commit 678bf6c64bc8a77c4351e35341159b6b543399cb Author: Donncha O'Cearbhaill <donncha@donncha.is> Date: Sat Feb 6 02:27:26 2016 +0100 Rewrite bundles2github to use releases --- gettor/utils.py | 86 ++++++++++----------- upload/bundles2github.py | 190 ++++++++++++++++++++++++----------------------- upload/github.cfg | 8 ++ 3 files changed, 142 insertions(+), 142 deletions(-) diff --git a/gettor/utils.py b/gettor/utils.py index c684097..c4f43c7 100644 --- a/gettor/utils.py +++ b/gettor/utils.py @@ -11,6 +11,7 @@ # :license: This is Free Software. See LICENSE for license information. import os +import re import hashlib """Common utilities for GetTor modules.""" @@ -19,6 +20,10 @@ import hashlib LOGGING_FORMAT = "[%(levelname)s] %(asctime)s; %(message)s" DATE_FORMAT = "%Y-%m-%d" # %H:%M:%S +windows_regex = '^torbrowser-install-\d\.\d\.\d_\w\w(-\w\w)?\.exe$' +linux_regex = '^tor-browser-linux\d\d-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz$' +osx_regex = '^TorBrowser-\d\.\d\.\d-osx\d\d_(\w\w)(-\w\w)?\.dmg$' + def get_logging_format(): """Get the logging format. @@ -49,7 +54,7 @@ def get_sha256(string): return str(hashlib.sha256(string).hexdigest()) -def get_bundle_info(file, osys): +def get_bundle_info(filename, osys=None): """Get the os, arch and lc from a bundle string. :param: file (string) the name of the file. @@ -60,63 +65,35 @@ def get_bundle_info(file, osys): :return: (list) the os, arch and lc. """ - if(osys == 'windows'): - m = re.search( - 'torbrowser-install-\d\.\d\.\d_(\w\w)(-\w\w)?\.exe', - file) - if m: - lc = m.group(1) - return 'windows', '32/64', lc - else: - raise ValueError("Invalid bundle format %s" % file) - elif(osys == 'linux'): - m = re.search( - 'tor-browser-linux(\d\d)-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz', - file) - if m: - arch = m.group(1) - lc = m.group(2) - return 'linux', arch, lc - else: - raise ValueError("Invalid bundle format %s" % file) - elif(osys == 'osx'): - m = re.search( - 'TorBrowser-\d\.\d\.\d-osx(\d\d)_(\w\w)(-\w\w)?\.dmg', - file) - if m: - os = 'osx' - arch = m.group(1) - lc = m.group(2) - return 'osx', arch, lc - else: - raise ValueError("Invalid bundle format %s" % file) - - -def valid_format(file, osys): + m_windows = re.search(windows_regex, filename) + m_linux = re.search(linux_regex, filename) + m_osx = re.search(osx_regex, filename) + + if m_windows: + return 'windows', '32/64', m_windows.group(1) + elif m_linux: + return 'linux', m_linux.group(1), m_linux.group(2) + elif m_osx: + return 'osx', m_osx.group(1), m_osx.group(2) + else: + raise ValueError("Invalid bundle format %s" % file) + + +def valid_format(filename, osys=None): """Check for valid bundle format Check if the given file has a valid bundle format (e.g. tor-browser-linux32-3.6.2_es-ES.tar.xz) :param: file (string) the name of the file. - :param: osys (string) the OS. :return: (boolean) true if the bundle format is valid, false otherwise. """ - if(osys == 'windows'): - m = re.search( - 'torbrowser-install-\d\.\d\.\d_\w\w(-\w\w)?\.exe', - file) - elif(osys == 'linux'): - m = re.search( - 'tor-browser-linux\d\d-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz', - file) - elif(osys == 'osx'): - m = re.search( - 'TorBrowser-\d\.\d\.\d-osx\d\d_(\w\w)(-\w\w)?\.dmg', - file) - if m: + m_windows = re.search(windows_regex, filename) + m_linux = re.search(linux_regex, filename) + m_osx = re.search(osx_regex, filename) + if any((m_windows, m_linux, m_osx)): return True else: return False @@ -139,3 +116,16 @@ def get_file_sha256(file): hasher.update(buf) buf = afile.read(BLOCKSIZE) return hasher.hexdigest() + + +def find_files_to_upload(upload_dir): + """ + Find the files which are named correctly and have a .asc file + """ + files = [] + for name in os.listdir(upload_dir): + asc_file = os.path.join(upload_dir, "{}.asc".format(name)) + if valid_format(name) and os.path.isfile(asc_file): + files.extend([name, "{}.asc".format(name)]) + + return files diff --git a/upload/bundles2github.py b/upload/bundles2github.py index decb44e..8668c78 100644 --- a/upload/bundles2github.py +++ b/upload/bundles2github.py @@ -13,24 +13,51 @@ import os import re -import sh import sys -import time -import shutil -import hashlib import argparse +import ConfigParser -from libsaas.services import github +import github3 import gnupg import gettor.core -from gettor.utils import get_bundle_info, get_file_sha256 +from gettor.utils import (get_bundle_info, get_file_sha256, + find_files_to_upload) + +import urllib3 + +# Actually verify Github's cert! +urllib3.disable_warnings() + + +def upload_new_release(github_repo, version, upload_dir): + """ + Returns a Release object + """ + + # Create a new release of this TBB + release = target_repo.create_release( + 'v{}'.format(version), + target_commitish="master", + name='Tor Browser Bundle {}'.format(version), + body='', + draft=True, + ) + + for filename in find_files_to_upload(upload_dir): + # Upload each file for this release + file_path = os.path.join(upload_dir, filename) + print("Uploading file {}".format(filename)) + release.upload_asset('application/zip', + filename, open(file_path, 'rb')) + + return release if __name__ == '__main__': parser = argparse.ArgumentParser( description='Utility to upload Tor Browser to Github.' ) - + # with this we only get the links of files already uploaded # useful when somethings fail after uploading parser.add_argument( @@ -45,24 +72,33 @@ if __name__ == '__main__': args = parser.parse_args() + config = ConfigParser.ConfigParser() + config.read('github-local.cfg') + # this script should be called after fetching the latest Tor Browser, # and specifying the latest version - version = args.version + if args.version: + version = args.version + else: + tbb_version_config = ConfigParser.ConfigParser() + tbb_version_config.read('latest_torbrowser.cfg') + version = tbb_version_config.get('version', 'current') # the token allow us to run this script without GitHub user/pass - gh_token = '' + github_access_token = config.get('app', 'access_token') # path to the fingerprint that signed the packages - tb_key = os.path.abspath('torbrowser-key.asc') + tb_key = os.path.abspath('tbb-key-torbrowserteam.asc') # path to the latest version of Tor Browser - tb_path = os.path.abspath('upload/latest') + tb_path = os.path.abspath('latest') - # path to the repository where we upload Tor Browser - repo_path = os.path.abspath('gettorbrowser') + # user and repository where we upload Tor Browser + github_user = config.get('app', 'user') + github_repo = config.get('app', 'repo') - # wait time between pushing the files to GH and asking for its links - wait_time = 10 + gh = github3.login(token=github_access_token) + target_repo = gh.repository(github_user, github_repo) # import key fingerprint gpg = gnupg.GPG() @@ -74,86 +110,52 @@ if __name__ == '__main__': # e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J readable_fp = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4)) - # we should have previously created a repository on GitHub where we - # want to push the files using an SSH key (to avoid using user/pass) - remote = 'origin' - branch = 'master' - user = 'TheTorProject' - repo = 'gettorbrowser' - raw_content = 'https://raw.githubusercontent.com/%s/%s/%s/' %\ - (user, repo, branch) - - # steps: - # 1) copy folder with latest version of Tor Browser - # 2) add files via sh.git - # 3) make a commit for the new version - # 4) push the changes - - if not args.links: - shutil.copytree( - tb_path, - os.path.abspath('%s/%s' % (repo_path, version)) - ) - - git = sh.git.bake(_cwd=repo_path) - git.add('%s' % version) - git.commit(m=version) - git.push() - - # it takes a while to process the recently pushed files - print "Wait a few seconds before asking for the links to Github..." - time.sleep(wait_time) - - print "Trying to get the links" - gh = github.GitHub(gh_token, None) - repocontent = gh.repo( - user, - repo - ).contents().get('%s' % version) - - core = gettor.core.Core( - os.path.abspath('core.cfg') - ) - - # erase old links, if any + # Find any published releases with this version number + for release in target_repo.iter_releases(): + if release.tag_name == 'v{}'.format(version) and not release.draft: + print("Found an existing published release with this version. " + "Not uploading again unless you delete the published " + "release '{}'.".format(release.tag_name)) + break + else: + release = None + + if args.links or release: + # Only generating link file, should use previously published release + if not release: + print("Error occured! Could not find a published release for " + "version {}".format(version)) + sys.exit(1) + + else: + # Remove any drafts to clean broken uploads + print('Uploading release, please wait, this might take a while!') + # Upload the latest browser bundles to a new release + release = upload_new_release(target_repo, version, tb_path) + + # Upload success, publish the release + release.edit(draft=False) + + # Create the links file for this release + core = gettor.core.Core(os.path.abspath('../core.cfg')) + + # Erase old links if any and create a new empty one core.create_links_file('GitHub', readable_fp) - for file in repocontent: - # e.g. https://raw.githubusercontent.com/gettorbrowser/dl/master/4.0.7/TorBrowser-4... - m = re.search('%s.*\/(.*)' % raw_content, file[u'download_url']) - if m: - filename = m.group(1) - # get bundle info according to its OS - if re.match('.*\.exe$', filename): - osys, arch, lc = get_bundle_info(filename, 'windows') - filename_asc = filename.replace('exe', 'exe.asc') - - elif re.match('.*\.dmg$', filename): - osys, arch, lc = get_bundle_info(filename, 'osx') - filename_asc = filename.replace('dmg', 'dmg.asc') - - elif re.match('.*\.tar.xz$', filename): - osys, arch, lc = get_bundle_info(filename, 'linux') - filename_asc = filename.replace('tar.xz', 'tar.xz.asc') - - else: - # don't care about other files (asc or txt) - continue - - sha256 = get_file_sha256( - os.path.abspath( - '%s/%s/%s' % (repo, version, filename) - ) - ) - - # since the url is easy to construct and it doesn't involve any - # kind of unique hash or identifier, we get the link for the - # asc signature just by adding '.asc' - link_asc = file[u'download_url'].replace(filename, filename_asc) - - link = "%s$%s$%s$" % (file[u'download_url'], link_asc, sha256) - - print "Adding %s" % file[u'download_url'] - core.add_link('GitHub', osys, lc, link) + print("Creating releases") + for asset in release.assets: + filename = asset.download_url.split('/')[-1] + osys, arch, lc = get_bundle_info(asset.download_url) + sha256 = get_file_sha256( + os.path.abspath(os.path.join(tb_path, filename)) + ) + + link = "{}${}${}$".format( + asset.download_url, + asset.download_url+".asc", + sha256, + ) + print("Adding {}".format(asset.download_url)) + core.add_link('GitHub', osys, lc, link) print "Github links updated!" diff --git a/upload/github.cfg b/upload/github.cfg new file mode 100644 index 0000000..7bdbd90 --- /dev/null +++ b/upload/github.cfg @@ -0,0 +1,8 @@ +[general] +upload_dir: latest +tbb_key: torbrowser-key.asc + +[app] +access_token: suchtoken +user: username +repo: gettor-front