- tor-commits - lists.torproject.org

[gettor/master] Will the real TBB key please stand up
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit 102dc3949d3ce589aea6952bde40723ca674d9b8 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Feb 5 18:35:20 2016 +0000 Will the real TBB key please stand up --- upload/torbrowser-key.asc | Bin 193819 -> 25381 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/upload/torbrowser-key.asc b/upload/torbrowser-key.asc index bd13d7e..345f65d 100644 Binary files a/upload/torbrowser-key.asc and b/upload/torbrowser-key.asc differ

1 0

[gettor/master] Remove redundant filename check
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit a34db1d2c957c1d28130d39ddc4190a5c7ccd684 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Feb 5 19:02:34 2016 +0000 Remove redundant filename check --- upload/bundles2drive.py | 12 +++--------- upload/bundles2dropbox.py | 12 +++--------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/upload/bundles2drive.py b/upload/bundles2drive.py index e6ad04b..0acada5 100644 --- a/upload/bundles2drive.py +++ b/upload/bundles2drive.py @@ -50,25 +50,19 @@ def upload_files(client, basedir): """ files = [] - p = re.compile('.*\.tar.xz$') for name in os.listdir(basedir): path = os.path.abspath(os.path.join(basedir, name)) - if os.path.isfile(path) and p.match(path)\ - and valid_format(name, 'linux'): + if os.path.isfile(path) and valid_format(name, 'linux'): files.append(name) - p = re.compile('.*\.exe$') for name in os.listdir(basedir): path = os.path.abspath(os.path.join(basedir, name)) - if os.path.isfile(path) and p.match(path)\ - and valid_format(name, 'windows'): + if os.path.isfile(path) and valid_format(name, 'windows'): files.append(name) - p = re.compile('.*\.dmg$') for name in os.listdir(basedir): path = os.path.abspath(os.path.join(basedir, name)) - if os.path.isfile(path) and p.match(path)\ - and valid_format(name, 'osx'): + if os.path.isfile(path) and valid_format(name, 'osx'): files.append(name) # dictionary to store file names and IDs diff --git a/upload/bundles2dropbox.py b/upload/bundles2dropbox.py index 9007410..40157bc 100644 --- a/upload/bundles2dropbox.py +++ b/upload/bundles2dropbox.py @@ -39,25 +39,19 @@ def upload_files(basedir, client): """ files = [] - p = re.compile('.*\.tar.xz$') for name in os.listdir(basedir): path = os.path.abspath(os.path.join(basedir, name)) - if os.path.isfile(path) and p.match(path)\ - and valid_format(name, 'linux'): + if os.path.isfile(path) and valid_format(name, 'linux'): files.append(name) - p = re.compile('.*\.exe$') for name in os.listdir(basedir): path = os.path.abspath(os.path.join(basedir, name)) - if os.path.isfile(path) and p.match(path)\ - and valid_format(name, 'windows'): + if os.path.isfile(path) and valid_format(name, 'windows'): files.append(name) - p = re.compile('.*\.dmg$') for name in os.listdir(basedir): path = os.path.abspath(os.path.join(basedir, name)) - if os.path.isfile(path) and p.match(path)\ - and valid_format(name, 'osx'): + if os.path.isfile(path) and valid_format(name, 'osx'): files.append(name) for file in files:

1 0

[gettor/master] Rewrite bundles2github to use releases
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit 678bf6c64bc8a77c4351e35341159b6b543399cb Author: Donncha O'Cearbhaill <donncha(a)donncha.is> Date: Sat Feb 6 02:27:26 2016 +0100 Rewrite bundles2github to use releases --- gettor/utils.py | 86 ++++++++++----------- upload/bundles2github.py | 190 ++++++++++++++++++++++++----------------------- upload/github.cfg | 8 ++ 3 files changed, 142 insertions(+), 142 deletions(-) diff --git a/gettor/utils.py b/gettor/utils.py index c684097..c4f43c7 100644 --- a/gettor/utils.py +++ b/gettor/utils.py @@ -11,6 +11,7 @@ # :license: This is Free Software. See LICENSE for license information. import os +import re import hashlib """Common utilities for GetTor modules.""" @@ -19,6 +20,10 @@ import hashlib LOGGING_FORMAT = "[%(levelname)s] %(asctime)s; %(message)s" DATE_FORMAT = "%Y-%m-%d" # %H:%M:%S +windows_regex = '^torbrowser-install-\d\.\d\.\d_\w\w(-\w\w)?\.exe$' +linux_regex = '^tor-browser-linux\d\d-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz$' +osx_regex = '^TorBrowser-\d\.\d\.\d-osx\d\d_(\w\w)(-\w\w)?\.dmg$' + def get_logging_format(): """Get the logging format. @@ -49,7 +54,7 @@ def get_sha256(string): return str(hashlib.sha256(string).hexdigest()) -def get_bundle_info(file, osys): +def get_bundle_info(filename, osys=None): """Get the os, arch and lc from a bundle string. :param: file (string) the name of the file. @@ -60,63 +65,35 @@ def get_bundle_info(file, osys): :return: (list) the os, arch and lc. """ - if(osys == 'windows'): - m = re.search( - 'torbrowser-install-\d\.\d\.\d_(\w\w)(-\w\w)?\.exe', - file) - if m: - lc = m.group(1) - return 'windows', '32/64', lc - else: - raise ValueError("Invalid bundle format %s" % file) - elif(osys == 'linux'): - m = re.search( - 'tor-browser-linux(\d\d)-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz', - file) - if m: - arch = m.group(1) - lc = m.group(2) - return 'linux', arch, lc - else: - raise ValueError("Invalid bundle format %s" % file) - elif(osys == 'osx'): - m = re.search( - 'TorBrowser-\d\.\d\.\d-osx(\d\d)_(\w\w)(-\w\w)?\.dmg', - file) - if m: - os = 'osx' - arch = m.group(1) - lc = m.group(2) - return 'osx', arch, lc - else: - raise ValueError("Invalid bundle format %s" % file) - - -def valid_format(file, osys): + m_windows = re.search(windows_regex, filename) + m_linux = re.search(linux_regex, filename) + m_osx = re.search(osx_regex, filename) + + if m_windows: + return 'windows', '32/64', m_windows.group(1) + elif m_linux: + return 'linux', m_linux.group(1), m_linux.group(2) + elif m_osx: + return 'osx', m_osx.group(1), m_osx.group(2) + else: + raise ValueError("Invalid bundle format %s" % file) + + +def valid_format(filename, osys=None): """Check for valid bundle format Check if the given file has a valid bundle format (e.g. tor-browser-linux32-3.6.2_es-ES.tar.xz) :param: file (string) the name of the file. - :param: osys (string) the OS. :return: (boolean) true if the bundle format is valid, false otherwise. """ - if(osys == 'windows'): - m = re.search( - 'torbrowser-install-\d\.\d\.\d_\w\w(-\w\w)?\.exe', - file) - elif(osys == 'linux'): - m = re.search( - 'tor-browser-linux\d\d-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz', - file) - elif(osys == 'osx'): - m = re.search( - 'TorBrowser-\d\.\d\.\d-osx\d\d_(\w\w)(-\w\w)?\.dmg', - file) - if m: + m_windows = re.search(windows_regex, filename) + m_linux = re.search(linux_regex, filename) + m_osx = re.search(osx_regex, filename) + if any((m_windows, m_linux, m_osx)): return True else: return False @@ -139,3 +116,16 @@ def get_file_sha256(file): hasher.update(buf) buf = afile.read(BLOCKSIZE) return hasher.hexdigest() + + +def find_files_to_upload(upload_dir): + """ + Find the files which are named correctly and have a .asc file + """ + files = [] + for name in os.listdir(upload_dir): + asc_file = os.path.join(upload_dir, "{}.asc".format(name)) + if valid_format(name) and os.path.isfile(asc_file): + files.extend([name, "{}.asc".format(name)]) + + return files diff --git a/upload/bundles2github.py b/upload/bundles2github.py index decb44e..8668c78 100644 --- a/upload/bundles2github.py +++ b/upload/bundles2github.py @@ -13,24 +13,51 @@ import os import re -import sh import sys -import time -import shutil -import hashlib import argparse +import ConfigParser -from libsaas.services import github +import github3 import gnupg import gettor.core -from gettor.utils import get_bundle_info, get_file_sha256 +from gettor.utils import (get_bundle_info, get_file_sha256, + find_files_to_upload) + +import urllib3 + +# Actually verify Github's cert! +urllib3.disable_warnings() + + +def upload_new_release(github_repo, version, upload_dir): + """ + Returns a Release object + """ + + # Create a new release of this TBB + release = target_repo.create_release( + 'v{}'.format(version), + target_commitish="master", + name='Tor Browser Bundle {}'.format(version), + body='', + draft=True, + ) + + for filename in find_files_to_upload(upload_dir): + # Upload each file for this release + file_path = os.path.join(upload_dir, filename) + print("Uploading file {}".format(filename)) + release.upload_asset('application/zip', + filename, open(file_path, 'rb')) + + return release if __name__ == '__main__': parser = argparse.ArgumentParser( description='Utility to upload Tor Browser to Github.' ) - + # with this we only get the links of files already uploaded # useful when somethings fail after uploading parser.add_argument( @@ -45,24 +72,33 @@ if __name__ == '__main__': args = parser.parse_args() + config = ConfigParser.ConfigParser() + config.read('github-local.cfg') + # this script should be called after fetching the latest Tor Browser, # and specifying the latest version - version = args.version + if args.version: + version = args.version + else: + tbb_version_config = ConfigParser.ConfigParser() + tbb_version_config.read('latest_torbrowser.cfg') + version = tbb_version_config.get('version', 'current') # the token allow us to run this script without GitHub user/pass - gh_token = '' + github_access_token = config.get('app', 'access_token') # path to the fingerprint that signed the packages - tb_key = os.path.abspath('torbrowser-key.asc') + tb_key = os.path.abspath('tbb-key-torbrowserteam.asc') # path to the latest version of Tor Browser - tb_path = os.path.abspath('upload/latest') + tb_path = os.path.abspath('latest') - # path to the repository where we upload Tor Browser - repo_path = os.path.abspath('gettorbrowser') + # user and repository where we upload Tor Browser + github_user = config.get('app', 'user') + github_repo = config.get('app', 'repo') - # wait time between pushing the files to GH and asking for its links - wait_time = 10 + gh = github3.login(token=github_access_token) + target_repo = gh.repository(github_user, github_repo) # import key fingerprint gpg = gnupg.GPG() @@ -74,86 +110,52 @@ if __name__ == '__main__': # e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J readable_fp = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4)) - # we should have previously created a repository on GitHub where we - # want to push the files using an SSH key (to avoid using user/pass) - remote = 'origin' - branch = 'master' - user = 'TheTorProject' - repo = 'gettorbrowser' - raw_content = 'https://raw.githubusercontent.com/%s/%s/%s/' %\ - (user, repo, branch) - - # steps: - # 1) copy folder with latest version of Tor Browser - # 2) add files via sh.git - # 3) make a commit for the new version - # 4) push the changes - - if not args.links: - shutil.copytree( - tb_path, - os.path.abspath('%s/%s' % (repo_path, version)) - ) - - git = sh.git.bake(_cwd=repo_path) - git.add('%s' % version) - git.commit(m=version) - git.push() - - # it takes a while to process the recently pushed files - print "Wait a few seconds before asking for the links to Github..." - time.sleep(wait_time) - - print "Trying to get the links" - gh = github.GitHub(gh_token, None) - repocontent = gh.repo( - user, - repo - ).contents().get('%s' % version) - - core = gettor.core.Core( - os.path.abspath('core.cfg') - ) - - # erase old links, if any + # Find any published releases with this version number + for release in target_repo.iter_releases(): + if release.tag_name == 'v{}'.format(version) and not release.draft: + print("Found an existing published release with this version. " + "Not uploading again unless you delete the published " + "release '{}'.".format(release.tag_name)) + break + else: + release = None + + if args.links or release: + # Only generating link file, should use previously published release + if not release: + print("Error occured! Could not find a published release for " + "version {}".format(version)) + sys.exit(1) + + else: + # Remove any drafts to clean broken uploads + print('Uploading release, please wait, this might take a while!') + # Upload the latest browser bundles to a new release + release = upload_new_release(target_repo, version, tb_path) + + # Upload success, publish the release + release.edit(draft=False) + + # Create the links file for this release + core = gettor.core.Core(os.path.abspath('../core.cfg')) + + # Erase old links if any and create a new empty one core.create_links_file('GitHub', readable_fp) - for file in repocontent: - # e.g. https://raw.githubusercontent.com/gettorbrowser/dl/master/4.0.7/TorBrowser-… - m = re.search('%s.*\/(.*)' % raw_content, file[u'download_url']) - if m: - filename = m.group(1) - # get bundle info according to its OS - if re.match('.*\.exe$', filename): - osys, arch, lc = get_bundle_info(filename, 'windows') - filename_asc = filename.replace('exe', 'exe.asc') - - elif re.match('.*\.dmg$', filename): - osys, arch, lc = get_bundle_info(filename, 'osx') - filename_asc = filename.replace('dmg', 'dmg.asc') - - elif re.match('.*\.tar.xz$', filename): - osys, arch, lc = get_bundle_info(filename, 'linux') - filename_asc = filename.replace('tar.xz', 'tar.xz.asc') - - else: - # don't care about other files (asc or txt) - continue - - sha256 = get_file_sha256( - os.path.abspath( - '%s/%s/%s' % (repo, version, filename) - ) - ) - - # since the url is easy to construct and it doesn't involve any - # kind of unique hash or identifier, we get the link for the - # asc signature just by adding '.asc' - link_asc = file[u'download_url'].replace(filename, filename_asc) - - link = "%s$%s$%s$" % (file[u'download_url'], link_asc, sha256) - - print "Adding %s" % file[u'download_url'] - core.add_link('GitHub', osys, lc, link) + print("Creating releases") + for asset in release.assets: + filename = asset.download_url.split('/')[-1] + osys, arch, lc = get_bundle_info(asset.download_url) + sha256 = get_file_sha256( + os.path.abspath(os.path.join(tb_path, filename)) + ) + + link = "{}${}${}$".format( + asset.download_url, + asset.download_url+".asc", + sha256, + ) + print("Adding {}".format(asset.download_url)) + core.add_link('GitHub', osys, lc, link) print "Github links updated!" diff --git a/upload/github.cfg b/upload/github.cfg new file mode 100644 index 0000000..7bdbd90 --- /dev/null +++ b/upload/github.cfg @@ -0,0 +1,8 @@ +[general] +upload_dir: latest +tbb_key: torbrowser-key.asc + +[app] +access_token: suchtoken +user: username +repo: gettor-front

1 0

[gettor/master] Read correct keyfile name
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit d8a9d30dea77c91da06915c5836cfbebe27ed002 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Feb 5 19:04:08 2016 +0000 Read correct keyfile name --- upload/bundles2github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upload/bundles2github.py b/upload/bundles2github.py index a5beea5..decb44e 100644 --- a/upload/bundles2github.py +++ b/upload/bundles2github.py @@ -53,7 +53,7 @@ if __name__ == '__main__': gh_token = '' # path to the fingerprint that signed the packages - tb_key = os.path.abspath('tbb-key-torbrowserteam.asc') + tb_key = os.path.abspath('torbrowser-key.asc') # path to the latest version of Tor Browser tb_path = os.path.abspath('upload/latest')

1 0

[gettor/master] Working script to add TBB' to releases
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit fb48a05736b4a84c3f1e4752e63750a2e6b26ec6 Author: Donncha O'Cearbhaill <donncha(a)donncha.is> Date: Sat Feb 6 03:33:42 2016 +0100 Working script to add TBB' to releases --- upload/bundles2github.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/upload/bundles2github.py b/upload/bundles2github.py index 8668c78..1252163 100644 --- a/upload/bundles2github.py +++ b/upload/bundles2github.py @@ -142,20 +142,23 @@ if __name__ == '__main__': # Erase old links if any and create a new empty one core.create_links_file('GitHub', readable_fp) - print("Creating releases") + print("Creating links file") for asset in release.assets: - filename = asset.download_url.split('/')[-1] - osys, arch, lc = get_bundle_info(asset.download_url) + url = ("https://github.com/{user}/{repo}/releases/download/" + "v{tag}/{file}".format( + user=github_user, + repo=github_repo, + tag=version, + file=asset.name, + )) + osys, arch, lc = get_bundle_info(asset.name) sha256 = get_file_sha256( - os.path.abspath(os.path.join(tb_path, filename)) + os.path.abspath(os.path.join(tb_path, asset.name)) ) - link = "{}${}${}$".format( - asset.download_url, - asset.download_url+".asc", - sha256, - ) - print("Adding {}".format(asset.download_url)) + link = "{}${}${}$".format(url, url + ".asc", sha256) + + print("Adding {}".format(url)) core.add_link('GitHub', osys, lc, link) print "Github links updated!"

1 0

[gettor/master] Merge pull request #11 from aagbsn/prune_stale_uploads
by ilv＠torproject.org 15 Feb '16

15 Feb '16

1 0

[gettor/master] Merge pull request #14 from DonnchaC/github-new
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit 645104175972a6be7b1933e0a5edc75103056791 Merge: 4103139 cfde870 Author: Israel Leiva <ilv(a)users.noreply.github.com> Date: Fri Feb 12 20:49:40 2016 +0000 Merge pull request #14 from DonnchaC/github-new Host TBB bundles on Github Releases gettor/utils.py | 86 ++++++++++----------- upload/bundles2github.py | 192 ++++++++++++++++++++++++----------------------- upload/github.cfg | 8 ++ 3 files changed, 144 insertions(+), 142 deletions(-)

1 0

[gettor/master] Merge pull request #12 from aagbsn/update_tbb_key
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit 09ea264f984c581d847d40260cfa960c91c78bad Merge: 4e6e580 102dc39 Author: Israel Leiva <ilv(a)users.noreply.github.com> Date: Mon Feb 8 22:39:44 2016 +0000 Merge pull request #12 from aagbsn/update_tbb_key Update tbb key upload/torbrowser-key.asc | Bin 193819 -> 25381 bytes 1 file changed, 0 insertions(+), 0 deletions(-)

1 0

[gettor/master] Merge pull request #13 from aagbsn/deduplicate_utility_methods
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit 41031394261d544680f5d69ff64aba4bf29caa73 Merge: 09ea264 d8a9d30 Author: Israel Leiva <ilv(a)users.noreply.github.com> Date: Mon Feb 8 22:41:19 2016 +0000 Merge pull request #13 from aagbsn/deduplicate_utility_methods Deduplicate utility methods gettor/utils.py | 92 ++++++++++++++++++++++++++++++++++++++++ upload/bundles2drive.py | 105 ++-------------------------------------------- upload/bundles2dropbox.py | 104 ++------------------------------------------- upload/bundles2github.py | 58 +------------------------ 4 files changed, 102 insertions(+), 257 deletions(-)

1 0

[gettor/master] Merge branch 'develop'
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit 7b052ebac1ed2f73014df126be93eb60a44aafd9 Merge: 6451041 7735f5b Author: ilv <ilv(a)users.noreply.github.com> Date: Mon Feb 15 17:09:02 2016 -0300 Merge branch 'develop' gettor/core.py | 22 +- gettor/http.py | 506 ++++++++++++++++++++++++++++++++++++ gettor/smtp.py | 8 +- lang/smtp/i18n/en/LC_MESSAGES/en.po | 36 +-- process_http.py | 14 + report.py | 129 +++++++++ scripts/create_api_mirror.py | 305 ++++++++++++++++++++++ scripts/sample-htaccess-api-mirror | 21 ++ 8 files changed, 1009 insertions(+), 32 deletions(-)

1 0