commit b60b5ad92d17d64f15214d80e20aa08b50fc96d1 Author: Cecylia Bocovich <cohosh@torproject.org> Date: Tue Nov 19 19:27:17 2019 -0500 Changed update_files to upload releases We were running out of space in both of our github and gitlab repositories due to storage limits. This change switches to using the github REST API to upload torbrowser binaries as releases. Releases are not subject to the space limits of repositories and, even better, do not count against our bandwidth limits when users download them. --- scripts/update_files | 113 +++++++++++++++++++++++---------------------------- 1 file changed, 51 insertions(+), 62 deletions(-) diff --git a/scripts/update_files b/scripts/update_files index 6e9e013..3fa35b2 100755 --- a/scripts/update_files +++ b/scripts/update_files @@ -1,62 +1,51 @@ -#!/bin/bash -# -# This file is part of GetTor, a Tor Browser distribution system. -# -# :authors: hiro <hiro@torproject.org> -# see also AUTHORS file -# -# :copyright: (c) 2008-2019, The Tor Project, Inc. -# -# :license: This is Free Software. See LICENSE for license information. - -cd ~/releases -git checkout master -git branch -D torbrowser-releases -git push github --delete torbrowser-releases - -git fetch --all --prune - -git add . -git commit -am 'Create release branch' -git checkout -b torbrowser-releases -git push -f --follow-tags origin torbrowser-releases -git push -f --follow-tags github torbrowser-releases - - -for row in $( - curl -s 'https://aus1.torproject.org/torbrowser/update_3/release/downloads.json' | - jq -r '.downloads' - ); do - r=$( - echo ${row} | - egrep -o 'https?://[^ ]+' | - tr -d '",' - ); - if [[ $r = *[!\ ]* ]]; then - git fetch --all - wget $r - git add . - git commit -m '[dist ci] commit from CI runner - update with new torbrowser downloads' - diffs=$(git diff origin/torbrowser-releases) - if [ -z "$diffs" ]; then - echo "No new releases" - else - git push -f --follow-tags origin torbrowser-releases - fi - diffs=$(git diff github/torbrowser-releases) - if [ -z "$diffs" ]; then - echo "No new releases" - else - git push -f --follow-tags github torbrowser-releases - fi - fi; -done - -rclone delete gdrive:releases - -for f in $(ls); do - # Update Google Drive - rclone copy $f gdrive:releases - # Update Internet Archive - ia upload ${f} $f --remote-name=$f --metadata="title:${f}" --metadata="mediatype:software" --metadata="collection:open_source_software" -done +#!/usr/bin/env python3 + +from github import Github + +import sys +import json +import urllib +import subprocess + +REPO_NAME = "TheTorProject/gettorbrowser" + +def delete_old_releases(repo): + for release in repo.get_releases(): + release.delete_release() + +#Download list of tor browser releases and upload them to github +def upload_files(release): + url = urllib.request.urlopen("https://aus1.torproject.org/torbrowser/update_3/release/downloads.json") + data = json.loads(url.read().decode()) + for arch in data['downloads']: + for locale in data['downloads'][arch]: + for asset in data['downloads'][arch][locale]: + url = data['downloads'][arch][locale][asset] + filename = url.split('/')[-1] + print("Downloading " + filename) + subprocess.check_call(["/usr/bin/wget", url]) + release.upload_asset(filename) + + +def main(token): + + #Initialize a new release + g = Github(token) + repo = g.get_repo(REPO_NAME) + + delete_old_releases(repo) + + #Create a new release + release = repo.create_git_release("torbrowser-release", "Tor Browser releases", "These releases were uploaded to be distributed with gettor.") + upload_files(release) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: {} AUTH_TOKEN".format(sys.argv[0]), file=sys.stderr) + print("\nAUTH_TOKEN should be an authentication token for a user" + "with access to the gettor repository.", file=sys.stderr) + sys.exit(1) + token = sys.argv[1] + + sys.exit(main(token))