commit 9fabf2a523347c0b654e7a17c30c703e6902b2e9 Author: Cecylia Bocovich cohosh@torproject.org Date: Tue Dec 10 12:27:24 2019 -0500

Restored old update_files script

This script had important commands for updating internet archine and google docs. The github release script has been renamed to update_github. --- scripts/update_files | 141 ++++++++++++++++++++++---------------------------- scripts/update_github | 79 ++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 79 deletions(-)

diff --git a/scripts/update_files b/scripts/update_files index 6d087e6..6e9e013 100755 --- a/scripts/update_files +++ b/scripts/update_files @@ -1,79 +1,62 @@ -#!/usr/bin/env python3 - -#pip3 install PyGithub - -# Before running, place the authentication token for a user with push access to the repository -# in an environment variable 'GITHUB_AUTH' - -from github import Github - -import sys -import json -import urllib -import subprocess -import os - -REPO_NAME = "TheTorProject/gettorbrowser" - -failure = False - -failed_uploads = [] - -def delete_old_releases(repo): - for release in repo.get_releases(): - for asset in release.get_assets(): - asset.delete_asset() - release.delete_release() - -#Download list of tor browser releases and upload them to github -def upload_files(release): - url = urllib.request.urlopen("https://aus1.torproject.org/torbrowser/update_3/release/downloads.json") - data = json.loads(url.read().decode()) - for arch in data['downloads']: - for locale in data['downloads'][arch]: - for asset in data['downloads'][arch][locale]: - url = data['downloads'][arch][locale][asset] - filename = url.split('/')[-1] - try: - subprocess.check_call(["/usr/bin/wget", "--quiet", url]) - release.upload_asset(filename) - os.remove(filename) - except: - print("Error: failed to update "+url+". Will retry later.") - failed_uploads.append(url) - #Retry failed uploads - for url in failed_uploads: - filename = url.split('/')[-1] - try: - subprocess.check_call(["/usr/bin/wget", "--quiet", url]) - release.upload_asset(filename) - os.remove(filename) - except: - print("Error: failed to update "+url+". Please upload this file manually.") - failure = True - - -def main(token): - - #Initialize a new release - g = Github(token) - repo = g.get_repo(REPO_NAME) - - delete_old_releases(repo) - - #Create a new release - release = repo.create_git_release("torbrowser-release", "Tor Browser releases", "These releases were uploaded to be distributed with gettor.") - upload_files(release) - - if failure: - sys.exit(1) - - -if __name__ == "__main__": - if 'GITHUB_AUTH' not in os.environ: - print("Usage: {}".format(sys.argv[0]), file=sys.stderr) - print("\nThe authentication token for github should be placed in the environment" - "variable 'GITHUB_AUTH'", file=sys.stderr) - sys.exit(1) - token = os.environ['GITHUB_AUTH'] - sys.exit(main(token)) +#!/bin/bash +# +# This file is part of GetTor, a Tor Browser distribution system. +# +# :authors: hiro hiro@torproject.org +# see also AUTHORS file +# +# :copyright: (c) 2008-2019, The Tor Project, Inc. +# +# :license: This is Free Software. See LICENSE for license information. + +cd ~/releases +git checkout master +git branch -D torbrowser-releases +git push github --delete torbrowser-releases + +git fetch --all --prune + +git add . +git commit -am 'Create release branch' +git checkout -b torbrowser-releases +git push -f --follow-tags origin torbrowser-releases +git push -f --follow-tags github torbrowser-releases + + +for row in $( + curl -s 'https://aus1.torproject.org/torbrowser/update_3/release/downloads.json' | + jq -r '.downloads' + ); do + r=$( + echo ${row} | + egrep -o 'https?://[^ ]+' | + tr -d '",' + ); + if [[ $r = *[!\ ]* ]]; then + git fetch --all + wget $r + git add . + git commit -m '[dist ci] commit from CI runner - update with new torbrowser downloads' + diffs=$(git diff origin/torbrowser-releases) + if [ -z "$diffs" ]; then + echo "No new releases" + else + git push -f --follow-tags origin torbrowser-releases + fi + diffs=$(git diff github/torbrowser-releases) + if [ -z "$diffs" ]; then + echo "No new releases" + else + git push -f --follow-tags github torbrowser-releases + fi + fi; +done + +rclone delete gdrive:releases + +for f in $(ls); do + # Update Google Drive + rclone copy $f gdrive:releases + # Update Internet Archive + ia upload ${f} $f --remote-name=$f --metadata="title:${f}" --metadata="mediatype:software" --metadata="collection:open_source_software" +done diff --git a/scripts/update_github b/scripts/update_github new file mode 100755 index 0000000..6d087e6 --- /dev/null +++ b/scripts/update_github @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +#pip3 install PyGithub + +# Before running, place the authentication token for a user with push access to the repository +# in an environment variable 'GITHUB_AUTH' + +from github import Github + +import sys +import json +import urllib +import subprocess +import os + +REPO_NAME = "TheTorProject/gettorbrowser" + +failure = False + +failed_uploads = [] + +def delete_old_releases(repo): + for release in repo.get_releases(): + for asset in release.get_assets(): + asset.delete_asset() + release.delete_release() + +#Download list of tor browser releases and upload them to github +def upload_files(release): + url = urllib.request.urlopen("https://aus1.torproject.org/torbrowser/update_3/release/downloads.json") + data = json.loads(url.read().decode()) + for arch in data['downloads']: + for locale in data['downloads'][arch]: + for asset in data['downloads'][arch][locale]: + url = data['downloads'][arch][locale][asset] + filename = url.split('/')[-1] + try: + subprocess.check_call(["/usr/bin/wget", "--quiet", url]) + release.upload_asset(filename) + os.remove(filename) + except: + print("Error: failed to update "+url+". Will retry later.") + failed_uploads.append(url) + #Retry failed uploads + for url in failed_uploads: + filename = url.split('/')[-1] + try: + subprocess.check_call(["/usr/bin/wget", "--quiet", url]) + release.upload_asset(filename) + os.remove(filename) + except: + print("Error: failed to update "+url+". Please upload this file manually.") + failure = True + + +def main(token): + + #Initialize a new release + g = Github(token) + repo = g.get_repo(REPO_NAME) + + delete_old_releases(repo) + + #Create a new release + release = repo.create_git_release("torbrowser-release", "Tor Browser releases", "These releases were uploaded to be distributed with gettor.") + upload_files(release) + + if failure: + sys.exit(1) + + +if __name__ == "__main__": + if 'GITHUB_AUTH' not in os.environ: + print("Usage: {}".format(sys.argv[0]), file=sys.stderr) + print("\nThe authentication token for github should be placed in the environment" + "variable 'GITHUB_AUTH'", file=sys.stderr) + sys.exit(1) + token = os.environ['GITHUB_AUTH'] + sys.exit(main(token))