[tor-commits] [gettor/master] Restored old update_files script
cohosh at torproject.org
cohosh at torproject.org
Tue Dec 10 17:30:53 UTC 2019
commit 9fabf2a523347c0b654e7a17c30c703e6902b2e9
Author: Cecylia Bocovich <cohosh at torproject.org>
Date: Tue Dec 10 12:27:24 2019 -0500
Restored old update_files script
This script had important commands for updating internet archine and
google docs. The github release script has been renamed to
update_github.
---
scripts/update_files | 141 ++++++++++++++++++++++----------------------------
scripts/update_github | 79 ++++++++++++++++++++++++++++
2 files changed, 141 insertions(+), 79 deletions(-)
diff --git a/scripts/update_files b/scripts/update_files
index 6d087e6..6e9e013 100755
--- a/scripts/update_files
+++ b/scripts/update_files
@@ -1,79 +1,62 @@
-#!/usr/bin/env python3
-
-#pip3 install PyGithub
-
-# Before running, place the authentication token for a user with push access to the repository
-# in an environment variable 'GITHUB_AUTH'
-
-from github import Github
-
-import sys
-import json
-import urllib
-import subprocess
-import os
-
-REPO_NAME = "TheTorProject/gettorbrowser"
-
-failure = False
-
-failed_uploads = []
-
-def delete_old_releases(repo):
- for release in repo.get_releases():
- for asset in release.get_assets():
- asset.delete_asset()
- release.delete_release()
-
-#Download list of tor browser releases and upload them to github
-def upload_files(release):
- url = urllib.request.urlopen("https://aus1.torproject.org/torbrowser/update_3/release/downloads.json")
- data = json.loads(url.read().decode())
- for arch in data['downloads']:
- for locale in data['downloads'][arch]:
- for asset in data['downloads'][arch][locale]:
- url = data['downloads'][arch][locale][asset]
- filename = url.split('/')[-1]
- try:
- subprocess.check_call(["/usr/bin/wget", "--quiet", url])
- release.upload_asset(filename)
- os.remove(filename)
- except:
- print("Error: failed to update "+url+". Will retry later.")
- failed_uploads.append(url)
- #Retry failed uploads
- for url in failed_uploads:
- filename = url.split('/')[-1]
- try:
- subprocess.check_call(["/usr/bin/wget", "--quiet", url])
- release.upload_asset(filename)
- os.remove(filename)
- except:
- print("Error: failed to update "+url+". Please upload this file manually.")
- failure = True
-
-
-def main(token):
-
- #Initialize a new release
- g = Github(token)
- repo = g.get_repo(REPO_NAME)
-
- delete_old_releases(repo)
-
- #Create a new release
- release = repo.create_git_release("torbrowser-release", "Tor Browser releases", "These releases were uploaded to be distributed with gettor.")
- upload_files(release)
-
- if failure:
- sys.exit(1)
-
-
-if __name__ == "__main__":
- if 'GITHUB_AUTH' not in os.environ:
- print("Usage: {}".format(sys.argv[0]), file=sys.stderr)
- print("\nThe authentication token for github should be placed in the environment"
- "variable 'GITHUB_AUTH'", file=sys.stderr)
- sys.exit(1)
- token = os.environ['GITHUB_AUTH']
- sys.exit(main(token))
+#!/bin/bash
+#
+# This file is part of GetTor, a Tor Browser distribution system.
+#
+# :authors: hiro <hiro at torproject.org>
+# see also AUTHORS file
+#
+# :copyright: (c) 2008-2019, The Tor Project, Inc.
+#
+# :license: This is Free Software. See LICENSE for license information.
+
+cd ~/releases
+git checkout master
+git branch -D torbrowser-releases
+git push github --delete torbrowser-releases
+
+git fetch --all --prune
+
+git add .
+git commit -am 'Create release branch'
+git checkout -b torbrowser-releases
+git push -f --follow-tags origin torbrowser-releases
+git push -f --follow-tags github torbrowser-releases
+
+
+for row in $(
+ curl -s 'https://aus1.torproject.org/torbrowser/update_3/release/downloads.json' |
+ jq -r '.downloads'
+ ); do
+ r=$(
+ echo ${row} |
+ egrep -o 'https?://[^ ]+' |
+ tr -d '",'
+ );
+ if [[ $r = *[!\ ]* ]]; then
+ git fetch --all
+ wget $r
+ git add .
+ git commit -m '[dist ci] commit from CI runner - update with new torbrowser downloads'
+ diffs=$(git diff origin/torbrowser-releases)
+ if [ -z "$diffs" ]; then
+ echo "No new releases"
+ else
+ git push -f --follow-tags origin torbrowser-releases
+ fi
+ diffs=$(git diff github/torbrowser-releases)
+ if [ -z "$diffs" ]; then
+ echo "No new releases"
+ else
+ git push -f --follow-tags github torbrowser-releases
+ fi
+ fi;
+done
+
+rclone delete gdrive:releases
+
+for f in $(ls); do
+ # Update Google Drive
+ rclone copy $f gdrive:releases
+ # Update Internet Archive
+ ia upload ${f} $f --remote-name=$f --metadata="title:${f}" --metadata="mediatype:software" --metadata="collection:open_source_software"
+done
diff --git a/scripts/update_github b/scripts/update_github
new file mode 100755
index 0000000..6d087e6
--- /dev/null
+++ b/scripts/update_github
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+#pip3 install PyGithub
+
+# Before running, place the authentication token for a user with push access to the repository
+# in an environment variable 'GITHUB_AUTH'
+
+from github import Github
+
+import sys
+import json
+import urllib
+import subprocess
+import os
+
+REPO_NAME = "TheTorProject/gettorbrowser"
+
+failure = False
+
+failed_uploads = []
+
+def delete_old_releases(repo):
+ for release in repo.get_releases():
+ for asset in release.get_assets():
+ asset.delete_asset()
+ release.delete_release()
+
+#Download list of tor browser releases and upload them to github
+def upload_files(release):
+ url = urllib.request.urlopen("https://aus1.torproject.org/torbrowser/update_3/release/downloads.json")
+ data = json.loads(url.read().decode())
+ for arch in data['downloads']:
+ for locale in data['downloads'][arch]:
+ for asset in data['downloads'][arch][locale]:
+ url = data['downloads'][arch][locale][asset]
+ filename = url.split('/')[-1]
+ try:
+ subprocess.check_call(["/usr/bin/wget", "--quiet", url])
+ release.upload_asset(filename)
+ os.remove(filename)
+ except:
+ print("Error: failed to update "+url+". Will retry later.")
+ failed_uploads.append(url)
+ #Retry failed uploads
+ for url in failed_uploads:
+ filename = url.split('/')[-1]
+ try:
+ subprocess.check_call(["/usr/bin/wget", "--quiet", url])
+ release.upload_asset(filename)
+ os.remove(filename)
+ except:
+ print("Error: failed to update "+url+". Please upload this file manually.")
+ failure = True
+
+
+def main(token):
+
+ #Initialize a new release
+ g = Github(token)
+ repo = g.get_repo(REPO_NAME)
+
+ delete_old_releases(repo)
+
+ #Create a new release
+ release = repo.create_git_release("torbrowser-release", "Tor Browser releases", "These releases were uploaded to be distributed with gettor.")
+ upload_files(release)
+
+ if failure:
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ if 'GITHUB_AUTH' not in os.environ:
+ print("Usage: {}".format(sys.argv[0]), file=sys.stderr)
+ print("\nThe authentication token for github should be placed in the environment"
+ "variable 'GITHUB_AUTH'", file=sys.stderr)
+ sys.exit(1)
+ token = os.environ['GITHUB_AUTH']
+ sys.exit(main(token))
More information about the tor-commits
mailing list