[tor-commits] [gettor/master] Extend script to update gitlab remote

cohosh at torproject.org cohosh at torproject.org
Thu Jan 23 01:50:16 UTC 2020


commit c63cb34499ffb348711104e1c5adbc1c7f168eef
Author: Cecylia Bocovich <cohosh at torproject.org>
Date:   Fri Jan 10 17:12:31 2020 -0500

    Extend script to update gitlab remote
    
    Updated and refactored the script to upload tor browser binaries to the
    gitlab remote using the GitLab REST API.
---
 scripts/update_github | 140 ++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 107 insertions(+), 33 deletions(-)

diff --git a/scripts/update_github b/scripts/update_github
index 34eb6c0..f896cd0 100755
--- a/scripts/update_github
+++ b/scripts/update_github
@@ -1,17 +1,22 @@
 #!/usr/bin/env python3
 
-#pip3 install PyGithub
+# Dependencies:
+#  pip3 install PyGithub
+#  pip3 install python-gitlab
 
-# Before running, place the authentication token for a user with push access to the repository
-# in an environment variable 'GITHUB_AUTH'
+# Before running, place the authentication token for a user with push access to
+# the Github repository in an environment variable 'GITHUB_AUTH' and a token to
+# access the GitLab repository in an environment variable 'GITLAB_AUTH'
 
 from github import Github
+from gitlab import Gitlab
 
 import sys
 import json
 import urllib
 import subprocess
 import os
+import base64
 
 REPO_NAME = "torproject/torbrowser-releases"
 
@@ -19,14 +24,73 @@ failure = False
 
 failed_uploads = []
 
-def delete_old_releases(repo):
-    for release in repo.get_releases():
-        for asset in release.get_assets():
-            asset.delete_asset()
-        release.delete_release()
+class GitlabRemote:
+    def __init__(self, token):
+        self.gl = Gitlab("https://gitlab.com", private_token=token)
+
+    def delete_old_projects(self):
+        projects = self.gl.projects.list(owned=True)
+        for project in projects:
+            project.delete()
+
+    def create_projects(self):
+        # Create one project for each platform
+        self.windows = self.gl.projects.create({'name': 'torbrowser-windows'})
+        self.mac = self.gl.projects.create({'name': 'torbrowser-osx'})
+        self.linux = self.gl.projects.create({'name': 'torbrowser-linux'})
+
+    def upload(self, filename, arch):
+        with open(filename, 'rb') as f:
+            contents = f.read()
+        data = {
+            'branch': 'master',
+            'commit_message': 'GetTor automated commit - update with new Tor Browser downloads',
+            'actions': [
+                {
+                    # Binary files need to be base64 encoded
+                    'action': 'create',
+                    'file_path': filename,
+                    'content': base64.b64encode(contents),
+                    'encoding': 'base64',
+                }
+            ]
+        }
+
+        if arch == 'linux32' or arch == 'linux64':
+            commit = self.linux.commits.create(data)
+        elif arch == 'win32' or arch == 'win64':
+            commit = self.windows.commits.create(data)
+        elif arch == 'osx64':
+            commit = self.mac.commits.create(data)
+        else:
+            # Something changed with the structure of the downloads json file
+            print("Unkown architecture: " + arch)
+            sys.exit(1)
+
+
+class GithubRemote:
+    def __init__(self, token):
+        g = Github(token)
+        self.repo = g.get_repo(REPO_NAME)
+
+    def delete_old_releases(self):
+        for release in self.repo.get_releases():
+            for asset in release.get_assets():
+                asset.delete_asset()
+            release.delete_release()
+
+    def create_release(self):
+        self.release = self.repo.create_git_release("torbrowser-release", "Tor Browser releases", "These releases were uploaded to be distributed with gettor.")
+
+    def upload(self, filename, arch):
+        self.release.upload_asset(filename)
 
 #Download list of tor browser releases and upload them to github
-def upload_files(release):
+def upload_files(remotes):
+    if len(remotes) == 0:
+        print("ERROR: No remotes to update", file=sys.stderr)
+        return 1
+
     url = urllib.request.urlopen("https://aus1.torproject.org/torbrowser/update_3/release/downloads.json")
     data = json.loads(url.read().decode())
     for arch in data['downloads']:
@@ -36,44 +100,54 @@ def upload_files(release):
                 filename = url.split('/')[-1]
                 try:
                     subprocess.check_call(["/usr/bin/wget", "--quiet", url])
-                    release.upload_asset(filename)
-                    os.remove(filename)
+
                 except:
-                    print("Error: failed to update "+url+". Will retry later.")
+                    print("Error: failed to fetch "+url+". Will retry later.")
                     failed_uploads.append(url)
+                    continue
+
+                for remote in remotes:
+                    remote.upload(filename, arch)
+                os.remove(filename)
     #Retry failed uploads
     for url in failed_uploads:
         filename = url.split('/')[-1]
         try:
             subprocess.check_call(["/usr/bin/wget", "--quiet", url])
-            release.upload_asset(filename)
+            for remote in remotes:
+                remote.upload(filename, arch)
             os.remove(filename)
         except:
             print("Error: failed to update "+url+". Please upload this file manually.")
             failure = True
 
-
-def main(token):
-
-    #Initialize a new release
-    g = Github(token)
-    repo = g.get_repo(REPO_NAME)
-
-    delete_old_releases(repo)
-
-    #Create a new release
-    release = repo.create_git_release("torbrowser-release", "Tor Browser releases", "These releases were uploaded to be distributed with gettor.")
-    upload_files(release)
-
     if failure:
-        sys.exit(1)
+        return 1
 
 
-if __name__ == "__main__":
+def main():
+    remotes = []
     if 'GITHUB_AUTH' not in os.environ:
-        print("Usage: {}".format(sys.argv[0]), file=sys.stderr)
-        print("\nThe authentication token for github should be placed in the environment"
+        print("WARNING: No Github authentication token given", file=sys.stderr)
+        print("The authentication token for github should be placed in the environment"
                 "variable 'GITHUB_AUTH'", file=sys.stderr)
-        sys.exit(1)
-    token = os.environ['GITHUB_AUTH']
-    sys.exit(main(token))
+    else:
+        github = GithubRemote(os.environ['GITHUB_AUTH'])
+        github.delete_old_releases()
+        github.create_release()
+        remotes.append(github)
+
+    if 'GITLAB_AUTH' not in os.environ:
+        print("WARNING: No GitLab authenticatin token given", file=sys.stderr)
+        print("The authentication token for gitlab should be placed in the environment"
+                "variable 'GITLAB_AUTH'", file=sys.stderr)
+    else:
+        gitlab = GitlabRemote(os.environ['GITLAB_AUTH'])
+        gitlab.delete_old_projects()
+        gitlab.create_projects()
+        remotes.append(gitlab)
+
+    return upload_files(remotes)
+
+if __name__ == "__main__":
+    sys.exit(main())





More information about the tor-commits mailing list