[tor-commits] [gettor/master] implemented google drive as a provider
ilv at torproject.org
ilv at torproject.org
Tue Sep 22 23:39:13 UTC 2015
commit a09ca92dce752eee8311879fbd568e49bff428b6
Author: poly <poly at darkdepths.net>
Date: Fri Dec 12 17:03:14 2014 +0400
implemented google drive as a provider
---
UPLOAD-GOOGLE-DRIVE | 21 ++++
bundles2drive.py | 335 +++++++++++++++++++++++++++++++++++++++++++++++++++
drive.cfg | 9 ++
drive.links | 25 ++++
providers.txt | 4 +-
5 files changed, 392 insertions(+), 2 deletions(-)
diff --git a/UPLOAD-GOOGLE-DRIVE b/UPLOAD-GOOGLE-DRIVE
new file mode 100644
index 0000000..c9a7184
--- /dev/null
+++ b/UPLOAD-GOOGLE-DRIVE
@@ -0,0 +1,21 @@
+1) Clone into the latest version of gettor:
+
+$ git clone https://github.com/ileiva/gettor.git
+
+2) Get the PGP key that signs the Tor Browser Bundle
+
+2) Visit https://console.developers.google.com//start/api?id=drive&credential=client_key and follow OAUTH2 process to get client ID and client secret for 'other' desktop application.
+
+3) Edit drive.cfg to with new client-id and secret. Leave refresh_token empty.
+
+4) Install the google drive API python client:
+
+$ pip install --upgrade google-api-python-client
+
+5) Run the script:
+
+$ python bundles2drive.py
+
+The first time the script is run, you will have to authorize it through a web browser. You will be prompted with a URL. Once that is done, a refresh token will be stored locally so that re-authorzing is unnesaccary.
+
+The script will then look for files in upload_dir (as specified in drive.cfg) and upload them to google drive. If no errors occur, it will then add formatted links and hash information to the drive.links file.
diff --git a/bundles2drive.py b/bundles2drive.py
new file mode 100644
index 0000000..6cf9513
--- /dev/null
+++ b/bundles2drive.py
@@ -0,0 +1,335 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of GetTor, a Tor Browser distribution system.
+#
+# :authors: poly <poly at darkdepths.net>
+# Israel Leiva <ilv at riseup.net>
+# see also AUTHORS file
+#
+# :copyright: (c) 2008-2014, The Tor Project, Inc.
+# (c) 2014, Poly
+# (c) 2014, Israel Leiva
+#
+# :license: This is Free Software. See LICENSE for license information.
+
+import re
+import os
+import gnupg
+import hashlib
+import ConfigParser
+import gettor.core
+
+#import google drive libs
+import httplib2
+from apiclient.discovery import build
+from apiclient.http import MediaFileUpload
+from oauth2client.client import OAuth2WebServerFlow
+from oauth2client.client import Credentials
+
+
+def valid_format(file, osys):
+ """Check for valid bundle format
+
+ Check if the given file has a valid bundle format
+ (e.g. tor-browser-linux32-3.6.2_es-ES.tar.xz)
+
+ :param: file (string) the name of the file.
+ :param: osys (string) the OS.
+
+ :return: (boolean) true if the bundle format is valid, false otherwise.
+
+ """
+ if(osys == 'windows'):
+ m = re.search(
+ 'torbrowser-install-\d\.\d\.\d_\w\w(-\w\w)?\.exe',
+ file)
+ elif(osys == 'linux'):
+ m = re.search(
+ 'tor-browser-linux\d\d-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz',
+ file)
+ elif(osys == 'osx'):
+ m = re.search(
+ 'TorBrowser-\d\.\d\.\d-osx\d\d_(\w\w)(-\w\w)?\.dmg',
+ file)
+ if m:
+ return True
+ else:
+ return False
+
+
+def get_bundle_info(file, osys):
+ """Get the os, arch and lc from a bundle string.
+
+ :param: file (string) the name of the file.
+ :param: osys (string) the OS.
+
+ :raise: ValueError if the bundle doesn't have a valid bundle format.
+
+ :return: (list) the os, arch and lc.
+
+ """
+ if(osys == 'windows'):
+ m = re.search(
+ 'torbrowser-install-\d\.\d\.\d_(\w\w)(-\w\w)?\.exe',
+ file)
+ if m:
+ lc = m.group(1)
+ return 'windows', '32/64', lc
+ else:
+ raise ValueError("Invalid bundle format %s" % file)
+ elif(osys == 'linux'):
+ m = re.search(
+ 'tor-browser-linux(\d\d)-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz',
+ file)
+ if m:
+ arch = m.group(1)
+ lc = m.group(2)
+ return 'linux', arch, lc
+ else:
+ raise ValueError("Invalid bundle format %s" % file)
+ elif(osys == 'osx'):
+ m = re.search(
+ 'TorBrowser-\d\.\d\.\d-osx(\d\d)_(\w\w)(-\w\w)?\.dmg',
+ file)
+ if m:
+ os = 'osx'
+ arch = m.group(1)
+ lc = m.group(2)
+ return 'osx', arch, lc
+ else:
+ raise ValueError("Invalid bundle format %s" % file)
+
+
+def get_file_sha256(file):
+ """Get the sha256 of a file.
+
+ :param: file (string) the path of the file.
+
+ :return: (string) the sha256 hash.
+
+ """
+ # as seen on the internetz
+ BLOCKSIZE = 65536
+ hasher = hashlib.sha256()
+ with open(file, 'rb') as afile:
+ buf = afile.read(BLOCKSIZE)
+ while len(buf) > 0:
+ hasher.update(buf)
+ buf = afile.read(BLOCKSIZE)
+ return hasher.hexdigest()
+
+
+def upload_files(client, basedir):
+ """Upload files to Google Drive.
+
+ Looks for tor browser files inside basedir.
+
+ :param: basedir (string) path of the folder with the files to be
+ uploaded.
+ :param: client (object) Google Drive object.
+
+ :raise: UploadError if something goes wrong while uploading the
+ files to Google Drive. All files are uploaded to '/'.
+
+ :return: (dict) the names of the uploaded files as the keys,
+ and file id as the value
+
+ """
+ files = []
+
+ p = re.compile('.*\.tar.xz$')
+ for name in os.listdir(basedir):
+ path = os.path.abspath(os.path.join(basedir, name))
+ if os.path.isfile(path) and p.match(path)\
+ and valid_format(name, 'linux'):
+ files.append(name)
+
+ p = re.compile('.*\.exe$')
+ for name in os.listdir(basedir):
+ path = os.path.abspath(os.path.join(basedir, name))
+ if os.path.isfile(path) and p.match(path)\
+ and valid_format(name, 'windows'):
+ files.append(name)
+
+ p = re.compile('.*\.dmg$')
+ for name in os.listdir(basedir):
+ path = os.path.abspath(os.path.join(basedir, name))
+ if os.path.isfile(path) and p.match(path)\
+ and valid_format(name, 'osx'):
+ files.append(name)
+
+ # dictionary to store file names and IDs
+ files_dict = dict()
+
+ for file in files:
+ asc = "%s.asc" % file
+ abs_file = os.path.abspath(os.path.join(basedir, file))
+ abs_asc = os.path.abspath(os.path.join(basedir, asc))
+
+ if not os.path.isfile(abs_asc):
+ # there are some .mar files that don't have .asc, don't upload it
+ continue
+
+ # upload tor browser installer
+ file_body = MediaFileUpload(abs_file, resumable=True)
+ body = {
+ 'title': file
+ }
+ print "Uploading '%s'..." % file
+ try:
+ file_data = drive_service.files().insert(body=body, media_body=file_body).execute()
+ except:
+ raise UploadError
+
+ # upload signature
+ asc_body = MediaFileUpload(abs_asc, resumable=True)
+ asc_head = {
+ 'title': "%s.asc" % file
+ }
+ print "Uploading '%s'..." % asc
+ try:
+ asc_data = drive_service.files().insert(body=asc_head, media_body=asc_body).execute()
+ except:
+ raise UploadError
+
+ # add filenames and file id to dict
+ files_dict[file] = file_data['id']
+ files_dict[asc] = asc_data['id']
+
+ return files_dict
+
+def share_file(service, file_id):
+ """Make files public
+
+ For a given file-id, sets role 'reader' to 'anyone'. Returns public
+ link to file.
+
+ :param: file_id (string)
+
+ :return: (string) url to shared file
+
+ """
+ permission = {
+ 'type': "anyone",
+ 'role': "reader",
+ 'withLink': True
+ }
+
+ try:
+ service.permissions().insert(
+ fileId=file_id, body=permission).execute()
+ except errors.HttpError, error:
+ print('An error occured while sharing: %s' % file_id)
+
+ try:
+ file = service.files().get(fileId=file_id).execute()
+ except errors.HttpError, error:
+ print('Error occured while fetch public link for file: %s' % file_id)
+
+ print("Uploaded to %s" % file['webContentLink'])
+ return file['webContentLink']
+
+
+if __name__ == '__main__':
+ config = ConfigParser.ConfigParser()
+ config.read('drive.cfg')
+
+ client_id = config.get('app', 'client-id')
+ app_secret = config.get('app', 'secret')
+ refresh_token = config.get('app', 'refresh_token')
+ upload_dir = config.get('general', 'upload_dir')
+
+ # important: this key must be the one that signed the packages
+ tbb_key = config.get('general', 'tbb_key')
+
+ # requests full access to drive account
+ OAUTH_SCOPE = 'https://www.googleapis.com/auth/drive'
+ REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'
+
+ print "Authenticating..."
+
+ flow = OAuth2WebServerFlow(client_id, app_secret, OAUTH_SCOPE,
+ redirect_uri=REDIRECT_URI)
+
+ # If no valid token found, need to prompt user.
+ # this should only occur once
+ if not refresh_token:
+ flow.params['access_type'] = 'offline'
+ flow.params['approval_prompt'] = 'force'
+ authorize_url = flow.step1_get_authorize_url()
+ print 'Go to the following link in your browser: ' + authorize_url
+ code = raw_input('Enter verification code: ').strip()
+ credentials = flow.step2_exchange(code)
+
+ # oauth2 credentials instance must be stored as json string
+ config.set('app', 'refresh_token', credentials.to_json())
+ with open('drive.cfg', 'wb') as configfile:
+ config.write(configfile)
+ else:
+ # we already have a valid token
+ credentials = Credentials.new_from_json(refresh_token)
+
+ # authenticate with oauth2
+ http = httplib2.Http()
+ http = credentials.authorize(http)
+
+ # initialize drive instance
+ drive_service = build('drive', 'v2', http=http)
+
+
+ # import key fingerprint
+ gpg = gnupg.GPG()
+ key_data = open(tbb_key).read()
+ import_result = gpg.import_keys(key_data)
+ fp = import_result.results[0]['fingerprint']
+
+ # make groups of four characters to make fingerprint more readable
+ # e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J
+ readable = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4))
+
+ try:
+ uploaded_files = upload_files(drive_service, upload_dir)
+ # use default config
+ core = gettor.core.Core('/home/gettor/core.cfg')
+
+ # erase old links
+ core.create_links_file('Drive', readable)
+
+ # recognize file OS by its extension
+ p1 = re.compile('.*\.tar.xz$')
+ p2 = re.compile('.*\.exe$')
+ p3 = re.compile('.*\.dmg$')
+ p4 = re.compile('.*\.asc$')
+
+ for file in uploaded_files.keys():
+ # only run for tor browser installers
+ if p4.match(file):
+ continue
+ asc = "%s.asc" % file
+ abs_file = os.path.abspath(os.path.join(upload_dir, file))
+ abs_asc = os.path.abspath(os.path.join(upload_dir, asc))
+
+ sha_file = get_file_sha256(abs_file)
+
+ # build links
+ link_file = share_file(drive_service,
+ uploaded_files[file])
+ link_asc = share_file(drive_service,
+ uploaded_files["%s.asc" % file])
+
+ if p1.match(file):
+ osys, arch, lc = get_bundle_info(file, 'linux')
+ elif p2.match(file):
+ osys, arch, lc = get_bundle_info(file, 'windows')
+ elif p3.match(file):
+ osys, arch, lc = get_bundle_info(file, 'osx')
+
+ link = "Package (%s-bit): %s\nASC signature (%s-bit): %s\n"\
+ "Package SHA256 checksum (%s-bit): %s\n" %\
+ (arch, link_file, arch, link_asc,
+ arch, sha_file)
+
+ # note that you should only upload bundles for supported locales
+ core.add_link('Drive', osys, lc, link)
+ except (ValueError, RuntimeError) as e:
+ print str(e)
diff --git a/drive.cfg b/drive.cfg
new file mode 100644
index 0000000..b400af0
--- /dev/null
+++ b/drive.cfg
@@ -0,0 +1,9 @@
+[general]
+upload_dir = upload
+tbb_key = tbb-key.asc
+
+[app]
+client-id =
+secret =
+refresh_token =
+
diff --git a/drive.links b/drive.links
new file mode 100644
index 0000000..5f4fa89
--- /dev/null
+++ b/drive.links
@@ -0,0 +1,25 @@
+[provider]
+name = Drive
+
+[key]
+fingerprint = A3C4 F0F9 79CA A22C DBA8 F512 EE8C BC9E 886D DD89
+
+[linux]
+en = Package (32-bit): https://docs.google.com/uc?id=0B7humfJYX-LjNTJPanBRekI5Slk&export=download
+ ASC signature (32-bit): https://docs.google.com/uc?id=0B7humfJYX-LjTXVONkhvZ1VRY2c&export=download
+ Package SHA256 checksum (32-bit): 03a4ecab9ffb4b579c9d03a0f800d9bcd4446b49eff21eb3e1db255fa9d9b930,
+ Package (64-bit): https://docs.google.com/uc?id=0B7humfJYX-LjbXp2MUh5RmxTa28&export=download
+ ASC signature (64-bit): https://docs.google.com/uc?id=0B7humfJYX-LjNTVZVnJUTnNCOEk&export=download
+ Package SHA256 checksum (64-bit): 03a4ecab9ffb4b579c9d03a0f800d9bcd4446b49eff21eb3e1db255fa9d9b930
+
+
+[windows]
+en = Package (32/64-bit): https://docs.google.com/uc?id=0B7humfJYX-LjakM4a21iRlQzQVE&export=download
+ ASC signature (32/64-bit): https://docs.google.com/uc?id=0B7humfJYX-LjZy1FLVpTLW9INFU&export=download
+ Package SHA256 checksum (32/64-bit): 21e54d964366e8a67f379ed402007f59cb3aebfddbc3a30dacc3ac64171145a6
+
+[osx]
+en = Package (32-bit): https://docs.google.com/uc?id=0B7humfJYX-LjenRobzJtcWw0UnM&export=download
+ ASC signature (32-bit): https://docs.google.com/uc?id=0B7humfJYX-LjZ0R4Z0I1ZlBRUHc&export=download
+ Package SHA256 checksum (32-bit): dd86727270ba236d85673194782b4cc9860bd886e4abdf2dfc6d716dabddb68f
+
diff --git a/providers.txt b/providers.txt
index 0f57719..2e8f4cd 100644
--- a/providers.txt
+++ b/providers.txt
@@ -1,10 +1,10 @@
================================== CURRENT ====================================
* Dropbox: Implemented.
+* Google Drive: Implemented.
-================================== TO BE IMPLEMENTED =========================
+============================= TO BE IMPLEMENTED ===============================
-* Google Drive: Not implemented yet.
================================== IDEAS =====================================
More information about the tor-commits
mailing list