tor-commits November 2015

tor-commits@lists.torproject.org

20 participants
856 discussions

[translation/tails-greeter_completed] Update translations for tails-greeter_completed
by translation＠torproject.org 03 Nov '15

03 Nov '15

commit 630b48d9f550a2417a58cb54d058982c4ea862b3 Author: Translation commit bot <translation(a)torproject.org> Date: Tue Nov 3 20:15:31 2015 +0000 Update translations for tails-greeter_completed --- tr/tr.po | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tr/tr.po b/tr/tr.po index 3d5e0e1..cb2a7c3 100644 --- a/tr/tr.po +++ b/tr/tr.po @@ -18,8 +18,8 @@ msgid "" msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2015-09-20 18:20+0200\n" -"PO-Revision-Date: 2015-10-17 18:25+0000\n" +"POT-Creation-Date: 2015-11-02 21:29+0100\n" +"PO-Revision-Date: 2015-11-03 20:06+0000\n" "Last-Translator: Kaya Zeren <kayazeren(a)gmail.com>\n" "Language-Team: Turkish (http://www.transifex.com/otf/torproject/language/tr/)\n" "MIME-Version: 1.0\n" @@ -168,6 +168,10 @@ msgid "" "need to configure bridge, firewall, or proxy settings." msgstr "Bu bilgisayarın İnternet bağlantısı sansürleniyor, süzülüyor ya da vekil sunucuya bağlı. Köprü, güvenlik duvarı ya da vekil sunucu ayarlarını yapmanız gerekir." +#: ../glade/optionswindow.glade.h:23 +msgid "Disable all networking" +msgstr "Tüm ağı devre dışı bırak" + #: ../glade/langpanel.glade.h:1 msgid " " msgstr " "

1 0

[translation/tails-greeter] Update translations for tails-greeter
by translation＠torproject.org 03 Nov '15

03 Nov '15

commit d39667f4dc9daecc6679866821df8e96ab2572c1 Author: Translation commit bot <translation(a)torproject.org> Date: Tue Nov 3 20:15:27 2015 +0000 Update translations for tails-greeter --- tr/tr.po | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tr/tr.po b/tr/tr.po index ed3d8b4..cb2a7c3 100644 --- a/tr/tr.po +++ b/tr/tr.po @@ -18,9 +18,9 @@ msgid "" msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2015-10-26 17:51+0100\n" -"PO-Revision-Date: 2015-10-27 10:10+0000\n" -"Last-Translator: carolyn <carolyn(a)anhalt.org>\n" +"POT-Creation-Date: 2015-11-02 21:29+0100\n" +"PO-Revision-Date: 2015-11-03 20:06+0000\n" +"Last-Translator: Kaya Zeren <kayazeren(a)gmail.com>\n" "Language-Team: Turkish (http://www.transifex.com/otf/torproject/language/tr/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -170,7 +170,7 @@ msgstr "Bu bilgisayarın İnternet bağlantısı sansürleniyor, süzülüyor ya #: ../glade/optionswindow.glade.h:23 msgid "Disable all networking" -msgstr "" +msgstr "Tüm ağı devre dışı bırak" #: ../glade/langpanel.glade.h:1 msgid " "

1 0

[gettor/master] Improvements on Twitter channel
by ilv＠torproject.org 03 Nov '15

03 Nov '15

commit a14cad3472e312fc939ddb7db8803048c6d26e4e Author: ilv <ilv(a)users.noreply.github.com> Date: Mon Aug 31 20:12:31 2015 -0300 Improvements on Twitter channel --- gettor/twitter.py | 595 +++++++++++++++++++++++++++++++++++++++++++++++++++++ process_tweets.py | 35 ++++ 2 files changed, 630 insertions(+) diff --git a/gettor/twitter.py b/gettor/twitter.py new file mode 100644 index 0000000..05320ef --- /dev/null +++ b/gettor/twitter.py @@ -0,0 +1,595 @@ +# -*- coding: utf-8 -*- +# +# This file is part of GetTor, a Tor Browser distribution system. +# +# :authors: Israel Leiva <ilv(a)riseup.net> +# Based on BridgeDB Twitter distributor (PoC) by wfn +# - https://github.com/wfn/twidibot +# +# :copyright: (c) 2008-2015, The Tor Project, Inc. +# (c) 2015, Israel Leiva +# +# :license: This is Free Software. See LICENSE for license information. + +import sys +import json +import time +import signal +import tweepy +import gettext + +from tweepy.models import Status + +import core +import utils +import blacklist + +"""Twitter module for processing requests. Forked from BridgeDB Twitter +distributor by wfn (https://github.com/wfn/twidibot)""" + + +class ConfigError(Exception): + pass + + +class InternalError(Exception): + pass + + +class TwitterBotStreamListener(tweepy.StreamListener): + """Listener for twitter's Streaming API.""" + + def __init__(self, bot, api=None): + self.bot = bot + self.processing_data = False + + super(TwitterBotStreamListener, self).__init__(api) + + def on_data(self, raw_data): + """Called when raw data is received from connection. + + This is where all the data comes first. Normally we could use + (inherit) the on_data() in tweepy.StreamListener, but it unnecessarily + and naively reports unknown event types as errors (to simple log); + also, we might want to tweak it further later on. + + But for now, this is basically taken from tweepy's on_data(). + + Return False to stop stream and close connection. + + """ + + self.processing_data = True + + data = json.loads(raw_data) + + if 'in_reply_to_status_id' in data: + status = Status.parse(self.api, data) + if self.on_status(status) is False: + return False + elif 'delete' in data: + delete = data['delete']['status'] + if self.on_delete(delete['id'], delete['user_id']) is False: + return False + elif 'event' in data: + status = Status.parse(self.api, data) + if self.on_event(status) is False: + return False + elif 'direct_message' in data: + status = Status.parse(self.api, data) + if self.on_direct_message(status) is False: + return False + elif 'limit' in data: + if self.on_limit(data['limit']['track']) is False: + return False + elif 'disconnect' in data: + if self.on_disconnect(data['disconnect']) is False: + return False + else: + # we really are ok to receive unknown stream/event types. + # log to debug? + log.debug('TwitterBotStreamListener::on_data(): got event/stream' + ' data of unknown type. Raw data follows:\n%s', data) + + self.processing_data = False + + def on_status(self, status): + """Called when a new status arrives""" + + #log.debug('Got status: %s', status) + return + + def on_event(self, status): + """Called when a new event arrives""" + + #log.debug('Got event: %s', status) + + # XXX make sure tweepy's given 'status.event' unicode string can + # always be safely converted to ascii + + # now it seems one can reply to dm without following the account + # if str(status.event) == 'follow': + # self.bot.handleFollowEvent(status) + + return + + def on_direct_message(self, status): + """Called when a new direct message arrives or is sent from us + + TODO: make a pull request for tweepy or something, because they + say it's only when a direct message is *received* (implying, 'by us') + + """ + + # doing twitter user comparisons using id_str makes sense here - it's + # safe and id_str's are guaranteed to be unique (re: latter, just like + # id's.) maybe consider deciding how comparisons should be made for sure, + # and then extend tweepy.models.User to include __eq__? + if status.direct_message['sender']['id_str'] != self.bot.bot_info.id_str: + self.bot.handleDirectMessage(status) + else: + # log.debug('Caught a direct message sent *from* us') + pass + + return + + def on_connect(self): + """Called once connected to streaming server. + + This will be invoked once a successful response + is received from the server. Allows the listener + to perform some work prior to entering the read loop. + + """ + pass + + def on_exception(self, exception): + """Called when an unhandled exception occurs.""" + return + + def on_delete(self, status_id, user_id): + """Called when a delete notice arrives for a status""" + return + + def on_limit(self, track): + """Called when a limitation notice arrvies""" + return + + def on_error(self, status_code): + """Called when a non-200 status code is returned""" + return False + + def on_timeout(self): + """Called when stream connection times out""" + return + + def on_disconnect(self, notice): + """Called when twitter sends a disconnect notice + + Disconnect codes are listed here: + https://dev.twitter.com/docs/streaming-apis/messages + #Disconnect_messages_disconnect + + """ + return + + +class TwitterBot(object): + """Main interface between the stateful listener and Twitter APIs.""" + + # TODO: think about secure ways of storing twitter access config. + # For one, app itself should ideally not be able to have write access + # to it. For another, ideally it would request details from some other + # component, authenticate, and not be able to re-authenticate to twitter + + """ + default_access_config = { + 'api_key': config.API_KEY, + 'api_secret': config.API_SECRET, + 'access_token': config.ACCESS_TOKEN, + 'token_secret': config.TOKEN_SECRET + }""" + + def __init__(self, **kw): + """Constructor that accepts custom access config as named arguments + + Easy to test things from interactive shell this way. + Probably won't be needed in production code. + + """ + + """ + self.access_config = dict() + for key, default in self.default_access_config.iteritems(): + self.access_config[key] = kw.get(key, default) + """ + + default_cfg = 'twitter.cfg' + config = ConfigParser.ConfigParser() + + if cfg is None or not os.path.isfile(cfg): + cfg = default_cfg + + try: + with open(cfg) as f: + config.readfp(f) + except IOError: + raise ConfigError("File %s not found!" % cfg) + + try: + self.api_key = config.get('access_config', 'api_key') + self.api_secret = config.get('access_config', 'api_secret') + self.access_token = config.get('access_config', 'access_token') + self.token_secret = config.get('access_config', 'token_secret') + + self.async_streaming = config.get('api', 'async_streaming') + self.char_limit = config.get('api', 'char_limit') + + self.mirrors = config.get('general', 'mirrors') + self.max_words = config.get('general', 'max_words') + self.i18ndir = config.get('i18n', 'dir') + + logdir = config.get('log', 'dir') + logfile = os.path.join(logdir, 'twitter.log') + loglevel = config.get('log', 'level') + + blacklist_cfg = config.get('blacklist', 'cfg') + self.bl = blacklist.Blacklist(blacklist_cfg) + self.bl_max_req = config.get('blacklist', 'max_requests') + self.bl_max_req = int(self.bl_max_req) + self.bl_wait_time = config.get('blacklist', 'wait_time') + self.bl_wait_time = int(self.bl_wait_time) + + core_cfg = config.get('general', 'core_cfg') + self.core = core.Core(core_cfg) + + except ConfigParser.Error as e: + raise ConfigError("Configuration error: %s" % str(e)) + except blacklist.ConfigError as e: + raise InternalError("Blacklist error: %s" % str(e)) + except core.ConfigError as e: + raise InternalError("Core error: %s" % str(e)) + + # logging + log = logging.getLogger(__name__) + + logging_format = utils.get_logging_format() + date_format = utils.get_date_format() + formatter = logging.Formatter(logging_format, date_format) + + log.info('Redirecting SMTP logging to %s' % logfile) + logfileh = logging.FileHandler(logfile, mode='a+') + logfileh.setFormatter(formatter) + logfileh.setLevel(logging.getLevelName(loglevel)) + log.addHandler(logfileh) + + # stop logging on stdout from now on + log.propagate = False + self.log = log + + self.setSignalHandlers() + self.msg = Messages() + + + def _is_blacklisted(self, account): + """Check if a user is blacklisted. + + :param: addr (string) the hashed address of the user. + + :return: true is the address is blacklisted, false otherwise. + + """ + anon_acc = utils.get_sha256(account) + + try: + self.bl.is_blacklisted( + anon_acc, 'Twitter', self.bl_max_req, self.bl_wait_time + ) + return False + except blacklist.BlacklistError as e: + return True + + + def _get_msg(self, msgid, lc): + """Get message identified by msgid in a specific locale. + + Params: msgid: the identifier of a string. + lc: the locale. + + Return: a string containing the given message. + + """ + self.log.debug("Getting message '%s' for locale %s" % (msgid, lc)) + try: + t = gettext.translation(lc, self.i18ndir, languages=[lc]) + _ = t.ugettext + + msgstr = _(msgid) + return msgstr + except IOError as e: + raise ConfigError("%s" % str(e)) + + + def _parse_request(self, message): + """ """ + self.log.debug("Parsing text.") + + # core knows what OS are supported + supported_os = self.core.get_supported_os() + supported_lc = self.core.get_supported_lc() + + # default values + req = {} + req['lc'] = 'en' + req['os'] = None + req['type'] = 'help' + + found_lc = False + found_os = False + found_mirrors = False + + # analyze every word + for word in message.split(' '): + # look for lc and os + if not found_lc: + for lc in supported_lc: + if re.match(lc, word, re.IGNORECASE): + found_lc = True + req['lc'] = lc + if not found_os: + for os in supported_os: + if re.match(os, word, re.IGNORECASE): + found_os = True + req['os'] = os + req['type'] = 'links' + # mirrors + if not found_mirrors: + if re.match("mirrors?", word, re.IGNORECASE): + found_mirrors = True + req['type'] = 'mirrors' + if (found_lc and found_os) or (found_lc and found_mirrors): + break + + return req + + + def setSignalHandlers(self): + """Set up relevant SIG* handlers for the bot. + + Note: if we want to handle some specific signal and not exit after + catching it, we may need to store the original CPython handler + (signified by signal.SIG_DFL), and restore it after handling the + signal in question. For now, we'll only care about signals after + which the program does exit. + + """ + + # for now, we'll only handle SIGTERM. it might make sense to handle + # SIGINT as well, though. + + signal.signal(signal.SIGTERM, self.handleSIGTERM) + self.log.debug("SIGTERM handler is set.") + + def handleSIGTERM(self, sig_number, stack_frame): + """Callback function called upon SIGTERM""" + + self.log.info("TwitterBot::handleSIGTERM(): caught SIGTERM signal.") + + self.log.info("Stopping bot listener.") + self.listener.running = False + while self.listener.processing_data: + self.log.info( + "Waiting for TwitterBotStreamListener to finish processing" + " a data request/package" + ) + time.sleep(0.5) + + self.log.info("Closing down storage controller.") + self.storage_controller.closeAll() + + self.log.info("Exiting program.") + sys.exit(0) + + def authenticate(self, auth=None): + """Authenticate to Twitter API, get API handle, and remember it.""" + + if auth: + self.auth = auth + else: + self.auth = tweepy.OAuthHandler( + self.api_key, + self.api_secret + ) + + self.auth.set_access_token( + self.access_token, + self.token_secret + ) + + try: + self.api = tweepy.API(self.auth) + except Exception as e: + self.log.fatal('Exception while authenticating to Twitter and ' + 'getting API handle: %s', e) + self.api = None + finally: + # del self.auth # ideally we'd be able to delete this, but + # presently - no; anything? + pass + + if self.api: + self.log.info('Authenticated and got the RESTful API handle') + self.bot_info = self.api.me() + #api.update_status('hello world!') + + def subscribeToStreams(self): + """Subscribe to relevant streams in the Streaming API.""" + + self.listener = TwitterBotStreamListener( + bot=self, + api=self.api + ) + + self.stream = tweepy.Stream(self.auth, self.listener) + + # user stream gives us direct messages and follow events + self.stream.userstream(async=self.async_streaming) + # stream.filter may be useful, but we don't need it for now + + # the following will not be executed if we're not going async - + # userstream() blocks, its event handler loop takes over: + self.log.info('Subscribed to relevant streams via Streaming API') + + def handleFollowEvent(self, event): + """ + user_id = event.source['id'] # 'id' is unique big int + + if user_id != self.bot_info.id: + user = self.api.get_user(id=user_id) + user.follow() + + if config.RESPOND_AFTER_FOLLOW: + # the following line *blocks* the thread that we care about. + # we should not do this, ever. as long as we're just testing + # with a few cat accounts, it's ok. + + # TODO: use sched.scheduler, or threading.Timer (or sth) + time.sleep(config.WAIT_TIME_AFTER_FOLLOW) + + # for now, english by default + self.sendMessage( + user_id, + get_msg('welcome', 'en') + ) + """ + # it seems that we don't need to be followed to send dm + pass + + + def handleDirectMessage(self, status): + """ Handle direct messages received (i.e. parse request). """ + sender_id = status.direct_message['sender_id'] + message = status.direct_message['text'].strip().lower() + + self.log.debug("Parsing request") + + try: + if self._is_blacklisted(str(sender_id)): + self.log.info("Request from blacklisted account!") + status = 'blacklisted' + bogus_request = True + + # first let's find out how many words are in the message + # request shouldn't be longer than 3 words, but just in case + words = message.split(' ') + if len(words) > self.max_words: + bogus_request = True + self.log.info("Message way too long") + status = 'error' + reply = self._get_msg('message_error', 'en') + + if not bogus_request: + self.log.debug("Request seems legit, let's parse it") + # let's try to guess what the user is asking + req = self._parse_text(str(msg)) + + if req['type'] == 'help': + self.log.debug("Type of request: help") + status = 'success' + reply = self._get_msg('help', 'en') + + elif req['type'] == 'mirrors': + self.log.debug("Type of request: mirrors") + status = 'success' + reply = self._get_msg('mirrors', 'en') + try: + with open(self.mirrors, "r") as list_mirrors: + mirrors = list_mirrors.read() + reply = reply % mirrors + + except IOError as e: + reply = self._get_msg('mirrors_unavailable', 'en') + + elif req['type'] == 'links': + self.log.debug("Type of request: help") + links = self.core.get_links( + "Twitter", + req['os'], + req['lc'] + ) + reply = self._get_msg('links', 'en') + reply = reply % (req['os'], req['lc'], links) + self.sendMessage(sender_id, reply) + + status = 'success' + + # send whatever the reply is + self.sendMessage(sender_id, reply) + + except (core.ConfigError, core.InternalError) as e: + # if core failes, send the user an error message, but keep going + self.log.error("Something went wrong internally: %s" % str(e)) + status = 'core_error' + reply = self._get_msg('internal_error', req['lc']) + + finally: + # keep stats + if req: + self.log.debug("Adding request to database... ") + self.core.add_request_to_db() + + def sendMessage(self, target_id, message): + # this is quick and ugly. primary splits (if needed) at newlines. + """ + try: + cur_message = '' + for line in message.split('\n'): + if len(cur_message + ('\n' if cur_message else '') + line)\ + > config.CHARACTER_LIMIT: + self._split_in_chunks_and_send(target_id, cur_message) + cur_message = '' + else: + cur_message += ('\n' if cur_message else '') + line + if cur_message: + self._split_in_chunks_and_send(target_id, cur_message) + except Exception as e: + # again, scrubbing 'target_id' should be an option, etc. + log.warning('Failed to send a direct message to %s. Exception:\n%s', + str(target_id), e) + return False + return True + """ + # with new twitter limit of direct messages, we can send messages + # without any trouble + self.api.send_direct_message( + user_id=target_id, + text=message + ) + + """ + def _split_in_chunks_and_send(self, target_id, message): + # assume any decent humane splitting has been done beforehand. + # we have to do with what we have here. + # exception handling at higher call stack. + + while message: + self.api.send_direct_message(user_id=target_id, + text=message[:config.CHARACTER_LIMIT]) + message = message[config.CHARACTER_LIMIT:] + """ + + """ + def followAllFollowers(self): + # Start following everyone who is following us. + + for follower in tweepy.Cursor(self.api.followers).items(): + follower.follow() + """ + + """ + def unfollowAllFollowers(self): + # Unfollow everyone who is following us. + + for follower in tweepy.Cursor(self.api.followers).items(): + follower.unfollow() + """ diff --git a/process_tweets.py b/process_tweets.py new file mode 100644 index 0000000..9e14f8b --- /dev/null +++ b/process_tweets.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import logging + +import gettor.twitter + +def main(): + """Quick way of running the thing. + + Note that default right now is 'no async', which means, function won't + return; on the other hand everything "will just work." + + If async is off, then we can: + >>> from quick_run import quick_run + >>> bot = quick_run() # authenticate, subscribe to streaming api, get handle + """ + + try: + bot = TwitterBot() + bot.authenticate() + # bot.api.update_status('hello world!') + bot.subscribeToStreams() + return bot + except gettor.twitter.ConfigError as e: + print "Configuration error: %s" % str(e) + except gettor.twitter.InternalError as e: + print "Core module not working: %s" % str(e) + except Exception as e: + # in case something unexpected happens + print "Unexpected error: %s" % str(e) + +if __name__ == '__main__': + main()

1 0

[gettor/master] Minor changes
by ilv＠torproject.org 03 Nov '15

03 Nov '15

commit 1e8645959172ce050e5890ecb5e0505ac72494d1 Author: ilv <ilv(a)users.noreply.github.com> Date: Thu Aug 27 15:19:40 2015 -0300 Minor changes --- upload/bundles2drive.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/upload/bundles2drive.py b/upload/bundles2drive.py index 9612382..1cf6a17 100644 --- a/upload/bundles2drive.py +++ b/upload/bundles2drive.py @@ -44,15 +44,15 @@ def valid_format(file, osys): """ if(osys == 'windows'): m = re.search( - 'torbrowser-install-\d\.\d\.\d_\w\w(-\w\w)?\.exe', + 'torbrowser-install-\d\.\d\.?\d?_\w\w(-\w\w)?\.exe', file) elif(osys == 'linux'): m = re.search( - 'tor-browser-linux\d\d-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz', + 'tor-browser-linux\d\d-\d\.\d\.?\d?_(\w\w)(-\w\w)?\.tar\.xz', file) elif(osys == 'osx'): m = re.search( - 'TorBrowser-\d\.\d\.\d-osx\d\d_(\w\w)(-\w\w)?\.dmg', + 'TorBrowser-\d\.\d\.?\d?-osx\d\d_(\w\w)(-\w\w)?\.dmg', file) if m: return True @@ -73,7 +73,7 @@ def get_bundle_info(file, osys): """ if(osys == 'windows'): m = re.search( - 'torbrowser-install-\d\.\d\.\d_(\w\w)(-\w\w)?\.exe', + 'torbrowser-install-\d\.\d\.?\d?_(\w\w)(-\w\w)?\.exe', file) if m: lc = m.group(1) @@ -82,7 +82,7 @@ def get_bundle_info(file, osys): raise ValueError("Invalid bundle format %s" % file) elif(osys == 'linux'): m = re.search( - 'tor-browser-linux(\d\d)-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz', + 'tor-browser-linux(\d\d)-\d\.\d\.?\d?_(\w\w)(-\w\w)?\.tar\.xz', file) if m: arch = m.group(1) @@ -92,7 +92,7 @@ def get_bundle_info(file, osys): raise ValueError("Invalid bundle format %s" % file) elif(osys == 'osx'): m = re.search( - 'TorBrowser-\d\.\d\.\d-osx(\d\d)_(\w\w)(-\w\w)?\.dmg', + 'TorBrowser-\d\.\d\.?\d?-osx(\d\d)_(\w\w)(-\w\w)?\.dmg', file) if m: os = 'osx' @@ -351,10 +351,10 @@ if __name__ == '__main__': elif p3.match(file): osys, arch, lc = get_bundle_info(file, 'osx') - link = "Package (%s-bit): %s\nASC signature (%s-bit): %s\n"\ - "Package SHA256 checksum (%s-bit): %s\n" %\ - (arch, link_file, arch, link_asc, - arch, sha_file) + link = "Tor Browser %s-bit:\n%s$Tor Browser's signature %s-bit:"\ + "\n%s$SHA256 checksum of Tor Browser %s-bit (advanced):"\ + "\n%s$" %\ + (arch, link_file, arch, link_asc, arch, sha_file) # note that you should only upload bundles for supported locales core.add_link('Drive', osys, lc, link)

1 0

[gettor/master] Split words on whitespace
by ilv＠torproject.org 03 Nov '15

03 Nov '15

commit 79359bd87d2450c7a15eac2c053a8d9b06fa4563 Author: Arlo Breault <arlolra(a)gmail.com> Date: Fri Oct 2 12:01:35 2015 +0200 Split words on whitespace --- gettor/smtp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gettor/smtp.py b/gettor/smtp.py index 34477ef..cfabfda 100644 --- a/gettor/smtp.py +++ b/gettor/smtp.py @@ -271,8 +271,8 @@ class SMTP(object): # search for OS or mirrors request # if nothing is found, help by default found_request = False - lines = msg.split(' ') - for word in lines: + words = re.split('\s+', test.strip()) + for word in words: if not found_request: # OS first for os in supported_os:

1 0

[gettor/master] Fixed format of message constructed by core
by ilv＠torproject.org 03 Nov '15

03 Nov '15

commit ce352ebdbe15d02f838fccd50a26723238a164aa Author: ilv <ilv(a)users.noreply.github.com> Date: Sat Oct 31 15:05:05 2015 -0300 Fixed format of message constructed by core --- gettor/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gettor/core.py b/gettor/core.py index 49fbbb2..6336e04 100644 --- a/gettor/core.py +++ b/gettor/core.py @@ -290,7 +290,7 @@ class Core(object): all_links.append(msg) - msg = "\n\n\nTor Browser's signature %s-bit (in the same order):" %\ + msg = "\n\n\nTor Browser's signature %s-bit:" %\ arch for link in links32: msg = "%s\n%s" % (msg, links32[link]) @@ -300,7 +300,7 @@ class Core(object): if osys == 'linux': msg = "\n\n\nTor Browser's signature 64-bit:" for link in links64: - msg = "%s\n%s" % (msg, links64[link]) + msg = "%s%s" % (msg, links64[link]) all_links.append(msg)

1 0

[gettor/master] Added option to get links only, and fixed format according to current template for links file.
by ilv＠torproject.org 03 Nov '15

03 Nov '15

commit 00a4e6a96a35a99e60f7ef7540235cdfc9d29099 Author: ilv <ilv(a)users.noreply.github.com> Date: Sat Oct 31 15:10:55 2015 -0300 Added option to get links only, and fixed format according to current template for links file. --- upload/bundles2github.py | 60 +++++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/upload/bundles2github.py b/upload/bundles2github.py index 41cd8d7..ee24edb 100644 --- a/upload/bundles2github.py +++ b/upload/bundles2github.py @@ -18,6 +18,7 @@ import sys import time import shutil import hashlib +import argparse from libsaas.services import github import gnupg @@ -80,22 +81,39 @@ def get_bundle_info(file, osys): return 'osx', arch, lc if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Utility to upload Tor Browser to Github.' + ) + + # with this we only get the links of files already uploaded + # useful when somethings fail after uploading + parser.add_argument( + '-l', '--links', default=None, + help='Create links file with files already uploaded.' + ) + + parser.add_argument( + '-v', '--version', default=None, + help='Version of Tor Browser.' + ) + + args = parser.parse_args() # this script should be called after fetching the latest Tor Browser, # and specifying the latest version - version = sys.argv[1] + version = args.version # the token allow us to run this script without GitHub user/pass gh_token = '' # path to the fingerprint that signed the packages - tb_key = os.path.abspath('tbb-key.asc') + tb_key = os.path.abspath('tbb-key-torbrowserteam.asc') # path to the latest version of Tor Browser tb_path = os.path.abspath('upload/latest') # path to the repository where we upload Tor Browser - repo_path = os.path.abspath('dl') + repo_path = os.path.abspath('gettorbrowser') # wait time between pushing the files to GH and asking for its links wait_time = 10 @@ -114,8 +132,8 @@ if __name__ == '__main__': # want to push the files using an SSH key (to avoid using user/pass) remote = 'origin' branch = 'master' - user = 'gettorbrowser' - repo = 'dl' + user = 'TheTorProject' + repo = 'gettorbrowser' raw_content = 'https://raw.githubusercontent.com/%s/%s/%s/' %\ (user, repo, branch) @@ -125,20 +143,22 @@ if __name__ == '__main__': # 3) make a commit for the new version # 4) push the changes - shutil.copytree( - tb_path, - os.path.abspath('%s/%s' % (repo_path, version)) - ) + if not args.links: + shutil.copytree( + tb_path, + os.path.abspath('%s/%s' % (repo_path, version)) + ) - git = sh.git.bake(_cwd=repo_path) - git.add('%s' % version) - git.commit(m=version) - git.push() + git = sh.git.bake(_cwd=repo_path) + git.add('%s' % version) + git.commit(m=version) + git.push() - # it takes a while to process the recently pushed files - print "Wait a few seconds before asking for the links to Github..." - time.sleep(wait_time) + # it takes a while to process the recently pushed files + print "Wait a few seconds before asking for the links to Github..." + time.sleep(wait_time) + print "Trying to get the links" gh = github.GitHub(gh_token, None) repocontent = gh.repo( user, @@ -176,7 +196,7 @@ if __name__ == '__main__': sha256 = get_file_sha256( os.path.abspath( - 'dl/%s/%s' % (version, filename) + '%s/%s/%s' % (repo, version, filename) ) ) @@ -185,11 +205,7 @@ if __name__ == '__main__': # asc signature just by adding '.asc' link_asc = file[u'download_url'].replace(filename, filename_asc) - link = "Tor Browser %s-bit:\n%s$Tor Browser's signature %s-bit:"\ - "\n%s$SHA256 checksum of Tor Browser %s-bit (advanced):"\ - "\n%s$" %\ - (arch, file[u'download_url'], arch, link_asc, - arch, sha256) + link = "%s$%s$%s$" % (file[u'download_url'], link_asc, sha256) print "Adding %s" % file[u'download_url'] core.add_link('GitHub', osys, lc, link)

1 0

[gettor/master] Fixed format for text plain messages in smtp
by ilv＠torproject.org 03 Nov '15

03 Nov '15

commit 00f122e3daa47aa890c7a5a8aebab086f4569a0b Author: ilv <ilv(a)users.noreply.github.com> Date: Sat Oct 31 15:04:20 2015 -0300 Fixed format for text plain messages in smtp --- gettor/smtp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gettor/smtp.py b/gettor/smtp.py index cfabfda..5476189 100644 --- a/gettor/smtp.py +++ b/gettor/smtp.py @@ -308,7 +308,7 @@ class SMTP(object): email_obj['From'] = from_addr email_obj['To'] = to_addr - msg_attach = MIMEText(msg, 'text') + msg_attach = MIMEText(msg, 'plain') email_obj.attach(msg_attach) return email_obj

1 0

[gettor/master] Updates to Twitter module, now it works. Still pending logging.
by ilv＠torproject.org 03 Nov '15

03 Nov '15

commit 9f2a5194c126f431579099185d5e1a12cbcec878 Author: ilv <ilv(a)users.noreply.github.com> Date: Fri Oct 30 22:11:52 2015 -0300 Updates to Twitter module, now it works. Still pending logging. --- gettor/twitter.py | 507 ++++++++++++----------------------------------------- process_tweets.py | 17 +- 2 files changed, 110 insertions(+), 414 deletions(-) diff --git a/gettor/twitter.py b/gettor/twitter.py index 05320ef..a197ac1 100644 --- a/gettor/twitter.py +++ b/gettor/twitter.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # -# This file is part of GetTor, a Tor Browser distribution system. +# This file is part of GetTor. # -# :authors: Israel Leiva <ilv(a)riseup.net> +# :authors: Israel Leiva <ilv(a)torproject.org> # Based on BridgeDB Twitter distributor (PoC) by wfn # - https://github.com/wfn/twidibot # @@ -11,22 +11,18 @@ # # :license: This is Free Software. See LICENSE for license information. -import sys -import json -import time -import signal +import os +import re import tweepy +import logging import gettext - -from tweepy.models import Status +import ConfigParser import core import utils import blacklist -"""Twitter module for processing requests. Forked from BridgeDB Twitter -distributor by wfn (https://github.com/wfn/twidibot)""" - +"""Twitter channel for distributing links to download Tor Browser.""" class ConfigError(Exception): pass @@ -36,172 +32,24 @@ class InternalError(Exception): pass -class TwitterBotStreamListener(tweepy.StreamListener): - """Listener for twitter's Streaming API.""" - - def __init__(self, bot, api=None): +class GetTorStreamListener(tweepy.StreamListener): + """ Basic listener for Twitter's streaming API.""" + def __init__(self, bot): self.bot = bot - self.processing_data = False - - super(TwitterBotStreamListener, self).__init__(api) - - def on_data(self, raw_data): - """Called when raw data is received from connection. - - This is where all the data comes first. Normally we could use - (inherit) the on_data() in tweepy.StreamListener, but it unnecessarily - and naively reports unknown event types as errors (to simple log); - also, we might want to tweak it further later on. - - But for now, this is basically taken from tweepy's on_data(). - - Return False to stop stream and close connection. - - """ - - self.processing_data = True - - data = json.loads(raw_data) - - if 'in_reply_to_status_id' in data: - status = Status.parse(self.api, data) - if self.on_status(status) is False: - return False - elif 'delete' in data: - delete = data['delete']['status'] - if self.on_delete(delete['id'], delete['user_id']) is False: - return False - elif 'event' in data: - status = Status.parse(self.api, data) - if self.on_event(status) is False: - return False - elif 'direct_message' in data: - status = Status.parse(self.api, data) - if self.on_direct_message(status) is False: - return False - elif 'limit' in data: - if self.on_limit(data['limit']['track']) is False: - return False - elif 'disconnect' in data: - if self.on_disconnect(data['disconnect']) is False: - return False - else: - # we really are ok to receive unknown stream/event types. - # log to debug? - log.debug('TwitterBotStreamListener::on_data(): got event/stream' - ' data of unknown type. Raw data follows:\n%s', data) - - self.processing_data = False - - def on_status(self, status): - """Called when a new status arrives""" - - #log.debug('Got status: %s', status) - return - - def on_event(self, status): - """Called when a new event arrives""" - - #log.debug('Got event: %s', status) - - # XXX make sure tweepy's given 'status.event' unicode string can - # always be safely converted to ascii - - # now it seems one can reply to dm without following the account - # if str(status.event) == 'follow': - # self.bot.handleFollowEvent(status) - - return + super(GetTorStreamListener, self).__init__(self.bot.api) def on_direct_message(self, status): - """Called when a new direct message arrives or is sent from us - - TODO: make a pull request for tweepy or something, because they - say it's only when a direct message is *received* (implying, 'by us') - - """ - - # doing twitter user comparisons using id_str makes sense here - it's - # safe and id_str's are guaranteed to be unique (re: latter, just like - # id's.) maybe consider deciding how comparisons should be made for sure, - # and then extend tweepy.models.User to include __eq__? + """ Right now we only care about direct messages. """ if status.direct_message['sender']['id_str'] != self.bot.bot_info.id_str: - self.bot.handleDirectMessage(status) - else: - # log.debug('Caught a direct message sent *from* us') - pass - - return - - def on_connect(self): - """Called once connected to streaming server. - - This will be invoked once a successful response - is received from the server. Allows the listener - to perform some work prior to entering the read loop. - - """ - pass - - def on_exception(self, exception): - """Called when an unhandled exception occurs.""" - return - - def on_delete(self, status_id, user_id): - """Called when a delete notice arrives for a status""" - return - - def on_limit(self, track): - """Called when a limitation notice arrvies""" - return - - def on_error(self, status_code): - """Called when a non-200 status code is returned""" - return False - - def on_timeout(self): - """Called when stream connection times out""" - return - - def on_disconnect(self, notice): - """Called when twitter sends a disconnect notice + self.bot.parse_request(status.direct_message) - Disconnect codes are listed here: - https://dev.twitter.com/docs/streaming-apis/messages - #Disconnect_messages_disconnect - - """ - return - class TwitterBot(object): - """Main interface between the stateful listener and Twitter APIs.""" - - # TODO: think about secure ways of storing twitter access config. - # For one, app itself should ideally not be able to have write access - # to it. For another, ideally it would request details from some other - # component, authenticate, and not be able to re-authenticate to twitter - - """ - default_access_config = { - 'api_key': config.API_KEY, - 'api_secret': config.API_SECRET, - 'access_token': config.ACCESS_TOKEN, - 'token_secret': config.TOKEN_SECRET - }""" - - def __init__(self, **kw): - """Constructor that accepts custom access config as named arguments - - Easy to test things from interactive shell this way. - Probably won't be needed in production code. + """ Receive and reply requests via Twitter. """ + def __init__(self, cfg=None): + """ Create new object by reading a configuration file. - """ - - """ - self.access_config = dict() - for key, default in self.default_access_config.iteritems(): - self.access_config[key] = kw.get(key, default) + :param: cfg (string) the path of the configuration file. """ default_cfg = 'twitter.cfg' @@ -222,11 +70,7 @@ class TwitterBot(object): self.access_token = config.get('access_config', 'access_token') self.token_secret = config.get('access_config', 'token_secret') - self.async_streaming = config.get('api', 'async_streaming') - self.char_limit = config.get('api', 'char_limit') - self.mirrors = config.get('general', 'mirrors') - self.max_words = config.get('general', 'max_words') self.i18ndir = config.get('i18n', 'dir') logdir = config.get('log', 'dir') @@ -235,8 +79,8 @@ class TwitterBot(object): blacklist_cfg = config.get('blacklist', 'cfg') self.bl = blacklist.Blacklist(blacklist_cfg) - self.bl_max_req = config.get('blacklist', 'max_requests') - self.bl_max_req = int(self.bl_max_req) + self.bl_max_request = config.get('blacklist', 'max_requests') + self.bl_max_request = int(self.bl_max_request) self.bl_wait_time = config.get('blacklist', 'wait_time') self.bl_wait_time = int(self.bl_wait_time) @@ -251,13 +95,14 @@ class TwitterBot(object): raise InternalError("Core error: %s" % str(e)) # logging + """ log = logging.getLogger(__name__) logging_format = utils.get_logging_format() date_format = utils.get_date_format() formatter = logging.Formatter(logging_format, date_format) - log.info('Redirecting SMTP logging to %s' % logfile) + log.info('Redirecting Twitter logging to %s' % logfile) logfileh = logging.FileHandler(logfile, mode='a+') logfileh.setFormatter(formatter) logfileh.setLevel(logging.getLevelName(loglevel)) @@ -265,31 +110,29 @@ class TwitterBot(object): # stop logging on stdout from now on log.propagate = False - self.log = log - - self.setSignalHandlers() - self.msg = Messages() - - - def _is_blacklisted(self, account): + #self.log = log""" + + def _is_blacklisted(self, username): """Check if a user is blacklisted. - :param: addr (string) the hashed address of the user. + :param: addr (string) the hashed username. - :return: true is the address is blacklisted, false otherwise. + :return: true is the username is blacklisted, false otherwise. """ - anon_acc = utils.get_sha256(account) + hashed_username = utils.get_sha256(username) try: self.bl.is_blacklisted( - anon_acc, 'Twitter', self.bl_max_req, self.bl_wait_time + hashed_username, + 'Twitter', + self.bl_max_request, + self.bl_wait_time ) return False except blacklist.BlacklistError as e: return True - def _get_msg(self, msgid, lc): """Get message identified by msgid in a specific locale. @@ -299,7 +142,7 @@ class TwitterBot(object): Return: a string containing the given message. """ - self.log.debug("Getting message '%s' for locale %s" % (msgid, lc)) + #self.log.debug("Getting message '%s' for locale %s" % (msgid, lc)) try: t = gettext.translation(lc, self.i18ndir, languages=[lc]) _ = t.ugettext @@ -308,11 +151,17 @@ class TwitterBot(object): return msgstr except IOError as e: raise ConfigError("%s" % str(e)) - - - def _parse_request(self, message): - """ """ - self.log.debug("Parsing text.") + + def parse_text(self, msg): + """ Parse the text part of a message. + + Split the message in words and look for patterns for locale, + operating system and mirrors requests. + + :param: msg (string) the message received. + + :return: request (list) 3-tuple with locale, os and type of request. + """ # core knows what OS are supported supported_os = self.core.get_supported_os() @@ -329,7 +178,8 @@ class TwitterBot(object): found_mirrors = False # analyze every word - for word in message.split(' '): + words = re.split('\s+', msg.strip()) + for word in words: # look for lc and os if not found_lc: for lc in supported_lc: @@ -352,154 +202,49 @@ class TwitterBot(object): return req + def parse_request(self, dm): + """ Process the request received. - def setSignalHandlers(self): - """Set up relevant SIG* handlers for the bot. + Check if the user is not blacklisted and then check the body of + the message to find out what is asking. - Note: if we want to handle some specific signal and not exit after - catching it, we may need to store the original CPython handler - (signified by signal.SIG_DFL), and restore it after handling the - signal in question. For now, we'll only care about signals after - which the program does exit. + :param: dm (status.direct_message) the direct message object received + via Twitter API. """ - # for now, we'll only handle SIGTERM. it might make sense to handle - # SIGINT as well, though. - - signal.signal(signal.SIGTERM, self.handleSIGTERM) - self.log.debug("SIGTERM handler is set.") - - def handleSIGTERM(self, sig_number, stack_frame): - """Callback function called upon SIGTERM""" - - self.log.info("TwitterBot::handleSIGTERM(): caught SIGTERM signal.") - - self.log.info("Stopping bot listener.") - self.listener.running = False - while self.listener.processing_data: - self.log.info( - "Waiting for TwitterBotStreamListener to finish processing" - " a data request/package" - ) - time.sleep(0.5) - - self.log.info("Closing down storage controller.") - self.storage_controller.closeAll() - - self.log.info("Exiting program.") - sys.exit(0) - - def authenticate(self, auth=None): - """Authenticate to Twitter API, get API handle, and remember it.""" - - if auth: - self.auth = auth - else: - self.auth = tweepy.OAuthHandler( - self.api_key, - self.api_secret - ) - - self.auth.set_access_token( - self.access_token, - self.token_secret - ) - - try: - self.api = tweepy.API(self.auth) - except Exception as e: - self.log.fatal('Exception while authenticating to Twitter and ' - 'getting API handle: %s', e) - self.api = None - finally: - # del self.auth # ideally we'd be able to delete this, but - # presently - no; anything? - pass - - if self.api: - self.log.info('Authenticated and got the RESTful API handle') - self.bot_info = self.api.me() - #api.update_status('hello world!') - - def subscribeToStreams(self): - """Subscribe to relevant streams in the Streaming API.""" - - self.listener = TwitterBotStreamListener( - bot=self, - api=self.api - ) - - self.stream = tweepy.Stream(self.auth, self.listener) - - # user stream gives us direct messages and follow events - self.stream.userstream(async=self.async_streaming) - # stream.filter may be useful, but we don't need it for now - - # the following will not be executed if we're not going async - - # userstream() blocks, its event handler loop takes over: - self.log.info('Subscribed to relevant streams via Streaming API') - - def handleFollowEvent(self, event): - """ - user_id = event.source['id'] # 'id' is unique big int - - if user_id != self.bot_info.id: - user = self.api.get_user(id=user_id) - user.follow() - - if config.RESPOND_AFTER_FOLLOW: - # the following line *blocks* the thread that we care about. - # we should not do this, ever. as long as we're just testing - # with a few cat accounts, it's ok. - - # TODO: use sched.scheduler, or threading.Timer (or sth) - time.sleep(config.WAIT_TIME_AFTER_FOLLOW) - - # for now, english by default - self.sendMessage( - user_id, - get_msg('welcome', 'en') - ) - """ - # it seems that we don't need to be followed to send dm - pass - - - def handleDirectMessage(self, status): - """ Handle direct messages received (i.e. parse request). """ - sender_id = status.direct_message['sender_id'] - message = status.direct_message['text'].strip().lower() - - self.log.debug("Parsing request") + sender_id = dm['sender']['id_str'] + msg = dm['text'].strip().lower() + bogus_request = False + request = None + status = '' try: if self._is_blacklisted(str(sender_id)): - self.log.info("Request from blacklisted account!") + #self.log.info("Request from blacklisted account!") status = 'blacklisted' bogus_request = True - # first let's find out how many words are in the message - # request shouldn't be longer than 3 words, but just in case - words = message.split(' ') - if len(words) > self.max_words: - bogus_request = True - self.log.info("Message way too long") - status = 'error' - reply = self._get_msg('message_error', 'en') - if not bogus_request: - self.log.debug("Request seems legit, let's parse it") + #self.log.debug("Request seems legit, let's parse it") # let's try to guess what the user is asking - req = self._parse_text(str(msg)) - - if req['type'] == 'help': - self.log.debug("Type of request: help") - status = 'success' - reply = self._get_msg('help', 'en') + request = self.parse_text(str(msg)) + + # possible options: links, mirrors, help + if request['type'] == 'links': + #self.log.debug("Type of request: help") + links = self.core.get_links( + 'Twitter', + request['os'], + request['lc'] + ) + + reply = self._get_msg('links', 'en') + reply = reply % (request['os'], request['lc'], links) + - elif req['type'] == 'mirrors': - self.log.debug("Type of request: mirrors") + elif request['type'] == 'mirrors': + #self.log.debug("Type of request: mirrors") status = 'success' reply = self._get_msg('mirrors', 'en') try: @@ -510,86 +255,50 @@ class TwitterBot(object): except IOError as e: reply = self._get_msg('mirrors_unavailable', 'en') - elif req['type'] == 'links': - self.log.debug("Type of request: help") - links = self.core.get_links( - "Twitter", - req['os'], - req['lc'] - ) - reply = self._get_msg('links', 'en') - reply = reply % (req['os'], req['lc'], links) - self.sendMessage(sender_id, reply) - + else: + #self.log.debug("Type of request: help") status = 'success' - - # send whatever the reply is - self.sendMessage(sender_id, reply) + reply = self._get_msg('help', 'en') + + self.api.send_direct_message( + user_id=sender_id, + text=reply + ) except (core.ConfigError, core.InternalError) as e: # if core failes, send the user an error message, but keep going - self.log.error("Something went wrong internally: %s" % str(e)) + #self.log.error("Something went wrong internally: %s" % str(e)) status = 'core_error' - reply = self._get_msg('internal_error', req['lc']) + reply = self._get_msg('internal_error', 'en') finally: # keep stats - if req: - self.log.debug("Adding request to database... ") + if request: + #self.log.debug("Adding request to database... ") self.core.add_request_to_db() - def sendMessage(self, target_id, message): - # this is quick and ugly. primary splits (if needed) at newlines. - """ - try: - cur_message = '' - for line in message.split('\n'): - if len(cur_message + ('\n' if cur_message else '') + line)\ - > config.CHARACTER_LIMIT: - self._split_in_chunks_and_send(target_id, cur_message) - cur_message = '' - else: - cur_message += ('\n' if cur_message else '') + line - if cur_message: - self._split_in_chunks_and_send(target_id, cur_message) - except Exception as e: - # again, scrubbing 'target_id' should be an option, etc. - log.warning('Failed to send a direct message to %s. Exception:\n%s', - str(target_id), e) - return False - return True + def start(self): + """ Start the bot for handling requests. + + Start a new Twitter bot. """ - # with new twitter limit of direct messages, we can send messages - # without any trouble - self.api.send_direct_message( - user_id=target_id, - text=message + self.auth = tweepy.OAuthHandler( + self.api_key, + self.api_secret + ) + + self.auth.set_access_token( + self.access_token, + self.token_secret ) - """ - def _split_in_chunks_and_send(self, target_id, message): - # assume any decent humane splitting has been done beforehand. - # we have to do with what we have here. - # exception handling at higher call stack. - - while message: - self.api.send_direct_message(user_id=target_id, - text=message[:config.CHARACTER_LIMIT]) - message = message[config.CHARACTER_LIMIT:] - """ - - """ - def followAllFollowers(self): - # Start following everyone who is following us. - - for follower in tweepy.Cursor(self.api.followers).items(): - follower.follow() - """ - - """ - def unfollowAllFollowers(self): - # Unfollow everyone who is following us. - - for follower in tweepy.Cursor(self.api.followers).items(): - follower.unfollow() - """ + self.api = tweepy.API(self.auth) + self.bot_info = self.api.me() + + stream = tweepy.Stream( + auth = self.api.auth, + listener=GetTorStreamListener(self) + ) + + stream.userstream() + diff --git a/process_tweets.py b/process_tweets.py index 9e14f8b..484cb97 100644 --- a/process_tweets.py +++ b/process_tweets.py @@ -7,22 +7,9 @@ import logging import gettor.twitter def main(): - """Quick way of running the thing. - - Note that default right now is 'no async', which means, function won't - return; on the other hand everything "will just work." - - If async is off, then we can: - >>> from quick_run import quick_run - >>> bot = quick_run() # authenticate, subscribe to streaming api, get handle - """ - try: - bot = TwitterBot() - bot.authenticate() - # bot.api.update_status('hello world!') - bot.subscribeToStreams() - return bot + bot = gettor.twitter.TwitterBot() + bot.start() except gettor.twitter.ConfigError as e: print "Configuration error: %s" % str(e) except gettor.twitter.InternalError as e:

1 0

[gettor/master] Added option to get links only, and fixed format according to current template for links file.
by ilv＠torproject.org 03 Nov '15

03 Nov '15

commit 24b655c4177eccc4439ac198ff098b7883425067 Author: ilv <ilv(a)users.noreply.github.com> Date: Sat Oct 31 15:09:37 2015 -0300 Added option to get links only, and fixed format according to current template for links file. --- upload/bundles2drive.py | 62 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/upload/bundles2drive.py b/upload/bundles2drive.py index 1cf6a17..bac6fea 100644 --- a/upload/bundles2drive.py +++ b/upload/bundles2drive.py @@ -17,6 +17,7 @@ import os import gnupg import hashlib import logging +import argparse import ConfigParser import gettor.core @@ -242,11 +243,56 @@ def share_file(service, file_id): except errors.HttpError, error: print('Error occured while fetch public link for file: %s' % file_id) - print("Uploaded to %s" % file['webContentLink']) + print "Uploaded %s to %s" % (file['title'], file['webContentLink']) return file['webContentLink'] +def get_files_links(service, v): + """Print links of uploaded files. + + :param: service (object): Goolge Drive service object. + :param: v (string): Version of Tor Browser to look for. + + """ + + windows_re = 'torbrowser-install-%s_\w\w(-\w\w)?\.exe(\.asc)?' % v + linux_re = 'tor-browser-linux\d\d-%s_(\w\w)(-\w\w)?\.tar\.xz(\.asc)?' % v + osx_re = 'TorBrowser-%s-osx\d\d_(\w\w)(-\w\w)?\.dmg(\.asc)?' % v + + # dictionary to store file names and IDs + files_dict = dict() + + print "Trying to fetch links of uploaded files..." + links = service.files().list().execute() + items = links.get('items', []) + + if not items: + raise ValueError('No files found.') + + else: + for item in items: + if re.search(windows_re, item['title']): + files_dict[item['title']] = item['id'] + elif re.search(linux_re, item['title']): + files_dict[item['title']] = item['id'] + elif re.search(osx_re, item['title']): + files_dict[item['title']] = item['id'] + return files_dict + if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Utility to upload Tor Browser to Google Drive.' + ) + + # if no LC specified, download all + parser.add_argument( + '-l', '--links', default=None, + help='Create links file with files already uploaded and '\ + 'matching the specified version. ' + ) + + args = parser.parse_args() + config = ConfigParser.ConfigParser() config.read('drive.cfg') @@ -310,7 +356,14 @@ if __name__ == '__main__': readable = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4)) try: - uploaded_files = upload_files(drive_service, upload_dir) + # helpful when something fails but files are uploaded. + if args.links: + uploaded_files = get_files_links(drive_service, args.links) + + if not uploaded_files: + raise ValueError("There are no files for that version") + else: + uploaded_files = upload_files(drive_service, upload_dir) # use default config core = gettor.core.Core('/home/gettor/core.cfg') @@ -351,10 +404,7 @@ if __name__ == '__main__': elif p3.match(file): osys, arch, lc = get_bundle_info(file, 'osx') - link = "Tor Browser %s-bit:\n%s$Tor Browser's signature %s-bit:"\ - "\n%s$SHA256 checksum of Tor Browser %s-bit (advanced):"\ - "\n%s$" %\ - (arch, link_file, arch, link_asc, arch, sha_file) + link = "%s$%s$%s$" % (link_file, link_asc, sha_file) # note that you should only upload bundles for supported locales core.add_link('Drive', osys, lc, link)

1 0

← Newer
1
...
72
73
74
75
76
77
78
...
86
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits November 2015