December 2015 - tor-commits - lists.torproject.org

[gettor/master] Minor fixes
by ilv＠torproject.org 04 Dec '15

04 Dec '15

commit eb33e8f3aef58677e9b1cd2fd16c81643d7a0a7b Author: ilv <ilv(a)users.noreply.github.com> Date: Tue Nov 17 13:33:54 2015 -0300 Minor fixes --- gettor/smtp.py | 5 +++-- gettor/utils.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/gettor/smtp.py b/gettor/smtp.py index d6de2ed..ed6a4fc 100644 --- a/gettor/smtp.py +++ b/gettor/smtp.py @@ -504,7 +504,7 @@ class SMTP(object): elif req['type'] == 'links': self.log.debug("Trying to obtain the links...") - self.log.info('links; %s; %s' % (req['os '], req['lc'])) + self.log.info('links; %s; %s' % (req['os'], req['lc'])) try: links = self.core.get_links( @@ -525,4 +525,5 @@ class SMTP(object): self.log.debug("FAILED: %s" % str(e)) raise SendEmailError("Something's wrong with the SMTP " "server: %s" % str(e)) - self.log.debug("Mail sent!") + finally: + self.log.debug("Request processed") diff --git a/gettor/utils.py b/gettor/utils.py index 8224f25..3f69e08 100644 --- a/gettor/utils.py +++ b/gettor/utils.py @@ -16,7 +16,7 @@ import hashlib """Common utilities for GetTor modules.""" -LOGGING_FORMAT = "[%(levelname)s] DATE: %(asctime)s; %(message)s" +LOGGING_FORMAT = "[%(levelname)s] %(asctime)s; %(message)s" DATE_FORMAT = "%Y-%m-%d" # %H:%M:%S

1 0

[gettor/master] Adapted requests table to new stats
by ilv＠torproject.org 04 Dec '15

04 Dec '15

commit f6ed22811e74b615c048c8d8cd75c0eb01b9cd55 Author: ilv <ilv(a)users.noreply.github.com> Date: Fri Dec 4 15:21:37 2015 -0300 Adapted requests table to new stats --- scripts/create_db.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/create_db.py b/scripts/create_db.py index ec4ba32..d6e0b0f 100644 --- a/scripts/create_db.py +++ b/scripts/create_db.py @@ -38,15 +38,16 @@ def main(): with con: cur = con.cursor() # table for handling users (i.e. blacklist) - cur.execute("CREATE TABLE users(id TEXT, service TEXT, times INT," - " blocked INT, last_request TEXT)") - # table for stats - # cur.execute("CREATE TABLE requests(service TEXT, type TEXT," - # " os TEXT, lc TEXT, pt INT, year INT, month INT," - # " day INT, status TEXT, logfile TEXT)") + cur.execute( + "CREATE TABLE users(id TEXT, service TEXT, times INT," + "blocked INT, last_request TEXT)" + ) + + cur.execute( + "CREATE TABLE requests(date TEXT, request TEXT, os TEXT," + " locale TEXT, channel TEXT, PRIMARY KEY (date, channel))" + ) - # for now we will use a counter - cur.execute("CREATE TABLE requests(id INT, counter INT)") print "Database %s created" % os.path.abspath(args.create) elif args.delete: os.remove(os.path.abspath(args.delete))

1 0

[gettor/master] Merge branch 'develop'
by ilv＠torproject.org 04 Dec '15

04 Dec '15

commit 977bc9ef248ac581f2f5f34ed59a7796fe9e2dbc Merge: b1a11bd f6ed228 Author: ilv <ilv(a)users.noreply.github.com> Date: Fri Dec 4 15:23:14 2015 -0300 Merge branch 'develop' gettor/core.py | 11 ----------- gettor/smtp.py | 38 +++++++------------------------------- gettor/twitter.py | 48 +++++++++++++++++------------------------------- gettor/utils.py | 15 +++++++++------ gettor/xmpp.py | 25 +++++-------------------- log/smtp.log | 6 ++++++ scripts/create_db.py | 17 +++++++++-------- 7 files changed, 53 insertions(+), 107 deletions(-)

1 0

[gettor/master] Updated sample log file with blacklist and invalid requests examples
by ilv＠torproject.org 04 Dec '15

04 Dec '15

commit 4bea6bb708e6d0a9f6e17b173d9c4411c62e89e9 Author: ilv <ilv(a)users.noreply.github.com> Date: Mon Nov 16 15:23:00 2015 -0300 Updated sample log file with blacklist and invalid requests examples --- log/smtp.log | 3 +++ 1 file changed, 3 insertions(+) diff --git a/log/smtp.log b/log/smtp.log index 3950925..1e436c1 100644 --- a/log/smtp.log +++ b/log/smtp.log @@ -1,3 +1,6 @@ [INFO] DATE: 2015-11-16; REQUEST: help; OS: none; LC: fa [INFO] DATE: 2015-11-16; REQUEST: links; OS: windows; LC: zh +[INFO] DATE: 2015-11-16; REQUEST: blacklist; OS: none; LC: none [INFO] DATE: 2015-11-16; REQUEST: mirrors; OS: none; LC: tr +[INFO] DATE: 2015-11-16; REQUEST: invalid; OS: none; LC: none +

1 0

[gettor/master] Deleted database interaction from channels and core. Added new logging strings for stats purposes (See #17588).
by ilv＠torproject.org 04 Dec '15

04 Dec '15

commit 553519a8d4fd0f1514cca164def386a96e393328 Author: ilv <ilv(a)users.noreply.github.com> Date: Mon Nov 16 15:19:44 2015 -0300 Deleted database interaction from channels and core. Added new logging strings for stats purposes (See #17588). --- gettor/core.py | 11 ----------- gettor/smtp.py | 40 +++++++++------------------------------- gettor/twitter.py | 51 ++++++++++++++++++++------------------------------- gettor/utils.py | 15 +++++++++------ gettor/xmpp.py | 28 ++++++++-------------------- log/smtp.log | 3 +++ 6 files changed, 49 insertions(+), 99 deletions(-) diff --git a/gettor/core.py b/gettor/core.py index 6336e04..3d946c9 100644 --- a/gettor/core.py +++ b/gettor/core.py @@ -469,14 +469,3 @@ class Core(object): else: self.log.debug("FAILED (links file doesn't seem legit)") raise LinkFileError("No links file for %s" % provider) - - def add_request_to_db(self): - """Add request to database.""" - self.log.debug("Trying to add request to database") - try: - self.db.connect() - self.db.add_request() - self.log.debug("Request added!") - except db.DBError as e: - self.log.debug("FAILED %s" % str(e)) - raise InternalError("Couldn't add request to database %s" % str(e)) diff --git a/gettor/smtp.py b/gettor/smtp.py index 3697786..9203039 100644 --- a/gettor/smtp.py +++ b/gettor/smtp.py @@ -461,24 +461,15 @@ class SMTP(object): self.log.debug("Normalizing address...") norm_from_addr = self._get_normalized_address(from_addr) except AddressError as e: - status = 'malformed' bogus_request = True - self.log.debug("Address is malformed!") - # it might be interesting to know what triggered this - # we are not logging this for now - # logfile = self._log_email('malformed', content) + self.log.info('REQUEST: invalid; OS: none; LC: none') if norm_from_addr: - self.log.debug("Anonymizing address...") anon_addr = utils.get_sha256(norm_from_addr) if self._is_blacklisted(anon_addr): - status = 'blacklisted' bogus_request = True - self.log.debug("Address is blacklisted!") - # it might be interesting to know extra info - # we are not logging this for now - # logfile = self._log_email(anon_addr, content) + self.log.info('REQUEST: blacklist; OS: none; LC: none') if not bogus_request: # try to figure out what the user is asking @@ -487,45 +478,43 @@ class SMTP(object): # our address should have the locale requested our_addr = "gettor+%s@%s" % (req['lc'], self.our_domain) - self.log.debug("Replying from %s" % our_addr) # possible options: help, links, mirrors if req['type'] == 'help': - self.log.debug("Type of request: help") self.log.debug("Trying to send help...") + self.log.info('REQUEST: help; OS: none; LC: %s' % + req['lc']) # make sure we can send emails try: self._send_help('en', our_addr, norm_from_addr) - status = 'success' except SendEmailError as e: - status = 'internal_error' self.log.debug("FAILED: %s" % str(e)) raise InternalError("Something's wrong with the SMTP " "server: %s" % str(e)) elif req['type'] == 'mirrors': - self.log.debug("Type of request: mirrors") self.log.debug("Trying to send the mirrors...") + self.log.info('REQUEST: mirrors; OS: none; LC: %s' % + req['lc']) # make sure we can send emails try: self._send_mirrors('en', our_addr, norm_from_addr) - status = 'success' except SendEmailError as e: - status = 'internal_error' self.log.debug("FAILED: %s" % str(e)) raise SendEmailError("Something's wrong with the SMTP " "server: %s" % str(e)) elif req['type'] == 'links': - self.log.debug("Type of request: links") self.log.debug("Trying to obtain the links...") + self.log.info('REQUEST: links; OS: %s; LC: %s' % + req['os '], req['lc']) + try: links = self.core.get_links( 'SMTP', req['os'], req['lc'] ) # if core fails, we fail too except (core.InternalError, core.ConfigError) as e: - status = 'core_error' self.log.debug("FAILED: %s" % str(e)) # something went wrong with the core raise InternalError("Error obtaining the links") @@ -535,19 +524,8 @@ class SMTP(object): try: self._send_links(links, req['lc'], req['os'], our_addr, norm_from_addr) - status = 'success' except SendEmailError as e: - status = 'internal_error' self.log.debug("FAILED: %s" % str(e)) raise SendEmailError("Something's wrong with the SMTP " "server: %s" % str(e)) self.log.debug("Mail sent!") - finally: - # keep stats - if req: - self.log.debug("Adding request to database... ") - try: - self.core.add_request_to_db() - self.log.debug("Request added") - except InternalError as e: - self.log.debug("FAILED: %s" % str(e)) diff --git a/gettor/twitter.py b/gettor/twitter.py index 13666a3..85cad47 100644 --- a/gettor/twitter.py +++ b/gettor/twitter.py @@ -24,6 +24,7 @@ import blacklist """Twitter channel for distributing links to download Tor Browser.""" + class ConfigError(Exception): pass @@ -51,7 +52,7 @@ class TwitterBot(object): :param: cfg (string) the path of the configuration file. """ - + default_cfg = 'twitter.cfg' config = ConfigParser.ConfigParser() @@ -107,10 +108,8 @@ class TwitterBot(object): logfileh.setLevel(logging.getLevelName(loglevel)) log.addHandler(logfileh) - # stop logging on stdout from now on - #log.propagate = False self.log = log - + def _is_blacklisted(self, username): """Check if a user is blacklisted. @@ -141,7 +140,6 @@ class TwitterBot(object): Return: a string containing the given message. """ - #self.log.debug("Getting message '%s' for locale %s" % (msgid, lc)) try: t = gettext.translation(lc, self.i18ndir, languages=[lc]) _ = t.ugettext @@ -161,7 +159,7 @@ class TwitterBot(object): :return: request (list) 3-tuple with locale, os and type of request. """ - + # core knows what OS are supported supported_os = self.core.get_supported_os() supported_lc = self.core.get_supported_lc() @@ -198,7 +196,7 @@ class TwitterBot(object): req['type'] = 'mirrors' if (found_lc and found_os) or (found_lc and found_mirrors): break - + return req def parse_request(self, dm): @@ -220,31 +218,30 @@ class TwitterBot(object): try: if self._is_blacklisted(str(sender_id)): - self.log.info("Request from blacklisted account!") - status = 'blacklisted' + self.log.info('REQUEST: blacklist; OS: none; LC: none') bogus_request = True if not bogus_request: self.log.debug("Request seems legit, let's parse it") # let's try to guess what the user is asking request = self.parse_text(str(msg)) - + # possible options: links, mirrors, help if request['type'] == 'links': - self.log.debug("Type of request: links") + self.log.info('REQUEST: links; OS: %s; LC: %s' % + (req['os'], req['lc'])) links = self.core.get_links( 'Twitter', request['os'], request['lc'] ) - + reply = self._get_msg('links', 'en') reply = reply % (request['os'], request['lc'], links) - elif request['type'] == 'mirrors': - self.log.debug("Type of request: mirrors") - status = 'success' + self.log.info('REQUEST: mirrors; OS: none; LC: %s' % + req['lc']) reply = self._get_msg('mirrors', 'en') try: with open(self.mirrors, "r") as list_mirrors: @@ -255,10 +252,10 @@ class TwitterBot(object): reply = self._get_msg('mirrors_unavailable', 'en') else: - self.log.debug("Type of request: help") - status = 'success' + self.log.info('REQUEST: help; OS: none; LC: %s' % + req['lc']) reply = self._get_msg('help', 'en') - + self.api.send_direct_message( user_id=sender_id, text=reply @@ -267,25 +264,18 @@ class TwitterBot(object): except (core.ConfigError, core.InternalError) as e: # if core failes, send the user an error message, but keep going self.log.error("Something went wrong internally: %s" % str(e)) - status = 'core_error' reply = self._get_msg('internal_error', 'en') - finally: - # keep stats - if request: - self.log.debug("Adding request to database... ") - self.core.add_request_to_db() - def start(self): """ Start the bot for handling requests. Start a new Twitter bot. """ self.auth = tweepy.OAuthHandler( - self.api_key, + self.api_key, self.api_secret ) - + self.auth.set_access_token( self.access_token, self.token_secret @@ -293,11 +283,10 @@ class TwitterBot(object): self.api = tweepy.API(self.auth) self.bot_info = self.api.me() - + stream = tweepy.Stream( - auth = self.api.auth, + auth=self.api.auth, listener=GetTorStreamListener(self) ) - - stream.userstream() + stream.userstream() diff --git a/gettor/utils.py b/gettor/utils.py index b4ab9c6..8224f25 100644 --- a/gettor/utils.py +++ b/gettor/utils.py @@ -16,25 +16,28 @@ import hashlib """Common utilities for GetTor modules.""" -LOGGING_FORMAT = "[%(levelname)s] %(asctime)s - %(message)s" -DATE_FORMAT = "%Y-%m-%d" # %H:%M:%S +LOGGING_FORMAT = "[%(levelname)s] DATE: %(asctime)s; %(message)s" +DATE_FORMAT = "%Y-%m-%d" # %H:%M:%S + def get_logging_format(): """Get the logging format. - + :return: (string) the logging format. - + """ return LOGGING_FORMAT + def get_date_format(): """Get the date format for logging. - + :return: (string) the date format for logging. - + """ return DATE_FORMAT + def get_sha256(string): """Get sha256 of a string. diff --git a/gettor/xmpp.py b/gettor/xmpp.py index a605f50..072d988 100644 --- a/gettor/xmpp.py +++ b/gettor/xmpp.py @@ -277,8 +277,7 @@ class XMPP(object): self.log.debug("Parsing request") try: if self._is_blacklisted(str(account)): - self.log.info("Request from blacklisted account!") - status = 'blacklisted' + self.log.info('REQUEST: blacklist; OS: none; LC: none') bogus_request = True # first let's find out how many words are in the message @@ -287,7 +286,7 @@ class XMPP(object): if len(words) > self.max_words: bogus_request = True self.log.info("Message way too long") - status = 'error' + self.log.info('REQUEST: invalid; OS: none; LC: none') reply = self._get_msg('message_error', 'en') if not bogus_request: @@ -296,13 +295,13 @@ class XMPP(object): req = self._parse_text(str(msg)) if req['type'] == 'help': - self.log.debug("Type of request: help") - status = 'success' + self.log.info('REQUEST: help; OS: none; LC: %s' % + req['lc']) reply = self._get_msg('help', 'en') elif req['type'] == 'mirrors': - self.log.debug("Type of request: mirrors") - status = 'success' + self.log.info('REQUEST: mirrors; OS: none; LC: %s' % + req['lc']) reply = self._get_msg('mirrors', 'en') try: with open(self.mirrors, "r") as list_mirrors: @@ -312,7 +311,8 @@ class XMPP(object): reply = self._get_msg('mirrors_unavailable', 'en') elif req['type'] == 'links': - self.log.debug("Type of request: help") + self.log.info('REQUEST: links; OS: %s; LC: %s' % + (req['os'], req['lc'])) links = self.core.get_links( "XMPP", req['os'], @@ -321,22 +321,10 @@ class XMPP(object): reply = self._get_msg('links', 'en') reply = reply % (req['os'], req['lc'], links) - status = 'success' - except (core.ConfigError, core.InternalError) as e: # if core failes, send the user an error message, but keep going self.log.error("Something went wrong internally: %s" % str(e)) - status = 'core_error' reply = self._get_msg('internal_error', req['lc']) finally: - # keep stats - if req: - self.log.debug("Adding request to database... ") - self.core.add_request_to_db() - - if reply: - self.log.debug("Everything seems OK. Sending back the reply") - else: - self.log.debug("Nothing to reply!") return reply diff --git a/log/smtp.log b/log/smtp.log index e69de29..3950925 100644 --- a/log/smtp.log +++ b/log/smtp.log @@ -0,0 +1,3 @@ +[INFO] DATE: 2015-11-16; REQUEST: help; OS: none; LC: fa +[INFO] DATE: 2015-11-16; REQUEST: links; OS: windows; LC: zh +[INFO] DATE: 2015-11-16; REQUEST: mirrors; OS: none; LC: tr

1 0

[gettor/master] Simple logging string for stats (Deleted redundant info)
by ilv＠torproject.org 04 Dec '15

04 Dec '15

commit 9c10511a8f0ae3e88a7ae9fa3785108995fe66e1 Author: ilv <ilv(a)users.noreply.github.com> Date: Tue Nov 17 00:37:49 2015 -0300 Simple logging string for stats (Deleted redundant info) --- gettor/smtp.py | 13 +++++-------- gettor/twitter.py | 11 ++++------- gettor/xmpp.py | 13 +++++-------- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/gettor/smtp.py b/gettor/smtp.py index 9203039..d6de2ed 100644 --- a/gettor/smtp.py +++ b/gettor/smtp.py @@ -462,14 +462,14 @@ class SMTP(object): norm_from_addr = self._get_normalized_address(from_addr) except AddressError as e: bogus_request = True - self.log.info('REQUEST: invalid; OS: none; LC: none') + self.log.info('invalid; none; none') if norm_from_addr: anon_addr = utils.get_sha256(norm_from_addr) if self._is_blacklisted(anon_addr): bogus_request = True - self.log.info('REQUEST: blacklist; OS: none; LC: none') + self.log.info('blacklist; none; none') if not bogus_request: # try to figure out what the user is asking @@ -482,8 +482,7 @@ class SMTP(object): # possible options: help, links, mirrors if req['type'] == 'help': self.log.debug("Trying to send help...") - self.log.info('REQUEST: help; OS: none; LC: %s' % - req['lc']) + self.log.info('help; none; %s' % req['lc']) # make sure we can send emails try: self._send_help('en', our_addr, norm_from_addr) @@ -494,8 +493,7 @@ class SMTP(object): elif req['type'] == 'mirrors': self.log.debug("Trying to send the mirrors...") - self.log.info('REQUEST: mirrors; OS: none; LC: %s' % - req['lc']) + self.log.info('mirrors; none; %s' % req['lc']) # make sure we can send emails try: self._send_mirrors('en', our_addr, norm_from_addr) @@ -506,8 +504,7 @@ class SMTP(object): elif req['type'] == 'links': self.log.debug("Trying to obtain the links...") - self.log.info('REQUEST: links; OS: %s; LC: %s' % - req['os '], req['lc']) + self.log.info('links; %s; %s' % (req['os '], req['lc'])) try: links = self.core.get_links( diff --git a/gettor/twitter.py b/gettor/twitter.py index 85cad47..f4b934f 100644 --- a/gettor/twitter.py +++ b/gettor/twitter.py @@ -218,7 +218,7 @@ class TwitterBot(object): try: if self._is_blacklisted(str(sender_id)): - self.log.info('REQUEST: blacklist; OS: none; LC: none') + self.log.info('blacklist; none; none') bogus_request = True if not bogus_request: @@ -228,8 +228,7 @@ class TwitterBot(object): # possible options: links, mirrors, help if request['type'] == 'links': - self.log.info('REQUEST: links; OS: %s; LC: %s' % - (req['os'], req['lc'])) + self.log.info('links; %s; %s' % (req['os'], req['lc'])) links = self.core.get_links( 'Twitter', request['os'], @@ -240,8 +239,7 @@ class TwitterBot(object): reply = reply % (request['os'], request['lc'], links) elif request['type'] == 'mirrors': - self.log.info('REQUEST: mirrors; OS: none; LC: %s' % - req['lc']) + self.log.info('mirrors; none; %s' % req['lc']) reply = self._get_msg('mirrors', 'en') try: with open(self.mirrors, "r") as list_mirrors: @@ -252,8 +250,7 @@ class TwitterBot(object): reply = self._get_msg('mirrors_unavailable', 'en') else: - self.log.info('REQUEST: help; OS: none; LC: %s' % - req['lc']) + self.log.info('help; none; %s' % req['lc']) reply = self._get_msg('help', 'en') self.api.send_direct_message( diff --git a/gettor/xmpp.py b/gettor/xmpp.py index 072d988..1bea716 100644 --- a/gettor/xmpp.py +++ b/gettor/xmpp.py @@ -277,7 +277,7 @@ class XMPP(object): self.log.debug("Parsing request") try: if self._is_blacklisted(str(account)): - self.log.info('REQUEST: blacklist; OS: none; LC: none') + self.log.info('blacklist; none; none') bogus_request = True # first let's find out how many words are in the message @@ -286,7 +286,7 @@ class XMPP(object): if len(words) > self.max_words: bogus_request = True self.log.info("Message way too long") - self.log.info('REQUEST: invalid; OS: none; LC: none') + self.log.info('invalid; none; none') reply = self._get_msg('message_error', 'en') if not bogus_request: @@ -295,13 +295,11 @@ class XMPP(object): req = self._parse_text(str(msg)) if req['type'] == 'help': - self.log.info('REQUEST: help; OS: none; LC: %s' % - req['lc']) + self.log.info('help; none; %s' % req['lc']) reply = self._get_msg('help', 'en') elif req['type'] == 'mirrors': - self.log.info('REQUEST: mirrors; OS: none; LC: %s' % - req['lc']) + self.log.info('mirrors; none; %s' % req['lc']) reply = self._get_msg('mirrors', 'en') try: with open(self.mirrors, "r") as list_mirrors: @@ -311,8 +309,7 @@ class XMPP(object): reply = self._get_msg('mirrors_unavailable', 'en') elif req['type'] == 'links': - self.log.info('REQUEST: links; OS: %s; LC: %s' % - (req['os'], req['lc'])) + self.log.info('links; %s; %s' % (req['os'], req['lc'])) links = self.core.get_links( "XMPP", req['os'],

1 0

[gettor/develop] Adapted requests table to new stats
by ilv＠torproject.org 04 Dec '15

04 Dec '15

commit f6ed22811e74b615c048c8d8cd75c0eb01b9cd55 Author: ilv <ilv(a)users.noreply.github.com> Date: Fri Dec 4 15:21:37 2015 -0300 Adapted requests table to new stats --- scripts/create_db.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/create_db.py b/scripts/create_db.py index ec4ba32..d6e0b0f 100644 --- a/scripts/create_db.py +++ b/scripts/create_db.py @@ -38,15 +38,16 @@ def main(): with con: cur = con.cursor() # table for handling users (i.e. blacklist) - cur.execute("CREATE TABLE users(id TEXT, service TEXT, times INT," - " blocked INT, last_request TEXT)") - # table for stats - # cur.execute("CREATE TABLE requests(service TEXT, type TEXT," - # " os TEXT, lc TEXT, pt INT, year INT, month INT," - # " day INT, status TEXT, logfile TEXT)") + cur.execute( + "CREATE TABLE users(id TEXT, service TEXT, times INT," + "blocked INT, last_request TEXT)" + ) + + cur.execute( + "CREATE TABLE requests(date TEXT, request TEXT, os TEXT," + " locale TEXT, channel TEXT, PRIMARY KEY (date, channel))" + ) - # for now we will use a counter - cur.execute("CREATE TABLE requests(id INT, counter INT)") print "Database %s created" % os.path.abspath(args.create) elif args.delete: os.remove(os.path.abspath(args.delete))

1 0

[metrics-lib/master] Prevent disappearance of torperf annotation.
by karsten＠torproject.org 04 Dec '15

04 Dec '15

commit 1e942ab15d636ca3307fe5327896afff807b52f8 Author: iwakeh <iwakeh(a)users.ourproject.org> Date: Thu Nov 26 10:00:00 2015 +0000 Prevent disappearance of torperf annotation. --- .../descriptor/impl/TorperfResultImpl.java | 12 ++- .../descriptor/impl/TorperfResultImplTest.java | 86 ++++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/src/org/torproject/descriptor/impl/TorperfResultImpl.java b/src/org/torproject/descriptor/impl/TorperfResultImpl.java index d865802..6da985f 100644 --- a/src/org/torproject/descriptor/impl/TorperfResultImpl.java +++ b/src/org/torproject/descriptor/impl/TorperfResultImpl.java @@ -27,6 +27,7 @@ public class TorperfResultImpl extends DescriptorImpl List<Descriptor> parsedDescriptors = new ArrayList<Descriptor>(); String descriptorString = new String(rawDescriptorBytes); Scanner s = new Scanner(descriptorString).useDelimiter("\n"); + String typeAnnotation = ""; while (s.hasNext()) { String line = s.next(); if (line.startsWith("@type torperf ")) { @@ -40,9 +41,12 @@ public class TorperfResultImpl extends DescriptorImpl throw new DescriptorParseException("Unsupported version in " + " line '" + line + "'."); } + typeAnnotation = line + "\n"; } else { - parsedDescriptors.add(new TorperfResultImpl(line.getBytes(), + parsedDescriptors.add(new TorperfResultImpl( + (typeAnnotation + line).getBytes(), failUnrecognizedDescriptorLines)); + typeAnnotation = ""; } } return parsedDescriptors; @@ -55,8 +59,12 @@ public class TorperfResultImpl extends DescriptorImpl this.parseTorperfResultLine(new String(rawDescriptorBytes)); } - private void parseTorperfResultLine(String line) + private void parseTorperfResultLine(String inputLine) throws DescriptorParseException { + String line = inputLine; + while (line.startsWith("@") && line.contains("\n")) { + line = line.split("\n")[1]; + } if (line.isEmpty()) { throw new DescriptorParseException("Blank lines are not allowed."); } diff --git a/test/org/torproject/descriptor/impl/TorperfResultImplTest.java b/test/org/torproject/descriptor/impl/TorperfResultImplTest.java new file mode 100644 index 0000000..be9221f --- /dev/null +++ b/test/org/torproject/descriptor/impl/TorperfResultImplTest.java @@ -0,0 +1,86 @@ +/* Copyright 2015 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.descriptor.impl; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.util.List; + +import org.junit.Test; +import org.torproject.descriptor.Descriptor; + +public class TorperfResultImplTest { + + @Test() + public void testAnnotatedInput() throws Exception{ + TorperfResultImpl result = (TorperfResultImpl) + (TorperfResultImpl.parseTorperfResults((torperfAnnotation + input) + .getBytes("US-ASCII"), false).get(0)); + assertEquals("Expected one annotation.", 1, + result.getAnnotations().size()); + assertEquals(torperfAnnotation.substring(0, 17), + result.getAnnotations().get(0)); + int count = 0; + for (Long l: result.getDataPercentiles().values()) { + assertNotNull(l); + assertEquals(l.longValue(), deciles[count++]); + } + } + + @Test() + public void testPartiallyAnnotatedInput() throws Exception{ + byte[] asciiBytes = (torperfAnnotation + + input + input + input).getBytes("US-ASCII"); + List<Descriptor> result = TorperfResultImpl.parseTorperfResults( + asciiBytes, false); + assertEquals("Expected one annotation.", 1, + ((TorperfResultImpl)(result.get(0))).getAnnotations().size()); + assertEquals(3, result.size()); + assertEquals("Expected zero annotations.", 0, + ((TorperfResultImpl)(result.get(1))).getAnnotations().size()); + assertEquals("Expected zero annotations.", 0, + ((TorperfResultImpl)(result.get(2))).getAnnotations().size()); + } + + @Test() + public void testAllAnnotatedInput() throws Exception{ + byte[] asciiBytes = (torperfAnnotation + input + + torperfAnnotation + input + + torperfAnnotation + input).getBytes("US-ASCII"); + List<Descriptor> result = TorperfResultImpl.parseTorperfResults( + asciiBytes, false); + assertEquals("Expected one annotation.", 1, + ((TorperfResultImpl)(result.get(0))).getAnnotations().size()); + assertEquals(3, result.size()); + assertEquals("Expected one annotation.", 1, + ((TorperfResultImpl)(result.get(1))).getAnnotations().size()); + assertEquals("Expected one annotation.", 1, + ((TorperfResultImpl)(result.get(2))).getAnnotations().size()); + } + + private static long[] deciles = new long[] { + 1441065602980L, 1441065603030L, 1441065603090L, 1441065603120L, + 1441065603230L, 1441065603250L, 1441065603310L, 1441065603370L, + 1441065603370L }; + + private static final String torperfAnnotation = "@type torperf 1.0\n"; + + private static final String input = + "BUILDTIMES=0.872834920883,1.09103679657,1.49180984497 " + + "CIRC_ID=1228 CONNECT=1441065601.86 DATACOMPLETE=1441065603.39 " + + "DATAPERC10=1441065602.98 DATAPERC20=1441065603.03 " + + "DATAPERC30=1441065603.09 DATAPERC40=1441065603.12 " + + "DATAPERC50=1441065603.23 DATAPERC60=1441065603.25 " + + "DATAPERC70=1441065603.31 DATAPERC80=1441065603.37 " + + "DATAPERC90=1441065603.37 DATAREQUEST=1441065602.38 " + + "DATARESPONSE=1441065602.84 DIDTIMEOUT=0 FILESIZE=51200 " + + "LAUNCH=1441065361.30 NEGOTIATE=1441065601.86 " + + "PATH=$C4C9C332D25B3546BEF4E1250CF410E97EF996E6," + + "$C43FA6474A9F071E9120DF63ED6EB8FDBA105234," + + "$7C0AA4E3B73E407E9F5FEB1912F8BE26D8AA124D QUANTILE=0.800000 " + + "READBYTES=51416 REQUEST=1441065601.86 RESPONSE=1441065602.38 " + + "SOCKET=1441065601.86 SOURCE=moria START=1441065601.86 " + + "TIMEOUT=1500 USED_AT=1441065603.40 USED_BY=2475 WRITEBYTES=75\n"; +} +

1 0

[metrics-lib/master] Require Java 7 now rather than after the first release.
by karsten＠torproject.org 02 Dec '15

02 Dec '15

commit 125d5cb92ef9ad86ccae10d6924e9ba951e76d7d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Dec 2 15:08:49 2015 +0100 Require Java 7 now rather than after the first release. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index eb043db..cef43c5 100644 --- a/build.xml +++ b/build.xml @@ -7,7 +7,7 @@ <property name="jarfile" value="descriptor-${release.version}.jar" /> <property name="jarsourcesfile" value="descriptor-${release.version}-sources.jar" /> - <property name="source-and-target-java-version" value="1.6" /> + <property name="source-and-target-java-version" value="1.7" /> <property name="release.tarball" value="descriptor-${release.version}.tar.gz" /> <property file="build.properties" />

1 0

[metrics-lib/master] Document signing and verifying releases.
by karsten＠torproject.org 02 Dec '15

02 Dec '15

commit c3202bdc5d84abf526d11dbde7ab8ffd344d6c54 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Dec 2 14:44:21 2015 +0100 Document signing and verifying releases. --- CONTRIB.md | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 57 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+) diff --git a/CONTRIB.md b/CONTRIB.md index 574c549..62c7cf8 100644 --- a/CONTRIB.md +++ b/CONTRIB.md @@ -154,6 +154,90 @@ Regarding version numbers, we'll start with 1.0.0 and bump to 1.0.1, previously deprecated feature, making a backwards-incompatible change, we'll bump to 2.0.0. +Releases are cryptographically signed on multiple levels: the Git tag +created for the release is signed using GnuPG, the produced .jar files are +signed using Java's jarsigner tool, and the produced tarball is again +signed using GnuPG. We'll assume that you're familiar with GnuPG and how +to manage keys for it, but we're including some sample commands for +managing keys for jarsigner using the less commonly known Java keytool +below. + +First, generate a new key pair and create a certificate that expires after +90 days using reasonable (yet compatible) cryptographic algorithms and +parameters: + +``` +keytool -genkeypair -alias karsten -keyalg RSA -keysize 2048 \ + -sigalg SHA256withRSA -validity 90 \ + -dname "CN=Karsten Loesing, O=The Tor Project\, Inc, L=Cambridge, ST=MA, C=US" +``` + +Extend the certificate for the existing key pair when it expires: + +``` +keytool -selfcert -alias karsten +``` + +Export the certificate to a local file called CERT: + +``` +keytool -exportcert -alias karsten -rfc -file CERT +``` + +Also, in order to sign releases using Ant, you'll have to create a file +`build.properties` with content similar to the content below (note that +without such a file, the Ant targets starting at `signjar` won't work): + +``` +jarsigner.alias=karsten +jarsigner.storepass=password +``` + +Putting out a new release requires a series of steps: + +Edit `build.xml` and raise the `release.version` property to the desired +new release. + +Edit `CHANGELOG.md` and make sure it contains the correct date for the +release. + +Commit these changes. + +Create a signed Git tag for the new release: + +``` +git tag -s descriptor-1.0.0 -m "DescripTor 1.0.0" +``` + +Push the branch. Ideally, verify the tag signature by cloning it on +another system and running the following command: + +``` +git verify-tag descriptor-1.0.0 +``` + +Use Ant to first clean up and then create a tarball containing all sources +and signed .jar files: + +``` +ant clean compile test jar signjar tar +``` + +Sign the produced tarball using GnuPG: + +``` +gpg --detach-sign --armor --local-user 0x4EFD4FDC3F46D41E \ + descriptor-1.0.0.tar.gz +``` + +Verify the signed tarball, ideally on a different system, as described in +`README.md`. + +Upload the tarball and signature file and announce the new version. + +Edit `build.xml` again and raise `release.version` to the current release +plus `-dev`, e.g., `1.0.0-dev`. + Packages -------- diff --git a/README.md b/README.md index cbf2a11..764f143 100644 --- a/README.md +++ b/README.md @@ -24,3 +24,60 @@ running, batch-processing applications and not for continuously running applications that rely on learning about changes to an underlying descriptor source. + +Verifying releases +------------------ + +Releases can be cryptographically verified to get some more confidence that +they were put together by a Tor developer. The following steps explain the +verification process by example. + +Download the release tarball and the separate signature file: + +``` +wget https://dist.torproject.org/descriptor/1.0.0/descriptor-1.0.0.tar.gz +wget https://dist.torproject.org/descriptor/1.0.0/descriptor-1.0.0.tar.gz.asc +``` + +Attempt to verify the signature on the tarball: + +``` +gpg --verify descriptor-1.0.0.tar.gz.asc +``` + +If the signature cannot be verified due to the public key of the signer +not being locally available, download that public key from one of the key +servers and retry: + +``` +gpg --keyserver pgp.mit.edu --recv-key 0x4EFD4FDC3F46D41E +gpg --verify descriptor-1.0.0.tar.gz.asc +``` + +If the signature still cannot be verified, something is wrong! + +But note that even if it can be verified, you now only know that the +signature was made by the person claiming to own this key, which could be +anyone. You'll need a trust path to the owner of this key in order to +trust this signature, but that's clearly out of scope here. In short, +your best chance is to meet a Tor developer in real life and enter the web +of trust. + +If you want to go one step further in the verification game, you can +verify the signature on the .jar files. + +Print and then import the provided X.509 certificate: + +``` +keytool -printcert -file CERT +keytool -importcert -alias karsten -file CERT +``` + +Verify the signatures on the contained .jar files using Java's jarsigner +tool: + +``` +jarsigner -verify descriptor-1.0.0.jar +jarsigner -verify descriptor-1.0.0-sources.jar +``` +

1 0