- tor-commits - lists.torproject.org

[tor/master] Check whether a client port is a Unix socket before using its IP addr
by nickm＠torproject.org 21 Oct '11

21 Oct '11

commit 9df99bbb9171c15b00c5148f54397351eccac95b Author: Robert Ransom <rransom.8774(a)gmail.com> Date: Thu Oct 20 03:17:23 2011 -0700 Check whether a client port is a Unix socket before using its IP addr Bugfix on commit c1ac0695d5bc64b555c345e4be87b18bab3ae56b, not yet in any release. Fixes bug 4091; bug reported by SwissTorHelp. --- src/or/config.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diff --git a/src/or/config.c b/src/or/config.c index 388c45f..59bbe1d 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -5128,7 +5128,9 @@ static void warn_nonlocal_client_ports(const smartlist_t *ports, const char *portname) { SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) { - if (!tor_addr_is_internal(&port->addr, 1)) { + if (port->is_unix_addr) { + /* Unix sockets aren't accessible over a network. */ + } else if (!tor_addr_is_internal(&port->addr, 1)) { log_warn(LD_CONFIG, "You specified a public address for %sPort. " "Other people on the Internet might find your computer and " "use it as an open proxy. Please don't allow this unless you "

1 0

[tor/master] Merge remote-tracking branch 'rransom-tor/bug4091'
by nickm＠torproject.org 21 Oct '11

21 Oct '11

commit 29825f0bfd301185744d6961c1b62b9597d165d6 Merge: 169c818 9df99bb Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Oct 20 14:26:54 2011 -0400 Merge remote-tracking branch 'rransom-tor/bug4091' src/or/config.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-)

1 0

[translation/gettor] Update translations for gettor
by translation＠torproject.org 21 Oct '11

21 Oct '11

commit 36d246006a03e9c4c906ebdde51f4a233c696732 Author: Translation commit bot <translation(a)torproject.org> Date: Thu Oct 20 18:15:24 2011 +0000 Update translations for gettor --- ar/gettor.po | 27 +++++++++++++++++++++++++-- 1 files changed, 25 insertions(+), 2 deletions(-) diff --git a/ar/gettor.po b/ar/gettor.po index 5a654db..7b921b2 100644 --- a/ar/gettor.po +++ b/ar/gettor.po @@ -12,8 +12,8 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2011-09-10 01:52-0400\n" -"PO-Revision-Date: 2011-10-11 06:30+0000\n" -"Last-Translator: modafar <modafar22(a)hotmail.com>\n" +"PO-Revision-Date: 2011-10-20 18:07+0000\n" +"Last-Translator: redeemer <fadi.redeemer.mansour(a)gmail.com>\n" "Language-Team: LANGUAGE <LL(a)li.org>\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -275,6 +275,11 @@ msgid "" "\n" " gpg --verify tor-browser-1.3.24_en-US.exe.asc tor-browser-1.3.24_en-US.exe" msgstr "" +"التأكد من التوقيع\n" +"========\n" +"إن كان حاسبك يحوي غنو بي جي (GnuPG)، قم باستخدام الأمر gpg بالشكل التالي بعد فك ضغط ملف zip:\n" +"\n" +" gpg --verify tor-browser-1.3.24_en-US.exe.asc tor-browser-1.3.24-en-US.exe" #: lib/gettor/i18n.py:143 msgid "" @@ -282,6 +287,10 @@ msgid "" "\n" " gpg: Good signature from 'Erinn Clark <...>'" msgstr "" +"يجب أن تكون النتيجة شيء من هذا القبيل:\n" +"\n" +" gpg: Good signature from 'Erinn Clark <...>'\n" +" gpg: توقيع جيد من 'إرين كلارك <...>'" #: lib/gettor/i18n.py:147 msgid "" @@ -290,12 +299,17 @@ msgid "" "\n" " http://www.gnupg.org/related_software/frontends.html" msgstr "" +"إن لم تكن معتاداً على استخدام أدوات سطر الأوامر، حاول البحث عن واجهة رسومية لغنو بي جي على هذا الموقع:\n" +"\n" +" http://www.gnupg.org/related_software/frontends.html" #: lib/gettor/i18n.py:152 msgid "" "BLOCKED ACCESS / CENSORSHIP\n" "===========================" msgstr "" +"الوصول المحجوب/الرقابة\n" +"=============" #: lib/gettor/i18n.py:155 msgid "" @@ -319,6 +333,9 @@ msgid "" "\n" " bridges(a)torproject.org" msgstr "" +"يمكنك الحصول على جسر بإرسال بريد يحوي \"get bridges\" في متن البريد وعنونه للبريد التالي:\n" +"\n" +" bridges(a)torproject.org" #: lib/gettor/i18n.py:167 msgid "" @@ -354,12 +371,16 @@ msgid "" "It was successfully understood. Your request is currently being processed.\n" "Your package (%s) should arrive within the next ten minutes." msgstr "" +"تم الفهم بنجاح. تتم حالياً معالجة طلبك. ستصلك رزمتك (%s) خلال الدقائق العشر " +"التالية." #: lib/gettor/i18n.py:182 msgid "" "If it doesn't arrive, the package might be too big for your mail provider.\n" "Try resending the mail from a GMAIL.COM, YAHOO.CN or YAHOO.COM account." msgstr "" +"إن لم تصل، لا بد أن الرزمة أكبر مما يسمح به مزود بريدك. حاول أن تعيد إرسال " +"الرسالة من حساب بريد من GMAIL.COM، YAHOO.CN أو YAHOO.COM." #: lib/gettor/i18n.py:185 msgid "" @@ -385,6 +406,8 @@ msgid "" "UNPACKING THE FILES\n" "===================" msgstr "" +"فك ضغط الملفات\n" +"=========" #: lib/gettor/i18n.py:196 msgid ""

1 0

[torspec/master] Oops; commutative, not transitive.
by nickm＠torproject.org 21 Oct '11

21 Oct '11

commit 9cd2dd3f9a5cc07e4caab92a8e81b10c15fc7717 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Oct 20 13:58:37 2011 -0400 Oops; commutative, not transitive. --- proposals/188-bridge-guards.txt | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/proposals/188-bridge-guards.txt b/proposals/188-bridge-guards.txt index 249d794..3c53cfb 100644 --- a/proposals/188-bridge-guards.txt +++ b/proposals/188-bridge-guards.txt @@ -134,7 +134,7 @@ Status: Open before passing it to Guillaume. Note that this design does not require that our stream cipher - operations be transitive, even though they are. + operations be commutative, even though they are. Note also that this design requires no change in behavior from any node other than Bob the bridge.

1 0

[stem/master] Moving tor launcher into stem function
by atagar＠torproject.org 21 Oct '11

21 Oct '11

commit 3a913b6b92796f2195c28e55ead532a0fef906e6 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Oct 20 09:58:37 2011 -0700 Moving tor launcher into stem function In writing the stem integ tests I needed a function for starting tor then blocking for its bootstraping to complete. Yesterday Jake mentioned that he could use a launch_tor() function in TorCtl so I'm generalizing this code and moving it into the stem lib. --- stem/__init__.py | 2 +- stem/process.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ test/runner.py | 48 ++++++++++------------------------------- 3 files changed, 74 insertions(+), 37 deletions(-) diff --git a/stem/__init__.py b/stem/__init__.py index 01c350c..5ac77ed 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -2,5 +2,5 @@ Library for working with the tor process. """ -__all__ = ["types"] +__all__ = ["process", "types"] diff --git a/stem/process.py b/stem/process.py new file mode 100644 index 0000000..f049610 --- /dev/null +++ b/stem/process.py @@ -0,0 +1,61 @@ +""" +Helper functions for working with tor as a process. These are mostly os +dependent, only working on linux, osx, and bsd. +""" + +import os +import signal +import subprocess + +# number of seconds before we time out our attempt to start a tor instance +TOR_INIT_TIMEOUT = 90 + +def launch_tor(torrc_path, init_msg_handler = None): + """ + Initializes a tor process. This blocks until initialization completes or we + error out. + + Arguments: + torrc_path (str) - location of the torrc for us to use + init_msg_handler (functor) - optional functor that will be provided with + tor's initialization stdout as we get it + + Returns: + subprocess.Popen instance for the tor subprocess + + Raises: + OSError if we either fail to create the tor process or reached a timeout + without success + """ + + # double check that we have a torrc to work with + if not os.path.exists(torrc_path): + raise OSError("torrc doesn't exist (%s)" % torrc_path) + + # starts a tor subprocess, raising an OSError if it fails + tor_process = subprocess.Popen(["tor", "-f", torrc_path], stdout = subprocess.PIPE, stderr = subprocess.PIPE) + + # time ourselves out if we reach TOR_INIT_TIMEOUT + def timeout_handler(signum, frame): + # terminates the uninitialized tor process and raise on timeout + tor_process.kill() + raise OSError("reached a %i second timeout without success" % TOR_INIT_TIMEOUT) + + signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(TOR_INIT_TIMEOUT) + + while True: + init_line = tor_process.stdout.readline().strip() + + # this will provide empty results if the process is terminated + if not init_line: + tor_process.kill() # ... but best make sure + raise OSError("process terminated") + + # provide the caller with the initialization message if they want it + if init_msg_handler: init_msg_handler(init_line) + + # return the process if we're done with bootstrapping + if init_line.endswith("Bootstrapped 100%: Done."): + return tor_process + diff --git a/test/runner.py b/test/runner.py index 6f76d01..c795ea8 100644 --- a/test/runner.py +++ b/test/runner.py @@ -6,9 +6,9 @@ import os import sys import time import shutil -import signal import tempfile -import subprocess + +import stem.process from stem.util import term @@ -87,44 +87,20 @@ class Runner: without success """ + def print_init_line(init_line): + """ + Prints output from tor's stdout while it starts up. + """ + + print term.format(" %s" % init_line, term.Color.BLUE) + print term.format("Starting tor...", term.Color.BLUE, term.Attr.BOLD) start_time = time.time() try: - # terminate our previous instance before continuing if we had one - if self._tor_process: self._tor_process.kill() - - # double check that we have a torrc to work with - torrc_dst = self.get_torrc_path() - if not os.path.exists(torrc_dst): - raise OSError("torrc doesn't exist (%s)" % torrc_dst) - - # starts a tor subprocess, raising an OSError if it fails - self._tor_process = subprocess.Popen(["tor", "-f", torrc_dst], stdout = subprocess.PIPE, stderr = subprocess.PIPE) - - # time ourselves out if we reach TOR_INIT_TIMEOUT - def timeout_handler(signum, frame): - # terminates the uninitialized tor process and raise on timeout - self._tor_process.kill() - raise OSError("reached a %i second timeout without success" % TOR_INIT_TIMEOUT) - - signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(TOR_INIT_TIMEOUT) - - while True: - init_line = self._tor_process.stdout.readline().strip() - - # this will provide empty results if the process is terminated - if not init_line: - self._tor_process.kill() # ... but best make sure - raise OSError("process terminated") - - print term.format(" %s" % init_line, term.Color.BLUE) - - # return the process if we're done with bootstrapping - if init_line.endswith("Bootstrapped 100%: Done."): - print term.format(" done (%i seconds)" % (time.time() - start_time), term.Color.BLUE, term.Attr.BOLD) - return + self._tor_process = stem.process.launch_tor(self.get_torrc_path(), print_init_line) + print term.format(" done (%i seconds)" % (time.time() - start_time), term.Color.BLUE, term.Attr.BOLD) + return except OSError, exc: print term.format(" failed to start tor: %s" % exc, term.Color.RED, term.Attr.BOLD) raise exc

1 0

[stem/master] Accidently suppressing integ asserts
by atagar＠torproject.org 21 Oct '11

21 Oct '11

commit 0a083c8b73a663d38f473e0180b0009d28e1b5d9 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Oct 20 10:09:36 2011 -0700 Accidently suppressing integ asserts A previous commit to suppress errors from the close() method in python 2.7 also suppressed the following asserts. These asserts still work with 2.7 so removing them from the try block. --- test/integ/message.py | 12 ++++++------ 1 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/integ/message.py b/test/integ/message.py index 053e9df..ae92af6 100644 --- a/test/integ/message.py +++ b/test/integ/message.py @@ -67,13 +67,13 @@ class TestMessageFunctions(unittest.TestCase): try: control_socket_file.close() control_socket_file.write("GETINFO version\r\n") - - # receives: AttributeError: 'NoneType' object has no attribute 'sendall' - self.assertRaises(AttributeError, control_socket_file.flush) - - # receives: stem.types.ControlSocketClosed: socket file has been closed - self.assertRaises(stem.types.ControlSocketClosed, stem.types.read_message, control_socket_file) except: pass + + # receives: AttributeError: 'NoneType' object has no attribute 'sendall' + self.assertRaises(AttributeError, control_socket_file.flush) + + # receives: stem.types.ControlSocketClosed: socket file has been closed + self.assertRaises(stem.types.ControlSocketClosed, stem.types.read_message, control_socket_file) def test_invalid_command(self): """

1 0

[stem/master] Message integ tests compatability issues
by atagar＠torproject.org 21 Oct '11

21 Oct '11

commit 26374bafd5957d3039cf10596de0f7a021f1afb9 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Oct 20 09:49:45 2011 -0700 Message integ tests compatability issues The message integration tests had compatability issues with... - Newer python versions (2.6 -> 2.7) due to an unexpected failure when calling close() on the socket's file object. We're operating with a closed socket at that time so all bets are off about if this should/shouldn't raise so the difference in behavior just warranted a testing workaround. - Older tor versions don't support 'GETINFO config-text'. I've left this as a TODO note for now and will add a version check for that test later. --- test/integ/message.py | 24 ++++++++++++++++-------- 1 files changed, 16 insertions(+), 8 deletions(-) diff --git a/test/integ/message.py b/test/integ/message.py index 46f20d2..053e9df 100644 --- a/test/integ/message.py +++ b/test/integ/message.py @@ -60,15 +60,20 @@ class TestMessageFunctions(unittest.TestCase): self.assertRaises(stem.types.ControlSocketClosed, stem.types.read_message, control_socket_file) # Closing the file handler, however, will cause a different type of error. + # This seems to depend on the python version, in 2.6 we get an + # AttributeError and in 2.7 the close() call raises... + # error: [Errno 32] Broken pipe - control_socket_file.close() - control_socket_file.write("GETINFO version\r\n") - - # receives: AttributeError: 'NoneType' object has no attribute 'sendall' - self.assertRaises(AttributeError, control_socket_file.flush) - - # receives: stem.types.ControlSocketClosed: socket file has been closed - self.assertRaises(stem.types.ControlSocketClosed, stem.types.read_message, control_socket_file) + try: + control_socket_file.close() + control_socket_file.write("GETINFO version\r\n") + + # receives: AttributeError: 'NoneType' object has no attribute 'sendall' + self.assertRaises(AttributeError, control_socket_file.flush) + + # receives: stem.types.ControlSocketClosed: socket file has been closed + self.assertRaises(stem.types.ControlSocketClosed, stem.types.read_message, control_socket_file) + except: pass def test_invalid_command(self): """ @@ -135,6 +140,9 @@ class TestMessageFunctions(unittest.TestCase): Parses the 'GETINFO config-text' response. """ + # TODO: add a 0.2.2.7-alpha version check for this test (that was when + # 'GETINFO config-text' was added) + # We can't be certain of the order, and there may be extra config-text # entries as per... # https://trac.torproject.org/projects/tor/ticket/2362

1 0

[metrics-web/master] Stop archiving consensuses and votes after checking them.
by karsten＠torproject.org 21 Oct '11

21 Oct '11

commit 29dbe7fbb197c7c22f85d3fcd610daa8613a2de2 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Oct 20 14:30:47 2011 +0200 Stop archiving consensuses and votes after checking them. The idea of archiving consensus and votes was that we could compare them over time to see if, e.g., a consensus parameter gets added or removed. This works in theory, but in practice, this approach makes the code more complex than it needs to be. We can as well change valid consensus parameters in the code. --- src/org/torproject/chc/Archiver.java | 120 ---------------------- src/org/torproject/chc/Downloader.java | 15 +--- src/org/torproject/chc/Main.java | 34 +------ src/org/torproject/chc/MetricsWebsiteReport.java | 5 - src/org/torproject/chc/NagiosReport.java | 19 ---- src/org/torproject/chc/Parser.java | 11 -- src/org/torproject/chc/Report.java | 4 - src/org/torproject/chc/Status.java | 9 -- src/org/torproject/chc/StdOutReport.java | 19 ---- 9 files changed, 5 insertions(+), 231 deletions(-) diff --git a/src/org/torproject/chc/Archiver.java b/src/org/torproject/chc/Archiver.java deleted file mode 100644 index 6271348..0000000 --- a/src/org/torproject/chc/Archiver.java +++ /dev/null @@ -1,120 +0,0 @@ -/* Copyright 2011 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.chc; - -import java.io.*; -import java.util.*; - -/* Load the last network status consensus and corresponding votes from - * disk and save newly downloaded ones to disk. */ -public class Archiver { - - /* Local directory containing cached consensuses and votes. */ - private File cachedFilesDirectory = new File("chc-cache"); - - /* Copy of the most recent cached consensus and votes. */ - private String loadedConsensus; - private List<String> loadedVotes = new ArrayList<String>(); - - /* Load the most recent cached consensus and votes to memory. */ - public void loadLastFromDisk() { - if (!this.cachedFilesDirectory.exists()) { - return; - } - String lastValidAfter = this.findLastConsensusPrefix(); - if (lastValidAfter != null) { - for (File file : this.cachedFilesDirectory.listFiles()) { - if (file.isDirectory() || - !file.getName().startsWith(lastValidAfter)) { - continue; - } - String content = null; - try { - FileInputStream fis = new FileInputStream(file); - BufferedInputStream bis = new BufferedInputStream(fis); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int len; - byte[] data = new byte[1024]; - while ((len = bis.read(data, 0, 1024)) >= 0) { - baos.write(data, 0, len); - } - bis.close(); - byte[] allData = baos.toByteArray(); - content = new String(allData); - } catch (IOException e) { - System.err.println("Could not read cached status from file '" - + file.getAbsolutePath() + "'. Skipping."); - continue; - } - if (file.getName().contains("-consensus")) { - this.loadedConsensus = content; - } else if (file.getName().contains("-vote-")) { - this.loadedVotes.add(content); - } - } - } - } - - /* Delete all cached consensuses and votes but the last consensus and - * corresponding votes. */ - public void deleteAllButLast() { - if (!this.cachedFilesDirectory.exists()) { - return; - } - String lastValidAfter = this.findLastConsensusPrefix(); - if (lastValidAfter != null) { - for (File file : this.cachedFilesDirectory.listFiles()) { - if (!file.getName().startsWith(lastValidAfter)) { - file.delete(); - } - } - } - } - - /* Save a status to disk under the given file name. */ - public void saveStatusStringToDisk(String content, String fileName) { - try { - this.cachedFilesDirectory.mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter(new File( - this.cachedFilesDirectory, fileName))); - bw.write(content); - bw.close(); - } catch (IOException e) { - System.err.println("Could not save status to file '" - + this.cachedFilesDirectory.getAbsolutePath() + "/" + fileName - + "'. Ignoring."); - } - } - - /* Find and return the timestamp prefix of the last published - * consensus. */ - private String findLastConsensusPrefix() { - String lastValidAfter = null; - for (File file : this.cachedFilesDirectory.listFiles()) { - if (file.isDirectory() || - file.getName().length() != - "yyyy-MM-dd-HH-mm-ss-consensus".length() || - !file.getName().endsWith("-consensus")) { - continue; - } - String prefix = file.getName().substring(0, - "yyyy-MM-dd-HH-mm-ss".length()); - if (lastValidAfter == null || - prefix.compareTo(lastValidAfter) > 0) { - lastValidAfter = prefix; - } - } - return lastValidAfter; - } - - /* Return the previously loaded (unparsed) consensus string. */ - public String getConsensusString() { - return this.loadedConsensus; - } - - /* Return the previously loaded (unparsed) vote strings. */ - public List<String> getVoteStrings() { - return this.loadedVotes; - } -} - diff --git a/src/org/torproject/chc/Downloader.java b/src/org/torproject/chc/Downloader.java index bdd23fe..cf451ee 100644 --- a/src/org/torproject/chc/Downloader.java +++ b/src/org/torproject/chc/Downloader.java @@ -20,21 +20,8 @@ public class Downloader { + "208.83.223.34:443,128.31.0.34:9131,194.109.206.212"). split(","))); - /* Set the last known valid-after time to avoid downloading a new - * consensus if there cannot be a new one yet. */ - private long lastKnownValidAfterMillis; - public void setLastKnownValidAfterMillis( - long lastKnownValidAfterMillis) { - this.lastKnownValidAfterMillis = lastKnownValidAfterMillis; - } - - /* Download a new consensus and corresponding votes if we expect them to - * be newer than the ones we have. */ + /* Download a new consensus and corresponding votes. */ public void downloadFromAuthorities() { - if (System.currentTimeMillis() - lastKnownValidAfterMillis < - 60L * 60L * 1000L) { - return; - } this.downloadConsensus(); if (this.downloadedConsensus != null) { this.parseConsensusToFindReferencedVotes(); diff --git a/src/org/torproject/chc/Main.java b/src/org/torproject/chc/Main.java index 5aadc6e..d42ee44 100644 --- a/src/org/torproject/chc/Main.java +++ b/src/org/torproject/chc/Main.java @@ -16,26 +16,13 @@ public class Main { reports.add(new NagiosReport("stats/consensus-health")); reports.add(new StdOutReport()); - /* Load last-known consensus and votes from disk and parse them. */ - Archiver archiver = new Archiver(); - archiver.loadLastFromDisk(); - Parser parser = new Parser(); - Status parsedCachedConsensus = parser.parse( - archiver.getConsensusString(), archiver.getVoteStrings()); - if (parsedCachedConsensus != null) { - for (Report report : reports) { - report.processCachedConsensus(parsedCachedConsensus); - } - } - /* Download consensus and corresponding votes from the directory - * authorities and parse them, too. */ + * authorities. */ Downloader downloader = new Downloader(); - if (parsedCachedConsensus != null) { - downloader.setLastKnownValidAfterMillis( - parsedCachedConsensus.getValidAfterMillis()); - } downloader.downloadFromAuthorities(); + + /* Parse consensus and votes and pass them to the reports. */ + Parser parser = new Parser(); Status parsedDownloadedConsensus = parser.parse( downloader.getConsensusString(), downloader.getVoteStrings()); if (parsedDownloadedConsensus != null) { @@ -44,19 +31,6 @@ public class Main { } } - /* Save the new consensus and corresponding votes to disk and delete - * all previous ones. */ - if (parsedDownloadedConsensus != null) { - archiver.saveStatusStringToDisk( - parsedDownloadedConsensus.getUnparsedString(), - parsedDownloadedConsensus.getFileName()); - for (Status vote : parsedDownloadedConsensus.getVotes()) { - archiver.saveStatusStringToDisk(vote.getUnparsedString(), - vote.getFileName()); - } - archiver.deleteAllButLast(); - } - /* Finish writing reports. */ for (Report report : reports) { report.writeReport(); diff --git a/src/org/torproject/chc/MetricsWebsiteReport.java b/src/org/torproject/chc/MetricsWebsiteReport.java index 2a54927..f77b89f 100644 --- a/src/org/torproject/chc/MetricsWebsiteReport.java +++ b/src/org/torproject/chc/MetricsWebsiteReport.java @@ -25,11 +25,6 @@ public class MetricsWebsiteReport implements Report { this.htmlOutputFile = new File(htmlOutputFilename); } - /* Process the last but one consensus (ignored in this report). */ - public void processCachedConsensus(Status cachedConsensus) { - /* Do nothing. */ - } - /* Store the downloaded consensus and corresponding votes for later * processing. */ private Status downloadedConsensus; diff --git a/src/org/torproject/chc/NagiosReport.java b/src/org/torproject/chc/NagiosReport.java index 11bdaa3..2b5a6d3 100644 --- a/src/org/torproject/chc/NagiosReport.java +++ b/src/org/torproject/chc/NagiosReport.java @@ -18,12 +18,6 @@ public class NagiosReport implements Report { this.nagiosOutputFile = new File(nagiosOutputFilename); } - /* Store the cached consensus for processing. */ - private Status cachedConsensus; - public void processCachedConsensus(Status cachedConsensus) { - this.cachedConsensus = cachedConsensus; - } - /* Store the current consensus and corresponding votes for * processing. */ private Status downloadedConsensus; @@ -58,8 +52,6 @@ public class NagiosReport implements Report { this.checkMissingVotes(); this.checkBandwidthScanners(); } - } else if (this.cachedConsensus != null) { - this.checkConsensusValid(this.cachedConsensus); } else { this.nagiosUnknowns.add("No consensus known"); } @@ -204,17 +196,6 @@ public class NagiosReport implements Report { } } - /* Check that the most recent consensus is not more than 3 hours - * old. */ - public void checkConsensusValid(Status consensus) { - if (consensus.getValidAfterMillis() < - System.currentTimeMillis() - 3L * 60L * 60L * 1000L) { - this.nagiosCriticals.add("The last known consensus published at " - + dateTimeFormat.format(consensus.getValidAfterMillis()) - + " is more than 3 hours old and therefore not valid anymore"); - } - } - /* Write all output to the Nagios status file. The most severe status * goes in the first line of the output file and the same status and all * log messages in the second line. */ diff --git a/src/org/torproject/chc/Parser.java b/src/org/torproject/chc/Parser.java index f7e9b57..47609a8 100644 --- a/src/org/torproject/chc/Parser.java +++ b/src/org/torproject/chc/Parser.java @@ -28,12 +28,9 @@ public class Parser { /* Date-time formats to parse and format timestamps. */ private static SimpleDateFormat dateTimeFormat; - private static SimpleDateFormat fileNameFormat; static { dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - fileNameFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); - fileNameFormat.setTimeZone(TimeZone.getTimeZone("UTC")); } /* Parse a consensus or vote string into a Status instance. */ @@ -143,14 +140,6 @@ public class Parser { br.close(); status.setRunningRelays(runningRelays); status.setBandwidthWeights(bandwidthWeights); - if (isConsensus) { - status.setFileName(fileNameFormat.format( - status.getValidAfterMillis()) + "-consensus"); - } else { - status.setFileName(fileNameFormat.format( - status.getValidAfterMillis()) + "-vote-" - + status.getFingerprint()); - } } catch (IOException e) { System.err.println("Caught an IOException while parsing a " + (isConsensus ? "consensus" : "vote") + " string. Skipping."); diff --git a/src/org/torproject/chc/Report.java b/src/org/torproject/chc/Report.java index 28483cc..c3937e0 100644 --- a/src/org/torproject/chc/Report.java +++ b/src/org/torproject/chc/Report.java @@ -8,10 +8,6 @@ import java.util.*; * some form. */ public interface Report { - /* Process the cached consensus and corresponding votes to compare them - * to the downloaded ones. */ - public abstract void processCachedConsensus(Status cachedConsensus); - /* Process the downloaded current consensus and corresponding votes to * find irregularities between them. */ public abstract void processDownloadedConsensus( diff --git a/src/org/torproject/chc/Status.java b/src/org/torproject/chc/Status.java index 51a9052..cbf7395 100644 --- a/src/org/torproject/chc/Status.java +++ b/src/org/torproject/chc/Status.java @@ -29,15 +29,6 @@ public class Status implements Comparable<Status> { return this.unparsedString; } - /* File name when writing this status to disk. */ - private String fileName; - public void setFileName(String fileName) { - this.fileName = fileName; - } - public String getFileName() { - return this.fileName; - } - /* Votes published at the same time as this consensus; votes don't * reference any statuses. */ private SortedSet<Status> votes = new TreeSet<Status>(); diff --git a/src/org/torproject/chc/StdOutReport.java b/src/org/torproject/chc/StdOutReport.java index f99c2eb..15a9148 100644 --- a/src/org/torproject/chc/StdOutReport.java +++ b/src/org/torproject/chc/StdOutReport.java @@ -21,12 +21,6 @@ public class StdOutReport implements Report { dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); } - /* Cached consensus for later processing. */ - private Status cachedConsensus; - public void processCachedConsensus(Status cachedConsensus) { - this.cachedConsensus = cachedConsensus; - } - /* Downloaded consensus and corresponding votes for later * processing. */ private Status downloadedConsensus; @@ -49,8 +43,6 @@ public class StdOutReport implements Report { this.checkMissingVotes(); this.checkBandwidthScanners(); } - } else if (this.cachedConsensus != null) { - this.checkConsensusValid(this.cachedConsensus); } else { this.warnings.put("No consensus known", 0L); } @@ -242,17 +234,6 @@ public class StdOutReport implements Report { } } - /* Check if the most recent consensus is older than 3 hours. */ - private void checkConsensusValid(Status consensus) { - if (consensus.getValidAfterMillis() < - System.currentTimeMillis() - 3L * 60L * 60L * 1000L) { - this.warnings.put("The last known consensus published at " - + dateTimeFormat.format(consensus.getValidAfterMillis()) - + " is more than 3 hours old and therefore not valid anymore", - 0L); - } - } - /* Prepare a report to be written to stdout. */ private String preparedReport = null; private void prepareReport() {

1 0

[metrics-web/master] Fix a String encoding problem with OpenJDK.
by karsten＠torproject.org 21 Oct '11

21 Oct '11

commit 8d02e8529ffc757be17dea188d239c333a62eee1 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Oct 20 13:22:07 2011 +0200 Fix a String encoding problem with OpenJDK. --- .../cron/RelayDescriptorDatabaseImporter.java | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diff --git a/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java b/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java index f8f1e9a..4a9f97c 100644 --- a/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java +++ b/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java @@ -496,8 +496,12 @@ public final class RelayDescriptorDatabaseImporter { this.psD.setLong(7, bandwidthAvg); this.psD.setLong(8, bandwidthBurst); this.psD.setLong(9, bandwidthObserved); + /* Remove all non-ASCII characters from the platform string, or + * we'll make Postgres unhappy. Sun's JDK and OpenJDK behave + * differently when creating a new String with a given encoding. + * That's what the regexp below is for. */ this.psD.setString(10, new String(platform.getBytes(), - "US-ASCII")); + "US-ASCII").replaceAll("[^\\p{ASCII}]","")); this.psD.setTimestamp(11, new Timestamp(published), cal); this.psD.setLong(12, uptime); this.psD.setString(13, extraInfoDigest);

1 0

[metrics-web/master] Download consensuses from different authorities in parallel.
by karsten＠torproject.org 21 Oct '11

21 Oct '11

commit f5eceb840d660595dea66b25510e9b934b0e908f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Oct 20 18:33:45 2011 +0200 Download consensuses from different authorities in parallel. --- src/org/torproject/chc/Downloader.java | 84 +++++++++++++++++++++++-------- 1 files changed, 62 insertions(+), 22 deletions(-) diff --git a/src/org/torproject/chc/Downloader.java b/src/org/torproject/chc/Downloader.java index 17ae71c..e56d20d 100644 --- a/src/org/torproject/chc/Downloader.java +++ b/src/org/torproject/chc/Downloader.java @@ -40,20 +40,28 @@ public class Downloader { private SortedMap<String, String> downloadedConsensuses = new TreeMap<String, String>(); private void downloadConsensus() { + Map<String, String> urls = new HashMap<String, String>(); for (Map.Entry<String, String> e : this.authorities.entrySet()) { String nickname = e.getKey(); String address = e.getValue(); String resource = "/tor/status-vote/current/consensus.z"; String fullUrl = "http://" + address + resource; - String response = this.downloadFromAuthority(fullUrl); - if (response != null) { + urls.put(nickname, fullUrl); + } + Map<String, String> responses = + this.downloadFromAuthority(new HashSet<String>(urls.values())); + for (Map.Entry<String, String> e : urls.entrySet()) { + String nickname = e.getKey(); + String url = e.getValue(); + if (responses.containsKey(url)) { + String response = responses.get(url); this.downloadedConsensuses.put(nickname, response); } else { System.err.println("Could not download consensus from directory " + "authority " + nickname + ". Ignoring."); } } - if (this.downloadedConsensuses.isEmpty()) { + if (responses.isEmpty()) { System.err.println("Could not download consensus from any of the " + "directory authorities. Ignoring."); } @@ -65,14 +73,14 @@ public class Downloader { Thread mainThread; String url; String response; - boolean interrupted = false; + boolean finished = false; public DownloadRunnable(String url) { this.mainThread = Thread.currentThread(); this.url = url; } public void run() { try { - URL u = new URL(url); + URL u = new URL(this.url); HttpURLConnection huc = (HttpURLConnection) u.openConnection(); huc.setRequestMethod("GET"); huc.connect(); @@ -83,37 +91,65 @@ public class Downloader { ByteArrayOutputStream baos = new ByteArrayOutputStream(); int len; byte[] data = new byte[1024]; - while (!this.interrupted && + while (!this.finished && (len = in.read(data, 0, 1024)) >= 0) { baos.write(data, 0, len); } - if (this.interrupted) { + if (this.finished) { return; } in.close(); byte[] allData = baos.toByteArray(); this.response = new String(allData); + this.finished = true; this.mainThread.interrupt(); } } catch (IOException e) { /* Can't do much except leaving this.response at null. */ } + this.finished = true; } } - /* Download a consensus or vote from a directory authority using a - * timeout of 60 seconds. */ - private String downloadFromAuthority(final String url) { - DownloadRunnable downloadRunnable = new DownloadRunnable(url); - new Thread(downloadRunnable).start(); - try { - Thread.sleep(60L * 1000L); - } catch (InterruptedException e) { - /* Do nothing. */ + /* Download one or more consensuses or votes from one or more directory + * authorities using a timeout of 60 seconds. */ + private Map<String, String> downloadFromAuthority(Set<String> urls) { + Set<DownloadRunnable> downloadRunnables = + new HashSet<DownloadRunnable>(); + for (String url : urls) { + DownloadRunnable downloadRunnable = new DownloadRunnable(url); + downloadRunnables.add(downloadRunnable); + new Thread(downloadRunnable).start(); + } + long started = System.currentTimeMillis(), sleep; + while ((sleep = started + 60L * 1000L - System.currentTimeMillis()) + > 0L) { + try { + Thread.sleep(sleep); + } catch (InterruptedException e) { + /* Do nothing. */ + } + boolean unfinished = false; + for (DownloadRunnable downloadRunnable : downloadRunnables) { + if (!downloadRunnable.finished) { + unfinished = true; + break; + } + } + if (!unfinished) { + break; + } } - String response = downloadRunnable.response; - downloadRunnable.interrupted = true; - return response; + Map<String, String> responses = new HashMap<String, String>(); + for (DownloadRunnable downloadRunnable : downloadRunnables) { + String url = downloadRunnable.url; + String response = downloadRunnable.response; + if (response != null) { + responses.put(url, response); + } + downloadRunnable.finished = true; + } + return responses; } /* Date-time formats to parse and format timestamps. */ @@ -125,7 +161,7 @@ public class Downloader { /* Parse the downloaded consensus to find fingerprints of directory * authorities publishing the corresponding votes. */ - private List<String> fingerprints = new ArrayList<String>(); + private SortedSet<String> fingerprints = new TreeSet<String>(); private void parseConsensusToFindReferencedVotes() { for (String downloadedConsensus : this.downloadedConsensuses.values()) { @@ -189,8 +225,12 @@ public class Downloader { String resource = "/tor/status-vote/current/" + fingerprint + ".z"; String fullUrl = "http://" + authority + resource; - downloadedVote = this.downloadFromAuthority(fullUrl); - if (downloadedVote != null) { + Set<String> urls = new HashSet<String>(); + urls.add(fullUrl); + Map<String, String> downloadedVotes = + this.downloadFromAuthority(urls); + if (downloadedVotes.containsKey(fullUrl)) { + downloadedVote = downloadedVotes.get(fullUrl); this.downloadedVotes.add(downloadedVote); } }

1 0