- tor-commits - lists.torproject.org

[tor/master] Abort writing cached-microdescs if a failed write has occurred.
by nickm＠torproject.org 31 May '12

31 May '12

commit 0da40bba8858337c27b4ca0e40e0d9be00df9131 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu May 24 11:07:01 2012 -0400 Abort writing cached-microdescs if a failed write has occurred. Bug 2954; fix on 0.2.2.6-alpha. --- changes/bug2954_more | 4 ++++ src/or/microdesc.c | 14 ++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/changes/bug2954_more b/changes/bug2954_more new file mode 100644 index 0000000..68fdf13 --- /dev/null +++ b/changes/bug2954_more @@ -0,0 +1,4 @@ + o Minor bugfixes: + - If we fail to write a microdescriptor to disk, do not continue + replacing the old microdescriptor file. Fix for bug 2954; bugfix + on 0.2.2.6-alpha. diff --git a/src/or/microdesc.c b/src/or/microdesc.c index be9b997..59b7d7b 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -80,7 +80,12 @@ dump_microdescriptor(FILE *f, microdesc_t *md, size_t *annotation_len_out) char annotation[ISO_TIME_LEN+32]; format_iso_time(buf, md->last_listed); tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf); - fputs(annotation, f); + if (fputs(annotation, f) < 0) { + log_warn(LD_DIR, + "Couldn't write microdescriptor annotation: %s", + strerror(ferror(f))); + return -1; + } r += strlen(annotation); *annotation_len_out = r; } else { @@ -226,9 +231,10 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, size_t annotation_len; size = dump_microdescriptor(f, md, &annotation_len); if (size < 0) { - /* XXX handle errors from dump_microdescriptor() */ - /* log? return -1? die? coredump the universe? */ - continue; + /* we already warned in dump_microdescriptor; */ + abort_writing_to_file(open_file); + smartlist_clear(added); + return added; } md->saved_location = SAVED_IN_JOURNAL; cache->journal_len += size;

1 0

[tor/master] Merge remote-tracking branch 'public/bug2954_more'
by nickm＠torproject.org 31 May '12

31 May '12

commit 11bf5585aafa4ea590b4f4b62a75323fb5907d95 Merge: 32d6aca 0da40bb Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu May 31 12:22:02 2012 -0400 Merge remote-tracking branch 'public/bug2954_more' changes/bug2954_more | 4 ++++ src/or/microdesc.c | 14 ++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-)

1 0

[ooni-probe/master] Write test case for OONITest
by art＠torproject.org 31 May '12

31 May '12

commit eb6a0f732a60f2d340de758deebf67781ae1358e Author: Arturo Filastò <hellais(a)torproject.org> Date: Thu May 31 17:43:20 2012 +0200 Write test case for OONITest * Make the test result conform to the spec --- docs/writing_tests.txt | 12 ------------ ooni/plugoo/tests.py | 2 +- tests/test_tests.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/docs/writing_tests.txt b/docs/writing_tests.txt deleted file mode 100644 index 1692f82..0000000 --- a/docs/writing_tests.txt +++ /dev/null @@ -1,12 +0,0 @@ -There are two modes of writing tests. - -The first kind of mechanism relied of writing a test that uses blocking code -and each instance of it is run inside of a separate thread. Beware that your -blocking code must be thread safe to run properly (as is obvious :P) - -The other menthod invloves having some knowledge about twisted. The test you -will write will be written in twisted and should implement async style non -blocking architecture. - -It is recommended that tests are written using the second pattern and the first -should only be used for runnign tests that have been previously written. diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 7bb2321..8e0ab8e 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -44,7 +44,7 @@ class OONITest(object): result['start_time'] = self.start_time result['end_time'] = self.end_time result['run_time'] = self.end_time - self.start_time - result['control'] = control + result['result'] = control log.msg("FINISHED %s" % result) self.report(result) return result diff --git a/tests/test_tests.py b/tests/test_tests.py new file mode 100644 index 0000000..702e6a3 --- /dev/null +++ b/tests/test_tests.py @@ -0,0 +1,33 @@ +from twisted.trial import unittest + +from ooni.plugoo import work, tests + +class TestsTestCase(unittest.TestCase): + def setUp(self): + class dummyReport: + def __call__(self, *args, **kw): + pass + self.dummyreport = dummyReport() + self.callbackResults = None + self.errbackResults = None + + def _callback(self, *args, **kw): + self.callbackResults = args, kw + + def _errback(self, *args, **kw): + pass + + def test_fallThrough(self): + """ + This tests to make sure that what is returned from the experiment + method falls all the way through to control and finish. + """ + test_dict = {"hello": "world"} + class DummyTest(tests.OONITest): + def experiment(self, args): + return test_dict + + test = DummyTest(None, None, self.dummyreport) + test.startTest(None).addCallback(self._callback) + self.assertEqual(self.callbackResults[0][0]['result'], test_dict) +

1 0

[metrics-web/master] Don't attempt to parse a correct @type gettor line.
by karsten＠torproject.org 31 May '12

31 May '12

commit d077b20e9bbbb3e0c20a25df81c40ee6aa44e417 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu May 31 16:37:17 2012 +0200 Don't attempt to parse a correct @type gettor line. --- src/org/torproject/ernie/cron/GetTorProcessor.java | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/src/org/torproject/ernie/cron/GetTorProcessor.java b/src/org/torproject/ernie/cron/GetTorProcessor.java index 0be6e12..a9ce00b 100644 --- a/src/org/torproject/ernie/cron/GetTorProcessor.java +++ b/src/org/torproject/ernie/cron/GetTorProcessor.java @@ -49,6 +49,7 @@ public class GetTorProcessor { + "Aborting."); break; } + continue; } String[] parts = line.split(" "); String date = parts[0];

1 0

[metrics-web/master] Fix broken HTML...
by karsten＠torproject.org 31 May '12

31 May '12

commit c1ac9e2e0e9b715fde1002f16b6dc7b3899c46a6 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu May 31 16:19:43 2012 +0200 Fix broken HTML... --- web/WEB-INF/data.jsp | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/WEB-INF/data.jsp b/web/WEB-INF/data.jsp index e8fc449..a1d0b49 100644 --- a/web/WEB-INF/data.jsp +++ b/web/WEB-INF/data.jsp @@ -284,8 +284,8 @@ file</a></h3> <br> <p>GetTor allows users to fetch the Tor software via email. - This <a href="data/gettor_stats.txt">file contains statistics on - the number of packages requested from GetTor per day. + This <a href="data/gettor_stats.txt">file</a> contains statistics + on the number of packages requested from GetTor per day. The data format is described <a href="formats.html#gettor">here</a>.</p> </div>

1 0

[metrics-web/master] The GetTor stats file now has a @type annotation, too.
by karsten＠torproject.org 31 May '12

31 May '12

commit bc1e8763d0119cd636ef857960c08ecf5beae7dc Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu May 31 16:18:25 2012 +0200 The GetTor stats file now has a @type annotation, too. --- web/WEB-INF/formats.jsp | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/web/WEB-INF/formats.jsp b/web/WEB-INF/formats.jsp index 9ac7741..5fbffab 100644 --- a/web/WEB-INF/formats.jsp +++ b/web/WEB-INF/formats.jsp @@ -86,7 +86,7 @@ are recent: <li><tt>@type bridge-extra-info 1.0</tt></li> <li><tt>@type torperf 1.0</tt></li> <li><tt>@type bridge-pool-assignment 1.0</tt></li> -<li><tt><font color="gray">@type gettor 1.0</font></tt></li> +<li><tt>@type gettor 1.0</tt></li> <li><tt><font color="gray">@type tordnsel 1.0</font></tt></li> </ul>

1 0

[metrics-web/master] Add GetTor stats file to data.html.
by karsten＠torproject.org 31 May '12

31 May '12

commit fee0d324da5a82b84d50a21a6bafca43e2df1c71 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu May 31 16:17:48 2012 +0200 Add GetTor stats file to data.html. --- web/WEB-INF/data.jsp | 11 +++++++++++ 1 files changed, 11 insertions(+), 0 deletions(-) diff --git a/web/WEB-INF/data.jsp b/web/WEB-INF/data.jsp index 06176db..e8fc449 100644 --- a/web/WEB-INF/data.jsp +++ b/web/WEB-INF/data.jsp @@ -28,6 +28,7 @@ <li><a href="#bridgeassignments">Bridge pool assignments</a></li> <li><a href="#performance">Performance data</a></li> <li><a href="#exitlist">Exit lists</a></li> + <li><a href="#gettor">GetTor statistics file</a></li> </ul> <p>The tarballs listed on this page and the raw files that were published on the last three days are also available via @@ -277,6 +278,16 @@ </tr> </c:forEach> </table> + <br> + <a name="gettor"></a> + <h3><a href="#gettor" class="anchor">GetTor statistics + file</a></h3> + <br> + <p>GetTor allows users to fetch the Tor software via email. + This <a href="data/gettor_stats.txt">file contains statistics on + the number of packages requested from GetTor per day. + The data format is described + <a href="formats.html#gettor">here</a>.</p> </div> </div> <div class="bottom" id="bottom">

1 0

[ooni-probe/master] Rename TwistedTest to OONITest
by art＠torproject.org 31 May '12

31 May '12

commit eb3f925af7baeeee8330e610797337aaf7423ee0 Author: Arturo Filastò <hellais(a)torproject.org> Date: Thu May 31 15:56:53 2012 +0200 Rename TwistedTest to OONITest --- ooni/plugins/blocking.py | 4 ++-- ooni/plugins/httphost.py | 4 ++-- ooni/plugins/skel.py | 4 ++-- ooni/plugoo/tests.py | 4 ++-- ooni/scaffolding.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py index 4638834..b6b63b6 100644 --- a/ooni/plugins/blocking.py +++ b/ooni/plugins/blocking.py @@ -3,14 +3,14 @@ from twisted.python import usage from twisted.plugin import IPlugin from ooni.plugoo.assets import Asset -from ooni.plugoo.tests import ITest, TwistedTest +from ooni.plugoo.tests import ITest, OONITest class BlockingArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], ['resume', 'r', 0, 'Resume at this index'], ['shit', 'o', None, 'Other arguments']] -class BlockingTest(TwistedTest): +class BlockingTest(OONITest): implements(IPlugin, ITest) shortName = "blocking" diff --git a/ooni/plugins/httphost.py b/ooni/plugins/httphost.py index 9b4de22..2c7f8af 100644 --- a/ooni/plugins/httphost.py +++ b/ooni/plugins/httphost.py @@ -19,7 +19,7 @@ from twisted.python import usage from twisted.plugin import IPlugin from ooni.plugoo.assets import Asset -from ooni.plugoo.tests import ITest, TwistedTest +from ooni.plugoo.tests import ITest, OONITest class HTTPHostArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], @@ -49,7 +49,7 @@ class HTTPHostAsset(Asset): return line.split(',')[1].replace('\n','') -class HTTPHostTest(TwistedTest): +class HTTPHostTest(OONITest): implements(IPlugin, ITest) shortName = "httphost" diff --git a/ooni/plugins/skel.py b/ooni/plugins/skel.py index c5eec47..de95b48 100644 --- a/ooni/plugins/skel.py +++ b/ooni/plugins/skel.py @@ -1,7 +1,7 @@ from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin -from plugoo.tests import ITest, TwistedTest +from plugoo.tests import ITest, OONITest from ooni import log class SkelArgs(usage.Options): @@ -9,7 +9,7 @@ class SkelArgs(usage.Options): ['resume', 'r', 0, 'Resume at this index'], ['other', 'o', None, 'Other arguments']] -class SkelTest(TwistedTest): +class SkelTest(OONITest): implements(IPlugin, ITest) shortName = "skeleton" diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 3f63476..7bb2321 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -16,7 +16,7 @@ from ooni.plugoo.reports import Report from ooni.plugoo.interface import ITest -class TwistedTest(object): +class OONITest(object): blocking = False def __init__(self, local_options, global_options, report, ooninet=None): @@ -34,7 +34,7 @@ class TwistedTest(object): return {'asset': None} def __repr__(self): - return "<TwistedTest %s %s %s>" % (self.options, self.global_options, + return "<OONITest %s %s %s>" % (self.options, self.global_options, self.assets) def finished(self, control): diff --git a/ooni/scaffolding.py b/ooni/scaffolding.py index b1d3f59..9004754 100755 --- a/ooni/scaffolding.py +++ b/ooni/scaffolding.py @@ -15,13 +15,13 @@ Safe hacking :). from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin -from plugoo.tests import ITest, TwistedTest +from plugoo.tests import ITest, OONITest class %(testName)sArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], ['resume', 'r', 0, 'Resume at this index']] -class %(testName)sTest(TwistedTest): +class %(testName)sTest(OONITest): implements(IPlugin, ITest) shortName = "%(testShortname)s"

1 0

[metrics-lib/master] Parse exit lists with @type annotation and Downloaded line.
by karsten＠torproject.org 31 May '12

31 May '12

commit 194768b33ffd36a3690a714cdbccb3b27a762fb5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu May 31 16:00:09 2012 +0200 Parse exit lists with @type annotation and Downloaded line. --- src/org/torproject/descriptor/ExitList.java | 4 ++-- .../torproject/descriptor/impl/DescriptorImpl.java | 3 ++- .../torproject/descriptor/impl/ExitListImpl.java | 20 ++++++++++++-------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/org/torproject/descriptor/ExitList.java b/src/org/torproject/descriptor/ExitList.java index 5372b8b..dc4a1f9 100644 --- a/src/org/torproject/descriptor/ExitList.java +++ b/src/org/torproject/descriptor/ExitList.java @@ -7,8 +7,8 @@ import java.util.Set; /* Exit list containing all known exit scan results at a given time. */ public interface ExitList extends Descriptor { - /* Return the publication time of the exit list. */ - public long getPublishedMillis(); + /* Return the download time of the exit list. */ + public long getDownloadedMillis(); /* Return the unordered set of exit scan results. */ public Set<ExitListEntry> getExitListEntries(); diff --git a/src/org/torproject/descriptor/impl/DescriptorImpl.java b/src/org/torproject/descriptor/impl/DescriptorImpl.java index 00a4b16..c1c4a9a 100644 --- a/src/org/torproject/descriptor/impl/DescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/DescriptorImpl.java @@ -70,7 +70,8 @@ public abstract class DescriptorImpl implements Descriptor { parsedDescriptors.addAll(DirectoryKeyCertificateImpl. parseDescriptors(rawDescriptorBytes, failUnrecognizedDescriptorLines)); - } else if (firstLines.startsWith("ExitNode ") || + } else if (firstLines.startsWith("@type tordnsel 1.") || + firstLines.startsWith("ExitNode ") || firstLines.contains("\nExitNode ")) { parsedDescriptors.add(new ExitListImpl(rawDescriptorBytes, fileName, failUnrecognizedDescriptorLines)); diff --git a/src/org/torproject/descriptor/impl/ExitListImpl.java b/src/org/torproject/descriptor/impl/ExitListImpl.java index 277112f..8683cce 100644 --- a/src/org/torproject/descriptor/impl/ExitListImpl.java +++ b/src/org/torproject/descriptor/impl/ExitListImpl.java @@ -21,24 +21,25 @@ public class ExitListImpl extends DescriptorImpl implements ExitList { boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { super(rawDescriptorBytes, failUnrecognizedDescriptorLines, false); - this.setPublishedMillisFromFileName(fileName); this.splitAndParseExitListEntries(rawDescriptorBytes); + this.setPublishedMillisFromFileName(fileName); } private void setPublishedMillisFromFileName(String fileName) throws DescriptorParseException { - if (fileName.length() == "2012-02-01-04-06-24".length()) { + if (this.downloadedMillis == 0L && + fileName.length() == "2012-02-01-04-06-24".length()) { try { SimpleDateFormat fileNameFormat = new SimpleDateFormat( "yyyy-MM-dd-HH-mm-ss"); fileNameFormat.setLenient(false); fileNameFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - this.publishedMillis = fileNameFormat.parse(fileName).getTime(); + this.downloadedMillis = fileNameFormat.parse(fileName).getTime(); } catch (ParseException e) { /* Handle below. */ } } - if (this.publishedMillis == 0L) { + if (this.downloadedMillis == 0L) { throw new DescriptorParseException("Unrecognized exit list file " + "name '" + fileName + "'."); } @@ -56,7 +57,10 @@ public class ExitListImpl extends DescriptorImpl implements ExitList { String line = s.next(); String[] parts = line.split(" "); String keyword = parts[0]; - if (keyword.equals("ExitNode")) { + if (keyword.equals("Downloaded")) { + this.downloadedMillis = ParseHelper.parseTimestampAtIndex(line, + parts, 1, 2); + } else if (keyword.equals("ExitNode")) { sb = new StringBuilder(); sb.append(line + "\n"); } else if (keyword.equals("Published")) { @@ -94,9 +98,9 @@ public class ExitListImpl extends DescriptorImpl implements ExitList { } } - private long publishedMillis; - public long getPublishedMillis() { - return this.publishedMillis; + private long downloadedMillis; + public long getDownloadedMillis() { + return this.downloadedMillis; } private Set<ExitListEntry> exitListEntries =

1 0

[metrics-db/master] Add @type annotation and Downloaded line to exit lists.
by karsten＠torproject.org 31 May '12

31 May '12

commit b920fb0798759d063786762eac3a5f8c7e7df617 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu May 31 15:44:52 2012 +0200 Add @type annotation and Downloaded line to exit lists. --- .../torproject/ernie/db/ExitListDownloader.java | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-) diff --git a/src/org/torproject/ernie/db/ExitListDownloader.java b/src/org/torproject/ernie/db/ExitListDownloader.java index 82bde11..01a554f 100644 --- a/src/org/torproject/ernie/db/ExitListDownloader.java +++ b/src/org/torproject/ernie/db/ExitListDownloader.java @@ -40,11 +40,18 @@ public class ExitListDownloader { SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss"); printFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + Date downloadedDate = new Date(); File exitListFile = new File("exitlist/" + printFormat.format( - new Date())); + downloadedDate)); exitListFile.getParentFile().mkdirs(); + SimpleDateFormat dateTimeFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); BufferedWriter bw = new BufferedWriter(new FileWriter( exitListFile)); + bw.write("@type tordnsel 1.0\n"); + bw.write("Downloaded " + dateTimeFormat.format(downloadedDate) + + "\n"); int len; byte[] data = new byte[1024]; while ((len = in.read(data, 0, 1024)) >= 0) {

1 0