- tor-commits - lists.torproject.org

[metrics-lib/master] Add Onionoo to list of software using metrics-lib.
by karsten＠torproject.org 21 Mar '12

21 Mar '12

commit 60d8ce92448f648708866182df2e6d6c5efc8186 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Mar 21 11:23:57 2012 +0100 Add Onionoo to list of software using metrics-lib. --- README | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/README b/README index f192501..93c2cee 100644 --- a/README +++ b/README @@ -33,4 +33,5 @@ during the initial development phase, ask for your software to be added to the list below (software name, repository, contact person): - DocTor, https://git.torproject.org/doctor.git, karsten + - Onionoo, https://git.torproject.org/onionoo.git, karsten

1 0

[metrics-web/master] Make R object generation thread-safe.
by karsten＠torproject.org 21 Mar '12

21 Mar '12

commit b0b2f22e160750f9c7acc9de509d2467d3b5c572 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Mar 21 10:53:55 2012 +0100 Make R object generation thread-safe. Problems with concurrent R object generation were highly unlikely before this patch. But that makes them even harder to track down. Now, we're generating all R objects in a separate thread that we join. Before starting a new thread we check if there's already a thread running to generate the same object, and if so, we join that one. --- src/org/torproject/ernie/web/RObjectGenerator.java | 200 +++++++++++--------- 1 files changed, 108 insertions(+), 92 deletions(-) diff --git a/src/org/torproject/ernie/web/RObjectGenerator.java b/src/org/torproject/ernie/web/RObjectGenerator.java index 011eccd..3482dce 100644 --- a/src/org/torproject/ernie/web/RObjectGenerator.java +++ b/src/org/torproject/ernie/web/RObjectGenerator.java @@ -4,11 +4,13 @@ package org.torproject.ernie.web; import java.io.BufferedInputStream; import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileReader; import java.io.IOException; +import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -181,48 +183,11 @@ public class RObjectGenerator implements ServletContextListener { /* See if we need to generate this graph. */ File imageFile = new File(this.cachedGraphsDirectory + "/" + imageFilename); - long now = System.currentTimeMillis(); - if (!checkCache || !imageFile.exists() || - imageFile.lastModified() < now - this.maxCacheAge * 1000L) { - - /* We do. Update the R query to contain the absolute path to the - * file to be generated, create a connection to Rserve, run the R - * query, and close the connection. The generated graph will be on - * disk. */ - rQuery = String.format(rQuery, imageFile.getAbsolutePath()); - try { - RConnection rc = new RConnection(rserveHost, rservePort); - rc.eval(rQuery); - rc.close(); - } catch (RserveException e) { - return null; - } - - /* Check that we really just generated the file */ - if (!imageFile.exists() || imageFile.lastModified() < now - - this.maxCacheAge * 1000L) { - return null; - } - } - - /* Read the image from disk and write it to a byte array. */ - byte[] result = null; - try { - BufferedInputStream bis = new BufferedInputStream( - new FileInputStream(imageFile), 1024); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - byte[] buffer = new byte[1024]; - int length; - while ((length = bis.read(buffer)) > 0) { - baos.write(buffer, 0, length); - } - result = baos.toByteArray(); - } catch (IOException e) { - return null; - } + byte[] imageBytes = this.generateRObject(rQuery, imageFile, + checkCache); /* Return the graph bytes. */ - return result; + return imageBytes; } public SortedSet<String> getAvailableCsvFiles() { @@ -245,35 +210,14 @@ public class RObjectGenerator implements ServletContextListener { /* See if we need to generate this .csv file. */ File csvFile = new File(this.cachedGraphsDirectory + "/" + csvFilename); - long now = System.currentTimeMillis(); - if (!checkCache || !csvFile.exists() || - csvFile.lastModified() < now - this.maxCacheAge * 1000L) { - - /* We do. Update the R query to contain the absolute path to the - * file to be generated, create a connection to Rserve, run the R - * query, and close the connection. The generated csv file will be - * on disk in the same directory as the generated graphs. */ - rQuery = String.format(rQuery, csvFile.getAbsolutePath()); - try { - RConnection rc = new RConnection(rserveHost, rservePort); - rc.eval(rQuery); - rc.close(); - } catch (RserveException e) { - return null; - } - - /* Check that we really just generated the file */ - if (!csvFile.exists() || csvFile.lastModified() < now - - this.maxCacheAge * 1000L) { - return null; - } - } + byte[] csvBytes = this.generateRObject(rQuery, csvFile, checkCache); /* Read the text file from disk and write it to a string. */ String result = null; try { StringBuilder sb = new StringBuilder(); - BufferedReader br = new BufferedReader(new FileReader(csvFile)); + BufferedReader br = new BufferedReader(new InputStreamReader( + new ByteArrayInputStream(csvBytes))); String line = null; while ((line = br.readLine()) != null) { sb.append(line + "\n"); @@ -283,7 +227,7 @@ public class RObjectGenerator implements ServletContextListener { return null; } - /* Return the csv file. */ + /* Return the csv file content. */ return result; } @@ -343,36 +287,15 @@ public class RObjectGenerator implements ServletContextListener { /* See if we need to generate this table. */ File tableFile = new File(this.cachedGraphsDirectory + "/" + tableFilename); - long now = System.currentTimeMillis(); - if (!checkCache || !tableFile.exists() || - tableFile.lastModified() < now - this.maxCacheAge * 1000L) { - - /* We do. Update the R query to contain the absolute path to the - * file to be generated, create a connection to Rserve, run the R - * query, and close the connection. The generated csv file will be - * on disk in the same directory as the generated graphs. */ - rQuery = String.format(rQuery, tableFile.getAbsolutePath()); - try { - RConnection rc = new RConnection(rserveHost, rservePort); - rc.eval(rQuery); - rc.close(); - } catch (RserveException e) { - return null; - } + byte[] tableBytes = this.generateRObject(rQuery, tableFile, + checkCache); - /* Check that we really just generated the file */ - if (!tableFile.exists() || tableFile.lastModified() < now - - this.maxCacheAge * 1000L) { - return null; - } - } - - /* Read the text file from disk and write the table content to a - * map. */ + /* Write the table content to a map. */ List<Map<String, String>> result = null; try { result = new ArrayList<Map<String, String>>(); - BufferedReader br = new BufferedReader(new FileReader(tableFile)); + BufferedReader br = new BufferedReader(new InputStreamReader( + new ByteArrayInputStream(tableBytes))); String line = br.readLine(); if (line != null) { List<String> headers = new ArrayList<String>(Arrays.asList( @@ -396,5 +319,98 @@ public class RObjectGenerator implements ServletContextListener { /* Return table values. */ return result; } -} + /* Generate an R object in a separate worker thread, or wait for an + * already running worker thread to finish and get its result. */ + private byte[] generateRObject(String rQuery, File rObjectFile, + boolean checkCache) { + RObjectGeneratorWorker worker = null; + synchronized (this.rObjectGeneratorThreads) { + if (this.rObjectGeneratorThreads.containsKey(rQuery)) { + worker = this.rObjectGeneratorThreads.get(rQuery); + } else { + worker = new RObjectGeneratorWorker(rQuery, rObjectFile, + checkCache); + this.rObjectGeneratorThreads.put(rQuery, worker); + worker.start(); + } + } + try { + worker.join(); + } catch (InterruptedException e) { + } + synchronized (this.rObjectGeneratorThreads) { + if (this.rObjectGeneratorThreads.containsKey(rQuery) && + this.rObjectGeneratorThreads.get(rQuery) == worker) { + this.rObjectGeneratorThreads.remove(rQuery); + } + } + return worker.getRObjectBytes(); + } + + private Map<String, RObjectGeneratorWorker> rObjectGeneratorThreads = + new HashMap<String, RObjectGeneratorWorker>(); + + private class RObjectGeneratorWorker extends Thread { + + private String rQuery; + private File rObjectFile; + private boolean checkCache; + private byte[] result = null; + + public RObjectGeneratorWorker(String rQuery, File rObjectFile, + boolean checkCache) { + this.rQuery = rQuery; + this.rObjectFile = rObjectFile; + this.checkCache = checkCache; + } + + public void run() { + + /* See if we need to generate this R object. */ + long now = System.currentTimeMillis(); + if (!this.checkCache || !this.rObjectFile.exists() || + this.rObjectFile.lastModified() < now - maxCacheAge * 1000L) { + + /* We do. Update the R query to contain the absolute path to the + * file to be generated, create a connection to Rserve, run the R + * query, and close the connection. The generated object will be + * on disk. */ + this.rQuery = String.format(this.rQuery, + this.rObjectFile.getAbsolutePath()); + try { + RConnection rc = new RConnection(rserveHost, rservePort); + rc.eval(this.rQuery); + rc.close(); + } catch (RserveException e) { + return; + } + + /* Check that we really just generated the R object. */ + if (!this.rObjectFile.exists() || this.rObjectFile.lastModified() + < now - maxCacheAge * 1000L) { + return; + } + } + + /* Read the R object from disk and write it to a byte array. */ + try { + BufferedInputStream bis = new BufferedInputStream( + new FileInputStream(this.rObjectFile), 1024); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int length; + while ((length = bis.read(buffer)) > 0) { + baos.write(buffer, 0, length); + } + this.result = baos.toByteArray(); + } catch (IOException e) { + return; + } + } + + public byte[] getRObjectBytes() { + return result; + } + } +}

1 0

[ooni-probe/master] Move REAMDE to top level directory
by art＠torproject.org 21 Mar '12

21 Mar '12

commit 15c33719cf6dfea1b4482a9436d83820d0dfbf88 Author: Arturo Filastò <hellais(a)gmail.com> Date: Tue Mar 20 20:58:33 2012 -0700 Move REAMDE to top level directory --- README | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ old/README | 57 --------------------------------------------------------- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/README b/README new file mode 100644 index 0000000..476aa40 --- /dev/null +++ b/README @@ -0,0 +1,57 @@ +ooni-probe - Open Observatory of Network Interference + +"The Net interprets censorship as damage and routes around it." + - John Gilmore; TIME magazine (6 December 1993) + +The goal of ooni-probe is to collect data about censorship around +the world. + +OONI uses open methodologies and the data will be provided in raw +format to allow any researcher to indipendently draw their conclusions +from the results OONI tests. + +There are currently projects aimed at measuring censorship in one +way or another but they either use non open methodologies or their +tools are not open sources. OONI aims at filling up this gap by +creating the first open source framework for developing network +tests and collecting data on censorship. + +OONI revolves around three major concepts: Assets, Tests and +Reports. + +# Assets + +Assets are the inputs used inside Tests to detect censorship events. +These can be URL lists, keywords, ip addresses, packets or any kind +of set of data. +In the python specific implementation this is represented as a python +iterable object. This means that the Testing framework will be able +to iterate through every element in the Asset. + +# Tests + +This is the core of OONI. These are the actual tests that will be run +using as input (if an input is required) the Assets. +Tests can be summarized as an experiment and a control. The control +represents the expected result and the experiment is the network operation +being performed on the live network. If the experiment does not match up +with the control then a censorship event had occured. + +OONI probe provides some useful functionality to the application developer +that may be useful when developing censorship detection tests. For example +it is possible to make a request over the Tor network easily or use a fast +and flexible non-blocking HTTP client implementation. + +# Reports + +This is the data that is collected from the test. OONI probe provides a +flexible means of storing results and uploading this data to a remote +server or a flat file. + +The Test developer should include in the report as much data as possible +and can contain raw packet dumps as well as structured synthetic results. + +In future on top of ooni-probe Reports it will be possible to develop +flexible post-processing tools to allow data-visualization guru's to +properly visualize and contextualize the resulting data. + diff --git a/old/README b/old/README deleted file mode 100644 index 476aa40..0000000 --- a/old/README +++ /dev/null @@ -1,57 +0,0 @@ -ooni-probe - Open Observatory of Network Interference - -"The Net interprets censorship as damage and routes around it." - - John Gilmore; TIME magazine (6 December 1993) - -The goal of ooni-probe is to collect data about censorship around -the world. - -OONI uses open methodologies and the data will be provided in raw -format to allow any researcher to indipendently draw their conclusions -from the results OONI tests. - -There are currently projects aimed at measuring censorship in one -way or another but they either use non open methodologies or their -tools are not open sources. OONI aims at filling up this gap by -creating the first open source framework for developing network -tests and collecting data on censorship. - -OONI revolves around three major concepts: Assets, Tests and -Reports. - -# Assets - -Assets are the inputs used inside Tests to detect censorship events. -These can be URL lists, keywords, ip addresses, packets or any kind -of set of data. -In the python specific implementation this is represented as a python -iterable object. This means that the Testing framework will be able -to iterate through every element in the Asset. - -# Tests - -This is the core of OONI. These are the actual tests that will be run -using as input (if an input is required) the Assets. -Tests can be summarized as an experiment and a control. The control -represents the expected result and the experiment is the network operation -being performed on the live network. If the experiment does not match up -with the control then a censorship event had occured. - -OONI probe provides some useful functionality to the application developer -that may be useful when developing censorship detection tests. For example -it is possible to make a request over the Tor network easily or use a fast -and flexible non-blocking HTTP client implementation. - -# Reports - -This is the data that is collected from the test. OONI probe provides a -flexible means of storing results and uploading this data to a remote -server or a flat file. - -The Test developer should include in the report as much data as possible -and can contain raw packet dumps as well as structured synthetic results. - -In future on top of ooni-probe Reports it will be possible to develop -flexible post-processing tools to allow data-visualization guru's to -properly visualize and contextualize the resulting data. -

1 0

r25563: {website} update the mirrors table. (website/trunk/include)
by Andrew Lewman 21 Mar '12

21 Mar '12

Author: phobos Date: 2012-03-21 03:36:34 +0000 (Wed, 21 Mar 2012) New Revision: 25563 Modified: website/trunk/include/mirrors-table.wmi Log: update the mirrors table. Modified: website/trunk/include/mirrors-table.wmi =================================================================== --- website/trunk/include/mirrors-table.wmi 2012-03-21 03:36:14 UTC (rev 25562) +++ website/trunk/include/mirrors-table.wmi 2012-03-21 03:36:34 UTC (rev 25563) @@ -1,15 +1,15 @@ <tr> - <td>IS</td> + <td>NL</td> - <td>torproject.is</td> + <td>Amorphis</td> <td>Up to date</td> <td> - </td> - <td><a href="http://torproject.is/dist/">http</a></td> - <td><a href="http://torproject.is/">http</a></td> + <td><a href="http://tor.amorphis.eu/dist/">http</a></td> + <td><a href="http://tor.amorphis.eu/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -18,32 +18,32 @@ <tr> - <td>IT</td> + <td>NL</td> - <td></td> + <td>torproject.nl</td> <td>Up to date</td> <td> - </td> - <td><a href="http://torproject.jcsh.it/dist/">http</a></td> - <td><a href="http://torproject.jcsh.it/">http</a></td> + <td><a href="http://www.torproject.nl/dist/">http</a></td> + <td><a href="http://www.torproject.nl/">http</a></td> + <td><a href="https://www.torproject.nl/dist/">https</a></td> + <td><a href="https://www.torproject.nl/">https</a></td> <td> - </td> <td> - </td> - <td> - </td> - <td> - </td> </tr> <tr> - <td>AT</td> + <td>NL</td> - <td></td> + <td>CCC</td> <td>Up to date</td> <td> - </td> - <td><a href="http://torproject.ph3x.at/dist/">http</a></td> - <td><a href="http://torproject.ph3x.at/">http</a></td> + <td><a href="http://tor.ccc.de/dist/">http</a></td> + <td><a href="http://tor.ccc.de/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -52,15 +52,15 @@ <tr> - <td>AT</td> + <td>DK</td> - <td></td> + <td>Zentrum der Gesundheit</td> <td>Up to date</td> <td> - </td> - <td><a href="http://tor.dont-know-me.at/dist/">http</a></td> - <td><a href="http://tor.dont-know-me.at/">http</a></td> + <td><a href="http://tor.idnr.ws/dist/">http</a></td> + <td><a href="http://tor.idnr.ws/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -69,15 +69,15 @@ <tr> - <td>EU</td> + <td>IT</td> <td></td> <td>Up to date</td> <td> - </td> - <td><a href="http://tor.unregistered.eu/dist/">http</a></td> - <td><a href="http://tor.unregistered.eu/">http</a></td> + <td><a href="http://torproject.jcsh.it/dist/">http</a></td> + <td><a href="http://torproject.jcsh.it/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -86,15 +86,15 @@ <tr> - <td>US</td> + <td>AT</td> - <td>searchprivate</td> + <td></td> <td>Up to date</td> <td> - </td> - <td><a href="http://tor.searchprivate.com/dist/">http</a></td> - <td><a href="http://tor.searchprivate.com/">http</a></td> + <td><a href="http://torproject.ph3x.at/dist/">http</a></td> + <td><a href="http://torproject.ph3x.at/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -103,100 +103,100 @@ <tr> - <td>DE</td> + <td>US</td> - <td>beme it</td> + <td>BarkerJr</td> <td>Up to date</td> <td> - </td> - <td><a href="http://tor.beme-it.de/dist/">http</a></td> - <td><a href="http://tor.beme-it.de/">http</a></td> - <td><a href="https://tor.beme-it.de/dist/">https</a></td> - <td><a href="https://tor.beme-it.de/">https</a></td> + <td><a href="http://www.torproject.us/dist/">http</a></td> + <td><a href="http://www.torproject.us/">http</a></td> + <td><a href="https://www.torproject.us/dist/">https</a></td> + <td><a href="https://www.torproject.us/">https</a></td> <td> - </td> - <td><a href="rsync://tor.beme-it.de/tor">rsync</a></td> + <td><a href="rsync://rsync.torproject.us/tor">rsync</a></td> </tr> <tr> - <td>US</td> + <td>AT</td> - <td>NW Linux</td> + <td></td> <td>Up to date</td> <td> - </td> - <td><a href="http://torproject.nwlinux.us/dist/">http</a></td> - <td><a href="http://torproject.nwlinux.us/">http</a></td> + <td><a href="http://tor.dont-know-me.at/dist/">http</a></td> + <td><a href="http://tor.dont-know-me.at/">http</a></td> <td> - </td> <td> - </td> <td> - </td> - <td><a href="rsync://nwlinux.us/tor-web">rsync</a></td> + <td> - </td> </tr> <tr> <td>DE</td> - <td>borgmann.tv</td> + <td>torservers</td> <td>Up to date</td> <td> - </td> - <td><a href="http://tor.borgmann.tv/dist/">http</a></td> - <td><a href="http://tor.borgmann.tv/">http</a></td> + <td><a href="http://www.torservers.net/mirrors/torproject.org/dist/">http</a></td> + <td><a href="http://www.torservers.net/mirrors/torproject.org/">http</a></td> + <td><a href="https://www.torservers.net/mirrors/torproject.org/dist/">https</a></td> + <td><a href="https://www.torservers.net/mirrors/torproject.org/">https</a></td> <td> - </td> <td> - </td> - <td> - </td> - <td> - </td> </tr> <tr> - <td>IL</td> + <td>IS</td> - <td>Host4site</td> + <td>myRL.net</td> <td>Up to date</td> <td> - </td> - <td><a href="http://mirror.host4site.co.il/torproject.org/dist/">http</a></td> - <td><a href="http://mirror.host4site.co.il/torproject.org/">http</a></td> + <td><a href="http://tor.myrl.net/dist/">http</a></td> + <td><a href="http://tor.myrl.net/">http</a></td> + <td><a href="https://tor.myrl.net/dist/">https</a></td> + <td><a href="https://tor.myrl.net/">https</a></td> <td> - </td> <td> - </td> - <td> - </td> - <td> - </td> </tr> <tr> - <td>GB</td> + <td>US</td> - <td>torland</td> + <td>hessmo</td> <td>Up to date</td> <td> - </td> - <td><a href="http://mirror.torland.me/torproject.org/dist/">http</a></td> - <td><a href="http://mirror.torland.me/torproject.org/">http</a></td> - <td><a href="https://mirror.torland.me/torproject.org/dist/">https</a></td> - <td><a href="https://mirror.torland.me/torproject.org/">https</a></td> + <td><a href="http://mirror.hessmo.com/tor/dist/">http</a></td> + <td><a href="http://mirror.hessmo.com/tor/">http</a></td> <td> - </td> <td> - </td> + <td> - </td> + <td> - </td> </tr> <tr> - <td>IS</td> + <td>LT</td> - <td>TheOnionRouter</td> + <td></td> <td>Up to date</td> <td> - </td> - <td><a href="http://theonionrouter.com/dist/">http</a></td> - <td><a href="http://theonionrouter.com/">http</a></td> + <td><a href="http://tor.vesta.nu/dist/">http</a></td> + <td><a href="http://tor.vesta.nu/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -205,15 +205,15 @@ <tr> - <td>INT</td> + <td>EU</td> - <td>CoralCDN</td> + <td></td> <td>Up to date</td> <td> - </td> - <td><a href="http://www.torproject.org.nyud.net/dist/">http</a></td> - <td><a href="http://www.torproject.org.nyud.net/">http</a></td> + <td><a href="http://tor.unregistered.eu/dist/">http</a></td> + <td><a href="http://tor.unregistered.eu/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -222,49 +222,49 @@ <tr> - <td>NL</td> + <td>FR</td> - <td>torproject.nl</td> + <td>BarkerJr</td> <td>Up to date</td> <td> - </td> - <td><a href="http://www.torproject.nl/dist/">http</a></td> - <td><a href="http://www.torproject.nl/">http</a></td> - <td><a href="https://www.torproject.nl/dist/">https</a></td> - <td><a href="https://www.torproject.nl/">https</a></td> + <td><a href="http://www.oignon.net/dist/">http</a></td> + <td><a href="http://www.oignon.net/">http</a></td> + <td><a href="https://www.oignon.net/dist/">https</a></td> + <td><a href="https://www.oignon.net/">https</a></td> <td> - </td> <td> - </td> </tr> <tr> - <td>NL</td> + <td>AT</td> - <td>CCC</td> + <td>cyberarmy</td> <td>Up to date</td> <td> - </td> - <td><a href="http://tor.ccc.de/dist/">http</a></td> - <td><a href="http://tor.ccc.de/">http</a></td> <td> - </td> + <td><a href="http://tor.cyberarmy.at/">http</a></td> <td> - </td> <td> - </td> <td> - </td> + <td> - </td> </tr> <tr> - <td>EE</td> + <td>US</td> - <td>CyberSIDE</td> + <td>searchprivate</td> <td>Up to date</td> <td> - </td> - <td><a href="http://cyberside.net.ee/tor/">http</a></td> - <td><a href="http://cyberside.planet.ee/tor/">http</a></td> + <td><a href="http://tor.searchprivate.com/dist/">http</a></td> + <td><a href="http://tor.searchprivate.com/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -273,49 +273,49 @@ <tr> - <td>US</td> + <td>DE</td> - <td>BarkerJr</td> + <td></td> <td>Up to date</td> <td> - </td> - <td><a href="http://www.torproject.us/dist/">http</a></td> - <td><a href="http://www.torproject.us/">http</a></td> - <td><a href="https://www.torproject.us/dist/">https</a></td> - <td><a href="https://www.torproject.us/">https</a></td> + <td><a href="http://mirror.open-networx.org/torproject.org/dist/">http</a></td> + <td><a href="http://mirror.open-networx.org/torproject.org/">http</a></td> + <td><a href="https://mirror.open-networx.org/torproject.org/dist/">https</a></td> + <td><a href="https://mirror.open-networx.org/torproject.org/">https</a></td> <td> - </td> - <td><a href="rsync://rsync.torproject.us/tor">rsync</a></td> + <td> - </td> </tr> <tr> - <td>IS</td> + <td>DE</td> - <td>myRL.net</td> + <td>factor.cc</td> <td>Up to date</td> <td> - </td> - <td><a href="http://tor.myrl.net/dist/">http</a></td> - <td><a href="http://tor.myrl.net/">http</a></td> - <td><a href="https://tor.myrl.net/dist/">https</a></td> - <td><a href="https://tor.myrl.net/">https</a></td> + <td><a href="http://tor.factor.cc/dist/">http</a></td> + <td><a href="http://tor.factor.cc/">http</a></td> + <td><a href="https://factor.cc/tor/dist/">https</a></td> + <td><a href="https://factor.cc/tor/">https</a></td> <td> - </td> <td> - </td> </tr> <tr> - <td>US</td> + <td>DE</td> - <td>hessmo</td> + <td>chaos darmstadt</td> <td>Up to date</td> <td> - </td> - <td><a href="http://mirror.hessmo.com/tor/dist/">http</a></td> - <td><a href="http://mirror.hessmo.com/tor/">http</a></td> + <td><a href="http://mirrors.chaos-darmstadt.de/tor-mirror/dist/">http</a></td> + <td><a href="http://mirrors.chaos-darmstadt.de/tor-mirror/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -324,15 +324,15 @@ <tr> - <td>LT</td> + <td>HU</td> - <td></td> + <td>Unknown</td> <td>Up to date</td> <td> - </td> - <td><a href="http://tor.vesta.nu/dist/">http</a></td> - <td><a href="http://tor.vesta.nu/">http</a></td> + <td><a href="http://mirror.tor.hu/dist/">http</a></td> + <td><a href="http://mirror.tor.hu/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -341,100 +341,83 @@ <tr> - <td>FR</td> + <td>DE</td> - <td>BarkerJr</td> + <td>beme it</td> <td>Up to date</td> <td> - </td> - <td><a href="http://www.oignon.net/dist/">http</a></td> - <td><a href="http://www.oignon.net/">http</a></td> - <td><a href="https://www.oignon.net/dist/">https</a></td> - <td><a href="https://www.oignon.net/">https</a></td> + <td><a href="http://tor.beme-it.de/dist/">http</a></td> + <td><a href="http://tor.beme-it.de/">http</a></td> + <td><a href="https://tor.beme-it.de/dist/">https</a></td> + <td><a href="https://tor.beme-it.de/">https</a></td> <td> - </td> - <td> - </td> + <td><a href="rsync://tor.beme-it.de/tor">rsync</a></td> </tr> <tr> - <td>AT</td> + <td>US</td> - <td>cyberarmy</td> + <td>NW Linux</td> <td>Up to date</td> <td> - </td> - <td><a href="http://tor.cyberarmy.at/dist/">http</a></td> - <td><a href="http://tor.cyberarmy.at/">http</a></td> + <td><a href="http://torproject.nwlinux.us/dist/">http</a></td> + <td><a href="http://torproject.nwlinux.us/">http</a></td> <td> - </td> <td> - </td> <td> - </td> - <td> - </td> + <td><a href="rsync://nwlinux.us/tor-web">rsync</a></td> </tr> <tr> <td>DE</td> - <td></td> + <td>borgmann.tv</td> <td>Up to date</td> <td> - </td> - <td><a href="http://mirror.open-networx.org/torproject.org/dist/">http</a></td> - <td><a href="http://mirror.open-networx.org/torproject.org/">http</a></td> - <td><a href="https://mirror.open-networx.org/torproject.org/dist/">https</a></td> - <td><a href="https://mirror.open-networx.org/torproject.org/">https</a></td> + <td><a href="http://tor.borgmann.tv/dist/">http</a></td> + <td><a href="http://tor.borgmann.tv/">http</a></td> <td> - </td> <td> - </td> -</tr> - -<tr> - - <td>DE</td> - - <td>factor.cc</td> - - <td>Up to date</td> - <td> - </td> - <td><a href="http://tor.factor.cc/dist/">http</a></td> - <td><a href="http://tor.factor.cc/">http</a></td> - <td><a href="https://factor.cc/tor/dist/">https</a></td> - <td><a href="https://factor.cc/tor/">https</a></td> <td> - </td> - <td> - </td> </tr> <tr> - <td>DE</td> + <td>US</td> - <td>chaos darmstadt</td> + <td>Xpdm</td> <td>Up to date</td> <td> - </td> - <td><a href="http://mirrors.chaos-darmstadt.de/tor-mirror/dist/">http</a></td> - <td><a href="http://mirrors.chaos-darmstadt.de/tor-mirror/">http</a></td> + <td><a href="http://torproj.xpdm.us/dist/">http</a></td> + <td><a href="http://torproj.xpdm.us/">http</a></td> + <td><a href="https://torproj.xpdm.us/dist/">https</a></td> + <td><a href="https://torproj.xpdm.us/">https</a></td> <td> - </td> <td> - </td> - <td> - </td> - <td> - </td> </tr> <tr> - <td>HU</td> + <td>IL</td> - <td>Unknown</td> + <td>Host4site</td> <td>Up to date</td> <td> - </td> - <td><a href="http://mirror.tor.hu/dist/">http</a></td> - <td><a href="http://mirror.tor.hu/">http</a></td> + <td><a href="http://mirror.host4site.co.il/torproject.org/dist/">http</a></td> + <td><a href="http://mirror.host4site.co.il/torproject.org/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -445,23 +428,6 @@ <td>US</td> - <td>Xpdm</td> - - <td>Up to date</td> - - <td> - </td> - <td><a href="http://torproj.xpdm.us/dist/">http</a></td> - <td><a href="http://torproj.xpdm.us/">http</a></td> - <td><a href="https://torproj.xpdm.us/dist/">https</a></td> - <td><a href="https://torproj.xpdm.us/">https</a></td> - <td> - </td> - <td> - </td> -</tr> - -<tr> - - <td>US</td> - <td>AskApache</td> <td>Up to date</td> @@ -511,66 +477,83 @@ <tr> - <td>DE</td> + <td>GB</td> - <td>[[:bbs:]]</td> + <td>torland</td> <td>Up to date</td> <td> - </td> + <td><a href="http://mirror.torland.me/torproject.org/dist/">http</a></td> + <td><a href="http://mirror.torland.me/torproject.org/">http</a></td> + <td><a href="https://mirror.torland.me/torproject.org/dist/">https</a></td> + <td><a href="https://mirror.torland.me/torproject.org/">https</a></td> <td> - </td> - <td><a href="http://tor.blingblingsquad.net/">http</a></td> <td> - </td> - <td><a href="https://tor.blingblingsquad.net/">https</a></td> +</tr> + +<tr> + + <td>IS</td> + + <td>TheOnionRouter</td> + + <td>Up to date</td> + <td> - </td> + <td><a href="http://theonionrouter.com/dist/">http</a></td> + <td><a href="http://theonionrouter.com/">http</a></td> <td> - </td> + <td> - </td> + <td> - </td> + <td> - </td> </tr> <tr> - <td>IS</td> + <td>SE</td> - <td>crypto.is</td> + <td>homosu</td> <td>Up to date</td> <td> - </td> - <td><a href="https://torproject.crypto.is/dist/">http</a></td> - <td><a href="https://torproject.crypto.is/">http</a></td> - <td><a href="https://torproject.crypto.is/dist/">https</a></td> - <td><a href="https://torproject.crypto.is/">https</a></td> + <td><a href="http://tor.homosu.net/dist/">http</a></td> + <td><a href="http://tor.homosu.net/">http</a></td> <td> - </td> <td> - </td> + <td> - </td> + <td> - </td> </tr> <tr> - <td>NL</td> + <td>DE</td> - <td>Amorphis</td> + <td>[[:bbs:]]</td> <td>Up to date</td> <td> - </td> - <td><a href="http://tor.amorphis.eu/dist/">http</a></td> - <td><a href="http://tor.amorphis.eu/">http</a></td> <td> - </td> + <td><a href="http://tor.blingblingsquad.net/">http</a></td> <td> - </td> + <td><a href="https://tor.blingblingsquad.net/">https</a></td> <td> - </td> <td> - </td> </tr> <tr> - <td>DK</td> + <td>INT</td> - <td>Zentrum der Gesundheit</td> + <td>CoralCDN</td> <td>Up to date</td> <td> - </td> - <td><a href="http://tor.idnr.ws/dist/">http</a></td> - <td><a href="http://tor.idnr.ws/">http</a></td> + <td><a href="http://www.torproject.org.nyud.net/dist/">http</a></td> + <td><a href="http://www.torproject.org.nyud.net/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -579,15 +562,15 @@ <tr> - <td>SE</td> + <td>EE</td> - <td>homosu</td> + <td>CyberSIDE</td> <td>Up to date</td> <td> - </td> - <td><a href="http://tor.homosu.net/dist/">http</a></td> - <td><a href="http://tor.homosu.net/">http</a></td> + <td><a href="http://cyberside.net.ee/tor/">http</a></td> + <td><a href="http://cyberside.planet.ee/tor/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -596,19 +579,19 @@ <tr> - <td>DE</td> + <td>IS</td> - <td>torservers</td> + <td>torproject.is</td> - <td>Out of date</td> + <td>Up to date</td> <td> - </td> - <td><a href="http://www.torservers.net/mirrors/torproject.org/dist/">http</a></td> - <td><a href="http://www.torservers.net/mirrors/torproject.org/">http</a></td> - <td><a href="https://www.torservers.net/mirrors/torproject.org/dist/">https</a></td> - <td><a href="https://www.torservers.net/mirrors/torproject.org/">https</a></td> + <td><a href="http://torproject.is/dist/">http</a></td> + <td><a href="http://torproject.is/">http</a></td> <td> - </td> <td> - </td> + <td> - </td> + <td> - </td> </tr> <tr> @@ -617,7 +600,7 @@ <td>India Tor Fans</td> - <td>Out of date</td> + <td>Up to date</td> <td> - </td> <td><a href="http://www.torproject.org.in/dist/">http</a></td> @@ -698,34 +681,34 @@ <tr> - <td>AT</td> + <td>NL</td> <td></td> - <td>Unknown</td> + <td>Out of date</td> <td> - </td> - <td><a href="http://tor.mirror.reisenb.at/dist/">http</a></td> - <td><a href="http://tor.mirror.reisenb.at/">http</a></td> <td> - </td> <td> - </td> + <td><a href="https://www.coevoet.nl/tor/dist/">https</a></td> <td> - </td> <td> - </td> + <td> - </td> </tr> <tr> - <td>NL</td> + <td>IS</td> - <td></td> + <td>crypto.is</td> - <td>Out of date</td> + <td>Unknown</td> <td> - </td> + <td><a href="https://torproject.crypto.is/dist/">http</a></td> + <td><a href="https://torproject.crypto.is/">http</a></td> + <td><a href="https://torproject.crypto.is/dist/">https</a></td> + <td><a href="https://torproject.crypto.is/">https</a></td> <td> - </td> <td> - </td> - <td><a href="https://www.coevoet.nl/tor/dist/">https</a></td> - <td> - </td> - <td> - </td> - <td> - </td> </tr>

1 0

r25562: {website} fix the link to sponsors in the press release from 4 years a (website/trunk/press/en)
by Andrew Lewman 21 Mar '12

21 Mar '12

Author: phobos Date: 2012-03-21 03:36:14 +0000 (Wed, 21 Mar 2012) New Revision: 25562 Modified: website/trunk/press/en/2008-12-19-roadmap-press-release.wml Log: fix the link to sponsors in the press release from 4 years ago. Modified: website/trunk/press/en/2008-12-19-roadmap-press-release.wml =================================================================== --- website/trunk/press/en/2008-12-19-roadmap-press-release.wml 2012-03-21 03:33:49 UTC (rev 25561) +++ website/trunk/press/en/2008-12-19-roadmap-press-release.wml 2012-03-21 03:36:14 UTC (rev 25562) @@ -53,7 +53,7 @@ handle all the people who want to use it."</p> <p>Tor welcomes additional sponsors to join <a - href="https://www.torproject.org/sponsors">current sponsors</a>; + href="<page about/sponsors>">current sponsors</a>; such as the Broadcasting Board of Governors, the NLnet Foundation, and hundreds of individual donors. While existing funders are enough to get the items on the roadmap started, an additional $2.1 million over the

1 0

r25561: {website} clean up some obvious dead links that are frequently hit by (in website/trunk: . about/en docs/en getinvolved/en torbutton/en/design)
by Andrew Lewman 21 Mar '12

21 Mar '12

Author: phobos Date: 2012-03-21 03:33:49 +0000 (Wed, 21 Mar 2012) New Revision: 25561 Modified: website/trunk/about/en/contributors.wml website/trunk/about/en/gsoc.wml website/trunk/about/en/sponsors.wml website/trunk/about/en/volunteers.wml website/trunk/docs/en/tor-doc-osx.wml website/trunk/docs/en/trademark-faq.wml website/trunk/getinvolved/en/volunteer.wml website/trunk/torbutton/en/design/index.html.en website/trunk/update-mirrors.pl Log: clean up some obvious dead links that are frequently hit by users. Modified: website/trunk/about/en/contributors.wml =================================================================== --- website/trunk/about/en/contributors.wml 2012-03-20 15:12:10 UTC (rev 25560) +++ website/trunk/about/en/contributors.wml 2012-03-21 03:33:49 UTC (rev 25561) @@ -14,8 +14,8 @@ <dt>Carolyn Anhalt</dt><dd>Coordinated translations and translators for all of tor's content; website, products, and materials.</dd> <dt>Domenik Bork</dt><dd> Worked on -Configuration of Hidden Services with User Authorization in Vidalia (<a -href="http://trac.vidalia-project.net/browser/vidalia/branches/hidden-services">svn</a>) as part of Google Summer of Code 2008.</dd> +Configuration of Hidden Services with User Authorization in Vidalia as +part of Google Summer of Code 2008.</dd> <dt>Benedikt Boss</dt><dd>Worked during the 2007 Google Summer of Code on <a href="https://svn.torproject.org/svn/topf/trunk/README">TOPF</a>, a fuzzer for Tor; mentored by Roger.</dd> Modified: website/trunk/about/en/gsoc.wml =================================================================== --- website/trunk/about/en/gsoc.wml 2012-03-20 15:12:10 UTC (rev 25560) +++ website/trunk/about/en/gsoc.wml 2012-03-21 03:33:49 UTC (rev 25561) @@ -226,8 +226,6 @@ </p> <ul> - <li><h4><a href="http://tor.spanning-tree.org/proposal.html">DNSEL Rewrite</a> by Harry Bock</h4></li> - <li><h4><a href="http://kjb.homeunix.com/wp-content/uploads/2010/05/KevinBerry-GSoC2010-TorP…">Extending Tor Network Metrics</a> by Kevin Berry</h4></li> <li><h4><a href="../about/gsocProposal/gsoc10-proposal-soat.txt">SOAT Expansion</a> by John Schanck</h4></li> <li><h4><a href="http://inspirated.com/uploads/tor-gsoc-11.pdf">GTK+ Frontend and Client Mode Improvements for arm</a> by Kamran Khan</h4></li> <li><h4><a href="http://www.gsathya.in/gsoc11.html">Orbot + ORLib</a> by Sathya Gunasekaran</h4></li> Modified: website/trunk/about/en/sponsors.wml =================================================================== --- website/trunk/about/en/sponsors.wml 2012-03-20 15:12:10 UTC (rev 25560) +++ website/trunk/about/en/sponsors.wml 2012-03-21 03:33:49 UTC (rev 25561) @@ -30,7 +30,7 @@ <h3><i>Asparagales</i> (up to $500k)</h3> <ul> - <li><a href="http://www.internews.fr/">Internews Europe</a> (2006-2008)</li> + <li>Internews Europe (2006-2008)</li> <li><a href="http://nsf.gov/">National Science Foundation via Drexel University</a> (2009-2011)</li> </ul> Modified: website/trunk/about/en/volunteers.wml =================================================================== --- website/trunk/about/en/volunteers.wml 2012-03-20 15:12:10 UTC (rev 25560) +++ website/trunk/about/en/volunteers.wml 2012-03-21 03:33:49 UTC (rev 25561) @@ -16,7 +16,7 @@ Brandon Wiley - <a href="http://stepthreeprivacy.org/">Blocking-resistant Transport Evaluation Framework</a><br /> George Kadianakis - <a href="https://gitorious.org/obfsproxy/pages/Home">Pluggable transports</a><br /> Julien Voisin - <a href="http://mat-tor.blogspot.com/">Metadata Anonymisation Toolkit</a><br /> -Kamran Khan - <a href="http://code.inspirated.com/gsoc/">GTK+ Frontend and Client Mode Improvements for arm</a><br /> +Kamran Khan - <a href="http://code.inspirated.com/">GTK+ Frontend and Client Mode Improvements for arm</a><br /> Max - <a href="https://tails.boum.org/todo/TailsGreeter/blog/">Custom GDM3 startup menu, aka. tails-greeter</a><br /> Sathyanarayanan Gunasekaran - <a href="http://gsathya.in/blog/">GSoC : Orbot + ORLib</a></dd> <dt>Anonym, bertagaz, and intrigeri</dt><dd>Maintainers of <a Modified: website/trunk/docs/en/tor-doc-osx.wml =================================================================== --- website/trunk/docs/en/tor-doc-osx.wml 2012-03-20 15:12:10 UTC (rev 25560) +++ website/trunk/docs/en/tor-doc-osx.wml 2012-03-21 03:33:49 UTC (rev 25561) @@ -91,8 +91,7 @@ port 9050), but see <a href="<wikifaq>#SOCKSAndDNS">this FAQ entry</a> for why this may be dangerous. For applications that support neither SOCKS nor HTTP, take a look at <a - href="http://www.taiyo.co.jp/~gotoh/ssh/connect.html">connect</a> - or <a href="http://www.dest-unreach.org/socat/">socat</a>.</p> + href="http://www.dest-unreach.org/socat/">socat</a>.</p> <p>For information on how to Torify other applications, check out the <a href="<wiki>doc/TorifyHOWTO">Torify HOWTO</a>. Modified: website/trunk/docs/en/trademark-faq.wml =================================================================== --- website/trunk/docs/en/trademark-faq.wml 2012-03-20 15:12:10 UTC (rev 25560) +++ website/trunk/docs/en/trademark-faq.wml 2012-03-21 03:33:49 UTC (rev 25561) @@ -76,10 +76,7 @@ <ul> <li><a href="https://tails.boum.org/">The Amnesic Incognito Live System</a></li> - <li>Portable Tor</li> - <li><a href="http://torstatus.kgprog.com/">Kprog Tor Status</a></li> <li><a href="<page projects/vidalia>">Vidalia</a></li> - <li><a href="http://www.anonymityanywhere.com/tork/">TorK</a></li> </ul> </div>  Modified: website/trunk/getinvolved/en/volunteer.wml =================================================================== --- website/trunk/getinvolved/en/volunteer.wml 2012-03-20 15:12:10 UTC (rev 25560) +++ website/trunk/getinvolved/en/volunteer.wml 2012-03-21 03:33:49 UTC (rev 25561) @@ -114,7 +114,7 @@ <td>Usability</td> <td>Sys Admin</td> <td>Heavy</td> - <td><a href="https://tails.boum.org/chat/">#tails</a></td> + <td><a href="https://tails.boum.org/">#tails</a></td> </tr> <tr> @@ -1238,7 +1238,7 @@ Skill Level: <i>Medium</i> <br> Likely Mentors: <i>Karsten</i> - <p>The <a href="https://metrics.torproject.org/data.html">Metrics data archive</a> of Tor relay descriptors and other Tor-related network data has grown to over 100G in size, bz2-compressed. We have developed two search interfaces: the <a href="https://metrics.torproject.org/relay-search.html">relay search</a> finds relays by nickname, fingerprint, or IP address in a given month; <a href="https://metrics.torproject.org/exonerator-beta.html">ExoneraTor</a> finds whether a given IP address was a relay on a given day.</p> + <p>The <a href="https://metrics.torproject.org/data.html">Metrics data archive</a> of Tor relay descriptors and other Tor-related network data has grown to over 100G in size, bz2-compressed. We have developed two search interfaces: the <a href="https://metrics.torproject.org/relay-search.html">relay search</a> finds relays by nickname, fingerprint, or IP address in a given month; <a href="https://metrics.torproject.org/exonerator.html">ExoneraTor</a> finds whether a given IP address was a relay on a given day.</p> <p>We'd like to have a more general search application for Tor descriptors and metrics data. There are more <a href="https://metrics.torproject.org/formats.html">descriptor types</a> that we'd like to include in the search. The search application should handle most of them and understand some semantics like what's a timestamp, what's an IP address, and what's a link to another descriptor. Users should then be able to search for arbitrary strings or limit their search to given time periods or IP address ranges. Descriptors that reference other descriptors should contain links, and descriptors should be able to say from where they are linked. The goal is to make the archive easily browsable.</p> Modified: website/trunk/torbutton/en/design/index.html.en =================================================================== --- website/trunk/torbutton/en/design/index.html.en 2012-03-20 15:12:10 UTC (rev 25560) +++ website/trunk/torbutton/en/design/index.html.en 2012-03-21 03:33:49 UTC (rev 25561) @@ -1315,13 +1315,6 @@ proxy-bypass exploits. It is a fully automated test suite maintained by <a class="ulink" href="http://digitaloffense.net/" target="_top">HD Moore</a> as a service for people to use to test their anonymity systems. - </p></li><li class="listitem"><a class="ulink" href="http://deanonymizer.com/" target="_top">Deanonymizer.com</a><p> - -Deanonymizer.com is another automated test suite that tests for proxy bypass -and other information disclosure vulnerabilities. It is maintained by Kyle -Williams, the author of <a class="ulink" href="http://www.janusvm.com/" target="_top">JanusVM</a> -and <a class="ulink" href="http://www.januspa.com/" target="_top">JanusPA</a>. - </p></li><li class="listitem"><a class="ulink" href="https://www.jondos.de/en/anontest" target="_top">JonDos AnonTest</a><p> Modified: website/trunk/update-mirrors.pl =================================================================== --- website/trunk/update-mirrors.pl 2012-03-20 15:12:10 UTC (rev 25560) +++ website/trunk/update-mirrors.pl 2012-03-21 03:33:49 UTC (rev 25561) @@ -492,7 +492,7 @@ httpsWebsiteMirror => "", rsyncWebsiteMirror => "", ftpWebsiteMirror => "", - httpDistMirror => "http://tor.cyberarmy.at/dist/", + httpDistMirror => "", httpsDistMirror => "", rsyncDistMirror => "", hiddenServiceMirror => "", @@ -713,24 +713,6 @@ rsyncDistMirror => "", hiddenServiceMirror => "", }, - mirror047 => { - adminContact => "", - orgName => "", - isoCC => "AT", - subRegion => "", - region => "AT", - ipv4 => "True", - ipv6 => "False", - loadBalanced => "No", - httpWebsiteMirror => "http://tor.mirror.reisenb.at/", - httpsWebsiteMirror => "", - rsyncWebsiteMirror => "", - ftpWebsiteMirror => "", - httpDistMirror => "http://tor.mirror.reisenb.at/dist/", - httpsDistMirror => "", - rsyncDistMirror => "", - hiddenServiceMirror => "", - }, mirror048 => { adminContact => "", orgName => "",

1 0

[ooni-probe/master] * Make TCPScan use gevent sockets
by art＠torproject.org 21 Mar '12

21 Mar '12

commit 79ef12a4ed5db353b8341b1977707dc6e9ae9d28 Author: Arturo Filastò <hellais(a)gmail.com> Date: Thu Mar 15 00:08:20 2012 -0700 * Make TCPScan use gevent sockets * Make the log level be configurable from config file --- ooni-probe.conf | 3 ++- plugoo/assets.py | 4 ---- tests/tcpscan.py | 2 +- utils.py | 2 +- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/ooni-probe.conf b/ooni-probe.conf index 9563493..8702353 100644 --- a/ooni-probe.conf +++ b/ooni-probe.conf @@ -9,10 +9,11 @@ assetdir = assets/ testdir = tests/ loglevel = DEBUG +consoleloglevel = DEBUG proxyaddress = 127.0.0.1:9050 # The following configurations are for searching for PlanetLab -# nodes, adding them to a slice, and PlanetLab general API +# nodes, adding them to a slice, and PlanetLab general API # authentication: pl_username = yourusername pl_password = yourpassword diff --git a/plugoo/assets.py b/plugoo/assets.py index 5ed0399..eded997 100644 --- a/plugoo/assets.py +++ b/plugoo/assets.py @@ -41,13 +41,9 @@ class Asset: # clean me up please... line = self.fh.readline() if line: -<<<<<<< HEAD parsed_line = self.parse_line(line) if parsed_line: return parsed_line -======= - return self.parse_line(line) ->>>>>>> 74f25ec... Add a line by line parser else: self.fh.seek(0) raise StopIteration diff --git a/tests/tcpscan.py b/tests/tcpscan.py index 9e962ca..26e854b 100644 --- a/tests/tcpscan.py +++ b/tests/tcpscan.py @@ -6,7 +6,7 @@ """ import os -import socket +from gevent import socket from datetime import datetime import socks diff --git a/utils.py b/utils.py index 79edadb..e9c55f1 100644 --- a/utils.py +++ b/utils.py @@ -61,7 +61,7 @@ def get_logger(config): filemode='w') console = logging.StreamHandler() - console.setLevel(logging.INFO) + console.setLevel(getattr(logging, config.consoleloglevel.upper())) # Set the console logger to a different format formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') console.setFormatter(formatter)

1 0

[ooni-probe/master] Add a method for line by line asset parsing
by art＠torproject.org 21 Mar '12

21 Mar '12

commit 91e0bbf22cf60173e31add4f61ce1e628325105d Author: Arturo Filastò <hellais(a)gmail.com> Date: Wed Mar 14 23:26:56 2012 -0700 Add a method for line by line asset parsing (I think I did a mess with git and may have made noise on tor-commits...) --- plugoo/assets.py | 15 ++++++++++++++- 1 files changed, 14 insertions(+), 1 deletions(-) diff --git a/plugoo/assets.py b/plugoo/assets.py index 6f566fe..5ed0399 100644 --- a/plugoo/assets.py +++ b/plugoo/assets.py @@ -26,6 +26,13 @@ class Asset: self.fh.seek(0) return i + 1 + def parse_line(self, line): + """ + Override this method if you need line + by line parsing of an Asset. + """ + return line.replace('\n','') + def next_asset(self): """ Return the next asset. @@ -34,7 +41,13 @@ class Asset: # clean me up please... line = self.fh.readline() if line: - return line.replace('\n','') +<<<<<<< HEAD + parsed_line = self.parse_line(line) + if parsed_line: + return parsed_line +======= + return self.parse_line(line) +>>>>>>> 74f25ec... Add a line by line parser else: self.fh.seek(0) raise StopIteration

1 0

[ooni-probe/master] * Start implementing worker
by art＠torproject.org 21 Mar '12

21 Mar '12

commit 2e6edd21f3a7b272c2c487b8ce399fe2e0fde3a3 Author: Arturo Filastò <hellais(a)gmail.com> Date: Fri Mar 16 20:16:55 2012 -0700 * Start implementing worker * Add hackish resume support --- plugoo/tests.py | 25 ++++++++++++++++++++----- plugoo/workers.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/plugoo/tests.py b/plugoo/tests.py index d57c0a8..3d6b36b 100644 --- a/plugoo/tests.py +++ b/plugoo/tests.py @@ -32,7 +32,7 @@ class Test: """ pass - def load_assets(self, assets): + def load_assets(self, assets, index=None): """ Takes as input an array of Asset objects and outputs an iterator for the loaded assets. @@ -57,21 +57,36 @@ class Test: smallassets = list(assets) smallassets.pop(bigidx) + i = 0 for x in assets[bigidx]: if asset_count > 1: # XXX this will only work in python 2.6, maybe refactor? for comb in itertools.product(*smallassets): - yield (x,) + comb + if index and i < index: + i += 1 + else: + yield (x,) + comb else: - yield (x) + if index and i < index: + i += 1 + else: + yield (x) - def run(self, assets=None, buffer=10, timeout=100000): + def run(self, assets=None, extradata=None, buffer=10, timeout=100000): self.logger.info("Starting %s", self.name) jobs = [] if assets: self.logger.debug("Running through tests") - for i, data in enumerate(self.load_assets(assets)): + + if extradata['index']: + index = extradata['index'] + else: + index = None + + for i, data in enumerate(self.load_assets(assets, index)): args = {'data': data} + if extradata: + args = dict(args.items()+extradata.items()) # Append to the job queue jobs.append(gevent.spawn(self.experiment, **args)) # If the buffer is full run the jobs diff --git a/plugoo/workers.py b/plugoo/workers.py new file mode 100644 index 0000000..3c8397d --- /dev/null +++ b/plugoo/workers.py @@ -0,0 +1,36 @@ +import gevent +from gevent.pool import Pool + +class WorkFactory: + """ + This class is responsible for producing + units of work. + """ + def __init__(self, assets=None, + nodes=None, rule=None): + pass + + def _process_rule(self): + pass + + def get_work_unit(): + pass + +class UnitOfWork: + def __init__(self, tests, poolsize=20, + unit_of_work=None): + pass + + def _read_unit_of_work(self): + pass + + def _build_pools(self): + for i, x in enumerate(self.tests): + if i % self.poolsize == 0: + + + def do(self): + with gevent.Timeout(): + self.pool.join() + +

1 0

[ooni-probe/master] Implement HTTP Host based filtering detection
by art＠torproject.org 21 Mar '12

21 Mar '12

commit d5fdae906d37f86c8b2adf084cc58fa2a9a53094 Author: Arturo Filastò <hellais(a)gmail.com> Date: Fri Mar 16 20:17:19 2012 -0700 Implement HTTP Host based filtering detection Used to detect censorship on T-Mobile --- tests/httphost.py | 132 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 132 insertions(+), 0 deletions(-) diff --git a/tests/httphost.py b/tests/httphost.py new file mode 100644 index 0000000..6446e1f --- /dev/null +++ b/tests/httphost.py @@ -0,0 +1,132 @@ +""" + HTTP Host based filtering + ************************* + + This test detect HTTP Host field + based filtering. + It is used to detect censorship on + performed with Web Guard (used by + T-Mobile US). +""" +import os +from datetime import datetime +from gevent import monkey + +import urllib2 +import httplib +# WARNING! Using gevent's socket +# introduces the 0x20 DNS "feature". +# This will result is weird DNS requests +# appearing on the wire. +monkey.patch_socket() + +from BeautifulSoup import BeautifulSoup + +from plugoo.assets import Asset +from plugoo.tests import Test + +__plugoo__ = "HTTP Host" +__desc__ = "This detects HTTP Host field based filtering" + +class HTTPHostAsset(Asset): + """ + This is the asset that should be used by the Test. It will + contain all the code responsible for parsing the asset file + and should be passed on instantiation to the test. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + return line.split(',')[1].replace('\n','') + +class HTTPHost(Test): + """ + The main Test class + """ + + def check_response(self, response): + soup = BeautifulSoup(response) + if soup.head.title.string == "Content Filtered": + # Response indicates censorship + return True + else: + # Response does not indicate censorship + return False + + + def is_censored(self, response): + if response: + soup = BeautifulSoup(response) + censored = self.check_response(response) + else: + censored = "unreachable" + return censored + + def urllib2_test(self, control_server, host): + req = urllib2.Request(control_server) + req.add_header('Host', host) + try: + r = urllib2.urlopen(req) + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + + return censored + + def httplib_test(self, control_server, host): + try: + conn = httplib.HTTPConnection(control_server) + conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True) + conn.putheader("Host", host) + conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6") + conn.endheaders() + r = conn.getresponse() + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + + return censored + + + def experiment(self, *a, **kw): + """ + Try to connect to the control server with + the specified host field. + """ + host = kw['data'] + control_server = kw['control_server'] + self.logger.info("Testing %s (%s)" % (host, control_server)) + + #censored = self.urllib2_test(control_server, host) + censored = self.httplib_test(control_server, host) + + self.logger.info("%s: %s" % (host, censored)) + return {'Time': datetime.now(), + 'Host': host, + 'Censored': censored} + + +def run(ooni): + """ + This is the function that will be called by OONI + and it is responsible for instantiating and passing + the arguments to the Test class. + """ + config = ooni.config + + # This the assets array to be passed to the run function of + # the test + assets = [HTTPHostAsset(os.path.join(config.main.assetdir, \ + "top-1m.csv"))] + + # Instantiate the Test + thetest = HTTPHost(ooni) + ooni.logger.info("starting HTTP Host Test...") + # Run the test with argument assets + thetest.run(assets, {'index': 5825, 'control_server': '195.85.254.203:8080'}) + ooni.logger.info("finished.") + +

1 0