March 2012 - tor-commits - lists.torproject.org

[translation/vidalia_help] Update translations for vidalia_help
by translation＠torproject.org 06 Mar '12

06 Mar '12

commit af33759cb17a894867a66a034e5dbd8ebab01672 Author: Translation commit bot <translation(a)torproject.org> Date: Tue Mar 6 17:45:19 2012 +0000 Update translations for vidalia_help --- eu/netview.po | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eu/netview.po b/eu/netview.po index 5add400..f6980ab 100644 --- a/eu/netview.po +++ b/eu/netview.po @@ -6,7 +6,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2010-06-26 17:00+0200\n" -"PO-Revision-Date: 2012-03-02 23:44+0000\n" +"PO-Revision-Date: 2012-03-06 17:19+0000\n" "Last-Translator: Antxon Baldarra <baldarra(a)euskalerria.org>\n" "Language-Team: LANGUAGE <LL(a)li.org>\n" "MIME-Version: 1.0\n" @@ -112,7 +112,7 @@ msgid "" "On the left side of the network view, you will see a list of relays in the " "Tor network. Next to each relay is an icon indicating that relay's status. " "The following table summarizes the possible relay status icons:" -msgstr "" +msgstr "Sare ikuspegiaren ezkerraldean, Tor sarean dauden errele zerrenda bat ikusiko duzu. Errele bakoitzaren alboan ikono bat dago errelearen egoera adieraziz. Hurrengo taulak errele egoera ikono ezberdinak laburbiltzen ditu:" #. type: Content of: <html><body><table><tr><td> #: en/netview.html:78 @@ -124,7 +124,7 @@ msgstr "Errelea lineaz kanpo dago edo besterik gabe ez du erantzuten." msgid "" "The relay is hibernating, meaning it is online, but has used up as " "much bandwidth as the operator is willing to allow for a given time period." -msgstr "" +msgstr "Errelea hibernatzen dago, honek esan nahi du online dagoela, baina operadoreak momentu zehatz batean baimendu dezakeen banda-zabalera guztia erabili duela." #. type: Content of: <html><body><table><tr><td> #: en/netview.html:92

1 0

[torperf/master] Truncate .data and .extradata files to contain only the last 4 days.
by karsten＠torproject.org 06 Mar '12

06 Mar '12

commit 178f753a451959cb78c6d0bf13a929ed1940a2e1 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Mar 5 16:33:41 2012 +0100 Truncate .data and .extradata files to contain only the last 4 days. --- extra_stats.py | 57 +++++++++++++++++++++++++++++++++++---- measurements-HOWTO | 18 ++++++++---- truncate-data.py | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 12 deletions(-) diff --git a/extra_stats.py b/extra_stats.py index 69bb8a5..662cf43 100755 --- a/extra_stats.py +++ b/extra_stats.py @@ -1,6 +1,6 @@ #!/usr/bin/python -import sys, time +import os, re, sys, time import TorCtl.TorUtil as TorUtil import TorCtl.TorCtl as TorCtl @@ -22,15 +22,18 @@ class Circuit: self.stream_fail_reason = None class WriteStats(TorCtl.PostEventListener): - def __init__(self, port, filename): + def __init__(self, port, filename, truncate): TorCtl.PostEventListener.__init__(self) self._port = int(port) self._filename = filename + self.truncate = truncate + self.first_launched = None self._conn = None self.all_circs = {} self.ignore_streams = {} self.current_timeout = None self.current_quantile = None + self.truncate_statsfile() def connect(self): self._conn = TorCtl.connect(HOST, self._port) @@ -52,6 +55,7 @@ class WriteStats(TorCtl.PostEventListener): self.current_quantile = b.cutoff_quantile result = b.event_name + " " +b.body self.write_result(result) + self.truncate_statsfile() def circ_status_event(self, c): if c.status == "LAUNCHED": @@ -94,6 +98,7 @@ class WriteStats(TorCtl.PostEventListener): (self.current_timeout, self.current_quantile) self.write_result(result) + self.truncate_statsfile() def stream_status_event(self, event): if event.status == "NEW": @@ -140,19 +145,59 @@ class WriteStats(TorCtl.PostEventListener): def write_result(self, result): # XXX: hrmm. seems wasteful to keep opening+closing.. + # XXX: When changing this, also change truncated_statsfile(). statsfile = open(self._filename, 'a') statsfile.write(result+"\n") statsfile.close() + def truncate_statsfile(self): + if not self.truncate: + return + launched_str = "^.*LAUNCH=([\\d]*).*$" + if not self.first_launched: + if os.path.isfile(self._filename): + launched_re = re.compile(launched_str) + with open(self._filename) as statsfile: + for line in statsfile: + m = launched_re.match(line) + if m: + self.first_launched = int(m.group(1)) + break + if not self.first_launched: + self.first_launched = time.time() + now = time.time() + if self.first_launched < now - 7 * 24 * 60 * 60: + copylines = False + statsfilebak_path = self._filename + ".bak" + statsfilebak_file = open(statsfilebak_path, "w") + launched_re = re.compile(launched_str) + with open(self._filename) as statsfile: + for line in statsfile: + if copylines: + statsfilebak_file.write(line) + else: + m = launched_re.match(line) + if m and int(m.group(1)) >= now - 4 * 24 * 60 * 60: + statsfilebak_file.write(line) + copylines = True + statsfilebak_file.close() + os.rename(statsfilebak_path, self._filename) + def main(): - if len(sys.argv) < 3: + if len(sys.argv) < 3 or len(sys.argv) > 4: print "Bad arguments" sys.exit(1) - port = sys.argv[1] - filename = sys.argv[2] + truncate = False + if (sys.argv[1] == "--truncate"): + truncate = True + port = sys.argv[2] + filename = sys.argv[3] + else: + port = sys.argv[1] + filename = sys.argv[2] - stats = WriteStats(port, filename) + stats = WriteStats(port, filename, truncate) stats.connect() stats.setup_listener() try: diff --git a/measurements-HOWTO b/measurements-HOWTO index 67de0c5..2cefb68 100644 --- a/measurements-HOWTO +++ b/measurements-HOWTO @@ -103,14 +103,17 @@ cd ~/torperf/torclient50kb && tor -f ~/torperf/torclient50kb/torrc cd ~/torperf/torclient1mb && tor -f ~/torperf/torclient1mb/torrc cd ~/torperf/torclient5mb && tor -f ~/torperf/torclient5mb/torrc sleep 5 -cd ~/torperf/torclient50kb && python ../extra_stats.py 10020 +cd ~/torperf/torclient50kb && python ../extra_stats.py --truncate 10020 ../50kb.extradata & -cd ~/torperf/torclient1mb && python ../extra_stats.py 10021 +cd ~/torperf/torclient1mb && python ../extra_stats.py --truncate 10021 ../1mb.extradata & -cd ~/torperf/torclient5mb && python ../extra_stats.py 10022 +cd ~/torperf/torclient5mb && python ../extra_stats.py --truncate 10022 ../5mb.extradata & EOF +(Omit the --truncate switch if you don't want .extradata files to be +truncated once per week to contain only the last 4 days of data.) + $ chmod a+x start-tors $ ./start-tors @@ -125,13 +128,16 @@ $ crontab -e */5 * * * * timeout -s2 295 ~/torperf/trivsocks-client torperf.torproject.org 127.0.0.1:9020 /.50kbfile >> ~/torperf/50kb.data - 2>/dev/null + 2>/dev/null; ~/torperf/truncate-data.py ~/torperf/50kb.data 2,32 * * * * timeout -s2 1795 ~/torperf/trivsocks-client torperf.torproject.org 127.0.0.1:9021 /.1mbfile >> ~/torperf/1mb.data - 2>/dev/null + 2>/dev/null; ~/torperf/truncate-data.py ~/torperf/1mb.data 8 * * * * timeout -s2 3595 ~/torperf/trivsocks-client torperf.torproject.org 127.0.0.1:9022 /.5mbfile >> ~/torperf/5mb.data - 2>/dev/null + 2>/dev/null; ~/torperf/truncate-data.py ~/torperf/5mb.data + +(Omit the truncate-data.py command if you don't want .data files to be +truncated once per week to contain only the last 4 days of data.) From now on, the three files 50kb.data, 1mb.data, and 5mb.data should accumulate lines like this (50kb.data shown here; line breaks are only for diff --git a/truncate-data.py b/truncate-data.py new file mode 100755 index 0000000..a70e3fc --- /dev/null +++ b/truncate-data.py @@ -0,0 +1,75 @@ +#!/usr/bin/python +import os +import re +import sys +import time + +# Truncate Torperf .data file by deleting lines older than 4 days, but +# only truncate once a week. +def main(): + + # Check usage. + if len(sys.argv) != 2: + print "Usage: ./truncate.py <.data file>" + return + data_path = sys.argv[1] + if not os.path.isfile(data_path): + print "%s is not a .data file." % data_path + return + + # Prepare for parsing. + parselines = False + copylines = False + databak_path = data_path + ".bak" + databak_file = None + started_re = re.compile('(^[\\d]*) .*') + now = time.time() + + # Parse the .data file line by line, possibly stopping early if the + # first timestamp we find isn't older than a week. + with open(data_path) as data_file: + for line in data_file: + + # Copy lines written in the past 4 days. We have already decided to + # copy this part of the .data file before, so just copy the line and + # continue. + if copylines: + databak_file.write(line) + continue + + # Skip empty lines. + if line.strip() == "": + continue + + # Extract the first timestamp from the current line. + m = started_re.match(line) + if not m: + print "%s is not a valid .data file." % data_path + return + started_ts = int(m.group(1)) + + # Decide whether to start copying lines. We have already decided to + # truncate this file before. + if parselines: + if started_ts >= now - 4 * 24 * 60 * 60: + databak_file.write(line) + copylines = True + continue + + # Decide whether to truncate this file at all. + if started_ts >= now - 7 * 24 * 60 * 60: + return + + # Open a .bak file to write into and start parsing lines to copy in + # the next iteration. + databak_file = open(databak_path, "w") + parselines = True + + # Close the .bak file and replace the original .data file with it. + if databak_file: + databak_file.close() + os.rename(databak_path, data_path) + +if __name__ == "__main__": + main() +

1 0

[metrics-web/master] Export graph data as JSON objects.
by karsten＠torproject.org 06 Mar '12

06 Mar '12

commit b889394b3cfbbcb8a038d7a285f39c9b68c283cd Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Mar 6 15:23:23 2012 +0100 Export graph data as JSON objects. --- etc/web.xml | 11 + src/org/torproject/ernie/web/GraphDataServlet.java | 238 ++++++++++++++++++++ 2 files changed, 249 insertions(+), 0 deletions(-) diff --git a/etc/web.xml b/etc/web.xml index 2c6752b..fea3df6 100644 --- a/etc/web.xml +++ b/etc/web.xml @@ -393,6 +393,17 @@ <url-pattern>/custom-graph.html</url-pattern> </servlet-mapping> + <servlet> + <servlet-name>GraphData</servlet-name> + <servlet-class> + org.torproject.ernie.web.GraphDataServlet + </servlet-class> + </servlet> + <servlet-mapping> + <servlet-name>GraphData</servlet-name> + <url-pattern>/graphs/*</url-pattern> + </servlet-mapping> + <welcome-file-list> <welcome-file>index.html</welcome-file> </welcome-file-list> diff --git a/src/org/torproject/ernie/web/GraphDataServlet.java b/src/org/torproject/ernie/web/GraphDataServlet.java new file mode 100644 index 0000000..ec3c93b --- /dev/null +++ b/src/org/torproject/ernie/web/GraphDataServlet.java @@ -0,0 +1,238 @@ +package org.torproject.ernie.web; + +import java.io.*; +import java.text.*; +import java.util.*; +import java.util.logging.*; + +import javax.servlet.*; +import javax.servlet.http.*; + +/** + * Servlet that reads an HTTP request for a JSON-formatted graph data + * document, asks the RObjectGenerator to generate the CSV file behind it, + * converts it to a JSON object, and returns it to the client. + */ +public class GraphDataServlet extends HttpServlet { + + private RObjectGenerator rObjectGenerator; + + /* Available graph data files. */ + private SortedMap<String, String> availableGraphDataFiles; + + /* Variable columns in CSV files that are in long form, not wide. */ + private SortedMap<String, String> variableColumns; + + /* Value columns in CSV files if only specific value columns shall be + * included in results. */ + private SortedMap<String, String> valueColumns; + + private Logger logger; + + public void init() { + + /* Initialize logger. */ + this.logger = Logger.getLogger(GraphDataServlet.class.toString()); + + /* Initialize map of available graph data files and corresponding CSV + * files. */ + this.availableGraphDataFiles = new TreeMap<String, String>(); + this.availableGraphDataFiles.put("relays", "networksize"); + this.availableGraphDataFiles.put("bridges", "networksize"); + this.availableGraphDataFiles.put("relays-by-country", + "relaycountries"); + this.availableGraphDataFiles.put("relays-by-flags", "relayflags"); + this.availableGraphDataFiles.put("relays-by-version", "versions"); + this.availableGraphDataFiles.put("relays-by-platform", "platforms"); + this.availableGraphDataFiles.put("relay-bandwidth", "bandwidth"); + this.availableGraphDataFiles.put("relay-dir-bandwidth", "dirbytes"); + this.availableGraphDataFiles.put("relay-bandwidth-history-by-flags", + "bwhist-flags"); + this.availableGraphDataFiles.put("direct-users-by-country", + "direct-users"); + this.availableGraphDataFiles.put("bridge-users-by-country", + "bridge-users"); + this.availableGraphDataFiles.put("gettor", "gettor"); + this.availableGraphDataFiles.put("torperf", "torperf"); + + /* Initialize map of graphs with specific variable columns. */ + this.variableColumns = new TreeMap<String, String>(); + this.variableColumns.put("relays-by-country", "country"); + this.variableColumns.put("relay-bandwidth-history-by-flags", + "isexit,isguard"); + this.variableColumns.put("torperf", "source"); + + /* Initialize map of graphs with specific value columns. */ + this.valueColumns = new TreeMap<String, String>(); + this.valueColumns.put("relays", "relays"); + this.valueColumns.put("bridges", "bridges"); + + /* Get a reference to the R object generator that we need to generate + * CSV files. */ + this.rObjectGenerator = (RObjectGenerator) getServletContext(). + getAttribute("RObjectGenerator"); + } + + public void doGet(HttpServletRequest request, + HttpServletResponse response) throws IOException, + ServletException { + + /* Find out which JSON file was requested and make sure we know this + * JSON file type. */ + String requestedJsonFile = request.getRequestURI(); + if (requestedJsonFile.contains("/")) { + requestedJsonFile = requestedJsonFile.substring(requestedJsonFile. + lastIndexOf("/") + 1); + } + if (!availableGraphDataFiles.containsKey(requestedJsonFile)) { + logger.info("Did not recognize requested .csv file from request " + + "URI: '" + request.getRequestURI() + "'. Responding with 404 " + + "Not Found."); + response.sendError(HttpServletResponse.SC_NOT_FOUND); + return; + } + String requestedCsvFile = this.availableGraphDataFiles.get( + requestedJsonFile); + logger.fine("CSV file '" + requestedCsvFile + ".csv' requested."); + + /* Prepare filename and R query string. */ + String rQuery = "export_" + requestedCsvFile.replaceAll("-", "_") + + "(path = '%s')"; + String csvFilename = requestedCsvFile + ".csv"; + + /* Request CSV file from R object generator, which asks Rserve to + * generate it. */ + String csvFileContent = this.rObjectGenerator.generateCsv(rQuery, + csvFilename); + + /* Make sure that we have a CSV to convert into JSON. */ + if (csvFileContent == null) { + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return; + } + + /* Convert CSV to JSON format. */ + String jsonString = null; + try { + BufferedReader br = new BufferedReader(new StringReader( + csvFileContent)); + String line; + String[] columns = null; + int dateCol = -1; + SortedSet<Integer> variableCols = new TreeSet<Integer>(); + SortedSet<Integer> valueCols = new TreeSet<Integer>(); + if ((line = br.readLine()) != null) { + columns = line.split(","); + for (int i = 0; i < columns.length; i++) { + if (columns[i].equals("date")) { + dateCol = i; + } else if (this.variableColumns.containsKey(requestedJsonFile) + && this.variableColumns.get(requestedJsonFile).contains( + columns[i])) { + variableCols.add(i); + } else if (!this.valueColumns.containsKey(requestedJsonFile) || + this.valueColumns.get(requestedJsonFile).contains( + columns[i])) { + valueCols.add(i); + } + } + } + if (columns == null || dateCol < 0 || valueCols.isEmpty()) { + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return; + } + SortedMap<String, SortedSet<String>> graphs = + new TreeMap<String, SortedSet<String>>(); + while ((line = br.readLine()) != null) { + String[] elements = line.split(","); + if (elements.length != columns.length) { + response.sendError( + HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return; + } + String date = elements[dateCol]; + String variable = ""; + if (!variableCols.isEmpty()) { + for (int variableCol : variableCols) { + String variableString = elements[variableCol]; + if (variableString.equals("TRUE")) { + variable += columns[variableCol] + "_"; + } else if (variableString.equals("FALSE")) { + variable += "not" + columns[variableCol] + "_"; + } else { + variable += variableString + "_"; + } + } + } + for (int valueCol : valueCols) { + if (elements[valueCol].equals("NA")) { + continue; + } + String graphName = variable + columns[valueCol]; + if (!graphs.containsKey(graphName)) { + graphs.put(graphName, new TreeSet<String>()); + } + String dateAndValue = date + "=" + elements[valueCol]; + graphs.get(graphName).add(dateAndValue); + } + } + StringBuilder sb = new StringBuilder(); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + for (Map.Entry<String, SortedSet<String>> e : graphs.entrySet()) { + String graphName = e.getKey(); + SortedSet<String> datesAndValues = e.getValue(); + if (datesAndValues.isEmpty()) { + continue; + } + String[] firstDateAndValue = datesAndValues.first().split("="); + String firstDate = firstDateAndValue[0]; + String firstValue = firstDateAndValue[1]; + String lastDate = datesAndValues.last().split("=")[0]; + sb.append(",\n\"" + graphName + "\":{" + + "\"first\":\"" + firstDate + "\"," + + "\"last\":\"" + lastDate + "\"," + + "\"values\":["); + int written = 0; + String previousDate = firstDate; + long previousDateMillis = dateFormat.parse(firstDate).getTime(); + for (String dateAndValue : datesAndValues) { + String parts[] = dateAndValue.split("="); + String date = parts[0]; + long dateMillis = dateFormat.parse(date).getTime(); + String value = parts[1]; + while (dateMillis - 86400L * 1000L > previousDateMillis) { + sb.append((written++ > 0 ? "," : "") + "null"); + previousDateMillis += 86400L * 1000L; + previousDate = dateFormat.format(previousDateMillis); + } + sb.append((written++ > 0 ? "," : "") + value); + previousDate = date; + previousDateMillis = dateMillis; + } + sb.append("]}"); + } + br.close(); + jsonString = "[" + sb.toString().substring(1) + "\n]"; + } catch (IOException e) { + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return; + } catch (ParseException e) { + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return; + } + + /* Make sure that we have a graph to return. */ + if (jsonString == null) { + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return; + } + + /* Write JSON file to response. */ + response.setHeader("Access-Control-Allow-Origin", "*"); + response.setContentType("application/json"); + response.setCharacterEncoding("utf-8"); + response.getWriter().print(jsonString); + } +} +

1 0

[metrics-web/master] Remove remaining links to network status page.
by karsten＠torproject.org 06 Mar '12

06 Mar '12

commit df1ab644fd067c072098697a908a73e349783281 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Mar 6 14:44:30 2012 +0100 Remove remaining links to network status page. --- .../ernie/web/ExoneraTorBetaServlet.java | 1 - .../torproject/ernie/web/ExoneraTorServlet.java | 1 - web/WEB-INF/error.jsp | 1 - 3 files changed, 0 insertions(+), 3 deletions(-) diff --git a/src/org/torproject/ernie/web/ExoneraTorBetaServlet.java b/src/org/torproject/ernie/web/ExoneraTorBetaServlet.java index a02b68d..f81d522 100644 --- a/src/org/torproject/ernie/web/ExoneraTorBetaServlet.java +++ b/src/org/torproject/ernie/web/ExoneraTorBetaServlet.java @@ -67,7 +67,6 @@ public class ExoneraTorBetaServlet extends HttpServlet { + " <a href=\"status.html\">Status</a>\n" + " \n" + " \n" - + " <a href=\"networkstatus.html\">Network Status</a>\n" + " <a class=\"current\">ExoneraTor</a>\n" + " <a href=\"relay-search.html\">Relay Search</a>\n" + " <a href=\"consensus-health.html\">Consensus " diff --git a/src/org/torproject/ernie/web/ExoneraTorServlet.java b/src/org/torproject/ernie/web/ExoneraTorServlet.java index da17f3f..aa46367 100644 --- a/src/org/torproject/ernie/web/ExoneraTorServlet.java +++ b/src/org/torproject/ernie/web/ExoneraTorServlet.java @@ -66,7 +66,6 @@ public class ExoneraTorServlet extends HttpServlet { + " <a href=\"status.html\">Status</a>\n" + " \n" + " \n" - + " <a href=\"networkstatus.html\">Network Status</a>\n" + " <a class=\"current\">ExoneraTor</a>\n" + " <a href=\"relay-search.html\">Relay Search</a>\n" + " <a href=\"consensus-health.html\">Consensus " diff --git a/web/WEB-INF/error.jsp b/web/WEB-INF/error.jsp index 71544bb..8e15d35 100644 --- a/web/WEB-INF/error.jsp +++ b/web/WEB-INF/error.jsp @@ -57,7 +57,6 @@ Maybe you find what you're looking for on our sitemap: </ul></li> <li><a href="status.html">Status</a> <ul> -<li><a href="networkstatus.html">Network Status</a></li> <li><a href="exonerator.html">ExoneraTor</a></li> <li><a href="relay-search.html">Relay Search</a></li> <li><a href="consensus-health.html">Consensus Health</a></li>

1 0

[metrics-db/master] Only skip downloading Torperf files younger than 5:30 hours.
by karsten＠torproject.org 06 Mar '12

06 Mar '12

commit c3690c76cff8437b25169d80897fc3ed94167f70 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Mar 6 12:20:02 2012 +0100 Only skip downloading Torperf files younger than 5:30 hours. --- src/org/torproject/ernie/db/TorperfDownloader.java | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/src/org/torproject/ernie/db/TorperfDownloader.java b/src/org/torproject/ernie/db/TorperfDownloader.java index 95d2e30..9506eee 100644 --- a/src/org/torproject/ernie/db/TorperfDownloader.java +++ b/src/org/torproject/ernie/db/TorperfDownloader.java @@ -58,7 +58,7 @@ public class TorperfDownloader { String lastTimestampLine = null; int linesAfterLastTimestampLine = 0; if (outputFile.exists() && outputFile.lastModified() > - System.currentTimeMillis() - 6L * 60L * 60L * 1000L) { + System.currentTimeMillis() - 330L * 60L * 1000L) { return; } else if (outputFile.exists()) { try {

1 0

[metrics-db/master] Update .gitignore.
by karsten＠torproject.org 06 Mar '12

06 Mar '12

commit 53285440eb9433fc45fb0748ac7960aa873d7661 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Mar 6 09:21:20 2012 +0100 Update .gitignore. --- .gitignore | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 03b5dbc..d5a5c14 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ archives/ bridges/ # compiled Java class files -bin/ +classes/ # generated R input files stats/ @@ -44,6 +44,9 @@ dist/ # Possibly modified config file config -# Generated raw database import files -pg-import/ +# Gettor files +gettor/ + +# Files provided via rsync +rsync/

1 0

[metrics-db/master] Download and merge possibly truncated Torperf files.
by karsten＠torproject.org 06 Mar '12

06 Mar '12

commit 138b3397ec300ba64ac48a13175435f7673d65fd Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Mar 6 11:53:39 2012 +0100 Download and merge possibly truncated Torperf files. --- .gitignore | 3 + config.template | 21 ++++ src/org/torproject/ernie/db/Configuration.java | 58 ++++++++++- src/org/torproject/ernie/db/Main.java | 7 + src/org/torproject/ernie/db/TorperfDownloader.java | 116 ++++++++++++++++++++ 5 files changed, 204 insertions(+), 1 deletions(-) diff --git a/.gitignore b/.gitignore index d5a5c14..e119284 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,9 @@ config # Gettor files gettor/ +# Torperf files +torperf/ + # Files provided via rsync rsync/ diff --git a/config.template b/config.template index 6cfdce5..62ff828 100644 --- a/config.template +++ b/config.template @@ -122,4 +122,25 @@ # ## Relative path to the directory that shall be provided via rsync #RsyncDirectory rsync/ +# +# +######## Torperf downloader ######## +# +## Download and merge Torperf .data and .extradata files +#ProcessTorperfFiles 0 +# +## Relative path to the directory to store Torperf files in +#TorperfOutputDirectory torperf/ +# +## Torperf source names and base URLs (option can be contained multiple +## times) +#TorperfSource torperf http://torperf.torproject.org/ +# +## Torperf .data files available on a given source (option can be +## contained multiple times) +#TorperfDataFiles torperf 50kb.data 1mb.data 5mb.data +# +## Torperf .extradata files available on a given source (option can be +## contained multiple times) +#TorperfExtradataFiles torperf 50kb.extradata 1mb.extradata 5mb.extradata diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java index 7197c7d..91f00b1 100644 --- a/src/org/torproject/ernie/db/Configuration.java +++ b/src/org/torproject/ernie/db/Configuration.java @@ -49,6 +49,11 @@ public class Configuration { private boolean processBridgePoolAssignments = false; private String assignmentsDirectory = "assignments/"; private String sanitizedAssignmentsDirectory = "sanitized-assignments/"; + private boolean processTorperfFiles = false; + private String torperfOutputDirectory = "torperf/"; + private SortedMap<String, String> torperfSources = null; + private SortedMap<String, List<String>> torperfDataFiles = null; + private SortedMap<String, List<String>> torperfExtradataFiles = null; private boolean provideFilesViaRsync = false; private String rsyncDirectory = "rsync"; public Configuration() { @@ -170,6 +175,41 @@ public class Configuration { this.assignmentsDirectory = line.split(" ")[1]; } else if (line.startsWith("SanitizedAssignmentsDirectory")) { this.sanitizedAssignmentsDirectory = line.split(" ")[1]; + } else if (line.startsWith("ProcessTorperfFiles")) { + this.processTorperfFiles = Integer.parseInt(line.split(" ")[1]) + != 0; + } else if (line.startsWith("TorperfOutputDirectory")) { + } else if (line.startsWith("TorperfSource")) { + if (this.torperfSources == null) { + this.torperfSources = new TreeMap<String, String>(); + } + String[] parts = line.split(" "); + String sourceName = parts[1]; + String baseUrl = parts[2]; + this.torperfSources.put(sourceName, baseUrl); + } else if (line.startsWith("TorperfDataFiles")) { + if (this.torperfDataFiles == null) { + this.torperfDataFiles = new TreeMap<String, List<String>>(); + } + String[] parts = line.split(" "); + String sourceName = parts[1]; + List<String> dataFiles = new ArrayList<String>(); + for (int i = 2; i < parts.length; i++) { + dataFiles.add(parts[i]); + } + this.torperfDataFiles.put(sourceName, dataFiles); + } else if (line.startsWith("TorperfExtradataFiles")) { + if (this.torperfExtradataFiles == null) { + this.torperfExtradataFiles = + new TreeMap<String, List<String>>(); + } + String[] parts = line.split(" "); + String sourceName = parts[1]; + List<String> extradataFiles = new ArrayList<String>(); + for (int i = 2; i < parts.length; i++) { + extradataFiles.add(parts[i]); + } + this.torperfExtradataFiles.put(sourceName, extradataFiles); } else if (line.startsWith("ProvideFilesViaRsync")) { this.provideFilesViaRsync = Integer.parseInt( line.split(" ")[1]) != 0; @@ -205,7 +245,8 @@ public class Configuration { !this.importDirectoryArchives && !this.downloadRelayDescriptors && !this.importBridgeSnapshots && !this.downloadGetTorStats && !this.downloadExitList && !this.processBridgePoolAssignments && - !this.writeDirectoryArchives && !this.writeSanitizedBridges) { + !this.writeDirectoryArchives && !this.writeSanitizedBridges && + !this.processTorperfFiles) { logger.warning("We have not been configured to read data from any " + "data source or write data to any data sink. You need to " + "edit your config file (" + configFile.getAbsolutePath() @@ -324,6 +365,21 @@ public class Configuration { public String getSanitizedAssignmentsDirectory() { return sanitizedAssignmentsDirectory; } + public boolean getProcessTorperfFiles() { + return this.processTorperfFiles; + } + public String getTorperfOutputDirectory() { + return this.torperfOutputDirectory; + } + public SortedMap<String, String> getTorperfSources() { + return this.torperfSources; + } + public SortedMap<String, List<String>> getTorperfDataFiles() { + return this.torperfDataFiles; + } + public SortedMap<String, List<String>> getTorperfExtradataFiles() { + return this.torperfExtradataFiles; + } public boolean getProvideFilesViaRsync() { return this.provideFilesViaRsync; } diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java index ab79174..a5b5080 100644 --- a/src/org/torproject/ernie/db/Main.java +++ b/src/org/torproject/ernie/db/Main.java @@ -136,6 +136,13 @@ public class Main { new File(config.getSanitizedAssignmentsDirectory())); } + // Process Torperf files + if (config.getProcessTorperfFiles()) { + new TorperfDownloader(new File(config.getTorperfOutputDirectory()), + config.getTorperfSources(), config.getTorperfDataFiles(), + config.getTorperfExtradataFiles()); + } + // Copy recently published files to a local directory that can then // be served via rsync. if (config.getProvideFilesViaRsync()) { diff --git a/src/org/torproject/ernie/db/TorperfDownloader.java b/src/org/torproject/ernie/db/TorperfDownloader.java new file mode 100644 index 0000000..95d2e30 --- /dev/null +++ b/src/org/torproject/ernie/db/TorperfDownloader.java @@ -0,0 +1,116 @@ +/* Copyright 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.db; + +import java.io.*; +import java.net.*; +import java.util.*; +import java.util.logging.*; + +/* Download possibly truncated Torperf .data and .extradata files from + * configured sources and append them to the files we already have. */ +public class TorperfDownloader { + + private File torperfOutputDirectory = null; + private SortedMap<String, String> torperfSources = null; + private SortedMap<String, List<String>> torperfDataFiles = null; + private SortedMap<String, List<String>> torperfExtradataFiles = null; + private Logger logger = null; + + public TorperfDownloader(File torperfOutputDirectory, + SortedMap<String, String> torperfSources, + SortedMap<String, List<String>> torperfDataFiles, + SortedMap<String, List<String>> torperfExtradataFiles) { + if (torperfOutputDirectory == null) { + throw new IllegalArgumentException(); + } + this.torperfOutputDirectory = torperfOutputDirectory; + this.torperfSources = torperfSources; + this.torperfDataFiles = torperfDataFiles; + this.torperfExtradataFiles = torperfExtradataFiles; + if (!this.torperfOutputDirectory.exists()) { + this.torperfOutputDirectory.mkdirs(); + } + this.logger = Logger.getLogger(TorperfDownloader.class.getName()); + this.downloadAndMergeFiles(this.torperfDataFiles, true); + this.downloadAndMergeFiles(this.torperfExtradataFiles, false); + } + + private void downloadAndMergeFiles( + SortedMap<String, List<String>> dataOrExtradataFiles, + boolean isDataFile) { + for (Map.Entry<String, List<String>> e : + dataOrExtradataFiles.entrySet()) { + String sourceName = e.getKey(); + String sourceBaseUrl = torperfSources.get(sourceName); + List<String> files = e.getValue(); + for (String file : files) { + String url = sourceBaseUrl + file; + File outputFile = new File(torperfOutputDirectory, + sourceName + "-" + file); + this.downloadAndMergeFile(url, outputFile, isDataFile); + } + } + } + + private void downloadAndMergeFile(String url, File outputFile, + boolean isDataFile) { + String lastTimestampLine = null; + int linesAfterLastTimestampLine = 0; + if (outputFile.exists() && outputFile.lastModified() > + System.currentTimeMillis() - 6L * 60L * 60L * 1000L) { + return; + } else if (outputFile.exists()) { + try { + BufferedReader br = new BufferedReader(new FileReader( + outputFile)); + String line; + while ((line = br.readLine()) != null) { + if (isDataFile || line.contains(" LAUNCH")) { + lastTimestampLine = line; + linesAfterLastTimestampLine = 0; + } else { + linesAfterLastTimestampLine++; + } + } + br.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed reading '" + + outputFile.getAbsolutePath() + "' to find the last line to " + + "append to.", e); + return; + } + } + try { + this.logger.fine("Downloading " + (isDataFile ? ".data" : + ".extradata") + " file from '" + url + "' and merging it into " + + "'" + outputFile.getAbsolutePath() + "'."); + URL u = new URL(url); + HttpURLConnection huc = (HttpURLConnection) u.openConnection(); + huc.setRequestMethod("GET"); + huc.connect(); + BufferedReader br = new BufferedReader(new InputStreamReader( + huc.getInputStream())); + String line; + BufferedWriter bw = new BufferedWriter(new FileWriter(outputFile, + true)); + boolean copyLines = lastTimestampLine == null; + while ((line = br.readLine()) != null) { + if (copyLines && linesAfterLastTimestampLine == 0) { + bw.write(line + "\n"); + } else if (copyLines && linesAfterLastTimestampLine > 0) { + linesAfterLastTimestampLine--; + } else if (line.equals(lastTimestampLine)) { + copyLines = true; + } + } + bw.close(); + br.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed downloading and merging '" + url + + "'.", e); + return; + } + } +} +

1 0

r25508: {check} fix the path to the state files for tordnsel for now. (check/trunk/cgi-bin)
by Andrew Lewman 06 Mar '12

06 Mar '12

Author: phobos Date: 2012-03-06 04:28:13 +0000 (Tue, 06 Mar 2012) New Revision: 25508 Modified: check/trunk/cgi-bin/TorBulkExitList.py Log: fix the path to the state files for tordnsel for now. Modified: check/trunk/cgi-bin/TorBulkExitList.py =================================================================== --- check/trunk/cgi-bin/TorBulkExitList.py 2012-03-05 22:13:51 UTC (rev 25507) +++ check/trunk/cgi-bin/TorBulkExitList.py 2012-03-06 04:28:13 UTC (rev 25508) @@ -72,7 +72,7 @@ # RawExitListURL = "http://exitlist.torproject.org/exitAddresses" # Currently fake this and return a static file: - RawExitList = '/srv/check.torproject.org/tordnsel/state/exit-addresses' + RawExitList = '/srv/tordnsel.torproject.org/srv/tordnsel.torproject.org/state/exit-addresses' return RawExitList

1 0

[ooni-probe/master] Add a HACKING file.
by art＠torproject.org 06 Mar '12

06 Mar '12

commit 374d2a6c241d56a3ee20204f1b44f81bc005cec1 Author: Arturo Filastò <hellais(a)gmail.com> Date: Mon Mar 5 18:16:22 2012 -0800 Add a HACKING file. * Describes how the code is organized and what style conventions should be used * TODO: make all the code conform to the style conventions --- HACKING | 253 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 253 insertions(+), 0 deletions(-) diff --git a/HACKING b/HACKING new file mode 100644 index 0000000..ed8d5dc --- /dev/null +++ b/HACKING @@ -0,0 +1,253 @@ +Hacking on OONI +*************** + +This documents gives guidelines on where to start looking +for helping out in developing OONI and what guidelines you +should follow when writing code. + +We try to follow the general python best practices and styling +guides as specified in PEP. + + Beautiful is better than ugly. + Explicit is better than implicit. + Simple is better than complex. + Complex is better than complicated. + Flat is better than nested. + Sparse is better than dense. + Readability counts. + Special cases aren't special enough to break the rules. + Although practicality beats purity. + Errors should never pass silently. + Unless explicitly silenced. + In the face of ambiguity, refuse the temptation to guess. + There should be one-- and preferably only one --obvious way to do it. + Although that way may not be obvious at first unless you're Dutch. + Now is better than never. + Although never is often better than *right* now. + If the implementation is hard to explain, it's a bad idea. + If the implementation is easy to explain, it may be a good idea. + Namespaces are one honking great idea -- let's do more of those! + + - Tim Peters, The Zen of Python + +Code Structure +--------- + +- HACKING + The document you are currently reading. + +- assets/ + Where we store all the asset files that are + used when running OONI tests. + +- config.py + Parts of the code related to parsing OONI + configuration files and making them accessible + to other components of the software. + +- logo.py + File containing some funny ASCII art. Yes, we + do enjoy ASCII art and are not afraid to admit it! + +- nodes.conf + The configuration file for nodes. This contains the + list of network and code execution nodes that can be + used to run tests off of. + +- ooniprobe.py + The main OONI-probe command line interface. This is + responsible for parsing the command line arguments and + passing the arguments to the underlying components. + +- ooni-probe.conf + The main OONI-probe configuration file. This can be used + to configure your OONI CLI, tell it where it should report + to, where the asset files are located, what should be used + for control, etc. + +- plugoo/__init__.py + All the necessary "goo" for making OONI probe work. This + means loading Assets, creating Reports, running Tests, + interacting with Nodes. + +- plugoo/assets.py + This is a python object representation of the data that is + located inside the asset directory. + +- plugoo/nodes.py + The part of code responsible for interacting with OONI Nodes. + Nodes can be Network or Code Execution. Network nodes are + capable of receiving packets and fowarding them onto their + network. This means that to run a test on X network nodes you + consume X*test_cost bandwith. Code Execution nodes accept units + of work, they are therefore capable of receiving a set of tests + that should be completed by a set of Network nodes or run locally. + +- plugoo/reports.py + Takes care of transforming the output of a test into a report. This + may mean pushing the result data to a remote backend or simply writing + a local file. + +- plugoo/tests.py + The main OONI probe test class. This provides all the necessary scaffold + to write your own test based on the OONI paradigm. + +- tests/ + Contains all the "offical" OONI tests that are shipped. + +- utils.py + Helper functions that don't fit into any place, but are not big enough to + be a dependency by themselves. + +Style guide +----------- + +This is an extract of the most important parts of PEP-8. When in doubt on +what code style should be followed first consult this doc, then PEP-8 and +if all fails use your best judgement or ask for help. + - Art. + +Indentation +........... + + Use 4 spaces per indentation level. + + This can be setup in vi with: + set tabstop=4 + set shiftwidth=4 + set expandtab + + + Continuation lines should be wrapper like this: + + foo = long_function_name(var_one, var_two, + var_three, var_four) + + or this: + + def long_function_name(var_one, + var_two, var_three, + var_four): + print(var_one) + + + They should NOT be wrapper like this: + + foo = long_function_name(var_one, var_two, + var_three, var_four) + + and NOT like this: + + # See how it creates confusion with what is inside the function? + def long_function_name(var_one, + var_two, var_three, + var_four): + print(var_one) + + +Tabs or Spaces? +............... + +Everytime you insert a \t into any piece of code a kitten dies. + +Only spaces. Please. + +(code should be run with python -tt) + +Maximum Line Length +................... + +Maximum of 79 characters. 72 characters for long blocks of text is recommended. + +Blank Lines +........... + +Separate top-level function and class definitions with two blank lines. + +Method definitions inside of class are separated by a single blank line. + + +Encoding +........ + +Always use UTF-8 encoding. This can be specified by add the encoding cookie +to the beginning of your python files: + + # -*- coding: UTF-8 + +All identifiers should be ASCII-only. All doc strings and comments should also +only be in ASCII. Non ASCII characters are allowed when they are related to +testing non-ASCII features or for the names of authors. + + +Imports +....... + +Import should be one per line as so: + + import os + import sys + from subprocess import Popen, PIPE + +Imports are always at the top of the file just after any module comments +and docstrings, berfore module globals and constants. + +Imports should be grouped in the following order: + +1. standard library imports +2. related third party imports +3. local application/library specific imports + +You should put a blank line between each group of imports. + + +Comments +........ + +Comments should always be up to date with the code. Don't have +comments that contraddict with the code. + +Comments should always be written in English. + +Blocks comments are indented to the same level of the code that +they refer to. They start with # and are followed by a single space. + +Use inline comments sparingly. # Gotcha? + + +Documentation strings +..................... + +Write docstrings for all public modules, functions, classes and +methods. Even better if you write them also for non-public methods. + +Place docstrings under the def. + +For a better overview on how to write docstrings consult: PEP-257 + + +Naming convention +................. + +Avoid using 'l' (lowercase letter el), 'O' (uppercase letter oh) or +I (uppercase letter eye) as single character variable names. + +Module names should have short, all-lowercase names. Underscores can be +used in the module name if it improves readability. Python packages should +also have short, all-lowercase names, although the use of underscores is +discouraged. + +Class names should follow the CapWords convention. +Note: When using abbreviations in CapWords, capitalize all the letters + of the abbreviation. Thus HTTPServerError is better than + HttpServerError. + +Exception names should follow the class names convention as exceptions +should be classes. + +Function names should be all lowercase with words separated by underscores +to improve readability. The same goes for Global Variable names. + +Method names should be all lowercase. Non-public methods should start with +an underscore. The same applies to instance variables. +

1 0

[obfsproxy/master] Update changelog for #5312.
by asn＠torproject.org 06 Mar '12

06 Mar '12

commit a1ea237f723cf92f45f77122148fd667a813e2e9 Author: George Kadianakis <desnacked(a)riseup.net> Date: Mon Mar 5 17:25:46 2012 -0800 Update changelog for #5312. --- ChangeLog | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0b8663e..93a9bff 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,10 @@ Changes in version 0.1.1 - <not yet released> - Add trailing slash to the torproject URL in the man page. Fixes bug #5311. Patch by Peter Palfrader. + - Add LICENSE and Doxyfile to the release archives from now on. + Fixes bug #5312. Patch by Peter Palfrader. + + Changes in version 0.1.0 - 2012-03-05 - Initial release.

1 0