- tor-commits - lists.torproject.org

[metrics-tasks/master] #6329 Only show families as accepted by consensus (mutually listed relays)
by karsten＠torproject.org 07 Aug '12

07 Aug '12

commit 4f74b76c9f669db5dbebd2e8ed09b20c5e2d54a0 Author: Michael Zeltner <m(a)niij.org> Date: Tue Aug 7 14:35:32 2012 +0200 #6329 Only show families as accepted by consensus (mutually listed relays) --- task-6329/tor-relays-stats.py | 21 ++++++++++----------- 1 files changed, 10 insertions(+), 11 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index d966f66..733bb55 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -44,17 +44,16 @@ class FamilyFilter(BaseFilter): self._family_relays = [self._family_fingerprint] + found_relay.get('family', []) def accept(self, relay): - fingerprint = '$%s' % relay['fingerprint'] - mentions = [fingerprint] + relay.get('family', []) - if fingerprint in self._family_relays: - return True - if 'Named' in relay['flags'] and relay['nickname'] in self._family_relays: - return True - if self._family_fingerprint in mentions: - return True - if self._family_nickname in mentions: - return True - return False + fingerprint = '$%s' % relay['fingerprint'] + mentions = [fingerprint] + relay.get('family', []) + # Only show families as accepted by consensus (mutually listed relays) + listed = fingerprint in self._family_relays + listed = listed or 'Named' in relay['flags'] and relay['nickname'] in self._family_relays + mentioned = self._family_fingerprint in mentions + mentioned = mentioned or self._family_nickname in mentions + if listed and mentioned: + return True + return False class CountryFilter(BaseFilter): def __init__(self, countries=[]):

1 0

[metrics-web/master] Remove unused imports.
by karsten＠torproject.org 07 Aug '12

07 Aug '12

commit 15363f970e8e200eb0dd81de9eaa96aab80e4125 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Aug 7 12:34:07 2012 +0200 Remove unused imports. --- src/org/torproject/ernie/web/CsvServlet.java | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) diff --git a/src/org/torproject/ernie/web/CsvServlet.java b/src/org/torproject/ernie/web/CsvServlet.java index 6fdcaac..2467d55 100644 --- a/src/org/torproject/ernie/web/CsvServlet.java +++ b/src/org/torproject/ernie/web/CsvServlet.java @@ -2,10 +2,7 @@ * See LICENSE for licensing information */ package org.torproject.ernie.web; -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStreamReader; import java.util.SortedSet; import java.util.logging.Logger;

1 0

[metrics-web/master] Remove GetTor statistics processing code.
by karsten＠torproject.org 07 Aug '12

07 Aug '12

commit 27869c49223db85aa05baad14ffc619708d9e661 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Aug 7 12:33:09 2012 +0200 Remove GetTor statistics processing code. --- README | 25 +--- config.template | 6 - db/tordir.sql | 9 -- rserve/csv.R | 35 ----- rserve/graphs.R | 39 ----- src/org/torproject/ernie/cron/Configuration.java | 13 -- src/org/torproject/ernie/cron/GetTorProcessor.java | 152 -------------------- src/org/torproject/ernie/cron/Main.java | 7 - src/org/torproject/ernie/web/GraphDataServlet.java | 1 - src/org/torproject/ernie/web/RObjectGenerator.java | 2 - 10 files changed, 1 insertions(+), 288 deletions(-) diff --git a/README b/README index 8b3651f..1c551e0 100644 --- a/README +++ b/README @@ -20,8 +20,7 @@ or only the database and the graphing engine, if desired. The metrics database contains data about the Tor Network coming from different sources, including the Tor directory authorities, Torperf -performance measurement installations, the GetTor software package -delivery service, and others. +performance measurement installations, and others. 1.1. Preparing the operating system @@ -289,28 +288,6 @@ Run the database import: $ ./run.sh -1.10. Importing GetTor statistics -================================= - -WARNING: The GetTor statistics are not available for download yet, so that -this section only applies to the official metrics website. - -GetTor is a software distribution service that allows users to fetch the -Tor software via email. GetTor produces daily statistics of requested -packages that can be imported into the metrics database. - -Put the GetTor statistics file into /srv/metrics-web/gettor/ . - -Edit /srv/metrics-web/config to contain the following options: - -ProcessGetTorStats 1 -GetTorDirectory gettor/ - -Run the database import: - -$ ./run.sh - - 2. Installing the graphing engine ================================= diff --git a/config.template b/config.template index 528b36f..0491431 100644 --- a/config.template +++ b/config.template @@ -45,12 +45,6 @@ ## Relative path to directory to import torperf results from #TorperfDirectory in/torperf/ # -## Process GetTor stats and import them into the database -#ProcessGetTorStats 0 -# -## Relative path to directory where to find GetTor stats -#GetTorDirectory in/gettor/ -# ## JDBC string for ExoneraTor database #ExoneraTorDatabaseJdbc jdbc:postgresql://localhost/exonerator?user=metrics&password=password # diff --git a/db/tordir.sql b/db/tordir.sql index 94c9320..93820bc 100644 --- a/db/tordir.sql +++ b/db/tordir.sql @@ -899,15 +899,6 @@ CREATE TABLE torperf_stats ( CONSTRAINT torperf_stats_pkey PRIMARY KEY("date", source) ); --- TABLE gettor_stats --- Packages requested from GetTor -CREATE TABLE gettor_stats ( - "date" DATE NOT NULL, - bundle CHARACTER VARYING(64) NOT NULL, - downloads INTEGER NOT NULL, - CONSTRAINT gettor_stats_pkey PRIMARY KEY("date", bundle) -); - -- Refresh all statistics in the database. CREATE OR REPLACE FUNCTION refresh_all() RETURNS INTEGER AS $$ BEGIN diff --git a/rserve/csv.R b/rserve/csv.R index 280d7b6..eec76d9 100644 --- a/rserve/csv.R +++ b/rserve/csv.R @@ -182,41 +182,6 @@ export_bridge_users <- function(path) { write.csv(bridgeusers, path, quote = FALSE, row.names = FALSE) } -export_gettor <- function(path) { - drv <- dbDriver("PostgreSQL") - con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) - q <- paste("SELECT date, bundle, downloads FROM gettor_stats", - "WHERE date < current_date - 1") - rs <- dbSendQuery(con, q) - downloads <- fetch(rs, n = -1) - dbDisconnect(con) - dbUnloadDriver(drv) - downloads_total <- downloads[downloads$bundle != "none", ] - downloads_total <- aggregate(downloads_total$downloads, - by = list(date = downloads_total$date), sum) - downloads_en <- downloads[grep("*_en", downloads$bundle), ] - downloads_en <- aggregate(downloads_en$downloads, - by = list(date = downloads_en$date), sum) - downloads_zh_cn <- downloads[grep("*_zh_cn", downloads$bundle), ] - downloads_zh_cn <- aggregate(downloads_zh_cn$downloads, - by = list(date = downloads_zh_cn$date), sum) - downloads_fa <- downloads[grep("*_fa", downloads$bundle), ] - downloads_fa <- aggregate(downloads_fa$downloads, - by = list(date = downloads_fa$date), sum) - downloads <- rbind( - data.frame(date = downloads_total$date, - bundle = "total", downloads = downloads_total$x), - data.frame(date = downloads_en$date, - bundle = "en", downloads = downloads_en$x), - data.frame(date = downloads_zh_cn$date, - bundle = "zh_cn", downloads = downloads_zh_cn$x), - data.frame(date = downloads_fa$date, - bundle = "fa", downloads = downloads_fa$x)) - downloads <- cast(downloads, date ~ bundle, value = "downloads") - downloads <- downloads[order(downloads$date), ] - write.csv(downloads, path, quote = FALSE, row.names = FALSE) -} - export_torperf <- function(path) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) diff --git a/rserve/graphs.R b/rserve/graphs.R index a441d86..87917aa 100644 --- a/rserve/graphs.R +++ b/rserve/graphs.R @@ -781,45 +781,6 @@ plot_bridge_users <- function(start, end, country, path, dpi) { ggsave(filename = path, width = 8, height = 5, dpi = as.numeric(dpi)) } -plot_gettor <- function(start, end, language, path, dpi) { - drv <- dbDriver("PostgreSQL") - con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) - condition <- ifelse(language == "all", "<> 'none'", - paste("LIKE '%_", tolower(language), "'", sep = "")) - q <- paste("SELECT date, SUM(downloads) AS downloads ", - "FROM gettor_stats WHERE bundle ", condition, " AND date >= '", - start, "' AND date <= '", end, - "' AND date < current_date - 1 GROUP BY date", sep = "") - rs <- dbSendQuery(con, q) - downloads <- fetch(rs, n = -1) - dbDisconnect(con) - dbUnloadDriver(drv) - dates <- seq(from = as.Date(start, "%Y-%m-%d"), - to = as.Date(end, "%Y-%m-%d"), by="1 day") - missing <- setdiff(dates, downloads$date) - if (length(missing) > 0) - downloads <- rbind(downloads, - data.frame(date = as.Date(missing, origin = "1970-01-01"), - downloads = NA)) - title <- ifelse(language == "all", - "Total packages requested from GetTor per day\n", - paste(languagename(language), " (", language, - ") packages requested from GetTor per day\n", sep = "")) - date_breaks <- date_breaks( - as.numeric(max(as.Date(downloads$date, "%Y-%m-%d")) - - min(as.Date(downloads$date, "%Y-%m-%d")))) - ggplot(downloads, aes(x = as.Date(date, "%Y-%m-%d"), y = downloads)) + - geom_line(size = 1) + - scale_x_date(name = paste("\nThe Tor Project - ", - "https://metrics.torproject.org/", sep = ""), - format = date_breaks$format, major = date_breaks$major, - minor = date_breaks$minor) + - scale_y_continuous(name = "", limits = c(0, max(downloads$downloads, - na.rm = TRUE))) + - opts(title = title) - ggsave(filename = path, width = 8, height = 5, dpi = as.numeric(dpi)) -} - plot_torperf <- function(start, end, source, filesize, path, dpi) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) diff --git a/src/org/torproject/ernie/cron/Configuration.java b/src/org/torproject/ernie/cron/Configuration.java index 5cdadaf..878e882 100644 --- a/src/org/torproject/ernie/cron/Configuration.java +++ b/src/org/torproject/ernie/cron/Configuration.java @@ -30,8 +30,6 @@ public class Configuration { private boolean writeBridgeStats = false; private boolean importWriteTorperfStats = false; private String torperfDirectory = "in/torperf/"; - private boolean processGetTorStats = false; - private String getTorDirectory = "in/gettor/"; private String exoneraTorDatabaseJdbc = "jdbc:postgresql:" + "//localhost/exonerator?user=metrics&password=password"; private String exoneraTorImportDirectory = "exonerator-import/"; @@ -87,11 +85,6 @@ public class Configuration { line.split(" ")[1]) != 0; } else if (line.startsWith("TorperfDirectory")) { this.torperfDirectory = line.split(" ")[1]; - } else if (line.startsWith("ProcessGetTorStats")) { - this.processGetTorStats = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("GetTorDirectory")) { - this.getTorDirectory = line.split(" ")[1]; } else if (line.startsWith("ExoneraTorDatabaseJdbc")) { this.exoneraTorDatabaseJdbc = line.split(" ")[1]; } else if (line.startsWith("ExoneraTorImportDirectory")) { @@ -160,12 +153,6 @@ public class Configuration { public String getTorperfDirectory() { return this.torperfDirectory; } - public boolean getProcessGetTorStats() { - return this.processGetTorStats; - } - public String getGetTorDirectory() { - return this.getTorDirectory; - } public String getExoneraTorDatabaseJdbc() { return this.exoneraTorDatabaseJdbc; } diff --git a/src/org/torproject/ernie/cron/GetTorProcessor.java b/src/org/torproject/ernie/cron/GetTorProcessor.java deleted file mode 100644 index d561784..0000000 --- a/src/org/torproject/ernie/cron/GetTorProcessor.java +++ /dev/null @@ -1,152 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.cron; - -import java.io.File; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.text.SimpleDateFormat; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; -import org.torproject.descriptor.DescriptorReader; -import org.torproject.descriptor.DescriptorSourceFactory; -import org.torproject.descriptor.GetTorStatistics; - -public class GetTorProcessor { - public GetTorProcessor(File getTorDirectory, String connectionURL) { - - Logger logger = Logger.getLogger(GetTorProcessor.class.getName()); - - /* Parse stats file. */ - File getTorFile = new File(getTorDirectory, "gettor_stats.txt"); - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - if (!getTorFile.exists() || getTorFile.isDirectory()) { - logger.warning("Could not read GetTor stats"); - return; - } - SortedSet<String> columns = new TreeSet<String>(); - SortedMap<String, Map<String, Integer>> data = - new TreeMap<String, Map<String, Integer>>(); - - logger.fine("Importing GetTor stats files in directory " - + getTorDirectory + "/..."); - DescriptorReader reader = - DescriptorSourceFactory.createDescriptorReader(); - reader.addDirectory(getTorDirectory); - Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - if (descriptorFile.getException() != null) { - logger.log(Level.WARNING, "Could not parse descriptor file '" - + descriptorFile.getFileName() + "'. Skipping.", - descriptorFile.getException()); - continue; - } - if (descriptorFile.getDescriptors() != null) { - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (!(descriptor instanceof GetTorStatistics)) { - continue; - } - GetTorStatistics stats = (GetTorStatistics) descriptor; - String date = dateFormat.format(stats.getDateMillis()); - Map<String, Integer> obs = new HashMap<String, Integer>(); - for (Map.Entry<String, Integer> e : - stats.getDownloadedPackages().entrySet()) { - columns.add(e.getKey().toLowerCase()); - obs.put(e.getKey().toLowerCase(), e.getValue()); - } - data.put(date, obs); - } - } - } - - /* Write results to database. */ - if (connectionURL != null) { - try { - Map<String, Integer> updateRows = new HashMap<String, Integer>(), - insertRows = new HashMap<String, Integer>(); - for (Map.Entry<String, Map<String, Integer>> e : - data.entrySet()) { - String date = e.getKey(); - Map<String, Integer> obs = e.getValue(); - for (String column : columns) { - if (obs.containsKey(column)) { - Integer value = obs.get(column); - String key = date + "," + column; - insertRows.put(key, value); - } - } - } - Connection conn = DriverManager.getConnection(connectionURL); - PreparedStatement psI = conn.prepareStatement( - "INSERT INTO gettor_stats (downloads, date, bundle) " - + "VALUES (?, ?, ?)"); - PreparedStatement psU = conn.prepareStatement( - "UPDATE gettor_stats SET downloads = ? " - + "WHERE date = ? AND bundle = ?"); - conn.setAutoCommit(false); - Statement statement = conn.createStatement(); - ResultSet rs = statement.executeQuery( - "SELECT date, bundle, downloads FROM gettor_stats"); - while (rs.next()) { - String date = rs.getDate(1).toString(); - String bundle = rs.getString(2); - String key = date + "," + bundle; - if (insertRows.containsKey(key)) { - int insertRow = insertRows.remove(key); - int oldCount = rs.getInt(3); - if (insertRow != oldCount) { - updateRows.put(key, insertRow); - } - } - } - for (Map.Entry<String, Integer> e : updateRows.entrySet()) { - String[] keyParts = e.getKey().split(","); - java.sql.Date date = java.sql.Date.valueOf(keyParts[0]); - String bundle = keyParts[1]; - int downloads = e.getValue(); - psU.clearParameters(); - psU.setLong(1, downloads); - psU.setDate(2, date); - psU.setString(3, bundle); - psU.executeUpdate(); - } - for (Map.Entry<String, Integer> e : insertRows.entrySet()) { - String[] keyParts = e.getKey().split(","); - java.sql.Date date = java.sql.Date.valueOf(keyParts[0]); - String bundle = keyParts[1]; - int downloads = e.getValue(); - psI.clearParameters(); - psI.setLong(1, downloads); - psI.setDate(2, date); - psI.setString(3, bundle); - psI.executeUpdate(); - } - conn.commit(); - conn.close(); - } catch (SQLException e) { - logger.log(Level.WARNING, "Failed to add GetTor stats to " - + "database.", e); - } - } - - logger.info("Finished processing statistics on Tor packages " - + "delivered by GetTor.\nLast date in statistics is " - + (data.isEmpty() ? "(null)" : data.lastKey()) + "."); - } -} - diff --git a/src/org/torproject/ernie/cron/Main.java b/src/org/torproject/ernie/cron/Main.java index 8022fcc..6c98602 100644 --- a/src/org/torproject/ernie/cron/Main.java +++ b/src/org/torproject/ernie/cron/Main.java @@ -84,13 +84,6 @@ public class Main { statsDirectory, config.getRelayDescriptorDatabaseJDBC()); } - // Download and process GetTor stats - if (config.getProcessGetTorStats()) { - new GetTorProcessor( - new File(config.getGetTorDirectory()), - config.getRelayDescriptorDatabaseJDBC()); - } - // Remove lock file lf.releaseLock(); diff --git a/src/org/torproject/ernie/web/GraphDataServlet.java b/src/org/torproject/ernie/web/GraphDataServlet.java index 5b79c1b..11800d7 100644 --- a/src/org/torproject/ernie/web/GraphDataServlet.java +++ b/src/org/torproject/ernie/web/GraphDataServlet.java @@ -68,7 +68,6 @@ public class GraphDataServlet extends HttpServlet { "direct-users"); this.availableGraphDataFiles.put("bridge-users-by-country", "bridge-users"); - this.availableGraphDataFiles.put("gettor", "gettor"); this.availableGraphDataFiles.put("torperf", "torperf"); /* Initialize map of graphs with specific variable columns. */ diff --git a/src/org/torproject/ernie/web/RObjectGenerator.java b/src/org/torproject/ernie/web/RObjectGenerator.java index 3a856c6..d8517c1 100644 --- a/src/org/torproject/ernie/web/RObjectGenerator.java +++ b/src/org/torproject/ernie/web/RObjectGenerator.java @@ -61,7 +61,6 @@ public class RObjectGenerator implements ServletContextListener { this.availableCsvFiles.add("direct-users"); this.availableCsvFiles.add("dirreq-stats"); this.availableCsvFiles.add("dirbytes"); - this.availableCsvFiles.add("gettor"); this.availableCsvFiles.add("monthly-users-average"); this.availableCsvFiles.add("monthly-users-peak"); this.availableCsvFiles.add("networksize"); @@ -95,7 +94,6 @@ public class RObjectGenerator implements ServletContextListener { "start,end,country,events,filename,nocutoff,dpi"); this.availableGraphs.put("bridge-users", "start,end,country,filename,dpi"); - this.availableGraphs.put("gettor", "start,end,language,filename,dpi"); this.availableGraphs.put("torperf", "start,end,source,filesize,filename,dpi"); this.availableGraphs.put("torperf-failures",

1 0

[metrics-db/master] Remove GetTor statistics processing code.
by karsten＠torproject.org 07 Aug '12

07 Aug '12

commit 8d8bdf1331699dc5ea3a97a2bc9436548ccfae3a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Aug 7 12:46:12 2012 +0200 Remove GetTor statistics processing code. --- .gitignore | 4 - config.template | 12 -- src/org/torproject/ernie/db/Configuration.java | 25 +---- src/org/torproject/ernie/db/GetTorDownloader.java | 115 -------------------- src/org/torproject/ernie/db/Main.java | 8 -- src/org/torproject/ernie/db/RsyncDataProvider.java | 17 +--- 6 files changed, 2 insertions(+), 179 deletions(-) diff --git a/.gitignore b/.gitignore index 8c9553c..51ae9b7 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,6 @@ website/graphs/bridge-users/*.png website/graphs/descriptors/*.png website/graphs/direct-users/*.png website/graphs/exit/*.png -website/graphs/gettor/*.png website/graphs/networksize/*.png website/graphs/new-users/*.png website/graphs/torperf/*.png @@ -47,9 +46,6 @@ dist/ # Possibly modified config file config -# Gettor files -gettor/ - # Torperf files torperf/ diff --git a/config.template b/config.template index aecc67d..52ee56f 100644 --- a/config.template +++ b/config.template @@ -96,18 +96,6 @@ #SanitizedAssignmentsDirectory sanitized-assignments/ # # -######## GetTor download statistics ######## -# -## Download GetTor stats -#DownloadGetTorStats 0 -# -## URL to download GetTor stats from -#GetTorStatsURL http://gettor.torproject.org:8080/~gettor/gettor_stats.txt -# -## Relative path to directory to store GetTor stats in -#GetTorDirectory gettor/ -# -# ######## Exit lists ######## # ## Download exit list and store it to disk diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java index 63349c8..e130dab 100644 --- a/src/org/torproject/ernie/db/Configuration.java +++ b/src/org/torproject/ernie/db/Configuration.java @@ -48,10 +48,6 @@ public class Configuration { private boolean downloadAllServerDescriptors = false; private boolean downloadAllExtraInfoDescriptors = false; private boolean compressRelayDescriptorDownloads; - private boolean downloadGetTorStats = false; - private String getTorStatsUrl = "http://gettor.torproject.org:8080/" - + "~gettor/gettor_stats.txt"; - private String getTorDirectory = "gettor/"; private boolean downloadExitList = false; private boolean processBridgePoolAssignments = false; private String assignmentsDirectory = "assignments/"; @@ -161,16 +157,6 @@ public class Configuration { } else if (line.startsWith("CompressRelayDescriptorDownloads")) { this.compressRelayDescriptorDownloads = Integer.parseInt( line.split(" ")[1]) != 0; - } else if (line.startsWith("DownloadGetTorStats")) { - this.downloadGetTorStats = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("GetTorStatsURL")) { - String newUrl = line.split(" ")[1]; - /* Test if URL has correct format. */ - new URL(newUrl); - this.getTorStatsUrl = newUrl; - } else if (line.startsWith("GetTorDirectory")) { - this.getTorDirectory = line.split(" ")[1]; } else if (line.startsWith("DownloadExitList")) { this.downloadExitList = Integer.parseInt( line.split(" ")[1]) != 0; @@ -238,7 +224,7 @@ public class Configuration { /** Make some checks if configuration is valid. */ if (!this.importCachedRelayDescriptors && !this.importDirectoryArchives && !this.downloadRelayDescriptors && - !this.importBridgeSnapshots && !this.downloadGetTorStats && + !this.importBridgeSnapshots && !this.downloadExitList && !this.processBridgePoolAssignments && !this.writeDirectoryArchives && !this.writeSanitizedBridges && !this.processTorperfFiles) { @@ -339,15 +325,6 @@ public class Configuration { public boolean getCompressRelayDescriptorDownloads() { return this.compressRelayDescriptorDownloads; } - public boolean getDownloadGetTorStats() { - return this.downloadGetTorStats; - } - public String getGetTorStatsUrl() { - return this.getTorStatsUrl; - } - public String getGetTorDirectory() { - return this.getTorDirectory; - } public boolean getDownloadExitList() { return this.downloadExitList; } diff --git a/src/org/torproject/ernie/db/GetTorDownloader.java b/src/org/torproject/ernie/db/GetTorDownloader.java deleted file mode 100644 index f1e1b77..0000000 --- a/src/org/torproject/ernie/db/GetTorDownloader.java +++ /dev/null @@ -1,115 +0,0 @@ -/* Copyright 2010--2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.db; - -import java.io.BufferedInputStream; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.StringReader; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.logging.Level; -import java.util.logging.Logger; - -public class GetTorDownloader { - - public GetTorDownloader(String gettorStatsUrl, File getTorDirectory) { - - Logger logger = Logger.getLogger(GetTorDownloader.class.getName()); - - File getTorFile = new File(getTorDirectory, "gettor_stats.txt"); - SortedMap<String, String> getTorStats = new TreeMap<String, String>(); - - if (getTorFile.exists() && !getTorFile.isDirectory()) { - try { - logger.fine("Reading local gettor_stats.txt file..."); - BufferedReader br = new BufferedReader(new FileReader( - getTorFile)); - String line = null; - while ((line = br.readLine()) != null) { - if (line.startsWith("@type gettor 1.")) { - continue; - } - String date = line.split(" ")[0]; - getTorStats.put(date, line); - } - br.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed parsing local GetTor stats!", - e); - return; - } - } - - String unparsed = null; - try { - logger.fine("Downloading GetTor stats..."); - URL u = new URL(gettorStatsUrl); - HttpURLConnection huc = (HttpURLConnection) u.openConnection(); - huc.setRequestMethod("GET"); - huc.connect(); - int response = huc.getResponseCode(); - if (response == 200) { - BufferedInputStream in = new BufferedInputStream( - huc.getInputStream()); - StringBuilder sb = new StringBuilder(); - int len; - byte[] data = new byte[1024]; - while ((len = in.read(data, 0, 1024)) >= 0) { - sb.append(new String(data, 0, len)); - } - in.close(); - unparsed = sb.toString(); - } - logger.fine("Finished downloading GetTor stats."); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed downloading GetTor stats", e); - return; - } - - try { - logger.fine("Parsing downloaded GetTor stats..."); - BufferedReader br = new BufferedReader(new StringReader(unparsed)); - String line = null; - while ((line = br.readLine()) != null) { - String date = line.split(" ")[0]; - getTorStats.put(date, line); - } - br.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed parsing downloaded GetTor stats!", - e); - return; - } - - try { - logger.fine("Writing GetTor stats to local gettor_stats.txt " - + "file..."); - if (!getTorDirectory.exists()) { - getTorDirectory.mkdirs(); - } - BufferedWriter bw = new BufferedWriter(new FileWriter(getTorFile)); - bw.write("@type gettor 1.0\n"); - for (String line : getTorStats.values()) { - bw.write(line + "\n"); - } - bw.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed writing GetTor stats to local " - + "gettor_stats.txt file", e); - return; - } - - logger.info("Finished downloading and processing statistics on Tor " - + "packages delivered by GetTor.\nDownloaded " + unparsed.length() - + " bytes. Last date in statistics is " + getTorStats.lastKey() - + "."); - } -} - diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java index 3fd6656..04cc868 100644 --- a/src/org/torproject/ernie/db/Main.java +++ b/src/org/torproject/ernie/db/Main.java @@ -118,12 +118,6 @@ public class Main { sbw = null; } - // Download and process GetTor stats - if (config.getDownloadGetTorStats()) { - new GetTorDownloader(config.getGetTorStatsUrl(), - new File(config.getGetTorDirectory())); - } - // Download exit list and store it to disk if (config.getDownloadExitList()) { new ExitListDownloader(); @@ -153,8 +147,6 @@ public class Main { !config.getProcessBridgePoolAssignments() ? null : new File(config.getSanitizedAssignmentsDirectory()), config.getDownloadExitList(), - !config.getDownloadGetTorStats() ? null : - new File(config.getGetTorDirectory()), !config.getProcessTorperfFiles() ? null : new File(config.getTorperfOutputDirectory()), new File(config.getRsyncDirectory())); diff --git a/src/org/torproject/ernie/db/RsyncDataProvider.java b/src/org/torproject/ernie/db/RsyncDataProvider.java index 4cb8e99..2f9632e 100644 --- a/src/org/torproject/ernie/db/RsyncDataProvider.java +++ b/src/org/torproject/ernie/db/RsyncDataProvider.java @@ -21,7 +21,7 @@ public class RsyncDataProvider { public RsyncDataProvider(File directoryArchivesOutputDirectory, File sanitizedBridgesWriteDirectory, File sanitizedAssignmentsDirectory, - boolean downloadExitList, File getTorDirectory, + boolean downloadExitList, File torperfOutputDirectory, File rsyncDirectory) { /* Initialize logger. */ @@ -174,21 +174,6 @@ public class RsyncDataProvider { + fileNamesInRsync.size() + " files left in " + rsyncDirectory.getAbsolutePath() + "."); - /* Copy GetTor stats. */ - if (getTorDirectory != null) { - String getTorFileName = "gettor_stats.txt"; - File getTorFile = new File(getTorDirectory, getTorFileName); - if (getTorFile.exists() && - getTorFile.lastModified() >= cutOffMillis) { - this.copyFile(getTorFile, new File(rsyncDirectory, "gettor/" - + getTorFileName)); - fileNamesInRsync.remove(getTorFileName); - } - } - logger.info("After copying the GetTor stats file, there are still " - + fileNamesInRsync.size() + " files left in " - + rsyncDirectory.getAbsolutePath() + "."); - /* Delete all files that we didn't (over-)write in this run. */ files.add(rsyncDirectory); while (!files.isEmpty()) {

1 0

[metrics-lib/master] Remove GetTor statistics parsing code.
by karsten＠torproject.org 07 Aug '12

07 Aug '12

commit ba8cb725d2f4cfb9f1495925d79aeaa7c05b2482 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Aug 7 12:25:42 2012 +0200 Remove GetTor statistics parsing code. --- README | 6 +- .../torproject/descriptor/GetTorStatistics.java | 13 --- .../torproject/descriptor/impl/DescriptorImpl.java | 3 - .../descriptor/impl/GetTorStatisticsImpl.java | 78 -------------------- 4 files changed, 3 insertions(+), 97 deletions(-) diff --git a/README b/README index 1450048..c8bab00 100644 --- a/README +++ b/README @@ -8,9 +8,9 @@ network data and for building services and applications. The descriptor types supported by DescripTor include relay and bridge descriptors which are part of Tor's directory protocol as well as Torperf -data files, GetTor statistics files, and TorDNSEL's exit lists. Access to -these descriptors is unified to facilitate access to publicly available -data about the Tor network. +data files and TorDNSEL's exit lists. Access to these descriptors is +unified to facilitate access to publicly available data about the Tor +network. This API is designed for Java programs that process Tor descriptors in batches. A Java program using this API first sets up a descriptor source diff --git a/src/org/torproject/descriptor/GetTorStatistics.java b/src/org/torproject/descriptor/GetTorStatistics.java deleted file mode 100644 index b6ba1e7..0000000 --- a/src/org/torproject/descriptor/GetTorStatistics.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.torproject.descriptor; - -import java.util.SortedMap; - -public interface GetTorStatistics { - - /* Return the date of these GetTor statistics in milliseconds since - * 1970-01-01 00:00:00. */ - public long getDateMillis(); - - /* Return the number of downloaded packages. */ - public SortedMap<String, Integer> getDownloadedPackages(); -} diff --git a/src/org/torproject/descriptor/impl/DescriptorImpl.java b/src/org/torproject/descriptor/impl/DescriptorImpl.java index bb6e415..b1e4408 100644 --- a/src/org/torproject/descriptor/impl/DescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/DescriptorImpl.java @@ -88,9 +88,6 @@ public abstract class DescriptorImpl implements Descriptor { } else if (firstLines.startsWith("@type torperf 1.")) { parsedDescriptors.addAll(TorperfResultImpl.parseTorperfResults( rawDescriptorBytes, failUnrecognizedDescriptorLines)); - } else if (firstLines.startsWith("@type gettor 1.")) { - parsedDescriptors.addAll(GetTorStatisticsImpl.parseGetTorStatistics( - rawDescriptorBytes, failUnrecognizedDescriptorLines)); } else { throw new DescriptorParseException("Could not detect descriptor " + "type in descriptor starting with '" + firstLines + "'."); diff --git a/src/org/torproject/descriptor/impl/GetTorStatisticsImpl.java b/src/org/torproject/descriptor/impl/GetTorStatisticsImpl.java deleted file mode 100644 index 202cf03..0000000 --- a/src/org/torproject/descriptor/impl/GetTorStatisticsImpl.java +++ /dev/null @@ -1,78 +0,0 @@ -/* Copyright 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.descriptor.impl; - -import java.util.ArrayList; -import java.util.List; -import java.util.Scanner; -import java.util.SortedMap; -import java.util.TreeMap; - -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.GetTorStatistics; - -public class GetTorStatisticsImpl extends DescriptorImpl - implements GetTorStatistics { - - public static List<Descriptor> parseGetTorStatistics( - byte[] rawDescriptorBytes, boolean failUnrecognizedDescriptorLines) - throws DescriptorParseException { - if (rawDescriptorBytes.length == 0) { - throw new DescriptorParseException("Descriptor is empty."); - } - List<Descriptor> parsedDescriptors = new ArrayList<Descriptor>(); - String descriptorString = new String(rawDescriptorBytes); - Scanner s = new Scanner(descriptorString).useDelimiter("\n"); - while (s.hasNext()) { - String line = s.next(); - if (line.startsWith("@type gettor ")) { - String[] parts = line.split(" "); - if (parts.length != 3) { - throw new DescriptorParseException("Illegal line '" + line - + "'."); - } - String version = parts[2]; - if (!version.startsWith("1.")) { - throw new DescriptorParseException("Unsupported version in " - + " line '" + line + "'."); - } - } else { - parsedDescriptors.add(new GetTorStatisticsImpl(line.getBytes(), - failUnrecognizedDescriptorLines)); - } - } - return parsedDescriptors; - } - - protected GetTorStatisticsImpl(byte[] rawDescriptorBytes, - boolean failUnrecognizedDescriptorLines) - throws DescriptorParseException { - super(rawDescriptorBytes, failUnrecognizedDescriptorLines, false); - this.parseGetTorStatisticsLine(new String(rawDescriptorBytes)); - } - - private void parseGetTorStatisticsLine(String line) - throws DescriptorParseException { - if (line.isEmpty()) { - throw new DescriptorParseException("Blank lines are not allowed."); - } - String[] parts = line.split(" "); - if (parts.length < 3) { - throw new DescriptorParseException("Illegal GetTor statistics line " - + "'" + line + "'."); - } - this.dateMillis = ParseHelper.parseDateAtIndex(line, parts, 0); - this.downloadedPackages = ParseHelper.parseKeyValuePairs(line, parts, - 2, ":"); - } - - private long dateMillis; - public long getDateMillis() { - return this.dateMillis; - } - - private SortedMap<String, Integer> downloadedPackages; - public SortedMap<String, Integer> getDownloadedPackages() { - return new TreeMap<String, Integer>(this.downloadedPackages); - } -}

1 0

[metrics-tasks/master] task-6329: refactor relay selection
by karsten＠torproject.org 07 Aug '12

07 Aug '12

commit fa237df1e9b60ce12867df91f655cc24f4aca7d2 Author: delber <delber(a)riseup.net> Date: Thu Aug 2 10:35:22 2012 +0000 task-6329: refactor relay selection Instead of doing all our tests in RelayStats.get_relay(), we now have a set of filters, each of them with its own class, that will be added to a filter chain depending on command-line options. --- task-6329/tor-relays-stats.py | 199 +++++++++++++++++++++++++---------------- 1 files changed, 122 insertions(+), 77 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 80d1f06..4a6989b 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -13,10 +13,103 @@ import os.path from optparse import OptionParser, OptionGroup import urllib import re +from abc import abstractmethod + +class BaseFilter(object): + @abstractmethod + def accept(self, relay): + pass + +class RunningFilter(BaseFilter): + def accept(self, relay): + return relay['running'] + +class FamilyFilter(BaseFilter): + def __init__(self, family, all_relays): + self._family_fingerprint = None + self._family_nickname = None + self._family_relays = [] + found_relay = None + for relay in all_relays: + if len(family) == 40 and relay['fingerprint'] == family: + found_relay = relay + break + if len(family) < 20 and 'Named' in relay['flags'] and relay['nickname'] == family: + found_relay = relay + break + if found_relay: + self._family_fingerprint = '$%s' % found_relay['fingerprint'] + if 'Named' in found_relay['flags']: + self._family_nickname = found_relay['nickname'] + self._family_relays = [self._family_fingerprint] + found_relay.get('family', []) + + def accept(self, relay): + fingerprint = '$%s' % relay['fingerprint'] + mentions = [fingerprint] + relay.get('family', []) + if fingerprint in self._family_relays: + return True + if 'Named' in relay['flags'] and relay['nickname'] in self._family_relays: + return True + if self._family_fingerprint in mentions: + return True + if self._family_nickname in mentions: + return True + return False + +class CountryFilter(BaseFilter): + def __init__(self, countries=[]): + self._countries = [x.lower() for x in countries] + + def accept(self, relay): + return relay.get('country', None) in self._countries + +class ASFilter(BaseFilter): + def __init__(self, as_sets=[]): + self._as_sets = [x if not x.isdigit() else "AS" + x for x in as_sets] + + def accept(self, relay): + return relay.get('as_number', None) in self._as_sets + +class ExitFilter(BaseFilter): + def accept(self, relay): + return relay.get('exit_probability', -1) > 0.0 + +class GuardFilter(BaseFilter): + def accept(self, relay): + return relay.get('guard_probability', -1) > 0.0 + +class FastExitFilter(BaseFilter): + def accept(self, relay): + if relay.get('bandwidth_rate', -1) < 12500 * 1024: + return False + if relay.get('advertised_bandwidth', -1) < 5000 * 1024: + return False + relevant_ports = set([80, 443, 554, 1755]) + summary = relay.get('exit_policy_summary', {}) + if 'accept' in summary: + portlist = summary['accept'] + elif 'reject' in summary: + portlist = summary['reject'] + else: + return False + ports = [] + for p in portlist: + if '-' in p: + ports.extend(range(int(p.split('-')[0]), + int(p.split('-')[1]) + 1)) + else: + ports.append(int(p)) + policy_ports = set(ports) + if 'accept' in summary and not relevant_ports.issubset(policy_ports): + return False + if 'reject' in summary and not relevant_ports.isdisjoint(policy_ports): + return False + return True class RelayStats(object): - def __init__(self): + def __init__(self, options): self._data = None + self._filters = self._create_filters(options) @property def data(self): @@ -24,76 +117,34 @@ class RelayStats(object): self._data = json.load(file('details.json')) return self._data - def get_relays(self, countries=[], as_sets=[], exits_only=False, guards_only=False, inactive=False, fast_exits_only=False, family=None): + def _create_filters(self, options): + filters = [] + if not options.inactive: + filters.append(RunningFilter()) + if options.family: + filters.append(FamilyFilter(options.family, self.data['relays'])) + if options.country: + filters.append(CountryFilter(options.country)) + if options.ases: + filters.append(ASFilter(options.ases)) + if options.exits_only: + filters.append(ExitFilter()) + if options.guards_only: + filters.append(GuardFilter()) + if options.fast_exits_only: + filters.append(FastExitFilter()) + return filters + + def get_relays(self): relays = [] - family_fingerprint = None - family_nickname = None - family_relays = [] - if countries: - countries = [x.lower() for x in countries] - if as_sets: - as_sets = [x if not x.isdigit() else "AS" + x for x in as_sets] - if family: - fingerprint = family if len(family) == 40 else None - nickname = family if len(family) < 20 else None - found_relay = None - for relay in self.data['relays']: - if fingerprint and relay['fingerprint'] == fingerprint: - found_relay = relay - break - if nickname and 'Named' in relay['flags'] and relay['nickname'] == nickname: - found_relay = relay - break - if found_relay: - family_fingerprint = '$%s' % found_relay.get('fingerprint') - family_nickname = found_relay['nickname'] if 'Named' in found_relay['flags'] else None - family_relays = found_relay.get('family', []) - family_relays.append(family_fingerprint) for relay in self.data['relays']: - if family: - mentions = relay.get('family', []) - mentions.append('$%s' % relay['fingerprint']) - if ('$%s' % relay['fingerprint'] not in family_relays and \ - relay['nickname'] not in family_relays if 'Named' in relay['flags'] else '') or \ - (family_fingerprint not in mentions and \ - family_nickname not in mentions): - continue - if not inactive and inactive == relay['running']: - continue - if countries and not relay.get('country', ' ') in countries: - continue - if as_sets and not relay.get('as_number', ' ') in as_sets: - continue - if exits_only and not relay.get('exit_probability', -1) > 0.0: - continue - if guards_only and not relay.get('guard_probability', -1) > 0.0: - continue - if fast_exits_only: - if relay.get('bandwidth_rate', -1) < 12500 * 1024: - continue - if relay.get('advertised_bandwidth', -1) < 5000 * 1024: - continue - relevant_ports = set([80, 443, 554, 1755]) - summary = relay.get('exit_policy_summary', {}) - if 'accept' in summary: - portlist = summary['accept'] - elif 'reject' in summary: - portlist = summary['reject'] - else: - continue - ports = [] - for p in portlist: - if '-' in p: - ports.extend(range(int(p.split('-')[0]), - int(p.split('-')[1]) + 1)) - else: - ports.append(int(p)) - policy_ports = set(ports) - if 'accept' in summary and not relevant_ports.issubset(policy_ports): - continue - if 'reject' in summary and not relevant_ports.isdisjoint(policy_ports): - continue - relays.append(relay) + accepted = True + for f in self._filters: + if not f.accept(relay): + accepted = False + break + if accepted: + relays.append(relay) return relays def group_relays(self, relays, by_country=False, by_as_number=False): @@ -248,14 +299,8 @@ if '__main__' == __name__: if not os.path.exists('details.json'): parser.error("Did not find details.json. Re-run with --download.") - stats = RelayStats() - relays = stats.get_relays(countries=options.country, - as_sets=options.ases, - exits_only=options.exits_only, - guards_only=options.guards_only, - inactive=options.inactive, - fast_exits_only=options.fast_exits_only, - family=options.family) + stats = RelayStats(options) + relays = stats.get_relays() grouped_relays = stats.group_relays(relays, by_country=options.by_country, by_as_number=options.by_as)

1 0

[metrics-tasks/master] task-6329: setup a function to select in which group belongs a relay
by karsten＠torproject.org 07 Aug '12

07 Aug '12

commit 9c595ce248ae6c2fbca7aa8f0987434f1d0548ca Author: delber <delber(a)riseup.net> Date: Thu Aug 2 10:55:40 2012 +0000 task-6329: setup a function to select in which group belongs a relay --- task-6329/tor-relays-stats.py | 26 ++++++++++++++------------ 1 files changed, 14 insertions(+), 12 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 4a6989b..da9eb6e 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -110,6 +110,7 @@ class RelayStats(object): def __init__(self, options): self._data = None self._filters = self._create_filters(options) + self._get_group = self._get_group_function(options) @property def data(self): @@ -135,6 +136,16 @@ class RelayStats(object): filters.append(FastExitFilter()) return filters + def _get_group_function(self, options): + if options.by_country and options.by_as: + return lambda relay: (relay.get('country', None), relay.get('as_number', None)) + elif options.by_country: + return lambda relay: relay.get('country', None) + elif options.by_as: + return lambda relay: relay.get('as_number', None) + else: + return lambda relay: relay.get('fingerprint') + def get_relays(self): relays = [] for relay in self.data['relays']: @@ -147,17 +158,10 @@ class RelayStats(object): relays.append(relay) return relays - def group_relays(self, relays, by_country=False, by_as_number=False): + def group_relays(self, relays): grouped_relays = {} for relay in relays: - if by_country and by_as_number: - key = (relay.get('country', None), relay.get('as_number', None)) - elif by_country: - key = relay.get('country', None) - elif by_as_number: - key = relay.get('as_number', None) - else: - key = relay.get('fingerprint') + key = self._get_group(relay) if key not in grouped_relays: grouped_relays[key] = [] grouped_relays[key].append(relay) @@ -301,9 +305,7 @@ if '__main__' == __name__: stats = RelayStats(options) relays = stats.get_relays() - grouped_relays = stats.group_relays(relays, - by_country=options.by_country, - by_as_number=options.by_as) + grouped_relays = stats.group_relays(relays) sorted_groups = stats.format_and_sort_groups(grouped_relays, by_country=options.by_country, by_as_number=options.by_as,

1 0

[metrics-tasks/master] task-6929: fix family selection when relay is mentioned by fingerprint
by karsten＠torproject.org 07 Aug '12

07 Aug '12

commit 68e3405a78b8866c2f49cf7697a43e8091074751 Author: delber <delber(a)riseup.net> Date: Sat Aug 4 15:39:30 2012 +0000 task-6929: fix family selection when relay is mentioned by fingerprint --- task-6329/tor-relays-stats.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index b6dd24c..80d1f06 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -52,7 +52,7 @@ class RelayStats(object): for relay in self.data['relays']: if family: mentions = relay.get('family', []) - mentions.append('%s' % relay['fingerprint']) + mentions.append('$%s' % relay['fingerprint']) if ('$%s' % relay['fingerprint'] not in family_relays and \ relay['nickname'] not in family_relays if 'Named' in relay['flags'] else '') or \ (family_fingerprint not in mentions and \

1 0

[metrics-tasks/master] task-6329: move OptionParser creation to its own function
by karsten＠torproject.org 07 Aug '12

07 Aug '12

commit 32c5f0be8774d2556a650d143f03b855e8ea17c5 Author: delber <delber(a)riseup.net> Date: Thu Aug 2 11:25:14 2012 +0000 task-6329: move OptionParser creation to its own function --- task-6329/tor-relays-stats.py | 20 ++++++++++++-------- 1 files changed, 12 insertions(+), 8 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 9706d80..d966f66 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -249,14 +249,7 @@ class RelayStats(object): selection_weights[2] * 100.0, selection_weights[3] * 100.0, selection_weights[4] * 100.0) -def download_details_file(): - url = urllib.urlopen('https://onionoo.torproject.org/details?type=relay') - details_file = open("details.json", 'w') - details_file.write(url.read()) - url.close() - details_file.close() - -if '__main__' == __name__: +def create_option_parser(): parser = OptionParser() parser.add_option("-d", "--download", action="store_true", help="download details.json from Onionoo service") @@ -291,6 +284,17 @@ if '__main__' == __name__: group.add_option("-s", "--short", action="store_true", help="cut the length of the line output at 70 chars") parser.add_option_group(group) + return parser + +def download_details_file(): + url = urllib.urlopen('https://onionoo.torproject.org/details?type=relay') + details_file = open("details.json", 'w') + details_file.write(url.read()) + url.close() + details_file.close() + +if '__main__' == __name__: + parser = create_option_parser() (options, args) = parser.parse_args() if len(args) > 0: parser.error("Did not understand positional argument(s), use options instead.")

1 0

[metrics-tasks/master] task-6329: group relays while doing the selection
by karsten＠torproject.org 07 Aug '12

07 Aug '12

commit cb39df4d4011900be387c797bd091f5cf77a388f Author: delber <delber(a)riseup.net> Date: Thu Aug 2 11:05:43 2012 +0000 task-6329: group relays while doing the selection --- task-6329/tor-relays-stats.py | 46 ++++++++++++++++++++-------------------- 1 files changed, 23 insertions(+), 23 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index da9eb6e..9706d80 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -111,6 +111,7 @@ class RelayStats(object): self._data = None self._filters = self._create_filters(options) self._get_group = self._get_group_function(options) + self._relays = None @property def data(self): @@ -118,6 +119,22 @@ class RelayStats(object): self._data = json.load(file('details.json')) return self._data + @property + def relays(self): + if self._relays: + return self._relays + + self._relays = {} + for relay in self.data['relays']: + accepted = True + for f in self._filters: + if not f.accept(relay): + accepted = False + break + if accepted: + self.add_relay(relay) + return self._relays + def _create_filters(self, options): filters = [] if not options.inactive: @@ -146,26 +163,11 @@ class RelayStats(object): else: return lambda relay: relay.get('fingerprint') - def get_relays(self): - relays = [] - for relay in self.data['relays']: - accepted = True - for f in self._filters: - if not f.accept(relay): - accepted = False - break - if accepted: - relays.append(relay) - return relays - - def group_relays(self, relays): - grouped_relays = {} - for relay in relays: - key = self._get_group(relay) - if key not in grouped_relays: - grouped_relays[key] = [] - grouped_relays[key].append(relay) - return grouped_relays + def add_relay(self, relay): + key = self._get_group(relay) + if key not in self._relays: + self._relays[key] = [] + self._relays[key].append(relay) def format_and_sort_groups(self, grouped_relays, by_country=False, by_as_number=False, links=False): formatted_groups = {} @@ -304,9 +306,7 @@ if '__main__' == __name__: parser.error("Did not find details.json. Re-run with --download.") stats = RelayStats(options) - relays = stats.get_relays() - grouped_relays = stats.group_relays(relays) - sorted_groups = stats.format_and_sort_groups(grouped_relays, + sorted_groups = stats.format_and_sort_groups(stats.relays, by_country=options.by_country, by_as_number=options.by_as, links=options.links)

1 0