April 2011 - tor-commits - lists.torproject.org

[tor/master] Fix a compile warning on OS X 10.6
by nickm＠torproject.org 07 Apr '11

07 Apr '11

commit 286d44402e829488abe802efd1607eabbc98f1d1 Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Thu Apr 7 05:30:49 2011 +0200 Fix a compile warning on OS X 10.6 --- src/or/circuituse.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 447ec84..3d5db98 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -291,7 +291,7 @@ circuit_expire_building(void) long ms = tor_lround(msec); \ struct timeval diff; \ diff.tv_sec = ms / 1000; \ - diff.tv_usec = (ms % 1000) * 1000; \ + diff.tv_usec = (int)((ms % 1000) * 1000); \ timersub(&now, &diff, &target); \ } while (0)

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.2'
by nickm＠torproject.org 07 Apr '11

07 Apr '11

commit 09759a66eae044e640f477ebe96b1c3edbb98c94 Merge: 53d02a6 286d444 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Apr 7 11:43:36 2011 -0400 Merge remote-tracking branch 'origin/maint-0.2.2' src/or/circuituse.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)

1 0

[tor/maint-0.2.2] Fix a compile warning on OS X 10.6
by nickm＠torproject.org 07 Apr '11

07 Apr '11

commit 286d44402e829488abe802efd1607eabbc98f1d1 Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Thu Apr 7 05:30:49 2011 +0200 Fix a compile warning on OS X 10.6 --- src/or/circuituse.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 447ec84..3d5db98 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -291,7 +291,7 @@ circuit_expire_building(void) long ms = tor_lround(msec); \ struct timeval diff; \ diff.tv_sec = ms / 1000; \ - diff.tv_usec = (ms % 1000) * 1000; \ + diff.tv_usec = (int)((ms % 1000) * 1000); \ timersub(&now, &diff, &target); \ } while (0)

1 0

r24572: {arm} Fixing man page typo (caught by tagnaq) (arm/trunk)
by Damian Johnson 07 Apr '11

07 Apr '11

Author: atagar Date: 2011-04-07 14:20:26 +0000 (Thu, 07 Apr 2011) New Revision: 24572 Modified: arm/trunk/arm.1 Log: Fixing man page typo (caught by tagnaq) Modified: arm/trunk/arm.1 =================================================================== --- arm/trunk/arm.1 2011-04-07 01:30:11 UTC (rev 24571) +++ arm/trunk/arm.1 2011-04-07 14:20:26 UTC (rev 24572) @@ -53,7 +53,7 @@ 67890 torctl runlevel+ U Unknown Events .TP -\fB\-v\fR, \fB\-\-verion\fR +\fB\-v\fR, \fB\-\-version\fR provides version information .TP

1 0

[metrics-web/master] Add GeoIP database to generate fancy relay visualizations.
by karsten＠torproject.org 07 Apr '11

07 Apr '11

commit 4c34249760470099475e9830a3b8255b2aa5f3a3 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Apr 4 21:09:40 2011 +0200 Add GeoIP database to generate fancy relay visualizations. --- build.xml | 9 ++ db/tordir.sql | 50 ++++++++++ rserve/csv.R | 12 +++ .../ernie/cron/GeoipDatabaseImporter.java | 95 ++++++++++++++++++++ src/org/torproject/ernie/web/CsvServlet.java | 1 + 5 files changed, 167 insertions(+), 0 deletions(-) diff --git a/build.xml b/build.xml index bd06daf..4b4a7e9 100644 --- a/build.xml +++ b/build.xml @@ -42,6 +42,15 @@ </javac> </target> +  + <target name="geoipdb" depends="compile"> + <java fork="true" + maxmemory="1024m" + classname="org.torproject.ernie.cron.GeoipDatabaseImporter"> + <classpath refid="classpath"/> + </java> + </target> +  <target name="run" depends="compile"> <java fork="true" diff --git a/db/tordir.sql b/db/tordir.sql index c47d19a..d5238ea 100644 --- a/db/tordir.sql +++ b/db/tordir.sql @@ -239,6 +239,56 @@ CREATE TABLE updates ( date DATE ); +-- GeoIP database that helps resolve IP addresses to country codes, +-- latitudes, and longitudes. +CREATE TABLE geoipdb ( + id SERIAL, + ipstart INET, + ipend INET, + country CHARACTER(2) NOT NULL, + latitude NUMERIC(7, 4) NOT NULL, + longitude NUMERIC(7, 4) NOT NULL +); + +-- Indexes to speed up looking up IP addresses in the GeoIP database. +CREATE INDEX geoip_ipstart ON geoipdb (ipstart); +CREATE INDEX geoip_ipend ON geoipdb (ipend); + +-- Result type for GeoIP lookups that encapsulates the country code, +-- latitude, and longitude. +CREATE TYPE geoip_result AS (country CHARACTER(2), + latitude NUMERIC(7, 4), longitude NUMERIC(7, 4)); + +-- GeoIP database lookup function. +CREATE OR REPLACE FUNCTION geoip_lookup (CHARACTER VARYING) +RETURNS geoip_result AS $$ + SELECT country, latitude, longitude FROM geoipdb + WHERE INET($1) + BETWEEN geoipdb.ipstart AND geoipdb.ipend LIMIT 1; +$$ LANGUAGE SQL; + +-- View that contains the relays of the first known consensuses of all +-- months in the database. +CREATE OR REPLACE VIEW relays_monthly_snapshots AS +SELECT status.validafter, status.fingerprint, status.nickname, + status.address, (status.geoip).country, (status.geoip).latitude, + (status.geoip).longitude, status.isexit, status.isfast, + status.isguard, status.isstable, status.version, status.ports, + descriptor.bandwidthavg, descriptor.bandwidthburst, + descriptor.bandwidthobserved +FROM ( + SELECT *, geoip_lookup(address) AS geoip + FROM statusentry + WHERE validafter IN ( + SELECT MIN(validafter) + FROM consensus + GROUP BY DATE_TRUNC('month', validafter) + ) +) AS status +LEFT JOIN descriptor +ON status.descriptor = descriptor.descriptor +ORDER BY validafter, fingerprint; + CREATE LANGUAGE plpgsql; -- FUNCTION refresh_relay_statuses_per_day() diff --git a/rserve/csv.R b/rserve/csv.R index 44138ac..e68971c 100644 --- a/rserve/csv.R +++ b/rserve/csv.R @@ -267,3 +267,15 @@ export_connbidirect <- function(path) { quote = FALSE, row.names = FALSE) } +export_relays_monthly_snapshots <- function(path) { + drv <- dbDriver("PostgreSQL") + con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) + q <- paste("SELECT * FROM relays_monthly_snapshots") + rs <- dbSendQuery(con, q) + c <- fetch(rs, n = -1) + dbDisconnect(con) + dbUnloadDriver(drv) + write.csv(format(c, trim = TRUE, scientific = FALSE), path, + quote = FALSE, row.names = FALSE) +} + diff --git a/src/org/torproject/ernie/cron/GeoipDatabaseImporter.java b/src/org/torproject/ernie/cron/GeoipDatabaseImporter.java new file mode 100644 index 0000000..afe5245 --- /dev/null +++ b/src/org/torproject/ernie/cron/GeoipDatabaseImporter.java @@ -0,0 +1,95 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.cron; + +import java.io.*; +import java.sql.*; +import java.util.*; +import java.util.logging.*; + +/** + * Import a Maxmind GeoLite City database to resolve resolve IP addresses + * to country codes, latitudes, and longitudes. + */ +public class GeoipDatabaseImporter { + public static void main(String[] args) throws IOException, + SQLException { + + /* Check if the GeoIP database files are in place. */ + File locationsFile = new File("GeoLiteCity-Location.csv"), + blocksFile = new File("GeoLiteCity-Blocks.csv"); + if (!locationsFile.exists() || !blocksFile.exists()) { + System.out.println("Could not find GeoLiteCity-Location.csv and/or " + + "GeoLiteCity-Blocks.csv in the working directory! Exiting!"); + System.exit(1); + } + + /* Initialize configuration to learn JDBC string. */ + Configuration config = new Configuration(); + String jdbcString = config.getRelayDescriptorDatabaseJDBC(); + + /* Connect to database. */ + Connection c = DriverManager.getConnection(jdbcString); + + /* Start by reading location information to memory. */ + BufferedReader br = new BufferedReader(new FileReader(locationsFile)); + String line; + Map<Integer, String> locations = new HashMap<Integer, String>(); + while ((line = br.readLine()) != null) { + if (line.startsWith("Copyright") || line.startsWith("locId")) { + continue; + } + String[] parts = line.split(","); + int locId = Integer.parseInt(parts[0]); + String country = parts[1].replaceAll("\"", ""); + String latitude = parts[5]; + String longitude = parts[6]; + locations.put(locId, country + "," + latitude + "," + longitude); + } + br.close(); + + /* Parse block information and add it to the database together with + * the location information. */ + PreparedStatement ps = c.prepareStatement("INSERT INTO geoipdb " + + "(ipstart, ipend, country, latitude, longitude) VALUES " + + "(?, ?, ?, ?, ?)"); + Statement s = c.createStatement(); + s.execute("DELETE FROM geoipdb"); + /* TODO The import takes 30+ minutes. Perform the import in a single + * transaction, or requests will return strange results in these 30+ + * minutes. */ + br = new BufferedReader(new FileReader(blocksFile)); + while ((line = br.readLine()) != null) { + if (line.startsWith("Copyright") || + line.startsWith("startIpNum")) { + continue; + } + String[] parts = line.replaceAll("\"", "").split(","); + long startIpNum = Long.parseLong(parts[0]); + String startIp = "" + startIpNum / 256 / 256 / 256 + "." + + startIpNum / 256 / 256 % 256 + "." + startIpNum / 256 % 256 + + "." + startIpNum % 256; + long endIpNum = Long.parseLong(parts[1]); + String endIp = "" + endIpNum / 256 / 256 / 256 + "." + + endIpNum / 256 / 256 % 256 + "." + endIpNum / 256 % 256 + "." + + endIpNum % 256; + int locId = Integer.parseInt(parts[2]); + if (!locations.containsKey(locId)) { + System.out.println("Cannot find locId=" + locId + + " in locations file!"); + continue; + } + String[] locationParts = locations.get(locId).split(","); + String country = locationParts[0]; + double latitude = Double.parseDouble(locationParts[1]); + double longitude = Double.parseDouble(locationParts[2]); + ps.setObject(1, startIp, Types.OTHER); + ps.setObject(2, endIp, Types.OTHER); + ps.setString(3, country); + ps.setDouble(4, latitude); + ps.setDouble(5, longitude); + ps.execute(); + } + } +} + diff --git a/src/org/torproject/ernie/web/CsvServlet.java b/src/org/torproject/ernie/web/CsvServlet.java index 33c098f..c68a2d0 100644 --- a/src/org/torproject/ernie/web/CsvServlet.java +++ b/src/org/torproject/ernie/web/CsvServlet.java @@ -40,6 +40,7 @@ public class CsvServlet extends HttpServlet { this.availableCsvFiles.add("monthly-users-peak"); this.availableCsvFiles.add("networksize"); this.availableCsvFiles.add("platforms"); + this.availableCsvFiles.add("relays-monthly-snapshots"); this.availableCsvFiles.add("relayflags"); this.availableCsvFiles.add("relayflags-hour"); this.availableCsvFiles.add("torperf");

1 0

r24571: {arm} Hotfix for r24570. (in arm/release: . src)
by Damian Johnson 07 Apr '11

07 Apr '11

Author: atagar Date: 2011-04-07 01:30:11 +0000 (Thu, 07 Apr 2011) New Revision: 24571 Modified: arm/release/setup.py arm/release/src/version.py Log: Hotfix for r24570. Modified: arm/release/setup.py =================================================================== --- arm/release/setup.py 2011-04-07 01:28:38 UTC (rev 24570) +++ arm/release/setup.py 2011-04-07 01:30:11 UTC (rev 24571) @@ -79,7 +79,7 @@ author='Damian Johnson', author_email='atagar(a)torproject.org', url='http://www.atagar.com/arm/', - packages=['arm', 'arm.interface', 'arm.interface.graphing', 'arm.util', 'arm.TorCtl'], + packages=['arm', 'arm.interface', 'arm.interface.graphing', 'arm.interface.connections', 'arm.util', 'arm.TorCtl'], package_dir={'arm': 'src'}, data_files=[("/usr/bin", ["arm"]), ("/usr/share/man/man1", [manFilename]), Modified: arm/release/src/version.py =================================================================== --- arm/release/src/version.py 2011-04-07 01:28:38 UTC (rev 24570) +++ arm/release/src/version.py 2011-04-07 01:30:11 UTC (rev 24571) @@ -2,6 +2,6 @@ Provides arm's version and release date. """ -VERSION = '1.4.2.1' -LAST_MODIFIED = "April 4, 2011" +VERSION = '1.4.2.2' +LAST_MODIFIED = "April 6, 2011"

1 0

r24570: {arm} Missing new connection components from installations (caught (arm/trunk)
by Damian Johnson 07 Apr '11

07 Apr '11

Author: atagar Date: 2011-04-07 01:28:38 +0000 (Thu, 07 Apr 2011) New Revision: 24570 Modified: arm/trunk/setup.py Log: Missing new connection components from installations (caught by Anthony Basile) Modified: arm/trunk/setup.py =================================================================== --- arm/trunk/setup.py 2011-04-06 13:53:01 UTC (rev 24569) +++ arm/trunk/setup.py 2011-04-07 01:28:38 UTC (rev 24570) @@ -79,7 +79,7 @@ author='Damian Johnson', author_email='atagar(a)torproject.org', url='http://www.atagar.com/arm/', - packages=['arm', 'arm.interface', 'arm.interface.graphing', 'arm.util', 'arm.TorCtl'], + packages=['arm', 'arm.interface', 'arm.interface.graphing', 'arm.interface.connections', 'arm.util', 'arm.TorCtl'], package_dir={'arm': 'src'}, data_files=[("/usr/bin", ["arm"]), ("/usr/share/man/man1", [manFilename]),

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.2'
by nickm＠torproject.org 06 Apr '11

06 Apr '11

commit 53d02a63dfee248669c402d3480af7184758cf52 Merge: 13f60e1 de1c36f Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Apr 6 16:20:32 2011 -0400 Merge remote-tracking branch 'origin/maint-0.2.2'

1 0

[tor/master] Backport: Generate version tags using Git, not (broken) svn revisions.
by nickm＠torproject.org 06 Apr '11

06 Apr '11

commit 02d93caa09656b1ef73838608258afad5c090105 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jan 25 14:01:04 2011 -0500 Backport: Generate version tags using Git, not (broken) svn revisions. Partial backport of daa0326aaaa85a760be94ee2360cfa61a9fb5be2 . Resolves bug 2402. Bugfix on 0.2.1.15 (for the part where we switched to git) and on 0.2.1.30 (for the part where we dumped micro-revisions.) --- changes/bug2402_redux | 6 +++++ src/or/Makefile.am | 60 ++++++++++-------------------------------------- src/or/config.c | 4 +- src/or/router.c | 2 - 4 files changed, 21 insertions(+), 51 deletions(-) diff --git a/changes/bug2402_redux b/changes/bug2402_redux new file mode 100644 index 0000000..84be04a --- /dev/null +++ b/changes/bug2402_redux @@ -0,0 +1,6 @@ + o Minor bugfixes + - Use micro-revision numbers in 0.2.1.x instead: apparently, they + were more used than we had known. (Bugfix on 0.2.1.30). + - Instead of generating our micro-version numbers using SVN revisions, + use git revisions instead. Bugfix on 0.2.1.15-rc; fixes bug 2402. + diff --git a/src/or/Makefile.am b/src/or/Makefile.am index ad2476f..51c1b0f 100644 --- a/src/or/Makefile.am +++ b/src/or/Makefile.am @@ -55,53 +55,19 @@ noinst_HEADERS = or.h eventdns.h eventdns_tor.h micro-revision.i tor_main.o: micro-revision.i micro-revision.i: FORCE - @svkdir=$$SVKROOT; \ - if test "x$$svkdir" = x ; then \ - svkdir=$$HOME/.svk; \ - fi; \ - if test -d ../../.git && test -x "`which git 2>&1;true`" ; then \ - if test -d ../../.git/svn && test -x "`which git-svn 2>&1;true`" ; then \ - git-svn info ../../README | \ - sed -n 's/^Revision: $[0-9][0-9]*$.*/"\1"/p' \ - > micro-revision.tmp \ - || true; \ - fi; \ - elif test -d ../../.svn && test -x "`which svn 2>&1;true`" ; then \ - svn info ../.. | \ - sed -n 's/^Revision: $[0-9][0-9]*$.*/"\1"/p' > micro-revision.tmp \ - || true; \ - elif test -x "`which svk 2>&1;true`" && test -d $$svkdir/local; then \ - location=../..; \ - rev=x; \ - while test x$$rev = xx; do \ - x=`svk info $$location | \ - sed -n 's/^Mirrored From:.*, Rev\. $[0-9][0-9]*$/\1/p'`; \ - if test x$$x != x; then \ - rev=$$x; \ - break; \ - else \ - loc=`svk info $$location | \ - sed -n 's/^Copied From: $.*$, Rev\. [0-9][0-9]*/\1/p' | \ - head -1`; \ - if test x$$loc = x; then \ - break; \ - else \ - location=/$$loc; \ - fi; \ - fi; \ - done; \ - if test x$$rev != xx; then \ - echo \"$$rev\" > micro-revision.tmp; \ - fi; \ - fi; \ - if test ! -f micro-revision.tmp ; then \ - if test ! -f micro-revision.i ; then \ - echo '""' > micro-revision.i; \ - fi; \ - elif test ! -f micro-revision.i || \ - test x"`cat micro-revision.tmp`" != x"`cat micro-revision.i`"; then \ - mv micro-revision.tmp micro-revision.i; \ - fi; true + @rm -f micro-revision.tmp; \ + if test -d ../../.git && test -x "`which git 2>&1;true`"; then \ + HASH="`git rev-parse --short=16 HEAD`"; \ + echo \"$$HASH\" > micro-revision.tmp; \ + fi; \ + if test ! -f micro-revision.tmp ; then \ + if test ! -f micro-revision.i ; then \ + echo '""' > micro-revision.i; \ + fi; \ + elif test ! -f micro-revision.i || \ + test x"`cat micro-revision.tmp`" != x"`cat micro-revision.i`"; then \ + mv micro-revision.tmp micro-revision.i; \ + fi; true #Dummy target to ensure that micro-revision.i _always_ gets built. FORCE: diff --git a/src/or/config.c b/src/or/config.c index 8397b23..9cd8149 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -829,9 +829,9 @@ get_version(void) { if (_version == NULL) { if (strlen(tor_svn_revision)) { - size_t len = strlen(VERSION)+strlen(tor_svn_revision)+8; + size_t len = strlen(VERSION)+strlen(tor_svn_revision)+16; _version = tor_malloc(len); - tor_snprintf(_version, len, "%s (r%s)", VERSION, tor_svn_revision); + tor_snprintf(_version, len, "%s (git-%s)", VERSION, tor_svn_revision); } else { _version = tor_strdup(VERSION); } diff --git a/src/or/router.c b/src/or/router.c index 75f592d..4ce4f85 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1586,8 +1586,6 @@ router_guess_address_from_dir_headers(uint32_t *guess) return -1; } -extern const char tor_svn_revision[]; /* from tor_main.c */ - /** Set <b>platform</b> (max length <b>len</b>) to a NUL-terminated short * string describing the version of Tor and the operating system we're * currently running on.

1 0

[tor/master] Merge maint-0.2.1 for bug2402_again fixes . Use "-s ours" so we don't actually take the patch.
by nickm＠torproject.org 06 Apr '11

06 Apr '11

commit de1c36ff6062ee4b45e1e1f637a7f168e104d7b1 Merge: 9d2b384 32341ac Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Apr 6 16:15:48 2011 -0400 Merge maint-0.2.1 for bug2402_again fixes . Use "-s ours" so we don't actually take the patch.

1 0