[tor-commits] [metrics-tasks/master] Add George's censorship detector script.
karsten at torproject.org
karsten at torproject.org
Fri Mar 25 09:34:19 UTC 2011
commit f5d257dff6af41dbbe33ab20c5bbb218a38c8cd8
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Fri Mar 25 10:29:38 2011 +0100
Add George's censorship detector script.
---
task-2718/detector.py | 306 +++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 306 insertions(+), 0 deletions(-)
diff --git a/task-2718/detector.py b/task-2718/detector.py
new file mode 100644
index 0000000..0370d02
--- /dev/null
+++ b/task-2718/detector.py
@@ -0,0 +1,306 @@
+## Copyright (c) 2011 George Danezis <gdane at microsoft.com>
+##
+## All rights reserved.
+##
+## Redistribution and use in source and binary forms, with or without
+## modification, are permitted (subject to the limitations in the
+## disclaimer below) provided that the following conditions are met:
+##
+## * Redistributions of source code must retain the above copyright
+## notice, this list of conditions and the following disclaimer.
+##
+## * Redistributions in binary form must reproduce the above copyright
+## notice, this list of conditions and the following disclaimer in the
+## documentation and/or other materials provided with the
+## distribution.
+##
+## * Neither the name of <Owner Organization> nor the names of its
+## contributors may be used to endorse or promote products derived
+## from this software without specific prior written permission.
+##
+## NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE
+## GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT
+## HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+## WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+## MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+## DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+## LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+## CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+## SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+## BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+## WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+## OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+## IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+##
+## (Clear BSD license: http://labs.metacarta.com/license-explanation.html#license)
+
+## This script reads a .csv file of the number of Tor users and finds
+## anomalies that might be indicative of censorship.
+
+# Dep: matplotlib
+from pylab import *
+import matplotlib
+
+# Dep: numpy
+import numpy
+
+# Dep: scipy
+import scipy.stats
+from scipy.stats.distributions import norm
+from scipy.stats.distributions import poisson
+
+# Std lib
+from datetime import date
+from datetime import timedelta
+import os.path
+
+days = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
+
+# read the .csv file
+class torstatstore:
+ def __init__(self, file_name):
+ f = file(file_name)
+ country_codes = f.readline()
+ country_codes = country_codes.strip().split(",")
+
+ store = {}
+ MAX_INDEX = 0
+ for i, line in enumerate(f):
+ MAX_INDEX += 1
+ line_parsed = line.strip().split(",")
+ for j, (ccode, val) in enumerate(zip(country_codes,line_parsed)):
+ processed_val = None
+ if ccode == "date":
+ try:
+ year, month, day = int(val[:4]), int(val[5:7]), int(val[8:10])
+ processed_val = date(year, month, day)
+ except Exception, e:
+ print "Parsing error (ignoring line %s):" % j
+ print "%s" % val,e
+ break
+
+ elif val != "NA":
+ processed_val = int(val)
+ store[(ccode, i)] = processed_val
+
+ # min and max
+ date_min = store[("date", 0)]
+ date_max = store[("date", i)]
+
+ all_dates = []
+ d = date_min
+ dt = timedelta(days=1)
+ while d <= date_max:
+ all_dates += [d]
+ d = d + dt
+
+ # Save for later
+ self.store = store
+ self.all_dates = all_dates
+ self.country_codes = country_codes
+ self.MAX_INDEX = MAX_INDEX
+ self.date_min = date_min
+ self.date_max = date_max
+
+ def get_country_series(self, ccode):
+ assert ccode in self.country_codes
+ series = {}
+ for d in self.all_dates:
+ series[d] = None
+ for i in range(self.MAX_INDEX):
+ series[self.store[("date", i)]] = self.store[(ccode, i)]
+ sx = []
+ for d in self.all_dates:
+ sx += [series[d]]
+ return sx
+
+ def get_largest(self, number):
+ exclude = set(["all", "??", "date"])
+ l = [(self.store[(c, self.MAX_INDEX-1)], c) for c in self.country_codes if c not in exclude]
+ l.sort()
+ l.reverse()
+ return l[:number]
+
+ def get_largest_locations(self, number):
+ l = self.get_largest(number)
+ res = {}
+ for _, ccode in l[:number]:
+ res[ccode] = self.get_country_series(ccode)
+ return res
+
+# Computes the difference between today and a number of days in the past
+def n_day_rel(series, days):
+ rel = []
+ for i, v in enumerate(series):
+ if series[i] is None:
+ rel += [None]
+ continue
+
+ if i - days < 0 or series[i-days] is None or series[i-days] == 0:
+ rel += [None]
+ else:
+ rel += [ float(series[i]) / series[i-days]]
+ return rel
+
+# Main model: computes the expected min / max range of number of users
+def make_tendencies_minmax(l, INTERVAL = 1):
+ lminus1 = dict([(ccode, n_day_rel(l[ccode], INTERVAL)) for ccode in l])
+ c = lminus1[lminus1.keys()[0]]
+ dists = []
+ minx = []
+ maxx = []
+ for i in range(len(c)):
+ vals = [lminus1[ccode][i] for ccode in lminus1.keys() if lminus1[ccode][i] != None]
+ if len(vals) < 8:
+ dists += [None]
+ minx += [None]
+ maxx += [None]
+ else:
+ vals.sort()
+ median = vals[len(vals)/2]
+ q1 = vals[len(vals)/4]
+ q2 = vals[(3*len(vals))/4]
+ qd = q2 - q1
+ vals = [v for v in vals if median - qd*4 < v and v < median + qd*4]
+ if len(vals) < 8:
+ dists += [None]
+ minx += [None]
+ maxx += [None]
+ continue
+ mu, signma = norm.fit(vals)
+ dists += [(mu, signma)]
+ maxx += [norm.ppf(0.9999, mu, signma)]
+ minx += [norm.ppf(1 - 0.9999, mu, signma)]
+ ## print minx[-1], maxx[-1]
+ return minx, maxx
+
+# Makes pretty plots
+def raw_plot(series, minc, maxc, labels, xtitle):
+ assert len(xtitle) == 3
+ fname, stitle, slegend = xtitle
+
+ font = {'family' : 'Bitstream Vera Sans',
+ 'weight' : 'normal',
+ 'size' : 8}
+ matplotlib.rc('font', **font)
+
+ ylim( (-max(series)*0.1, max(series)*1.1) )
+ plot(labels, series, linewidth=1.0, label="Users")
+
+ wherefill = []
+ for mm,mx in zip(minc, maxc):
+ wherefill += [not (mm == None and mx == None)]
+ assert mm < mx or (mm == None and mx == None)
+
+ fill_between(labels, minc, maxc, where=wherefill, color="gray", label="Prediction")
+
+ vdown = []
+ vup = []
+ for i,v in enumerate(series):
+ if minc[i] != None and v < minc[i]:
+ vdown += [v]
+ vup += [None]
+ elif maxc[i] != None and v > maxc[i]:
+ vdown += [None]
+ vup += [v]
+ else:
+ vup += [None]
+ vdown += [None]
+
+ plot(labels, vdown, 'o', ms=10, lw=2, alpha=0.5, mfc='orange', label="Downturns")
+ plot(labels, vup, 'o', ms=10, lw=2, alpha=0.5, mfc='green', label="Upturns")
+
+ legend(loc=2)
+
+ xlabel('Time (days)')
+ ylabel('Users')
+ title(stitle)
+ grid(True)
+ F = gcf()
+
+ F.set_size_inches(10,5)
+ F.savefig(fname, format="png", dpi = (150))
+ close()
+
+def absolute_plot(series, minc, maxc, labels,INTERVAL, xtitle):
+ in_minc = []
+ in_maxc = []
+ for i, v in enumerate(series):
+ if i > 0 and i - INTERVAL >= 0 and series[i] != None and series[i-INTERVAL] != None and series[i-INTERVAL] != 0 and minc[i]!= None and maxc[i]!= None:
+ in_minc += [minc[i] * poisson.ppf(1-0.9999, series[i-INTERVAL])]
+ in_maxc += [maxc[i] * poisson.ppf(0.9999, series[i-INTERVAL])]
+ if not in_minc[-1] < in_maxc[-1]:
+ print in_minc[-1], in_maxc[-1], series[i-INTERVAL], minc[i], maxc[i]
+ assert in_minc[-1] < in_maxc[-1]
+ else:
+ in_minc += [None]
+ in_maxc += [None]
+ raw_plot(series, in_minc, in_maxc, labels, xtitle)
+
+# Censorship score by jurisdiction
+def censor_score(series, minc, maxc, INTERVAL):
+ upscore = 0
+ downscore = 0
+ for i, v in enumerate(series):
+ if i > 0 and i - INTERVAL >= 0 and series[i] != None and series[i-INTERVAL] != None and series[i-INTERVAL] != 0 and minc[i]!= None and maxc[i]!= None:
+ in_minc = minc[i] * poisson.ppf(1-0.9999, series[i-INTERVAL])
+ in_maxc = maxc[i] * poisson.ppf(0.9999, series[i-INTERVAL])
+ downscore += 1 if minc[i] != None and v < in_minc else 0
+ upscore += 1 if maxc[i] != None and v > in_maxc else 0
+ return downscore, upscore
+
+def plot_target(tss, TARGET, xtitle, minx, maxx, DAYS=365, INTERV = 7):
+ ctarget = tss.get_country_series(TARGET)
+ c = n_day_rel(ctarget, INTERV)
+ absolute_plot(ctarget[-DAYS:], minx[-DAYS:], maxx[-DAYS:], tss.all_dates[-DAYS:],INTERV, xtitle = xtitle)
+
+
+## Make a league table of censorship + nice graphs
+def plot_all(tss, minx, maxx, INTERV, DAYS=None, rdir="img"):
+ rdir = os.path.realpath(rdir)
+ if not os.path.exists(rdir) or not os.path.isdir(rdir):
+ print "ERROR: %s does not exist or is not a directory." % rdir
+ return
+
+ summary_file = file(os.path.join(rdir, "summary.txt"), "w")
+
+ if DAYS == None:
+ DAYS = 6*31
+
+ s = tss.get_largest(200)
+ scores = []
+ for num, li in s:
+ print ".",
+ ds,us = censor_score(tss.get_country_series(li)[-DAYS:], minx[-DAYS:], maxx[-DAYS:], INTERV)
+ # print ds, us
+ scores += [(ds,num, us, li)]
+ scores.sort()
+ scores.reverse()
+ s = "\n=======================\n"
+ s+= "Report for %s to %s\n" % (tss.all_dates[-DAYS], tss.all_dates[-1])
+ s+= "=======================\n"
+ print s
+ summary_file.write(s)
+ for a,nx, b,c in scores:
+ if a > 0:
+ s = "%s -- down: %2d (up: %2d affected: %s)" % (c, a, b, nx)
+ print s
+ summary_file.write(s + "\n")
+ xtitle = (os.path.join(rdir, "%03d-%s-censor.png" % (a,c)), "Tor report for %s -- down: %2d (up: %2d affected: %s)" % (c, a, b, nx),"")
+ plot_target(tss, c,xtitle, minx, maxx, DAYS, INTERV)
+ summary_file.close()
+
+def main():
+ # Change these to customize script
+ CSV_FILE = "direct-users.csv"
+ GRAPH_DIR = "img"
+ INTERV = 7
+ DAYS= 6 * 31
+
+ tss = torstatstore(CSV_FILE)
+ l = tss.get_largest_locations(50)
+ minx, maxx = make_tendencies_minmax(l, INTERV)
+ plot_all(tss, minx, maxx, INTERV, DAYS, rdir=GRAPH_DIR)
+
+if __name__ == "__main__":
+ main()
More information about the tor-commits
mailing list