March 2016 - tor-commits - lists.torproject.org

[translation/bridgedb] Update translations for bridgedb
by translation＠torproject.org 18 Mar '16

18 Mar '16

commit a5e2dd962a0c5a0081e7d6d462768d3c858894ab Author: Translation commit bot <translation(a)torproject.org> Date: Fri Mar 18 13:45:04 2016 +0000 Update translations for bridgedb --- lt/LC_MESSAGES/bridgedb.po | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lt/LC_MESSAGES/bridgedb.po b/lt/LC_MESSAGES/bridgedb.po index a461f49..9369b17 100644 --- a/lt/LC_MESSAGES/bridgedb.po +++ b/lt/LC_MESSAGES/bridgedb.po @@ -13,7 +13,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: 'https://trac.torproject.org/projects/tor/newticket?component=BridgeDB&keywo…'\n" "POT-Creation-Date: 2015-07-25 03:40+0000\n" -"PO-Revision-Date: 2016-03-18 13:15+0000\n" +"PO-Revision-Date: 2016-03-18 13:20+0000\n" "Last-Translator: AG <aiste.gvildyte(a)gmail.com>\n" "Language-Team: Lithuanian (http://www.transifex.com/otf/torproject/language/lt/)\n" "MIME-Version: 1.0\n" @@ -77,7 +77,7 @@ msgstr "" #: bridgedb/https/templates/bridges.html:68 msgid "It seems there was an error getting your QRCode." -msgstr "" +msgstr "Įvyko klaida įkeliant jūsų QR kodą." #: bridgedb/https/templates/bridges.html:73 msgid "" @@ -275,7 +275,7 @@ msgstr "" #: bridgedb/strings.py:115 msgid "My bridges don't work! I need help!" -msgstr "" +msgstr "Mano tiltai neveikia! Man reikia pagalbos!" #. TRANSLATORS: Please DO NOT translate "Tor". #: bridgedb/strings.py:117

1 0

[translation/bridgedb] Update translations for bridgedb
by translation＠torproject.org 18 Mar '16

18 Mar '16

commit 1d2cbce18dd87267ab7c33a75124cba122611b17 Author: Translation commit bot <translation(a)torproject.org> Date: Fri Mar 18 13:15:04 2016 +0000 Update translations for bridgedb --- lt/LC_MESSAGES/bridgedb.po | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lt/LC_MESSAGES/bridgedb.po b/lt/LC_MESSAGES/bridgedb.po index b274592..a461f49 100644 --- a/lt/LC_MESSAGES/bridgedb.po +++ b/lt/LC_MESSAGES/bridgedb.po @@ -3,6 +3,7 @@ # This file is distributed under the same license as the BridgeDB project. # # Translators: +# AG <aiste.gvildyte(a)gmail.com>, 2016 # Moo, 2015 # Saule Papeckyte <saule.papeckyte(a)gmail.com>, 2015 # Edgaras7 <system(a)vippsystems.com>, 2014 @@ -12,8 +13,8 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: 'https://trac.torproject.org/projects/tor/newticket?component=BridgeDB&keywo…'\n" "POT-Creation-Date: 2015-07-25 03:40+0000\n" -"PO-Revision-Date: 2015-12-02 23:02+0000\n" -"Last-Translator: Moo\n" +"PO-Revision-Date: 2016-03-18 13:15+0000\n" +"Last-Translator: AG <aiste.gvildyte(a)gmail.com>\n" "Language-Team: Lithuanian (http://www.transifex.com/otf/torproject/language/lt/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -55,11 +56,11 @@ msgstr "Kontaktai" #: bridgedb/https/templates/bridges.html:35 msgid "Select All" -msgstr "" +msgstr "Pažymėti viską" #: bridgedb/https/templates/bridges.html:40 msgid "Show QRCode" -msgstr "" +msgstr "Rodyti QR kodą" #: bridgedb/https/templates/bridges.html:52 msgid "QRCode for your bridge lines"

1 0

[translation/torbutton-torbuttonproperties] Update translations for torbutton-torbuttonproperties
by translation＠torproject.org 18 Mar '16

18 Mar '16

commit c2676d3f68e750f0640cc21ac8bfd0f0d67dd3ec Author: Translation commit bot <translation(a)torproject.org> Date: Fri Mar 18 09:15:49 2016 +0000 Update translations for torbutton-torbuttonproperties --- hr/torbutton.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hr/torbutton.properties b/hr/torbutton.properties index d303488..750e733 100644 --- a/hr/torbutton.properties +++ b/hr/torbutton.properties @@ -62,7 +62,7 @@ torbutton.slider_notification_button = Open security settings torbutton.maximize_warning = Maximizing Tor Browser can allow websites to determine your monitor size, which can be used to track you. We recommend that you leave Tor Browser windows in their original default size. # Canvas permission prompt. Strings are kept here for ease of translation. -canvas.siteprompt=This website (%S) attempted to extract HTML5 canvas image data, which may be used to uniquely identify your computer.\n\nShould Tor Browser allow this website to extract HTML5 canvas image data? +canvas.siteprompt=Ova stranica (%S) je pokušala izvući podatke HTML5 slike, koji mogu biti korišteni kako bi se unikatno identificiralo Vaše računalo.\n\nTreba li Tor Browser dopustiti ovoj stranici izvlačenje podataka HTML5 slike? canvas.notNow=Ne sada canvas.notNowAccessKey=N canvas.allow=Dozvoli ubuduce

1 0

[translation/torbutton-aboutdialogdtd] Update translations for torbutton-aboutdialogdtd
by translation＠torproject.org 18 Mar '16

18 Mar '16

commit 70ad50e375f0c61ed4060607b48fe20b647dcf72 Author: Translation commit bot <translation(a)torproject.org> Date: Fri Mar 18 08:46:17 2016 +0000 Update translations for torbutton-aboutdialogdtd --- hr/aboutdialog.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hr/aboutdialog.dtd b/hr/aboutdialog.dtd index 5099ad7..ba59803 100644 --- a/hr/aboutdialog.dtd +++ b/hr/aboutdialog.dtd @@ -1,4 +1,4 @@ -<!ENTITY project.start "&brandShortName; is developed by "> +<!ENTITY project.start "&brandShortName; razvija">  <!ENTITY project.tpoLink "the &vendorShortName;"> <!ENTITY project.end ", a nonprofit working to defend your privacy and freedom online.">

1 0

[translation/tor-and-https] Update translations for tor-and-https
by translation＠torproject.org 18 Mar '16

18 Mar '16

commit 68115a5ee5df4510bb3af27b8a335abb652c5a14 Author: Translation commit bot <translation(a)torproject.org> Date: Fri Mar 18 08:46:09 2016 +0000 Update translations for tor-and-https --- hr.po | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hr.po b/hr.po index 8446c65..e0cc440 100644 --- a/hr.po +++ b/hr.po @@ -1,13 +1,14 @@ # # Translators: +# Igor <lyricaltumor(a)gmail.com>, 2016 # Vlado V. <vlays(a)mail.com>, 2015 msgid "" msgstr "" "Project-Id-Version: The Tor Project\n" "POT-Creation-Date: 2014-07-17 14:23+0000\n" -"PO-Revision-Date: 2015-04-28 15:33+0000\n" -"Last-Translator: Vlado V. <vlays(a)mail.com>\n" -"Language-Team: Croatian (http://www.transifex.com/projects/p/torproject/language/hr/)\n" +"PO-Revision-Date: 2016-03-18 08:19+0000\n" +"Last-Translator: Igor <lyricaltumor(a)gmail.com>\n" +"Language-Team: Croatian (http://www.transifex.com/otf/torproject/language/hr/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" @@ -20,7 +21,7 @@ msgstr "" #. languages (e.g. Arabic or Hebrew). #: C/tor-and-https.svg:3 msgid "ltr" -msgstr "" +msgstr "s lijeva nadesno" #. (itstool) path: svg/title #: C/tor-and-https.svg:14

1 0

[translation/torbutton-torbuttondtd] Update translations for torbutton-torbuttondtd
by translation＠torproject.org 18 Mar '16

18 Mar '16

commit 23945f4e6d354942dbb8face04feba7189dbe47a Author: Translation commit bot <translation(a)torproject.org> Date: Fri Mar 18 08:46:00 2016 +0000 Update translations for torbutton-torbuttondtd --- hr/torbutton.dtd | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/hr/torbutton.dtd b/hr/torbutton.dtd index d493f00..ad18093 100644 --- a/hr/torbutton.dtd +++ b/hr/torbutton.dtd @@ -24,7 +24,7 @@ <!ENTITY torbutton.pref_connection_more_info.text "Torbutton je trenutno omogućen. Ukoliko želite promijeniti Vaše ne-Tor proxy postavke, molimo Vas da onemogućite Torbutton i vratite se ovdje. Ukoliko želite promijeniti Vaše Tor postavke, molimo Vas, koristite prozor Torbutton postavki."> <!ENTITY torbutton.context_menu.new_identity "New Identity"> <!ENTITY torbutton.context_menu.new_identity_key "I"> -<!ENTITY torbutton.context_menu.new_circuit "New Tor Circuit for this Site"> +<!ENTITY torbutton.context_menu.new_circuit "Novi Tor krug za ovu stranicu"> <!ENTITY torbutton.context_menu.new_circuit_key "Z"> <!ENTITY torbutton.context_menu.toggle "Promijeni Tor status"> <!ENTITY torbutton.context_menu.toggle.key "P"> @@ -36,7 +36,7 @@ <!ENTITY torbutton.context_menu.networksettings.key "S"> <!ENTITY torbutton.context_menu.downloadUpdate "Provjeri Nadogradnje za Tor Pretraživač.."> <!ENTITY torbutton.context_menu.downloadUpdate.key "O"> -<!ENTITY torbutton.context_menu.cookieProtections "Cookie Protections…"> +<!ENTITY torbutton.context_menu.cookieProtections "Zaštita kolačića..."> <!ENTITY torbutton.context_menu.cookieProtections.key "Z"> <!ENTITY torbutton.context_menu.copyTor "Kopiraj Tor URL"> <!ENTITY torbutton.context_menu.copyTor.key "k"> @@ -152,34 +152,34 @@ <!ENTITY torbutton.prefs.sec_caption "Nivo Sigurnosti"> <!ENTITY torbutton.prefs.sec_low "Nisko (zadano)"> <!ENTITY torbutton.prefs.sec_low_usable_desc "Ovo pruža najupotrebljiviji doživljaj."> -<!ENTITY torbutton.prefs.sec_low_desc "At this security level, all browser features are enabled."> -<!ENTITY torbutton.prefs.sec_font_rend_svg_tooltip "The SVG OpenType font rendering mechanism is disabled."> +<!ENTITY torbutton.prefs.sec_low_desc "Na ovoj sigurnosnoj razini, sve značajke preglednika su omogućene."> +<!ENTITY torbutton.prefs.sec_font_rend_svg_tooltip "Mehanizam za prikaz SVG OpenType fonta je onemogućen."> <!ENTITY torbutton.prefs.sec_med_low "Srednje-Nisko"> <!ENTITY torbutton.prefs.sec_gen_desc "Na ovom sigurnosnom nivou, primjenjuju se sljedeće promjene (prijeđi mišem za detalje)"> <!ENTITY torbutton.prefs.sec_html5_desc "HTML5 video i audio mediji postaju klik-za-pokrenuti prek NoScripte."> -<!ENTITY torbutton.prefs.sec_html5_tooltip "On some sites, you might need to use the NoScript toolbar button to enable these media objects."> +<!ENTITY torbutton.prefs.sec_html5_tooltip "Možda ćete morati koristiti NoScript alatnu traku na nekim stranicama kako bi omogućili ove medijske objekte."> <!ENTITY torbutton.prefs.sec_some_jit_desc "Neke JavaScript optimizacije performansi su isključene."> <!ENTITY torbutton.prefs.sec_jit_desc_tooltip "ION JIT, Type Inference, ASM.JS."> <!ENTITY torbutton.prefs.sec_baseline_jit_desc_tooltip "Baseline JIT."> -<!ENTITY torbutton.prefs.sec_jit_slower_desc "Scripts on some sites may run slower."> +<!ENTITY torbutton.prefs.sec_jit_slower_desc "Skripte mogu biti sporije na nekim stranicama."> <!ENTITY torbutton.prefs.sec_jar_desc "Udaljene JAR datoteke su blokirane."> <!ENTITY torbutton.prefs.sec_jar_tooltip "JAR datoteke su ekstremno rijetke ne webu, ali mogu biti izvor XSS i ostalih napada."> -<!ENTITY torbutton.prefs.sec_mathml_desc "Some mechanisms of displaying math equations are disabled."> +<!ENTITY torbutton.prefs.sec_mathml_desc "Neki mehanizmi prikaza matematičkih jednadžbi su onemogućeni."> <!ENTITY torbutton.prefs.sec_mathml_desc_tooltip "MathML je onemogućen."> <!ENTITY torbutton.prefs.sec_med_high "Srednje-Visoko"> <!ENTITY torbutton.prefs.sec_all_jit_desc "Sve JavaScript optimizacije performansi su onemogućene."> -<!ENTITY torbutton.prefs.sec_font_rend_desc "Some font rendering features are disabled."> -<!ENTITY torbutton.prefs.sec_font_rend_graphite_tooltip "The Graphite font rendering mechanism is disabled."> +<!ENTITY torbutton.prefs.sec_font_rend_desc "Neke značajke prikazivanja fontoca su onemogućene."> +<!ENTITY torbutton.prefs.sec_font_rend_graphite_tooltip "Mehanizam za prikaz Graphite fonta je onemogućen."> <!ENTITY torbutton.prefs.sec_svg_desc "Neki tipovi slika su onemogućeni."> <!ENTITY torbutton.prefs.sec_svg_desc_tooltip "SVG format je onemogućen."> <!ENTITY torbutton.prefs.sec_js_https_desc "JavaScript je onemogućen po zadanom na svim ne-HTTPS stranicama."> -<!ENTITY torbutton.prefs.sec_js_desc_tooltip "JavaScript can be enabled on a per-site basis via the NoScript toolbar button."> +<!ENTITY torbutton.prefs.sec_js_desc_tooltip "JavaScript može biti omogućen posebice za svaku stranicu putem gumba na NoScript alatnoj traci."> <!ENTITY torbutton.prefs.sec_high "Visoko"> <!ENTITY torbutton.prefs.sec_all_js_desc "JavaScript je onemogućen po zadanom na svim stranicama."> <!ENTITY torbutton.prefs.sec_audio_video_desc "Većina audio i video formata su onemogućeni."> -<!ENTITY torbutton.prefs.sec_audio_video_desc_tooltip "WebM is the only codec that remains enabled."> +<!ENTITY torbutton.prefs.sec_audio_video_desc_tooltip "WebM je jedini kodek koji ostaje omogućen."> <!ENTITY torbutton.prefs.sec_webfonts_desc "Neki fontovi i ikone mogu biti netočno prikazani."> <!ENTITY torbutton.prefs.sec_webfonts_desc_tooltip "Fontovi pružani od strane web-stranica su blokirani."> -<!ENTITY torbutton.prefs.sec_custom "Custom Values"> -<!ENTITY torbutton.circuit_display.title "Tor circuit for this site"> +<!ENTITY torbutton.prefs.sec_custom "Prilagođene vrijednosti"> +<!ENTITY torbutton.circuit_display.title "Tor krug za ovu stranicu">

1 0

[translation/tor-launcher-progress_completed] Update translations for tor-launcher-progress_completed
by translation＠torproject.org 18 Mar '16

18 Mar '16

commit d921d5e10d3f8fc82ce9cca80c2f1a445588c094 Author: Translation commit bot <translation(a)torproject.org> Date: Fri Mar 18 08:45:41 2016 +0000 Update translations for tor-launcher-progress_completed --- hr/progress.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hr/progress.dtd b/hr/progress.dtd index 9c3caaf..da8dda4 100644 --- a/hr/progress.dtd +++ b/hr/progress.dtd @@ -1,4 +1,4 @@ <!ENTITY torprogress.dialog.title "Status Tor-a"> <!ENTITY torprogress.openSettings "Otvori Postavke"> <!ENTITY torprogress.heading "Spajanje na Tor mrežu"> -<!ENTITY torprogress.pleaseWait "Molimo pričekajte dok uspostavimo konekciju sa Tor mrežom."> +<!ENTITY torprogress.pleaseWait "Molimo pričekajte dok uspostavljamo vezu sa Tor mrežom.   Ovo može potrajati nekoliko minuta.">

1 0

[translation/tor-launcher-progress] Update translations for tor-launcher-progress
by translation＠torproject.org 18 Mar '16

18 Mar '16

commit 12265536e2f397664a9a0b5e01c81225b1bfd56d Author: Translation commit bot <translation(a)torproject.org> Date: Fri Mar 18 08:45:36 2016 +0000 Update translations for tor-launcher-progress --- hr/progress.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hr/progress.dtd b/hr/progress.dtd index 4bacee4..da8dda4 100644 --- a/hr/progress.dtd +++ b/hr/progress.dtd @@ -1,4 +1,4 @@ <!ENTITY torprogress.dialog.title "Status Tor-a"> <!ENTITY torprogress.openSettings "Otvori Postavke"> <!ENTITY torprogress.heading "Spajanje na Tor mrežu"> -<!ENTITY torprogress.pleaseWait "Please wait while we establish a connection to the Tor network.  This may take several minutes."> +<!ENTITY torprogress.pleaseWait "Molimo pričekajte dok uspostavljamo vezu sa Tor mrežom.   Ovo može potrajati nekoliko minuta.">

1 0

[metrics-web/master] Add new module on disagreement among dirauths.
by karsten＠torproject.org 18 Mar '16

18 Mar '16

commit 19572294cf53147693a41cf2cac3916f4bb2a633 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Mar 17 13:58:41 2016 +0100 Add new module on disagreement among dirauths. --- .../org/torproject/metrics/collectdescs/Main.java | 1 + modules/disagreement/.gitignore | 2 + modules/disagreement/build.xml | 37 ++ .../org/torproject/metrics/disagreement/Main.java | 618 +++++++++++++++++++++ shared/bin/20-run-disagreement-stats.sh | 5 + shared/bin/99-copy-stats-files.sh | 1 + website/etc/metrics.json | 22 + website/etc/web.xml | 1 + .../metrics/web/research/ResearchStatsServlet.java | 1 + 9 files changed, 688 insertions(+) diff --git a/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java b/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java index c95df18..dc6ef82 100644 --- a/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java +++ b/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java @@ -21,6 +21,7 @@ public class Main { "/recent/relay-descriptors/consensuses/", "/recent/relay-descriptors/extra-infos/", "/recent/relay-descriptors/server-descriptors/", + "/recent/relay-descriptors/votes/", "/recent/torperf/" }, 0L, new File("../../shared/in"), true); } } diff --git a/modules/disagreement/.gitignore b/modules/disagreement/.gitignore new file mode 100644 index 0000000..bb84c80 --- /dev/null +++ b/modules/disagreement/.gitignore @@ -0,0 +1,2 @@ +stats/ + diff --git a/modules/disagreement/build.xml b/modules/disagreement/build.xml new file mode 100644 index 0000000..ce1c022 --- /dev/null +++ b/modules/disagreement/build.xml @@ -0,0 +1,37 @@ +<project default="run" name="disagreement" basedir="."> + + <property name="disagreement-sources" value="src/main/java"/> + <property name="disagreement-classes" value="classes"/> + <path id="classpath"> + <pathelement path="${disagreement-classes}"/> + <fileset dir="../../shared/lib"> + <include name="descriptor-1.1.0.jar"/> + <include name="commons-compress-1.4.1.jar"/> + <include name="xz-1.0.jar"/> + </fileset> + </path> + + <target name="compile"> + <mkdir dir="${disagreement-classes}"/> + <javac destdir="${disagreement-classes}" + srcdir="${disagreement-sources}" + source="1.7" + target="1.7" + debug="true" + deprecation="true" + optimize="false" + failonerror="true" + includeantruntime="false"> + <classpath refid="classpath"/> + </javac> + </target> + + <target name="run" depends="compile"> + <java fork="true" + maxmemory="2g" + classname="org.torproject.metrics.disagreement.Main"> + <classpath refid="classpath"/> + </java> + </target> +</project> + diff --git a/modules/disagreement/src/main/java/org/torproject/metrics/disagreement/Main.java b/modules/disagreement/src/main/java/org/torproject/metrics/disagreement/Main.java new file mode 100644 index 0000000..b612c43 --- /dev/null +++ b/modules/disagreement/src/main/java/org/torproject/metrics/disagreement/Main.java @@ -0,0 +1,618 @@ +package org.torproject.metrics.disagreement; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.LineNumberReader; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.TimeZone; +import java.util.TreeMap; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorFile; +import org.torproject.descriptor.DescriptorReader; +import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.NetworkStatusEntry; +import org.torproject.descriptor.RelayNetworkStatusVote; + +/* Read all relay network status votes from the in/ subdirectory with a + * valid-after time of 12:00:00, extract attributes like relay flags or + * bandwidth measurements that the directory authorities assigned to + * relays, and output aggregate statistics on disagreement among the + * directory authorities. + * + * When initializing from descriptor archives, put tarballs in the in/ + * subdirectory, run this code, and then move the tarballs away. + * Otherwise, tarballs will be re-processed in each subsequent execution. + * + * Recent descriptors can stay in the in/ subdirectory and be re-processed + * in each execution. */ +public class Main { + + public static void main(String[] args) throws Exception { + new Main().run(); + } + + public void run() throws Exception { + readResults(); + readDescriptors(); + aggregate(); + writeResults(); + } + + /* We're processing a lot of strings, including authority identities, + * relay fingerprints, and attributes like relay flags. It would be + * wasteful to store more than one instance of these strings in memory. + * We also want to store them as part of long integers below. + * + * That's why we're resolving all strings to integers and keeping maps + * from string to integers. In case of authority identities and relay + * fingerprints we don't need to resolve integers back to strings, but + * in case of attributes we need to put attribute strings into the + * output file, so we're also keeping a list of attribute strings in + * insertion order. */ + private Map<String, Integer> authorityIndexes = + new HashMap<String, Integer>(); + private Map<String, Integer> fingerprintIndexes = + new HashMap<String, Integer>(); + private Map<String, Integer> attributeIndexes = + new HashMap<String, Integer>(); + private List<String> attributeStrings = new ArrayList<String>(); + + public Main() { + + /* Initialize maps from strings to integers and back by adding the + * empty string as 0-th element. This is necessary, because we want + * to be able to treat 0 as special case below. Once we add non-empty + * strings to maps, they'll be indexed starting at 1. */ + this.getAuthorityIndexForString(""); + this.getFingerprintIndexForString(""); + this.getAttributeIndexForString(""); + } + + /* Resolve the given authority string to its authority index, possibly + * after adding it to the mapping if it wasn't contained before. */ + protected int getAuthorityIndexForString(String authorityString) { + return putToMapsAndReturnIndex(this.authorityIndexes, null, + authorityString); + } + + /* Resolve the given fingerprint string to its fingerprint index, + * possibly after adding it to the mapping if it wasn't contained + * before. */ + protected int getFingerprintIndexForString(String fingerprintString) { + return putToMapsAndReturnIndex(this.fingerprintIndexes, null, + fingerprintString); + } + + /* Resolve the given attribute string to its attribute index, possibly + * after adding it to the mapping if it wasn't contained before. */ + protected int getAttributeIndexForString(String attributeString) { + return putToMapsAndReturnIndex(this.attributeIndexes, + this.attributeStrings, attributeString); + } + + /* Resolve the given attribute index to its attribute string, or return + * null if this attribute index doesn't exist. */ + protected String getAttributeStringForIndex(int attributeIndex) { + if (attributeIndex < 0 || attributeIndex >= attributeStrings.size()) { + return null; + } else { + return this.attributeStrings.get(attributeIndex); + } + } + + /* Helper method: Return an index for a given string and possibly put + * the string into the map and corresponding list if it was not + * contained before. */ + protected int putToMapsAndReturnIndex( + Map<String, Integer> stringToIntMap, List<String> strings, + String string) { + Integer index = stringToIntMap.get(string); + if (index == null) { + if (strings != null) { + strings.add(string); + } + stringToIntMap.put(string, stringToIntMap.size()); + } + return stringToIntMap.get(string); + } + + /* The following code is heavily optimized towards low memory usage, + * because we're potentially processing a lot of data, especially when + * initializing from descriptor archives. + * + * We need to be able to store weeks or even months of votes in memory, + * which is really a lot of data. The reason is that we need all votes + * published at the same hour to be present when running aggregations, + * but we cannot rely on the order of incoming votes. That's why we + * have to process votes in two steps: first, we extract everything we + * need from votes and store it in memory, and second, we aggregate what + * we have in memory and output aggregate results. + * + * We're going rather low-level here by converting each attribute that + * an authority assigned to a relay at a certain vote valid-after time + * into a single 64-bit signed long integer. We're doing this by + * subdividing (most of) the available bits into ranges for the + * different parts we want to store. As an added requirement we're + * arranging ranges in a way that we can later process long values in + * sorted order without keeping much state. Ranges are: + * - 22 bits for the valid-after time in half hours since 1970-01-01 + * 00:00:00 (which won't overflow until year 2209), + * - 8 bits for the attribute index (which allows up to 253 different + * relay flags in addition to reserved 0, "Listed", and + * "Measured"), + * - 24 bits for the fingerprint index (which allows over 16 million + * different relay fingerprints in the current execution), and + * - 6 bits for the authority index (which allows up to 63 different + * authorities in addition to the reserved 0). + * + * If any of these numbers overflows during the execution, we'll detect + * that, suggest to process less data at once, and exit with an error. + * Still, it should be possible to process months of data in a single + * execution. For example, one set of votes published at the same + * valid-after hour in December 2015 required to keep 420,000 long + * values in memory, which is roughly 3.2 MiB plus list overhead. */ + protected List<Long> assignments = new ArrayList<Long>(); + protected final static int VALIDAFTER_LEN = 22, ATTRIBUTE_LEN = 8, + FINGERPRINT_LEN = 24, AUTHORITY_LEN = 6; + protected final static int + VALIDAFTER_SHIFT = ATTRIBUTE_LEN + FINGERPRINT_LEN + AUTHORITY_LEN, + ATTRIBUTE_SHIFT = FINGERPRINT_LEN + AUTHORITY_LEN, + FINGERPRINT_SHIFT = AUTHORITY_LEN, + AUTHORITY_SHIFT = 0; + + /* Define some constants for timestamp math. */ + protected final static long HALF_HOUR = 30L * 60L * 1000L, + ONE_HOUR = 2L * HALF_HOUR, HALF_DAY = 12L * ONE_HOUR, + ONE_DAY = 2L * HALF_DAY; + + /* Convert the given valid-after time in milliseconds, attribute index, + * fingerprint index, and authority index to a long integer following + * the conversion rules stated above. Return -1 in case of + * overflows. */ + protected static long convertToLongValue(long validAfterMillis, + int attributeIndex, int fingerprintIndex, int authorityIndex) { + long validAfterHalfHours = validAfterMillis / HALF_HOUR; + if (validAfterHalfHours < 0L || + validAfterHalfHours >= (1L << VALIDAFTER_LEN)) { + return -1; + } + if (attributeIndex < 0 || attributeIndex >= (1 << ATTRIBUTE_LEN)) { + return -1; + } + if (fingerprintIndex < 0 || + fingerprintIndex >= (1 << FINGERPRINT_LEN)) { + return -1; + } + if (authorityIndex < 0 || authorityIndex >= (1 << AUTHORITY_LEN)) { + return -1; + } + long longValue = (validAfterHalfHours << VALIDAFTER_SHIFT) + + ((long) attributeIndex << ATTRIBUTE_SHIFT) + + ((long) fingerprintIndex << FINGERPRINT_SHIFT) + + ((long) authorityIndex << AUTHORITY_SHIFT); + return longValue; + } + + /* Extract the valid-after time in milliseconds from the given long + * integer value. */ + protected static long extractValidAfterMillisFromLongValue( + long longValue) { + return (longValue >> VALIDAFTER_SHIFT) * HALF_HOUR; + } + + /* Extract the attribute index from the given long integer value. */ + protected static int extractAttributeIndexFromLongValue( + long longValue) { + return (int) ((longValue >> ATTRIBUTE_SHIFT) % (1 << ATTRIBUTE_LEN)); + } + + /* Extract the fingerprint index from the given long integer value. */ + protected static int extractFingerprintIndexFromLongValue( + long longValue) { + return (int) ((longValue >> FINGERPRINT_SHIFT) % + (1 << FINGERPRINT_LEN)); + } + + /* Extract the authority index from the given long integer value. */ + protected static int extractAuthorityIndexFromLongValue( + long longValue) { + return (int) ((longValue >> AUTHORITY_SHIFT) % (1 << AUTHORITY_LEN)); + } + + /* Keep all aggregated results in memory, so that we easily merge new + * results obtained in the current execution. + * + * Another reason for keeping all results in memory is that we may have + * processed some votes from a given valid-after time before, and now we + * need to decide whether we keep old results or replace them with new + * results. We decide this by using results that are based on more + * votes, which could be old or new results. + * + * Map keys are valid-after times formatted as strings, map values are + * sorted lists of all subsequent columns starting with that valid-after + * time. */ + protected SortedMap<String, List<String>> results = + new TreeMap<String, List<String>>(); + + /* Store all results in this .csv file. */ + protected File resultsFile = new File("stats/disagreement.csv"); + + /* Use the following .csv header line for results. */ + protected String resultsHeaderLine = + "validafter,attribute,votes,required,max,relays"; + + /* Read results from the previous execution to memory and do some minor + * validation of the file format while doing so. Return immediately + * without error if the file does not exist (yet). */ + private void readResults() throws Exception { + if (!this.resultsFile.exists()) { + return; + } + LineNumberReader lnr = new LineNumberReader(new BufferedReader( + new FileReader(this.resultsFile))); + String line; + if ((line = lnr.readLine()) == null || + !line.equals(this.resultsHeaderLine)) { + lnr.close(); + throw new IOException("Unexpected line " + lnr.getLineNumber() + + " in " + this.resultsFile + "."); + } + while ((line = lnr.readLine()) != null) { + if (!line.contains(",")) { + lnr.close(); + throw new IOException("Cannot parse invalid line " + + lnr.getLineNumber() + " in " + this.resultsFile + "."); + } + int indexOfFirstComma = line.indexOf(","); + String validafter = line.substring(0, indexOfFirstComma); + String otherColumns = line.substring(indexOfFirstComma); + if (!this.results.containsKey(validafter)) { + this.results.put(validafter, new ArrayList<String>()); + } + this.results.get(validafter).add(otherColumns); + } + lnr.close(); + } + + /* Write results to the results file. More precisely, write them to a + * temporary file and rename it to the target file to avoid failing half + * way through and losing the previous results file. */ + private void writeResults() throws Exception { + this.resultsFile.getParentFile().mkdirs(); + File resultsTmpFile = new File(this.resultsFile + ".tmp"); + BufferedWriter bw = new BufferedWriter(new FileWriter( + resultsTmpFile)); + bw.write(this.resultsHeaderLine + "\n"); + for (Map.Entry<String, List<String>> e : this.results.entrySet()) { + String validafter = e.getKey(); + for (String otherColumns : e.getValue()) { + bw.write(validafter + otherColumns + "\n"); + } + } + bw.close(); + resultsTmpFile.renameTo(this.resultsFile); + } + + /* Read relay network status votes from this directory. */ + protected File[] inDirectories = new File[] { + new File("../../shared/in/archive/relay-descriptors/votes"), + new File("../../shared/in/recent/relay-descriptors/votes") + }; + + /* Read relay network status votes from disk and extract all relevant + * pieces from them. */ + protected void readDescriptors() throws Exception { + DescriptorReader descriptorReader = + DescriptorSourceFactory.createDescriptorReader(); + descriptorReader.setMaxDescriptorFilesInQueue(5); + for (File inDirectory : this.inDirectories) { + descriptorReader.addDirectory(inDirectory); + } + Iterator<DescriptorFile> descriptorFiles = + descriptorReader.readDescriptors(); + while (descriptorFiles.hasNext()) { + DescriptorFile descriptorFile = descriptorFiles.next(); + for (Descriptor descriptor : descriptorFile.getDescriptors()) { + if ((descriptor instanceof RelayNetworkStatusVote)) { + RelayNetworkStatusVote vote = + (RelayNetworkStatusVote) descriptor; + processVote(vote); + } + } + } + } + + private static final String LISTED_ATTRIBUTE = "Listed", + MEASURED_ATTRIBUTE = "Measured"; + + /* Process a single relay network status vote. */ + private void processVote(RelayNetworkStatusVote vote) throws Exception { + long validAfterMillis = vote.getValidAfterMillis(); + if (validAfterMillis % ONE_DAY != HALF_DAY) { + /* Only process votes with a valid-after time of 12:00:00 as a means + * to reduce the overall amount of data. */ + return; + } + + /* Use the authority's identity to distinguish votes. */ + String authorityIdentity = vote.getIdentity(); + + /* Collect a set of all attributes that the authority assigns in this + * vote, which includes all known flags, the general "Listed" + * attribute for listing relays, and possibly the "Measured" attribute + * for bandwidth-measured relays. */ + Set<String> knownAttributes = new HashSet<String>( + vote.getKnownFlags()); + knownAttributes.add(LISTED_ATTRIBUTE); + + /* Go through all status entries in this vote and remember which + * attributes this authority assigns to which relays. */ + for (NetworkStatusEntry entry : + vote.getStatusEntries().values()) { + + /* Use the relay's fingerprint to distinguish relays. */ + String fingerprint = entry.getFingerprint(); + + /* Compile a set of all attributes assigned to this relay, including + * all relay flags, "Listed", and possibly "Measured". */ + Set<String> attributes = new HashSet<String>(entry.getFlags()); + attributes.add(LISTED_ATTRIBUTE); + if (entry.getMeasured() >= 0L) { + attributes.add(MEASURED_ATTRIBUTE); + knownAttributes.add(MEASURED_ATTRIBUTE); + } + + /* Remember all attributes assigned to this relay. */ + this.addAssignedAttributes(validAfterMillis, attributes, + fingerprint, authorityIdentity); + } + + /* Remember all attributes assigned by the authority in this vote. */ + this.addKnownAttributes(validAfterMillis, knownAttributes, + authorityIdentity); + + /* Remember all fingerprints voted on by the authority at the given + * valid-after time. */ + this.addKnownFingerprints(validAfterMillis, + vote.getStatusEntries().keySet()); + } + + /* Remember all attributes assigned to a given relay by an authority at + * a given valid-after time. These are converted to long integers + * with all components being non-zero. */ + protected void addAssignedAttributes(long validAfterMillis, + Set<String> attributes, String fingerprint, + String authorityIdentity) throws Exception { + int fingerprintIndex = this.getFingerprintIndexForString(fingerprint); + int authorityIndex = this.getAuthorityIndexForString( + authorityIdentity); + for (String attribute : attributes) { + int attributeIndex = this.getAttributeIndexForString(attribute); + long longValue = convertToLongValue(validAfterMillis, + attributeIndex, fingerprintIndex, authorityIndex); + if (longValue < 0L) { + throw new Exception("Could not convert vote data to the internal " + + "format. Try processing fewer votes at once."); + } + this.assignments.add(longValue); + } + } + + /* Remember all attributes voted on by an authority at a given + * valid-after time. These are converted to long integers with + * fingerprint component being zero. */ + protected void addKnownAttributes(long validAfterMillis, + Set<String> knownAttributes, String authorityIdentity) + throws Exception { + int authorityIndex = this.getAuthorityIndexForString( + authorityIdentity); + for (String attribute : knownAttributes) { + int attributeIndex = this.getAttributeIndexForString(attribute); + long longValue = convertToLongValue(validAfterMillis, + attributeIndex, 0, authorityIndex); + if (longValue < 0L) { + throw new Exception("Could not convert vote data to the internal " + + "format. Try processing fewer votes at once."); + } + this.assignments.add(longValue); + } + } + + /* Remember all fingerprints known by an authority at a given + * valid-after time. These are converted to long integers with + * attribute and authority component being zero. */ + protected void addKnownFingerprints(long validAfterMillis, + Set<String> knownFingerprints) throws Exception { + for (String fingerprint : knownFingerprints) { + int fingerprintIndex = this.getFingerprintIndexForString( + fingerprint); + long longValue = convertToLongValue(validAfterMillis, 0, + fingerprintIndex, 0); + if (longValue < 0L) { + throw new Exception("Could not convert vote data to the internal " + + "format. Try processing fewer votes at once."); + } + this.assignments.add(longValue); + } + } + + /* Aggregate everything we extracted from votes earlier into statistics + * on disagreement among directory authorities. */ + protected void aggregate() { + + /* Initialize a date format for formatting valid-after times. */ + DateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss", Locale.US); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + + /* Sort long integer values and append the largest possible long + * value, so that we can process values in the following order + * (columns are valid-after time, attribute, fingerprint, authority; + * + stands for a value > 0, 0 stands for 0): + * (+, 0, +, 0): all fingerprints known at a given valid-after time + * (+, +, 0, +): all attributes known by a given authority + * (+, +, +, +): attributes assigned to relays by authority + * ... + * (MAX_VALUE): end-of-list marker */ + Collections.sort(this.assignments); + this.assignments.add(Long.MAX_VALUE); + + /* Remember long value and some of its components from the last + * iteration. */ + long lastLongValue = -1L, lastValidAfterMillis = -1L; + int lastAttributeIndex = -1, lastFingerprintIndex = -1; + + /* Keep a list of all output lines for a single valid-after time. */ + List<String> outputLines = new ArrayList<String>(); + + /* Keep counters for the number of fingerprints seen at a valid-after + * time, the number of authorities voting on an attribute, and the + * number of votes that a relay received for a given attribute. */ + int knownFingerprintsByAllAuthorities = 0, + authoritiesVotingOnAttribute = 0, votesForAttribute = 0; + + /* Keep counters of relays receiving a given number of votes on an + * attribute. The number at element i is the number of relays + * receiving i votes. */ + int[] relays = new int[(1 << AUTHORITY_LEN)]; + + /* Go through all long values in ascending order. */ + for (long longValue : this.assignments) { + + /* Skip duplicate values. */ + if (lastLongValue == longValue) { + continue; + } + + /* If we're looking at attributes assigned to relays and just moved + * from one non-zero fingerprint to the next, we need to wrap up + * results for the last fingerprint before moving on. */ + int fingerprintIndex = extractFingerprintIndexFromLongValue( + longValue); + if (lastAttributeIndex > 0 && lastFingerprintIndex > 0 && + lastFingerprintIndex != fingerprintIndex) { + + /* This relay received at least one vote for the given attribute, + * or otherwise it wouldn't be contained in the list of long + * values. Increment the counter for the number of votes we + * counted for this attribute, and then reset that counter. */ + relays[0]--; + relays[votesForAttribute]++; + votesForAttribute = 0; + } + + /* If we're looking at attributes and just moved from one non-zero + * attribute to the next, we need to wrap up results for the last + * attribute before moving on. And if we just moved to the first + * attribute, initialize counters. */ + int attributeIndex = extractAttributeIndexFromLongValue(longValue); + if (lastAttributeIndex >= 0 && + lastAttributeIndex != attributeIndex) { + + /* If we just finished a non-zero attribute, wrap it up. + * Determine the number of votes required for getting into the + * consensus, which is typically the majority of votes, except for + * the "Measured" attribute, where it's set to 3. Format one + * output line for each possible number of votes, from 0 to the + * total number of authorities voting on the attribute. */ + if (lastAttributeIndex > 0) { + String lastAttribute = this.getAttributeStringForIndex( + lastAttributeIndex); + int required = lastAttribute.equals(MEASURED_ATTRIBUTE) ? 3 + : (authoritiesVotingOnAttribute / 2) + 1; + for (int i = 0; i <= authoritiesVotingOnAttribute; i++) { + outputLines.add(String.format(",%s,%d,%d,%d,%d", + lastAttribute, i, required, authoritiesVotingOnAttribute, + relays[i])); + } + } + + /* Reset counters and initialize the bucket at index 0 with the + * total number of fingerprints known at this valid-after time. */ + authoritiesVotingOnAttribute = 0; + relays = new int[(1 << AUTHORITY_LEN)]; + relays[0] = knownFingerprintsByAllAuthorities; + } + + /* If we just moved from one valid-after time to the next, we need + * to wrap up results for the last valid-after time before moving + * on. */ + long validAfterMillis = extractValidAfterMillisFromLongValue( + longValue); + if (lastValidAfterMillis >= 0L && + lastValidAfterMillis < validAfterMillis) { + + /* Check if results already contain lines for this valid-after + * time. If so, only replace them with new results lines if there + * are more new lines than old lines. The rationale is that more + * lines are very likely based on more votes, and we want to + * include as many votes as possible in the aggregation. */ + String validAfterString = dateTimeFormat.format( + lastValidAfterMillis); + if (!this.results.containsKey(validAfterString) || + this.results.get(validAfterString).size() < + outputLines.size()) { + + /* Sort results lines, and then put them in. */ + Collections.sort(outputLines); + this.results.put(validAfterString, outputLines); + } + + /* Prepare for aggregating votes from the next valid-after + * time. */ + outputLines = new ArrayList<String>(); + knownFingerprintsByAllAuthorities = 0; + } + + /* If we reached our end-of-list marker, stop here. */ + if (longValue == Long.MAX_VALUE) { + break; + } + + /* Look at the current indexes and increment one of the three + * counters. If this value doesn't contain an attribute index, it + * was put in for counting all known fingerprints at this + * valid-after time. */ + if (attributeIndex == 0) { + knownFingerprintsByAllAuthorities++; + } + + /* Otherwise, if this value doesn't contain a fingerprint index, it + * was put in for counting authorities voting on a given attribute + * at the current valid-after time. */ + else if (fingerprintIndex == 0) { + authoritiesVotingOnAttribute++; + } + + /* Otherwise, if both indexes are non-zero, this value was put in to + * count how many authorities assign the attribute to this relay at + * this valid-after time. */ + else { + votesForAttribute++; + } + + /* Prepare moving on to the next value. */ + lastLongValue = longValue; + lastValidAfterMillis = validAfterMillis; + lastAttributeIndex = attributeIndex; + lastFingerprintIndex = fingerprintIndex; + } + } +} + diff --git a/shared/bin/20-run-disagreement-stats.sh b/shared/bin/20-run-disagreement-stats.sh new file mode 100755 index 0000000..9abe1e2 --- /dev/null +++ b/shared/bin/20-run-disagreement-stats.sh @@ -0,0 +1,5 @@ +#!/bin/sh +cd modules/disagreement/ +ant | grep "\[java\]" +cd ../../ + diff --git a/shared/bin/99-copy-stats-files.sh b/shared/bin/99-copy-stats-files.sh index 22018ba..23d46cf 100755 --- a/shared/bin/99-copy-stats-files.sh +++ b/shared/bin/99-copy-stats-files.sh @@ -5,4 +5,5 @@ cp -a modules/connbidirect/stats/connbidirect2.csv shared/stats/ cp -a modules/advbwdist/stats/advbwdist.csv shared/stats/ cp -a modules/hidserv/stats/hidserv.csv shared/stats/ cp -a modules/clients/stats/clients.csv shared/stats/ +cp -a modules/disagreement/stats/disagreement.csv shared/stats/ diff --git a/website/etc/metrics.json b/website/etc/metrics.json index 2e1ee4f..03e8b85 100644 --- a/website/etc/metrics.json +++ b/website/etc/metrics.json @@ -834,5 +834,27 @@ "type": "Link", "level": "Basic", "description": "Uncharted made a visualization of data flow in the Tor network where they place each <a href=\"about.html#relay\">relay</a> on a world map and illustrate traffic exchanged between relays as animated dots. More details can be found on the <a href=\"https://torflow.uncharted.software/\">Uncharted website</a>.<a href=\"https://torflow.uncharted.software/\"><img src=\"images/uncharted-data-flow.png\" alt=\"Data flow in the Tor network\"></a>" + }, + { + "id": "disagreement-data", + "title": "Disagreement among the directory authorities", + "tags": [ + "Relays" + ], + "type": "Data", + "level": "Advanced", + "description": "The following data file contains statistics about agreement or disagreement among the <a href=\"about.html#directory-authority\">directory authorities</a>. Once per hour the directory authorities exchange votes with their view of the <a href=\"about.html#relay\">relays</a> in the network including attributes like <a href=\"about.html#relay-flag\">relay flags</a> or bandwidth measurements. This data file includes counts of relays by number of directory authorities assigning them a given attribute.", + "data_file": "stats/disagreement.csv", + "data_column_spec": [ + "validafter: UTC timestamp (YYYY-MM-DD HH:MM:SS) after which votes became valid, which may be used as the vote publication time.", + "attribute: Attribute assigned to relays by directory authorities, which includes relay flags like \"Exit\" or \"BadExit\" as well as \"Listed\" for relays being listed in a vote and \"Measured\" for relays being measured by the bandwidth authorities.", + "votes: Number of votes assigning the attribute to relays.", + "required: Required number of votes for the attribute to be assigned to a relay for being included in the consensus.", + "max: Maximum number of possible votes assigning the attribute to relays.", + "relays: Number of relays that got the given number of votes for the given attribute." + ], + "related": [ + "relayflags" + ] } ] diff --git a/website/etc/web.xml b/website/etc/web.xml index 674be4c..8587947 100644 --- a/website/etc/web.xml +++ b/website/etc/web.xml @@ -77,6 +77,7 @@ <url-pattern>/connbidirect-data.html</url-pattern> <url-pattern>/connbidirect2-data.html</url-pattern> <url-pattern>/hidserv-data.html</url-pattern> + <url-pattern>/disagreement-data.html</url-pattern> </servlet-mapping> <servlet> diff --git a/website/src/org/torproject/metrics/web/research/ResearchStatsServlet.java b/website/src/org/torproject/metrics/web/research/ResearchStatsServlet.java index 1e045c1..a4194f2 100644 --- a/website/src/org/torproject/metrics/web/research/ResearchStatsServlet.java +++ b/website/src/org/torproject/metrics/web/research/ResearchStatsServlet.java @@ -36,6 +36,7 @@ public class ResearchStatsServlet extends HttpServlet { this.availableStatisticsFiles.add("connbidirect2"); this.availableStatisticsFiles.add("advbwdist"); this.availableStatisticsFiles.add("hidserv"); + this.availableStatisticsFiles.add("disagreement"); } public long getLastModified(HttpServletRequest request) {

1 0

[metrics-web/master] Ignore votes in most of the modules.
by karsten＠torproject.org 18 Mar '16

18 Mar '16

commit afd9ccb9212389d7a785d60586bdd35c2185366d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Mar 17 20:58:38 2016 +0100 Ignore votes in most of the modules. --- .../src/org/torproject/metrics/clients/Main.java | 8 ++++++-- modules/legacy/build.xml | 2 +- modules/legacy/config.template | 6 ++++-- .../org/torproject/ernie/cron/Configuration.java | 19 +++++++++++++++---- .../legacy/src/org/torproject/ernie/cron/Main.java | 2 +- .../cron/RelayDescriptorDatabaseImporter.java | 22 ++++++++++++++-------- 6 files changed, 41 insertions(+), 18 deletions(-) diff --git a/modules/clients/src/org/torproject/metrics/clients/Main.java b/modules/clients/src/org/torproject/metrics/clients/Main.java index 430cb1e..fcd18a3 100644 --- a/modules/clients/src/org/torproject/metrics/clients/Main.java +++ b/modules/clients/src/org/torproject/metrics/clients/Main.java @@ -62,9 +62,13 @@ public class Main { descriptorReader.setExcludeFiles(new File( "status/relay-descriptors")); descriptorReader.addDirectory(new File( - "../../shared/in/recent/relay-descriptors")); + "../../shared/in/recent/relay-descriptors/consensuses")); descriptorReader.addDirectory(new File( - "../../shared/in/archive/relay-descriptors")); + "../../shared/in/recent/relay-descriptors/extra-infos")); + descriptorReader.addDirectory(new File( + "../../shared/in/archive/relay-descriptors/consensuses")); + descriptorReader.addDirectory(new File( + "../../shared/in/archive/relay-descriptors/extra-infos")); Iterator<DescriptorFile> descriptorFiles = descriptorReader.readDescriptors(); while (descriptorFiles.hasNext()) { diff --git a/modules/legacy/build.xml b/modules/legacy/build.xml index b0dc32b..3ecf147 100644 --- a/modules/legacy/build.xml +++ b/modules/legacy/build.xml @@ -46,7 +46,7 @@  <target name="run" depends="compile"> <java fork="true" - maxmemory="1024m" + maxmemory="2g" classname="org.torproject.ernie.cron.Main"> <classpath refid="classpath"/> </java> diff --git a/modules/legacy/config.template b/modules/legacy/config.template index ce779d9..743fe1e 100644 --- a/modules/legacy/config.template +++ b/modules/legacy/config.template @@ -1,8 +1,10 @@ ## Import directory archives from disk, if available #ImportDirectoryArchives 0 # -## Relative path to directory to import directory archives from -#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/ +## Relative paths to directories to import directory archives from +#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/consensuses/ +#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/server-descriptors/ +#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/extra-infos/ # ## Keep a history of imported directory archive files to know which files ## have been imported before. This history can be useful when importing diff --git a/modules/legacy/src/org/torproject/ernie/cron/Configuration.java b/modules/legacy/src/org/torproject/ernie/cron/Configuration.java index 878e882..86d5d10 100644 --- a/modules/legacy/src/org/torproject/ernie/cron/Configuration.java +++ b/modules/legacy/src/org/torproject/ernie/cron/Configuration.java @@ -7,6 +7,9 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -17,7 +20,8 @@ import java.util.logging.Logger; */ public class Configuration { private boolean importDirectoryArchives = false; - private String directoryArchivesDirectory = "in/relay-descriptors/"; + private List<String> directoryArchivesDirectories = + new ArrayList<String>(); private boolean keepDirectoryArchiveImportHistory = false; private boolean importSanitizedBridges = false; private String sanitizedBridgesDirectory = "in/bridge-descriptors/"; @@ -55,7 +59,7 @@ public class Configuration { this.importDirectoryArchives = Integer.parseInt( line.split(" ")[1]) != 0; } else if (line.startsWith("DirectoryArchivesDirectory")) { - this.directoryArchivesDirectory = line.split(" ")[1]; + this.directoryArchivesDirectories.add(line.split(" ")[1]); } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) { this.keepDirectoryArchiveImportHistory = Integer.parseInt( line.split(" ")[1]) != 0; @@ -117,8 +121,15 @@ public class Configuration { public boolean getImportDirectoryArchives() { return this.importDirectoryArchives; } - public String getDirectoryArchivesDirectory() { - return this.directoryArchivesDirectory; + public List<String> getDirectoryArchivesDirectories() { + if (this.directoryArchivesDirectories.isEmpty()) { + String prefix = "../../shared/in/recent/relay-descriptors/"; + return Arrays.asList( + (prefix + "consensuses/," + prefix + "server-descriptors/," + + prefix + "extra-infos/").split(",")); + } else { + return this.directoryArchivesDirectories; + } } public boolean getKeepDirectoryArchiveImportHistory() { return this.keepDirectoryArchiveImportHistory; diff --git a/modules/legacy/src/org/torproject/ernie/cron/Main.java b/modules/legacy/src/org/torproject/ernie/cron/Main.java index fb0697c..7319efa 100644 --- a/modules/legacy/src/org/torproject/ernie/cron/Main.java +++ b/modules/legacy/src/org/torproject/ernie/cron/Main.java @@ -45,7 +45,7 @@ public class Main { config.getRelayDescriptorDatabaseJDBC() : null, config.getWriteRelayDescriptorsRawFiles() ? config.getRelayDescriptorRawFilesDirectory() : null, - new File(config.getDirectoryArchivesDirectory()), + config.getDirectoryArchivesDirectories(), statsDirectory, config.getKeepDirectoryArchiveImportHistory()) : null; if (rddi != null) { diff --git a/modules/legacy/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java b/modules/legacy/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java index 19f2a0f..35128e7 100644 --- a/modules/legacy/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java +++ b/modules/legacy/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java @@ -173,7 +173,7 @@ public final class RelayDescriptorDatabaseImporter { private boolean importIntoDatabase; private boolean writeRawImportFiles; - private File archivesDirectory; + private List<String> archivesDirectories; private File statsDirectory; private boolean keepImportHistory; @@ -182,14 +182,14 @@ public final class RelayDescriptorDatabaseImporter { * preparing statements. */ public RelayDescriptorDatabaseImporter(String connectionURL, - String rawFilesDirectory, File archivesDirectory, + String rawFilesDirectory, List<String> archivesDirectories, File statsDirectory, boolean keepImportHistory) { - if (archivesDirectory == null || + if (archivesDirectories == null || statsDirectory == null) { throw new IllegalArgumentException(); } - this.archivesDirectory = archivesDirectory; + this.archivesDirectories = archivesDirectories; this.statsDirectory = statsDirectory; this.keepImportHistory = keepImportHistory; @@ -805,12 +805,18 @@ public final class RelayDescriptorDatabaseImporter { } public void importRelayDescriptors() { - if (archivesDirectory.exists()) { - logger.fine("Importing files in directory " + archivesDirectory - + "/..."); + logger.fine("Importing files in directories " + archivesDirectories + + "/..."); + if (!this.archivesDirectories.isEmpty()) { DescriptorReader reader = DescriptorSourceFactory.createDescriptorReader(); - reader.addDirectory(archivesDirectory); + reader.setMaxDescriptorFilesInQueue(10); + for (String archivesPath : this.archivesDirectories) { + File archivesDirectory = new File(archivesPath); + if (archivesDirectory.exists()) { + reader.addDirectory(archivesDirectory); + } + } if (keepImportHistory) { reader.setExcludeFiles(new File(statsDirectory, "database-importer-relay-descriptor-history"));

1 0