November 2019 - tor-commits - lists.torproject.org

[translation/support-portal] https://gitweb.torproject.org/translation.git/commit/?h=support-portal
by translation＠torproject.org 10 Nov '19

10 Nov '19

commit 616c510af401c86257765f48b07b5aa12bf7413c Author: Translation commit bot <translation(a)torproject.org> Date: Sun Nov 10 03:53:06 2019 +0000 https://gitweb.torproject.org/translation.git/commit/?h=support-portal --- contents+tr.po | 49 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/contents+tr.po b/contents+tr.po index 189cc350c..8d7bc3653 100644 --- a/contents+tr.po +++ b/contents+tr.po @@ -953,8 +953,8 @@ msgstr "" msgid "" "Does Tor remove personal information from the data my application sends?" msgstr "" -"Tor kullandığım uygulamanın gönderdiği veriler içinden kişisel bilgilerimi " -"siler mı?" +"Tor uygulaması diğer uygulamaların gönderdiği veriler içinden kişisel " +"bilgilerimi siler mı?" #: https//support.torproject.org/about/no-data-scrubbing/ #: (content/about/no-data-scrubbing/contents+en.lrquestion.description) @@ -1841,7 +1841,7 @@ msgstr "" #: https//support.torproject.org/tbb/tbb-15/ #: (content/tbb/tbb-15/contents+en.lrquestion.title) msgid "Can I download Tor Browser for Chrome OS?" -msgstr "Chrome OS için Tor Browser indirebilir miyim?" +msgstr "Chrome OS için Tor Browser uygulamasını indirebilir miyim?" #: https//support.torproject.org/tbb/tbb-15/ #: (content/tbb/tbb-15/contents+en.lrquestion.description) @@ -3370,8 +3370,8 @@ msgid "" "What is the difference between using Tor Browser and 'Incognito mode' or " "private tabs?" msgstr "" -"Tor Browser kullanmak ile 'Gizli Pencere' (Chrome, Incognito) ya da Gizli " -"Gezinti Firefox, Private) kullanmak arasındaki fark nedir?" +"Tor Browser kullanmak ile 'Gizli Pencere' (Chrome, Incognito Mode) ya da " +"'Gizli Gezinti' (Firefox, Private Tabs) kullanmak arasındaki fark nedir?" #: https//support.torproject.org/tbb/tbb-and-incognito-mode/ #: (content/tbb/tbb-and-incognito-mode/contents+en.lrquestion.description) @@ -3424,7 +3424,7 @@ msgid "" "Browsing](https://support.mozilla.org/en-US/kb/common-myths-about-private-" "browsing)." msgstr "" -"Gizli pencere ya da Gizli gezinti özelliklerindeki sınırlamalar hakkında " +"Gizli Pencere ya da Gizli Gezinti özelliklerindeki sınırlamalar hakkında " "ayrıntılı bilgi almak için, Common Myths about Private " "Browsing](https://support.mozilla.org/en-US/kb/common-myths-about-private-" "browsing) Mozilla makalesine bakabilirsiniz." @@ -6042,8 +6042,8 @@ msgid "" "* If you still can't handle the memory load, consider reducing the amount of" " bandwidth your relay advertises." msgstr "" -"* Hala bellek kullanımı konusunda ilerleme sağlayamıyorsanız, aktarıcınızın " -"sunduğu bant genişliğini azaltmayı düşünün." +"* Buna rağmen bellek kullanımını kontrol altına alamıyorsanız, aktarıcınızın" +" sunduğu bant genişliğini azaltmayı düşünün." #: https//support.torproject.org/operators/relay-memory/ #: (content/operators/relay-memory/contents+en.lrquestion.description) @@ -6159,8 +6159,8 @@ msgstr "" #: (content/operators/upgrade-or-move/contents+en.lrquestion.title) msgid "I want to upgrade/move my relay. How do I keep the same key?" msgstr "" -"Aktarıcımı yükseltmek ya da taşımak istiyorum. Aynı anahtarı kullanmasını " -"nasıl sağlayabilirim?" +"Aktarıcımda sürüm yükseltmek ya da başka bilgisayara taşımak istiyorum. Aynı" +" anahtarın kullanılmasını nasıl sağlayabilirim?" #: https//support.torproject.org/operators/upgrade-or-move/ #: (content/operators/upgrade-or-move/contents+en.lrquestion.description) @@ -6193,6 +6193,9 @@ msgid "" "torrc and the same DataDirectory, then the upgrade should just work and your" " relay will keep using the same key." msgstr "" +"Özetle Tor aktarıcınızın sürümünü yükseltirken aynı torrc dosyasını aynı " +"Data klasöründe tutarsanız sürüm yükseltme işleminden sonra aktarıcınız aynı" +" anahtarı kullanmayı sürdürür." #: https//support.torproject.org/operators/upgrade-or-move/ #: (content/operators/upgrade-or-move/contents+en.lrquestion.description) @@ -6200,6 +6203,9 @@ msgid "" "If you need to pick a new DataDirectory, be sure to copy your old " "keys/ed25519_master_id_secret_key and keys/secret_id_key over." msgstr "" +"Yeni bir Data klasörü kullanmanız gerekiyorsa, eski " +"keys/ed25519_master_id_secret_key ve keys/secret_id_key dosyalarını bu " +"klasöre kopyaladığınızdan emin olun." #: https//support.torproject.org/operators/upgrade-or-move/ #: (content/operators/upgrade-or-move/contents+en.lrquestion.description) @@ -6207,6 +6213,8 @@ msgid "" "Note: As of Tor 0.2.7 we are using new generation identities for relays " "based on ed25519 elliptic curve cryptography." msgstr "" +"Not: Tor 0.2.7 sürümünden sonra aktarıcılar için ed25519 eliptik eğri " +"şifrelemesi ile çalışan yeni nesil kimlikler kullanılıyor." #: https//support.torproject.org/operators/upgrade-or-move/ #: (content/operators/upgrade-or-move/contents+en.lrquestion.description) @@ -6214,6 +6222,8 @@ msgid "" "Eventually they will replace the old RSA identities, but that will happen in" " time, to ensure compatibility with older versions." msgstr "" +"Bunlar eski RSA kimliklerinin yerini aldı. Ancak eski sürümler ile " +"uyumluluğu kaybetmemek için zaman içinde tamamen değiştirilecek." #: https//support.torproject.org/operators/upgrade-or-move/ #: (content/operators/upgrade-or-move/contents+en.lrquestion.description) @@ -6222,6 +6232,9 @@ msgid "" "file: keys/ed25519_master_id_secret_key) and a RSA identity (identity key " "file: keys/secret_id_key)." msgstr "" +"O zamana kadar tüm aktarıcılarda bir ed25519 kimliği (kimlik anahtarı " +"dosyası: keys/ed25519_master_id_secret_key) ve bir RSA kimliği (kimlik " +"anahtarı dosyası: keys/secret_id_key) bulunacak." #: https//support.torproject.org/operators/upgrade-or-move/ #: (content/operators/upgrade-or-move/contents+en.lrquestion.description) @@ -6229,11 +6242,14 @@ msgid "" "You need to copy / backup both of them in order to restore your relay, " "change your DataDirectory or migrate the relay on a new computer." msgstr "" +"Data klasörünü değiştirdiğinizde ya da aktarıcınızı yeni bir bilgisayara " +"taşıdığınızda, aktarıcınızı geri yükleyebilmek için iki anahtar dosyasını da" +" kopyalamanız ya da yedeklemeniz gerekir. " #: https//support.torproject.org/operators/what-is-the-bad-exit-flag/ #: (content/operators/what-is-the-bad-exit-flag/contents+en.lrquestion.title) msgid "What is the BadExit flag?" -msgstr "" +msgstr "Kötü Çıkış (BadExit) işareti nedir?" #: https//support.torproject.org/operators/what-is-the-bad-exit-flag/ #: (content/operators/what-is-the-bad-exit-flag/contents+en.lrquestion.description) @@ -6242,6 +6258,10 @@ msgid "" "This tells Tor to avoid exiting through that relay. In effect, relays with " "this flag become non-exits." msgstr "" +"Bir çıkış aktarıcısı hatalı yapılandırıldığında ya da kötü niyetli " +"davranışlar sergilediğinde Kötü Çıkış (BadExit) olarak işaretlenir. Böylece " +"Tor ağının bu aktarıcıdan çıkış yapması engellenir. Bu şekilde işaretlenmiş " +"aktarıcılar yokmuş gibi davranılır." #: https//support.torproject.org/operators/what-is-the-bad-exit-flag/ #: (content/operators/what-is-the-bad-exit-flag/contents+en.lrquestion.description) @@ -6252,11 +6272,16 @@ msgid "" "team](https://community.torproject.org/relay/community-resources/bad-" "relays/) so we can sort out the issue." msgstr "" +"Aktarıcınız kötü çıkış olarak işaretlenmiş ise, aktarıcınızda trafik " +"yöneltmesi sırasında bir sorun ya da şüpheli bir işlem algılanmış ve sizinle" +" iletişim kurulamamış demektir. Lütfen sorunu çözebilmemiz için [kötü " +"aktarıcılar ekibi](https://community.torproject.org/relay/community-" +"resources/bad-relays/) ile görüşün." #: https//support.torproject.org/operators/what-type-of-relays-are-most-needed/ #: (content/operators/what-type-of-relays-are-most-needed/contents+en.lrquestion.title) msgid "What type of relays are most needed?" -msgstr "" +msgstr "En fazla ihtiyaç duyulan aktarıcı türüne hangisidir?" #: https//support.torproject.org/operators/what-type-of-relays-are-most-needed/ #: (content/operators/what-type-of-relays-are-most-needed/contents+en.lrquestion.description)

1 0

[translation/tpo-web_completed] https://gitweb.torproject.org/translation.git/commit/?h=tpo-web_completed
by translation＠torproject.org 10 Nov '19

10 Nov '19

commit 04e9b8b1821a5952c1bfa6e2d7b64676c758716f Author: Translation commit bot <translation(a)torproject.org> Date: Sun Nov 10 03:52:44 2019 +0000 https://gitweb.torproject.org/translation.git/commit/?h=tpo-web_completed --- contents+tr.po | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contents+tr.po b/contents+tr.po index b9bff99c5..ea3ab5f25 100644 --- a/contents+tr.po +++ b/contents+tr.po @@ -425,8 +425,7 @@ msgstr "" "**Tıpkı Tor kullanıcıları gibi, Tor projesini hayata geçirenler de " "geliştiriciler, araştırmacılar ve kurucular gibi çok farklı kişilerden " "oluşur. Ancak Tor projesine katılan herkesin ortak bir inancı vardır: " -"İnsanların izlenmeden ve sansüre uğramadan İnternet kullanabilmesi " -"gerekir.**" +"İnsanlar izlenmeden ve sansüre uğramadan İnternet kullanabilmelidir.**" #: https//www.torproject.org/about/history/ #: (content/about/history/contents+en.lrpage.body)

1 0

[translation/tpo-web] https://gitweb.torproject.org/translation.git/commit/?h=tpo-web
by translation＠torproject.org 10 Nov '19

10 Nov '19

commit 7922b4216529a581278dc09401f28ea4cb7ea6bd Author: Translation commit bot <translation(a)torproject.org> Date: Sun Nov 10 03:52:34 2019 +0000 https://gitweb.torproject.org/translation.git/commit/?h=tpo-web --- contents+tr.po | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contents+tr.po b/contents+tr.po index b9bff99c5..ea3ab5f25 100644 --- a/contents+tr.po +++ b/contents+tr.po @@ -425,8 +425,7 @@ msgstr "" "**Tıpkı Tor kullanıcıları gibi, Tor projesini hayata geçirenler de " "geliştiriciler, araştırmacılar ve kurucular gibi çok farklı kişilerden " "oluşur. Ancak Tor projesine katılan herkesin ortak bir inancı vardır: " -"İnsanların izlenmeden ve sansüre uğramadan İnternet kullanabilmesi " -"gerekir.**" +"İnsanlar izlenmeden ve sansüre uğramadan İnternet kullanabilmelidir.**" #: https//www.torproject.org/about/history/ #: (content/about/history/contents+en.lrpage.body)

1 0

[metrics-web/master] Bump version to 1.3.0-dev.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 4ae4da944dbfe8b50085b43c9ef9f764959d3867 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Nov 9 22:46:28 2019 +0100 Bump version to 1.3.0-dev. --- CHANGELOG.md | 3 +++ build.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cccaab..fe74cf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# Changes in version 1.?.? - 2019-1?-?? + + # Changes in version 1.3.0 - 2019-11-09 * Medium changes diff --git a/build.xml b/build.xml index e08a44a..83e69e7 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ <property name="javadoc-title" value="MetricsWeb API Documentation"/> <property name="implementation-title" value="metrics-web" /> - <property name="release.version" value="1.3.0" /> + <property name="release.version" value="1.3.0-dev" /> <property name="metricslibversion" value="2.9.1" /> <property name="exoneratorversion" value="4.2.0" /> <property name="jetty.version" value="-9.2.21.v20170120" />

1 0

[exonerator/master] Bump version to 4.2.0-dev.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit ec51d14df47ee9dae29a9e2cbfbc9a3ba55459f9 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Nov 9 22:45:32 2019 +0100 Bump version to 4.2.0-dev. --- CHANGELOG.md | 3 +++ build.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 632f622..9c0cf51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# Changes in version 4.?.? - 2019-1?-?? + + # Changes in version 4.2.0 - 2019-11-09 * Medium changes diff --git a/build.xml b/build.xml index 38fee16..312f249 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ <property name="javadoc-title" value="ExoneraTor API Documentation"/> <property name="implementation-title" value="ExoneraTor" /> - <property name="release.version" value="4.2.0" /> + <property name="release.version" value="4.2.0-dev" /> <property name="metricslibversion" value="2.9.1" /> <property name="jetty.version" value="-9.2.21.v20170120" /> <property name="warfile"

1 0

[metrics-web/release] Run modules from Java only.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 829863f48fb4e624f849df4c67bc970331014b0d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jan 24 10:08:02 2019 +0100 Run modules from Java only. Implements #29166. --- build.xml | 136 +-------------------- .../web/images => R/rserver}/no-data-available.pdf | Bin .../web/images => R/rserver}/no-data-available.png | Bin .../web/images => R/rserver}/no-data-available.xcf | Bin src/main/R/rserver/rserve-init.R | 6 +- .../torproject/metrics/stats/advbwdist/Main.java | 15 ++- .../metrics/stats/bwhist/Configuration.java | 18 --- .../org/torproject/metrics/stats/bwhist/Main.java | 31 +++-- .../torproject/metrics/stats/clients/Database.java | 10 +- .../torproject/metrics/stats/clients/Detector.java | 7 +- .../org/torproject/metrics/stats/clients/Main.java | 37 +++--- .../metrics/stats/collectdescs/Main.java | 5 +- .../metrics/stats/connbidirect/Main.java | 15 ++- .../org/torproject/metrics/stats/hidserv/Main.java | 14 ++- .../org/torproject/metrics/stats/main/Main.java | 122 ++++++++++++++++++ .../torproject/metrics/stats/onionperf/Main.java | 26 ++-- .../metrics/stats/servers/Configuration.java | 18 --- .../torproject/metrics/stats/servers/Database.java | 10 +- .../org/torproject/metrics/stats/servers/Main.java | 45 +++---- .../metrics/stats/totalcw/Configuration.java | 18 --- .../torproject/metrics/stats/totalcw/Database.java | 10 +- .../org/torproject/metrics/stats/totalcw/Main.java | 29 +++-- .../torproject/metrics/stats/webstats/Main.java | 23 ++-- 23 files changed, 296 insertions(+), 299 deletions(-) diff --git a/build.xml b/build.xml index 42965bf..254a71f 100644 --- a/build.xml +++ b/build.xml @@ -18,7 +18,8 @@ <property name="name" value="metrics-web"/> <property name="project-main-class" - value="org.torproject.TBD" /> + value="org.torproject.metrics.stats.main.Main" /> + <property name="jarincludes" value="logback.xml" /> <property name="additional2sign" value="${warfile}" /> <property name="tardepends" value="war" /> @@ -26,11 +27,6 @@ <property name="specdir" value="${basedir}/generated/spec" /> -  - <property name="metrics-web.deployment.base" - value="/srv/metrics.torproject.org/metrics" /> -  <target name="coverage-check"> <cobertura-check totallinerate="0" totalbranchrate="0" > @@ -301,85 +297,6 @@ <delete file="${specdir}/${specfile}.tmp2" quiet="true" /> </target> -  - <property name="prepare.deployment" - value="/srv/metrics.torproject.org/metrics/work" /> - -  - <property name="modulebase" value="${prepare.deployment}/modules" /> - -  - <target name="run-web-prepare" depends="init" > - <mkdir dir="${prepare.deployment}/modules" /> - <antcall target="collectdescs" /> - <antcall target="connbidirect" /> - <antcall target="onionperf" /> - <antcall target="bwhist" /> - <antcall target="advbwdist" /> - <antcall target="hidserv" /> - <antcall target="clients" /> - <antcall target="servers" /> - <antcall target="webstats" /> - <antcall target="totalcw" /> - <antcall target="make-data-available" /> - </target> - - <target name="collectdescs" > - <property name="module.name" value="collectdescs" /> - <antcall target="run-java" /> - </target> - - <target name="connbidirect" > - <property name="module.name" value="connbidirect" /> - <antcall target="run-java" /> - </target> - - <target name="onionperf" > - <property name="module.name" value="onionperf" /> - <antcall target="run-java" /> - </target> - - <target name="bwhist" > - <property name="module.name" value="bwhist" /> - <antcall target="run-java" /> - </target> - - <target name="advbwdist"> - <property name="module.name" value="advbwdist" /> - <antcall target="run-java" /> - </target> - - <target name="hidserv" > - <property name="module.name" value="hidserv" /> - <antcall target="run-java" /> - </target> - - <target name="clients" > - <property name="module.name" value="clients" /> - <antcall target="run-java" /> - </target> - - <target name="servers" > - <property name="module.name" value="servers" /> - <antcall target="run-java" > - <param name="module.main" - value="org.torproject.metrics.stats.servers.Main" /> - </antcall> - </target> - - <target name="webstats" > - <property name="module.name" value="webstats" /> - <antcall target="run-java" /> - </target> - - <target name="totalcw" > - <property name="module.name" value="totalcw" /> - <antcall target="run-java" > - <param name="module.main" - value="org.torproject.metrics.stats.totalcw.Main" /> - </antcall> - </target> -  - <target name="run-java"> - <echo message="Running java module ${module.name} ... " /> - <available file="${dist}/${jarfile}" property="have.jar"/> - <fail unless="have.jar" message="Please run 'ant jar' first."/> - <condition property="mainclass" - value="${module.main}" - else="org.torproject.metrics.stats.${module.name}.Main" > - <isset property="module.main"/> - </condition> - <property name="workingdir" value="${modulebase}/${module.name}" /> - <mkdir dir="${workingdir}" /> - <java dir="${workingdir}" - fork="true" - maxmemory="4g" - classname="${mainclass}"> - <classpath> - <pathelement location="${dist}/${jarfile}"/> - <pathelement location="${resources}"/> - </classpath> - <jvmarg value="-DLOGBASE=../logs"/> - <jvmarg value="-Duser.language=us" /> - <jvmarg value="-Duser.region=US" /> - </java> - <echo message="Java module ${module.name} finished. " /> - </target> -  diff --git a/src/main/resources/web/images/no-data-available.pdf b/src/main/R/rserver/no-data-available.pdf similarity index 100% rename from src/main/resources/web/images/no-data-available.pdf rename to src/main/R/rserver/no-data-available.pdf diff --git a/src/main/resources/web/images/no-data-available.png b/src/main/R/rserver/no-data-available.png similarity index 100% rename from src/main/resources/web/images/no-data-available.png rename to src/main/R/rserver/no-data-available.png diff --git a/src/main/resources/web/images/no-data-available.xcf b/src/main/R/rserver/no-data-available.xcf similarity index 100% rename from src/main/resources/web/images/no-data-available.xcf rename to src/main/R/rserver/no-data-available.xcf diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R index c412e4c..5a47550 100644 --- a/src/main/R/rserver/rserve-init.R +++ b/src/main/R/rserver/rserve-init.R @@ -333,7 +333,7 @@ copyright_notice <- "The Tor Project - https://metrics.torproject.org/" stats_dir <- "/srv/metrics.torproject.org/metrics/shared/stats/" -rdata_dir <- "/srv/metrics.torproject.org/metrics/shared/RData/" +no_data_available_dir <- "/srv/metrics.torproject.org/metrics/src/main/R/rserver/" # Helper function that copies the appropriate no data object to filename. copy_no_data <- function(filename) { @@ -342,8 +342,8 @@ copy_no_data <- function(filename) { if (".csv" == extension) { write("# No data available for the given parameters.", file=filename) } else { - file.copy(paste(rdata_dir, "no-data-available", extension, sep = ""), - filename) + file.copy(paste(no_data_available_dir, "no-data-available", extension, + sep = ""), filename) } } diff --git a/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java b/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java index 6c4f4ac..3afcb80 100644 --- a/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java +++ b/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java @@ -32,6 +32,9 @@ import java.util.TreeMap; public class Main { + private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "advbwdist"); + /** Executes this data-processing module. */ public static void main(String[] args) throws IOException { @@ -41,7 +44,8 @@ public class Main { DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); for (Descriptor descriptor : descriptorReader.readDescriptors(new File( - "../../shared/in/recent/relay-descriptors/server-descriptors"))) { + org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/relay-descriptors/server-descriptors"))) { if (!(descriptor instanceof ServerDescriptor)) { continue; } @@ -56,9 +60,9 @@ public class Main { /* Parse consensuses, keeping a parse history. */ descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - File historyFile = new File("status/parsed-consensuses"); + File historyFile = new File(baseDir, "status/parsed-consensuses"); descriptorReader.setHistoryFile(historyFile); - File resultsFile = new File("stats/advbwdist-validafter.csv"); + File resultsFile = new File(baseDir, "stats/advbwdist-validafter.csv"); resultsFile.getParentFile().mkdirs(); boolean writeHeader = !resultsFile.exists(); BufferedWriter bw = new BufferedWriter(new FileWriter(resultsFile, @@ -70,7 +74,8 @@ public class Main { "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); for (Descriptor descriptor : descriptorReader.readDescriptors(new File( - "../../shared/in/recent/relay-descriptors/consensuses"))) { + org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/relay-descriptors/consensuses"))) { if (!(descriptor instanceof RelayNetworkStatusConsensus)) { continue; } @@ -165,7 +170,7 @@ public class Main { preAggregatedValues.get(keyWithoutTime).add(value); } } - File aggregateResultsFile = new File("stats/advbwdist.csv"); + File aggregateResultsFile = new File(baseDir, "stats/advbwdist.csv"); aggregateResultsFile.getParentFile().mkdirs(); try (BufferedWriter bw2 = new BufferedWriter( new FileWriter(aggregateResultsFile))) { diff --git a/src/main/java/org/torproject/metrics/stats/bwhist/Configuration.java b/src/main/java/org/torproject/metrics/stats/bwhist/Configuration.java deleted file mode 100644 index 2a0fbc5..0000000 --- a/src/main/java/org/torproject/metrics/stats/bwhist/Configuration.java +++ /dev/null @@ -1,18 +0,0 @@ -/* Copyright 2011--2018 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.metrics.stats.bwhist; - -/** Configuration options parsed from Java properties with reasonable hard-coded - * defaults. */ -public class Configuration { - static String descriptors = System.getProperty("bwhist.descriptors", - "../../shared/in/"); - static String database = System.getProperty("bwhist.database", - "jdbc:postgresql:tordir"); - static String history = System.getProperty("bwhist.history", - "status/read-descriptors"); - static String output = System.getProperty("bwhist.output", - "stats/"); -} - diff --git a/src/main/java/org/torproject/metrics/stats/bwhist/Main.java b/src/main/java/org/torproject/metrics/stats/bwhist/Main.java index 61c1435..30358f2 100644 --- a/src/main/java/org/torproject/metrics/stats/bwhist/Main.java +++ b/src/main/java/org/torproject/metrics/stats/bwhist/Main.java @@ -7,7 +7,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.nio.file.Paths; import java.util.Arrays; /** @@ -18,11 +17,19 @@ public class Main { private static Logger log = LoggerFactory.getLogger(Main.class); - private static String[][] paths = { - {"recent", "relay-descriptors", "consensuses"}, - {"recent", "relay-descriptors", "extra-infos"}, - {"archive", "relay-descriptors", "consensuses"}, - {"archive", "relay-descriptors", "extra-infos"}}; + private static String[] paths = { + "recent/relay-descriptors/consensuses", + "recent/relay-descriptors/extra-infos", + "archive/relay-descriptors/consensuses", + "archive/relay-descriptors/extra-infos" }; + + private static final String jdbcString = String.format( + "jdbc:postgresql://localhost/tordir?user=%s&password=%s", + System.getProperty("metrics.dbuser", "metrics"), + System.getProperty("metrics.dbpass", "password")); + + private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "bwhist"); /** Executes this data-processing module. */ public static void main(String[] args) throws Exception { @@ -31,20 +38,20 @@ public class Main { log.info("Reading descriptors and inserting relevant parts into the " + "database."); - File[] descriptorDirectories = Arrays.stream(paths).map((String[] path) - -> Paths.get(Configuration.descriptors, path).toFile()) - .toArray(File[]::new); - File historyFile = new File(Configuration.history); + File[] descriptorDirectories = Arrays.stream(paths).map((String path) + -> new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + path)).toArray(File[]::new); + File historyFile = new File(baseDir, "status/read-descriptors"); RelayDescriptorDatabaseImporter database = new RelayDescriptorDatabaseImporter(descriptorDirectories, - historyFile, Configuration.database); + historyFile, jdbcString); database.importRelayDescriptors(); log.info("Aggregating database entries."); database.aggregate(); log.info("Querying aggregated statistics from the database."); - new Writer().write(Paths.get(Configuration.output, "bandwidth.csv"), + new Writer().write(new File(baseDir, "stats/bandwidth.csv").toPath(), database.queryBandwidth()); log.info("Closing database connection."); diff --git a/src/main/java/org/torproject/metrics/stats/clients/Database.java b/src/main/java/org/torproject/metrics/stats/clients/Database.java index 7e783dc..f8bca92 100644 --- a/src/main/java/org/torproject/metrics/stats/clients/Database.java +++ b/src/main/java/org/torproject/metrics/stats/clients/Database.java @@ -22,7 +22,10 @@ import java.util.TimeZone; class Database implements AutoCloseable { /** Database connection string. */ - private String jdbcString; + private static final String jdbcString = String.format( + "jdbc:postgresql://localhost/userstats?user=%s&password=%s", + System.getProperty("metrics.dbuser", "metrics"), + System.getProperty("metrics.dbpass", "password")); /** Connection object for all interactions with the database. */ private Connection connection; @@ -33,14 +36,13 @@ class Database implements AutoCloseable { /** Create a new Database instance and prepare for inserting or querying * data. */ - Database(String jdbcString) throws SQLException { - this.jdbcString = jdbcString; + Database() throws SQLException { this.connect(); this.prepareStatements(); } private void connect() throws SQLException { - this.connection = DriverManager.getConnection(this.jdbcString); + this.connection = DriverManager.getConnection(jdbcString); this.connection.setAutoCommit(false); } diff --git a/src/main/java/org/torproject/metrics/stats/clients/Detector.java b/src/main/java/org/torproject/metrics/stats/clients/Detector.java index 1a523c2..cceac5e 100644 --- a/src/main/java/org/torproject/metrics/stats/clients/Detector.java +++ b/src/main/java/org/torproject/metrics/stats/clients/Detector.java @@ -53,7 +53,6 @@ import java.io.FileWriter; import java.io.IOException; import java.io.LineNumberReader; import java.nio.file.Path; -import java.nio.file.Paths; import java.time.LocalDate; import java.time.format.DateTimeParseException; import java.util.ArrayList; @@ -70,10 +69,12 @@ import java.util.stream.Collectors; public class Detector { /** Input file. */ - private static final Path INPUT_PATH = Paths.get("stats", "userstats.csv"); + private static final Path INPUT_PATH = new File(Main.baseDir, + "stats/userstats.csv").toPath(); /** Output file. */ - private static final Path OUTPUT_PATH = Paths.get("stats", "clients.csv"); + private static final Path OUTPUT_PATH = new File(Main.baseDir, + "stats/clients.csv").toPath(); /** Number of largest locations to be included in the detection algorithm. */ private static final int NUM_LARGEST_LOCATIONS = 50; diff --git a/src/main/java/org/torproject/metrics/stats/clients/Main.java b/src/main/java/org/torproject/metrics/stats/clients/Main.java index 0f1087b..2f22f72 100644 --- a/src/main/java/org/torproject/metrics/stats/clients/Main.java +++ b/src/main/java/org/torproject/metrics/stats/clients/Main.java @@ -16,7 +16,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.nio.file.Paths; import java.sql.SQLException; import java.util.Map; import java.util.SortedMap; @@ -26,18 +25,18 @@ public class Main { private static Logger log = LoggerFactory.getLogger(Main.class); - private static final String jdbcString - = System.getProperty("clients.database", "jdbc:postgresql:userstats"); - private static Database database; + static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "clients"); + /** Executes this data-processing module. */ public static void main(String[] args) throws Exception { log.info("Starting clients module."); log.info("Connecting to database."); - database = new Database(jdbcString); + database = new Database(); log.info("Reading relay descriptors and importing relevant parts into the " + "database."); @@ -52,10 +51,10 @@ public class Main { database.commit(); log.info("Querying aggregated statistics from the database."); - new Writer().write(Paths.get("stats", "userstats.csv"), + new Writer().write(new File(baseDir, "stats/userstats.csv").toPath(), database.queryEstimated()); - new Writer().write(Paths.get("stats", "userstats-combined.csv"), - database.queryCombined()); + new Writer().write(new File(baseDir, "stats/userstats-combined.csv") + .toPath(), database.queryCombined()); log.info("Disconnecting from database."); database.close(); @@ -75,13 +74,17 @@ public class Main { private static void parseRelayDescriptors() throws Exception { DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - File historyFile = new File("status/relay-descriptors"); + File historyFile = new File(baseDir, "status/relay-descriptors"); descriptorReader.setHistoryFile(historyFile); for (Descriptor descriptor : descriptorReader.readDescriptors( - new File("../../shared/in/recent/relay-descriptors/consensuses"), - new File("../../shared/in/recent/relay-descriptors/extra-infos"), - new File("../../shared/in/archive/relay-descriptors/consensuses"), - new File("../../shared/in/archive/relay-descriptors/extra-infos"))) { + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/relay-descriptors/consensuses"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/relay-descriptors/extra-infos"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "archive/relay-descriptors/consensuses"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "archive/relay-descriptors/extra-infos"))) { if (descriptor instanceof ExtraInfoDescriptor) { parseRelayExtraInfoDescriptor((ExtraInfoDescriptor) descriptor); } else if (descriptor instanceof RelayNetworkStatusConsensus) { @@ -209,11 +212,13 @@ public class Main { private static void parseBridgeDescriptors() throws Exception { DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - File historyFile = new File("status/bridge-descriptors"); + File historyFile = new File(baseDir, "status/bridge-descriptors"); descriptorReader.setHistoryFile(historyFile); for (Descriptor descriptor : descriptorReader.readDescriptors( - new File("../../shared/in/recent/bridge-descriptors"), - new File("../../shared/in/archive/bridge-descriptors"))) { + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/bridge-descriptors"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "archive/bridge-descriptors"))) { if (descriptor instanceof ExtraInfoDescriptor) { parseBridgeExtraInfoDescriptor( (ExtraInfoDescriptor) descriptor); diff --git a/src/main/java/org/torproject/metrics/stats/collectdescs/Main.java b/src/main/java/org/torproject/metrics/stats/collectdescs/Main.java index a9620f5..1d0d840 100644 --- a/src/main/java/org/torproject/metrics/stats/collectdescs/Main.java +++ b/src/main/java/org/torproject/metrics/stats/collectdescs/Main.java @@ -10,6 +10,9 @@ import java.io.File; public class Main { + private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "collectdescs"); + /** Executes this data-processing module. */ public static void main(String[] args) { /* Fetch recent descriptors from CollecTor. */ @@ -27,7 +30,7 @@ public class Main { "/recent/relay-descriptors/votes/", "/recent/torperf/", "/recent/webstats/" - }, 0L, new File("../../shared/in"), true); + }, 0L, org.torproject.metrics.stats.main.Main.descriptorsDir, true); } } diff --git a/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java b/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java index 468ffba..c7fefee 100644 --- a/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java +++ b/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java @@ -129,13 +129,18 @@ public class Main { static final long ONE_DAY_IN_MILLIS = 86400000L; + private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "connbidirect"); + /** Executes this data-processing module. */ public static void main(String[] args) throws IOException { - File parseHistoryFile = new File("stats/parse-history"); - File aggregateStatsFile = new File("stats/connbidirect2.csv"); + File parseHistoryFile = new File(baseDir, "stats/parse-history"); + File aggregateStatsFile = new File(baseDir, "stats/connbidirect2.csv"); File[] descriptorsDirectories = new File[] { - new File("../../shared/in/archive/relay-descriptors/extra-infos"), - new File("../../shared/in/recent/relay-descriptors/extra-infos")}; + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "archive/relay-descriptors/extra-infos"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/relay-descriptors/extra-infos")}; SortedMap<String, Long> parseHistory = parseParseHistory( readStringFromFile(parseHistoryFile)); if (parseHistory == null) { @@ -160,7 +165,7 @@ public class Main { + "leave out those descriptors in future runs."); return; } - File rawStatsFile = new File("stats/raw-stats"); + File rawStatsFile = new File(baseDir, "stats/raw-stats"); SortedSet<RawStat> rawStats = parseRawStats( readStringFromFile(rawStatsFile)); if (rawStats == null) { diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/Main.java b/src/main/java/org/torproject/metrics/stats/hidserv/Main.java index b6a4e43..5076353 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/Main.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/Main.java @@ -16,6 +16,9 @@ public class Main { private static Logger log = LoggerFactory.getLogger(Main.class); + private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "hidserv"); + /** Parses new descriptors, extrapolate contained statistics using * computed network fractions, aggregate results, and writes results to * disk. */ @@ -23,9 +26,11 @@ public class Main { /* Initialize directories and file paths. */ File[] inDirectories = new File[] { - new File("../../shared/in/recent/relay-descriptors/consensuses"), - new File("../../shared/in/recent/relay-descriptors/extra-infos") }; - File statusDirectory = new File("status"); + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/relay-descriptors/consensuses"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/relay-descriptors/extra-infos") }; + File statusDirectory = new File(baseDir, "status"); /* Initialize parser and read parse history to avoid parsing * descriptor files that haven't changed since the last execution. */ @@ -71,7 +76,8 @@ public class Main { * other statistics. Write the result to a .csv file that can be * processed by other tools. */ log.info("Aggregating statistics..."); - File hidservStatsExtrapolatedCsvFile = new File("stats/hidserv.csv"); + File hidservStatsExtrapolatedCsvFile = new File(baseDir, + "stats/hidserv.csv"); Aggregator aggregator = new Aggregator(statusDirectory, extrapolatedHidServStatsStore, hidservStatsExtrapolatedCsvFile); aggregator.aggregateHidServStats(); diff --git a/src/main/java/org/torproject/metrics/stats/main/Main.java b/src/main/java/org/torproject/metrics/stats/main/Main.java new file mode 100644 index 0000000..3dc2775 --- /dev/null +++ b/src/main/java/org/torproject/metrics/stats/main/Main.java @@ -0,0 +1,122 @@ +package org.torproject.metrics.stats.main; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; + +public class Main { + + private static final Logger log = LoggerFactory.getLogger(Main.class); + + private static final String baseDir = System.getProperty("metrics.basedir", + "/srv/metrics.torproject.org/metrics"); + + public static final File modulesDir = new File(baseDir, "work/modules"); + + public static final File descriptorsDir = new File(baseDir, "work/shared/in"); + + private static final File statsDir = new File(baseDir, "shared/stats"); + + /** Start the metrics update run. */ + public static void main(String[] args) { + + log.info("Starting metrics update run."); + + File[] outputDirs = new File[] { modulesDir, statsDir }; + for (File outputDir : outputDirs) { + if (outputDir.exists()) { + continue; + } + if (outputDir.mkdirs()) { + log.info("Successfully created module base directory {} and any " + + "nonexistent parent directories.", + outputDir.getAbsolutePath()); + } else { + log.error("Unable to create module base directory {} and any " + + "nonexistent parent directories. Exiting.", + outputDir.getAbsolutePath()); + return; + } + } + + Class<?>[] modules = new Class<?>[] { + org.torproject.metrics.stats.collectdescs.Main.class, + org.torproject.metrics.stats.connbidirect.Main.class, + org.torproject.metrics.stats.onionperf.Main.class, + org.torproject.metrics.stats.bwhist.Main.class, + org.torproject.metrics.stats.advbwdist.Main.class, + org.torproject.metrics.stats.hidserv.Main.class, + org.torproject.metrics.stats.clients.Main.class, + org.torproject.metrics.stats.servers.Main.class, + org.torproject.metrics.stats.webstats.Main.class, + org.torproject.metrics.stats.totalcw.Main.class + }; + + for (Class<?> module : modules) { + try { + log.info("Starting {} module.", module.getName()); + module.getDeclaredMethod("main", String[].class) + .invoke(null, (Object) args); + log.info("Completed {} module.", module.getName()); + } catch (NoSuchMethodException | IllegalAccessException + | InvocationTargetException e) { + log.warn("Caught an exception when invoking the main method of the {} " + + "module. Moving on to the next module, if available.", e); + } + } + + log.info("Making module data available."); + File[] moduleStatsDirs = new File[] { + new File(modulesDir, "connbidirect/stats"), + new File(modulesDir, "onionperf/stats"), + new File(modulesDir, "bwhist/stats"), + new File(modulesDir, "advbwdist/stats/advbwdist.csv"), + new File(modulesDir, "hidserv/stats"), + new File(modulesDir, "clients/stats/clients.csv"), + new File(modulesDir, "clients/stats/userstats-combined.csv"), + new File(modulesDir, "servers/stats"), + new File(modulesDir, "webstats/stats"), + new File(modulesDir, "totalcw/stats") + }; + List<String> copiedFiles = new ArrayList<>(); + for (File moduleStatsDir : moduleStatsDirs) { + if (!moduleStatsDir.exists()) { + log.warn("Skipping nonexistent module stats dir {}.", moduleStatsDir); + continue; + } + File[] moduleStatsFiles = moduleStatsDir.isDirectory() + ? moduleStatsDir.listFiles() : new File[] { moduleStatsDir }; + if (null == moduleStatsFiles) { + log.warn("Skipping nonexistent module stats dir {}.", moduleStatsDir); + continue; + } + for (File statsFile : moduleStatsFiles) { + if (!statsFile.isFile() || !statsFile.getName().endsWith(".csv")) { + continue; + } + try { + Files.copy(statsFile.toPath(), + new File(statsDir, statsFile.getName()).toPath(), + StandardCopyOption.REPLACE_EXISTING); + copiedFiles.add(statsFile.getName()); + } catch (IOException e) { + log.warn("Unable to copy module stats file {} to stats output " + + "directory {}. Skipping.", statsFile, statsDir, e); + } + } + } + if (!copiedFiles.isEmpty()) { + log.info("Successfully copied {} files to stats output directory: {}", + copiedFiles.size(), copiedFiles); + } + + log.info("Completed metrics update run."); + } +} diff --git a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java index 02b70af..a970434 100644 --- a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java +++ b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java @@ -13,7 +13,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; @@ -37,23 +36,30 @@ public class Main { /** Logger for this class. */ private static Logger log = LoggerFactory.getLogger(Main.class); + private static final String jdbcString = String.format( + "jdbc:postgresql://localhost/onionperf?user=%s&password=%s", + System.getProperty("metrics.dbuser", "metrics"), + System.getProperty("metrics.dbpass", "password")); + + private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "onionperf"); + /** Executes this data-processing module. */ public static void main(String[] args) throws Exception { log.info("Starting onionperf module."); - String dbUrlString = "jdbc:postgresql:onionperf"; - Connection connection = connectToDatabase(dbUrlString); + Connection connection = connectToDatabase(); importOnionPerfFiles(connection); - writeStatistics(Paths.get("stats", "torperf-1.1.csv"), + writeStatistics(new File(baseDir, "stats/torperf-1.1.csv").toPath(), queryOnionPerf(connection)); - writeStatistics(Paths.get("stats", "buildtimes.csv"), + writeStatistics(new File(baseDir, "stats/buildtimes.csv").toPath(), queryBuildTimes(connection)); - writeStatistics(Paths.get("stats", "latencies.csv"), + writeStatistics(new File(baseDir, "stats/latencies.csv").toPath(), queryLatencies(connection)); disconnectFromDatabase(connection); log.info("Terminated onionperf module."); } - private static Connection connectToDatabase(String jdbcString) + private static Connection connectToDatabase() throws SQLException { log.info("Connecting to database."); Connection connection = DriverManager.getConnection(jdbcString); @@ -91,8 +97,10 @@ public class Main { Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); DescriptorReader dr = DescriptorSourceFactory.createDescriptorReader(); for (Descriptor d : dr.readDescriptors( - new File("../../shared/in/archive/torperf"), - new File("../../shared/in/recent/torperf"))) { + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "archive/torperf"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/torperf"))) { if (!(d instanceof TorperfResult)) { continue; } diff --git a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java b/src/main/java/org/torproject/metrics/stats/servers/Configuration.java deleted file mode 100644 index 1b7c217..0000000 --- a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java +++ /dev/null @@ -1,18 +0,0 @@ -/* Copyright 2017--2018 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.metrics.stats.servers; - -/** Configuration options parsed from Java properties with reasonable hard-coded - * defaults. */ -class Configuration { - static String descriptors = System.getProperty("servers.descriptors", - "../../shared/in/"); - static String database = System.getProperty("servers.database", - "jdbc:postgresql:ipv6servers"); - static String history = System.getProperty("servers.history", - "status/read-descriptors"); - static String output = System.getProperty("servers.output", - "stats/"); -} - diff --git a/src/main/java/org/torproject/metrics/stats/servers/Database.java b/src/main/java/org/torproject/metrics/stats/servers/Database.java index 9c9bda3..f7f1f0f 100644 --- a/src/main/java/org/torproject/metrics/stats/servers/Database.java +++ b/src/main/java/org/torproject/metrics/stats/servers/Database.java @@ -29,7 +29,10 @@ import java.util.TimeZone; class Database implements AutoCloseable { /** Database connection string. */ - private String jdbcString; + private static final String jdbcString = String.format( + "jdbc:postgresql://localhost/ipv6servers?user=%s&password=%s", + System.getProperty("metrics.dbuser", "metrics"), + System.getProperty("metrics.dbpass", "password")); /** Connection object for all interactions with the database. */ private Connection connection; @@ -87,15 +90,14 @@ class Database implements AutoCloseable { /** Create a new Database instance and prepare for inserting or querying * data. */ - Database(String jdbcString) throws SQLException { - this.jdbcString = jdbcString; + Database() throws SQLException { this.connect(); this.prepareStatements(); this.initializeCaches(); } private void connect() throws SQLException { - this.connection = DriverManager.getConnection(this.jdbcString); + this.connection = DriverManager.getConnection(jdbcString); this.connection.setAutoCommit(false); } diff --git a/src/main/java/org/torproject/metrics/stats/servers/Main.java b/src/main/java/org/torproject/metrics/stats/servers/Main.java index 54f44d0..30010a2 100644 --- a/src/main/java/org/torproject/metrics/stats/servers/Main.java +++ b/src/main/java/org/torproject/metrics/stats/servers/Main.java @@ -14,7 +14,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.nio.file.Paths; import java.sql.SQLException; import java.util.Arrays; @@ -25,15 +24,18 @@ public class Main { private static Logger log = LoggerFactory.getLogger(Main.class); - private static String[][] paths = { - {"recent", "relay-descriptors", "consensuses"}, - {"recent", "relay-descriptors", "server-descriptors"}, - {"recent", "bridge-descriptors", "statuses"}, - {"recent", "bridge-descriptors", "server-descriptors"}, - {"archive", "relay-descriptors", "consensuses"}, - {"archive", "relay-descriptors", "server-descriptors"}, - {"archive", "bridge-descriptors", "statuses"}, - {"archive", "bridge-descriptors", "server-descriptors"}}; + private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "servers"); + + private static String[] paths = { + "recent/relay-descriptors/consensuses", + "recent/relay-descriptors/server-descriptors", + "recent/bridge-descriptors/statuses", + "recent/bridge-descriptors/server-descriptors", + "archive/relay-descriptors/consensuses", + "archive/relay-descriptors/server-descriptors", + "archive/bridge-descriptors/statuses", + "archive/bridge-descriptors/server-descriptors" }; /** Run the module. */ public static void main(String[] args) throws Exception { @@ -43,15 +45,15 @@ public class Main { log.info("Reading descriptors and inserting relevant parts into the " + "database."); DescriptorReader reader = DescriptorSourceFactory.createDescriptorReader(); - File historyFile = new File(Configuration.history); + File historyFile = new File(baseDir, "status/read-descriptors"); reader.setHistoryFile(historyFile); Parser parser = new Parser(); - try (Database database = new Database(Configuration.database)) { + try (Database database = new Database()) { try { for (Descriptor descriptor : reader.readDescriptors( - Arrays.stream(paths).map((String[] path) - -> Paths.get(Configuration.descriptors, path).toFile()) - .toArray(File[]::new))) { + Arrays.stream(paths).map((String path) -> new File( + org.torproject.metrics.stats.main.Main.descriptorsDir, path)) + .toArray(File[]::new))) { if (descriptor instanceof ServerDescriptor) { database.insertServerDescriptor(parser.parseServerDescriptor( (ServerDescriptor) descriptor)); @@ -86,17 +88,18 @@ public class Main { reader.saveHistoryFile(historyFile); log.info("Querying aggregated statistics from the database."); - new Writer().write(Paths.get(Configuration.output, "ipv6servers.csv"), + File outputDir = new File(baseDir, "stats"); + new Writer().write(new File(outputDir, "ipv6servers.csv").toPath(), database.queryServersIpv6()); - new Writer().write(Paths.get(Configuration.output, "advbw.csv"), + new Writer().write(new File(outputDir, "advbw.csv").toPath(), database.queryAdvbw()); - new Writer().write(Paths.get(Configuration.output, "networksize.csv"), + new Writer().write(new File(outputDir, "networksize.csv").toPath(), database.queryNetworksize()); - new Writer().write(Paths.get(Configuration.output, "relayflags.csv"), + new Writer().write(new File(outputDir, "relayflags.csv").toPath(), database.queryRelayflags()); - new Writer().write(Paths.get(Configuration.output, "versions.csv"), + new Writer().write(new File(outputDir, "versions.csv").toPath(), database.queryVersions()); - new Writer().write(Paths.get(Configuration.output, "platforms.csv"), + new Writer().write(new File(outputDir, "platforms.csv").toPath(), database.queryPlatforms()); log.info("Terminating servers module."); diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Configuration.java b/src/main/java/org/torproject/metrics/stats/totalcw/Configuration.java deleted file mode 100644 index a424f84..0000000 --- a/src/main/java/org/torproject/metrics/stats/totalcw/Configuration.java +++ /dev/null @@ -1,18 +0,0 @@ -/* Copyright 2018 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.metrics.stats.totalcw; - -/** Configuration options parsed from Java properties with reasonable hard-coded - * defaults. */ -class Configuration { - static String descriptors = System.getProperty("totalcw.descriptors", - "../../shared/in/"); - static String database = System.getProperty("totalcw.database", - "jdbc:postgresql:totalcw"); - static String history = System.getProperty("totalcw.history", - "status/read-descriptors"); - static String output = System.getProperty("totalcw.output", - "stats/totalcw.csv"); -} - diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java index e842bc6..eeb24a0 100644 --- a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java +++ b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java @@ -25,7 +25,10 @@ import java.util.TimeZone; class Database implements AutoCloseable { /** Database connection string. */ - private String jdbcString; + private static final String jdbcString = String.format( + "jdbc:postgresql://localhost/totalcw?user=%s&password=%s", + System.getProperty("metrics.dbuser", "metrics"), + System.getProperty("metrics.dbpass", "password")); /** Connection object for all interactions with the database. */ private Connection connection; @@ -46,14 +49,13 @@ class Database implements AutoCloseable { /** Create a new Database instance and prepare for inserting or querying * data. */ - Database(String jdbcString) throws SQLException { - this.jdbcString = jdbcString; + Database() throws SQLException { this.connect(); this.prepareStatements(); } private void connect() throws SQLException { - this.connection = DriverManager.getConnection(this.jdbcString); + this.connection = DriverManager.getConnection(jdbcString); this.connection.setAutoCommit(false); } diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Main.java b/src/main/java/org/torproject/metrics/stats/totalcw/Main.java index e5ba8ab..54296be 100644 --- a/src/main/java/org/torproject/metrics/stats/totalcw/Main.java +++ b/src/main/java/org/torproject/metrics/stats/totalcw/Main.java @@ -13,7 +13,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.nio.file.Paths; import java.sql.SQLException; import java.util.Arrays; @@ -24,11 +23,14 @@ public class Main { private static Logger log = LoggerFactory.getLogger(Main.class); - private static String[][] paths = { - {"recent", "relay-descriptors", "consensuses"}, - {"archive", "relay-descriptors", "consensuses"}, - {"recent", "relay-descriptors", "votes"}, - {"archive", "relay-descriptors", "votes"}}; + private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "totalcw"); + + private static String[] paths = { + "recent/relay-descriptors/consensuses", + "archive/relay-descriptors/consensuses", + "recent/relay-descriptors/votes", + "archive/relay-descriptors/votes" }; /** Run the module. */ public static void main(String[] args) throws Exception { @@ -38,15 +40,15 @@ public class Main { log.info("Reading consensuses and votes and inserting relevant parts into " + "the database."); DescriptorReader reader = DescriptorSourceFactory.createDescriptorReader(); - File historyFile = new File(Configuration.history); + File historyFile = new File(baseDir, "status/read-descriptors"); reader.setHistoryFile(historyFile); Parser parser = new Parser(); - try (Database database = new Database(Configuration.database)) { + try (Database database = new Database()) { try { for (Descriptor descriptor : reader.readDescriptors( - Arrays.stream(paths).map((String[] path) - -> Paths.get(Configuration.descriptors, path).toFile()) - .toArray(File[]::new))) { + Arrays.stream(paths).map((String path) -> new File( + org.torproject.metrics.stats.main.Main.descriptorsDir, path)) + .toArray(File[]::new))) { if (descriptor instanceof RelayNetworkStatusConsensus) { database.insertConsensus(parser.parseRelayNetworkStatusConsensus( (RelayNetworkStatusConsensus) descriptor)); @@ -71,9 +73,10 @@ public class Main { log.info("Querying aggregated statistics from the database."); Iterable<OutputLine> output = database.queryTotalcw(); - log.info("Writing aggregated statistics to {}.", Configuration.output); + File outputFile = new File(baseDir, "stats/totalcw.csv"); + log.info("Writing aggregated statistics to {}.", outputFile); if (null != output) { - new Writer().write(Paths.get(Configuration.output), output); + new Writer().write(outputFile.toPath(), output); } log.info("Terminating totalcw module."); diff --git a/src/main/java/org/torproject/metrics/stats/webstats/Main.java b/src/main/java/org/torproject/metrics/stats/webstats/Main.java index fb0a903..87b643a 100644 --- a/src/main/java/org/torproject/metrics/stats/webstats/Main.java +++ b/src/main/java/org/torproject/metrics/stats/webstats/Main.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.sql.Connection; import java.sql.Date; import java.sql.DriverManager; @@ -48,6 +47,11 @@ public class Main { /** Logger for this class. */ private static Logger log = LoggerFactory.getLogger(Main.class); + private static final String jdbcString = String.format( + "jdbc:postgresql://localhost/webstats?user=%s&password=%s", + System.getProperty("metrics.dbuser", "metrics"), + System.getProperty("metrics.dbpass", "password")); + private static final String LOG_DATE = "log_date"; private static final String REQUEST_TYPE = "request_type"; @@ -66,22 +70,27 @@ public class Main { + PLATFORM + "," + CHANNEL + "," + LOCALE + "," + INCREMENTAL + "," + COUNT; + private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "webstats"); + /** Executes this data-processing module. */ public static void main(String[] args) throws Exception { log.info("Starting webstats module."); - String dbUrlString = "jdbc:postgresql:webstats"; - Connection connection = connectToDatabase(dbUrlString); + Connection connection = connectToDatabase(); SortedSet<String> skipFiles = queryImportedFileNames(connection); importLogFiles(connection, skipFiles, - new File("../../shared/in/recent/webstats"), - new File("../../shared/in/archive/webstats")); + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/webstats"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "archive/webstats")); SortedSet<String> statistics = queryWebstats(connection); - writeStatistics(Paths.get("stats", "webstats.csv"), statistics); + writeStatistics(new File(baseDir, "stats/webstats.csv").toPath(), + statistics); disconnectFromDatabase(connection); log.info("Terminated webstats module."); } - private static Connection connectToDatabase(String jdbcString) + private static Connection connectToDatabase() throws SQLException { log.info("Connecting to database."); Connection connection = DriverManager.getConnection(jdbcString);

1 0

[metrics-web/release] Add three new @type annotations.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit e7da8f637cbdd2d851d06657f7959befbb68dcf1 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jan 24 11:27:29 2019 +0100 Add three new @type annotations. Resolves #28615. --- src/main/resources/web/jsps/collector.jsp | 58 +++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/main/resources/web/jsps/collector.jsp b/src/main/resources/web/jsps/collector.jsp index 3e98785..1adc9b3 100644 --- a/src/main/resources/web/jsps/collector.jsp +++ b/src/main/resources/web/jsps/collector.jsp @@ -92,6 +92,11 @@ <td><a href="/collector/archive/relay-descriptors/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr> + <td><a href="#type-detached-signature-3">Detached Signatures</a></td> + <td><code>@type detached-signature-3 1.0</code></td> + <td></td> +</tr> +<tr> <td><a href="#type-network-status-microdesc-consensus-3">Microdescriptor Consensuses</a></td> <td><code>@type network-status-microdesc-consensus-3 1.0</code></td> <td><a href="/collector/recent/relay-descriptors/microdescs/consensus-microdesc/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> @@ -104,6 +109,11 @@ <a href="/collector/archive/relay-descriptors/microdescs/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr> + <td><a href="#type-network-status-entry-3">Network Status Entries</a></td> + <td><code>@type network-status-entry-3 1.0</code></td> + <td></td> +</tr> +<tr> <td><a href="#type-network-status-2">Version 2 Network Statuses</a></td> <td><code>@type network-status-2 1.0</code></td> <td><a href="/collector/archive/relay-descriptors/statuses/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> @@ -177,6 +187,14 @@ <td><a href="/collector/recent/webstats/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/webstats/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> +<tr class="tableHeadline"> + <td colspan="3"><b><a href="#bandwidth-files">Bandwidth Files</a></b></td> +</tr> +<tr> + <td><a href="#type-bandwidth-file">Bandwidth Files</a></td> + <td><code>@type bandwidth-file 1.2</code></td> + <td></td> +</tr> </tbody> </table> @@ -298,6 +316,17 @@ These key certificates change once every few months, so they are only available in a single descriptor archive tarball. </p> +<h3 id="type-detached-signature-3" class="hover">Detached Signatures +<small><code>@type detached-signature-3 1.0</code></small> +<a href="#type-detached-signature-3" class="anchor">#</a> +</h3> + +<p> +Detached signature as per section 3.10 of the dir-spec, and downloadable for +DistSeconds every consensus freshness period (usually five minutes each hour) +via the <code>/tor/status-vote/next/consensus-signatures</code> resource. +</p> + <h3 id="type-network-status-microdesc-consensus-3" class="hover">Microdescriptor Consensuses <small><code>@type network-status-microdesc-consensus-3 1.0</code></small> <a href="/collector/recent/relay-descriptors/microdescs/" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> @@ -335,6 +364,17 @@ contain all descriptors collected in an hour concatenated into a single file. </p> +<h3 id="type-network-status-entry-3" class="hover">Network Status Entries +<small><code>@type network-status-entry-3 1.0</code></small> +<a href="#type-network-status-entry-3" class="anchor">#</a> +</h3> + +<p> +Individual router status entry from an unflavored v3 network status document. +These are available from Tor's control port <code>GETINFO ns/*</code> commands +and NS events. +</p> + <h3 id="type-network-status-2" class="hover">Version 2 Network Statuses <small><code>@type network-status-2 1.0</code></small> <a href="/collector/archive/relay-descriptors/statuses/" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a> @@ -732,6 +772,24 @@ detail on a separate <a href="web-server-logs.html">page</a>. </p> +<br> +<h2 id="bandwidth-files" class="hover">Bandwidth Files +<a href="#bandwidth-files" class="anchor">#</a> +</h2> + +<h3 id="type-bandwidth-file" class="hover">Bandwidth Files +<small><code>@type bandwidth-file 1.2</code></small> +<a href="#type-bandwidth-file" class="anchor">#</a> +</h3> + +<p> +Bandwidth authority metrics as defined in the <a href="https://gitweb.torproject.org/torspec.git/tree/bandwidth-file-spec.txt">bandwidth-file-spec</a>. +These are available from a DirPort's +<code>/tor/status-vote/next/bandwidth</code> url and CollecTor (both are +<a href="https://trac.torproject.org/projects/tor/ticket/21377">pending</a>). +</p> + + </div> <br>

1 0

[metrics-web/release] Make bandwidth.csv independent of import order.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 1fd062c11e45ca5c4cb166f6db8a27eabaf60e48 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Mar 8 15:08:08 2019 +0100 Make bandwidth.csv independent of import order. With this patch we're not overwriting bandwidth history parts with whichever history comes last, but we're computing the maximum value for each 15-minute interval of all imported bandwidth histories. This makes bandwidth.csv independent of descriptor import order. --- src/main/sql/bwhist/tordir.sql | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/main/sql/bwhist/tordir.sql b/src/main/sql/bwhist/tordir.sql index dfe7b5d..047f18e 100644 --- a/src/main/sql/bwhist/tordir.sql +++ b/src/main/sql/bwhist/tordir.sql @@ -122,6 +122,24 @@ CREATE TABLE updates ( date DATE ); +-- Return an array as the result of merging two arrays: if an array index is +-- only contained in one array, that array element is included in the result; +-- if an array index is contained in both arrays, the greater of the two +-- elements is included. +CREATE OR REPLACE FUNCTION array_merge(first BIGINT[], second BIGINT[]) +RETURNS BIGINT[] AS $$ +DECLARE + merged BIGINT[]; +BEGIN + FOR i IN LEAST(array_lower(first, 1), array_lower(second, 1)).. + GREATEST(array_upper(first, 1), array_upper(second, 1)) LOOP + merged[i] := GREATEST(first[i], second[i]); + END LOOP; +RETURN merged; +END; +$$ LANGUAGE plpgsql +STABLE RETURNS NULL ON NULL INPUT; + CREATE OR REPLACE FUNCTION array_sum (BIGINT[]) RETURNS BIGINT AS $$ SELECT SUM($1[i])::bigint FROM generate_series(array_lower($1, 1), array_upper($1, 1)) index(i); @@ -143,25 +161,10 @@ CREATE OR REPLACE FUNCTION insert_bwhist( ELSE BEGIN UPDATE bwhist - SET read[array_lower(insert_read, 1): - array_upper(insert_read, 1)] = insert_read, - written[array_lower(insert_written, 1): - array_upper(insert_written, 1)] = insert_written, - dirread[array_lower(insert_dirread, 1): - array_upper(insert_dirread, 1)] = insert_dirread, - dirwritten[array_lower(insert_dirwritten, 1): - array_upper(insert_dirwritten, 1)] = insert_dirwritten - WHERE fingerprint = insert_fingerprint AND date = insert_date; - -- Updating twice is an ugly workaround for PostgreSQL bug 5840 - UPDATE bwhist - SET read[array_lower(insert_read, 1): - array_upper(insert_read, 1)] = insert_read, - written[array_lower(insert_written, 1): - array_upper(insert_written, 1)] = insert_written, - dirread[array_lower(insert_dirread, 1): - array_upper(insert_dirread, 1)] = insert_dirread, - dirwritten[array_lower(insert_dirwritten, 1): - array_upper(insert_dirwritten, 1)] = insert_dirwritten + SET read = array_merge(read, insert_read), + written = array_merge(written, insert_written), + dirread = array_merge(dirread, insert_dirread), + dirwritten = array_merge(dirwritten, insert_dirwritten) WHERE fingerprint = insert_fingerprint AND date = insert_date; END; END IF;

1 0

[metrics-web/release] Make very thin ribbons more visible.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 24aa37f07397bd1ba5022842eb5b2ce10d6ce6dd Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Mar 6 11:13:48 2019 +0100 Make very thin ribbons more visible. Fixes #29655. --- src/main/R/rserver/rserve-init.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R index 57e14f5..c412e4c 100644 --- a/src/main/R/rserver/rserve-init.R +++ b/src/main/R/rserver/rserve-init.R @@ -1029,7 +1029,7 @@ plot_userstats_bridge_combined <- function(start_p, end_p, country_p, path_p) { title <- paste("Bridge users by transport from ", countryname(country_p), sep = "") ggplot(u, aes(x = as.Date(date), ymin = low, ymax = high, - fill = transport)) + + fill = transport, colour = transport)) + geom_ribbon(alpha = 0.5, size = 0.5) + scale_x_date(name = "", breaks = custom_breaks, labels = custom_labels, minor_breaks = custom_minor_breaks) +

1 0

[metrics-web/release] Add levels = NULL to col_factor().
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit a9fac06cc120a13dd35ba77983dc2d54aacc75ed Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jan 11 09:30:53 2019 +0100 Add levels = NULL to col_factor(). This is not required for readr_1.3.0 which runs locally here but required for readr_1.1.1 which runs on the server. --- src/main/R/rserver/graphs.R | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R index 205afbe..18a9d3e 100644 --- a/src/main/R/rserver/graphs.R +++ b/src/main/R/rserver/graphs.R @@ -682,7 +682,7 @@ prepare_connbidirect <- function(start_p = NULL, end_p = NULL) { read_csv(file = paste(stats_dir, "connbidirect2.csv", sep = ""), col_types = cols( date = col_date(format = ""), - direction = col_factor(), + direction = col_factor(levels = NULL), quantile = col_double(), fraction = col_double())) %>% filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% @@ -1109,7 +1109,7 @@ prepare_hidserv_dir_onions_seen <- function(start_p = NULL, end_p = NULL) { read_csv(file = paste(stats_dir, "hidserv.csv", sep = ""), col_types = cols( date = col_date(format = ""), - type = col_factor(), + type = col_factor(levels = NULL), wmean = col_skip(), wmedian = col_skip(), wiqm = col_double(), @@ -1137,7 +1137,7 @@ prepare_hidserv_rend_relayed_cells <- function(start_p = NULL, end_p = NULL) { read_csv(file = paste(stats_dir, "hidserv.csv", sep = ""), col_types = cols( date = col_date(format = ""), - type = col_factor(), + type = col_factor(levels = NULL), wmean = col_skip(), wmedian = col_skip(), wiqm = col_double(), @@ -1167,7 +1167,7 @@ prepare_webstats_tb <- function(start_p = NULL, end_p = NULL) { read_csv(file = paste(stats_dir, "webstats.csv", sep = ""), col_types = cols( log_date = col_date(format = ""), - request_type = col_factor(), + request_type = col_factor(levels = NULL), platform = col_skip(), channel = col_skip(), locale = col_skip(), @@ -1210,8 +1210,8 @@ prepare_webstats_tb_platform <- function(start_p = NULL, end_p = NULL) { read_csv(file = paste(stats_dir, "webstats.csv", sep = ""), col_types = cols( log_date = col_date(format = ""), - request_type = col_factor(), - platform = col_factor(), + request_type = col_factor(levels = NULL), + platform = col_factor(levels = NULL), channel = col_skip(), locale = col_skip(), incremental = col_skip(), @@ -1253,10 +1253,10 @@ prepare_webstats_tb_locale <- function(start_p = NULL, end_p = NULL) { read_csv(file = paste(stats_dir, "webstats.csv", sep = ""), col_types = cols( log_date = col_date(format = ""), - request_type = col_factor(), + request_type = col_factor(levels = NULL), platform = col_skip(), channel = col_skip(), - locale = col_factor(), + locale = col_factor(levels = NULL), incremental = col_skip(), count = col_double())) %>% filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>% @@ -1308,7 +1308,7 @@ prepare_webstats_tm <- function(start_p = NULL, end_p = NULL) { read_csv(file = paste(stats_dir, "webstats.csv", sep = ""), col_types = cols( log_date = col_date(format = ""), - request_type = col_factor(), + request_type = col_factor(levels = NULL), platform = col_skip(), channel = col_skip(), locale = col_skip(), @@ -1348,7 +1348,7 @@ prepare_relays_ipv6 <- function(start_p = NULL, end_p = NULL) { read_csv(file = paste(stats_dir, "ipv6servers.csv", sep = ""), col_types = cols( valid_after_date = col_date(format = ""), - server = col_factor(), + server = col_factor(levels = NULL), guard_relay = col_skip(), exit_relay = col_skip(), announced_ipv6 = col_logical(), @@ -1395,7 +1395,7 @@ prepare_bridges_ipv6 <- function(start_p = NULL, end_p = NULL) { read_csv(file = paste(stats_dir, "ipv6servers.csv", sep = ""), col_types = cols( valid_after_date = col_date(format = ""), - server = col_factor(), + server = col_factor(levels = NULL), guard_relay = col_skip(), exit_relay = col_skip(), announced_ipv6 = col_logical(), @@ -1436,7 +1436,7 @@ prepare_advbw_ipv6 <- function(start_p = NULL, end_p = NULL) { read_csv(file = paste(stats_dir, "ipv6servers.csv", sep = ""), col_types = cols( valid_after_date = col_date(format = ""), - server = col_factor(), + server = col_factor(levels = NULL), guard_relay = col_logical(), exit_relay = col_logical(), announced_ipv6 = col_logical(),

1 0