September 2016 - tor-commits - lists.torproject.org

[collector/master] Implements task-19895 run-once functionality.
by karsten＠torproject.org 02 Sep '16

02 Sep '16

commit 95bba76f5adb9defa2b9ae23762b0ff1f0bb737e Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Aug 31 11:52:38 2016 +0200 Implements task-19895 run-once functionality. CollecTorMain implements Callable and scheduler waits for termination of all started Callables. --- .../torproject/collector/cron/CollecTorMain.java | 11 ++++- .../org/torproject/collector/cron/Scheduler.java | 57 +++++++++++++--------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/torproject/collector/cron/CollecTorMain.java b/src/main/java/org/torproject/collector/cron/CollecTorMain.java index 26c9671..c7d0c7e 100644 --- a/src/main/java/org/torproject/collector/cron/CollecTorMain.java +++ b/src/main/java/org/torproject/collector/cron/CollecTorMain.java @@ -14,9 +14,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Observable; import java.util.Observer; +import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicBoolean; -public abstract class CollecTorMain implements Observer, Runnable { +public abstract class CollecTorMain implements Callable<Object>, Observer, + Runnable { private static final Logger logger = LoggerFactory.getLogger( CollecTorMain.class); @@ -58,6 +60,13 @@ public abstract class CollecTorMain implements Observer, Runnable { logger.info("Terminating {} module of CollecTor.", module()); } + /** Wrapper for <code>run</code>. */ + @Override + public final Object call() { + run(); + return null; + } + @Override public synchronized void update(Observable obs, Object obj) { newConfigAvailable.set(true); diff --git a/src/main/java/org/torproject/collector/cron/Scheduler.java b/src/main/java/org/torproject/collector/cron/Scheduler.java index 78789f4..12fe1bb 100644 --- a/src/main/java/org/torproject/collector/cron/Scheduler.java +++ b/src/main/java/org/torproject/collector/cron/Scheduler.java @@ -11,8 +11,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.Callable; import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; @@ -51,6 +53,7 @@ public final class Scheduler implements ThreadFactory { */ public void scheduleModuleRuns(Map<Key, Class<? extends CollecTorMain>> collecTorMains, Configuration conf) { + List<Callable<Object>> runOnceMains = new ArrayList<>(); for (Map.Entry<Key, Class<? extends CollecTorMain>> ctmEntry : collecTorMains.entrySet()) { try { @@ -58,9 +61,15 @@ public final class Scheduler implements ThreadFactory { String prefix = ctmEntry.getKey().name().replace(ACTIVATED, ""); CollecTorMain ctm = ctmEntry.getValue() .getConstructor(Configuration.class).newInstance(conf); - scheduleExecutions(conf.getBool(Key.RunOnce), ctm, - conf.getInt(Key.valueOf(prefix + OFFSETMIN)), - conf.getInt(Key.valueOf(prefix + PERIODMIN))); + if (conf.getBool(Key.RunOnce)) { + logger.info("Prepare single run for " + ctm.getClass().getName() + + "."); + runOnceMains.add(Executors.callable(ctm)); + } else { + scheduleExecutions(ctm, + conf.getInt(Key.valueOf(prefix + OFFSETMIN)), + conf.getInt(Key.valueOf(prefix + PERIODMIN))); + } } } catch (ConfigurationException | IllegalAccessException | InstantiationException | InvocationTargetException @@ -70,30 +79,32 @@ public final class Scheduler implements ThreadFactory { + ". Reason: " + ex.getMessage(), ex); } } + try { + if (conf.getBool(Key.RunOnce)) { + scheduler.invokeAll(runOnceMains); + } + } catch (ConfigurationException | InterruptedException + | RejectedExecutionException | NullPointerException ex) { + logger.error("Cannot schedule run-once: " + ex.getMessage(), ex); + } } private static final long MILLIS_IN_A_MINUTE = 60_000L; - private void scheduleExecutions(boolean runOnce, CollecTorMain ctm, - int offset, int period) { - if (runOnce) { - logger.info("Single run for " + ctm.getClass().getName() + "."); - this.scheduler.execute(ctm); - } else { - logger.info("Periodic updater started for " + ctm.getClass().getName() - + "; offset=" + offset + ", period=" + period + "."); - long periodMillis = period * MILLIS_IN_A_MINUTE; - long initialDelayMillis = computeInitialDelayMillis( - System.currentTimeMillis(), offset * MILLIS_IN_A_MINUTE, periodMillis); - - /* Run after initialDelay delay and then every period min. */ - logger.info("Periodic updater will first run in {} and then every {} " - + "minutes.", initialDelayMillis < MILLIS_IN_A_MINUTE - ? "under 1 minute" - : (initialDelayMillis / MILLIS_IN_A_MINUTE) + " minute(s)", period); - this.scheduler.scheduleAtFixedRate(ctm, initialDelayMillis, periodMillis, - TimeUnit.MILLISECONDS); - } + private void scheduleExecutions(CollecTorMain ctm, int offset, int period) { + logger.info("Periodic updater started for " + ctm.getClass().getName() + + "; offset=" + offset + ", period=" + period + "."); + long periodMillis = period * MILLIS_IN_A_MINUTE; + long initialDelayMillis = computeInitialDelayMillis( + System.currentTimeMillis(), offset * MILLIS_IN_A_MINUTE, periodMillis); + + /* Run after initialDelay delay and then every period min. */ + logger.info("Periodic updater will first run in {} and then every {} " + + "minutes.", initialDelayMillis < MILLIS_IN_A_MINUTE + ? "under 1 minute" + : (initialDelayMillis / MILLIS_IN_A_MINUTE) + " minute(s)", period); + this.scheduler.scheduleAtFixedRate(ctm, initialDelayMillis, periodMillis, + TimeUnit.MILLISECONDS); } protected static long computeInitialDelayMillis(long currentMillis,

1 0

[metrics-web/master] Include more dependencies for the #20049 fix.
by karsten＠torproject.org 02 Sep '16

02 Sep '16

commit b7c1615cc8e0c58b72b1876e2a630dd29f4ea7cf Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Sep 2 12:05:50 2016 +0200 Include more dependencies for the #20049 fix. --- modules/legacy/build.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/legacy/build.xml b/modules/legacy/build.xml index 1e4049b..07ca673 100644 --- a/modules/legacy/build.xml +++ b/modules/legacy/build.xml @@ -13,6 +13,9 @@ <include name="postgresql-jdbc3-9.2.jar"/> <include name="commons-lang-2.6.jar"/> <include name="descriptor-1.4.0.jar"/> + <include name="logback-classic-1.1.2.jar"/> + <include name="logback-core-1.1.2.jar"/> + <include name="slf4j-api-1.7.7.jar"/> </fileset> </path>

1 0

[collector/master] Add support for Bifroest's bridge descriptor tarballs.
by karsten＠torproject.org 02 Sep '16

02 Sep '16

commit 46ab6609854e1e2f0c3d55a52f9ada1c2dd59793 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Aug 26 12:23:22 2016 +0200 Add support for Bifroest's bridge descriptor tarballs. Implements #20037. --- CHANGELOG.md | 3 ++ .../bridgedescs/BridgeDescriptorParser.java | 5 +-- .../bridgedescs/BridgeSnapshotReader.java | 36 +++++++++++++++++++--- .../bridgedescs/SanitizedBridgesWriter.java | 6 ++-- 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8be9e2..cf1ea12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changes in version 1.1.0 - 2016-08-31 + * Medium changes + - Add support for Bifroest's bridge descriptor tarballs. + * Minor changes - Remove quotes around base URL in index.json. diff --git a/src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java b/src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java index 511b171..d40052e 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java +++ b/src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java @@ -27,7 +27,7 @@ public class BridgeDescriptorParser { /** Parses the first line of the given descriptor data to determine the * descriptor type and passes it to the sanitized bridges writer. */ - public void parse(byte[] allData, String dateTime) + public void parse(byte[] allData, String dateTime, String authorityFingerprint) throws ConfigurationException { try { BufferedReader br = new BufferedReader(new StringReader( @@ -45,7 +45,8 @@ public class BridgeDescriptorParser { } } else { if (this.sbw != null) { - this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime); + this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime, + authorityFingerprint); } } } catch (IOException e) { diff --git a/src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java b/src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java index 25bb2ea..046db7d 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java +++ b/src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java @@ -100,9 +100,35 @@ public class BridgeSnapshotReader { } BufferedInputStream bis = new BufferedInputStream(tais); String fn = pop.getName(); - String dateTime = fn.substring(11, 21) + " " - + fn.substring(22, 24) + ":" + fn.substring(24, 26) - + ":" + fn.substring(26, 28); + String[] fnParts = fn.split("-"); + if (fnParts.length != 5) { + logger.warn("Invalid bridge descriptor tarball file name: " + + fn + ". Skipping."); + continue; + } + String authorityPart = String.format("%s-%s-", fnParts[0], + fnParts[1]); + String datePart = String.format("%s-%s-%s", fnParts[2], + fnParts[3], fnParts[4]); + String authorityFingerprint; + switch (authorityPart) { + case "from-tonga-": + authorityFingerprint = + "4A0CCD2DDC7995083D73F5D667100C8A5831F16D"; + break; + case "from-bifroest-": + authorityFingerprint = + "1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1"; + break; + default: + logger.warn("Did not recognize the bridge authority that " + + "generated " + fn + ". Skipping."); + continue; + } + String dateTime = datePart.substring(0, 10) + " " + + datePart.substring(11, 13) + ":" + + datePart.substring(13, 15) + ":" + + datePart.substring(15, 17); while ((tais.getNextTarEntry()) != null) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); int len; @@ -130,7 +156,7 @@ public class BridgeSnapshotReader { if (firstLine.startsWith("published ") || firstLine.startsWith("flag-thresholds ") || firstLine.startsWith("r ")) { - bdp.parse(allData, dateTime); + bdp.parse(allData, dateTime, authorityFingerprint); parsedStatuses++; } else if (descriptorImportHistory.contains(fileDigest)) { /* Skip server descriptors or extra-info descriptors if @@ -167,7 +193,7 @@ public class BridgeSnapshotReader { DigestUtils.sha(descBytes)); if (!descriptorImportHistory.contains( descriptorDigest)) { - bdp.parse(descBytes, dateTime); + bdp.parse(descBytes, dateTime, authorityFingerprint); descriptorImportHistory.add(descriptorDigest); if (firstLine.startsWith("router ")) { parsedServerDescriptors++; diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index dcac024..b787f78 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -379,7 +379,8 @@ public class SanitizedBridgesWriter extends CollecTorMain { * Sanitizes a network status and writes it to disk. */ public void sanitizeAndStoreNetworkStatus(byte[] data, - String publicationTime) throws ConfigurationException { + String publicationTime, String authorityFingerprint) + throws ConfigurationException { if (this.persistenceProblemWithSecrets) { /* There's a persistence problem, so we shouldn't scrub more IP @@ -546,8 +547,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { File tarballFile = new File( this.sanitizedBridgesDirectory.getAbsolutePath() + "/" + syear + "/" + smonth + "/statuses/" + sday + "/" + syear + smonth - + sday + "-" + stime + "-" - + "4A0CCD2DDC7995083D73F5D667100C8A5831F16D"); + + sday + "-" + stime + "-" + authorityFingerprint); File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(), "bridge-descriptors/statuses/" + tarballFile.getName()); File[] outputFiles = new File[] { tarballFile, rsyncFile };

1 0

[collector/master] Remove an .htaccess option.
by karsten＠torproject.org 02 Sep '16

02 Sep '16

commit 0f5266a571c4cebb0f18d4b1a193db05e3f3f64e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Sep 2 11:12:34 2016 +0200 Remove an .htaccess option. This reflects what's currently running on the main CollecTor instance. --- src/main/webapp/.htaccess | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/webapp/.htaccess b/src/main/webapp/.htaccess index 0d6bd98..5983f0f 100644 --- a/src/main/webapp/.htaccess +++ b/src/main/webapp/.htaccess @@ -1,4 +1,3 @@ -Options +Indexes IndexOptions IgnoreCase FancyIndexing FoldersFirst HTMLTable NameWidth=* DescriptionWidth=* SuppressIcon SuppressDescription SuppressHTMLPreamble IndexOrderDefault Descending Name IndexIgnore *.tmp

1 0

[tor-browser-bundle/hardened-builds] Don't hardcode particular SF mirror
by gk＠torproject.org 02 Sep '16

02 Sep '16

commit cbf19c98632116c7ac197feb72856deb04b18142 Author: Georg Koppen <gk(a)torproject.org> Date: Fri Sep 2 07:21:07 2016 +0000 Don't hardcode particular SF mirror --- gitian/versions | 6 +++++- gitian/versions.alpha | 2 +- gitian/versions.beta | 2 +- gitian/versions.nightly | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/gitian/versions b/gitian/versions index 22b52ff..46aad1b 100755 --- a/gitian/versions +++ b/gitian/versions @@ -84,6 +84,8 @@ SETUPTOOLS_PACKAGE=setuptools-${SETUPTOOLS_VER}.tar.gz PARSLEY_PACKAGE=Parsley-${PARSLEY_VER}.tar.gz GO14_PACKAGE=go${GO14_VER}.src.tar.gz GO_PACKAGE=go${GO_VER}.src.tar.gz +NSIS_PACKAGE=nsis-${NSIS_VER}-src.tar.bz2 +NSIS_DEBIAN_PACKAGE=nsis_${NSIS_VER}-1.debian.tar.xz STIXMATHFONT_PACKAGE=STIXv1.1.1-latex.zip NOTOEMOJIFONT_PACKAGE=NotoEmoji-Regular.ttf NOTOJPFONT_PACKAGE=NotoSansJP-Regular.otf @@ -143,7 +145,9 @@ SETUPTOOLS_URL=https://pypi.python.org/packages/source/s/setuptools/${SETUP… PARSLEY_URL=https://pypi.python.org/packages/source/P/Parsley/${PARSLEY_PAC… GO14_URL=https://golang.org/dl/${GO14_PACKAGE} GO_URL=https://golang.org/dl/${GO_PACKAGE} -STIXMATHFONT_URL=http://iweb.dl.sourceforge.net/project/stixfonts/Current%20Release/${STIXMATHFONT_PACKAGE} +NSIS_URL=http://downloads.sourceforge.net/nsis/${NSIS_PACKAGE} +NSIS_DEBIAN_URL=http://http.debian.net/debian/pool/main/n/nsis/${NSIS_DEBIAN_PACKAGE} +STIXMATHFONT_URL=http://downloads.sourceforge.net/stixfonts/${STIXMATHFONT_PACKAGE} NOTOEMOJIFONT_URL=https://github.com/googlei18n/noto-emoji/raw/2f1ffdd6fbbd… NOTOCJKBASE_URL=https://github.com/googlei18n/noto-cjk/raw/f36eda03dfa5582a… NOTOJPFONT_URL=${NOTOCJKBASE_URL}/${NOTOJPFONT_PACKAGE} diff --git a/gitian/versions.alpha b/gitian/versions.alpha index fae60f8..964e529 100755 --- a/gitian/versions.alpha +++ b/gitian/versions.alpha @@ -155,7 +155,7 @@ GO14_URL=https://golang.org/dl/${GO14_PACKAGE} GO_URL=https://golang.org/dl/${GO_PACKAGE} NSIS_URL=http://downloads.sourceforge.net/nsis/${NSIS_PACKAGE} NSIS_DEBIAN_URL=http://http.debian.net/debian/pool/main/n/nsis/${NSIS_DEBIA… -STIXMATHFONT_URL=http://heanet.dl.sourceforge.net/project/stixfonts/Current%20Release/${STIXMATHFONT_PACKAGE} +STIXMATHFONT_URL=http://downloads.sourceforge.net/stixfonts/${STIXMATHFONT_PACKAGE} NOTOEMOJIFONT_URL=https://github.com/googlei18n/noto-emoji/raw/2f1ffdd6fbbd… NOTOCJKBASE_URL=https://github.com/googlei18n/noto-cjk/raw/f36eda03dfa5582a… NOTOJPFONT_URL=${NOTOCJKBASE_URL}/${NOTOJPFONT_PACKAGE} diff --git a/gitian/versions.beta b/gitian/versions.beta index 92761a4..a686402 100755 --- a/gitian/versions.beta +++ b/gitian/versions.beta @@ -132,7 +132,7 @@ SETUPTOOLS_URL=https://pypi.python.org/packages/source/s/setuptools/${SETUP… PARSLEY_URL=https://pypi.python.org/packages/source/P/Parsley/${PARSLEY_PAC… GO14_URL=https://golang.org/dl/${GO14_PACKAGE} GO_URL=https://golang.org/dl/${GO_PACKAGE} -STIXMATHFONT_URL=http://iweb.dl.sourceforge.net/project/stixfonts/Current%20Release/${STIXMATHFONT_PACKAGE} +STIXMATHFONT_URL=http://downloads.sourceforge.net/stixfonts/${STIXMATHFONT_PACKAGE} NOTOEMOJIFONT_URL=https://github.com/googlei18n/noto-emoji/raw/2f1ffdd6fbbd… NOTOCJKBASE_URL=https://github.com/googlei18n/noto-cjk/raw/f36eda03dfa5582a… NOTOJPFONT_URL=${NOTOCJKBASE_URL}/${NOTOJPFONT_PACKAGE} diff --git a/gitian/versions.nightly b/gitian/versions.nightly index 1d216fa..27059d7 100755 --- a/gitian/versions.nightly +++ b/gitian/versions.nightly @@ -158,7 +158,7 @@ GO14_URL=https://golang.org/dl/${GO14_PACKAGE} GO_URL=https://golang.org/dl/${GO_PACKAGE} NSIS_URL=http://downloads.sourceforge.net/nsis/${NSIS_PACKAGE} NSIS_DEBIAN_URL=http://http.debian.net/debian/pool/main/n/nsis/${NSIS_DEBIA… -STIXMATHFONT_URL=http://heanet.dl.sourceforge.net/project/stixfonts/Current%20Release/${STIXMATHFONT_PACKAGE} +STIXMATHFONT_URL=http://downloads.sourceforge.net/stixfonts/${STIXMATHFONT_PACKAGE} NOTOEMOJIFONT_URL=https://github.com/googlei18n/noto-emoji/raw/2f1ffdd6fbbd… NOTOCJKBASE_URL=https://github.com/googlei18n/noto-cjk/raw/f36eda03dfa5582a… NOTOJPFONT_URL=${NOTOCJKBASE_URL}/${NOTOJPFONT_PACKAGE}

1 0

[tor-browser-bundle/maint-6.0] Don't hard-code particular SF mirror
by gk＠torproject.org 02 Sep '16

02 Sep '16

commit 0d0c51f747667c7cdb54f27ef789b55fd45d0e66 Author: Georg Koppen <gk(a)torproject.org> Date: Fri Sep 2 07:14:33 2016 +0000 Don't hard-code particular SF mirror --- gitian/versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitian/versions b/gitian/versions index 67c114c..03f1dfa 100755 --- a/gitian/versions +++ b/gitian/versions @@ -143,7 +143,7 @@ PARSLEY_URL=https://pypi.python.org/packages/source/P/Parsley/${PARSLEY_PAC… GO_URL=https://golang.org/dl/${GO_PACKAGE} NSIS_URL=http://downloads.sourceforge.net/nsis/${NSIS_PACKAGE} NSIS_DEBIAN_URL=http://http.debian.net/debian/pool/main/n/nsis/${NSIS_DEBIA… -STIXMATHFONT_URL=http://heanet.dl.sourceforge.net/project/stixfonts/Current%20Release/${STIXMATHFONT_PACKAGE} +STIXMATHFONT_URL=http://downloads.sourceforge.net/stixfonts/${STIXMATHFONT_PACKAGE} NOTOEMOJIFONT_URL=https://github.com/googlei18n/noto-emoji/raw/2f1ffdd6fbbd… NOTOCJKBASE_URL=https://github.com/googlei18n/noto-cjk/raw/f36eda03dfa5582a… NOTOJPFONT_URL=${NOTOCJKBASE_URL}/${NOTOJPFONT_PACKAGE}

1 0

[tor-browser-bundle/master] Don't hard-code particular SF mirror
by gk＠torproject.org 02 Sep '16

02 Sep '16

commit 11a8d116c244b26c82cdfd040f3a35bf359f5450 Author: Georg Koppen <gk(a)torproject.org> Date: Fri Sep 2 07:09:54 2016 +0000 Don't hard-code particular SF mirror --- gitian/versions | 2 +- gitian/versions.alpha | 2 +- gitian/versions.beta | 2 +- gitian/versions.nightly | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gitian/versions b/gitian/versions index c3fdb32..febeb06 100755 --- a/gitian/versions +++ b/gitian/versions @@ -148,7 +148,7 @@ GO14_URL=https://golang.org/dl/${GO14_PACKAGE} GO_URL=https://golang.org/dl/${GO_PACKAGE} NSIS_URL=http://downloads.sourceforge.net/nsis/${NSIS_PACKAGE} NSIS_DEBIAN_URL=http://http.debian.net/debian/pool/main/n/nsis/${NSIS_DEBIA… -STIXMATHFONT_URL=http://heanet.dl.sourceforge.net/project/stixfonts/Current%20Release/${STIXMATHFONT_PACKAGE} +STIXMATHFONT_URL=http://downloads.sourceforge.net/stixfonts/${STIXMATHFONT_PACKAGE} NOTOEMOJIFONT_URL=https://github.com/googlei18n/noto-emoji/raw/2f1ffdd6fbbd… NOTOCJKBASE_URL=https://github.com/googlei18n/noto-cjk/raw/f36eda03dfa5582a… NOTOJPFONT_URL=${NOTOCJKBASE_URL}/${NOTOJPFONT_PACKAGE} diff --git a/gitian/versions.alpha b/gitian/versions.alpha index 4be792f..acd15b3 100755 --- a/gitian/versions.alpha +++ b/gitian/versions.alpha @@ -148,7 +148,7 @@ GO14_URL=https://golang.org/dl/${GO14_PACKAGE} GO_URL=https://golang.org/dl/${GO_PACKAGE} NSIS_URL=http://downloads.sourceforge.net/nsis/${NSIS_PACKAGE} NSIS_DEBIAN_URL=http://http.debian.net/debian/pool/main/n/nsis/${NSIS_DEBIA… -STIXMATHFONT_URL=http://heanet.dl.sourceforge.net/project/stixfonts/Current%20Release/${STIXMATHFONT_PACKAGE} +STIXMATHFONT_URL=http://downloads.sourceforge.net/stixfonts/${STIXMATHFONT_PACKAGE} NOTOEMOJIFONT_URL=https://github.com/googlei18n/noto-emoji/raw/2f1ffdd6fbbd… NOTOCJKBASE_URL=https://github.com/googlei18n/noto-cjk/raw/f36eda03dfa5582a… NOTOJPFONT_URL=${NOTOCJKBASE_URL}/${NOTOJPFONT_PACKAGE} diff --git a/gitian/versions.beta b/gitian/versions.beta index 89003ff..6e5485e 100755 --- a/gitian/versions.beta +++ b/gitian/versions.beta @@ -124,7 +124,7 @@ SETUPTOOLS_URL=https://pypi.python.org/packages/source/s/setuptools/${SETUP… PARSLEY_URL=https://pypi.python.org/packages/source/P/Parsley/${PARSLEY_PAC… GO14_URL=https://golang.org/dl/${GO14_PACKAGE} GO_URL=https://golang.org/dl/${GO_PACKAGE} -STIXMATHFONT_URL=http://heanet.dl.sourceforge.net/project/stixfonts/Current%20Release/${STIXMATHFONT_PACKAGE} +STIXMATHFONT_URL=http://downloads.sourceforge.net/stixfonts/${STIXMATHFONT_PACKAGE} NOTOEMOJIFONT_URL=https://github.com/googlei18n/noto-emoji/raw/2f1ffdd6fbbd… NOTOCJKBASE_URL=https://github.com/googlei18n/noto-cjk/raw/f36eda03dfa5582a… NOTOJPFONT_URL=${NOTOCJKBASE_URL}/${NOTOJPFONT_PACKAGE} diff --git a/gitian/versions.nightly b/gitian/versions.nightly index f92a471..86f8b06 100755 --- a/gitian/versions.nightly +++ b/gitian/versions.nightly @@ -155,7 +155,7 @@ GO14_URL=https://golang.org/dl/${GO14_PACKAGE} GO_URL=https://golang.org/dl/${GO_PACKAGE} NSIS_URL=http://downloads.sourceforge.net/nsis/${NSIS_PACKAGE} NSIS_DEBIAN_URL=http://http.debian.net/debian/pool/main/n/nsis/${NSIS_DEBIA… -STIXMATHFONT_URL=http://heanet.dl.sourceforge.net/project/stixfonts/Current%20Release/${STIXMATHFONT_PACKAGE} +STIXMATHFONT_URL=http://downloads.sourceforge.net/stixfonts/${STIXMATHFONT_PACKAGE} NOTOEMOJIFONT_URL=https://github.com/googlei18n/noto-emoji/raw/2f1ffdd6fbbd… NOTOCJKBASE_URL=https://github.com/googlei18n/noto-cjk/raw/f36eda03dfa5582a… NOTOJPFONT_URL=${NOTOCJKBASE_URL}/${NOTOJPFONT_PACKAGE}

1 0

[metrics-web/master] Make another minor tweak to the #20049 fix.
by karsten＠torproject.org 01 Sep '16

01 Sep '16

commit 62ad25feaf8a0514cee7e82e35d60c1577d6f69f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Sep 1 21:14:06 2016 +0200 Make another minor tweak to the #20049 fix. --- .../torproject/ernie/cron/network/ConsensusStatsFileHandler.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java b/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java index 824e1eb..3e71348 100644 --- a/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java +++ b/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java @@ -139,6 +139,8 @@ public class ConsensusStatsFileHandler { + "! Aborting to read this file!"); break; } + /* Assume that all lines without authority nickname are based on + * Tonga's network status, not Bifroest's. */ String key = parts[0] + "," + (parts.length < 4 ? "Tonga" : parts[1]); String value = null; if (parts.length == 2) { @@ -147,11 +149,7 @@ public class ConsensusStatsFileHandler { value = key + "," + parts[1] + "," + parts[2]; } else if (parts.length == 4) { value = key + "," + parts[2] + "," + parts[3]; - } else { - /* Impossible, we already checked the range above. */ - } - /* Assume that all lines without authority nickname are based on - * Tonga's network status, not Bifroest's. */ + } /* No more cases as we already checked the range above. */ this.bridgesRaw.put(key, value); } br.close();

1 0

[metrics-web/master] Process bridge network statuses from two authorities.
by karsten＠torproject.org 01 Sep '16

01 Sep '16

commit d1de2a89cd96c0f9dc6615042ac47c984fbbd590 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Sep 1 15:56:56 2016 +0200 Process bridge network statuses from two authorities. Fixes #20049. --- modules/legacy/build.xml | 2 +- .../cron/network/ConsensusStatsFileHandler.java | 160 ++++++++++++--------- 2 files changed, 96 insertions(+), 66 deletions(-) diff --git a/modules/legacy/build.xml b/modules/legacy/build.xml index 6cacf50..1e4049b 100644 --- a/modules/legacy/build.xml +++ b/modules/legacy/build.xml @@ -12,7 +12,7 @@ <include name="commons-compress-1.9.jar"/> <include name="postgresql-jdbc3-9.2.jar"/> <include name="commons-lang-2.6.jar"/> - <include name="descriptor-1.2.0.jar"/> + <include name="descriptor-1.4.0.jar"/> </fileset> </path> diff --git a/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java b/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java index 6222859..824e1eb 100644 --- a/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java +++ b/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java @@ -51,15 +51,16 @@ public class ConsensusStatsFileHandler { private File bridgeConsensusStatsRawFile; /** - * Number of running bridges in a given bridge status. Map keys are - * bridge status times formatted as "yyyy-MM-dd HH:mm:ss", map values - * are lines as read from <code>stats/bridge-consensus-stats-raw</code>. + * Number of running bridges in a given bridge status. Map keys are the bridge + * status time formatted as "yyyy-MM-dd HH:mm:ss", a comma, and the bridge + * authority nickname, map values are lines as read from + * <code>stats/bridge-consensus-stats-raw</code>. */ private SortedMap<String, String> bridgesRaw; /** * Average number of running bridges per day. Map keys are dates - * formatted as "yyyy-MM-dd", map values are the last column as written + * formatted as "yyyy-MM-dd", map values are the remaining columns as written * to <code>stats/consensus-stats</code>. */ private SortedMap<String, String> bridgesPerDay; @@ -132,17 +133,26 @@ public class ConsensusStatsFileHandler { continue; } String[] parts = line.split(","); - String dateTime = parts[0]; - if (parts.length == 2) { - this.bridgesRaw.put(dateTime, line + ",0"); - } else if (parts.length == 3) { - this.bridgesRaw.put(dateTime, line); - } else { + if (parts.length < 2 || parts.length > 4) { this.logger.warning("Corrupt line '" + line + "' in file " + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "! Aborting to read this file!"); break; } + String key = parts[0] + "," + (parts.length < 4 ? "Tonga" : parts[1]); + String value = null; + if (parts.length == 2) { + value = key + "," + parts[1] + ",0"; + } else if (parts.length == 3) { + value = key + "," + parts[1] + "," + parts[2]; + } else if (parts.length == 4) { + value = key + "," + parts[2] + "," + parts[3]; + } else { + /* Impossible, we already checked the range above. */ + } + /* Assume that all lines without authority nickname are based on + * Tonga's network status, not Bifroest's. */ + this.bridgesRaw.put(key, value); } br.close(); this.logger.fine("Finished reading file " @@ -159,20 +169,21 @@ public class ConsensusStatsFileHandler { * Adds the intermediate results of the number of running bridges in a * given bridge status to the existing observations. */ - public void addBridgeConsensusResults(long publishedMillis, int running, - int runningEc2Bridges) { - String published = dateTimeFormat.format(publishedMillis); - String line = published + "," + running + "," + runningEc2Bridges; - if (!this.bridgesRaw.containsKey(published)) { + public void addBridgeConsensusResults(long publishedMillis, + String authorityNickname, int running, int runningEc2Bridges) { + String publishedAuthority = dateTimeFormat.format(publishedMillis) + "," + + authorityNickname; + String line = publishedAuthority + "," + running + "," + runningEc2Bridges; + if (!this.bridgesRaw.containsKey(publishedAuthority)) { this.logger.finer("Adding new bridge numbers: " + line); - this.bridgesRaw.put(published, line); + this.bridgesRaw.put(publishedAuthority, line); this.bridgeResultsAdded++; - } else if (!line.equals(this.bridgesRaw.get(published))) { + } else if (!line.equals(this.bridgesRaw.get(publishedAuthority))) { this.logger.warning("The numbers of running bridges we were just " + "given (" + line + ") are different from what we learned " - + "before (" + this.bridgesRaw.get(published) + ")! " + + "before (" + this.bridgesRaw.get(publishedAuthority) + ")! " + "Overwriting!"); - this.bridgesRaw.put(published, line); + this.bridgesRaw.put(publishedAuthority, line); } } @@ -191,10 +202,24 @@ public class ConsensusStatsFileHandler { while (descriptorFiles.hasNext()) { DescriptorFile descriptorFile = descriptorFiles.next(); if (descriptorFile.getDescriptors() != null) { + String authority = null; + if (descriptorFile.getFileName().contains( + "4A0CCD2DDC7995083D73F5D667100C8A5831F16D")) { + authority = "Tonga"; + } else if (descriptorFile.getFileName().contains( + "1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1")) { + authority = "Bifroest"; + } for (Descriptor descriptor : descriptorFile.getDescriptors()) { if (descriptor instanceof BridgeNetworkStatus) { + if (authority == null) { + this.logger.warning("Did not recognize the bridge authority " + + "that generated " + descriptorFile.getFileName() + + ". Skipping."); + continue; + } this.addBridgeNetworkStatus( - (BridgeNetworkStatus) descriptor); + (BridgeNetworkStatus) descriptor, authority); } } } @@ -203,7 +228,8 @@ public class ConsensusStatsFileHandler { } } - private void addBridgeNetworkStatus(BridgeNetworkStatus status) { + private void addBridgeNetworkStatus(BridgeNetworkStatus status, + String authority) { int runningBridges = 0; int runningEc2Bridges = 0; for (NetworkStatusEntry statusEntry @@ -215,7 +241,7 @@ public class ConsensusStatsFileHandler { } } } - this.addBridgeConsensusResults(status.getPublishedMillis(), + this.addBridgeConsensusResults(status.getPublishedMillis(), authority, runningBridges, runningEc2Bridges); } @@ -225,49 +251,51 @@ public class ConsensusStatsFileHandler { */ public void writeFiles() { - /* Go through raw observations of numbers of running bridges in bridge - * statuses, calculate averages per day, and add these averages to - * final results. */ - if (!this.bridgesRaw.isEmpty()) { - String tempDate = null; + /* Go through raw observations and put everything into nested maps by day + * and bridge authority. */ + Map<String, Map<String, int[]>> bridgesPerDayAndAuthority = + new HashMap<String, Map<String, int[]>>(); + for (String bridgesRawLine : this.bridgesRaw.values()) { + String date = bridgesRawLine.substring(0, 10); + if (!bridgesPerDayAndAuthority.containsKey(date)) { + bridgesPerDayAndAuthority.put(date, new TreeMap<String, int[]>()); + } + String[] parts = bridgesRawLine.split(","); + String authority = parts[1]; + if (!bridgesPerDayAndAuthority.get(date).containsKey(authority)) { + bridgesPerDayAndAuthority.get(date).put(authority, new int[3]); + } + int[] bridges = bridgesPerDayAndAuthority.get(date).get(authority); + bridges[0] += Integer.parseInt(parts[2]); + bridges[1] += Integer.parseInt(parts[3]); + bridges[2]++; + } + + /* Sum up average numbers of running bridges per day reported by all bridge + * authorities and add these averages to final results. */ + for (Map.Entry<String, Map<String, int[]>> perDay + : bridgesPerDayAndAuthority.entrySet()) { + String date = perDay.getKey(); int brunning = 0; int brunningEc2 = 0; - int statuses = 0; - Iterator<String> it = this.bridgesRaw.values().iterator(); - boolean haveWrittenFinalLine = false; - while (it.hasNext() || !haveWrittenFinalLine) { - String next = it.hasNext() ? it.next() : null; - /* Finished reading a day or even all lines? */ - if (tempDate != null && (next == null - || !next.substring(0, 10).equals(tempDate))) { - /* Only write results if we have seen at least half of all - * statuses. */ - if (statuses >= 24) { - String line = "," + (brunning / statuses) + "," - + (brunningEc2 / statuses); - /* Are our results new? */ - if (!this.bridgesPerDay.containsKey(tempDate)) { - this.logger.finer("Adding new average bridge numbers: " - + tempDate + line); - this.bridgesPerDay.put(tempDate, line); - } else if (!line.equals(this.bridgesPerDay.get(tempDate))) { - this.logger.finer("Replacing existing average bridge " - + "numbers (" + this.bridgesPerDay.get(tempDate) - + " with new numbers: " + line); - this.bridgesPerDay.put(tempDate, line); - } - } - brunning = brunningEc2 = statuses = 0; - haveWrittenFinalLine = (next == null); - } - /* Sum up number of running bridges. */ - if (next != null) { - tempDate = next.substring(0, 10); - statuses++; - String[] parts = next.split(","); - brunning += Integer.parseInt(parts[1]); - brunningEc2 += Integer.parseInt(parts[2]); + for (int[] perAuthority : perDay.getValue().values()) { + int statuses = perAuthority[2]; + if (statuses < 12) { + /* Only write results if we have seen at least a dozen statuses. */ + continue; } + brunning += perAuthority[0] / statuses; + brunningEc2 += perAuthority[1] / statuses; + } + String line = "," + brunning + "," + brunningEc2; + /* Are our results new? */ + if (!this.bridgesPerDay.containsKey(date)) { + this.logger.finer("Adding new average bridge numbers: " + date + line); + this.bridgesPerDay.put(date, line); + } else if (!line.equals(this.bridgesPerDay.get(date))) { + this.logger.finer("Replacing existing average bridge numbers (" + + this.bridgesPerDay.get(date) + " with new numbers: " + line); + this.bridgesPerDay.put(date, line); } } @@ -278,9 +306,11 @@ public class ConsensusStatsFileHandler { this.bridgeConsensusStatsRawFile.getParentFile().mkdirs(); BufferedWriter bw = new BufferedWriter( new FileWriter(this.bridgeConsensusStatsRawFile)); - bw.append("datetime,brunning,brunningec2\n"); + bw.append("datetime,authority,brunning,brunningec2"); + bw.newLine(); for (String line : this.bridgesRaw.values()) { - bw.append(line + "\n"); + bw.append(line); + bw.newLine(); } bw.close(); this.logger.fine("Finished writing file " @@ -376,7 +406,7 @@ public class ConsensusStatsFileHandler { + "old: " + this.bridgesRaw.lastKey()); } } catch (ParseException e) { - /* Can't parse the timestamp? Whatever. */ + logger.warning("Can't parse the timestamp? Reason: " + e); } } logger.info(dumpStats.toString());

1 0

[tor-messenger-build/updater] Update aboutDialog.xul to remove link
by sukhbir＠torproject.org 01 Sep '16

01 Sep '16

commit 7810bee320ccadbc6e07c9dc3cf38dc4d1647293 Author: Sukhbir Singh <sukhbir(a)torproject.org> Date: Thu Sep 1 14:50:48 2016 -0400 Update aboutDialog.xul to remove link --- projects/instantbird/aboutDialog.xul | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/projects/instantbird/aboutDialog.xul b/projects/instantbird/aboutDialog.xul index 5e1f51f..ba924b9 100644 --- a/projects/instantbird/aboutDialog.xul +++ b/projects/instantbird/aboutDialog.xul @@ -110,10 +110,7 @@ #endif <description class="text-blurb" id="projectDesc"> &project.start; - <label class="text-link" - href="https://www.torproject.org/"> - &project.tpoLink; - </label>&project.end; + &project.tpoLink;&project.end; </description> <description class="text-blurb" id="helpDesc"> &help.donate;

1 0