August 2016 - tor-commits - lists.torproject.org

[tor-browser-bundle/hardened-builds] Updating CCTOOLS source url
by gk＠torproject.org 01 Aug '16

01 Aug '16

commit efbddbb105bfc7c3197c8ea79e7039a95a55cfbc Author: Georg Koppen <gk(a)torproject.org> Date: Mon Aug 1 13:16:01 2016 +0000 Updating CCTOOLS source url --- gitian/fetch-inputs.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gitian/fetch-inputs.sh b/gitian/fetch-inputs.sh index 1bf69a2..4d48089 100755 --- a/gitian/fetch-inputs.sh +++ b/gitian/fetch-inputs.sh @@ -6,6 +6,7 @@ MIRROR_URL=https://people.torproject.org/~mikeperry/mirrors/sources/ MIRROR_URL_DCF=https://people.torproject.org/~dcf/mirrors/sources/ MIRROR_URL_ASN=https://people.torproject.org/~asn/mirrors/sources/ +MIRROR_URL_GK=https://people.torproject.org/~asn/mirrors/sources/ set -e set -u umask 0022 @@ -138,6 +139,13 @@ do get "${!PACKAGE}" "${MIRROR_URL}${!PACKAGE}" done +for i in CCTOOLS +do + PACKAGE="${i}_PACKAGE" + URL="${MIRROR_URL_GK}${!PACKAGE}" + get "${!PACKAGE}" "${MIRROR_URL_GK}${!PACKAGE}" +done + # XXX: Omit googlecode.com packages because Google won't allow wget -N # and because the download seems to 404 about 50% of the time. for i in ARGPARSE

1 0

[collector/master] Check available disk space in relaydescs module.
by karsten＠torproject.org 01 Aug '16

01 Aug '16

commit 072474496192f37cb023147f8862d5515ecbad96 Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jul 27 17:37:32 2016 +0200 Check available disk space in relaydescs module. Check if there's enough space before and after running the relaydescs module, and warn if less than 200 MiB are left. Implements #18865. --- .../torproject/collector/cron/CollecTorMain.java | 26 ++++++++++++++++++++++ .../collector/relaydescs/ArchiveWriter.java | 2 ++ 2 files changed, 28 insertions(+) diff --git a/src/main/java/org/torproject/collector/cron/CollecTorMain.java b/src/main/java/org/torproject/collector/cron/CollecTorMain.java index 5fe6376..e869dae 100644 --- a/src/main/java/org/torproject/collector/cron/CollecTorMain.java +++ b/src/main/java/org/torproject/collector/cron/CollecTorMain.java @@ -10,7 +10,10 @@ import org.torproject.collector.conf.Key; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Calendar; import java.util.Map; import java.util.concurrent.Executors; @@ -21,6 +24,8 @@ public abstract class CollecTorMain implements Runnable { private static Logger log = LoggerFactory.getLogger(CollecTorMain.class); + private static final long LIMIT_MB = 200; + protected Configuration config; public CollecTorMain(Configuration conf) { @@ -50,5 +55,26 @@ public abstract class CollecTorMain implements Runnable { * Returns the module name for logging purposes. */ public abstract String module(); + + /** + * Checks the available space for the storage the given path is located on and + * logs a warning, if 200 MiB or less are available, and otherwise logs + * available space in TRACE level. + */ + public static void checkAvailableSpace(Path location) { + try { + long megaBytes = (long) (Files.getFileStore(location).getUsableSpace() + / 1024 / 1024); + if (megaBytes < LIMIT_MB) { + log.warn("Available storage critical for {}; only {} MiB left.", + location, megaBytes); + } else { + log.trace("Available storage for {}: {} MiB", location, megaBytes); + } + } catch (IOException ioe) { + log.warn("Cannot access {}; reason: {}", location, ioe.getMessage(), + ioe); + } + } } diff --git a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java index 24b7ab5..9053372 100644 --- a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java @@ -118,6 +118,7 @@ public class ArchiveWriter extends CollecTorMain { @Override protected void startProcessing() throws ConfigurationException { recentPath = config.getPath(Key.RecentPath); + CollecTorMain.checkAvailableSpace(recentPath); recentPathName = recentPath.toString(); File statsDir = config.getPath(Key.StatsPath).toFile(); storedServerDescriptorsFile @@ -192,6 +193,7 @@ public class ArchiveWriter extends CollecTorMain { new ReferenceChecker(recentPath.toFile(), new File(statsDir, "references"), new File(statsDir, "references-history")).check(); + CollecTorMain.checkAvailableSpace(recentPath); } private void loadDescriptorDigests() {

1 0

[metrics-lib/master] Bump version to 1.3.1-dev.
by karsten＠torproject.org 01 Aug '16

01 Aug '16

commit c68b09070a271a31c799fed6e12af731ac53e468 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 1 14:48:57 2016 +0200 Bump version to 1.3.1-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 23d56a9..5e08766 100644 --- a/build.xml +++ b/build.xml @@ -1,5 +1,5 @@ <project default="jar" name="descriptor" basedir="."> - <property name="release.version" value="1.3.1" /> + <property name="release.version" value="1.3.1-dev" /> <property name="sources" value="src/main/java"/> <property name="resources" value="src/main/resources"/> <property name="generated" value="generated/"/>

1 0

[tor-browser-bundle/hardened-builds] Add Selfrando sources for to versions.alpha
by gk＠torproject.org 01 Aug '16

01 Aug '16

commit d4e78124c4aef4220d59acd03df49c94c67801d1 Author: Georg Koppen <gk(a)torproject.org> Date: Mon Aug 1 12:42:22 2016 +0000 Add Selfrando sources for to versions.alpha --- gitian/versions.alpha | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitian/versions.alpha b/gitian/versions.alpha index dba74e7..fae60f8 100755 --- a/gitian/versions.alpha +++ b/gitian/versions.alpha @@ -45,6 +45,7 @@ GO_X_CRYPTO_TAG=4ed45ec682102c643324fae5dff8dab085b6c300 GO_X_NET_TAG=7dbad50ab5b31073856416cdcfeb2796d682f844 OBFS4_TAG=obfs4proxy-0.0.5 NOTOFONTS_TAG=720e34851382ee3c1ef024d8dffb68ffbfb234c2 +SELFRANDO_TAG=8e49e7aa664625d8e4793579864231b151589d0f GITIAN_TAG=tor-browser-builder-4-1 @@ -68,6 +69,7 @@ PARSLEY_VER=1.2 GO14_VER=1.4.3 GO_VER=1.6.3 NSIS_VER=2.51 +ELFUTILS_VER=0.160 ## File names for the source packages OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz @@ -99,6 +101,7 @@ NOTOJPFONT_PACKAGE=NotoSansJP-Regular.otf NOTOKRFONT_PACKAGE=NotoSansKR-Regular.otf NOTOSCFONT_PACKAGE=NotoSansSC-Regular.otf NOTOTCFONT_PACKAGE=NotoSansTC-Regular.otf +ELFUTILS_PACKAGE=elfutils-${ELFUTILS_VER}.tar.bz2 # Hashes for packages with weak sigs or no sigs OPENSSL_HASH=4a6ee491a2fdb22e519c76fdc2a628bb3cec12762cd456861d207996c8a07088 @@ -159,3 +162,4 @@ NOTOJPFONT_URL=${NOTOCJKBASE_URL}/${NOTOJPFONT_PACKAGE} NOTOKRFONT_URL=${NOTOCJKBASE_URL}/${NOTOKRFONT_PACKAGE} NOTOSCFONT_URL=${NOTOCJKBASE_URL}/${NOTOSCFONT_PACKAGE} NOTOTCFONT_URL=${NOTOCJKBASE_URL}/${NOTOTCFONT_PACKAGE} +ELFUTILS_URL=https://fedorahosted.org/releases/e/l/elfutils/${ELFUTILS_VER}/${ELFUTILS_PACKAGE}

1 0

[metrics-lib/master] Update unit tests for 17943b0.
by karsten＠torproject.org 01 Aug '16

01 Aug '16

commit 6ac45212ce6a262235c2bc889fb2d981dc18e70c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 1 14:38:01 2016 +0200 Update unit tests for 17943b0. --- .../impl/DescriptorCollectorImplTest.java | 26 +++++----------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/src/test/java/org/torproject/descriptor/impl/DescriptorCollectorImplTest.java b/src/test/java/org/torproject/descriptor/impl/DescriptorCollectorImplTest.java index 4355888..77fb259 100644 --- a/src/test/java/org/torproject/descriptor/impl/DescriptorCollectorImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/DescriptorCollectorImplTest.java @@ -24,7 +24,7 @@ public class DescriptorCollectorImplTest { + "<img src=\"/icons/unknown.gif\" alt=\"[ ]\"></td><td>" + "<a href=\"" + remoteFilename + "\">" + "2015-05-24-12-00-00-consensus</a></td>" - + "<td align=\"right\">24-May-2015 12:08 </td>" + + "<td align=\"right\">2015-05-24 12:08 </td>" + "<td align=\"right\">1.5M</td><td> </td></tr>"; SortedMap<String, Long> remoteFiles = new DescriptorCollectorImpl().parseDirectoryListing( @@ -40,8 +40,8 @@ public class DescriptorCollectorImplTest { @Test() public void testSameFileTwoTimestampsLastWins() { String remoteFilename = "2015-05-24-12-00-00-consensus"; - String firstTimestamp = "24-May-2015 12:04"; - String secondTimestamp = "24-May-2015 12:08"; + String firstTimestamp = "2015-05-24 12:04"; + String secondTimestamp = "2015-05-24 12:08"; String lineFormat = "<tr><td valign=\"top\">" + "<img src=\"/icons/unknown.gif\" alt=\"[ ]\"></td><td>" + "<a href=\"%s\">2015-05-24-12-00-00-consensus</a></td>" @@ -65,7 +65,7 @@ public class DescriptorCollectorImplTest { String directoryListing = "<tr><td valign=\"top\">" + "<img src=\"/icons/folder.gif\" alt=\"[DIR]\"></td><td>" + "<a href=\"subdir/\">subdir/</a></td>" - + "<td align=\"right\">27-May-2015 14:07 </td>" + + "<td align=\"right\">2015-05-27 14:07 </td>" + "<td align=\"right\"> - </td><td> </td></tr>"; DescriptorCollectorImpl collector = new DescriptorCollectorImpl(); SortedMap<String, Long> remoteFiles = collector.parseDirectoryListing( @@ -94,7 +94,7 @@ public class DescriptorCollectorImplTest { + "<img src=\"/icons/unknown.gif\" alt=\"[ ]\"></td><td>" + "<a href=\"2015-05-24-12-00-00-consensus\">" + "2015-05-24-12-00-00-consensus</a></td>" - + "<td align=\"right\">2015-05-24 12:08 </td>" + + "<td align=\"right\">24-May-2015 12:08 </td>" + "<td align=\"right\">1.5M</td><td> </td></tr>"; SortedMap<String, Long> remoteFiles = new DescriptorCollectorImpl().parseDirectoryListing( @@ -109,21 +109,7 @@ public class DescriptorCollectorImplTest { + "<img src=\"/icons/unknown.gif\" alt=\"[ ]\"></td><td>" + "<a href=\"2015-05-24-12-00-00-consensus\">" + "2015-05-24-12-00-00-consensus</a></td>" - + "<td align=\"right\">34-May-2015 12:08 </td>" - + "<td align=\"right\">1.5M</td><td> </td></tr>"; - SortedMap<String, Long> remoteFiles = - new DescriptorCollectorImpl().parseDirectoryListing( - REMOTE_DIRECTORY_CONSENSUSES, directoryListing); - assertNull(remoteFiles); - } - - @Test() - public void testInvalidLocaleDe() { - String directoryListing = "<tr><td valign=\"top\">" - + "<img src=\"/icons/unknown.gif\" alt=\"[ ]\"></td><td>" - + "<a href=\"2015-05-24-12-00-00-consensus\">" - + "2015-05-24-12-00-00-consensus</a></td>" - + "<td align=\"right\">24-Mai-2015 12:08 </td>" + + "<td align=\"right\">2015-05-34 12:08 </td>" + "<td align=\"right\">1.5M</td><td> </td></tr>"; SortedMap<String, Long> remoteFiles = new DescriptorCollectorImpl().parseDirectoryListing(

1 0

[metrics-lib/master] Add change log entry for 17943b0.
by karsten＠torproject.org 01 Aug '16

01 Aug '16

commit 6aa37108e2891de74931901bc88b39fac308d21b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 1 14:26:28 2016 +0200 Add change log entry for 17943b0. --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d71b148..f034b5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# Changes in version 1.3.1 - 2016-08-01 + + * Medium changes + - Adapt to CollecTor's new date format to make DescriptorCollector + work again. + + # Changes in version 1.3.0 - 2016-07-06 * Medium changes

1 0

[metrics-lib/master] Prepare for 1.3.1 release.
by karsten＠torproject.org 01 Aug '16

01 Aug '16

commit ec268b51e323137a184962154b64e5417c91d887 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 1 14:38:36 2016 +0200 Prepare for 1.3.1 release. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index cd0f921..23d56a9 100644 --- a/build.xml +++ b/build.xml @@ -1,5 +1,5 @@ <project default="jar" name="descriptor" basedir="."> - <property name="release.version" value="1.3.0-dev" /> + <property name="release.version" value="1.3.1" /> <property name="sources" value="src/main/java"/> <property name="resources" value="src/main/resources"/> <property name="generated" value="generated/"/>

1 0

[tor-browser-bundle/hardened-builds] updating gk's key
by gk＠torproject.org 01 Aug '16

01 Aug '16

commit 1911d4e4ae2ed9de4bc466e63ae1cbdaff409d77 Author: Georg Koppen <gk(a)torproject.org> Date: Mon Aug 1 11:50:36 2016 +0000 updating gk's key --- gitian/gpg/gk.gpg | Bin 72418 -> 113673 bytes gitian/gpg/torbutton.gpg | Bin 94114 -> 135369 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/gitian/gpg/gk.gpg b/gitian/gpg/gk.gpg index 2f78b14..359ea69 100644 Binary files a/gitian/gpg/gk.gpg and b/gitian/gpg/gk.gpg differ diff --git a/gitian/gpg/torbutton.gpg b/gitian/gpg/torbutton.gpg index da40f76..17b6c70 100644 Binary files a/gitian/gpg/torbutton.gpg and b/gitian/gpg/torbutton.gpg differ

1 0

[tor-browser-bundle/hardened-builds] Bug 19737: Allow git tag signatures made using an expired key
by gk＠torproject.org 01 Aug '16

01 Aug '16

commit c6458e49f9dd51708e22c84f26195c50d0ff2d0b Author: Nicolas Vigier <boklm(a)torproject.org> Date: Wed Jul 27 22:53:19 2016 +0200 Bug 19737: Allow git tag signatures made using an expired key We are adding a gitian/git-gpg-wrapper script which validates signatures made using an expired key. To do that, we use gpgv rather than gpg, and any EXPKEYSIG line from the gpg `--status-fd` output is replaced by a GOODSIG line. --- gitian/git-gpg-wrapper | 13 +++++++++++++ gitian/verify-tags.sh | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/gitian/git-gpg-wrapper b/gitian/git-gpg-wrapper new file mode 100644 index 0000000..f137d6d --- /dev/null +++ b/gitian/git-gpg-wrapper @@ -0,0 +1,13 @@ +#!/bin/bash +# This wrapper script is used by git to verify signatures made using +# an expired key. +# https://bugs.torproject.org/19737 +set -e +if [ $# -eq 4 ] && [ "$1" = '--status-fd=1' ] \ + && [ "$2" = '--verify' ] +then + gpgv "$1" "$3" "$4" | sed 's/^\[GNUPG:\] EXPKEYSIG /\[GNUPG:\] GOODSIG /' + exit ${PIPESTATUS[0]} +else + exec gpg "$@" +fi diff --git a/gitian/verify-tags.sh b/gitian/verify-tags.sh index 5908801..8277fca 100755 --- a/gitian/verify-tags.sh +++ b/gitian/verify-tags.sh @@ -37,10 +37,11 @@ verify_git() { local gpghome=$(mktemp -d) echo "rm -rf '$gpghome'" >> "$CLEANUP" - GNUPGHOME="$gpghome" gpg --import "$keyring" + GNUPGHOME="$gpghome" gpg --no-default-keyring --keyring trustedkeys.gpg --import "$keyring" pushd . cd "$dir" + git config --local gpg.program "$WRAPPER_DIR/git-gpg-wrapper" if ! GNUPGHOME="$gpghome" git tag -v "$tag"; then echo >&2 "$dir: verification of tag $tag against $keyring failed!" exit 1

1 0

[tor-browser-bundle/hardened-builds] Making the git-gpg-wrapper executable
by gk＠torproject.org 01 Aug '16

01 Aug '16

commit 166f1c2a301c5f8ca8d949e32ad2728c6bdb42af Author: Georg Koppen <gk(a)torproject.org> Date: Sat Jul 30 10:00:11 2016 +0000 Making the git-gpg-wrapper executable --- gitian/git-gpg-wrapper | 0 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/gitian/git-gpg-wrapper b/gitian/git-gpg-wrapper old mode 100644 new mode 100755

1 0