January 2017 - tor-commits - lists.torproject.org

[onionoo/master] Implements task-20596: use metrics-base and reduced build.xml,
by karsten＠torproject.org 05 Jan '17

05 Jan '17

commit 0442032f24c8764d6c055ed3b9f4e047fbaa886e Author: iwakeh <iwakeh(a)torproject.org> Date: Mon Dec 26 13:27:48 2016 +0100 Implements task-20596: use metrics-base and reduced build.xml, added bootstrap script. --- .gitmodules | 3 + build.xml | 362 +++++----------------------- src/build | 1 + src/main/resources/bootstrap-development.sh | 10 + 4 files changed, 69 insertions(+), 307 deletions(-) diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..e87aef6 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "src/build"] + path = src/build + url = https://git.torproject.org/metrics-base diff --git a/build.xml b/build.xml index bc68da9..5f1d798 100644 --- a/build.xml +++ b/build.xml @@ -1,35 +1,47 @@ -<project default="dist" name="onionoo" basedir="."> +<?xml version="1.0"?> +<!DOCTYPE project [ + <!ENTITY base SYSTEM "src/build/java/base.xml"> +]> + +<project default="usage" name="Onionoo" basedir="."> + + <property name="javadoc-title" value="Onionoo API Documentation"/> + <property name="implementation-title" value="Onionoo" /> <property name="onionoo.protocol.version" value="3.1"/> <property name="release.version" value="${onionoo.protocol.version}-1.0.1-dev"/> <property name="descriptorversion" value="1.5.0"/> - <property name="jetty.version" value="8.1.16.v20140903" /> - <property name="javasources" value="src/main/java"/> - <property name="testsources" value="src/test/java"/> - <property name="generated" value="generated"/> - <property name="classes" value="${generated}/classes"/> - <property name="testclasses" value="${generated}/testclasses/"/> - <property name="resources" value="src/main/resources/"/> - <property name="testresources" value="src/test/resources/"/> - <property name="testresult" value="${generated}/test-results"/> - <property name="coverageresult" value="${generated}/coverage-report/"/> - <property name="instrument" value="${generated}/instrument/"/> - <property name="cobertura.ser.file" value="${basedir}/cobertura.ser" /> - <property name="dist" value="${generated}/dist"/> - <property name="signed" value="${dist}/signed"/> - <property name="docs" value="${generated}/javadoc/"/> - <property name="libs" value="lib"/> + <property name="jetty.version" value="" /> <property name="warfile" value="onionoo-${release.version}.war"/> - <property name="onionoo.jarfile" - value="onionoo-${release.version}.jar"/> - <property name="source-and-target-java-version" value="1.7" /> - <property name="release.tarball" - value="${dist}/onionoo-${release.version}.tar.gz" /> - <property file="build.properties" /> - <patternset id="runtime" > +  + <property name="name" value="onionoo"/> + +  + <property name="project-main-class" + value="org.torproject.onionoo.cron.Main" /> + + <property name="jarincludes" value="logback.xml" /> + <property name="jarpatternprop" value="library" /> + <property name="additional2sign" value="${warfile}" /> + + <target name="coverage-check"> + <cobertura-check branchrate="0" linerate="0" + totallinerate="34" totalbranchrate="30" > + <regex pattern="org.torproject.onionoo.server" + branchrate="67" linerate="77"/> + <regex pattern="org.torproject.onionoo.docs" + branchrate="19" linerate="29"/> + <regex pattern="org.torproject.onionoo.updater" + branchrate="17" linerate="18"/> + <regex pattern="org.torproject.onionoo.writer" + branchrate="22" linerate="20"/> + </cobertura-check> + </target> + + <patternset id="library" > <include name="descriptor-${descriptorversion}.jar"/> <include name="commons-codec-1.9.jar"/> <include name="commons-compress-1.9.jar"/> @@ -41,180 +53,26 @@ </patternset> <patternset id="web" > - <include name="servlet-api-3.0.jar"/> - <include name="jetty8-server-${jetty.version}.jar"/> - <include name="jetty8-servlets-${jetty.version}.jar"/> - <include name="jetty8-servlet-${jetty.version}.jar"/> - <include name="jetty8-util-${jetty.version}.jar"/> - <include name="jetty8-http-${jetty.version}.jar"/> - <include name="jetty8-io-${jetty.version}.jar"/> - <include name="jetty8-xml-${jetty.version}.jar"/> - <include name="jetty8-webapp-${jetty.version}.jar"/> - <include name="jetty8-continuation-${jetty.version}.jar"/> - <include name="jetty8-security-${jetty.version}.jar"/> + <include name="servlet-api-3.0.jar"/> + <include name="jetty8-server${jetty.version}.jar"/> + <include name="jetty8-servlets${jetty.version}.jar"/> + <include name="jetty8-servlet${jetty.version}.jar"/> + <include name="jetty8-util${jetty.version}.jar"/> + <include name="jetty8-http${jetty.version}.jar"/> + <include name="jetty8-io${jetty.version}.jar"/> + <include name="jetty8-xml${jetty.version}.jar"/> + <include name="jetty8-webapp${jetty.version}.jar"/> + <include name="jetty8-continuation${jetty.version}.jar"/> + <include name="jetty8-security${jetty.version}.jar"/> </patternset> - <patternset id="test" > - <include name="junit4-4.11.jar"/> - <include name="hamcrest-core-1.3.jar"/> + <patternset id="runtime" > + <patternset refid="library" /> + <patternset refid="web" /> </patternset> - <path id="classpath"> - <pathelement path="${classes}"/> - <fileset dir="${libs}"> - <patternset refid="runtime" /> - <patternset refid="web" /> - </fileset> - <fileset dir="${resources}"> - <include name="logback.xml,jetty.xml"/> - </fileset> - </path> - - <path id="test.classpath"> - <path refid="classpath" /> - <pathelement path="${testclasses}"/> - <fileset dir="${libs}"> - <patternset refid="runtime" /> - <patternset refid="test" /> - <patternset refid="web" /> - </fileset> - </path> - - <path id="checkstyle.classpath" > - <fileset dir="${libs}"> - <include name="checkstyle-6.17-all.jar" /> - </fileset> - </path> - - <path id="cobertura.classpath"> - <fileset dir="${libs}"> - <include name="descriptor-${descriptorversion}.jar"/> - <include name="cobertura-2.1.1.jar" /> - <include name="slf4j-api-1.7.7.jar" /> - <include name="commons-lang3-3.3.2.jar" /> - <include name="asm4-5.0.3.jar" /> - <include name="asm4-util-5.0.3.jar" /> - <include name="asm4-tree-5.0.3.jar" /> - <include name="asm4-commons-5.0.3.jar" /> - <include name="asm4-analysis-5.0.3.jar" /> - <include name="oro-2.0.8.jar" /> - <include name="logback-core-1.1.2.jar" /> - <include name="logback-classic-1.1.2.jar" /> - </fileset> - </path> - - <path id="cobertura.test.classpath"> - <path location="${instrument}" /> - <path refid="test.classpath" /> - <path refid="cobertura.classpath" /> - </path> - - <target name="find-revision"> - <exec executable="git" outputproperty="git.revision"> - <arg value="rev-parse" /> - <arg value="--short" /> - <arg value="HEAD" /> - </exec> - </target> - - <target name="init"> - <mkdir dir="${classes}"/> - <mkdir dir="${testclasses}"/> - <mkdir dir="${testresult}"/> - <mkdir dir="${docs}"/> - <mkdir dir="${dist}"/> - <mkdir dir="${signed}"/> - </target> - - <target name="clean" > - <delete includeEmptyDirs="true" quiet="true" > - <fileset dir="${generated}" defaultexcludes="false" includes="**" /> - </delete> - <delete file="${cobertura.ser.file}" quiet="true"/> - </target> - - <target name="compile" - depends="init"> - <javac destdir="${classes}" - srcdir="${javasources}" - source="${source-and-target-java-version}" - target="${source-and-target-java-version}" - debug="true" - deprecation="true" - optimize="false" - failonerror="true" - includeantruntime="false"> - <classpath refid="classpath"/> - </javac> - </target> - - <target name="compile-tests" depends="compile"> - <javac destdir="${testclasses}" - srcdir="${testsources}" - source="${source-and-target-java-version}" - target="${source-and-target-java-version}" - debug="true" debuglevel="lines,source" - deprecation="true" - optimize="false" - failonerror="true" - includeantruntime="false"> - <classpath refid="test.classpath"/> - </javac> - </target> - - <target name="docs" depends="init"> - <javadoc destdir="${docs}" - footer="&copy; 2016 The Tor Project" - doctitle="Onionoo Documentation" - use="true" - windowtitle="Onionoo Documentation"> - <classpath refid="classpath"/> - <fileset dir="${javasources}/" includes="**/*.java" /> - </javadoc> - </target> - - <target name="test" depends="compile,compile-tests"> - <junit fork="true" haltonfailure="true" printsummary="on"> - <classpath refid="test.classpath" /> - <formatter type="plain" usefile="false"/> - <batchtest> - <fileset dir="${testclasses}" - includes="**/*Test.class"/> - </batchtest> - </junit> - </target> - - <taskdef resource="com/puppycrawl/tools/checkstyle/ant/checkstyle-ant-task.properties"> - <classpath refid="checkstyle.classpath" /> - </taskdef> - <target name="checks" depends="compile,compile-tests"> - <checkstyle config="${testresources}/metrics_checks.xml"> - <fileset dir="${javasources}" - includes="**/*.java" /> - <fileset dir="${testsources}" - includes="**/*.java" /> - <classpath> - <path refid="classpath" /> - <path refid="checkstyle.classpath" /> - </classpath> - <formatter type="plain" - toFile="${generated}/checkstyle_report.txt"/> - </checkstyle> - <exec executable="cat" outputproperty="checkstyle.result"> - <arg value="${generated}/checkstyle_report.txt" /> - </exec> - <fail message="Checkstyle complaints: ${checkstyle.result}" > - <condition> - <not> - <length string="${checkstyle.result}" - length="29" /> - </not> - </condition> - </fail> - </target> - <target name="war" - depends="compile,find-revision"> + depends="compile,gitrev"> <war destfile="${dist}/${warfile}" webxml="${resources}/web.xml" basedir="${classes}"> @@ -227,7 +85,6 @@ <zips> <fileset dir="${libs}"> <patternset refid="runtime" /> - <patternset refid="web" /> </fileset> </zips> </archives> @@ -247,125 +104,16 @@ <attribute name="Implementation-Title" value="Onionoo Server"/> <attribute name="Implementation-Version" value="${release.version}-${git.revision}"/> - <attribute name="Onionoo-Protocol-Version" - value="${onionoo.protocol.version}" /> <attribute name="Main-Class" value="org.torproject.onionoo.server.ServerMain" /> </manifest> </war> </target> - <target name="jar" depends="compile,find-revision"> - <jar destfile="${dist}/${onionoo.jarfile}" - basedir="${classes}"> - <restrict> - <not> - <name name="META-INF/*" /> - </not> - <archives> - <zips> - <fileset dir="${libs}"> - <patternset refid="runtime" /> - </fileset> - </zips> - </archives> - </restrict> - <fileset dir="${resources}" includes="logback.xml" /> - <exclude name="**/Test*.class"/> - <manifest> - <attribute name="Created-By" value="The Tor Project" /> - <attribute name="Implementation-Title" value="Onionoo Server"/> - <attribute name="Implementation-Version" - value="${release.version}-${git.revision}"/> - <attribute name="Onionoo-Protocol-Version" - value="${onionoo.protocol.version}" /> - <attribute name="Main-Class" - value="org.torproject.onionoo.cron.Main" /> - </manifest> - </jar> - </target> - - <target name="signjar" depends="jar,war"> - <signjar alias="${jarsigner.alias}" - storepass="${jarsigner.storepass}" - sigalg="SHA256withRSA" - destdir="${signed}" - digestalg="SHA-256" - tsaurl="http://timestamp.digicert.com"> - <path> - <fileset dir="${dist}" includes="${onionoo.jarfile} ${warfile}"/> - </path> - </signjar> - <fail message="Signing failed." > - <condition> - <not> - <and> - <issigned file="${signed}/${onionoo.jarfile}"/> - <issigned file="${signed}/${warfile}"/> - </and> - </not> - </condition> - </fail> - </target> - - <target name="tar" depends="signjar"> - <tar destfile="${release.tarball}" compression="gzip"> - <tarfileset dir="." prefix="onionoo-${release.version}"> - <include name="${signed}/${onionoo.jarfile}"/> - <include name="${signed}/${warfile}"/> - <include name="build.xml" /> - <include name="LICENSE" /> - <include name="INSTALL" /> - <include name="DESIGN" /> - <include name="CERT" /> - <include name="*.md" /> - </tarfileset> - <tarfileset dir="${javasources}" - prefix="onionoo-${release.version}/${javasources}" /> - <tarfileset dir="${resources}" - prefix="onionoo-${release.version}/${resources}" /> - <tarfileset dir="${testsources}" - prefix="onionoo-${release.version}/${testsources}" /> - <tarfileset dir="${testresources}" - prefix="onionoo-${release.version}/${testresources}" /> - <tarfileset dir="${libs}" - prefix="onionoo-${release.version}/${libs}" /> - </tar> - </target> - - <taskdef classpathref="cobertura.classpath" resource="tasks.properties" /> - <target name="coverage" depends="compile,compile-tests"> - <copy todir="${instrument}" > - <fileset dir="${classes}" > - <include name="**/*"/> - </fileset> - </copy> - <cobertura-instrument ignoreTrivial="true"> - <fileset dir="${instrument}"> - <include name="**/**/*.class" /> - </fileset> - </cobertura-instrument> - <junit fork="true" haltonfailure="false" printsummary="on"> - <sysproperty key="net.sourceforge.cobertura.datafile" - file="${cobertura.ser.file}" /> - <classpath refid="cobertura.test.classpath" /> - <formatter type="xml" /> - <batchtest toDir="${testresult}" > - <fileset dir="${testclasses}" includes="**/*Test.class" /> - </batchtest> - </junit> - <cobertura-report format="html" destdir="${coverageresult}" > - <fileset dir="${javasources}"> - <include name="**/*.java" /> - </fileset> - </cobertura-report> - <cobertura-check branchrate="0" linerate="0" totallinerate="34" totalbranchrate="30" > - <regex pattern="org.torproject.onionoo.server" branchrate="67" linerate="77"/> - <regex pattern="org.torproject.onionoo.docs" branchrate="19" linerate="29"/> - <regex pattern="org.torproject.onionoo.updater" branchrate="17" linerate="18"/> - <regex pattern="org.torproject.onionoo.writer" branchrate="22" linerate="20"/> - </cobertura-check> - </target> +  + &base; </project> diff --git a/src/build b/src/build new file mode 160000 index 0000000..7a01d68 --- /dev/null +++ b/src/build @@ -0,0 +1 @@ +Subproject commit 7a01d68f813cb5ae2904e73dbc81999ce0622eca diff --git a/src/main/resources/bootstrap-development.sh b/src/main/resources/bootstrap-development.sh new file mode 100644 index 0000000..33a1a50 --- /dev/null +++ b/src/main/resources/bootstrap-development.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# +# Copyright 2016 The Tor Project +# See LICENSE for licensing information +# +# Use for submodule initialization and checkout. +# Only necessary after very first checkout without recursion. +# +git submodule update --init --remote +

1 0

[onionoo/master] squash! squash! Implements task-20596: use metrics-base and reduced build.xml, added executable bootstrap script. Removed obsolete DESIGN document and metrics_checks.xml.
by karsten＠torproject.org 05 Jan '17

05 Jan '17

commit df833f880ef50dbd5cf1d6c52d9df2166ca1e799 Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jan 4 12:51:41 2017 +0100 squash! squash! Implements task-20596: use metrics-base and reduced build.xml, added executable bootstrap script. Removed obsolete DESIGN document and metrics_checks.xml. Make target war run before signing. --- build.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/build.xml b/build.xml index 26620d8..620fb3b 100644 --- a/build.xml +++ b/build.xml @@ -26,6 +26,7 @@ <property name="jarincludes" value="logback.xml" /> <property name="jarpatternprop" value="library" /> <property name="additional2sign" value="${warfile}" /> + <property name="tardepends" value="war" /> <target name="coverage-check"> <cobertura-check branchrate="0" linerate="0"

1 0

[onionoo/master] Update to latest metrics-base.
by karsten＠torproject.org 05 Jan '17

05 Jan '17

commit 97b456895e58de911d8ec60af558d29e1598c9f2 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jan 5 15:27:44 2017 +0100 Update to latest metrics-base. --- src/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build b/src/build index 7a01d68..bb0e1e5 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 7a01d68f813cb5ae2904e73dbc81999ce0622eca +Subproject commit bb0e1e50ba75dfa15add334706d8ce288fbf2b0b

1 0

[metrics-base/master] Only include resources in binary jar, neither in javadoc nor sources jars.
by karsten＠torproject.org 05 Jan '17

05 Jan '17

commit bb0e1e50ba75dfa15add334706d8ce288fbf2b0b Author: iwakeh <iwakeh(a)torproject.org> Date: Thu Jan 5 14:31:31 2017 +0100 Only include resources in binary jar, neither in javadoc nor sources jars. Improved comments in build template. --- java/base.xml | 5 ++++- java/build.xml.template | 12 +++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/java/base.xml b/java/base.xml index feba6b0..47540e0 100644 --- a/java/base.xml +++ b/java/base.xml @@ -242,6 +242,7 @@ <param name="usepattern" value="empty" /> <param name="uselibpath" value="${libs}" /> <param name="uselibpattern" value="${jarpattern}" /> + <param name="resourceincludes" value="${jarincludes}" /> </antcall> <antcall target="jarring" > <param name="destfile" value="${dist}/${jarsourcesfile}" /> @@ -250,6 +251,7 @@ <param name="usepattern" value="**/*.java" /> <param name="uselibpath" value="${libs}" /> <param name="uselibpattern" value="empty" /> + <param name="resourceincludes" value="X" /> </antcall> <antcall target="jarring" > <param name="destfile" value="${dist}/${jardocsfile}" /> @@ -258,6 +260,7 @@ <param name="usepattern" value="*" /> <param name="uselibpath" value="${libs}" /> <param name="uselibpattern" value="empty" /> + <param name="resourceincludes" value="X" /> </antcall> </target> @@ -273,7 +276,7 @@ <jar destfile="${destfile}" basedir="${usebase}" manifest="${manifestfile}" > - <fileset dir="${resources}" includes="${jarincludes}" /> + <fileset dir="${resources}" includes="${resourceincludes}" /> <restrict> <not> <name name="META-INF/*" /> diff --git a/java/build.xml.template b/java/build.xml.template index cdc5582..4f5444d 100644 --- a/java/build.xml.template +++ b/java/build.xml.template @@ -22,20 +22,22 @@ <property name="tardepends" value="XXXX" /> --> -  +  <property name="name" value="XXXX"/> -  +   + space separated list of files. + Leave it as XXXX, if there are none. --> <property name="jarincludes" value="XXXX" />  + jar task uses 'runtime', if not set. <property name="jarpatternprop" value="XXXX" /> + -->  <target name="coverage-check">

1 0

[tor-browser-bundle/hardened-builds] We just want to have 64bit things for hardened
by gk＠torproject.org 05 Jan '17

05 Jan '17

commit 2eca5d42245a98b4572943b0307a74392bfa4f5c Author: Georg Koppen <gk(a)torproject.org> Date: Thu Jan 5 11:55:16 2017 +0000 We just want to have 64bit things for hardened --- gitian/descriptors/linux/gitian-pluggable-transports.yml | 3 --- gitian/descriptors/linux/gitian-webrtc.yml | 3 --- gitian/mkbundle-linux.sh | 3 +-- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/gitian/descriptors/linux/gitian-pluggable-transports.yml b/gitian/descriptors/linux/gitian-pluggable-transports.yml index 31aa9bb..9b4a278 100644 --- a/gitian/descriptors/linux/gitian-pluggable-transports.yml +++ b/gitian/descriptors/linux/gitian-pluggable-transports.yml @@ -59,11 +59,8 @@ files: - "dzip.sh" - "gmp-linux64-utils.zip" - "openssl-linux64-utils.zip" -- "binutils-linux32-utils.zip" - "binutils-linux64-utils.zip" -- "gcc-linux32-utils.zip" - "gcc-linux64-utils.zip" -- "webrtc-linux32-gbuilt.zip" - "webrtc-linux64-gbuilt.zip" script: | INSTDIR="$HOME/install" diff --git a/gitian/descriptors/linux/gitian-webrtc.yml b/gitian/descriptors/linux/gitian-webrtc.yml index ba654c5..f18da52 100644 --- a/gitian/descriptors/linux/gitian-webrtc.yml +++ b/gitian/descriptors/linux/gitian-webrtc.yml @@ -4,7 +4,6 @@ distro: "debian" suites: - "wheezy" architectures: -- "i386" - "amd64" packages: - "unzip" @@ -24,9 +23,7 @@ remotes: "dir": "depot_tools" files: - "dzip.sh" -- "binutils-linux32-utils.zip" - "binutils-linux64-utils.zip" -- "gcc-linux32-utils.zip" - "gcc-linux64-utils.zip" - "webrtc.tar.gz" script: | diff --git a/gitian/mkbundle-linux.sh b/gitian/mkbundle-linux.sh index e5be697..a9f4cea 100755 --- a/gitian/mkbundle-linux.sh +++ b/gitian/mkbundle-linux.sh @@ -187,8 +187,7 @@ else echo fi -if [ ! -f inputs/webrtc-linux32-gbuilt.zip -o \ - ! -f inputs/webrtc-linux64-gbuilt.zip ]; +if [ ! -f inputs/webrtc-linux64-gbuilt.zip ]; then echo echo "****** Starting WebRTC Component of Linux Bundle (4/6 for Linux) ******"

1 0

[tor-browser-bundle/hardened-builds] Bug 20735: Add snowflake pt to alpha Linux builds
by gk＠torproject.org 05 Jan '17

05 Jan '17

commit f78866cad71122fa851ebdf6c63053605c565826 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Apr 30 15:42:21 2016 -0700 Bug 20735: Add snowflake pt to alpha Linux builds * Snowflake depends on go-webrtc, which in turn depends on libwebrtc, which adds significantly to the build. webrtc is special, having its own build system that brings in lots of Chromium dependencies. * Further, building cgo reproducibly requires go 1.7.3 and a compiler that supports the following flags, -gno-record-gcc-switches -fdebug-prefix-map=$WORK=/tmp/go-build --- Bundle-Data/PTConfigs/bridge_prefs.js | 2 + .../PTConfigs/linux/torrc-defaults-appendix | 3 + gitian/Makefile | 5 +- gitian/check-prerequisites.sh | 8 +- .../linux/gitian-pluggable-transports.yml | 79 +++++++++++++ gitian/descriptors/linux/gitian-webrtc.yml | 125 +++++++++++++++++++++ gitian/fetch-inputs.sh | 28 +++++ gitian/mkbundle-linux.sh | 44 ++++++-- gitian/verify-tags.sh | 6 + gitian/versions | 13 ++- gitian/versions.alpha | 6 + gitian/versions.beta | 5 + gitian/versions.nightly | 6 + 13 files changed, 312 insertions(+), 18 deletions(-) diff --git a/Bundle-Data/PTConfigs/bridge_prefs.js b/Bundle-Data/PTConfigs/bridge_prefs.js index 6831555..bd92585 100644 --- a/Bundle-Data/PTConfigs/bridge_prefs.js +++ b/Bundle-Data/PTConfigs/bridge_prefs.js @@ -38,3 +38,5 @@ pref("extensions.torlauncher.default_bridge.obfs4.20", "obfs4 85.17.30.79:443 FC pref("extensions.torlauncher.default_bridge.meek-amazon.1", "meek 0.0.2.0:2 B9E7141C594AF25699E0079C1F0146F409495296 url=https://d2zfqthxsdq309.cloudfront.net/ front=a0.awsstatic.com"); pref("extensions.torlauncher.default_bridge.meek-azure.1", "meek 0.0.2.0:3 A2C13B7DFCAB1CBF3A884B6EB99A98067AB6EF44 url=https://az786092.vo.msecnd.net/ front=ajax.aspnetcdn.com"); + +pref("extensions.torlauncher.default_bridge.snowflake.1", "snowflake 0.0.3.0:1 B05EC36590AC0374F0E4D36DDA77908FD59D97F3"); diff --git a/Bundle-Data/PTConfigs/linux/torrc-defaults-appendix b/Bundle-Data/PTConfigs/linux/torrc-defaults-appendix index c79efde..7fbf63c 100644 --- a/Bundle-Data/PTConfigs/linux/torrc-defaults-appendix +++ b/Bundle-Data/PTConfigs/linux/torrc-defaults-appendix @@ -6,3 +6,6 @@ ClientTransportPlugin obfs2,obfs3,obfs4,scramblesuit exec ./TorBrowser/Tor/Plugg ## meek configuration ClientTransportPlugin meek exec ./TorBrowser/Tor/PluggableTransports/meek-client-torbrowser -- ./TorBrowser/Tor/PluggableTransports/meek-client + +## snowflake configuration +ClientTransportPlugin snowflake exec ./TorBrowser/Tor/PluggableTransports/snowflake-client -url https://snowflake-reg.appspot.com/ -front www.google.com -ice stun:stun.l.google.com:19302 diff --git a/gitian/Makefile b/gitian/Makefile index 1aac5ed..b321eab 100644 --- a/gitian/Makefile +++ b/gitian/Makefile @@ -93,7 +93,7 @@ prep-alpha: $(TORSOCKS) ./fetch-inputs.sh ../../gitian-builder/inputs/ versions.alpha -clean: clean-utils clean-tor clean-browser clean-pt clean-bundle +clean: clean-utils clean-tor clean-browser clean-webrtc clean-pt clean-bundle rm -f ../../gitian-builder/inputs/*.yml rm -f ../../gitian-builder/inputs/bundle.inputs rm -f ../../gitian-builder/inputs/versions* @@ -112,6 +112,9 @@ clean-browser: clean-bundle rm -f ../../gitian-builder/inputs/tor-browser-win* rm -f ../../gitian-builder/inputs/mar-tools-* +clean-webrtc: clean-bundle + rm -f ../../gitian-builder/inputs/webrtc-*-gbuilt.zip + clean-pt: clean-bundle rm -f ../../gitian-builder/inputs/pluggable-transports* diff --git a/gitian/check-prerequisites.sh b/gitian/check-prerequisites.sh index 858fd4d..530e160 100755 --- a/gitian/check-prerequisites.sh +++ b/gitian/check-prerequisites.sh @@ -17,7 +17,7 @@ then VERSION=`cat /etc/issue | grep -Eo '[0-9]{2}' | head -1` if [ "$VERSION" -ge "14" ]; then - dpkg -s ruby apache2 git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion debian-archive-keyring torsocks tor 2>/dev/null >/dev/null + dpkg -s ruby apache2 git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion debian-archive-keyring curl pkg-config libgtk2.0-dev libglib2.0-dev torsocks tor 2>/dev/null >/dev/null if [ $? -ne 0 ]; then @@ -25,7 +25,7 @@ then echo echo "Please run:" echo " sudo apt-get install torsocks tor" - echo " sudo torsocks apt-get install ruby apache2 git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion debian-archive-keyring" + echo " sudo torsocks apt-get install ruby apache2 git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion debian-archive-keyring curl pkg-config libgtk2.0-dev libglib2.0-dev" exit 1 fi else @@ -34,7 +34,7 @@ then fi elif [ $DISTRO = "Debian" ]; then - dpkg -s ruby git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot zip unzip torsocks tor python-cheetah debootstrap parted kpartx rsync 2>/dev/null >/dev/null + dpkg -s ruby git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot zip unzip python-cheetah debootstrap parted kpartx rsync curl pkg-config libgtk2.0-dev libglib2.0-dev torsocks tor 2>/dev/null >/dev/null if [ $? -ne 0 ]; then @@ -42,7 +42,7 @@ then echo echo "Please run" echo " sudo apt-get install torsocks tor" - echo " sudo torsocks apt-get install ruby git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot zip unzip python-cheetah debootstrap parted kpartx rsync" + echo " sudo torsocks apt-get install ruby git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot zip unzip python-cheetah debootstrap parted kpartx rsync curl pkg-config libgtk2.0-dev libglib2.0-dev" exit 1 fi else diff --git a/gitian/descriptors/linux/gitian-pluggable-transports.yml b/gitian/descriptors/linux/gitian-pluggable-transports.yml index 2793a76..31aa9bb 100644 --- a/gitian/descriptors/linux/gitian-pluggable-transports.yml +++ b/gitian/descriptors/linux/gitian-pluggable-transports.yml @@ -13,6 +13,8 @@ packages: - "swig" - "zip" - "hardening-wrapper" +- "pkg-config" +- "libx11-dev" reference_datetime: "2000-01-01 00:00:00" remotes: - "url": "https://git.torproject.org/pluggable-transports/pyptlib.git" @@ -39,6 +41,12 @@ remotes: "dir": "goxnet" - "url": "https://git.torproject.org/pluggable-transports/obfs4.git" "dir": "obfs4" +- "url": "https://github.com/keroserene/go-webrtc.git" + "dir": "go-webrtc" +- "url": "https://git.torproject.org/pluggable-transports/snowflake.git" + "dir": "snowflake" +- "url": "https://github.com/dchest/uniuri.git" + "dir": "uniuri" files: - "pycrypto.tar.gz" - "argparse.tar.gz" @@ -51,6 +59,12 @@ files: - "dzip.sh" - "gmp-linux64-utils.zip" - "openssl-linux64-utils.zip" +- "binutils-linux32-utils.zip" +- "binutils-linux64-utils.zip" +- "gcc-linux32-utils.zip" +- "gcc-linux64-utils.zip" +- "webrtc-linux32-gbuilt.zip" +- "webrtc-linux64-gbuilt.zip" script: | INSTDIR="$HOME/install" PTDIR="$INSTDIR/Tor/PluggableTransports" @@ -73,6 +87,35 @@ script: | # FTE only needs libgmp.so.10 and no libgmpxx anymore. cp $INSTDIR/gmp/lib/libgmp.so.10 $INSTDIR/Tor + # Preparing Binutils and GCC for webrtc + unzip -d $INSTDIR binutils-linux$GBUILD_BITS-utils.zip + # Make sure gold is used with the hardening wrapper for full RELRO, see + # #13031. + cd $INSTDIR/binutils/bin + rm ld + cp /usr/bin/hardened-ld ./ + mv ld.gold ld.gold.real + ln -sf hardened-ld ld.gold + ln -sf ld.gold ld + cd ~/build + unzip -d $INSTDIR gcc-linux$GBUILD_BITS-utils.zip + # Make sure we use the hardening wrapper when compiling Tor Browser. + cd $INSTDIR/gcc/bin + cp /usr/bin/hardened-cc ./ + mv gcc gcc.real + mv c++ c++.real + ln -sf hardened-cc gcc + ln -sf hardened-cc c++ + cd ~/build + export PATH=$INSTDIR/gcc/bin:$INSTDIR/binutils/bin:$PATH + ARCH="" + if [ $GBUILD_BITS == "64" ]; + then + ARCH="64" + fi + # Make sure our GCC gets properly used. + export LD_LIBRARY_PATH=$INSTDIR/gcc/lib$ARCH + # Building go 1.4.x # This is needed to bootstrap the go that we actually use # https://golang.org/doc/install/source#go14 @@ -231,6 +274,14 @@ script: | go install github.com/dchest/siphash cd .. + # Building go uniuri + cd uniuri + find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" + mkdir -p "$GOPATH/src/github.com/dchest/" + ln -sf "$PWD" "$GOPATH/src/github.com/dchest/uniuri" + go install github.com/dchest/uniuri + cd .. + # Building golang.org/x/crypto (obfs4proxy > 0.0.3 || Go >= 1.4) cd goxcrypto find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" @@ -259,6 +310,34 @@ script: | cp -a obfs4proxy $PTDIR cd ../.. + unzip webrtc-linux$GBUILD_BITS-gbuilt.zip + + # Building go-webrtc + cd go-webrtc + # Replace the prebuilt webrtc library with our own one. + rm -rf include/ lib/ + ln -sf ../webrtc/{include,lib} . + find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" + mkdir -p "$GOPATH/src/github.com/keroserene/" + ln -sf "$PWD" "$GOPATH/src/github.com/keroserene/go-webrtc" + # The go-webrtc package sets _GLIBCXX_USE_CXX11_ABI=0 for compatibility with + # the prebuilt libwebrtc-magic.a libraries. Since we build our own + # libwebrtc-magic.a with the C++11 ABI, we have to undo that setting here, + # using CGO_CXXFLAGS. + CGO_CXXFLAGS=-D_GLIBCXX_USE_CXX11_ABI=1 go install github.com/keroserene/go-webrtc + cd .. + + # Building snowflake + cd snowflake + find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" + cd client + go build -ldflags '-s' + cp -a client $PTDIR/snowflake-client + cd .. + mkdir -p $INSTDIR/Docs/snowflake + cp -a README.md LICENSE $INSTDIR/Docs/snowflake + cd .. + # Grabbing the results and making sure timestamps don't spoil them cd $INSTDIR ~/build/dzip.sh pluggable-transports-linux$GBUILD_BITS-gbuilt.zip Tor/ Docs/ diff --git a/gitian/descriptors/linux/gitian-webrtc.yml b/gitian/descriptors/linux/gitian-webrtc.yml new file mode 100644 index 0000000..ba654c5 --- /dev/null +++ b/gitian/descriptors/linux/gitian-webrtc.yml @@ -0,0 +1,125 @@ +--- +name: "webrtc-linux" +distro: "debian" +suites: +- "wheezy" +architectures: +- "i386" +- "amd64" +packages: +- "unzip" +- "zip" +- "hardening-wrapper" +- "libasound2-dev" +- "libexpat1-dev" +- "libudev-dev" +- "libx11-dev" +- "libxext-dev" +- "libxrandr-dev" +- "default-jre-headless" +- "default-jdk" +reference_datetime: "2000-01-01 00:00:00" +remotes: +- "url": "https://chromium.googlesource.com/chromium/tools/depot_tools.git" + "dir": "depot_tools" +files: +- "dzip.sh" +- "binutils-linux32-utils.zip" +- "binutils-linux64-utils.zip" +- "gcc-linux32-utils.zip" +- "gcc-linux64-utils.zip" +- "webrtc.tar.gz" +script: | + INSTDIR="$HOME/install" + export REFERENCE_DATETIME + export TZ=UTC + export LC_ALL=C + umask 0022 + # + mkdir -p $OUTDIR/ + + # Config options for hardening-wrapper + export DEB_BUILD_HARDENING=1 + export DEB_BUILD_HARDENING_STACKPROTECTOR=1 + export DEB_BUILD_HARDENING_FORTIFY=1 + export DEB_BUILD_HARDENING_FORMAT=1 + export DEB_BUILD_HARDENING_PIE=1 + + # Preparing Binutils and GCC for webrtc + unzip -d $INSTDIR binutils-linux$GBUILD_BITS-utils.zip + # Make sure gold is used with the hardening wrapper for full RELRO, see + # #13031. + cd $INSTDIR/binutils/bin + rm ld + cp /usr/bin/hardened-ld ./ + mv ld.gold ld.gold.real + ln -sf hardened-ld ld.gold + ln -sf ld.gold ld + cd ~/build + unzip -d $INSTDIR gcc-linux$GBUILD_BITS-utils.zip + # Make sure we use the hardening wrapper when compiling Tor Browser. + cd $INSTDIR/gcc/bin + cp /usr/bin/hardened-cc ./ + mv gcc gcc.real + mv c++ c++.real + ln -sf hardened-cc gcc + ln -sf hardened-cc c++ + cd ~/build + export PATH=$INSTDIR/gcc/bin:$INSTDIR/binutils/bin:$PATH + ARCH="" + if [ $GBUILD_BITS == "64" ]; + then + ARCH="64" + fi + # Make sure our GCC gets properly used. + export LD_LIBRARY_PATH=$INSTDIR/gcc/lib$ARCH + + # Setting up depot_tools + # https://dev.chromium.org/developers/how-tos/install-depot-tools + export PATH="$PATH:$PWD/depot_tools" + # Disable automatic updating. + export DEPOT_TOOLS_UPDATE=0 + + # Building webrtc + tar xaf webrtc.tar.gz + cd webrtc/src + export GYP_CROSSCOMPILE=1 + export GYP_DEFINES="OS=linux target_arch=$(if [ $GBUILD_BITS = 32 ]; then echo ia32; else echo x64; fi)" + # Do not use bundled utilities. + GYP_DEFINES+=" clang=0 host_clang=0 use_sysroot=0" + GYP_DEFINES+=" linux_use_bundled_gold=0 linux_use_gold_flags=0 linux_use_bundled_binutils=0" + # Avoid some dependencies. Some of these don't seem to work fully; for + # example even with use_alsa=0 we need libasound2-dev. + GYP_DEFINES+=" use_x11=0 use_gtk=0 use_gconf=0 use_alsa=0 use_pulseaudio=0 use_libpci=0 include_internal_audio_device=0 enabled_libjingle_device_manager=0" + # examples and tests also bring in dependencies. + GYP_DEFINES+=" include_examples=0 include_tests=0" + # embedded=1 is supposed to turn of various features; see + # https://bugs.chromium.org/p/chromium/issues/detail?id=318413. + GYP_DEFINES+=" embedded=1" + GYP_DEFINES+=" werror=" + JAVA_HOME=/usr/lib/jvm/default-java webrtc/build/gyp_webrtc.py + ninja -C out/Release + # Run ninja once more, without include_tests=0, in order to build just the + # FakeAudioCaptureModule that go-webrtc uses. + GYP_DEFINES="$(echo "$GYP_DEFINES" | sed -e 's/include_tests=0//g')" + JAVA_HOME=/usr/lib/jvm/default-java webrtc/build/gyp_webrtc.py + ninja -C out/Release obj/webrtc/api/test/peerconnection_unittests.fakeaudiocapturemodule.o + # https://github.com/keroserene/go-webrtc/issues/23#issuecomment-175312648 + # dump_syms_regtest.o is actually an executable, not an object file. If not + # excluded, it results in the error: + # libwebrtc-linux-386-magic.a(dump_syms_regtest.o): unsupported ELF file type 2 + ar crs libwebrtc-magic.a $(find . -name '*.o' -not -name '*.main.o' -not -name 'dump_syms_regtest.o' | sort) + cd ../.. + + # Grabbing the result + cd $INSTDIR + mkdir -p webrtc/include webrtc/lib + cp -f $HOME/build/webrtc/src/libwebrtc-magic.a webrtc/lib/libwebrtc-linux-$(if [ $GBUILD_BITS = 32 ]; then echo 386; else echo amd64; fi)-magic.a + INCLUDE_DIR="$PWD/webrtc/include" + (cd $HOME/build/webrtc/src && for h in $(find talk/ webrtc/ -type f -name '*.h'); do + mkdir -p "$INCLUDE_DIR/$(dirname $h)" + cp -f "$h" "$INCLUDE_DIR/$h" + done) + + ~/build/dzip.sh webrtc-linux$GBUILD_BITS-gbuilt.zip webrtc + cp webrtc-linux$GBUILD_BITS-gbuilt.zip $OUTDIR/ diff --git a/gitian/fetch-inputs.sh b/gitian/fetch-inputs.sh index 65ebc11..01ec802 100755 --- a/gitian/fetch-inputs.sh +++ b/gitian/fetch-inputs.sh @@ -273,6 +273,7 @@ https-everywhere https://git.torproject.org/https-everywhere.git $HTTPSE_TA torbutton https://git.torproject.org/torbutton.git $TORBUTTON_TAG tor-launcher https://git.torproject.org/tor-launcher.git $TORLAUNCHER_TAG tor-browser https://git.torproject.org/tor-browser.git $TORBROWSER_TAG +libcxx https://github.com/llvm-mirror/libcxx $LIBCXX_TAG mingw-w64-git http://git.code.sf.net/p/mingw-w64/mingw-w64 $MINGW_TAG pyptlib https://git.torproject.org/pluggable-transports/pyptlib.git $PYPTLIB_TAG obfsproxy https://git.torproject.org/pluggable-transports/obfsproxy.git $OBFSPROXY_TAG @@ -290,6 +291,10 @@ goxnet https://go.googlesource.com/net $GO_X_NET_TAG obfs4 https://git.torproject.org/pluggable-transports/obfs4.git $OBFS4_TAG noto-fonts https://github.com/googlei18n/noto-fonts $NOTOFONTS_TAG selfrando https://github.com/immunant/selfrando.git $SELFRANDO_TAG +depot_tools https://chromium.googlesource.com/chromium/tools/depot_tools.git $DEPOT_TOOLS_TAG +go-webrtc https://github.com/keroserene/go-webrtc $GO_WEBRTC_TAG +snowflake https://git.torproject.org/pluggable-transports/snowflake.git $SNOWFLAKE_TAG +uniuri https://github.com/dchest/uniuri $UNIURI_TAG EOF # HTTPS-Everywhere is special, too. We need to initialize the git submodules and @@ -299,5 +304,28 @@ git submodule init git submodule update cd .. +# WebRTC is special, having its own build system that brings in lots of Chromium dependencies. +# https://webrtc.org/native-code/development/ +# depot_tools must have been downloaded before running this code. +dir=webrtc +PATH="$PATH:$PWD/depot_tools" +# GYP_CROSSCOMPILE=1 and GYP_DEFINES="use_x11=0" prevent probing for certain dependencies. +# Use --no-history because the whole checkout with history is about 12 GB. +export GYP_CROSSCOMPILE=1 +export GYP_DEFINES="use_x11=0" +mkdir -p "$dir" +cd "$dir" +if [ ! -d "src" ]; +then + # "fetch" is part of depot_tools. + fetch --nohooks --no-history webrtc +fi +# "gclient" is part of depot_tools. This download takes a long time the first time. +# JAVA_HOME is needed in a hook for libjingle. The readlink line tries to find the current JRE. +# default-java comes from the package default-jdk-headless. +JAVA_HOME=/usr/lib/jvm/default-java gclient sync --with_branch_heads --no-history -r $WEBRTC_TAG +cd .. +tar --exclude .git -czf webrtc.tar.gz webrtc + exit 0 diff --git a/gitian/mkbundle-linux.sh b/gitian/mkbundle-linux.sh index 48aac24..e5be697 100755 --- a/gitian/mkbundle-linux.sh +++ b/gitian/mkbundle-linux.sh @@ -108,7 +108,7 @@ if [ ! -f inputs/binutils-$BINUTILS_VER-linux64-utils.zip -o \ ! -f inputs/self-rando-utils.zip ]; then echo - echo "****** Starting Utilities Component of Linux Bundle (1/5 for Linux) ******" + echo "****** Starting Utilities Component of Linux Bundle (1/6 for Linux) ******" echo ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit libevent=$LIBEVENT_TAG,selfrando=$SELFRANDO_TAG $DESCRIPTOR_DIR/linux/gitian-utils.yml @@ -129,7 +129,7 @@ then #cp -a result/utils-linux-res.yml inputs/ else echo - echo "****** SKIPPING already built Utilities Component of Linux Bundle (1/5 for Linux) ******" + echo "****** SKIPPING already built Utilities Component of Linux Bundle (1/6 for Linux) ******" echo # We might have built the utilities in the past but maybe the links are # pointing to the wrong version. Refresh them. @@ -145,7 +145,7 @@ fi if [ ! -f inputs/tor-linux64-gbuilt.zip ]; then echo - echo "****** Starting Tor Component of Linux Bundle (2/5 for Linux) ******" + echo "****** Starting Tor Component of Linux Bundle (2/6 for Linux) ******" echo ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor=$TOR_TAG $DESCRIPTOR_DIR/linux/gitian-tor.yml @@ -160,14 +160,14 @@ then #cp -a result/tor-linux-res.yml inputs/ else echo - echo "****** SKIPPING already built Tor Component of Linux Bundle (2/5 for Linux) ******" + echo "****** SKIPPING already built Tor Component of Linux Bundle (2/6 for Linux) ******" echo fi if [ ! -f inputs/tor-browser-linux64-gbuilt.zip ]; then echo - echo "****** Starting TorBrowser Component of Linux Bundle (3/5 for Linux) ******" + echo "****** Starting TorBrowser Component of Linux Bundle (3/6 for Linux) ******" echo ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor-browser=$TORBROWSER_TAG,faketime=$FAKETIME_TAG,selfrando=$SELFRANDO_TAG $DESCRIPTOR_DIR/linux/gitian-firefox.yml @@ -183,17 +183,39 @@ then #cp -a result/torbrowser-linux-res.yml inputs/ else echo - echo "****** SKIPPING already built TorBrowser Component of Linux Bundle (3/5 for Linux) ******" + echo "****** SKIPPING already built TorBrowser Component of Linux Bundle (3/6 for Linux) ******" + echo +fi + +if [ ! -f inputs/webrtc-linux32-gbuilt.zip -o \ + ! -f inputs/webrtc-linux64-gbuilt.zip ]; +then + echo + echo "****** Starting WebRTC Component of Linux Bundle (4/6 for Linux) ******" + echo + + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit depot_tools=$DEPOT_TOOLS_TAG $DESCRIPTOR_DIR/linux/gitian-webrtc.yml + if [ $? -ne 0 ]; + then + #mv var/build.log ./webrtc-fail-linux.log.`date +%Y%m%d%H%M%S` + exit 1 + fi + + cp -a build/out/webrtc-linux*-gbuilt.zip inputs/ + #cp -a result/webrtc-linux-res.yml inputs/ +else + echo + echo "****** SKIPPING already built WebRTC Component of Linux Bundle (4/6 for Linux) ******" echo fi if [ ! -f inputs/pluggable-transports-linux64-gbuilt.zip ]; then echo - echo "****** Starting Pluggable Transports Component of Linux Bundle (4/5 for Linux) ******" + echo "****** Starting Pluggable Transports Component of Linux Bundle (5/6 for Linux) ******" echo - ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit pyptlib=$PYPTLIB_TAG,obfsproxy=$OBFSPROXY_TAG,libfte=$LIBFTE_TAG,fteproxy=$FTEPROXY_TAG,txsocksx=$TXSOCKSX_TAG,goptlib=$GOPTLIB_TAG,meek=$MEEK_TAG,ed25519=$GOED25519_TAG,siphash=$GOSIPHASH_TAG,goxcrypto=$GO_X_CRYPTO_TAG,goxnet=$GO_X_NET_TAG,obfs4=$OBFS4_TAG $DESCRIPTOR_DIR/linux/gitian-pluggable-transports.yml + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit pyptlib=$PYPTLIB_TAG,obfsproxy=$OBFSPROXY_TAG,libfte=$LIBFTE_TAG,fteproxy=$FTEPROXY_TAG,txsocksx=$TXSOCKSX_TAG,goptlib=$GOPTLIB_TAG,meek=$MEEK_TAG,ed25519=$GOED25519_TAG,siphash=$GOSIPHASH_TAG,goxcrypto=$GO_X_CRYPTO_TAG,goxnet=$GO_X_NET_TAG,obfs4=$OBFS4_TAG,go-webrtc=$GO_WEBRTC_TAG,snowflake=$SNOWFLAKE_TAG,uniuri=$UNIURI_TAG $DESCRIPTOR_DIR/linux/gitian-pluggable-transports.yml if [ $? -ne 0 ]; then #mv var/build.log ./pluggable-transports-fail-linux.log.`date +%Y%m%d%H%M%S` @@ -204,14 +226,14 @@ then #cp -a result/pluggable-transports-linux-res.yml inputs/ else echo - echo "****** SKIPPING already built Pluggable Transports Component of Linux Bundle (4/5 for Linux) ******" + echo "****** SKIPPING already built Pluggable Transports Component of Linux Bundle (5/6 for Linux) ******" echo fi if [ ! -f inputs/bundle-linux.gbuilt ]; then echo - echo "****** Starting Bundling+Localization of Linux Bundle (5/5 for Linux) ******" + echo "****** Starting Bundling+Localization of Linux Bundle (6/6 for Linux) ******" echo cd $WRAPPER_DIR && ./record-inputs.sh $VERSIONS_FILE && cd $GITIAN_DIR @@ -231,7 +253,7 @@ then touch inputs/bundle-linux.gbuilt else echo - echo "****** SKIPPING already built Bundling+Localization of Linux Bundle (5/5 for Linux) ******" + echo "****** SKIPPING already built Bundling+Localization of Linux Bundle (6/6 for Linux) ******" echo fi diff --git a/gitian/verify-tags.sh b/gitian/verify-tags.sh index b7c45c4..575901b 100755 --- a/gitian/verify-tags.sh +++ b/gitian/verify-tags.sh @@ -108,6 +108,7 @@ EOF while read dir commit; do check_git_hash "$dir" "$commit" done << EOF +libcxx $LIBCXX_TAG mingw-w64-git $MINGW_TAG libdmg-hfsplus $LIBDMG_TAG libfte $LIBFTE_TAG @@ -120,6 +121,10 @@ goxcrypto $GO_X_CRYPTO_TAG goxnet $GO_X_NET_TAG noto-fonts $NOTOFONTS_TAG selfrando $SELFRANDO_TAG +depot_tools $DEPOT_TOOLS_TAG +go-webrtc $GO_WEBRTC_TAG +snowflake $SNOWFLAKE_TAG +uniuri $UNIURI_TAG EOF # Verify signatures on signed packages @@ -152,6 +157,7 @@ do fi done +# webrtc? Its fetch brings in sources from many repos. cd "$INPUTS_DIR" verify_git "." "$WRAPPER_DIR/gpg/torbutton.gpg" "$GITIAN_TAG" diff --git a/gitian/versions b/gitian/versions index 3b84b63..1b88910 100755 --- a/gitian/versions +++ b/gitian/versions @@ -19,8 +19,12 @@ TORBUTTON_TAG=1.9.3.2 HTTPSE_TAG=5.1.0 NSIS_TAG=v0.3 ZLIB_TAG=v1.2.8 -LIBEVENT_TAG=release-2.0.21-stable -MINGW_TAG=a883b47a45ff74ced41dfbd9f748d5c2c61f3c01 # due to bug 1156131 +LIBEVENT_TAG=release-2.0.22-stable +CMAKE_TAG=v2.8.12.2 +LLVM_TAG=8f188e0ea735ac9383a65a0d1c846eb790c2ec74 # r247539 +CLANG_TAG=592b43b609b42cffd1531a700c140e10766bf049 # r247539 +LIBCXX_TAG=af9a44f256be54de6874d1eefd1d282d0671c0cf # r247539 +MINGW_TAG=a0cd5afeb60be3be0860e9a203314c10485bb9b8 PYPTLIB_TAG=pyptlib-0.0.6 OBFSPROXY_TAG=obfsproxy-0.2.12 FLASHPROXY_TAG=1.6 @@ -37,6 +41,11 @@ GO_X_CRYPTO_TAG=4ed45ec682102c643324fae5dff8dab085b6c300 GO_X_NET_TAG=7dbad50ab5b31073856416cdcfeb2796d682f844 OBFS4_TAG=obfs4proxy-0.0.5 NOTOFONTS_TAG=720e34851382ee3c1ef024d8dffb68ffbfb234c2 +DEPOT_TOOLS_TAG=28216cd14b44716db5c83634afbdc6e90492652b +WEBRTC_TAG=f33698296719f956497d2dbff81b5080864a8804 # https://chromium.googlesource.com/external/webrtc.git/+/refs/branch-heads/52 +GO_WEBRTC_TAG=69c5ca70533142494d145b43ef2f46ca94715557 +SNOWFLAKE_TAG=6cecd31fd896eb26e64ad8bab8a9ea510ec3b21d +UNIURI_TAG=8902c56451e9b58ff940bbe5fec35d5f9c04584a GITIAN_TAG=tor-browser-builder-3.x-8 diff --git a/gitian/versions.alpha b/gitian/versions.alpha index f553468..d9ce0cb 100755 --- a/gitian/versions.alpha +++ b/gitian/versions.alpha @@ -29,6 +29,7 @@ LIBEVENT_TAG=release-2.0.22-stable CMAKE_TAG=v2.8.12.2 LLVM_TAG=8f188e0ea735ac9383a65a0d1c846eb790c2ec74 # r247539 CLANG_TAG=592b43b609b42cffd1531a700c140e10766bf049 # r247539 +LIBCXX_TAG=af9a44f256be54de6874d1eefd1d282d0671c0cf # r247539 MINGW_TAG=a0cd5afeb60be3be0860e9a203314c10485bb9b8 PYPTLIB_TAG=pyptlib-0.0.6 OBFSPROXY_TAG=obfsproxy-0.2.12 @@ -46,6 +47,11 @@ GO_X_NET_TAG=7dbad50ab5b31073856416cdcfeb2796d682f844 OBFS4_TAG=obfs4proxy-0.0.5 NOTOFONTS_TAG=720e34851382ee3c1ef024d8dffb68ffbfb234c2 SELFRANDO_TAG=44ac0df8c0e8eb291f2d913e2dd147000bf54863 +DEPOT_TOOLS_TAG=28216cd14b44716db5c83634afbdc6e90492652b +WEBRTC_TAG=f33698296719f956497d2dbff81b5080864a8804 # https://chromium.googlesource.com/external/webrtc.git/+/refs/branch-heads/52 +GO_WEBRTC_TAG=69c5ca70533142494d145b43ef2f46ca94715557 +SNOWFLAKE_TAG=6cecd31fd896eb26e64ad8bab8a9ea510ec3b21d +UNIURI_TAG=8902c56451e9b58ff940bbe5fec35d5f9c04584a GITIAN_TAG=tor-browser-builder-4-1 diff --git a/gitian/versions.beta b/gitian/versions.beta index 2bcde03..a9c988e 100755 --- a/gitian/versions.beta +++ b/gitian/versions.beta @@ -30,6 +30,11 @@ TXSOCKSX_TAG=216eb0894a1755872f4789f9458aa6cf543b8433 # unsigned habnabit/1.13.0 GOPTLIB_TAG=0.2 MEEK_TAG=0.25 NOTOFONTS_TAG=720e34851382ee3c1ef024d8dffb68ffbfb234c2 +DEPOT_TOOLS_TAG=28216cd14b44716db5c83634afbdc6e90492652b +WEBRTC_TAG=f33698296719f956497d2dbff81b5080864a8804 # https://chromium.googlesource.com/external/webrtc.git/+/refs/branch-heads/52 +GO_WEBRTC_TAG=69c5ca70533142494d145b43ef2f46ca94715557 +SNOWFLAKE_TAG=6cecd31fd896eb26e64ad8bab8a9ea510ec3b21d +UNIURI_TAG=8902c56451e9b58ff940bbe5fec35d5f9c04584a GITIAN_TAG=tor-browser-builder-3.x-6 diff --git a/gitian/versions.nightly b/gitian/versions.nightly index d606efc..ad3f017 100755 --- a/gitian/versions.nightly +++ b/gitian/versions.nightly @@ -32,6 +32,7 @@ LIBEVENT_TAG=release-2.0.22-stable CMAKE_TAG=v2.8.12.2 LLVM_TAG=8f188e0ea735ac9383a65a0d1c846eb790c2ec74 # r247539 CLANG_TAG=592b43b609b42cffd1531a700c140e10766bf049 # r247539 +LIBCXX_TAG=af9a44f256be54de6874d1eefd1d282d0671c0cf # r247539 MINGW_TAG=a0cd5afeb60be3be0860e9a203314c10485bb9b8 PYPTLIB_TAG=master OBFSPROXY_TAG=master @@ -49,6 +50,11 @@ GO_X_NET_TAG=master OBFS4_TAG=master NOTOFONTS_TAG=720e34851382ee3c1ef024d8dffb68ffbfb234c2 SELFRANDO_TAG=44ac0df8c0e8eb291f2d913e2dd147000bf54863 +DEPOT_TOOLS_TAG=master +WEBRTC_TAG=f33698296719f956497d2dbff81b5080864a8804 # https://chromium.googlesource.com/external/webrtc.git/+/refs/branch-heads/52 +GO_WEBRTC_TAG=master +SNOWFLAKE_TAG=master +UNIURI_TAG=master GITIAN_TAG=tor-browser-builder-4

1 0

[tor-browser-bundle/master] Bug 20735: Add snowflake pt to alpha Linux builds
by gk＠torproject.org 05 Jan '17

05 Jan '17

commit 761b0dbabd8fd14e8d9149634b52869f0c68375b Author: David Fifield <david(a)bamsoftware.com> Date: Mon Dec 19 13:10:04 2016 -0800 Bug 20735: Add snowflake pt to alpha Linux builds * Snowflake depends on go-webrtc, which in turn depends on libwebrtc, which adds significantly to the build. webrtc is special, having its own build system that brings in lots of Chromium dependencies. * Further, building cgo reproducibly requires go 1.7.3 and a compiler that supports the following flags, -gno-record-gcc-switches -fdebug-prefix-map=$WORK=/tmp/go-build --- Bundle-Data/PTConfigs/bridge_prefs.js | 2 + .../PTConfigs/linux/torrc-defaults-appendix | 3 + gitian/Makefile | 5 +- gitian/check-prerequisites.sh | 8 +- .../linux/gitian-pluggable-transports.yml | 79 +++++++++++++ gitian/descriptors/linux/gitian-webrtc.yml | 125 +++++++++++++++++++++ gitian/fetch-inputs.sh | 28 +++++ gitian/mkbundle-linux.sh | 48 +++++--- gitian/verify-tags.sh | 6 + gitian/versions | 6 + gitian/versions.alpha | 6 + gitian/versions.beta | 5 + gitian/versions.nightly | 6 + 13 files changed, 309 insertions(+), 18 deletions(-) diff --git a/Bundle-Data/PTConfigs/bridge_prefs.js b/Bundle-Data/PTConfigs/bridge_prefs.js index 6831555..bd92585 100644 --- a/Bundle-Data/PTConfigs/bridge_prefs.js +++ b/Bundle-Data/PTConfigs/bridge_prefs.js @@ -38,3 +38,5 @@ pref("extensions.torlauncher.default_bridge.obfs4.20", "obfs4 85.17.30.79:443 FC pref("extensions.torlauncher.default_bridge.meek-amazon.1", "meek 0.0.2.0:2 B9E7141C594AF25699E0079C1F0146F409495296 url=https://d2zfqthxsdq309.cloudfront.net/ front=a0.awsstatic.com"); pref("extensions.torlauncher.default_bridge.meek-azure.1", "meek 0.0.2.0:3 A2C13B7DFCAB1CBF3A884B6EB99A98067AB6EF44 url=https://az786092.vo.msecnd.net/ front=ajax.aspnetcdn.com"); + +pref("extensions.torlauncher.default_bridge.snowflake.1", "snowflake 0.0.3.0:1 B05EC36590AC0374F0E4D36DDA77908FD59D97F3"); diff --git a/Bundle-Data/PTConfigs/linux/torrc-defaults-appendix b/Bundle-Data/PTConfigs/linux/torrc-defaults-appendix index c79efde..7fbf63c 100644 --- a/Bundle-Data/PTConfigs/linux/torrc-defaults-appendix +++ b/Bundle-Data/PTConfigs/linux/torrc-defaults-appendix @@ -6,3 +6,6 @@ ClientTransportPlugin obfs2,obfs3,obfs4,scramblesuit exec ./TorBrowser/Tor/Plugg ## meek configuration ClientTransportPlugin meek exec ./TorBrowser/Tor/PluggableTransports/meek-client-torbrowser -- ./TorBrowser/Tor/PluggableTransports/meek-client + +## snowflake configuration +ClientTransportPlugin snowflake exec ./TorBrowser/Tor/PluggableTransports/snowflake-client -url https://snowflake-reg.appspot.com/ -front www.google.com -ice stun:stun.l.google.com:19302 diff --git a/gitian/Makefile b/gitian/Makefile index 5694cbf..666dc53 100644 --- a/gitian/Makefile +++ b/gitian/Makefile @@ -101,7 +101,7 @@ prep-alpha: ./check-prerequisites.sh $(TORSOCKS) ./fetch-inputs.sh ../../gitian-builder/inputs/ versions.alpha -clean: clean-utils clean-tor clean-browser clean-pt clean-sandbox clean-bundle +clean: clean-utils clean-tor clean-browser clean-webrtc clean-pt clean-sandbox clean-bundle rm -f ../../gitian-builder/inputs/*.yml rm -f ../../gitian-builder/inputs/bundle.inputs rm -f ../../gitian-builder/inputs/versions* @@ -120,6 +120,9 @@ clean-browser: clean-bundle rm -f ../../gitian-builder/inputs/tor-browser-win* rm -f ../../gitian-builder/inputs/mar-tools-* +clean-webrtc: clean-bundle + rm -f ../../gitian-builder/inputs/webrtc-*-gbuilt.zip + clean-pt: clean-bundle rm -f ../../gitian-builder/inputs/pluggable-transports* diff --git a/gitian/check-prerequisites.sh b/gitian/check-prerequisites.sh index 858fd4d..530e160 100755 --- a/gitian/check-prerequisites.sh +++ b/gitian/check-prerequisites.sh @@ -17,7 +17,7 @@ then VERSION=`cat /etc/issue | grep -Eo '[0-9]{2}' | head -1` if [ "$VERSION" -ge "14" ]; then - dpkg -s ruby apache2 git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion debian-archive-keyring torsocks tor 2>/dev/null >/dev/null + dpkg -s ruby apache2 git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion debian-archive-keyring curl pkg-config libgtk2.0-dev libglib2.0-dev torsocks tor 2>/dev/null >/dev/null if [ $? -ne 0 ]; then @@ -25,7 +25,7 @@ then echo echo "Please run:" echo " sudo apt-get install torsocks tor" - echo " sudo torsocks apt-get install ruby apache2 git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion debian-archive-keyring" + echo " sudo torsocks apt-get install ruby apache2 git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion debian-archive-keyring curl pkg-config libgtk2.0-dev libglib2.0-dev" exit 1 fi else @@ -34,7 +34,7 @@ then fi elif [ $DISTRO = "Debian" ]; then - dpkg -s ruby git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot zip unzip torsocks tor python-cheetah debootstrap parted kpartx rsync 2>/dev/null >/dev/null + dpkg -s ruby git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot zip unzip python-cheetah debootstrap parted kpartx rsync curl pkg-config libgtk2.0-dev libglib2.0-dev torsocks tor 2>/dev/null >/dev/null if [ $? -ne 0 ]; then @@ -42,7 +42,7 @@ then echo echo "Please run" echo " sudo apt-get install torsocks tor" - echo " sudo torsocks apt-get install ruby git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot zip unzip python-cheetah debootstrap parted kpartx rsync" + echo " sudo torsocks apt-get install ruby git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot zip unzip python-cheetah debootstrap parted kpartx rsync curl pkg-config libgtk2.0-dev libglib2.0-dev" exit 1 fi else diff --git a/gitian/descriptors/linux/gitian-pluggable-transports.yml b/gitian/descriptors/linux/gitian-pluggable-transports.yml index c790a9f..ecf49de 100644 --- a/gitian/descriptors/linux/gitian-pluggable-transports.yml +++ b/gitian/descriptors/linux/gitian-pluggable-transports.yml @@ -12,6 +12,8 @@ packages: - "python-dev" - "zip" - "hardening-wrapper" +- "pkg-config" +- "libx11-dev" reference_datetime: "2000-01-01 00:00:00" remotes: - "url": "https://git.torproject.org/pluggable-transports/pyptlib.git" @@ -38,6 +40,12 @@ remotes: "dir": "goxnet" - "url": "https://git.torproject.org/pluggable-transports/obfs4.git" "dir": "obfs4" +- "url": "https://github.com/keroserene/go-webrtc.git" + "dir": "go-webrtc" +- "url": "https://git.torproject.org/pluggable-transports/snowflake.git" + "dir": "snowflake" +- "url": "https://github.com/dchest/uniuri.git" + "dir": "uniuri" files: - "pycrypto.tar.gz" - "argparse.tar.gz" @@ -52,6 +60,12 @@ files: - "openssl-linux64-utils.zip" - "go-linux32-utils.zip" - "go-linux64-utils.zip" +- "binutils-linux32-utils.zip" +- "binutils-linux64-utils.zip" +- "gcc-linux32-utils.zip" +- "gcc-linux64-utils.zip" +- "webrtc-linux32-gbuilt.zip" +- "webrtc-linux64-gbuilt.zip" script: | INSTDIR="$HOME/install" PTDIR="$INSTDIR/Tor/PluggableTransports" @@ -78,6 +92,35 @@ script: | # FTE only needs libgmp.so.10 and no libgmpxx anymore. cp $INSTDIR/gmp/lib/libgmp.so.10 $INSTDIR/Tor + # Preparing Binutils and GCC for webrtc + unzip -d $INSTDIR binutils-linux$GBUILD_BITS-utils.zip + # Make sure gold is used with the hardening wrapper for full RELRO, see + # #13031. + cd $INSTDIR/binutils/bin + rm ld + cp /usr/bin/hardened-ld ./ + mv ld.gold ld.gold.real + ln -sf hardened-ld ld.gold + ln -sf ld.gold ld + cd ~/build + unzip -d $INSTDIR gcc-linux$GBUILD_BITS-utils.zip + # Make sure we use the hardening wrapper when compiling Tor Browser. + cd $INSTDIR/gcc/bin + cp /usr/bin/hardened-cc ./ + mv gcc gcc.real + mv c++ c++.real + ln -sf hardened-cc gcc + ln -sf hardened-cc c++ + cd ~/build + export PATH=$INSTDIR/gcc/bin:$INSTDIR/binutils/bin:$PATH + ARCH="" + if [ $GBUILD_BITS == "64" ]; + then + ARCH="64" + fi + # Make sure our GCC gets properly used. + export LD_LIBRARY_PATH=$INSTDIR/gcc/lib$ARCH + # Building pyptlib cd pyptlib find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" @@ -216,6 +259,14 @@ script: | go install github.com/dchest/siphash cd .. + # Building go uniuri + cd uniuri + find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" + mkdir -p "$GOPATH/src/github.com/dchest/" + ln -sf "$PWD" "$GOPATH/src/github.com/dchest/uniuri" + go install github.com/dchest/uniuri + cd .. + # Building golang.org/x/crypto (obfs4proxy > 0.0.3 || Go >= 1.4) cd goxcrypto find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" @@ -244,6 +295,34 @@ script: | cp -a obfs4proxy $PTDIR cd ../.. + unzip webrtc-linux$GBUILD_BITS-gbuilt.zip + + # Building go-webrtc + cd go-webrtc + # Replace the prebuilt webrtc library with our own one. + rm -rf include/ lib/ + ln -sf ../webrtc/{include,lib} . + find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" + mkdir -p "$GOPATH/src/github.com/keroserene/" + ln -sf "$PWD" "$GOPATH/src/github.com/keroserene/go-webrtc" + # The go-webrtc package sets _GLIBCXX_USE_CXX11_ABI=0 for compatibility with + # the prebuilt libwebrtc-magic.a libraries. Since we build our own + # libwebrtc-magic.a with the C++11 ABI, we have to undo that setting here, + # using CGO_CXXFLAGS. + CGO_CXXFLAGS=-D_GLIBCXX_USE_CXX11_ABI=1 go install github.com/keroserene/go-webrtc + cd .. + + # Building snowflake + cd snowflake + find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" + cd client + go build -ldflags '-s' + cp -a client $PTDIR/snowflake-client + cd .. + mkdir -p $INSTDIR/Docs/snowflake + cp -a README.md LICENSE $INSTDIR/Docs/snowflake + cd .. + # Grabbing the results and making sure timestamps don't spoil them cd $INSTDIR ~/build/dzip.sh pluggable-transports-linux$GBUILD_BITS-gbuilt.zip Tor/ Docs/ diff --git a/gitian/descriptors/linux/gitian-webrtc.yml b/gitian/descriptors/linux/gitian-webrtc.yml new file mode 100644 index 0000000..ba654c5 --- /dev/null +++ b/gitian/descriptors/linux/gitian-webrtc.yml @@ -0,0 +1,125 @@ +--- +name: "webrtc-linux" +distro: "debian" +suites: +- "wheezy" +architectures: +- "i386" +- "amd64" +packages: +- "unzip" +- "zip" +- "hardening-wrapper" +- "libasound2-dev" +- "libexpat1-dev" +- "libudev-dev" +- "libx11-dev" +- "libxext-dev" +- "libxrandr-dev" +- "default-jre-headless" +- "default-jdk" +reference_datetime: "2000-01-01 00:00:00" +remotes: +- "url": "https://chromium.googlesource.com/chromium/tools/depot_tools.git" + "dir": "depot_tools" +files: +- "dzip.sh" +- "binutils-linux32-utils.zip" +- "binutils-linux64-utils.zip" +- "gcc-linux32-utils.zip" +- "gcc-linux64-utils.zip" +- "webrtc.tar.gz" +script: | + INSTDIR="$HOME/install" + export REFERENCE_DATETIME + export TZ=UTC + export LC_ALL=C + umask 0022 + # + mkdir -p $OUTDIR/ + + # Config options for hardening-wrapper + export DEB_BUILD_HARDENING=1 + export DEB_BUILD_HARDENING_STACKPROTECTOR=1 + export DEB_BUILD_HARDENING_FORTIFY=1 + export DEB_BUILD_HARDENING_FORMAT=1 + export DEB_BUILD_HARDENING_PIE=1 + + # Preparing Binutils and GCC for webrtc + unzip -d $INSTDIR binutils-linux$GBUILD_BITS-utils.zip + # Make sure gold is used with the hardening wrapper for full RELRO, see + # #13031. + cd $INSTDIR/binutils/bin + rm ld + cp /usr/bin/hardened-ld ./ + mv ld.gold ld.gold.real + ln -sf hardened-ld ld.gold + ln -sf ld.gold ld + cd ~/build + unzip -d $INSTDIR gcc-linux$GBUILD_BITS-utils.zip + # Make sure we use the hardening wrapper when compiling Tor Browser. + cd $INSTDIR/gcc/bin + cp /usr/bin/hardened-cc ./ + mv gcc gcc.real + mv c++ c++.real + ln -sf hardened-cc gcc + ln -sf hardened-cc c++ + cd ~/build + export PATH=$INSTDIR/gcc/bin:$INSTDIR/binutils/bin:$PATH + ARCH="" + if [ $GBUILD_BITS == "64" ]; + then + ARCH="64" + fi + # Make sure our GCC gets properly used. + export LD_LIBRARY_PATH=$INSTDIR/gcc/lib$ARCH + + # Setting up depot_tools + # https://dev.chromium.org/developers/how-tos/install-depot-tools + export PATH="$PATH:$PWD/depot_tools" + # Disable automatic updating. + export DEPOT_TOOLS_UPDATE=0 + + # Building webrtc + tar xaf webrtc.tar.gz + cd webrtc/src + export GYP_CROSSCOMPILE=1 + export GYP_DEFINES="OS=linux target_arch=$(if [ $GBUILD_BITS = 32 ]; then echo ia32; else echo x64; fi)" + # Do not use bundled utilities. + GYP_DEFINES+=" clang=0 host_clang=0 use_sysroot=0" + GYP_DEFINES+=" linux_use_bundled_gold=0 linux_use_gold_flags=0 linux_use_bundled_binutils=0" + # Avoid some dependencies. Some of these don't seem to work fully; for + # example even with use_alsa=0 we need libasound2-dev. + GYP_DEFINES+=" use_x11=0 use_gtk=0 use_gconf=0 use_alsa=0 use_pulseaudio=0 use_libpci=0 include_internal_audio_device=0 enabled_libjingle_device_manager=0" + # examples and tests also bring in dependencies. + GYP_DEFINES+=" include_examples=0 include_tests=0" + # embedded=1 is supposed to turn of various features; see + # https://bugs.chromium.org/p/chromium/issues/detail?id=318413. + GYP_DEFINES+=" embedded=1" + GYP_DEFINES+=" werror=" + JAVA_HOME=/usr/lib/jvm/default-java webrtc/build/gyp_webrtc.py + ninja -C out/Release + # Run ninja once more, without include_tests=0, in order to build just the + # FakeAudioCaptureModule that go-webrtc uses. + GYP_DEFINES="$(echo "$GYP_DEFINES" | sed -e 's/include_tests=0//g')" + JAVA_HOME=/usr/lib/jvm/default-java webrtc/build/gyp_webrtc.py + ninja -C out/Release obj/webrtc/api/test/peerconnection_unittests.fakeaudiocapturemodule.o + # https://github.com/keroserene/go-webrtc/issues/23#issuecomment-175312648 + # dump_syms_regtest.o is actually an executable, not an object file. If not + # excluded, it results in the error: + # libwebrtc-linux-386-magic.a(dump_syms_regtest.o): unsupported ELF file type 2 + ar crs libwebrtc-magic.a $(find . -name '*.o' -not -name '*.main.o' -not -name 'dump_syms_regtest.o' | sort) + cd ../.. + + # Grabbing the result + cd $INSTDIR + mkdir -p webrtc/include webrtc/lib + cp -f $HOME/build/webrtc/src/libwebrtc-magic.a webrtc/lib/libwebrtc-linux-$(if [ $GBUILD_BITS = 32 ]; then echo 386; else echo amd64; fi)-magic.a + INCLUDE_DIR="$PWD/webrtc/include" + (cd $HOME/build/webrtc/src && for h in $(find talk/ webrtc/ -type f -name '*.h'); do + mkdir -p "$INCLUDE_DIR/$(dirname $h)" + cp -f "$h" "$INCLUDE_DIR/$h" + done) + + ~/build/dzip.sh webrtc-linux$GBUILD_BITS-gbuilt.zip webrtc + cp webrtc-linux$GBUILD_BITS-gbuilt.zip $OUTDIR/ diff --git a/gitian/fetch-inputs.sh b/gitian/fetch-inputs.sh index b3d5c26..729e492 100755 --- a/gitian/fetch-inputs.sh +++ b/gitian/fetch-inputs.sh @@ -277,6 +277,7 @@ tor-browser https://git.torproject.org/tor-browser.git $TORBR cmake https://cmake.org/cmake.git $CMAKE_TAG llvm https://github.com/llvm-mirror/llvm $LLVM_TAG clang https://github.com/llvm-mirror/clang $CLANG_TAG +libcxx https://github.com/llvm-mirror/libcxx $LIBCXX_TAG mingw-w64-git http://git.code.sf.net/p/mingw-w64/mingw-w64 $MINGW_TAG pyptlib https://git.torproject.org/pluggable-transports/pyptlib.git $PYPTLIB_TAG obfsproxy https://git.torproject.org/pluggable-transports/obfsproxy.git $OBFSPROXY_TAG @@ -296,6 +297,10 @@ noto-fonts https://github.com/googlei18n/noto-fonts $NOTOFONTS_TAG errors https://github.com/pkg/errors $ERRORS_TAG gb https://github.com/constabulary/gb $GB_TAG sandbox https://git.schwanenlied.me/yawning/sandboxed-tor-browser $SANDBOX_TAG +depot_tools https://chromium.googlesource.com/chromium/tools/depot_tools.git $DEPOT_TOOLS_TAG +go-webrtc https://github.com/keroserene/go-webrtc $GO_WEBRTC_TAG +snowflake https://git.torproject.org/pluggable-transports/snowflake.git $SNOWFLAKE_TAG +uniuri https://github.com/dchest/uniuri $UNIURI_TAG EOF # HTTPS-Everywhere is special, too. We need to initialize the git submodules and @@ -305,5 +310,28 @@ git submodule init git submodule update cd .. +# WebRTC is special, having its own build system that brings in lots of Chromium dependencies. +# https://webrtc.org/native-code/development/ +# depot_tools must have been downloaded before running this code. +dir=webrtc +PATH="$PATH:$PWD/depot_tools" +# GYP_CROSSCOMPILE=1 and GYP_DEFINES="use_x11=0" prevent probing for certain dependencies. +# Use --no-history because the whole checkout with history is about 12 GB. +export GYP_CROSSCOMPILE=1 +export GYP_DEFINES="use_x11=0" +mkdir -p "$dir" +cd "$dir" +if [ ! -d "src" ]; +then + # "fetch" is part of depot_tools. + fetch --nohooks --no-history webrtc +fi +# "gclient" is part of depot_tools. This download takes a long time the first time. +# JAVA_HOME is needed in a hook for libjingle. The readlink line tries to find the current JRE. +# default-java comes from the package default-jdk-headless. +JAVA_HOME=/usr/lib/jvm/default-java gclient sync --with_branch_heads --no-history -r $WEBRTC_TAG +cd .. +tar --exclude .git -czf webrtc.tar.gz webrtc + exit 0 diff --git a/gitian/mkbundle-linux.sh b/gitian/mkbundle-linux.sh index 27cccd0..f1da2e0 100755 --- a/gitian/mkbundle-linux.sh +++ b/gitian/mkbundle-linux.sh @@ -114,7 +114,7 @@ if [ ! -f inputs/binutils-$BINUTILS_VER-linux32-utils.zip -o \ ! -f inputs/go-$GO_VER-linux64-utils.zip ]; then echo - echo "****** Starting Utilities Component of Linux Bundle (1/6 for Linux) ******" + echo "****** Starting Utilities Component of Linux Bundle (1/7 for Linux) ******" echo ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit libevent=$LIBEVENT_TAG $DESCRIPTOR_DIR/linux/gitian-utils.yml @@ -142,7 +142,7 @@ then #cp -a result/utils-linux-res.yml inputs/ else echo - echo "****** SKIPPING already built Utilities Component of Linux Bundle (1/6 for Linux) ******" + echo "****** SKIPPING already built Utilities Component of Linux Bundle (1/7 for Linux) ******" echo # We might have built the utilities in the past but maybe the links are # pointing to the wrong version. Refresh them. @@ -166,7 +166,7 @@ if [ ! -f inputs/tor-linux32-gbuilt.zip -o \ ! -f inputs/tor-linux64-gbuilt.zip ]; then echo - echo "****** Starting Tor Component of Linux Bundle (2/6 for Linux) ******" + echo "****** Starting Tor Component of Linux Bundle (2/7 for Linux) ******" echo ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor=$TOR_TAG $DESCRIPTOR_DIR/linux/gitian-tor.yml @@ -181,7 +181,7 @@ then #cp -a result/tor-linux-res.yml inputs/ else echo - echo "****** SKIPPING already built Tor Component of Linux Bundle (2/6 for Linux) ******" + echo "****** SKIPPING already built Tor Component of Linux Bundle (2/7 for Linux) ******" echo fi @@ -190,7 +190,7 @@ if [ ! -f inputs/tor-browser-linux32-gbuilt.zip -o \ ! -f inputs/tor-browser-linux64-gbuilt.zip ]; then echo - echo "****** Starting TorBrowser Component of Linux Bundle (3/6 for Linux) ******" + echo "****** Starting TorBrowser Component of Linux Bundle (3/7 for Linux) ******" echo ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor-browser=$TORBROWSER_TAG,faketime=$FAKETIME_TAG $DESCRIPTOR_DIR/linux/gitian-firefox.yml @@ -206,7 +206,29 @@ then #cp -a result/torbrowser-linux-res.yml inputs/ else echo - echo "****** SKIPPING already built TorBrowser Component of Linux Bundle (3/6 for Linux) ******" + echo "****** SKIPPING already built TorBrowser Component of Linux Bundle (3/7 for Linux) ******" + echo +fi + +if [ ! -f inputs/webrtc-linux32-gbuilt.zip -o \ + ! -f inputs/webrtc-linux64-gbuilt.zip ]; +then + echo + echo "****** Starting WebRTC Component of Linux Bundle (4/7 for Linux) ******" + echo + + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit depot_tools=$DEPOT_TOOLS_TAG $DESCRIPTOR_DIR/linux/gitian-webrtc.yml + if [ $? -ne 0 ]; + then + #mv var/build.log ./webrtc-fail-linux.log.`date +%Y%m%d%H%M%S` + exit 1 + fi + + cp -a build/out/webrtc-linux*-gbuilt.zip inputs/ + #cp -a result/webrtc-linux-res.yml inputs/ +else + echo + echo "****** SKIPPING already built WebRTC Component of Linux Bundle (4/7 for Linux) ******" echo fi @@ -214,10 +236,10 @@ if [ ! -f inputs/pluggable-transports-linux32-gbuilt.zip -o \ ! -f inputs/pluggable-transports-linux64-gbuilt.zip ]; then echo - echo "****** Starting Pluggable Transports Component of Linux Bundle (4/6 for Linux) ******" + echo "****** Starting Pluggable Transports Component of Linux Bundle (5/7 for Linux) ******" echo - ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit pyptlib=$PYPTLIB_TAG,obfsproxy=$OBFSPROXY_TAG,libfte=$LIBFTE_TAG,fteproxy=$FTEPROXY_TAG,txsocksx=$TXSOCKSX_TAG,goptlib=$GOPTLIB_TAG,meek=$MEEK_TAG,ed25519=$GOED25519_TAG,siphash=$GOSIPHASH_TAG,goxcrypto=$GO_X_CRYPTO_TAG,goxnet=$GO_X_NET_TAG,obfs4=$OBFS4_TAG $DESCRIPTOR_DIR/linux/gitian-pluggable-transports.yml + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit pyptlib=$PYPTLIB_TAG,obfsproxy=$OBFSPROXY_TAG,libfte=$LIBFTE_TAG,fteproxy=$FTEPROXY_TAG,txsocksx=$TXSOCKSX_TAG,goptlib=$GOPTLIB_TAG,meek=$MEEK_TAG,ed25519=$GOED25519_TAG,siphash=$GOSIPHASH_TAG,goxcrypto=$GO_X_CRYPTO_TAG,goxnet=$GO_X_NET_TAG,obfs4=$OBFS4_TAG,go-webrtc=$GO_WEBRTC_TAG,snowflake=$SNOWFLAKE_TAG,uniuri=$UNIURI_TAG $DESCRIPTOR_DIR/linux/gitian-pluggable-transports.yml if [ $? -ne 0 ]; then #mv var/build.log ./pluggable-transports-fail-linux.log.`date +%Y%m%d%H%M%S` @@ -228,7 +250,7 @@ then #cp -a result/pluggable-transports-linux-res.yml inputs/ else echo - echo "****** SKIPPING already built Pluggable Transports Component of Linux Bundle (4/6 for Linux) ******" + echo "****** SKIPPING already built Pluggable Transports Component of Linux Bundle (5/7 for Linux) ******" echo fi @@ -243,7 +265,7 @@ if [ ! -f inputs/sandbox-$SB_TAG-linux32.zip -o \ ! -f inputs/sandbox-$SB_TAG-linux64.zip ]; then echo - echo "****** Starting Sandbox Component of Linux Bundle (5/6 for Linux) ******" + echo "****** Starting Sandbox Component of Linux Bundle (6/7 for Linux) ******" echo ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit errors=$ERRORS_TAG,gb=$GB_TAG,sandbox=$SANDBOX_TAG $DESCRIPTOR_DIR/linux/gitian-sandbox.yml @@ -256,14 +278,14 @@ then #cp -a result/sandbox-linux-res.yml inputs/ else echo - echo "****** SKIPPING already built Sandbox Component of Linux Bundle (5/6 for Linux) ******" + echo "****** SKIPPING already built Sandbox Component of Linux Bundle (6/7 for Linux) ******" echo fi if [ ! -f inputs/bundle-linux.gbuilt ]; then echo - echo "****** Starting Bundling+Localization of Linux Bundle (6/6 for Linux) ******" + echo "****** Starting Bundling+Localization of Linux Bundle (7/7 for Linux) ******" echo cd $WRAPPER_DIR && ./record-inputs.sh $VERSIONS_FILE && cd $GITIAN_DIR @@ -284,7 +306,7 @@ then touch inputs/bundle-linux.gbuilt else echo - echo "****** SKIPPING already built Bundling+Localization of Linux Bundle (6/6 for Linux) ******" + echo "****** SKIPPING already built Bundling+Localization of Linux Bundle (7/7 for Linux) ******" echo fi diff --git a/gitian/verify-tags.sh b/gitian/verify-tags.sh index d95b0e0..368a990 100755 --- a/gitian/verify-tags.sh +++ b/gitian/verify-tags.sh @@ -111,6 +111,7 @@ while read dir commit; do done << EOF llvm $LLVM_TAG clang $CLANG_TAG +libcxx $LIBCXX_TAG mingw-w64-git $MINGW_TAG libdmg-hfsplus $LIBDMG_TAG libfte $LIBFTE_TAG @@ -122,6 +123,10 @@ siphash $GOSIPHASH_TAG goxcrypto $GO_X_CRYPTO_TAG goxnet $GO_X_NET_TAG noto-fonts $NOTOFONTS_TAG +depot_tools $DEPOT_TOOLS_TAG +go-webrtc $GO_WEBRTC_TAG +snowflake $SNOWFLAKE_TAG +uniuri $UNIURI_TAG EOF # Verify signatures on signed packages @@ -154,6 +159,7 @@ do fi done +# webrtc? Its fetch brings in sources from many repos. cd "$INPUTS_DIR" verify_git "." "$WRAPPER_DIR/gpg/torbutton.gpg" "$GITIAN_TAG" diff --git a/gitian/versions b/gitian/versions index 81963bb..598a945 100755 --- a/gitian/versions +++ b/gitian/versions @@ -25,6 +25,7 @@ LIBEVENT_TAG=release-2.0.22-stable CMAKE_TAG=v2.8.12.2 LLVM_TAG=8f188e0ea735ac9383a65a0d1c846eb790c2ec74 # r247539 CLANG_TAG=592b43b609b42cffd1531a700c140e10766bf049 # r247539 +LIBCXX_TAG=af9a44f256be54de6874d1eefd1d282d0671c0cf # r247539 MINGW_TAG=a0cd5afeb60be3be0860e9a203314c10485bb9b8 PYPTLIB_TAG=pyptlib-0.0.6 OBFSPROXY_TAG=obfsproxy-0.2.12 @@ -41,6 +42,11 @@ GO_X_CRYPTO_TAG=4ed45ec682102c643324fae5dff8dab085b6c300 GO_X_NET_TAG=7dbad50ab5b31073856416cdcfeb2796d682f844 OBFS4_TAG=obfs4proxy-0.0.5 NOTOFONTS_TAG=720e34851382ee3c1ef024d8dffb68ffbfb234c2 +DEPOT_TOOLS_TAG=28216cd14b44716db5c83634afbdc6e90492652b +WEBRTC_TAG=f33698296719f956497d2dbff81b5080864a8804 # https://chromium.googlesource.com/external/webrtc.git/+/refs/branch-heads/52 +GO_WEBRTC_TAG=69c5ca70533142494d145b43ef2f46ca94715557 +SNOWFLAKE_TAG=6cecd31fd896eb26e64ad8bab8a9ea510ec3b21d +UNIURI_TAG=8902c56451e9b58ff940bbe5fec35d5f9c04584a GITIAN_TAG=tor-browser-builder-4-1 diff --git a/gitian/versions.alpha b/gitian/versions.alpha index a8a08c1..14ed662 100755 --- a/gitian/versions.alpha +++ b/gitian/versions.alpha @@ -25,6 +25,7 @@ LIBEVENT_TAG=release-2.0.22-stable CMAKE_TAG=v2.8.12.2 LLVM_TAG=8f188e0ea735ac9383a65a0d1c846eb790c2ec74 # r247539 CLANG_TAG=592b43b609b42cffd1531a700c140e10766bf049 # r247539 +LIBCXX_TAG=af9a44f256be54de6874d1eefd1d282d0671c0cf # r247539 MINGW_TAG=a0cd5afeb60be3be0860e9a203314c10485bb9b8 PYPTLIB_TAG=pyptlib-0.0.6 OBFSPROXY_TAG=obfsproxy-0.2.12 @@ -44,6 +45,11 @@ NOTOFONTS_TAG=720e34851382ee3c1ef024d8dffb68ffbfb234c2 ERRORS_TAG=248dadf4e9068a0b3e79f02ed0a610d935de5302 GB_TAG=06cc925cce6592e922dcc4839a8b44feb384e71e SANDBOX_TAG=sandboxed-tor-browser-0.0.2 +DEPOT_TOOLS_TAG=28216cd14b44716db5c83634afbdc6e90492652b +WEBRTC_TAG=f33698296719f956497d2dbff81b5080864a8804 # https://chromium.googlesource.com/external/webrtc.git/+/refs/branch-heads/52 +GO_WEBRTC_TAG=69c5ca70533142494d145b43ef2f46ca94715557 +SNOWFLAKE_TAG=6cecd31fd896eb26e64ad8bab8a9ea510ec3b21d +UNIURI_TAG=8902c56451e9b58ff940bbe5fec35d5f9c04584a GITIAN_TAG=tor-browser-builder-4-2 diff --git a/gitian/versions.beta b/gitian/versions.beta index 577b802..b1342b9 100755 --- a/gitian/versions.beta +++ b/gitian/versions.beta @@ -29,6 +29,11 @@ TXSOCKSX_TAG=216eb0894a1755872f4789f9458aa6cf543b8433 # unsigned habnabit/1.13.0 GOPTLIB_TAG=0.2 MEEK_TAG=0.25 NOTOFONTS_TAG=720e34851382ee3c1ef024d8dffb68ffbfb234c2 +DEPOT_TOOLS_TAG=28216cd14b44716db5c83634afbdc6e90492652b +WEBRTC_TAG=f33698296719f956497d2dbff81b5080864a8804 # https://chromium.googlesource.com/external/webrtc.git/+/refs/branch-heads/52 +GO_WEBRTC_TAG=69c5ca70533142494d145b43ef2f46ca94715557 +SNOWFLAKE_TAG=6cecd31fd896eb26e64ad8bab8a9ea510ec3b21d +UNIURI_TAG=8902c56451e9b58ff940bbe5fec35d5f9c04584a GITIAN_TAG=tor-browser-builder-3.x-6 diff --git a/gitian/versions.nightly b/gitian/versions.nightly index c661b0e..1b1b346 100755 --- a/gitian/versions.nightly +++ b/gitian/versions.nightly @@ -32,6 +32,7 @@ LIBEVENT_TAG=release-2.0.22-stable CMAKE_TAG=v2.8.12.2 LLVM_TAG=8f188e0ea735ac9383a65a0d1c846eb790c2ec74 # r247539 CLANG_TAG=592b43b609b42cffd1531a700c140e10766bf049 # r247539 +LIBCXX_TAG=af9a44f256be54de6874d1eefd1d282d0671c0cf # r247539 MINGW_TAG=a0cd5afeb60be3be0860e9a203314c10485bb9b8 PYPTLIB_TAG=master OBFSPROXY_TAG=master @@ -51,6 +52,11 @@ NOTOFONTS_TAG=720e34851382ee3c1ef024d8dffb68ffbfb234c2 ERRORS_TAG=248dadf4e9068a0b3e79f02ed0a610d935de5302 GB_TAG=06cc925cce6592e922dcc4839a8b44feb384e71e SANDBOX_TAG=master +DEPOT_TOOLS_TAG=master +WEBRTC_TAG=f33698296719f956497d2dbff81b5080864a8804 # https://chromium.googlesource.com/external/webrtc.git/+/refs/branch-heads/52 +GO_WEBRTC_TAG=master +SNOWFLAKE_TAG=master +UNIURI_TAG=master GITIAN_TAG=tor-browser-builder-4

1 0

[metrics-lib/master] Log more, throw fewer RuntimeExceptions.
by karsten＠torproject.org 05 Jan '17

05 Jan '17

commit 9a6198305599d86e8f76d6b3f5bd6cdb8e15925c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jan 3 16:15:34 2017 +0100 Log more, throw fewer RuntimeExceptions. --- .../descriptor/index/DescriptorIndexCollector.java | 65 +++++++++++++++++----- .../org/torproject/descriptor/index/FileNode.java | 6 +- .../index/DescriptorIndexCollectorTest.java | 19 ++----- 3 files changed, 59 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java b/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java index d32f811..9b269fe 100644 --- a/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java +++ b/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java @@ -43,14 +43,23 @@ public class DescriptorIndexCollector implements DescriptorCollector { public void collectDescriptors(String collecTorIndexUrlString, String[] remoteDirectories, long minLastModified, File localDirectory, boolean deleteExtraneousLocalFiles) { + log.info("Starting descriptor collection."); if (minLastModified < 0) { throw new IllegalArgumentException("A negative minimum " + "last-modified time is not permitted."); } - if (null == localDirectory || localDirectory.isFile()) { - throw new IllegalArgumentException("Local directory already exists " - + "and is not a directory."); + if (null == collecTorIndexUrlString || null == remoteDirectories + || null == localDirectory) { + throw new IllegalArgumentException("Null values are not permitted for " + + "CollecTor base URL, remote directories, or local directory."); } + if (localDirectory.isFile()) { + throw new IllegalArgumentException("A non-directory file exists at {} " + + "which is supposed to be the local directory for storing remotely " + + "fetched files. Move this file away or delete it. Aborting " + + "descriptor collection."); + } + log.info("Indexing local directory {}.", localDirectory.getAbsolutePath()); SortedMap<String, Long> localFiles = statLocalDirectory(localDirectory); SortedMap<String, FileNode> remoteFiles = null; IndexNode index = null; @@ -61,20 +70,29 @@ public class DescriptorIndexCollector implements DescriptorCollector { if (indexUrl.getPath().isEmpty()) { indexUrlString += "/index/index.json"; } + log.info("Fetching remote index file {}.", indexUrlString); index = IndexNode.fetchIndex(indexUrlString); remoteFiles = index.retrieveFilesIn(remoteDirectories); } catch (Exception ex) { - throw new RuntimeException("Cannot fetch index from '" - + indexUrlString + "'.", ex); + log.warn("Cannot fetch index file {} and hence cannot determine which " + + "remote files to fetch. Aborting descriptor collection.", + indexUrlString, ex); + return; + } + log.info("Fetching remote files from {}.", index.path); + if (!this.fetchRemoteFiles(index.path, remoteFiles, minLastModified, + localDirectory, localFiles)) { + return; } - this.fetchRemoteFiles(index.path, remoteFiles, minLastModified, - localDirectory, localFiles); if (deleteExtraneousLocalFiles) { - this.deleteExtraneousLocalFiles(remoteFiles, localDirectory, localFiles); + log.info("Deleting extraneous files from local directory {}.", + localDirectory); + deleteExtraneousLocalFiles(remoteFiles, localDirectory, localFiles); } + log.info("Finished descriptor collection."); } - void fetchRemoteFiles(String baseUrl, SortedMap<String, FileNode> remotes, + boolean fetchRemoteFiles(String baseUrl, SortedMap<String, FileNode> remotes, long minLastModified, File localDir, SortedMap<String, Long> locals) { for (Map.Entry<String, FileNode> entry : remotes.entrySet()) { String filepathname = entry.getKey(); @@ -88,10 +106,17 @@ public class DescriptorIndexCollector implements DescriptorCollector { continue; } if (!filepath.exists() && !filepath.mkdirs()) { - throw new RuntimeException("Cannot create dir: " + filepath); + log.warn("Cannot create local directory {} to store remote file {}. " + + "Aborting descriptor collection.", filepath, filename); + return false; } File destinationFile = new File(filepath, filename); File tempDestinationFile = new File(filepath, "." + filename); + log.debug("Fetching remote file {} with expected size of {} bytes from " + + "{}, storing locally to temporary file {}, then renaming to {}.", + filepathname, entry.getValue().size, baseUrl, + tempDestinationFile.getAbsolutePath(), + destinationFile.getAbsolutePath()); try (InputStream is = new URL(baseUrl + "/" + filepathname) .openStream()) { Files.copy(is, tempDestinationFile.toPath()); @@ -99,13 +124,18 @@ public class DescriptorIndexCollector implements DescriptorCollector { tempDestinationFile.renameTo(destinationFile); destinationFile.setLastModified(lastModifiedMillis); } else { - log.warn("File sizes don't match. Expected {}, but received {}.", - entry.getValue().size, tempDestinationFile.length()); + log.warn("Fetched remote file {} from {} has a size of {} bytes " + + "which is different from the expected {} bytes. Not storing " + + "this file.", + filename, baseUrl, tempDestinationFile.length(), + entry.getValue().size); } } catch (IOException e) { - log.error("Cannot fetch remote file: {}", filename, e); + log.warn("Cannot fetch remote file {} from {}. Skipping that file.", + filename, baseUrl, e); } } + return true; } static void deleteExtraneousLocalFiles( @@ -113,7 +143,10 @@ public class DescriptorIndexCollector implements DescriptorCollector { File localDir, SortedMap<String, Long> locals) { for (String localPath : locals.keySet()) { if (!remoteFiles.containsKey(localPath)) { - new File(localDir, localPath).delete(); + File extraneousLocalFile = new File(localDir, localPath); + log.debug("Deleting extraneous local file {}.", + extraneousLocalFile.getAbsolutePath()); + extraneousLocalFile.delete(); } } } @@ -137,7 +170,9 @@ public class DescriptorIndexCollector implements DescriptorCollector { } }); } catch (IOException ioe) { - log.error("Cannot stat local directory.", ioe); + log.warn("Cannot index local directory {} to skip any remote files that " + + "already exist locally. Continuing with an either empty or " + + "incomplete index of local files.", ioe); } return locals; } diff --git a/src/main/java/org/torproject/descriptor/index/FileNode.java b/src/main/java/org/torproject/descriptor/index/FileNode.java index 02e82da..6af5aa3 100644 --- a/src/main/java/org/torproject/descriptor/index/FileNode.java +++ b/src/main/java/org/torproject/descriptor/index/FileNode.java @@ -67,8 +67,10 @@ public class FileNode implements Comparable<FileNode> { try { lastModifiedMillis = dateTimeFormat.parse(this.lastModified).getTime(); } catch (ParseException ex) { - log.warn("Cannot parse date-time. Setting lastModifiedMillis to -1L.", - ex); + log.warn("Cannot parse last-modified time {} of remote file entry {}. " + + "Fetching remote file regardless of configured last-modified " + + "time. The following error message provides more details.", + this.lastModified, this.path, ex); this.lastModifiedMillis = -1L; } } diff --git a/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java b/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java index c5a6a5c..8886c85 100644 --- a/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java +++ b/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java @@ -210,11 +210,12 @@ public class DescriptorIndexCollectorTest { assertTrue("found " + res, res.isEmpty()); } - @Test(expected = RuntimeException.class) + @Test() public void testMinimalArgs() throws IOException { File fakeDir = tmpf.newFolder("fantasy-dir"); new DescriptorIndexCollector() - .collectDescriptors(null, new String[]{}, 100L, fakeDir, true); + .collectDescriptors("http://localhost", new String[]{}, 100L, fakeDir, + true); } @Test(expected = IllegalArgumentException.class) @@ -227,7 +228,7 @@ public class DescriptorIndexCollectorTest { public void testIllegalDirectory() throws IOException { File fakeDir = tmpf.newFile("fantasy-dir"); new DescriptorIndexCollector().collectDescriptors( - null, new String[]{}, 100L, fakeDir, false); + "", new String[]{}, 100L, fakeDir, false); } @Test(expected = IllegalArgumentException.class) @@ -236,24 +237,14 @@ public class DescriptorIndexCollectorTest { null, new String[]{}, 100L, null, false); } - @Test(expected = IllegalArgumentException.class) - public void testExistingFile() throws IOException { - File fakeDir = tmpf.newFile("fantasy-dir"); - new DescriptorIndexCollector() - .collectDescriptors(null, null, 100L, fakeDir, false); - } - @Test() public void testExistingDir() throws IOException { File dir = tmpf.newFolder(); dir.setWritable(false); SortedMap<String, FileNode> fm = new TreeMap<>(); fm.put("readonly", new FileNode("w", 2L, "2100-01-01 01:01")); - thrown.expect(RuntimeException.class); - thrown.expectMessage("Cannot create dir: " + dir.toString() - + "/readonly"); new DescriptorIndexCollector() - .fetchRemoteFiles(null, fm, 100L, dir, new TreeMap<String, Long>()); + .fetchRemoteFiles("", fm, 100L, dir, new TreeMap<String, Long>()); } }

1 0

[metrics-base/master] squash! Corrected source-jar contents, enabled additonal target execution before signing using a new property, and cleaned up jar-task.
by karsten＠torproject.org 05 Jan '17

05 Jan '17

commit fe608f854d0d8e2ca2f6e1dde6946b6869174c2b Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jan 4 17:33:57 2017 +0100 squash! Corrected source-jar contents, enabled additonal target execution before signing using a new property, and cleaned up jar-task. Added Implemented-By property to manifest. --- java/base.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/base.xml b/java/base.xml index 6a3734d..c11dd74 100644 --- a/java/base.xml +++ b/java/base.xml @@ -286,7 +286,7 @@ </restrict> <fileset dir="${usepath}" includes="${usepattern}" /> <manifest> - <attribute name="Created-By" value="The Tor Project" /> + <attribute name="Implemented-By" value="The Tor Project" /> <attribute name="Implementation-Title" value="${implementation-title}"/> <attribute name="Implementation-Version" value="${release.version}-${git.revision}"/>

1 0

[metrics-base/master] squash! squash! Corrected source-jar contents, enabled additonal target execution before signing using a new property, and cleaned up jar-task. Corrected path for javadoc-jar.
by karsten＠torproject.org 05 Jan '17

05 Jan '17

commit 60dabe9036bfe79297b0228ac28825079c9ed2b7 Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jan 4 17:08:39 2017 +0100 squash! squash! Corrected source-jar contents, enabled additonal target execution before signing using a new property, and cleaned up jar-task. Corrected path for javadoc-jar. Removed wrong property definition. --- java/base.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/java/base.xml b/java/base.xml index 0975b25..6a3734d 100644 --- a/java/base.xml +++ b/java/base.xml @@ -28,7 +28,6 @@ value="${dist}/${name}-${release.version}.tar.gz" /> <property file="build.properties" /> <property name="emptymanifest" value="${generated}/emptymanifest" /> - <property name="tardepends" value="" />  <path id="classpath">

1 0