commit 8f8a4cd59bfe07e5b8c12e3990627544d9caeab4 Author: Kevin P. Dyer kpdyer@gmail.com Date: Sun Feb 9 11:47:03 2014 -0800
Initial fteproxy integration --- Bundle-Data/beta/linux/torrc-defaults-appendix | 5 +++ gitian/build-helpers/wine-wrappers/g++.py | 39 +++++++++++++++++++ gitian/build-helpers/wine-wrappers/setup.py | 2 +- .../linux/gitian-pluggable-transports.yml | 22 +++++++++++ .../mac/gitian-pluggable-transports.yml | 28 ++++++++++++- .../windows/gitian-pluggable-transports.yml | 41 +++++++++++++++++++- gitian/fetch-inputs.sh | 8 ++-- gitian/mkbundle-linux.sh | 2 +- gitian/mkbundle-mac.sh | 2 +- gitian/mkbundle-windows.sh | 2 +- gitian/versions.beta | 9 ++++- 11 files changed, 149 insertions(+), 11 deletions(-)
diff --git a/Bundle-Data/beta/linux/torrc-defaults-appendix b/Bundle-Data/beta/linux/torrc-defaults-appendix index 2f5ad32..b43cd59 100644 --- a/Bundle-Data/beta/linux/torrc-defaults-appendix +++ b/Bundle-Data/beta/linux/torrc-defaults-appendix @@ -1,4 +1,9 @@
+## fteproxy configuration +# Bridge fte bridges.fteproxy.org:8080 + +ClientTransportPlugin fte exec ./Tor/PluggableTransports/fteproxy --managed + ## obfsproxy configuration # # Bridge obfs3 83.212.101.2:42782 2ADFE7AA8D272C520D1FBFBF4E413F3A1B26313D diff --git a/gitian/build-helpers/wine-wrappers/g++.py b/gitian/build-helpers/wine-wrappers/g++.py new file mode 100755 index 0000000..4738420 --- /dev/null +++ b/gitian/build-helpers/wine-wrappers/g++.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +# A wrapper for i686-w64-mingw32-g++ that removes -mno-cygwin and converts +# Windows paths to Unix paths, so that the w64-mingw32 g++ can be called by +# Python distutils. + +import os +import subprocess +import sys + +import common + +args = ["/usr/bin/i686-w64-mingw32-g++"] +sys.argv.pop(0) +while sys.argv: + a = sys.argv.pop(0) + if not a.startswith("-"): + args.append(common.winepath(a)) + continue + if a == "-mno-cygwin": + continue + if a == "--output-lib": + o = sys.argv.pop(0) + a = "-out-implib="+o + if a in ("-I", "-L"): + args.append(a) + args.append(common.winepath(sys.argv.pop(0))) + continue + o = common.search_startswith(a, ("-I", "-L")) + if o is not None: + path = a[len(o):] + args.append("%s%s" % (o, common.winepath(path))) + continue + args.append(a) +p = common.popen_faketime(args, stderr=subprocess.PIPE) +stderr = p.stderr.read() +sys.stderr.write(stderr) +if " error: " in stderr: + sys.exit(1) diff --git a/gitian/build-helpers/wine-wrappers/setup.py b/gitian/build-helpers/wine-wrappers/setup.py index 797015e..77a9526 100644 --- a/gitian/build-helpers/wine-wrappers/setup.py +++ b/gitian/build-helpers/wine-wrappers/setup.py @@ -1,7 +1,7 @@ from distutils.core import setup import py2exe setup( - console=["gcc.py", "dllwrap.py", "swig.py"], + console=["gcc.py", "g++.py", "dllwrap.py", "swig.py"], zipfile=None, options={"py2exe": {"bundle_files": 1, "compressed": True}} ) diff --git a/gitian/descriptors/linux/gitian-pluggable-transports.yml b/gitian/descriptors/linux/gitian-pluggable-transports.yml index 9aafc57..c6c65dd 100644 --- a/gitian/descriptors/linux/gitian-pluggable-transports.yml +++ b/gitian/descriptors/linux/gitian-pluggable-transports.yml @@ -10,6 +10,7 @@ packages: - "unzip" - "zip" - "swig" +- "m4" - "python-setuptools" - "python-dev" - "faketime" @@ -22,8 +23,11 @@ remotes: "dir": "obfsproxy" - "url": "https://git.torproject.org/flashproxy.git" "dir": "flashproxy" +- "url": "https://github.com/kpdyer/fteproxy.git" + "dir": "fteproxy" files: - "openssl.tar.gz" +- "gmp.tar.bz2" - "pycrypto.tar.gz" - "argparse.tar.gz" - "zope.interface.zip" @@ -50,6 +54,15 @@ script: | mkdir -p $PTDIR mkdir -p $INSTDIR/Debug/ # + tar xvf gmp.tar.bz2 + cd gmp-* + find -type f | xargs touch --date="$REFERENCE_DATETIME" + ./configure --prefix=$INSTDIR/gmp --disable-static --enable-shared --enable-cxx + make + make install + cp -an $INSTDIR/gmp/lib/*.so* $INSTDIR/Tor + cd .. + # cd pyptlib find -type f | xargs touch --date="$REFERENCE_DATETIME" python setup.py build --build-lib build @@ -118,6 +131,15 @@ script: | cp -a {doc/*,README,LICENSE,ChangeLog} $INSTDIR/Docs/FlashProxy cd .. # + cd fteproxy + find -type f | xargs touch --date="$REFERENCE_DATETIME" + CFLAGS="-L$INSTDIR/Tor -I$INSTDIR/gmp/include" CXXFLAGS="-L$INSTDIR/Tor -I$INSTDIR/gmp/include" make + cp -a bin/fteproxy $PTDIR/ + cp -ra fte $PTDIR/ + mkdir -p $INSTDIR/Docs/fteproxy + cp -a {COPYING,README.md} $INSTDIR/Docs/fteproxy + cd .. + # cd $INSTDIR # ~/build/dzip.sh pluggable-transports-linux$GBUILD_BITS-gbuilt.zip Tor/ Docs/ diff --git a/gitian/descriptors/mac/gitian-pluggable-transports.yml b/gitian/descriptors/mac/gitian-pluggable-transports.yml index d44193c..5aaf364 100644 --- a/gitian/descriptors/mac/gitian-pluggable-transports.yml +++ b/gitian/descriptors/mac/gitian-pluggable-transports.yml @@ -9,7 +9,9 @@ packages: - "unzip" - "zip" - "swig" +- "m4" - "python-setuptools" +- "python-dev" - "faketime" - "libtool" #- "libssl0.9.8" @@ -21,8 +23,11 @@ remotes: "dir": "obfsproxy" - "url": "https://git.torproject.org/flashproxy.git" "dir": "flashproxy" +- "url": "https://github.com/kpdyer/fteproxy.git" + "dir": "fteproxy" files: - "openssl.tar.gz" +- "gmp.tar.bz2" - "pycrypto.tar.gz" - "argparse.tar.gz" - "zope.interface.zip" @@ -47,10 +52,13 @@ script: | #export CXXFLAGS=$CFLAGS #export LDFLAGS=$CFLAGS export PATH="$PATH:/usr/apple-osx/bin/" + export AR=/usr/apple-osx/bin/i686-apple-darwin11-ar export CC=/usr/apple-osx/bin/i686-apple-darwin11-gcc + export CXX=/usr/apple-osx/bin/i686-apple-darwin11-g++ # http://bugs.python.org/issue9437 export LDSHARED="$CC -pthread -shared" - export CFLAGS="-I/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/include/ -I/usr/lib/gcc/i686-apple-darwin10/4.2.1/include/ -I. -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/ -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/system/ -F/usr/lib/apple/SDKs/MacOSX10.6.sdk/System/Library/Frameworks -mmacosx-version-min=10.5" + export CFLAGS="-I/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/include/ -I/usr/lib/gcc/i686-apple-darwin10/4.2.1/include/ -I. -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/ -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/system/ -F/usr/lib/apple/SDKs/MacOSX10.6.sdk/System/Library/Frameworks -mmacosx-version-min=10.5 -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/i686-apple-darwin10/4.2.1 -I$INSTDIR/gmp/include -L$INSTDIR/gmp/lib" + export CXXFLAGS="-I/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/include/ -I/usr/lib/gcc/i686-apple-darwin10/4.2.1/include/ -I. -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/ -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/system/ -F/usr/lib/apple/SDKs/MacOSX10.6.sdk/System/Library/Frameworks -mmacosx-version-min=10.5 -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/i686-apple-darwin10/4.2.1 -I$INSTDIR/gmp/include -L$INSTDIR/gmp/lib" export LDFLAGS="-L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/ -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/system/ -F/usr/lib/apple/SDKs/MacOSX10.6.sdk/System/Library/Frameworks -mmacosx-version-min=10.5" # mkdir -p $PTDIR/ @@ -63,6 +71,15 @@ script: | # For OpenSSL sudo ln -s /usr/apple-osx/bin/apple-osx-gcc /usr/apple-osx/bin/i686-apple-darwin11-cc # + tar xvf gmp.tar.bz2 + cd gmp-* + find -type f | xargs touch --date="$REFERENCE_DATETIME" + ./configure --host=i686-apple-darwin11 --prefix=$INSTDIR/gmp --disable-static --enable-shared --enable-cxx + make + make install + cp $INSTDIR/gmp/lib/*dylib* $INSTDIR/TorBrowserBundle.app/Tor/ + cd .. + # cd pyptlib find -type f | xargs touch --date="$REFERENCE_DATETIME" python setup.py build --build-lib build @@ -141,6 +158,15 @@ script: | cp -a {doc/*,README,LICENSE,ChangeLog} $INSTDIR/TorBrowserBundle.app/Docs/FlashProxy cd .. # + cd fteproxy + find -type f | xargs touch --date="$REFERENCE_DATETIME" + CFLAGS="-L$INSTDIR/Tor -I$INSTDIR/gmp/include $CFLAGS" CXXFLAGS="-L$INSTDIR/Tor -I$INSTDIR/gmp/include $CXXFLAGS" make + cp -a bin/fteproxy $PTDIR/ + cp -ra fte $PTDIR/ + mkdir -p $INSTDIR/Docs/fteproxy + cp -a {COPYING,README.md} $INSTDIR/Docs/fteproxy + cd .. + # cd $INSTDIR ~/build/dzip.sh pluggable-transports-mac$GBUILD_BITS-gbuilt.zip TorBrowserBundle.app cp pluggable-transports-mac$GBUILD_BITS-gbuilt.zip $OUTDIR/ diff --git a/gitian/descriptors/windows/gitian-pluggable-transports.yml b/gitian/descriptors/windows/gitian-pluggable-transports.yml index a8535c6..92dddeb 100644 --- a/gitian/descriptors/windows/gitian-pluggable-transports.yml +++ b/gitian/descriptors/windows/gitian-pluggable-transports.yml @@ -10,7 +10,10 @@ packages: - "zip" - "swig" - "p7zip-full" +- "m4" - "mingw-w64" +- "g++-mingw-w64" +- "python-dev" - "faketime" - "libtool" reference_datetime: "2000-01-01 00:00:00" @@ -21,8 +24,11 @@ remotes: "dir": "obfsproxy" - "url": "https://git.torproject.org/flashproxy.git" "dir": "flashproxy" +- "url": "https://github.com/kpdyer/fteproxy.git" + "dir": "fteproxy" files: - "openssl.tar.gz" +- "gmp.tar.bz2" - "setuptools.tar.gz" - "pycrypto.tar.gz" - "zope.interface.zip" @@ -71,6 +77,18 @@ script: | rm -rf "$tmpdir" } # + # + tar xvf gmp.tar.bz2 + cd gmp-* + find -type f | xargs touch --date="$REFERENCE_DATETIME" + ./configure --prefix=$INSTDIR/gmp --host=i686-w64-mingw32 --enable-cxx --disable-static --enable-shared + make + make install + cp -an $INSTDIR/gmp/bin/*dll* $INSTDIR/Tor + cp -an $INSTDIR/gmp/lib/*dll* $INSTDIR/Tor + cd .. + # + # # Install a Wine new enough to have a fix for http://bugs.winehq.org/show_bug.cgi?id=29764; # otherwise Python run under Wine constantly crashes in _PyVerify_fd, which is # called by such common operations as io.open and os.fstat (anything involving @@ -103,7 +121,7 @@ script: | mkdir -p build/bdist.win32/winexe/bundle-2.7/ cp -a $INSTDIR/python/python27.dll build/bdist.win32/winexe/bundle-2.7/ LD_PRELOAD= $INSTPYTHON setup.py py2exe - cp -a dist/gcc.exe dist/dllwrap.exe dist/swig.exe $WINEROOT/windows/ + cp -a dist/gcc.exe dist/g++.exe dist/dllwrap.exe dist/swig.exe $WINEROOT/windows/ cd .. # tar xzf setuptools.tar.gz @@ -156,6 +174,7 @@ script: | cd obfsproxy find -type f | xargs touch --date="$REFERENCE_DATETIME" LD_PRELOAD= $INSTPYTHON setup_py2exe.py py2exe + LD_PRELOAD= $INSTPYTHON setup.py install py2exe_zip_timestomp py2exe_bundle/dist/obfsproxy.zip cp -an py2exe_bundle/dist/{*.pyd,*.exe,*.zip} $PTDIR/ mkdir -p $INSTDIR/Docs/Obfsproxy @@ -188,6 +207,26 @@ script: | cp {doc/*,README,LICENSE,ChangeLog} $INSTDIR/Docs/FlashProxy cd .. # + cd fteproxy + cp $INSTDIR/gmp/bin/*dll* . + cp $INSTDIR/gmp/lib/*dll* . + ln -s $INSTDIR/gmp thirdparty/gmp + find -type f | xargs touch --date="$REFERENCE_DATETIME" + make win32 + make thirdparty/re2/obj/libre2.a + LD_PRELOAD= $INSTPYTHON setup.py build_ext -c mingw32 --inplace + LD_PRELOAD= $INSTPYTHON setup.py py2exe + mkdir -p $PTDIR/fte + mkdir -p $PTDIR/fte/tests + cp -an dist/{*.pyd,*.exe,*.zip} $PTDIR/ + cp -an fte/VERSION $PTDIR/fte + cp -ran fte/defs $PTDIR/fte + cp -ran fte/tests/dfas $PTDIR/fte/tests + cp -an thirdparty/gmp/bin/*.dll $PTDIR/ + mkdir -p $INSTDIR/Docs/fteproxy + cp -an {COPYING,README.md} $INSTDIR/Docs/fteproxy + cd .. + # # http://bugs.winehq.org/show_bug.cgi?id=3591 cp -a $INSTDIR/python/python27.dll $PTDIR/ # diff --git a/gitian/fetch-inputs.sh b/gitian/fetch-inputs.sh index d4eda0e..d757f4d 100755 --- a/gitian/fetch-inputs.sh +++ b/gitian/fetch-inputs.sh @@ -132,7 +132,7 @@ do fi done
-for i in BINUTILS GCC PYTHON PYCRYPTO M2CRYPTO PYTHON_MSI +for i in BINUTILS GCC PYTHON PYCRYPTO M2CRYPTO PYTHON_MSI GMP do PACKAGE="${i}_PACKAGE" URL="${i}_URL" @@ -151,7 +151,6 @@ do fi done
- for i in TOOLCHAIN4 OSXSDK MSVCR100 do PACKAGE="${i}_PACKAGE" @@ -195,6 +194,7 @@ do exit 1 fi done + cd ..
# NoScript and PDF.JS are magikal and special: @@ -217,7 +217,7 @@ fi
# Verify packages with weak or no signatures via direct sha256 check # (OpenSSL is signed with MD5, and OSXSDK is not signed at all) -for i in OSXSDK TOOLCHAIN4 NOSCRIPT MINGW MSVCR100 PYCRYPTO ARGPARSE ZOPEINTERFACE TWISTED M2CRYPTO SETUPTOOLS OPENSSL +for i in OSXSDK TOOLCHAIN4 NOSCRIPT MINGW MSVCR100 PYCRYPTO ARGPARSE ZOPEINTERFACE TWISTED M2CRYPTO SETUPTOOLS OPENSSL GMP do PACKAGE="${i}_PACKAGE" HASH="${i}_HASH" @@ -265,6 +265,7 @@ ln -sf "$TWISTED_PACKAGE" twisted.tar.bz2 ln -sf "$M2CRYPTO_PACKAGE" m2crypto.tar.gz ln -sf "$PY2EXE_PACKAGE" py2exe.exe ln -sf "$SETUPTOOLS_PACKAGE" setuptools.tar.gz +ln -sf "$GMP_PACKAGE" gmp.tar.bz2
# Fetch latest gitian-builder itself # XXX - this is broken if a non-standard inputs dir is selected using the command line flag. @@ -290,6 +291,7 @@ tor-browser https://git.torproject.org/tor-browser.git $TORBR pyptlib https://git.torproject.org/pluggable-transports/pyptlib.git obfsproxy https://git.torproject.org/pluggable-transports/obfsproxy.git flashproxy https://git.torproject.org/flashproxy.git +fteproxy https://github.com/kpdyer/fteproxy.git EOF
exit 0 diff --git a/gitian/mkbundle-linux.sh b/gitian/mkbundle-linux.sh index 960d33c..d2901d8 100755 --- a/gitian/mkbundle-linux.sh +++ b/gitian/mkbundle-linux.sh @@ -129,7 +129,7 @@ then echo "****** Starting Pluggable Transports Component of Linux Bundle (3/4 for Linux) ******" echo
- ./bin/gbuild -j $NUM_PROCS --commit pyptlib=$PYPTLIB_TAG,obfsproxy=$OBFSPROXY_TAG,flashproxy=$FLASHPROXY_TAG $DESCRIPTOR_DIR/linux/gitian-pluggable-transports.yml + ./bin/gbuild -j $NUM_PROCS --commit pyptlib=$PYPTLIB_TAG,obfsproxy=$OBFSPROXY_TAG,flashproxy=$FLASHPROXY_TAG,fteproxy=$FTEPROXY_TAG $DESCRIPTOR_DIR/linux/gitian-pluggable-transports.yml if [ $? -ne 0 ]; then #mv var/build.log ./pluggable-transports-fail-linux.log.`date +%Y%m%d%H%M%S` diff --git a/gitian/mkbundle-mac.sh b/gitian/mkbundle-mac.sh index 21e88d2..1f5eb18 100755 --- a/gitian/mkbundle-mac.sh +++ b/gitian/mkbundle-mac.sh @@ -126,7 +126,7 @@ then echo "****** Starting Pluggable Transports Component of Mac Bundle (3/4 for Mac) ******" echo
- ./bin/gbuild -j $NUM_PROCS --commit pyptlib=$PYPTLIB_TAG,obfsproxy=$OBFSPROXY_TAG,flashproxy=$FLASHPROXY_TAG $DESCRIPTOR_DIR/mac/gitian-pluggable-transports.yml + ./bin/gbuild -j $NUM_PROCS --commit pyptlib=$PYPTLIB_TAG,obfsproxy=$OBFSPROXY_TAG,flashproxy=$FLASHPROXY_TAG,fteproxy=$FTEPROXY_TAG $DESCRIPTOR_DIR/mac/gitian-pluggable-transports.yml if [ $? -ne 0 ]; then #mv var/build.log ./firefox-fail-mac.log.`date +%Y%m%d%H%M%S` diff --git a/gitian/mkbundle-windows.sh b/gitian/mkbundle-windows.sh index 3760b01..c30ccf1 100755 --- a/gitian/mkbundle-windows.sh +++ b/gitian/mkbundle-windows.sh @@ -127,7 +127,7 @@ then echo "****** Starting Pluggable Transports Component of Windows Bundle (3/4 for Windows) ******" echo
- ./bin/gbuild -j $NUM_PROCS --commit pyptlib=$PYPTLIB_TAG,obfsproxy=$OBFSPROXY_TAG,flashproxy=$FLASHPROXY_TAG $DESCRIPTOR_DIR/windows/gitian-pluggable-transports.yml + ./bin/gbuild -j $NUM_PROCS --commit pyptlib=$PYPTLIB_TAG,obfsproxy=$OBFSPROXY_TAG,flashproxy=$FLASHPROXY_TAG,fteproxy=$FTEPROXY_TAG $DESCRIPTOR_DIR/windows/gitian-pluggable-transports.yml if [ $? -ne 0 ]; then #mv var/build.log ./pluggable-transports-fail-win32.log.`date +%Y%m%d%H%M%S` diff --git a/gitian/versions.beta b/gitian/versions.beta index 7cb474e..775d7f9 100755 --- a/gitian/versions.beta +++ b/gitian/versions.beta @@ -13,13 +13,15 @@ NSIS_TAG=v0.1 ZLIB_TAG=v1.2.8 LIBEVENT_TAG=release-2.0.21-stable MINGW_REV=6184 -PYPTLIB_TAG=pyptlib-0.0.4 -OBFSPROXY_TAG=obfsproxy-0.2.3 +PYPTLIB_TAG=pyptlib-0.0.5 +OBFSPROXY_TAG=obfsproxy-0.2.4 FLASHPROXY_TAG=1.4 +FTEPROXY_TAG=0.2.6
GITIAN_TAG=tor-browser-builder-3.0-4
OPENSSL_VER=1.0.1f +GMP_VER=5.1.3 FIREFOX_LANG_VER=24.3.0esr BINUTILS_VER=2.22 GCC_VER=4.6.3 @@ -34,6 +36,7 @@ SETUPTOOLS_VER=1.4
## File names for the source packages OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz +GMP_PACKAGE=gmp-${GMP_VER}.tar.bz2 NOSCRIPT_PACKAGE=noscript_security_suite-2.6.8.13-fn+sm+fx.xpi TOOLCHAIN4_PACKAGE=x86_64-apple-darwin10.tar.xz OSXSDK_PACKAGE=apple-uni-sdk-10.6_20110407-0.flosoft1_i386.deb @@ -53,6 +56,7 @@ SETUPTOOLS_PACKAGE=setuptools-${SETUPTOOLS_VER}.tar.gz
# Hashes for packages with weak sigs or no sigs OPENSSL_HASH=6cc2a80b17d64de6b7bac985745fdaba971d54ffd7d38d3556f998d7c0c9cb5a +GMP_HASH=752079520b4690531171d0f4532e40f08600215feefede70b24fabdc6f1ab160 OSXSDK_HASH=6602d8d5ddb371fbc02e2a5967d9bd0cd7358d46f9417753c8234b923f2ea6fc TOOLCHAIN4_HASH=7b71bfe02820409b994c5c33a7eab81a81c72550f5da85ff7af70da3da244645 NOSCRIPT_HASH=0d90f283ee9e3be5bb1ebc315fab12994ff5a15380bd3a2fdff62750286b4ea1 @@ -68,6 +72,7 @@ SETUPTOOLS_HASH=75d288687066ed124311d6ca5f40ffa92a0e81adcd7fff318c6e84082713cf39
## Non-git package URLs OPENSSL_URL=https://www.openssl.org/source/$%7BOPENSSL_PACKAGE%7D +GMP_URL=https://ftp.gnu.org/gnu/gmp/$%7BGMP_PACKAGE%7D TOOLCHAIN4_URL=https://people.torproject.org/~mikeperry/mirrors/sources/$%7BTOOLCHAIN4_PACK... OSXSDK_URL=https://launchpad.net/~flosoft/+archive/cross-apple/+files/$%7BOSXSDK_PACKAG... BINUTILS_URL=https://ftp.gnu.org/gnu/binutils/$%7BBINUTILS_PACKAGE%7D
tor-commits@lists.torproject.org
-
mikeperry＠torproject.org