[tor-commits] [tor-browser-bundle/master] Build go-webrtc and snowflake in the mac pluggable-transports descriptor.

Fri Jul 14 07:17:47 UTC 2017

commit 26e0cd44f2886bfad1c3d30844ff7a21eb9d0478
Author: David Fifield <david at bamsoftware.com>
Date:   Wed Jul 5 22:57:22 2017 -0700

    Build go-webrtc and snowflake in the mac pluggable-transports descriptor.
    
    I had to apply two tricks to get a reproducible snowflake-client.
    
    The first is to use faketime to eliminate some timestamps. There were 11
    variable timestamps in the file. Through experimentation, I found that
    10 of them were dependent on the Go runtime (recompiling Go caused them
    to change) and 1 was dependent on snowflake-client itself (recompiling
    snowflake-client with the same runtime changed only that 1 timestamp).
    The underlying issue has to do with clang 3.8.0 on Darwin embedding
    timestamps, unsolved in the Go issue tracker as of 13 days ago.
    https://github.com/golang/go/issues/9206#issuecomment-310476743
    
    The second is a sed command to clobber embedded paths of the form
    /tmp/go-buildXXXXXXXXX and /tmp/go-link-XXXXXXXXX. Their presence is
    caused by some combination of Clang and Darwin, and there is as yet no
    known workaround upstream.
---
 .../mac/gitian-pluggable-transports.yml            | 71 +++++++++++++++++++++-
 gitian/mkbundle-mac.sh                             |  2 +-
 2 files changed, 71 insertions(+), 2 deletions(-)

diff --git a/gitian/descriptors/mac/gitian-pluggable-transports.yml b/gitian/descriptors/mac/gitian-pluggable-transports.yml
index 75ad899..ebfcaa9 100644
--- a/gitian/descriptors/mac/gitian-pluggable-transports.yml
+++ b/gitian/descriptors/mac/gitian-pluggable-transports.yml
@@ -6,7 +6,9 @@ suites:
 architectures:
 - "amd64"
 packages:
+- "faketime"
 - "unzip"
+- "pkg-config"
 - "zip"
 reference_datetime: "2000-01-01 00:00:00"
 remotes:
@@ -24,6 +26,12 @@ remotes:
   "dir": "goxnet"
 - "url": "https://git.torproject.org/pluggable-transports/obfs4.git"
   "dir": "obfs4"
+- "url": "https://github.com/keroserene/go-webrtc.git"
+  "dir": "go-webrtc"
+- "url": "https://git.torproject.org/pluggable-transports/snowflake.git"
+  "dir": "snowflake"
+- "url": "https://github.com/dchest/uniuri.git"
+  "dir": "uniuri"
 files:
 - "versions"
 - "go14.tar.gz"
@@ -31,6 +39,7 @@ files:
 - "clang-linux64-jessie-utils.zip"
 - "cctools.tar.gz"
 - "MacOSX10.7.sdk.tar.gz"
+- "webrtc-mac64-gbuilt.zip"
 - "dzip.sh"
 script: |
   INSTDIR="$HOME/install"
@@ -59,6 +68,8 @@ script: |
   tar xaf MacOSX10.7.sdk.tar.gz
   # Preparing clang for cross-compilation, setting the proper flags and
   # variables
+  # "go link" expects to find a program called "dsymutil" exactly.
+  ln -sf x86_64-apple-darwin10-dsymutil $HOME/build/cctools/bin/dsymutil
   # ld needs libLTO.so from llvm
   export LD_LIBRARY_PATH="$HOME/build/clang/lib"
   export PATH="$HOME/build/cctools/bin:$PATH"
@@ -93,7 +104,12 @@ script: |
   export GOARCH=amd64
   tar xvf go.tar.gz
   cd go/src
-  CGO_ENABLED=1 CC_FOR_TARGET="$CC_FOR_TARGET" CC= CFLAGS= LDFLAGS= ./make.bash
+  # faketime is needed because clang 3.8.0 on Darwin embeds the timestamps of
+  # certain intermediate object files (including those that arise here while
+  # compiling the Go runtime itself). Without this, variable timestamps would
+  # end up in snowflake-client.
+  # https://github.com/golang/go/issues/9206#issuecomment-310476743
+  CGO_ENABLED=1 CC_FOR_TARGET="$CC_FOR_TARGET" CC= CFLAGS= LDFLAGS= faketime -f "$REFERENCE_DATETIME" ./make.bash
   cd ../..
   export PATH="$PATH:$PWD/go/bin"
 
@@ -138,6 +154,14 @@ script: |
   go install github.com/dchest/siphash
   cd ..
 
+  # Building go uniuri
+  cd uniuri
+  find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME"
+  mkdir -p "$GOPATH/src/github.com/dchest/"
+  ln -sf "$PWD" "$GOPATH/src/github.com/dchest/uniuri"
+  go install github.com/dchest/uniuri
+  cd ..
+
   # Building golang.org/x/crypto (obfs4proxy > 0.0.3 || Go >= 1.4)
   cd goxcrypto
   find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME"
@@ -166,6 +190,51 @@ script: |
   cp -a obfs4proxy $PTDIR
   cd ../..
 
+  unzip webrtc-mac64-gbuilt.zip
+  export SDKROOT="$PWD/MacOSX10.7.sdk"
+
+  # Building go-webrtc
+  cd go-webrtc
+  # Replace the prebuilt webrtc library with our own one.
+  rm -rf include/ lib/
+  ln -sf ../webrtc/{include,lib} .
+  find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME"
+  mkdir -p "$GOPATH/src/github.com/keroserene/"
+  ln -sf "$PWD" "$GOPATH/src/github.com/keroserene/go-webrtc"
+  CFLAGS="$FLAGS -mmacosx-version-min=10.7"
+  CXXFLAGS="$FLAGS -stdlib=libc++ -mmacosx-version-min=10.7"
+  LDFLAGS="$FLAGS -stdlib=libc++ -mmacosx-version-min=10.7"
+  GOARCH=amd64 CGO_ENABLED=1 CGO_CFLAGS="$CFLAGS" CGO_CXXFLAGS="$CXXFLAGS" CGO_LDFLAGS="$LDFLAGS" CC="$HOME/build/clang/bin/clang" CXX="$HOME/build/clang/bin/clang++" go install github.com/keroserene/go-webrtc
+  cd ..
+
+  # Building snowflake
+  cd snowflake
+  find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME"
+  cd client
+  # See the faketime comment above. Without faketime, snowflake-client would
+  # contain the timestamp of the temporary client.a file created during
+  # "go build".
+  GOARCH=amd64 CGO_ENABLED=1 CGO_CFLAGS="$CFLAGS" CGO_CXXFLAGS="$CXXFLAGS" CGO_LDFLAGS="$LDFLAGS" CC="$HOME/build/clang/bin/clang" CXX="$HOME/build/clang/bin/clang++" faketime -f "$REFERENCE_DATETIME" go build -ldflags '-s'
+  # Hack: Overwrite variable absolute paths embedded in the binary. clang 3.8.0
+  # on Darwin embeds such paths and the issue is unsolved in upstream Go as of
+  # 2016-06-28:
+  # https://github.com/golang/go/issues/9206#issuecomment-310476743
+  # The two kinds of paths are ("000000000" stands for 9 random digits):
+  #   /tmp/go-build000000000
+  #   /tmp/go-link-000000000
+  # Such paths are the output of ioutil.TempDir("", "go-build") and
+  # ioutil.TempDir("", "go-link-").
+  cp -a client client.stomped
+  sed -i -E -e 's#(/tmp/go-build|/tmp/go-link-)[0-9]{9}/#\1XXXXXXXXX/#g' client.stomped
+  # Sanity check: make sure the file actually changed. If it did not, it could
+  # mean that a change in go or clang has made this step unnecessary.
+  cmp client client.stomped && (echo "No paths replaced in snowflake-client. Maybe the replacement failed or is no longer needed. Check descriptors/mac/gitian-pluggable-transports.yml"; exit 1)
+  cp -a client.stomped $PTDIR/snowflake-client
+  cd ..
+  mkdir -p $INSTDIR/Docs/snowflake
+  cp -a README.md LICENSE $INSTDIR/Docs/snowflake
+  cd ..
+
   # Grabbing the result
   cd $INSTDIR
   ~/build/dzip.sh pluggable-transports-mac64-gbuilt.zip TorBrowserBundle.app
diff --git a/gitian/mkbundle-mac.sh b/gitian/mkbundle-mac.sh
index 646aca6..7bf546e 100755
--- a/gitian/mkbundle-mac.sh
+++ b/gitian/mkbundle-mac.sh
@@ -240,7 +240,7 @@ then
   echo "****** Starting Pluggable Transports Component of Mac Bundle (5/6 for Mac) ******"
   echo
 
-  ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit goptlib=$GOPTLIB_TAG,meek=$MEEK_TAG,ed25519=$GOED25519_TAG,siphash=$GOSIPHASH_TAG,goxcrypto=$GO_X_CRYPTO_TAG,goxnet=$GO_X_NET_TAG,obfs4=$OBFS4_TAG $DESCRIPTOR_DIR/mac/gitian-pluggable-transports.yml
+  ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit goptlib=$GOPTLIB_TAG,meek=$MEEK_TAG,ed25519=$GOED25519_TAG,siphash=$GOSIPHASH_TAG,goxcrypto=$GO_X_CRYPTO_TAG,goxnet=$GO_X_NET_TAG,obfs4=$OBFS4_TAG,go-webrtc=$GO_WEBRTC_TAG,snowflake=$SNOWFLAKE_TAG,uniuri=$UNIURI_TAG $DESCRIPTOR_DIR/mac/gitian-pluggable-transports.yml
   if [ $? -ne 0 ];
   then
     #mv var/build.log ./firefox-fail-mac.log.`date +%Y%m%d%H%M%S`



