- tor-commits - lists.torproject.org

[translation/tor-launcher-progress_completed] Update translations for tor-launcher-progress_completed
by translation＠torproject.org 13 May '16

13 May '16

commit 65ee68c34a61d092a790d4ffd7760df5b302d985 Author: Translation commit bot <translation(a)torproject.org> Date: Fri May 13 12:45:41 2016 +0000 Update translations for tor-launcher-progress_completed --- es_AR/progress.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/es_AR/progress.dtd b/es_AR/progress.dtd index 10a52dd..223d4a5 100644 --- a/es_AR/progress.dtd +++ b/es_AR/progress.dtd @@ -1,4 +1,4 @@ <!ENTITY torprogress.dialog.title "Estado de Tor"> <!ENTITY torprogress.openSettings "Abrir Configuraciones"> <!ENTITY torprogress.heading "Conectando a la red Tor"> -<!ENTITY torprogress.pleaseWait "Por favor, espere mientras establecemos una conexión a la red Tor."> +<!ENTITY torprogress.pleaseWait "Por favor, espere mientras establecemos una conexión a la red Tor.  Puede tardar varios minutos.">

1 0

[translation/tor-launcher-progress] Update translations for tor-launcher-progress
by translation＠torproject.org 13 May '16

13 May '16

commit bec1dc2e5b7e7a5fc30fe82231761c9e2ad6a4b5 Author: Translation commit bot <translation(a)torproject.org> Date: Fri May 13 12:45:37 2016 +0000 Update translations for tor-launcher-progress --- es_AR/progress.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/es_AR/progress.dtd b/es_AR/progress.dtd index f8029aa..223d4a5 100644 --- a/es_AR/progress.dtd +++ b/es_AR/progress.dtd @@ -1,4 +1,4 @@ <!ENTITY torprogress.dialog.title "Estado de Tor"> <!ENTITY torprogress.openSettings "Abrir Configuraciones"> <!ENTITY torprogress.heading "Conectando a la red Tor"> -<!ENTITY torprogress.pleaseWait "Please wait while we establish a connection to the Tor network.  This may take several minutes."> +<!ENTITY torprogress.pleaseWait "Por favor, espere mientras establecemos una conexión a la red Tor.  Puede tardar varios minutos.">

1 0

[tor-browser/tor-browser-45.1.0esr-6.0-1] Bug 18958: Spoof screen.orientation values
by gk＠torproject.org 13 May '16

13 May '16

commit a086a5d0070821b1c4dfd2492bdc993c91353a2b Author: Arthur Edelstein <arthuredelstein(a)gmail.com> Date: Sun May 8 22:23:42 2016 -0700 Bug 18958: Spoof screen.orientation values Make sure that screen.orientation.angle -> 0 and screen.orientation.type -> "landscape-primary" Also refactors screen.mozOrientation. --- dom/base/ScreenOrientation.cpp | 29 +++++++++++++++++++++++++++-- dom/base/ScreenOrientation.h | 2 ++ dom/base/nsGlobalWindow.cpp | 7 ++++--- dom/base/nsScreen.cpp | 38 ++++++++++++++++++++------------------ 4 files changed, 53 insertions(+), 23 deletions(-) diff --git a/dom/base/ScreenOrientation.cpp b/dom/base/ScreenOrientation.cpp index 868a1d5..fdea67a 100644 --- a/dom/base/ScreenOrientation.cpp +++ b/dom/base/ScreenOrientation.cpp @@ -15,6 +15,7 @@ #include "mozilla/Preferences.h" #include "mozilla/dom/Promise.h" +#include "nsContentUtils.h" using namespace mozilla; using namespace mozilla::dom; @@ -403,18 +404,23 @@ ScreenOrientation::UnlockDeviceOrientation() OrientationType ScreenOrientation::DeviceType() const { - return mType; + return ShouldResistFingerprinting() ? OrientationType::Landscape_primary + : mType; } uint16_t ScreenOrientation::DeviceAngle() const { - return mAngle; + return ShouldResistFingerprinting() ? 0 : mAngle; } OrientationType ScreenOrientation::GetType(ErrorResult& aRv) const { + if (ShouldResistFingerprinting()) { + return OrientationType::Landscape_primary; + } + nsIDocument* doc = GetResponsibleDocument(); if (!doc) { aRv.Throw(NS_ERROR_UNEXPECTED); @@ -427,6 +433,10 @@ ScreenOrientation::GetType(ErrorResult& aRv) const uint16_t ScreenOrientation::GetAngle(ErrorResult& aRv) const { + if (ShouldResistFingerprinting()) { + return 0; + } + nsIDocument* doc = GetResponsibleDocument(); if (!doc) { aRv.Throw(NS_ERROR_UNEXPECTED); @@ -489,6 +499,10 @@ ScreenOrientation::GetResponsibleDocument() const void ScreenOrientation::Notify(const hal::ScreenConfiguration& aConfiguration) { + if (ShouldResistFingerprinting()) { + return; + } + nsIDocument* doc = GetResponsibleDocument(); if (!doc) { return; @@ -564,6 +578,17 @@ ScreenOrientation::WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) return ScreenOrientationBinding::Wrap(aCx, this, aGivenProto); } +bool +ScreenOrientation::ShouldResistFingerprinting() const +{ + bool resist = false; + nsCOMPtr<nsPIDOMWindow> owner = GetOwner(); + if (owner) { + resist = nsContentUtils::ShouldResistFingerprinting(owner->GetDocShell()); + } + return resist; +} + NS_IMPL_ISUPPORTS(ScreenOrientation::VisibleEventListener, nsIDOMEventListener) NS_IMETHODIMP diff --git a/dom/base/ScreenOrientation.h b/dom/base/ScreenOrientation.h index afb5e77..a6e51c7 100644 --- a/dom/base/ScreenOrientation.h +++ b/dom/base/ScreenOrientation.h @@ -102,6 +102,8 @@ private: void DispatchChangeEvent(); + bool ShouldResistFingerprinting() const; + LockPermission GetLockOrientationPermission(bool aCheckSandbox) const; // Gets the responsible document as defined in the spec. diff --git a/dom/base/nsGlobalWindow.cpp b/dom/base/nsGlobalWindow.cpp index ed42538..6dc06b1 100644 --- a/dom/base/nsGlobalWindow.cpp +++ b/dom/base/nsGlobalWindow.cpp @@ -12820,8 +12820,8 @@ void nsGlobalWindow::EnableOrientationChangeListener() { MOZ_ASSERT(IsInnerWindow()); - - if (!mOrientationChangeObserver) { + if (!nsContentUtils::ShouldResistFingerprinting(mDocShell) + && !mOrientationChangeObserver) { mOrientationChangeObserver = new WindowOrientationObserver(this); } @@ -13671,7 +13671,8 @@ nsGlobalWindow::IsModalContentWindow(JSContext* aCx, JSObject* aGlobal) int16_t nsGlobalWindow::Orientation() const { - return WindowOrientationObserver::OrientationAngle(); + return nsContentUtils::ShouldResistFingerprinting(mDocShell) ? + 0 : WindowOrientationObserver::OrientationAngle(); } #endif diff --git a/dom/base/nsScreen.cpp b/dom/base/nsScreen.cpp index ad1642b..49c12eb 100644 --- a/dom/base/nsScreen.cpp +++ b/dom/base/nsScreen.cpp @@ -166,25 +166,21 @@ nsScreen::Orientation() const void nsScreen::GetMozOrientation(nsString& aOrientation) const { - if (ShouldResistFingerprinting()) { + switch (mScreenOrientation->DeviceType()) { + case OrientationType::Portrait_primary: + aOrientation.AssignLiteral("portrait-primary"); + break; + case OrientationType::Portrait_secondary: + aOrientation.AssignLiteral("portrait-secondary"); + break; + case OrientationType::Landscape_primary: aOrientation.AssignLiteral("landscape-primary"); - } else { - switch (mScreenOrientation->DeviceType()) { - case OrientationType::Portrait_primary: - aOrientation.AssignLiteral("portrait-primary"); - break; - case OrientationType::Portrait_secondary: - aOrientation.AssignLiteral("portrait-secondary"); - break; - case OrientationType::Landscape_primary: - aOrientation.AssignLiteral("landscape-primary"); - break; - case OrientationType::Landscape_secondary: - aOrientation.AssignLiteral("landscape-secondary"); - break; - default: - MOZ_CRASH("Unacceptable screen orientation type."); - } + break; + case OrientationType::Landscape_secondary: + aOrientation.AssignLiteral("landscape-secondary"); + break; + default: + MOZ_CRASH("Unacceptable screen orientation type."); } } @@ -236,6 +232,9 @@ bool nsScreen::MozLockOrientation(const Sequence<nsString>& aOrientations, ErrorResult& aRv) { + if (ShouldResistFingerprinting()) { + return false; + } ScreenOrientationInternal orientation = eScreenOrientation_None; for (uint32_t i = 0; i < aOrientations.Length(); ++i) { @@ -283,6 +282,9 @@ nsScreen::MozLockOrientation(const Sequence<nsString>& aOrientations, void nsScreen::MozUnlockOrientation() { + if (ShouldResistFingerprinting()) { + return; + } UpdateDocShellOrientationLock(GetOwner(), eScreenOrientation_None); mScreenOrientation->UnlockDeviceOrientation(); }

1 0

[tor-browser/tor-browser-45.1.0esr-6.0-1] fixup! Regression tests for #5856: Do not expose physical screen info via window & window.screen.
by gk＠torproject.org 13 May '16

13 May '16

commit ec74b173f8a00e59aa42e0cf32e65ec2266cb477 Author: Arthur Edelstein <arthuredelstein(a)gmail.com> Date: Sun May 8 23:33:39 2016 -0700 fixup! Regression tests for #5856: Do not expose physical screen info via window & window.screen. --- tbb-tests/test_tor_bug5856.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tbb-tests/test_tor_bug5856.html b/tbb-tests/test_tor_bug5856.html index 2eae453..7a9ab6f 100644 --- a/tbb-tests/test_tor_bug5856.html +++ b/tbb-tests/test_tor_bug5856.html @@ -27,7 +27,10 @@ https://trac.torproject.org/projects/tor/ticket/5856 ["window.screen.availLeft", 0], ["window.screen.availTop", 0], ["window.screen.width", "window.innerWidth"], - ["window.screen.height", "window.innerHeight"] + ["window.screen.height", "window.innerHeight"], + ["window.screen.orientation.type", "'landscape-primary'"], + ["window.screen.orientation.angle", 0], + ["window.screen.mozOrientation", "'landscape-primary'"], ]; pairs.map(checkPair);

1 0

[translation/tor-launcher-properties_completed] Update translations for tor-launcher-properties_completed
by translation＠torproject.org 13 May '16

13 May '16

commit 4e584ca26221ab3dc1a0aed522eaa4bd9fb7b5a3 Author: Translation commit bot <translation(a)torproject.org> Date: Fri May 13 05:45:36 2016 +0000 Update translations for tor-launcher-properties_completed --- el/torlauncher.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/el/torlauncher.properties b/el/torlauncher.properties index 90fdc3f..b9ce8ce 100644 --- a/el/torlauncher.properties +++ b/el/torlauncher.properties @@ -36,6 +36,7 @@ torlauncher.quit_win=Έξοδος torlauncher.done=Τελος torlauncher.forAssistance=Για βοήθεια, επικοινωνήστε με το% S +torlauncher.forAssistance2=Για βοήθεια, επισκεφτείτε %S torlauncher.copiedNLogMessages=Η αντιγραφή ολοκληρώθηκε. %S μηνύματα καταγραφής είναι έτοιμα να επικολληθούν σε ένα κειμενογράφο ή ένα μήνυμα ηλεκτρονικού ταχυδρομείου.

1 0

[translation/tor-launcher-properties] Update translations for tor-launcher-properties
by translation＠torproject.org 13 May '16

13 May '16

commit 49f306ef4eff5c1640445c2a7d3a4505dedbf8b8 Author: Translation commit bot <translation(a)torproject.org> Date: Fri May 13 05:45:32 2016 +0000 Update translations for tor-launcher-properties --- el/torlauncher.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/el/torlauncher.properties b/el/torlauncher.properties index b0a53e6..b9ce8ce 100644 --- a/el/torlauncher.properties +++ b/el/torlauncher.properties @@ -36,7 +36,7 @@ torlauncher.quit_win=Έξοδος torlauncher.done=Τελος torlauncher.forAssistance=Για βοήθεια, επικοινωνήστε με το% S -torlauncher.forAssistance2=For assistance, visit %S +torlauncher.forAssistance2=Για βοήθεια, επισκεφτείτε %S torlauncher.copiedNLogMessages=Η αντιγραφή ολοκληρώθηκε. %S μηνύματα καταγραφής είναι έτοιμα να επικολληθούν σε ένα κειμενογράφο ή ένα μήνυμα ηλεκτρονικού ταχυδρομείου.

1 0

[webwml/master] Add ooni dev job posting
by sebastian＠torproject.org 13 May '16

13 May '16

commit 65f9bd914997ef3fcffb76dd208119340e3d3be8 Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Fri May 13 03:47:39 2016 +0200 Add ooni dev job posting --- about/en/jobs-ooni.wml | 142 +++++++++++++++++++++++++++++++++++++++++++++++++ about/en/jobs.wml | 18 +++---- 2 files changed, 151 insertions(+), 9 deletions(-) diff --git a/about/en/jobs-ooni.wml b/about/en/jobs-ooni.wml new file mode 100644 index 0000000..2db748c --- /dev/null +++ b/about/en/jobs-ooni.wml @@ -0,0 +1,142 @@ +## translation metadata +# Revision: $Revision$ +# Translation-Priority: 3-low + +#include "head.wmi" TITLE="Tor Project: Jobs (core developer)" CHARSET="UTF-8" +<div id="content" class="clearfix"> + <div id="breadcrumbs"> + <a href="<page index>">Home » </a> + <a href="<page about/overview>">About » </a> + <a href="<page about/jobs>">Jobs</a> + </div> + <div id="maincol"> +<h1>The Tor Project is looking for another dedicated core developer to +work on OONI!</h1> + +<p> +Your job would be to work as a software engineer on the Open Observatory +of Network Interference (OONI), a global network measurement platform. +Your initial focus will be on the design and implementation of a +mechanism for instrumenting probes on which measurements should be run +and re-engineering the data processing pipeline to scale to the growing +OONI user-base. + +Moreover you will be expected to be working on improving the existing +measurement instrument (ooni-probe) and the data collection backend +component (ooni-backend). +</p> + +<p> +This would be a contractor position starting as soon as possible (by +mid-June 2016, latest) for a duration of 12 months. +</p> + +<p> +Any candidate must: +</p> + +<ul> + <li>Have extensive experience in python and several other + programming languages.</li> + <li>Be comfortable working remotely.</li> + <li>Be self-directed.</li> + <li>Be comfortable and experienced in getting code and design + reviewed and reviewing the code and design of others.</li> + <li>Be comfortable and experienced in documenting and designing + protocols.</li> + <li>Be comfortable with transparency: as a non-profit, everything + we do is in public, including your name and pay rate.</li> +</ul> + +<p> +An ideal candidate would also: +</p> + +<ul> + <li>Have prior experience with designing and implementing scalable + complex network applications.</li> + <li>Not be afraid of refactoring code and writing unit-tests.</li> + <li>Be familiar with the challenges of developing and scaling data + processing pipelines.</li> + <li>Be familiar with the network measurement field.</li> + <li>Have experience with open-source software development, including + working with distributed teams across different time-zones + containing employees and volunteers of differing skill levels.</li> + <li>Have basic familiarity with distributed version control + systems.</li> + <li>Have contributed significant chunks of code to multiple + open-source projects in the past.</li> + <li>Be passionate about internet freedom and interested in + contributing to it in a concrete way.</li> +</ul> + +<p> +Other notes: +</p> + +<ul> + <li>You'll need to be comfortable in this environment! We coordinate + via IRC, instant messaging, email, and bug trackers.</li> + <li>Academic degrees are great, but not required if you have the + right experience.</li> + <li>We only write free and open source software, and we don't + believe in software patents.</li> +</ul> + +<p> +How to apply +</p> + +<ul> + <li>Link to a sample of code you've written in the past that you're + allowed to show us and ideally also your github profile.</li> + <li>Provide a CV explaining your background, experience, skills, and + other relevant qualifications.</li> + <li>List some people who can tell us more about you: these + references could be employers or co-workers, open source projects, + etc.</li> + <li>Email the above to jobs(a)torproject.org</li> +</ul> + +<p> +About the project: +</p> + +<p> +The Open Observatory of Network Interference (OONI) is a free software +project under the Tor Project which collects and processes network +measurements with the aim of detecting network interferences, such as +censorship, surveillance and traffic manipulation. Since late 2012, OONI +has collected millions of measurements across more than 90 countries +around the world. +</p> + +<p> +About the company: +</p> + +<p> +The Tor Project is a US 501(c)(3) non-profit dedicated to research, +development, and education about online anonymity and privacy. The Tor +network's 3000 volunteer relays carry 16 Gbps for upwards of half a +million daily users, including ordinary citizens who want protection +from identity theft and prying corporations, corporations who want to +look at a competitor's website in private, people around the world whose +Internet connections are censored, and even governments and law +enforcement. Tor has a staff of 14 paid developers, researchers, and +advocates, plus many dozen volunteers who help out on a daily basis. Tor +is funded in part by government research and development grants, and in +part by individual and corporate donations. +</p> + + </div> +  + <div id = "sidecol"> +#include "side.wmi" +#include "info.wmi" + </div> +  +</div> + +#include <foot.wmi> + diff --git a/about/en/jobs.wml b/about/en/jobs.wml index b859d4d..eab3af4 100644 --- a/about/en/jobs.wml +++ b/about/en/jobs.wml @@ -12,18 +12,18 @@ <div id="maincol"> <h1>The Tor Project is always looking for more great people to join our team!</h1> -<p>At the moment, we don't have any official open positions. Please check back -soon, though! In the meantime, you may want to glance at our -<a href="<page getinvolved/volunteer>">volunteers</a> page. +#<p>At the moment, we don't have any official open positions. Please check back +#soon, though! In the meantime, you may want to glance at our +#<a href="<page getinvolved/volunteer>">volunteers</a> page. -#<p>Specifically, we're looking for the following people currently: -#</p> -#<ul> -#<li><a href="<page about/jobs-hr>">Human Resources Manager</a></li> +<p>Specifically, we're looking for the following people currently: +</p> +<ul> +<li><a href="<page about/jobs-ooni>">Core OONI Developer</a></li> #<li><a href="<page about/jobs-editor>">Writer/Editor</a></li> #<li><a href="<page about/jobs-admin-assistant>">Administrative Assistant</a></li> -#</ul> -#</p> +</ul> +</p> <p>In the past, we looked for the following people:</p>

1 0

[tor-browser-bundle/hardened-builds] Bug 17406: Include SelfRando support in our nightlies
by gk＠torproject.org 12 May '16

12 May '16

commit 138470ca7ce810f62761af8c3989654ec4a611fa Author: Georg Koppen <gk(a)torproject.org> Date: Thu May 12 20:25:02 2016 +0000 Bug 17406: Include SelfRando support in our nightlies SelfRando is a new defense against code reuse attacks developed by the Radactor and Readactor++ people. We should give it a wider testing audience by including it in the nightly. --- RelativeLink/start-tor-browser | 3 + gitian/descriptors/linux/gitian-firefox.yml | 7 + gitian/descriptors/linux/gitian-utils.yml | 38 +++- gitian/fetch-inputs.sh | 4 +- gitian/gpg/ELFUTILS.gpg | Bin 0 -> 10483 bytes gitian/mkbundle-linux.sh | 9 +- gitian/patches/libfaketime-asan.patch | 268 ++++++++++++++++++++++++++-- gitian/verify-tags.sh | 3 +- gitian/versions.nightly | 4 + 9 files changed, 313 insertions(+), 23 deletions(-) diff --git a/RelativeLink/start-tor-browser b/RelativeLink/start-tor-browser index 2e7805d..19c5cfc 100755 --- a/RelativeLink/start-tor-browser +++ b/RelativeLink/start-tor-browser @@ -270,6 +270,9 @@ fi LD_LIBRARY_PATH="${HOME}/TorBrowser/Tor/" export LD_LIBRARY_PATH +export SELFRANDO_write_layout_file= +export NSS_DISABLE_HW_AES=1 + # We need to disable LSan which is enabled by default now. Otherwise we'll get # a crash during shutdown: https://bugs.torproject.org/10599#comment:59 ASAN_OPTIONS="detect_leaks=0" diff --git a/gitian/descriptors/linux/gitian-firefox.yml b/gitian/descriptors/linux/gitian-firefox.yml index e24a740..22686b7 100644 --- a/gitian/descriptors/linux/gitian-firefox.yml +++ b/gitian/descriptors/linux/gitian-firefox.yml @@ -29,12 +29,15 @@ remotes: "dir": "tor-browser" - "url": "https://github.com/wolfcw/libfaketime" "dir": "faketime" +- "url": "https://github.com/immunant/selfrando.git" + "dir": "selfrando" files: - "binutils-linux64-utils.zip" - "gcc-linux64-utils.zip" - "re-dzip.sh" - "dzip.sh" - "versions" +- "self-rando-utils.zip" script: | source versions INSTDIR="$HOME/install" @@ -49,6 +52,7 @@ script: | export DEB_BUILD_HARDENING_FORMAT=1 export DEB_BUILD_HARDENING_PIE=1 # + unzip -d $INSTDIR self-rando-utils.zip # Preparing Binutils and GCC for Tor Browser unzip -d $INSTDIR binutils-linux$GBUILD_BITS-utils.zip # Make sure gold is used with the hardening wrapper for full RELRO, see @@ -96,6 +100,9 @@ script: | # that feature it would hang sometimes for unknown but to libfaketime related # reasons. export LD_PRELOAD="" + # Self-Rando wrapper + export PATH="$HOME/build/selfrando/Tools/TorBrowser/tc-wrapper/:$PATH" + export SELFRANDO_skip_shuffle= make -f client.mk configure CONFIGURE_ARGS="--with-tor-browser-version=${TORBROWSER_VERSION} --enable-update-channel=${TORBROWSER_UPDATE_CHANNEL} --enable-bundled-fonts" # We need libfaketime for all the timestamps e.g. written into the libraries. # BUT we need to exclude |make build| from it. Otherwise the build fails close diff --git a/gitian/descriptors/linux/gitian-utils.yml b/gitian/descriptors/linux/gitian-utils.yml index 72ead1b..15b26f4 100644 --- a/gitian/descriptors/linux/gitian-utils.yml +++ b/gitian/descriptors/linux/gitian-utils.yml @@ -24,10 +24,14 @@ packages: - "libssl-dev" # Needed for binutils (64bit) as we are building with PIE enabled. - "libstdc++6-4.7-pic" +# Needed for Selfrando +- "scons" reference_datetime: "2000-01-01 00:00:00" remotes: - "url": "https://github.com/libevent/libevent.git" "dir": "libevent" +- "url": "https://github.com/immunant/selfrando.git" + "dir": "selfrando" files: - "binutils.tar.bz2" - "gcc.tar.bz2" @@ -36,6 +40,7 @@ files: - "versions" - "dzip.sh" - "libfaketime-asan.patch" +- "elfutils.tar.bz2" script: | INSTDIR="$HOME/install" source versions @@ -56,11 +61,8 @@ script: | # The libstdc++ shipped by default is non-PIC which breaks the binutils build # if we build with DEB_BUILD_HARDENING_PIE=1. We need to install a PIC one AND # make sure it gets used before the non-PIC one would. - if [ $GBUILD_BITS == "64" ]; - then - ln -s /usr/lib/gcc/x86_64-linux-gnu/4.7/libstdc++_pic.a libstdc++.a - export LDFLAGS="-L/home/debian -lstdc++" - fi + ln -s /usr/lib/gcc/x86_64-linux-gnu/4.7/libstdc++_pic.a libstdc++.a + export LDFLAGS="-L/home/debian -lstdc++" cd binutils* # We want to use gold as the linker in our toolchain mainly as it is way # faster when linking Tor Browser code (especially libxul). But apart from @@ -90,6 +92,31 @@ script: | cd .. export DEB_BUILD_HARDENING_FORMAT=1 + OPATH="$PATH" + OLD_LIBRARY_PATH="$LD_LIBRARY_PATH" + export PATH="$INSTDIR/binutils/bin:$INSTDIR/gcc/bin:$PATH" + export LD_LIBRARY_PATH="$INSTDIR/gcc/lib64/:$LD_LIBRARY_PATH" + + # Building Elfutils + tar xjf elfutils.tar.bz2 + cd elfutils*/ + ./configure --prefix=$INSTDIR/elfutils + make $MAKEOPTS + make install + cd .. + + # Building Selfrando + cd selfrando + scons -Q arch=x86_64 ADDN_CCFLAGS="-include $(ls $HOME/build/elfutils-*/libelf/elf.h) -I $INSTDIR/elfutils/include -fno-reorder-functions" ADDN_LINKFLAGS="-L $INSTDIR/elfutils/lib" + mkdir -p $INSTDIR/self-rando + cp sconsRelease/x86_64/bin/* $INSTDIR/self-rando/ + cd .. + + PATH="$OPATH" + unset OPATH + LD_LIBRARY_PATH="$OLD_LIBRARY_PATH" + unset OLD_LIBRARY_PATH + # Building Libevent cd libevent ./autogen.sh @@ -134,4 +161,5 @@ script: | ~/build/dzip.sh openssl-$OPENSSL_VER-linux$GBUILD_BITS-utils.zip openssl ~/build/dzip.sh libevent-${LIBEVENT_TAG#release-}-linux$GBUILD_BITS-utils.zip libevent ~/build/dzip.sh gmp-$GMP_VER-linux$GBUILD_BITS-utils.zip gmp + ~/build/dzip.sh self-rando-utils.zip self-rando cp *utils.zip $OUTDIR/ diff --git a/gitian/fetch-inputs.sh b/gitian/fetch-inputs.sh index 027a47e..4f63374 100755 --- a/gitian/fetch-inputs.sh +++ b/gitian/fetch-inputs.sh @@ -112,7 +112,7 @@ update_git() { ############################################################################## # Get+verify sigs that exist -for i in OPENSSL BINUTILS GCC PYCRYPTO PYTHON_MSI GMP +for i in OPENSSL BINUTILS GCC PYCRYPTO PYTHON_MSI GMP ELFUTILS do PACKAGE="${i}_PACKAGE" URL="${i}_URL" @@ -236,6 +236,7 @@ ln -sf "$SETUPTOOLS_PACKAGE" setuptools.tar.gz ln -sf "$GMP_PACKAGE" gmp.tar.bz2 ln -sf "$PARSLEY_PACKAGE" parsley.tar.gz ln -sf "$GO_PACKAGE" go.tar.gz +ln -sf "$ELFUTILS_PACKAGE" elfutils.tar.bz2 # Fetch latest gitian-builder itself # XXX - this is broken if a non-standard inputs dir is selected using the command line flag. @@ -273,6 +274,7 @@ goxcrypto https://go.googlesource.com/crypto $GO_X_CRYPTO_TAG goxnet https://go.googlesource.com/net $GO_X_NET_TAG obfs4 https://git.torproject.org/pluggable-transports/obfs4.git $OBFS4_TAG noto-fonts https://github.com/googlei18n/noto-fonts $NOTOFONTS_TAG +selfrando https://github.com/immunant/selfrando.git $SELFRANDO_TAG EOF # HTTPS-Everywhere is special, too. We need to initialize the git submodules and diff --git a/gitian/gpg/ELFUTILS.gpg b/gitian/gpg/ELFUTILS.gpg new file mode 100644 index 0000000..f1cd4b3 Binary files /dev/null and b/gitian/gpg/ELFUTILS.gpg differ diff --git a/gitian/mkbundle-linux.sh b/gitian/mkbundle-linux.sh index a1674dd..48aac24 100755 --- a/gitian/mkbundle-linux.sh +++ b/gitian/mkbundle-linux.sh @@ -35,7 +35,7 @@ fi if [ -z "$VM_MEMORY" ]; then - export VM_MEMORY=4000 + export VM_MEMORY=6000 fi ./make-vms.sh @@ -104,13 +104,14 @@ if [ ! -f inputs/binutils-$BINUTILS_VER-linux64-utils.zip -o \ ! -f inputs/gcc-$GCC_VER-linux64-utils.zip -o \ ! -f inputs/openssl-$OPENSSL_VER-linux64-utils.zip -o \ ! -f inputs/libevent-${LIBEVENT_TAG_ORIG#release-}-linux64-utils.zip -o \ - ! -f inputs/gmp-$GMP_VER-linux64-utils.zip ]; + ! -f inputs/gmp-$GMP_VER-linux64-utils.zip -o \ + ! -f inputs/self-rando-utils.zip ]; then echo echo "****** Starting Utilities Component of Linux Bundle (1/5 for Linux) ******" echo - ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit libevent=$LIBEVENT_TAG $DESCRIPTOR_DIR/linux/gitian-utils.yml + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit libevent=$LIBEVENT_TAG,selfrando=$SELFRANDO_TAG $DESCRIPTOR_DIR/linux/gitian-utils.yml if [ $? -ne 0 ]; then #mv var/build.log ./utils-fail-linux.log.`date +%Y%m%d%H%M%S` @@ -169,7 +170,7 @@ then echo "****** Starting TorBrowser Component of Linux Bundle (3/5 for Linux) ******" echo - ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor-browser=$TORBROWSER_TAG,faketime=$FAKETIME_TAG $DESCRIPTOR_DIR/linux/gitian-firefox.yml + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor-browser=$TORBROWSER_TAG,faketime=$FAKETIME_TAG,selfrando=$SELFRANDO_TAG $DESCRIPTOR_DIR/linux/gitian-firefox.yml if [ $? -ne 0 ]; then #mv var/build.log ./firefox-fail-linux.log.`date +%Y%m%d%H%M%S` diff --git a/gitian/patches/libfaketime-asan.patch b/gitian/patches/libfaketime-asan.patch index 5346eae..ff3e62b 100644 --- a/gitian/patches/libfaketime-asan.patch +++ b/gitian/patches/libfaketime-asan.patch @@ -1,12 +1,3 @@ -From 0ef83e9a0631e25e62e1dbc483108742fc125304 Mon Sep 17 00:00:00 2001 -From: Georg Koppen <gk(a)torproject.org> -Date: Mon, 14 Sep 2015 14:27:36 +0000 -Subject: [PATCH] No dying anymore - ---- - libsanitizer/asan/asan_linux.cc | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - diff --git a/libsanitizer/asan/asan_linux.cc b/libsanitizer/asan/asan_linux.cc index c504168..1d1a6dd 100644 --- a/libsanitizer/asan/asan_linux.cc @@ -23,6 +14,259 @@ index c504168..1d1a6dd 100644 } } --- -1.7.10.4 - +diff --git a/libsanitizer/sanitizer_common/SelfRandoAddressTranslator.h b/libsanitizer/sanitizer_common/SelfRandoAddressTranslator.h +new file mode 100644 +index 0000000..5f13870 +--- /dev/null ++++ b/libsanitizer/sanitizer_common/SelfRandoAddressTranslator.h +@@ -0,0 +1,155 @@ ++// ++// Created by tf on 10/12/2015. ++// ++ ++#pragma once ++#include "sanitizer_internal_defs.h" ++#include "sanitizer_libc.h" ++#include "sanitizer_common.h" ++ ++#define PROT_READ 0x1 /* Page can be read. */ ++#define PROT_WRITE 0x2 /* Page can be written. */ ++#define PROT_EXEC 0x4 /* Page can be executed. */ ++#define PROT_NONE 0x0 /* Page can not be accessed. */ ++#define MAP_SHARED 0x01 /* Share changes. */ ++ ++namespace __sanitizer { ++ class SelfRandoAddressTranslator { ++ struct __attribute__((__packed__)) FunctionRecord { ++ void *undiv_addr; ++ void *div_addr; ++ u32 size; ++ }; ++ ++ struct __attribute__((__packed__)) LibraryRecord { ++ u32 version; ++ u32 seed; ++ void *file_base; ++ void *mem_base; ++ void *size; ++ ++ private: ++ char _name; ++ ++ public: ++ inline char *name() { return &_name; } ++ ++ inline FunctionRecord *function_records() { ++ return (FunctionRecord *) (&_name + internal_strlen(&_name) + 1); ++ } ++ ++ u8 *record_end() { ++ FunctionRecord *fr = function_records(); ++ while (fr->undiv_addr) { ++ fr++; ++ } ++ return (u8 *) fr + sizeof(void *); ++ } ++ }; ++ ++ // 16K loaded modules should be enough for everyone. ++ LibraryRecord* library_records_idx[1<<14]; ++ int library_records_idx_size; ++ u8 *library_records_end; ++ void add_library_record(LibraryRecord* lr); ++ ++ void read_layout_file(); ++ u8 *record_end(int i); ++ ++ public: ++ SelfRandoAddressTranslator(); ++ ++ uptr convertAddress(uptr original_address); ++ }; ++ ++ ++ ++ ++ SelfRandoAddressTranslator::SelfRandoAddressTranslator() { ++ Printf("SRAT init\n"); ++ read_layout_file(); ++ } ++ ++ void SelfRandoAddressTranslator::add_library_record(LibraryRecord* lr) { ++ library_records_idx_size++; ++ CHECK(library_records_idx_size <= 1<<14); ++ library_records_idx[library_records_idx_size - 1] = lr; ++ } ++ ++ u8* SelfRandoAddressTranslator::record_end(int i) { ++ if (i < library_records_idx_size - 1) { ++ return (u8 *) library_records_idx[i + 1] - sizeof(void *); ++ } else { ++ return library_records_end - sizeof(void*); ++ } ++ } ++ ++ void SelfRandoAddressTranslator::read_layout_file() { ++ uptr pid = internal_getpid(); ++ char filename[32]; ++ internal_snprintf(filename, 32, "/tmp/%d.mlf", pid); ++ ++ u8 buf[200]; // Current size of struct stat: 144 ++ if(0 != internal_stat(filename, &buf)) { ++ Printf("SRAT warning: cannot find Memory Layout File. Stack trace will be probably wrong.\n"); ++ return; ++ } ++ ++ uptr fd = OpenFile(filename, /*write*/ false); ++ uptr size = *(uptr*) (buf+48); ++ CHECK_NE(fd, -1); ++ u8 *f = (u8 *) internal_mmap(0, size, PROT_READ, MAP_SHARED, (int) fd, 0); ++ CHECK_NE(f, 0); ++ library_records_end = f + size; ++ ++ u8 *p = f; ++ while (p < library_records_end) { ++ LibraryRecord *lr = (LibraryRecord *) p; ++ add_library_record(lr); ++ CHECK_EQ(lr->version, 0x00000101); ++ p = lr->record_end(); ++ } ++ ++ } ++ ++ uptr SelfRandoAddressTranslator::convertAddress(uptr p) { ++// Printf("conv %p", p); ++ for (int i = 0; i < library_records_idx_size; ++i) { ++ LibraryRecord *lr = library_records_idx[i]; ++ ++ if (p > (uptr) lr->mem_base && p < (uptr) lr->mem_base + (uptr) lr->size) { ++ // found the correct library ++ ++ FunctionRecord *function_records = lr->function_records(); ++ ++ int a = 0; ++ int b = (int) ((FunctionRecord *) record_end(i) - function_records); ++// Printf(" (lib=>%s< a=%d b=%d)", lr->name(), a, b); ++ CHECK_EQ((u8 *) (function_records + b), record_end(i)); ++ ++ while (b - a > 1) { // bisection ++ int c = a + (b - a) / 2; ++ FunctionRecord *fr = &function_records[c]; ++ ++ if ((uptr) fr->div_addr > p) { // too high ++ b = c; ++ } else if ((uptr) fr->div_addr + (uptr) fr->size < p) { // too low ++ a = c; ++ } else { // found ++ b = c + 1; ++ a = c; ++ } ++ } ++ FunctionRecord *fr = &function_records[a]; ++ CHECK((uptr) fr->div_addr < p && p < (uptr) fr->div_addr + (uptr) fr->size); ++ ++ p = (p - (uptr) fr->div_addr + (uptr) fr->undiv_addr); ++ ++ break; ++ } ++ } ++ ++// Printf(" => %p\n", p); ++ return p; ++ } ++} +\ No newline at end of file +diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc +index 5cc21d3..393a494 100644 +--- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc ++++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc +@@ -21,6 +21,7 @@ + #include "sanitizer_procmaps.h" + #include "sanitizer_symbolizer.h" + #include "sanitizer_symbolizer_libbacktrace.h" ++#include "SelfRandoAddressTranslator.h" + + #include <errno.h> + #include <stdlib.h> +@@ -512,7 +513,7 @@ class POSIXSymbolizer : public Symbolizer { + internal_symbolizer_(internal_symbolizer), + libbacktrace_symbolizer_(libbacktrace_symbolizer) {} + +- uptr SymbolizePC(uptr addr, AddressInfo *frames, uptr max_frames) { ++ virtual uptr SymbolizePC(uptr addr, AddressInfo *frames, uptr max_frames) { + BlockingMutexLock l(&mu_); + if (max_frames == 0) + return 0; +@@ -580,7 +581,7 @@ class POSIXSymbolizer : public Symbolizer { + return frame_id; + } + +- bool SymbolizeData(uptr addr, DataInfo *info) { ++ virtual bool SymbolizeData(uptr addr, DataInfo *info) { + BlockingMutexLock l(&mu_); + LoadedModule *module = FindModuleForAddress(addr); + if (module == 0) +@@ -606,7 +607,7 @@ class POSIXSymbolizer : public Symbolizer { + return true; + } + +- bool GetModuleNameAndOffsetForPC(uptr pc, const char **module_name, ++ virtual bool GetModuleNameAndOffsetForPC(uptr pc, const char **module_name, + uptr *module_address) { + BlockingMutexLock l(&mu_); + return FindModuleNameAndOffsetForAddress(pc, module_name, module_address); +@@ -720,9 +721,40 @@ class POSIXSymbolizer : public Symbolizer { + LibbacktraceSymbolizer *libbacktrace_symbolizer_; // Leaked. + }; + ++ ++ class SelfRandoPOSIXSymbolizer : public POSIXSymbolizer { ++ SelfRandoAddressTranslator translator; ++ ++ public: ++ SelfRandoPOSIXSymbolizer(ExternalSymbolizerInterface *external_symbolizer, ++ InternalSymbolizer *internal_symbolizer, ++ LibbacktraceSymbolizer *libbacktrace_symbolizer) ++ : POSIXSymbolizer(external_symbolizer, internal_symbolizer, libbacktrace_symbolizer), ++ translator() { } ++ ++ ++ private: ++ virtual uptr SymbolizePC(uptr addr, AddressInfo *frames, uptr max_frames) override { ++ uptr conv_addr = translator.convertAddress(addr); ++ uptr res = POSIXSymbolizer::SymbolizePC(conv_addr, frames, max_frames); ++ if (res == 1) frames->address = addr; ++ return res; ++ } ++ ++ virtual bool SymbolizeData(uptr addr, DataInfo *info) override { ++ addr = translator.convertAddress(addr); ++ return POSIXSymbolizer::SymbolizeData(addr, info); ++ } ++ ++ virtual bool GetModuleNameAndOffsetForPC(uptr pc, const char **module_name, uptr *module_address) override { ++ pc = translator.convertAddress(pc); ++ return POSIXSymbolizer::GetModuleNameAndOffsetForPC(pc, module_name, module_address); ++ }; ++ }; ++ + Symbolizer *Symbolizer::PlatformInit() { + if (!common_flags()->symbolize) { +- return new(symbolizer_allocator_) POSIXSymbolizer(0, 0, 0); ++ return new(symbolizer_allocator_) SelfRandoPOSIXSymbolizer(0, 0, 0); + } + InternalSymbolizer* internal_symbolizer = + InternalSymbolizer::get(&symbolizer_allocator_); +@@ -754,10 +786,11 @@ Symbolizer *Symbolizer::PlatformInit() { + } + } + +- return new(symbolizer_allocator_) POSIXSymbolizer( ++ return new(symbolizer_allocator_) SelfRandoPOSIXSymbolizer( + external_symbolizer, internal_symbolizer, libbacktrace_symbolizer); + } + ++ + } // namespace __sanitizer + + #endif // SANITIZER_POSIX diff --git a/gitian/verify-tags.sh b/gitian/verify-tags.sh index 46cb01a..5268f85 100755 --- a/gitian/verify-tags.sh +++ b/gitian/verify-tags.sh @@ -118,10 +118,11 @@ siphash $GOSIPHASH_TAG goxcrypto $GO_X_CRYPTO_TAG goxnet $GO_X_NET_TAG noto-fonts $NOTOFONTS_TAG +selfrando $SELFRANDO_TAG EOF # Verify signatures on signed packages -for i in OPENSSL BINUTILS GCC PYCRYPTO PYTHON_MSI GMP +for i in OPENSSL BINUTILS GCC PYCRYPTO PYTHON_MSI GMP ELFUTILS do PACKAGE="${i}_PACKAGE" URL="${i}_URL" diff --git a/gitian/versions.nightly b/gitian/versions.nightly index 088625b..9ff7e8f 100755 --- a/gitian/versions.nightly +++ b/gitian/versions.nightly @@ -48,6 +48,7 @@ GO_X_CRYPTO_TAG=master GO_X_NET_TAG=master OBFS4_TAG=master NOTOFONTS_TAG=720e34851382ee3c1ef024d8dffb68ffbfb234c2 +SELFRANDO_TAG=8e49e7aa664625d8e4793579864231b151589d0f GITIAN_TAG=tor-browser-builder-4 @@ -69,6 +70,7 @@ SETUPTOOLS_VER=1.4 PARSLEY_VER=1.2 GO_VER=1.4.2 NSIS_VER=2.51 +ELFUTILS_VER=0.160 ## File names for the source packages OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz @@ -99,6 +101,7 @@ NOTOJPFONT_PACKAGE=NotoSansJP-Regular.otf NOTOKRFONT_PACKAGE=NotoSansKR-Regular.otf NOTOSCFONT_PACKAGE=NotoSansSC-Regular.otf NOTOTCFONT_PACKAGE=NotoSansTC-Regular.otf +ELFUTILS_PACKAGE=elfutils-${ELFUTILS_VER}.tar.bz2 # Hashes for packages with weak sigs or no sigs OPENSSL_HASH=e7e81d82f3cd538ab0cdba494006d44aab9dd96b7f6233ce9971fb7c7916d511 @@ -157,3 +160,4 @@ NOTOJPFONT_URL=${NOTOCJKBASE_URL}/${NOTOJPFONT_PACKAGE} NOTOKRFONT_URL=${NOTOCJKBASE_URL}/${NOTOKRFONT_PACKAGE} NOTOSCFONT_URL=${NOTOCJKBASE_URL}/${NOTOSCFONT_PACKAGE} NOTOTCFONT_URL=${NOTOCJKBASE_URL}/${NOTOTCFONT_PACKAGE} +ELFUTILS_URL=https://fedorahosted.org/releases/e/l/elfutils/${ELFUTILS_VER}/${ELFUTILS_PACKAGE}

1 0

[collector/master] Remove unused bridge pool assignments code.
by karsten＠torproject.org 12 May '16

12 May '16

commit 393a8d6d5b8f4d037127bcdddca49e6fc43076d5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu May 12 21:43:59 2016 +0200 Remove unused bridge pool assignments code. --- bin/run-bridgepools | 3 - config.template | 10 - src/org/torproject/collector/Main.java | 2 - .../BridgePoolAssignmentsProcessor.java | 273 --------------------- .../torproject/collector/main/Configuration.java | 15 -- 5 files changed, 303 deletions(-) diff --git a/bin/run-bridgepools b/bin/run-bridgepools deleted file mode 100755 index 62d71f7..0000000 --- a/bin/run-bridgepools +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -`dirname $0`/run-all `basename $0` - diff --git a/config.template b/config.template index cd31802..88407a2 100644 --- a/config.template +++ b/config.template @@ -85,16 +85,6 @@ #SanitizedBridgesWriteDirectory out/bridge-descriptors/ # # -######## Bridge pool assignments ######## -# -## Relative path to directory to read bridge pool assignment files from -#AssignmentsDirectory in/bridge-pool-assignments/ -# -## Relative path to directory to write sanitized bridge pool assignment -## files to -#SanitizedAssignmentsDirectory out/bridge-pool-assignments/ -# -# ######## Exit lists ######## # ## (No options available) diff --git a/src/org/torproject/collector/Main.java b/src/org/torproject/collector/Main.java index 44182ed..9c64696 100644 --- a/src/org/torproject/collector/Main.java +++ b/src/org/torproject/collector/Main.java @@ -4,7 +4,6 @@ package org.torproject.collector; import org.torproject.collector.bridgedescs.SanitizedBridgesWriter; -import org.torproject.collector.bridgepools.BridgePoolAssignmentsProcessor; import org.torproject.collector.exitlists.ExitListDownloader; import org.torproject.collector.index.CreateIndexJson; import org.torproject.collector.relaydescs.ArchiveWriter; @@ -33,7 +32,6 @@ public class Main { static { // add a new main class here collecTorMains.put("bridgedescs", SanitizedBridgesWriter.class); - collecTorMains.put("bridgepools", BridgePoolAssignmentsProcessor.class); collecTorMains.put("exitlists", ExitListDownloader.class); collecTorMains.put("updateindex", CreateIndexJson.class); collecTorMains.put("relaydescs", ArchiveWriter.class); diff --git a/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java deleted file mode 100644 index 2b6c262..0000000 --- a/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java +++ /dev/null @@ -1,273 +0,0 @@ -/* Copyright 2011--2016 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.collector.bridgepools; - -import org.torproject.collector.main.Configuration; -import org.torproject.collector.main.LockFile; - -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStreamReader; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.SortedSet; -import java.util.Stack; -import java.util.TimeZone; -import java.util.TreeSet; -import java.util.logging.Level; -import java.util.logging.Logger; - -public class BridgePoolAssignmentsProcessor extends Thread { - - public static void main(String[] args) { - - Logger logger = Logger.getLogger( - BridgePoolAssignmentsProcessor.class.getName()); - logger.info("Starting bridge-pool-assignments module of CollecTor."); - - // Initialize configuration - Configuration config = new Configuration(); - - // Use lock file to avoid overlapping runs - LockFile lf = new LockFile("bridge-pool-assignments"); - if (!lf.acquireLock()) { - logger.severe("Warning: CollecTor is already running or has not exited " - + "cleanly! Exiting!"); - System.exit(1); - } - - // Process bridge pool assignments - new BridgePoolAssignmentsProcessor(config).run(); - - // Remove lock file - lf.releaseLock(); - - logger.info("Terminating bridge-pool-assignments module of CollecTor."); - } - - private Configuration config; - - public BridgePoolAssignmentsProcessor(Configuration config) { - this.config = config; - } - - public void run() { - - File assignmentsDirectory = - new File(config.getAssignmentsDirectory()); - File sanitizedAssignmentsDirectory = - new File(config.getSanitizedAssignmentsDirectory()); - - Logger logger = - Logger.getLogger(BridgePoolAssignmentsProcessor.class.getName()); - if (assignmentsDirectory == null - || sanitizedAssignmentsDirectory == null) { - IllegalArgumentException e = new IllegalArgumentException("Neither " - + "assignmentsDirectory nor sanitizedAssignmentsDirectory may " - + "be null!"); - throw e; - } - - List<File> assignmentFiles = new ArrayList<File>(); - Stack<File> files = new Stack<File>(); - files.add(assignmentsDirectory); - while (!files.isEmpty()) { - File file = files.pop(); - if (file.isDirectory()) { - files.addAll(Arrays.asList(file.listFiles())); - } else if (file.getName().equals("assignments.log")) { - assignmentFiles.add(file); - } - } - - SimpleDateFormat assignmentFormat = - new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - assignmentFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - SimpleDateFormat filenameFormat = - new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss"); - filenameFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String duplicateFingerprint = null; - long maxBridgePoolAssignmentTime = 0L; - for (File assignmentFile : assignmentFiles) { - logger.info("Processing bridge pool assignment file '" - + assignmentFile.getAbsolutePath() + "'..."); - try { - BufferedReader br = null; - if (assignmentFile.getName().endsWith(".gz")) { - br = new BufferedReader(new InputStreamReader( - new GzipCompressorInputStream(new FileInputStream( - assignmentFile)))); - } else { - br = new BufferedReader(new FileReader(assignmentFile)); - } - String line; - String bridgePoolAssignmentLine = null; - SortedSet<String> sanitizedAssignments = new TreeSet<String>(); - boolean wroteLastLine = false; - boolean skipBefore20120504125947 = true; - Set<String> hashedFingerprints = null; - while ((line = br.readLine()) != null || !wroteLastLine) { - if (line != null && line.startsWith("bridge-pool-assignment ")) { - String[] parts = line.split(" "); - if (parts.length != 3) { - continue; - } - /* TODO Take out this temporary hack to ignore all assignments - * coming from ponticum when byblos was still the official - * BridgeDB host. */ - if (line.compareTo( - "bridge-pool-assignment 2012-05-04 12:59:47") >= 0) { - skipBefore20120504125947 = false; - } - } - if (skipBefore20120504125947) { - if (line == null) { - break; - } else { - continue; - } - } - if (line == null - || line.startsWith("bridge-pool-assignment ")) { - if (bridgePoolAssignmentLine != null) { - try { - long bridgePoolAssignmentTime = assignmentFormat.parse( - bridgePoolAssignmentLine.substring( - "bridge-pool-assignment ".length())).getTime(); - maxBridgePoolAssignmentTime = Math.max( - maxBridgePoolAssignmentTime, - bridgePoolAssignmentTime); - File tarballFile = new File( - sanitizedAssignmentsDirectory, filenameFormat.format( - bridgePoolAssignmentTime)); - File rsyncFile = new File( - "recent/bridge-pool-assignments/" - + tarballFile.getName()); - File[] outputFiles = new File[] { tarballFile, - rsyncFile }; - for (File outputFile : outputFiles) { - if (!outputFile.exists()) { - outputFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - outputFile)); - bw.write("@type bridge-pool-assignment 1.0\n"); - bw.write(bridgePoolAssignmentLine + "\n"); - for (String assignmentLine : sanitizedAssignments) { - bw.write(assignmentLine + "\n"); - } - bw.close(); - } - } - } catch (IOException e) { - logger.log(Level.WARNING, "Could not write sanitized " - + "bridge pool assignment file for line '" - + bridgePoolAssignmentLine + "' to disk. Skipping " - + "bridge pool assignment file '" - + assignmentFile.getAbsolutePath() + "'.", e); - break; - } catch (ParseException e) { - logger.log(Level.WARNING, "Could not write sanitized " - + "bridge pool assignment file for line '" - + bridgePoolAssignmentLine + "' to disk. Skipping " - + "bridge pool assignment file '" - + assignmentFile.getAbsolutePath() + "'.", e); - break; - } - sanitizedAssignments.clear(); - } - if (line == null) { - wroteLastLine = true; - } else { - bridgePoolAssignmentLine = line; - hashedFingerprints = new HashSet<String>(); - } - } else { - String[] parts = line.split(" "); - if (parts.length < 2 || parts[0].length() < 40) { - logger.warning("Unrecognized line '" + line - + "'. Aborting."); - break; - } - String hashedFingerprint = null; - try { - hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex( - line.split(" ")[0].toCharArray())).toLowerCase(); - } catch (DecoderException e) { - logger.warning("Unable to decode hex fingerprint in line '" - + line + "'. Aborting."); - break; - } - if (hashedFingerprints.contains(hashedFingerprint)) { - duplicateFingerprint = bridgePoolAssignmentLine; - } - hashedFingerprints.add(hashedFingerprint); - String assignmentDetails = line.substring(40); - sanitizedAssignments.add(hashedFingerprint - + assignmentDetails); - } - } - br.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Could not read bridge pool assignment " - + "file '" + assignmentFile.getAbsolutePath() - + "'. Skipping.", e); - } - } - - if (duplicateFingerprint != null) { - logger.warning("At least one bridge pool assignment list contained " - + "duplicate fingerprints. Last found in assignment list " - + "starting with '" + duplicateFingerprint + "'."); - } - - if (maxBridgePoolAssignmentTime > 0L - && maxBridgePoolAssignmentTime + 330L * 60L * 1000L - < System.currentTimeMillis()) { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - logger.warning("The last known bridge pool assignment list was " - + "published at " - + dateTimeFormat.format(maxBridgePoolAssignmentTime) - + ", which is more than 5:30 hours in the past."); - } - - this.cleanUpRsyncDirectory(); - - logger.info("Finished processing bridge pool assignment file(s)."); - } - - /* Delete all files from the rsync directory that have not been modified - * in the last three days. */ - public void cleanUpRsyncDirectory() { - long cutOffMillis = System.currentTimeMillis() - - 3L * 24L * 60L * 60L * 1000L; - Stack<File> allFiles = new Stack<File>(); - allFiles.add(new File("recent/bridge-pool-assignments")); - while (!allFiles.isEmpty()) { - File file = allFiles.pop(); - if (file.isDirectory()) { - allFiles.addAll(Arrays.asList(file.listFiles())); - } else if (file.lastModified() < cutOffMillis) { - file.delete(); - } - } - } -} - diff --git a/src/org/torproject/collector/main/Configuration.java b/src/org/torproject/collector/main/Configuration.java index 9a511d3..aee1d02 100644 --- a/src/org/torproject/collector/main/Configuration.java +++ b/src/org/torproject/collector/main/Configuration.java @@ -62,9 +62,6 @@ public class Configuration { private boolean downloadAllServerDescriptors = false; private boolean downloadAllExtraInfoDescriptors = false; private boolean compressRelayDescriptorDownloads; - private String assignmentsDirectory = "in/bridge-pool-assignments/"; - private String sanitizedAssignmentsDirectory = - "out/bridge-pool-assignments/"; private String torperfOutputDirectory = "out/torperf/"; private SortedMap<String, String> torperfSources = null; private List<String> torperfFiles = null; @@ -170,10 +167,6 @@ public class Configuration { } else if (line.startsWith("CompressRelayDescriptorDownloads")) { this.compressRelayDescriptorDownloads = Integer.parseInt( line.split(" ")[1]) != 0; - } else if (line.startsWith("AssignmentsDirectory")) { - this.assignmentsDirectory = line.split(" ")[1]; - } else if (line.startsWith("SanitizedAssignmentsDirectory")) { - this.sanitizedAssignmentsDirectory = line.split(" ")[1]; } else if (line.startsWith("TorperfOutputDirectory")) { this.torperfOutputDirectory = line.split(" ")[1]; } else if (line.startsWith("TorperfSource")) { @@ -310,14 +303,6 @@ public class Configuration { return this.compressRelayDescriptorDownloads; } - public String getAssignmentsDirectory() { - return this.assignmentsDirectory; - } - - public String getSanitizedAssignmentsDirectory() { - return this.sanitizedAssignmentsDirectory; - } - public String getTorperfOutputDirectory() { return this.torperfOutputDirectory; }

1 0

[collector/master] implements task-18922, make logging configurable.
by karsten＠torproject.org 12 May '16

12 May '16

commit f85683536bb0918f449387bf73f02385a3495bb5 Author: iwakeh <iwakeh(a)torproject.org> Date: Mon May 9 22:00:26 2016 +0000 implements task-18922, make logging configurable. --- bin/logging.properties | 20 +++++ bin/run-all | 16 ++++ bin/run-bridgedescs | 2 +- bin/run-bridgepools | 2 +- bin/run-exitlists | 2 +- bin/run-relaydescs | 2 +- bin/run-torperf | 2 +- bin/update-index | 2 +- .../bridgedescs/SanitizedBridgesWriter.java | 3 - .../BridgePoolAssignmentsProcessor.java | 3 - .../collector/exitlists/ExitListDownloader.java | 3 - .../collector/main/LoggingConfiguration.java | 96 ---------------------- .../collector/relaydescs/ArchiveWriter.java | 3 - .../collector/torperf/TorperfDownloader.java | 3 - 14 files changed, 42 insertions(+), 117 deletions(-) diff --git a/bin/logging.properties b/bin/logging.properties new file mode 100644 index 0000000..5aae3cb --- /dev/null +++ b/bin/logging.properties @@ -0,0 +1,20 @@ +handlers= java.util.logging.FileHandler, java.util.logging.ConsoleHandler + +.level = FINER + +# if the following dir pattern is changed, LOGGINGPATH='log' in run-all +# needs to be adapted. +java.util.logging.FileHandler.pattern = ./log/collector-%g.log +java.util.logging.FileHandler.limit = 5000000 +java.util.logging.FileHandler.count = 9 +java.util.logging.FileHandler.append = true +java.util.logging.FileHandler.level = FINER +java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter + +java.util.logging.ConsoleHandler.level = WARNING +java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter + +java.util.logging.SimpleFormatter.format = %1$tF %1$tT %4$s %2$s %5$s%n + +sun.net.www.level = SEVERE +sun.level = SEVERE diff --git a/bin/run-all b/bin/run-all new file mode 100755 index 0000000..469aced --- /dev/null +++ b/bin/run-all @@ -0,0 +1,16 @@ +#!/bin/sh + +TOKEN=`expr substr $1 5 20` + +# if the path is changed in properties, this needs to be adapted. +LOGGINGPATH='log' + +if ! test -d $LOGGINGPATH +then mkdir $LOGGINGPATH +fi + +# the following uses one logging configuration for all modules +java -Xmx2g -Djava.util.logging.config.file=./bin/logging.properties -jar collector-0.9.0-dev.jar $TOKEN + +# this would use a special log config for each module +# java -Xmx2g -Djava.util.logging.config.file=./bin/logging.$TOKEN.properties -jar collector-0.9.0-dev.jar $TOKEN diff --git a/bin/run-bridgedescs b/bin/run-bridgedescs index 37eeea5..62d71f7 100755 --- a/bin/run-bridgedescs +++ b/bin/run-bridgedescs @@ -1,3 +1,3 @@ #!/bin/sh -java -Xmx2g -jar collector-0.9.0-dev.jar bridgedescs +`dirname $0`/run-all `basename $0` diff --git a/bin/run-bridgepools b/bin/run-bridgepools index 9c5c53a..62d71f7 100755 --- a/bin/run-bridgepools +++ b/bin/run-bridgepools @@ -1,3 +1,3 @@ #!/bin/sh -java -Xmx2g -jar collector-0.9.0-dev.jar bridgepools +`dirname $0`/run-all `basename $0` diff --git a/bin/run-exitlists b/bin/run-exitlists index 3bdef11..62d71f7 100755 --- a/bin/run-exitlists +++ b/bin/run-exitlists @@ -1,3 +1,3 @@ #!/bin/sh -java -Xmx2g -jar collector-0.9.0-dev.jar exitlists +`dirname $0`/run-all `basename $0` diff --git a/bin/run-relaydescs b/bin/run-relaydescs index 8ad7506..62d71f7 100755 --- a/bin/run-relaydescs +++ b/bin/run-relaydescs @@ -1,3 +1,3 @@ #!/bin/sh -java -Xmx2g -jar collector-0.9.0-dev.jar relaydescs +`dirname $0`/run-all `basename $0` diff --git a/bin/run-torperf b/bin/run-torperf index 267df58..62d71f7 100755 --- a/bin/run-torperf +++ b/bin/run-torperf @@ -1,3 +1,3 @@ #!/bin/sh -java -Xmx2g -jar collector-0.9.0-dev.jar torperf +`dirname $0`/run-all `basename $0` diff --git a/bin/update-index b/bin/update-index index 70b0067..6e6c488 100755 --- a/bin/update-index +++ b/bin/update-index @@ -1,3 +1,3 @@ #!/bin/sh -java -Xmx2g -jar collector-0.9.0-dev.jar updateindex +`dirname $0`/run-all run-updateindex diff --git a/src/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index f8d61c0..3214715 100644 --- a/src/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -5,7 +5,6 @@ package org.torproject.collector.bridgedescs; import org.torproject.collector.main.Configuration; import org.torproject.collector.main.LockFile; -import org.torproject.collector.main.LoggingConfiguration; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Base64; @@ -52,8 +51,6 @@ public class SanitizedBridgesWriter extends Thread { public static void main(String[] args) { - /* Initialize logging configuration. */ - new LoggingConfiguration("bridge-descriptors"); Logger logger = Logger.getLogger( SanitizedBridgesWriter.class.getName()); logger.info("Starting bridge-descriptors module of CollecTor."); diff --git a/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java index 8afaa43..2b6c262 100644 --- a/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java +++ b/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java @@ -5,7 +5,6 @@ package org.torproject.collector.bridgepools; import org.torproject.collector.main.Configuration; import org.torproject.collector.main.LockFile; -import org.torproject.collector.main.LoggingConfiguration; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; @@ -38,8 +37,6 @@ public class BridgePoolAssignmentsProcessor extends Thread { public static void main(String[] args) { - /* Initialize logging configuration. */ - new LoggingConfiguration("bridge-pool-assignments"); Logger logger = Logger.getLogger( BridgePoolAssignmentsProcessor.class.getName()); logger.info("Starting bridge-pool-assignments module of CollecTor."); diff --git a/src/org/torproject/collector/exitlists/ExitListDownloader.java b/src/org/torproject/collector/exitlists/ExitListDownloader.java index d328850..54fd50f 100644 --- a/src/org/torproject/collector/exitlists/ExitListDownloader.java +++ b/src/org/torproject/collector/exitlists/ExitListDownloader.java @@ -5,7 +5,6 @@ package org.torproject.collector.exitlists; import org.torproject.collector.main.Configuration; import org.torproject.collector.main.LockFile; -import org.torproject.collector.main.LoggingConfiguration; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DescriptorParser; @@ -34,8 +33,6 @@ public class ExitListDownloader extends Thread { public static void main(String[] args) { - /* Initialize logging configuration. */ - new LoggingConfiguration("exit-lists"); Logger logger = Logger.getLogger(ExitListDownloader.class.getName()); logger.info("Starting exit-lists module of CollecTor."); diff --git a/src/org/torproject/collector/main/LoggingConfiguration.java b/src/org/torproject/collector/main/LoggingConfiguration.java deleted file mode 100644 index b8a91c7..0000000 --- a/src/org/torproject/collector/main/LoggingConfiguration.java +++ /dev/null @@ -1,96 +0,0 @@ -/* Copyright 2010--2016 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.collector.main; - -import java.io.File; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.TimeZone; -import java.util.logging.ConsoleHandler; -import java.util.logging.FileHandler; -import java.util.logging.Formatter; -import java.util.logging.Handler; -import java.util.logging.Level; -import java.util.logging.LogRecord; -import java.util.logging.Logger; - -/** - * Initialize logging configuration. - * - * Log levels used by CollecTor: - * - * - SEVERE: An event made it impossible to continue program execution. - * - WARNING: A potential problem occurred that requires the operator to - * look after the otherwise unattended setup - * - INFO: Messages on INFO level are meant to help the operator in making - * sure that operation works as expected. - * - FINE: Debug messages that are used to identify problems and which are - * turned on by default. - * - FINER: More detailed debug messages to investigate problems in more - * detail. Not turned on by default. Increase log file limit when using - * FINER. - * - FINEST: Most detailed debug messages. Not used. - */ -public class LoggingConfiguration { - public LoggingConfiguration(String moduleName) { - - /* Remove default console handler. */ - for (Handler h : Logger.getLogger("").getHandlers()) { - Logger.getLogger("").removeHandler(h); - } - - /* Disable logging of internal Sun classes. */ - Logger.getLogger("sun").setLevel(Level.OFF); - - /* Set minimum log level we care about from INFO to FINER. */ - Logger.getLogger("").setLevel(Level.FINER); - - /* Create log handler that writes messages on WARNING or higher to the - * console. */ - final SimpleDateFormat dateTimeFormat = - new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - Formatter cf = new Formatter() { - public String format(LogRecord record) { - return dateTimeFormat.format(new Date(record.getMillis())) + " " - + record.getMessage() + "\n"; - } - }; - Handler ch = new ConsoleHandler(); - ch.setFormatter(cf); - ch.setLevel(Level.WARNING); - Logger.getLogger("").addHandler(ch); - - /* Initialize own logger for this class. */ - Logger logger = Logger.getLogger( - LoggingConfiguration.class.getName()); - - /* Create log handler that writes all messages on FINE or higher to a - * local file. */ - Formatter ff = new Formatter() { - public String format(LogRecord record) { - return dateTimeFormat.format(new Date(record.getMillis())) + " " - + record.getLevel() + " " + record.getSourceClassName() + " " - + record.getSourceMethodName() + " " + record.getMessage() - + (record.getThrown() != null ? " " + record.getThrown() : "") - + "\n"; - } - }; - try { - new File("log").mkdirs(); - FileHandler fh = new FileHandler("log/" + moduleName, 5000000, 5, - true); - fh.setFormatter(ff); - fh.setLevel(Level.FINE); - Logger.getLogger("").addHandler(fh); - } catch (SecurityException e) { - logger.log(Level.WARNING, "No permission to create log file. " - + "Logging to file is disabled.", e); - } catch (IOException e) { - logger.log(Level.WARNING, "Could not write to log file. Logging to " - + "file is disabled.", e); - } - } -} diff --git a/src/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/org/torproject/collector/relaydescs/ArchiveWriter.java index 370b5ea..ee87b12 100644 --- a/src/org/torproject/collector/relaydescs/ArchiveWriter.java +++ b/src/org/torproject/collector/relaydescs/ArchiveWriter.java @@ -5,7 +5,6 @@ package org.torproject.collector.relaydescs; import org.torproject.collector.main.Configuration; import org.torproject.collector.main.LockFile; -import org.torproject.collector.main.LoggingConfiguration; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DescriptorParser; import org.torproject.descriptor.DescriptorSourceFactory; @@ -39,8 +38,6 @@ public class ArchiveWriter extends Thread { public static void main(String[] args) { - /* Initialize logging configuration. */ - new LoggingConfiguration("relay-descriptors"); Logger logger = Logger.getLogger(ArchiveWriter.class.getName()); logger.info("Starting relay-descriptors module of CollecTor."); diff --git a/src/org/torproject/collector/torperf/TorperfDownloader.java b/src/org/torproject/collector/torperf/TorperfDownloader.java index a2c94ef..7bcfbf3 100644 --- a/src/org/torproject/collector/torperf/TorperfDownloader.java +++ b/src/org/torproject/collector/torperf/TorperfDownloader.java @@ -5,7 +5,6 @@ package org.torproject.collector.torperf; import org.torproject.collector.main.Configuration; import org.torproject.collector.main.LockFile; -import org.torproject.collector.main.LoggingConfiguration; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -34,8 +33,6 @@ public class TorperfDownloader extends Thread { public static void main(String[] args) { - /* Initialize logging configuration. */ - new LoggingConfiguration("torperf"); Logger logger = Logger.getLogger(TorperfDownloader.class.getName()); logger.info("Starting torperf module of CollecTor.");

1 0