May 2016 - tor-commits - lists.torproject.org

[tor-browser-bundle/hardened-builds] Bug 17406: Include SelfRando support in our nightlies
by gk＠torproject.org 12 May '16

12 May '16

commit 138470ca7ce810f62761af8c3989654ec4a611fa Author: Georg Koppen <gk(a)torproject.org> Date: Thu May 12 20:25:02 2016 +0000 Bug 17406: Include SelfRando support in our nightlies SelfRando is a new defense against code reuse attacks developed by the Radactor and Readactor++ people. We should give it a wider testing audience by including it in the nightly. --- RelativeLink/start-tor-browser | 3 + gitian/descriptors/linux/gitian-firefox.yml | 7 + gitian/descriptors/linux/gitian-utils.yml | 38 +++- gitian/fetch-inputs.sh | 4 +- gitian/gpg/ELFUTILS.gpg | Bin 0 -> 10483 bytes gitian/mkbundle-linux.sh | 9 +- gitian/patches/libfaketime-asan.patch | 268 ++++++++++++++++++++++++++-- gitian/verify-tags.sh | 3 +- gitian/versions.nightly | 4 + 9 files changed, 313 insertions(+), 23 deletions(-) diff --git a/RelativeLink/start-tor-browser b/RelativeLink/start-tor-browser index 2e7805d..19c5cfc 100755 --- a/RelativeLink/start-tor-browser +++ b/RelativeLink/start-tor-browser @@ -270,6 +270,9 @@ fi LD_LIBRARY_PATH="${HOME}/TorBrowser/Tor/" export LD_LIBRARY_PATH +export SELFRANDO_write_layout_file= +export NSS_DISABLE_HW_AES=1 + # We need to disable LSan which is enabled by default now. Otherwise we'll get # a crash during shutdown: https://bugs.torproject.org/10599#comment:59 ASAN_OPTIONS="detect_leaks=0" diff --git a/gitian/descriptors/linux/gitian-firefox.yml b/gitian/descriptors/linux/gitian-firefox.yml index e24a740..22686b7 100644 --- a/gitian/descriptors/linux/gitian-firefox.yml +++ b/gitian/descriptors/linux/gitian-firefox.yml @@ -29,12 +29,15 @@ remotes: "dir": "tor-browser" - "url": "https://github.com/wolfcw/libfaketime" "dir": "faketime" +- "url": "https://github.com/immunant/selfrando.git" + "dir": "selfrando" files: - "binutils-linux64-utils.zip" - "gcc-linux64-utils.zip" - "re-dzip.sh" - "dzip.sh" - "versions" +- "self-rando-utils.zip" script: | source versions INSTDIR="$HOME/install" @@ -49,6 +52,7 @@ script: | export DEB_BUILD_HARDENING_FORMAT=1 export DEB_BUILD_HARDENING_PIE=1 # + unzip -d $INSTDIR self-rando-utils.zip # Preparing Binutils and GCC for Tor Browser unzip -d $INSTDIR binutils-linux$GBUILD_BITS-utils.zip # Make sure gold is used with the hardening wrapper for full RELRO, see @@ -96,6 +100,9 @@ script: | # that feature it would hang sometimes for unknown but to libfaketime related # reasons. export LD_PRELOAD="" + # Self-Rando wrapper + export PATH="$HOME/build/selfrando/Tools/TorBrowser/tc-wrapper/:$PATH" + export SELFRANDO_skip_shuffle= make -f client.mk configure CONFIGURE_ARGS="--with-tor-browser-version=${TORBROWSER_VERSION} --enable-update-channel=${TORBROWSER_UPDATE_CHANNEL} --enable-bundled-fonts" # We need libfaketime for all the timestamps e.g. written into the libraries. # BUT we need to exclude |make build| from it. Otherwise the build fails close diff --git a/gitian/descriptors/linux/gitian-utils.yml b/gitian/descriptors/linux/gitian-utils.yml index 72ead1b..15b26f4 100644 --- a/gitian/descriptors/linux/gitian-utils.yml +++ b/gitian/descriptors/linux/gitian-utils.yml @@ -24,10 +24,14 @@ packages: - "libssl-dev" # Needed for binutils (64bit) as we are building with PIE enabled. - "libstdc++6-4.7-pic" +# Needed for Selfrando +- "scons" reference_datetime: "2000-01-01 00:00:00" remotes: - "url": "https://github.com/libevent/libevent.git" "dir": "libevent" +- "url": "https://github.com/immunant/selfrando.git" + "dir": "selfrando" files: - "binutils.tar.bz2" - "gcc.tar.bz2" @@ -36,6 +40,7 @@ files: - "versions" - "dzip.sh" - "libfaketime-asan.patch" +- "elfutils.tar.bz2" script: | INSTDIR="$HOME/install" source versions @@ -56,11 +61,8 @@ script: | # The libstdc++ shipped by default is non-PIC which breaks the binutils build # if we build with DEB_BUILD_HARDENING_PIE=1. We need to install a PIC one AND # make sure it gets used before the non-PIC one would. - if [ $GBUILD_BITS == "64" ]; - then - ln -s /usr/lib/gcc/x86_64-linux-gnu/4.7/libstdc++_pic.a libstdc++.a - export LDFLAGS="-L/home/debian -lstdc++" - fi + ln -s /usr/lib/gcc/x86_64-linux-gnu/4.7/libstdc++_pic.a libstdc++.a + export LDFLAGS="-L/home/debian -lstdc++" cd binutils* # We want to use gold as the linker in our toolchain mainly as it is way # faster when linking Tor Browser code (especially libxul). But apart from @@ -90,6 +92,31 @@ script: | cd .. export DEB_BUILD_HARDENING_FORMAT=1 + OPATH="$PATH" + OLD_LIBRARY_PATH="$LD_LIBRARY_PATH" + export PATH="$INSTDIR/binutils/bin:$INSTDIR/gcc/bin:$PATH" + export LD_LIBRARY_PATH="$INSTDIR/gcc/lib64/:$LD_LIBRARY_PATH" + + # Building Elfutils + tar xjf elfutils.tar.bz2 + cd elfutils*/ + ./configure --prefix=$INSTDIR/elfutils + make $MAKEOPTS + make install + cd .. + + # Building Selfrando + cd selfrando + scons -Q arch=x86_64 ADDN_CCFLAGS="-include $(ls $HOME/build/elfutils-*/libelf/elf.h) -I $INSTDIR/elfutils/include -fno-reorder-functions" ADDN_LINKFLAGS="-L $INSTDIR/elfutils/lib" + mkdir -p $INSTDIR/self-rando + cp sconsRelease/x86_64/bin/* $INSTDIR/self-rando/ + cd .. + + PATH="$OPATH" + unset OPATH + LD_LIBRARY_PATH="$OLD_LIBRARY_PATH" + unset OLD_LIBRARY_PATH + # Building Libevent cd libevent ./autogen.sh @@ -134,4 +161,5 @@ script: | ~/build/dzip.sh openssl-$OPENSSL_VER-linux$GBUILD_BITS-utils.zip openssl ~/build/dzip.sh libevent-${LIBEVENT_TAG#release-}-linux$GBUILD_BITS-utils.zip libevent ~/build/dzip.sh gmp-$GMP_VER-linux$GBUILD_BITS-utils.zip gmp + ~/build/dzip.sh self-rando-utils.zip self-rando cp *utils.zip $OUTDIR/ diff --git a/gitian/fetch-inputs.sh b/gitian/fetch-inputs.sh index 027a47e..4f63374 100755 --- a/gitian/fetch-inputs.sh +++ b/gitian/fetch-inputs.sh @@ -112,7 +112,7 @@ update_git() { ############################################################################## # Get+verify sigs that exist -for i in OPENSSL BINUTILS GCC PYCRYPTO PYTHON_MSI GMP +for i in OPENSSL BINUTILS GCC PYCRYPTO PYTHON_MSI GMP ELFUTILS do PACKAGE="${i}_PACKAGE" URL="${i}_URL" @@ -236,6 +236,7 @@ ln -sf "$SETUPTOOLS_PACKAGE" setuptools.tar.gz ln -sf "$GMP_PACKAGE" gmp.tar.bz2 ln -sf "$PARSLEY_PACKAGE" parsley.tar.gz ln -sf "$GO_PACKAGE" go.tar.gz +ln -sf "$ELFUTILS_PACKAGE" elfutils.tar.bz2 # Fetch latest gitian-builder itself # XXX - this is broken if a non-standard inputs dir is selected using the command line flag. @@ -273,6 +274,7 @@ goxcrypto https://go.googlesource.com/crypto $GO_X_CRYPTO_TAG goxnet https://go.googlesource.com/net $GO_X_NET_TAG obfs4 https://git.torproject.org/pluggable-transports/obfs4.git $OBFS4_TAG noto-fonts https://github.com/googlei18n/noto-fonts $NOTOFONTS_TAG +selfrando https://github.com/immunant/selfrando.git $SELFRANDO_TAG EOF # HTTPS-Everywhere is special, too. We need to initialize the git submodules and diff --git a/gitian/gpg/ELFUTILS.gpg b/gitian/gpg/ELFUTILS.gpg new file mode 100644 index 0000000..f1cd4b3 Binary files /dev/null and b/gitian/gpg/ELFUTILS.gpg differ diff --git a/gitian/mkbundle-linux.sh b/gitian/mkbundle-linux.sh index a1674dd..48aac24 100755 --- a/gitian/mkbundle-linux.sh +++ b/gitian/mkbundle-linux.sh @@ -35,7 +35,7 @@ fi if [ -z "$VM_MEMORY" ]; then - export VM_MEMORY=4000 + export VM_MEMORY=6000 fi ./make-vms.sh @@ -104,13 +104,14 @@ if [ ! -f inputs/binutils-$BINUTILS_VER-linux64-utils.zip -o \ ! -f inputs/gcc-$GCC_VER-linux64-utils.zip -o \ ! -f inputs/openssl-$OPENSSL_VER-linux64-utils.zip -o \ ! -f inputs/libevent-${LIBEVENT_TAG_ORIG#release-}-linux64-utils.zip -o \ - ! -f inputs/gmp-$GMP_VER-linux64-utils.zip ]; + ! -f inputs/gmp-$GMP_VER-linux64-utils.zip -o \ + ! -f inputs/self-rando-utils.zip ]; then echo echo "****** Starting Utilities Component of Linux Bundle (1/5 for Linux) ******" echo - ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit libevent=$LIBEVENT_TAG $DESCRIPTOR_DIR/linux/gitian-utils.yml + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit libevent=$LIBEVENT_TAG,selfrando=$SELFRANDO_TAG $DESCRIPTOR_DIR/linux/gitian-utils.yml if [ $? -ne 0 ]; then #mv var/build.log ./utils-fail-linux.log.`date +%Y%m%d%H%M%S` @@ -169,7 +170,7 @@ then echo "****** Starting TorBrowser Component of Linux Bundle (3/5 for Linux) ******" echo - ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor-browser=$TORBROWSER_TAG,faketime=$FAKETIME_TAG $DESCRIPTOR_DIR/linux/gitian-firefox.yml + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor-browser=$TORBROWSER_TAG,faketime=$FAKETIME_TAG,selfrando=$SELFRANDO_TAG $DESCRIPTOR_DIR/linux/gitian-firefox.yml if [ $? -ne 0 ]; then #mv var/build.log ./firefox-fail-linux.log.`date +%Y%m%d%H%M%S` diff --git a/gitian/patches/libfaketime-asan.patch b/gitian/patches/libfaketime-asan.patch index 5346eae..ff3e62b 100644 --- a/gitian/patches/libfaketime-asan.patch +++ b/gitian/patches/libfaketime-asan.patch @@ -1,12 +1,3 @@ -From 0ef83e9a0631e25e62e1dbc483108742fc125304 Mon Sep 17 00:00:00 2001 -From: Georg Koppen <gk(a)torproject.org> -Date: Mon, 14 Sep 2015 14:27:36 +0000 -Subject: [PATCH] No dying anymore - ---- - libsanitizer/asan/asan_linux.cc | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - diff --git a/libsanitizer/asan/asan_linux.cc b/libsanitizer/asan/asan_linux.cc index c504168..1d1a6dd 100644 --- a/libsanitizer/asan/asan_linux.cc @@ -23,6 +14,259 @@ index c504168..1d1a6dd 100644 } } --- -1.7.10.4 - +diff --git a/libsanitizer/sanitizer_common/SelfRandoAddressTranslator.h b/libsanitizer/sanitizer_common/SelfRandoAddressTranslator.h +new file mode 100644 +index 0000000..5f13870 +--- /dev/null ++++ b/libsanitizer/sanitizer_common/SelfRandoAddressTranslator.h +@@ -0,0 +1,155 @@ ++// ++// Created by tf on 10/12/2015. ++// ++ ++#pragma once ++#include "sanitizer_internal_defs.h" ++#include "sanitizer_libc.h" ++#include "sanitizer_common.h" ++ ++#define PROT_READ 0x1 /* Page can be read. */ ++#define PROT_WRITE 0x2 /* Page can be written. */ ++#define PROT_EXEC 0x4 /* Page can be executed. */ ++#define PROT_NONE 0x0 /* Page can not be accessed. */ ++#define MAP_SHARED 0x01 /* Share changes. */ ++ ++namespace __sanitizer { ++ class SelfRandoAddressTranslator { ++ struct __attribute__((__packed__)) FunctionRecord { ++ void *undiv_addr; ++ void *div_addr; ++ u32 size; ++ }; ++ ++ struct __attribute__((__packed__)) LibraryRecord { ++ u32 version; ++ u32 seed; ++ void *file_base; ++ void *mem_base; ++ void *size; ++ ++ private: ++ char _name; ++ ++ public: ++ inline char *name() { return &_name; } ++ ++ inline FunctionRecord *function_records() { ++ return (FunctionRecord *) (&_name + internal_strlen(&_name) + 1); ++ } ++ ++ u8 *record_end() { ++ FunctionRecord *fr = function_records(); ++ while (fr->undiv_addr) { ++ fr++; ++ } ++ return (u8 *) fr + sizeof(void *); ++ } ++ }; ++ ++ // 16K loaded modules should be enough for everyone. ++ LibraryRecord* library_records_idx[1<<14]; ++ int library_records_idx_size; ++ u8 *library_records_end; ++ void add_library_record(LibraryRecord* lr); ++ ++ void read_layout_file(); ++ u8 *record_end(int i); ++ ++ public: ++ SelfRandoAddressTranslator(); ++ ++ uptr convertAddress(uptr original_address); ++ }; ++ ++ ++ ++ ++ SelfRandoAddressTranslator::SelfRandoAddressTranslator() { ++ Printf("SRAT init\n"); ++ read_layout_file(); ++ } ++ ++ void SelfRandoAddressTranslator::add_library_record(LibraryRecord* lr) { ++ library_records_idx_size++; ++ CHECK(library_records_idx_size <= 1<<14); ++ library_records_idx[library_records_idx_size - 1] = lr; ++ } ++ ++ u8* SelfRandoAddressTranslator::record_end(int i) { ++ if (i < library_records_idx_size - 1) { ++ return (u8 *) library_records_idx[i + 1] - sizeof(void *); ++ } else { ++ return library_records_end - sizeof(void*); ++ } ++ } ++ ++ void SelfRandoAddressTranslator::read_layout_file() { ++ uptr pid = internal_getpid(); ++ char filename[32]; ++ internal_snprintf(filename, 32, "/tmp/%d.mlf", pid); ++ ++ u8 buf[200]; // Current size of struct stat: 144 ++ if(0 != internal_stat(filename, &buf)) { ++ Printf("SRAT warning: cannot find Memory Layout File. Stack trace will be probably wrong.\n"); ++ return; ++ } ++ ++ uptr fd = OpenFile(filename, /*write*/ false); ++ uptr size = *(uptr*) (buf+48); ++ CHECK_NE(fd, -1); ++ u8 *f = (u8 *) internal_mmap(0, size, PROT_READ, MAP_SHARED, (int) fd, 0); ++ CHECK_NE(f, 0); ++ library_records_end = f + size; ++ ++ u8 *p = f; ++ while (p < library_records_end) { ++ LibraryRecord *lr = (LibraryRecord *) p; ++ add_library_record(lr); ++ CHECK_EQ(lr->version, 0x00000101); ++ p = lr->record_end(); ++ } ++ ++ } ++ ++ uptr SelfRandoAddressTranslator::convertAddress(uptr p) { ++// Printf("conv %p", p); ++ for (int i = 0; i < library_records_idx_size; ++i) { ++ LibraryRecord *lr = library_records_idx[i]; ++ ++ if (p > (uptr) lr->mem_base && p < (uptr) lr->mem_base + (uptr) lr->size) { ++ // found the correct library ++ ++ FunctionRecord *function_records = lr->function_records(); ++ ++ int a = 0; ++ int b = (int) ((FunctionRecord *) record_end(i) - function_records); ++// Printf(" (lib=>%s< a=%d b=%d)", lr->name(), a, b); ++ CHECK_EQ((u8 *) (function_records + b), record_end(i)); ++ ++ while (b - a > 1) { // bisection ++ int c = a + (b - a) / 2; ++ FunctionRecord *fr = &function_records[c]; ++ ++ if ((uptr) fr->div_addr > p) { // too high ++ b = c; ++ } else if ((uptr) fr->div_addr + (uptr) fr->size < p) { // too low ++ a = c; ++ } else { // found ++ b = c + 1; ++ a = c; ++ } ++ } ++ FunctionRecord *fr = &function_records[a]; ++ CHECK((uptr) fr->div_addr < p && p < (uptr) fr->div_addr + (uptr) fr->size); ++ ++ p = (p - (uptr) fr->div_addr + (uptr) fr->undiv_addr); ++ ++ break; ++ } ++ } ++ ++// Printf(" => %p\n", p); ++ return p; ++ } ++} +\ No newline at end of file +diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc +index 5cc21d3..393a494 100644 +--- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc ++++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc +@@ -21,6 +21,7 @@ + #include "sanitizer_procmaps.h" + #include "sanitizer_symbolizer.h" + #include "sanitizer_symbolizer_libbacktrace.h" ++#include "SelfRandoAddressTranslator.h" + + #include <errno.h> + #include <stdlib.h> +@@ -512,7 +513,7 @@ class POSIXSymbolizer : public Symbolizer { + internal_symbolizer_(internal_symbolizer), + libbacktrace_symbolizer_(libbacktrace_symbolizer) {} + +- uptr SymbolizePC(uptr addr, AddressInfo *frames, uptr max_frames) { ++ virtual uptr SymbolizePC(uptr addr, AddressInfo *frames, uptr max_frames) { + BlockingMutexLock l(&mu_); + if (max_frames == 0) + return 0; +@@ -580,7 +581,7 @@ class POSIXSymbolizer : public Symbolizer { + return frame_id; + } + +- bool SymbolizeData(uptr addr, DataInfo *info) { ++ virtual bool SymbolizeData(uptr addr, DataInfo *info) { + BlockingMutexLock l(&mu_); + LoadedModule *module = FindModuleForAddress(addr); + if (module == 0) +@@ -606,7 +607,7 @@ class POSIXSymbolizer : public Symbolizer { + return true; + } + +- bool GetModuleNameAndOffsetForPC(uptr pc, const char **module_name, ++ virtual bool GetModuleNameAndOffsetForPC(uptr pc, const char **module_name, + uptr *module_address) { + BlockingMutexLock l(&mu_); + return FindModuleNameAndOffsetForAddress(pc, module_name, module_address); +@@ -720,9 +721,40 @@ class POSIXSymbolizer : public Symbolizer { + LibbacktraceSymbolizer *libbacktrace_symbolizer_; // Leaked. + }; + ++ ++ class SelfRandoPOSIXSymbolizer : public POSIXSymbolizer { ++ SelfRandoAddressTranslator translator; ++ ++ public: ++ SelfRandoPOSIXSymbolizer(ExternalSymbolizerInterface *external_symbolizer, ++ InternalSymbolizer *internal_symbolizer, ++ LibbacktraceSymbolizer *libbacktrace_symbolizer) ++ : POSIXSymbolizer(external_symbolizer, internal_symbolizer, libbacktrace_symbolizer), ++ translator() { } ++ ++ ++ private: ++ virtual uptr SymbolizePC(uptr addr, AddressInfo *frames, uptr max_frames) override { ++ uptr conv_addr = translator.convertAddress(addr); ++ uptr res = POSIXSymbolizer::SymbolizePC(conv_addr, frames, max_frames); ++ if (res == 1) frames->address = addr; ++ return res; ++ } ++ ++ virtual bool SymbolizeData(uptr addr, DataInfo *info) override { ++ addr = translator.convertAddress(addr); ++ return POSIXSymbolizer::SymbolizeData(addr, info); ++ } ++ ++ virtual bool GetModuleNameAndOffsetForPC(uptr pc, const char **module_name, uptr *module_address) override { ++ pc = translator.convertAddress(pc); ++ return POSIXSymbolizer::GetModuleNameAndOffsetForPC(pc, module_name, module_address); ++ }; ++ }; ++ + Symbolizer *Symbolizer::PlatformInit() { + if (!common_flags()->symbolize) { +- return new(symbolizer_allocator_) POSIXSymbolizer(0, 0, 0); ++ return new(symbolizer_allocator_) SelfRandoPOSIXSymbolizer(0, 0, 0); + } + InternalSymbolizer* internal_symbolizer = + InternalSymbolizer::get(&symbolizer_allocator_); +@@ -754,10 +786,11 @@ Symbolizer *Symbolizer::PlatformInit() { + } + } + +- return new(symbolizer_allocator_) POSIXSymbolizer( ++ return new(symbolizer_allocator_) SelfRandoPOSIXSymbolizer( + external_symbolizer, internal_symbolizer, libbacktrace_symbolizer); + } + ++ + } // namespace __sanitizer + + #endif // SANITIZER_POSIX diff --git a/gitian/verify-tags.sh b/gitian/verify-tags.sh index 46cb01a..5268f85 100755 --- a/gitian/verify-tags.sh +++ b/gitian/verify-tags.sh @@ -118,10 +118,11 @@ siphash $GOSIPHASH_TAG goxcrypto $GO_X_CRYPTO_TAG goxnet $GO_X_NET_TAG noto-fonts $NOTOFONTS_TAG +selfrando $SELFRANDO_TAG EOF # Verify signatures on signed packages -for i in OPENSSL BINUTILS GCC PYCRYPTO PYTHON_MSI GMP +for i in OPENSSL BINUTILS GCC PYCRYPTO PYTHON_MSI GMP ELFUTILS do PACKAGE="${i}_PACKAGE" URL="${i}_URL" diff --git a/gitian/versions.nightly b/gitian/versions.nightly index 088625b..9ff7e8f 100755 --- a/gitian/versions.nightly +++ b/gitian/versions.nightly @@ -48,6 +48,7 @@ GO_X_CRYPTO_TAG=master GO_X_NET_TAG=master OBFS4_TAG=master NOTOFONTS_TAG=720e34851382ee3c1ef024d8dffb68ffbfb234c2 +SELFRANDO_TAG=8e49e7aa664625d8e4793579864231b151589d0f GITIAN_TAG=tor-browser-builder-4 @@ -69,6 +70,7 @@ SETUPTOOLS_VER=1.4 PARSLEY_VER=1.2 GO_VER=1.4.2 NSIS_VER=2.51 +ELFUTILS_VER=0.160 ## File names for the source packages OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz @@ -99,6 +101,7 @@ NOTOJPFONT_PACKAGE=NotoSansJP-Regular.otf NOTOKRFONT_PACKAGE=NotoSansKR-Regular.otf NOTOSCFONT_PACKAGE=NotoSansSC-Regular.otf NOTOTCFONT_PACKAGE=NotoSansTC-Regular.otf +ELFUTILS_PACKAGE=elfutils-${ELFUTILS_VER}.tar.bz2 # Hashes for packages with weak sigs or no sigs OPENSSL_HASH=e7e81d82f3cd538ab0cdba494006d44aab9dd96b7f6233ce9971fb7c7916d511 @@ -157,3 +160,4 @@ NOTOJPFONT_URL=${NOTOCJKBASE_URL}/${NOTOJPFONT_PACKAGE} NOTOKRFONT_URL=${NOTOCJKBASE_URL}/${NOTOKRFONT_PACKAGE} NOTOSCFONT_URL=${NOTOCJKBASE_URL}/${NOTOSCFONT_PACKAGE} NOTOTCFONT_URL=${NOTOCJKBASE_URL}/${NOTOTCFONT_PACKAGE} +ELFUTILS_URL=https://fedorahosted.org/releases/e/l/elfutils/${ELFUTILS_VER}/${ELFUTILS_PACKAGE}

1 0

[collector/master] Remove unused bridge pool assignments code.
by karsten＠torproject.org 12 May '16

12 May '16

commit 393a8d6d5b8f4d037127bcdddca49e6fc43076d5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu May 12 21:43:59 2016 +0200 Remove unused bridge pool assignments code. --- bin/run-bridgepools | 3 - config.template | 10 - src/org/torproject/collector/Main.java | 2 - .../BridgePoolAssignmentsProcessor.java | 273 --------------------- .../torproject/collector/main/Configuration.java | 15 -- 5 files changed, 303 deletions(-) diff --git a/bin/run-bridgepools b/bin/run-bridgepools deleted file mode 100755 index 62d71f7..0000000 --- a/bin/run-bridgepools +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -`dirname $0`/run-all `basename $0` - diff --git a/config.template b/config.template index cd31802..88407a2 100644 --- a/config.template +++ b/config.template @@ -85,16 +85,6 @@ #SanitizedBridgesWriteDirectory out/bridge-descriptors/ # # -######## Bridge pool assignments ######## -# -## Relative path to directory to read bridge pool assignment files from -#AssignmentsDirectory in/bridge-pool-assignments/ -# -## Relative path to directory to write sanitized bridge pool assignment -## files to -#SanitizedAssignmentsDirectory out/bridge-pool-assignments/ -# -# ######## Exit lists ######## # ## (No options available) diff --git a/src/org/torproject/collector/Main.java b/src/org/torproject/collector/Main.java index 44182ed..9c64696 100644 --- a/src/org/torproject/collector/Main.java +++ b/src/org/torproject/collector/Main.java @@ -4,7 +4,6 @@ package org.torproject.collector; import org.torproject.collector.bridgedescs.SanitizedBridgesWriter; -import org.torproject.collector.bridgepools.BridgePoolAssignmentsProcessor; import org.torproject.collector.exitlists.ExitListDownloader; import org.torproject.collector.index.CreateIndexJson; import org.torproject.collector.relaydescs.ArchiveWriter; @@ -33,7 +32,6 @@ public class Main { static { // add a new main class here collecTorMains.put("bridgedescs", SanitizedBridgesWriter.class); - collecTorMains.put("bridgepools", BridgePoolAssignmentsProcessor.class); collecTorMains.put("exitlists", ExitListDownloader.class); collecTorMains.put("updateindex", CreateIndexJson.class); collecTorMains.put("relaydescs", ArchiveWriter.class); diff --git a/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java deleted file mode 100644 index 2b6c262..0000000 --- a/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java +++ /dev/null @@ -1,273 +0,0 @@ -/* Copyright 2011--2016 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.collector.bridgepools; - -import org.torproject.collector.main.Configuration; -import org.torproject.collector.main.LockFile; - -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStreamReader; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.SortedSet; -import java.util.Stack; -import java.util.TimeZone; -import java.util.TreeSet; -import java.util.logging.Level; -import java.util.logging.Logger; - -public class BridgePoolAssignmentsProcessor extends Thread { - - public static void main(String[] args) { - - Logger logger = Logger.getLogger( - BridgePoolAssignmentsProcessor.class.getName()); - logger.info("Starting bridge-pool-assignments module of CollecTor."); - - // Initialize configuration - Configuration config = new Configuration(); - - // Use lock file to avoid overlapping runs - LockFile lf = new LockFile("bridge-pool-assignments"); - if (!lf.acquireLock()) { - logger.severe("Warning: CollecTor is already running or has not exited " - + "cleanly! Exiting!"); - System.exit(1); - } - - // Process bridge pool assignments - new BridgePoolAssignmentsProcessor(config).run(); - - // Remove lock file - lf.releaseLock(); - - logger.info("Terminating bridge-pool-assignments module of CollecTor."); - } - - private Configuration config; - - public BridgePoolAssignmentsProcessor(Configuration config) { - this.config = config; - } - - public void run() { - - File assignmentsDirectory = - new File(config.getAssignmentsDirectory()); - File sanitizedAssignmentsDirectory = - new File(config.getSanitizedAssignmentsDirectory()); - - Logger logger = - Logger.getLogger(BridgePoolAssignmentsProcessor.class.getName()); - if (assignmentsDirectory == null - || sanitizedAssignmentsDirectory == null) { - IllegalArgumentException e = new IllegalArgumentException("Neither " - + "assignmentsDirectory nor sanitizedAssignmentsDirectory may " - + "be null!"); - throw e; - } - - List<File> assignmentFiles = new ArrayList<File>(); - Stack<File> files = new Stack<File>(); - files.add(assignmentsDirectory); - while (!files.isEmpty()) { - File file = files.pop(); - if (file.isDirectory()) { - files.addAll(Arrays.asList(file.listFiles())); - } else if (file.getName().equals("assignments.log")) { - assignmentFiles.add(file); - } - } - - SimpleDateFormat assignmentFormat = - new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - assignmentFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - SimpleDateFormat filenameFormat = - new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss"); - filenameFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String duplicateFingerprint = null; - long maxBridgePoolAssignmentTime = 0L; - for (File assignmentFile : assignmentFiles) { - logger.info("Processing bridge pool assignment file '" - + assignmentFile.getAbsolutePath() + "'..."); - try { - BufferedReader br = null; - if (assignmentFile.getName().endsWith(".gz")) { - br = new BufferedReader(new InputStreamReader( - new GzipCompressorInputStream(new FileInputStream( - assignmentFile)))); - } else { - br = new BufferedReader(new FileReader(assignmentFile)); - } - String line; - String bridgePoolAssignmentLine = null; - SortedSet<String> sanitizedAssignments = new TreeSet<String>(); - boolean wroteLastLine = false; - boolean skipBefore20120504125947 = true; - Set<String> hashedFingerprints = null; - while ((line = br.readLine()) != null || !wroteLastLine) { - if (line != null && line.startsWith("bridge-pool-assignment ")) { - String[] parts = line.split(" "); - if (parts.length != 3) { - continue; - } - /* TODO Take out this temporary hack to ignore all assignments - * coming from ponticum when byblos was still the official - * BridgeDB host. */ - if (line.compareTo( - "bridge-pool-assignment 2012-05-04 12:59:47") >= 0) { - skipBefore20120504125947 = false; - } - } - if (skipBefore20120504125947) { - if (line == null) { - break; - } else { - continue; - } - } - if (line == null - || line.startsWith("bridge-pool-assignment ")) { - if (bridgePoolAssignmentLine != null) { - try { - long bridgePoolAssignmentTime = assignmentFormat.parse( - bridgePoolAssignmentLine.substring( - "bridge-pool-assignment ".length())).getTime(); - maxBridgePoolAssignmentTime = Math.max( - maxBridgePoolAssignmentTime, - bridgePoolAssignmentTime); - File tarballFile = new File( - sanitizedAssignmentsDirectory, filenameFormat.format( - bridgePoolAssignmentTime)); - File rsyncFile = new File( - "recent/bridge-pool-assignments/" - + tarballFile.getName()); - File[] outputFiles = new File[] { tarballFile, - rsyncFile }; - for (File outputFile : outputFiles) { - if (!outputFile.exists()) { - outputFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - outputFile)); - bw.write("@type bridge-pool-assignment 1.0\n"); - bw.write(bridgePoolAssignmentLine + "\n"); - for (String assignmentLine : sanitizedAssignments) { - bw.write(assignmentLine + "\n"); - } - bw.close(); - } - } - } catch (IOException e) { - logger.log(Level.WARNING, "Could not write sanitized " - + "bridge pool assignment file for line '" - + bridgePoolAssignmentLine + "' to disk. Skipping " - + "bridge pool assignment file '" - + assignmentFile.getAbsolutePath() + "'.", e); - break; - } catch (ParseException e) { - logger.log(Level.WARNING, "Could not write sanitized " - + "bridge pool assignment file for line '" - + bridgePoolAssignmentLine + "' to disk. Skipping " - + "bridge pool assignment file '" - + assignmentFile.getAbsolutePath() + "'.", e); - break; - } - sanitizedAssignments.clear(); - } - if (line == null) { - wroteLastLine = true; - } else { - bridgePoolAssignmentLine = line; - hashedFingerprints = new HashSet<String>(); - } - } else { - String[] parts = line.split(" "); - if (parts.length < 2 || parts[0].length() < 40) { - logger.warning("Unrecognized line '" + line - + "'. Aborting."); - break; - } - String hashedFingerprint = null; - try { - hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex( - line.split(" ")[0].toCharArray())).toLowerCase(); - } catch (DecoderException e) { - logger.warning("Unable to decode hex fingerprint in line '" - + line + "'. Aborting."); - break; - } - if (hashedFingerprints.contains(hashedFingerprint)) { - duplicateFingerprint = bridgePoolAssignmentLine; - } - hashedFingerprints.add(hashedFingerprint); - String assignmentDetails = line.substring(40); - sanitizedAssignments.add(hashedFingerprint - + assignmentDetails); - } - } - br.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Could not read bridge pool assignment " - + "file '" + assignmentFile.getAbsolutePath() - + "'. Skipping.", e); - } - } - - if (duplicateFingerprint != null) { - logger.warning("At least one bridge pool assignment list contained " - + "duplicate fingerprints. Last found in assignment list " - + "starting with '" + duplicateFingerprint + "'."); - } - - if (maxBridgePoolAssignmentTime > 0L - && maxBridgePoolAssignmentTime + 330L * 60L * 1000L - < System.currentTimeMillis()) { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - logger.warning("The last known bridge pool assignment list was " - + "published at " - + dateTimeFormat.format(maxBridgePoolAssignmentTime) - + ", which is more than 5:30 hours in the past."); - } - - this.cleanUpRsyncDirectory(); - - logger.info("Finished processing bridge pool assignment file(s)."); - } - - /* Delete all files from the rsync directory that have not been modified - * in the last three days. */ - public void cleanUpRsyncDirectory() { - long cutOffMillis = System.currentTimeMillis() - - 3L * 24L * 60L * 60L * 1000L; - Stack<File> allFiles = new Stack<File>(); - allFiles.add(new File("recent/bridge-pool-assignments")); - while (!allFiles.isEmpty()) { - File file = allFiles.pop(); - if (file.isDirectory()) { - allFiles.addAll(Arrays.asList(file.listFiles())); - } else if (file.lastModified() < cutOffMillis) { - file.delete(); - } - } - } -} - diff --git a/src/org/torproject/collector/main/Configuration.java b/src/org/torproject/collector/main/Configuration.java index 9a511d3..aee1d02 100644 --- a/src/org/torproject/collector/main/Configuration.java +++ b/src/org/torproject/collector/main/Configuration.java @@ -62,9 +62,6 @@ public class Configuration { private boolean downloadAllServerDescriptors = false; private boolean downloadAllExtraInfoDescriptors = false; private boolean compressRelayDescriptorDownloads; - private String assignmentsDirectory = "in/bridge-pool-assignments/"; - private String sanitizedAssignmentsDirectory = - "out/bridge-pool-assignments/"; private String torperfOutputDirectory = "out/torperf/"; private SortedMap<String, String> torperfSources = null; private List<String> torperfFiles = null; @@ -170,10 +167,6 @@ public class Configuration { } else if (line.startsWith("CompressRelayDescriptorDownloads")) { this.compressRelayDescriptorDownloads = Integer.parseInt( line.split(" ")[1]) != 0; - } else if (line.startsWith("AssignmentsDirectory")) { - this.assignmentsDirectory = line.split(" ")[1]; - } else if (line.startsWith("SanitizedAssignmentsDirectory")) { - this.sanitizedAssignmentsDirectory = line.split(" ")[1]; } else if (line.startsWith("TorperfOutputDirectory")) { this.torperfOutputDirectory = line.split(" ")[1]; } else if (line.startsWith("TorperfSource")) { @@ -310,14 +303,6 @@ public class Configuration { return this.compressRelayDescriptorDownloads; } - public String getAssignmentsDirectory() { - return this.assignmentsDirectory; - } - - public String getSanitizedAssignmentsDirectory() { - return this.sanitizedAssignmentsDirectory; - } - public String getTorperfOutputDirectory() { return this.torperfOutputDirectory; }

1 0

[collector/master] implements task-18922, make logging configurable.
by karsten＠torproject.org 12 May '16

12 May '16

commit f85683536bb0918f449387bf73f02385a3495bb5 Author: iwakeh <iwakeh(a)torproject.org> Date: Mon May 9 22:00:26 2016 +0000 implements task-18922, make logging configurable. --- bin/logging.properties | 20 +++++ bin/run-all | 16 ++++ bin/run-bridgedescs | 2 +- bin/run-bridgepools | 2 +- bin/run-exitlists | 2 +- bin/run-relaydescs | 2 +- bin/run-torperf | 2 +- bin/update-index | 2 +- .../bridgedescs/SanitizedBridgesWriter.java | 3 - .../BridgePoolAssignmentsProcessor.java | 3 - .../collector/exitlists/ExitListDownloader.java | 3 - .../collector/main/LoggingConfiguration.java | 96 ---------------------- .../collector/relaydescs/ArchiveWriter.java | 3 - .../collector/torperf/TorperfDownloader.java | 3 - 14 files changed, 42 insertions(+), 117 deletions(-) diff --git a/bin/logging.properties b/bin/logging.properties new file mode 100644 index 0000000..5aae3cb --- /dev/null +++ b/bin/logging.properties @@ -0,0 +1,20 @@ +handlers= java.util.logging.FileHandler, java.util.logging.ConsoleHandler + +.level = FINER + +# if the following dir pattern is changed, LOGGINGPATH='log' in run-all +# needs to be adapted. +java.util.logging.FileHandler.pattern = ./log/collector-%g.log +java.util.logging.FileHandler.limit = 5000000 +java.util.logging.FileHandler.count = 9 +java.util.logging.FileHandler.append = true +java.util.logging.FileHandler.level = FINER +java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter + +java.util.logging.ConsoleHandler.level = WARNING +java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter + +java.util.logging.SimpleFormatter.format = %1$tF %1$tT %4$s %2$s %5$s%n + +sun.net.www.level = SEVERE +sun.level = SEVERE diff --git a/bin/run-all b/bin/run-all new file mode 100755 index 0000000..469aced --- /dev/null +++ b/bin/run-all @@ -0,0 +1,16 @@ +#!/bin/sh + +TOKEN=`expr substr $1 5 20` + +# if the path is changed in properties, this needs to be adapted. +LOGGINGPATH='log' + +if ! test -d $LOGGINGPATH +then mkdir $LOGGINGPATH +fi + +# the following uses one logging configuration for all modules +java -Xmx2g -Djava.util.logging.config.file=./bin/logging.properties -jar collector-0.9.0-dev.jar $TOKEN + +# this would use a special log config for each module +# java -Xmx2g -Djava.util.logging.config.file=./bin/logging.$TOKEN.properties -jar collector-0.9.0-dev.jar $TOKEN diff --git a/bin/run-bridgedescs b/bin/run-bridgedescs index 37eeea5..62d71f7 100755 --- a/bin/run-bridgedescs +++ b/bin/run-bridgedescs @@ -1,3 +1,3 @@ #!/bin/sh -java -Xmx2g -jar collector-0.9.0-dev.jar bridgedescs +`dirname $0`/run-all `basename $0` diff --git a/bin/run-bridgepools b/bin/run-bridgepools index 9c5c53a..62d71f7 100755 --- a/bin/run-bridgepools +++ b/bin/run-bridgepools @@ -1,3 +1,3 @@ #!/bin/sh -java -Xmx2g -jar collector-0.9.0-dev.jar bridgepools +`dirname $0`/run-all `basename $0` diff --git a/bin/run-exitlists b/bin/run-exitlists index 3bdef11..62d71f7 100755 --- a/bin/run-exitlists +++ b/bin/run-exitlists @@ -1,3 +1,3 @@ #!/bin/sh -java -Xmx2g -jar collector-0.9.0-dev.jar exitlists +`dirname $0`/run-all `basename $0` diff --git a/bin/run-relaydescs b/bin/run-relaydescs index 8ad7506..62d71f7 100755 --- a/bin/run-relaydescs +++ b/bin/run-relaydescs @@ -1,3 +1,3 @@ #!/bin/sh -java -Xmx2g -jar collector-0.9.0-dev.jar relaydescs +`dirname $0`/run-all `basename $0` diff --git a/bin/run-torperf b/bin/run-torperf index 267df58..62d71f7 100755 --- a/bin/run-torperf +++ b/bin/run-torperf @@ -1,3 +1,3 @@ #!/bin/sh -java -Xmx2g -jar collector-0.9.0-dev.jar torperf +`dirname $0`/run-all `basename $0` diff --git a/bin/update-index b/bin/update-index index 70b0067..6e6c488 100755 --- a/bin/update-index +++ b/bin/update-index @@ -1,3 +1,3 @@ #!/bin/sh -java -Xmx2g -jar collector-0.9.0-dev.jar updateindex +`dirname $0`/run-all run-updateindex diff --git a/src/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index f8d61c0..3214715 100644 --- a/src/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -5,7 +5,6 @@ package org.torproject.collector.bridgedescs; import org.torproject.collector.main.Configuration; import org.torproject.collector.main.LockFile; -import org.torproject.collector.main.LoggingConfiguration; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Base64; @@ -52,8 +51,6 @@ public class SanitizedBridgesWriter extends Thread { public static void main(String[] args) { - /* Initialize logging configuration. */ - new LoggingConfiguration("bridge-descriptors"); Logger logger = Logger.getLogger( SanitizedBridgesWriter.class.getName()); logger.info("Starting bridge-descriptors module of CollecTor."); diff --git a/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java index 8afaa43..2b6c262 100644 --- a/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java +++ b/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java @@ -5,7 +5,6 @@ package org.torproject.collector.bridgepools; import org.torproject.collector.main.Configuration; import org.torproject.collector.main.LockFile; -import org.torproject.collector.main.LoggingConfiguration; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; @@ -38,8 +37,6 @@ public class BridgePoolAssignmentsProcessor extends Thread { public static void main(String[] args) { - /* Initialize logging configuration. */ - new LoggingConfiguration("bridge-pool-assignments"); Logger logger = Logger.getLogger( BridgePoolAssignmentsProcessor.class.getName()); logger.info("Starting bridge-pool-assignments module of CollecTor."); diff --git a/src/org/torproject/collector/exitlists/ExitListDownloader.java b/src/org/torproject/collector/exitlists/ExitListDownloader.java index d328850..54fd50f 100644 --- a/src/org/torproject/collector/exitlists/ExitListDownloader.java +++ b/src/org/torproject/collector/exitlists/ExitListDownloader.java @@ -5,7 +5,6 @@ package org.torproject.collector.exitlists; import org.torproject.collector.main.Configuration; import org.torproject.collector.main.LockFile; -import org.torproject.collector.main.LoggingConfiguration; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DescriptorParser; @@ -34,8 +33,6 @@ public class ExitListDownloader extends Thread { public static void main(String[] args) { - /* Initialize logging configuration. */ - new LoggingConfiguration("exit-lists"); Logger logger = Logger.getLogger(ExitListDownloader.class.getName()); logger.info("Starting exit-lists module of CollecTor."); diff --git a/src/org/torproject/collector/main/LoggingConfiguration.java b/src/org/torproject/collector/main/LoggingConfiguration.java deleted file mode 100644 index b8a91c7..0000000 --- a/src/org/torproject/collector/main/LoggingConfiguration.java +++ /dev/null @@ -1,96 +0,0 @@ -/* Copyright 2010--2016 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.collector.main; - -import java.io.File; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.TimeZone; -import java.util.logging.ConsoleHandler; -import java.util.logging.FileHandler; -import java.util.logging.Formatter; -import java.util.logging.Handler; -import java.util.logging.Level; -import java.util.logging.LogRecord; -import java.util.logging.Logger; - -/** - * Initialize logging configuration. - * - * Log levels used by CollecTor: - * - * - SEVERE: An event made it impossible to continue program execution. - * - WARNING: A potential problem occurred that requires the operator to - * look after the otherwise unattended setup - * - INFO: Messages on INFO level are meant to help the operator in making - * sure that operation works as expected. - * - FINE: Debug messages that are used to identify problems and which are - * turned on by default. - * - FINER: More detailed debug messages to investigate problems in more - * detail. Not turned on by default. Increase log file limit when using - * FINER. - * - FINEST: Most detailed debug messages. Not used. - */ -public class LoggingConfiguration { - public LoggingConfiguration(String moduleName) { - - /* Remove default console handler. */ - for (Handler h : Logger.getLogger("").getHandlers()) { - Logger.getLogger("").removeHandler(h); - } - - /* Disable logging of internal Sun classes. */ - Logger.getLogger("sun").setLevel(Level.OFF); - - /* Set minimum log level we care about from INFO to FINER. */ - Logger.getLogger("").setLevel(Level.FINER); - - /* Create log handler that writes messages on WARNING or higher to the - * console. */ - final SimpleDateFormat dateTimeFormat = - new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - Formatter cf = new Formatter() { - public String format(LogRecord record) { - return dateTimeFormat.format(new Date(record.getMillis())) + " " - + record.getMessage() + "\n"; - } - }; - Handler ch = new ConsoleHandler(); - ch.setFormatter(cf); - ch.setLevel(Level.WARNING); - Logger.getLogger("").addHandler(ch); - - /* Initialize own logger for this class. */ - Logger logger = Logger.getLogger( - LoggingConfiguration.class.getName()); - - /* Create log handler that writes all messages on FINE or higher to a - * local file. */ - Formatter ff = new Formatter() { - public String format(LogRecord record) { - return dateTimeFormat.format(new Date(record.getMillis())) + " " - + record.getLevel() + " " + record.getSourceClassName() + " " - + record.getSourceMethodName() + " " + record.getMessage() - + (record.getThrown() != null ? " " + record.getThrown() : "") - + "\n"; - } - }; - try { - new File("log").mkdirs(); - FileHandler fh = new FileHandler("log/" + moduleName, 5000000, 5, - true); - fh.setFormatter(ff); - fh.setLevel(Level.FINE); - Logger.getLogger("").addHandler(fh); - } catch (SecurityException e) { - logger.log(Level.WARNING, "No permission to create log file. " - + "Logging to file is disabled.", e); - } catch (IOException e) { - logger.log(Level.WARNING, "Could not write to log file. Logging to " - + "file is disabled.", e); - } - } -} diff --git a/src/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/org/torproject/collector/relaydescs/ArchiveWriter.java index 370b5ea..ee87b12 100644 --- a/src/org/torproject/collector/relaydescs/ArchiveWriter.java +++ b/src/org/torproject/collector/relaydescs/ArchiveWriter.java @@ -5,7 +5,6 @@ package org.torproject.collector.relaydescs; import org.torproject.collector.main.Configuration; import org.torproject.collector.main.LockFile; -import org.torproject.collector.main.LoggingConfiguration; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DescriptorParser; import org.torproject.descriptor.DescriptorSourceFactory; @@ -39,8 +38,6 @@ public class ArchiveWriter extends Thread { public static void main(String[] args) { - /* Initialize logging configuration. */ - new LoggingConfiguration("relay-descriptors"); Logger logger = Logger.getLogger(ArchiveWriter.class.getName()); logger.info("Starting relay-descriptors module of CollecTor."); diff --git a/src/org/torproject/collector/torperf/TorperfDownloader.java b/src/org/torproject/collector/torperf/TorperfDownloader.java index a2c94ef..7bcfbf3 100644 --- a/src/org/torproject/collector/torperf/TorperfDownloader.java +++ b/src/org/torproject/collector/torperf/TorperfDownloader.java @@ -5,7 +5,6 @@ package org.torproject.collector.torperf; import org.torproject.collector.main.Configuration; import org.torproject.collector.main.LockFile; -import org.torproject.collector.main.LoggingConfiguration; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -34,8 +33,6 @@ public class TorperfDownloader extends Thread { public static void main(String[] args) { - /* Initialize logging configuration. */ - new LoggingConfiguration("torperf"); Logger logger = Logger.getLogger(TorperfDownloader.class.getName()); logger.info("Starting torperf module of CollecTor.");

1 0

[tor/release-0.2.8] Fix out-of-bounds write during voting with duplicate ed25519 keys
by nickm＠torproject.org 12 May '16

12 May '16

commit bf3e32a45288b64e5535e02f40bd2bcb93c8a520 Author: John Brooks <special(a)torproject.org> Date: Wed May 11 12:13:22 2016 -0400 Fix out-of-bounds write during voting with duplicate ed25519 keys In dirserv_compute_performance_thresholds, we allocate arrays based on the length of 'routers', a list of routerinfo_t, but loop over the nodelist. The 'routers' list may be shorter when relays were filtered by routers_make_ed_keys_unique, leading to an out-of-bounds write on directory authorities. This bug was originally introduced in 26e89742, but it doesn't look possible to trigger until routers_make_ed_keys_unique was introduced in 13a31e72. Fixes bug 19032; bugfix on tor 0.2.8.2-alpha. --- changes/bug19032 | 4 ++++ src/or/dirserv.c | 31 ++++++++++++++++--------------- src/or/dirserv.h | 2 +- src/or/networkstatus.c | 3 +-- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/changes/bug19032 b/changes/bug19032 new file mode 100644 index 0000000..93f17c2 --- /dev/null +++ b/changes/bug19032 @@ -0,0 +1,4 @@ + o Major bugfixes (security, directory authorities): + - Fix a crash and out-of-bounds write during authority voting, when the + list of relays includes duplicate ed25519 identity keys. Fixes bug 19032; + bugfix on 0.2.8.2-alpha. diff --git a/src/or/dirserv.c b/src/or/dirserv.c index ae67e8e..a367669 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1422,13 +1422,13 @@ router_counts_toward_thresholds(const node_t *node, time_t now, * * Also, set the is_exit flag of each router appropriately. */ static void -dirserv_compute_performance_thresholds(const smartlist_t *routers, - digestmap_t *omit_as_sybil) +dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil) { int n_active, n_active_nonexit, n_familiar; uint32_t *uptimes, *bandwidths_kb, *bandwidths_excluding_exits_kb; long *tks; double *mtbfs, *wfus; + smartlist_t *nodelist; time_t now = time(NULL); const or_options_t *options = get_options(); @@ -1446,27 +1446,28 @@ dirserv_compute_performance_thresholds(const smartlist_t *routers, guard_tk = 0; guard_wfu = 0; + nodelist_assert_ok(); + nodelist = nodelist_get_list(); + /* Initialize arrays that will hold values for each router. We'll * sort them and use that to compute thresholds. */ n_active = n_active_nonexit = 0; /* Uptime for every active router. */ - uptimes = tor_calloc(smartlist_len(routers), sizeof(uint32_t)); + uptimes = tor_calloc(smartlist_len(nodelist), sizeof(uint32_t)); /* Bandwidth for every active router. */ - bandwidths_kb = tor_calloc(smartlist_len(routers), sizeof(uint32_t)); + bandwidths_kb = tor_calloc(smartlist_len(nodelist), sizeof(uint32_t)); /* Bandwidth for every active non-exit router. */ bandwidths_excluding_exits_kb = - tor_calloc(smartlist_len(routers), sizeof(uint32_t)); + tor_calloc(smartlist_len(nodelist), sizeof(uint32_t)); /* Weighted mean time between failure for each active router. */ - mtbfs = tor_calloc(smartlist_len(routers), sizeof(double)); + mtbfs = tor_calloc(smartlist_len(nodelist), sizeof(double)); /* Time-known for each active router. */ - tks = tor_calloc(smartlist_len(routers), sizeof(long)); + tks = tor_calloc(smartlist_len(nodelist), sizeof(long)); /* Weighted fractional uptime for each active router. */ - wfus = tor_calloc(smartlist_len(routers), sizeof(double)); - - nodelist_assert_ok(); + wfus = tor_calloc(smartlist_len(nodelist), sizeof(double)); /* Now, fill in the arrays. */ - SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) { + SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) { if (options->BridgeAuthoritativeDir && node->ri && node->ri->purpose != ROUTER_PURPOSE_BRIDGE) @@ -1542,7 +1543,7 @@ dirserv_compute_performance_thresholds(const smartlist_t *routers, * fill wfus with the wfu of every such "familiar" router. */ n_familiar = 0; - SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) { + SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) { if (router_counts_toward_thresholds(node, now, omit_as_sybil, require_mbw)) { routerinfo_t *ri = node->ri; @@ -1596,11 +1597,11 @@ dirserv_compute_performance_thresholds(const smartlist_t *routers, * networkstatus_getinfo_by_purpose(). */ void -dirserv_compute_bridge_flag_thresholds(const smartlist_t *routers) +dirserv_compute_bridge_flag_thresholds() { digestmap_t *omit_as_sybil = digestmap_new(); - dirserv_compute_performance_thresholds(routers, omit_as_sybil); + dirserv_compute_performance_thresholds(omit_as_sybil); digestmap_free(omit_as_sybil, NULL); } @@ -2870,7 +2871,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, * this must come before dirserv_compute_performance_thresholds() */ dirserv_count_measured_bws(routers); - dirserv_compute_performance_thresholds(routers, omit_as_sybil); + dirserv_compute_performance_thresholds(omit_as_sybil); routerstatuses = smartlist_new(); microdescriptors = smartlist_new(); diff --git a/src/or/dirserv.h b/src/or/dirserv.h index b16a67c..4bb3072 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -50,7 +50,7 @@ int list_server_status_v1(smartlist_t *routers, char **router_status_out, int dirserv_dump_directory_to_string(char **dir_out, crypto_pk_t *private_key); char *dirserv_get_flag_thresholds_line(void); -void dirserv_compute_bridge_flag_thresholds(const smartlist_t *routers); +void dirserv_compute_bridge_flag_thresholds(void); int directory_fetches_from_authorities(const or_options_t *options); int directory_fetches_dir_info_early(const or_options_t *options); diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index a9b22ed..f72e9d5 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1697,11 +1697,10 @@ networkstatus_dump_bridge_status_to_file(time_t now) char *fname = NULL; char *thresholds = NULL; char *published_thresholds_and_status = NULL; - routerlist_t *rl = router_get_routerlist(); char published[ISO_TIME_LEN+1]; format_iso_time(published, now); - dirserv_compute_bridge_flag_thresholds(rl->routers); + dirserv_compute_bridge_flag_thresholds(); thresholds = dirserv_get_flag_thresholds_line(); tor_asprintf(&published_thresholds_and_status, "published %s\nflag-thresholds %s\n%s",

1 0

[tor/release-0.2.8] Merge remote-tracking branch 'special/bug19032-027' into maint-0.2.7
by nickm＠torproject.org 12 May '16

12 May '16

commit 4e94580591da579e53461aeedfb1340836427c02 Merge: 039fc84 bf3e32a Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed May 11 13:13:23 2016 -0400 Merge remote-tracking branch 'special/bug19032-027' into maint-0.2.7 changes/bug19032 | 4 ++++ src/or/dirserv.c | 31 ++++++++++++++++--------------- src/or/dirserv.h | 2 +- src/or/networkstatus.c | 3 +-- 4 files changed, 22 insertions(+), 18 deletions(-)

1 0

[tor/release-0.2.8] Merge branch 'maint-0.2.7' into maint-0.2.8
by nickm＠torproject.org 12 May '16

12 May '16

commit 24fbb9a81b19cc20df9a204d3203d1b914fb267d Merge: 28e1aa1 4e94580 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed May 11 13:15:17 2016 -0400 Merge branch 'maint-0.2.7' into maint-0.2.8 changes/bug19032 | 4 ++++ src/or/dirserv.c | 31 ++++++++++++++++--------------- src/or/dirserv.h | 2 +- src/or/networkstatus.c | 3 +-- 4 files changed, 22 insertions(+), 18 deletions(-)

1 0

[tor/release-0.2.8] Merge remote-tracking branch 'teor/bug18816_simplify' into maint-0.2.8
by nickm＠torproject.org 12 May '16

12 May '16

commit 8d962233f65022dc7fbc0466c981e3e7f2aea0c8 Merge: 24fbb9a 797ece0 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed May 11 13:20:51 2016 -0400 Merge remote-tracking branch 'teor/bug18816_simplify' into maint-0.2.8 changes/bug18816 | 2 -- src/or/routerlist.c | 16 ++++++---------- 2 files changed, 6 insertions(+), 12 deletions(-)

1 0

[tor/release-0.2.8] Merge branch 'maint-0.2.7' into maint-0.2.8
by nickm＠torproject.org 12 May '16

12 May '16

commit 163cee1b6489c8966681b420d6da49e8ff4a9d1f Merge: 8d96223 d40e869 Author: Roger Dingledine <arma(a)torproject.org> Date: Wed May 11 13:42:40 2016 -0400 Merge branch 'maint-0.2.7' into maint-0.2.8 src/or/dirserv.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)

1 0

[tor/release-0.2.8] minor touchups on nick's ticket 17621 changes
by nickm＠torproject.org 12 May '16

12 May '16

commit 4a62d7aabc4e00d22f27d188f4cb2bb6efb3b674 Author: Roger Dingledine <arma(a)torproject.org> Date: Wed May 11 16:35:36 2016 -0400 minor touchups on nick's ticket 17621 changes --- changes/doc17621 | 2 +- doc/tor.1.txt | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/changes/doc17621 b/changes/doc17621 index af61d81..ab37d29 100644 --- a/changes/doc17621 +++ b/changes/doc17621 @@ -1,3 +1,3 @@ o Documentation: - Document the contents of the 'datadir/keys' subdirectory in the manual - page. Closes ticekt 17621. + page. Closes ticket 17621. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 5d85935..74915b7 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -2703,18 +2703,18 @@ __DataDirectory__**/keys/***:: Only used by servers. Holds identity keys and onion keys. __DataDirectory__**/keys/authority_identity_key**:: - A directory authority's master identity key, used to authenticate its + A v3 directory authority's master identity key, used to authenticate its signing key. Tor doesn't use this while it's running. The tor-gencert program uses this. If you're running an authority, you should keep this key offline, and not actually put it here. __DataDirectory__**/keys/authority_certificate**:: - A directory authority's certificate, which authenticates the authority's + A v3 directory authority's certificate, which authenticates the authority's current vote- and consensus-signing key using its master identity key. Only directory authorities use this file. __DataDirectory__**/keys/authority_signing_key**:: - A directory authority's signing key, used to sign votes and consensuses. + A v3 directory authority's signing key, used to sign votes and consensuses. Only directory authorities use this file. Corresponds to the **authority_certificate** cert. @@ -2765,9 +2765,8 @@ __DataDirectory__**/hashed-fingerprint**:: identity key. (That is, the hash of the hash of the identity key.) __DataDirectory__**/v3-status-votes**:: - Only for authoritative directory servers. This file contains status votes - from all the authoritative directory servers and is used to generate the - network consensus document. + Only for v3 authoritative directory servers. This file contains + status votes from all the authoritative directory servers. __DataDirectory__**/unverified-consensus**:: This file contains a network consensus document that has been downloaded,

1 0

[tor/release-0.2.8] Merge branch 'bug18977_024_v2' into bug18977_026_v2
by nickm＠torproject.org 12 May '16

12 May '16

commit 20b01cece8cd2ceb7f8051aee04294bdba471f1a Merge: 0b477bf e57f26c Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu May 12 14:39:06 2016 -0400 Merge branch 'bug18977_024_v2' into bug18977_026_v2 Had conflicts related to other correct_tm bugs in 0.2.6. Added wday for another case. changes/bug18977 | 4 ++++ src/common/compat.c | 4 ++++ src/common/util.c | 2 ++ 3 files changed, 10 insertions(+) diff --cc src/common/compat.c index 1788e32,90132fc..103ce24 --- a/src/common/compat.c +++ b/src/common/compat.c @@@ -2683,18 -2461,11 +2683,20 @@@ correct_tm(int islocal, const time_t *t r->tm_year = 8099; r->tm_mon = 11; r->tm_mday = 31; - r->tm_yday = 365; + r->tm_yday = 364; + r->tm_wday = 6; r->tm_hour = 23; r->tm_min = 59; r->tm_sec = 59; + } else if (r->tm_year < (1-1900)) { + r->tm_year = (1-1900); + r->tm_mon = 0; + r->tm_mday = 1; + r->tm_yday = 0; ++ r->tm_wday = 0; + r->tm_hour = 0; + r->tm_min = 0; + r->tm_sec = 0; } return r; } @@@ -2708,7 -2479,8 +2710,8 @@@ r->tm_year = 70; /* 1970 CE */ r->tm_mon = 0; r->tm_mday = 1; - r->tm_yday = 1; + r->tm_yday = 0; + r->tm_wday = 0; r->tm_hour = 0; r->tm_min = 0 ; r->tm_sec = 0; @@@ -2721,7 -2493,8 +2724,8 @@@ r->tm_year = 137; /* 2037 CE */ r->tm_mon = 11; r->tm_mday = 31; - r->tm_yday = 365; + r->tm_yday = 364; + r->tm_wday = 6; r->tm_hour = 23; r->tm_min = 59; r->tm_sec = 59;

1 0