August 2011 - tor-commits - lists.torproject.org

[torouter/master] torouter-prep package
by ioerror＠torproject.org 05 Aug '11

05 Aug '11

commit 66def778a541f7d7241c9ba02be774adbc495dd8 Author: Jacob Appelbaum <jacob(a)appelbaum.net> Date: Fri Aug 5 15:45:36 2011 -0700 torouter-prep package --- packages/torouter-prep/Makefile | 14 ++ packages/torouter-prep/debian/README | 6 + packages/torouter-prep/debian/README.Debian | 8 + packages/torouter-prep/debian/changelog | 5 + packages/torouter-prep/debian/compat | 1 + packages/torouter-prep/debian/control | 14 ++ packages/torouter-prep/debian/copyright | 36 ++++ packages/torouter-prep/debian/dirs | 1 + packages/torouter-prep/debian/files | 1 + packages/torouter-prep/debian/postinst | 39 +++++ packages/torouter-prep/debian/postrm | 37 ++++ packages/torouter-prep/debian/preinst | 35 ++++ packages/torouter-prep/debian/prerm | 38 +++++ packages/torouter-prep/debian/rules | 6 + packages/torouter-prep/debian/source/format | 1 + packages/torouter-prep/debian/torouter.default | 11 ++ packages/torouter-prep/debian/torouter.substvars | 1 + packages/torouter-prep/src/linux-tor-prio.sh | 192 ++++++++++++++++++++++ packages/torouter-prep/src/torouter_config.sh | 157 ++++++++++++++++++ packages/torouter-prep/src/torouter_takeover.sh | 3 + 20 files changed, 606 insertions(+), 0 deletions(-) diff --git a/packages/torouter-prep/Makefile b/packages/torouter-prep/Makefile new file mode 100644 index 0000000..1d80201 --- /dev/null +++ b/packages/torouter-prep/Makefile @@ -0,0 +1,14 @@ +default: build + +build: + echo "nothing to build" + +install: + install -o root -g root -m 750 src/torouter_config.sh $(DESTDIR)/usr/bin + install -o root -g root -m 750 src/torouter_takeover.sh $(DESTDIR)/usr/bin + +clean: + echo "nothing to clean" + +deb: + dpkg-buildpackage -us -uc diff --git a/packages/torouter-prep/debian/README b/packages/torouter-prep/debian/README new file mode 100644 index 0000000..bf746d1 --- /dev/null +++ b/packages/torouter-prep/debian/README @@ -0,0 +1,6 @@ +The Debian Package torouter +---------------------------- + +This package is for the Torouter project from the Tor Project. + + -- Jacob Appelbaum <jacob(a)torproject.org> Wed, 27 Jul 2011 12:27:38 -0400 diff --git a/packages/torouter-prep/debian/README.Debian b/packages/torouter-prep/debian/README.Debian new file mode 100644 index 0000000..27e430c --- /dev/null +++ b/packages/torouter-prep/debian/README.Debian @@ -0,0 +1,8 @@ +torouter for Debian +------------------- + +This is a package for Debian that converts a DreamPlug into a Torouter. + +It will configure the system without prompting the user for any input. + + -- Jacob Appelbaum <jacob(a)torproject.org> Wed, 27 Jul 2011 09:51:21 -0400 diff --git a/packages/torouter-prep/debian/changelog b/packages/torouter-prep/debian/changelog new file mode 100644 index 0000000..f62a4bb --- /dev/null +++ b/packages/torouter-prep/debian/changelog @@ -0,0 +1,5 @@ +torouter (0.0.1-1) unstable; urgency=low + + * Initial release (Closes: #nnnn) <nnnn is the bug number of your ITP> + + -- Jacob Appelbaum <jacob(a)appelbaum.net> Wed, 27 Jul 2011 10:08:02 -0400 diff --git a/packages/torouter-prep/debian/compat b/packages/torouter-prep/debian/compat new file mode 100644 index 0000000..7f8f011 --- /dev/null +++ b/packages/torouter-prep/debian/compat @@ -0,0 +1 @@ +7 diff --git a/packages/torouter-prep/debian/control b/packages/torouter-prep/debian/control new file mode 100644 index 0000000..f9930d6 --- /dev/null +++ b/packages/torouter-prep/debian/control @@ -0,0 +1,14 @@ +Source: torouter-prep +Section: unknown +Priority: extra +Maintainer: Jacob Appelbaum <jacob(a)torproject.org> +Build-Depends: debhelper (>= 7.0.50~) +Standards-Version: 3.9.1 +Homepage: <https://www.torproject.org/> + +Package: torouter +Architecture: all +Depends: ${misc:Depends}, apt-utils, openssh-server, isc-dhcp-client, cron-apt +Description: Torouter meta-package to take over the DreamPlug + DreamPlug devices may use this package to convert to a Torouter. + This package only works if the DreamPlug is already running Debian. diff --git a/packages/torouter-prep/debian/copyright b/packages/torouter-prep/debian/copyright new file mode 100644 index 0000000..9f4ebf2 --- /dev/null +++ b/packages/torouter-prep/debian/copyright @@ -0,0 +1,36 @@ +Format: http://dep.debian.net/deps/dep5 +Upstream-Name: torouter +Source: <https://www.torproject.org/> + +Files: * +Copyright: 2011 Jacob Appelbaum <jacob(a)appelbaum.net> +License: BSD-3-Clause + +Files: debian/* +Copyright: 2011 Jacob Appelbaum <jacob(a)torproject.org> +License: BSD-3-Clause + +License: BSD-3-Clause + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Neither the name of the University nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + . + THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. diff --git a/packages/torouter-prep/debian/dirs b/packages/torouter-prep/debian/dirs new file mode 100644 index 0000000..e772481 --- /dev/null +++ b/packages/torouter-prep/debian/dirs @@ -0,0 +1 @@ +usr/bin diff --git a/packages/torouter-prep/debian/files b/packages/torouter-prep/debian/files new file mode 100644 index 0000000..894b9ed --- /dev/null +++ b/packages/torouter-prep/debian/files @@ -0,0 +1 @@ +torouter_0.0.1-1_all.deb unknown extra diff --git a/packages/torouter-prep/debian/postinst b/packages/torouter-prep/debian/postinst new file mode 100644 index 0000000..97e9113 --- /dev/null +++ b/packages/torouter-prep/debian/postinst @@ -0,0 +1,39 @@ +#!/bin/sh +# postinst script for torouter +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * <postinst> `configure' <most-recently-configured-version> +# * <old-postinst> `abort-upgrade' <new version> +# * <conflictor's-postinst> `abort-remove' `in-favour' <package> +# <new-version> +# * <postinst> `abort-remove' +# * <deconfigured's-postinst> `abort-deconfigure' `in-favour' +# <failed-install-package> <version> `removing' +# <conflicting-package> <version> +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + configure) + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/packages/torouter-prep/debian/postrm b/packages/torouter-prep/debian/postrm new file mode 100644 index 0000000..632ac80 --- /dev/null +++ b/packages/torouter-prep/debian/postrm @@ -0,0 +1,37 @@ +#!/bin/sh +# postrm script for torouter +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * <postrm> `remove' +# * <postrm> `purge' +# * <old-postrm> `upgrade' <new-version> +# * <new-postrm> `failed-upgrade' <old-version> +# * <new-postrm> `abort-install' +# * <new-postrm> `abort-install' <old-version> +# * <new-postrm> `abort-upgrade' <old-version> +# * <disappearer's-postrm> `disappear' <overwriter> +# <overwriter-version> +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + purge|remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + ;; + + *) + echo "postrm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/packages/torouter-prep/debian/preinst b/packages/torouter-prep/debian/preinst new file mode 100644 index 0000000..1cc598e --- /dev/null +++ b/packages/torouter-prep/debian/preinst @@ -0,0 +1,35 @@ +#!/bin/sh +# preinst script for torouter +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * <new-preinst> `install' +# * <new-preinst> `install' <old-version> +# * <new-preinst> `upgrade' <old-version> +# * <old-preinst> `abort-upgrade' <new-version> +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + install|upgrade) + ;; + + abort-upgrade) + ;; + + *) + echo "preinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/packages/torouter-prep/debian/prerm b/packages/torouter-prep/debian/prerm new file mode 100644 index 0000000..4bb95dd --- /dev/null +++ b/packages/torouter-prep/debian/prerm @@ -0,0 +1,38 @@ +#!/bin/sh +# prerm script for torouter +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * <prerm> `remove' +# * <old-prerm> `upgrade' <new-version> +# * <new-prerm> `failed-upgrade' <old-version> +# * <conflictor's-prerm> `remove' `in-favour' <package> <new-version> +# * <deconfigured's-prerm> `deconfigure' `in-favour' +# <package-being-installed> <version> `removing' +# <conflicting-package> <version> +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + remove|upgrade|deconfigure) + ;; + + failed-upgrade) + ;; + + *) + echo "prerm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/packages/torouter-prep/debian/rules b/packages/torouter-prep/debian/rules new file mode 100755 index 0000000..74439ab --- /dev/null +++ b/packages/torouter-prep/debian/rules @@ -0,0 +1,6 @@ +#!/usr/bin/make -f +# -*- makefile -*- +#export DH_VERBOSE=1 + +%: + dh $@ diff --git a/packages/torouter-prep/debian/source/format b/packages/torouter-prep/debian/source/format new file mode 100644 index 0000000..89ae9db --- /dev/null +++ b/packages/torouter-prep/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/packages/torouter-prep/debian/torouter.default b/packages/torouter-prep/debian/torouter.default new file mode 100644 index 0000000..d5fcdf1 --- /dev/null +++ b/packages/torouter-prep/debian/torouter.default @@ -0,0 +1,11 @@ +# Defaults for torouter initscript +# sourced by /etc/init.d/torouter +# installed at /etc/default/torouter by the maintainer scripts + +# +# This is a POSIX shell fragment +# + +# When the package is configured - we set this accordingly +#DREAMPLUG="" +#CONFIGURED="" diff --git a/packages/torouter-prep/debian/torouter.substvars b/packages/torouter-prep/debian/torouter.substvars new file mode 100644 index 0000000..abd3ebe --- /dev/null +++ b/packages/torouter-prep/debian/torouter.substvars @@ -0,0 +1 @@ +misc:Depends= diff --git a/packages/torouter-prep/src/linux-tor-prio.sh b/packages/torouter-prep/src/linux-tor-prio.sh new file mode 100644 index 0000000..ea9e0dd --- /dev/null +++ b/packages/torouter-prep/src/linux-tor-prio.sh @@ -0,0 +1,192 @@ +#!/bin/bash +# Written by Marco Bonetti & Mike Perry +# Based on instructions from Dan Singletary's ADSL BW Management HOWTO: +# http://www.faqs.org/docs/Linux-HOWTO/ADSL-Bandwidth-Management-HOWTO.html +# This script is Public Domain. + +############################### README ################################# + +# This script provides prioritization of Tor traffic below other +# traffic on a Linux server. It has two modes of operation: UID based +# and IP based. + +# UID BASED PRIORITIZATION +# +# The UID based method requires that Tor be launched from +# a specific user ID. The "User" Tor config setting is +# insufficient, as it sets the UID after the socket is created. +# Here is a C wrapper you can use to execute Tor and drop privs before +# it creates any sockets. +# +# Compile with: +# gcc -DUID=`id -u tor` -DGID=`id -g tor` tor_wrap.c -o tor_wrap +# +# #include <unistd.h> +# int main(int argc, char **argv) { +# if(initgroups("tor", GID) == -1) { perror("initgroups"); return 1; } +# if(setresgid(GID, GID, GID) == -1) { perror("setresgid"); return 1; } +# if(setresuid(UID, UID, UID) == -1) { perror("setresuid"); return 1; } +# execl("/bin/tor", "/bin/tor", "-f", "/etc/tor/torrc", NULL); +# perror("execl"); return 1; +# } + +# IP BASED PRIORITIZATION +# +# The IP setting requires that a separate IP address be dedicated to Tor. +# Your Torrc should be set to bind to this IP for "OutboundBindAddress", +# "ListenAddress", and "Address". + +# GENERAL USAGE +# +# You should also tune the individual connection rate parameters below +# to your individual connection. In particular, you should leave *some* +# minimum amount of bandwidth for Tor, so that Tor users are not +# completely choked out when you use your server's bandwidth. 30% is +# probably a reasonable choice. More is better of course. +# +# To start the shaping, run it as: +# ./linux-tor-prio.sh +# +# To get status information (useful to verify packets are getting marked +# and prioritized), run: +# ./linux-tor-prio.sh status +# +# And to stop prioritization: +# ./linux-tor-prio.sh stop +# +######################################################################## + +# BEGIN USER TUNABLE PARAMETERS + +DEV=eth0 + +# NOTE! You must START Tor under this UID. Using the Tor User +# config setting is NOT sufficient. See above. +TOR_UID=$(id -u tor) + +# If the UID mechanism doesn't work for you, you can set this parameter +# instead. If set, it will take precedence over the UID setting. Note that +# you need multiple IPs with one specifically devoted to Tor for this to +# work. +#TOR_IP="42.42.42.42" + +# Average ping to most places on the net, milliseconds +RTT_LATENCY=40 + +# RATE_UP must be less than your connection's upload capacity in +# kbits/sec. If it is larger, then the bottleneck will be at your +# router's queue, which you do not control. This will cause congestion +# and a revert to normal TCP fairness no matter what the queing +# priority is. +RATE_UP=5000 + +# RATE_UP_TOR is the minimum speed your Tor connections will have in +# kbits/sec. They will have at least this much bandwidth for upload. +# In general, you probably shouldn't set this too low, or else Tor +# users who use your node will be completely choked out whenever your +# machine does any other network activity. That is not very fun. +RATE_UP_TOR=1500 + +# RATE_UP_TOR_CEIL is the maximum rate allowed for all Tor trafic in +# kbits/sec. +RATE_UP_TOR_CEIL=5000 + +CHAIN=OUTPUT +#CHAIN=PREROUTING +#CHAIN=POSTROUTING + +MTU=1500 +AVG_PKT=900 # should be more like 600 for non-exit nodes + +# END USER TUNABLE PARAMETERS + + + +# The queue size should be no larger than your bandwidth-delay +# product. This is RT latency*bandwidth/MTU/2 + +BDP=$(expr $RTT_LATENCY \* $RATE_UP / $AVG_PKT) + +# Further research indicates that the BDP calculations should use +# RTT/sqrt(n) where n is the expected number of active connections.. + +BDP=$(expr $BDP / 4) + +if [ "$1" = "status" ] +then + echo "[qdisc]" + tc -s qdisc show dev $DEV + tc -s qdisc show dev imq0 + echo "[class]" + tc -s class show dev $DEV + tc -s class show dev imq0 + echo "[filter]" + tc -s filter show dev $DEV + tc -s filter show dev imq0 + echo "[iptables]" + iptables -t mangle -L TORSHAPER-OUT -v -x 2> /dev/null + exit +fi + + +# Reset everything to a known state (cleared) +tc qdisc del dev $DEV root 2> /dev/null > /dev/null +tc qdisc del dev imq0 root 2> /dev/null > /dev/null +iptables -t mangle -D POSTROUTING -o $DEV -j TORSHAPER-OUT 2> /dev/null > /dev/null +iptables -t mangle -D PREROUTING -o $DEV -j TORSHAPER-OUT 2> /dev/null > /dev/null +iptables -t mangle -D OUTPUT -o $DEV -j TORSHAPER-OUT 2> /dev/null > /dev/null +iptables -t mangle -F TORSHAPER-OUT 2> /dev/null > /dev/null +iptables -t mangle -X TORSHAPER-OUT 2> /dev/null > /dev/null +ip link set imq0 down 2> /dev/null > /dev/null +rmmod imq 2> /dev/null > /dev/null + +if [ "$1" = "stop" ] +then + echo "Shaping removed on $DEV." + exit +fi + +# Outbound Shaping (limits total bandwidth to RATE_UP) + +ip link set dev $DEV qlen $BDP + +# Add HTB root qdisc, default is high prio +tc qdisc add dev $DEV root handle 1: htb default 20 + +# Add main rate limit class +tc class add dev $DEV parent 1: classid 1:1 htb rate ${RATE_UP}kbit + +# Create the two classes, giving Tor at least RATE_UP_TOR kbit and capping +# total upstream at RATE_UP so the queue is under our control. +tc class add dev $DEV parent 1:1 classid 1:20 htb rate $(expr $RATE_UP - $RATE_UP_TOR)kbit ceil ${RATE_UP}kbit prio 0 +tc class add dev $DEV parent 1:1 classid 1:21 htb rate $[$RATE_UP_TOR]kbit ceil ${RATE_UP_TOR_CEIL}kbit prio 10 + +# Start up pfifo +tc qdisc add dev $DEV parent 1:20 handle 20: pfifo limit $BDP +tc qdisc add dev $DEV parent 1:21 handle 21: pfifo limit $BDP + +# filter traffic into classes by fwmark +tc filter add dev $DEV parent 1:0 prio 0 protocol ip handle 20 fw flowid 1:20 +tc filter add dev $DEV parent 1:0 prio 0 protocol ip handle 21 fw flowid 1:21 + +# add TORSHAPER-OUT chain to the mangle table in iptables +iptables -t mangle -N TORSHAPER-OUT +iptables -t mangle -I $CHAIN -o $DEV -j TORSHAPER-OUT + + +# Set firewall marks +# Low priority to Tor +if [ ""$TOR_IP == "" ] +then + echo "Using UID-based QoS. UID $TOR_UID marked as low priority." + iptables -t mangle -A TORSHAPER-OUT -m owner --uid-owner $TOR_UID -j MARK --set-mark 21 +else + echo "Using IP-based QoS. $TOR_IP marked as low priority." + iptables -t mangle -A TORSHAPER-OUT -s $TOR_IP -j MARK --set-mark 21 +fi + +# High prio for everything else +iptables -t mangle -A TORSHAPER-OUT -m mark --mark 0 -j MARK --set-mark 20 + +echo "Outbound shaping added to $DEV. Rate for Tor upload at least: ${RATE_UP_TOR}Kbyte/sec." + diff --git a/packages/torouter-prep/src/torouter_config.sh b/packages/torouter-prep/src/torouter_config.sh new file mode 100644 index 0000000..7a6a581 --- /dev/null +++ b/packages/torouter-prep/src/torouter_config.sh @@ -0,0 +1,157 @@ +#!/bin/bash -x + +echo "This program will reconfigure your Debian system into a Torouter" +exit 0 +echo "This is where we'd take over the entire Torouter system" + +# For every file we touch, move it to the temp_dir and then tar it up in the end +temp_dir="`mktemp -d`" + +# Add a user +ADMINUSER="toradmin" +ADMINGROUP="toradmin" + +# Install the Tor repo key +gpg --keyserver keys.gnupg.net --recv 886DDD89 +gpg --export A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89 | apt-key add - + +cp /etc/hosts $temp_dir/ +# Stomp on the hosts file +cat << EOF > /etc/hosts +127.0.0.1 localhost +EOF + +cp /etc/hostname $temp_dir/ +# Set us to have a default host name +echo "torouter" > /etc/hostname + +# We need to prep apt to understand that we want packages from other repos +# We append to the current package list +cat << EOF >> /etc/apt/sources.list +# Tor's debian package repo: +deb http://deb.torproject.org/torproject.org squeeze main +deb http://deb.torproject.org/torproject.org experimental-squeeze main + +# Add Debian backports for OpenNTPD, libminiupnpc-dev, libminiupnpc5 +# http://packages.debian.org/squeeze-backports/libminiupnpc-dev +deb http://backports.debian.org/debian-backports squeeze-backports main contrib non-free + +# Add Debian experimental for libnatpmp0 +# http://packages.debian.org/experimental/libnatpmp0 +deb http://ftp.debian.org/debian experimental main +deb-src http://ftp.debian.org/debian experimental main + +EOF + +# We're creating this file to ensure we get updates +cat << 'EOF' > /etc/apt/preferences.d/backports +Package: * +Pin: release a=squeeze-backports +Pin-Priority: 200 +EOF + +apt-get -y update + +# Install some other packages here: +apt-get -y install denyhosts ufw + +# Allow us to set the clock: +apt-get -y -t squeeze-backports install openntpd + +# Install Tor and deps: +apt-get -y install tor tor-geoipdb + +# To build with natpmp support +apt-get -y -t experimental install libnatpmp0 + +# To build with miniupnpc support +apt-get -y -t squeeze-backports install libminiupnpc-dev +apt-get -y -t squeeze-backports install libminiupnpc5 + +# XXX +# We want to apt-get source tor and build it for the 0.2.3.x branch +# + +# Install a Tor controller: +apt-get -y install tor-arm + +# Install a normal dns cache for eth1 +apt-get -y install dnsmasq + +## +## Configuration stage of the script +## + +# Configure arm +zcat /usr/share/doc/tor-arm/armrc.sample.gz > ~$(ADMINUSER)/.armrc +# XXX This is where we will call torrc-takeover.py when it is packaged + +# XXX We should reconfigure /etc/inittab here + +# Configure the network +# eth0 is our "internet" interface with a dhcp client +cat << 'EOF' > /etc/network/interfaces +# The primary network interface +allow-hotplug eth0 +iface eth0 inet dhcp + +# +# XXX Configure eth1 and ap0 here +# + +EOF + +# XXX We should configure ufw here +# ufw allow +# XXX We should configure denyhosts +# XXX We should configure dnsmasq +# XXX We should configure the DHCP server here + +cp /etc/tor/torrc $temp_dir/ +# configure Tor and stomp on the current Tor config +cat << 'EOF' > /etc/tor/torrc +# Run Tor as a bridge/relay only, not as a client +SocksPort 0 + +# What port to advertise for incoming Tor connections +ORPort 443 + +# We're on a flash file system +AvoidDiskWrites 1 + +# Be a bridge +BridgeRelay 1 + +# Rate limited +BandwidthRate 50KB + +# Don't allow any Tor traffic to exit +Exitpolicy reject *:* + +# Allow a controller (tor-arm) on this system to configure Tor: +ControlPort 9051 +ControlListenAddress 127.0.0.1:9051 +CookieAuthentication 1 +EOF + +# Remove a bunch of stuff: +apt-get -y remove exim4-base exim4-config exim4-daemon-light dbus + +## Disable ipv6 support +cp /etc/sysctl.d/disableipv6.conf $temp_dir/ +echo net.ipv6.conf.all.disable_ipv6=1 > /etc/sysctl.d/disableipv6.conf +cp /etc/sshd_config $temp_dir/ +echo "AddressFamily inet" >> /etc/ssh/ssh_config + +## +## Restart services here +## + +/etc/init.d/ssh restart +/etc/init.d/tor restart + +## +## Touch a stamp to show that we're now a Torouter +## + +echo "torouter" > /etc/torouter diff --git a/packages/torouter-prep/src/torouter_takeover.sh b/packages/torouter-prep/src/torouter_takeover.sh new file mode 100644 index 0000000..2b76502 --- /dev/null +++ b/packages/torouter-prep/src/torouter_takeover.sh @@ -0,0 +1,3 @@ +# This program will wrap debtakeover: +# http://www.hadrons.org/~guillem/debian/debtakeover/ +

1 0

[metrics-web/master] Be more precise about using the database together with Tor Status.
by karsten＠torproject.org 05 Aug '11

05 Aug '11

commit 57dae5651a2f9d8499c66913ba7f896997194799 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Aug 5 14:03:20 2011 +0200 Be more precise about using the database together with Tor Status. Tor Status will need extra-info descriptors, so we should tell Tor to download them. --- README | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/README b/README index 111543a..8b3651f 100644 --- a/README +++ b/README @@ -168,6 +168,7 @@ directory. Configure a local Tor client to fetch all known descriptors as early as possible by adding these config options to its torrc file: +DownloadExtraInfo 1 FetchUselessDescriptors 1 FetchDirInfoEarly 1 FetchDirInfoExtraEarly 1

1 0

[tor/master] Separate generation of an entry-stats string from writing it to disk.
by nickm＠torproject.org 04 Aug '11

04 Aug '11

commit 88083463cb895df0286210830e14c635c9e026a5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Aug 4 14:14:01 2011 +0200 Separate generation of an entry-stats string from writing it to disk. This commit is similar to the previous two commits for dirreq-stats, but for entry-stats. --- src/or/geoip.c | 75 +++++++++++++++++++++++++++++++++++------------------- src/or/geoip.h | 2 + src/test/test.c | 49 +++++++++++++++++++++++++++++++++++- 3 files changed, 98 insertions(+), 28 deletions(-) diff --git a/src/or/geoip.c b/src/or/geoip.c index acc00a8..e525b2a 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -1332,25 +1332,51 @@ geoip_entry_stats_init(time_t now) start_of_entry_stats_interval = now; } +/** Reset counters for entry stats. */ +void +geoip_reset_entry_stats(time_t now) +{ + client_history_clear(); + start_of_entry_stats_interval = now; +} + /** Stop collecting entry stats in a way that we can re-start doing so in * geoip_entry_stats_init(). */ void geoip_entry_stats_term(void) { - client_history_clear(); - start_of_entry_stats_interval = 0; + geoip_reset_entry_stats(0); +} + +/** Return a newly allocated string containing the entry statistics + * until now, or NULL if we're not collecting entry stats. */ +char * +geoip_format_entry_stats(time_t now) +{ + char t[ISO_TIME_LEN+1]; + char *data = NULL; + char *result; + + if (!start_of_entry_stats_interval) + return NULL; /* Not initialized. */ + + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + format_iso_time(t, now); + tor_asprintf(&result, "entry-stats-end %s (%u s)\nentry-ips %s\n", + t, (unsigned) (now - start_of_entry_stats_interval), + data ? data : ""); + tor_free(data); + return result; } -/** Write entry statistics to $DATADIR/stats/entry-stats and return time - * when we would next want to write. */ +/** If 24 hours have passed since the beginning of the current entry stats + * period, write entry stats to $DATADIR/stats/entry-stats (possibly + * overwriting an existing file) and reset counters. Return when we would + * next want to write entry stats or 0 if we never want to write. */ time_t geoip_entry_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL; - char *data = NULL; - char written[ISO_TIME_LEN+1]; - open_file_t *open_file = NULL; - FILE *out; + char *statsdir = NULL, *filename = NULL, *str = NULL; if (!start_of_entry_stats_interval) return 0; /* Not initialized. */ @@ -1360,31 +1386,26 @@ geoip_entry_stats_write(time_t now) /* Discard all items in the client history that are too old. */ geoip_remove_old_clients(start_of_entry_stats_interval); + /* Generate history string .*/ + str = geoip_format_entry_stats(now); + + /* Write entry-stats string to disk. */ statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) + if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { + log_warn(LD_HIST, "Unable to create stats/ directory!"); goto done; + } filename = get_datadir_fname2("stats", "entry-stats"); - data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); - format_iso_time(written, now); - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE | O_TEXT, - 0600, &open_file); - if (!out) - goto done; - if (fprintf(out, "entry-stats-end %s (%u s)\nentry-ips %s\n", - written, (unsigned) (now - start_of_entry_stats_interval), - data ? data : "") < 0) - goto done; + if (write_str_to_file(filename, str, 0) < 0) + log_warn(LD_HIST, "Unable to write entry statistics to disk!"); - start_of_entry_stats_interval = now; + /* Reset measurement interval start. */ + geoip_reset_entry_stats(now); - finish_writing_to_file(open_file); - open_file = NULL; done: - if (open_file) - abort_writing_to_file(open_file); - tor_free(filename); tor_free(statsdir); - tor_free(data); + tor_free(filename); + tor_free(str); return start_of_entry_stats_interval + WRITE_STATS_INTERVAL; } diff --git a/src/or/geoip.h b/src/or/geoip.h index 3472ee2..3372260 100644 --- a/src/or/geoip.h +++ b/src/or/geoip.h @@ -50,6 +50,8 @@ void geoip_dirreq_stats_term(void); void geoip_entry_stats_init(time_t now); time_t geoip_entry_stats_write(time_t now); void geoip_entry_stats_term(void); +void geoip_reset_entry_stats(time_t now); +char *geoip_format_entry_stats(time_t now); void geoip_bridge_stats_init(time_t now); time_t geoip_bridge_stats_write(time_t now); void geoip_bridge_stats_term(void); diff --git a/src/test/test.c b/src/test/test.c index 9e34984..fe8146a 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -1538,7 +1538,13 @@ test_geoip(void) "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" "dirreq-v2-direct-dl complete=0,timeout=0,running=0\n" "dirreq-v3-tunneled-dl complete=0,timeout=0,running=4\n" - "dirreq-v2-tunneled-dl complete=0,timeout=0,running=0\n"; + "dirreq-v2-tunneled-dl complete=0,timeout=0,running=0\n", + *entry_stats_1 = + "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "entry-ips ab=8\n", + *entry_stats_2 = + "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "entry-ips \n"; /* Populate the DB a bit. Add these in order, since we can't do the final * 'sort' step. These aren't very good IP addresses, but they're perfectly @@ -1590,6 +1596,7 @@ test_geoip(void) /* Stop being a bridge and start being a directory mirror that gathers * directory request statistics. */ + geoip_bridge_stats_term(); get_options_mutable()->BridgeRelay = 0; get_options_mutable()->BridgeRecordUsageByCountry = 0; get_options_mutable()->DirReqStatistics = 1; @@ -1637,6 +1644,46 @@ test_geoip(void) s = geoip_format_dirreq_stats(now + 86400); test_streq(dirreq_stats_4, s); + /* Stop collecting directory request statistics and start gathering + * entry stats. */ + geoip_dirreq_stats_term(); + get_options_mutable()->DirReqStatistics = 0; + get_options_mutable()->EntryStatistics = 1; + + /* Start testing entry statistics by making sure that we don't collect + * anything without initializing entry stats. */ + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, 100, now); + s = geoip_format_entry_stats(now + 86400); + test_assert(!s); + + /* Initialize stats, note one connecting client, and generate the + * entry-stats history string. */ + geoip_entry_stats_init(now); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, 100, now); + s = geoip_format_entry_stats(now + 86400); + test_streq(entry_stats_1, s); + tor_free(s); + + /* Stop collecting stats, add another connecting client, and ensure we + * don't generate a history string. */ + geoip_entry_stats_term(); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, 101, now); + s = geoip_format_entry_stats(now + 86400); + test_assert(!s); + + /* Re-start stats, add a connecting client, reset stats, and make sure + * that we get an all empty history string. */ + geoip_entry_stats_init(now); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, 100, now); + geoip_reset_entry_stats(now); + s = geoip_format_entry_stats(now + 86400); + test_streq(entry_stats_2, s); + tor_free(s); + + /* Stop collecting entry statistics. */ + geoip_entry_stats_term(); + get_options_mutable()->EntryStatistics = 0; + done: tor_free(s); }

1 0

[tor/master] Separate generation of a dirreq-stats string from writing it to disk.
by nickm＠torproject.org 04 Aug '11

04 Aug '11

commit 2174fc0ba036f3b0143d14626b431fa93ee6653d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Aug 4 12:28:12 2011 +0200 Separate generation of a dirreq-stats string from writing it to disk. This patch separates the generation of a dirreq-stats string from actually writing it to disk. The new geoip_format_dirreq_stats() generates a dirreq-stats string that geoip_dirreq_stats_write() writes to disk. All the state changing (e.g., resetting the dirreq-stats history and initializing the next measurement interval) takes place in geoip_dirreq_stats_write(). That allows us to finally test the dirreq-stats code better. --- src/or/geoip.c | 84 ++++++++++++++++++++++++------------------ src/or/geoip.h | 2 + src/test/test.c | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 156 insertions(+), 37 deletions(-) diff --git a/src/or/geoip.c b/src/or/geoip.c index aff636d..acc00a8 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -930,10 +930,9 @@ geoip_dirreq_stats_init(time_t now) start_of_dirreq_stats_interval = now; } -/** Stop collecting directory request stats in a way that we can re-start - * doing so in geoip_dirreq_stats_init(). */ +/** Reset counters for dirreq stats. */ void -geoip_dirreq_stats_term(void) +geoip_reset_dirreq_stats(time_t now) { SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { c->n_v2_ns_requests = c->n_v3_ns_requests = 0; @@ -965,19 +964,24 @@ geoip_dirreq_stats_term(void) tor_free(this); } } - start_of_dirreq_stats_interval = 0; + start_of_dirreq_stats_interval = now; } -/** Write dirreq statistics to $DATADIR/stats/dirreq-stats and return when - * we would next want to write. */ -time_t -geoip_dirreq_stats_write(time_t now) +/** Stop collecting directory request stats in a way that we can re-start + * doing so in geoip_dirreq_stats_init(). */ +void +geoip_dirreq_stats_term(void) +{ + geoip_reset_dirreq_stats(0); +} + +/** Return a newly allocated string containing the dirreq statistics + * until now, or NULL if we're not collecting dirreq stats. */ +char * +geoip_format_dirreq_stats(time_t now) { - char *statsdir = NULL, *filename = NULL; - open_file_t *open_file = NULL; char t[ISO_TIME_LEN+1]; double v2_share = 0.0, v3_share = 0.0; - FILE *out; int i; char *v3_ips_string, *v2_ips_string, *v3_reqs_string, *v2_reqs_string, *v2_share_string = NULL, *v3_share_string = NULL, @@ -986,12 +990,7 @@ geoip_dirreq_stats_write(time_t now) char *result; if (!start_of_dirreq_stats_interval) - return 0; /* Not initialized. */ - if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now) - goto done; /* Not ready to write. */ - - /* Discard all items in the client history that are too old. */ - geoip_remove_old_clients(start_of_dirreq_stats_interval); + return NULL; /* Not initialized. */ format_iso_time(t, now); v2_ips_string = geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS_V2); @@ -1078,34 +1077,47 @@ geoip_dirreq_stats_write(time_t now) tor_free(v3_tunneled_dl_string); tor_free(v2_tunneled_dl_string); - /* Reset history to prepare for the next measurement interval. */ - SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - c->n_v2_ns_requests = c->n_v3_ns_requests = 0; - }); - memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); - memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); - start_of_dirreq_stats_interval = now; + return result; +} + +/** If 24 hours have passed since the beginning of the current dirreq + * stats period, write dirreq stats to $DATADIR/stats/dirreq-stats + * (possibly overwriting an existing file) and reset counters. Return + * when we would next want to write dirreq stats or 0 if we never want to + * write. */ +time_t +geoip_dirreq_stats_write(time_t now) +{ + char *statsdir = NULL, *filename = NULL, *str = NULL; + + if (!start_of_dirreq_stats_interval) + return 0; /* Not initialized. */ + if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_dirreq_stats_interval); + + /* Generate history string .*/ + str = geoip_format_dirreq_stats(now); /* Write dirreq-stats string to disk. */ statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) + if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { + log_warn(LD_HIST, "Unable to create stats/ directory!"); goto done; + } filename = get_datadir_fname2("stats", "dirreq-stats"); - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE | O_TEXT, - 0600, &open_file); - if (!out) - goto done; - if (fprintf(out, "%s", result) < 0) - goto done; - finish_writing_to_file(open_file); - open_file = NULL; + if (write_str_to_file(filename, str, 0) < 0) + log_warn(LD_HIST, "Unable to write dirreq statistics to disk!"); + + /* Reset measurement interval start. */ + geoip_reset_dirreq_stats(now); done: - if (open_file) - abort_writing_to_file(open_file); tor_free(statsdir); tor_free(filename); - tor_free(result); + tor_free(str); return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL; } diff --git a/src/or/geoip.h b/src/or/geoip.h index b50da74..3472ee2 100644 --- a/src/or/geoip.h +++ b/src/or/geoip.h @@ -43,6 +43,8 @@ void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, dirreq_state_t new_state); void geoip_dirreq_stats_init(time_t now); +void geoip_reset_dirreq_stats(time_t now); +char *geoip_format_dirreq_stats(time_t now); time_t geoip_dirreq_stats_write(time_t now); void geoip_dirreq_stats_term(void); void geoip_entry_stats_init(time_t now); diff --git a/src/test/test.c b/src/test/test.c index 4ef87a0..9e34984 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -1481,8 +1481,64 @@ static void test_geoip(void) { int i, j; - time_t now = time(NULL); + time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */ char *s = NULL; + const char *dirreq_stats_1 = + "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "dirreq-v3-ips ab=8\n" + "dirreq-v2-ips \n" + "dirreq-v3-reqs ab=8\n" + "dirreq-v2-reqs \n" + "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0," + "not-modified=0,busy=0\n" + "dirreq-v2-resp ok=0,unavailable=0,not-found=0,not-modified=0," + "busy=0\n" + "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v2-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n" + "dirreq-v2-tunneled-dl complete=0,timeout=0,running=0\n", + *dirreq_stats_2 = + "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "dirreq-v3-ips \n" + "dirreq-v2-ips \n" + "dirreq-v3-reqs \n" + "dirreq-v2-reqs \n" + "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0," + "not-modified=0,busy=0\n" + "dirreq-v2-resp ok=0,unavailable=0,not-found=0,not-modified=0," + "busy=0\n" + "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v2-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n" + "dirreq-v2-tunneled-dl complete=0,timeout=0,running=0\n", + *dirreq_stats_3 = + "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "dirreq-v3-ips \n" + "dirreq-v2-ips \n" + "dirreq-v3-reqs \n" + "dirreq-v2-reqs \n" + "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0," + "not-modified=0,busy=0\n" + "dirreq-v2-resp ok=0,unavailable=0,not-found=0,not-modified=0," + "busy=0\n" + "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v2-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n" + "dirreq-v2-tunneled-dl complete=0,timeout=0,running=0\n", + *dirreq_stats_4 = + "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "dirreq-v3-ips \n" + "dirreq-v2-ips \n" + "dirreq-v3-reqs \n" + "dirreq-v2-reqs \n" + "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0," + "not-modified=0,busy=0\n" + "dirreq-v2-resp ok=0,unavailable=0,not-found=0,not-modified=0," + "busy=0\n" + "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v2-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v3-tunneled-dl complete=0,timeout=0,running=4\n" + "dirreq-v2-tunneled-dl complete=0,timeout=0,running=0\n"; /* Populate the DB a bit. Add these in order, since we can't do the final * 'sort' step. These aren't very good IP addresses, but they're perfectly @@ -1532,6 +1588,55 @@ test_geoip(void) test_assert(s); test_streq("zz=24,xy=8", s); + /* Stop being a bridge and start being a directory mirror that gathers + * directory request statistics. */ + get_options_mutable()->BridgeRelay = 0; + get_options_mutable()->BridgeRecordUsageByCountry = 0; + get_options_mutable()->DirReqStatistics = 1; + + /* Start testing dirreq statistics by making sure that we don't collect + * dirreq stats without initializing them. */ + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, 100, now); + s = geoip_format_dirreq_stats(now + 86400); + test_assert(!s); + + /* Initialize stats, note one connecting client, and generate the + * dirreq-stats history string. */ + geoip_dirreq_stats_init(now); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, 100, now); + s = geoip_format_dirreq_stats(now + 86400); + test_streq(dirreq_stats_1, s); + tor_free(s); + + /* Stop collecting stats, add another connecting client, and ensure we + * don't generate a history string. */ + geoip_dirreq_stats_term(); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, 101, now); + s = geoip_format_dirreq_stats(now + 86400); + test_assert(!s); + + /* Re-start stats, add a connecting client, reset stats, and make sure + * that we get an all empty history string. */ + geoip_dirreq_stats_init(now); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, 100, now); + geoip_reset_dirreq_stats(now); + s = geoip_format_dirreq_stats(now + 86400); + test_streq(dirreq_stats_2, s); + tor_free(s); + + /* Note a successful network status response and make sure that it + * appears in the history string. */ + geoip_note_ns_response(GEOIP_CLIENT_NETWORKSTATUS, GEOIP_SUCCESS); + s = geoip_format_dirreq_stats(now + 86400); + test_streq(dirreq_stats_3, s); + tor_free(s); + + /* Start a tunneled directory request. */ + geoip_start_dirreq((uint64_t) 1, 1024, GEOIP_CLIENT_NETWORKSTATUS, + DIRREQ_TUNNELED); + s = geoip_format_dirreq_stats(now + 86400); + test_streq(dirreq_stats_4, s); + done: tor_free(s); }

1 0

[tor/master] Clean up bridge-stats code.
by nickm＠torproject.org 04 Aug '11

04 Aug '11

commit d690a99dfe65ba164aa3de0662be2a8fedc69b01 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Aug 4 14:45:24 2011 +0200 Clean up bridge-stats code. Only write a bridge-stats string if bridge stats have been initialized. This behavior is similar to dirreq-stats, entry-stats, etc. Also add a few unit tests for the bridge-stats code. --- src/or/geoip.c | 8 +++++--- src/or/geoip.h | 1 + src/test/test.c | 23 ++++++++++++++++++++++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/or/geoip.c b/src/or/geoip.c index e525b2a..67dea96 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -1197,8 +1197,8 @@ static char *bridge_stats_extrainfo = NULL; /** Return a newly allocated string holding our bridge usage stats by country * in a format suitable for inclusion in an extrainfo document. Return NULL on * failure. */ -static char * -format_bridge_stats_extrainfo(time_t now) +char * +geoip_format_bridge_stats(time_t now) { char *out = NULL, *data = NULL; long duration = now - start_of_bridge_stats_interval; @@ -1206,6 +1206,8 @@ format_bridge_stats_extrainfo(time_t now) if (duration < 0) return NULL; + if (!start_of_bridge_stats_interval) + return NULL; /* Not initialized. */ format_iso_time(written, now); data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); @@ -1255,7 +1257,7 @@ geoip_bridge_stats_write(time_t now) geoip_remove_old_clients(start_of_bridge_stats_interval); /* Generate formatted string */ - val = format_bridge_stats_extrainfo(now); + val = geoip_format_bridge_stats(now); if (val == NULL) goto done; diff --git a/src/or/geoip.h b/src/or/geoip.h index 3372260..ce38419 100644 --- a/src/or/geoip.h +++ b/src/or/geoip.h @@ -53,6 +53,7 @@ void geoip_entry_stats_term(void); void geoip_reset_entry_stats(time_t now); char *geoip_format_entry_stats(time_t now); void geoip_bridge_stats_init(time_t now); +char *geoip_format_bridge_stats(time_t now); time_t geoip_bridge_stats_write(time_t now); void geoip_bridge_stats_term(void); const char *geoip_get_bridge_stats_extrainfo(time_t); diff --git a/src/test/test.c b/src/test/test.c index fe8146a..605e44c 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -1483,7 +1483,10 @@ test_geoip(void) int i, j; time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */ char *s = NULL; - const char *dirreq_stats_1 = + const char *bridge_stats_1 = + "bridge-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "bridge-ips zz=24,xy=8\n", + *dirreq_stats_1 = "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" "dirreq-v3-ips ab=8\n" "dirreq-v2-ips \n" @@ -1594,6 +1597,24 @@ test_geoip(void) test_assert(s); test_streq("zz=24,xy=8", s); + /* Start testing bridge statistics by making sure that we don't output + * bridge stats without initializing them. */ + s = geoip_format_bridge_stats(now + 86400); + test_assert(!s); + + /* Initialize stats and generate the bridge-stats history string out of + * the connecting clients added above. */ + geoip_bridge_stats_init(now); + s = geoip_format_bridge_stats(now + 86400); + test_streq(bridge_stats_1, s); + tor_free(s); + + /* Stop collecting bridge stats and make sure we don't write a history + * string anymore. */ + geoip_bridge_stats_term(); + s = geoip_format_bridge_stats(now + 86400); + test_assert(!s); + /* Stop being a bridge and start being a directory mirror that gathers * directory request statistics. */ geoip_bridge_stats_term();

1 0

[tor/master] Replace files in stats/ rather than appending to them.
by nickm＠torproject.org 04 Aug '11

04 Aug '11

commit a6a127c833eace1100aca7ab8ad118862bb8a8b9 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Aug 4 20:53:53 2011 +0200 Replace files in stats/ rather than appending to them. Right now, we append statistics to files in the stats/ directory for half of the statistics, whereas we overwrite these files for the other half. In particular, we append buffer, dirreq, and entry stats and overwrite exit, connection, and bridge stats. Appending to files was useful when we didn't include stats in extra-info descriptors, because otherwise we'd have to copy them away to prevent Tor from overwriting them. But now that we include statistics in extra-info descriptors, it makes no sense to keep the old statistics forever. We should change the behavior to overwriting instead of appending for all statistics. Implements #2930. --- changes/bug2930 | 5 +++++ src/or/geoip.c | 4 ++-- src/or/rephist.c | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/changes/bug2930 b/changes/bug2930 new file mode 100644 index 0000000..8d28664 --- /dev/null +++ b/changes/bug2930 @@ -0,0 +1,5 @@ + o Minor features: + - Replace files in stats/ rather than appending to them. Now that we + include statistics in extra-info descriptors, it makes no sense to + keep old statistics forever. Implements #2930. + diff --git a/src/or/geoip.c b/src/or/geoip.c index 5596ff3..d02666b 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -996,7 +996,7 @@ geoip_dirreq_stats_write(time_t now) data_v2 = geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS_V2); data_v3 = geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS); format_iso_time(written, now); - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND | O_TEXT, + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE | O_TEXT, 0600, &open_file); if (!out) goto done; @@ -1329,7 +1329,7 @@ geoip_entry_stats_write(time_t now) filename = get_datadir_fname2("stats", "entry-stats"); data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); format_iso_time(written, now); - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND | O_TEXT, + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE | O_TEXT, 0600, &open_file); if (!out) goto done; diff --git a/src/or/rephist.c b/src/or/rephist.c index 1ad08fe..672e3f3 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -2490,7 +2490,7 @@ rep_hist_buffer_stats_write(time_t now) if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) goto done; filename = get_datadir_fname2("stats", "buffer-stats"); - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND | O_TEXT, + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE | O_TEXT, 0600, &open_file); if (!out) goto done;

1 0

[tor/master] Create cell-stats history even if we didn't see a single circuit.
by nickm＠torproject.org 04 Aug '11

04 Aug '11

commit 8f4e5fa2b5e78d04ac8d656f58db22110f2e450d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Aug 3 13:33:48 2011 +0200 Create cell-stats history even if we didn't see a single circuit. So far, if we didn't see a single circuit, we refrained from generating a cell-stats string and logged a warning. Nobody will notice the warning, and people will wonder why there's no cell-stats string in the extra-info descriptor. The better behavior is to generate a cell-stats string with all zeros. --- src/or/rephist.c | 35 +++++++++++++++++------------------ 1 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index 672e3f3..a5a828d 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -2455,32 +2455,31 @@ rep_hist_buffer_stats_write(time_t now) /* add current circuits to stats */ for (circ = _circuit_get_global_list(); circ; circ = circ->next) rep_hist_buffer_stats_add_circ(circ, now); - /* calculate deciles */ + + /* Calculate deciles if we saw at least one circuit. */ memset(processed_cells, 0, SHARES * sizeof(int)); memset(circs_in_share, 0, SHARES * sizeof(int)); memset(queued_cells, 0, SHARES * sizeof(double)); memset(time_in_queue, 0, SHARES * sizeof(double)); if (!circuits_for_buffer_stats) circuits_for_buffer_stats = smartlist_create(); - smartlist_sort(circuits_for_buffer_stats, - _buffer_stats_compare_entries); number_of_circuits = smartlist_len(circuits_for_buffer_stats); - if (number_of_circuits < 1) { - log_info(LD_HIST, "Attempt to write cell statistics to disk failed. " - "We haven't seen a single circuit to report about."); - goto done; - } - i = 0; - SMARTLIST_FOREACH_BEGIN(circuits_for_buffer_stats, - circ_buffer_stats_t *, stat) - { - int share = i++ * SHARES / number_of_circuits; - processed_cells[share] += stat->processed_cells; - queued_cells[share] += stat->mean_num_cells_in_queue; - time_in_queue[share] += stat->mean_time_cells_in_queue; - circs_in_share[share]++; + if (number_of_circuits > 0) { + smartlist_sort(circuits_for_buffer_stats, + _buffer_stats_compare_entries); + i = 0; + SMARTLIST_FOREACH_BEGIN(circuits_for_buffer_stats, + circ_buffer_stats_t *, stat) + { + int share = i++ * SHARES / number_of_circuits; + processed_cells[share] += stat->processed_cells; + queued_cells[share] += stat->mean_num_cells_in_queue; + time_in_queue[share] += stat->mean_time_cells_in_queue; + circs_in_share[share]++; + } + SMARTLIST_FOREACH_END(stat); } - SMARTLIST_FOREACH_END(stat); + /* clear buffer stats history */ SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, stat, tor_free(stat));

1 0

[tor/master] Clean up rep_hist_buffer_stats_write().
by nickm＠torproject.org 04 Aug '11

04 Aug '11

commit 95ebd01e6225fecc2af6d9adb0167f1ab6a363fc Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Aug 3 13:34:29 2011 +0200 Clean up rep_hist_buffer_stats_write(). We later want to split this function into one function that generates the buffer-stats string and one that writes it to disk. --- src/or/rephist.c | 95 +++++++++++++++++++++++++++++------------------------- 1 files changed, 51 insertions(+), 44 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index a5a828d..ce0152e 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -2434,24 +2434,25 @@ time_t rep_hist_buffer_stats_write(time_t now) { char *statsdir = NULL, *filename = NULL; - char written[ISO_TIME_LEN+1]; open_file_t *open_file = NULL; FILE *out; #define SHARES 10 int processed_cells[SHARES], circs_in_share[SHARES], number_of_circuits, i; double queued_cells[SHARES], time_in_queue[SHARES]; - smartlist_t *str_build = NULL; - char *str = NULL, *buf = NULL; + char *buf = NULL; circuit_t *circ; + smartlist_t *processed_cells_strings, *queued_cells_strings, + *time_in_queue_strings; + char *processed_cells_string, *queued_cells_string, + *time_in_queue_string; + char t[ISO_TIME_LEN+1]; if (!start_of_buffer_stats_interval) return 0; /* Not initialized. */ if (start_of_buffer_stats_interval + WRITE_STATS_INTERVAL > now) goto done; /* Not ready to write */ - str_build = smartlist_create(); - /* add current circuits to stats */ for (circ = _circuit_get_global_list(); circ; circ = circ->next) rep_hist_buffer_stats_add_circ(circ, now); @@ -2484,53 +2485,61 @@ rep_hist_buffer_stats_write(time_t now) SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, stat, tor_free(stat)); smartlist_clear(circuits_for_buffer_stats); - /* write to file */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) - goto done; - filename = get_datadir_fname2("stats", "buffer-stats"); - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE | O_TEXT, - 0600, &open_file); - if (!out) - goto done; - format_iso_time(written, now); - if (fprintf(out, "cell-stats-end %s (%d s)\n", written, - (unsigned) (now - start_of_buffer_stats_interval)) < 0) - goto done; + + /* Write deciles to strings. */ + processed_cells_strings = smartlist_create(); + queued_cells_strings = smartlist_create(); + time_in_queue_strings = smartlist_create(); for (i = 0; i < SHARES; i++) { tor_asprintf(&buf,"%d", !circs_in_share[i] ? 0 : processed_cells[i] / circs_in_share[i]); - smartlist_add(str_build, buf); + smartlist_add(processed_cells_strings, buf); } - str = smartlist_join_strings(str_build, ",", 0, NULL); - if (fprintf(out, "cell-processed-cells %s\n", str) < 0) - goto done; - tor_free(str); - SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); - smartlist_clear(str_build); for (i = 0; i < SHARES; i++) { tor_asprintf(&buf, "%.2f", circs_in_share[i] == 0 ? 0.0 : queued_cells[i] / (double) circs_in_share[i]); - smartlist_add(str_build, buf); + smartlist_add(queued_cells_strings, buf); } - str = smartlist_join_strings(str_build, ",", 0, NULL); - if (fprintf(out, "cell-queued-cells %s\n", str) < 0) - goto done; - tor_free(str); - SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); - smartlist_clear(str_build); for (i = 0; i < SHARES; i++) { tor_asprintf(&buf, "%.0f", circs_in_share[i] == 0 ? 0.0 : time_in_queue[i] / (double) circs_in_share[i]); - smartlist_add(str_build, buf); + smartlist_add(time_in_queue_strings, buf); } - str = smartlist_join_strings(str_build, ",", 0, NULL); - if (fprintf(out, "cell-time-in-queue %s\n", str) < 0) + + /* Join all observations in single strings. */ + processed_cells_string = smartlist_join_strings(processed_cells_strings, + ",", 0, NULL); + queued_cells_string = smartlist_join_strings(queued_cells_strings, + ",", 0, NULL); + time_in_queue_string = smartlist_join_strings(time_in_queue_strings, + ",", 0, NULL); + SMARTLIST_FOREACH(processed_cells_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(queued_cells_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(time_in_queue_strings, char *, cp, tor_free(cp)); + smartlist_free(processed_cells_strings); + smartlist_free(queued_cells_strings); + smartlist_free(time_in_queue_strings); + + /* Write everything to disk. */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) + goto done; + filename = get_datadir_fname2("stats", "buffer-stats"); + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE | O_TEXT, + 0600, &open_file); + if (!out) + goto done; + format_iso_time(t, now); + if (fprintf(out, "cell-stats-end %s (%d s)\n", t, + (unsigned) (now - start_of_buffer_stats_interval)) < 0) + goto done; + if (fprintf(out, "cell-processed-cells %s\n", processed_cells_string) + < 0) + goto done; + if (fprintf(out, "cell-queued-cells %s\n", queued_cells_string) < 0) + goto done; + if (fprintf(out, "cell-time-in-queue %s\n", time_in_queue_string) < 0) goto done; - tor_free(str); - SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); - smartlist_free(str_build); - str_build = NULL; if (fprintf(out, "cell-circuits-per-decile %d\n", (number_of_circuits + SHARES - 1) / SHARES) < 0) goto done; @@ -2542,11 +2551,9 @@ rep_hist_buffer_stats_write(time_t now) abort_writing_to_file(open_file); tor_free(filename); tor_free(statsdir); - if (str_build) { - SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); - smartlist_free(str_build); - } - tor_free(str); + tor_free(processed_cells_string); + tor_free(queued_cells_string); + tor_free(time_in_queue_string); #undef SHARES return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL; }

1 0

[tor/master] Separate generation of a buffer-stats string from writing it to disk.
by nickm＠torproject.org 04 Aug '11

04 Aug '11

commit 07dc46e7fc7275fa1fcd365e34f318aa164fe7f1 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Aug 3 13:26:49 2011 +0200 Separate generation of a buffer-stats string from writing it to disk. The new rep_hist_format_buffer_stats() generates a buffer-stats string that rep_hist_buffer_stats_write() writes to disk. All the state changing (e.g., resetting the buffer-stats history and initializing the next measurement interval) takes place in rep_hist_buffer_stats_write(). That allows us to finally test the buffer-stats code better. --- src/or/rephist.c | 119 +++++++++++++++++++++++++++++++----------------------- src/or/rephist.h | 2 + 2 files changed, 71 insertions(+), 50 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index ce0152e..1b43e5c 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -2420,42 +2420,41 @@ _buffer_stats_compare_entries(const void **_a, const void **_b) void rep_hist_buffer_stats_term(void) { - start_of_buffer_stats_interval = 0; + rep_hist_reset_buffer_stats(0); +} + +/** Clear history of circuit statistics and set the measurement interval + * start to now. */ +void +rep_hist_reset_buffer_stats(time_t now) +{ if (!circuits_for_buffer_stats) circuits_for_buffer_stats = smartlist_create(); SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, stat, tor_free(stat)); smartlist_clear(circuits_for_buffer_stats); + start_of_buffer_stats_interval = now; } -/** Write buffer statistics to $DATADIR/stats/buffer-stats and return when - * we would next want to write exit stats. */ -time_t -rep_hist_buffer_stats_write(time_t now) +/** Return a newly allocated string containing the buffer statistics until + * now, or NULL if we're not collecting buffer stats. */ +char * +rep_hist_format_buffer_stats(time_t now) { - char *statsdir = NULL, *filename = NULL; - open_file_t *open_file = NULL; - FILE *out; #define SHARES 10 int processed_cells[SHARES], circs_in_share[SHARES], number_of_circuits, i; double queued_cells[SHARES], time_in_queue[SHARES]; char *buf = NULL; - circuit_t *circ; smartlist_t *processed_cells_strings, *queued_cells_strings, *time_in_queue_strings; char *processed_cells_string, *queued_cells_string, *time_in_queue_string; char t[ISO_TIME_LEN+1]; + char *result; if (!start_of_buffer_stats_interval) - return 0; /* Not initialized. */ - if (start_of_buffer_stats_interval + WRITE_STATS_INTERVAL > now) - goto done; /* Not ready to write */ - - /* add current circuits to stats */ - for (circ = _circuit_get_global_list(); circ; circ = circ->next) - rep_hist_buffer_stats_add_circ(circ, now); + return NULL; /* Not initialized. */ /* Calculate deciles if we saw at least one circuit. */ memset(processed_cells, 0, SHARES * sizeof(int)); @@ -2481,11 +2480,6 @@ rep_hist_buffer_stats_write(time_t now) SMARTLIST_FOREACH_END(stat); } - /* clear buffer stats history */ - SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, - stat, tor_free(stat)); - smartlist_clear(circuits_for_buffer_stats); - /* Write deciles to strings. */ processed_cells_strings = smartlist_create(); queued_cells_strings = smartlist_create(); @@ -2520,41 +2514,66 @@ rep_hist_buffer_stats_write(time_t now) smartlist_free(queued_cells_strings); smartlist_free(time_in_queue_strings); - /* Write everything to disk. */ + /* Put everything together. */ + format_iso_time(t, now); + tor_asprintf(&result, "cell-stats-end %s (%d s)\n" + "cell-processed-cells %s\n" + "cell-queued-cells %s\n" + "cell-time-in-queue %s\n" + "cell-circuits-per-decile %d\n", + t, (unsigned) (now - start_of_buffer_stats_interval), + processed_cells_string, + queued_cells_string, + time_in_queue_string, + (number_of_circuits + SHARES - 1) / SHARES); + tor_free(processed_cells_string); + tor_free(queued_cells_string); + tor_free(time_in_queue_string); + return result; +#undef SHARES +} + +/** If 24 hours have passed since the beginning of the current buffer + * stats period, write buffer stats to $DATADIR/stats/buffer-stats + * (possibly overwriting an existing file) and reset counters. Return + * when we would next want to write buffer stats or 0 if we never want to + * write. */ +time_t +rep_hist_buffer_stats_write(time_t now) +{ + circuit_t *circ; + char *statsdir = NULL, *filename = NULL, *str = NULL; + + if (!start_of_buffer_stats_interval) + return 0; /* Not initialized. */ + if (start_of_buffer_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write */ + + /* Add open circuits to the history. */ + for (circ = _circuit_get_global_list(); circ; circ = circ->next) { + rep_hist_buffer_stats_add_circ(circ, now); + } + + /* Generate history string. */ + str = rep_hist_format_buffer_stats(now); + + /* Reset both buffer history and counters of open circuits. */ + rep_hist_reset_buffer_stats(now); + + /* Try to write to disk. */ statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) + if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { + log_warn(LD_HIST, "Unable to create stats/ directory!"); goto done; + } filename = get_datadir_fname2("stats", "buffer-stats"); - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE | O_TEXT, - 0600, &open_file); - if (!out) - goto done; - format_iso_time(t, now); - if (fprintf(out, "cell-stats-end %s (%d s)\n", t, - (unsigned) (now - start_of_buffer_stats_interval)) < 0) - goto done; - if (fprintf(out, "cell-processed-cells %s\n", processed_cells_string) - < 0) - goto done; - if (fprintf(out, "cell-queued-cells %s\n", queued_cells_string) < 0) - goto done; - if (fprintf(out, "cell-time-in-queue %s\n", time_in_queue_string) < 0) - goto done; - if (fprintf(out, "cell-circuits-per-decile %d\n", - (number_of_circuits + SHARES - 1) / SHARES) < 0) - goto done; - finish_writing_to_file(open_file); - open_file = NULL; - start_of_buffer_stats_interval = now; + if (write_str_to_file(filename, str, 0) < 0) + log_warn(LD_HIST, "Unable to write buffer stats to disk!"); + done: - if (open_file) - abort_writing_to_file(open_file); + tor_free(str); tor_free(filename); tor_free(statsdir); - tor_free(processed_cells_string); - tor_free(queued_cells_string); - tor_free(time_in_queue_string); -#undef SHARES return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL; } diff --git a/src/or/rephist.h b/src/or/rephist.h index 5748748..13dee15 100644 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@ -77,6 +77,8 @@ void rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval); time_t rep_hist_buffer_stats_write(time_t now); void rep_hist_buffer_stats_term(void); +char *rep_hist_format_buffer_stats(time_t now); +void rep_hist_reset_buffer_stats(time_t now); void rep_hist_conn_stats_init(time_t now); void rep_hist_note_or_conn_bytes(uint64_t conn_id, size_t num_read,

1 0

[tor/master] Add unit tests for buffer-stats.
by nickm＠torproject.org 04 Aug '11

04 Aug '11

commit 3d3ed853e79d7aea2f41d34813a7fa5a3517dc92 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Aug 3 13:29:03 2011 +0200 Add unit tests for buffer-stats. Now that formatting the buffer-stats string is separate from writing it to disk, we can also decouple the logic to extract stats from circuits and finally write some unit tests for the history code. --- src/or/rephist.c | 35 ++++++++++++++++++++++++++------- src/or/rephist.h | 2 + src/test/test.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 8 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index 1b43e5c..25aece3 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -2364,23 +2364,41 @@ typedef struct circ_buffer_stats_t { /** List of circ_buffer_stats_t. */ static smartlist_t *circuits_for_buffer_stats = NULL; +/** Remember cell statistics mean_num_cells_in_queue, + * mean_time_cells_in_queue, and processed_cells of a + * circuit. */ +void +rep_hist_add_buffer_stats(double mean_num_cells_in_queue, + double mean_time_cells_in_queue, uint32_t processed_cells) +{ + circ_buffer_stats_t *stat; + if (!start_of_buffer_stats_interval) + return; /* Not initialized. */ + stat = tor_malloc_zero(sizeof(circ_buffer_stats_t)); + stat->mean_num_cells_in_queue = mean_num_cells_in_queue; + stat->mean_time_cells_in_queue = mean_time_cells_in_queue; + stat->processed_cells = processed_cells; + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + smartlist_add(circuits_for_buffer_stats, stat); +} + /** Remember cell statistics for circuit circ at time * end_of_interval and reset cell counters in case the circuit * remains open in the next measurement interval. */ void rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval) { - circ_buffer_stats_t *stat; time_t start_of_interval; int interval_length; or_circuit_t *orcirc; + double mean_num_cells_in_queue, mean_time_cells_in_queue; + uint32_t processed_cells; if (CIRCUIT_IS_ORIGIN(circ)) return; orcirc = TO_OR_CIRCUIT(circ); if (!orcirc->processed_cells) return; - if (!circuits_for_buffer_stats) - circuits_for_buffer_stats = smartlist_create(); start_of_interval = (circ->timestamp_created.tv_sec > start_of_buffer_stats_interval) ? circ->timestamp_created.tv_sec : @@ -2388,17 +2406,18 @@ rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval) interval_length = (int) (end_of_interval - start_of_interval); if (interval_length <= 0) return; - stat = tor_malloc_zero(sizeof(circ_buffer_stats_t)); - stat->processed_cells = orcirc->processed_cells; + processed_cells = orcirc->processed_cells; /* 1000.0 for s -> ms; 2.0 because of app-ward and exit-ward queues */ - stat->mean_num_cells_in_queue = (double) orcirc->total_cell_waiting_time / + mean_num_cells_in_queue = (double) orcirc->total_cell_waiting_time / (double) interval_length / 1000.0 / 2.0; - stat->mean_time_cells_in_queue = + mean_time_cells_in_queue = (double) orcirc->total_cell_waiting_time / (double) orcirc->processed_cells; - smartlist_add(circuits_for_buffer_stats, stat); orcirc->total_cell_waiting_time = 0; orcirc->processed_cells = 0; + rep_hist_add_buffer_stats(mean_num_cells_in_queue, + mean_time_cells_in_queue, + processed_cells); } /** Sorting helper: return -1, 1, or 0 based on comparison of two diff --git a/src/or/rephist.h b/src/or/rephist.h index 13dee15..e590659 100644 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@ -77,6 +77,8 @@ void rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval); time_t rep_hist_buffer_stats_write(time_t now); void rep_hist_buffer_stats_term(void); +void rep_hist_add_buffer_stats(double mean_num_cells_in_queue, + double mean_time_cells_in_queue, uint32_t processed_cells); char *rep_hist_format_buffer_stats(time_t now); void rep_hist_reset_buffer_stats(time_t now); diff --git a/src/test/test.c b/src/test/test.c index e7d4ccb..4ef87a0 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -1635,6 +1635,61 @@ test_stats(void) rep_hist_reset_conn_stats(now); s = rep_hist_format_conn_stats(now + 86400); test_streq("conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,0,0\n", s); + tor_free(s); + + /* Continue with testing buffer statistics; we shouldn't collect buffer + * stats without initializing them. */ + rep_hist_add_buffer_stats(2.0, 2.0, 20); + s = rep_hist_format_buffer_stats(now + 86400); + test_assert(!s); + + /* Initialize stats, add statistics for a single circuit, and generate + * the history string. */ + rep_hist_buffer_stats_init(now); + rep_hist_add_buffer_stats(2.0, 2.0, 20); + s = rep_hist_format_buffer_stats(now + 86400); + test_streq("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "cell-processed-cells 20,0,0,0,0,0,0,0,0,0\n" + "cell-queued-cells 2.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00," + "0.00,0.00\n" + "cell-time-in-queue 2,0,0,0,0,0,0,0,0,0\n" + "cell-circuits-per-decile 1\n", s); + tor_free(s); + + /* Add nineteen more circuit statistics to the one that's already in the + * history to see that the math works correctly. */ + for (i = 21; i < 30; i++) + rep_hist_add_buffer_stats(2.0, 2.0, i); + for (i = 20; i < 30; i++) + rep_hist_add_buffer_stats(3.5, 3.5, i); + s = rep_hist_format_buffer_stats(now + 86400); + test_streq("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "cell-processed-cells 29,28,27,26,25,24,23,22,21,20\n" + "cell-queued-cells 2.75,2.75,2.75,2.75,2.75,2.75,2.75,2.75," + "2.75,2.75\n" + "cell-time-in-queue 3,3,3,3,3,3,3,3,3,3\n" + "cell-circuits-per-decile 2\n", s); + tor_free(s); + + /* Stop collecting stats, add statistics for one circuit, and ensure we + * don't generate a history string. */ + rep_hist_buffer_stats_term(); + rep_hist_add_buffer_stats(2.0, 2.0, 20); + s = rep_hist_format_buffer_stats(now + 86400); + test_assert(!s); + + /* Re-start stats, add statistics for one circuit, reset stats, and make + * sure that the history has all zeros. */ + rep_hist_buffer_stats_init(now); + rep_hist_add_buffer_stats(2.0, 2.0, 20); + rep_hist_reset_buffer_stats(now); + s = rep_hist_format_buffer_stats(now + 86400); + test_streq("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "cell-processed-cells 0,0,0,0,0,0,0,0,0,0\n" + "cell-queued-cells 0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00," + "0.00,0.00\n" + "cell-time-in-queue 0,0,0,0,0,0,0,0,0,0\n" + "cell-circuits-per-decile 0\n", s); done: tor_free(s);

1 0