April 2014 - tor-commits - lists.torproject.org

[meek/master] Move a couple programs to be named after themselves.
by dcf＠torproject.org 11 Apr '14

11 Apr '14

commit 0f40b62cba0aa8581432b37f867d197ba2c32fba Author: David Fifield <david(a)bamsoftware.com> Date: Thu Apr 10 08:41:12 2014 -0700 Move a couple programs to be named after themselves. --- meek-client-torbrowser/main.go | 181 -------------------- meek-client-torbrowser/meek-client-torbrowser.go | 181 ++++++++++++++++++++ terminateprocess-buffer/main.go | 38 ---- terminateprocess-buffer/terminateprocess-buffer.go | 38 ++++ 4 files changed, 219 insertions(+), 219 deletions(-) diff --git a/meek-client-torbrowser/main.go b/meek-client-torbrowser/main.go deleted file mode 100644 index 1d55eff..0000000 --- a/meek-client-torbrowser/main.go +++ /dev/null @@ -1,181 +0,0 @@ -// The meek-client-torbrowser program starts a copy of Tor Browser running -// meek-http-helper in a special profile, and then starts meek-client set up to -// use the browser helper. -// -// Arguments to this program are passed unmodified to meek-client, with the -// addition of a --helper option pointing to the browser helper. -package main - -import ( - "bufio" - "io" - "io/ioutil" - "log" - "os" - "os/exec" - "os/signal" - "path/filepath" - "regexp" - "syscall" -) - -var helperAddrPattern *regexp.Regexp - -// Log a call to os.Process.Kill. -func logKill(p *os.Process) error { - log.Printf("killing PID %d", p.Pid) - err := p.Kill() - if err != nil { - log.Print(err) - } - return err -} - -// Log a call to os.Process.Signal. -func logSignal(p *os.Process, sig os.Signal) error { - log.Printf("sending signal %s to PID %d", sig, p.Pid) - err := p.Signal(sig) - if err != nil { - log.Print(err) - } - return err -} - -// Run firefox and return its exec.Cmd and stdout pipe. -func runFirefox() (cmd *exec.Cmd, stdout io.Reader, err error) { - var profilePath string - // Mac OS X needs an absolute profile path. - profilePath, err = filepath.Abs(firefoxProfilePath) - if err != nil { - return - } - cmd = exec.Command(firefoxPath, "-no-remote", "-profile", profilePath) - cmd.Stderr = os.Stderr - stdout, err = cmd.StdoutPipe() - if err != nil { - return - } - log.Printf("running firefox command %q", cmd.Args) - err = cmd.Start() - if err != nil { - return - } - log.Printf("firefox started with pid %d", cmd.Process.Pid) - return cmd, stdout, nil -} - -// Look for the magic meek-http-helper address string in the Reader, and return -// the address it contains. Start a goroutine to continue reading and discarding -// output of the Reader before returning. -func grepHelperAddr(r io.Reader) (string, error) { - var helperAddr string - scanner := bufio.NewScanner(r) - for scanner.Scan() { - line := scanner.Text() - if m := helperAddrPattern.FindStringSubmatch(line); m != nil { - helperAddr = m[1] - break - } - } - err := scanner.Err() - if err != nil { - return "", err - } - // Ran out of input before finding the pattern. - if helperAddr == "" { - return "", io.EOF - } - // Keep reading from the browser to avoid its output buffer filling. - go io.Copy(ioutil.Discard, r) - return helperAddr, nil -} - -// Run meek-client and return its exec.Cmd. -func runMeekClient(helperAddr string) (cmd *exec.Cmd, err error) { - args := os.Args[1:] - args = append(args, []string{"--helper", helperAddr}...) - cmd = exec.Command(meekClientPath, args...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - log.Printf("running meek-client command %q", cmd.Args) - err = cmd.Start() - if err != nil { - return - } - log.Printf("meek-client started with pid %d", cmd.Process.Pid) - return cmd, nil -} - -func main() { - var err error - - f, err := os.OpenFile("meek-client-torbrowser.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) - if err != nil { - log.Fatal(err) - } - defer f.Close() - log.SetOutput(f) - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - - // This magic string is emitted by meek-http-helper. - helperAddrPattern, err = regexp.Compile(`^meek-http-helper: listen (127\.0\.0\.1:\d+)$`) - if err != nil { - log.Fatal(err) - } - - // Start firefox. - firefoxCmd, stdout, err := runFirefox() - if err != nil { - log.Print(err) - return - } - defer logKill(firefoxCmd.Process) - - // Find out the helper's listening address. - helperAddr, err := grepHelperAddr(stdout) - if err != nil { - log.Print(err) - return - } - - // Start meek-client with the helper address. - meekClientCmd, err := runMeekClient(helperAddr) - if err != nil { - log.Print(err) - return - } - defer logKill(meekClientCmd.Process) - - if exitOnStdinEOF { - // On Windows, we don't get a SIGINT or SIGTERM, rather we are killed - // without a chance to clean up our subprocesses. When run inside - // terminateprocess-buffer, it is instead terminateprocess-buffer that - // is killed, and we can detect that event by that our stdin gets - // closed. - // https://trac.torproject.org/projects/tor/ticket/9330 - go func() { - io.Copy(ioutil.Discard, os.Stdin) - log.Printf("synthesizing SIGTERM because of stdin close") - sigChan <- syscall.SIGTERM - }() - } - - sig := <-sigChan - log.Printf("sig %s", sig) - err = logSignal(meekClientCmd.Process, sig) - if err != nil { - log.Print(err) - } - - // If SIGINT, wait for a second SIGINT. - if sig == syscall.SIGINT { - sig := <-sigChan - log.Printf("sig %s", sig) - err = logSignal(meekClientCmd.Process, sig) - if err != nil { - log.Print(err) - } - } -} diff --git a/meek-client-torbrowser/meek-client-torbrowser.go b/meek-client-torbrowser/meek-client-torbrowser.go new file mode 100644 index 0000000..1d55eff --- /dev/null +++ b/meek-client-torbrowser/meek-client-torbrowser.go @@ -0,0 +1,181 @@ +// The meek-client-torbrowser program starts a copy of Tor Browser running +// meek-http-helper in a special profile, and then starts meek-client set up to +// use the browser helper. +// +// Arguments to this program are passed unmodified to meek-client, with the +// addition of a --helper option pointing to the browser helper. +package main + +import ( + "bufio" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "os/signal" + "path/filepath" + "regexp" + "syscall" +) + +var helperAddrPattern *regexp.Regexp + +// Log a call to os.Process.Kill. +func logKill(p *os.Process) error { + log.Printf("killing PID %d", p.Pid) + err := p.Kill() + if err != nil { + log.Print(err) + } + return err +} + +// Log a call to os.Process.Signal. +func logSignal(p *os.Process, sig os.Signal) error { + log.Printf("sending signal %s to PID %d", sig, p.Pid) + err := p.Signal(sig) + if err != nil { + log.Print(err) + } + return err +} + +// Run firefox and return its exec.Cmd and stdout pipe. +func runFirefox() (cmd *exec.Cmd, stdout io.Reader, err error) { + var profilePath string + // Mac OS X needs an absolute profile path. + profilePath, err = filepath.Abs(firefoxProfilePath) + if err != nil { + return + } + cmd = exec.Command(firefoxPath, "-no-remote", "-profile", profilePath) + cmd.Stderr = os.Stderr + stdout, err = cmd.StdoutPipe() + if err != nil { + return + } + log.Printf("running firefox command %q", cmd.Args) + err = cmd.Start() + if err != nil { + return + } + log.Printf("firefox started with pid %d", cmd.Process.Pid) + return cmd, stdout, nil +} + +// Look for the magic meek-http-helper address string in the Reader, and return +// the address it contains. Start a goroutine to continue reading and discarding +// output of the Reader before returning. +func grepHelperAddr(r io.Reader) (string, error) { + var helperAddr string + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + if m := helperAddrPattern.FindStringSubmatch(line); m != nil { + helperAddr = m[1] + break + } + } + err := scanner.Err() + if err != nil { + return "", err + } + // Ran out of input before finding the pattern. + if helperAddr == "" { + return "", io.EOF + } + // Keep reading from the browser to avoid its output buffer filling. + go io.Copy(ioutil.Discard, r) + return helperAddr, nil +} + +// Run meek-client and return its exec.Cmd. +func runMeekClient(helperAddr string) (cmd *exec.Cmd, err error) { + args := os.Args[1:] + args = append(args, []string{"--helper", helperAddr}...) + cmd = exec.Command(meekClientPath, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + log.Printf("running meek-client command %q", cmd.Args) + err = cmd.Start() + if err != nil { + return + } + log.Printf("meek-client started with pid %d", cmd.Process.Pid) + return cmd, nil +} + +func main() { + var err error + + f, err := os.OpenFile("meek-client-torbrowser.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + log.Fatal(err) + } + defer f.Close() + log.SetOutput(f) + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + + // This magic string is emitted by meek-http-helper. + helperAddrPattern, err = regexp.Compile(`^meek-http-helper: listen (127\.0\.0\.1:\d+)$`) + if err != nil { + log.Fatal(err) + } + + // Start firefox. + firefoxCmd, stdout, err := runFirefox() + if err != nil { + log.Print(err) + return + } + defer logKill(firefoxCmd.Process) + + // Find out the helper's listening address. + helperAddr, err := grepHelperAddr(stdout) + if err != nil { + log.Print(err) + return + } + + // Start meek-client with the helper address. + meekClientCmd, err := runMeekClient(helperAddr) + if err != nil { + log.Print(err) + return + } + defer logKill(meekClientCmd.Process) + + if exitOnStdinEOF { + // On Windows, we don't get a SIGINT or SIGTERM, rather we are killed + // without a chance to clean up our subprocesses. When run inside + // terminateprocess-buffer, it is instead terminateprocess-buffer that + // is killed, and we can detect that event by that our stdin gets + // closed. + // https://trac.torproject.org/projects/tor/ticket/9330 + go func() { + io.Copy(ioutil.Discard, os.Stdin) + log.Printf("synthesizing SIGTERM because of stdin close") + sigChan <- syscall.SIGTERM + }() + } + + sig := <-sigChan + log.Printf("sig %s", sig) + err = logSignal(meekClientCmd.Process, sig) + if err != nil { + log.Print(err) + } + + // If SIGINT, wait for a second SIGINT. + if sig == syscall.SIGINT { + sig := <-sigChan + log.Printf("sig %s", sig) + err = logSignal(meekClientCmd.Process, sig) + if err != nil { + log.Print(err) + } + } +} diff --git a/terminateprocess-buffer/main.go b/terminateprocess-buffer/main.go deleted file mode 100644 index 16a7297..0000000 --- a/terminateprocess-buffer/main.go +++ /dev/null @@ -1,38 +0,0 @@ -// This program is designed to sit between tor and a transport plugin on -// Windows. On Windows, transport plugins are killed with a TerminateProcess, -// which doesn't give them a chance to clean up before exiting. -// https://trac.torproject.org/projects/tor/ticket/9330 -// The idea of this program is that the transport plugin can read from its -// standard input, which will be closed when this program is terminated. The -// transport plugin can then treat the stdin-closed event like a SIGTERM. -package main - -import ( - "io" - "log" - "os" - "os/exec" -) - -func main() { - args := os.Args[1:] - if len(args) < 1 { - log.Fatalf("%s needs a command to run", os.Args[0]) - } - cmd := exec.Command(args[0], args[1:]...) - stdin, err := cmd.StdinPipe() - if err != nil { - log.Fatal(err) - } - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err = cmd.Start() - if err != nil { - log.Fatal(err) - } - io.Copy(stdin, os.Stdin) - err = cmd.Wait() - if err != nil { - log.Fatal(err) - } -} diff --git a/terminateprocess-buffer/terminateprocess-buffer.go b/terminateprocess-buffer/terminateprocess-buffer.go new file mode 100644 index 0000000..16a7297 --- /dev/null +++ b/terminateprocess-buffer/terminateprocess-buffer.go @@ -0,0 +1,38 @@ +// This program is designed to sit between tor and a transport plugin on +// Windows. On Windows, transport plugins are killed with a TerminateProcess, +// which doesn't give them a chance to clean up before exiting. +// https://trac.torproject.org/projects/tor/ticket/9330 +// The idea of this program is that the transport plugin can read from its +// standard input, which will be closed when this program is terminated. The +// transport plugin can then treat the stdin-closed event like a SIGTERM. +package main + +import ( + "io" + "log" + "os" + "os/exec" +) + +func main() { + args := os.Args[1:] + if len(args) < 1 { + log.Fatalf("%s needs a command to run", os.Args[0]) + } + cmd := exec.Command(args[0], args[1:]...) + stdin, err := cmd.StdinPipe() + if err != nil { + log.Fatal(err) + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Start() + if err != nil { + log.Fatal(err) + } + io.Copy(stdin, os.Stdin) + err = cmd.Wait() + if err != nil { + log.Fatal(err) + } +}

1 0

[meek/master] Fix meek-client-torbrowser makefile target.
by dcf＠torproject.org 11 Apr '14

11 Apr '14

commit 19bbe6ec48f3732d749287efed545975fda138bb Author: David Fifield <david(a)bamsoftware.com> Date: Thu Apr 10 08:59:12 2014 -0700 Fix meek-client-torbrowser makefile target. --- meek-client-torbrowser/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meek-client-torbrowser/Makefile b/meek-client-torbrowser/Makefile index 7ce013b..9d0a70f 100644 --- a/meek-client-torbrowser/Makefile +++ b/meek-client-torbrowser/Makefile @@ -4,9 +4,9 @@ BINDIR = $(PREFIX)/bin GOBUILDFLAGS = -all: meek-client +all: meek-client-torbrowser -meek-client: *.go +meek-client-torbrowser: *.go go build $(GOBUILDFLAGS) clean:

1 0

[translation/tails-iuk] Update translations for tails-iuk
by translation＠torproject.org 11 Apr '14

11 Apr '14

commit b99f5ca97ec222e01601ed125d7c25618a35683c Author: Translation commit bot <translation(a)torproject.org> Date: Fri Apr 11 09:45:48 2014 +0000 Update translations for tails-iuk --- hu.po | 57 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/hu.po b/hu.po index fe45c05..064312a 100644 --- a/hu.po +++ b/hu.po @@ -3,13 +3,14 @@ # This file is distributed under the same license as the PACKAGE package. # # Translators: +# lajos <mrlajos(a)gmail.com>, 2014 msgid "" msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: Tails developers <tails(a)boum.org>\n" "POT-Creation-Date: 2014-03-05 15:11+0100\n" -"PO-Revision-Date: 2014-03-06 08:17+0000\n" -"Last-Translator: runasand <runa.sandvik(a)gmail.com>\n" +"PO-Revision-Date: 2014-04-11 09:40+0000\n" +"Last-Translator: lajos <mrlajos(a)gmail.com>\n" "Language-Team: Hungarian (http://www.transifex.com/projects/p/torproject/language/hu/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -24,7 +25,7 @@ msgstr "" #: ../lib/Tails/IUK/Frontend.pm:216 msgid "Error while checking for upgrades" -msgstr "" +msgstr "Hiba frissítések ellenőrzése közben" #: ../lib/Tails/IUK/Frontend.pm:219 msgid "" @@ -37,15 +38,15 @@ msgstr "" #: ../lib/Tails/IUK/Frontend.pm:234 msgid "no automatic upgrade is available from our website for this version" -msgstr "" +msgstr "Nincs elérhető automatikus frissítés a weboldalunkon ehhez a verzióhoz" #: ../lib/Tails/IUK/Frontend.pm:240 msgid "your device was not created using Tails Installer" -msgstr "" +msgstr "Az eszközöd nem a Tails telepítővel volt létrehozva" #: ../lib/Tails/IUK/Frontend.pm:245 msgid "Tails was started from a DVD or a read-only device" -msgstr "" +msgstr "A Tails DVD-ről vagy egy csak olvasható eszközről lett indítva" #: ../lib/Tails/IUK/Frontend.pm:250 msgid "there is not enough free space on the Tails system partition" @@ -58,15 +59,15 @@ msgstr "" #: ../lib/Tails/IUK/Frontend.pm:261 #, perl-brace-format msgid "No explanation available for reason '%{reason}s'." -msgstr "" +msgstr "Nincs elérhető magyarázat erre '%{reason}s'." #: ../lib/Tails/IUK/Frontend.pm:281 msgid "The system is up-to-date" -msgstr "" +msgstr "A rendszer naprakész" #: ../lib/Tails/IUK/Frontend.pm:286 msgid "This version of Tails is outdated, and may have security issues." -msgstr "" +msgstr "A Tails ezen verziója nem naprakész és biztonsági hibákat tartalmazhat." #: ../lib/Tails/IUK/Frontend.pm:318 #, perl-brace-format @@ -90,7 +91,7 @@ msgstr "" #: ../lib/Tails/IUK/Frontend.pm:360 msgid "Error while detecting available upgrades" -msgstr "" +msgstr "Hiba történt elérhető frissítések keresésekor" #: ../lib/Tails/IUK/Frontend.pm:370 #, perl-brace-format @@ -110,15 +111,15 @@ msgstr "" #: ../lib/Tails/IUK/Frontend.pm:385 msgid "Upgrade available" -msgstr "" +msgstr "Frissítés elérhető" #: ../lib/Tails/IUK/Frontend.pm:386 msgid "Upgrade now" -msgstr "" +msgstr "Frissítés most" #: ../lib/Tails/IUK/Frontend.pm:387 msgid "Upgrade later" -msgstr "" +msgstr "Frissítés később" #: ../lib/Tails/IUK/Frontend.pm:395 #, perl-brace-format @@ -134,16 +135,16 @@ msgstr "" #: ../lib/Tails/IUK/Frontend.pm:411 msgid "New version available" -msgstr "" +msgstr "Új verzió elérhető" #: ../lib/Tails/IUK/Frontend.pm:468 msgid "Downloading upgrade" -msgstr "" +msgstr "Frissítés letöltése" #: ../lib/Tails/IUK/Frontend.pm:471 #, perl-brace-format msgid "Downloading the upgrade to %{name}s %{version}s..." -msgstr "" +msgstr "Frissítés letöltése %{name}s %{version}s..." #: ../lib/Tails/IUK/Frontend.pm:512 msgid "" @@ -155,7 +156,7 @@ msgstr "" #: ../lib/Tails/IUK/Frontend.pm:528 ../lib/Tails/IUK/Frontend.pm:547 msgid "Error while downloading the upgrade" -msgstr "" +msgstr "Hiba frissítés letöltésekor" #: ../lib/Tails/IUK/Frontend.pm:540 #, perl-brace-format @@ -166,11 +167,11 @@ msgstr "" #: ../lib/Tails/IUK/Frontend.pm:559 msgid "Error while creating temporary downloading directory" -msgstr "" +msgstr "Hiba történt az ideiglenes letöltési könyvtár létrehozásakor" #: ../lib/Tails/IUK/Frontend.pm:562 msgid "Failed to create temporary download directory" -msgstr "" +msgstr "Nem lehet létrehozni az ideiglenes letöltési könyvtárat" #: ../lib/Tails/IUK/Frontend.pm:574 msgid "" @@ -184,35 +185,35 @@ msgstr "" #: ../lib/Tails/IUK/Frontend.pm:579 msgid "Restart Tails" -msgstr "" +msgstr "Tails újraindítása" #: ../lib/Tails/IUK/Frontend.pm:580 msgid "Restart now" -msgstr "" +msgstr "Újraindítás most" #: ../lib/Tails/IUK/Frontend.pm:581 msgid "Restart later" -msgstr "" +msgstr "Újraindítás később" #: ../lib/Tails/IUK/Frontend.pm:592 msgid "Error while restarting the system" -msgstr "" +msgstr "Hiba a rendszer újraindításakor" #: ../lib/Tails/IUK/Frontend.pm:595 msgid "Failed to restart the system" -msgstr "" +msgstr "Nem sikerült a rendszer újraindítása" #: ../lib/Tails/IUK/Frontend.pm:610 msgid "Error while shutting down the network" -msgstr "" +msgstr "Hiba a hálózat leállításakor" #: ../lib/Tails/IUK/Frontend.pm:613 msgid "Failed to shutdown network" -msgstr "" +msgstr "Nem sikerült a hálózat leállítása" #: ../lib/Tails/IUK/Frontend.pm:623 msgid "Upgrading the system" -msgstr "" +msgstr "Rendszer frissítés" #: ../lib/Tails/IUK/Frontend.pm:625 msgid "" @@ -231,4 +232,4 @@ msgstr "" #: ../lib/Tails/IUK/Frontend.pm:672 msgid "Error while installing the upgrade" -msgstr "" +msgstr "Hiba frissítés telepítésekor"

1 0

r26703: {website} Update links to TorStatus websites. (website/trunk/projects/en)
by Karsten Loesing 11 Apr '14

11 Apr '14

Author: kloesing Date: 2014-04-11 08:13:12 +0000 (Fri, 11 Apr 2014) New Revision: 26703 Modified: website/trunk/projects/en/onionoo.wml Log: Update links to TorStatus websites. Modified: website/trunk/projects/en/onionoo.wml =================================================================== --- website/trunk/projects/en/onionoo.wml 2014-04-11 00:52:48 UTC (rev 26702) +++ website/trunk/projects/en/onionoo.wml 2014-04-11 08:13:12 UTC (rev 26703) @@ -103,8 +103,9 @@ <a href="https://svn.torproject.org/svn/torstatus/trunk/">unmaintained</a> website that displays Tor relay information similar to <a href="http://atlas.torproject.org/">Atlas</a>. There are still a - few <a href="http://torstatus.all.de/">TorStatus websites</a> - running.</p> + few <a href="http://torstatus.blutmagie.de/">TorStatus</a> + <a href="https://torstatus.rueckgr.at/">websites</a> + <a href="http://tns.hermetix.org/">running</a>.</p> <p>There's another project from summer 2011 called TorStatus which is a <a href="https://gitweb.torproject.org/torstatus.git">rewrite</a> of

1 0

[onionoo/master] Split bandwidth data writer into two classes.
by karsten＠torproject.org 11 Apr '14

11 Apr '14

commit 4afcccf3de9d923332d6b043c9dd43bb2ee82b22 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Apr 10 18:35:12 2014 +0200 Split bandwidth data writer into two classes. --- .../torproject/onionoo/BandwidthDataWriter.java | 395 -------------------- .../onionoo/BandwidthDocumentWriter.java | 199 ++++++++++ src/org/torproject/onionoo/BandwidthStatus.java | 74 +++- .../torproject/onionoo/BandwidthStatusUpdater.java | 152 ++++++++ src/org/torproject/onionoo/DocumentStore.java | 6 +- src/org/torproject/onionoo/Main.java | 11 +- 6 files changed, 435 insertions(+), 402 deletions(-) diff --git a/src/org/torproject/onionoo/BandwidthDataWriter.java b/src/org/torproject/onionoo/BandwidthDataWriter.java deleted file mode 100644 index 227df2b..0000000 --- a/src/org/torproject/onionoo/BandwidthDataWriter.java +++ /dev/null @@ -1,395 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Scanner; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TimeZone; -import java.util.TreeMap; - -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.ExtraInfoDescriptor; - -/* Write bandwidth data files to disk and delete bandwidth files of relays - * or bridges that fell out of the summary list. - * - * Bandwidth history data is available in different resolutions, depending - * on the considered time interval. Data for the past 72 hours is - * available for 15 minute detail, data for the past week in 1 hour - * detail, data for the past month in 4 hour detail, data for the past 3 - * months in 12 hour detail, data for the past year in 2 day detail, and - * earlier data in 10 day detail. These detail levels have been chosen to - * provide between 92 and 192 data points for graphing the bandwidth of - * the past day, past week, past month, past three months, past year, and - * past five years. - * - * Only update bandwidth data files for which new bandwidth histories are - * available. There's no point in updating bandwidth documents when we - * don't have newer bandwidth data to add. This means that, e.g., the - * last 3 days in the bandwidth document may not be equivalent to the last - * 3 days as of publishing the document, but that's something clients can - * work around. */ -public class BandwidthDataWriter implements DescriptorListener, - StatusUpdater, FingerprintListener, DocumentWriter { - - private DescriptorSource descriptorSource; - - private DocumentStore documentStore; - - private long now; - - private SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - - public BandwidthDataWriter(DescriptorSource descriptorSource, - DocumentStore documentStore, Time time) { - this.descriptorSource = descriptorSource; - this.documentStore = documentStore; - this.now = time.currentTimeMillis(); - this.dateTimeFormat.setLenient(false); - this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - this.registerDescriptorListeners(); - this.registerFingerprintListeners(); - } - - private void registerDescriptorListeners() { - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.RELAY_EXTRA_INFOS); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.BRIDGE_EXTRA_INFOS); - } - - private void registerFingerprintListeners() { - /* TODO Not used yet. - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_EXTRA_INFOS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_EXTRA_INFOS);*/ - } - - public void processDescriptor(Descriptor descriptor, boolean relay) { - if (descriptor instanceof ExtraInfoDescriptor) { - this.parseDescriptor((ExtraInfoDescriptor) descriptor); - } - } - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - /* TODO Not used yet. */ - } - - public void updateStatuses() { - /* Status files are already updated while processing descriptors. */ - } - - public void writeDocuments() { - /* Document files are already updated while processing descriptors. */ - } - - private void parseDescriptor(ExtraInfoDescriptor descriptor) { - String fingerprint = descriptor.getFingerprint(); - boolean updateHistory = false; - SortedMap<Long, long[]> writeHistory = new TreeMap<Long, long[]>(), - readHistory = new TreeMap<Long, long[]>(); - if (descriptor.getWriteHistory() != null) { - parseHistoryLine(descriptor.getWriteHistory().getLine(), - writeHistory); - updateHistory = true; - } - if (descriptor.getReadHistory() != null) { - parseHistoryLine(descriptor.getReadHistory().getLine(), - readHistory); - updateHistory = true; - } - if (updateHistory) { - this.readHistoryFromDisk(fingerprint, writeHistory, readHistory); - this.compressHistory(writeHistory); - this.compressHistory(readHistory); - this.writeHistoryToDisk(fingerprint, writeHistory, readHistory); - this.writeBandwidthDataFileToDisk(fingerprint, writeHistory, - readHistory); - } - } - - private void parseHistoryLine(String line, - SortedMap<Long, long[]> history) { - String[] parts = line.split(" "); - if (parts.length < 6) { - return; - } - try { - long endMillis = this.dateTimeFormat.parse(parts[1] + " " - + parts[2]).getTime(); - long intervalMillis = Long.parseLong(parts[3].substring(1)) * 1000L; - String[] values = parts[5].split(","); - for (int i = values.length - 1; i >= 0; i--) { - long bandwidthValue = Long.parseLong(values[i]); - long startMillis = endMillis - intervalMillis; - history.put(startMillis, new long[] { startMillis, endMillis, - bandwidthValue }); - endMillis -= intervalMillis; - } - } catch (ParseException e) { - System.err.println("Could not parse timestamp in line '" + line - + "'. Skipping."); - } - } - - private void readHistoryFromDisk(String fingerprint, - SortedMap<Long, long[]> writeHistory, - SortedMap<Long, long[]> readHistory) { - BandwidthStatus bandwidthStatus = this.documentStore.retrieve( - BandwidthStatus.class, false, fingerprint); - if (bandwidthStatus == null) { - return; - } - String historyString = bandwidthStatus.documentString; - try { - Scanner s = new Scanner(historyString); - while (s.hasNextLine()) { - String line = s.nextLine(); - String[] parts = line.split(" "); - if (parts.length != 6) { - System.err.println("Illegal line '" + line + "' in bandwidth " - + "history for fingerprint '" + fingerprint + "'. " - + "Skipping this line."); - continue; - } - SortedMap<Long, long[]> history = parts[0].equals("r") - ? readHistory : writeHistory; - long startMillis = this.dateTimeFormat.parse(parts[1] + " " - + parts[2]).getTime(); - long endMillis = this.dateTimeFormat.parse(parts[3] + " " - + parts[4]).getTime(); - long bandwidth = Long.parseLong(parts[5]); - long previousEndMillis = history.headMap(startMillis).isEmpty() - ? startMillis - : history.get(history.headMap(startMillis).lastKey())[1]; - long nextStartMillis = history.tailMap(startMillis).isEmpty() - ? endMillis : history.tailMap(startMillis).firstKey(); - if (previousEndMillis <= startMillis && - nextStartMillis >= endMillis) { - history.put(startMillis, new long[] { startMillis, endMillis, - bandwidth }); - } - } - s.close(); - } catch (ParseException e) { - System.err.println("Could not parse timestamp while reading " - + "bandwidth history for fingerprint '" + fingerprint + "'. " - + "Skipping."); - e.printStackTrace(); - } - } - - private void compressHistory( - SortedMap<Long, long[]> history) { - SortedMap<Long, long[]> uncompressedHistory = - new TreeMap<Long, long[]>(history); - history.clear(); - long lastStartMillis = 0L, lastEndMillis = 0L, lastBandwidth = 0L; - SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String lastMonthString = "1970-01"; - for (long[] v : uncompressedHistory.values()) { - long startMillis = v[0], endMillis = v[1], bandwidth = v[2]; - long intervalLengthMillis; - if (this.now - endMillis <= 72L * 60L * 60L * 1000L) { - intervalLengthMillis = 15L * 60L * 1000L; - } else if (this.now - endMillis <= 7L * 24L * 60L * 60L * 1000L) { - intervalLengthMillis = 60L * 60L * 1000L; - } else if (this.now - endMillis <= 31L * 24L * 60L * 60L * 1000L) { - intervalLengthMillis = 4L * 60L * 60L * 1000L; - } else if (this.now - endMillis <= 92L * 24L * 60L * 60L * 1000L) { - intervalLengthMillis = 12L * 60L * 60L * 1000L; - } else if (this.now - endMillis <= 366L * 24L * 60L * 60L * 1000L) { - intervalLengthMillis = 2L * 24L * 60L * 60L * 1000L; - } else { - intervalLengthMillis = 10L * 24L * 60L * 60L * 1000L; - } - String monthString = dateTimeFormat.format(startMillis); - if (lastEndMillis == startMillis && - ((lastEndMillis - 1L) / intervalLengthMillis) == - ((endMillis - 1L) / intervalLengthMillis) && - lastMonthString.equals(monthString)) { - lastEndMillis = endMillis; - lastBandwidth += bandwidth; - } else { - if (lastStartMillis > 0L) { - history.put(lastStartMillis, new long[] { lastStartMillis, - lastEndMillis, lastBandwidth }); - } - lastStartMillis = startMillis; - lastEndMillis = endMillis; - lastBandwidth = bandwidth; - } - lastMonthString = monthString; - } - if (lastStartMillis > 0L) { - history.put(lastStartMillis, new long[] { lastStartMillis, - lastEndMillis, lastBandwidth }); - } - } - - private void writeHistoryToDisk(String fingerprint, - SortedMap<Long, long[]> writeHistory, - SortedMap<Long, long[]> readHistory) { - StringBuilder sb = new StringBuilder(); - for (long[] v : writeHistory.values()) { - sb.append("w " + this.dateTimeFormat.format(v[0]) + " " - + this.dateTimeFormat.format(v[1]) + " " - + String.valueOf(v[2]) + "\n"); - } - for (long[] v : readHistory.values()) { - sb.append("r " + this.dateTimeFormat.format(v[0]) + " " - + this.dateTimeFormat.format(v[1]) + " " - + String.valueOf(v[2]) + "\n"); - } - BandwidthStatus bandwidthStatus = new BandwidthStatus(); - bandwidthStatus.documentString = sb.toString(); - this.documentStore.store(bandwidthStatus, fingerprint); - } - - private void writeBandwidthDataFileToDisk(String fingerprint, - SortedMap<Long, long[]> writeHistory, - SortedMap<Long, long[]> readHistory) { - String writeHistoryString = formatHistoryString(writeHistory); - String readHistoryString = formatHistoryString(readHistory); - StringBuilder sb = new StringBuilder(); - sb.append("{\"fingerprint\":\"" + fingerprint + "\",\n" - + "\"write_history\":{\n" + writeHistoryString + "},\n" - + "\"read_history\":{\n" + readHistoryString + "}}\n"); - BandwidthDocument bandwidthDocument = new BandwidthDocument(); - bandwidthDocument.documentString = sb.toString(); - this.documentStore.store(bandwidthDocument, fingerprint); - } - - private String[] graphNames = new String[] { - "3_days", - "1_week", - "1_month", - "3_months", - "1_year", - "5_years" }; - - private long[] graphIntervals = new long[] { - 72L * 60L * 60L * 1000L, - 7L * 24L * 60L * 60L * 1000L, - 31L * 24L * 60L * 60L * 1000L, - 92L * 24L * 60L * 60L * 1000L, - 366L * 24L * 60L * 60L * 1000L, - 5L * 366L * 24L * 60L * 60L * 1000L }; - - private long[] dataPointIntervals = new long[] { - 15L * 60L * 1000L, - 60L * 60L * 1000L, - 4L * 60L * 60L * 1000L, - 12L * 60L * 60L * 1000L, - 2L * 24L * 60L * 60L * 1000L, - 10L * 24L * 60L * 60L * 1000L }; - - private String formatHistoryString(SortedMap<Long, long[]> history) { - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < this.graphIntervals.length; i++) { - String graphName = this.graphNames[i]; - long graphInterval = this.graphIntervals[i]; - long dataPointInterval = this.dataPointIntervals[i]; - List<Long> dataPoints = new ArrayList<Long>(); - long intervalStartMillis = ((this.now - graphInterval) - / dataPointInterval) * dataPointInterval; - long totalMillis = 0L, totalBandwidth = 0L; - for (long[] v : history.values()) { - long startMillis = v[0], endMillis = v[1], bandwidth = v[2]; - if (endMillis < intervalStartMillis) { - continue; - } - while ((intervalStartMillis / dataPointInterval) != - (endMillis / dataPointInterval)) { - dataPoints.add(totalMillis * 5L < dataPointInterval - ? -1L : (totalBandwidth * 1000L) / totalMillis); - totalBandwidth = 0L; - totalMillis = 0L; - intervalStartMillis += dataPointInterval; - } - totalBandwidth += bandwidth; - totalMillis += (endMillis - startMillis); - } - dataPoints.add(totalMillis * 5L < dataPointInterval - ? -1L : (totalBandwidth * 1000L) / totalMillis); - long maxValue = 1L; - int firstNonNullIndex = -1, lastNonNullIndex = -1; - for (int j = 0; j < dataPoints.size(); j++) { - long dataPoint = dataPoints.get(j); - if (dataPoint >= 0L) { - if (firstNonNullIndex < 0) { - firstNonNullIndex = j; - } - lastNonNullIndex = j; - if (dataPoint > maxValue) { - maxValue = dataPoint; - } - } - } - if (firstNonNullIndex < 0) { - continue; - } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) * dataPointInterval - + dataPointInterval / 2L; - if (i > 0 && - firstDataPointMillis >= this.now - graphIntervals[i - 1]) { - /* Skip bandwidth history object, because it doesn't contain - * anything new that wasn't already contained in the last - * bandwidth history object(s). */ - continue; - } - long lastDataPointMillis = firstDataPointMillis - + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; - double factor = ((double) maxValue) / 999.0; - int count = lastNonNullIndex - firstNonNullIndex + 1; - StringBuilder sb2 = new StringBuilder(); - sb2.append("\"" + graphName + "\":{" - + "\"first\":\"" - + this.dateTimeFormat.format(firstDataPointMillis) + "\"," - + "\"last\":\"" - + this.dateTimeFormat.format(lastDataPointMillis) + "\"," - +"\"interval\":" + String.valueOf(dataPointInterval / 1000L) - + ",\"factor\":" + String.format(Locale.US, "%.3f", factor) - + ",\"count\":" + String.valueOf(count) + ",\"values\":["); - int written = 0, previousNonNullIndex = -2; - boolean foundTwoAdjacentDataPoints = false; - for (int j = firstNonNullIndex; j <= lastNonNullIndex; j++) { - long dataPoint = dataPoints.get(j); - if (dataPoint >= 0L) { - if (j - previousNonNullIndex == 1) { - foundTwoAdjacentDataPoints = true; - } - previousNonNullIndex = j; - } - sb2.append((written++ > 0 ? "," : "") + (dataPoint < 0L ? "null" : - String.valueOf((dataPoint * 999L) / maxValue))); - } - sb2.append("]},\n"); - if (foundTwoAdjacentDataPoints) { - sb.append(sb2.toString()); - } - } - String result = sb.toString(); - if (result.length() >= 2) { - result = result.substring(0, result.length() - 2) + "\n"; - } - return result; - } - - public String getStatsString() { - /* TODO Add statistics string. */ - return null; - } -} - diff --git a/src/org/torproject/onionoo/BandwidthDocumentWriter.java b/src/org/torproject/onionoo/BandwidthDocumentWriter.java new file mode 100644 index 0000000..754c8f3 --- /dev/null +++ b/src/org/torproject/onionoo/BandwidthDocumentWriter.java @@ -0,0 +1,199 @@ +/* Copyright 2011--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TimeZone; + +public class BandwidthDocumentWriter implements FingerprintListener, + DocumentWriter{ + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + private SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + + public BandwidthDocumentWriter(DescriptorSource descriptorSource, + DocumentStore documentStore, Time time) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + this.now = time.currentTimeMillis(); + this.dateTimeFormat.setLenient(false); + this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + this.registerFingerprintListeners(); + } + + private void registerFingerprintListeners() { + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_EXTRA_INFOS); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_EXTRA_INFOS); + } + + private Set<String> updateBandwidthDocuments = new HashSet<String>(); + + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + this.updateBandwidthDocuments.addAll(fingerprints); + } + + public void writeDocuments() { + for (String fingerprint : this.updateBandwidthDocuments) { + BandwidthStatus bandwidthStatus = this.documentStore.retrieve( + BandwidthStatus.class, true, fingerprint); + if (bandwidthStatus == null) { + continue; + } + this.writeBandwidthDataFileToDisk(fingerprint, + bandwidthStatus.writeHistory, bandwidthStatus.readHistory); + } + Logger.printStatusTime("Wrote bandwidth document files"); + } + + private void writeBandwidthDataFileToDisk(String fingerprint, + SortedMap<Long, long[]> writeHistory, + SortedMap<Long, long[]> readHistory) { + String writeHistoryString = formatHistoryString(writeHistory); + String readHistoryString = formatHistoryString(readHistory); + StringBuilder sb = new StringBuilder(); + sb.append("{\"fingerprint\":\"" + fingerprint + "\",\n" + + "\"write_history\":{\n" + writeHistoryString + "},\n" + + "\"read_history\":{\n" + readHistoryString + "}}\n"); + BandwidthDocument bandwidthDocument = new BandwidthDocument(); + bandwidthDocument.documentString = sb.toString(); + this.documentStore.store(bandwidthDocument, fingerprint); + } + + private String[] graphNames = new String[] { + "3_days", + "1_week", + "1_month", + "3_months", + "1_year", + "5_years" }; + + private long[] graphIntervals = new long[] { + 72L * 60L * 60L * 1000L, + 7L * 24L * 60L * 60L * 1000L, + 31L * 24L * 60L * 60L * 1000L, + 92L * 24L * 60L * 60L * 1000L, + 366L * 24L * 60L * 60L * 1000L, + 5L * 366L * 24L * 60L * 60L * 1000L }; + + private long[] dataPointIntervals = new long[] { + 15L * 60L * 1000L, + 60L * 60L * 1000L, + 4L * 60L * 60L * 1000L, + 12L * 60L * 60L * 1000L, + 2L * 24L * 60L * 60L * 1000L, + 10L * 24L * 60L * 60L * 1000L }; + + private String formatHistoryString(SortedMap<Long, long[]> history) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < this.graphIntervals.length; i++) { + String graphName = this.graphNames[i]; + long graphInterval = this.graphIntervals[i]; + long dataPointInterval = this.dataPointIntervals[i]; + List<Long> dataPoints = new ArrayList<Long>(); + long intervalStartMillis = ((this.now - graphInterval) + / dataPointInterval) * dataPointInterval; + long totalMillis = 0L, totalBandwidth = 0L; + for (long[] v : history.values()) { + long startMillis = v[0], endMillis = v[1], bandwidth = v[2]; + if (endMillis < intervalStartMillis) { + continue; + } + while ((intervalStartMillis / dataPointInterval) != + (endMillis / dataPointInterval)) { + dataPoints.add(totalMillis * 5L < dataPointInterval + ? -1L : (totalBandwidth * 1000L) / totalMillis); + totalBandwidth = 0L; + totalMillis = 0L; + intervalStartMillis += dataPointInterval; + } + totalBandwidth += bandwidth; + totalMillis += (endMillis - startMillis); + } + dataPoints.add(totalMillis * 5L < dataPointInterval + ? -1L : (totalBandwidth * 1000L) / totalMillis); + long maxValue = 1L; + int firstNonNullIndex = -1, lastNonNullIndex = -1; + for (int j = 0; j < dataPoints.size(); j++) { + long dataPoint = dataPoints.get(j); + if (dataPoint >= 0L) { + if (firstNonNullIndex < 0) { + firstNonNullIndex = j; + } + lastNonNullIndex = j; + if (dataPoint > maxValue) { + maxValue = dataPoint; + } + } + } + if (firstNonNullIndex < 0) { + continue; + } + long firstDataPointMillis = (((this.now - graphInterval) + / dataPointInterval) + firstNonNullIndex) * dataPointInterval + + dataPointInterval / 2L; + if (i > 0 && + firstDataPointMillis >= this.now - graphIntervals[i - 1]) { + /* Skip bandwidth history object, because it doesn't contain + * anything new that wasn't already contained in the last + * bandwidth history object(s). */ + continue; + } + long lastDataPointMillis = firstDataPointMillis + + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; + double factor = ((double) maxValue) / 999.0; + int count = lastNonNullIndex - firstNonNullIndex + 1; + StringBuilder sb2 = new StringBuilder(); + sb2.append("\"" + graphName + "\":{" + + "\"first\":\"" + + this.dateTimeFormat.format(firstDataPointMillis) + "\"," + + "\"last\":\"" + + this.dateTimeFormat.format(lastDataPointMillis) + "\"," + +"\"interval\":" + String.valueOf(dataPointInterval / 1000L) + + ",\"factor\":" + String.format(Locale.US, "%.3f", factor) + + ",\"count\":" + String.valueOf(count) + ",\"values\":["); + int written = 0, previousNonNullIndex = -2; + boolean foundTwoAdjacentDataPoints = false; + for (int j = firstNonNullIndex; j <= lastNonNullIndex; j++) { + long dataPoint = dataPoints.get(j); + if (dataPoint >= 0L) { + if (j - previousNonNullIndex == 1) { + foundTwoAdjacentDataPoints = true; + } + previousNonNullIndex = j; + } + sb2.append((written++ > 0 ? "," : "") + (dataPoint < 0L ? "null" : + String.valueOf((dataPoint * 999L) / maxValue))); + } + sb2.append("]},\n"); + if (foundTwoAdjacentDataPoints) { + sb.append(sb2.toString()); + } + } + String result = sb.toString(); + if (result.length() >= 2) { + result = result.substring(0, result.length() - 2) + "\n"; + } + return result; + } + + public String getStatsString() { + /* TODO Add statistics string. */ + return null; + } +} diff --git a/src/org/torproject/onionoo/BandwidthStatus.java b/src/org/torproject/onionoo/BandwidthStatus.java index bf6f504..fd3c36e 100644 --- a/src/org/torproject/onionoo/BandwidthStatus.java +++ b/src/org/torproject/onionoo/BandwidthStatus.java @@ -1,7 +1,79 @@ -/* Copyright 2013 The Tor Project +/* Copyright 2013--2014 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Scanner; +import java.util.SortedMap; +import java.util.TimeZone; +import java.util.TreeMap; + class BandwidthStatus extends Document { + + SortedMap<Long, long[]> writeHistory = new TreeMap<Long, long[]>(); + + SortedMap<Long, long[]> readHistory = new TreeMap<Long, long[]>(); + + public void fromDocumentString(String documentString) { + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setLenient(false); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + try { + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + String[] parts = line.split(" "); + if (parts.length != 6) { + System.err.println("Illegal line '" + line + "' in bandwidth " + + "history. Skipping this line."); + continue; + } + SortedMap<Long, long[]> history = parts[0].equals("r") + ? readHistory : writeHistory; + long startMillis = dateTimeFormat.parse(parts[1] + " " + + parts[2]).getTime(); + long endMillis = dateTimeFormat.parse(parts[3] + " " + + parts[4]).getTime(); + long bandwidth = Long.parseLong(parts[5]); + long previousEndMillis = history.headMap(startMillis).isEmpty() + ? startMillis + : history.get(history.headMap(startMillis).lastKey())[1]; + long nextStartMillis = history.tailMap(startMillis).isEmpty() + ? endMillis : history.tailMap(startMillis).firstKey(); + if (previousEndMillis <= startMillis && + nextStartMillis >= endMillis) { + history.put(startMillis, new long[] { startMillis, endMillis, + bandwidth }); + } + } + s.close(); + } catch (ParseException e) { + System.err.println("Could not parse timestamp while reading " + + "bandwidth history. Skipping."); + e.printStackTrace(); + } + + } + + public String toDocumentString() { + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setLenient(false); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + StringBuilder sb = new StringBuilder(); + for (long[] v : writeHistory.values()) { + sb.append("w " + dateTimeFormat.format(v[0]) + " " + + dateTimeFormat.format(v[1]) + " " + + String.valueOf(v[2]) + "\n"); + } + for (long[] v : readHistory.values()) { + sb.append("r " + dateTimeFormat.format(v[0]) + " " + + dateTimeFormat.format(v[1]) + " " + + String.valueOf(v[2]) + "\n"); + } + return sb.toString(); + } } diff --git a/src/org/torproject/onionoo/BandwidthStatusUpdater.java b/src/org/torproject/onionoo/BandwidthStatusUpdater.java new file mode 100644 index 0000000..6254260 --- /dev/null +++ b/src/org/torproject/onionoo/BandwidthStatusUpdater.java @@ -0,0 +1,152 @@ +/* Copyright 2011--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.SortedMap; +import java.util.TimeZone; +import java.util.TreeMap; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.ExtraInfoDescriptor; + +public class BandwidthStatusUpdater implements DescriptorListener, + StatusUpdater { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + private SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + + public BandwidthStatusUpdater(DescriptorSource descriptorSource, + DocumentStore documentStore, Time time) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + this.now = time.currentTimeMillis(); + this.dateTimeFormat.setLenient(false); + this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + this.registerDescriptorListeners(); + } + + private void registerDescriptorListeners() { + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.RELAY_EXTRA_INFOS); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_EXTRA_INFOS); + } + + public void processDescriptor(Descriptor descriptor, boolean relay) { + if (descriptor instanceof ExtraInfoDescriptor) { + this.parseDescriptor((ExtraInfoDescriptor) descriptor); + } + } + + public void updateStatuses() { + /* Status files are already updated while processing descriptors. */ + } + + private void parseDescriptor(ExtraInfoDescriptor descriptor) { + String fingerprint = descriptor.getFingerprint(); + BandwidthStatus bandwidthStatus = this.documentStore.retrieve( + BandwidthStatus.class, true, fingerprint); + if (bandwidthStatus == null) { + bandwidthStatus = new BandwidthStatus(); + } + if (descriptor.getWriteHistory() != null) { + parseHistoryLine(descriptor.getWriteHistory().getLine(), + bandwidthStatus.writeHistory); + } + if (descriptor.getReadHistory() != null) { + parseHistoryLine(descriptor.getReadHistory().getLine(), + bandwidthStatus.readHistory); + } + this.compressHistory(bandwidthStatus.writeHistory); + this.compressHistory(bandwidthStatus.readHistory); + this.documentStore.store(bandwidthStatus, fingerprint); + } + + private void parseHistoryLine(String line, + SortedMap<Long, long[]> history) { + String[] parts = line.split(" "); + if (parts.length < 6) { + return; + } + try { + long endMillis = this.dateTimeFormat.parse(parts[1] + " " + + parts[2]).getTime(); + long intervalMillis = Long.parseLong(parts[3].substring(1)) * 1000L; + String[] values = parts[5].split(","); + for (int i = values.length - 1; i >= 0; i--) { + long bandwidthValue = Long.parseLong(values[i]); + long startMillis = endMillis - intervalMillis; + /* TODO Should we first check whether an interval is already + * contained in history? */ + history.put(startMillis, new long[] { startMillis, endMillis, + bandwidthValue }); + endMillis -= intervalMillis; + } + } catch (ParseException e) { + System.err.println("Could not parse timestamp in line '" + line + + "'. Skipping."); + } + } + + private void compressHistory(SortedMap<Long, long[]> history) { + SortedMap<Long, long[]> uncompressedHistory = + new TreeMap<Long, long[]>(history); + history.clear(); + long lastStartMillis = 0L, lastEndMillis = 0L, lastBandwidth = 0L; + SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + String lastMonthString = "1970-01"; + for (long[] v : uncompressedHistory.values()) { + long startMillis = v[0], endMillis = v[1], bandwidth = v[2]; + long intervalLengthMillis; + if (this.now - endMillis <= 72L * 60L * 60L * 1000L) { + intervalLengthMillis = 15L * 60L * 1000L; + } else if (this.now - endMillis <= 7L * 24L * 60L * 60L * 1000L) { + intervalLengthMillis = 60L * 60L * 1000L; + } else if (this.now - endMillis <= 31L * 24L * 60L * 60L * 1000L) { + intervalLengthMillis = 4L * 60L * 60L * 1000L; + } else if (this.now - endMillis <= 92L * 24L * 60L * 60L * 1000L) { + intervalLengthMillis = 12L * 60L * 60L * 1000L; + } else if (this.now - endMillis <= 366L * 24L * 60L * 60L * 1000L) { + intervalLengthMillis = 2L * 24L * 60L * 60L * 1000L; + } else { + intervalLengthMillis = 10L * 24L * 60L * 60L * 1000L; + } + String monthString = dateTimeFormat.format(startMillis); + if (lastEndMillis == startMillis && + ((lastEndMillis - 1L) / intervalLengthMillis) == + ((endMillis - 1L) / intervalLengthMillis) && + lastMonthString.equals(monthString)) { + lastEndMillis = endMillis; + lastBandwidth += bandwidth; + } else { + if (lastStartMillis > 0L) { + history.put(lastStartMillis, new long[] { lastStartMillis, + lastEndMillis, lastBandwidth }); + } + lastStartMillis = startMillis; + lastEndMillis = endMillis; + lastBandwidth = bandwidth; + } + lastMonthString = monthString; + } + if (lastStartMillis > 0L) { + history.put(lastStartMillis, new long[] { lastStartMillis, + lastEndMillis, lastBandwidth }); + } + } + + public String getStatsString() { + /* TODO Add statistics string. */ + return null; + } +} + diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java index 5da7267..be6abd5 100644 --- a/src/org/torproject/onionoo/DocumentStore.java +++ b/src/org/torproject/onionoo/DocumentStore.java @@ -196,7 +196,8 @@ public class DocumentStore { document instanceof UptimeDocument) { Gson gson = new Gson(); documentString = gson.toJson(this); - } else if (document instanceof WeightsStatus || + } else if (document instanceof BandwidthStatus || + document instanceof WeightsStatus || document instanceof ClientsStatus || document instanceof UptimeStatus) { documentString = document.toDocumentString(); @@ -290,7 +291,8 @@ public class DocumentStore { documentType.equals(UptimeDocument.class)) { return this.retrieveParsedDocumentFile(documentType, documentString); - } else if (documentType.equals(WeightsStatus.class) || + } else if (documentType.equals(BandwidthStatus.class) || + documentType.equals(WeightsStatus.class) || documentType.equals(ClientsStatus.class) || documentType.equals(UptimeStatus.class)) { return this.retrieveParsedStatusFile(documentType, documentString); diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 434d90c..60db116 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -32,16 +32,18 @@ public class Main { Logger.printStatusTime("Initialized reverse domain name resolver"); NodeDataWriter ndw = new NodeDataWriter(dso, rdnr, ls, ds, t); Logger.printStatusTime("Initialized node data writer"); - BandwidthDataWriter bdw = new BandwidthDataWriter(dso, ds, t); - Logger.printStatusTime("Initialized bandwidth data writer"); + BandwidthStatusUpdater bsu = new BandwidthStatusUpdater(dso, ds, t); + Logger.printStatusTime("Initialized bandwidth status updater"); WeightsStatusUpdater wsu = new WeightsStatusUpdater(dso, ds, t); Logger.printStatusTime("Initialized weights status updater"); ClientsStatusUpdater csu = new ClientsStatusUpdater(dso, ds, t); Logger.printStatusTime("Initialized clients status updater"); UptimeStatusUpdater usu = new UptimeStatusUpdater(dso, ds); Logger.printStatusTime("Initialized uptime status updater"); - StatusUpdater[] sus = new StatusUpdater[] { ndw, bdw, wsu, csu, usu }; + StatusUpdater[] sus = new StatusUpdater[] { ndw, bsu, wsu, csu, usu }; + BandwidthDocumentWriter bdw = new BandwidthDocumentWriter(dso, ds, t); + Logger.printStatusTime("Initialized bandwidth document writer"); WeightsDocumentWriter wdw = new WeightsDocumentWriter(dso, ds, t); Logger.printStatusTime("Initialized weights document writer"); ClientsDocumentWriter cdw = new ClientsDocumentWriter(dso, ds, t); @@ -95,7 +97,8 @@ public class Main { } /* TODO Print status updater statistics for *all* status updaters once * all data writers have been separated. */ - for (DocumentWriter dw : new DocumentWriter[] { wdw, cdw, udw }) { + for (DocumentWriter dw : new DocumentWriter[] { bdw, wdw, cdw, + udw }) { String statsString = dw.getStatsString(); if (statsString != null) { Logger.printStatistics(dw.getClass().getSimpleName(),

1 0

[onionoo/master] Split node data writer into two classes.
by karsten＠torproject.org 11 Apr '14

11 Apr '14

commit e152b6761b125ddcbfe57fc8713043e2af0ad2a3 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Mar 14 15:53:54 2014 +0100 Split node data writer into two classes. --- .../torproject/onionoo/DetailsDocumentWriter.java | 380 ++++++++ src/org/torproject/onionoo/Main.java | 15 +- src/org/torproject/onionoo/NodeDataWriter.java | 959 -------------------- .../onionoo/NodeDetailsStatusUpdater.java | 634 +++++++++++++ 4 files changed, 1022 insertions(+), 966 deletions(-) diff --git a/src/org/torproject/onionoo/DetailsDocumentWriter.java b/src/org/torproject/onionoo/DetailsDocumentWriter.java new file mode 100644 index 0000000..0fd47c5 --- /dev/null +++ b/src/org/torproject/onionoo/DetailsDocumentWriter.java @@ -0,0 +1,380 @@ +package org.torproject.onionoo; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Scanner; +import java.util.Set; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeSet; + +import org.apache.commons.lang.StringEscapeUtils; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.ExitList; +import org.torproject.descriptor.ExitListEntry; + +public class DetailsDocumentWriter implements DescriptorListener, + FingerprintListener, DocumentWriter { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + public DetailsDocumentWriter(DescriptorSource descriptorSource, + DocumentStore documentStore, Time time) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + this.now = time.currentTimeMillis(); + this.registerDescriptorListeners(); + this.registerFingerprintListeners(); + } + + private void registerDescriptorListeners() { + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.EXIT_LISTS); + } + + public void processDescriptor(Descriptor descriptor, boolean relay) { + if (descriptor instanceof ExitList) { + this.processExitList((ExitList) descriptor); + } + } + + private Map<String, Set<ExitListEntry>> exitListEntries = + new HashMap<String, Set<ExitListEntry>>(); + + /* TODO Processing descriptors should really be done in + * NodeDetailsStatusUpdater, not here. This is also a bug, because + * we're only considering newly published exit lists. */ + private void processExitList(ExitList exitList) { + for (ExitListEntry exitListEntry : exitList.getExitListEntries()) { + if (exitListEntry.getScanMillis() < + this.now - 24L * 60L * 60L * 1000L) { + continue; + } + String fingerprint = exitListEntry.getFingerprint(); + if (!this.exitListEntries.containsKey(fingerprint)) { + this.exitListEntries.put(fingerprint, + new HashSet<ExitListEntry>()); + } + this.exitListEntries.get(fingerprint).add(exitListEntry); + } + } + + private void registerFingerprintListeners() { + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_SERVER_DESCRIPTORS); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_STATUSES); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_SERVER_DESCRIPTORS); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_POOL_ASSIGNMENTS); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.EXIT_LISTS); + } + + private SortedSet<String> newRelays = new TreeSet<String>(), + newBridges = new TreeSet<String>(); + + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + if (relay) { + this.newRelays.addAll(fingerprints); + } else { + this.newBridges.addAll(fingerprints); + } + } + + public void writeDocuments() { + this.updateRelayDetailsFiles(); + this.updateBridgeDetailsFiles(); + Logger.printStatusTime("Wrote details document files"); + } + + private void updateRelayDetailsFiles() { + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + for (String fingerprint : this.newRelays) { + + /* Generate network-status-specific part. */ + NodeStatus entry = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (entry == null) { + continue; + } + String nickname = entry.getNickname(); + String address = entry.getAddress(); + List<String> orAddresses = new ArrayList<String>(); + orAddresses.add(address + ":" + entry.getOrPort()); + orAddresses.addAll(entry.getOrAddressesAndPorts()); + StringBuilder orAddressesAndPortsBuilder = new StringBuilder(); + int addressesWritten = 0; + for (String orAddress : orAddresses) { + orAddressesAndPortsBuilder.append( + (addressesWritten++ > 0 ? "," : "") + "\"" + + orAddress.toLowerCase() + "\""); + } + String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); + String firstSeen = dateTimeFormat.format( + entry.getFirstSeenMillis()); + String lastChangedOrAddress = dateTimeFormat.format( + entry.getLastChangedOrAddress()); + String running = entry.getRunning() ? "true" : "false"; + int dirPort = entry.getDirPort(); + String countryCode = entry.getCountryCode(); + String latitude = entry.getLatitude(); + String longitude = entry.getLongitude(); + String countryName = entry.getCountryName(); + String regionName = entry.getRegionName(); + String cityName = entry.getCityName(); + String aSNumber = entry.getASNumber(); + String aSName = entry.getASName(); + long consensusWeight = entry.getConsensusWeight(); + String hostName = entry.getHostName(); + double advertisedBandwidthFraction = + entry.getAdvertisedBandwidthFraction(); + double consensusWeightFraction = entry.getConsensusWeightFraction(); + double guardProbability = entry.getGuardProbability(); + double middleProbability = entry.getMiddleProbability(); + double exitProbability = entry.getExitProbability(); + String defaultPolicy = entry.getDefaultPolicy(); + String portList = entry.getPortList(); + Boolean recommendedVersion = entry.getRecommendedVersion(); + StringBuilder sb = new StringBuilder(); + sb.append("{\"version\":1,\n" + + "\"nickname\":\"" + nickname + "\",\n" + + "\"fingerprint\":\"" + fingerprint + "\",\n" + + "\"or_addresses\":[" + orAddressesAndPortsBuilder.toString() + + "]"); + if (dirPort != 0) { + sb.append(",\n\"dir_address\":\"" + address + ":" + dirPort + + "\""); + } + sb.append(",\n\"last_seen\":\"" + lastSeen + "\""); + sb.append(",\n\"first_seen\":\"" + firstSeen + "\""); + sb.append(",\n\"last_changed_address_or_port\":\"" + + lastChangedOrAddress + "\""); + sb.append(",\n\"running\":" + running); + SortedSet<String> relayFlags = entry.getRelayFlags(); + if (!relayFlags.isEmpty()) { + sb.append(",\n\"flags\":["); + int written = 0; + for (String relayFlag : relayFlags) { + sb.append((written++ > 0 ? "," : "") + "\"" + relayFlag + "\""); + } + sb.append("]"); + } + if (countryCode != null) { + sb.append(",\n\"country\":\"" + countryCode + "\""); + } + if (latitude != null) { + sb.append(",\n\"latitude\":" + latitude); + } + if (longitude != null) { + sb.append(",\n\"longitude\":" + longitude); + } + if (countryName != null) { + sb.append(",\n\"country_name\":\"" + + escapeJSON(countryName) + "\""); + } + if (regionName != null) { + sb.append(",\n\"region_name\":\"" + + escapeJSON(regionName) + "\""); + } + if (cityName != null) { + sb.append(",\n\"city_name\":\"" + + escapeJSON(cityName) + "\""); + } + if (aSNumber != null) { + sb.append(",\n\"as_number\":\"" + + escapeJSON(aSNumber) + "\""); + } + if (aSName != null) { + sb.append(",\n\"as_name\":\"" + + escapeJSON(aSName) + "\""); + } + if (consensusWeight >= 0L) { + sb.append(",\n\"consensus_weight\":" + + String.valueOf(consensusWeight)); + } + if (hostName != null) { + sb.append(",\n\"host_name\":\"" + + escapeJSON(hostName) + "\""); + } + if (advertisedBandwidthFraction >= 0.0) { + sb.append(String.format( + ",\n\"advertised_bandwidth_fraction\":%.9f", + advertisedBandwidthFraction)); + } + if (consensusWeightFraction >= 0.0) { + sb.append(String.format(",\n\"consensus_weight_fraction\":%.9f", + consensusWeightFraction)); + } + if (guardProbability >= 0.0) { + sb.append(String.format(",\n\"guard_probability\":%.9f", + guardProbability)); + } + if (middleProbability >= 0.0) { + sb.append(String.format(",\n\"middle_probability\":%.9f", + middleProbability)); + } + if (exitProbability >= 0.0) { + sb.append(String.format(",\n\"exit_probability\":%.9f", + exitProbability)); + } + if (defaultPolicy != null && (defaultPolicy.equals("accept") || + defaultPolicy.equals("reject")) && portList != null) { + sb.append(",\n\"exit_policy_summary\":{\"" + defaultPolicy + + "\":["); + int portsWritten = 0; + for (String portOrPortRange : portList.split(",")) { + sb.append((portsWritten++ > 0 ? "," : "") + + "\"" + portOrPortRange + "\""); + } + sb.append("]}"); + } + if (recommendedVersion != null) { + sb.append(",\n\"recommended_version\":" + (recommendedVersion ? + "true" : "false")); + } + + /* Add exit addresses if at least one of them is distinct from the + * onion-routing addresses. */ + if (exitListEntries.containsKey(fingerprint)) { + for (ExitListEntry exitListEntry : + exitListEntries.get(fingerprint)) { + entry.addExitAddress(exitListEntry.getExitAddress()); + } + } + if (!entry.getExitAddresses().isEmpty()) { + sb.append(",\n\"exit_addresses\":["); + int written = 0; + for (String exitAddress : entry.getExitAddresses()) { + sb.append((written++ > 0 ? "," : "") + "\"" + + exitAddress.toLowerCase() + "\""); + } + sb.append("]"); + } + + /* Append descriptor-specific part from details status file, and + * update contact in node status. */ + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, false, fingerprint); + if (detailsStatus != null && + detailsStatus.documentString.length() > 0) { + sb.append(",\n" + detailsStatus.documentString); + String contact = null; + Scanner s = new Scanner(detailsStatus.documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + if (!line.startsWith("\"contact\":")) { + continue; + } + int start = "\"contact\":\"".length(), end = line.length() - 1; + if (line.endsWith(",")) { + end--; + } + contact = unescapeJSON(line.substring(start, end)); + break; + } + s.close(); + entry.setContact(contact); + } + + /* Finish details string. */ + sb.append("\n}\n"); + + /* Write details file to disk. */ + DetailsDocument detailsDocument = new DetailsDocument(); + detailsDocument.documentString = sb.toString(); + this.documentStore.store(detailsDocument, fingerprint); + } + } + + private void updateBridgeDetailsFiles() { + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + for (String fingerprint : this.newBridges) { + + /* Generate network-status-specific part. */ + NodeStatus entry = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (entry == null) { + continue; + } + String nickname = entry.getNickname(); + String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); + String firstSeen = dateTimeFormat.format( + entry.getFirstSeenMillis()); + String running = entry.getRunning() ? "true" : "false"; + String address = entry.getAddress(); + List<String> orAddresses = new ArrayList<String>(); + orAddresses.add(address + ":" + entry.getOrPort()); + orAddresses.addAll(entry.getOrAddressesAndPorts()); + StringBuilder orAddressesAndPortsBuilder = new StringBuilder(); + int addressesWritten = 0; + for (String orAddress : orAddresses) { + orAddressesAndPortsBuilder.append( + (addressesWritten++ > 0 ? "," : "") + "\"" + + orAddress.toLowerCase() + "\""); + } + StringBuilder sb = new StringBuilder(); + sb.append("{\"version\":1,\n" + + "\"nickname\":\"" + nickname + "\",\n" + + "\"hashed_fingerprint\":\"" + fingerprint + "\",\n" + + "\"or_addresses\":[" + orAddressesAndPortsBuilder.toString() + + "],\n\"last_seen\":\"" + lastSeen + "\",\n\"first_seen\":\"" + + firstSeen + "\",\n\"running\":" + running); + + SortedSet<String> relayFlags = entry.getRelayFlags(); + if (!relayFlags.isEmpty()) { + sb.append(",\n\"flags\":["); + int written = 0; + for (String relayFlag : relayFlags) { + sb.append((written++ > 0 ? "," : "") + "\"" + relayFlag + "\""); + } + sb.append("]"); + } + + /* Append descriptor-specific part from details status file. */ + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, false, fingerprint); + if (detailsStatus != null && + detailsStatus.documentString.length() > 0) { + sb.append(",\n" + detailsStatus.documentString); + } + + /* Finish details string. */ + sb.append("\n}\n"); + + /* Write details file to disk. */ + DetailsDocument detailsDocument = new DetailsDocument(); + detailsDocument.documentString = sb.toString(); + this.documentStore.store(detailsDocument, fingerprint); + } + } + + private static String escapeJSON(String s) { + return StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'"); + } + + private static String unescapeJSON(String s) { + return StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'")); + } + + public String getStatsString() { + /* TODO Add statistics string. */ + return null; + } +} diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 60db116..355baac 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -30,7 +30,8 @@ public class Main { Logger.printStatusTime("Initialized Geoip lookup service"); ReverseDomainNameResolver rdnr = new ReverseDomainNameResolver(t); Logger.printStatusTime("Initialized reverse domain name resolver"); - NodeDataWriter ndw = new NodeDataWriter(dso, rdnr, ls, ds, t); + NodeDetailsStatusUpdater ndsu = new NodeDetailsStatusUpdater(dso, + rdnr, ls, ds, t); Logger.printStatusTime("Initialized node data writer"); BandwidthStatusUpdater bsu = new BandwidthStatusUpdater(dso, ds, t); Logger.printStatusTime("Initialized bandwidth status updater"); @@ -40,8 +41,11 @@ public class Main { Logger.printStatusTime("Initialized clients status updater"); UptimeStatusUpdater usu = new UptimeStatusUpdater(dso, ds); Logger.printStatusTime("Initialized uptime status updater"); - StatusUpdater[] sus = new StatusUpdater[] { ndw, bsu, wsu, csu, usu }; + StatusUpdater[] sus = new StatusUpdater[] { ndsu, bsu, wsu, csu, + usu }; + DetailsDocumentWriter ddw = new DetailsDocumentWriter(dso, ds, t); + Logger.printStatusTime("Initialized details document writer"); BandwidthDocumentWriter bdw = new BandwidthDocumentWriter(dso, ds, t); Logger.printStatusTime("Initialized bandwidth document writer"); WeightsDocumentWriter wdw = new WeightsDocumentWriter(dso, ds, t); @@ -50,7 +54,7 @@ public class Main { Logger.printStatusTime("Initialized clients document writer"); UptimeDocumentWriter udw = new UptimeDocumentWriter(dso, ds, t); Logger.printStatusTime("Initialized uptime document writer"); - DocumentWriter[] dws = new DocumentWriter[] { ndw, bdw, wdw, cdw, + DocumentWriter[] dws = new DocumentWriter[] { ddw, bdw, wdw, cdw, udw }; Logger.printStatus("Reading descriptors."); @@ -95,10 +99,7 @@ public class Main { statsString); } } - /* TODO Print status updater statistics for *all* status updaters once - * all data writers have been separated. */ - for (DocumentWriter dw : new DocumentWriter[] { bdw, wdw, cdw, - udw }) { + for (DocumentWriter dw : dws) { String statsString = dw.getStatsString(); if (statsString != null) { Logger.printStatistics(dw.getClass().getSimpleName(), diff --git a/src/org/torproject/onionoo/NodeDataWriter.java b/src/org/torproject/onionoo/NodeDataWriter.java deleted file mode 100644 index 5941b1c..0000000 --- a/src/org/torproject/onionoo/NodeDataWriter.java +++ /dev/null @@ -1,959 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Scanner; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TimeZone; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.apache.commons.lang.StringEscapeUtils; -import org.torproject.descriptor.BridgeNetworkStatus; -import org.torproject.descriptor.BridgePoolAssignment; -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.ExitList; -import org.torproject.descriptor.ExitListEntry; -import org.torproject.descriptor.NetworkStatusEntry; -import org.torproject.descriptor.RelayNetworkStatusConsensus; -import org.torproject.descriptor.ServerDescriptor; -import org.torproject.onionoo.LookupService.LookupResult; - -/* Write updated summary and details data files to disk. - * - * The parts of details files coming from server descriptors always come - * from the last known descriptor of a relay or bridge, not from the - * descriptor that was last referenced in a network status. */ -public class NodeDataWriter implements DescriptorListener, StatusUpdater, - FingerprintListener, DocumentWriter { - - private DescriptorSource descriptorSource; - - private ReverseDomainNameResolver reverseDomainNameResolver; - - private LookupService lookupService; - - private DocumentStore documentStore; - - private long now; - - private SortedMap<String, NodeStatus> knownNodes = - new TreeMap<String, NodeStatus>(); - - private SortedMap<String, NodeStatus> relays; - - private SortedMap<String, NodeStatus> bridges; - - private long relaysLastValidAfterMillis = -1L; - - private long bridgesLastPublishedMillis = -1L; - - private SortedMap<String, Integer> lastBandwidthWeights = null; - - private int relayConsensusesProcessed = 0, bridgeStatusesProcessed = 0; - - public NodeDataWriter(DescriptorSource descriptorSource, - ReverseDomainNameResolver reverseDomainNameResolver, - LookupService lookupService, DocumentStore documentStore, - Time time) { - this.descriptorSource = descriptorSource; - this.reverseDomainNameResolver = reverseDomainNameResolver; - this.lookupService = lookupService; - this.documentStore = documentStore; - this.now = time.currentTimeMillis(); - this.registerDescriptorListeners(); - this.registerFingerprintListeners(); - } - - private void registerDescriptorListeners() { - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.RELAY_SERVER_DESCRIPTORS); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.BRIDGE_STATUSES); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.BRIDGE_SERVER_DESCRIPTORS); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.BRIDGE_POOL_ASSIGNMENTS); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.EXIT_LISTS); - } - - private void registerFingerprintListeners() { - /* TODO Not used yet. - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_SERVER_DESCRIPTORS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_STATUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_SERVER_DESCRIPTORS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_POOL_ASSIGNMENTS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.EXIT_LISTS);*/ - } - - public void processDescriptor(Descriptor descriptor, boolean relay) { - if (descriptor instanceof RelayNetworkStatusConsensus) { - this.updateRelayNetworkStatusConsensus( - (RelayNetworkStatusConsensus) descriptor); - } else if (descriptor instanceof ServerDescriptor && relay) { - this.processRelayServerDescriptor((ServerDescriptor) descriptor); - } else if (descriptor instanceof BridgeNetworkStatus) { - this.updateBridgeNetworkStatus((BridgeNetworkStatus) descriptor); - } else if (descriptor instanceof ServerDescriptor && !relay) { - this.processBridgeServerDescriptor((ServerDescriptor) descriptor); - } else if (descriptor instanceof BridgePoolAssignment) { - this.processBridgePoolAssignment((BridgePoolAssignment) descriptor); - } else if (descriptor instanceof ExitList) { - this.processExitList((ExitList) descriptor); - } - } - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - /* TODO Not used yet. */ - } - - public void updateStatuses() { - this.readStatusSummary(); - Logger.printStatusTime("Read status summary"); - this.setCurrentNodes(); - Logger.printStatusTime("Set current node fingerprints"); - this.startReverseDomainNameLookups(); - Logger.printStatusTime("Started reverse domain name lookups"); - this.lookUpCitiesAndASes(); - Logger.printStatusTime("Looked up cities and ASes"); - this.setRunningBits(); - Logger.printStatusTime("Set running bits"); - this.calculatePathSelectionProbabilities(); - Logger.printStatusTime("Calculated path selection probabilities"); - this.finishReverseDomainNameLookups(); - Logger.printStatusTime("Finished reverse domain name lookups"); - this.writeStatusSummary(); - Logger.printStatusTime("Wrote status summary"); - this.writeOutSummary(); - Logger.printStatusTime("Wrote out summary"); - } - - public void writeDocuments() { - this.writeOutDetails(); - Logger.printStatusTime("Wrote detail data files"); - } - - private void updateRelayNetworkStatusConsensus( - RelayNetworkStatusConsensus consensus) { - long validAfterMillis = consensus.getValidAfterMillis(); - if (validAfterMillis > this.relaysLastValidAfterMillis) { - this.relaysLastValidAfterMillis = validAfterMillis; - } - Set<String> recommendedVersions = null; - if (consensus.getRecommendedServerVersions() != null) { - recommendedVersions = new HashSet<String>(); - for (String recommendedVersion : - consensus.getRecommendedServerVersions()) { - recommendedVersions.add("Tor " + recommendedVersion); - } - } - for (NetworkStatusEntry entry : - consensus.getStatusEntries().values()) { - String nickname = entry.getNickname(); - String fingerprint = entry.getFingerprint(); - String address = entry.getAddress(); - SortedSet<String> orAddressesAndPorts = new TreeSet<String>( - entry.getOrAddresses()); - int orPort = entry.getOrPort(); - int dirPort = entry.getDirPort(); - SortedSet<String> relayFlags = entry.getFlags(); - long consensusWeight = entry.getBandwidth(); - String defaultPolicy = entry.getDefaultPolicy(); - String portList = entry.getPortList(); - Boolean recommendedVersion = (recommendedVersions == null || - entry.getVersion() == null) ? null : - recommendedVersions.contains(entry.getVersion()); - NodeStatus newNodeStatus = new NodeStatus(true, nickname, - fingerprint, address, orAddressesAndPorts, null, - validAfterMillis, orPort, dirPort, relayFlags, consensusWeight, - null, null, -1L, defaultPolicy, portList, validAfterMillis, - validAfterMillis, null, null, recommendedVersion); - if (this.knownNodes.containsKey(fingerprint)) { - this.knownNodes.get(fingerprint).update(newNodeStatus); - } else { - this.knownNodes.put(fingerprint, newNodeStatus); - } - } - this.relayConsensusesProcessed++; - if (this.relaysLastValidAfterMillis == validAfterMillis) { - this.lastBandwidthWeights = consensus.getBandwidthWeights(); - } - } - - private void updateBridgeNetworkStatus(BridgeNetworkStatus status) { - long publishedMillis = status.getPublishedMillis(); - if (publishedMillis > this.bridgesLastPublishedMillis) { - this.bridgesLastPublishedMillis = publishedMillis; - } - for (NetworkStatusEntry entry : status.getStatusEntries().values()) { - String nickname = entry.getNickname(); - String fingerprint = entry.getFingerprint(); - String address = entry.getAddress(); - SortedSet<String> orAddressesAndPorts = new TreeSet<String>( - entry.getOrAddresses()); - int orPort = entry.getOrPort(); - int dirPort = entry.getDirPort(); - SortedSet<String> relayFlags = entry.getFlags(); - NodeStatus newNodeStatus = new NodeStatus(false, nickname, - fingerprint, address, orAddressesAndPorts, null, - publishedMillis, orPort, dirPort, relayFlags, -1L, "??", null, - -1L, null, null, publishedMillis, -1L, null, null, null); - if (this.knownNodes.containsKey(fingerprint)) { - this.knownNodes.get(fingerprint).update(newNodeStatus); - } else { - this.knownNodes.put(fingerprint, newNodeStatus); - } - } - this.bridgeStatusesProcessed++; - } - - private void readStatusSummary() { - SortedSet<String> fingerprints = this.documentStore.list( - NodeStatus.class, true); - for (String fingerprint : fingerprints) { - NodeStatus node = this.documentStore.retrieve(NodeStatus.class, - true, fingerprint); - if (node.isRelay()) { - this.relaysLastValidAfterMillis = Math.max( - this.relaysLastValidAfterMillis, node.getLastSeenMillis()); - } else { - this.bridgesLastPublishedMillis = Math.max( - this.bridgesLastPublishedMillis, node.getLastSeenMillis()); - } - if (this.knownNodes.containsKey(fingerprint)) { - this.knownNodes.get(fingerprint).update(node); - } else { - this.knownNodes.put(fingerprint, node); - } - } - } - - private void setRunningBits() { - for (NodeStatus node : this.knownNodes.values()) { - if (node.isRelay() && node.getRelayFlags().contains("Running") && - node.getLastSeenMillis() == this.relaysLastValidAfterMillis) { - node.setRunning(true); - } - if (!node.isRelay() && node.getRelayFlags().contains("Running") && - node.getLastSeenMillis() == this.bridgesLastPublishedMillis) { - node.setRunning(true); - } - } - } - - private void lookUpCitiesAndASes() { - SortedSet<String> addressStrings = new TreeSet<String>(); - for (NodeStatus node : this.knownNodes.values()) { - if (node.isRelay()) { - addressStrings.add(node.getAddress()); - } - } - if (addressStrings.isEmpty()) { - System.err.println("No relay IP addresses to resolve to cities or " - + "ASN."); - return; - } - SortedMap<String, LookupResult> lookupResults = - this.lookupService.lookup(addressStrings); - for (NodeStatus node : knownNodes.values()) { - if (!node.isRelay()) { - continue; - } - String addressString = node.getAddress(); - if (lookupResults.containsKey(addressString)) { - LookupResult lookupResult = lookupResults.get(addressString); - node.setCountryCode(lookupResult.countryCode); - node.setCountryName(lookupResult.countryName); - node.setRegionName(lookupResult.regionName); - node.setCityName(lookupResult.cityName); - node.setLatitude(lookupResult.latitude); - node.setLongitude(lookupResult.longitude); - node.setASNumber(lookupResult.aSNumber); - node.setASName(lookupResult.aSName); - } - } - } - - private void writeStatusSummary() { - this.writeSummary(true); - } - - private void writeOutSummary() { - this.writeSummary(false); - } - - private void writeSummary(boolean includeArchive) { - SortedMap<String, NodeStatus> nodes = includeArchive - ? this.knownNodes : this.getCurrentNodes(); - for (Map.Entry<String, NodeStatus> e : nodes.entrySet()) { - this.documentStore.store(e.getValue(), e.getKey()); - } - } - - private SortedMap<String, NodeStatus> getCurrentNodes() { - long cutoff = Math.max(this.relaysLastValidAfterMillis, - this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L; - SortedMap<String, NodeStatus> currentNodes = - new TreeMap<String, NodeStatus>(); - for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { - if (e.getValue().getLastSeenMillis() >= cutoff) { - currentNodes.put(e.getKey(), e.getValue()); - } - } - return currentNodes; - } - - private void processRelayServerDescriptor( - ServerDescriptor descriptor) { - String fingerprint = descriptor.getFingerprint(); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setLenient(false); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String publishedDateTime = - dateTimeFormat.format(descriptor.getPublishedMillis()); - if (detailsStatus != null) { - String detailsString = detailsStatus.documentString; - String descPublishedLine = "\"desc_published\":\"" - + publishedDateTime + "\","; - Scanner s = new Scanner(detailsString); - while (s.hasNextLine()) { - String line = s.nextLine(); - if (line.startsWith("\"desc_published\":\"")) { - if (descPublishedLine.compareTo(line) < 0) { - return; - } else { - break; - } - } - } - s.close(); - } - StringBuilder sb = new StringBuilder(); - String lastRestartedString = dateTimeFormat.format( - descriptor.getPublishedMillis() - descriptor.getUptime() * 1000L); - int bandwidthRate = descriptor.getBandwidthRate(); - int bandwidthBurst = descriptor.getBandwidthBurst(); - int observedBandwidth = descriptor.getBandwidthObserved(); - int advertisedBandwidth = Math.min(bandwidthRate, - Math.min(bandwidthBurst, observedBandwidth)); - sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n" - + "\"last_restarted\":\"" + lastRestartedString + "\",\n" - + "\"bandwidth_rate\":" + bandwidthRate + ",\n" - + "\"bandwidth_burst\":" + bandwidthBurst + ",\n" - + "\"observed_bandwidth\":" + observedBandwidth + ",\n" - + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n" - + "\"exit_policy\":["); - int written = 0; - for (String exitPolicyLine : descriptor.getExitPolicyLines()) { - sb.append((written++ > 0 ? "," : "") + "\n \"" + exitPolicyLine - + "\""); - } - sb.append("\n]"); - if (descriptor.getContact() != null) { - sb.append(",\n\"contact\":\"" - + escapeJSON(descriptor.getContact()) + "\""); - } - if (descriptor.getPlatform() != null) { - sb.append(",\n\"platform\":\"" - + escapeJSON(descriptor.getPlatform()) + "\""); - } - if (descriptor.getFamilyEntries() != null) { - sb.append(",\n\"family\":["); - written = 0; - for (String familyEntry : descriptor.getFamilyEntries()) { - sb.append((written++ > 0 ? "," : "") + "\n \"" + familyEntry - + "\""); - } - sb.append("\n]"); - } - if (descriptor.getIpv6DefaultPolicy() != null && - (descriptor.getIpv6DefaultPolicy().equals("accept") || - descriptor.getIpv6DefaultPolicy().equals("reject")) && - descriptor.getIpv6PortList() != null) { - sb.append(",\n\"exit_policy_v6_summary\":{\"" - + descriptor.getIpv6DefaultPolicy() + "\":["); - int portsWritten = 0; - for (String portOrPortRange : - descriptor.getIpv6PortList().split(",")) { - sb.append((portsWritten++ > 0 ? "," : "") + "\"" + portOrPortRange - + "\""); - } - sb.append("]}"); - } - if (descriptor.isHibernating()) { - sb.append(",\n\"hibernating\":true"); - } - detailsStatus = new DetailsStatus(); - detailsStatus.documentString = sb.toString(); - this.documentStore.store(detailsStatus, fingerprint); - } - - private void processBridgeServerDescriptor( - ServerDescriptor descriptor) { - String fingerprint = descriptor.getFingerprint(); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setLenient(false); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String publishedDateTime = - dateTimeFormat.format(descriptor.getPublishedMillis()); - String poolAssignmentLine = null; - if (detailsStatus != null) { - String detailsString = detailsStatus.documentString; - String descPublishedLine = "\"desc_published\":\"" - + publishedDateTime + "\","; - Scanner s = new Scanner(detailsString); - while (s.hasNextLine()) { - String line = s.nextLine(); - if (line.startsWith("\"pool_assignment\":")) { - poolAssignmentLine = line; - } else if (line.startsWith("\"desc_published\":") && - descPublishedLine.compareTo(line) < 0) { - return; - } - } - s.close(); - } - StringBuilder sb = new StringBuilder(); - String lastRestartedString = dateTimeFormat.format( - descriptor.getPublishedMillis() - descriptor.getUptime() * 1000L); - int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(), - Math.min(descriptor.getBandwidthBurst(), - descriptor.getBandwidthObserved())); - sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n" - + "\"last_restarted\":\"" + lastRestartedString + "\",\n" - + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n" - + "\"platform\":\"" + escapeJSON(descriptor.getPlatform()) - + "\""); - if (poolAssignmentLine != null) { - sb.append(",\n" + poolAssignmentLine); - } - detailsStatus = new DetailsStatus(); - detailsStatus.documentString = sb.toString(); - this.documentStore.store(detailsStatus, fingerprint); - } - - private void processBridgePoolAssignment( - BridgePoolAssignment bridgePoolAssignment) { - for (Map.Entry<String, String> e : - bridgePoolAssignment.getEntries().entrySet()) { - String fingerprint = e.getKey(); - String details = e.getValue(); - StringBuilder sb = new StringBuilder(); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); - if (detailsStatus != null) { - String detailsString = detailsStatus.documentString; - Scanner s = new Scanner(detailsString); - int linesWritten = 0; - boolean endsWithComma = false; - while (s.hasNextLine()) { - String line = s.nextLine(); - if (!line.startsWith("\"pool_assignment\":")) { - sb.append((linesWritten++ > 0 ? "\n" : "") + line); - endsWithComma = line.endsWith(","); - } - } - s.close(); - if (sb.length() > 0) { - sb.append((endsWithComma ? "" : ",") + "\n"); - } - } - sb.append("\"pool_assignment\":\"" + details + "\""); - detailsStatus = new DetailsStatus(); - detailsStatus.documentString = sb.toString(); - this.documentStore.store(detailsStatus, fingerprint); - } - } - - private Map<String, Set<ExitListEntry>> exitListEntries = - new HashMap<String, Set<ExitListEntry>>(); - - private void processExitList(ExitList exitList) { - for (ExitListEntry exitListEntry : exitList.getExitListEntries()) { - if (exitListEntry.getScanMillis() < - this.now - 24L * 60L * 60L * 1000L) { - continue; - } - String fingerprint = exitListEntry.getFingerprint(); - if (!this.exitListEntries.containsKey(fingerprint)) { - this.exitListEntries.put(fingerprint, - new HashSet<ExitListEntry>()); - } - this.exitListEntries.get(fingerprint).add(exitListEntry); - } - } - - private void setCurrentNodes() { - SortedMap<String, NodeStatus> currentNodes = this.getCurrentNodes(); - this.relays = new TreeMap<String, NodeStatus>(); - this.bridges = new TreeMap<String, NodeStatus>(); - for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) { - if (e.getValue().isRelay()) { - this.relays.put(e.getKey(), e.getValue()); - } else { - this.bridges.put(e.getKey(), e.getValue()); - } - } - } - - private void startReverseDomainNameLookups() { - Map<String, Long> addressLastLookupTimes = - new HashMap<String, Long>(); - for (NodeStatus relay : relays.values()) { - addressLastLookupTimes.put(relay.getAddress(), - relay.getLastRdnsLookup()); - } - this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes); - this.reverseDomainNameResolver.startReverseDomainNameLookups(); - } - - private void finishReverseDomainNameLookups() { - this.reverseDomainNameResolver.finishReverseDomainNameLookups(); - Map<String, String> lookupResults = - this.reverseDomainNameResolver.getLookupResults(); - long startedRdnsLookups = - this.reverseDomainNameResolver.getLookupStartMillis(); - for (NodeStatus relay : relays.values()) { - if (lookupResults.containsKey(relay.getAddress())) { - relay.setHostName(lookupResults.get(relay.getAddress())); - relay.setLastRdnsLookup(startedRdnsLookups); - } - } - } - - private void calculatePathSelectionProbabilities() { - boolean consensusContainsBandwidthWeights = false; - double wgg = 0.0, wgd = 0.0, wmg = 0.0, wmm = 0.0, wme = 0.0, - wmd = 0.0, wee = 0.0, wed = 0.0; - if (this.lastBandwidthWeights != null) { - SortedSet<String> weightKeys = new TreeSet<String>(Arrays.asList( - "Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(","))); - weightKeys.removeAll(this.lastBandwidthWeights.keySet()); - if (weightKeys.isEmpty()) { - consensusContainsBandwidthWeights = true; - wgg = ((double) this.lastBandwidthWeights.get("Wgg")) / 10000.0; - wgd = ((double) this.lastBandwidthWeights.get("Wgd")) / 10000.0; - wmg = ((double) this.lastBandwidthWeights.get("Wmg")) / 10000.0; - wmm = ((double) this.lastBandwidthWeights.get("Wmm")) / 10000.0; - wme = ((double) this.lastBandwidthWeights.get("Wme")) / 10000.0; - wmd = ((double) this.lastBandwidthWeights.get("Wmd")) / 10000.0; - wee = ((double) this.lastBandwidthWeights.get("Wee")) / 10000.0; - wed = ((double) this.lastBandwidthWeights.get("Wed")) / 10000.0; - } - } else { - System.err.println("Could not determine most recent Wxx parameter " - + "values, probably because we didn't parse a consensus in " - + "this execution. All relays' guard/middle/exit weights are " - + "going to be 0.0."); - } - SortedMap<String, Double> - advertisedBandwidths = new TreeMap<String, Double>(), - consensusWeights = new TreeMap<String, Double>(), - guardWeights = new TreeMap<String, Double>(), - middleWeights = new TreeMap<String, Double>(), - exitWeights = new TreeMap<String, Double>(); - double totalAdvertisedBandwidth = 0.0; - double totalConsensusWeight = 0.0; - double totalGuardWeight = 0.0; - double totalMiddleWeight = 0.0; - double totalExitWeight = 0.0; - for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { - String fingerprint = e.getKey(); - NodeStatus relay = e.getValue(); - if (!relay.getRunning()) { - continue; - } - boolean isExit = relay.getRelayFlags().contains("Exit") && - !relay.getRelayFlags().contains("BadExit"); - boolean isGuard = relay.getRelayFlags().contains("Guard"); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); - if (detailsStatus != null) { - double advertisedBandwidth = -1.0; - String detailsString = detailsStatus.documentString; - Scanner s = new Scanner(detailsString); - while (s.hasNextLine()) { - String line = s.nextLine(); - if (!line.startsWith("\"advertised_bandwidth\":")) { - continue; - } - try { - advertisedBandwidth = (double) Integer.parseInt( - line.split(":")[1].replaceAll(",", "")); - } catch (NumberFormatException ex) { - /* Handle below. */ - } - break; - } - s.close(); - if (advertisedBandwidth >= 0.0) { - advertisedBandwidths.put(fingerprint, advertisedBandwidth); - totalAdvertisedBandwidth += advertisedBandwidth; - } - } - double consensusWeight = (double) relay.getConsensusWeight(); - consensusWeights.put(fingerprint, consensusWeight); - totalConsensusWeight += consensusWeight; - if (consensusContainsBandwidthWeights) { - double guardWeight = consensusWeight, - middleWeight = consensusWeight, - exitWeight = consensusWeight; - if (isGuard && isExit) { - guardWeight *= wgd; - middleWeight *= wmd; - exitWeight *= wed; - } else if (isGuard) { - guardWeight *= wgg; - middleWeight *= wmg; - exitWeight = 0.0; - } else if (isExit) { - guardWeight = 0.0; - middleWeight *= wme; - exitWeight *= wee; - } else { - guardWeight = 0.0; - middleWeight *= wmm; - exitWeight = 0.0; - } - guardWeights.put(fingerprint, guardWeight); - middleWeights.put(fingerprint, middleWeight); - exitWeights.put(fingerprint, exitWeight); - totalGuardWeight += guardWeight; - totalMiddleWeight += middleWeight; - totalExitWeight += exitWeight; - } - } - for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { - String fingerprint = e.getKey(); - NodeStatus relay = e.getValue(); - if (advertisedBandwidths.containsKey(fingerprint)) { - relay.setAdvertisedBandwidthFraction(advertisedBandwidths.get( - fingerprint) / totalAdvertisedBandwidth); - } - if (consensusWeights.containsKey(fingerprint)) { - relay.setConsensusWeightFraction(consensusWeights.get(fingerprint) - / totalConsensusWeight); - } - if (guardWeights.containsKey(fingerprint)) { - relay.setGuardProbability(guardWeights.get(fingerprint) - / totalGuardWeight); - } - if (middleWeights.containsKey(fingerprint)) { - relay.setMiddleProbability(middleWeights.get(fingerprint) - / totalMiddleWeight); - } - if (exitWeights.containsKey(fingerprint)) { - relay.setExitProbability(exitWeights.get(fingerprint) - / totalExitWeight); - } - } - } - - private void writeOutDetails() { - this.updateRelayDetailsFiles(); - this.updateBridgeDetailsFiles(); - } - - private static String escapeJSON(String s) { - return StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'"); - } - - private static String unescapeJSON(String s) { - return StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'")); - } - - private void updateRelayDetailsFiles() { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - for (Map.Entry<String, NodeStatus> relay : this.relays.entrySet()) { - String fingerprint = relay.getKey(); - - /* Generate network-status-specific part. */ - NodeStatus entry = relay.getValue(); - String nickname = entry.getNickname(); - String address = entry.getAddress(); - List<String> orAddresses = new ArrayList<String>(); - orAddresses.add(address + ":" + entry.getOrPort()); - orAddresses.addAll(entry.getOrAddressesAndPorts()); - StringBuilder orAddressesAndPortsBuilder = new StringBuilder(); - int addressesWritten = 0; - for (String orAddress : orAddresses) { - orAddressesAndPortsBuilder.append( - (addressesWritten++ > 0 ? "," : "") + "\"" - + orAddress.toLowerCase() + "\""); - } - String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); - String firstSeen = dateTimeFormat.format( - entry.getFirstSeenMillis()); - String lastChangedOrAddress = dateTimeFormat.format( - entry.getLastChangedOrAddress()); - String running = entry.getRunning() ? "true" : "false"; - int dirPort = entry.getDirPort(); - String countryCode = entry.getCountryCode(); - String latitude = entry.getLatitude(); - String longitude = entry.getLongitude(); - String countryName = entry.getCountryName(); - String regionName = entry.getRegionName(); - String cityName = entry.getCityName(); - String aSNumber = entry.getASNumber(); - String aSName = entry.getASName(); - long consensusWeight = entry.getConsensusWeight(); - String hostName = entry.getHostName(); - double advertisedBandwidthFraction = - entry.getAdvertisedBandwidthFraction(); - double consensusWeightFraction = entry.getConsensusWeightFraction(); - double guardProbability = entry.getGuardProbability(); - double middleProbability = entry.getMiddleProbability(); - double exitProbability = entry.getExitProbability(); - String defaultPolicy = entry.getDefaultPolicy(); - String portList = entry.getPortList(); - Boolean recommendedVersion = entry.getRecommendedVersion(); - StringBuilder sb = new StringBuilder(); - sb.append("{\"version\":1,\n" - + "\"nickname\":\"" + nickname + "\",\n" - + "\"fingerprint\":\"" + fingerprint + "\",\n" - + "\"or_addresses\":[" + orAddressesAndPortsBuilder.toString() - + "]"); - if (dirPort != 0) { - sb.append(",\n\"dir_address\":\"" + address + ":" + dirPort - + "\""); - } - sb.append(",\n\"last_seen\":\"" + lastSeen + "\""); - sb.append(",\n\"first_seen\":\"" + firstSeen + "\""); - sb.append(",\n\"last_changed_address_or_port\":\"" - + lastChangedOrAddress + "\""); - sb.append(",\n\"running\":" + running); - SortedSet<String> relayFlags = entry.getRelayFlags(); - if (!relayFlags.isEmpty()) { - sb.append(",\n\"flags\":["); - int written = 0; - for (String relayFlag : relayFlags) { - sb.append((written++ > 0 ? "," : "") + "\"" + relayFlag + "\""); - } - sb.append("]"); - } - if (countryCode != null) { - sb.append(",\n\"country\":\"" + countryCode + "\""); - } - if (latitude != null) { - sb.append(",\n\"latitude\":" + latitude); - } - if (longitude != null) { - sb.append(",\n\"longitude\":" + longitude); - } - if (countryName != null) { - sb.append(",\n\"country_name\":\"" - + escapeJSON(countryName) + "\""); - } - if (regionName != null) { - sb.append(",\n\"region_name\":\"" - + escapeJSON(regionName) + "\""); - } - if (cityName != null) { - sb.append(",\n\"city_name\":\"" - + escapeJSON(cityName) + "\""); - } - if (aSNumber != null) { - sb.append(",\n\"as_number\":\"" - + escapeJSON(aSNumber) + "\""); - } - if (aSName != null) { - sb.append(",\n\"as_name\":\"" - + escapeJSON(aSName) + "\""); - } - if (consensusWeight >= 0L) { - sb.append(",\n\"consensus_weight\":" - + String.valueOf(consensusWeight)); - } - if (hostName != null) { - sb.append(",\n\"host_name\":\"" - + escapeJSON(hostName) + "\""); - } - if (advertisedBandwidthFraction >= 0.0) { - sb.append(String.format( - ",\n\"advertised_bandwidth_fraction\":%.9f", - advertisedBandwidthFraction)); - } - if (consensusWeightFraction >= 0.0) { - sb.append(String.format(",\n\"consensus_weight_fraction\":%.9f", - consensusWeightFraction)); - } - if (guardProbability >= 0.0) { - sb.append(String.format(",\n\"guard_probability\":%.9f", - guardProbability)); - } - if (middleProbability >= 0.0) { - sb.append(String.format(",\n\"middle_probability\":%.9f", - middleProbability)); - } - if (exitProbability >= 0.0) { - sb.append(String.format(",\n\"exit_probability\":%.9f", - exitProbability)); - } - if (defaultPolicy != null && (defaultPolicy.equals("accept") || - defaultPolicy.equals("reject")) && portList != null) { - sb.append(",\n\"exit_policy_summary\":{\"" + defaultPolicy - + "\":["); - int portsWritten = 0; - for (String portOrPortRange : portList.split(",")) { - sb.append((portsWritten++ > 0 ? "," : "") - + "\"" + portOrPortRange + "\""); - } - sb.append("]}"); - } - if (recommendedVersion != null) { - sb.append(",\n\"recommended_version\":" + (recommendedVersion ? - "true" : "false")); - } - - /* Add exit addresses if at least one of them is distinct from the - * onion-routing addresses. */ - if (exitListEntries.containsKey(fingerprint)) { - for (ExitListEntry exitListEntry : - exitListEntries.get(fingerprint)) { - entry.addExitAddress(exitListEntry.getExitAddress()); - } - } - if (!entry.getExitAddresses().isEmpty()) { - sb.append(",\n\"exit_addresses\":["); - int written = 0; - for (String exitAddress : entry.getExitAddresses()) { - sb.append((written++ > 0 ? "," : "") + "\"" - + exitAddress.toLowerCase() + "\""); - } - sb.append("]"); - } - - /* Append descriptor-specific part from details status file, and - * update contact in node status. */ - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); - if (detailsStatus != null && - detailsStatus.documentString.length() > 0) { - sb.append(",\n" + detailsStatus.documentString); - String contact = null; - Scanner s = new Scanner(detailsStatus.documentString); - while (s.hasNextLine()) { - String line = s.nextLine(); - if (!line.startsWith("\"contact\":")) { - continue; - } - int start = "\"contact\":\"".length(), end = line.length() - 1; - if (line.endsWith(",")) { - end--; - } - contact = unescapeJSON(line.substring(start, end)); - break; - } - s.close(); - entry.setContact(contact); - } - - /* Finish details string. */ - sb.append("\n}\n"); - - /* Write details file to disk. */ - DetailsDocument detailsDocument = new DetailsDocument(); - detailsDocument.documentString = sb.toString(); - this.documentStore.store(detailsDocument, fingerprint); - } - } - - private void updateBridgeDetailsFiles() { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - for (Map.Entry<String, NodeStatus> bridge : this.bridges.entrySet()) { - String fingerprint = bridge.getKey(); - - /* Generate network-status-specific part. */ - NodeStatus entry = bridge.getValue(); - String nickname = entry.getNickname(); - String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); - String firstSeen = dateTimeFormat.format( - entry.getFirstSeenMillis()); - String running = entry.getRunning() ? "true" : "false"; - String address = entry.getAddress(); - List<String> orAddresses = new ArrayList<String>(); - orAddresses.add(address + ":" + entry.getOrPort()); - orAddresses.addAll(entry.getOrAddressesAndPorts()); - StringBuilder orAddressesAndPortsBuilder = new StringBuilder(); - int addressesWritten = 0; - for (String orAddress : orAddresses) { - orAddressesAndPortsBuilder.append( - (addressesWritten++ > 0 ? "," : "") + "\"" - + orAddress.toLowerCase() + "\""); - } - StringBuilder sb = new StringBuilder(); - sb.append("{\"version\":1,\n" - + "\"nickname\":\"" + nickname + "\",\n" - + "\"hashed_fingerprint\":\"" + fingerprint + "\",\n" - + "\"or_addresses\":[" + orAddressesAndPortsBuilder.toString() - + "],\n\"last_seen\":\"" + lastSeen + "\",\n\"first_seen\":\"" - + firstSeen + "\",\n\"running\":" + running); - - SortedSet<String> relayFlags = entry.getRelayFlags(); - if (!relayFlags.isEmpty()) { - sb.append(",\n\"flags\":["); - int written = 0; - for (String relayFlag : relayFlags) { - sb.append((written++ > 0 ? "," : "") + "\"" + relayFlag + "\""); - } - sb.append("]"); - } - - /* Append descriptor-specific part from details status file. */ - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); - if (detailsStatus != null && - detailsStatus.documentString.length() > 0) { - sb.append(",\n" + detailsStatus.documentString); - } - - /* Finish details string. */ - sb.append("\n}\n"); - - /* Write details file to disk. */ - DetailsDocument detailsDocument = new DetailsDocument(); - detailsDocument.documentString = sb.toString(); - this.documentStore.store(detailsDocument, fingerprint); - } - } - - public String getStatsString() { - StringBuilder sb = new StringBuilder(); - sb.append(" " + Logger.formatDecimalNumber( - relayConsensusesProcessed) + " relay consensuses processed\n"); - sb.append(" " + Logger.formatDecimalNumber(bridgeStatusesProcessed) - + " bridge statuses processed\n"); - return sb.toString(); - } -} - diff --git a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java new file mode 100644 index 0000000..3c67aea --- /dev/null +++ b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java @@ -0,0 +1,634 @@ +/* Copyright 2011, 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Scanner; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.commons.lang.StringEscapeUtils; +import org.torproject.descriptor.BridgeNetworkStatus; +import org.torproject.descriptor.BridgePoolAssignment; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.NetworkStatusEntry; +import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.descriptor.ServerDescriptor; +import org.torproject.onionoo.LookupService.LookupResult; + +public class NodeDetailsStatusUpdater implements DescriptorListener, + StatusUpdater { + + private DescriptorSource descriptorSource; + + private ReverseDomainNameResolver reverseDomainNameResolver; + + private LookupService lookupService; + + private DocumentStore documentStore; + + private long now; + + private SortedMap<String, NodeStatus> knownNodes = + new TreeMap<String, NodeStatus>(); + + private SortedMap<String, NodeStatus> relays; + + private SortedMap<String, NodeStatus> bridges; + + private long relaysLastValidAfterMillis = -1L; + + private long bridgesLastPublishedMillis = -1L; + + private SortedMap<String, Integer> lastBandwidthWeights = null; + + private int relayConsensusesProcessed = 0, bridgeStatusesProcessed = 0; + + public NodeDetailsStatusUpdater(DescriptorSource descriptorSource, + ReverseDomainNameResolver reverseDomainNameResolver, + LookupService lookupService, DocumentStore documentStore, + Time time) { + this.descriptorSource = descriptorSource; + this.reverseDomainNameResolver = reverseDomainNameResolver; + this.lookupService = lookupService; + this.documentStore = documentStore; + this.now = time.currentTimeMillis(); + this.registerDescriptorListeners(); + } + + private void registerDescriptorListeners() { + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.RELAY_SERVER_DESCRIPTORS); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_STATUSES); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_SERVER_DESCRIPTORS); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_POOL_ASSIGNMENTS); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.EXIT_LISTS); + } + + public void processDescriptor(Descriptor descriptor, boolean relay) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + this.processRelayNetworkStatusConsensus( + (RelayNetworkStatusConsensus) descriptor); + } else if (descriptor instanceof ServerDescriptor && relay) { + this.processRelayServerDescriptor((ServerDescriptor) descriptor); + } else if (descriptor instanceof BridgeNetworkStatus) { + this.processBridgeNetworkStatus((BridgeNetworkStatus) descriptor); + } else if (descriptor instanceof ServerDescriptor && !relay) { + this.processBridgeServerDescriptor((ServerDescriptor) descriptor); + } else if (descriptor instanceof BridgePoolAssignment) { + this.processBridgePoolAssignment((BridgePoolAssignment) descriptor); + } + } + + public void updateStatuses() { + this.readStatusSummary(); + Logger.printStatusTime("Read status summary"); + this.setCurrentNodes(); + Logger.printStatusTime("Set current node fingerprints"); + this.startReverseDomainNameLookups(); + Logger.printStatusTime("Started reverse domain name lookups"); + this.lookUpCitiesAndASes(); + Logger.printStatusTime("Looked up cities and ASes"); + this.setRunningBits(); + Logger.printStatusTime("Set running bits"); + this.calculatePathSelectionProbabilities(); + Logger.printStatusTime("Calculated path selection probabilities"); + this.finishReverseDomainNameLookups(); + Logger.printStatusTime("Finished reverse domain name lookups"); + this.writeStatusSummary(); + Logger.printStatusTime("Wrote status summary"); + /* TODO Does anything break if we take the following out? + * Like, does DocumentStore make sure there's a status/summary with + * all node statuses and an out/summary with only recent ones? + this.writeOutSummary(); + Logger.printStatusTime("Wrote out summary");*/ + } + + private void processRelayNetworkStatusConsensus( + RelayNetworkStatusConsensus consensus) { + long validAfterMillis = consensus.getValidAfterMillis(); + if (validAfterMillis > this.relaysLastValidAfterMillis) { + this.relaysLastValidAfterMillis = validAfterMillis; + } + Set<String> recommendedVersions = null; + if (consensus.getRecommendedServerVersions() != null) { + recommendedVersions = new HashSet<String>(); + for (String recommendedVersion : + consensus.getRecommendedServerVersions()) { + recommendedVersions.add("Tor " + recommendedVersion); + } + } + for (NetworkStatusEntry entry : + consensus.getStatusEntries().values()) { + String nickname = entry.getNickname(); + String fingerprint = entry.getFingerprint(); + String address = entry.getAddress(); + SortedSet<String> orAddressesAndPorts = new TreeSet<String>( + entry.getOrAddresses()); + int orPort = entry.getOrPort(); + int dirPort = entry.getDirPort(); + SortedSet<String> relayFlags = entry.getFlags(); + long consensusWeight = entry.getBandwidth(); + String defaultPolicy = entry.getDefaultPolicy(); + String portList = entry.getPortList(); + Boolean recommendedVersion = (recommendedVersions == null || + entry.getVersion() == null) ? null : + recommendedVersions.contains(entry.getVersion()); + NodeStatus newNodeStatus = new NodeStatus(true, nickname, + fingerprint, address, orAddressesAndPorts, null, + validAfterMillis, orPort, dirPort, relayFlags, consensusWeight, + null, null, -1L, defaultPolicy, portList, validAfterMillis, + validAfterMillis, null, null, recommendedVersion); + if (this.knownNodes.containsKey(fingerprint)) { + this.knownNodes.get(fingerprint).update(newNodeStatus); + } else { + this.knownNodes.put(fingerprint, newNodeStatus); + } + } + this.relayConsensusesProcessed++; + if (this.relaysLastValidAfterMillis == validAfterMillis) { + this.lastBandwidthWeights = consensus.getBandwidthWeights(); + } + } + + private void processBridgeNetworkStatus(BridgeNetworkStatus status) { + long publishedMillis = status.getPublishedMillis(); + if (publishedMillis > this.bridgesLastPublishedMillis) { + this.bridgesLastPublishedMillis = publishedMillis; + } + for (NetworkStatusEntry entry : status.getStatusEntries().values()) { + String nickname = entry.getNickname(); + String fingerprint = entry.getFingerprint(); + String address = entry.getAddress(); + SortedSet<String> orAddressesAndPorts = new TreeSet<String>( + entry.getOrAddresses()); + int orPort = entry.getOrPort(); + int dirPort = entry.getDirPort(); + SortedSet<String> relayFlags = entry.getFlags(); + NodeStatus newNodeStatus = new NodeStatus(false, nickname, + fingerprint, address, orAddressesAndPorts, null, + publishedMillis, orPort, dirPort, relayFlags, -1L, "??", null, + -1L, null, null, publishedMillis, -1L, null, null, null); + if (this.knownNodes.containsKey(fingerprint)) { + this.knownNodes.get(fingerprint).update(newNodeStatus); + } else { + this.knownNodes.put(fingerprint, newNodeStatus); + } + } + this.bridgeStatusesProcessed++; + } + + private void readStatusSummary() { + SortedSet<String> fingerprints = this.documentStore.list( + NodeStatus.class, true); + for (String fingerprint : fingerprints) { + NodeStatus node = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (node.isRelay()) { + this.relaysLastValidAfterMillis = Math.max( + this.relaysLastValidAfterMillis, node.getLastSeenMillis()); + } else { + this.bridgesLastPublishedMillis = Math.max( + this.bridgesLastPublishedMillis, node.getLastSeenMillis()); + } + if (this.knownNodes.containsKey(fingerprint)) { + this.knownNodes.get(fingerprint).update(node); + } else { + this.knownNodes.put(fingerprint, node); + } + } + } + + private void setRunningBits() { + for (NodeStatus node : this.knownNodes.values()) { + if (node.isRelay() && node.getRelayFlags().contains("Running") && + node.getLastSeenMillis() == this.relaysLastValidAfterMillis) { + node.setRunning(true); + } + if (!node.isRelay() && node.getRelayFlags().contains("Running") && + node.getLastSeenMillis() == this.bridgesLastPublishedMillis) { + node.setRunning(true); + } + } + } + + private void lookUpCitiesAndASes() { + SortedSet<String> addressStrings = new TreeSet<String>(); + for (NodeStatus node : this.knownNodes.values()) { + if (node.isRelay()) { + addressStrings.add(node.getAddress()); + } + } + if (addressStrings.isEmpty()) { + System.err.println("No relay IP addresses to resolve to cities or " + + "ASN."); + return; + } + SortedMap<String, LookupResult> lookupResults = + this.lookupService.lookup(addressStrings); + for (NodeStatus node : knownNodes.values()) { + if (!node.isRelay()) { + continue; + } + String addressString = node.getAddress(); + if (lookupResults.containsKey(addressString)) { + LookupResult lookupResult = lookupResults.get(addressString); + node.setCountryCode(lookupResult.countryCode); + node.setCountryName(lookupResult.countryName); + node.setRegionName(lookupResult.regionName); + node.setCityName(lookupResult.cityName); + node.setLatitude(lookupResult.latitude); + node.setLongitude(lookupResult.longitude); + node.setASNumber(lookupResult.aSNumber); + node.setASName(lookupResult.aSName); + } + } + } + + private void writeStatusSummary() { + this.writeSummary(true); + } + + private void writeSummary(boolean includeArchive) { + SortedMap<String, NodeStatus> nodes = includeArchive + ? this.knownNodes : this.getCurrentNodes(); + for (Map.Entry<String, NodeStatus> e : nodes.entrySet()) { + this.documentStore.store(e.getValue(), e.getKey()); + } + } + + private SortedMap<String, NodeStatus> getCurrentNodes() { + long cutoff = Math.max(this.relaysLastValidAfterMillis, + this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L; + SortedMap<String, NodeStatus> currentNodes = + new TreeMap<String, NodeStatus>(); + for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { + if (e.getValue().getLastSeenMillis() >= cutoff) { + currentNodes.put(e.getKey(), e.getValue()); + } + } + return currentNodes; + } + + private void processRelayServerDescriptor( + ServerDescriptor descriptor) { + String fingerprint = descriptor.getFingerprint(); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, false, fingerprint); + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setLenient(false); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + String publishedDateTime = + dateTimeFormat.format(descriptor.getPublishedMillis()); + if (detailsStatus != null) { + String detailsString = detailsStatus.documentString; + String descPublishedLine = "\"desc_published\":\"" + + publishedDateTime + "\","; + Scanner s = new Scanner(detailsString); + while (s.hasNextLine()) { + String line = s.nextLine(); + if (line.startsWith("\"desc_published\":\"")) { + if (descPublishedLine.compareTo(line) < 0) { + return; + } else { + break; + } + } + } + s.close(); + } + StringBuilder sb = new StringBuilder(); + String lastRestartedString = dateTimeFormat.format( + descriptor.getPublishedMillis() - descriptor.getUptime() * 1000L); + int bandwidthRate = descriptor.getBandwidthRate(); + int bandwidthBurst = descriptor.getBandwidthBurst(); + int observedBandwidth = descriptor.getBandwidthObserved(); + int advertisedBandwidth = Math.min(bandwidthRate, + Math.min(bandwidthBurst, observedBandwidth)); + sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n" + + "\"last_restarted\":\"" + lastRestartedString + "\",\n" + + "\"bandwidth_rate\":" + bandwidthRate + ",\n" + + "\"bandwidth_burst\":" + bandwidthBurst + ",\n" + + "\"observed_bandwidth\":" + observedBandwidth + ",\n" + + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n" + + "\"exit_policy\":["); + int written = 0; + for (String exitPolicyLine : descriptor.getExitPolicyLines()) { + sb.append((written++ > 0 ? "," : "") + "\n \"" + exitPolicyLine + + "\""); + } + sb.append("\n]"); + if (descriptor.getContact() != null) { + sb.append(",\n\"contact\":\"" + + escapeJSON(descriptor.getContact()) + "\""); + } + if (descriptor.getPlatform() != null) { + sb.append(",\n\"platform\":\"" + + escapeJSON(descriptor.getPlatform()) + "\""); + } + if (descriptor.getFamilyEntries() != null) { + sb.append(",\n\"family\":["); + written = 0; + for (String familyEntry : descriptor.getFamilyEntries()) { + sb.append((written++ > 0 ? "," : "") + "\n \"" + familyEntry + + "\""); + } + sb.append("\n]"); + } + if (descriptor.getIpv6DefaultPolicy() != null && + (descriptor.getIpv6DefaultPolicy().equals("accept") || + descriptor.getIpv6DefaultPolicy().equals("reject")) && + descriptor.getIpv6PortList() != null) { + sb.append(",\n\"exit_policy_v6_summary\":{\"" + + descriptor.getIpv6DefaultPolicy() + "\":["); + int portsWritten = 0; + for (String portOrPortRange : + descriptor.getIpv6PortList().split(",")) { + sb.append((portsWritten++ > 0 ? "," : "") + "\"" + portOrPortRange + + "\""); + } + sb.append("]}"); + } + if (descriptor.isHibernating()) { + sb.append(",\n\"hibernating\":true"); + } + detailsStatus = new DetailsStatus(); + detailsStatus.documentString = sb.toString(); + this.documentStore.store(detailsStatus, fingerprint); + } + + private void processBridgeServerDescriptor( + ServerDescriptor descriptor) { + String fingerprint = descriptor.getFingerprint(); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, false, fingerprint); + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setLenient(false); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + String publishedDateTime = + dateTimeFormat.format(descriptor.getPublishedMillis()); + String poolAssignmentLine = null; + if (detailsStatus != null) { + String detailsString = detailsStatus.documentString; + String descPublishedLine = "\"desc_published\":\"" + + publishedDateTime + "\","; + Scanner s = new Scanner(detailsString); + while (s.hasNextLine()) { + String line = s.nextLine(); + if (line.startsWith("\"pool_assignment\":")) { + poolAssignmentLine = line; + } else if (line.startsWith("\"desc_published\":") && + descPublishedLine.compareTo(line) < 0) { + return; + } + } + s.close(); + } + StringBuilder sb = new StringBuilder(); + String lastRestartedString = dateTimeFormat.format( + descriptor.getPublishedMillis() - descriptor.getUptime() * 1000L); + int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(), + Math.min(descriptor.getBandwidthBurst(), + descriptor.getBandwidthObserved())); + sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n" + + "\"last_restarted\":\"" + lastRestartedString + "\",\n" + + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n" + + "\"platform\":\"" + escapeJSON(descriptor.getPlatform()) + + "\""); + if (poolAssignmentLine != null) { + sb.append(",\n" + poolAssignmentLine); + } + detailsStatus = new DetailsStatus(); + detailsStatus.documentString = sb.toString(); + this.documentStore.store(detailsStatus, fingerprint); + } + + private static String escapeJSON(String s) { + return StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'"); + } + + private void processBridgePoolAssignment( + BridgePoolAssignment bridgePoolAssignment) { + for (Map.Entry<String, String> e : + bridgePoolAssignment.getEntries().entrySet()) { + String fingerprint = e.getKey(); + String details = e.getValue(); + StringBuilder sb = new StringBuilder(); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, false, fingerprint); + if (detailsStatus != null) { + String detailsString = detailsStatus.documentString; + Scanner s = new Scanner(detailsString); + int linesWritten = 0; + boolean endsWithComma = false; + while (s.hasNextLine()) { + String line = s.nextLine(); + if (!line.startsWith("\"pool_assignment\":")) { + sb.append((linesWritten++ > 0 ? "\n" : "") + line); + endsWithComma = line.endsWith(","); + } + } + s.close(); + if (sb.length() > 0) { + sb.append((endsWithComma ? "" : ",") + "\n"); + } + } + sb.append("\"pool_assignment\":\"" + details + "\""); + detailsStatus = new DetailsStatus(); + detailsStatus.documentString = sb.toString(); + this.documentStore.store(detailsStatus, fingerprint); + } + } + + private void setCurrentNodes() { + SortedMap<String, NodeStatus> currentNodes = this.getCurrentNodes(); + this.relays = new TreeMap<String, NodeStatus>(); + this.bridges = new TreeMap<String, NodeStatus>(); + for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) { + if (e.getValue().isRelay()) { + this.relays.put(e.getKey(), e.getValue()); + } else { + this.bridges.put(e.getKey(), e.getValue()); + } + } + } + + private void startReverseDomainNameLookups() { + Map<String, Long> addressLastLookupTimes = + new HashMap<String, Long>(); + for (NodeStatus relay : relays.values()) { + addressLastLookupTimes.put(relay.getAddress(), + relay.getLastRdnsLookup()); + } + this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes); + this.reverseDomainNameResolver.startReverseDomainNameLookups(); + } + + private void finishReverseDomainNameLookups() { + this.reverseDomainNameResolver.finishReverseDomainNameLookups(); + Map<String, String> lookupResults = + this.reverseDomainNameResolver.getLookupResults(); + long startedRdnsLookups = + this.reverseDomainNameResolver.getLookupStartMillis(); + for (NodeStatus relay : relays.values()) { + if (lookupResults.containsKey(relay.getAddress())) { + relay.setHostName(lookupResults.get(relay.getAddress())); + relay.setLastRdnsLookup(startedRdnsLookups); + } + } + } + + private void calculatePathSelectionProbabilities() { + boolean consensusContainsBandwidthWeights = false; + double wgg = 0.0, wgd = 0.0, wmg = 0.0, wmm = 0.0, wme = 0.0, + wmd = 0.0, wee = 0.0, wed = 0.0; + if (this.lastBandwidthWeights != null) { + SortedSet<String> weightKeys = new TreeSet<String>(Arrays.asList( + "Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(","))); + weightKeys.removeAll(this.lastBandwidthWeights.keySet()); + if (weightKeys.isEmpty()) { + consensusContainsBandwidthWeights = true; + wgg = ((double) this.lastBandwidthWeights.get("Wgg")) / 10000.0; + wgd = ((double) this.lastBandwidthWeights.get("Wgd")) / 10000.0; + wmg = ((double) this.lastBandwidthWeights.get("Wmg")) / 10000.0; + wmm = ((double) this.lastBandwidthWeights.get("Wmm")) / 10000.0; + wme = ((double) this.lastBandwidthWeights.get("Wme")) / 10000.0; + wmd = ((double) this.lastBandwidthWeights.get("Wmd")) / 10000.0; + wee = ((double) this.lastBandwidthWeights.get("Wee")) / 10000.0; + wed = ((double) this.lastBandwidthWeights.get("Wed")) / 10000.0; + } + } else { + System.err.println("Could not determine most recent Wxx parameter " + + "values, probably because we didn't parse a consensus in " + + "this execution. All relays' guard/middle/exit weights are " + + "going to be 0.0."); + } + SortedMap<String, Double> + advertisedBandwidths = new TreeMap<String, Double>(), + consensusWeights = new TreeMap<String, Double>(), + guardWeights = new TreeMap<String, Double>(), + middleWeights = new TreeMap<String, Double>(), + exitWeights = new TreeMap<String, Double>(); + double totalAdvertisedBandwidth = 0.0; + double totalConsensusWeight = 0.0; + double totalGuardWeight = 0.0; + double totalMiddleWeight = 0.0; + double totalExitWeight = 0.0; + for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { + String fingerprint = e.getKey(); + NodeStatus relay = e.getValue(); + if (!relay.getRunning()) { + continue; + } + boolean isExit = relay.getRelayFlags().contains("Exit") && + !relay.getRelayFlags().contains("BadExit"); + boolean isGuard = relay.getRelayFlags().contains("Guard"); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, false, fingerprint); + if (detailsStatus != null) { + double advertisedBandwidth = -1.0; + String detailsString = detailsStatus.documentString; + Scanner s = new Scanner(detailsString); + while (s.hasNextLine()) { + String line = s.nextLine(); + if (!line.startsWith("\"advertised_bandwidth\":")) { + continue; + } + try { + advertisedBandwidth = (double) Integer.parseInt( + line.split(":")[1].replaceAll(",", "")); + } catch (NumberFormatException ex) { + /* Handle below. */ + } + break; + } + s.close(); + if (advertisedBandwidth >= 0.0) { + advertisedBandwidths.put(fingerprint, advertisedBandwidth); + totalAdvertisedBandwidth += advertisedBandwidth; + } + } + double consensusWeight = (double) relay.getConsensusWeight(); + consensusWeights.put(fingerprint, consensusWeight); + totalConsensusWeight += consensusWeight; + if (consensusContainsBandwidthWeights) { + double guardWeight = consensusWeight, + middleWeight = consensusWeight, + exitWeight = consensusWeight; + if (isGuard && isExit) { + guardWeight *= wgd; + middleWeight *= wmd; + exitWeight *= wed; + } else if (isGuard) { + guardWeight *= wgg; + middleWeight *= wmg; + exitWeight = 0.0; + } else if (isExit) { + guardWeight = 0.0; + middleWeight *= wme; + exitWeight *= wee; + } else { + guardWeight = 0.0; + middleWeight *= wmm; + exitWeight = 0.0; + } + guardWeights.put(fingerprint, guardWeight); + middleWeights.put(fingerprint, middleWeight); + exitWeights.put(fingerprint, exitWeight); + totalGuardWeight += guardWeight; + totalMiddleWeight += middleWeight; + totalExitWeight += exitWeight; + } + } + for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { + String fingerprint = e.getKey(); + NodeStatus relay = e.getValue(); + if (advertisedBandwidths.containsKey(fingerprint)) { + relay.setAdvertisedBandwidthFraction(advertisedBandwidths.get( + fingerprint) / totalAdvertisedBandwidth); + } + if (consensusWeights.containsKey(fingerprint)) { + relay.setConsensusWeightFraction(consensusWeights.get(fingerprint) + / totalConsensusWeight); + } + if (guardWeights.containsKey(fingerprint)) { + relay.setGuardProbability(guardWeights.get(fingerprint) + / totalGuardWeight); + } + if (middleWeights.containsKey(fingerprint)) { + relay.setMiddleProbability(middleWeights.get(fingerprint) + / totalMiddleWeight); + } + if (exitWeights.containsKey(fingerprint)) { + relay.setExitProbability(exitWeights.get(fingerprint) + / totalExitWeight); + } + } + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + Logger.formatDecimalNumber( + relayConsensusesProcessed) + " relay consensuses processed\n"); + sb.append(" " + Logger.formatDecimalNumber(bridgeStatusesProcessed) + + " bridge statuses processed\n"); + return sb.toString(); + } +} +

1 0

[onionoo/master] Simplify weights status updater by taking out threading code.
by karsten＠torproject.org 11 Apr '14

11 Apr '14

commit 97fbdee60d66a627fa9b378fc20767864bf275cc Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Mar 14 12:10:39 2014 +0100 Simplify weights status updater by taking out threading code. The threading code was never used, because it's broken. No reason to keep it. --- .../torproject/onionoo/WeightsStatusUpdater.java | 69 +++++--------------- 1 file changed, 15 insertions(+), 54 deletions(-) diff --git a/src/org/torproject/onionoo/WeightsStatusUpdater.java b/src/org/torproject/onionoo/WeightsStatusUpdater.java index 5e890ef..79296d3 100644 --- a/src/org/torproject/onionoo/WeightsStatusUpdater.java +++ b/src/org/torproject/onionoo/WeightsStatusUpdater.java @@ -3,11 +3,9 @@ package org.torproject.onionoo; import java.text.SimpleDateFormat; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; @@ -66,6 +64,7 @@ public class WeightsStatusUpdater implements DescriptorListener, private void processRelayNetworkConsensus( RelayNetworkStatusConsensus consensus) { + // TODO This does not scale for bulk imports. this.consensuses.add(consensus); } @@ -107,62 +106,24 @@ public class WeightsStatusUpdater implements DescriptorListener, } } - // TODO Use 4 workers once threading problems are solved. - private static final int HISTORY_UPDATER_WORKERS_NUM = 1; private void updateWeightsHistory(long validAfterMillis, long freshUntilMillis, SortedMap<String, double[]> pathSelectionWeights) { - List<HistoryUpdateWorker> historyUpdateWorkers = - new ArrayList<HistoryUpdateWorker>(); - for (int i = 0; i < HISTORY_UPDATER_WORKERS_NUM; i++) { - HistoryUpdateWorker historyUpdateWorker = - new HistoryUpdateWorker(validAfterMillis, freshUntilMillis, - pathSelectionWeights, this); - historyUpdateWorkers.add(historyUpdateWorker); - historyUpdateWorker.setDaemon(true); - historyUpdateWorker.start(); - } - for (HistoryUpdateWorker historyUpdateWorker : historyUpdateWorkers) { - try { - historyUpdateWorker.join(); - } catch (InterruptedException e) { - /* This is not something that we can take care of. Just leave the - * worker thread alone. */ - } - } - } - - private class HistoryUpdateWorker extends Thread { - private long validAfterMillis; - private long freshUntilMillis; - private SortedMap<String, double[]> pathSelectionWeights; - private WeightsStatusUpdater parent; - public HistoryUpdateWorker(long validAfterMillis, - long freshUntilMillis, - SortedMap<String, double[]> pathSelectionWeights, - WeightsStatusUpdater parent) { - this.validAfterMillis = validAfterMillis; - this.freshUntilMillis = freshUntilMillis; - this.pathSelectionWeights = pathSelectionWeights; - this.parent = parent; - } - public void run() { - String fingerprint = null; - double[] weights = null; - do { - fingerprint = null; - synchronized (pathSelectionWeights) { - if (!pathSelectionWeights.isEmpty()) { - fingerprint = pathSelectionWeights.firstKey(); - weights = pathSelectionWeights.remove(fingerprint); - } - } - if (fingerprint != null) { - this.parent.addToHistory(fingerprint, this.validAfterMillis, - this.freshUntilMillis, weights); + String fingerprint = null; + double[] weights = null; + do { + fingerprint = null; + synchronized (pathSelectionWeights) { + if (!pathSelectionWeights.isEmpty()) { + fingerprint = pathSelectionWeights.firstKey(); + weights = pathSelectionWeights.remove(fingerprint); } - } while (fingerprint != null); - } + } + if (fingerprint != null) { + this.addToHistory(fingerprint, validAfterMillis, + freshUntilMillis, weights); + } + } while (fingerprint != null); } private SortedMap<String, double[]> calculatePathSelectionProbabilities(

1 0

[onionoo/master] Split weights data writer into two classes.
by karsten＠torproject.org 11 Apr '14

11 Apr '14

commit beb3a4e7e9983d42d4044d11c6589e6bcc58edaa Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Apr 10 20:09:56 2014 +0200 Split weights data writer into two classes. --- src/org/torproject/onionoo/Main.java | 10 +- src/org/torproject/onionoo/WeightsDataWriter.java | 582 -------------------- .../torproject/onionoo/WeightsDocumentWriter.java | 222 ++++++++ .../torproject/onionoo/WeightsStatusUpdater.java | 394 +++++++++++++ 4 files changed, 622 insertions(+), 586 deletions(-) diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index af33124..434d90c 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -34,14 +34,16 @@ public class Main { Logger.printStatusTime("Initialized node data writer"); BandwidthDataWriter bdw = new BandwidthDataWriter(dso, ds, t); Logger.printStatusTime("Initialized bandwidth data writer"); - WeightsDataWriter wdw = new WeightsDataWriter(dso, ds, t); - Logger.printStatusTime("Initialized weights data writer"); + WeightsStatusUpdater wsu = new WeightsStatusUpdater(dso, ds, t); + Logger.printStatusTime("Initialized weights status updater"); ClientsStatusUpdater csu = new ClientsStatusUpdater(dso, ds, t); Logger.printStatusTime("Initialized clients status updater"); UptimeStatusUpdater usu = new UptimeStatusUpdater(dso, ds); Logger.printStatusTime("Initialized uptime status updater"); - StatusUpdater[] sus = new StatusUpdater[] { ndw, bdw, wdw, csu, usu }; + StatusUpdater[] sus = new StatusUpdater[] { ndw, bdw, wsu, csu, usu }; + WeightsDocumentWriter wdw = new WeightsDocumentWriter(dso, ds, t); + Logger.printStatusTime("Initialized weights document writer"); ClientsDocumentWriter cdw = new ClientsDocumentWriter(dso, ds, t); Logger.printStatusTime("Initialized clients document writer"); UptimeDocumentWriter udw = new UptimeDocumentWriter(dso, ds, t); @@ -93,7 +95,7 @@ public class Main { } /* TODO Print status updater statistics for *all* status updaters once * all data writers have been separated. */ - for (DocumentWriter dw : new DocumentWriter[] { cdw, udw }) { + for (DocumentWriter dw : new DocumentWriter[] { wdw, cdw, udw }) { String statsString = dw.getStatsString(); if (statsString != null) { Logger.printStatistics(dw.getClass().getSimpleName(), diff --git a/src/org/torproject/onionoo/WeightsDataWriter.java b/src/org/torproject/onionoo/WeightsDataWriter.java deleted file mode 100644 index 0d7b815..0000000 --- a/src/org/torproject/onionoo/WeightsDataWriter.java +++ /dev/null @@ -1,582 +0,0 @@ -/* Copyright 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TimeZone; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.NetworkStatusEntry; -import org.torproject.descriptor.RelayNetworkStatusConsensus; -import org.torproject.descriptor.ServerDescriptor; - -public class WeightsDataWriter implements DescriptorListener, - StatusUpdater, FingerprintListener, DocumentWriter { - - private DescriptorSource descriptorSource; - - private DocumentStore documentStore; - - private long now; - - public WeightsDataWriter(DescriptorSource descriptorSource, - DocumentStore documentStore, Time time) { - this.descriptorSource = descriptorSource; - this.documentStore = documentStore; - this.now = time.currentTimeMillis(); - this.registerDescriptorListeners(); - this.registerFingerprintListeners(); - } - - private void registerDescriptorListeners() { - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.RELAY_SERVER_DESCRIPTORS); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_SERVER_DESCRIPTORS); - } - - public void processDescriptor(Descriptor descriptor, boolean relay) { - if (descriptor instanceof ServerDescriptor) { - this.processRelayServerDescriptor((ServerDescriptor) descriptor); - } else if (descriptor instanceof RelayNetworkStatusConsensus) { - this.processRelayNetworkConsensus( - (RelayNetworkStatusConsensus) descriptor); - } - } - - public void updateStatuses() { - this.updateWeightsHistories(); - Logger.printStatusTime("Updated weights histories"); - this.updateWeightsStatuses(); - Logger.printStatusTime("Updated weights status files"); - } - - public void writeDocuments() { - this.writeWeightsDataFiles(); - Logger.printStatusTime("Wrote weights document files"); - } - - private Set<RelayNetworkStatusConsensus> consensuses = - new HashSet<RelayNetworkStatusConsensus>(); - - private void processRelayNetworkConsensus( - RelayNetworkStatusConsensus consensus) { - this.consensuses.add(consensus); - } - - private Set<String> updateWeightsStatuses = new HashSet<String>(); - - private Set<String> updateWeightsDocuments = new HashSet<String>(); - - private Map<String, Set<String>> descriptorDigestsByFingerprint = - new HashMap<String, Set<String>>(); - - private Map<String, Integer> advertisedBandwidths = - new HashMap<String, Integer>(); - - private void processRelayServerDescriptor( - ServerDescriptor serverDescriptor) { - String digest = serverDescriptor.getServerDescriptorDigest(). - toUpperCase(); - int advertisedBandwidth = Math.min(Math.min( - serverDescriptor.getBandwidthBurst(), - serverDescriptor.getBandwidthObserved()), - serverDescriptor.getBandwidthRate()); - this.advertisedBandwidths.put(digest, advertisedBandwidth); - String fingerprint = serverDescriptor.getFingerprint(); - this.updateWeightsStatuses.add(fingerprint); - if (!this.descriptorDigestsByFingerprint.containsKey( - fingerprint)) { - this.descriptorDigestsByFingerprint.put(fingerprint, - new HashSet<String>()); - } - this.descriptorDigestsByFingerprint.get(fingerprint).add(digest); - } - - private void updateWeightsHistories() { - for (RelayNetworkStatusConsensus consensus : this.consensuses) { - long validAfterMillis = consensus.getValidAfterMillis(), - freshUntilMillis = consensus.getFreshUntilMillis(); - SortedMap<String, double[]> pathSelectionWeights = - this.calculatePathSelectionProbabilities(consensus); - this.updateWeightsHistory(validAfterMillis, freshUntilMillis, - pathSelectionWeights); - } - } - - // TODO Use 4 workers once threading problems are solved. - private static final int HISTORY_UPDATER_WORKERS_NUM = 1; - private void updateWeightsHistory(long validAfterMillis, - long freshUntilMillis, - SortedMap<String, double[]> pathSelectionWeights) { - List<HistoryUpdateWorker> historyUpdateWorkers = - new ArrayList<HistoryUpdateWorker>(); - for (int i = 0; i < HISTORY_UPDATER_WORKERS_NUM; i++) { - HistoryUpdateWorker historyUpdateWorker = - new HistoryUpdateWorker(validAfterMillis, freshUntilMillis, - pathSelectionWeights, this); - historyUpdateWorkers.add(historyUpdateWorker); - historyUpdateWorker.setDaemon(true); - historyUpdateWorker.start(); - } - for (HistoryUpdateWorker historyUpdateWorker : historyUpdateWorkers) { - try { - historyUpdateWorker.join(); - } catch (InterruptedException e) { - /* This is not something that we can take care of. Just leave the - * worker thread alone. */ - } - } - } - - private class HistoryUpdateWorker extends Thread { - private long validAfterMillis; - private long freshUntilMillis; - private SortedMap<String, double[]> pathSelectionWeights; - private WeightsDataWriter parent; - public HistoryUpdateWorker(long validAfterMillis, - long freshUntilMillis, - SortedMap<String, double[]> pathSelectionWeights, - WeightsDataWriter parent) { - this.validAfterMillis = validAfterMillis; - this.freshUntilMillis = freshUntilMillis; - this.pathSelectionWeights = pathSelectionWeights; - this.parent = parent; - } - public void run() { - String fingerprint = null; - double[] weights = null; - do { - fingerprint = null; - synchronized (pathSelectionWeights) { - if (!pathSelectionWeights.isEmpty()) { - fingerprint = pathSelectionWeights.firstKey(); - weights = pathSelectionWeights.remove(fingerprint); - } - } - if (fingerprint != null) { - this.parent.addToHistory(fingerprint, this.validAfterMillis, - this.freshUntilMillis, weights); - } - } while (fingerprint != null); - } - } - - private SortedMap<String, double[]> calculatePathSelectionProbabilities( - RelayNetworkStatusConsensus consensus) { - double wgg = 1.0, wgd = 1.0, wmg = 1.0, wmm = 1.0, wme = 1.0, - wmd = 1.0, wee = 1.0, wed = 1.0; - SortedMap<String, Integer> bandwidthWeights = - consensus.getBandwidthWeights(); - if (bandwidthWeights != null) { - SortedSet<String> missingWeightKeys = new TreeSet<String>( - Arrays.asList("Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(","))); - missingWeightKeys.removeAll(bandwidthWeights.keySet()); - if (missingWeightKeys.isEmpty()) { - wgg = ((double) bandwidthWeights.get("Wgg")) / 10000.0; - wgd = ((double) bandwidthWeights.get("Wgd")) / 10000.0; - wmg = ((double) bandwidthWeights.get("Wmg")) / 10000.0; - wmm = ((double) bandwidthWeights.get("Wmm")) / 10000.0; - wme = ((double) bandwidthWeights.get("Wme")) / 10000.0; - wmd = ((double) bandwidthWeights.get("Wmd")) / 10000.0; - wee = ((double) bandwidthWeights.get("Wee")) / 10000.0; - wed = ((double) bandwidthWeights.get("Wed")) / 10000.0; - } - } - SortedMap<String, Double> - advertisedBandwidths = new TreeMap<String, Double>(), - consensusWeights = new TreeMap<String, Double>(), - guardWeights = new TreeMap<String, Double>(), - middleWeights = new TreeMap<String, Double>(), - exitWeights = new TreeMap<String, Double>(); - double totalAdvertisedBandwidth = 0.0; - double totalConsensusWeight = 0.0; - double totalGuardWeight = 0.0; - double totalMiddleWeight = 0.0; - double totalExitWeight = 0.0; - for (NetworkStatusEntry relay : - consensus.getStatusEntries().values()) { - String fingerprint = relay.getFingerprint(); - if (!relay.getFlags().contains("Running")) { - continue; - } - boolean isExit = relay.getFlags().contains("Exit") && - !relay.getFlags().contains("BadExit"); - boolean isGuard = relay.getFlags().contains("Guard"); - String serverDescriptorDigest = relay.getDescriptor(). - toUpperCase(); - double advertisedBandwidth = 0.0; - if (!this.advertisedBandwidths.containsKey( - serverDescriptorDigest)) { - WeightsStatus weightsStatus = this.documentStore.retrieve( - WeightsStatus.class, true, fingerprint); - if (weightsStatus != null) { - if (!this.descriptorDigestsByFingerprint.containsKey( - fingerprint)) { - this.descriptorDigestsByFingerprint.put(fingerprint, - new HashSet<String>()); - } - this.descriptorDigestsByFingerprint.get(fingerprint).addAll( - weightsStatus.advertisedBandwidths.keySet()); - this.advertisedBandwidths.putAll( - weightsStatus.advertisedBandwidths); - } - } - if (this.advertisedBandwidths.containsKey( - serverDescriptorDigest)) { - advertisedBandwidth = (double) this.advertisedBandwidths.get( - serverDescriptorDigest); - } - double consensusWeight = (double) relay.getBandwidth(); - double guardWeight = (double) relay.getBandwidth(); - double middleWeight = (double) relay.getBandwidth(); - double exitWeight = (double) relay.getBandwidth(); - if (isGuard && isExit) { - guardWeight *= wgd; - middleWeight *= wmd; - exitWeight *= wed; - } else if (isGuard) { - guardWeight *= wgg; - middleWeight *= wmg; - exitWeight = 0.0; - } else if (isExit) { - guardWeight = 0.0; - middleWeight *= wme; - exitWeight *= wee; - } else { - guardWeight = 0.0; - middleWeight *= wmm; - exitWeight = 0.0; - } - advertisedBandwidths.put(fingerprint, advertisedBandwidth); - consensusWeights.put(fingerprint, consensusWeight); - guardWeights.put(fingerprint, guardWeight); - middleWeights.put(fingerprint, middleWeight); - exitWeights.put(fingerprint, exitWeight); - totalAdvertisedBandwidth += advertisedBandwidth; - totalConsensusWeight += consensusWeight; - totalGuardWeight += guardWeight; - totalMiddleWeight += middleWeight; - totalExitWeight += exitWeight; - } - SortedMap<String, double[]> pathSelectionProbabilities = - new TreeMap<String, double[]>(); - for (NetworkStatusEntry relay : - consensus.getStatusEntries().values()) { - String fingerprint = relay.getFingerprint(); - double[] probabilities = new double[] { - advertisedBandwidths.get(fingerprint) - / totalAdvertisedBandwidth, - consensusWeights.get(fingerprint) / totalConsensusWeight, - guardWeights.get(fingerprint) / totalGuardWeight, - middleWeights.get(fingerprint) / totalMiddleWeight, - exitWeights.get(fingerprint) / totalExitWeight }; - pathSelectionProbabilities.put(fingerprint, probabilities); - } - return pathSelectionProbabilities; - } - - private void addToHistory(String fingerprint, long validAfterMillis, - long freshUntilMillis, double[] weights) { - WeightsStatus weightsStatus = this.documentStore.retrieve( - WeightsStatus.class, true, fingerprint); - if (weightsStatus == null) { - weightsStatus = new WeightsStatus(); - } - SortedMap<long[], double[]> history = weightsStatus.history; - long[] interval = new long[] { validAfterMillis, freshUntilMillis }; - if ((history.headMap(interval).isEmpty() || - history.headMap(interval).lastKey()[1] <= validAfterMillis) && - (history.tailMap(interval).isEmpty() || - history.tailMap(interval).firstKey()[0] >= freshUntilMillis)) { - history.put(interval, weights); - this.compressHistory(weightsStatus); - this.addAdvertisedBandwidths(weightsStatus, fingerprint); - this.documentStore.store(weightsStatus, fingerprint); - this.updateWeightsStatuses.remove(fingerprint); - } - } - - private void addAdvertisedBandwidths(WeightsStatus weightsStatus, - String fingerprint) { - if (this.descriptorDigestsByFingerprint.containsKey(fingerprint)) { - for (String descriptorDigest : - this.descriptorDigestsByFingerprint.get(fingerprint)) { - if (this.advertisedBandwidths.containsKey(descriptorDigest)) { - int advertisedBandwidth = - this.advertisedBandwidths.get(descriptorDigest); - weightsStatus.advertisedBandwidths.put(descriptorDigest, - advertisedBandwidth); - } - } - } - } - - private void compressHistory(WeightsStatus weightsStatus) { - SortedMap<long[], double[]> history = weightsStatus.history; - SortedMap<long[], double[]> compressedHistory = - new TreeMap<long[], double[]>(history.comparator()); - long lastStartMillis = 0L, lastEndMillis = 0L; - double[] lastWeights = null; - SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String lastMonthString = "1970-01"; - for (Map.Entry<long[], double[]> e : history.entrySet()) { - long startMillis = e.getKey()[0], endMillis = e.getKey()[1]; - double[] weights = e.getValue(); - long intervalLengthMillis; - if (this.now - endMillis <= 7L * 24L * 60L * 60L * 1000L) { - intervalLengthMillis = 60L * 60L * 1000L; - } else if (this.now - endMillis <= 31L * 24L * 60L * 60L * 1000L) { - intervalLengthMillis = 4L * 60L * 60L * 1000L; - } else if (this.now - endMillis <= 92L * 24L * 60L * 60L * 1000L) { - intervalLengthMillis = 12L * 60L * 60L * 1000L; - } else if (this.now - endMillis <= 366L * 24L * 60L * 60L * 1000L) { - intervalLengthMillis = 2L * 24L * 60L * 60L * 1000L; - } else { - intervalLengthMillis = 10L * 24L * 60L * 60L * 1000L; - } - String monthString = dateTimeFormat.format(startMillis); - if (lastEndMillis == startMillis && - ((lastEndMillis - 1L) / intervalLengthMillis) == - ((endMillis - 1L) / intervalLengthMillis) && - lastMonthString.equals(monthString)) { - double lastIntervalInHours = (double) ((lastEndMillis - - lastStartMillis) / 60L * 60L * 1000L); - double currentIntervalInHours = (double) ((endMillis - - startMillis) / 60L * 60L * 1000L); - double newIntervalInHours = (double) ((endMillis - - lastStartMillis) / 60L * 60L * 1000L); - for (int i = 0; i < lastWeights.length; i++) { - lastWeights[i] *= lastIntervalInHours; - lastWeights[i] += weights[i] * currentIntervalInHours; - lastWeights[i] /= newIntervalInHours; - } - lastEndMillis = endMillis; - } else { - if (lastStartMillis > 0L) { - compressedHistory.put(new long[] { lastStartMillis, - lastEndMillis }, lastWeights); - } - lastStartMillis = startMillis; - lastEndMillis = endMillis; - lastWeights = weights; - } - lastMonthString = monthString; - } - if (lastStartMillis > 0L) { - compressedHistory.put(new long[] { lastStartMillis, lastEndMillis }, - lastWeights); - } - weightsStatus.history = compressedHistory; - } - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - if (relay) { - this.updateWeightsDocuments.addAll(fingerprints); - } - } - - private void writeWeightsDataFiles() { - for (String fingerprint : this.updateWeightsDocuments) { - WeightsStatus weightsStatus = this.documentStore.retrieve( - WeightsStatus.class, true, fingerprint); - if (weightsStatus == null) { - continue; - } - SortedMap<long[], double[]> history = weightsStatus.history; - WeightsDocument weightsDocument = new WeightsDocument(); - weightsDocument.documentString = this.formatHistoryString( - fingerprint, history); - this.documentStore.store(weightsDocument, fingerprint); - } - Logger.printStatusTime("Wrote weights document files"); - } - - private String[] graphTypes = new String[] { - "advertised_bandwidth_fraction", - "consensus_weight_fraction", - "guard_probability", - "middle_probability", - "exit_probability" - }; - - private String[] graphNames = new String[] { - "1_week", - "1_month", - "3_months", - "1_year", - "5_years" }; - - private long[] graphIntervals = new long[] { - 7L * 24L * 60L * 60L * 1000L, - 31L * 24L * 60L * 60L * 1000L, - 92L * 24L * 60L * 60L * 1000L, - 366L * 24L * 60L * 60L * 1000L, - 5L * 366L * 24L * 60L * 60L * 1000L }; - - private long[] dataPointIntervals = new long[] { - 60L * 60L * 1000L, - 4L * 60L * 60L * 1000L, - 12L * 60L * 60L * 1000L, - 2L * 24L * 60L * 60L * 1000L, - 10L * 24L * 60L * 60L * 1000L }; - - private String formatHistoryString(String fingerprint, - SortedMap<long[], double[]> history) { - StringBuilder sb = new StringBuilder(); - sb.append("{\"fingerprint\":\"" + fingerprint + "\""); - for (int graphTypeIndex = 0; graphTypeIndex < this.graphTypes.length; - graphTypeIndex++) { - String graphType = this.graphTypes[graphTypeIndex]; - sb.append(",\n\"" + graphType + "\":{"); - int graphIntervalsWritten = 0; - for (int graphIntervalIndex = 0; graphIntervalIndex < - this.graphIntervals.length; graphIntervalIndex++) { - String timeline = this.formatTimeline(graphTypeIndex, - graphIntervalIndex, history); - if (timeline != null) { - sb.append((graphIntervalsWritten++ > 0 ? "," : "") + "\n" - + timeline); - } - } - sb.append("}"); - } - sb.append("\n}\n"); - return sb.toString(); - } - - private String formatTimeline(int graphTypeIndex, - int graphIntervalIndex, SortedMap<long[], double[]> history) { - String graphName = this.graphNames[graphIntervalIndex]; - long graphInterval = this.graphIntervals[graphIntervalIndex]; - long dataPointInterval = - this.dataPointIntervals[graphIntervalIndex]; - List<Double> dataPoints = new ArrayList<Double>(); - long intervalStartMillis = ((this.now - graphInterval) - / dataPointInterval) * dataPointInterval; - long totalMillis = 0L; - double totalWeightTimesMillis = 0.0; - for (Map.Entry<long[], double[]> e : history.entrySet()) { - long startMillis = e.getKey()[0], endMillis = e.getKey()[1]; - double weight = e.getValue()[graphTypeIndex]; - if (endMillis < intervalStartMillis) { - continue; - } - while ((intervalStartMillis / dataPointInterval) != - (endMillis / dataPointInterval)) { - dataPoints.add(totalMillis * 5L < dataPointInterval - ? -1.0 : totalWeightTimesMillis / (double) totalMillis); - totalWeightTimesMillis = 0.0; - totalMillis = 0L; - intervalStartMillis += dataPointInterval; - } - totalWeightTimesMillis += weight - * ((double) (endMillis - startMillis)); - totalMillis += (endMillis - startMillis); - } - dataPoints.add(totalMillis * 5L < dataPointInterval - ? -1.0 : totalWeightTimesMillis / (double) totalMillis); - double maxValue = 0.0; - int firstNonNullIndex = -1, lastNonNullIndex = -1; - for (int dataPointIndex = 0; dataPointIndex < dataPoints.size(); - dataPointIndex++) { - double dataPoint = dataPoints.get(dataPointIndex); - if (dataPoint >= 0.0) { - if (firstNonNullIndex < 0) { - firstNonNullIndex = dataPointIndex; - } - lastNonNullIndex = dataPointIndex; - if (dataPoint > maxValue) { - maxValue = dataPoint; - } - } - } - if (firstNonNullIndex < 0) { - return null; - } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) * dataPointInterval - + dataPointInterval / 2L; - if (graphIntervalIndex > 0 && firstDataPointMillis >= - this.now - graphIntervals[graphIntervalIndex - 1]) { - /* Skip weights history object, because it doesn't contain - * anything new that wasn't already contained in the last - * weights history object(s). */ - return null; - } - long lastDataPointMillis = firstDataPointMillis - + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; - double factor = ((double) maxValue) / 999.0; - int count = lastNonNullIndex - firstNonNullIndex + 1; - StringBuilder sb = new StringBuilder(); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - sb.append("\"" + graphName + "\":{" - + "\"first\":\"" + dateTimeFormat.format(firstDataPointMillis) - + "\",\"last\":\"" + dateTimeFormat.format(lastDataPointMillis) - + "\",\"interval\":" + String.valueOf(dataPointInterval / 1000L) - + ",\"factor\":" + String.format(Locale.US, "%.9f", factor) - + ",\"count\":" + String.valueOf(count) + ",\"values\":["); - int dataPointsWritten = 0, previousNonNullIndex = -2; - boolean foundTwoAdjacentDataPoints = false; - for (int dataPointIndex = firstNonNullIndex; dataPointIndex <= - lastNonNullIndex; dataPointIndex++) { - double dataPoint = dataPoints.get(dataPointIndex); - if (dataPoint >= 0.0) { - if (dataPointIndex - previousNonNullIndex == 1) { - foundTwoAdjacentDataPoints = true; - } - previousNonNullIndex = dataPointIndex; - } - sb.append((dataPointsWritten++ > 0 ? "," : "") - + (dataPoint < 0.0 ? "null" : - String.valueOf((long) ((dataPoint * 999.0) / maxValue)))); - } - sb.append("]}"); - if (foundTwoAdjacentDataPoints) { - return sb.toString(); - } else { - return null; - } - } - - private void updateWeightsStatuses() { - for (String fingerprint : this.updateWeightsStatuses) { - WeightsStatus weightsStatus = this.documentStore.retrieve( - WeightsStatus.class, true, fingerprint); - if (weightsStatus == null) { - weightsStatus = new WeightsStatus(); - } - this.addAdvertisedBandwidths(weightsStatus, fingerprint); - this.documentStore.store(weightsStatus, fingerprint); - } - } - - public String getStatsString() { - /* TODO Add statistics string. */ - return null; - } -} - diff --git a/src/org/torproject/onionoo/WeightsDocumentWriter.java b/src/org/torproject/onionoo/WeightsDocumentWriter.java new file mode 100644 index 0000000..9e5355f --- /dev/null +++ b/src/org/torproject/onionoo/WeightsDocumentWriter.java @@ -0,0 +1,222 @@ +/* Copyright 2012--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TimeZone; + +public class WeightsDocumentWriter implements FingerprintListener, + DocumentWriter { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + public WeightsDocumentWriter(DescriptorSource descriptorSource, + DocumentStore documentStore, Time time) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + this.now = time.currentTimeMillis(); + this.registerFingerprintListeners(); + } + + private void registerFingerprintListeners() { + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_SERVER_DESCRIPTORS); + } + + private Set<String> updateWeightsDocuments = new HashSet<String>(); + + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + if (relay) { + this.updateWeightsDocuments.addAll(fingerprints); + } + } + + public void writeDocuments() { + this.writeWeightsDataFiles(); + Logger.printStatusTime("Wrote weights document files"); + } + + private void writeWeightsDataFiles() { + for (String fingerprint : this.updateWeightsDocuments) { + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus == null) { + continue; + } + SortedMap<long[], double[]> history = weightsStatus.history; + WeightsDocument weightsDocument = new WeightsDocument(); + weightsDocument.documentString = this.formatHistoryString( + fingerprint, history); + this.documentStore.store(weightsDocument, fingerprint); + } + } + + private String[] graphTypes = new String[] { + "advertised_bandwidth_fraction", + "consensus_weight_fraction", + "guard_probability", + "middle_probability", + "exit_probability" + }; + + private String[] graphNames = new String[] { + "1_week", + "1_month", + "3_months", + "1_year", + "5_years" }; + + private long[] graphIntervals = new long[] { + 7L * 24L * 60L * 60L * 1000L, + 31L * 24L * 60L * 60L * 1000L, + 92L * 24L * 60L * 60L * 1000L, + 366L * 24L * 60L * 60L * 1000L, + 5L * 366L * 24L * 60L * 60L * 1000L }; + + private long[] dataPointIntervals = new long[] { + 60L * 60L * 1000L, + 4L * 60L * 60L * 1000L, + 12L * 60L * 60L * 1000L, + 2L * 24L * 60L * 60L * 1000L, + 10L * 24L * 60L * 60L * 1000L }; + + private String formatHistoryString(String fingerprint, + SortedMap<long[], double[]> history) { + StringBuilder sb = new StringBuilder(); + sb.append("{\"fingerprint\":\"" + fingerprint + "\""); + for (int graphTypeIndex = 0; graphTypeIndex < this.graphTypes.length; + graphTypeIndex++) { + String graphType = this.graphTypes[graphTypeIndex]; + sb.append(",\n\"" + graphType + "\":{"); + int graphIntervalsWritten = 0; + for (int graphIntervalIndex = 0; graphIntervalIndex < + this.graphIntervals.length; graphIntervalIndex++) { + String timeline = this.formatTimeline(graphTypeIndex, + graphIntervalIndex, history); + if (timeline != null) { + sb.append((graphIntervalsWritten++ > 0 ? "," : "") + "\n" + + timeline); + } + } + sb.append("}"); + } + sb.append("\n}\n"); + return sb.toString(); + } + + private String formatTimeline(int graphTypeIndex, + int graphIntervalIndex, SortedMap<long[], double[]> history) { + String graphName = this.graphNames[graphIntervalIndex]; + long graphInterval = this.graphIntervals[graphIntervalIndex]; + long dataPointInterval = + this.dataPointIntervals[graphIntervalIndex]; + List<Double> dataPoints = new ArrayList<Double>(); + long intervalStartMillis = ((this.now - graphInterval) + / dataPointInterval) * dataPointInterval; + long totalMillis = 0L; + double totalWeightTimesMillis = 0.0; + for (Map.Entry<long[], double[]> e : history.entrySet()) { + long startMillis = e.getKey()[0], endMillis = e.getKey()[1]; + double weight = e.getValue()[graphTypeIndex]; + if (endMillis < intervalStartMillis) { + continue; + } + while ((intervalStartMillis / dataPointInterval) != + (endMillis / dataPointInterval)) { + dataPoints.add(totalMillis * 5L < dataPointInterval + ? -1.0 : totalWeightTimesMillis / (double) totalMillis); + totalWeightTimesMillis = 0.0; + totalMillis = 0L; + intervalStartMillis += dataPointInterval; + } + totalWeightTimesMillis += weight + * ((double) (endMillis - startMillis)); + totalMillis += (endMillis - startMillis); + } + dataPoints.add(totalMillis * 5L < dataPointInterval + ? -1.0 : totalWeightTimesMillis / (double) totalMillis); + double maxValue = 0.0; + int firstNonNullIndex = -1, lastNonNullIndex = -1; + for (int dataPointIndex = 0; dataPointIndex < dataPoints.size(); + dataPointIndex++) { + double dataPoint = dataPoints.get(dataPointIndex); + if (dataPoint >= 0.0) { + if (firstNonNullIndex < 0) { + firstNonNullIndex = dataPointIndex; + } + lastNonNullIndex = dataPointIndex; + if (dataPoint > maxValue) { + maxValue = dataPoint; + } + } + } + if (firstNonNullIndex < 0) { + return null; + } + long firstDataPointMillis = (((this.now - graphInterval) + / dataPointInterval) + firstNonNullIndex) * dataPointInterval + + dataPointInterval / 2L; + if (graphIntervalIndex > 0 && firstDataPointMillis >= + this.now - graphIntervals[graphIntervalIndex - 1]) { + /* Skip weights history object, because it doesn't contain + * anything new that wasn't already contained in the last + * weights history object(s). */ + return null; + } + long lastDataPointMillis = firstDataPointMillis + + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; + double factor = ((double) maxValue) / 999.0; + int count = lastNonNullIndex - firstNonNullIndex + 1; + StringBuilder sb = new StringBuilder(); + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + sb.append("\"" + graphName + "\":{" + + "\"first\":\"" + dateTimeFormat.format(firstDataPointMillis) + + "\",\"last\":\"" + dateTimeFormat.format(lastDataPointMillis) + + "\",\"interval\":" + String.valueOf(dataPointInterval / 1000L) + + ",\"factor\":" + String.format(Locale.US, "%.9f", factor) + + ",\"count\":" + String.valueOf(count) + ",\"values\":["); + int dataPointsWritten = 0, previousNonNullIndex = -2; + boolean foundTwoAdjacentDataPoints = false; + for (int dataPointIndex = firstNonNullIndex; dataPointIndex <= + lastNonNullIndex; dataPointIndex++) { + double dataPoint = dataPoints.get(dataPointIndex); + if (dataPoint >= 0.0) { + if (dataPointIndex - previousNonNullIndex == 1) { + foundTwoAdjacentDataPoints = true; + } + previousNonNullIndex = dataPointIndex; + } + sb.append((dataPointsWritten++ > 0 ? "," : "") + + (dataPoint < 0.0 ? "null" : + String.valueOf((long) ((dataPoint * 999.0) / maxValue)))); + } + sb.append("]}"); + if (foundTwoAdjacentDataPoints) { + return sb.toString(); + } else { + return null; + } + } + + public String getStatsString() { + /* TODO Add statistics string. */ + return null; + } +} diff --git a/src/org/torproject/onionoo/WeightsStatusUpdater.java b/src/org/torproject/onionoo/WeightsStatusUpdater.java new file mode 100644 index 0000000..5e890ef --- /dev/null +++ b/src/org/torproject/onionoo/WeightsStatusUpdater.java @@ -0,0 +1,394 @@ +/* Copyright 2012--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.NetworkStatusEntry; +import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.descriptor.ServerDescriptor; + +public class WeightsStatusUpdater implements DescriptorListener, + StatusUpdater { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + public WeightsStatusUpdater(DescriptorSource descriptorSource, + DocumentStore documentStore, Time time) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + this.now = time.currentTimeMillis(); + this.registerDescriptorListeners(); + } + + private void registerDescriptorListeners() { + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.RELAY_SERVER_DESCRIPTORS); + } + + public void processDescriptor(Descriptor descriptor, boolean relay) { + if (descriptor instanceof ServerDescriptor) { + this.processRelayServerDescriptor((ServerDescriptor) descriptor); + } else if (descriptor instanceof RelayNetworkStatusConsensus) { + this.processRelayNetworkConsensus( + (RelayNetworkStatusConsensus) descriptor); + } + } + + public void updateStatuses() { + this.updateWeightsHistories(); + Logger.printStatusTime("Updated weights histories"); + this.updateWeightsStatuses(); + Logger.printStatusTime("Updated weights status files"); + } + + private Set<RelayNetworkStatusConsensus> consensuses = + new HashSet<RelayNetworkStatusConsensus>(); + + private void processRelayNetworkConsensus( + RelayNetworkStatusConsensus consensus) { + this.consensuses.add(consensus); + } + + private Set<String> updateWeightsStatuses = new HashSet<String>(); + + private Map<String, Set<String>> descriptorDigestsByFingerprint = + new HashMap<String, Set<String>>(); + + private Map<String, Integer> advertisedBandwidths = + new HashMap<String, Integer>(); + + private void processRelayServerDescriptor( + ServerDescriptor serverDescriptor) { + String digest = serverDescriptor.getServerDescriptorDigest(). + toUpperCase(); + int advertisedBandwidth = Math.min(Math.min( + serverDescriptor.getBandwidthBurst(), + serverDescriptor.getBandwidthObserved()), + serverDescriptor.getBandwidthRate()); + this.advertisedBandwidths.put(digest, advertisedBandwidth); + String fingerprint = serverDescriptor.getFingerprint(); + this.updateWeightsStatuses.add(fingerprint); + if (!this.descriptorDigestsByFingerprint.containsKey( + fingerprint)) { + this.descriptorDigestsByFingerprint.put(fingerprint, + new HashSet<String>()); + } + this.descriptorDigestsByFingerprint.get(fingerprint).add(digest); + } + + private void updateWeightsHistories() { + for (RelayNetworkStatusConsensus consensus : this.consensuses) { + long validAfterMillis = consensus.getValidAfterMillis(), + freshUntilMillis = consensus.getFreshUntilMillis(); + SortedMap<String, double[]> pathSelectionWeights = + this.calculatePathSelectionProbabilities(consensus); + this.updateWeightsHistory(validAfterMillis, freshUntilMillis, + pathSelectionWeights); + } + } + + // TODO Use 4 workers once threading problems are solved. + private static final int HISTORY_UPDATER_WORKERS_NUM = 1; + private void updateWeightsHistory(long validAfterMillis, + long freshUntilMillis, + SortedMap<String, double[]> pathSelectionWeights) { + List<HistoryUpdateWorker> historyUpdateWorkers = + new ArrayList<HistoryUpdateWorker>(); + for (int i = 0; i < HISTORY_UPDATER_WORKERS_NUM; i++) { + HistoryUpdateWorker historyUpdateWorker = + new HistoryUpdateWorker(validAfterMillis, freshUntilMillis, + pathSelectionWeights, this); + historyUpdateWorkers.add(historyUpdateWorker); + historyUpdateWorker.setDaemon(true); + historyUpdateWorker.start(); + } + for (HistoryUpdateWorker historyUpdateWorker : historyUpdateWorkers) { + try { + historyUpdateWorker.join(); + } catch (InterruptedException e) { + /* This is not something that we can take care of. Just leave the + * worker thread alone. */ + } + } + } + + private class HistoryUpdateWorker extends Thread { + private long validAfterMillis; + private long freshUntilMillis; + private SortedMap<String, double[]> pathSelectionWeights; + private WeightsStatusUpdater parent; + public HistoryUpdateWorker(long validAfterMillis, + long freshUntilMillis, + SortedMap<String, double[]> pathSelectionWeights, + WeightsStatusUpdater parent) { + this.validAfterMillis = validAfterMillis; + this.freshUntilMillis = freshUntilMillis; + this.pathSelectionWeights = pathSelectionWeights; + this.parent = parent; + } + public void run() { + String fingerprint = null; + double[] weights = null; + do { + fingerprint = null; + synchronized (pathSelectionWeights) { + if (!pathSelectionWeights.isEmpty()) { + fingerprint = pathSelectionWeights.firstKey(); + weights = pathSelectionWeights.remove(fingerprint); + } + } + if (fingerprint != null) { + this.parent.addToHistory(fingerprint, this.validAfterMillis, + this.freshUntilMillis, weights); + } + } while (fingerprint != null); + } + } + + private SortedMap<String, double[]> calculatePathSelectionProbabilities( + RelayNetworkStatusConsensus consensus) { + double wgg = 1.0, wgd = 1.0, wmg = 1.0, wmm = 1.0, wme = 1.0, + wmd = 1.0, wee = 1.0, wed = 1.0; + SortedMap<String, Integer> bandwidthWeights = + consensus.getBandwidthWeights(); + if (bandwidthWeights != null) { + SortedSet<String> missingWeightKeys = new TreeSet<String>( + Arrays.asList("Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(","))); + missingWeightKeys.removeAll(bandwidthWeights.keySet()); + if (missingWeightKeys.isEmpty()) { + wgg = ((double) bandwidthWeights.get("Wgg")) / 10000.0; + wgd = ((double) bandwidthWeights.get("Wgd")) / 10000.0; + wmg = ((double) bandwidthWeights.get("Wmg")) / 10000.0; + wmm = ((double) bandwidthWeights.get("Wmm")) / 10000.0; + wme = ((double) bandwidthWeights.get("Wme")) / 10000.0; + wmd = ((double) bandwidthWeights.get("Wmd")) / 10000.0; + wee = ((double) bandwidthWeights.get("Wee")) / 10000.0; + wed = ((double) bandwidthWeights.get("Wed")) / 10000.0; + } + } + SortedMap<String, Double> + advertisedBandwidths = new TreeMap<String, Double>(), + consensusWeights = new TreeMap<String, Double>(), + guardWeights = new TreeMap<String, Double>(), + middleWeights = new TreeMap<String, Double>(), + exitWeights = new TreeMap<String, Double>(); + double totalAdvertisedBandwidth = 0.0; + double totalConsensusWeight = 0.0; + double totalGuardWeight = 0.0; + double totalMiddleWeight = 0.0; + double totalExitWeight = 0.0; + for (NetworkStatusEntry relay : + consensus.getStatusEntries().values()) { + String fingerprint = relay.getFingerprint(); + if (!relay.getFlags().contains("Running")) { + continue; + } + boolean isExit = relay.getFlags().contains("Exit") && + !relay.getFlags().contains("BadExit"); + boolean isGuard = relay.getFlags().contains("Guard"); + String serverDescriptorDigest = relay.getDescriptor(). + toUpperCase(); + double advertisedBandwidth = 0.0; + if (!this.advertisedBandwidths.containsKey( + serverDescriptorDigest)) { + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus != null) { + if (!this.descriptorDigestsByFingerprint.containsKey( + fingerprint)) { + this.descriptorDigestsByFingerprint.put(fingerprint, + new HashSet<String>()); + } + this.descriptorDigestsByFingerprint.get(fingerprint).addAll( + weightsStatus.advertisedBandwidths.keySet()); + this.advertisedBandwidths.putAll( + weightsStatus.advertisedBandwidths); + } + } + if (this.advertisedBandwidths.containsKey( + serverDescriptorDigest)) { + advertisedBandwidth = (double) this.advertisedBandwidths.get( + serverDescriptorDigest); + } + double consensusWeight = (double) relay.getBandwidth(); + double guardWeight = (double) relay.getBandwidth(); + double middleWeight = (double) relay.getBandwidth(); + double exitWeight = (double) relay.getBandwidth(); + if (isGuard && isExit) { + guardWeight *= wgd; + middleWeight *= wmd; + exitWeight *= wed; + } else if (isGuard) { + guardWeight *= wgg; + middleWeight *= wmg; + exitWeight = 0.0; + } else if (isExit) { + guardWeight = 0.0; + middleWeight *= wme; + exitWeight *= wee; + } else { + guardWeight = 0.0; + middleWeight *= wmm; + exitWeight = 0.0; + } + advertisedBandwidths.put(fingerprint, advertisedBandwidth); + consensusWeights.put(fingerprint, consensusWeight); + guardWeights.put(fingerprint, guardWeight); + middleWeights.put(fingerprint, middleWeight); + exitWeights.put(fingerprint, exitWeight); + totalAdvertisedBandwidth += advertisedBandwidth; + totalConsensusWeight += consensusWeight; + totalGuardWeight += guardWeight; + totalMiddleWeight += middleWeight; + totalExitWeight += exitWeight; + } + SortedMap<String, double[]> pathSelectionProbabilities = + new TreeMap<String, double[]>(); + for (NetworkStatusEntry relay : + consensus.getStatusEntries().values()) { + String fingerprint = relay.getFingerprint(); + double[] probabilities = new double[] { + advertisedBandwidths.get(fingerprint) + / totalAdvertisedBandwidth, + consensusWeights.get(fingerprint) / totalConsensusWeight, + guardWeights.get(fingerprint) / totalGuardWeight, + middleWeights.get(fingerprint) / totalMiddleWeight, + exitWeights.get(fingerprint) / totalExitWeight }; + pathSelectionProbabilities.put(fingerprint, probabilities); + } + return pathSelectionProbabilities; + } + + private void addToHistory(String fingerprint, long validAfterMillis, + long freshUntilMillis, double[] weights) { + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus == null) { + weightsStatus = new WeightsStatus(); + } + SortedMap<long[], double[]> history = weightsStatus.history; + long[] interval = new long[] { validAfterMillis, freshUntilMillis }; + if ((history.headMap(interval).isEmpty() || + history.headMap(interval).lastKey()[1] <= validAfterMillis) && + (history.tailMap(interval).isEmpty() || + history.tailMap(interval).firstKey()[0] >= freshUntilMillis)) { + history.put(interval, weights); + this.compressHistory(weightsStatus); + this.addAdvertisedBandwidths(weightsStatus, fingerprint); + this.documentStore.store(weightsStatus, fingerprint); + this.updateWeightsStatuses.remove(fingerprint); + } + } + + private void addAdvertisedBandwidths(WeightsStatus weightsStatus, + String fingerprint) { + if (this.descriptorDigestsByFingerprint.containsKey(fingerprint)) { + for (String descriptorDigest : + this.descriptorDigestsByFingerprint.get(fingerprint)) { + if (this.advertisedBandwidths.containsKey(descriptorDigest)) { + int advertisedBandwidth = + this.advertisedBandwidths.get(descriptorDigest); + weightsStatus.advertisedBandwidths.put(descriptorDigest, + advertisedBandwidth); + } + } + } + } + + private void compressHistory(WeightsStatus weightsStatus) { + SortedMap<long[], double[]> history = weightsStatus.history; + SortedMap<long[], double[]> compressedHistory = + new TreeMap<long[], double[]>(history.comparator()); + long lastStartMillis = 0L, lastEndMillis = 0L; + double[] lastWeights = null; + SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + String lastMonthString = "1970-01"; + for (Map.Entry<long[], double[]> e : history.entrySet()) { + long startMillis = e.getKey()[0], endMillis = e.getKey()[1]; + double[] weights = e.getValue(); + long intervalLengthMillis; + if (this.now - endMillis <= 7L * 24L * 60L * 60L * 1000L) { + intervalLengthMillis = 60L * 60L * 1000L; + } else if (this.now - endMillis <= 31L * 24L * 60L * 60L * 1000L) { + intervalLengthMillis = 4L * 60L * 60L * 1000L; + } else if (this.now - endMillis <= 92L * 24L * 60L * 60L * 1000L) { + intervalLengthMillis = 12L * 60L * 60L * 1000L; + } else if (this.now - endMillis <= 366L * 24L * 60L * 60L * 1000L) { + intervalLengthMillis = 2L * 24L * 60L * 60L * 1000L; + } else { + intervalLengthMillis = 10L * 24L * 60L * 60L * 1000L; + } + String monthString = dateTimeFormat.format(startMillis); + if (lastEndMillis == startMillis && + ((lastEndMillis - 1L) / intervalLengthMillis) == + ((endMillis - 1L) / intervalLengthMillis) && + lastMonthString.equals(monthString)) { + double lastIntervalInHours = (double) ((lastEndMillis + - lastStartMillis) / 60L * 60L * 1000L); + double currentIntervalInHours = (double) ((endMillis + - startMillis) / 60L * 60L * 1000L); + double newIntervalInHours = (double) ((endMillis + - lastStartMillis) / 60L * 60L * 1000L); + for (int i = 0; i < lastWeights.length; i++) { + lastWeights[i] *= lastIntervalInHours; + lastWeights[i] += weights[i] * currentIntervalInHours; + lastWeights[i] /= newIntervalInHours; + } + lastEndMillis = endMillis; + } else { + if (lastStartMillis > 0L) { + compressedHistory.put(new long[] { lastStartMillis, + lastEndMillis }, lastWeights); + } + lastStartMillis = startMillis; + lastEndMillis = endMillis; + lastWeights = weights; + } + lastMonthString = monthString; + } + if (lastStartMillis > 0L) { + compressedHistory.put(new long[] { lastStartMillis, lastEndMillis }, + lastWeights); + } + weightsStatus.history = compressedHistory; + } + + private void updateWeightsStatuses() { + for (String fingerprint : this.updateWeightsStatuses) { + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus == null) { + weightsStatus = new WeightsStatus(); + } + this.addAdvertisedBandwidths(weightsStatus, fingerprint); + this.documentStore.store(weightsStatus, fingerprint); + } + } + + public String getStatsString() { + /* TODO Add statistics string. */ + return null; + } +} +

1 0

[onionoo/master] Split clients data writer into two classes.
by karsten＠torproject.org 11 Apr '14

11 Apr '14

commit 864c148564a6ec3be0d979b126b8ca88f01c4912 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Apr 8 21:13:02 2014 +0200 Split clients data writer into two classes. --- src/org/torproject/onionoo/ClientsDataWriter.java | 486 -------------------- .../torproject/onionoo/ClientsDocumentWriter.java | 302 ++++++++++++ .../torproject/onionoo/ClientsStatusUpdater.java | 230 +++++++++ src/org/torproject/onionoo/Main.java | 10 +- 4 files changed, 538 insertions(+), 490 deletions(-) diff --git a/src/org/torproject/onionoo/ClientsDataWriter.java b/src/org/torproject/onionoo/ClientsDataWriter.java deleted file mode 100644 index 7662f54..0000000 --- a/src/org/torproject/onionoo/ClientsDataWriter.java +++ /dev/null @@ -1,486 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TimeZone; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.ExtraInfoDescriptor; - -/* - * Example extra-info descriptor used as input: - * - * extra-info ndnop2 DE6397A047ABE5F78B4C87AF725047831B221AAB - * dirreq-stats-end 2014-02-16 16:42:11 (86400 s) - * dirreq-v3-resp ok=856,not-enough-sigs=0,unavailable=0,not-found=0, - * not-modified=40,busy=0 - * bridge-stats-end 2014-02-16 16:42:17 (86400 s) - * bridge-ips ??=8,in=8,se=8 - * bridge-ip-versions v4=8,v6=0 - * - * Clients status file produced as intermediate output: - * - * 2014-02-15 16:42:11 2014-02-16 00:00:00 - * 259.042 in=86.347,se=86.347 v4=259.042 - * 2014-02-16 00:00:00 2014-02-16 16:42:11 - * 592.958 in=197.653,se=197.653 v4=592.958 - * - * Clients document file produced as output: - * - * "1_month":{ - * "first":"2014-02-03 12:00:00", - * "last":"2014-02-28 12:00:00", - * "interval":86400, - * "factor":0.139049349, - * "count":26, - * "values":[371,354,349,374,432,null,485,458,493,536,null,null,524,576, - * 607,622,null,635,null,566,774,999,945,690,656,681], - * "countries":{"cn":0.0192,"in":0.1768,"ir":0.2487,"ru":0.0104, - * "se":0.1698,"sy":0.0325,"us":0.0406}, - * "transports":{"obfs2":0.4581}, - * "versions":{"v4":1.0000}} - */ -public class ClientsDataWriter implements DescriptorListener, - StatusUpdater, FingerprintListener, DocumentWriter { - - private DescriptorSource descriptorSource; - - private DocumentStore documentStore; - - private long now; - - public ClientsDataWriter(DescriptorSource descriptorSource, - DocumentStore documentStore, Time time) { - this.descriptorSource = descriptorSource; - this.documentStore = documentStore; - this.now = time.currentTimeMillis(); - this.registerDescriptorListeners(); - this.registerFingerprintListeners(); - } - - private void registerDescriptorListeners() { - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.BRIDGE_EXTRA_INFOS); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_EXTRA_INFOS); - } - - public void processDescriptor(Descriptor descriptor, boolean relay) { - if (descriptor instanceof ExtraInfoDescriptor && !relay) { - this.processBridgeExtraInfoDescriptor( - (ExtraInfoDescriptor) descriptor); - } - } - - private static final long ONE_HOUR_MILLIS = 60L * 60L * 1000L, - ONE_DAY_MILLIS = 24L * ONE_HOUR_MILLIS; - - private SortedMap<String, SortedSet<ClientsHistory>> newResponses = - new TreeMap<String, SortedSet<ClientsHistory>>(); - - private void processBridgeExtraInfoDescriptor( - ExtraInfoDescriptor descriptor) { - long dirreqStatsEndMillis = descriptor.getDirreqStatsEndMillis(); - long dirreqStatsIntervalLengthMillis = - descriptor.getDirreqStatsIntervalLength() * 1000L; - SortedMap<String, Integer> responses = descriptor.getDirreqV3Resp(); - if (dirreqStatsEndMillis < 0L || - dirreqStatsIntervalLengthMillis != ONE_DAY_MILLIS || - responses == null || !responses.containsKey("ok")) { - return; - } - double okResponses = (double) (responses.get("ok") - 4); - if (okResponses < 0.0) { - return; - } - String hashedFingerprint = descriptor.getFingerprint().toUpperCase(); - long dirreqStatsStartMillis = dirreqStatsEndMillis - - dirreqStatsIntervalLengthMillis; - long utcBreakMillis = (dirreqStatsEndMillis / ONE_DAY_MILLIS) - * ONE_DAY_MILLIS; - for (int i = 0; i < 2; i++) { - long startMillis = i == 0 ? dirreqStatsStartMillis : utcBreakMillis; - long endMillis = i == 0 ? utcBreakMillis : dirreqStatsEndMillis; - if (startMillis >= endMillis) { - continue; - } - double totalResponses = okResponses - * ((double) (endMillis - startMillis)) - / ((double) ONE_DAY_MILLIS); - SortedMap<String, Double> responsesByCountry = - this.weightResponsesWithUniqueIps(totalResponses, - descriptor.getBridgeIps(), "??"); - SortedMap<String, Double> responsesByTransport = - this.weightResponsesWithUniqueIps(totalResponses, - descriptor.getBridgeIpTransports(), "<??>"); - SortedMap<String, Double> responsesByVersion = - this.weightResponsesWithUniqueIps(totalResponses, - descriptor.getBridgeIpVersions(), ""); - ClientsHistory newResponseHistory = new ClientsHistory( - startMillis, endMillis, totalResponses, responsesByCountry, - responsesByTransport, responsesByVersion); - if (!this.newResponses.containsKey(hashedFingerprint)) { - this.newResponses.put(hashedFingerprint, - new TreeSet<ClientsHistory>()); - } - this.newResponses.get(hashedFingerprint).add( - newResponseHistory); - } - } - - private SortedMap<String, Double> weightResponsesWithUniqueIps( - double totalResponses, SortedMap<String, Integer> uniqueIps, - String omitString) { - SortedMap<String, Double> weightedResponses = - new TreeMap<String, Double>(); - int totalUniqueIps = 0; - if (uniqueIps != null) { - for (Map.Entry<String, Integer> e : uniqueIps.entrySet()) { - if (e.getValue() > 4) { - totalUniqueIps += e.getValue() - 4; - } - } - } - if (totalUniqueIps > 0) { - for (Map.Entry<String, Integer> e : uniqueIps.entrySet()) { - if (!e.getKey().equals(omitString) && e.getValue() > 4) { - weightedResponses.put(e.getKey(), - (((double) (e.getValue() - 4)) * totalResponses) - / ((double) totalUniqueIps)); - } - } - } - return weightedResponses; - } - - public void updateStatuses() { - for (Map.Entry<String, SortedSet<ClientsHistory>> e : - this.newResponses.entrySet()) { - String hashedFingerprint = e.getKey(); - ClientsStatus clientsStatus = this.documentStore.retrieve( - ClientsStatus.class, true, hashedFingerprint); - if (clientsStatus == null) { - clientsStatus = new ClientsStatus(); - } - this.addToHistory(clientsStatus, e.getValue()); - this.compressHistory(clientsStatus); - this.documentStore.store(clientsStatus, hashedFingerprint); - } - Logger.printStatusTime("Updated clients status files"); - } - - private void addToHistory(ClientsStatus clientsStatus, - SortedSet<ClientsHistory> newIntervals) { - SortedSet<ClientsHistory> history = clientsStatus.history; - for (ClientsHistory interval : newIntervals) { - if ((history.headSet(interval).isEmpty() || - history.headSet(interval).last().endMillis <= - interval.startMillis) && - (history.tailSet(interval).isEmpty() || - history.tailSet(interval).first().startMillis >= - interval.endMillis)) { - history.add(interval); - } - } - } - - private void compressHistory(ClientsStatus clientsStatus) { - SortedSet<ClientsHistory> history = clientsStatus.history; - SortedSet<ClientsHistory> compressedHistory = - new TreeSet<ClientsHistory>(); - ClientsHistory lastResponses = null; - SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String lastMonthString = "1970-01"; - for (ClientsHistory responses : history) { - long intervalLengthMillis; - if (this.now - responses.endMillis <= - 92L * 24L * 60L * 60L * 1000L) { - intervalLengthMillis = 24L * 60L * 60L * 1000L; - } else if (this.now - responses.endMillis <= - 366L * 24L * 60L * 60L * 1000L) { - intervalLengthMillis = 2L * 24L * 60L * 60L * 1000L; - } else { - intervalLengthMillis = 10L * 24L * 60L * 60L * 1000L; - } - String monthString = dateTimeFormat.format(responses.startMillis); - if (lastResponses != null && - lastResponses.endMillis == responses.startMillis && - ((lastResponses.endMillis - 1L) / intervalLengthMillis) == - ((responses.endMillis - 1L) / intervalLengthMillis) && - lastMonthString.equals(monthString)) { - lastResponses.addResponses(responses); - } else { - if (lastResponses != null) { - compressedHistory.add(lastResponses); - } - lastResponses = responses; - } - lastMonthString = monthString; - } - if (lastResponses != null) { - compressedHistory.add(lastResponses); - } - clientsStatus.history = compressedHistory; - } - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - if (!relay) { - this.updateDocuments.addAll(fingerprints); - } - } - - private SortedSet<String> updateDocuments = new TreeSet<String>(); - - public void writeDocuments() { - for (String hashedFingerprint : this.updateDocuments) { - ClientsStatus clientsStatus = this.documentStore.retrieve( - ClientsStatus.class, true, hashedFingerprint); - if (clientsStatus == null) { - continue; - } - SortedSet<ClientsHistory> history = clientsStatus.history; - ClientsDocument clientsDocument = new ClientsDocument(); - clientsDocument.documentString = this.formatHistoryString( - hashedFingerprint, history); - this.documentStore.store(clientsDocument, hashedFingerprint); - } - Logger.printStatusTime("Wrote clients document files"); - } - - private String[] graphNames = new String[] { - "1_week", - "1_month", - "3_months", - "1_year", - "5_years" }; - - private long[] graphIntervals = new long[] { - 7L * 24L * 60L * 60L * 1000L, - 31L * 24L * 60L * 60L * 1000L, - 92L * 24L * 60L * 60L * 1000L, - 366L * 24L * 60L * 60L * 1000L, - 5L * 366L * 24L * 60L * 60L * 1000L }; - - private long[] dataPointIntervals = new long[] { - 24L * 60L * 60L * 1000L, - 24L * 60L * 60L * 1000L, - 24L * 60L * 60L * 1000L, - 2L * 24L * 60L * 60L * 1000L, - 10L * 24L * 60L * 60L * 1000L }; - - private String formatHistoryString(String hashedFingerprint, - SortedSet<ClientsHistory> history) { - StringBuilder sb = new StringBuilder(); - sb.append("{\"fingerprint\":\"" + hashedFingerprint + "\""); - sb.append(",\n\"average_clients\":{"); - int graphIntervalsWritten = 0; - for (int graphIntervalIndex = 0; graphIntervalIndex < - this.graphIntervals.length; graphIntervalIndex++) { - String timeline = this.formatTimeline(graphIntervalIndex, history); - if (timeline != null) { - sb.append((graphIntervalsWritten++ > 0 ? "," : "") + "\n" - + timeline); - } - } - sb.append("}"); - sb.append("\n}\n"); - return sb.toString(); - } - - private String formatTimeline(int graphIntervalIndex, - SortedSet<ClientsHistory> history) { - String graphName = this.graphNames[graphIntervalIndex]; - long graphInterval = this.graphIntervals[graphIntervalIndex]; - long dataPointInterval = - this.dataPointIntervals[graphIntervalIndex]; - List<Double> dataPoints = new ArrayList<Double>(); - long intervalStartMillis = ((this.now - graphInterval) - / dataPointInterval) * dataPointInterval; - long millis = 0L; - double responses = 0.0, totalResponses = 0.0; - SortedMap<String, Double> - totalResponsesByCountry = new TreeMap<String, Double>(), - totalResponsesByTransport = new TreeMap<String, Double>(), - totalResponsesByVersion = new TreeMap<String, Double>(); - for (ClientsHistory hist : history) { - if (hist.endMillis < intervalStartMillis) { - continue; - } - while ((intervalStartMillis / dataPointInterval) != - (hist.endMillis / dataPointInterval)) { - dataPoints.add(millis * 2L < dataPointInterval - ? -1.0 : responses * ((double) ONE_DAY_MILLIS) - / (((double) millis) * 10.0)); - responses = 0.0; - millis = 0L; - intervalStartMillis += dataPointInterval; - } - responses += hist.totalResponses; - totalResponses += hist.totalResponses; - for (Map.Entry<String, Double> e : - hist.responsesByCountry.entrySet()) { - if (!totalResponsesByCountry.containsKey(e.getKey())) { - totalResponsesByCountry.put(e.getKey(), 0.0); - } - totalResponsesByCountry.put(e.getKey(), e.getValue() - + totalResponsesByCountry.get(e.getKey())); - } - for (Map.Entry<String, Double> e : - hist.responsesByTransport.entrySet()) { - if (!totalResponsesByTransport.containsKey(e.getKey())) { - totalResponsesByTransport.put(e.getKey(), 0.0); - } - totalResponsesByTransport.put(e.getKey(), e.getValue() - + totalResponsesByTransport.get(e.getKey())); - } - for (Map.Entry<String, Double> e : - hist.responsesByVersion.entrySet()) { - if (!totalResponsesByVersion.containsKey(e.getKey())) { - totalResponsesByVersion.put(e.getKey(), 0.0); - } - totalResponsesByVersion.put(e.getKey(), e.getValue() - + totalResponsesByVersion.get(e.getKey())); - } - millis += (hist.endMillis - hist.startMillis); - } - dataPoints.add(millis * 2L < dataPointInterval - ? -1.0 : responses * ((double) ONE_DAY_MILLIS) - / (((double) millis) * 10.0)); - double maxValue = 0.0; - int firstNonNullIndex = -1, lastNonNullIndex = -1; - for (int dataPointIndex = 0; dataPointIndex < dataPoints.size(); - dataPointIndex++) { - double dataPoint = dataPoints.get(dataPointIndex); - if (dataPoint >= 0.0) { - if (firstNonNullIndex < 0) { - firstNonNullIndex = dataPointIndex; - } - lastNonNullIndex = dataPointIndex; - if (dataPoint > maxValue) { - maxValue = dataPoint; - } - } - } - if (firstNonNullIndex < 0) { - return null; - } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) * dataPointInterval - + dataPointInterval / 2L; - if (graphIntervalIndex > 0 && firstDataPointMillis >= - this.now - graphIntervals[graphIntervalIndex - 1]) { - /* Skip clients history object, because it doesn't contain - * anything new that wasn't already contained in the last - * clients history object(s). */ - return null; - } - long lastDataPointMillis = firstDataPointMillis - + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; - double factor = ((double) maxValue) / 999.0; - int count = lastNonNullIndex - firstNonNullIndex + 1; - StringBuilder sb = new StringBuilder(); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - sb.append("\"" + graphName + "\":{" - + "\"first\":\"" + dateTimeFormat.format(firstDataPointMillis) - + "\",\"last\":\"" + dateTimeFormat.format(lastDataPointMillis) - + "\",\"interval\":" + String.valueOf(dataPointInterval / 1000L) - + ",\"factor\":" + String.format(Locale.US, "%.9f", factor) - + ",\"count\":" + String.valueOf(count) + ",\"values\":["); - int dataPointsWritten = 0, previousNonNullIndex = -2; - boolean foundTwoAdjacentDataPoints = false; - for (int dataPointIndex = firstNonNullIndex; dataPointIndex <= - lastNonNullIndex; dataPointIndex++) { - double dataPoint = dataPoints.get(dataPointIndex); - if (dataPoint >= 0.0) { - if (dataPointIndex - previousNonNullIndex == 1) { - foundTwoAdjacentDataPoints = true; - } - previousNonNullIndex = dataPointIndex; - } - sb.append((dataPointsWritten++ > 0 ? "," : "") - + (dataPoint < 0.0 ? "null" : - String.valueOf((long) ((dataPoint * 999.0) / maxValue)))); - } - sb.append("]"); - if (!totalResponsesByCountry.isEmpty()) { - sb.append(",\"countries\":{"); - int written = 0; - for (Map.Entry<String, Double> e : - totalResponsesByCountry.entrySet()) { - if (e.getValue() > totalResponses / 100.0) { - sb.append((written++ > 0 ? "," : "") + "\"" + e.getKey() - + "\":" + String.format(Locale.US, "%.4f", - e.getValue() / totalResponses)); - } - } - sb.append("}"); - } - if (!totalResponsesByTransport.isEmpty()) { - sb.append(",\"transports\":{"); - int written = 0; - for (Map.Entry<String, Double> e : - totalResponsesByTransport.entrySet()) { - if (e.getValue() > totalResponses / 100.0) { - sb.append((written++ > 0 ? "," : "") + "\"" + e.getKey() - + "\":" + String.format(Locale.US, "%.4f", - e.getValue() / totalResponses)); - } - } - sb.append("}"); - } - if (!totalResponsesByVersion.isEmpty()) { - sb.append(",\"versions\":{"); - int written = 0; - for (Map.Entry<String, Double> e : - totalResponsesByVersion.entrySet()) { - if (e.getValue() > totalResponses / 100.0) { - sb.append((written++ > 0 ? "," : "") + "\"" + e.getKey() - + "\":" + String.format(Locale.US, "%.4f", - e.getValue() / totalResponses)); - } - } - sb.append("}"); - } - sb.append("}"); - if (foundTwoAdjacentDataPoints) { - return sb.toString(); - } else { - return null; - } - } - - public String getStatsString() { - int newIntervals = 0; - for (SortedSet<ClientsHistory> hist : this.newResponses.values()) { - newIntervals += hist.size(); - } - StringBuilder sb = new StringBuilder(); - sb.append(" " - + Logger.formatDecimalNumber(newIntervals / 2) - + " client statistics processed from extra-info descriptors\n"); - sb.append(" " - + Logger.formatDecimalNumber(this.newResponses.size()) - + " client status files updated\n"); - sb.append(" " - + Logger.formatDecimalNumber(this.updateDocuments.size()) - + " client document files updated\n"); - return sb.toString(); - } -} - diff --git a/src/org/torproject/onionoo/ClientsDocumentWriter.java b/src/org/torproject/onionoo/ClientsDocumentWriter.java new file mode 100644 index 0000000..9d3b8dc --- /dev/null +++ b/src/org/torproject/onionoo/ClientsDocumentWriter.java @@ -0,0 +1,302 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeMap; +import java.util.TreeSet; + +/* + * Clients status file produced as intermediate output: + * + * 2014-02-15 16:42:11 2014-02-16 00:00:00 + * 259.042 in=86.347,se=86.347 v4=259.042 + * 2014-02-16 00:00:00 2014-02-16 16:42:11 + * 592.958 in=197.653,se=197.653 v4=592.958 + * + * Clients document file produced as output: + * + * "1_month":{ + * "first":"2014-02-03 12:00:00", + * "last":"2014-02-28 12:00:00", + * "interval":86400, + * "factor":0.139049349, + * "count":26, + * "values":[371,354,349,374,432,null,485,458,493,536,null,null,524,576, + * 607,622,null,635,null,566,774,999,945,690,656,681], + * "countries":{"cn":0.0192,"in":0.1768,"ir":0.2487,"ru":0.0104, + * "se":0.1698,"sy":0.0325,"us":0.0406}, + * "transports":{"obfs2":0.4581}, + * "versions":{"v4":1.0000}} + */ +public class ClientsDocumentWriter implements FingerprintListener, + DocumentWriter { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + public ClientsDocumentWriter(DescriptorSource descriptorSource, + DocumentStore documentStore, Time time) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + this.now = time.currentTimeMillis(); + this.registerFingerprintListeners(); + } + + private void registerFingerprintListeners() { + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_EXTRA_INFOS); + } + + private SortedSet<String> updateDocuments = new TreeSet<String>(); + + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + if (!relay) { + this.updateDocuments.addAll(fingerprints); + } + } + + private int writtenDocuments = 0; + + public void writeDocuments() { + for (String hashedFingerprint : this.updateDocuments) { + ClientsStatus clientsStatus = this.documentStore.retrieve( + ClientsStatus.class, true, hashedFingerprint); + if (clientsStatus == null) { + continue; + } + SortedSet<ClientsHistory> history = clientsStatus.history; + ClientsDocument clientsDocument = new ClientsDocument(); + clientsDocument.documentString = this.formatHistoryString( + hashedFingerprint, history); + this.documentStore.store(clientsDocument, hashedFingerprint); + this.writtenDocuments++; + } + Logger.printStatusTime("Wrote clients document files"); + } + + private String[] graphNames = new String[] { + "1_week", + "1_month", + "3_months", + "1_year", + "5_years" }; + + private long[] graphIntervals = new long[] { + 7L * 24L * 60L * 60L * 1000L, + 31L * 24L * 60L * 60L * 1000L, + 92L * 24L * 60L * 60L * 1000L, + 366L * 24L * 60L * 60L * 1000L, + 5L * 366L * 24L * 60L * 60L * 1000L }; + + private long[] dataPointIntervals = new long[] { + 24L * 60L * 60L * 1000L, + 24L * 60L * 60L * 1000L, + 24L * 60L * 60L * 1000L, + 2L * 24L * 60L * 60L * 1000L, + 10L * 24L * 60L * 60L * 1000L }; + + private String formatHistoryString(String hashedFingerprint, + SortedSet<ClientsHistory> history) { + StringBuilder sb = new StringBuilder(); + sb.append("{\"fingerprint\":\"" + hashedFingerprint + "\""); + sb.append(",\n\"average_clients\":{"); + int graphIntervalsWritten = 0; + for (int graphIntervalIndex = 0; graphIntervalIndex < + this.graphIntervals.length; graphIntervalIndex++) { + String timeline = this.formatTimeline(graphIntervalIndex, history); + if (timeline != null) { + sb.append((graphIntervalsWritten++ > 0 ? "," : "") + "\n" + + timeline); + } + } + sb.append("}"); + sb.append("\n}\n"); + return sb.toString(); + } + + private static final long ONE_HOUR_MILLIS = 60L * 60L * 1000L, + ONE_DAY_MILLIS = 24L * ONE_HOUR_MILLIS; + + private String formatTimeline(int graphIntervalIndex, + SortedSet<ClientsHistory> history) { + String graphName = this.graphNames[graphIntervalIndex]; + long graphInterval = this.graphIntervals[graphIntervalIndex]; + long dataPointInterval = + this.dataPointIntervals[graphIntervalIndex]; + List<Double> dataPoints = new ArrayList<Double>(); + long intervalStartMillis = ((this.now - graphInterval) + / dataPointInterval) * dataPointInterval; + long millis = 0L; + double responses = 0.0, totalResponses = 0.0; + SortedMap<String, Double> + totalResponsesByCountry = new TreeMap<String, Double>(), + totalResponsesByTransport = new TreeMap<String, Double>(), + totalResponsesByVersion = new TreeMap<String, Double>(); + for (ClientsHistory hist : history) { + if (hist.endMillis < intervalStartMillis) { + continue; + } + while ((intervalStartMillis / dataPointInterval) != + (hist.endMillis / dataPointInterval)) { + dataPoints.add(millis * 2L < dataPointInterval + ? -1.0 : responses * ((double) ONE_DAY_MILLIS) + / (((double) millis) * 10.0)); + responses = 0.0; + millis = 0L; + intervalStartMillis += dataPointInterval; + } + responses += hist.totalResponses; + totalResponses += hist.totalResponses; + for (Map.Entry<String, Double> e : + hist.responsesByCountry.entrySet()) { + if (!totalResponsesByCountry.containsKey(e.getKey())) { + totalResponsesByCountry.put(e.getKey(), 0.0); + } + totalResponsesByCountry.put(e.getKey(), e.getValue() + + totalResponsesByCountry.get(e.getKey())); + } + for (Map.Entry<String, Double> e : + hist.responsesByTransport.entrySet()) { + if (!totalResponsesByTransport.containsKey(e.getKey())) { + totalResponsesByTransport.put(e.getKey(), 0.0); + } + totalResponsesByTransport.put(e.getKey(), e.getValue() + + totalResponsesByTransport.get(e.getKey())); + } + for (Map.Entry<String, Double> e : + hist.responsesByVersion.entrySet()) { + if (!totalResponsesByVersion.containsKey(e.getKey())) { + totalResponsesByVersion.put(e.getKey(), 0.0); + } + totalResponsesByVersion.put(e.getKey(), e.getValue() + + totalResponsesByVersion.get(e.getKey())); + } + millis += (hist.endMillis - hist.startMillis); + } + dataPoints.add(millis * 2L < dataPointInterval + ? -1.0 : responses * ((double) ONE_DAY_MILLIS) + / (((double) millis) * 10.0)); + double maxValue = 0.0; + int firstNonNullIndex = -1, lastNonNullIndex = -1; + for (int dataPointIndex = 0; dataPointIndex < dataPoints.size(); + dataPointIndex++) { + double dataPoint = dataPoints.get(dataPointIndex); + if (dataPoint >= 0.0) { + if (firstNonNullIndex < 0) { + firstNonNullIndex = dataPointIndex; + } + lastNonNullIndex = dataPointIndex; + if (dataPoint > maxValue) { + maxValue = dataPoint; + } + } + } + if (firstNonNullIndex < 0) { + return null; + } + long firstDataPointMillis = (((this.now - graphInterval) + / dataPointInterval) + firstNonNullIndex) * dataPointInterval + + dataPointInterval / 2L; + if (graphIntervalIndex > 0 && firstDataPointMillis >= + this.now - graphIntervals[graphIntervalIndex - 1]) { + /* Skip clients history object, because it doesn't contain + * anything new that wasn't already contained in the last + * clients history object(s). */ + return null; + } + long lastDataPointMillis = firstDataPointMillis + + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; + double factor = ((double) maxValue) / 999.0; + int count = lastNonNullIndex - firstNonNullIndex + 1; + StringBuilder sb = new StringBuilder(); + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + sb.append("\"" + graphName + "\":{" + + "\"first\":\"" + dateTimeFormat.format(firstDataPointMillis) + + "\",\"last\":\"" + dateTimeFormat.format(lastDataPointMillis) + + "\",\"interval\":" + String.valueOf(dataPointInterval / 1000L) + + ",\"factor\":" + String.format(Locale.US, "%.9f", factor) + + ",\"count\":" + String.valueOf(count) + ",\"values\":["); + int dataPointsWritten = 0, previousNonNullIndex = -2; + boolean foundTwoAdjacentDataPoints = false; + for (int dataPointIndex = firstNonNullIndex; dataPointIndex <= + lastNonNullIndex; dataPointIndex++) { + double dataPoint = dataPoints.get(dataPointIndex); + if (dataPoint >= 0.0) { + if (dataPointIndex - previousNonNullIndex == 1) { + foundTwoAdjacentDataPoints = true; + } + previousNonNullIndex = dataPointIndex; + } + sb.append((dataPointsWritten++ > 0 ? "," : "") + + (dataPoint < 0.0 ? "null" : + String.valueOf((long) ((dataPoint * 999.0) / maxValue)))); + } + sb.append("]"); + if (!totalResponsesByCountry.isEmpty()) { + sb.append(",\"countries\":{"); + int written = 0; + for (Map.Entry<String, Double> e : + totalResponsesByCountry.entrySet()) { + if (e.getValue() > totalResponses / 100.0) { + sb.append((written++ > 0 ? "," : "") + "\"" + e.getKey() + + "\":" + String.format(Locale.US, "%.4f", + e.getValue() / totalResponses)); + } + } + sb.append("}"); + } + if (!totalResponsesByTransport.isEmpty()) { + sb.append(",\"transports\":{"); + int written = 0; + for (Map.Entry<String, Double> e : + totalResponsesByTransport.entrySet()) { + if (e.getValue() > totalResponses / 100.0) { + sb.append((written++ > 0 ? "," : "") + "\"" + e.getKey() + + "\":" + String.format(Locale.US, "%.4f", + e.getValue() / totalResponses)); + } + } + sb.append("}"); + } + if (!totalResponsesByVersion.isEmpty()) { + sb.append(",\"versions\":{"); + int written = 0; + for (Map.Entry<String, Double> e : + totalResponsesByVersion.entrySet()) { + if (e.getValue() > totalResponses / 100.0) { + sb.append((written++ > 0 ? "," : "") + "\"" + e.getKey() + + "\":" + String.format(Locale.US, "%.4f", + e.getValue() / totalResponses)); + } + } + sb.append("}"); + } + sb.append("}"); + if (foundTwoAdjacentDataPoints) { + return sb.toString(); + } else { + return null; + } + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments) + + " clients document files updated\n"); + return sb.toString(); + } +} diff --git a/src/org/torproject/onionoo/ClientsStatusUpdater.java b/src/org/torproject/onionoo/ClientsStatusUpdater.java new file mode 100644 index 0000000..e15c11a --- /dev/null +++ b/src/org/torproject/onionoo/ClientsStatusUpdater.java @@ -0,0 +1,230 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.text.SimpleDateFormat; +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.ExtraInfoDescriptor; + +/* + * Example extra-info descriptor used as input: + * + * extra-info ndnop2 DE6397A047ABE5F78B4C87AF725047831B221AAB + * dirreq-stats-end 2014-02-16 16:42:11 (86400 s) + * dirreq-v3-resp ok=856,not-enough-sigs=0,unavailable=0,not-found=0, + * not-modified=40,busy=0 + * bridge-stats-end 2014-02-16 16:42:17 (86400 s) + * bridge-ips ??=8,in=8,se=8 + * bridge-ip-versions v4=8,v6=0 + * + * Clients status file produced as intermediate output: + * + * 2014-02-15 16:42:11 2014-02-16 00:00:00 + * 259.042 in=86.347,se=86.347 v4=259.042 + * 2014-02-16 00:00:00 2014-02-16 16:42:11 + * 592.958 in=197.653,se=197.653 v4=592.958 + */ +public class ClientsStatusUpdater implements DescriptorListener, + StatusUpdater { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + public ClientsStatusUpdater(DescriptorSource descriptorSource, + DocumentStore documentStore, Time time) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + this.now = time.currentTimeMillis(); + this.registerDescriptorListeners(); + } + + private void registerDescriptorListeners() { + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_EXTRA_INFOS); + } + + public void processDescriptor(Descriptor descriptor, boolean relay) { + if (descriptor instanceof ExtraInfoDescriptor && !relay) { + this.processBridgeExtraInfoDescriptor( + (ExtraInfoDescriptor) descriptor); + } + } + + private static final long ONE_HOUR_MILLIS = 60L * 60L * 1000L, + ONE_DAY_MILLIS = 24L * ONE_HOUR_MILLIS; + + private SortedMap<String, SortedSet<ClientsHistory>> newResponses = + new TreeMap<String, SortedSet<ClientsHistory>>(); + + private void processBridgeExtraInfoDescriptor( + ExtraInfoDescriptor descriptor) { + long dirreqStatsEndMillis = descriptor.getDirreqStatsEndMillis(); + long dirreqStatsIntervalLengthMillis = + descriptor.getDirreqStatsIntervalLength() * 1000L; + SortedMap<String, Integer> responses = descriptor.getDirreqV3Resp(); + if (dirreqStatsEndMillis < 0L || + dirreqStatsIntervalLengthMillis != ONE_DAY_MILLIS || + responses == null || !responses.containsKey("ok")) { + return; + } + double okResponses = (double) (responses.get("ok") - 4); + if (okResponses < 0.0) { + return; + } + String hashedFingerprint = descriptor.getFingerprint().toUpperCase(); + long dirreqStatsStartMillis = dirreqStatsEndMillis + - dirreqStatsIntervalLengthMillis; + long utcBreakMillis = (dirreqStatsEndMillis / ONE_DAY_MILLIS) + * ONE_DAY_MILLIS; + for (int i = 0; i < 2; i++) { + long startMillis = i == 0 ? dirreqStatsStartMillis : utcBreakMillis; + long endMillis = i == 0 ? utcBreakMillis : dirreqStatsEndMillis; + if (startMillis >= endMillis) { + continue; + } + double totalResponses = okResponses + * ((double) (endMillis - startMillis)) + / ((double) ONE_DAY_MILLIS); + SortedMap<String, Double> responsesByCountry = + this.weightResponsesWithUniqueIps(totalResponses, + descriptor.getBridgeIps(), "??"); + SortedMap<String, Double> responsesByTransport = + this.weightResponsesWithUniqueIps(totalResponses, + descriptor.getBridgeIpTransports(), "<??>"); + SortedMap<String, Double> responsesByVersion = + this.weightResponsesWithUniqueIps(totalResponses, + descriptor.getBridgeIpVersions(), ""); + ClientsHistory newResponseHistory = new ClientsHistory( + startMillis, endMillis, totalResponses, responsesByCountry, + responsesByTransport, responsesByVersion); + if (!this.newResponses.containsKey(hashedFingerprint)) { + this.newResponses.put(hashedFingerprint, + new TreeSet<ClientsHistory>()); + } + this.newResponses.get(hashedFingerprint).add( + newResponseHistory); + } + } + + private SortedMap<String, Double> weightResponsesWithUniqueIps( + double totalResponses, SortedMap<String, Integer> uniqueIps, + String omitString) { + SortedMap<String, Double> weightedResponses = + new TreeMap<String, Double>(); + int totalUniqueIps = 0; + if (uniqueIps != null) { + for (Map.Entry<String, Integer> e : uniqueIps.entrySet()) { + if (e.getValue() > 4) { + totalUniqueIps += e.getValue() - 4; + } + } + } + if (totalUniqueIps > 0) { + for (Map.Entry<String, Integer> e : uniqueIps.entrySet()) { + if (!e.getKey().equals(omitString) && e.getValue() > 4) { + weightedResponses.put(e.getKey(), + (((double) (e.getValue() - 4)) * totalResponses) + / ((double) totalUniqueIps)); + } + } + } + return weightedResponses; + } + + public void updateStatuses() { + for (Map.Entry<String, SortedSet<ClientsHistory>> e : + this.newResponses.entrySet()) { + String hashedFingerprint = e.getKey(); + ClientsStatus clientsStatus = this.documentStore.retrieve( + ClientsStatus.class, true, hashedFingerprint); + if (clientsStatus == null) { + clientsStatus = new ClientsStatus(); + } + this.addToHistory(clientsStatus, e.getValue()); + this.compressHistory(clientsStatus); + this.documentStore.store(clientsStatus, hashedFingerprint); + } + Logger.printStatusTime("Updated clients status files"); + } + + private void addToHistory(ClientsStatus clientsStatus, + SortedSet<ClientsHistory> newIntervals) { + SortedSet<ClientsHistory> history = clientsStatus.history; + for (ClientsHistory interval : newIntervals) { + if ((history.headSet(interval).isEmpty() || + history.headSet(interval).last().endMillis <= + interval.startMillis) && + (history.tailSet(interval).isEmpty() || + history.tailSet(interval).first().startMillis >= + interval.endMillis)) { + history.add(interval); + } + } + } + + private void compressHistory(ClientsStatus clientsStatus) { + SortedSet<ClientsHistory> history = clientsStatus.history; + SortedSet<ClientsHistory> compressedHistory = + new TreeSet<ClientsHistory>(); + ClientsHistory lastResponses = null; + SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + String lastMonthString = "1970-01"; + for (ClientsHistory responses : history) { + long intervalLengthMillis; + if (this.now - responses.endMillis <= + 92L * 24L * 60L * 60L * 1000L) { + intervalLengthMillis = 24L * 60L * 60L * 1000L; + } else if (this.now - responses.endMillis <= + 366L * 24L * 60L * 60L * 1000L) { + intervalLengthMillis = 2L * 24L * 60L * 60L * 1000L; + } else { + intervalLengthMillis = 10L * 24L * 60L * 60L * 1000L; + } + String monthString = dateTimeFormat.format(responses.startMillis); + if (lastResponses != null && + lastResponses.endMillis == responses.startMillis && + ((lastResponses.endMillis - 1L) / intervalLengthMillis) == + ((responses.endMillis - 1L) / intervalLengthMillis) && + lastMonthString.equals(monthString)) { + lastResponses.addResponses(responses); + } else { + if (lastResponses != null) { + compressedHistory.add(lastResponses); + } + lastResponses = responses; + } + lastMonthString = monthString; + } + if (lastResponses != null) { + compressedHistory.add(lastResponses); + } + clientsStatus.history = compressedHistory; + } + + public String getStatsString() { + int newIntervals = 0; + for (SortedSet<ClientsHistory> hist : this.newResponses.values()) { + newIntervals += hist.size(); + } + StringBuilder sb = new StringBuilder(); + sb.append(" " + + Logger.formatDecimalNumber(newIntervals / 2) + + " client statistics processed from extra-info descriptors\n"); + sb.append(" " + + Logger.formatDecimalNumber(this.newResponses.size()) + + " client status files updated\n"); + return sb.toString(); + } +} + diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 1a5d083..af33124 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -36,12 +36,14 @@ public class Main { Logger.printStatusTime("Initialized bandwidth data writer"); WeightsDataWriter wdw = new WeightsDataWriter(dso, ds, t); Logger.printStatusTime("Initialized weights data writer"); - ClientsDataWriter cdw = new ClientsDataWriter(dso, ds, t); - Logger.printStatusTime("Initialized clients data writer"); + ClientsStatusUpdater csu = new ClientsStatusUpdater(dso, ds, t); + Logger.printStatusTime("Initialized clients status updater"); UptimeStatusUpdater usu = new UptimeStatusUpdater(dso, ds); Logger.printStatusTime("Initialized uptime status updater"); - StatusUpdater[] sus = new StatusUpdater[] { ndw, bdw, wdw, cdw, usu }; + StatusUpdater[] sus = new StatusUpdater[] { ndw, bdw, wdw, csu, usu }; + ClientsDocumentWriter cdw = new ClientsDocumentWriter(dso, ds, t); + Logger.printStatusTime("Initialized clients document writer"); UptimeDocumentWriter udw = new UptimeDocumentWriter(dso, ds, t); Logger.printStatusTime("Initialized uptime document writer"); DocumentWriter[] dws = new DocumentWriter[] { ndw, bdw, wdw, cdw, @@ -91,7 +93,7 @@ public class Main { } /* TODO Print status updater statistics for *all* status updaters once * all data writers have been separated. */ - for (DocumentWriter dw : new DocumentWriter[] { udw }) { + for (DocumentWriter dw : new DocumentWriter[] { cdw, udw }) { String statsString = dw.getStatsString(); if (statsString != null) { Logger.printStatistics(dw.getClass().getSimpleName(),

1 0

[onionoo/master] Move (de-)serialization code into WeightsStatus.
by karsten＠torproject.org 11 Apr '14

11 Apr '14

commit d3638d4c9cf350ab5eb6f319e453a94163838236 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Apr 8 23:58:18 2014 +0200 Move (de-)serialization code into WeightsStatus. --- src/org/torproject/onionoo/DocumentStore.java | 6 +- src/org/torproject/onionoo/WeightsDataWriter.java | 161 ++++++--------------- src/org/torproject/onionoo/WeightsStatus.java | 91 ++++++++++++ 3 files changed, 143 insertions(+), 115 deletions(-) diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java index 387f34c..5da7267 100644 --- a/src/org/torproject/onionoo/DocumentStore.java +++ b/src/org/torproject/onionoo/DocumentStore.java @@ -196,7 +196,8 @@ public class DocumentStore { document instanceof UptimeDocument) { Gson gson = new Gson(); documentString = gson.toJson(this); - } else if (document instanceof ClientsStatus || + } else if (document instanceof WeightsStatus || + document instanceof ClientsStatus || document instanceof UptimeStatus) { documentString = document.toDocumentString(); } else { @@ -289,7 +290,8 @@ public class DocumentStore { documentType.equals(UptimeDocument.class)) { return this.retrieveParsedDocumentFile(documentType, documentString); - } else if (documentType.equals(ClientsStatus.class) || + } else if (documentType.equals(WeightsStatus.class) || + documentType.equals(ClientsStatus.class) || documentType.equals(UptimeStatus.class)) { return this.retrieveParsedStatusFile(documentType, documentString); } else { diff --git a/src/org/torproject/onionoo/WeightsDataWriter.java b/src/org/torproject/onionoo/WeightsDataWriter.java index d230662..0d7b815 100644 --- a/src/org/torproject/onionoo/WeightsDataWriter.java +++ b/src/org/torproject/onionoo/WeightsDataWriter.java @@ -2,17 +2,14 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; -import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Scanner; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; @@ -230,7 +227,19 @@ public class WeightsDataWriter implements DescriptorListener, double advertisedBandwidth = 0.0; if (!this.advertisedBandwidths.containsKey( serverDescriptorDigest)) { - this.readHistoryFromDisk(fingerprint); + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus != null) { + if (!this.descriptorDigestsByFingerprint.containsKey( + fingerprint)) { + this.descriptorDigestsByFingerprint.put(fingerprint, + new HashSet<String>()); + } + this.descriptorDigestsByFingerprint.get(fingerprint).addAll( + weightsStatus.advertisedBandwidths.keySet()); + this.advertisedBandwidths.putAll( + weightsStatus.advertisedBandwidths); + } } if (this.advertisedBandwidths.containsKey( serverDescriptorDigest)) { @@ -288,93 +297,42 @@ public class WeightsDataWriter implements DescriptorListener, private void addToHistory(String fingerprint, long validAfterMillis, long freshUntilMillis, double[] weights) { - SortedMap<long[], double[]> history = - this.readHistoryFromDisk(fingerprint); + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus == null) { + weightsStatus = new WeightsStatus(); + } + SortedMap<long[], double[]> history = weightsStatus.history; long[] interval = new long[] { validAfterMillis, freshUntilMillis }; if ((history.headMap(interval).isEmpty() || history.headMap(interval).lastKey()[1] <= validAfterMillis) && (history.tailMap(interval).isEmpty() || history.tailMap(interval).firstKey()[0] >= freshUntilMillis)) { history.put(interval, weights); - history = this.compressHistory(history); - this.writeHistoryToDisk(fingerprint, history); + this.compressHistory(weightsStatus); + this.addAdvertisedBandwidths(weightsStatus, fingerprint); + this.documentStore.store(weightsStatus, fingerprint); this.updateWeightsStatuses.remove(fingerprint); } } - private SortedMap<long[], double[]> readHistoryFromDisk( + private void addAdvertisedBandwidths(WeightsStatus weightsStatus, String fingerprint) { - SortedMap<long[], double[]> history = - new TreeMap<long[], double[]>(new Comparator<long[]>() { - public int compare(long[] a, long[] b) { - return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0; - } - }); - WeightsStatus weightsStatus = this.documentStore.retrieve( - WeightsStatus.class, false, fingerprint); - if (weightsStatus != null) { - String historyString = weightsStatus.documentString; - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setLenient(false); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - try { - Scanner s = new Scanner(historyString); - while (s.hasNextLine()) { - String line = s.nextLine(); - String[] parts = line.split(" "); - if (parts.length == 2) { - String descriptorDigest = parts[0]; - int advertisedBandwidth = Integer.parseInt(parts[1]); - if (!this.descriptorDigestsByFingerprint.containsKey( - fingerprint)) { - this.descriptorDigestsByFingerprint.put(fingerprint, - new HashSet<String>()); - } - this.descriptorDigestsByFingerprint.get(fingerprint).add( - descriptorDigest); - this.advertisedBandwidths.put(descriptorDigest, - advertisedBandwidth); - continue; - } - if (parts.length != 9) { - System.err.println("Illegal line '" + line + "' in weights " - + "history for fingerprint '" + fingerprint + "'. " - + "Skipping this line."); - continue; - } - if (parts[4].equals("NaN")) { - /* Remove corrupt lines written on 2013-07-07 and the days - * after. */ - continue; - } - long validAfterMillis = dateTimeFormat.parse(parts[0] - + " " + parts[1]).getTime(); - long freshUntilMillis = dateTimeFormat.parse(parts[2] - + " " + parts[3]).getTime(); - long[] interval = new long[] { validAfterMillis, - freshUntilMillis }; - double[] weights = new double[] { - Double.parseDouble(parts[4]), - Double.parseDouble(parts[5]), - Double.parseDouble(parts[6]), - Double.parseDouble(parts[7]), - Double.parseDouble(parts[8]) }; - history.put(interval, weights); + if (this.descriptorDigestsByFingerprint.containsKey(fingerprint)) { + for (String descriptorDigest : + this.descriptorDigestsByFingerprint.get(fingerprint)) { + if (this.advertisedBandwidths.containsKey(descriptorDigest)) { + int advertisedBandwidth = + this.advertisedBandwidths.get(descriptorDigest); + weightsStatus.advertisedBandwidths.put(descriptorDigest, + advertisedBandwidth); } - s.close(); - } catch (ParseException e) { - System.err.println("Could not parse timestamp while reading " - + "weights history for fingerprint '" + fingerprint + "'. " - + "Skipping."); - e.printStackTrace(); } } - return history; } - private SortedMap<long[], double[]> compressHistory( - SortedMap<long[], double[]> history) { + private void compressHistory(WeightsStatus weightsStatus) { + SortedMap<long[], double[]> history = weightsStatus.history; SortedMap<long[], double[]> compressedHistory = new TreeMap<long[], double[]>(history.comparator()); long lastStartMillis = 0L, lastEndMillis = 0L; @@ -429,39 +387,7 @@ public class WeightsDataWriter implements DescriptorListener, compressedHistory.put(new long[] { lastStartMillis, lastEndMillis }, lastWeights); } - return compressedHistory; - } - - private void writeHistoryToDisk(String fingerprint, - SortedMap<long[], double[]> history) { - StringBuilder sb = new StringBuilder(); - if (this.descriptorDigestsByFingerprint.containsKey(fingerprint)) { - for (String descriptorDigest : - this.descriptorDigestsByFingerprint.get(fingerprint)) { - if (this.advertisedBandwidths.containsKey(descriptorDigest)) { - int advertisedBandwidth = - this.advertisedBandwidths.get(descriptorDigest); - sb.append(descriptorDigest + " " - + String.valueOf(advertisedBandwidth) + "\n"); - } - } - } - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - for (Map.Entry<long[], double[]> e : history.entrySet()) { - long[] fresh = e.getKey(); - double[] weights = e.getValue(); - sb.append(dateTimeFormat.format(fresh[0]) + " " - + dateTimeFormat.format(fresh[1])); - for (double weight : weights) { - sb.append(String.format(" %.12f", weight)); - } - sb.append("\n"); - } - WeightsStatus weightsStatus = new WeightsStatus(); - weightsStatus.documentString = sb.toString(); - this.documentStore.store(weightsStatus, fingerprint); + weightsStatus.history = compressedHistory; } public void processFingerprints(SortedSet<String> fingerprints, @@ -473,13 +399,18 @@ public class WeightsDataWriter implements DescriptorListener, private void writeWeightsDataFiles() { for (String fingerprint : this.updateWeightsDocuments) { - SortedMap<long[], double[]> history = - this.readHistoryFromDisk(fingerprint); + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus == null) { + continue; + } + SortedMap<long[], double[]> history = weightsStatus.history; WeightsDocument weightsDocument = new WeightsDocument(); weightsDocument.documentString = this.formatHistoryString( fingerprint, history); this.documentStore.store(weightsDocument, fingerprint); } + Logger.printStatusTime("Wrote weights document files"); } private String[] graphTypes = new String[] { @@ -633,9 +564,13 @@ public class WeightsDataWriter implements DescriptorListener, private void updateWeightsStatuses() { for (String fingerprint : this.updateWeightsStatuses) { - SortedMap<long[], double[]> history = - this.readHistoryFromDisk(fingerprint); - this.writeHistoryToDisk(fingerprint, history); + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus == null) { + weightsStatus = new WeightsStatus(); + } + this.addAdvertisedBandwidths(weightsStatus, fingerprint); + this.documentStore.store(weightsStatus, fingerprint); } } diff --git a/src/org/torproject/onionoo/WeightsStatus.java b/src/org/torproject/onionoo/WeightsStatus.java index f8f78ad..4d92f30 100644 --- a/src/org/torproject/onionoo/WeightsStatus.java +++ b/src/org/torproject/onionoo/WeightsStatus.java @@ -1,5 +1,96 @@ package org.torproject.onionoo; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.Scanner; +import java.util.SortedMap; +import java.util.TimeZone; +import java.util.TreeMap; + class WeightsStatus extends Document { + + SortedMap<long[], double[]> history = new TreeMap<long[], double[]>( + new Comparator<long[]>() { + public int compare(long[] a, long[] b) { + return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0; + } + }); + + Map<String, Integer> advertisedBandwidths = + new HashMap<String, Integer>(); + + public void fromDocumentString(String documentString) { + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setLenient(false); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + try { + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + String[] parts = line.split(" "); + if (parts.length == 2) { + String descriptorDigest = parts[0]; + int advertisedBandwidth = Integer.parseInt(parts[1]); + this.advertisedBandwidths.put(descriptorDigest, + advertisedBandwidth); + continue; + } + if (parts.length != 9) { + System.err.println("Illegal line '" + line + "' in weights " + + "status file. Skipping this line."); + continue; + } + if (parts[4].equals("NaN")) { + /* Remove corrupt lines written on 2013-07-07 and the days + * after. */ + continue; + } + long validAfterMillis = dateTimeFormat.parse(parts[0] + + " " + parts[1]).getTime(); + long freshUntilMillis = dateTimeFormat.parse(parts[2] + + " " + parts[3]).getTime(); + long[] interval = new long[] { validAfterMillis, + freshUntilMillis }; + double[] weights = new double[] { + Double.parseDouble(parts[4]), + Double.parseDouble(parts[5]), + Double.parseDouble(parts[6]), + Double.parseDouble(parts[7]), + Double.parseDouble(parts[8]) }; + this.history.put(interval, weights); + } + s.close(); + } catch (ParseException e) { + System.err.println("Could not parse timestamp while reading " + + "weights status file. Skipping."); + e.printStackTrace(); + } + } + + public String toDocumentString() { + StringBuilder sb = new StringBuilder(); + for (Map.Entry<String, Integer> e : + this.advertisedBandwidths.entrySet()) { + sb.append(e.getKey() + " " + String.valueOf(e.getValue()) + "\n"); + } + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + for (Map.Entry<long[], double[]> e : history.entrySet()) { + long[] fresh = e.getKey(); + double[] weights = e.getValue(); + sb.append(dateTimeFormat.format(fresh[0]) + " " + + dateTimeFormat.format(fresh[1])); + for (double weight : weights) { + sb.append(String.format(" %.12f", weight)); + } + sb.append("\n"); + } + return sb.toString(); + } }

1 0