commit 2d1c226bfcd160ff91a98b36cbe2ac0cb3a83a79 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Sun May 25 08:34:33 2014 +0200 Switch to parsing concatenated descriptors from rsync recent. --- bin/update.sh | 2 +- build.xml | 2 +- deps/metrics-lib | 2 +- src/org/torproject/onionoo/DescriptorSource.java | 17 +++++++++++++---- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/bin/update.sh b/bin/update.sh index 0d48e59..9f1424f 100755 --- a/bin/update.sh +++ b/bin/update.sh @@ -1,4 +1,4 @@ #!/bin/bash -rsync -az --delete metrics.torproject.org::'metrics-recent/relay-descriptors/consensuses/ metrics-recent/relay-descriptors/server-descriptors/ metrics-recent/relay-descriptors/extra-infos/ metrics-recent/bridge-descriptors/statuses/ metrics-recent/bridge-descriptors/server-descriptors/ metrics-recent/bridge-descriptors/extra-infos/ metrics-recent/bridge-pool-assignments/ metrics-recent/exit-lists' in +rsync -az --delete --exclude 'relay-descriptors/votes' --exclude 'relay-descriptors/microdescs' --exclude 'relay-descriptors/server-descriptors' --exclude 'relay-descriptors/extra-infos' --exclude 'bridge-descriptors/server-descriptors' --exclude 'bridge-descriptors/extra-infos' --exclude 'torperf' metrics.torproject.org::metrics-recent in ant run >> log diff --git a/build.xml b/build.xml index e772da4..500204d 100644 --- a/build.xml +++ b/build.xml @@ -92,7 +92,7 @@ </target> <target name="run" depends="compile"> <java fork="true" - maxmemory="2048m" + maxmemory="4g" classname="org.torproject.onionoo.Main"> <classpath refid="classpath"/> </java> diff --git a/deps/metrics-lib b/deps/metrics-lib index 3e60ccd..a472d7f 160000 --- a/deps/metrics-lib +++ b/deps/metrics-lib @@ -1 +1 @@ -Subproject commit 3e60ccdaaba598cabb7281d45f9a415299b8e3e3 +Subproject commit a472d7f3425feec9b9bed552b05af33dab20fee1 diff --git a/src/org/torproject/onionoo/DescriptorSource.java b/src/org/torproject/onionoo/DescriptorSource.java index f4ff2fc..0692854 100644 --- a/src/org/torproject/onionoo/DescriptorSource.java +++ b/src/org/torproject/onionoo/DescriptorSource.java @@ -106,24 +106,29 @@ class DescriptorQueue { public void addDirectory(DescriptorType descriptorType) { String directoryName = null; + int maxDescriptorFilesInQueue = -1; switch (descriptorType) { case RELAY_CONSENSUSES: directoryName = "relay-descriptors/consensuses"; break; case RELAY_SERVER_DESCRIPTORS: - directoryName = "relay-descriptors/server-descriptors"; + directoryName = "relay-descriptors/server-descriptors-cat"; + maxDescriptorFilesInQueue = 1; break; case RELAY_EXTRA_INFOS: - directoryName = "relay-descriptors/extra-infos"; + directoryName = "relay-descriptors/extra-infos-cat"; + maxDescriptorFilesInQueue = 1; break; case BRIDGE_STATUSES: directoryName = "bridge-descriptors/statuses"; break; case BRIDGE_SERVER_DESCRIPTORS: - directoryName = "bridge-descriptors/server-descriptors"; + directoryName = "bridge-descriptors/server-descriptors-cat"; + maxDescriptorFilesInQueue = 1; break; case BRIDGE_EXTRA_INFOS: - directoryName = "bridge-descriptors/extra-infos"; + directoryName = "bridge-descriptors/extra-infos-cat"; + maxDescriptorFilesInQueue = 1; break; case BRIDGE_POOL_ASSIGNMENTS: directoryName = "bridge-pool-assignments"; @@ -139,6 +144,10 @@ class DescriptorQueue { File directory = new File(this.inDir, directoryName); if (directory.exists() && directory.isDirectory()) { this.descriptorReader.addDirectory(directory); + if (maxDescriptorFilesInQueue >= 1) { + this.descriptorReader.setMaxDescriptorFilesInQueue( + maxDescriptorFilesInQueue); + } } else { System.err.println("Directory " + directory.getAbsolutePath() + " either does not exist or is not a directory. Not adding "