December 2017 - tor-commits - lists.torproject.org

[metrics-web/master] Create ant task for running connbidirect module.
by karsten＠torproject.org 20 Dec '17

20 Dec '17

commit 478f35413065bf1f6b32a2bafcbe654099c1e6d1 Author: iwakeh <iwakeh(a)torproject.org> Date: Mon Dec 4 11:14:21 2017 +0000 Create ant task for running connbidirect module. Move the sources and test sources. Delete old script and build.xml. Create task connbidirect. --- build.xml | 7 ++++++- modules/connbidirect/.gitignore | 3 --- modules/connbidirect/build.xml | 13 ------------- shared/bin/10-run-connbidirect-stats.sh | 5 ----- .../main/java/org/torproject/metrics/connbidirect/Main.java | 0 .../java/org/torproject/metrics/connbidirect/MainTest.java | 0 6 files changed, 6 insertions(+), 22 deletions(-) diff --git a/build.xml b/build.xml index 7f2abe9..0ed97ba 100644 --- a/build.xml +++ b/build.xml @@ -86,6 +86,11 @@ <antcall target="run-java" /> </target> + <target name="connbidirect" > + <property name="module.name" value="connbidirect" /> + <antcall target="run-java" /> + </target> + <target name="run-rserver"> <exec executable="R" dir="${Rsources}/rserver" @@ -103,7 +108,7 @@ <target name="run-java"> <echo message="Running java module ${module.name} ... " /> <available file="${dist}/${jarfile}" property="have.jar"/> - <fail unless="have.jar" message="Please run 'ant jar'."/> + <fail unless="have.jar" message="Please run 'ant jar' first."/> <condition property="mainclass" value="${module.main}" else="org.torproject.metrics.${module.name}.Main" > diff --git a/modules/connbidirect/.gitignore b/modules/connbidirect/.gitignore deleted file mode 100644 index b359b59..0000000 --- a/modules/connbidirect/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -classes/ -stats/ - diff --git a/modules/connbidirect/build.xml b/modules/connbidirect/build.xml deleted file mode 100644 index 4c9c1f4..0000000 --- a/modules/connbidirect/build.xml +++ /dev/null @@ -1,13 +0,0 @@ -<project default="run" name="connbidirect" basedir="."> - - <property name="mainclass" value="org.torproject.metrics.connbidirect.Main"/> - - <include file="../../shared/build-base.xml" as="basetask"/> - <target name="clean" depends="basetask.clean"/> - <target name="compile" depends="basetask.compile"/> - <target name="testcompile" depends="basetask.testcompile"/> - <target name="test" depends="basetask.test"/> - <target name="run" depends="basetask.run"/> - -</project> - diff --git a/shared/bin/10-run-connbidirect-stats.sh b/shared/bin/10-run-connbidirect-stats.sh deleted file mode 100755 index d59c315..0000000 --- a/shared/bin/10-run-connbidirect-stats.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -cd modules/connbidirect/ -ant | grep "\[java\]" -cd ../../ - diff --git a/modules/connbidirect/src/main/java/org/torproject/metrics/connbidirect/Main.java b/src/main/java/org/torproject/metrics/connbidirect/Main.java similarity index 100% rename from modules/connbidirect/src/main/java/org/torproject/metrics/connbidirect/Main.java rename to src/main/java/org/torproject/metrics/connbidirect/Main.java diff --git a/modules/connbidirect/src/test/java/org/torproject/metrics/connbidirect/MainTest.java b/src/test/java/org/torproject/metrics/connbidirect/MainTest.java similarity index 100% rename from modules/connbidirect/src/test/java/org/torproject/metrics/connbidirect/MainTest.java rename to src/test/java/org/torproject/metrics/connbidirect/MainTest.java

1 0

[metrics-web/master] Begin creation of standard build environment.
by karsten＠torproject.org 20 Dec '17

20 Dec '17

commit c350ba41caaf910d345e130de89aec25af90312a Author: iwakeh <iwakeh(a)torproject.org> Date: Mon Dec 4 11:14:12 2017 +0000 Begin creation of standard build environment. Add bootstrap script, metrics-base submodule, move metrics-lib submodule. --- .gitmodules | 7 +++++-- src/build | 1 + src/main/resources/bootstrap-development.sh | 10 ++++++++++ {submods => src/submods}/metrics-lib | 0 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 7e4601a..99bca1b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ -[submodule "submods/metrics-lib"] - path = submods/metrics-lib +[submodule "src/build"] + path = src/build + url = https://git.torproject.org/metrics-base.git +[submodule "src/submods/metrics-lib"] + path = src/submods/metrics-lib url = https://git.torproject.org/metrics-lib.git diff --git a/src/build b/src/build new file mode 160000 index 0000000..23c6e0b --- /dev/null +++ b/src/build @@ -0,0 +1 @@ +Subproject commit 23c6e0be5fab9463f137615053ef412e4da2315e diff --git a/src/main/resources/bootstrap-development.sh b/src/main/resources/bootstrap-development.sh new file mode 100755 index 0000000..28fb975 --- /dev/null +++ b/src/main/resources/bootstrap-development.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# +# Copyright 2016--2017 The Tor Project +# See LICENSE for licensing information +# +# Use for submodule initialization and checkout. +# Only necessary after very first checkout without recursion. +# +git submodule update --init --remote + diff --git a/submods/metrics-lib b/src/submods/metrics-lib similarity index 100% rename from submods/metrics-lib rename to src/submods/metrics-lib

1 0

[metrics-web/master] Delete HACKING file.
by karsten＠torproject.org 20 Dec '17

20 Dec '17

commit dc9df6642d0581aa9c944d124cae925e6d824a68 Author: iwakeh <iwakeh(a)torproject.org> Date: Mon Dec 4 11:14:31 2017 +0000 Delete HACKING file. The information in HACKING is obsolete and current information is available on the web-site and Metrics wiki pages. --- HACKING | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/HACKING b/HACKING deleted file mode 100644 index f981750..0000000 --- a/HACKING +++ /dev/null @@ -1,20 +0,0 @@ -Some notes on code style: - -General: -- Lines do not exceed 74 characters unless things become really ugly. -- Files end with two newlines. - -Java code: -Please refer to the Metrics Team Java Style Guide: -https://trac.torproject.org/projects/tor/wiki/org/teams/MetricsTeam/MetricsJavaStyleGuide#CodingStyle - - -SQL code: -- All SQL keywords and PostgreSQL functions are written in capital - letters. -- Never execute a query with SELECT * and expect the result columns to be - in a specific order. - -R: -- Add a space before and after '='. -

1 0

[metrics-web/master] Replace start script for Rserver by ant script.
by karsten＠torproject.org 20 Dec '17

20 Dec '17

commit ba8e25e19a77d53b82582f2a7f2b1025b4c011f7 Author: iwakeh <iwakeh(a)torproject.org> Date: Mon Dec 4 11:14:17 2017 +0000 Replace start script for Rserver by ant script. This includes moving the necessary R sources and config file to a location in the main source tree and removing the now obsolete script. --- build.xml | 14 ++++++++++++++ {website/rserve => src/main/R/rserver}/Rserv.conf | 0 {website/rserve => src/main/R/rserver}/graphs.R | 0 {website/rserve => src/main/R/rserver}/rserve-init.R | 0 {website/rserve => src/main/R/rserver}/tables.R | 0 website/rserve/start.sh | 3 --- 6 files changed, 14 insertions(+), 3 deletions(-) diff --git a/build.xml b/build.xml index bef7e1b..7ebe764 100644 --- a/build.xml +++ b/build.xml @@ -74,6 +74,20 @@ </exec> </target> +  + <target name="run-rserver"> + <exec executable="R" + dir="${Rsources}/rserver" + failonerror="true" > + <arg value="CMD"/> + <arg value="Rserve" /> + <arg value="--no-save" /> + <arg value="--quiet" /> + <arg value="--RS-conf" /> + <arg value="Rserv.conf" /> + </exec> + </target> +  diff --git a/website/rserve/Rserv.conf b/src/main/R/rserver/Rserv.conf similarity index 100% rename from website/rserve/Rserv.conf rename to src/main/R/rserver/Rserv.conf diff --git a/website/rserve/graphs.R b/src/main/R/rserver/graphs.R similarity index 100% rename from website/rserve/graphs.R rename to src/main/R/rserver/graphs.R diff --git a/website/rserve/rserve-init.R b/src/main/R/rserver/rserve-init.R similarity index 100% rename from website/rserve/rserve-init.R rename to src/main/R/rserver/rserve-init.R diff --git a/website/rserve/tables.R b/src/main/R/rserver/tables.R similarity index 100% rename from website/rserve/tables.R rename to src/main/R/rserver/tables.R diff --git a/website/rserve/start.sh b/website/rserve/start.sh deleted file mode 100755 index f353885..0000000 --- a/website/rserve/start.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -dir=`pwd` -R CMD Rserve --no-save --RS-conf $dir/Rserv.conf >> rserve.log 2>&1

1 0

[metrics-web/master] Delete CONTRIB.md.
by karsten＠torproject.org 20 Dec '17

20 Dec '17

commit 294a6d097505e0d254ee7019d553b89a41643645 Author: iwakeh <iwakeh(a)torproject.org> Date: Mon Dec 4 11:14:30 2017 +0000 Delete CONTRIB.md. CONTRIB.md is partially obsolete and the information it contains is already available on the public web-pages. --- CONTRIB.md | 89 -------------------------------------------------------------- 1 file changed, 89 deletions(-) diff --git a/CONTRIB.md b/CONTRIB.md deleted file mode 100644 index 4f36afe..0000000 --- a/CONTRIB.md +++ /dev/null @@ -1,89 +0,0 @@ -Contributor's guide to the Metrics website - -Dear contributor to the Metrics website. This guide shall help you -understand the design decisions behind building the Metrics website and -give you starting points where you should look to make it bigger and -better. - -First of all, let's talk briefly about the scope of the Metrics website, -which we'll be calling Metrics in the following. - - - What Metrics is: Metrics is supposed to provide easy access to Tor - network data. The typical Metrics user is neither a researcher nor a - developer and is just looking for an easy way to learn more about this - Tor network they have been hearing about. Metrics is giving them data - in visual or tabular form, together with explanations that are easy to - understand with as little technical language as possible. - - - What Metrics is not: The typical Tor researcher or Tor developer would - probably want to dive deeper into the data to learn even more about the - Tor network. But in contrast to the average Metrics user they could - just fetch the original data from CollecTor and run their own analysis. - Metrics is not trying to be the solution for everyone. If we have to - choose, we're aiming for simplicity instead of comprehensiveness. - -Now let's take a quick tour of the components that Metrics is made of. - - - Data-processing modules: The bulk of Metrics code is running in the - background, invisible to Metrics users. It's the code that takes - CollecTor data as input and that produces .csv files that are the basis - for graphs and tables on Metrics. There's usually one such module per - generated .csv file that focuses on a different aspect of Tor network - data. All these modules are periodically executed by the system's cron - daemon, independent of user requests to the website part of Metrics. - See the modules/ subdirectory for the existing data-processing modules. - Note that modules don't have to be written in Java even though that's - currently the case for all of them. The only requirement is that - there's a shell script to run the module using packages available in - Debian stable. The remaining components of Metrics are all related to - its website part. - - - Start page: The website part of Metrics is organized into one page per - metric, which can be a graph, table, data file, or external link, and - the start page to browse available metrics. Each metric has attributes - like a descriptive name, one or more tags (relays, bridges, etc.), a - type (graph, table, etc.), and a level (basic or advanced). All - metrics are defined in `website/etc/metrics.json` and displayed in the - table on the start page. - - - Graph pages: The bulk of graph pages consist of graphing methods in - `website/rserve/graphs.R` that are written in R and using the ggplot2 - graphing library. These methods read one or more of the .csv files - produced by data-processing modules and produce a graph image as - output. Graphs have a few additional attributes in - `website/etc/metrics.json` like a description and parameters to - customize the graph. As of writing this guide, there's one exception - with the bubble graph which is implemented using JavaScript library - D3.js and which might soon be generated on the server using Node.js. - - - Table pages: Metrics also provides a few aspects of Tor network data in - tabular form with customization options. Like graphs, the data in - these tables is provided using R by reading the previously generated - .csv files. All relevant R code for generating table data is located - in `website/rserve/tables.R`. Again, there are additional attributes - in `website/etc/metrics.json` that define what parameters are available - to customize table contents and how to format results. - - - Data pages: While most Metrics user are not expected to run their own - analyses based on raw Tor network data, some of them might want to look - deeper into the data they saw in a graph or table. Metrics provides - all pre-aggregated output from its data-processing modules as - downloadable .csv files and also documents these file formats in - sufficient detail for Metrics users to use them. - - - Link pages: Metrics is not the only game in town, and it's great that - other developers take the publicly available Tor network data and - visualize it in a different way. Metrics acknowledges these efforts by - adding link pages with thumbnails to make it easy for Metrics users to - find those external visualizations. - - - About page: Most Metrics users have a basic understanding of how Tor - works, most likely from reading the main Tor website. But Metrics - should give its users enough explanations to understand where all the - Tor network data comes from and how that data is used to learn - interesting facts about the Tor network. That's where the About page - comes into play. The About page consists of a list of frequently used - terms and a second list of frequently asked questions. There could be - more documentation, but more text doesn't necessarily mean that users - will read more. -

1 0

[metrics-web/master] Add ant task for legacy run.
by karsten＠torproject.org 20 Dec '17

20 Dec '17

commit 9554c558608741ec0bc400744b8d25b82a46f275 Author: iwakeh <iwakeh(a)torproject.org> Date: Mon Dec 4 11:14:27 2017 +0000 Add ant task for legacy run. Move related sources to main src tree, delete old build.xml and shell script. Update .gitignore to ignore legacy.config. The psql executions can contain all command line parameters of psql, i.e., it is also possible to add username and more. Currently, only the db name is set. This situation could be made easier/more consistent, but this is a new task not part of the current. --- .gitignore | 2 +- build.xml | 46 ++++++++++++++++++++++ modules/legacy/build.xml | 24 ----------- shared/bin/50-run-legacy-stats.sh | 9 ----- .../org/torproject/ernie/cron/Configuration.java | 0 .../java/org/torproject/ernie/cron/LockFile.java | 0 .../ernie/cron/LoggingConfiguration.java | 0 .../main/java/org/torproject/ernie/cron/Main.java | 0 .../cron/RelayDescriptorDatabaseImporter.java | 0 .../cron/network/ConsensusStatsFileHandler.java | 0 .../main/resources/legacy.config.template | 0 .../legacy/db => src/main/sql/legacy}/tordir.sql | 0 12 files changed, 47 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 8234621..b546cef 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,7 @@ cobertura.ser /lib/ # Copied and possibly modified config file -config +legacy.config # other *~ diff --git a/build.xml b/build.xml index b793a10..09857ba 100644 --- a/build.xml +++ b/build.xml @@ -96,6 +96,52 @@ <antcall target="run-java" /> </target> +  + <target name="legacy-create-config" > + <copy file="${resources}/legacy.config.template" + tofile="${basedir}/legacy.config"/> + </target> + +  + <target name="legacy" > + <property name="module.name" value="legacy" /> + <property name="localmoddir" value="${modulebase}/${module.name}" /> + <property name="statsdir" + value="${localmoddir}/stats" /> + <mkdir dir="${statsdir}" /> + + <copy file="${basedir}/legacy.config" + tofile="${localmoddir}/config"/> + + <antcall target="run-java"> + <param name="module.main" + value="org.torproject.ernie.cron.Main" /> + </antcall> + + <exec executable="psql" + dir="${localmoddir}" + failonerror="true" > + <arg value="--dbname=tordir"/> + <arg value="-c SELECT * FROM refresh_all();" /> + </exec> + + <exec executable="psql" + dir="${localmoddir}" + failonerror="true" > + <arg value="-c COPY (SELECT * FROM stats_servers) TO STDOUT WITH CSV HEADER;" /> + <arg value="--dbname=tordir"/> + <arg value="--output=${statsdir}/servers.csv" /> + </exec> + + <exec executable="psql" + dir="${localmoddir}" + failonerror="true" > + <arg value="-c COPY (SELECT * FROM stats_bandwidth) TO STDOUT WITH CSV HEADER;" /> + <arg value="--dbname=tordir"/> + <arg value="--output=${statsdir}/bandwidth.csv" /> + </exec> + </target> + <target name="advbwdist"> <property name="module.name" value="advbwdist" /> <antcall target="run-java" /> diff --git a/modules/legacy/build.xml b/modules/legacy/build.xml deleted file mode 100644 index bc2b674..0000000 --- a/modules/legacy/build.xml +++ /dev/null @@ -1,24 +0,0 @@ -<project default="run" name="metrics-web" basedir="."> - - <property name="mainclass" value="org.torproject.ernie.cron.Main"/> - - <include file="../../shared/build-base.xml" as="basetask"/> - <target name="clean" depends="basetask.clean"/> - <target name="compile" depends="basetask.compile"/> - <target name="run" depends="initrun,basetask.run"/> - - <path id="classpath"> - <pathelement path="${classes}"/> - <path refid="base.classpath" /> - <fileset dir="${libs}"> - <include name="postgresql-jdbc3-9.2.jar"/> - </fileset> - </path> - -  - <target name="initrun" > - <copy file="config.template" tofile="config"/> - </target> - -</project> - diff --git a/shared/bin/50-run-legacy-stats.sh b/shared/bin/50-run-legacy-stats.sh deleted file mode 100755 index 0fd5699..0000000 --- a/shared/bin/50-run-legacy-stats.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -cd modules/legacy/ -ant | grep "\[java\]" -psql -U metrics tordir -c 'SELECT * FROM refresh_all();' -mkdir -p stats -psql -c 'COPY (SELECT * FROM stats_servers) TO STDOUT WITH CSV HEADER;' tordir > stats/servers.csv -psql -c 'COPY (SELECT * FROM stats_bandwidth) TO STDOUT WITH CSV HEADER;' tordir > stats/bandwidth.csv -cd ../../ - diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java b/src/main/java/org/torproject/ernie/cron/Configuration.java similarity index 100% rename from modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java rename to src/main/java/org/torproject/ernie/cron/Configuration.java diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/LockFile.java b/src/main/java/org/torproject/ernie/cron/LockFile.java similarity index 100% rename from modules/legacy/src/main/java/org/torproject/ernie/cron/LockFile.java rename to src/main/java/org/torproject/ernie/cron/LockFile.java diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/LoggingConfiguration.java b/src/main/java/org/torproject/ernie/cron/LoggingConfiguration.java similarity index 100% rename from modules/legacy/src/main/java/org/torproject/ernie/cron/LoggingConfiguration.java rename to src/main/java/org/torproject/ernie/cron/LoggingConfiguration.java diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/Main.java b/src/main/java/org/torproject/ernie/cron/Main.java similarity index 100% rename from modules/legacy/src/main/java/org/torproject/ernie/cron/Main.java rename to src/main/java/org/torproject/ernie/cron/Main.java diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java b/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java similarity index 100% rename from modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java rename to src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java b/src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java similarity index 100% rename from modules/legacy/src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java rename to src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java diff --git a/modules/legacy/config.template b/src/main/resources/legacy.config.template similarity index 100% rename from modules/legacy/config.template rename to src/main/resources/legacy.config.template diff --git a/modules/legacy/db/tordir.sql b/src/main/sql/legacy/tordir.sql similarity index 100% rename from modules/legacy/db/tordir.sql rename to src/main/sql/legacy/tordir.sql

1 0

[metrics-web/master] Add ant task for collectdescs run, move related sources.
by karsten＠torproject.org 20 Dec '17

20 Dec '17

commit 2d7e8cf4dadfbf100126570f8e2e7066de38c016 Author: iwakeh <iwakeh(a)torproject.org> Date: Mon Dec 4 11:14:20 2017 +0000 Add ant task for collectdescs run, move related sources. Delete old build.xml and shell script; add some deployment related properties to build.xml; add ant helper task for running java modules. --- build.xml | 35 ++++++++++++++++++++++ modules/collectdescs/build.xml | 11 ------- shared/bin/01-collectdescs.sh | 5 ---- .../org/torproject/metrics/collectdescs/Main.java | 0 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/build.xml b/build.xml index 7ebe764..7f2abe9 100644 --- a/build.xml +++ b/build.xml @@ -74,7 +74,18 @@ </exec> </target> +  + <property name="prepare.deployment" value="${basedir}/generated" /> + +  + <property name="modulebase" value="${prepare.deployment}/modules" /> +  + <target name="collectdescs" > + <property name="module.name" value="collectdescs" /> + <antcall target="run-java" /> + </target> + <target name="run-rserver"> <exec executable="R" dir="${Rsources}/rserver" @@ -88,6 +99,30 @@ </exec> </target> +  + <target name="run-java"> + <echo message="Running java module ${module.name} ... " /> + <available file="${dist}/${jarfile}" property="have.jar"/> + <fail unless="have.jar" message="Please run 'ant jar'."/> + <condition property="mainclass" + value="${module.main}" + else="org.torproject.metrics.${module.name}.Main" > + <isset property="module.main"/> + </condition> + <property name="workingdir" value="${modulebase}/${module.name}" /> + <mkdir dir="${workingdir}" /> + <java dir="${workingdir}" + fork="true" + maxmemory="2g" + classname="${mainclass}"> + <classpath> + <pathelement location="${dist}/${jarfile}"/> + <pathelement location="${resources}"/> + </classpath> + <jvmarg value="-DLOGBASE=../logs"/> + </java> + </target> +  diff --git a/modules/collectdescs/build.xml b/modules/collectdescs/build.xml deleted file mode 100644 index 1499df6..0000000 --- a/modules/collectdescs/build.xml +++ /dev/null @@ -1,11 +0,0 @@ -<project default="run" name="collectdescs" basedir="."> - - <property name="mainclass" value="org.torproject.metrics.collectdescs.Main"/> - - <include file="../../shared/build-base.xml" as="basetask"/> - <target name="clean" depends="basetask.clean"/> - <target name="compile" depends="basetask.compile"/> - <target name="run" depends="basetask.run"/> - -</project> - diff --git a/shared/bin/01-collectdescs.sh b/shared/bin/01-collectdescs.sh deleted file mode 100755 index 18b900f..0000000 --- a/shared/bin/01-collectdescs.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -cd modules/collectdescs/ -ant | grep "\[java\]" -cd ../../ - diff --git a/modules/collectdescs/src/main/java/org/torproject/metrics/collectdescs/Main.java b/src/main/java/org/torproject/metrics/collectdescs/Main.java similarity index 100% rename from modules/collectdescs/src/main/java/org/torproject/metrics/collectdescs/Main.java rename to src/main/java/org/torproject/metrics/collectdescs/Main.java

1 0

[metrics-web/master] Replace web-prepare.sh by ant task.
by karsten＠torproject.org 20 Dec '17

20 Dec '17

commit 5523617d54cc31acec49947184daa9334cdded31 Author: iwakeh <iwakeh(a)torproject.org> Date: Mon Dec 4 11:14:14 2017 +0000 Replace web-prepare.sh by ant task. This is a preparational move. The ant task will later be called from the task preparing the war, i.e., it is only an internal helper task. --- build.xml | 10 ++++++++++ website/src/main/resources/web-prepare.sh | 15 --------------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/build.xml b/build.xml index 26bbf0e..bef7e1b 100644 --- a/build.xml +++ b/build.xml @@ -64,6 +64,16 @@ <patternset refid="library" /> </patternset> + <target name="generate-ml-javadoc" depends="submoduleupdate" > + <mkdir dir="${basedir}/src/submods/metrics-lib/lib" /> + <exec executable="ant" + dir="${basedir}/src/submods/metrics-lib" + failonerror="true" > + <arg value="clean" /> + <arg value="docs" /> + </exec> + </target> +  diff --git a/website/src/main/resources/web-prepare.sh b/website/src/main/resources/web-prepare.sh deleted file mode 100755 index 02b604b..0000000 --- a/website/src/main/resources/web-prepare.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -# -# Expecting the submodule path as argument -# - -cd $1 - -for x in metrics-lib ; do - cd $1/$x - src/main/resources/bootstrap-development.sh - if ! [ -d lib ] ; then - mkdir lib - fi; - ant clean docs -done;

1 0

[metrics-web/master] Replace run-web and copy scripts with ant tasks.
by karsten＠torproject.org 20 Dec '17

20 Dec '17

commit 4ea3758cfffa70a63c5721185f4b110d131cc663 Author: iwakeh <iwakeh(a)torproject.org> Date: Mon Dec 4 11:14:29 2017 +0000 Replace run-web and copy scripts with ant tasks. --- build.xml | 32 ++++++++++++++++++++++++++++++++ run-web.sh | 6 ------ shared/bin/99-copy-stats-files.sh | 15 --------------- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/build.xml b/build.xml index 78f17d0..ed2125a 100644 --- a/build.xml +++ b/build.xml @@ -81,6 +81,19 @@ <property name="modulebase" value="${prepare.deployment}/modules" />  + <target name="run-web-prepare" depends="init" > + <mkdir dir="${prepare.deployment}/modules" /> + <antcall target="collectdescs" /> + <antcall target="connbidirect" /> + <antcall target="onionperf" /> + <antcall target="legacy" /> + <antcall target="advbwdist" /> + <antcall target="hidserv" /> + <antcall target="clients" /> + <antcall target="webstats" /> + <antcall target="make-data-available" /> + </target> + <target name="collectdescs" > <property name="module.name" value="collectdescs" /> <antcall target="run-java" /> @@ -237,6 +250,25 @@ </exec> </target> + <target name="make-data-available" > + <property name="statsdir" value="${metrics-web.deployment.base}/shared/stats/" /> + <mkdir dir="${statsdir}" /> + <property name="rdatadir" value="${metrics-web.deployment.base}/shared/RData" /> + <mkdir dir="${rdatadir}" /> + <copy todir="${statsdir}" > + <fileset dir="${modulebase}/onionperf/stats" includes="*.csv" /> + <fileset dir="${modulebase}/connbidirect/stats" includes="connbidirect2.csv" /> + <fileset dir="${modulebase}/advbwdist/stats" includes="advbwdist.csv" /> + <fileset dir="${modulebase}/legacy/stats" includes="*.csv" /> + <fileset dir="${modulebase}/hidserv/stats" includes="hidserv.csv" /> + <fileset dir="${modulebase}/clients/stats" + includes="clients*.csv userstats-combined.csv" /> + <fileset dir="${modulebase}/webstats/stats" includes="webstats.csv" /> + <fileset dir="${modulebase}/clients/RData" includes="*.RData" /> + <fileset dir="${modulebase}/webstats/RData" includes="*.RData" /> + </copy> + </target> +  <target name="run-java"> <echo message="Running java module ${module.name} ... " /> diff --git a/run-web.sh b/run-web.sh deleted file mode 100755 index bd0bb5c..0000000 --- a/run-web.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -for i in $(ls shared/bin/[0-9]* | sort); do - echo `date` "Running $i." - ./$i -done - diff --git a/shared/bin/99-copy-stats-files.sh b/shared/bin/99-copy-stats-files.sh deleted file mode 100755 index cc0c9fc..0000000 --- a/shared/bin/99-copy-stats-files.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -mkdir -p shared/stats -cp -a modules/onionperf/stats/*.csv shared/stats/ -cp -a modules/legacy/stats/*.csv shared/stats/ -cp -a modules/connbidirect/stats/connbidirect2.csv shared/stats/ -cp -a modules/advbwdist/stats/advbwdist.csv shared/stats/ -cp -a modules/hidserv/stats/hidserv.csv shared/stats/ -cp -a modules/clients/stats/clients*.csv shared/stats/ -cp -a modules/clients/stats/userstats-combined.csv shared/stats/ -cp -a modules/webstats/stats/webstats.csv shared/stats/ - -mkdir -p shared/RData -cp -a modules/clients/RData/*.RData shared/RData/ -cp -a modules/webstats/RData/*.RData shared/RData/ -

1 0

[metrics-web/master] Raise memory from 2g to 4g for all module runs.
by karsten＠torproject.org 20 Dec '17

20 Dec '17

commit 51ea118abe6c416af01125dfdc2265db40f89f86 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Dec 19 10:27:36 2017 +0100 Raise memory from 2g to 4g for all module runs. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 070ec67..f77048a 100644 --- a/build.xml +++ b/build.xml @@ -388,7 +388,7 @@ <mkdir dir="${workingdir}" /> <java dir="${workingdir}" fork="true" - maxmemory="2g" + maxmemory="4g" classname="${mainclass}"> <classpath> <pathelement location="${dist}/${jarfile}"/>

1 0