commit c0e7cd031c0b06b6e61c926ddb3f4b15d14f9eaa Author: iwakeh iwakeh@torproject.org Date: Mon Dec 4 11:14:28 2017 +0000
Add ant task for clients run.
Move related sources to main src tree, delete old build.xml and shell script. --- build.xml | 58 ++++++++++++++++++++++ modules/clients/.gitignore | 2 - modules/clients/build.xml | 11 ---- shared/bin/80-run-clients-stats.sh | 34 ------------- {modules => src/main/R}/clients/merge-clients.R | 0 {modules => src/main/R}/clients/split-clients.R | 0 .../main/R}/clients/userstats-detector.R | 0 .../java/org/torproject/metrics/clients/Main.java | 0 .../main/python}/clients/country_info.py | 0 {modules => src/main/python}/clients/detector.py | 0 .../main/sql}/clients/init-userstats.sql | 0 .../main/sql}/clients/test-userstats.sql | 0 12 files changed, 58 insertions(+), 47 deletions(-)
diff --git a/build.xml b/build.xml index 09857ba..78f17d0 100644 --- a/build.xml +++ b/build.xml @@ -155,6 +155,64 @@ <antcall target="run-java" /> </target>
+ <target name="clients" > + <property name="module.name" value="clients" /> + <property name="localmoddir" value="${modulebase}/${module.name}" /> + + <property name="rdatadir" value="${localmoddir}/RData" /> + <mkdir dir="${rdatadir}" /> + <property name="statsdir" + value="${localmoddir}/stats" /> + <mkdir dir="${statsdir}" /> + + <antcall target="run-java" /> + <apply executable="psql" failonerror="true" > + <arg value="--dbname=userstats"/> + <arg value="-f"/> + <fileset dir="${localmoddir}/out" + includes="*.sql" /> + </apply> + + <exec executable="psql" + dir="${localmoddir}" + failonerror="true" > + <arg value="-c COPY (SELECT * FROM estimated) TO STDOUT WITH CSV HEADER;" /> + <arg value="--dbname=userstats"/> + <arg value="--output=userstats.csv" /> + </exec> + + <exec executable="psql" + dir="${localmoddir}" + failonerror="true" > + <arg value="-c COPY (SELECT * FROM combined) TO STDOUT WITH CSV HEADER;" /> + <arg value="--dbname=userstats"/> + <arg value="--output=userstats-combined.csv" /> + </exec> + + <antcall target="run-R" > + <param name="module.Rscript" value="userstats-detector.R" /> + </antcall> + + <exec executable="python" + dir="${localmoddir}" + failonerror="true" > + <arg value="${pysources}/${module.name}/detector.py" /> + <arg value="userstats-detector.csv" /> + <arg value="userstats-ranges.csv" /> + </exec> + + <antcall target="run-R" > + <param name="module.Rscript" value="merge-clients.R" /> + </antcall> + + <copy file="clients.csv" todir="${statsdir}" /> + <copy file="userstats-combined.csv" todir="${statsdir}" /> + + <antcall target="run-R" > + <param name="module.Rscript" value="split-clients.R" /> + </antcall> + </target> + <target name="webstats" > <property name="module.name" value="webstats" /> <property name="rdatadir" value="${modulebase}/${module.name}/RData" /> diff --git a/modules/clients/.gitignore b/modules/clients/.gitignore deleted file mode 100644 index 29a7166..0000000 --- a/modules/clients/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.csv - diff --git a/modules/clients/build.xml b/modules/clients/build.xml deleted file mode 100644 index f67bdae..0000000 --- a/modules/clients/build.xml +++ /dev/null @@ -1,11 +0,0 @@ -<project default="run" name="clients" basedir="."> - - <property name="mainclass" value="org.torproject.metrics.clients.Main"/> - - <include file="../../shared/build-base.xml" as="basetask"/> - <target name="clean" depends="basetask.clean"/> - <target name="compile" depends="basetask.compile"/> - <target name="run" depends="basetask.run"/> - -</project> - diff --git a/shared/bin/80-run-clients-stats.sh b/shared/bin/80-run-clients-stats.sh deleted file mode 100755 index fe93e44..0000000 --- a/shared/bin/80-run-clients-stats.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh - -cd modules/clients/ - -echo `date` "Parsing descriptors." -ant | grep "[java]" - -for i in $(ls out/*.sql) -do - echo `date` "Importing $i." - psql -f $i userstats -done - -echo `date` "Exporting results." -psql -c 'COPY (SELECT * FROM estimated) TO STDOUT WITH CSV HEADER;' userstats > userstats.csv -psql -c 'COPY (SELECT * FROM combined) TO STDOUT WITH CSV HEADER;' userstats > userstats-combined.csv - -echo `date` "Running censorship detector." -R --slave -f userstats-detector.R > /dev/null 2>&1 -python detector.py - -echo `date` "Merging censorship detector results." -R --slave -f merge-clients.R > /dev/null 2>&1 -mkdir -p stats/ -cp clients.csv stats/ -cp userstats-combined.csv stats/ - -echo `date` "Saving results as .RData files." -R --slave -f split-clients.R > /dev/null 2>&1 - -echo `date` "Terminating." - -cd ../../ - diff --git a/modules/clients/merge-clients.R b/src/main/R/clients/merge-clients.R similarity index 100% rename from modules/clients/merge-clients.R rename to src/main/R/clients/merge-clients.R diff --git a/modules/clients/split-clients.R b/src/main/R/clients/split-clients.R similarity index 100% rename from modules/clients/split-clients.R rename to src/main/R/clients/split-clients.R diff --git a/modules/clients/userstats-detector.R b/src/main/R/clients/userstats-detector.R similarity index 100% rename from modules/clients/userstats-detector.R rename to src/main/R/clients/userstats-detector.R diff --git a/modules/clients/src/main/java/org/torproject/metrics/clients/Main.java b/src/main/java/org/torproject/metrics/clients/Main.java similarity index 100% rename from modules/clients/src/main/java/org/torproject/metrics/clients/Main.java rename to src/main/java/org/torproject/metrics/clients/Main.java diff --git a/modules/clients/country_info.py b/src/main/python/clients/country_info.py similarity index 100% rename from modules/clients/country_info.py rename to src/main/python/clients/country_info.py diff --git a/modules/clients/detector.py b/src/main/python/clients/detector.py similarity index 100% rename from modules/clients/detector.py rename to src/main/python/clients/detector.py diff --git a/modules/clients/init-userstats.sql b/src/main/sql/clients/init-userstats.sql similarity index 100% rename from modules/clients/init-userstats.sql rename to src/main/sql/clients/init-userstats.sql diff --git a/modules/clients/test-userstats.sql b/src/main/sql/clients/test-userstats.sql similarity index 100% rename from modules/clients/test-userstats.sql rename to src/main/sql/clients/test-userstats.sql