commit 7274522c2fb8e0cc9d6332c7b300156903988048 Author: Arlo Breault <arlolra@gmail.com> Date: Tue Dec 16 19:45:06 2014 -0800 Move away from rsync * @kloesing shut down the server. * Closes #24 on github --- Makefile | 75 +++++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/Makefile b/Makefile index d9c0ed4..0b2d255 100644 --- a/Makefile +++ b/Makefile @@ -3,10 +3,10 @@ SHELL := /bin/bash start: exits i18n @./check -rsync_server = metrics.torproject.org -consensuses_dir = metrics-recent/relay-descriptors/consensuses/ -exit_lists_dir = metrics-recent/exit-lists/ -descriptors_dir = metrics-recent/relay-descriptors/server-descriptors/ +collector_url = https://collector.torproject.org/recent/ +consensuses_dir = relay-descriptors/consensuses/ +exit_lists_dir = exit-lists/ +descriptors_dir = relay-descriptors/server-descriptors/ data/: @mkdir -p data @@ -21,18 +21,27 @@ data/exit-lists/: data/ @mkdir -p data/exit-lists data/consensus: data/consensuses/ - @echo Getting latest consensus documents - @rsync -avz $(rsync_server)::$(consensuses_dir) --delete ./data/consensuses/ + @echo "Getting latest consensus documents" + @pushd data/consensuses/; \ + wget -r -nH -nd -nc --no-parent --reject "index.html*" \ + $(collector_url)$(consensuses_dir); \ + popd @echo Consensuses written data/exit-addresses: data/exit-lists/ - @echo Getting latest exit lists - @rsync -avz $(rsync_server)::$(exit_lists_dir) --delete ./data/exit-lists/ + @echo "Getting latest exit lists" + @pushd data/exit-lists/; \ + wget -r -nH -nd -nc --no-parent --reject "index.html*" \ + $(collector_url)$(exit_lists_dir); \ + popd @echo Exit lists written -exits: data/consensus data/exit-addresses data/cached-descriptors - @echo Generating exit-policies file - @python scripts/exitips.py +descriptors: data/descriptors/ + @echo "Getting latest descriptors (This may take a while)" + @pushd data/descriptors/; \ + wget -r -nH -nd -nc --no-parent --reject "index.html*" \ + $(collector_url)$(descriptors_dir); \ + popd @echo Done data/cached-descriptors: descriptors @@ -41,16 +50,30 @@ data/cached-descriptors: descriptors find data/descriptors -type f -mmin -60 | xargs cat > data/cached-descriptors @echo "Done" -descriptors_cutoff = $(shell date -v-1H -v-30M "+%Y/%m/%d %H:%M:%S") -descriptors: data/descriptors/ - @echo "Getting latest descriptors (This may take a while)" - @find data/descriptors -type f -mmin +90 -delete - @rsync $(rsync_server)::$(descriptors_dir) | awk 'BEGIN { before="$(descriptors_cutoff)"; } before < ($$3 " " $$4) && ($$5 != ".") { print $$5; }' | rsync -avz --files-from=- $(rsync_server)::$(descriptors_dir) ./data/descriptors/ +exits: data/consensus data/exit-addresses data/cached-descriptors + @echo Generating exit-policies file + @python scripts/exitips.py @echo Done +locale/: + rm -rf locale + git clone -b torcheck_completed \ + https://git.torproject.org/translation.git locale + pushd locale; \ + for f in *; do \ + if [ "$$f" != "templates" ]; then \ + pushd "$$f"; \ + mkdir LC_MESSAGES; \ + msgfmt -o LC_MESSAGES/check.mo torcheck.po; \ + popd; \ + fi \ + done + data/langs: data/ curl -k https://www.transifex.com/api/2/languages/ > data/langs +i18n: locale/ data/langs + build: go fmt go build @@ -68,22 +91,10 @@ bench: build go test -benchtime 10s -bench "$(filter)" -benchmem profile: build - go test -cpuprofile ../../cpu.prof -memprofile ../../mem.prof -benchtime 40s -bench "$(filter)" - -i18n: locale/ data/langs - -locale/: - rm -rf locale - git clone -b torcheck_completed https://git.torproject.org/translation.git locale - pushd locale; \ - for f in *; do \ - if [ "$$f" != "templates" ]; then \ - pushd "$$f"; \ - mkdir LC_MESSAGES; \ - msgfmt -o LC_MESSAGES/check.mo torcheck.po; \ - popd; \ - fi \ - done + go test \ + -cpuprofile ../../cpu.prof \ + -memprofile ../../mem.prof \ + -benchtime 40s -bench "$(filter)" install: build mv check /usr/local/bin/check