[tor-commits] [check/master] Move away from rsync

[arlo at torproject.org]

commit 7274522c2fb8e0cc9d6332c7b300156903988048
Author: Arlo Breault <arlolra at gmail.com>
Date:   Tue Dec 16 19:45:06 2014 -0800

    Move away from rsync
    
     * @kloesing shut down the server.
    
     * Closes #24 on github
---
 Makefile |   75 +++++++++++++++++++++++++++++++++++---------------------------
 1 file changed, 43 insertions(+), 32 deletions(-)

diff --git a/Makefile b/Makefile
index d9c0ed4..0b2d255 100644
--- a/Makefile
+++ b/Makefile
@@ -3,10 +3,10 @@ SHELL := /bin/bash
 start: exits i18n
 	@./check
 
-rsync_server = metrics.torproject.org
-consensuses_dir = metrics-recent/relay-descriptors/consensuses/
-exit_lists_dir = metrics-recent/exit-lists/
-descriptors_dir = metrics-recent/relay-descriptors/server-descriptors/
+collector_url = https://collector.torproject.org/recent/
+consensuses_dir = relay-descriptors/consensuses/
+exit_lists_dir = exit-lists/
+descriptors_dir = relay-descriptors/server-descriptors/
 
 data/:
 	@mkdir -p data
@@ -21,18 +21,27 @@ data/exit-lists/: data/
 	@mkdir -p data/exit-lists
 
 data/consensus: data/consensuses/
-	@echo Getting latest consensus documents
-	@rsync -avz $(rsync_server)::$(consensuses_dir) --delete ./data/consensuses/
+	@echo "Getting latest consensus documents"
+	@pushd data/consensuses/; \
+		wget -r -nH -nd -nc --no-parent --reject "index.html*" \
+			$(collector_url)$(consensuses_dir); \
+		popd
 	@echo Consensuses written
 
 data/exit-addresses: data/exit-lists/
-	@echo Getting latest exit lists
-	@rsync -avz $(rsync_server)::$(exit_lists_dir) --delete ./data/exit-lists/
+	@echo "Getting latest exit lists"
+	@pushd data/exit-lists/; \
+		wget -r -nH -nd -nc --no-parent --reject "index.html*" \
+			$(collector_url)$(exit_lists_dir); \
+		popd
 	@echo Exit lists written
 
-exits: data/consensus data/exit-addresses data/cached-descriptors
-	@echo Generating exit-policies file
-	@python scripts/exitips.py
+descriptors: data/descriptors/
+	@echo "Getting latest descriptors (This may take a while)"
+	@pushd data/descriptors/; \
+		wget -r -nH -nd -nc --no-parent --reject "index.html*" \
+			$(collector_url)$(descriptors_dir); \
+		popd
 	@echo Done
 
 data/cached-descriptors: descriptors
@@ -41,16 +50,30 @@ data/cached-descriptors: descriptors
 	find data/descriptors -type f -mmin -60 | xargs cat > data/cached-descriptors
 	@echo "Done"
 
-descriptors_cutoff = $(shell date -v-1H -v-30M "+%Y/%m/%d %H:%M:%S")
-descriptors: data/descriptors/
-	@echo "Getting latest descriptors (This may take a while)"
-	@find data/descriptors -type f -mmin +90 -delete
-	@rsync $(rsync_server)::$(descriptors_dir) | awk 'BEGIN { before="$(descriptors_cutoff)"; } before < ($$3 " " $$4) && ($$5 != ".") { print $$5; }' | rsync -avz --files-from=- $(rsync_server)::$(descriptors_dir) ./data/descriptors/
+exits: data/consensus data/exit-addresses data/cached-descriptors
+	@echo Generating exit-policies file
+	@python scripts/exitips.py
 	@echo Done
 
+locale/:
+	rm -rf locale
+	git clone -b torcheck_completed \
+		https://git.torproject.org/translation.git locale
+	pushd locale; \
+	for f in *; do \
+		if [ "$$f" != "templates" ]; then \
+			pushd "$$f"; \
+			mkdir LC_MESSAGES; \
+			msgfmt -o LC_MESSAGES/check.mo torcheck.po; \
+			popd; \
+		fi \
+	done
+
 data/langs: data/
 	curl -k https://www.transifex.com/api/2/languages/ > data/langs
 
+i18n: locale/ data/langs
+
 build:
 	go fmt
 	go build
@@ -68,22 +91,10 @@ bench: build
 	go test -benchtime 10s -bench "$(filter)" -benchmem
 
 profile: build
-	go test -cpuprofile ../../cpu.prof -memprofile ../../mem.prof -benchtime 40s -bench "$(filter)"
-
-i18n: locale/ data/langs
-
-locale/:
-	rm -rf locale
-	git clone -b torcheck_completed https://git.torproject.org/translation.git locale
-	pushd locale; \
-	for f in *; do \
-		if [ "$$f" != "templates" ]; then \
-			pushd "$$f"; \
-			mkdir LC_MESSAGES; \
-			msgfmt -o LC_MESSAGES/check.mo torcheck.po; \
-			popd; \
-		fi \
-	done
+	go test \
+		-cpuprofile ../../cpu.prof \
+		-memprofile ../../mem.prof \
+		-benchtime 40s -bench "$(filter)"
 
 install: build
 	mv check /usr/local/bin/check