[tor-bugs] #13566 [Ooni]: Work on script that copies the reports from the collectors to the /raw directory

[Tor Bug Tracker & Wiki]

#13566: Work on script that copies the reports from the collectors to the /raw
directory
-------------------------+--------------------------------------
     Reporter:  hellais  |      Owner:  hellais
         Type:  task     |     Status:  new
     Priority:  normal   |  Milestone:
    Component:  Ooni     |    Version:
   Resolution:           |   Keywords:  ooni_data_analytics_team
Actual Points:           |  Parent ID:
       Points:           |
-------------------------+--------------------------------------

Comment (by cypherpunks):

 reports2raw.py
 ---------------

 #!/usr/bin/python
 # -*- coding: utf-8 -*-

 """Script to move the reports generated by ooni to a "raw" directory,
 either in
 a remote collector or from a remote collector to local.
 """
 __author__ = "fucking somebody"
 __copyright__ = "GPL v3"

 import os.path
 import subprocess
 from settings import *

 data_bouncer_path = os.path.join(src_path, data_bouncer_rel_path)
 data_col_path = os.path.join(src_path, data_col_rel_path)
 raw_path = os.path.join(dst_path, raw_rel_path)

 src_paths = ' '.join([data_bouncer_path, data_col_path])

 path_args =[remote_user + "@" + remote_host +":" + src_paths, raw_path]
 print "Running rsync with " + str(path_args)
 r = subprocess.call(["rsync"] + rsync_args + path_args)
 if r == 0:
     print "Sucessfully moved reports "
 else:
     print "Error trying to move reports"

 # TODO:
 # remove the empty directories
 # find src_path -depth -type d -empty -delete
 # or remove the reports root directory
 # rm -rf data

 settings.py
 ---------------

 # settings for reports2raw.py

 # ooni report directories are currently:
 #   data/bouncer-*/archive/
 #   data_col_rel_path = 'data/collector/archive/'

 data_bouncer_rel_path = 'data/bouncer-*/archive/'
 data_col_rel_path = 'data/collector/archive/'
 raw_rel_path = 'raw'

 src_path = '/'
 dst_path = '/'

 #just for testing
 import os.path
 src_path = os.path.realpath('.')
 dst_path = os.path.realpath('.')


 rsync_args=["-rv", "--remove-source-files"]
 remote_user = 'user'
 # if the report data is in localhost, just use localhost here
 # and this will do rsync in local
 remote_host = 'localhost'

--
Ticket URL: <https://trac.torproject.org/projects/tor/ticket/13566#comment:3>
Tor Bug Tracker & Wiki <https://trac.torproject.org/>
The Tor Project: anonymity online