December 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Use txsocksx as a SOCKS proxy
by art＠torproject.org 26 Dec '12

26 Dec '12

commit 5c6581d5ac8dd4aa8d8a8d80442753e88764bce7 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Dec 24 02:43:19 2012 +0100 Use txsocksx as a SOCKS proxy Minor refactoring of txagentwithsocks --- ooni/nettest.py | 3 +- ooni/templates/httpt.py | 2 +- ooni/utils/txagentwithsocks.py | 127 ++-------------------------------------- 3 files changed, 9 insertions(+), 123 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 062a88f..e67b1ba 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -15,7 +15,8 @@ from twisted.internet.defer import TimeoutError as DeferTimeoutError from twisted.web._newclient import ResponseNeverReceived from ooni.utils import log -from ooni.utils.txagentwithsocks import SOCKSError + +from txsocksx.errors import SOCKSError from socket import gaierror diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 089aea5..7486210 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -18,7 +18,7 @@ from ooni import config from ooni.utils.net import BodyReceiver, StringProducer, userAgents -from ooni.utils.txagentwithsocks import Agent, SOCKSError, TrueHeaders +from ooni.utils.txagentwithsocks import Agent, TrueHeaders from ooni.nettest import handleAllFailures diff --git a/ooni/utils/txagentwithsocks.py b/ooni/utils/txagentwithsocks.py index 7c34288..af46b0f 100644 --- a/ooni/utils/txagentwithsocks.py +++ b/ooni/utils/txagentwithsocks.py @@ -16,122 +16,9 @@ from twisted.internet.endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint, _ from twisted.internet import interfaces, defer from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred -from ooni.utils import log - -class SOCKSError(Exception): - def __init__(self, value): - Exception.__init__(self) - self.code = value - -class SOCKSv5ClientProtocol(_WrappingProtocol): - state = 0 - - def __init__(self, connectedDeferred, wrappedProtocol, host, port): - _WrappingProtocol.__init__(self, connectedDeferred, wrappedProtocol) - self._host = host - self._port = port - self.ready = False - - def logPrefix(self): - return 'SOCKSv5ClientProtocol' - - def socks_state_0(self, data): - # error state - self._connectedDeferred.errback(SOCKSError(0x00)) - return - - def socks_state_1(self, data): - if data != "\x05\x00": - self._connectedDeferred.errback(SOCKSError(0x00)) - return - - # Anonymous access allowed - let's issue connect - self.transport.write(struct.pack("!BBBBB", 5, 1, 0, 3, - len(self._host)) + - self._host + - struct.pack("!H", self._port)) - - def socks_state_2(self, data): - if data[:2] != "\x05\x00": - # Anonymous access denied - - errcode = ord(data[1]) - self._connectedDeferred.errback(SOCKSError(errcode)) - - return - - self.ready = True - self._wrappedProtocol.transport = self.transport - self._wrappedProtocol.connectionMade() - - self._connectedDeferred.callback(self._wrappedProtocol) - - def connectionMade(self): - # We implement only Anonymous access - self.transport.write(struct.pack("!BB", 5, len("\x00")) + "\x00") - - self.state = self.state + 1 - - def write(self, data): - if self.ready: - self.transport.write(data) - else: - self.buf.append(data) - - def dataReceived(self, data): - if self.state != 3: - getattr(self, 'socks_state_%s' % (self.state), - self.socks_state_0)(data) - self.state = self.state + 1 - else: - self._wrappedProtocol.dataReceived(data) - -class SOCKSv5ClientFactory(_WrappingFactory): - protocol = SOCKSv5ClientProtocol - - def __init__(self, wrappedFactory, host, port): - _WrappingFactory.__init__(self, wrappedFactory) - self._host, self._port = host, port - - def logPrefix(self): - return 'SOCKSv5ClientFactory' - - def buildProtocol(self, addr): - try: - proto = self._wrappedFactory.buildProtocol(addr) - except: - self._onConnection.errback() - else: - return self.protocol(self._onConnection, proto, - self._host, self._port) - -class SOCKS5ClientEndpoint(object): - implements(interfaces.IStreamClientEndpoint) - - def __init__(self, reactor, sockshost, socksport, - host, port, timeout=30, bindAddress=None): - - self._reactor = reactor - self._sockshost = sockshost - self._socksport = socksport - self._host = host - self._port = port - self._timeout = timeout - self._bindAddress = bindAddress - - def logPrefix(self): - return 'SOCKSv5ClientEndpoint' - - def connect(self, protocolFactory): - try: - wf = SOCKSv5ClientFactory(protocolFactory, self._host, self._port) - self._reactor.connectTCP( - self._sockshost, self._socksport, wf, - timeout=self._timeout, bindAddress=self._bindAddress) - return wf._onConnection - except: - return defer.fail() +from txsocksx.client import SOCKS5ClientEndpoint +from ooni.utils import log class TrueHeaders(http_headers.Headers): def __init__(self, rawHeaders=None): @@ -295,12 +182,10 @@ class Agent(client.Agent): kwargs['bindAddress'] = self._bindAddress if scheme == 'http': return TCP4ClientEndpoint(self._reactor, host, port, **kwargs) - elif scheme == 'shttp': - return SOCKS5ClientEndpoint(self._reactor, self._sockshost, - self._socksport, host, port, **kwargs) - elif scheme == 'httpo': - return SOCKS5ClientEndpoint(self._reactor, self._sockshost, - self._socksport, host, port, **kwargs) + elif scheme == 'shttp' or scheme == 'httpo': + socksProxy = TCP4ClientEndpoint(self._reactor, self._sockshost, + self._socksport) + return SOCKS5ClientEndpoint(host, port, socksProxy) elif scheme == 'https': return SSL4ClientEndpoint(self._reactor, host, port, self._wrapContextFactory(host, port), **kwargs)

1 0

[ooni-probe/master] Merge branch 'txsocksx'
by art＠torproject.org 26 Dec '12

26 Dec '12

commit 3513f45d2996154465085b74555cddc885c9279f Merge: 51e9ff3 b5d31dc Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Dec 26 17:34:18 2012 +0100 Merge branch 'txsocksx' * txsocksx: Add txsocksx as a dependency Properly handle closure of filedescriptors in runner.py More detailed error handling for SOCKS Use txsocksx as a SOCKS proxy Minor refactoring of txagentwithsocks README.md | 1 + ooni/nettest.py | 60 ++++++++++++++++--- ooni/runner.py | 6 +- ooni/templates/httpt.py | 2 +- ooni/utils/txagentwithsocks.py | 127 ++-------------------------------------- requirements.txt | 1 + 6 files changed, 65 insertions(+), 132 deletions(-)

1 0

[ooni-probe/master] Add txsocksx as a dependency
by art＠torproject.org 26 Dec '12

26 Dec '12

commit b5d31dc92082e23228269d697cb4732b301375ea Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Dec 26 17:34:00 2012 +0100 Add txsocksx as a dependency --- README.md | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+), 0 deletions(-) diff --git a/README.md b/README.md index 34beb6a..8d90f7f 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ The python dependencies required for running ooniprobe are: * libdnet: https://code.google.com/p/libdnet/ * BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ * txtorcon: https://github.com/meejah/txtorcon + * txsocksx: https://github.com/hellais/txsocksx ## Install Tor diff --git a/requirements.txt b/requirements.txt index 1243806..633dbe5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,7 @@ pyOpenSSL>=0.13 pygeoip>=0.2.5 repoze.sphinx.autointerface>=0.7.1 https://hg.secdev.org/scapy/archive/tip.zip#egg=scapy +https://github.com/hellais/txsocksx/archive/master.zip storm>=0.19 transaction>=1.3.0 txtorcon>=0.7

1 0

r25971: {website} update mirrors table. (website/trunk/include)
by Andrew Lewman 26 Dec '12

26 Dec '12

Author: phobos Date: 2012-12-26 15:40:44 +0000 (Wed, 26 Dec 2012) New Revision: 25971 Modified: website/trunk/include/mirrors-table.wmi Log: update mirrors table. Modified: website/trunk/include/mirrors-table.wmi =================================================================== --- website/trunk/include/mirrors-table.wmi 2012-12-26 15:29:58 UTC (rev 25970) +++ website/trunk/include/mirrors-table.wmi 2012-12-26 15:40:44 UTC (rev 25971) @@ -69,23 +69,6 @@ <tr> - <td>PT</td> - - <td>5ª Coluna</td> - - <td>Up to date</td> - - <td> - </td> - <td><a href="http://torproject.pt/dist/">http</a></td> - <td><a href="http://torproject.pt/">http</a></td> - <td> - </td> - <td> - </td> - <td> - </td> - <td> - </td> -</tr> - -<tr> - <td>NL</td> <td>CCC</td> @@ -103,15 +86,15 @@ <tr> - <td>DK</td> + <td>EE</td> - <td>Zentrum der Gesundheit</td> + <td>CyberSIDE</td> <td>Up to date</td> <td> - </td> - <td><a href="http://tor.idnr.ws/dist/">http</a></td> - <td><a href="http://tor.idnr.ws/">http</a></td> + <td><a href="http://cyberside.net.ee/tor/">http</a></td> + <td><a href="http://cyberside.planet.ee/tor/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -314,12 +297,12 @@ <td>Up to date</td> <td> - </td> - <td> - </td> + <td><a href="http://tor.li/dist/">http</a></td> <td><a href="http://tor.li/">http</a></td> + <td><a href="https://tor.li/dist/">https</a></td> + <td><a href="https://tor.li/">https</a></td> <td> - </td> <td> - </td> - <td> - </td> - <td> - </td> </tr> <tr> @@ -341,6 +324,23 @@ <tr> + <td>FR</td> + + <td></td> + + <td>Up to date</td> + + <td> - </td> + <td><a href="http://tor.hardkor.info/dist/">http</a></td> + <td><a href="http://tor.hardkor.info/">http</a></td> + <td> - </td> + <td> - </td> + <td> - </td> + <td> - </td> +</tr> + +<tr> + <td>UK</td> <td></td> @@ -732,6 +732,23 @@ <tr> + <td>INT</td> + + <td>CoralCDN</td> + + <td>Up to date</td> + + <td> - </td> + <td><a href="http://www.torproject.org.nyud.net/dist/">http</a></td> + <td><a href="http://www.torproject.org.nyud.net/">http</a></td> + <td> - </td> + <td> - </td> + <td> - </td> + <td> - </td> +</tr> + +<tr> + <td>US</td> <td></td> @@ -749,15 +766,15 @@ <tr> - <td>EE</td> + <td>DK</td> - <td>CyberSIDE</td> + <td>Zentrum der Gesundheit</td> <td>Up to date</td> <td> - </td> - <td><a href="http://cyberside.net.ee/tor/">http</a></td> - <td><a href="http://cyberside.planet.ee/tor/">http</a></td> + <td><a href="http://tor.idnr.ws/dist/">http</a></td> + <td><a href="http://tor.idnr.ws/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -885,15 +902,15 @@ <tr> - <td>NL</td> + <td>PT</td> - <td>BIT BV</td> + <td>5ª Coluna</td> <td>Unknown</td> - <td><a href="ftp://ftp.bit.nl/mirror/tor/">ftp</a></td> - <td><a href="http://ftp.bit.nl/mirror/tor/">http</a></td> <td> - </td> + <td><a href="http://torproject.pt/dist/">http</a></td> + <td><a href="http://torproject.pt/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -902,15 +919,15 @@ <tr> - <td>UA</td> + <td>NL</td> - <td></td> + <td>BIT BV</td> <td>Unknown</td> + <td><a href="ftp://ftp.bit.nl/mirror/tor/">ftp</a></td> + <td><a href="http://ftp.bit.nl/mirror/tor/">http</a></td> <td> - </td> - <td><a href="http://torua.reactor-xg.kiev.ua/dist/">http</a></td> - <td><a href="http://torua.reactor-xg.kiev.ua/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -919,15 +936,15 @@ <tr> - <td>FR</td> + <td>UA</td> <td></td> <td>Unknown</td> <td> - </td> - <td><a href="http://tor.hardkor.info/dist/">http</a></td> - <td><a href="http://tor.hardkor.info/">http</a></td> + <td><a href="http://torua.reactor-xg.kiev.ua/dist/">http</a></td> + <td><a href="http://torua.reactor-xg.kiev.ua/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -970,34 +987,34 @@ <tr> - <td>NL</td> + <td>DE</td> <td></td> - <td>Out of date</td> + <td>Unknown</td> <td> - </td> + <td><a href="http://mirror.vieth-server.de/torproject.org/dist/">http</a></td> + <td><a href="http://mirror.vieth-server.de/torproject.org">http</a></td> <td> - </td> <td> - </td> - <td><a href="https://www.coevoet.nl/tor/dist/">https</a></td> <td> - </td> <td> - </td> - <td> - </td> </tr> <tr> - <td>INT</td> + <td>NL</td> - <td>CoralCDN</td> + <td></td> - <td>Unknown</td> + <td>Out of date</td> <td> - </td> - <td><a href="http://www.torproject.org.nyud.net/dist/">http</a></td> - <td><a href="http://www.torproject.org.nyud.net/">http</a></td> <td> - </td> <td> - </td> + <td><a href="https://www.coevoet.nl/tor/dist/">https</a></td> <td> - </td> <td> - </td> + <td> - </td> </tr>

1 0

[ooni-probe/master] Fix bug that made HTTP based tests stall
by art＠torproject.org 26 Dec '12

26 Dec '12

commit 47d61ebfa76d87c4ca071bbfea728454b4be888b Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Dec 26 16:28:56 2012 +0100 Fix bug that made HTTP based tests stall The issue occurred when the server was configured to keep the connection alive. Since I did not evaluate the content length field when passing it to the body delivery Protocol connection lost would never occur and the test would stall indefinitely. --- ooni/templates/httpt.py | 8 ++++++-- ooni/utils/net.py | 8 +++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 089aea5..a0e8eb1 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -220,11 +220,15 @@ class HTTPTest(NetTestCase): else: self.processResponseHeaders(response_headers_dict) + try: + content_length = response.headers.getRawHeaders('content-length') + except IndexError: + content_length = None + finished = defer.Deferred() - response.deliverBody(BodyReceiver(finished)) + response.deliverBody(BodyReceiver(finished, content_length)) finished.addCallback(self._processResponseBody, request, response, body_processor) - return finished def doRequest(self, url, method="GET", diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 824d720..865977c 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -61,11 +61,17 @@ class StringProducer(object): pass class BodyReceiver(protocol.Protocol): - def __init__(self, finished): + def __init__(self, finished, content_length=None): self.finished = finished self.data = "" + self.bytes_remaining = content_length def dataReceived(self, bytes): + if self.bytes_remaining: + if self.bytes_remaining == 0: + self.connectionLost(None) + else: + self.bytes_remaining -= len(bytes) self.data += bytes def connectionLost(self, reason):

1 0

[ooni-probe/master] Add more detailed error handling to runner.py
by art＠torproject.org 26 Dec '12

26 Dec '12

commit 51e9ff3db497f1494a45dbb7525576f8eb1cd278 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Dec 26 16:30:34 2012 +0100 Add more detailed error handling to runner.py --- ooni/runner.py | 12 +++++++++++- 1 files changed, 11 insertions(+), 1 deletions(-) diff --git a/ooni/runner.py b/ooni/runner.py index 7787992..d064756 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -177,7 +177,12 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, return yaml_reporter.testDone(test_instance, test_name) d1 = oonib_reporter.testDone(test_instance, test_name) d2 = yaml_reporter.testDone(test_instance, test_name) - return defer.DeferredList([d1, d2]) + dl = defer.DeferredList([d1, d2]) + @dl.addErrback + def reportingFailed(failure): + log.err("Error in reporting %s" % test_name) + log.exception(failure) + return dl def test_done(result, test_instance, test_name): log.msg("Finished running %s" % test_name) @@ -232,6 +237,11 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, test_methods_d = defer.DeferredList(dl) test_methods_d.addCallback(tests_done, test_cases[0][0]) + @test_methods_d.addErrback + def deferredListFailed(failure): + log.err("Error Test Method Deferred List") + log.exception(failure) + return test_methods_d def runTestCasesWithInputUnit(test_cases, input_unit, yaml_reporter,

1 0

r25970: {website} add vieth-server mirror. (website/trunk)
by Andrew Lewman 26 Dec '12

26 Dec '12

Author: phobos Date: 2012-12-26 15:29:58 +0000 (Wed, 26 Dec 2012) New Revision: 25970 Modified: website/trunk/update-mirrors.pl Log: add vieth-server mirror. Modified: website/trunk/update-mirrors.pl =================================================================== --- website/trunk/update-mirrors.pl 2012-12-26 15:27:14 UTC (rev 25969) +++ website/trunk/update-mirrors.pl 2012-12-26 15:29:58 UTC (rev 25970) @@ -1178,6 +1178,24 @@ httpsDistMirror => "", rsyncDistMirror => "", hiddenServiceMirror => "", + }, + mirror083 => { + adminContact => "", + orgName => "", + isoCC => "DE", + subRegion => "", + region => "DE", + ipv4 => "True", + ipv6 => "False", + loadBalanced => "No", + httpWebsiteMirror => "http://mirror.vieth-server.de/torproject.org", + httpsWebsiteMirror => "", + rsyncWebsiteMirror => "", + ftpWebsiteMirror => "", + httpDistMirror => "http://mirror.vieth-server.de/torproject.org/dist/", + httpsDistMirror => "", + rsyncDistMirror => "", + hiddenServiceMirror => "", } );

1 0

r25969: {website} update the tor.li mirror (website/trunk)
by Andrew Lewman 26 Dec '12

26 Dec '12

Author: phobos Date: 2012-12-26 15:27:14 +0000 (Wed, 26 Dec 2012) New Revision: 25969 Modified: website/trunk/update-mirrors.pl Log: update the tor.li mirror Modified: website/trunk/update-mirrors.pl =================================================================== --- website/trunk/update-mirrors.pl 2012-12-26 00:51:41 UTC (rev 25968) +++ website/trunk/update-mirrors.pl 2012-12-26 15:27:14 UTC (rev 25969) @@ -1153,11 +1153,11 @@ ipv6 => "False", loadBalanced => "No", httpWebsiteMirror => "http://tor.li/", - httpsWebsiteMirror => "", + httpsWebsiteMirror => "https://tor.li/", rsyncWebsiteMirror => "", ftpWebsiteMirror => "", - httpDistMirror => "", - httpsDistMirror => "", + httpDistMirror => "http://tor.li/dist/", + httpsDistMirror => "https://tor.li/dist/", rsyncDistMirror => "", hiddenServiceMirror => "", },

1 0

[tor/master] Rate-limit "No circuits are opened" message to once-per-hour
by nickm＠torproject.org 26 Dec '12

26 Dec '12

commit 127cb39ffcacbcd99633671e92d299ef53bbac46 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Dec 26 10:05:45 2012 -0500 Rate-limit "No circuits are opened" message to once-per-hour mr-4 reports on #7799 that he was seeing it several times per second, which suggests that things had gone very wrong. This isn't a real fix, but it should make Tor usable till we can figure out the real issue. --- changes/bug7799.ratelim | 6 ++++++ src/or/circuituse.c | 11 ++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/changes/bug7799.ratelim b/changes/bug7799.ratelim new file mode 100644 index 0000000..cb7742c --- /dev/null +++ b/changes/bug7799.ratelim @@ -0,0 +1,6 @@ + o Minor bugfixes: + - Rate-limit the "No circuits are opened. Relaxed timeout for a + circuit with channel state open..." message to once per hour to + keep it from filling the notice logs. Mitigates bug 7799 but does + not fix the underlying cause. Bugfix on 0.2.4.7-alpha. + diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 57ecef1..c3495b4 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -478,13 +478,18 @@ circuit_expire_building(void) } continue; } else { - log_notice(LD_CIRC, + static ratelim_t relax_timeout_limit = RATELIM_INIT(3600); + char *m; + if ((m = rate_limit_log(&relax_timeout_limit, approx_time()))) { + log_notice(LD_CIRC, "No circuits are opened. Relaxed timeout for " "a circuit with channel state %s to %ldms. " "However, it appears the circuit has timed out anyway. " - "%d guards are live. ", + "%d guards are live. %s", channel_state_to_string(victim->n_chan->state), - (long)circ_times.close_ms, num_live_entry_guards(0)); + (long)circ_times.close_ms, num_live_entry_guards(0), m); + tor_free(m); + } } }

1 0

[ooni-probe/master] Refactor to raise exceptions instead of checking return code
by art＠torproject.org 26 Dec '12

26 Dec '12

commit 6c616ba759c6c2ac4d14b8a63f95084b0ec17ad7 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Dec 26 08:12:51 2012 +0100 Refactor to raise exceptions instead of checking return code --- scripts/archive_reports.py | 27 +++++++++++++++------------ 1 files changed, 15 insertions(+), 12 deletions(-) diff --git a/scripts/archive_reports.py b/scripts/archive_reports.py index f48fa08..8246822 100755 --- a/scripts/archive_reports.py +++ b/scripts/archive_reports.py @@ -36,13 +36,15 @@ def filter_reports_by_age(report): except (InvalidTimestampFormat, ValueError): return False +class InvalidReportField(Exception): + pass + def validate_fields(fields): log.debug("Report fields are: %s" % fields) # check report version if fields['test_version'] not in valid_test_versions: - log.err("Report submitted with invalid report version!") - return False + raise InvalidReportField('test_version') # check report CC #XXX: confirm what value we use for default CC and whether @@ -50,33 +52,28 @@ def validate_fields(fields): if fields['probe_cc'] is None: fields['probe_cc'] = default_probe_cc if not re.match('[A-Z\?]{2,4}', fields['probe_cc'].upper()): - log.err("Report submitted with invalid CC!") - return False + raise InvalidReportField('probe_cc') # check report ASN if fields['probe_asn'] is None: fields['probe_asn'] = 'AS0' if not re.match('^AS[0-9]{1,10}', fields['probe_asn'].upper()): - log.err("Report submitted with invalid AS Number!") - return False + raise InvalidReportField('probe_asn') # check report timestamp try: datetime_ts = datetime.fromtimestamp(fields['start_time']) datetime_str = timestamp(datetime_ts) except InvalidTimestampFormat: - log.err("Report submitted with invalid timestamp!") - return False + raise InvalidReportField('start_time') # check report IP try: IPAddress(fields['probe_ip']) except ValueError: - log.err("Report submitted with invalid IP Address!") - return False + raise InvalidReportField('probe_ip') # all looks good! - return True def get_report_header_fields(report_header): required_fields = ['probe_asn', 'probe_cc', 'probe_ip', 'start_time', @@ -141,7 +138,13 @@ for report in reports_to_archive: yamloo = yaml.safe_load_all(f) report_header = yamloo.next() fields = get_report_header_fields(report_header) - if not validate_fields(fields): + try: + validate_fields(fields) + except InvalidReportField, field_name: + log.err("Report %s contains invalid field called %s" % (report, field_name)) + continue + except: + log.err("An unhandled error occurred while processing %s" % report) continue # get a target filename or fail

1 0