commit 026e43b08656d78398b15742ddf510f6b17f859e Author: Arlo Breault <arlolra@gmail.com> Date: Sun Nov 24 15:16:16 2013 -0800 Keep all the addresses we've seen per relay. Stored as an address list. Will result in more false positives but probably better than false negatives. See discussion in #21. Also, trac #7342. --- datastore.go | 93 +++++++++++++++++++++++++++++++++++++--------------- datastore_test.go | 30 ++++++++--------- handlers.go | 4 +-- scripts/exitips.py | 12 ++++--- 4 files changed, 90 insertions(+), 49 deletions(-) diff --git a/datastore.go b/datastore.go index 17aab92..34106a6 100644 --- a/datastore.go +++ b/datastore.go @@ -50,27 +50,51 @@ type AddressPort struct { Port int } +type CanExitCache struct { + ap AddressPort + can bool +} + type Policy struct { Fingerprint string - Address string + Address []string Rules []Rule IsAllowedDefault bool Tminus int + CacheLast CanExitCache } -func (p Policy) CanExit(ap AddressPort) bool { +func (p Policy) CanExit(ap AddressPort) (can bool) { + if p.CacheLast.ap == ap { + can = p.CacheLast.can + return + } + + // update the cache *after* we return + defer func() { + p.CacheLast = CanExitCache{ap, can} + }() + addr := net.ParseIP(ap.Address) if addr != nil && ValidPort(ap.Port) { for _, rule := range p.Rules { if rule.IsMatch(addr, ap.Port) { - return rule.IsAccept + can = rule.IsAccept + return } } } - return p.IsAllowedDefault + + can = p.IsAllowedDefault + return +} + +type PolicyAddress struct { + Policy Policy + Address string } -type PolicyList []Policy +type PolicyList []PolicyAddress func (p PolicyList) Less(i, j int) bool { return p[i].Address < p[j].Address @@ -127,8 +151,8 @@ func (e *Exits) DumpJSON(w io.Writer, tminus int, ip string, port int) { func (e *Exits) GetAllExits(ap AddressPort, tminus int, fn func(string, string, int)) { ind := 0 for _, val := range e.List { - if val.Tminus <= tminus && val.CanExit(ap) { - fn(val.Address, val.Fingerprint, ind) + if val.Policy.Tminus <= tminus && val.Policy.CanExit(ap) { + fn(val.Address, val.Policy.Fingerprint, ind) ind += 1 } } @@ -149,29 +173,53 @@ func (e *Exits) IsTor(remoteAddr string) (fingerprint string, ok bool) { return } -func (e *Exits) Update(exits PolicyList) PolicyList { +func InsertUnique(arr *[]string, a string) { + for _, b := range *arr { + if a == b { + return + } + } + *arr = append(*arr, a) +} + +func (e *Exits) Update(exits []Policy, update bool) { m := make(map[string]Policy) - for _, p := range e.List { - p.Tminus = p.Tminus + 1 - m[p.Fingerprint] = p + // bump entries by an hour that aren't in the new exit list + if update { + for _, p := range e.List { + if _, ok := m[p.Policy.Fingerprint]; !ok { + p.Policy.Tminus = p.Policy.Tminus + 1 + m[p.Policy.Fingerprint] = p.Policy + } + } } + // keep all unique ips we've seen for _, p := range exits { + if q, ok := m[p.Fingerprint]; ok { + for _, a := range q.Address { + InsertUnique(&p.Address, a) + } + } m[p.Fingerprint] = p } - i := 0 - exits = make(PolicyList, len(m)) + var pl PolicyList for _, p := range m { - exits[i] = p - i = i + 1 + for _, a := range p.Address { + pl = append(pl, PolicyAddress{p, a}) + } } - return exits + + // sort -n + sort.Sort(pl) + + e.List = pl } func (e *Exits) Load(source io.Reader, update bool) error { - var exits PolicyList + var exits []Policy dec := json.NewDecoder(source) for { @@ -195,18 +243,9 @@ func (e *Exits) Load(source io.Reader, update bool) error { exits = append(exits, p) } - // bump entries by an hour that aren't in this list - if update { - exits = e.Update(exits) - } - - // sort -n - sort.Sort(exits) - - e.List = exits + e.Update(exits, update) e.UpdateTime = time.Now() e.PreComputeTorList() - return nil } diff --git a/datastore_test.go b/datastore_test.go index ccde0a5..07afc8a 100644 --- a/datastore_test.go +++ b/datastore_test.go @@ -65,8 +65,8 @@ func setupExitList(t *testing.T, testData string) (e *Exits) { } func TestExitListLoading(t *testing.T) { - testData := `{"Rules": [{"IsAccept": true, "MinPort": 706, "MaxPort": 706, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 993, "MaxPort": 993, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 995, "MaxPort": 995, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": "83.227.52.198"} - {"Rules": [{"IsAccept": true, "MinPort": 20, "MaxPort": 23, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 43, "MaxPort": 43, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 53, "MaxPort": 53, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 79, "MaxPort": 81, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 88, "MaxPort": 88, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 110, "MaxPort": 110, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 143, "MaxPort": 143, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 194, "MaxPort": 194, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 220, "MaxPort": 220, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 443, "MaxPort": 443, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 4 64, "MaxPort": 465, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 543, "MaxPort": 544, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 563, "MaxPort": 563, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 587, "MaxPort": 587, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 706, "MaxPort": 706, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 749, "MaxPort": 749, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 873, "MaxPort": 873, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 902, "MaxPort": 904, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 981, "MaxPort": 981, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 989, "MaxPort": 995, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 1194, "MaxPort": 1194, "Address": null , "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 1220, "MaxPort": 1220, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 1293, "MaxPort": 1293, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 1500, "MaxPort": 1500, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 1723, "MaxPort": 1723, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 1863, "MaxPort": 1863, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 2082, "MaxPort": 2083, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 2086, "MaxPort": 2087, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 2095, "MaxPort": 2096, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 3128, "MaxPort": 3128, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 3389, "MaxPort": 3389, "Address": null, "IsAddressWildc ard": true}, {"IsAccept": true, "MinPort": 3690, "MaxPort": 3690, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 4321, "MaxPort": 4321, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 4643, "MaxPort": 4643, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 5050, "MaxPort": 5050, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 5190, "MaxPort": 5190, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 5222, "MaxPort": 5223, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 5228, "MaxPort": 5228, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 5900, "MaxPort": 5900, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 6666, "MaxPort": 6667, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 6679, "MaxPort": 6679, "Address": null, "IsAddressWildcard": true}, {"Is Accept": true, "MinPort": 6697, "MaxPort": 6697, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 8000, "MaxPort": 8000, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 8008, "MaxPort": 8008, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 8080, "MaxPort": 8080, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 8087, "MaxPort": 8088, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 8443, "MaxPort": 8443, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 8888, "MaxPort": 8888, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 9418, "MaxPort": 9418, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 9999, "MaxPort": 10000, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 19294, "MaxPort": 19294, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 19638, "MaxPort": 19638, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": "91.121.43.80"}` + testData := `{"Rules": [{"IsAccept": true, "MinPort": 706, "MaxPort": 706, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 993, "MaxPort": 993, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 995, "MaxPort": 995, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": ["83.227.52.198"], "Fingerprint": "1"} + {"Rules": [{"IsAccept": true, "MinPort": 20, "MaxPort": 23, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 43, "MaxPort": 43, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 53, "MaxPort": 53, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 79, "MaxPort": 81, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 88, "MaxPort": 88, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 110, "MaxPort": 110, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 143, "MaxPort": 143, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 194, "MaxPort": 194, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 220, "MaxPort": 220, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 443, "MaxPort": 443, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 4 64, "MaxPort": 465, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 543, "MaxPort": 544, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 563, "MaxPort": 563, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 587, "MaxPort": 587, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 706, "MaxPort": 706, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 749, "MaxPort": 749, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 873, "MaxPort": 873, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 902, "MaxPort": 904, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 981, "MaxPort": 981, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 989, "MaxPort": 995, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 1194, "MaxPort": 1194, "Address": null , "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 1220, "MaxPort": 1220, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 1293, "MaxPort": 1293, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 1500, "MaxPort": 1500, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 1723, "MaxPort": 1723, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 1863, "MaxPort": 1863, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 2082, "MaxPort": 2083, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 2086, "MaxPort": 2087, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 2095, "MaxPort": 2096, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 3128, "MaxPort": 3128, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 3389, "MaxPort": 3389, "Address": null, "IsAddressWildc ard": true}, {"IsAccept": true, "MinPort": 3690, "MaxPort": 3690, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 4321, "MaxPort": 4321, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 4643, "MaxPort": 4643, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 5050, "MaxPort": 5050, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 5190, "MaxPort": 5190, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 5222, "MaxPort": 5223, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 5228, "MaxPort": 5228, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 5900, "MaxPort": 5900, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 6666, "MaxPort": 6667, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 6679, "MaxPort": 6679, "Address": null, "IsAddressWildcard": true}, {"Is Accept": true, "MinPort": 6697, "MaxPort": 6697, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 8000, "MaxPort": 8000, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 8008, "MaxPort": 8008, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 8080, "MaxPort": 8080, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 8087, "MaxPort": 8088, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 8443, "MaxPort": 8443, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 8888, "MaxPort": 8888, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 9418, "MaxPort": 9418, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 9999, "MaxPort": 10000, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 19294, "MaxPort": 19294, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 19638, "MaxPort": 19638, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": ["91.121.43.80"], "Fingerprint": "2"}` exits := setupExitList(t, testData) // Valid tor exit @@ -87,8 +87,8 @@ func expectDump(t *testing.T, e *Exits, ip string, port int, expected ...string) } func TestIsAcceptRules(t *testing.T) { - testData := `{"Rules": [{"IsAccept": false, "MinPort": 706, "MaxPort": 706, "Address": null, "IsAddressWildcard": true}, {"IsAccept": false, "MinPort": 5000, "MaxPort": 55000, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": "111.111.111.111"} - {"Rules": [{"IsAccept": true, "MinPort": 706, "MaxPort": 706, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 5000, "MaxPort": 55000, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": "222.222.222.222"}` + testData := `{"Rules": [{"IsAccept": false, "MinPort": 706, "MaxPort": 706, "Address": null, "IsAddressWildcard": true}, {"IsAccept": false, "MinPort": 5000, "MaxPort": 55000, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": ["111.111.111.111"], "Fingerprint": "1"} + {"Rules": [{"IsAccept": true, "MinPort": 706, "MaxPort": 706, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 5000, "MaxPort": 55000, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": ["222.222.222.222"], "Fingerprint": "2"}` exits := setupExitList(t, testData) // one should fail, the other should be OK @@ -98,8 +98,8 @@ func TestIsAcceptRules(t *testing.T) { } func TestIsDefaultAllowedPolicy(t *testing.T) { - testData := `{"Rules": [{"IsAccept": false, "MinPort": 706, "MaxPort": 706, "Address": null, "IsAddressWildcard": true}, {"IsAccept": false, "MinPort": 5000, "MaxPort": 55000, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": true, "Address": "111.111.111.111"} - {"Rules": [{"IsAccept": true, "MinPort": 706, "MaxPort": 706, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 5000, "MaxPort": 55000, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": "222.222.222.222"}` + testData := `{"Rules": [{"IsAccept": false, "MinPort": 706, "MaxPort": 706, "Address": null, "IsAddressWildcard": true}, {"IsAccept": false, "MinPort": 5000, "MaxPort": 55000, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": true, "Address": ["111.111.111.111"], "Fingerprint": "1"} + {"Rules": [{"IsAccept": true, "MinPort": 706, "MaxPort": 706, "Address": null, "IsAddressWildcard": true}, {"IsAccept": true, "MinPort": 5000, "MaxPort": 55000, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": ["222.222.222.222"], "Fingerprint": "2"}` exits := setupExitList(t, testData) // first one should be allowing everything but his blocked port, @@ -112,8 +112,8 @@ func TestIsDefaultAllowedPolicy(t *testing.T) { func TestRulesNonWildcard(t *testing.T) { // Testing load - testData := `{"Rules": [{"IsAccept": false, "MinPort": 706, "MaxPort": 706, "Address": "38.229.70.31"}, {"IsAccept": false, "MinPort": 5000, "MaxPort": 55000, "Address": "38.229.70.31"}], "IsAllowedDefault": true, "Address": "111.111.111.111"} - {"Rules": [{"IsAccept": true, "MinPort": 706, "MaxPort": 706, "Address": "38.229.70.31"}, {"IsAccept": true, "MinPort": 5000, "MaxPort": 55000, "Address": "38.229.70.31"}], "IsAllowedDefault": false, "Address": "222.222.222.222"}` + testData := `{"Rules": [{"IsAccept": false, "MinPort": 706, "MaxPort": 706, "Address": "38.229.70.31"}, {"IsAccept": false, "MinPort": 5000, "MaxPort": 55000, "Address": "38.229.70.31"}], "IsAllowedDefault": true, "Address": ["111.111.111.111"], "Fingerprint": "1"} + {"Rules": [{"IsAccept": true, "MinPort": 706, "MaxPort": 706, "Address": "38.229.70.31"}, {"IsAccept": true, "MinPort": 5000, "MaxPort": 55000, "Address": "38.229.70.31"}], "IsAllowedDefault": false, "Address": ["222.222.222.222"], "Fingerprint": "2"}` exits := setupExitList(t, testData) // first one should reject due to ip @@ -132,7 +132,7 @@ func TestRulesNonWildcard(t *testing.T) { } func TestMaskedIP(t *testing.T) { - testData := `{"Rules": [{"MaxPort": 65535, "IsAddressWildcard": false, "Mask": "255.0.0.0", "Address": "0.0.0.0", "IsAccept": false, "MinPort": 1}, {"MaxPort": 65535, "IsAddressWildcard": false, "Mask": "255.255.0.0", "Address": "169.254.0.0", "IsAccept": false, "MinPort": 1}], "IsAllowedDefault": true, "Address": "111.111.111.111"}` + testData := `{"Rules": [{"MaxPort": 65535, "IsAddressWildcard": false, "Mask": "255.0.0.0", "Address": "0.0.0.0", "IsAccept": false, "MinPort": 1}, {"MaxPort": 65535, "IsAddressWildcard": false, "Mask": "255.255.0.0", "Address": "169.254.0.0", "IsAccept": false, "MinPort": 1}], "IsAllowedDefault": true, "Address": ["111.111.111.111"], "Fingerprint": "1"}` exits := setupExitList(t, testData) expectDump(t, exits, "0.1.2.3", 123) expectDump(t, exits, "169.254.111.111", 345) @@ -140,13 +140,13 @@ func TestMaskedIP(t *testing.T) { } func TestDoubleReject(t *testing.T) { - testData := `{"Rules": [{"IsAccept": false, "MinPort": 80, "MaxPort": 80, "Address": "222.222.222.222"}, {"IsAccept": false, "MinPort": 80, "MaxPort": 80, "Address": "123.123.123.123"}], "IsAllowedDefault": true, "Address": "111.111.111.111"}` + testData := `{"Rules": [{"IsAccept": false, "MinPort": 80, "MaxPort": 80, "Address": "222.222.222.222"}, {"IsAccept": false, "MinPort": 80, "MaxPort": 80, "Address": "123.123.123.123"}], "IsAllowedDefault": true, "Address": ["111.111.111.111"], "Fingerprint": "1"}` exits := setupExitList(t, testData) expectDump(t, exits, "222.222.222.222", 80) } func TestRejectWithDefaultReject(t *testing.T) { - testData := `{"Rules": [{"IsAccept": false, "MinPort": 80, "MaxPort": 80, "Address": "222.222.222.222"}, {"IsAccept": true, "MinPort": 80, "MaxPort": 80, "Address": "", "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": "111.111.111.111"}` + testData := `{"Rules": [{"IsAccept": false, "MinPort": 80, "MaxPort": 80, "Address": "222.222.222.222"}, {"IsAccept": true, "MinPort": 80, "MaxPort": 80, "Address": "", "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": ["111.111.111.111"], "Fingerprint": "1"}` exits := setupExitList(t, testData) // Should reject expectDump(t, exits, "222.222.222.222", 80) @@ -162,20 +162,20 @@ func TestMatchedRuleOrdering(t *testing.T) { such entry. The rules are considered in order; if no rule matches, the address will be accepted. For clarity, the last such entry SHOULD be accept : or reject :. */ - testData := `{"Rules": [{"IsAccept": false, "MinPort": 80, "MaxPort": 80, "Address": "222.222.222.222"}, {"IsAccept": true, "MinPort": 80, "MaxPort": 80, "Address": "222.222.222.222"}], "IsAllowedDefault": false, "Address": "111.111.111.111"}` + testData := `{"Rules": [{"IsAccept": false, "MinPort": 80, "MaxPort": 80, "Address": "222.222.222.222"}, {"IsAccept": true, "MinPort": 80, "MaxPort": 80, "Address": "222.222.222.222"}], "IsAllowedDefault": false, "Address": ["111.111.111.111"], "Fingerprint": "1"}` exits := setupExitList(t, testData) // Should match the reject rule first expectDump(t, exits, "222.222.222.222", 80) - testData = `{"Rules": [{"IsAccept": true, "MinPort": 80, "MaxPort": 80, "Address": "222.222.222.222"}, {"IsAccept": false, "MinPort": 80, "MaxPort": 80, "Address": "222.222.222.222"}], "IsAllowedDefault": false, "Address": "111.111.111.111"}` + testData = `{"Rules": [{"IsAccept": true, "MinPort": 80, "MaxPort": 80, "Address": "222.222.222.222"}, {"IsAccept": false, "MinPort": 80, "MaxPort": 80, "Address": "222.222.222.222"}], "IsAllowedDefault": false, "Address": ["111.111.111.111"], "Fingerprint": "1"}` exits = setupExitList(t, testData) // Should match the accept rule first expectDump(t, exits, "222.222.222.222", 80, "111.111.111.111") } func TestPastHours(t *testing.T) { - testData := `{"Rules": [{"IsAccept": true, "MinPort": 80, "MaxPort": 80, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": "111.111.111.111", "Tminus": 4} - {"Rules": [{"IsAccept": true, "MinPort": 80, "MaxPort": 80, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": "222.222.222.222", "Tminus": 17}` + testData := `{"Rules": [{"IsAccept": true, "MinPort": 80, "MaxPort": 80, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": ["111.111.111.111"], "Fingerprint": "1", "Tminus": 4} + {"Rules": [{"IsAccept": true, "MinPort": 80, "MaxPort": 80, "Address": null, "IsAddressWildcard": true}], "IsAllowedDefault": false, "Address": ["222.222.222.222"], "Fingerprint": "2", "Tminus": 17}` exits := setupExitList(t, testData) // Should reject expectDump(t, exits, "123.123.123.123", 80, "111.111.111.111") diff --git a/handlers.go b/handlers.go index f852a0a..be9bd6a 100644 --- a/handlers.go +++ b/handlers.go @@ -11,8 +11,8 @@ import ( "net/http" "regexp" "strconv" - "time" "strings" + "time" ) // page model @@ -48,7 +48,7 @@ func RootHandler(Layout *template.Template, Exits *Exits, domain *gettext.Domain if len(host) > 0 { parts := strings.Split(host, ",") // apache will append the remote address - host = strings.TrimSpace(parts[len(parts) - 1]) + host = strings.TrimSpace(parts[len(parts)-1]) } else { host, _, err = net.SplitHostPort(r.RemoteAddr) } diff --git a/scripts/exitips.py b/scripts/exitips.py index 3c5bf45..81059b1 100755 --- a/scripts/exitips.py +++ b/scripts/exitips.py @@ -18,7 +18,7 @@ from stem.exit_policy import AddressType class Router(): def __init__(self, router, tminus): self.Fingerprint = router.fingerprint - self.Address = router.address + self.Address = [router.address] self.IsAllowedDefault = router.exit_policy._is_allowed_default self.IsAllowed = router.exit_policy.is_exiting_allowed() self.Rules = [] @@ -73,11 +73,13 @@ def main(consensuses, exit_lists): # update exit addresses with data from TorDNSEL for descriptor in parse_file("data/exit-lists/" + m[0], "tordnsel 1.0"): - descriptor.exit_addresses.sort(key=operator.itemgetter(1), - reverse=True) e = exits.get(descriptor.fingerprint, None) - if e is not None and e.Tminus == t: - e.Address = descriptor.exit_addresses[0][0] + if e is not None: + if e.Tminus == t: + e.Address = [] + for a in descriptor.exit_addresses: + if a[0] not in e.Address: + e.Address.append(a[0]) # update all with server descriptor info for descriptor in parse_file("data/cached-descriptors",