commit d93caf7f0a82d833e58b5a139f081a02e33c20c6 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Mon May 13 15:32:58 2013 +0200 Update GeoIP file script and manual substitutions. Script changes now catch more cases when our manual substitutions need to be updated. Manual substitutions are updated to May 2013 database. --- src/config/deanonymind.py | 27 +++++++--- src/config/geoip-manual | 121 +++++++++++++++++---------------------------- 2 files changed, 65 insertions(+), 83 deletions(-) diff --git a/src/config/deanonymind.py b/src/config/deanonymind.py index c86dadc..31d0658 100755 --- a/src/config/deanonymind.py +++ b/src/config/deanonymind.py @@ -156,23 +156,34 @@ def apply_manual_changes(assignments, manual_assignments): entry['end_num'] == manual_entry['end_num']: if len(manual_entry['country_code']) != 2: print '-%s' % (line, ) # only remove, don't replace - else: + del manual_dict[start_num] + elif entry['country_code'] != \ + manual_entry['country_code']: new_line = format_line_with_other_country(entry, manual_entry) print '-%s\n+%s' % (line, new_line, ) result.append(new_line) - del manual_dict[start_num] + del manual_dict[start_num] + else: + print ('Warning: not applying ineffective manual ' + 'change:\n %s\n %s' % (line, manual_line, )) + result.append(line) else: - print ('Warning: only partial match between ' - 'original/automatically replaced assignment and ' - 'manual assignment:\n %s\n %s\nNot applying ' - 'manual change.' % (line, manual_line, )) + print ('Warning: not applying manual change that is only ' + 'a partial match:\n %s\n %s' % + (line, manual_line, )) result.append(line) + elif 'country_code' in entry and \ + entry['country_code'] == 'A1': + print ('Warning: no manual replacement for A1 entry:\n %s' + % (line, )) + result.append(line) else: result.append(line) if len(manual_dict) > 0: - print ('Warning: could not apply all manual assignments: %s' % - ('\n '.join(manual_dict.values())), ) + print 'Warning: could not apply all manual assignments:' + for line in manual_dict.values(): + print ' %s' % (line, ) return result def write_file(path, assignments, long_format=True): diff --git a/src/config/geoip-manual b/src/config/geoip-manual index fe5cc13..6dfac26 100644 --- a/src/config/geoip-manual +++ b/src/config/geoip-manual @@ -3,17 +3,14 @@ # directory to process this file when producing a new geoip file. See # README.geoip in the same directory for details. -# Remove MaxMind entry 0.116.0.0-0.119.255.255 which MaxMind says is AT, -# but which is part of reserved range 0.0.0.0/8. -KL 2012-06-13 -# Disabled, because MaxMind apparently removed this range from their -# database. -KL 2013-02-08 -#"0.116.0.0","0.119.255.255","7602176","7864319","","" - # IN, though could as well be UA. Previous MaxMind entry # 5.56.23.0-5.56.23.127 is IN, next MaxMind entry 5.56.24.0-5.56.31.255 is # UA, and RIR delegation files say the entire block # 5.56.16.0-5.56.23.255 is DE. -KL 2013-04-08 -"5.56.23.128","5.56.23.255","87562112","87562239","IN","India" +# Changed to DE, because range has changed, next MaxMind entry +# 5.56.23.192-5.56.23.255 is DE, and RIR delegation files say the entire +# block 5.56.16.0-5.56.23.255 is DE. -KL 2013-05-13 +"5.56.23.128","5.56.23.191","87562112","87562175","DE","Germany" # GB, because previous MaxMind entry 31.6.16.0-31.6.25.255 is GB, and RIR # delegation files say entire range 31.6.0.0-31.6.63.255 is GB. @@ -31,6 +28,11 @@ # -KL 2012-11-27 "37.139.64.0","37.139.64.0","629882880","629882880","EU","Europe" +# US, because next MaxMind entry 38.99.145.0-38.99.149.255 is US and +# RIR delegation files say entire range 38.0.0.0-38.255.255.255 is US. +# -KL 2013-05-13 +"38.99.144.0","38.99.144.255","644059136","644059391","US","United States" + # GB, because RIR delegation files say exactly this range # 46.16.32.0-46.16.39.255 is GB, even though neither previous nor next # MaxMind range is GB. Both previous and next MaxMind ranges match RIR @@ -52,22 +54,6 @@ # GB. -KL 2013-03-07 "46.166.135.0","46.166.139.255","782665472","782666751","GB","United Kingdom" -# SE, because previous MaxMind entry 46.246.66.0-46.246.71.255 is SE, and -# RIR delegation files say entire range 46.246.0.0-46.246.127.255 is SE. -# -KL 2013-03-07 -"46.246.72.0","46.246.79.255","787892224","787894271","SE","Sweden" - -# SE, because MaxMind range 46.246.88.0-46.246.127.255 is subdivided into -# a mix of SE and A1, and RIR delegation files say entire range -# 46.246.0.0-46.246.127.255 is SE. -KL 2013-04-08 -"46.246.88.0","46.246.88.92","787896320","787896412","SE","Sweden" - -# US, though could as well be CA. Previous MaxMind entry -# 64.237.32.52-64.237.34.127 is US, next MaxMind entry -# 64.237.34.144-64.237.34.151 is CA, and RIR delegation files say the -# entire block 64.237.32.0-64.237.63.255 is US. -KL 2012-11-27 -"64.237.34.128","64.237.34.143","1089282688","1089282703","US","United States" - # Removing, because RIR delegation files don't even have an entry for this # single-address range, and there's no previous or next range in MaxMind. # -KL 2013-03-07 @@ -83,6 +69,9 @@ # delegation files say 67.43.144.0-67.43.159.255 is US. # -KL 2012-11-27 "67.43.144.0","67.43.144.255","1126928384","1126928639","US","United States" +# US, because previous MaxMind entry 68.68.101.232-68.68.102.71 is US, and +# RIR delegation files say 68.68.64.0-68.68.79.255 is US. -KL 2013-05-13 +"68.68.102.72","68.68.102.79","1145333320","1145333327","US","United States" # US, because previous MaxMind entry 70.159.21.51-70.232.244.255 is US, # because next MaxMind entry 70.232.245.58-70.232.245.59 is A2 ("Satellite @@ -103,26 +92,6 @@ # -KL 2013-03-07 "77.242.196.0","77.242.196.255","1307755520","1307755775","GB","United Kingdom" -# GB, despite neither previous (FI) nor next (FR) MaxMind entry being GB, -# but because RIR delegation files say entire range -# 79.141.160.0-79.141.175.255 is GB. -KL 2013-04-08 -"79.141.160.0","79.141.162.255","1334681600","1334682367","GB","United Kingdom" - -# GB, despite neither previous (FR) nor next (DK) MaxMind entry being GB, -# but because RIR delegation files say entire range -# 79.141.160.0-79.141.175.255 is GB. -KL 2013-04-08 -"79.141.164.0","79.141.164.255","1334682624","1334682879","GB","United Kingdom" - -# GB, despite neither previous (DK) nor next (CH) MaxMind entry being GB, -# but because RIR delegation files say entire range -# 79.141.160.0-79.141.175.255 is GB. -KL 2013-04-08 -"79.141.166.0","79.141.166.255","1334683136","1334683391","GB","United Kingdom" - -# GB, despite neither previous (CH) nor next (IT) MaxMind entry being GB, -# but because RIR delegation files say entire range -# 79.141.160.0-79.141.175.255 is GB. -KL 2013-04-08 -"79.141.168.0","79.141.175.255","1334683648","1334685695","GB","United Kingdom" - # SE, because previous MaxMind entry 80.67.11.200-80.67.13.255 is SE, and # RIR delegation files say entire range 80.67.0.0-80.67.15.255 is SE. # -KL 2013-03-07 @@ -154,25 +123,6 @@ # -KL 2012-11-27 "91.238.214.0","91.238.215.255","1542379008","1542379519","GB","United Kingdom" -# GB, despite neither previous (RU) nor next (FR) MaxMind entry being GB, -# but because RIR delegation files say entire range -# 95.141.16.0-95.141.31.255 is GB. -KL 2013-04-08 -"95.141.16.0","95.141.16.255","1603080192","1603080447","GB","United Kingdom" - -# GB, despite neither previous (FR) nor next (LU) MaxMind entry being GB, -# but because RIR delegation files say entire range -# 95.141.16.0-95.141.31.255 is GB. -KL 2013-04-08 -"95.141.18.0","95.141.29.47","1603080704","1603083567","GB","United Kingdom" - -# GB, despite neither previous (LU) nor next (IT) MaxMind entry being GB, -# but because RIR delegation files say entire range -# 95.141.16.0-95.141.31.255 is GB. -KL 2013-04-08 -"95.141.30.48","95.141.31.255","1603083824","1603084287","GB","United Kingdom" - -# US, because next MaxMind entry 173.0.16.0-173.0.65.255 is US, and RIR -# delegation files say 173.0.0.0-173.0.15.255 is US. -KL 2012-11-27 -"173.0.0.0","173.0.15.255","2902458368","2902462463","US","United States" - # FR, because RIR delegation files say 176.31.0.0-176.31.255.255 is FR, # even though neither previous nor next MaxMind range is FR. This may be # wrong, but we don't have any better information to say this range is ES @@ -180,6 +130,15 @@ # -KL 2013-03-07 "176.31.59.140","176.31.59.143","2954836876","2954836879","FR","France" +# NL, because next MaxMind entry 176.56.173.0-176.56.173.63 is NL, and RIR +# delegation files say 176.56.160.0-176.56.191.255 is NL. -KL 2013-05-13 +"176.56.172.0","176.56.172.255","2956504064","2956504319","NL","Netherlands" + +# NL, despite neither previous (RU) nor next (GB) MaxMind entry being NL, +# but because RIR delegation files say entire range +# 176.56.160.0-176.56.191.255 is NL. -KL 2013-05-13 +"176.56.174.0","176.56.174.255","2956504576","2956504831","NL","Netherlands" + # US, because next MaxMind entry 176.67.84.0-176.67.84.79 is US, and RIR # delegation files say 176.67.80.0-176.67.87.255 is US. -KL 2012-11-27 "176.67.80.0","176.67.83.255","2957201408","2957202431","US","United States" @@ -194,15 +153,11 @@ # -KL 2013-03-07 "178.33.213.212","178.33.213.215","2988561876","2988561879","FR","France" -# SE, because next MaxMind entry 178.73.196.113-178.73.196.126 is SE, and -# RIR delegation files say entire range 178.73.192.0-178.73.255.255 is SE. -# -KL 2013-04-08 -"178.73.192.0","178.73.196.112","2991177728","2991178864","SE","Sweden" - -# SE, because previous MaxMind entry 178.73.213.0-178.73.215.255 is SE, -# and RIR delegation files say entire range 178.73.192.0-178.73.255.255 is -# SE. -KL 2013-03-07 -"178.73.216.0","178.73.223.255","2991183872","2991185919","SE","Sweden" +# GB, because RIR delegation files say exactly this range +# 185.25.84.0-185.25.87.255 is GB, even though neither previous nor next +# MaxMind range is GB. Both previous and next MaxMind ranges match RIR +# delegation files, too. -KL 2013-05-13 +"185.25.84.0","185.25.87.255","3105444864","3105445887","GB","United Kingdom" # GB, because RIR delegation files say exactly this range # 193.104.113.0-193.104.113.255 is GB, even though neither previous nor @@ -216,15 +171,26 @@ # -KL 2012-11-27 "193.200.150.0","193.200.150.255","3251148288","3251148543","EU","Europe" -# GB, because next MaxMind entry 195.13.64.0-195.13.68.159 is GB, and RIR -# delegation files say exact entry is GB. -KL 2013-04-08 -"195.13.60.0","195.13.63.255","3272424448","3272425471","GB","United Kingdom" - # US, because previous MaxMind entry 199.96.68.0-199.96.87.127 is US, and # RIR delegation files say 199.96.80.0-199.96.87.255 is US. # -KL 2012-11-27 "199.96.87.128","199.96.87.255","3344979840","3344979967","US","United States" +# US, because next MaxMind entry 199.101.193.0-199.101.195.255 is US, and, +# together with next entries, matches RIR delegation file entry +# 199.101.192.0-199.101.199.255 which is US. -KL 2013-05-13 +"199.101.192.0","199.101.192.255","3345334272","3345334527","US","United States" + +# AU, because previous MaxMind entry 203.98.64.0-203.98.94.255 is AU, and +# RIR delegation files say 203.98.82.0-203.98.95.255 is AU. +# -KL 2013-05-13 +"203.98.95.0","203.98.95.255","3412221696","3412221951","AU","Australia" + +# US, because next MaxMind entry 204.12.162.0-204.12.197.119 is US, and +# RIR delegation files say 204.12.160.0-204.12.191.255 is US. +# -KL 2013-05-13 +"204.12.160.0","204.12.161.255","3423379456","3423379967","US","United States" + # US, because previous MaxMind entry 209.58.176.144-209.59.31.255 is US, # and RIR delegation files say 209.59.32.0-209.59.63.255 is US. # -KL 2012-11-27 @@ -237,6 +203,11 @@ # -KL 2013-03-07 "212.118.232.0","212.118.232.255","3564562432","3564562687","GB","United Kingdom" +# EU, despite neither previous (RU) nor next (SE) MaxMind entry being EU, +# but because RIR delegation files agree with previous MaxMind entry and +# say EU for 217.15.160.0-217.15.175.255. -KL 2013-05-13 +"217.15.160.0","217.15.164.255","3641679872","3641681151","EU","Europe" + # FR, because previous MaxMind entry 217.15.166.0-217.15.166.255 is FR, # and RIR delegation files contain a block 217.15.160.0-217.15.175.255 # which, however, is EU, not FR. But merging with next MaxMind entry