[tor-commits] r26450: {website} new update mirrors script, see ticket 10269, update mirrors (in website/trunk: . include)

4 Dec 2013

Author: phobos
Date: 2013-12-04 22:12:29 +0000 (Wed, 04 Dec 2013)
New Revision: 26450
Modified:
   website/trunk/include/mirrors-table.wmi
   website/trunk/update-mirrors.pl
Log:
new update mirrors script, see ticket 10269, update mirrors table with the new version.
Modified: website/trunk/include/mirrors-table.wmi
===================================================================

--- website/trunk/include/mirrors-table.wmi	2013-12-04 10:57:57 UTC (rev 26449)
+++ website/trunk/include/mirrors-table.wmi	2013-12-04 22:12:29 UTC (rev 26450)
@@ -1,23 +1,6 @@
<tr>
-         <td>INT</td>
-
-         <td>CoralCDN</td>
-
-         <td>Up to date</td>
-
-    <td> - </td>
-    <td><a href="http://www.torproject.org.nyud.net/dist/">http</a></td>
-    <td><a href="http://www.torproject.org.nyud.net/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
          <td>CZ</td>
<td>Lightning-bolt.net</td>
@@ -120,23 +103,6 @@
<tr>
-         <td>LT</td>
-
-         <td></td>
-
-         <td>Up to date</td>
-
-    <td> - </td>
-    <td><a href="http://tor.vesta.nu/dist/">http</a></td>
-    <td><a href="http://tor.vesta.nu/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
          <td>DE</td>
<td></td>
@@ -341,40 +307,6 @@
<tr>
-         <td>US</td>
-
-         <td>AskApache</td>
-
-         <td>Up to date</td>
-
-    <td> - </td>
-    <td><a href="http://tor.askapache.com/dist/">http</a></td>
-    <td><a href="http://tor.askapache.com/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>DE</td>
-
-         <td>Netzkonstrukt Berlin</td>
-
-         <td>Up to date</td>
-
-    <td> - </td>
-    <td><a href="http://mirror.ntzk.de/torproject.org/dist/">http</a></td>
-    <td><a href="http://mirror.ntzk.de/torproject.org/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
          <td>RU</td>
<td>Soviet Anonymous</td>
@@ -443,36 +375,36 @@
<tr>
-         <td>AT</td>
+         <td>INT</td>
-         <td>TechAsk.IT</td>
+         <td>CoralCDN</td>
<td>Up to date</td>
<td> - </td>
-    <td><a href="http://www.unicorncloud.org/public/torproject.org/dist">http</a></td>
-    <td><a href="http://www.unicorncloud.org/public/torproject.org/">http</a></td>
-    <td><a href="https://www.unicorncloud.org/public/torproject.org/dist">https</a></td>
-    <td><a href="https://www.unicorncloud.org/public/torproject.org/">https</a></td>
+    <td><a href="http://www.torproject.org.nyud.net/dist/">http</a></td>
+    <td><a href="http://www.torproject.org.nyud.net/">http</a></td>
     <td> - </td>
     <td> - </td>
+    <td> - </td>
+    <td> - </td>
 </tr>
<tr>
-         <td>NL</td>
+         <td>AT</td>
-         <td>Amorphis</td>
+         <td>TechAsk.IT</td>
<td>Up to date</td>
<td> - </td>
-    <td><a href="http://tor.amorphis.eu/dist/">http</a></td>
-    <td><a href="http://tor.amorphis.eu/">http</a></td>
+    <td><a href="http://www.unicorncloud.org/public/torproject.org/dist">http</a></td>
+    <td><a href="http://www.unicorncloud.org/public/torproject.org/">http</a></td>
+    <td><a href="https://www.unicorncloud.org/public/torproject.org/dist">https</a></td>
+    <td><a href="https://www.unicorncloud.org/public/torproject.org/">https</a></td>
     <td> - </td>
     <td> - </td>
-    <td> - </td>
-    <td> - </td>
 </tr>
<tr>
@@ -630,6 +562,23 @@
<tr>
+         <td>MX</td>
+
+         <td></td>
+
+         <td>Up to date</td>
+
+    <td> - </td>
+    <td><a href="http://fbnaia.homelinux.net/torproject/dist/">http</a></td>
+    <td><a href="http://fbnaia.homelinux.net/torproject/">http</a></td>
+    <td><a href="https://fbnaia.homelinux.net/torproject//dist/">https</a></td>
+    <td><a href="https://fbnaia.homelinux.net/torproject/">https</a></td>
+    <td> - </td>
+    <td> - </td>
+</tr>
+     
+<tr>
+
          <td>NL</td>
<td>BBLN</td>
@@ -834,34 +783,17 @@
<tr>
-         <td>IN</td>
+         <td>LT</td>
-         <td>India Tor Fans</td>
-
-         <td>DO NOT USE. Out of date.</td>
-
-    <td> - </td>
-    <td><a href="http://www.torproject.org.in/dist/">http</a></td>
-    <td><a href="http://www.torproject.org.in/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>TN</td>
-
          <td></td>
-         <td>DO NOT USE. Out of date.</td>
+         <td>Up to date</td>
<td> - </td>
+    <td><a href="http://tor.vesta.nu/dist/">http</a></td>
+    <td><a href="http://tor.vesta.nu/">http</a></td>
     <td> - </td>
-    <td><a href="http://torproject.antagonism.org/">http</a></td>
     <td> - </td>
-    <td><a href="https://torproject.antagonism.org/">https</a></td>
     <td> - </td>
     <td> - </td>
 </tr>
@@ -870,13 +802,13 @@
<td>US</td>
-         <td>searchprivate</td>
+         <td>AskApache</td>
-         <td>DO NOT USE. Out of date.</td>
+         <td>Up to date</td>
<td> - </td>
-    <td><a href="http://tor.searchprivate.com/dist/">http</a></td>
-    <td><a href="http://tor.searchprivate.com/">http</a></td>
+    <td><a href="http://tor.askapache.com/dist/">http</a></td>
+    <td><a href="http://tor.askapache.com/">http</a></td>
     <td> - </td>
     <td> - </td>
     <td> - </td>
@@ -885,289 +817,34 @@
<tr>
-         <td>DE</td>
-
-         <td></td>
-
-         <td>Unknown</td>
-
-    <td> - </td>
-    <td><a href="http://tor.taskserver.de/dist/">http</a></td>
-    <td><a href="http://tor.taskserver.de/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>US</td>
-
-         <td></td>
-
-         <td>Unknown</td>
-
-    <td> - </td>
-    <td><a href="http://tor.mirrors.whitedholdings.org/dist/">http</a></td>
-    <td><a href="http://tor.mirrors.whitedholdings.org/">http</a></td>
-    <td><a href="https://tor.mirrors.whitedholdings.org/dist/">https</a></td>
-    <td><a href="https://tor.mirrors.whitedholdings.org/">https</a></td>
-    <td><a href="rsync://tor.mirrors.whitedholdings.org/tor-dist">rsync</a></td>
-    <td><a href="rsync://mirrors.whitedholdings.org/tor">rsync</a></td>
-</tr>
-     
-<tr>
-
-         <td>US</td>
-
-         <td></td>
-
-         <td>Unknown</td>
-
-    <td><a href="ftp://mirrors.go-parts.com/tor/">ftp</a></td>
-    <td><a href="http://mirrors.go-parts.com/tor/dist/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td><a href="rsync://mirrors.go-parts.com/mirrors/tor/">rsync</a></td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>US</td>
-
-         <td></td>
-
-         <td>Unknown</td>
-
-    <td> - </td>
-    <td><a href="http://tor.loritsu.com/dist/">http</a></td>
-    <td><a href="http://tor.loritsu.com/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>US</td>
-
-         <td>NW Linux</td>
-
-         <td>Unknown</td>
-
-    <td> - </td>
-    <td><a href="http://torproject.nwlinux.us/dist/">http</a></td>
-    <td><a href="http://torproject.nwlinux.us/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td><a href="rsync://nwlinux.us/tor-dist">rsync</a></td>
-    <td><a href="rsync://nwlinux.us/tor-web">rsync</a></td>
-</tr>
-     
-<tr>
-
          <td>NL</td>
-         <td></td>
+         <td>Amorphis</td>
-         <td>DO NOT USE. Out of date.</td>
+         <td>Up to date</td>
<td> - </td>
+    <td><a href="http://tor.amorphis.eu/dist/">http</a></td>
+    <td><a href="http://tor.amorphis.eu/">http</a></td>
     <td> - </td>
     <td> - </td>
-    <td><a href="https://www.coevoet.nl/tor/dist/">https</a></td>
     <td> - </td>
     <td> - </td>
-    <td> - </td>
 </tr>
<tr>
-         <td>LU</td>
+         <td>IN</td>
-         <td></td>
+         <td>India Tor Fans</td>
-         <td>Unknown</td>
-
-    <td> - </td>
-    <td><a href="http://torproject.adamas.ai/dist/">http</a></td>
-    <td><a href="http://torproject.adamas.ai/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>FR</td>
-
-         <td>LazyTiger</td>
-
-         <td>Unknown</td>
-
-    <td> - </td>
-    <td><a href="http://tor.taiga-san.net/dist/">http</a></td>
-    <td><a href="http://tor.taiga-san.net/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>DE</td>
-
-         <td>[[:bbs:]]</td>
-
-         <td>Unknown</td>
-
-    <td> - </td>
-    <td><a href="http://tor.blingblingsquad.net/dist/">http</a></td>
-    <td><a href="http://tor.blingblingsquad.net/">http</a></td>
-    <td><a href="https://tor.blingblingsquad.net/dist/">https</a></td>
-    <td><a href="https://tor.blingblingsquad.net/">https</a></td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>US</td>
-
-         <td></td>
-
          <td>DO NOT USE. Out of date.</td>
<td> - </td>
-    <td><a href="http://www.netgull.com/torproject/">http</a></td>
+    <td><a href="http://www.torproject.org.in/dist/">http</a></td>
+    <td><a href="http://www.torproject.org.in/">http</a></td>
     <td> - </td>
     <td> - </td>
     <td> - </td>
     <td> - </td>
-    <td> - </td>
 </tr>
-     
-<tr>
-
-         <td>US</td>
-
-         <td></td>
-
-         <td>Unknown</td>
-
-    <td> - </td>
-    <td><a href="http://tor.minibofh.org/dist/">http</a></td>
-    <td><a href="http://tor.minibofh.org/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>FR</td>
-
-         <td></td>
-
-         <td>Unknown</td>
-
-    <td> - </td>
-    <td><a href="http://37.187.0.127/tormirror/dist/">http</a></td>
-    <td><a href="http://37.187.0.127/tormirror/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>MX</td>
-
-         <td></td>
-
-         <td>Unknown</td>
-
-    <td> - </td>
-    <td><a href="http://fbnaia.homelinux.net/torproject/dist/">http</a></td>
-    <td><a href="http://fbnaia.homelinux.net/torproject/">http</a></td>
-    <td><a href="https://fbnaia.homelinux.net/torproject//dist/">https</a></td>
-    <td><a href="https://fbnaia.homelinux.net/torproject/">https</a></td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>EE</td>
-
-         <td></td>
-
-         <td>Unknown</td>
-
-    <td> - </td>
-    <td><a href="http://tor.li/dist/">http</a></td>
-    <td><a href="http://tor.li/">http</a></td>
-    <td><a href="https://tor.li/dist/">https</a></td>
-    <td><a href="https://tor.li/">https</a></td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>UA</td>
-
-         <td></td>
-
-         <td>Unknown</td>
-
-    <td> - </td>
-    <td><a href="http://torua.reactor-xg.kiev.ua/dist/">http</a></td>
-    <td><a href="http://torua.reactor-xg.kiev.ua/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>UK</td>
-
-         <td></td>
-
-         <td>Unknown</td>
-
-    <td> - </td>
-    <td><a href="http://tor.mage.me.uk/dist/">http</a></td>
-    <td><a href="http://tor.mage.me.uk/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>DE</td>
-
-         <td>tor-mirror.de</td>
-
-         <td>Unknown</td>
-
-    <td> - </td>
-    <td><a href="http://tor-mirror.de/dist/">http</a></td>
-    <td><a href="http://tor-mirror.de/">http</a></td>
-    <td><a href="https://tor-mirror.de/dist/">https</a></td>
-    <td><a href="https://tor-mirror.de/">https</a></td>
-    <td> - </td>
-    <td> - </td>
-</tr>
Modified: website/trunk/update-mirrors.pl
===================================================================
--- website/trunk/update-mirrors.pl	2013-12-04 10:57:57 UTC (rev 26449)
+++ website/trunk/update-mirrors.pl	2013-12-04 22:12:29 UTC (rev 26450)
@@ -1,23 +1,16 @@
 #!/usr/bin/perl -w
 use warnings;
 use strict;
+use Data::Dumper;
 use LWP::Simple;
+use HTML::LinkExtor;
 use LWP;
 use Date::Parse;
 use Date::Format;
+use Digest::SHA qw(sha256_hex);
-#
-# A quick hack by Jacob Appelbaum jacob@appelbaum.net
-# LWP suggestions by Leigh Honeywell
 # This is Free Software (GPLv3)
 # http://www.gnu.org/licenses/gpl-3.0.txt
-#
-# CHANGELOG
-# 20091003 Code changes to elimiate the need for a trailing slash in addresses for script runtime
-# 20091004 Code changes to increase out of date tolerance to 48 hours
-# 20091028 Code changes to increase timout to 30 seconds (attempting to # resolve "unknown" status')
-# 20091028 Code changes to change user agent of script
-# 20100807 Remove dead mirrors.
print "Creating LWP agent ($LWP::VERSION)...\n";
 my $lua = LWP::UserAgent->new(
@@ -37,45 +30,71 @@
return $cleanedData;
 }
+sub ExtractLinks {
+    my $content = shift; 
+    my $url     = shift;
+    my @links;
-sub FetchDate {
-    my $url = shift; # Base url for mirror
-    my $trace = "project/trace/www-master.torproject.org"; # this file should always exist
-    $url = "$url/$trace";
+    my $parser = HTML::LinkExtor->new(undef, $url);
+    $parser->parse($content);
+    foreach my $linkarray($parser->links)
+    {
+         my ($elt_type, $attr_name, $attr_value) = @$linkarray;
+         if ($elt_type eq 'a' && $attr_name eq 'href' && $attr_value =~ //$/ && $attr_value =~ /^$url/)
+         {
+         	push @links, Fetch($attr_value, &ExtractLinks);
+         }
+	 elsif ($attr_value =~ /.(xpi|dmg|exe|tar.gz)$/)
+	 #elsif ($attr_value =~ /.(asc)$/)
+         {
+         	push @links, $attr_value;
+         }
+    }
+    return @links;
+}
-    print "Fetching possible date from: $url\n";
+sub ExtractDate {
+    my $content = shift;  
+    $content    = sanitize($content);
+    my $date    = str2time($content);
+    print "Extracting possible date from: $content\n";
+    if ($date) {
+        print "We've fetched a date $date.\n";
+        return $date;
+    } else {
+        print "We haven't fetched a date.\n";
+	return undef;
+    }
+}
+
+sub ExtractSig {
+    my $content = shift;
+    return sha256_hex($content); 
+}
+
+sub Fetch {
+    my ($url, $sub) = @_; # Base url for mirror
+    print "Fetch $url\n";
+
     my $request = new HTTP::Request GET => "$url";
     my $result = $lua->request($request);
     my $code = $result->code();
-    print "Result code $code\n";
+    print "\tResult code $code\n";
if ($result->is_success && $code eq "200"){
-       my $taint = $result->content;
-       my $content = sanitize($taint);
+       my $content = $result->content;
        if ($content) {
-
-            my $date = str2time($content);
-
-            if ($date) {
-                print "We've fetched a date $date.\n";
-                return $date;
-            } else {
-                print "We've haven't fetched a date.\n";
-                return "Unknown";
-            }
-
+	    return $sub->($content, $url);
         } else {
-            print "Unable to fetch date, empty content returned.\n";
-            return "Unknown";
+            print "Unable to fetch $url, empty content returned.\n";
         }
} else {
        print "Our request failed, we had no result.\n";
-       return "Unknown";
     }
-    return "Unknown";
+    return undef;
 }
# This is the list of all known Tor mirrors
@@ -1349,34 +1368,74 @@
 print "We have a total of $count mirrors\n";
 print "Fetching the last updated date for each mirror.\n";
-my $tortime;
-$tortime = FetchDate("https://www.torproject.org/");
+my $tortime = Fetch("https://www.torproject.org/project/trace/www-master.torproject.org", &ExtractDate);
+my @torfiles = Fetch("https://www.torproject.org/dist/", &ExtractLinks); 
+my %randomtorfiles;
+
+for (1 .. 1)
+{
+	my $r = int(rand(scalar(@torfiles)));
+	my $suffix = $torfiles[$r];
+	$suffix =~ s/^https://www.torproject.org//;
+	$randomtorfiles{$suffix} = Fetch($torfiles[$r], &ExtractSig);
+}
+
+print "Using these files for sig matching:\n";
+print join("\n", keys %randomtorfiles);
+
 # Adjust offical Tor time by out-of-date offset: number of days * seconds per day
 $tortime -= 1 * 172800;
 print "The official time for Tor is $tortime. \n";
+my %todelete;
foreach my $server ( keys %m ) {
-    print "Attempting to fetch from $m{$server}{'orgName'}\n";
+    print "Attempting to fetch from $server: $m{$server}{'orgName'}\n";
-    if ($m{$server}{'httpWebsiteMirror'}) {
-        print "Attempt to fetch via HTTP.\n";
-        $m{$server}{"updateDate"} = FetchDate("$m{$server}{'httpWebsiteMirror'}");
-    } elsif ($m{$server}{'httpsWebsiteMirror'}) {
-        print "Attempt to fetch via HTTPS.\n";
-        $m{$server}{"updateDate"} = FetchDate("$m{$server}{'httpsWebsiteMirror'}");
-    } elsif ($m{$server}{'ftpWebsiteMirror'}) {
-        print "Attempt to fetch via FTP.\n";
-        $m{$server}{"updateDate"} = FetchDate("$m{$server}{'ftpWebsiteMirror'}");
+    foreach my $serverType('httpWebsiteMirror', 'httpsWebsiteMirror', 'ftpWebsiteMirror')
+    {
+        if ($m{$server}{$serverType}) {
+            print "Attempt to fetch via $serverType.\n";
+            $m{$server}{"updateDate"} = Fetch("$m{$server}{$serverType}/project/trace/www-master.torproject.org", &ExtractDate);
+            if ($m{$server}{updateDate}) {
+                foreach my $randomtorfile(keys %randomtorfiles) {
+                	my $sig = Fetch("$m{$server}{$serverType}/$randomtorfile", &ExtractSig);
+            		if (!$sig) {
+			    $todelete{$server} = "Unreadable $randomtorfile";
+		        } elsif ($sig ne $randomtorfiles{$randomtorfile}) {
+			    $todelete{$server} = "Sig mismatch on $randomtorfile";
+            		} else {
+        	    	    print "Sig $sig matches for $randomtorfile on $m{$server}{$serverType}\n";
+			}
+            	        last;
+		}
+            } else {
+		delete $m{$server}{updateDate};
+		$todelete{$server} = "Unreadable date";
+            }
+	    last;
+        } 
+    }
+    if (exists $m{$server}{updateDate}) {
+        print "We fetched and stored the following: $m{$server}{'updateDate'}\n";
     } else {
-        print "We were unable to fetch or store anything. We still have the following: $m{$server}{'updateDate'}\n";
-    }
+	$todelete{$server} = "Unreadable date";
+    } 
+ }
-    print "We fetched and stored the following: $m{$server}{'updateDate'}\n";
+foreach my $outdated(keys %todelete)
+{
+    print "Deleted $outdated due to $todelete{$outdated}\n";
+    print Dumper($m{$outdated});
+    delete  $m{$outdated};
+}
- }
+foreach my $k(keys %m)
+{
+    print "$k\n";
+    print "$m{$k}{updateDate}\n";
+}
-
 print "We sorted the following mirrors by their date of last update: \n";
 foreach my $server ( sort { $m{$b}{'updateDate'} <=> $m{$a}{'updateDate'}} keys %m ) {
@@ -1394,7 +1453,7 @@
 open(OUT, "> $outFile") or die "Can't open $outFile: $!";
# Here's where we open a file and print some wml include goodness
-# This is storted from last known recent update to unknown update times
+# This is sorted from last known recent update to unknown update times
 foreach my $server ( sort { $m{$b}{'updateDate'} <=> $m{$a}{'updateDate'}} keys %m ) {
my $time;

    

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

[tor-commits] r26450: {website} new update mirrors script, see ticket 10269, update mirrors (in website/trunk: . include)