r26455: {website} update mirror script again, run it and update mirrors table. (in website/trunk: . include) - tor-commits

6 Dec 2013

Author: phobos
Date: 2013-12-06 03:49:41 +0000 (Fri, 06 Dec 2013)
New Revision: 26455
Modified:
   website/trunk/include/mirrors-table.wmi
   website/trunk/update-mirrors.pl
Log:
update mirror script again, run it and update mirrors table.
Modified: website/trunk/include/mirrors-table.wmi
===================================================================

--- website/trunk/include/mirrors-table.wmi	2013-12-06 02:57:45 UTC (rev 26454)
+++ website/trunk/include/mirrors-table.wmi	2013-12-06 03:49:41 UTC (rev 26455)
@@ -18,6 +18,23 @@
<tr>
+         <td>DK</td>
+
+         <td>Zentrum der Gesundheit</td>
+
+         <td>Up to date</td>
+
+    <td> - </td>
+    <td><a href="http://tor.idnr.ws/dist/">http</a></td>
+    <td><a href="http://tor.idnr.ws/">http</a></td>
+    <td> - </td>
+    <td> - </td>
+    <td> - </td>
+    <td> - </td>
+</tr>
+     
+<tr>
+
          <td>NO</td>
<td>MultiNet AS</td>
@@ -307,23 +324,6 @@
<tr>
-         <td>US</td>
-
-         <td>Xpdm</td>
-
-         <td>Up to date</td>
-
-    <td> - </td>
-    <td><a href="http://torproj.xpdm.us/dist/">http</a></td>
-    <td><a href="http://torproj.xpdm.us/">http</a></td>
-    <td><a href="https://torproj.xpdm.us/dist/">https</a></td>
-    <td><a href="https://torproj.xpdm.us/">https</a></td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
          <td>DE</td>
<td>beme it</td>
@@ -358,42 +358,25 @@
<tr>
-         <td>LT</td>
+         <td>MX</td>
<td></td>
<td>Up to date</td>
<td> - </td>
-    <td><a href="http://tor.vesta.nu/dist/">http</a></td>
-    <td><a href="http://tor.vesta.nu/">http</a></td>
+    <td><a href="http://fbnaia.homelinux.net/torproject/dist/">http</a></td>
+    <td><a href="http://fbnaia.homelinux.net/torproject/">http</a></td>
+    <td><a href="https://fbnaia.homelinux.net/torproject//dist/">https</a></td>
+    <td><a href="https://fbnaia.homelinux.net/torproject/">https</a></td>
     <td> - </td>
     <td> - </td>
-    <td> - </td>
-    <td> - </td>
 </tr>
<tr>
<td>US</td>
-         <td>AskApache</td>
-
-         <td>Up to date</td>
-
-    <td> - </td>
-    <td><a href="http://tor.askapache.com/dist/">http</a></td>
-    <td><a href="http://tor.askapache.com/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>US</td>
-
          <td>hessmo</td>
<td>Up to date</td>
@@ -494,6 +477,23 @@
<tr>
+         <td>NL</td>
+
+         <td>Amorphis</td>
+
+         <td>Up to date</td>
+
+    <td> - </td>
+    <td><a href="http://tor.amorphis.eu/dist/">http</a></td>
+    <td><a href="http://tor.amorphis.eu/">http</a></td>
+    <td> - </td>
+    <td> - </td>
+    <td> - </td>
+    <td> - </td>
+</tr>
+     
+<tr>
+
          <td>US</td>
<td>HackThisSite.org</td>
@@ -664,23 +664,6 @@
<tr>
-         <td>US</td>
-
-         <td></td>
-
-         <td>Up to date</td>
-
-    <td><a href="ftp://mirrors.go-parts.com/tor/">ftp</a></td>
-    <td><a href="http://mirrors.go-parts.com/tor/dist/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td><a href="rsync://mirrors.go-parts.com/mirrors/tor/">rsync</a></td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
          <td>DE</td>
<td>torservers</td>
@@ -749,53 +732,36 @@
<tr>
-         <td>IS</td>
+         <td>SE</td>
-         <td>myRL.net</td>
+         <td>homosu</td>
<td>Up to date</td>
<td> - </td>
-    <td><a href="http://tor.myrl.net/dist/">http</a></td>
-    <td><a href="http://tor.myrl.net/">http</a></td>
-    <td><a href="https://tor.myrl.net/dist/">https</a></td>
-    <td><a href="https://tor.myrl.net/">https</a></td>
+    <td><a href="http://tor.homosu.net/dist/">http</a></td>
+    <td><a href="http://tor.homosu.net/">http</a></td>
     <td> - </td>
     <td> - </td>
-</tr>
-     
-<tr>
-
-         <td>NL</td>
-
-         <td>Amorphis</td>
-
-         <td>Up to date</td>
-
     <td> - </td>
-    <td><a href="http://tor.amorphis.eu/dist/">http</a></td>
-    <td><a href="http://tor.amorphis.eu/">http</a></td>
     <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
 </tr>
<tr>
-         <td>DK</td>
+         <td>IS</td>
-         <td>Zentrum der Gesundheit</td>
+         <td>myRL.net</td>
<td>Up to date</td>
<td> - </td>
-    <td><a href="http://tor.idnr.ws/dist/">http</a></td>
-    <td><a href="http://tor.idnr.ws/">http</a></td>
+    <td><a href="http://tor.myrl.net/dist/">http</a></td>
+    <td><a href="http://tor.myrl.net/">http</a></td>
+    <td><a href="https://tor.myrl.net/dist/">https</a></td>
+    <td><a href="https://tor.myrl.net/">https</a></td>
     <td> - </td>
     <td> - </td>
-    <td> - </td>
-    <td> - </td>
 </tr>
<tr>
@@ -817,15 +783,15 @@
<tr>
-         <td>SE</td>
+         <td>LT</td>
-         <td>homosu</td>
+         <td></td>
<td>Up to date</td>
<td> - </td>
-    <td><a href="http://tor.homosu.net/dist/">http</a></td>
-    <td><a href="http://tor.homosu.net/">http</a></td>
+    <td><a href="http://tor.vesta.nu/dist/">http</a></td>
+    <td><a href="http://tor.vesta.nu/">http</a></td>
     <td> - </td>
     <td> - </td>
     <td> - </td>
@@ -834,15 +800,15 @@
<tr>
-         <td>IN</td>
+         <td>US</td>
-         <td>India Tor Fans</td>
+         <td>AskApache</td>
-         <td>DO NOT USE. Out of date.</td>
+         <td>Up to date</td>
<td> - </td>
-    <td><a href="http://www.torproject.org.in/dist/">http</a></td>
-    <td><a href="http://www.torproject.org.in/">http</a></td>
+    <td><a href="http://tor.askapache.com/dist/">http</a></td>
+    <td><a href="http://tor.askapache.com/">http</a></td>
     <td> - </td>
     <td> - </td>
     <td> - </td>
@@ -851,23 +817,6 @@
<tr>
-         <td>TN</td>
-
-         <td></td>
-
-         <td>DO NOT USE. Out of date.</td>
-
-    <td> - </td>
-    <td> - </td>
-    <td><a href="http://torproject.antagonism.org/">http</a></td>
-    <td> - </td>
-    <td><a href="https://torproject.antagonism.org/">https</a></td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
          <td>US</td>
<td></td>
@@ -902,23 +851,6 @@
<tr>
-         <td>US</td>
-
-         <td>searchprivate</td>
-
-         <td>DO NOT USE. Out of date.</td>
-
-    <td> - </td>
-    <td><a href="http://tor.searchprivate.com/dist/">http</a></td>
-    <td><a href="http://tor.searchprivate.com/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
          <td>DE</td>
<td></td>
@@ -953,23 +885,6 @@
<tr>
-         <td>DE</td>
-
-         <td>Netzkonstrukt Berlin</td>
-
-         <td>DO NOT USE. Out of date.</td>
-
-    <td> - </td>
-    <td><a href="http://mirror.ntzk.de/torproject.org/dist/">http</a></td>
-    <td><a href="http://mirror.ntzk.de/torproject.org/">http</a></td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
          <td>UA</td>
<td></td>
@@ -987,23 +902,6 @@
<tr>
-         <td>MX</td>
-
-         <td></td>
-
-         <td>Unknown</td>
-
-    <td> - </td>
-    <td><a href="http://fbnaia.homelinux.net/torproject/dist/">http</a></td>
-    <td><a href="http://fbnaia.homelinux.net/torproject/">http</a></td>
-    <td><a href="https://fbnaia.homelinux.net/torproject//dist/">https</a></td>
-    <td><a href="https://fbnaia.homelinux.net/torproject/">https</a></td>
-    <td> - </td>
-    <td> - </td>
-</tr>
-     
-<tr>
-
          <td>FR</td>
<td></td>
@@ -1171,3 +1069,37 @@
     <td> - </td>
     <td> - </td>
 </tr>
+     
+<tr>
+
+         <td>US</td>
+
+         <td>Xpdm</td>
+
+         <td>Failed signature check</td>
+
+    <td> - </td>
+    <td><a href="http://torproj.xpdm.us/dist/">http</a></td>
+    <td><a href="http://torproj.xpdm.us/">http</a></td>
+    <td><a href="https://torproj.xpdm.us/dist/">https</a></td>
+    <td><a href="https://torproj.xpdm.us/">https</a></td>
+    <td> - </td>
+    <td> - </td>
+</tr>
+     
+<tr>
+
+         <td>US</td>
+
+         <td></td>
+
+         <td>Failed signature check</td>
+
+    <td><a href="ftp://mirrors.go-parts.com/tor/">ftp</a></td>
+    <td><a href="http://mirrors.go-parts.com/tor/dist/">http</a></td>
+    <td> - </td>
+    <td> - </td>
+    <td> - </td>
+    <td><a href="rsync://mirrors.go-parts.com/mirrors/tor/">rsync</a></td>
+    <td> - </td>
+</tr>
Modified: website/trunk/update-mirrors.pl
===================================================================
--- website/trunk/update-mirrors.pl	2013-12-06 02:57:45 UTC (rev 26454)
+++ website/trunk/update-mirrors.pl	2013-12-06 03:49:41 UTC (rev 26455)
@@ -59,26 +59,31 @@
     my $date    = str2time($content);
if ($date) {
-    	print "ExtractDate($content) = $date\n";
+    	print "\tExtractDate($content) = $date\n";
         return $date;
     } else {
-    	print "ExtractDate($content) = ?\n";
+    	print "\tExtractDate($content) = ?\n";
    return undef;
     }
 }
sub ExtractSig {
-    my $content = shift;
-    return sha256_hex($content); 
+    my $content = shift; 
+    my $url     = shift;
+    my $sig = sha256_hex($content);
+    print "\tExtractSig($url) = $sig\n";
+    return $sig;
 }
sub Fetch {
     my ($url, $sub) = @_; # Base url for mirror
+    $|++; # unbuffer stdout to show progress
+    print "\nGET $url: ";
     my $request = new HTTP::Request GET => "$url";
     my $result = $lua->request($request);
     my $code = $result->code();
-    print "\nGET $url: $code\n";
+    print "$code\n";
if ($result->is_success && $code eq "200"){
        my $content = $result->content;
@@ -146,8 +151,9 @@
print "Using these files for sig matching:\n";
 print join("\n", keys %randomtorfiles);
+print "\n";
-# Adjust official Tor time by out-of-date offset: number of days * seconds per day
+# Adjust offical Tor time by out-of-date offset: number of days * seconds per day
 $tortime -= 1 * 172800;
 print "The official time for Tor is $tortime. \n";
@@ -163,12 +169,10 @@
                 foreach my $randomtorfile(keys %randomtorfiles) {
                     my $sig = Fetch("$m[$server]->{$serverType}/$randomtorfile", &ExtractSig);
             	    if (!$sig) {
-		        print STDERR "Unreadable $randomtorfile on $m[$server]->{$serverType}";
    		$m[$server]->{sigMatched} = 0;
             	    	last;
    	    } elsif ($sig ne $randomtorfiles{$randomtorfile}) {
    		$m[$server]->{sigMatched} = 0;
-		        print STDERR "Sig mismatch of $randomtorfile on $m[$server]->{$serverType}";
             	    	last;
             	    }
    	}
@@ -181,15 +185,13 @@
 sub PrintServer {
      my $server = shift;
      my $time;
-     if ( $server->{'updateDate'} ) {
-	  if ( $server->{'updateDate'} > $tortime ) {
-	    $time = "Up to date";
-	  } else { $time = "DO NOT USE. Out of date."; }
-     } else { $time = "Unknown"; }
+     if ( $server->{updateDate} && $server->{sigMatched} ) { $time = "Up to date"; } 
+     elsif (!$server->{updateDate}) 			   { $time = "Unknown"; }
+     else 			    			   { $time = "Failed signature check"; }
 print OUT <<"END";
      \n<tr>\n
-         <td>$server->{'isoCC'}</td>\n
-         <td>$server->{'orgName'}</td>\n
+         <td>$server->{isoCC}</td>\n
+         <td>$server->{orgName}</td>\n
          <td>$time</td>\n
 END
@@ -204,7 +206,7 @@
foreach my $precious ( sort keys %prettyNames )
      {
-        if ($server->{"$precious"}) {
+        if ($server->{$precious}) {
             print OUT "    <td><a href="" . $server->{$precious} . "">" .
                       "$prettyNames{$precious}</a></td>\n";
         } else { print OUT "    <td> - </td>\n"; }
@@ -219,12 +221,16 @@
# Here's where we open a file and print some wml include goodness
 # This is sorted from last known recent update to unknown update times
-foreach my $server ( sort { $b->{'updateDate'} <=> $a->{'updateDate'}} grep {$_->{updateDate} && $_->{sigMatched}} @m ) {
+foreach my $server ( sort { $b->{updateDate} <=> $a->{updateDate} } grep {$_->{updateDate} && $_->{updateDate} > $tortime && $_->{sigMatched}} @m ) {
     PrintServer($server);
 }
-foreach my $server ( grep {!$_->{updateDate} || !$_->{sigMatched}} @m ) {
+foreach my $server ( grep {!$_->{updateDate}}  @m ) {
     PrintServer($server);
 }
+foreach my $server ( grep {!$_->{sigMatched} && $_->{updateDate} && $_->{updateDate} > $tortime} @m ) {
+    PrintServer($server);
+}
+# That leaves those servers whose updateDate < torTime. Check the csv for those.
DumpMirrors(@m);

    