- tor-commits - lists.torproject.org

[gettor/master] Script and sample htaccess for creating GetTor API static mirror
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit 7735f5b69383ce26da39dc0dcdfd82f364ec10b8 Author: ilv <ilv(a)users.noreply.github.com> Date: Thu Dec 31 18:46:29 2015 -0300 Script and sample htaccess for creating GetTor API static mirror --- scripts/create_api_mirror.py | 305 +++++++++++++++++++++++++++++++++++++ scripts/sample-htaccess-api-mirror | 21 +++ 2 files changed, 326 insertions(+) diff --git a/scripts/create_api_mirror.py b/scripts/create_api_mirror.py new file mode 100644 index 0000000..e80ec4a --- /dev/null +++ b/scripts/create_api_mirror.py @@ -0,0 +1,305 @@ +# -*- coding: utf-8 -*- +# +# This file is part of GetTor. +# +# :authors: Israel Leiva <ilv(a)torproject.org> +# see also AUTHORS file +# +# :copyright: (c) 2008-2015, The Tor Project, Inc. +# (c) 2015, Israel Leiva +# +# :license: This is Free Software. See LICENSE for license information. + +import os +import re +import json +import codecs +import urllib2 +import ConfigParser + +from time import gmtime, strftime + + +"""Script to build a static mirror of GetTor RESTful API""" + +# currently supported locales for Tor Browser +LC = ['ar', 'de', 'en-US', 'es-ES', 'fa', 'fr', 'it', 'ko', 'nl', 'pl', + 'pt-PT', 'ru', 'tr', 'vi', 'zh-CN'] + +# https://gitweb.tpo/tor-browser-spec.git/tree/processes/VersionNumbers +# does not say anything about operating systems, so it's possible the +# notation might change in the future. We should always use the same three +# strings though: linux, windows, osx. +OS = { + 'Linux': 'linux', + 'Windows': 'windows', + 'MacOS': 'osx' +} + +# based on +# https://gitweb.tpo.org/tor-browser-spec.git/tree/processes/VersionNumbers +# except for the first one, which is based on current RecommendedTBBVersions +RE = { + 'os': '(.*)-(\w+)', + 'alpha': '\d\.\d(\.\d)*a\d+', + 'beta': '\d\.\d(\.\d)*b\d+', + 'stable': '\d\.\d(\.\d)*' +} + +# strings to build names of packages depending on OS. +PKG = { + 'windows': 'torbrowser-install-%s_%s.exe', + 'linux': 'tor-browser-linux%s-%s_%s.tar.xz', + 'osx': 'TorBrowser-%s-osx64_%s.dmg' +} + +# bin and asc are used to build the download links for each version, os and lc +URL = { + 'version': 'https://gettor.torproject.org/api/latest.json', + 'mirrors': 'https://gettor.torproject.org/api/mirrors.json', + 'providers': 'https://gettor.torproject.org/api/providers.json' +} + +GETTOR_URL = 'http://gettor.torproject.org' +SERVER_URL = 'https://yourprivateserver.com' +API_TREE = '/home/ilv/Proyectos/tor/static_api/' + +class ConfigError(Exception): + pass + + +class InternalError(Exception): + pass + + +class APIMirror(object): + """ Provide useful resources via RESTful API. """ + def __init__(self, cfg=None): + self.url = SERVER_URL + self.tree = API_TREE + + def _is_json(self, my_json): + """ Check if json generated is valid. + + :param: my_json (string) data to ve verified. + + :return: (bool) true if data is json-valid, false otherwise. + + """ + try: + json_object = json.loads(my_json) + except ValueError, e: + return False + return True + + def _write_json(self, path, content): + """ + """ + try: + with codecs.open( + path, + "w", + "utf-8" + ) as jsonfile: + # Make pretty json + json.dump( + content, + jsonfile, + sort_keys=True, + indent=4, + separators=(',', ': '), + encoding="utf-8", + ) + except IOError as e: + #logging.error("Couldn't write json: %s" % str(e)) + print "Error building %s: %s" % (path, str(e)) + print "%s built" % path + + def _get_provider_name(self, p): + """ Return simplified version of provider's name. + + :param: p (string) provider's name. + + :return: (string) provider's name in lowercase and without spaces. + + """ + p = p.replace(' ', '-') + return p.lower() + + + def _load_latest_version(self): + """ Load latest version data from GetTor API. """ + response = urllib2.urlopen(URL['version']) + json_response = json.load(response) + self.lv = json_response + + def _load_links(self): + """ Load links and providers data. """ + + response = urllib2.urlopen(URL['providers']) + json_response = json.load(response) + + links = {} + + for provider in json_response: + if provider != 'updated_at': + provider_url = json_response[provider] + + provider_response = urllib2.urlopen("%s.json" % provider_url) + provider_content = json.load(provider_response) + + json_response[provider] = provider_url.replace( + GETTOR_URL, + self.url + ) + + + links[provider] = provider_content + + self.providers = json_response + self.links = links + + def _load_mirrors(self): + """ Load mirrors data from GetTor API. """ + response = urllib2.urlopen(URL['mirrors']) + json_response = json.load(response) + self.mirrors = json_response + + def _load_resources(self): + """ Load available resources data. """ + + self.resources = { + 'providers': '%s/providers' % self.url, + 'mirrors': '%s/mirrors' % self.url, + 'latest_version': '%s/latest' % self.url, + 'updated_at': strftime("%Y-%m-%d %H:%M:%S", gmtime()) + } + + def load_data(self): + """ Load all data.""" + self._load_links() + self._load_mirrors() + self._load_resources() + self._load_latest_version() + + def build(self): + """ Build RESTful API. """ + + print "Building API mirror" + + # resources + self._write_json( + os.path.join(self.tree, 'api'), + self.resources + ) + + api_path = os.path.join(self.tree, 'api-content') + if not os.path.isdir(api_path): + os.mkdir(api_path) + + # providers + self._write_json( + os.path.join(api_path, 'providers'), + self.providers + ) + + providers_path = os.path.join(api_path, 'providers-content') + if not os.path.isdir(providers_path): + os.mkdir(providers_path) + + for provider in self.links: + if provider == 'updated_at': + continue + + self._write_json( + os.path.join(providers_path, provider), + self.links[provider] + ) + + provider_path = os.path.join( + providers_path, + "%s-content" % provider + ) + + if not os.path.isdir(provider_path): + os.mkdir(provider_path) + + for osys in self.links[provider]: + self._write_json( + os.path.join(provider_path, osys), + self.links[provider][osys] + ) + + provider_os_path = os.path.join( + provider_path, "%s-content" % osys + ) + + if not os.path.isdir(provider_os_path): + os.mkdir(provider_os_path) + + for lc in self.links[provider][osys]: + self._write_json( + os.path.join(provider_os_path, lc), + self.links[provider][osys][lc] + ) + + # latest version + self._write_json( + os.path.join(api_path, 'latest'), + self.lv + ) + + lv_path = os.path.join(api_path, 'latest-content') + if not os.path.isdir(lv_path): + os.mkdir(lv_path) + + for release in self.lv: + if release == 'updated_at': + continue + + self._write_json( + os.path.join(lv_path, release), + self.lv[release] + ) + + release_path = os.path.join( + lv_path, + "%s-content" % release + ) + + if not os.path.isdir(release_path): + os.mkdir(release_path) + + for osys in self.lv[release]['downloads']: + self._write_json( + os.path.join(release_path, osys), + self.lv[release]['downloads'][osys] + ) + + release_os_path = os.path.join( + release_path, + "%s-content" % osys + ) + + if not os.path.isdir(release_os_path): + os.mkdir(release_os_path) + + for lc in self.lv[release]['downloads'][osys]: + self._write_json( + os.path.join(release_os_path, lc), + self.lv[release]['downloads'][osys][lc] + ) + + # mirrors + self._write_json( + os.path.join(api_path, 'mirrors'), + self.mirrors + ) + +def main(): + api = APIMirror() + api.load_data() + api.build() + +if __name__ == '__main__': + main() diff --git a/scripts/sample-htaccess-api-mirror b/scripts/sample-htaccess-api-mirror new file mode 100644 index 0000000..41594db --- /dev/null +++ b/scripts/sample-htaccess-api-mirror @@ -0,0 +1,21 @@ +Options +FollowSymLinks +RewriteEngine On + +# Operating System and Locale filters for providers +RewriteCond %{QUERY_STRING} os=([a-z]+)&lc=([a-z]+) +RewriteRule ^api/providers/([a-z\-]+)$ api-content/providers-content/$1-content/%1-content/%2.json +RewriteCond %{QUERY_STRING} os=([a-z]+) +RewriteRule ^api/providers/([a-z\-]+)$ api-content/providers-content/$1-content/%1 + +# Operating System and Locale filters for TPO downloads +RewriteCond %{QUERY_STRING} os=([a-z]+)&lc=([a-zA-Z\-]+) +RewriteRule ^api/latest/([a-z]+)$ api-content/latest-content/$1-content/%1-content/%2.json +RewriteCond %{QUERY_STRING} os=([a-z]+) +RewriteRule ^api/latest/([a-z]+)$ api-content/latest-content/$1-content/%1 + +# General rules +RewriteRule ^api/providers/([a-z\-]+)$ api-content/providers-content/$1 [NC] +RewriteRule ^api/latest/([a-z]+)$ api-content/latest-content/$1 [NC] +RewriteRule ^api/([a-z]+)$ api-content/$1 [NC] + +ErrorDocument 404 404-file

1 0

[metrics-db/master] Add more robust descriptor reference checker.
by karsten＠torproject.org 15 Feb '16

15 Feb '16

commit 56288d49746ace3fdc03411c79f7d9b52d66c35d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Feb 11 22:41:52 2016 +0100 Add more robust descriptor reference checker. The new reference checker reads files in recent/relay-descriptors/, rather than keeping data structures of newly written descriptors in memory and storing them to disk later. This should be more robust against unplanned reboots. --- .../ernie/db/relaydescs/ArchiveWriter.java | 4 + .../ernie/db/relaydescs/ReferenceChecker.java | 310 +++++++++++++++++++++ 2 files changed, 314 insertions(+) diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java index 90b41c3..01b4d06 100644 --- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java +++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java @@ -57,6 +57,10 @@ public class ArchiveWriter extends Thread { // Import/download relay descriptors from the various sources new ArchiveWriter(config).run(); + new ReferenceChecker(new File("recent/relay-descriptors"), + new File("stats/references"), + new File("stats/references-history")).check(); + // Remove lock file lf.releaseLock(); diff --git a/src/org/torproject/ernie/db/relaydescs/ReferenceChecker.java b/src/org/torproject/ernie/db/relaydescs/ReferenceChecker.java new file mode 100644 index 0000000..4bafa76 --- /dev/null +++ b/src/org/torproject/ernie/db/relaydescs/ReferenceChecker.java @@ -0,0 +1,310 @@ +package org.torproject.ernie.db.relaydescs; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Locale; +import java.util.Set; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeSet; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorFile; +import org.torproject.descriptor.DescriptorReader; +import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.DirSourceEntry; +import org.torproject.descriptor.ExtraInfoDescriptor; +import org.torproject.descriptor.Microdescriptor; +import org.torproject.descriptor.NetworkStatusEntry; +import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.descriptor.RelayNetworkStatusVote; +import org.torproject.descriptor.ServerDescriptor; + +import com.google.gson.Gson; + +public class ReferenceChecker { + + private Logger log = Logger.getLogger(ReferenceChecker.class.getName()); + + private File descriptorsDir; + + private File referencesFile; + + private File historyFile; + + public ReferenceChecker(File descriptorsDir, File referencesFile, + File historyFile) { + this.descriptorsDir = descriptorsDir; + this.referencesFile = referencesFile; + this.historyFile = historyFile; + } + + public void check() { + this.getCurrentTimeMillis(); + this.readReferencesFile(); + this.readNewDescriptors(); + this.dropStaleReferences(); + this.checkReferences(); + this.writeReferencesFile(); + } + + private long currentTimeMillis; + + private void getCurrentTimeMillis() { + this.currentTimeMillis = System.currentTimeMillis(); + } + + private static class Reference implements Comparable<Reference> { + + private String referencing; + + private String referenced; + + private double weight; + + private long expiresAfterMillis; + + public Reference(String referencing, String referenced, double weight, + long expiresAfterMillis) { + this.referencing = referencing; + this.referenced = referenced; + this.weight = weight; + this.expiresAfterMillis = expiresAfterMillis; + } + + @Override + public boolean equals(Object otherObject) { + if (!(otherObject instanceof Reference)) { + return false; + } + Reference other = (Reference) otherObject; + return this.referencing.equals(other.referencing) && + this.referenced.equals(other.referenced); + } + + @Override + public int hashCode() { + return this.referencing.hashCode() + this.referenced.hashCode(); + } + + @Override + public int compareTo(Reference other) { + int result = this.referencing.compareTo(other.referencing); + if (result == 0) { + result = this.referenced.compareTo(other.referenced); + } + return result; + } + } + + private SortedSet<Reference> references = new TreeSet<Reference>(); + + private void addReference(String referencing, String referenced, + double weight, long expiresAfterMillis) { + this.references.add(new Reference(referencing.toUpperCase(), + referenced.toUpperCase(), weight, expiresAfterMillis)); + } + + private void readReferencesFile() { + if (!this.referencesFile.exists()) { + return; + } + Gson gson = new Gson(); + try { + FileReader fr = new FileReader(this.referencesFile); + this.references.addAll(Arrays.asList(gson.fromJson(fr, + Reference[].class))); + fr.close(); + } catch (IOException e) { + this.log.log(Level.WARNING, "Cannot read existing references file " + + "from previous run.", e); + } + } + + private void readNewDescriptors() { + DescriptorReader descriptorReader = + DescriptorSourceFactory.createDescriptorReader(); + descriptorReader.addDirectory(this.descriptorsDir); + descriptorReader.setExcludeFiles(this.historyFile); + Iterator<DescriptorFile> descriptorFiles = + descriptorReader.readDescriptors(); + while (descriptorFiles.hasNext()) { + DescriptorFile descriptorFile = descriptorFiles.next(); + for (Descriptor descriptor : descriptorFile.getDescriptors()) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + RelayNetworkStatusConsensus consensus = + (RelayNetworkStatusConsensus) descriptor; + String consensusFlavor = consensus.getConsensusFlavor(); + if (consensusFlavor == null) { + this.readRelayNetworkStatusConsensusUnflavored(consensus); + } else if (consensusFlavor.equals("microdesc")) { + this.readRelayNetworkStatusConsensusMicrodesc(consensus); + } else { + /* Ignore unknown consensus flavors. */ + } + } else if (descriptor instanceof RelayNetworkStatusVote) { + this.readRelayNetworkStatusVote( + (RelayNetworkStatusVote) descriptor); + } else if (descriptor instanceof ServerDescriptor) { + this.readServerDescriptor((ServerDescriptor) descriptor); + } else if (descriptor instanceof ExtraInfoDescriptor) { + this.readExtraInfoDescriptor((ExtraInfoDescriptor) descriptor); + } else if (descriptor instanceof Microdescriptor) { + readMicrodescriptor((Microdescriptor) descriptor); + } else { + /* Ignore unknown descriptors. */ + } + } + } + } + + private static DateFormat dateTimeFormat; + static { + dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", + Locale.US); + dateTimeFormat.setLenient(false); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + private static final long ONE_HOUR = 60L * 60L * 1000L, + THREE_HOURS = 3L * ONE_HOUR, SIX_HOURS = 6L * ONE_HOUR, + ONE_DAY = 24L * ONE_HOUR, THIRTY_DAYS = 30L * ONE_DAY; + + private void readRelayNetworkStatusConsensusUnflavored( + RelayNetworkStatusConsensus consensus) { + String validAfter = dateTimeFormat.format( + consensus.getValidAfterMillis()); + String referencing = String.format("C-%s", validAfter); + this.addReference(referencing, String.format("M-%s", validAfter), 1.0, + consensus.getValidAfterMillis() + THREE_HOURS); + for (DirSourceEntry dirSourceEntry : + consensus.getDirSourceEntries().values()) { + if (!dirSourceEntry.isLegacy()) { + this.addReference(referencing, String.format("V-%s-%s", + validAfter, dirSourceEntry.getIdentity()), 1.0, + consensus.getValidAfterMillis() + THREE_HOURS); + } + } + double entryWeight = 200.0 / + ((double) consensus.getStatusEntries().size()); + for (NetworkStatusEntry entry : + consensus.getStatusEntries().values()) { + this.addReference(referencing, + String.format("S-%s", entry.getDescriptor()), entryWeight, + entry.getPublishedMillis() + THREE_HOURS); + } + } + + + private void readRelayNetworkStatusConsensusMicrodesc( + RelayNetworkStatusConsensus consensus) { + String validAfter = dateTimeFormat.format( + consensus.getValidAfterMillis()); + String referencing = String.format("M-%s", validAfter); + this.addReference(referencing, String.format("C-%s", validAfter), 1.0, + consensus.getValidAfterMillis() + THREE_HOURS); + double entryWeight = 200.0 / + ((double) consensus.getStatusEntries().size()); + for (NetworkStatusEntry entry : + consensus.getStatusEntries().values()) { + for (String digest : entry.getMicrodescriptorDigests()) { + this.addReference(referencing, String.format("D-%s", digest), + entryWeight, entry.getPublishedMillis() + THREE_HOURS); + } + } + } + + private void readRelayNetworkStatusVote(RelayNetworkStatusVote vote) { + String validAfter = dateTimeFormat.format(vote.getValidAfterMillis()); + String referencing = String.format("V-%s-%s", validAfter, + vote.getIdentity()); + double entryWeight = 200.0 / + ((double) vote.getStatusEntries().size()); + for (NetworkStatusEntry entry : vote.getStatusEntries().values()) { + this.addReference(referencing, + String.format("S-%s", entry.getDescriptor()), entryWeight, + entry.getPublishedMillis() + SIX_HOURS); + } + } + + private void readServerDescriptor(ServerDescriptor serverDescriptor) { + String referenced = serverDescriptor.getExtraInfoDigest() == null ? "" + : String.format("E-%s", serverDescriptor.getExtraInfoDigest()); + this.addReference(String.format("S-%s", + serverDescriptor.getServerDescriptorDigest()), referenced, 0.01, + serverDescriptor.getPublishedMillis() + SIX_HOURS); + } + + private void readExtraInfoDescriptor( + ExtraInfoDescriptor extraInfoDescriptor) { + this.addReference(String.format("E-%s", + extraInfoDescriptor.getExtraInfoDigest()), "", 0.005, + extraInfoDescriptor.getPublishedMillis() + SIX_HOURS); + } + + private void readMicrodescriptor(Microdescriptor microdesc) { + this.addReference( + String.format("D-%s", microdesc.getMicrodescriptorDigest()), "", + 0.0, this.currentTimeMillis + THIRTY_DAYS); + } + + private void dropStaleReferences() { + SortedSet<Reference> recentReferences = new TreeSet<Reference>(); + for (Reference reference : this.references) { + if (this.currentTimeMillis <= reference.expiresAfterMillis) { + recentReferences.add(reference); + } + } + this.references = recentReferences; + } + + private void checkReferences() { + Set<String> knownDescriptors = new HashSet<String>(); + for (Reference reference : this.references) { + knownDescriptors.add(reference.referencing); + } + double totalMissingDescriptorsWeight = 0.0; + Set<String> missingDescriptors = new TreeSet<String>(); + StringBuilder sb = new StringBuilder("Missing referenced " + + "descriptors:"); + for (Reference reference : this.references) { + if (reference.referenced.length() > 0 && + !knownDescriptors.contains(reference.referenced)) { + if (!missingDescriptors.contains(reference.referenced)) { + totalMissingDescriptorsWeight += reference.weight; + } + missingDescriptors.add(reference.referenced); + sb.append(String.format("%n%s -> %s (%.4f -> %.4f)", + reference.referencing, reference.referenced, reference.weight, + totalMissingDescriptorsWeight)); + } + } + this.log.log(Level.INFO, sb.toString()); + if (totalMissingDescriptorsWeight > 0.999) { + this.log.log(Level.WARNING, "Missing too many referenced " + + "descriptors (" + totalMissingDescriptorsWeight + ")."); + } + } + + private void writeReferencesFile() { + Gson gson = new Gson(); + try { + FileWriter fw = new FileWriter(this.referencesFile); + gson.toJson(this.references, fw); + fw.close(); + } catch (IOException e) { + this.log.log(Level.WARNING, "Cannot write references file for next " + + "run.", e); + } + } +} +

1 0

[metrics-db/master] Keep microdescriptors in recent/ for up to 30 days.
by karsten＠torproject.org 15 Feb '16

15 Feb '16

commit 7f0b062bcd0dee7658092784706cec7b860a1b66 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Feb 12 11:06:04 2016 +0100 Keep microdescriptors in recent/ for up to 30 days. We'll need to keep microdescriptors around longer than three days, because they change less frequently and are referenced much longer than, for example, server descriptors. --- src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java index 01b4d06..a85408c 100644 --- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java +++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java @@ -802,17 +802,23 @@ public class ArchiveWriter extends Thread { } /* Delete all files from the rsync directory that have not been modified - * in the last three days, and remove the .tmp extension from newly + * in the last three days (except for microdescriptors which are kept + * for up to thirty days), and remove the .tmp extension from newly * written files. */ public void cleanUpRsyncDirectory() { long cutOffMillis = System.currentTimeMillis() - 3L * 24L * 60L * 60L * 1000L; + long cutOffMicroMillis = cutOffMillis - 27L * 24L * 60L * 60L * 1000L; Stack<File> allFiles = new Stack<File>(); allFiles.add(new File("recent/relay-descriptors")); while (!allFiles.isEmpty()) { File file = allFiles.pop(); if (file.isDirectory()) { allFiles.addAll(Arrays.asList(file.listFiles())); + } else if (file.getName().endsWith("-micro")) { + if (file.lastModified() < cutOffMicroMillis) { + file.delete(); + } } else if (file.lastModified() < cutOffMillis) { file.delete(); } else if (file.getName().endsWith(".tmp")) {

1 0

[metrics-db/master] Tone down missing-descriptor warnings.
by karsten＠torproject.org 15 Feb '16

15 Feb '16

commit 0d873c4bf95d15cc0fa67160bea38eb9ba5bac50 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Feb 12 11:12:30 2016 +0100 Tone down missing-descriptor warnings. We should remove these checks in favor of the more robust reference check, but let's keep the log lines for now as a backup. --- src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java index a85408c..60cfca9 100644 --- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java +++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java @@ -731,7 +731,7 @@ public class ArchiveWriter extends Thread { } this.logger.info(sb.toString()); if (missingDescriptors) { - this.logger.warning("We are missing at least 0.5% of server or " + this.logger.fine("We are missing at least 0.5% of server or " + "extra-info descriptors referenced from a consensus or " + "vote or at least 0.5% of microdescriptors referenced from a " + "microdesc consensus."); @@ -739,13 +739,13 @@ public class ArchiveWriter extends Thread { if (missingVotes) { /* TODO Shouldn't warn if we're not trying to archive votes at * all. */ - this.logger.warning("We are missing at least one vote that was " + this.logger.fine("We are missing at least one vote that was " + "referenced from a consensus."); } if (missingMicrodescConsensus) { /* TODO Shouldn't warn if we're not trying to archive microdesc * consensuses at all. */ - this.logger.warning("We are missing at least one microdesc " + this.logger.fine("We are missing at least one microdesc " + "consensus that was published together with a known " + "consensus."); }

1 0

[tor-browser-bundle/hardened-builds] Changelog update
by gk＠torproject.org 15 Feb '16

15 Feb '16

commit 05c2dcf6ad89442e1ce46d79d93cce51e24243e2 Author: Georg Koppen <gk(a)torproject.org> Date: Mon Feb 15 14:53:48 2016 +0000 Changelog update --- Bundle-Data/Docs/ChangeLog.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Bundle-Data/Docs/ChangeLog.txt b/Bundle-Data/Docs/ChangeLog.txt index d2b7420..d115483 100644 --- a/Bundle-Data/Docs/ChangeLog.txt +++ b/Bundle-Data/Docs/ChangeLog.txt @@ -13,6 +13,7 @@ Tor Browser 6.0a2-hardened -- February 15 2016 * Build System * Linux * Bug 15578: Switch to Debian Wheezy guest VMs (10.04 LTS is EOL) + * Bug 18198: Building the hardened Tor Browser in a Debian Wheezy VM is broken Tor Browser 6.0a1-hardened -- January 27 2016 * All Platforms

1 0

[nyx/master] Drop unused get_data_directory()
by atagar＠torproject.org 15 Feb '16

15 Feb '16

commit 57a3e6ef083da10db12d0f12d9580e4e1991ef1a Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Feb 10 11:28:55 2016 -0800 Drop unused get_data_directory() Method is unused. If it wasnn't then our util's DATA_DIR is the thing to use. --- nyx/controller.py | 19 +------------------ nyx/log_panel.py | 5 +---- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/nyx/controller.py b/nyx/controller.py index cfd40e8..a5f82a1 100644 --- a/nyx/controller.py +++ b/nyx/controller.py @@ -22,10 +22,10 @@ import nyx.util.tracker import stem from stem.control import State +from stem.util import conf, log from nyx.util import panel, tor_controller, ui_tools -from stem.util import conf, log NYX_CONTROLLER = None @@ -39,7 +39,6 @@ def conf_handler(key, value): CONFIG = conf.config_dict('nyx', { 'startup.events': 'N3', - 'startup.data_directory': '~/.nyx', 'features.acsSupport': True, 'features.panels.show.graph': True, 'features.panels.show.log': True, @@ -422,22 +421,6 @@ class Controller: else: self._force_redraw = True - def get_data_directory(self): - """ - Provides the path where nyx's resources are being placed. The path ends - with a slash and is created if it doesn't already exist. - """ - - data_dir = os.path.expanduser(CONFIG['startup.data_directory']) - - if not data_dir.endswith('/'): - data_dir += '/' - - if not os.path.exists(data_dir): - os.makedirs(data_dir) - - return data_dir - def quit(self): self.quit_signal = True diff --git a/nyx/log_panel.py b/nyx/log_panel.py index 4fdb8ad..37e9e21 100644 --- a/nyx/log_panel.py +++ b/nyx/log_panel.py @@ -20,9 +20,7 @@ from stem.util import conf, log def conf_handler(key, value): - if key == 'features.log.maxLineWrap': - return max(1, value) - elif key == 'features.log.prepopulateReadLimit': + if key == 'features.log.prepopulateReadLimit': return max(0, value) elif key == 'features.log.maxRefreshRate': return max(10, value) @@ -33,7 +31,6 @@ def conf_handler(key, value): CONFIG = conf.config_dict('nyx', { 'features.logFile': '', 'features.log.showDuplicateEntries': False, - 'features.log.maxLineWrap': 6, 'features.log.prepopulate': True, 'features.log.prepopulateReadLimit': 5000, 'features.log.maxRefreshRate': 300,

1 0

[nyx/master] Drop unused get_last_redraw_time()
by atagar＠torproject.org 15 Feb '16

15 Feb '16

commit 71bab1d655dcfb16b7607f8ce1df000775d8c786 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Feb 10 11:51:24 2016 -0800 Drop unused get_last_redraw_time() The private attribute is still used in one spot internally, but method is unused. --- nyx/controller.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/nyx/controller.py b/nyx/controller.py index a5f82a1..aadad15 100644 --- a/nyx/controller.py +++ b/nyx/controller.py @@ -382,14 +382,6 @@ class Controller: self._force_redraw = True - def get_last_redraw_time(self): - """ - Provides the time when the content was last redrawn, zero if the content - has never been drawn. - """ - - return self._last_drawn - def set_msg(self, msg = None, attr = None, redraw = False): """ Sets the message displayed in the interfaces control panel. This uses our

1 0

[nyx/master] Drop request_redraw()
by atagar＠torproject.org 15 Feb '16

15 Feb '16

commit cfa6b91044bfcb62a543aaf267f377429a7dc1de Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Feb 10 12:01:50 2016 -0800 Drop request_redraw() Really not needed in the one spot we use it. --- nyx/controller.py | 7 ------- nyx/popups.py | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/nyx/controller.py b/nyx/controller.py index aadad15..dc9f53b 100644 --- a/nyx/controller.py +++ b/nyx/controller.py @@ -375,13 +375,6 @@ class Controller: if force: self._last_drawn = current_time - def request_redraw(self): - """ - Requests that all content is redrawn when the interface is next rendered. - """ - - self._force_redraw = True - def set_msg(self, msg = None, attr = None, redraw = False): """ Sets the message displayed in the interfaces control panel. This uses our diff --git a/nyx/popups.py b/nyx/popups.py index c25b321..d6549a6 100644 --- a/nyx/popups.py +++ b/nyx/popups.py @@ -68,8 +68,8 @@ def popup_window(height = -1, width = -1, top = 0, left = 0, below_static = True return (None, 0, 0) def __exit__(self, exit_type, value, traceback): - nyx.controller.get_controller().request_redraw() panel.CURSES_LOCK.release() + nyx.controller.get_controller().redraw(True) return _Popup()

1 0

[nyx/master] Merge heartbeat_check() into header panel
by atagar＠torproject.org 15 Feb '16

15 Feb '16

commit 2013e771145df9ee845e4098011e308cafeb55d7 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Feb 10 19:35:20 2016 -0800 Merge heartbeat_check() into header panel Our header panel already provides notices for file descriptor usage. Heartbeat tracking is pretty similar. --- nyx/controller.py | 25 ------------------------- nyx/header_panel.py | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 27 deletions(-) diff --git a/nyx/controller.py b/nyx/controller.py index 07cab4d..0f972b3 100644 --- a/nyx/controller.py +++ b/nyx/controller.py @@ -400,29 +400,6 @@ class Controller: self.quit_signal = True -def heartbeat_check(is_unresponsive): - """ - Logs if its been ten seconds since the last BW event. - - Arguments: - is_unresponsive - flag for if we've indicated to be responsive or not - """ - - controller = tor_controller() - last_heartbeat = controller.get_latest_heartbeat() - - if controller.is_alive(): - if not is_unresponsive and (time.time() - last_heartbeat) >= 10: - is_unresponsive = True - log.notice('Relay unresponsive (last heartbeat: %s)' % time.ctime(last_heartbeat)) - elif is_unresponsive and (time.time() - last_heartbeat) < 10: - # really shouldn't happen (meant Tor froze for a bit) - is_unresponsive = False - log.notice('Relay resumed') - - return is_unresponsive - - def start_nyx(stdscr): """ Main draw loop context. @@ -469,11 +446,9 @@ def start_nyx(stdscr): # main draw loop override_key = None # uses this rather than waiting on user input - is_unresponsive = False # flag for heartbeat responsiveness check while not control.quit_signal: display_panels = control.get_display_panels() - is_unresponsive = heartbeat_check(is_unresponsive) # sets panel visability diff --git a/nyx/header_panel.py b/nyx/header_panel.py index 4a0841e..bbc5ede 100644 --- a/nyx/header_panel.py +++ b/nyx/header_panel.py @@ -43,6 +43,7 @@ class HeaderPanel(panel.Panel, threading.Thread): self._pause_condition = threading.Condition() self._halt = False # terminates thread if true + self._reported_inactive = False tor_controller().add_status_listener(self.reset_listener) @@ -217,7 +218,8 @@ class HeaderPanel(panel.Panel, threading.Thread): """ x = self.addstr(y, x, 'Tor Disconnected', curses.A_BOLD, 'red') - self.addstr(y, x, vals.format(' ({last_heartbeat}, press r to reconnect)')) + last_heartbeat = time.strftime('%H:%M %m/%d/%Y', time.localtime(vals.last_heartbeat)) + self.addstr(y, x, ' (%s, press r to reconnect)' % last_heartbeat) def _draw_resource_usage(self, x, y, width, vals): """ @@ -388,6 +390,14 @@ class HeaderPanel(panel.Panel, threading.Thread): log_msg = msg('panel.header.fd_used_at_sixty_percent', percentage = fd_percent) log.log_once('fd_used_at_sixty_percent', log.NOTICE, log_msg) + if self._vals.is_connected: + if not self._reported_inactive and (time.time() - self._vals.last_heartbeat) >= 10: + self._reported_inactive = True + log.notice('Relay unresponsive (last heartbeat: %s)' % time.ctime(self._vals.last_heartbeat)) + elif self._reported_inactive and (time.time() - self._vals.last_heartbeat) < 10: + self._reported_inactive = False + log.notice('Relay resumed') + if previous_height != self.get_height(): # We're toggling between being a relay and client, causing the height # of this panel to change. Redraw all content so we don't get @@ -436,7 +446,7 @@ def get_sampling(last_sampling = None): 'retrieved': retrieved, 'is_connected': controller.is_alive(), 'connection_time': controller.connection_time(), - 'last_heartbeat': time.strftime('%H:%M %m/%d/%Y', time.localtime(controller.get_latest_heartbeat())), + 'last_heartbeat': controller.get_latest_heartbeat(), 'fingerprint': controller.get_info('fingerprint', 'Unknown'), 'nickname': controller.get_conf('Nickname', ''),

1 0

[nyx/master] Drop conn_reset_listener()
by atagar＠torproject.org 15 Feb '16

15 Feb '16

commit 16dc6d613c2ad7f9a982f351ff4a746d58f6deb3 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Feb 10 19:05:06 2016 -0800 Drop conn_reset_listener() Moving the last remnents of the controller's status listener into other spots. --- nyx/controller.py | 12 ------------ nyx/header_panel.py | 5 ++++- nyx/settings/strings.cfg | 1 + nyx/util/tracker.py | 3 +++ 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/nyx/controller.py b/nyx/controller.py index fcc6239..07cab4d 100644 --- a/nyx/controller.py +++ b/nyx/controller.py @@ -3,7 +3,6 @@ Main interface loop for nyx, periodically redrawing the screen and issuing user input to the proper panels. """ -import os import time import curses import threading @@ -20,7 +19,6 @@ import nyx.util.tracker import stem -from stem.control import State from stem.util import conf, log from nyx.util import panel, tor_controller, ui_tools @@ -111,11 +109,6 @@ def init_controller(stdscr, start_time): if CONFIG['features.panels.show.connection']: page_panels.append([nyx.connection_panel.ConnectionPanel(stdscr)]) - controller = tor_controller() - controller.add_status_listener(conn_reset_listener) - resolver = nyx.util.tracker.get_connection_tracker() - log.info('Operating System: %s, Connection Resolvers: %s' % (os.uname()[0], ', '.join(resolver._resolvers))) - # third page: config if CONFIG['features.panels.show.config']: @@ -430,11 +423,6 @@ def heartbeat_check(is_unresponsive): return is_unresponsive -def conn_reset_listener(controller, event_type, _): - if event_type == State.CLOSED: - log.notice('Tor control port closed') - - def start_nyx(stdscr): """ Main draw loop context. diff --git a/nyx/header_panel.py b/nyx/header_panel.py index 4eb0a84..4a0841e 100644 --- a/nyx/header_panel.py +++ b/nyx/header_panel.py @@ -15,7 +15,7 @@ import stem import nyx.controller import nyx.popups -from stem.control import Listener +from stem.control import Listener, State from stem.util import conf, log, proc, str_tools, system from nyx.util import msg, tor_controller, panel, tracker @@ -370,6 +370,9 @@ class HeaderPanel(panel.Panel, threading.Thread): def reset_listener(self, controller, event_type, _): self._update() + if event_type == State.CLOSED: + log.notice('Tor control port closed') + def _update(self): previous_height = self.get_height() self._vals = get_sampling(self._vals) diff --git a/nyx/settings/strings.cfg b/nyx/settings/strings.cfg index bfe375f..f669e43 100644 --- a/nyx/settings/strings.cfg +++ b/nyx/settings/strings.cfg @@ -37,6 +37,7 @@ msg.setup.unknown_event_types nyx doesn't recognize the following event types: { msg.setup.color_support_available Terminal color support detected and enabled msg.setup.color_support_unavailable Terminal color support unavailable +msg.tracker.available_resolvers Operating System: {os}, Connection Resolvers: {resolvers} msg.tracker.abort_getting_resources Failed three attempts to get process resource usage from {resolver}, {response} ({exc}) msg.tracker.abort_getting_port_usage Failed three attempts to determine the process using active ports ({exc}) msg.tracker.lookup_rate_increased connection lookup time increasing to {seconds} seconds per call diff --git a/nyx/util/tracker.py b/nyx/util/tracker.py index 8aa1181..2b2b155 100644 --- a/nyx/util/tracker.py +++ b/nyx/util/tracker.py @@ -47,6 +47,7 @@ Background tasks for gathering information about the tor process. """ import collections +import os import time import threading @@ -501,6 +502,8 @@ class ConnectionTracker(Daemon): else: self._resolvers = [CustomResolver.INFERENCE] + log.info('tracker.available_resolvers', os = os.uname()[0], resolvers = ', '.join(self._resolvers)) + def _task(self, process_pid, process_name): if self._custom_resolver: resolver = self._custom_resolver

1 0