August 2017 - tor-commits - lists.torproject.org

[onionoo/master] Prepare for 4.1-1.4.0 release.
by karsten＠torproject.org 31 Aug '17

31 Aug '17

commit b3bb600f3495603ea4f13e94d9b99556c0be9dc0 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Aug 30 22:35:48 2017 +0200 Prepare for 4.1-1.4.0 release. --- CERT | 18 +++++++++--------- CHANGELOG.md | 6 +++--- build.xml | 2 +- .../org/torproject/onionoo/server/ResponseBuilder.java | 2 +- 4 files changed, 14 insertions(+), 14 … [View More]deletions(-) diff --git a/CERT b/CERT index 6cc5a93..d94c0c3 100644 --- a/CERT +++ b/CERT @@ -1,8 +1,8 @@ -----BEGIN CERTIFICATE----- -MIIDaTCCAlGgAwIBAgIEYQnM/zANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV +MIIDaTCCAlGgAwIBAgIEZTniETANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV UzELMAkGA1UECBMCV0ExEDAOBgNVBAcTB1NlYXR0bGUxHTAbBgNVBAoTFFRoZSBU b3IgUHJvamVjdCwgSW5jMRgwFgYDVQQDEw9LYXJzdGVuIExvZXNpbmcwHhcNMTcw -MTAyMTYzNTQ2WhcNMTcwNDAyMTYzNTQ2WjBlMQswCQYDVQQGEwJVUzELMAkGA1UE +ODE3MTg1MDQ0WhcNMTcxMTE1MTg1MDQ0WjBlMQswCQYDVQQGEwJVUzELMAkGA1UE CBMCV0ExEDAOBgNVBAcTB1NlYXR0bGUxHTAbBgNVBAoTFFRoZSBUb3IgUHJvamVj dCwgSW5jMRgwFgYDVQQDEw9LYXJzdGVuIExvZXNpbmcwggEiMA0GCSqGSIb3DQEB AQUAA4IBDwAwggEKAoIBAQChXn+IUp+o6G+k4ffxk3TkxZb3iXfiG7byNsG63olU @@ -11,11 +11,11 @@ Qw+VAhKTcEIv4yiR0BWapQyR07pgmKirYVjN6s6ef8NJzUptpxLlaYJ3ZfQfc4aE MXzScgaccwDFIWQ661lzLGCfeSxxa3Xy4wWsGwzNzLITYrrABcbg7yogLo2btNvD oEwGL3/baQdhl0dra6biVCZr9ydn3Hg57S55pUU0rBY25id78zUO8xrfNHw54wwX lOblGt75OOkahP/ZZSBxxoiknJ6y5VQV8y+noA4vigXFAgMBAAGjITAfMB0GA1Ud -DgQWBBSeh60M+/wMYyYhlxtuff2Hk9n7bzANBgkqhkiG9w0BAQsFAAOCAQEAi2K5 -o3UnRBLdk9zkBEDbqPWxq8eiiIacbSoorspVlv4vqNO2UsTLc2sA4zZpY1L7Mkc6 -y7pcd1jFc6e9Wzt0agoIU9i3pyqkUx11ktE8a8CiVjDOns05GHe1DsLYcSLG8aJR -2pNEFjZEndHTXGRKwxtTmHimsIIWiy8tYFkC7SkBQj+BnXBqzCbJPhRDpMdhyJmQ -CLUhTTgU30G32x3vOhddWHAH8UDsXNlBASQOFs+A/1Mx5DI5AXLtwjp/rweTWoBk -11SfkVS/mfMZNRGOA9DEfoRYHPT0Rn4URAQrBr73CKQaVVYyeuDOoEs69KRH2JSO -IRcgAfpjtg8cry/eiw== +DgQWBBSeh60M+/wMYyYhlxtuff2Hk9n7bzANBgkqhkiG9w0BAQsFAAOCAQEAbsAc +gwl5KJH3pVKw4b+ACCOMgW+27MisCFbT1Izq2Wx+JcLMt3N//MoIpYOZWhsIeazW +/NE0fNbkLi0IYA0F1nUC9pHl44Hd8Gjfqa/YQUi9ALtgsY7l6W0sceW8WnZ8bu8J +DfrqnmB0bD2xc9ZjOn58al8dVjVWs95M87D9WCRU6LiaKFj5c45wciABQsTmC0qD +pyHYOaSGtXxXKDw5pAntdtHkCbowV5tDi/QQ8Tg7i5O7xwSh71Q7TZiNFMpLomBL +QllHfTZryFmoHyGn5MfngBUVCVHig5nXmk0dUMGuLiK4789dkgiPRz0vpB5Yf8Yy +CCE2jB6VBi2g5fMx0w== -----END CERTIFICATE----- diff --git a/CHANGELOG.md b/CHANGELOG.md index 9538287..20ddb1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 4.1-1.?.? - 2017-0?-?? +# Changes in version 4.1-1.4.0 - 2017-08-30 * Medium changes - Reset IPv6 exit-policy summary in details status if a newer @@ -6,10 +6,10 @@ - Remove optional fields "countries", "transports", and "versions" from clients objects which were still labeled as beta. - Add new "version" parameter to filter for Tor version. - - Switch from our own CollecTor downloader to metrics-lib's - DescriptorCollector. * Minor changes + - Switch from our own CollecTor downloader to metrics-lib's + DescriptorCollector. - Add a new Java property "onionoo.basedir" to re-configure the base directory used by the web server component. diff --git a/build.xml b/build.xml index 24010dd..41f11c0 100644 --- a/build.xml +++ b/build.xml @@ -10,7 +10,7 @@ <property name="implementation-title" value="Onionoo" /> <property name="onionoo.protocol.version" value="4.1"/> <property name="release.version" - value="${onionoo.protocol.version}-1.3.0-dev"/> + value="${onionoo.protocol.version}-1.4.0"/> <property name="metricslibversion" value="2.0.0"/> <property name="jetty.version" value="-9.2.21.v20170120" /> <property name="warfile" diff --git a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java index c9cec32..efa02c3 100644 --- a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java +++ b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java @@ -102,7 +102,7 @@ public class ResponseBuilder { return this.charsWritten; } - private static final String PROTOCOL_VERSION = "4.0"; + private static final String PROTOCOL_VERSION = "4.1"; private static final String NEXT_MAJOR_VERSION_SCHEDULED = null; [View Less]

1 0

[collector/master] Use metrics-lib's JSON handling classes.
by karsten＠torproject.org 30 Aug '17

30 Aug '17

commit d2677cba06aaa94f67a6645a1da20d96502e34ba Author: iwakeh <iwakeh(a)torproject.org> Date: Fri Aug 18 13:03:04 2017 +0000 Use metrics-lib's JSON handling classes. Implements task-23286 using metrics-lib-2.0.0 --- .../collector/index/CreateIndexJson.java | 86 ++++------------------ 1 file changed, 15 insertions(+), 71 deletions(-) diff --git a/src/main/java/org/torproject/collector/index/CreateIndexJson.java b/src/main/java/org/torproject/collector/index/… [View More]CreateIndexJson.java index 4b1b8f0..4b55d51 100644 --- a/src/main/java/org/torproject/collector/index/CreateIndexJson.java +++ b/src/main/java/org/torproject/collector/index/CreateIndexJson.java @@ -7,29 +7,27 @@ import org.torproject.collector.conf.Configuration; import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; import org.torproject.collector.cron.CollecTorMain; +import org.torproject.descriptor.index.DirectoryNode; +import org.torproject.descriptor.index.FileNode; +import org.torproject.descriptor.index.FileType; +import org.torproject.descriptor.index.IndexNode; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; -import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; -import java.io.FileWriter; -import java.io.IOException; import java.io.OutputStreamWriter; -import java.io.Writer; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Locale; import java.util.SortedSet; import java.util.TimeZone; import java.util.TreeSet; -import java.util.zip.GZIPOutputStream; /* Create a fresh index.json containing all directories and files in the * archive/ and recent/ directories. @@ -87,57 +85,6 @@ public class CreateIndexJson extends CollecTorMain { } } - private class DirectoryNode implements Comparable<DirectoryNode> { - String path; - SortedSet<FileNode> files; - SortedSet<DirectoryNode> directories; - - DirectoryNode(String path, SortedSet<FileNode> files, - SortedSet<DirectoryNode> directories) { - this.path = path; - this.files = files; - this.directories = directories; - } - - public int compareTo(DirectoryNode other) { - return this.path.compareTo(other.path); - } - } - - @SuppressWarnings({"checkstyle:membername", "checkstyle:parametername"}) - private class IndexNode { - String index_created; - String path; - SortedSet<FileNode> files; - SortedSet<DirectoryNode> directories; - - IndexNode(String index_created, String path, - SortedSet<FileNode> files, - SortedSet<DirectoryNode> directories) { - this.index_created = index_created; - this.path = path; - this.files = files; - this.directories = directories; - } - } - - @SuppressWarnings({"checkstyle:membername", "checkstyle:parametername"}) - private class FileNode implements Comparable<FileNode> { - String path; - long size; - String last_modified; - - FileNode(String path, long size, String last_modified) { - this.path = path; - this.size = size; - this.last_modified = last_modified; - } - - public int compareTo(FileNode other) { - return this.path.compareTo(other.path); - } - } - private static DateFormat dateTimeFormat; static { @@ -198,24 +145,21 @@ public class CreateIndexJson extends CollecTorMain { return fileNode; } - private void writeIndex(IndexNode indexNode) throws IOException { + private void writeIndex(IndexNode indexNode) throws Exception { indexJsonFile.getParentFile().mkdirs(); Gson gson = new GsonBuilder().create(); String indexNodeString = gson.toJson(indexNode); - Writer[] writers = new Writer[] { - new FileWriter(indexJsonFile), - new OutputStreamWriter(new GZIPOutputStream( - new FileOutputStream(indexJsonFile + ".gz"))), - new OutputStreamWriter(new XZCompressorOutputStream( - new FileOutputStream(indexJsonFile + ".xz"))), - new OutputStreamWriter(new BZip2CompressorOutputStream( - new FileOutputStream(indexJsonFile + ".bz2"))) - }; - for (Writer writer : writers) { - BufferedWriter bufferedWriter = new BufferedWriter(writer); - bufferedWriter.write(indexNodeString); - bufferedWriter.close(); + for (String filename : new String[] {indexJsonFile.toString(), + indexJsonFile + ".gz", indexJsonFile + ".xz", indexJsonFile + ".bz2"}) { + FileType type = FileType.valueOf( + filename.substring(filename.lastIndexOf(".") + 1).toUpperCase()); + try (BufferedWriter bufferedWriter + = new BufferedWriter(new OutputStreamWriter(type.outputStream( + new FileOutputStream(filename))))) { + bufferedWriter.write(indexNodeString); + } } } + } [View Less]

1 0

[onionoo/master] Add system property for all paths.
by karsten＠torproject.org 30 Aug '17

30 Aug '17

commit 14f6192ec88971dbd42ad1cc59e271c029a5646e Author: iwakeh <iwakeh(a)torproject.org> Date: Tue Aug 22 07:41:02 2017 +0000 Add system property for all paths. The system property is 'onionoo.basedir'. If not set, backwards compatible defaults apply. Implements task-14201. --- CHANGELOG.md | 4 ++++ src/main/java/org/torproject/onionoo/cron/Main.java | 3 +-- src/main/java/org/torproject/onionoo/server/… [View More]NodeIndexer.java | 7 +++---- src/main/resources/web.xml | 5 ----- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e028dd1..9538287 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ - Switch from our own CollecTor downloader to metrics-lib's DescriptorCollector. + * Minor changes + - Add a new Java property "onionoo.basedir" to re-configure the + base directory used by the web server component. + # Changes in version 4.0-1.3.0 - 2017-08-04 diff --git a/src/main/java/org/torproject/onionoo/cron/Main.java b/src/main/java/org/torproject/onionoo/cron/Main.java index 685da35..f6f7099 100644 --- a/src/main/java/org/torproject/onionoo/cron/Main.java +++ b/src/main/java/org/torproject/onionoo/cron/Main.java @@ -22,8 +22,7 @@ import java.util.concurrent.TimeUnit; /* Update search data and status data files. */ public class Main implements Runnable { - private Main() { - } + private Main() {} private Logger log = LoggerFactory.getLogger(Main.class); diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index 7ed9ea6..db13ed9 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -20,7 +20,6 @@ import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; -import javax.servlet.ServletContext; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; @@ -31,11 +30,11 @@ public class NodeIndexer implements ServletContextListener, Runnable { @Override public void contextInitialized(ServletContextEvent contextEvent) { - ServletContext servletContext = contextEvent.getServletContext(); - File outDir = new File(servletContext.getInitParameter("outDir")); + File outDir = new File(System.getProperty("onionoo.basedir", + "/srv/onionoo.torproject.org/onionoo"), "out"); if (!outDir.exists() || !outDir.isDirectory()) { log.error("\n\n\tOut-dir not found! Expected directory: " + outDir - + "\n\tVerify the configuration in ./etc/web.xml.template"); + + "\n\tSet system property 'onionoo.basedir'."); System.exit(1); } DocumentStore documentStore = DocumentStoreFactory.getDocumentStore(); diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml index cefdfae..da1431b 100644 --- a/src/main/resources/web.xml +++ b/src/main/resources/web.xml @@ -41,11 +41,6 @@ <url-pattern>/uptime</url-pattern> </servlet-mapping> - <context-param> - <param-name>outDir</param-name> - <param-value>/srv/onionoo.torproject.org/onionoo/out/</param-value> - </context-param> - <listener> <listener-class> org.torproject.onionoo.server.NodeIndexer [View Less]

1 0

[stem/master] Recaching manual if schema is out of date
by atagar＠torproject.org 30 Aug '17

30 Aug '17

commit 8ecb75a957fbb721e0053597bb0b6ab40a990f63 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Aug 29 10:51:42 2017 -0700 Recaching manual if schema is out of date --- cache_manual.py | 3 +++ stem/cached_tor_manual.sqlite | Bin 217088 -> 217088 bytes stem/manual.py | 20 +++++++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cache_manual.py b/cache_manual.py index 3112879a..1022648b 100755 --- a/… [View More]cache_manual.py +++ b/cache_manual.py @@ -41,6 +41,9 @@ if __name__ == '__main__': try: cached_manual = stem.manual.Manual.from_cache() + except stem.manual.SchemeMismatch as exc: + print('Cached database schema is out of date (was %s, but current version is %s)' % (exc.database_schema, exc.library_schema)) + cached_manual = None except IOError: cached_manual = None # local copy has been deleted diff --git a/stem/cached_tor_manual.sqlite b/stem/cached_tor_manual.sqlite index 81d0b230..bc50d083 100644 Binary files a/stem/cached_tor_manual.sqlite and b/stem/cached_tor_manual.sqlite differ diff --git a/stem/manual.py b/stem/manual.py index e520f2f8..21eb68ee 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -97,6 +97,22 @@ CATEGORY_SECTIONS = OrderedDict(( )) +class SchemeMismatch(IOError): + """ + Database schema doesn't match what Stem supports. + + .. versionadded:: 1.6.0 + + :var int database_schema: schema of the database + :var int library_schema: schema of the library + """ + + def __init__(self, message, database_schema, library_schema): + super(SchemeMismatch, self).__init__(message) + self.database_schema = database_schema + self.library_schema = library_schema + + def query(query, *param): """ Performs the given query on our sqlite manual cache. This database should @@ -339,6 +355,7 @@ class Manual(object): self.config_options = config_options self.man_commit = None self.stem_commit = None + self.schema = None @staticmethod def from_cache(path = None): @@ -384,7 +401,7 @@ class Manual(object): raise IOError('Failed to read database metadata from %s: %s' % (path, exc)) if schema != SCHEMA_VERSION: - raise IOError("Stem's current manual schema version is %s, but %s was version %s" % (SCHEMA_VERSION, path, schema)) + raise SchemeMismatch("Stem's current manual schema version is %s, but %s was version %s" % (SCHEMA_VERSION, path, schema), schema, SCHEMA_VERSION) commandline = dict(conn.execute('SELECT name, description FROM commandline').fetchall()) signals = dict(conn.execute('SELECT name, description FROM signals').fetchall()) @@ -399,6 +416,7 @@ class Manual(object): manual = Manual(name, synopsis, description, commandline, signals, files, config_options) manual.man_commit = man_commit manual.stem_commit = stem_commit + manual.schema = schema return manual [View Less]

1 0

[stem/master] Dictionaries could cause equality checks to fail
by atagar＠torproject.org 30 Aug '17

30 Aug '17

commit d4ae5b775941b54b9303a88a33e21831f0e6fffa Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Aug 29 10:57:15 2017 -0700 Dictionaries could cause equality checks to fail Dictionaries, of course, aren't assured to have the same key ordering unless we're using an OrderedDict. Previously we constructed dictionaries in the same order because configs are ordered, but this is no longer the case with sqlite. Fixing our hashing function to not care … [View More]

1 0

[stem/master] Make schema a constant, near the version
by atagar＠torproject.org 30 Aug '17

30 Aug '17

commit 19b8393652a3cba6741d1051d49af02b735c883e Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Aug 29 13:19:10 2017 -0700 Make schema a constant, near the version Placing the schema version with the schema itself will help us remember to bump it when the later changes. --- stem/cached_tor_manual.sqlite | Bin 217088 -> 218112 bytes stem/manual.py | 41 ++++++++++++++++++++++++++++------------- 2 files changed, 28 insertions(+), 13 … [View More]deletions(-) diff --git a/stem/cached_tor_manual.sqlite b/stem/cached_tor_manual.sqlite index bc50d083..898b09bc 100644 Binary files a/stem/cached_tor_manual.sqlite and b/stem/cached_tor_manual.sqlite differ diff --git a/stem/manual.py b/stem/manual.py index 21eb68ee..366b0ea9 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -84,7 +84,18 @@ Category = stem.util.enum.Enum('GENERAL', 'CLIENT', 'RELAY', 'DIRECTORY', 'AUTHO GITWEB_MANUAL_URL = 'https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt' CACHE_PATH = os.path.join(os.path.dirname(__file__), 'cached_tor_manual.sqlite') DATABASE = None # cache database connections -SCHEMA_VERSION = 1 # version of the schema used by our sqlite cache + +SCHEMA_VERSION = 1 # version of our scheme, bump this if you change the following +SCHEMA = ( + 'CREATE TABLE schema(version NUMBER)', + 'INSERT INTO schema(version) VALUES (%i)' % SCHEMA_VERSION, + + 'CREATE TABLE metadata(name TEXT, synopsis TEXT, description TEXT, man_commit TEXT, stem_commit TEXT)', + 'CREATE TABLE commandline(name TEXT PRIMARY KEY, description TEXT)', + 'CREATE TABLE signals(name TEXT PRIMARY KEY, description TEXT)', + 'CREATE TABLE files(name TEXT PRIMARY KEY, description TEXT)', + 'CREATE TABLE torrc(key TEXT PRIMARY KEY, name TEXT, category TEXT, usage TEXT, summary TEXT, description TEXT, position NUMBER)', +) CATEGORY_SECTIONS = OrderedDict(( ('GENERAL OPTIONS', Category.GENERAL), @@ -396,13 +407,15 @@ class Manual(object): with sqlite3.connect(path) as conn: try: - name, synopsis, description, man_commit, stem_commit, schema = conn.execute('SELECT name, synopsis, description, man_commit, stem_commit, schema FROM metadata').fetchone() + schema = conn.execute('SELECT version FROM schema').fetchone()[0] + + if schema != SCHEMA_VERSION: + raise SchemeMismatch("Stem's current manual schema version is %s, but %s was version %s" % (SCHEMA_VERSION, path, schema), schema, SCHEMA_VERSION) + + name, synopsis, description, man_commit, stem_commit = conn.execute('SELECT name, synopsis, description, man_commit, stem_commit FROM metadata').fetchone() except sqlite3.OperationalError as exc: raise IOError('Failed to read database metadata from %s: %s' % (path, exc)) - if schema != SCHEMA_VERSION: - raise SchemeMismatch("Stem's current manual schema version is %s, but %s was version %s" % (SCHEMA_VERSION, path, schema), schema, SCHEMA_VERSION) - commandline = dict(conn.execute('SELECT name, description FROM commandline').fetchall()) signals = dict(conn.execute('SELECT name, description FROM signals').fetchall()) files = dict(conn.execute('SELECT name, description FROM files').fetchall()) @@ -553,14 +566,16 @@ class Manual(object): return self._save_as_config(path) def _save_as_sqlite(self, path): - with sqlite3.connect(path + '.new') as conn: - conn.execute('CREATE TABLE metadata(name TEXT, synopsis TEXT, description TEXT, man_commit TEXT, stem_commit TEXT, schema NUMBER)') - conn.execute('CREATE TABLE commandline(name TEXT PRIMARY KEY, description TEXT)') - conn.execute('CREATE TABLE signals(name TEXT PRIMARY KEY, description TEXT)') - conn.execute('CREATE TABLE files(name TEXT PRIMARY KEY, description TEXT)') - conn.execute('CREATE TABLE torrc(key TEXT PRIMARY KEY, name TEXT, category TEXT, usage TEXT, summary TEXT, description TEXT, position NUMBER)') + tmp_path = path + '.new' + + if os.path.exists(tmp_path): + os.remove(tmp_path) + + with sqlite3.connect(tmp_path) as conn: + for cmd in SCHEMA: + conn.execute(cmd) - conn.execute('INSERT INTO metadata(name, synopsis, description, man_commit, stem_commit, schema) VALUES (?,?,?,?,?,?)', (self.name, self.synopsis, self.description, self.man_commit, self.stem_commit, SCHEMA_VERSION)) + conn.execute('INSERT INTO metadata(name, synopsis, description, man_commit, stem_commit) VALUES (?,?,?,?,?)', (self.name, self.synopsis, self.description, self.man_commit, self.stem_commit)) for k, v in self.commandline_options.items(): conn.execute('INSERT INTO commandline(name, description) VALUES (?,?)', (k, v)) @@ -577,7 +592,7 @@ class Manual(object): if os.path.exists(path): os.remove(path) - os.rename(path + '.new', path) + os.rename(tmp_path, path) def _save_as_config(self, path): conf = stem.util.conf.Config() [View Less]

1 0

[stem/master] Let library read multiple schemas
by atagar＠torproject.org 30 Aug '17

30 Aug '17

commit a937507c68e82e655243b898256789072480b6f1 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Aug 29 13:26:24 2017 -0700 Let library read multiple schemas We'll probably want to provide backward compatability with prior minor version releases, but not development commits. Changing the exception so we can provide a tuple of what we support rather than just a single schema version. --- cache_manual.py | 12 ++++++++---- stem/manual.py | 8 ++++---- 2 … [View More]files changed, 12 insertions(+), 8 deletions(-) diff --git a/cache_manual.py b/cache_manual.py index 1022648b..0d07fa5f 100755 --- a/cache_manual.py +++ b/cache_manual.py @@ -41,11 +41,15 @@ if __name__ == '__main__': try: cached_manual = stem.manual.Manual.from_cache() - except stem.manual.SchemeMismatch as exc: - print('Cached database schema is out of date (was %s, but current version is %s)' % (exc.database_schema, exc.library_schema)) - cached_manual = None + db_schema = cached_manual.schema + except stem.manual.SchemaMismatch as exc: + cached_manual, db_schema = None, exc.database_schema except IOError: - cached_manual = None # local copy has been deleted + cached_manual, db_schema = None, None # local copy has been deleted + + if db_schema != stem.manual.SCHEMA_VERSION: + print('Cached database schema is out of date (was %s, but current version is %s)' % (exc.database_schema, stem.manual.SCHEMA_VERSION)) + cached_manual = None latest_manual = stem.manual.Manual.from_remote() diff --git a/stem/manual.py b/stem/manual.py index 366b0ea9..290688e6 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -108,18 +108,18 @@ CATEGORY_SECTIONS = OrderedDict(( )) -class SchemeMismatch(IOError): +class SchemaMismatch(IOError): """ Database schema doesn't match what Stem supports. .. versionadded:: 1.6.0 :var int database_schema: schema of the database - :var int library_schema: schema of the library + :var tuple supported_schemas: schemas library supports """ def __init__(self, message, database_schema, library_schema): - super(SchemeMismatch, self).__init__(message) + super(SchemaMismatch, self).__init__(message) self.database_schema = database_schema self.library_schema = library_schema @@ -410,7 +410,7 @@ class Manual(object): schema = conn.execute('SELECT version FROM schema').fetchone()[0] if schema != SCHEMA_VERSION: - raise SchemeMismatch("Stem's current manual schema version is %s, but %s was version %s" % (SCHEMA_VERSION, path, schema), schema, SCHEMA_VERSION) + raise SchemaMismatch("Stem's current manual schema version is %s, but %s was version %s" % (SCHEMA_VERSION, path, schema), schema, (SCHEMA_VERSION,)) name, synopsis, description, man_commit, stem_commit = conn.execute('SELECT name, synopsis, description, man_commit, stem_commit FROM metadata').fetchone() except sqlite3.OperationalError as exc: [View Less]

1 0

[tor-launcher/master] Bug 22232: fix typo in README-BOOTSTRAP.
by brade＠torproject.org 30 Aug '17

30 Aug '17

commit a120bb02f4dbcb3e26fe0c031d383a9a8681678a Author: Kathy Brade <brade(a)pearlcrescent.com> Date: Wed Aug 30 10:43:34 2017 -0400 Bug 22232: fix typo in README-BOOTSTRAP. --- README-BOOTSTRAP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-BOOTSTRAP b/README-BOOTSTRAP index 8bc2a76..a5354e7 100644 --- a/README-BOOTSTRAP +++ b/README-BOOTSTRAP @@ -35,7 +35,7 @@ mBootstrapErrorOccurred (Boolean) This is set to true when a mIsBootstrapDone is set to … [View More]

1 0

[tor-browser-build/master] README: mention that rbm is included as a git submodule
by boklm＠torproject.org 30 Aug '17

30 Aug '17

commit da7485a0b8f89b9ee2278d6132fc7bb53d5220e8 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Wed Aug 30 14:45:07 2017 +0200 README: mention that rbm is included as a git submodule --- README | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README b/README index 4381074..10cadec 100644 --- a/README +++ b/README @@ -46,6 +46,10 @@ If you are running Debian or Ubuntu, you can install them with: libdigest-sha-perl libdata-uuid-perl libdata-dump-perl \… [View More]

1 0

[onionoo/master] Switch to metrics-lib's DescriptorCollector.
by karsten＠torproject.org 30 Aug '17

30 Aug '17

commit 32992eea202770065b4cb61d4dc39bbee1b87628 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed May 17 15:58:22 2017 +0200 Switch to metrics-lib's DescriptorCollector. Implements #22287. --- CHANGELOG.md | 2 + .../onionoo/updater/DescriptorDownloader.java | 184 --------------------- .../onionoo/updater/DescriptorSource.java | 41 ++--- 3 files changed, 13 insertions(+), 214 deletions(-) diff --git a/… [View More]CHANGELOG.md b/CHANGELOG.md index bb38dd0..e028dd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ - Remove optional fields "countries", "transports", and "versions" from clients objects which were still labeled as beta. - Add new "version" parameter to filter for Tor version. + - Switch from our own CollecTor downloader to metrics-lib's + DescriptorCollector. # Changes in version 4.0-1.3.0 - 2017-08-04 diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorDownloader.java b/src/main/java/org/torproject/onionoo/updater/DescriptorDownloader.java deleted file mode 100644 index 1e41f25..0000000 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorDownloader.java +++ /dev/null @@ -1,184 +0,0 @@ -/* Copyright 2016--2017 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.onionoo.updater; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.zip.GZIPInputStream; - -class DescriptorDownloader { - - private static Logger log = LoggerFactory.getLogger( - DescriptorDownloader.class); - - private final String protocolHostNameResourcePrefix = - "https://collector.torproject.org/recent/"; - - private String directory; - - private final File inDir = new File("in/recent"); - - public DescriptorDownloader(DescriptorType descriptorType) { - switch (descriptorType) { - case RELAY_CONSENSUSES: - this.directory = "relay-descriptors/consensuses/"; - break; - case RELAY_SERVER_DESCRIPTORS: - this.directory = "relay-descriptors/server-descriptors/"; - break; - case RELAY_EXTRA_INFOS: - this.directory = "relay-descriptors/extra-infos/"; - break; - case EXIT_LISTS: - this.directory = "exit-lists/"; - break; - case BRIDGE_STATUSES: - this.directory = "bridge-descriptors/statuses/"; - break; - case BRIDGE_SERVER_DESCRIPTORS: - this.directory = "bridge-descriptors/server-descriptors/"; - break; - case BRIDGE_EXTRA_INFOS: - this.directory = "bridge-descriptors/extra-infos/"; - break; - default: - log.error("Unknown descriptor type."); - return; - } - } - - private SortedSet<String> localFiles = new TreeSet<>(); - - public int statLocalFiles() { - File localDirectory = new File(this.inDir, this.directory); - if (localDirectory.exists()) { - for (File file : localDirectory.listFiles()) { - this.localFiles.add(file.getName()); - } - } - return this.localFiles.size(); - } - - private SortedSet<String> remoteFiles = new TreeSet<>(); - - public int fetchRemoteDirectory() { - String directoryUrl = this.protocolHostNameResourcePrefix - + this.directory; - try { - URL url = new URL(directoryUrl); - HttpURLConnection huc = (HttpURLConnection) url.openConnection(); - huc.setRequestMethod("GET"); - huc.connect(); - if (huc.getResponseCode() != 200) { - log.error("Could not fetch " + directoryUrl - + ": " + huc.getResponseCode() + " " - + huc.getResponseMessage() + ". Skipping."); - return 0; - } - try (BufferedReader br = new BufferedReader(new InputStreamReader( - huc.getInputStream()))) { - String line; - while ((line = br.readLine()) != null) { - if (!line.trim().startsWith("<tr>") - || !line.contains("<a href=\"")) { - continue; - } - String linePart = line.substring( - line.indexOf("<a href=\"") + "<a href=\"".length()); - if (!linePart.contains("\"")) { - continue; - } - linePart = linePart.substring(0, linePart.indexOf("\"")); - if (linePart.endsWith("/")) { - continue; - } - this.remoteFiles.add(linePart); - } - } - } catch (IOException e) { - log.error("Could not fetch or parse " + directoryUrl - + ". Skipping. Reason: " + e.getMessage()); - } - return this.remoteFiles.size(); - } - - public int fetchRemoteFiles() { - int fetchedFiles = 0; - for (String remoteFile : this.remoteFiles) { - if (this.localFiles.contains(remoteFile)) { - continue; - } - String fileUrl = this.protocolHostNameResourcePrefix - + this.directory + remoteFile; - File localTempFile = new File(this.inDir, this.directory - + remoteFile + ".tmp"); - File localFile = new File(this.inDir, this.directory + remoteFile); - try { - localFile.getParentFile().mkdirs(); - URL url = new URL(fileUrl); - HttpURLConnection huc = (HttpURLConnection) url.openConnection(); - huc.setRequestMethod("GET"); - huc.addRequestProperty("Accept-Encoding", "gzip"); - huc.connect(); - if (huc.getResponseCode() != 200) { - log.error("Could not fetch \n\t" + fileUrl - + ": " + huc.getResponseCode() + " " - + huc.getResponseMessage() + ". Skipping."); - continue; - } - InputStream is; - if (huc.getContentEncoding() != null - && huc.getContentEncoding().equalsIgnoreCase("gzip")) { - is = new GZIPInputStream(huc.getInputStream()); - } else { - is = huc.getInputStream(); - } - try (BufferedInputStream bis = new BufferedInputStream(is); - BufferedOutputStream bos = new BufferedOutputStream( - new FileOutputStream(localTempFile))) { - int len; - byte[] data = new byte[1024]; - while ((len = bis.read(data, 0, 1024)) >= 0) { - bos.write(data, 0, len); - } - } - localTempFile.renameTo(localFile); - long lastModified = huc.getHeaderFieldDate("Last-Modified", -1L); - if (lastModified >= 0) { - localFile.setLastModified(lastModified); - } - fetchedFiles++; - } catch (IOException e) { - log.error("Could not fetch or store \n\t" + fileUrl - + ". Skipping.\n\tReason: " + e.getMessage()); - } - } - return fetchedFiles; - } - - public int deleteOldLocalFiles() { - int deletedFiles = 0; - for (String localFile : this.localFiles) { - if (!this.remoteFiles.contains(localFile)) { - new File(this.inDir, this.directory + localFile).delete(); - deletedFiles++; - } - } - return deletedFiles; - } -} - diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java index 45b40ee..d32727f 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java @@ -4,6 +4,7 @@ package org.torproject.onionoo.updater; import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorCollector; import org.torproject.onionoo.util.FormattingUtils; import org.slf4j.Logger; @@ -22,9 +23,11 @@ public class DescriptorSource { private static final Logger log = LoggerFactory.getLogger( DescriptorSource.class); - private final File inRecentDir = new File("in/recent"); + private final File inDir = new File("in"); - private final File inArchiveDir = new File("in/archive"); + private final File inRecentDir = new File(inDir, "recent"); + + private final File inArchiveDir = new File(inDir, "archive"); private final File statusDir = new File("status"); @@ -65,28 +68,14 @@ public class DescriptorSource { /** Downloads descriptors from CollecTor. */ public void downloadDescriptors() { + List<String> remoteDirectories = new ArrayList<>(); for (DescriptorType descriptorType : DescriptorType.values()) { - log.info("Loading: " + descriptorType); - this.downloadDescriptors(descriptorType); + remoteDirectories.add("/recent/" + descriptorType.getDir()); } - } - - private int localFilesBefore = 0; - - private int foundRemoteFiles = 0; - - private int downloadedFiles = 0; - - private int deletedLocalFiles = 0; - - private void downloadDescriptors(DescriptorType descriptorType) { - DescriptorDownloader descriptorDownloader = - new DescriptorDownloader(descriptorType); - this.localFilesBefore += descriptorDownloader.statLocalFiles(); - this.foundRemoteFiles += - descriptorDownloader.fetchRemoteDirectory(); - this.downloadedFiles += descriptorDownloader.fetchRemoteFiles(); - this.deletedLocalFiles += descriptorDownloader.deleteOldLocalFiles(); + DescriptorCollector dc = org.torproject.descriptor.DescriptorSourceFactory + .createDescriptorCollector(); + dc.collectDescriptors("https://collector.torproject.org", + remoteDirectories.toArray(new String[0]), 0L, inDir, true); } /** Reads archived and recent descriptors from disk and feeds them into @@ -206,14 +195,6 @@ public class DescriptorSource { * descriptors during the current execution. */ public String getStatsString() { StringBuilder sb = new StringBuilder(); - sb.append(" ").append(this.localFilesBefore) - .append(" recent descriptor files ").append("found locally\n"); - sb.append(" ").append(this.foundRemoteFiles) - .append(" recent descriptor files ").append("found remotely\n"); - sb.append(" ").append(this.downloadedFiles) - .append(" recent descriptor files ").append("downloaded from remote\n"); - sb.append(" ").append(this.deletedLocalFiles) - .append(" recent descriptor ").append("files deleted locally\n"); sb.append(" ").append(this.descriptorQueues.size()) .append(" descriptor ").append("queues created for recent descriptors\n"); int historySizeBefore = 0; [View Less]

1 0