January 2017 - tor-commits - lists.torproject.org

[tor/master] Merge branch 'maint-0.2.9'
by nickm＠torproject.org 23 Jan '17

23 Jan '17

commit 7dd5fcb707652fd761bc4b0d40cbb0c1559a84b0 Merge: 0668d29 0d4d9b6 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 23 14:34:33 2017 -0500 Merge branch 'maint-0.2.9'

1 0

[tor/master] Bump to 0.3.0.2-alpha-dev
by nickm＠torproject.org 23 Jan '17

23 Jan '17

commit c4cc11a9df48defe73eccfa8296e7553bfd4da50 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 23 14:38:10 2017 -0500 Bump to 0.3.0.2-alpha-dev --- configure.ac | 2 +- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index c5f39ee..7399b5b 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2015, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.3.0.2-alpha]) +AC_INIT([tor],[0.3.0.2-alpha-dev]) AC_CONFIG_SRCDIR([src/or/main.c]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 776e51f..c98aa78 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.3.0.2-alpha" +!define VERSION "0.3.0.2-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index c42d54d..e111d96 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.3.0.2-alpha" +#define VERSION "0.3.0.2-alpha-dev"

1 0

[tor/release-0.2.9] Bump version to 0.2.9.9-dev
by nickm＠torproject.org 23 Jan '17

23 Jan '17

commit 0d4d9b6d889de6dc85ca8870ada123aaf41b0053 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 23 14:34:08 2017 -0500 Bump version to 0.2.9.9-dev --- configure.ac | 2 +- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 1506373..2134a41 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2015, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.2.9.9]) +AC_INIT([tor],[0.2.9.9-dev]) AC_CONFIG_SRCDIR([src/or/main.c]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 9b86f72..91eb5f2 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.2.9.9" +!define VERSION "0.2.9.9-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 17e1e54..80d5cf0 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.9.9" +#define VERSION "0.2.9.9-dev"

1 0

[translation/tor-browser-manual] Update translations for tor-browser-manual
by translation＠torproject.org 23 Jan '17

23 Jan '17

commit b2157101c28f6c545499e4ca228b92b0ca5a7ad8 Author: Translation commit bot <translation(a)torproject.org> Date: Mon Jan 23 19:18:22 2017 +0000 Update translations for tor-browser-manual --- ro/ro.po | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/ro/ro.po b/ro/ro.po index f3ad6e0..3808a72 100644 --- a/ro/ro.po +++ b/ro/ro.po @@ -893,7 +893,7 @@ msgstr "" #: plugins.page:58 msgid "Browser Add-ons" -msgstr "" +msgstr "Suplimente navigator" #: plugins.page:59 msgid "" @@ -1026,6 +1026,8 @@ msgid "" "external ref='media/security-slider/slider.png' " "md5='3c469cd3ed9f60ebb6bbbc63daa90082'" msgstr "" +"external ref='media/security-slider/slider.png' " +"md5='3c469cd3ed9f60ebb6bbbc63daa90082'" #: security-slider.page:25 msgid "" @@ -1047,6 +1049,8 @@ msgid "" "external ref='media/security-slider/slider_window.png' " "md5='c733bdccd1731ed1a772777b25bae7a1'" msgstr "" +"external ref='media/security-slider/slider_window.png' " +"md5='c733bdccd1731ed1a772777b25bae7a1'" #: security-slider.page:36 msgid "" @@ -1096,7 +1100,7 @@ msgstr "" #: security-slider.page:73 msgid "Low" -msgstr "" +msgstr "Scăzut" #: security-slider.page:74 msgid "" @@ -1106,7 +1110,7 @@ msgstr "" #: transports.page:6 transports.page:20 msgid "Types of pluggable transport" -msgstr "" +msgstr "Tipuri de transporturi conectabile" #: transports.page:10 msgid "Pluggable Transports" @@ -1128,7 +1132,7 @@ msgstr "" #: transports.page:28 msgid "obfs3" -msgstr "" +msgstr "obfs3" #: transports.page:33 msgid "" @@ -1138,7 +1142,7 @@ msgstr "" #: transports.page:42 msgid "obfs4" -msgstr "" +msgstr "obfs4" #: transports.page:47 msgid "" @@ -1149,7 +1153,7 @@ msgstr "" #: transports.page:56 msgid "Scramblesuit" -msgstr "" +msgstr "Scramblesuit" #: transports.page:61 msgid "ScrambleSuit is similar to obfs4 but has a different set of bridges." @@ -1157,7 +1161,7 @@ msgstr "" #: transports.page:69 msgid "FTE" -msgstr "" +msgstr "FTE" #: transports.page:74 msgid "" @@ -1167,7 +1171,7 @@ msgstr "" #: transports.page:82 msgid "meek" -msgstr "" +msgstr "meek" #: transports.page:87 msgid "" @@ -1179,7 +1183,7 @@ msgstr "" #: troubleshooting.page:6 msgid "What to do if Tor Browser doesn’t work" -msgstr "" +msgstr "Ce fac dacă navigatorul Tor nu funcționează" #: troubleshooting.page:12 msgid "" @@ -1190,7 +1194,7 @@ msgstr "" #: troubleshooting.page:21 msgid "Quick fixes" -msgstr "" +msgstr "Rezolvări rapide" #: troubleshooting.page:22 msgid "" @@ -1229,7 +1233,7 @@ msgstr "" #: troubleshooting.page:64 msgid "Is your connection censored?" -msgstr "" +msgstr "Este conexiunea ta cenzurată?" #: troubleshooting.page:65 msgid "" @@ -1292,7 +1296,7 @@ msgstr "" #: updating.page:6 msgid "How to update Tor Browser" -msgstr "" +msgstr "Cum să actualizezi navigatorul Tor" #: updating.page:10 msgid "Updating"

1 0

[metrics-web/master] Add webstats module and webstats-tb graph.
by karsten＠torproject.org 23 Jan '17

23 Jan '17

commit 1c0ec1e13a507baa9621156645d2dc28d85c8748 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jan 23 20:09:38 2017 +0100 Add webstats module and webstats-tb graph. Implements #21236. --- modules/webstats/.gitignore | 3 + modules/webstats/build.xml | 27 ++ .../java/org/torproject/metrics/webstats/Main.java | 406 +++++++++++++++++++++ .../webstats/src/main/resources/init-webstats.sql | 164 +++++++++ modules/webstats/src/main/resources/write-RData.R | 9 + .../org/torproject/metrics/webstats/MainTest.java | 93 +++++ shared/bin/90-run-webstats-stats.sh | 10 + shared/bin/99-copy-stats-files.sh | 2 + shared/build-base.xml | 42 +++ shared/build.xml | 2 +- website/etc/categories.json | 5 +- website/etc/metrics.json | 14 + website/etc/web.xml | 4 + website/rserve/graphs.R | 31 ++ .../metrics/web/research/ResearchStatsServlet.java | 1 + website/web/WEB-INF/sources.jsp | 6 + website/web/WEB-INF/stats.jsp | 34 +- 17 files changed, 849 insertions(+), 4 deletions(-) diff --git a/modules/webstats/.gitignore b/modules/webstats/.gitignore new file mode 100644 index 0000000..a8e4d02 --- /dev/null +++ b/modules/webstats/.gitignore @@ -0,0 +1,3 @@ +/stats/*.csv +/RData/*.RData + diff --git a/modules/webstats/build.xml b/modules/webstats/build.xml new file mode 100644 index 0000000..bcfe251 --- /dev/null +++ b/modules/webstats/build.xml @@ -0,0 +1,27 @@ +<project default="run" name="webstats" basedir="."> + + <property name="sources" value="src/main/java"/> + <property name="testsources" value="src/test/java"/> + + <include file="../../shared/build-base.xml" as="basetask"/> + <target name="clean" depends="basetask.clean"/> + <target name="compile" depends="basetask.compile"/> + <target name="test" depends="basetask.test"/> + + <path id="classpath"> + <pathelement path="${classes}"/> + <path refid="base.classpath" /> + <fileset dir="${libs}"> + <include name="postgresql-jdbc3-9.2.jar"/> + </fileset> + </path> + + <target name="run" depends="compile"> + <java fork="true" + maxmemory="1g" + classname="org.torproject.metrics.webstats.Main"> + <classpath refid="classpath"/> + </java> + </target> +</project> + diff --git a/modules/webstats/src/main/java/org/torproject/metrics/webstats/Main.java b/modules/webstats/src/main/java/org/torproject/metrics/webstats/Main.java new file mode 100644 index 0000000..b6e2f96 --- /dev/null +++ b/modules/webstats/src/main/java/org/torproject/metrics/webstats/Main.java @@ -0,0 +1,406 @@ +/* Copyright 2016--2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.webstats; + +import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Date; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** Main class of the webstats module that downloads log files from the server, + * imports them into a database, and exports aggregate statistics to a CSV + * file. */ +public class Main { + + /** Logger for this class. */ + private static Logger log = LoggerFactory.getLogger(Main.class); + + /** Pattern for links contained in directory listings. */ + static final Pattern URL_STRING_PATTERN = + Pattern.compile(".*<a href=\"([^\"]+)\">.*"); + + static final Pattern LOG_FILE_URL_PATTERN = + Pattern.compile("^.*/([^/]+)/([^/]+)-access.log-(\\d{8}).xz$"); + + private static DateFormat logDateFormat; + + static { + logDateFormat = new SimpleDateFormat("yyyyMMdd"); + logDateFormat.setLenient(false); + logDateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + static final Pattern LOG_LINE_PATTERN = Pattern.compile( + "^0.0.0.[01] - - \\[\\d{2}/\\w{3}/\\d{4}:00:00:00 \\+0000\\] " + + "\"(GET|HEAD) ([^ ]+) HTTP[^ ]+\" (\\d+) (-|\\d+) \"-\" \"-\" -$"); + + private static final String LOG_DATE = "log_date"; + + private static final String REQUEST_TYPE = "request_type"; + + private static final String PLATFORM = "platform"; + + private static final String CHANNEL = "channel"; + + private static final String LOCALE = "locale"; + + private static final String INCREMENTAL = "incremental"; + + private static final String COUNT = "count"; + + private static final String ALL_COLUMNS = LOG_DATE + "," + REQUEST_TYPE + "," + + PLATFORM + "," + CHANNEL + "," + LOCALE + "," + INCREMENTAL + "," + + COUNT; + + /** Executes this data-processing module. */ + public static void main(String[] args) throws Exception { + log.info("Starting webstats module."); + String dbUrlString = "jdbc:postgresql:webstats"; + Connection connection = connectToDatabase(dbUrlString); + SortedSet<String> previouslyImportedLogFileUrls = + queryImportedFiles(connection); + String baseUrl = "https://webstats.torproject.org/out/"; + SortedSet<String> newLogFileUrls = downloadDirectoryListings(baseUrl, + previouslyImportedLogFileUrls); + importLogFiles(connection, newLogFileUrls); + SortedSet<String> statistics = queryWebstats(connection); + writeStatistics(Paths.get("stats", "webstats.csv"), statistics); + disconnectFromDatabase(connection); + log.info("Terminated webstats module."); + } + + private static Connection connectToDatabase(String jdbcString) + throws SQLException { + log.info("Connecting to database."); + Connection connection = DriverManager.getConnection(jdbcString); + connection.setAutoCommit(false); + log.info("Successfully connected to database."); + return connection; + } + + static SortedSet<String> queryImportedFiles(Connection connection) + throws SQLException { + log.info("Querying URLs of previously imported log files."); + SortedSet<String> importedLogFileUrls = new TreeSet<>(); + Statement st = connection.createStatement(); + String queryString = "SELECT url FROM files"; + try (ResultSet rs = st.executeQuery(queryString)) { + while (rs.next()) { + importedLogFileUrls.add(rs.getString(1)); + } + } + log.info("Found {} URLs of previously imported log files.", + importedLogFileUrls.size()); + return importedLogFileUrls; + } + + static SortedSet<String> downloadDirectoryListings(String baseUrl, + SortedSet<String> importedLogFileUrls) throws IOException { + log.info("Downloading directory listings from {}.", baseUrl); + List<String> directoryListings = new ArrayList<>(); + directoryListings.add(baseUrl); + SortedSet<String> newLogFileUrls = new TreeSet<>(); + while (!directoryListings.isEmpty()) { + String urlString = directoryListings.remove(0); + if (urlString.endsWith("/")) { + directoryListings.addAll(downloadDirectoryListing(urlString)); + } else if (!urlString.endsWith(".xz")) { + log.debug("Skipping unrecognized URL {}.", urlString); + } else if (!importedLogFileUrls.contains(urlString)) { + newLogFileUrls.add(urlString); + } + } + log.info("Found {} URLs of log files that have not yet been imported.", + newLogFileUrls.size()); + return newLogFileUrls; + } + + static List<String> downloadDirectoryListing(String urlString) + throws IOException { + log.debug("Downloading directory listing from {}.", urlString); + List<String> urlStrings = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new InputStreamReader( + new URL(urlString).openStream()))) { + String line; + while ((line = br.readLine()) != null) { + Matcher matcher = URL_STRING_PATTERN.matcher(line); + if (matcher.matches() && !matcher.group(1).startsWith("/")) { + urlStrings.add(urlString + matcher.group(1)); + } + } + } + return urlStrings; + } + + static void importLogFiles(Connection connection, + SortedSet<String> newLogFileUrls) { + log.info("Downloading, parsing, and importing {} log files.", + newLogFileUrls.size()); + for (String urlString : newLogFileUrls) { + try { + Object[] metaData = parseMetaData(urlString); + if (metaData == null) { + continue; + } + List<String> downloadedLogLines = downloadLogFile(urlString); + Map<String, Integer> parsedLogLines = parseLogLines(urlString, + downloadedLogLines); + importLogLines(connection, urlString, metaData, parsedLogLines); + } catch (IOException | ParseException exc) { + log.warn("Cannot download or parse log file with URL {}. Retrying " + + "in the next run.", urlString, exc); + } catch (SQLException exc) { + log.warn("Cannot import log file with URL {} into the database. " + + "Rolling back and retrying in the next run.", urlString, exc); + try { + connection.rollback(); + } catch (SQLException exceptionWhileRollingBack) { + /* Ignore. */ + } + } + } + } + + private static Object[] parseMetaData(String urlString) + throws ParseException { + log.debug("Importing log file {}.", urlString); + if (urlString.contains("-ssl-access.log-")) { + log.debug("Skipping log file containing SSL requests with URL {}.", + urlString); + return null; + } + Matcher logFileUrlMatcher = LOG_FILE_URL_PATTERN.matcher(urlString); + if (!logFileUrlMatcher.matches()) { + log.debug("Skipping log file with unrecognized URL {}.", urlString); + return null; + } + String server = logFileUrlMatcher.group(1); + String site = logFileUrlMatcher.group(2); + long logDateMillis = logDateFormat.parse(logFileUrlMatcher.group(3)) + .getTime(); + return new Object[] { server, site, new Long(logDateMillis) }; + } + + static List<String> downloadLogFile(String urlString) throws IOException { + List<String> downloadedLogLines = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new InputStreamReader( + new XZCompressorInputStream(new URL(urlString).openStream())))) { + String line; + while ((line = br.readLine()) != null) { + downloadedLogLines.add(line); + } + } + return downloadedLogLines; + } + + static Map<String, Integer> parseLogLines(String urlString, + List<String> logLines) { + int skippedLines = 0; + Map<String, Integer> parsedLogLines = new HashMap<>(); + for (String logLine : logLines) { + Matcher logLineMatcher = LOG_LINE_PATTERN.matcher(logLine); + if (!logLineMatcher.matches()) { + skippedLines++; + continue; + } + String method = logLineMatcher.group(1); + String resource = logLineMatcher.group(2); + int responseCode = Integer.parseInt(logLineMatcher.group(3)); + String combined = String.format("%s %s %d", method, resource, + responseCode); + if (!parsedLogLines.containsKey(combined)) { + parsedLogLines.put(combined, 1); + } else { + parsedLogLines.put(combined, parsedLogLines.get(combined) + 1); + } + } + if (skippedLines > 0) { + log.debug("Skipped {} lines while parsing log file {}.", skippedLines, + urlString); + } + return parsedLogLines; + } + + private static void importLogLines(Connection connection, String urlString, + Object[] metaData, Map<String, Integer> parsedLogLines) + throws SQLException { + PreparedStatement psFiles = connection.prepareStatement( + "INSERT INTO files (url, server, site, " + LOG_DATE + ") " + + "VALUES (?, ?, ?, ?)", Statement.RETURN_GENERATED_KEYS); + PreparedStatement psResourcesSelect = connection.prepareStatement( + "SELECT resource_id FROM resources WHERE resource_string = ?"); + PreparedStatement psResourcesInsert = connection.prepareStatement( + "INSERT INTO resources (resource_string) VALUES (?)", + Statement.RETURN_GENERATED_KEYS); + PreparedStatement psRequests = connection.prepareStatement( + "INSERT INTO requests (file_id, method, resource_id, response_code, " + + COUNT + ") VALUES (?, CAST(? AS method), ?, ?, ?)"); + String server = (String) metaData[0]; + String site = (String) metaData[1]; + long logDateMillis = (long) metaData[2]; + int fileId = insertFile(psFiles, urlString, server, site, logDateMillis); + if (fileId < 0) { + log.debug("Skipping previously imported log file {}.", urlString); + return; + } + for (Map.Entry<String, Integer> requests : parsedLogLines.entrySet()) { + String[] keyParts = requests.getKey().split(" "); + String method = keyParts[0]; + String resource = keyParts[1]; + int responseCode = Integer.parseInt(keyParts[2]); + int count = requests.getValue(); + int resourceId = insertResource(psResourcesSelect, psResourcesInsert, + resource); + if (resourceId < 0) { + log.error("Could not retrieve auto-generated key for new resources " + + "entry."); + connection.rollback(); + return; + } + insertRequest(psRequests, fileId, method, resourceId, responseCode, + count); + } + connection.commit(); + log.debug("Finished importing log file with URL {} into database.", + urlString); + } + + private static int insertFile(PreparedStatement psFiles, String urlString, + String server, String site, long logDateMillis) throws SQLException { + int fileId = -1; + psFiles.clearParameters(); + psFiles.setString(1, truncateString(urlString, 2048)); + psFiles.setString(2, truncateString(server, 32)); + psFiles.setString(3, truncateString(site, 128)); + psFiles.setDate(4, new Date(logDateMillis)); + psFiles.execute(); + try (ResultSet rs = psFiles.getGeneratedKeys()) { + if (rs.next()) { + fileId = rs.getInt(1); + } + } + return fileId; + } + + private static void insertRequest(PreparedStatement psRequests, int fileId, + String method, int resourceId, int responseCode, int count) + throws SQLException { + psRequests.clearParameters(); + psRequests.setInt(1, fileId); + psRequests.setString(2, method); + psRequests.setInt(3, resourceId); + psRequests.setInt(4, responseCode); + psRequests.setInt(5, count); + psRequests.execute(); + } + + private static int insertResource(PreparedStatement psResourcesSelect, + PreparedStatement psResourcesInsert, String resource) + throws SQLException { + int resourceId = -1; + String truncatedResource = truncateString(resource, 2048); + psResourcesSelect.clearParameters(); + psResourcesSelect.setString(1, truncatedResource); + try (ResultSet rs = psResourcesSelect.executeQuery()) { + if (rs.next()) { + resourceId = rs.getInt(1); + } + } + if (resourceId < 0) { + /* There's a small potential for a race condition between the previous + * SELECT and this INSERT INTO, but that will be resolved by the UNIQUE + * constraint when committing the transaction. */ + psResourcesInsert.clearParameters(); + psResourcesInsert.setString(1, truncatedResource); + psResourcesInsert.execute(); + try (ResultSet rs = psResourcesInsert.getGeneratedKeys()) { + if (rs.next()) { + resourceId = rs.getInt(1); + } + } + } + return resourceId; + } + + private static String truncateString(String originalString, + int truncateAfter) { + if (originalString.length() > truncateAfter) { + originalString = originalString.substring(0, truncateAfter); + } + return originalString; + } + + static SortedSet<String> queryWebstats(Connection connection) + throws SQLException { + log.info("Querying statistics from database."); + SortedSet<String> statistics = new TreeSet<>(); + Statement st = connection.createStatement(); + String queryString = "SELECT " + ALL_COLUMNS + " FROM webstats"; + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.US); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + try (ResultSet rs = st.executeQuery(queryString)) { + while (rs.next()) { + statistics.add(String.format("%s,%s,%s,%s,%s,%s,%d", + dateFormat.format(rs.getDate(LOG_DATE)), + emptyNull(rs.getString(REQUEST_TYPE)), + emptyNull(rs.getString(PLATFORM)), + emptyNull(rs.getString(CHANNEL)), + emptyNull(rs.getString(LOCALE)), + emptyNull(rs.getString(INCREMENTAL)), + rs.getLong(COUNT))); + } + } + return statistics; + } + + private static String emptyNull(String text) { + return null == text ? "" : text; + } + + static void writeStatistics(Path webstatsPath, + SortedSet<String> statistics) throws IOException { + webstatsPath.toFile().getParentFile().mkdirs(); + List<String> lines = new ArrayList<String>(); + lines.add(ALL_COLUMNS); + lines.addAll(statistics); + log.info("Writing {} lines to {}.", lines.size(), + webstatsPath.toFile().getAbsolutePath()); + Files.write(webstatsPath, lines, StandardCharsets.UTF_8); + } + + private static void disconnectFromDatabase(Connection connection) + throws SQLException { + log.info("Disconnecting from database."); + connection.close(); + } +} + diff --git a/modules/webstats/src/main/resources/init-webstats.sql b/modules/webstats/src/main/resources/init-webstats.sql new file mode 100644 index 0000000..98bb758 --- /dev/null +++ b/modules/webstats/src/main/resources/init-webstats.sql @@ -0,0 +1,164 @@ +-- Copyright 2016--2017 The Tor Project +-- See LICENSE for licensing information + +CREATE TYPE method AS ENUM ('GET', 'HEAD'); + +CREATE TABLE files ( + file_id SERIAL PRIMARY KEY, + url CHARACTER VARYING(2048) UNIQUE NOT NULL, + server CHARACTER VARYING(32) NOT NULL, + site CHARACTER VARYING(128) NOT NULL, + log_date DATE NOT NULL, + UNIQUE (server, site, log_date) +); + +CREATE TABLE resources ( + resource_id SERIAL PRIMARY KEY, + resource_string CHARACTER VARYING(2048) UNIQUE NOT NULL +); + +CREATE TABLE requests ( + file_id INTEGER REFERENCES files (file_id) NOT NULL, + method METHOD NOT NULL, + resource_id INTEGER REFERENCES resources (resource_id) NOT NULL, + response_code SMALLINT NOT NULL, + count INTEGER NOT NULL, + UNIQUE (file_id, method, resource_id, response_code) +); + +CREATE OR REPLACE VIEW webstats AS + SELECT log_date, + CASE WHEN resource_string LIKE '%.asc' THEN 'tbsd' + ELSE 'tbid' END AS request_type, + CASE WHEN resource_string LIKE '%.exe%' THEN 'w' + WHEN resource_string LIKE '%.dmg%' THEN 'm' + WHEN resource_string LIKE '%.tar.xz%' THEN 'l' + ELSE 'o' END AS platform, + CASE WHEN resource_string LIKE '%-hardened%' THEN 'h' + WHEN resource_string LIKE '%/%.%a%/%' THEN 'a' + ELSE 'r' END AS channel, + COALESCE(SUBSTRING(resource_string + FROM '.*_([a-zA-Z]{2}|[a-zA-Z]{2}-[a-zA-Z]{2})[\._-].*'), '??') AS locale, + NULL::BOOLEAN AS incremental, + SUM(count) AS count + FROM files NATURAL JOIN requests NATURAL JOIN resources + WHERE (resource_string LIKE '%/torbrowser/%.exe' + OR resource_string LIKE '%/torbrowser/%.dmg' + OR resource_string LIKE '%/torbrowser/%.tar.xz' + OR resource_string LIKE '%/torbrowser/%.exe.asc' + OR resource_string LIKE '%/torbrowser/%.dmg.asc' + OR resource_string LIKE '%/torbrowser/%.tar.xz.asc') + AND response_code = 200 + AND method = 'GET' + GROUP BY log_date, request_type, platform, channel, locale, incremental + UNION + SELECT log_date, + 'tbup' AS request_type, + CASE WHEN resource_string LIKE '%/WINNT%' THEN 'w' + WHEN resource_string LIKE '%/Darwin%' THEN 'm' + ELSE 'l' END AS platform, + CASE WHEN resource_string LIKE '%/hardened/%' THEN 'h' + WHEN resource_string LIKE '%/alpha/%' THEN 'a' + WHEN resource_string LIKE '%/release/%' THEN 'r' + ELSE 'o' END AS channel, + COALESCE(SUBSTRING(resource_string + FROM '.*/([a-zA-Z]{2}|[a-zA-Z]{2}-[a-zA-Z]{2})\??$'), '??') AS locale, + NULL::BOOLEAN AS incremental, + SUM(count) AS count + FROM files NATURAL JOIN requests NATURAL JOIN resources + WHERE resource_string LIKE '%/torbrowser/update_2/%' + AND resource_string NOT LIKE '%.xml' + AND response_code = 200 + AND method = 'GET' + GROUP BY log_date, request_type, platform, channel, locale, incremental + UNION + SELECT log_date, + 'tbur' AS request_type, + CASE WHEN resource_string LIKE '%-win32-%' THEN 'w' + WHEN resource_string LIKE '%-osx%' THEN 'm' + ELSE 'l' END AS platform, + CASE WHEN resource_string LIKE '%-hardened%' THEN 'h' + WHEN resource_string LIKE '%/%.%a%/%' THEN 'a' + ELSE 'r' END AS channel, + COALESCE(SUBSTRING(resource_string + FROM '.*_([a-zA-Z]{2}|[a-zA-Z]{2}-[a-zA-Z]{2})[\._-].*'), '??') AS locale, + CASE WHEN resource_string LIKE '%.incremental.%' THEN TRUE + ELSE FALSE END AS incremental, + SUM(count) AS count + FROM files NATURAL JOIN requests NATURAL JOIN resources + WHERE resource_string LIKE '%/torbrowser/%.mar' + AND response_code = 302 + AND method = 'GET' + GROUP BY log_date, request_type, platform, channel, locale, incremental + UNION + SELECT log_date, + 'tmid' AS request_type, + CASE WHEN resource_string LIKE '%.exe' THEN 'w' + WHEN resource_string LIKE '%.dmg' THEN 'm' + WHEN resource_string LIKE '%.tar.xz' THEN 'l' + ELSE 'o' END AS platform, + NULL AS channel, + COALESCE(SUBSTRING(resource_string + FROM '.*_([a-zA-Z]{2}|[a-zA-Z]{2}-[a-zA-Z]{2})[\._-].*'), '??') AS locale, + NULL::BOOLEAN AS incremental, + SUM(count) AS count + FROM files NATURAL JOIN requests NATURAL JOIN resources + WHERE (resource_string LIKE '%/tormessenger/%.exe' + OR resource_string LIKE '%/tormessenger/%.dmg' + OR resource_string LIKE '%/tormessenger/%.tar.xz') + AND response_code = 200 + AND method = 'GET' + GROUP BY log_date, request_type, platform, channel, locale, incremental + UNION + SELECT log_date, + 'tmup' AS request_type, + CASE WHEN resource_string LIKE '%/WINNT%' THEN 'w' + WHEN resource_string LIKE '%/Darwin%' THEN 'm' + WHEN resource_string LIKE '%/Linux%' THEN 'l' + ELSE 'o' END AS platform, + NULL AS channel, + COALESCE(SUBSTRING(resource_string + FROM '.*/([a-zA-Z]{2}|[a-zA-Z]{2}-[a-zA-Z]{2})\??$'), '??') AS locale, + NULL::BOOLEAN AS incremental, + SUM(count) AS count + FROM files NATURAL JOIN requests NATURAL JOIN resources + WHERE resource_string LIKE '%/tormessenger/update_2/%' + AND resource_string NOT LIKE '%.xml' + AND resource_string NOT LIKE '%/' + AND resource_string NOT LIKE '%/?' + AND response_code = 200 + AND method = 'GET' + GROUP BY log_date, request_type, platform, channel, locale, incremental + UNION + SELECT log_date, + 'twhph' AS request_type, + NULL AS platform, + NULL AS channel, + NULL AS locale, + NULL::BOOLEAN AS incremental, + SUM(count) AS count + FROM files NATURAL JOIN requests NATURAL JOIN resources + WHERE (resource_string = '/' + OR resource_string LIKE '/index%') + AND response_code = 200 + AND (site = 'torproject.org' + OR site = 'www.torproject.org') + AND method = 'GET' + GROUP BY log_date, request_type, platform, channel, locale, incremental + UNION + SELECT log_date, + 'twdph' AS request_type, + NULL AS platform, + NULL AS channel, + NULL AS locale, + NULL::BOOLEAN AS incremental, + SUM(count) AS count + FROM files NATURAL JOIN requests NATURAL JOIN resources + WHERE (resource_string LIKE '/download/download%' + OR resource_string LIKE '/projects/torbrowser.html%') + AND response_code = 200 + AND (site = 'torproject.org' + OR site = 'www.torproject.org') + AND method = 'GET' + GROUP BY log_date, request_type, platform, channel, locale, incremental; + diff --git a/modules/webstats/src/main/resources/write-RData.R b/modules/webstats/src/main/resources/write-RData.R new file mode 100644 index 0000000..2cb8917 --- /dev/null +++ b/modules/webstats/src/main/resources/write-RData.R @@ -0,0 +1,9 @@ +dir.create("RData", showWarnings = FALSE) + +d <- read.csv("stats/webstats.csv", stringsAsFactors = FALSE) +d <- d[d$request_type %in% c('tbid', 'tbsd', 'tbup', 'tbur'), ] +data <- aggregate(list(count = d$count), + by = list(log_date = as.Date(d$log_date), request_type = d$request_type), + FUN = sum) +save(data, file = "RData/webstats-tb.RData") + diff --git a/modules/webstats/src/test/java/org/torproject/metrics/webstats/MainTest.java b/modules/webstats/src/test/java/org/torproject/metrics/webstats/MainTest.java new file mode 100644 index 0000000..1c4f0bc --- /dev/null +++ b/modules/webstats/src/test/java/org/torproject/metrics/webstats/MainTest.java @@ -0,0 +1,93 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.webstats; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +import java.util.regex.Matcher; + +public class MainTest { + + static final String SAMPLE_LOG_FILE_NAME = + "metrics.torproject.org-access.log-20170117.xz"; + + static final String SAMPLE_SUBDIRECTORY_NAME = "meronense.torproject.org/"; + + static final String SAMPLE_LOG_FILE_URL = + "https://webstats.torproject.org/out/meronense.torproject.org/" + + "metrics.torproject.org-access.log-20170117.xz"; + + static final String[] SAMPLE_LOG_LINES = new String[] { + "0.0.0.0 - - [17/Jan/2017:00:00:00 +0000] " + + "\"GET / HTTP/1.0\" 200 10532 \"-\" \"-\" -", + "0.0.0.0 - - [17/Jan/2017:00:00:00 +0000] " + + "\"HEAD /bubbles.html HTTP/1.1\" 200 - \"-\" \"-\" -" + }; + + @Test + public void testUrlStringPatternComplete() { + Matcher matcher = Main.URL_STRING_PATTERN.matcher( + "<img src=\"/icons/unknown.gif\" alt=\"[ ]\"> " + + "<a href=\"" + SAMPLE_LOG_FILE_NAME + "\">" + SAMPLE_LOG_FILE_NAME + + "</a> 2017-01-19 19:43 5.6K "); + assertTrue(matcher.matches()); + assertEquals(SAMPLE_LOG_FILE_NAME, matcher.group(1)); + } + + @Test + public void testUrlStringPatternOnlyATag() { + Matcher matcher = Main.URL_STRING_PATTERN.matcher("<a href=\"" + + SAMPLE_LOG_FILE_NAME + "\">" + SAMPLE_LOG_FILE_NAME + "</a>"); + assertTrue(matcher.matches()); + assertEquals(SAMPLE_LOG_FILE_NAME, matcher.group(1)); + } + + @Test + public void testUrlStringPatternSubdirectory() { + Matcher matcher = Main.URL_STRING_PATTERN.matcher( + "<a href=\"" + SAMPLE_SUBDIRECTORY_NAME + "\">" + + SAMPLE_SUBDIRECTORY_NAME + "/</a>"); + assertTrue(matcher.matches()); + assertEquals(SAMPLE_SUBDIRECTORY_NAME, matcher.group(1)); + } + + @Test + public void testUrlStringPatternAnythingBetweenDoubleQuotesHtml() { + Matcher matcher = Main.URL_STRING_PATTERN.matcher( + "<a href=\"anything-between-double-quotes.html\">Link/</a>"); + assertTrue(matcher.matches()); + assertEquals("anything-between-double-quotes.html", matcher.group(1)); + } + + @Test + public void testLogFileUrlPatternComplete() { + Matcher matcher = Main.LOG_FILE_URL_PATTERN.matcher(SAMPLE_LOG_FILE_URL); + assertTrue(matcher.matches()); + assertEquals("meronense.torproject.org", matcher.group(1)); + assertEquals("metrics.torproject.org", matcher.group(2)); + assertEquals("20170117", matcher.group(3)); + } + + @Test + public void testLogLinePatternGetSlash() { + Matcher matcher = Main.LOG_LINE_PATTERN.matcher(SAMPLE_LOG_LINES[0]); + assertTrue(matcher.matches()); + assertEquals("GET", matcher.group(1)); + assertEquals("/", matcher.group(2)); + assertEquals("200", matcher.group(3)); + } + + @Test + public void testLogLinePatternHeadBubbles() { + Matcher matcher = Main.LOG_LINE_PATTERN.matcher(SAMPLE_LOG_LINES[1]); + assertTrue(matcher.matches()); + assertEquals("HEAD", matcher.group(1)); + assertEquals("/bubbles.html", matcher.group(2)); + assertEquals("200", matcher.group(3)); + } +} + diff --git a/shared/bin/90-run-webstats-stats.sh b/shared/bin/90-run-webstats-stats.sh new file mode 100755 index 0000000..37091b4 --- /dev/null +++ b/shared/bin/90-run-webstats-stats.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +cd modules/webstats/ + +ant run | grep "\[java\]" + +R --slave -f src/main/resources/write-RData.R > /dev/null 2>&1 + +cd ../../ + diff --git a/shared/bin/99-copy-stats-files.sh b/shared/bin/99-copy-stats-files.sh index d236630..a828686 100755 --- a/shared/bin/99-copy-stats-files.sh +++ b/shared/bin/99-copy-stats-files.sh @@ -6,7 +6,9 @@ cp -a modules/advbwdist/stats/advbwdist.csv shared/stats/ cp -a modules/hidserv/stats/hidserv.csv shared/stats/ cp -a modules/clients/stats/clients*.csv shared/stats/ cp -a modules/clients/stats/userstats-combined.csv shared/stats/ +cp -a modules/webstats/stats/webstats.csv shared/stats/ mkdir -p shared/RData cp -a modules/clients/RData/*.RData shared/RData/ +cp -a modules/webstats/RData/*.RData shared/RData/ diff --git a/shared/build-base.xml b/shared/build-base.xml index 7051f49..759e1d0 100644 --- a/shared/build-base.xml +++ b/shared/build-base.xml @@ -1,9 +1,11 @@ <project basedir="."> <property name="sources" value="src"/> + <property name="testsources" value="src/test/java"/> <property name="libs" value="../../shared/lib"/> <property name="generated" value="generated"/> <property name="classes" value="${generated}/classes/"/> + <property name="testclasses" value="${generated}/test-classes/"/> <property name="source-and-target-java-version" value="1.7" /> <property name="descriptorversion" value="1.5.0" /> @@ -21,6 +23,21 @@ </fileset> </path> + <path id="base.testclasspath"> + <pathelement path="${base.testclasses}"/> + <pathelement path="base.classpath"/> + <fileset dir="${libs}"> + <include name="hamcrest-core-1.3.jar"/> + <include name="junit4-4.11.jar"/> + </fileset> + </path> + + <path id="testclasspath"> + <pathelement path="${testclasses}"/> + <path refid="base.testclasspath" /> + <path refid="base.classpath" /> + </path> + <target name="clean"> <delete includeEmptyDirs="true" quiet="true"> <fileset dir="${generated}" defaultexcludes="false" includes="**" /> @@ -29,6 +46,7 @@ <target name="init"> <mkdir dir="${classes}"/> + <mkdir dir="${testclasses}"/> </target> <target name="compile" depends="init" > @@ -45,6 +63,30 @@ </javac> </target> + <target name="testcompile" depends="compile" > + <javac destdir="${testclasses}" + srcdir="${testsources}" + source="${source-and-target-java-version}" + target="${source-and-target-java-version}" + debug="true" debuglevel="lines,source" + deprecation="true" + optimize="false" + failonerror="true" + includeantruntime="false"> + <classpath refid="testclasspath"/> + </javac> + </target> + + <target name="test" depends="testcompile"> + <junit fork="true" haltonfailure="true" printsummary="off"> + <classpath refid="testclasspath"/> + <formatter type="plain" usefile="false"/> + <batchtest> + <fileset dir="${testclasses}" + includes="**/*Test.class"/> + </batchtest> + </junit> + </target> </project> diff --git a/shared/build.xml b/shared/build.xml index 13a09f7..cb51d5f 100644 --- a/shared/build.xml +++ b/shared/build.xml @@ -26,9 +26,9 @@ <fileset dir="../modules/clients/src" includes="**/*.java"/> <fileset dir="../modules/collectdescs/src" includes="**/*.java"/> <fileset dir="../modules/connbidirect/src" includes="**/*.java"/> - <fileset dir="../modules/disagreement/src" includes="**/*.java"/> <fileset dir="../modules/hidserv/src" includes="**/*.java"/> <fileset dir="../modules/legacy/src" includes="**/*.java"/> + <fileset dir="../modules/webstats/src" includes="**/*.java"/> <classpath> <path refid="checkstyle.classpath" /> </classpath> diff --git a/website/etc/categories.json b/website/etc/categories.json index 8b4ea77..6825634 100644 --- a/website/etc/categories.json +++ b/website/etc/categories.json @@ -78,6 +78,9 @@ "icon": "fa-download", "header": "Applications", "summary": "How many Tor applications, like Tor Browser, have been downloaded or updated.", - "metrics": [] + "description": "The following application statistics are based on the analysis of requests to <code>torproject.org</code> web servers.", + "metrics": [ + "webstats-tb" + ] } ] diff --git a/website/etc/metrics.json b/website/etc/metrics.json index f7666be..4a97ca1 100644 --- a/website/etc/metrics.json +++ b/website/etc/metrics.json @@ -402,5 +402,19 @@ "title": "Network churn rate by relay flag", "type": "Link", "description": "This image shows the churn rate of the Tor network by <a href=\"glossary.html#relay-flag\">relay flag</a> in a given month. The churn rate, a value in the interval [0,1], captures the rate of <a href=\"glossary.html#relay\">relays</a> joining and leaving the network from one <a href=\"glossary.html#consensus\">consensus</a> to the next (that is, within one hour). The complete image gallery can be found on <a href=\"https://nymity.ch/sybilhunting/churn-values/\">Philipp Winter's homepage</a>.<a href=\"https://nymity.ch/sybilhunting/churn-values/\"><img src=\"images/networkchurn.png\" alt=\"Network churn rate by relay flag\"></a>" + }, + { + "id": "webstats-tb", + "title": "Tor Browser downloads and updates", + "type": "Graph", + "description": "This graph shows absolute numbers of requests to Tor's web servers by request type. It is based on data from <a href=\"https://webstats.torproject.org/\" target=\"_blank\"><code>webstats.torproject.org</code></a> which collects logs from <code>torproject.org</code> web servers and provides them as a stripped-down version of Apache's "combined" log format without IP addresses, log times, HTTP parameters, referers, and user agent strings. Initial downloads and signature downloads are requests made by the user to download a Tor Browser executable or a corresponding signature file from the Tor website. Update pings and update requests are requests made by Tor Browser to check whether a newer version is available or to download a newer version.", + "function": "plot_webstats_tb", + "parameters": [ + "start", + "end" + ], + "data": [ + "webstats" + ] } ] diff --git a/website/etc/web.xml b/website/etc/web.xml index 7444cf5..916984c 100644 --- a/website/etc/web.xml +++ b/website/etc/web.xml @@ -46,6 +46,7 @@ <url-pattern>/hidserv-dir-onions-seen.html</url-pattern> <url-pattern>/hidserv-rend-relayed-cells.html</url-pattern> <url-pattern>/hidserv-frac-reporting.html</url-pattern> + <url-pattern>/webstats-tb.html</url-pattern> </servlet-mapping> <servlet> @@ -177,6 +178,9 @@ <url-pattern>/hidserv-frac-reporting.png</url-pattern> <url-pattern>/hidserv-frac-reporting.pdf</url-pattern> <url-pattern>/hidserv-frac-reporting.svg</url-pattern> + <url-pattern>/webstats-tb.png</url-pattern> + <url-pattern>/webstats-tb.pdf</url-pattern> + <url-pattern>/webstats-tb.svg</url-pattern> </servlet-mapping> <servlet> diff --git a/website/rserve/graphs.R b/website/rserve/graphs.R index 9f8daa7..fc6cfb6 100644 --- a/website/rserve/graphs.R +++ b/website/rserve/graphs.R @@ -1095,3 +1095,34 @@ plot_hidserv_frac_reporting <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 72) } +plot_webstats_tb <- function(start, end, path) { + end <- min(end, as.character(Sys.Date() - 2)) + load("/srv/metrics.torproject.org/metrics/shared/RData/webstats-tb.RData") + d <- data + d <- d[d$log_date >= start & d$log_date <= end, ] + date_breaks <- date_breaks(as.numeric(max(d$log_date) - min(d$log_date))) + d$request_type <- factor(d$request_type) + levels(d$request_type) <- list( + 'Initial downloads' = 'tbid', + 'Signature downloads' = 'tbsd', + 'Update pings' = 'tbup', + 'Update requests' = 'tbur') + formatter <- function(x, ...) { + format(x, ..., scientific = FALSE, big.mark = ' ') } + ggplot(d, aes(x = log_date, y = count)) + + geom_point() + + geom_line() + + expand_limits(y = 0) + + facet_grid(request_type ~ ., scales = "free_y") + + scale_x_date(name = paste("\nThe Tor Project - ", + "https://metrics.torproject.org/", sep = ""), + labels = date_format(date_breaks$format), + breaks = date_breaks$major, + minor_breaks = date_breaks$minor) + + scale_y_continuous(name = 'Requests per day\n', labels = formatter) + + theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)), + strip.background = element_rect(fill = NA)) + + ggtitle("Tor Browser downloads and updates\n") + ggsave(filename = path, width = 8, height = 5, dpi = 72) +} + diff --git a/website/src/org/torproject/metrics/web/research/ResearchStatsServlet.java b/website/src/org/torproject/metrics/web/research/ResearchStatsServlet.java index 8f5c399..de5715e 100644 --- a/website/src/org/torproject/metrics/web/research/ResearchStatsServlet.java +++ b/website/src/org/torproject/metrics/web/research/ResearchStatsServlet.java @@ -39,6 +39,7 @@ public class ResearchStatsServlet extends HttpServlet { this.availableStatisticsFiles.add("advbwdist"); this.availableStatisticsFiles.add("hidserv"); this.availableStatisticsFiles.add("disagreement"); + this.availableStatisticsFiles.add("webstats"); } @Override diff --git a/website/web/WEB-INF/sources.jsp b/website/web/WEB-INF/sources.jsp index 64ca85b..b36d43f 100644 --- a/website/web/WEB-INF/sources.jsp +++ b/website/web/WEB-INF/sources.jsp @@ -39,6 +39,12 @@ </div> <div class="container"> + <ul> + <li><a href="https://webstats.torproject.org/" target="_blank"><code>webstats.torproject.org</code></a> collects logs from <code>torproject.org</code> web servers and provides them as a stripped-down version of Apache's "combined" log format without IP addresses, log times, HTTP parameters, referers, and user agent strings.</li> + </ul> + </div> + + <div class="container"> <h2>Measurement tools <a href="#measurement" name="measurement" class="anchor">#</a></h2> The following tools perform active measurements in the Tor network. <ul> diff --git a/website/web/WEB-INF/stats.jsp b/website/web/WEB-INF/stats.jsp index fc676ba..fbecc0f 100644 --- a/website/web/WEB-INF/stats.jsp +++ b/website/web/WEB-INF/stats.jsp @@ -483,8 +483,38 @@ given attribute.</li> </ul> - </div> - </div> +</div> + +<div class="container"> +<h2>Requests to <code>torproject.org</code> web servers <a href="#webstats" name="webstats" class="anchor">#</a></h2> + +The following data file contains aggregate statistics on requests to <code>torproject.org</code> web servers. + +Download as <a href="stats/webstats.csv">CSV file</a>. + +The statistics file contains the following columns: +<ul> +<li>log_date: UTC date (YYYY-MM-DD) when requests to <code>torproject.org</code> web servers have been logged.</li> +<li>request_type: Request type with fixed identifiers as follows: +<ul> +<li>"tbid": Tor Browser initial downloads: GET requests to all sites with resource strings <code>'%/torbrowser/%.exe'</code>, <code>'%/torbrowser/%.dmg'</code>, and <code>'%/torbrowser/%.tar.xz'</code> and response code 200.</li> +<li>"tbsd": Tor Browser signature downloads: GET requests to all sites with resource strings <code>'%/torbrowser/%.exe.asc'</code>, <code>'%/torbrowser/%.dmg.asc'</code>, and <code>'%/torbrowser/%.tar.xz.asc'</code> and response code 200.</li> +<li>"tbup": Tor Browser update pings: GET requests to all sites with resource strings <code>'%/torbrowser/update_2/%'</code> and response code 200.</li> +<li>"tbur": Tor Browser update requests: GET requests to all sites with resource strings <code>'%/torbrowser/%.mar'</code> and response code 302.</li> +<li>"tmid": Tor Messenger initial downloads: GET requests to all sites with resource strings <code>'%/tormessenger/%.exe'</code>, <code>'%/tormessenger/%.dmg'</code>, and <code>'%/tormessenger/%.tar.xz'</code> and response code 200.</li> +<li>"tmup": Tor Messenger update pings: GET requests to all sites with resource strings <code>'%/tormessenger/update_2/%'</code> and response code 200.</li> +<li>"twhph": Tor website home page hits: GET requests to sites <code>'torproject.org'</code> and <code>'www.torproject.org'</code> with resource strings <code>'/'</code> and <code>'/index%'</code> and response code 200.</li> +<li>"twdph": Tor website download page hits: GET requests to sites <code>'torproject.org'</code> and <code>'www.torproject.org'</code> with resource strings <code>'/download/download%'</code> and <code>'/projects/torbrowser.html%'</code> and response code 200.</li> +</ul> +</li> +<li>platform: Platform string, like "w" for Windows, "m" for macOS, or "l" for Linux, "o" for other platforms, and the empty string for all platforms.</li> +<li>channel: Release channel, like "r" for stable releases, "a" for alpha releases, "h" for hardened releases, and the empty string for all channels.</li> +<li>locale: Locale, like "en-US" for English (United States), "de" for German, etc., "??" for unrecognized locales, and the empty string for all locales.</li> +<li>incremental: Incremental update, with "t" for incremental updates, "f" for non-incremental (full) updates, and the empty string for all updates types.</li> +<li>count: Number of request for the given request type, platform, etc.</li> +</ul> + +</div> <jsp:include page="bottom.jsp"/>

1 0

[translation/tor-browser-manual_completed] Update translations for tor-browser-manual_completed
by translation＠torproject.org 23 Jan '17

23 Jan '17

commit b7de3ac2a7cbc89b9b41c1d6023c4f06bcba6f7a Author: Translation commit bot <translation(a)torproject.org> Date: Mon Jan 23 18:48:29 2017 +0000 Update translations for tor-browser-manual_completed --- zh_CN/zh_CN.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zh_CN/zh_CN.po b/zh_CN/zh_CN.po index 7f4ff53..e7b2e7b 100644 --- a/zh_CN/zh_CN.po +++ b/zh_CN/zh_CN.po @@ -1376,7 +1376,7 @@ msgstr "移除掉 Tor 浏览器后再重新安装，若是要更新版本的话 #: troubleshooting.page:64 msgid "Is your connection censored?" -msgstr "您的连接受到审查？" +msgstr "您的连接是否受到审查？" #: troubleshooting.page:65 msgid ""

1 0

[translation/tor-browser-manual] Update translations for tor-browser-manual
by translation＠torproject.org 23 Jan '17

23 Jan '17

commit ad258ee78e86ce093cd95df8ee9163bc75a3726f Author: Translation commit bot <translation(a)torproject.org> Date: Mon Jan 23 18:48:26 2017 +0000 Update translations for tor-browser-manual --- nl/nl.po | 8 ++++---- ro/ro.po | 30 +++++++++++++++++------------- zh_CN/zh_CN.po | 2 +- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/nl/nl.po b/nl/nl.po index aa51e57..063603a 100644 --- a/nl/nl.po +++ b/nl/nl.po @@ -210,7 +210,7 @@ msgstr "" #: circumvention.page:6 msgid "What to do if the Tor network is blocked" -msgstr "" +msgstr "Wat te doen als het Tor netwerk geblokkeerd is" #: circumvention.page:10 msgid "Circumvention" @@ -310,7 +310,7 @@ msgstr "" #: downloading.page:7 msgid "How to download Tor Browser" -msgstr "" +msgstr "Hoe de Tor Browser downloaden" #: downloading.page:10 msgid "Downloading" @@ -434,11 +434,11 @@ msgstr "" #: first-time.page:7 msgid "Learn how to use Tor Browser for the first time" -msgstr "" +msgstr "Leer hoe de Tor Browser te gebruiken voor de eerste keer" #: first-time.page:10 msgid "Running Tor Browser for the first time" -msgstr "" +msgstr "De Tor Browser gebruiken voor de eerste keer" #: first-time.page:12 msgid "" diff --git a/ro/ro.po b/ro/ro.po index ae9d7cb..f3ad6e0 100644 --- a/ro/ro.po +++ b/ro/ro.po @@ -6,12 +6,13 @@ # Iordache Livia-Serena <iordacheliviaserena(a)gmail.com>, 2016 # laura berindei <lauraagavriloae(a)yahoo.com>, 2016 # Viorel <f_viorel(a)yahoo.com>, 2016 +# drazvan <razvan.net(a)gmail.com>, 2017 msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "POT-Creation-Date: 2016-12-06 16:36-0600\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: Viorel <f_viorel(a)yahoo.com>, 2016\n" +"Last-Translator: drazvan <razvan.net(a)gmail.com>, 2017\n" "Language-Team: Romanian (https://www.transifex.com/otf/teams/1519/ro/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -104,7 +105,7 @@ msgstr "" #: bridges.page:10 msgid "Bridges" -msgstr "" +msgstr "Punți" #: bridges.page:12 msgid "" @@ -303,7 +304,7 @@ msgstr "" #: downloading.page:27 msgid "GetTor" -msgstr "" +msgstr "GetTor" #: downloading.page:28 msgid "" @@ -356,7 +357,7 @@ msgstr "" #: downloading.page:84 msgid "Satori" -msgstr "" +msgstr "Satori" #: downloading.page:85 msgid "" @@ -497,7 +498,7 @@ msgstr "" #: index.page:6 msgid "Tor Browser User Manual" -msgstr "" +msgstr "Manual de utilizare pentru navigatorul Tor" #: known-issues.page:6 msgid "A list of known issues." @@ -505,7 +506,7 @@ msgstr "" #: known-issues.page:10 msgid "Known Issues" -msgstr "" +msgstr "Probleme cunoscute" #: known-issues.page:14 msgid "" @@ -528,11 +529,11 @@ msgstr "" #: known-issues.page:29 msgid "Sophos Antivirus for Mac" -msgstr "" +msgstr "Antivirusul Sophos pentru Mac" #: known-issues.page:32 msgid "Microsoft Security Essentials" -msgstr "" +msgstr "Microsoft Security Essentials" #: known-issues.page:37 msgid "" @@ -571,6 +572,9 @@ msgid "" " ./start-tor-browser.desktop\n" " " msgstr "" +"\n" +" ./start-tor-browser.desktop\n" +" " #: managing-identities.page:6 msgid "Learn how to control personally-identifying information in Tor Browser" @@ -578,7 +582,7 @@ msgstr "" #: managing-identities.page:10 msgid "Managing identities" -msgstr "" +msgstr "Administrarea identităților" #: managing-identities.page:12 msgid "" @@ -736,7 +740,7 @@ msgstr "" #: onionsites.page:10 msgid "Onion Services" -msgstr "" +msgstr "Servicii Onion" #: onionsites.page:11 msgid "" @@ -795,7 +799,7 @@ msgstr "" #: onionsites.page:58 troubleshooting.page:10 msgid "Troubleshooting" -msgstr "" +msgstr "Depanare" #: onionsites.page:59 msgid "" @@ -828,7 +832,7 @@ msgstr "" #: plugins.page:13 msgid "Flash Player" -msgstr "" +msgstr "Player Flash" #: plugins.page:14 msgid "" @@ -849,7 +853,7 @@ msgstr "" #: plugins.page:31 msgid "JavaScript" -msgstr "" +msgstr "JavaScript" #: plugins.page:32 msgid "" diff --git a/zh_CN/zh_CN.po b/zh_CN/zh_CN.po index 7f4ff53..e7b2e7b 100644 --- a/zh_CN/zh_CN.po +++ b/zh_CN/zh_CN.po @@ -1376,7 +1376,7 @@ msgstr "移除掉 Tor 浏览器后再重新安装，若是要更新版本的话 #: troubleshooting.page:64 msgid "Is your connection censored?" -msgstr "您的连接受到审查？" +msgstr "您的连接是否受到审查？" #: troubleshooting.page:65 msgid ""

1 0

[translation/tails-greeter-2] Update translations for tails-greeter-2
by translation＠torproject.org 23 Jan '17

23 Jan '17

commit 06874ad908dd6b5f084b3ea7a33a2a86da69b855 Author: Translation commit bot <translation(a)torproject.org> Date: Mon Jan 23 18:48:13 2017 +0000 Update translations for tails-greeter-2 --- ro/ro.po | 83 +++++++++++++++++++++++++++++++++++----------------------------- 1 file changed, 45 insertions(+), 38 deletions(-) diff --git a/ro/ro.po b/ro/ro.po index 7584572..fd57891 100644 --- a/ro/ro.po +++ b/ro/ro.po @@ -10,7 +10,7 @@ msgstr "" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2016-12-03 13:14+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: clopotel <yo_sergiu05(a)yahoo.com>, 2016\n" +"Last-Translator: drazvan <razvan.net(a)gmail.com>, 2017\n" "Language-Team: Romanian (https://www.transifex.com/otf/teams/1519/ro/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -20,7 +20,7 @@ msgstr "" #: ../data/greeter.ui.h:1 msgid "Administration Password" -msgstr "" +msgstr "Parolă administrare" #: ../data/greeter.ui.h:2 msgid "" @@ -31,24 +31,24 @@ msgstr "" #: ../data/greeter.ui.h:3 msgid "Enter an administration password" -msgstr "" +msgstr "Introdu o parolă de administrare" #: ../data/greeter.ui.h:4 msgid "Confirm" -msgstr "" +msgstr "Confirmă" #: ../data/greeter.ui.h:5 msgid "Confirm your administration password" -msgstr "" +msgstr "Confimă parola ta de administrare" #: ../data/greeter.ui.h:6 msgid "Disable" -msgstr "" +msgstr "Dezactivează" #. This string will never be displayed in the 1st version of the greeter. #: ../data/greeter.ui.h:8 msgid "Windows Camouflage" -msgstr "" +msgstr "Camuflare Windows" #. This string will never be displayed in the 1st version of the greeter. #: ../data/greeter.ui.h:10 @@ -56,15 +56,17 @@ msgid "" "This option makes Tails look like Microsoft Windows 10. This can be useful " "to avoid attracting suspicion in public places." msgstr "" +"Această opțiune face ca Tails să semene mai mult cu Microsoft Windows 10. " +"Acest lucru poate fi util în locuri publice pentru a evita suspiciunile." #. This string will never be displayed in the 1st version of the greeter. #: ../data/greeter.ui.h:12 msgid "Microsoft Windows 10 camouflage" -msgstr "" +msgstr "Camuflare Microsoft Windows 10" #: ../data/greeter.ui.h:13 msgid "MAC Address Spoofing" -msgstr "" +msgstr "Falsificare adresă MAC" #: ../data/greeter.ui.h:14 msgid "" @@ -76,15 +78,15 @@ msgstr "" #: ../data/greeter.ui.h:15 msgid "Spoof all MAC addresses (default)" -msgstr "" +msgstr "Falsifică toate adresele MAC (implicit)" #: ../data/greeter.ui.h:16 msgid "Don't spoof MAC addresses" -msgstr "" +msgstr "Nu falsifica adresele MAC" #: ../data/greeter.ui.h:17 msgid "Welcome to Tails!" -msgstr "" +msgstr "Bun venit la Tails!" #. This string will never be displayed in the 1st version of the greeter. #: ../data/greeter.ui.h:19 @@ -95,15 +97,15 @@ msgstr "" #: ../data/greeter.ui.h:20 msgid "Language & Region" -msgstr "" +msgstr "Limbă și Regiune" #: ../data/greeter.ui.h:21 msgid "Default Settings" -msgstr "" +msgstr "Setări implicite" #: ../data/greeter.ui.h:22 msgid "Save Language & Region Settings" -msgstr "" +msgstr "Salvează setările de Limbă și Regiune" #: ../data/greeter.ui.h:23 msgid "Language" @@ -115,52 +117,52 @@ msgstr "" #: ../data/greeter.ui.h:25 msgid "Formats" -msgstr "" +msgstr "Formate" #: ../data/greeter.ui.h:26 msgid "Time Zone" -msgstr "" +msgstr "Fus orar" #: ../data/greeter.ui.h:27 msgid "Encrypted Persistent Storage" -msgstr "" +msgstr "Stocare persistentă criptată" #: ../data/greeter.ui.h:28 msgid "Show Passphrase" -msgstr "" +msgstr "Arată parola" #: ../data/greeter.ui.h:29 msgid "Configure Persistent Storage" -msgstr "" +msgstr "Configurează stocarea persistentă" #: ../data/greeter.ui.h:30 msgid "Enter your passphrase to unlock the persistent storage" -msgstr "" +msgstr "Introdu parola ta pentru a debloca stocarea persistentă." #: ../data/greeter.ui.h:31 ../tailsgreeter/gui.py:798 #: ../tailsgreeter/gui.py:962 msgid "Unlock" -msgstr "" +msgstr "Deblochează" #: ../data/greeter.ui.h:32 msgid "Relock Persistent Storage" -msgstr "" +msgstr "Blochează din nou Stocarea persistentă" #: ../data/greeter.ui.h:33 ../tailsgreeter/gui.py:514 msgid "Additional Settings" -msgstr "" +msgstr "Setări adiționale" #: ../data/greeter.ui.h:34 msgid "Save Additional Settings" -msgstr "" +msgstr "Salvează Setările adiționale" #: ../data/greeter.ui.h:35 msgid "Add an additional setting" -msgstr "" +msgstr "Adaugă o setare adițională" #: ../data/greeter.ui.h:36 msgid "Network Configuration" -msgstr "" +msgstr "Configurare rețea" #: ../data/greeter.ui.h:37 msgid "" @@ -168,14 +170,17 @@ msgid "" "configure a Tor bridge or a local proxy. To work completely offline, you can" " disable all networking." msgstr "" +"Dacă conexiunea ta de Internet este cenzurată, filtrată, sau trecută printr-" +"un proxy, poți configura o punte Tor sau un proxy local. Pentru a lucra " +"complet în mod deconectat, poți dezactiva rețeaua." #: ../data/greeter.ui.h:38 msgid "Connect directly to the Tor network (default)" -msgstr "" +msgstr "Conectează-te direct la rețeaua Tor (implicit)" #: ../data/greeter.ui.h:39 msgid "Configure a Tor bridge, firewall, or proxy settings" -msgstr "" +msgstr "Configurează o punte Tor, un firewall, sau setările proxy" #: ../data/greeter.ui.h:40 msgid "Disable all networking" @@ -186,22 +191,24 @@ msgid "" "The default settings are safe in most situations. To add a custom setting, " "press the \"+\" button below." msgstr "" +"Setările implicite sunt sigure în majoritatea cazurilor. Pentru a adăuga o " +"setare personalizată, apasă pe butonul \"+\" de mai jos." #: ../data/greeter.ui.h:42 ../tailsgreeter/gui.py:717 msgid "Off (default)" -msgstr "" +msgstr "Dezactivat (implicit)" #: ../data/greeter.ui.h:43 ../tailsgreeter/gui.py:715 msgid "On (default)" -msgstr "" +msgstr "Activat (implicit)" #: ../data/greeter.ui.h:44 msgid "Network Connection" -msgstr "" +msgstr "Conexiune rețea" #: ../data/greeter.ui.h:45 ../tailsgreeter/gui.py:914 msgid "Direct (default)" -msgstr "" +msgstr "Direct (implicit)" #: ../tailsgreeter/gui.py:456 msgid "Shutdown" @@ -209,7 +216,7 @@ msgstr "Inchide" #: ../tailsgreeter/gui.py:462 msgid "Start Tails" -msgstr "" +msgstr "Porneşte Tails" #: ../tailsgreeter/gui.py:523 msgid "Cancel" @@ -225,7 +232,7 @@ msgstr "Înapoi" #: ../tailsgreeter/gui.py:652 msgid "Unlocking…" -msgstr "" +msgstr "Se deblochează..." #: ../tailsgreeter/gui.py:680 msgid "" @@ -235,11 +242,11 @@ msgstr "" #: ../tailsgreeter/gui.py:720 msgid "On" -msgstr "" +msgstr "Activat" #: ../tailsgreeter/gui.py:722 msgid "Off" -msgstr "" +msgstr "Dezactivat" #: ../tailsgreeter/gui.py:801 msgid "Failed to relock persistent storage." @@ -247,7 +254,7 @@ msgstr "" #: ../tailsgreeter/gui.py:920 msgid "Bridge & Proxy" -msgstr "" +msgstr "Bridge și Proxy" #: ../tailsgreeter/gui.py:926 msgid "Offline"

1 0

[translation/torbutton-torbuttondtd] Update translations for torbutton-torbuttondtd
by translation＠torproject.org 23 Jan '17

23 Jan '17

commit 972d1029568d9ced72238345af0b4bdc14a22c2e Author: Translation commit bot <translation(a)torproject.org> Date: Mon Jan 23 18:46:40 2017 +0000 Update translations for torbutton-torbuttondtd --- ro/torbutton.dtd | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ro/torbutton.dtd b/ro/torbutton.dtd index 7ce65eb..b3f4631 100644 --- a/ro/torbutton.dtd +++ b/ro/torbutton.dtd @@ -1,4 +1,4 @@ -<!ENTITY torbutton.context_menu.new_identity "Identitate noua"> +<!ENTITY torbutton.context_menu.new_identity "Identitate nouă"> <!ENTITY torbutton.context_menu.new_identity_key "l"> <!ENTITY torbutton.context_menu.new_circuit "Circuit Tor nou pentru acest site"> <!ENTITY torbutton.context_menu.new_circuit_key "C"> @@ -11,9 +11,9 @@ <!ENTITY torbutton.context_menu.cookieProtections "Protecție cookie..."> <!ENTITY torbutton.context_menu.cookieProtections.key "C"> <!ENTITY torbutton.button.tooltip "Apasă pentru pornirea Torbutton"> -<!ENTITY torbutton.prefs.security_settings "Tor Browser Security Settings"> +<!ENTITY torbutton.prefs.security_settings "Setări de securitate pentru navigatorul Tor"> <!ENTITY torbutton.prefs.restore_defaults "Revino la setările inițiale"> -<!ENTITY torbutton.prefs.custom_warning "Your custom browser preferences have resulted in unusual security settings. For security and privacy reasons, we recommend you choose one of the default security levels."> +<!ENTITY torbutton.prefs.custom_warning "Preferințele tale de browser personalizate au avut ca rezultat setări de securitate neobișnuite. Din motive de securitate și confidențialitate, vă recomandăm să alegeți una dintre nivelurile de securitate implicite."> <!ENTITY torbutton.cookiedialog.title "Gestionează Protecţii Cookie"> <!ENTITY torbutton.cookiedialog.lockCol "Protejat"> <!ENTITY torbutton.cookiedialog.domainCol "Gazda"> @@ -46,7 +46,7 @@ <!ENTITY torbutton.prefs.sec_jit_slower_desc "Scripturile pe unele site-uri poate rula încet."> <!ENTITY torbutton.prefs.sec_mathml_desc "Anumite mecanisme de afișare ecuații matematice sunt dezactivate."> <!ENTITY torbutton.prefs.sec_mathml_desc_tooltip "MathML este dezactivat."> -<!ENTITY torbutton.prefs.sec_medium "Medium"> +<!ENTITY torbutton.prefs.sec_medium "Mediu"> <!ENTITY torbutton.prefs.sec_all_jit_desc "Sunt dezactivate optimizări de performanță pe pagini unde JavaScript este activat."> <!ENTITY torbutton.prefs.sec_font_rend_desc "Anumite caracteristici de expunere fonturi sunt dezactivate."> <!ENTITY torbutton.prefs.sec_font_rend_graphite_tooltip "Mecanismul de expunere fonturi Graphite este dezactivat.">

1 0

[translation/abouttor-homepage_completed] Update translations for abouttor-homepage_completed
by translation＠torproject.org 23 Jan '17

23 Jan '17

commit 716a3ad70f1d8786ced13b80362a93ca8a3506ad Author: Translation commit bot <translation(a)torproject.org> Date: Mon Jan 23 18:46:22 2017 +0000 Update translations for abouttor-homepage_completed --- ro/aboutTor.dtd | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ro/aboutTor.dtd b/ro/aboutTor.dtd index eaeb6f4..33ce08e 100644 --- a/ro/aboutTor.dtd +++ b/ro/aboutTor.dtd @@ -19,10 +19,7 @@ <!ENTITY aboutTor.failure2.label "Tor nu funcționează în acest browser."> <!ENTITY aboutTor.search.label "Căutare"> - -<!ENTITY aboutTor.searchSPPost.link "https://startpage.com/rth/search"> -<!ENTITY aboutTor.searchDDGPost.link "https://duckduckgo.com/html/"> -<!ENTITY aboutTor.searchDCPost.link "https://search.disconnect.me/searchTerms/search?ses=Google&location_opt…"> +<!ENTITY aboutTor.searchDDGPost.link "https://duckduckgo.com"> <!ENTITY aboutTor.torInfo1.label "Informații suplimentare:"> <!ENTITY aboutTor.torInfo2.label "Țara și adresa IP:"> @@ -32,6 +29,9 @@ <!ENTITY aboutTor.whatnextAnswer.label "Tor NU este tot ce ai nevoie pentru a naviga anonim! S-ar putea să trebuiască să îți schimbi unele obiceiuri de navigare, ca să fii sigur că identitatea îți rămâne protejată."> <!ENTITY aboutTor.whatnext.label "Ponturi despre cum să rămâi anonim >>"> <!ENTITY aboutTor.whatnext.link "https://www.torproject.org/download/download.html.en#warning"> +<!ENTITY aboutTor.torbrowser_user_manual.accesskey "M"> +<!ENTITY aboutTor.torbrowser_user_manual.label "Manual de utilizare pentru navigatorul Tor"> +<!ENTITY aboutTor.torbrowser_user_manual.link "https://tb-manual.torproject.org/en-US"> <!ENTITY aboutTor.helpInfo1.label "Poți să ajuți!"> <!ENTITY aboutTor.helpInfo2.label "Sunt multe căi prin care tu poți ajuta pentru a face Tor Network mai rapid și mai puternic:"> <!ENTITY aboutTor.helpInfo3.label "Rulează un Nod Releu Tor >>">

1 0