[tor-commits] [metrics-db/master] Add an index.json with all served contents.

[karsten at torproject.org]

commit addf07db2ae47cf7c5e395d4bd7f503a40944e6c
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Mon Oct 19 09:22:35 2015 +0200

    Add an index.json with all served contents.
    
    Implements #17321.
---
 bin/update-index                                   |    3 +
 build.xml                                          |    8 +
 .../collector/index/CreateIndexJson.java           |  161 ++++++++++++++++++++
 3 files changed, 172 insertions(+)

diff --git a/bin/update-index b/bin/update-index
new file mode 100755
index 0000000..4a3f2f8
--- /dev/null
+++ b/bin/update-index
@@ -0,0 +1,3 @@
+#!/bin/sh
+ant updateindex | grep "\[java\]"
+
diff --git a/build.xml b/build.xml
index 218d6b5..4d7b986 100644
--- a/build.xml
+++ b/build.xml
@@ -8,6 +8,8 @@
     <fileset dir="/usr/share/java">
       <include name="commons-codec.jar"/>
       <include name="commons-compress.jar"/>
+      <include name="gson.jar"/>
+      <include name="xz.jar"/>
     </fileset>
     <fileset dir="deps/metrics-lib">
       <include name="descriptor.jar"/>
@@ -71,5 +73,11 @@
       <classpath refid="classpath"/>
     </java>
   </target>
+  <target name="updateindex" depends="compile">
+    <java fork="true"
+          classname="org.torproject.collector.index.CreateIndexJson">
+      <classpath refid="classpath"/>
+    </java>
+  </target>
 </project>
 
diff --git a/src/org/torproject/collector/index/CreateIndexJson.java b/src/org/torproject/collector/index/CreateIndexJson.java
new file mode 100644
index 0000000..cf2e0bb
--- /dev/null
+++ b/src/org/torproject/collector/index/CreateIndexJson.java
@@ -0,0 +1,161 @@
+/* Copyright 2015 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.collector.index;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Locale;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeSet;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+/* Create a fresh index.json containing all directories and files in the
+ * archive/ and recent/ directories.
+ *
+ * Note that if this ever takes longer than a few seconds, we'll have to
+ * cache index parts of directories or files that haven't changed.
+ * Example: if we parse include cryptographic hashes or @type information,
+ * we'll likely have to do that. */
+public class CreateIndexJson {
+
+  static final File indexJsonFile = new File("index.json");
+
+  static final String basePath = "https://collector.torproject.org";
+
+  static final File[] indexedDirectories = new File[] {
+      new File("archive"), new File("recent") };
+
+  static final String dateTimePattern = "yyyy-MM-dd HH:mm";
+
+  static final Locale dateTimeLocale = Locale.US;
+
+  static final TimeZone dateTimezone = TimeZone.getTimeZone("UTC");
+
+  public static void main(String[] args) throws IOException {
+    writeIndex(indexDirectories());
+  }
+
+  static class DirectoryNode implements Comparable<DirectoryNode> {
+    String path;
+    SortedSet<FileNode> files;
+    SortedSet<DirectoryNode> directories;
+    DirectoryNode(String path, SortedSet<FileNode> files,
+        SortedSet<DirectoryNode> directories) {
+      this.path = path;
+      this.files = files;
+      this.directories = directories;
+    }
+    public int compareTo(DirectoryNode o) {
+      return this.path.compareTo(o.path);
+    }
+  }
+
+  static class IndexNode {
+    String index_created;
+    String path;
+    SortedSet<FileNode> files;
+    SortedSet<DirectoryNode> directories;
+    IndexNode(String index_created, String path,
+        SortedSet<FileNode> files,
+        SortedSet<DirectoryNode> directories) {
+      this.index_created = index_created;
+      this.path = path;
+      this.files = files;
+      this.directories = directories;
+    }
+  }
+
+  static class FileNode implements Comparable<FileNode> {
+    String path;
+    long size;
+    String last_modified;
+    FileNode(String path, long size, String last_modified) {
+      this.path = path;
+      this.size = size;
+      this.last_modified = last_modified;
+    }
+    public int compareTo(FileNode o) {
+      return this.path.compareTo(o.path);
+    }
+  }
+
+  static DateFormat dateTimeFormat;
+  static {
+    dateTimeFormat = new SimpleDateFormat(dateTimePattern,
+        dateTimeLocale);
+    dateTimeFormat.setLenient(false);
+    dateTimeFormat.setTimeZone(dateTimezone);
+  }
+
+  static IndexNode indexDirectories() {
+    SortedSet<DirectoryNode> directoryNodes =
+        new TreeSet<DirectoryNode>();
+    for (File directory : indexedDirectories) {
+      if (directory.exists() && directory.isDirectory()) {
+        directoryNodes.add(indexDirectory(directory));
+      }
+    }
+    return new IndexNode(dateTimeFormat.format(
+        System.currentTimeMillis()), basePath, null, directoryNodes);
+  }
+
+  static DirectoryNode indexDirectory(File directory) {
+    SortedSet<FileNode> fileNodes = new TreeSet<FileNode>();
+    SortedSet<DirectoryNode> directoryNodes =
+        new TreeSet<DirectoryNode>();
+    for (File fileOrDirectory : directory.listFiles()) {
+      if (fileOrDirectory.getName().startsWith(".")) {
+        continue;
+      }
+      if (fileOrDirectory.isFile()) {
+        fileNodes.add(indexFile(fileOrDirectory));
+      } else {
+        directoryNodes.add(indexDirectory(fileOrDirectory));
+      }
+    }
+    DirectoryNode directoryNode = new DirectoryNode(
+        directory.getName(), fileNodes.isEmpty() ? null : fileNodes,
+        directoryNodes.isEmpty() ? null : directoryNodes);
+    return directoryNode;
+  }
+
+  static FileNode indexFile(File file) {
+    FileNode fileNode = new FileNode(file.getName(), file.length(),
+        dateTimeFormat.format(file.lastModified()));
+    return fileNode;
+  }
+
+  static void writeIndex(IndexNode indexNode) throws IOException {
+    Gson gson = new GsonBuilder().create();
+    String indexNodeString = gson.toJson(indexNode);
+    Writer[] writers = new Writer[] {
+        new FileWriter(indexJsonFile),
+        new OutputStreamWriter(new GZIPOutputStream(
+            new FileOutputStream(indexJsonFile + ".gz"))),
+        new OutputStreamWriter(new XZCompressorOutputStream(
+            new FileOutputStream(indexJsonFile + ".xz"))),
+        new OutputStreamWriter(new BZip2CompressorOutputStream(
+            new FileOutputStream(indexJsonFile + ".bz2")))
+    };
+    for (Writer writer : writers) {
+      BufferedWriter bufferedWriter = new BufferedWriter(writer);
+      bufferedWriter.write(indexNodeString);
+      bufferedWriter.close();
+    }
+  }
+}
+