commit addf07db2ae47cf7c5e395d4bd7f503a40944e6c Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Mon Oct 19 09:22:35 2015 +0200 Add an index.json with all served contents. Implements #17321. --- bin/update-index | 3 + build.xml | 8 + .../collector/index/CreateIndexJson.java | 161 ++++++++++++++++++++ 3 files changed, 172 insertions(+) diff --git a/bin/update-index b/bin/update-index new file mode 100755 index 0000000..4a3f2f8 --- /dev/null +++ b/bin/update-index @@ -0,0 +1,3 @@ +#!/bin/sh +ant updateindex | grep "\[java\]" + diff --git a/build.xml b/build.xml index 218d6b5..4d7b986 100644 --- a/build.xml +++ b/build.xml @@ -8,6 +8,8 @@ <fileset dir="/usr/share/java"> <include name="commons-codec.jar"/> <include name="commons-compress.jar"/> + <include name="gson.jar"/> + <include name="xz.jar"/> </fileset> <fileset dir="deps/metrics-lib"> <include name="descriptor.jar"/> @@ -71,5 +73,11 @@ <classpath refid="classpath"/> </java> </target> + <target name="updateindex" depends="compile"> + <java fork="true" + classname="org.torproject.collector.index.CreateIndexJson"> + <classpath refid="classpath"/> + </java> + </target> </project> diff --git a/src/org/torproject/collector/index/CreateIndexJson.java b/src/org/torproject/collector/index/CreateIndexJson.java new file mode 100644 index 0000000..cf2e0bb --- /dev/null +++ b/src/org/torproject/collector/index/CreateIndexJson.java @@ -0,0 +1,161 @@ +/* Copyright 2015 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.collector.index; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Locale; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeSet; +import java.util.zip.GZIPOutputStream; + +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +/* Create a fresh index.json containing all directories and files in the + * archive/ and recent/ directories. + * + * Note that if this ever takes longer than a few seconds, we'll have to + * cache index parts of directories or files that haven't changed. + * Example: if we parse include cryptographic hashes or @type information, + * we'll likely have to do that. */ +public class CreateIndexJson { + + static final File indexJsonFile = new File("index.json"); + + static final String basePath = "https://collector.torproject.org"; + + static final File[] indexedDirectories = new File[] { + new File("archive"), new File("recent") }; + + static final String dateTimePattern = "yyyy-MM-dd HH:mm"; + + static final Locale dateTimeLocale = Locale.US; + + static final TimeZone dateTimezone = TimeZone.getTimeZone("UTC"); + + public static void main(String[] args) throws IOException { + writeIndex(indexDirectories()); + } + + static class DirectoryNode implements Comparable<DirectoryNode> { + String path; + SortedSet<FileNode> files; + SortedSet<DirectoryNode> directories; + DirectoryNode(String path, SortedSet<FileNode> files, + SortedSet<DirectoryNode> directories) { + this.path = path; + this.files = files; + this.directories = directories; + } + public int compareTo(DirectoryNode o) { + return this.path.compareTo(o.path); + } + } + + static class IndexNode { + String index_created; + String path; + SortedSet<FileNode> files; + SortedSet<DirectoryNode> directories; + IndexNode(String index_created, String path, + SortedSet<FileNode> files, + SortedSet<DirectoryNode> directories) { + this.index_created = index_created; + this.path = path; + this.files = files; + this.directories = directories; + } + } + + static class FileNode implements Comparable<FileNode> { + String path; + long size; + String last_modified; + FileNode(String path, long size, String last_modified) { + this.path = path; + this.size = size; + this.last_modified = last_modified; + } + public int compareTo(FileNode o) { + return this.path.compareTo(o.path); + } + } + + static DateFormat dateTimeFormat; + static { + dateTimeFormat = new SimpleDateFormat(dateTimePattern, + dateTimeLocale); + dateTimeFormat.setLenient(false); + dateTimeFormat.setTimeZone(dateTimezone); + } + + static IndexNode indexDirectories() { + SortedSet<DirectoryNode> directoryNodes = + new TreeSet<DirectoryNode>(); + for (File directory : indexedDirectories) { + if (directory.exists() && directory.isDirectory()) { + directoryNodes.add(indexDirectory(directory)); + } + } + return new IndexNode(dateTimeFormat.format( + System.currentTimeMillis()), basePath, null, directoryNodes); + } + + static DirectoryNode indexDirectory(File directory) { + SortedSet<FileNode> fileNodes = new TreeSet<FileNode>(); + SortedSet<DirectoryNode> directoryNodes = + new TreeSet<DirectoryNode>(); + for (File fileOrDirectory : directory.listFiles()) { + if (fileOrDirectory.getName().startsWith(".")) { + continue; + } + if (fileOrDirectory.isFile()) { + fileNodes.add(indexFile(fileOrDirectory)); + } else { + directoryNodes.add(indexDirectory(fileOrDirectory)); + } + } + DirectoryNode directoryNode = new DirectoryNode( + directory.getName(), fileNodes.isEmpty() ? null : fileNodes, + directoryNodes.isEmpty() ? null : directoryNodes); + return directoryNode; + } + + static FileNode indexFile(File file) { + FileNode fileNode = new FileNode(file.getName(), file.length(), + dateTimeFormat.format(file.lastModified())); + return fileNode; + } + + static void writeIndex(IndexNode indexNode) throws IOException { + Gson gson = new GsonBuilder().create(); + String indexNodeString = gson.toJson(indexNode); + Writer[] writers = new Writer[] { + new FileWriter(indexJsonFile), + new OutputStreamWriter(new GZIPOutputStream( + new FileOutputStream(indexJsonFile + ".gz"))), + new OutputStreamWriter(new XZCompressorOutputStream( + new FileOutputStream(indexJsonFile + ".xz"))), + new OutputStreamWriter(new BZip2CompressorOutputStream( + new FileOutputStream(indexJsonFile + ".bz2"))) + }; + for (Writer writer : writers) { + BufferedWriter bufferedWriter = new BufferedWriter(writer); + bufferedWriter.write(indexNodeString); + bufferedWriter.close(); + } + } +} +