commit 5cedec578e04116aa588eb791faf08969a69f712 Author: Karsten Loesing karsten.loesing@gmx.net Date: Sun Apr 26 17:06:39 2015 +0200
Add four new execution modes.
New modes are:
--single-run Run steps 1--3 only for a single time, then exit. --download-only Only run step 1: download recent descriptors, then exit. --update-only Only run step 2: update internal status files, then exit. --write-only Only run step 3: write output document files, then exit.
Default mode is:
[no argument] Run steps 1--3 repeatedly once per hour.
Use the lock file in default periodic-update mode, too, because we don't want other execution modes to interfere with an ongoing periodic update either.
Implements part of #13600. --- .../java/org/torproject/onionoo/cron/Main.java | 276 ++++++++++++++------ .../onionoo/updater/DescriptorSource.java | 3 - 2 files changed, 200 insertions(+), 79 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/cron/Main.java b/src/main/java/org/torproject/onionoo/cron/Main.java index c098dfe..2f45cea 100644 --- a/src/main/java/org/torproject/onionoo/cron/Main.java +++ b/src/main/java/org/torproject/onionoo/cron/Main.java @@ -1,4 +1,4 @@ -/* Copyright 2011, 2012 The Tor Project +/* Copyright 2011--2015 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.cron;
@@ -18,102 +18,226 @@ import org.torproject.onionoo.util.LockFile; import org.torproject.onionoo.writer.DocumentWriterRunner;
/* Update search data and status data files. */ -public class Main { - - private static Logger log = LoggerFactory.getLogger(Main.class); +public class Main implements Runnable {
private Main() { }
- private static final ScheduledExecutorService scheduler = - Executors.newScheduledThreadPool(1); + private Logger log = LoggerFactory.getLogger(Main.class);
public static void main(String[] args) { - boolean runOnce = "true".equals(System.getProperty( - "onionoo.cron.runonce", "true")); - if (runOnce){ - log.info("Going to run one-time updater ... "); - LockFile lf = new LockFile(); - log.info("Initializing."); - if (lf.acquireLock()) { - log.info("Acquired lock"); - } else { - log.error("Could not acquire lock. Is Onionoo already running? " - + "Terminating"); - return; - } - new Updater().run(); - log.info("Releasing lock."); - if (lf.releaseLock()) { - log.info("Released lock"); - } else { - log.error("Could not release lock. The next execution may not " - + "start as expected"); + Main main = new Main(); + main.parseArgsOrExit(args); + main.runOrScheduleExecutions(); + } + + boolean defaultMode = false, singleRun = false, downloadOnly = false, + updateOnly = false, writeOnly = false; + + /* TODO Parsing command-line arguments is only a workaround until we're + * more certain what kind of options we want to support. We should then + * switch to some library that parses options for us. */ + private void parseArgsOrExit(String[] args) { + boolean validArgs = true; + if (args.length == 0) { + this.defaultMode = true; + } else if (args.length == 1) { + switch (args[0]) { + case "--help": + this.printUsageAndExit(0); + break; + case "--single-run": + this.singleRun = true; + break; + case "--download-only": + this.downloadOnly = true; + break; + case "--update-only": + this.updateOnly = true; + break; + case "--write-only": + this.writeOnly = true; + break; + default: + validArgs = false; } - return; + } else if (args.length > 1) { + validArgs = false; + } + if (!validArgs) { + this.printUsageAndExit(1); + } + } + + private void printUsageAndExit(int status) { + System.err.println("Please provide only a single execution:"); + System.err.println(" [no argument] Run steps 1--3 repeatedly " + + "once per hour."); + System.err.println(" --single-run Run steps 1--3 only for a " + + "single time, then exit."); + System.err.println(" --download-only Only run step 1: download " + + "recent descriptors, then exit."); + System.err.println(" --update-only Only run step 2: update " + + "internal status files, then exit."); + System.err.println(" --write-only Only run step 3: write " + + "output document files, then exit."); + System.err.println(" --help Print out this help message " + + "and exit."); + System.exit(status); + } + + private void runOrScheduleExecutions() { + if (!this.defaultMode) { + this.log.info("Going to run one-time updater ... "); + this.run(); } else { - log.info("Periodic updater started."); - final Runnable updater = new Updater(); - int currentMinute = Calendar.getInstance().get(Calendar.MINUTE); - int initialDelay = (75 - currentMinute + currentMinute % 5) % 60; + this.scheduleExecutions(); + } + } + + private final ScheduledExecutorService scheduler = + Executors.newScheduledThreadPool(1); + + private void scheduleExecutions() { + this.log.info("Periodic updater started."); + final Runnable mainRunnable = this; + int currentMinute = Calendar.getInstance().get(Calendar.MINUTE); + int initialDelay = (75 - currentMinute + currentMinute % 5) % 60; + + /* Run after initialDelay delay and then every hour. */ + this.log.info("Periodic updater will start every hour at minute " + + ((currentMinute + initialDelay) % 60) + "."); + this.scheduler.scheduleAtFixedRate(mainRunnable, initialDelay, 60, + TimeUnit.MINUTES); + } + + public void run() { + this.acquireLockOrExit(); + this.initialize(); + this.downloadDescriptors(); + this.updateStatuses(); + this.writeDocuments(); + this.shutDown(); + this.gatherStatistics(); + this.cleanUp(); + this.releaseLock(); + } + + private LockFile lf;
- /* Run after initialDelay delay and then every hour. */ - log.info("Periodic updater will start every hour at minute " - + ((currentMinute + initialDelay) % 60) + "."); - scheduler.scheduleAtFixedRate(updater, initialDelay, 60, - TimeUnit.MINUTES); + private void acquireLockOrExit() { + this.log.info("Initializing."); + this.lf = new LockFile(); + if (this.lf.acquireLock()) { + this.log.info("Acquired lock"); + } else { + this.log.error("Could not acquire lock. Is Onionoo already " + + "running? Terminating"); + System.exit(1); } }
- private static class Updater implements Runnable{ + private DescriptorSource dso;
- private Logger log = LoggerFactory.getLogger(Main.class); + private DocumentStore ds;
- public void run() { - log.debug("Started update ..."); + private StatusUpdateRunner sur;
- DescriptorSource dso = - DescriptorSourceFactory.getDescriptorSource(); - log.info("Initialized descriptor source"); - DocumentStore ds = DocumentStoreFactory.getDocumentStore(); - log.info("Initialized document store"); - StatusUpdateRunner sur = new StatusUpdateRunner(); - log.info("Initialized status update runner"); - DocumentWriterRunner dwr = new DocumentWriterRunner(); - log.info("Initialized document writer runner"); + private DocumentWriterRunner dwr;
- log.info("Downloading descriptors."); - dso.downloadDescriptors(); + private void initialize() { + this.log.debug("Started update ..."); + if (!this.writeOnly) { + this.dso = DescriptorSourceFactory.getDescriptorSource(); + this.log.info("Initialized descriptor source"); + } + if (!this.downloadOnly) { + this.ds = DocumentStoreFactory.getDocumentStore(); + this.log.info("Initialized document store"); + } + if (!this.downloadOnly && !this.writeOnly) { + this.sur = new StatusUpdateRunner(); + this.log.info("Initialized status update runner"); + } + if (!this.downloadOnly && !this.updateOnly) { + this.dwr = new DocumentWriterRunner(); + this.log.info("Initialized document writer runner"); + } + }
- log.info("Reading descriptors."); - dso.readDescriptors(); + private void downloadDescriptors() { + if (this.updateOnly || this.writeOnly) { + return; + } + this.log.info("Downloading descriptors."); + this.dso.downloadDescriptors(); + }
- log.info("Updating internal status files."); - sur.updateStatuses(); + private void updateStatuses() { + if (this.downloadOnly || this.writeOnly) { + return; + } + this.log.info("Reading descriptors."); + this.dso.readDescriptors(); + this.log.info("Updating internal status files."); + this.sur.updateStatuses(); + }
- log.info("Updating document files."); - dwr.writeDocuments(); + private void writeDocuments() { + if (this.downloadOnly || this.updateOnly) { + return; + } + log.info("Updating document files."); + this.dwr.writeDocuments(); + }
- log.info("Shutting down."); - dso.writeHistoryFiles(); + private void shutDown() { + log.info("Shutting down."); + if (this.dso != null) { + this.dso.writeHistoryFiles(); log.info("Wrote parse histories"); - ds.flushDocumentCache(); - log.info("Flushed document cache"); - - log.info("Gathering statistics."); - sur.logStatistics(); - dwr.logStatistics(); - log.info("Descriptor source\n" + dso.getStatsString()); - log.info("Document store\n" + ds.getStatsString()); - - /* Clean up to prevent out-of-memory exception, and to ensure that - * the next execution starts with a fresh descriptor source. */ - log.info("Cleaning up."); - ds.invalidateDocumentCache(); - DocumentStoreFactory.setDocumentStore(null); - DescriptorSourceFactory.setDescriptorSource(null); - - log.info("Done."); + } + if (this.ds != null) { + this.ds.flushDocumentCache(); + this.log.info("Flushed document cache"); + } + } + + private void gatherStatistics() { + this.log.info("Gathering statistics."); + if (this.sur != null) { + this.sur.logStatistics(); + } + if (this.dwr != null) { + this.dwr.logStatistics(); + } + if (this.dso != null) { + this.log.info("Descriptor source\n" + this.dso.getStatsString()); + } + if (this.ds != null) { + this.log.info("Document store\n" + this.ds.getStatsString()); + } + } + + private void cleanUp() { + /* Clean up to prevent out-of-memory exception, and to ensure that the + * next execution starts with a fresh descriptor source. */ + this.log.info("Cleaning up."); + if (this.ds != null) { + this.ds.invalidateDocumentCache(); + } + DocumentStoreFactory.setDocumentStore(null); + DescriptorSourceFactory.setDescriptorSource(null); + this.log.info("Done."); + } + + private void releaseLock() { + this.log.info("Releasing lock."); + if (this.lf.releaseLock()) { + this.log.info("Released lock"); + } else { + this.log.error("Could not release lock. The next execution may " + + "not start as expected"); } } } diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java index 26b7b10..52e00c9 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java @@ -68,9 +68,6 @@ public class DescriptorSource { downloadedFiles = 0, deletedLocalFiles = 0;
private void downloadDescriptors(DescriptorType descriptorType) { - if (!this.descriptorListeners.containsKey(descriptorType)) { - return; - } DescriptorDownloader descriptorDownloader = new DescriptorDownloader(descriptorType); this.localFilesBefore += descriptorDownloader.statLocalFiles();