[tor-commits] [tor/master] Functionality to ensure there is space to add files to cache.
nickm at torproject.org
nickm at torproject.org
Fri Apr 28 15:08:22 UTC 2017
commit 7a0964279fafc9912ed5cc4b82abfb81328f5150
Author: Nick Mathewson <nickm at torproject.org>
Date: Wed Apr 26 10:42:11 2017 -0400
Functionality to ensure there is space to add files to cache.
---
src/common/storagedir.c | 10 ++++++
src/common/storagedir.h | 1 +
src/or/conscache.c | 13 ++++++++
src/or/conscache.h | 1 +
src/or/consdiffmgr.c | 81 +++++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 106 insertions(+)
diff --git a/src/common/storagedir.c b/src/common/storagedir.c
index 9d3c32e..309d42d 100644
--- a/src/common/storagedir.c
+++ b/src/common/storagedir.c
@@ -530,3 +530,13 @@ storage_dir_remove_all(storage_dir_t *d)
return storage_dir_shrink(d, 0, d->max_files);
}
+/**
+ * Return the largest number of non-temporary files we're willing to
+ * store in <b>d</b>.
+ */
+int
+storage_dir_get_max_files(storage_dir_t *d)
+{
+ return d->max_files;
+}
+
diff --git a/src/common/storagedir.h b/src/common/storagedir.h
index 7811944..db25057 100644
--- a/src/common/storagedir.h
+++ b/src/common/storagedir.h
@@ -45,6 +45,7 @@ int storage_dir_shrink(storage_dir_t *d,
uint64_t target_size,
int min_to_remove);
int storage_dir_remove_all(storage_dir_t *d);
+int storage_dir_get_max_files(storage_dir_t *d);
#endif
diff --git a/src/or/conscache.c b/src/or/conscache.c
index 7760d13..5ffa129 100644
--- a/src/or/conscache.c
+++ b/src/or/conscache.c
@@ -400,6 +400,19 @@ consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff)
}
/**
+ * Return the number of currently unused filenames available in this cache.
+ */
+int
+consensus_cache_get_n_filenames_available(consensus_cache_t *cache)
+{
+ tor_assert(cache);
+ int max = storage_dir_get_max_files(cache->dir);
+ int used = smartlist_len(storage_dir_list(cache->dir));
+ tor_assert_nonfatal(max >= used);
+ return max - used;
+}
+
+/**
* Delete every element of <b>cache</b> has been marked with
* consensus_cache_entry_mark_for_removal. If <b>force</b> is false,
* retain those entries which are not in use except by the cache.
diff --git a/src/or/conscache.h b/src/or/conscache.h
index ea27733..aef5420 100644
--- a/src/or/conscache.h
+++ b/src/or/conscache.h
@@ -19,6 +19,7 @@ int consensus_cache_register_with_sandbox(consensus_cache_t *cache,
void consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff);
void consensus_cache_delete_pending(consensus_cache_t *cache,
int force);
+int consensus_cache_get_n_filenames_available(consensus_cache_t *cache);
consensus_cache_entry_t *consensus_cache_add(consensus_cache_t *cache,
const config_line_t *labels,
const uint8_t *data,
diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c
index 1f3915f..ba52650 100644
--- a/src/or/consdiffmgr.c
+++ b/src/or/consdiffmgr.c
@@ -107,6 +107,7 @@ static consdiff_cfg_t consdiff_cfg = {
/* .cache_max_num = */ 128
};
+static int consdiffmgr_ensure_space_for_files(int n);
static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
consensus_cache_entry_t *diff_to);
static void consdiffmgr_set_cache_flags(void);
@@ -413,6 +414,8 @@ consdiffmgr_add_consensus(const char *consensus,
}
/* We don't have it. Add it to the cache. */
+ consdiffmgr_ensure_space_for_files(1);
+
{
size_t bodylen = strlen(consensus);
config_line_t *labels = NULL;
@@ -853,6 +856,82 @@ consdiffmgr_rescan(void)
}
/**
+ * Helper: compare two files by their from-valid-after and valid-after labels,
+ * trying to sort in ascending order by from-valid-after (when present) and
+ * valid-after (when not). Place everything that has neither label first in
+ * the list.
+ */
+static int
+compare_by_staleness_(const void **a, const void **b)
+{
+ const consensus_cache_entry_t *e1 = *a;
+ const consensus_cache_entry_t *e2 = *b;
+ const char *va1, *fva1, *va2, *fva2;
+ va1 = consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER);
+ va2 = consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER);
+ fva1 = consensus_cache_entry_get_value(e1, LABEL_FROM_VALID_AFTER);
+ fva2 = consensus_cache_entry_get_value(e2, LABEL_FROM_VALID_AFTER);
+
+ if (fva1)
+ va1 = fva1;
+ if (fva2)
+ va2 = fva2;
+
+ /* See note about iso-encoded values in compare_by_valid_after_. Also note
+ * that missing dates will get placed first. */
+ return strcmp_opt(va1, va2);
+}
+
+/** If there are not enough unused filenames to store <b>n</b> files, then
+ * delete old consensuses until there are. (We have to keep track of the
+ * number of filenames because of the way that the seccomp2 cache works.)
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+static int
+consdiffmgr_ensure_space_for_files(int n)
+{
+ consensus_cache_t *cache = cdm_cache_get();
+ if (consensus_cache_get_n_filenames_available(cache) >= n) {
+ // there are already enough unused filenames.
+ return 0;
+ }
+ // Try a cheap deletion of stuff that's waiting to get deleted.
+ consensus_cache_delete_pending(cache, 0);
+ if (consensus_cache_get_n_filenames_available(cache) >= n) {
+ // okay, _that_ made enough filenames available.
+ return 0;
+ }
+ // Let's get more assertive: clean out unused stuff, and force-remove
+ // the files.
+ consdiffmgr_cleanup();
+ consensus_cache_delete_pending(cache, 1);
+ const int n_to_remove = n - consensus_cache_get_n_filenames_available(cache);
+ if (n_to_remove <= 0) {
+ // okay, finally!
+ return 0;
+ }
+
+ // At this point, we're going to have to throw out objects that will be
+ // missed. Too bad!
+ smartlist_t *objects = smartlist_new();
+ consensus_cache_find_all(objects, cache, NULL, NULL);
+ smartlist_sort(objects, compare_by_staleness_);
+ int n_marked = 0;
+ SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
+ consensus_cache_entry_mark_for_removal(ent);
+ if (++n_marked >= n_to_remove)
+ break;
+ } SMARTLIST_FOREACH_END(ent);
+
+ consensus_cache_delete_pending(cache, 1);
+ if (BUG(n_marked < n_to_remove))
+ return -1;
+ else
+ return 0;
+}
+
+/**
* Set consensus cache flags on the objects in this consdiffmgr.
*/
static void
@@ -1066,6 +1145,8 @@ consensus_diff_worker_replyfn(void *work_)
/* Success! Store the results */
log_info(LD_DIRSERV, "Adding consensus diff from %s to %s",
lv_from_digest, lv_to_digest);
+
+ consdiffmgr_ensure_space_for_files(1);
consensus_cache_entry_t *ent =
consensus_cache_add(cdm_cache_get(), job->labels_out,
job->body_out,
More information about the tor-commits
mailing list