- tor-commits - lists.torproject.org

[tor/maint-0.2.2] Rebuild the microdesc cache when a sufficient number of bytes are dropped
by nickm＠torproject.org 03 May '11

03 May '11

commit cb6c909664f97f751fe78c0aa3205a9042760c53 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 17:03:49 2011 -0400 Rebuild the microdesc cache when a sufficient number of bytes are dropped Previously on 0.2.2, we'd never clean the cache. Now that we can clean it, we want to add a condition to rebuild it: that should happen whenever we have dropped enough microdescriptors that we could save a lot of space. No changes file, since 0.2.3 doesn't need one and 0.2.2 already has some changes files for the backport of the microdesc_clean_cahce() function. --- src/or/microdesc.c | 45 +++++++++++++++++++++++++++++++++------------ src/or/microdesc.h | 2 +- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 58fbe37..c6bb3c6 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -177,6 +177,8 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, if (md2->last_listed < md->last_listed) md2->last_listed = md->last_listed; microdesc_free(md); + if (where != SAVED_NOWHERE) + cache->bytes_dropped += size; continue; } @@ -206,14 +208,7 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, if (f) finish_writing_to_file(open_file); /*XXX Check me.*/ - { - size_t old_content_len = - cache->cache_content ? cache->cache_content->size : 0; - if (cache->journal_len > 16384 + old_content_len && - cache->journal_len > old_content_len / 2) { - microdesc_cache_rebuild(cache); - } - } + microdesc_cache_rebuild(cache, 0/* only as needed */); return added; } @@ -275,6 +270,9 @@ microdesc_cache_reload(microdesc_cache_t *cache) } log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", total); + + microdesc_cache_clean(cache, 0, 0); + return 0; } @@ -323,11 +321,30 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) } } +static int +should_rebuild_md_cache(microdesc_cache_t *cache) +{ + const size_t old_len = + cache->cache_content ? cache->cache_content->size : 0; + const size_t journal_len = cache->journal_len; + const size_t dropped = cache->bytes_dropped; + + if (journal_len < 16384) + return 0; /* Don't bother, not enough has happened yet. */ + if (dropped > (journal_len + old_len) / 3) + return 1; /* We could save 1/3 or more of the currently used space. */ + if (journal_len > old_len / 2) + return 1; /* We should append to the regular file */ + + return 0; +} + /** Regenerate the main cache file for cache, clear the journal file, * and update every microdesc_t in the cache with pointers to its new - * location. */ + * location. If force is true, do this unconditionally. If + * force is false, do it only if we expect to save space on disk. */ int -microdesc_cache_rebuild(microdesc_cache_t *cache) +microdesc_cache_rebuild(microdesc_cache_t *cache, int force) { open_file_t *open_file; FILE *f; @@ -337,11 +354,14 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) off_t off = 0; int orig_size, new_size; - log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); - /* Remove dead descriptors */ microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/); + if (!force && !should_rebuild_md_cache(cache)) + return 0; + + log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); orig_size += (int)cache->journal_len; @@ -396,6 +416,7 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) write_str_to_file(cache->journal_fname, "", 1); cache->journal_len = 0; + cache->bytes_dropped = 0; new_size = (int)cache->cache_content->size; log_info(LD_DIR, "Done rebuilding microdesc cache. " diff --git a/src/or/microdesc.h b/src/or/microdesc.h index 30cb25d..77ce853 100644 --- a/src/or/microdesc.h +++ b/src/or/microdesc.h @@ -22,7 +22,7 @@ smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, int no_save); void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force); -int microdesc_cache_rebuild(microdesc_cache_t *cache); +int microdesc_cache_rebuild(microdesc_cache_t *cache, int force); int microdesc_cache_reload(microdesc_cache_t *cache); void microdesc_cache_clear(microdesc_cache_t *cache);

1 0

[tor/master] Backport microdesc_cache_clean to 0.2.2
by nickm＠torproject.org 03 May '11

03 May '11

commit 56fbd728c2df7f189390b9a417b4c3ecc3690ede Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 16:45:15 2011 -0400 Backport microdesc_cache_clean to 0.2.2 Otherwise we have no way to keep authorities' microdesc caches in 0.2.2 from growing without bound. --- changes/bug2230_clean_1 | 4 +++ src/or/microdesc.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++ src/or/microdesc.h | 1 + 3 files changed, 56 insertions(+), 0 deletions(-) diff --git a/changes/bug2230_clean_1 b/changes/bug2230_clean_1 new file mode 100644 index 0000000..a4edf94 --- /dev/null +++ b/changes/bug2230_clean_1 @@ -0,0 +1,4 @@ + o Minor features + - Backport code from 0.2.3.x to allow directory authorities to clean + their microdescriptor caches. + diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 0ceb134..58fbe37 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -23,6 +23,8 @@ struct microdesc_cache_t { tor_mmap_t *cache_content; /** Number of bytes used in the journal file. */ size_t journal_len; + /** Number of bytes in descriptors removed as too old. */ + size_t bytes_dropped; /** Total bytes of microdescriptor bodies we have added to this cache */ uint64_t total_len_seen; @@ -276,6 +278,51 @@ microdesc_cache_reload(microdesc_cache_t *cache) return 0; } +/** By default, we remove any microdescriptors that have gone at least this + * long without appearing in a current consensus. */ +#define TOLERATE_MICRODESC_AGE (7*24*60*60) + +/** Remove all microdescriptors from cache that haven't been listed for + * a long time. Does not rebuild the cache on disk. If cutoff is + * positive, specifically remove microdescriptors that have been unlisted + * since cutoff. If force is true, remove microdescriptors even + * if we have no current live microdescriptor consensus. + */ +void +microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) +{ + microdesc_t **mdp, *victim; + int dropped=0, kept=0; + size_t bytes_dropped = 0; + time_t now = time(NULL); + + (void) force; + /* In 0.2.2, we let this proceed unconditionally: only authorities have + * microdesc caches. */ + + if (cutoff <= 0) + cutoff = now - TOLERATE_MICRODESC_AGE; + + for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) { + if ((*mdp)->last_listed < cutoff) { + ++dropped; + victim = *mdp; + mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp); + bytes_dropped += victim->bodylen; + microdesc_free(victim); + } else { + ++kept; + mdp = HT_NEXT(microdesc_map, &cache->map, mdp); + } + } + + if (dropped) { + log_notice(LD_DIR, "Removed %d/%d microdescriptors as old.", + dropped,dropped+kept); + cache->bytes_dropped += bytes_dropped; + } +} + /** Regenerate the main cache file for cache, clear the journal file, * and update every microdesc_t in the cache with pointers to its new * location. */ @@ -291,6 +338,10 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) int orig_size, new_size; log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + + /* Remove dead descriptors */ + microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); orig_size += (int)cache->journal_len; diff --git a/src/or/microdesc.h b/src/or/microdesc.h index b3e12f8..30cb25d 100644 --- a/src/or/microdesc.h +++ b/src/or/microdesc.h @@ -21,6 +21,7 @@ smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, int no_save); +void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force); int microdesc_cache_rebuild(microdesc_cache_t *cache); int microdesc_cache_reload(microdesc_cache_t *cache); void microdesc_cache_clear(microdesc_cache_t *cache);

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.2'
by nickm＠torproject.org 03 May '11

03 May '11

commit 97a55b80894b2816f435a1ee06a2549f0da72939 Merge: 1f97e83 56fbd72 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 16:46:28 2011 -0400 Merge remote-tracking branch 'origin/maint-0.2.2'

1 0

[tor/maint-0.2.2] Backport microdesc_cache_clean to 0.2.2
by nickm＠torproject.org 03 May '11

03 May '11

commit 56fbd728c2df7f189390b9a417b4c3ecc3690ede Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 16:45:15 2011 -0400 Backport microdesc_cache_clean to 0.2.2 Otherwise we have no way to keep authorities' microdesc caches in 0.2.2 from growing without bound. --- changes/bug2230_clean_1 | 4 +++ src/or/microdesc.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++ src/or/microdesc.h | 1 + 3 files changed, 56 insertions(+), 0 deletions(-) diff --git a/changes/bug2230_clean_1 b/changes/bug2230_clean_1 new file mode 100644 index 0000000..a4edf94 --- /dev/null +++ b/changes/bug2230_clean_1 @@ -0,0 +1,4 @@ + o Minor features + - Backport code from 0.2.3.x to allow directory authorities to clean + their microdescriptor caches. + diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 0ceb134..58fbe37 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -23,6 +23,8 @@ struct microdesc_cache_t { tor_mmap_t *cache_content; /** Number of bytes used in the journal file. */ size_t journal_len; + /** Number of bytes in descriptors removed as too old. */ + size_t bytes_dropped; /** Total bytes of microdescriptor bodies we have added to this cache */ uint64_t total_len_seen; @@ -276,6 +278,51 @@ microdesc_cache_reload(microdesc_cache_t *cache) return 0; } +/** By default, we remove any microdescriptors that have gone at least this + * long without appearing in a current consensus. */ +#define TOLERATE_MICRODESC_AGE (7*24*60*60) + +/** Remove all microdescriptors from cache that haven't been listed for + * a long time. Does not rebuild the cache on disk. If cutoff is + * positive, specifically remove microdescriptors that have been unlisted + * since cutoff. If force is true, remove microdescriptors even + * if we have no current live microdescriptor consensus. + */ +void +microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) +{ + microdesc_t **mdp, *victim; + int dropped=0, kept=0; + size_t bytes_dropped = 0; + time_t now = time(NULL); + + (void) force; + /* In 0.2.2, we let this proceed unconditionally: only authorities have + * microdesc caches. */ + + if (cutoff <= 0) + cutoff = now - TOLERATE_MICRODESC_AGE; + + for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) { + if ((*mdp)->last_listed < cutoff) { + ++dropped; + victim = *mdp; + mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp); + bytes_dropped += victim->bodylen; + microdesc_free(victim); + } else { + ++kept; + mdp = HT_NEXT(microdesc_map, &cache->map, mdp); + } + } + + if (dropped) { + log_notice(LD_DIR, "Removed %d/%d microdescriptors as old.", + dropped,dropped+kept); + cache->bytes_dropped += bytes_dropped; + } +} + /** Regenerate the main cache file for cache, clear the journal file, * and update every microdesc_t in the cache with pointers to its new * location. */ @@ -291,6 +338,10 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) int orig_size, new_size; log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + + /* Remove dead descriptors */ + microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); orig_size += (int)cache->journal_len; diff --git a/src/or/microdesc.h b/src/or/microdesc.h index b3e12f8..30cb25d 100644 --- a/src/or/microdesc.h +++ b/src/or/microdesc.h @@ -21,6 +21,7 @@ smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, int no_save); +void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force); int microdesc_cache_rebuild(microdesc_cache_t *cache); int microdesc_cache_reload(microdesc_cache_t *cache); void microdesc_cache_clear(microdesc_cache_t *cache);

1 0

[tor/master] Add missing code to set cache->journal_len when reading microdesc journal
by nickm＠torproject.org 03 May '11

03 May '11

commit 698fa0fc67118bea2ea7720455ebbdee705c8c1a Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 16:22:31 2011 -0400 Add missing code to set cache->journal_len when reading microdesc journal This could be one reason that authorities' journals would grow without bound; related to bug 2230. Bugfix on 0.2.2.6-alpha. Fix by "cypherpunks". --- changes/bug2230_part1 | 7 +++++++ src/or/microdesc.c | 1 + 2 files changed, 8 insertions(+), 0 deletions(-) diff --git a/changes/bug2230_part1 b/changes/bug2230_part1 new file mode 100644 index 0000000..79f7254 --- /dev/null +++ b/changes/bug2230_part1 @@ -0,0 +1,7 @@ + o Minor bugfixes + - When loading the microdesc journal, remember its current size. + In 0.2.2, this helps prevent the microdesc journal from growing + without limit on authorities (who are the only ones to use it in + 0.2.2). Fixes a part of bug 2230; bugfix on 0.2.2.6-alpha. + Fix posted by "cypherpunks." + diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 2c4b343..3566277 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -261,6 +261,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) journal_content = read_file_to_str(cache->journal_fname, RFTS_IGNORE_MISSING, &st); if (journal_content) { + cache->journal_len = (size_t) st.st_size; added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, SAVED_IN_JOURNAL, 0);

1 0

[tor/master] Fix a check for when to rebuild the microdesc cache. (Backport from 0.2.3.
by nickm＠torproject.org 03 May '11

03 May '11

commit 970715dd8f52a79bbaddfcfa5af1f11608ddacf9 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 16:29:39 2011 -0400 Fix a check for when to rebuild the microdesc cache. (Backport from 0.2.3. --- changes/bug2230_part2 | 5 +++++ src/or/microdesc.c | 2 +- 2 files changed, 6 insertions(+), 1 deletions(-) diff --git a/changes/bug2230_part2 b/changes/bug2230_part2 new file mode 100644 index 0000000..2664ecc --- /dev/null +++ b/changes/bug2230_part2 @@ -0,0 +1,5 @@ + o Minor bugfixes + - The microdesc journal is supposed to get rebuilt only if it is + at least _half_ the length of the store, not _twice_ the length + of the store. Bugfix on 0.2.2.6-alpha; fixes part of bug 2230. + diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 3566277..0ceb134 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -208,7 +208,7 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, size_t old_content_len = cache->cache_content ? cache->cache_content->size : 0; if (cache->journal_len > 16384 + old_content_len && - cache->journal_len > old_content_len * 2) { + cache->journal_len > old_content_len / 2) { microdesc_cache_rebuild(cache); } }

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.2'
by nickm＠torproject.org 03 May '11

03 May '11

commit 1f97e8322fe040066c630073a71a59ff0c161beb Merge: 5b96773 970715d Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 16:33:41 2011 -0400 Merge remote-tracking branch 'origin/maint-0.2.2' Conflicts: src/or/microdesc.c changes/bug2230_part1 | 7 +++++++ changes/bug2230_part2 | 5 +++++ src/or/microdesc.c | 6 ++++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --cc src/or/microdesc.c index 469c2fc,0ceb134..521ee07 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@@ -246,17 -207,12 +246,18 @@@ microdescs_add_list_to_cache(microdesc_ { size_t old_content_len = cache->cache_content ? cache->cache_content->size : 0; - if ((cache->journal_len > 16384 + old_content_len && - cache->journal_len > old_content_len / 2)) + if (cache->journal_len > 16384 + old_content_len && + cache->journal_len > old_content_len / 2) { microdesc_cache_rebuild(cache); + } } + { + networkstatus_t *ns = networkstatus_get_latest_consensus(); + if (ns && ns->flavor == FLAV_MICRODESC) + SMARTLIST_FOREACH(added, microdesc_t *, md, nodelist_add_microdesc(md)); + } + return added; } @@@ -305,9 -261,10 +306,10 @@@ microdesc_cache_reload(microdesc_cache_ journal_content = read_file_to_str(cache->journal_fname, RFTS_IGNORE_MISSING, &st); if (journal_content) { + cache->journal_len = (size_t) st.st_size; added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, - SAVED_IN_JOURNAL, 0); + SAVED_IN_JOURNAL, 0, -1, NULL); if (added) { total += smartlist_len(added); smartlist_free(added);

1 0

[tor/maint-0.2.2] Add missing code to set cache->journal_len when reading microdesc journal
by nickm＠torproject.org 03 May '11

03 May '11

commit 698fa0fc67118bea2ea7720455ebbdee705c8c1a Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 16:22:31 2011 -0400 Add missing code to set cache->journal_len when reading microdesc journal This could be one reason that authorities' journals would grow without bound; related to bug 2230. Bugfix on 0.2.2.6-alpha. Fix by "cypherpunks". --- changes/bug2230_part1 | 7 +++++++ src/or/microdesc.c | 1 + 2 files changed, 8 insertions(+), 0 deletions(-) diff --git a/changes/bug2230_part1 b/changes/bug2230_part1 new file mode 100644 index 0000000..79f7254 --- /dev/null +++ b/changes/bug2230_part1 @@ -0,0 +1,7 @@ + o Minor bugfixes + - When loading the microdesc journal, remember its current size. + In 0.2.2, this helps prevent the microdesc journal from growing + without limit on authorities (who are the only ones to use it in + 0.2.2). Fixes a part of bug 2230; bugfix on 0.2.2.6-alpha. + Fix posted by "cypherpunks." + diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 2c4b343..3566277 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -261,6 +261,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) journal_content = read_file_to_str(cache->journal_fname, RFTS_IGNORE_MISSING, &st); if (journal_content) { + cache->journal_len = (size_t) st.st_size; added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, SAVED_IN_JOURNAL, 0);

1 0

[arm/master] Adding procName util to readme and package
by atagar＠torproject.org 03 May '11

03 May '11

commit 908acf3170058fc4deacba1fc2f82549d2d54c02 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue May 3 09:49:42 2011 -0700 Adding procName util to readme and package --- README | 1 + src/util/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletions(-) diff --git a/README b/README index 50657a1..ff03b37 100644 --- a/README +++ b/README @@ -172,6 +172,7 @@ Layout: log.py - aggregator for application events panel.py - wrapper for safely working with curses subwindows procTools.py - queries process & system information from /proc contents + procName.py - renames our process to a friendlier name sysTools.py - helper for system calls, providing client side caching torConfig.py - functions for working with the torrc and config options torTools.py - TorCtl wrapper, providing caching and derived information diff --git a/src/util/__init__.py b/src/util/__init__.py index dd65242..365e82a 100644 --- a/src/util/__init__.py +++ b/src/util/__init__.py @@ -4,5 +4,5 @@ application's status, making cross platform system calls, parsing tor data, and safely working with curses (hiding some of the gory details). """ -__all__ = ["conf", "connections", "enum", "hostnames", "log", "panel", "procTools", "sysTools", "torConfig", "torTools", "uiTools"] +__all__ = ["conf", "connections", "enum", "hostnames", "log", "panel", "procTools", "procName", "sysTools", "torConfig", "torTools", "uiTools"]

1 0

[arm/master] Renaming procname to procName
by atagar＠torproject.org 03 May '11

03 May '11

commit 6c07b110c6b9b0d2285a5875d98647abaf8d67b8 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue May 3 09:47:52 2011 -0700 Renaming procname to procName The rest of the utils follow a camel-case convention so this one should too. --- src/starter.py | 4 +- src/util/procName.py | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/util/procname.py | 115 -------------------------------------------------- 3 files changed, 117 insertions(+), 117 deletions(-) diff --git a/src/starter.py b/src/starter.py index 5b73a73..be97d0e 100644 --- a/src/starter.py +++ b/src/starter.py @@ -391,8 +391,8 @@ if __name__ == '__main__': # "arm <input args>" try: - from util import procname - procname.renameProcess("arm\0%s" % "\0".join(sys.argv[1:])) + from util import procName + procName.renameProcess("arm\0%s" % "\0".join(sys.argv[1:])) except: pass cli.controller.startTorMonitor(time.time() - initTime, expandedEvents, param["startup.blindModeEnabled"]) diff --git a/src/util/procName.py b/src/util/procName.py new file mode 100644 index 0000000..97dc707 --- /dev/null +++ b/src/util/procName.py @@ -0,0 +1,115 @@ +# Module to allow for arbitrary renaming of our python process. This is mostly +# based on: +# http://www.rhinocerus.net/forum/lang-python/569677-setting-program-name-lik… +# and an adaptation by Jake: https://github.com/ioerror/chameleon +# +# A cleaner implementation is available at: +# https://github.com/cream/libs/blob/b38970e2a6f6d2620724c828808235be0445b799… +# but I'm not quite clear on their implementation, and it only does targeted +# argument replacement (ie, replace argv[0], argv[1], etc but with a string +# the same size). + +import sys +import ctypes +import ctypes.util + +# flag for setting the process name, found in '/usr/include/linux/prctl.h' +PR_SET_NAME = 15 + +argc_t = ctypes.POINTER(ctypes.c_char_p) + +Py_GetArgcArgv = ctypes.pythonapi.Py_GetArgcArgv +Py_GetArgcArgv.restype = None +Py_GetArgcArgv.argtypes = [ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(argc_t)] + +# tracks the last name we've changed the process to +currentProcessName = None +maxNameLength = -1 + +def renameProcess(processName): + """ + Renames our current process from "python <args>" to a custom name. + + Arguments: + processName - new name for our process + """ + + _setArgv(processName) + if sys.platform == "linux2": + _setPrctlName(processName) + elif sys.platform == "freebsd7": + _setProcTitle(processName) + +def _setArgv(processName): + """ + Overwrites our argv in a similar fashion to how it's done in C with: + strcpy(argv[0], "new_name"); + """ + + global currentProcessName, maxNameLength + + argv = ctypes.c_int(0) + argc = argc_t() + Py_GetArgcArgv(argv, ctypes.pointer(argc)) + + # The original author did the memset for 256, while Jake did it for the + # processName length (capped at 1608). I'm not sure of the reasons for + # either of these limits, but setting it to anything higher than than the + # length of the null terminated process name should be pointless, so opting + # for Jake's implementation on this. + + if currentProcessName == None: + # Getting argv via... + # currentProcessName = " ".join(["python"] + sys.argv) + # + # doesn't do the trick since this will miss interpretor arguments like... + # python -W ignore::DeprecationWarning myScript.py + # + # Hence we're fetching this via our ctypes argv. Alternatively we could + # use ps, though this is less desirable: + # "ps -p %i -o args" % os.getpid() + + args = [] + for i in range(100): + if argc[i] == None: break + args.append(str(argc[i])) + + currentProcessName = " ".join(args) + maxNameLength = len(currentProcessName) + + if len(processName) > maxNameLength: + msg = "can't rename process to something longer than our initial name since this would overwrite memory used for the env" + raise IOError(msg) + + # space we need to clear + zeroSize = max(len(currentProcessName), len(processName)) + + ctypes.memset(argc.contents, 0, zeroSize + 1) # null terminate the string's end + ctypes.memmove(argc.contents, processName, len(processName)) + currentProcessName = processName + +def _setPrctlName(processName): + """ + Sets the prctl name, which is used by top and killall. This appears to be + Linux specific and has the max of 15 characters. Source: + http://stackoverflow.com/questions/564695/is-there-a-way-to-change-effectiv… + """ + + libc = ctypes.CDLL(ctypes.util.find_library("c")) + nameBuffer = ctypes.create_string_buffer(len(processName)+1) + nameBuffer.value = processName + libc.prctl(PR_SET_NAME, ctypes.byref(nameBuffer), 0, 0, 0) + +def _setProcTitle(processName): + """ + BSD specific calls (should be compataible with both FreeBSD and OpenBSD: + http://fxr.watson.org/fxr/source/gen/setproctitle.c?v=FREEBSD-LIBC + http://www.rootr.net/man/man/setproctitle/3 + """ + + libc = ctypes.CDLL(ctypes.util.find_library("c")) + nameBuffer = ctypes.create_string_buffer(len(processName)+1) + nameBuffer.value = processName + libc.setproctitle(ctypes.byref(nameBuffer)) + diff --git a/src/util/procname.py b/src/util/procname.py deleted file mode 100644 index 97dc707..0000000 --- a/src/util/procname.py +++ /dev/null @@ -1,115 +0,0 @@ -# Module to allow for arbitrary renaming of our python process. This is mostly -# based on: -# http://www.rhinocerus.net/forum/lang-python/569677-setting-program-name-lik… -# and an adaptation by Jake: https://github.com/ioerror/chameleon -# -# A cleaner implementation is available at: -# https://github.com/cream/libs/blob/b38970e2a6f6d2620724c828808235be0445b799… -# but I'm not quite clear on their implementation, and it only does targeted -# argument replacement (ie, replace argv[0], argv[1], etc but with a string -# the same size). - -import sys -import ctypes -import ctypes.util - -# flag for setting the process name, found in '/usr/include/linux/prctl.h' -PR_SET_NAME = 15 - -argc_t = ctypes.POINTER(ctypes.c_char_p) - -Py_GetArgcArgv = ctypes.pythonapi.Py_GetArgcArgv -Py_GetArgcArgv.restype = None -Py_GetArgcArgv.argtypes = [ctypes.POINTER(ctypes.c_int), - ctypes.POINTER(argc_t)] - -# tracks the last name we've changed the process to -currentProcessName = None -maxNameLength = -1 - -def renameProcess(processName): - """ - Renames our current process from "python <args>" to a custom name. - - Arguments: - processName - new name for our process - """ - - _setArgv(processName) - if sys.platform == "linux2": - _setPrctlName(processName) - elif sys.platform == "freebsd7": - _setProcTitle(processName) - -def _setArgv(processName): - """ - Overwrites our argv in a similar fashion to how it's done in C with: - strcpy(argv[0], "new_name"); - """ - - global currentProcessName, maxNameLength - - argv = ctypes.c_int(0) - argc = argc_t() - Py_GetArgcArgv(argv, ctypes.pointer(argc)) - - # The original author did the memset for 256, while Jake did it for the - # processName length (capped at 1608). I'm not sure of the reasons for - # either of these limits, but setting it to anything higher than than the - # length of the null terminated process name should be pointless, so opting - # for Jake's implementation on this. - - if currentProcessName == None: - # Getting argv via... - # currentProcessName = " ".join(["python"] + sys.argv) - # - # doesn't do the trick since this will miss interpretor arguments like... - # python -W ignore::DeprecationWarning myScript.py - # - # Hence we're fetching this via our ctypes argv. Alternatively we could - # use ps, though this is less desirable: - # "ps -p %i -o args" % os.getpid() - - args = [] - for i in range(100): - if argc[i] == None: break - args.append(str(argc[i])) - - currentProcessName = " ".join(args) - maxNameLength = len(currentProcessName) - - if len(processName) > maxNameLength: - msg = "can't rename process to something longer than our initial name since this would overwrite memory used for the env" - raise IOError(msg) - - # space we need to clear - zeroSize = max(len(currentProcessName), len(processName)) - - ctypes.memset(argc.contents, 0, zeroSize + 1) # null terminate the string's end - ctypes.memmove(argc.contents, processName, len(processName)) - currentProcessName = processName - -def _setPrctlName(processName): - """ - Sets the prctl name, which is used by top and killall. This appears to be - Linux specific and has the max of 15 characters. Source: - http://stackoverflow.com/questions/564695/is-there-a-way-to-change-effectiv… - """ - - libc = ctypes.CDLL(ctypes.util.find_library("c")) - nameBuffer = ctypes.create_string_buffer(len(processName)+1) - nameBuffer.value = processName - libc.prctl(PR_SET_NAME, ctypes.byref(nameBuffer), 0, 0, 0) - -def _setProcTitle(processName): - """ - BSD specific calls (should be compataible with both FreeBSD and OpenBSD: - http://fxr.watson.org/fxr/source/gen/setproctitle.c?v=FREEBSD-LIBC - http://www.rootr.net/man/man/setproctitle/3 - """ - - libc = ctypes.CDLL(ctypes.util.find_library("c")) - nameBuffer = ctypes.create_string_buffer(len(processName)+1) - nameBuffer.value = processName - libc.setproctitle(ctypes.byref(nameBuffer)) -

1 0