May 2011 - tor-commits - lists.torproject.org

[arm/master] Moving torrc sighup reloads to the torrc panel
by atagar＠torproject.org 14 May '11

14 May '11

commit 7fbbbad2e682d617157bd931ce18c1bcd3504ead Author: Damian Johnson <atagar(a)torproject.org> Date: Fri May 13 19:29:00 2011 -0700 Moving torrc sighup reloads to the torrc panel Using the new style listeners for torrc sighup listening rather than the old sighup tracker. --- src/cli/controller.py | 9 --------- src/cli/torrcPanel.py | 20 +++++++++++++++++++- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/cli/controller.py b/src/cli/controller.py index 5696daa..63f62e5 100644 --- a/src/cli/controller.py +++ b/src/cli/controller.py @@ -432,15 +432,6 @@ def drawTorMonitor(stdscr, startTime, loggedEvents, isBlindMode): if panels["graph"].currentDisplay == "bandwidth": panels["graph"].setHeight(panels["graph"].stats["bandwidth"].getContentHeight()) - # TODO: should redraw the torrcPanel - #panels["torrc"].loadConfig() - - # reload the torrc if it's previously been loaded - if loadedTorrc.isLoaded(): - try: - loadedTorrc.load(True) - if page == 3: panels["torrc"].redraw(True) - except: pass sighupTracker.isReset = False diff --git a/src/cli/torrcPanel.py b/src/cli/torrcPanel.py index af6f4d6..e14a16a 100644 --- a/src/cli/torrcPanel.py +++ b/src/cli/torrcPanel.py @@ -8,7 +8,7 @@ import threading import popups -from util import conf, enum, panel, torConfig, uiTools +from util import conf, enum, panel, torConfig, torTools, uiTools DEFAULT_CONFIG = {"features.config.file.showScrollbars": True, "features.config.file.maxLinesPerEntry": 8} @@ -40,6 +40,24 @@ class TorrcPanel(panel.Panel): # _lastContentHeightArgs is None or differs from the current dimensions) self._lastContentHeight = 1 self._lastContentHeightArgs = None + + # listens for tor reload (sighup) events + torTools.getConn().addStatusListener(self.resetListener) + + def resetListener(self, conn, eventType): + """ + Reloads and displays the torrc on tor reload (sighup) events. + + Arguments: + conn - tor controller + eventType - type of event detected + """ + + if eventType == torTools.State.INIT: + try: + torConfig.getTorrc().load(True) + self.redraw(True) + except: pass def handleKey(self, key): self.valsLock.acquire()

1 0

[arm/master] Expanding connection line selection by a space
by atagar＠torproject.org 14 May '11

14 May '11

commit 860511ce71e2ad947014271bd2517624fb38289e Author: Damian Johnson <atagar(a)torproject.org> Date: Fri May 13 18:21:43 2011 -0700 Expanding connection line selection by a space The connection panel looks a little better if selection hilighting stretches all the way to the left window edge. --- src/cli/connections/circEntry.py | 4 +++- src/cli/connections/connEntry.py | 6 ++++-- src/cli/connections/connPanel.py | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/cli/connections/circEntry.py b/src/cli/connections/circEntry.py index b15b26a..82ef0ff 100644 --- a/src/cli/connections/circEntry.py +++ b/src/cli/connections/circEntry.py @@ -177,13 +177,14 @@ class CircLine(connEntry.ConnectionLine): lineFormat = uiTools.getColor(connEntry.CATEGORY_COLOR[self.getType()]) # The required widths are the sum of the following: + # initial space (1 character) # bracketing (3 characters) # placementLabel (14 characters) # gap between etc and placement label (5 characters) if self.isLast: bracket = (curses.ACS_LLCORNER, curses.ACS_HLINE, ord(' ')) else: bracket = (curses.ACS_VLINE, ord(' '), ord(' ')) - baselineSpace = len(bracket) + 14 + 5 + baselineSpace = len(bracket) + 14 + 5 + 1 dst, etc = "", "" if listingType == entries.ListingType.IP_ADDRESS: @@ -212,5 +213,6 @@ class CircLine(connEntry.ConnectionLine): drawEntry = uiTools.DrawEntry(" " * (width - baselineSpace - len(dst) - len(etc) + 5), lineFormat, drawEntry) drawEntry = uiTools.DrawEntry(dst + etc, lineFormat, drawEntry) drawEntry = uiTools.DrawEntry(bracket, curses.A_NORMAL, drawEntry, lockFormat = True) + drawEntry = uiTools.DrawEntry(" ", curses.A_NORMAL, drawEntry, lockFormat = True) return drawEntry diff --git a/src/cli/connections/connEntry.py b/src/cli/connections/connEntry.py index ac45656..05e2a45 100644 --- a/src/cli/connections/connEntry.py +++ b/src/cli/connections/connEntry.py @@ -288,7 +288,7 @@ class ConnectionLine(entries.ConnectionPanelLine): timePrefix = "+" if self.isInitialConnection else " " else: timePrefix = "" - timeEntry = myListing.getNext() + timeEntry = myListing.getNext().getNext() timeEntry.text = timePrefix + "%5s" % uiTools.getTimeLabel(currentTime - self.startTime, 1) return myListing @@ -304,6 +304,7 @@ class ConnectionLine(entries.ConnectionPanelLine): entryType = self.getType() # Lines are split into the following components in reverse: + # init gap - " " # content - "<src> --> <dst> <etc> " # time - "<uptime>" # preType - " (" @@ -317,7 +318,8 @@ class ConnectionLine(entries.ConnectionPanelLine): drawEntry = uiTools.DrawEntry(entryType.upper(), lineFormat | curses.A_BOLD, drawEntry) drawEntry = uiTools.DrawEntry(" (", lineFormat, drawEntry) drawEntry = uiTools.DrawEntry(" " * timeWidth, lineFormat, drawEntry) - drawEntry = uiTools.DrawEntry(self._getListingContent(width - (12 + timeWidth), listingType), lineFormat, drawEntry) + drawEntry = uiTools.DrawEntry(self._getListingContent(width - (12 + timeWidth) - 1, listingType), lineFormat, drawEntry) + drawEntry = uiTools.DrawEntry(" ", lineFormat, drawEntry) return drawEntry def _getDetails(self, width): diff --git a/src/cli/connections/connPanel.py b/src/cli/connections/connPanel.py index 7fab33e..45750c3 100644 --- a/src/cli/connections/connPanel.py +++ b/src/cli/connections/connPanel.py @@ -278,9 +278,9 @@ class ConnectionPanel(panel.Panel, threading.Thread): title = "Connection Details:" if self._showDetails else self._title self.addstr(0, 0, title, curses.A_STANDOUT) - scrollOffset = 1 + scrollOffset = 0 if isScrollbarVisible: - scrollOffset = 3 + scrollOffset = 2 self.addScrollBar(scrollLoc, scrollLoc + height - detailPanelOffset - 1, len(self._entryLines), 1 + detailPanelOffset) if self.isPaused() or not self._isTorRunning:

1 0

[arm/master] Banishing addstr_wrap from the controller
by atagar＠torproject.org 14 May '11

14 May '11

commit e1c3ca099effe77b00c7ad86872598e54f66aa51 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri May 13 18:29:16 2011 -0700 Banishing addstr_wrap from the controller The descriptorPopup is destined to be dropped from the codebase once I write the controller prompt, and this is also the last usage of the deprecated addstr_wrap function. Rather than rewriting the popup to use an alternative I'm just puting all the soon-to-be-dead code to one place. --- src/cli/controller.py | 29 ----------------------------- src/cli/descriptorPopup.py | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/cli/controller.py b/src/cli/controller.py index 5927e17..eb1df4f 100644 --- a/src/cli/controller.py +++ b/src/cli/controller.py @@ -182,35 +182,6 @@ class ControlPanel(panel.Panel): self.addstr(0, xLoc + 1, " " * barProgress, curses.A_STANDOUT | uiTools.getColor("red")) self.addstr(0, xLoc + barWidth + 1, "]", curses.A_BOLD) -def addstr_wrap(panel, y, x, text, formatting, startX = 0, endX = -1, maxY = -1): - """ - Writes text with word wrapping, returning the ending y/x coordinate. - y: starting write line - x: column offset from startX - text / formatting: content to be written - startX / endX: column bounds in which text may be written - """ - - # moved out of panel (trying not to polute new code!) - # TODO: unpleaseantly complex usage - replace with something else when - # rewriting confPanel and descriptorPopup (the only places this is used) - if not text: return (y, x) # nothing to write - if endX == -1: endX = panel.maxX # defaults to writing to end of panel - if maxY == -1: maxY = panel.maxY + 1 # defaults to writing to bottom of panel - lineWidth = endX - startX # room for text - while True: - if len(text) > lineWidth - x - 1: - chunkSize = text.rfind(" ", 0, lineWidth - x) - writeText = text[:chunkSize] - text = text[chunkSize:].strip() - - panel.addstr(y, x + startX, writeText, formatting) - y, x = y + 1, 0 - if y >= maxY: return (y, x) - else: - panel.addstr(y, x + startX, text, formatting) - return (y, x + len(text)) - class sighupListener(TorCtl.PostEventListener): """ Listens for reload signal (hup), which is produced by: diff --git a/src/cli/descriptorPopup.py b/src/cli/descriptorPopup.py index 920f2bc..2a51905 100644 --- a/src/cli/descriptorPopup.py +++ b/src/cli/descriptorPopup.py @@ -22,6 +22,35 @@ SIG_END_KEYS = ["-----END RSA PUBLIC KEY-----", "-----END SIGNATURE-----"] UNRESOLVED_MSG = "No consensus data available" ERROR_MSG = "Unable to retrieve data" +def addstr_wrap(panel, y, x, text, formatting, startX = 0, endX = -1, maxY = -1): + """ + Writes text with word wrapping, returning the ending y/x coordinate. + y: starting write line + x: column offset from startX + text / formatting: content to be written + startX / endX: column bounds in which text may be written + """ + + # moved out of panel (trying not to polute new code!) + # TODO: unpleaseantly complex usage - replace with something else when + # rewriting confPanel and descriptorPopup (the only places this is used) + if not text: return (y, x) # nothing to write + if endX == -1: endX = panel.maxX # defaults to writing to end of panel + if maxY == -1: maxY = panel.maxY + 1 # defaults to writing to bottom of panel + lineWidth = endX - startX # room for text + while True: + if len(text) > lineWidth - x - 1: + chunkSize = text.rfind(" ", 0, lineWidth - x) + writeText = text[:chunkSize] + text = text[chunkSize:].strip() + + panel.addstr(y, x + startX, writeText, formatting) + y, x = y + 1, 0 + if y >= maxY: return (y, x) + else: + panel.addstr(y, x + startX, text, formatting) + return (y, x + len(text)) + class PopupProperties: """ State attributes of popup window for consensus descriptions. @@ -176,8 +205,8 @@ def draw(popup, properties): keyword, remainder = lineText, "" keywordFormat = uiTools.getColor(SIG_COLOR) - lineNum, xLoc = controller.addstr_wrap(popup, lineNum, 0, keyword, keywordFormat, xOffset + numOffset, popup.maxX - 1, popup.maxY - 1) - lineNum, xLoc = controller.addstr_wrap(popup, lineNum, xLoc, remainder, remainderFormat, xOffset + numOffset, popup.maxX - 1, popup.maxY - 1) + lineNum, xLoc = addstr_wrap(popup, lineNum, 0, keyword, keywordFormat, xOffset + numOffset, popup.maxX - 1, popup.maxY - 1) + lineNum, xLoc = addstr_wrap(popup, lineNum, xLoc, remainder, remainderFormat, xOffset + numOffset, popup.maxX - 1, popup.maxY - 1) lineNum += 1 if lineNum > pageHeight: break

1 0

r24749: {projects} And of course there was a typo. (projects/articles/browser-privacy)
by Mike Perry 13 May '11

13 May '11

Author: mikeperry Date: 2011-05-13 22:30:01 +0000 (Fri, 13 May 2011) New Revision: 24749 Modified: projects/articles/browser-privacy/W3CIdentity-Tor-final.pdf projects/articles/browser-privacy/W3CIdentity.tex Log: And of course there was a typo. Modified: projects/articles/browser-privacy/W3CIdentity-Tor-final.pdf =================================================================== (Binary files differ) Modified: projects/articles/browser-privacy/W3CIdentity.tex =================================================================== --- projects/articles/browser-privacy/W3CIdentity.tex 2011-05-13 20:50:21 UTC (rev 24748) +++ projects/articles/browser-privacy/W3CIdentity.tex 2011-05-13 22:30:01 UTC (rev 24749) @@ -386,7 +386,7 @@ makes it impossible to expect privacy-by-design approaches to function properly. We cannot expect users who do not even understand the basic properties of these tracking mechanisms to effectively use privacy mechanisms -to avoid, opt out of, or decline such tracking. Therefore, browser inteface +to avoid, opt out of, or decline such tracking. Therefore, browser interface and behavior must be brought in line with user expectations. \bibliographystyle{plain} \bibliography{W3CIdentity}

1 0

r24748: {} Add final submission PDF. (projects/articles/browser-privacy)
by Mike Perry 13 May '11

13 May '11

Author: mikeperry Date: 2011-05-13 20:50:21 +0000 (Fri, 13 May 2011) New Revision: 24748 Added: projects/articles/browser-privacy/W3CIdentity-Tor-final.pdf Log: Add final submission PDF. Added: projects/articles/browser-privacy/W3CIdentity-Tor-final.pdf =================================================================== (Binary files differ) Property changes on: projects/articles/browser-privacy/W3CIdentity-Tor-final.pdf ___________________________________________________________________ Added: svn:mime-type + application/octet-stream

1 0

r24747: {projects} Add a sentence to the conclusion, tighten some other languag (projects/articles/browser-privacy)
by Mike Perry 13 May '11

13 May '11

Author: mikeperry Date: 2011-05-13 20:47:03 +0000 (Fri, 13 May 2011) New Revision: 24747 Modified: projects/articles/browser-privacy/W3CIdentity.tex Log: Add a sentence to the conclusion, tighten some other language for space. Modified: projects/articles/browser-privacy/W3CIdentity.tex =================================================================== --- projects/articles/browser-privacy/W3CIdentity.tex 2011-05-13 16:10:10 UTC (rev 24746) +++ projects/articles/browser-privacy/W3CIdentity.tex 2011-05-13 20:47:03 UTC (rev 24747) @@ -83,7 +83,7 @@ Both of these directions must be pursued to provide users with the ability to properly use the web in a privacy-preserving way. -\footnotetext{We only consider implementations that involve privacy-by-design. +\footnotetext{We only consider privacy-by-design approaches. Privacy-by-policy approaches such as Do Not Track will not be discussed.} \section{User Privacy on the Web} @@ -122,13 +122,12 @@ pages. The default experience is such that all of this data exchange is concealed from the user. -So then what comprises a user's web identity for tracking purposes? In terms -of authentication, it would at first appear to be limited to cookies, HTTP -Auth tokens, and client TLS certificates. However, this identifier-based -approach breaks down quickly on the modern web. High-security websites are -already using fingerprinting as an auxiliary second factor of -authentication~\cite{security-fingerprinting}, and online data aggregators -utilize everything they can to build complete portraits of users' +So then what comprises a user's web identity for tracking purposes? At first +glance, it appears to be limited to cookies, HTTP Auth tokens, and client TLS +certificates. However, this identifier-based approach breaks down quickly on +the modern web. High-security websites are already using fingerprinting as a +second factor of authentication~\cite{security-fingerprinting}, and data +aggregators utilize everything they can to build complete portraits of users' identities~\cite{tracking-identity}. Despite what the user may believe, her actual web identity then is a superset @@ -150,7 +149,7 @@ When privacy is expanded to cover all items that enable or substantially contribute to linkability, a lot more components of the browser are now in -scope. We will briefly enumerate these four categories of components. +scope. We will briefly enumerate these categories of components. First, the obvious properties are found in the state of the browser: cookies, DOM storage, cache, cryptographic tokens and cryptographic state, and @@ -182,23 +181,21 @@ distribution of each of several key attributes to determine how many bits of identifying information each attribute provided. -While not perfect\footnotemark, this metric allows us to prioritize our efforts -on the -components that have the most potential for linkability. +While not perfect\footnotemark, this metric allows us to prioritize our +efforts on the components that have the most potential for linkability. % -\footnotetext{In particular, the test does not take in all aspects of +\footnotetext{In particular, Panopticlick did not measure all aspects of resolution information. It did not calculate the size of widgets, window decoration, or toolbar size. We believe these resolution-related properties -may add high amounts of entropy to the resolution component. They also did not +may add high amounts of entropy to the resolution component. It also did not measure clock offset and other time-based fingerprints. Furthermore, as new -browser features are added, the experiment should be repeated to include +browser features are added, the experiment should be repeated to measure them.} % -It also shows the benefits of standardizing on -implementations of fingerprinting resistance where possible. More -implementations using the same defenses means more users with similar -fingerprints, which means less entropy in the metric. Similarly, uniform -feature deployment leads to less entropy. +It also shows the benefits of standardizing on implementations of +fingerprinting resistance where possible. More implementations using the same +defenses means more users with similar fingerprints, which means less entropy +in the metric. Similarly, uniform feature deployment leads to less entropy. \section{Matching User Perception with Reality} @@ -222,15 +219,14 @@ indicate that their current set of accumulated linkable state comprise a single, trackable web identity that can be changed or cleared. -We believe that the user interface of the browser should convey a sense of -persistent identity prominently to the user in the form of a visual cue. This -cue can either be an abstract image, graphic or theme (such as the user's -choice of Firefox Persona~\cite{firefox-personas}), or it can be a text area -with the user's current pseudonym. This idea of identity should then be -integrated with the browsing experience. Users should be able to click a -button to get a clean slate for a new identity, and should be able to log into -and out of password-protected stored identities, which would contain the -entire state of the browser. +We believe that the browser UI should convey a sense of persistent identity +prominently to the user in the form of a visual cue. This cue can either be an +image, graphic or theme (such as the user's choice of Firefox +Persona~\cite{firefox-personas}), or it can be a text area with the user's +pseudonym. This idea of identity should then be integrated with the browsing +experience. Users should be able to click a button to get a clean slate for a +new identity, and should be able to log into and out of password-protected +stored identities, which would contain the entire state of the browser. To this user, the Private Browsing Mode would be no more than a special case of this identity UI---a special identity that they can trust not to store @@ -351,9 +347,9 @@ window could have a user-intuitive way of representing the user's relationship with different top-level origins, perhaps by using only the `favicon' of that top-level origin to represent all of the browser state accumulated by that -origin. The user could delete the entire set of browser state (cookies, cache, -storage, cryptographic tokens) associated with a site simply by removing its -favicon from her privacy info panel. +origin. The user could delete the entire set of top-level origin state +(cookies, cache, storage, cryptographic tokens) simply by removing the favicon +from her privacy info panel. Linkability based on fingerprintable browser properties is also amenable to improvement under this model. In particular, one can imagine per-origin plugin @@ -364,9 +360,8 @@ of the web closer to what the user assumes is happening, they must be deployed uniformly, with a consistent top-level origin restriction model. Uniform deployment will take significant coordination and standardization efforts. -Furthermore, even a vastly improved origin model cannot prevent -instances of explicit tracking partnerships between sites and third-party -content providers. Therefore, both origin improvements and +Furthermore, even an improved origin model cannot protect against large +multi-service top-level origins. Therefore, both origin improvements and identity-isolation approaches are necessary. \section{Conclusions} @@ -378,20 +373,21 @@ is so extreme that it raises moral issues about the level of consent actually involved in web use and associated tracking. -In fact, standardization efforts seemed to realize this problem early on but -failed to create feasible recommendations for improving the situation. -RFC~2965 governing HTTP State Management mandated in Section~3.3.6 that -third-party origins must not cause the browser to transmit cookies unless the -interaction is ``verifiable'' and readily apparent to the user~\cite{rfc2965}. -In Section~6, it also strongly suggested that informed consent and user -control should govern the interaction of users to tracking identifiers. +In fact, standardization efforts realized this problem early on but failed to +create feasible recommendations for improving the situation. RFC~2965 +governing HTTP State Management mandated in Section~3.3.6 that third-party +origins must not cause the browser to transmit cookies unless the interaction +is ``verifiable'' and readily apparent to the user~\cite{rfc2965}. In +Section~6, it also strongly suggested that informed consent and user control +should govern the interaction of users to tracking identifiers. Without changes to both browser behavior and browser interface, informed consent is simply not possible on today's web. The lack of informed consent makes it impossible to expect privacy-by-design approaches to function properly. We cannot expect users who do not even understand the basic -properties of these tracking mechanisms to effectively -use privacy mechanisms to avoid, opt out of, or decline such tracking. +properties of these tracking mechanisms to effectively use privacy mechanisms +to avoid, opt out of, or decline such tracking. Therefore, browser inteface +and behavior must be brought in line with user expectations. \bibliographystyle{plain} \bibliography{W3CIdentity}

1 0

[arm/master] fix: Control message lagging being state
by atagar＠torproject.org 13 May '11

13 May '11

commit 6a22d0c090846fe2ffb3dbb5a971cd6831d79bfb Author: Damian Johnson <atagar(a)torproject.org> Date: Fri May 13 07:35:13 2011 -0700 fix: Control message lagging being state The control panel is the last of the old-style panels and doesn't yet support non-forced redraws. --- src/cli/controller.py | 6 +++++- src/util/procName.py | 22 ++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/cli/controller.py b/src/cli/controller.py index 51b64c7..5927e17 100644 --- a/src/cli/controller.py +++ b/src/cli/controller.py @@ -586,7 +586,11 @@ def drawTorMonitor(stdscr, startTime, loggedEvents, isBlindMode): newSize = stdscr.getmaxyx() isResize = lastSize != newSize lastSize = newSize - panels[panelKey].redraw(isResize) + + if panelKey != "control": + panels[panelKey].redraw(isResize) + else: + panels[panelKey].redraw(True) stdscr.refresh() diff --git a/src/util/procName.py b/src/util/procName.py index 97dc707..fab0b1e 100644 --- a/src/util/procName.py +++ b/src/util/procName.py @@ -1,13 +1,15 @@ -# Module to allow for arbitrary renaming of our python process. This is mostly -# based on: -# http://www.rhinocerus.net/forum/lang-python/569677-setting-program-name-lik… -# and an adaptation by Jake: https://github.com/ioerror/chameleon -# -# A cleaner implementation is available at: -# https://github.com/cream/libs/blob/b38970e2a6f6d2620724c828808235be0445b799… -# but I'm not quite clear on their implementation, and it only does targeted -# argument replacement (ie, replace argv[0], argv[1], etc but with a string -# the same size). +""" +Module to allow for arbitrary renaming of our python process. This is mostly +based on: +http://www.rhinocerus.net/forum/lang-python/569677-setting-program-name-like-0-perl.html#post2272369 +and an adaptation by Jake: https://github.com/ioerror/chameleon + +A cleaner implementation is available at: +https://github.com/cream/libs/blob/b38970e2a6f6d2620724c828808235be0445b799/cream/util/procname.py +but I'm not quite clear on their implementation, and it only does targeted +argument replacement (ie, replace argv[0], argv[1], etc but with a string +the same size). +""" import sys import ctypes

1 0

[arm/master] fix: Log panel busy wait during curses lock
by atagar＠torproject.org 13 May '11

13 May '11

commit 6f46b55a97e67922d5d6aa1b1ec8340e5085a406 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri May 13 08:17:13 2011 -0700 fix: Log panel busy wait during curses lock If we registered events while the curses lock was held (for instance with a popup or prompt for user input) then arm would peg the cpu. The reason for this was that the log panel was attempting to redraw itself, but not registering any of the attempts since they all failed so it became a busy wait. --- src/cli/logPanel.py | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diff --git a/src/cli/logPanel.py b/src/cli/logPanel.py index 34bf759..6d54ab4 100644 --- a/src/cli/logPanel.py +++ b/src/cli/logPanel.py @@ -1079,6 +1079,10 @@ class LogPanel(panel.Panel, threading.Thread): else: lastDay = currentDay self.redraw(True) + + # makes sure that we register this as an update, otherwise lacking the + # curses lock can cause a busy wait here + self._lastUpdate = time.time() def stop(self): """

1 0

r24746: {website} Asking org preference on the GSoC app template to help with (website/trunk/about/en)
by Damian Johnson 13 May '11

13 May '11

Author: atagar Date: 2011-05-13 16:10:10 +0000 (Fri, 13 May 2011) New Revision: 24746 Modified: website/trunk/about/en/gsoc.wml Log: Asking org preference on the GSoC app template to help with deduplication next year Modified: website/trunk/about/en/gsoc.wml =================================================================== --- website/trunk/about/en/gsoc.wml 2011-05-13 15:51:39 UTC (rev 24745) +++ website/trunk/about/en/gsoc.wml 2011-05-13 16:10:10 UTC (rev 24746) @@ -166,6 +166,11 @@ Interacting with us on IRC will help us get to know you, and help you get to know our community.</li> + <li>Are you applying to other projects for GSoC and, if so, what would be + your preference if you're accepted to both? Having a stated preference + helps with the deduplication process and will not impact if we accept your + application or not.</li> + <li>Is there anything else we should know that will make us like your project more?</li>

1 0

r24745: {website} Adding gsoc proposals for this year's students (in website/trunk/about: en gsocProposal)
by Damian Johnson 13 May '11

13 May '11

Author: atagar Date: 2011-05-13 15:51:39 +0000 (Fri, 13 May 2011) New Revision: 24745 Added: website/trunk/about/gsocProposal/gsoc11-proposal-metadataToolkit.txt Modified: website/trunk/about/en/gsoc.wml Log: Adding gsoc proposals for this year's students Modified: website/trunk/about/en/gsoc.wml =================================================================== --- website/trunk/about/en/gsoc.wml 2011-05-12 14:09:44 UTC (rev 24744) +++ website/trunk/about/en/gsoc.wml 2011-05-13 15:51:39 UTC (rev 24745) @@ -212,6 +212,10 @@ <li><h4><a href="http://tor.spanning-tree.org/proposal.html">DNSEL Rewrite</a> by Harry Bock</h4></li> <li><h4><a href="http://kjb.homeunix.com/wp-content/uploads/2010/05/KevinBerry-GSoC2010-TorP…">Extending Tor Network Metrics</a> by Kevin Berry</h4></li> <li><h4><a href="../about/gsocProposal/gsoc10-proposal-soat.txt">SOAT Expansion</a> by John Schanck</h4></li> + <li><h4><a href="http://inspirated.com/uploads/tor-gsoc-11.pdf">GTK+ Frontend and Client Mode Improvements for arm</a> by Kamran Khan</h4></li> + <li><h4><a href="http://www.gsathya.in/gsoc11.html">Orbot + ORLib</a> by Sathya Gunasekaran</h4></li> + <li><h4><a href="http://blanu.net/TorSummerOfCodeProposal.pdf">Blocking-resistant Transport Evaluation Framework</a> by Brandon Wiley</h4></li> + <li><h4><a href="../about/gsocProposal/gsoc11-proposal-metadataToolkit.txt">Metadata Anonymisation Toolkit</a> by Julien Voisin</h4></li> <li><h4><a href="http://www.atagar.com/misc/gsocBlog09/">Website Pootle Translation</a> by Damian Johnson</h4></li> </ul> Added: website/trunk/about/gsocProposal/gsoc11-proposal-metadataToolkit.txt =================================================================== --- website/trunk/about/gsocProposal/gsoc11-proposal-metadataToolkit.txt (rev 0) +++ website/trunk/about/gsocProposal/gsoc11-proposal-metadataToolkit.txt 2011-05-13 15:51:39 UTC (rev 24745) @@ -0,0 +1,231 @@ +Hello, +I am Julien Voisin, undergraduate computer science student from France. + +I am interested to work on the “Meta-data anonymizing toolkit for file publication” project. +I know there is already a student interested in by this project, but I really want to do it : +I needed it for my own and have already thought about a potential design some time ago. + +I would like to work for the EFF, because I am very concerned about privacy issues on the Internet. +I think privacy is an essential right, and not just an option. +Especially, I would really enjoy to work for the Tor project (or Tails, since it's heavily based on him). +I am using it for quite some time and would like to get more involved and contribute back! + +I use F/OSS on a daily basis (Ubuntu, Debian, Archlinux and Gentoo). +So far my major contributions were the writing of documentations for archLinux, openmw, xda-forum and Ubuntu. +Recently I have released a little matrix manipulation library written in C, +originally for an academic project (http://dustri.org/lib/) +I am interested to do the debian package, but heard that it can be quite tricky, so a little +help would be much appreciated + +I do not have any major plan for this summer (but my holidays only begins the june 4th), so I can fully focus on the project and reasonably think that I could commit 5-6 hours per day on it. + +Requirement/Deliverables: + + * A command line and a GUI tool having both the following capabilities (in order of importance): + o Listing the metadatas embedded in a given file + o A batch mode to handle a whole directory (or set of directories) + o The ability to scan files packed in the most common archive formats + o A nice binding for srm (Secure ReMoval) or shred (GNU utils) to properly remove the original file containing the evil metas + o Let the user delete/modify a specific meta + + * Should run on the most common OS/architectures (And especially on Debian Squeeze, since Tails is based on it.) + * The whole thing should be easily extensible (especially it should be easy to add support for new file formats) + * The proper functioning of the software should be easily testable + +I'd like to do this project in Python, because I already have done some personal projects whith it (for which I also used subversion) : an IRC bot tailored for logging (dustri.org/tor/depluie.tar.bz2 still under heavy WIP), a battery monitor, a simple search engine indexing FTP servers, ... + +Why is Python a good choice for implementing this project ? + + 1. I am experienced with the language + 2. There are plenty of libraries to read/write metadatas, among them is Hachoir (https://bitbucket.org/haypo/hachoir/)that looks very promising since it supports quite a few file formats + 3. It is easy to wrap other libraries for our needs (even if they are not written in Python !) + 4. Runs on almost every OS/architecture, what is a great benefit for portability + 5. It is easy to make unit tests (thanks to the built-in Unittest module) + +Proposed design: + +The proposed design has three main components : one lib, a command line tool and a GUI tool. + +The aim of the library (described with more details in the next part) is to make the development of tools easy. A special attention will be made on the API that it exposes. The ultimate goal being to be able to add the support of new file format in the library without changing the whole code of the tools. + +Meta reading/writing library : + +A library to read and write metas for various file formats. The main goal is to provide an abstraction interface (for the file format and for the underlying libraries used). +At first it would only wrap Hachoir. +Why hachoir : + + * Autofix: Hachoir is able to open invalid / truncated files + * Lazy: Open a file is very fast since no information is read from file, data are read and/or computed when the user ask for it + + * Types: Hachoir has many predefined field types (integer, bit, string, etc.) and supports string with charset (ISO-8859-1, UTF-8, UTF-16, ...) + * Addresses and sizes are stored in bit, so flags are stored as classic fields + * Editor: Using Hachoir representation of data, you can edit, insert, remove data and then save in a new file. + * Meta : Support a very large scale of file format + +But we could also wrap other libraries to support a particular file format. Or write ourself the support for a format, although this should be avoided if possible (it looks simple at first, but supporting different versions of the format and maintaining the thing over time is extremely time consuming) +The must would be to make the children libraries optional dependencies. + +One typical use case of the lib is to ask for metadatas for a file, if the format is supported a list (or maybe a tree) of metas is returned. + +Both the GUI and the cmdline tool will use this lib. + +The cmdline/GUI tool features: + + * List all the meta + * Removing all the meta + * Anonymising all the meta + * Let the user chose wich meta he wants to modify + * Support archives anonymisation + * Secure removal + * Cleaning wholes folder recursively + +GUI: +Essentially the GUI tool would do the same features as for the cmd line too. +I do not have a significant GUI development experience, but I'm planing to fix that point during community bonding period. + +Proposed development methods: +One way to develop this would be to do it layer by layer : first implementing the meta reading/writing lib for ale the formats in one shot, then making the command line application, … + +However for this project, developing feature by feature seems more appropriate : +Starting by a skeleton implementing a thin slice of functionality that traverses most of the layers. +For example, I could start by focussing only on EXIF metas : make sure that the meta reading/writing library supports EXIF, then make the command line tool using the previous library. +And only then, when the skeleton is actually working, add supports for other features/formats. + +This allows a more incremental development flow and after only a few weeks I would be able to deliver a working system. The list of features supported in the first iterations would be ridiculously short but at least that would enable me to get feedbacks and notice quickly if I am on the wrong tracks. +Also, since I would add one feature at a time, the structure of the system will tend to easily accommodate that. Thanks to this, adding the support of a new file format will be made easy, even after the end of the GSOC. + + +Testing +For such a tool, since the smallest crack could compromise the user, testing is critically important. I plan to implement two main kind of testings : + +Unit tests + +To test the proper working of the meta accessing library. Maintaining a collection of files with metas for every format we should support. Using Python's unittest I can setup expectations to make sure that the library will finds the good fields. + +End-to-End testing: + +A script (or set of script) to test the proper working of the command line tool. One way is to run the tool in batch mode on the input test files set. If in the output we are still able to find metas, the system is not doing his job right. + +It is possible to write this script in Python, possibly using again the unittest lib (even if here the goal is to execute an external executable and see how it's behaving) to get output of the results that is consistent with the unit tests. + +The goal is to be able to automatically test the whole system in a few commands (one for the unit tests one for the end-to-end tests). It will be easy to add a new document in the test set, so if someone from the community provides a file that was not cleaned properly, we can easily reproduce the problem and then decide on the proper action to take. The ability to tests everything easily might help a user to make sure that everything works fine on his OS/architecture. Also, if for one reason after some months it is decided to change from Hachoir to another underlying library (yes a pretty radical decision, just given as example) we still have the tests, so valuable to check than everything is still working with the new library. + +Timeline: + + * Community Bonding Period (in order of importance) + o Playing around with pygobject + o Playing with Hachoir + o Learning git + + * First two weeks : + o create the structure in the repository (directories, README, ..) + o Create a skeleton + + o Objectives : to have a deployable working system as soon as possible(even if the list of features is ridiculous). So that I can show you my work in an incremental way thereafter and get feedbacks early. + o The lib will handle reading/writing EXIF fields (using Hachoir) + o A set of tests files (and automated unit tests) to demonstrate that the lib does the job + o The beginning of the command line tool, at this point must list and delete EXIF meta + o An automated end-to-end test to show that the command line tool does properly remove the EXIF + +After this first step (making the skeleton) I should be able to deliver a working system right after adding each of the following features. I Hope to get feedbacks so can fix problems quickly + + * 3 weeks + o adding support for (in order of importance) pdf, zip/tar/bzip (just the meta, not the content yet), jpeg/png/bmp, ogg/mpeg1-2-3, exe... + o For every type of meta, that involves : + + Creating some input test files with meta data + + Implementing the feature in the library + + Asserting that the lib does the job with unit tests + + Modifying the cmd line tool to support the feature (if necessary) + + Checking that the cmd line tool can properly delete this type of meta with automated end-to-end test + + * about one day + o Enable the command line tool to set a specific meta to a chosen value + + * about 1 day + o Implementation of the “batch mode” in the cmdline tool, to clean a whole folder + o Implementation of secure removal + + * about 2 days : + o Add support for deep archive cleanup + + Clean the content of the archives + + Make a list of non supported format, for which we warn the user that only the container can be cleaned from meta, not the content (at first that will include rar, 7zip, ..) + + The supported formats will be those supported natively by Python ( bzip2, gzip, tar ) + + Create some test archives for each supported format containing various files with metas + + Implement the deep cleanup for the format + + Assert that the command line passes the end-to-end tests (that is, it can correctly clean the content of the test archives) + + * about 2 days + o Add support for complete deletion of the original files + o Make a binding nice for shred (should not be to hard using Python) + o Implement the feature in the command line tool + + * 3 weeks + o Implementation of the GUI tool + o At this stage, I can use the experience from implementing the cmdline tool to implement the GUI tool, having the same features. + + * 1 week + o Add support for more format (might be based on requests from the community) + + * Remaining weeks + o I want to keep those remaining week in case of problems, and for + + Remaining/polishing cleanup + + Bugfixing + + Integration work + + Missing features + + Packaging + + Final documentation + * Every Week-end : + o Documentation time : both end-user, and design. I do not like to document my code while I'm coding it : it slows a lot the development process, but it's not a good thing to delay it too much : week-ends seems fine for this. + o A blog-post, and a mail on the mailing list about what I have done in the week. + +About the anonymisation process : + +Since I'll relay on Hachoir in a first time, I don't know how much it's +effective on every case. +I am planing to do some tests during the Community Bonding Period. + +The plan is to first rely on the capabilities of Hachoir. I don't know yet +how effective it is for each case. I am planing to do some tests during the +Community Bonding Period. Following the test strategy described before, it +will be easy to add a new document in the test set. If I could make a test +file with metas not supported by Hachoir (or someone from the community +provide such a file), we could then decide on the proper action : propose a +patch to Hachoir or use another library for this specific format. + +Doing R&D about supporting exotics fields, or improve existing support will +depend of my progress. Speaking of the fingerprints, the subject of the +project is “metadata anonymisation”, and not “fingerprint detection” : they +are too many softwares, too many subtle ways to alter/mark a file (and I +don't even speak of steganography). + +So, nop, I'm not planing to implement fingerprinting detection. Not that it +is not interesting, it's just that it's not realistic to support it +correctly (actually it's not realistic to support it at all, given that I +must first support the metas, in such a short time frame). Would you agree +that it is better to first focus on making a working tool that does the job +for the known metadatas ? + +That done, if the design is good we could easily add support for more exotic +fields (or some kind of fingerprinting). I think we should never loose track +of the sad truth : no matter how big the effort we spend on this, making a +comprehensive tool able of detecting every kind of meta and every pattern of +fingerprinting is just not feasible. + +About archives anonymisation + +Task order for anonymisation of an archive : + + 1. Extract the archive + 2. Anonymise his content + 3. Recompress the archive + 4. Securely deleting the extracted content + 5. Anonymise the new created archive + +And I'm planing to support the extraction only of archive formats supported by python (without this limitation, the tool will not be portable). If the user try to anonymize a .rar for example, the tool will popup “WARNING - only the container will be anonymize, and not the content - WARNING” + +As for what I expect from my mentor, I think he should try to be available (not immediately but in the 48 hours) when I need him specifically (e.g. technical questions no one else on IRC can answer) but he doesn't need to check on me all the time. I'm fine with mails to (since intrigeri is not always “online”, it will be I think, the best solution to communicate with him). I'd like to do a weekly reunion, on irc or jabber, to discuss things more smoothly. + +I'm using irc quite a lot, and I'm hanging on #tor, and #tor-dev (nick : jvoisin). +I'm planing to do a blogpost every week-end, about the advancement of the project. +You can mail me at : julien.voisin(a)dustri.org for more information. +

1 0