- tbb-commits - lists.torproject.org

[torbutton/master] Revert "Bug 13742: Remove SafeCache code (in favor of C++ implementation)"
by gk＠torproject.org 02 Dec '14

02 Dec '14

commit 7b93a59893957db4495abe3541036c760ed119f9 Author: Georg Koppen <gk(a)torproject.org> Date: Tue Dec 2 14:42:13 2014 +0000 Revert "Bug 13742: Remove SafeCache code (in favor of C++ implementation)" This reverts commit 1dedacb01855cab10835d165578575603a208cec. --- src/chrome/content/stanford-safecache.js | 448 ++++++++++++++++++++++++++++++ src/chrome/content/torbutton.xul | 4 + 2 files changed, 452 insertions(+) diff --git a/src/chrome/content/stanford-safecache.js b/src/chrome/content/stanford-safecache.js new file mode 100644 index 0000000..0ef52ad --- /dev/null +++ b/src/chrome/content/stanford-safecache.js @@ -0,0 +1,448 @@ +// Bug 1506 P3: This file provides important cache isolation properties, +// but it is not very useful without a patched Firefox to go with it. +// It could probably also use some refactoring into an XPCOM component. +// It is currently registered from torbutton.js's "main window" observers, +// which is not the right place. + +/* + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of Stanford University nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ +// Dual-keyed cache/cookie same-origin policy +// Ensures that cache and cookies set in HTTP headers cannot be used for +// non-cooperative or semi-cooperative tracking. +// +// Author: Edward Pastuszenski +// +// Based on Stanford SafeCache +// Author: Collin Jackson +// Other contributors: Andrew Bortz, John Mitchell, Dan Boneh +// + +////////////////////////////////////////////////////////////////////////////// +// Constants + +const kSSC_ENABLED_PREF = "extensions.torbutton.safecache"; +const kSSC_SC_ENABLED_PREF = "extensions.torbutton.dual_key_cookies"; +const kSSC_TORBUTTON_PREF = "extensions.torbutton.tor_enabled"; +const kSSC_COOKIE_JS_PREF = "extensions.torbutton.cookie_js_allow"; + +//////////////////////////////////////////////////////////////////////////// +// Debug stuff + +/** + * Dump information to the console? + */ +var SSC_debug = true; + +/** + * Sends data to the console if we're in debug mode + * @param msg The string containing the message to display + */ +function SSC_dump(msg) { + if (SSC_debug) + torbutton_log(3, "SSC: " + msg); +} + +//////////////////////////////////////////////////////////////////////////// +// "Major" objects/classes + +/** + * SafeCache HTTP Request Listener + * Watches for the authentication requests and presents password dialog + */ + +function SSC_RequestListener(controller) { + this.controller = controller; + this.thirdPartyUtil = Cc["@mozilla.org/thirdpartyutil;1"]. + getService(Ci.mozIThirdPartyUtil); + this.cookie_permissions = Cc["@mozilla.org/cookie/permission;1"]. + getService(Ci.nsICookiePermission); +} + +SSC_RequestListener.prototype = +{ + controller: null, // The SSC_Controller that created this + + observe: function(subject, topic, data) { + try { + if(this.controller.getEnabled() == 2) return; + if(this.controller.getEnabled() == 1 + && !this.controller.getTorButton()) return; + if (topic == 'http-on-modify-request') { + subject.QueryInterface(Components.interfaces.nsIHttpChannel); + subject.QueryInterface(Components.interfaces.nsIHttpChannelInternal); + subject.QueryInterface(Components.interfaces.nsICachingChannel); + subject.QueryInterface(Components.interfaces.nsIChannel); + this.onModifyRequest(subject); + } else if (topic == 'http-on-examine-response') { + subject.QueryInterface(Components.interfaces.nsIHttpChannel); + subject.QueryInterface(Components.interfaces.nsIHttpChannelInternal); + subject.QueryInterface(Components.interfaces.nsICachingChannel); + subject.QueryInterface(Components.interfaces.nsIChannel); + this.onExamineResponse(subject); + } + } catch(e) {try {torbutton_log(4, "SSC: "+e);} catch(ex) {}} + }, + + bypassCache: function(channel) { + channel.loadFlags |= channel.LOAD_BYPASS_CACHE; + // INHIBIT_PERSISTENT_CACHING instead? + channel.cacheKey = this.newCacheKey(0); + SSC_dump("Bypassed cache for " + channel.URI.spec); + }, + + setCacheKey: function(channel, str) { + try { + channel.cacheDomain = str; + SSC_dump("Set cacheDomain to "+str+" for "+channel.URI.spec); + } catch(e) { + var oldData = this.readCacheKey(channel); + var newKey = this.newCacheKey(this.getHash(str) + oldData); + channel.cacheKey = newKey; + SSC_dump("Set cache key to hash(" + str + ") = " + + newKey.data + " for " + channel.URI.spec); + } + }, + + onModifyRequest: function(channel) { + var parent_host = null; + var parent_spec = null; + if (channel.notificationCallbacks || + channel.loadGroup && channel.loadGroup.notificationCallbacks) { + var callbacks = null; + if (channel.notificationCallbacks) { + callbacks = channel.notificationCallbacks; + } else { + callbacks = channel.loadGroup.notificationCallbacks; + } + try { + var wind = callbacks.QueryInterface( + Components.interfaces.nsIInterfaceRequestor).getInterface( + Components.interfaces.nsIDOMWindow); + parent_host = wind.window.top.location.hostname; + parent_spec = wind.window.top.location.href; + } catch(e) { + } + SSC_dump("Parent "+parent_host+" for "+ channel.URI.spec); + } + + if (channel.documentURI && channel.documentURI == channel.URI) { + parent_host = null; // first party interaction + parent_spec = null; + } else if(!parent_host) { + // Questionable first party interaction.. + try { + var anuri = null; + try { + anuri = this.thirdPartyUtil.getFirstPartyURIFromChannel(channel, false); + } catch (e) { + torbutton_safelog(2, "FirstParty API failed to get parent: "+e+" ", channel.URI.spec); + // We are not using the TBB based on ESR 24. Falling back to the old + // method. + anuri = this.cookie_permissions.getOriginatingURI(channel); + } + parent_host = anuri.host; + parent_spec = anuri.spec; + } catch(e) { + // XXX: This can still fail for OCSP and other windowless requests.. + torbutton_safelog(3, "Cookie API failed to get parent: "+e+" ",channel.URI.spec); + if (!channel.referrer) { + torbutton_safelog(3, "SSC: No cache isolation parent for ", channel.URI.spec); + } else { + parent_host = channel.referrer.host; + parent_spec = channel.referrer.spec; + } + } + } + + // Same-origin policy + var referrer; + if (parent_host && parent_host != channel.URI.host) { + SSC_dump("Segmenting " + channel.URI.spec + + " content loaded by " + parent_host); + this.setCacheKey(channel, parent_host); + referrer = parent_host; + try { + // Disable 3rd party http auth, but exempt the browser (for favicon loads) + // FIXME: Hrmm, this is just going to disable auth for 3rd party domains. + // It would be better if we could isolate the auth, but still + // allow it to be transmitted.. But I guess, who still uses http auth anyways? + if (channel.getRequestHeader("Authorization") !== null) { + if (parent_spec == "chrome://browser/content/browser.xul") { + torbutton_log(3, "Allowing auth for browser load of "+channel.URI.spec); + } else { + torbutton_safelog(4, "Removing 3rd party HTTP auth for url ", + channel.URI.spec+", parent: "+parent_spec); + channel.setRequestHeader("Authorization", null, false); + channel.setRequestHeader("Pragma", null, false); + channel.setRequestHeader("Cache-Control", null, false); + } + } + } catch (e) {} + } else { + referrer = channel.URI.host; + if(!this.readCacheKey(channel)) { + this.setCacheKey(channel, channel.URI.host); + } else { + SSC_dump("Existing cache key detected; leaving it unchanged."); + } + } + + if (this.controller.getBlockThirdPartyCache()) { + if(parent_host && parent_host != channel.URI.host) { + //SSC_dump("Third party cache blocked for " + channel.URI.spec + + //" content loaded by " + parent_host); + this.bypassCache(channel); + } + } + + var cookie = null; + if (this.controller.getSafeCookieEnabled()) { + try{ + cookie = channel.getRequestHeader("Cookie"); + //SSC_dump("Cookie: " + cookie); + } catch(e) {cookie = null;} + } + + if(cookie) { + //Strip the secondary key from every referrer-matched cookie + var newHeader = ""; + var i = 0; + var lastStart = 0; + //State 0: no information on next cookie + //State 1: cookie will be sent. + //State 2: cookie will not be sent. + var state = 0; + while (i < cookie.length) { + //Dual-keyed cookie + if(state == 0 && cookie.charAt(i) == '|'){ + //If referrers match, strip key and send cookie + var cookieReferrer = cookie.toString().substr(lastStart, i - lastStart); + if (referrer == cookieReferrer){ + lastStart = i+1; + state = 1; + } else { + state = 2; + } + } + //Single-keyed cookie that was set via scripting. + if (state == 0 && cookie.charAt(i) == '='){ + if(this.controller.getCookieJS()) state = 1; + else { + if (referrer == channel.getRequestHeader("Host")) state = 1; + else state = 2; + } + } + //End of a cookie + if (cookie.charAt(i) == ';') { + var thisCookie = cookie.toString().substr(lastStart, i - lastStart + 2); + if (state == 1){ + newHeader += thisCookie; + } + if (state == 2){ + SSC_dump("Declining to send " + thisCookie + + " for request by embedded domain " + channel.URI.host + + + " " + channel.getRequestHeader("Host") + + " on embedding page " + referrer); + } + lastStart = i+2; + state = 0; + } + //End of the string + if (i == cookie.length - 1){ + thisCookie = cookie.toString().substr(lastStart, i - lastStart + 1); + if (state == 1){ + newHeader += thisCookie; + } + if (state == 2){ + SSC_dump("Declining to send " + thisCookie + + " for request by embedded domain " + channel.URI.host + + + " " + channel.getRequestHeader("Host") + + " on embedding page " + referrer); + } + lastStart = i+1; + } + i++; + } + channel.setRequestHeader("Cookie", newHeader, false); + } + + }, + + onExamineResponse: function(channel) { + var setCookie; + try{ + setCookie = channel.getResponseHeader("Set-Cookie"); + } catch(e) {setCookie = null;} + + if(setCookie) { + var parent = null; + // XXX: need to use loadGroup here if null.. + if (channel.notificationCallbacks) { + var wind = channel.notificationCallbacks.QueryInterface( + Components.interfaces.nsIInterfaceRequestor).getInterface( + Components.interfaces.nsIDOMWindow); + parent = wind.window.top.location; + } + + if (channel.documentURI && channel.documentURI == channel.URI) { + parent = null; // first party interaction + } + + var referrer; + // Same-origin policy + if (parent && parent.hostname != channel.URI.host) { + //SSC_dump("Segmenting " + channel.URI.host + + //" content loaded by " + parent.host); + referrer = parent.hostname; + } else { + referrer = channel.URI.host; + } + //Dual-key each cookie set in the header + var newHeader = ""; + var i = 0; + var lastStart = 0; + //Some messy code that prevents multiple embedding-domain keys + //from being concatenated to cookie names. + var passedname = false; + var namebar = false; + while (i < setCookie.length) { + if (setCookie.charAt(i) == '=') passedname = true; + else if (setCookie.charAt(i) == '|' && passedname == false) + namebar = true; + if (i == setCookie.length - 1 || setCookie.charAt(i) == '\n'){ + if(!namebar){ + newHeader += referrer + "|" + + setCookie.toString().substr(lastStart, i - lastStart + 1); + } + lastStart = i+1; + passedname = false; + namebar = false; + } + i++; + } + //SSC_dump("MODIFIED Set-Cookie: " + newHeader); + channel.setResponseHeader("Set-Cookie", newHeader, false); + } + }, + + // Read the integer data contained in a cache key + readCacheKey: function(channel) { + try { + return channel.cacheDomain; + } catch(e) { + channel.cacheKey.QueryInterface(Components.interfaces.nsISupportsPRUint32); + return channel.cacheKey.data; + } + }, + + // Construct a new cache key with some integer data + newCacheKey: function(data) { + var cacheKey = + Components.classes["@mozilla.org/supports-PRUint32;1"] + .createInstance(Components.interfaces.nsISupportsPRUint32); + cacheKey.data = data; + return cacheKey; + }, + + strMD5: function(str) { + var converter = + Components.classes["@mozilla.org/intl/scriptableunicodeconverter"]. + createInstance(Components.interfaces.nsIScriptableUnicodeConverter); + + // nsIURI.host is UTF-8 + converter.charset = "UTF-8"; + // result.value will contain the array length + var result = {}; + // data is an array of bytes + var data = converter.convertToByteArray(str, result); + var ch = Components.classes["@mozilla.org/security/hash;1"] + .createInstance(Components.interfaces.nsICryptoHash); + ch.init(ch.MD5); + ch.update(data, data.length); + return ch.finish(false); + }, + + // Get an integer hash of a string + getHash: function(str) { + var hash = this.strMD5(str); + var intHash = 0; + for(var i = 0; i < hash.length && i < 8; i++) + intHash += hash.charCodeAt(i) << (i << 3); + return intHash; + }, +} + +/** + * Master control object. Adds and removes the RequestListener + */ +function SSC_Controller() { + this.prefs = Components.classes["@mozilla.org/preferences-service;1"] + .getService(Components.interfaces.nsIPrefService); + this.addListener(new SSC_RequestListener(this)); +} + +SSC_Controller.prototype = { + + getEnabled: function() { + return (this.prefs.getIntPref(kSSC_ENABLED_PREF)); + }, + + getSafeCookieEnabled: function() { + return (this.prefs.getBoolPref(kSSC_SC_ENABLED_PREF)); + }, + + getBlockThirdPartyCache: function() { + // Meh, no pref for now. Always allow. + return false; + }, + + getTorButton: function() { + return (this.prefs.getBoolPref(kSSC_TORBUTTON_PREF)); + }, + + getCookieJS: function() { + return (this.prefs.getBoolPref(kSSC_COOKIE_JS_PREF)); + }, + + addListener: function(listener) { + this.listener = listener; + var observerService = + Components.classes["@mozilla.org/observer-service;1"] + .getService(Components.interfaces.nsIObserverService); + observerService.addObserver(listener, "http-on-modify-request", false); + // XXX: We need an observer to add this listener when the pref gets set + if (this.getSafeCookieEnabled()) { + observerService.addObserver(listener, "http-on-examine-response", false); + } + }, + + removeListener: function() { + var observerService = + Components.classes["@mozilla.org/observer-service;1"] + .getService(Components.interfaces.nsIObserverService); + observerService.removeObserver(this.listener, "http-on-modify-request"); + if (this.getSafeCookieEnabled()) { + observerService.removeObserver(this.listener, "http-on-examine-response"); + } + }, +} + +//////////////////////////////////////////////////////////////////////////// +// Global stuff +// "What script would be complete without a couple of globals?" --Fritz + +var SSC_controller; + +function SSC_startup() { + if(!SSC_controller) SSC_controller = new SSC_Controller(); + SSC_dump("Loaded controller"); +} diff --git a/src/chrome/content/torbutton.xul b/src/chrome/content/torbutton.xul index 95624f0..00dc6f0 100644 --- a/src/chrome/content/torbutton.xul +++ b/src/chrome/content/torbutton.xul @@ -6,6 +6,10 @@ <overlay id="torbutton-overlay" xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"> + +  + <script src="chrome://torbutton/content/stanford-safecache.js" /> + <script type="application/x-javascript" src="chrome://torbutton/content/torbutton_util.js" /> <script type="application/x-javascript" src="chrome://torbutton/content/tor-circuit-display.js" /> <script type="application/x-javascript" src="chrome://torbutton/content/torbutton.js" />

1 0

[torbutton/master] Revert "Bug 13751: Get rid of remaining SafeCache parts."
by gk＠torproject.org 02 Dec '14

02 Dec '14

commit b426228c016a443cc5b2cc29b30f84efc4346ab1 Author: Georg Koppen <gk(a)torproject.org> Date: Tue Dec 2 14:43:36 2014 +0000 Revert "Bug 13751: Get rid of remaining SafeCache parts." This reverts commit e2cce93493a857b062af83f5e8f9130fb76933a0. --- src/CHANGELOG | 3 --- src/chrome/content/torbutton.js | 6 ++++++ src/install.rdf | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/CHANGELOG b/src/CHANGELOG index 1966ab7..4510b0c 100644 --- a/src/CHANGELOG +++ b/src/CHANGELOG @@ -1,6 +1,3 @@ -1.8.1.1 - * Bug 13751: Get rid of remaining SafeCache parts - 1.8.1.0 * Bug 13746: Properly link Torbutton UI to thirdparty pref. * Bug 13742: Remove SafeCache code (in favor of C++ implementation) diff --git a/src/chrome/content/torbutton.js b/src/chrome/content/torbutton.js index 8dba1fd..da39864 100644 --- a/src/chrome/content/torbutton.js +++ b/src/chrome/content/torbutton.js @@ -2690,6 +2690,9 @@ function torbutton_do_main_window_startup() torbutton_unique_pref_observer.register(); + // Bug 1506: This is probably the most important observer in this function + // XXX: We should fold this into our code/move it to its own component + SSC_startup(); } // Bug 1506 P4: Most of this function is now useless, save @@ -2955,6 +2958,9 @@ function torbutton_close_window(event) { progress.removeProgressListener(torbutton_weblistener); torbutton_unique_pref_observer.unregister(); + // XXX: We should fold this into our code.. + SSC_controller.removeListener(); + if(m_tb_is_main_window) { // main window not reset above // This happens on Mac OS because they allow firefox // to still persist without a navigator window diff --git a/src/install.rdf b/src/install.rdf index b39f0de..402b99c 100644 --- a/src/install.rdf +++ b/src/install.rdf @@ -6,7 +6,7 @@ <em:name>Torbutton</em:name> <em:creator>Mike Perry</em:creator> <em:id>torbutton(a)torproject.org</em:id> - <em:version>1.8.1.1</em:version> + <em:version>1.8.1.0</em:version> <em:homepageURL>https://www.torproject.org/projects/torbrowser.html.en</em:homepageURL> <em:optionsURL>chrome://torbutton/content/preferences.xul</em:optionsURL> <em:iconURL>chrome://torbutton/skin/tor.png</em:iconURL>

1 0

[torbutton/master] #13671: fix circuit display when bridges are used
by gk＠torproject.org 02 Dec '14

02 Dec '14

commit 35ea75545037574c568df386798efb6ea21790d5 Author: Arthur Edelstein <arthuredelstein(a)gmail.com> Date: Wed Nov 12 12:15:01 2014 -0800 #13671: fix circuit display when bridges are used --- src/chrome/content/tor-circuit-display.js | 142 ++++++++++++++++++---------- src/modules/tor-control-port.js | 146 +++++++++++++++++------------ 2 files changed, 178 insertions(+), 110 deletions(-) diff --git a/src/chrome/content/tor-circuit-display.js b/src/chrome/content/tor-circuit-display.js index 0817aa6..caf6886 100644 --- a/src/chrome/content/tor-circuit-display.js +++ b/src/chrome/content/tor-circuit-display.js @@ -27,6 +27,7 @@ let createTorCircuitDisplay = (function () { // Mozilla utilities const Cu = Components.utils; Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/Task.jsm"); // Import the controller code. let { controller } = Cu.import("resource://torbutton/modules/tor-control-port.js"); @@ -39,51 +40,90 @@ let logger = Cc["@torproject.org/torbutton-logger;1"] // A mutable map that stores the current nodes for each domain. let domainToNodeDataMap = {}, - // A mutable map that records what circuits are already known. - knownCircuitIDs = {}; + // A mutable map that reports `true` for IDs of "mature" circuits + // (those that have conveyed a stream).. + knownCircuitIDs = {}, + // A map from bridge fingerprint to its IP address. + bridgeIDtoIPmap = new Map(); // __trimQuotes(s)__. // Removes quotation marks around a quoted string. let trimQuotes = s => s ? s.match(/^\"(.*)\"$/)[1] : undefined; -// nodeDataForID(controller, id, onResult)__. -// Requests the IP, country code, and name of a node with given ID. -// Returns result via onResult. -// Example: nodeData(["20BC91DC525C3DC9974B29FBEAB51230DE024C44"], show); -let nodeDataForID = function (controller, ids, onResult) { - let idRequests = ids.map(id => "ns/id/" + id); - controller.getInfoMultiple(idRequests, function (statusMaps) { - let IPs = statusMaps.map(statusMap => statusMap.IP), - countryRequests = IPs.map(ip => "ip-to-country/" + ip); - controller.getInfoMultiple(countryRequests, function (countries) { - let results = []; - for (let i = 0; i < ids.length; ++i) { - results.push({ name : statusMaps[i].nickname, id : ids[i] , - ip : statusMaps[i].IP , country : countries[i] }); - } - onResult(results); - }); - }); +// __readBridgeIPs(controller)__. +// Gets a map from bridge ID to bridge IP, and stores it +// in `bridgeIDtoIPmap`. +let readBridgeIPs = function (controller) { + Task.spawn(function* () { + let configText = yield controller.getInfo("config-text"), + bridgeEntries = configText.Bridge; + if (bridgeEntries) { + bridgeEntries.map(entry => { + let IPplusPort, ID, + tokens = entry.split(/\s+/); + // First check if we have a "vanilla" bridge: + if (tokens[0].match(/^\d+\.\d+\.\d+\.\d+/)) { + [IPplusPort, ID] = tokens; + // Several bridge types have a similar format: + } else if (["fte", "obfs3", "obfs4", "scramblesuit"] + .indexOf(tokens[0]) >= 0) { + [IPplusPort, ID] = tokens.slice(1); + } + // (For now, we aren't dealing with meek bridges and flashproxy.) + if (IPplusPort && ID) { + let IP = IPplusPort.split(":")[0]; + bridgeIDtoIPmap.set(ID.toUpperCase(), IP); + } + }); + } + }).then(null, Cu.reportError); }; -// __nodeDataForCircuit(controller, circuitEvent, onResult)__. +// nodeDataForID(controller, id)__. +// Returns the type, IP and country code of a node with given ID. +// Example: `nodeData(controller, "20BC91DC525C3DC9974B29FBEAB51230DE024C44")` +// => `{ type : "default" , ip : "12.23.34.45" , countryCode : "fr" }` +let nodeDataForID = function* (controller, id) { + let result = {}; // ip, type, countryCode; + if (bridgeIDtoIPmap.has(id.toUpperCase())) { + result.ip = bridgeIDtoIPmap.get(id.toUpperCase()); + result.type = "bridge"; + } else { + // Get the IP address for the given node ID. + try { + let statusMap = yield controller.getInfo("ns/id/" + id); + result.ip = statusMap.IP; + } catch (e) { } + result.type = "default"; + } + if (result.ip) { + // Get the country code for the node's IP address. + try { + result.countryCode = yield controller.getInfo("ip-to-country/" + result.ip); + } catch (e) { } + } + return result; +}; + +// __nodeDataForCircuit(controller, circuitEvent)__. // Gets the information for a circuit. -let nodeDataForCircuit = function (controller, circuitEvent, onResult) { - let ids = circuitEvent.circuit.map(circ => circ[0]); - nodeDataForID(controller, ids, onResult); +let nodeDataForCircuit = function* (controller, circuitEvent) { + let rawIDs = circuitEvent.circuit.map(circ => circ[0]), + // Remove the leading '$' if present. + ids = rawIDs.map(id => id[0] === "$" ? id.substring(1) : id); + // Get the node data for all IDs in circuit. + return [for (id of ids) yield nodeDataForID(controller, id)]; }; -// __getCircuitStatusByID(aController, circuitID, onCircuitStatus)__ -// Returns the circuit status for the circuit with the given ID -// via onCircuitStatus(status). -let getCircuitStatusByID = function(aController, circuitID, onCircuitStatus) { - aController.getInfo("circuit-status", function (circuitStatuses) { - for (let circuitStatus of circuitStatuses) { - if (circuitStatus.id === circuitID) { - onCircuitStatus(circuitStatus); - } +// __getCircuitStatusByID(aController, circuitID)__ +// Returns the circuit status for the circuit with the given ID. +let getCircuitStatusByID = function* (aController, circuitID) { + let circuitStatuses = yield aController.getInfo("circuit-status"); + for (let circuitStatus of circuitStatuses) { + if (circuitStatus.id === circuitID) { + return circuitStatus; } - }); + } }; // __collectIsolationData(aController)__. @@ -95,20 +135,18 @@ let collectIsolationData = function (aController) { aController.watchEvent( "STREAM", streamEvent => streamEvent.StreamStatus === "SENTCONNECT", - function (streamEvent) { + streamEvent => Task.spawn(function* () { if (!knownCircuitIDs[streamEvent.CircuitID]) { logger.eclog(3, "streamEvent.CircuitID: " + streamEvent.CircuitID); knownCircuitIDs[streamEvent.CircuitID] = true; - getCircuitStatusByID(aController, streamEvent.CircuitID, function (circuitStatus) { - let domain = trimQuotes(circuitStatus.SOCKS_USERNAME); - if (domain) { - nodeDataForCircuit(aController, circuitStatus, function (nodeData) { - domainToNodeDataMap[domain] = nodeData; - }); - } - }); + let circuitStatus = yield getCircuitStatusByID(aController, streamEvent.CircuitID), + domain = trimQuotes(circuitStatus.SOCKS_USERNAME); + if (domain) { + let nodeData = yield nodeDataForCircuit(aController, circuitStatus); + domainToNodeDataMap[domain] = nodeData; + } } - }); + }).then(null, Cu.reportError)); }; // ## User interface @@ -122,7 +160,7 @@ let regionBundle = Services.strings.createBundle( // Convert a country code to a localized country name. // Example: `'de'` -> `'Deutschland'` in German locale. let localizedCountryNameFromCode = function (countryCode) { - if (typeof(countryCode) === "undefined") return ""; + if (typeof(countryCode) === "undefined") return undefined; try { return regionBundle.GetStringFromName(countryCode.toLowerCase()); } catch (e) { @@ -144,10 +182,13 @@ let showCircuitDisplay = function (show) { // `"France (12.34.56.78)"`. let nodeLines = function (nodeData) { let result = ["This browser"]; - for (let {ip, country} of nodeData) { - result.push(localizedCountryNameFromCode(country) + " (" + ip + ")"); + for (let {ip, countryCode, type} of nodeData) { + let bridge = type === "bridge"; + result.push((countryCode ? localizedCountryNameFromCode(countryCode) + : "Unknown country") + + " (" + (bridge ? "Bridge" : (ip || "IP unknown")) + ")"); } - result[4] = ("Internet"); + result[4] = "Internet"; return result; }; @@ -207,7 +248,9 @@ let syncDisplayWithSelectedTab = (function() { updateCircuitDisplay(); } else { // Stop syncing. - gBrowser.tabContainer.removeEventListener("TabSelect", listener1); + if (gBrowser.tabContainer) { + gBrowser.tabContainer.removeEventListener("TabSelect", listener1); + } gBrowser.removeTabsProgressListener(listener2); // Hide the display. showCircuitDisplay(false); @@ -269,6 +312,7 @@ let setupDisplay = function (host, port, password, enablePrefName) { logger.eclog(5, "Disabling tor display circuit because of an error."); stop(); }); + readBridgeIPs(myController); syncDisplayWithSelectedTab(true); collectIsolationData(myController); } diff --git a/src/modules/tor-control-port.js b/src/modules/tor-control-port.js index eae62f5..d356ebb 100644 --- a/src/modules/tor-control-port.js +++ b/src/modules/tor-control-port.js @@ -23,10 +23,16 @@ let {classes: Cc, interfaces: Ci, results: Cr, Constructor: CC, utils: Cu } = Co // ### Import Mozilla Services Cu.import("resource://gre/modules/Services.jsm"); -// ## torbutton logger -let logger = Cc["@torproject.org/torbutton-logger;1"] - .getService(Components.interfaces.nsISupports).wrappedJSObject, - log = x => logger.eclog(3, x); +// __log__. +// Logging function +let log; +if ((typeof console) !== "undefined") { + log = x => console.log(typeof(x) === "string" ? x : JSON.stringify(x)); +} else { + let logger = Cc["@torproject.org/torbutton-logger;1"] + .getService(Components.interfaces.nsISupports).wrappedJSObject; + log = x => logger.eclog(3, x); +} // ### announce this file log("Loading tor-control-port.js\n"); @@ -58,8 +64,8 @@ io.asyncSocketStreams = function (host, port) { // asynchronously pumps incoming data to the onInputData callback. io.pumpInputStream = function (inputStream, onInputData, onError) { // Wrap raw inputStream with a "ScriptableInputStream" so we can read incoming data. - let ScriptableInputStream = CC("@mozilla.org/scriptableinputstream;1", - "nsIScriptableInputStream", "init"), + let ScriptableInputStream = Components.Constructor( + "@mozilla.org/scriptableinputstream;1", "nsIScriptableInputStream", "init"), scriptableInputStream = new ScriptableInputStream(inputStream), // A private method to read all data available on the input stream. readAll = function() { @@ -171,11 +177,12 @@ io.onLineFromOnMessage = function (onMessage) { }; // __io.callbackDispatcher()__. -// Returns [onString, dispatcher] where the latter is an object with two member functions: -// dispatcher.addCallback(regex, callback), and dispatcher.removeCallback(callback). -// Pass onString to another function that needs a callback with a single string argument. -// Whenever dispatcher.onString receives a string, the dispatcher will check for any -// regex matches and pass the string on to the corresponding callback(s). +// Returns dispatcher object with three member functions: +// dispatcher.addCallback(regex, callback), and dispatcher.removeCallback(callback), +// and dispatcher.pushMessage(message). +// Pass pushMessage to another function that needs a callback with a single string +// argument. Whenever dispatcher.pushMessage receives a string, the dispatcher will +// check for any regex matches and pass the string on to the corresponding callback(s). io.callbackDispatcher = function () { let callbackPairs = [], removeCallback = function (aCallback) { @@ -189,34 +196,38 @@ io.callbackDispatcher = function () { } return function () { removeCallback(callback); }; }, - onString = function (message) { + pushMessage = function (message) { for (let [regex, callback] of callbackPairs) { if (message.match(regex)) { callback(message); } } }; - return [onString, {addCallback : addCallback, removeCallback : removeCallback}]; + return { pushMessage : pushMessage, removeCallback : removeCallback, + addCallback : addCallback }; }; -// __io.matchRepliesToCommands(asyncSend)__. -// Takes asyncSend(message), an asynchronous send function, and returns two functions -// sendCommand(command, replyCallback) and onReply(response). If we call sendCommand, -// then when onReply is called, the corresponding replyCallback will be called. -io.matchRepliesToCommands = function (asyncSend) { +// __io.matchRepliesToCommands(asyncSend, dispatcher)__. +// Takes asyncSend(message), an asynchronous send function, and the callback +// displatcher, and returns a function Promise<response> sendCommand(command). +io.matchRepliesToCommands = function (asyncSend, dispatcher) { let commandQueue = [], - sendCommand = function (command, replyCallback) { - commandQueue.push([command, replyCallback]); + sendCommand = function (command, replyCallback, errorCallback) { + commandQueue.push([command, replyCallback, errorCallback]); asyncSend(command); - }, - onReply = function (reply) { - let [command, replyCallback] = commandQueue.shift(); - if (replyCallback) { replyCallback(reply); } - }, - onFailure = function () { - commandQueue.shift(); }; - return [sendCommand, onReply, onFailure]; + // Watch for responses (replies or error messages) + dispatcher.addCallback(/^[245]\d\d/, function (message) { + let [command, replyCallback, errorCallback] = commandQueue.shift(); + if (message.match(/^2/) && replyCallback) replyCallback(message); + if (message.match(/^[45]/) && errorCallback) { + errorCallback(new Error(command + " -> " + message)); + } + }); + // Create and return a version of sendCommand that returns a Promise. + return command => new Promise(function (replyCallback, errorCallback) { + sendCommand(command, replyCallback, errorCallback); + }); }; // __io.controlSocket(host, port, password, onError)__. @@ -228,7 +239,7 @@ io.matchRepliesToCommands = function (asyncSend) { // let socket = controlSocket("127.0.0.1", 9151, "MyPassw0rd", // function (error) { console.log(error.message || error); }); // // Send command and receive "250" reply or error message -// socket.sendCommand(commandText, replyCallback); +// socket.sendCommand(commandText, replyCallback, errorCallback); // // Register or deregister for "650" notifications // // that match regex // socket.addNotificationCallback(regex, callback); @@ -237,26 +248,20 @@ io.matchRepliesToCommands = function (asyncSend) { // socket.close(); io.controlSocket = function (host, port, password, onError) { // Produce a callback dispatcher for Tor messages. - let [onMessage, mainDispatcher] = io.callbackDispatcher(), + let mainDispatcher = io.callbackDispatcher(), // Open the socket and convert format to Tor messages. socket = io.asyncSocket(host, port, - io.onDataFromOnLine(io.onLineFromOnMessage(onMessage)), + io.onDataFromOnLine( + io.onLineFromOnMessage(mainDispatcher.pushMessage)), onError), // Tor expects any commands to be terminated by CRLF. writeLine = function (text) { socket.write(text + "\r\n"); }, - // Ensure we return the correct reply for each sendCommand. - [sendCommand, onReply, onFailure] = io.matchRepliesToCommands(writeLine), + // Create a sendCommand method from writeLine. + sendCommand = io.matchRepliesToCommands(writeLine, mainDispatcher), // Create a secondary callback dispatcher for Tor notification messages. - [onNotification, notificationDispatcher] = io.callbackDispatcher(); - // Pass successful reply back to sendCommand callback. - mainDispatcher.addCallback(/^2\d\d/, onReply); - // Pass error message to sendCommand callback. - mainDispatcher.addCallback(/^[45]\d\d/, function (message) { - onFailure(); - onError(new Error(message)); - }); + notificationDispatcher = io.callbackDispatcher(); // Pass asynchronous notifications to notification dispatcher. - mainDispatcher.addCallback(/^650/, onNotification); + mainDispatcher.addCallback(/^650/, notificationDispatcher.pushMessage); // Log in to control port. sendCommand("authenticate " + (password || "")); // Activate needed events. @@ -310,6 +315,16 @@ utils.splitLines = function (string) { return string.split(/\r?\n/); }; // inside pairs of quotation marks. utils.splitAtSpaces = utils.extractor(/((\S*?"(.*?)")+\S*|\S+)/g); +// __utils.splitAtFirst(string, regex)__. +// Splits a string at the first instance of regex match. If no match is +// found, returns the whole string. +utils.splitAtFirst = function (string, regex) { + let match = string.match(regex); + return match ? [ string.substring(0, match.index), + string.substring(match.index + match[0].length) ] + : string; +}; + // __utils.splitAtEquals(string)__. // Splits a string into chunks between equals. Does not split at equals // inside pairs of quotation marks. @@ -321,7 +336,7 @@ utils.splitAtEquals = utils.extractor(/(([^=]*?"(.*?)")+[^=]*|[^=]+)/g); utils.mergeObjects = function (arrayOfObjects) { let result = {}; for (let obj of arrayOfObjects) { - for (var key in obj) { + for (let key in obj) { result[key] = obj[key]; } } @@ -433,6 +448,20 @@ info.streamStatusParser = function (text) { "CircuitID", "Target"]); }; +// __info.configTextParser(text)__. +// Parse the output of a `getinfo config-text`. +info.configTextParser = function(text) { + let result = {}; + utils.splitLines(text).map(function(line) { + let [name, value] = utils.splitAtFirst(line, /\s/); + if (name) { + if (!result.hasOwnProperty(name)) result[name] = []; + result[name].push(value); + } + }); + return result; +}; + // __info.parsers__. // A map of GETINFO keys to parsing function, which convert result strings to JavaScript // data. @@ -440,7 +469,7 @@ info.parsers = { "version" : utils.identity, "config-file" : utils.identity, "config-defaults-file" : utils.identity, - "config-text" : utils.identity, + "config-text" : info.configTextParser, "ns/id/" : info.routerStatusParser, "ns/name/" : info.routerStatusParser, "ip-to-country/" : utils.identity, @@ -471,10 +500,9 @@ info.stringToValue = function (string) { return info.getParser(key)(valueString); }; -// __info.getInfoMultiple(aControlSocket, keys, onData)__. -// Sends GETINFO for an array of keys. Passes onData an array of their respective results, -// in order. -info.getInfoMultiple = function (aControlSocket, keys, onData) { +// __info.getInfoMultiple(aControlSocket, keys)__. +// Sends GETINFO for an array of keys. Returns a promise with an array of results. +info.getInfoMultiple = function (aControlSocket, keys) { /* if (!(keys instanceof Array)) { throw new Error("keys argument should be an array"); @@ -490,14 +518,14 @@ info.getInfoMultiple = function (aControlSocket, keys, onData) { throw new Error("unsupported key"); } */ - aControlSocket.sendCommand("getinfo " + keys.join(" "), function (message) { - onData(info.keyValueStringsFromMessage(message).map(info.stringToValue)); - }); + return aControlSocket.sendCommand("getinfo " + keys.join(" ")) + .then(message => info.keyValueStringsFromMessage(message) + .map(info.stringToValue)); }; -// __info.getInfo(controlSocket, key, onValue)__. -// Sends GETINFO for a single key. Passes onValue the value for that key. -info.getInfo = function (aControlSocket, key, onValue) { +// __info.getInfo(controlSocket, key)__. +// Sends GETINFO for a single key. Returns a promise with the result. +info.getInfo = function (aControlSocket, key) { /* if (!utils.isString(key)) { throw new Error("key argument should be a string"); @@ -506,9 +534,7 @@ info.getInfo = function (aControlSocket, key, onValue) { throw new Error("onValue argument should be a function"); } */ - info.getInfoMultiple(aControlSocket, [key], function (data) { - onValue(data[0]); - }); + return info.getInfoMultiple(aControlSocket, [key]).then(data => data[0]); }; // ## event @@ -559,10 +585,8 @@ tor.controllerCache = {}; tor.controller = function (host, port, password, onError) { let socket = io.controlSocket(host, port, password, onError), isOpen = true; - return { getInfo : function (key, log) { info.getInfo(socket, key, log); } , - getInfoMultiple : function (keys, log) { - info.getInfoMultiple(socket, keys, log); - }, + return { getInfo : key => info.getInfo(socket, key), + getInfoMultiple : keys => info.getInfoMultiple(socket, keys), watchEvent : function (type, filter, onData) { event.watchEvent(socket, type, filter, onData); },

1 0

[torbutton/master] Bug #13672. bind tor circuit display to a pref
by gk＠torproject.org 02 Dec '14

02 Dec '14

commit bdc987b01c977c177d567eb850ce4c98d02e3538 Author: Arthur Edelstein <arthuredelstein(a)gmail.com> Date: Wed Nov 5 23:21:59 2014 -0800 Bug #13672. bind tor circuit display to a pref --- src/chrome/content/tor-circuit-display.js | 256 +++++++++++++++++++---------- src/chrome/content/torbutton.js | 3 +- src/defaults/preferences/preferences.js | 1 + src/modules/tor-control-port.js | 33 ++-- 4 files changed, 188 insertions(+), 105 deletions(-) diff --git a/src/chrome/content/tor-circuit-display.js b/src/chrome/content/tor-circuit-display.js index 49cd832..0817aa6 100644 --- a/src/chrome/content/tor-circuit-display.js +++ b/src/chrome/content/tor-circuit-display.js @@ -15,9 +15,12 @@ /* global document, gBrowser, Components */ // ### Main function -// __runTorCircuitDisplay(host, port, password)__. -// The single function we run to activate automatic display of the Tor circuit.. -let runTorCircuitDisplay = (function () { +// __createTorCircuitDisplay(host, port, password, enablePrefName)__. +// The single function that prepares tor circuit display. Connects to a tor +// control port with the given host, port, and password, and binds to +// a named bool pref whose value determines whether the circuit display +// is enabled or disabled. +let createTorCircuitDisplay = (function () { "use strict"; @@ -32,28 +35,12 @@ let { controller } = Cu.import("resource://torbutton/modules/tor-control-port.js let logger = Cc["@torproject.org/torbutton-logger;1"] .getService(Components.interfaces.nsISupports).wrappedJSObject; -// __regionBundle__. -// A list of localized region (country) names. -let regionBundle = Services.strings.createBundle( - "chrome://global/locale/regionNames.properties"); - -// __localizedCountryNameFromCode(countryCode)__. -// Convert a country code to a localized country name. -// Example: `'de'` -> `'Deutschland'` in German locale. -let localizedCountryNameFromCode = function (countryCode) { - if (typeof(countryCode) === "undefined") return ""; - try { - return regionBundle.GetStringFromName(countryCode.toLowerCase()); - } catch (e) { - return countryCode.toUpperCase(); - } -}; +// ## Circuit/stream domain and node monitoring -// __domainToNodeDataMap__. // A mutable map that stores the current nodes for each domain. -let domainToNodeDataMap = {}; - -let knownCircuitIDs = {}; +let domainToNodeDataMap = {}, + // A mutable map that records what circuits are already known. + knownCircuitIDs = {}; // __trimQuotes(s)__. // Removes quotation marks around a quoted string. @@ -86,6 +73,70 @@ let nodeDataForCircuit = function (controller, circuitEvent, onResult) { nodeDataForID(controller, ids, onResult); }; +// __getCircuitStatusByID(aController, circuitID, onCircuitStatus)__ +// Returns the circuit status for the circuit with the given ID +// via onCircuitStatus(status). +let getCircuitStatusByID = function(aController, circuitID, onCircuitStatus) { + aController.getInfo("circuit-status", function (circuitStatuses) { + for (let circuitStatus of circuitStatuses) { + if (circuitStatus.id === circuitID) { + onCircuitStatus(circuitStatus); + } + } + }); +}; + +// __collectIsolationData(aController)__. +// Watches for STREAM SENTCONNECT events. When a SENTCONNECT event occurs, then +// we assume isolation settings (SOCKS username+password) are now fixed for the +// corresponding circuit. Whenever the first stream on a new circuit is seen, +// looks up u+p and records the node data in the domainToNodeDataMap. +let collectIsolationData = function (aController) { + aController.watchEvent( + "STREAM", + streamEvent => streamEvent.StreamStatus === "SENTCONNECT", + function (streamEvent) { + if (!knownCircuitIDs[streamEvent.CircuitID]) { + logger.eclog(3, "streamEvent.CircuitID: " + streamEvent.CircuitID); + knownCircuitIDs[streamEvent.CircuitID] = true; + getCircuitStatusByID(aController, streamEvent.CircuitID, function (circuitStatus) { + let domain = trimQuotes(circuitStatus.SOCKS_USERNAME); + if (domain) { + nodeDataForCircuit(aController, circuitStatus, function (nodeData) { + domainToNodeDataMap[domain] = nodeData; + }); + } + }); + } + }); +}; + +// ## User interface + +// __regionBundle__. +// A list of localized region (country) names. +let regionBundle = Services.strings.createBundle( + "chrome://global/locale/regionNames.properties"); + +// __localizedCountryNameFromCode(countryCode)__. +// Convert a country code to a localized country name. +// Example: `'de'` -> `'Deutschland'` in German locale. +let localizedCountryNameFromCode = function (countryCode) { + if (typeof(countryCode) === "undefined") return ""; + try { + return regionBundle.GetStringFromName(countryCode.toLowerCase()); + } catch (e) { + return countryCode.toUpperCase(); + } +}; + +// __showCircuitDisplay(show)__. +// If show === true, makes the circuit display visible. +let showCircuitDisplay = function (show) { + document.querySelector("svg#tor-circuit").style.display = show ? + 'block' : 'none'; +}; + // __nodeLines(nodeData)__. // Takes a nodeData array of three items each like // `{ ip : "12.34.56.78", country : "fr" }` @@ -121,7 +172,6 @@ let updateCircuitDisplay = function () { document.querySelector("svg#tor-circuit text#domain").innerHTML = "(" + domain + "):"; // Update the displayed information for the relay nodes. let diagramNodes = document.querySelectorAll("svg#tor-circuit text.node-text"), - //diagramCircles = document.querySelectorAll("svg#tor-circuit .node-circule"), lines = nodeLines(nodeData); for (let i = 0; i < diagramNodes.length; ++i) { let line = lines[i]; @@ -130,85 +180,109 @@ let updateCircuitDisplay = function () { } } // Only show the Tor circuit if we have a domain and node data. - document.querySelector("svg#tor-circuit").style.display = (domain && nodeData) ? - 'block' : 'none'; + showCircuitDisplay(domain && nodeData); } }; -// __getCircuitStatusByID(aController, circuitID, onCircuitStatus)__ -// Returns the circuit status for the circuit with the given ID -// via onCircuitStatus(status). -let getCircuitStatusByID = function(aController, circuitID, onCircuitStatus) { - aController.getInfo("circuit-status", function (circuitStatuses) { - for (let circuitStatus of circuitStatuses) { - if (circuitStatus.id === circuitID) { - onCircuitStatus(circuitStatus); - } - } - }); -}; - -// __collectIsolationData(aController)__. -// Watches for STREAM SENTCONNECT events. When a SENTCONNECT event occurs, then -// the isolation settings (SOCKS username+password) become fixed for the -// corresponding circuit. Whenever the first stream on a new circuit is seen, -// looks up u+p and records the node data in the domainToNodeDataMap. -let collectIsolationData = function (aController) { - aController.watchEvent( - "STREAM", - streamEvent => streamEvent.StreamStatus === "SENTCONNECT", - function (streamEvent) { - if (!knownCircuitIDs[streamEvent.CircuitID]) { - logger.eclog(4, "streamEvent.CircuitID: " + streamEvent.CircuitID); - knownCircuitIDs[streamEvent.CircuitID] = true; - getCircuitStatusByID(aController, streamEvent.CircuitID, function (circuitStatus) { - let domain = trimQuotes(circuitStatus.SOCKS_USERNAME); - if (domain) { - nodeDataForCircuit(aController, circuitStatus, function (nodeData) { - domainToNodeDataMap[domain] = nodeData; - updateCircuitDisplay(); - }); - } else { - updateCircuitDisplay(); - } - }); - } - }); -}; - -// __syncDisplayWithSelectedTab()__. +// __syncDisplayWithSelectedTab(syncOn)__. // We may have multiple tabs, but there is only one instance of TorButton's popup // panel for displaying the Tor circuit UI. Therefore we need to update the display // to show the currently selected tab at its current location. -let syncDisplayWithSelectedTab = function () { - // Whenever a different tab is selected, change the circuit display - // to show the circuit for that tab's domain. - gBrowser.tabContainer.addEventListener("TabSelect", function (event) { - updateCircuitDisplay(); - }); - // If the currently selected tab has been sent to a new location, - // update the circuit to reflect that. - gBrowser.addTabsProgressListener({ onLocationChange : function (aBrowser) { - if (aBrowser == gBrowser.selectedBrowser) { +let syncDisplayWithSelectedTab = (function() { + let listener1 = event => { updateCircuitDisplay(); }, + listener2 = { onLocationChange : function (aBrowser) { + if (aBrowser === gBrowser.selectedBrowser) { + updateCircuitDisplay(); + } + } }; + return function (syncOn) { + if (syncOn) { + // Whenever a different tab is selected, change the circuit display + // to show the circuit for that tab's domain. + gBrowser.tabContainer.addEventListener("TabSelect", listener1); + // If the currently selected tab has been sent to a new location, + // update the circuit to reflect that. + gBrowser.addTabsProgressListener(listener2); + // Get started with a correct display. updateCircuitDisplay(); + } else { + // Stop syncing. + gBrowser.tabContainer.removeEventListener("TabSelect", listener1); + gBrowser.removeTabsProgressListener(listener2); + // Hide the display. + showCircuitDisplay(false); } - } }); + }; +})(); + +// ## Pref utils - // Get started with a correct display. - updateCircuitDisplay(); +// __prefs__. A shortcut to Mozilla Services.prefs. +let prefs = Services.prefs; + +// __getPrefValue(prefName)__ +// Returns the current value of a preference, regardless of its type. +let getPrefValue = function (prefName) { + switch(prefs.getPrefType(prefName)) { + case prefs.PREF_BOOL: return prefs.getBoolPref(prefName); + case prefs.PREF_INT: return prefs.getIntPref(prefName); + case prefs.PREF_STRING: return prefs.getCharPref(prefName); + default: return null; + } }; -// __display(host, port, password)__. -// The main function for activating automatic display of the Tor circuit. -// A reference to this function (called runTorCircuitDisplay) is exported as a global. -let display = function (host, port, password) { - let myController = controller(host, port || 9151, password, function (x) { logger.eclog(5, x); }); - syncDisplayWithSelectedTab(); - collectIsolationData(myController); +// __bindPrefAndInit(prefName, prefHandler)__ +// Applies prefHandler to the current value of pref specified by prefName. +// Re-applies prefHandler whenever the value of the pref changes. +// Returns a zero-arg function that unbinds the pref. +let bindPrefAndInit = function (prefName, prefHandler) { + let update = () => { prefHandler(getPrefValue(prefName)); }, + observer = { observe : function (subject, topic, data) { + if (data === prefName) { + update(); + } + } }; + prefs.addObserver(prefName, observer, false); + update(); + return () => { prefs.removeObserver(prefName, observer); }; }; -return display; +// ## Main function -// Finish runTorCircuitDisplay() -})(); +// setupDisplay(host, port, password, enablePrefName)__. +// Returns a function that lets you start/stop automatic display of the Tor circuit. +// A reference to this function (called createTorCircuitDisplay) is exported as a global. +let setupDisplay = function (host, port, password, enablePrefName) { + let myController = null, + stop = function() { + if (myController) { + syncDisplayWithSelectedTab(false); + myController.close(); + myController = null; + } + }, + start = function () { + if (!myController) { + myController = controller(host, port || 9151, password, function (err) { + // An error has occurred. + logger.eclog(5, err); + logger.eclog(5, "Disabling tor display circuit because of an error."); + stop(); + }); + syncDisplayWithSelectedTab(true); + collectIsolationData(myController); + } + }; + try { + let unbindPref = bindPrefAndInit(enablePrefName, on => { if (on) start(); else stop(); }); + // When this chrome window is unloaded, we need to unbind the pref. + window.addEventListener("unload", unbindPref); + } catch (e) { + logger.eclog(5, "Error: " + e.message + "\n" + e.stack); + } +}; + +return setupDisplay; +// Finish createTorCircuitDisplay() +})(); diff --git a/src/chrome/content/torbutton.js b/src/chrome/content/torbutton.js index 2e921da..8dba1fd 100644 --- a/src/chrome/content/torbutton.js +++ b/src/chrome/content/torbutton.js @@ -619,7 +619,8 @@ function torbutton_init() { torbutton_update_statusbar(mode); torbutton_notify_if_update_needed(); - runTorCircuitDisplay(m_tb_control_host, m_tb_control_port, m_tb_control_pass); + createTorCircuitDisplay(m_tb_control_host, m_tb_control_port, m_tb_control_pass, + "extensions.torbutton.display_circuit"); torbutton_log(3, 'init completed'); } diff --git a/src/defaults/preferences/preferences.js b/src/defaults/preferences/preferences.js index d527d7e..3e02b77 100644 --- a/src/defaults/preferences/preferences.js +++ b/src/defaults/preferences/preferences.js @@ -4,6 +4,7 @@ pref("extensions.torbutton.loglevel",4); pref("extensions.torbutton.logmethod",1); // 0=stdout, 1=errorconsole, 2=debuglog // Display prefs +pref("extensions.torbutton.display_circuit", true); pref("extensions.torbutton.display_panel",true); pref("extensions.torbutton.panel_style",'text'); pref("extensions.torbutton(a)torproject.org.description", "chrome://torbutton/locale/torbutton.properties"); diff --git a/src/modules/tor-control-port.js b/src/modules/tor-control-port.js index ada80b5..eae62f5 100644 --- a/src/modules/tor-control-port.js +++ b/src/modules/tor-control-port.js @@ -156,15 +156,16 @@ io.onLineFromOnMessage = function (onMessage) { // Add to the list of pending lines. pendingLines.push(line); // If line is the last in a message, then pass on the full multiline message. - if (line.match(/^\d\d\d /) && (pendingLines.length == 1 || - pendingLines[0].startsWith(line.substring(0,3)))) { + if (line.match(/^\d\d\d /) && + (pendingLines.length === 1 || + pendingLines[0].substring(0,3) === line.substring(0,3))) { // Combine pending lines to form message. let message = pendingLines.join("\r\n"); + log("controlPort >> " + message); // Wipe pendingLines before we call onMessage, in case onMessage throws an error. pendingLines = []; // Pass multiline message to onMessage. onMessage(message); - log("controlPort >> " + message); } }; }; @@ -547,11 +548,17 @@ event.watchEvent = function (controlSocket, type, filter, onData) { // Things related to the main controller. let tor = tor || {}; +// __tor.controllerCache__. +// A map from "host:port" to controller objects. Prevents redundant instantiation +// of control sockets. +tor.controllerCache = {}; + // __tor.controller(host, port, password, onError)__. // Creates a tor controller at the given host and port, with the given password. // onError returns asynchronously whenever a connection error occurs. tor.controller = function (host, port, password, onError) { - let socket = io.controlSocket(host, port, password, onError); + let socket = io.controlSocket(host, port, password, onError), + isOpen = true; return { getInfo : function (key, log) { info.getInfo(socket, key, log); } , getInfoMultiple : function (keys, log) { info.getInfoMultiple(socket, keys, log); @@ -559,14 +566,11 @@ tor.controller = function (host, port, password, onError) { watchEvent : function (type, filter, onData) { event.watchEvent(socket, type, filter, onData); }, - close : socket.close }; + isOpen : () => isOpen, + close : () => { isOpen = false; socket.close(); } + }; }; -// __tor.controllerCache__. -// A map from "host:port" to controller objects. Prevents redundant instantiation -// of control sockets. -tor.controllerCache = {}; - // ## Export // __controller(host, port, password, onError)__. @@ -584,9 +588,12 @@ tor.controllerCache = {}; // // Close the controller permanently // c.close(); let controller = function (host, port, password, onError) { - let dest = host + ":" + port; - return (tor.controllerCache[dest] = tor.controllerCache[dest] || - tor.controller(host, port, password, onError)); + let dest = host + ":" + port, + maybeController = tor.controllerCache[dest]; + return (tor.controllerCache[dest] = + (maybeController && maybeController.isOpen()) ? + maybeController : + tor.controller(host, port, password, onError)); }; // Export the controller function for external use.

1 0

[tor-browser-bundle/master] Bump versions for nightlies.
by gk＠torproject.org 02 Dec '14

02 Dec '14

commit 6800ca4a0e89a59afecae2b931ab6795b3292eaf Author: Georg Koppen <gk(a)torproject.org> Date: Tue Dec 2 13:25:47 2014 +0000 Bump versions for nightlies. --- gitian/versions.nightly | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gitian/versions.nightly b/gitian/versions.nightly index e6ea2a1..1b6aff4 100755 --- a/gitian/versions.nightly +++ b/gitian/versions.nightly @@ -4,7 +4,7 @@ BUILD_PT_BUNDLES=1 VERIFY_TAGS=0 -FIREFOX_VERSION=31.2.0esr +FIREFOX_VERSION=31.3.0esr TORBROWSER_UPDATE_CHANNEL=default @@ -51,13 +51,13 @@ PY2EXE_VER=0.6.9 SETUPTOOLS_VER=1.4 LXML_VER=3.3.5 PARSLEY_VER=1.2 -HTTPSE_VER=5.0development.0 +HTTPSE_VER=5.0development.1 GO_VER=1.3.3 ## File names for the source packages OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz GMP_PACKAGE=gmp-${GMP_VER}.tar.bz2 -NOSCRIPT_PACKAGE=noscript_security_suite-2.6.9.1-fn+fx+sm.xpi +NOSCRIPT_PACKAGE=noscript_security_suite-2.6.9.5-sm+fn+fx.xpi HTTPSE_PACKAGE=https-everywhere-${HTTPSE_VER}.xpi TOOLCHAIN4_PACKAGE=x86_64-apple-darwin10.tar.xz TOOLCHAIN4_OLD_PACKAGE=multiarch-darwin11-cctools127.2-gcc42-5666.3-llvmgcc42-2336.1-Linux-120724.tar.xz @@ -87,8 +87,8 @@ GMP_HASH=752079520b4690531171d0f4532e40f08600215feefede70b24fabdc6f1ab160 OSXSDK_HASH=6602d8d5ddb371fbc02e2a5967d9bd0cd7358d46f9417753c8234b923f2ea6fc TOOLCHAIN4_HASH=7b71bfe02820409b994c5c33a7eab81a81c72550f5da85ff7af70da3da244645 TOOLCHAIN4_OLD_HASH=65c1b2d302358a6b95a26c6828a66908a199276193bb0b268f2dcc1a997731e9 -NOSCRIPT_HASH=92bfa1f8a5673ffc82fe54db43c5f410c1cf403e38b82780f01eeeeae8233d0b -HTTPSE_HASH=6feb4982d13f89656b61435b6b49e1996c61bc79e52d046d02312879376028e3 +NOSCRIPT_HASH=6db840d4ecf60985aa9eb7ef0142e67d78503d76a778f8f4e7e33e1810a2701 +HTTPSE_HASH=d97ec822641d4ebf404fee1179497e1d80f2b8344d5fa485169ef045ba184db0 MSVCR100_HASH=1221a09484964a6f38af5e34ee292b9afefccb3dc6e55435fd3aaf7c235d9067 PYCRYPTO_HASH=f2ce1e989b272cfcb677616763e0a2e7ec659effa67a88aa92b3a65528f60a3c ARGPARSE_HASH=ddaf4b0a618335a32b6664d4ae038a1de8fbada3b25033f9021510ed2b3941a4

1 0

[tor-browser-bundle/master] Bump lang pack version, too.
by gk＠torproject.org 02 Dec '14

02 Dec '14

commit ad84f4205a95dfdbf72a9949014c469c3316f72c Author: Georg Koppen <gk(a)torproject.org> Date: Tue Dec 2 13:29:02 2014 +0000 Bump lang pack version, too. --- gitian/versions.nightly | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitian/versions.nightly b/gitian/versions.nightly index 1b6aff4..2ba0141 100755 --- a/gitian/versions.nightly +++ b/gitian/versions.nightly @@ -37,7 +37,7 @@ GITIAN_TAG=tor-browser-builder-3.x-7 OPENSSL_VER=1.0.1i GMP_VER=5.1.3 FIREFOX_LANG_VER=$FIREFOX_VERSION -FIREFOX_LANG_BUILD=build1 +FIREFOX_LANG_BUILD=build2 BINUTILS_VER=2.24 GCC_VER=4.9.1 PYTHON_VER=2.7.5

1 0

[tor-browser-bundle/master] Peparing 4.5-alpha-2.
by gk＠torproject.org 02 Dec '14

02 Dec '14

commit 7af137f526da8495bbaebffc59077f2e9422d71a Author: Georg Koppen <gk(a)torproject.org> Date: Tue Dec 2 13:55:13 2014 +0000 Peparing 4.5-alpha-2. Bump Torbutton, NoScript and HTTPS Everywhere version. Update config.yml and ChangeLog. --- Bundle-Data/Docs/ChangeLog.txt | 30 ++++++++++++++++++++++++++++++ gitian/versions.alpha | 18 +++++++++--------- tools/update-responses/config.yml | 19 +++++++++---------- 3 files changed, 48 insertions(+), 19 deletions(-) diff --git a/Bundle-Data/Docs/ChangeLog.txt b/Bundle-Data/Docs/ChangeLog.txt index f226adc..f1511b0 100644 --- a/Bundle-Data/Docs/ChangeLog.txt +++ b/Bundle-Data/Docs/ChangeLog.txt @@ -1,3 +1,33 @@ +Tor Browser 4.5-alpha-2 -- Dec 5 2014 + * All Platforms + * Update Firefox to 31.3.0esr + * Update NoScript to 2.6.9.5 + * Update HTTPS Everywhere to 5.0development.1 + * Update Torbutton to 1.8.1.2 + * Bug 13672: Make circuit display optional + * Bug 13671: Make bridges visible on circuit display + * Bug 9387: Incorporate user feedback + * Bug 13784: Remove third party authentication tokens + +Tor Browser 4.0.2 -- Dec 2 2014 + * All Platforms + * Update Firefox to 31.3.0esr + * Update NoScript to 2.6.9.5 + * Update HTTPS Everywhere to 4.0.2 + * Update Torbutton to 1.7.0.2 + * Bug 13019: Synchronize locale spoofing pref with our Firefox patch + * Bug 13746: Properly link Torbutton UI to thirdparty pref. + * Bug 13742: Fix domain isolation for content cache and disk-enabled +browsing mode + * Bug 5926: Prevent JS engine locale leaks (by setting the C library +locale) + * Bug 13504: Remove unreliable/unreachable non-public bridges + * Bug 13435: Remove our custom POODLE fix + * Windows + * Bug 13443: Re-enable DirectShow; fix crash with mingw patch. + * Bug 13558: Fix crash on Windows XP during download folder changing + * Bug 13594: Fix update failure for Windows XP users + Tor Browser 4.5-alpha-1 -- Nov 14 2014 * All Platforms * Bug 3455: Patch Firefox SOCKS and proxy filters to allow user+pass isolation diff --git a/gitian/versions.alpha b/gitian/versions.alpha index edb59c2..a25af43 100755 --- a/gitian/versions.alpha +++ b/gitian/versions.alpha @@ -1,17 +1,17 @@ -TORBROWSER_VERSION=4.5-alpha-1 +TORBROWSER_VERSION=4.5-alpha-2 BUNDLE_LOCALES="ar de es-ES fa fr it ko nl pl pt-PT ru tr vi zh-CN" BUILD_PT_BUNDLES=1 VERIFY_TAGS=1 -FIREFOX_VERSION=31.2.0esr +FIREFOX_VERSION=31.3.0esr TORBROWSER_UPDATE_CHANNEL=alpha -TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-4.5-1-build6 +TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-4.5-1-build1 TOR_TAG=tor-0.2.6.1-alpha TORLAUNCHER_TAG=0.2.7.0.1 -TORBUTTON_TAG=1.8.1.1 +TORBUTTON_TAG=1.8.1.2 HTTPSE_TAG=3.5.3 # XXX: HTTPSE_VER is used instead, pending #11630 NSIS_TAG=v0.2 ZLIB_TAG=v1.2.8 @@ -36,7 +36,7 @@ GITIAN_TAG=tor-browser-builder-3.x-7 OPENSSL_VER=1.0.1i GMP_VER=5.1.3 FIREFOX_LANG_VER=$FIREFOX_VERSION -FIREFOX_LANG_BUILD=build1 +FIREFOX_LANG_BUILD=build2 BINUTILS_VER=2.24 GCC_VER=4.9.1 PYTHON_VER=2.7.5 @@ -50,13 +50,13 @@ PY2EXE_VER=0.6.9 SETUPTOOLS_VER=1.4 LXML_VER=3.3.5 PARSLEY_VER=1.2 -HTTPSE_VER=5.0development.0 +HTTPSE_VER=5.0development.1 GO_VER=1.3.3 ## File names for the source packages OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz GMP_PACKAGE=gmp-${GMP_VER}.tar.bz2 -NOSCRIPT_PACKAGE=noscript_security_suite-2.6.9.3-sm+fn+fx.xpi +NOSCRIPT_PACKAGE=noscript_security_suite-2.6.9.5-sm+fn+fx.xpi HTTPSE_PACKAGE=https-everywhere-${HTTPSE_VER}.xpi TOOLCHAIN4_PACKAGE=x86_64-apple-darwin10.tar.xz TOOLCHAIN4_OLD_PACKAGE=multiarch-darwin11-cctools127.2-gcc42-5666.3-llvmgcc42-2336.1-Linux-120724.tar.xz @@ -86,8 +86,8 @@ GMP_HASH=752079520b4690531171d0f4532e40f08600215feefede70b24fabdc6f1ab160 OSXSDK_HASH=6602d8d5ddb371fbc02e2a5967d9bd0cd7358d46f9417753c8234b923f2ea6fc TOOLCHAIN4_HASH=7b71bfe02820409b994c5c33a7eab81a81c72550f5da85ff7af70da3da244645 TOOLCHAIN4_OLD_HASH=65c1b2d302358a6b95a26c6828a66908a199276193bb0b268f2dcc1a997731e9 -NOSCRIPT_HASH=d399d724953fe032ccca758e577c2136b8fd6c377cea0e3f42ed999b796d290a -HTTPSE_HASH=6feb4982d13f89656b61435b6b49e1996c61bc79e52d046d02312879376028e3 +NOSCRIPT_HASH=6db840d4ecf60985aa9eb7ef0142e67d78503d76a778f8f4e7e33e1810a2701 +HTTPSE_HASH=d97ec822641d4ebf404fee1179497e1d80f2b8344d5fa485169ef045ba184db0 MSVCR100_HASH=1221a09484964a6f38af5e34ee292b9afefccb3dc6e55435fd3aaf7c235d9067 PYCRYPTO_HASH=f2ce1e989b272cfcb677616763e0a2e7ec659effa67a88aa92b3a65528f60a3c ARGPARSE_HASH=ddaf4b0a618335a32b6664d4ae038a1de8fbada3b25033f9021510ed2b3941a4 diff --git a/tools/update-responses/config.yml b/tools/update-responses/config.yml index 557b197..6b0018d 100644 --- a/tools/update-responses/config.yml +++ b/tools/update-responses/config.yml @@ -6,20 +6,19 @@ build_targets: osx32: Darwin_x86-gcc3 osx64: Darwin_x86_64-gcc3 channels: - release: 4.0 - alpha: 4.5-alpha-1 + release: 4.0.2 + alpha: 4.5-alpha-2 versions: - 4.0: - platformVersion: 31.2.0 + 4.0.2: + platformVersion: 31.3.0 detailsURL: https://www.torproject.org/projects/torbrowser.html.en - download_url: https://www.torproject.org/dist/torbrowser/4.0 - 4.5-alpha-1: - platformVersion: 31.2.0 + download_url: https://www.torproject.org/dist/torbrowser/4.0.2 + 4.5-alpha-2: + platformVersion: 31.3.0 detailsURL: https://www.torproject.org/projects/torbrowser.html.en - download_url: https://www.torproject.org/dist/torbrowser/4.5-alpha-1 + download_url: https://www.torproject.org/dist/torbrowser/4.5-alpha-2 incremental_from: - - 4.0 - - 4.0.1 + - 4.5-alpha-1 migrate_archs: osx32: osx64 osx32:

1 0

[tor-browser-bundle/maint-4.0] Bumping tor-browser tag for build2.
by gk＠torproject.org 01 Dec '14

01 Dec '14

commit 8501e871a574856e962bc1732dfea05d206c3656 Author: Georg Koppen <gk(a)torproject.org> Date: Mon Dec 1 09:32:56 2014 +0000 Bumping tor-browser tag for build2. --- Bundle-Data/Docs/ChangeLog.txt | 1 + gitian/versions | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Bundle-Data/Docs/ChangeLog.txt b/Bundle-Data/Docs/ChangeLog.txt index 93d20d7..50f7974 100644 --- a/Bundle-Data/Docs/ChangeLog.txt +++ b/Bundle-Data/Docs/ChangeLog.txt @@ -9,6 +9,7 @@ Tor Browser 4.0.2 -- Dec 2 2014 * Bug 13742: Fix domain isolation for content cache and disk-enabled browsing mode * Bug 5926: Prevent JS engine locale leaks (by setting the C library locale) * Bug 13504: Remove unreliable/unreachable non-public bridges + * Bug 13435: Remove our custom POODLE fix * Windows * Bug 13443: Re-enable DirectShow; fix crash with mingw patch. * Bug 13558: Fix crash on Windows XP during download folder changing diff --git a/gitian/versions b/gitian/versions index ad8fd74..6f9d97b 100755 --- a/gitian/versions +++ b/gitian/versions @@ -8,7 +8,7 @@ FIREFOX_VERSION=31.3.0esr TORBROWSER_UPDATE_CHANNEL=release -TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-4.0-1-build1 +TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-4.0-1-build2 TOR_TAG=tor-0.2.5.10 TORLAUNCHER_TAG=0.2.7.0.1 TORBUTTON_TAG=1.7.0.2

1 0

[tor-browser/tor-browser-31.3.0esr-4.0-1] Bug 13435: Remove our custom POODLE fix.
by gk＠torproject.org 01 Dec '14

01 Dec '14

commit 91ef6ad6972aac82b7ce539664a27fd471251733 Author: Georg Koppen <gk(a)torproject.org> Date: Mon Dec 1 09:20:53 2014 +0000 Bug 13435: Remove our custom POODLE fix. --- browser/app/profile/000-tor-browser.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/browser/app/profile/000-tor-browser.js b/browser/app/profile/000-tor-browser.js index 43b894d..db85aa0 100644 --- a/browser/app/profile/000-tor-browser.js +++ b/browser/app/profile/000-tor-browser.js @@ -171,8 +171,6 @@ pref("media.audio_data.enabled", false); // Enable TLS 1.1 and 1.2: // https://trac.torproject.org/projects/tor/ticket/11253 pref("security.tls.version.max", 3); -// POODLE hotfix: Disable SSLv3 -pref("security.tls.version.min", 1); #ifdef TOR_BROWSER_VERSION #expand pref("torbrowser.version", __TOR_BROWSER_VERSION__);

1 0

[tor-browser/tor-browser-31.3.0esr-4.0-1] Revert "Bug 762358 - Re-run configure when mozconfig changed in a significant way. r=gps a=NPOTB"
by gk＠torproject.org 01 Dec '14

01 Dec '14

commit 811add0090f95de6e1fa3700fb55e8bc57802a52 Author: Georg Koppen <gk(a)torproject.org> Date: Mon Dec 1 08:59:35 2014 +0100 Revert "Bug 762358 - Re-run configure when mozconfig changed in a significant way. r=gps a=NPOTB" This reverts commit 87d09ba73a620931ffbe2576064aae55a961b097. It seems it breaks our deterministic build setup running |configure| again during |make build|. The former does not like being run under libfaketime but the latter needs to be in order to get reproducible builds. --- build/autoconf/altoptions.m4 | 43 +------ build/autoconf/mozconfig-find | 76 +++++++++++++ build/autoconf/mozconfig2client-mk | 76 +++++++++++++ build/autoconf/mozconfig2configure | 103 +++++++++++++++++ client.mk | 50 ++++++--- python/mozbuild/mozbuild/base.py | 38 ++----- python/mozbuild/mozbuild/mach_commands.py | 118 ++++---------------- python/mozbuild/mozbuild/mozconfig.py | 101 ++++++----------- python/mozbuild/mozbuild/mozconfig_loader | 8 -- python/mozbuild/mozbuild/test/backend/common.py | 8 -- .../mozbuild/test/frontend/test_emitter.py | 8 -- python/mozbuild/mozbuild/test/test_base.py | 75 +++---------- python/mozbuild/mozbuild/test/test_mozconfig.py | 56 +--------- testing/xpcshell/selftest.py | 1 - 14 files changed, 381 insertions(+), 380 deletions(-) diff --git a/build/autoconf/altoptions.m4 b/build/autoconf/altoptions.m4 index 6c40ad6..3dc8c6e 100644 --- a/build/autoconf/altoptions.m4 +++ b/build/autoconf/altoptions.m4 @@ -116,42 +116,7 @@ echo " dnl MOZ_READ_MYCONFIG() - Read in 'myconfig.sh' file AC_DEFUN([MOZ_READ_MOZCONFIG], [AC_REQUIRE([AC_INIT_BINSH])dnl -inserted= -dnl Shell is hard, so here is what the following does: -dnl - Reset $@ (command line arguments) -dnl - Add the configure options from mozconfig to $@ one by one -dnl - Add the original command line arguments after that, one by one -dnl -dnl There are several tricks involved: -dnl - It is not possible to preserve the whitespaces in $@ by assigning to -dnl another variable, so the two first steps above need to happen in the first -dnl iteration of the third step. -dnl - We always want the configure options to be added, so the loop must be -dnl iterated at least once, so we add a dummy argument first, and discard it. -dnl - something | while read line ... makes the while run in a subshell, meaning -dnl that anything it does is not propagated to the main shell, so we can't do -dnl set -- foo there. As a consequence, what the while loop reading mach -dnl environment output does is output a set of shell commands for the main shell -dnl to eval. -dnl - Extra care is due when lines from mach environment output contain special -dnl shell characters, so we use ' for quoting and ensure no ' end up in between -dnl the quoting mark unescaped. -dnl Some of the above is directly done in mach environment --format=configure. -failed_eval() { - echo "Failed eval'ing the following:" - $(dirname [$]0)/[$1]/mach environment --format=configure - exit 1 -} - -set -- dummy "[$]@" -for ac_option -do - if test -z "$inserted"; then - set -- - eval "$($(dirname [$]0)/[$1]/mach environment --format=configure)" || failed_eval - inserted=1 - else - set -- "[$]@" "$ac_option" - fi -done -]) +# Read in '.mozconfig' script to set the initial options. +# See the mozconfig2configure script for more details. +_AUTOCONF_TOOLS_DIR=`dirname [$]0`/[$1]/build/autoconf +. $_AUTOCONF_TOOLS_DIR/mozconfig2configure]) diff --git a/build/autoconf/mozconfig-find b/build/autoconf/mozconfig-find new file mode 100755 index 0000000..97dd90c --- /dev/null +++ b/build/autoconf/mozconfig-find @@ -0,0 +1,76 @@ +#! /bin/sh +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# mozconfigfind - Loads options from .mozconfig onto configure's +# command-line. The .mozconfig file is searched for in the +# order: +# If $MOZCONFIG is set, use that. +# If one of $TOPSRCDIR/.mozconfig or $TOPSRCDIR/mozconfig exists, use it. +# If both exist, or if various legacy locations contain a mozconfig, error. +# Otherwise, use the default build options. +# +topsrcdir=$1 + +abspath() { + if uname -s | grep -q MINGW; then + # We have no way to figure out whether we're in gmake or pymake right + # now. gmake gives us Unix-style paths while pymake gives us Windows-style + # paths, so attempt to handle both. + regexes='^$[A-Za-z]:\|\\\\\|\/$ ^\/' + else + regexes='^\/' + fi + + for regex in $regexes; do + if echo $1 | grep -q $regex; then + echo $1 + return + fi + done + + # If we're at this point, we have a relative path + echo `pwd`/$1 +} + +if [ -n "$MOZCONFIG" ] && ! [ -f "$MOZCONFIG" ]; then + echo "Specified MOZCONFIG \"$MOZCONFIG\" does not exist!" 1>&2 + exit 1 +fi + +if [ -n "$MOZ_MYCONFIG" ]; then + echo "Your environment currently has the MOZ_MYCONFIG variable set to \"$MOZ_MYCONFIG\". MOZ_MYCONFIG is no longer supported. Please use MOZCONFIG instead." 1>&2 + exit 1 +fi + +if [ -z "$MOZCONFIG" ] && [ -f "$topsrcdir/.mozconfig" ] && [ -f "$topsrcdir/mozconfig" ]; then + echo "Both \$topsrcdir/.mozconfig and \$topsrcdir/mozconfig are supported, but you must choose only one. Please remove the other." 1>&2 + exit 1 +fi + +for _config in "$MOZCONFIG" \ + "$topsrcdir/.mozconfig" \ + "$topsrcdir/mozconfig" +do + if test -f "$_config"; then + abspath $_config + exit 0 + fi +done + +# We used to support a number of other implicit .mozconfig locations. We now +# detect if we were about to use any of these locations and issue an error if we +# find any. +for _config in "$topsrcdir/mozconfig.sh" \ + "$topsrcdir/myconfig.sh" \ + "$HOME/.mozconfig" \ + "$HOME/.mozconfig.sh" \ + "$HOME/.mozmyconfig.sh" +do + if test -f "$_config"; then + echo "You currently have a mozconfig at \"$_config\". This implicit location is no longer supported. Please move it to $topsrcdir/.mozconfig or specify it explicitly via \$MOZCONFIG." 1>&2 + exit 1 + fi +done diff --git a/build/autoconf/mozconfig2client-mk b/build/autoconf/mozconfig2client-mk new file mode 100755 index 0000000..aaf8de1 --- /dev/null +++ b/build/autoconf/mozconfig2client-mk @@ -0,0 +1,76 @@ +#! /bin/sh +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# mozconfig2client-mk - Translates .mozconfig into options for client.mk. +# Prints defines to stdout. +# +# See mozconfig2configure for more details + +print_header() { + cat <<EOF +# gmake +# This file is automatically generated for client.mk. +# Do not edit. Edit $FOUND_MOZCONFIG instead. + +EOF +} + +ac_add_options() { + for _opt + do + case "$_opt" in + --target=*) + echo $_opt | sed s/--target/CONFIG_GUESS/ + ;; + *) + echo "# $_opt is used by configure (not client.mk)" + ;; + esac + done +} + +ac_add_app_options() { + echo "# $* is used by configure (not client.mk)" +} + +mk_add_options() { + for _opt + do + # Escape shell characters, space, tab, dollar, quote, backslash, + # and substitute '@<word>@' with '$(<word>)'. + _opt=`echo "$_opt" | sed -e 's/$[\"\\]$/\\\\\1/g; s/@$[^@]*$@/\$(\1)/g;'` + echo $_opt; + done +} + +# Main +#-------------------------------------------------- + +scriptdir=`dirname $0` +topsrcdir=$1 + +# If the path changes, configure should be rerun +echo "# PATH=$PATH" + +# If FOUND_MOZCONFIG isn't set, look for it and make sure the script doesn't error out +isfoundset=${FOUND_MOZCONFIG+yes} +if [ -z $isfoundset ]; then + FOUND_MOZCONFIG=`$scriptdir/mozconfig-find $topsrcdir` + if [ $? -ne 0 ]; then + echo '$(error Fix above errors before continuing.)' + else + isfoundset=yes + fi +fi + +if [ -n $isfoundset ]; then + if [ "$FOUND_MOZCONFIG" ] + then + print_header + . "$FOUND_MOZCONFIG" + echo "FOUND_MOZCONFIG := $FOUND_MOZCONFIG" + fi +fi diff --git a/build/autoconf/mozconfig2configure b/build/autoconf/mozconfig2configure new file mode 100755 index 0000000..99623b6 --- /dev/null +++ b/build/autoconf/mozconfig2configure @@ -0,0 +1,103 @@ +#! /bin/sh +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# mozconfig2configure - Loads options from .mozconfig onto configure's +# command-line. See mozconfig-find for how the config file is +# found +# +# The options from .mozconfig are inserted into the command-line +# before the real command-line options. This way the real options +# will override any .mozconfig options. +# +# .mozconfig is a shell script. To add an option to configure's +# command-line use the pre-defined function, ac_add_options, +# +# ac_add_options <configure-option> [<configure-option> ... ] +# +# For example, +# +# ac_add_options --with-pthreads --enable-debug +# +# ac_add_options can be called multiple times in .mozconfig. +# Each call adds more options to configure's command-line. + +# Note: $_AUTOCONF_TOOLS_DIR must be defined in the script that includes this. + +ac_add_options() { + for _opt + do + # Escape shell characters, space, tab, dollar, quote, backslash, parentheses. + _opt=`echo $_opt | sed -e 's/$[\ \ \$\"\\\($]\)/\\\\\1/g;s/@$[^@]*$@/\$\1/g;'` + _opt=`echo $_opt | sed -e 's/@$[^@]*$@/\$(\1)/g'` + + # Avoid adding duplicates + case "$ac_options" in + # Note that all options in $ac_options are enclosed in quotes, + # so there will always be a last character to match [^-A-Za-z0-9_] + *"\"$_opt[^-A-Za-z0-9_]"* ) ;; + * ) mozconfig_ac_options="$mozconfig_ac_options $_opt" ;; + esac + done +} + +ac_add_app_options() { + APP=$1 + shift; + if [ "$APP" = "$MOZ_BUILD_APP" ]; then + ac_add_options "$*"; + fi +} + +mk_add_options() { + # These options are for client.mk + # configure can safely ignore them. + : +} + +ac_echo_options() { + echo "Adding configure options from $FOUND_MOZCONFIG:" + eval "set -- $mozconfig_ac_options" + for _opt + do + echo " $_opt" + done +} + +# Main +#-------------------------------------------------- +topsrcdir=$(cd `dirname $0`; pwd -W 2>/dev/null || pwd) +ac_options= +mozconfig_ac_options= + +# Save the real command-line options +for _opt +do + # Escape shell characters, space, tab, dollar, quote, backslash. + _opt=`echo $_opt | sed -e 's/$[\ \ \$\"\\]$/\\\\\1/g;'` + ac_options="$ac_options \"$_opt\"" +done + + +# If FOUND_MOZCONFIG isn't set, look for it and make sure the script doesn't error out +isfoundset=${FOUND_MOZCONFIG+yes} +if [ -z $isfoundset ]; then + FOUND_MOZCONFIG=`$_AUTOCONF_TOOLS_DIR/mozconfig-find $topsrcdir` + if [ $? -ne 0 ]; then + echo "Fix above errors before continuing." 1>&2 + exit 1 + fi +fi + +if [ "$FOUND_MOZCONFIG" ]; then + . "$FOUND_MOZCONFIG" +fi +export FOUND_MOZCONFIG + +if [ "$mozconfig_ac_options" ]; then + ac_echo_options 1>&2 +fi + +eval "set -- $mozconfig_ac_options $ac_options" diff --git a/client.mk b/client.mk index 5cbaf6b..f737c5b 100644 --- a/client.mk +++ b/client.mk @@ -50,6 +50,7 @@ endif ifndef TOPSRCDIR ifeq (,$(wildcard client.mk)) TOPSRCDIR := $(patsubst %/,%,$(dir $(MAKEFILE_LIST))) +MOZ_OBJDIR = . else TOPSRCDIR := $(CWD) endif @@ -98,6 +99,8 @@ endif # See build pages, http://www.mozilla.org/build/ for how to set up mozconfig. +MOZCONFIG_LOADER := build/autoconf/mozconfig2client-mk + define CR @@ -108,9 +111,7 @@ endef # before evaluation. $(shell) replacing newlines with spaces, || is always # followed by a space (since sed doesn't remove newlines), except on the # last line, so replace both '|| ' and '||'. -# Also, make MOZ_PGO available to mozconfig when passed on make command line. -# Likewise for MOZ_CURRENT_PROJECT. -MOZCONFIG_CONTENT := $(subst ||,$(CR),$(subst || ,$(CR),$(shell $(addprefix MOZ_CURRENT_PROJECT=,$(MOZ_CURRENT_PROJECT)) MOZ_PGO=$(MOZ_PGO) $(TOPSRCDIR)/mach environment --format=client.mk | sed 's/$$/||/'))) +MOZCONFIG_CONTENT := $(subst ||,$(CR),$(subst || ,$(CR),$(shell $(TOPSRCDIR)/$(MOZCONFIG_LOADER) $(TOPSRCDIR) | sed 's/$$/||/'))) $(eval $(MOZCONFIG_CONTENT)) export FOUND_MOZCONFIG @@ -141,18 +142,35 @@ ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS))) endif +ifndef MOZ_OBJDIR + MOZ_OBJDIR = obj-$(CONFIG_GUESS) +else +# On Windows Pymake builds check MOZ_OBJDIR doesn't start with "/" + ifneq (,$(findstring mingw,$(CONFIG_GUESS))) + ifeq (1_a,$(.PYMAKE)_$(firstword a$(subst /, ,$(MOZ_OBJDIR)))) + $(error For Windows Pymake builds, MOZ_OBJDIR must be a Windows [and not MSYS] style path.) + endif + endif +endif + ifdef MOZ_BUILD_PROJECTS ifdef MOZ_CURRENT_PROJECT + OBJDIR = $(MOZ_OBJDIR)/$(MOZ_CURRENT_PROJECT) + MOZ_MAKE = $(MAKE) $(MOZ_MAKE_FLAGS) -C $(OBJDIR) BUILD_PROJECT_ARG = MOZ_BUILD_APP=$(MOZ_CURRENT_PROJECT) - export MOZ_CURRENT_PROJECT else + OBJDIR = $(error Cannot find the OBJDIR when MOZ_CURRENT_PROJECT is not set.) MOZ_MAKE = $(error Cannot build in the OBJDIR when MOZ_CURRENT_PROJECT is not set.) endif -endif # MOZ_BUILD_PROJECTS +else # MOZ_BUILD_PROJECTS + +OBJDIR = $(MOZ_OBJDIR) MOZ_MAKE = $(MAKE) $(MOZ_MAKE_FLAGS) -C $(OBJDIR) +endif # MOZ_BUILD_PROJECTS + # 'configure' scripts generated by autoconf. CONFIGURES := $(TOPSRCDIR)/configure CONFIGURES += $(TOPSRCDIR)/js/src/configure @@ -184,7 +202,7 @@ WANT_MOZCONFIG_MK = 1 endif ifdef WANT_MOZCONFIG_MK -# For now, only output "export" lines from mach environment --format=client.mk output. +# For now, only output "export" lines from mozconfig2client-mk output. MOZCONFIG_MK_LINES := $(filter export||%,$(MOZCONFIG_OUT_LINES)) $(OBJDIR)/.mozconfig.mk: $(FOUND_MOZCONFIG) $(call mkdir_deps,$(OBJDIR)) $(OBJDIR)/CLOBBER $(if $(MOZCONFIG_MK_LINES),( $(foreach line,$(MOZCONFIG_MK_LINES), echo '$(subst ||, ,$(line))';) )) > $@ @@ -217,11 +235,17 @@ everything: clean build # This is up here, outside of the MOZ_CURRENT_PROJECT logic so that this # is usable in multi-pass builds, where you might not have a runnable # application until all the build passes and postflight scripts have run. +ifdef MOZ_OBJDIR + PGO_OBJDIR = $(MOZ_OBJDIR) +else + PGO_OBJDIR := $(TOPSRCDIR) +endif + profiledbuild:: $(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_GENERATE=1 MOZ_PGO_INSTRUMENTED=1 - $(MAKE) -C $(OBJDIR) package MOZ_PGO_INSTRUMENTED=1 MOZ_INTERNAL_SIGNING_FORMAT= MOZ_EXTERNAL_SIGNING_FORMAT= - rm -f $(OBJDIR)/jarlog/en-US.log - MOZ_PGO_INSTRUMENTED=1 JARLOG_FILE=jarlog/en-US.log EXTRA_TEST_ARGS=10 $(MAKE) -C $(OBJDIR) pgo-profile-run + $(MAKE) -C $(PGO_OBJDIR) package MOZ_PGO_INSTRUMENTED=1 MOZ_INTERNAL_SIGNING_FORMAT= MOZ_EXTERNAL_SIGNING_FORMAT= + rm -f ${PGO_OBJDIR}/jarlog/en-US.log + MOZ_PGO_INSTRUMENTED=1 JARLOG_FILE=jarlog/en-US.log EXTRA_TEST_ARGS=10 $(MAKE) -C $(PGO_OBJDIR) pgo-profile-run $(MAKE) -f $(TOPSRCDIR)/client.mk maybe_clobber_profiledbuild $(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_USE=1 @@ -302,7 +326,6 @@ CONFIG_STATUS_DEPS := \ $(TOPSRCDIR)/build/virtualenv_packages.txt \ $(TOPSRCDIR)/python/mozbuild/mozbuild/virtualenv.py \ $(TOPSRCDIR)/testing/mozbase/packages.txt \ - $(OBJDIR)/.mozconfig.json \ $(NULL) CONFIGURE_ENV_ARGS += \ @@ -330,13 +353,8 @@ configure-preqs = \ $(call mkdir_deps,$(OBJDIR)) \ $(if $(MOZ_BUILD_PROJECTS),$(call mkdir_deps,$(MOZ_OBJDIR))) \ save-mozconfig \ - $(OBJDIR)/.mozconfig.json \ $(NULL) -CREATE_MOZCONFIG_JSON := $(shell $(TOPSRCDIR)/mach environment --format=json -o $(OBJDIR)/.mozconfig.json) -$(OBJDIR)/.mozconfig.json: $(call mkdir_deps,$(OBJDIR)) - @$(TOPSRCDIR)/mach environment --format=json -o $@ - save-mozconfig: $(FOUND_MOZCONFIG) -cp $(FOUND_MOZCONFIG) $(OBJDIR)/.mozconfig @@ -355,7 +373,7 @@ $(OBJDIR)/config.status: $(CONFIG_STATUS_DEPS) else $(OBJDIR)/Makefile: $(CONFIG_STATUS_DEPS) endif - @$(MAKE) -f $(TOPSRCDIR)/client.mk configure CREATE_MOZCONFIG_JSON= + @$(MAKE) -f $(TOPSRCDIR)/client.mk configure ifneq (,$(CONFIG_STATUS)) $(OBJDIR)/config/autoconf.mk: $(TOPSRCDIR)/config/autoconf.mk.in diff --git a/python/mozbuild/mozbuild/base.py b/python/mozbuild/mozbuild/base.py index 14c9972..d00d987 100644 --- a/python/mozbuild/mozbuild/base.py +++ b/python/mozbuild/mozbuild/base.py @@ -161,8 +161,7 @@ class MozbuildObject(ProcessExecutionMixin): # environment. If no mozconfig is present, the config will not have # much defined. loader = MozconfigLoader(topsrcdir) - current_project = os.environ.get('MOZ_CURRENT_PROJECT') - config = loader.read_mozconfig(mozconfig, moz_build_app=current_project) + config = loader.read_mozconfig(mozconfig) config_topobjdir = MozbuildObject.resolve_mozconfig_topobjdir( topsrcdir, config) @@ -172,30 +171,13 @@ class MozbuildObject(ProcessExecutionMixin): # inside an objdir you probably want to perform actions on that objdir, # not another one. This prevents accidental usage of the wrong objdir # when the current objdir is ambiguous. - # However, if the found mozconfig resolves to another objdir that - # doesn't exist, we may be in a subtree like when building mozilla/ - # under c-c, and the objdir was defined as a relative path. Try again - # adjusting for that. - if topobjdir and config_topobjdir: - if not os.path.exists(config_topobjdir): - config_topobjdir = MozbuildObject.resolve_mozconfig_topobjdir( - os.path.dirname(topsrcdir), config) - if current_project: - config_topobjdir = os.path.join(config_topobjdir, - current_project) - config_topobjdir = os.path.join(config_topobjdir, - os.path.basename(topsrcdir)) - elif current_project: - config_topobjdir = os.path.join(config_topobjdir, current_project) - - _config_topobjdir = config_topobjdir - mozilla_dir = os.path.join(_config_topobjdir, 'mozilla') - if not samepath(topobjdir, _config_topobjdir) \ - and (not os.path.exists(mozilla_dir) or not samepath(topobjdir, + mozilla_dir = os.path.join(config_topobjdir, 'mozilla') + if not samepath(topobjdir, config_topobjdir) \ + and (os.path.exists(mozilla_dir) and not samepath(topobjdir, mozilla_dir)): - raise ObjdirMismatchException(topobjdir, _config_topobjdir) + raise ObjdirMismatchException(topobjdir, config_topobjdir) topobjdir = topobjdir or config_topobjdir if topobjdir: @@ -251,8 +233,7 @@ class MozbuildObject(ProcessExecutionMixin): """ if self._mozconfig is None: loader = MozconfigLoader(self.topsrcdir) - self._mozconfig = loader.read_mozconfig( - moz_build_app=os.environ.get('MOZ_CURRENT_PROJECT')) + self._mozconfig = loader.read_mozconfig() return self._mozconfig @@ -567,13 +548,8 @@ class MachCommandBase(MozbuildObject): # more reliable than mozconfig when cwd is inside an objdir. topsrcdir = context.topdir topobjdir = None - detect_virtualenv_mozinfo = True - if hasattr(context, 'detect_virtualenv_mozinfo'): - detect_virtualenv_mozinfo = getattr(context, - 'detect_virtualenv_mozinfo') try: - dummy = MozbuildObject.from_environment(cwd=context.cwd, - detect_virtualenv_mozinfo=detect_virtualenv_mozinfo) + dummy = MozbuildObject.from_environment(cwd=context.cwd) topsrcdir = dummy.topsrcdir topobjdir = dummy._topobjdir except BuildEnvironmentNotFoundException: diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py index 0b19815..0bfe4a6 100644 --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -10,8 +10,6 @@ import operator import os import sys -import mozpack.path as mozpath - from mach.decorators import ( CommandArgument, CommandProvider, @@ -919,53 +917,37 @@ class Makefiles(MachCommandBase): class MachDebug(MachCommandBase): @Command('environment', category='build-dev', description='Show info about the mach and build environment.') - @CommandArgument('--format', default='pretty', - choices=['pretty', 'client.mk', 'configure', 'json'], - help='Print data in the given format.') - @CommandArgument('--output', '-o', type=str, - help='Output to the given file.') @CommandArgument('--verbose', '-v', action='store_true', help='Print verbose output.') - def environment(self, format, output=None, verbose=False): - func = getattr(self, '_environment_%s' % format.replace('.', '_')) - - if output: - # We want to preserve mtimes if the output file already exists - # and the content hasn't changed. - from mozbuild.util import FileAvoidWrite - with FileAvoidWrite(output) as out: - return func(out, verbose) - return func(sys.stdout, verbose) - - def _environment_pretty(self, out, verbose): + def environment(self, verbose=False): state_dir = self._mach_context.state_dir import platform - print('platform:\n\t%s' % platform.platform(), file=out) - print('python version:\n\t%s' % sys.version, file=out) - print('python prefix:\n\t%s' % sys.prefix, file=out) - print('mach cwd:\n\t%s' % self._mach_context.cwd, file=out) - print('os cwd:\n\t%s' % os.getcwd(), file=out) - print('mach directory:\n\t%s' % self._mach_context.topdir, file=out) - print('state directory:\n\t%s' % state_dir, file=out) + print('platform:\n\t%s' % platform.platform()) + print('python version:\n\t%s' % sys.version) + print('python prefix:\n\t%s' % sys.prefix) + print('mach cwd:\n\t%s' % self._mach_context.cwd) + print('os cwd:\n\t%s' % os.getcwd()) + print('mach directory:\n\t%s' % self._mach_context.topdir) + print('state directory:\n\t%s' % state_dir) - print('object directory:\n\t%s' % self.topobjdir, file=out) + print('object directory:\n\t%s' % self.topobjdir) if self.mozconfig['path']: - print('mozconfig path:\n\t%s' % self.mozconfig['path'], file=out) + print('mozconfig path:\n\t%s' % self.mozconfig['path']) if self.mozconfig['configure_args']: - print('mozconfig configure args:', file=out) + print('mozconfig configure args:') for arg in self.mozconfig['configure_args']: - print('\t%s' % arg, file=out) + print('\t%s' % arg) if self.mozconfig['make_extra']: - print('mozconfig extra make args:', file=out) + print('mozconfig extra make args:') for arg in self.mozconfig['make_extra']: - print('\t%s' % arg, file=out) + print('\t%s' % arg) if self.mozconfig['make_flags']: - print('mozconfig make flags:', file=out) + print('mozconfig make flags:') for arg in self.mozconfig['make_flags']: - print('\t%s' % arg, file=out) + print('\t%s' % arg) config = None @@ -976,70 +958,14 @@ class MachDebug(MachCommandBase): pass if config: - print('config topsrcdir:\n\t%s' % config.topsrcdir, file=out) - print('config topobjdir:\n\t%s' % config.topobjdir, file=out) + print('config topsrcdir:\n\t%s' % config.topsrcdir) + print('config topobjdir:\n\t%s' % config.topobjdir) if verbose: - print('config substitutions:', file=out) + print('config substitutions:') for k in sorted(config.substs): - print('\t%s: %s' % (k, config.substs[k]), file=out) + print('\t%s: %s' % (k, config.substs[k])) - print('config defines:', file=out) + print('config defines:') for k in sorted(config.defines): - print('\t%s' % k, file=out) - - def _environment_client_mk(self, out, verbose): - if self.mozconfig['make_extra']: - for arg in self.mozconfig['make_extra']: - print(arg, file=out) - objdir = mozpath.normsep(self.topobjdir) - print('MOZ_OBJDIR=%s' % objdir, file=out) - if 'MOZ_CURRENT_PROJECT' in os.environ: - objdir = mozpath.join(objdir, os.environ['MOZ_CURRENT_PROJECT']) - print('OBJDIR=%s' % objdir, file=out) - if self.mozconfig['path']: - print('FOUND_MOZCONFIG=%s' % mozpath.normsep(self.mozconfig['path']), - file=out) - - def _environment_configure(self, out, verbose): - if self.mozconfig['path']: - # Replace ' with '"'"', so that shell quoting e.g. - # a'b becomes 'a'"'"'b'. - quote = lambda s: s.replace("'", """'"'"'""") - print('echo Adding configure options from %s' % - mozpath.normsep(self.mozconfig['path']), file=out) - if self.mozconfig['configure_args']: - for arg in self.mozconfig['configure_args']: - quoted_arg = quote(arg) - print("echo ' %s'" % quoted_arg, file=out) - print("""set -- "$@" '%s'""" % quoted_arg, file=out) - for key, value in self.mozconfig['env']['added'].items(): - print("export %s='%s'" % (key, quote(value)), file=out) - for key, (old, value) in self.mozconfig['env']['modified'].items(): - print("export %s='%s'" % (key, quote(value)), file=out) - for key, value in self.mozconfig['vars']['added'].items(): - print("%s='%s'" % (key, quote(value)), file=out) - for key, (old, value) in self.mozconfig['vars']['modified'].items(): - print("%s='%s'" % (key, quote(value)), file=out) - for key in self.mozconfig['env']['removed'].keys() + \ - self.mozconfig['vars']['removed'].keys(): - print("unset %s" % key, file=out) - - def _environment_json(self, out, verbose): - import json - class EnvironmentEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, MozbuildObject): - result = { - 'topsrcdir': obj.topsrcdir, - 'topobjdir': obj.topobjdir, - 'mozconfig': obj.mozconfig, - } - if verbose: - result['substs'] = obj.substs - result['defines'] = obj.defines - return result - elif isinstance(obj, set): - return list(obj) - return json.JSONEncoder.default(self, obj) - json.dump(self, cls=EnvironmentEncoder, sort_keys=True, fp=out) + print('\t%s' % k) diff --git a/python/mozbuild/mozbuild/mozconfig.py b/python/mozbuild/mozbuild/mozconfig.py index 9feca70..3c1e3eb 100644 --- a/python/mozbuild/mozbuild/mozconfig.py +++ b/python/mozbuild/mozbuild/mozconfig.py @@ -65,11 +65,7 @@ class MozconfigLoader(ProcessExecutionMixin): DEPRECATED_TOPSRCDIR_PATHS = ('mozconfig.sh', 'myconfig.sh') DEPRECATED_HOME_PATHS = ('.mozconfig', '.mozconfig.sh', '.mozmyconfig.sh') - IGNORE_SHELL_VARIABLES = {'_'} - - ENVIRONMENT_VARIABLES = { - 'CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'MOZ_OBJDIR', - } + IGNORE_SHELL_VARIABLES = ('_') def __init__(self, topsrcdir): self.topsrcdir = topsrcdir @@ -200,7 +196,6 @@ class MozconfigLoader(ProcessExecutionMixin): 'make_flags': None, 'make_extra': None, 'env': None, - 'vars': None, } if path is None: @@ -236,49 +231,36 @@ class MozconfigLoader(ProcessExecutionMixin): parsed = self._parse_loader_output(output) - def diff_vars(vars_before, vars_after): - set1 = set(vars_before.keys()) - self.IGNORE_SHELL_VARIABLES - set2 = set(vars_after.keys()) - self.IGNORE_SHELL_VARIABLES - added = set2 - set1 - removed = set1 - set2 - maybe_modified = set1 & set2 - changed = { - 'added': {}, - 'removed': {}, - 'modified': {}, - 'unmodified': {}, - } - - for key in added: - changed['added'][key] = vars_after[key] - - for key in removed: - changed['removed'][key] = vars_before[key] - - for key in maybe_modified: - if vars_before[key] != vars_after[key]: - changed['modified'][key] = ( - vars_before[key], vars_after[key]) - elif key in self.ENVIRONMENT_VARIABLES: - # In order for irrelevant environment variable changes not - # to incur in re-running configure, only a set of - # environment variables are stored when they are - # unmodified. Otherwise, changes such as using a different - # terminal window, or even rebooting, would trigger - # reconfigures. - changed['unmodified'][key] = vars_after[key] - - return changed - - result['env'] = diff_vars(parsed['env_before'], parsed['env_after']) - - # Environment variables also appear as shell variables, but that's - # uninteresting duplication of information. Filter them out. - filt = lambda x, y: {k: v for k, v in x.items() if k not in y} - result['vars'] = diff_vars( - filt(parsed['vars_before'], parsed['env_before']), - filt(parsed['vars_after'], parsed['env_after']) - ) + all_variables = set(parsed['vars_before'].keys()) + all_variables |= set(parsed['vars_after'].keys()) + + changed = { + 'added': {}, + 'removed': {}, + 'modified': {}, + 'unmodified': {}, + } + + for key in all_variables: + if key in self.IGNORE_SHELL_VARIABLES: + continue + + if key not in parsed['vars_before']: + changed['added'][key] = parsed['vars_after'][key] + continue + + if key not in parsed['vars_after']: + changed['removed'][key] = parsed['vars_before'][key] + continue + + if parsed['vars_before'][key] != parsed['vars_after'][key]: + changed['modified'][key] = ( + parsed['vars_before'][key], parsed['vars_after'][key]) + continue + + changed['unmodified'][key] = parsed['vars_after'][key] + + result['env'] = changed result['configure_args'] = [self._expand(o) for o in parsed['ac']] @@ -286,9 +268,6 @@ class MozconfigLoader(ProcessExecutionMixin): result['configure_args'].extend(self._expand(o) for o in parsed['ac_app'][moz_build_app]) - if 'MOZ_OBJDIR' in parsed['env_before']: - result['topobjdir'] = parsed['env_before']['MOZ_OBJDIR'] - mk = [self._expand(o) for o in parsed['mk']] for o in mk: @@ -318,8 +297,6 @@ class MozconfigLoader(ProcessExecutionMixin): ac_app_options = defaultdict(list) before_source = {} after_source = {} - env_before_source = {} - env_after_source = {} current = None current_type = None @@ -362,14 +339,7 @@ class MozconfigLoader(ProcessExecutionMixin): assert current_type is not None - vars_mapping = { - 'BEFORE_SOURCE': before_source, - 'AFTER_SOURCE': after_source, - 'ENV_BEFORE_SOURCE': env_before_source, - 'ENV_AFTER_SOURCE': env_after_source, - } - - if current_type in vars_mapping: + if current_type in ('BEFORE_SOURCE', 'AFTER_SOURCE'): # mozconfigs are sourced using the Bourne shell (or at least # in Bourne shell mode). This means |set| simply lists # variables from the current shell (not functions). (Note that @@ -430,7 +400,10 @@ class MozconfigLoader(ProcessExecutionMixin): assert name is not None - vars_mapping[current_type][name] = value + if current_type == 'BEFORE_SOURCE': + before_source[name] = value + else: + after_source[name] = value current = [] @@ -444,8 +417,6 @@ class MozconfigLoader(ProcessExecutionMixin): 'ac_app': ac_app_options, 'vars_before': before_source, 'vars_after': after_source, - 'env_before': env_before_source, - 'env_after': env_after_source, } def _expand(self, s): diff --git a/python/mozbuild/mozbuild/mozconfig_loader b/python/mozbuild/mozbuild/mozconfig_loader index 569c690..d12c7fa 100755 --- a/python/mozbuild/mozbuild/mozconfig_loader +++ b/python/mozbuild/mozbuild/mozconfig_loader @@ -44,10 +44,6 @@ mk_add_options() { done } -echo "------BEGIN_ENV_BEFORE_SOURCE" -env -echo "------END_ENV_BEFORE_SOURCE" - echo "------BEGIN_BEFORE_SOURCE" set echo "------END_BEFORE_SOURCE" @@ -62,7 +58,3 @@ echo "------BEGIN_AFTER_SOURCE" set echo "------END_AFTER_SOURCE" -echo "------BEGIN_ENV_AFTER_SOURCE" -env -echo "------END_ENV_AFTER_SOURCE" - diff --git a/python/mozbuild/mozbuild/test/backend/common.py b/python/mozbuild/mozbuild/test/backend/common.py index b6e1566..7cfec3c 100644 --- a/python/mozbuild/mozbuild/test/backend/common.py +++ b/python/mozbuild/mozbuild/test/backend/common.py @@ -83,14 +83,6 @@ CONFIGS = DefaultOnReadDict({ class BackendTester(unittest.TestCase): - def setUp(self): - self._old_env = dict(os.environ) - os.environ.pop('MOZ_OBJDIR', None) - - def tearDown(self): - os.environ.clear() - os.environ.update(self._old_env) - def _get_environment(self, name): """Obtain a new instance of a ConfigEnvironment for a known profile. diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/mozbuild/mozbuild/test/frontend/test_emitter.py index 9e5fef1..25b69d4 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py +++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py @@ -41,14 +41,6 @@ data_path = mozpath.join(data_path, 'data') class TestEmitterBasic(unittest.TestCase): - def setUp(self): - self._old_env = dict(os.environ) - os.environ.pop('MOZ_OBJDIR', None) - - def tearDown(self): - os.environ.clear() - os.environ.update(self._old_env) - def reader(self, name): config = MockConfig(mozpath.join(data_path, name), extra_substs=dict( ENABLE_TESTS='1', diff --git a/python/mozbuild/mozbuild/test/test_base.py b/python/mozbuild/mozbuild/test/test_base.py index 3735213..24118cd 100644 --- a/python/mozbuild/mozbuild/test/test_base.py +++ b/python/mozbuild/mozbuild/test/test_base.py @@ -22,15 +22,14 @@ from mozbuild.base import ( BadEnvironmentException, MachCommandBase, MozbuildObject, - ObjdirMismatchException, PathArgument, ) from mozbuild.backend.configenvironment import ConfigEnvironment -from buildconfig import topsrcdir, topobjdir curdir = os.path.dirname(__file__) +topsrcdir = os.path.abspath(os.path.join(curdir, '..', '..', '..', '..')) log_manager = LoggingManager() @@ -39,15 +38,14 @@ class TestMozbuildObject(unittest.TestCase): self._old_cwd = os.getcwd() self._old_env = dict(os.environ) os.environ.pop('MOZCONFIG', None) - os.environ.pop('MOZ_OBJDIR', None) def tearDown(self): os.chdir(self._old_cwd) os.environ.clear() os.environ.update(self._old_env) - def get_base(self, topobjdir=None): - return MozbuildObject(topsrcdir, None, log_manager, topobjdir=topobjdir) + def get_base(self): + return MozbuildObject(topsrcdir, None, log_manager) def test_objdir_config_guess(self): base = self.get_base() @@ -74,6 +72,7 @@ class TestMozbuildObject(unittest.TestCase): 'foo')) self.assertTrue(base.topobjdir.endswith('foo')) + @unittest.skip('Failing on buildbot.') def test_objdir_config_status(self): """Ensure @CONFIG_GUESS@ is handled when loading mozconfig.""" base = self.get_base() @@ -107,17 +106,16 @@ class TestMozbuildObject(unittest.TestCase): mozconfig=mozconfig, ), fh) - os.environ[b'MOZCONFIG'] = mozconfig.encode('utf-8') + os.environ[b'MOZCONFIG'] = mozconfig os.chdir(topobjdir) - obj = MozbuildObject.from_environment( - detect_virtualenv_mozinfo=False) + obj = MozbuildObject.from_environment() self.assertEqual(obj.topobjdir, topobjdir) finally: - os.chdir(self._old_cwd) shutil.rmtree(d) + @unittest.skip('Failing on buildbot.') def test_relative_objdir(self): """Relative defined objdirs are loaded properly.""" d = os.path.realpath(tempfile.mkdtemp()) @@ -136,18 +134,16 @@ class TestMozbuildObject(unittest.TestCase): mozconfig=mozconfig, ), fh) - os.environ[b'MOZCONFIG'] = mozconfig.encode('utf-8') + os.environ[b'MOZCONFIG'] = mozconfig child = os.path.join(topobjdir, 'foo', 'bar') os.makedirs(child) os.chdir(child) - obj = MozbuildObject.from_environment( - detect_virtualenv_mozinfo=False) + obj = MozbuildObject.from_environment() self.assertEqual(obj.topobjdir, topobjdir) finally: - os.chdir(self._old_cwd) shutil.rmtree(d) @unittest.skipIf(not hasattr(os, 'symlink'), 'symlinks not available.') @@ -181,9 +177,9 @@ class TestMozbuildObject(unittest.TestCase): self.assertEqual(obj.topobjdir, topobjdir_real) finally: - os.chdir(self._old_cwd) shutil.rmtree(d) + @unittest.skip('Failed on buildbot (bug 853954)') def test_mach_command_base_inside_objdir(self): """Ensure a MachCommandBase constructed from inside the objdir works.""" @@ -212,7 +208,6 @@ class TestMozbuildObject(unittest.TestCase): context.topdir = topsrcdir context.settings = None context.log_manager = None - context.detect_virtualenv_mozinfo=False o = MachCommandBase(context) @@ -220,9 +215,9 @@ class TestMozbuildObject(unittest.TestCase): self.assertEqual(o.topsrcdir, topsrcdir) finally: - os.chdir(self._old_cwd) shutil.rmtree(d) + @unittest.skip('Failing on buildbot.') def test_objdir_is_srcdir_rejected(self): """Ensure the srcdir configurations are rejected.""" d = os.path.realpath(tempfile.mkdtemp()) @@ -240,41 +235,6 @@ class TestMozbuildObject(unittest.TestCase): MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) finally: - os.chdir(self._old_cwd) - shutil.rmtree(d) - - def test_objdir_mismatch(self): - """Ensure MachCommandBase throwing on objdir mismatch.""" - d = os.path.realpath(tempfile.mkdtemp()) - - try: - real_topobjdir = os.path.join(d, 'real-objdir') - os.makedirs(real_topobjdir) - - topobjdir = os.path.join(d, 'objdir') - os.makedirs(topobjdir) - - topsrcdir = os.path.join(d, 'srcdir') - os.makedirs(topsrcdir) - - mozconfig = os.path.join(d, 'mozconfig') - with open(mozconfig, 'wt') as fh: - fh.write('mk_add_options MOZ_OBJDIR=%s' % real_topobjdir) - - mozinfo = os.path.join(topobjdir, 'mozinfo.json') - with open(mozinfo, 'wt') as fh: - json.dump(dict( - topsrcdir=topsrcdir, - mozconfig=mozconfig, - ), fh) - - os.chdir(topobjdir) - - with self.assertRaises(ObjdirMismatchException): - MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) - - finally: - os.chdir(self._old_cwd) shutil.rmtree(d) def test_config_guess(self): @@ -286,8 +246,9 @@ class TestMozbuildObject(unittest.TestCase): self.assertIsNotNone(result) self.assertGreater(len(result), 0) + @unittest.skip('Failing on buildbot (bug 853954).') def test_config_environment(self): - base = self.get_base(topobjdir=topobjdir) + base = self.get_base() ce = base.config_environment self.assertIsInstance(ce, ConfigEnvironment) @@ -298,17 +259,15 @@ class TestMozbuildObject(unittest.TestCase): self.assertIsInstance(base.defines, dict) self.assertIsInstance(base.substs, dict) + @unittest.skip('Failing on buildbot (bug 853954).') def test_get_binary_path(self): - base = self.get_base(topobjdir=topobjdir) + base = self.get_base() platform = sys.platform # We should ideally use the config.status from the build. Let's install # a fake one. - substs = [ - ('MOZ_APP_NAME', 'awesomeapp'), - ('MOZ_BUILD_APP', 'awesomeapp'), - ] + substs = [('MOZ_APP_NAME', 'awesomeapp')] if sys.platform.startswith('darwin'): substs.append(('OS_ARCH', 'Darwin')) substs.append(('BIN_SUFFIX', '')) @@ -343,7 +302,7 @@ class TestMozbuildObject(unittest.TestCase): if platform.startswith('darwin'): self.assertTrue(p.endswith('awesomeapp/Nightly.app/Contents/MacOS/awesomeapp')) elif platform.startswith(('win32', 'cygwin')): - self.assertTrue(p.endswith('awesomeapp\\awesomeapp.exe')) + self.assertTrue(p.endswith('awesomeapp/awesomeapp.exe')) else: self.assertTrue(p.endswith('awesomeapp/awesomeapp')) diff --git a/python/mozbuild/mozbuild/test/test_mozconfig.py b/python/mozbuild/mozbuild/test/test_mozconfig.py index 744424e..429523d 100644 --- a/python/mozbuild/mozbuild/test/test_mozconfig.py +++ b/python/mozbuild/mozbuild/test/test_mozconfig.py @@ -29,7 +29,6 @@ class TestMozconfigLoader(unittest.TestCase): def setUp(self): self._old_env = dict(os.environ) os.environ.pop('MOZCONFIG', None) - os.environ.pop('MOZ_OBJDIR', None) os.environ.pop('CC', None) os.environ.pop('CXX', None) self._temp_dirs = set() @@ -244,7 +243,6 @@ class TestMozconfigLoader(unittest.TestCase): 'make_flags': None, 'make_extra': None, 'env': None, - 'vars': None, }) def test_read_empty_mozconfig(self): @@ -258,10 +256,9 @@ class TestMozconfigLoader(unittest.TestCase): self.assertEqual(result['make_extra'], []) for f in ('added', 'removed', 'modified'): - self.assertEqual(len(result['vars'][f]), 0) self.assertEqual(len(result['env'][f]), 0) - self.assertEqual(result['env']['unmodified'], {}) + self.assertGreater(len(result['env']['unmodified']), 0) def test_read_capture_ac_options(self): """Ensures ac_add_options calls are captured.""" @@ -319,22 +316,6 @@ class TestMozconfigLoader(unittest.TestCase): self.assertEqual(result['make_flags'], '-j8') self.assertEqual(result['make_extra'], ['FOO=BAR BAZ', 'BIZ=1']) - def test_read_empty_mozconfig_objdir_environ(self): - os.environ[b'MOZ_OBJDIR'] = b'obj-firefox' - with NamedTemporaryFile(mode='w') as mozconfig: - result = self.get_loader().read_mozconfig(mozconfig.name) - self.assertEqual(result['topobjdir'], 'obj-firefox') - - def test_read_capture_mk_options_objdir_environ(self): - """Ensures mk_add_options calls are captured and override the environ.""" - os.environ[b'MOZ_OBJDIR'] = b'obj-firefox' - with NamedTemporaryFile(mode='w') as mozconfig: - mozconfig.write('mk_add_options MOZ_OBJDIR=/foo/bar\n') - mozconfig.flush() - - result = self.get_loader().read_mozconfig(mozconfig.name) - self.assertEqual(result['topobjdir'], '/foo/bar') - def test_read_moz_objdir_substitution(self): """Ensure @TOPSRCDIR@ substitution is recognized in MOZ_OBJDIR.""" with NamedTemporaryFile(mode='w') as mozconfig: @@ -356,10 +337,9 @@ class TestMozconfigLoader(unittest.TestCase): result = self.get_loader().read_mozconfig(mozconfig.name) - self.assertEqual(result['vars']['added'], { + self.assertEqual(result['env']['added'], { 'CC': '/usr/local/bin/clang', 'CXX': '/usr/local/bin/clang++'}) - self.assertEqual(result['env']['added'], {}) def test_read_exported_variables(self): """Exported variables are caught as new variables.""" @@ -369,7 +349,6 @@ class TestMozconfigLoader(unittest.TestCase): result = self.get_loader().read_mozconfig(mozconfig.name) - self.assertEqual(result['vars']['added'], {}) self.assertEqual(result['env']['added'], { 'MY_EXPORTED': 'woot'}) @@ -383,25 +362,10 @@ class TestMozconfigLoader(unittest.TestCase): result = self.get_loader().read_mozconfig(mozconfig.name) - self.assertEqual(result['vars']['modified'], {}) self.assertEqual(result['env']['modified'], { 'CC': ('/usr/bin/gcc', '/usr/local/bin/clang') }) - def test_read_unmodified_variables(self): - """Variables modified by mozconfig are detected.""" - os.environ[b'CC'] = b'/usr/bin/gcc' - - with NamedTemporaryFile(mode='w') as mozconfig: - mozconfig.flush() - - result = self.get_loader().read_mozconfig(mozconfig.name) - - self.assertEqual(result['vars']['unmodified'], {}) - self.assertEqual(result['env']['unmodified'], { - 'CC': '/usr/bin/gcc' - }) - def test_read_removed_variables(self): """Variables unset by the mozconfig are detected.""" os.environ[b'CC'] = b'/usr/bin/clang' @@ -412,7 +376,6 @@ class TestMozconfigLoader(unittest.TestCase): result = self.get_loader().read_mozconfig(mozconfig.name) - self.assertEqual(result['vars']['removed'], {}) self.assertEqual(result['env']['removed'], { 'CC': '/usr/bin/clang'}) @@ -425,11 +388,10 @@ class TestMozconfigLoader(unittest.TestCase): result = self.get_loader().read_mozconfig(mozconfig.name) - self.assertEqual(result['vars']['added'], { + self.assertEqual(result['env']['added'], { 'multi': 'foo\nbar', 'single': '1' }) - self.assertEqual(result['env']['added'], {}) def test_read_topsrcdir_defined(self): """Ensure $topsrcdir references work as expected.""" @@ -440,25 +402,19 @@ class TestMozconfigLoader(unittest.TestCase): loader = self.get_loader() result = loader.read_mozconfig(mozconfig.name) - self.assertEqual(result['vars']['added']['TEST'], + self.assertEqual(result['env']['added']['TEST'], loader.topsrcdir.replace(os.sep, '/')) - self.assertEqual(result['env']['added'], {}) def test_read_empty_variable_value(self): """Ensure empty variable values are parsed properly.""" with NamedTemporaryFile(mode='w') as mozconfig: mozconfig.write('EMPTY=\n') - mozconfig.write('export EXPORT_EMPTY=\n') mozconfig.flush() result = self.get_loader().read_mozconfig(mozconfig.name) - self.assertEqual(result['vars']['added'], { - 'EMPTY': '', - }) - self.assertEqual(result['env']['added'], { - 'EXPORT_EMPTY': '' - }) + self.assertIn('EMPTY', result['env']['added']) + self.assertEqual(result['env']['added']['EMPTY'], '') def test_read_load_exception(self): """Ensure non-0 exit codes in mozconfigs are handled properly.""" diff --git a/testing/xpcshell/selftest.py b/testing/xpcshell/selftest.py index 78e1aa9..df2ba24 100644 --- a/testing/xpcshell/selftest.py +++ b/testing/xpcshell/selftest.py @@ -12,7 +12,6 @@ from StringIO import StringIO from xml.etree.ElementTree import ElementTree from mozbuild.base import MozbuildObject -os.environ.pop('MOZ_OBJDIR') build_obj = MozbuildObject.from_environment() from runxpcshelltests import XPCShellTests

1 0