August 2017 - tbb-commits - lists.torproject.org

[tor-browser/tor-browser-52.2.0esr-7.5-1] fixup! Bug 19273: Avoid JavaScript patching of the external app helper dialog.
by gk＠torproject.org 04 Aug '17

04 Aug '17

commit 0c48149c2a4d4603f066c26c1b54f5e4fe554454 Author: Kathy Brade <brade(a)pearlcrescent.com> Date: Thu Jul 20 17:04:44 2017 -0400 fixup! Bug 19273: Avoid JavaScript patching of the external app helper dialog. When handling an external URI or downloading a file, invoke Torbutton's external app blocker component (which will present a download warning dialog unless the user has checked the "Automatically download files from now on" box). For e10s compatibility, avoid using a modal dialog and instead use a callback interface (nsIHelperAppWarningLauncher) to allow Torbutton to indicate the user's desire to cancel or continue each request. --- .../exthandler/nsExternalHelperAppService.cpp | 162 ++++++++++++++++++--- uriloader/exthandler/nsExternalHelperAppService.h | 3 + .../exthandler/nsIExternalHelperAppService.idl | 47 ++++++ 3 files changed, 195 insertions(+), 17 deletions(-) diff --git a/uriloader/exthandler/nsExternalHelperAppService.cpp b/uriloader/exthandler/nsExternalHelperAppService.cpp index 51a7ee0f6ab6..df08905304fb 100644 --- a/uriloader/exthandler/nsExternalHelperAppService.cpp +++ b/uriloader/exthandler/nsExternalHelperAppService.cpp @@ -132,6 +132,9 @@ static const char NEVER_ASK_FOR_SAVE_TO_DISK_PREF[] = static const char NEVER_ASK_FOR_OPEN_FILE_PREF[] = "browser.helperApps.neverAsk.openFile"; +static const char WARNING_DIALOG_CONTRACT_ID[] = + "@torproject.org/torbutton-extAppBlocker;1"; + // Helper functions for Content-Disposition headers /** @@ -594,6 +597,107 @@ static const nsDefaultMimeTypeEntry nonDecodableExtensions[] = { { APPLICATION_GZIP, "svgz" } }; +////////////////////////////////////////////////////////////////////////////////////////////////////// +// begin nsExternalLoadURIHandler class definition and implementation +////////////////////////////////////////////////////////////////////////////////////////////////////// +class nsExternalLoadURIHandler final : public nsIHelperAppWarningLauncher +{ +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIHELPERAPPWARNINGLAUNCHER + + nsExternalLoadURIHandler(nsIInterfaceRequestor * aWindowContext, + nsIURI *aURI, + nsIHandlerInfo *aHandlerInfo); + +protected: + ~nsExternalLoadURIHandler(); + + nsCOMPtr<nsIInterfaceRequestor> mWindowContext; + nsCOMPtr<nsIURI> mURI; + nsCOMPtr<nsIHandlerInfo> mHandlerInfo; + nsCOMPtr<nsIHelperAppWarningDialog> mWarningDialog; +}; + +NS_IMPL_ADDREF(nsExternalLoadURIHandler) +NS_IMPL_RELEASE(nsExternalLoadURIHandler) + +NS_INTERFACE_MAP_BEGIN(nsExternalLoadURIHandler) + NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsIHelperAppWarningLauncher) + NS_INTERFACE_MAP_ENTRY(nsIHelperAppWarningLauncher) +NS_INTERFACE_MAP_END_THREADSAFE + +nsExternalLoadURIHandler::nsExternalLoadURIHandler( + nsIInterfaceRequestor *aWindowContext, + nsIURI *aURI, + nsIHandlerInfo *aHandlerInfo) +: mWindowContext(aWindowContext) +, mURI(aURI) +, mHandlerInfo(aHandlerInfo) +{ + nsresult rv = NS_OK; + mWarningDialog = do_CreateInstance(WARNING_DIALOG_CONTRACT_ID, &rv); + if (NS_SUCCEEDED(rv) && mWarningDialog) { + // This will create a reference cycle (the dialog holds a reference to us + // as nsIHelperAppWarningLauncher), which will be broken in ContinueRequest + // or CancelRequest. + rv = mWarningDialog->MaybeShow(this, aWindowContext); + } + + if (NS_FAILED(rv)) { + // If for some reason we could not open the download warning prompt, + // continue with the request. + ContinueRequest(); + } +} + +nsExternalLoadURIHandler::~nsExternalLoadURIHandler() +{ +} + +NS_IMETHODIMP nsExternalLoadURIHandler::ContinueRequest() +{ + MOZ_ASSERT(mURI); + MOZ_ASSERT(mHandlerInfo); + + // Break our reference cycle with the download warning dialog (set up in + // LoadURI). + mWarningDialog = nullptr; + + nsHandlerInfoAction preferredAction; + mHandlerInfo->GetPreferredAction(&preferredAction); + bool alwaysAsk = true; + mHandlerInfo->GetAlwaysAskBeforeHandling(&alwaysAsk); + + // if we are not supposed to ask, and the preferred action is to use + // a helper app or the system default, we just launch the URI. + if (!alwaysAsk && (preferredAction == nsIHandlerInfo::useHelperApp || + preferredAction == nsIHandlerInfo::useSystemDefault)) + return mHandlerInfo->LaunchWithURI(mURI, mWindowContext); + + nsresult rv = NS_OK; + nsCOMPtr<nsIContentDispatchChooser> chooser = + do_CreateInstance("@mozilla.org/content-dispatch-chooser;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + + return chooser->Ask(mHandlerInfo, mWindowContext, mURI, + nsIContentDispatchChooser::REASON_CANNOT_HANDLE); +} + +NS_IMETHODIMP nsExternalLoadURIHandler::CancelRequest(nsresult aReason) +{ + NS_ENSURE_ARG(NS_FAILED(aReason)); + + // Break our reference cycle with the download warning dialog (set up in + // LoadURI). + mWarningDialog = nullptr; + + return NS_OK; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////// +// nsExternalHelperAppService definition and implementation +////////////////////////////////////////////////////////////////////////////////////////////////////// NS_IMPL_ISUPPORTS( nsExternalHelperAppService, nsIExternalHelperAppService, @@ -1014,23 +1118,13 @@ nsExternalHelperAppService::LoadURI(nsIURI *aURI, rv = GetProtocolHandlerInfo(scheme, getter_AddRefs(handler)); NS_ENSURE_SUCCESS(rv, rv); - nsHandlerInfoAction preferredAction; - handler->GetPreferredAction(&preferredAction); - bool alwaysAsk = true; - handler->GetAlwaysAskBeforeHandling(&alwaysAsk); + RefPtr<nsExternalLoadURIHandler> h = + new nsExternalLoadURIHandler(aWindowContext, uri, handler); + if (!h) { + return NS_ERROR_OUT_OF_MEMORY; + } - // if we are not supposed to ask, and the preferred action is to use - // a helper app or the system default, we just launch the URI. - if (!alwaysAsk && (preferredAction == nsIHandlerInfo::useHelperApp || - preferredAction == nsIHandlerInfo::useSystemDefault)) - return handler->LaunchWithURI(uri, aWindowContext); - - nsCOMPtr<nsIContentDispatchChooser> chooser = - do_CreateInstance("@mozilla.org/content-dispatch-chooser;1", &rv); - NS_ENSURE_SUCCESS(rv, rv); - - return chooser->Ask(handler, aWindowContext, uri, - nsIContentDispatchChooser::REASON_CANNOT_HANDLE); + return NS_OK; } NS_IMETHODIMP nsExternalHelperAppService::GetApplicationDescription(const nsACString& aScheme, nsAString& _retval) @@ -1198,6 +1292,7 @@ NS_INTERFACE_MAP_BEGIN(nsExternalAppHandler) NS_INTERFACE_MAP_ENTRY(nsIStreamListener) NS_INTERFACE_MAP_ENTRY(nsIRequestObserver) NS_INTERFACE_MAP_ENTRY(nsIHelperAppLauncher) + NS_INTERFACE_MAP_ENTRY(nsIHelperAppWarningLauncher) NS_INTERFACE_MAP_ENTRY(nsICancelable) NS_INTERFACE_MAP_ENTRY(nsITimerCallback) NS_INTERFACE_MAP_ENTRY(nsIBackgroundFileSaverObserver) @@ -1671,6 +1766,29 @@ NS_IMETHODIMP nsExternalAppHandler::OnStartRequest(nsIRequest *request, nsISuppo httpInternal->SetChannelIsForDownload(true); } + mWarningDialog = do_CreateInstance(WARNING_DIALOG_CONTRACT_ID, &rv); + if (NS_SUCCEEDED(rv) && mWarningDialog) { + // This will create a reference cycle (the dialog holds a reference to us + // as nsIHelperAppWarningLauncher), which will be broken in ContinueRequest + // or CancelRequest. + rv = mWarningDialog->MaybeShow(this, GetDialogParent()); + } + + if (NS_FAILED(rv)) { + // If for some reason we could not open the download warning prompt, + // continue with the request. + ContinueRequest(); + } + + return NS_OK; +} + +NS_IMETHODIMP nsExternalAppHandler::ContinueRequest() +{ + // Break our reference cycle with the download warning dialog (set up in + // OnStartRequest) + mWarningDialog = nullptr; + // now that the temp file is set up, find out if we need to invoke a dialog // asking the user what they want us to do with this content... @@ -1731,6 +1849,7 @@ NS_IMETHODIMP nsExternalAppHandler::OnStartRequest(nsIRequest *request, nsISuppo action = nsIMIMEInfo::saveToDisk; } + nsresult rv = NS_OK; if (alwaysAsk) { // Display the dialog @@ -1786,6 +1905,15 @@ NS_IMETHODIMP nsExternalAppHandler::OnStartRequest(nsIRequest *request, nsISuppo return NS_OK; } +NS_IMETHODIMP nsExternalAppHandler::CancelRequest(nsresult aReason) +{ + // Break our reference cycle with the download warning dialog (set up in + // OnStartRequest) + mWarningDialog = nullptr; + + return Cancel(aReason); +} + // Convert error info into proper message text and send OnStatusChange // notification to the dialog progress listener or nsITransfer implementation. void nsExternalAppHandler::SendStatusChange(ErrorType type, nsresult rv, nsIRequest *aRequest, const nsAFlatString &path) @@ -2477,7 +2605,7 @@ NS_IMETHODIMP nsExternalAppHandler::Cancel(nsresult aReason) } // Break our reference cycle with the helper app dialog (set up in - // OnStartRequest) + // ContinueRequest) mDialog = nullptr; mRequest = nullptr; diff --git a/uriloader/exthandler/nsExternalHelperAppService.h b/uriloader/exthandler/nsExternalHelperAppService.h index ceec66661dd4..3146407dcbe5 100644 --- a/uriloader/exthandler/nsExternalHelperAppService.h +++ b/uriloader/exthandler/nsExternalHelperAppService.h @@ -210,6 +210,7 @@ private: */ class nsExternalAppHandler final : public nsIStreamListener, public nsIHelperAppLauncher, + public nsIHelperAppWarningLauncher, public nsITimerCallback, public nsIBackgroundFileSaverObserver { @@ -218,6 +219,7 @@ public: NS_DECL_NSISTREAMLISTENER NS_DECL_NSIREQUESTOBSERVER NS_DECL_NSIHELPERAPPLAUNCHER + NS_DECL_NSIHELPERAPPWARNINGLAUNCHER NS_DECL_NSICANCELABLE NS_DECL_NSITIMERCALLBACK NS_DECL_NSIBACKGROUNDFILESAVEROBSERVER @@ -478,6 +480,7 @@ protected: nsCOMPtr<nsIChannel> mOriginalChannel; /**< in the case of a redirect, this will be the pre-redirect channel. */ nsCOMPtr<nsIHelperAppLauncherDialog> mDialog; + nsCOMPtr<nsIHelperAppWarningDialog> mWarningDialog; /** * Keep request alive in case when helper non-modal dialog shown. diff --git a/uriloader/exthandler/nsIExternalHelperAppService.idl b/uriloader/exthandler/nsIExternalHelperAppService.idl index bfdfff5ceaa8..546252f04ba3 100644 --- a/uriloader/exthandler/nsIExternalHelperAppService.idl +++ b/uriloader/exthandler/nsIExternalHelperAppService.idl @@ -152,3 +152,50 @@ interface nsIHelperAppLauncher : nsICancelable */ readonly attribute int64_t contentLength; }; + +/** + * nsIHelperAppWarningLauncher is implemented by two classes: + * nsExternalLoadURIHandler + * nsExternalAppHandler + */ +[scriptable, uuid(cffd508b-4aaf-43ad-99c6-671d35cbc558)] +interface nsIHelperAppWarningLauncher : nsISupports +{ + /** + * Callback invoked by the external app warning dialog to continue the + * request. + * NOTE: This will release the reference to the nsIHelperAppWarningDialog. + */ + void continueRequest(); + + /** + * Callback invoked by the external app warning dialog to cancel the request. + * NOTE: This will release the reference to the nsIHelperAppWarningDialog. + * + * @param aReason + * Pass a failure code to indicate the reason why this operation is + * being canceled. It is an error to pass a success code. + */ + void cancelRequest(in nsresult aReason); +}; + +/** + * nsIHelperAppWarningDialog is implemented by Torbutton's external app + * blocker (src/components/external-app-blocker.js). + */ +[scriptable, uuid(f4899a3f-0df3-42cc-9db8-bdf599e5a208)] +interface nsIHelperAppWarningDialog : nsISupports +{ + /** + * Possibly show a launch warning dialog (it will not be shown if the user + * has chosen to not see the warning again). + * + * @param aLauncher + * A nsIHelperAppWarningLauncher to be invoked after the user confirms + * or cancels the download. + * @param aWindowContext + * The window associated with the download. + */ + void maybeShow(in nsIHelperAppWarningLauncher aLauncher, + in nsISupports aWindowContext); +};

1 0

[tor-browser/tor-browser-52.2.0esr-7.5-1] Revert "Bug 19273: Avoid JavaScript patching of the external app helper dialog."
by gk＠torproject.org 04 Aug '17

04 Aug '17

commit 47df7aa33aa1ea4a72ac029169725609f2f518aa Author: Kathy Brade <brade(a)pearlcrescent.com> Date: Tue Jul 18 16:22:32 2017 -0400 Revert "Bug 19273: Avoid JavaScript patching of the external app helper dialog." This reverts commit 648d842b127e6ca6457d43e45244c65ee1773aa3. --- netwerk/protocol/http/HttpBaseChannel.cpp | 7 --- netwerk/protocol/http/HttpBaseChannel.h | 1 - netwerk/protocol/http/NullHttpChannel.cpp | 6 --- netwerk/protocol/http/nsIHttpChannel.idl | 9 +--- .../protocol/viewsource/nsViewSourceChannel.cpp | 11 ----- .../exthandler/nsExternalHelperAppService.cpp | 57 ---------------------- 6 files changed, 1 insertion(+), 90 deletions(-) diff --git a/netwerk/protocol/http/HttpBaseChannel.cpp b/netwerk/protocol/http/HttpBaseChannel.cpp index 5ea6a002a40f..e0f7edec05f9 100644 --- a/netwerk/protocol/http/HttpBaseChannel.cpp +++ b/netwerk/protocol/http/HttpBaseChannel.cpp @@ -1233,13 +1233,6 @@ NS_IMETHODIMP HttpBaseChannel::SetTopLevelContentWindowId(uint64_t aWindowId) return NS_OK; } -NS_IMETHODIMP HttpBaseChannel::IsPendingUnforced(bool *aIsPendingUnforced) -{ - NS_ENSURE_ARG_POINTER(aIsPendingUnforced); - *aIsPendingUnforced = mIsPending; - return NS_OK; -} - NS_IMETHODIMP HttpBaseChannel::GetTransferSize(uint64_t *aTransferSize) { diff --git a/netwerk/protocol/http/HttpBaseChannel.h b/netwerk/protocol/http/HttpBaseChannel.h index 3a1a8ba525b0..c8184a601b9e 100644 --- a/netwerk/protocol/http/HttpBaseChannel.h +++ b/netwerk/protocol/http/HttpBaseChannel.h @@ -190,7 +190,6 @@ public: NS_IMETHOD SetChannelId(const nsACString& aChannelId) override; NS_IMETHOD GetTopLevelContentWindowId(uint64_t *aContentWindowId) override; NS_IMETHOD SetTopLevelContentWindowId(uint64_t aContentWindowId) override; - NS_IMETHOD IsPendingUnforced(bool *aIsPendingUnforced) override; // nsIHttpChannelInternal NS_IMETHOD GetDocumentURI(nsIURI **aDocumentURI) override; diff --git a/netwerk/protocol/http/NullHttpChannel.cpp b/netwerk/protocol/http/NullHttpChannel.cpp index 8b362cb24233..8c048a6b5a7e 100644 --- a/netwerk/protocol/http/NullHttpChannel.cpp +++ b/netwerk/protocol/http/NullHttpChannel.cpp @@ -82,12 +82,6 @@ NullHttpChannel::SetTopLevelContentWindowId(uint64_t aWindowId) } NS_IMETHODIMP -NullHttpChannel::IsPendingUnforced(bool *_retval) -{ - return NS_ERROR_NOT_IMPLEMENTED; -} - -NS_IMETHODIMP NullHttpChannel::GetTransferSize(uint64_t *aTransferSize) { return NS_ERROR_NOT_IMPLEMENTED; diff --git a/netwerk/protocol/http/nsIHttpChannel.idl b/netwerk/protocol/http/nsIHttpChannel.idl index 0de0676ad156..75ec2c73989a 100644 --- a/netwerk/protocol/http/nsIHttpChannel.idl +++ b/netwerk/protocol/http/nsIHttpChannel.idl @@ -14,7 +14,7 @@ interface nsIHttpHeaderVisitor; * the inspection of the resulting HTTP response status and headers when they * become available. */ -[builtinclass, scriptable, uuid(e0d8071b-5389-48c2-92c7-6708c968044d)] +[builtinclass, scriptable, uuid(c5a4a073-4539-49c7-a3f2-cec3f0619c6c)] interface nsIHttpChannel : nsIChannel { /************************************************************************** @@ -469,11 +469,4 @@ interface nsIHttpChannel : nsIChannel * this channels is being load in. */ attribute uint64_t topLevelContentWindowId; - - /** - * Returns true if a request is pending due to "natural" causes and - * not just because ForcePending() has been called. See isPending() - * in nsIRequest.idl for more details about pending requests. - */ - boolean isPendingUnforced(); }; diff --git a/netwerk/protocol/viewsource/nsViewSourceChannel.cpp b/netwerk/protocol/viewsource/nsViewSourceChannel.cpp index 9f9b89438fba..9ed71c4ef001 100644 --- a/netwerk/protocol/viewsource/nsViewSourceChannel.cpp +++ b/netwerk/protocol/viewsource/nsViewSourceChannel.cpp @@ -740,17 +740,6 @@ nsViewSourceChannel::SetTopLevelContentWindowId(uint64_t aWindowId) } NS_IMETHODIMP -nsViewSourceChannel::IsPendingUnforced(bool *result) -{ - if (mHttpChannel) { - return mHttpChannel->IsPendingUnforced(result); - } - - NS_ENSURE_TRUE(mChannel, NS_ERROR_FAILURE); - return mChannel->IsPending(result); -} - -NS_IMETHODIMP nsViewSourceChannel::GetRequestMethod(nsACString & aRequestMethod) { return !mHttpChannel ? NS_ERROR_NULL_POINTER : diff --git a/uriloader/exthandler/nsExternalHelperAppService.cpp b/uriloader/exthandler/nsExternalHelperAppService.cpp index 4a7fd7f92f15..51a7ee0f6ab6 100644 --- a/uriloader/exthandler/nsExternalHelperAppService.cpp +++ b/uriloader/exthandler/nsExternalHelperAppService.cpp @@ -41,7 +41,6 @@ #include "nsThreadUtils.h" #include "nsAutoPtr.h" #include "nsIMutableArray.h" -#include "nsISupportsPrimitives.h" // for nsISupportsPRBool // used to access our datastore of user-configured helper applications #include "nsIHandlerService.h" @@ -106,7 +105,6 @@ #include "mozilla/Preferences.h" #include "mozilla/ipc/URIUtils.h" -#include "mozilla/Unused.h" using namespace mozilla; using namespace mozilla::ipc; @@ -436,22 +434,6 @@ static nsresult GetDownloadDirectory(nsIFile **_directory, return NS_OK; } -static nsresult shouldCancel(bool *aShouldCancel) -{ - NS_ENSURE_ARG_POINTER(aShouldCancel); - - nsCOMPtr<nsISupportsPRBool> cancelObj = - do_CreateInstance(NS_SUPPORTS_PRBOOL_CONTRACTID); - cancelObj->SetData(false); - nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService(); - if (!obs) - return NS_ERROR_FAILURE; - - obs->NotifyObservers(cancelObj, "external-app-requested", nullptr); - cancelObj->GetData(aShouldCancel); - return NS_OK; -} - /** * Structure for storing extension->type mappings. * @see defaultMimeEntries @@ -1028,14 +1010,6 @@ nsExternalHelperAppService::LoadURI(nsIURI *aURI, return NS_OK; // explicitly denied } - // Give other modules, including extensions, a chance to cancel. - bool doCancel = false; - rv = shouldCancel(&doCancel); - NS_ENSURE_SUCCESS(rv, rv); - if (doCancel) { - return NS_OK; - } - nsCOMPtr<nsIHandlerInfo> handler; rv = GetProtocolHandlerInfo(scheme, getter_AddRefs(handler)); NS_ENSURE_SUCCESS(rv, rv); @@ -1669,37 +1643,6 @@ NS_IMETHODIMP nsExternalAppHandler::OnStartRequest(nsIRequest *request, nsISuppo return NS_OK; } - // Give other modules, including extensions, a chance to cancel. - // To avoid a problem where OnDataAvailable fires but is not handled - // correctly while a modal dialog displayed by Torbutton is open, we - // suspend and then we either cancel or resume active requests. - // See bugs 21766 and 21886. - bool isPending = false; - nsCOMPtr<nsIHttpChannel> httpChan = do_QueryInterface(request); - if (httpChan) { - rv = httpChan->IsPendingUnforced(&isPending); - } else { - rv = request->IsPending(&isPending); - } - NS_ENSURE_SUCCESS(rv, rv); - - if (isPending) { - Unused << request->Suspend(); // Best effort: ignore failures. - } - - bool doCancel = false; - rv = shouldCancel(&doCancel); - NS_ENSURE_SUCCESS(rv, rv); - if (doCancel) { - mCanceled = true; - request->Cancel(NS_BINDING_ABORTED); - return NS_OK; - } - - if (isPending) { - Unused << request->Resume(); // Best effort: ignore failures. - } - rv = SetUpTempFile(aChannel); if (NS_FAILED(rv)) { nsresult transferError = rv;

1 0

[tor-browser/tor-browser-52.2.0esr-7.5-1] Bug 21830: Copying large test from web console leaks to /tmp
by gk＠torproject.org 04 Aug '17

04 Aug '17

commit 06580161f901dedf60e6dcb9252e7ce6b3e7b37b Author: Georg Koppen <gk(a)torproject.org> Date: Fri Aug 4 05:55:49 2017 +0000 Bug 21830: Copying large test from web console leaks to /tmp Patch written by Neill Miller --- widget/nsTransferable.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/widget/nsTransferable.cpp b/widget/nsTransferable.cpp index e99d45401df8..976180c7581e 100644 --- a/widget/nsTransferable.cpp +++ b/widget/nsTransferable.cpp @@ -39,6 +39,7 @@ Notes to self: #include "nsIFile.h" #include "nsILoadContext.h" #include "mozilla/UniquePtr.h" +#include "mozilla/Preferences.h" NS_IMPL_ISUPPORTS(nsTransferable, nsITransferable) @@ -248,6 +249,11 @@ nsTransferable::Init(nsILoadContext* aContext) if (aContext) { mPrivateData = aContext->UsePrivateBrowsing(); + } else { + // without aContext here to provide PrivateBrowsing information, + // we defer to the active configured setting + mPrivateData = + mozilla::Preferences::GetBool("browser.privatebrowsing.autostart"); } #ifdef DEBUG mInitialized = true;

1 0

[torbutton/master] Changelog update and version bump
by gk＠torproject.org 04 Aug '17

04 Aug '17

commit 6c76c3083f5ed221deddc51d13e09e8f8b6418f4 Author: Georg Koppen <gk(a)torproject.org> Date: Fri Aug 4 05:33:06 2017 +0000 Changelog update and version bump --- src/CHANGELOG | 13 +++++++++++++ src/install.rdf | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/CHANGELOG b/src/CHANGELOG index 775884c..ddd40d8 100644 --- a/src/CHANGELOG +++ b/src/CHANGELOG @@ -1,3 +1,16 @@ +1.9.8 + * Bug 22610: Avoid crashes when canceling external helper app related downloads + * Bug 22472: Fix FTP downloads when external helper app dialog is shown + * Bug 22471: Downloading pdf files via the PDF viewer download button is broken + * Bug 22618: Downloading pdf file via file:/// is stalling + * Bug 22542: Resize slider window to work without scrollbars + * Bug 21999: Fix display of language prompt in non-en-US locales + * Bug 18193: Don't let about:tor have chrome privileges + * Bug 22535: Search on about:tor discards search query + * Bug 21948: Going back to about:tor page gives "Address isn't valid" error + * Code clean-up + * Translations update + 1.9.7.4 * Bug 22542: Security Settings window too small on macOS 10.12 diff --git a/src/install.rdf b/src/install.rdf index 868faea..c501b6a 100644 --- a/src/install.rdf +++ b/src/install.rdf @@ -6,7 +6,7 @@ <em:name>Torbutton</em:name> <em:creator>Mike Perry</em:creator> <em:id>torbutton(a)torproject.org</em:id> - <em:version>1.9.7.4</em:version> + <em:version>1.9.8</em:version> <em:multiprocessCompatible>true</em:multiprocessCompatible> <em:homepageURL>https://www.torproject.org/projects/torbrowser.html.en</em:homepageURL> <em:optionsURL>chrome://torbutton/content/preferences.xul</em:optionsURL>

1 0

[torbutton/master] fixup! Bug 19273: Avoid JavaScript patching of the external app helper dialog.
by gk＠torproject.org 04 Aug '17

04 Aug '17

commit d48388d248ddf23150e70c7f0e4b84109e793a0d Author: Kathy Brade <brade(a)pearlcrescent.com> Date: Thu Jul 20 17:11:53 2017 -0400 fixup! Bug 19273: Avoid JavaScript patching of the external app helper dialog. For compatibility with recent changes to our browser patch, change the external app blocker module so it implements the new nsIHelperAppWarningDialog interface. Since the external app blocker module is no longer a service, split the drag and drop filter into a separate component (which remains a service). --- src/chrome.manifest | 7 +- src/components/dragDropFilter.js | 86 +++++++++++++++++ src/components/external-app-blocker.js | 169 ++++++++++++++++----------------- 3 files changed, 175 insertions(+), 87 deletions(-) diff --git a/src/chrome.manifest b/src/chrome.manifest index 272401b..65f55a9 100644 --- a/src/chrome.manifest +++ b/src/chrome.manifest @@ -143,8 +143,11 @@ skin torbutton classic/1.0 chrome/skin/ style chrome://global/content/customizeToolbar.xul chrome://torbutton/skin/torbutton.css # Firefox 4-style component registration +component {f605ec27-d867-44b5-ad97-2a29276642c3} components/dragDropFilter.js +contract @torproject.org/torbutton-dragDropFilter;1 {f605ec27-d867-44b5-ad97-2a29276642c3} + component {3da0269f-fc29-4e9e-a678-c3b1cafcf13f} components/external-app-blocker.js -contract @torproject.org/torbutton-extAppBlockerService;1 {3da0269f-fc29-4e9e-a678-c3b1cafcf13f} +contract @torproject.org/torbutton-extAppBlocker;1 {3da0269f-fc29-4e9e-a678-c3b1cafcf13f} component {06322def-6fde-4c06-aef6-47ae8e799629} components/startup-observer.js contract @torproject.org/startup-observer;1 {06322def-6fde-4c06-aef6-47ae8e799629} @@ -165,4 +168,4 @@ category profile-after-change CookieJarSelector @torproject.org/cookie-jar-selec category profile-after-change StartupObserver @torproject.org/startup-observer;1 category profile-after-change DomainIsolator @torproject.org/domain-isolator;1 -category profile-after-change ExtAppBlockerService @torproject.org/torbutton-extAppBlockerService;1 +category profile-after-change DragDropFilter @torproject.org/torbutton-dragDropFilter;1 diff --git a/src/components/dragDropFilter.js b/src/components/dragDropFilter.js new file mode 100644 index 0000000..22dde86 --- /dev/null +++ b/src/components/dragDropFilter.js @@ -0,0 +1,86 @@ +/************************************************************************* + * Drag and Drop Handler. + * + * Implements an observer that filters drag events to prevent OS + * access to URLs (a potential proxy bypass vector). + *************************************************************************/ + +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cu = Components.utils; + +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); + +// Module specific constants +const kMODULE_NAME = "Torbutton Drag and Drop Handler"; +const kCONTRACT_ID = "@torproject.org/torbutton-dragDropFilter;1"; +const kMODULE_CID = Components.ID("f605ec27-d867-44b5-ad97-2a29276642c3"); + +const kInterfaces = [Ci.nsIObserver, Ci.nsIClassInfo]; + +function DragDropFilter() { + this.logger = Cc["@torproject.org/torbutton-logger;1"] + .getService(Ci.nsISupports).wrappedJSObject; + this.logger.log(3, "Component Load 0: New DragDropFilter."); + + try { + var observerService = Cc["@mozilla.org/observer-service;1"]. + getService(Ci.nsIObserverService); + observerService.addObserver(this, "on-datatransfer-available", false); + } catch(e) { + this.logger.log(5, "Failed to register drag observer"); + } +} + +DragDropFilter.prototype = +{ + QueryInterface: XPCOMUtils.generateQI([Ci.nsISupports, Ci.nsIObserver]), + + // make this an nsIClassInfo object + flags: Ci.nsIClassInfo.DOM_OBJECT, + classDescription: kMODULE_NAME, + contractID: kCONTRACT_ID, + classID: kMODULE_CID, + + // method of nsIClassInfo + getInterfaces: function(count) { + count.value = kInterfaces.length; + return kInterfaces; + }, + + // method of nsIClassInfo + getHelperForLanguage: function(count) { return null; }, + + // method of nsIObserver + observe: function(subject, topic, data) { + if (topic == "on-datatransfer-available") { + this.logger.log(3, "The DataTransfer is available"); + return this.filterDataTransferURLs(subject); + } + }, + + filterDataTransferURLs: function(aDataTransfer) { + var types = null; + var type = ""; + var count = aDataTransfer.mozItemCount; + var len = 0; + for (var i = 0; i < count; ++i) { + this.logger.log(3, "Inspecting the data transfer: " + i); + types = aDataTransfer.mozTypesAt(i); + len = types.length; + for (var j = 0; j < len; ++j) { + type = types[j]; + this.logger.log(3, "Type is: " + type); + if (type == "text/x-moz-url" || + type == "text/x-moz-url-data" || + type == "text/uri-list" || + type == "application/x-moz-file-promise-url") { + aDataTransfer.clearData(type); + this.logger.log(3, "Removing " + type); + } + } + } + } +}; + +var NSGetFactory = XPCOMUtils.generateNSGetFactory([DragDropFilter]); diff --git a/src/components/external-app-blocker.js b/src/components/external-app-blocker.js index e56fed3..ca86580 100644 --- a/src/components/external-app-blocker.js +++ b/src/components/external-app-blocker.js @@ -7,19 +7,24 @@ * Handles displaying confirmation dialogs for external apps and protocols * due to Firefox Bug https://bugzilla.mozilla.org/show_bug.cgi?id=440892 * - * Also implements an observer that filters drag events to prevent OS - * access to URLs (a potential proxy bypass vector). + * An instance of this module is created each time the browser starts to + * download a file that may be opened by another or application and when + * an external application may be invoked to handle an URL (e.g., when the + * user clicks on a mailto: URL). *************************************************************************/ const Cc = Components.classes; const Ci = Components.interfaces; +const Cr = Components.results; const Cu = Components.utils; Cu.import("resource://gre/modules/XPCOMUtils.jsm"); +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/SharedPromptUtils.jsm"); // Module specific constants const kMODULE_NAME = "Torbutton External App Handler"; -const kCONTRACT_ID = "@torproject.org/torbutton-extAppBlockerService;1"; +const kCONTRACT_ID = "@torproject.org/torbutton-extAppBlocker;1"; const kMODULE_CID = Components.ID("3da0269f-fc29-4e9e-a678-c3b1cafcf13f"); const kInterfaces = [Ci.nsIObserver, Ci.nsIClassInfo]; @@ -28,23 +33,14 @@ function ExternalAppBlocker() { this.logger = Cc["@torproject.org/torbutton-logger;1"] .getService(Ci.nsISupports).wrappedJSObject; this.logger.log(3, "Component Load 0: New ExternalAppBlocker."); - - this._prefs = Cc["@mozilla.org/preferences-service;1"] - .getService(Ci.nsIPrefBranch); - - try { - var observerService = Cc["@mozilla.org/observer-service;1"]. - getService(Ci.nsIObserverService); - observerService.addObserver(this, "external-app-requested", false); - observerService.addObserver(this, "on-datatransfer-available", false); - } catch(e) { - this.logger.log(5, "Failed to register external app observer or drag observer"); - } } ExternalAppBlocker.prototype = { - QueryInterface: XPCOMUtils.generateQI([Ci.nsISupports, Ci.nsIObserver]), + _helperAppLauncher: undefined, + + QueryInterface: XPCOMUtils.generateQI([Ci.nsISupports, Ci.nsIObserver, + Ci.nsIHelperAppWarningDialog]), // make this an nsIClassInfo object flags: Ci.nsIClassInfo.DOM_OBJECT, @@ -58,83 +54,86 @@ ExternalAppBlocker.prototype = return kInterfaces; }, - // method of nsIClassInfo + // method of nsIClassInfo getHelperForLanguage: function(count) { return null; }, - // Returns true if launch should proceed. - _confirmLaunch: function() { - if (!this._prefs.getBoolPref("extensions.torbutton.launch_warning")) { - return true; - } - - var wm = Cc["@mozilla.org/appshell/window-mediator;1"] - .getService(Ci.nsIWindowMediator); - var chrome = wm.getMostRecentWindow("navigator:browser"); - - var prompts = Cc["@mozilla.org/embedcomp/prompt-service;1"] - .getService(Ci.nsIPromptService); - var flags = prompts.BUTTON_POS_0 * prompts.BUTTON_TITLE_IS_STRING + - prompts.BUTTON_POS_1 * prompts.BUTTON_TITLE_IS_STRING + - prompts.BUTTON_DELAY_ENABLE + - prompts.BUTTON_POS_1_DEFAULT; - - var title = chrome.torbutton_get_property_string("torbutton.popup.external.title"); - var app = chrome.torbutton_get_property_string("torbutton.popup.external.app"); - var note = chrome.torbutton_get_property_string("torbutton.popup.external.note"); - var suggest = chrome.torbutton_get_property_string("torbutton.popup.external.suggest"); - var launch = chrome.torbutton_get_property_string("torbutton.popup.launch"); - var cancel = chrome.torbutton_get_property_string("torbutton.popup.cancel"); - var dontask = chrome.torbutton_get_property_string("torbutton.popup.dontask"); - - var check = {value: false}; - var result = prompts.confirmEx(chrome, title, app+note+suggest+" ", - flags, launch, cancel, "", dontask, check); - - if (check.value) { - this._prefs.setBoolPref("extensions.torbutton.launch_warning", false); + // method of nsIHelperAppWarningDialog + maybeShow: function(aLauncher, aWindowContext) + { + // Hold a reference to the object that called this component. This is + // important not just because we need to later invoke the + // continueRequest() or cancelRequest() callback on aLauncher, but also + // so that the launcher object (which is a reference counted object) is + // not released too soon. + this._helperAppLauncher = aLauncher; + + if (!Services.prefs.getBoolPref("extensions.torbutton.launch_warning")) { + this._helperAppLauncher.continueRequest(); + return; } - return (0 == result); - }, - - observe: function(subject, topic, data) { - if (topic == "external-app-requested") { - this.logger.log(3, "External app requested"); - // subject.data is true if the launch should be canceled. - if ((subject instanceof Ci.nsISupportsPRBool) - && !subject.data /* not canceled already */ - && !this._confirmLaunch()) { - subject.data = true; // The user said to cancel the launch. - } - } else if (topic == "on-datatransfer-available") { - this.logger.log(3, "The DataTransfer is available"); - return this.filterDataTransferURLs(subject); - } + this._showPrompt(aWindowContext); }, - filterDataTransferURLs: function(aDataTransfer) { - var types = null; - var type = ""; - var count = aDataTransfer.mozItemCount; - var len = 0; - for (var i = 0; i < count; ++i) { - this.logger.log(3, "Inspecting the data transfer: " + i); - types = aDataTransfer.mozTypesAt(i); - len = types.length; - for (var j = 0; j < len; ++j) { - type = types[j]; - this.logger.log(3, "Type is: " + type); - if (type == "text/x-moz-url" || - type == "text/x-moz-url-data" || - type == "text/uri-list" || - type == "application/x-moz-file-promise-url") { - aDataTransfer.clearData(type); - this.logger.log(3, "Removing " + type); - } - } + /* + * The _showPrompt() implementation uses some XUL and JS that is part of the + * browser's confirmEx() implementation. Specifically, _showPrompt() depends + * on chrome://global/content/commonDialog.xul as well as some of the code + * in resource://gre/modules/SharedPromptUtils.jsm. + */ + _showPrompt: function(aWindowContext) { + let parentWin; + try { + parentWin = aWindowContext.getInterface(Ci.nsIDOMWindow); + } catch (e) { + parentWin = Services.wm.getMostRecentWindow("navigator:browser"); } - } + let title = parentWin.torbutton_get_property_string("torbutton.popup.external.title"); + let app = parentWin.torbutton_get_property_string("torbutton.popup.external.app"); + let note = parentWin.torbutton_get_property_string("torbutton.popup.external.note"); + let suggest = parentWin.torbutton_get_property_string("torbutton.popup.external.suggest"); + let launch = parentWin.torbutton_get_property_string("torbutton.popup.launch"); + let cancel = parentWin.torbutton_get_property_string("torbutton.popup.cancel"); + let dontask = parentWin.torbutton_get_property_string("torbutton.popup.dontask"); + + let args = { + promptType: "confirmEx", + title: title, + text: app+note+suggest+" ", + checkLabel: dontask, + checked: false, + ok: false, + button0Label: launch, + button1Label: cancel, + defaultButtonNum: 1, // Cancel + buttonNumClicked: 1, // Cancel + enableDelay: true, + }; + + let propBag = PromptUtils.objectToPropBag(args); + let uri = "chrome://global/content/commonDialog.xul"; + let promptWin = Services.ww.openWindow(parentWin, uri, "_blank", + "centerscreen,chrome,titlebar", propBag); + promptWin.addEventListener("load", aEvent => { + promptWin.addEventListener("unload", aEvent => { + PromptUtils.propBagToObject(propBag, args); + + if (0 == args.buttonNumClicked) { + // Save the checkbox value and tell the browser's external helper app + // module about the user's choice. + if (args.checked) { + Services.prefs.setBoolPref("extensions.torbutton.launch_warning", + false); + } + + this._helperAppLauncher.continueRequest(); + } else { + this._helperAppLauncher.cancelRequest(Cr.NS_BINDING_ABORTED); + } + }, false); + }, false); + }, }; var NSGetFactory = XPCOMUtils.generateNSGetFactory([ExternalAppBlocker]);

1 0

[tor-browser-build/master] Add var/release, var/alpha, var/nightly options
by boklm＠torproject.org 03 Aug '17

03 Aug '17

commit bec3345342b0a60a5a8e908e50a37dbdd4e0697b Author: Nicolas Vigier <boklm(a)torproject.org> Date: Thu Aug 3 23:44:17 2017 +0200 Add var/release, var/alpha, var/nightly options Add some options to be able to write things like this in templates: [% IF c("var/release") -%] do something on release channel [% ELSIF c("var/alpha") -%] do something on alpha channel [% ELSIF c("var/nightly") -%] do something on nightly channel [% END -%] --- rbm.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rbm.conf b/rbm.conf index c93019e..327cd39 100644 --- a/rbm.conf +++ b/rbm.conf @@ -98,8 +98,15 @@ targets: noint: debug: 0 + release: + var: + release: 1 + alpha: + var: + alpha:1 nightly: var: + nightly: 1 multi_lingual: 1 torbrowser-testbuild:

1 0

[tor-browser-build/master] firefox: add var/torbrowser_update_channel value for the release channel
by boklm＠torproject.org 03 Aug '17

03 Aug '17

commit 47552843a3111c640c809524b34054246f9e4806 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Thu Aug 3 23:49:08 2017 +0200 firefox: add var/torbrowser_update_channel value for the release channel --- projects/firefox/config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/firefox/config b/projects/firefox/config index 50d7a4e..b2400c9 100644 --- a/projects/firefox/config +++ b/projects/firefox/config @@ -22,6 +22,10 @@ var: use_container: 1 targets: + release: + var: + torbrowser_update_channel: release + nightly: git_hash: 'tor-browser-[% c("var/firefox_version") %]-[% c("var/torbrowser_branch") %]-1' tag_gpg_id: 0

1 0

[tor-browser-build/master] Add a README.HACKING file
by boklm＠torproject.org 03 Aug '17

03 Aug '17

commit 41dc4091f04a16754b208af8edbb2e501a0a3831 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Fri Aug 4 00:19:47 2017 +0200 Add a README.HACKING file Add a file to list the main things to know when making changes to the Tor Browser build. --- README | 7 ++ README.HACKING | 223 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 230 insertions(+) diff --git a/README b/README index e5fc45d..7a2a671 100644 --- a/README +++ b/README @@ -205,3 +205,10 @@ Common Build Errors You can look at the README.BUILD_ERRORS file for a list of common build errors and their solutions. + +Hacking on the Tor Browser build +-------------------------------- + +The file README.HACKING tries to list the main things to know when +making changes to the Tor Browser build. + diff --git a/README.HACKING b/README.HACKING new file mode 100644 index 0000000..2687ed1 --- /dev/null +++ b/README.HACKING @@ -0,0 +1,223 @@ +Hacking on Tor Browser Build +============================ + +This file tries to list the main things to know when making changes to +the Tor Browser build. + + +rbm documentation +----------------- + +If you go to directory rbm/doc, you can find the rbm documentation. You +can build it with 'make html' or 'make man'. + + +Using and defining options +-------------------------- + +Options can be used in templates with the following syntax: + + [% c("option_name") %] + +More details about the templates syntax can be found in +rbm/doc/rbm_templates.7 and [http://template-toolkit.org/docs/manual/index.html] + +Some options have a specific meaning to rbm. You can see their descriptions +in rbm/doc/rbm_config.7 and rbm/doc/rbm_input_files.7. + +When the option does not have a specific meaning to rbm, it is a good +idea to define it under var/ to avoid potential conflicts in option names +with a future version of rbm. + +The options can be defined in different places: + +- rbm.conf for options available to all components +- project/$project/config for options available to one component +- rbm.local.conf for user defined options + +In each of those places, an option can be defined: + +- at the root of the document +- under targets/$target/, in which case the definition only applies when + a specific target is defined + +The targets are usually used to select: + +- the platform: torbrowser-linux-x86_64, torbrowser-linux-i686, + torbrowser-windows-i686, torbrowser-osx-x86_64 +- the channel: release, nightly, alpha + +The targets torbrowser-linux-x86_64, torbrowser-linux-i686, +torbrowser-windows-i686, torbrowser-osx-x86_64 are special cases. They +do not contain options directly, instead they contain a list of other +targets. For instance, the torbrowser-linux-x86_64 target is pointing +to the linux-x86_64 and linux targets. You should define an option under +the linux target if it applies to Linux on both architectures, or under +the linux-x86_64 if it only applies to the x86_64 architecture. + +An option that is defined at the root of rbm.conf can be overrided by +an other definition under a target, or inside projects/$project/config. +You can find the complete priority order in rbm/doc/rbm_config.7. + + +Defining a project's filename +----------------------------- + +The filename option defines the output file or directory from the build +of a component. If the file or directory exists in the out/$component +directory, the component will not be rebuilt when it is used as a +dependency from an other component. + +The filename is usually something like this: + + filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' + +The var/build_id value is a hash that is based on the build inputs of +the component: + - the build script (after template evaluation) + - the container definition + - the input files (the filename when it is a dependency on an other + project, the filename and hash of its content otherwise) +This means that each time the build script, the container or one of the +dependencies is modified, the output filename will change and a rebuild +of the component will be required. + +The version and var/osname values could be removed from the filename, +but they are useful to get an idea of what a file contains. + + +Adding some Linux/Windows/OSX specific changes to a build script +---------------------------------------------------------------- + +You can use the following template syntax in the build scripts: + + [% IF c("var/linux") -%] + # do something for linux + [% ELSIF c("var/windows") -%] + # do something for windows + [% ELSIF c("var/osx") -%] + # do something for osx + [% END -%] + +You can also use var/linux-x86_64 and var/linux-i686 for things that +only apply to x86_64 and i686 linux builds. You can use the var/release, +var/alpha and var/nightly options to do things depending on the channel. + +As an alternative you can define an option with a different value +depending on the target: + + targets: + linux: + var: + do_something: 'do something for linux' + windows-i686: + var: + do_something: 'do something for windows' + osx-x86_64: + var: + do_something: 'do something for osx' + +And in the build script, use: + + [% c("var/do_something") %] + + +Evaluating a component's build script +------------------------------------- + +If you want to look at the build script used to build a component for a +specific platform/channel, you can use the following command: + + $ ./rbm/rbm showconf $component build --target $channel --target $platform + +$component should be replaced by the name of the component. + +$channel should be one of the following: + + - release + + - alpha + + - nightly + +$platform should be one of the following: + + - torbrowser-linux-x86_64 + + - torbrowser-linux-i686 + + - torbrowser-windows-i686 + + - torbrowser-osx-x86_64 + +For example, to see the tor's build script for linux x86_64 on the alpha +channel, you can use: + + $ ./rbm/rbm showconf tor build --target alpha --target \ + torbrowser-linux-x86_64 + +If the component you are looking at has many dependencies, the display +can take some time as various build_id values need to be computed. If +you don't care about the accuracy of input and output file names, you +can add '--target no_build_id' to the command line. For instance if you +want to look at the build script for the tor-browser component (which +has a lot of dependencies), you can use: + + $ ./rbm/rbm showconf tor build --target alpha --target \ + torbrowser-linux-x86_64 --target no_build_id + +The same type of commands can be used to look at any option values, +replacing build with the name of the option you want to look at. For +instance if you want to know the output filename of tor on linux-x86_64 +on the alpha channel, you can use: + + $ ./rbm/rbm showconf tor filename --target alpha --target \ + torbrowser-linux-x86_64 + + +Building a single component +--------------------------- + +When you are working on some changes to a component, before creating a +full bundle with those changes, you might want to try a build of the +component you modified only. + +This can be done with the following command: + + $ ./rbm/rbm build $component --target $channel --target $platform + +See the previous section "Evaluating a component's build script" for a +list of possible values for $channel and $platform. + +For instance if you want to build tor for linux-x86_64 on the alpha +channel, you can run: + + $ ./rbm/rbm build tor --target alpha --target torbrowser-linux-x86_64 + +To find the resulting file from the build, you can use 'ls -ltr out/tor/' +to find the file with the last modification time. + + +Debugging a build error +----------------------- + +If the build fails, a debugging shell will automatically be opened in +the container used for the build (unless you set debug to 0 in +rbm.local.conf). + +Before using the debugging shell, you can first look at the build logs +in an other terminal (the filename of the log file is indicated in the +rbm output). + +In the debug shell, you can type 'bash' to get a shell with completion. +You can then look at the 'build' file, and copy paste some of its commands +to debug the problem. If you need to edit files, vim is not installed +in the build containers, but vi can be used instead. + +When you press ctrl+d (twice if you opened a bash shell) the debug shell +is closed and the container containing the failed build is removed. + +If you want to make a backup of the container's rootfs with the failed +build to look at it later, you should be able to find it in +/tmp/$UID/rbm-containers. +

1 0

[tor-browser-build/master] Add comment to explain why we build https-e once for all platforms
by boklm＠torproject.org 03 Aug '17

03 Aug '17

commit 84fd513bee8b345d636a3f9d6529fe683c40f0a1 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Thu Aug 3 21:43:42 2017 +0200 Add comment to explain why we build https-e once for all platforms --- projects/https-everywhere/config | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/projects/https-everywhere/config b/projects/https-everywhere/config index 3652aaa..6ee1ee9 100644 --- a/projects/https-everywhere/config +++ b/projects/https-everywhere/config @@ -7,6 +7,14 @@ gpg_keyring: https-everywhere.gpg tag_gpg_id: 1 filename: "[% project %]-[% c('version') %]-[% c('var/build_id') %].xpi" var: + # HTTPS Everywhere is expected to be the same on all platforms. To avoid + # building the same thing 4 times, using 4 different container images + # (each one with a different suite or architecture), we set the container + # to wheezy/amd64 for all platforms. This allows us to create only one + # container image, and also build the extension only one time as the + # filename does not contain the platform, and var/build_id should be + # the same since there is now nothing platform specific in the build + # inputs. This allows us to save a little time and disk space. container: use_container: 1 suite: wheezy

1 0

[tor-browser-build/master] Bug 22499: build obfsproxy and fteproxy for Windows
by boklm＠torproject.org 03 Aug '17

03 Aug '17

commit 6221fac747287361a0e4432c3ebf6590461bbbcf Author: Nicolas Vigier <boklm(a)torproject.org> Date: Thu Aug 3 17:56:39 2017 +0200 Bug 22499: build obfsproxy and fteproxy for Windows Also finish including the Linux build of obfsproxy and fteproxy. Based on gitian/descriptors/windows/gitian-pluggable-transports.yml from tor-browser-bundle.git. --- keyring/winpython.gpg | Bin 0 -> 1167 bytes projects/argparse/build | 20 +++++++--- projects/argparse/config | 7 ++++ projects/fteproxy/build | 21 +++++++++- projects/fteproxy/config | 9 +++++ projects/libfte/build | 23 +++++++++-- projects/libfte/config | 8 ++++ projects/obfsproxy/build | 32 ++++++++++++++-- projects/obfsproxy/config | 25 +++++++++++- projects/parsley/build | 21 +++++++--- projects/parsley/config | 7 ++++ projects/pycrypto/build | 29 +++++++++++--- projects/pycrypto/config | 7 ++++ projects/pyptlib/build | 21 +++++++--- projects/pyptlib/config | 7 ++++ projects/pyyaml/build | 21 +++++++--- projects/pyyaml/config | 7 ++++ projects/tor-browser/build | 4 ++ projects/tor-browser/config | 3 ++ projects/twisted/build | 32 +++++++++++++--- projects/twisted/config | 10 +++++ projects/txsocksx/build | 21 +++++++--- projects/txsocksx/config | 7 ++++ projects/winpython/build | 36 ++++++++++++++++++ projects/winpython/config | 55 +++++++++++++++++++++++++++ projects/winpython/pyc-timestamp.sh | 18 +++++++++ projects/winpython/wine-wrappers/common.py | 27 +++++++++++++ projects/winpython/wine-wrappers/dllwrap.py | 36 ++++++++++++++++++ projects/winpython/wine-wrappers/g++.py | 39 +++++++++++++++++++ projects/winpython/wine-wrappers/gcc.py | 36 ++++++++++++++++++ projects/winpython/wine-wrappers/settings.py | 2 + projects/winpython/wine-wrappers/setup.py | 7 ++++ projects/zope.interface/build | 27 ++++++++++--- projects/zope.interface/config | 9 +++++ rbm.conf | 29 ++++++++++++++ 35 files changed, 612 insertions(+), 51 deletions(-) diff --git a/keyring/winpython.gpg b/keyring/winpython.gpg new file mode 100644 index 0000000..97abdc1 Binary files /dev/null and b/keyring/winpython.gpg differ diff --git a/projects/argparse/build b/projects/argparse/build index 938c5e9..29e4221 100644 --- a/projects/argparse/build +++ b/projects/argparse/build @@ -1,14 +1,24 @@ #!/bin/bash [% c("var/set_default_env") -%] distdir="/var/tmp/dist/[% project %]" -[% c("var/set_PTDIR_DOCSDIR") -%] -mkdir -p $PTDIR +[% IF c("var/windows") -%] + [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] +[% ELSE -%] + [% c("var/set_PTDIR_DOCSDIR") -%] + mkdir -p $PTDIR +[% END -%] mkdir -p /var/tmp/build tar -C /var/tmp/build -xf $rootdir/[% project %]-[% c('version') %].tar.gz cd /var/tmp/build/[% project %]-[% c('version') %] -export PYTHON=python2 -$PYTHON setup.py build --build-lib build -cp -a build/argparse.py $PTDIR/ +[% IF c("var/windows") -%] + pydir="$distdir/python" + mkdir -p $pydir/Lib/site-packages + export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages" + $PYTHON setup.py install --prefix=$(winepath -w $pydir) +[% ELSE -%] + python2 setup.py build --build-lib build + cp -a build/argparse.py $PTDIR/ +[% END -%] cd $distdir [% c('tar', { tar_src => [ '.' ], diff --git a/projects/argparse/config b/projects/argparse/config index 87f80b0..c4d9dec 100644 --- a/projects/argparse/config +++ b/projects/argparse/config @@ -12,8 +12,15 @@ targets: arch_deps: - python-setuptools - python-dev + windows-i686: + var: + compiler: winpython + post_pkginst: '[% c("var/install_wine_ppa") %]' input_files: - project: container-image - URL: 'https://pypi.python.org/packages/source/a/argparse/argparse-[% c("version") %].tar.gz' sha256sum: ddaf4b0a618335a32b6664d4ae038a1de8fbada3b25033f9021510ed2b3941a4 + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]' + enable: '[% c("var/windows") %]' diff --git a/projects/fteproxy/build b/projects/fteproxy/build index b33c3a5..0d9baf1 100644 --- a/projects/fteproxy/build +++ b/projects/fteproxy/build @@ -3,11 +3,28 @@ distdir="/var/tmp/dist/[% project %]" [% c("var/set_PTDIR_DOCSDIR") -%] mkdir -p $PTDIR $DOCSDIR +[% IF c("var/windows") -%] + [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] +[% END -%] +tar -C $distdir -xf $rootdir/[% c('input_files_by_name/libfte') %] mkdir -p /var/tmp/build tar -C /var/tmp/build -xf $rootdir/[% project %]-[% c('version') %].tar.gz cd /var/tmp/build/[% project %]-[% c('version') %] -cp -a bin/fteproxy $PTDIR/fteproxy.bin -cp -ra fteproxy $PTDIR/ +[% IF c("var/windows") -%] + pydir="$distdir/python" + export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages" + $PYTHON setup_tbb.py py2exe + py2exe_zip_timestomp dist/fteproxy.zip + cp -an dist/{*.pyd,*.exe,*.zip} $PTDIR/ + mkdir -p $PTDIR/fteproxy + cp -a fteproxy/VERSION $PTDIR/fteproxy + mkdir -p $PTDIR/fteproxy/defs + cp -a fteproxy/defs/*.json $PTDIR/fteproxy/defs + rm -Rf $pydir +[% ELSE -%] + cp -a bin/fteproxy $PTDIR/fteproxy.bin + cp -ra fteproxy $PTDIR/ +[% END -%] cp -a {COPYING,README.md} $DOCSDIR/ cd $distdir [% c('tar', { diff --git a/projects/fteproxy/config b/projects/fteproxy/config index 059381b..6a13e0d 100644 --- a/projects/fteproxy/config +++ b/projects/fteproxy/config @@ -15,6 +15,15 @@ targets: arch_deps: - python-setuptools - python-dev + windows-i686: + var: + compiler: winpython + post_pkginst: '[% c("var/install_wine_ppa") %]' input_files: - project: container-image + - project: libfte + name: libfte + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]' + enable: '[% c("var/windows") %]' diff --git a/projects/libfte/build b/projects/libfte/build index fc7b9f6..62c8fce 100644 --- a/projects/libfte/build +++ b/projects/libfte/build @@ -10,9 +10,26 @@ tar -C /var/tmp/build -xf $rootdir/[% project %]-[% c('version') %].tar.gz cd /var/tmp/build/[% project %]-[% c('version') %] tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/gmp') %] ln -s /var/tmp/dist/gmp thirdparty/gmp -export PYTHON=python2 -make -cp -ra fte $PTDIR/ +mkdir -p $distdir +tar -C $distdir -xf $rootdir/[% c('input_files_by_name/obfsproxy') %] +[% IF c("var/windows") -%] + pydir=$distdir/python + export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]" + export LD_PRELOAD= + export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages" + # FTE only needs libgmp-10.dll and no libgmpxx anymore. + cp -a /var/tmp/dist/gmp/bin/libgmp-10.dll . + cp -a /var/tmp/dist/gmp/bin/libgmp-10.dll $distdir/TorBrowser/Tor/ + WINDOWS_BUILD=1 CROSS_COMPILE=1 make + $PYTHON setup.py install --prefix=$(winepath -w "$pydir") + $PYTHON setup.py install +[% ELSE -%] + export PYTHON=python2 + make + cp -ra fte $PTDIR/ + mkdir -p $distdir/TorBrowser/Tor + cp /var/tmp/dist/gmp/lib/libgmp.so.10 $distdir/TorBrowser/Tor/ +[% END -%] cp -a {LICENSE,README.md} $DOCSDIR/ cp -a thirdparty/re2/LICENSE $DOCSDIR/LICENSE.re2 cd $distdir diff --git a/projects/libfte/config b/projects/libfte/config index e2219a7..4c34116 100644 --- a/projects/libfte/config +++ b/projects/libfte/config @@ -14,6 +14,12 @@ targets: arch_deps: - python-setuptools - python-dev + windows-i686: + var: + compiler: winpython + post_pkginst: '[% c("var/install_wine_ppa") %]' + arch_deps: + - faketime input_files: - project: container-image @@ -21,3 +27,5 @@ input_files: project: '[% c("var/compiler") %]' - project: gmp name: gmp + - project: obfsproxy + name: obfsproxy diff --git a/projects/obfsproxy/build b/projects/obfsproxy/build index 841c608..9d64aa9 100644 --- a/projects/obfsproxy/build +++ b/projects/obfsproxy/build @@ -3,13 +3,37 @@ distdir="/var/tmp/dist/[% project %]" [% c("var/set_PTDIR_DOCSDIR") -%] mkdir -p $PTDIR $DOCSDIR +[% IF c("var/windows") -%] + [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] +[% ELSE -%] +[% END -%] +mkdir -p $distdir +tar -C $distdir -xf $rootdir/[% c('input_files_by_name/pycrypto') %] +tar -C $distdir -xf $rootdir/[% c('input_files_by_name/twisted') %] +tar -C $distdir -xf $rootdir/[% c('input_files_by_name/argparse') %] +tar -C $distdir -xf $rootdir/[% c('input_files_by_name/parsley') %] +tar -C $distdir -xf $rootdir/[% c('input_files_by_name/pyptlib') %] +tar -C $distdir -xf $rootdir/[% c('input_files_by_name/pyyaml') %] +tar -C $distdir -xf $rootdir/[% c('input_files_by_name/txsocksx') %] +tar -C $distdir -xf $rootdir/[% c('input_files_by_name/zope.interface') %] mkdir -p /var/tmp/build tar -C /var/tmp/build -xf $rootdir/[% project %]-[% c('version') %].tar.gz cd /var/tmp/build/[% project %]-[% c('version') %] -export PYTHON=python2 -$PYTHON setup.py build --build-lib build -cp -a build/obfsproxy $PTDIR/ -cp -a bin/obfsproxy $PTDIR/obfsproxy.bin +[% IF c("var/windows") -%] + pydir="$distdir/python" + export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages" + $PYTHON setup_py2exe.py py2exe + $PYTHON setup.py install --prefix=$(winepath -w $pydir) + py2exe_zip_timestomp py2exe_bundle/dist/obfsproxy.zip + cp -an py2exe_bundle/dist/{*.pyd,*.exe,*.zip} $PTDIR/ + # http://bugs.winehq.org/show_bug.cgi?id=3591 + cp -a /var/tmp/dist/winpython/python27.dll $PTDIR +[% ELSE -%] + export PYTHON=python2 + $PYTHON setup.py build --build-lib build + cp -a build/obfsproxy $PTDIR/ + cp -a bin/obfsproxy $PTDIR/obfsproxy.bin +[% END -%] cp -a {LICENSE,README} $DOCSDIR cd $distdir [% c('tar', { diff --git a/projects/obfsproxy/config b/projects/obfsproxy/config index ad436dd..9134de1 100644 --- a/projects/obfsproxy/config +++ b/projects/obfsproxy/config @@ -7,7 +7,7 @@ gpg_keyring: obfsproxy.gpg filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' var: - DOCSDIR_project: Obsfproxy + DOCSDIR_project: Obfsproxy container: use_container: 1 @@ -17,6 +17,29 @@ targets: arch_deps: - python-setuptools - python-dev + windows-i686: + var: + compiler: winpython + post_pkginst: '[% c("var/install_wine_ppa") %]' input_files: - project: container-image + - project: pycrypto + name: pycrypto + - project: twisted + name: twisted + - project: argparse + name: argparse + - project: parsley + name: parsley + - project: pyptlib + name: pyptlib + - project: pyyaml + name: pyyaml + - project: txsocksx + name: txsocksx + - project: zope.interface + name: zope.interface + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]' + enable: '[% c("var/windows") %]' diff --git a/projects/parsley/build b/projects/parsley/build index ea5f19a..1d52f9a 100644 --- a/projects/parsley/build +++ b/projects/parsley/build @@ -1,14 +1,25 @@ #!/bin/bash [% c("var/set_default_env") -%] distdir="/var/tmp/dist/[% project %]" -[% c("var/set_PTDIR_DOCSDIR") -%] -mkdir -p $PTDIR +[% IF c("var/windows") -%] + [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] +[% ELSE -%] + [% c("var/set_PTDIR_DOCSDIR") -%] + mkdir -p $PTDIR +[% END -%] mkdir -p /var/tmp/build tar -C /var/tmp/build -xf $rootdir/Parsley-[% c('version') %].tar.gz cd /var/tmp/build/Parsley-[% c('version') %] -export PYTHON=python2 -$PYTHON setup.py build --build-lib build -cp -a build/parsley.py build/ometa build/terml $PTDIR/ +[% IF c("var/windows") -%] + pydir="$distdir/python" + mkdir -p $pydir/Lib/site-packages + export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages" + $PYTHON setup.py install --prefix=$(winepath -w $pydir) +[% ELSE -%] + PYTHON=python2 + $PYTHON setup.py build --build-lib build + cp -a build/parsley.py build/ometa build/terml $PTDIR/ +[% END -%] cd $distdir [% c('tar', { tar_src => [ '.' ], diff --git a/projects/parsley/config b/projects/parsley/config index 72d5bfc..b788899 100644 --- a/projects/parsley/config +++ b/projects/parsley/config @@ -12,8 +12,15 @@ targets: arch_deps: - python-setuptools - python-dev + windows-i686: + var: + compiler: winpython + post_pkginst: '[% c("var/install_wine_ppa") %]' input_files: - project: container-image - URL: 'https://pypi.python.org/packages/source/P/Parsley/Parsley-[% c("version") %].tar.gz' sha256sum: 50d30cee70770fd44db7cea421cb2fb75af247c3a1cd54885c06b30a7c85dd23 + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]' + enable: '[% c("var/windows") %]' diff --git a/projects/pycrypto/build b/projects/pycrypto/build index 9b005ab..51ec5ce 100644 --- a/projects/pycrypto/build +++ b/projects/pycrypto/build @@ -1,22 +1,39 @@ #!/bin/bash +set -e [% c("var/set_default_env") -%] [% c("var/setarch") -%] [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] distdir="/var/tmp/dist/[% project %]" -[% c("var/set_PTDIR_DOCSDIR") -%] -mkdir -p $PTDIR +[% IF c("var/linux") -%] + [% c("var/set_PTDIR_DOCSDIR") -%] + mkdir -p $PTDIR +[% END -%] mkdir -p /var/tmp/build tar -C /var/tmp/build -xf $rootdir/[% project %]-[% c('version') %].tar.gz cd /var/tmp/build/[% project %]-[% c('version') %] -export PYTHON=python2 [% IF c("var/linux-i686") -%] export CFLAGS=-m32 export CXXFLAGS=-m32 export LDFLAGS=-m32 [% END -%] -./configure --build=i686-linux-gnu [% c("var/configure_opt") %] -$PYTHON setup.py build --build-lib build -cp -a build/Crypto $PTDIR/ +[% IF c("var/windows") -%] + pydir="$distdir/python" + mkdir -p "$pydir" + export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]" + export LD_PRELOAD=[% c("var/faketime_path") %] + # This is bogus, that we run the configure script in the build environment, + # but it seems to work. https://bugs.launchpad.net/pycrypto/+bug/1096207 for + # ac_cv_func_malloc_0_nonnull. + ac_cv_func_malloc_0_nonnull=yes sh configure --host=i686-w64-mingw32 + export LD_PRELOAD= + $PYTHON setup.py build_ext -c mingw32 + $PYTHON setup.py install --prefix=$(winepath -w $pydir) +[% ELSE -%] + export PYTHON=python2 + ./configure --build=i686-linux-gnu [% c("var/configure_opt") %] + python2 setup.py build --build-lib build + cp -a build/Crypto $PTDIR/ +[% END -%] cd $distdir [% c('tar', { tar_src => [ '.' ], diff --git a/projects/pycrypto/config b/projects/pycrypto/config index 3a9f24b..ca314c8 100644 --- a/projects/pycrypto/config +++ b/projects/pycrypto/config @@ -12,6 +12,13 @@ targets: - python-dev - python-setuptools + windows-i686: + var: + compiler: winpython + post_pkginst: '[% c("var/install_wine_ppa") %]' + arch_deps: + - faketime + input_files: - project: container-image - name: '[% c("var/compiler") %]' diff --git a/projects/pyptlib/build b/projects/pyptlib/build index 1dc0c07..de70c5b 100644 --- a/projects/pyptlib/build +++ b/projects/pyptlib/build @@ -1,14 +1,25 @@ #!/bin/bash [% c("var/set_default_env") -%] distdir="/var/tmp/dist/[% project %]" -[% c("var/set_PTDIR_DOCSDIR") -%] -mkdir -p $PTDIR +[% IF c("var/linux") -%] + [% c("var/set_PTDIR_DOCSDIR") -%] + mkdir -p $PTDIR +[% END -%] +[% IF c("var/windows") -%] + [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] +[% END -%] mkdir -p /var/tmp/build tar -C /var/tmp/build -xf $rootdir/[% project %]-[% c('version') %].tar.gz cd /var/tmp/build/[% project %]-[% c('version') %] -export PYTHON=python2 -$PYTHON setup.py build --build-lib build -cp -a build/pyptlib $PTDIR/ +[% IF c("var/windows") -%] + pydir="$distdir/python" + mkdir -p "$pydir" + $PYTHON setup.py install --single-version-externally-managed --record /dev/null --prefix=$(winepath -w $pydir) +[% ELSE -%] + export PYTHON=python2 + $PYTHON setup.py build --build-lib build + cp -a build/pyptlib $PTDIR/ +[% END -%] cd $distdir [% c('tar', { tar_src => [ '.' ], diff --git a/projects/pyptlib/config b/projects/pyptlib/config index e8ca100..7ae1d68 100644 --- a/projects/pyptlib/config +++ b/projects/pyptlib/config @@ -16,6 +16,13 @@ targets: arch_deps: - python-setuptools - python-dev + windows-i686: + var: + compiler: winpython + post_pkginst: '[% c("var/install_wine_ppa") %]' input_files: - project: container-image + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]' + enable: '[% c("var/windows") %]' diff --git a/projects/pyyaml/build b/projects/pyyaml/build index c10855c..d3d60d7 100644 --- a/projects/pyyaml/build +++ b/projects/pyyaml/build @@ -1,14 +1,25 @@ #!/bin/bash [% c("var/set_default_env") -%] distdir="/var/tmp/dist/[% project %]" -[% c("var/set_PTDIR_DOCSDIR") -%] -mkdir -p $PTDIR +[% IF c("var/windows") -%] + [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] +[% ELSE -%] + [% c("var/set_PTDIR_DOCSDIR") -%] + mkdir -p $PTDIR +[% END -%] mkdir -p /var/tmp/build tar -C /var/tmp/build -xf $rootdir/PyYAML-[% c('version') %].tar.gz cd /var/tmp/build/PyYAML-[% c('version') %] -export PYTHON=python2 -$PYTHON setup.py build --build-lib build -cp -a build/yaml $PTDIR/ +[% IF c("var/windows") -%] + pydir="$distdir/python" + mkdir -p $pydir/Lib/site-packages + export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages" + $PYTHON setup.py install --prefix=$(winepath -w $pydir) +[% ELSE -%] + export PYTHON=python2 + $PYTHON setup.py build --build-lib build + cp -a build/yaml $PTDIR/ +[% END -%] cd $distdir [% c('tar', { tar_src => [ '.' ], diff --git a/projects/pyyaml/config b/projects/pyyaml/config index 5324c10..8d43bf6 100644 --- a/projects/pyyaml/config +++ b/projects/pyyaml/config @@ -12,8 +12,15 @@ targets: arch_deps: - python-setuptools - python-dev + windows-i686: + var: + compiler: winpython + post_pkginst: '[% c("var/install_wine_ppa") %]' input_files: - project: container-image - URL: 'https://pypi.python.org/packages/source/P/PyYAML/PyYAML-[% c("version") %].tar.gz' sha256sum: c36c938a872e5ff494938b33b14aaa156cb439ec67548fcab3535bb78b0846e8 + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]' + enable: '[% c("var/windows") %]' diff --git a/projects/tor-browser/build b/projects/tor-browser/build index b9537a3..77a02ea 100644 --- a/projects/tor-browser/build +++ b/projects/tor-browser/build @@ -56,6 +56,10 @@ mv $TBDIR/meek-http-helper(a)bamsoftware.com.xpi $TBDIR/$MEEKPROFILEPATH/extension tar -C $TBDIR -xf [% c('input_files_by_name/snowflake') -%] [% END -%] +[% IF c("var/fteproxy") -%] + tar -C $TBDIR -xf [% c('input_files_by_name/fteproxy') %] +[% END -%] + tar -C $TBDIR[% IF c("var/osx") %]/Contents/Resources[% END %] -xf [% c('input_files_by_name/fonts') %] [% IF c("var/linux") %] diff --git a/projects/tor-browser/config b/projects/tor-browser/config index fef279b..4f708bc 100644 --- a/projects/tor-browser/config +++ b/projects/tor-browser/config @@ -50,6 +50,9 @@ input_files: name: meek - project: obfs4 name: obfs4 + - project: fteproxy + name: fteproxy + enable: '[% c("var/fteproxy") %]' - project: snowflake name: snowflake enable: '[% c("var/snowflake") %]' diff --git a/projects/twisted/build b/projects/twisted/build index acc5c9d..0d9154f 100644 --- a/projects/twisted/build +++ b/projects/twisted/build @@ -1,14 +1,36 @@ #!/bin/bash [% c("var/set_default_env") -%] distdir="/var/tmp/dist/[% project %]" -[% c("var/set_PTDIR_DOCSDIR") -%] -mkdir -p $PTDIR +[% IF c("var/windows") -%] + [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] +[% ELSE -%] + [% c("var/set_PTDIR_DOCSDIR") -%] + mkdir -p $PTDIR +[% END -%] mkdir -p /var/tmp/build tar -C /var/tmp/build -xf $rootdir/Twisted-[% c('version') %].tar.bz2 cd /var/tmp/build/Twisted-[% c('version') %] -export PYTHON=python2 -$PYTHON setup.py build --build-lib build -cp -a build/twisted $PTDIR/ +[% IF c("var/windows") -%] + pydir="$distdir/python" + export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]" + export LD_PRELOAD=[% c("var/faketime_path") %] + # twisted/internet/iocpreactor/iocpsupport/iocpsupport.c includes "python.h" + # rather than "Python.h". + ln -sf Python.h /var/tmp/dist/winpython/include/python.h + # We need to set the "mingw32" compiler to avoid an error in build_ext, but + # Twisted's "install" command calls build_ext unconditionally, whether the + # extensions have been built already or not, so we can't just call build_ext + # separately as with other packages. The "install" command doesn't recognize + # the -c option, so we set the compiler in a configuration file. + echo $'[build_ext]\ncompiler=mingw32' > setup.cfg + mkdir -p $pydir/Lib/site-packages + export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages" + LD_PRELOAD= $PYTHON setup.py install --single-version-externally-managed --record /dev/null --prefix=$(winepath -w $pydir) +[% ELSE -%] + export PYTHON=python2 + $PYTHON setup.py build --build-lib build + cp -a build/twisted $PTDIR/ +[% END -%] cd $distdir [% c('tar', { tar_src => [ '.' ], diff --git a/projects/twisted/config b/projects/twisted/config index f2f7de8..2f35166 100644 --- a/projects/twisted/config +++ b/projects/twisted/config @@ -12,8 +12,18 @@ targets: arch_deps: - python-setuptools - python-dev + windows-i686: + var: + compiler: winpython + post_pkginst: '[% c("var/install_wine_ppa") %]' + arch_deps: + - p7zip-full + - faketime input_files: - project: container-image - URL: 'https://pypi.python.org/packages/source/T/Twisted/Twisted-[% c("version") %].tar.bz2' sha256sum: 095175638c019ac7c0604f4c291724a16ff1acd062e181b01293bf4dcbc62cf3 + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]' + enable: '[% c("var/windows") %]' diff --git a/projects/txsocksx/build b/projects/txsocksx/build index 3491a26..31d7b18 100644 --- a/projects/txsocksx/build +++ b/projects/txsocksx/build @@ -1,8 +1,12 @@ #!/bin/bash [% c("var/set_default_env") -%] distdir="/var/tmp/dist/[% project %]" -[% c("var/set_PTDIR_DOCSDIR") -%] -mkdir -p $PTDIR +[% IF c("var/windows") -%] + [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] +[% ELSE -%] + [% c("var/set_PTDIR_DOCSDIR") -%] + mkdir -p $PTDIR +[% END -%] mkdir -p /var/tmp/build tar -C /var/tmp/build -xf $rootdir/[% project %]-[% c('version') %].tar.gz cd /var/tmp/build/[% project %]-[% c('version') %] @@ -11,9 +15,16 @@ cd /var/tmp/build/[% project %]-[% c('version') %] # dependency should be fine here as txsocksx catches the exception due to # missing __version__ and __sha__ . mkdir vcversioner-1.14.1.1-py2.7.egg -export PYTHON=python2 -$PYTHON setup.py build --build-lib build -cp -a build/txsocksx $PTDIR/ +[% IF c("var/windows") -%] + pydir="$distdir/python" + mkdir -p $pydir/Lib/site-packages + export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages" + $PYTHON setup.py install_lib --install-dir=$(winepath -w "$pydir/Lib/site-packages") +[% ELSE -%] + export PYTHON=python2 + $PYTHON setup.py build --build-lib build + cp -a build/txsocksx $PTDIR/ +[% END -%] cd $distdir [% c('tar', { tar_src => [ '.' ], diff --git a/projects/txsocksx/config b/projects/txsocksx/config index 6142943..18fc4a8 100644 --- a/projects/txsocksx/config +++ b/projects/txsocksx/config @@ -14,6 +14,13 @@ targets: arch_deps: - python-setuptools - python-dev + windows-i686: + var: + compiler: winpython + post_pkginst: '[% c("var/install_wine_ppa") %]' input_files: - project: container-image + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]' + enable: '[% c("var/windows") %]' diff --git a/projects/winpython/build b/projects/winpython/build new file mode 100644 index 0000000..2a30acc --- /dev/null +++ b/projects/winpython/build @@ -0,0 +1,36 @@ +#!/bin/bash +set -e +[% c("var/set_default_env") -%] +[% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] +distdir="/var/tmp/dist/[% project %]" +mkdir -p $distdir +WINEROOT=$HOME/.wine/drive_c +wineboot -i +msiexec /qn /i python-[% c("version") %].msi TARGETDIR=$distdir +sed -i 's/self.dll_libraries = get_msvcr()/pass#self.dll_libraries = get_msvcr()/g' $distdir/Lib/distutils/cygwinccompiler.py +tar xf setuptools-*.tar.gz +rm setuptools-*.tar.gz +cd setuptools-* +wine $distdir/python.exe setup.py install +cd $rootdir +7z x $rootdir/py2exe-*.exe +cp -a PLATLIB/* $distdir/Lib/site-packages/ +export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]" +export LD_PRELOAD=[% c("var/faketime_path") %] +cd wine-wrappers +# Push our config into wine-wrappers. +> settings.py +echo "LD_PRELOAD = \"$LD_PRELOAD\"" >> settings.py +echo "FAKETIME = \"$FAKETIME\"" >> settings.py +# Must pre-copy python27.dll into the build directory, or else py2exe can't find it. +mkdir -p build/bdist.win32/winexe/bundle-2.7/ +cp -a $distdir/python27.dll build/bdist.win32/winexe/bundle-2.7/ +LD_PRELOAD= $distdir/python.exe setup.py py2exe +mkdir -p $distdir/wineroot/windows/ +cp -a dist/gcc.exe dist/g++.exe dist/dllwrap.exe $distdir/wineroot/windows/ +cp $rootdir/pyc-timestamp.sh $distdir/ +cd $distdir/.. +[% c('tar', { + tar_src => [ project, 'mingw-w64' ], + tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'), + }) %] diff --git a/projects/winpython/config b/projects/winpython/config new file mode 100644 index 0000000..dcd21f6 --- /dev/null +++ b/projects/winpython/config @@ -0,0 +1,55 @@ +# vim: filetype=yaml sw=2 +version: 2.7.5 +filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' + +var: + compiler: mingw-w64 + post_pkginst: '[% c("var/install_wine_ppa") %]' + arch_deps: + - p7zip-full + - faketime + container: + use_container: 1 + setup: | + [% pc('mingw-w64', 'var/setup') %] + # wine path ($HOME/.wine) gets included in some binaries, so set HOME + # to a fixed path. + export HOME=/var/tmp/home + mkdir -p $HOME + WINEROOT=$HOME/.wine/drive_c + wineboot -i + cp -a /var/tmp/dist/winpython/wineroot/windows/* $WINEROOT/windows/ + export PYTHON="wine /var/tmp/dist/winpython/python.exe" + + # Set the timestamp on every .pyc file in a zip file, and re-dzip the zip file. + function py2exe_zip_timestomp { + ZIPFILE="$1" + local tmpdir="$(mktemp -d)" + local tmpzip="$(mktemp -u)" + unzip -d "$tmpdir" "$ZIPFILE" + cd "$tmpdir" + find . -name '*.pyc' -print0 | xargs -0 /var/tmp/dist/winpython/pyc-timestamp.sh "2000-01-01 00:00:00" + [% c('zip', { + zip_src => [ '.' ], + zip_args => '$tmpzip', + }) %] + cd - + mv -f "$tmpzip" "$ZIPFILE" + rm -rf "$tmpdir" + } + + +input_files: + - project: container-image + - URL: 'https://www.python.org/ftp/python/[% c("version") %]/python-[% c("version") %].msi' + file_gpg_id: 1 + sig_ext: asc + gpg_keyring: winpython.gpg + - URL: https://pypi.python.org/packages/source/s/setuptools/setuptools-1.4.tar.gz + sha256sum: 75d288687066ed124311d6ca5f40ffa92a0e81adcd7fff318c6e84082713cf39 + - URL: http://downloads.sourceforge.net/py2exe/0.6.9/py2exe-0.6.9.win32-py2.7.exe + sha256sum: 610a8800de3d973ed5ed4ac505ab42ad058add18a68609ac09e6cf3598ef056c + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]' + - filename: wine-wrappers + - filename: pyc-timestamp.sh diff --git a/projects/winpython/pyc-timestamp.sh b/projects/winpython/pyc-timestamp.sh new file mode 100755 index 0000000..e092268 --- /dev/null +++ b/projects/winpython/pyc-timestamp.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Usage: pyc-timestamp.sh "2001-01-01" FILENAMES... +# Overwrite (in place) the timestamp in .pyc Python bytecode files. +# +# http://hg.python.org/cpython/file/2.7/Lib/py_compile.py#l123 +# http://nedbatchelder.com/blog/200804/the_structure_of_pyc_files.html +# http://benno.id.au/blog/2013/01/15/python-determinism + +TIMESPEC="$1" +shift + +hex=$(printf 0x%08x $(date +%s --date="$TIMESPEC")) +# Write little-endian. +esc=$(printf "\\\\x%02x\\\\x%02x\\\\x%02x\\\\x%02x" $(($hex&0xff)) $((($hex>>8)&0xff)) $((($hex>>16)&0xff)) $((($hex>>24)&0xff))) +for filename in "$@"; do + echo $filename + echo -n -e "$esc" | dd of="$filename" bs=1 seek=4 conv=notrunc +done diff --git a/projects/winpython/wine-wrappers/common.py b/projects/winpython/wine-wrappers/common.py new file mode 100644 index 0000000..19e994b --- /dev/null +++ b/projects/winpython/wine-wrappers/common.py @@ -0,0 +1,27 @@ +import os +import subprocess + +import settings + +def winepath(windowspath): + """Convert a Windows path to a Unix path.""" + return subprocess.check_output(["winepath", "-u", windowspath])[:-1] + +def search_startswith(a, elems): + """Search for the first element of the array a that startswith any of the + members of elems, and return it. Return None if no match was found.""" + for e in elems: + if a.startswith(e): + return e + return None + +def popen_faketime(*args, **kwargs): + """Does subprocess.Popen after setting faketime environment variables.""" + ld_preload = os.getenv("LD_PRELOAD") + if ld_preload: + # ld.so(8): "The items of [LD_PRELOAD] can be separated by spaces or colons." + ld_preload += ":" + ld_preload += settings.LD_PRELOAD + os.putenv("LD_PRELOAD", ld_preload) + os.putenv("FAKETIME", settings.FAKETIME) + return subprocess.Popen(*args, **kwargs) diff --git a/projects/winpython/wine-wrappers/dllwrap.py b/projects/winpython/wine-wrappers/dllwrap.py new file mode 100755 index 0000000..300a3bd --- /dev/null +++ b/projects/winpython/wine-wrappers/dllwrap.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +# A wrapper for i686-w64-mingw32-dllwrap that removes -mno-cygwin and converts +# Windows paths to Unix paths, so that the w64-mingw32 dllwrap can be called by +# Python distutils. + +import os +import subprocess +import sys + +import common + +args = ["/var/tmp/dist/mingw-w64/bin/i686-w64-mingw32-dllwrap"] +sys.argv.pop(0) +while sys.argv: + a = sys.argv.pop(0) + if not a.startswith("-"): + args.append(common.winepath(a)) + continue + if a == "-mno-cygwin": + continue + if a in ("-I", "-L", "-s", "--def", "--output-lib"): + args.append(a) + args.append(common.winepath(sys.argv.pop(0))) + continue + o = common.search_startswith(a, ("-I", "-L", "--def=", "--output-lib=")) + if o is not None: + path = a[len(o):] + args.append("%s%s" % (o, common.winepath(path))) + continue + args.append(a) +p = common.popen_faketime(args, stderr=subprocess.PIPE) +stderr = p.stderr.read() +sys.stderr.write(stderr) +if " error: " in stderr: + sys.exit(1) diff --git a/projects/winpython/wine-wrappers/g++.py b/projects/winpython/wine-wrappers/g++.py new file mode 100755 index 0000000..a71c4e3 --- /dev/null +++ b/projects/winpython/wine-wrappers/g++.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +# A wrapper for i686-w64-mingw32-g++ that removes -mno-cygwin and converts +# Windows paths to Unix paths, so that the w64-mingw32 g++ can be called by +# Python distutils. + +import os +import subprocess +import sys + +import common + +args = ["/var/tmp/dist/mingw-w64/bin/i686-w64-mingw32-g++"] +sys.argv.pop(0) +while sys.argv: + a = sys.argv.pop(0) + if not a.startswith("-"): + args.append(common.winepath(a)) + continue + if a == "-mno-cygwin": + continue + if a == "--output-lib": + o = sys.argv.pop(0) + a = "-out-implib="+o + if a in ("-I", "-L"): + args.append(a) + args.append(common.winepath(sys.argv.pop(0))) + continue + o = common.search_startswith(a, ("-I", "-L")) + if o is not None: + path = a[len(o):] + args.append("%s%s" % (o, common.winepath(path))) + continue + args.append(a) +p = common.popen_faketime(args, stderr=subprocess.PIPE) +stderr = p.stderr.read() +sys.stderr.write(stderr) +if " error: " in stderr: + sys.exit(1) diff --git a/projects/winpython/wine-wrappers/gcc.py b/projects/winpython/wine-wrappers/gcc.py new file mode 100755 index 0000000..3db89b4 --- /dev/null +++ b/projects/winpython/wine-wrappers/gcc.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +# A wrapper for i686-w64-mingw32-gcc that removes -mno-cygwin and converts +# Windows paths to Unix paths, so that the w64-mingw32 gcc can be called by +# Python distutils. + +import os +import subprocess +import sys + +import common + +args = ["/var/tmp/dist/mingw-w64/bin/i686-w64-mingw32-gcc"] +sys.argv.pop(0) +while sys.argv: + a = sys.argv.pop(0) + if not a.startswith("-"): + args.append(common.winepath(a)) + continue + if a == "-mno-cygwin": + continue + if a in ("-I", "-L"): + args.append(a) + args.append(common.winepath(sys.argv.pop(0))) + continue + o = common.search_startswith(a, ("-I", "-L")) + if o is not None: + path = a[len(o):] + args.append("%s%s" % (o, common.winepath(path))) + continue + args.append(a) +p = common.popen_faketime(args, stderr=subprocess.PIPE) +stderr = p.stderr.read() +sys.stderr.write(stderr) +if " error: " in stderr: + sys.exit(1) diff --git a/projects/winpython/wine-wrappers/settings.py b/projects/winpython/wine-wrappers/settings.py new file mode 100644 index 0000000..f395433 --- /dev/null +++ b/projects/winpython/wine-wrappers/settings.py @@ -0,0 +1,2 @@ +LD_PRELOAD = "/usr/lib/faketime/libfaketime.so.1" +FAKETIME = "2000-01-01 00:00:00" diff --git a/projects/winpython/wine-wrappers/setup.py b/projects/winpython/wine-wrappers/setup.py new file mode 100644 index 0000000..f80f2c0 --- /dev/null +++ b/projects/winpython/wine-wrappers/setup.py @@ -0,0 +1,7 @@ +from distutils.core import setup +import py2exe +setup( + console=["gcc.py", "g++.py", "dllwrap.py"], + zipfile=None, + options={"py2exe": {"bundle_files": 1, "compressed": True}} +) diff --git a/projects/zope.interface/build b/projects/zope.interface/build index 23501a3..29406fb 100644 --- a/projects/zope.interface/build +++ b/projects/zope.interface/build @@ -1,14 +1,31 @@ #!/bin/bash [% c("var/set_default_env") -%] distdir="/var/tmp/dist/[% project %]" -[% c("var/set_PTDIR_DOCSDIR") -%] -mkdir -p $PTDIR +[% IF c("var/linux") -%] + [% c("var/set_PTDIR_DOCSDIR") -%] + mkdir -p $PTDIR +[% END -%] +[% IF c("var/windows") -%] + [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] +[% END -%] mkdir -p /var/tmp/build unzip -d /var/tmp/build $rootdir/[% project %]-[% c('version') %].zip cd /var/tmp/build/[% project %]-[% c('version') %] -export PYTHON=python2 -$PYTHON setup.py build --build-lib build -cp -a build/zope $PTDIR/ +[% IF c("var/windows") -%] + find -type f -print0 | xargs -0 [% c("var/touch") %] + pydir="$distdir/python" + mkdir -p $pydir/Lib/site-packages + export PYTHONPATH="$(winepath -w $pydir)\\Lib\\site-packages" + export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]" + LD_PRELOAD= $PYTHON setup.py build_ext -c mingw32 + $PYTHON setup.py install --single-version-externally-managed --record /dev/null --prefix=$(winepath -w $pydir) + # Must create this file in order for py2exe to find the package. + touch $pydir/Lib/site-packages/zope/__init__.py +[% ELSE -%] + export PYTHON=python2 + $PYTHON setup.py build --build-lib build + cp -a build/zope $PTDIR/ +[% END -%] cd $distdir [% c('tar', { tar_src => [ '.' ], diff --git a/projects/zope.interface/config b/projects/zope.interface/config index 9743b79..4ae3cff 100644 --- a/projects/zope.interface/config +++ b/projects/zope.interface/config @@ -12,8 +12,17 @@ targets: arch_deps: - python-setuptools - python-dev + windows-i686: + var: + compiler: winpython + post_pkginst: '[% c("var/install_wine_ppa") %]' + arch_deps: + - faketime input_files: - project: container-image - URL: 'https://pypi.python.org/packages/source/z/zope.interface/zope.interface-[% c("version") %].zip' sha256sum: 1a7c84716bbd9981915b64a81d8a3f076a5934a8c8df4224655469b3564940cc + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]' + enable: '[% c("var/windows") %]' diff --git a/rbm.conf b/rbm.conf index ae8c5b2..c93019e 100644 --- a/rbm.conf +++ b/rbm.conf @@ -144,6 +144,7 @@ targets: compiler: gcc # We only build snowflake on linux and osx for now snowflake: 1 + fteproxy: 1 container: suite: wheezy deps: @@ -186,6 +187,34 @@ targets: export RBM_SETARCH=1 exec setarch i686 ./build fi + fteproxy: 1 + install_wine_ppa: | + # Install a Wine new enough to have a fix for + # http://bugs.winehq.org/show_bug.cgi?id=29764; otherwise Python run under + # Wine constantly crashes in _PyVerify_fd, which is called by such common + # operations as io.open and os.fstat (anything involving a file descriptor + # number). Ubuntu's main repository only has wine1.4, and the issue was fixed + # in 1.5.29. + echo 'deb http://ppa.launchpad.net/ubuntu-wine/ppa/ubuntu precise main' >> /etc/apt/sources.list + # This key is from https://launchpad.net/~ubuntu-wine/+archive/ppa and + # http://keyserver.ubuntu.com:11371/pks/lookup?op=get&search=0x5A9A06AEF9CB8D…. + apt-key add - << EOF + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1 + + mI0ESXVzlQEEAN1BxiR961SiFrJ7tacrAImCmDdxs4OSifgpBAp8q0fe0iLcSeAo + WdS7H+7Y4T+/2t3XEw7+3cD831WBu8c/Pv0ldx5TyDyOQmEtUBlMqy33WdKVxsci + rnag8ShrNM9PtG/vAQU/JtfQl68dXfD3BsCtrXVrjqcO2AxNYBSvd9hpABEBAAG0 + IkxhdW5jaHBhZCBQUEEgZm9yIFVidW50dSBXaW5lIFRlYW2ItgQTAQIAIAUCSXVz + lQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEFqaBq75y42wflcD/jLMihWM + zRCO60S/a7SqD0QNqV/nAYEOsma/Y2C/uhQ44j0np+iOB6+PDRbyJ8GVgIjpaIbt + l4sReXcf7bS9Dhhn5Gbe/n6VQr7xoMr1Io0PrXdmWpmsOfCuebWU4bV1w+YBvHFF + qTu5QF2Y0Fj9bRxQdQ1M2HcnXOiLq82hKlb+ + =Z9DY + -----END PGP PUBLIC KEY BLOCK----- + EOF + apt-get update + apt-get --no-install-recommends -y install wine torbrowser-osx-x86_64: - osx-x86_64

1 0