commit f520a5037c96de05f208552158802fa4840055af Author: Mike Perry <mikeperry-git@fscked.org> Date: Tue Apr 2 16:02:56 2013 -0700 Bug 8470: Increase pipeline randomization. --- build-scripts/config/pound_tor.js | 3 + ...ize-HTTP-request-order-and-pipeline-depth.patch | 708 +++++++++++++++++--- 2 files changed, 623 insertions(+), 88 deletions(-) diff --git a/build-scripts/config/pound_tor.js b/build-scripts/config/pound_tor.js index ab98b46..db382a6 100644 --- a/build-scripts/config/pound_tor.js +++ b/build-scripts/config/pound_tor.js @@ -97,6 +97,9 @@ pref("network.http.proxy.pipelining", true); pref("security.ssl.enable_false_start", true); pref("network.http.keep-alive.timeout", 20); pref("network.http.connection-retry-timeout", 0); +pref("network.http.max-persistent-connections-per-proxy", 256); +// Hacked pref: Now means "Attempt to pipeline at least this many requests together" +pref("network.http.pipelining.max-optimistic-requests", 3); // Extension support pref("extensions.autoDisableScopes", 0); diff --git a/src/current-patches/firefox/0017-Randomize-HTTP-request-order-and-pipeline-depth.patch b/src/current-patches/firefox/0017-Randomize-HTTP-request-order-and-pipeline-depth.patch index ddfb184..a5d2957 100644 --- a/src/current-patches/firefox/0017-Randomize-HTTP-request-order-and-pipeline-depth.patch +++ b/src/current-patches/firefox/0017-Randomize-HTTP-request-order-and-pipeline-depth.patch @@ -1,7 +1,7 @@ -From cd87b7f64f035f67ec883c1b1ed4746454892781 Mon Sep 17 00:00:00 2001 +From a88adc5c3000c3e2a18b1065da77dd83d2b3ae7b Mon Sep 17 00:00:00 2001 From: Mike Perry <mikeperry-git@torproject.org> Date: Tue, 4 Dec 2012 17:38:51 -0800 -Subject: [PATCH 17/27] Randomize HTTP request order and pipeline depth. +Subject: [PATCH 17/28] Randomize HTTP request order and pipeline depth. This is an experimental defense against http://lorre.uni.lu/~andriy/papers/acmccs-wpes11-fingerprinting.pdf @@ -13,18 +13,21 @@ This defense has been improved since that blog post to additionally randomize the order and concurrency of non-pipelined HTTP requests. This patch is also different from the 10.x ESR patch, as the pipelining -code has changed. We may want to set network.http.pipelining.aggressive to get -similar behavior... +code has changed significantly. -The good news is we now randomize SPDY request order as well as pipeline -request order (though SPDY is still disabled by default in TBB). +This patch may have some minor impact on SPDY request order, but the SPDY +implementation has not been altered directly. It has several stream queues +that may also benefit from reordering. --- - netwerk/protocol/http/nsHttpConnectionMgr.cpp | 67 +++++++++++++++++++++++-- - netwerk/protocol/http/nsHttpConnectionMgr.h | 3 + - 2 files changed, 65 insertions(+), 5 deletions(-) + netwerk/protocol/http/nsHttpConnectionMgr.cpp | 288 +++++++++++++++++-------- + netwerk/protocol/http/nsHttpConnectionMgr.h | 15 +- + netwerk/protocol/http/nsHttpHandler.h | 2 + + netwerk/protocol/http/nsHttpPipeline.cpp | 62 +++++- + netwerk/protocol/http/nsHttpPipeline.h | 3 + + 5 files changed, 272 insertions(+), 98 deletions(-) diff --git a/netwerk/protocol/http/nsHttpConnectionMgr.cpp b/netwerk/protocol/http/nsHttpConnectionMgr.cpp -index 133c301..59d03c0 100644 +index 133c301..872d505 100644 --- a/netwerk/protocol/http/nsHttpConnectionMgr.cpp +++ b/netwerk/protocol/http/nsHttpConnectionMgr.cpp @@ -20,6 +20,8 @@ @@ -36,108 +39,465 @@ index 133c301..59d03c0 100644 using namespace mozilla; using namespace mozilla::net; -@@ -39,15 +41,46 @@ InsertTransactionSorted(nsTArray<nsHttpTransaction*> &pendingQ, nsHttpTransactio +@@ -39,15 +41,26 @@ InsertTransactionSorted(nsTArray<nsHttpTransaction*> &pendingQ, nsHttpTransactio // insert into queue with smallest valued number first. search in reverse // order under the assumption that many of the existing transactions will // have the same priority (usually 0). + uint32_t len = pendingQ.Length(); -+ uint32_t begin = 0, end = len+1; -+ int found_begin = 0; - for (int32_t i=pendingQ.Length()-1; i>=0; --i) { +- nsHttpTransaction *t = pendingQ[i]; +- if (trans->Priority() >= t->Priority()) { +- pendingQ.InsertElementAt(i+1, trans); +- return; +- } + if (pendingQ.IsEmpty()) { + pendingQ.InsertElementAt(0, trans); + return; -+ } + } + -+// #define PRESERVE_PRIORITY_ORDER -+#ifdef PRESERVE_PRIORITY_ORDER -+ // XXX: Untested -+ for (uint32_t i=0; i < len; ++i) { - nsHttpTransaction *t = pendingQ[i]; -- if (trans->Priority() >= t->Priority()) { -- pendingQ.InsertElementAt(i+1, trans); -- return; + pendingQ.InsertElementAt(0, trans); + -+ /* As soon as we see a priority >= us, our insertion -+ * range starts there */ -+ if (!found_begin && t->Priority() >= trans->Priority()) { -+ begin = i; -+ found_begin = 1; -+ } -+ /* As soon as we see a priority > us, our insertion -+ * range ends there */ -+ if (t->Priority() > trans->Priority()) { -+ end = i; -+ break; ++ // FIXME: Refactor into standalone helper (for nsHttpPipeline) ++ // Or at least simplify this function if this shuffle ends up ++ // being an improvement. ++ uint32_t i = 0; ++ for (i=0; i < len; ++i) { ++ uint32_t ridx = rand() % len; ++ ++ nsHttpTransaction *tmp = pendingQ[i]; ++ pendingQ[i] = pendingQ[ridx]; ++ pendingQ[ridx] = tmp; ++ } + } + + //----------------------------------------------------------------------------- +@@ -919,6 +932,8 @@ nsHttpConnectionMgr::ProcessPendingQForEntry(nsConnectionEntry *ent) + nsHttpTransaction *trans; + nsresult rv; + bool dispatchedSuccessfully = false; ++ int dispatchCount = 0; ++ int total = count; + + // iterate the pending list until one is dispatched successfully. Keep + // iterating afterwards only until a transaction fails to dispatch. +@@ -953,16 +968,29 @@ nsHttpConnectionMgr::ProcessPendingQForEntry(nsConnectionEntry *ent) + dispatchedSuccessfully = true; + count = ent->mPendingQ.Length(); + --i; ++ dispatchCount++; + continue; } + +- if (dispatchedSuccessfully) +- return true; ++ // We want to keep walking the dispatch table to ensure requests ++ // get combined properly. ++ //if (dispatchedSuccessfully) { ++ // return true; ++ //} + + NS_ABORT_IF_FALSE(count == ((int32_t) ent->mPendingQ.Length()), + "something mutated pending queue from " + "GetConnection()"); } -- pendingQ.InsertElementAt(0, trans); + -+ // XXX Verify that begin..end are all == trans->Priority() ++#ifdef WTF_DEBUG ++ if (dispatchedSuccessfully) { ++ fprintf(stderr, "WTF-queue: Dispatched %d/%d pending transactions for %s\n", ++ dispatchCount, total, ent->mConnInfo->Host()); ++ return true; ++ } +#endif + -+ // Choose random destination begin..end -+ uint32_t count = end - begin; -+ if (count == 0) count = 1; -+ -+ // FIXME: rand() is not crypto-secure.. but meh, this code will probably -+ // change like 2 dozen more times before merge, and rand() is probably -+ // good enough for our purposes anyways. -+ pendingQ.InsertElementAt(begin + (rand()%count), trans); + return false; } - //----------------------------------------------------------------------------- -@@ -68,6 +101,12 @@ nsHttpConnectionMgr::nsHttpConnectionMgr() - mCT.Init(); - mAlternateProtocolHash.Init(16); - mSpdyPreferredHash.Init(); -+ -+ nsresult rv; -+ mRandomGenerator = do_GetService("@mozilla.org/security/random-generator;1", &rv); -+ if (NS_FAILED(rv)) { -+ mRandomGenerator = nullptr; -+ } +@@ -1263,7 +1291,7 @@ nsHttpConnectionMgr::MakeNewConnection(nsConnectionEntry *ent, } - nsHttpConnectionMgr::~nsHttpConnectionMgr() -@@ -1120,6 +1159,19 @@ nsHttpConnectionMgr::AtActiveConnectionLimit(nsConnectionEntry *ent, uint8_t cap + bool +-nsHttpConnectionMgr::AddToShortestPipeline(nsConnectionEntry *ent, ++nsHttpConnectionMgr::AddToBestPipeline(nsConnectionEntry *ent, + nsHttpTransaction *trans, + nsHttpTransaction::Classifier classification, + uint16_t depthLimit) +@@ -1300,40 +1328,92 @@ nsHttpConnectionMgr::AddToShortestPipeline(nsConnectionEntry *ent, + if (maxdepth < 2) + return false; + +- nsAHttpTransaction *activeTrans; ++ // Find out how many requests of this class we have ++ uint32_t sameClass = 0; ++ uint32_t allClasses = ent->mPendingQ.Length(); ++ for (uint32_t i = 0; i < allClasses; ++i) { ++ if (trans != ent->mPendingQ[i] && ++ classification == ent->mPendingQ[i]->Classification()) { ++ sameClass++; ++ } ++ } + ++ nsAHttpTransaction *activeTrans; ++ nsHttpPipeline *pipeline; + nsHttpConnection *bestConn = nullptr; + uint32_t activeCount = ent->mActiveConns.Length(); +- uint32_t bestConnLength = 0; +- uint32_t connLength; ++ uint32_t pipelineDepth; ++ uint32_t requestLen; ++ uint32_t totalDepth = 0; ++ ++ // Now, try to find the best pipeline ++ nsTArray<nsHttpConnection *> validConns; ++ nsTArray<nsHttpConnection *> betterConns; ++ nsTArray<nsHttpConnection *> bestConns; ++ uint32_t numPipelines = 0; + + for (uint32_t i = 0; i < activeCount; ++i) { + nsHttpConnection *conn = ent->mActiveConns[i]; +- if (!conn->SupportsPipelining()) +- continue; - LOG((" connection count = %d, limit %d\n", totalCount, maxPersistConns)); +- if (conn->Classification() != classification) ++ if (!conn->SupportsPipelining()) + continue; -+ // Fuzz maxConns for website fingerprinting attack -+ // We create a range of maxConns/5 up to 6*maxConns/5 -+ // because this function is called repeatedly, and we'll -+ // end up converging on the high side of concurrent connections -+ // after a short while. -+ PRUint8 *bytes = nullptr; -+ nsresult rv = mRandomGenerator->GenerateRandomBytes(1, &bytes); -+ NS_ENSURE_SUCCESS(rv, rv); + activeTrans = conn->Transaction(); + -+ bytes[0] = bytes[0] % (maxPersistConns + 1); -+ maxPersistConns = (maxPersistConns/5) + bytes[0]; -+ NS_Free(bytes); + if (!activeTrans || + activeTrans->IsDone() || + NS_FAILED(activeTrans->Status())) + continue; + +- connLength = activeTrans->PipelineDepth(); ++ pipeline = activeTrans->QueryPipeline(); ++ if (!pipeline) ++ continue; ++ ++ numPipelines++; + - // use >= just to be safe - bool result = (totalCount >= maxPersistConns); - LOG((" result: %s", result ? "true" : "false")); -@@ -1297,6 +1349,11 @@ nsHttpConnectionMgr::AddToShortestPipeline(nsConnectionEntry *ent, ++ pipelineDepth = activeTrans->PipelineDepth(); ++ requestLen = pipeline->RequestDepth(); - maxdepth = PR_MIN(maxdepth, depthLimit); +- if (maxdepth <= connLength) ++ totalDepth += pipelineDepth; ++ ++ // Only count in-flight requests towards maxdepth. ++ if (maxdepth <= (pipelineDepth - requestLen)) + continue; -+ if (maxdepth/2 > 1) { -+ // This is a crazy hack to randomize pipeline depth a bit more.. -+ maxdepth = 1 + maxdepth/2 + (rand() % (maxdepth/2)); +- if (!bestConn || (connLength < bestConnLength)) { +- bestConn = conn; +- bestConnLength = connLength; +- } +- } ++ validConns.AppendElement(conn); ++ ++ // Prefer a pipeline that either has at least two requests ++ // queued already, or for which we can add multiple requests ++ if (requestLen + allClasses < mMaxOptimisticPipelinedRequests) ++ continue; + +- if (!bestConn) ++ betterConns.AppendElement(conn); ++ ++ // Prefer a pipeline with the same classification if ++ // our current classes will put it over the line ++ if (conn->Classification() != classification) ++ continue; ++ if (requestLen + sameClass < mMaxOptimisticPipelinedRequests) ++ continue; ++ ++ bestConns.AppendElement(conn); + } + - if (maxdepth < 2) ++ const char *type; ++ if (bestConns.Length()) { ++ type = "best"; ++ bestConn = bestConns[rand()%bestConns.Length()]; ++ } else if (betterConns.Length()) { ++ type = "better"; ++ bestConn = betterConns[rand()%betterConns.Length()]; ++ } else if (validConns.Length() && totalDepth == 0) { ++ // We only use valid conns if it's a last resort ++ // (No other requests are pending or in flight) ++ type = "valid"; ++ bestConn = validConns[rand()%validConns.Length()]; ++ } else { return false; ++ } + + activeTrans = bestConn->Transaction(); + nsresult rv = activeTrans->AddTransaction(trans); +@@ -1343,6 +1423,14 @@ nsHttpConnectionMgr::AddToShortestPipeline(nsConnectionEntry *ent, + LOG((" scheduling trans %p on pipeline at position %d\n", + trans, trans->PipelinePosition())); + ++#ifdef WTF_DEBUG ++ pipeline = activeTrans->QueryPipeline(); ++ fprintf(stderr, "WTF-depth: Added trans to %s of %d/%d/%d/%d pipelines. Request len %d/%d for %s\n", ++ type, bestConns.Length(), betterConns.Length(), validConns.Length(), ++ numPipelines, pipeline->RequestDepth(), activeTrans->PipelineDepth(), ++ ent->mConnInfo->Host()); ++#endif ++ + if ((ent->PipelineState() == PS_YELLOW) && (trans->PipelinePosition() > 1)) + ent->SetYellowConnection(bestConn); + return true; +@@ -1403,26 +1491,12 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, + nsHttpTransaction::Classifier classification = trans->Classification(); + uint8_t caps = trans->Caps(); + ++ bool allowNewPipelines = true; ++ + // no keep-alive means no pipelines either + if (!(caps & NS_HTTP_ALLOW_KEEPALIVE)) + caps = caps & ~NS_HTTP_ALLOW_PIPELINING; + +- // 0 - If this should use spdy then dispatch it post haste. +- // 1 - If there is connection pressure then see if we can pipeline this on +- // a connection of a matching type instead of using a new conn +- // 2 - If there is an idle connection, use it! +- // 3 - if class == reval or script and there is an open conn of that type +- // then pipeline onto shortest pipeline of that class if limits allow +- // 4 - If we aren't up against our connection limit, +- // then open a new one +- // 5 - Try a pipeline if we haven't already - this will be unusual because +- // it implies a low connection pressure situation where +- // MakeNewConnection() failed.. that is possible, but unlikely, due to +- // global limits +- // 6 - no connection is available - queue it +- +- bool attemptedOptimisticPipeline = !(caps & NS_HTTP_ALLOW_PIPELINING); +- + // step 0 + // look for existing spdy connection - that's always best because it is + // essentially pipelining without head of line blocking +@@ -1436,20 +1510,27 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, + } + } + +- // step 1 +- // If connection pressure, then we want to favor pipelining of any kind +- if (IsUnderPressure(ent, classification) && !attemptedOptimisticPipeline) { +- attemptedOptimisticPipeline = true; +- if (AddToShortestPipeline(ent, trans, +- classification, +- mMaxOptimisticPipelinedRequests)) { +- return NS_OK; +- } ++ // step 1: Try a pipeline ++ if (caps & NS_HTTP_ALLOW_PIPELINING && ++ AddToBestPipeline(ent, trans, classification, ++ mMaxPipelinedRequests)) { ++ return NS_OK; + } + +- // step 2 +- // consider an idle persistent connection +- if (caps & NS_HTTP_ALLOW_KEEPALIVE) { ++ // Step 2: Decide if we should forbid new pipeline creation. ++ // ++ // FIXME: We repurposed mMaxOptimisticPipelinedRequests here to mean: ++ // "Don't make a new pipeline until you have this many requests pending and ++ // no potential connections to put them on". It might be nice to give this ++ // its own pref.. ++ if (HasPipelines(ent) && ++ ent->mPendingQ.Length() < mMaxOptimisticPipelinedRequests && ++ trans->Classification() != nsAHttpTransaction::CLASS_SOLO && ++ caps & NS_HTTP_ALLOW_PIPELINING) ++ allowNewPipelines = false; ++ ++ // step 3: consider an idle persistent connection ++ if (allowNewPipelines && (caps & NS_HTTP_ALLOW_KEEPALIVE)) { + nsRefPtr<nsHttpConnection> conn; + while (!conn && (ent->mIdleConns.Length() > 0)) { + conn = ent->mIdleConns[0]; +@@ -1483,21 +1564,8 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, + } + } + +- // step 3 +- // consider pipelining scripts and revalidations +- if (!attemptedOptimisticPipeline && +- (classification == nsHttpTransaction::CLASS_REVALIDATION || +- classification == nsHttpTransaction::CLASS_SCRIPT)) { +- attemptedOptimisticPipeline = true; +- if (AddToShortestPipeline(ent, trans, +- classification, +- mMaxOptimisticPipelinedRequests)) { +- return NS_OK; +- } +- } +- +- // step 4 +- if (!onlyReusedConnection) { ++ // step 4: Maybe make a connection? ++ if (!onlyReusedConnection && allowNewPipelines) { + nsresult rv = MakeNewConnection(ent, trans); + if (NS_SUCCEEDED(rv)) { + // this function returns NOT_AVAILABLE for asynchronous connects +@@ -1510,17 +1578,16 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, + return rv; + } + } ++ ++ // XXX: We dequeue and queue the same url here sometimes.. ++#ifdef WTF_DEBUG ++ nsHttpRequestHead *head = trans->RequestHead(); ++ fprintf(stderr, "WTF: Queuing url %s%s\n", ++ ent->mConnInfo->Host(), ++ head ? head->RequestURI().BeginReading() : "<unknown?>"); ++#endif + +- // step 5 +- if (caps & NS_HTTP_ALLOW_PIPELINING) { +- if (AddToShortestPipeline(ent, trans, +- classification, +- mMaxPipelinedRequests)) { +- return NS_OK; +- } +- } +- +- // step 6 ++ // step 6: Queue it + return NS_ERROR_NOT_AVAILABLE; /* queue it */ + } + +@@ -1590,10 +1657,20 @@ nsHttpConnectionMgr::DispatchAbstractTransaction(nsConnectionEntry *ent, + if (!NS_SUCCEEDED(rv)) + return rv; + transaction = pipeline; ++#ifdef WTF_DEBUG ++ fprintf(stderr, "WTF: New pipeline created from %d idle conns for host %s\n", ++ ent->mIdleConns.Length(), ent->mConnInfo->Host()); ++#endif + } + else { + LOG((" not using pipeline datastructure due to class solo.\n")); + transaction = aTrans; ++#ifdef WTF_TEST ++ nsHttpRequestHead *head = transaction->RequestHead(); ++ fprintf(stderr, "WTF-order: Pipeline forbidden for url %s%s\n", ++ ent->mConnInfo->Host(), ++ head ? head->RequestURI().BeginReading() : "<unknown?>"); ++#endif + } + + nsRefPtr<nsConnectionHandle> handle = new nsConnectionHandle(conn); +@@ -1692,27 +1769,15 @@ nsHttpConnectionMgr::ProcessNewTransaction(nsHttpTransaction *trans) + "Sticky Connection Not In Active List"); + trans->SetConnection(nullptr); + rv = DispatchTransaction(ent, trans, conn); +- } +- else +- rv = TryDispatchTransaction(ent, false, trans); +- +- if (NS_SUCCEEDED(rv)) { +- LOG((" ProcessNewTransaction Dispatch Immediately trans=%p\n", trans)); + return rv; + } +- +- if (rv == NS_ERROR_NOT_AVAILABLE) { +- LOG((" adding transaction to pending queue " +- "[trans=%p pending-count=%u]\n", +- trans, ent->mPendingQ.Length()+1)); +- // put this transaction on the pending queue... ++ else { ++ // XXX: maybe check the queue first and directly call TryDispatch? + InsertTransactionSorted(ent->mPendingQ, trans); + NS_ADDREF(trans); ++ ProcessPendingQForEntry(ent); + return NS_OK; + } +- +- LOG((" ProcessNewTransaction Hard Error trans=%p rv=%x\n", trans, rv)); +- return rv; + } + + +@@ -2311,13 +2376,37 @@ nsHttpConnectionMgr::OnMsgSpeculativeConnect(int32_t, void *param) + if (preferredEntry) + ent = preferredEntry; ++ /* Only speculative connect if we're not pipelining */ + if (!ent->mIdleConns.Length() && !RestrictConnections(ent) && +- !AtActiveConnectionLimit(ent, trans->Caps())) { ++ !HasPipelines(ent) && !AtActiveConnectionLimit(ent, trans->Caps())) { + CreateTransport(ent, trans, trans->Caps(), true); + } + } + + bool ++nsHttpConnectionMgr::HasPipelines(nsConnectionEntry *ent) ++{ ++ uint32_t activeCount = ent->mActiveConns.Length(); ++ ++ if (!ent->SupportsPipelining()) { ++ return false; ++ } ++ ++ for (uint32_t i = 0; i < activeCount; ++i) { ++ nsHttpConnection *conn = ent->mActiveConns[i]; ++ if (!conn->SupportsPipelining()) ++ continue; ++ ++ nsAHttpTransaction *activeTrans = conn->Transaction(); ++ ++ if (activeTrans && !activeTrans->IsDone() && ++ !NS_FAILED(activeTrans->Status())) ++ return true; ++ } ++ return false; ++} ++ ++bool + nsHttpConnectionMgr::nsConnectionHandle::IsPersistent() + { + return mConn->IsPersistent(); +@@ -2852,9 +2941,12 @@ nsConnectionEntry::nsConnectionEntry(nsHttpConnectionInfo *ci) + { + NS_ADDREF(mConnInfo); + if (gHttpHandler->GetPipelineAggressive()) { +- mGreenDepth = kPipelineUnlimited; ++ // Randomize the pipeline depth (3..32) ++ mGreenDepth = gHttpHandler->GetMaxOptimisticPipelinedRequests() ++ + rand() % gHttpHandler->GetMaxPipelinedRequests(); + mPipelineState = PS_GREEN; + } ++ + mInitialGreenDepth = mGreenDepth; + memset(mPipeliningClassPenalty, 0, sizeof(int16_t) * nsAHttpTransaction::CLASS_MAX); + } +@@ -2892,8 +2984,9 @@ nsConnectionEntry::OnPipelineFeedbackInfo( + LOG(("Transaction completed at pipeline depth of %d. Host = %s\n", + depth, mConnInfo->Host())); + +- if (depth >= 3) +- mGreenDepth = kPipelineUnlimited; ++ // Don't set this. We want to keep our initial random value.. ++ //if (depth >= 3) ++ // mGreenDepth = kPipelineUnlimited; + } + + nsAHttpTransaction::Classifier classification; +@@ -2921,6 +3014,11 @@ nsConnectionEntry::OnPipelineFeedbackInfo( + mPipelineState, mConnInfo->Host())); + mPipelineState = PS_RED; + mPipeliningPenalty = 0; ++#ifdef WTF_TEST ++ fprintf(stderr, "WTF-bad: Red pipeline status disabled host %s\n", ++ mConnInfo->Host()); ++#endif ++ + } + + if (mLastCreditTime.IsNull()) diff --git a/netwerk/protocol/http/nsHttpConnectionMgr.h b/netwerk/protocol/http/nsHttpConnectionMgr.h -index 580710a..b22c669 100644 +index 580710a..7aecb68 100644 --- a/netwerk/protocol/http/nsHttpConnectionMgr.h +++ b/netwerk/protocol/http/nsHttpConnectionMgr.h -@@ -23,6 +23,7 @@ +@@ -23,11 +23,23 @@ #include "nsIObserver.h" #include "nsITimer.h" #include "nsIX509Cert3.h" @@ -145,15 +505,187 @@ index 580710a..b22c669 100644 class nsHttpPipeline; -@@ -585,6 +586,8 @@ private: - uint64_t mTimeOfNextWakeUp; - // Timer for next pruning of dead connections. - nsCOMPtr<nsITimer> mTimer; -+ // Random number generator for reordering HTTP pipeline -+ nsCOMPtr<nsIRandomGenerator> mRandomGenerator; + class nsIHttpUpgradeListener; + ++// We need our own optional debug define because pipelining behavior ++// is significantly altered by rendering speed (which is abysmal on ++// debug builds) ++#ifdef DEBUG ++# define WTF_DEBUG ++#endif ++ ++#ifdef WTF_DEBUG ++# define WTF_TEST ++#endif ++ + //----------------------------------------------------------------------------- + + class nsHttpConnectionMgr : public nsIObserver +@@ -478,6 +490,7 @@ private: + nsresult BuildPipeline(nsConnectionEntry *, + nsAHttpTransaction *, + nsHttpPipeline **); ++ bool HasPipelines(nsConnectionEntry *); + bool RestrictConnections(nsConnectionEntry *); + nsresult ProcessNewTransaction(nsHttpTransaction *); + nsresult EnsureSocketThreadTargetIfOnline(); +@@ -492,7 +505,7 @@ private: + + nsresult MakeNewConnection(nsConnectionEntry *ent, + nsHttpTransaction *trans); +- bool AddToShortestPipeline(nsConnectionEntry *ent, ++ bool AddToBestPipeline(nsConnectionEntry *ent, + nsHttpTransaction *trans, + nsHttpTransaction::Classifier classification, + uint16_t depthLimit); +diff --git a/netwerk/protocol/http/nsHttpHandler.h b/netwerk/protocol/http/nsHttpHandler.h +index 2963195..cd79069 100644 +--- a/netwerk/protocol/http/nsHttpHandler.h ++++ b/netwerk/protocol/http/nsHttpHandler.h +@@ -215,6 +215,8 @@ public: + nsCString& hostLine); + + bool GetPipelineAggressive() { return mPipelineAggressive; } ++ uint32_t GetMaxPipelinedRequests() { return mMaxPipelinedRequests; } ++ uint32_t GetMaxOptimisticPipelinedRequests() { return mMaxOptimisticPipelinedRequests; } + void GetMaxPipelineObjectSize(int64_t *outVal) + { + *outVal = mMaxPipelineObjectSize; +diff --git a/netwerk/protocol/http/nsHttpPipeline.cpp b/netwerk/protocol/http/nsHttpPipeline.cpp +index 9e59878..a9e9911 100644 +--- a/netwerk/protocol/http/nsHttpPipeline.cpp ++++ b/netwerk/protocol/http/nsHttpPipeline.cpp +@@ -87,6 +87,32 @@ nsHttpPipeline::~nsHttpPipeline() + free(mPushBackBuf); + } + ++// Generate a shuffled request ordering sequence ++void ++nsHttpPipeline::ShuffleTransOrder(uint32_t count) ++{ ++ if (count < 2) ++ return; ++ ++ uint32_t pos = mRequestQ[0]->PipelinePosition(); ++ uint32_t i = 0; ++ ++ for (i=0; i < count; ++i) { ++ uint32_t ridx = rand() % count; ++ ++ nsAHttpTransaction *tmp = mRequestQ[i]; ++ mRequestQ[i] = mRequestQ[ridx]; ++ mRequestQ[ridx] = tmp; ++ } ++ ++ for (i=0; i < count; ++i) { ++ mRequestQ[i]->SetPipelinePosition(pos); ++ pos++; ++ } ++ ++ LOG(("nsHttpPipeline::ShuffleTransOrder: Shuffled %d transactions.\n", count)); ++} ++ + nsresult + nsHttpPipeline::AddTransaction(nsAHttpTransaction *trans) + { +@@ -112,6 +138,8 @@ nsHttpPipeline::AddTransaction(nsAHttpTransaction *trans) + // the pipeline object. + trans->SetConnection(this); + ++ ShuffleTransOrder(mRequestQ.Length()); ++ + if (mConnection && !mClosed && mRequestQ.Length() == 1) + mConnection->ResumeSend(); + +@@ -760,8 +788,11 @@ nsHttpPipeline::CancelPipeline(nsresult originalReason) + if (respLen > 1) + mResponseQ.TruncateLength(1); + +- DontReuse(); +- Classify(nsAHttpTransaction::CLASS_SOLO); ++ /* Don't flag timed out connections as unreusable.. Tor is just slow :( */ ++ if (originalReason != NS_ERROR_NET_TIMEOUT) { ++ DontReuse(); ++ Classify(nsAHttpTransaction::CLASS_SOLO); ++ } + + return total; + } +@@ -842,8 +873,19 @@ nsHttpPipeline::FillSendBuf() + + uint32_t n; + uint64_t avail; ++ uint64_t totalAvailable = Available(); ++ uint64_t totalSent = 0; ++ uint64_t reqsSent = 0; ++ uint64_t alreadyPending = 0; ++ ++ mSendBufIn->Available(&alreadyPending); ++ + nsAHttpTransaction *trans; + nsITransport *transport = Transport(); ++#ifdef WTF_TEST ++ nsRefPtr<nsHttpConnectionInfo> ci; ++ GetConnectionInfo(getter_AddRefs(ci)); ++#endif + + while ((trans = Request(0)) != nullptr) { + avail = trans->Available(); +@@ -864,6 +906,7 @@ nsHttpPipeline::FillSendBuf() + } + + mSendingToProgress += n; ++ totalSent += n; + if (!mSuppressSendEvents && transport) { + // Simulate a SENDING_TO event + trans->OnTransportStatus(transport, +@@ -874,6 +917,14 @@ nsHttpPipeline::FillSendBuf() + + avail = trans->Available(); + if (avail == 0) { ++#ifdef WTF_TEST ++ nsHttpRequestHead *head = trans->RequestHead(); ++ fprintf(stderr, "WTF-order: Pipelined req %d/%d (%dB). Url: %s%s\n", ++ trans->PipelinePosition(), PipelineDepth(), n, ++ ci->Host(), head ? head->RequestURI().BeginReading() : "<unknown?>"); ++#endif ++ reqsSent++; ++ + // move transaction from request queue to response queue + mRequestQ.RemoveElementAt(0); + mResponseQ.AppendElement(trans); +@@ -893,5 +944,12 @@ nsHttpPipeline::FillSendBuf() + else + mRequestIsPartial = true; + } ++ ++#ifdef WTF_TEST ++ if (totalSent) ++ fprintf(stderr, "WTF-combine: Sent %d/%d bytes of %d combined pipelined requests for host %s\n", ++ alreadyPending+totalSent, totalAvailable, reqsSent, ci->Host()); ++#endif ++ + return NS_OK; + } +diff --git a/netwerk/protocol/http/nsHttpPipeline.h b/netwerk/protocol/http/nsHttpPipeline.h +index 746a196..4dc06c1 100644 +--- a/netwerk/protocol/http/nsHttpPipeline.h ++++ b/netwerk/protocol/http/nsHttpPipeline.h +@@ -27,11 +27,14 @@ public: + nsHttpPipeline(); + virtual ~nsHttpPipeline(); + ++ uint32_t RequestDepth() { return mRequestQ.Length(); } ++ + private: + nsresult FillSendBuf(); + + static NS_METHOD ReadFromPipe(nsIInputStream *, void *, const char *, + uint32_t, uint32_t, uint32_t *); ++ void ShuffleTransOrder(uint32_t); - // A 1s tick to call nsHttpConnection::ReadTimeoutTick on - // active http/1 connections and check for orphaned half opens. + // convenience functions + nsAHttpTransaction *Request(int32_t i) -- -1.7.5.4 +1.7.9.5