commit bedc9b317052aeabc1adca379acaba80634cab87 Author: Mike Perry <mikeperry-git@fscked.org> Date: Tue Jan 1 00:00:00 2013 -0800 Fix a minor issue with the pipeline depth patch. We were accidentally creating too many connections at once, which impacts pipelining. --- ...ize-HTTP-request-order-and-pipeline-depth.patch | 151 +++++++++++++++----- 1 files changed, 112 insertions(+), 39 deletions(-) diff --git a/src/current-patches/firefox/0017-Randomize-HTTP-request-order-and-pipeline-depth.patch b/src/current-patches/firefox/0017-Randomize-HTTP-request-order-and-pipeline-depth.patch index a82e429..11d0e47 100644 --- a/src/current-patches/firefox/0017-Randomize-HTTP-request-order-and-pipeline-depth.patch +++ b/src/current-patches/firefox/0017-Randomize-HTTP-request-order-and-pipeline-depth.patch @@ -1,4 +1,4 @@ -From 6043bc14b28d49618e7882707c56b2fac4ada2a9 Mon Sep 17 00:00:00 2001 +From aa6534432718dae0e3810dc441fc797ecf809c6e Mon Sep 17 00:00:00 2001 From: Mike Perry <mikeperry-git@torproject.org> Date: Tue, 4 Dec 2012 17:38:51 -0800 Subject: [PATCH 17/28] Randomize HTTP request order and pipeline depth. @@ -36,15 +36,15 @@ request representation that will allow more requests to be packed inside Tor cells. If you have interest in evaluating SPDY in a study of Website Traffic Fingerprinting, please contact me. --- - netwerk/protocol/http/nsHttpConnectionMgr.cpp | 298 +++++++++++++++++-------- + netwerk/protocol/http/nsHttpConnectionMgr.cpp | 351 +++++++++++++++++-------- netwerk/protocol/http/nsHttpConnectionMgr.h | 15 +- netwerk/protocol/http/nsHttpHandler.h | 2 + netwerk/protocol/http/nsHttpPipeline.cpp | 62 ++++- netwerk/protocol/http/nsHttpPipeline.h | 3 + - 5 files changed, 282 insertions(+), 98 deletions(-) + 5 files changed, 327 insertions(+), 106 deletions(-) diff --git a/netwerk/protocol/http/nsHttpConnectionMgr.cpp b/netwerk/protocol/http/nsHttpConnectionMgr.cpp -index 133c301..69de433 100644 +index 133c301..c98894c 100644 --- a/netwerk/protocol/http/nsHttpConnectionMgr.cpp +++ b/netwerk/protocol/http/nsHttpConnectionMgr.cpp @@ -20,6 +20,8 @@ @@ -89,7 +89,7 @@ index 133c301..69de433 100644 } //----------------------------------------------------------------------------- -@@ -919,6 +932,8 @@ nsHttpConnectionMgr::ProcessPendingQForEntry(nsConnectionEntry *ent) +@@ -919,22 +932,27 @@ nsHttpConnectionMgr::ProcessPendingQForEntry(nsConnectionEntry *ent) nsHttpTransaction *trans; nsresult rv; bool dispatchedSuccessfully = false; @@ -98,7 +98,33 @@ index 133c301..69de433 100644 // iterate the pending list until one is dispatched successfully. Keep // iterating afterwards only until a transaction fails to dispatch. -@@ -953,16 +968,29 @@ nsHttpConnectionMgr::ProcessPendingQForEntry(nsConnectionEntry *ent) + for (uint32_t i = 0; i < count; ++i) { + trans = ent->mPendingQ[i]; + +- // When this transaction has already established a half-open ++ // When this entry has already established a half-open + // connection, we want to prevent any duplicate half-open + // connections from being established and bound to this +- // transaction. Allow only use of an idle persistent connection +- // (if found) for transactions referred by a half-open connection. ++ // transaction. + bool alreadyHalfOpen = false; +- for (int32_t j = 0; j < ((int32_t) ent->mHalfOpens.Length()); ++j) { +- if (ent->mHalfOpens[j]->Transaction() == trans) { +- alreadyHalfOpen = true; +- break; ++ if (ent->SupportsPipelining()) { ++ alreadyHalfOpen = (ent->UnconnectedHalfOpens() > 0); ++ } else { ++ for (int32_t j = 0; j < ((int32_t) ent->mHalfOpens.Length()); ++j) { ++ if (ent->mHalfOpens[j]->Transaction() == trans) { ++ alreadyHalfOpen = true; ++ break; ++ } + } + } + +@@ -953,16 +971,29 @@ nsHttpConnectionMgr::ProcessPendingQForEntry(nsConnectionEntry *ent) dispatchedSuccessfully = true; count = ent->mPendingQ.Length(); --i; @@ -130,7 +156,18 @@ index 133c301..69de433 100644 return false; } -@@ -1263,7 +1291,7 @@ nsHttpConnectionMgr::MakeNewConnection(nsConnectionEntry *ent, +@@ -1247,6 +1278,10 @@ nsHttpConnectionMgr::MakeNewConnection(nsConnectionEntry *ent, + if (AtActiveConnectionLimit(ent, trans->Caps())) + return NS_ERROR_NOT_AVAILABLE; + ++#ifdef WTF_DEBUG ++ fprintf(stderr, "WTF: MakeNewConnection() is creating a transport (pipelines %d) for host %s\n", ++ ent->SupportsPipelining(), ent->mConnInfo->Host()); ++#endif + nsresult rv = CreateTransport(ent, trans, trans->Caps(), false); + if (NS_FAILED(rv)) { + /* hard failure */ +@@ -1263,7 +1298,7 @@ nsHttpConnectionMgr::MakeNewConnection(nsConnectionEntry *ent, } bool @@ -139,7 +176,7 @@ index 133c301..69de433 100644 nsHttpTransaction *trans, nsHttpTransaction::Classifier classification, uint16_t depthLimit) -@@ -1300,40 +1328,100 @@ nsHttpConnectionMgr::AddToShortestPipeline(nsConnectionEntry *ent, +@@ -1300,40 +1335,100 @@ nsHttpConnectionMgr::AddToShortestPipeline(nsConnectionEntry *ent, if (maxdepth < 2) return false; @@ -192,11 +229,11 @@ index 133c301..69de433 100644 + continue; + + numPipelines++; -+ -+ pipelineDepth = activeTrans->PipelineDepth(); -+ requestLen = pipeline->RequestDepth(); - if (maxdepth <= connLength) ++ pipelineDepth = activeTrans->PipelineDepth(); ++ requestLen = pipeline->RequestDepth(); ++ + totalDepth += pipelineDepth; + + // If we're within striking distance of our pipeline @@ -222,8 +259,7 @@ index 133c301..69de433 100644 + // queued already, or for which we can add multiple requests + if (requestLen + allClasses < mMaxOptimisticPipelinedRequests) + continue; - -- if (!bestConn) ++ + betterConns.AppendElement(conn); + + // Prefer a pipeline with the same classification if @@ -232,7 +268,8 @@ index 133c301..69de433 100644 + continue; + if (requestLen + sameClass < mMaxOptimisticPipelinedRequests) + continue; -+ + +- if (!bestConn) + bestConns.AppendElement(conn); + } + @@ -254,7 +291,7 @@ index 133c301..69de433 100644 activeTrans = bestConn->Transaction(); nsresult rv = activeTrans->AddTransaction(trans); -@@ -1343,6 +1431,15 @@ nsHttpConnectionMgr::AddToShortestPipeline(nsConnectionEntry *ent, +@@ -1343,6 +1438,15 @@ nsHttpConnectionMgr::AddToShortestPipeline(nsConnectionEntry *ent, LOG((" scheduling trans %p on pipeline at position %d\n", trans, trans->PipelinePosition())); @@ -270,7 +307,7 @@ index 133c301..69de433 100644 if ((ent->PipelineState() == PS_YELLOW) && (trans->PipelinePosition() > 1)) ent->SetYellowConnection(bestConn); return true; -@@ -1403,26 +1500,12 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, +@@ -1403,26 +1507,12 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, nsHttpTransaction::Classifier classification = trans->Classification(); uint8_t caps = trans->Caps(); @@ -299,7 +336,7 @@ index 133c301..69de433 100644 // step 0 // look for existing spdy connection - that's always best because it is // essentially pipelining without head of line blocking -@@ -1436,20 +1519,27 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, +@@ -1436,20 +1526,27 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, } } @@ -339,7 +376,7 @@ index 133c301..69de433 100644 nsRefPtr<nsHttpConnection> conn; while (!conn && (ent->mIdleConns.Length() > 0)) { conn = ent->mIdleConns[0]; -@@ -1483,21 +1573,8 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, +@@ -1483,21 +1580,8 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, } } @@ -363,7 +400,7 @@ index 133c301..69de433 100644 nsresult rv = MakeNewConnection(ent, trans); if (NS_SUCCEEDED(rv)) { // this function returns NOT_AVAILABLE for asynchronous connects -@@ -1510,17 +1587,16 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, +@@ -1510,17 +1594,16 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent, return rv; } } @@ -390,13 +427,21 @@ index 133c301..69de433 100644 return NS_ERROR_NOT_AVAILABLE; /* queue it */ } -@@ -1590,10 +1666,20 @@ nsHttpConnectionMgr::DispatchAbstractTransaction(nsConnectionEntry *ent, +@@ -1590,10 +1673,28 @@ nsHttpConnectionMgr::DispatchAbstractTransaction(nsConnectionEntry *ent, if (!NS_SUCCEEDED(rv)) return rv; transaction = pipeline; +#ifdef WTF_DEBUG -+ fprintf(stderr, "WTF: New pipeline created from %d idle conns for host %s\n", -+ ent->mIdleConns.Length(), ent->mConnInfo->Host()); ++ if (HasPipelines(ent) && ++ ent->mPendingQ.Length()+1 < mMaxOptimisticPipelinedRequests) { ++ fprintf(stderr, "WTF-new-bug: New pipeline created from %d idle conns for host %s with %d/%d pending\n", ++ ent->mIdleConns.Length(), ent->mConnInfo->Host(), ent->mPendingQ.Length(), ++ mMaxOptimisticPipelinedRequests); ++ } else { ++ fprintf(stderr, "WTF-new: New pipeline created from %d idle conns for host %s with %d/%d pending\n", ++ ent->mIdleConns.Length(), ent->mConnInfo->Host(), ent->mPendingQ.Length(), ++ mMaxOptimisticPipelinedRequests); ++ } +#endif } else { @@ -411,9 +456,14 @@ index 133c301..69de433 100644 } nsRefPtr<nsConnectionHandle> handle = new nsConnectionHandle(conn); -@@ -1692,27 +1778,15 @@ nsHttpConnectionMgr::ProcessNewTransaction(nsHttpTransaction *trans) +@@ -1691,28 +1792,20 @@ nsHttpConnectionMgr::ProcessNewTransaction(nsHttpTransaction *trans) + NS_ABORT_IF_FALSE(((int32_t)ent->mActiveConns.IndexOf(conn)) != -1, "Sticky Connection Not In Active List"); trans->SetConnection(nullptr); ++#ifdef WTF_TEST ++ fprintf(stderr, "WTF-bad: Sticky connection status on 1 transaction to host %s\n", ++ ent->mConnInfo->Host()); ++#endif rv = DispatchTransaction(ent, trans, conn); - } - else @@ -442,19 +492,34 @@ index 133c301..69de433 100644 } -@@ -2311,13 +2385,37 @@ nsHttpConnectionMgr::OnMsgSpeculativeConnect(int32_t, void *param) +@@ -2311,10 +2404,48 @@ nsHttpConnectionMgr::OnMsgSpeculativeConnect(int32_t, void *param) if (preferredEntry) ent = preferredEntry; -+ /* Only speculative connect if we're not pipelining */ - if (!ent->mIdleConns.Length() && !RestrictConnections(ent) && +- if (!ent->mIdleConns.Length() && !RestrictConnections(ent) && - !AtActiveConnectionLimit(ent, trans->Caps())) { -+ !HasPipelines(ent) && !AtActiveConnectionLimit(ent, trans->Caps())) { ++ if (ent->SupportsPipelining()) { ++ /* Only speculative connect if we're not pipelining and have no other pending ++ * unconnected half-opens.. */ ++ if (ent->UnconnectedHalfOpens() == 0 && ent->mIdleConns.Length() == 0 ++ && !RestrictConnections(ent) && !HasPipelines(ent) ++ && !AtActiveConnectionLimit(ent, trans->Caps())) { ++#ifdef WTF_DEBUG ++ fprintf(stderr, "WTF: Creating speculative connection because we have no pipelines\n"); ++#endif ++ CreateTransport(ent, trans, trans->Caps(), true); ++ } ++ } else if (!ent->mIdleConns.Length() && !RestrictConnections(ent) && ++ !AtActiveConnectionLimit(ent, trans->Caps())) { ++#ifdef WTF_DEBUG ++ fprintf(stderr, "WTF: Creating speculative connection because we can't pipeline\n"); ++#endif CreateTransport(ent, trans, trans->Caps(), true); } - } - - bool ++ ++} ++ ++bool +nsHttpConnectionMgr::HasPipelines(nsConnectionEntry *ent) +{ + uint32_t activeCount = ent->mActiveConns.Length(); @@ -475,13 +540,21 @@ index 133c301..69de433 100644 + return true; + } + return false; -+} -+ -+bool - nsHttpConnectionMgr::nsConnectionHandle::IsPersistent() - { - return mConn->IsPersistent(); -@@ -2852,9 +2950,13 @@ nsConnectionEntry::nsConnectionEntry(nsHttpConnectionInfo *ci) + } + + bool +@@ -2661,6 +2792,10 @@ nsHalfOpenSocket::OnOutputStreamReady(nsIAsyncOutputStream *out) + nsRefPtr<nsHttpTransaction> temp = dont_AddRef(mEnt->mPendingQ[index]); + mEnt->mPendingQ.RemoveElementAt(index); + gHttpHandler->ConnMgr()->AddActiveConn(conn, mEnt); ++#ifdef WTF_DEBUG ++ fprintf(stderr, "WTF: Speculative half-opened connection is now ready for %s (pipelines %d)\n", ++ mEnt->mConnInfo->Host(), mEnt->SupportsPipelining()); ++#endif + rv = gHttpHandler->ConnMgr()->DispatchTransaction(mEnt, temp, conn); + } + else { +@@ -2852,9 +2987,13 @@ nsConnectionEntry::nsConnectionEntry(nsHttpConnectionInfo *ci) { NS_ADDREF(mConnInfo); if (gHttpHandler->GetPipelineAggressive()) { @@ -496,7 +569,7 @@ index 133c301..69de433 100644 mInitialGreenDepth = mGreenDepth; memset(mPipeliningClassPenalty, 0, sizeof(int16_t) * nsAHttpTransaction::CLASS_MAX); } -@@ -2892,8 +2994,9 @@ nsConnectionEntry::OnPipelineFeedbackInfo( +@@ -2892,8 +3031,9 @@ nsConnectionEntry::OnPipelineFeedbackInfo( LOG(("Transaction completed at pipeline depth of %d. Host = %s\n", depth, mConnInfo->Host())); @@ -508,7 +581,7 @@ index 133c301..69de433 100644 } nsAHttpTransaction::Classifier classification; -@@ -2921,6 +3024,11 @@ nsConnectionEntry::OnPipelineFeedbackInfo( +@@ -2921,6 +3061,11 @@ nsConnectionEntry::OnPipelineFeedbackInfo( mPipelineState, mConnInfo->Host())); mPipelineState = PS_RED; mPipeliningPenalty = 0;