[tor-commits] [snowflake/master] Add proxy type to stats exported by broker
cohosh at torproject.org
cohosh at torproject.org
Thu Nov 28 18:54:16 UTC 2019
commit 981abffbd93673c483a0d41631e472303b3fd0aa
Author: Cecylia Bocovich <cohosh at torproject.org>
Date: Wed Nov 20 13:27:04 2019 -0500
Add proxy type to stats exported by broker
---
broker/broker.go | 2 +-
broker/metrics.go | 73 +++++++++++++++++++++++++++++++++++------
broker/snowflake-broker_test.go | 58 +++++++++++++++++++++++++-------
3 files changed, 111 insertions(+), 22 deletions(-)
diff --git a/broker/broker.go b/broker/broker.go
index 3edfe84..e897ffd 100644
--- a/broker/broker.go
+++ b/broker/broker.go
@@ -173,7 +173,7 @@ func proxyPolls(ctx *BrokerContext, w http.ResponseWriter, r *http.Request) {
if err != nil {
log.Println("Error processing proxy IP: ", err.Error())
} else {
- ctx.metrics.UpdateCountryStats(remoteIP)
+ ctx.metrics.UpdateCountryStats(remoteIP, ptype)
}
// Wait for a client to avail an offer to the snowflake, or timeout if nil.
diff --git a/broker/metrics.go b/broker/metrics.go
index 15a4a88..b5c423c 100644
--- a/broker/metrics.go
+++ b/broker/metrics.go
@@ -19,6 +19,24 @@ We export metrics in the following format:
A count of the total number of unique IP addresses of snowflake
proxies that have polled.
+ "snowflake-ips-standalone" NUM NL
+ [At most once.]
+
+ A count of the total number of unique IP addresses of snowflake
+ proxies of type "standalone" that have polled.
+
+ "snowflake-ips-badge" NUM NL
+ [At most once.]
+
+ A count of the total number of unique IP addresses of snowflake
+ proxies of type "badge" that have polled.
+
+ "snowflake-ips-webext" NUM NL
+ [At most once.]
+
+ A count of the total number of unique IP addresses of snowflake
+ proxies of type "webext" that have polled.
+
"snowflake-idle-count" NUM NL
[At most once.]
@@ -58,8 +76,11 @@ var (
const metricsResolution = 60 * 60 * 24 * time.Second //86400 seconds
type CountryStats struct {
- addrs map[string]bool
- counts map[string]int
+ standalone map[string]bool
+ badge map[string]bool
+ webext map[string]bool
+ unknown map[string]bool
+ counts map[string]int
}
// Implements Observable
@@ -89,13 +110,27 @@ func (s CountryStats) Display() string {
return output
}
-func (m *Metrics) UpdateCountryStats(addr string) {
+func (m *Metrics) UpdateCountryStats(addr string, ptype string) {
var country string
var ok bool
- if m.countryStats.addrs[addr] {
- return
+ if ptype == "standalone" {
+ if m.countryStats.standalone[addr] {
+ return
+ }
+ } else if ptype == "badge" {
+ if m.countryStats.badge[addr] {
+ return
+ }
+ } else if ptype == "webext" {
+ if m.countryStats.webext[addr] {
+ return
+ }
+ } else {
+ if m.countryStats.unknown[addr] {
+ return
+ }
}
ip := net.ParseIP(addr)
@@ -118,7 +153,15 @@ func (m *Metrics) UpdateCountryStats(addr string) {
//update map of unique ips and counts
m.countryStats.counts[country]++
- m.countryStats.addrs[addr] = true
+ if ptype == "standalone" {
+ m.countryStats.standalone[addr] = true
+ } else if ptype == "badge" {
+ m.countryStats.badge[addr] = true
+ } else if ptype == "webext" {
+ m.countryStats.webext[addr] = true
+ } else {
+ m.countryStats.unknown[addr] = true
+ }
}
@@ -148,8 +191,11 @@ func NewMetrics(metricsLogger *log.Logger) (*Metrics, error) {
m := new(Metrics)
m.countryStats = CountryStats{
- counts: make(map[string]int),
- addrs: make(map[string]bool),
+ counts: make(map[string]int),
+ standalone: make(map[string]bool),
+ badge: make(map[string]bool),
+ webext: make(map[string]bool),
+ unknown: make(map[string]bool),
}
m.logger = metricsLogger
@@ -172,7 +218,11 @@ func (m *Metrics) logMetrics() {
func (m *Metrics) printMetrics() {
m.logger.Println("snowflake-stats-end", time.Now().UTC().Format("2006-01-02 15:04:05"), fmt.Sprintf("(%d s)", int(metricsResolution.Seconds())))
m.logger.Println("snowflake-ips", m.countryStats.Display())
- m.logger.Println("snowflake-ips-total", len(m.countryStats.addrs))
+ m.logger.Println("snowflake-ips-total", len(m.countryStats.standalone)+
+ len(m.countryStats.badge)+len(m.countryStats.webext)+len(m.countryStats.unknown))
+ m.logger.Println("snowflake-ips-standalone", len(m.countryStats.standalone))
+ m.logger.Println("snowflake-ips-badge", len(m.countryStats.badge))
+ m.logger.Println("snowflake-ips-webext", len(m.countryStats.webext))
m.logger.Println("snowflake-idle-count", binCount(m.proxyIdleCount))
m.logger.Println("client-denied-count", binCount(m.clientDeniedCount))
m.logger.Println("client-snowflake-match-count", binCount(m.clientProxyMatchCount))
@@ -184,7 +234,10 @@ func (m *Metrics) zeroMetrics() {
m.clientDeniedCount = 0
m.clientProxyMatchCount = 0
m.countryStats.counts = make(map[string]int)
- m.countryStats.addrs = make(map[string]bool)
+ m.countryStats.standalone = make(map[string]bool)
+ m.countryStats.badge = make(map[string]bool)
+ m.countryStats.webext = make(map[string]bool)
+ m.countryStats.unknown = make(map[string]bool)
}
// Rounds up a count to the nearest multiple of 8.
diff --git a/broker/snowflake-broker_test.go b/broker/snowflake-broker_test.go
index cb5f34f..b23e688 100644
--- a/broker/snowflake-broker_test.go
+++ b/broker/snowflake-broker_test.go
@@ -387,7 +387,7 @@ func TestGeoip(t *testing.T) {
if err := ctx.metrics.LoadGeoipDatabases("invalid_filename", "invalid_filename6"); err != nil {
log.Printf("loading geo ip databases returned error: %v", err)
}
- ctx.metrics.UpdateCountryStats("127.0.0.1")
+ ctx.metrics.UpdateCountryStats("127.0.0.1", "")
So(ctx.metrics.tablev4, ShouldEqual, nil)
})
@@ -408,7 +408,6 @@ func TestMetrics(t *testing.T) {
w := httptest.NewRecorder()
data := bytes.NewReader([]byte("{\"Sid\":\"ymbcCMto7KHNGYlp\",\"Version\":\"1.0\"}"))
r, err := http.NewRequest("POST", "snowflake.broker/proxy", data)
- r.Header.Set("X-Session-ID", "test")
r.RemoteAddr = "129.97.208.23:8888" //CA geoip
So(err, ShouldBeNil)
go func(ctx *BrokerContext) {
@@ -419,8 +418,47 @@ func TestMetrics(t *testing.T) {
p.offerChannel <- nil
<-done
+ w = httptest.NewRecorder()
+ data = bytes.NewReader([]byte(`{"Sid":"ymbcCMto7KHNGYlp","Version":"1.0","Type":"standalone"}`))
+ r, err = http.NewRequest("POST", "snowflake.broker/proxy", data)
+ r.RemoteAddr = "129.97.208.23:8888" //CA geoip
+ So(err, ShouldBeNil)
+ go func(ctx *BrokerContext) {
+ proxyPolls(ctx, w, r)
+ done <- true
+ }(ctx)
+ p = <-ctx.proxyPolls //manually unblock poll
+ p.offerChannel <- nil
+ <-done
+
+ w = httptest.NewRecorder()
+ data = bytes.NewReader([]byte(`{"Sid":"ymbcCMto7KHNGYlp","Version":"1.0","Type":"badge"}`))
+ r, err = http.NewRequest("POST", "snowflake.broker/proxy", data)
+ r.RemoteAddr = "129.97.208.23:8888" //CA geoip
+ So(err, ShouldBeNil)
+ go func(ctx *BrokerContext) {
+ proxyPolls(ctx, w, r)
+ done <- true
+ }(ctx)
+ p = <-ctx.proxyPolls //manually unblock poll
+ p.offerChannel <- nil
+ <-done
+
+ w = httptest.NewRecorder()
+ data = bytes.NewReader([]byte(`{"Sid":"ymbcCMto7KHNGYlp","Version":"1.0","Type":"webext"}`))
+ r, err = http.NewRequest("POST", "snowflake.broker/proxy", data)
+ r.RemoteAddr = "129.97.208.23:8888" //CA geoip
+ So(err, ShouldBeNil)
+ go func(ctx *BrokerContext) {
+ proxyPolls(ctx, w, r)
+ done <- true
+ }(ctx)
+ p = <-ctx.proxyPolls //manually unblock poll
+ p.offerChannel <- nil
+ <-done
ctx.metrics.printMetrics()
- So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips CA=1\nsnowflake-ips-total 1\nsnowflake-idle-count 8\nclient-denied-count 0\nclient-snowflake-match-count 0\n")
+ So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips CA=4\nsnowflake-ips-total 4\nsnowflake-ips-standalone 1\nsnowflake-ips-badge 1\nsnowflake-ips-webext 1\nsnowflake-idle-count 8\nclient-denied-count 0\nclient-snowflake-match-count 0\n")
+
})
//Test addition of client failures
@@ -433,13 +471,13 @@ func TestMetrics(t *testing.T) {
clientOffers(ctx, w, r)
ctx.metrics.printMetrics()
- So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-idle-count 0\nclient-denied-count 8\nclient-snowflake-match-count 0\n")
+ So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-ips-standalone 0\nsnowflake-ips-badge 0\nsnowflake-ips-webext 0\nsnowflake-idle-count 0\nclient-denied-count 8\nclient-snowflake-match-count 0\n")
// Test reset
buf.Reset()
ctx.metrics.zeroMetrics()
ctx.metrics.printMetrics()
- So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-idle-count 0\nclient-denied-count 0\nclient-snowflake-match-count 0\n")
+ So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-ips-standalone 0\nsnowflake-ips-badge 0\nsnowflake-ips-webext 0\nsnowflake-idle-count 0\nclient-denied-count 0\nclient-snowflake-match-count 0\n")
})
//Test addition of client matches
Convey("for client-proxy match", func() {
@@ -460,7 +498,7 @@ func TestMetrics(t *testing.T) {
<-done
ctx.metrics.printMetrics()
- So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-idle-count 0\nclient-denied-count 0\nclient-snowflake-match-count 8\n")
+ So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-ips-standalone 0\nsnowflake-ips-badge 0\nsnowflake-ips-webext 0\nsnowflake-idle-count 0\nclient-denied-count 0\nclient-snowflake-match-count 8\n")
})
//Test rounding boundary
Convey("binning boundary", func() {
@@ -479,12 +517,12 @@ func TestMetrics(t *testing.T) {
clientOffers(ctx, w, r)
ctx.metrics.printMetrics()
- So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-idle-count 0\nclient-denied-count 8\nclient-snowflake-match-count 0\n")
+ So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-ips-standalone 0\nsnowflake-ips-badge 0\nsnowflake-ips-webext 0\nsnowflake-idle-count 0\nclient-denied-count 8\nclient-snowflake-match-count 0\n")
clientOffers(ctx, w, r)
buf.Reset()
ctx.metrics.printMetrics()
- So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-idle-count 0\nclient-denied-count 16\nclient-snowflake-match-count 0\n")
+ So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-ips-standalone 0\nsnowflake-ips-badge 0\nsnowflake-ips-webext 0\nsnowflake-idle-count 0\nclient-denied-count 16\nclient-snowflake-match-count 0\n")
})
//Test unique ip
@@ -492,7 +530,6 @@ func TestMetrics(t *testing.T) {
w := httptest.NewRecorder()
data := bytes.NewReader([]byte(`{"Sid":"ymbcCMto7KHNGYlp","Version":"1.0"}`))
r, err := http.NewRequest("POST", "snowflake.broker/proxy", data)
- r.Header.Set("X-Session-ID", "test")
r.RemoteAddr = "129.97.208.23:8888" //CA geoip
So(err, ShouldBeNil)
go func(ctx *BrokerContext) {
@@ -508,7 +545,6 @@ func TestMetrics(t *testing.T) {
if err != nil {
log.Printf("unable to get NewRequest with error: %v", err)
}
- r.Header.Set("X-Session-ID", "test")
r.RemoteAddr = "129.97.208.23:8888" //CA geoip
go func(ctx *BrokerContext) {
proxyPolls(ctx, w, r)
@@ -519,7 +555,7 @@ func TestMetrics(t *testing.T) {
<-done
ctx.metrics.printMetrics()
- So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips CA=1\nsnowflake-ips-total 1\nsnowflake-idle-count 8\nclient-denied-count 0\nclient-snowflake-match-count 0\n")
+ So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips CA=1\nsnowflake-ips-total 1\nsnowflake-ips-standalone 0\nsnowflake-ips-badge 0\nsnowflake-ips-webext 0\nsnowflake-idle-count 8\nclient-denied-count 0\nclient-snowflake-match-count 0\n")
})
})
}
More information about the tor-commits
mailing list