- tor-commits - lists.torproject.org

[gettor/master] Changed to tpo address
by ilv＠torproject.org 16 Feb '16

16 Feb '16

commit 519708f3947b7d906086362f7e52c7a44ff67025 Author: ilv <ilv(a)users.noreply.github.com> Date: Mon Feb 15 21:19:25 2016 -0300 Changed to tpo address --- AUTHORS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 6e3edb0..347db0b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,5 +1,5 @@ Current maintainer/core developers: - Israel Leiva <ilv(a)riseup.net> 4096R/540BFC0E + Israel Leiva <ilv(a)torproject.org> 4096R/540BFC0E Past core developers: Jacob Appelbaum <jacob(a)appelbaum.net>

1 0

[gettor/master] aagbsn and donncha made some nice contributions
by ilv＠torproject.org 16 Feb '16

16 Feb '16

commit c6c019fb18f2527867bf13529e641fa25fb6f8c6 Author: ilv <ilv(a)users.noreply.github.com> Date: Mon Feb 15 21:22:19 2016 -0300 aagbsn and donncha made some nice contributions --- AUTHORS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 347db0b..3d84253 100644 --- a/AUTHORS +++ b/AUTHORS @@ -11,3 +11,5 @@ Contributors: Nima Fatemi <nima(a)torproject.org> Poly <poly(a)darkdepths.net> Sukhbir Singh <sukhbir(a)torproject.org> 4096R/B297B391 + Aaron Gibson + Donncha O'Cearbhaill

1 0

[gettor/master] Template files used for generating the landing page and README in Github mirrors
by ilv＠torproject.org 16 Feb '16

16 Feb '16

commit 608e6ccd6acd0bd3e57e307848add257d552a492 Author: ilv <ilv(a)users.noreply.github.com> Date: Tue Feb 16 01:21:09 2016 -0300 Template files used for generating the landing page and README in Github mirrors --- upload/landing_gh.tpl | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++ upload/readme_gh.tpl | 40 +++++++++++++++++++++ 2 files changed, 136 insertions(+) diff --git a/upload/landing_gh.tpl b/upload/landing_gh.tpl new file mode 100644 index 0000000..9e083f1 --- /dev/null +++ b/upload/landing_gh.tpl @@ -0,0 +1,96 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="description" content=""> + <meta name="author" content=""> + <title>GetTor | Download Tor Browser for Android, Windows, Linux, OS X</title> + <link href="css/bootstrap.min.css" rel="stylesheet"> + <link href="css/gettor.css" rel="stylesheet"> + </head> + + <body> + + <nav class="navbar navbar-inverse navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <span class="navbar-brand"> + <span id="head-title">GetTor</span> + </span> + </div> + <div class="navbar-right"> + <span class="navbar-brand"> + <span id="head-link"><a href="https://www.torproject.org">The Tor Project</a></span> + </span> + </div> + </div> + </nav> + + <div id="main"> + <div class="container text-justify"> + <div id="description"> + <p> + <h1>Download Tor Browser</h1> + <hr /> + Below you will find links to download the latest version of Tor Browser (%VERSION%). + </p> + </div> + <br /> + <h2>Direct downloads</h2> +  + <div class="row well"> + <div class="col-lg-2"> + <img src="img/windows-logo.png" class="pad-big-right"> + </div> + <div class="col-lg-10"> + <a href="%WINDOWS%" class="big-link">Tor Browser for Windows</a><br />(<a href="%WINDOWS_SIG%">signature file</a>) + </div> + </div> + <div class="row well"> + <div class="col-lg-2"> + <img src="img/osx-logo.png" class="pad-big-right"> + </div> + <div class="col-lg-10"> + <a href="%OSX%" class="big-link">Tor Browser for OS X</a><br />(<a href="%OSX_SIG%">signature file</a>) + </div> + </div> + <div class="row well"> + <div class="col-lg-2"> + <img src="img/linux-logo.png" class="pad-big-right"> + </div> + <div class="col-lg-10"> + <a href="%LINUX32%" class="big-link">Tor Browser for Linux 32-bit</a><br />(<a href="%LINUX32_SIG%">signature file</a>) + </div> + </div> + <div class="row well"> + <div class="col-lg-2"> + <img src="img/linux-logo.png" class="pad-big-right"> + </div> + <div class="col-lg-10"> + <a href="%LINUX64%" class="big-link">Tor Browser for Linux 64-bit</a><br />(<a href="%LINUX64_SIG%">signature file</a>) + </div> + </div> + </div> + </div> + + <hr /> + + <footer class="pad-big-left"> + <p>© The Tor Project</p> + </footer> + + </div> + + </body> +</html> diff --git a/upload/readme_gh.tpl b/upload/readme_gh.tpl new file mode 100644 index 0000000..80952f3 --- /dev/null +++ b/upload/readme_gh.tpl @@ -0,0 +1,40 @@ +# Download Tor Browser + +In this repository you will find links to download the latest version of +Tor Browser, which currently is %VERSION%. Please select one of the links below: + +## Windows +[Download Tor Browser](%WINDOWS_EN%) (English) ([signature file](%WINDOWS_EN_SIG%)). + +[Download Tor Browser](%WINDOWS_FA%) (Farsi) ([signature file](%WINDOWS_FA_SIG%)). + +[Download Tor Browser](%WINDOWS_TR%) (Turkish) ([signature file](%WINDOWS_TR_SIG%)). + +[Download Tor Browser](%WINDOWS_ZH%) (Chinese) ([signature file](%WINDOWS_ZH_SIG%)). + +## OS X +[Download Tor Browser](%OSX_EN%) (English) ([signature file](%OSX_EN_SIG%)). + +[Download Tor Browser](%OSX_FA%) (Farsi) ([signature file](%OSX_FA_SIG%)). + +[Download Tor Browser](%OSX_TR%) (Turkish) ([signature file](%OSX_TR_SIG%)). + +[Download Tor Browser](%OSX_ZH%) (Chinese) ([signature file](%OSX_ZH_SIG%)). + +## Linux 32-bit +[Download Tor Browser](%LINUX32_EN%) (English) ([signature file](%LINUX32_EN_SIG%)). + +[Download Tor Browser](%LINUX32_FA%) (Farsi) ([signature file](%LINUX32_FA_SIG%)). + +[Download Tor Browser](%LINUX32_TR%) (Turkish) ([signature file](%LINUX32_TR_SIG%)). + +[Download Tor Browser](%LINUX32_ZH%) (Chinese) ([signature file](%LINUX32_ZH_SIG%)). + +## Linux 64-bit +[Download Tor Browser](%LINUX64_EN%) (English) ([signature file](%LINUX64_EN_SIG%)). + +[Download Tor Browser](%LINUX64_FA%) (Farsi) ([signature file](%LINUX64_FA_SIG%)). + +[Download Tor Browser](%LINUX64_TR%) (Turkish) ([signature file](%LINUX64_TR_SIG%)). + +[Download Tor Browser](%LINUX64_ZH%) (Chinese) ([signature file](%LINUX64_ZH_SIG%)).

1 0

[gettor/master] Latest version of github3 replaces iter_releases() by releases()
by ilv＠torproject.org 16 Feb '16

16 Feb '16

commit bd107baa96f3a6f2663a9fde999e1f6b690d6726 Author: ilv <ilv(a)users.noreply.github.com> Date: Tue Feb 16 01:19:52 2016 -0300 Latest version of github3 replaces iter_releases() by releases() --- upload/bundles2github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upload/bundles2github.py b/upload/bundles2github.py index e670f85..2e3c888 100644 --- a/upload/bundles2github.py +++ b/upload/bundles2github.py @@ -108,7 +108,7 @@ if __name__ == '__main__': readable_fp = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4)) # Find any published releases with this version number - for release in target_repo.iter_releases(): + for release in target_repo.releases(): if release.tag_name == 'v{}'.format(version) and not release.draft: print("Found an existing published release with this version. " "Not uploading again unless you delete the published "

1 0

[gettor/master] Script for generating landing page in gettor gh-pages and readme file in gettorbrowser
by ilv＠torproject.org 16 Feb '16

16 Feb '16

commit c6740894c25f5b367a01b67d940047fe34decd67 Author: ilv <ilv(a)users.noreply.github.com> Date: Tue Feb 16 01:25:46 2016 -0300 Script for generating landing page in gettor gh-pages and readme file in gettorbrowser --- create_gh_mirrors.py | 181 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) diff --git a/create_gh_mirrors.py b/create_gh_mirrors.py new file mode 100644 index 0000000..a341940 --- /dev/null +++ b/create_gh_mirrors.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- +# +# This file is part of GetTor +# +# :authors: Israel Leiva <ilv(a)torproject.org> +# see also AUTHORS file +# +# :license: This is Free Software. See LICENSE for license information. + +"""create_gh_mirrors -- Create landing page and readme for Github.""" + +import os +import ConfigParser + +import github3 + +import gettor.core + + +def create_readme(tpl_path, md_path, version, links): + """Create README file with links stored in github.links. + + :param: tpl_path (string) path to file used as template. + :param: md_path (string) path to file generated. + :param: version (string) tor browser version. + :param: links (object) github links. + + """ + lcs = ['FA', 'ZH', 'TR', 'EN'] + + md_file = open(md_path, 'w') + with open(tpl_path, 'r') as tpl_file: + content_md = tpl_file.read() + + for lc in lcs: + win_link = links.get('windows', lc.lower()) + win_pkg, win_sig, win_sha = [e for e in win_link.split("$") if e] + + osx_link = links.get('osx', lc.lower()) + osx_pkg, osx_sig, osx_sha = [e for e in osx_link.split("$") if e] + + linux_links = links.get('linux', lc.lower()) + linux32_link, linux64_link = linux_links.split(',') + linux32_pkg, linux32_sig, linux32_sha = [ + e for e in linux32_link.split("$") if e + ] + linux64_pkg, linux64_sig, linux64_sha = [ + e for e in linux64_link.split("$") if e + ] + + content_md = content_md.replace( + "%WINDOWS_{}%".format(lc), win_pkg + ) + content_md = content_md.replace( + "%WINDOWS_{}_SIG%".format(lc), win_sig + ) + + content_md = content_md.replace( + "%OSX_{}%".format(lc), osx_pkg + ) + content_md = content_md.replace( + "%OSX_{}_SIG%".format(lc), osx_sig + ) + + content_md = content_md.replace( + "%LINUX32_{}%".format(lc), linux32_pkg + ) + content_md = content_md.replace( + "%LINUX32_{}_SIG%".format(lc), linux32_sig + ) + content_md = content_md.replace( + "%LINUX64_{}%".format(lc), linux64_pkg + ) + content_md = content_md.replace( + "%LINUX64_{}_SIG%".format(lc), linux64_sig + ) + + content_md = content_md.replace("%VERSION%", version) + md_file.write(content_md) + + print "README generated with version %s" % version + + +def create_landing_html(tpl_path, html_path, version, links): + """Create README file with links stored in github.links. + + :param: tpl_path (string) path to file used as template. + :param: html_path (string) path to file generated. + :param: version (string) tor browser version. + :param: links (object) github links. + + """ + win_link = links.get('windows', 'en') + win_pkg, win_sig, win_sha = [e for e in win_link.split("$") if e] + + osx_link = links.get('osx', 'en') + osx_pkg, osx_sig, osx_sha = [e for e in osx_link.split("$") if e] + + linux_links = links.get('linux', 'en') + linux32_link, linux64_link = linux_links.split(',') + linux32_pkg, linux32_sig, linux32_sha = [ + e for e in linux32_link.split("$") if e + ] + linux64_pkg, linux64_sig, linux64_sha = [ + e for e in linux64_link.split("$") if e + ] + + html_file = open(html_path, 'w') + with open(tpl_path, 'r') as tpl_file: + content_tpl = tpl_file.read().replace('\n', '') + content_html = '' + + content_html = content_tpl.replace("%WINDOWS%", win_pkg) + content_html = content_html.replace("%WINDOWS_SIG%", win_sig) + + content_html = content_html.replace("%OSX%", osx_pkg) + content_html = content_html.replace("%OSX_SIG%", osx_sig) + + content_html = content_html.replace("%LINUX32%", linux32_pkg) + content_html = content_html.replace("%LINUX32_SIG%", linux32_sig) + content_html = content_html.replace("%LINUX64%", linux64_pkg) + content_html = content_html.replace("%LINUX64_SIG%", linux64_sig) + + content_html = content_html.replace("%VERSION%", version) + html_file.write(content_html) + + print "HTML generated with version %s" % version + + +def main(): + """Generate HTML and md files and update it in Github.""" + github_links = 'providers/github.links' + tbb_version_path = 'latest_torbrowser.cfg' + md_path = 'upload/readme_gh.md' + html_path = 'upload/landing_gh.html' + md_tpl_path = 'upload/readme_gh.tpl' + html_tpl_path = 'upload/landing_gh.tpl' + github_access_token = '' + + tbb_version_config = ConfigParser.ConfigParser() + tbb_version_config.read(tbb_version_path) + version = tbb_version_config.get('version', 'current') + + links = ConfigParser.ConfigParser() + links.read(github_links) + + create_landing_html(html_tpl_path, html_path, version, links) + create_readme(md_tpl_path, md_path, version, links) + + landing = open(html_path, 'r') + content_landing = landing.read().replace('\n', '') + + readme = open(md_path, 'r') + content_readme = readme.read() + + gh = github3.login(token=github_access_token) + repo_landing = gh.repository('thetorproject', 'gettor') + repo_readme = gh.repository('thetorproject', 'gettorbrowser') + + file_landing_gh = repo_landing.file_contents('index.html') + file_readme_gh = repo_readme.file_contents('README.md') + + data_landing = { + 'message': 'Updating landing page.', + 'content': content_landing, + 'branch': 'gh-pages' + } + + data_readme = { + 'message': 'Updating README.', + 'content': content_readme + } + + file_landing_gh.update(**data_landing) + print "Landing page updated in gettor" + + file_readme_gh.update(**data_readme) + print "README updated in gettorbrowser" + +if __name__ == "__main__": + main()

1 0

[gettor/master] Changes for production in github uploader
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit c66101e83598eda5b2d12841b7520a6f64264810 Author: ilv <ilv(a)users.noreply.github.com> Date: Mon Feb 15 21:00:36 2016 -0300 Changes for production in github uploader --- gettor/utils.py | 12 ++++++------ upload/bundles2github.py | 31 +++++++++++++------------------ 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/gettor/utils.py b/gettor/utils.py index c4f43c7..2780564 100644 --- a/gettor/utils.py +++ b/gettor/utils.py @@ -20,9 +20,9 @@ import hashlib LOGGING_FORMAT = "[%(levelname)s] %(asctime)s; %(message)s" DATE_FORMAT = "%Y-%m-%d" # %H:%M:%S -windows_regex = '^torbrowser-install-\d\.\d\.\d_\w\w(-\w\w)?\.exe$' -linux_regex = '^tor-browser-linux\d\d-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz$' -osx_regex = '^TorBrowser-\d\.\d\.\d-osx\d\d_(\w\w)(-\w\w)?\.dmg$' +windows_regex = '^torbrowser-install-\d\.\d(\.\d)?_(\w\w)(-\w\w)?\.exe$' +linux_regex = '^tor-browser-linux(\d\d)-\d\.\d(\.\d)?_(\w\w)(-\w\w)?\.tar\.xz$' +osx_regex = '^TorBrowser-\d\.\d(\.\d)?-osx\d\d_(\w\w)(-\w\w)?\.dmg$' def get_logging_format(): @@ -70,11 +70,11 @@ def get_bundle_info(filename, osys=None): m_osx = re.search(osx_regex, filename) if m_windows: - return 'windows', '32/64', m_windows.group(1) + return 'windows', '32/64', m_windows.group(2) elif m_linux: - return 'linux', m_linux.group(1), m_linux.group(2) + return 'linux', m_linux.group(1), m_linux.group(3) elif m_osx: - return 'osx', m_osx.group(1), m_osx.group(2) + return 'osx', '64', m_osx.group(2) else: raise ValueError("Invalid bundle format %s" % file) diff --git a/upload/bundles2github.py b/upload/bundles2github.py index d081ddd..e670f85 100644 --- a/upload/bundles2github.py +++ b/upload/bundles2github.py @@ -67,33 +67,28 @@ if __name__ == '__main__': help='Create links file with files already uploaded.' ) - parser.add_argument( - '-v', '--version', default=None, - help='Version of Tor Browser.' - ) - args = parser.parse_args() config = ConfigParser.ConfigParser() - config.read('github-local.cfg') + config.read('github.cfg') - # this script should be called after fetching the latest Tor Browser, - # and specifying the latest version - if args.version: - version = args.version - else: - tbb_version_config = ConfigParser.ConfigParser() - tbb_version_config.read('latest_torbrowser.cfg') - version = tbb_version_config.get('version', 'current') + tbb_version_path = config.get('general', 'version_cfg_path') + + tbb_version_config = ConfigParser.ConfigParser() + tbb_version_config.read(tbb_version_path) + version = tbb_version_config.get('version', 'current') # the token allow us to run this script without GitHub user/pass github_access_token = config.get('app', 'access_token') # path to the fingerprint that signed the packages - tb_key = os.path.abspath('tbb-key-torbrowserteam.asc') + tb_key = config.get('general', 'tbb_key_path') # path to the latest version of Tor Browser - tb_path = os.path.abspath('latest') + tb_path = config.get('general', 'latest_path') + + # path to gettor code configuration + core_path = config.get('general', 'core_path') # user and repository where we upload Tor Browser github_user = config.get('app', 'user') @@ -139,13 +134,13 @@ if __name__ == '__main__': release.edit(draft=False) # Create the links file for this release - core = gettor.core.Core(os.path.abspath('../core.cfg')) + core = gettor.core.Core(core_path) # Erase old links if any and create a new empty one core.create_links_file('GitHub', readable_fp) print("Creating links file") - for asset in release.assets: + for asset in release.assets(): url = asset.browser_download_url if url.endswith('.asc'): continue

1 0

[snowflake/master] Begin the Broker sub-README page
by arlo＠torproject.org 15 Feb '16

15 Feb '16

commit 4acff9983cca4d3a7778e3a49f073b0e342e5cf4 Author: Serene Han <keroserene+git(a)gmail.com> Date: Thu Feb 11 21:39:20 2016 -0800 Begin the Broker sub-README page --- broker/README.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/broker/README.md b/broker/README.md index e69de29..ab6af99 100644 --- a/broker/README.md +++ b/broker/README.md @@ -0,0 +1,39 @@ +This is the Broker component of Snowflake. + +### Overview + +The Broker handles the rendezvous by matching Snowflake +Clients with Proxies, and passing their WebRTC Session Descriptions +(the "signaling" step). This allows Clients and Proxies to establish +a Peer connection. + +It is analogous to Flashproxy's +[Facilitator](https://trac.torproject.org/projects/tor/wiki/FlashProxyFAQ), +but bidirectional and domain-fronted. + +The Broker expects: + +- Clients to send their SDP offer in a POST request, which will then block + until the Broker responds with the answer of the matched Proxy. +- Proxies to announce themselves with a POST request, to which the Broker + responds with some Client's SDP offer. The Proxy should then send a second + POST request soon after containing its SDP answer, which the Broker passes + back to the same Client. + +### Running your own + +You can run your own Broker on either localhost or appengine. +(Other CDNs will be supported soon.) + + +To run on localhost, run `dev_appserver.py` or equivalent from this +directory. (on arch, I use the wrapper script `dev_appserver-go`) + +To run on appengine, you can spin up your own instance with an arbitrary +name, and use `appcfg.py`. + +In both cases, you'll need to provide the URL of the custom broker +to the client plugin using the `--url $URL` flag. + +See more detailed appengine instructions +[here](https://cloud.google.com/appengine/docs/go/).

1 0

[snowflake/master] Add context to HTTP handlers, attempt to support localhost Broker.
by arlo＠torproject.org 15 Feb '16

15 Feb '16

commit b04d1f67fb9ec8888da295a0fcf908fd8c08b84b Author: Serene Han <keroserene+git(a)gmail.com> Date: Fri Feb 12 15:38:28 2016 -0800 Add context to HTTP handlers, attempt to support localhost Broker. Seems unlikely to work due to dev_appserver single inflight request limitation --- broker/broker.go | 127 +++++++++++++++++++++++++++++----------- broker/snowflake-broker_test.go | 8 +++ client/torrc-localhost | 7 +++ proxy/broker.coffee | 3 +- 4 files changed, 110 insertions(+), 35 deletions(-) diff --git a/broker/broker.go b/broker/broker.go index ae039a5..a83e78a 100644 --- a/broker/broker.go +++ b/broker/broker.go @@ -11,6 +11,7 @@ package snowflake_broker import ( "container/heap" + "fmt" "io/ioutil" "log" "net" @@ -23,27 +24,68 @@ const ( ProxyTimeout = 10 ) -// This is minimum viable client-proxy registration. -// TODO(#13): better, more secure registration corresponding to what's in -// the python flashproxy facilitator. -var snowflakes *SnowflakeHeap +type SnowflakeContext struct { + snowflakes *SnowflakeHeap + // Map keeping track of snowflakeIDs required to match SDP answers from + // the second http POST. + snowflakeMap map[string]*Snowflake +} + +type SnowflakeHandler struct { + *SnowflakeContext + h func(*SnowflakeContext, http.ResponseWriter, *http.Request) +} + +func (sh SnowflakeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + sh.h(sh.SnowflakeContext, w, r) +} -// Map keeping track of snowflakeIDs required to match SDP answers from -// the second http POST. -var snowflakeMap map[string]*Snowflake +type ProxyRequest struct { + id string + offerChan chan []byte +} + +var createChan = make(chan *ProxyRequest) // Create and add a Snowflake to the heap. -func AddSnowflake(id string) *Snowflake { +func (sc *SnowflakeContext) AddSnowflake(id string) *Snowflake { + log.Println(sc.snowflakes) + snowflake := new(Snowflake) snowflake.id = id snowflake.clients = 0 snowflake.offerChannel = make(chan []byte) snowflake.answerChannel = make(chan []byte) - heap.Push(snowflakes, snowflake) - snowflakeMap[id] = snowflake + heap.Push(sc.snowflakes, snowflake) + sc.snowflakeMap[id] = snowflake + + log.Println("Total snowflakes available: ", sc.snowflakes.Len()) + log.Println(sc.snowflakes) + log.Println(sc.snowflakeMap) return snowflake } +func (sc *SnowflakeContext) Broker(proxies <-chan *ProxyRequest) { + for p := range proxies { + log.Println("adding ", p.id) + snowflake := sc.AddSnowflake(p.id) + // Wait for a client to avail an offer to the snowflake, or timeout + // and ask the snowflake to poll later. + go func(p *ProxyRequest) { + select { + case offer := <-snowflake.offerChannel: + log.Println("Passing client offer to snowflake.") + p.offerChan <- offer + case <-time.After(time.Second * ProxyTimeout): + // This snowflake is no longer available to serve clients. + heap.Remove(sc.snowflakes, snowflake.index) + delete(sc.snowflakeMap, snowflake.id) + p.offerChan <- nil + } + }(p) + } +} + func robotsTxtHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Write([]byte("User-agent: *\nDisallow:\n")) @@ -73,7 +115,7 @@ Expects a WebRTC SDP offer in the Request to give to an assigned snowflake proxy, which responds with the SDP answer to be sent in the HTTP response back to the client. */ -func clientHandler(w http.ResponseWriter, r *http.Request) { +func clientHandler(ctx *SnowflakeContext, w http.ResponseWriter, r *http.Request) { offer, err := ioutil.ReadAll(r.Body) if nil != err { log.Println("Invalid data.") @@ -85,23 +127,24 @@ func clientHandler(w http.ResponseWriter, r *http.Request) { // Find the most available snowflake proxy, and pass the offer to it. // TODO: Needs improvement - maybe shouldn' - snowflake := heap.Pop(snowflakes).(*Snowflake) - if nil == snowflake { + if ctx.snowflakes.Len() <= 0 { + log.Println("Client: No snowflake proxies available.") w.WriteHeader(http.StatusServiceUnavailable) - // w.Write([]byte("no snowflake proxies available")) return } + snowflake := heap.Pop(ctx.snowflakes).(*Snowflake) snowflake.offerChannel <- offer // Wait for the answer to be returned on the channel. select { case answer := <-snowflake.answerChannel: - log.Println("Retrieving answer") + log.Println("Client: Retrieving answer") w.Write(answer) // Only remove from the snowflake map once the answer is set. - delete(snowflakeMap, snowflake.id) + delete(ctx.snowflakeMap, snowflake.id) case <-time.After(time.Second * ClientTimeout): + log.Println("Client: Timed out.") w.WriteHeader(http.StatusGatewayTimeout) w.Write([]byte("timed out waiting for answer!")) } @@ -110,7 +153,7 @@ func clientHandler(w http.ResponseWriter, r *http.Request) { /* For snowflake proxies to request a client from the Broker. */ -func proxyHandler(w http.ResponseWriter, r *http.Request) { +func proxyHandler(ctx *SnowflakeContext, w http.ResponseWriter, r *http.Request) { if isPreflight(w, r) { return } @@ -126,21 +169,22 @@ func proxyHandler(w http.ResponseWriter, r *http.Request) { } // Maybe confirm that X-Session-ID is the same. log.Println("Received snowflake: ", id) - snowflake := AddSnowflake(id) + + p := new(ProxyRequest) + p.id = id + p.offerChan = make(chan []byte) + createChan <- p // Wait for a client to avail an offer to the snowflake, or timeout // and ask the snowflake to poll later. - select { - case offer := <-snowflake.offerChannel: - log.Println("Passing client offer to snowflake.") - w.Write(offer) - - case <-time.After(time.Second * ProxyTimeout): - // This snowflake is no longer available to serve clients. - heap.Remove(snowflakes, snowflake.index) - delete(snowflakeMap, snowflake.id) + offer := <-p.offerChan + if nil == offer { + log.Println("Proxy " + id + " did not receive a Client offer.") w.WriteHeader(http.StatusGatewayTimeout) + return } + log.Println("Passing client offer to snowflake.") + w.Write(offer) } /* @@ -148,12 +192,12 @@ Expects snowflake proxes which have previously successfully received an offer from proxyHandler to respond with an answer in an HTTP POST, which the broker will pass back to the original client. */ -func answerHandler(w http.ResponseWriter, r *http.Request) { +func answerHandler(ctx *SnowflakeContext, w http.ResponseWriter, r *http.Request) { if isPreflight(w, r) { return } id := r.Header.Get("X-Session-ID") - snowflake, ok := snowflakeMap[id] + snowflake, ok := ctx.snowflakeMap[id] if !ok || nil == snowflake { // The snowflake took too long to respond with an answer, // and the designated client is no longer around / recognized by the Broker. @@ -170,15 +214,30 @@ func answerHandler(w http.ResponseWriter, r *http.Request) { snowflake.answerChannel <- body } +func debugHandler(ctx *SnowflakeContext, w http.ResponseWriter, r *http.Request) { + s := fmt.Sprintf("current: %d", ctx.snowflakes.Len()) + w.Write([]byte(s)) +} + func init() { - snowflakes = new(SnowflakeHeap) - snowflakeMap = make(map[string]*Snowflake) + // snowflakeMap = make(map[string]*Snowflake) + snowflakes := new(SnowflakeHeap) heap.Init(snowflakes) + ctx := &SnowflakeContext{ + snowflakes: snowflakes, + snowflakeMap: make(map[string]*Snowflake), + } + + go ctx.Broker(createChan) http.HandleFunc("/robots.txt", robotsTxtHandler) http.HandleFunc("/ip", ipHandler) - http.HandleFunc("/client", clientHandler) - http.HandleFunc("/proxy", proxyHandler) - http.HandleFunc("/answer", answerHandler) + // http.HandleFunc("/client", clientHandler) + // http.HandleFunc("/proxy", proxyHandler) + // http.HandleFunc("/answer", answerHandler) + http.Handle("/client", SnowflakeHandler{ctx, clientHandler}) + http.Handle("/proxy", SnowflakeHandler{ctx, proxyHandler}) + http.Handle("/answer", SnowflakeHandler{ctx, answerHandler}) + http.Handle("/debug", SnowflakeHandler{ctx, debugHandler}) } diff --git a/broker/snowflake-broker_test.go b/broker/snowflake-broker_test.go index 03f876b..47533d0 100644 --- a/broker/snowflake-broker_test.go +++ b/broker/snowflake-broker_test.go @@ -22,6 +22,8 @@ func TestSnowflakeHeap(t *testing.T) { s4.clients = 1 heap.Push(h, s1) + if 1 != h.Len() { + } heap.Push(h, s2) heap.Push(h, s3) heap.Push(h, s4) @@ -36,6 +38,9 @@ func TestSnowflakeHeap(t *testing.T) { } r := heap.Pop(h).(*Snowflake) + if 2 != h.Len() { + t.Error("Unexpected length.") + } if r.clients != 3 { t.Error("Unexpected clients: ", r.clients) } @@ -44,6 +49,9 @@ func TestSnowflakeHeap(t *testing.T) { } r = heap.Pop(h).(*Snowflake) + if 1 != h.Len() { + t.Error("Unexpected length.") + } if r.clients != 4 { t.Error("Unexpected clients: ", r.clients) } diff --git a/client/torrc-localhost b/client/torrc-localhost new file mode 100644 index 0000000..7d539fb --- /dev/null +++ b/client/torrc-localhost @@ -0,0 +1,7 @@ +UseBridges 1 +DataDirectory datadir + +ClientTransportPlugin snowflake exec ./client \ +-url http://localhost:8080/ \ + +Bridge snowflake 0.0.3.0:1 diff --git a/proxy/broker.coffee b/proxy/broker.coffee index e989d57..71c3186 100644 --- a/proxy/broker.coffee +++ b/proxy/broker.coffee @@ -28,7 +28,8 @@ class Broker @clients = 0 @id = genSnowflakeID() # Ensure url has the right protocol + trailing slash. - @url = 'https://' + @url if 0 != @url.indexOf('https://', 0) + @url = 'http://' + @url if 0 == @url.indexOf('localhost', 0) + @url = 'https://' + @url if 0 != @url.indexOf('http', 0) @url += '/' if '/' != @url.substr -1 # Promises some client SDP Offer.

1 0

[snowflake/master] Convert Broker SnowflakeHeap test to goconvey, and async test for client handler
by arlo＠torproject.org 15 Feb '16

15 Feb '16

commit 0e1c5a17568a8017e7e650a0a8d38829d0f202f3 Author: Serene Han <keroserene+git(a)gmail.com> Date: Sat Feb 13 11:47:57 2016 -0800 Convert Broker SnowflakeHeap test to goconvey, and async test for client handler --- broker/broker.go | 48 +++++------- broker/snowflake-broker_test.go | 164 ++++++++++++++++++++++++++-------------- 2 files changed, 127 insertions(+), 85 deletions(-) diff --git a/broker/broker.go b/broker/broker.go index a83e78a..c81392b 100644 --- a/broker/broker.go +++ b/broker/broker.go @@ -24,20 +24,29 @@ const ( ProxyTimeout = 10 ) -type SnowflakeContext struct { +type BrokerContext struct { snowflakes *SnowflakeHeap // Map keeping track of snowflakeIDs required to match SDP answers from // the second http POST. snowflakeMap map[string]*Snowflake } +func NewBrokerContext() *BrokerContext { + snowflakes := new(SnowflakeHeap) + heap.Init(snowflakes) + return &BrokerContext{ + snowflakes: snowflakes, + snowflakeMap: make(map[string]*Snowflake), + } +} + type SnowflakeHandler struct { - *SnowflakeContext - h func(*SnowflakeContext, http.ResponseWriter, *http.Request) + *BrokerContext + h func(*BrokerContext, http.ResponseWriter, *http.Request) } func (sh SnowflakeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - sh.h(sh.SnowflakeContext, w, r) + sh.h(sh.BrokerContext, w, r) } type ProxyRequest struct { @@ -48,9 +57,7 @@ type ProxyRequest struct { var createChan = make(chan *ProxyRequest) // Create and add a Snowflake to the heap. -func (sc *SnowflakeContext) AddSnowflake(id string) *Snowflake { - log.Println(sc.snowflakes) - +func (sc *BrokerContext) AddSnowflake(id string) *Snowflake { snowflake := new(Snowflake) snowflake.id = id snowflake.clients = 0 @@ -58,16 +65,12 @@ func (sc *SnowflakeContext) AddSnowflake(id string) *Snowflake { snowflake.answerChannel = make(chan []byte) heap.Push(sc.snowflakes, snowflake) sc.snowflakeMap[id] = snowflake - - log.Println("Total snowflakes available: ", sc.snowflakes.Len()) - log.Println(sc.snowflakes) - log.Println(sc.snowflakeMap) return snowflake } -func (sc *SnowflakeContext) Broker(proxies <-chan *ProxyRequest) { +// Match proxies to clients. +func (sc *BrokerContext) Broker(proxies <-chan *ProxyRequest) { for p := range proxies { - log.Println("adding ", p.id) snowflake := sc.AddSnowflake(p.id) // Wait for a client to avail an offer to the snowflake, or timeout // and ask the snowflake to poll later. @@ -115,7 +118,7 @@ Expects a WebRTC SDP offer in the Request to give to an assigned snowflake proxy, which responds with the SDP answer to be sent in the HTTP response back to the client. */ -func clientHandler(ctx *SnowflakeContext, w http.ResponseWriter, r *http.Request) { +func clientHandler(ctx *BrokerContext, w http.ResponseWriter, r *http.Request) { offer, err := ioutil.ReadAll(r.Body) if nil != err { log.Println("Invalid data.") @@ -153,7 +156,7 @@ func clientHandler(ctx *SnowflakeContext, w http.ResponseWriter, r *http.Request /* For snowflake proxies to request a client from the Broker. */ -func proxyHandler(ctx *SnowflakeContext, w http.ResponseWriter, r *http.Request) { +func proxyHandler(ctx *BrokerContext, w http.ResponseWriter, r *http.Request) { if isPreflight(w, r) { return } @@ -192,7 +195,7 @@ Expects snowflake proxes which have previously successfully received an offer from proxyHandler to respond with an answer in an HTTP POST, which the broker will pass back to the original client. */ -func answerHandler(ctx *SnowflakeContext, w http.ResponseWriter, r *http.Request) { +func answerHandler(ctx *BrokerContext, w http.ResponseWriter, r *http.Request) { if isPreflight(w, r) { return } @@ -214,28 +217,19 @@ func answerHandler(ctx *SnowflakeContext, w http.ResponseWriter, r *http.Request snowflake.answerChannel <- body } -func debugHandler(ctx *SnowflakeContext, w http.ResponseWriter, r *http.Request) { +func debugHandler(ctx *BrokerContext, w http.ResponseWriter, r *http.Request) { s := fmt.Sprintf("current: %d", ctx.snowflakes.Len()) w.Write([]byte(s)) } func init() { - // snowflakeMap = make(map[string]*Snowflake) - snowflakes := new(SnowflakeHeap) - heap.Init(snowflakes) - ctx := &SnowflakeContext{ - snowflakes: snowflakes, - snowflakeMap: make(map[string]*Snowflake), - } + ctx := NewBrokerContext() go ctx.Broker(createChan) http.HandleFunc("/robots.txt", robotsTxtHandler) http.HandleFunc("/ip", ipHandler) - // http.HandleFunc("/client", clientHandler) - // http.HandleFunc("/proxy", proxyHandler) - // http.HandleFunc("/answer", answerHandler) http.Handle("/client", SnowflakeHandler{ctx, clientHandler}) http.Handle("/proxy", SnowflakeHandler{ctx, proxyHandler}) http.Handle("/answer", SnowflakeHandler{ctx, answerHandler}) diff --git a/broker/snowflake-broker_test.go b/broker/snowflake-broker_test.go index 47533d0..7d8f169 100644 --- a/broker/snowflake-broker_test.go +++ b/broker/snowflake-broker_test.go @@ -1,67 +1,115 @@ package snowflake_broker import ( + "bytes" "container/heap" + . "github.com/smartystreets/goconvey/convey" + "net/http" + "net/http/httptest" "testing" ) +func TestBroker(t *testing.T) { + + Convey("Context", t, func() { + ctx := NewBrokerContext() + + Convey("Adds Snowflake", func() { + ctx := NewBrokerContext() + So(ctx.snowflakes.Len(), ShouldEqual, 0) + So(len(ctx.snowflakeMap), ShouldEqual, 0) + ctx.AddSnowflake("foo") + So(ctx.snowflakes.Len(), ShouldEqual, 1) + So(len(ctx.snowflakeMap), ShouldEqual, 1) + }) + + Convey("Responds to client offers...", func() { + + w := httptest.NewRecorder() + data := bytes.NewReader([]byte("test")) + r, err := http.NewRequest("POST", "broker.com/client", data) + So(err, ShouldBeNil) + + Convey("with 503 when no snowflakes are available.", func() { + clientHandler(ctx, w, r) + h := w.Header() + So(h["Access-Control-Allow-Headers"], ShouldNotBeNil) + So(w.Code, ShouldEqual, http.StatusServiceUnavailable) + So(w.Body.String(), ShouldEqual, "") + }) + + Convey("with a proxy answer if available.", func() { + done := make(chan bool) + // Prepare a fake proxy to respond with. + snowflake := ctx.AddSnowflake("fake") + go func() { + clientHandler(ctx, w, r) + done <- true + }() + offer := <-snowflake.offerChannel + So(offer, ShouldResemble, []byte("test")) + snowflake.answerChannel <- []byte("fake answer") + <-done + So(w.Body.String(), ShouldEqual, "fake answer") + So(w.Code, ShouldEqual, http.StatusOK) + }) + + Convey("Times out when no proxy responds.", func() { + done := make(chan bool) + snowflake := ctx.AddSnowflake("fake") + go func() { + clientHandler(ctx, w, r) + done <- true + }() + offer := <-snowflake.offerChannel + So(offer, ShouldResemble, []byte("test")) + <-done + So(w.Code, ShouldEqual, http.StatusGatewayTimeout) + }) + + }) + }) +} + func TestSnowflakeHeap(t *testing.T) { - h := new(SnowflakeHeap) - heap.Init(h) - if 0 != h.Len() { - t.Error("Unexpected length.") - } - s1 := new(Snowflake) - s2 := new(Snowflake) - s3 := new(Snowflake) - s4 := new(Snowflake) - - s1.clients = 4 - s2.clients = 5 - s3.clients = 3 - s4.clients = 1 - - heap.Push(h, s1) - if 1 != h.Len() { - } - heap.Push(h, s2) - heap.Push(h, s3) - heap.Push(h, s4) - - if 4 != h.Len() { - t.Error("Unexpected length.") - } - - heap.Remove(h, 0) - if 3 != h.Len() { - t.Error("Unexpected length.") - } - - r := heap.Pop(h).(*Snowflake) - if 2 != h.Len() { - t.Error("Unexpected length.") - } - if r.clients != 3 { - t.Error("Unexpected clients: ", r.clients) - } - if r.index != -1 { - t.Error("Unexpected index: ", r.index) - } - - r = heap.Pop(h).(*Snowflake) - if 1 != h.Len() { - t.Error("Unexpected length.") - } - if r.clients != 4 { - t.Error("Unexpected clients: ", r.clients) - } - - r = heap.Pop(h).(*Snowflake) - if r.clients != 5 { - t.Error("Unexpected clients: ", r.clients) - } - - if 0 != h.Len() { - t.Error("Unexpected length.") - } + Convey("SnowflakeHeap", t, func() { + h := new(SnowflakeHeap) + heap.Init(h) + So(h.Len(), ShouldEqual, 0) + s1 := new(Snowflake) + s2 := new(Snowflake) + s3 := new(Snowflake) + s4 := new(Snowflake) + s1.clients = 4 + s2.clients = 5 + s3.clients = 3 + s4.clients = 1 + + heap.Push(h, s1) + So(h.Len(), ShouldEqual, 1) + heap.Push(h, s2) + So(h.Len(), ShouldEqual, 2) + heap.Push(h, s3) + So(h.Len(), ShouldEqual, 3) + heap.Push(h, s4) + So(h.Len(), ShouldEqual, 4) + + heap.Remove(h, 0) + So(h.Len(), ShouldEqual, 3) + + r := heap.Pop(h).(*Snowflake) + So(h.Len(), ShouldEqual, 2) + So(r.clients, ShouldEqual, 3) + So(r.index, ShouldEqual, -1) + + r = heap.Pop(h).(*Snowflake) + So(h.Len(), ShouldEqual, 1) + So(r.clients, ShouldEqual, 4) + So(r.index, ShouldEqual, -1) + + r = heap.Pop(h).(*Snowflake) + So(h.Len(), ShouldEqual, 0) + So(r.clients, ShouldEqual, 5) + So(r.index, ShouldEqual, -1) + }) }

1 0

[snowflake/master] Update travis to have goconvey
by arlo＠torproject.org 15 Feb '16

15 Feb '16

commit e13b35faf312cc5c65e089c7d54257d9524d3600 Author: Serene Han <keroserene+git(a)gmail.com> Date: Sat Feb 13 11:56:27 2016 -0800 Update travis to have goconvey --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 7f4e265..6b49a19 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ env: install: - rm -rf ~/.nvm && git clone https://github.com/creationix/nvm.git ~/.nvm && (cd ~/.nvm && git checkout `git describe --abbrev=0 --tags`) && source ~/.nvm/nvm.sh && nvm install $TRAVIS_NODE_VERSION + - go get -u github.com/smartystreets/goconvey before_script: npm install -g coffee-script coffeelint jasmine

1 0