October 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Implement collector for classifying website with domclass.
by art＠torproject.org 07 Oct '12

07 Oct '12

commit 3d1afe284b3437ee460841828e7a7175f93564e3 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Oct 7 17:05:33 2012 +0000 Implement collector for classifying website with domclass. Next step run this on all the sites we want to test. --- nettests/experimental/domclass_collector.py | 33 +++++++++++++++++++++++++++ ooni/kit/__init__.py | 1 + ooni/kit/domclass.py | 30 ++++++++++++++++-------- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/nettests/experimental/domclass_collector.py b/nettests/experimental/domclass_collector.py new file mode 100644 index 0000000..9b2c8d8 --- /dev/null +++ b/nettests/experimental/domclass_collector.py @@ -0,0 +1,33 @@ +# -*- encoding: utf-8 -*- +# +# The purpose of this collector is to compute the eigenvector for the input +# file containing a list of sites. +# +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from twisted.internet import threads, defer + +from ooni.kit import domclass +from ooni.templates import httpt + +class DOMClassCollector(httpt.HTTPTest): + name = "DOM class collector" + author = "Arturo Filastò" + version = 0.1 + + inputs = ['http://news.google.com/', 'http://wikileaks.org/'] + #inputFile = ['f', 'file', None, 'The list of urls to build a domclass for'] + + def test_collect(self): + if self.input: + url = self.input + return self.doRequest(url) + else: + raise Exception("No input specified") + + def processResponseBody(self, body): + eigenvalues = domclass.compute_eigenvalues_from_DOM(content=body) + self.report['eigenvalues'] = eigenvalues + diff --git a/ooni/kit/__init__.py b/ooni/kit/__init__.py new file mode 100644 index 0000000..55374c9 --- /dev/null +++ b/ooni/kit/__init__.py @@ -0,0 +1 @@ +__all__ = ['domclass'] diff --git a/ooni/kit/domclass.py b/ooni/kit/domclass.py index 02c26dc..1cf33a0 100644 --- a/ooni/kit/domclass.py +++ b/ooni/kit/domclass.py @@ -108,7 +108,7 @@ def compute_eigenvalues(matrix): """ return numpy.linalg.eigvals(matrix) -def readDOM(content=None, filename=None): +def readDOM(content=None, filename=None, debug=False): """ Parses the DOM of the HTML page and returns an array of parent, child pairs. @@ -124,20 +124,30 @@ def readDOM(content=None, filename=None): content = ''.join(f.readlines()) f.close() - start = time.time() - print "Running BeautifulSoup on content" + if debug: + start = time.time() + print "Running BeautifulSoup on content" dom = BeautifulSoup(content) - print "done in %s" % (time.time() - start) + if debug: + print "done in %s" % (time.time() - start) - start = time.time() - print "Creating couples matrix" + if debug: + start = time.time() + print "Creating couples matrix" couples = [] for x in dom.findAll(): couples.append((str(x.parent.name), str(x.name))) - print "done in %s" % (time.time() - start) + if debug: + print "done in %s" % (time.time() - start) return couples +def compute_eigenvalues_from_DOM(*arg,**kw): + dom = readDOM(*arg, **kw) + probability_matrix = compute_probability_matrix(dom) + eigenvalues = compute_eigenvalues(probability_matrix) + return eigenvalues + def compute_correlation(matrix_a, matrix_b): correlation = numpy.vdot(matrix_a, matrix_b) correlation /= numpy.linalg.norm(matrix_a)*numpy.linalg.norm(matrix_b) @@ -192,13 +202,13 @@ def benchmark(): """ start = time.time() print "Read file B" - site_a = readDOM(filename='filea.txt') + site_a = readDOM(filename='filea.txt', debug=True) print "--------" print "total done in %s" % (time.time() - start) start = time.time() print "Read file A" - site_b = readDOM(filename='fileb.txt') + site_b = readDOM(filename='fileb.txt', debug=True) print "--------" print "total done in %s" % (time.time() - start) @@ -233,4 +243,4 @@ def benchmark(): print "Corelation: %s" % correlation -benchmark() +#benchmark()

1 0

[translation/liveusb-creator] Update translations for liveusb-creator
by translation＠torproject.org 07 Oct '12

07 Oct '12

commit 03c912364cc68b94fbc2d54ff1f6639632646a2b Author: Translation commit bot <translation(a)torproject.org> Date: Sun Oct 7 16:45:37 2012 +0000 Update translations for liveusb-creator --- hu/hu.po | 67 +++++++++++++++++++++++++++++++------------------------------ 1 files changed, 34 insertions(+), 33 deletions(-) diff --git a/hu/hu.po b/hu/hu.po index 2220ffc..25abf03 100644 --- a/hu/hu.po +++ b/hu/hu.po @@ -4,13 +4,14 @@ # # Translators: # Sulyok Péter <peti(a)fedoraproject.org>, 2009. +# <viktor.varga(a)gmail.com>, 2012. msgid "" msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2012-04-13 14:16+0200\n" -"PO-Revision-Date: 2012-10-03 18:09+0000\n" -"Last-Translator: Tor Project <tor-assistants(a)torproject.org>\n" +"PO-Revision-Date: 2012-10-07 16:45+0000\n" +"Last-Translator: vargaviktor <viktor.varga(a)gmail.com>\n" "Language-Team: LANGUAGE <LL(a)li.org>\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -31,7 +32,7 @@ msgstr "%s letöltése…" #: ../liveusb/gui.py:197 #, python-format msgid "Device is too small: it must be at least %s MiB." -msgstr "" +msgstr "Az eszköz túl kicsi: legalább %s MiB méretű kell legyen." #: ../liveusb/gui.py:226 msgid "" @@ -42,7 +43,7 @@ msgstr "Hiba: A címkét nem lehet beállítani, vagy az eszköz UUID-jét leké #: ../liveusb/gui.py:276 #, python-format msgid "Installation complete! (%s)" -msgstr "" +msgstr "Telepítés kész! (%s)" #: ../liveusb/gui.py:281 msgid "LiveUSB creation failed!" @@ -53,67 +54,67 @@ msgid "" "Warning: This tool needs to be run as an Administrator. To do this, right " "click on the icon and open the Properties. Under the Compatibility tab, " "check the \"Run this program as an administrator\" box." -msgstr "" +msgstr "Figyelem: Ennek az eszköznek Administrator jogosultsággal kell indulnia. Ahhoz, hogy így futhasson kattintson jobb gombbal az ikonján, nyissa meg a Tulajdonságok ablakot, és a Kompatibilitás fülön jelölje a \"Futtatás rendszergazdaként\" opciót." #: ../liveusb/gui.py:413 msgid "Unable to find any USB drive" -msgstr "" +msgstr "Nem található egyetlen USB meghajtó sem" #: ../liveusb/gui.py:521 msgid "" "Device is not yet mounted, so we cannot determine the amount of free space." " Setting a maximum limit of 8G for the persistent storage." -msgstr "" +msgstr "Az eszköz még nem csatolt, így nem eldönthető, hogy mennyi rajta a szabad hely. Az állandó tároló számára 8 Gbyte beállítva." #: ../liveusb/gui.py:528 #, python-format msgid "No free space on device %(device)s" -msgstr "" +msgstr "Nincs szabad hely a %(device)s eszközön." #: ../liveusb/gui.py:533 msgid "Partition is FAT16; Restricting overlay size to 2G" -msgstr "" +msgstr "A partíció FAT16; Korlátozott a 2G feletti méret elérése." #: ../liveusb/gui.py:577 msgid "Persistent Storage" -msgstr "" +msgstr "Állandó tároló" #: ../liveusb/gui.py:586 msgid "Installation complete!" -msgstr "" +msgstr "Telepítés kész!" #: ../liveusb/gui.py:587 msgid "Installation was completed. Press OK to close this program." -msgstr "" +msgstr "A telepítés befejeződött. Kattintson az OK gombra a program bezárásához." #: ../liveusb/gui.py:610 msgid "" "The Master Boot Record on your device is blank. Pressing 'Create Live USB' " "again will reset the MBR on this device." -msgstr "" +msgstr "A Master Boot Record az eszközön üres. A 'Live USB létrehozása' gomb megnyomásával alaphelyzetbe állítja az MBR-t ezen az eszközön." #: ../liveusb/gui.py:624 msgid "" "Warning: The Master Boot Record on your device does not match your system's " "syslinux MBR. If you have trouble booting this stick, try running the " "liveusb-creator with the --reset-mbr option." -msgstr "" +msgstr "Figyelem: A Master Boot Record az eszközén nem egyezik a rendszere syslinux MBR-jével. Ha problémája van ezzel az eszközzel, próbálja meg a liveusb-creator-t futtatni a --reser-mbr opcióval." #: ../liveusb/gui.py:638 msgid "Unable to mount device" -msgstr "" +msgstr "Nem lehetséges az eszköz csatolása" #: ../liveusb/gui.py:644 msgid "Warning: All data on the selected drive will be lost." -msgstr "" +msgstr "Figyelem: Minden adat a meghajtón törlésre kerül." #: ../liveusb/gui.py:646 ../liveusb/gui.py:663 msgid "Press 'Next' if you wish to continue." -msgstr "" +msgstr "Kattintson a 'Tovább' gombra, ha folytatni szeretné." #: ../liveusb/gui.py:648 ../liveusb/gui.py:665 msgid "Next" -msgstr "" +msgstr "Tovább" #: ../liveusb/gui.py:658 msgid "" @@ -157,28 +158,28 @@ msgstr "" #: ../liveusb/gui.py:743 #, python-format msgid "%(filename)s selected" -msgstr "" +msgstr "%(filename)s kiválasztva" #: ../liveusb/creator.py:91 msgid "You must run this application as root" -msgstr "" +msgstr "Az alkalmazást root-ként kell futtatnia" #: ../liveusb/creator.py:130 msgid "" "Source type does not support verification of ISO MD5 checksum, skipping" -msgstr "" +msgstr "A forrás típusa nem teszi lehetővé az ISO MD5 összeg ellenőrzését, kihagyva" #: ../liveusb/creator.py:132 msgid "Verifying ISO MD5 checksum" -msgstr "" +msgstr "Az ISO MD5 összeg ellenőrzése" #: ../liveusb/creator.py:137 msgid "ISO MD5 checksum verification failed" -msgstr "" +msgstr "Az ISO MD5 összeg ellenőrzése sikertelen" #: ../liveusb/creator.py:139 msgid "ISO MD5 checksum passed" -msgstr "" +msgstr "Az ISO MD5 összeg ellenőrzése sikeres" #: ../liveusb/creator.py:144 msgid "Extracting live image to USB device..." @@ -234,7 +235,7 @@ msgstr "" #: ../liveusb/creator.py:497 msgid "Removing existing Live OS" -msgstr "" +msgstr "A meglévő Live operációs rendszer eltávolítása" #: ../liveusb/creator.py:506 ../liveusb/creator.py:517 #, python-format @@ -298,21 +299,21 @@ msgstr "" #: ../liveusb/creator.py:811 #, python-format msgid "Unable to mount device: %(message)s" -msgstr "" +msgstr "%(message)s csatolása nem lehetséges" #: ../liveusb/creator.py:823 msgid "No mount points found" -msgstr "" +msgstr "Nem található csatolásai pont" #: ../liveusb/creator.py:834 #, python-format msgid "Unmounting %(device)s" -msgstr "" +msgstr "%(device)s lecsatolása" #: ../liveusb/creator.py:846 #, python-format msgid "Mount %s exists after unmounting" -msgstr "" +msgstr "%s csatolás létezik lecsatolás után" #: ../liveusb/creator.py:883 ../liveusb/creator.py:1203 msgid "Verifying filesystem..." @@ -321,21 +322,21 @@ msgstr "Fájlrendszer ellenőrzése…" #: ../liveusb/creator.py:907 #, python-format msgid "Unable to change volume label: %(message)s" -msgstr "" +msgstr "Nem változtatható meg a meghajtó címkéje: %(message)s" #: ../liveusb/creator.py:912 ../liveusb/creator.py:1236 msgid "Installing bootloader..." -msgstr "" +msgstr "Bootloader telepítése..." #: ../liveusb/creator.py:935 ../liveusb/creator.py:1254 #, python-format msgid "Removing %(file)s" -msgstr "" +msgstr "%(file)s eltávolítása" #: ../liveusb/creator.py:1017 #, python-format msgid "%s already bootable" -msgstr "" +msgstr "%s már bootolható" #: ../liveusb/creator.py:1037 msgid "Unable to find partition"

1 0

[translation/https_everywhere_completed] Update translations for https_everywhere_completed
by translation＠torproject.org 07 Oct '12

07 Oct '12

commit d7ac4db055ee4f3be2043b327336a98f6a431dce Author: Translation commit bot <translation(a)torproject.org> Date: Sun Oct 7 16:45:35 2012 +0000 Update translations for https_everywhere_completed --- hu/https-everywhere.properties | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/hu/https-everywhere.properties b/hu/https-everywhere.properties index 5b05b0b..1606fbf 100644 --- a/hu/https-everywhere.properties +++ b/hu/https-everywhere.properties @@ -1,2 +1,5 @@ https-everywhere.menu.globalEnable = HTTPS Everywhere bekapcsolása https-everywhere.menu.globalDisable = HTTPS Everywhere kikapcsolása +https-everywhere.menu.enableDisable = Szabályok Engedélyezése / Tiltása +https-everywhere.menu.noRules = (nincs szabály ehhez az oldalhoz) +https-everywhere.menu.unknownRules = (az oldalhoz tartozó szabályok ismeretlenek)

1 0

[translation/https_everywhere] Update translations for https_everywhere
by translation＠torproject.org 07 Oct '12

07 Oct '12

commit 6772b958477e5b17b0c924ebac14ca9902683a0c Author: Translation commit bot <translation(a)torproject.org> Date: Sun Oct 7 16:45:32 2012 +0000 Update translations for https_everywhere --- hu/https-everywhere.properties | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hu/https-everywhere.properties b/hu/https-everywhere.properties index 1154994..1606fbf 100644 --- a/hu/https-everywhere.properties +++ b/hu/https-everywhere.properties @@ -1,5 +1,5 @@ https-everywhere.menu.globalEnable = HTTPS Everywhere bekapcsolása https-everywhere.menu.globalDisable = HTTPS Everywhere kikapcsolása -# https-everywhere.menu.enableDisable = Enable / Disable Rules -# https-everywhere.menu.noRules = (No Rules for This Page) -# https-everywhere.menu.unknownRules = (Rules for This Page Unknown) +https-everywhere.menu.enableDisable = Szabályok Engedélyezése / Tiltása +https-everywhere.menu.noRules = (nincs szabály ehhez az oldalhoz) +https-everywhere.menu.unknownRules = (az oldalhoz tartozó szabályok ismeretlenek)

1 0

[translation/whisperback_completed] Update translations for whisperback_completed
by translation＠torproject.org 07 Oct '12

07 Oct '12

commit 915bda7d669024f60fc02a001cf810c0939859b8 Author: Translation commit bot <translation(a)torproject.org> Date: Sun Oct 7 16:45:29 2012 +0000 Update translations for whisperback_completed --- hu/hu.po | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hu/hu.po b/hu/hu.po index 483dfea..223ebfb 100644 --- a/hu/hu.po +++ b/hu/hu.po @@ -9,14 +9,14 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2012-02-16 22:33+0100\n" -"PO-Revision-Date: 2012-04-21 11:41+0000\n" +"PO-Revision-Date: 2012-10-07 16:21+0000\n" "Last-Translator: vargaviktor <viktor.varga(a)gmail.com>\n" "Language-Team: LANGUAGE <LL(a)li.org>\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Language: hu\n" -"Plural-Forms: nplurals=2; plural=(n != 1)\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" #. XXX use a better exception #: ../whisperBack/whisperback.py:63 @@ -157,8 +157,8 @@ msgid "Send feedback in an encrypted mail." msgstr "Visszajelzés küldése titkosított emailben." #: ../whisperBack/gui.py:405 -msgid "Copyright © 2009-2011 Tails developpers (tails(a)boum.org)" -msgstr "Copyright © 2009-2011 Tails fejlesztők (tails(a)boum.org)" +msgid "Copyright © 2009-2012 Tails developpers (tails(a)boum.org)" +msgstr "Copyright © 2009-2012 Tails fejlesztők (tails(a)boum.org)" #: ../whisperBack/gui.py:406 msgid "Tails developers <tails(a)boum.org>" @@ -182,8 +182,8 @@ msgid "Bug description" msgstr "Hiba leírás" #: ../data/whisperback.ui.h:3 -msgid "Copyright © 2009-2011 tails(a)boum.org" -msgstr "Copyright © 2009-2011 tails(a)boum.org" +msgid "Copyright © 2009-2012 tails(a)boum.org" +msgstr "Copyright © 2009-2012 tails(a)boum.org" #: ../data/whisperback.ui.h:4 msgid "Help" @@ -214,7 +214,7 @@ msgstr "A csatolandó technikai adatok" #: ../data/whisperback.ui.h:12 msgid "" "WhisperBack - Send feedback in an encrypted mail\n" -"Copyright (C) 2009-2011 Tails developers <tails(a)boum.org>\n" +"Copyright (C) 2009-2012 Tails developers <tails(a)boum.org>\n" "\n" "This program is free software; you can redistribute it and/or modify\n" "it under the terms of the GNU General Public License as published by\n" @@ -228,7 +228,7 @@ msgid "" "\n" "You should have received a copy of the GNU General Public License\n" "along with this program. If not, see <http://www.gnu.org/licenses/>.\n" -msgstr "WhisperBack - Visszajelzés küldése titkosított email címen keresztül\nCopyright (C) 2009-2011 Tails developers <tails(a)boum.org>\n\nEz a program egy szabad szoftver; terjesztheti, módosíthatja a GNU General Public License alapjám, amit a \nFree Software Foundation publikált; a 3-as vagy későbbi licence verzió alapján.\n\nEz a program abban a reményben került tejresztésr, hogy hasznos, de SEMMILYEN GARANCIA NEM JÁR HOZZÁ, beleértve a kereskedelmi, és egyéb rész célokat is.\nTekintse meg a GNU General Public License a részletekért.\n\nEzzel a szoftverrel kellett, hogy kapjon egy példányt, a GNU General Public License -ből. Ha nem találja tekintse meg a <http://www.gnu.org/licenses/>oldalon.\n" +msgstr "WhisperBack - Visszajelzés küldése titkosított emailben\nCopyright (C) 2009-2012 Tails fejlesztők <tails(a)boum.org>\n\nEz a program szabad szoftver; terjesztheti és/vagy módosíthatja azt a GNU General Public License szabályait betartva, mely licencet a Free Software Foundation készítette; a 3-as verziójú Licence vagy (választása szerint) későbbi változata alapján.\n\nEz a program annak reményében került terjesztésre, hogy hasznos lehet valakinek, de NINCS RÁ GARANCIA beleértve a beleértett garanciáit egy adott célra használatnak vagy közlésnek. További részletekért forduljon a GNU General Public License-hez.\n\nA programhoz mellékelve kel találjon egy másolatát a GNU General Public License-nek. Ha mégsem találja, tekintse meg a <http://www.gnu.org/licenses/> oldalon.\n" #: ../data/whisperback.ui.h:28 msgid "debugging info"

1 0

[translation/whisperback] Update translations for whisperback
by translation＠torproject.org 07 Oct '12

07 Oct '12

commit 1881aa81db70ccb28b3ee48c4b448add67e1df31 Author: Translation commit bot <translation(a)torproject.org> Date: Sun Oct 7 16:45:28 2012 +0000 Update translations for whisperback --- hu/hu.po | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hu/hu.po b/hu/hu.po index e22d5ba..223ebfb 100644 --- a/hu/hu.po +++ b/hu/hu.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2012-02-16 22:33+0100\n" -"PO-Revision-Date: 2012-10-07 16:15+0000\n" +"PO-Revision-Date: 2012-10-07 16:21+0000\n" "Last-Translator: vargaviktor <viktor.varga(a)gmail.com>\n" "Language-Team: LANGUAGE <LL(a)li.org>\n" "MIME-Version: 1.0\n" @@ -228,7 +228,7 @@ msgid "" "\n" "You should have received a copy of the GNU General Public License\n" "along with this program. If not, see <http://www.gnu.org/licenses/>.\n" -msgstr "" +msgstr "WhisperBack - Visszajelzés küldése titkosított emailben\nCopyright (C) 2009-2012 Tails fejlesztők <tails(a)boum.org>\n\nEz a program szabad szoftver; terjesztheti és/vagy módosíthatja azt a GNU General Public License szabályait betartva, mely licencet a Free Software Foundation készítette; a 3-as verziójú Licence vagy (választása szerint) későbbi változata alapján.\n\nEz a program annak reményében került terjesztésre, hogy hasznos lehet valakinek, de NINCS RÁ GARANCIA beleértve a beleértett garanciáit egy adott célra használatnak vagy közlésnek. További részletekért forduljon a GNU General Public License-hez.\n\nA programhoz mellékelve kel találjon egy másolatát a GNU General Public License-nek. Ha mégsem találja, tekintse meg a <http://www.gnu.org/licenses/> oldalon.\n" #: ../data/whisperback.ui.h:28 msgid "debugging info"

1 0

[translation/tsum] Update translations for tsum
by translation＠torproject.org 07 Oct '12

07 Oct '12

commit cf5d2e0b7e00b4f044b3943902a54f652d1ea18c Author: Translation commit bot <translation(a)torproject.org> Date: Sun Oct 7 16:45:07 2012 +0000 Update translations for tsum --- hu/short-user-manual_hu_noimg.xhtml | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/hu/short-user-manual_hu_noimg.xhtml b/hu/short-user-manual_hu_noimg.xhtml index 7824f86..92aa5ec 100644 --- a/hu/short-user-manual_hu_noimg.xhtml +++ b/hu/short-user-manual_hu_noimg.xhtml @@ -27,7 +27,7 @@ Mielőtt a Tor Böngésző csomagot elindítaná, ellenőriznie kell, hogy a megfelelő verzióval rendelkezik-e. A szoftverhez amit letöltött található egy fájl, ami meg azonos névvel,de .asc kiterjesztéssel rendelkezik. Ez a .asc fájl egy GPG aláírás, és lehetővé teszi, hogy ellenőrizhesse, hogy a fájl ténylegesen az a fájl-e amit mi szerettünk volna, hogy beszerezhessen. Mielőtt ellenőrizni tudná az aláírást, szüksége lehet a GnuPG csomag letöltésére és telepítésére: - Windows: <a href="http://gpg4win.org/download.html">http://gpg4win.org/download.html</a> Mac OS X: <a href="http://www.gpgtools.org/">http://www.gpgtools.org/</a> Linux: Most Linux distributions come with GnuPG preinstalled. + Windows: <a href="http://gpg4win.org/download.html">http://gpg4win.org/download.html</a> Mac OS X: <a href="http://www.gpgtools.org/">http://www.gpgtools.org/</a> Linux: A legtöbb Linux disztribúció a GnuPG-t előre telepítve tartalmazza. Ne felejtse el, hogy lehet, szüksége lesz az alábbi parancsok esetén az útvonalat módosítására. Erinn Clark írja alá a Tor Böngésző csomagot a 0x63FEE659 kulccsal. Erinn kulcsának importáláshoz futtassa a következőt: <pre>

1 0

[ooni-probe/master] Do some more thorough benchmarks that identify the issue in BS
by art＠torproject.org 07 Oct '12

07 Oct '12

commit e977bc9e2c03c198d256f3cb1db745024791be63 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Oct 7 16:44:14 2012 +0000 Do some more thorough benchmarks that identify the issue in BS --- ooni/kit/benchmarks.txt | 35 +++++++++++++++++++ ooni/kit/domclass.py | 87 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 95 insertions(+), 27 deletions(-) diff --git a/ooni/kit/benchmarks.txt b/ooni/kit/benchmarks.txt new file mode 100644 index 0000000..c2070e5 --- /dev/null +++ b/ooni/kit/benchmarks.txt @@ -0,0 +1,35 @@ +Read file B +done in 0.74356508255 +Read file A +done in 0.94336104393 +Computing prob matrix +done in 0.0432229042053 +Computing eigenvalues +done in 0.00188422203064 +Corelation: 0.999999079331 + + +Read file B +done in 0.946599960327 +Read file A +done in 0.909075975418 +Computing prob matrix +done in 0.0541450977325 +Computing eigenvalues +done in 0.00259518623352 +Computing prob matrix B +done in 0.060467004776 +Computing eigen B +done in 0.00102496147156 +Computing correlation +done in 0.000158071517944 +Corelation: 0.999999079331 + +this was with: +683 filea.txt +678 fileb.txt + +diff file* | wc -l +283 + + diff --git a/ooni/kit/domclass.py b/ooni/kit/domclass.py index 33c960a..02c26dc 100644 --- a/ooni/kit/domclass.py +++ b/ooni/kit/domclass.py @@ -32,6 +32,7 @@ import yaml import numpy from bs4 import BeautifulSoup +import time # All HTML4 tags # XXX add link to W3C page where these came from @@ -123,10 +124,17 @@ def readDOM(content=None, filename=None): content = ''.join(f.readlines()) f.close() + start = time.time() + print "Running BeautifulSoup on content" dom = BeautifulSoup(content) + print "done in %s" % (time.time() - start) + + start = time.time() + print "Creating couples matrix" couples = [] for x in dom.findAll(): couples.append((str(x.parent.name), str(x.name))) + print "done in %s" % (time.time() - start) return couples @@ -138,66 +146,91 @@ def compute_correlation(matrix_a, matrix_b): def benchmark(): """ - Running some very basic benchmarks we assets this: - - Read file B - done in 0.74356508255 - Read file A - done in 0.94336104393 - Computing prob matrix - done in 0.0432229042053 - Computing eigenvalues - done in 0.00188422203064 - Corelation: 0.999999079331 + Running some very basic benchmarks on this input data: - this was with: + Data files: 683 filea.txt 678 fileb.txt diff file* | wc -l 283 + We get such results: + + Read file B + Running BeautifulSoup on content + done in 0.768223047256 + Creating couples matrix + done in 0.023903131485 + -------- + total done in 0.796372890472 + Read file A + Running BeautifulSoup on content + done in 0.752885818481 + Creating couples matrix + done in 0.0163578987122 + -------- + total done in 0.770951986313 + Computing prob matrix + done in 0.0475239753723 + Computing eigenvalues + done in 0.00161099433899 + Computing prob matrix B + done in 0.0408289432526 + Computing eigen B + done in 0.000268936157227 + Computing correlation + done in 0.00016713142395 + Corelation: 0.999999079331 What this means is that the bottleneck is not in the maths, but is rather in the computation of the DOM tree matrix. - XXX We should focus on optimizing the parsing of the HTML and the - computation of the couple matrix. + XXX We should focus on optimizing the parsing of the HTML (this depends on + beautiful soup). Perhaps we can find and alternative to it that is + sufficient for us. """ - import time start = time.time() - print "Read file B" site_a = readDOM(filename='filea.txt') - print "done in %s" % (time.time() - start) - start = time.time() + print "--------" + print "total done in %s" % (time.time() - start) + start = time.time() print "Read file A" site_b = readDOM(filename='fileb.txt') - print "done in %s" % (time.time() - start) - start = time.time() - - + print "--------" + print "total done in %s" % (time.time() - start) a = {} + b = {} + + start = time.time() print "Computing prob matrix" a['matrix'] = compute_probability_matrix(site_a) - print "done in %s" % (time.time() - start) start = time.time() print "Computing eigenvalues" a['eigen'] = compute_eigenvalues(a['matrix']) - print "done in %s" % (time.time() - start) start = time.time() - - b = {} + start = time.time() + print "Computing prob matrix B" b['matrix'] = compute_probability_matrix(site_b) + print "done in %s" % (time.time() - start) + + start = time.time() + print "Computing eigen B" b['eigen'] = compute_eigenvalues(b['matrix']) + print "done in %s" % (time.time() - start) + start = time.time() + print "Computing correlation" correlation = compute_correlation(a['eigen'], b['eigen']) + print "done in %s" % (time.time() - start) + print "Corelation: %s" % correlation -#benchmark() +benchmark()

1 0

[ooni-probe/master] Do some benckmarks on domclass and figure out where optimization should be
by art＠torproject.org 07 Oct '12

07 Oct '12

commit c30049b0d9daf748378f3da717411f4c0b85a23d Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Oct 7 16:32:00 2012 +0000 Do some benckmarks on domclass and figure out where optimization should be done. (We must optimize how the DOM tree is parsed and how we compute the couple matrix) --- ooni/kit/domclass.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 files changed, 51 insertions(+), 2 deletions(-) diff --git a/ooni/kit/domclass.py b/ooni/kit/domclass.py index d50647c..33c960a 100644 --- a/ooni/kit/domclass.py +++ b/ooni/kit/domclass.py @@ -136,14 +136,63 @@ def compute_correlation(matrix_a, matrix_b): correlation = (correlation + 1)/2 return correlation -def example(): +def benchmark(): + """ + Running some very basic benchmarks we assets this: + + Read file B + done in 0.74356508255 + Read file A + done in 0.94336104393 + Computing prob matrix + done in 0.0432229042053 + Computing eigenvalues + done in 0.00188422203064 + Corelation: 0.999999079331 + + this was with: + 683 filea.txt + 678 fileb.txt + + diff file* | wc -l + 283 + + + What this means is that the bottleneck is not in the maths, but is rather + in the computation of the DOM tree matrix. + + XXX We should focus on optimizing the parsing of the HTML and the + computation of the couple matrix. + """ + import time + start = time.time() + + print "Read file B" site_a = readDOM(filename='filea.txt') + print "done in %s" % (time.time() - start) + start = time.time() + + print "Read file A" site_b = readDOM(filename='fileb.txt') + print "done in %s" % (time.time() - start) + start = time.time() + + a = {} + print "Computing prob matrix" a['matrix'] = compute_probability_matrix(site_a) + + print "done in %s" % (time.time() - start) + start = time.time() + + print "Computing eigenvalues" a['eigen'] = compute_eigenvalues(a['matrix']) + print "done in %s" % (time.time() - start) + start = time.time() + + b = {} b['matrix'] = compute_probability_matrix(site_b) b['eigen'] = compute_eigenvalues(b['matrix']) @@ -151,4 +200,4 @@ def example(): correlation = compute_correlation(a['eigen'], b['eigen']) print "Corelation: %s" % correlation - +#benchmark()

1 0

[ooni-probe/master] Turn DOMClass into a kit
by art＠torproject.org 07 Oct '12

07 Oct '12

commit 9590d92f9e36b10b6960f83246f35e73d72f55e5 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Oct 7 16:24:46 2012 +0000 Turn DOMClass into a kit --- ooni/kit/domclass.py | 154 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 154 insertions(+), 0 deletions(-) diff --git a/ooni/kit/domclass.py b/ooni/kit/domclass.py new file mode 100644 index 0000000..d50647c --- /dev/null +++ b/ooni/kit/domclass.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python +#-*- encoding: utf-8 -*- +# +# domclass +# ******** +# +# :copyright: (c) 2012 by Arturo Filastò +# :license: see LICENSE for more details. +# +# how this works +# -------------- +# +# This classifier uses the DOM structure of a website to determine how similar +# the two sites are. +# The procedure we use is the following: +# * First we parse all the DOM tree of the web page and we build a list of +# TAG parent child relationships (ex. <html><a></a><c></c></html> => +# (html, a), (a, b), (html, c)). +# +# * We then use this information to build a matrix (M) where m[i][j] = P(of +# transitioning from tag[i] to tag[j]). If tag[i] does not exists P() = 0. +# Note: M is a square matrix that is number_of_tags wide. +# +# * We then calculate the eigenvectors (v_i) and eigenvalues (e) of M. +# +# * The corelation between page A and B is given via this formula: +# correlation = dot_product(e_A, e_B), where e_A and e_B are +# resepectively the eigenvalues for the probability matrix A and the +# probability matrix B. +# + +import yaml +import numpy +from bs4 import BeautifulSoup + +# All HTML4 tags +# XXX add link to W3C page where these came from +alltags = ['A', 'ABBR', 'ACRONYM', 'ADDRESS', 'APPLET', 'AREA', 'B', 'BASE', + 'BASEFONT', 'BD', 'BIG', 'BLOCKQUOTE', 'BODY', 'BR', 'BUTTON', 'CAPTION', + 'CENTER', 'CITE', 'CODE', 'COL', 'COLGROUP', 'DD', 'DEL', 'DFN', 'DIR', 'DIV', + 'DL', 'DT', 'E M', 'FIELDSET', 'FONT', 'FORM', 'FRAME', 'FRAMESET', 'H1', 'H2', + 'H3', 'H4', 'H5', 'H6', 'HEAD', 'HR', 'HTML', 'I', 'IFRAME ', 'IMG', + 'INPUT', 'INS', 'ISINDEX', 'KBD', 'LABEL', 'LEGEND', 'LI', 'LINK', 'MAP', + 'MENU', 'META', 'NOFRAMES', 'NOSCRIPT', 'OBJECT', 'OL', 'OPTGROUP', 'OPTION', + 'P', 'PARAM', 'PRE', 'Q', 'S', 'SAMP', 'SCRIPT', 'SELECT', 'SMALL', 'SPAN', + 'STRIKE', 'STRONG', 'STYLE', 'SUB', 'SUP', 'TABLE', 'TBODY', 'TD', + 'TEXTAREA', 'TFOOT', 'TH', 'THEAD', 'TITLE', 'TR', 'TT', 'U', 'UL', 'VAR'] + +# Reduced subset of only the most common tags +commontags = ['A', 'B', 'BLOCKQUOTE', 'BODY', 'BR', 'BUTTON', 'CAPTION', + 'CENTER', 'CITE', 'CODE', 'COL', 'DD', 'DIV', + 'DL', 'DT', 'EM', 'FIELDSET', 'FONT', 'FORM', 'FRAME', 'FRAMESET', 'H1', 'H2', + 'H3', 'H4', 'H5', 'H6', 'HEAD', 'HR', 'HTML', 'IFRAME ', 'IMG', + 'INPUT', 'INS', 'LABEL', 'LEGEND', 'LI', 'LINK', 'MAP', + 'MENU', 'META', 'NOFRAMES', 'NOSCRIPT', 'OBJECT', 'OL', 'OPTION', + 'P', 'PRE', 'SCRIPT', 'SELECT', 'SMALL', 'SPAN', + 'STRIKE', 'STRONG', 'STYLE', 'SUB', 'SUP', 'TABLE', 'TBODY', 'TD', + 'TEXTAREA', 'TFOOT', 'TH', 'THEAD', 'TITLE', 'TR', 'TT', 'U', 'UL'] + +# The tags we are intested in using for our analysis +thetags = ['A', 'DIV', 'FRAME', 'H1', 'H2', + 'H3', 'H4', 'IFRAME ', 'INPUT', + 'LABEL','LI', 'P', 'SCRIPT', 'SPAN', + 'STYLE', 'TR'] + +def compute_probability_matrix(dataset): + """ + Compute the probability matrix based on the input dataset. + + :dataset: an array of pairs representing the parent child relationships. + """ + import itertools + ret = {} + matrix = numpy.zeros((len(thetags) + 1, len(thetags) + 1)) + + for data in dataset: + x = data[0].upper() + y = data[1].upper() + try: + x = thetags.index(x) + except: + x = len(thetags) + + try: + y = thetags.index(y) + except: + y = len(thetags) + + matrix[x,y] += 1 + + for x in xrange(len(thetags) + 1): + possibilities = 0 + for y in matrix[x]: + possibilities += y + + for i in xrange(len(matrix[x])): + if possibilities != 0: + matrix[x][i] = matrix[x][i]/possibilities + + return matrix + +def compute_eigenvalues(matrix): + """ + Returns the eigenvalues of the supplied square matrix. + + :matrix: must be a square matrix and diagonalizable. + """ + return numpy.linalg.eigvals(matrix) + +def readDOM(content=None, filename=None): + """ + Parses the DOM of the HTML page and returns an array of parent, child + pairs. + + :content: the content of the HTML page to be read. + + :filename: the filename to be read from for getting the content of the + page. + """ + + if filename: + f = open(filename) + content = ''.join(f.readlines()) + f.close() + + dom = BeautifulSoup(content) + couples = [] + for x in dom.findAll(): + couples.append((str(x.parent.name), str(x.name))) + + return couples + +def compute_correlation(matrix_a, matrix_b): + correlation = numpy.vdot(matrix_a, matrix_b) + correlation /= numpy.linalg.norm(matrix_a)*numpy.linalg.norm(matrix_b) + correlation = (correlation + 1)/2 + return correlation + +def example(): + site_a = readDOM(filename='filea.txt') + site_b = readDOM(filename='fileb.txt') + + a = {} + a['matrix'] = compute_probability_matrix(site_a) + a['eigen'] = compute_eigenvalues(a['matrix']) + + b = {} + b['matrix'] = compute_probability_matrix(site_b) + b['eigen'] = compute_eigenvalues(b['matrix']) + + correlation = compute_correlation(a['eigen'], b['eigen']) + print "Corelation: %s" % correlation + +

1 0