This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 95445f49f1be510e16b4cfc7d5312f69bf46d9da Author: David Goulet dgoulet@torproject.org AuthorDate: Mon Jun 27 14:46:21 2022 -0400

ext: Add Equi-X library

Signed-off-by: David Goulet dgoulet@torproject.org --- .gitignore | 1 + Makefile.am | 10 +- configure.ac | 2 + src/ext/equix/.gitmodules | 3 + src/ext/equix/CMakeLists.txt | 82 ++++ src/ext/equix/LICENSE | 165 +++++++ src/ext/equix/README.md | 77 +++ src/ext/equix/build.sh | 7 + src/ext/equix/devlog.md | 178 +++++++ src/ext/equix/hashx/.gitignore | 9 + src/ext/equix/hashx/CMakeLists.txt | 99 ++++ src/ext/equix/hashx/LICENSE | 165 +++++++ src/ext/equix/hashx/README.md | 135 ++++++ src/ext/equix/hashx/include/hashx.h | 140 ++++++ src/ext/equix/hashx/src/bench.c | 135 ++++++ src/ext/equix/hashx/src/blake2.c | 467 +++++++++++++++++++ src/ext/equix/hashx/src/blake2.h | 73 +++ src/ext/equix/hashx/src/compiler.c | 18 + src/ext/equix/hashx/src/compiler.h | 41 ++ src/ext/equix/hashx/src/compiler_a64.c | 154 ++++++ src/ext/equix/hashx/src/compiler_x86.c | 151 ++++++ src/ext/equix/hashx/src/context.c | 81 ++++ src/ext/equix/hashx/src/context.h | 45 ++ src/ext/equix/hashx/src/force_inline.h | 9 + src/ext/equix/hashx/src/hashx.c | 134 ++++++ src/ext/equix/hashx/src/hashx_endian.h | 103 +++++ src/ext/equix/hashx/src/hashx_thread.c | 27 ++ src/ext/equix/hashx/src/hashx_thread.h | 27 ++ src/ext/equix/hashx/src/hashx_time.c | 35 ++ src/ext/equix/hashx/src/hashx_time.h | 9 + src/ext/equix/hashx/src/instruction.h | 31 ++ src/ext/equix/hashx/src/program.c | 771 +++++++++++++++++++++++++++++++ src/ext/equix/hashx/src/program.h | 48 ++ src/ext/equix/hashx/src/program_exec.c | 152 ++++++ src/ext/equix/hashx/src/siphash.c | 66 +++ src/ext/equix/hashx/src/siphash.h | 35 ++ src/ext/equix/hashx/src/siphash_rng.c | 31 ++ src/ext/equix/hashx/src/siphash_rng.h | 30 ++ src/ext/equix/hashx/src/test_utils.h | 60 +++ src/ext/equix/hashx/src/tests.c | 219 +++++++++ src/ext/equix/hashx/src/unreachable.h | 9 + src/ext/equix/hashx/src/virtual_memory.c | 126 +++++ src/ext/equix/hashx/src/virtual_memory.h | 19 + src/ext/equix/include/equix.h | 145 ++++++ src/ext/equix/src/bench.c | 175 +++++++ src/ext/equix/src/context.c | 57 +++ src/ext/equix/src/context.h | 18 + src/ext/equix/src/equix.c | 96 ++++ src/ext/equix/src/solver.c | 270 +++++++++++ src/ext/equix/src/solver.h | 30 ++ src/ext/equix/src/solver_heap.h | 108 +++++ src/ext/equix/src/tests.c | 124 +++++ 52 files changed, 5200 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore index 94988ed982..737ab72bf5 100644 --- a/.gitignore +++ b/.gitignore @@ -152,6 +152,7 @@ core.* # /src/ext/ /src/ext/ed25519/ref10/libed25519_ref10.lib /src/ext/ed25519/donna/libed25519_donna.lib +/src/ext/equix/build /src/ext/keccak-tiny/libkeccak-tiny.lib

# /src/app diff --git a/Makefile.am b/Makefile.am index c7b8b16d35..28a1c8e57d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -124,6 +124,10 @@ TOR_CRYPTO_TESTING_LIBS = \ $(LIBDONNA) endif

+EQUIX_LIBS = \ + src/ext/equix/build/libequix.a \ + src/ext/equix/build/hashx/libhashx.a + # All static libraries used to link tor. TOR_INTERNAL_LIBS = \ src/core/libtor-app.a \ @@ -132,7 +136,8 @@ TOR_INTERNAL_LIBS = \ $(TOR_CRYPTO_LIBS) \ $(TOR_UTIL_LIBS) \ src/trunnel/libor-trunnel.a \ - src/lib/libtor-trace.a + src/lib/libtor-trace.a \ + $(EQUIX_LIBS)

libtor.a: $(TOR_INTERNAL_LIBS) $(AM_V_AR) export AR="$(AR)"; \ @@ -152,7 +157,8 @@ TOR_INTERNAL_TESTING_LIBS = \ $(TOR_CRYPTO_TESTING_LIBS) \ $(TOR_UTIL_TESTING_LIBS) \ src/trunnel/libor-trunnel-testing.a \ - src/lib/libtor-trace.a + src/lib/libtor-trace.a \ + $(EQUIX_LIBS)

src/test/libtor-testing.a: $(TOR_INTERNAL_TESTING_LIBS) $(AM_V_AR) export AR="$(AR)"; \ diff --git a/configure.ac b/configure.ac index f2e5f72e97..c0b531c81d 100644 --- a/configure.ac +++ b/configure.ac @@ -31,6 +31,8 @@ tor_incr_n_warnings() { tor_ac_n_warnings=`expr $tor_ac_n_warnings + 1` }

+AC_CONFIG_COMMANDS([equix], [./src/ext/equix/build.sh]) + m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) AC_CONFIG_HEADERS([orconfig.h])

diff --git a/src/ext/equix/.gitmodules b/src/ext/equix/.gitmodules new file mode 100644 index 0000000000..6f1277ee9f --- /dev/null +++ b/src/ext/equix/.gitmodules @@ -0,0 +1,3 @@ +[submodule "hashx"] + path = hashx + url = https://github.com/tevador/hashx diff --git a/src/ext/equix/CMakeLists.txt b/src/ext/equix/CMakeLists.txt new file mode 100644 index 0000000000..032989d804 --- /dev/null +++ b/src/ext/equix/CMakeLists.txt @@ -0,0 +1,82 @@ +# Copyright (c) 2020 tevador tevador@gmail.com +# See LICENSE for licensing information + +cmake_minimum_required(VERSION 2.8.8) + +set(EQUIX_VERSION 1) +set(EQUIX_VERSION_MINOR 0) +set(EQUIX_VERSION_PATCH 0) +set(EQUIX_VERSION_STR "${EQUIX_VERSION}.${EQUIX_VERSION_MINOR}.${EQUIX_VERSION_PATCH}") + +project(equix) + +add_definitions(-DHASHX_SIZE=8) + +add_subdirectory("hashx") + +set(equix_sources +src/context.c +src/equix.c +src/solver.c) + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) + message(STATUS "Setting default build type: ${CMAKE_BUILD_TYPE}") +endif() + +add_library(equix SHARED ${equix_sources}) +set_property(TARGET equix PROPERTY POSITION_INDEPENDENT_CODE ON) +set_property(TARGET equix PROPERTY PUBLIC_HEADER include/equix.h) +include_directories(equix + include/ + hashx/include/ + hashx/src/) +target_compile_definitions(equix PRIVATE HASHX_STATIC) +target_compile_definitions(equix PRIVATE EQUIX_SHARED) +target_link_libraries(equix + PRIVATE hashx_static) +set_target_properties(equix PROPERTIES VERSION ${EQUIX_VERSION_STR} + SOVERSION ${EQUIX_VERSION}) + +add_library(equix_static STATIC ${equix_sources}) +set_property(TARGET equix_static PROPERTY POSITION_INDEPENDENT_CODE ON) +set_target_properties(equix_static PROPERTIES OUTPUT_NAME equix) +target_compile_definitions(equix_static PRIVATE HASHX_STATIC) +target_compile_definitions(equix_static PRIVATE EQUIX_STATIC) +include_directories(equix_static + include/ + hashx/include/ + hashx/src/) +target_link_libraries(equix_static + PRIVATE hashx_static) + +include(GNUInstallDirs) +install(TARGETS equix equix_static + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + +add_executable(equix-tests + src/tests.c) +include_directories(equix-tests + include/) +target_compile_definitions(equix-tests PRIVATE EQUIX_STATIC) +target_link_libraries(equix-tests + PRIVATE equix_static) + +if(NOT Threads_FOUND AND UNIX AND NOT APPLE) + set(THREADS_PREFER_PTHREAD_FLAG ON) + find_package(Threads) +endif() + +add_executable(equix-bench + src/bench.c + hashx/src/hashx_thread.c + hashx/src/hashx_time.c) +include_directories(equix-bench + include/ + hashx/src/) +target_compile_definitions(equix-bench PRIVATE EQUIX_STATIC) +target_link_libraries(equix-bench + PRIVATE equix_static + PRIVATE ${CMAKE_THREAD_LIBS_INIT}) diff --git a/src/ext/equix/LICENSE b/src/ext/equix/LICENSE new file mode 100644 index 0000000000..0a041280bd --- /dev/null +++ b/src/ext/equix/LICENSE @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. https://fsf.org/ + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/src/ext/equix/README.md b/src/ext/equix/README.md new file mode 100644 index 0000000000..d3e8c93125 --- /dev/null +++ b/src/ext/equix/README.md @@ -0,0 +1,77 @@ +# Equi-X + +Equi-X is a CPU-friendly [client puzzle](https://en.wikipedia.org/wiki/Client_Puzzle_Protocol) +with fast verification and small solution size (16 bytes). It is based on Equihash(60,3) with +two major changes: + +1. Blake2b hash function is replaced with [HashX](https://github.com/tevador/hashx). +2. XOR is replaced with modular addition. + +An Equi-X solution for nonce `X` is a set of eight 16-bit indices <code>i₀, ..., i₇</code> such that: + +<code>H_X(i₀) + H_X(i₁) + H_X(i₂) + H_X(i₃) + H_X(i₄) + H_X(i₅) + H_X(i₆) + H_X(i₇) = 0 (mod 2⁶⁰)</code> + +where <code>H_X</code> is a HashX function generated for nonce `X`. Equi-X is therefore a variant of the [subset sum problem](https://en.wikipedia.org/wiki/Subset_sum_problem). Each nonce value provides 2 solutions on average. + +Equi-X also has additional requirements that prove that the solution was found using the Wagner's algorithm. See the [Equihash paper](https://eprint.iacr.org/2015/946.pdf) for details. + +### Example solution + +``` +H(0x6c31) = 0xcfa5375c0a7f5d7 \ + (+) = 0xa73d9777f110000 \ +H(0x8803) = 0xd798601be690a29 / | + (+) = 0xefdaadb00000000 \ +H(0x80c2) = 0xcabd8974bbee8d5 \ | | + (+) = 0x489d16380ef0000 / | +H(0xa1db) = 0x7ddf8cc3530172b / | + (+) = 0 +H(0x6592) = 0x348a96fd685dcba \ | + (+) = 0x357120e8ffb8000 \ | +H(0x76b7) = 0x00e689eb975a346 / | | + (+) = 0x102552500000000 / +H(0x74a6) = 0xacccc4ad2d06bcd \ | + (+) = 0xdab431670048000 / +H(0xe259) = 0x2de76cb9d341433 / +``` + +## Performance + +|Algorithm |n |k |memory |solution size|verification ¹|CPU perf. ²|GPU perf. ³| +|----------|---|---|-------|-------------|------------|-----------|----------| +|**Equi-X**|60 |3 |1.8 MiB|16 bytes |~50 μs |2400 Sol/s| ? | +|Zcash |200|9 |144 MiB|1344 bytes |>150 μs |30 Sol/s |~400 Sol/s ⁴| +|BTG |144|5 |2.5 GiB|100 bytes |~10 μs |1 Sol/s |~45 Sol/s ⁵| + +1. Using AMD Ryzen 1700 with 1 thread. +1. Using AMD Ryzen 1700 with 16 threads. +1. Using NVIDIA GTX 1660 Ti. +1. Estimated from http://www.zcashbenchmarks.info/ (GTX 1070) +1. Estimated from https://miniz.ch/features/ + +## Build + +``` +git clone --recursive https://github.com/tevador/equix.git +cd equix +mkdir build +cd build +cmake .. +make +``` +``` +./equix-tests +./equix-bench --help +``` + +## Design notes + +See [devlog.md](devlog.md) + +## Donations + +You can support the development of Equi-X by sending XMR to this address: + +``` +85GkKXSD8EQ22EMC2ZKF64S6L6Lcm8Gr23VbAKB1zg6FUW81sUEmrvvRPoM3GCpUZSC9azCLdeityW2N3CVsV4CAC3p1evV +``` \ No newline at end of file diff --git a/src/ext/equix/build.sh b/src/ext/equix/build.sh new file mode 100755 index 0000000000..753e3a138e --- /dev/null +++ b/src/ext/equix/build.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +cd ./src/ext/equix +mkdir build +cd build +cmake .. +make diff --git a/src/ext/equix/devlog.md b/src/ext/equix/devlog.md new file mode 100644 index 0000000000..fbd9da346e --- /dev/null +++ b/src/ext/equix/devlog.md @@ -0,0 +1,178 @@ +# DoS protection for onion services: from RandomX to Equi-X + +In early May 2020, I was contacted by two Tor developers to help them with their denial-of-service (DoS) mitigation proposal. Onion services have been plagued by DoS for many years and there are no easy solutions due to the anonymous nature of the Tor network. The vulnerability stems from the fact that the service has to perform a series of expensive operations for each connecting client, so an attacker can simply flood the service with connection requests, which are comparatively cheap [...] + +## Proof of work + +One of the strategies that can be used against such attacks are client puzzles, which require the client to prove that they performed certain amount of computational work before the request is accepted. This is not a new idea – using a client puzzle to combat email spam was first proposed in 1997 [[1](https://cypherpunks.venona.com/date/1997/03/msg00774.html)] and there have been proposals to add a client puzzle to the TLS protocol that we all use every day to browse the web [[2](https:/ [...] + +The goal of the current Tor proposal [[3](https://lists.torproject.org/pipermail/tor-dev/2020-April/014215.html)] is to integrate some kind of client puzzle as part of the protocol used to connect to onion services. The client would download the hidden service descriptor, find that the service is protected by a client puzzle, calculate the solution and send it as part of the connection request. The service would give the request a priority value based on the "difficulty" of the puzzle solution. + +The main issue is to find a suitable puzzle. There are three main requirements that the algorithm must meet: + +1. The solution proof must be smaller than about 200 bytes. +2. Solution verification must be fast. +3. GPUs and FPGAs should not provide a large advantage for solving the puzzle. + +The first requirement is due to the limited space in the message that the client sends to the hidden service to negotiate a connection. This doesn't pose a big problem in practice. + +The second requirement is needed to prevent the client puzzle from becoming a DoS vector on its own. The service must be able to quickly verify if a proof provided by a client is valid or not, otherwise the attacker could just flood the service with invalid proofs. + +The third requirement is the most important one. Users of the Tor browser are equipped with an ordinary general-purpose CPU and an attacker must not be able to use similarly priced hardware that offers a much higher performance for solving the puzzle – otherwise it would be easy for an attacker to simulate many legitimate clients connecting to the service. + +It should be noted that the puzzle does not need to be resistant to speed-up by specialized fixed-function hardware (ASIC). This is called "ASIC resistance" and there are two main reasons why it's not strictly required in this case: + +1. The cost to produce a specialized chip for a given algorithm exceeds USD $1 million for the 28 nm process [[4](https://www.electronicdesign.com/technologies/embedded-revolution/article/21...)]. This is in contrast with the current state when bringing down an onion service requires very little resources [[5](https://www.reddit.com/r/darknet/comments/b3qvbq/this_ddos_is_massive_its_go...)]. +2. If an ASIC for the client puzzle is developed, the algorithm can be quickly changed with a simple patch, rendering the attacker's investment useless. This is in contrast with cryptocurrencies, which have complex consensus protocols and can take months or years to change their proof of work algorithm [[6](https://ethereum-magicians.org/t/eip-progpow-a-programmatic-proof-of-work/27...)]. + +Therefore the client puzzle algorithm must be CPU-friendly and minimize the advantage of programmable hardware such as GPUs and FPGAs. + +## RandomX + +In 2019, we developed a new CPU-friendly proof of work algorithm called RandomX to be used by Monero [[7](https://web.getmonero.org/resources/moneropedia/randomx.html)]. It has been audited and field tested, which makes it a good candidate to be considered for a Tor client puzzle. + +However, during the development of RandomX, we focused mainly on the "CPU-friendliness" of the algorithm to minimize the advantage of specialized hardware. Other aspects such as verification performance and memory requirements were secondary and only had upper limits. The result is that RandomX has quite high solution verification demands – over 2 GiB of memory and around 2 milliseconds to verify a hash. Alternatively, it is possible to use only 256 MiB of memory, but then verification t [...] + +So I began slimming down RandomX to make it less demanding for verification. I halved the memory requirements and reduced the number of rounds from 8 to 2, which is the minimum that can be considered safe from optimizations. The result is RandomX-Tor [[8](https://github.com/tevador/RandomX/tree/tor-pow)], which takes around 0.5 ms to verify with slightly over 1 GiB of memory. GPUs perform very poorly when running RandomX-Tor – similar to full RandomX. + +However, I see two main problems with RandomX-Tor: + +1. 2000 verifications per second still may not be fast enough to prevent flooding attacks on the service, especially since each connection request to an onion service requires very little network bandwidth (512 bytes). +1. The 1 GB memory requirement is too high. Onion services would have to allocate over 2 GB just to verify puzzles because two different puzzles may be active at a given moment to allow for network synchronization delays. + +So I decided to continue my quest and look for a better puzzle. + +## Alternatives + +I briefly tested Argon2 [[9](https://github.com/p-h-c/phc-winner-argon2)] and Yespower [[10](https://www.openwall.com/yespower/)], which are password hashing algorithms claiming to be "GPU-resistant". However, both have slower verification performance than RandomX-Tor and both run faster on GPUs, so they are not suitable. See Appendix for details. + +Then I researched Equihash, which is an asymmetric client puzzle offering very fast solution verification [[11](https://eprint.iacr.org/2015/946.pdf)]. The main problem with Equihash is that GPUs perform up to 100x faster than CPUs, so it is unusable as a puzzle for CPU-based clients. + +## HashX + +I was ready to give up when I remembered SuperscalarHash, which is a part of RandomX that's used only to initialize the DAG [[12](https://github.com/tevador/RandomX/blob/master/doc/specs.md#6-superscalarhas...)]. It could be considered a lightweight version of RandomX with only integer operations and no memory accesses. I began refactoring the code with the goal of creating a fast GPU-resistant hash function. The main idea is that each instance of the hash function would use a completely d [...] + +I named the new algorithm "HashX" and in the course of the next week, I made several other changes compared to the old SuperscalarHash: + +* improved code generator that can fully saturate the simulated 3-way superscalar pipeline +* removed reciprocal multiplication, which was sometimes causing front-end stalls +* implemented a more optimized JIT compiler for x86-64 +* added input-dependent branches to further hinder GPU performance + +I also made a few tweaks to the hash finalizer to pass SMHasher [[13](https://github.com/rurban/smhasher)]. The result is a CPU-friendly algorithm suitable to be used as a partial hash inversion client puzzle. + +## Equi-X + +I still felt a bit uneasy about HashX using no memory. This means that logic-only FPGAs could be a viable option to run HashX. Additionally, introducing the right amount of memory-hardness could affect GPUs more than CPUs. I remembered Equihash, which is a memory-hard algorithm, but still allows for fast solution verification. I could simply replace the Blake2b hash in the default implementation with HashX. Each solution attempt would then use a different randomly generated HashX instance. + +Equihash has two main parameters that determine the difficulty of the generalized birthday problem: `n` is the hash size in bits and <code>2^k</code> is the number of hashes that need to XOR to zero. The memory hardness is proportional to <code>2^n/(k+1)</code>. To get fast verification and a small proof size, `k` should be as low as possible. I chose `k=3`, which means the goal would be to find preimages to 8 hashes that XOR to zero. + +I first tried with `n=96`, which would require about 2 GB of memory for an efficient solver. However, the problem is that each HashX instance would be used for <code>2²⁵</code> hashes, which could make it feasible for GPUs to compile an optimized kernel for each new solution attempt. Additionally, GPUs are much better at the sorting phase of Equihash than CPUs due to their higher memory bandwidth. + +After some more benchmarking, I estimated that <code>2¹⁶</code> hashes per solution attempt would be the sweet spot for CPUs because one solution attempt would take roughly 6-8 ms, the HashX compilation overhead would still be under 1% and the expected memory usage would be under 2 MiB. This can fit entirely into the CPU cache, which is the only case when CPUs can compete with GPUs in memory bandwidth. The optimal Equihash parameters are then `n=60` and `k=3`. I call this algo [...] + +However, when testing Equi-X, I discovered a fatal flaw: the generalized birthday problem, as used by Equihash, requires a collision resistant hash function. While HashX is preimage-resistant, it is not collision resistant. Some small fraction of HashX instances produce a significant number of hash collisions. With a 2^k-XOR, finding 2^k-1 pairs of colliding hashes automatically produces a valid solution because equal values XOR to zero. But it gets worse. If the numb [...] + +Some modifications of HashX reduced the number of collisions, but I could not completely eliminate them, so I needed another solution. I began researching the generalized birthday problem and I found this paragraph in the seminal paper by David Wagner [[14](https://people.eecs.berkeley.edu/~daw/papers/genbday.html)]: + + + +The advantage of 2^k-SUM compared to 2^k-XOR is that identical hashes no longer produce valid solutions (except for `0x000000000000000` and `0x800000000000000`, but these are very rare). There are two additional advantages: + +1. While XOR and addition have the same performance in CPUs, XOR is much faster in custom hardware. This means, for example, that an FPGA-based solver will have to use slightly more resources to calculate the modular additions. +1. With 2^k-SUM, we can eliminate checking for duplicate indices, which simplifies both the solver and the verifier [[15](https://github.com/tevador/equix/issues/1)]. + +With this modification, an Equi-X solution for nonce `X` is a set of eight 16-bit indices <code>i₀, ..., i₇</code> such that: + +<code>H_X(i₀) + H_X(i₁) + H_X(i₂) + H_X(i₃) + H_X(i₄) + H_X(i₅) + H_X(i₆) + H_X(i₇) = 0 (mod 2⁶⁰)</code> + +where <code>H_X</code> is a HashX function generated for nonce `X`. + +Example of a solution: + +``` +H(0x6c31) = 0xcfa5375c0a7f5d7 \ + (+) = 0xa73d9777f110000 \ +H(0x8803) = 0xd798601be690a29 / | + (+) = 0xefdaadb00000000 \ +H(0x80c2) = 0xcabd8974bbee8d5 \ | | + (+) = 0x489d16380ef0000 / | +H(0xa1db) = 0x7ddf8cc3530172b / | + (+) = 0 +H(0x6592) = 0x348a96fd685dcba \ | + (+) = 0x357120e8ffb8000 \ | +H(0x76b7) = 0x00e689eb975a346 / | | + (+) = 0x102552500000000 / +H(0x74a6) = 0xacccc4ad2d06bcd \ | + (+) = 0xdab431670048000 / +H(0xe259) = 0x2de76cb9d341433 / +``` + +The following table compares Equi-X with the most common variants of Equihash: + +|Algorithm |n |k |memory|solution size|verification ¹|CPU perf. ²|GPU perf. ³| +|----------|---|---|-------|-------------|------------|-----------|----------| +|**Equi-X**|60 |3 |1.8 MiB|16 bytes |~50 μs |2400 Sol/s| ? | +|Zcash |200|9 |144 MiB|1344 bytes |>150 μs |30 Sol/s |~400 Sol/s ⁴| +|BTG |144|5 |2.5 GiB|100 bytes |~10 μs |1 Sol/s |~45 Sol/s ⁵| + +1. Using AMD Ryzen 1700 with 1 thread. +1. Using AMD Ryzen 1700 with 16 threads. +1. Using NVIDIA GTX 1660 Ti. +1. Estimated from http://www.zcashbenchmarks.info/ (GTX 1070) +1. Estimated from https://miniz.ch/features/ + +## References + +[1] https://cypherpunks.venona.com/date/1997/03/msg00774.html + +[2] https://tools.ietf.org/html/draft-nygren-tls-client-puzzles-02 + +[3] https://lists.torproject.org/pipermail/tor-dev/2020-April/014215.html + +[4] https://www.electronicdesign.com/technologies/embedded-revolution/article/21... + +[5] https://www.reddit.com/r/darknet/comments/b3qvbq/this_ddos_is_massive_its_go... + +[6] https://ethereum-magicians.org/t/eip-progpow-a-programmatic-proof-of-work/27... + +[7] https://web.getmonero.org/resources/moneropedia/randomx.html + +[8] https://github.com/tevador/RandomX/tree/tor-pow + +[9] https://github.com/p-h-c/phc-winner-argon2 + +[10] https://www.openwall.com/yespower/ + +[11] https://eprint.iacr.org/2015/946.pdf + +[12] https://github.com/tevador/RandomX/blob/master/doc/specs.md#6-superscalarhas... + +[13] https://github.com/rurban/smhasher + +[14] https://people.eecs.berkeley.edu/~daw/papers/genbday.html + +[15] https://github.com/tevador/equix/issues/1 + +## Appendix: Comparison of GPU-resistant client puzzles + +|Algorithm|GPU speed ⁶|Solver memory|Verifier memory|Verif./sec. ¹¹|Solution size ¹²| +|------------|---------|-------------|---------------|---------------|-------------| +|**Equi-X** ¹|<50% ⁷|1.81 MiB|< 20 KiB|19600| 32 bytes| +|**HashX** ²|<50% ⁷|< 20 KiB|< 20 KiB|19700|16 bytes| +|RandomX-Tor ³|10% ⁸| 1041 MiB|1041 MiB|2200|16 bytes| +|Argon2 ⁴|~300% ⁹|2.00 MiB|2.00 MiB|1020|16 bytes| +|Yespower ⁵|~200% ¹⁰|2.00 MiB|2.00 MiB|780|16 bytes| + +1. https://github.com/tevador/equix +1. https://github.com/tevador/hashx +1. https://github.com/tevador/RandomX/tree/tor-pow +1. https://github.com/p-h-c/phc-winner-argon2 +1. https://www.openwall.com/yespower/ +1. Performance of NVIDIA GTX 1660 Ti compared to AMD Ryzen 1700 +1. No GPU implementation exists; upper bound based on RandomX performance +1. Benchmarked using https://github.com/SChernykh/RandomX_CUDA +1. Estimated from https://github.com/WebDollar/argon2-gpu +1. Estimated from https://nlpool.nl/bench?algo=yescrypt +1. Benchmarked on AMD Ryzen 1700 @ 3.3 GHz using 1 thread +1. Including a 16-byte nonce. diff --git a/src/ext/equix/hashx/.gitignore b/src/ext/equix/hashx/.gitignore new file mode 100644 index 0000000000..ec94c2c699 --- /dev/null +++ b/src/ext/equix/hashx/.gitignore @@ -0,0 +1,9 @@ +bin/ +obj/ +*.user +*.suo +.vs +x64/ +Release/ +Debug/ +build/ diff --git a/src/ext/equix/hashx/CMakeLists.txt b/src/ext/equix/hashx/CMakeLists.txt new file mode 100644 index 0000000000..5eb694d5dd --- /dev/null +++ b/src/ext/equix/hashx/CMakeLists.txt @@ -0,0 +1,99 @@ +# Copyright (c) 2020 tevador tevador@gmail.com +# See LICENSE for licensing information + +cmake_minimum_required(VERSION 2.8.8) + +set(HASHX_VERSION 1) +set(HASHX_VERSION_MINOR 0) +set(HASHX_VERSION_PATCH 0) +set(HASHX_VERSION_STR "${HASHX_VERSION}.${HASHX_VERSION_MINOR}.${HASHX_VERSION_PATCH}") + +project(hashx) + +set(hashx_sources +src/blake2.c +src/compiler.c +src/compiler_a64.c +src/compiler_x86.c +src/context.c +src/hashx.c +src/program.c +src/program_exec.c +src/siphash.c +src/siphash_rng.c +src/virtual_memory.c) + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) + message(STATUS "Setting default build type: ${CMAKE_BUILD_TYPE}") +endif() + +option(HASHX_BLOCK_MODE "Hash function for block mode" OFF) + +if(HASHX_BLOCK_MODE) + add_definitions(-DHASHX_BLOCK_MODE) +endif() + +set(HASHX_SIZE CACHE STRING "Hash function output size in bytes") + +if(HASHX_SIZE) + if(HASHX_SIZE GREATER 32) + message(SEND_ERROR "The maximum hash size is 32 bytes") + else() + add_definitions(-DHASHX_SIZE=${HASHX_SIZE}) + endif() +endif() + +set(HASHX_SALT CACHE STRING "Implementation-specific salt value") + +if(HASHX_SALT) + string(LENGTH ${HASHX_SALT} HASHX_SALT_LENGTH) + if(HASHX_SALT_LENGTH GREATER 15) + message(SEND_ERROR "The maximum salt length is 15 characters") + else() + add_definitions(-DHASHX_SALT=${HASHX_SALT}) + endif() +endif() + +add_library(hashx SHARED ${hashx_sources}) +set_property(TARGET hashx PROPERTY POSITION_INDEPENDENT_CODE ON) +set_property(TARGET hashx PROPERTY PUBLIC_HEADER include/hashx.h) +include_directories(hashx + include/) +target_compile_definitions(hashx PRIVATE HASHX_SHARED) +set_target_properties(hashx PROPERTIES VERSION ${HASHX_VERSION_STR} + SOVERSION ${HASHX_VERSION}) + +add_library(hashx_static STATIC ${hashx_sources}) +set_property(TARGET hashx_static PROPERTY POSITION_INDEPENDENT_CODE ON) +set_target_properties(hashx_static PROPERTIES OUTPUT_NAME hashx) + +include(GNUInstallDirs) +install(TARGETS hashx hashx_static + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + +add_executable(hashx-tests + src/tests.c) +include_directories(hashx-tests + include/) +target_compile_definitions(hashx-tests PRIVATE HASHX_STATIC) +target_link_libraries(hashx-tests + PRIVATE hashx_static) + +if(NOT Threads_FOUND AND UNIX AND NOT APPLE) + set(THREADS_PREFER_PTHREAD_FLAG ON) + find_package(Threads) +endif() + +add_executable(hashx-bench + src/bench.c + src/hashx_thread.c + src/hashx_time.c) +include_directories(hashx-bench + include/) +target_compile_definitions(hashx-bench PRIVATE HASHX_STATIC) +target_link_libraries(hashx-bench + PRIVATE hashx_static + PRIVATE ${CMAKE_THREAD_LIBS_INIT}) diff --git a/src/ext/equix/hashx/LICENSE b/src/ext/equix/hashx/LICENSE new file mode 100644 index 0000000000..0a041280bd --- /dev/null +++ b/src/ext/equix/hashx/LICENSE @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. https://fsf.org/ + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/src/ext/equix/hashx/README.md b/src/ext/equix/hashx/README.md new file mode 100644 index 0000000000..1d8ea47652 --- /dev/null +++ b/src/ext/equix/hashx/README.md @@ -0,0 +1,135 @@ +# HashX + +HashX is an algorithm designed for client puzzles and proof-of-work schemes. +While traditional cryptographic hash functions use a fixed one-way compression +function, each HashX instance represents a unique pseudorandomly generated +one-way function. + +HashX functions are generated as a carefully crafted sequence of integer +operations to fully saturate a 3-way superscalar CPU pipeline (modeled after +the Intel Ivy Bridge architecture). Extra care is taken to avoid optimizations +and to ensure that each function takes exactly the same number of CPU cycles +(currently 512 instructions over 192 cycles). + +## API + +The API consists of 4 functions and is documented in the public header file +[hashx.h](include/hashx.h). + +Example of usage: + +```c +#include <hashx.h> +#include <stdio.h> + +int main() { + char seed[] = "this is a seed that will generate a hash function"; + char hash[HASHX_SIZE]; + hashx_ctx* ctx = hashx_alloc(HASHX_COMPILED); + if (ctx == HASHX_NOTSUPP) + ctx = hashx_alloc(HASHX_INTERPRETED); + if (ctx == NULL) + return 1; + if (!hashx_make(ctx, seed, sizeof(seed))) /* generate a hash function */ + return 1; + hashx_exec(ctx, 123456789, hash); /* calculate the hash of a nonce value */ + hashx_free(ctx); + for (unsigned i = 0; i < HASHX_SIZE; ++i) + printf("%02x", hash[i] & 0xff); + printf("\n"); + return 0; +} +``` + +## Build + +A C99-compatible compiler and `cmake` are required. + +``` +git clone https://github.com/tevador/hashx.git +cd hashx +mkdir build +cd build +cmake .. [-DHASHX_BLOCK_MODE=ON] [-DHASHX_SIZE=<1-32>] [-DHASHX_SALT="my custom hash"] +make +``` + +### Block mode (default: off) + +Because HashX is meant to be used in proof-of-work schemes and client puzzles, +the input is a 64-bit counter value. If you need to hash arbitrary data, build +with `-DHASHX_BLOCK_MODE=ON`. This will change the API to accept `const void*, size_t` instead of `uint64_t`. +However, it is strongly recommended to use the counter mode, which is almost twice faster for short inputs. + +### Hash size (default: 32) + +The default hash output size is 32 bytes (256 bits). If you want to reduce the output +size, build with, for example, `-DHASHX_SIZE=20`. Output sizes in the range of 1-32 bytes +are supported. Shorter output sizes are formed by simply truncating the full 256-bit hash. + +### Generator salt (default: "HashX v1") + +An implementation-specific salt value may be specified when building, for example: `-DHASHX_SALT="my custom hash"`. +This value is used as a salt when generating hash instances. The maximum supported +salt size is 15 characters. + +## Performance + +HashX was designed for fast verification. Generating a hash function from seed +takes about 50 μs and a 64-bit nonce can be hashed in under 100 ns (in compiled +mode) or in about 1-2 μs (in interpreted mode). + +A benchmark executable is included: +``` +./hashx-bench --seeds 500 +``` + +## Security + +HashX should provide strong preimage resistance. No other security guarantees are made. About + 99% of HashX instances pass [SMHasher](https://github.com/tevador/smhasher), + but using HashX as a generic hash function is not recommended. + +Known vulnerabilities that should not affect intended use cases: + +1. HashX is not collision resistant. Around 0.2% of seeds produce "weak" hash functions for +which hash collisions are plentiful. +2. Secret values should not be used as inputs to HashX because the generated instructions +include data-dependent branches by design. + +## Protocols based on HashX + +Here are two examples of how HashX can be used in practice: + +### Interactive client puzzle + +Client puzzles are protocols designed to protect server resources from abuse. +A client requesting a resource from a server may be asked to solve a puzzle +before the request is accepted. + +One of the first proposed client puzzles is [Hashcash](https://en.wikipedia.org/wiki/Hashcash), +which requires the client to find a partial SHA-1 hash inversion. However, +because of the static nature of cryptographic hash functions, an attacker can +offload hashing to a GPU or FPGA to gain a significant advantage over legitimate +clients equipped only with a CPU. + +In a HashX-based interactive client puzzle, the server sends each client +a 256-bit challenge used to generate a unique HashX function. The client then +has to find a 64-bit nonce value such that the resulting hash has a predefined +number of leading zeroes. An attacker cannot easily parallelize the workload +because each request would require a new GPU kernel or FPGA bistream. + +### Non-interactive proof-of-work + +In the absence of a central authority handing out challenges (for example in +a cryptocurrency), the client takes some public information `T` (for example +a block template) and combines it with a chosen 64-bit nonce `N1`. +The resulting string `X = T||N1` is then used to generate a HashX function +<code>H_X</code>. The client then tries to find a 16-bit nonce `N2` +such that <code>r = H_X(N2)</code> meets the difficulty target of +the protocol. If no `N2` value is successful, the client increments `N1` and +tries with a different hash function. + +In this protocol, each HashX function provides only 65536 attempts before it +must be discarded. This limits the parallelization advantage of GPUs and FPGAs. +A CPU core will be able to test about 200 different hash functions per second. diff --git a/src/ext/equix/hashx/include/hashx.h b/src/ext/equix/hashx/include/hashx.h new file mode 100644 index 0000000000..c95fd295ef --- /dev/null +++ b/src/ext/equix/hashx/include/hashx.h @@ -0,0 +1,140 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +/* + * HashX is an algorithm designed for client puzzles and proof-of-work schemes. + * While traditional cryptographic hash functions use a fixed one-way + * compression function, each HashX instance represents a unique pseudorandomly + * generated one-way function. + * + * Example of usage: + * + #include <hashx.h> + #include <stdio.h> + + int main() { + char seed[] = "this is a seed that will generate a hash function"; + char hash[HASHX_SIZE]; + hashx_ctx* ctx = hashx_alloc(HASHX_COMPILED); + if (ctx == HASHX_NOTSUPP) + ctx = hashx_alloc(HASHX_INTERPRETED); + if (ctx == NULL) + return 1; + if (!hashx_make(ctx, seed, sizeof(seed))) + return 1; + hashx_exec(ctx, 123456789, hash); + hashx_free(ctx); + for (unsigned i = 0; i < HASHX_SIZE; ++i) + printf("%02x", hash[i] & 0xff); + printf("\n"); + return 0; + } + * + */ + +#ifndef HASHX_H +#define HASHX_H + +#include <stdint.h> +#include <stddef.h> + +/* + * Input of the hash function. + * + * Counter mode (default): a 64-bit unsigned integer + * Block mode: pointer to a buffer and the number of bytes to be hashed +*/ +#ifndef HASHX_BLOCK_MODE +#define HASHX_INPUT uint64_t input +#else +#define HASHX_INPUT const void* input, size_t size +#endif + +/* The default (and maximum) hash size is 32 bytes */ +#ifndef HASHX_SIZE +#define HASHX_SIZE 32 +#endif + +/* Opaque struct representing a HashX instance */ +typedef struct hashx_ctx hashx_ctx; + +/* Type of hash function */ +typedef enum hashx_type { + HASHX_INTERPRETED, + HASHX_COMPILED +} hashx_type; + +/* Sentinel value used to indicate unsupported type */ +#define HASHX_NOTSUPP ((hashx_ctx*)-1) + +#if defined(_WIN32) || defined(__CYGWIN__) +#define HASHX_WIN +#endif + +/* Shared/static library definitions */ +#ifdef HASHX_WIN + #ifdef HASHX_SHARED + #define HASHX_API __declspec(dllexport) + #elif !defined(HASHX_STATIC) + #define HASHX_API __declspec(dllimport) + #else + #define HASHX_API + #endif + #define HASHX_PRIVATE +#else + #ifdef HASHX_SHARED + #define HASHX_API __attribute__ ((visibility ("default"))) + #else + #define HASHX_API __attribute__ ((visibility ("hidden"))) + #endif + #define HASHX_PRIVATE __attribute__ ((visibility ("hidden"))) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Allocate a HashX instance. + * + * @param type is the type of instance to be created. + * + * @return pointer to a new HashX instance. Returns NULL on memory allocation + * failure and HASHX_NOTSUPP if the requested type is not supported. +*/ +HASHX_API hashx_ctx* hashx_alloc(hashx_type type); + +/* + * Create a new HashX function from seed. + * + * @param ctx is pointer to a HashX instance. + * @param seed is a pointer to the seed value. + * @param size is the size of the seed. + * + * @return 1 on success, 0 on failure. +*/ +HASHX_API int hashx_make(hashx_ctx* ctx, const void* seed, size_t size); + +/* + * Execute the HashX function. + * + * @param ctx is pointer to a HashX instance. A HashX function must have + * been previously created by calling hashx_make. + * @param HASHX_INPUT is the input to be hashed (see definition above). + * @param output is a pointer to the result buffer. HASHX_SIZE bytes will be + * written. + s*/ +HASHX_API void hashx_exec(const hashx_ctx* ctx, HASHX_INPUT, void* output); + +/* + * Free a HashX instance. + * + * @param ctx is pointer to a HashX instance. +*/ +HASHX_API void hashx_free(hashx_ctx* ctx); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/ext/equix/hashx/src/bench.c b/src/ext/equix/hashx/src/bench.c new file mode 100644 index 0000000000..fbcd41a064 --- /dev/null +++ b/src/ext/equix/hashx/src/bench.c @@ -0,0 +1,135 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include "test_utils.h" +#include "hashx_thread.h" +#include "hashx_endian.h" +#include "hashx_time.h" +#include <limits.h> +#include <inttypes.h> + +typedef struct worker_job { + int id; + hashx_thread thread; + hashx_ctx* ctx; + int64_t total_hashes; + uint64_t best_hash; + uint64_t threshold; + int start; + int step; + int end; + int nonces; +} worker_job; + +static hashx_thread_retval worker(void* args) { + worker_job* job = (worker_job*)args; + job->total_hashes = 0; + job->best_hash = UINT64_MAX; + for (int seed = job->start; seed < job->end; seed += job->step) { + if (!hashx_make(job->ctx, &seed, sizeof(seed))) { + continue; + } + for (int nonce = 0; nonce < job->nonces; ++nonce) { + uint8_t hash[HASHX_SIZE] = { 0 }; +#ifndef HASHX_BLOCK_MODE + hashx_exec(job->ctx, nonce, hash); +#else + hashx_exec(job->ctx, &nonce, sizeof(nonce), hash); +#endif + uint64_t hashval = load64(hash); + if (hashval < job->best_hash) { + job->best_hash = hashval; + } + if (hashval < job->threshold) { + printf("[thread %2i] Hash (%5i, %5i) below threshold:" + " ...%02x%02x%02x%02x%02x%02x%02x%02x\n", + job->id, + seed, + nonce, + hash[0], + hash[1], + hash[2], + hash[3], + hash[4], + hash[5], + hash[6], + hash[7]); + } + } + job->total_hashes += job->nonces; + } + return HASHX_THREAD_SUCCESS; +} + +int main(int argc, char** argv) { + int nonces, seeds, start, diff, threads; + bool interpret; + read_int_option("--diff", argc, argv, &diff, INT_MAX); + read_int_option("--start", argc, argv, &start, 0); + read_int_option("--seeds", argc, argv, &seeds, 500); + read_int_option("--nonces", argc, argv, &nonces, 65536); + read_int_option("--threads", argc, argv, &threads, 1); + read_option("--interpret", argc, argv, &interpret); + hashx_type flags = HASHX_INTERPRETED; + if (!interpret) { + flags = HASHX_COMPILED; + } + uint64_t best_hash = UINT64_MAX; + uint64_t diff_ex = (uint64_t)diff * 1000ULL; + uint64_t threshold = UINT64_MAX / diff_ex; + int seeds_end = seeds + start; + int64_t total_hashes = 0; + printf("Interpret: %i, Target diff.: %" PRIu64 ", Threads: %i\n", interpret, diff_ex, threads); + printf("Testing seeds %i-%i with %i nonces each ...\n", start, seeds_end - 1, nonces); + double time_start, time_end; + worker_job* jobs = malloc(sizeof(worker_job) * threads); + if (jobs == NULL) { + printf("Error: memory allocation failure\n"); + return 1; + } + for (int thd = 0; thd < threads; ++thd) { + jobs[thd].ctx = hashx_alloc(flags); + if (jobs[thd].ctx == NULL) { + printf("Error: memory allocation failure\n"); + return 1; + } + if (jobs[thd].ctx == HASHX_NOTSUPP) { + printf("Error: not supported. Try with --interpret\n"); + return 1; + } + jobs[thd].id = thd; + jobs[thd].start = start + thd; + jobs[thd].step = threads; + jobs[thd].end = seeds_end; + jobs[thd].nonces = nonces; + jobs[thd].threshold = threshold; + } + time_start = hashx_time(); + if (threads > 1) { + for (int thd = 0; thd < threads; ++thd) { + jobs[thd].thread = hashx_thread_create(&worker, &jobs[thd]); + } + for (int thd = 0; thd < threads; ++thd) { + hashx_thread_join(jobs[thd].thread); + } + } + else { + worker(jobs); + } + time_end = hashx_time(); + for (int thd = 0; thd < threads; ++thd) { + total_hashes += jobs[thd].total_hashes; + if (jobs[thd].best_hash < best_hash) { + best_hash = jobs[thd].best_hash; + } + } + double elapsed = time_end - time_start; + printf("Total hashes: %" PRIi64 "\n", total_hashes); + printf("%f hashes/sec.\n", total_hashes / elapsed); + printf("%f seeds/sec.\n", seeds / elapsed); + printf("Best hash: ..."); + output_hex((char*)&best_hash, sizeof(best_hash)); + printf(" (diff: %" PRIu64 ")\n", UINT64_MAX / best_hash); + free(jobs); + return 0; +} diff --git a/src/ext/equix/hashx/src/blake2.c b/src/ext/equix/hashx/src/blake2.c new file mode 100644 index 0000000000..f353cb3064 --- /dev/null +++ b/src/ext/equix/hashx/src/blake2.c @@ -0,0 +1,467 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +/* Original code from Argon2 reference source code package used under CC0 + * https://github.com/P-H-C/phc-winner-argon2 + * Copyright 2015 + * Daniel Dinu, Dmitry Khovratovich, Jean-Philippe Aumasson, and Samuel Neves +*/ + +#include <stdint.h> +#include <string.h> +#include <stdio.h> +#include <stdlib.h> + +#include "blake2.h" +#include "hashx_endian.h" + +static const uint64_t blake2b_IV[8] = { + UINT64_C(0x6a09e667f3bcc908), UINT64_C(0xbb67ae8584caa73b), + UINT64_C(0x3c6ef372fe94f82b), UINT64_C(0xa54ff53a5f1d36f1), + UINT64_C(0x510e527fade682d1), UINT64_C(0x9b05688c2b3e6c1f), + UINT64_C(0x1f83d9abfb41bd6b), UINT64_C(0x5be0cd19137e2179) }; + +#define BLAKE2_SIGMA_0_0 0 +#define BLAKE2_SIGMA_0_1 1 +#define BLAKE2_SIGMA_0_2 2 +#define BLAKE2_SIGMA_0_3 3 +#define BLAKE2_SIGMA_0_4 4 +#define BLAKE2_SIGMA_0_5 5 +#define BLAKE2_SIGMA_0_6 6 +#define BLAKE2_SIGMA_0_7 7 +#define BLAKE2_SIGMA_0_8 8 +#define BLAKE2_SIGMA_0_9 9 +#define BLAKE2_SIGMA_0_10 10 +#define BLAKE2_SIGMA_0_11 11 +#define BLAKE2_SIGMA_0_12 12 +#define BLAKE2_SIGMA_0_13 13 +#define BLAKE2_SIGMA_0_14 14 +#define BLAKE2_SIGMA_0_15 15 + +#define BLAKE2_SIGMA_1_0 14 +#define BLAKE2_SIGMA_1_1 10 +#define BLAKE2_SIGMA_1_2 4 +#define BLAKE2_SIGMA_1_3 8 +#define BLAKE2_SIGMA_1_4 9 +#define BLAKE2_SIGMA_1_5 15 +#define BLAKE2_SIGMA_1_6 13 +#define BLAKE2_SIGMA_1_7 6 +#define BLAKE2_SIGMA_1_8 1 +#define BLAKE2_SIGMA_1_9 12 +#define BLAKE2_SIGMA_1_10 0 +#define BLAKE2_SIGMA_1_11 2 +#define BLAKE2_SIGMA_1_12 11 +#define BLAKE2_SIGMA_1_13 7 +#define BLAKE2_SIGMA_1_14 5 +#define BLAKE2_SIGMA_1_15 3 + +#define BLAKE2_SIGMA_2_0 11 +#define BLAKE2_SIGMA_2_1 8 +#define BLAKE2_SIGMA_2_2 12 +#define BLAKE2_SIGMA_2_3 0 +#define BLAKE2_SIGMA_2_4 5 +#define BLAKE2_SIGMA_2_5 2 +#define BLAKE2_SIGMA_2_6 15 +#define BLAKE2_SIGMA_2_7 13 +#define BLAKE2_SIGMA_2_8 10 +#define BLAKE2_SIGMA_2_9 14 +#define BLAKE2_SIGMA_2_10 3 +#define BLAKE2_SIGMA_2_11 6 +#define BLAKE2_SIGMA_2_12 7 +#define BLAKE2_SIGMA_2_13 1 +#define BLAKE2_SIGMA_2_14 9 +#define BLAKE2_SIGMA_2_15 4 + +#define BLAKE2_SIGMA_3_0 7 +#define BLAKE2_SIGMA_3_1 9 +#define BLAKE2_SIGMA_3_2 3 +#define BLAKE2_SIGMA_3_3 1 +#define BLAKE2_SIGMA_3_4 13 +#define BLAKE2_SIGMA_3_5 12 +#define BLAKE2_SIGMA_3_6 11 +#define BLAKE2_SIGMA_3_7 14 +#define BLAKE2_SIGMA_3_8 2 +#define BLAKE2_SIGMA_3_9 6 +#define BLAKE2_SIGMA_3_10 5 +#define BLAKE2_SIGMA_3_11 10 +#define BLAKE2_SIGMA_3_12 4 +#define BLAKE2_SIGMA_3_13 0 +#define BLAKE2_SIGMA_3_14 15 +#define BLAKE2_SIGMA_3_15 8 + +#define BLAKE2_SIGMA_4_0 9 +#define BLAKE2_SIGMA_4_1 0 +#define BLAKE2_SIGMA_4_2 5 +#define BLAKE2_SIGMA_4_3 7 +#define BLAKE2_SIGMA_4_4 2 +#define BLAKE2_SIGMA_4_5 4 +#define BLAKE2_SIGMA_4_6 10 +#define BLAKE2_SIGMA_4_7 15 +#define BLAKE2_SIGMA_4_8 14 +#define BLAKE2_SIGMA_4_9 1 +#define BLAKE2_SIGMA_4_10 11 +#define BLAKE2_SIGMA_4_11 12 +#define BLAKE2_SIGMA_4_12 6 +#define BLAKE2_SIGMA_4_13 8 +#define BLAKE2_SIGMA_4_14 3 +#define BLAKE2_SIGMA_4_15 13 + +#define BLAKE2_SIGMA_5_0 2 +#define BLAKE2_SIGMA_5_1 12 +#define BLAKE2_SIGMA_5_2 6 +#define BLAKE2_SIGMA_5_3 10 +#define BLAKE2_SIGMA_5_4 0 +#define BLAKE2_SIGMA_5_5 11 +#define BLAKE2_SIGMA_5_6 8 +#define BLAKE2_SIGMA_5_7 3 +#define BLAKE2_SIGMA_5_8 4 +#define BLAKE2_SIGMA_5_9 13 +#define BLAKE2_SIGMA_5_10 7 +#define BLAKE2_SIGMA_5_11 5 +#define BLAKE2_SIGMA_5_12 15 +#define BLAKE2_SIGMA_5_13 14 +#define BLAKE2_SIGMA_5_14 1 +#define BLAKE2_SIGMA_5_15 9 + +#define BLAKE2_SIGMA_6_0 12 +#define BLAKE2_SIGMA_6_1 5 +#define BLAKE2_SIGMA_6_2 1 +#define BLAKE2_SIGMA_6_3 15 +#define BLAKE2_SIGMA_6_4 14 +#define BLAKE2_SIGMA_6_5 13 +#define BLAKE2_SIGMA_6_6 4 +#define BLAKE2_SIGMA_6_7 10 +#define BLAKE2_SIGMA_6_8 0 +#define BLAKE2_SIGMA_6_9 7 +#define BLAKE2_SIGMA_6_10 6 +#define BLAKE2_SIGMA_6_11 3 +#define BLAKE2_SIGMA_6_12 9 +#define BLAKE2_SIGMA_6_13 2 +#define BLAKE2_SIGMA_6_14 8 +#define BLAKE2_SIGMA_6_15 11 + +#define BLAKE2_SIGMA_7_0 13 +#define BLAKE2_SIGMA_7_1 11 +#define BLAKE2_SIGMA_7_2 7 +#define BLAKE2_SIGMA_7_3 14 +#define BLAKE2_SIGMA_7_4 12 +#define BLAKE2_SIGMA_7_5 1 +#define BLAKE2_SIGMA_7_6 3 +#define BLAKE2_SIGMA_7_7 9 +#define BLAKE2_SIGMA_7_8 5 +#define BLAKE2_SIGMA_7_9 0 +#define BLAKE2_SIGMA_7_10 15 +#define BLAKE2_SIGMA_7_11 4 +#define BLAKE2_SIGMA_7_12 8 +#define BLAKE2_SIGMA_7_13 6 +#define BLAKE2_SIGMA_7_14 2 +#define BLAKE2_SIGMA_7_15 10 + +#define BLAKE2_SIGMA_8_0 6 +#define BLAKE2_SIGMA_8_1 15 +#define BLAKE2_SIGMA_8_2 14 +#define BLAKE2_SIGMA_8_3 9 +#define BLAKE2_SIGMA_8_4 11 +#define BLAKE2_SIGMA_8_5 3 +#define BLAKE2_SIGMA_8_6 0 +#define BLAKE2_SIGMA_8_7 8 +#define BLAKE2_SIGMA_8_8 12 +#define BLAKE2_SIGMA_8_9 2 +#define BLAKE2_SIGMA_8_10 13 +#define BLAKE2_SIGMA_8_11 7 +#define BLAKE2_SIGMA_8_12 1 +#define BLAKE2_SIGMA_8_13 4 +#define BLAKE2_SIGMA_8_14 10 +#define BLAKE2_SIGMA_8_15 5 + +#define BLAKE2_SIGMA_9_0 10 +#define BLAKE2_SIGMA_9_1 2 +#define BLAKE2_SIGMA_9_2 8 +#define BLAKE2_SIGMA_9_3 4 +#define BLAKE2_SIGMA_9_4 7 +#define BLAKE2_SIGMA_9_5 6 +#define BLAKE2_SIGMA_9_6 1 +#define BLAKE2_SIGMA_9_7 5 +#define BLAKE2_SIGMA_9_8 15 +#define BLAKE2_SIGMA_9_9 11 +#define BLAKE2_SIGMA_9_10 9 +#define BLAKE2_SIGMA_9_11 14 +#define BLAKE2_SIGMA_9_12 3 +#define BLAKE2_SIGMA_9_13 12 +#define BLAKE2_SIGMA_9_14 13 +#define BLAKE2_SIGMA_9_15 0 + +#define BLAKE2_SIGMA_10_0 0 +#define BLAKE2_SIGMA_10_1 1 +#define BLAKE2_SIGMA_10_2 2 +#define BLAKE2_SIGMA_10_3 3 +#define BLAKE2_SIGMA_10_4 4 +#define BLAKE2_SIGMA_10_5 5 +#define BLAKE2_SIGMA_10_6 6 +#define BLAKE2_SIGMA_10_7 7 +#define BLAKE2_SIGMA_10_8 8 +#define BLAKE2_SIGMA_10_9 9 +#define BLAKE2_SIGMA_10_10 10 +#define BLAKE2_SIGMA_10_11 11 +#define BLAKE2_SIGMA_10_12 12 +#define BLAKE2_SIGMA_10_13 13 +#define BLAKE2_SIGMA_10_14 14 +#define BLAKE2_SIGMA_10_15 15 + +#define BLAKE2_SIGMA_11_0 14 +#define BLAKE2_SIGMA_11_1 10 +#define BLAKE2_SIGMA_11_2 4 +#define BLAKE2_SIGMA_11_3 8 +#define BLAKE2_SIGMA_11_4 9 +#define BLAKE2_SIGMA_11_5 15 +#define BLAKE2_SIGMA_11_6 13 +#define BLAKE2_SIGMA_11_7 6 +#define BLAKE2_SIGMA_11_8 1 +#define BLAKE2_SIGMA_11_9 12 +#define BLAKE2_SIGMA_11_10 0 +#define BLAKE2_SIGMA_11_11 2 +#define BLAKE2_SIGMA_11_12 11 +#define BLAKE2_SIGMA_11_13 7 +#define BLAKE2_SIGMA_11_14 5 +#define BLAKE2_SIGMA_11_15 3 + +static FORCE_INLINE uint64_t rotr64(const uint64_t w, const unsigned c) { + return (w >> c) | (w << (64 - c)); +} + +static FORCE_INLINE void blake2b_set_lastblock(blake2b_state* S) { + S->f[0] = (uint64_t)-1; +} + +static FORCE_INLINE void blake2b_increment_counter(blake2b_state* S, + uint64_t inc) { + S->t[0] += inc; + S->t[1] += (S->t[0] < inc); +} + +static FORCE_INLINE void blake2b_invalidate_state(blake2b_state* S) { + //clear_internal_memory(S, sizeof(*S)); /* wipe */ + blake2b_set_lastblock(S); /* invalidate for further use */ +} + +static FORCE_INLINE void blake2b_init0(blake2b_state* S) { + memset(S, 0, sizeof(*S)); + memcpy(S->h, blake2b_IV, sizeof(S->h)); +} + +int hashx_blake2b_init_param(blake2b_state* S, const blake2b_param* P) { + const unsigned char* p = (const unsigned char*)P; + unsigned int i; + + if (NULL == P || NULL == S) { + return -1; + } + + blake2b_init0(S); + /* IV XOR Parameter Block */ + for (i = 0; i < 8; ++i) { + S->h[i] ^= load64(&p[i * sizeof(S->h[i])]); + } + S->outlen = P->digest_length; + return 0; +} + +#define SIGMA(r, k) BLAKE2_SIGMA_ ## r ## _ ## k + +#define G(r, i, j, a, b, c, d) \ + do { \ + a = a + b + m[SIGMA(r, i)]; \ + d = rotr64(d ^ a, 32); \ + c = c + d; \ + b = rotr64(b ^ c, 24); \ + a = a + b + m[SIGMA(r, j)]; \ + d = rotr64(d ^ a, 16); \ + c = c + d; \ + b = rotr64(b ^ c, 63); \ + } while ((void)0, 0) + +#define ROUND_INNER(r) \ + do { \ + G(r, 0, 1, v[0], v[4], v[8], v[12]); \ + G(r, 2, 3, v[1], v[5], v[9], v[13]); \ + G(r, 4, 5, v[2], v[6], v[10], v[14]); \ + G(r, 6, 7, v[3], v[7], v[11], v[15]); \ + G(r, 8, 9, v[0], v[5], v[10], v[15]); \ + G(r, 10, 11, v[1], v[6], v[11], v[12]); \ + G(r, 12, 13, v[2], v[7], v[8], v[13]); \ + G(r, 14, 15, v[3], v[4], v[9], v[14]); \ + } while ((void)0, 0) + +#define ROUND(r) ROUND_INNER(r) + +static void blake2b_compress(blake2b_state* S, const uint8_t* block) { + uint64_t m[16]; + uint64_t v[16]; + unsigned int i; + + for (i = 0; i < 16; ++i) { + m[i] = load64(block + i * sizeof(m[i])); + } + + for (i = 0; i < 8; ++i) { + v[i] = S->h[i]; + } + + v[8] = blake2b_IV[0]; + v[9] = blake2b_IV[1]; + v[10] = blake2b_IV[2]; + v[11] = blake2b_IV[3]; + v[12] = blake2b_IV[4] ^ S->t[0]; + v[13] = blake2b_IV[5] ^ S->t[1]; + v[14] = blake2b_IV[6] ^ S->f[0]; + v[15] = blake2b_IV[7] ^ S->f[1]; + + ROUND(0); + ROUND(1); + ROUND(2); + ROUND(3); + ROUND(4); + ROUND(5); + ROUND(6); + ROUND(7); + ROUND(8); + ROUND(9); + ROUND(10); + ROUND(11); + + for (i = 0; i < 8; ++i) { + S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; + } +} + +static void blake2b_compress_4r(blake2b_state* S, const uint8_t* block) { + uint64_t m[16]; + uint64_t v[16]; + unsigned int i; + + for (i = 0; i < 16; ++i) { + m[i] = load64(block + i * sizeof(m[i])); + } + + for (i = 0; i < 8; ++i) { + v[i] = S->h[i]; + } + + v[8] = blake2b_IV[0]; + v[9] = blake2b_IV[1]; + v[10] = blake2b_IV[2]; + v[11] = blake2b_IV[3]; + v[12] = blake2b_IV[4] ^ S->t[0]; + v[13] = blake2b_IV[5] ^ S->t[1]; + v[14] = blake2b_IV[6] ^ S->f[0]; + v[15] = blake2b_IV[7] ^ S->f[1]; + + ROUND(0); + ROUND(1); + ROUND(2); + ROUND(3); + + for (i = 0; i < 8; ++i) { + S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; + } +} + +int hashx_blake2b_update(blake2b_state* S, const void* in, size_t inlen) { + const uint8_t* pin = (const uint8_t*)in; + + if (inlen == 0) { + return 0; + } + + /* Sanity check */ + if (S == NULL || in == NULL) { + return -1; + } + + /* Is this a reused state? */ + if (S->f[0] != 0) { + return -1; + } + + if (S->buflen + inlen > BLAKE2B_BLOCKBYTES) { + /* Complete current block */ + size_t left = S->buflen; + size_t fill = BLAKE2B_BLOCKBYTES - left; + memcpy(&S->buf[left], pin, fill); + blake2b_increment_counter(S, BLAKE2B_BLOCKBYTES); + blake2b_compress(S, S->buf); + S->buflen = 0; + inlen -= fill; + pin += fill; + /* Avoid buffer copies when possible */ + while (inlen > BLAKE2B_BLOCKBYTES) { + blake2b_increment_counter(S, BLAKE2B_BLOCKBYTES); + blake2b_compress(S, pin); + inlen -= BLAKE2B_BLOCKBYTES; + pin += BLAKE2B_BLOCKBYTES; + } + } + memcpy(&S->buf[S->buflen], pin, inlen); + S->buflen += (unsigned int)inlen; + return 0; +} + +int hashx_blake2b_final(blake2b_state* S, void* out, size_t outlen) { + uint8_t buffer[BLAKE2B_OUTBYTES] = { 0 }; + unsigned int i; + + /* Sanity checks */ + if (S == NULL || out == NULL || outlen < S->outlen) { + return -1; + } + + /* Is this a reused state? */ + if (S->f[0] != 0) { + return -1; + } + + blake2b_increment_counter(S, S->buflen); + blake2b_set_lastblock(S); + memset(&S->buf[S->buflen], 0, BLAKE2B_BLOCKBYTES - S->buflen); /* Padding */ + blake2b_compress(S, S->buf); + + for (i = 0; i < 8; ++i) { /* Output full hash to temp buffer */ + store64(buffer + sizeof(S->h[i]) * i, S->h[i]); + } + + memcpy(out, buffer, S->outlen); + + return 0; +} + +/* 4-round version of Blake2b */ +void hashx_blake2b_4r(const blake2b_param* params, const void* in, + size_t inlen, void* out) { + + blake2b_state state; + const uint8_t* p = (const uint8_t*)params; + + blake2b_init0(&state); + /* IV XOR Parameter Block */ + for (unsigned i = 0; i < 8; ++i) { + state.h[i] ^= load64(&p[i * sizeof(state.h[i])]); + } + //state.outlen = blake_params.digest_length; + + const uint8_t* pin = (const uint8_t*)in; + + while (inlen > BLAKE2B_BLOCKBYTES) { + blake2b_increment_counter(&state, BLAKE2B_BLOCKBYTES); + blake2b_compress_4r(&state, pin); + inlen -= BLAKE2B_BLOCKBYTES; + pin += BLAKE2B_BLOCKBYTES; + } + + memcpy(state.buf, pin, inlen); + blake2b_increment_counter(&state, inlen); + blake2b_set_lastblock(&state); + blake2b_compress_4r(&state, state.buf); + + /* Output hash */ + memcpy(out, state.h, sizeof(state.h)); +} diff --git a/src/ext/equix/hashx/src/blake2.h b/src/ext/equix/hashx/src/blake2.h new file mode 100644 index 0000000000..ba535b21fb --- /dev/null +++ b/src/ext/equix/hashx/src/blake2.h @@ -0,0 +1,73 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +/* Original code from Argon2 reference source code package used under CC0 Licence + * https://github.com/P-H-C/phc-winner-argon2 + * Copyright 2015 + * Daniel Dinu, Dmitry Khovratovich, Jean-Philippe Aumasson, and Samuel Neves +*/ + +#ifndef PORTABLE_BLAKE2_H +#define PORTABLE_BLAKE2_H + +#include <stdint.h> +#include <limits.h> +#include <stddef.h> +#include <hashx.h> + +#if defined(__cplusplus) +extern "C" { +#endif + +enum blake2b_constant { + BLAKE2B_BLOCKBYTES = 128, + BLAKE2B_OUTBYTES = 64, + BLAKE2B_KEYBYTES = 64, + BLAKE2B_SALTBYTES = 16, + BLAKE2B_PERSONALBYTES = 16 +}; + +#pragma pack(push, 1) +typedef struct blake2b_param { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint64_t node_offset; /* 16 */ + uint8_t node_depth; /* 17 */ + uint8_t inner_length; /* 18 */ + uint8_t reserved[14]; /* 32 */ + uint8_t salt[BLAKE2B_SALTBYTES]; /* 48 */ + uint8_t personal[BLAKE2B_PERSONALBYTES]; /* 64 */ +} blake2b_param; +#pragma pack(pop) + +typedef struct blake2b_state { + uint64_t h[8]; + uint64_t t[2]; + uint64_t f[2]; + uint8_t buf[BLAKE2B_BLOCKBYTES]; + unsigned buflen; + unsigned outlen; + uint8_t last_node; +} blake2b_state; + +/* Ensure param structs have not been wrongly padded */ +/* Poor man's static_assert */ +enum { + blake2_size_check_0 = 1 / !!(CHAR_BIT == 8), + blake2_size_check_2 = + 1 / !!(sizeof(blake2b_param) == sizeof(uint64_t) * CHAR_BIT) +}; + +HASHX_PRIVATE int hashx_blake2b_init_param(blake2b_state* S, const blake2b_param* P); +HASHX_PRIVATE int hashx_blake2b_update(blake2b_state* S, const void* in, size_t inlen); +HASHX_PRIVATE int hashx_blake2b_final(blake2b_state* S, void* out, size_t outlen); +HASHX_PRIVATE void hashx_blake2b_4r(const blake2b_param* P, const void* in, size_t inlen, void* out); + +#if defined(__cplusplus) +} +#endif + +#endif diff --git a/src/ext/equix/hashx/src/compiler.c b/src/ext/equix/hashx/src/compiler.c new file mode 100644 index 0000000000..f180bf2d25 --- /dev/null +++ b/src/ext/equix/hashx/src/compiler.c @@ -0,0 +1,18 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include <stdbool.h> + +#include "compiler.h" +#include "virtual_memory.h" +#include "program.h" +#include "context.h" + +bool hashx_compiler_init(hashx_ctx* ctx) { + ctx->code = hashx_vm_alloc(COMP_CODE_SIZE); + return ctx->code != NULL; +} + +void hashx_compiler_destroy(hashx_ctx* ctx) { + hashx_vm_free(ctx->code, COMP_CODE_SIZE); +} diff --git a/src/ext/equix/hashx/src/compiler.h b/src/ext/equix/hashx/src/compiler.h new file mode 100644 index 0000000000..ef0201be02 --- /dev/null +++ b/src/ext/equix/hashx/src/compiler.h @@ -0,0 +1,41 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifndef COMPILER_H +#define COMPILER_H + +#include <stdint.h> +#include <stdbool.h> +#include <hashx.h> +#include "virtual_memory.h" +#include "program.h" + +HASHX_PRIVATE void hashx_compile_x86(const hashx_program* program, uint8_t* code); + +HASHX_PRIVATE void hashx_compile_a64(const hashx_program* program, uint8_t* code); + +#if defined(_M_X64) || defined(__x86_64__) +#define HASHX_COMPILER 1 +#define HASHX_COMPILER_X86 +#define hashx_compile hashx_compile_x86 +#elif defined(__aarch64__) +#define HASHX_COMPILER 1 +#define HASHX_COMPILER_A64 +#define hashx_compile hashx_compile_a64 +#else +#define HASHX_COMPILER 0 +#define hashx_compile +#endif + +HASHX_PRIVATE bool hashx_compiler_init(hashx_ctx* compiler); +HASHX_PRIVATE void hashx_compiler_destroy(hashx_ctx* compiler); + +#define COMP_PAGE_SIZE 4096 +#define COMP_RESERVE_SIZE 1024 +#define COMP_AVG_INSTR_SIZE 5 +#define COMP_CODE_SIZE \ + ALIGN_SIZE( \ + HASHX_PROGRAM_MAX_SIZE * COMP_AVG_INSTR_SIZE + COMP_RESERVE_SIZE, \ + COMP_PAGE_SIZE) + +#endif diff --git a/src/ext/equix/hashx/src/compiler_a64.c b/src/ext/equix/hashx/src/compiler_a64.c new file mode 100644 index 0000000000..48f743b988 --- /dev/null +++ b/src/ext/equix/hashx/src/compiler_a64.c @@ -0,0 +1,154 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include <string.h> +#include <assert.h> + +#include "compiler.h" +#include "program.h" +#include "virtual_memory.h" +#include "unreachable.h" + +#define EMIT(p,x) do { \ + memcpy(p, x, sizeof(x)); \ + p += sizeof(x); \ + } while (0) +#define EMIT_U32(p,x) *((uint32_t*)(p)) = x; p += sizeof(uint32_t) +#define EMIT_IMM32(p,x) \ + EMIT_U32(p, 0x9280000c | \ + ((x <= INT32_MAX) << 30) | \ + (((x <= INT32_MAX) ? (x & 0xFFFF) : (~x & 0xFFFF)) << 5)); \ + EMIT_U32(p, 0xf2a0000c | \ + (((x >> 16) & 0xFFFF) << 5)); + +#ifdef HASHX_COMPILER_A64 + +static const uint8_t a64_prologue[] = { + 0x07, 0x1c, 0x40, 0xf9, /* ldr x7, [x0, #56] */ + 0x06, 0x18, 0x40, 0xf9, /* ldr x6, [x0, #48] */ + 0x05, 0x14, 0x40, 0xf9, /* ldr x5, [x0, #40] */ + 0x04, 0x10, 0x40, 0xf9, /* ldr x4, [x0, #32] */ + 0x03, 0x0c, 0x40, 0xf9, /* ldr x3, [x0, #24] */ + 0x02, 0x08, 0x40, 0xf9, /* ldr x2, [x0, #16] */ + 0x01, 0x04, 0x40, 0xf9, /* ldr x1, [x0, #8] */ + 0xe8, 0x03, 0x00, 0xaa, /* mov x8, x0 */ + 0x00, 0x00, 0x40, 0xf9, /* ldr x0, [x0] */ + 0xe9, 0x03, 0x1f, 0x2a, /* mov w9, wzr */ +}; + +static const uint8_t a64_epilogue[] = { + 0x00, 0x01, 0x00, 0xf9, /* str x0, [x8] */ + 0x01, 0x05, 0x00, 0xf9, /* str x1, [x8, #8] */ + 0x02, 0x09, 0x00, 0xf9, /* str x2, [x8, #16] */ + 0x03, 0x0d, 0x00, 0xf9, /* str x3, [x8, #24] */ + 0x04, 0x11, 0x00, 0xf9, /* str x4, [x8, #32] */ + 0x05, 0x15, 0x00, 0xf9, /* str x5, [x8, #40] */ + 0x06, 0x19, 0x00, 0xf9, /* str x6, [x8, #48] */ + 0x07, 0x1d, 0x00, 0xf9, /* str x7, [x8, #56] */ + 0xc0, 0x03, 0x5f, 0xd6, /* ret */ +}; + +void hashx_compile_a64(const hashx_program* program, uint8_t* code) { + hashx_vm_rw(code, COMP_CODE_SIZE); + uint8_t* pos = code; + uint8_t* target = NULL; + int creg = -1; + EMIT(pos, a64_prologue); + for (int i = 0; i < program->code_size; ++i) { + const instruction* instr = &program->code[i]; + switch (instr->opcode) + { + case INSTR_UMULH_R: + EMIT_U32(pos, 0x9bc07c00 | + (instr->src << 16) | + (instr->dst << 5) | + (instr->dst)); + if (target != NULL) { + creg = instr->dst; + } + break; + case INSTR_SMULH_R: + EMIT_U32(pos, 0x9b407c00 | + (instr->src << 16) | + (instr->dst << 5) | + (instr->dst)); + if (target != NULL) { + creg = instr->dst; + } + break; + case INSTR_MUL_R: + assert(creg != instr->dst); + EMIT_U32(pos, 0x9b007c00 | + (instr->src << 16) | + (instr->dst << 5) | + (instr->dst)); + break; + case INSTR_SUB_R: + assert(creg != instr->dst); + EMIT_U32(pos, 0xcb000000 | + (instr->src << 16) | + (instr->dst << 5) | + (instr->dst)); + break; + case INSTR_XOR_R: + assert(creg != instr->dst); + EMIT_U32(pos, 0xca000000 | + (instr->src << 16) | + (instr->dst << 5) | + (instr->dst)); + break; + case INSTR_ADD_RS: + assert(creg != instr->dst); + EMIT_U32(pos, 0x8b000000 | + (instr->src << 16) | + (instr->imm32 << 10) | + (instr->dst << 5) | + (instr->dst)); + break; + case INSTR_ROR_C: + assert(creg != instr->dst); + EMIT_U32(pos, 0x93c00000 | + (instr->dst << 16) | + (instr->imm32 << 10) | + (instr->dst << 5) | + (instr->dst)); + break; + case INSTR_ADD_C: + assert(creg != instr->dst); + EMIT_IMM32(pos, instr->imm32); + EMIT_U32(pos, 0x8b0c0000 | + (instr->dst << 5) | + (instr->dst)); + break; + case INSTR_XOR_C: + assert(creg != instr->dst); + EMIT_IMM32(pos, instr->imm32); + EMIT_U32(pos, 0xca0c0000 | + (instr->dst << 5) | + (instr->dst)); + break; + case INSTR_TARGET: + target = pos; + break; + case INSTR_BRANCH: + EMIT_IMM32(pos, instr->imm32); + EMIT_U32(pos, 0x2a00012b | (creg << 16)); + EMIT_U32(pos, 0x6a0c017f); + EMIT_U32(pos, 0x5a891129); + EMIT_U32(pos, 0x54000000 | + ((((uint32_t)(target - pos)) >> 2) & 0x7FFFF) << 5); + target = NULL; + creg = -1; + break; + default: + UNREACHABLE; + } + } + EMIT(pos, a64_epilogue); + hashx_vm_rx(code, COMP_CODE_SIZE); +#ifdef __GNUC__ + __builtin___clear_cache(code, pos); +#endif +} + +#endif diff --git a/src/ext/equix/hashx/src/compiler_x86.c b/src/ext/equix/hashx/src/compiler_x86.c new file mode 100644 index 0000000000..0a1d9efd8f --- /dev/null +++ b/src/ext/equix/hashx/src/compiler_x86.c @@ -0,0 +1,151 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include <string.h> + +#include "compiler.h" +#include "program.h" +#include "virtual_memory.h" +#include "unreachable.h" + +#if defined(_WIN32) || defined(__CYGWIN__) +#define WINABI +#endif + +#define EMIT(p,x) do { \ + memcpy(p, x, sizeof(x)); \ + p += sizeof(x); \ + } while (0) +#define EMIT_BYTE(p,x) *((p)++) = x +#define EMIT_U16(p,x) *((uint16_t*)(p)) = x; p += sizeof(uint16_t) +#define EMIT_U32(p,x) *((uint32_t*)(p)) = x; p += sizeof(uint32_t) +#define EMIT_U64(p,x) *((uint64_t*)(p)) = x; p += sizeof(uint64_t) + +#define GEN_SIB(scale, index, base) ((scale << 6) | (index << 3) | base) + +#ifdef HASHX_COMPILER_X86 + +static const uint8_t x86_prologue[] = { +#ifndef WINABI + 0x48, 0x89, 0xF9, /* mov rcx, rdi */ + 0x48, 0x83, 0xEC, 0x20, /* sub rsp, 32 */ + 0x4C, 0x89, 0x24, 0x24, /* mov qword ptr [rsp+0], r12 */ + 0x4C, 0x89, 0x6C, 0x24, 0x08, /* mov qword ptr [rsp+8], r13 */ + 0x4C, 0x89, 0x74, 0x24, 0x10, /* mov qword ptr [rsp+16], r14 */ + 0x4C, 0x89, 0x7C, 0x24, 0x18, /* mov qword ptr [rsp+24], r15 */ +#else + 0x4C, 0x89, 0x64, 0x24, 0x08, /* mov qword ptr [rsp+8], r12 */ + 0x4C, 0x89, 0x6C, 0x24, 0x10, /* mov qword ptr [rsp+16], r13 */ + 0x4C, 0x89, 0x74, 0x24, 0x18, /* mov qword ptr [rsp+24], r14 */ + 0x4C, 0x89, 0x7C, 0x24, 0x20, /* mov qword ptr [rsp+32], r15 */ + 0x48, 0x83, 0xEC, 0x10, /* sub rsp, 16 */ + 0x48, 0x89, 0x34, 0x24, /* mov qword ptr [rsp+0], rsi */ + 0x48, 0x89, 0x7C, 0x24, 0x08, /* mov qword ptr [rsp+8], rdi */ +#endif + 0x31, 0xF6, /* xor esi, esi */ + 0x8D, 0x7E, 0xFF, /* lea edi, [rsi-1] */ + 0x4C, 0x8B, 0x01, /* mov r8, qword ptr [rcx+0] */ + 0x4C, 0x8B, 0x49, 0x08, /* mov r9, qword ptr [rcx+8] */ + 0x4C, 0x8B, 0x51, 0x10, /* mov r10, qword ptr [rcx+16] */ + 0x4C, 0x8B, 0x59, 0x18, /* mov r11, qword ptr [rcx+24] */ + 0x4C, 0x8B, 0x61, 0x20, /* mov r12, qword ptr [rcx+32] */ + 0x4C, 0x8B, 0x69, 0x28, /* mov r13, qword ptr [rcx+40] */ + 0x4C, 0x8B, 0x71, 0x30, /* mov r14, qword ptr [rcx+48] */ + 0x4C, 0x8B, 0x79, 0x38 /* mov r15, qword ptr [rcx+56] */ +}; + +static const uint8_t x86_epilogue[] = { + 0x4C, 0x89, 0x01, /* mov qword ptr [rcx+0], r8 */ + 0x4C, 0x89, 0x49, 0x08, /* mov qword ptr [rcx+8], r9 */ + 0x4C, 0x89, 0x51, 0x10, /* mov qword ptr [rcx+16], r10 */ + 0x4C, 0x89, 0x59, 0x18, /* mov qword ptr [rcx+24], r11 */ + 0x4C, 0x89, 0x61, 0x20, /* mov qword ptr [rcx+32], r12 */ + 0x4C, 0x89, 0x69, 0x28, /* mov qword ptr [rcx+40], r13 */ + 0x4C, 0x89, 0x71, 0x30, /* mov qword ptr [rcx+48], r14 */ + 0x4C, 0x89, 0x79, 0x38, /* mov qword ptr [rcx+56], r15 */ +#ifndef WINABI + 0x4C, 0x8B, 0x24, 0x24, /* mov r12, qword ptr [rsp+0] */ + 0x4C, 0x8B, 0x6C, 0x24, 0x08, /* mov r13, qword ptr [rsp+8] */ + 0x4C, 0x8B, 0x74, 0x24, 0x10, /* mov r14, qword ptr [rsp+16] */ + 0x4C, 0x8B, 0x7C, 0x24, 0x18, /* mov r15, qword ptr [rsp+24] */ + 0x48, 0x83, 0xC4, 0x20, /* add rsp, 32 */ +#else + 0x48, 0x8B, 0x34, 0x24, /* mov rsi, qword ptr [rsp+0] */ + 0x48, 0x8B, 0x7C, 0x24, 0x08, /* mov rdi, qword ptr [rsp+8] */ + 0x48, 0x83, 0xC4, 0x10, /* add rsp, 16 */ + 0x4C, 0x8B, 0x64, 0x24, 0x08, /* mov r12, qword ptr [rsp+8] */ + 0x4C, 0x8B, 0x6C, 0x24, 0x10, /* mov r13, qword ptr [rsp+16] */ + 0x4C, 0x8B, 0x74, 0x24, 0x18, /* mov r14, qword ptr [rsp+24] */ + 0x4C, 0x8B, 0x7C, 0x24, 0x20, /* mov r15, qword ptr [rsp+32] */ +#endif + 0xC3 /* ret */ +}; + +void hashx_compile_x86(const hashx_program* program, uint8_t* code) { + hashx_vm_rw(code, COMP_CODE_SIZE); + uint8_t* pos = code; + uint8_t* target = NULL; + EMIT(pos, x86_prologue); + for (int i = 0; i < program->code_size; ++i) { + const instruction* instr = &program->code[i]; + switch (instr->opcode) + { + case INSTR_UMULH_R: + EMIT_U64(pos, 0x8b4ce0f749c08b49 | + (((uint64_t)instr->src) << 40) | + (((uint64_t)instr->dst) << 16)); + EMIT_BYTE(pos, 0xc2 + 8 * instr->dst); + break; + case INSTR_SMULH_R: + EMIT_U64(pos, 0x8b4ce8f749c08b49 | + (((uint64_t)instr->src) << 40) | + (((uint64_t)instr->dst) << 16)); + EMIT_BYTE(pos, 0xc2 + 8 * instr->dst); + break; + case INSTR_MUL_R: + EMIT_U32(pos, 0xc0af0f4d | (instr->dst << 27) | (instr->src << 24)); + break; + case INSTR_SUB_R: + EMIT_U16(pos, 0x2b4d); + EMIT_BYTE(pos, 0xc0 | (instr->dst << 3) | instr->src); + break; + case INSTR_XOR_R: + EMIT_U16(pos, 0x334d); + EMIT_BYTE(pos, 0xc0 | (instr->dst << 3) | instr->src); + break; + case INSTR_ADD_RS: + EMIT_U32(pos, 0x00048d4f | + (instr->dst << 19) | + GEN_SIB(instr->imm32, instr->src, instr->dst) << 24); + break; + case INSTR_ROR_C: + EMIT_U32(pos, 0x00c8c149 | (instr->dst << 16) | (instr->imm32 << 24)); + break; + case INSTR_ADD_C: + EMIT_U16(pos, 0x8149); + EMIT_BYTE(pos, 0xc0 | instr->dst); + EMIT_U32(pos, instr->imm32); + break; + case INSTR_XOR_C: + EMIT_U16(pos, 0x8149); + EMIT_BYTE(pos, 0xf0 | instr->dst); + EMIT_U32(pos, instr->imm32); + break; + case INSTR_TARGET: + target = pos; /* +2 */ + EMIT_U32(pos, 0x440fff85); + EMIT_BYTE(pos, 0xf7); + break; + case INSTR_BRANCH: + EMIT_U64(pos, ((uint64_t)instr->imm32) << 32 | 0xc2f7f209); + EMIT_U16(pos, ((target - pos) << 8) | 0x74); + break; + default: + UNREACHABLE; + } + } + EMIT(pos, x86_epilogue); + hashx_vm_rx(code, COMP_CODE_SIZE); +} + +#endif diff --git a/src/ext/equix/hashx/src/context.c b/src/ext/equix/hashx/src/context.c new file mode 100644 index 0000000000..de2144d46c --- /dev/null +++ b/src/ext/equix/hashx/src/context.c @@ -0,0 +1,81 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include <stdlib.h> +#include <string.h> + +#include <hashx.h> +#include "context.h" +#include "compiler.h" +#include "program.h" + +#define STRINGIZE_INNER(x) #x +#define STRINGIZE(x) STRINGIZE_INNER(x) + +/* Salt used when generating hash functions. Useful for domain separation. */ +#ifndef HASHX_SALT +#define HASHX_SALT HashX v1 +#endif + +/* Blake2b params used to generate program keys */ +const blake2b_param hashx_blake2_params = { + .digest_length = 64, + .key_length = 0, + .fanout = 1, + .depth = 1, + .leaf_length = 0, + .node_offset = 0, + .node_depth = 0, + .inner_length = 0, + .reserved = { 0 }, + .salt = STRINGIZE(HASHX_SALT), + .personal = { 0 } +}; + +hashx_ctx* hashx_alloc(hashx_type type) { + if (!HASHX_COMPILER && (type & HASHX_COMPILED)) { + return HASHX_NOTSUPP; + } + hashx_ctx* ctx = malloc(sizeof(hashx_ctx)); + if (ctx == NULL) { + goto failure; + } + ctx->code = NULL; + if (type & HASHX_COMPILED) { + if (!hashx_compiler_init(ctx)) { + goto failure; + } + ctx->type = HASHX_COMPILED; + } + else { + ctx->program = malloc(sizeof(hashx_program)); + if (ctx->program == NULL) { + goto failure; + } + ctx->type = HASHX_INTERPRETED; + } +#ifdef HASHX_BLOCK_MODE + memcpy(&ctx->params, &hashx_blake2_params, 32); +#endif +#ifndef NDEBUG + ctx->has_program = false; +#endif + return ctx; +failure: + hashx_free(ctx); + return NULL; +} + +void hashx_free(hashx_ctx* ctx) { + if (ctx != NULL && ctx != HASHX_NOTSUPP) { + if (ctx->code != NULL) { + if (ctx->type & HASHX_COMPILED) { + hashx_compiler_destroy(ctx); + } + else { + free(ctx->program); + } + } + free(ctx); + } +} diff --git a/src/ext/equix/hashx/src/context.h b/src/ext/equix/hashx/src/context.h new file mode 100644 index 0000000000..40736397f8 --- /dev/null +++ b/src/ext/equix/hashx/src/context.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifndef CONTEXT_H +#define CONTEXT_H + +#include <stdbool.h> + +#include "hashx.h" +#include "blake2.h" +#include "siphash.h" + +typedef void program_func(uint64_t r[8]); + +#ifdef __cplusplus +extern "C" { +#endif + +HASHX_PRIVATE extern const blake2b_param hashx_blake2_params; + +#ifdef __cplusplus +} +#endif + +typedef struct hashx_program hashx_program; + +/* HashX context. */ +typedef struct hashx_ctx { + union { + uint8_t* code; + program_func* func; + hashx_program* program; + }; + hashx_type type; +#ifndef HASHX_BLOCK_MODE + siphash_state keys; +#else + blake2b_param params; +#endif +#ifndef NDEBUG + bool has_program; +#endif +} hashx_ctx; + +#endif diff --git a/src/ext/equix/hashx/src/force_inline.h b/src/ext/equix/hashx/src/force_inline.h new file mode 100644 index 0000000000..d9af3f32e8 --- /dev/null +++ b/src/ext/equix/hashx/src/force_inline.h @@ -0,0 +1,9 @@ +#ifndef FORCE_INLINE +#if defined(_MSC_VER) +#define FORCE_INLINE __inline +#elif defined(__GNUC__) || defined(__clang__) +#define FORCE_INLINE __inline__ +#else +#define FORCE_INLINE +#endif +#endif \ No newline at end of file diff --git a/src/ext/equix/hashx/src/hashx.c b/src/ext/equix/hashx/src/hashx.c new file mode 100644 index 0000000000..1f5715dce8 --- /dev/null +++ b/src/ext/equix/hashx/src/hashx.c @@ -0,0 +1,134 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#include <hashx.h> +#include "blake2.h" +#include "hashx_endian.h" +#include "program.h" +#include "context.h" +#include "compiler.h" + +#if HASHX_SIZE > 32 +#error HASHX_SIZE cannot be more than 32 +#endif + +#ifndef HASHX_BLOCK_MODE +#define HASHX_INPUT_ARGS input +#else +#define HASHX_INPUT_ARGS input, size +#endif + +static int initialize_program(hashx_ctx* ctx, hashx_program* program, + siphash_state keys[2]) { + + if (!hashx_program_generate(&keys[0], program)) { + return 0; + } +#ifndef HASHX_BLOCK_MODE + memcpy(&ctx->keys, &keys[1], 32); +#else + memcpy(&ctx->params.salt, &keys[1], 32); +#endif +#ifndef NDEBUG + ctx->has_program = true; +#endif + return 1; +} + +int hashx_make(hashx_ctx* ctx, const void* seed, size_t size) { + assert(ctx != NULL && ctx != HASHX_NOTSUPP); + assert(seed != NULL || size == 0); + siphash_state keys[2]; + blake2b_state hash_state; + hashx_blake2b_init_param(&hash_state, &hashx_blake2_params); + hashx_blake2b_update(&hash_state, seed, size); + hashx_blake2b_final(&hash_state, &keys, sizeof(keys)); + if (ctx->type & HASHX_COMPILED) { + hashx_program program; + if (!initialize_program(ctx, &program, keys)) { + return 0; + } + hashx_compile(&program, ctx->code); + return 1; + } + return initialize_program(ctx, ctx->program, keys); +} + +void hashx_exec(const hashx_ctx* ctx, HASHX_INPUT, void* output) { + assert(ctx != NULL && ctx != HASHX_NOTSUPP); + assert(output != NULL); + assert(ctx->has_program); + uint64_t r[8]; +#ifndef HASHX_BLOCK_MODE + hashx_siphash24_ctr_state512(&ctx->keys, input, r); +#else + hashx_blake2b_4r(&ctx->params, input, size, r); +#endif + + if (ctx->type & HASHX_COMPILED) { + ctx->func(r); + } + else { + hashx_program_execute(ctx->program, r); + } + + /* Hash finalization to remove bias toward 0 caused by multiplications */ +#ifndef HASHX_BLOCK_MODE + r[0] += ctx->keys.v0; + r[1] += ctx->keys.v1; + r[6] += ctx->keys.v2; + r[7] += ctx->keys.v3; +#else + const uint8_t* p = (const uint8_t*)&ctx->params; + r[0] ^= load64(&p[8 * 0]); + r[1] ^= load64(&p[8 * 1]); + r[2] ^= load64(&p[8 * 2]); + r[3] ^= load64(&p[8 * 3]); + r[4] ^= load64(&p[8 * 4]); + r[5] ^= load64(&p[8 * 5]); + r[6] ^= load64(&p[8 * 6]); + r[7] ^= load64(&p[8 * 7]); +#endif + /* 1 SipRound per 4 registers is enough to pass SMHasher. */ + SIPROUND(r[0], r[1], r[2], r[3]); + SIPROUND(r[4], r[5], r[6], r[7]); + + /* output */ +#if HASHX_SIZE > 0 + /* optimized output for hash sizes that are multiples of 8 */ +#if HASHX_SIZE % 8 == 0 + uint8_t* temp_out = (uint8_t*)output; +#if HASHX_SIZE >= 8 + store64(temp_out + 0, r[0] ^ r[4]); +#endif +#if HASHX_SIZE >= 16 + store64(temp_out + 8, r[1] ^ r[5]); +#endif +#if HASHX_SIZE >= 24 + store64(temp_out + 16, r[2] ^ r[6]); +#endif +#if HASHX_SIZE >= 32 + store64(temp_out + 24, r[3] ^ r[7]); +#endif +#else /* any output size */ + uint8_t temp_out[32]; +#if HASHX_SIZE > 0 + store64(temp_out + 0, r[0] ^ r[4]); +#endif +#if HASHX_SIZE > 8 + store64(temp_out + 8, r[1] ^ r[5]); +#endif +#if HASHX_SIZE > 16 + store64(temp_out + 16, r[2] ^ r[6]); +#endif +#if HASHX_SIZE > 24 + store64(temp_out + 24, r[3] ^ r[7]); +#endif + memcpy(output, temp_out, HASHX_SIZE); +#endif +#endif +} diff --git a/src/ext/equix/hashx/src/hashx_endian.h b/src/ext/equix/hashx/src/hashx_endian.h new file mode 100644 index 0000000000..23537cfba5 --- /dev/null +++ b/src/ext/equix/hashx/src/hashx_endian.h @@ -0,0 +1,103 @@ +#ifndef ENDIAN_H +#define ENDIAN_H + +#include <stdint.h> +#include <string.h> +#include "force_inline.h" + +/* Argon2 Team - Begin Code */ +/* + Not an exhaustive list, but should cover the majority of modern platforms + Additionally, the code will always be correct---this is only a performance + tweak. +*/ +#if (defined(__BYTE_ORDER__) && \ + (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) || \ + defined(__LITTLE_ENDIAN__) || defined(__ARMEL__) || defined(__MIPSEL__) || \ + defined(__AARCH64EL__) || defined(__amd64__) || defined(__i386__) || \ + defined(_M_IX86) || defined(_M_X64) || defined(_M_AMD64) || \ + defined(_M_ARM) +#define NATIVE_LITTLE_ENDIAN +#endif +/* Argon2 Team - End Code */ + +static FORCE_INLINE uint32_t load32(const void* src) { +#if defined(NATIVE_LITTLE_ENDIAN) + uint32_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t* p = (const uint8_t*)src; + uint32_t w = *p++; + w |= (uint32_t)(*p++) << 8; + w |= (uint32_t)(*p++) << 16; + w |= (uint32_t)(*p++) << 24; + return w; +#endif +} + +static FORCE_INLINE uint64_t load64_native(const void* src) { + uint64_t w; + memcpy(&w, src, sizeof w); + return w; +} + +static FORCE_INLINE uint64_t load64(const void* src) { +#if defined(NATIVE_LITTLE_ENDIAN) + return load64_native(src); +#else + const uint8_t* p = (const uint8_t*)src; + uint64_t w = *p++; + w |= (uint64_t)(*p++) << 8; + w |= (uint64_t)(*p++) << 16; + w |= (uint64_t)(*p++) << 24; + w |= (uint64_t)(*p++) << 32; + w |= (uint64_t)(*p++) << 40; + w |= (uint64_t)(*p++) << 48; + w |= (uint64_t)(*p++) << 56; + return w; +#endif +} + +static FORCE_INLINE void store32(void* dst, uint32_t w) { +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t* p = (uint8_t*)dst; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; +#endif +} + +static FORCE_INLINE void store64_native(void* dst, uint64_t w) { + memcpy(dst, &w, sizeof w); +} + +static FORCE_INLINE void store64(void* dst, uint64_t w) { +#if defined(NATIVE_LITTLE_ENDIAN) + store64_native(dst, w); +#else + uint8_t* p = (uint8_t*)dst; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; +#endif +} +#endif diff --git a/src/ext/equix/hashx/src/hashx_thread.c b/src/ext/equix/hashx/src/hashx_thread.c new file mode 100644 index 0000000000..6618378e66 --- /dev/null +++ b/src/ext/equix/hashx/src/hashx_thread.c @@ -0,0 +1,27 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include "hashx_thread.h" + +hashx_thread hashx_thread_create(hashx_thread_func* func, void* args) { +#ifdef HASHX_WIN + return CreateThread(NULL, 0, func, args, 0, NULL); +#else + hashx_thread thread; + if (pthread_create(&thread, NULL, func, args) != 0) + { + thread = 0; + } + return thread; +#endif +} + +void hashx_thread_join(hashx_thread thread) { +#ifdef HASHX_WIN + WaitForSingleObject(thread, INFINITE); + CloseHandle(thread); +#else + void* retval; + pthread_join(thread, &retval); +#endif +} diff --git a/src/ext/equix/hashx/src/hashx_thread.h b/src/ext/equix/hashx/src/hashx_thread.h new file mode 100644 index 0000000000..01556626d8 --- /dev/null +++ b/src/ext/equix/hashx/src/hashx_thread.h @@ -0,0 +1,27 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifndef HASHX_THREAD_H +#define HASHX_THREAD_H + +#include <hashx.h> + +#ifdef HASHX_WIN +#include <Windows.h> +typedef HANDLE hashx_thread; +typedef DWORD hashx_thread_retval; +#define HASHX_THREAD_SUCCESS 0 +#else +#include <pthread.h> +typedef pthread_t hashx_thread; +typedef void* hashx_thread_retval; +#define HASHX_THREAD_SUCCESS NULL +#endif + +typedef hashx_thread_retval hashx_thread_func(void* args); + +hashx_thread hashx_thread_create(hashx_thread_func* func, void* args); + +void hashx_thread_join(hashx_thread thread); + +#endif diff --git a/src/ext/equix/hashx/src/hashx_time.c b/src/ext/equix/hashx/src/hashx_time.c new file mode 100644 index 0000000000..d3eaaed10e --- /dev/null +++ b/src/ext/equix/hashx/src/hashx_time.c @@ -0,0 +1,35 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include "hashx_time.h" +#include <hashx.h> + +#if defined(HASHX_WIN) +#include <windows.h> +#else +#include <sys/time.h> +#endif + +double hashx_time() { +#ifdef HASHX_WIN + static double freq = 0; + if (freq == 0) { + LARGE_INTEGER freq_long; + if (!QueryPerformanceFrequency(&freq_long)) { + return 0; + } + freq = freq_long.QuadPart; + } + LARGE_INTEGER time; + if (!QueryPerformanceCounter(&time)) { + return 0; + } + return time.QuadPart / freq; +#else + struct timeval time; + if (gettimeofday(&time, NULL) != 0) { + return 0; + } + return (double)time.tv_sec + (double)time.tv_usec * 1.0e-6; +#endif +} diff --git a/src/ext/equix/hashx/src/hashx_time.h b/src/ext/equix/hashx/src/hashx_time.h new file mode 100644 index 0000000000..da1ca746fc --- /dev/null +++ b/src/ext/equix/hashx/src/hashx_time.h @@ -0,0 +1,9 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifndef HASHX_TIME_H +#define HASHX_TIME_H + +double hashx_time(void); + +#endif diff --git a/src/ext/equix/hashx/src/instruction.h b/src/ext/equix/hashx/src/instruction.h new file mode 100644 index 0000000000..f17582ffea --- /dev/null +++ b/src/ext/equix/hashx/src/instruction.h @@ -0,0 +1,31 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifndef INSTRUCTION_H +#define INSTRUCTION_H + +#include <stdint.h> + +typedef enum instr_type { + INSTR_UMULH_R, /* unsigned high multiplication by a register */ + INSTR_SMULH_R, /* signed high multiplication by a register */ + INSTR_MUL_R, /* multiplication by a register */ + INSTR_SUB_R, /* subtraction of a register */ + INSTR_XOR_R, /* xor with a register */ + INSTR_ADD_RS, /* addition of a shifted register */ + INSTR_ROR_C, /* rotation by a constant */ + INSTR_ADD_C, /* addition of a constant */ + INSTR_XOR_C, /* xor with a constant */ + INSTR_TARGET, /* branch instruction target */ + INSTR_BRANCH, /* conditional branch */ +} instr_type; + +typedef struct instruction { + instr_type opcode; + int src; + int dst; + uint32_t imm32; + uint32_t op_par; +} instruction; + +#endif diff --git a/src/ext/equix/hashx/src/program.c b/src/ext/equix/hashx/src/program.c new file mode 100644 index 0000000000..7f4b0cef0a --- /dev/null +++ b/src/ext/equix/hashx/src/program.c @@ -0,0 +1,771 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "program.h" +#include "unreachable.h" +#include "siphash_rng.h" + +/* instructions are generated until this CPU cycle */ +#define TARGET_CYCLE 192 + +/* requirements for the program to be acceptable */ +#define REQUIREMENT_SIZE 512 +#define REQUIREMENT_MUL_COUNT 192 +#define REQUIREMENT_LATENCY 195 + +/* R5 (x86 = r13) register cannot be used as the destination of INSTR_ADD_RS */ +#define REGISTER_NEEDS_DISPLACEMENT 5 + +#define PORT_MAP_SIZE (TARGET_CYCLE + 4) +#define NUM_PORTS 3 +#define MAX_RETRIES 1 +#define LOG2_BRANCH_PROB 4 +#define BRANCH_MASK 0x80000000 + +#define TRACE false +#define TRACE_PRINT(...) do { if (TRACE) printf(__VA_ARGS__); } while (false) + +#define MAX(a,b) ((a) > (b) ? (a) : (b)) + +/* If the instruction is a multiplication. */ +static inline bool is_mul(instr_type type) { + return type <= INSTR_MUL_R; +} + +/* If the instruction is a 64x64->128 bit multiplication. */ +static inline bool is_wide_mul(instr_type type) { + return type < INSTR_MUL_R; +} + +/* Ivy Bridge integer execution ports: P0, P1, P5 */ +typedef enum execution_port { + PORT_NONE = 0, + PORT_P0 = 1, + PORT_P1 = 2, + PORT_P5 = 4, + PORT_P01 = PORT_P0 | PORT_P1, + PORT_P05 = PORT_P0 | PORT_P5, + PORT_P015 = PORT_P0 | PORT_P1 | PORT_P5 +} execution_port; + +static const char* execution_port_names[] = { + "PORT_NONE", "PORT_P0", "PORT_P1", "PORT_P01", "PORT_P5", "PORT_P05", "PORT_P15", "PORT_P015" +}; + +typedef struct instr_template { + instr_type type; /* instruction type */ + const char* x86_asm; /* x86 assembly */ + int x86_size; /* x86 code size */ + int latency; /* latency in cycles */ + execution_port uop1; /* ex. ports for the 1st uop */ + execution_port uop2; /* ex. ports for the 2nd uop */ + uint32_t immediate_mask; /* mask for imm32 */ + instr_type group; /* instruction group */ + bool imm_can_be_0; /* if imm32 can be zero */ + bool distinct_dst; /* if dst and src must be distinct */ + bool op_par_src; /* operation parameter is equal to src */ + bool has_src; /* if the instruction has a source operand */ + bool has_dst; /* if the instr. has a destination operand */ +} instr_template; + +typedef struct register_info { + int latency; /* cycle when the register value will be ready */ + instr_type last_op; /* last op applied to the register */ + int last_op_par; /* parameter of the last op (-1 = constant) */ +} register_info; + +typedef struct program_item { + const instr_template** templates; + uint32_t mask0; + uint32_t mask1; + bool duplicates; +} program_item; + +typedef struct generator_ctx { + int cycle; + int sub_cycle; + int mul_count; + bool chain_mul; + int latency; + siphash_rng gen; + register_info registers[8]; + execution_port ports[PORT_MAP_SIZE][NUM_PORTS]; +} generator_ctx; + +const static instr_template tpl_umulh_r = { + .type = INSTR_UMULH_R, + .x86_asm = "mul r", + .x86_size = 9, /* mov, mul, mov */ + .latency = 4, + .uop1 = PORT_P1, + .uop2 = PORT_P5, + .immediate_mask = 0, + .group = INSTR_UMULH_R, + .imm_can_be_0 = false, + .distinct_dst = false, + .op_par_src = false, + .has_src = true, + .has_dst = true, +}; + +const static instr_template tpl_smulh_r = { + .type = INSTR_SMULH_R, + .x86_asm = "imul r", + .x86_size = 9, /* mov, mul, mov */ + .latency = 4, + .uop1 = PORT_P1, + .uop2 = PORT_P5, + .immediate_mask = 0, + .group = INSTR_SMULH_R, + .imm_can_be_0 = false, + .distinct_dst = false, + .op_par_src = false, + .has_src = true, + .has_dst = true, +}; + +const static instr_template tpl_mul_r = { + .type = INSTR_MUL_R, + .x86_asm = "imul r,r", + .x86_size = 4, + .latency = 3, + .uop1 = PORT_P1, + .uop2 = PORT_NONE, + .immediate_mask = 0, + .group = INSTR_MUL_R, + .imm_can_be_0 = false, + .distinct_dst = true, + .op_par_src = true, + .has_src = true, + .has_dst = true, +}; + +const static instr_template tpl_sub_r = { + .type = INSTR_SUB_R, + .x86_asm = "sub r,r", + .x86_size = 3, + .latency = 1, + .uop1 = PORT_P015, + .uop2 = PORT_NONE, + .immediate_mask = 0, + .group = INSTR_ADD_RS, + .imm_can_be_0 = false, + .distinct_dst = true, + .op_par_src = true, + .has_src = true, + .has_dst = true, +}; + +const static instr_template tpl_xor_r = { + .type = INSTR_XOR_R, + .x86_asm = "xor r,r", + .x86_size = 3, + .latency = 1, + .uop1 = PORT_P015, + .uop2 = PORT_NONE, + .immediate_mask = 0, + .group = INSTR_XOR_R, + .imm_can_be_0 = false, + .distinct_dst = true, + .op_par_src = true, + .has_src = true, + .has_dst = true, +}; + +const static instr_template tpl_add_rs = { + .type = INSTR_ADD_RS, + .x86_asm = "lea r,r+r*s", + .x86_size = 4, + .latency = 1, + .uop1 = PORT_P01, + .uop2 = PORT_NONE, + .immediate_mask = 3, + .group = INSTR_ADD_RS, + .imm_can_be_0 = true, + .distinct_dst = true, + .op_par_src = true, + .has_src = true, + .has_dst = true, +}; + +const static instr_template tpl_ror_c = { + .type = INSTR_ROR_C, + .x86_asm = "ror r,i", + .x86_size = 4, + .latency = 1, + .uop1 = PORT_P05, + .uop2 = PORT_NONE, + .immediate_mask = 63, + .group = INSTR_ROR_C, + .imm_can_be_0 = false, + .distinct_dst = true, + .op_par_src = false, + .has_src = false, + .has_dst = true, +}; + +const static instr_template tpl_add_c = { + .type = INSTR_ADD_C, + .x86_asm = "add r,i", + .x86_size = 7, + .latency = 1, + .uop1 = PORT_P015, + .uop2 = PORT_NONE, + .immediate_mask = UINT32_MAX, + .group = INSTR_ADD_C, + .imm_can_be_0 = false, + .distinct_dst = true, + .op_par_src = false, + .has_src = false, + .has_dst = true, +}; + +const static instr_template tpl_xor_c = { + .type = INSTR_XOR_C, + .x86_asm = "xor r,i", + .x86_size = 7, + .latency = 1, + .uop1 = PORT_P015, + .uop2 = PORT_NONE, + .immediate_mask = UINT32_MAX, + .group = INSTR_XOR_C, + .imm_can_be_0 = false, + .distinct_dst = true, + .op_par_src = false, + .has_src = false, + .has_dst = true, +}; + + +const static instr_template tpl_target = { + .type = INSTR_TARGET, + .x86_asm = "cmovz esi, edi", + .x86_size = 5, /* test, cmovz */ + .latency = 1, + .uop1 = PORT_P015, + .uop2 = PORT_P015, + .immediate_mask = 0, + .group = INSTR_TARGET, + .imm_can_be_0 = false, + .distinct_dst = true, + .op_par_src = false, + .has_src = false, + .has_dst = false, +}; + +const static instr_template tpl_branch = { + .type = INSTR_BRANCH, + .x86_asm = "jz target", + .x86_size = 10, /* or, test, jz */ + .latency = 1, + .uop1 = PORT_P015, + .uop2 = PORT_P015, + .immediate_mask = BRANCH_MASK, + .group = INSTR_BRANCH, + .imm_can_be_0 = false, + .distinct_dst = true, + .op_par_src = false, + .has_src = false, + .has_dst = false, +}; + +const static instr_template* instr_lookup[] = { + &tpl_ror_c, + &tpl_xor_c, + &tpl_add_c, + &tpl_add_c, + &tpl_sub_r, + &tpl_xor_r, + &tpl_xor_c, + &tpl_add_rs, +}; + +const static instr_template* wide_mul_lookup[] = { + &tpl_smulh_r, + &tpl_umulh_r +}; + +const static instr_template* mul_lookup = &tpl_mul_r; +const static instr_template* target_lookup = &tpl_target; +const static instr_template* branch_lookup = &tpl_branch; + +const static program_item item_mul = { + .templates = &mul_lookup, + .mask0 = 0, + .mask1 = 0, + .duplicates = true +}; + +const static program_item item_target = { + .templates = &target_lookup, + .mask0 = 0, + .mask1 = 0, + .duplicates = true +}; + +const static program_item item_branch = { + .templates = &branch_lookup, + .mask0 = 0, + .mask1 = 0, + .duplicates = true +}; + +const static program_item item_wide_mul = { + .templates = wide_mul_lookup, + .mask0 = 1, + .mask1 = 1, + .duplicates = true +}; + +const static program_item item_any = { + .templates = instr_lookup, + .mask0 = 7, + .mask1 = 3, /* instructions that don't need a src register */ + .duplicates = false +}; + +const static program_item* program_layout[] = { + &item_mul, + &item_target, + &item_any, + &item_mul, + &item_any, + &item_any, + &item_mul, + &item_any, + &item_any, + &item_mul, + &item_any, + &item_any, + &item_wide_mul, + &item_any, + &item_any, + &item_mul, + &item_any, + &item_any, + &item_mul, + &item_branch, + &item_any, + &item_mul, + &item_any, + &item_any, + &item_wide_mul, + &item_any, + &item_any, + &item_mul, + &item_any, + &item_any, + &item_mul, + &item_any, + &item_any, + &item_mul, + &item_any, + &item_any, +}; + +static const instr_template* select_template(generator_ctx* ctx, instr_type last_instr, int attempt) { + const program_item* item = program_layout[ctx->sub_cycle % 36]; + const instr_template* tpl; + do { + int index = item->mask0 ? hashx_siphash_rng_u8(&ctx->gen) & (attempt > 0 ? item->mask1 : item->mask0) : 0; + tpl = item->templates[index]; + } while (!item->duplicates && tpl->group == last_instr); + return tpl; +} + +static uint32_t branch_mask(siphash_rng* gen) { + uint32_t mask = 0; + int popcnt = 0; + while (popcnt < LOG2_BRANCH_PROB) { + int bit = hashx_siphash_rng_u8(gen) % 32; + uint32_t bitmask = 1U << bit; + if (!(mask & bitmask)) { + mask |= bitmask; + popcnt++; + } + } + return mask; +} + +static void instr_from_template(const instr_template* tpl, siphash_rng* gen, instruction* instr) { + instr->opcode = tpl->type; + if (tpl->immediate_mask) { + if (tpl->immediate_mask == BRANCH_MASK) { + instr->imm32 = branch_mask(gen); + } + else do { + instr->imm32 = hashx_siphash_rng_u32(gen) & tpl->immediate_mask; + } while (instr->imm32 == 0 && !tpl->imm_can_be_0); + } + if (!tpl->op_par_src) { + if (tpl->distinct_dst) { + instr->op_par = UINT32_MAX; + } + else { + instr->op_par = hashx_siphash_rng_u32(gen); + } + } + if (!tpl->has_src) { + instr->src = -1; + } + if (!tpl->has_dst) { + instr->dst = -1; + } +} + +static bool select_register(int available_regs[8], int regs_count, siphash_rng* gen, int* reg_out) { + if (regs_count == 0) + return false; + + int index; + + if (regs_count > 1) { + index = hashx_siphash_rng_u32(gen) % regs_count; + } + else { + index = 0; + } + *reg_out = available_regs[index]; + return true; +} + +static bool select_destination(const instr_template* tpl, instruction* instr, generator_ctx* ctx, int cycle) { + int available_regs[8]; + int regs_count = 0; + /* Conditions for the destination register: + // * value must be ready at the required cycle + // * cannot be the same as the source register unless the instruction allows it + // - this avoids optimizable instructions such as "xor r, r" or "sub r, r" + // * register cannot be multiplied twice in a row unless chain_mul is true + // - this avoids accumulation of trailing zeroes in registers due to excessive multiplication + // - allowChainedMul is set to true if an attempt to find source/destination registers failed (this is quite rare, but prevents a catastrophic failure of the generator) + // * either the last instruction applied to the register or its source must be different than this instruction + // - this avoids optimizable instruction sequences such as "xor r1, r2; xor r1, r2" or "ror r, C1; ror r, C2" or "add r, C1; add r, C2" + // * register r5 cannot be the destination of the IADD_RS instruction (limitation of the x86 lea instruction) */ + for (int i = 0; i < 8; ++i) { + bool available = ctx->registers[i].latency <= cycle; + available &= ((!tpl->distinct_dst) | (i != instr->src)); + available &= (ctx->chain_mul | (tpl->group != INSTR_MUL_R) | (ctx->registers[i].last_op != INSTR_MUL_R)); + available &= ((ctx->registers[i].last_op != tpl->group) | (ctx->registers[i].last_op_par != instr->op_par)); + available &= ((instr->opcode != INSTR_ADD_RS) | (i != REGISTER_NEEDS_DISPLACEMENT)); + available_regs[regs_count] = available ? i : 0; + regs_count += available; + } + return select_register(available_regs, regs_count, &ctx->gen, &instr->dst); +} + +static bool select_source(const instr_template* tpl, instruction* instr, generator_ctx* ctx, int cycle) { + int available_regs[8]; + int regs_count = 0; + /* all registers that are ready at the cycle */ + for (int i = 0; i < 8; ++i) { + if (ctx->registers[i].latency <= cycle) + available_regs[regs_count++] = i; + } + /* if there are only 2 available registers for ADD_RS and one of them is r5, select it as the source because it cannot be the destination */ + if (regs_count == 2 && instr->opcode == INSTR_ADD_RS) { + if (available_regs[0] == REGISTER_NEEDS_DISPLACEMENT || available_regs[1] == REGISTER_NEEDS_DISPLACEMENT) { + instr->op_par = instr->src = REGISTER_NEEDS_DISPLACEMENT; + return true; + } + } + if (select_register(available_regs, regs_count, &ctx->gen, &instr->src)) { + if (tpl->op_par_src) + instr->op_par = instr->src; + return true; + } + return false; +} + +static int schedule_uop(execution_port uop, generator_ctx* ctx, int cycle, bool commit) { + /* The scheduling here is done optimistically by checking port availability in order P5 -> P0 -> P1 to not overload + port P1 (multiplication) by instructions that can go to any port. */ + for (; cycle < PORT_MAP_SIZE; ++cycle) { + if ((uop & PORT_P5) && !ctx->ports[cycle][2]) { + if (commit) { + ctx->ports[cycle][2] = uop; + } + TRACE_PRINT("%s scheduled to port P5 at cycle %i (commit = %i)\n", execution_port_names[uop], cycle, commit); + return cycle; + } + if ((uop & PORT_P0) && !ctx->ports[cycle][0]) { + if (commit) { + ctx->ports[cycle][0] = uop; + } + TRACE_PRINT("%s scheduled to port P0 at cycle %i (commit = %i)\n", execution_port_names[uop], cycle, commit); + return cycle; + } + if ((uop & PORT_P1) != 0 && !ctx->ports[cycle][1]) { + if (commit) { + ctx->ports[cycle][1] = uop; + } + TRACE_PRINT("%s scheduled to port P1 at cycle %i (commit = %i)\n", execution_port_names[uop], cycle, commit); + return cycle; + } + } + return -1; +} + +static int schedule_instr(const instr_template* tpl, generator_ctx* ctx, bool commit) { + if (tpl->uop2 == PORT_NONE) { + /* this instruction has only one uOP */ + return schedule_uop(tpl->uop1, ctx, ctx->cycle, commit); + } + else { + /* instructions with 2 uOPs are scheduled conservatively by requiring both uOPs to execute in the same cycle */ + for (int cycle = ctx->cycle; cycle < PORT_MAP_SIZE; ++cycle) { + + int cycle1 = schedule_uop(tpl->uop1, ctx, cycle, false); + int cycle2 = schedule_uop(tpl->uop2, ctx, cycle, false); + + if (cycle1 >= 0 && cycle1 == cycle2) { + if (commit) { + schedule_uop(tpl->uop1, ctx, cycle, true); + schedule_uop(tpl->uop2, ctx, cycle, true); + } + return cycle1; + } + } + } + + return -1; +} + +static void print_registers(const generator_ctx* ctx) { + for (int i = 0; i < 8; ++i) { + printf(" R%i = %i\n", i, ctx->registers[i].latency); + } +} + +bool hashx_program_generate(const siphash_state* key, hashx_program* program) { + generator_ctx ctx = { + .cycle = 0, + .sub_cycle = 0, /* 3 sub-cycles = 1 cycle */ + .mul_count = 0, + .chain_mul = false, + .latency = 0, + .ports = { 0 } + }; + hashx_siphash_rng_init(&ctx.gen, key); + for (int i = 0; i < 8; ++i) { + ctx.registers[i].last_op = -1; + ctx.registers[i].latency = 0; + ctx.registers[i].last_op_par = -1; + } + program->code_size = 0; + + int attempt = 0; + instr_type last_instr = -1; +#ifdef HASHX_PROGRAM_STATS + program->x86_size = 0; +#endif + + while (program->code_size < HASHX_PROGRAM_MAX_SIZE) { + instruction* instr = &program->code[program->code_size]; + TRACE_PRINT("CYCLE: %i/%i\n", ctx.sub_cycle, ctx.cycle); + + /* select an instruction template */ + const instr_template* tpl = select_template(&ctx, last_instr, attempt); + last_instr = tpl->group; + + TRACE_PRINT("Template: %s\n", tpl->x86_asm); + + instr_from_template(tpl, &ctx.gen, instr); + + /* calculate the earliest cycle when this instruction (all of its uOPs) can be scheduled for execution */ + int scheduleCycle = schedule_instr(tpl, &ctx, false); + if (scheduleCycle < 0) { + TRACE_PRINT("Unable to map operation '%s' to execution port (cycle %i)\n", tpl->x86_asm, ctx.cycle); + /* __debugbreak(); */ + break; + } + + ctx.chain_mul = attempt > 0; + + /* find a source register (if applicable) that will be ready when this instruction executes */ + if (tpl->has_src) { + if (!select_source(tpl, instr, &ctx, scheduleCycle)) { + TRACE_PRINT("; src STALL (attempt %i)\n", attempt); + if (attempt++ < MAX_RETRIES) { + continue; + } + if (TRACE) { + printf("; select_source FAILED at cycle %i\n", ctx.cycle); + print_registers(&ctx); + /* __debugbreak(); */ + } + ctx.sub_cycle += 3; + ctx.cycle = ctx.sub_cycle / 3; + attempt = 0; + continue; + } + TRACE_PRINT("; src = r%i\n", instr->src); + } + + /* find a destination register that will be ready when this instruction executes */ + if (tpl->has_dst) { + if (!select_destination(tpl, instr, &ctx, scheduleCycle)) { + TRACE_PRINT("; dst STALL (attempt %i)\n", attempt); + if (attempt++ < MAX_RETRIES) { + continue; + } + if (TRACE) { + printf("; select_destination FAILED at cycle %i\n", ctx.cycle); + print_registers(&ctx); + /* __debugbreak(); */ + } + ctx.sub_cycle += 3; + ctx.cycle = ctx.sub_cycle / 3; + attempt = 0; + continue; + } + TRACE_PRINT("; dst = r%i\n", instr->dst); + } + attempt = 0; + + /* recalculate when the instruction can be scheduled for execution based on operand availability */ + scheduleCycle = schedule_instr(tpl, &ctx, true); + + if (scheduleCycle < 0) { + TRACE_PRINT("Unable to map operation '%s' to execution port (cycle %i)\n", tpl->x86_asm, ctx.cycle); + break; + } + + /* terminating condition */ + if (scheduleCycle >= TARGET_CYCLE) { + break; + } + + if (tpl->has_dst) { + register_info* ri = &ctx.registers[instr->dst]; + int retireCycle = scheduleCycle + tpl->latency; + ri->latency = retireCycle; + ri->last_op = tpl->group; + ri->last_op_par = instr->op_par; + ctx.latency = MAX(retireCycle, ctx.latency); + TRACE_PRINT("; RETIRED at cycle %i\n", retireCycle); + } + + program->code_size++; +#ifdef HASHX_PROGRAM_STATS + program->x86_size += tpl->x86_size; +#endif + + ctx.mul_count += is_mul(instr->opcode); + + ++ctx.sub_cycle; + ctx.sub_cycle += (tpl->uop2 != PORT_NONE); + ctx.cycle = ctx.sub_cycle / 3; + } + +#ifdef HASHX_PROGRAM_STATS + memset(program->asic_latencies, 0, sizeof(program->asic_latencies)); + + program->counter = ctx.gen.counter; + program->wide_mul_count = 0; + program->mul_count = ctx.mul_count; + + /* Calculate ASIC latency: + Assumes 1 cycle latency for all operations and unlimited parallelization. */ + for (int i = 0; i < program->code_size; ++i) { + instruction* instr = &program->code[i]; + if (instr->dst < 0) + continue; + int last_dst = program->asic_latencies[instr->dst] + 1; + int lat_src = instr->dst != instr->src ? program->asic_latencies[instr->src] + 1 : 0; + program->asic_latencies[instr->dst] = MAX(last_dst, lat_src); + program->wide_mul_count += is_wide_mul(instr->opcode); + } + + program->asic_latency = 0; + program->cpu_latency = 0; + for (int i = 0; i < 8; ++i) { + program->asic_latency = MAX(program->asic_latency, program->asic_latencies[i]); + program->cpu_latencies[i] = ctx.registers[i].latency; + program->cpu_latency = MAX(program->cpu_latency, program->cpu_latencies[i]); + } + + program->ipc = program->code_size / (double)program->cpu_latency; + program->branch_count = 0; + memset(program->branches, 0, sizeof(program->branches)); + + if (TRACE) { + printf("; ALU port utilization:\n"); + printf("; (* = in use, _ = idle)\n"); + for (int i = 0; i < PORT_MAP_SIZE; ++i) { + printf("; %3i ", i); + for (int j = 0; j < NUM_PORTS; ++j) { + printf("%c", (ctx.ports[i][j] ? '*' : '_')); + } + printf("\n"); + } + } +#endif + + /* reject programs that don't meet the uniform complexity requirements */ + /* this happens in less than 1 seed out of 10000 */ + return + (program->code_size == REQUIREMENT_SIZE) & + (ctx.mul_count == REQUIREMENT_MUL_COUNT) & + (ctx.latency == REQUIREMENT_LATENCY - 1); /* cycles are numbered from 0 */ +} + +static const char* x86_reg_map[] = { "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" }; + +void hashx_program_asm_x86(const hashx_program* program) { + int target = 0; + for (unsigned i = 0; i < program->code_size; ++i) { + const instruction* instr = &program->code[i]; + switch (instr->opcode) + { + case INSTR_SUB_R: + printf("sub %s, %s\n", x86_reg_map[instr->dst], x86_reg_map[instr->src]); + break; + case INSTR_XOR_R: + printf("xor %s, %s\n", x86_reg_map[instr->dst], x86_reg_map[instr->src]); + break; + case INSTR_ADD_RS: + printf("lea %s, [%s+%s*%u]\n", x86_reg_map[instr->dst], x86_reg_map[instr->dst], x86_reg_map[instr->src], 1 << instr->imm32); + break; + case INSTR_MUL_R: + printf("imul %s, %s\n", x86_reg_map[instr->dst], x86_reg_map[instr->src]); + break; + case INSTR_ROR_C: + printf("ror %s, %u\n", x86_reg_map[instr->dst], instr->imm32); + break; + case INSTR_ADD_C: + printf("add %s, %i\n", x86_reg_map[instr->dst], instr->imm32); + break; + case INSTR_XOR_C: + printf("xor %s, %i\n", x86_reg_map[instr->dst], instr->imm32); + break; + case INSTR_UMULH_R: + printf("mov rax, %s\n", x86_reg_map[instr->dst]); + printf("mul %s\n", x86_reg_map[instr->src]); + printf("mov %s, rdx\n", x86_reg_map[instr->dst]); + break; + case INSTR_SMULH_R: + printf("mov rax, %s\n", x86_reg_map[instr->dst]); + printf("imul %s\n", x86_reg_map[instr->src]); + printf("mov %s, rdx\n", x86_reg_map[instr->dst]); + break; + case INSTR_TARGET: + printf("test edi, edi\n"); + printf("target_%i: cmovz esi, edi\n", i); + target = i; + break; + case INSTR_BRANCH: + printf("or edx, esi\n"); + printf("test edx, %i\n", instr->imm32); + printf("jz target_%i\n", target); + break; + default: + UNREACHABLE; + } + } +} diff --git a/src/ext/equix/hashx/src/program.h b/src/ext/equix/hashx/src/program.h new file mode 100644 index 0000000000..096cc4ee0a --- /dev/null +++ b/src/ext/equix/hashx/src/program.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifndef PROGRAM_H +#define PROGRAM_H + +#include <stdint.h> +#include <stdbool.h> +#include <hashx.h> +#include "instruction.h" +#include "siphash.h" +#include "blake2.h" + +#define HASHX_PROGRAM_MAX_SIZE 512 + +typedef struct hashx_program { + instruction code[HASHX_PROGRAM_MAX_SIZE]; + size_t code_size; +#ifdef HASHX_PROGRAM_STATS + unsigned counter; + double ipc; + int x86_size; + int cpu_latency; + int asic_latency; + int mul_count; + int wide_mul_count; + int cpu_latencies[8]; + int asic_latencies[8]; + int branch_count; + int branches[16]; +#endif +} hashx_program; + +#ifdef __cplusplus +extern "C" { +#endif + +HASHX_PRIVATE bool hashx_program_generate(const siphash_state* key, hashx_program* program); + +HASHX_PRIVATE void hashx_program_execute(const hashx_program* program, uint64_t r[8]); + +HASHX_PRIVATE void hashx_program_asm_x86(const hashx_program* program); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/ext/equix/hashx/src/program_exec.c b/src/ext/equix/hashx/src/program_exec.c new file mode 100644 index 0000000000..f8b991e01e --- /dev/null +++ b/src/ext/equix/hashx/src/program_exec.c @@ -0,0 +1,152 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include "program.h" +#include "force_inline.h" +#include "unreachable.h" +#include "siphash.h" +#include "hashx_endian.h" + +#if defined(__SIZEOF_INT128__) +typedef unsigned __int128 uint128_t; +typedef __int128 int128_t; +static FORCE_INLINE uint64_t umulh(uint64_t a, uint64_t b) { + return ((uint128_t)a * b) >> 64; + } +static FORCE_INLINE int64_t smulh(int64_t a, int64_t b) { + return ((int128_t)a * b) >> 64; +} +#define HAVE_UMULH +#define HAVE_SMULH +#endif + +#if defined(_MSC_VER) +#pragma warning (disable : 4146) /* unary minus applied to unsigned type */ +#define HAS_VALUE(X) X ## 0 +#define EVAL_DEFINE(X) HAS_VALUE(X) +#include <intrin.h> +#include <stdlib.h> + +static FORCE_INLINE uint64_t rotr64(uint64_t x, unsigned int c) { + return _rotr64(x, c); +} + +#define HAVE_ROTR + +#if EVAL_DEFINE(__MACHINEARM64_X64(1)) +static FORCE_INLINE uint64_t umulh(uint64_t a, uint64_t b) { + return __umulh(a, b); +} +#define HAVE_UMULH +#endif + +#if EVAL_DEFINE(__MACHINEX64(1)) +static FORCE_INLINE int64_t smulh(int64_t a, int64_t b) { + int64_t hi; + _mul128(a, b, &hi); + return hi; +} +#define HAVE_SMULH +#endif + +#endif + +#ifndef HAVE_ROTR +static FORCE_INLINE uint64_t rotr64(uint64_t a, unsigned int b) { + return (a >> b) | (a << (64 - b)); +} +#define HAVE_ROTR +#endif + +#ifndef HAVE_UMULH +#define LO(x) ((x)&0xffffffff) +#define HI(x) ((x)>>32) +uint64_t umulh(uint64_t a, uint64_t b) { + uint64_t ah = HI(a), al = LO(a); + uint64_t bh = HI(b), bl = LO(b); + uint64_t x00 = al * bl; + uint64_t x01 = al * bh; + uint64_t x10 = ah * bl; + uint64_t x11 = ah * bh; + uint64_t m1 = LO(x10) + LO(x01) + HI(x00); + uint64_t m2 = HI(x10) + HI(x01) + LO(x11) + HI(m1); + uint64_t m3 = HI(x11) + HI(m2); + + return (m3 << 32) + LO(m2); +} +#undef LO +#undef HI +#define HAVE_UMULH +#endif + +#ifndef HAVE_SMULH +int64_t smulh(int64_t a, int64_t b) { + int64_t hi = umulh(a, b); + if (a < 0LL) hi -= b; + if (b < 0LL) hi -= a; + return hi; +} +#define HAVE_SMULH +#endif + +static FORCE_INLINE uint64_t sign_extend_2s_compl(uint32_t x) { + return (-1 == ~0) ? + (int64_t)(int32_t)(x) : + (x > INT32_MAX ? (x | 0xffffffff00000000ULL) : (uint64_t)x); +} + +void hashx_program_execute(const hashx_program* program, uint64_t r[8]) { + int target = 0; + bool branch_enable = true; + uint32_t result = 0; + int branch_idx = 0; + for (int i = 0; i < program->code_size; ++i) { + const instruction* instr = &program->code[i]; + switch (instr->opcode) + { + case INSTR_UMULH_R: + result = r[instr->dst] = umulh(r[instr->dst], r[instr->src]); + break; + case INSTR_SMULH_R: + result = r[instr->dst] = smulh(r[instr->dst], r[instr->src]); + break; + case INSTR_MUL_R: + r[instr->dst] *= r[instr->src]; + break; + case INSTR_SUB_R: + r[instr->dst] -= r[instr->src]; + break; + case INSTR_XOR_R: + r[instr->dst] ^= r[instr->src]; + break; + case INSTR_ADD_RS: + r[instr->dst] += r[instr->src] << instr->imm32; + break; + case INSTR_ROR_C: + r[instr->dst] = rotr64(r[instr->dst], instr->imm32); + break; + case INSTR_ADD_C: + r[instr->dst] += sign_extend_2s_compl(instr->imm32); + break; + case INSTR_XOR_C: + r[instr->dst] ^= sign_extend_2s_compl(instr->imm32); + break; + case INSTR_TARGET: + target = i; + break; + case INSTR_BRANCH: + if (branch_enable && (result & instr->imm32) == 0) { + i = target; + branch_enable = false; +#ifdef HASHX_PROGRAM_STATS + ((hashx_program*)program)->branch_count++; + ((hashx_program*)program)->branches[branch_idx]++; +#endif + } + branch_idx++; + break; + default: + UNREACHABLE; + } + } +} diff --git a/src/ext/equix/hashx/src/siphash.c b/src/ext/equix/hashx/src/siphash.c new file mode 100644 index 0000000000..0acfca8814 --- /dev/null +++ b/src/ext/equix/hashx/src/siphash.c @@ -0,0 +1,66 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include "siphash.h" +#include "hashx_endian.h" +#include "unreachable.h" + +uint64_t hashx_siphash13_ctr(uint64_t input, const siphash_state* keys) { + uint64_t v0 = keys->v0; + uint64_t v1 = keys->v1; + uint64_t v2 = keys->v2; + uint64_t v3 = keys->v3; + + v3 ^= input; + + SIPROUND(v0, v1, v2, v3); + + v0 ^= input; + v2 ^= 0xff; + + SIPROUND(v0, v1, v2, v3); + SIPROUND(v0, v1, v2, v3); + SIPROUND(v0, v1, v2, v3); + + return (v0 ^ v1) ^ (v2 ^ v3); +} + +void hashx_siphash24_ctr_state512(const siphash_state* keys, uint64_t input, + uint64_t state_out[8]) { + + uint64_t v0 = keys->v0; + uint64_t v1 = keys->v1; + uint64_t v2 = keys->v2; + uint64_t v3 = keys->v3; + + v1 ^= 0xee; + v3 ^= input; + + SIPROUND(v0, v1, v2, v3); + SIPROUND(v0, v1, v2, v3); + + v0 ^= input; + v2 ^= 0xee; + + SIPROUND(v0, v1, v2, v3); + SIPROUND(v0, v1, v2, v3); + SIPROUND(v0, v1, v2, v3); + SIPROUND(v0, v1, v2, v3); + + state_out[0] = v0; + state_out[1] = v1; + state_out[2] = v2; + state_out[3] = v3; + + v1 ^= 0xdd; + + SIPROUND(v0, v1, v2, v3); + SIPROUND(v0, v1, v2, v3); + SIPROUND(v0, v1, v2, v3); + SIPROUND(v0, v1, v2, v3); + + state_out[4] = v0; + state_out[5] = v1; + state_out[6] = v2; + state_out[7] = v3; +} diff --git a/src/ext/equix/hashx/src/siphash.h b/src/ext/equix/hashx/src/siphash.h new file mode 100644 index 0000000000..bb468402c3 --- /dev/null +++ b/src/ext/equix/hashx/src/siphash.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifndef SIPHASH_H +#define SIPHASH_H + +#include <stdint.h> +#include <hashx.h> + +#define ROTL(x, b) (((x) << (b)) | ((x) >> (64 - (b)))) +#define SIPROUND(v0, v1, v2, v3) \ + do { \ + v0 += v1; v2 += v3; v1 = ROTL(v1, 13); \ + v3 = ROTL(v3, 16); v1 ^= v0; v3 ^= v2; \ + v0 = ROTL(v0, 32); v2 += v1; v0 += v3; \ + v1 = ROTL(v1, 17); v3 = ROTL(v3, 21); \ + v1 ^= v2; v3 ^= v0; v2 = ROTL(v2, 32); \ + } while (0) + +typedef struct siphash_state { + uint64_t v0, v1, v2, v3; +} siphash_state; + +#ifdef __cplusplus +extern "C" { +#endif + +HASHX_PRIVATE uint64_t hashx_siphash13_ctr(uint64_t input, const siphash_state* keys); +HASHX_PRIVATE void hashx_siphash24_ctr_state512(const siphash_state* keys, uint64_t input, uint64_t state_out[8]); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/ext/equix/hashx/src/siphash_rng.c b/src/ext/equix/hashx/src/siphash_rng.c new file mode 100644 index 0000000000..f1ec23bf47 --- /dev/null +++ b/src/ext/equix/hashx/src/siphash_rng.c @@ -0,0 +1,31 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include "siphash_rng.h" + +void hashx_siphash_rng_init(siphash_rng* gen, const siphash_state* state) { + gen->keys = *state; + gen->counter = 0; + gen->count8 = 0; + gen->count32 = 0; +} + +uint8_t hashx_siphash_rng_u8(siphash_rng* gen) { + if (gen->count8 == 0) { + gen->buffer8 = hashx_siphash13_ctr(gen->counter, &gen->keys); + gen->counter++; + gen->count8 = sizeof(gen->buffer8); + } + gen->count8--; + return gen->buffer8 >> (gen->count8 * 8); +} + +uint32_t hashx_siphash_rng_u32(siphash_rng* gen) { + if (gen->count32 == 0) { + gen->buffer32 = hashx_siphash13_ctr(gen->counter, &gen->keys); + gen->counter++; + gen->count32 = sizeof(gen->buffer32) / sizeof(uint32_t); + } + gen->count32--; + return gen->buffer32 >> (gen->count32 * 32); +} diff --git a/src/ext/equix/hashx/src/siphash_rng.h b/src/ext/equix/hashx/src/siphash_rng.h new file mode 100644 index 0000000000..638b177e06 --- /dev/null +++ b/src/ext/equix/hashx/src/siphash_rng.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifndef SIPHASH_GENERATOR_H +#define SIPHASH_GENERATOR_H + +#include <stdint.h> +#include <hashx.h> +#include "siphash.h" + +typedef struct siphash_rng { + siphash_state keys; + uint64_t counter; + uint64_t buffer8, buffer32; + unsigned count8, count32; +} siphash_rng; + +#ifdef __cplusplus +extern "C" { +#endif + +HASHX_PRIVATE void hashx_siphash_rng_init(siphash_rng* gen, const siphash_state* state); +HASHX_PRIVATE uint32_t hashx_siphash_rng_u32(siphash_rng* gen); +HASHX_PRIVATE uint8_t hashx_siphash_rng_u8(siphash_rng* gen); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/ext/equix/hashx/src/test_utils.h b/src/ext/equix/hashx/src/test_utils.h new file mode 100644 index 0000000000..54c2f7ec80 --- /dev/null +++ b/src/ext/equix/hashx/src/test_utils.h @@ -0,0 +1,60 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifndef TEST_UTILS_H +#define TEST_UTILS_H + +#include <stdio.h> +#include <stdbool.h> +#include <string.h> +#include <stdlib.h> +#include <hashx.h> + +static inline void read_option(const char* option, int argc, char** argv, bool* out) { + for (int i = 0; i < argc; ++i) { + if (strcmp(argv[i], option) == 0) { + *out = true; + return; + } + } + *out = false; +} + +static inline void read_int_option(const char* option, int argc, char** argv, int* out, int default_val) { + for (int i = 0; i < argc - 1; ++i) { + if (strcmp(argv[i], option) == 0 && (*out = atoi(argv[i + 1])) > 0) { + return; + } + } + *out = default_val; +} + +static inline char parse_nibble(char hex) { + hex &= ~0x20; + return (hex & 0x40) ? hex - ('A' - 10) : hex & 0xf; +} + +static inline void hex2bin(const char* in, int length, char* out) { + for (int i = 0; i < length; i += 2) { + char nibble1 = parse_nibble(*in++); + char nibble2 = parse_nibble(*in++); + *out++ = nibble1 << 4 | nibble2; + } +} + +static inline void output_hex(const char* data, int length) { + for (unsigned i = 0; i < length; ++i) + printf("%02x", data[i] & 0xff); +} + +static inline bool hashes_equal(char* a, char* b) { + return memcmp(a, b, HASHX_SIZE) == 0; +} + +static inline bool equals_hex(const void* hash, const char* hex) { + char reference[HASHX_SIZE]; + hex2bin(hex, 2 * HASHX_SIZE, reference); + return memcmp(hash, reference, sizeof(reference)) == 0; +} + +#endif diff --git a/src/ext/equix/hashx/src/tests.c b/src/ext/equix/hashx/src/tests.c new file mode 100644 index 0000000000..f04d8b9d8f --- /dev/null +++ b/src/ext/equix/hashx/src/tests.c @@ -0,0 +1,219 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifdef NDEBUG +#undef NDEBUG +#endif + +#include <assert.h> +#include "test_utils.h" + +typedef bool test_func(); + +static int test_no = 0; + +static hashx_ctx* ctx_int = NULL; +static hashx_ctx* ctx_cmp = NULL; + +static const char seed1[] = "This is a test"; +static const char seed2[] = "Lorem ipsum dolor sit amet"; + +static const uint64_t counter1 = 0; +static const uint64_t counter2 = 123456; +static const uint64_t counter3 = 987654321123456789; + +static const unsigned char long_input[] = { + 0x0b, 0x0b, 0x98, 0xbe, 0xa7, 0xe8, 0x05, 0xe0, 0x01, 0x0a, 0x21, 0x26, + 0xd2, 0x87, 0xa2, 0xa0, 0xcc, 0x83, 0x3d, 0x31, 0x2c, 0xb7, 0x86, 0x38, + 0x5a, 0x7c, 0x2f, 0x9d, 0xe6, 0x9d, 0x25, 0x53, 0x7f, 0x58, 0x4a, 0x9b, + 0xc9, 0x97, 0x7b, 0x00, 0x00, 0x00, 0x00, 0x66, 0x6f, 0xd8, 0x75, 0x3b, + 0xf6, 0x1a, 0x86, 0x31, 0xf1, 0x29, 0x84, 0xe3, 0xfd, 0x44, 0xf4, 0x01, + 0x4e, 0xca, 0x62, 0x92, 0x76, 0x81, 0x7b, 0x56, 0xf3, 0x2e, 0x9b, 0x68, + 0xbd, 0x82, 0xf4, 0x16 +}; + +#define RUN_TEST(x) run_test(#x, &x) + +static void run_test(const char* name, test_func* func) { + printf("[%2i] %-40s ... ", ++test_no, name); + printf(func() ? "PASSED\n" : "SKIPPED\n"); +} + +static bool test_alloc() { + ctx_int = hashx_alloc(HASHX_INTERPRETED); + assert(ctx_int != NULL && ctx_int != HASHX_NOTSUPP); + return true; +} + +static bool test_free() { + hashx_free(ctx_int); + hashx_free(ctx_cmp); + return true; +} + +static bool test_make1() { + int result = hashx_make(ctx_int, seed1, sizeof(seed1)); + assert(result == 1); + return true; +} + +static bool test_hash_ctr1() { +#ifdef HASHX_SALT + return false; +#endif +#ifndef HASHX_BLOCK_MODE + char hash[HASHX_SIZE]; + hashx_exec(ctx_int, counter2, hash); + /* printf("\n"); + output_hex(hash, HASHX_SIZE); + printf("\n"); */ + assert(equals_hex(hash, "aebdd50aa67c93afb82a4c534603b65e46decd584c55161c526ebc099415ccf1")); + return true; +#else + return false; +#endif +} + +static bool test_hash_ctr2() { +#ifdef HASHX_SALT + return false; +#endif +#ifndef HASHX_BLOCK_MODE + char hash[HASHX_SIZE]; + hashx_exec(ctx_int, counter1, hash); + assert(equals_hex(hash, "2b2f54567dcbea98fdb5d5e5ce9a65983c4a4e35ab1464b1efb61e83b7074bb2")); + return true; +#else + return false; +#endif +} + +static bool test_make2() { + int result = hashx_make(ctx_int, seed2, sizeof(seed2)); + assert(result == 1); + return true; +} + +static bool test_hash_ctr3() { +#ifdef HASHX_SALT + return false; +#endif +#ifndef HASHX_BLOCK_MODE + char hash[HASHX_SIZE]; + hashx_exec(ctx_int, counter2, hash); + assert(equals_hex(hash, "ab3d155bf4bbb0aa3a71b7801089826186e44300e6932e6ffd287cf302bbb0ba")); + return true; +#else + return false; +#endif +} + +static bool test_hash_ctr4() { +#ifdef HASHX_SALT + return false; +#endif +#ifndef HASHX_BLOCK_MODE + char hash[HASHX_SIZE]; + hashx_exec(ctx_int, counter3, hash); + assert(equals_hex(hash, "8dfef0497c323274a60d1d93292b68d9a0496379ba407b4341cf868a14d30113")); + return true; +#else + return false; +#endif +} + +static bool test_hash_block1() { +#ifdef HASHX_SALT + return false; +#endif +#ifndef HASHX_BLOCK_MODE + return false; +#else + char hash[HASHX_SIZE]; + hashx_exec(ctx_int, long_input, sizeof(long_input), hash); + assert(equals_hex(hash, "d0b232b832459501ca1ac9dc0429fd931414ead7624a457e375a43ea3e5e737a")); + return true; +#endif +} + +static bool test_alloc_compiler() { + ctx_cmp = hashx_alloc(HASHX_COMPILED); + assert(ctx_cmp != NULL); + return ctx_cmp != HASHX_NOTSUPP; +} + +static bool test_make3() { + if (ctx_cmp == HASHX_NOTSUPP) + return false; + + int result = hashx_make(ctx_cmp, seed2, sizeof(seed2)); + assert(result == 1); + return true; +} + +static bool test_compiler_ctr1() { + if (ctx_cmp == HASHX_NOTSUPP) + return false; + +#ifndef HASHX_BLOCK_MODE + char hash1[HASHX_SIZE]; + char hash2[HASHX_SIZE]; + hashx_exec(ctx_int, counter2, hash1); + hashx_exec(ctx_cmp, counter2, hash2); + assert(hashes_equal(hash1, hash2)); + return true; +#else + return false; +#endif +} + +static bool test_compiler_ctr2() { + if (ctx_cmp == HASHX_NOTSUPP) + return false; + +#ifndef HASHX_BLOCK_MODE + char hash1[HASHX_SIZE]; + char hash2[HASHX_SIZE]; + hashx_exec(ctx_int, counter1, hash1); + hashx_exec(ctx_cmp, counter1, hash2); + assert(hashes_equal(hash1, hash2)); + return true; +#else + return false; +#endif +} + +static bool test_compiler_block1() { + if (ctx_cmp == HASHX_NOTSUPP) + return false; +#ifndef HASHX_BLOCK_MODE + return false; +#else + char hash1[HASHX_SIZE]; + char hash2[HASHX_SIZE]; + hashx_exec(ctx_int, long_input, sizeof(long_input), hash1); + hashx_exec(ctx_cmp, long_input, sizeof(long_input), hash2); + assert(hashes_equal(hash1, hash2)); + return true; +#endif +} + +int main() { + RUN_TEST(test_alloc); + RUN_TEST(test_make1); + RUN_TEST(test_hash_ctr1); + RUN_TEST(test_hash_ctr2); + RUN_TEST(test_make2); + RUN_TEST(test_hash_ctr3); + RUN_TEST(test_hash_ctr4); + RUN_TEST(test_alloc_compiler); + RUN_TEST(test_make3); + RUN_TEST(test_compiler_ctr1); + RUN_TEST(test_compiler_ctr2); + RUN_TEST(test_hash_block1); + RUN_TEST(test_compiler_block1); + RUN_TEST(test_free); + + printf("\nAll tests were successful\n"); + return 0; +} diff --git a/src/ext/equix/hashx/src/unreachable.h b/src/ext/equix/hashx/src/unreachable.h new file mode 100644 index 0000000000..69807fb395 --- /dev/null +++ b/src/ext/equix/hashx/src/unreachable.h @@ -0,0 +1,9 @@ +#ifndef UNREACHABLE +#ifdef __GNUC__ +#define UNREACHABLE __builtin_unreachable() +#elif _MSC_VER +#define UNREACHABLE __assume(0) +#else +#define UNREACHABLE +#endif +#endif diff --git a/src/ext/equix/hashx/src/virtual_memory.c b/src/ext/equix/hashx/src/virtual_memory.c new file mode 100644 index 0000000000..7dc63e1e68 --- /dev/null +++ b/src/ext/equix/hashx/src/virtual_memory.c @@ -0,0 +1,126 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include "virtual_memory.h" + +#ifdef HASHX_WIN +#include <windows.h> +#else +#ifdef __APPLE__ +#include <mach/vm_statistics.h> +#endif +#include <sys/types.h> +#include <sys/mman.h> +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif +#define PAGE_READONLY PROT_READ +#define PAGE_READWRITE (PROT_READ | PROT_WRITE) +#define PAGE_EXECUTE_READ (PROT_READ | PROT_EXEC) +#define PAGE_EXECUTE_READWRITE (PROT_READ | PROT_WRITE | PROT_EXEC) +#endif + +#ifdef HASHX_WIN + +static int set_privilege(const char* pszPrivilege, BOOL bEnable) { + HANDLE hToken; + TOKEN_PRIVILEGES tp; + BOOL status; + DWORD error; + + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES + | TOKEN_QUERY, &hToken)) + return 0; + + if (!LookupPrivilegeValue(NULL, pszPrivilege, &tp.Privileges[0].Luid)) + return 0; + + tp.PrivilegeCount = 1; + + if (bEnable) + tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + else + tp.Privileges[0].Attributes = 0; + + status = AdjustTokenPrivileges(hToken, FALSE, &tp, 0, + (PTOKEN_PRIVILEGES)NULL, 0); + error = GetLastError(); + + CloseHandle(hToken); + + return status && (error == ERROR_SUCCESS); +} +#endif + +void* hashx_vm_alloc(size_t bytes) { + void* mem; +#ifdef HASHX_WIN + mem = VirtualAlloc(NULL, bytes, MEM_COMMIT, PAGE_READWRITE); +#else + mem = mmap(NULL, bytes, PAGE_READWRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (mem == MAP_FAILED) + return NULL; +#endif + return mem; +} + +static inline int page_protect(void* ptr, size_t bytes, int rules) { +#ifdef HASHX_WIN + DWORD oldp; + if (!VirtualProtect(ptr, bytes, (DWORD)rules, &oldp)) { + return 0; + } +#else + if (-1 == mprotect(ptr, bytes, rules)) + return 0; +#endif + return 1; +} + +void hashx_vm_rw(void* ptr, size_t bytes) { + page_protect(ptr, bytes, PAGE_READWRITE); +} + +void hashx_vm_rx(void* ptr, size_t bytes) { + page_protect(ptr, bytes, PAGE_EXECUTE_READ); +} + +void* hashx_vm_alloc_huge(size_t bytes) { + void* mem; +#ifdef HASHX_WIN + set_privilege("SeLockMemoryPrivilege", 1); + SIZE_T page_min = GetLargePageMinimum(); + if (page_min > 0) { + mem = VirtualAlloc(NULL, ALIGN_SIZE(bytes, page_min), MEM_COMMIT + | MEM_RESERVE | MEM_LARGE_PAGES, PAGE_READWRITE); + } + else { + mem = NULL; + } +#else +#ifdef __APPLE__ + mem = mmap(NULL, bytes, PAGE_READWRITE, MAP_PRIVATE | MAP_ANONYMOUS, + VM_FLAGS_SUPERPAGE_SIZE_2MB, 0); +#elif defined(__FreeBSD__) + mem = mmap(NULL, bytes, PAGE_READWRITE, MAP_PRIVATE | MAP_ANONYMOUS + | MAP_ALIGNED_SUPER, -1, 0); +#elif defined(__OpenBSD__) + mem = MAP_FAILED; // OpenBSD does not support huge pages +#else + mem = mmap(NULL, bytes, PAGE_READWRITE, MAP_PRIVATE | MAP_ANONYMOUS + | MAP_HUGETLB | MAP_POPULATE, -1, 0); +#endif + if (mem == MAP_FAILED) { + mem = NULL; + } +#endif + return mem; +} + +void hashx_vm_free(void* ptr, size_t bytes) { +#ifdef HASHX_WIN + VirtualFree(ptr, 0, MEM_RELEASE); +#else + munmap(ptr, bytes); +#endif +} diff --git a/src/ext/equix/hashx/src/virtual_memory.h b/src/ext/equix/hashx/src/virtual_memory.h new file mode 100644 index 0000000000..d08f74dcc6 --- /dev/null +++ b/src/ext/equix/hashx/src/virtual_memory.h @@ -0,0 +1,19 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifndef VIRTUAL_MEMORY_H +#define VIRTUAL_MEMORY_H + +#include <stdint.h> +#include <stddef.h> +#include <hashx.h> + +#define ALIGN_SIZE(pos, align) ((((pos) - 1) / (align) + 1) * (align)) + +HASHX_PRIVATE void* hashx_vm_alloc(size_t size); +HASHX_PRIVATE void hashx_vm_rw(void* ptr, size_t size); +HASHX_PRIVATE void hashx_vm_rx(void* ptr, size_t size); +HASHX_PRIVATE void* hashx_vm_alloc_huge(size_t size); +HASHX_PRIVATE void hashx_vm_free(void* ptr, size_t size); + +#endif diff --git a/src/ext/equix/include/equix.h b/src/ext/equix/include/equix.h new file mode 100644 index 0000000000..01ab249437 --- /dev/null +++ b/src/ext/equix/include/equix.h @@ -0,0 +1,145 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifndef EQUIX_H +#define EQUIX_H + +#include <stdint.h> +#include <stddef.h> + +/* + * The solver will return at most this many solutions. + */ +#define EQUIX_MAX_SOLS 8 + +/* + * The number of indices. + */ +#define EQUIX_NUM_IDX 8 + +/* + * 16-bit index. + */ +typedef uint16_t equix_idx; + +/* + * The solution. + */ +typedef struct equix_solution { + equix_idx idx[EQUIX_NUM_IDX]; +} equix_solution; + +/* + * Solution verification results + */ +typedef enum equix_result { + EQUIX_OK, /* Solution is valid */ + EQUIX_CHALLENGE, /* The challenge is invalid (the internal hash + function doesn't pass validation). */ + EQUIX_ORDER, /* Indices are not in the correct order. */ + EQUIX_PARTIAL_SUM, /* The partial sums of the hash values don't + have the required number of trailing zeroes. */ + EQUIX_FINAL_SUM /* The hash values don't sum to zero. */ +} equix_result; + +/* + * Opaque struct that holds the Equi-X context + */ +typedef struct equix_ctx equix_ctx; + +/* + * Flags for context creation +*/ +typedef enum equix_ctx_flags { + EQUIX_CTX_VERIFY = 0, /* Context for verification */ + EQUIX_CTX_SOLVE = 1, /* Context for solving */ + EQUIX_CTX_COMPILE = 2, /* Compile internal hash function */ + EQUIX_CTX_HUGEPAGES = 4, /* Allocate solver memory using HugePages */ +} equix_ctx_flags; + +/* Sentinel value used to indicate unsupported type */ +#define EQUIX_NOTSUPP ((equix_ctx*)-1) + +#if defined(_WIN32) || defined(__CYGWIN__) +#define EQUIX_WIN +#endif + +/* Shared/static library definitions */ +#ifdef EQUIX_WIN + #ifdef EQUIX_SHARED + #define EQUIX_API __declspec(dllexport) + #elif !defined(EQUIX_STATIC) + #define EQUIX_API __declspec(dllimport) + #else + #define EQUIX_API + #endif + #define EQUIX_PRIVATE +#else + #ifdef EQUIX_SHARED + #define EQUIX_API __attribute__ ((visibility ("default"))) + #else + #define EQUIX_API __attribute__ ((visibility ("hidden"))) + #endif + #define EQUIX_PRIVATE __attribute__ ((visibility ("hidden"))) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Allocate an Equi-X context. + * + * @param flags is the type of context to be created + * + * @return pointer to a newly created context. Returns NULL on memory + * allocation failure and EQUIX_NOTSUPP if the requested type + * is not supported. + */ +EQUIX_API equix_ctx* equix_alloc(equix_ctx_flags flags); + +/* +* Free an Equi-X a context. +* +* @param ctx is a pointer to the context +*/ +EQUIX_API void equix_free(equix_ctx* ctx); + +/* + * Find Equi-X solutions for the given challenge. + * + * @param ctx pointer to an Equi-X context + * @param challenge pointer to the challenge data + * @param challenge_size size of the challenge + * @param output pointer to the output array where solutions will be + * stored + * + * @return the number of solutions found + */ +EQUIX_API int equix_solve( + equix_ctx* ctx, + const void* challenge, + size_t challenge_size, + equix_solution output[EQUIX_MAX_SOLS]); + +/* + * Verify an Equi-X solution. + * + * @param ctx pointer to an Equi-X context + * @param challenge pointer to the challenge data + * @param challenge_size size of the challenge + * @param solution pointer to the solution to be verified + * + * @return verification result +*/ +EQUIX_API equix_result equix_verify( + equix_ctx* ctx, + const void* challenge, + size_t challenge_size, + const equix_solution* solution); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/ext/equix/src/bench.c b/src/ext/equix/src/bench.c new file mode 100644 index 0000000000..e5b925c3d2 --- /dev/null +++ b/src/ext/equix/src/bench.c @@ -0,0 +1,175 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include <stdio.h> +#include <stdbool.h> +#include <string.h> +#include <stdlib.h> +#include <equix.h> +#include <test_utils.h> +#include <hashx_thread.h> +#include <hashx_time.h> + +typedef struct solver_output { + equix_solution sols[EQUIX_MAX_SOLS]; + int count; +} solver_output; + +typedef struct worker_job { + int id; + hashx_thread thread; + equix_ctx* ctx; + int64_t total_sols; + int start; + int step; + int end; + solver_output* output; +} worker_job; + +static hashx_thread_retval worker(void* args) { + worker_job* job = (worker_job*)args; + job->total_sols = 0; + solver_output* outptr = job->output; + for (int seed = job->start; seed < job->end; seed += job->step) { + int count = equix_solve(job->ctx, &seed, sizeof(seed), outptr->sols); + outptr->count = count; + job->total_sols += count; + outptr++; + } + return HASHX_THREAD_SUCCESS; +} + +static void print_solution(int nonce, const equix_solution* sol) { + output_hex((char*)&nonce, sizeof(nonce)); + printf(" : { "); + for (int idx = 0; idx < EQUIX_NUM_IDX; ++idx) { + printf("%#06x%s", sol->idx[idx], + idx != EQUIX_NUM_IDX - 1 ? ", " : ""); + } + printf(" }\n"); +} + +static const char* result_names[] = { + "OK", + "Invalid nonce", + "Indices out of order", + "Nonzero partial sum", + "Nonzero final sum" +}; + +static void print_help(char* executable) { + printf("Usage: %s [OPTIONS]\n", executable); + printf("Supported options:\n"); + printf(" --help show this message\n"); + printf(" --nonces N solve N nonces (default: N=500)\n"); + printf(" --start S start with nonce S (default: S=0)\n"); + printf(" --threads T use T threads (default: T=1)\n"); + printf(" --interpret use HashX interpreter\n"); + printf(" --hugepages use hugepages\n"); + printf(" --sols print all solutions\n"); +} + +int main(int argc, char** argv) { + int nonces, start, threads; + bool interpret, huge_pages, print_sols, help; + read_option("--help", argc, argv, &help); + if (help) { + print_help(argv[0]); + return 0; + } + read_int_option("--nonces", argc, argv, &nonces, 500); + read_int_option("--start", argc, argv, &start, 0); + read_option("--interpret", argc, argv, &interpret); + read_option("--hugepages", argc, argv, &huge_pages); + read_option("--sols", argc, argv, &print_sols); + read_int_option("--threads", argc, argv, &threads, 1); + equix_ctx_flags flags = EQUIX_CTX_SOLVE; + if (!interpret) { + flags |= EQUIX_CTX_COMPILE; + } + if (huge_pages) { + flags |= EQUIX_CTX_HUGEPAGES; + } + worker_job* jobs = malloc(sizeof(worker_job) * threads); + if (jobs == NULL) { + printf("Error: memory allocation failure\n"); + return 1; + } + int per_thread = (nonces + threads - 1) / threads; + for (int thd = 0; thd < threads; ++thd) { + jobs[thd].ctx = equix_alloc(flags); + if (jobs[thd].ctx == NULL) { + printf("Error: memory allocation failure\n"); + return 1; + } + if (jobs[thd].ctx == EQUIX_NOTSUPP) { + printf("Error: not supported. Try with --interpret\n"); + return 1; + } + jobs[thd].id = thd; + jobs[thd].start = start + thd; + jobs[thd].step = threads; + jobs[thd].end = start + nonces; + jobs[thd].output = malloc(sizeof(solver_output) * per_thread); + if (jobs[thd].output == NULL) { + printf("Error: memory allocation failure\n"); + return 1; + } + } + printf("Solving nonces %i-%i (interpret: %i, hugepages: %i, threads: %i) ...\n", start, start + nonces - 1, interpret, huge_pages, threads); + int total_sols = 0; + double time_start, time_end; + time_start = hashx_time(); + if (threads > 1) { + for (int thd = 0; thd < threads; ++thd) { + jobs[thd].thread = hashx_thread_create(&worker, &jobs[thd]); + } + for (int thd = 0; thd < threads; ++thd) { + hashx_thread_join(jobs[thd].thread); + } + } + else { + worker(jobs); + } + time_end = hashx_time(); + for (int thd = 0; thd < threads; ++thd) { + total_sols += jobs[thd].total_sols; + } + double elapsed = time_end - time_start; + printf("%f solutions/nonce\n", total_sols / (double)nonces); + printf("%f solutions/sec. (%i thread%s)\n", total_sols / elapsed, threads, threads > 1 ? "s" : ""); + if (print_sols) { + for (int thd = 0; thd < threads; ++thd) { + worker_job* job = &jobs[thd]; + solver_output* outptr = job->output; + for (int seed = job->start; seed < job->end; seed += job->step) { + for (int sol = 0; sol < outptr->count; ++sol) { + print_solution(seed, &outptr->sols[sol]); + } + outptr++; + } + } + } + time_start = hashx_time(); + for (int thd = 0; thd < threads; ++thd) { + worker_job* job = &jobs[thd]; + solver_output* outptr = job->output; + for (int seed = job->start; seed < job->end; seed += job->step) { + for (int sol = 0; sol < outptr->count; ++sol) { + equix_result result = equix_verify(job->ctx, &seed, sizeof(seed), &outptr->sols[sol]); + if (result != EQUIX_OK) { + printf("Invalid solution (%s):\n", result_names[result]); + print_solution(seed, &outptr->sols[sol]); + } + } + outptr++; + } + } + time_end = hashx_time(); + printf("%f verifications/sec. (1 thread)\n", total_sols / (time_end - time_start)); + for (int thd = 0; thd < threads; ++thd) { + free(jobs[thd].output); + } + free(jobs); + return 0; +} diff --git a/src/ext/equix/src/context.c b/src/ext/equix/src/context.c new file mode 100644 index 0000000000..b0aa2d40e5 --- /dev/null +++ b/src/ext/equix/src/context.c @@ -0,0 +1,57 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include <stdlib.h> +#include <equix.h> +#include <virtual_memory.h> +#include "context.h" +#include "solver_heap.h" + +equix_ctx* equix_alloc(equix_ctx_flags flags) { + equix_ctx* ctx_failure = NULL; + equix_ctx* ctx = malloc(sizeof(equix_ctx)); + if (ctx == NULL) { + goto failure; + } + ctx->flags = flags & EQUIX_CTX_COMPILE; + ctx->hash_func = hashx_alloc(flags & EQUIX_CTX_COMPILE ? + HASHX_COMPILED : HASHX_INTERPRETED); + if (ctx->hash_func == NULL) { + goto failure; + } + if (ctx->hash_func == HASHX_NOTSUPP) { + ctx_failure = EQUIX_NOTSUPP; + goto failure; + } + if (flags & EQUIX_CTX_SOLVE) { + if (flags & EQUIX_CTX_HUGEPAGES) { + ctx->heap = hashx_vm_alloc_huge(sizeof(solver_heap)); + } + else { + ctx->heap = malloc(sizeof(solver_heap)); + } + if (ctx->heap == NULL) { + goto failure; + } + } + ctx->flags = flags; + return ctx; +failure: + equix_free(ctx); + return ctx_failure; +} + +void equix_free(equix_ctx* ctx) { + if (ctx != NULL && ctx != EQUIX_NOTSUPP) { + if (ctx->flags & EQUIX_CTX_SOLVE) { + if (ctx->flags & EQUIX_CTX_HUGEPAGES) { + hashx_vm_free(ctx->heap, sizeof(solver_heap)); + } + else { + free(ctx->heap); + } + } + hashx_free(ctx->hash_func); + free(ctx); + } +} diff --git a/src/ext/equix/src/context.h b/src/ext/equix/src/context.h new file mode 100644 index 0000000000..69dadb8069 --- /dev/null +++ b/src/ext/equix/src/context.h @@ -0,0 +1,18 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifndef CONTEXT_H +#define CONTEXT_H + +#include <equix.h> +#include <hashx.h> + +typedef struct solver_heap solver_heap; + +typedef struct equix_ctx { + hashx_ctx* hash_func; + solver_heap* heap; + equix_ctx_flags flags; +} equix_ctx; + +#endif diff --git a/src/ext/equix/src/equix.c b/src/ext/equix/src/equix.c new file mode 100644 index 0000000000..71ebd9ca38 --- /dev/null +++ b/src/ext/equix/src/equix.c @@ -0,0 +1,96 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include <stdlib.h> +#include <stdbool.h> +#include <string.h> + +#include <equix.h> +#include <hashx.h> +#include "context.h" +#include "solver.h" +#include <hashx_endian.h> + +static bool verify_order(const equix_solution* solution) { + return + tree_cmp4(&solution->idx[0], &solution->idx[4]) & + tree_cmp2(&solution->idx[0], &solution->idx[2]) & + tree_cmp2(&solution->idx[4], &solution->idx[6]) & + tree_cmp1(&solution->idx[0], &solution->idx[1]) & + tree_cmp1(&solution->idx[2], &solution->idx[3]) & + tree_cmp1(&solution->idx[4], &solution->idx[5]) & + tree_cmp1(&solution->idx[6], &solution->idx[7]); +} + +static uint64_t sum_pair(hashx_ctx* hash_func, equix_idx left, equix_idx right) { + uint8_t hash_left[HASHX_SIZE]; + uint8_t hash_right[HASHX_SIZE]; + hashx_exec(hash_func, left, hash_left); + hashx_exec(hash_func, right, hash_right); + return load64(hash_left) + load64(hash_right); +} + +static equix_result verify_internal(hashx_ctx* hash_func, const equix_solution* solution) { + uint64_t pair0 = sum_pair(hash_func, solution->idx[0], solution->idx[1]); + if (pair0 & EQUIX_STAGE1_MASK) { + return EQUIX_PARTIAL_SUM; + } + uint64_t pair1 = sum_pair(hash_func, solution->idx[2], solution->idx[3]); + if (pair1 & EQUIX_STAGE1_MASK) { + return EQUIX_PARTIAL_SUM; + } + uint64_t pair4 = pair0 + pair1; + if (pair4 & EQUIX_STAGE2_MASK) { + return EQUIX_PARTIAL_SUM; + } + uint64_t pair2 = sum_pair(hash_func, solution->idx[4], solution->idx[5]); + if (pair2 & EQUIX_STAGE1_MASK) { + return EQUIX_PARTIAL_SUM; + } + uint64_t pair3 = sum_pair(hash_func, solution->idx[6], solution->idx[7]); + if (pair3 & EQUIX_STAGE1_MASK) { + return EQUIX_PARTIAL_SUM; + } + uint64_t pair5 = pair2 + pair3; + if (pair5 & EQUIX_STAGE2_MASK) { + return EQUIX_PARTIAL_SUM; + } + uint64_t pair6 = pair4 + pair5; + if (pair6 & EQUIX_FULL_MASK) { + return EQUIX_FINAL_SUM; + } + return EQUIX_OK; +} + +int equix_solve( + equix_ctx* ctx, + const void* challenge, + size_t challenge_size, + equix_solution output[EQUIX_MAX_SOLS]) +{ + if ((ctx->flags & EQUIX_CTX_SOLVE) == 0) { + return 0; + } + + if (!hashx_make(ctx->hash_func, challenge, challenge_size)) { + return 0; + } + + return equix_solver_solve(ctx->hash_func, ctx->heap, output); +} + + +equix_result equix_verify( + equix_ctx* ctx, + const void* challenge, + size_t challenge_size, + const equix_solution* solution) +{ + if (!verify_order(solution)) { + return EQUIX_ORDER; + } + if (!hashx_make(ctx->hash_func, challenge, challenge_size)) { + return EQUIX_CHALLENGE; + } + return verify_internal(ctx->hash_func, solution); +} diff --git a/src/ext/equix/src/solver.c b/src/ext/equix/src/solver.c new file mode 100644 index 0000000000..480c699c60 --- /dev/null +++ b/src/ext/equix/src/solver.c @@ -0,0 +1,270 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#include "solver.h" +#include "context.h" +#include "solver_heap.h" +#include <hashx_endian.h> +#include <string.h> +#include <stdbool.h> +#include <assert.h> +#include <stdio.h> + +#ifdef _MSC_VER +#pragma warning (disable : 4146) /* unary minus applied to unsigned type */ +#endif + +#define CLEAR(x) memset(&x, 0, sizeof(x)) +#define MAKE_ITEM(bucket, left, right) ((left) << 17 | (right) << 8 | (bucket)) +#define ITEM_BUCKET(item) (item) % NUM_COARSE_BUCKETS +#define ITEM_LEFT_IDX(item) (item) >> 17 +#define ITEM_RIGHT_IDX(item) ((item) >> 8) & 511 +#define INVERT_BUCKET(idx) -(idx) % NUM_COARSE_BUCKETS +#define INVERT_SCRATCH(idx) -(idx) % NUM_FINE_BUCKETS +#define STAGE1_IDX(buck, pos) heap->stage1_indices.buckets[buck].items[pos] +#define STAGE2_IDX(buck, pos) heap->stage2_indices.buckets[buck].items[pos] +#define STAGE3_IDX(buck, pos) heap->stage3_indices.buckets[buck].items[pos] +#define STAGE1_DATA(buck, pos) heap->stage1_data.buckets[buck].items[pos] +#define STAGE2_DATA(buck, pos) heap->stage2_data.buckets[buck].items[pos] +#define STAGE3_DATA(buck, pos) heap->stage3_data.buckets[buck].items[pos] +#define STAGE1_SIZE(buck) heap->stage1_indices.counts[buck] +#define STAGE2_SIZE(buck) heap->stage2_indices.counts[buck] +#define STAGE3_SIZE(buck) heap->stage3_indices.counts[buck] +#define SCRATCH(buck, pos) heap->scratch_ht.buckets[buck].items[pos] +#define SCRATCH_SIZE(buck) heap->scratch_ht.counts[buck] +#define SWAP_IDX(a, b) \ + do { \ + equix_idx temp = a; \ + a = b; \ + b = temp; \ + } while(0) +#define CARRY (bucket_idx != 0) +#define BUCK_START 0 +#define BUCK_END (NUM_COARSE_BUCKETS / 2 + 1) + +typedef uint32_t u32; +typedef stage1_idx_item s1_idx; +typedef stage2_idx_item s2_idx; +typedef stage3_idx_item s3_idx; + +static FORCE_INLINE uint64_t hash_value(hashx_ctx* hash_func, equix_idx index) { + char hash[HASHX_SIZE]; + hashx_exec(hash_func, index, hash); + return load64(hash); +} + +static void build_solution_stage1(equix_idx* output, solver_heap* heap, s2_idx root) { + u32 bucket = ITEM_BUCKET(root); + u32 bucket_inv = INVERT_BUCKET(bucket); + u32 left_parent_idx = ITEM_LEFT_IDX(root); + u32 right_parent_idx = ITEM_RIGHT_IDX(root); + s1_idx left_parent = STAGE1_IDX(bucket, left_parent_idx); + s1_idx right_parent = STAGE1_IDX(bucket_inv, right_parent_idx); + output[0] = left_parent; + output[1] = right_parent; + if (!tree_cmp1(&output[0], &output[1])) { + SWAP_IDX(output[0], output[1]); + } +} + +static void build_solution_stage2(equix_idx* output, solver_heap* heap, s3_idx root) { + u32 bucket = ITEM_BUCKET(root); + u32 bucket_inv = INVERT_BUCKET(bucket); + u32 left_parent_idx = ITEM_LEFT_IDX(root); + u32 right_parent_idx = ITEM_RIGHT_IDX(root); + s2_idx left_parent = STAGE2_IDX(bucket, left_parent_idx); + s2_idx right_parent = STAGE2_IDX(bucket_inv, right_parent_idx); + build_solution_stage1(&output[0], heap, left_parent); + build_solution_stage1(&output[2], heap, right_parent); + if (!tree_cmp2(&output[0], &output[2])) { + SWAP_IDX(output[0], output[2]); + SWAP_IDX(output[1], output[3]); + } +} + +static void build_solution(equix_solution* solution, solver_heap* heap, s3_idx left, s3_idx right) { + build_solution_stage2(&solution->idx[0], heap, left); + build_solution_stage2(&solution->idx[4], heap, right); + if (!tree_cmp4(&solution->idx[0], &solution->idx[4])) { + SWAP_IDX(solution->idx[0], solution->idx[4]); + SWAP_IDX(solution->idx[1], solution->idx[5]); + SWAP_IDX(solution->idx[2], solution->idx[6]); + SWAP_IDX(solution->idx[3], solution->idx[7]); + } +} + +static void solve_stage0(hashx_ctx* hash_func, solver_heap* heap) { + CLEAR(heap->stage1_indices.counts); + for (u32 i = 0; i < INDEX_SPACE; ++i) { + uint64_t value = hash_value(hash_func, i); + u32 bucket_idx = value % NUM_COARSE_BUCKETS; + u32 item_idx = STAGE1_SIZE(bucket_idx); + if (item_idx >= COARSE_BUCKET_ITEMS) + continue; + STAGE1_SIZE(bucket_idx) = item_idx + 1; + STAGE1_IDX(bucket_idx, item_idx) = i; + STAGE1_DATA(bucket_idx, item_idx) = value / NUM_COARSE_BUCKETS; /* 52 bits */ + } +} + +#define MAKE_PAIRS1 \ + stage1_data_item value = STAGE1_DATA(bucket_idx, item_idx) + CARRY; \ + u32 fine_buck_idx = value % NUM_FINE_BUCKETS; \ + u32 fine_cpl_bucket = INVERT_SCRATCH(fine_buck_idx); \ + u32 fine_cpl_size = SCRATCH_SIZE(fine_cpl_bucket); \ + for (u32 fine_idx = 0; fine_idx < fine_cpl_size; ++fine_idx) { \ + u32 cpl_index = SCRATCH(fine_cpl_bucket, fine_idx); \ + stage1_data_item cpl_value = STAGE1_DATA(cpl_bucket, cpl_index); \ + stage1_data_item sum = value + cpl_value; \ + assert((sum % NUM_FINE_BUCKETS) == 0); \ + sum /= NUM_FINE_BUCKETS; /* 45 bits */ \ + u32 s2_buck_id = sum % NUM_COARSE_BUCKETS; \ + u32 s2_item_id = STAGE2_SIZE(s2_buck_id); \ + if (s2_item_id >= COARSE_BUCKET_ITEMS) \ + continue; \ + STAGE2_SIZE(s2_buck_id) = s2_item_id + 1; \ + STAGE2_IDX(s2_buck_id, s2_item_id) = \ + MAKE_ITEM(bucket_idx, item_idx, cpl_index); \ + STAGE2_DATA(s2_buck_id, s2_item_id) = \ + sum / NUM_COARSE_BUCKETS; /* 37 bits */ \ + } \ + +static void solve_stage1(solver_heap* heap) { + CLEAR(heap->stage2_indices.counts); + for (u32 bucket_idx = BUCK_START; bucket_idx < BUCK_END; ++bucket_idx) { + u32 cpl_bucket = INVERT_BUCKET(bucket_idx); + CLEAR(heap->scratch_ht.counts); + u32 cpl_buck_size = STAGE1_SIZE(cpl_bucket); + for (u32 item_idx = 0; item_idx < cpl_buck_size; ++item_idx) { + stage1_data_item value = STAGE1_DATA(cpl_bucket, item_idx); + u32 fine_buck_idx = value % NUM_FINE_BUCKETS; + u32 fine_item_idx = SCRATCH_SIZE(fine_buck_idx); + if (fine_item_idx >= FINE_BUCKET_ITEMS) + continue; + SCRATCH_SIZE(fine_buck_idx) = fine_item_idx + 1; + SCRATCH(fine_buck_idx, fine_item_idx) = item_idx; + if (cpl_bucket == bucket_idx) { + MAKE_PAIRS1 + } + } + if (cpl_bucket != bucket_idx) { + u32 buck_size = STAGE1_SIZE(bucket_idx); + for (u32 item_idx = 0; item_idx < buck_size; ++item_idx) { + MAKE_PAIRS1 + } + } + } +} + +#define MAKE_PAIRS2 \ + stage2_data_item value = STAGE2_DATA(bucket_idx, item_idx) + CARRY; \ + u32 fine_buck_idx = value % NUM_FINE_BUCKETS; \ + u32 fine_cpl_bucket = INVERT_SCRATCH(fine_buck_idx); \ + u32 fine_cpl_size = SCRATCH_SIZE(fine_cpl_bucket); \ + for (u32 fine_idx = 0; fine_idx < fine_cpl_size; ++fine_idx) { \ + u32 cpl_index = SCRATCH(fine_cpl_bucket, fine_idx); \ + stage2_data_item cpl_value = STAGE2_DATA(cpl_bucket, cpl_index); \ + stage2_data_item sum = value + cpl_value; \ + assert((sum % NUM_FINE_BUCKETS) == 0); \ + sum /= NUM_FINE_BUCKETS; /* 30 bits */ \ + u32 s3_buck_id = sum % NUM_COARSE_BUCKETS; \ + u32 s3_item_id = STAGE3_SIZE(s3_buck_id); \ + if (s3_item_id >= COARSE_BUCKET_ITEMS) \ + continue; \ + STAGE3_SIZE(s3_buck_id) = s3_item_id + 1; \ + STAGE3_IDX(s3_buck_id, s3_item_id) = \ + MAKE_ITEM(bucket_idx, item_idx, cpl_index); \ + STAGE3_DATA(s3_buck_id, s3_item_id) = \ + sum / NUM_COARSE_BUCKETS; /* 22 bits */ \ + } \ + +static void solve_stage2(solver_heap* heap) { + CLEAR(heap->stage3_indices.counts); + for (u32 bucket_idx = BUCK_START; bucket_idx < BUCK_END; ++bucket_idx) { + u32 cpl_bucket = INVERT_BUCKET(bucket_idx); + CLEAR(heap->scratch_ht.counts); + u32 cpl_buck_size = STAGE2_SIZE(cpl_bucket); + for (u32 item_idx = 0; item_idx < cpl_buck_size; ++item_idx) { + stage2_data_item value = STAGE2_DATA(cpl_bucket, item_idx); + u32 fine_buck_idx = value % NUM_FINE_BUCKETS; + u32 fine_item_idx = SCRATCH_SIZE(fine_buck_idx); + if (fine_item_idx >= FINE_BUCKET_ITEMS) + continue; + SCRATCH_SIZE(fine_buck_idx) = fine_item_idx + 1; + SCRATCH(fine_buck_idx, fine_item_idx) = item_idx; + if (cpl_bucket == bucket_idx) { + MAKE_PAIRS2 + } + } + if (cpl_bucket != bucket_idx) { + u32 buck_size = STAGE2_SIZE(bucket_idx); + for (u32 item_idx = 0; item_idx < buck_size; ++item_idx) { + MAKE_PAIRS2 + } + } + } +} + +#define MAKE_PAIRS3 \ + stage3_data_item value = STAGE3_DATA(bucket_idx, item_idx) + CARRY; \ + u32 fine_buck_idx = value % NUM_FINE_BUCKETS; \ + u32 fine_cpl_bucket = INVERT_SCRATCH(fine_buck_idx); \ + u32 fine_cpl_size = SCRATCH_SIZE(fine_cpl_bucket); \ + for (u32 fine_idx = 0; fine_idx < fine_cpl_size; ++fine_idx) { \ + u32 cpl_index = SCRATCH(fine_cpl_bucket, fine_idx); \ + stage3_data_item cpl_value = STAGE3_DATA(cpl_bucket, cpl_index); \ + stage3_data_item sum = value + cpl_value; \ + assert((sum % NUM_FINE_BUCKETS) == 0); \ + sum /= NUM_FINE_BUCKETS; /* 15 bits */ \ + if ((sum & EQUIX_STAGE1_MASK) == 0) { \ + /* we have a solution */ \ + s3_idx item_left = STAGE3_IDX(bucket_idx, item_idx); \ + s3_idx item_right = STAGE3_IDX(cpl_bucket, cpl_index); \ + build_solution(&output[sols_found], heap, item_left, item_right); \ + if (++(sols_found) >= EQUIX_MAX_SOLS) { \ + return sols_found; \ + } \ + } \ + } \ + +static int solve_stage3(solver_heap* heap, equix_solution output[EQUIX_MAX_SOLS]) { + int sols_found = 0; + + for (u32 bucket_idx = BUCK_START; bucket_idx < BUCK_END; ++bucket_idx) { + u32 cpl_bucket = -bucket_idx & (NUM_COARSE_BUCKETS - 1); + bool nodup = cpl_bucket == bucket_idx; + CLEAR(heap->scratch_ht.counts); + u32 cpl_buck_size = STAGE3_SIZE(cpl_bucket); + for (u32 item_idx = 0; item_idx < cpl_buck_size; ++item_idx) { + stage3_data_item value = STAGE3_DATA(cpl_bucket, item_idx); + u32 fine_buck_idx = value % NUM_FINE_BUCKETS; + u32 fine_item_idx = SCRATCH_SIZE(fine_buck_idx); + if (fine_item_idx >= FINE_BUCKET_ITEMS) + continue; + SCRATCH_SIZE(fine_buck_idx) = fine_item_idx + 1; + SCRATCH(fine_buck_idx, fine_item_idx) = item_idx; + if (cpl_bucket == bucket_idx) { + MAKE_PAIRS3 + } + } + if (cpl_bucket != bucket_idx) { + u32 buck_size = STAGE3_SIZE(bucket_idx); + for (u32 item_idx = 0; item_idx < buck_size; ++item_idx) { + MAKE_PAIRS3 + } + } + } + + return sols_found; +} + +int equix_solver_solve( + hashx_ctx* hash_func, + solver_heap* heap, + equix_solution output[EQUIX_MAX_SOLS]) +{ + solve_stage0(hash_func, heap); + solve_stage1(heap); + solve_stage2(heap); + return solve_stage3(heap, output); +} diff --git a/src/ext/equix/src/solver.h b/src/ext/equix/src/solver.h new file mode 100644 index 0000000000..ad69951929 --- /dev/null +++ b/src/ext/equix/src/solver.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifndef SOLVER_H +#define SOLVER_H + +#include <equix.h> +#include <hashx_endian.h> +#include <stdbool.h> +#include "context.h" + +#define EQUIX_STAGE1_MASK ((1ull << 15) - 1) +#define EQUIX_STAGE2_MASK ((1ull << 30) - 1) +#define EQUIX_FULL_MASK ((1ull << 60) - 1) + +static inline bool tree_cmp1(const equix_idx* left, const equix_idx* right) { + return *left <= *right; +} + +static inline bool tree_cmp2(const equix_idx* left, const equix_idx* right) { + return load32(left) <= load32(right); +} + +static inline bool tree_cmp4(const equix_idx* left, const equix_idx* right) { + return load64(left) <= load64(right); +} + +EQUIX_PRIVATE int equix_solver_solve(hashx_ctx* hash_func, solver_heap* heap, equix_solution output[EQUIX_MAX_SOLS]); + +#endif diff --git a/src/ext/equix/src/solver_heap.h b/src/ext/equix/src/solver_heap.h new file mode 100644 index 0000000000..71c5f0ca48 --- /dev/null +++ b/src/ext/equix/src/solver_heap.h @@ -0,0 +1,108 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifndef SOLVER_HEAP_H +#define SOLVER_HEAP_H + +#include <stdint.h> +#include <equix.h> + +#define INDEX_SPACE (UINT32_C(1) << 16) +#define NUM_COARSE_BUCKETS 256 +#define NUM_FINE_BUCKETS 128 +#define COARSE_BUCKET_ITEMS 336 +#define FINE_BUCKET_ITEMS 12 + +typedef uint16_t fine_item; + +typedef struct fine_bucket { + fine_item items[FINE_BUCKET_ITEMS]; +} fine_bucket; + +typedef struct fine_hashtab { + uint8_t counts[NUM_FINE_BUCKETS]; + fine_bucket buckets[NUM_FINE_BUCKETS]; +} fine_hashtab; + +typedef equix_idx stage1_idx_item; /* 16 bits */ + +typedef uint64_t stage1_data_item; /* 52 bits */ + +typedef struct stage1_idx_bucket { + stage1_idx_item items[COARSE_BUCKET_ITEMS]; +} stage1_idx_bucket; + +typedef struct stage1_data_bucket { + stage1_data_item items[COARSE_BUCKET_ITEMS]; +} stage1_data_bucket; + +typedef struct stage1_idx_hashtab { + uint16_t counts[NUM_COARSE_BUCKETS]; + stage1_idx_bucket buckets[NUM_COARSE_BUCKETS]; +} stage1_idx_hashtab; + +typedef struct stage1_data_hashtab { + stage1_data_bucket buckets[NUM_COARSE_BUCKETS]; +} stage1_data_hashtab; + +typedef uint32_t stage2_idx_item; /* 26 bits: 8 bits = left bucket index + 9 bits = left item index + 9 bits = right item index */ + +typedef struct stage2_idx_bucket { + stage2_idx_item items[COARSE_BUCKET_ITEMS]; +} stage2_idx_bucket; + +typedef struct stage2_idx_hashtab { + uint16_t counts[NUM_COARSE_BUCKETS]; + stage2_idx_bucket buckets[NUM_COARSE_BUCKETS]; +} stage2_idx_hashtab; + +#ifdef SOLVER_PACKED_STAGE2 +#pragma pack(push, 1) +typedef struct stage2_data_item { + uint32_t upper; /* 22 bits */ + uint8_t middle; /* 8 bits */ + uint8_t lower; /* 7 bits */ +} stage2_data_item; +#pragma pack(pop) +#else +typedef uint64_t stage2_data_item; /* 37 bits */ +#endif + +typedef struct stage2_data_bucket { + stage2_data_item items[COARSE_BUCKET_ITEMS]; +} stage2_data_bucket; + +typedef struct stage2_data_hashtab { + stage2_data_bucket buckets[NUM_COARSE_BUCKETS]; +} stage2_data_hashtab; + +typedef uint32_t stage3_data_item; /* 22 bits */ + +typedef struct stage3_data_bucket { + stage3_data_item items[COARSE_BUCKET_ITEMS]; +} stage3_data_bucket; + +typedef struct stage3_data_hashtab { + stage3_data_bucket buckets[NUM_COARSE_BUCKETS]; +} stage3_data_hashtab; + +typedef stage2_idx_hashtab stage3_idx_hashtab; +typedef stage2_idx_item stage3_idx_item; + +typedef struct solver_heap { + stage1_idx_hashtab stage1_indices; /* 172 544 bytes */ + stage2_idx_hashtab stage2_indices; /* 344 576 bytes */ + stage2_data_hashtab stage2_data; /* 688 128 bytes */ + union { + stage1_data_hashtab stage1_data; /* 688 128 bytes */ + struct { + stage3_idx_hashtab stage3_indices; /* 344 576 bytes */ + stage3_data_hashtab stage3_data; /* 344 064 bytes */ + }; + }; + fine_hashtab scratch_ht; /* 3 200 bytes */ +} solver_heap; /* TOTAL: 1 897 088 bytes */ + +#endif diff --git a/src/ext/equix/src/tests.c b/src/ext/equix/src/tests.c new file mode 100644 index 0000000000..63fb5bdb1e --- /dev/null +++ b/src/ext/equix/src/tests.c @@ -0,0 +1,124 @@ +/* Copyright (c) 2020 tevador tevador@gmail.com */ +/* See LICENSE for licensing information */ + +#ifdef NDEBUG +#undef NDEBUG +#endif + +#include <assert.h> +#include <equix.h> +#include <stdbool.h> +#include <stdio.h> + +typedef bool test_func(); + +static equix_ctx* ctx = NULL; +static equix_solution solution[EQUIX_MAX_SOLS]; +static int nonce; +static int valid_count = 0; +static int test_no = 0; + +#define SWAP_IDX(a, b) \ + do { \ + equix_idx temp = a; \ + a = b; \ + b = temp; \ + } while(0) + +static bool test_alloc() { + ctx = equix_alloc(EQUIX_CTX_SOLVE); + assert(ctx != NULL && ctx != EQUIX_NOTSUPP); + return true; +} + +static bool test_free() { + equix_free(ctx); + return true; +} + +static bool test_solve() { + int num_solutions = 0; + for (nonce = 0; num_solutions == 0 && nonce < 20; ++nonce) { + num_solutions = equix_solve(ctx, &nonce, sizeof(nonce), solution); + } + --nonce; + assert(num_solutions > 0); + return true; +} + +static bool test_verify1() { + equix_result result = equix_verify(ctx, &nonce, sizeof(nonce), &solution[0]); + assert(result == EQUIX_OK); + return true; +} + +static bool test_verify2() { + SWAP_IDX(solution[0].idx[0], solution[0].idx[1]); + equix_result result = equix_verify(ctx, &nonce, sizeof(nonce), &solution[0]); + assert(result == EQUIX_ORDER); + return true; +} + +static bool test_verify3() { + SWAP_IDX(solution[0].idx[0], solution[0].idx[4]); + SWAP_IDX(solution[0].idx[1], solution[0].idx[5]); + SWAP_IDX(solution[0].idx[2], solution[0].idx[6]); + SWAP_IDX(solution[0].idx[3], solution[0].idx[7]); + equix_result result = equix_verify(ctx, &nonce, sizeof(nonce), &solution[0]); + assert(result == EQUIX_ORDER); + SWAP_IDX(solution[0].idx[0], solution[0].idx[4]); + SWAP_IDX(solution[0].idx[1], solution[0].idx[5]); + SWAP_IDX(solution[0].idx[2], solution[0].idx[6]); + SWAP_IDX(solution[0].idx[3], solution[0].idx[7]); + return true; +} + +static bool test_verify4() { + SWAP_IDX(solution[0].idx[1], solution[0].idx[2]); + equix_result result = equix_verify(ctx, &nonce, sizeof(nonce), &solution[0]); + assert(result == EQUIX_PARTIAL_SUM); + SWAP_IDX(solution[0].idx[1], solution[0].idx[2]); + return true; +} + +static void permute_idx(int start) { + if (start == EQUIX_NUM_IDX - 1) { + equix_result result = equix_verify(ctx, &nonce, sizeof(nonce), &solution[0]); + valid_count += result == EQUIX_OK; + } + else { + for (int i = start; i < EQUIX_NUM_IDX; ++i) { + SWAP_IDX(solution[0].idx[start], solution[0].idx[i]); + permute_idx(start + 1); + SWAP_IDX(solution[0].idx[start], solution[0].idx[i]); + } + } +} + +static bool test_permutations() { + permute_idx(0); + assert(valid_count == 1); /* check that only one of the 40320 possible + permutations of indices is a valid solution */ + return true; +} + +#define RUN_TEST(x) run_test(#x, &x) + +static void run_test(const char* name, test_func* func) { + printf("[%2i] %-40s ... ", ++test_no, name); + printf(func() ? "PASSED\n" : "SKIPPED\n"); +} + +int main() { + RUN_TEST(test_alloc); + RUN_TEST(test_solve); + RUN_TEST(test_verify1); + RUN_TEST(test_verify2); + RUN_TEST(test_verify3); + RUN_TEST(test_verify4); + RUN_TEST(test_permutations); + RUN_TEST(test_free); + + printf("\nAll tests were successful\n"); + return 0; +}

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit d79814f1b15368ef70f87bb696db6492e8a36c9e Author: David Goulet dgoulet@torproject.org AuthorDate: Mon Jun 27 14:04:06 2022 -0400

hs: PoW extension encoding

Signed-off-by: David Goulet dgoulet@torproject.org --- src/feature/hs/hs_cell.c | 79 +++++++++++++++++++++++++++++++++++++++++++++++- src/feature/hs/hs_cell.h | 2 ++ 2 files changed, 80 insertions(+), 1 deletion(-)

diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c index 490f05e54f..32da706a63 100644 --- a/src/feature/hs/hs_cell.c +++ b/src/feature/hs/hs_cell.c @@ -374,6 +374,79 @@ introduce1_encrypt_and_encode(trn_cell_introduce1_t *cell, tor_free(encrypted); }

+/** Build the PoW cell extension and put it in the given extensions object. + * Return 0 on success, -1 on failure. */ +static int +build_introduce_pow_extension(const hs_pow_solution_t *pow_solution, + trn_extension_t *extensions) +{ + ssize_t ret; + size_t pow_ext_encoded_len; + uint8_t *field_array; + trn_extension_field_t *field = NULL; + trn_cell_extension_pow_t *pow_ext = NULL; + + tor_assert(pow_solution); + tor_assert(extensions); + + /* We are creating a cell extension field of type PoW solution. */ + field = trn_extension_field_new(); + trn_extension_field_set_field_type(field, TRUNNEL_CELL_EXTENSION_TYPE_POW); + + /* Build PoW extension field. */ + pow_ext = trn_cell_extension_pow_new(); + + /* Copy PoW solution values into PoW extension cell. */ + + /* Equi-X base scheme */ + trn_cell_extension_pow_set_pow_version(pow_ext, TRUNNEL_POW_EQUIX); + + memcpy(trn_cell_extension_pow_getarray_pow_nonce(pow_ext), + &pow_solution->nonce, TRUNNEL_POW_NONCE_LEN); + + trn_cell_extension_pow_set_pow_effort(pow_ext, pow_solution->effort); + trn_cell_extension_pow_set_pow_seed(pow_ext, pow_solution->seed_head); + + memcpy(trn_cell_extension_pow_getarray_pow_solution(pow_ext), + &pow_solution->equix_solution, TRUNNEL_POW_SOLUTION_LEN); + + /* Set the field with the encoded PoW extension. */ + ret = trn_cell_extension_pow_encoded_len(pow_ext); + if (BUG(ret <= 0)) { + goto err; + } + pow_ext_encoded_len = ret; + + /* Set length field and the field array size length. */ + trn_extension_field_set_field_len(field, pow_ext_encoded_len); + trn_extension_field_setlen_field(field, pow_ext_encoded_len); + /* Encode the PoW extension into the cell extension field. */ + field_array = trn_extension_field_getarray_field(field); + ret = trn_cell_extension_pow_encode(field_array, + trn_extension_field_getlen_field(field), pow_ext); + if (BUG(ret <= 0)) { + goto err; + } + tor_assert(ret == (ssize_t)pow_ext_encoded_len); + + /* Finally, encode field into the cell extension. */ + trn_extension_add_fields(extensions, field); + + /* We've just add an extension field to the cell extensions so increment the + * total number. */ + trn_extension_set_num(extensions, trn_extension_get_num(extensions) + 1); + + /* Cleanup. PoW extension has been encoded at this point. */ + trn_cell_extension_pow_free(pow_ext); + + return 0; + +err: + trn_extension_field_free(field); + trn_cell_extension_pow_free(pow_ext); + return -1; +} + /** Build and set the INTRODUCE congestion control extension in the given * extensions. */ static void @@ -412,10 +485,14 @@ introduce1_set_encrypted(trn_cell_introduce1_t *cell, /* Setup extension(s) if any. */ ext = trn_extension_new(); tor_assert(ext); - /* Build congestion control extension is enabled. */ + /* Build congestion control extension if enabled. */ if (data->cc_enabled) { build_introduce_cc_extension(ext); } + /* Build PoW extension if present. */ + if (data->pow_solution) { + build_introduce_pow_extension(data->pow_solution, ext); + } trn_cell_introduce_encrypted_set_extensions(enc_cell, ext);

/* Set the rendezvous cookie. */ diff --git a/src/feature/hs/hs_cell.h b/src/feature/hs/hs_cell.h index c76a0690a8..538a135491 100644 --- a/src/feature/hs/hs_cell.h +++ b/src/feature/hs/hs_cell.h @@ -42,6 +42,8 @@ typedef struct hs_cell_introduce1_data_t { smartlist_t *link_specifiers; /** Congestion control parameters. */ unsigned int cc_enabled : 1; + /** PoW solution (Can be NULL if disabled). */ + const hs_pow_solution_t *pow_solution; } hs_cell_introduce1_data_t;

/** This data structure contains data that we need to parse an INTRODUCE2 cell

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 51ce0bb6ef60781cfe20fcaa16b36d438f264db7 Author: David Goulet dgoulet@torproject.org AuthorDate: Mon Jun 27 16:03:54 2022 -0400

hs: Add solve and verify PoW functions

Signed-off-by: David Goulet dgoulet@torproject.org --- configure.ac | 3 + src/app/include.am | 6 +- src/feature/hs/hs_pow.c | 280 ++++++++++++++++++++++++++++++++++++++++++++++ src/feature/hs/hs_pow.h | 9 ++ src/feature/hs/include.am | 1 + src/test/fuzz/include.am | 3 +- src/test/include.am | 16 +-- 7 files changed, 307 insertions(+), 11 deletions(-)

diff --git a/configure.ac b/configure.ac index c0b531c81d..babc65fa8f 100644 --- a/configure.ac +++ b/configure.ac @@ -403,6 +403,9 @@ AC_DEFUN([ADD_MODULE], [ m4_foreach_w([module], MODULES, [ADD_MODULE([module])]) AC_SUBST(TOR_MODULES_ALL_ENABLED)

+dnl Blake2 check for Equi-X support. +PKG_CHECK_MODULES([LIBB2], [libb2]) + dnl check for the correct "ar" when cross-compiling. dnl (AM_PROG_AR was new in automake 1.11.2, which we do not yet require, dnl so kludge up a replacement for the case where it isn't there yet.) diff --git a/src/app/include.am b/src/app/include.am index 5494d904a3..33365bfcac 100644 --- a/src/app/include.am +++ b/src/app/include.am @@ -20,7 +20,8 @@ src_app_tor_LDADD = libtor.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ - @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@ \ + @LIBB2_LIBS@

if COVERAGE_ENABLED src_app_tor_cov_SOURCES = $(src_app_tor_SOURCES) @@ -32,5 +33,6 @@ src_app_tor_cov_LDADD = src/test/libtor-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ \ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ - @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@ \ + @LIBB2_LIBS@ endif diff --git a/src/feature/hs/hs_pow.c b/src/feature/hs/hs_pow.c new file mode 100644 index 0000000000..2b36da93db --- /dev/null +++ b/src/feature/hs/hs_pow.c @@ -0,0 +1,280 @@ +/* Copyright (c) 2017-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file hs_pow.c + * \brief Contains code to handle proof-of-work computations + * when a hidden service is defending against DoS attacks. + **/ + +typedef unsigned __int128 uint128_t; + +#include <blake2.h> +#include <stdio.h> + +#include "ext/ht.h" +#include "feature/hs/hs_descriptor.h" +#include "feature/hs/hs_pow.h" +#include "lib/crypt_ops/crypto_rand.h" + +/** Replay cache set up */ +/** Cache entry for (nonce, seed) replay protection. */ +typedef struct nonce_cache_entry_t { + HT_ENTRY(nonce_cache_entry_t) node; + uint128_t nonce; + uint32_t seed_head; +} nonce_cache_entry_t; + +/** Return true if the two (nonce, seed) replay cache entries are the same */ +static inline int +nonce_cache_entries_eq_(const struct nonce_cache_entry_t *entry1, + const struct nonce_cache_entry_t *entry2) +{ + return entry1->nonce == entry2->nonce && + entry1->seed_head == entry2->seed_head; +} + +/** Hash function to hash the (nonce, seed) tuple entry. */ +static inline unsigned +nonce_cache_entry_hash_(const struct nonce_cache_entry_t *ent) +{ + return (unsigned)siphash24g(&ent->nonce, HS_POW_NONCE_LEN) + ent->seed_head; +} + +static HT_HEAD(nonce_cache_table_ht, nonce_cache_entry_t) + nonce_cache_table = HT_INITIALIZER(); + +HT_PROTOTYPE(nonce_cache_table_ht, nonce_cache_entry_t, node, + nonce_cache_entry_hash_, nonce_cache_entries_eq_); + +HT_GENERATE2(nonce_cache_table_ht, nonce_cache_entry_t, node, + nonce_cache_entry_hash_, nonce_cache_entries_eq_, 0.6, + tor_reallocarray_, tor_free_); + +/** We use this to check if an entry in the replay cache is for a particular + * seed head, so we know to remove it once the seed is no longer in use. */ +static int +nonce_cache_entry_has_seed(nonce_cache_entry_t *ent, void *data) +{ + /* Returning nonzero makes HT_FOREACH_FN remove the element from the HT */ + return ent->seed_head == *(uint32_t *)data; +} + +/** Helper: Increment a given nonce and set it in the challenge at the right + * offset. Use by the solve function. */ +static inline void +increment_and_set_nonce(uint128_t *nonce, uint8_t *challenge) +{ + (*nonce)++; + memcpy(challenge + HS_POW_SEED_LEN, nonce, HS_POW_NONCE_LEN); +} + +/* Helper: Build EquiX challenge (C || N || INT_32(E)) and return a newly + * allocated buffer containing it. */ +static uint8_t * +build_equix_challenge(const uint8_t *seed, const uint128_t nonce, + const uint32_t effort) +{ + /* Build EquiX challenge (C || N || INT_32(E)). */ + size_t offset = 0; + uint8_t *challenge = tor_malloc_zero(HS_POW_CHALLENGE_LEN); + + memcpy(challenge, seed, HS_POW_SEED_LEN); + offset += HS_POW_SEED_LEN; + memcpy(challenge + offset, &nonce, HS_POW_NONCE_LEN); + offset += HS_POW_NONCE_LEN; + set_uint32(challenge + offset, tor_htonl(effort)); + offset += HS_POW_EFFORT_LEN; + tor_assert(HS_POW_CHALLENGE_LEN == offset); + + return challenge; +} + +/** Helper: Return true iff the given challenge and solution for the given + * effort do validate as in: R * E <= UINT32_MAX. */ +static bool +validate_equix_challenge(const uint8_t *challenge, const equix_solution *sol, + const uint32_t effort) +{ + /* Fail if R * E > UINT32_MAX. */ + uint8_t hash_result[HS_POW_HASH_LEN]; + blake2b_state b2_state; + + if (BUG(blake2b_init(&b2_state, HS_POW_HASH_LEN) < 0)) { + return false; + } + + /* Construct: blake2b(C || N || E || S) */ + blake2b_update(&b2_state, challenge, HS_POW_CHALLENGE_LEN); + blake2b_update(&b2_state, (const uint8_t *) sol, HS_POW_EQX_SOL_LEN); + blake2b_final(&b2_state, hash_result, HS_POW_HASH_LEN); + + /* Scale to 64 bit so we can avoid 32 bit overflow. */ + uint64_t RE = tor_htonl(get_uint32(hash_result)) * (uint64_t) effort; + + return RE <= UINT32_MAX; +} + +/** Solve the EquiX/blake2b PoW scheme using the parameters in pow_params, and + * store the solution in pow_solution_out. Returns 0 on success and -1 + * otherwise. Called by a client. */ +int +hs_pow_solve(const hs_pow_desc_params_t *pow_params, + hs_pow_solution_t *pow_solution_out) +{ + int ret = -1; + uint128_t nonce; + uint8_t *challenge = NULL; + equix_ctx *ctx = NULL; + + tor_assert(pow_params); + tor_assert(pow_solution_out); + + /* Select E (just using suggested for now) */ + uint32_t effort = pow_params->suggested_effort; + + /* Generate a random nonce N. */ + crypto_rand((char *)&nonce, sizeof(uint128_t)); + + /* Build EquiX challenge (C || N || INT_32(E)). */ + challenge = build_equix_challenge(pow_params->seed, nonce, effort); + + ctx = equix_alloc(EQUIX_CTX_SOLVE); + equix_solution solution[EQUIX_MAX_SOLS]; + + /* We'll do a maximum of the nonce size iterations here which is the maximum + * number of nonce we can try in an attempt to find a valid solution. */ + log_debug(LD_REND, "Solving proof of work"); + for (uint64_t i = 0; i < UINT64_MAX; i++) { + /* Calculate S = equix_solve(C || N || E) */ + if (!equix_solve(ctx, challenge, HS_POW_CHALLENGE_LEN, solution)) { + ret = -1; + goto end; + } + const equix_solution *sol = &solution[0]; + + equix_result result = equix_verify(ctx, challenge, + HS_POW_CHALLENGE_LEN, sol); + if (result != EQUIX_OK) { + /* Go again with a new nonce. */ + increment_and_set_nonce(&nonce, challenge); + continue; + } + + /* Validate the challenge against the solution. */ + if (validate_equix_challenge(challenge, sol, effort)) { + /* Store the nonce N. */ + pow_solution_out->nonce = nonce; + /* Store the effort E. */ + pow_solution_out->effort = effort; + /* We only store the first 4 bytes of the seed C. */ + pow_solution_out->seed_head = get_uint32(pow_params->seed); + /* Store the solution S */ + memcpy(&pow_solution_out->equix_solution, sol, + sizeof(pow_solution_out->equix_solution)); + + /* Indicate success and we are done. */ + ret = 0; + break; + } + + /* Did not pass the R * E <= UINT32_MAX check. Increment the nonce and + * try again. */ + increment_and_set_nonce(&nonce, challenge); + } + + end: + tor_free(challenge); + equix_free(ctx); + return ret; +} + +/** Verify the solution in pow_solution using the service's current PoW + * parameters found in pow_state. Returns 0 on success and -1 otherwise. Called + * by the service. */ +int +hs_pow_verify(const hs_pow_service_state_t *pow_state, + const hs_pow_solution_t *pow_solution) +{ + int ret = -1; + uint8_t *challenge = NULL; + nonce_cache_entry_t search, *entry = NULL; + equix_ctx *ctx = NULL; + const uint8_t *seed = NULL; + + tor_assert(pow_state); + tor_assert(pow_solution); + + /* Notice, but don't fail, if E = POW_EFFORT is lower than the minimum + * effort. We will take whatever valid cells arrive, put them into the + * pqueue, and get to whichever ones we get to. */ + if (pow_solution->effort < pow_state->min_effort) { + log_info(LD_REND, "Effort %d used in solution is less than the minimum " + "effort %d required by the service. That's ok.", + pow_solution->effort, pow_state->min_effort); + } + + /* Find a valid seed C that starts with the seed head. Fail if no such seed + * exists. */ + if (get_uint32(pow_state->seed_current) == pow_solution->seed_head) { + seed = pow_state->seed_current; + } else if (get_uint32(pow_state->seed_previous) == pow_solution->seed_head) { + seed = pow_state->seed_previous; + } else { + log_err(LD_REND, "Seed head didn't match either seed."); + goto done; + } + + /* Fail if N = POW_NONCE is present in the replay cache. */ + search.nonce = pow_solution->nonce; + search.seed_head = pow_solution->seed_head; + entry = HT_FIND(nonce_cache_table_ht, &nonce_cache_table, &search); + if (entry) { + log_warn(LD_REND, "Found (nonce, seed) tuple in the replay cache."); + goto done; + } + + /* Build the challenge with the param we have. */ + challenge = build_equix_challenge(seed, pow_solution->nonce, + pow_solution->effort); + + if (!validate_equix_challenge(challenge, &pow_solution->equix_solution, + pow_solution->effort)) { + log_warn(LD_REND, "Equi-X solution and effort was too large."); + goto done; + } + + /* Fail if equix_verify(C || N || E, S) != EQUIX_OK */ + ctx = equix_alloc(EQUIX_CTX_SOLVE); + + equix_result result = equix_verify(ctx, challenge, HS_POW_CHALLENGE_LEN, + &pow_solution->equix_solution); + if (result != EQUIX_OK) { + log_warn(LD_REND, "Verification of EquiX solution in PoW failed."); + goto done; + } + + /* PoW verified successfully. */ + ret = 0; + + /* Add the (nonce, seed) tuple to the replay cache. */ + entry = tor_malloc_zero(sizeof(nonce_cache_entry_t)); + entry->nonce = pow_solution->nonce; + entry->seed_head = pow_solution->seed_head; + HT_INSERT(nonce_cache_table_ht, &nonce_cache_table, entry); + + done: + tor_free(challenge); + equix_free(ctx); + return ret; +} + +/** Remove entries from the (nonce, seed) replay cache which are for the seed + * beginning with seed_head. */ +void +hs_pow_remove_seed_from_cache(uint32_t seed) +{ + /* If nonce_cache_entry_has_seed returns 1, the entry is removed. */ + HT_FOREACH_FN(nonce_cache_table_ht, &nonce_cache_table, + nonce_cache_entry_has_seed, &seed); +} diff --git a/src/feature/hs/hs_pow.h b/src/feature/hs/hs_pow.h index 2ad5c2b04c..cd4f228565 100644 --- a/src/feature/hs/hs_pow.h +++ b/src/feature/hs/hs_pow.h @@ -110,4 +110,13 @@ typedef struct hs_pow_solution_t { equix_solution equix_solution; } hs_pow_solution_t;

+/* API */ +int hs_pow_solve(const hs_pow_desc_params_t *pow_params, + hs_pow_solution_t *pow_solution_out); + +int hs_pow_verify(const hs_pow_service_state_t *pow_state, + const hs_pow_solution_t *pow_solution); + +void hs_pow_remove_seed_from_cache(uint32_t seed); + #endif /* !defined(TOR_HS_POW_H) */ diff --git a/src/feature/hs/include.am b/src/feature/hs/include.am index efe85543cd..f4966e6c54 100644 --- a/src/feature/hs/include.am +++ b/src/feature/hs/include.am @@ -15,6 +15,7 @@ LIBTOR_APP_A_SOURCES += \ src/feature/hs/hs_intropoint.c \ src/feature/hs/hs_metrics.c \ src/feature/hs/hs_ob.c \ + src/feature/hs/hs_pow.c \ src/feature/hs/hs_service.c \ src/feature/hs/hs_stats.c \ src/feature/hs/hs_sys.c \ diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index 9fece7d004..a97dca1489 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -14,7 +14,8 @@ FUZZING_LIBS = \ @TOR_SYSTEMD_LIBS@ \ @TOR_LZMA_LIBS@ \ @TOR_ZSTD_LIBS@ \ - @TOR_TRACE_LIBS@ + @TOR_TRACE_LIBS@ \ + @LIBB2_LIBS@

oss-fuzz-prereqs: \ src/test/libtor-testing.a diff --git a/src/test/include.am b/src/test/include.am index deff450490..2ecea43333 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -301,7 +301,7 @@ src_test_test_switch_id_LDADD = \ $(TOR_UTIL_TESTING_LIBS) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_USERENV@ \ - @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@ @LIBB2_LIBS@ src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) \ @TOR_LDFLAGS_libevent@ src_test_test_LDADD = \ @@ -309,7 +309,7 @@ src_test_test_LDADD = \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ \ - @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@ + @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@ @LIBB2_LIBS@

src_test_test_slow_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS) @@ -337,7 +337,7 @@ src_test_bench_LDADD = \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ \ - @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@ + @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@ @LIBB2_LIBS@

src_test_test_workqueue_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) \ @TOR_LDFLAGS_libevent@ @@ -346,7 +346,7 @@ src_test_test_workqueue_LDADD = \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ \ - @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@ @LIBB2_LIBS@

src_test_test_timers_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_timers_CFLAGS = $(src_test_test_CFLAGS) @@ -357,7 +357,7 @@ src_test_test_timers_LDADD = \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ \ - @TOR_LZMA_LIBS@ @TOR_TRACE_LIBS@ + @TOR_LZMA_LIBS@ @TOR_TRACE_LIBS@ @LIBB2_LIBS@ src_test_test_timers_LDFLAGS = $(src_test_test_LDFLAGS)

# ADD_C_FILE: INSERT HEADERS HERE. @@ -391,7 +391,7 @@ src_test_test_ntor_cl_LDADD = \ libtor.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ - @CURVE25519_LIBS@ @TOR_LZMA_LIBS@ @TOR_TRACE_LIBS@ + @CURVE25519_LIBS@ @TOR_LZMA_LIBS@ @TOR_TRACE_LIBS@ @LIBB2_LIBS@ src_test_test_ntor_cl_AM_CPPFLAGS = \ $(AM_CPPFLAGS)

@@ -401,7 +401,7 @@ src_test_test_hs_ntor_cl_LDADD = \ libtor.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ \ - @CURVE25519_LIBS@ @TOR_TRACE_LIBS@ + @CURVE25519_LIBS@ @TOR_TRACE_LIBS@ @LIBB2_LIBS@ src_test_test_hs_ntor_cl_AM_CPPFLAGS = \ $(AM_CPPFLAGS)

@@ -413,7 +413,7 @@ src_test_test_bt_cl_LDADD = \ $(TOR_UTIL_TESTING_LIBS) \ @TOR_LIB_MATH@ \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ - @TOR_TRACE_LIBS@ + @TOR_TRACE_LIBS@ @LIBB2_LIBS@ src_test_test_bt_cl_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_test_test_bt_cl_CPPFLAGS= $(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS) endif

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 8b41e09a775e882096364210317813c830160a5b Author: David Goulet dgoulet@torproject.org AuthorDate: Tue Jun 28 11:42:35 2022 -0400

hs: Client now solve PoW if present

At this commit, the tor main loop solves it. We might consider moving this to the CPU pool at some point.

Signed-off-by: David Goulet dgoulet@torproject.org --- src/core/or/origin_circuit_st.h | 6 ++++++ src/feature/hs/hs_circuit.c | 6 +++++- src/feature/hs/hs_circuit.h | 3 ++- src/feature/hs/hs_client.c | 19 ++++++++++++++++++- src/test/test_hs_service.c | 8 ++++---- 5 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/src/core/or/origin_circuit_st.h b/src/core/or/origin_circuit_st.h index c5c255bb49..fd5424c450 100644 --- a/src/core/or/origin_circuit_st.h +++ b/src/core/or/origin_circuit_st.h @@ -212,6 +212,12 @@ struct origin_circuit_t { * (in host byte order) for response comparison. */ uint32_t pathbias_probe_nonce;

+ /** Set iff this is a hidden-service circuit for a HS with PoW defenses + * enabled, so that we know to be more lenient with timing out the + * circuit-build to allow the service time to work through the queue of + * requests. */ + unsigned int hs_with_pow_circ : 1; + /** Set iff this circuit has been given a relaxed timeout because * no circuits have opened. Used to prevent spamming logs. */ unsigned int relaxed_timeout : 1; diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 006ba964fe..3f8f16955f 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -1095,7 +1095,8 @@ int hs_circ_send_introduce1(origin_circuit_t *intro_circ, origin_circuit_t *rend_circ, const hs_desc_intro_point_t *ip, - const hs_subcredential_t *subcredential) + const hs_subcredential_t *subcredential, + const hs_pow_solution_t *pow_solution) { int ret = -1; ssize_t payload_len; @@ -1129,6 +1130,9 @@ hs_circ_send_introduce1(origin_circuit_t *intro_circ, goto close; }

+ /* Set the PoW solution if any. */ + intro1_data.pow_solution = pow_solution; + /* If the rend circ was set up for congestion control, add that to the * intro data, to signal it in an extension */ if (TO_CIRCUIT(rend_circ)->ccontrol) { diff --git a/src/feature/hs/hs_circuit.h b/src/feature/hs/hs_circuit.h index afbff7b894..3c84abaad2 100644 --- a/src/feature/hs/hs_circuit.h +++ b/src/feature/hs/hs_circuit.h @@ -55,7 +55,8 @@ int hs_circ_handle_introduce2(const hs_service_t *service, int hs_circ_send_introduce1(origin_circuit_t *intro_circ, origin_circuit_t *rend_circ, const hs_desc_intro_point_t *ip, - const struct hs_subcredential_t *subcredential); + const struct hs_subcredential_t *subcredential, + const hs_pow_solution_t *pow_solution); int hs_circ_send_establish_rendezvous(origin_circuit_t *circ);

/* e2e circuit API. */ diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index 7cee3480d5..e241e6218d 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -613,6 +613,7 @@ send_introduce1(origin_circuit_t *intro_circ, char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1]; const ed25519_public_key_t *service_identity_pk = NULL; const hs_desc_intro_point_t *ip; + hs_pow_solution_t *pow_solution = NULL;

tor_assert(rend_circ); if (intro_circ_is_ok(intro_circ) < 0) { @@ -668,9 +669,24 @@ send_introduce1(origin_circuit_t *intro_circ, goto perm_err; }

+ /* If the descriptor contains PoW parameters then the service is + * expecting a PoW solution in the INTRODUCE cell, which we solve here. */ + if (desc->encrypted_data.pow_params) { + log_debug(LD_REND, "PoW params present in descriptor."); + pow_solution = tor_malloc_zero(sizeof(hs_pow_solution_t)); + if (hs_pow_solve(desc->encrypted_data.pow_params, pow_solution)) { + log_warn(LD_REND, "Haven't solved the PoW yet."); + goto tran_err; + } + /* Set flag to reflect that the HS we are attempting to rendezvous has PoW + * defenses enabled, and as such we will need to be more lenient with + * timing out while waiting for the circuit to be built. */ + rend_circ->hs_with_pow_circ = 1; + } + /* Send the INTRODUCE1 cell. */ if (hs_circ_send_introduce1(intro_circ, rend_circ, ip, - &desc->subcredential) < 0) { + &desc->subcredential, pow_solution) < 0) { if (TO_CIRCUIT(intro_circ)->marked_for_close) { /* If the introduction circuit was closed, we were unable to send the * cell for some reasons. In any case, the intro circuit has to be @@ -724,6 +740,7 @@ send_introduce1(origin_circuit_t *intro_circ,

end: memwipe(onion_address, 0, sizeof(onion_address)); + tor_free(pow_solution); return status; }

diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index 03a4800f25..4a8a758b3f 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -2406,7 +2406,7 @@ test_intro2_handling(void *arg) /* Create INTRODUCE1 */ tt_assert(fast_mem_is_zero(relay_payload, sizeof(relay_payload))); retval = hs_circ_send_introduce1(intro_circ, &rend_circ, - alice_ip, &x_subcred); + alice_ip, &x_subcred, NULL);

/* Check that the payload was written successfully */ tt_int_op(retval, OP_EQ, 0); @@ -2447,7 +2447,7 @@ test_intro2_handling(void *arg) /* Create INTRODUCE1 from Alice to X through Z */ memset(relay_payload, 0, sizeof(relay_payload)); retval = hs_circ_send_introduce1(intro_circ, &rend_circ, - alice_ip, &z_subcred); + alice_ip, &z_subcred, NULL);

/* Check that the payload was written successfully */ tt_int_op(retval, OP_EQ, 0); @@ -2484,7 +2484,7 @@ test_intro2_handling(void *arg) /* Create INTRODUCE1 from Alice to X using X's subcred. */ memset(relay_payload, 0, sizeof(relay_payload)); retval = hs_circ_send_introduce1(intro_circ, &rend_circ, - alice_ip, &x_subcred); + alice_ip, &x_subcred, NULL);

/* Check that the payload was written successfully */ tt_int_op(retval, OP_EQ, 0); @@ -2577,7 +2577,7 @@ test_intro2_handling(void *arg) * service!) */ memset(relay_payload, 0, sizeof(relay_payload)); retval = hs_circ_send_introduce1(intro_circ, &rend_circ, - alice_ip, &y_subcred); + alice_ip, &y_subcred, NULL); tt_int_op(retval, OP_EQ, 0);

/* Check that the payload was written successfully */

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit f0b63ca242a66cb5172e6b11a9f068ed348f601b Author: David Goulet dgoulet@torproject.org AuthorDate: Wed Jun 29 11:05:35 2022 -0400

hs: Move rendezvous circuit data structure

When parsing an INTRODUCE2 cell, we extract data in order to launch the rendezvous circuit. This commit creates a data structure just for that data so it can be used by future commits for prop327 in order to copy that data over a priority queue instead of the whole intro data data structure which contains pointers that could dissapear.

Signed-off-by: David Goulet dgoulet@torproject.org --- src/feature/hs/hs_cell.c | 23 ++++++++++++----------- src/feature/hs/hs_cell.h | 27 +++++++++++++++++---------- src/feature/hs/hs_circuit.c | 25 ++++++++++++++----------- 3 files changed, 43 insertions(+), 32 deletions(-)

diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c index 32da706a63..603d997c42 100644 --- a/src/feature/hs/hs_cell.c +++ b/src/feature/hs/hs_cell.c @@ -812,7 +812,7 @@ get_introduce2_keys_and_verify_mac(hs_cell_introduce2_data_t *data, data->n_subcredentials, data->subcredentials, encrypted_section, - &data->client_pk); + &data->rdv_data.client_pk); if (intro_keys == NULL) { log_info(LD_REND, "Invalid INTRODUCE2 encrypted data. Unable to " "compute key material"); @@ -875,9 +875,9 @@ parse_introduce_cell_extension(hs_cell_introduce2_data_t *data, switch (trn_extension_field_get_field_type(field)) { case TRUNNEL_EXT_TYPE_CC_FIELD_REQUEST: /* CC requests, enable it. */ - data->cc_enabled = 1; + data->rdv_data.cc_enabled = 1; data->pv.protocols_known = 1; - data->pv.supports_congestion_control = data->cc_enabled; + data->pv.supports_congestion_control = data->rdv_data.cc_enabled; break; default: break; @@ -944,7 +944,7 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data, * guaranteed to exist because of the length check above). We are gonna use * the client public key to compute the ntor keys and decrypt the payload: */ - memcpy(&data->client_pk.public_key, encrypted_section, + memcpy(&data->rdv_data.client_pk.public_key, encrypted_section, CURVE25519_PUBKEY_LEN);

/* Get the right INTRODUCE2 ntor keys and verify the cell MAC */ @@ -960,12 +960,13 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data, { /* The ENCRYPTED_DATA section starts just after the CLIENT_PK. */ const uint8_t *encrypted_data = - encrypted_section + sizeof(data->client_pk); + encrypted_section + sizeof(data->rdv_data.client_pk); /* It's symmetric encryption so it's correct to use the ENCRYPTED length * for decryption. Computes the length of ENCRYPTED_DATA meaning removing * the CLIENT_PK and MAC length. */ size_t encrypted_data_len = - encrypted_section_len - (sizeof(data->client_pk) + DIGEST256_LEN); + encrypted_section_len - + (sizeof(data->rdv_data.client_pk) + DIGEST256_LEN);

/* This decrypts the ENCRYPTED_DATA section of the cell. */ decrypted = decrypt_introduce2(intro_keys->enc_key, @@ -992,12 +993,12 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,

/* Extract onion key and rendezvous cookie from the cell used for the * rendezvous point circuit e2e encryption. */ - memcpy(data->onion_pk.public_key, + memcpy(data->rdv_data.onion_pk.public_key, trn_cell_introduce_encrypted_getconstarray_onion_key(enc_cell), CURVE25519_PUBKEY_LEN); - memcpy(data->rendezvous_cookie, + memcpy(data->rdv_data.rendezvous_cookie, trn_cell_introduce_encrypted_getconstarray_rend_cookie(enc_cell), - sizeof(data->rendezvous_cookie)); + sizeof(data->rdv_data.rendezvous_cookie));

/* Extract rendezvous link specifiers. */ for (size_t idx = 0; @@ -1011,7 +1012,7 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data, if (BUG(!lspec_dup)) { goto done; } - smartlist_add(data->link_specifiers, lspec_dup); + smartlist_add(data->rdv_data.link_specifiers, lspec_dup); }

/* Extract any extensions. */ @@ -1031,7 +1032,7 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,

/* If the client asked for congestion control, but we don't support it, * that's a failure. It should not have asked, based on our descriptor. */ - if (data->cc_enabled && !congestion_control_enabled()) { + if (data->rdv_data.cc_enabled && !congestion_control_enabled()) { goto done; }

diff --git a/src/feature/hs/hs_cell.h b/src/feature/hs/hs_cell.h index 2735401c05..61c0a94b20 100644 --- a/src/feature/hs/hs_cell.h +++ b/src/feature/hs/hs_cell.h @@ -47,6 +47,21 @@ typedef struct hs_cell_introduce1_data_t { const hs_pow_solution_t *pow_solution; } hs_cell_introduce1_data_t;

+/** Introduction data needed to launch a rendezvous circuit. This is set after + * receiving an INTRODUCE2 valid cell. */ +typedef struct hs_cell_intro_rdv_data_t { + /** Onion public key computed using the INTRODUCE2 encrypted section. */ + curve25519_public_key_t onion_pk; + /** Rendezvous cookie taken from the INTRODUCE2 encrypted section. */ + uint8_t rendezvous_cookie[REND_COOKIE_LEN]; + /** Client public key from the INTRODUCE2 encrypted section. */ + curve25519_public_key_t client_pk; + /** Link specifiers of the rendezvous point. Contains link_specifier_t. */ + smartlist_t *link_specifiers; + /** Congestion control parameters. */ + unsigned int cc_enabled : 1; +} hs_cell_intro_rdv_data_t; + /** This data structure contains data that we need to parse an INTRODUCE2 cell * which is used by the INTRODUCE2 cell parsing function. On a successful * parsing, the onion_pk and rendezvous_cookie will be populated with the @@ -77,20 +92,12 @@ typedef struct hs_cell_introduce2_data_t {

/*** Mutable Section: Set upon parsing INTRODUCE2 cell. ***/

- /** Onion public key computed using the INTRODUCE2 encrypted section. */ - curve25519_public_key_t onion_pk; - /** Rendezvous cookie taken from the INTRODUCE2 encrypted section. */ - uint8_t rendezvous_cookie[REND_COOKIE_LEN]; - /** Client public key from the INTRODUCE2 encrypted section. */ - curve25519_public_key_t client_pk; - /** Link specifiers of the rendezvous point. Contains link_specifier_t. */ - smartlist_t *link_specifiers; + /** Data needed to launch a rendezvous circuit. */ + hs_cell_intro_rdv_data_t rdv_data; /** Replay cache of the introduction point. */ replaycache_t *replay_cache; /** Flow control negotiation parameters. */ protover_summary_flags_t pv; - /** Congestion control parameters. */ - unsigned int cc_enabled : 1; } hs_cell_introduce2_data_t;

/* Build cell API. */ diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 3f8f16955f..835cd366ad 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -326,8 +326,8 @@ launch_rendezvous_point_circuit,(const hs_service_t *service,

/* Get the extend info data structure for the chosen rendezvous point * specified by the given link specifiers. */ - info = hs_get_extend_info_from_lspecs(data->link_specifiers, - &data->onion_pk, + info = hs_get_extend_info_from_lspecs(data->rdv_data.link_specifiers, + &data->rdv_data.onion_pk, service->config.is_single_onion); if (info == NULL) { /* We are done here, we can't extend to the rendezvous point. */ @@ -374,7 +374,8 @@ launch_rendezvous_point_circuit,(const hs_service_t *service, log_info(LD_REND, "Rendezvous circuit launched to %s with cookie %s " "for %s service %s", safe_str_client(extend_info_describe(info)), - safe_str_client(hex_str((const char *) data->rendezvous_cookie, + safe_str_client(hex_str((const char *) + data->rdv_data.rendezvous_cookie, REND_COOKIE_LEN)), get_service_anonymity_string(service), safe_str_client(service->onion_address)); @@ -393,7 +394,8 @@ launch_rendezvous_point_circuit,(const hs_service_t *service, curve25519_keypair_generate(&ephemeral_kp, 0); if (hs_ntor_service_get_rendezvous1_keys(&ip->auth_key_kp.pubkey, &ip->enc_key_kp, - &ephemeral_kp, &data->client_pk, + &ephemeral_kp, + &data->rdv_data.client_pk, &keys) < 0) { /* This should not really happened but just in case, don't make tor * freak out, close the circuit and move on. */ @@ -404,15 +406,15 @@ launch_rendezvous_point_circuit,(const hs_service_t *service, goto end; } circ->hs_ident = create_rp_circuit_identifier(service, - data->rendezvous_cookie, - &ephemeral_kp.pubkey, &keys); + data->rdv_data.rendezvous_cookie, + &ephemeral_kp.pubkey, &keys); memwipe(&ephemeral_kp, 0, sizeof(ephemeral_kp)); memwipe(&keys, 0, sizeof(keys)); tor_assert(circ->hs_ident); }

/* Setup congestion control if asked by the client from the INTRO cell. */ - if (data->cc_enabled) { + if (data->rdv_data.cc_enabled) { hs_circ_setup_congestion_control(circ, congestion_control_sendme_inc(), service->config.is_single_onion); } @@ -1003,9 +1005,9 @@ hs_circ_handle_introduce2(const hs_service_t *service, data.enc_kp = &ip->enc_key_kp; data.payload = payload; data.payload_len = payload_len; - data.link_specifiers = smartlist_new(); data.replay_cache = ip->replay_cache; - data.cc_enabled = 0; + data.rdv_data.link_specifiers = smartlist_new(); + data.rdv_data.cc_enabled = 0;

if (get_subcredential_for_handling_intro2_cell(service, &data, subcredential)) { @@ -1022,7 +1024,8 @@ hs_circ_handle_introduce2(const hs_service_t *service, /* Check whether we've seen this REND_COOKIE before to detect repeats. */ if (replaycache_add_test_and_elapsed( service->state.replay_cache_rend_cookie, - data.rendezvous_cookie, sizeof(data.rendezvous_cookie), + data.rdv_data.rendezvous_cookie, + sizeof(data.rdv_data.rendezvous_cookie), &elapsed)) { /* A Tor client will send a new INTRODUCE1 cell with the same REND_COOKIE * as its previous one if its intro circ times out while in state @@ -1048,7 +1051,7 @@ hs_circ_handle_introduce2(const hs_service_t *service, ret = 0;

done: - link_specifier_smartlist_free(data.link_specifiers); + link_specifier_smartlist_free(data.rdv_data.link_specifiers); memwipe(&data, 0, sizeof(data)); return ret; }

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 35227a7a15dd1ff2e993c21b2a5da8d6498c0a3e Author: David Goulet dgoulet@torproject.org AuthorDate: Wed Jun 29 12:00:41 2022 -0400

trunnel: Centralize the INTRO1 extension type

Signed-off-by: David Goulet dgoulet@torproject.org --- src/feature/hs/hs_cell.c | 4 ++-- src/trunnel/hs/cell_introduce1.h | 1 + src/trunnel/hs/cell_introduce1.trunnel | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c index 004a7fcbe1..ab9283dc1b 100644 --- a/src/feature/hs/hs_cell.c +++ b/src/feature/hs/hs_cell.c @@ -457,7 +457,7 @@ build_introduce_cc_extension(trn_extension_t *extensions) /* Build CC request extension. */ field = trn_extension_field_new(); trn_extension_field_set_field_type(field, - TRUNNEL_EXT_TYPE_CC_FIELD_REQUEST); + TRUNNEL_EXT_TYPE_CC_REQUEST);

/* No payload indicating a request to use congestion control. */ trn_extension_field_set_field_len(field, 0); @@ -933,7 +933,7 @@ parse_introduce_cell_extension(const hs_service_t *service, tor_assert(field);

switch (trn_extension_field_get_field_type(field)) { - case TRUNNEL_EXT_TYPE_CC_FIELD_REQUEST: + case TRUNNEL_EXT_TYPE_CC_REQUEST: /* CC requests, enable it. */ data->rdv_data.cc_enabled = 1; data->pv.protocols_known = 1; diff --git a/src/trunnel/hs/cell_introduce1.h b/src/trunnel/hs/cell_introduce1.h index 90d34f37f2..827c107b6b 100644 --- a/src/trunnel/hs/cell_introduce1.h +++ b/src/trunnel/hs/cell_introduce1.h @@ -19,6 +19,7 @@ struct link_specifier_st; #define TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY1 1 #define TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519 2 #define TRUNNEL_HS_INTRO_ONION_KEY_TYPE_NTOR 1 +#define TRUNNEL_EXT_TYPE_CC_REQUEST 1 #define TRUNNEL_EXT_TYPE_POW 2 #define TRUNNEL_POW_NONCE_LEN 16 #define TRUNNEL_POW_SOLUTION_LEN 16 diff --git a/src/trunnel/hs/cell_introduce1.trunnel b/src/trunnel/hs/cell_introduce1.trunnel index 35e00bed94..a92fc76ab5 100644 --- a/src/trunnel/hs/cell_introduce1.trunnel +++ b/src/trunnel/hs/cell_introduce1.trunnel @@ -78,6 +78,8 @@ struct trn_cell_introduce_encrypted { * INTRODUCE1 cell (encrypted section) extensions. */

+/* Cell extenstion type Congestion Control Request. */ +const TRUNNEL_EXT_TYPE_CC_REQUEST = 0x01; /* Cell extension type PoW. */ const TRUNNEL_EXT_TYPE_POW = 0x02;

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit bc9fe5a6f851a0749c141d3f1d4673ac12e3ca39 Author: David Goulet dgoulet@torproject.org AuthorDate: Thu Jun 30 09:53:41 2022 -0400

hs: Handle multiple rend request per mainloop run

Signed-off-by: David Goulet dgoulet@torproject.org --- src/feature/hs/hs_circuit.c | 51 +++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 18 deletions(-)

diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index cbb3e0bfdd..3d9892e482 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -617,32 +617,47 @@ compare_rend_request_by_effort_(const void *_a, const void *_b) } }

+/** How many rendezvous request we handle per mainloop event. Per prop327, + * handling an INTRODUCE2 cell takes on average 5.56msec on an average CPU and + * so it means that launching this max amount of circuits is well below 0.08 + * seconds which we believe is negligable on the whole mainloop. */ +#define MAX_REND_REQUEST_PER_MAINLOOP 16 + static void handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) { - (void) ev; /* Not using the returned event, make compiler happy. */ + int count = 0; hs_service_t *service = arg; hs_pow_service_state_t *pow_state = service->state.pow_state;

- /* Pop next request by effort. */ - pending_rend_t *req = - smartlist_pqueue_pop(pow_state->rend_request_pqueue, - compare_rend_request_by_effort_, - offsetof(pending_rend_t, idx)); - - log_info(LD_REND, "Dequeued pending rendezvous request with effort: %u. " - "Remaining requests: %u", - req->rdv_data.pow_effort, - smartlist_len(pow_state->rend_request_pqueue)); + (void) ev; /* Not using the returned event, make compiler happy. */

- /* Launch the rendezvous circuit. */ - launch_rendezvous_point_circuit(service, &req->ip_auth_pubkey, - &req->ip_enc_key_kp, &req->rdv_data); + /* Process rendezvous request until the maximum per mainloop run. */ + while (smartlist_len(pow_state->rend_request_pqueue) > 0) { + if (++count == MAX_REND_REQUEST_PER_MAINLOOP) { + break; + }

- /* Clean memory. */ - link_specifier_smartlist_free(req->rdv_data.link_specifiers); - memwipe(req, 0, sizeof(pending_rend_t)); - tor_free(req); + /* Pop next request by effort. */ + pending_rend_t *req = + smartlist_pqueue_pop(pow_state->rend_request_pqueue, + compare_rend_request_by_effort_, + offsetof(pending_rend_t, idx)); + + log_info(LD_REND, "Dequeued pending rendezvous request with effort: %u. " + "Remaining requests: %u", + req->rdv_data.pow_effort, + smartlist_len(pow_state->rend_request_pqueue)); + + /* Launch the rendezvous circuit. */ + launch_rendezvous_point_circuit(service, &req->ip_auth_pubkey, + &req->ip_enc_key_kp, &req->rdv_data); + + /* Clean memory. */ + link_specifier_smartlist_free(req->rdv_data.link_specifiers); + memwipe(req, 0, sizeof(pending_rend_t)); + tor_free(req); + }

/* If there are still some pending rendezvous circuits in the pqueue then * reschedule the event in order to continue handling them. */

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 4571faf0c343e3224c0113225eca14e3eacde7b2 Author: Roger Dingledine arma@torproject.org AuthorDate: Thu Jun 30 16:10:56 2022 -0400

pass time around as a parameter

should help with unit testing --- src/feature/hs/hs_circuit.c | 22 +++++++++++----------- src/feature/hs/hs_circuit.h | 3 ++- src/test/test_hs_service.c | 4 +++- 3 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index da852d7107..f111462276 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -325,10 +325,10 @@ MOCK_IMPL(STATIC void, launch_rendezvous_point_circuit,(const hs_service_t *service, const ed25519_public_key_t *ip_auth_pubkey, const curve25519_keypair_t *ip_enc_key_kp, - const hs_cell_intro_rdv_data_t *rdv_data)) + const hs_cell_intro_rdv_data_t *rdv_data, + time_t now)) { int circ_needs_uptime; - time_t now = time(NULL); extend_info_t *info = NULL; origin_circuit_t *circ;

@@ -632,10 +632,8 @@ compare_rend_request_by_effort_(const void *_a, const void *_b)

/** Remove too old entries from the given rendezvous request priority queue. */ static void -trim_rend_pqueue(smartlist_t *pqueue) +trim_rend_pqueue(smartlist_t *pqueue, time_t now) { - time_t now = time(NULL); - SMARTLIST_FOREACH_BEGIN(pqueue, pending_rend_t *, req) { if ((req->enqueued_ts + MAX_REND_TIMEOUT) < now) { SMARTLIST_DEL_CURRENT_KEEPORDER(pqueue, req); @@ -656,12 +654,13 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) int count = 0; hs_service_t *service = arg; hs_pow_service_state_t *pow_state = service->state.pow_state; + time_t now = time(NULL);

(void) ev; /* Not using the returned event, make compiler happy. */

/* Before we process rendezvous request, trim the list to remove out dated * entries. */ - trim_rend_pqueue(pow_state->rend_request_pqueue); + trim_rend_pqueue(pow_state->rend_request_pqueue, now);

/* Process rendezvous request until the maximum per mainloop run. */ while (smartlist_len(pow_state->rend_request_pqueue) > 0) { @@ -682,7 +681,7 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg)

/* Launch the rendezvous circuit. */ launch_rendezvous_point_circuit(service, &req->ip_auth_pubkey, - &req->ip_enc_key_kp, &req->rdv_data); + &req->ip_enc_key_kp, &req->rdv_data, now); free_pending_rend(req); }

@@ -705,7 +704,7 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) * Return 0 if we successfully enqueued the request else -1. */ static int enqueue_rend_request(const hs_service_t *service, hs_service_intro_point_t *ip, - hs_cell_introduce2_data_t *data) + hs_cell_introduce2_data_t *data, time_t now) { hs_pow_service_state_t *pow_state = NULL; pending_rend_t *req = NULL; @@ -733,7 +732,7 @@ enqueue_rend_request(const hs_service_t *service, hs_service_intro_point_t *ip, * doesn't get freed under us. */ data->rdv_data.link_specifiers = NULL; req->idx = -1; - req->enqueued_ts = time(NULL); + req->enqueued_ts = now;

/* Enqueue the rendezvous request. */ smartlist_pqueue_add(pow_state->rend_request_pqueue, @@ -1149,6 +1148,7 @@ hs_circ_handle_introduce2(const hs_service_t *service, int ret = -1; time_t elapsed; hs_cell_introduce2_data_t data; + time_t now = time(NULL);

tor_assert(service); tor_assert(circ); @@ -1208,7 +1208,7 @@ hs_circ_handle_introduce2(const hs_service_t *service, if (service->config.has_pow_defenses_enabled) { log_debug(LD_REND, "Adding introduction request to pqueue with effort: %u", data.rdv_data.pow_effort); - if (enqueue_rend_request(service, ip, &data) < 0) { + if (enqueue_rend_request(service, ip, &data, now) < 0) { goto done; }

@@ -1222,7 +1222,7 @@ hs_circ_handle_introduce2(const hs_service_t *service,

/* Launch rendezvous circuit with the onion key and rend cookie. */ launch_rendezvous_point_circuit(service, &ip->auth_key_kp.pubkey, - &ip->enc_key_kp, &data.rdv_data); + &ip->enc_key_kp, &data.rdv_data, now); /* Success. */ ret = 0;

diff --git a/src/feature/hs/hs_circuit.h b/src/feature/hs/hs_circuit.h index cbd48c5b33..d61ddcede8 100644 --- a/src/feature/hs/hs_circuit.h +++ b/src/feature/hs/hs_circuit.h @@ -106,7 +106,8 @@ MOCK_DECL(STATIC void, launch_rendezvous_point_circuit,(const hs_service_t *service, const ed25519_public_key_t *ip_auth_pubkey, const curve25519_keypair_t *ip_enc_key_kp, - const hs_cell_intro_rdv_data_t *rdv_data)); + const hs_cell_intro_rdv_data_t *rdv_data, + time_t now));

#endif /* defined(HS_CIRCUIT_PRIVATE) */

diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index eb905714c1..dc60c7ca29 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -2281,12 +2281,14 @@ static void mock_launch_rendezvous_point_circuit(const hs_service_t *service, const ed25519_public_key_t *ip_auth_pubkey, const curve25519_keypair_t *ip_enc_key_kp, - const hs_cell_intro_rdv_data_t *rdv_data) + const hs_cell_intro_rdv_data_t *rdv_data, + time_t now) { (void) service; (void) ip_auth_pubkey; (void) ip_enc_key_kp; (void) rdv_data; + (void) now; return; }

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 8042379c44abff3510ad4552cb27ad7d08538410 Author: Roger Dingledine arma@torproject.org AuthorDate: Thu Jun 30 17:25:16 2022 -0400

new design for handling too many pending rend reqs

now we let ourselves queue up to twice as many as we expect, and when we get to the limit we make a new pqueue and move over the first n elements that we like most.

(the old approach, of calling SMARTLIST_DEL_CURRENT_KEEPORDER() on elements in a pqueue, will destroy its heapify property.)

we also discard elements that are too old, either during the trimming process or if they come up as the next request to respond to.

lastly, fix a fencepost error on how many rend reqs we would handle per iteration. --- src/feature/hs/hs_circuit.c | 93 +++++++++++++++++++++++++++++++++------------ 1 file changed, 69 insertions(+), 24 deletions(-)

diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index f111462276..26163b5ca7 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -630,16 +630,63 @@ compare_rend_request_by_effort_(const void *_a, const void *_b) } }

-/** Remove too old entries from the given rendezvous request priority queue. */ +/** Return 1 if a request waiting in our service-side pqueue is old + * enough that we should just discard it rather than trying to respond, + * or 0 if we still like it. As a heuristic, choose half of the total + * permitted time interval (so we don't approve trying to respond to + * requests when we will then give up on them a moment later). + */ +static int +queued_rend_request_is_too_old(pending_rend_t *req, time_t now) +{ + if ((req->enqueued_ts + MAX_REND_TIMEOUT/2) < now) + return 1; + return 0; +} + +/** Maximum number of rendezvous requests we enqueue per service. We allow the + * average amount of INTRODUCE2 that a service can process in a second times + * the rendezvous timeout. Then we let it grow to twice that before + * discarding the bottom half in trim_rend_pqueue(). */ +#define QUEUED_REND_REQUEST_HIGH_WATER (2 * 180 * MAX_REND_TIMEOUT) + +/** Our rendezvous request priority queue is too full; keep the first + * QUEUED_REND_REQUEST_HIGH_WATER/2 entries and discard the rest. + */ static void -trim_rend_pqueue(smartlist_t *pqueue, time_t now) +trim_rend_pqueue(hs_pow_service_state_t *pow_state, time_t now) { - SMARTLIST_FOREACH_BEGIN(pqueue, pending_rend_t *, req) { - if ((req->enqueued_ts + MAX_REND_TIMEOUT) < now) { - SMARTLIST_DEL_CURRENT_KEEPORDER(pqueue, req); + smartlist_t *old_pqueue = pow_state->rend_request_pqueue; + smartlist_t *new_pqueue = pow_state->rend_request_pqueue = smartlist_new(); + + log_notice(LD_REND, "Rendezvous request priority queue has " + "reached capacity (%d). Discarding the bottom half.", + smartlist_len(old_pqueue)); + + while (smartlist_len(old_pqueue) && + smartlist_len(new_pqueue) < QUEUED_REND_REQUEST_HIGH_WATER/2) { + /* while there are still old ones, and the new one isn't full yet */ + pending_rend_t *req = + smartlist_pqueue_pop(old_pqueue, + compare_rend_request_by_effort_, + offsetof(pending_rend_t, idx)); + if (queued_rend_request_is_too_old(req, now)) { + log_info(LD_REND, "While trimming, rend request has been pending " + "for too long; discarding."); free_pending_rend(req); + } else { + smartlist_pqueue_add(new_pqueue, + compare_rend_request_by_effort_, + offsetof(pending_rend_t, idx), req); } + } + + /* Ok, we have rescued all the entries we want to keep. The rest are + * all excess. */ + SMARTLIST_FOREACH_BEGIN(old_pqueue, pending_rend_t *, req) { + free_pending_rend(req); } SMARTLIST_FOREACH_END(req); + smartlist_free(old_pqueue); }

/** How many rendezvous request we handle per mainloop event. Per prop327, @@ -658,16 +705,8 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg)

(void) ev; /* Not using the returned event, make compiler happy. */

- /* Before we process rendezvous request, trim the list to remove out dated - * entries. */ - trim_rend_pqueue(pow_state->rend_request_pqueue, now); - /* Process rendezvous request until the maximum per mainloop run. */ while (smartlist_len(pow_state->rend_request_pqueue) > 0) { - if (++count == MAX_REND_REQUEST_PER_MAINLOOP) { - break; - } - /* Pop next request by effort. */ pending_rend_t *req = smartlist_pqueue_pop(pow_state->rend_request_pqueue, @@ -679,10 +718,21 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) req->rdv_data.pow_effort, smartlist_len(pow_state->rend_request_pqueue));

+ if (queued_rend_request_is_too_old(req, now)) { + log_info(LD_REND, "Top rend request has been pending for too long; " + "discarding and moving to the next one."); + free_pending_rend(req); + continue; /* do not increment count, this one's free */ + } + /* Launch the rendezvous circuit. */ launch_rendezvous_point_circuit(service, &req->ip_auth_pubkey, &req->ip_enc_key_kp, &req->rdv_data, now); free_pending_rend(req); + + if (++count == MAX_REND_REQUEST_PER_MAINLOOP) { + break; + } }

/* If there are still some pending rendezvous circuits in the pqueue then @@ -692,11 +742,6 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) } }

-/** Maximum number of rendezvous requests we enqueue per service. We allow the - * average amount of INTRODUCE2 that a service can process in a second times - * the rendezvous timeout. */ -#define MAX_REND_REQUEST (180 * MAX_REND_TIMEOUT) - /** Given the information needed to launch a rendezvous circuit and an * effort value, enqueue the rendezvous request in the service's PoW priority * queue with the effort being the priority. @@ -716,12 +761,6 @@ enqueue_rend_request(const hs_service_t *service, hs_service_intro_point_t *ip, /* Ease our lives */ pow_state = service->state.pow_state;

- if (smartlist_len(pow_state->rend_request_pqueue) >= MAX_REND_REQUEST) { - log_info(LD_REND, "Rendezvous request priority queue has " - "reached capacity."); - return -1; - } - req = tor_malloc_zero(sizeof(pending_rend_t));

/* Copy over the rendezvous request the needed data to launch a circuit. */ @@ -753,6 +792,12 @@ enqueue_rend_request(const hs_service_t *service, hs_service_intro_point_t *ip, /* Activate event, we just enqueued a rendezvous request. */ mainloop_event_activate(pow_state->pop_pqueue_ev);

+ /* See if there are so many cells queued that we need to cull. */ + if (smartlist_len(pow_state->rend_request_pqueue) >= + QUEUED_REND_REQUEST_HIGH_WATER) { + trim_rend_pqueue(pow_state, now); + } + return 0; }

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit d0c2d4cb431e86e7637ad6d917a52ae4ea25b256 Author: Roger Dingledine arma@torproject.org AuthorDate: Thu Jun 30 18:28:10 2022 -0400

add a log line for when client succeeds --- src/feature/hs/hs_client.c | 1 + 1 file changed, 1 insertion(+)

diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index e241e6218d..1cecacaf9f 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -678,6 +678,7 @@ send_introduce1(origin_circuit_t *intro_circ, log_warn(LD_REND, "Haven't solved the PoW yet."); goto tran_err; } + log_notice(LD_REND, "Got a PoW solution we like! Shipping it!"); /* Set flag to reflect that the HS we are attempting to rendezvous has PoW * defenses enabled, and as such we will need to be more lenient with * timing out while waiting for the circuit to be built. */

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit b95bd5017f09dd9385e54e9b3b6a49474b4dc9a8 Author: Roger Dingledine arma@torproject.org AuthorDate: Thu Jun 30 22:01:15 2022 -0400

track how many in-flight hs-side rend circs

not used in decision-making yet, but it's all ready to use in a "don't dequeue any more if we have too many in-flight" kind of way --- src/feature/hs/hs_circuit.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+)

diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 0c71443879..26dd600f1d 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -689,6 +689,27 @@ trim_rend_pqueue(hs_pow_service_state_t *pow_state, time_t now) smartlist_free(old_pqueue); }

+/** Count up how many pending outgoing (CIRCUIT_PURPOSE_S_CONNECT_REND) + * circuits there are for this service. Used in the PoW rate limiting + * world to decide whether it's time to launch any new ones. + */ +static int +count_service_rp_circuits_pending(hs_service_t *service) +{ + origin_circuit_t *ocirc = NULL; + int count = 0; + while ((ocirc = circuit_get_next_by_purpose(ocirc, + CIRCUIT_PURPOSE_S_CONNECT_REND))) { + /* Count up circuits that are v3 and for this service. */ + if (ocirc->hs_ident != NULL && + ed25519_pubkey_eq(&ocirc->hs_ident->identity_pk, + &service->keys.identity_pk)) { + count++; + } + } + return count; +} + /** How many rendezvous request we handle per mainloop event. Per prop327, * handling an INTRODUCE2 cell takes on average 5.56msec on an average CPU and * so it means that launching this max amount of circuits is well below 0.08 @@ -705,6 +726,11 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg)

(void) ev; /* Not using the returned event, make compiler happy. */

+ log_notice(LD_REND, "Considering launching more rendezvous responses. " + "%d in-flight, %d pending.", + count_service_rp_circuits_pending(service), + smartlist_len(pow_state->rend_request_pqueue)); + /* Process rendezvous request until the maximum per mainloop run. */ while (smartlist_len(pow_state->rend_request_pqueue) > 0) { /* Pop next request by effort. */

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 13f62582456cc9e7f1e8fd5993dd0e33b7da196b Author: Roger Dingledine arma@torproject.org AuthorDate: Fri Jul 1 15:56:46 2022 -0400

rate-limit low-effort rendezvous responses

specifically, if we have 16 in-flight rend circs, and the next one at the top of the pqueue is lower than our suggested effort, then don't launch it yet.

this way we always launch adequate-effort requests immediately, and we always handle *some* low-effort requests, but we are ready at any moment to handle a few new adequate-effort requests. --- src/feature/hs/hs_circuit.c | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-)

diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 4e31a534b0..c5eb22d5cd 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -710,12 +710,37 @@ count_service_rp_circuits_pending(hs_service_t *service) return count; }

+/** Peek at the top entry on the pending rend pqueue. If its level of + * effort is at least what we're suggesting for that service right now, + * return 1, else return 0. + */ +static int +top_of_rend_pqueue_is_worthwhile(hs_pow_service_state_t *pow_state) +{ + tor_assert(pow_state->rend_request_pqueue); + tor_assert(smartlist_len(pow_state->rend_request_pqueue)); + + pending_rend_t *req = + smartlist_get(pow_state->rend_request_pqueue, 0); + + if (req->rdv_data.pow_effort >= pow_state->suggested_effort) + return 1; + + return 0; +} + /** How many rendezvous request we handle per mainloop event. Per prop327, * handling an INTRODUCE2 cell takes on average 5.56msec on an average CPU and * so it means that launching this max amount of circuits is well below 0.08 * seconds which we believe is negligable on the whole mainloop. */ #define MAX_REND_REQUEST_PER_MAINLOOP 16

+/** What is the threshold of in-progress (CIRCUIT_PURPOSE_S_CONNECT_REND) + * rendezvous responses above which we won't launch new low-effort rendezvous + * responses? (Intro2 cells with suitable PoW effort are not affected + * by this threshold.) */ +#define MAX_CHEAP_REND_CIRCUITS_IN_PROGRESS 16 + static void handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) { @@ -723,16 +748,31 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) hs_service_t *service = arg; hs_pow_service_state_t *pow_state = service->state.pow_state; time_t now = time(NULL); + int in_flight = count_service_rp_circuits_pending(service);

(void) ev; /* Not using the returned event, make compiler happy. */

log_notice(LD_REND, "Considering launching more rendezvous responses. " "%d in-flight, %d pending.", - count_service_rp_circuits_pending(service), + in_flight, smartlist_len(pow_state->rend_request_pqueue));

/* Process rendezvous request until the maximum per mainloop run. */ while (smartlist_len(pow_state->rend_request_pqueue) > 0) { + + /* first, peek at the top result to see if we want to pop it */ + if (in_flight >= MAX_CHEAP_REND_CIRCUITS_IN_PROGRESS && + !top_of_rend_pqueue_is_worthwhile(pow_state)) { + /* We have queued requests, but they are all low priority, and also + * we have too many in-progress rendezvous responses. Don't launch + * any more. Schedule ourselves to reassess in a bit. */ + log_notice(LD_REND, "Next request to launch is low priority, and " + "%d in-flight already. Waiting to launch more.", in_flight); + const struct timeval delay_tv = { 0, 100000 }; + mainloop_event_schedule(pow_state->pop_pqueue_ev, &delay_tv); + return; /* done here! no cleanup needed. */ + } + /* Pop next request by effort. */ pending_rend_t *req = smartlist_pqueue_pop(pow_state->rend_request_pqueue, @@ -756,6 +796,7 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) &req->ip_enc_key_kp, &req->rdv_data, now); free_pending_rend(req);

+ ++in_flight; if (++count == MAX_REND_REQUEST_PER_MAINLOOP) { break; }

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit e436ce2a3c6a67e466090d9b7db136d7dd0f8385 Author: Roger Dingledine arma@torproject.org AuthorDate: Fri Jul 1 17:04:14 2022 -0400

drop the default min effort to 20

effort 100 is really quite expensive --- src/feature/hs/hs_config.h | 2 +- src/feature/hs/hs_pow.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/feature/hs/hs_config.h b/src/feature/hs/hs_config.h index 578aa468e2..15af172674 100644 --- a/src/feature/hs/hs_config.h +++ b/src/feature/hs/hs_config.h @@ -27,7 +27,7 @@

/* Default values for the HS anti-DoS PoW defenses. */ #define HS_CONFIG_V3_POW_DEFENSES_DEFAULT 0 -#define HS_CONFIG_V3_POW_DEFENSES_MIN_EFFORT_DEFAULT 100 +#define HS_CONFIG_V3_POW_DEFENSES_MIN_EFFORT_DEFAULT 20 #define HS_CONFIG_V3_POW_DEFENSES_SVC_BOTTOM_CAPACITY_DEFAULT 100

/* API */ diff --git a/src/feature/hs/hs_pow.h b/src/feature/hs/hs_pow.h index 679332e644..ee7b3c45d8 100644 --- a/src/feature/hs/hs_pow.h +++ b/src/feature/hs/hs_pow.h @@ -17,7 +17,7 @@ typedef unsigned __int128 uint128_t; #include "lib/evloop/compat_libevent.h" #include "lib/smartlist_core/smartlist_core.h"

-#define HS_POW_SUGGESTED_EFFORT_DEFAULT 100 // HRPR TODO 5000 +#define HS_POW_SUGGESTED_EFFORT_DEFAULT 20 // HRPR TODO 5000 /* Service updates the suggested effort every HS_UPDATE_PERIOD seconds. */ #define HS_UPDATE_PERIOD 300 // HRPR TODO Should be consensus

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit e605620744e3f752791dd6da4fd92d187af0d4a6 Author: Roger Dingledine arma@torproject.org AuthorDate: Fri Jul 1 18:32:04 2022 -0400

clients defend themselves from absurd pow requests

if asked for higher than a cap, we just solve it at the cap

i picked 500 for now but maybe we'll pick a better number in the future. --- src/feature/hs/hs_client.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)

diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index 1cecacaf9f..2ba2692941 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -600,6 +600,10 @@ find_desc_intro_point_by_legacy_id(const char *legacy_id, return ret_ip; }

+/** Set a client-side cap on the highest effort of PoW we will try to + * tackle. If asked for higher, we solve it at this cap. */ +#define CLIENT_MAX_POW_EFFORT 500 + /** Send an INTRODUCE1 cell along the intro circuit and populate the rend * circuit identifier with the needed key material for the e2e encryption. * Return 0 on success, -1 if there is a transient error such that an action @@ -674,6 +678,20 @@ send_introduce1(origin_circuit_t *intro_circ, if (desc->encrypted_data.pow_params) { log_debug(LD_REND, "PoW params present in descriptor."); pow_solution = tor_malloc_zero(sizeof(hs_pow_solution_t)); + + /* make sure we can't be tricked into hopeless quests */ + if (desc->encrypted_data.pow_params->suggested_effort > + CLIENT_MAX_POW_EFFORT) { + log_notice(LD_REND, "Onion service suggested effort %d which is " + "higher than we want to solve. Solving at %d instead.", + desc->encrypted_data.pow_params->suggested_effort, + CLIENT_MAX_POW_EFFORT); + + /* clobber it in-place. hopefully this won't have bad side effects. */ + desc->encrypted_data.pow_params->suggested_effort = + CLIENT_MAX_POW_EFFORT; + } + if (hs_pow_solve(desc->encrypted_data.pow_params, pow_solution)) { log_warn(LD_REND, "Haven't solved the PoW yet."); goto tran_err;

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 5b3a067fe37bce66f2f3cc4e8972ce428535e175 Author: Mike Perry mikeperry-git@torproject.org AuthorDate: Wed Jul 13 20:50:38 2022 +0000

Replace the constant bottom-half rate with handled count.

This allows us to more accurately estimate effort, based on real bottom-half throughput over the duration of a descriptor update. --- src/feature/hs/hs_circuit.c | 2 ++ src/feature/hs/hs_config.h | 1 - src/feature/hs/hs_pow.h | 4 ++-- src/feature/hs/hs_service.c | 14 +++++--------- src/feature/hs/hs_service.h | 1 - 5 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 948dab6933..acb33bbf88 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -803,7 +803,9 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) &req->ip_enc_key_kp, &req->rdv_data, now); free_pending_rend(req);

+ ++pow_state->rend_handled; ++in_flight; + if (++count == MAX_REND_REQUEST_PER_MAINLOOP) { break; } diff --git a/src/feature/hs/hs_config.h b/src/feature/hs/hs_config.h index 15af172674..119a91565b 100644 --- a/src/feature/hs/hs_config.h +++ b/src/feature/hs/hs_config.h @@ -28,7 +28,6 @@ /* Default values for the HS anti-DoS PoW defenses. */ #define HS_CONFIG_V3_POW_DEFENSES_DEFAULT 0 #define HS_CONFIG_V3_POW_DEFENSES_MIN_EFFORT_DEFAULT 20 -#define HS_CONFIG_V3_POW_DEFENSES_SVC_BOTTOM_CAPACITY_DEFAULT 100

/* API */

diff --git a/src/feature/hs/hs_pow.h b/src/feature/hs/hs_pow.h index ee7b3c45d8..019fea400e 100644 --- a/src/feature/hs/hs_pow.h +++ b/src/feature/hs/hs_pow.h @@ -90,8 +90,8 @@ typedef struct hs_pow_service_state_t { /* The following values are used when calculating and updating the suggested * effort every HS_UPDATE_PERIOD seconds. */

- /* Number of intro requests the service can handle per second. */ - uint32_t svc_bottom_capacity; + /* Number of intro requests the service handled since last update. */ + uint32_t rend_handled; /* The next time at which to update the suggested effort. */ time_t next_effort_update; /* Sum of effort of all valid requests received since the last update. */ diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index 65f6dce2c6..045022f182 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -265,8 +265,6 @@ set_service_default_config(hs_service_config_t *c, /* PoW default options. */ c->has_dos_defense_enabled = HS_CONFIG_V3_POW_DEFENSES_DEFAULT; c->pow_min_effort = HS_CONFIG_V3_POW_DEFENSES_MIN_EFFORT_DEFAULT; - c->pow_svc_bottom_capacity = - HS_CONFIG_V3_POW_DEFENSES_SVC_BOTTOM_CAPACITY_DEFAULT; }

/** Initialize PoW defenses */ @@ -286,7 +284,7 @@ initialize_pow_defenses(hs_service_t *service) /* We recalculate and update the suggested effort every HS_UPDATE_PERIOD * seconds. */ pow_state->suggested_effort = HS_POW_SUGGESTED_EFFORT_DEFAULT; - pow_state->svc_bottom_capacity = service->config.pow_svc_bottom_capacity; + pow_state->rend_handled = 0; pow_state->total_effort = 0; pow_state->next_effort_update = (time(NULL) + HS_UPDATE_PERIOD);

@@ -2677,15 +2675,12 @@ rotate_pow_seeds(hs_service_t *service, time_t now) static void update_suggested_effort(hs_service_t *service, time_t now) { - uint64_t denom; - /* Make life easier */ hs_pow_service_state_t *pow_state = service->state.pow_state;

/* Calculate the new suggested effort. */ - /* TODO Check for overflow in denominator? */ - denom = (pow_state->svc_bottom_capacity * HS_UPDATE_PERIOD); - pow_state->suggested_effort = (pow_state->total_effort / denom); + /* TODO Check for overflow? */ + pow_state->suggested_effort = (uint32_t)(pow_state->total_effort / pow_state->rend_handled);

log_debug(LD_REND, "Recalculated suggested effort: %u", pow_state->suggested_effort); @@ -2695,8 +2690,9 @@ update_suggested_effort(hs_service_t *service, time_t now) pow_state->suggested_effort = pow_state->min_effort; }

- /* Reset the total effort sum for this update period. */ + /* Reset the total effort sum and number of rends for this update period. */ pow_state->total_effort = 0; + pow_state->rend_handled = 0; pow_state->next_effort_update = now + HS_UPDATE_PERIOD; }

diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h index 817fa67718..465d9fba80 100644 --- a/src/feature/hs/hs_service.h +++ b/src/feature/hs/hs_service.h @@ -265,7 +265,6 @@ typedef struct hs_service_config_t { /** True iff PoW anti-DoS defenses are enabled. */ unsigned int has_pow_defenses_enabled : 1; uint32_t pow_min_effort; - uint32_t pow_svc_bottom_capacity;

/** If set, contains the Onion Balance master ed25519 public key (taken from * an .onion addresses) that this tor instance serves as backend. */

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit d36144ba31e9841a3b8ebb1650406f72256a540b Author: Mike Perry mikeperry-git@torproject.org AuthorDate: Thu Jul 14 17:34:32 2022 +0000

Initialize startup effort at 0.

If it works correctly, auto-tuning should set a non-zero effort once an attack begins. --- src/feature/hs/hs_service.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index 4f8a689290..80f0863183 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -283,7 +283,7 @@ initialize_pow_defenses(hs_service_t *service)

/* We recalculate and update the suggested effort every HS_UPDATE_PERIOD * seconds. */ - pow_state->suggested_effort = HS_POW_SUGGESTED_EFFORT_DEFAULT; + pow_state->suggested_effort = 0; pow_state->rend_handled = 0; pow_state->total_effort = 0; pow_state->next_effort_update = (time(NULL) + HS_UPDATE_PERIOD);

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit a5b0c7b4045d02acc15375a7d8bbad727b36e0de Author: Roger Dingledine arma@torproject.org AuthorDate: Sun Sep 4 00:48:42 2022 -0400

start the cpuworkers always, even for clients

prepares the way for client-side pow cpuworkers

also happens to resolve bug https://bugs.torproject.org/tpo/core/tor/40617 (which went into 0.4.7.4-alpha) because now we survive initing the cpuworker subsystem when we're not a relay. --- src/app/main/main.c | 8 ++++---- src/core/crypto/onion_crypto.c | 3 +++ src/core/mainloop/cpuworker.c | 2 +- src/core/mainloop/cpuworker.h | 4 +--- src/feature/relay/relay_config.c | 6 ------ 5 files changed, 9 insertions(+), 14 deletions(-)

diff --git a/src/app/main/main.c b/src/app/main/main.c index 838e129d04..a50a0aad6f 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1237,10 +1237,10 @@ run_tor_main_loop(void) const time_t now = time(NULL); directory_info_has_arrived(now, 1, 0);

- if (server_mode(get_options()) || dir_server_mode(get_options())) { - /* launch cpuworkers. Need to do this *after* we've read the onion key. */ - cpu_init(); - } + /* launch cpuworkers. Need to do this *after* we've read the onion key. */ + /* launch them always for all tors, now that clients can solve onion PoWs. */ + cpuworker_init(); + consdiffmgr_enable_background_compression();

/* Setup shared random protocol subsystem. */ diff --git a/src/core/crypto/onion_crypto.c b/src/core/crypto/onion_crypto.c index 81e4e1b078..0839d8903f 100644 --- a/src/core/crypto/onion_crypto.c +++ b/src/core/crypto/onion_crypto.c @@ -64,6 +64,9 @@ static const size_t NTOR3_CIRC_VERIFICATION_LEN = 14; server_onion_keys_t * server_onion_keys_new(void) { + if (!get_master_identity_key()) + return NULL; + server_onion_keys_t *keys = tor_malloc_zero(sizeof(server_onion_keys_t)); memcpy(keys->my_identity, router_get_my_id_digest(), DIGEST_LEN); ed25519_pubkey_copy(&keys->my_ed_identity, get_master_identity_key()); diff --git a/src/core/mainloop/cpuworker.c b/src/core/mainloop/cpuworker.c index 9ad8939e4d..4a22790b44 100644 --- a/src/core/mainloop/cpuworker.c +++ b/src/core/mainloop/cpuworker.c @@ -117,7 +117,7 @@ cpuworker_consensus_has_changed(const networkstatus_t *ns) * during Tor's lifetime. */ void -cpu_init(void) +cpuworker_init(void) { if (!replyqueue) { replyqueue = replyqueue_new(0); diff --git a/src/core/mainloop/cpuworker.h b/src/core/mainloop/cpuworker.h index 9eee287c1f..7821f5612f 100644 --- a/src/core/mainloop/cpuworker.h +++ b/src/core/mainloop/cpuworker.h @@ -12,9 +12,7 @@ #ifndef TOR_CPUWORKER_H #define TOR_CPUWORKER_H

-#include "feature/nodelist/networkstatus_st.h" - -void cpu_init(void); +void cpuworker_init(void); void cpuworkers_rotate_keyinfo(void);

void cpuworker_consensus_has_changed(const networkstatus_t *ns); diff --git a/src/feature/relay/relay_config.c b/src/feature/relay/relay_config.c index aa9d48beac..553b269ecf 100644 --- a/src/feature/relay/relay_config.c +++ b/src/feature/relay/relay_config.c @@ -1327,12 +1327,6 @@ options_act_relay(const or_options_t *old_options) "Worker-related options changed. Rotating workers."); const int server_mode_turned_on = server_mode(options) && !server_mode(old_options); - const int dir_server_mode_turned_on = - dir_server_mode(options) && !dir_server_mode(old_options); - - if (server_mode_turned_on || dir_server_mode_turned_on) { - cpu_init(); - }

if (server_mode_turned_on) { ip_address_changed(0);

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit eba919093320a995a10637170fcc881a4c2c2dd9 Author: Roger Dingledine arma@torproject.org AuthorDate: Sun Sep 4 03:55:27 2022 -0400

compute the client-side pow in a cpuworker thread

We mark the intro circuit with a new flag saying that the pow is in the cpuworker queue. When the cpuworker comes back, it either has a solution, in which case we proceed with sending the intro1 cell, or it has no solution, in which case we unmark the intro circuit and let the whole process restart on the next iteration of connection_ap_handshake_attach_circuit(). --- src/core/mainloop/cpuworker.c | 3 +- src/core/or/circuituse.c | 4 +- src/core/or/origin_circuit_st.h | 4 ++ src/feature/hs/hs_client.c | 31 ++++---- src/feature/hs/hs_client.h | 4 ++ src/feature/hs/hs_pow.c | 155 +++++++++++++++++++++++++++++++++++++++- src/feature/hs/hs_pow.h | 4 ++ src/lib/evloop/workqueue.c | 5 +- 8 files changed, 191 insertions(+), 19 deletions(-)

diff --git a/src/core/mainloop/cpuworker.c b/src/core/mainloop/cpuworker.c index 4a22790b44..a42dbb528d 100644 --- a/src/core/mainloop/cpuworker.c +++ b/src/core/mainloop/cpuworker.c @@ -14,7 +14,8 @@ * Right now, we use this infrastructure * <ul><li>for processing onionskins in onion.c * <li>for compressing consensuses in consdiffmgr.c, - * <li>and for calculating diffs and compressing them in consdiffmgr.c. + * <li>for calculating diffs and compressing them in consdiffmgr.c. + * <li>and for solving onion service PoW challenges in pow.c. * </ul> **/ #include "core/or/or.h" diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index d5879a21eb..b78f72e835 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -3043,8 +3043,8 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn) if (introcirc->base_.state == CIRCUIT_STATE_OPEN) { int ret; log_info(LD_REND, "Found open intro circ %u (id: %" PRIu32 "). " - "Rend circuit %u (id: %" PRIu32 "); Sending " - "introduction. (stream %d sec old)", + "Rend circuit %u (id: %" PRIu32 "); Considering " + "sending introduction. (stream %d sec old)", (unsigned) TO_CIRCUIT(introcirc)->n_circ_id, introcirc->global_identifier, (unsigned) TO_CIRCUIT(rendcirc)->n_circ_id, diff --git a/src/core/or/origin_circuit_st.h b/src/core/or/origin_circuit_st.h index fd5424c450..3b3fcc9b42 100644 --- a/src/core/or/origin_circuit_st.h +++ b/src/core/or/origin_circuit_st.h @@ -218,6 +218,10 @@ struct origin_circuit_t { * requests. */ unsigned int hs_with_pow_circ : 1;

+ /** Set iff this intro circ required a pow, and it has already queued + * the pow with the cpuworker and is awaiting a reply. */ + unsigned int hs_currently_solving_pow : 1; + /** Set iff this circuit has been given a relaxed timeout because * no circuits have opened. Used to prevent spamming logs. */ unsigned int relaxed_timeout : 1; diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index d7cfad7cd5..038f76c5b8 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -541,7 +541,7 @@ intro_circ_is_ok(const origin_circuit_t *circ)

/** Find a descriptor intro point object that matches the given ident in the * given descriptor desc. Return NULL if not found. */ -static const hs_desc_intro_point_t * +const hs_desc_intro_point_t * find_desc_intro_point_by_ident(const hs_ident_circuit_t *ident, const hs_descriptor_t *desc) { @@ -670,7 +670,6 @@ consider_sending_introduce1(origin_circuit_t *intro_circ, char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1]; const ed25519_public_key_t *service_identity_pk = NULL; const hs_desc_intro_point_t *ip; - hs_pow_solution_t *pow_solution = NULL;

tor_assert(rend_circ); if (intro_circ_is_ok(intro_circ) < 0) { @@ -682,9 +681,15 @@ consider_sending_introduce1(origin_circuit_t *intro_circ, * version number but for now there is none because it's all v3. */ hs_build_address(service_identity_pk, HS_VERSION_THREE, onion_address);

- log_info(LD_REND, "Sending INTRODUCE1 cell to service %s on circuit %u", + log_info(LD_REND, "Considering sending INTRODUCE1 cell to service %s " + "on circuit %u", safe_str_client(onion_address), TO_CIRCUIT(intro_circ)->n_circ_id);

+ /* if it's already waiting on the cpuworker farm, don't queue it again */ + if (intro_circ->hs_currently_solving_pow) { + goto tran_err; + } + /* 1) Get descriptor from our cache. */ const hs_descriptor_t *desc = hs_cache_lookup_as_client(service_identity_pk); @@ -731,7 +736,6 @@ consider_sending_introduce1(origin_circuit_t *intro_circ, if (desc->encrypted_data.pow_params && desc->encrypted_data.pow_params->suggested_effort > 0) { log_debug(LD_REND, "PoW params present in descriptor."); - pow_solution = tor_malloc_zero(sizeof(hs_pow_solution_t));

/* make sure we can't be tricked into hopeless quests */ if (desc->encrypted_data.pow_params->suggested_effort > @@ -746,15 +750,15 @@ consider_sending_introduce1(origin_circuit_t *intro_circ, CLIENT_MAX_POW_EFFORT; }

- if (hs_pow_solve(desc->encrypted_data.pow_params, pow_solution)) { - log_warn(LD_REND, "Haven't solved the PoW yet."); - goto tran_err; - } - log_notice(LD_REND, "Got a PoW solution we like! Shipping it!"); - /* Set flag to reflect that the HS we are attempting to rendezvous has PoW - * defenses enabled, and as such we will need to be more lenient with - * timing out while waiting for the circuit to be built. */ - rend_circ->hs_with_pow_circ = 1; + /* send it to the client-side pow cpuworker for solving. */ + intro_circ->hs_currently_solving_pow = 1; + pow_queue_work(intro_circ->global_identifier, + rend_circ->global_identifier, + desc->encrypted_data.pow_params); + + /* can't proceed with the intro1 cell yet, so yield back to the + * main loop */ + goto tran_err; }

/* move on to the next phase: actually try to send it */ @@ -781,7 +785,6 @@ consider_sending_introduce1(origin_circuit_t *intro_circ,

end: memwipe(onion_address, 0, sizeof(onion_address)); - tor_free(pow_solution); return status; }

diff --git a/src/feature/hs/hs_client.h b/src/feature/hs/hs_client.h index 37daeab943..e87cc00b75 100644 --- a/src/feature/hs/hs_client.h +++ b/src/feature/hs/hs_client.h @@ -78,6 +78,10 @@ typedef struct hs_client_service_authorization_t { int flags; } hs_client_service_authorization_t;

+const hs_desc_intro_point_t * +find_desc_intro_point_by_ident(const hs_ident_circuit_t *ident, + const hs_descriptor_t *desc); + hs_client_register_auth_status_t hs_client_register_auth_credentials(hs_client_service_authorization_t *creds);

diff --git a/src/feature/hs/hs_pow.c b/src/feature/hs/hs_pow.c index 49577617e6..2e94f9bced 100644 --- a/src/feature/hs/hs_pow.c +++ b/src/feature/hs/hs_pow.c @@ -13,9 +13,15 @@ typedef unsigned __int128 uint128_t; #include <stdio.h>

#include "ext/ht.h" +#include "core/or/circuitlist.h" +#include "core/or/origin_circuit_st.h" +#include "feature/hs/hs_cache.h" #include "feature/hs/hs_descriptor.h" +#include "feature/hs/hs_client.h" #include "feature/hs/hs_pow.h" #include "lib/crypt_ops/crypto_rand.h" +#include "core/mainloop/cpuworker.h" +#include "lib/evloop/workqueue.h"

/** Replay cache set up */ /** Cache entry for (nonce, seed) replay protection. */ @@ -144,7 +150,7 @@ hs_pow_solve(const hs_pow_desc_params_t *pow_params,

/* We'll do a maximum of the nonce size iterations here which is the maximum * number of nonce we can try in an attempt to find a valid solution. */ - log_notice(LD_REND, "Solving proof of work"); + log_notice(LD_REND, "Solving proof of work (effort %u)", effort); for (uint64_t i = 0; i < UINT64_MAX; i++) { /* Calculate S = equix_solve(C || N || E) */ if (!equix_solve(ctx, challenge, HS_POW_CHALLENGE_LEN, solution)) { @@ -290,3 +296,150 @@ hs_pow_free_service_state(hs_pow_service_state_t *state) mainloop_event_free(state->pop_pqueue_ev); tor_free(state); } + +/* ===== + Thread workers + =====*/ + +/** + * An object passed to a worker thread that will try to solve the pow. + */ +typedef struct pow_worker_job_t { + + /** Input: The pow challenge we need to solve. */ + hs_pow_desc_params_t *pow_params; + + /** State: we'll look these up to figure out how to proceed after. */ + uint32_t intro_circ_identifier; + uint32_t rend_circ_identifier; + + /** Output: The worker thread will malloc and write its answer here, + * or set it to NULL if it produced no useful answer. */ + hs_pow_solution_t *pow_solution_out; + +} pow_worker_job_t; + +/** + * Worker function. This function runs inside a worker thread and receives + * a pow_worker_job_t as its input. + */ +static workqueue_reply_t +pow_worker_threadfn(void *state_, void *work_) +{ + (void)state_; + pow_worker_job_t *job = work_; + job->pow_solution_out = tor_malloc_zero(sizeof(hs_pow_solution_t)); + + if (hs_pow_solve(job->pow_params, job->pow_solution_out)) { + log_info(LD_REND, "Haven't solved the PoW yet. Returning."); + tor_free(job->pow_solution_out); + job->pow_solution_out = NULL; /* how we signal that we came up empty */ + return WQ_RPL_REPLY; + } + + /* we have a winner! */ + log_info(LD_REND, "cpuworker pow: we have a winner!"); + return WQ_RPL_REPLY; +} + +/** + * Helper: release all storage held in <b>job</b>. + */ +static void +pow_worker_job_free(pow_worker_job_t *job) +{ + if (!job) + return; + tor_free(job->pow_params); + tor_free(job->pow_solution_out); + tor_free(job); +} + +/** + * Worker function: This function runs in the main thread, and receives + * a pow_worker_job_t that the worker thread has already processed. + */ +static void +pow_worker_replyfn(void *work_) +{ + tor_assert(in_main_thread()); + tor_assert(work_); + + pow_worker_job_t *job = work_; + + // look up the circuits that we're going to use this pow in + origin_circuit_t *intro_circ = + circuit_get_by_global_id(job->intro_circ_identifier); + origin_circuit_t *rend_circ = + circuit_get_by_global_id(job->rend_circ_identifier); + + /* try to re-create desc and ip */ + const ed25519_public_key_t *service_identity_pk = NULL; + const hs_descriptor_t *desc = NULL; + const hs_desc_intro_point_t *ip = NULL; + if (intro_circ) + service_identity_pk = &intro_circ->hs_ident->identity_pk; + if (service_identity_pk) + desc = hs_cache_lookup_as_client(service_identity_pk); + if (desc) + ip = find_desc_intro_point_by_ident(intro_circ->hs_ident, desc); + + if (intro_circ && rend_circ && service_identity_pk && desc && ip && + job->pow_solution_out) { /* successful pow solve, and circs still here */ + + log_notice(LD_REND, "Got a PoW solution we like! Shipping it!"); + /* Set flag to reflect that the HS we are attempting to rendezvous has PoW + * defenses enabled, and as such we will need to be more lenient with + * timing out while waiting for the service-side circuit to be built. */ + rend_circ->hs_with_pow_circ = 1; + + // and then send that intro cell + if (send_introduce1(intro_circ, rend_circ, + desc, job->pow_solution_out, ip) < 0) { + /* if it failed, mark the intro point as ready to start over */ + intro_circ->hs_currently_solving_pow = 0; + } + + } else { /* unsuccessful pow solve. put it back on the queue. */ + log_notice(LD_REND, + "PoW cpuworker returned with no solution. Will retry soon."); + if (intro_circ) { + intro_circ->hs_currently_solving_pow = 0; + } + /* We could imagine immediately re-launching a follow-up worker + * here too, but for now just let the main intro loop find the + * not-being-serviced request and it can start everything again. For + * the sake of complexity, maybe that's the best long-term solution + * too, and we can tune the cpuworker job to try for longer if we want + * to improve efficiency. */ + } + + pow_worker_job_free(job); +} + +/** + * Queue the job of solving the pow in a worker thread. + */ +int +pow_queue_work(uint32_t intro_circ_identifier, + uint32_t rend_circ_identifier, + const hs_pow_desc_params_t *pow_params) +{ + tor_assert(in_main_thread()); + + pow_worker_job_t *job = tor_malloc_zero(sizeof(*job)); + job->intro_circ_identifier = intro_circ_identifier; + job->rend_circ_identifier = rend_circ_identifier; + job->pow_params = tor_memdup(pow_params, sizeof(hs_pow_desc_params_t)); + + workqueue_entry_t *work; + work = cpuworker_queue_work(WQ_PRI_LOW, + pow_worker_threadfn, + pow_worker_replyfn, + job); + if (!work) { + pow_worker_job_free(job); + return -1; + } + return 0; +} diff --git a/src/feature/hs/hs_pow.h b/src/feature/hs/hs_pow.h index bc0b823fd9..587cae6155 100644 --- a/src/feature/hs/hs_pow.h +++ b/src/feature/hs/hs_pow.h @@ -130,4 +130,8 @@ int hs_pow_verify(const hs_pow_service_state_t *pow_state, void hs_pow_remove_seed_from_cache(uint32_t seed); void hs_pow_free_service_state(hs_pow_service_state_t *state);

+int pow_queue_work(uint32_t intro_circ_identifier, + uint32_t rend_circ_identifier, + const hs_pow_desc_params_t *pow_params); + #endif /* !defined(TOR_HS_POW_H) */ diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c index bc929148eb..9a0c02fbd0 100644 --- a/src/lib/evloop/workqueue.c +++ b/src/lib/evloop/workqueue.c @@ -20,7 +20,10 @@ * The main thread can also queue an "update" that will be handled by all the * workers. This is useful for updating state that all the workers share. * - * In Tor today, there is currently only one thread pool, used in cpuworker.c. + * In Tor today, there is currently only one thread pool, managed + * in cpuworker.c and handling a variety of types of work, from the original + * "onion skin" circuit handshakes, to consensus diff computation, to + * client-side onion service PoW generation. */

#include "orconfig.h"

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 48c67263d9b3779e1f3296564192b13b6b0895b4 Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Fri Feb 24 18:25:25 2023 -0800

hs_metrics: Proof of Work pqueue depth, suggested effort

Adds two new metrics for hs_pow, and an internal parameter within hs_metrics for implementing gauge parameters that reset before every update.

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/feature/hs/hs_circuit.c | 8 +++++++ src/feature/hs/hs_metrics.c | 11 ++++++--- src/feature/hs/hs_metrics.h | 47 ++++++++++++++++++++++++--------------- src/feature/hs/hs_metrics_entry.c | 12 ++++++++++ src/feature/hs/hs_metrics_entry.h | 10 ++++++--- src/feature/hs/hs_service.c | 2 ++ src/test/test_hs_metrics.c | 20 +++++++++++++---- 7 files changed, 82 insertions(+), 28 deletions(-)

diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 0ac47ee19f..3684def697 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -791,6 +791,9 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) compare_rend_request_by_effort_, offsetof(pending_rend_t, idx));

+ hs_metrics_pow_pqueue_rdv(service, + smartlist_len(pow_state->rend_request_pqueue)); + log_notice(LD_REND, "Dequeued pending rendezvous request with effort: %u. " "Waited %d. " "Remaining requests: %u", @@ -870,6 +873,9 @@ enqueue_rend_request(const hs_service_t *service, hs_service_intro_point_t *ip, compare_rend_request_by_effort_, offsetof(pending_rend_t, idx), req);

+ hs_metrics_pow_pqueue_rdv(service, + smartlist_len(pow_state->rend_request_pqueue)); + log_notice(LD_REND, "Enqueued rendezvous request with effort: %u. " "Queued requests: %u", req->rdv_data.pow_effort, @@ -888,6 +894,8 @@ enqueue_rend_request(const hs_service_t *service, hs_service_intro_point_t *ip, if (smartlist_len(pow_state->rend_request_pqueue) >= QUEUED_REND_REQUEST_HIGH_WATER) { trim_rend_pqueue(pow_state, now); + hs_metrics_pow_pqueue_rdv(service, + smartlist_len(pow_state->rend_request_pqueue)); }

return 0; diff --git a/src/feature/hs/hs_metrics.c b/src/feature/hs/hs_metrics.c index 46c72cf539..19a330a01e 100644 --- a/src/feature/hs/hs_metrics.c +++ b/src/feature/hs/hs_metrics.c @@ -68,6 +68,8 @@ add_metric_with_labels(hs_service_t *service, hs_metrics_key_t metric, case HS_METRICS_NUM_ESTABLISHED_RDV: FALLTHROUGH; case HS_METRICS_NUM_RDV: FALLTHROUGH; case HS_METRICS_NUM_ESTABLISHED_INTRO: FALLTHROUGH; + case HS_METRICS_POW_NUM_PQUEUE_RDV: FALLTHROUGH; + case HS_METRICS_POW_SUGGESTED_EFFORT: FALLTHROUGH; case HS_METRICS_INTRO_CIRC_BUILD_TIME: FALLTHROUGH; case HS_METRICS_REND_CIRC_BUILD_TIME: FALLTHROUGH; default: @@ -146,7 +148,7 @@ void hs_metrics_update_by_service(const hs_metrics_key_t key, const hs_service_t *service, uint16_t port, const char *reason, - int64_t n, int64_t obs) + int64_t n, int64_t obs, bool reset) { tor_assert(service);

@@ -167,6 +169,9 @@ hs_metrics_update_by_service(const hs_metrics_key_t key, entry, metrics_format_label("port", port_to_str(port)))) && ((!reason || metrics_store_entry_has_label( entry, metrics_format_label("reason", reason))))) { + if (reset) { + metrics_store_entry_reset(entry); + }

if (metrics_store_entry_is_histogram(entry)) { metrics_store_hist_entry_update(entry, n, obs); @@ -190,7 +195,7 @@ void hs_metrics_update_by_ident(const hs_metrics_key_t key, const ed25519_public_key_t *ident_pk, const uint16_t port, const char *reason, - int64_t n, int64_t obs) + int64_t n, int64_t obs, bool reset) { hs_service_t *service;

@@ -204,7 +209,7 @@ hs_metrics_update_by_ident(const hs_metrics_key_t key, * service and thus the only way to know is to lookup the service. */ return; } - hs_metrics_update_by_service(key, service, port, reason, n, obs); + hs_metrics_update_by_service(key, service, port, reason, n, obs, reset); }

/** Return a list of all the onion service metrics stores. This is the diff --git a/src/feature/hs/hs_metrics.h b/src/feature/hs/hs_metrics.h index 4eff4cb498..f2e5dbd9ec 100644 --- a/src/feature/hs/hs_metrics.h +++ b/src/feature/hs/hs_metrics.h @@ -27,75 +27,86 @@ const smartlist_t *hs_metrics_get_stores(void); void hs_metrics_update_by_ident(const hs_metrics_key_t key, const ed25519_public_key_t *ident_pk, const uint16_t port, const char *reason, - int64_t n, int64_t obs); + int64_t n, int64_t obs, bool reset); void hs_metrics_update_by_service(const hs_metrics_key_t key, const hs_service_t *service, uint16_t port, const char *reason, - int64_t n, int64_t obs); + int64_t n, int64_t obs, bool reset);

/** New introducion request received. */ #define hs_metrics_new_introduction(s) \ - hs_metrics_update_by_service(HS_METRICS_NUM_INTRODUCTIONS, (s), 0, NULL, 1, \ - 0) + hs_metrics_update_by_service(HS_METRICS_NUM_INTRODUCTIONS, (s), \ + 0, NULL, 1, 0, false)

/** Introducion request rejected. */ #define hs_metrics_reject_intro_req(s, reason) \ hs_metrics_update_by_service(HS_METRICS_NUM_REJECTED_INTRO_REQ, (s), 0, \ - (reason), 1, 0) + (reason), 1, 0, false)

/** Number of bytes written to the application from the service. */ #define hs_metrics_app_write_bytes(i, port, n) \ hs_metrics_update_by_ident(HS_METRICS_APP_WRITE_BYTES, (i), (port), NULL, \ - (n), 0) + (n), 0, false)

/** Number of bytes read from the application to the service. */ #define hs_metrics_app_read_bytes(i, port, n) \ - hs_metrics_update_by_ident(HS_METRICS_APP_READ_BYTES, (i), (port), NULL, \ - (n), 0) + hs_metrics_update_by_ident(HS_METRICS_APP_READ_BYTES, (i), \ + (port), NULL, (n), 0, false)

/** Newly established rendezvous. This is called as soon as the circuit purpose * is REND_JOINED which is when the RENDEZVOUS2 cell is sent. */ #define hs_metrics_new_established_rdv(s) \ - hs_metrics_update_by_service(HS_METRICS_NUM_ESTABLISHED_RDV, (s), 0, NULL, \ - 1, 0) + hs_metrics_update_by_service(HS_METRICS_NUM_ESTABLISHED_RDV, (s), \ + 0, NULL, 1, 0, false)

/** New rendezvous circuit failure. */ #define hs_metrics_failed_rdv(i, reason) \ - hs_metrics_update_by_ident(HS_METRICS_NUM_FAILED_RDV, (i), 0, (reason), 1, 0) + hs_metrics_update_by_ident(HS_METRICS_NUM_FAILED_RDV, (i), \ + 0, (reason), 1, 0, false)

/** Established rendezvous closed. This is called when the circuit in * REND_JOINED state is marked for close. */ #define hs_metrics_close_established_rdv(i) \ - hs_metrics_update_by_ident(HS_METRICS_NUM_ESTABLISHED_RDV, (i), 0, NULL, \ - -1, 0) + hs_metrics_update_by_ident(HS_METRICS_NUM_ESTABLISHED_RDV, (i), \ + 0, NULL, -1, 0, false)

/** New rendezvous circuit being launched. */ #define hs_metrics_new_rdv(i) \ - hs_metrics_update_by_ident(HS_METRICS_NUM_RDV, (i), 0, NULL, 1, 0) + hs_metrics_update_by_ident(HS_METRICS_NUM_RDV, (i), 0, NULL, 1, 0, false) + +/** Update depth of rendezvous pqueue any time new work is enqueued. */ +#define hs_metrics_pow_pqueue_rdv(s, n) \ + hs_metrics_update_by_service(HS_METRICS_POW_NUM_PQUEUE_RDV, (s), 0, \ + NULL, (n), 0, true) + +/** Update the suggested effort we include in proof-of-work state */ +#define hs_metrics_pow_suggested_effort(s, n) \ + hs_metrics_update_by_service(HS_METRICS_POW_SUGGESTED_EFFORT, (s), 0, \ + NULL, (n), 0, true)

/** New introduction circuit has been established. This is called when the * INTRO_ESTABLISHED has been received by the service. */ #define hs_metrics_new_established_intro(s) \ hs_metrics_update_by_service(HS_METRICS_NUM_ESTABLISHED_INTRO, (s), 0, \ - NULL, 1, 0) + NULL, 1, 0, false)

/** Established introduction circuit closes. This is called when * INTRO_ESTABLISHED circuit is marked for close. */ #define hs_metrics_close_established_intro(i) \ hs_metrics_update_by_ident(HS_METRICS_NUM_ESTABLISHED_INTRO, (i), 0, NULL, \ - -1, 0) + -1, 0, false)

/** Record an introduction circuit build time duration. This is called * when the INTRO_ESTABLISHED has been received by the service. */ #define hs_metrics_intro_circ_build_time(s, obs) \ hs_metrics_update_by_service(HS_METRICS_INTRO_CIRC_BUILD_TIME, (s), 0, \ - NULL, 1, obs) + NULL, 1, obs, false)

/** Record a rendezvous circuit build time duration. This is called as soon as * the circuit purpose is REND_JOINED which is when the RENDEZVOUS2 cell is * sent. */ #define hs_metrics_rdv_circ_build_time(s, obs) \ hs_metrics_update_by_service(HS_METRICS_REND_CIRC_BUILD_TIME, (s), 0, NULL, \ - 1, obs) + 1, obs, false)

#endif /* !defined(TOR_FEATURE_HS_HS_METRICS_H) */ diff --git a/src/feature/hs/hs_metrics_entry.c b/src/feature/hs/hs_metrics_entry.c index 3524d72334..2268ba4c59 100644 --- a/src/feature/hs/hs_metrics_entry.c +++ b/src/feature/hs/hs_metrics_entry.c @@ -104,6 +104,18 @@ const hs_metrics_entry_t base_metrics[] = .bucket_count = hs_metrics_circ_build_time_buckets_size, .help = "The rendezvous circuit build time in milliseconds", }, + { + .key = HS_METRICS_POW_NUM_PQUEUE_RDV, + .type = METRICS_TYPE_GAUGE, + .name = METRICS_NAME(hs_rdv_pow_pqueue_count), + .help = "Number of requests waiting in the proof of work priority queue", + }, + { + .key = HS_METRICS_POW_SUGGESTED_EFFORT, + .type = METRICS_TYPE_GAUGE, + .name = METRICS_NAME(hs_pow_suggested_effort), + .help = "Suggested effort for requests with a proof of work client puzzle", + }, };

/** Size of base_metrics array that is number of entries. */ diff --git a/src/feature/hs/hs_metrics_entry.h b/src/feature/hs/hs_metrics_entry.h index 4c9abd06d7..1a1bc701ec 100644 --- a/src/feature/hs/hs_metrics_entry.h +++ b/src/feature/hs/hs_metrics_entry.h @@ -48,11 +48,11 @@ typedef enum { HS_METRICS_APP_WRITE_BYTES = 1, /** Number of bytes read from application to onion service. */ HS_METRICS_APP_READ_BYTES = 2, - /** Number of established rendezsvous. */ + /** Number of established rendezvous. */ HS_METRICS_NUM_ESTABLISHED_RDV = 3, - /** Number of rendezsvous circuits created. */ + /** Number of rendezvous circuits created. */ HS_METRICS_NUM_RDV = 4, - /** Number of failed rendezsvous. */ + /** Number of failed rendezvous. */ HS_METRICS_NUM_FAILED_RDV = 5, /** Number of established introducton points. */ HS_METRICS_NUM_ESTABLISHED_INTRO = 6, @@ -62,6 +62,10 @@ typedef enum { HS_METRICS_INTRO_CIRC_BUILD_TIME = 8, /** Rendezvous circuit build time in milliseconds. */ HS_METRICS_REND_CIRC_BUILD_TIME = 9, + /** Number of requests waiting in the proof of work priority queue. */ + HS_METRICS_POW_NUM_PQUEUE_RDV = 10, + /** Suggested effort for requests with a proof of work client puzzle. */ + HS_METRICS_POW_SUGGESTED_EFFORT = 11, } hs_metrics_key_t;

/** The metadata of an HS metrics. */ diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index d67fead791..fda0162958 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -2700,6 +2700,8 @@ update_suggested_effort(hs_service_t *service, time_t now) pow_state->suggested_effort = 2*pow_state->suggested_effort/3; }

+ hs_metrics_pow_suggested_effort(service, pow_state->suggested_effort); + log_debug(LD_REND, "Recalculated suggested effort: %u", pow_state->suggested_effort);

diff --git a/src/test/test_hs_metrics.c b/src/test/test_hs_metrics.c index c3c7ef57bc..acb0649434 100644 --- a/src/test/test_hs_metrics.c +++ b/src/test/test_hs_metrics.c @@ -40,7 +40,8 @@ test_metrics(void *arg)

/* Update entry by identifier. */ hs_metrics_update_by_ident(HS_METRICS_NUM_INTRODUCTIONS, - &service->keys.identity_pk, 0, NULL, 42, 0); + &service->keys.identity_pk, 0, NULL, 42, + 0, false);

/* Confirm the entry value. */ const smartlist_t *entries = metrics_store_get_all(service->metrics.store, @@ -53,14 +54,15 @@ test_metrics(void *arg)

/* Update entry by service now. */ hs_metrics_update_by_service(HS_METRICS_NUM_INTRODUCTIONS, - service, 0, NULL, 42, 0); + service, 0, NULL, 42, 0, false); tt_int_op(metrics_store_entry_get_value(entry), OP_EQ, 84);

const char *reason = HS_METRICS_ERR_INTRO_REQ_BAD_AUTH_KEY;

/* Update tor_hs_intro_rejected_intro_req_count */ hs_metrics_update_by_ident(HS_METRICS_NUM_REJECTED_INTRO_REQ, - &service->keys.identity_pk, 0, reason, 112, 0); + &service->keys.identity_pk, 0, + reason, 112, 0, false);

entries = metrics_store_get_all(service->metrics.store, "tor_hs_intro_rejected_intro_req_count"); @@ -75,9 +77,19 @@ test_metrics(void *arg)

/* Update tor_hs_intro_rejected_intro_req_count entry by service now. */ hs_metrics_update_by_service(HS_METRICS_NUM_REJECTED_INTRO_REQ, service, 0, - reason, 10, 0); + reason, 10, 0, false); tt_int_op(metrics_store_entry_get_value(entry), OP_EQ, 122);

+ /* So far these have been relative updates. Test updates with reset */ + hs_metrics_update_by_service(HS_METRICS_NUM_REJECTED_INTRO_REQ, + service, 0, reason, 10, 0, true); + tt_int_op(metrics_store_entry_get_value(entry), OP_EQ, 10); + + hs_metrics_update_by_ident(HS_METRICS_NUM_REJECTED_INTRO_REQ, + &service->keys.identity_pk, 0, reason, + 345, 0, true); + tt_int_op(metrics_store_entry_get_value(entry), OP_EQ, 345); + done: hs_free_all(); }

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 98299e0f8b872825cffa5afd007ee7fd5fd2a39a Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Mon Feb 27 15:36:22 2023 -0800

manpage: document HiddenServicePoWDefensesEnabled option

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- doc/man/tor.1.txt | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/doc/man/tor.1.txt b/doc/man/tor.1.txt index 57992cd8d2..a62c7c7d82 100644 --- a/doc/man/tor.1.txt +++ b/doc/man/tor.1.txt @@ -3021,14 +3021,14 @@ Denial of Service mitigation subsystem described above. (Default: auto)

-As for onion services, only one possible mitigation exists. It was intended to -protect the network first and thus do not help the service availability or -reachability. +For onion services, mitigations are a work in progress and multiple options +are currently available.

-The mitigation we put in place is a rate limit of the amount of introduction -that happens at the introduction point for a service. In other words, it rates -limit the number of clients that are attempting to reach the service at the -introduction point instead of at the service itself. +The introduction point defense is a rate limit on the number of introduction +requests that will be forwarded to a service by each of its honest +introduction point routers. This can prevent some types of overwhelming floods +from reaching the service, but it will also prevent legitimate clients from +establishing new connections.

The following options are per onion service:

@@ -3082,6 +3082,23 @@ The bottom line is that this protects the network by preventing an onion service to flood the network with new rendezvous circuits that is reducing load on the network.

+A secondary mitigation is available, based on prioritized dispatch of rendezvous +circuits for new connections. The queue is ordered based on effort a client +chooses to spend at computing a proof-of-work function. + +The following options are per onion service: + +[[HiddenServicePoWDefensesEnabled]] **HiddenServicePoWDefensesEnabled** **0**|**1**:: + + Enable proof-of-work based service DoS mitigation. If set to 1 (enabled), + tor will include parameters for an optional client puzzle in the encrypted + portion of this hidden service's descriptor. Incoming rendezvous requests + will be prioritized based on the amount of effort a client chooses to make + when computing a solution to the puzzle. The service will periodically update + a suggested amount of effort, based on attack load, and disable the puzzle + entirely when the service is not overloaded. + (Default: 0) +

== DIRECTORY AUTHORITY SERVER OPTIONS

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit f3b98116b6f331ec9b849867dff8dec957ce7edc Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Mon Feb 27 18:39:43 2023 -0800

hs_pow: Rate limited dequeue

This adds a token bucket ratelimiter on the dequeue side of hs_pow's priority queue. It adds config options and docs for those options. (HiddenServicePoWQueueRate/Burst)

I'm testing this as a way to limit the overhead of circuit creation when we're experiencing a flood of rendezvous requests.

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- doc/man/tor.1.txt | 13 +++++++++++++ src/app/config/config.c | 2 ++ src/feature/hs/hs_circuit.c | 19 +++++++++++++++++++ src/feature/hs/hs_config.c | 18 +++++++++++++++++- src/feature/hs/hs_options.inc | 2 ++ src/feature/hs/hs_pow.h | 8 ++++++++ src/feature/hs/hs_service.c | 9 +++++++++ src/feature/hs/hs_service.h | 2 ++ 8 files changed, 72 insertions(+), 1 deletion(-)

diff --git a/doc/man/tor.1.txt b/doc/man/tor.1.txt index a62c7c7d82..2ac6a8471c 100644 --- a/doc/man/tor.1.txt +++ b/doc/man/tor.1.txt @@ -3099,6 +3099,19 @@ The following options are per onion service: entirely when the service is not overloaded. (Default: 0)

+[[HiddenServicePoWQueueRate]] **HiddenServicePoWQueueRate** __NUM__:: + + The sustained rate of rendezvous requests to dispatch per second from + the priority queue. Has no effect when proof-of-work is disabled. + If this is set to 0 there's no explicit limit and we will process + requests as quickly as possible. + (Default: 250) + +[[HiddenServicePoWQueueBurst]] **HiddenServicePoWQueueBurst** __NUM__:: + + The maximum burst size for rendezvous requests handled from the + priority queue at once. (Default: 2500) +

== DIRECTORY AUTHORITY SERVER OPTIONS

diff --git a/src/app/config/config.c b/src/app/config/config.c index e035c6d6f3..0618622db9 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -509,6 +509,8 @@ static const config_var_t option_vars_[] = { VAR("HiddenServiceOnionBalanceInstance", LINELIST_S, RendConfigLines, NULL), VAR("HiddenServicePoWDefensesEnabled", LINELIST_S, RendConfigLines, NULL), + VAR("HiddenServicePoWQueueRate", LINELIST_S, RendConfigLines, NULL), + VAR("HiddenServicePoWQueueBurst", LINELIST_S, RendConfigLines, NULL), VAR("HiddenServiceStatistics", BOOL, HiddenServiceStatistics_option, "1"), V(ClientOnionAuthDir, FILENAME, NULL), OBSOLETE("CloseHSClientCircuitsImmediatelyOnTimeout"), diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 3684def697..55b992ee28 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -785,6 +785,20 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) return; /* done here! no cleanup needed. */ }

+ if (pow_state->using_pqueue_bucket) { + token_bucket_ctr_refill(&pow_state->pqueue_bucket, + (uint32_t) approx_time()); + + if (token_bucket_ctr_get(&pow_state->pqueue_bucket) > 0) { + token_bucket_ctr_dec(&pow_state->pqueue_bucket, 1); + } else { + /* Waiting for pqueue rate limit to refill, come back later */ + const struct timeval delay_tv = { 0, 100000 }; + mainloop_event_schedule(pow_state->pop_pqueue_ev, &delay_tv); + return; + } + } + /* Pop next request by effort. */ pending_rend_t *req = smartlist_pqueue_pop(pow_state->rend_request_pqueue, @@ -816,6 +830,11 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) ++pow_state->rend_handled; ++in_flight;

+ if (pow_state->using_pqueue_bucket && + token_bucket_ctr_get(&pow_state->pqueue_bucket) < 1) { + break; + } + if (++count == MAX_REND_REQUEST_PER_MAINLOOP) { break; } diff --git a/src/feature/hs/hs_config.c b/src/feature/hs/hs_config.c index 4561bd3e48..0f5a8cf49a 100644 --- a/src/feature/hs/hs_config.c +++ b/src/feature/hs/hs_config.c @@ -320,6 +320,13 @@ config_validate_service(const hs_service_config_t *config) config->intro_dos_burst_per_sec, config->intro_dos_rate_per_sec); goto invalid; } + if (config->has_pow_defenses_enabled && + (config->pow_queue_burst < config->pow_queue_rate)) { + log_warn(LD_CONFIG, "Hidden service PoW queue burst (%" PRIu32 ") can " + "not be smaller than the rate value (%" PRIu32 ").", + config->pow_queue_burst, config->pow_queue_rate); + goto invalid; + }

/* Valid. */ return 0; @@ -394,8 +401,17 @@ config_service_v3(const hs_opts_t *hs_opts,

/* Are the PoW anti-DoS defenses enabled? */ config->has_pow_defenses_enabled = hs_opts->HiddenServicePoWDefensesEnabled; - log_info(LD_REND, "Service PoW defenses are %s.", + config->pow_queue_rate = hs_opts->HiddenServicePoWQueueRate; + config->pow_queue_burst = hs_opts->HiddenServicePoWQueueBurst; + + log_info(LD_REND, "Service PoW defenses are %s", config->has_pow_defenses_enabled ? "enabled" : "disabled"); + if (config->has_pow_defenses_enabled) { + log_info(LD_REND, "Service PoW queue rate set to: %" PRIu32, + config->pow_queue_rate); + log_info(LD_REND, "Service PoW queue burst set to: %" PRIu32, + config->pow_queue_burst); + }

/* We do not load the key material for the service at this stage. This is * done later once tor can confirm that it is in a running state. */ diff --git a/src/feature/hs/hs_options.inc b/src/feature/hs/hs_options.inc index 2eb76db40f..4ec62d592b 100644 --- a/src/feature/hs/hs_options.inc +++ b/src/feature/hs/hs_options.inc @@ -32,5 +32,7 @@ CONF_VAR(HiddenServiceEnableIntroDoSRatePerSec, POSINT, 0, "25") CONF_VAR(HiddenServiceEnableIntroDoSBurstPerSec, POSINT, 0, "200") CONF_VAR(HiddenServiceOnionBalanceInstance, BOOL, 0, "0") CONF_VAR(HiddenServicePoWDefensesEnabled, BOOL, 0, "0") +CONF_VAR(HiddenServicePoWQueueRate, POSINT, 0, "250") +CONF_VAR(HiddenServicePoWQueueBurst, POSINT, 0, "2500")

END_CONF_STRUCT(hs_opts_t) diff --git a/src/feature/hs/hs_pow.h b/src/feature/hs/hs_pow.h index 587cae6155..4eb9c5faa6 100644 --- a/src/feature/hs/hs_pow.h +++ b/src/feature/hs/hs_pow.h @@ -15,6 +15,7 @@ typedef unsigned __int128 uint128_t; #include "ext/equix/include/equix.h"

#include "lib/evloop/compat_libevent.h" +#include "lib/evloop/token_bucket.h" #include "lib/smartlist_core/smartlist_core.h"

#define HS_POW_SUGGESTED_EFFORT_DEFAULT 20 // HRPR TODO 5000 @@ -70,6 +71,9 @@ typedef struct hs_pow_service_state_t { * the service's priority queue; higher effort is higher priority. */ mainloop_event_t *pop_pqueue_ev;

+ /* Token bucket for rate limiting the priority queue */ + token_bucket_ctr_t pqueue_bucket; + /* The current seed being used in the PoW defenses. */ uint8_t seed_current[HS_POW_SEED_LEN];

@@ -99,8 +103,12 @@ typedef struct hs_pow_service_state_t { time_t next_effort_update; /* Sum of effort of all valid requests received since the last update. */ uint64_t total_effort; + /* Did we have elements waiting in the queue during this period? */ bool had_queue; + /* Are we using pqueue_bucket to rate limit the pqueue? */ + bool using_pqueue_bucket; + } hs_pow_service_state_t;

/* Struct to store a solution to the PoW challenge. */ diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index fda0162958..dd360d3659 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -279,6 +279,15 @@ initialize_pow_defenses(hs_service_t *service) pow_state->rend_request_pqueue = smartlist_new(); pow_state->pop_pqueue_ev = NULL;

+ if (service->config.pow_queue_rate > 0 && + service->config.pow_queue_burst >= service->config.pow_queue_rate) { + pow_state->using_pqueue_bucket = 1; + token_bucket_ctr_init(&pow_state->pqueue_bucket, + service->config.pow_queue_rate, + service->config.pow_queue_burst, + (uint32_t) approx_time()); + } + pow_state->min_effort = service->config.pow_min_effort;

/* We recalculate and update the suggested effort every HS_UPDATE_PERIOD diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h index 465d9fba80..37984bd6c8 100644 --- a/src/feature/hs/hs_service.h +++ b/src/feature/hs/hs_service.h @@ -265,6 +265,8 @@ typedef struct hs_service_config_t { /** True iff PoW anti-DoS defenses are enabled. */ unsigned int has_pow_defenses_enabled : 1; uint32_t pow_min_effort; + uint32_t pow_queue_rate; + uint32_t pow_queue_burst;

/** If set, contains the Onion Balance master ed25519 public key (taken from * an .onion addresses) that this tor instance serves as backend. */

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 557eb814863285986039699ce92c6c4f11ee7426 Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Wed Mar 1 16:46:25 2023 -0800

hs_pow_solve: use equix_solve more efficiently

This was apparently misinterpreting "zero solutions" as an error instead of just moving on to the next nonce. Additionally, equix could have been returning up to 8 solutions and we would only give one of those a chance to succeed.

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/feature/hs/hs_pow.c | 61 ++++++++++++++++++++----------------------------- 1 file changed, 25 insertions(+), 36 deletions(-)

diff --git a/src/feature/hs/hs_pow.c b/src/feature/hs/hs_pow.c index 2e94f9bced..3c02a4851e 100644 --- a/src/feature/hs/hs_pow.c +++ b/src/feature/hs/hs_pow.c @@ -146,46 +146,35 @@ hs_pow_solve(const hs_pow_desc_params_t *pow_params, challenge = build_equix_challenge(pow_params->seed, nonce, effort);

ctx = equix_alloc(EQUIX_CTX_SOLVE); - equix_solution solution[EQUIX_MAX_SOLS]; + equix_solution solutions[EQUIX_MAX_SOLS];

- /* We'll do a maximum of the nonce size iterations here which is the maximum - * number of nonce we can try in an attempt to find a valid solution. */ log_notice(LD_REND, "Solving proof of work (effort %u)", effort); - for (uint64_t i = 0; i < UINT64_MAX; i++) { - /* Calculate S = equix_solve(C || N || E) */ - if (!equix_solve(ctx, challenge, HS_POW_CHALLENGE_LEN, solution)) { - ret = -1; - goto end; - } - const equix_solution *sol = &solution[0]; - - equix_result result = equix_verify(ctx, challenge, - HS_POW_CHALLENGE_LEN, sol); - if (result != EQUIX_OK) { - /* Go again with a new nonce. */ - increment_and_set_nonce(&nonce, challenge); - continue; - } - - /* Validate the challenge against the solution. */ - if (validate_equix_challenge(challenge, sol, effort)) { - /* Store the nonce N. */ - pow_solution_out->nonce = nonce; - /* Store the effort E. */ - pow_solution_out->effort = effort; - /* We only store the first 4 bytes of the seed C. */ - pow_solution_out->seed_head = get_uint32(pow_params->seed); - /* Store the solution S */ - memcpy(&pow_solution_out->equix_solution, sol, - sizeof(pow_solution_out->equix_solution)); - - /* Indicate success and we are done. */ - ret = 0; - break; + for (;;) { + /* Calculate solutions to S = equix_solve(C || N || E), */ + int count = equix_solve(ctx, challenge, HS_POW_CHALLENGE_LEN, solutions); + for (int i = 0; i < count; i++) { + const equix_solution *sol = &solutions[i]; + + /* Check an Equi-X solution against the effort threshold */ + if (validate_equix_challenge(challenge, sol, effort)) { + /* Store the nonce N. */ + pow_solution_out->nonce = nonce; + /* Store the effort E. */ + pow_solution_out->effort = effort; + /* We only store the first 4 bytes of the seed C. */ + pow_solution_out->seed_head = get_uint32(pow_params->seed); + /* Store the solution S */ + memcpy(&pow_solution_out->equix_solution, sol, + sizeof(pow_solution_out->equix_solution)); + + /* Indicate success and we are done. */ + ret = 0; + goto end; + } }

- /* Did not pass the R * E <= UINT32_MAX check. Increment the nonce and - * try again. */ + /* No solutions for this nonce and/or none that passed the effort + * threshold, increment and try again. */ increment_and_set_nonce(&nonce, challenge); }

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit dcb9c4df67d116dc16f5361c8e4cd6e21fbb9abf Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Wed Mar 8 15:44:55 2023 -0800

hs_pow: Make proof-of-work support optional in configure

This adds a new "pow" module for the user-visible proof of work support in ./configure, and this disables src/feature/hs/hs_pow at compile-time.

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- configure.ac | 37 ++++++++++++++++++++++++++---- src/app/config/config.c | 20 ++++++++++++----- src/core/include.am | 1 + src/feature/hs/hs_circuit.c | 2 +- src/feature/hs/hs_client.c | 11 +++++---- src/feature/hs/hs_config.c | 6 +++++ src/feature/hs/hs_pow.c | 6 ++--- src/feature/hs/hs_pow.h | 55 ++++++++++++++++++++++++++++++++++++++++++--- src/feature/hs/hs_service.c | 8 ++++--- src/feature/hs/include.am | 9 +++++++- src/test/test_parseconf.sh | 6 +++++ 11 files changed, 137 insertions(+), 24 deletions(-)

diff --git a/configure.ac b/configure.ac index a9ea853611..eaef9d2703 100644 --- a/configure.ac +++ b/configure.ac @@ -373,7 +373,7 @@ dnl Tor modules options. These options are namespaced with --disable-module-XXX dnl ---

dnl All our modules. -m4_define(MODULES, relay dirauth dircache) +m4_define([MODULES], [relay dirauth dircache pow])

# Some modules are only disabled through another option. For those, we don't # want to print the help in the summary at the end of the configure. Any entry @@ -382,6 +382,9 @@ m4_define(MODULES, relay dirauth dircache) m4_set_add_all([MODULES_WITH_NO_OPTIONS], [dircache])

dnl Relay module. +m4_define([module_option_hints(relay)], + [AS_IF([test "x$value" = x1], [HINT_OPT([--disable-module-relay])], + [HINT_NONE])]) AC_ARG_ENABLE([module-relay], AS_HELP_STRING([--disable-module-relay], [Build tor without the Relay modules: tor can not run as a relay, bridge, or authority. Implies --disable-module-dirauth])) @@ -399,6 +402,9 @@ AM_COND_IF(BUILD_MODULE_DIRCACHE, [Compile with directory cache support]))

dnl Directory Authority module. +m4_define([module_option_hints(dirauth)], + [AS_IF([test "x$value" = x1], [HINT_OPT([--disable-module-dirauth])], + [HINT_NONE])]) AC_ARG_ENABLE([module-dirauth], AS_HELP_STRING([--disable-module-dirauth], [Build tor without the Directory Authority module: tor can not run as a directory authority or bridge authority])) @@ -407,6 +413,19 @@ AM_COND_IF(BUILD_MODULE_DIRAUTH, AC_DEFINE([HAVE_MODULE_DIRAUTH], [1], [Compile with Directory Authority feature support]))

+dnl Hidden Service Proof-of-Work module. +m4_define([module_option_hints(pow)], + [AS_IF([test "x$value" = x1], [HINT_OPT([--disable-module-pow])], + [AS_IF([test "x$license_option" != "xGPL"], [HINT_OPT([requires --enable-gpl])], + [HINT_NONE])])]) +AC_ARG_ENABLE([module-pow], + AS_HELP_STRING([--disable-module-pow], + [Build tor without proof-of-work denial of service mitigation, normally available when building with --enable-gpl])) +AM_CONDITIONAL(BUILD_MODULE_POW, + [test "x$license_option" = "xGPL" && test "x$enable_module_pow" != "xno"]) +AM_COND_IF(BUILD_MODULE_POW, + AC_DEFINE([HAVE_MODULE_POW], [1], [Compile with proof-of-work support])) + dnl Helper variables. TOR_MODULES_ALL_ENABLED= AC_DEFUN([ADD_MODULE], [ @@ -2733,12 +2752,22 @@ PPRINT_PROP_BOOL([Fragile Hardening (--enable-fragile-hardening, dev only)], $va AS_ECHO PPRINT_SUBTITLE([Modules])

+# Modules have documentation hints indicating how they can be enabled +# or disabled, and those hints can select which version of our message +# to show based on variables at configure-time. +# +# Each "module_option_hints(<name>)" macro, if it exists, must +# visit exactly one HINT_* macro using shell conditionals. + m4_foreach_w([mname], MODULES, [ AM_COND_IF(m4_join([], [BUILD_MODULE_], m4_toupper([]mname[])), value=1, value=0) - m4_set_contains([MODULES_WITH_NO_OPTIONS], mname, - PPRINT_PROP_BOOL([mname], $value), - PPRINT_PROP_BOOL([mname (--disable-module-mname)], $value)) + m4_pushdef([HINT_OPT], [PPRINT_PROP_BOOL](mname ($1), [[$value]])) + m4_pushdef([HINT_NONE], [PPRINT_PROP_BOOL](mname, [[$value]])) + m4_ifdef([module_option_hints](mname), + [m4_indir([module_option_hints](mname))], + [HINT_NONE]) + m4_popdef([HINT_OPT], [HINT_NONE]) ] )

diff --git a/src/app/config/config.c b/src/app/config/config.c index cb71d0fb6d..24321b764f 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -88,9 +88,11 @@ #include "feature/control/control.h" #include "feature/control/control_auth.h" #include "feature/control/control_events.h" +#include "feature/dircache/dirserv.h" #include "feature/dirclient/dirclient_modes.h" #include "feature/hibernate/hibernate.h" #include "feature/hs/hs_config.h" +#include "feature/hs/hs_pow.h" #include "feature/metrics/metrics.h" #include "feature/nodelist/dirlist.h" #include "feature/nodelist/networkstatus.h" @@ -2731,11 +2733,19 @@ list_deprecated_options(void) static void list_enabled_modules(void) { - printf("%s: %s\n", "relay", have_module_relay() ? "yes" : "no"); - printf("%s: %s\n", "dirauth", have_module_dirauth() ? "yes" : "no"); - // We don't list dircache, because it cannot be enabled or disabled - // independently from relay. Listing it here would proliferate - // test variants in test_parseconf.sh to no useful purpose. + static const struct { + const char *name; + bool have; + } list[] = { + { "relay", have_module_relay() }, + { "dirauth", have_module_dirauth() }, + { "dircache", have_module_dircache() }, + { "pow", have_module_pow() } + }; + + for (unsigned i = 0; i < sizeof list / sizeof list[0]; i++) { + printf("%s: %s\n", list[i].name, list[i].have ? "yes" : "no"); + } }

/** Prints compile-time and runtime library versions. */ diff --git a/src/core/include.am b/src/core/include.am index 7752a7974b..d24e5d5137 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -17,6 +17,7 @@ if UNITTESTS_ENABLED LIBTOR_APP_TESTING_A_SOURCES += $(MODULE_RELAY_SOURCES) LIBTOR_APP_TESTING_A_SOURCES += $(MODULE_DIRCACHE_SOURCES) LIBTOR_APP_TESTING_A_SOURCES += $(MODULE_DIRAUTH_SOURCES) +LIBTOR_APP_TESTING_A_SOURCES += $(MODULE_POW_SOURCES)

src_core_libtor_app_testing_a_SOURCES = $(LIBTOR_APP_TESTING_A_SOURCES) else diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 55b992ee28..f7ab6442b9 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -1369,7 +1369,7 @@ hs_circ_handle_introduce2(const hs_service_t *service,

/* Add the rendezvous request to the priority queue if PoW defenses are * enabled, otherwise rendezvous as usual. */ - if (service->config.has_pow_defenses_enabled) { + if (have_module_pow() && service->config.has_pow_defenses_enabled) { log_notice(LD_REND, "Adding introduction request to pqueue with effort: %u", data.rdv_data.pow_effort); diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index 56547de7e7..6a404395ea 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -733,7 +733,8 @@ consider_sending_introduce1(origin_circuit_t *intro_circ,

/* If the descriptor contains PoW parameters then the service is * expecting a PoW solution in the INTRODUCE cell, which we solve here. */ - if (desc->encrypted_data.pow_params && + if (have_module_pow() && + desc->encrypted_data.pow_params && desc->encrypted_data.pow_params->suggested_effort > 0) { log_debug(LD_REND, "PoW params present in descriptor.");

@@ -752,9 +753,11 @@ consider_sending_introduce1(origin_circuit_t *intro_circ,

/* send it to the client-side pow cpuworker for solving. */ intro_circ->hs_currently_solving_pow = 1; - pow_queue_work(intro_circ->global_identifier, - rend_circ->global_identifier, - desc->encrypted_data.pow_params); + if (0 != hs_pow_queue_work(intro_circ->global_identifier, + rend_circ->global_identifier, + desc->encrypted_data.pow_params)) { + log_debug(LD_REND, "Failed to enqueue PoW request"); + }

/* can't proceed with the intro1 cell yet, so yield back to the * main loop */ diff --git a/src/feature/hs/hs_config.c b/src/feature/hs/hs_config.c index 0f5a8cf49a..296941138b 100644 --- a/src/feature/hs/hs_config.c +++ b/src/feature/hs/hs_config.c @@ -327,6 +327,12 @@ config_validate_service(const hs_service_config_t *config) config->pow_queue_burst, config->pow_queue_rate); goto invalid; } + if (config->has_pow_defenses_enabled && !have_module_pow()) { + log_warn(LD_CONFIG, "Hidden service proof-of-work defenses are enabled " + "in our configuration but this build of tor does not " + "include the required 'pow' module."); + goto invalid; + }

/* Valid. */ return 0; diff --git a/src/feature/hs/hs_pow.c b/src/feature/hs/hs_pow.c index 3c02a4851e..8ca121762f 100644 --- a/src/feature/hs/hs_pow.c +++ b/src/feature/hs/hs_pow.c @@ -410,9 +410,9 @@ pow_worker_replyfn(void *work_) * Queue the job of solving the pow in a worker thread. */ int -pow_queue_work(uint32_t intro_circ_identifier, - uint32_t rend_circ_identifier, - const hs_pow_desc_params_t *pow_params) +hs_pow_queue_work(uint32_t intro_circ_identifier, + uint32_t rend_circ_identifier, + const hs_pow_desc_params_t *pow_params) { tor_assert(in_main_thread());

diff --git a/src/feature/hs/hs_pow.h b/src/feature/hs/hs_pow.h index 92ea011b2b..b27bd7441c 100644 --- a/src/feature/hs/hs_pow.h +++ b/src/feature/hs/hs_pow.h @@ -127,6 +127,9 @@ typedef struct hs_pow_solution_t { equix_solution equix_solution; } hs_pow_solution_t;

+#ifdef HAVE_MODULE_POW +#define have_module_pow() (1) + /* API */ int hs_pow_solve(const hs_pow_desc_params_t *pow_params, hs_pow_solution_t *pow_solution_out); @@ -137,8 +140,54 @@ int hs_pow_verify(const hs_pow_service_state_t *pow_state, void hs_pow_remove_seed_from_cache(uint32_t seed); void hs_pow_free_service_state(hs_pow_service_state_t *state);

-int pow_queue_work(uint32_t intro_circ_identifier, - uint32_t rend_circ_identifier, - const hs_pow_desc_params_t *pow_params); +int hs_pow_queue_work(uint32_t intro_circ_identifier, + uint32_t rend_circ_identifier, + const hs_pow_desc_params_t *pow_params); + +#else /* !defined(HAVE_MODULE_POW) */ +#define have_module_pow() (0) + +static inline int +hs_pow_solve(const hs_pow_desc_params_t *pow_params, + hs_pow_solution_t *pow_solution_out) +{ + (void)pow_params; + (void)pow_solution_out; + return -1; +} + +static inline int +hs_pow_verify(const hs_pow_service_state_t *pow_state, + const hs_pow_solution_t *pow_solution) +{ + (void)pow_state; + (void)pow_solution; + return -1; +} + +static inline void +hs_pow_remove_seed_from_cache(uint32_t seed) +{ + (void)seed; +} + +static inline void +hs_pow_free_service_state(hs_pow_service_state_t *state) +{ + (void)state; +} + +static inline int +hs_pow_queue_work(uint32_t intro_circ_identifier, + uint32_t rend_circ_identifier, + const hs_pow_desc_params_t *pow_params) +{ + (void)intro_circ_identifier; + (void)rend_circ_identifier; + (void)pow_params; + return -1; +} + +#endif /* defined(HAVE_MODULE_POW) */

#endif /* !defined(TOR_HS_POW_H) */ diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index dd360d3659..a9070024cb 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -2899,7 +2899,7 @@ run_housekeeping_event(time_t now)

/* Check if we need to initialize or update PoW parameters, if the * defenses are enabled. */ - if (service->config.has_pow_defenses_enabled) { + if (have_module_pow() && service->config.has_pow_defenses_enabled) { pow_housekeeping(service, now); }

@@ -2937,8 +2937,10 @@ run_build_descriptor_event(time_t now) * is useful for newly built descriptors. */ update_all_descriptors_intro_points(now);

- /* Update the PoW params if needed. */ - update_all_descriptors_pow_params(now); + if (have_module_pow()) { + /* Update the PoW params if needed. */ + update_all_descriptors_pow_params(now); + } }

/** For the given service, launch any intro point circuits that could be diff --git a/src/feature/hs/include.am b/src/feature/hs/include.am index f4966e6c54..b64ab1b41c 100644 --- a/src/feature/hs/include.am +++ b/src/feature/hs/include.am @@ -15,12 +15,19 @@ LIBTOR_APP_A_SOURCES += \ src/feature/hs/hs_intropoint.c \ src/feature/hs/hs_metrics.c \ src/feature/hs/hs_ob.c \ - src/feature/hs/hs_pow.c \ src/feature/hs/hs_service.c \ src/feature/hs/hs_stats.c \ src/feature/hs/hs_sys.c \ src/feature/hs/hs_metrics_entry.c

+# Proof of Work module +MODULE_POW_SOURCES = \ + src/feature/hs/hs_pow.c + +if BUILD_MODULE_POW +LIBTOR_APP_A_SOURCES += $(MODULE_POW_SOURCES) +endif + # ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/feature/hs/hs_cache.h \ diff --git a/src/test/test_parseconf.sh b/src/test/test_parseconf.sh index c02b8b23c0..85a8cbbf0c 100755 --- a/src/test/test_parseconf.sh +++ b/src/test/test_parseconf.sh @@ -98,6 +98,9 @@ # want to encode that knowledge in this test script, so we supply a # separate result file for every combination of disabled modules that # has a different result.) +# +# This logic ignores modules that are not listed by --list-modules +# (dircache) and some that do not currently affect config parsing (pow).

umask 077 set -e @@ -197,6 +200,8 @@ echo "This pattern should not match any log messages" \ "$NON_EMPTY"

STANDARD_LIBS="libevent\|openssl\|zlib" +MODULES_WITHOUT_CONFIG_TESTS="dircache\|pow" + # Lib names are restricted to [a-z0-9]* at the moment # We don't actually want to support foreign accents here # shellcheck disable=SC2018,SC2019 @@ -229,6 +234,7 @@ TOR_LIBS_ENABLED_SEARCH="$(echo "$TOR_LIBS_ENABLED_SEARCH" | tr ' ' '\n' \ | grep -v '^_*$' | tr '\n' ' ')"

TOR_MODULES_DISABLED="$("$TOR_BINARY" --list-modules | grep ': no' \ + | grep -v "$MODULES_WITHOUT_CONFIG_TESTS" \ | cut -d ':' -f1 | sort | tr '\n' '_')" # Remove the last underscore, if there is one TOR_MODULES_DISABLED=${TOR_MODULES_DISABLED%_}

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit ffa8531fe0f495e45ade4910b7a1c0d7e56d78ba Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Thu Mar 9 15:11:28 2023 -0800

test_crypto: add equix and hashx tests

This adds test vectors for the Equi-X proof of work algorithm and the Hash-X function it's based on. The overall Equi-X test takes about 10 seconds to run on my machine, so it's in test_crypto_slow. The hashx test still covers both the compiled and interpreted versions of the hash function.

There aren't any official test vectors for Equi-X or for its particular configuration of Hash-X, so I made some up based on the current implementation.

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/ext/include.am | 5 ++ src/test/test_crypto.c | 72 ++++++++++++++++++++++++ src/test/test_crypto_slow.c | 134 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+)

diff --git a/src/ext/include.am b/src/ext/include.am index 8b646b1b4e..f7dc9fe59a 100644 --- a/src/ext/include.am +++ b/src/ext/include.am @@ -1,6 +1,11 @@

AM_CPPFLAGS += -I$(srcdir)/src/ext -Isrc/ext

+# TODO: put this with other equix/hashx ext defs when those happen, +# and also add it to the autoconf config header. For now this is here +# just for test_crypto's benefit. +AM_CPPFLAGS += -DHASHX_SIZE=8 + EXTRA_DIST += src/ext/ext.md

EXTHEADERS = \ diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 6583fbfbda..1f66a6251a 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -10,6 +10,7 @@ #include "test/test.h" #include "lib/crypt_ops/aes.h" #include "siphash.h" +#include "ext/equix/hashx/include/hashx.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_dh.h" #include "lib/crypt_ops/crypto_ed25519.h" @@ -2957,6 +2958,76 @@ test_crypto_blake2b(void *arg) ; }

+static void +test_crypto_hashx(void *arg) +{ + (void)arg; + + /* Specifically test the embedded instance of HashX inside Equi-X. + * It uses a non-default setting of HASHX_SIZE=8 */ + static const struct { + const char *seed_literal; + uint64_t hash_input; + const char *out_hex; + } vectors[] = { + { "", 0, "466cc2021c268560" }, + { "a", 0, "b2a110ee695c475c" }, + { "ab", 0, "57c77f7e0d2c1727" }, + { "abc", 0, "ef560991338086d1" }, + { "", 999, "304068b62bc4874e" }, + { "a", 999, "c8b66a8eb4bba304" }, + { "ab", 999, "26c1f7031f0b3645" }, + { "abc", 999, "de84f9d286b39ab5" }, + { "abc", UINT64_MAX, "f756c266a3cb3b5a" } + }; + + static const struct { + hashx_type type; + } variations[] = { + { HASHX_INTERPRETED }, +#if defined(_M_X64) || defined(__x86_64__) || defined(__aarch64__) + { HASHX_COMPILED }, +#endif + }; + + const unsigned num_vectors = sizeof vectors / sizeof vectors[0]; + const unsigned num_variations = sizeof variations / sizeof variations[0]; + + for (unsigned vec_i = 0; vec_i < num_vectors; vec_i++) { + const char *seed_literal = vectors[vec_i].seed_literal; + const uint64_t hash_input = vectors[vec_i].hash_input; + const char *out_hex = vectors[vec_i].out_hex; + const size_t seed_len = strlen(seed_literal); + const size_t out_hex_len = strlen(out_hex); + + int retval = -1; + uint8_t out_expected[HASHX_SIZE] = { 0 }; + tt_int_op(out_hex_len, OP_EQ, 2 * HASHX_SIZE); + retval = base16_decode((char*)out_expected, HASHX_SIZE, + out_hex, out_hex_len); + tt_int_op(retval, OP_EQ, HASHX_SIZE); + + for (unsigned vari_i = 0; vari_i < num_variations; vari_i++) { + uint8_t out_actual[HASHX_SIZE] = { 0 }; + + hashx_ctx *ctx = hashx_alloc(variations[vari_i].type); + tt_ptr_op(ctx, OP_NE, NULL); + tt_ptr_op(ctx, OP_NE, HASHX_NOTSUPP); + retval = hashx_make(ctx, seed_literal, seed_len); + tt_int_op(retval, OP_EQ, 1); + + memset(out_actual, 0xa5, sizeof out_actual); + hashx_exec(ctx, hash_input, out_actual); + tt_mem_op(out_actual, OP_EQ, out_expected, sizeof out_actual); + + hashx_free(ctx); + } + } + + done: + ; +} + /* We want the likelihood that the random buffer exhibits any regular pattern * to be far less than the memory bit error rate in the int return value. * Using 2048 bits provides a failure rate of 1/(3 * 10^616), and we call @@ -3187,6 +3258,7 @@ struct testcase_t crypto_tests[] = { { "ed25519_storage", test_crypto_ed25519_storage, 0, NULL, NULL }, { "siphash", test_crypto_siphash, 0, NULL, NULL }, { "blake2b", test_crypto_blake2b, 0, NULL, NULL }, + { "hashx", test_crypto_hashx, 0, NULL, NULL }, { "failure_modes", test_crypto_failure_modes, TT_FORK, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_crypto_slow.c b/src/test/test_crypto_slow.c index bcfea10cf6..5ffd199813 100644 --- a/src/test/test_crypto_slow.c +++ b/src/test/test_crypto_slow.c @@ -7,6 +7,7 @@ #define CRYPTO_S2K_PRIVATE #include "core/or/or.h" #include "test/test.h" +#include "ext/equix/include/equix.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_ed25519.h" #include "lib/crypt_ops/crypto_s2k.h" @@ -584,6 +585,138 @@ test_crypto_ed25519_fuzz_donna(void *arg) ; }

+static void +test_crypto_equix(void *arg) +{ + (void)arg; + + static const struct { + const char *challenge_literal; + size_t num_solutions; + equix_solution solutions[EQUIX_MAX_SOLS]; + } vectors[] = { + { "zzz", 1, { + {{ 0xae21, 0xd392, 0x3215, 0xdd9c, 0x2f08, 0x93df, 0x232c, 0xe5dc }}, + }}, + { "rrr", 1, { + {{ 0x0873, 0x57a8, 0x73e0, 0x912e, 0x1ca8, 0xad96, 0x9abd, 0xd7de }}, + }}, + { "qqq", 0, {{{ 0 }}} }, + { "0123456789", 0, {{{ 0 }}} }, + { "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 0, {{{ 0 }}} }, + { "", 3, { + {{ 0x0098, 0x3a4d, 0xc489, 0xcfba, 0x7ef3, 0xa498, 0xa00f, 0xec20 }}, + {{ 0x78d8, 0x8611, 0xa4df, 0xec19, 0x0927, 0xa729, 0x842f, 0xf771 }}, + {{ 0x54b5, 0xcc11, 0x1593, 0xe624, 0x9357, 0xb339, 0xb138, 0xed99 }}, + }}, + { "a", 3, { + {{ 0x4b38, 0x8c81, 0x9255, 0xad99, 0x5ce7, 0xeb3e, 0xc635, 0xee38 }}, + {{ 0x3f9e, 0x659b, 0x9ae6, 0xb891, 0x63ae, 0x777c, 0x06ca, 0xc593 }}, + {{ 0x2227, 0xa173, 0x365a, 0xb47d, 0x1bb2, 0xa077, 0x0d5e, 0xf25f }}, + }}, + { "abc", 2, { + {{ 0x371f, 0x8865, 0x8189, 0xfbc3, 0x26df, 0xe4c0, 0xab39, 0xfe5a }}, + {{ 0x2101, 0xb88f, 0xc525, 0xccb3, 0x5785, 0xa41e, 0x4fba, 0xed18 }}, + }}, + { "abce", 4, { + {{ 0x4fca, 0x72eb, 0x101f, 0xafab, 0x1add, 0x2d71, 0x75a3, 0xc978 }}, + {{ 0x17f1, 0x7aa6, 0x23e3, 0xab00, 0x7e2f, 0x917e, 0x16da, 0xda9e }}, + {{ 0x70ee, 0x7757, 0x8a54, 0xbd2b, 0x90e4, 0xe31e, 0x2085, 0xe47e }}, + {{ 0x62c5, 0x86d1, 0x5752, 0xe1f0, 0x12da, 0x8f33, 0x7336, 0xf161 }}, + }}, + { "01234567890123456789", 5, { + {{ 0x4803, 0x6775, 0xc5c9, 0xd1b0, 0x1bc3, 0xe4f6, 0x4027, 0xf5ad }}, + {{ 0x5a8a, 0x9542, 0xef99, 0xf0b9, 0x4905, 0x4e29, 0x2da5, 0xfbd5 }}, + {{ 0x4c79, 0xc935, 0x2bcb, 0xcd0f, 0x0362, 0x9fa9, 0xa62e, 0xf83a }}, + {{ 0x5878, 0x6edf, 0x1e00, 0xf5e3, 0x43de, 0x9212, 0xd01e, 0xfd11 }}, + {{ 0x0b69, 0x2d17, 0x01be, 0x6cb4, 0x0fba, 0x4a9e, 0x8d75, 0xa50f }}, + }}, + }; + + static const struct { + equix_ctx_flags flags; + equix_result expected; + } variations[] = { + {0, EQUIX_OK}, + {0, EQUIX_ORDER}, + {0, EQUIX_PARTIAL_SUM}, +#if defined(_M_X64) || defined(__x86_64__) || defined(__aarch64__) + {EQUIX_CTX_COMPILE, EQUIX_OK}, + {EQUIX_CTX_COMPILE, EQUIX_ORDER}, + {EQUIX_CTX_COMPILE, EQUIX_PARTIAL_SUM}, +#endif + }; + + const unsigned num_vectors = sizeof vectors / sizeof vectors[0]; + const unsigned num_variations = sizeof variations / sizeof variations[0]; + + for (unsigned vec_i = 0; vec_i < num_vectors; vec_i++) { + const char *challenge_literal = vectors[vec_i].challenge_literal; + const size_t challenge_len = strlen(challenge_literal); + + const size_t num_sols = vectors[vec_i].num_solutions; + const equix_solution *sols_expected = vectors[vec_i].solutions; + + for (unsigned vari_i = 0; vari_i < num_variations; vari_i++) { + const equix_ctx_flags flags = variations[vari_i].flags; + const equix_result expected = variations[vari_i].expected; + + equix_solution sols_actual[EQUIX_MAX_SOLS]; + equix_ctx *solve_ctx = NULL, *verify_ctx = NULL; + + solve_ctx = equix_alloc(EQUIX_CTX_SOLVE | flags); + tt_ptr_op(solve_ctx, OP_NE, NULL); + tt_ptr_op(solve_ctx, OP_NE, EQUIX_NOTSUPP); + + /* Solve phase: Make sure the test vector matches */ + memset(sols_actual, 0xa5, sizeof sols_actual); + int retval = equix_solve(solve_ctx, challenge_literal, + challenge_len, sols_actual); + tt_int_op(retval, OP_EQ, num_sols); + tt_mem_op(sols_actual, OP_EQ, sols_expected, + num_sols * sizeof(equix_solution)); + + verify_ctx = equix_alloc(EQUIX_CTX_VERIFY | flags); + tt_ptr_op(verify_ctx, OP_NE, NULL); + tt_ptr_op(verify_ctx, OP_NE, EQUIX_NOTSUPP); + + /* Use each solution for positive and negative tests of verify */ + for (size_t sol_i = 0; sol_i < num_sols; sol_i++) { + equix_result result; + equix_idx tmp_idx; + equix_solution *sol = &sols_actual[sol_i]; + + switch (expected) { + case EQUIX_ORDER: + /* Swap two otherwise valid indices, to trigger an order error */ + tmp_idx = sol->idx[0]; + sol->idx[0] = sol->idx[1]; + sol->idx[1] = tmp_idx; + break; + case EQUIX_FINAL_SUM: + case EQUIX_PARTIAL_SUM: + /* Most changes to the solution will cause a partial sum error */ + sol->idx[0]++; + break; + case EQUIX_OK: + case EQUIX_CHALLENGE: + break; + } + + result = equix_verify(verify_ctx, challenge_literal, + challenge_len, sol); + tt_int_op(expected, OP_EQ, result); + } + + equix_free(solve_ctx); + equix_free(verify_ctx); + } + } + + done: + ; +} + #ifndef COCCI #define CRYPTO_LEGACY(name) \ { #name, test_crypto_ ## name , 0, NULL, NULL } @@ -619,5 +752,6 @@ struct testcase_t slow_crypto_tests[] = { { "pbkdf2_vectors", test_crypto_pbkdf2_vectors, 0, NULL, NULL }, { "pwbox", test_crypto_pwbox, 0, NULL, NULL }, ED25519_TEST(fuzz_donna, TT_FORK), + { "equix", test_crypto_equix, 0, NULL, NULL }, END_OF_TESTCASES };

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 246ced3a8ce7bbfb60f14dec7e138670afdab647 Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Thu Mar 9 19:33:41 2023 -0800

ext: build equix and hashx using automake

This replaces the sketchy cmake invocation we had inside configure

The libs are always built and always used in unit tests, but only included in libtor and tor when --enable-gpl is set.

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- .gitignore | 1 - Makefile.am | 17 ++++++------ configure.ac | 13 +++++++-- src/ext/equix/.gitmodules | 3 --- src/ext/equix/build.sh | 7 ----- src/ext/equix/hashx/.gitignore | 9 ------- src/ext/include.am | 60 +++++++++++++++++++++++++++++++++++++----- 7 files changed, 73 insertions(+), 37 deletions(-)

diff --git a/.gitignore b/.gitignore index 737ab72bf5..94988ed982 100644 --- a/.gitignore +++ b/.gitignore @@ -152,7 +152,6 @@ core.* # /src/ext/ /src/ext/ed25519/ref10/libed25519_ref10.lib /src/ext/ed25519/donna/libed25519_donna.lib -/src/ext/equix/build /src/ext/keccak-tiny/libkeccak-tiny.lib

# /src/app diff --git a/Makefile.am b/Makefile.am index 28a1c8e57d..21f71e2e05 100644 --- a/Makefile.am +++ b/Makefile.am @@ -114,6 +114,10 @@ TOR_CRYPTO_LIBS = \ $(LIBKECCAK_TINY) \ $(LIBDONNA)

+if BUILD_MODULE_POW +TOR_CRYPTO_LIBS += $(EQUIX_LIBS) +endif + # Variants of the above for linking the testing variant of tor (for coverage # and tests) if UNITTESTS_ENABLED @@ -121,13 +125,10 @@ TOR_CRYPTO_TESTING_LIBS = \ src/lib/libtor-tls-testing.a \ src/lib/libtor-crypt-ops-testing.a \ $(LIBKECCAK_TINY) \ - $(LIBDONNA) + $(LIBDONNA) \ + $(EQUIX_LIBS) endif

-EQUIX_LIBS = \ - src/ext/equix/build/libequix.a \ - src/ext/equix/build/hashx/libhashx.a - # All static libraries used to link tor. TOR_INTERNAL_LIBS = \ src/core/libtor-app.a \ @@ -136,8 +137,7 @@ TOR_INTERNAL_LIBS = \ $(TOR_CRYPTO_LIBS) \ $(TOR_UTIL_LIBS) \ src/trunnel/libor-trunnel.a \ - src/lib/libtor-trace.a \ - $(EQUIX_LIBS) + src/lib/libtor-trace.a

libtor.a: $(TOR_INTERNAL_LIBS) $(AM_V_AR) export AR="$(AR)"; \ @@ -157,8 +157,7 @@ TOR_INTERNAL_TESTING_LIBS = \ $(TOR_CRYPTO_TESTING_LIBS) \ $(TOR_UTIL_TESTING_LIBS) \ src/trunnel/libor-trunnel-testing.a \ - src/lib/libtor-trace.a \ - $(EQUIX_LIBS) + src/lib/libtor-trace.a

src/test/libtor-testing.a: $(TOR_INTERNAL_TESTING_LIBS) $(AM_V_AR) export AR="$(AR)"; \ diff --git a/configure.ac b/configure.ac index 2b20965d68..7f01e5b076 100644 --- a/configure.ac +++ b/configure.ac @@ -31,8 +31,6 @@ tor_incr_n_warnings() { tor_ac_n_warnings=`expr $tor_ac_n_warnings + 1` }

-AC_CONFIG_COMMANDS([equix], [./src/ext/equix/build.sh]) - m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) AC_CONFIG_HEADERS([orconfig.h])

@@ -2602,6 +2600,17 @@ if test "$enable_coverage" = "yes" && test "$have_clang" = "no"; then esac fi

+# These HashX parameter definitions are needed in CPPFLAGS when compiling +# the equix and hashx ext modules, but elsewhere in tor we can use orconfig.h + +m4_define([equix_hashx_size], [8]) +[HASHX_SIZE=]equix_hashx_size +AC_SUBST([HASHX_SIZE]) +AC_DEFINE([HASHX_SIZE], equix_hashx_size, + [Output size in bytes for the internal customization of HashX]) +AC_DEFINE([HASHX_STATIC], [1], [We statically link with HashX]) +AC_DEFINE([EQUIX_STATIC], [1], [We statically link with EquiX]) + CPPFLAGS="$CPPFLAGS $TOR_CPPFLAGS_libevent $TOR_CPPFLAGS_openssl $TOR_CPPFLAGS_zlib"

AC_CONFIG_FILES([ diff --git a/src/ext/equix/.gitmodules b/src/ext/equix/.gitmodules deleted file mode 100644 index 6f1277ee9f..0000000000 --- a/src/ext/equix/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "hashx"] - path = hashx - url = https://github.com/tevador/hashx diff --git a/src/ext/equix/build.sh b/src/ext/equix/build.sh deleted file mode 100755 index 753e3a138e..0000000000 --- a/src/ext/equix/build.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -cd ./src/ext/equix -mkdir build -cd build -cmake .. -make diff --git a/src/ext/equix/hashx/.gitignore b/src/ext/equix/hashx/.gitignore deleted file mode 100644 index ec94c2c699..0000000000 --- a/src/ext/equix/hashx/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -bin/ -obj/ -*.user -*.suo -.vs -x64/ -Release/ -Debug/ -build/ diff --git a/src/ext/include.am b/src/ext/include.am index dea8e4419b..dad6a592b7 100644 --- a/src/ext/include.am +++ b/src/ext/include.am @@ -1,14 +1,8 @@

AM_CPPFLAGS += \ -I$(srcdir)/src/ext/ \ - -I$(srcdir)/src/ext/equix/include/ \ -I$(srcdir)/src/ext/equix/hashx/include/

-# TODO: put this with other equix/hashx ext defs when those happen, -# and also add it to the autoconf config header. For now this is here -# just for test_crypto's benefit. -AM_CPPFLAGS += -DHASHX_SIZE=8 - EXTRA_DIST += src/ext/ext.md

EXTHEADERS = \ @@ -152,6 +146,60 @@ noinst_HEADERS += $(ED25519_DONNA_HDRS) LIBED25519_DONNA=src/ext/ed25519/donna/libed25519_donna.a noinst_LIBRARIES += $(LIBED25519_DONNA)

+src_ext_equix_libhashx_a_CPPFLAGS = \ + -I$(srcdir)/src/ext/equix/hashx/include/ \ + -I$(srcdir)/src/ext/equix/hashx/src/ \ + -DHASHX_SIZE=@HASHX_SIZE@ \ + -DEQUIX_STATIC=1 -DHASHX_STATIC=1 + +src_ext_equix_libhashx_a_SOURCES = \ + src/ext/equix/hashx/src/blake2.c \ + src/ext/equix/hashx/src/compiler.c \ + src/ext/equix/hashx/src/compiler_a64.c \ + src/ext/equix/hashx/src/compiler_x86.c \ + src/ext/equix/hashx/src/context.c \ + src/ext/equix/hashx/src/hashx.c \ + src/ext/equix/hashx/src/program.c \ + src/ext/equix/hashx/src/program_exec.c \ + src/ext/equix/hashx/src/siphash.c \ + src/ext/equix/hashx/src/siphash_rng.c \ + src/ext/equix/hashx/src/virtual_memory.c + +src_ext_equix_libequix_a_CPPFLAGS = \ + -I$(srcdir)/src/ext/equix/include/ \ + -I$(srcdir)/src/ext/equix/src/ \ + $(src_ext_equix_libhashx_a_CPPFLAGS) + +src_ext_equix_libequix_a_SOURCES = \ + src/ext/equix/src/context.c \ + src/ext/equix/src/equix.c \ + src/ext/equix/src/solver.c + +EQUIX_HDRS = \ + src/ext/equix/hashx/include/hashx.h \ + src/ext/equix/hashx/src/blake2.h \ + src/ext/equix/hashx/src/compiler.h \ + src/ext/equix/hashx/src/context.h \ + src/ext/equix/hashx/src/force_inline.h \ + src/ext/equix/hashx/src/hashx_endian.h \ + src/ext/equix/hashx/src/instruction.h \ + src/ext/equix/hashx/src/program.h \ + src/ext/equix/hashx/src/siphash_rng.h \ + src/ext/equix/hashx/src/siphash.h \ + src/ext/equix/hashx/src/unreachable.h \ + src/ext/equix/hashx/src/virtual_memory.h \ + src/ext/equix/include/equix.h \ + src/ext/equix/src/context.h \ + src/ext/equix/src/solver_heap.h \ + src/ext/equix/src/solver.h + +EQUIX_LIBS = \ + src/ext/equix/libhashx.a \ + src/ext/equix/libequix.a + +noinst_HEADERS += $(EQUIX_HDRS) +noinst_LIBRARIES += $(EQUIX_LIBS) + if BUILD_KECCAK_TINY src_ext_keccak_tiny_libkeccak_tiny_a_CFLAGS=\ @CFLAGS_CONSTTIME@

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit ae86d98815ad61c447cd81c0060d403641e3071a Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Wed Mar 15 13:34:21 2023 -0700

equix: Portability fixes for big endian platforms

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/ext/equix/hashx/src/hashx.c | 18 +++++++++++++++--- src/ext/equix/src/solver.h | 18 ++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/src/ext/equix/hashx/src/hashx.c b/src/ext/equix/hashx/src/hashx.c index 1f5715dce8..5f17ccd522 100644 --- a/src/ext/equix/hashx/src/hashx.c +++ b/src/ext/equix/hashx/src/hashx.c @@ -41,12 +41,24 @@ static int initialize_program(hashx_ctx* ctx, hashx_program* program,

int hashx_make(hashx_ctx* ctx, const void* seed, size_t size) { assert(ctx != NULL && ctx != HASHX_NOTSUPP); - assert(seed != NULL || size == 0); - siphash_state keys[2]; + assert(seed != NULL || size == 0); + + uint8_t keys_bytes[2 * sizeof(siphash_state)]; blake2b_state hash_state; hashx_blake2b_init_param(&hash_state, &hashx_blake2_params); hashx_blake2b_update(&hash_state, seed, size); - hashx_blake2b_final(&hash_state, &keys, sizeof(keys)); + hashx_blake2b_final(&hash_state, keys_bytes, sizeof(keys_bytes)); + + siphash_state keys[2]; + keys[0].v0 = load64(keys_bytes + 0 * sizeof(uint64_t)); + keys[0].v1 = load64(keys_bytes + 1 * sizeof(uint64_t)); + keys[0].v2 = load64(keys_bytes + 2 * sizeof(uint64_t)); + keys[0].v3 = load64(keys_bytes + 3 * sizeof(uint64_t)); + keys[1].v0 = load64(keys_bytes + 4 * sizeof(uint64_t)); + keys[1].v1 = load64(keys_bytes + 5 * sizeof(uint64_t)); + keys[1].v2 = load64(keys_bytes + 6 * sizeof(uint64_t)); + keys[1].v3 = load64(keys_bytes + 7 * sizeof(uint64_t)); + if (ctx->type & HASHX_COMPILED) { hashx_program program; if (!initialize_program(ctx, &program, keys)) { diff --git a/src/ext/equix/src/solver.h b/src/ext/equix/src/solver.h index ad69951929..4cf4105679 100644 --- a/src/ext/equix/src/solver.h +++ b/src/ext/equix/src/solver.h @@ -17,12 +17,26 @@ static inline bool tree_cmp1(const equix_idx* left, const equix_idx* right) { return *left <= *right; }

+static inline uint32_t tree_idx2(const equix_idx* idx) { + return + (uint32_t)idx[1] << 1*16 | + (uint32_t)idx[0] << 0*16; +} + static inline bool tree_cmp2(const equix_idx* left, const equix_idx* right) { - return load32(left) <= load32(right); + return tree_idx2(left) <= tree_idx2(right); +} + +static inline uint64_t tree_idx4(const equix_idx* idx) { + return + (uint64_t)idx[3] << 3*16 | + (uint64_t)idx[2] << 2*16 | + (uint64_t)idx[1] << 1*16 | + (uint64_t)idx[0] << 0*16; }

static inline bool tree_cmp4(const equix_idx* left, const equix_idx* right) { - return load64(left) <= load64(right); + return tree_idx4(left) <= tree_idx4(right); }

EQUIX_PRIVATE int equix_solver_solve(hashx_ctx* hash_func, solver_heap* heap, equix_solution output[EQUIX_MAX_SOLS]);

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit c6b168e141e9b2a80c80254a9cf3f2a5583fac8c Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Fri Mar 10 14:09:45 2023 -0800

test_hs_pow: add test vectors for our hs_pow client puzzle

This adds test vectors for the overall client puzzle at the hs_pow and hs_cell layers.

These are similar to the crypto/equix tests, but they also cover particulars of our hs_pow format like the conversion to byte arrays, the replay cache, the effort test, and the formatting of the equix challenge string.

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/test/include.am | 2 + src/test/test.c | 1 + src/test/test.h | 2 + src/test/test_hs_pow.c | 479 ++++++++++++++++++++++++++++++++++++++++++++ src/test/test_hs_pow_slow.c | 238 ++++++++++++++++++++++ src/test/test_slow.c | 1 + 6 files changed, 723 insertions(+)

diff --git a/src/test/include.am b/src/test/include.am index deff450490..4d4bfb8938 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -188,6 +188,7 @@ src_test_test_SOURCES += \ src/test/test_hs_descriptor.c \ src/test/test_hs_dos.c \ src/test/test_hs_metrics.c \ + src/test/test_hs_pow.c \ src/test/test_keypin.c \ src/test/test_link_handshake.c \ src/test/test_logging.c \ @@ -266,6 +267,7 @@ src_test_test_slow_SOURCES += \ src/test/test_slow.c \ src/test/test_crypto_slow.c \ src/test/test_process_slow.c \ + src/test/test_hs_pow_slow.c \ src/test/test_prob_distr.c \ src/test/ptr_helpers.c \ src/test/test_ptr_slow.c \ diff --git a/src/test/test.c b/src/test/test.c index 170dafe0c6..5ffb06e882 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -824,6 +824,7 @@ struct testgroup_t testgroups[] = { { "hs_metrics/", hs_metrics_tests }, { "hs_ntor/", hs_ntor_tests }, { "hs_ob/", hs_ob_tests }, + { "hs_pow/", hs_pow_tests }, { "hs_service/", hs_service_tests }, { "keypin/", keypin_tests }, { "link-handshake/", link_handshake_tests }, diff --git a/src/test/test.h b/src/test/test.h index d6c06c658f..ffac069e39 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -147,6 +147,7 @@ extern struct testcase_t hs_intropoint_tests[]; extern struct testcase_t hs_metrics_tests[]; extern struct testcase_t hs_ntor_tests[]; extern struct testcase_t hs_ob_tests[]; +extern struct testcase_t hs_pow_tests[]; extern struct testcase_t hs_service_tests[]; extern struct testcase_t keypin_tests[]; extern struct testcase_t link_handshake_tests[]; @@ -207,6 +208,7 @@ extern struct testcase_t voting_schedule_tests[]; extern struct testcase_t x509_tests[];

extern struct testcase_t slow_crypto_tests[]; +extern struct testcase_t slow_hs_pow_tests[]; extern struct testcase_t slow_process_tests[]; extern struct testcase_t slow_ptr_tests[];

diff --git a/src/test/test_hs_pow.c b/src/test/test_hs_pow.c new file mode 100644 index 0000000000..706ad2db05 --- /dev/null +++ b/src/test/test_hs_pow.c @@ -0,0 +1,479 @@ +/* Copyright (c) 2020-2023, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_hs_pow.c + * \brief Tests for service proof-of-work verification and wire protocol. + */ + +#define HS_SERVICE_PRIVATE +#define HS_CIRCUIT_PRIVATE + +#include "lib/cc/compat_compiler.h" +#include "lib/cc/torint.h" + +#include "test/hs_test_helpers.h" +#include "test/log_test_helpers.h" +#include "test/test_helpers.h" +#include "test/test.h" + +#include "app/config/config.h" +#include "core/or/circuitbuild.h" +#include "core/or/circuitlist.h" +#include "core/or/relay.h" +#include "feature/hs/hs_cell.h" +#include "feature/hs/hs_circuit.h" +#include "feature/hs/hs_metrics.h" +#include "feature/hs/hs_pow.h" +#include "feature/hs/hs_service.h" +#include "feature/nodelist/nodelist.h" + +#include "core/or/crypt_path_st.h" +#include "core/or/origin_circuit_st.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/routerinfo_st.h" + +#include "trunnel/hs/cell_introduce1.h" + +static int test_rend_launch_count; +static uint32_t test_rend_launch_expect_effort; + +static void +mock_launch_rendezvous_point_circuit(const hs_service_t *service, + const ed25519_public_key_t *ip_auth_pubkey, + const curve25519_keypair_t *ip_enc_key_kp, + const hs_cell_intro_rdv_data_t *rdv_data, + time_t now) +{ + (void) service; + (void) ip_auth_pubkey; + (void) ip_enc_key_kp; + (void) rdv_data; + (void) now; + + tt_int_op(test_rend_launch_expect_effort, OP_EQ, rdv_data->pow_effort); + test_rend_launch_count++; + +done: + ; +} + +static node_t *fake_node = NULL; + +static const node_t * +mock_build_state_get_exit_node(cpath_build_state_t *state) +{ + (void) state; + + if (!fake_node) { + curve25519_secret_key_t seckey; + curve25519_secret_key_generate(&seckey, 0); + + fake_node = tor_malloc_zero(sizeof(node_t)); + fake_node->ri = tor_malloc_zero(sizeof(routerinfo_t)); + fake_node->ri->onion_curve25519_pkey = + tor_malloc_zero(sizeof(curve25519_public_key_t)); + curve25519_public_key_generate(fake_node->ri->onion_curve25519_pkey, + &seckey); + } + + return fake_node; +} + +static smartlist_t * +mock_node_get_link_specifier_smartlist(const node_t *node, bool direct_conn) +{ + (void) node; + (void) direct_conn; + + smartlist_t *lspecs = smartlist_new(); + link_specifier_t *ls_legacy = link_specifier_new(); + smartlist_add(lspecs, ls_legacy); + + return lspecs; +} + +static size_t relay_payload_len; +static uint8_t relay_payload[RELAY_PAYLOAD_SIZE]; + +static int +mock_relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ, + uint8_t relay_command, const char *payload, + size_t payload_len, + crypt_path_t *cpath_layer, + const char *filename, int lineno) +{ + (void) stream_id; + (void) circ; + (void) relay_command; + (void) payload; + (void) payload_len; + (void) cpath_layer; + (void) filename; + (void) lineno; + + memcpy(relay_payload, payload, payload_len); + relay_payload_len = payload_len; + + return 0; +} + +typedef struct testing_hs_pow_service_t { + hs_service_t service; + hs_subcredential_t subcred; + hs_service_intro_point_t *service_ip; + hs_desc_intro_point_t *desc_ip; + hs_ntor_intro_cell_keys_t intro_keys; + origin_circuit_t *intro_circ; + origin_circuit_t *rend_circ; +} testing_hs_pow_service_t; + +/* Common test setup */ +static testing_hs_pow_service_t * +testing_hs_pow_service_new(void) +{ + MOCK(build_state_get_exit_node, mock_build_state_get_exit_node); + MOCK(relay_send_command_from_edge_, mock_relay_send_command_from_edge); + MOCK(launch_rendezvous_point_circuit, mock_launch_rendezvous_point_circuit); + MOCK(node_get_link_specifier_smartlist, + mock_node_get_link_specifier_smartlist); + + testing_hs_pow_service_t *tsvc = tor_malloc_zero(sizeof *tsvc); + hs_metrics_service_init(&tsvc->service); + + ed25519_keypair_t identity_keypair; + ed25519_keypair_generate(&identity_keypair, 0); + hs_helper_get_subcred_from_identity_keypair(&identity_keypair, + &tsvc->subcred); + + curve25519_secret_key_t seckey; + curve25519_public_key_t pkey; + curve25519_secret_key_generate(&seckey, 0); + curve25519_public_key_generate(&pkey, &seckey); + + node_t intro_node; + memset(&intro_node, 0, sizeof(intro_node)); + routerinfo_t ri; + memset(&ri, 0, sizeof(routerinfo_t)); + ri.onion_curve25519_pkey = &pkey; + intro_node.ri = &ri; + + hs_service_intro_point_t *svc_ip = service_intro_point_new(&intro_node); + const ed25519_public_key_t *ip_auth_pubkey = &svc_ip->auth_key_kp.pubkey; + const curve25519_public_key_t *ip_enc_pubkey = &svc_ip->enc_key_kp.pubkey; + tsvc->service_ip = svc_ip; + + ed25519_keypair_t signing_kp; + ed25519_keypair_generate(&signing_kp, 0); + tsvc->desc_ip = hs_helper_build_intro_point(&signing_kp, 0, "1.2.3.4", 0, + &svc_ip->auth_key_kp, + &svc_ip->enc_key_kp); + + tsvc->intro_circ = origin_circuit_new(); + tsvc->rend_circ = origin_circuit_new(); + + tsvc->intro_circ->cpath = tor_malloc_zero(sizeof(crypt_path_t)); + + struct hs_ident_circuit_t *hs_ident = tor_malloc_zero(sizeof *hs_ident); + tsvc->rend_circ->hs_ident = hs_ident; + tsvc->intro_circ->hs_ident = hs_ident; + curve25519_keypair_generate(&hs_ident->rendezvous_client_kp, 0); + tt_int_op(0, OP_EQ, + hs_ntor_client_get_introduce1_keys(ip_auth_pubkey, + ip_enc_pubkey, + &hs_ident->rendezvous_client_kp, + &tsvc->subcred, + &tsvc->intro_keys)); + done: + return tsvc; +} + +static void +testing_hs_pow_service_free(testing_hs_pow_service_t *tsvc) +{ + hs_metrics_service_free(&tsvc->service); + service_intro_point_free(tsvc->service_ip); + hs_desc_intro_point_free(tsvc->desc_ip); + tor_free(tsvc->service.state.pow_state); + tor_free(tsvc); + + if (fake_node) { + tor_free(fake_node->ri->onion_curve25519_pkey); + tor_free(fake_node->ri); + tor_free(fake_node); + } + + UNMOCK(build_state_get_exit_node); + UNMOCK(relay_send_command_from_edge_); + UNMOCK(launch_rendezvous_point_circuit); + UNMOCK(node_get_link_specifier_smartlist); +} + +/* Make sure we can send a PoW extension to a service without PoW enabled */ +static void +test_hs_pow_unsolicited(void *arg) +{ + (void)arg; + + testing_hs_pow_service_t *tsvc = testing_hs_pow_service_new(); + + /* Try this twice, changing only the presence or lack of PoW solution */ + for (int test_variant = 0; test_variant < 2; test_variant++) { + + relay_payload_len = 0; + test_rend_launch_count = 0; + test_rend_launch_expect_effort = 0; + memset(relay_payload, 0, sizeof relay_payload); + + hs_pow_solution_t solution = { 0 }; + int retval; + + retval = hs_circ_send_introduce1(tsvc->intro_circ, tsvc->rend_circ, + tsvc->desc_ip, &tsvc->subcred, + test_variant == 0 ? &solution : NULL); + + tt_int_op(retval, OP_EQ, 0); + tt_assert(!fast_mem_is_zero((const char*)relay_payload, + sizeof relay_payload)); + tt_int_op(relay_payload_len, OP_NE, 0); + + retval = hs_circ_handle_introduce2(&tsvc->service, tsvc->intro_circ, + tsvc->service_ip, &tsvc->subcred, + relay_payload, + relay_payload_len); + + tt_int_op(retval, OP_EQ, test_variant == 0 ? -1 : 0); + tt_int_op(test_rend_launch_count, OP_EQ, test_variant == 0 ? 0 : 1); + } + + done: + testing_hs_pow_service_free(tsvc); +} + +static void +test_hs_pow_vectors(void *arg) +{ + (void)arg; + + /* This covers encoding, wire protocol, and verification for PoW-extended + * introduction cells. The solutions here can be generated using the + * setup in test_hs_pow_slow. + */ + static const struct { + uint32_t claimed_effort; + uint32_t validated_effort; + int expected_retval; + const char *seed_hex; + const char *nonce_hex; + const char *sol_hex; + const char *encoded_hex; + } vectors[] = { + { + /* All zero, expect invalid */ + 1, 0, -1, + "0000000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000", "00000000000000000000000000000000", + "01" + "00000000000000000000000000000000" + "00000001" "00000000" + "00000000000000000000000000000000" + }, + { + /* Valid zero-effort solution */ + 0, 0, 0, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "55555555555555555555555555555555", "fd57d7676238c0ad1d5473aa2d0cbff5", + "01" + "55555555555555555555555555555555" + "00000000" "aaaaaaaa" + "fd57d7676238c0ad1d5473aa2d0cbff5" + }, + { + /* Valid high-effort solution */ + 1000000, 1000000, 0, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "16505855555555555555555555555555", "bf2c2d345e5773b5c32ec5596244bdbc", + "01" + "16505855555555555555555555555555" + "000f4240" "aaaaaaaa" + "bf2c2d345e5773b5c32ec5596244bdbc" + }, + { + /* Reject replays */ + 1000000, 0, -1, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "16505855555555555555555555555555", "bf2c2d345e5773b5c32ec5596244bdbc", + "01" + "16505855555555555555555555555555" + "000f4240" "aaaaaaaa" + "bf2c2d345e5773b5c32ec5596244bdbc" + }, + { + /* The claimed effort must exactly match what's in the challenge */ + 99999, 0, -1, + "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", + "cdd49fdbc34326d9d2f18ed277469c63", "7f153437c58620d3ea4717746093dde6", + "01" + "cdd49fdbc34326d9d2f18ed277469c63" + "0001869f" "cf0afb86" + "7f153437c58620d3ea4717746093dde6" + }, + { + /* Otherwise good solution but with a corrupted nonce */ + 100000, 0, -1, + "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", + "cdd49fdbc34326d9d2f18ed270469c63", "7f153437c58620d3ea4717746093dde6", + "01" + "cdd49fdbc34326d9d2f18ed270469c63" + "000186a0" "cf0afb86" + "7f153437c58620d3ea4717746093dde6" + }, + { + /* Corrected version of above */ + 100000, 100000, 0, + "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", + "cdd49fdbc34326d9d2f18ed277469c63", "7f153437c58620d3ea4717746093dde6", + "01" + "cdd49fdbc34326d9d2f18ed277469c63" + "000186a0" "cf0afb86" + "7f153437c58620d3ea4717746093dde6" + } + }; + + testing_hs_pow_service_t *tsvc = testing_hs_pow_service_new(); + hs_pow_service_state_t *pow_state = tor_malloc_zero(sizeof *pow_state); + tsvc->service.state.pow_state = pow_state; + + char *mem_op_hex_tmp = NULL; + uint8_t *decrypted = NULL; + trn_cell_introduce_encrypted_t *enc_cell = NULL; + trn_cell_introduce1_t *cell = NULL; + + const unsigned num_vectors = sizeof vectors / sizeof vectors[0]; + for (unsigned vec_i = 0; vec_i < num_vectors; vec_i++) { + const int expected_retval = vectors[vec_i].expected_retval; + const char *seed_hex = vectors[vec_i].seed_hex; + const char *nonce_hex = vectors[vec_i].nonce_hex; + const char *sol_hex = vectors[vec_i].sol_hex; + const char *encoded_hex = vectors[vec_i].encoded_hex; + + relay_payload_len = 0; + test_rend_launch_count = 0; + test_rend_launch_expect_effort = vectors[vec_i].validated_effort; + memset(relay_payload, 0, sizeof relay_payload); + + hs_pow_solution_t solution = { + .effort = vectors[vec_i].claimed_effort, + }; + int retval; + + tt_int_op(strlen(seed_hex), OP_EQ, 2 * HS_POW_SEED_LEN); + tt_int_op(strlen(nonce_hex), OP_EQ, 2 * sizeof solution.nonce); + tt_int_op(strlen(sol_hex), OP_EQ, 2 * sizeof solution.equix_solution); + + tt_int_op(base16_decode((char*)pow_state->seed_previous, HS_POW_SEED_LEN, + seed_hex, 2 * HS_POW_SEED_LEN), + OP_EQ, HS_POW_SEED_LEN); + tt_int_op(base16_decode((char*)&solution.nonce, sizeof solution.nonce, + nonce_hex, 2 * sizeof solution.nonce), + OP_EQ, HS_POW_NONCE_LEN); + tt_int_op(base16_decode((char*)&solution.equix_solution, + sizeof solution.equix_solution, + sol_hex, 2 * sizeof solution.equix_solution), + OP_EQ, HS_POW_EQX_SOL_LEN); + + memcpy(&solution.seed_head, pow_state->seed_previous, + sizeof solution.seed_head); + + /* Try to encode 'solution' into a relay cell */ + + retval = hs_circ_send_introduce1(tsvc->intro_circ, tsvc->rend_circ, + tsvc->desc_ip, &tsvc->subcred, + &solution); + + tt_int_op(retval, OP_EQ, 0); + tt_assert(!fast_mem_is_zero((const char*)relay_payload, + sizeof relay_payload)); + tt_int_op(relay_payload_len, OP_NE, 0); + + /* Check the service's response to this introduction */ + + retval = hs_circ_handle_introduce2(&tsvc->service, tsvc->intro_circ, + tsvc->service_ip, &tsvc->subcred, + relay_payload, + relay_payload_len); + tt_int_op(retval, OP_EQ, expected_retval); + tt_int_op(test_rend_launch_count, OP_EQ, expected_retval == 0 ? 1 : 0); + + /* Start unpacking the cell ourselves so we can check the PoW data */ + + trn_cell_introduce1_free(cell); + cell = NULL; + tt_int_op(trn_cell_introduce1_parse(&cell, relay_payload, + relay_payload_len), OP_GT, 0); + + size_t encrypted_section_len; + const uint8_t *encrypted_section; + encrypted_section = trn_cell_introduce1_getconstarray_encrypted(cell); + encrypted_section_len = trn_cell_introduce1_getlen_encrypted(cell); + tt_int_op(encrypted_section_len, OP_GT, + DIGEST256_LEN + CURVE25519_PUBKEY_LEN); + + /* Decrypt the encrypted portion of the INTRODUCE1 */ + + crypto_cipher_t *cipher = NULL; + cipher = crypto_cipher_new_with_bits((char *) tsvc->intro_keys.enc_key, + CURVE25519_PUBKEY_LEN * 8); + tt_ptr_op(cipher, OP_NE, NULL); + + size_t decrypted_len = encrypted_section_len + - DIGEST256_LEN - CURVE25519_PUBKEY_LEN; + tor_free(decrypted); + decrypted = tor_malloc_zero(decrypted_len); + retval = crypto_cipher_decrypt(cipher, (char *) decrypted, + (const char *) encrypted_section + + CURVE25519_PUBKEY_LEN, + decrypted_len); + crypto_cipher_free(cipher); + tt_int_op(retval, OP_EQ, 0); + + /* Parse the outer layer of the encrypted payload */ + + trn_cell_introduce_encrypted_free(enc_cell); + enc_cell = NULL; + tt_int_op(trn_cell_introduce_encrypted_parse(&enc_cell, decrypted, + decrypted_len), OP_GT, 0); + + /* Check for the expected single extension */ + + const trn_extension_t *extensions = + trn_cell_introduce_encrypted_get_extensions(enc_cell); + tt_int_op(trn_extension_get_num(extensions), OP_EQ, 1); + + const trn_extension_field_t *field = + trn_extension_getconst_fields(extensions, 0); + tt_int_op(trn_extension_field_get_field_type(field), + OP_EQ, TRUNNEL_EXT_TYPE_POW); + + const uint8_t *field_data = trn_extension_field_getconstarray_field(field); + size_t field_len = trn_extension_field_getlen_field(field); + + /* Our test vectors cover the packed data in the single extension */ + + tt_int_op(field_len * 2, OP_EQ, strlen(encoded_hex)); + test_memeq_hex(field_data, encoded_hex); + } + + done: + tor_free(mem_op_hex_tmp); + tor_free(decrypted); + trn_cell_introduce1_free(cell); + trn_cell_introduce_encrypted_free(enc_cell); + testing_hs_pow_service_free(tsvc); +} + +struct testcase_t hs_pow_tests[] = { + { "unsolicited", test_hs_pow_unsolicited, TT_FORK, NULL, NULL }, + { "vectors", test_hs_pow_vectors, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_hs_pow_slow.c b/src/test/test_hs_pow_slow.c new file mode 100644 index 0000000000..4d28765ba9 --- /dev/null +++ b/src/test/test_hs_pow_slow.c @@ -0,0 +1,238 @@ +/* Copyright (c) 2020-2023, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_hs_pow_slow.c + * \brief Slower (solve + verify) tests for service proof-of-work defenses. + */ + +#define HS_SERVICE_PRIVATE + +#include "lib/cc/compat_compiler.h" +#include "lib/cc/torint.h" + +#include "test/test.h" +#include "test/test_helpers.h" +#include "test/log_test_helpers.h" +#include "test/rng_test_helpers.h" + +#include "app/config/config.h" +#include "feature/hs/hs_pow.h" + +static int +testing_one_hs_pow_solution(const hs_pow_solution_t *ref_solution, + const uint8_t *seed) +{ + int retval = -1; + hs_pow_solution_t sol_buffer; + hs_pow_service_state_t *s = tor_malloc_zero(sizeof(hs_pow_service_state_t)); + uint32_t seed_head; + + memcpy(s->seed_previous, seed, HS_POW_SEED_LEN); + memcpy(&seed_head, seed, sizeof seed_head); + + const unsigned num_variants = 10; + const unsigned num_attempts = 3; + + for (unsigned variant = 0; variant < num_variants; variant++) { + hs_pow_remove_seed_from_cache(seed_head); + + for (unsigned attempt = 0; attempt < num_attempts; attempt++) { + int expected = -1; + memcpy(&sol_buffer, ref_solution, sizeof sol_buffer); + + /* One positive test, and a few negative tests of corrupted solutions */ + if (variant == 0) { + if (attempt == 0) { + /* Only the first attempt should succeed (nonce replay) */ + expected = 0; + } + } else if (variant & 1) { + sol_buffer.nonce += variant; + } else { + sol_buffer.equix_solution.idx[variant % EQUIX_NUM_IDX]++; + } + + tt_int_op(expected, OP_EQ, hs_pow_verify(s, &sol_buffer)); + } + } + + retval = 0; +done: + hs_pow_free_service_state(s); + return retval; +} + +static void +test_hs_pow_vectors(void *arg) +{ + (void)arg; + + /* All test vectors include a solve, verify, and fail-verify phase + * as well as a test of the nonce replay cache. The initial nonce for the + * solution search is set via the solver's RNG data. The amount of solve + * time during test execution can be tuned based on how far away from the + * winning nonce our solve_rng value is set. + */ + static const struct { + uint32_t effort; + const char *solve_rng_hex; + const char *seed_hex; + const char *nonce_hex; + const char *sol_hex; + } vectors[] = { + { + 0, "55555555555555555555555555555555", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "55555555555555555555555555555555", "fd57d7676238c0ad1d5473aa2d0cbff5" + }, + { + 1, "55555555555555555555555555555555", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "55555555555555555555555555555555", "703d8bc75492e8f90d836dd21bde61fc" + }, + { + 2, "55555555555555555555555555555555", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "56555555555555555555555555555555", "c2374478d35040b53e4eb9aa9f16e9ec" + }, + { + 10, "55555555555555555555555555555555", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "5c555555555555555555555555555555", "b167af85e25a0c961928eff53672c1f8" + }, + { + 10, "ffffffffffffffffffffffffffffffff", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "02000000000000000000000000000000", "954e4464715842d391712bb3b2289ff8" + }, + { + 1337, "7fffffffffffffffffffffffffffffff", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "eaffffffffffffffffffffffffffffff", "dbab3eb9045f85f8162c482d43f7d6fc" + }, + { + 31337, "00410000000000000000000000000000", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "23410000000000000000000000000000", "545ddd60e33bfa73ec75aada68608ee8" + }, + { + 100, "6b555555555555555555555555555555", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "6b555555555555555555555555555555", "7e14e98fed2f35a1b293b39d56b260e9" + }, + { + 1000, "0e565555555555555555555555555555", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "0e565555555555555555555555555555", "514963616e0b986afb1414afa88b85ff" + }, + { + 10000, "80835555555555555555555555555555", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "89835555555555555555555555555555", "7a5164905f8aaec152126258a2462ae6" + }, + { + 100000, "fd995655555555555555555555555555", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "fd995655555555555555555555555555", "8b27f2664340bc88dd5335821a68f5ff" + }, + { + 1000000, "15505855555555555555555555555555", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "16505855555555555555555555555555", "bf2c2d345e5773b5c32ec5596244bdbc" + }, + { + 1, "d0aec1669384bfe5ed39cd724d6c7954", + "c52be1f8a5e6cc3b8fb71cfdbe272cbc91d4d035400f2f94fb0d0074794e0a07", + "d0aec1669384bfe5ed39cd724d6c7954", "9e062190e23b34a80562818b14cf4ae5" + }, + { + 1, "b4d0e611e6935750fcf9406aae131f62", + "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", + "b4d0e611e6935750fcf9406aae131f62", "a01cf4457a016488df4fa45f0864b6fb" + }, + { + 1, "b4d0e611e6935750fcf9406aae131f62", + "9dfbd06d86fed8e12de3ab214e1a63ea61f46253fe08346a20378da70c4a327d", + "b5d0e611e6935750fcf9406aae131f62", "5944a260423392780f10b25b7e2502d3" + }, + { + 1, "40559fdbc34326d9d2f18ed277469c63", + "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", + "40559fdbc34326d9d2f18ed277469c63", "31139564ca5262a4f82b9385b2832fce" + }, + { + 10000, "70559fdbc34326d9d2f18ed277469c63", + "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", + "72559fdbc34326d9d2f18ed277469c63", "262c6c82025c53b69b0bf255606ca3e2" + }, + { + 100000, "c0d49fdbc34326d9d2f18ed277469c63", + "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", + "cdd49fdbc34326d9d2f18ed277469c63", "7f153437c58620d3ea4717746093dde6" + }, + { + 1000000, "40fdb1dbc34326d9d2f18ed277469c63", + "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", + "4cfdb1dbc34326d9d2f18ed277469c63", "b31bbb45340e17a14c2156c0b66780e7" + }, + }; + + const unsigned num_vectors = sizeof vectors / sizeof vectors[0]; + for (unsigned vec_i = 0; vec_i < num_vectors; vec_i++) { + const char *seed_hex = vectors[vec_i].seed_hex; + const char *solve_rng_hex = vectors[vec_i].solve_rng_hex; + const char *nonce_hex = vectors[vec_i].nonce_hex; + const char *sol_hex = vectors[vec_i].sol_hex; + + uint8_t rng_bytes[HS_POW_NONCE_LEN]; + hs_pow_solution_t output; + hs_pow_solution_t solution = { 0 }; + hs_pow_desc_params_t params = { + .type = HS_POW_DESC_V1, + .suggested_effort = vectors[vec_i].effort, + }; + + tt_int_op(strlen(seed_hex), OP_EQ, 2 * sizeof params.seed); + tt_int_op(strlen(solve_rng_hex), OP_EQ, 2 * sizeof rng_bytes); + tt_int_op(strlen(nonce_hex), OP_EQ, 2 * sizeof solution.nonce); + tt_int_op(strlen(sol_hex), OP_EQ, 2 * sizeof solution.equix_solution); + + tt_int_op(base16_decode((char*)params.seed, HS_POW_SEED_LEN, + seed_hex, 2 * HS_POW_SEED_LEN), + OP_EQ, HS_POW_SEED_LEN); + tt_int_op(base16_decode((char*)rng_bytes, sizeof rng_bytes, + solve_rng_hex, 2 * sizeof rng_bytes), + OP_EQ, HS_POW_NONCE_LEN); + tt_int_op(base16_decode((char*)&solution.nonce, sizeof solution.nonce, + nonce_hex, 2 * sizeof solution.nonce), + OP_EQ, HS_POW_NONCE_LEN); + tt_int_op(base16_decode((char*)&solution.equix_solution, + sizeof solution.equix_solution, + sol_hex, 2 * sizeof solution.equix_solution), + OP_EQ, HS_POW_EQX_SOL_LEN); + memcpy(&solution.seed_head, params.seed, sizeof solution.seed_head); + + memset(&output, 0xaa, sizeof output); + testing_enable_prefilled_rng(rng_bytes, HS_POW_NONCE_LEN); + tt_int_op(0, OP_EQ, hs_pow_solve(&params, &output)); + testing_disable_prefilled_rng(); + + tt_mem_op(params.seed, OP_EQ, &output.seed_head, + sizeof output.seed_head); + tt_mem_op(&solution.nonce, OP_EQ, &output.nonce, + sizeof output.nonce); + tt_mem_op(&solution.equix_solution, OP_EQ, &output.equix_solution, + sizeof output.equix_solution); + + tt_int_op(testing_one_hs_pow_solution(&output, params.seed), OP_EQ, 0); + } + + done: + testing_disable_prefilled_rng(); +} + +struct testcase_t slow_hs_pow_tests[] = { + { "vectors", test_hs_pow_vectors, 0, NULL, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_slow.c b/src/test/test_slow.c index 5f42b43103..08366416ca 100644 --- a/src/test/test_slow.c +++ b/src/test/test_slow.c @@ -21,6 +21,7 @@ struct testgroup_t testgroups[] = { { "slow/crypto/", slow_crypto_tests }, { "slow/process/", slow_process_tests }, + { "slow/hs_pow/", slow_hs_pow_tests }, { "slow/prob_distr/", slow_stochastic_prob_distr_tests }, { "slow/ptr/", slow_ptr_tests }, END_OF_GROUPS

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 037dea2252e5122059bd189c57ad606f05c7b0b7 Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Tue Mar 14 09:01:14 2023 -0700

hs_pow: fix assert in services that receive unsolicited proof of work

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/feature/hs/hs_cell.c | 5 +++++ 1 file changed, 5 insertions(+)

diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c index cadafd2c90..190f0f3b5a 100644 --- a/src/feature/hs/hs_cell.c +++ b/src/feature/hs/hs_cell.c @@ -807,6 +807,11 @@ handle_introduce2_encrypted_cell_pow_extension(const hs_service_t *service,

tor_assert(field);

+ if (!service->state.pow_state) { + log_info(LD_REND, "Unsolicited PoW solution in INTRODUCE2 request."); + goto end; + } + if (trn_cell_extension_pow_parse(&pow, trn_extension_field_getconstarray_field(field), trn_extension_field_getlen_field(field)) < 0) {

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 209a59face791159e167837214e22b6eaa3375b4 Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Tue Mar 14 16:16:27 2023 -0700

hs_pow: Don't require uint128_t

We were using a native uint128_t to represent the hs_pow nonce, but as the comments note it's more portable and more flexible to use a byte array. Indeed the uint128_t was a problem for 32-bit platforms. This swaps in a new implementation that uses multiple machine words to implement the nonce incrementation.

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/feature/hs/hs_pow.c | 34 +++++++++++++++++++--------------- src/feature/hs/hs_pow.h | 6 +----- src/test/test_hs_pow_slow.c | 2 +- 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/feature/hs/hs_pow.c b/src/feature/hs/hs_pow.c index 1fc9de1268..e41f271fec 100644 --- a/src/feature/hs/hs_pow.c +++ b/src/feature/hs/hs_pow.c @@ -7,8 +7,6 @@ * when a hidden service is defending against DoS attacks. **/

-typedef unsigned __int128 uint128_t; - #include <stdio.h>

#include "ext/ht.h" @@ -20,6 +18,7 @@ typedef unsigned __int128 uint128_t; #include "feature/hs/hs_client.h" #include "feature/hs/hs_pow.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/cc/ctassert.h" #include "core/mainloop/cpuworker.h" #include "lib/evloop/workqueue.h"

@@ -27,7 +26,7 @@ typedef unsigned __int128 uint128_t; /** Cache entry for (nonce, seed) replay protection. */ typedef struct nonce_cache_entry_t { HT_ENTRY(nonce_cache_entry_t) node; - uint128_t nonce; + uint8_t nonce[HS_POW_NONCE_LEN]; uint32_t seed_head; } nonce_cache_entry_t;

@@ -36,7 +35,7 @@ static inline int nonce_cache_entries_eq_(const struct nonce_cache_entry_t *entry1, const struct nonce_cache_entry_t *entry2) { - return entry1->nonce == entry2->nonce && + return fast_memeq(entry1->nonce, entry2->nonce, HS_POW_NONCE_LEN) && entry1->seed_head == entry2->seed_head; }

@@ -44,7 +43,7 @@ nonce_cache_entries_eq_(const struct nonce_cache_entry_t *entry1, static inline unsigned nonce_cache_entry_hash_(const struct nonce_cache_entry_t *ent) { - return (unsigned)siphash24g(&ent->nonce, HS_POW_NONCE_LEN) + ent->seed_head; + return (unsigned)siphash24g(ent->nonce, HS_POW_NONCE_LEN) + ent->seed_head; }

static HT_HEAD(nonce_cache_table_ht, nonce_cache_entry_t) @@ -69,9 +68,14 @@ nonce_cache_entry_has_seed(nonce_cache_entry_t *ent, void *data) /** Helper: Increment a given nonce and set it in the challenge at the right * offset. Use by the solve function. */ static inline void -increment_and_set_nonce(uint128_t *nonce, uint8_t *challenge) +increment_and_set_nonce(uint8_t *nonce, uint8_t *challenge) { - (*nonce)++; + for (unsigned i = 0; i < HS_POW_NONCE_LEN; i++) { + uint8_t prev = nonce[i]; + if (++nonce[i] > prev) { + break; + } + } memcpy(challenge + HS_POW_SEED_LEN, nonce, HS_POW_NONCE_LEN); }

@@ -95,7 +99,7 @@ build_equix_ctx(equix_ctx_flags flags) /* Helper: Build EquiX challenge (C || N || INT_32(E)) and return a newly * allocated buffer containing it. */ static uint8_t * -build_equix_challenge(const uint8_t *seed, const uint128_t nonce, +build_equix_challenge(const uint8_t *seed, const uint8_t *nonce, const uint32_t effort) { /* Build EquiX challenge (C || N || INT_32(E)). */ @@ -104,7 +108,7 @@ build_equix_challenge(const uint8_t *seed, const uint128_t nonce,

memcpy(challenge, seed, HS_POW_SEED_LEN); offset += HS_POW_SEED_LEN; - memcpy(challenge + offset, &nonce, HS_POW_NONCE_LEN); + memcpy(challenge + offset, nonce, HS_POW_NONCE_LEN); offset += HS_POW_NONCE_LEN; set_uint32(challenge + offset, tor_htonl(effort)); offset += HS_POW_EFFORT_LEN; @@ -146,7 +150,7 @@ hs_pow_solve(const hs_pow_desc_params_t *pow_params, hs_pow_solution_t *pow_solution_out) { int ret = -1; - uint128_t nonce; + uint8_t nonce[HS_POW_NONCE_LEN]; uint8_t *challenge = NULL; equix_ctx *ctx = NULL;

@@ -157,7 +161,7 @@ hs_pow_solve(const hs_pow_desc_params_t *pow_params, uint32_t effort = pow_params->suggested_effort;

/* Generate a random nonce N. */ - crypto_rand((char *)&nonce, sizeof(uint128_t)); + crypto_rand((char *)nonce, sizeof nonce);

/* Build EquiX challenge (C || N || INT_32(E)). */ challenge = build_equix_challenge(pow_params->seed, nonce, effort); @@ -178,7 +182,7 @@ hs_pow_solve(const hs_pow_desc_params_t *pow_params, /* Check an Equi-X solution against the effort threshold */ if (validate_equix_challenge(challenge, sol, effort)) { /* Store the nonce N. */ - pow_solution_out->nonce = nonce; + memcpy(pow_solution_out->nonce, nonce, HS_POW_NONCE_LEN); /* Store the effort E. */ pow_solution_out->effort = effort; /* We only store the first 4 bytes of the seed C. */ @@ -195,7 +199,7 @@ hs_pow_solve(const hs_pow_desc_params_t *pow_params,

/* No solutions for this nonce and/or none that passed the effort * threshold, increment and try again. */ - increment_and_set_nonce(&nonce, challenge); + increment_and_set_nonce(nonce, challenge); }

end: @@ -243,7 +247,7 @@ hs_pow_verify(const hs_pow_service_state_t *pow_state, }

/* Fail if N = POW_NONCE is present in the replay cache. */ - search.nonce = pow_solution->nonce; + memcpy(search.nonce, pow_solution->nonce, HS_POW_NONCE_LEN); search.seed_head = pow_solution->seed_head; entry = HT_FIND(nonce_cache_table_ht, &nonce_cache_table, &search); if (entry) { @@ -279,7 +283,7 @@ hs_pow_verify(const hs_pow_service_state_t *pow_state,

/* Add the (nonce, seed) tuple to the replay cache. */ entry = tor_malloc_zero(sizeof(nonce_cache_entry_t)); - entry->nonce = pow_solution->nonce; + memcpy(entry->nonce, pow_solution->nonce, HS_POW_NONCE_LEN); entry->seed_head = pow_solution->seed_head; HT_INSERT(nonce_cache_table_ht, &nonce_cache_table, entry);

diff --git a/src/feature/hs/hs_pow.h b/src/feature/hs/hs_pow.h index b27bd7441c..6eb03ef64e 100644 --- a/src/feature/hs/hs_pow.h +++ b/src/feature/hs/hs_pow.h @@ -10,8 +10,6 @@ #ifndef TOR_HS_POW_H #define TOR_HS_POW_H

-typedef unsigned __int128 uint128_t; - #include "ext/equix/include/equix.h"

#include "lib/evloop/compat_libevent.h" @@ -112,10 +110,8 @@ typedef struct hs_pow_service_state_t {

/* Struct to store a solution to the PoW challenge. */ typedef struct hs_pow_solution_t { - /** HRPR TODO are we best off storing this as a byte array, as trunnel doesnt - * support uint128 (?) */ /* The 16 byte nonce used in the solution. */ - uint128_t nonce; + uint8_t nonce[HS_POW_NONCE_LEN];

/* The effort used in the solution. */ uint32_t effort; diff --git a/src/test/test_hs_pow_slow.c b/src/test/test_hs_pow_slow.c index 8ccbf8025c..716501dffd 100644 --- a/src/test/test_hs_pow_slow.c +++ b/src/test/test_hs_pow_slow.c @@ -48,7 +48,7 @@ testing_one_hs_pow_solution(const hs_pow_solution_t *ref_solution, expected = 0; } } else if (variant & 1) { - sol_buffer.nonce += variant; + sol_buffer.nonce[variant % HS_POW_NONCE_LEN]++; } else { sol_buffer.equix_solution.idx[variant % EQUIX_NUM_IDX]++; }

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 700814a3a117652682ccdf1ea591584b5eca1ff6 Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Wed Mar 15 10:41:22 2023 -0700

hs_pow: Fix nonce cache entry leak

This leak was showing up in address sanitizer runs of test_hs_pow, but it will also happen during normal operation as seeds are rotated.

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/feature/hs/hs_pow.c | 19 ++++++++++++------- src/test/test_hs_pow.c | 1 + src/test/test_hs_pow_slow.c | 1 + 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/feature/hs/hs_pow.c b/src/feature/hs/hs_pow.c index 77fec689e4..fef809e369 100644 --- a/src/feature/hs/hs_pow.c +++ b/src/feature/hs/hs_pow.c @@ -57,13 +57,18 @@ HT_GENERATE2(nonce_cache_table_ht, nonce_cache_entry_t, node, nonce_cache_entry_hash_, nonce_cache_entries_eq_, 0.6, tor_reallocarray_, tor_free_);

-/** We use this to check if an entry in the replay cache is for a particular - * seed head, so we know to remove it once the seed is no longer in use. */ +/** This is a callback used to check replay cache entries against a provided + * seed head, or NULL to operate on the entire cache. Matching entries return + * 1 and their internal cache entry is freed, non-matching entries return 0. */ static int -nonce_cache_entry_has_seed(nonce_cache_entry_t *ent, void *data) +nonce_cache_entry_match_seed_and_free(nonce_cache_entry_t *ent, void *data) { - /* Returning nonzero makes HT_FOREACH_FN remove the element from the HT */ - return fast_memeq(ent->bytes.seed_head, data, HS_POW_SEED_HEAD_LEN); + if (data == NULL || + fast_memeq(ent->bytes.seed_head, data, HS_POW_SEED_HEAD_LEN)) { + tor_free(ent); + return 1; + } + return 0; }

/** Helper: Increment a given nonce and set it in the challenge at the right @@ -298,13 +303,13 @@ hs_pow_verify(const hs_pow_service_state_t *pow_state, }

/** Remove entries from the (nonce, seed) replay cache which are for the seed - * beginning with seed_head. */ + * beginning with seed_head. If seed_head is NULL, remove all cache entries. */ void hs_pow_remove_seed_from_cache(const uint8_t *seed_head) { /* If nonce_cache_entry_has_seed returns 1, the entry is removed. */ HT_FOREACH_FN(nonce_cache_table_ht, &nonce_cache_table, - nonce_cache_entry_has_seed, (void*)seed_head); + nonce_cache_entry_match_seed_and_free, (void*)seed_head); }

/** Free a given PoW service state. */ diff --git a/src/test/test_hs_pow.c b/src/test/test_hs_pow.c index 877f022a79..edda3cd643 100644 --- a/src/test/test_hs_pow.c +++ b/src/test/test_hs_pow.c @@ -469,6 +469,7 @@ test_hs_pow_vectors(void *arg) trn_cell_introduce1_free(cell); trn_cell_introduce_encrypted_free(enc_cell); testing_hs_pow_service_free(tsvc); + hs_pow_remove_seed_from_cache(NULL); }

struct testcase_t hs_pow_tests[] = { diff --git a/src/test/test_hs_pow_slow.c b/src/test/test_hs_pow_slow.c index 0c83922646..13d7111e88 100644 --- a/src/test/test_hs_pow_slow.c +++ b/src/test/test_hs_pow_slow.c @@ -228,6 +228,7 @@ test_hs_pow_vectors(void *arg)

done: testing_disable_prefilled_rng(); + hs_pow_remove_seed_from_cache(NULL); }

struct testcase_t slow_hs_pow_tests[] = {

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 2de98a7f4ea86c234990747fb566e70fef6aee15 Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Wed Mar 15 12:31:50 2023 -0700

hs_pow: Represent equix_solution as a byte array

This patch is intended to clarify the points at which we convert between the internal representation of an equix_solution and a portable but opaque byte array representation.

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/feature/hs/hs_cell.c | 8 ++++---- src/feature/hs/hs_pow.c | 43 +++++++++++++++++++++++++++++++++++-------- src/feature/hs/hs_pow.h | 6 ++---- src/test/test_hs_pow.c | 9 ++++----- src/test/test_hs_pow_slow.c | 4 ++-- 5 files changed, 47 insertions(+), 23 deletions(-)

diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c index dce360fd91..d3639cb92f 100644 --- a/src/feature/hs/hs_cell.c +++ b/src/feature/hs/hs_cell.c @@ -409,7 +409,7 @@ build_introduce_pow_extension(const hs_pow_solution_t *pow_solution, memcpy(trn_cell_extension_pow_getarray_pow_seed(pow_ext), pow_solution->seed_head, TRUNNEL_POW_SEED_HEAD_LEN); memcpy(trn_cell_extension_pow_getarray_pow_solution(pow_ext), - &pow_solution->equix_solution, TRUNNEL_POW_SOLUTION_LEN); + pow_solution->equix_solution, TRUNNEL_POW_SOLUTION_LEN);

/* Set the field with the encoded PoW extension. */ ret = trn_cell_extension_pow_encoded_len(pow_ext); @@ -830,13 +830,13 @@ handle_introduce2_encrypted_cell_pow_extension(const hs_service_t *service, /* Effort E */ sol.effort = trn_cell_extension_pow_get_pow_effort(pow); /* Seed C */ - memcpy(&sol.seed_head, trn_cell_extension_pow_getconstarray_pow_seed(pow), + memcpy(sol.seed_head, trn_cell_extension_pow_getconstarray_pow_seed(pow), HS_POW_SEED_HEAD_LEN); /* Nonce N */ - memcpy(&sol.nonce, trn_cell_extension_pow_getconstarray_pow_nonce(pow), + memcpy(sol.nonce, trn_cell_extension_pow_getconstarray_pow_nonce(pow), HS_POW_NONCE_LEN); /* Solution S */ - memcpy(&sol.equix_solution, + memcpy(sol.equix_solution, trn_cell_extension_pow_getconstarray_pow_solution(pow), HS_POW_EQX_SOL_LEN);

diff --git a/src/feature/hs/hs_pow.c b/src/feature/hs/hs_pow.c index fef809e369..b5a359940c 100644 --- a/src/feature/hs/hs_pow.c +++ b/src/feature/hs/hs_pow.c @@ -13,11 +13,13 @@ #include "ext/compat_blake2.h" #include "core/or/circuitlist.h" #include "core/or/origin_circuit_st.h" +#include "ext/equix/include/equix.h" #include "feature/hs/hs_cache.h" #include "feature/hs/hs_descriptor.h" #include "feature/hs/hs_client.h" #include "feature/hs/hs_pow.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/arch/bytes.h" #include "lib/cc/ctassert.h" #include "core/mainloop/cpuworker.h" #include "lib/evloop/workqueue.h" @@ -126,7 +128,8 @@ build_equix_challenge(const uint8_t *seed, const uint8_t *nonce, /** Helper: Return true iff the given challenge and solution for the given * effort do validate as in: R * E <= UINT32_MAX. */ static bool -validate_equix_challenge(const uint8_t *challenge, const equix_solution *sol, +validate_equix_challenge(const uint8_t *challenge, + const uint8_t *solution_bytes, const uint32_t effort) { /* Fail if R * E > UINT32_MAX. */ @@ -139,7 +142,7 @@ validate_equix_challenge(const uint8_t *challenge, const equix_solution *sol,

/* Construct: blake2b(C || N || E || S) */ blake2b_update(&b2_state, challenge, HS_POW_CHALLENGE_LEN); - blake2b_update(&b2_state, (const uint8_t *) sol, HS_POW_EQX_SOL_LEN); + blake2b_update(&b2_state, solution_bytes, HS_POW_EQX_SOL_LEN); blake2b_final(&b2_state, hash_result, HS_POW_HASH_LEN);

/* Scale to 64 bit so we can avoid 32 bit overflow. */ @@ -148,6 +151,28 @@ validate_equix_challenge(const uint8_t *challenge, const equix_solution *sol, return RE <= UINT32_MAX; }

+/** Helper: Convert equix_solution to a byte array in little-endian order */ +static void +pack_equix_solution(const equix_solution *sol_in, + uint8_t *bytes_out) +{ + for (unsigned i = 0; i < EQUIX_NUM_IDX; i++) { + bytes_out[i*2+0] = (uint8_t)sol_in->idx[i]; + bytes_out[i*2+1] = (uint8_t)(sol_in->idx[i] >> 8); + } +} + +/** Helper: Build an equix_solution from its corresponding byte array. */ +static void +unpack_equix_solution(const uint8_t *bytes_in, + equix_solution *sol_out) +{ + for (unsigned i = 0; i < EQUIX_NUM_IDX; i++) { + sol_out->idx[i] = (uint16_t)bytes_in[i*2+0] | + (uint16_t)bytes_in[i*2+1] << 8; + } +} + /** Solve the EquiX/blake2b PoW scheme using the parameters in pow_params, and * store the solution in pow_solution_out. Returns 0 on success and -1 * otherwise. Called by a client. */ @@ -177,16 +202,17 @@ hs_pow_solve(const hs_pow_desc_params_t *pow_params, goto end; } equix_solution solutions[EQUIX_MAX_SOLS]; + uint8_t sol_bytes[HS_POW_EQX_SOL_LEN];

log_notice(LD_REND, "Solving proof of work (effort %u)", effort); for (;;) { /* Calculate solutions to S = equix_solve(C || N || E), */ int count = equix_solve(ctx, challenge, HS_POW_CHALLENGE_LEN, solutions); for (int i = 0; i < count; i++) { - const equix_solution *sol = &solutions[i]; + pack_equix_solution(&solutions[i], sol_bytes);

/* Check an Equi-X solution against the effort threshold */ - if (validate_equix_challenge(challenge, sol, effort)) { + if (validate_equix_challenge(challenge, sol_bytes, effort)) { /* Store the nonce N. */ memcpy(pow_solution_out->nonce, nonce, HS_POW_NONCE_LEN); /* Store the effort E. */ @@ -195,8 +221,7 @@ hs_pow_solve(const hs_pow_desc_params_t *pow_params, memcpy(pow_solution_out->seed_head, pow_params->seed, sizeof(pow_solution_out->seed_head)); /* Store the solution S */ - memcpy(&pow_solution_out->equix_solution, sol, - sizeof(pow_solution_out->equix_solution)); + memcpy(&pow_solution_out->equix_solution, sol_bytes, sizeof sol_bytes);

/* Indicate success and we are done. */ ret = 0; @@ -267,7 +292,7 @@ hs_pow_verify(const hs_pow_service_state_t *pow_state, challenge = build_equix_challenge(seed, pow_solution->nonce, pow_solution->effort);

- if (!validate_equix_challenge(challenge, &pow_solution->equix_solution, + if (!validate_equix_challenge(challenge, pow_solution->equix_solution, pow_solution->effort)) { log_warn(LD_REND, "Equi-X solution and effort was too large."); goto done; @@ -279,8 +304,10 @@ hs_pow_verify(const hs_pow_service_state_t *pow_state, }

/* Fail if equix_verify(C || N || E, S) != EQUIX_OK */ + equix_solution equix_sol; + unpack_equix_solution(pow_solution->equix_solution, &equix_sol); equix_result result = equix_verify(ctx, challenge, HS_POW_CHALLENGE_LEN, - &pow_solution->equix_solution); + &equix_sol); if (result != EQUIX_OK) { log_warn(LD_REND, "Verification of EquiX solution in PoW failed."); goto done; diff --git a/src/feature/hs/hs_pow.h b/src/feature/hs/hs_pow.h index 3d7018ab30..de4d9df94c 100644 --- a/src/feature/hs/hs_pow.h +++ b/src/feature/hs/hs_pow.h @@ -10,8 +10,6 @@ #ifndef TOR_HS_POW_H #define TOR_HS_POW_H

-#include "ext/equix/include/equix.h" - #include "lib/evloop/compat_libevent.h" #include "lib/evloop/token_bucket.h" #include "lib/smartlist_core/smartlist_core.h" @@ -121,8 +119,8 @@ typedef struct hs_pow_solution_t { /* A prefix of the seed used in this solution, so it can be identified. */ uint8_t seed_head[HS_POW_SEED_HEAD_LEN];

- /* The Equi-X solution used in the solution. */ - equix_solution equix_solution; + /* The Equi-X solution used in this PoW solution. */ + uint8_t equix_solution[HS_POW_EQX_SOL_LEN]; } hs_pow_solution_t;

#ifdef HAVE_MODULE_POW diff --git a/src/test/test_hs_pow.c b/src/test/test_hs_pow.c index edda3cd643..ca470e2b5d 100644 --- a/src/test/test_hs_pow.c +++ b/src/test/test_hs_pow.c @@ -374,12 +374,11 @@ test_hs_pow_vectors(void *arg) tt_int_op(base16_decode((char*)pow_state->seed_previous, HS_POW_SEED_LEN, seed_hex, 2 * HS_POW_SEED_LEN), OP_EQ, HS_POW_SEED_LEN); - tt_int_op(base16_decode((char*)&solution.nonce, sizeof solution.nonce, - nonce_hex, 2 * sizeof solution.nonce), + tt_int_op(base16_decode((char*)solution.nonce, HS_POW_NONCE_LEN, + nonce_hex, 2 * HS_POW_NONCE_LEN), OP_EQ, HS_POW_NONCE_LEN); - tt_int_op(base16_decode((char*)&solution.equix_solution, - sizeof solution.equix_solution, - sol_hex, 2 * sizeof solution.equix_solution), + tt_int_op(base16_decode((char*)solution.equix_solution, HS_POW_EQX_SOL_LEN, + sol_hex, 2 * HS_POW_EQX_SOL_LEN), OP_EQ, HS_POW_EQX_SOL_LEN);

memcpy(solution.seed_head, pow_state->seed_previous, HS_POW_SEED_HEAD_LEN); diff --git a/src/test/test_hs_pow_slow.c b/src/test/test_hs_pow_slow.c index 13d7111e88..690ce0cf78 100644 --- a/src/test/test_hs_pow_slow.c +++ b/src/test/test_hs_pow_slow.c @@ -46,9 +46,9 @@ testing_one_hs_pow_solution(const hs_pow_solution_t *ref_solution, expected = 0; } } else if (variant & 1) { - sol_buffer.nonce[variant % HS_POW_NONCE_LEN]++; + sol_buffer.nonce[variant / 2 % HS_POW_NONCE_LEN]++; } else { - sol_buffer.equix_solution.idx[variant % EQUIX_NUM_IDX]++; + sol_buffer.equix_solution[variant / 2 % HS_POW_EQX_SOL_LEN]++; }

tt_int_op(expected, OP_EQ, hs_pow_verify(s, &sol_buffer));

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit d15bbf32dafc3ce1ae2265668e476e7f5dbf5b08 Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Thu Mar 16 08:44:15 2023 -0700

changes: Ticket 40634 (hs_pow)

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- changes/ticket40634 | 3 +++ 1 file changed, 3 insertions(+)

diff --git a/changes/ticket40634 b/changes/ticket40634 new file mode 100644 index 0000000000..42baa6509a --- /dev/null +++ b/changes/ticket40634 @@ -0,0 +1,3 @@ + o Major features (onion services): + - Proof-of-work client puzzles for DoS mitigation, from proposal 327. + Closes ticket 40634. \ No newline at end of file

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit ac29c7209dbdaf0d317f1c4eb67d5e330386552b Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Tue Mar 21 12:56:10 2023 -0700

hs_pow: bump client-side effort limit from 500 to 10000

500 was quite low, but this limit was helpful when the suggested-effort estimation algorithm was likely to give us large abrupt increases. Now that this should be fixed, let's allow spending a bit more time on the client puzzles if it's actually necessary.

Solving a puzzle with effort=10000 usually completes within a minute on my old x86_64 machine. We may want to fine tune this further, and it should probably be made into a config option.

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/feature/hs/hs_client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index 6a404395ea..2a620da953 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -655,7 +655,7 @@ send_introduce1(origin_circuit_t *intro_circ,

/** Set a client-side cap on the highest effort of PoW we will try to * tackle. If asked for higher, we solve it at this cap. */ -#define CLIENT_MAX_POW_EFFORT 500 +#define CLIENT_MAX_POW_EFFORT 10000

/** Send an INTRODUCE1 cell along the intro circuit and populate the rend * circuit identifier with the needed key material for the e2e encryption.

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 903c6cf1ab5ba115bbfd33385e0acd5f88ad2ba3 Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Fri Mar 31 13:36:58 2023 -0700

hs_pow: client side effort adjustment

The goal of this patch is to add an additional mechanism for adjusting PoW effort upwards, where clients rather than services can choose to solve their puzzles at a higher effort than what was suggested in the descriptor.

I wanted to use hs_cache's existing unreachability stats to drive this effort bump, but this revealed some cases where a circuit (intro or rend) closed early on can end up in hs_cache with an all zero intro point key, where nobody will find it. This moves intro_auth_pk initialization earlier in a couple places and adds nonfatal asserts to catch the problem if it shows up elsewhere.

The actual effort adjustment method I chose is to multiply the suggested effort by (1 + unresponsive_count), then ensure the result is at least 1. If a service has suggested effort of 0 but we fail to connect, retries will all use an effort of 1. If the suggestion was 50, we'll try 50, 100, 150, 200, etc. This is bounded both by our client effort limit and by the limit on unresponsive_count (currently 5).

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/core/or/circuituse.c | 23 +++++--- src/feature/hs/hs_cache.c | 2 + src/feature/hs/hs_circuit.c | 1 + src/feature/hs/hs_client.c | 130 ++++++++++++++++++++++++++++---------------- src/feature/hs/hs_client.h | 2 + src/feature/hs/hs_pow.c | 27 ++++----- src/feature/hs/hs_pow.h | 23 ++++++-- src/test/test_hs_client.c | 2 + src/test/test_hs_pow_slow.c | 15 +++-- 9 files changed, 142 insertions(+), 83 deletions(-)

diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index b78f72e835..ac9005e1d4 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -564,14 +564,6 @@ circuit_expire_building(void) continue; }

- /* Ignore circuits that are waiting for an introduction to a service with - * PoW enabled, it can take an arbitrary amount of time. They will get - * cleaned up if the SOCKS connection is closed. */ - if (TO_ORIGIN_CIRCUIT(victim)->hs_with_pow_circ && - victim->purpose == CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) { - continue; - } - build_state = TO_ORIGIN_CIRCUIT(victim)->build_state; if (build_state && build_state->onehop_tunnel) cutoff = begindir_cutoff; @@ -2560,6 +2552,11 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, circ->hs_ident = hs_ident_circuit_new(&edge_conn->hs_ident->identity_pk); } + if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) { + if (hs_client_setup_intro_circ_auth_key(circ) < 0) { + return 0; + } + } if (circ->base_.purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND && circ->base_.state == CIRCUIT_STATE_OPEN) circuit_has_opened(circ); @@ -3012,6 +3009,16 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn) conn, CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT, &introcirc); if (retval < 0) return -1; /* failed */

+ if (rendcirc && introcirc) { + /* Let's fill out the hs_ident fully as soon as possible, so that + * unreachability counts can be updated properly even if circuits close + * early. */ + tor_assert_nonfatal(!ed25519_public_key_is_zero( + &introcirc->hs_ident->intro_auth_pk)); + ed25519_pubkey_copy(&rendcirc->hs_ident->intro_auth_pk, + &introcirc->hs_ident->intro_auth_pk); + } + if (retval > 0) { /* one has already sent the intro. keep waiting. */ tor_assert(introcirc); diff --git a/src/feature/hs/hs_cache.c b/src/feature/hs/hs_cache.c index dcca1d7086..0cc7dfd031 100644 --- a/src/feature/hs/hs_cache.c +++ b/src/feature/hs/hs_cache.c @@ -581,6 +581,8 @@ cache_client_intro_state_lookup(const ed25519_public_key_t *service_pk,

tor_assert(service_pk); tor_assert(auth_key); + tor_assert_nonfatal(!ed25519_public_key_is_zero(service_pk)); + tor_assert_nonfatal(!ed25519_public_key_is_zero(auth_key));

/* Lookup the intro state cache for this service key. */ cache = digest256map_get(hs_cache_client_intro_state, service_pk->pubkey); diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 92217d760a..d5188b514f 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -251,6 +251,7 @@ create_intro_circuit_identifier(const hs_service_t *service,

ident = hs_ident_circuit_new(&service->keys.identity_pk); ed25519_pubkey_copy(&ident->intro_auth_pk, &ip->auth_key_kp.pubkey); + tor_assert_nonfatal(!ed25519_public_key_is_zero(&ident->intro_auth_pk));

return ident; } diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index 2a620da953..6303b8fe75 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -549,6 +549,7 @@ find_desc_intro_point_by_ident(const hs_ident_circuit_t *ident,

tor_assert(ident); tor_assert(desc); + tor_assert_nonfatal(!ed25519_public_key_is_zero(&ident->intro_auth_pk));

SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points, const hs_desc_intro_point_t *, ip) { @@ -634,15 +635,8 @@ send_introduce1(origin_circuit_t *intro_circ, return -1; /* transient failure */ }

- /* Cell has been sent successfully. Copy the introduction point - * authentication and encryption key in the rendezvous circuit identifier so - * we can compute the ntor keys when we receive the RENDEZVOUS2 cell. */ - memcpy(&rend_circ->hs_ident->intro_enc_pk, &ip->enc_key, - sizeof(rend_circ->hs_ident->intro_enc_pk)); - ed25519_pubkey_copy(&rend_circ->hs_ident->intro_auth_pk, - &intro_circ->hs_ident->intro_auth_pk); - - /* Now, we wait for an ACK or NAK on this circuit. */ + /* Cell has been sent successfully. + * Now, we wait for an ACK or NAK on this circuit. */ circuit_change_purpose(TO_CIRCUIT(intro_circ), CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT); /* Set timestamp_dirty, because circuit_expire_building expects it to @@ -657,6 +651,17 @@ send_introduce1(origin_circuit_t *intro_circ, * tackle. If asked for higher, we solve it at this cap. */ #define CLIENT_MAX_POW_EFFORT 10000

+/** Set a client-side minimum effort. If the client is choosing to increase + * effort on retry, it will always pick a value >= this lower limit. */ +#define CLIENT_MIN_RETRY_POW_EFFORT 8 + +/** Client effort will double on every retry until this level is hit */ +#define CLIENT_POW_EFFORT_DOUBLE_UNTIL 1000 + +/** After we reach DOUBLE_UNTIL, client effort is multiplied by this amount + * on every retry until we reach MAX_POW_EFFORT. */ +#define CLIENT_POW_RETRY_MULTIPLIER (1.5f) + /** Send an INTRODUCE1 cell along the intro circuit and populate the rend * circuit identifier with the needed key material for the e2e encryption. * Return 0 on success, -1 if there is a transient error such that an action @@ -731,37 +736,76 @@ consider_sending_introduce1(origin_circuit_t *intro_circ, goto perm_err; }

- /* If the descriptor contains PoW parameters then the service is - * expecting a PoW solution in the INTRODUCE cell, which we solve here. */ - if (have_module_pow() && - desc->encrypted_data.pow_params && - desc->encrypted_data.pow_params->suggested_effort > 0) { - log_debug(LD_REND, "PoW params present in descriptor."); + /* Copy the introduction point authentication and encryption key + * in the rendezvous circuit identifier so we can compute the ntor keys + * when we receive the RENDEZVOUS2 cell. */ + memcpy(&rend_circ->hs_ident->intro_enc_pk, &ip->enc_key, + sizeof(rend_circ->hs_ident->intro_enc_pk));

- /* make sure we can't be tricked into hopeless quests */ - if (desc->encrypted_data.pow_params->suggested_effort > - CLIENT_MAX_POW_EFFORT) { + /* Optionally choose to solve a client puzzle for this connection. This + * is only available if we have PoW support at compile time, and if the + * service has provided a PoW seed in its descriptor. The puzzle is enabled + * any time effort is nonzero, which can be recommended by the service or + * self-imposed as a result of previous timeouts. + */ + if (have_module_pow() && desc->encrypted_data.pow_params) { + hs_pow_solver_inputs_t pow_inputs = { + .effort = desc->encrypted_data.pow_params->suggested_effort + }; + memcpy(pow_inputs.seed, desc->encrypted_data.pow_params->seed, + sizeof pow_inputs.seed); + log_debug(LD_REND, "PoW params present in descriptor, suggested_effort=%u", + pow_inputs.effort); + + if (pow_inputs.effort > CLIENT_MAX_POW_EFFORT) { log_notice(LD_REND, "Onion service suggested effort %d which is " "higher than we want to solve. Solving at %d instead.", - desc->encrypted_data.pow_params->suggested_effort, - CLIENT_MAX_POW_EFFORT); - - /* clobber it in-place. hopefully this won't have bad side effects. */ - desc->encrypted_data.pow_params->suggested_effort = - CLIENT_MAX_POW_EFFORT; + pow_inputs.effort, CLIENT_MAX_POW_EFFORT); + pow_inputs.effort = CLIENT_MAX_POW_EFFORT; }

- /* send it to the client-side pow cpuworker for solving. */ - intro_circ->hs_currently_solving_pow = 1; - if (0 != hs_pow_queue_work(intro_circ->global_identifier, - rend_circ->global_identifier, - desc->encrypted_data.pow_params)) { - log_debug(LD_REND, "Failed to enqueue PoW request"); + const hs_cache_intro_state_t *state = + hs_cache_client_intro_state_find(&intro_circ->hs_ident->identity_pk, + &intro_circ->hs_ident->intro_auth_pk); + uint32_t unreachable_count = state ? state->unreachable_count : 0; + if (state) { + log_debug(LD_REND, "hs_cache state during PoW consideration, " + "error=%d timed_out=%d unreachable_count=%u", + state->error, state->timed_out, state->unreachable_count); + } + uint64_t new_effort = pow_inputs.effort; + for (unsigned n_retry = 0; n_retry < unreachable_count; n_retry++) { + if (new_effort >= CLIENT_MAX_POW_EFFORT) { + break; + } + if (new_effort < CLIENT_POW_EFFORT_DOUBLE_UNTIL) { + new_effort <<= 1; + } else { + new_effort = (uint64_t) (CLIENT_POW_RETRY_MULTIPLIER * new_effort); + } + new_effort = MAX((uint64_t)CLIENT_MIN_RETRY_POW_EFFORT, new_effort); + new_effort = MIN((uint64_t)CLIENT_MAX_POW_EFFORT, new_effort); + } + if (pow_inputs.effort != (uint32_t)new_effort) { + log_notice(LD_REND, "Increasing PoW effort from %d to %d after intro " + "point unreachable_count=%d", + pow_inputs.effort, (int)new_effort, unreachable_count); + pow_inputs.effort = (uint32_t)new_effort; }

- /* can't proceed with the intro1 cell yet, so yield back to the - * main loop */ - goto tran_err; + if (pow_inputs.effort > 0) { + /* send it to the client-side pow cpuworker for solving. */ + intro_circ->hs_currently_solving_pow = 1; + if (hs_pow_queue_work(intro_circ->global_identifier, + rend_circ->global_identifier, + &pow_inputs) != 0) { + log_warn(LD_REND, "Failed to enqueue PoW request"); + } + + /* can't proceed with the intro1 cell yet, so yield back to the + * main loop */ + goto tran_err; + } }

/* move on to the next phase: actually try to send it */ @@ -796,8 +840,8 @@ consider_sending_introduce1(origin_circuit_t *intro_circ, * * Return 0 if everything went well, otherwise return -1 in the case of errors. */ -static int -setup_intro_circ_auth_key(origin_circuit_t *circ) +int +hs_client_setup_intro_circ_auth_key(origin_circuit_t *circ) { const hs_descriptor_t *desc; const hs_desc_intro_point_t *ip; @@ -843,13 +887,6 @@ client_intro_circ_has_opened(origin_circuit_t *circ) log_info(LD_REND, "Introduction circuit %u has opened. Attaching streams.", (unsigned int) TO_CIRCUIT(circ)->n_circ_id);

- /* This is an introduction circuit so we'll attach the correct - * authentication key to the circuit identifier so it can be identified - * properly later on. */ - if (setup_intro_circ_auth_key(circ) < 0) { - return; - } - connection_ap_attach_pending(1); }

@@ -2047,6 +2084,7 @@ hs_client_circuit_cleanup_on_free(const circuit_t *circ)

orig_circ = CONST_TO_ORIGIN_CIRCUIT(circ); tor_assert(orig_circ->hs_ident); + const ed25519_public_key_t *intro_pk = &orig_circ->hs_ident->intro_auth_pk;

has_timed_out = (circ->marked_for_close_orig_reason == END_CIRC_REASON_TIMEOUT); @@ -2061,22 +2099,22 @@ hs_client_circuit_cleanup_on_free(const circuit_t *circ) safe_str_client(ed25519_fmt(&orig_circ->hs_ident->identity_pk)), safe_str_client(build_state_get_exit_nickname(orig_circ->build_state)), failure); + tor_assert_nonfatal(!ed25519_public_key_is_zero(intro_pk)); hs_cache_client_intro_state_note(&orig_circ->hs_ident->identity_pk, - &orig_circ->hs_ident->intro_auth_pk, - failure); + intro_pk, failure); break; case CIRCUIT_PURPOSE_C_INTRODUCING: if (has_timed_out || !orig_circ->build_state) { break; } + tor_assert_nonfatal(!ed25519_public_key_is_zero(intro_pk)); failure = INTRO_POINT_FAILURE_UNREACHABLE; log_info(LD_REND, "Failed v3 intro circ for service %s to intro point %s " "(while building circuit). Marking as unreachable.", safe_str_client(ed25519_fmt(&orig_circ->hs_ident->identity_pk)), safe_str_client(build_state_get_exit_nickname(orig_circ->build_state))); hs_cache_client_intro_state_note(&orig_circ->hs_ident->identity_pk, - &orig_circ->hs_ident->intro_auth_pk, - failure); + intro_pk, failure); break; default: break; diff --git a/src/feature/hs/hs_client.h b/src/feature/hs/hs_client.h index e87cc00b75..234306a3c3 100644 --- a/src/feature/hs/hs_client.h +++ b/src/feature/hs/hs_client.h @@ -119,6 +119,8 @@ int hs_client_any_intro_points_usable(const ed25519_public_key_t *service_pk, int hs_client_refetch_hsdesc(const ed25519_public_key_t *identity_pk); void hs_client_dir_info_changed(void);

+int hs_client_setup_intro_circ_auth_key(origin_circuit_t *circ); + int hs_client_send_introduce1(origin_circuit_t *intro_circ, origin_circuit_t *rend_circ);

diff --git a/src/feature/hs/hs_pow.c b/src/feature/hs/hs_pow.c index 199c290004..1f6932a55d 100644 --- a/src/feature/hs/hs_pow.c +++ b/src/feature/hs/hs_pow.c @@ -177,7 +177,7 @@ unpack_equix_solution(const uint8_t *bytes_in, * store the solution in pow_solution_out. Returns 0 on success and -1 * otherwise. Called by a client. */ int -hs_pow_solve(const hs_pow_desc_params_t *pow_params, +hs_pow_solve(const hs_pow_solver_inputs_t *pow_inputs, hs_pow_solution_t *pow_solution_out) { int ret = -1; @@ -185,17 +185,15 @@ hs_pow_solve(const hs_pow_desc_params_t *pow_params, uint8_t *challenge = NULL; equix_ctx *ctx = NULL;

- tor_assert(pow_params); + tor_assert(pow_inputs); tor_assert(pow_solution_out); - - /* Select E (just using suggested for now) */ - uint32_t effort = pow_params->suggested_effort; + const uint32_t effort = pow_inputs->effort;

/* Generate a random nonce N. */ crypto_rand((char *)nonce, sizeof nonce);

/* Build EquiX challenge (C || N || INT_32(E)). */ - challenge = build_equix_challenge(pow_params->seed, nonce, effort); + challenge = build_equix_challenge(pow_inputs->seed, nonce, effort);

ctx = build_equix_ctx(EQUIX_CTX_SOLVE); if (!ctx) { @@ -218,7 +216,7 @@ hs_pow_solve(const hs_pow_desc_params_t *pow_params, /* Store the effort E. */ pow_solution_out->effort = effort; /* We only store the first 4 bytes of the seed C. */ - memcpy(pow_solution_out->seed_head, pow_params->seed, + memcpy(pow_solution_out->seed_head, pow_inputs->seed, sizeof(pow_solution_out->seed_head)); /* Store the solution S */ memcpy(&pow_solution_out->equix_solution, sol_bytes, sizeof sol_bytes); @@ -353,8 +351,8 @@ hs_pow_free_service_state(hs_pow_service_state_t *state) */ typedef struct pow_worker_job_t {

- /** Input: The pow challenge we need to solve. */ - hs_pow_desc_params_t *pow_params; + /** Inputs for the PoW solver (seed, chosen effort) */ + hs_pow_solver_inputs_t pow_inputs;

/** State: we'll look these up to figure out how to proceed after. */ uint32_t intro_circ_identifier; @@ -377,15 +375,15 @@ pow_worker_threadfn(void *state_, void *work_) pow_worker_job_t *job = work_; job->pow_solution_out = tor_malloc_zero(sizeof(hs_pow_solution_t));

- if (hs_pow_solve(job->pow_params, job->pow_solution_out)) { - log_info(LD_REND, "Haven't solved the PoW yet. Returning."); + if (hs_pow_solve(&job->pow_inputs, job->pow_solution_out)) { + log_warn(LD_REND, "Failed to run the proof of work solver"); tor_free(job->pow_solution_out); job->pow_solution_out = NULL; /* how we signal that we came up empty */ return WQ_RPL_REPLY; }

/* we have a winner! */ - log_info(LD_REND, "cpuworker pow: we have a winner!"); + log_info(LD_REND, "cpuworker has a proof of work solution"); return WQ_RPL_REPLY; }

@@ -397,7 +395,6 @@ pow_worker_job_free(pow_worker_job_t *job) { if (!job) return; - tor_free(job->pow_params); tor_free(job->pow_solution_out); tor_free(job); } @@ -470,14 +467,14 @@ pow_worker_replyfn(void *work_) int hs_pow_queue_work(uint32_t intro_circ_identifier, uint32_t rend_circ_identifier, - const hs_pow_desc_params_t *pow_params) + const hs_pow_solver_inputs_t *pow_inputs) { tor_assert(in_main_thread());

pow_worker_job_t *job = tor_malloc_zero(sizeof(*job)); job->intro_circ_identifier = intro_circ_identifier; job->rend_circ_identifier = rend_circ_identifier; - job->pow_params = tor_memdup(pow_params, sizeof(hs_pow_desc_params_t)); + memcpy(&job->pow_inputs, pow_inputs, sizeof job->pow_inputs);

workqueue_entry_t *work; work = cpuworker_queue_work(WQ_PRI_LOW, diff --git a/src/feature/hs/hs_pow.h b/src/feature/hs/hs_pow.h index fe78a48d9a..6e3611be69 100644 --- a/src/feature/hs/hs_pow.h +++ b/src/feature/hs/hs_pow.h @@ -56,6 +56,17 @@ typedef struct hs_pow_desc_params_t { time_t expiration_time; } hs_pow_desc_params_t;

+/** The inputs to the PoW solver, derived from the descriptor data and the + * client's per-connection effort choices. */ +typedef struct hs_pow_solver_inputs_t { + /** Seed value from a current descriptor */ + uint8_t seed[HS_POW_SEED_LEN]; + + /** Effort chosen by the client. May be higher or ower than + * suggested_effort in the descriptor. */ + uint32_t effort; +} hs_pow_solver_inputs_t; + /** State and parameters of PoW defenses, stored in the service state. */ typedef struct hs_pow_service_state_t { /* If PoW defenses are enabled this is a priority queue containing acceptable @@ -124,7 +135,7 @@ typedef struct hs_pow_solution_t { #define have_module_pow() (1)

/* API */ -int hs_pow_solve(const hs_pow_desc_params_t *pow_params, +int hs_pow_solve(const hs_pow_solver_inputs_t *pow_inputs, hs_pow_solution_t *pow_solution_out);

int hs_pow_verify(const hs_pow_service_state_t *pow_state, @@ -135,16 +146,16 @@ void hs_pow_free_service_state(hs_pow_service_state_t *state);

int hs_pow_queue_work(uint32_t intro_circ_identifier, uint32_t rend_circ_identifier, - const hs_pow_desc_params_t *pow_params); + const hs_pow_solver_inputs_t *pow_inputs);

#else /* !defined(HAVE_MODULE_POW) */ #define have_module_pow() (0)

static inline int -hs_pow_solve(const hs_pow_desc_params_t *pow_params, +hs_pow_solve(const hs_pow_solver_inputs_t *pow_inputs, hs_pow_solution_t *pow_solution_out) { - (void)pow_params; + (void)pow_inputs; (void)pow_solution_out; return -1; } @@ -173,11 +184,11 @@ hs_pow_free_service_state(hs_pow_service_state_t *state) static inline int hs_pow_queue_work(uint32_t intro_circ_identifier, uint32_t rend_circ_identifier, - const hs_pow_desc_params_t *pow_params) + const hs_pow_solver_inputs_t *pow_inputs) { (void)intro_circ_identifier; (void)rend_circ_identifier; - (void)pow_params; + (void)pow_inputs; return -1; }

diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c index 11a5589d21..f873d90212 100644 --- a/src/test/test_hs_client.c +++ b/src/test/test_hs_client.c @@ -177,6 +177,7 @@ helper_get_circ_and_stream_for_test(origin_circuit_t **circ_out,

/* prop224: Setup hs ident on the circuit */ or_circ->hs_ident = hs_ident_circuit_new(&service_pk); + or_circ->hs_ident->intro_auth_pk.pubkey[0] = 42;

TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;

@@ -1186,6 +1187,7 @@ test_socks_hs_errors(void *arg) circ->purpose = CIRCUIT_PURPOSE_C_REND_READY; ocirc = TO_ORIGIN_CIRCUIT(circ); ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey); + ocirc->hs_ident->intro_auth_pk.pubkey[0] = 42; ocirc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t)); /* Code path will log this exit so build it. */ ocirc->build_state->chosen_exit = extend_info_new("TestNickname", digest, diff --git a/src/test/test_hs_pow_slow.c b/src/test/test_hs_pow_slow.c index fdade2d3fa..e7d1311cee 100644 --- a/src/test/test_hs_pow_slow.c +++ b/src/test/test_hs_pow_slow.c @@ -187,17 +187,16 @@ test_hs_pow_vectors(void *arg) uint8_t rng_bytes[HS_POW_NONCE_LEN]; hs_pow_solution_t output; hs_pow_solution_t solution = { 0 }; - hs_pow_desc_params_t params = { - .type = HS_POW_DESC_V1, - .suggested_effort = vectors[vec_i].effort, + hs_pow_solver_inputs_t input = { + .effort = vectors[vec_i].effort, };

- tt_int_op(strlen(seed_hex), OP_EQ, 2 * sizeof params.seed); + tt_int_op(strlen(seed_hex), OP_EQ, 2 * sizeof input.seed); tt_int_op(strlen(solve_rng_hex), OP_EQ, 2 * sizeof rng_bytes); tt_int_op(strlen(nonce_hex), OP_EQ, 2 * sizeof solution.nonce); tt_int_op(strlen(sol_hex), OP_EQ, 2 * sizeof solution.equix_solution);

- tt_int_op(base16_decode((char*)params.seed, HS_POW_SEED_LEN, + tt_int_op(base16_decode((char*)input.seed, HS_POW_SEED_LEN, seed_hex, 2 * HS_POW_SEED_LEN), OP_EQ, HS_POW_SEED_LEN); tt_int_op(base16_decode((char*)rng_bytes, sizeof rng_bytes, @@ -210,11 +209,11 @@ test_hs_pow_vectors(void *arg) sizeof solution.equix_solution, sol_hex, 2 * sizeof solution.equix_solution), OP_EQ, HS_POW_EQX_SOL_LEN); - memcpy(solution.seed_head, params.seed, HS_POW_SEED_HEAD_LEN); + memcpy(solution.seed_head, input.seed, HS_POW_SEED_HEAD_LEN);

memset(&output, 0xaa, sizeof output); testing_enable_prefilled_rng(rng_bytes, HS_POW_NONCE_LEN); - tt_int_op(0, OP_EQ, hs_pow_solve(&params, &output)); + tt_int_op(0, OP_EQ, hs_pow_solve(&input, &output)); testing_disable_prefilled_rng();

tt_mem_op(solution.seed_head, OP_EQ, output.seed_head, @@ -224,7 +223,7 @@ test_hs_pow_vectors(void *arg) tt_mem_op(&solution.equix_solution, OP_EQ, &output.equix_solution, sizeof output.equix_solution);

- tt_int_op(testing_one_hs_pow_solution(&output, params.seed), OP_EQ, 0); + tt_int_op(testing_one_hs_pow_solution(&output, input.seed), OP_EQ, 0); }

done:

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit ee63863dcad8eb88584e67beaa2618cbb9f24127 Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Thu Apr 6 08:34:47 2023 -0700

hs_pow: Lower several logs from notice to info

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/feature/hs/hs_circuit.c | 26 +++++++++++++------------- src/feature/hs/hs_client.c | 6 +++--- src/feature/hs/hs_pow.c | 4 ++-- src/feature/hs/hs_service.c | 4 ++-- 4 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index d5188b514f..3978c69567 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -665,7 +665,7 @@ trim_rend_pqueue(hs_pow_service_state_t *pow_state, time_t now) smartlist_t *old_pqueue = pow_state->rend_request_pqueue; smartlist_t *new_pqueue = pow_state->rend_request_pqueue = smartlist_new();

- log_notice(LD_REND, "Rendezvous request priority queue has " + log_info(LD_REND, "Rendezvous request priority queue has " "reached capacity (%d). Discarding the bottom half.", smartlist_len(old_pqueue));

@@ -776,10 +776,10 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg)

(void) ev; /* Not using the returned event, make compiler happy. */

- log_notice(LD_REND, "Considering launching more rendezvous responses. " - "%d in-flight, %d pending.", - in_flight, - smartlist_len(pow_state->rend_request_pqueue)); + log_info(LD_REND, "Considering launching more rendezvous responses. " + "%d in-flight, %d pending.", + in_flight, + smartlist_len(pow_state->rend_request_pqueue));

/* Process rendezvous request until the maximum per mainloop run. */ while (smartlist_len(pow_state->rend_request_pqueue) > 0) { @@ -790,8 +790,8 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) /* We have queued requests, but they are all low priority, and also * we have too many in-progress rendezvous responses. Don't launch * any more. Schedule ourselves to reassess in a bit. */ - log_notice(LD_REND, "Next request to launch is low priority, and " - "%d in-flight already. Waiting to launch more.", in_flight); + log_info(LD_REND, "Next request to launch is low priority, and " + "%d in-flight already. Waiting to launch more.", in_flight); const struct timeval delay_tv = { 0, 100000 }; mainloop_event_schedule(pow_state->pop_pqueue_ev, &delay_tv); return; /* done here! no cleanup needed. */ @@ -820,7 +820,7 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) hs_metrics_pow_pqueue_rdv(service, smartlist_len(pow_state->rend_request_pqueue));

- log_notice(LD_REND, "Dequeued pending rendezvous request with effort: %u. " + log_info(LD_REND, "Dequeued pending rendezvous request with effort: %u. " "Waited %d. " "Remaining requests: %u", req->rdv_data.pow_effort, @@ -828,7 +828,7 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) smartlist_len(pow_state->rend_request_pqueue));

if (queued_rend_request_is_too_old(req, now)) { - log_notice(LD_REND, "Top rend request has been pending for too long; " + log_info(LD_REND, "Top rend request has been pending for too long; " "discarding and moving to the next one."); free_pending_rend(req); continue; /* do not increment count, this one's free */ @@ -907,7 +907,7 @@ enqueue_rend_request(const hs_service_t *service, hs_service_intro_point_t *ip, hs_metrics_pow_pqueue_rdv(service, smartlist_len(pow_state->rend_request_pqueue));

- log_notice(LD_REND, "Enqueued rendezvous request with effort: %u. " + log_info(LD_REND, "Enqueued rendezvous request with effort: %u. " "Queued requests: %u", req->rdv_data.pow_effort, smartlist_len(pow_state->rend_request_pqueue)); @@ -1382,9 +1382,9 @@ hs_circ_handle_introduce2(const hs_service_t *service, /* Add the rendezvous request to the priority queue if PoW defenses are * enabled, otherwise rendezvous as usual. */ if (have_module_pow() && service->config.has_pow_defenses_enabled) { - log_notice(LD_REND, - "Adding introduction request to pqueue with effort: %u", - data.rdv_data.pow_effort); + log_info(LD_REND, + "Adding introduction request to pqueue with effort: %u", + data.rdv_data.pow_effort); if (enqueue_rend_request(service, ip, &data, now) < 0) { goto done; } diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index 6303b8fe75..e77e1a3fbd 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -787,9 +787,9 @@ consider_sending_introduce1(origin_circuit_t *intro_circ, new_effort = MIN((uint64_t)CLIENT_MAX_POW_EFFORT, new_effort); } if (pow_inputs.effort != (uint32_t)new_effort) { - log_notice(LD_REND, "Increasing PoW effort from %d to %d after intro " - "point unreachable_count=%d", - pow_inputs.effort, (int)new_effort, unreachable_count); + log_info(LD_REND, "Increasing PoW effort from %d to %d after intro " + "point unreachable_count=%d", + pow_inputs.effort, (int)new_effort, unreachable_count); pow_inputs.effort = (uint32_t)new_effort; }

diff --git a/src/feature/hs/hs_pow.c b/src/feature/hs/hs_pow.c index 1f6932a55d..3cd14c5246 100644 --- a/src/feature/hs/hs_pow.c +++ b/src/feature/hs/hs_pow.c @@ -202,7 +202,7 @@ hs_pow_solve(const hs_pow_solver_inputs_t *pow_inputs, equix_solution solutions[EQUIX_MAX_SOLS]; uint8_t sol_bytes[HS_POW_EQX_SOL_LEN];

- log_notice(LD_REND, "Solving proof of work (effort %u)", effort); + log_info(LD_REND, "Solving proof of work (effort %u)", effort); for (;;) { /* Calculate solutions to S = equix_solve(C || N || E), */ int count = equix_solve(ctx, challenge, HS_POW_CHALLENGE_LEN, solutions); @@ -431,7 +431,7 @@ pow_worker_replyfn(void *work_) if (intro_circ && rend_circ && service_identity_pk && desc && ip && job->pow_solution_out) { /* successful pow solve, and circs still here */

- log_notice(LD_REND, "Got a PoW solution we like! Shipping it!"); + log_info(LD_REND, "Got a PoW solution we like! Shipping it!"); /* Set flag to reflect that the HS we are attempting to rendezvous has PoW * defenses enabled, and as such we will need to be more lenient with * timing out while waiting for the service-side circuit to be built. */ diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index 1a40e34cf7..43598ad768 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -298,7 +298,7 @@ initialize_pow_defenses(hs_service_t *service) * seed to be predictable even if it doesn't really exist yet, and it needs * to be different to the current nonce for the replay cache scrubbing to * function correctly. */ - log_notice(LD_REND, "Generating both PoW seeds..."); + log_info(LD_REND, "Generating both PoW seeds..."); crypto_rand((char *)&pow_state->seed_current, HS_POW_SEED_LEN); crypto_rand((char *)&pow_state->seed_previous, HS_POW_SEED_LEN);

@@ -2450,7 +2450,7 @@ update_all_descriptors_pow_params(time_t now) * initialise pow_params in the descriptors. If this runs the next if * statement will run and set the correct values. */ if (!encrypted->pow_params) { - log_notice(LD_REND, "Initializing pow_params in descriptor..."); + log_info(LD_REND, "Initializing pow_params in descriptor..."); encrypted->pow_params = tor_malloc_zero(sizeof(hs_pow_desc_params_t)); }

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 50313d114f12769579e7f26258e0956a5bbcaa00 Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Thu Apr 6 14:27:18 2023 -0700

hs_pow: faster hs_circuitmap lookup for rend in pow_worker_job_t

The worker job queue for hs_pow needs what's effectively a weak pointer to two circuits, but there's not a generic mechanism for this in c-tor. The previous approach of circuit_get_by_global_id() is straightforward but not efficient. These global IDs are normally only used by the control port protocol. To reduce the number of O(N) lookups we have over the whole circuit list, we can use hs_circuitmap to look up the rend circuit by its auth cookie.

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/feature/hs/hs_client.c | 2 +- src/feature/hs/hs_pow.c | 20 +++++++++++++++----- src/feature/hs/hs_pow.h | 6 +++--- 3 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index e77e1a3fbd..d8df1d0772 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -797,7 +797,7 @@ consider_sending_introduce1(origin_circuit_t *intro_circ, /* send it to the client-side pow cpuworker for solving. */ intro_circ->hs_currently_solving_pow = 1; if (hs_pow_queue_work(intro_circ->global_identifier, - rend_circ->global_identifier, + rend_circ->hs_ident->rendezvous_cookie, &pow_inputs) != 0) { log_warn(LD_REND, "Failed to enqueue PoW request"); } diff --git a/src/feature/hs/hs_pow.c b/src/feature/hs/hs_pow.c index 3cd14c5246..15152bc649 100644 --- a/src/feature/hs/hs_pow.c +++ b/src/feature/hs/hs_pow.c @@ -16,6 +16,7 @@ #include "ext/equix/include/equix.h" #include "feature/hs/hs_cache.h" #include "feature/hs/hs_descriptor.h" +#include "feature/hs/hs_circuitmap.h" #include "feature/hs/hs_client.h" #include "feature/hs/hs_pow.h" #include "lib/crypt_ops/crypto_rand.h" @@ -356,7 +357,7 @@ typedef struct pow_worker_job_t {

/** State: we'll look these up to figure out how to proceed after. */ uint32_t intro_circ_identifier; - uint32_t rend_circ_identifier; + uint8_t rend_circ_cookie[HS_REND_COOKIE_LEN];

/** Output: The worker thread will malloc and write its answer here, * or set it to NULL if it produced no useful answer. */ @@ -411,11 +412,17 @@ pow_worker_replyfn(void *work_)

pow_worker_job_t *job = work_;

- // look up the circuits that we're going to use this pow in + /* Look up the circuits that we're going to use this pow in. + * There's room for improvement here. We already had a fast mapping to + * rend circuits from some kind of identifier that we can keep in a + * pow_worker_job_t, but we don't have that index for intro circs at this + * time. If the linear search in circuit_get_by_global_id() is ever a + * noticeable bottleneck we should add another map. + */ origin_circuit_t *intro_circ = circuit_get_by_global_id(job->intro_circ_identifier); origin_circuit_t *rend_circ = - circuit_get_by_global_id(job->rend_circ_identifier); + hs_circuitmap_get_established_rend_circ_client_side(job->rend_circ_cookie);

/* try to re-create desc and ip */ const ed25519_public_key_t *service_identity_pk = NULL; @@ -466,14 +473,17 @@ pow_worker_replyfn(void *work_) */ int hs_pow_queue_work(uint32_t intro_circ_identifier, - uint32_t rend_circ_identifier, + const uint8_t *rend_circ_cookie, const hs_pow_solver_inputs_t *pow_inputs) { tor_assert(in_main_thread()); + tor_assert(rend_circ_cookie); + tor_assert(pow_inputs);

pow_worker_job_t *job = tor_malloc_zero(sizeof(*job)); job->intro_circ_identifier = intro_circ_identifier; - job->rend_circ_identifier = rend_circ_identifier; + memcpy(&job->rend_circ_cookie, rend_circ_cookie, + sizeof job->rend_circ_cookie); memcpy(&job->pow_inputs, pow_inputs, sizeof job->pow_inputs);

workqueue_entry_t *work; diff --git a/src/feature/hs/hs_pow.h b/src/feature/hs/hs_pow.h index 6e3611be69..357f527c34 100644 --- a/src/feature/hs/hs_pow.h +++ b/src/feature/hs/hs_pow.h @@ -145,7 +145,7 @@ void hs_pow_remove_seed_from_cache(const uint8_t *seed_head); void hs_pow_free_service_state(hs_pow_service_state_t *state);

int hs_pow_queue_work(uint32_t intro_circ_identifier, - uint32_t rend_circ_identifier, + const uint8_t *rend_circ_cookie, const hs_pow_solver_inputs_t *pow_inputs);

#else /* !defined(HAVE_MODULE_POW) */ @@ -183,11 +183,11 @@ hs_pow_free_service_state(hs_pow_service_state_t *state)

static inline int hs_pow_queue_work(uint32_t intro_circ_identifier, - uint32_t rend_circ_identifier, + const uint8_t *rend_circ_cookie, const hs_pow_solver_inputs_t *pow_inputs) { (void)intro_circ_identifier; - (void)rend_circ_identifier; + (void)rend_circ_cookie; (void)pow_inputs; return -1; }

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit a13d7bd5e96765ac7c660415a498d9d9100ade62 Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Tue Apr 11 10:28:09 2023 -0700

hs_pow: always give other events a chance to run between rend requests

This dequeue path has been through a few revisions by now, first limiting us to a fixed number per event loop callback, then an additional limit based on a token bucket, then the current version which has only the token bucket.

The thinking behing processing multiple requests per callback was to optimize our usage of libevent, but in effect this creates a prioritization problem. I think even a small fixed limit would be less reliable than just backing out this optimization and always allowing other callbacks to interrupt us in-between dequeues.

With this patch I'm seeing much smoother queueing behavior when I add artificial delays to the main thread in testing.

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/feature/hs/hs_circuit.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index c3f2fbfc1e..ccd6711041 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -767,7 +767,9 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg) in_flight, smartlist_len(pow_state->rend_request_pqueue));

- /* Process rendezvous request until the maximum per mainloop run. */ + /* Process only one rend request per callback, so that this work will not + * be prioritized over other event loop callbacks. We may need to retry + * in order to find one request that's still viable. */ while (smartlist_len(pow_state->rend_request_pqueue) > 0) {

/* first, peek at the top result to see if we want to pop it */ @@ -827,11 +829,7 @@ handle_rend_pqueue_cb(mainloop_event_t *ev, void *arg)

++pow_state->rend_handled; ++in_flight; - - if (pow_state->using_pqueue_bucket && - token_bucket_ctr_get(&pow_state->pqueue_bucket) < 1) { - break; - } + break; }

/* If there are still some pending rendezvous circuits in the pqueue then

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit 971de27c0744deac7017d8da720eea39ad960a6f Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Tue Apr 25 09:02:45 2023 -0700

hs_pow: fix error path with outdated assumption

This error path with the "PoW cpuworker returned with no solution. Will retry soon." message was usually lying. It's concerning now because we expect to always find a solution no matter how long it takes, rather than re-enter the solver repeatedly, so any exit without a solution is a sign of a problem.

In fact when this error path gets hit, we are usually missing a circuit instead because the request is quite old and the circuits have been destroyed. This is not an emergency, it's just a sign of client-side overload.

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/feature/hs/hs_pow.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/feature/hs/hs_pow.c b/src/feature/hs/hs_pow.c index 15152bc649..4f662b58d9 100644 --- a/src/feature/hs/hs_pow.c +++ b/src/feature/hs/hs_pow.c @@ -436,9 +436,11 @@ pow_worker_replyfn(void *work_) ip = find_desc_intro_point_by_ident(intro_circ->hs_ident, desc);

if (intro_circ && rend_circ && service_identity_pk && desc && ip && - job->pow_solution_out) { /* successful pow solve, and circs still here */ + job->pow_solution_out) {

+ /* successful pow solve, and circs still here */ log_info(LD_REND, "Got a PoW solution we like! Shipping it!"); + /* Set flag to reflect that the HS we are attempting to rendezvous has PoW * defenses enabled, and as such we will need to be more lenient with * timing out while waiting for the service-side circuit to be built. */ @@ -451,18 +453,16 @@ pow_worker_replyfn(void *work_) intro_circ->hs_currently_solving_pow = 0; }

- } else { /* unsuccessful pow solve. put it back on the queue. */ - log_notice(LD_REND, - "PoW cpuworker returned with no solution. Will retry soon."); + } else { + if (!job->pow_solution_out) { + log_warn(LD_REND, "PoW cpuworker returned with no solution"); + } else { + log_info(LD_REND, "PoW solution completed but we can " + "no longer locate its circuit"); + } if (intro_circ) { intro_circ->hs_currently_solving_pow = 0; } - /* We could imagine immediately re-launching a follow-up worker - * here too, but for now just let the main intro loop find the - * not-being-serviced request and it can start everything again. For - * the sake of complexity, maybe that's the best long-term solution - * too, and we can tune the cpuworker job to try for longer if we want - * to improve efficiency. */ }

pow_worker_job_free(job);

This is an automated email from the git hooks/post-receive script.

dgoulet pushed a commit to branch main in repository tor.

commit e643a708793f748bf7c3dd4978762429e51411cf Author: Micah Elizabeth Scott beth@torproject.org AuthorDate: Mon May 8 23:58:30 2023 -0700

hs_pow: Modify challenge format, include blinded HS id

This is a protocol breaking change that implements nickm's changes to prop 327 to add an algorithm personalization string and blinded HS id to the EquiX challenge string for our onion service client puzzle.

This corresponds with the spec changes in torspec!130, and it fixes a proposed vulnerability documented in ticket tor#40789.

Clients and services prior to this patch will no longer be compatible with the proposed "v1" proof-of-work protocol.

Signed-off-by: Micah Elizabeth Scott beth@torproject.org --- src/feature/hs/hs_cell.c | 16 +++++--- src/feature/hs/hs_cell.h | 3 +- src/feature/hs/hs_circuit.c | 2 +- src/feature/hs/hs_client.c | 2 + src/feature/hs/hs_pow.c | 51 +++++++++++++++++------- src/feature/hs/hs_pow.h | 25 +++++++++--- src/feature/hs/hs_service.c | 8 ++++ src/feature/hs/hs_service.h | 4 ++ src/test/test_hs_pow.c | 55 ++++++++++++++++++-------- src/test/test_hs_pow_slow.c | 95 +++++++++++++++++++++++++++++++-------------- 10 files changed, 187 insertions(+), 74 deletions(-)

diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c index d3639cb92f..0039825f3c 100644 --- a/src/feature/hs/hs_cell.c +++ b/src/feature/hs/hs_cell.c @@ -799,14 +799,16 @@ hs_cell_parse_intro_established(const uint8_t *payload, size_t payload_len) * including if PoW couldn't be verified. */ static int handle_introduce2_encrypted_cell_pow_extension(const hs_service_t *service, - const trn_extension_field_t *field, - hs_cell_introduce2_data_t *data) + const hs_service_intro_point_t *ip, + const trn_extension_field_t *field, + hs_cell_introduce2_data_t *data) { int ret = -1; trn_cell_extension_pow_t *pow = NULL; hs_pow_solution_t sol;

tor_assert(field); + tor_assert(ip);

if (!service->state.pow_state) { log_info(LD_REND, "Unsolicited PoW solution in INTRODUCE2 request."); @@ -840,7 +842,7 @@ handle_introduce2_encrypted_cell_pow_extension(const hs_service_t *service, trn_cell_extension_pow_getconstarray_pow_solution(pow), HS_POW_EQX_SOL_LEN);

- if (hs_pow_verify(service->state.pow_state, &sol)) { + if (hs_pow_verify(&ip->blinded_id, service->state.pow_state, &sol)) { log_info(LD_REND, "PoW INTRODUCE2 request failed to verify."); goto end; } @@ -930,6 +932,7 @@ get_introduce2_keys_and_verify_mac(hs_cell_introduce2_data_t *data, * includes a PoW that doesn't verify). */ static int parse_introduce_cell_extension(const hs_service_t *service, + const hs_service_intro_point_t *ip, hs_cell_introduce2_data_t *data, const trn_extension_field_t *field) { @@ -948,7 +951,7 @@ parse_introduce_cell_extension(const hs_service_t *service, break; case TRUNNEL_EXT_TYPE_POW: /* PoW request. If successful, the effort is put in the data. */ - if (handle_introduce2_encrypted_cell_pow_extension(service, + if (handle_introduce2_encrypted_cell_pow_extension(service, ip, field, data) < 0) { log_fn(LOG_PROTOCOL_WARN, LD_REND, "Invalid PoW cell extension."); ret = -1; @@ -969,7 +972,8 @@ parse_introduce_cell_extension(const hs_service_t *service, ssize_t hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data, const origin_circuit_t *circ, - const hs_service_t *service) + const hs_service_t *service, + const hs_service_intro_point_t *ip) { int ret = -1; time_t elapsed; @@ -1102,7 +1106,7 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data, /* The number of extensions should match the number of fields. */ break; } - if (parse_introduce_cell_extension(service, data, field) < 0) { + if (parse_introduce_cell_extension(service, ip, data, field) < 0) { goto done; } } diff --git a/src/feature/hs/hs_cell.h b/src/feature/hs/hs_cell.h index 7b547991e6..02631cf376 100644 --- a/src/feature/hs/hs_cell.h +++ b/src/feature/hs/hs_cell.h @@ -122,7 +122,8 @@ ssize_t hs_cell_parse_intro_established(const uint8_t *payload, size_t payload_len); ssize_t hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data, const origin_circuit_t *circ, - const hs_service_t *service); + const hs_service_t *service, + const hs_service_intro_point_t *ip); int hs_cell_parse_introduce_ack(const uint8_t *payload, size_t payload_len); int hs_cell_parse_rendezvous2(const uint8_t *payload, size_t payload_len, uint8_t *handshake_info, diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 9311a26169..4c27f417c5 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -1333,7 +1333,7 @@ hs_circ_handle_introduce2(const hs_service_t *service, goto done; }

- if (hs_cell_parse_introduce2(&data, circ, service) < 0) { + if (hs_cell_parse_introduce2(&data, circ, service, ip) < 0) { hs_metrics_reject_intro_req(service, HS_METRICS_ERR_INTRO_REQ_INTRODUCE2); goto done; } diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index d8df1d0772..0e4b3ca0c3 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -752,6 +752,8 @@ consider_sending_introduce1(origin_circuit_t *intro_circ, hs_pow_solver_inputs_t pow_inputs = { .effort = desc->encrypted_data.pow_params->suggested_effort }; + ed25519_pubkey_copy(&pow_inputs.service_blinded_id, + &desc->plaintext_data.blinded_pubkey); memcpy(pow_inputs.seed, desc->encrypted_data.pow_params->seed, sizeof pow_inputs.seed); log_debug(LD_REND, "PoW params present in descriptor, suggested_effort=%u", diff --git a/src/feature/hs/hs_pow.c b/src/feature/hs/hs_pow.c index 1a23c69836..f75b3cb119 100644 --- a/src/feature/hs/hs_pow.c +++ b/src/feature/hs/hs_pow.c @@ -9,6 +9,8 @@

#include <stdio.h>

+#include "core/or/or.h" +#include "app/config/config.h" #include "ext/ht.h" #include "ext/compat_blake2.h" #include "core/or/circuitlist.h" @@ -20,6 +22,7 @@ #include "feature/hs/hs_client.h" #include "feature/hs/hs_pow.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_format.h" #include "lib/arch/bytes.h" #include "lib/cc/ctassert.h" #include "core/mainloop/cpuworker.h" @@ -85,7 +88,7 @@ increment_and_set_nonce(uint8_t *nonce, uint8_t *challenge) break; } } - memcpy(challenge + HS_POW_SEED_LEN, nonce, HS_POW_NONCE_LEN); + memcpy(challenge + HS_POW_NONCE_OFFSET, nonce, HS_POW_NONCE_LEN); }

/* Helper: Allocate an EquiX context, using the much faster compiled @@ -105,18 +108,32 @@ build_equix_ctx(equix_ctx_flags flags) return ctx; }

-/* Helper: Build EquiX challenge (C || N || INT_32(E)) and return a newly - * allocated buffer containing it. */ +/* Helper: Build EquiX challenge (P || ID || C || N || INT_32(E)) and return + * a newly allocated buffer containing it. */ static uint8_t * -build_equix_challenge(const uint8_t *seed, const uint8_t *nonce, +build_equix_challenge(const ed25519_public_key_t *blinded_id, + const uint8_t *seed, const uint8_t *nonce, const uint32_t effort) { - /* Build EquiX challenge (C || N || INT_32(E)). */ size_t offset = 0; uint8_t *challenge = tor_malloc_zero(HS_POW_CHALLENGE_LEN);

- memcpy(challenge, seed, HS_POW_SEED_LEN); + CTASSERT(HS_POW_ID_LEN == sizeof *blinded_id); + tor_assert_nonfatal(!ed25519_public_key_is_zero(blinded_id)); + + log_debug(LD_REND, + "Constructing EquiX challenge with " + "blinded service id %s, effort: %d", + safe_str_client(ed25519_fmt(blinded_id)), + effort); + + memcpy(challenge + offset, HS_POW_PSTRING, HS_POW_PSTRING_LEN); + offset += HS_POW_PSTRING_LEN; + memcpy(challenge + offset, blinded_id, HS_POW_ID_LEN); + offset += HS_POW_ID_LEN; + memcpy(challenge + offset, seed, HS_POW_SEED_LEN); offset += HS_POW_SEED_LEN; + tor_assert(HS_POW_NONCE_OFFSET == offset); memcpy(challenge + offset, nonce, HS_POW_NONCE_LEN); offset += HS_POW_NONCE_LEN; set_uint32(challenge + offset, tor_htonl(effort)); @@ -193,8 +210,9 @@ hs_pow_solve(const hs_pow_solver_inputs_t *pow_inputs, /* Generate a random nonce N. */ crypto_rand((char *)nonce, sizeof nonce);

- /* Build EquiX challenge (C || N || INT_32(E)). */ - challenge = build_equix_challenge(pow_inputs->seed, nonce, effort); + /* Build EquiX challenge string */ + challenge = build_equix_challenge(&pow_inputs->service_blinded_id, + pow_inputs->seed, nonce, effort);

ctx = build_equix_ctx(EQUIX_CTX_SOLVE); if (!ctx) { @@ -243,7 +261,8 @@ hs_pow_solve(const hs_pow_solver_inputs_t *pow_inputs, * parameters found in pow_state. Returns 0 on success and -1 otherwise. Called * by the service. */ int -hs_pow_verify(const hs_pow_service_state_t *pow_state, +hs_pow_verify(const ed25519_public_key_t *service_blinded_id, + const hs_pow_service_state_t *pow_state, const hs_pow_solution_t *pow_solution) { int ret = -1; @@ -254,6 +273,8 @@ hs_pow_verify(const hs_pow_service_state_t *pow_state,

tor_assert(pow_state); tor_assert(pow_solution); + tor_assert(service_blinded_id); + tor_assert_nonfatal(!ed25519_public_key_is_zero(service_blinded_id));

/* Find a valid seed C that starts with the seed head. Fail if no such seed * exists. */ @@ -278,13 +299,13 @@ hs_pow_verify(const hs_pow_service_state_t *pow_state, goto done; }

- /* Build the challenge with the param we have. */ - challenge = build_equix_challenge(seed, pow_solution->nonce, - pow_solution->effort); + /* Build the challenge with the params we have. */ + challenge = build_equix_challenge(service_blinded_id, seed, + pow_solution->nonce, pow_solution->effort);

if (!validate_equix_challenge(challenge, pow_solution->equix_solution, pow_solution->effort)) { - log_warn(LD_REND, "Equi-X solution and effort was too large."); + log_warn(LD_REND, "Verification of challenge effort in PoW failed."); goto done; }

@@ -293,7 +314,7 @@ hs_pow_verify(const hs_pow_service_state_t *pow_state, goto done; }

- /* Fail if equix_verify(C || N || E, S) != EQUIX_OK */ + /* Fail if equix_verify() != EQUIX_OK */ equix_solution equix_sol; unpack_equix_solution(pow_solution->equix_solution, &equix_sol); equix_result result = equix_verify(ctx, challenge, HS_POW_CHALLENGE_LEN, @@ -482,6 +503,8 @@ hs_pow_queue_work(uint32_t intro_circ_identifier, tor_assert(in_main_thread()); tor_assert(rend_circ_cookie); tor_assert(pow_inputs); + tor_assert_nonfatal( + !ed25519_public_key_is_zero(&pow_inputs->service_blinded_id));

pow_worker_job_t *job = tor_malloc_zero(sizeof(*job)); job->intro_circ_identifier = intro_circ_identifier; diff --git a/src/feature/hs/hs_pow.h b/src/feature/hs/hs_pow.h index 481c293cc5..3418d7a0cb 100644 --- a/src/feature/hs/hs_pow.h +++ b/src/feature/hs/hs_pow.h @@ -13,6 +13,7 @@ #include "lib/evloop/compat_libevent.h" #include "lib/evloop/token_bucket.h" #include "lib/smartlist_core/smartlist_core.h" +#include "lib/crypt_ops/crypto_ed25519.h"

/* Service updates the suggested effort every HS_UPDATE_PERIOD seconds. * This parameter controls how often we can change hs descriptor data to @@ -30,17 +31,27 @@ #define HS_POW_EQX_SOL_LEN 16 /** Length of blake2b hash result (R) used in the PoW scheme. */ #define HS_POW_HASH_LEN 4 +/** Length of algorithm personalization string (P) used in the PoW scheme */ +#define HS_POW_PSTRING_LEN 16 +/** Algorithm personalization string (P) */ +#define HS_POW_PSTRING "Tor hs intro v1\0" +/** Length of the blinded public ID for the onion service (ID) */ +#define HS_POW_ID_LEN 32 /** Length of random seed used in the PoW scheme. */ #define HS_POW_SEED_LEN 32 /** Length of seed identification heading in the PoW scheme. */ #define HS_POW_SEED_HEAD_LEN 4 /** Length of an effort value */ #define HS_POW_EFFORT_LEN sizeof(uint32_t) +/** Offset of the nonce value within the challenge string */ +#define HS_POW_NONCE_OFFSET \ + (HS_POW_PSTRING_LEN + HS_POW_ID_LEN + HS_POW_SEED_LEN) /** Length of a PoW challenge. Construction as per prop327 is: - * (C || N || INT_32(E)) + * (P || ID || C || N || INT_32(E)) */ #define HS_POW_CHALLENGE_LEN \ - (HS_POW_SEED_LEN + HS_POW_NONCE_LEN + HS_POW_EFFORT_LEN) + (HS_POW_PSTRING_LEN + HS_POW_ID_LEN + \ + HS_POW_SEED_LEN + HS_POW_NONCE_LEN + HS_POW_EFFORT_LEN)

/** Type of PoW in the descriptor. */ typedef enum { @@ -68,7 +79,8 @@ typedef struct hs_pow_desc_params_t { typedef struct hs_pow_solver_inputs_t { /** Seed value from a current descriptor */ uint8_t seed[HS_POW_SEED_LEN]; - + /** Blinded public ID for the onion service this puzzle is bound to */ + ed25519_public_key_t service_blinded_id; /** Effort chosen by the client. May be higher or ower than * suggested_effort in the descriptor. */ uint32_t effort; @@ -152,7 +164,8 @@ typedef struct hs_pow_solution_t { int hs_pow_solve(const hs_pow_solver_inputs_t *pow_inputs, hs_pow_solution_t *pow_solution_out);

-int hs_pow_verify(const hs_pow_service_state_t *pow_state, +int hs_pow_verify(const ed25519_public_key_t *service_blinded_id, + const hs_pow_service_state_t *pow_state, const hs_pow_solution_t *pow_solution);

void hs_pow_remove_seed_from_cache(const uint8_t *seed_head); @@ -175,9 +188,11 @@ hs_pow_solve(const hs_pow_solver_inputs_t *pow_inputs, }

static inline int -hs_pow_verify(const hs_pow_service_state_t *pow_state, +hs_pow_verify(const ed25519_public_key_t *service_blinded_id, + const hs_pow_service_state_t *pow_state, const hs_pow_solution_t *pow_solution) { + (void)service_blinded_id; (void)pow_state; (void)pow_solution; return -1; diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index a7e4e40a71..3ef2a9120c 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -2304,6 +2304,14 @@ pick_needed_intro_points(hs_service_t *service, safe_str_client(service->onion_address)); goto done; } + + /* Save a copy of the specific version of the blinded ID that we + * use to reach this intro point. Needed to validate proof-of-work + * solutions that are bound to this specific service. */ + tor_assert(desc->desc); + ed25519_pubkey_copy(&ip->blinded_id, + &desc->desc->plaintext_data.blinded_pubkey); + /* Valid intro point object, add it to the descriptor current map. */ service_intro_point_add(desc->intro_points.map, ip); } diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h index e7e53e73a3..36d67719ca 100644 --- a/src/feature/hs/hs_service.h +++ b/src/feature/hs/hs_service.h @@ -62,6 +62,10 @@ typedef struct hs_service_intro_point_t { /** Encryption keypair for the "ntor" type. */ curve25519_keypair_t enc_key_kp;

+ /** Blinded public ID for this service, from this intro point's + * active time period. */ + ed25519_public_key_t blinded_id; + /** Legacy key if that intro point doesn't support v3. This should be used if * the base object legacy flag is set. */ crypto_pk_t *legacy_key; diff --git a/src/test/test_hs_pow.c b/src/test/test_hs_pow.c index 909a2b569e..072fbacff4 100644 --- a/src/test/test_hs_pow.c +++ b/src/test/test_hs_pow.c @@ -264,6 +264,7 @@ test_hs_pow_vectors(void *arg) uint32_t validated_effort; int expected_retval; const char *seed_hex; + const char *service_blinded_id_hex; const char *nonce_hex; const char *sol_hex; const char *encoded_hex; @@ -272,6 +273,7 @@ test_hs_pow_vectors(void *arg) /* All zero, expect invalid */ 1, 0, -1, "0000000000000000000000000000000000000000000000000000000000000000", + "1111111111111111111111111111111111111111111111111111111111111111", "00000000000000000000000000000000", "00000000000000000000000000000000", "01" "00000000000000000000000000000000" @@ -282,67 +284,74 @@ test_hs_pow_vectors(void *arg) /* Valid zero-effort solution */ 0, 0, 0, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "55555555555555555555555555555555", "fd57d7676238c0ad1d5473aa2d0cbff5", + "1111111111111111111111111111111111111111111111111111111111111111", + "55555555555555555555555555555555", "4312f87ceab844c78e1c793a913812d7", "01" "55555555555555555555555555555555" "00000000" "aaaaaaaa" - "fd57d7676238c0ad1d5473aa2d0cbff5" + "4312f87ceab844c78e1c793a913812d7" }, { /* Valid high-effort solution */ 1000000, 1000000, 0, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "16505855555555555555555555555555", "bf2c2d345e5773b5c32ec5596244bdbc", + "1111111111111111111111111111111111111111111111111111111111111111", + "59217255555555555555555555555555", "0f3db97b9cac20c1771680a1a34848d3", "01" - "16505855555555555555555555555555" + "59217255555555555555555555555555" "000f4240" "aaaaaaaa" - "bf2c2d345e5773b5c32ec5596244bdbc" + "0f3db97b9cac20c1771680a1a34848d3" }, { /* Reject replays */ 1000000, 0, -1, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "16505855555555555555555555555555", "bf2c2d345e5773b5c32ec5596244bdbc", + "1111111111111111111111111111111111111111111111111111111111111111", + "59217255555555555555555555555555", "0f3db97b9cac20c1771680a1a34848d3", "01" - "16505855555555555555555555555555" + "59217255555555555555555555555555" "000f4240" "aaaaaaaa" - "bf2c2d345e5773b5c32ec5596244bdbc" + "0f3db97b9cac20c1771680a1a34848d3" }, { /* The claimed effort must exactly match what's in the challenge */ 99999, 0, -1, "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", - "cdd49fdbc34326d9d2f18ed277469c63", "7f153437c58620d3ea4717746093dde6", + "bfd298428562e530c52bdb36d81a0e293ef4a0e94d787f0f8c0c611f4f9e78ed", + "2eff9fdbc34326d9d2f18ed277469c63", "400cb091139f86b352119f6e131802d6", "01" - "cdd49fdbc34326d9d2f18ed277469c63" + "2eff9fdbc34326d9d2f18ed277469c63" "0001869f" "86fb0acf" - "7f153437c58620d3ea4717746093dde6" + "400cb091139f86b352119f6e131802d6" }, { /* Otherwise good solution but with a corrupted nonce */ 100000, 0, -1, "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", - "cdd49fdbc34326d9d2f18ed270469c63", "7f153437c58620d3ea4717746093dde6", + "bfd298428562e530c52bdb36d81a0e293ef4a0e94d787f0f8c0c611f4f9e78ed", + "2eff9fdbc34326d9a2f18ed277469c63", "400cb091139f86b352119f6e131802d6", "01" - "cdd49fdbc34326d9d2f18ed270469c63" + "2eff9fdbc34326d9a2f18ed277469c63" "000186a0" "86fb0acf" - "7f153437c58620d3ea4717746093dde6" + "400cb091139f86b352119f6e131802d6" }, { /* Corrected version of above */ 100000, 100000, 0, "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", - "cdd49fdbc34326d9d2f18ed277469c63", "7f153437c58620d3ea4717746093dde6", + "bfd298428562e530c52bdb36d81a0e293ef4a0e94d787f0f8c0c611f4f9e78ed", + "2eff9fdbc34326d9d2f18ed277469c63", "400cb091139f86b352119f6e131802d6", "01" - "cdd49fdbc34326d9d2f18ed277469c63" + "2eff9fdbc34326d9d2f18ed277469c63" "000186a0" "86fb0acf" - "7f153437c58620d3ea4717746093dde6" + "400cb091139f86b352119f6e131802d6" } };

testing_hs_pow_service_t *tsvc = testing_hs_pow_service_new(); hs_pow_service_state_t *pow_state = tor_malloc_zero(sizeof *pow_state); tsvc->service.state.pow_state = pow_state; + tsvc->service.desc_current = service_descriptor_new(); pow_state->rend_request_pqueue = smartlist_new();

char *mem_op_hex_tmp = NULL; @@ -353,6 +362,7 @@ test_hs_pow_vectors(void *arg) const unsigned num_vectors = sizeof vectors / sizeof vectors[0]; for (unsigned vec_i = 0; vec_i < num_vectors; vec_i++) { const int expected_retval = vectors[vec_i].expected_retval; + const char *service_blinded_id_hex = vectors[vec_i].service_blinded_id_hex; const char *seed_hex = vectors[vec_i].seed_hex; const char *nonce_hex = vectors[vec_i].nonce_hex; const char *sol_hex = vectors[vec_i].sol_hex; @@ -368,10 +378,19 @@ test_hs_pow_vectors(void *arg) }; int retval;

+ tt_int_op(strlen(service_blinded_id_hex), OP_EQ, 2 * HS_POW_ID_LEN); tt_int_op(strlen(seed_hex), OP_EQ, 2 * HS_POW_SEED_LEN); tt_int_op(strlen(nonce_hex), OP_EQ, 2 * sizeof solution.nonce); tt_int_op(strlen(sol_hex), OP_EQ, 2 * sizeof solution.equix_solution);

+ tt_assert(tsvc->service.desc_current); + ed25519_public_key_t *desc_blinded_pubkey = + &tsvc->service.desc_current->desc->plaintext_data.blinded_pubkey; + + tt_int_op(base16_decode((char*)desc_blinded_pubkey->pubkey, + HS_POW_ID_LEN, service_blinded_id_hex, + 2 * HS_POW_ID_LEN), + OP_EQ, HS_POW_ID_LEN); tt_int_op(base16_decode((char*)pow_state->seed_previous, HS_POW_SEED_LEN, seed_hex, 2 * HS_POW_SEED_LEN), OP_EQ, HS_POW_SEED_LEN); @@ -382,6 +401,7 @@ test_hs_pow_vectors(void *arg) sol_hex, 2 * HS_POW_EQX_SOL_LEN), OP_EQ, HS_POW_EQX_SOL_LEN);

+ ed25519_pubkey_copy(&tsvc->service_ip->blinded_id, desc_blinded_pubkey); memcpy(solution.seed_head, pow_state->seed_previous, HS_POW_SEED_HEAD_LEN);

/* Try to encode 'solution' into a relay cell */ @@ -468,6 +488,7 @@ test_hs_pow_vectors(void *arg) tor_free(decrypted); trn_cell_introduce1_free(cell); trn_cell_introduce_encrypted_free(enc_cell); + service_descriptor_free(tsvc->service.desc_current); testing_hs_pow_service_free(tsvc); hs_pow_remove_seed_from_cache(NULL); } diff --git a/src/test/test_hs_pow_slow.c b/src/test/test_hs_pow_slow.c index e7d1311cee..e21eee3395 100644 --- a/src/test/test_hs_pow_slow.c +++ b/src/test/test_hs_pow_slow.c @@ -21,6 +21,7 @@

static int testing_one_hs_pow_solution(const hs_pow_solution_t *ref_solution, + const ed25519_public_key_t *service_blinded_id, const uint8_t *seed) { int retval = -1; @@ -52,7 +53,8 @@ testing_one_hs_pow_solution(const hs_pow_solution_t *ref_solution, sol_buffer.equix_solution[variant / 2 % HS_POW_EQX_SOL_LEN]++; }

- tt_int_op(expected, OP_EQ, hs_pow_verify(s, &sol_buffer)); + tt_int_op(expected, OP_EQ, + hs_pow_verify(service_blinded_id, s, &sol_buffer)); } }

@@ -77,109 +79,136 @@ test_hs_pow_vectors(void *arg) uint32_t effort; const char *solve_rng_hex; const char *seed_hex; + const char *service_blinded_id_hex; const char *nonce_hex; const char *sol_hex; } vectors[] = { { 0, "55555555555555555555555555555555", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "55555555555555555555555555555555", "fd57d7676238c0ad1d5473aa2d0cbff5" + "1111111111111111111111111111111111111111111111111111111111111111", + "55555555555555555555555555555555", "4312f87ceab844c78e1c793a913812d7" }, { 1, "55555555555555555555555555555555", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "55555555555555555555555555555555", "703d8bc75492e8f90d836dd21bde61fc" + "1111111111111111111111111111111111111111111111111111111111111111", + "55555555555555555555555555555555", "84355542ab2b3f79532ef055144ac5ab" + }, + { + 1, "55555555555555555555555555555555", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "1111111111111111111111111111111111111111111111111111111111111110", + "55555555555555555555555555555555", "115e4b70da858792fc205030b8c83af9" }, { 2, "55555555555555555555555555555555", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "56555555555555555555555555555555", "c2374478d35040b53e4eb9aa9f16e9ec" + "1111111111111111111111111111111111111111111111111111111111111111", + "55555555555555555555555555555555", "4600a93a535ed76dc746c99942ab7de2" }, { 10, "55555555555555555555555555555555", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "5c555555555555555555555555555555", "b167af85e25a0c961928eff53672c1f8" + "1111111111111111111111111111111111111111111111111111111111111111", + "56555555555555555555555555555555", "128bbda5df2929c3be086de2aad34aed" }, { 10, "ffffffffffffffffffffffffffffffff", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "02000000000000000000000000000000", "954e4464715842d391712bb3b2289ff8" + "1111111111111111111111111111111111111111111111111111111111111111", + "01000000000000000000000000000000", "203af985537fadb23f3ed5873b4c81ce" }, { 1337, "7fffffffffffffffffffffffffffffff", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "eaffffffffffffffffffffffffffffff", "dbab3eb9045f85f8162c482d43f7d6fc" + "4111111111111111111111111111111111111111111111111111111111111111", + "01000000000000000000000000000000", "31c377cb72796ed80ae77df6ac1d6bfd" }, { - 31337, "00410000000000000000000000000000", + 31337, "34a20000000000000000000000000000", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "23410000000000000000000000000000", "545ddd60e33bfa73ec75aada68608ee8" + "1111111111111111111111111111111111111111111111111111111111111111", + "36a20000000000000000000000000000", "ca6899b91113aaf7536f28db42526bff" }, { - 100, "6b555555555555555555555555555555", + 100, "55555555555555555555555555555555", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "6b555555555555555555555555555555", "7e14e98fed2f35a1b293b39d56b260e9" + "1111111111111111111111111111111111111111111111111111111111111111", + "56555555555555555555555555555555", "3a4122a240bd7abfc922ab3cbb9479ed" }, { - 1000, "0e565555555555555555555555555555", + 1000, "d3555555555555555555555555555555", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "0e565555555555555555555555555555", "514963616e0b986afb1414afa88b85ff" + "1111111111111111111111111111111111111111111111111111111111111111", + "d4555555555555555555555555555555", "338cc08f57697ce8ac2e4b453057d6e9" }, { - 10000, "80835555555555555555555555555555", + 10000, "c5715555555555555555555555555555", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "89835555555555555555555555555555", "7a5164905f8aaec152126258a2462ae6" + "1111111111111111111111111111111111111111111111111111111111111111", + "c8715555555555555555555555555555", "9f2d3d4ed831ac96ad34c25fb59ff3e2" }, { - 100000, "fd995655555555555555555555555555", + 100000, "418d5655555555555555555555555555", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "fd995655555555555555555555555555", "8b27f2664340bc88dd5335821a68f5ff" + "1111111111111111111111111111111111111111111111111111111111111111", + "428d5655555555555555555555555555", "9863f3acd2d15adfd244a7ca61d4c6ff" }, { - 1000000, "15505855555555555555555555555555", + 1000000, "58217255555555555555555555555555", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "16505855555555555555555555555555", "bf2c2d345e5773b5c32ec5596244bdbc" + "1111111111111111111111111111111111111111111111111111111111111111", + "59217255555555555555555555555555", "0f3db97b9cac20c1771680a1a34848d3" }, { 1, "d0aec1669384bfe5ed39cd724d6c7954", "c52be1f8a5e6cc3b8fb71cfdbe272cbc91d4d035400f2f94fb0d0074794e0a07", - "d0aec1669384bfe5ed39cd724d6c7954", "9e062190e23b34a80562818b14cf4ae5" + "bfd298428562e530c52bdb36d81a0e293ef4a0e94d787f0f8c0c611f4f9e78ed", + "d1aec1669384bfe5ed39cd724d6c7954", "462606e5f8c2f3f844127b8bfdd6b4ff" }, { 1, "b4d0e611e6935750fcf9406aae131f62", "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", - "b4d0e611e6935750fcf9406aae131f62", "a01cf4457a016488df4fa45f0864b6fb" + "bfd298428562e530c52bdb36d81a0e293ef4a0e94d787f0f8c0c611f4f9e78ed", + "b4d0e611e6935750fcf9406aae131f62", "9f3fbd50b1a83fb63284bde44318c0fd" }, { 1, "b4d0e611e6935750fcf9406aae131f62", "9dfbd06d86fed8e12de3ab214e1a63ea61f46253fe08346a20378da70c4a327d", - "b5d0e611e6935750fcf9406aae131f62", "5944a260423392780f10b25b7e2502d3" + "bec632eb76123956f99a06d394fcbee8f135b8ed01f2e90aabe404cb0346744a", + "b4d0e611e6935750fcf9406aae131f62", "161baa7490356292d020065fdbe55ffc" }, { 1, "40559fdbc34326d9d2f18ed277469c63", "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", - "40559fdbc34326d9d2f18ed277469c63", "31139564ca5262a4f82b9385b2832fce" + "bfd298428562e530c52bdb36d81a0e293ef4a0e94d787f0f8c0c611f4f9e78ed", + "40559fdbc34326d9d2f18ed277469c63", "fa649c6a2c5c0bb6a3511b9ea4b448d1" }, { - 10000, "70559fdbc34326d9d2f18ed277469c63", + 10000, "34569fdbc34326d9d2f18ed277469c63", "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", - "72559fdbc34326d9d2f18ed277469c63", "262c6c82025c53b69b0bf255606ca3e2" + "bfd298428562e530c52bdb36d81a0e293ef4a0e94d787f0f8c0c611f4f9e78ed", + "36569fdbc34326d9d2f18ed277469c63", "2802951e623c74adc443ab93e99633ee" }, { - 100000, "c0d49fdbc34326d9d2f18ed277469c63", + 100000, "2cff9fdbc34326d9d2f18ed277469c63", "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", - "cdd49fdbc34326d9d2f18ed277469c63", "7f153437c58620d3ea4717746093dde6" + "bfd298428562e530c52bdb36d81a0e293ef4a0e94d787f0f8c0c611f4f9e78ed", + "2eff9fdbc34326d9d2f18ed277469c63", "400cb091139f86b352119f6e131802d6" }, { - 1000000, "40fdb1dbc34326d9d2f18ed277469c63", + 1000000, "5243b3dbc34326d9d2f18ed277469c63", "86fb0acf4932cda44dbb451282f415479462dd10cb97ff5e7e8e2a53c3767a7f", - "4cfdb1dbc34326d9d2f18ed277469c63", "b31bbb45340e17a14c2156c0b66780e7" + "bfd298428562e530c52bdb36d81a0e293ef4a0e94d787f0f8c0c611f4f9e78ed", + "5543b3dbc34326d9d2f18ed277469c63", "b47c718b56315e9697173a6bac1feaa4" }, };

const unsigned num_vectors = sizeof vectors / sizeof vectors[0]; for (unsigned vec_i = 0; vec_i < num_vectors; vec_i++) { const char *seed_hex = vectors[vec_i].seed_hex; + const char *service_blinded_id_hex = vectors[vec_i].service_blinded_id_hex; const char *solve_rng_hex = vectors[vec_i].solve_rng_hex; const char *nonce_hex = vectors[vec_i].nonce_hex; const char *sol_hex = vectors[vec_i].sol_hex; @@ -191,11 +220,16 @@ test_hs_pow_vectors(void *arg) .effort = vectors[vec_i].effort, };

+ tt_int_op(strlen(service_blinded_id_hex), OP_EQ, 2 * HS_POW_ID_LEN); tt_int_op(strlen(seed_hex), OP_EQ, 2 * sizeof input.seed); tt_int_op(strlen(solve_rng_hex), OP_EQ, 2 * sizeof rng_bytes); tt_int_op(strlen(nonce_hex), OP_EQ, 2 * sizeof solution.nonce); tt_int_op(strlen(sol_hex), OP_EQ, 2 * sizeof solution.equix_solution);

+ tt_int_op(base16_decode((char*)input.service_blinded_id.pubkey, + HS_POW_ID_LEN, service_blinded_id_hex, + 2 * HS_POW_ID_LEN), + OP_EQ, HS_POW_ID_LEN); tt_int_op(base16_decode((char*)input.seed, HS_POW_SEED_LEN, seed_hex, 2 * HS_POW_SEED_LEN), OP_EQ, HS_POW_SEED_LEN); @@ -223,7 +257,8 @@ test_hs_pow_vectors(void *arg) tt_mem_op(&solution.equix_solution, OP_EQ, &output.equix_solution, sizeof output.equix_solution);

- tt_int_op(testing_one_hs_pow_solution(&output, input.seed), OP_EQ, 0); + tt_int_op(testing_one_hs_pow_solution(&output, &input.service_blinded_id, + input.seed), OP_EQ, 0); }

done: