[tor] branch main updated (d5dea2202c -> 415c0354b2)
This is an automated email from the git hooks/post-receive script. dgoulet pushed a change to branch main in repository tor. from d5dea2202c changes: Add file for ticket 40797 new c40c5adec2 test_sandbox: equix crypto test case for issue 40794 new 941613c663 hashx: minor, another logical operator change new 6fd5ca4914 hashx: allow hashx_compile to fail, avoid segfault without changing API new 5a4f92ea7b hashx: API changes to allow recovery from late compile failures new a3513dea54 equix: API changes for new result codes and hashx compatibility new a397a92be2 hs_pow: Update for equix API to fix issue 40794 new 415c0354b2 hs_pow: Add CompiledProofOfWorkHash torrc option The 7 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: doc/man/tor.1.txt | 15 ++++ src/app/config/config.c | 1 + src/app/config/or_options_st.h | 5 ++ src/ext/equix/hashx/README.md | 37 +++++++-- src/ext/equix/hashx/include/hashx.h | 72 ++++++++++++----- src/ext/equix/hashx/src/bench.c | 34 +++++--- src/ext/equix/hashx/src/compiler.c | 9 ++- src/ext/equix/hashx/src/compiler.h | 11 +-- src/ext/equix/hashx/src/compiler_a64.c | 9 ++- src/ext/equix/hashx/src/compiler_x86.c | 7 +- src/ext/equix/hashx/src/context.c | 45 +++-------- src/ext/equix/hashx/src/context.h | 16 ++-- src/ext/equix/hashx/src/hashx.c | 78 ++++++++++++------ src/ext/equix/hashx/src/program.c | 4 +- src/ext/equix/hashx/src/tests.c | 116 +++++++++++++++++++-------- src/ext/equix/hashx/src/virtual_memory.c | 31 +++++--- src/ext/equix/hashx/src/virtual_memory.h | 5 +- src/ext/equix/include/equix.h | 57 ++++++++----- src/ext/equix/src/bench.c | 44 ++++++----- src/ext/equix/src/context.c | 25 +++--- src/ext/equix/src/equix.c | 70 +++++++++++----- src/ext/equix/src/solver.c | 3 +- src/ext/equix/src/tests.c | 61 +++++++------- src/feature/hs/hs_client.c | 3 +- src/feature/hs/hs_pow.c | 132 ++++++++++++++++++++----------- src/feature/hs/hs_pow.h | 2 + src/test/test_crypto.c | 10 +-- src/test/test_crypto_slow.c | 65 +++++++-------- src/test/test_hs_pow_slow.c | 1 + src/test/test_sandbox.c | 56 +++++++++++++ 30 files changed, 676 insertions(+), 348 deletions(-) -- To stop receiving notification emails like this one, please contact the administrator of this repository.
This is an automated email from the git hooks/post-receive script. dgoulet pushed a commit to branch main in repository tor. commit c40c5adec21aa02ba21fdf530aeff81b3523cb63 Author: Micah Elizabeth Scott <beth@torproject.org> AuthorDate: Tue May 23 19:18:50 2023 -0700 test_sandbox: equix crypto test case for issue 40794 This is an additional test case for test_sandbox that runs a small subset of test_crypto_equix() inside the syscall sandbox, where mprotect() is filtered. It's reasonable for the sandbox to disallow JIT. We could revise this policy if we want, but it seems a good default for now. The problem in issue 40794 is that both equix and hashx need improvements in their API to handle failures after allocation time, and this failure occurs while the hash function is being compiled. With this commit only, the segfault from issue 40794 is reproduced. Subsequent commits will fix the segfault and revise the API. Signed-off-by: Micah Elizabeth Scott <beth@torproject.org> --- src/test/test_sandbox.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/test/test_sandbox.c b/src/test/test_sandbox.c index 7ec08a3546..a28c9b6e41 100644 --- a/src/test/test_sandbox.c +++ b/src/test/test_sandbox.c @@ -12,6 +12,8 @@ #include "orconfig.h" #include "lib/sandbox/sandbox.h" +#include "lib/crypt_ops/crypto_rand.h" +#include "ext/equix/include/equix.h" #ifdef USE_LIBSECCOMP @@ -292,6 +294,63 @@ test_sandbox_stat_filename(void *arg) (void)0; } +/** This is a simplified subset of test_crypto_equix(), running one solve + * and one verify from inside the sandbox. The sandbox restricts mprotect, and + * hashx will experience a failure at runtime which this test case exercises. + * The result of the solve and verify should both still be correct, since we + * expect it to cleanly fall back on an interpreted implementation which has + * no operating system dependencies. */ +static void +test_sandbox_crypto_equix(void *arg) +{ + (void)arg; + + const char *challenge_literal = "abce"; + const size_t challenge_len = strlen(challenge_literal); + const size_t num_sols = 4; + static const equix_solution sols_expected[EQUIX_MAX_SOLS] = { + {{ 0x4fca, 0x72eb, 0x101f, 0xafab, 0x1add, 0x2d71, 0x75a3, 0xc978 }}, + {{ 0x17f1, 0x7aa6, 0x23e3, 0xab00, 0x7e2f, 0x917e, 0x16da, 0xda9e }}, + {{ 0x70ee, 0x7757, 0x8a54, 0xbd2b, 0x90e4, 0xe31e, 0x2085, 0xe47e }}, + {{ 0x62c5, 0x86d1, 0x5752, 0xe1f0, 0x12da, 0x8f33, 0x7336, 0xf161 }}, + }; + + equix_solution sols_actual[EQUIX_MAX_SOLS] = { 0 }; + equix_ctx *solve_ctx = NULL, *verify_ctx = NULL; + + /* TODO: A subsequent change will modify these flags to use an auto fallback + * that will be built into our fork of equix. (This implements a + * performant and low-complexity way to share the generated program + * state during fallback instead of re-generating it.) + */ + solve_ctx = equix_alloc(EQUIX_CTX_SOLVE | EQUIX_CTX_COMPILE); + tt_ptr_op(solve_ctx, OP_NE, NULL); + tt_ptr_op(solve_ctx, OP_NE, EQUIX_NOTSUPP); + + int retval = equix_solve(solve_ctx, challenge_literal, + challenge_len, sols_actual); + tt_int_op(retval, OP_EQ, num_sols); + tt_mem_op(sols_actual, OP_EQ, sols_expected, + num_sols * sizeof(equix_solution)); + + verify_ctx = equix_alloc(EQUIX_CTX_VERIFY | EQUIX_CTX_COMPILE); + tt_ptr_op(verify_ctx, OP_NE, NULL); + tt_ptr_op(verify_ctx, OP_NE, EQUIX_NOTSUPP); + + /* Test one of the solutions randomly */ + equix_result result; + const unsigned sol_i = crypto_rand_int(num_sols); + equix_solution *sol = &sols_actual[sol_i]; + + result = equix_verify(verify_ctx, challenge_literal, + challenge_len, sol); + tt_int_op(EQUIX_OK, OP_EQ, result); + + done: + equix_free(solve_ctx); + equix_free(verify_ctx); +} + #define SANDBOX_TEST_SKIPPED(name) \ { #name, test_sandbox_ ## name, TT_SKIP, NULL, NULL } @@ -343,6 +402,8 @@ struct testcase_t sandbox_tests[] = { #else SANDBOX_TEST_SKIPPED(stat_filename), #endif + + SANDBOX_TEST_IN_SANDBOX(crypto_equix), END_OF_TESTCASES }; -- To stop receiving notification emails like this one, please contact the administrator of this repository.
This is an automated email from the git hooks/post-receive script. dgoulet pushed a commit to branch main in repository tor. commit 941613c663785aaa401b723671bb8023cbb29eab Author: Micah Elizabeth Scott <beth@torproject.org> AuthorDate: Wed May 24 13:20:58 2023 -0700 hashx: minor, another logical operator change The code style in equix and hashx sometimes uses bitwise operators in place of logical ones in cases where it doesn't really matter either way. This sometimes annoys our static analyzer tools. Signed-off-by: Micah Elizabeth Scott <beth@torproject.org> --- src/ext/equix/hashx/src/program.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ext/equix/hashx/src/program.c b/src/ext/equix/hashx/src/program.c index f144ce14a0..b44bdb855a 100644 --- a/src/ext/equix/hashx/src/program.c +++ b/src/ext/equix/hashx/src/program.c @@ -712,8 +712,8 @@ bool hashx_program_generate(const siphash_state* key, hashx_program* program) { /* reject programs that don't meet the uniform complexity requirements */ /* this happens in less than 1 seed out of 10000 */ return - (program->code_size == REQUIREMENT_SIZE) & - (ctx.mul_count == REQUIREMENT_MUL_COUNT) & + (program->code_size == REQUIREMENT_SIZE) && + (ctx.mul_count == REQUIREMENT_MUL_COUNT) && (ctx.latency == REQUIREMENT_LATENCY - 1); /* cycles are numbered from 0 */ } -- To stop receiving notification emails like this one, please contact the administrator of this repository.
This is an automated email from the git hooks/post-receive script. dgoulet pushed a commit to branch main in repository tor. commit 6fd5ca491405807c0ac92ff9f343377b1c11a5c8 Author: Micah Elizabeth Scott <beth@torproject.org> AuthorDate: Wed May 24 13:25:54 2023 -0700 hashx: allow hashx_compile to fail, avoid segfault without changing API This is a minimal portion of the fix for tor issue #40794, in which hashx segfaults due to denial of mprotect() syscalls at runtime. Prior to this fix, hashx makes the assumption that if the JIT is supported on the current architecture, it will also be usable at runtime. This isn't true if mprotect fails on linux, which it may for various reasons: the tor built-in sandbox, the shadow simulator, or external security software that implements a syscall filter. The necessary error propagation was missing internally in hashx, causing us to obliviously call into code which was never made executable. With this fix, hashx_make() will instead fail by returning zero. A proper fix will require API changes so that callers can discern between different types of failures. Zero already means that a program couldn't be constructed, which requires a different response: choosing a different seed, vs switching implementations. Callers would also benefit from a way to use one context (with its already-built program) to run in either compiled or interpreted mode. Signed-off-by: Micah Elizabeth Scott <beth@torproject.org> --- src/ext/equix/hashx/src/compiler.h | 6 +++--- src/ext/equix/hashx/src/compiler_a64.c | 9 ++++++--- src/ext/equix/hashx/src/compiler_x86.c | 7 ++++--- src/ext/equix/hashx/src/hashx.c | 4 +++- src/ext/equix/hashx/src/virtual_memory.c | 28 +++++++++++++++------------- src/ext/equix/hashx/src/virtual_memory.h | 5 +++-- 6 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/ext/equix/hashx/src/compiler.h b/src/ext/equix/hashx/src/compiler.h index a100806fea..140f797a58 100644 --- a/src/ext/equix/hashx/src/compiler.h +++ b/src/ext/equix/hashx/src/compiler.h @@ -10,9 +10,9 @@ #include "virtual_memory.h" #include "program.h" -HASHX_PRIVATE void hashx_compile_x86(const hashx_program* program, uint8_t* code); +HASHX_PRIVATE bool hashx_compile_x86(const hashx_program* program, uint8_t* code); -HASHX_PRIVATE void hashx_compile_a64(const hashx_program* program, uint8_t* code); +HASHX_PRIVATE bool hashx_compile_a64(const hashx_program* program, uint8_t* code); #if defined(_M_X64) || defined(__x86_64__) #define HASHX_COMPILER 1 @@ -24,7 +24,7 @@ HASHX_PRIVATE void hashx_compile_a64(const hashx_program* program, uint8_t* code #define hashx_compile(p,c) hashx_compile_a64(p,c) #else #define HASHX_COMPILER 0 -#define hashx_compile(p,c) +#define hashx_compile(p,c) (false) #endif HASHX_PRIVATE bool hashx_compiler_init(hashx_ctx* compiler); diff --git a/src/ext/equix/hashx/src/compiler_a64.c b/src/ext/equix/hashx/src/compiler_a64.c index 48f743b988..94635ad1b7 100644 --- a/src/ext/equix/hashx/src/compiler_a64.c +++ b/src/ext/equix/hashx/src/compiler_a64.c @@ -48,8 +48,9 @@ static const uint8_t a64_epilogue[] = { 0xc0, 0x03, 0x5f, 0xd6, /* ret */ }; -void hashx_compile_a64(const hashx_program* program, uint8_t* code) { - hashx_vm_rw(code, COMP_CODE_SIZE); +bool hashx_compile_a64(const hashx_program* program, uint8_t* code) { + if (!hashx_vm_rw(code, COMP_CODE_SIZE)) + return false; uint8_t* pos = code; uint8_t* target = NULL; int creg = -1; @@ -145,10 +146,12 @@ void hashx_compile_a64(const hashx_program* program, uint8_t* code) { } } EMIT(pos, a64_epilogue); - hashx_vm_rx(code, COMP_CODE_SIZE); + if (!hashx_vm_rx(code, COMP_CODE_SIZE)) + return false; #ifdef __GNUC__ __builtin___clear_cache(code, pos); #endif + return true; } #endif diff --git a/src/ext/equix/hashx/src/compiler_x86.c b/src/ext/equix/hashx/src/compiler_x86.c index f03b17cca4..12f59a1d0b 100644 --- a/src/ext/equix/hashx/src/compiler_x86.c +++ b/src/ext/equix/hashx/src/compiler_x86.c @@ -81,8 +81,9 @@ static const uint8_t x86_epilogue[] = { 0xC3 /* ret */ }; -void hashx_compile_x86(const hashx_program* program, uint8_t* code) { - hashx_vm_rw(code, COMP_CODE_SIZE); +bool hashx_compile_x86(const hashx_program* program, uint8_t* code) { + if (!hashx_vm_rw(code, COMP_CODE_SIZE)) + return false; uint8_t* pos = code; uint8_t* target = NULL; EMIT(pos, x86_prologue); @@ -145,7 +146,7 @@ void hashx_compile_x86(const hashx_program* program, uint8_t* code) { } } EMIT(pos, x86_epilogue); - hashx_vm_rx(code, COMP_CODE_SIZE); + return hashx_vm_rx(code, COMP_CODE_SIZE); } #endif diff --git a/src/ext/equix/hashx/src/hashx.c b/src/ext/equix/hashx/src/hashx.c index da84aa51f3..372df09581 100644 --- a/src/ext/equix/hashx/src/hashx.c +++ b/src/ext/equix/hashx/src/hashx.c @@ -64,7 +64,9 @@ int hashx_make(hashx_ctx* ctx, const void* seed, size_t size) { if (!initialize_program(ctx, &program, keys)) { return 0; } - hashx_compile(&program, ctx->code); + if (!hashx_compile(&program, ctx->code)) { + return 0; + } return 1; } return initialize_program(ctx, ctx->program, keys); diff --git a/src/ext/equix/hashx/src/virtual_memory.c b/src/ext/equix/hashx/src/virtual_memory.c index e01fd878b9..8d574740c5 100644 --- a/src/ext/equix/hashx/src/virtual_memory.c +++ b/src/ext/equix/hashx/src/virtual_memory.c @@ -22,7 +22,7 @@ #ifdef HASHX_WIN -static int set_privilege(const char* pszPrivilege, BOOL bEnable) { +static bool set_privilege(const char* pszPrivilege, BOOL bEnable) { HANDLE hToken; TOKEN_PRIVILEGES tp; BOOL status; @@ -30,10 +30,10 @@ static int set_privilege(const char* pszPrivilege, BOOL bEnable) { if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hToken)) - return 0; + return false; if (!LookupPrivilegeValue(NULL, pszPrivilege, &tp.Privileges[0].Luid)) - return 0; + return false; tp.PrivilegeCount = 1; @@ -64,31 +64,33 @@ void* hashx_vm_alloc(size_t bytes) { return mem; } -static inline int page_protect(void* ptr, size_t bytes, int rules) { +static inline bool page_protect(void* ptr, size_t bytes, int rules) { #ifdef HASHX_WIN DWORD oldp; if (!VirtualProtect(ptr, bytes, (DWORD)rules, &oldp)) { - return 0; + return false; } #else - if (-1 == mprotect(ptr, bytes, rules)) - return 0; + if (mprotect(ptr, bytes, rules) != 0) + return false; #endif - return 1; + return true; } -void hashx_vm_rw(void* ptr, size_t bytes) { - page_protect(ptr, bytes, PAGE_READWRITE); +bool hashx_vm_rw(void* ptr, size_t bytes) { + return page_protect(ptr, bytes, PAGE_READWRITE); } -void hashx_vm_rx(void* ptr, size_t bytes) { - page_protect(ptr, bytes, PAGE_EXECUTE_READ); +bool hashx_vm_rx(void* ptr, size_t bytes) { + return page_protect(ptr, bytes, PAGE_EXECUTE_READ); } void* hashx_vm_alloc_huge(size_t bytes) { void* mem; #ifdef HASHX_WIN - set_privilege("SeLockMemoryPrivilege", 1); + if (!set_privilege("SeLockMemoryPrivilege", 1)) { + /* Failed, but try the VirtualAlloc anyway */ + } SIZE_T page_min = GetLargePageMinimum(); if (page_min > 0) { mem = VirtualAlloc(NULL, ALIGN_SIZE(bytes, page_min), MEM_COMMIT diff --git a/src/ext/equix/hashx/src/virtual_memory.h b/src/ext/equix/hashx/src/virtual_memory.h index d08f74dcc6..9780d218c2 100644 --- a/src/ext/equix/hashx/src/virtual_memory.h +++ b/src/ext/equix/hashx/src/virtual_memory.h @@ -6,13 +6,14 @@ #include <stdint.h> #include <stddef.h> +#include <stdbool.h> #include <hashx.h> #define ALIGN_SIZE(pos, align) ((((pos) - 1) / (align) + 1) * (align)) HASHX_PRIVATE void* hashx_vm_alloc(size_t size); -HASHX_PRIVATE void hashx_vm_rw(void* ptr, size_t size); -HASHX_PRIVATE void hashx_vm_rx(void* ptr, size_t size); +HASHX_PRIVATE bool hashx_vm_rw(void* ptr, size_t size); +HASHX_PRIVATE bool hashx_vm_rx(void* ptr, size_t size); HASHX_PRIVATE void* hashx_vm_alloc_huge(size_t size); HASHX_PRIVATE void hashx_vm_free(void* ptr, size_t size); -- To stop receiving notification emails like this one, please contact the administrator of this repository.
This is an automated email from the git hooks/post-receive script. dgoulet pushed a commit to branch main in repository tor. commit 5a4f92ea7b9f743548e4dba8843b13fb258a7321 Author: Micah Elizabeth Scott <beth@torproject.org> AuthorDate: Thu May 25 17:37:52 2023 -0700 hashx: API changes to allow recovery from late compile failures This is an API breaking change to hashx, which modifies the error handling strategy. The main goal here is to allow unproblematic recovery from hashx_compile failures. hashx_alloc can no longer fail for reasons other than memory allocation. All platform-specific compile failures are now reported via hashx_make(), in order to both allow later failure and avoid requiring users of the API to maintain and test multiple failure paths. Note that late failures may be more common in actual use than early failures. Early failures represent architectures other than x86_64 and aarch64. Late failures could represent a number of system configurations where syscalls are restricted. The definition of a hashx context no longer tries to overlay storage for the different types of program, and instead allows one context to always contain an interpretable description of the program as well as an optional buffer for compiled code. The hashx_type enum is now used to mean either a specific type of hash function or a type of hashx context. You can allocate a context for use only with interpreted or compiled functions, or you can use HASHX_TRY_COMPILE to prefer the compiler with an automatic fallback on the interpreter. After calling hashx_make(), the new hashx_query_type() can be used if needed to determine which implementation was actually chosen. The error return types have been overhauled so that everyone uses the hashx_result enum, and seed failures vs compile failures are always clearly distinguishable. Signed-off-by: Micah Elizabeth Scott <beth@torproject.org> --- src/ext/equix/hashx/README.md | 37 ++++++++-- src/ext/equix/hashx/include/hashx.h | 72 ++++++++++++++----- src/ext/equix/hashx/src/bench.c | 34 ++++++--- src/ext/equix/hashx/src/compiler.c | 9 +-- src/ext/equix/hashx/src/compiler.h | 5 +- src/ext/equix/hashx/src/context.c | 45 +++--------- src/ext/equix/hashx/src/context.h | 16 ++--- src/ext/equix/hashx/src/hashx.c | 78 ++++++++++++++------- src/ext/equix/hashx/src/tests.c | 116 ++++++++++++++++++++++--------- src/ext/equix/hashx/src/virtual_memory.c | 3 + 10 files changed, 269 insertions(+), 146 deletions(-) diff --git a/src/ext/equix/hashx/README.md b/src/ext/equix/hashx/README.md index 1d8ea47652..f6352475b0 100644 --- a/src/ext/equix/hashx/README.md +++ b/src/ext/equix/hashx/README.md @@ -13,7 +13,7 @@ and to ensure that each function takes exactly the same number of CPU cycles ## API -The API consists of 4 functions and is documented in the public header file +The API consists of 5 functions and is documented in the public header file [hashx.h](include/hashx.h). Example of usage: @@ -25,13 +25,15 @@ Example of usage: int main() { char seed[] = "this is a seed that will generate a hash function"; char hash[HASHX_SIZE]; - hashx_ctx* ctx = hashx_alloc(HASHX_COMPILED); - if (ctx == HASHX_NOTSUPP) - ctx = hashx_alloc(HASHX_INTERPRETED); + hashx_type func_type; + hashx_ctx* ctx = hashx_alloc(HASHX_TRY_COMPILE); if (ctx == NULL) return 1; - if (!hashx_make(ctx, seed, sizeof(seed))) /* generate a hash function */ + /* generate a hash function */ + if (hashx_make(ctx, seed, sizeof(seed)) != HASHX_OK) return 1; + if (hashx_query_type(ctx, &func_type) == HASHX_OK && func_type == HASHX_TYPE_COMPILED) + printf("Using the compiled implementation of HashX\n"); hashx_exec(ctx, 123456789, hash); /* calculate the hash of a nonce value */ hashx_free(ctx); for (unsigned i = 0; i < HASHX_SIZE; ++i) @@ -84,6 +86,31 @@ A benchmark executable is included: ./hashx-bench --seeds 500 ``` +## Error fallback + +The compiled implementation of HashX is much faster (very roughly 20x) so it +should be used whenever possible. It may be necessary to use the interpreter +for multiple reasons: either the platform is not supported at compile time, +or various runtime policies disallow the memory protection changes that are +necessary to do just-in-time compilation. Failures may be detected late, so +the library provides a built-in mechanism to fall back from the compiled +implementation to interpreted quickly without duplicating the whole context. + +The `hashx_query_type()` function is optional, provided for users of the +`HASHX_TRY_COMPILE` context who need to know which implementation was +ultimately used. + +The actual hash function, `hashx_exec()`, returns an error code for +completeness in reporting programming errors, but if a caller has invoked +`hashx_make()` successfully it can be considered infallible. + +It is always possible for `hashx_make()` to fail. In addition to the +OS-specific failures you may see when forcing HashX to use the compiled +implementation with `hashx_alloc(HASHX_TYPE_COMPILED)`, it's always possible +for `hashx_make` to fail unpredictably for a particular seed value. These +seeds should be discarded and a new one attempted by the caller when +`hashx_make` returns `HASHX_FAIL_SEED`. + ## Security HashX should provide strong preimage resistance. No other security guarantees are made. About diff --git a/src/ext/equix/hashx/include/hashx.h b/src/ext/equix/hashx/include/hashx.h index 0d5521177a..2910515d9a 100644 --- a/src/ext/equix/hashx/include/hashx.h +++ b/src/ext/equix/hashx/include/hashx.h @@ -15,14 +15,13 @@ int main() { char seed[] = "this is a seed that will generate a hash function"; char hash[HASHX_SIZE]; - hashx_ctx* ctx = hashx_alloc(HASHX_COMPILED); - if (ctx == HASHX_NOTSUPP) - ctx = hashx_alloc(HASHX_INTERPRETED); + hashx_ctx* ctx = hashx_alloc(HASHX_TRY_COMPILE); if (ctx == NULL) return 1; - if (!hashx_make(ctx, seed, sizeof(seed))) + if (hashx_make(ctx, seed, sizeof(seed)) != EQUIX_OK) + return 1; + if (hashx_exec(ctx, 123456789, hash) != EQUIX_OK) return 1; - hashx_exec(ctx, 123456789, hash); hashx_free(ctx); for (unsigned i = 0; i < HASHX_SIZE; ++i) printf("%02x", hash[i] & 0xff); @@ -58,14 +57,21 @@ /* Opaque struct representing a HashX instance */ typedef struct hashx_ctx hashx_ctx; -/* Type of hash function */ +/* Type of hash context / type of compiled function */ typedef enum hashx_type { - HASHX_INTERPRETED, - HASHX_COMPILED + HASHX_TYPE_INTERPRETED = 1, /* Only the interpreted implementation */ + HASHX_TYPE_COMPILED, /* Require the compiler, fail if unavailable */ + HASHX_TRY_COMPILE, /* (hashx_alloc) Try compiler, don't require */ } hashx_type; -/* Sentinel value used to indicate unsupported type */ -#define HASHX_NOTSUPP ((hashx_ctx*)-1) +/* Result code for hashx_make and hashx_exec */ +typedef enum hashx_result { + HASHX_OK = 0, + HASHX_FAIL_UNPREPARED, /* Trying to run an unmade hash funciton */ + HASHX_FAIL_UNDEFINED, /* Unrecognized hashx_type enum value */ + HASHX_FAIL_SEED, /* Can't construct a hash function from this seed */ + HASHX_FAIL_COMPILE, /* Can't compile, and no fallback is enabled. */ +} hashx_result; #if defined(_WIN32) || defined(__CYGWIN__) #define HASHX_WIN @@ -100,35 +106,65 @@ extern "C" { * @param type is the type of instance to be created. * * @return pointer to a new HashX instance. Returns NULL on memory allocation - * failure and HASHX_NOTSUPP if the requested type is not supported. -*/ + * failures only. Other failures are reported in hashx_make. + */ HASHX_API hashx_ctx* hashx_alloc(hashx_type type); /* - * Create a new HashX function from seed. + * Create a new HashX function from a variable-length seed value. + * + * The seed value will be hashed internally in order to initialize the state + * of the HashX program generator and create a new unique hash function. * * @param ctx is pointer to a HashX instance. * @param seed is a pointer to the seed value. * @param size is the size of the seed. * - * @return 1 on success, 0 on failure. + * @return HASHX_OK on success, HASHX_FAIL_SEED if the specific seed is + * not associated with a valid hash program, and HASHX_FAIL_COMPILE + * if the compiler failed for OS-specific reasons and the interpreter + * fallback was disabled by allocating the context with + * HASHX_TYPE_COMPILED rather than HASHX_TRY_COMPILE. + */ +HASHX_API hashx_result hashx_make(hashx_ctx* ctx, + const void* seed, size_t size); + +/* + * Asks the specific implementation of a function created with hashx_make. + * + * This will equal the parameter to hashx_alloc() if a specific type was + * chosen there, but a context allocated with HASHX_TRY_COMPILE will allow + * the implementation to vary dynamically during hashx_make. + * + * @param ctx is pointer to a HashX instance. + * @param type_out is a pointer to which, on success, we write + * a HASHX_TYPE_* value. + * + * @return HASHX_OK on success, or HASHX_FAIL_UNPREPARED if hashx_make has not + * been invoked successfully on this context. */ -HASHX_API int hashx_make(hashx_ctx* ctx, const void* seed, size_t size); +HASHX_API hashx_result hashx_query_type(hashx_ctx* ctx, hashx_type *type_out); /* * Execute the HashX function. * * @param ctx is pointer to a HashX instance. A HashX function must have - * been previously created by calling hashx_make. + * been previously created by invoking hashx_make successfully. * @param HASHX_INPUT is the input to be hashed (see definition above). * @param output is a pointer to the result buffer. HASHX_SIZE bytes will be * written. - s*/ -HASHX_API void hashx_exec(const hashx_ctx* ctx, HASHX_INPUT, void* output); + * + * @return HASHX_OK on success, or HASHX_FAIL_UNPREPARED if hashx_make has not + * been invoked successfully on this context. + */ +HASHX_API hashx_result hashx_exec(const hashx_ctx* ctx, + HASHX_INPUT, void* output); /* * Free a HashX instance. * + * Has no effect if ctx is NULL. + * * @param ctx is pointer to a HashX instance. */ HASHX_API void hashx_free(hashx_ctx* ctx); diff --git a/src/ext/equix/hashx/src/bench.c b/src/ext/equix/hashx/src/bench.c index fbcd41a064..a440825436 100644 --- a/src/ext/equix/hashx/src/bench.c +++ b/src/ext/equix/hashx/src/bench.c @@ -5,6 +5,7 @@ #include "hashx_thread.h" #include "hashx_endian.h" #include "hashx_time.h" +#include <assert.h> #include <limits.h> #include <inttypes.h> @@ -26,16 +27,31 @@ static hashx_thread_retval worker(void* args) { job->total_hashes = 0; job->best_hash = UINT64_MAX; for (int seed = job->start; seed < job->end; seed += job->step) { - if (!hashx_make(job->ctx, &seed, sizeof(seed))) { - continue; + { + hashx_result result = hashx_make(job->ctx, &seed, sizeof(seed)); + if (result == HASHX_FAIL_SEED) { + continue; + } + if (result == HASHX_FAIL_COMPILE) { + printf("Error: not supported. Try with --interpret\n"); + } + assert(result == HASHX_OK); + if (result != HASHX_OK) + break; } for (int nonce = 0; nonce < job->nonces; ++nonce) { uint8_t hash[HASHX_SIZE] = { 0 }; + { #ifndef HASHX_BLOCK_MODE - hashx_exec(job->ctx, nonce, hash); + hashx_result result = hashx_exec(job->ctx, nonce, hash); #else - hashx_exec(job->ctx, &nonce, sizeof(nonce), hash); + hashx_result result = hashx_exec(job->ctx, + &nonce, sizeof(nonce), hash); #endif + assert(result == HASHX_OK); + if (result != HASHX_OK) + break; + } uint64_t hashval = load64(hash); if (hashval < job->best_hash) { job->best_hash = hashval; @@ -70,9 +86,9 @@ int main(int argc, char** argv) { read_int_option("--nonces", argc, argv, &nonces, 65536); read_int_option("--threads", argc, argv, &threads, 1); read_option("--interpret", argc, argv, &interpret); - hashx_type flags = HASHX_INTERPRETED; + hashx_type ctx_type = HASHX_TYPE_INTERPRETED; if (!interpret) { - flags = HASHX_COMPILED; + ctx_type = HASHX_TYPE_COMPILED; } uint64_t best_hash = UINT64_MAX; uint64_t diff_ex = (uint64_t)diff * 1000ULL; @@ -88,15 +104,11 @@ int main(int argc, char** argv) { return 1; } for (int thd = 0; thd < threads; ++thd) { - jobs[thd].ctx = hashx_alloc(flags); + jobs[thd].ctx = hashx_alloc(ctx_type); if (jobs[thd].ctx == NULL) { printf("Error: memory allocation failure\n"); return 1; } - if (jobs[thd].ctx == HASHX_NOTSUPP) { - printf("Error: not supported. Try with --interpret\n"); - return 1; - } jobs[thd].id = thd; jobs[thd].start = start + thd; jobs[thd].step = threads; diff --git a/src/ext/equix/hashx/src/compiler.c b/src/ext/equix/hashx/src/compiler.c index f180bf2d25..870f3654b6 100644 --- a/src/ext/equix/hashx/src/compiler.c +++ b/src/ext/equix/hashx/src/compiler.c @@ -8,11 +8,12 @@ #include "program.h" #include "context.h" -bool hashx_compiler_init(hashx_ctx* ctx) { - ctx->code = hashx_vm_alloc(COMP_CODE_SIZE); - return ctx->code != NULL; +void hashx_compiler_init(hashx_ctx* ctx) { + /* This can fail, but it's uncommon. We report this up the call chain + * later, at the same time as an mprotect or similar failure. */ + ctx->compiler_mem = hashx_vm_alloc(COMP_CODE_SIZE); } void hashx_compiler_destroy(hashx_ctx* ctx) { - hashx_vm_free(ctx->code, COMP_CODE_SIZE); + hashx_vm_free(ctx->compiler_mem, COMP_CODE_SIZE); } diff --git a/src/ext/equix/hashx/src/compiler.h b/src/ext/equix/hashx/src/compiler.h index 140f797a58..f248e8c1ea 100644 --- a/src/ext/equix/hashx/src/compiler.h +++ b/src/ext/equix/hashx/src/compiler.h @@ -15,19 +15,16 @@ HASHX_PRIVATE bool hashx_compile_x86(const hashx_program* program, uint8_t* code HASHX_PRIVATE bool hashx_compile_a64(const hashx_program* program, uint8_t* code); #if defined(_M_X64) || defined(__x86_64__) -#define HASHX_COMPILER 1 #define HASHX_COMPILER_X86 #define hashx_compile(p,c) hashx_compile_x86(p,c) #elif defined(__aarch64__) -#define HASHX_COMPILER 1 #define HASHX_COMPILER_A64 #define hashx_compile(p,c) hashx_compile_a64(p,c) #else -#define HASHX_COMPILER 0 #define hashx_compile(p,c) (false) #endif -HASHX_PRIVATE bool hashx_compiler_init(hashx_ctx* compiler); +HASHX_PRIVATE void hashx_compiler_init(hashx_ctx* compiler); HASHX_PRIVATE void hashx_compiler_destroy(hashx_ctx* compiler); #define COMP_PAGE_SIZE 4096 diff --git a/src/ext/equix/hashx/src/context.c b/src/ext/equix/hashx/src/context.c index 8548fb7ffa..03a9de57fd 100644 --- a/src/ext/equix/hashx/src/context.c +++ b/src/ext/equix/hashx/src/context.c @@ -33,50 +33,25 @@ const blake2b_param hashx_blake2_params = { }; hashx_ctx* hashx_alloc(hashx_type type) { - if (!HASHX_COMPILER && (type & HASHX_COMPILED)) { - return HASHX_NOTSUPP; - } hashx_ctx* ctx = malloc(sizeof(hashx_ctx)); - if (ctx == NULL) { - goto failure; - } - ctx->code = NULL; - ctx->type = 0; - if (type & HASHX_COMPILED) { - if (!hashx_compiler_init(ctx)) { - goto failure; - } - ctx->type = HASHX_COMPILED; - } - else { - ctx->program = malloc(sizeof(hashx_program)); - if (ctx->program == NULL) { - goto failure; - } - ctx->type = HASHX_INTERPRETED; + if (ctx == NULL) + return NULL; + + memset(ctx, 0, sizeof *ctx); + ctx->ctx_type = type; + if (type == HASHX_TYPE_COMPILED || type == HASHX_TRY_COMPILE) { + hashx_compiler_init(ctx); } + #ifdef HASHX_BLOCK_MODE memcpy(&ctx->params, &hashx_blake2_params, 32); -#endif -#ifndef NDEBUG - ctx->has_program = false; #endif return ctx; -failure: - hashx_free(ctx); - return NULL; } void hashx_free(hashx_ctx* ctx) { - if (ctx != NULL && ctx != HASHX_NOTSUPP) { - if (ctx->code != NULL) { - if (ctx->type & HASHX_COMPILED) { - hashx_compiler_destroy(ctx); - } - else { - free(ctx->program); - } - } + if (ctx != NULL) { + hashx_compiler_destroy(ctx); free(ctx); } } diff --git a/src/ext/equix/hashx/src/context.h b/src/ext/equix/hashx/src/context.h index 40736397f8..ad434eb66c 100644 --- a/src/ext/equix/hashx/src/context.h +++ b/src/ext/equix/hashx/src/context.h @@ -9,8 +9,7 @@ #include "hashx.h" #include "blake2.h" #include "siphash.h" - -typedef void program_func(uint64_t r[8]); +#include "program.h" #ifdef __cplusplus extern "C" { @@ -26,20 +25,15 @@ typedef struct hashx_program hashx_program; /* HashX context. */ typedef struct hashx_ctx { - union { - uint8_t* code; - program_func* func; - hashx_program* program; - }; - hashx_type type; + uint8_t* compiler_mem; + hashx_type ctx_type; + hashx_type func_type; + hashx_program program; #ifndef HASHX_BLOCK_MODE siphash_state keys; #else blake2b_param params; #endif -#ifndef NDEBUG - bool has_program; -#endif } hashx_ctx; #endif diff --git a/src/ext/equix/hashx/src/hashx.c b/src/ext/equix/hashx/src/hashx.c index 372df09581..36a32fc298 100644 --- a/src/ext/equix/hashx/src/hashx.c +++ b/src/ext/equix/hashx/src/hashx.c @@ -22,25 +22,20 @@ #define HASHX_INPUT_ARGS input, size #endif -static int initialize_program(hashx_ctx* ctx, hashx_program* program, - siphash_state keys[2]) { - - if (!hashx_program_generate(&keys[0], program)) { - return 0; +static bool initialize_program(hashx_ctx* ctx, siphash_state keys[2]) { + if (!hashx_program_generate(&keys[0], &ctx->program)) { + return false; } #ifndef HASHX_BLOCK_MODE memcpy(&ctx->keys, &keys[1], 32); #else memcpy(&ctx->params.salt, &keys[1], 32); #endif -#ifndef NDEBUG - ctx->has_program = true; -#endif - return 1; + return true; } -int hashx_make(hashx_ctx* ctx, const void* seed, size_t size) { - assert(ctx != NULL && ctx != HASHX_NOTSUPP); +hashx_result hashx_make(hashx_ctx* ctx, const void* seed, size_t size) { + assert(ctx != NULL); assert(seed != NULL || size == 0); uint8_t keys_bytes[2 * sizeof(siphash_state)]; @@ -59,23 +54,48 @@ int hashx_make(hashx_ctx* ctx, const void* seed, size_t size) { keys[1].v2 = load64(keys_bytes + 6 * sizeof(uint64_t)); keys[1].v3 = load64(keys_bytes + 7 * sizeof(uint64_t)); - if (ctx->type & HASHX_COMPILED) { - hashx_program program; - if (!initialize_program(ctx, &program, keys)) { - return 0; + ctx->func_type = (hashx_type)0; + if (!initialize_program(ctx, keys)) { + return HASHX_FAIL_SEED; + } + + switch (ctx->ctx_type) { + case HASHX_TYPE_INTERPRETED: + ctx->func_type = HASHX_TYPE_INTERPRETED; + return HASHX_OK; + case HASHX_TYPE_COMPILED: + case HASHX_TRY_COMPILE: + if (ctx->compiler_mem != NULL && + hashx_compile(&ctx->program, ctx->compiler_mem)) { + ctx->func_type = HASHX_TYPE_COMPILED; + return HASHX_OK; } - if (!hashx_compile(&program, ctx->code)) { - return 0; + if (ctx->ctx_type == HASHX_TRY_COMPILE) { + ctx->func_type = HASHX_TYPE_INTERPRETED; + return HASHX_OK; + } else { + return HASHX_FAIL_COMPILE; } - return 1; + default: + return HASHX_FAIL_UNDEFINED; + } +} + +hashx_result hashx_query_type(hashx_ctx* ctx, hashx_type *type_out) { + assert(ctx != NULL); + assert(type_out != NULL); + + if (ctx->func_type == (hashx_type)0) { + return HASHX_FAIL_UNPREPARED; } - return initialize_program(ctx, ctx->program, keys); + *type_out = ctx->func_type; + return HASHX_OK; } -void hashx_exec(const hashx_ctx* ctx, HASHX_INPUT, void* output) { - assert(ctx != NULL && ctx != HASHX_NOTSUPP); +hashx_result hashx_exec(const hashx_ctx* ctx, HASHX_INPUT, void* output) { + assert(ctx != NULL); assert(output != NULL); - assert(ctx->has_program); + uint64_t r[8]; #ifndef HASHX_BLOCK_MODE hashx_siphash24_ctr_state512(&ctx->keys, input, r); @@ -83,11 +103,14 @@ void hashx_exec(const hashx_ctx* ctx, HASHX_INPUT, void* output) { hashx_blake2b_4r(&ctx->params, input, size, r); #endif - if (ctx->type & HASHX_COMPILED) { - ctx->func(r); - } - else { - hashx_program_execute(ctx->program, r); + if (ctx->func_type == HASHX_TYPE_COMPILED) { + typedef void program_func(uint64_t r[8]); + assert(ctx->compiler_mem != NULL); + ((program_func*)ctx->compiler_mem)(r); + } else if (ctx->func_type == HASHX_TYPE_INTERPRETED) { + hashx_program_execute(&ctx->program, r); + } else { + return HASHX_FAIL_UNPREPARED; } /* Hash finalization to remove bias toward 0 caused by multiplications */ @@ -145,4 +168,5 @@ void hashx_exec(const hashx_ctx* ctx, HASHX_INPUT, void* output) { memcpy(output, temp_out, HASHX_SIZE); #endif #endif + return HASHX_OK; } diff --git a/src/ext/equix/hashx/src/tests.c b/src/ext/equix/hashx/src/tests.c index e1569844ac..f0a4ebe713 100644 --- a/src/ext/equix/hashx/src/tests.c +++ b/src/ext/equix/hashx/src/tests.c @@ -14,6 +14,7 @@ static int test_no = 0; static hashx_ctx* ctx_int = NULL; static hashx_ctx* ctx_cmp = NULL; +static hashx_ctx* ctx_auto = NULL; static const char seed1[] = "This is a test"; static const char seed2[] = "Lorem ipsum dolor sit amet"; @@ -42,20 +43,21 @@ static void run_test(const char* name, test_func* func) { } static bool test_alloc() { - ctx_int = hashx_alloc(HASHX_INTERPRETED); - assert(ctx_int != NULL && ctx_int != HASHX_NOTSUPP); + ctx_int = hashx_alloc(HASHX_TYPE_INTERPRETED); + assert(ctx_int != NULL); return true; } static bool test_free() { hashx_free(ctx_int); hashx_free(ctx_cmp); + hashx_free(ctx_auto); return true; } static bool test_make1() { - int result = hashx_make(ctx_int, seed1, sizeof(seed1)); - assert(result == 1); + hashx_result result = hashx_make(ctx_int, seed1, sizeof(seed1)); + assert(result == HASHX_OK); return true; } @@ -65,7 +67,8 @@ static bool test_hash_ctr1() { #endif #ifndef HASHX_BLOCK_MODE char hash[HASHX_SIZE]; - hashx_exec(ctx_int, counter2, hash); + hashx_result result = hashx_exec(ctx_int, counter2, hash); + assert(result == HASHX_OK); /* printf("\n"); output_hex(hash, HASHX_SIZE); printf("\n"); */ @@ -82,7 +85,8 @@ static bool test_hash_ctr2() { #endif #ifndef HASHX_BLOCK_MODE char hash[HASHX_SIZE]; - hashx_exec(ctx_int, counter1, hash); + hashx_result result = hashx_exec(ctx_int, counter1, hash); + assert(result == HASHX_OK); assert(equals_hex(hash, "2b2f54567dcbea98fdb5d5e5ce9a65983c4a4e35ab1464b1efb61e83b7074bb2")); return true; #else @@ -91,8 +95,8 @@ static bool test_hash_ctr2() { } static bool test_make2() { - int result = hashx_make(ctx_int, seed2, sizeof(seed2)); - assert(result == 1); + hashx_result result = hashx_make(ctx_int, seed2, sizeof(seed2)); + assert(result == HASHX_OK); return true; } @@ -102,7 +106,8 @@ static bool test_hash_ctr3() { #endif #ifndef HASHX_BLOCK_MODE char hash[HASHX_SIZE]; - hashx_exec(ctx_int, counter2, hash); + hashx_result result = hashx_exec(ctx_int, counter2, hash); + assert(result == HASHX_OK); assert(equals_hex(hash, "ab3d155bf4bbb0aa3a71b7801089826186e44300e6932e6ffd287cf302bbb0ba")); return true; #else @@ -116,7 +121,8 @@ static bool test_hash_ctr4() { #endif #ifndef HASHX_BLOCK_MODE char hash[HASHX_SIZE]; - hashx_exec(ctx_int, counter3, hash); + hashx_result result = hashx_exec(ctx_int, counter3, hash); + assert(result == HASHX_OK); assert(equals_hex(hash, "8dfef0497c323274a60d1d93292b68d9a0496379ba407b4341cf868a14d30113")); return true; #else @@ -132,36 +138,40 @@ static bool test_hash_block1() { return false; #else char hash[HASHX_SIZE]; - hashx_exec(ctx_int, long_input, sizeof(long_input), hash); + hashx_result result = hashx_exec(ctx_int, long_input, sizeof(long_input), hash); + assert(result == HASHX_OK); assert(equals_hex(hash, "d0b232b832459501ca1ac9dc0429fd931414ead7624a457e375a43ea3e5e737a")); return true; #endif } static bool test_alloc_compiler() { - ctx_cmp = hashx_alloc(HASHX_COMPILED); + ctx_cmp = hashx_alloc(HASHX_TYPE_COMPILED); assert(ctx_cmp != NULL); - return ctx_cmp != HASHX_NOTSUPP; + return true; } static bool test_make3() { - if (ctx_cmp == HASHX_NOTSUPP) + hashx_result result = hashx_make(ctx_cmp, seed2, sizeof(seed2)); + if (result == HASHX_FAIL_COMPILE) { return false; - - int result = hashx_make(ctx_cmp, seed2, sizeof(seed2)); - assert(result == 1); + } + assert(result == HASHX_OK); return true; } static bool test_compiler_ctr1() { - if (ctx_cmp == HASHX_NOTSUPP) - return false; - #ifndef HASHX_BLOCK_MODE + hashx_result result; char hash1[HASHX_SIZE]; char hash2[HASHX_SIZE]; - hashx_exec(ctx_int, counter2, hash1); - hashx_exec(ctx_cmp, counter2, hash2); + result = hashx_exec(ctx_int, counter2, hash1); + assert(result == HASHX_OK); + result = hashx_exec(ctx_cmp, counter2, hash2); + if (result == HASHX_FAIL_UNPREPARED) { + return false; + } + assert(result == HASHX_OK); assert(hashes_equal(hash1, hash2)); return true; #else @@ -170,14 +180,17 @@ static bool test_compiler_ctr1() { } static bool test_compiler_ctr2() { - if (ctx_cmp == HASHX_NOTSUPP) - return false; - #ifndef HASHX_BLOCK_MODE + hashx_result result; char hash1[HASHX_SIZE]; char hash2[HASHX_SIZE]; - hashx_exec(ctx_int, counter1, hash1); - hashx_exec(ctx_cmp, counter1, hash2); + result = hashx_exec(ctx_int, counter1, hash1); + assert(result == HASHX_OK); + result = hashx_exec(ctx_cmp, counter1, hash2); + if (result == HASHX_FAIL_UNPREPARED) { + return false; + } + assert(result == HASHX_OK); assert(hashes_equal(hash1, hash2)); return true; #else @@ -186,20 +199,58 @@ static bool test_compiler_ctr2() { } static bool test_compiler_block1() { - if (ctx_cmp == HASHX_NOTSUPP) - return false; #ifndef HASHX_BLOCK_MODE return false; #else + hashx_result result; char hash1[HASHX_SIZE]; char hash2[HASHX_SIZE]; - hashx_exec(ctx_int, long_input, sizeof(long_input), hash1); - hashx_exec(ctx_cmp, long_input, sizeof(long_input), hash2); + result = hashx_exec(ctx_int, long_input, sizeof(long_input), hash1); + assert(result == HASHX_OK); + result = hashx_exec(ctx_cmp, long_input, sizeof(long_input), hash2); + if (result == HASHX_FAIL_UNPREPARED) { + return false; + } + assert(result == HASHX_OK); assert(hashes_equal(hash1, hash2)); return true; #endif } +static bool test_alloc_automatic() { + ctx_auto = hashx_alloc(HASHX_TRY_COMPILE); + assert(ctx_auto != NULL); + return true; +} + +static bool test_auto_fallback() { + hashx_result result = hashx_make(ctx_auto, seed2, sizeof(seed2)); + assert(result == HASHX_OK); + hashx_type actual_type = (hashx_type)-1; + result = hashx_query_type(ctx_auto, &actual_type); + assert(result == HASHX_OK); + assert(actual_type == HASHX_TYPE_INTERPRETED || + actual_type == HASHX_TYPE_COMPILED); + return actual_type == HASHX_TYPE_INTERPRETED; +} + +static bool test_bad_seeds() { +#ifdef HASHX_SALT + return false; +#else + hashx_result result; + result = hashx_make(ctx_auto, "\xf8\x05\x00\x00", 4); + assert(result == HASHX_OK); + result = hashx_make(ctx_auto, "\xf9\x05\x00\x00", 4); + assert(result == HASHX_FAIL_SEED); + result = hashx_make(ctx_auto, "\x5d\x93\x02\x00", 4); + assert(result == HASHX_FAIL_SEED); + result = hashx_make(ctx_auto, "\x5e\x93\x02\x00", 4); + assert(result == HASHX_OK); + return true; +#endif +} + int main() { RUN_TEST(test_alloc); RUN_TEST(test_make1); @@ -214,6 +265,9 @@ int main() { RUN_TEST(test_compiler_ctr2); RUN_TEST(test_hash_block1); RUN_TEST(test_compiler_block1); + RUN_TEST(test_alloc_automatic); + RUN_TEST(test_auto_fallback); + RUN_TEST(test_bad_seeds); RUN_TEST(test_free); printf("\nAll tests were successful\n"); diff --git a/src/ext/equix/hashx/src/virtual_memory.c b/src/ext/equix/hashx/src/virtual_memory.c index 8d574740c5..e9df825c9f 100644 --- a/src/ext/equix/hashx/src/virtual_memory.c +++ b/src/ext/equix/hashx/src/virtual_memory.c @@ -120,6 +120,9 @@ void* hashx_vm_alloc_huge(size_t bytes) { } void hashx_vm_free(void* ptr, size_t bytes) { + if (!ptr) { + return; + } #ifdef HASHX_WIN (void)bytes; VirtualFree(ptr, 0, MEM_RELEASE); -- To stop receiving notification emails like this one, please contact the administrator of this repository.
This is an automated email from the git hooks/post-receive script. dgoulet pushed a commit to branch main in repository tor. commit a3513dea54c1de6da4f5224624f545cd0d527891 Author: Micah Elizabeth Scott <beth@torproject.org> AuthorDate: Sun May 28 14:05:45 2023 -0700 equix: API changes for new result codes and hashx compatibility This change adapts Equi-X to the corresponding HashX API changes that added HASHX_TRY_COMPILE. The new regularized HashX return codes are reflected by revised corresponding Equi-X return codes. Both solve and verify operations now return an error/success code, and a new equix_solutions_buffer struct includes both the solution buffer and information about the solution count and hashx implementation. With this change, it's possible to discern between hash construction failures (invalid seed) and some external error like an mprotect() failure. Signed-off-by: Micah Elizabeth Scott <beth@torproject.org> --- src/ext/equix/include/equix.h | 57 ++++++++++++++++++++++++----------- src/ext/equix/src/bench.c | 44 +++++++++++++++------------ src/ext/equix/src/context.c | 25 +++++++++------- src/ext/equix/src/equix.c | 70 +++++++++++++++++++++++++++++++------------ src/ext/equix/src/solver.c | 3 +- src/ext/equix/src/tests.c | 61 ++++++++++++++++++++----------------- 6 files changed, 165 insertions(+), 95 deletions(-) diff --git a/src/ext/equix/include/equix.h b/src/ext/equix/include/equix.h index 01ab249437..75b25a4d53 100644 --- a/src/ext/equix/include/equix.h +++ b/src/ext/equix/include/equix.h @@ -30,16 +30,35 @@ typedef struct equix_solution { } equix_solution; /* - * Solution verification results + * Extra informational flags returned by the solver + */ +typedef enum equix_solution_flags { + EQUIX_SOLVER_DID_USE_COMPILER = (1 << 0), +} equix_solution_flags; + +/* + * Fixed size buffer containing up to EQUIX_MAX_SOLS solutions. + */ +typedef struct equix_solutions_buffer { + unsigned count; + equix_solution_flags flags; + equix_solution sols[EQUIX_MAX_SOLS]; +} equix_solutions_buffer; + +/* + * Result type for solve and verify operations */ typedef enum equix_result { EQUIX_OK, /* Solution is valid */ - EQUIX_CHALLENGE, /* The challenge is invalid (the internal hash + EQUIX_FAIL_CHALLENGE, /* The challenge is invalid (the internal hash function doesn't pass validation). */ - EQUIX_ORDER, /* Indices are not in the correct order. */ - EQUIX_PARTIAL_SUM, /* The partial sums of the hash values don't + EQUIX_FAIL_ORDER, /* Indices are not in the correct order. */ + EQUIX_FAIL_PARTIAL_SUM, /* The partial sums of the hash values don't have the required number of trailing zeroes. */ - EQUIX_FINAL_SUM /* The hash values don't sum to zero. */ + EQUIX_FAIL_FINAL_SUM, /* The hash values don't sum to zero. */ + EQUIX_FAIL_COMPILE, /* Can't compile, and no fallback is enabled */ + EQUIX_FAIL_NO_SOLVER, /* Solve requested on a context with no solver */ + EQUIX_FAIL_INTERNAL, /* Internal error (bug) */ } equix_result; /* @@ -49,17 +68,15 @@ typedef struct equix_ctx equix_ctx; /* * Flags for context creation -*/ + */ typedef enum equix_ctx_flags { EQUIX_CTX_VERIFY = 0, /* Context for verification */ EQUIX_CTX_SOLVE = 1, /* Context for solving */ - EQUIX_CTX_COMPILE = 2, /* Compile internal hash function */ - EQUIX_CTX_HUGEPAGES = 4, /* Allocate solver memory using HugePages */ + EQUIX_CTX_MUST_COMPILE = 2, /* Must compile internal hash function */ + EQUIX_CTX_TRY_COMPILE = 4, /* Compile if possible */ + EQUIX_CTX_HUGEPAGES = 8, /* Allocate solver memory using HugePages */ } equix_ctx_flags; -/* Sentinel value used to indicate unsupported type */ -#define EQUIX_NOTSUPP ((equix_ctx*)-1) - #if defined(_WIN32) || defined(__CYGWIN__) #define EQUIX_WIN #endif @@ -93,8 +110,7 @@ extern "C" { * @param flags is the type of context to be created * * @return pointer to a newly created context. Returns NULL on memory - * allocation failure and EQUIX_NOTSUPP if the requested type - * is not supported. + * allocation failure. */ EQUIX_API equix_ctx* equix_alloc(equix_ctx_flags flags); @@ -114,13 +130,17 @@ EQUIX_API void equix_free(equix_ctx* ctx); * @param output pointer to the output array where solutions will be * stored * - * @return the number of solutions found + * @return On success, returns EQUIX_OK and sets output->count to the number + * of solutions found, with the solutions themselves written to the + * output buffer. If the challenge is unusable, returns + * EQUIX_FAIL_CHALLENGE. If the EQUIX_CTX_MUST_COMPILE flag is in use + * and the compiler fails, this can return EQUIX_FAIL_COMPILE. */ -EQUIX_API int equix_solve( +EQUIX_API equix_result equix_solve( equix_ctx* ctx, const void* challenge, size_t challenge_size, - equix_solution output[EQUIX_MAX_SOLS]); + equix_solutions_buffer *output); /* * Verify an Equi-X solution. @@ -130,8 +150,9 @@ EQUIX_API int equix_solve( * @param challenge_size size of the challenge * @param solution pointer to the solution to be verified * - * @return verification result -*/ + * @return Verification result. This can return EQUIX_OK or any of the + * EQUIX_FAIL_* error codes. + */ EQUIX_API equix_result equix_verify( equix_ctx* ctx, const void* challenge, diff --git a/src/ext/equix/src/bench.c b/src/ext/equix/src/bench.c index e5b925c3d2..8d3c82e855 100644 --- a/src/ext/equix/src/bench.c +++ b/src/ext/equix/src/bench.c @@ -10,11 +10,6 @@ #include <hashx_thread.h> #include <hashx_time.h> -typedef struct solver_output { - equix_solution sols[EQUIX_MAX_SOLS]; - int count; -} solver_output; - typedef struct worker_job { int id; hashx_thread thread; @@ -23,17 +18,29 @@ typedef struct worker_job { int start; int step; int end; - solver_output* output; + equix_solutions_buffer* output; } worker_job; static hashx_thread_retval worker(void* args) { worker_job* job = (worker_job*)args; job->total_sols = 0; - solver_output* outptr = job->output; + equix_solutions_buffer* outptr = job->output; for (int seed = job->start; seed < job->end; seed += job->step) { - int count = equix_solve(job->ctx, &seed, sizeof(seed), outptr->sols); - outptr->count = count; - job->total_sols += count; + equix_result result = equix_solve(job->ctx, &seed, + sizeof(seed), outptr); + if (result == EQUIX_OK) { + job->total_sols += outptr->count; + } else if (result == EQUIX_FAIL_CHALLENGE) { + outptr->count = 0; + } else if (result == EQUIX_FAIL_COMPILE) { + printf("Error: not supported. Try with --interpret\n"); + exit(1); + break; + } else { + printf("Error: unexpected solve failure (%d)\n", (int)result); + exit(1); + break; + } outptr++; } return HASHX_THREAD_SUCCESS; @@ -54,7 +61,10 @@ static const char* result_names[] = { "Invalid nonce", "Indices out of order", "Nonzero partial sum", - "Nonzero final sum" + "Nonzero final sum", + "HashX compiler failed", + "(Internal) Solver not allocated", + "(Internal error)" }; static void print_help(char* executable) { @@ -85,7 +95,7 @@ int main(int argc, char** argv) { read_int_option("--threads", argc, argv, &threads, 1); equix_ctx_flags flags = EQUIX_CTX_SOLVE; if (!interpret) { - flags |= EQUIX_CTX_COMPILE; + flags |= EQUIX_CTX_MUST_COMPILE; } if (huge_pages) { flags |= EQUIX_CTX_HUGEPAGES; @@ -102,15 +112,11 @@ int main(int argc, char** argv) { printf("Error: memory allocation failure\n"); return 1; } - if (jobs[thd].ctx == EQUIX_NOTSUPP) { - printf("Error: not supported. Try with --interpret\n"); - return 1; - } jobs[thd].id = thd; jobs[thd].start = start + thd; jobs[thd].step = threads; jobs[thd].end = start + nonces; - jobs[thd].output = malloc(sizeof(solver_output) * per_thread); + jobs[thd].output = malloc(sizeof(equix_solutions_buffer) * per_thread); if (jobs[thd].output == NULL) { printf("Error: memory allocation failure\n"); return 1; @@ -141,7 +147,7 @@ int main(int argc, char** argv) { if (print_sols) { for (int thd = 0; thd < threads; ++thd) { worker_job* job = &jobs[thd]; - solver_output* outptr = job->output; + equix_solutions_buffer* outptr = job->output; for (int seed = job->start; seed < job->end; seed += job->step) { for (int sol = 0; sol < outptr->count; ++sol) { print_solution(seed, &outptr->sols[sol]); @@ -153,7 +159,7 @@ int main(int argc, char** argv) { time_start = hashx_time(); for (int thd = 0; thd < threads; ++thd) { worker_job* job = &jobs[thd]; - solver_output* outptr = job->output; + equix_solutions_buffer* outptr = job->output; for (int seed = job->start; seed < job->end; seed += job->step) { for (int sol = 0; sol < outptr->count; ++sol) { equix_result result = equix_verify(job->ctx, &seed, sizeof(seed), &outptr->sols[sol]); diff --git a/src/ext/equix/src/context.c b/src/ext/equix/src/context.c index b0aa2d40e5..28edf5e104 100644 --- a/src/ext/equix/src/context.c +++ b/src/ext/equix/src/context.c @@ -8,21 +8,23 @@ #include "solver_heap.h" equix_ctx* equix_alloc(equix_ctx_flags flags) { - equix_ctx* ctx_failure = NULL; equix_ctx* ctx = malloc(sizeof(equix_ctx)); if (ctx == NULL) { goto failure; } - ctx->flags = flags & EQUIX_CTX_COMPILE; - ctx->hash_func = hashx_alloc(flags & EQUIX_CTX_COMPILE ? - HASHX_COMPILED : HASHX_INTERPRETED); - if (ctx->hash_func == NULL) { - goto failure; + ctx->flags = (equix_ctx_flags)0; + + if (flags & EQUIX_CTX_MUST_COMPILE) { + ctx->hash_func = hashx_alloc(HASHX_TYPE_COMPILED); + } else if (flags & EQUIX_CTX_TRY_COMPILE) { + ctx->hash_func = hashx_alloc(HASHX_TRY_COMPILE); + } else { + ctx->hash_func = hashx_alloc(HASHX_TYPE_INTERPRETED); } - if (ctx->hash_func == HASHX_NOTSUPP) { - ctx_failure = EQUIX_NOTSUPP; + if (ctx->hash_func == NULL) { goto failure; } + if (flags & EQUIX_CTX_SOLVE) { if (flags & EQUIX_CTX_HUGEPAGES) { ctx->heap = hashx_vm_alloc_huge(sizeof(solver_heap)); @@ -33,16 +35,19 @@ equix_ctx* equix_alloc(equix_ctx_flags flags) { if (ctx->heap == NULL) { goto failure; } + } else { + ctx->heap = NULL; } + ctx->flags = flags; return ctx; failure: equix_free(ctx); - return ctx_failure; + return NULL; } void equix_free(equix_ctx* ctx) { - if (ctx != NULL && ctx != EQUIX_NOTSUPP) { + if (ctx != NULL) { if (ctx->flags & EQUIX_CTX_SOLVE) { if (ctx->flags & EQUIX_CTX_HUGEPAGES) { hashx_vm_free(ctx->heap, sizeof(solver_heap)); diff --git a/src/ext/equix/src/equix.c b/src/ext/equix/src/equix.c index 5b314ba6ac..a254261509 100644 --- a/src/ext/equix/src/equix.c +++ b/src/ext/equix/src/equix.c @@ -4,6 +4,7 @@ #include <stdlib.h> #include <stdbool.h> #include <string.h> +#include <assert.h> #include <equix.h> #include <hashx.h> @@ -25,60 +26,88 @@ static bool verify_order(const equix_solution* solution) { static uint64_t sum_pair(hashx_ctx* hash_func, equix_idx left, equix_idx right) { uint8_t hash_left[HASHX_SIZE]; uint8_t hash_right[HASHX_SIZE]; - hashx_exec(hash_func, left, hash_left); - hashx_exec(hash_func, right, hash_right); + hashx_result r_left = hashx_exec(hash_func, left, hash_left); + hashx_result r_right = hashx_exec(hash_func, right, hash_right); + assert(r_left == HASHX_OK && r_right == HASHX_OK); return load64(hash_left) + load64(hash_right); } static equix_result verify_internal(hashx_ctx* hash_func, const equix_solution* solution) { uint64_t pair0 = sum_pair(hash_func, solution->idx[0], solution->idx[1]); if (pair0 & EQUIX_STAGE1_MASK) { - return EQUIX_PARTIAL_SUM; + return EQUIX_FAIL_PARTIAL_SUM; } uint64_t pair1 = sum_pair(hash_func, solution->idx[2], solution->idx[3]); if (pair1 & EQUIX_STAGE1_MASK) { - return EQUIX_PARTIAL_SUM; + return EQUIX_FAIL_PARTIAL_SUM; } uint64_t pair4 = pair0 + pair1; if (pair4 & EQUIX_STAGE2_MASK) { - return EQUIX_PARTIAL_SUM; + return EQUIX_FAIL_PARTIAL_SUM; } uint64_t pair2 = sum_pair(hash_func, solution->idx[4], solution->idx[5]); if (pair2 & EQUIX_STAGE1_MASK) { - return EQUIX_PARTIAL_SUM; + return EQUIX_FAIL_PARTIAL_SUM; } uint64_t pair3 = sum_pair(hash_func, solution->idx[6], solution->idx[7]); if (pair3 & EQUIX_STAGE1_MASK) { - return EQUIX_PARTIAL_SUM; + return EQUIX_FAIL_PARTIAL_SUM; } uint64_t pair5 = pair2 + pair3; if (pair5 & EQUIX_STAGE2_MASK) { - return EQUIX_PARTIAL_SUM; + return EQUIX_FAIL_PARTIAL_SUM; } uint64_t pair6 = pair4 + pair5; if (pair6 & EQUIX_FULL_MASK) { - return EQUIX_FINAL_SUM; + return EQUIX_FAIL_FINAL_SUM; } return EQUIX_OK; } -int equix_solve( +static equix_result equix_hashx_make( + equix_ctx* ctx, + const void* challenge, + size_t challenge_size) +{ + switch (hashx_make(ctx->hash_func, challenge, challenge_size)) { + case HASHX_OK: + return EQUIX_OK; + case HASHX_FAIL_SEED: + return EQUIX_FAIL_CHALLENGE; + case HASHX_FAIL_COMPILE: + return EQUIX_FAIL_COMPILE; + case HASHX_FAIL_UNDEFINED: + case HASHX_FAIL_UNPREPARED: + default: + return EQUIX_FAIL_INTERNAL; + } +} + +equix_result equix_solve( equix_ctx* ctx, const void* challenge, size_t challenge_size, - equix_solution output[EQUIX_MAX_SOLS]) + equix_solutions_buffer *output) { if ((ctx->flags & EQUIX_CTX_SOLVE) == 0) { - return 0; + return EQUIX_FAIL_NO_SOLVER; } - if (!hashx_make(ctx->hash_func, challenge, challenge_size)) { - return 0; + equix_result result = equix_hashx_make(ctx, challenge, challenge_size); + if (result != EQUIX_OK) { + return result; } - return equix_solver_solve(ctx->hash_func, ctx->heap, output); -} + output->flags = 0; + hashx_type func_type; + if (hashx_query_type(ctx->hash_func, &func_type) == HASHX_OK && + func_type == HASHX_TYPE_COMPILED) { + output->flags |= EQUIX_SOLVER_DID_USE_COMPILER; + } + output->count = equix_solver_solve(ctx->hash_func, ctx->heap, output->sols); + return EQUIX_OK; +} equix_result equix_verify( equix_ctx* ctx, @@ -87,10 +116,13 @@ equix_result equix_verify( const equix_solution* solution) { if (!verify_order(solution)) { - return EQUIX_ORDER; + return EQUIX_FAIL_ORDER; } - if (!hashx_make(ctx->hash_func, challenge, challenge_size)) { - return EQUIX_CHALLENGE; + + equix_result result = equix_hashx_make(ctx, challenge, challenge_size); + if (result != EQUIX_OK) { + return result; } + return verify_internal(ctx->hash_func, solution); } diff --git a/src/ext/equix/src/solver.c b/src/ext/equix/src/solver.c index 6824b59cc4..1beda06c74 100644 --- a/src/ext/equix/src/solver.c +++ b/src/ext/equix/src/solver.c @@ -49,7 +49,8 @@ typedef stage3_idx_item s3_idx; static FORCE_INLINE uint64_t hash_value(hashx_ctx* hash_func, equix_idx index) { char hash[HASHX_SIZE]; - hashx_exec(hash_func, index, hash); + hashx_result result = hashx_exec(hash_func, index, hash); + assert(result == HASHX_OK); return load64(hash); } diff --git a/src/ext/equix/src/tests.c b/src/ext/equix/src/tests.c index 63fb5bdb1e..75937a7995 100644 --- a/src/ext/equix/src/tests.c +++ b/src/ext/equix/src/tests.c @@ -13,7 +13,7 @@ typedef bool test_func(); static equix_ctx* ctx = NULL; -static equix_solution solution[EQUIX_MAX_SOLS]; +static equix_solutions_buffer output; static int nonce; static int valid_count = 0; static int test_no = 0; @@ -26,8 +26,8 @@ static int test_no = 0; } while(0) static bool test_alloc() { - ctx = equix_alloc(EQUIX_CTX_SOLVE); - assert(ctx != NULL && ctx != EQUIX_NOTSUPP); + ctx = equix_alloc(EQUIX_CTX_SOLVE | EQUIX_CTX_TRY_COMPILE); + assert(ctx != NULL); return true; } @@ -37,60 +37,65 @@ static bool test_free() { } static bool test_solve() { - int num_solutions = 0; - for (nonce = 0; num_solutions == 0 && nonce < 20; ++nonce) { - num_solutions = equix_solve(ctx, &nonce, sizeof(nonce), solution); + output.count = 0; + for (nonce = 0; output.count == 0 && nonce < 20; ++nonce) { + equix_result result = equix_solve(ctx, &nonce, sizeof(nonce), &output); + assert(result == EQUIX_OK); } --nonce; - assert(num_solutions > 0); + assert(output.count > 0); + assert(output.flags == EQUIX_SOLVER_DID_USE_COMPILER || output.flags == 0); + printf("(using %s HashX) ", + (EQUIX_SOLVER_DID_USE_COMPILER & output.flags) + ? "compiled" : "interpreted"); return true; } static bool test_verify1() { - equix_result result = equix_verify(ctx, &nonce, sizeof(nonce), &solution[0]); + equix_result result = equix_verify(ctx, &nonce, sizeof(nonce), &output.sols[0]); assert(result == EQUIX_OK); return true; } static bool test_verify2() { - SWAP_IDX(solution[0].idx[0], solution[0].idx[1]); - equix_result result = equix_verify(ctx, &nonce, sizeof(nonce), &solution[0]); - assert(result == EQUIX_ORDER); + SWAP_IDX(output.sols[0].idx[0], output.sols[0].idx[1]); + equix_result result = equix_verify(ctx, &nonce, sizeof(nonce), &output.sols[0]); + assert(result == EQUIX_FAIL_ORDER); return true; } static bool test_verify3() { - SWAP_IDX(solution[0].idx[0], solution[0].idx[4]); - SWAP_IDX(solution[0].idx[1], solution[0].idx[5]); - SWAP_IDX(solution[0].idx[2], solution[0].idx[6]); - SWAP_IDX(solution[0].idx[3], solution[0].idx[7]); - equix_result result = equix_verify(ctx, &nonce, sizeof(nonce), &solution[0]); - assert(result == EQUIX_ORDER); - SWAP_IDX(solution[0].idx[0], solution[0].idx[4]); - SWAP_IDX(solution[0].idx[1], solution[0].idx[5]); - SWAP_IDX(solution[0].idx[2], solution[0].idx[6]); - SWAP_IDX(solution[0].idx[3], solution[0].idx[7]); + SWAP_IDX(output.sols[0].idx[0], output.sols[0].idx[4]); + SWAP_IDX(output.sols[0].idx[1], output.sols[0].idx[5]); + SWAP_IDX(output.sols[0].idx[2], output.sols[0].idx[6]); + SWAP_IDX(output.sols[0].idx[3], output.sols[0].idx[7]); + equix_result result = equix_verify(ctx, &nonce, sizeof(nonce), &output.sols[0]); + assert(result == EQUIX_FAIL_ORDER); + SWAP_IDX(output.sols[0].idx[0], output.sols[0].idx[4]); + SWAP_IDX(output.sols[0].idx[1], output.sols[0].idx[5]); + SWAP_IDX(output.sols[0].idx[2], output.sols[0].idx[6]); + SWAP_IDX(output.sols[0].idx[3], output.sols[0].idx[7]); return true; } static bool test_verify4() { - SWAP_IDX(solution[0].idx[1], solution[0].idx[2]); - equix_result result = equix_verify(ctx, &nonce, sizeof(nonce), &solution[0]); - assert(result == EQUIX_PARTIAL_SUM); - SWAP_IDX(solution[0].idx[1], solution[0].idx[2]); + SWAP_IDX(output.sols[0].idx[1], output.sols[0].idx[2]); + equix_result result = equix_verify(ctx, &nonce, sizeof(nonce), &output.sols[0]); + assert(result == EQUIX_FAIL_PARTIAL_SUM); + SWAP_IDX(output.sols[0].idx[1], output.sols[0].idx[2]); return true; } static void permute_idx(int start) { if (start == EQUIX_NUM_IDX - 1) { - equix_result result = equix_verify(ctx, &nonce, sizeof(nonce), &solution[0]); + equix_result result = equix_verify(ctx, &nonce, sizeof(nonce), &output.sols[0]); valid_count += result == EQUIX_OK; } else { for (int i = start; i < EQUIX_NUM_IDX; ++i) { - SWAP_IDX(solution[0].idx[start], solution[0].idx[i]); + SWAP_IDX(output.sols[0].idx[start], output.sols[0].idx[i]); permute_idx(start + 1); - SWAP_IDX(solution[0].idx[start], solution[0].idx[i]); + SWAP_IDX(output.sols[0].idx[start], output.sols[0].idx[i]); } } } -- To stop receiving notification emails like this one, please contact the administrator of this repository.
This is an automated email from the git hooks/post-receive script. dgoulet pushed a commit to branch main in repository tor. commit a397a92be2032e781479fa4d53a04f9b369ea1ac Author: Micah Elizabeth Scott <beth@torproject.org> AuthorDate: Sun May 28 16:35:31 2023 -0700 hs_pow: Update for equix API to fix issue 40794 This change adapts the hs_pow layer and unit tests to API changes in hashx and equix which modify the fault recovery responsibilities and reporting behaivor. This and the corresponding implementation changes in hashx and equix form the fix for #40794, both solving the segfault and giving hashx a way to report those failures up the call chain without them being mistaken for a different error (unusable seed) that would warrant a retry. To handle these new late compiler failures with a minimum of fuss or inefficiency, the failover is delegated to the internals of hashx and tor needs only pass in a EQUIX_CTX_TRY_COMPILE flag to get the behavior that tor was previously responsible for implementing. Signed-off-by: Micah Elizabeth Scott <beth@torproject.org> --- src/feature/hs/hs_pow.c | 110 +++++++++++++++++++++++++++----------------- src/test/test_crypto.c | 10 ++-- src/test/test_crypto_slow.c | 65 +++++++++++++------------- src/test/test_sandbox.c | 27 +++++------ 4 files changed, 117 insertions(+), 95 deletions(-) diff --git a/src/feature/hs/hs_pow.c b/src/feature/hs/hs_pow.c index f75b3cb119..27d09cb0b4 100644 --- a/src/feature/hs/hs_pow.c +++ b/src/feature/hs/hs_pow.c @@ -27,6 +27,7 @@ #include "lib/cc/ctassert.h" #include "core/mainloop/cpuworker.h" #include "lib/evloop/workqueue.h" +#include "lib/time/compat_time.h" /** Replay cache set up */ /** Cache entry for (nonce, seed) replay protection. */ @@ -91,23 +92,6 @@ increment_and_set_nonce(uint8_t *nonce, uint8_t *challenge) memcpy(challenge + HS_POW_NONCE_OFFSET, nonce, HS_POW_NONCE_LEN); } -/* Helper: Allocate an EquiX context, using the much faster compiled - * implementation of hashx if it's available on this architecture. */ -static equix_ctx * -build_equix_ctx(equix_ctx_flags flags) -{ - equix_ctx *ctx = equix_alloc(flags | EQUIX_CTX_COMPILE); - if (ctx == EQUIX_NOTSUPP) { - ctx = equix_alloc(flags); - } - tor_assert_nonfatal(ctx != EQUIX_NOTSUPP); - tor_assert_nonfatal(ctx != NULL); - if (ctx == EQUIX_NOTSUPP) { - ctx = NULL; - } - return ctx; -} - /* Helper: Build EquiX challenge (P || ID || C || N || INT_32(E)) and return * a newly allocated buffer containing it. */ static uint8_t * @@ -214,36 +198,82 @@ hs_pow_solve(const hs_pow_solver_inputs_t *pow_inputs, challenge = build_equix_challenge(&pow_inputs->service_blinded_id, pow_inputs->seed, nonce, effort); - ctx = build_equix_ctx(EQUIX_CTX_SOLVE); + ctx = equix_alloc(EQUIX_CTX_SOLVE | EQUIX_CTX_TRY_COMPILE); if (!ctx) { goto end; } - equix_solution solutions[EQUIX_MAX_SOLS]; - uint8_t sol_bytes[HS_POW_EQX_SOL_LEN]; + uint8_t sol_bytes[HS_POW_EQX_SOL_LEN]; + monotime_t start_time; + monotime_get(&start_time); log_info(LD_REND, "Solving proof of work (effort %u)", effort); + for (;;) { /* Calculate solutions to S = equix_solve(C || N || E), */ - int count = equix_solve(ctx, challenge, HS_POW_CHALLENGE_LEN, solutions); - for (int i = 0; i < count; i++) { - pack_equix_solution(&solutions[i], sol_bytes); - - /* Check an Equi-X solution against the effort threshold */ - if (validate_equix_challenge(challenge, sol_bytes, effort)) { - /* Store the nonce N. */ - memcpy(pow_solution_out->nonce, nonce, HS_POW_NONCE_LEN); - /* Store the effort E. */ - pow_solution_out->effort = effort; - /* We only store the first 4 bytes of the seed C. */ - memcpy(pow_solution_out->seed_head, pow_inputs->seed, + equix_solutions_buffer buffer; + equix_result result; + result = equix_solve(ctx, challenge, HS_POW_CHALLENGE_LEN, &buffer); + switch (result) { + + case EQUIX_OK: + for (unsigned i = 0; i < buffer.count; i++) { + pack_equix_solution(&buffer.sols[i], sol_bytes); + + /* Check an Equi-X solution against the effort threshold */ + if (validate_equix_challenge(challenge, sol_bytes, effort)) { + /* Store the nonce N. */ + memcpy(pow_solution_out->nonce, nonce, HS_POW_NONCE_LEN); + /* Store the effort E. */ + pow_solution_out->effort = effort; + /* We only store the first 4 bytes of the seed C. */ + memcpy(pow_solution_out->seed_head, pow_inputs->seed, sizeof(pow_solution_out->seed_head)); - /* Store the solution S */ - memcpy(&pow_solution_out->equix_solution, sol_bytes, sizeof sol_bytes); + /* Store the solution S */ + memcpy(&pow_solution_out->equix_solution, + sol_bytes, sizeof sol_bytes); + + monotime_t end_time; + monotime_get(&end_time); + int64_t duration_usec = monotime_diff_usec(&start_time, &end_time); + log_info(LD_REND, "Proof of work solution (effort %u) found " + "using %s implementation in %u.%06u seconds", + effort, + (EQUIX_SOLVER_DID_USE_COMPILER & buffer.flags) + ? "compiled" : "interpreted", + (unsigned)(duration_usec / 1000000), + (unsigned)(duration_usec % 1000000)); + + /* Indicate success and we are done. */ + ret = 0; + goto end; + } + } + break; + + case EQUIX_FAIL_CHALLENGE: + /* This happens occasionally due to HashX rejecting some program + * configurations. For our purposes here it's the same as count==0. + * Increment the nonce and try again. */ + break; + + case EQUIX_FAIL_COMPILE: + /* The interpreter is disabled and the compiler failed */ + log_warn(LD_REND, "Proof of work solver failed, " + "compile error with no fallback enabled."); + goto end; - /* Indicate success and we are done. */ - ret = 0; + /* These failures are not applicable to equix_solve, but included for + * completeness and to satisfy exhaustive enum warnings. */ + case EQUIX_FAIL_ORDER: + case EQUIX_FAIL_PARTIAL_SUM: + case EQUIX_FAIL_FINAL_SUM: + /* And these really should not happen, and indicate + * programming errors if they do. */ + case EQUIX_FAIL_NO_SOLVER: + case EQUIX_FAIL_INTERNAL: + default: + tor_assert_nonfatal_unreached(); goto end; - } } /* No solutions for this nonce and/or none that passed the effort @@ -309,7 +339,7 @@ hs_pow_verify(const ed25519_public_key_t *service_blinded_id, goto done; } - ctx = build_equix_ctx(EQUIX_CTX_VERIFY); + ctx = equix_alloc(EQUIX_CTX_VERIFY | EQUIX_CTX_TRY_COMPILE); if (!ctx) { goto done; } @@ -401,11 +431,7 @@ pow_worker_threadfn(void *state_, void *work_) log_warn(LD_REND, "Failed to run the proof of work solver"); tor_free(job->pow_solution_out); job->pow_solution_out = NULL; /* how we signal that we came up empty */ - return WQ_RPL_REPLY; } - - /* we have a winner! */ - log_info(LD_REND, "cpuworker has a proof of work solution"); return WQ_RPL_REPLY; } diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 926d4178c1..db5a2db650 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -2982,9 +2982,9 @@ test_crypto_hashx(void *arg) static const struct { hashx_type type; } variations[] = { - { HASHX_INTERPRETED }, + { HASHX_TYPE_INTERPRETED }, #if defined(_M_X64) || defined(__x86_64__) || defined(__aarch64__) - { HASHX_COMPILED }, + { HASHX_TYPE_COMPILED }, #endif }; @@ -3013,12 +3013,12 @@ test_crypto_hashx(void *arg) ctx = hashx_alloc(variations[vari_i].type); tt_ptr_op(ctx, OP_NE, NULL); - tt_ptr_op(ctx, OP_NE, HASHX_NOTSUPP); retval = hashx_make(ctx, seed_literal, seed_len); - tt_int_op(retval, OP_EQ, 1); + tt_int_op(retval, OP_EQ, HASHX_OK); memset(out_actual, 0xa5, sizeof out_actual); - hashx_exec(ctx, hash_input, out_actual); + retval = hashx_exec(ctx, hash_input, out_actual); + tt_int_op(retval, OP_EQ, HASHX_OK); tt_mem_op(out_actual, OP_EQ, out_expected, sizeof out_actual); } } diff --git a/src/test/test_crypto_slow.c b/src/test/test_crypto_slow.c index 5ffd199813..23bc7a852f 100644 --- a/src/test/test_crypto_slow.c +++ b/src/test/test_crypto_slow.c @@ -636,14 +636,21 @@ test_crypto_equix(void *arg) static const struct { equix_ctx_flags flags; equix_result expected; + equix_solution_flags sol_flags; } variations[] = { - {0, EQUIX_OK}, - {0, EQUIX_ORDER}, - {0, EQUIX_PARTIAL_SUM}, + {0, EQUIX_OK, 0}, + {0, EQUIX_FAIL_ORDER, 0}, + {0, EQUIX_FAIL_PARTIAL_SUM, 0}, #if defined(_M_X64) || defined(__x86_64__) || defined(__aarch64__) - {EQUIX_CTX_COMPILE, EQUIX_OK}, - {EQUIX_CTX_COMPILE, EQUIX_ORDER}, - {EQUIX_CTX_COMPILE, EQUIX_PARTIAL_SUM}, + { EQUIX_CTX_MUST_COMPILE, EQUIX_OK, + EQUIX_SOLVER_DID_USE_COMPILER + }, + { EQUIX_CTX_MUST_COMPILE, EQUIX_FAIL_ORDER, + EQUIX_SOLVER_DID_USE_COMPILER + }, + { EQUIX_CTX_MUST_COMPILE, EQUIX_FAIL_PARTIAL_SUM, + EQUIX_SOLVER_DID_USE_COMPILER + }, #endif }; @@ -659,48 +666,42 @@ test_crypto_equix(void *arg) for (unsigned vari_i = 0; vari_i < num_variations; vari_i++) { const equix_ctx_flags flags = variations[vari_i].flags; + const equix_solution_flags sol_flags = variations[vari_i].sol_flags; const equix_result expected = variations[vari_i].expected; - equix_solution sols_actual[EQUIX_MAX_SOLS]; + equix_solutions_buffer output; equix_ctx *solve_ctx = NULL, *verify_ctx = NULL; solve_ctx = equix_alloc(EQUIX_CTX_SOLVE | flags); tt_ptr_op(solve_ctx, OP_NE, NULL); - tt_ptr_op(solve_ctx, OP_NE, EQUIX_NOTSUPP); /* Solve phase: Make sure the test vector matches */ - memset(sols_actual, 0xa5, sizeof sols_actual); - int retval = equix_solve(solve_ctx, challenge_literal, - challenge_len, sols_actual); - tt_int_op(retval, OP_EQ, num_sols); - tt_mem_op(sols_actual, OP_EQ, sols_expected, + memset(&output, 0xa5, sizeof output); + equix_result result; + result = equix_solve(solve_ctx, challenge_literal, + challenge_len, &output); + tt_int_op(result, OP_EQ, EQUIX_OK); + tt_int_op(output.count, OP_EQ, num_sols); + tt_int_op(output.flags, OP_EQ, sol_flags); + tt_mem_op(output.sols, OP_EQ, sols_expected, num_sols * sizeof(equix_solution)); verify_ctx = equix_alloc(EQUIX_CTX_VERIFY | flags); tt_ptr_op(verify_ctx, OP_NE, NULL); - tt_ptr_op(verify_ctx, OP_NE, EQUIX_NOTSUPP); /* Use each solution for positive and negative tests of verify */ for (size_t sol_i = 0; sol_i < num_sols; sol_i++) { - equix_result result; equix_idx tmp_idx; - equix_solution *sol = &sols_actual[sol_i]; - - switch (expected) { - case EQUIX_ORDER: - /* Swap two otherwise valid indices, to trigger an order error */ - tmp_idx = sol->idx[0]; - sol->idx[0] = sol->idx[1]; - sol->idx[1] = tmp_idx; - break; - case EQUIX_FINAL_SUM: - case EQUIX_PARTIAL_SUM: - /* Most changes to the solution will cause a partial sum error */ - sol->idx[0]++; - break; - case EQUIX_OK: - case EQUIX_CHALLENGE: - break; + equix_solution *sol = &output.sols[sol_i]; + + if (expected == EQUIX_FAIL_ORDER) { + /* Swap two otherwise valid indices, to trigger an order error */ + tmp_idx = sol->idx[0]; + sol->idx[0] = sol->idx[1]; + sol->idx[1] = tmp_idx; + } else if (expected == EQUIX_FAIL_PARTIAL_SUM) { + /* Most changes to the solution will cause a partial sum error */ + sol->idx[0]++; } result = equix_verify(verify_ctx, challenge_literal, diff --git a/src/test/test_sandbox.c b/src/test/test_sandbox.c index a28c9b6e41..64182ecc91 100644 --- a/src/test/test_sandbox.c +++ b/src/test/test_sandbox.c @@ -315,32 +315,27 @@ test_sandbox_crypto_equix(void *arg) {{ 0x62c5, 0x86d1, 0x5752, 0xe1f0, 0x12da, 0x8f33, 0x7336, 0xf161 }}, }; - equix_solution sols_actual[EQUIX_MAX_SOLS] = { 0 }; + equix_solutions_buffer output; equix_ctx *solve_ctx = NULL, *verify_ctx = NULL; - /* TODO: A subsequent change will modify these flags to use an auto fallback - * that will be built into our fork of equix. (This implements a - * performant and low-complexity way to share the generated program - * state during fallback instead of re-generating it.) - */ - solve_ctx = equix_alloc(EQUIX_CTX_SOLVE | EQUIX_CTX_COMPILE); + solve_ctx = equix_alloc(EQUIX_CTX_SOLVE | EQUIX_CTX_TRY_COMPILE); tt_ptr_op(solve_ctx, OP_NE, NULL); - tt_ptr_op(solve_ctx, OP_NE, EQUIX_NOTSUPP); - int retval = equix_solve(solve_ctx, challenge_literal, - challenge_len, sols_actual); - tt_int_op(retval, OP_EQ, num_sols); - tt_mem_op(sols_actual, OP_EQ, sols_expected, + equix_result result; + memset(&output, 0xEE, sizeof output); + result = equix_solve(solve_ctx, challenge_literal, challenge_len, &output); + tt_int_op(result, OP_EQ, EQUIX_OK); + tt_int_op(output.count, OP_EQ, num_sols); + tt_int_op(output.flags, OP_EQ, 0); /* EQUIX_SOLVER_DID_USE_COMPILER unset */ + tt_mem_op(output.sols, OP_EQ, sols_expected, num_sols * sizeof(equix_solution)); - verify_ctx = equix_alloc(EQUIX_CTX_VERIFY | EQUIX_CTX_COMPILE); + verify_ctx = equix_alloc(EQUIX_CTX_VERIFY | EQUIX_CTX_TRY_COMPILE); tt_ptr_op(verify_ctx, OP_NE, NULL); - tt_ptr_op(verify_ctx, OP_NE, EQUIX_NOTSUPP); /* Test one of the solutions randomly */ - equix_result result; const unsigned sol_i = crypto_rand_int(num_sols); - equix_solution *sol = &sols_actual[sol_i]; + equix_solution *sol = &output.sols[sol_i]; result = equix_verify(verify_ctx, challenge_literal, challenge_len, sol); -- To stop receiving notification emails like this one, please contact the administrator of this repository.
This is an automated email from the git hooks/post-receive script. dgoulet pushed a commit to branch main in repository tor. commit 415c0354b2e495f236ddf8f67accb3cba4249e21 Author: Micah Elizabeth Scott <beth@torproject.org> AuthorDate: Sun May 28 19:45:41 2023 -0700 hs_pow: Add CompiledProofOfWorkHash torrc option This exposes the new fallback behavior in hashx via a new AUTOBOOL configuration option, available to both clients and services. The default should be fine for nearly everyone, but it might be necessary to enable or disable the compiler manually for diagnostic purposes. Signed-off-by: Micah Elizabeth Scott <beth@torproject.org> --- doc/man/tor.1.txt | 15 +++++++++++++++ src/app/config/config.c | 1 + src/app/config/or_options_st.h | 5 +++++ src/feature/hs/hs_client.c | 3 ++- src/feature/hs/hs_pow.c | 26 ++++++++++++++++++++++---- src/feature/hs/hs_pow.h | 2 ++ src/test/test_hs_pow_slow.c | 1 + 7 files changed, 48 insertions(+), 5 deletions(-) diff --git a/doc/man/tor.1.txt b/doc/man/tor.1.txt index 19fdf90c90..1589809b1a 100644 --- a/doc/man/tor.1.txt +++ b/doc/man/tor.1.txt @@ -3119,6 +3119,21 @@ The following options are per onion service: The maximum burst size for rendezvous requests handled from the priority queue at once. (Default: 2500) +These options are applicable to both onion services and their clients: + +[[CompiledProofOfWorkHash]] **CompiledProofOfWorkHash** **0**|**1**|**auto**:: + When proof-of-work DoS mitigation is active, both the services themselves + and the clients which connect will use a dynamically generated hash + function as part of the puzzle computation. + + + If this option is set to 1, puzzles will only be solved and verified using + the compiled implementation (about 20x faster) and we choose to fail rather + than using a slower fallback. If it's 0, the compiler will never be used. + By default, the compiler is always tried if possible but the interpreter is + available as a fallback. (Default: auto) + +See also <<opt-list-modules,`--list-modules`>>, these proof of work options +have no effect unless the "`pow`" module is enabled at compile time. == DIRECTORY AUTHORITY SERVER OPTIONS diff --git a/src/app/config/config.c b/src/app/config/config.c index 10090f273d..4a703abaa3 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -380,6 +380,7 @@ static const config_var_t option_vars_[] = { V(ClientTransportPlugin, LINELIST, NULL), V(ClientUseIPv6, BOOL, "1"), V(ClientUseIPv4, BOOL, "1"), + V(CompiledProofOfWorkHash, AUTOBOOL, "auto"), V(ConfluxEnabled, AUTOBOOL, "auto"), VAR("ConfluxClientUX", STRING, ConfluxClientUX_option, "throughput"), diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index c8680bb49e..36b00662b5 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -723,6 +723,11 @@ struct or_options_t { * accessing this value directly. */ int ClientPreferIPv6DirPort; + /** If true, always use the compiled hash implementation. If false, always + * the interpreter. Default of "auto" allows a dynamic fallback from + * copmiler to interpreter. */ + int CompiledProofOfWorkHash; + /** If true, the tor client will use conflux for its general purpose * circuits which excludes onion service traffic. */ int ConfluxEnabled; diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index 0e4b3ca0c3..2bb59f078e 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -750,7 +750,8 @@ consider_sending_introduce1(origin_circuit_t *intro_circ, */ if (have_module_pow() && desc->encrypted_data.pow_params) { hs_pow_solver_inputs_t pow_inputs = { - .effort = desc->encrypted_data.pow_params->suggested_effort + .effort = desc->encrypted_data.pow_params->suggested_effort, + .CompiledProofOfWorkHash = get_options()->CompiledProofOfWorkHash }; ed25519_pubkey_copy(&pow_inputs.service_blinded_id, &desc->plaintext_data.blinded_pubkey); diff --git a/src/feature/hs/hs_pow.c b/src/feature/hs/hs_pow.c index 27d09cb0b4..5cee3b00d7 100644 --- a/src/feature/hs/hs_pow.c +++ b/src/feature/hs/hs_pow.c @@ -175,9 +175,24 @@ unpack_equix_solution(const uint8_t *bytes_in, } } +/** Helper: Map the CompiledProofOfWorkHash configuration option to its + * corresponding equix_ctx_flags bit. */ +static equix_ctx_flags +hs_pow_equix_option_flags(int CompiledProofOfWorkHash) +{ + if (CompiledProofOfWorkHash == 0) { + return 0; + } else if (CompiledProofOfWorkHash == 1) { + return EQUIX_CTX_MUST_COMPILE; + } else { + tor_assert_nonfatal(CompiledProofOfWorkHash == -1); + return EQUIX_CTX_TRY_COMPILE; + } +} + /** Solve the EquiX/blake2b PoW scheme using the parameters in pow_params, and * store the solution in pow_solution_out. Returns 0 on success and -1 - * otherwise. Called by a client. */ + * otherwise. Called by a client, from a cpuworker thread. */ int hs_pow_solve(const hs_pow_solver_inputs_t *pow_inputs, hs_pow_solution_t *pow_solution_out) @@ -198,7 +213,10 @@ hs_pow_solve(const hs_pow_solver_inputs_t *pow_inputs, challenge = build_equix_challenge(&pow_inputs->service_blinded_id, pow_inputs->seed, nonce, effort); - ctx = equix_alloc(EQUIX_CTX_SOLVE | EQUIX_CTX_TRY_COMPILE); + /* This runs on a cpuworker, let's not access global get_options(). + * Instead, the particular options we need are captured in pow_inputs. */ + ctx = equix_alloc(EQUIX_CTX_SOLVE | + hs_pow_equix_option_flags(pow_inputs->CompiledProofOfWorkHash)); if (!ctx) { goto end; } @@ -339,7 +357,8 @@ hs_pow_verify(const ed25519_public_key_t *service_blinded_id, goto done; } - ctx = equix_alloc(EQUIX_CTX_VERIFY | EQUIX_CTX_TRY_COMPILE); + ctx = equix_alloc(EQUIX_CTX_VERIFY | + hs_pow_equix_option_flags(get_options()->CompiledProofOfWorkHash)); if (!ctx) { goto done; } @@ -428,7 +447,6 @@ pow_worker_threadfn(void *state_, void *work_) job->pow_solution_out = tor_malloc_zero(sizeof(hs_pow_solution_t)); if (hs_pow_solve(&job->pow_inputs, job->pow_solution_out)) { - log_warn(LD_REND, "Failed to run the proof of work solver"); tor_free(job->pow_solution_out); job->pow_solution_out = NULL; /* how we signal that we came up empty */ } diff --git a/src/feature/hs/hs_pow.h b/src/feature/hs/hs_pow.h index b5949b7916..d47eba82ab 100644 --- a/src/feature/hs/hs_pow.h +++ b/src/feature/hs/hs_pow.h @@ -84,6 +84,8 @@ typedef struct hs_pow_solver_inputs_t { /** Effort chosen by the client. May be higher or lower than * suggested_effort in the descriptor. */ uint32_t effort; + /** Configuration option, choice of hash implementation. AUTOBOOL. */ + int CompiledProofOfWorkHash; } hs_pow_solver_inputs_t; /** State and parameters of PoW defenses, stored in the service state. */ diff --git a/src/test/test_hs_pow_slow.c b/src/test/test_hs_pow_slow.c index e21eee3395..ff715cf53e 100644 --- a/src/test/test_hs_pow_slow.c +++ b/src/test/test_hs_pow_slow.c @@ -218,6 +218,7 @@ test_hs_pow_vectors(void *arg) hs_pow_solution_t solution = { 0 }; hs_pow_solver_inputs_t input = { .effort = vectors[vec_i].effort, + .CompiledProofOfWorkHash = -1 }; tt_int_op(strlen(service_blinded_id_hex), OP_EQ, 2 * HS_POW_ID_LEN); -- To stop receiving notification emails like this one, please contact the administrator of this repository.
participants (1)
-
gitolite role