[tor/master] Merge branch 'tor-github/pr/1022' - tor-commits

23 May 2019

commit 29955f13e5bc8e61724759ec7245aae602672111
Merge: e13e2012b 9e5c27bd2
Author: David Goulet dgoulet@torproject.org
Date:   Thu May 23 09:50:28 2019 -0400
Merge branch 'tor-github/pr/1022'
.travis.yml                    |  2 +-
 changes/ticket28878            | 11 +++++++++++
 src/ext/tinytest.c             |  6 ++++++
 src/ext/tinytest.h             |  3 +++
 src/test/rng_test_helpers.c    | 36 ++++++++++++++++++++++++++++++++----
 src/test/rng_test_helpers.h    |  3 +--
 src/test/test_circuitpadding.c | 29 +++++++++++++++++++++++++++++
 src/test/test_prob_distr.c     | 29 ++++++++++-------------------
 8 files changed, 93 insertions(+), 26 deletions(-)
diff --cc src/test/test_circuitpadding.c
index 1ac2bd676,4dc284423..a2d192203

--- a/src/test/test_circuitpadding.c
+++ b/src/test/test_circuitpadding.c
@@@ -540,9 -528,10 +544,10 @@@ test_circuitpadding_token_removal_highe
    /* Mock it up */
    MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
    MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+   testing_enable_reproducible_rng();
/* Setup test environment (time etc.) */
 -  client_side = (circuit_t *)origin_circuit_new();
 +  client_side = TO_CIRCUIT(origin_circuit_new());
    client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
    monotime_enable_test_mocking();
@@@ -645,9 -635,10 +651,10 @@@ test_circuitpadding_token_removal_lower
    /* Mock it up */
    MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
    MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+   testing_enable_reproducible_rng();
/* Setup test environment (time etc.) */
 -  client_side = (circuit_t *)origin_circuit_new();
 +  client_side = TO_CIRCUIT(origin_circuit_new());
    client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
    monotime_enable_test_mocking();
@@@ -743,9 -735,10 +751,10 @@@ test_circuitpadding_closest_token_remov
    /* Mock it up */
    MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
    MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+   testing_enable_reproducible_rng();
/* Setup test environment (time etc.) */
 -  client_side = (circuit_t *)origin_circuit_new();
 +  client_side = TO_CIRCUIT(origin_circuit_new());
    client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
    monotime_enable_test_mocking();
@@@ -849,9 -843,10 +859,10 @@@ test_circuitpadding_closest_token_remov
    /* Mock it up */
    MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
    MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+   testing_enable_reproducible_rng();
/* Setup test environment (time etc.) */
 -  client_side = (circuit_t *)origin_circuit_new();
 +  client_side = TO_CIRCUIT(origin_circuit_new());
    client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
    monotime_enable_test_mocking();
@@@ -960,9 -956,10 +972,10 @@@ test_circuitpadding_token_removal_exact
    /* Mock it up */
    MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
    MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+   testing_enable_reproducible_rng();
/* Setup test environment (time etc.) */
 -  client_side = (circuit_t *)origin_circuit_new();
 +  client_side = TO_CIRCUIT(origin_circuit_new());
    client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
    monotime_enable_test_mocking();
@@@ -1299,11 -1300,12 +1316,12 @@@ test_circuitpadding_wronghop(void *arg
    /* Mock this function so that our cell counting tests don't get confused by
     * padding that gets sent by scheduled timers. */
    MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+   testing_enable_reproducible_rng();
-  client_side = (circuit_t *)origin_circuit_new();
 +  client_side = TO_CIRCUIT(origin_circuit_new());
    dummy_channel.cmux = circuitmux_alloc();
 -  relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel,
 -                                            &dummy_channel);
 +  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
 +                                            &dummy_channel));
    orig_client = TO_ORIGIN_CIRCUIT(client_side);
relay_side->purpose = CIRCUIT_PURPOSE_OR;
@@@ -2742,387 -2576,9 +2770,388 @@@ test_circuitpadding_reduce_disable(voi
    free_fake_orcirc(relay_side);
    circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
    circuitmux_free(dummy_channel.cmux);
+   testing_disable_reproducible_rng();
  }
+/** Just a basic machine whose whole purpose is to reach the END state */
 +static void
 +helper_create_ender_machine(void)
 +{
 +  /* Start, burst */
 +  circpad_machine_states_init(&circ_client_machine, 2);
 +
 +  circ_client_machine.states[CIRCPAD_STATE_START].
 +      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_END;
 +
 +  circ_client_machine.conditions.state_mask = CIRCPAD_STATE_ALL;
 +  circ_client_machine.conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
 +}
 +
 +static time_t mocked_timeofday;
 +/** Set timeval to a mock date and time. This is necessary
 + * to make tor_gettimeofday() mockable. */
 +static void
 +mock_tor_gettimeofday(struct timeval *timeval)
 +{
 +  timeval->tv_sec = mocked_timeofday;
 +  timeval->tv_usec = 0;
 +}
 +
 +/** Test manual managing of circuit lifetimes by the circuitpadding
 + *  subsystem. In particular this test goes through all the cases of the
 + *  circpad_marked_circuit_for_padding() function, via
 + *  circuit_mark_for_close() as well as
 + *  circuit_expire_old_circuits_clientside(). */
 +static void
 +test_circuitpadding_manage_circuit_lifetime(void *arg)
 +{
 +  circpad_machine_runtime_t *mi;
 +
 +  (void) arg;
 +
 +  client_side = (circuit_t *)origin_circuit_new();
 +  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
 +  monotime_enable_test_mocking();
 +  MOCK(tor_gettimeofday, mock_tor_gettimeofday);
 +  mocked_timeofday = 23;
 +
 +  helper_create_ender_machine();
 +
 +  /* Enable manual circuit lifetime manage for this test */
 +  circ_client_machine.manage_circ_lifetime = 1;
 +
 +  /* Test setup */
 +  client_side->padding_machine[0] = &circ_client_machine;
 +  client_side->padding_info[0] =
 +    circpad_circuit_machineinfo_new(client_side, 0);
 +  mi = client_side->padding_info[0];
 +
 +  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_START);
 +
 +  /* Check that the circuit is not marked for close */
 +  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
 +  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL);
 +
 +  /* Mark this circuit for close due to a remote reason */
 +  circuit_mark_for_close(client_side,
 +                         END_CIRC_REASON_FLAG_REMOTE|END_CIRC_REASON_NONE);
 +  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
 +  tt_int_op(client_side->marked_for_close, OP_NE, 0);
 +  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL);
 +  client_side->marked_for_close = 0;
 +
 +  /* Mark this circuit for close due to a protocol issue */
 +  circuit_mark_for_close(client_side, END_CIRC_REASON_TORPROTOCOL);
 +  tt_int_op(client_side->marked_for_close, OP_NE, 0);
 +  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL);
 +  client_side->marked_for_close = 0;
 +
 +  /* Mark a measurement circuit for close */
 +  client_side->purpose = CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT;
 +  circuit_mark_for_close(client_side, END_CIRC_REASON_NONE);
 +  tt_int_op(client_side->marked_for_close, OP_NE, 0);
 +  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
 +  client_side->marked_for_close = 0;
 +
 +  /* Mark a general circuit for close */
 +  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
 +  circuit_mark_for_close(client_side, END_CIRC_REASON_NONE);
 +
 +  /* Check that this circuit is still not marked for close since we are
 +   * managing the lifetime manually, but the circuit was tagged as such by the
 +   * circpadding subsystem */
 +  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
 +  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
 +
 +  /* We just tested case (1) from the comments of
 +   * circpad_circuit_should_be_marked_for_close() */
 +
 +  /* Transition the machine to the END state but did not delete its machine */
 +  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
 +  circpad_cell_event_nonpadding_received(client_side);
 +  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
 +
 +  /* We just tested case (3) from the comments of
 +   * circpad_circuit_should_be_marked_for_close().
 +   * Now let's go for case (2). */
 +
 +  /* Reset the close mark */
 +  client_side->marked_for_close = 0;
 +
 +  /* Mark this circuit for close */
 +  circuit_mark_for_close(client_side, 0);
 +
 +  /* See that the circ got closed since we are already in END state */
 +  tt_int_op(client_side->marked_for_close, OP_NE, 0);
 +
 +  /* We just tested case (2). Now let's see that case (4) is unreachable as
 +     that comment claims */
 +
 +  /* First, reset all close marks and tags */
 +  client_side->marked_for_close = 0;
 +  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
 +
 +  /* Now re-create the ender machine so that we can transition to END again */
 +  /* Free up some stuff first */
 +  circpad_circuit_free_all_machineinfos(client_side);
 +  tor_free(circ_client_machine.states);
 +  helper_create_ender_machine();
 +
 +  client_side->padding_machine[0] = &circ_client_machine;
 +  client_side->padding_info[0] =
 +    circpad_circuit_machineinfo_new(client_side, 0);
 +  mi = client_side->padding_info[0];
 +
 +  /* Check we are in START. */
 +  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_START);
 +
 +  /* Test that we don't expire this circuit yet */
 +  client_side->timestamp_dirty = 0;
 +  client_side->state = CIRCUIT_STATE_OPEN;
 +  tor_gettimeofday(&client_side->timestamp_began);
 +  TO_ORIGIN_CIRCUIT(client_side)->circuit_idle_timeout = 23;
 +  mocked_timeofday += 24;
 +  circuit_expire_old_circuits_clientside();
 +  circuit_expire_old_circuits_clientside();
 +  circuit_expire_old_circuits_clientside();
 +  tt_int_op(client_side->timestamp_dirty, OP_NE, 0);
 +  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
 +  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
 +
 +  /* Runaway circpad test: if the machine does not transition to end,
 +   * test that after CIRCPAD_DELAY_MAX_SECS, we get marked anyway */
 +  mocked_timeofday = client_side->timestamp_dirty
 +      + get_options()->MaxCircuitDirtiness + 2;
 +  client_side->padding_info[0]->last_cell_time_sec =
 +      approx_time()-(CIRCPAD_DELAY_MAX_SECS+10);
 +  circuit_expire_old_circuits_clientside();
 +  tt_int_op(client_side->marked_for_close, OP_NE, 0);
 +
 +  /* Test back to normal: if we had activity, we won't close */
 +  client_side->padding_info[0]->last_cell_time_sec = approx_time();
 +  client_side->marked_for_close = 0;
 +  circuit_expire_old_circuits_clientside();
 +  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
 +
 +  /* Transition to END, but before we're past the dirty timer */
 +  mocked_timeofday = client_side->timestamp_dirty;
 +  circpad_cell_event_nonpadding_received(client_side);
 +  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
 +
 +  /* Verify that the circuit was not closed. */
 +  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
 +
 +  /* Now that we are in END state, we can be closed by expiry, but via
 +   * the timestamp_dirty path, not the idle path. So first test not dirty
 +   * enough. */
 +  mocked_timeofday = client_side->timestamp_dirty;
 +  circuit_expire_old_circuits_clientside();
 +  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
 +  mocked_timeofday = client_side->timestamp_dirty
 +      + get_options()->MaxCircuitDirtiness + 2;
 +  circuit_expire_old_circuits_clientside();
 +  tt_int_op(client_side->marked_for_close, OP_NE, 0);
 +
 + done:
 +  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
 +  tor_free(circ_client_machine.states);
 +  monotime_disable_test_mocking();
 +  UNMOCK(tor_gettimeofday);
 +}
 +
 +/** Helper for the test_circuitpadding_hs_machines test:
 + *
 + *  - Create a client and relay circuit.
 + *  - Setup right circuit purpose and attach a machine to the client circuit.
 + *  - Verify that state transitions work as intended and state length gets
 + *    enforced.
 + *
 + *  This function is able to do this test both for intro and rend circuits
 + *  depending on the value of <b>test_intro_circs</b>.
 + */
 +static void
 +helper_test_hs_machines(bool test_intro_circs)
 +{
 +  /* Setup the circuits */
 +  origin_circuit_t *origin_client_side = origin_circuit_new();
 +  client_side = TO_CIRCUIT(origin_client_side);
 +  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
 +
 +  dummy_channel.cmux = circuitmux_alloc();
 +  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
 +  relay_side->purpose = CIRCUIT_PURPOSE_OR;
 +
 +  /* extend the client circ to two hops */
 +  simulate_single_hop_extend(client_side, relay_side, 1);
 +  simulate_single_hop_extend(client_side, relay_side, 1);
 +
 +  /* machines only apply on opened circuits */
 +  origin_client_side->has_opened = 1;
 +
 +  /************************************/
 +
 +  /* Attaching the client machine now won't work here because of a wrong
 +   * purpose */
 +  tt_assert(!client_side->padding_machine[0]);
 +  circpad_add_matching_machines(origin_client_side, origin_padding_machines);
 +  tt_assert(!client_side->padding_machine[0]);
 +
 +  /* Change the purpose, see the machine getting attached */
 +  client_side->purpose = test_intro_circs ?
 +    CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT : CIRCUIT_PURPOSE_C_REND_JOINED;
 +  circpad_add_matching_machines(origin_client_side, origin_padding_machines);
 +  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
 +  tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL);
 +
 +  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
 +  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
 +
 +  /* Verify that the right machine is attached */
 +  tt_str_op(client_side->padding_machine[0]->name, OP_EQ,
 +            test_intro_circs ? "client_ip_circ" : "client_rp_circ");
 +  tt_str_op(relay_side->padding_machine[0]->name, OP_EQ,
 +            test_intro_circs ? "relay_ip_circ": "relay_rp_circ");
 +
 +  /***********************************/
 +
 +  /* Intro machines are at START state, but rend machines have already skipped
 +   * to OBFUSCATE_CIRC_SETUP because of the sent PADDING_NEGOTIATE. */
 +  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
 +            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
 +  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
 +            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
 +
 +  /*Send non-padding to move the machines from START to OBFUSCATE_CIRC_SETUP */
 +  circpad_cell_event_nonpadding_received(client_side);
 +  circpad_cell_event_nonpadding_received(relay_side);
 +  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
 +            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
 +  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
 +            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
 +
 +  /* Check that the state lengths have been sampled and are within range */
 +  circpad_machine_runtime_t *client_machine_runtime =
 +    client_side->padding_info[0];
 +  circpad_machine_runtime_t *relay_machine_runtime =
 +    relay_side->padding_info[0];
 +
 +  if (test_intro_circs) {
 +    /* on the client side, we don't send any padding so
 +     * state length is not set */
 +    tt_i64_op(client_machine_runtime->state_length, OP_EQ, -1);
 +    /* relay side has state limits. check them */
 +    tt_i64_op(relay_machine_runtime->state_length, OP_GE,
 +              INTRO_MACHINE_MINIMUM_PADDING);
 +    tt_i64_op(relay_machine_runtime->state_length, OP_LT,
 +              INTRO_MACHINE_MAXIMUM_PADDING);
 +  } else {
 +    tt_i64_op(client_machine_runtime->state_length, OP_EQ, 1);
 +    tt_i64_op(relay_machine_runtime->state_length, OP_EQ, 1);
 +  }
 +
 +  if (test_intro_circs) {
 +    int i;
 +    /* Send state_length worth of padding from the relay and see that the
 +     * client state goes to END */
 +    for (i = (int) relay_machine_runtime->state_length ; i > 0 ; i--) {
 +      circpad_send_padding_cell_for_callback(relay_machine_runtime);
 +    }
 +    /* See that the machine has been teared down after all the length has been
 +     * exhausted (the padding info should now be null on both sides) */
 +    tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
 +    tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
 +  } else {
 +    int i;
 +    /* Send state_length worth of padding and see that the state goes to END */
 +    for (i = (int) client_machine_runtime->state_length ; i > 0 ; i--) {
 +      circpad_send_padding_cell_for_callback(client_machine_runtime);
 +    }
 +    /* See that the machine has been teared down after all the length has been
 +     * exhausted. */
 +    tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
 +              CIRCPAD_STATE_END);
 +  }
 +
 + done:
 +  free_fake_orcirc(relay_side);
 +  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
 +  circuitmux_free(dummy_channel.cmux);
 +  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
 +}
 +
 +/** Test that the HS circuit padding machines work as intended. */
 +static void
 +test_circuitpadding_hs_machines(void *arg)
 +{
 +  (void)arg;
 +
 +  /* Test logic:
 +   *
 +   * 1) Register the HS machines, which aim to hide the presense of
 +   *    onion service traffic on the client-side
 +   *
 +   * 2) Call helper_test_hs_machines() to perform tests for the intro circuit
 +   *    machines and for the rend circuit machines.
 +  */
 +
 +  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
 +  MOCK(circuit_package_relay_cell, circuit_package_relay_cell_mock);
 +  MOCK(circuit_get_nth_node, circuit_get_nth_node_mock);
 +  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
 +
 +  origin_padding_machines = smartlist_new();
 +  relay_padding_machines = smartlist_new();
 +
 +  nodes_init();
 +
 +  monotime_init();
 +  monotime_enable_test_mocking();
 +  monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC);
 +  monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC);
 +  curr_mocked_time = 1*TOR_NSEC_PER_USEC;
 +
 +  timers_initialize();
 +
 +  /* This is needed so that we are not considered to be dormant */
 +  note_user_activity(20);
 +
 +  /************************************/
 +
 +  /* Register the HS machines */
 +  circpad_machine_client_hide_intro_circuits(origin_padding_machines);
 +  circpad_machine_client_hide_rend_circuits(origin_padding_machines);
 +  circpad_machine_relay_hide_intro_circuits(relay_padding_machines);
 +  circpad_machine_relay_hide_rend_circuits(relay_padding_machines);
 +
 +  /***********************************/
 +
 +  /* Do the tests for the intro circuit machines */
 +  helper_test_hs_machines(true);
 +  /* Do the tests for the rend circuit machines */
 +  helper_test_hs_machines(false);
 +
 +  timers_shutdown();
 +  monotime_disable_test_mocking();
 +
 +  SMARTLIST_FOREACH_BEGIN(origin_padding_machines,
 +                          circpad_machine_spec_t *, m) {
 +    machine_spec_free(m);
 +  } SMARTLIST_FOREACH_END(m);
 +
 +  SMARTLIST_FOREACH_BEGIN(relay_padding_machines,
 +                          circpad_machine_spec_t *, m) {
 +    machine_spec_free(m);
 +  } SMARTLIST_FOREACH_END(m);
 +
 +  smartlist_free(origin_padding_machines);
 +  smartlist_free(relay_padding_machines);
 +
 +  UNMOCK(circuitmux_attach_circuit);
 +  UNMOCK(circuit_package_relay_cell);
 +  UNMOCK(circuit_get_nth_node);
 +  UNMOCK(circpad_machine_schedule_padding);
 +}
 +
  #define TEST_CIRCUITPADDING(name, flags) \
      { #name, test_##name, (flags), NULL, NULL }

    