runtime: more regression tests for Runtime::apply (#13170)

During the development of parallel transaction
processing (#12983), we encountered a bug when multiple transactions
from the same account using different keys were included in a single
chunk. In this scenario, the account balance was calculated incorrectly
because these transactions were processed in parallel.

This PR adds two new tests: one to verify the correct ordering of
transaction outcomes and another to validate transactions signed with
different keys from
the same account.

Author: miloserdow

Diff for: runtime/runtime/src/tests/apply.rs

@@ -61,8 +61,41 @@ fn setup_runtime(
     let epoch_info_provider = MockEpochInfoProvider::default();
     let shard_layout = epoch_info_provider.shard_layout(&EpochId::default()).unwrap();
     let shard_uid = shard_layout.shard_uids().next().unwrap();
+
+    let accounts_with_keys = initial_accounts
+        .into_iter()
+        .map(|account_id| {
+            let signer = Arc::new(InMemorySigner::test_signer(&account_id));
+            (account_id, vec![signer])
+        })
+        .collect::<Vec<_>>();
+
     let (runtime, tries, state_root, apply_state, signers) = setup_runtime_for_shard(
-        initial_accounts,
+        accounts_with_keys,
+        initial_balance,
+        initial_locked,
+        gas_limit,
+        shard_uid,
+        &shard_layout,
+    );
+
+    (runtime, tries, state_root, apply_state, signers, epoch_info_provider)
+}
+
+/// Same general idea as `setup_runtime`, but you can pass multiple keys
+/// for each account.
+fn setup_runtime_with_keys(
+    accounts_with_keys: Vec<(AccountId, Vec<Arc<Signer>>)>,
+    initial_balance: Balance,
+    initial_locked: Balance,
+    gas_limit: Gas,
+) -> (Runtime, ShardTries, CryptoHash, ApplyState, Vec<Arc<Signer>>, impl EpochInfoProvider) {
+    let epoch_info_provider = MockEpochInfoProvider::default();
+    let shard_layout = epoch_info_provider.shard_layout(&EpochId::default()).unwrap();
+    let shard_uid = shard_layout.shard_uids().next().unwrap();
+
+    let (runtime, tries, state_root, apply_state, signers) = setup_runtime_for_shard(
+        accounts_with_keys,
         initial_balance,
         initial_locked,
         gas_limit,
@@ -74,7 +107,7 @@ fn setup_runtime(
 }
 
 fn setup_runtime_for_shard(
-    initial_accounts: Vec<AccountId>,
+    accounts_with_keys: Vec<(AccountId, Vec<Arc<Signer>>)>,
     initial_balance: Balance,
     initial_locked: Balance,
     gas_limit: Gas,
@@ -84,23 +117,33 @@ fn setup_runtime_for_shard(
     let tries = TestTriesBuilder::new().build();
     let root = MerkleHash::default();
     let runtime = Runtime::new();
-    let mut signers = vec![];
     let mut initial_state = tries.new_trie_update(shard_uid, root);
-    for account_id in initial_accounts.into_iter() {
-        let signer: Arc<Signer> = Arc::new(InMemorySigner::test_signer(&account_id));
-        let mut initial_account = account_new(initial_balance, CryptoHash::default());
-        // For the account and a full access key
-        initial_account.set_storage_usage(182);
-        initial_account.set_locked(initial_locked);
-        set_account(&mut initial_state, account_id.clone(), &initial_account);
-        set_access_key(
-            &mut initial_state,
-            account_id,
-            signer.public_key(),
-            &AccessKey::full_access(),
-        );
-        signers.push(signer);
-    }
+
+    let signers = accounts_with_keys
+        .into_iter()
+        .flat_map(|(account_id, signers_for_account)| {
+            let mut initial_account = account_new(initial_balance, CryptoHash::default());
+
+            initial_account.set_storage_usage(182);
+            initial_account.set_locked(initial_locked);
+
+            set_account(&mut initial_state, account_id.clone(), &initial_account);
+
+            signers_for_account
+                .into_iter()
+                .map(|signer| {
+                    set_access_key(
+                        &mut initial_state,
+                        account_id.clone(),
+                        signer.public_key(),
+                        &AccessKey::full_access(),
+                    );
+                    signer
+                })
+                .collect::<Vec<_>>()
+        })
+        .collect::<Vec<_>>();
+
     initial_state.commit(StateChangeCause::InitialState);
     let trie_changes = initial_state.finalize().unwrap().trie_changes;
     let mut store_update = tries.store_update();
@@ -2433,8 +2476,17 @@ fn test_congestion_buffering() {
     let deposit = to_yocto(10_000);
     // execute a single receipt per chunk
     let gas_limit = 1;
+
+    let accounts_with_keys = accounts
+        .into_iter()
+        .map(|account| {
+            let signer = Arc::new(InMemorySigner::test_signer(&account));
+            (account, vec![signer])
+        })
+        .collect::<Vec<_>>();
+
     let (runtime, tries, mut root, mut apply_state, _) = setup_runtime_for_shard(
-        accounts,
+        accounts_with_keys,
         initial_balance,
         initial_locked,
         gas_limit,
@@ -2784,3 +2836,185 @@ fn test_deploy_and_call_local_receipts() {
         ActionErrorKind::FunctionCallError(FunctionCallError::MethodResolveError(_))
     );
 }
+
+/// Verifies that valid transactions from multiple accounts are processed in the correct order,
+/// while transactions with an invalid signer are dropped.
+#[test]
+fn test_transaction_ordering_with_apply() {
+    let alice_signer = InMemorySigner::test_signer(&alice_account());
+    let bob_signer = InMemorySigner::test_signer(&bob_account());
+    let alice_invalid_signer = InMemorySigner::from_seed(alice_account(), KeyType::ED25519, "seed");
+
+    // This transaction should be droped due to invalid signer.
+    let alice_invalid_tx = SignedTransaction::send_money(
+        1,
+        alice_account(),
+        alice_account(),
+        &alice_invalid_signer,
+        100,
+        CryptoHash::default(),
+    );
+    let alice_tx1 = SignedTransaction::send_money(
+        1,
+        alice_account(),
+        alice_account(),
+        &alice_signer,
+        200,
+        CryptoHash::default(),
+    );
+    let alice_tx2 = SignedTransaction::send_money(
+        2,
+        alice_account(),
+        bob_account(),
+        &alice_signer,
+        300,
+        CryptoHash::default(),
+    );
+    let bob_tx1 = SignedTransaction::send_money(
+        1,
+        bob_account(),
+        bob_account(),
+        &bob_signer,
+        400,
+        CryptoHash::default(),
+    );
+    let bob_tx2 = SignedTransaction::send_money(
+        2,
+        bob_account(),
+        alice_account(),
+        &bob_signer,
+        500,
+        CryptoHash::default(),
+    );
+    let bob_tx3 = SignedTransaction::send_money(
+        3,
+        bob_account(),
+        bob_account(),
+        &bob_signer,
+        600,
+        CryptoHash::default(),
+    );
+
+    let txs = vec![
+        bob_tx1.clone(),
+        alice_invalid_tx,
+        alice_tx1.clone(),
+        bob_tx2.clone(),
+        alice_tx2.clone(),
+        bob_tx3.clone(),
+    ];
+
+    let (runtime, tries, root, apply_state, _signers, epoch_info_provider) = setup_runtime(
+        vec![alice_account(), bob_account()],
+        to_yocto(1_000_000),
+        to_yocto(500_000),
+        10u64.pow(15),
+    );
+
+    let validity_flags = vec![true; txs.len()];
+    let signed_valid_period_txs = SignedValidPeriodTransactions::new(&txs, &validity_flags);
+    let apply_result = runtime
+        .apply(
+            tries.get_trie_for_shard(ShardUId::single_shard(), root),
+            &None,
+            &apply_state,
+            &[],
+            signed_valid_period_txs,
+            &epoch_info_provider,
+            Default::default(),
+        )
+        .expect("apply should succeed");
+
+    let expected_order = vec![
+        bob_tx1.get_hash(),
+        alice_tx1.get_hash(),
+        bob_tx2.get_hash(),
+        alice_tx2.get_hash(),
+        bob_tx3.get_hash(),
+    ];
+
+    // Note: The 3 local receipts are generated for valid transactions
+    // where signer_id == receiver_id - tx2, tx4, tx6 (not for tx1 as it is dropped).
+    assert_eq!(
+        apply_result.outcomes.len(),
+        8,
+        "should have processed 5 transactions and 3 local receipts"
+    );
+    let tx_outcomes = apply_result.outcomes.iter().take(5).map(|o| o.id).collect::<Vec<_>>();
+    assert_eq!(tx_outcomes, expected_order, "outcomes are not in expected sorted order");
+}
+
+/// Verifies proper ordering and balance update for transactions signed with multiple keys from one account.
+/// Alice is set up with 3 full-access keys.
+/// Six transactions from Alice to Bob are submitted using various nonces and keys.
+/// The test checks that outcomes are correctly ordered and Alice's final balance is within the expected range.
+#[test]
+fn test_transaction_multiple_access_keys_with_apply() {
+    let alice_signer1 = InMemorySigner::from_seed(alice_account(), KeyType::ED25519, "seed1");
+    let alice_signer2 = InMemorySigner::from_seed(alice_account(), KeyType::ED25519, "seed2");
+    let alice_signer3 = InMemorySigner::from_seed(alice_account(), KeyType::ED25519, "seed3");
+
+    let send_money_tx = |nonce, key| {
+        SignedTransaction::send_money(
+            nonce,
+            alice_account(),
+            bob_account(),
+            key,
+            to_yocto(1000),
+            CryptoHash::default(),
+        )
+    };
+
+    let txs = vec![
+        send_money_tx(1, &alice_signer1),
+        send_money_tx(1, &alice_signer2),
+        send_money_tx(1, &alice_signer3),
+        send_money_tx(2, &alice_signer3),
+        send_money_tx(2, &alice_signer1),
+        send_money_tx(3, &alice_signer1),
+    ];
+
+    let accounts_with_keys = vec![
+        (
+            alice_account(),
+            vec![Arc::new(alice_signer1), Arc::new(alice_signer2), Arc::new(alice_signer3)],
+        ),
+        (bob_account(), vec![]),
+    ];
+
+    let (runtime, tries, root, mut apply_state, _signers, epoch_info_provider) =
+        setup_runtime_with_keys(
+            accounts_with_keys,
+            to_yocto(1_000_000),
+            to_yocto(500_000),
+            10u64.pow(15),
+        );
+
+    let validity_flags = vec![true; txs.len()];
+    let signed_valid_period_txs = SignedValidPeriodTransactions::new(&txs, &validity_flags);
+    let apply_result = runtime
+        .apply(
+            tries.get_trie_for_shard(ShardUId::single_shard(), root),
+            &None,
+            &apply_state,
+            &[],
+            signed_valid_period_txs,
+            &epoch_info_provider,
+            Default::default(),
+        )
+        .expect("apply should succeed");
+
+    let expected_order = txs.iter().map(|tx| tx.get_hash()).collect::<Vec<_>>();
+
+    assert_eq!(apply_result.outcomes.len(), txs.len(), "should have processed 6 transactions");
+    let tx_outcomes = apply_result.outcomes.iter().map(|o| o.id).collect::<Vec<_>>();
+    assert_eq!(tx_outcomes, expected_order, "outcomes are not in expected sorted order");
+
+    let shard_uid = ShardUId::single_shard();
+    let root = commit_apply_result(&apply_result, &mut apply_state, &tries, shard_uid);
+    let state = tries.new_trie_update(shard_uid, root);
+    let account = get_account(&state, &alice_account()).unwrap().unwrap();
+
+    assert!(account.amount() < to_yocto(994_000));
+    assert!(account.amount() > to_yocto(993_000));
+}