pyquarkchain
pyquarkchain copied to clipboard
Published
20 hours ago •
QuarkChain
QuarkChain
Flaky xshard integration test
seeing flaky tests (like this run) sometimes. @qizhou can you take a look?
2020-07-15T01:03:14.5971924Z =================================== FAILURES ===================================
2020-07-15T01:03:14.5972127Z ___________ TestCluster.test_broadcast_cross_shard_transactions_2x1 ____________
2020-07-15T01:03:14.5972214Z
2020-07-15T01:03:14.5972435Z self = <quarkchain.cluster.tests.test_cluster.TestCluster testMethod=test_broadcast_cross_shard_transactions_2x1>
2020-07-15T01:03:14.5972588Z
2020-07-15T01:03:14.5972718Z def test_broadcast_cross_shard_transactions_2x1(self):
2020-07-15T01:03:14.5973590Z """ Test the cross shard transactions are broadcasted to the destination shards """
2020-07-15T01:03:14.5973860Z id1 = Identity.create_random_identity()
2020-07-15T01:03:14.5974051Z id2 = Identity.create_random_identity()
2020-07-15T01:03:14.5974240Z acc1 = Address.create_from_identity(id1, full_shard_key=0)
2020-07-15T01:03:14.5974452Z acc2 = Address.create_from_identity(id2, full_shard_key=1 << 16)
2020-07-15T01:03:14.5974647Z acc3 = Address.create_random_account(full_shard_key=2 << 16)
2020-07-15T01:03:14.5974838Z acc4 = Address.create_random_account(full_shard_key=1 << 16)
2020-07-15T01:03:14.5975027Z acc5 = Address.create_random_account(full_shard_key=0)
2020-07-15T01:03:14.5975150Z
2020-07-15T01:03:14.5975504Z with ClusterContext(
2020-07-15T01:03:14.5975701Z 1, acc1, chain_size=8, shard_size=1, mblock_coinbase_amount=1000000
2020-07-15T01:03:14.5975886Z ) as clusters:
2020-07-15T01:03:14.5976083Z master = clusters[0].master
2020-07-15T01:03:14.5976260Z slaves = clusters[0].slave_list
2020-07-15T01:03:14.5976428Z
2020-07-15T01:03:14.5976553Z # Add a root block first so that later minor blocks referring to this root
2020-07-15T01:03:14.5976751Z # can be broadcasted to other shards
2020-07-15T01:03:14.5976928Z root_block = call_async(
2020-07-15T01:03:14.5977106Z master.get_next_block_to_mine(
2020-07-15T01:03:14.5977301Z Address.create_empty_account(), branch_value=None
2020-07-15T01:03:14.5977496Z )
2020-07-15T01:03:14.5977658Z )
2020-07-15T01:03:14.5977827Z call_async(master.add_root_block(root_block))
2020-07-15T01:03:14.5977951Z
2020-07-15T01:03:14.5978109Z b0 = (
2020-07-15T01:03:14.5978273Z clusters[0]
2020-07-15T01:03:14.5978443Z .get_shard_state((1 << 16) + 1)
2020-07-15T01:03:14.5978747Z .create_block_to_mine(address=acc2)
2020-07-15T01:03:14.5978951Z )
2020-07-15T01:03:14.5979126Z call_async(clusters[0].get_shard((1 << 16) + 1).add_block(b0))
2020-07-15T01:03:14.5979253Z
2020-07-15T01:03:14.5979421Z self.assert_balance(master, [acc1, acc2], [1000000, 500000])
2020-07-15T01:03:14.5979596Z
2020-07-15T01:03:14.5979760Z tx1 = create_transfer_transaction(
2020-07-15T01:03:14.5979942Z shard_state=clusters[0].get_shard_state(1),
2020-07-15T01:03:14.5980185Z key=id1.get_key(),
2020-07-15T01:03:14.5980354Z from_address=acc1,
2020-07-15T01:03:14.5980550Z to_address=acc3,
2020-07-15T01:03:14.5980667Z value=54321,
2020-07-15T01:03:14.5980847Z gas=opcodes.GTXXSHARDCOST + opcodes.GTXCOST,
2020-07-15T01:03:14.5981021Z gas_price=1,
2020-07-15T01:03:14.5981186Z )
2020-07-15T01:03:14.5981360Z self.assertTrue(slaves[0].add_tx(tx1))
2020-07-15T01:03:14.5981535Z tx2 = create_transfer_transaction(
2020-07-15T01:03:14.5981714Z shard_state=clusters[0].get_shard_state(1),
2020-07-15T01:03:14.5981838Z key=id1.get_key(),
2020-07-15T01:03:14.5982019Z from_address=acc1,
2020-07-15T01:03:14.5982199Z to_address=acc3,
2020-07-15T01:03:14.5982368Z value=5555,
2020-07-15T01:03:14.5982543Z gas=opcodes.GTXXSHARDCOST + opcodes.GTXCOST,
2020-07-15T01:03:14.5982723Z nonce=tx1.tx.to_evm_tx().nonce + 1,
2020-07-15T01:03:14.5982893Z gas_price=3,
2020-07-15T01:03:14.5983054Z )
2020-07-15T01:03:14.5983170Z self.assertTrue(slaves[0].add_tx(tx2))
2020-07-15T01:03:14.5983346Z
2020-07-15T01:03:14.5983522Z tx3 = create_transfer_transaction(
2020-07-15T01:03:14.5983715Z shard_state=clusters[0].get_shard_state(1),
2020-07-15T01:03:14.5983891Z key=id2.get_key(),
2020-07-15T01:03:14.5984058Z from_address=acc2,
2020-07-15T01:03:14.5984224Z to_address=acc3,
2020-07-15T01:03:14.5984339Z value=7787,
2020-07-15T01:03:14.5984512Z gas=opcodes.GTXXSHARDCOST + opcodes.GTXCOST,
2020-07-15T01:03:14.5984685Z gas_price=2,
2020-07-15T01:03:14.5984859Z )
2020-07-15T01:03:14.5985026Z self.assertTrue(slaves[1].add_tx(tx3))
2020-07-15T01:03:14.5985206Z
2020-07-15T01:03:14.5985378Z b1 = clusters[0].get_shard_state(1).create_block_to_mine(address=acc5)
2020-07-15T01:03:14.5985560Z b2 = (
2020-07-15T01:03:14.5985672Z clusters[0]
2020-07-15T01:03:14.5985842Z .get_shard_state((1 << 16) + 1)
2020-07-15T01:03:14.5986084Z .create_block_to_mine(address=acc4)
2020-07-15T01:03:14.5986327Z )
2020-07-15T01:03:14.5986485Z
2020-07-15T01:03:14.5986655Z call_async(clusters[0].get_shard(1).add_block(b1))
2020-07-15T01:03:14.5986857Z call_async(clusters[0].get_shard((1 << 16) + 1).add_block(b2))
2020-07-15T01:03:14.5986981Z
2020-07-15T01:03:14.5987141Z self.assert_balance(
2020-07-15T01:03:14.5987309Z master,
2020-07-15T01:03:14.5987486Z [acc1, acc1.address_in_shard(2 << 16), acc2, acc4, acc5],
2020-07-15T01:03:14.5987665Z [
2020-07-15T01:03:14.5989154Z 1000000
2020-07-15T01:03:14.5989981Z - 54321
2020-07-15T01:03:14.5990238Z - 5555
2020-07-15T01:03:14.5990603Z - (opcodes.GTXXSHARDCOST + opcodes.GTXCOST) * 4,
2020-07-15T01:03:14.5990818Z 1000000,
2020-07-15T01:03:14.5991210Z 500000 - 7787 - (opcodes.GTXXSHARDCOST + opcodes.GTXCOST) * 2,
2020-07-15T01:03:14.5991401Z 500000 + opcodes.GTXCOST,
2020-07-15T01:03:14.5991681Z 500000 + opcodes.GTXCOST * 2,
2020-07-15T01:03:14.5991878Z ],
2020-07-15T01:03:14.5992040Z )
2020-07-15T01:03:14.5992151Z self.assertEqual(
2020-07-15T01:03:14.5992320Z call_async(
2020-07-15T01:03:14.5992495Z master.get_primary_account_data(acc3)
2020-07-15T01:03:14.5992694Z ).token_balances.balance_map,
2020-07-15T01:03:14.5992862Z {},
2020-07-15T01:03:14.5993021Z )
2020-07-15T01:03:14.5993191Z
2020-07-15T01:03:14.5993311Z # expect chain 2 got the CrossShardTransactionList of b1
2020-07-15T01:03:14.5993488Z xshard_tx_list = (
2020-07-15T01:03:14.5993655Z clusters[0]
2020-07-15T01:03:14.5993824Z .get_shard_state((2 << 16) | 1)
2020-07-15T01:03:14.5994011Z .db.get_minor_block_xshard_tx_list(b1.header.get_hash())
2020-07-15T01:03:14.5994203Z )
2020-07-15T01:03:14.5994382Z self.assertEqual(len(xshard_tx_list.tx_list), 2)
2020-07-15T01:03:14.5994573Z self.assertEqual(xshard_tx_list.tx_list[0].tx_hash, tx1.get_hash())
2020-07-15T01:03:14.5994714Z self.assertEqual(xshard_tx_list.tx_list[1].tx_hash, tx2.get_hash())
2020-07-15T01:03:14.5994907Z
2020-07-15T01:03:14.5995133Z xshard_tx_list = (
2020-07-15T01:03:14.5995296Z clusters[0]
2020-07-15T01:03:14.5995465Z .get_shard_state((2 << 16) | 1)
2020-07-15T01:03:14.5995647Z .db.get_minor_block_xshard_tx_list(b2.header.get_hash())
2020-07-15T01:03:14.5995840Z )
2020-07-15T01:03:14.5995961Z self.assertEqual(len(xshard_tx_list.tx_list), 1)
2020-07-15T01:03:14.5996148Z self.assertEqual(xshard_tx_list.tx_list[0].tx_hash, tx3.get_hash())
2020-07-15T01:03:14.5996349Z
2020-07-15T01:03:14.5996508Z root_block = call_async(
2020-07-15T01:03:14.5996690Z master.get_next_block_to_mine(address=acc1, branch_value=None)
2020-07-15T01:03:14.5996870Z )
2020-07-15T01:03:14.5997042Z call_async(master.add_root_block(root_block))
2020-07-15T01:03:14.5997214Z
2020-07-15T01:03:14.5997328Z # b3 should include the deposits of tx1, t2, t3
2020-07-15T01:03:14.5998197Z b3 = (
2020-07-15T01:03:14.5998384Z clusters[0]
2020-07-15T01:03:14.5998556Z .get_shard_state((2 << 16) | 1)
2020-07-15T01:03:14.5998686Z .create_block_to_mine(address=acc1.address_in_shard(2 << 16))
2020-07-15T01:03:14.5998870Z )
2020-07-15T01:03:14.5999032Z self.assertTrue(
2020-07-15T01:03:14.5999214Z call_async(master.add_raw_minor_block(b3.header.branch, b3.serialize()))
2020-07-15T01:03:14.5999403Z )
2020-07-15T01:03:14.5999675Z self.assert_balance(
2020-07-15T01:03:14.5999856Z master,
2020-07-15T01:03:14.5999983Z [acc1, acc1.address_in_shard(2 << 16), acc2, acc3, acc4, acc5],
2020-07-15T01:03:14.6000169Z [
2020-07-15T01:03:14.6000341Z 1000000 # minior block reward
2020-07-15T01:03:14.6000753Z - 54321
2020-07-15T01:03:14.6001053Z - 5555
2020-07-15T01:03:14.6001676Z - (opcodes.GTXXSHARDCOST + opcodes.GTXCOST) * 4,
2020-07-15T01:03:14.6001869Z 1500000 + opcodes.GTXXSHARDCOST * 3,
2020-07-15T01:03:14.6002292Z 500000 - 7787 - (opcodes.GTXXSHARDCOST + opcodes.GTXCOST) * 2,
2020-07-15T01:03:14.6002432Z 54321 + 5555 + 7787,
2020-07-15T01:03:14.6002609Z 500000 + opcodes.GTXCOST,
2020-07-15T01:03:14.6002783Z > 500000 + opcodes.GTXCOST * 2,
2020-07-15T01:03:14.6003003Z ],
2020-07-15T01:03:14.6003172Z )
2020-07-15T01:03:14.6003239Z
2020-07-15T01:03:14.6141761Z quarkchain/cluster/tests/test_cluster.py:1146:
2020-07-15T01:03:14.6142207Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2020-07-15T01:03:14.6142370Z quarkchain/cluster/tests/test_cluster.py:1001: in assert_balance
2020-07-15T01:03:14.6142518Z {genesis_token: balance_list[idx]},
2020-07-15T01:03:14.6142647Z E AssertionError: {35760: 1518000} != {35760: 1527000}
2020-07-15T01:03:14.6143321Z E - {35760: 1518000}
2020-07-15T01:03:14.6143453Z E ? ^^
2020-07-15T01:03:14.6143576Z E
2020-07-15T01:03:14.6143695Z E + {35760: 1527000}
2020-07-15T01:03:14.6143820Z E ? ^^
2020-07-15T01:03:14.6143963Z ===================== 1 failed, 31 passed in 69.45 seconds =====================
Looks like it is reproducible via: while [[ $? == 0 ]]; do pytest tests/test_cluster.py -k test_broadcast_cross_shard_transactions_2x1; done
The cause is that the root block only contains 2 minor blocks (expected 3, miss b2), and thus the corresponding xshard deposit is not processed.
Interesting why there is a race that a root block does not contain all minor blocks that are successfully added.