ArcticDB
ArcticDB copied to clipboard
Published
20 hours ago •
man-group
man-group
`test_symbol_list_parallel_stress_with_delete[False-2-3-10-2]` failed on CI for Linux Python 3.7
https://github.com/man-group/ArcticDB/actions/runs/7398766075/job/20129034406 In case logs expire:
_________ test_symbol_list_parallel_stress_with_delete[False-2-3-10-2] _________
lmdb_version_store_v1 = NativeVersionStore: Library: test_symbol_list_parallel_stre.224_2024-01-03T15_15_46_634705, Primary Storage: lmdb_storage.
list_freq = 2, delete_freq = 10, update_freq = 3, compaction_size = 2
same_symbols = False
@pytest.mark.parametrize("list_freq", [2, 100])
@pytest.mark.parametrize("delete_freq", [2, 10])
@pytest.mark.parametrize("update_freq", [3, 8])
@pytest.mark.parametrize("compaction_size", [2, 10, 200])
@pytest.mark.parametrize("same_symbols", [True, False])
def test_symbol_list_parallel_stress_with_delete(
lmdb_version_store_v1, list_freq, delete_freq, update_freq, compaction_size, same_symbols
):
pd.set_option("display.max_rows", None)
ScopedMaxDelta(compaction_size)
lib = lmdb_version_store_v1
num_pre_existing_symbols = 100
num_symbols = 10
num_workers = 5
num_cycles = 1
symbol_length = 6
pre_existing_symbols = random_strings(num_pre_existing_symbols, symbol_length)
for idx, existing in enumerate(pre_existing_symbols):
lib.write(existing, _tiny_df(idx))
if same_symbols:
frozen_symbols = random_strings(num_symbols, symbol_length)
symbols = [frozen_symbols for _ in range(num_workers)]
else:
symbols = [random_strings(num_symbols, symbol_length) for _ in range(num_workers)]
with Pool(num_workers) as p:
p.map(
_perform_actions,
[(lib, syms, idx, num_cycles, list_freq, delete_freq, update_freq) for idx, syms in enumerate(symbols)],
)
p.close()
p.join()
expected_symbols = set(lib.list_symbols(use_symbol_list=False))
got_symbols = set(lib.list_symbols())
extra_symbols = got_symbols - expected_symbols
assert len(extra_symbols) == 0
missing_symbols = expected_symbols - got_symbols
for sym in missing_symbols:
> assert not lib.version_store.indexes_sorted(sym)
E AssertionError: assert not True
E + where True = <bound method PyCapsule.indexes_sorted of <arcticdb_ext.version_store.PythonVersionStore object at 0x7f498088a3b0>>('wPd')
E + where <bound method PyCapsule.indexes_sorted of <arcticdb_ext.version_store.PythonVersionStore object at 0x7f498088a3b0>> = <arcticdb_ext.version_store.PythonVersionStore object at 0x7f498088a3b0>.indexes_sorted
E + where <arcticdb_ext.version_store.PythonVersionStore object at 0x7f498088a3b0> = NativeVersionStore: Library: test_symbol_list_parallel_stre.224_2024-01-03T15_15_46_634705, Primary Storage: lmdb_storage..version_store
test_symbol_list.py:347: AssertionError
----------------------------- Captured stderr call -----------------------------
20240103 15:15:47.144851 286 W arcticdb.version | Unexpected lock timestamp, 1704294947135532895 != 1704294947136246316
20240103 15:15:47.145360 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xb492ec66445c89dc@1704294946861950880[AssheV,_v2_]
20240103 15:15:47.145395 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x3c20ea158fa697f7@1704294946778279926[Ayr,_v2_]
20240103 15:15:47.145402 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xe8637cf5ea5a40f2@1704294946736478074[BPRhTSx,_v2_]
20240103 15:15:47.145407 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xa2b4e45ab9bfdd4@1704294946692781486[BfVULd,_v2_]
20240103 15:15:47.145412 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x93cce04853b63a4d@1704294946968133382[CZHVQMT,_v2_]
20240103 15:15:47.145417 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x38b46baeed8cb859@17042949467[2170](https://github.com/man-group/ArcticDB/actions/runs/7398766075/job/20129034406#step:11:2171)0081[ClN,_v2_]
20240103 15:15:47.145422 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x39cae7714aab2de@1704294946982150072[DZj,_v2_]
20240103 15:15:47.145427 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x8f9c30475dd730f0@1704294946885844280[DzsJpa,_v2_]
20240103 15:15:47.145431 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xb76ced47ad4b1f77@1704294946875293475[EgaiITo,_v2_]
20240103 15:15:47.145437 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x3a627c0e9d4bd69f@1704294946766914671[FP,_v2_]
20240103 15:15:47.145441 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x9a6bd1f4f32b5f98@1704294946843661807[GbwrGIA,_v2_]
20240103 15:15:47.145446 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xb976f28adb7bdfa2@1704294947033717504[GyaBDRy,_v2_]
20240103 15:15:47.145451 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x26b4b525dc725f5d@1704294946964586314[HuekifR,_v2_]
20240103 15:15:47.145456 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x13ec33b2866ea7d8@1704294946666542641[IGd,_v2_]
20240103 15:15:47.145460 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xcf5dd9752798db77@1704294946700552355[ILLMqGQ,_v2_]
20240103 15:15:47.145465 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x9920903d4f9ce825@1704294946704516100[JLoCtU,_v2_]
20240103 15:15:47.145470 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x5f79b935c44c90@1704294947026101073[JVBstP,_v2_]
20240103 15:15:47.145476 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xd3df0b9218732fa8@1704294947018633050[JjUMM,_v2_]
20240103 15:15:47.145481 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x3aae21d8cb72e84@1704294946740111984[JlZQee,_v2_]
20240103 15:15:47.145485 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xb1da95805932bbbe@1704294946725182799[JxmOaa,_v2_]
20240103 15:15:47.145490 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x5993060a34d9d97b@1704294947004379128[KKb,_v2_]
20240103 15:15:47.145495 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xc329b4cc670fca16@1704294946655902759[KTUDmj,_v2_]
20240103 15:15:47.145500 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x10a928edb52e628b@1704294946770598484[KWbj,_v2_]
20240103 15:15:47.145504 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x9bebbfad5ad63ba@1704294946708120275[Ke,_v2_]
20240103 15:15:47.145509 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x3371361a69d2a05d@1704294946728584325[Kg,_v2_]
20240103 15:15:47.145514 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xce01dc90bd8272f4@1704294947022114926[KicJ,_v2_]
20240103 15:15:47.145519 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xbf2dd023a9ef08d4@1704294946948509278[LEz,_v2_]
20240103 15:15:47.145531 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x3549e1d108c56edf@1704294946971970782[NBcz,_v2_]
20240103 15:15:47.145537 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x6869d95c6604ea6f@1704294946835435018[NvC,_v2_]
20240103 15:15:47.145542 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xf3399dc934f5dc17@1704294946819136714[NxNC,_v2_]
20240103 15:15:47.145547 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xa4a42bfa84e1fbbc@1704294946674993227[OzzKM,_v2_]
20240103 15:15:47.145551 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xacb2c65b05e3f1ec@1704294946939928646[PRCcxli,_v2_]
20240103 15:15:47.145557 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x401c62de9d67e764@1704294946922426722[Qgios,_v2_]
20240103 15:15:47.145562 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x88e5f515531dd632@1704294946712338485[SHJtur,_v2_]
20240103 15:15:47.145567 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x85a331ab6deb15b2@1704294946957435751[Sx,_v2_]
20240103 15:15:47.145572 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x87aa5c0cca841004@1704294947045832898[TDWuAg,_v2_]
20240103 15:15:47.145577 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xc9933dc6e2836efb@1704294946759337704[TGB,_v2_]
20240103 15:15:47.145582 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xe6fe270862bfbe31@1704294946811100049[TQnd,_v2_]
20240103 15:15:47.145587 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xfc9ff2989d333375@1704294946932890886[TYtev,_v2_]
20240103 15:15:47.145591 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xb43d9e71d375b7d5@1704294946858209269[Ta,_v2_]
20240103 15:15:47.145596 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xc8868090f1142c9b@1704294946986418736[TzwsZE,_v2_]
20240103 15:15:47.145601 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x38e4953a19b95184@1704294946747913710[Uc,_v2_]
20240103 15:15:47.145605 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x1751cac937e44561@1704294946806584725[VaY,_v2_]
20240103 15:15:47.145610 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x530cb54c3605d082@1704294946685298024[Vipume,_v2_]
20240103 15:15:47.145615 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x7685ab2335ef51dd@1704294946901095635[Vq,_v2_]
20240103 15:15:47.145619 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xf7a2bf1dc171250a@1704294946896794651[Vrt,_v2_]
20240103 15:15:47.145624 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xe31fbb41c03a17ae@1704294946851137445[Vs,_v2_]
20240103 15:15:47.145629 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x6c8bfab7b6035dab@1704294946847873986[Wnj,_v2_]
20240103 15:15:47.145633 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xe15ed4c5a2294428@1704294946681954064[XGRIlm,_v2_]
20240103 15:15:47.145638 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x474ce59515e60e9f@1704294946908608002[Xf,_v2_]
20240103 15:15:47.145643 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x23c2c7aa1ae4bebc@1704294946990407929[XqFdGD,_v2_]
20240103 15:15:47.145647 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x49c448eef31d05db@1704294946854625583[YkqRj,_v2_]
20240103 15:15:47.145652 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xa32d5f5828a2f5fe@1704294946995387562[ZFS,_v2_]
20240103 15:15:47.145656 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xa11e25dbf366edf5@1704294946889952726[ZGrg,_v2_]
20240103 15:15:47.145661 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xd71f44b7e1b0d64f@1704294946696880594[ZN,_v2_]
20240103 15:15:47.145671 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xcbd6a9e8889e207d@1704294946911978170[ZOvYFb,_v2_]
20240103 15:15:47.145675 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x936ee6aaba1fff24@1704294946663340566[ZYiOPC,_v2_]
20240103 15:15:47.145680 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xc62ed91d6daf3ce2@1704294947012154783[Zf,_v2_]
20240103 15:15:47.145685 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x8d5469bdb4825d2f@1704294946823440339[ZqENhe,_v2_]
20240103 15:15:47.145689 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xc284799978ec50aa@1704294946929276743[aU,_v2_]
20240103 15:15:47.145694 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x54aee76a857ed1ba@1704294947041697463[eUhlO,_v2_]
20240103 15:15:47.145699 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xd06b0df34e5e8088@1704294946670432378[fqmzX,_v2_]
20240103 15:15:47.145703 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xf0026212878cb9b6@1704294946732806603[gGSgul,_v2_]
20240103 15:15:47.145708 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x6f8ef2ed10ce7a88@1704294946919254896[gfwxWNO,_v2_]
20240103 15:15:47.145712 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x25064b1d4195167f@1704294946882130491[gsTqgag,_v2_]
20240103 15:15:47.145719 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x5dfe3e1ab479ec08@1704294946688885367[hIaWe,_v2_]
20240103 15:15:47.145724 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x11d35f82adccf641@1704294947015365914[hNJocQy,_v2_]
20240103 15:15:47.145729 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x8e0477[2172](https://github.com/man-group/ArcticDB/actions/runs/7398766075/job/20129034406#step:11:2173)6a93e5@1704294946678803617[hWl,_v2_]
20240103 15:15:47.145733 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x481440dab104494@1704294946743398305[hfINX,_v2_]
20240103 15:15:47.145738 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xb17ac9d82724f1ea@1704294946893296735[hto,_v2_]
20240103 15:15:47.145743 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x8e44cc13dba95208@1704294946915370569[iX,_v2_]
20240103 15:15:47.145747 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x990caed706569f4d@1704294946827186819[iqbVvn,_v2_]
20240103 15:15:47.145752 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xd08b65f60efc3259@1704294946659730861[iuEH,_v2_]
20240103 15:15:47.145757 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xd78cb69822ce1ebe@1704294947037802856[jTHry,_v2_]
20240103 15:15:47.145761 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xc294eb2014ffb9ee@1704294946961118842[jaTyhAu,_v2_]
20240103 15:15:47.145766 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x56a13f35e608358d@1704294947050499645[jdMV,_v2_]
20240103 15:15:47.145771 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x191ae6a08a6dffc7@1704294946755865576[jhh,_v2_]
20240103 15:15:47.145775 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x5b733e60423173e7@1704294946878901336[jp,_v2_]
20240103 15:15:47.145780 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xe5efc249da3f0593@1704294946716702708[kC,_v2_]
20240103 15:15:47.145785 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x3736c723b3232caf@1704294946952471490[lFNV,_v2_]
20240103 15:15:47.145789 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xe4d44a43e9395c78@1704294947008758757[mEGBAk,_v2_]
20240103 15:15:47.145794 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xda92297cad83714e@1704294947030229747[nMdQd,_v2_]
20240103 15:15:47.145804 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xd1d17538e706b596@1704294946871929809[ok,_v2_]
20240103 15:15:47.145813 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x2d65a2464705aba5@1704294946868123918[pHV,_v2_]
20240103 15:15:47.145818 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x733289bc8660fc20@1704294946652277325[qpt,_v2_]
20240103 15:15:47.145823 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xdc5fba302a0e1556@1704294946773718886[sk,_v2_]
20240103 15:15:47.145828 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x4b0d640495efdda6@1704294946999379720[tiyQdS,_v2_]
20240103 15:15:47.145832 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x2bf28fa2b242bb83@1704294946904891938[uER,_v2_]
20240103 15:15:47.145837 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x4d008fa97909d037@1704294946646710161[uGhlxJY,_v2_]
20240103 15:15:47.145842 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x4b67e8f11d4e50cb@1704294946936064339[uPIWWZ,_v2_]
20240103 15:15:47.145847 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x5c1dc7338c18a7e3@1704294947055437719[uUVel,_v2_]
20240103 15:15:47.145851 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xb24ea9cb6bc9b6f9@1704294946839423289[vNVy,_v2_]
20240103 15:15:47.145856 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xcfe19a0661c8e7ba@1704294946944555581[vVoi,_v2_]
20240103 15:15:47.145861 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xd985873cfe3f1071@1704294946763199890[vXZOqeQ,_v2_]
20240103 15:15:47.145865 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x6d38c55884b3bd5@1704294946831479476[vcQco,_v2_]
20240103 15:15:47.145871 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xa69f70015f7c7fa4@1704294946781837213[wvgznZ,_v2_]
20240103 15:15:47.145876 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x51ac983e3ab07b8e@1704294947062627233[yAX,_v2_]
20240103 15:15:47.145880 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xef3e381fa0d152f2@1704294946815138064[yP,_v2_]
20240103 15:15:47.145885 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x578f4304f12eada2@1704294947059039709[ytO,_v2_]
20240103 15:15:47.145890 25186 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x198e88080d67365e@1704294946925849388[zogKitf,_v2_]
20240103 15:15:47.148898 286 W arcticdb.symbol | Ignoring error while trying to compact the symbol list: Composite: l:__add__:0:0xb492ec66445c89dc@1704294946861950880[AssheV,_v2_], l:__add__:0:0x3c20ea158fa697f7@1704294946778279926[Ayr,_v2_], l:__add__:0:0xe8637cf5ea5a40f2@1704294946736478074[BPRhTSx,_v2_], l:__add__:0:0xa2b4e45ab9bfdd4@1704294946692781486[BfVULd,_v2_], l:__add__:0:0x93cce04853b63a4d@1704294946968133382[CZHVQMT,_v2_], l:__add__:0:0x38b46baeed8cb859@1704294946721700081[ClN,_v2_], l:__add__:0:0x39cae7714aab2de@1704294946982150072[DZj,_v2_], l:__add__:0:0x8f9c30475dd730f0@1704294946885844280[DzsJpa,_v2_], l:__add__:0:0xb76ced47ad4b1f77@1704294946875293475[EgaiITo,_v2_], l:__add__:0:0x3a627c0e9d4bd69f@1704294946766914671[FP,_v2_], l:__add__:0:0x9a6bd1f4f32b5f98@1704294946843661807[GbwrGIA,_v2_], l:__add__:0:0xb976f28adb7bdfa2@1704294947033717504[GyaBDRy,_v2_], l:__add__:0:0x26b4b525dc725f5d@1704294946964586314[HuekifR,_v2_], l:__add__:0:0x13ec33b2866ea7d8@1704294946666542641[IGd,_v2_], l:__add__:0:0xcf5dd9752798db77@1704294946700552355[ILLMqGQ,_v2_], l:__add__:0:0x9920903d4f9ce825@1704294946704516100[JLoCtU,_v2_], l:__add__:0:0x5f79b935c44c90@1704294947026101073[JVBstP,_v2_], l:__add__:0:0xd3df0b9[2187](https://github.com/man-group/ArcticDB/actions/runs/7398766075/job/20129034406#step:11:2188)32fa8@1704294947018633050[JjUMM,_v2_], l:__add__:0:0x3aae21d8cb72e84@1704294946740111984[JlZQee,_v2_], l:__add__:0:0xb1da95805932bbbe@1704294946725182799[JxmOaa,_v2_], l:__add__:0:0x5993060a34d9d97b@1704294947004379128[KKb,_v2_], l:__add__:0:0xc329b4cc670fca16@1704294946655902759[KTUDmj,_v2_], l:__add__:0:0x10a928edb52e628b@1704294946770598484[KWbj,_v2_], l:__add__:0:0x9bebbfad5ad63ba@1704294946708120275[Ke,_v2_], l:__add__:0:0x3371361a69d2a05d@1704294946728584325[Kg,_v2_], l:__add__:0:0xce01dc90bd8272f4@17042949470[2211](https://github.com/man-group/ArcticDB/actions/runs/7398766075/job/20129034406#step:11:2212)4926[KicJ,_v2_], l:__add__:0:0xbf2dd023a9ef08d4@1704294946948509278[LEz,_v2_], l:__add__:0:0x3549e1d108c56edf@1704294946971970782[NBcz,_v2_], l:__add__:0:0x6869d95c6604ea6f@1704294946835435018[NvC,_v2_], l:__add__:0:0xf3399dc934f5dc17@1704294946819136714[NxNC,_v2_], l:__add__:0:0xa4a42bfa84e1fbbc@1704294946674993227[OzzKM,_v2_], l:__add__:0:0xacb2c65b05e3f1ec@1704294946939928646[PRCcxli,_v2_], l:__add__:0:0x401c62de9d67e764@17042949469[2242](https://github.com/man-group/ArcticDB/actions/runs/7398766075/job/20129034406#step:11:2243)6722[Qgios,_v2_], l:__add__:0:0x88e5f515531dd632@1704294946712338485[SHJtur,_v2_], l:__add__:0:0x85a331ab6deb15b2@1704294946957435751[Sx,_v2_], l:__add__:0:0x87aa5c0cca841004@1704294947045832898[TDWuAg,_v2_], l:__add__:0:0xc9933dc6e2836efb@1704294946759337704[TGB,_v2_], l:__add__:0:0xe6fe270862bfbe31@1704294946811100049[TQnd,_v2_], l:__add__:0:0xfc9ff2989d333375@1704294946932890886[TYtev,_v2_], l:__add__:0:0xb43d9e71d375b7d5@1704294946858209269[Ta,_v2_], l:__add__:0:0xc8868090f1142c9b@1704294946986418736[TzwsZE,_v2_], l:__add__:0:0x38e4953a19b95184@1704294946747913710[Uc,_v2_], l:__add__:0:0x1751cac937e44561@1704294946806584725[VaY,_v2_], l:__add__:0:0x530cb54c3605d082@1704294946685298024[Vipume,_v2_], l:__add__:0:0x7685ab2335ef51dd@1704294946901095635[Vq,_v2_], l:__add__:0:0xf7a2bf1dc171250a@1704294946896794651[Vrt,_v2_], l:__add__:0:0xe31fbb41c03a17ae@1704294946851137445[Vs,_v2_], l:__add__:0:0x6c8bfab7b6035dab@1704294946847873986[Wnj,_v2_], l:__add__:0:0xe15ed4c5a2294428@1704294946681954064[XGRIlm,_v2_], l:__add__:0:0x474ce59515e60e9f@1704294946908608002[Xf,_v2_], l:__add__:0:0x23c2c7aa1ae4bebc@1704294946990407929[XqFdGD,_v2_], l:__add__:0:0x49c448eef31d05db@1704294946854625583[YkqRj,_v2_], l:__add__:0:0xa32d5f5828a2f5fe@1704294946995387562[ZFS,_v2_], l:__add__:0:0xa11e25dbf366edf5@1704294946889952726[ZGrg,_v2_], l:__add__:0:0xd71f44b7e1b0d64f@1704294946696880594[ZN,_v2_], l:__add__:0:0xcbd6a9e8889e207d@1704294946911978170[ZOvYFb,_v2_], l:__add__:0:0x936ee6aaba1fff24@1704294946663340566[ZYiOPC,_v2_], l:__add__:0:0xc62ed91d6daf3ce2@1704294947012154783[Zf,_v2_], l:__add__:0:0x8d5469bdb4825d2f@1704294946823440339[ZqENhe,_v2_], l:__add__:0:0xc284799978ec50aa@1704294946929276743[aU,_v2_], l:__add__:0:0x54aee76a857ed1ba@1704294947041697463[eUhlO,_v2_], l:__add__:0:0xd06b0df34e5e8088@1704294946670432378[fqmzX,_v2_], l:__add__:0:0xf0026212878cb9b6@1704294946732806603[gGSgul,_v2_], l:__add__:0:0x6f8ef2ed10ce7a88@1704294946919254896[gfwxWNO,_v2_], l:__add__:0:0x25064b1d4195167f@1704294946882130491[gsTqgag,_v2_], l:__add__:0:0x5dfe3e1ab479ec08@1704294946688885367[hIaWe,_v2_], l:__add__:0:0x11d35f82adccf641@1704294947015365914[hNJocQy,_v2_], l:__add__:0:0x8e047721726a93e5@1704294946678803617[hWl,_v2_], l:__add__:0:0x481440dab104494@1704294946743398305[hfINX,_v2_], l:__add__:0:0xb17ac9d82724f1ea@1704294946893296735[hto,_v2_], l:__add__:0:0x8e44cc13dba95208@1704294946915370569[iX,_v2_], l:__add__:0:0x990caed706569f4d@1704294946827186819[iqbVvn,_v2_], l:__add__:0:0xd08b65f60efc3259@1704294946659730861[iuEH,_v2_], l:__add__:0:0xd78cb69822ce1ebe@1704294947037802856[jTHry,_v2_], l:__add__:0:0xc294eb2014ffb9ee@1704294946961118842[jaTyhAu,_v2_], l:__add__:0:0x56a13f35e608358d@1704294947050499645[jdMV,_v2_], l:__add__:0:0x191ae6a08a6dffc7@1704294946755865576[jhh,_v2_], l:__add__:0:0x5b733e60423173e7@1704294946878901336[jp,_v2_], l:__add__:0:0xe5efc249da3f0593@1704294946716702708[kC,_v2_], l:__add__:0:0x3736c723b3232caf@1704294946952471490[lFNV,_v2_], l:__add__:0:0xe4d44a43e9395c78@1704294947008758757[mEGBAk,_v2_], l:__add__:0:0xda92297cad83714e@1704294947030229747[nMdQd,_v2_], l:__add__:0:0xd1d17538e706b596@1704294946871929809[ok,_v2_], l:__add__:0:0x2d65a2464705aba5@1704294946868123918[pHV,_v2_], l:__add__:0:0x733289bc8660fc20@170429494665[2277](https://github.com/man-group/ArcticDB/actions/runs/7398766075/job/20129034406#step:11:2278)325[qpt,_v2_], l:__add__:0:0xdc5fba302a0e1556@1704294946773718886[sk,_v2_], l:__add__:0:0x4b0d640495efdda6@1704294946999379720[tiyQdS,_v2_], l:__add__:0:0x2bf28fa2b242bb83@1704294946904891938[uER,_v2_], l:__add__:0:0x4d008fa97909d037@1704294946646710161[uGhlxJY,_v2_], l:__add__:0:0x4b67e8f11d4e50cb@1704294946936064339[uPIWWZ,_v2_], l:__add__:0:0x5c1dc7338c18a7e3@1704294947055437719[uUVel,_v2_], l:__add__:0:0xb24ea9cb6bc9b6f9@1704294946839423289[vNVy,_v2_], l:__add__:0:0xcfe19a0661c8e7ba@1704294946944555581[vVoi,_v2_], l:__add__:0:0xd985873cfe3f1071@1704294946763199890[vXZOqeQ,_v2_], l:__add__:0:0x6d38c55884b3bd5@1704294946831479476[vcQco,_v2_], l:__add__:0:0xa69f70015f7c7fa4@1704294946781837213[wvgznZ,_v2_], l:__add__:0:0x51ac983e3ab07b8e@1704294947062627233[yAX,_v2_], l:__add__:0:0xef3e381fa0d152f2@1704294946815138064[yP,_v2_], l:__add__:0:0x578f4304f12eada2@1704294947059039709[ytO,_v2_], l:__add__:0:0x198e88080d67365e@1704294946925849388[zogKitf,_v2_],
I think the failure is not limited to case False-2-3-10-2
In case the log expired:
=================================== FAILURES ===================================
_________ test_symbol_list_parallel_stress_with_delete[False-2-8-10-2] _________
lmdb_version_store_v1 = NativeVersionStore: Library: test_symbol_list_parallel_stre.544_2024-01-08T17_24_11_414833, Primary Storage: lmdb_storage.
list_freq = 2, delete_freq = 10, update_freq = 8, compaction_size = 2
same_symbols = False
@pytest.mark.parametrize("list_freq", [2, 100])
@pytest.mark.parametrize("delete_freq", [2, 10])
@pytest.mark.parametrize("update_freq", [3, 8])
@pytest.mark.parametrize("compaction_size", [2, 10, 200])
@pytest.mark.parametrize("same_symbols", [True, False])
def test_symbol_list_parallel_stress_with_delete(
lmdb_version_store_v1, list_freq, delete_freq, update_freq, compaction_size, same_symbols
):
pd.set_option("display.max_rows", None)
ScopedMaxDelta(compaction_size)
lib = lmdb_version_store_v1
num_pre_existing_symbols = 100
num_symbols = 10
num_workers = 5
num_cycles = 1
symbol_length = 6
pre_existing_symbols = random_strings(num_pre_existing_symbols, symbol_length)
for idx, existing in enumerate(pre_existing_symbols):
lib.write(existing, _tiny_df(idx))
if same_symbols:
frozen_symbols = random_strings(num_symbols, symbol_length)
symbols = [frozen_symbols for _ in range(num_workers)]
else:
symbols = [random_strings(num_symbols, symbol_length) for _ in range(num_workers)]
with Pool(num_workers) as p:
p.map(
_perform_actions,
[(lib, syms, idx, num_cycles, list_freq, delete_freq, update_freq) for idx, syms in enumerate(symbols)],
)
p.close()
p.join()
expected_symbols = set(lib.list_symbols(use_symbol_list=False))
got_symbols = set(lib.list_symbols())
extra_symbols = got_symbols - expected_symbols
assert len(extra_symbols) == 0
missing_symbols = expected_symbols - got_symbols
for sym in missing_symbols:
> assert not lib.version_store.indexes_sorted(sym)
E AssertionError: assert not True
E + where True = <bound method PyCapsule.indexes_sorted of <arcticdb_ext.version_store.PythonVersionStore object at 0x7f38901bfcb0>>('xKhZ')
E + where <bound method PyCapsule.indexes_sorted of <arcticdb_ext.version_store.PythonVersionStore object at 0x7f38901bfcb0>> = <arcticdb_ext.version_store.PythonVersionStore object at 0x7f38901bfcb0>.indexes_sorted
E + where <arcticdb_ext.version_store.PythonVersionStore object at 0x7f38901bfcb0> = NativeVersionStore: Library: test_symbol_list_parallel_stre.544_2024-01-08T17_24_11_414833, Primary Storage: lmdb_storage..version_store
test_symbol_list.py:347: AssertionError
----------------------------- Captured stderr call -----------------------------
20240108 17:24:11.837703 292 W arcticdb.version | Unexpected lock timestamp, 1704734651832599824 != 1704734651829850948
20240108 17:24:11.838156 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xa113080619843668@1704734651635681840[AZOOMi,_v2_]
20240108 17:24:11.838195 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x599659fc9851bcfe@1704734651665867885[AsRPRu,_v2_]
20240108 17:24:11.838203 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x3322dff958f2cfc1@1704734651565609718[CSZH,_v2_]
20240108 17:24:11.838212 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x3b88720222322d5d@1704734651433651474[Cj,_v2_]
20240108 17:24:11.838220 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xaeffcfd71cc18088@1704734651753364797[CmB,_v2_]
20240108 17:24:11.838228 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xf101f5774ae72f50@1704734651427696529[DFtm,_v2_]
20240108 17:24:11.838236 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x48438ad12b30b347@1704734651685350917[DRlNX,_v2_]
20240108 17:24:11.838244 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x8937c286bdc0cbd8@1704734651678579319[FHbgTk,_v2_]
20240108 17:24:11.838252 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xff8d012c647f8564@1704734651462510691[Gjffa,_v2_]
20240108 17:24:11.838262 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xb8bf3b58b8aacca3@1704734651430422551[HqIznpv,_v2_]
20240108 17:24:11.838270 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x78f769d4209bbaa6@1704734651682404661[Ilwekff,_v2_]
20240108 17:24:11.838277 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xdc1051a52e209f7@1704734651743730515[JPt,_v2_]
20240108 17:24:11.838285 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xa99ff36cbc001b6c@1704734651469575357[KCIIJ,_v2_]
20240108 17:24:11.838293 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x9ee0e567a09d4f0d@1704734651692224366[KMQQ,_v2_]
20240108 17:24:11.838300 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xac049097366894b3@1704734651442772653[KOwlckR,_v2_]
20240108 17:24:11.838308 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x27c48396a61f51b1@1704734651695322407[LTf,_v2_]
20240108 17:24:11.838316 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xe6d117b01d9ca5d@1704734651459161364[LvJf,_v2_]
20240108 17:24:11.838325 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xdd47acb40a511aa7@1704734651487357197[MHk,_v2_]
20240108 17:24:11.838333 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xb1347b13b9aa929@1704734651547023411[MI,_v2_]
20240108 17:24:11.838341 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x2b7b5dffca532d15@1704734651659376596[Md,_v2_]
20240108 17:24:11.838349 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x77287b6bcc55151b@1704734651576523519[Mp,_v2_]
20240108 17:24:11.838358 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x98e7c04f463b26a2@1704734651629261462[NYrbCI,_v2_]
20240108 17:24:11.838365 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xa0e64c269e98d036@1704734651593906118[NuV,_v2_]
20240108 17:24:11.838372 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x894858890ee49aa8@1704734651512534849[Optd,_v2_]
20240108 17:24:11.838379 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x741d00947ab966ae@1704734651662394086[OuhZ,_v2_]
20240108 17:24:11.838385 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x5153ab5e89693005@1704734651597579568[PGgxyWy,_v2_]
20240108 17:24:11.838392 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xdad67e5f1a357f04@1704734651526299200[PJ,_v2_]
20240108 17:24:11.838399 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xeac10edf4d992da9@1704734651572955635[QgG,_v2_]
20240108 17:24:11.838420 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xf3c06daa61ce5e11@1704734651746713125[QvisxQ,_v2_]
20240108 17:24:11.838426 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xe1e356a31019cb8f@1704734651473093196[RNQXsg,_v2_]
20240108 17:24:11.838431 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x5e233063ee4e7288@1704734651503352957[RRuzb,_v2_]
20240108 17:24:11.838436 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x8a98f0776c4a749a@1704734651516084879[RqCKbVz,_v2_]
20240108 17:24:11.838440 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xc0fe08278b541365@1704734651580182915[StTiDaW,_v2_]
20240108 17:24:11.838447 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x15aef14c1b32547a@1704734651625785079[TIkubUV,_v2_]
20240108 17:24:11.838451 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xba51c5d52b35eb13@1704734651550286860[TeqR,_v2_]
20240108 17:24:11.838456 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x4359b786679645cd@1704734651450041207[Uf,_v2_]
20240108 17:24:11.838461 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xaa8b8761dfdeb0ab@1704734651675605489[Uvu,_v2_]
20240108 17:24:11.838466 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xfa4537aa83cf9651@1704734651540184789[VNCRZly,_v2_]
20240108 17:24:11.838470 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xb270bb0dcb35adf2@1704734651493825182[Vxqphnt,_v2_]
20240108 17:24:11.838475 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x633c3fbc8162dd48@1704734651740495020[WbdnKX,_v2_]
20240108 17:24:11.838480 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x489e0f850e06d183@1704734651490915705[WePFnC,_v2_]
20240108 17:24:11.838485 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x1091f995f391d401@1704734651708836352[Wl,_v2_]
20240108 17:24:11.838489 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xba9e8d2b6d0db49d@1704734651479143225[XD,_v2_]
20240108 17:24:11.838494 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x7998ffb54dfe8012@1704734651757180438[XJXX,_v2_]
20240108 17:24:11.838499 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x3e0e02912450fc04@1704734651519303523[ZCuo,_v2_]
20240108 17:24:11.838504 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xc6f3600450590f8c@1704734651529558550[ZDYasGD,_v2_]
20240108 17:24:11.838509 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xebc803aca4de5a46@1704734651750122519[ZuinK,_v2_]
20240108 17:24:11.838514 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x79964f69f5f721e6@1704734651583263290[aCYe,_v2_]
20240108 17:24:11.838518 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x3cf01e357955d355@1704734651723348721[aajSn,_v2_]
20240108 17:24:11.838523 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xddd8dd0fc0df1ed8@1704734651559188369[cbmtv,_v2_]
20240108 17:24:11.838528 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x8372c31d3b25606b@1704734651453243817[ckTUNd,_v2_]
20240108 17:24:11.838533 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x1297eb7b46339039@1704734651586529704[dTrwHl,_v2_]
20240108 17:24:11.838537 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x1b50e21074820ab1@1704734651719942577[dWJcwD,_v2_]
20240108 17:24:11.838542 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xdc3ace4b7068e798@1704734651729965071[eXCzh,_v2_]
20240108 17:24:11.838547 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xd2977a168fa3a3b2@1704734651439971325[fFZRK,_v2_]
20240108 17:24:11.838558 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xd3feb15cc901a7aa@1704734651641832868[fbHvCR,_v2_]
20240108 17:24:11.838562 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xba261bae94f30ffa@1704734651543740257[fdfnas,_v2_]
20240108 17:24:11.838567 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x5703fec10c081a7c@1704734651436900345[fn,_v2_]
20240108 17:24:11.838572 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x9c27762a8e8e48a4@1704734651522516537[gE,_v2_]
20240108 17:24:11.838577 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x53905ac6634f382d@1704734651655504777[gRBqz,_v2_]
20240108 17:24:11.838581 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x947548c76ec5fc4@1704734651532863605[gW,_v2_]
20240108 17:24:11.838586 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x4837bd942a22be61@1704734651483877417[gg,_v2_]
20240108 17:24:11.838591 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x7db5832013b44e25@1704734651423547354[ggmdI,_v2_]
20240108 17:24:11.838596 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xf69184c44b371c66@1704734651648790888[hEdtZu,_v2_]
20240108 17:24:11.838600 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x83fb63e3036ff07e@1704734651476242712[hHvBo,_v2_]
20240108 17:24:11.838607 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x4778c30338679ac1@1704734651606648800[hMPOj,_v2_]
20240108 17:24:11.838612 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x17c3e658165b35c4@1704734651716046400[hSMCh,_v2_]
20240108 17:24:11.838617 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xdb95b9b994359604@1704734651672154013[iHwa,_v2_]
20240108 17:24:11.838621 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x32143474bbd41321@1704734651727012596[iTTz,_v2_]
20240108 17:24:11.838626 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x79a78ea4c268492c@1704734651466457439[ioD,_v2_]
20240108 17:24:11.838631 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x63d78007c656f30d@1704734651456113944[kMpZX,_v2_]
20240108 17:24:11.838635 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xf55853c29c49d9a8@1704734651698961692[lNHpk,_v2_]
20240108 17:24:11.838640 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xb26ea47e688e0aed@1704734651603773353[lh,_v2_]
20240108 17:24:11.838645 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x5c68fe9236a845f9@1704734651562509143[lm,_v2_]
20240108 17:24:11.838649 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x9fc4ce31f79ab652@1704734651645669405[ltbbN,_v2_]
20240108 17:24:11.838654 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x3de7569fe4498b94@1704734651610435180[mre,_v2_]
20240108 17:24:11.838659 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xebccd69f05811cec@1704734651617567571[oS,_v2_]
20240108 17:24:11.838664 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xd3977b9069c880ba@1704734651638653835[oexwNlI,_v2_]
20240108 17:24:11.838668 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xb79e0ccfacaa9ca@1704734651600817117[onKTl,_v2_]
20240108 17:24:11.838673 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x62768ed92301b6ce@170473465163[2201](https://github.com/man-group/ArcticDB/actions/runs/7450479371/job/20270513545#step:12:2202)709[qSPa,_v2_]
20240108 17:24:11.838678 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x50e6616b804c4e5e@1704734651536397265[qTQCtR,_v2_]
20240108 17:24:11.838682 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xef741ff22b69ef90@1704734651712835908[sK,_v2_]
20240108 17:24:11.838687 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x67b08deb65e071e@1704734651506534843[sWTTYvx,_v2_]
20240108 17:24:11.838696 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x758d39e7121df8c@1704734651614619019[sX,_v2_]
20240108 17:24:11.838700 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x549991ea053fdf36@1704734651669078404[sds,_v2_]
20240108 17:24:11.838705 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x64b692248cfc5d05@1704734651556211394[sqz,_v2_]
20240108 17:24:11.838710 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x8135f53cad469835@1704734651688580562[tOvK,_v2_]
20240108 17:24:11.838714 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x405245f497cf27d7@1704734651735856149[tsKV,_v2_]
20240108 17:24:11.838719 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x1be68b2b40c9a5c4@1704734651496816945[ujp,_v2_]
20240108 17:24:11.838724 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x137f52867c05a106@1704734651702084178[vanNp,_v2_]
20240108 17:24:11.838728 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xa89b58f73a3e48a9@1704734651500062469[vkSp,_v2_]
20240108 17:24:11.838733 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xfc3812822e181cd5@1704734651553225272[vyxL,_v2_]
20240108 17:24:11.838738 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xaad86665bfe662ba@1704734651732920797[wTkZpn,_v2_]
20240108 17:24:11.838742 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x87c9cda7724a47d1@1704734651651688963[wUsLVZ,_v2_]
20240108 17:24:11.838747 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xad16fbc2f30660fb@1704734651569446794[wvRr,_v2_]
20240108 17:24:11.838751 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x5398697a682e0940@1704734651509606975[xTqm,_v2_]
20240108 17:24:11.838756 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x869c432b9b5fd278@1704734651589940635[xaaimMZ,_v2_]
20240108 17:24:11.838761 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xf852b23913ec796a@1704734651446801659[zSFW,_v2_]
20240108 17:24:11.838765 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0xcb6b13d1660257e3@1704734651621596012[zov,_v2_]
20240108 17:24:11.838770 25385 W arcticdb.storage | Failed to delete segment for key l:__add__:0:0x37f95038376ad69b@1704734651760718526[zuxiRCz,_v2_]
20240108 17:24:11.840266 292 W arcticdb.symbol | Ignoring error while trying to compact the symbol list: Composite: l:__add__:0:0xa113080619843668@1704734651635681840[AZOOMi,_v2_], l:__add__:0:0x599659fc9851bcfe@1704734651665867885[AsRPRu,_v2_], l:__add__:0:0x3322dff958f2cfc1@1704734651565609718[CSZH,_v2_], l:__add__:0:0x3b88720[2223](https://github.com/man-group/ArcticDB/actions/runs/7450479371/job/20270513545#step:12:2224)22d5d@1704734651433651474[Cj,_v2_], l:__add__:0:0xaeffcfd71cc18088@1704734651753364797[CmB,_v2_], l:__add__:0:0xf101f5774ae72f50@1704734651427696529[DFtm,_v2_], l:__add__:0:0x48438ad12b30b347@1704734651685350917[DRlNX,_v2_], l:__add__:0:0x8937c286bdc0cbd8@1704734651678579319[FHbgTk,_v2_], l:__add__:0:0xff8d012c647f8564@1704734651462510691[Gjffa,_v2_], l:__add__:0:0xb8bf3b58b8aacca3@1704734651430422551[HqIznpv,_v2_], l:__add__:0:0x78f769d4209bbaa6@1704734651682404661[Ilwekff,_v2_], l:__add__:0:0xdc1051a52e209f7@1704734651743730515[JPt,_v2_], l:__add__:0:0xa99ff36cbc001b6c@1704734651469575357[KCIIJ,_v2_], l:__add__:0:0x9ee0e567a09d4f0d@170473465169[2224](https://github.com/man-group/ArcticDB/actions/runs/7450479371/job/20270513545#step:12:2225)366[KMQQ,_v2_], l:__add__:0:0xac049097366894b3@1704734651442772653[KOwlckR,_v2_], l:__add__:0:0x27c48396a61f51b1@17047346516953[2240](https://github.com/man-group/ArcticDB/actions/runs/7450479371/job/20270513545#step:12:2241)7[LTf,_v2_], l:__add__:0:0xe6d117b01d9ca5d@1704734651459161364[LvJf,_v2_], l:__add__:0:0xdd47acb40a511aa7@1704734651487357197[MHk,_v2_], l:__add__:0:0xb1347b13b9aa929@1704734651547023411[MI,_v2_], l:__add__:0:0x2b7b5dffca532d15@1704734651659376596[Md,_v2_], l:__add__:0:0x77287b6bcc55151b@1704734651576523519[Mp,_v2_], l:__add__:0:0x98e7c04f463b26a2@1704734651629261462[NYrbCI,_v2_], l:__add__:0:0xa0e64c269e98d036@1704734651593906118[NuV,_v2_], l:__add__:0:0x894858890ee49aa8@1704734651512534849[Optd,_v2_], l:__add__:0:0x741d00947ab966ae@1704734651662394086[OuhZ,_v2_], l:__add__:0:0x5153ab5e89693005@1704734651597579568[PGgxyWy,_v2_], l:__add__:0:0xdad67e5f1a357f04@1704734651526299200[PJ,_v2_], l:__add__:0:0xeac10edf4d992da9@1704734651572955635[QgG,_v2_], l:__add__:0:0xf3c06daa61ce5e11@1704734651746713125[QvisxQ,_v2_], l:__add__:0:0xe1e356a31019cb8f@1704734651473093196[RNQXsg,_v2_], l:__add__:0:0x5e233063ee4e7288@1704734651503352957[RRuzb,_v2_], l:__add__:0:0x8a98f0776c4a749a@1704734651516084879[RqCKbVz,_v2_], l:__add__:0:0xc0fe08278b541365@1704734651580182915[StTiDaW,_v2_], l:__add__:0:0x15aef14c1b32547a@1704734651625785079[TIkubUV,_v2_], l:__add__:0:0xba51c5d52b35eb13@1704734651550286860[TeqR,_v2_], l:__add__:0:0x4359b786679645cd@1704734651450041207[Uf,_v2_], l:__add__:0:0xaa8b8761dfdeb0ab@1704734651675605489[Uvu,_v2_], l:__add__:0:0xfa4537aa83cf9651@1704734651540184789[VNCRZly,_v2_], l:__add__:0:0xb270bb0dcb35adf2@1704734651493825182[Vxqphnt,_v2_], l:__add__:0:0x633c3fbc8162dd48@1704734651740495020[WbdnKX,_v2_], l:__add__:0:0x489e0f850e06d183@1704734651490915705[WePFnC,_v2_], l:__add__:0:0x1091f995f391d401@1704734651708836352[Wl,_v2_], l:__add__:0:0xba9e8d2b6d0db49d@1704734651479143225[XD,_v2_], l:__add__:0:0x7998ffb54dfe8012@1704734651757180438[XJXX,_v2_], l:__add__:0:0x3e0e02912450fc04@1704734651519303523[ZCuo,_v2_], l:__add__:0:0xc6f3600450590f8c@1704734651529558550[ZDYasGD,_v2_], l:__add__:0:0xebc803aca4de5a46@1704734651750122519[ZuinK,_v2_], l:__add__:0:0x79964f69f5f721e6@1704734651583263290[aCYe,_v2_], l:__add__:0:0x3cf01e357955d355@1704734651723348721[aajSn,_v2_], l:__add__:0:0xddd8dd0fc0df1ed8@1704734651559188369[cbmtv,_v2_], l:__add__:0:0x8372c31d3b25606b@1704734651453243817[ckTUNd,_v2_], l:__add__:0:0x1297eb7b46339039@1704734651586529704[dTrwHl,_v2_], l:__add__:0:0x1b50e21074820ab1@1704734651719942577[dWJcwD,_v2_], l:__add__:0:0xdc3ace4b7068e798@1704734651729965071[eXCzh,_v2_], l:__add__:0:0xd2977a168fa3a3b2@1704734651439971325[fFZRK,_v2_], l:__add__:0:0xd3feb15cc901a7aa@1704734651641832868[fbHvCR,_v2_], l:__add__:0:0xba261bae94f30ffa@1704734651543740257[fdfnas,_v2_], l:__add__:0:0x5703fec10c081a7c@1704734651436900345[fn,_v2_], l:__add__:0:0x9c27762a8e8e48a4@1704734651522516537[gE,_v2_], l:__add__:0:0x53905ac6634f382d@1704734651655504777[gRBqz,_v2_], l:__add__:0:0x947548c76ec5fc4@1704734651532863605[gW,_v2_], l:__add__:0:0x4837bd942a22be61@1704734651483877417[gg,_v2_], l:__add__:0:0x7db5832013b44e25@1704734651423547354[ggmdI,_v2_], l:__add__:0:0xf69184c44b371c66@1704734651648790888[hEdtZu,_v2_], l:__add__:0:0x83fb63e3036ff07e@1704734651476242712[hHvBo,_v2_], l:__add__:0:0x4778c30338679ac1@1704734651606648800[hMPOj,_v2_], l:__add__:0:0x17c3e658165b35c4@1704734651716046400[hSMCh,_v2_], l:__add__:0:0xdb95b9b994359604@1704734651672154013[iHwa,_v2_], l:__add__:0:0x32143474bbd41321@1704734651727012596[iTTz,_v2_], l:__add__:0:0x79a78ea4c268492c@1704734651466457439[ioD,_v2_], l:__add__:0:0x63d78007c656f30d@1704734651456113944[kMpZX,_v2_], l:__add__:0:0xf55853c29c49d9a8@1704734651698961692[lNHpk,_v2_], l:__add__:0:0xb26ea47e688e0aed@1704734651603773353[lh,_v2_], l:__add__:0:0x5c68fe9236a845f9@1704734651562509143[lm,_v2_], l:__add__:0:0x9fc4ce31f79ab652@1704734651645669405[ltbbN,_v2_], l:__add__:0:0x3de7569fe4498b94@1704734651610435180[mre,_v2_], l:__add__:0:0xebccd69f05811cec@1704734651617567571[oS,_v2_], l:__add__:0:0xd3977b9069c880ba@1704734651638653835[oexwNlI,_v2_], l:__add__:0:0xb79e0ccfacaa9ca@1704734651600817117[onKTl,_v2_], l:__add__:0:0x62768ed92301b6ce@1704734651632201709[qSPa,_v2_], l:__add__:0:0x50e6616b804c4e5e@1704734651536397265[qTQCtR,_v2_], l:__add__:0:0xef741ff22b69ef90@1704734651712835908[sK,_v2_], l:__add__:0:0x67b08deb65e071e@1704734651506534843[sWTTYvx,_v2_], l:__add__:0:0x758d39e7121df8c@1704734651614619019[sX,_v2_], l:__add__:0:0x549991ea053fdf36@1704734651669078404[sds,_v2_], l:__add__:0:0x64b69[2248](https://github.com/man-group/ArcticDB/actions/runs/7450479371/job/20270513545#step:12:2249)cfc5d05@1704734651556211394[sqz,_v2_], l:__add__:0:0x8135f53cad469835@1704734651688580562[tOvK,_v2_], l:__add__:0:0x405245f497cf27d7@1704734651735856149[tsKV,_v2_], l:__add__:0:0x1be68b2b40c9a5c4@1704734651496816945[ujp,_v2_], l:__add__:0:0x137f52867c05a106@1704734651702084178[vanNp,_v2_], l:__add__:0:0xa89b58f73a3e48a9@1704734651500062469[vkSp,_v2_], l:__add__:0:0xfc3812822e181cd5@1704734651553[2252](https://github.com/man-group/ArcticDB/actions/runs/7450479371/job/20270513545#step:12:2253)72[vyxL,_v2_], l:__add__:0:0xaad86665bfe662ba@1704734651732920797[wTkZpn,_v2_], l:__add__:0:0x87c9cda7724a47d1@1704734651651688963[wUsLVZ,_v2_], l:__add__:0:0xad16fbc2f30660fb@1704734651569446794[wvRr,_v2_], l:__add__:0:0x5398697a682e0940@1704734651509606975[xTqm,_v2_], l:__add__:0:0x869c432b9b5fd278@1704734651589940635[xaaimMZ,_v2_], l:__add__:0:0xf852b23913ec796a@1704734651446801659[zSFW,_v2_], l:__add__:0:0xcb6b13d1660257e3@1704734651621596012[zov,_v2_], l:__add__:0:0x37f95038376ad69b@1704734651760718526[zuxiRCz,_v2_],
Another instance of this happening here:
FAILED tests/integration/arcticdb/version_store/test_symbol_list.py::test_symbol_list_parallel_stress_with_delete[False-10-8-10-2] - AssertionError: assert not True
+ where True = <bound method PyCapsule.indexes_sorted of <arcticdb_ext.version_store.PythonVersionStore object at 0x7fe98097cf30>>('bHkeC')
+ where <bound method PyCapsule.indexes_sorted of <arcticdb_ext.version_store.PythonVersionStore object at 0x7fe98097cf30>> = <arcticdb_ext.version_store.PythonVersionStore object at 0x7fe98097cf30>.indexes_sorted
+ where <arcticdb_ext.version_store.PythonVersionStore object at 0x7fe98097cf30> = NativeVersionStore: Library: test_symbol_list_parallel_stre.266_2024-02-26T15_32_32_897272, Primary Storage: lmdb_storage..version_store
It is getting quite flaky recently and it is not limited to only [False-2-3-10-2]. Hit it multiple times with [True-2-8-2-100] as well