lightninglabs
[bug]: Multiple asset channels have been force-closed
litd -- v0.15.0-alpha tapd -- v0.6.0 lnd -- v0.19.1-beta
My node has established one BTC channel + one asset channel with multiple peers. Every 10 minutes, I use Keysend to send an asset payment to all peer asset channels. Since each asset transfer also includes 354 sats, the peers respond by sending back 354 sats to me via Keysend.
However, I don’t understand why some of these asset channels suddenly entered a force-closed state. Out of 200 asset channels, 7 have been force-closed, and they remain stuck in an unresolved state (unable to fully close).
"pending_force_closing_channels": [
{
"channel": {
"remote_node_pub": "02094b89eca31f623af2d4fe2bf0fe52828c9816bf23573bb94f3938ce44c5a621",
"channel_point": "02e194f26ed80c39363f3bf889271525ea98f37e1aba3656596c0a7d43d40c09:0",
"capacity": "20000",
"local_balance": "8855",
"remote_balance": "10000",
"local_chan_reserve_sat": "0",
"remote_chan_reserve_sat": "0",
"initiator": "INITIATOR_LOCAL",
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"num_forwarding_packages": "0",
"chan_status_flags": "",
"private": true,
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9837900184
}
],
"remote_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 162099816
}
],
"outgoing_htlcs": [],
"incoming_htlcs": [],
"capacity": 10000000000,
"local_balance": 9837900184,
"remote_balance": 162099816,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 0
}
},
"closing_txid": "937ab4d5c847eb7bde242e16a64f0a6d794dd3dba976b1b327a06f77b2feacc6",
"limbo_balance": "8855",
"maturity_height": 904956,
"blocks_til_maturity": -55,
"recovered_balance": "330",
"pending_htlcs": [],
"anchor": "RECOVERED"
},
{
"channel": {
"remote_node_pub": "020686ae2a3a54897ef8098ff3ac0541cd09b2ce6296dc8d5c8b757d5ce14d6d96",
"channel_point": "afc7179f513ed1651d0a950a184a05c68fc2b080b777d021418d7a9071dc4964:0",
"capacity": "20000",
"local_balance": "8088",
"remote_balance": "10708",
"local_chan_reserve_sat": "0",
"remote_chan_reserve_sat": "0",
"initiator": "INITIATOR_LOCAL",
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"num_forwarding_packages": "0",
"chan_status_flags": "",
"private": true,
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9999619484
}
],
"remote_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 380516
}
],
"outgoing_htlcs": [],
"incoming_htlcs": [],
"capacity": 10000000000,
"local_balance": 9999619484,
"remote_balance": 380516,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 0
}
},
"closing_txid": "70aa87e5a77013fcc51af00c3b2fb0c17aefc9116951d3a7b35f31d42a187bc7",
"limbo_balance": "8418",
"maturity_height": 904806,
"blocks_til_maturity": -205,
"recovered_balance": "0",
"pending_htlcs": [],
"anchor": "LIMBO"
},
{
"channel": {
"remote_node_pub": "021b161cc6bbdce722d07514f9c72badb6843433084d4c67bd1842fc48c2e363b6",
"channel_point": "db5b335293c4f7d736e5dab6ac2972f7c7023d0680dde71d9dccfb74d9125581:0",
"capacity": "20000",
"local_balance": "8855",
"remote_balance": "10000",
"local_chan_reserve_sat": "0",
"remote_chan_reserve_sat": "0",
"initiator": "INITIATOR_LOCAL",
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"num_forwarding_packages": "0",
"chan_status_flags": "",
"private": true,
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9814878966
}
],
"remote_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 185121034
}
],
"outgoing_htlcs": [],
"incoming_htlcs": [],
"capacity": 10000000000,
"local_balance": 9814878966,
"remote_balance": 185121034,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 0
}
},
"closing_txid": "6233e886a9d39c0d40cac8997c2f91b8224b70eb055440982c38580b5ca68596",
"limbo_balance": "8855",
"maturity_height": 904956,
"blocks_til_maturity": -55,
"recovered_balance": "330",
"pending_htlcs": [],
"anchor": "RECOVERED"
},
{
"channel": {
"remote_node_pub": "0319698624fba54edce6626bbfd7d0a74e2359436272efedbf35467d866cb57c01",
"channel_point": "00a26943dd10aa14368f7214a2710596cae67ec237ddb94ae7994e5e9909a5c2:0",
"capacity": "20000",
"local_balance": "8855",
"remote_balance": "10000",
"local_chan_reserve_sat": "0",
"remote_chan_reserve_sat": "0",
"initiator": "INITIATOR_LOCAL",
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"num_forwarding_packages": "0",
"chan_status_flags": "",
"private": true,
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9976978782
}
],
"remote_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 23021218
}
],
"outgoing_htlcs": [],
"incoming_htlcs": [],
"capacity": 10000000000,
"local_balance": 9976978782,
"remote_balance": 23021218,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 0
}
},
"closing_txid": "ad547d90dc2463d3eb55dc72baedc10d4bfef88d41c38381b5fcf3373f65bf9b",
"limbo_balance": "8855",
"maturity_height": 905068,
"blocks_til_maturity": 57,
"recovered_balance": "330",
"pending_htlcs": [],
"anchor": "RECOVERED"
},
{
"channel": {
"remote_node_pub": "03c1ea3853a96531f72954e0f4f66bcb3e0542f10c46c8b0da0e6f3191e97dbb88",
"channel_point": "d733afb11ef653f8063770605a23931258b886df77a4f9c91b2fc14db2447bce:0",
"capacity": "20000",
"local_balance": "8796",
"remote_balance": "10000",
"local_chan_reserve_sat": "0",
"remote_chan_reserve_sat": "0",
"initiator": "INITIATOR_LOCAL",
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"num_forwarding_packages": "0",
"chan_status_flags": "",
"private": true,
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9950913436
}
],
"remote_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 49086564
}
],
"outgoing_htlcs": [],
"incoming_htlcs": [],
"capacity": 10000000000,
"local_balance": 9950913436,
"remote_balance": 49086564,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 0
}
},
"closing_txid": "af7313d030694612df2ab0e363062413d274860fca844e74febea1e4df99cdbd",
"limbo_balance": "9126",
"maturity_height": 904835,
"blocks_til_maturity": -176,
"recovered_balance": "0",
"pending_htlcs": [],
"anchor": "LIMBO"
},
{
"channel": {
"remote_node_pub": "03c8a8d79e54eaee93c2b9ecc0fd4d2224c1a4e31c5e12bedc62ee0fb9b6e2ca69",
"channel_point": "032d3b9165353be967a2891a2b6e4c578b181ceae62165a0423e1da2f76cdada:0",
"capacity": "20000",
"local_balance": "8856",
"remote_balance": "10000",
"local_chan_reserve_sat": "0",
"remote_chan_reserve_sat": "0",
"initiator": "INITIATOR_LOCAL",
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"num_forwarding_packages": "0",
"chan_status_flags": "",
"private": true,
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9999238968
}
],
"remote_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 761032
}
],
"outgoing_htlcs": [],
"incoming_htlcs": [],
"capacity": 10000000000,
"local_balance": 9999238968,
"remote_balance": 761032,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 0
}
},
"closing_txid": "ef93c27b2fa151959f0d711a9498c0896bd607d04671963da095cf96bc43740b",
"limbo_balance": "9186",
"maturity_height": 905005,
"blocks_til_maturity": -6,
"recovered_balance": "0",
"pending_htlcs": [],
"anchor": "LIMBO"
},
{
"channel": {
"remote_node_pub": "0213843878d8c02d753966c985a1159ea6d7b61f92a7e8193142258ff3666bb180",
"channel_point": "f290d6b997fd0294cdb847d4beb4655c364319d910c6c1cd08e77d40aa39c1e6:0",
"capacity": "20000",
"local_balance": "8855",
"remote_balance": "10000",
"local_chan_reserve_sat": "0",
"remote_chan_reserve_sat": "0",
"initiator": "INITIATOR_LOCAL",
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"num_forwarding_packages": "0",
"chan_status_flags": "",
"private": true,
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9896119132
}
],
"remote_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 103880868
}
],
"outgoing_htlcs": [],
"incoming_htlcs": [],
"capacity": 10000000000,
"local_balance": 9896119132,
"remote_balance": 103880868,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 0
}
},
"closing_txid": "44529b0324f3f94f60db65bbed09dce0dc9d15adf2b7d8b183ae9f2e225255f5",
"limbo_balance": "8855",
"maturity_height": 905068,
"blocks_til_maturity": 57,
"recovered_balance": "330",
"pending_htlcs": [],
"anchor": "RECOVERED"
}
],
"waiting_close_channels": []
}
This is the log from the time period when one of the asset channels was force-closed. I extracted the logs from 2 minutes before and after the timestamp of the closing_txid (44529b0324f3f94f60db65bbed09dce0dc9d15adf2b7d8b183ae9f2e225255f5) on my node.
This is the log snippet from the time window when one of the asset channels was force-closed. I’ve extracted logs covering 2 minutes before and after the timestamp of the closing_txid (44529b0324f3f94f60db65bbed09dce0dc9d15adf2b7d8b183ae9f2e225255f5) on my node.
I haven't yet obtained the counterparty's logs, but I'll share them as soon as they become available. Please let me know if you need any additional information - I'll be happy to provide whatever is needed for troubleshooting.
I located the counterparty's log records, but their node appears to have been offline during the channel closure timeframe - there are no corresponding log entries from that period. The channel that was force-closed is shown in the following segment:
"pending_force_closing_channels": [
{
"channel": {
"remote_node_pub": "02c953421bc7f07be6052920e46843d11e6d3ffc9986177c91f140d76c6ed3a3d4",
"channel_point": "00a26943dd10aa14368f7214a2710596cae67ec237ddb94ae7994e5e9909a5c2:0",
"capacity": "20000",
"local_balance": "10000",
"remote_balance": "8855",
"local_chan_reserve_sat": "0",
"remote_chan_reserve_sat": "0",
"initiator": "INITIATOR_REMOTE",
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"num_forwarding_packages": "0",
"chan_status_flags": "",
"private": true,
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 23021218
}
],
"remote_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9976978782
}
],
"outgoing_htlcs": [],
"incoming_htlcs": [],
"capacity": 10000000000,
"local_balance": 23021218,
"remote_balance": 9976978782,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 0
}
},
"closing_txid": "ad547d90dc2463d3eb55dc72baedc10d4bfef88d41c38381b5fcf3373f65bf9b",
"limbo_balance": "10330",
"maturity_height": 0,
"blocks_til_maturity": 0,
"recovered_balance": "0",
"pending_htlcs": [],
"anchor": "LIMBO"
}
],
"waiting_close_channels": []
}
I observed that the closing transaction (closing_txid: ad547d90dc2463d3eb55dc72baedc10d4bfef88d41c38381b5fcf3373f65bf9b) was initiated at 2025-07-10 21:45:42, but there are no corresponding log entries from that exact timestamp in node's records.
Hi. Thanks for the report. It's possible you ran into the channel connection issue with lnd v0.19.0. This has been fixed in v0.19.1, but we haven't bundled that into a litd version yet. We'll do that soon.
But to confirm, can you please send us more of your log? The actual force close seems to have happened before the timestamp you mentioned, so it's not in the file you sent.
@shuishihang how many peers does your node have?
Looking at the latest logs you sent, it seems like the node may have having some network connectivity issues:
2025-07-11 11:10:46.572 [ERR] CNCT: Notify blockbeat for ChannelArbitrator failed: consumer ChannelArbitrator(00a26943dd10aa14368f7214a2710596cae67ec237ddb94ae7994e5e9909a5c2:0): process block timeout
2025-07-11 11:10:46.572 [ERR] CHIO: Height[905016]: Failed to process block: consumer ChainArbitrator: process block timeout
2025-07-11 11:10:46.572 [ERR] CHIO: Height[905016]: Notify block failed: queue=1 got err: consumer ChainArbitrator: process block timeout
2025-07-11 11:10:46.572 [INF] CNCT: ChannelArbitrator(c0a0d3a5735ccbdf9ff226fb847c98b80f55eb28a578f8aca93174a196462bc3:1) no close event
2025-07-11 11:10:47.164 [INF] SRVR: Established connection to: 02c953421bc7f07be6052920e46843d11e6d3ffc9986177c91f140d76c6ed3a3d4@X:9735
AFAICT, none of the on-chain resolves that handle output sweeping are getting notifications for new blocks.
My node may have over 300 peer connections.Why are these force-closed channels stuck in an unresolved state? Attached are the complete logs from the timeframe of the problematic node (closing_txid: ad547d90dc2463d3eb55dc72baedc10d4bfef88d41c38381b5fcf3373f65bf9b). I’m unsure if this contains the specific information you need—the logs are extensive, and I can’t pinpoint the exact timestamp.
Another channel on my node has entered the waiting_close_channels state. Below are its details and the logs recorded during that timeframe.
"waiting_close_channels": [
{
"channel": {
"remote_node_pub": "025eb73588b8a32b063a6d9eb0fdf22d82ee850c35f779aa682361a807a464ff32",
"channel_point": "b042ab0ab4d880ab5dba703a8dd4df587ac46ca6eab47fab0a380d58a40868e2:0",
"capacity": "20000",
"local_balance": "7252",
"remote_balance": "10000",
"local_chan_reserve_sat": "354",
"remote_chan_reserve_sat": "1062",
"initiator": "INITIATOR_LOCAL",
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"num_forwarding_packages": "0",
"chan_status_flags": "ChanStatusBorked|ChanStatusCommitBroadcasted|ChanStatusLocalCloseInitiator",
"private": true,
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9985730650
}
],
"remote_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 14269350
}
],
"outgoing_htlcs": [],
"incoming_htlcs": [],
"capacity": 10000000000,
"local_balance": 9985730650,
"remote_balance": 14269350,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 0
}
},
"limbo_balance": "7252",
"commitments": {
"local_txid": "3be78770eb223e9a7930d8698811185bf9f975fd00ca9861426b03f4cf9d975b",
"remote_txid": "daef6206d1a8cedf8032108805826b102a9a87e50ec72ddc7a8fd1156f6095fc",
"remote_pending_txid": "428bc2b60ba86532e139c08ff66dfef356f5331023511cbf8e19bfc030c29e5d",
"local_commit_fee_sat": "672",
"remote_commit_fee_sat": "485",
"remote_pending_commit_fee_sat": "0"
},
"closing_txid": "3be78770eb223e9a7930d8698811185bf9f975fd00ca9861426b03f4cf9d975b",
"closing_tx_hex": ""
}
]
}
Thanks for the logs. Unfortunately they still don't contain the actual cause for the force close. The earliest mention of one of the channels (for example afc7179f513ed1651d0a950a184a05c68fc2b080b777d021418d7a9071dc4964:0) is at 2025-07-10 20:29:28.784, but it looks like at that point the channel was already force closed:
2025-07-10 20:29:28.784 [INF] CNCT: ChannelArbitrator(afc7179f513ed1651d0a950a184a05c68fc2b080b777d021418d7a9071dc4964:0): skipping reading close events in state=StateWaitingFullResolution
Do you have log files that go further back than that date?
I'm still looking into why the channel state isn't resolved. Unfortunately there are no error messages related to that in either log file.
I also saw that you're using a custom build of litd with modifications. Can you confirm you're running the correct commit of tapd? If yes, can you please confirm the actual commit you compiled your litd with?
litd -- v0.15.0-alpha I have a closed asset channel (closing_txid: bbc9e6c24018088c58a2be957c07c76836cbcc8feb2d70b44f541d2bd2e505a8, timestamped 2025-07-14 02:18:55). Attached are the logs generated during the 3-hour window around its closure. These are the only recent logs I could retrieve for a closed asset channel, as older logs were automatically purged.
litd -- v0.15.0-alpha litd——v0.15.0-alpha I have a closed asset channel (closing_txid: bbc9e6c24018088c58a2be957c07c76836cbcc8feb2d70b44f541d2bd2e505a8, timestamped 2025-07-14 02:18:55). Attached are the logs generated during the 3-hour window around its closure. These are the only recent logs I could retrieve for a closed asset channel, as older logs were automatically purged.我有一个已关闭的资产通道（closing_txid：bbc9e6c24018088c58a2be957c07c76836cbcc8feb2d70b44f541d2bd2e505a8，时间戳为 2025-07-14 02:18:55）。附件是该通道关闭前后 3 小时内生成的日志。这些是我能检索到的关于已关闭资产通道的最新日志，因为旧日志已被自动清除。
If another asset channel gets force-closed again, I will carefully monitor the situation and provide you with all relevant logs for analysis
I'm not sure how you timestamp the closing transaction. But the channel 0b5fc3a49ecaa2a3420e84c7a7085df69fc41326abf5ee1c6d212d9ac4d95e54:0 was closed on 2025-07-13 (the log file is always in UTC). So again, the actual force close of 0b5fc3a49ecaa2a3420e84c7a7085df69fc41326abf5ee1c6d212d9ac4d95e54:0 (closing txid bbc9e6c24018088c58a2be957c07c76836cbcc8feb2d70b44f541d2bd2e505a8) is not in the files you sent...
What about the other questions above?
😅I searched for the closing_txid (ad547d90dc2463d3eb55dc72baedc10d4bfef88d41c38381b5fcf3373f65bf9b) in the mempool to check its timestamp, and I might have indeed mistaken the actual closure time. I’ll pay closer attention to this issue if another force-close occurs. However, I still don’t understand why the previously force-closed asset channels remain stuck in an unresolved state.
{
"channel": {
"remote_node_pub": "03e13380639fbf0a3a990e74185ff966bc87780f448d2e8653900eff8e47b047bf",
"channel_point": "9fa4ccbf4e0d48ae32e8bcbc0292e4b4c95aa8b948c70af528e49bd6d0faa0c8:0",
"capacity": "20000",
"local_balance": "8102",
"remote_balance": "10000",
"local_chan_reserve_sat": "0",
"remote_chan_reserve_sat": "0",
"initiator": "INITIATOR_LOCAL",
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"num_forwarding_packages": "0",
"chan_status_flags": "",
"private": true,
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9784437686
}
],
"remote_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 215562314
}
],
"outgoing_htlcs": [],
"incoming_htlcs": [],
"capacity": 10000000000,
"local_balance": 9784437686,
"remote_balance": 215562314,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 0
}
},
"closing_txid": "68e3255acfd9a3060a7afa3fe2db4c2e31deb9e688f2cbc56a5d6689bcb4b505",
"limbo_balance": "8432",
"maturity_height": 905673,
"blocks_til_maturity": 81,
"recovered_balance": "0",
"pending_htlcs": [],
"anchor": "LIMBO"
},
{
"channel": {
"remote_node_pub": "0280d7d16f739a5d137a0ba5ddc244a520337b787628e6c33c1a7194b1640b7176",
"channel_point": "9fb7c2c513cddfe03ede9fd5fdb1a17aa667e3eb51cbcbf5ee2c4cce0d7eb0be:0",
"capacity": "20000",
"local_balance": "8039",
"remote_balance": "10000",
"local_chan_reserve_sat": "0",
"remote_chan_reserve_sat": "0",
"initiator": "INITIATOR_LOCAL",
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"num_forwarding_packages": "0",
"chan_status_flags": "",
"private": true,
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9786720782
}
],
"remote_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 213279218
}
],
"outgoing_htlcs": [],
"incoming_htlcs": [],
"capacity": 10000000000,
"local_balance": 9786720782,
"remote_balance": 213279218,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 0
}
},
"closing_txid": "597029f2f4268f32069e1134b1e97c82469feac9a3a66180fba2264542bf19ca",
"limbo_balance": "8369",
"maturity_height": 905674,
"blocks_til_maturity": 29,
"recovered_balance": "0",
"pending_htlcs": [],
"anchor": "LIMBO"
},
This time, I found the force-closure information of this asset channel in the logs.
2025-07-14 23:17:59.730 [INF] CNCT: ChannelArbitrator(9fa4ccbf4e0d48ae32e8bcbc0292e4b4c95aa8b948c70af528e49bd6d0faa0c8:0): fail dangling htlc=0c052c70aaffcade1096a4a74dd9a13ad9dcf1cd382a0596edb7ab789bb68226 from local/remote commitments diff
Can you please just give me the full log file with everything as far back as possible? You can zip it if there's a size limit (or send it to me on another platform). It's really hard to find the cause for something that happens sporadically when we don't have the full log.
Also, please answer the other questions above. And having a bit more detail about your node also wouldn't help. What chain backend? Are some of your peers also nodes you control? For example, the log you provided of one peer that runs on Neutrino with many hundreds of channels... That's not really a stable configuration IMO, is that also one of your nodes?
I don’t have complete log records because my node automatically purges oversized LND logs. When the issue recurs, I’ll compile and share all existing intact logs. My node runs on Bitcoind, while my ~300 peers use Neutrino. These peers only open one asset channel and one BTC channel with my node. I did use a custom-modified LiTd version (v0.15.0-alpha), but without altering core functionalities. The Tapd commit is correct, and LiTd was compiled from the v0.15.0-alpha source. In Tapd, I only modified the satoshi capacity parameter for asset channel openings, without altering any other channel-related functionalities
Below are the forcibly closed asset channels that occurred today. The earliest logs I currently have cover from last night at 8 PM up to the present.
{
"channel": {
"remote_node_pub": "02246a884d38c94bdb3101890d2500749f734d843ea891c5399072f5237ce1300f",
"channel_point": "515a775c47984d8a72fb50f189fb36fdf7c1c3059bfb012a632b602592232147:0",
"capacity": "20000",
"local_balance": "7598",
"remote_balance": "10000",
"local_chan_reserve_sat": "0",
"remote_chan_reserve_sat": "0",
"initiator": "INITIATOR_LOCAL",
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"num_forwarding_packages": "0",
"chan_status_flags": "",
"private": true,
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9990106584
}
],
"remote_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9893416
}
],
"outgoing_htlcs": [],
"incoming_htlcs": [],
"capacity": 10000000000,
"local_balance": 9990106584,
"remote_balance": 9893416,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 0
}
},
"closing_txid": "7dbc18e80df8c0fd12d0cff54e7a946eea78cace4189daf3ae23a313bc36f0f7",
"limbo_balance": "7928",
"maturity_height": 906164,
"blocks_til_maturity": 82,
"recovered_balance": "0",
"pending_htlcs": [],
"anchor": "LIMBO"
},
{
"channel": {
"remote_node_pub": "02ce0e0d0b8d48535f979c1823ed500b7280885615b41a7bfdf4596536273644aa",
"channel_point": "d2ae128cbfa80a16cd45f9b5e84ac82627e90a4712dece668d24d604f9a5915c:0",
"capacity": "20000",
"local_balance": "7659",
"remote_balance": "10000",
"local_chan_reserve_sat": "0",
"remote_chan_reserve_sat": "0",
"initiator": "INITIATOR_LOCAL",
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"num_forwarding_packages": "0",
"chan_status_flags": "",
"private": true,
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9733448542
}
],
"remote_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 266551458
}
],
"outgoing_htlcs": [],
"incoming_htlcs": [],
"capacity": 10000000000,
"local_balance": 9733448542,
"remote_balance": 266551458,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 0
}
},
"closing_txid": "6f751a1c6997f73471bbba3bfdcd4fde2ce6986b9d1a962b6405a58f0fced42f",
"limbo_balance": "7989",
"maturity_height": 906163,
"blocks_til_maturity": 81,
"recovered_balance": "0",
"pending_htlcs": [],
"anchor": "LIMBO"
},
{
"channel": {
"remote_node_pub": "0284430163650adb660c69ac55a58e1b697ca6524bf8a43f12ab67ac089c9e48b5",
"channel_point": "82d08f9b1ae941ca7cbd8b1abe09c81e463e4f4971c48e873b24138e253715c5:0",
"capacity": "20000",
"local_balance": "7660",
"remote_balance": "10000",
"local_chan_reserve_sat": "0",
"remote_chan_reserve_sat": "0",
"initiator": "INITIATOR_LOCAL",
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"num_forwarding_packages": "0",
"chan_status_flags": "",
"private": true,
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9948630340
}
],
"remote_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 51369660
}
],
"outgoing_htlcs": [],
"incoming_htlcs": [],
"capacity": 10000000000,
"local_balance": 9948630340,
"remote_balance": 51369660,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 0
}
},
"closing_txid": "f318fe0be64ce75d215357cef3c908e1b7f88f2f2c25af47d97400567abe6f68",
"limbo_balance": "7990",
"maturity_height": 906166,
"blocks_til_maturity": 84,
"recovered_balance": "0",
"pending_htlcs": [],
"anchor": "LIMBO"
}
Can you please actually add the logs? All I see is the same comment three times...
Thanks for the logs. So it really looks like your node is somehow locked up and the HTLCs aren't getting processed (which eventually leads to force closes).
When did you last restart the node?
And did you update to the latest litd v0.15.1-alpha? If not, that would be my recommendation.
And then after update/restart, do you see any outputs in lncli wallet pendingsweeps?
Below is the output I observed after restarting with the updated version, using the command lncli wallet pendingsweeps,Can those asset channels that entered forced closure still be closed properly?
{
"pending_sweeps": [
{
"outpoint": "d5e484b386e0c11b17abdda7ce7114d3e2a71ba9b03fbf380afd895c5636b2ff:2",
"witness_type": "TAPROOT_LOCAL_COMMIT_SPEND",
"amount_sat": 8933,
"sat_per_vbyte": 0,
"broadcast_attempts": 0,
"requested_sat_per_vbyte": 0,
"immediate": false,
"budget": 4467,
"deadline_height": 907623,
"maturity_height": 906615,
"next_broadcast_height": 0,
"requested_conf_target": 0,
"force": false
},
{
"outpoint": "1b29c29af2492e9838904c33795f59fc1047b3f456059b8857d3a6ec3771ec60:2",
"witness_type": "TAPROOT_LOCAL_COMMIT_SPEND",
"amount_sat": 9051,
"sat_per_vbyte": 0,
"broadcast_attempts": 0,
"requested_sat_per_vbyte": 0,
"immediate": false,
"budget": 4526,
"deadline_height": 907720,
"maturity_height": 906712,
"next_broadcast_height": 0,
"requested_conf_target": 0,
"force": false
},
{
"outpoint": "e4957b95d2903ab1eaaf9f5f6f65be2892bed08300cf18c16d5eba36907d0496:2",
"witness_type": "TAPROOT_LOCAL_COMMIT_SPEND",
"amount_sat": 8957,
"sat_per_vbyte": 0,
"broadcast_attempts": 0,
"requested_sat_per_vbyte": 0,
"immediate": false,
"budget": 4479,
"deadline_height": 907670,
"maturity_height": 906662,
"next_broadcast_height": 0,
"requested_conf_target": 0,
"force": false
},
{
"outpoint": "3be78770eb223e9a7930d8698811185bf9f975fd00ca9861426b03f4cf9d975b:1",
"witness_type": "TAPROOT_ANCHOR_SWEEP_SPEND",
"amount_sat": 330,
"sat_per_vbyte": 0,
"broadcast_attempts": 1,
"requested_sat_per_vbyte": 0,
"immediate": false,
"budget": 684,
"deadline_height": 906608,
"maturity_height": 906607,
"next_broadcast_height": 0,
"requested_conf_target": 0,
"force": false
},
{
"outpoint": "e5edc0a6167f6f3bf28953cb1f253205bc98f42d1933882c0544d7146104fff9:2",
"witness_type": "TAPROOT_LOCAL_COMMIT_SPEND",
"amount_sat": 9008,
"sat_per_vbyte": 0,
"broadcast_attempts": 0,
"requested_sat_per_vbyte": 0,
"immediate": false,
"budget": 4504,
"deadline_height": 907743,
"maturity_height": 906735,
"next_broadcast_height": 0,
"requested_conf_target": 0,
"force": false
},
{
"outpoint": "01603dd1902b9e1be410287999e1d4a7bdf4ff88cc7c38f7d877109bfea00ebc:2",
"witness_type": "TAPROOT_LOCAL_COMMIT_SPEND",
"amount_sat": 8118,
"sat_per_vbyte": 0,
"broadcast_attempts": 0,
"requested_sat_per_vbyte": 0,
"immediate": false,
"budget": 4059,
"deadline_height": 907636,
"maturity_height": 906628,
"next_broadcast_height": 0,
"requested_conf_target": 0,
"force": false
},
{
"outpoint": "daef6206d1a8cedf8032108805826b102a9a87e50ec72ddc7a8fd1156f6095fc:0",
"witness_type": "TAPROOT_ANCHOR_SWEEP_SPEND",
"amount_sat": 330,
"sat_per_vbyte": 0,
"broadcast_attempts": 1,
"requested_sat_per_vbyte": 0,
"immediate": false,
"budget": 330,
"deadline_height": 907615,
"maturity_height": 906607,
"next_broadcast_height": 0,
"requested_conf_target": 0,
"force": false
},
{
"outpoint": "c73e6019f4e386f32fb314a84848ad2bfcfd3ad9e1eba85c95a29f0fbef84da4:2",
"witness_type": "TAPROOT_LOCAL_COMMIT_SPEND",
"amount_sat": 9008,
"sat_per_vbyte": 0,
"broadcast_attempts": 0,
"requested_sat_per_vbyte": 0,
"immediate": false,
"budget": 4504,
"deadline_height": 907746,
"maturity_height": 906738,
"next_broadcast_height": 0,
"requested_conf_target": 0,
"force": false
},
{
"outpoint": "428bc2b60ba86532e139c08ff66dfef356f5331023511cbf8e19bfc030c29e5d:0",
"witness_type": "TAPROOT_ANCHOR_SWEEP_SPEND",
"amount_sat": 330,
"sat_per_vbyte": 0,
"broadcast_attempts": 1,
"requested_sat_per_vbyte": 0,
"immediate": false,
"budget": 330,
"deadline_height": 907615,
"maturity_height": 906607,
"next_broadcast_height": 0,
"requested_conf_target": 0,
"force": false
},
{
"outpoint": "52a79aabec5307c33d83ac8f726976a050fff44b3261457b5f23984fca3bd7be:2",
"witness_type": "TAPROOT_LOCAL_COMMIT_SPEND",
"amount_sat": 9008,
"sat_per_vbyte": 0,
"broadcast_attempts": 0,
"requested_sat_per_vbyte": 0,
"immediate": false,
"budget": 4504,
"deadline_height": 907741,
"maturity_height": 906733,
"next_broadcast_height": 0,
"requested_conf_target": 0,
"force": false
}
]
}
Hmm, this looks similar to the issue https://github.com/lightningnetwork/lnd/issues/10053. There is a workaround (increasing your bitcoin node's -minrelaytxfee configuration). See this conversation here: https://github.com/lightningnetwork/lnd/discussions/10051#discussioncomment-13696583.
I encountered an issue after setting the -minrelaytxfee parameter to minrelaytxfee=0.00003 in my Bitcoin node. When attempting to open an asset channel, I received the error: rpc error: code = Unknown desc = error funding channel: error requesting delivery: unable to broadcast transaction c2323b368bd2dc6cb66b746b37ad521030c1f3bf4cb9f45f97abeb5bf24cefff: rpc error: code = Unknown desc = undefined: min relay fee not met.
The channel entered a pending state, but the funding transaction never appeared in the mempool. This happened multiple times. Even after removing the -minrelaytxfee configuration, these pending transactions still did not show up in the mempool.
"channel_point": "383d5e4b23daea0a8dc280dd42818056284e183870a76bc6a91c12f27001b7db:0"
"channel_point": "87f052b7cdfd6154e5de58b951d96da8cff9b649764d20aa43dc4d71a986db0a:0"
"channel_point": "488487f538fd7dea5f0d419124ddac063dae58137430c2589a29cf336c68c7bf:0"
"channel_point": "c2323b368bd2dc6cb66b746b37ad521030c1f3bf4cb9f45f97abeb5bf24cefff:0"
"channel_point": "c84523546eecde4d8f499ea2ec53566850395dc6776472a799542af75fbee561:0"
Did you specify --sat_per_vbyte to a higher value when opening the channel?
You can restart your node after removing the minrelaytxfee param to re-broadcast them.
Did the setting change anything related to the pending channels?
I set the --sat_per_vbyte parameter with a minimum of 3 and maximum of 8. After removing the minrelaytxfee setting and restarting the Bitcoin node, I still can't locate these transactions. This setting doesn't seem to have any effect on channels in pending_close state - perhaps the fees I set were too small
Or the transactions are conflicting with each other as they spent the same coins? Do you see any errors related to the attempted re-publish on startup?
Such logs appear repeatedly
2025-07-29 14:36:11.696 [WRN] BTWL: Transaction 84216b2231c0e917ce66973aa048107403f7f56254733fa76f175029c703a81b not accepted by mempool: txn-already-known
2025-07-29 14:36:11.700 [INF] BTWL: Inserting unconfirmed transaction 84216b2231c0e917ce66973aa048107403f7f56254733fa76f175029c703a81b
2025-07-29 14:36:11.709 [ERR] BTCN: Broadcast attempt failed: <nil>
2025-07-29 14:36:11.717 [WRN] BTWL: Transaction 3be78770eb223e9a7930d8698811185bf9f975fd00ca9861426b03f4cf9d975b not accepted by mempool: txn-already-known
2025-07-29 14:36:11.720 [WRN] BTWL: Transaction c84523546eecde4d8f499ea2ec53566850395dc6776472a799542af75fbee561 not accepted by mempool: missing-inputs
2025-07-29 14:36:11.720 [ERR] RPCS: [/walletrpc.WalletKit/PublishTransaction]: transaction rejected: output already spent
2025-07-29 14:36:11.720 [ERR] BTCN: Broadcast attempt failed: <nil>
2025-07-29 14:36:11.721 [WRN] BTWL: Transaction 18b7055ef696be2a4a0751429c992d5e33655adb61f0d22ba22064937d119dc7 not accepted by mempool: txn-already-in-mempool
2025-07-29 14:36:11.721 [ERR] BTCN: Broadcast attempt failed: <nil>
2025-07-29 14:36:11.722 [WRN] BTWL: Transaction 87f052b7cdfd6154e5de58b951d96da8cff9b649764d20aa43dc4d71a986db0a not accepted by mempool: missing-inputs
2025-07-29 14:36:11.722 [ERR] RPCS: [/walletrpc.WalletKit/PublishTransaction]: transaction rejected: output already spent
2025-07-29 14:36:11.723 [ERR] BTCN: Broadcast attempt failed: <nil>
2025-07-29 14:36:11.723 [WRN] BTWL: Transaction 383d5e4b23daea0a8dc280dd42818056284e183870a76bc6a91c12f27001b7db not accepted by mempool: missing-inputs
2025-07-29 14:36:11.723 [ERR] RPCS: [/walletrpc.WalletKit/PublishTransaction]: transaction rejected: output already spent
2025-07-29 14:36:11.726 [WRN] BTWL: Transaction 488487f538fd7dea5f0d419124ddac063dae58137430c2589a29cf336c68c7bf not accepted by mempool: missing-inputs
2025-07-29 14:36:11.726 [ERR] RPCS: [/walletrpc.WalletKit/PublishTransaction]: transaction rejected: output already spent
2025-07-29 14:36:11.727 [ERR] BTCN: Broadcast attempt failed: <nil>
2025-07-29 14:36:11.728 [WRN] BTWL: Transaction 7399a0c323c3df3d82ef976a1cb63418169c96fca8c51a9e88a26d3b1fad0c2e not accepted by mempool: txn-already-in-mempool
2025-07-29 14:36:11.728 [WRN] BTWL: Transaction c2323b368bd2dc6cb66b746b37ad521030c1f3bf4cb9f45f97abeb5bf24cefff not accepted by mempool: missing-inputs
2025-07-29 14:36:11.728 [ERR] RPCS: [/walletrpc.WalletKit/PublishTransaction]: transaction rejected: output already spent
2025-07-29 14:36:11.738 [INF] BTWL: Inserting unconfirmed transaction 3be78770eb223e9a7930d8698811185bf9f975fd00ca9861426b03f4cf9d975b
2025-07-29 14:36:11.761 [INF] BTWL: Inserting unconfirmed transaction 18b7055ef696be2a4a0751429c992d5e33655adb61f0d22ba22064937d119dc7
2025-07-29 14:36:11.781 [INF] BTWL: Inserting unconfirmed transaction 7399a0c323c3df3d82ef976a1cb63418169c96fca8c51a9e88a26d3b1fad0c2e
2025-07-29 14:36:11.819 [INF] NTFN: New confirmation subscription: conf_id=9, txid=18b7055ef696be2a4a0751429c992d5e33655adb61f0d22ba22064937d119dc7, num_confs=6 height_hint=907649
Yeah, then the inputs were already spent and those channels will never make it. You can use lncli abandonchannel on them (make sure to read lncli abandonchannel --help).
After using Keysend, the recipient has not successfully received the payment. I checked the asset channel and found that my local display shows the channel is online, but remotely it appears offline.
{
"active": true,
"remote_pubkey": "02bda35583eb853b235e14cc3edd24a919f68fbd790a7ca0fe742f9b46ef1e429a",
"channel_point": "9dcdc207b0f8ae16a87c6e98d0213f1fa16bacf3e78d39ef12a999d289c8c0bd:0",
"chan_id": "bdc0c889d299a912ef398de7f3ac6ba11f3f21d0986e7ca816aef8b007c2cd9d",
"scid": "997912355483680768",
"scid_str": "907596x2452x0",
"capacity": "20000",
"local_balance": "8322",
"remote_balance": "10000",
"commit_fee": "1018",
"commit_weight": "958",
"fee_per_kw": "1052",
"unsettled_balance": "0",
"total_satoshis_sent": "0",
"total_satoshis_received": "0",
"num_updates": "0",
"pending_htlcs": [],
"csv_delay": 144,
"private": true,
"initiator": true,
"chan_status_flags": "ChanStatusDefault",
"local_chan_reserve_sat": "354",
"remote_chan_reserve_sat": "1062",
"static_remote_key": false,
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"lifetime": "1826",
"uptime": "1782",
"close_address": "",
"push_amount_sat": "10000",
"thaw_height": 0,
"local_constraints": {
"csv_delay": 144,
"chan_reserve_sat": "354",
"dust_limit_sat": "354",
"max_pending_amt_msat": "19800000",
"min_htlc_msat": "1",
"max_accepted_htlcs": 83
},
"remote_constraints": {
"csv_delay": 144,
"chan_reserve_sat": "1062",
"dust_limit_sat": "354",
"max_pending_amt_msat": "19800000",
"min_htlc_msat": "1",
"max_accepted_htlcs": 83
},
"alias_scids": [
"17592186044416001394"
],
"zero_conf": false,
"zero_conf_confirmed_scid": "0",
"peer_alias": "LNT.CJ01",
"peer_scid_alias": "17592186044416000001",
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 10000000000
}
],
"remote_assets": [],
"outgoing_htlcs": [],
"incoming_htlcs": [],
"capacity": 10000000000,
"local_balance": 10000000000,
"remote_balance": 0,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 0
}
},
but the peer node sees it as offline. Below are the details:
{
"active": false,
"remote_pubkey": "02c953421bc7f07be6052920e46843d11e6d3ffc9986177c91f140d76c6ed3a3d4",
"channel_point": "9dcdc207b0f8ae16a87c6e98d0213f1fa16bacf3e78d39ef12a999d289c8c0bd:0",
"chan_id": "bdc0c889d299a912ef398de7f3ac6ba11f3f21d0986e7ca816aef8b007c2cd9d",
"scid": "997912355483680768",
"scid_str": "907596x2452x0",
"capacity": "20000",
"local_balance": "10000",
"remote_balance": "3508",
"commit_fee": "2646",
"commit_weight": "2506",
"fee_per_kw": "1052",
"unsettled_balance": "3186",
"total_satoshis_sent": "0",
"total_satoshis_received": "0",
"num_updates": "9",
"pending_htlcs": [
{
"incoming": true,
"amount": "354",
"hash_lock": "a606274d98dbeabfa92acf1ce98255a6aeb3e971acae93f3595d557eeddfaa79",
"expiration_height": 907702,
"htlc_index": "0",
"forwarding_channel": "0",
"forwarding_htlc_index": "0",
"locked_in": false
},
{
"incoming": true,
"amount": "354",
"hash_lock": "4881b7de958ae7b32a478306ecfce8b7705533d06ea9db499916e363a41ccc48",
"expiration_height": 907702,
"htlc_index": "1",
"forwarding_channel": "0",
"forwarding_htlc_index": "0",
"locked_in": false
},
{
"incoming": true,
"amount": "354",
"hash_lock": "6599ebae71ff124a6fc2fad8b716035aabc206529bc0c814bc049f46857216e5",
"expiration_height": 907702,
"htlc_index": "2",
"forwarding_channel": "0",
"forwarding_htlc_index": "0",
"locked_in": false
},
{
"incoming": true,
"amount": "354",
"hash_lock": "9c609c0d68fe6bd4cda637a1e41295710b3676806a9f8a5c48eb6b0c7c0b6433",
"expiration_height": 907702,
"htlc_index": "3",
"forwarding_channel": "0",
"forwarding_htlc_index": "0",
"locked_in": false
},
{
"incoming": true,
"amount": "354",
"hash_lock": "0f914a2ef1eadb1ce58317e977942c8144cac8bf251035972e72b3f9d467cd05",
"expiration_height": 907705,
"htlc_index": "4",
"forwarding_channel": "0",
"forwarding_htlc_index": "0",
"locked_in": false
},
{
"incoming": true,
"amount": "354",
"hash_lock": "4ab4a1b8046cd9ebb74488c69d8f71e812f0547e8268fd1086048b61876a8bc9",
"expiration_height": 907705,
"htlc_index": "5",
"forwarding_channel": "0",
"forwarding_htlc_index": "0",
"locked_in": false
},
{
"incoming": true,
"amount": "354",
"hash_lock": "40d2e4658c662367459524a2a05c3d24fdb9b9dfcc449213eb259a9c8f31aaf9",
"expiration_height": 907707,
"htlc_index": "6",
"forwarding_channel": "0",
"forwarding_htlc_index": "0",
"locked_in": false
},
{
"incoming": true,
"amount": "354",
"hash_lock": "6783669f695b275eb6e16efa8796405e9c7b11199cf1c998be61a6a3222e861a",
"expiration_height": 907708,
"htlc_index": "7",
"forwarding_channel": "0",
"forwarding_htlc_index": "0",
"locked_in": false
},
{
"incoming": true,
"amount": "354",
"hash_lock": "5f1826fef0f8205198a42f131d43ad77d7a85290548485d111024e2a56aa45a1",
"expiration_height": 907708,
"htlc_index": "8",
"forwarding_channel": "0",
"forwarding_htlc_index": "0",
"locked_in": false
}
],
"csv_delay": 144,
"private": true,
"initiator": false,
"chan_status_flags": "ChanStatusDefault",
"local_chan_reserve_sat": "1062",
"remote_chan_reserve_sat": "354",
"static_remote_key": false,
"commitment_type": "SIMPLE_TAPROOT_OVERLAY",
"lifetime": "13",
"uptime": "13",
"close_address": "",
"push_amount_sat": "10000",
"thaw_height": 0,
"local_constraints": {
"csv_delay": 144,
"chan_reserve_sat": "1062",
"dust_limit_sat": "354",
"max_pending_amt_msat": "19800000",
"min_htlc_msat": "1",
"max_accepted_htlcs": 83
},
"remote_constraints": {
"csv_delay": 144,
"chan_reserve_sat": "354",
"dust_limit_sat": "354",
"max_pending_amt_msat": "19800000",
"min_htlc_msat": "1",
"max_accepted_htlcs": 83
},
"alias_scids": [
"17592186044416000001"
],
"zero_conf": false,
"zero_conf_confirmed_scid": "0",
"peer_alias": "LNT.chengdu",
"peer_scid_alias": "0",
"memo": "",
"custom_channel_data": {
"funding_assets": [
{
"version": 1,
"asset_genesis": {
"genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
"name": "LNT",
"meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
},
"amount": 10000000000,
"script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
"decimal_display": 6
}
],
"local_assets": [],
"remote_assets": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 9998287678
}
],
"outgoing_htlcs": [],
"incoming_htlcs": [
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 190258
},
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 190258
},
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 190258
},
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 190258
},
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 190258
},
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 190258
},
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 190258
},
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 190258
},
{
"asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
"amount": 190258
}
],
"capacity": 10000000000,
"local_balance": 0,
"remote_balance": 9998287678,
"outgoing_htlc_balance": 0,
"incoming_htlc_balance": 1712322
}
}
]
}
Today when I checked this channel again, it had entered the pending_force_closing_channels state { "channel": { "remote_node_pub": "02bda35583eb853b235e14cc3edd24a919f68fbd790a7ca0fe742f9b46ef1e429a", "channel_point": "9dcdc207b0f8ae16a87c6e98d0213f1fa16bacf3e78d39ef12a999d289c8c0bd:0", "capacity": "20000", "local_balance": "8322", "remote_balance": "10000", "local_chan_reserve_sat": "0", "remote_chan_reserve_sat": "0", "initiator": "INITIATOR_LOCAL", "commitment_type": "SIMPLE_TAPROOT_OVERLAY", "num_forwarding_packages": "0", "chan_status_flags": "", "private": true, "memo": "", "custom_channel_data": { "funding_assets": [ { "version": 1, "asset_genesis": { "genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5", "name": "LNT", "meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4", "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c" }, "amount": 10000000000, "script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6", "decimal_display": 6 } ], "local_assets": [ { "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c", "amount": 10000000000 } ], "remote_assets": [], "outgoing_htlcs": [], "incoming_htlcs": [], "capacity": 10000000000, "local_balance": 10000000000, "remote_balance": 0, "outgoing_htlc_balance": 0, "incoming_htlc_balance": 0 } }, "closing_txid": "379abd9716daa49fd2dbd29f87d744552f706218e8fca21920343459db8890d0", "limbo_balance": "8322", "maturity_height": 907847, "blocks_til_maturity": 53, "recovered_balance": "0", "pending_htlcs": [], "anchor": "LOST" },
I just see this that's suspicious:
2025-07-29 06:05:48.586 [INF] PEER: Peer(02bda35583eb853b235e14cc3edd24a919f68fbd790a7ca0fe742f9b46ef1e429a): unable to read message from peer: read next header: EOF
2025-07-29 06:05:48.586 [INF] PEER: Peer(02bda35583eb853b235e14cc3edd24a919f68fbd790a7ca0fe742f9b46ef1e429a): disconnecting 02bda35583eb853b235e14cc3edd24a919f68fbd790a7ca0fe742f9b46ef1e429a@183.229.198.139:2320, reason: read handler closed
Which could be the gossip data rate limiter.
What value did you set for gossip.msg-rate-bytes=? You might want to increase that value by adding another zero. And use double that value in gossip.msg-burst-bytes=.