ccxt
ccxt copied to clipboard
ccxt
[Kucoin] Unable to recreate watchOrderbook after initial sync problem
Hi Fellows!
After initial error on watchOrderbook "kucoin failed to synchronize WebSocket feed with the snapshot for symbol COV/USDT in 3 attempts": (kucoin.js)
retryFetchOrderBookSnapshot (client, message, subscription) {
const symbol = this.safeString (subscription, 'symbol');
const messageHash = this.safeString (subscription, 'messageHash');
// console.log ('fetchOrderBookSnapshot', nonce, previousSequence, nonce >= previousSequence);
const options = this.safeValue (this.options, 'fetchOrderBookSnapshot', {});
const maxAttempts = this.safeInteger (options, 'maxAttempts', 3);
let numAttempts = this.safeInteger (subscription, 'numAttempts', 0);
// retry to syncrhonize if we haven't reached maxAttempts yet
if (numAttempts < maxAttempts) {
// safety guard
if (messageHash in client.subscriptions) {
numAttempts = this.sum (numAttempts, 1);
subscription['numAttempts'] = numAttempts;
client.subscriptions[messageHash] = subscription;
this.spawn (this.fetchOrderBookSnapshot, client, message, subscription);
}
} else {
if (messageHash in client.subscriptions) {
subscription['fetchingOrderBookSnapshot'] = false;
subscription['numAttempts'] = 0;
client.subscriptions[messageHash] = subscription;
}
const e = new InvalidNonce (this.id + ' failed to synchronize WebSocket feed with the snapshot for symbol ' + symbol + ' in ' + maxAttempts.toString () + ' attempts');
client.reject (e, messageHash);
}
}
there is no possibility to create watchOrderbook for the pair again, because the subscription exists, however is not handled. On new watchOrderbook, the if condition of following code is skipped to "return future", which never resolves (Exchange.js line 115-134)
connected.then (() => {
if (!client.subscriptions[subscribeHash]) {
client.subscriptions[subscribeHash] = subscription || true;
const options = this.safeValue (this.options, 'ws');
const cost = this.safeValue (options, 'cost', 1);
if (message) {
if (this.enableRateLimit && client.throttle) {
// add cost here |
// |
// V
client.throttle (cost).then (() => {
client.send (message);
}).catch ((e) => { throw e });
} else {
client.send (message);
}
}
}
})
return future;
I face memory leaks on kucoin and I suspect dead subscriptions like these might be the cause. Please investigate. Thanks.
- CCXT.PRO:1.2.6
Here's a little snippet for you (you might execute it repeatedly until failed sync occurs)
const exchangeName = 'kucoin';
const exchangeKroitor = new ccxtpro[exchangeName](params);
let orderbooksKroitor = {}
async function main() {
await exchangeKroitor.loadMarkets();
watchOrderBook(exchangeKroitor, 'COV/BTC')
watchOrderBook(exchangeKroitor, 'COV/USDT')
}
async function watchOrderBook(exchange, symbol) {
if (!exchange.markets[symbol]) {
return;
}
while (true) {
try {
const orderbook = await exchange.watchOrderBook(symbol)
orderbooksKroitor[symbol] = orderbook
} catch (e) {
console.log(e.message)
}
}
}
This is kind of broader snippet, I've got around 10% of failed sync out of the symbol list every time:
const exchangeName = 'kucoin';
const exchangeKroitor = new ccxtpro[exchangeName](params);
let orderbooksKroitor = {}
async function main() {
const symbols = ["BAX/ETH", "BAX/USDT", "SAND/BTC", "SAND/USDT", "COV/ETH", "COV/BTC", "COV/USDT", "SYLO/USDT", "BIFI/BTC", "ORBS/BTC", "YLD/BTC", "YLD/USDT", "DMT/ETH", "DMT/BTC", "INXT/BTC", "INXT/USDT", "REV/BTC", "REV/USDT", "AKRO/BTC", "JASMY/ETH", "JASMY/USDT", "SPI/BTC", "SPI/USDT", "BFC/BTC", "VEMP/USDT", "GAME/BTC", "GAME/USDT", "DAWN/BTC", "STPT/BTC", "MDT/BTC", "MDT/USDT", "SUKU/BTC", "SUKU/USDT", "SHR/BTC", "SHR/USDT", "UBT/ETH", "UBT/BTC", "DVI/BTC", "DVI/USDT", "PUNDIX/ETH", "PUNDIX/BTC", "PUNDIX/USDT", "XDB/BTC", "XDB/USDT", "GET/BTC", "GET/USDT", "SPC/BTC", "MYST/BTC", "MYST/USDT", "POLC/ETH", "POLC/USDT", "PYR/BTC", "PYR/USDT", "ANKR/ETH", "ANKR/BTC", "PMA/BTC", "PMA/USDT", "VEE/BTC", "TRAC/ETH", "TRAC/BTC", "TRAC/USDT", "VRA/ETH", "VRA/BTC", "VRA/USDT", "SRN/ETH", "SRN/BTC", "SXP/BTC", "RSV/USDT", "CND/BTC", "MFT/BTC", "MFT/USDT", "LRC/BTC", "UPP/BTC", "UQC/BTC", "UQC/USDT", "POWR/ETH", "POWR/BTC", "SOLVE/ETH", "SOLVE/BTC", "SOLVE/USDT", "MTL/BTC", "1INCH/ETH", "1INCH/BTC", "1INCH/USDT", "SUSHI/ETH", "SUSHI/BTC", "SUSHI/USDT", "REVV/BTC", "REVV/USDT", "CEL/ETH", "CEL/BTC", "CEL/USDT", "UTK/BTC", "USDP/BTC", "CVC/ETH", "CVC/BTC", "MET/ETH", "MET/BTC", "RSR/BTC", "RSR/USDT", "AMP/ETH", "AMP/BTC", "AMP/USDT", "RLC/BTC", "QNT/ETH", "QNT/BTC", "DRGN/BTC", "BOSON/BTC", "BOSON/USDT", "UMA/ETH", "UMA/BTC", "UMA/USDT", "UNI/ETH", "UNI/BTC", "UNI/USDT", "REN/ETH", "REN/BTC", "REN/USDT", "GNO/ETH", "GNO/BTC", "BAL/ETH", "BAL/BTC", "BAL/USDT", "DNT/BTC", "NMR/ETH", "NMR/BTC", "NMR/USDT", "NKN/BTC", "NKN/USDT", "GLM/ETH", "GLM/BTC", "DUSK/USDT", "RENBTC/ETH", "RENBTC/BTC", "RENBTC/USDT", "ANT/ETH", "ANT/BTC", "CRV/ETH", "CRV/BTC", "CRV/USDT", "RAMP/ETH", "RAMP/BTC", "RAMP/USDT", "COMP/ETH", "COMP/BTC", "COMP/USDT", "BAND/ETH", "BAND/BTC", "BAND/USDT", "SNT/ETH", "SNT/BTC", "TUSD/ETH", "TUSD/BTC", "TUSD/USDT", "OXT/BTC", "OXT/USDT", "USDC/ETH", "USDC/BTC", "USDC/USDT", "OGN/ETH", "OGN/BTC", "DEP/BTC", "DEP/USDT", "AAVE/ETH", "AAVE/BTC", "AAVE/USDT", "BNT/ETH", "BNT/BTC", "MATIC/ETH", "MATIC/BTC", "MATIC/USDT", "ENJ/ETH", "ENJ/BTC", "ENJ/USDT", "SNX/ETH", "SNX/BTC", "SNX/USDT", "MKR/ETH", "MKR/BTC", "MKR/USDT", "STORJ/BTC", "BAT/ETH", "BAT/BTC", "BAT/USDT", "MANA/ETH", "MANA/BTC", "LINK/ETH", "LINK/BTC", "LINK/USDT", "TYC/BTC", "TYC/USDT", "FOR/BTC", "ABYSS/BTC", "FLIXX/USDT", "INV/ETH", "CIV/USDT", "VID/BTC", "AGRS/BTC", "AGRS/USDT", "ENG/ETH", "ENG/BTC", "EDR/BTC", "STRK/BTC", "STRK/USDT", "UCT/BTC", "UCT/USDT", "ROOK/ETH", "ROOK/BTC", "ROOK/USDT", "CRTS/USDT", "BOA/BTC", "BOA/USDT", "PROM/BTC", "PROM/USDT", "SPWN/ETH", "SPWN/USDT", "DRC/ETH", "SKM/BTC", "SKM/USDT", "RFOX/BTC", "RFOX/USDT"]
await exchangeKroitor.loadMarkets();
const symbolsPromises = symbols.map(symbol => watchOrderBook(exchangeKroitor, symbol))
await Promise.all([symbolsPromises.concat(tick())])
}
async function watchOrderBook(exchange, symbol) {
if (!exchange.markets[symbol]) {
return;
}
while (true) {
try {
const orderbook = await exchange.watchOrderBook(symbol)
orderbooksKroitor[symbol] = orderbook
} catch (e) {
console.log(e.message)
}
}
}
async function tick() {
try {
const delay = 30000
while (true) {
const symbols = Object.keys(orderbooksKroitor)
console.log(new Date(), exchangeKroitor.id, 'orderbooks', symbols.length, symbols)
await exchangeKroitor.sleep(delay)
}
} catch (e) {
console.log(e);
}
}
@krychla1 Thank you very much; as temporary mitigation, can you try to increase maxAttempts and delay to see if it helps?
const exchange = new ccxtpro.kucoin ({
'apiKey': '',
'secret': ''
})
exchange.options['fetchOrderBookSnapshot']['maxAttemps'] = NEW_VALUE // currently 3
exchange.options['fetchOrderBookSnapshot']['delay'] = NEW_VALUE // currently 1000
@carlosmiei Thanks for your suggestions. Delay has no impact (even with extremes like 10 000), however maxAttempts (you've got a typo there) does help. It is strange as the nonces returned from fetchOrderbook are quite random, jumping from low to high to low.
//kucoin.js line 400
if (nonce < previousSequence) {
console.log(nonce, '<', previousSequence, 'retryFetch')
this.retryFetchOrderBookSnapshot (client, message, subscription);
} else {
console.log(nonce, '>', previousSequence, 'ok')
...
2889983 < 2890062 retryFetch
2889912 < 2890062 retryFetch
2889873 < 2890062 retryFetch
2890027 < 2890062 retryFetch
2890013 < 2890062 retryFetch
2890066 > 2890062 ok
Often it takes more than 5 tries to match previousSequence.
Nice workaround, however this issue needs to be addressed - once a subscription gets lost, it can't be resubscribed. And to reinitialize the whole kucoin client is not an option either, as with many subscriptions there are always a few problematic ones. Thank you.
Hi as the ccxt pro will be free very soon, Im working on a project which need socket conndction and ig is wasteful to buy it only fof few weeks. I will be thankful if @krychla1 send me a download able link or package to my email: [email protected]
@myehrajat Hi. Nothing is wasteful, you would support a very professional project that will save your time and make money if you're clever enough. Don't try to find ways to cheat it, be thankful you can enjoy it for lifetime paying just for a few months.