zenbot
zenbot copied to clipboard
DeviaVir
Huge speedup
I've been profiling zenbot (in simulation mode) It was spending most of it's time in lolex doing Object key lookups. I solved that by storing lolex timers in a queue instead of an Object, giving an over 10x speed increase.
However, after further review, it was still taking most of it's time handling "fake" setTimeouts to call checkorder again and again. I replaced that mechanism with a .done() .fail() mechanism instead (required a small change to the sim exchange, instead of polling for an order to be done, the exchange calls the callback once the order is done), and now simulations that at the beginning of the day took over 30 seconds, now run in well under a second (they are seemingly instant).
I think there's some more I can do to improve still, but for the moment this is enough for my needs (ie make it fast enough for genetic optimization to be pratical/not take days).
The source is available to anyone who wants the speed upgrade, but isn't good enough for a PR yet (that's the plan though): I don't think the current code would work outside of simulations (ie trade/paper).
This is the version of lolex adapted for tinyqueue:
"use strict";
const TINY_QUEUE = require('./tinyqueue.js');
function withGlobal(_global) { var userAgent = _global.navigator && _global.navigator.userAgent; var isRunningInIE = userAgent && userAgent.indexOf("MSIE ") > -1; var maxTimeout = Math.pow(2, 31) - 1; //see https://heycam.github.io/webidl/#abstract-opdef-converttoint
// Make properties writable in IE, as per
// http://www.adequatelygood.com/Replacing-setTimeout-Globally.html
if (isRunningInIE) {
_global.setTimeout = _global.setTimeout;
_global.clearTimeout = _global.clearTimeout;
_global.setInterval = _global.setInterval;
_global.clearInterval = _global.clearInterval;
_global.Date = _global.Date;
}
// setImmediate is not a standard function
// avoid adding the prop to the window object if not present
if (_global.setImmediate !== undefined) {
_global.setImmediate = _global.setImmediate;
_global.clearImmediate = _global.clearImmediate;
}
// node expects setTimeout/setInterval to return a fn object w/ .ref()/.unref()
// browsers, a number.
// see https://github.com/cjohansen/Sinon.JS/pull/436
var NOOP = function () { return undefined; };
var timeoutResult = _global.setTimeout(NOOP, 0);
var addTimerReturnsObject = typeof timeoutResult === "object";
var hrtimePresent = (_global.process && typeof _global.process.hrtime === "function");
var nextTickPresent = (_global.process && typeof _global.process.nextTick === "function");
var performancePresent = (_global.performance && typeof _global.performance.now === "function");
var performanceConstructorExists = (_global.Performance && typeof _global.Performance === "function");
var requestAnimationFramePresent = (
_global.requestAnimationFrame && typeof _global.requestAnimationFrame === "function"
);
var cancelAnimationFramePresent = (
_global.cancelAnimationFrame && typeof _global.cancelAnimationFrame === "function"
);
_global.clearTimeout(timeoutResult);
var NativeDate = _global.Date;
var uniqueTimerId = 1;
function isNumberFinite(num) {
if (Number.isFinite) {
return Number.isFinite(num);
}
if (typeof num !== "number") {
return false;
}
return isFinite(num);
}
/**
* Parse strings like "01:10:00" (meaning 1 hour, 10 minutes, 0 seconds) into
* number of milliseconds. This is used to support human-readable strings passed
* to clock.tick()
*/
function parseTime(str) {
if (!str) {
return 0;
}
var strings = str.split(":");
var l = strings.length;
var i = l;
var ms = 0;
var parsed;
if (l > 3 || !/^(\d\d:){0,2}\d\d?$/.test(str)) {
throw new Error("tick only understands numbers, 'm:s' and 'h:m:s'. Each part must be two digits");
}
while (i--) {
parsed = parseInt(strings[i], 10);
if (parsed >= 60) {
throw new Error("Invalid time " + str);
}
ms += parsed * Math.pow(60, (l - i - 1));
}
return ms * 1000;
}
/**
* Floor function that also works for negative numbers
*/
function fixedFloor(n) {
return (n >= 0 ? Math.floor(n) : Math.ceil(n));
}
/**
* % operator that also works for negative numbers
*/
function fixedModulo(n, m) {
return ((n % m) + m) % m;
}
/**
* Used to grok the `now` parameter to createClock.
* @param epoch {Date|number} the system time
*/
function getEpoch(epoch) {
if (!epoch) { return 0; }
if (typeof epoch.getTime === "function") { return epoch.getTime(); }
if (typeof epoch === "number") { return epoch; }
throw new TypeError("now should be milliseconds since UNIX epoch");
}
function inRange(from, to, timer) {
return timer && timer.callAt >= from && timer.callAt <= to;
}
function mirrorDateProperties(target, source) {
var prop;
for (prop in source) {
if (source.hasOwnProperty(prop)) {
target[prop] = source[prop];
}
}
// set special now implementation
if (source.now) {
target.now = function now() {
return target.clock.now;
};
} else {
delete target.now;
}
// set special toSource implementation
if (source.toSource) {
target.toSource = function toSource() {
return source.toSource();
};
} else {
delete target.toSource;
}
// set special toString implementation
target.toString = function toString() {
return source.toString();
};
target.prototype = source.prototype;
target.parse = source.parse;
target.UTC = source.UTC;
target.prototype.toUTCString = source.prototype.toUTCString;
return target;
}
function createDate() {
function ClockDate(year, month, date, hour, minute, second, ms) {
// Defensive and verbose to avoid potential harm in passing
// explicit undefined when user does not pass argument
switch (arguments.length) {
case 0:
return new NativeDate(ClockDate.clock.now);
case 1:
return new NativeDate(year);
case 2:
return new NativeDate(year, month);
case 3:
return new NativeDate(year, month, date);
case 4:
return new NativeDate(year, month, date, hour);
case 5:
return new NativeDate(year, month, date, hour, minute);
case 6:
return new NativeDate(year, month, date, hour, minute, second);
default:
return new NativeDate(year, month, date, hour, minute, second, ms);
}
}
return mirrorDateProperties(ClockDate, NativeDate);
}
function enqueueJob(clock, job) {
// enqueues a microtick-deferred task - ecma262/#sec-enqueuejob
if (!clock.jobs) {
clock.jobs = [];
}
clock.jobs.push(job);
}
function runJobs(clock) {
// runs all microtick-deferred tasks - ecma262/#sec-runjobs
if (!clock.jobs) return;
for (var i = 0; i < clock.jobs.length; i++) {
var job = clock.jobs[i];
job.func.apply(null, job.args);
if (clock.loopLimit && i > clock.loopLimit) {
throw new Error("Aborting after running " + clock.loopLimit + " timers, assuming an infinite loop!");
}
}
clock.jobs = [];
}
function addTimer(clock, timer) {
if (timer.func === undefined) {
throw new Error("Callback must be provided to timer calls");
}
timer.type = timer.immediate ? "Immediate" : "Timeout";
if (timer.hasOwnProperty("delay")) {
if (!isNumberFinite(timer.delay)) {
timer.delay = 0;
}
timer.delay = timer.delay > maxTimeout ? 1 : timer.delay;
timer.delay = Math.max(0, timer.delay);
}
if (timer.hasOwnProperty("interval")) {
timer.type = "Interval";
timer.interval = timer.interval > maxTimeout ? 1 : timer.interval;
}
if (timer.hasOwnProperty("animation")) {
timer.type = "AnimationFrame";
timer.animation = true;
}
if (!clock.timers) {
clock.timers = {};
}
timer.id = uniqueTimerId++;
timer.createdAt = clock.now;
timer.callAt = clock.now + (parseInt(timer.delay) || (clock.duringTick ? 1 : 0));
// TODO:
// clock.timers[timer.id] = timer;
clock.timersq.push(timer);
if (addTimerReturnsObject) {
return {
id: timer.id,
ref: NOOP,
unref: NOOP
};
}
return timer.id;
}
/* eslint consistent-return: "off" */
function compareTimers(a, b) {
// Sort first by absolute timing
if (a.callAt < b.callAt) {
return -1;
}
if (a.callAt > b.callAt) {
return 1;
}
// Sort next by immediate, immediate timers take precedence
if (a.immediate && !b.immediate) {
return -1;
}
if (!a.immediate && b.immediate) {
return 1;
}
// Sort next by creation time, earlier-created timers take precedence
if (a.createdAt < b.createdAt) {
return -1;
}
if (a.createdAt > b.createdAt) {
return 1;
}
// Sort next by id, lower-id timers take precedence
if (a.id < b.id) {
return -1;
}
if (a.id > b.id) {
return 1;
}
// As timer ids are unique, no fallback `0` is necessary
}
function firstTimerInRange(clock, from, to) {
let timer = null;
// TODO: Check if stopping after the first one ("continue/break") would work here or if we actually need to go through the entire loop for our seek here
for(let id = 0;Â id < clock.timersq.data.length; id++){
if (inRange(from, to, clock.timersq.data[id]) && (!timer || compareTimers(timer, clock.timersq.data[id]) === 1)) {
timer = clock.timersq.data[id];
}
}
return timer;
}
function firstTimer(clock) {
let timer = clock.timersq.peek();
return timer;
// var timers = clock.timers;
// var timer = null;
// var id;
// for (id in timers) {
// if (timers.hasOwnProperty(id)) {
// if (!timer || compareTimers(timer, timers[id]) === 1) {
// timer = timers[id];
// }
// }
// }
// return timer;
}
function lastTimer(clock) {
let timer = null;
for(let id = 0;Â id <= clock.timersq.data.length; id++){
if (!timer || compareTimers(timer, clock.timersq.data[id]) === 1) {
timer = clock.timersq.data[id];
}
}
return timer;
/* var timers = clock.timers;
var timer = null;
var id;
for (id in timers) {
if (timers.hasOwnProperty(id)) {
if (!timer || compareTimers(timer, timers[id]) === -1) {
timer = timers[id];
}
}
}
return timer; */
}
function callTimer(clock, timer) {
if (typeof timer.interval === "number") {
// TODO:
// TODO:Â WHYÂ do we pop here ???
clock.timerq.pop();
// clock.timers[timer.id].callAt += timer.interval;
clock.timersq.push(timer);
} else {
// delete clock.timers[timer.id];
clock.timersq.pop();
}
if (typeof timer.func === "function") {
timer.func.apply(null, timer.args);
} else {
/* eslint no-eval: "off" */
eval(timer.func);
}
}
function clearTimer(clock, timerId, ttype) {
if (!timerId) {
// null appears to be allowed in most browsers, and appears to be
// relied upon by some libraries, like Bootstrap carousel
return;
}
if (!clock.timers) {
clock.timers = {};
}
// in Node, timerId is an object with .ref()/.unref(), and
// its .id field is the actual timer id.
if (typeof timerId === "object") {
timerId = timerId.id;
}
if (clock.timers.hasOwnProperty(timerId)) {
// check that the ID matches a timer of the correct type
var timer = clock.timers[timerId];
if (timer.type === ttype) {
delete clock.timers[timerId];
} else {
var clear = ttype === "AnimationFrame" ? "cancelAnimationFrame" : "clear" + ttype;
var schedule = timer.type === "AnimationFrame" ? "requestAnimationFrame" : "set" + timer.type;
throw new Error("Cannot clear timer: timer created with " + schedule
+ "() but cleared with " + clear + "()");
}
}
}
function uninstall(clock, target, config) {
var method,
i,
l;
var installedHrTime = "_hrtime";
var installedNextTick = "_nextTick";
for (i = 0, l = clock.methods.length; i < l; i++) {
method = clock.methods[i];
if (method === "hrtime" && target.process) {
target.process.hrtime = clock[installedHrTime];
} else if (method === "nextTick" && target.process) {
target.process.nextTick = clock[installedNextTick];
} else if (method === "performance") {
Object.defineProperty(target, method, {
writeable: false,
value: clock["_" + method]
});
} else {
if (target[method] && target[method].hadOwnProperty) {
target[method] = clock["_" + method];
if (method === "clearInterval" && config.shouldAdvanceTime === true) {
target[method](clock.attachedInterval);
}
} else {
try {
delete target[method];
} catch (ignore) { /* eslint empty-block: "off" */ }
}
}
}
// Prevent multiple executions which will completely remove these props
clock.methods = [];
// return pending timers, to enable checking what timers remained on uninstall
if (!clock.timers) {
return [];
}
return Object.keys(clock.timers).map(function mapper(key) {
return clock.timers[key];
});
}
function hijackMethod(target, method, clock) {
var prop;
clock[method].hadOwnProperty = Object.prototype.hasOwnProperty.call(target, method);
clock["_" + method] = target[method];
if (method === "Date") {
var date = mirrorDateProperties(clock[method], target[method]);
target[method] = date;
} else if (method === "performance") {
Object.defineProperty(target, method, {
writeable: false,
value: clock[method]
});
} else {
target[method] = function () {
return clock[method].apply(clock, arguments);
};
for (prop in clock[method]) {
if (clock[method].hasOwnProperty(prop)) {
target[method][prop] = clock[method][prop];
}
}
}
target[method].clock = clock;
}
function doIntervalTick(clock, advanceTimeDelta) {
clock.tick(advanceTimeDelta);
}
var timers = {
setTimeout: _global.setTimeout,
clearTimeout: _global.clearTimeout,
setImmediate: _global.setImmediate,
clearImmediate: _global.clearImmediate,
setInterval: _global.setInterval,
clearInterval: _global.clearInterval,
Date: _global.Date
};
if (hrtimePresent) {
timers.hrtime = _global.process.hrtime;
}
if (nextTickPresent) {
timers.nextTick = _global.process.nextTick;
}
if (performancePresent) {
timers.performance = _global.performance;
}
if (requestAnimationFramePresent) {
timers.requestAnimationFrame = _global.requestAnimationFrame;
}
if (cancelAnimationFramePresent) {
timers.cancelAnimationFrame = _global.cancelAnimationFrame;
}
var keys = Object.keys || function (obj) {
var ks = [];
var key;
for (key in obj) {
if (obj.hasOwnProperty(key)) {
ks.push(key);
}
}
return ks;
};
/**
* @param start {Date|number} the system time
* @param loopLimit {number} maximum number of timers that will be run when calling runAll()
*/
function createClock(start, loopLimit) {
start = start || 0;
loopLimit = loopLimit || 1000;
var clock = {
now: getEpoch(start),
hrNow: 0,
timeouts: {},
Date: createDate(),
loopLimit: loopLimit,
timersq: new TINY_QUEUE([], compareTimers),
};
clock.Date.clock = clock;
function getTimeToNextFrame() {
return 16 - ((clock.now - start) % 16);
}
clock.setTimeout = function setTimeout(func, timeout) {
return addTimer(clock, {
func: func,
args: Array.prototype.slice.call(arguments, 2),
delay: timeout
});
};
clock.clearTimeout = function clearTimeout(timerId) {
return clearTimer(clock, timerId, "Timeout");
};
clock.nextTick = function nextTick(func) {
return enqueueJob(clock, {
func: func,
args: Array.prototype.slice.call(arguments, 1)
});
};
clock.setInterval = function setInterval(func, timeout) {
return addTimer(clock, {
func: func,
args: Array.prototype.slice.call(arguments, 2),
delay: timeout,
interval: timeout
});
};
clock.clearInterval = function clearInterval(timerId) {
return clearTimer(clock, timerId, "Interval");
};
clock.setImmediate = function setImmediate(func) {
return addTimer(clock, {
func: func,
args: Array.prototype.slice.call(arguments, 1),
immediate: true
});
};
clock.clearImmediate = function clearImmediate(timerId) {
return clearTimer(clock, timerId, "Immediate");
};
clock.requestAnimationFrame = function requestAnimationFrame(func) {
var result = addTimer(clock, {
func: func,
delay: getTimeToNextFrame(),
args: [clock.now + getTimeToNextFrame()],
animation: true
});
return result.id || result;
};
clock.cancelAnimationFrame = function cancelAnimationFrame(timerId) {
return clearTimer(clock, timerId, "AnimationFrame");
};
function updateHrTime(newNow) {
clock.hrNow += (newNow - clock.now);
}
clock.runMicrotasks = function runMicrotasks() {
runJobs(clock);
};
clock.tick = function tick(ms) {
ms = typeof ms === "number" ? ms : parseTime(ms);
var tickFrom = clock.now;
var tickTo = clock.now + ms;
var previous = clock.now;
var timer, firstException, oldNow;
clock.duringTick = true;
// perform process.nextTick()s
oldNow = clock.now;
runJobs(clock);
if (oldNow !== clock.now) {
// compensate for any setSystemTime() call during process.nextTick() callback
tickFrom += clock.now - oldNow;
tickTo += clock.now - oldNow;
}
// perform each timer in the requested range
timer = firstTimerInRange(clock, tickFrom, tickTo);
let loop = 0;
while (timer && tickFrom <= tickTo) {
/*console.log([loop, clock.timersq.data.length]);
loop++;
if(loop > 1000){
}*/
//if (clock.timers[timer.id]) {
updateHrTime(timer.callAt);
tickFrom = timer.callAt;
clock.now = timer.callAt;
oldNow = clock.now;
try {
runJobs(clock);
callTimer(clock, timer);
} catch (e) {
console.log(e);
firstException = firstException || e;
}
// compensate for any setSystemTime() call during timer callback
if (oldNow !== clock.now) {
tickFrom += clock.now - oldNow;
tickTo += clock.now - oldNow;
previous += clock.now - oldNow;
}
//}
timer = firstTimerInRange(clock, previous, tickTo);
previous = tickFrom;
}
// perform process.nextTick()s again
oldNow = clock.now;
runJobs(clock);
if (oldNow !== clock.now) {
// compensate for any setSystemTime() call during process.nextTick() callback
tickFrom += clock.now - oldNow;
tickTo += clock.now - oldNow;
}
clock.duringTick = false;
// corner case: during runJobs, new timers were scheduled which could be in the range [clock.now, tickTo]
timer = firstTimerInRange(clock, tickFrom, tickTo);
if (timer) {
try {
clock.tick(tickTo - clock.now); // do it all again - for the remainder of the requested range
} catch (e) {
firstException = firstException || e;
}
} else {
// no timers remaining in the requested range: move the clock all the way to the end
updateHrTime(tickTo);
clock.now = tickTo;
}
if (firstException) {
throw firstException;
}
return clock.now;
};
clock.next = function next() {
runJobs(clock);
var timer = firstTimer(clock);
if (!timer) {
return clock.now;
}
clock.duringTick = true;
try {
updateHrTime(timer.callAt);
clock.now = timer.callAt;
callTimer(clock, timer);
runJobs(clock);
return clock.now;
} catch(err){
console.log(err);
} finally {
clock.duringTick = false;
}
};
clock.runAll = function runAll() {
var numTimers, i;
runJobs(clock);
for (i = 0; i < clock.loopLimit; i++) {
if (!clock.timers) {
return clock.now;
}
numTimers = clock.timersq.length;
if (numTimers === 0) {
return clock.now;
}
clock.next();
}
throw new Error("Aborting after running " + clock.loopLimit + " timers, assuming an infinite loop!");
};
clock.runToFrame = function runToFrame() {
return clock.tick(getTimeToNextFrame());
};
clock.runToLast = function runToLast() {
var timer = lastTimer(clock);
if (!timer) {
runJobs(clock);
return clock.now;
}
return clock.tick(timer.callAt);
};
clock.reset = function reset() {
clock.timers = {};
clock.jobs = [];
clock.now = getEpoch(start);
};
clock.setSystemTime = function setSystemTime(systemTime) {
// determine time difference
var newNow = getEpoch(systemTime);
var difference = newNow - clock.now;
var id, timer;
// update 'system clock'
clock.now = newNow;
// update timers and intervals to keep them stable
for (id in clock.timers) {
if (clock.timers.hasOwnProperty(id)) {
timer = clock.timers[id];
timer.createdAt += difference;
timer.callAt += difference;
}
}
};
if (performancePresent) {
clock.performance = Object.create(_global.performance);
if (performanceConstructorExists) {
var proto = _global.Performance.prototype;
Object
.getOwnPropertyNames(proto)
.forEach(function (name) {
if (Object.getOwnPropertyDescriptor(proto, name).writable) {
clock.performance[name] = proto[name];
}
});
}
clock.performance.now = function lolexNow() {
return clock.hrNow;
};
}
if (hrtimePresent) {
clock.hrtime = function (prev) {
if (Array.isArray(prev)) {
var oldSecs = (prev[0] + prev[1] / 1e9);
var newSecs = (clock.hrNow / 1000);
var difference = (newSecs - oldSecs);
var secs = fixedFloor(difference);
var nanosecs = fixedModulo(difference * 1e9, 1e9);
return [
secs,
nanosecs
];
}
return [
fixedFloor(clock.hrNow / 1000),
fixedModulo(clock.hrNow * 1e6, 1e9)
];
};
}
return clock;
}
/**
* @param config {Object} optional config
* @param config.target {Object} the target to install timers in (default `window`)
* @param config.now {number|Date} a number (in milliseconds) or a Date object (default epoch)
* @param config.toFake {string[]} names of the methods that should be faked.
* @param config.loopLimit {number} the maximum number of timers that will be run when calling runAll()
* @param config.shouldAdvanceTime {Boolean} tells lolex to increment mocked time automatically (default false)
* @param config.advanceTimeDelta {Number} increment mocked time every <<advanceTimeDelta>> ms (default: 20ms)
*/
function install(config) {
if ( arguments.length > 1 || config instanceof Date || Array.isArray(config) || typeof config === "number") {
throw new TypeError("lolex.install called with " + String(config) +
" lolex 2.0+ requires an object parameter - see https://github.com/sinonjs/lolex");
}
config = typeof config !== "undefined" ? config : {};
config.shouldAdvanceTime = config.shouldAdvanceTime || false;
config.advanceTimeDelta = config.advanceTimeDelta || 20;
var i, l;
var target = config.target || _global;
var clock = createClock(config.now, config.loopLimit);
clock.uninstall = function () {
return uninstall(clock, target, config);
};
clock.methods = config.toFake || [];
if (clock.methods.length === 0) {
// do not fake nextTick by default - GitHub#126
clock.methods = keys(timers).filter(function (key) {return key !== "nextTick";});
}
for (i = 0, l = clock.methods.length; i < l; i++) {
if (clock.methods[i] === "hrtime") {
if (target.process && typeof target.process.hrtime === "function") {
hijackMethod(target.process, clock.methods[i], clock);
}
} else if (clock.methods[i] === "nextTick") {
if (target.process && typeof target.process.nextTick === "function") {
hijackMethod(target.process, clock.methods[i], clock);
}
} else {
if (clock.methods[i] === "setInterval" && config.shouldAdvanceTime === true) {
var intervalTick = doIntervalTick.bind(null, clock, config.advanceTimeDelta);
var intervalId = target[clock.methods[i]](
intervalTick,
config.advanceTimeDelta);
clock.attachedInterval = intervalId;
}
hijackMethod(target, clock.methods[i], clock);
}
}
return clock;
}
return {
timers: timers,
createClock: createClock,
install: install,
withGlobal: withGlobal
};
}
var defaultImplementation = withGlobal(global || window);
exports.timers = defaultImplementation.timers; exports.createClock = defaultImplementation.createClock; exports.install = defaultImplementation.install; exports.withGlobal = withGlobal;
Hi, thanks for posting this. I am trying to run many simulations in parallel and am finding I am CPU bound once around 15-20 are running, even though I have quite a lot of power available. So this change is of interest to me ;)
I pasted your source code above over the contents of node_modules/lolex/src/lolex-src.js and also added tinyqueue.js in the same directory, with the contents pasted from here: https://raw.githubusercontent.com/mourner/tinyqueue/master/index.js
The simulation starts ok but after a short while I get this error;
UnhandledPromiseRejectionWarning: MongoServerSelectionError: Server selection timed out after 30000 ms
at /home/zenbot/node_modules/mongodb/lib/core/sdam/topology.js:438:30
at callTimer (/home/zenbot/node_modules/lolex/src/lolex-src.js:367:20)
at Object.tick (/home/zenbot/node_modules/lolex/src/lolex-src.js:647:21)
at withOnPeriod (/home/zenbot/lib/engine.js:779:17)
at onTrade (/home/zenbot/lib/engine.js:950:7)
at /home/zenbot/lib/engine.js:904:5
at /home/zenbot/node_modules/async/dist/async.js:4011:13
at Object.process (/home/zenbot/node_modules/async/dist/async.js:1674:21)
at /home/zenbot/node_modules/async/dist/async.js:1532:23
at /home/zenbot/node_modules/async/dist/async.js:74:45
I am not using localhost as my Mongo address so I wonder if that is the cause. Did you have to work through this error when you implemented this?
By the way, just so you know, there are funny characters in the signature of two for loops in the above
for(let id = 0;Â id < clock.timersq.data.length; id++)
for(let id = 0;Â id <= clock.timersq.data.length; id++)
I didn't have this error, but my mongo was on localhost. If you really want to use zenbot (I'm now convinced it's not a viable option for what we both are trying to do), I strongly recommend you implement something where:
- You take the k-lines/trades from mongo and write them to a file
- You use that file instead of mongodb in zenbot when running a backtest It should improve performance immensely, and will make sure you can't have any mongodb error (the current mongodb code in zenbot is utter crap, just look at how it does things...) I actually changed a lot of how zenbot worked with mongo before I gave up on it, but lots of my work broke parts of zenbot I don't use (like paper), so I don't think it'd be helpful to others as-is. FYI, I'm now using bbgo instead, and it works incredibly well, and it's very actively developped (and supports grid, which is what I was adding to zenbot). The only issue is it's lacking documentation at the moment.
On Sun, Jan 23, 2022 at 2:05 PM Geoff Clayton @.***> wrote:
Hi, thanks for posting this. I am trying to run many simulations in parallel and am finding I am CPU bound once around 15-20 are running, even though I have quite a lot of power available. So this change is of interest to me ;)
I pasted your source code above over the contents of node_modules/lolex/src/lolex-src.js and also added tinyqueue.js in the same directory, with the contents pasted from here: https://raw.githubusercontent.com/mourner/tinyqueue/master/index.js
The simulation starts ok but after a short while I get this error;
UnhandledPromiseRejectionWarning: MongoServerSelectionError: Server selection timed out after 30000 ms
at /home/zenbot/node_modules/mongodb/lib/core/sdam/topology.js:438:30 at callTimer (/home/zenbot/node_modules/lolex/src/lolex-src.js:367:20) at Object.tick (/home/zenbot/node_modules/lolex/src/lolex-src.js:647:21) at withOnPeriod (/home/zenbot/lib/engine.js:779:17) at onTrade (/home/zenbot/lib/engine.js:950:7) at /home/zenbot/lib/engine.js:904:5 at /home/zenbot/node_modules/async/dist/async.js:4011:13 at Object.process (/home/zenbot/node_modules/async/dist/async.js:1674:21) at /home/zenbot/node_modules/async/dist/async.js:1532:23 at /home/zenbot/node_modules/async/dist/async.js:74:45I am not using localhost as my Mongo address so I wonder if that is the cause. Did you have to work through this error when you implemented this?
By the way, just so you know, there are funny characters in the signature of two for loops in the above for(let id = 0;Â id < clock.timersq.data.length; id++) for(let id = 0;Â id <= clock.timersq.data.length; id++)
— Reply to this email directly, view it on GitHub https://github.com/DeviaVir/zenbot/issues/2765#issuecomment-1019481398, or unsubscribe https://github.com/notifications/unsubscribe-auth/AAA2SFLPRFNNJPEQU7LCZ2DUXP4JFANCNFSM5I4T7LUQ . Triage notifications on the go with GitHub Mobile for iOS https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675 or Android https://play.google.com/store/apps/details?id=com.github.android&referrer=utm_campaign%3Dnotification-email%26utm_medium%3Demail%26utm_source%3Dgithub.
You are receiving this because you authored the thread.Message ID: @.***>
--
勇気とユーモア
Hi, thanks for the reply. Writing data to a file for Zenbot to read from sounds like a good idea. I already tried running Mongo with the inMemory storage engine to see if that would make a difference, but it made no difference at all.
My app is rather deeply integrated with Zenbot at this point so I am not going to give up on it just yeat. Though I may change the way in which I expect to use it to fit with the limitations on parallelisation. I did have a set up where I was using AWS Farscape with auto-scaling to handle parallelising up to hundreds of processes, and it worked quite well, but I was not keen on the costs.
I just had a look at bbgo and as far as I can see it is also limited to running one backtest at a time so would require similar tolling around it as I have built for Zenbot. I would need to do some analysis of resource usage of bbgo vs. Zenbot before committing to that route.
Much to think about!
On Sun, Jan 23, 2022 at 2:24 PM Geoff Clayton @.***> wrote:
Hi, thanks for the reply. Writing data to a file for Zenbot to read from sounds like a good idea. I already tried running Mongo with the inMemory storage engine to see if that would make a difference, but it made no difference at all.
Tried that too.
My app is rather deeply integrated with Zenbot at this point so I am not going to give up on it just yeat.
That's called the "sunk cost fallacy", look it up, it cost me 3 weeks of zenbot work...
Though I may change the way in which I expect to use it to fit with the limitations on parallelisation. I did have a set up where I was using AWS Farscape with auto-scaling to handle parallelising up to hundreds of processes, and it worked quite well, but I was not keen on the costs.
That's my long-term plan. For now a server with 40 cores does the job, though a bit slowly.
I just had a look at bbgo and as far as I can see it is also limited to running one backtest at a time
But it's very cheap in terms of ressources, and very fast at starting/going through it (insanely faster compared to zenbot). Especially after I reported a whole bunch of bugs and asked for features that make this easier. Currently, in my system, running a backtest on one day, takes under 0.1 second (starting, stopping, writing the report to disk as json, everything included). Zenbot was at least several seconds.
so would require similar tolling around it as I have built for Zenbot. I would need to do some analysis of resource usage of bbgo vs. Zenbot before committing to that route.
Much to think about!
— Reply to this email directly, view it on GitHub https://github.com/DeviaVir/zenbot/issues/2765#issuecomment-1019485003, or unsubscribe https://github.com/notifications/unsubscribe-auth/AAA2SFO2SLWZ35AHMQZPLVLUXP6RFANCNFSM5I4T7LUQ . Triage notifications on the go with GitHub Mobile for iOS https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675 or Android https://play.google.com/store/apps/details?id=com.github.android&referrer=utm_campaign%3Dnotification-email%26utm_medium%3Demail%26utm_source%3Dgithub.
You are receiving this because you authored the thread.Message ID: @.***>
--
勇気とユーモア
I can see how the sunk cost fallacy applies here, but I have limited to spend on this stuff, so if my choices are either
- Find a suitable use-case for the system I have already built
- Do nothing
...then I think I'd go for option 1 :)
I wish there were an option 3: spend a bunch more time on it, but there is not...
I will take another look at bbgo though. I like the idea of making the underlying sim engine in my framework selectable from a variety.
On Sun, Jan 23, 2022 at 2:53 PM Geoff Clayton @.***> wrote:
I can see how the sunk cost fallacy applies here, but I have limited to spend on this stuff, so if my choices are either
- Find a suitable use-case for the system I have already built
- Do nothing
...then I think I'd go for option 1 :)
I wish there were an option 3: spend a bunch more time on it, but there is not...
I will take another look at bbgo though. I like the idea of making the underlying sim engine in my framework selectable from a variety.
That's how I started: I made my bot class a parent, made a zenbot and a bbgo class that inherited it, rewrote the tests to run on both, and in a couple of days I had it switchable from one to the other. It was much easier than I first expected. Maybe give it a quick go/make this a side project.
— Reply to this email directly, view it on GitHub https://github.com/DeviaVir/zenbot/issues/2765#issuecomment-1019490426, or unsubscribe https://github.com/notifications/unsubscribe-auth/AAA2SFPGFOGDBR6CHWIKXA3UXQB3ZANCNFSM5I4T7LUQ . Triage notifications on the go with GitHub Mobile for iOS https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675 or Android https://play.google.com/store/apps/details?id=com.github.android&referrer=utm_campaign%3Dnotification-email%26utm_medium%3Demail%26utm_source%3Dgithub.
You are receiving this because you authored the thread.Message ID: @.***>
--
勇気とユーモア