Appending files / persisting logging data

[Andersama]

Some objects which are supposed to stick around are arrays, some of which expand. Obviously the pattern is general, the larger the object we want to persist the longer writeFile takes, degrading performance over time A hackish approach to this is something like below:

let data_store = storage.create({dir: 'db', ttl: false, logging: true});

data_store.logItem = function (key, datumValue, options = {}) {
    let data_str = this.stringify(datumValue);
    if (this.logging) {
        this.log(`log ('${key}': '${data_str}')`);
    }
    this.appendRawFile(this.getDatumPath(key), data_str+'\n');
};

data_store.getLog = function (key) {
    return this.getRawDatum(key);
};

data_store.appendRawFile = function (file, content) {
    return new Promise((resolve, reject) => {
        fs.appendFile(file, content, this.options.encoding, (err) => {
            if (err) {
                return reject(err);
            }
            resolve({file: file, content: content});
            this.log('appended: ' + file);
        });
    });
};

    //whenever we need to get data we do end up reading the entire file
    let target_array = [];
    let log_file = await data_store.getLog('raw_data');
    let log_array = log_file.split('\n');

    let l = log_array.length;
    //all data had '\n' appended, so the file is going to have an extra blank line
    for (let i = 0; (i+1) < log_array.length; i++) {
        try  {
            let log_item = JSON.parse(log_array[i]);
            target_array.push(log_item);
        } catch (err) {
            console.log(err, i, l, log_array[i]);
        }
    }

This reduces the cost of writing something like a log line to a file, by only writing data to the end as we need it. Does drop ttl and other features, and here I assume the target file is a newline delimited list of json data. One benefit here is we can drop badly parsed data (any incomplete appends / corrupted data) presuming data is non critical.

I don't know if it'd be worthwhile to add api functions for something like this.