Fastest way to create a huge object with NAPI

[ggreco]

This is not strictly an issue, more a performance issue... point me to a better location where to post this, if it's not the right place... I could not find one.

I need to parse huge JSONs in my node application, and node has a limit of 512MB on the string that can be used as input of a JSON.parse(), so I tried to implement a JSON parse on a file (or a remote buffer) using NAPI, but the result is quite slow, more than 10 times slower of JSON.parse() on JSON sizes that JSON.parse can handle, I'm using rapidjson to parse in C++, and this is pretty fast, benchmarking the code the bottleneck seems to be the allocation of the napi values I'm creating when parsing the JSON, so I'm asking if there is something I can do to speed up the allocation of a lot of small objects (something like a bulk allocation method or a sql-like begin/end transaction logic for operations like this one), here is the c++ code I wrote:

#include <napi.h>
#include "rapidjson/document.h"

static Napi::Value parseJSONNode(Napi::Env &env, rapidjson::Value &node, Napi::Object *parent = nullptr, const char *parentKey = nullptr, uint32_t parentIndex = 0) {
    Napi::Value v;
    if (node.IsArray()) {
        Napi::Array a = Napi::Array::New(env, node.Size());
        for (rapidjson::SizeType i = 0; i < node.Size(); ++i) {
            parseJSONNode(env, node[i], &a, nullptr, i);
        }
        v = a;
    } else if (node.IsObject()) {
        Napi::Object o = Napi::Object::New(env);
        for (auto itr = node.MemberBegin(); itr != node.MemberEnd(); ++itr) {
            parseJSONNode(env, itr->value, &o, itr->name.GetString());
        }
        v = o;
    } else if (node.IsString()) {
        v = Napi::String::New(env, node.GetString());
    } else if (node.IsNumber()) {
        v = Napi::Number::New(env, node.GetDouble());
    } else if (node.IsBool()) {
        v = Napi::Boolean::New(env, node.GetBool());
    }
    if (parent) {
        if (parentKey) // parent is an object
            (*parent)[parentKey] = v;
        else // ... or an array
            (*parent)[parentIndex] = v;
    }
    return v;
}

Napi::Value parseJSON(const Napi::CallbackInfo& info) {
    Napi::Env env = info.Env();
    if (info.Length() < 2) {
        Napi::TypeError::New(env, "Invalid argument count").ThrowAsJavaScriptException();
        return env.Undefined();
    }
    if (!info[1].IsNumber() || !info[0].IsBuffer()) {
        Napi::TypeError::New(env, "Invalid argument type").ThrowAsJavaScriptException();
        return env.Undefined();
    }
  
    try {
        rapidjson::Document doc;    
        char *buffer = info[0].As<Napi::Buffer<char>>().Data();
        double buffer_len = info[1].ToNumber();
        doc.Parse(buffer, (size_t)buffer_len);
   
        if (!doc.IsObject() && !doc.IsArray()) {
            Napi::TypeError::New(env, "Unable to parse JSON").ThrowAsJavaScriptException();
            return env.Undefined();
        }
        return parseJSONNode(env, doc);
    } catch (std::string &err) {
        Napi::TypeError::New(env, err).ThrowAsJavaScriptException();
        return env.Undefined();
    }
}

Napi::Object Init(Napi::Env env, Napi::Object exports) {
    exports.Set(Napi::String::New(env, "parseJSON"), Napi::Function::New(env, parseJSON));
    return exports;
}

NODE_API_MODULE(addon, Init)