ajv icon indicating copy to clipboard operation
ajv copied to clipboard

Unable to generate ESM-compatible standalone code

Open cjscheller opened this issue 2 years ago • 16 comments

The version of Ajv you are using 8.12

The environment you have the problem with node v16.16.0, ESM project ({type: module})

Your code (please make it as small as possible to reproduce the issue)

Code to compile AJV schema:

// compile.js

import Ajv from "ajv";
import standaloneCode from "ajv/dist/standalone/index.js";
import { writeFile } from "fs/promises";

const ajvOptions = {
    strict: true,
    allErrors: true,
    messages: true,
    code: {
        source: true,
        esm: true, // ESM support
    },
};

const ajv = new Ajv(ajvOptions);

const exampleSchema = {
    type: "object",
    properties: {
        body: {
            type: "object",
            properties: {
                firstName: {
                    type: "string",
                    minLength: 1,
                },
                lastName: {
                    type: "string",
                },
            },
            required: ["firstName", "lastName"],
            additionalProperties: false,
        },
    },
};

compileJsonSchema(exampleSchema);

// Note: excluded code that runs this function
async function compileJsonSchema(schema) {
    try {
        // Compile schema via AJV
        const compiled = ajv.compile(schema);

        // Build output module file content via AJV's standalone
        let moduleCode = standaloneCode(ajv, compiled);

        // Write file to parent directory of source file
        await writeFile("./schema.js", moduleCode, "utf-8");
    } catch (err) {
        console.error(err);
    }
}

Results in node.js v8+ Able to successfully compile standalone code; unable to import in ESM project

Results and error messages in your platform

I am using the standalone validation code functionality to generate compiled schema files at build time. I am compiling the code with the AJV library in a JS script and specifying esm: true to compile the standalone code for ESM. The compilation works and I am writing the compiled schema to a JS file, but I am unable to import the standalone code from this schema file in my ESM project because it imports the ucs2length utility via a require statement (const func2 = require("ajv/dist/runtime/ucs2length").default).

Here is an example schema compiled via standalone for ESM support (generated with the above script:

"use strict";export const validate = validate10;export default validate10;const schema11 = {"type":"object","properties":{"body":{"type":"object","properties":{"firstName":{"type":"string","minLength":1},"lastName":{"type":"string"}},"required":["firstName","lastName"],"additionalProperties":false}}};const func2 = require("ajv/dist/runtime/ucs2length").default;function validate10(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){let vErrors = null;let errors = 0;if(data && typeof data == "object" && !Array.isArray(data)){if(data.body !== undefined){let data0 = data.body;if(data0 && typeof data0 == "object" && !Array.isArray(data0)){if(data0.firstName === undefined){const err0 = {instancePath:instancePath+"/body",schemaPath:"#/properties/body/required",keyword:"required",params:{missingProperty: "firstName"},message:"must have required property '"+"firstName"+"'"};if(vErrors === null){vErrors = [err0];}else {vErrors.push(err0);}errors++;}if(data0.lastName === undefined){const err1 = {instancePath:instancePath+"/body",schemaPath:"#/properties/body/required",keyword:"required",params:{missingProperty: "lastName"},message:"must have required property '"+"lastName"+"'"};if(vErrors === null){vErrors = [err1];}else {vErrors.push(err1);}errors++;}for(const key0 in data0){if(!((key0 === "firstName") || (key0 === "lastName"))){const err2 = {instancePath:instancePath+"/body",schemaPath:"#/properties/body/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"};if(vErrors === null){vErrors = [err2];}else {vErrors.push(err2);}errors++;}}if(data0.firstName !== undefined){let data1 = data0.firstName;if(typeof data1 === "string"){if(func2(data1) < 1){const err3 = {instancePath:instancePath+"/body/firstName",schemaPath:"#/properties/body/properties/firstName/minLength",keyword:"minLength",params:{limit: 1},message:"must NOT have fewer than 1 characters"};if(vErrors === null){vErrors = [err3];}else {vErrors.push(err3);}errors++;}}else {const err4 = {instancePath:instancePath+"/body/firstName",schemaPath:"#/properties/body/properties/firstName/type",keyword:"type",params:{type: "string"},message:"must be string"};if(vErrors === null){vErrors = [err4];}else {vErrors.push(err4);}errors++;}}if(data0.lastName !== undefined){if(typeof data0.lastName !== "string"){const err5 = {instancePath:instancePath+"/body/lastName",schemaPath:"#/properties/body/properties/lastName/type",keyword:"type",params:{type: "string"},message:"must be string"};if(vErrors === null){vErrors = [err5];}else {vErrors.push(err5);}errors++;}}}else {const err6 = {instancePath:instancePath+"/body",schemaPath:"#/properties/body/type",keyword:"type",params:{type: "object"},message:"must be object"};if(vErrors === null){vErrors = [err6];}else {vErrors.push(err6);}errors++;}}}else {const err7 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"};if(vErrors === null){vErrors = [err7];}else {vErrors.push(err7);}errors++;}validate10.errors = vErrors;return errors === 0;}

I'm able to import and use the schema file by manually replacing the require with an import statement. Am I overlooking something to generate standalone validation code that is fully ESM-compatible?

cjscheller avatar Feb 07 '23 19:02 cjscheller

FYI: This is only a problem for JSON Schema generation. If I instead import Ajv from "ajv/dist/jtd.js" for JTD schemas, the code compiles fine.

pmorch avatar Apr 17 '23 12:04 pmorch

Any news on this one? This is a big issue that actually prevents using AJV to generate ESM validation modules.

ericmorand avatar May 26 '23 09:05 ericmorand

It's not just ucs2length. I also noticed it's requiring the equal module to be used when you have a schema like this:

{ 
  "type": "string",
  "const": "my_value"
}
  const func1 = require('ajv/dist/runtime/ucs2length').default;
+ const func0 = require('ajv/dist/runtime/equal').default;

Any news on this one? This is a big issue that actually prevents using AJV to generate ESM validation modules.

@ericmorand

Our fix for now is to configure Vite to transform the standalone validation files so that CJS Require statements get modified to look like ES import syntax.

import commonjs from 'vite-plugin-commonjs';

const config = defineConfig({
    plugins: [
        commonjs({
            filter: (id) => {
                const fileRegex = /precompiled.*validations\.js$/;
                return fileRegex.test(id);
            },
        }),
    ],
    // ...

The same approach can be done for Rollup users.

Weffe avatar Dec 18 '23 23:12 Weffe

I solved this issue in vite using this in vite.config.js:

import commonjs from '@rollup/plugin-commonjs';

const config = {
	build: {
		rollupOptions: {
			//... other options
			plugins: [commonjs({ transformMixedEsModules: true })],
		}
	}
};

As mentioned beforehand, you could also use { transformMixedEsModules: true } in a rollup config

patiboh avatar Feb 09 '24 15:02 patiboh

The workaround with the "commonjs" plugin did not work for me, maybe because I don't compile the validation code to a file, but in-memory via the "virtual" plugin. I had to monkey-patch the generated code and replace the require() lines with proper imports:

import func2 from 'ajv/dist/runtime/ucs2length'
import {fullFormats} from 'ajv-formats/dist/formats'
const formats0 = fullFormats.date

Actual code may vary depending on used features. Needless to say, this is a fragile hack and no substitute for a proper fix.

hschletz avatar Aug 02 '24 09:08 hschletz

I'm still having this issue while ESM is enabled, I see require() in the compiled file

meness avatar Dec 11 '24 11:12 meness

A workaround is compiling it in CJS format and then importing it in an ESM file like this.

import { createRequire } from 'module';

const require = createRequire(import.meta.url);

const { yourFunction } = require('compiled.cjs');

Note: Absolute paths don't work with this solution, you should use relative paths.

Importing the CJS like import { yourFunction } from 'compiled.cjs' doesn't work in AWS Lambda.

meness avatar Dec 11 '24 12:12 meness

It's 2025 and the issue still exists with AJV v8.17.1.

To reproduce just execute:

import Ajv from 'ajv'
import standaloneCode from 'ajv/dist/standalone'

const ajv = new Ajv({code: {source: true, esm: true},})
console.log(standaloneCode(ajv, {validateJsonSchemaV7: 'http://json-schema.org/draft-07/schema#'}))

And then unfortunately you can see this in the produced code:

const func0 = require("ajv/dist/runtime/equal").default;

That causes this runtime error:

ReferenceError: require is not defined

mirismaili avatar Jan 03 '25 14:01 mirismaili

The other issue is, isn't the produced code standalone? Then what is require (or even import) inside it from node_modules ("ajv/dist/runtime/equal" or "ajv/dist/runtime/ucs2length")?!

mirismaili avatar Jan 03 '25 14:01 mirismaili

isn't the produced code standalone?

As is, no it isn't if your schema has rules which requires (pun intended) the ajv/dist/runtime/equal and ajv/dist/runtime/ucs2length functions. Independently of generating commonjs or (incorrect) esm code, the ajv module will be needed as a dependency

PopGoesTheWza avatar Jan 07 '25 22:01 PopGoesTheWza

We have an awfully kludgy fix where we replace require("ajv/dist/runtime/equal").default and require("ajv/dist/runtime/ucs2length").default within the esm code with standalone function derived from the original ajv source code.

I'd gladfully assist on a PR, implementing a proper solution, if needed.

PopGoesTheWza avatar Jan 07 '25 22:01 PopGoesTheWza

I had the misfortune of running into this problem as well. I tried with the various vite/rollup common-js plugins but to no avail, and I think it's because ajv outputs requires but ajv-formats is even worse and generates straight-out invalid code ala

const formats96 = {"_items":["require(\"ajv-formats/dist/formats\").",{"str":"fullFormats"},""]}["date-time"];

To workaround this, I (also) had to do a bunch of regex'ing. I've pasted the crux of it here in the hope that someone else does not waste as much time as I have on this.

// assume `moduleCode` is generated from `standaloneCode(ajv, {...})`
const preamble = [
  `// @ts-nocheck`,
  `"use strict";`,
  `import { fullFormats } from "ajv-formats/dist/formats";`,
].join("\n");
const imports = new Set<string>();
const formatsRegex = /const (formats\d+)\s*=\s\{.+\}(.+);/g;
const requireRegex = /const (\S+)\s*=\s*require\((.+)\)\.(\S+);/g;
const replaced = moduleCode
  .replaceAll(
    formatsRegex,
    (_match, p1, p2) => `const ${p1} = fullFormats${p2}`
  )
  .replace(requireRegex, (_match, p1, p2, p3) => {
    imports.add(`import { ${p3} as ${p1} } from ${p2};`);
    return "";
  })
  // since `use strict` should be the first non-comment line, and we're adding
  // more lines in the preamble, we have moved it into the preamble.
  .replace(`"use strict";\n`, "");

const uglyOut = [preamble, Array.from(imports).join("\n"), "", replaced].join(
  "\n"
);

process.stdout.write(
  // prettify the generated code (not that it really matters).
  await prettier.format(uglyOut, { parser: "babel" }),
  "utf-8"
);

The code above also formats the output with prettier, which is of course not necessary.

adamschoenemann avatar Jan 08 '25 10:01 adamschoenemann

I tackled two different scenarios using separate approaches:

  1. Pre-build environment (with access to Node.js, node_modules, etc.):

    This was the easier case. I used esbuild to bundle the generated source:

    import Ajv from 'ajv'
    import standaloneCode from 'ajv/dist/standalone'
    import {build} from 'esbuild'
    
    const originalSource = standaloneCode(/* ... */)
    const {outputFiles} = await build({
      bundle: true,
      write: false,
      format: 'esm',
      sourcemap: false,
      stdin: {
        contents: originalSource,
        resolveDir: '/',
        sourcefile: 'input.js', // Virtual source file name
        loader: 'js',
      },
    })
    const bundledSource = outputFiles[0].text
    
  2. Browser environment (specifically in a service-worker, without access to Node.js, esbuild, etc.):

    This was more challenging. To resolve the issue, I manually replaced require() calls 😕:

    /**
     * Workaround for the issue of {@link import('ajv/dist/standalone').default AJV's `standaloneCode()`}.
     * @see https://github.com/ajv-validator/ajv/issues/2209
     */
    const UNRESOLVED_SOURCES_OF_AJV_STANDALONE = {
      /** @see https://www.unpkg.com/[email protected]/es6/index.js */
      ' require("ajv/dist/runtime/equal").default;': `
        function equal(a, b) {
          if (a === b) return true;
    
          if (a && b && typeof a == 'object' && typeof b == 'object') {
            if (a.constructor !== b.constructor) return false;
    
            var length, i, keys;
            if (Array.isArray(a)) {
              length = a.length;
              if (length != b.length) return false;
              for (i = length; i-- !== 0;)
                if (!equal(a[i], b[i])) return false;
              return true;
            }
    
    
            if ((a instanceof Map) && (b instanceof Map)) {
              if (a.size !== b.size) return false;
              for (i of a.entries())
                if (!b.has(i[0])) return false;
              for (i of a.entries())
                if (!equal(i[1], b.get(i[0]))) return false;
              return true;
            }
    
            if ((a instanceof Set) && (b instanceof Set)) {
              if (a.size !== b.size) return false;
              for (i of a.entries())
                if (!b.has(i[0])) return false;
              return true;
            }
    
            if (ArrayBuffer.isView(a) && ArrayBuffer.isView(b)) {
              length = a.length;
              if (length != b.length) return false;
              for (i = length; i-- !== 0;)
                if (a[i] !== b[i]) return false;
              return true;
            }
    
    
            if (a.constructor === RegExp) return a.source === b.source && a.flags === b.flags;
            if (a.valueOf !== Object.prototype.valueOf) return a.valueOf() === b.valueOf();
            if (a.toString !== Object.prototype.toString) return a.toString() === b.toString();
    
            keys = Object.keys(a);
            length = keys.length;
            if (length !== Object.keys(b).length) return false;
    
            for (i = length; i-- !== 0;)
              if (!Object.prototype.hasOwnProperty.call(b, keys[i])) return false;
    
            for (i = length; i-- !== 0;) {
              var key = keys[i];
    
              if (!equal(a[key], b[key])) return false;
            }
    
            return true;
          }
    
          // true if both NaN, false otherwise
          return a!==a && b!==b;
        };
      `.trim(),
    
      /** @see https://www.unpkg.com/[email protected]/dist/runtime/ucs2length.js */
      ' require("ajv/dist/runtime/ucs2length").default;': `
        function ucs2length(str) {
            const len = str.length;
            let length = 0;
            let pos = 0;
            let value;
            while (pos < len) {
                length++;
                value = str.charCodeAt(pos++);
                if (value >= 0xd800 && value <= 0xdbff && pos < len) {
                    // high surrogate, and there is a next character
                    value = str.charCodeAt(pos);
                    if ((value & 0xfc00) === 0xdc00)
                        pos++; // low surrogate
                }
            }
            return length;
        };
      `.trim(),
    }
    
    let validatorSource = standaloneCode(ajv, idNameMap) // https://ajv.js.org/standalone.html#generating-functions-s-for-multiple-schemas-using-the-js-library-es6-and-esm-exports
    
    // Patch `validatorSource` to work around AJV's `standaloneCode` issue: https://github.com/ajv-validator/ajv/issues/2209#issuecomment-2569293326
    for (const [toBeResolved, resolvedSource] of Object.entries(UNRESOLVED_SOURCES_OF_AJV_STANDALONE))
       validatorSource = validatorSource.replace(toBeResolved, resolvedSource)
    

mirismaili avatar Jan 09 '25 13:01 mirismaili

@mirismaili tried your approach, but got this error from esbuild

Error: Build failed with 3 errors:
../../../../../../../input.js:1:38114: ERROR: Could not resolve "ajv/dist/runtime/ucs2length"
../../../../../../../input.js:1:104921: ERROR: Could not resolve "ajv/dist/runtime/equal"
../../../../../../../input.js:1:104980: ERROR: Could not resolve "ajv-formats/dist/formats"
    at failureErrorWithLog (/Users/simone.dicola2/WebstormProjects/publishing-tools/node_modules/esbuild/lib/main.js:1477:15)
    at /Users/simone.dicola2/WebstormProjects/publishing-tools/node_modules/esbuild/lib/main.js:946:25
    at /Users/simone.dicola2/WebstormProjects/publishing-tools/node_modules/esbuild/lib/main.js:898:52
    at buildResponseToResult (/Users/simone.dicola2/WebstormProjects/publishing-tools/node_modules/esbuild/lib/main.js:944:7)
    at /Users/simone.dicola2/WebstormProjects/publishing-tools/node_modules/esbuild/lib/main.js:971:16
    at responseCallbacks.<computed> (/Users/simone.dicola2/WebstormProjects/publishing-tools/node_modules/esbuild/lib/main.js:623:9)
    at handleIncomingPacket (/Users/simone.dicola2/WebstormProjects/publishing-tools/node_modules/esbuild/lib/main.js:678:12)
    at Socket.readFromStdout (/Users/simone.dicola2/WebstormProjects/publishing-tools/node_modules/esbuild/lib/main.js:601:7)
    at Socket.emit (node:events:517:28)
    at addChunk (node:internal/streams/readable:368:12) {
  errors: [Getter/Setter],
  warnings: [Getter/Setter]
}

Node.js v18.20.6

s-di-cola avatar Apr 01 '25 13:04 s-di-cola

this is my solution without any additional dependency:

function generateStandaloneCode(ajv: Ajv, outputPath: string): void {
    // @ts-ignore
    let ajvStandAlone = standaloneCode(ajv, idMap);

    ajvStandAlone = ajvStandAlone.replace('const func0 = require("ajv/dist/runtime/equal").default;', `import func0 from "ajv/dist/runtime/equal";`)
        .replace('const formats0 = require("ajv-formats/dist/formats").fullFormats.uri;', `import { fullFormats } from "ajv-formats/dist/formats";  const formats0 = fullFormats.uri;`)
        .replace('const func3 = require("ajv/dist/runtime/ucs2length").default;', `const func3 = (str) => {
                                                                                                            const len = str.length
                                                                                                            let length = 0
                                                                                                            let pos = 0
                                                                                                            let value
                                                                                                            while (pos < len) {
                                                                                                                length++
                                                                                                                value = str.charCodeAt(pos++)
                                                                                                                if (value >= 0xd800 && value <= 0xdbff && pos < len) {
                                                                                                                    // high surrogate, and there is a next character
                                                                                                                    value = str.charCodeAt(pos)
                                                                                                                    if ((value & 0xfc00) === 0xdc00) pos++ // low surrogate
                                                                                                                }
                                                                                                            }
                                                                                                            return length
                                                                                                        }`);
    fs.writeFileSync(outputPath, ajvStandAlone);
}

function compileSchemas(): void {
    const ajv = new Ajv({
        allErrors: true,
        verbose: true,
        strict: false,
        allowUnionTypes: true,
        code: {source: true, esm: true},
    });
 
    Logger.info('Adding schemas...');
    addSchemas(ajv, path.resolve(dirname(import.meta), '../managed'));
    Logger.info('Generating standalone code...');
    generateStandaloneCode(ajv, path.resolve(dirname(import.meta), './validator.mjs'));
}

compileSchemas();

s-di-cola avatar Apr 02 '25 09:04 s-di-cola

The only way I was able to get the Ajv module to work within TypeScript, was rather than importing the default module, I imported the named module. This is already supported in the exports (you can confirm this in the node_modules/ajv/dist/ajv.d.ts file of your local project).

import { Ajv, ErrorObject } from 'ajv';

// This AJV is typescript-ready
const ajv = new Ajv({ allErrors: true });

Unrelated to above, but I also wanted to use ajv-formats library as well. Which also has it's issues. It is not handled well by TypeScript, and it doesn't export the named module in addition to the default. But it does export the type of the default module. So I ended up needed a hacky solution as:

import formatsPlugin, { FormatsPlugin } from 'ajv-formats';

const addFormats = (formatsPlugin as unknown as FormatsPlugin);
addFormats(ajv);

Happy Coding 😸

shmolf avatar Apr 27 '25 18:04 shmolf