cdxgen
cdxgen copied to clipboard
CycloneDX
[deno] deno lock file support
This would be very useful to our project. We use Deno, with no package.json or package-lock.json. We have dependencies from NPM, as well as JSR and HTTPS imports from https://esm.sh (which are built from NPM packages).
Currently I parse a deno.lock file myself and generate a CycloneDX SBOM file to be picked up by a downstream tool (Anchore), but they recently dropped support, and I'm sure I wasn't doing the best job myself in the first place.
@adamgreg could you kindly share the implementation you have? Feel free to use openai or Gemini to improve the code before submission too, since they have been doing alright recently.
Sure, no problem. See below.
When I said it's not probably doing the best job, I wasn't referring to general code quality, but to the fact that it is building the SBOM content quite manually, and probably missing lots of important aspects that I'm unaware of. A lot of the details of good practice for SBOMs are mysterious to me. The aim was just to add enough information so that the SBOM Cataloger of Anchore Enterprise would pick it up and add to the overall SBOM.
It builds pURLs for everything. For NPM packages and for https://esm.sh it uses the npm: scheme (with no record about the processing by esm.sh). Otherwise it makes Github pURLs. For JSR packages it links through to the Rekor attestation to identify a git repository, and will throw an exception if a package is missing that.
I use it as part of a build pipeline, so it picks up various arguments as environment variables.
import { chunk, distinctBy, mapNotNullish } from "jsr:@std/collections";
/** Partial type of items from Rekor's log entries API */
type RekorLogEntry = Record<string, {
logIndex: number;
attestation: { data: string };
}>;
/** Type for the subject section of https://in-toto.io/Statement/v1 */
type Subject = {
name: string;
digest: {
sha256: string;
};
};
/** Partial type of https://in-toto.io/Statement/v1 */
type Attestation = {
subject: Subject | [Subject];
};
// Common patterns for (optionally scoped) package names
// and for version specifiers (optionally with pre-release or build metadata)
const PKG_NAME_PATTERN = "((?:@[\\w.\\-]+/)?[\\w.\\-]+)";
const PKG_VER_PATTERN = "(v?[\\d.]+)(?:[+\\-][0-9A-Za-z\\-.]+)?";
const PKG_PATTERN = `${PKG_NAME_PATTERN}@${PKG_VER_PATTERN}`;
// Regular expressions used to identify packages in ES module URLs
const ESM_SH_URL_PATTERN = "https://esm.sh/(?:v\\d+|stable)/" + PKG_PATTERN;
const DENO_STD_URL_PATTERN = "https://deno.land/(std)@" + PKG_VER_PATTERN;
const DENO_X_URL_PATTERN = "https://deno.land/x/" + PKG_PATTERN;
const GH_RAW_URL_PATTERN = "https://raw.githubusercontent.com/(.+?/.+?)/(.+?)/";
// Union of the expressions, used to identify any URLs that don't match any pattern
const HANDLED_URL_PATTERN = new RegExp(
[
ESM_SH_URL_PATTERN,
DENO_STD_URL_PATTERN,
DENO_X_URL_PATTERN,
GH_RAW_URL_PATTERN,
"https://esm.sh/(?!v\\d+|stable)", // Such esm.sh URLs redirect to include API version
].join("|"),
);
// Get the name of the application being analysed
const serviceName = Deno.env.get("SERVICE_NAME");
if (!serviceName) {
throw Error(
"SERVICE_NAME environment variable must be defined as the name of the application",
);
}
// Get the version of the application being analysed
const serviceVersion = Deno.env.get("SERVICE_VERSION");
if (!serviceVersion) {
throw Error(
"SERVICE_VERSION environment variable must be defined as the version of the application",
);
}
// Get the path of the deno.lock file to analyse
const denoLockFile = Deno.env.get("DENO_LOCK_FILE");
if (!denoLockFile) {
throw Error(
"DENO_LOCK environment variable must reference a valid Deno lockfile for the application",
);
}
// Get the path of the SBOM file to create
const denoSbomFile = Deno.env.get("DENO_SBOM_FILE");
if (!denoSbomFile) {
throw Error(
"DENO_SBOM_FILE environment variable must reference an output path for the generated SBOM",
);
}
// Get the git repo name from an environment variable
const gitUrl = Deno.env.get("GIT_URL");
if (!gitUrl) {
throw Error(
"GIT_URL environment variable required, referring to the repo containing this script",
);
}
// Read the Deno lockfile
const denoLock = JSON.parse(await Deno.readTextFile(denoLockFile));
// Get all remote module URLs
const remoteModuleUrls = Object.keys(denoLock.remote);
/** Extract set of package names & versions where the module URL matches the pattern */
function findPackages(inputStrs: string[], pattern: string) {
const regex = new RegExp(pattern);
const allMatches = mapNotNullish(inputStrs, (x) => x.match(regex)?.slice(1));
return distinctBy(allMatches, (x) => x.join()) as [string, string][];
}
// Find dependency versions for deno_std, deps from esm.sh and deps from deno.land/x
const stdDeps = findPackages(remoteModuleUrls, DENO_STD_URL_PATTERN);
const esmDeps = findPackages(remoteModuleUrls, ESM_SH_URL_PATTERN);
const denolandDeps = findPackages(remoteModuleUrls, DENO_X_URL_PATTERN);
const githubRawDeps = findPackages(remoteModuleUrls, GH_RAW_URL_PATTERN);
// Check for any module URLs that were not matched to a package
const unhandledUrls = remoteModuleUrls.filter((url) =>
!HANDLED_URL_PATTERN.test(url)
);
if (unhandledUrls.length !== 0) {
console.error("Unhandled remote URLs:\n" + unhandledUrls.sort().join("\n"));
}
// Get all NPM package versions
const npmPackages = Object.keys(denoLock.npm ?? denoLock.packages?.npm ?? {});
const npmDeps = findPackages(npmPackages, PKG_PATTERN);
// Check for any NPM package keys that were not matched to a package
if (npmDeps.length !== npmPackages.length) {
const regex = new RegExp(PKG_PATTERN);
const unhandled = npmPackages.filter((x) => !regex.test(x)).sort();
throw Error(
"Could not parse NPM dependency keys:\n" + unhandled.join("\n"),
);
}
// Get all JSR package versions
const jsrPackages = Object.keys(denoLock.jsr ?? {});
const jsrDeps = findPackages(jsrPackages, PKG_PATTERN);
// Check for any JSR package keys that were not matched to a package
if (jsrDeps.length !== jsrPackages.length) {
const regex = new RegExp(PKG_PATTERN);
const unhandled = jsrPackages.filter((x) => !regex.test(x)).sort();
throw Error(
"Could not parse JSR dependency keys:\n" + unhandled.join("\n"),
);
}
// Create SBOM components for deno std from https://deno.land/std
const stdComponents = stdDeps.map(([, ver]) => (
{
type: "library",
name: "deno_std",
version: ver,
purl: `pkg:github/denoland/deno_std@${ver}`,
}
));
// Create SBOM components for packages from npm / esm.sh
const npmComponents = [...npmDeps, ...esmDeps].map(([name, ver]) => ({
type: "library",
name: name,
version: ver,
purl: `pkg:npm/${name}@${ver}`,
}));
// Create SBOM components for packages from deno.land/x
const denolandComponents = await Promise.all(
denolandDeps.map(async ([name, ver]) => {
// Use the repository's API to fetch information about the source of the package
const resp = await fetch(
`https://apiland.deno.dev/v2/modules/${name}/${ver}`,
);
const json = await resp.json();
if (!resp.ok) {
throw new Error(json.stack);
}
const { type, repository, ref } = json.upload_options;
return {
type: "library",
name: name,
version: ver,
purl: `pkg:${type}/${repository}@${ref}`,
};
}),
);
// Get SigStore Rekor transparency log IDs for JSR packages
const jsrRekorLogIds = new Map(
await Promise.all(
jsrDeps.map(async ([name, version]) => {
const [, scope, pkg] = name.match(/@([^\/]+)\/(.*)/)!;
const url =
`https://api.jsr.io/scopes/${scope}/packages/${pkg}/versions/${version}`;
const { rekorLogId } = await fetch(url).then((resp) => resp.json());
if (rekorLogId == null) {
throw Error(`No transparency log for ${name}/${version}`);
}
return [parseInt(rekorLogId), { name, version }] as const;
}),
),
);
// Fetch the SigStore Rekor logs for JSR packages
const rekorResp: RekorLogEntry[] = (await Promise.all(
chunk(jsrRekorLogIds.keys().toArray(), 10).map((ids) =>
fetch(
"https://rekor.sigstore.dev/api/v1/log/entries/retrieve",
{
method: "POST",
headers: {
accept: "application/json",
"Content-Type": "application/json",
},
body: JSON.stringify({ logIndexes: ids }),
},
).then((resp) => resp.json())
),
)).flat();
// Extract the attestations from the Rekor logs, and enrich with name & version
const jsrAttestations = rekorResp.flatMap((x) =>
Object.values(x).map(({ logIndex, attestation }) => ({
...jsrRekorLogIds.get(logIndex as number)!,
...JSON.parse(globalThis.atob(attestation.data)) as Attestation,
}))
);
// Create SBOM components for packages from JSR
const jsrComponents = jsrAttestations.map(({ name, version, subject }) => {
// Subject may be an array (with one member) or not
const { name: purl, digest } = Array.isArray(subject) ? subject[0] : subject;
return {
type: "library",
name,
version,
purl,
hashes: [{ alg: "SHA-256", content: digest.sha256 }],
};
});
// Create SBOM components for raw Github modules
const githubRawComponents = githubRawDeps.map(([name, ver]) => ({
type: "library",
name: name,
version: ver,
purl: `pkg:github/${name}@${ver}`,
}));
// Combine and sort all the library components
const libComponents = [
...stdComponents,
...denolandComponents,
...npmComponents,
...jsrComponents,
...githubRawComponents,
].sort((a, b) =>
a.name.localeCompare(b.name) * 10 + a.version.localeCompare(b.version)
);
// Build the SBOM
const sbom = {
bomFormat: "CycloneDX",
specVersion: "1.4",
serialNumber: `urn:uuid:${crypto.randomUUID()}`,
metadata: {
timestamp: new Date().toISOString(),
tools: [
{
name: "scripts/gen_deno_sbom.ts",
version: "1.0.0",
vendor: "Aguila Engineering ltd",
externalReferences: [
{
type: "vcs",
url: gitUrl,
},
],
},
],
component: {
type: "application",
name: serviceName,
version: serviceVersion,
},
},
components: [
{
type: "platform",
name: "Deno",
author: "Deno",
version: Deno.version.deno,
purl: `pkg:crates.io/deno@${Deno.version.deno}`,
components: [{
type: "platform",
name: "V8",
author: "Google",
version: Deno.version.v8,
purl: `pkg:github.com/v8/v8@${Deno.version.v8}`,
}],
},
...libComponents,
],
};
// Write the SBOM to file
await Deno.writeTextFile(denoSbomFile, JSON.stringify(sbom));