About the formats...

[ROBERT-MCDOWELL]

does ColorConverter.js recognizes RGB and YUV differences like "I420" "I420A" "I422" "I444" "NV12" "RGBA" "RGBX" "BGRA" "BGRX"?

[SimonWaldherr]

Unfortunately not at the moment, but if you want to implement this and create a pull request, I would accept it.

[ROBERT-MCDOWELL]

Hi Simon, I will try, but for now I'm struggling with manipulated RGBA pixels (create from getImageData()) from a VideoFrame with I420 format created from MediaStreamTrackProcessor(track).readable.getReader() to make it work in a new VideoFrame back to forma I420.... :(

[ROBERT-MCDOWELL]

for now I just try to manipulate pixels from an imageData in RGBA (without to use colorconverter) but when I return a new VideoFrame() create from this imageData I have weird screen results (see attached screenshot) here is the code

async function manipulatePixels(frame){
	try{
		if(avgRGB.length > 0 && frame.timestamp && frame.timestamp > 0){
			const init = {timestamp:frame.timestamp,codedWidth:frame.codedWidth,codedHeight:frame.codedHeight,displayWidth:frame.displayWidth,displayHeight:frame.displayHeight,format:frame.format};
			const options = {format:init.format,rect:{x:0,y:0,width:init.codedWidth,height:init.codedHeight}};
			const typedArray = new Uint8ClampedArray((init.codedWidth * init.codedHeight * 4));
			await frame.copyTo(typedArray,options);
			frame.close();
			const imageData = await new ImageData(typedArray,init.codedWidth,init.codedHeight);
			
			let pixels = imageData.data;
			const pxCount = pixels.length / 4;
			for(let i=0;i<pxCount;i++){
				const r = i * 4;
				const g = (i * 4) + 1;
				const b = (i * 4) + 2;
				const a = (i * 4) + 3;
				let pixel = [pixels[r],pixels[g],pixels[b],pixels[a]];
				if(avgRGB[3] == "invert"){
					pixel = pixelInvert(pixel);
				}else if(avgRGB[3] == "transparent"){
					pixel = pixelTransparent(pixel);
				}else if(avgRGB[3] == "backgroundImage"){
					pixel = pixelBackgroundImage(pixel);
				}else if(avgRGB[3] == "backgroundVideo"){
					pixel = pixelBackgroundVideo(pixel);
				}
				if(init.format == "I420"){
					//pixel = colorconv.RGB2YUV([pixel[0],pixel[1],pixel[2]]);
				}
				pixels[r] = pixel[0];
				pixels[g] = pixel[1];
				pixels[b] = pixel[2];
				pixels[a] = pixel[3];
			}
			return(await new VideoFrame(imageData.data.buffer,init));
		}
	}catch(e){
		console.log(e);
	}
	return frame;
}

any idea what's happening?