react-native-skia
react-native-skia copied to clipboard
Shopify
Skia frame processor turns screen white
I'm using Skia with react-native-vision-camera and react-native-fast-tflite for real-time pose detection. Everything is working great until I add the Skia frame processor – as soon as I do that, the screen just turns white and the camera feed doesn't show at all. No crash, just a white screen.
Also, while building and running the app, I got an error related to onLayout saying Vision Camera doesn't support it. I patched that by just removing the onLayout prop from one of the components inside node_modules, and it worked. But I'm not sure if that's the right way to fix it.
Would really appreciate any help or ideas on what might be causing the white screen with Skia frame processor!
Thanks in advance 🙏
Platform: Android
Specs :
Expo SDK: ~54.0.8
React Native: 0.81.4
React: 19.1.0
New Architecture: enabled ("newArchEnabled": true)
{
"android": {
"minSdkVersion": 33,
"compileSdkVersion": 35,
"targetSdkVersion": 35
},
"ios": {
"deploymentTarget": "15.1"
}
}
Full code :
import { Skia } from '@shopify/react-native-skia';
import React, { JSX, useEffect, useState } from 'react';
import { Alert, PermissionsAndroid, Platform, StyleSheet, Text, View } from 'react-native';
import { useTensorflowModel } from 'react-native-fast-tflite';
import { Camera, useCameraDevice, useCameraPermission, useSkiaFrameProcessor } from 'react-native-vision-camera';
import { useResizePlugin } from 'vision-camera-resize-plugin';
import modelAsset from '../assets/model/model.tflite';
const MIN_CONFIDENCE = 0.3;
export default function App(): JSX.Element {
const { hasPermission, requestPermission } = useCameraPermission();
const device = useCameraDevice('front');
const [debugInfo, setDebugInfo] = useState<string>('Initializing...');
const { resize } = useResizePlugin();
// Debug logging
useEffect(() => {
console.log('Camera permission:', hasPermission);
console.log('Camera device:', device);
let info = `Permission: ${hasPermission}\n`;
info += `Device: ${device ? 'Found' : 'None'}\n`;
if (device) {
info += `Device ID: ${device.id}\n`;
info += `Position: ${device.position}\n`;
}
setDebugInfo(info);
}, [hasPermission, device]);
useEffect(() => {
if (!hasPermission) {
console.log('Requesting camera permission...');
requestPermission().then((granted) => {
console.log('Permission granted:', granted);
setDebugInfo(prev => prev + `\nPermission result: ${granted}`);
});
}
}, [hasPermission, requestPermission]);
useEffect(() => {
const requestCameraPermission = async (): Promise<void> => {
if (Platform.OS === 'android') {
try {
const result = await PermissionsAndroid.request(
PermissionsAndroid.PERMISSIONS.CAMERA,
{
title: 'Camera Permission',
message: 'App needs camera permission',
buttonPositive: 'OK',
buttonNegative: 'Cancel',
}
);
console.log('Android permission result:', result);
setDebugInfo(prev => prev + `\nAndroid permission: ${result}`);
} catch (error) {
console.error('Permission error:', error);
Alert.alert('Permission Error', `${error}`);
}
}
};
if (!hasPermission) {
requestCameraPermission();
}
}, [hasPermission]);
// Load the TensorFlow Lite plugin and model
const delegate = Platform.OS === 'ios' ? 'core-ml' : undefined;
const plugin = useTensorflowModel(modelAsset, delegate);
useEffect(() => {
if (plugin) {
console.log('Plugin state:', plugin.state);
setDebugInfo(prev => prev + `\nModel state: ${plugin.state}`);
if (plugin.model) {
console.log('Model inputs:', plugin.model.inputs.map(input =>
`${input.dataType} [${input.shape}]`));
console.log('Model outputs:', plugin.model.outputs.map(output =>
`${output.dataType} [${output.shape}]`));
}
}
}, [plugin]);
// Get input dimensions from the model
const inputTensor = plugin.model?.inputs[0];
const inputWidth = inputTensor?.shape[1] ?? 192;
const inputHeight = inputTensor?.shape[2] ?? 192;
// Create paints for drawing
const linePaint = Skia.Paint();
linePaint.setColor(Skia.Color('white'));
linePaint.setStrokeWidth(3);
linePaint.setAntiAlias(true);
const pointPaint = Skia.Paint();
pointPaint.setColor(Skia.Color('red'));
pointPaint.setAntiAlias(true);
// Define pose connections (adjust based on your model's keypoint format)
const connections = [
[5, 7], [7, 9], // left arm
[6, 8], [8, 10], // right arm
[11, 13], [13, 15], // left leg
[12, 14], [14, 16], // right leg
[5, 6], // shoulders
[11, 12], // hips
[5, 11], [6, 12] // torso
];
const rotation = Platform.OS === 'ios' ? '0deg' : '270deg';
const frameProcessor = useSkiaFrameProcessor((frame) => {
'worklet'
try {
if (plugin?.state === 'loaded' && plugin.model) {
// Resize frame for model input
const resizedFrame = resize(frame, {
scale: {
width: inputWidth,
height: inputHeight,
},
pixelFormat: 'rgb',
dataType: 'uint8',
rotation: rotation,
});
// Run inference
const outputs = plugin.model.runSync([resizedFrame]);
const output = outputs[0] as Float32Array;
const frameWidth = frame.width;
const frameHeight = frame.height;
// Only draw overlays if we have valid output
if (output && output.length >= 51) { // 17 keypoints * 3 values = 51
console.log('Drawing pose with', output.length, 'values');
// Draw connections first
for (const connection of connections) {
const [startIdx, endIdx] = connection;
const startConf = output[startIdx * 3 + 2];
const endConf = output[endIdx * 3 + 2];
if (startConf > MIN_CONFIDENCE && endConf > MIN_CONFIDENCE) {
const startY = output[startIdx * 3] * frameHeight;
const startX = output[startIdx * 3 + 1] * frameWidth;
const endY = output[endIdx * 3] * frameHeight;
const endX = output[endIdx * 3 + 1] * frameWidth;
frame.drawLine(startX, startY, endX, endY, linePaint);
}
}
// Draw keypoints
for (let i = 0; i < 17; i++) {
const confidence = output[i * 3 + 2];
if (confidence > MIN_CONFIDENCE) {
const y = output[i * 3] * frameHeight;
const x = output[i * 3 + 1] * frameWidth;
frame.drawCircle(x, y, 6, pointPaint);
}
}
}
}
// Always draw a small indicator to show the frame processor is working
const indicatorPaint = Skia.Paint();
indicatorPaint.setColor(Skia.Color('lime'));
frame.drawCircle(50, 50, 10, indicatorPaint);
} catch (error) {
console.error('Frame processor error:', error);
// Draw error indicator
const errorPaint = Skia.Paint();
errorPaint.setColor(Skia.Color('red'));
frame.drawCircle(50, 50, 15, errorPaint);
}
}, [plugin, linePaint, pointPaint]);
// Show different states
if (!hasPermission) {
return (
<View style={styles.container}>
<Text style={styles.message}>Camera permission required</Text>
<Text style={styles.debug}>{debugInfo}</Text>
</View>
);
}
if (device == null) {
return (
<View style={styles.container}>
<Text style={styles.message}>No camera device found</Text>
<Text style={styles.debug}>{debugInfo}</Text>
</View>
);
}
return (
<View style={StyleSheet.absoluteFill}>
{/* Debug overlay */}
<View style={styles.debugOverlay}>
<Text style={styles.debugText}>{debugInfo}</Text>
<Text style={styles.debugText}>Camera: {device.position}</Text>
<Text style={styles.debugText}>Model: {inputWidth}x{inputHeight}</Text>
</View>
<Camera
style={StyleSheet.absoluteFill}
device={device}
isActive={true}
frameProcessor={frameProcessor}
pixelFormat={Platform.OS === 'ios' ? 'rgb' : 'yuv'}
onInitialized={() => {
console.log('Camera initialized!');
setDebugInfo(prev => prev + '\nCamera: Initialized');
}}
onError={(error) => {
console.error('Camera error:', error);
Alert.alert('Camera Error', `${error.message}`);
setDebugInfo(prev => prev + `\nError: ${error.message}`);
}}
/>
</View>
);
}
Package.json :
{
"name": "new-pose-detection",
"main": "expo-router/entry",
"version": "1.0.0",
"scripts": {
"start": "expo start",
"reset-project": "node ./scripts/reset-project.js",
"android": "expo run:android",
"ios": "expo run:ios",
"web": "expo start --web",
"lint": "expo lint"
},
"dependencies": {
"@expo/vector-icons": "^15.0.2",
"@react-navigation/bottom-tabs": "^7.4.0",
"@react-navigation/elements": "^2.6.3",
"@react-navigation/native": "^7.1.8",
"@shopify/react-native-skia": "^2.2.17",
"expo": "~54.0.8",
"expo-build-properties": "~1.0.8",
"expo-camera": "~17.0.8",
"expo-constants": "~18.0.9",
"expo-dev-client": "~6.0.12",
"expo-font": "~14.0.8",
"expo-haptics": "~15.0.7",
"expo-image": "~3.0.8",
"expo-linking": "~8.0.8",
"expo-router": "~6.0.6",
"expo-splash-screen": "~31.0.10",
"expo-status-bar": "~3.0.8",
"expo-symbols": "~1.0.7",
"expo-system-ui": "~6.0.7",
"expo-web-browser": "~15.0.7",
"react": "19.1.0",
"react-dom": "19.1.0",
"react-native": "0.81.4",
"react-native-fast-tflite": "^1.6.1",
"react-native-gesture-handler": "~2.28.0",
"react-native-reanimated": "~4.1.0",
"react-native-safe-area-context": "~5.6.0",
"react-native-screens": "~4.16.0",
"react-native-vision-camera": "^4.7.2",
"react-native-web": "~0.21.0",
"react-native-worklets": "0.5.1",
"react-native-worklets-core": "^1.6.2",
"vision-camera-resize-plugin": "^3.2.0"
},
"devDependencies": {
"@react-native-community/cli": "^20.0.2",
"@types/react": "~19.1.0",
"eslint": "^9.25.0",
"eslint-config-expo": "~10.0.0",
"typescript": "~5.9.2"
},
"private": true
}
