tflite_flutter_plugin icon indicating copy to clipboard operation
tflite_flutter_plugin copied to clipboard

No Output at All with Custom Model. No errors too?

Open mtgnoah opened this issue 3 years ago • 0 comments

I am very puzzled by this issue because it seems to run fine but I get no models and when I trained the model it was doing great. It is a yolov5 model that got converted into tflite using this. Here is what I get for the console when I run.

 Camera view is doing something
I/flutter ( 5720): Labels loaded successfully
I/flutter ( 5720): Model has loaded
I/flutter ( 5720): [Tensor{_tensor: Pointer<TfLiteTensor>: address=0x7655609f6140, name: Identity, type: TfLiteType.float32, 
shape: [1, 6300, 12], data:  302400]
I/flutter ( 5720): [Tensor{_tensor: Pointer<TfLiteTensor>: address=0x7655609ec040, name: input_1, type: TfLiteType.float32, shape: [1, 320, 320, 3], data:  1228800]
I/flutter ( 5720): 1
I/flutter ( 5720): [[], [1, 6300, 12]]
I/flutter ( 5720): [TfLiteType.float32]
I/flutter ( 5720): Outputs created
I/flutter ( 5720): Image is being scanned
I/flutter ( 5720): Scanning now
I/flutter ( 5720): Infering now
I/flutter ( 5720): Model has loaded
I/flutter ( 5720): [Tensor{_tensor: Pointer<TfLiteTensor>: address=0x7655609f6140, name: Identity, type: TfLiteType.float32, shape: [1, 6300, 12], data:  302400]
[Tensor{_tensor: Pointer<TfLiteTensor>: address=0x7655609ec040, name: input_1, type: TfLiteType.float32, shape: [1, 320, 320, 3], data:  1228800]
I/flutter ( 5720): 1
I/flutter ( 5720): [[], [1, 6300, 12]]
I/flutter ( 5720): [TfLiteType.float32]
I/flutter ( 5720): Outputs created
I/flutter ( 5720): Labels loaded successfully
I/flutter ( 5720): Image Processed
I/flutter ( 5720): This is results count: 0
I/flutter ( 5720): Scans List:[]
I/flutter ( 5720): This is the[]

Here is the code

import 'package:image/image.dart' as imageLib;
class IsolateUtils {
  static const String DEBUG_NAME = "InferenceIsolate";

  late Isolate _isolate;
  ReceivePort _receivePort = ReceivePort();
  late SendPort _sendPort;

  SendPort get sendPort => _sendPort;

  Future<void> start() async {
    _isolate = await Isolate.spawn<SendPort>(
      entryPoint,
      _receivePort.sendPort,
      debugName: DEBUG_NAME,
    );

    _sendPort = await _receivePort.first;
  }

  static void entryPoint(SendPort sendPort) async {
    final port = ReceivePort();
    sendPort.send(port.sendPort);

    await for (final IsolateData? isolateData in port) {
      if (isolateData != null) {
        Classifier classifier = Classifier(
            interpreter:
                Interpreter.fromAddress(isolateData.interpreterAddress),
            labels: isolateData.labels);
        imageLib.Image image = isolateData.cameraImage;

        if (Platform.isAndroid) {
          image = imageLib.copyRotate(image, 90);
        }
        Map<String, dynamic>? results = classifier.predict(image);
        isolateData.responsePort.send(results);
      }
    }
  }
}

/// Bundles data to pass between Isolate
class IsolateData {
  imageLib.Image cameraImage;
  int interpreterAddress;
  List<String> labels;
  late SendPort responsePort;

  IsolateData(
    this.cameraImage,
    this.interpreterAddress,
    this.labels,
  );
}
import 'package:image/image.dart' as imageLib;

/// [CameraView] sends each frame for inference
class CameraView extends StatefulWidget {
  /// Callback to pass results after inference to [HomeView]
  final Function(List<Scan> scans) resultsCallback;
  final String imageStarter;

  /// Callback to inference stats to [HomeView]

  /// Constructor
  const CameraView(this.resultsCallback, this.imageStarter);
  @override
  _CameraViewState createState() => _CameraViewState();
}

class _CameraViewState extends State<CameraView> with WidgetsBindingObserver {
  // /// List of available cameras
  // List<CameraDescription> cameras;

  // /// Controller
  // CameraController cameraController;

  /// true when inference is ongoing
  late bool predicting;

  /// Instance of [Classifier]
  late Classifier classifier;

  /// Instance of [IsolateUtils]
  late IsolateUtils isolateUtils;

  @override
  void initState() {
    super.initState();
    initStateAsync();
  }

  void initStateAsync() async {
    WidgetsBinding.instance!.addObserver(this);
    print("Camera view is doing something");
    // Spawn a new isolate
    isolateUtils = IsolateUtils();
    await isolateUtils.start();

    // Camera initialization
    //initializeCamera();
    startScan();
    // Create an instance of classifier to load model and labels
    classifier = Classifier();

    // Initially predicting = false
    predicting = false;
  }

  void startScan() async {
    final bytes = await File(widget.imageStarter).readAsBytes();
    final imageLib.Image? image = imageLib.decodeImage(bytes);
    print("Image is being scanned");
    await imageScanner(image!);
  }

  @override
  Widget build(BuildContext context) {
    // Return empty container while the camera is not initialized
    // if (cameraController == null || !cameraController.value.isInitialized) {
    //   return Container();
    // }

    return Container();
  }

  /// function to scan tickets fast
  imageScanner(imageLib.Image cameraImage) async {
    if (predicting) {
      return;
    }

    setState(() {
      predicting = true;
    });
    print("Scanning now");
    // var uiThreadTimeStart = DateTime.now().millisecondsSinceEpoch;

    // Data to be passed to inference isolate
    var isolateData = IsolateData(
      cameraImage,
      classifier.interpreter.address,
      classifier.labels,
    );

    // We could have simply used the compute method as well however
    // it would be as in-efficient as we need to continuously passing data
    // to another isolate.

    /// perform inference in separate isolate
    Map<String, dynamic> inferenceResults = await inference(isolateData);
    //print(inferenceResults["scans"]);
    // var uiThreadInferenceElapsedTime =
    //     DateTime.now().millisecondsSinceEpoch - uiThreadTimeStart;

    // pass results to HomeView
    widget.resultsCallback(inferenceResults["scans"]);

    // pass stats to HomeView
    // widget.statsCallback((inferenceResults["stats"] as Stats)
    //   ..totalElapsedTime = uiThreadInferenceElapsedTime);

    // set predicting to false to allow new frames
    setState(() {
      predicting = false;
    });
  }

  /// Runs inference in another isolate
  Future<Map<String, dynamic>> inference(IsolateData isolateData) async {
    print("Infering now");
    ReceivePort responsePort = ReceivePort();
    isolateUtils.sendPort
        .send(isolateData..responsePort = responsePort.sendPort);
    var results = await responsePort.first;
    return results;
  }

  // @override
  // void didChangeAppLifecycleState(AppLifecycleState state) async {
  //   switch (state) {
  //     case AppLifecycleState.paused:
  //       cameraController.stopImageStream();
  //       break;
  //     case AppLifecycleState.resumed:
  //       if (!cameraController.value.isStreamingImages) {
  //         await cameraController.startImageStream(onLatestImageAvailable);
  //       }
  //       break;
  //     default:
  //   }
  // }

  @override
  void dispose() {
    WidgetsBinding.instance!.removeObserver(this);
    // cameraController.dispose();
    super.dispose();
  }
}

mtgnoah avatar Aug 20 '21 16:08 mtgnoah