I am running into an `Undefined symbol` issue with dart-ffi and Go

[Oabraham1]

trafficstars

I am trying to use ffigen to develop a library that I can use to call a go package from dart code.

My current approach is to use the basic setup that you get from running the initial ffigen

flutter create --template=plugin_ffi --platforms=android,ios,linux,macos,windows project_name

I am compiling my go code(CGO) to generate a header file that I then pull into the basic header file provided by running the initial ffigen creation script and use in the C code from the initial basic setup.

Whenever I do this, I get the error Error (Xcode): Undefined symbol: _parse_data

Here is my Go code that generates the header file:

//main.go

package main

/*
#include <stdlib.h>
*/
import "C"
import (
    "bufio"
    "bytes"
    "encoding/json"
    "errors"
    "fmt"
    "io"
    "strings"

    "github.com/erh/gonmea/analyzer"
)

//export parse_data
func parse_data(byteArr []byte) *C.char {
    fmt.Println(string(byteArr))
    in := io.NopCloser(bytes.NewReader(byteArr))
    defer in.Close()

    parser, err := analyzer.NewParser()
    if err != nil {

        return C.CString(fmt.Sprintf("Error: %v", err))
    }

    reader := bufio.NewReader(in)
    for {
        line, _, err := reader.ReadLine()
        if err != nil {
            if errors.Is(err, io.EOF) {
                return nil
            }
            return C.CString(fmt.Sprintf("Error: %v", err))
        }
        line = []byte(strings.TrimSpace(string(line)))
        if len(line) == 0 {
            continue
        }
        msg, err := parser.ParseMessage(line)
        if err != nil {
            return C.CString(fmt.Sprintf("Error: %v", err))
        }
        md, err := json.MarshalIndent(msg, "", "  ")
        if err != nil {
            return C.CString(fmt.Sprintf("Error: %v", err))
        }
        fmt.Println(string(md))
        return C.CString(string(md))
    }
}

func main() {}

Here is the header file generated by the go code

/* Code generated by cmd/cgo; DO NOT EDIT. */

/* package command-line-arguments */


#line 1 "cgo-builtin-export-prolog"

#include <stddef.h>

#ifndef GO_CGO_EXPORT_PROLOGUE_H
#define GO_CGO_EXPORT_PROLOGUE_H

#ifndef GO_CGO_GOSTRING_TYPEDEF
typedef struct { const char *p; ptrdiff_t n; } _GoString_;
#endif

#endif

/* Start of preamble from import "C" comments.  */


#line 3 "main.go"

#include <stdlib.h>

#line 1 "cgo-generated-wrapper"


/* End of preamble from import "C" comments.  */


/* Start of boilerplate cgo prologue.  */
#line 1 "cgo-gcc-export-header-prolog"

#ifndef GO_CGO_PROLOGUE_H
#define GO_CGO_PROLOGUE_H

typedef signed char GoInt8;
typedef unsigned char GoUint8;
typedef short GoInt16;
typedef unsigned short GoUint16;
typedef int GoInt32;
typedef unsigned int GoUint32;
typedef long long GoInt64;
typedef unsigned long long GoUint64;
typedef GoInt64 GoInt;
typedef GoUint64 GoUint;
typedef size_t GoUintptr;
typedef float GoFloat32;
typedef double GoFloat64;
#ifdef _MSC_VER
#include <complex.h>
typedef _Fcomplex GoComplex64;
typedef _Dcomplex GoComplex128;
#else
typedef float _Complex GoComplex64;
typedef double _Complex GoComplex128;
#endif

/*
  static assertion to make sure the file is being used on architecture
  at least with matching size of GoInt.
*/
typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1];

#ifndef GO_CGO_GOSTRING_TYPEDEF
typedef _GoString_ GoString;
#endif
typedef void *GoMap;
typedef void *GoChan;
typedef struct { void *t; void *v; } GoInterface;
typedef struct { void *data; GoInt len; GoInt cap; } GoSlice;

#endif

/* End of boilerplate cgo prologue.  */

#ifdef __cplusplus
extern "C" {
#endif

extern char* parse_data(GoSlice byteArr);

#ifdef __cplusplus
}
#endif

Here is the header file generated from running the initial ffigen creation script

#ifndef GONMEA_FLUTTER_H
#define GONMEA_FLUTTER_H

#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "libgonmea_flutter.h"

#if _WIN32
#include <windows.h>
#else
#include <pthread.h>
#include <unistd.h>
#endif

#if _WIN32
#define FFI_PLUGIN_EXPORT __declspec(dllexport)
#else
#define FFI_PLUGIN_EXPORT __attribute__((visibility("default"))) __attribute__((used))
#endif

// A very short-lived native function.
FFI_PLUGIN_EXPORT intptr_t sum(intptr_t a, intptr_t b);

extern char* parse_data(GoSlice byteArr);

FFI_PLUGIN_EXPORT char* parse_data_wrapper(GoSlice byteArr);

#endif

Here is the C code

#include "gonmea_flutter.h"

FFI_PLUGIN_EXPORT intptr_t sum(intptr_t a, intptr_t b) { 
    return a + b; 
}

FFI_PLUGIN_EXPORT char* parse_data_wrapper(GoSlice byteArr) {
    return parse_data(byteArr);
}

Here is my ffigen.yaml file

# Run with `flutter pub run ffigen --config ffigen.yaml`.
name: GonmeaFlutterBindings
description: |
  Bindings for `src/gonmea_flutter.h`.

  Regenerate bindings with `flutter pub run ffigen --config ffigen.yaml`.
output: 'lib/gonmea_flutter_bindings_generated.dart'
headers:
  entry-points:
    - 'src/gonmea_flutter.h'
  include-directives:
    - 'src/gonmea_flutter.h'
    - 'src/libgonmea_flutter.h'
preamble: |
  // ignore_for_file: always_specify_types
  // ignore_for_file: camel_case_types
  // ignore_for_file: non_constant_identifier_names
comments:
  style: any
  length: full

Here is the autogenerated dart-ffi bindings generated by running flutter pub run ffigen --config ffigen.yaml

// ignore_for_file: always_specify_types
// ignore_for_file: camel_case_types
// ignore_for_file: non_constant_identifier_names

// AUTO GENERATED FILE, DO NOT EDIT.
//
// Generated by `package:ffigen`.
// ignore_for_file: type=lint
import 'dart:ffi' as ffi;

/// Bindings for `src/gonmea_flutter.h`.
///
/// Regenerate bindings with `flutter pub run ffigen --config ffigen.yaml`.
///
class GonmeaFlutterBindings {
  /// Holds the symbol lookup function.
  final ffi.Pointer<T> Function<T extends ffi.NativeType>(String symbolName)
      _lookup;

  /// The symbols are looked up in [dynamicLibrary].
  GonmeaFlutterBindings(ffi.DynamicLibrary dynamicLibrary)
      : _lookup = dynamicLibrary.lookup;

  /// The symbols are looked up with [lookup].
  GonmeaFlutterBindings.fromLookup(
      ffi.Pointer<T> Function<T extends ffi.NativeType>(String symbolName)
          lookup)
      : _lookup = lookup;

  ffi.Pointer<ffi.Char> parse_data(
    GoSlice byteArr,
  ) {
    return _parse_data(
      byteArr,
    );
  }

  late final _parse_dataPtr =
      _lookup<ffi.NativeFunction<ffi.Pointer<ffi.Char> Function(GoSlice)>>(
          'parse_data');
  late final _parse_data =
      _parse_dataPtr.asFunction<ffi.Pointer<ffi.Char> Function(GoSlice)>();

  /// A very short-lived native function.
  int sum(
    int a,
    int b,
  ) {
    return _sum(
      a,
      b,
    );
  }

  late final _sumPtr =
      _lookup<ffi.NativeFunction<ffi.IntPtr Function(ffi.IntPtr, ffi.IntPtr)>>(
          'sum');
  late final _sum = _sumPtr.asFunction<int Function(int, int)>();

  ffi.Pointer<ffi.Char> parse_data_wrapper(
    GoSlice byteArr,
  ) {
    return _parse_data_wrapper(
      byteArr,
    );
  }

  late final _parse_data_wrapperPtr =
      _lookup<ffi.NativeFunction<ffi.Pointer<ffi.Char> Function(GoSlice)>>(
          'parse_data_wrapper');
  late final _parse_data_wrapper = _parse_data_wrapperPtr
      .asFunction<ffi.Pointer<ffi.Char> Function(GoSlice)>();
}

final class _GoString_ extends ffi.Struct {
  external ffi.Pointer<ffi.Char> p;

  @ptrdiff_t()
  external int n;
}

typedef ptrdiff_t = __darwin_ptrdiff_t;
typedef __darwin_ptrdiff_t = ffi.Long;

final class GoInterface extends ffi.Struct {
  external ffi.Pointer<ffi.Void> t;

  external ffi.Pointer<ffi.Void> v;
}

final class GoSlice extends ffi.Struct {
  external ffi.Pointer<ffi.Void> data;

  @GoInt()
  external int len;

  @GoInt()
  external int cap;
}

typedef GoInt = GoInt64;
typedef GoInt64 = ffi.LongLong;

Finally, here is my implementation where I called the auto generated ffi bindings

import 'dart:convert';
import 'dart:ffi';
import 'dart:io';
import 'package:ffi/ffi.dart';

import 'package:flutter/foundation.dart';

import 'gonmea_flutter_bindings_generated.dart';

/// A very short-lived native function.
///
/// For very short-lived functions, it is fine to call them on the main isolate.
/// They will block the Dart execution while running the native function, so
/// only do this for native functions which are guaranteed to be short-lived.
int sum(int a, int b) => _bindings.sum(a, b);


String? _processDataHelper(Uint8List data) {
  final rawDataPtr = calloc<Uint8>(data.length);
  final rawDataList = rawDataPtr.asTypedList(data.length);
  rawDataList.setAll(0, data);

  final goSlicePtr = calloc<GoSlice>();
  goSlicePtr.ref.data = rawDataPtr.cast<Void>();
  goSlicePtr.ref.len = data.length;
  goSlicePtr.ref.cap = data.length;

  final resultPtr = _bindings.parse_data_wrapper(goSlicePtr.ref);

  String? result;
  if (resultPtr != nullptr) {
    result = resultPtr.cast<Utf8>().toDartString();
    calloc.free(resultPtr.cast<Void>());
  }

  // Free remaining allocated memory
  calloc.free(goSlicePtr);
  calloc.free(rawDataPtr);

  return result;
}

Map<dynamic, dynamic> _stringToMap(String dartString) {
  try {
    final jsonString = json.decode(dartString);
    return jsonString;
  } catch (e) {
    return {};
  }
}

Map<dynamic, dynamic> processData(Uint8List data) {
  final readingsString = _processDataHelper(data);
  if (readingsString != null) {
    return _stringToMap(readingsString);
  }
  return {};
}



const String _libName = 'gonmea_flutter';

/// The dynamic library in which the symbols for [GonmeaFlutterBindings] can be found.
final DynamicLibrary _dylib = () {
  if (Platform.isMacOS || Platform.isIOS) {
    return DynamicLibrary.open('$_libName.framework/$_libName');
  }
  if (Platform.isAndroid || Platform.isLinux) {
    return DynamicLibrary.open('lib$_libName.so');
  }
  if (Platform.isWindows) {
    return DynamicLibrary.open('$_libName.dll');
  }
  throw UnsupportedError('Unknown platform: ${Platform.operatingSystem}');
}();

/// The bindings to the native functions in [_dylib].
final GonmeaFlutterBindings _bindings = GonmeaFlutterBindings(_dylib);

Please let me know if there's anything missing 🙏🏻

I have tried to directly use the generated header file in the C code and remove the autogenerated code and that doesn't seem to fix it

[dcharkes]

//export parse_data

That should make the symbol visible.

I am compiling my go code(CGO) to generate a header file that I then pull into the basic header file provided by running the initial ffigen creation script and use in the C code from the initial basic setup.

The C header file only contains the API, no implementation. Are you also compiling the Go code to a dynamic library? How is the dynamic library that you're loading at runtime created?

[Oabraham1]

//export parse_data

That should make the symbol visible.

I am compiling my go code(CGO) to generate a header file that I then pull into the basic header file provided by running the initial ffigen creation script and use in the C code from the initial basic setup.

The C header file only contains the API, no implementation. Are you also compiling the Go code to a dynamic library? How is the dynamic library that you're loading at runtime created?

I am compiling the Go code to a header file and a static library. So, I updated my compilation steps to

My Makefile

LIB_NAME=gonmea_flutter

ios-x86_64-sim:
	GOARCH=amd64 \
	SDK=iphonesimulator \
	LIB_NAME=${LIB_NAME} \
	./build_ios.sh

ios-arm64-sim:
	GOARCH=arm64 \
	SDK=iphonesimulator \
	LIB_NAME=${LIB_NAME} \
	./build_ios.sh

ios-arm64:
	GOARCH=arm64 \
	SDK=iphoneos \
	LIB_NAME=${LIB_NAME} \
	./build_ios.sh

ios: ios-x86_64-sim ios-arm64-sim ios-arm64
	lipo \
    -create \
    ${LIB_NAME}_arm64_iphonesimulator.a \
    ${LIB_NAME}_amd64_iphonesimulator.a \
    -output ${LIB_NAME}_iphonesimulator.a
	rm ${LIB_NAME}_arm64_iphonesimulator.*
	rm ${LIB_NAME}_amd64_iphonesimulator.*

	mkdir -p ios-arm64
	mkdir -p ios-simulator
	mv ./${LIB_NAME}_arm64_iphoneos.a ./ios-arm64/${LIB_NAME}.a
	cp ./${LIB_NAME}_arm64_iphoneos.h ./ios-arm64/${LIB_NAME}.h
	mv ./${LIB_NAME}_iphonesimulator.a ./ios-simulator/${LIB_NAME}.a
	mv ./${LIB_NAME}_arm64_iphoneos.h ./ios-simulator/${LIB_NAME}.h

	xcodebuild -create-xcframework \
			-output ${LIB_NAME}.xcframework \
			-library ios-arm64/${LIB_NAME}.a \
			-headers ios-arm64/${LIB_NAME}.h \
			-library ios-simulator/${LIB_NAME}.a \
			-headers ios-simulator/${LIB_NAME}.h
	rm -rf ios-arm64
	rm -rf ios-arm64-simulator
	rm -rf ios-simulator
	rm -rf ../ios/${LIB_NAME}.xcframework
	mv ${LIB_NAME}.xcframework ../ios/

and my script to build the static library for ios

# build_ios.sh file
#!/bin/sh
export GOOS=ios
export CGO_ENABLED=1
# export CGO_CFLAGS="-fembed-bitcode"
# export MIN_VERSION=15

SDK_PATH=$(xcrun --sdk "$SDK" --show-sdk-path)

if [ "$GOARCH" = "amd64" ]; then
    CARCH="x86_64"
elif [ "$GOARCH" = "arm64" ]; then
    CARCH="arm64"
fi

if [ "$SDK" = "iphoneos" ]; then
  export TARGET="$CARCH-apple-ios$MIN_VERSION"
elif [ "$SDK" = "iphonesimulator" ]; then
  export TARGET="$CARCH-apple-ios$MIN_VERSION-simulator"
fi

CLANG=$(xcrun --sdk "$SDK" --find clang)
CC="$CLANG -target $TARGET -isysroot $SDK_PATH $@"
export CC

go build -trimpath -buildmode=c-archive -o ${LIB_NAME}_${GOARCH}_${SDK}.a

I also removed the C code since I'm just trying to use the compiled Go code directly

[dcharkes]

I am not familiar with how the static libraries inside xcframeworks work, so far I've been only using dynamic libraries.

Maybe the easiest way would be to use flutter channel master && flutter config --enable-native-assets && flutter create --template=package_ffi my_package and then modify hook/build.dart to invoke the go compiler to build a dynamic library and return that.

[shigomany]

I wasted whole day on combining Rust (static library) and Flutter and here's that I will tell you:

1. .xcframework not working in Flutter, prefer use separated libraries in .podspec file via vendored_libraries.
2. It's joke, but you should call every function in dummy functions like here

Then you can access in your library via DynamicLibrary.process()

P.S. You can check your exported functions in nm command line tool.