FTBFS on x32 architecture

[bigon]

Hello,

openh264 FTBFS on debian x32 architecture:

The issue is caused by two things:

1. The -m64 flag passed to the compiler, on this architecture it should either be -mx32or no flag at all should be passed
2. When calling nasm, the elf64 format is used, it should be elfx32 here again

A new flavour of x86 should be added here

[bigon]

The following patch fixes the build (ARCH=x32 make):

Index: openh264-2.2.0+dfsg/build/platform-linux.mk
===================================================================
--- openh264-2.2.0+dfsg.orig/build/platform-linux.mk
+++ openh264-2.2.0+dfsg/build/platform-linux.mk
@@ -13,6 +13,8 @@ AR_OPTS = crD $@
 ifeq ($(ASM_ARCH), x86)
 ifeq ($(ARCH), x86_64)
 ASMFLAGS += -f elf64
+else ifeq ($(ARCH), x32)
+ASMFLAGS += -f elfx32
 else
 ASMFLAGS += -f elf
 endif
Index: openh264-2.2.0+dfsg/codec/common/x86/asm_inc.asm
===================================================================
--- openh264-2.2.0+dfsg.orig/codec/common/x86/asm_inc.asm
+++ openh264-2.2.0+dfsg/codec/common/x86/asm_inc.asm
@@ -135,6 +135,9 @@ BITS 64
 %ifidn __OUTPUT_FORMAT__,elf64
 SECTION .note.GNU-stack noalloc noexec nowrite progbits ; Mark the stack as non-executable
 %endif
+%ifidn __OUTPUT_FORMAT__,elfx32
+SECTION .note.GNU-stack noalloc noexec nowrite progbits ; Mark the stack as non-executable
+%endif
 
 %define arg1 rdi
 %define arg2 rsi
Index: openh264-2.2.0+dfsg/build/x86-common.mk
===================================================================
--- openh264-2.2.0+dfsg.orig/build/x86-common.mk
+++ openh264-2.2.0+dfsg/build/x86-common.mk
@@ -1,5 +1,6 @@
 CFLAGS_M32=-m32
 CFLAGS_M64=-m64
+CFLAGS_MX32=-mx32
 ASM_INCLUDES = -I$(SRC_PATH)codec/common/x86/
 ifneq ($(ENABLE64BIT),)
 ifeq ($(ENABLE64BIT), Yes)
@@ -12,6 +13,10 @@ ifeq ($(ARCH), x86_64)
 CFLAGS += $(CFLAGS_M64)
 LDFLAGS += $(CFLAGS_M64)
 ASMFLAGS_PLATFORM = -DUNIX64
+else ifeq ($(ARCH), x32)
+CFLAGS += $(CFLAGS_MX32)
+LDFLAGS += $(CFLAGS_MX32)
+ASMFLAGS_PLATFORM = -DUNIX64
 else
 CFLAGS += $(CFLAGS_M32)
 LDFLAGS += $(CFLAGS_M32)
@@ -24,7 +29,7 @@ endif
 endif
 ifeq ($(USE_ASM),Yes)
 CFLAGS += -DX86_ASM
-ifneq ($(ARCH), x86_64)
+ifeq ($(filter x86_64 x32, $(ARCH)),)
 CFLAGS += -DX86_32_ASM
 endif
 ASM_ARCH = x86
Index: openh264-2.2.0+dfsg/Makefile
===================================================================
--- openh264-2.2.0+dfsg.orig/Makefile
+++ openh264-2.2.0+dfsg/Makefile
@@ -7,8 +7,8 @@ vpath %.S $(SRC_PATH)
 vpath %.rc $(SRC_PATH)
 vpath %.pc.in $(SRC_PATH)
 
-OS=$(shell uname | tr A-Z a-z | tr -d \\-0-9. | sed -E 's/^(net|open|free)bsd/bsd/')
-ARCH=$(shell uname -m)
+OS?=$(shell uname | tr A-Z a-z | tr -d \\-0-9. | sed -E 's/^(net|open|free)bsd/bsd/')
+ARCH?=$(shell uname -m)
 LIBPREFIX=lib
 LIBSUFFIX=a
 CCAS=$(CC)
Index: openh264-2.2.0+dfsg/build/arch.mk
===================================================================
--- openh264-2.2.0+dfsg.orig/build/arch.mk
+++ openh264-2.2.0+dfsg/build/arch.mk
@@ -1,7 +1,7 @@
 #for x86
 HAVE_AVX2 := Yes
 
-ifneq ($(filter %86 x86_64, $(ARCH)),)
+ifneq ($(filter %86 x86_64 x32, $(ARCH)),)
 include $(SRC_PATH)build/x86-common.mk
 ifeq ($(USE_ASM), Yes)
 ifeq ($(HAVE_AVX2), Yes)

For meson, it's not clear to me how this can be fixed

[bigon]

Hello,

Any news on this?

[GuangweiWang]

there are many x86 and x86_64 assembly code in openh264. I am not sure this code works fine on x32 ABI. have you done any testing about it?

[bigon]

Not really now, it just fixes the compilation

[AndreiCherniaev]

I am thinking how to fix it in meson, may be this lines should be fixed:

    asm_args += ['-DX86_32', '-DHAVE_AVX2']
    add_project_arguments('-DHAVE_AVX2', language: 'cpp')
    add_project_arguments('-DHAVE_AVX2', '-DX86_ASM', '-DX86_32_ASM', language: 'c')

All three HAVE_AVX2, X86_ASM, X86_32_ASM should be disabled by default because they call SIMD which can be not supported.

How to fix? Let's try to use SIMD module. I need help...

[amyspark]

@AndreiCherniaev I don't see any cpuid checks, so I assume the maximum instruction set is picked at build time. I think this should suffice:

diff --git a/meson.build b/meson.build
index 573f6c60..05e3f70e 100644
--- a/meson.build
+++ b/meson.build
@@ -66,15 +66,13 @@ if ['linux', 'android', 'ios', 'darwin'].contains(system)
   endif
   if cpu_family == 'x86'
     asm_format = asm_format32
-    asm_args += ['-DX86_32', '-DX86_32_PICASM', '-DHAVE_AVX2']
-    add_project_arguments('-DHAVE_AVX2', language: 'cpp')
-    add_project_arguments('-DHAVE_AVX2', '-DX86_ASM', '-DX86_32_ASM', language: 'c')
+    asm_args += ['-DX86_32', '-DX86_32_PICASM']
+    add_project_arguments('-DX86_ASM', '-DX86_32_ASM', language: 'c')
     asm_inc = join_paths(meson.current_source_dir(), 'codec', 'common', 'x86', '')
   elif cpu_family == 'x86_64'
     asm_format = asm_format64
-    asm_args += ['-DUNIX64', '-DHAVE_AVX2']
-    add_project_arguments('-DHAVE_AVX2', language: 'cpp')
-    add_project_arguments('-DHAVE_AVX2', '-DX86_ASM', language: 'c')
+    asm_args += ['-DUNIX64']
+    add_project_arguments('-DX86_ASM', language: 'c')
     asm_inc = join_paths(meson.current_source_dir(), 'codec', 'common', 'x86', '')
   elif cpu_family == 'arm'
     asm_format = asm_format32
@@ -143,6 +141,14 @@ else
   error('FIXME: Unhandled system @0@'.format(system))
 endif
 
+if cpu_family in ['x86', 'x86_64']
+  if cc.get_define('_M_IX86_FP') == '2' or cc.has_define('__AVX2__')
+    asm_args += ['-DHAVE_AVX2']
+    add_project_arguments('-DHAVE_AVX2', language: 'cpp')
+    add_project_arguments('-DHAVE_AVX2', language: 'c')
+  endif
+endif
+
 use_asm_gen = false
 if cpu_family in ['x86', 'x86_64']
   nasm = find_program('nasm')

The check needs to be done against the __AVX2__ macro and its equivalent in MSVC. It's the responsibility of the user to specify /arch:AVX2 or -mavx2 in c_args and cpp_args when configuring.