x86 Floating Point Catastrophically Broken

[tmikov]

Hi, This is totally unexpected, but we have found out that x86 floating point operations do not work correctly in the emulator. I have no idea how this is even possible.

The following very simple program does not print the correct output:

#include <stdio.h>

volatile double D = 0xFFFFFFFF;

int main(void) {
  volatile unsigned U = (unsigned)D;
  printf("%x\n", U);
  return 0;
}

A more detailed test similarly fails:

#include <stdio.h>

// INCORRECT OUTPUT: 
// D: 4294967295.000000, int: 7fffffff
// D: 4294967295.000000, unsigned: 7fffffff
// cvttsd2si: 7fffffff
// fisttp: 7fffffff
volatile double D = 0xFFFFFFFF;
int convertWithSSE2(double n) {
  int result;
  asm("cvttsd2si %1, %0" : "=r"(result) : "x"(n) :);
  return result;
}
int convertWithFPU(double n) {
  int result;
  asm("fldl %1\n\t"
      "fisttpl %0"
      : "=m"(result)
      : "m"(n)
      : "st");
  return result;
}
int main(void) {
  volatile int x = (int)D;
  printf("D: %f, int: %x\n", D, x);
  volatile unsigned ux = (unsigned)D;
  printf("D: %f, unsigned: %x\n", D, ux);
  int u1 = convertWithSSE2(D);
  printf("cvttsd2si: %x\n", u1);
  int u2 = convertWithFPU(D);
  printf("fisttp: %x\n", u2);
  return 0;
}