libnfs
libnfs copied to clipboard
sahlberg
Error in `/qa/libnfs/examples/.libs/lt-nfsclient-raw': free(): invalid next size (fast): 0x00000000020523f0
$ cat /etc/centos-release CentOS Linux release 7.4.1708 (Core) $ uname -r 3.10.0-693.el7.vastos.6.x86_64
Connect to RPC.RQUOTAD on 172.35.37.10:0 *** Error in `/qa/libnfs/examples/.libs/lt-nfsclient-raw': free(): invalid next size (fast): 0x00000000020523f0 *** ======= Backtrace: ========= /lib64/libc.so.6(+0x81609)[0x7f172f9d7609] /qa/libnfs/lib/.libs/libnfs.so.13(libnfs_zdr_destroy+0x23)[0x7f172ff45df3] /qa/libnfs/lib/.libs/libnfs.so.13(rpc_process_pdu+0x618)[0x7f172ff57aa8] /qa/libnfs/lib/.libs/libnfs.so.13(rpc_service+0x7f0)[0x7f172ff58bc0] /qa/libnfs/examples/.libs/lt-nfsclient-raw[0x400dab] /lib64/libc.so.6(__libc_start_main+0xf5)[0x7f172f978495] /qa/libnfs/examples/.libs/lt-nfsclient-raw[0x400e3a] ======= Memory map: ======== 00400000-00403000 r-xp 00000000 08:02 427568 /qa/libnfs/examples/.libs/lt-nfsclient-raw 00602000-00603000 r--p 00002000 08:02 427568 /qa/libnfs/examples/.libs/lt-nfsclient-raw 00603000-00604000 rw-p 00003000 08:02 427568 /qa/libnfs/examples/.libs/lt-nfsclient-raw 0204e000-02073000 rw-p 00000000 00:00 0 [heap] 7f1728000000-7f1728021000 rw-p 00000000 00:00 0 7f1728021000-7f172c000000 ---p 00000000 00:00 0 7f172f329000-7f172f33e000 r-xp 00000000 08:02 786452 /usr/lib64/libgcc_s-4.8.5-20150702.so.1 7f172f33e000-7f172f53d000 ---p 00015000 08:02 786452 /usr/lib64/libgcc_s-4.8.5-20150702.so.1 7f172f53d000-7f172f53e000 r--p 00014000 08:02 786452 /usr/lib64/libgcc_s-4.8.5-20150702.so.1 7f172f53e000-7f172f53f000 rw-p 00015000 08:02 786452 /usr/lib64/libgcc_s-4.8.5-20150702.so.1 7f172f53f000-7f172f54b000 r-xp 00000000 08:02 788468 /usr/lib64/libnss_files-2.17.so 7f172f54b000-7f172f74a000 ---p 0000c000 08:02 788468 /usr/lib64/libnss_files-2.17.so 7f172f74a000-7f172f74b000 r--p 0000b000 08:02 788468 /usr/lib64/libnss_files-2.17.so 7f172f74b000-7f172f74c000 rw-p 0000c000 08:02 788468 /usr/lib64/libnss_files-2.17.so 7f172f74c000-7f172f752000 rw-p 00000000 00:00 0 7f172f752000-7f172f754000 r-xp 00000000 08:02 788456 /usr/lib64/libdl-2.17.so 7f172f754000-7f172f954000 ---p 00002000 08:02 788456 /usr/lib64/libdl-2.17.so 7f172f954000-7f172f955000 r--p 00002000 08:02 788456 /usr/lib64/libdl-2.17.so 7f172f955000-7f172f956000 rw-p 00003000 08:02 788456 /usr/lib64/libdl-2.17.so 7f172f956000-7f172fb18000 r-xp 00000000 08:02 788450 /usr/lib64/libc-2.17.so 7f172fb18000-7f172fd18000 ---p 001c2000 08:02 788450 /usr/lib64/libc-2.17.so 7f172fd18000-7f172fd1c000 r--p 001c2000 08:02 788450 /usr/lib64/libc-2.17.so 7f172fd1c000-7f172fd1e000 rw-p 001c6000 08:02 788450 /usr/lib64/libc-2.17.so 7f172fd1e000-7f172fd23000 rw-p 00000000 00:00 0 7f172fd23000-7f172fd2c000 r-xp 00000000 08:02 789047 /usr/lib64/libpopt.so.0.0.0 7f172fd2c000-7f172ff2b000 ---p 00009000 08:02 789047 /usr/lib64/libpopt.so.0.0.0 7f172ff2b000-7f172ff2c000 r--p 00008000 08:02 789047 /usr/lib64/libpopt.so.0.0.0 7f172ff2c000-7f172ff2d000 rw-p 00009000 08:02 789047 /usr/lib64/libpopt.so.0.0.0 7f172ff2d000-7f172ff7d000 r-xp 00000000 08:02 427200 /qa/libnfs/lib/.libs/libnfs.so.13.0.0 7f172ff7d000-7f173017c000 ---p 00050000 08:02 427200 /qa/libnfs/lib/.libs/libnfs.so.13.0.0 7f173017c000-7f173017d000 r--p 0004f000 08:02 427200 /qa/libnfs/lib/.libs/libnfs.so.13.0.0 7f173017d000-7f173017f000 rw-p 00050000 08:02 427200 /qa/libnfs/lib/.libs/libnfs.so.13.0.0 7f173017f000-7f1730185000 r-xp 00000000 08:02 427601 /qa/libnfs/examples/ld_nfs.so 7f1730185000-7f1730385000 ---p 00006000 08:02 427601 /qa/libnfs/examples/ld_nfs.so 7f1730385000-7f1730386000 r--p 00006000 08:02 427601 /qa/libnfs/examples/ld_nfs.so 7f1730386000-7f1730387000 rw-p 00007000 08:02 427601 /qa/libnfs/examples/ld_nfs.so 7f1730387000-7f1730389000 rw-p 00000000 00:00 0 7f1730389000-7f17303ab000 r-xp 00000000 08:02 788443 /usr/lib64/ld-2.17.so 7f1730598000-7f173059c000 rw-p 00000000 00:00 0 7f17305a6000-7f17305aa000 rw-p 00000000 00:00 0 7f17305aa000-7f17305ab000 r--p 00021000 08:02 788443 /usr/lib64/ld-2.17.so 7f17305ab000-7f17305ac000 rw-p 00022000 08:02 788443 /usr/lib64/ld-2.17.so 7f17305ac000-7f17305ad000 rw-p 00000000 00:00 0 7ffc920bf000-7ffc920e0000 rw-p 00000000 00:00 0 [stack] 7ffc921d9000-7ffc921db000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] Aborted
This may be related to a problem I'm seeing. There's a buffer overrun in libnfs_zdr_int when processing zdr_mountres3_ok. Specifically, a zero-byte buffer is allocated in libnfs_zdr_array and is then written to in libnfs_zdr_int.
$ uname -a
Linux ff1 4.4.177-iio3 #1 SMP Sat Dec 14 14:26:11 CST 2019 x86_64 x86_64 x86_64 GNU/Linux
And the last commit in my local copy (given as a crude versioning):
commit a7b99949715cb6d62af7d9070a7a71220472c09a
Merge: ac60fea 4b12b5e
Author: Ronnie Sahlberg <[email protected]>
Date: Tue Oct 22 23:10:56 2019 +1000
Here's how the zero-byte buffer is allocated:
$ sudo gdb nfs-ls nfs://172.16.11.99/exports/share1/
(gdb) set args nfs://172.16.11.99/exports/share1/
(gdb) b zdr_mountres3_ok
(gdb) run
Breakpoint 1, zdr_mountres3_ok (zdrs=0x7fffffffe3c0, objp=0x609900) at libnfs-raw-mount.c:131
131 if (!zdr_fhandle3 (zdrs, &objp->fhandle))
(gdb) b libnfs-zdr.c:95
Breakpoint 2 at 0x7ffff7b90db9: file libnfs-zdr.c, line 95.
(gdb) b libnfs_zdr_array
Breakpoint 3 at 0x7ffff7b916a4: file libnfs-zdr.c, line 322.
(gdb) c
Continuing.
Breakpoint 3, libnfs_zdr_array (zdrs=0x7fffffffe3c0, arrp=0x609918, size=0x609910, maxsize=4294967295, elsize=4, proc=0x7ffff7b90ecf <libnfs_zdr_int>) at libnfs-zdr.c:322
322 s = (*size * elsize) & 0xffffffff;
(gdb) c
Continuing.
Breakpoint 2, zdr_malloc (zdrs=0x7fffffffe3c0, size=0) at libnfs-zdr.c:95
95 return &mem->buf[0];
(gdb) p *zdrs
$2 = {x_op = ZDR_DECODE, buf = 0x608870 "\200", size = 72, pos = 68, mem = 0x6075a0}
(gdb) p *zdrs->mem
$3 = {next = 0x0, size = 0, buf = ""}
The current set of frames:
(gdb) where
#0 zdr_malloc (zdrs=0x7fffffffe3c0, size=0) at libnfs-zdr.c:95
#1 0x00007ffff7b91707 in libnfs_zdr_array (zdrs=0x7fffffffe3c0, arrp=0x609918, size=0x609910, maxsize=4294967295, elsize=4, proc=0x7ffff7b90ecf <libnfs_zdr_int>)
at libnfs-zdr.c:332
#2 0x00007ffff7bafae7 in zdr_mountres3_ok (zdrs=0x7fffffffe3c0, objp=0x609900) at libnfs-raw-mount.c:133
#3 0x00007ffff7bafb4a in zdr_mountres3 (zdrs=0x7fffffffe3c0, objp=0x6098f8) at libnfs-raw-mount.c:146
#4 0x00007ffff7b91a06 in libnfs_accepted_reply (zdrs=0x7fffffffe3c0, ar=0x7fffffffe300) at libnfs-zdr.c:417
#5 0x00007ffff7b91b9f in libnfs_rpc_reply_body (rpc=0x6030b0, zdrs=0x7fffffffe3c0, rmb=0x7fffffffe2f8) at libnfs-zdr.c:473
#6 0x00007ffff7b91d36 in libnfs_rpc_msg (rpc=0x6030b0, zdrs=0x7fffffffe3c0, msg=0x7fffffffe2f0) at libnfs-zdr.c:516
#7 0x00007ffff7b91de5 in libnfs_zdr_replymsg (rpc=0x6030b0, zdrs=0x7fffffffe3c0, msg=0x7fffffffe2f0) at libnfs-zdr.c:536
#8 0x00007ffff7babdfe in rpc_process_reply (rpc=0x6030b0, pdu=0x609880, zdr=0x7fffffffe3c0) at pdu.c:296
#9 0x00007ffff7baca10 in rpc_process_pdu (rpc=0x6030b0, buf=0x608870 "\200", size=72) at pdu.c:575
#10 0x00007ffff7bad2af in rpc_read_from_socket (rpc=0x6030b0) at socket.c:360
#11 0x00007ffff7bad8e8 in rpc_service (rpc=0x6030b0, revents=1) at socket.c:505
#12 0x00007ffff7b898ab in nfs_service (nfs=0x603010, revents=1) at libnfs.c:246
#13 0x00007ffff7b8d93f in wait_for_nfs_reply (nfs=0x603010, cb_data=0x7fffffffe500) at libnfs-sync.c:186
#14 0x00007ffff7b8daa9 in nfs_mount (nfs=0x603010, server=0x607410 "172.16.11.99", export=0x607440 "/exports/share1/") at libnfs-sync.c:231
#15 0x0000000000401783 in main (argc=2, argv=0x7fffffffe6f8) at nfs-ls.c:246
And here's where the overrun begins:
[... snip stepping ...]
(gdb) s
40 if (!proc(zdrs, *arrp + i * elsize)) {
(gdb) p &zdrs->mem.buf
$8 = (char (*)[1]) 0x6075ac
(gdb) p *arrp
$9 = 0x6075ac ""
(gdb) p proc
$10 = (zdrproc_t) 0x7ffff7b90ecf <libnfs_zdr_int>
[... snip stepping ...]
(gdb) s
libnfs_zdr_u_int (zdrs=0x7fffffffe3c0, u=0x6075ac) at libnfs-zdr.c:109
109 if (zdrs->pos + 4 > zdrs->size) {
(gdb) n
113 switch (zdrs->x_op) {
(gdb) n
119 *u = ntohl(*(uint32_t *)(void *)&zdrs->buf[zdrs->pos]);
(gdb) p u
$11 = (uint32_t *) 0x6075ac
which is the address of the zero-byte buffer.
And the frames:
(gdb) where
#0 libnfs_zdr_u_int (zdrs=0x7fffffffe3c0, u=0x6075ac) at libnfs-zdr.c:119
#1 0x00007ffff7b90ef2 in libnfs_zdr_int (zdrs=0x7fffffffe3c0, i=0x6075ac) at libnfs-zdr.c:129
#2 0x00007ffff7b9176a in libnfs_zdr_array (zdrs=0x7fffffffe3c0, arrp=0x609918, size=0x609910, maxsize=4294967295, elsize=4, proc=0x7ffff7b90ecf <libnfs_zdr_int>)
at libnfs-zdr.c:340
#3 0x00007ffff7bafae7 in zdr_mountres3_ok (zdrs=0x7fffffffe3c0, objp=0x609900) at libnfs-raw-mount.c:133
#4 0x00007ffff7bafb4a in zdr_mountres3 (zdrs=0x7fffffffe3c0, objp=0x6098f8) at libnfs-raw-mount.c:146
#5 0x00007ffff7b91a06 in libnfs_accepted_reply (zdrs=0x7fffffffe3c0, ar=0x7fffffffe300) at libnfs-zdr.c:417
#6 0x00007ffff7b91b9f in libnfs_rpc_reply_body (rpc=0x6030b0, zdrs=0x7fffffffe3c0, rmb=0x7fffffffe2f8) at libnfs-zdr.c:473
#7 0x00007ffff7b91d36 in libnfs_rpc_msg (rpc=0x6030b0, zdrs=0x7fffffffe3c0, msg=0x7fffffffe2f0) at libnfs-zdr.c:516
#8 0x00007ffff7b91de5 in libnfs_zdr_replymsg (rpc=0x6030b0, zdrs=0x7fffffffe3c0, msg=0x7fffffffe2f0) at libnfs-zdr.c:536
#9 0x00007ffff7babdfe in rpc_process_reply (rpc=0x6030b0, pdu=0x609880, zdr=0x7fffffffe3c0) at pdu.c:296
#10 0x00007ffff7baca10 in rpc_process_pdu (rpc=0x6030b0, buf=0x608870 "\200", size=72) at pdu.c:575
#11 0x00007ffff7bad2af in rpc_read_from_socket (rpc=0x6030b0) at socket.c:360
#12 0x00007ffff7bad8e8 in rpc_service (rpc=0x6030b0, revents=1) at socket.c:505
#13 0x00007ffff7b898ab in nfs_service (nfs=0x603010, revents=1) at libnfs.c:246
#14 0x00007ffff7b8d93f in wait_for_nfs_reply (nfs=0x603010, cb_data=0x7fffffffe500) at libnfs-sync.c:186
#15 0x00007ffff7b8daa9 in nfs_mount (nfs=0x603010, server=0x607410 "172.16.11.99", export=0x607440 "/exports/share1/") at libnfs-sync.c:231
#16 0x0000000000401783 in main (argc=2, argv=0x7fffffffe6f8) at nfs-ls.c:246
This patch resolves the problem. It won't merge directly into master as I'm running slightly older code, but the same problem exists.
Allocation size s was computed before the decode via libnfs_zdr_u_int was done, so it was wrong--it would compute s bytes (0 bytes), decode size as 1, allocate 0 bytes, and then write 4 bytes (size 1 * sizeof(uint32)) into the allocation. This patch changes the order to decode (1 size), compute (4 bytes), allocate (4 bytes), write.
Tested in the real world (crash goes away) and under valgrind (no more reports of writing past the buffer).
diff --git a/lib/libnfs-zdr.c b/lib/libnfs-zdr.c
index 5f0b3ae..2ab4806 100644
--- a/lib/libnfs-zdr.c
+++ b/lib/libnfs-zdr.c
@@ -317,6 +317,11 @@ bool_t libnfs_zdr_string(ZDR *zdrs, char **strp, uint32_t maxsize)
bool_t libnfs_zdr_array(ZDR *zdrs, char **arrp, uint32_t *size, uint32_t maxsize, uint32_t elsize, zdrproc_t proc)
{
int i;
+
+ if (!libnfs_zdr_u_int(zdrs, size)) {
+ return FALSE;
+ }
+
uint32_t s;
s = (*size * elsize) & 0xffffffff;
@@ -324,10 +329,6 @@ bool_t libnfs_zdr_array(ZDR *zdrs, char **arrp, uint32_t *size, uint32_t maxsize
return FALSE;
}
- if (!libnfs_zdr_u_int(zdrs, size)) {
- return FALSE;
- }
-
if (zdrs->x_op == ZDR_DECODE) {
*arrp = zdr_malloc(zdrs, s);
if (*arrp == NULL) {