Optimize POST's transferFrom. (transferFrom is not doing zero copy and IO size is 8K)

Open zzmao opened this issue 7 years ago • 1 comments

An experiment was did by sending one blob to ambry. strace shows that POST is not doing zero copy and two system call was used ('read' and 'pwrite') multiple times when transferFrom is called.

transferFrom is transferring 8K each time. It's relatively small if average blob size is ~120K.

There is a space to optimize POST. The proposal is: Replace transferFrom by two individual calls in Java. The user level buffer for copy can be per thread. Maybe async since both read from network and write to file are blocking call.

@asinghai

read(22, "\212\226\234!\253\355\220\17\341`\235m\3\4~\264\361\342je\7z\343\7\312\5\207t\243r\315\331"..., 1377012) = 130936
epoll_wait(40, {{EPOLLIN, {u32=22, u64=8294161772240175126}}}, 8192, 300) = 1
read(22, ";\355G,\240\202\242wP\355\30\325\232\230\203\231\241\361\343]\30)\354\275\314nn\314\35S\373V"..., 1246076) = 130936
epoll_wait(40, {{EPOLLIN, {u32=22, u64=8294161772240175126}}}, 8192, 300) = 1
read(22, "aNT\22\326\324+\301\335RJn\306N~\324\32[\312\373xvC\207\361\242vz\306<-\357"..., 1115140) = 130936
epoll_wait(40, {{EPOLLIN, {u32=22, u64=8294161772240175126}}}, 8192, 300) = 1
read(22, "\24P=\27\262\213R\210\236\333l\247\21\253\352\215\331\217\312/\357\3777c\3\10H\354\n\3170C"..., 984204) = 130936
epoll_wait(40, {{EPOLLIN, {u32=22, u64=8294161772240175126}}}, 8192, 300) = 1
read(22, "\306\375\335\376\213%\274\347\336\352}\224\363\372J0J\255\274r!\35\25N\342,9\367\374\5\266\324"..., 853268) = 130936
epoll_wait(40, {{EPOLLIN, {u32=22, u64=8294161772240175126}}}, 8192, 300) = 1
futex(0x7f80482b7e54, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7f80482b7e50, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
read(22, "P\2V\247\213\223u\337P\347)\250\224\304z\334ak\222\370YP\235 \231_\327\307\7\275\270\203"..., 722332) = 130936
epoll_wait(40, {{EPOLLIN, {u32=22, u64=8294161772240175126}}}, 8192, 300) = 1
read(22, "\237\363\20\\\244^g\363J\375\255\\\371;\1\257)9\352\225\340b\367\364\303\336)\363\365\25\270\247"..., 591396) = 130936
epoll_wait(40, {{EPOLLIN, {u32=22, u64=8294161772240175126}}}, 8192, 300) = 1
read(22, "\354\250\253<\245\330O\362\34\377@-Fv\250B\271#\4\323\306EM\16WI\212 \207{u\32"..., 460460) = 130936
epoll_wait(40, {{EPOLLIN, {u32=22, u64=8294161772240175126}}}, 8192, 300) = 1
read(22, "\337\252\237\353h\345\201\353\237\252\237f\345\v~sX\247\266/\276\1\6\222\232\205e\241\364\302\216\322"..., 329524) = 130936
epoll_wait(40, {{EPOLLIN, {u32=22, u64=8294161772240175126}}}, 8192, 300) = 1
read(22, "\21\210ME!\325V\205p\323\354\255\325\r\304\314x?r4\321`\361\216(\333\20\357vh\313\330"..., 198588) = 130936
epoll_wait(40, {{EPOLLIN, {u32=22, u64=8294161772240175126}}}, 8192, 300) = 1
read(22, "CU\326\224\33<\303c$\314\23 \3615\247'\251\2528D\317\257$\343k\264/\273R\255\357\254"..., 67652) = 67652



pwrite(29, "\234\306\36207\312\200@\242k:\230\343\2722\31H\7\3322y\317\310 \3551?\265\341#B\237"..., 8192, 2187792) = 8192
pwrite(29, "\373\217\354\224\250\273\4P\245\216\242\341v\226=\326 \216\20A\303\16\316\243@)\330@#\234\2269"..., 8192, 2195984) = 8192
pwrite(29, "\363EG\n\343\266x\v\6\36\3G7\221\267aXAW\275\5\t\23t\211\273X\366.\301\364\27"..., 8192, 2204176) = 8192
pwrite(29, "\336\312fN6\222\26\0267\22\236ou\336\303a2\226\252\225\237\360v\231\202\211K\245K\343\327N"..., 8192, 2212368) = 8192
pwrite(29, "\322\351\251j\205\r\34s\234\214H\34\225\2052-~\253H\17\367\275\301\210C\306\326SC]0\261"..., 8192, 2220560) = 8192
pwrite(29, "\374\26NM\352\10\264Ss\257T\275$h\373\344\216e\207\263\267\330\273\vW\3319\272\246Xo\242"..., 8192, 2228752) = 8192
pwrite(29, "*i\340\252\30\17\260\355\335\270\260\212\217W\314qj\31\351DuFA\267\301\v\332\2647\314\241["..., 8192, 2236944) = 8192
pwrite(29, "\211\237)O\251\257\200\312|\301u\313\n={DL2O4R\243'\252\321,\243\23\f\345\t\275"..., 8192, 2245136) = 8192
pwrite(29, "\2016i\266_!b>MsPe\247\217}{\201{u)0\354\306\223~\324iT\324\211\3\341"..., 8192, 2253328) = 8192
pwrite(29, "\345\242\366\315\302$}\257\277`p\27WA\246+\333\235\20Mx\24\367\313\345w\226\222\26-,A"..., 8192, 2261520) = 8192
pwrite(29, "\36033\376\300\245\223\36j#,\356h\373M\320Q\267\317\333_-\373\262\222l+\207\341\342\250["..., 8192, 2269712) = 8192
pwrite(29, "\f\f\234\246*\1\345\227pr\275\10V\351\224\346\271\200\226\22\5i\21\242\3656\275r\300\214o\314"..., 8192, 2277904) = 8192
pwrite(29, "\212\351\227\252\234;\305\232p)\210\177q\323\322\26\317\244\334\255\10\4\275\20\354Cr\0\301\303\373X"..., 8192, 2286096) = 8192
pwrite(29, "\252\204j\227\222S\321\200;\0313F\235V\260\256\0\5\204\20\224\246\333\311`\226\34\357\336\263\200\321"..., 8192, 2294288) = 8192
pwrite(29, "\227@\2343\253[\340\322\202s5U\r\220,\215\334\\.\273\21\263\233J\305r\227\277\241Y\4S"..., 8192, 2302480) = 8192
pwrite(29, "S\vv/\320\211\35\177'\252?.\374\355q\336G\266\n\323\362\216\344\241\231\203\26f\265\6\23!"..., 8192, 2310672) = 8192
pwrite(29, "\364\356\fR\4s\26\233\231\21A\223\354\33B\211\5iF\265\261a\r\314g\343&\373\27\341fi"..., 8192, 2318864) = 8192
pwrite(29, "a\214t\342\334\f\32Q\316\207\374\333\335\340\202\272\226\244pK\252\27\200\324$b\315\2512]\364m"..., 8192, 2327056) = 8192
pwrite(29, "\31\240\0304\257\244|\373gv\7Aq\321\"\237Q\262\tI\2~\365\332\0o=]$\357\276\21"..., 8192, 2335248) = 8192
pwrite(29, "Wc\212\374al\202{\34\0238<\365~\343\36wj7\365\363\6\234\363\222\6\26\177Xt\20l"..., 8192, 2343440) = 8192
pwrite(29, "\254-\353\245,\2Qe\375\0357\205\343<}\312iQ'\213\n\276^\341I\314\34rz\230\377\336"..., 8192, 2351632) = 8192
pwrite(29, "o}{\226\204\302nee i\377+\213!\25\305\306\315/\r\2168\201\276\262-\247A\272\370\245"..., 264, 2359824) = 264

FileChannelImpl: private static final int TRANSFER_SIZE = 8192;

Jun 25 '18 17:06 zzmao

transferTo is good. It's doing zero-copy as expected. sendfile is observed when GET happen.

Jun 25 '18 18:06 zzmao