sendfile to nonblocking socket

[voron <voron@amhost.net>]

Hello

I'm testing a web server nginx for films sharing in my LAN. And I've got 
some interesting results. When I tried to download film or another big 
file via gigabit link, I've got sendfile block with nonblocking 
socket.   Strace log in attach. Some commens

#enabling nonblock on fd 3
20:51:20 ioctl(3, FIONBIO, [1])         = 0
#normal nonblocking sendfile, asked 2147480274 bytes, sent  236164 
bytes, so nonblocking works
20:51:20 sendfile(3, 8, [847150], 2147480274) = 236164
#sendfile 390 M, 6 seconds
20:51:22 sendfile(3, 8, [102578266], 2147481510) = 390115144
#sendfile 1000 M, 15 seconds
20:51:40 sendfile(3, 8, [1303409692], 2147482596) = 1008100764
#sendfile ~2G, 30 seconds
20:51:55 sendfile(3, 8, [2312288408], 1982678888) = 1982678888

As I see, nonblocking mode is enabled - sendfile sends less than asked.  
But 2G via single 30 seconds sendfile call - this is blocking call. How 
can I  avoid that?  I prefer sendfile as fastest way to send file 
content to network socket. The problem with sendfile block on 
nonblocking socket has place only when I'm using network connection, 
that is faster than my hard disk, for example  gigabit NIC or localhost. 
When I use 100Mbit NIC, which is slower, than my hard disk,  I got small 
and fast sendfile calls without blocking.

My kernel is  Linux 2.6.20.3-grsec x86_64. I verified that also on 
2.6.18 - same results. Please advise.

ps: I've did same tests with lighttpd's sendfile and got same results - 
block on nonblocking socket, when network is faster than disk.

Thank you,
Alex

strace -tp 10305
Process 10305 attached - interrupt to quit
20:51:17 write(13, "2007/04/23 20:51:17 [info] 10305"..., 85) = 85
20:51:17 epoll_wait(7, {{EPOLLIN, {u32=1054142480, u64=50471214837776}}}, 512, 4294967295) = 1
20:51:20 accept(10, {sa_family=AF_INET, sin_port=htons(55446), sin_addr=inet_addr("192.168.78.1")}, [5056729052170158096]) = 3
20:51:20 ioctl(3, FIONBIO, [1])         = 0
20:51:20 epoll_ctl(7, EPOLL_CTL_ADD, 3, {EPOLLIN|EPOLLET, {u32=1054142784, u64=50471214838080}}) = 0
20:51:20 epoll_wait(7, {{EPOLLIN, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:20 recvfrom(3, "GET /3.tmp HTTP/1.0\r\nUser-Agent:"..., 1024, 0, NULL, NULL) = 147
20:51:20 open("/var/www/cacti/htdocs/3.tmp", O_RDONLY) = 8
20:51:20 fstat(8, {st_mode=S_IFREG|0644, st_size=4294967296, ...}) = 0
20:51:20 setsockopt(3, SOL_TCP, TCP_CORK, [1], 4) = 0
20:51:20 writev(3, [{"HTTP/1.1 200 OK\r\nServer: nginx/0"..., 262}], 1) = 262
20:51:20 sendfile(3, 8, [0], 2147479552) = 20576
20:51:20 epoll_ctl(7, EPOLL_CTL_MOD, 3, {EPOLLIN|EPOLLOUT|EPOLLET, {u32=1054142784, u64=50471214838080}}) = 0
20:51:20 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:20 sendfile(3, 8, [20576], 2147483552) = 55568
20:51:20 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599981) = 1
20:51:20 sendfile(3, 8, [76144], 2147481232) = 27784
20:51:20 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599979) = 1
20:51:20 sendfile(3, 8, [103928], 2147482120) = 90298
20:51:20 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599979) = 1
20:51:20 sendfile(3, 8, [194226], 2147481934) = 652924
20:51:20 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599978) = 1
20:51:20 sendfile(3, 8, [847150], 2147480274) = 236164
20:51:20 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599970) = 1
20:51:21 sendfile(3, 8, [1083314], 2147481678) = 4549630
20:51:21 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599960) = 1
20:51:21 sendfile(3, 8, [5632944], 2147482704) = 38529462
20:51:21 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599894) = 1
20:51:21 sendfile(3, 8, [44162406], 2147480218) = 33451936
20:51:22 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:22 sendfile(3, 8, [77614342], 2147480314) = 20393456
20:51:22 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:22 sendfile(3, 8, [98007798], 2147480842) = 583464
20:51:22 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:22 sendfile(3, 8, [98591262], 2147483106) = 687654
20:51:22 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599959) = 1
20:51:22 sendfile(3, 8, [99278916], 2147483580) = 798790
20:51:22 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599945) = 1
20:51:22 sendfile(3, 8, [100077706], 2147483510) = 2500560
20:51:22 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599931) = 1
20:51:22 sendfile(3, 8, [102578266], 2147481510) = 390115144
20:51:28 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599910) = 1
20:51:28 sendfile(3, 8, [492693410], 2147481694) = 1007170
20:51:28 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:28 sendfile(3, 8, [493700580], 2147482140) = 1028008
20:51:28 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599984) = 1
20:51:28 sendfile(3, 8, [494728588], 2147482228) = 804430152
20:51:40 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599971) = 1
20:51:40 sendfile(3, 8, [1299158740], 2147481900) = 1069684
20:51:40 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:40 sendfile(3, 8, [1300228424], 2147481272) = 1062738
20:51:40 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599976) = 1
20:51:40 sendfile(3, 8, [1301291162], 2147483494) = 1007170
20:51:40 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599954) = 1
20:51:40 sendfile(3, 8, [1302298332], 2147479844) = 1111360
20:51:40 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599931) = 1
20:51:40 sendfile(3, 8, [1303409692], 2147482596) = 1008100764
20:51:55 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 599905) = 1
20:51:55 sendfile(3, 8, [2311510456], 1983456840) = 777952
20:51:55 epoll_wait(7, {{EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 600000) = 1
20:51:55 sendfile(3, 8, [2312288408], 1982678888) = 1982678888
20:52:25 write(16, "192.168.78.1 - voron [23/Apr/200"..., 114) = 114
20:52:25 close(8)                       = 0
20:52:25 setsockopt(3, SOL_TCP, TCP_CORK, [0], 4) = 0
20:52:25 recvfrom(3, 0x57d880, 1024, 0, 0, 0) = -1 EAGAIN (Resource temporarily unavailable)
20:52:25 epoll_wait(7, {{EPOLLIN|EPOLLOUT, {u32=1054142784, u64=50471214838080}}}, 512, 75000) = 1
20:52:25 recvfrom(3, "", 1024, 0, NULL, NULL) = 0
20:52:25 close(3)