nchan Worker crash

This time we had a crash loop: worker crashes, lots of clients try to reconnect –> worker crashes again.

The env details are same as in #498

These are the stack traces that I've got: 1.

(gdb) where
#0  chanhead_messages_gc_custom (ch=ch@entry=0xa789bd0, max_messages=7733018412696365634) at /home/deploy/nginx/nchan/src/store/memory/memstore.c:1939
#1  0x00000000005190ec in memstore_chanhead_messages_gc (ch=0xa789bd0) at /home/deploy/nginx/nchan/src/store/memory/memstore.c:1952
#2  nchan_memstore_chanhead_ready_to_reap (ch=0xa789bd0, force=<optimized out>) at /home/deploy/nginx/nchan/src/store/memory/memstore.c:186
#3  0x0000000000500034 in its_reaping_time (rp=rp@entry=0x7d47f8 <mdata+408>, force=force@entry=0 '\000') at /home/deploy/nginx/nchan/src/util/nchan_reaper.c:209
#4  0x0000000000500502 in reaper_timer_handler (ev=<optimized out>) at /home/deploy/nginx/nchan/src/util/nchan_reaper.c:308
#5  0x00000000004418c3 in ngx_event_expire_timers () at src/event/ngx_event_timer.c:94
#6  0x000000000044151d in ngx_process_events_and_timers (cycle=cycle@entry=0x175deb0) at src/event/ngx_event.c:256
#7  0x0000000000449af5 in ngx_worker_process_cycle (cycle=cycle@entry=0x175deb0, data=data@entry=0x0) at src/os/unix/ngx_process_cycle.c:750
#8  0x000000000044811c in ngx_spawn_process (cycle=cycle@entry=0x175deb0, proc=proc@entry=0x449ab0 <ngx_worker_process_cycle>, data=data@entry=0x0, 
    name=name@entry=0x53b9f3 "worker process", respawn=respawn@entry=-3) at src/os/unix/ngx_process.c:199
#9  0x00000000004492b4 in ngx_start_worker_processes (cycle=cycle@entry=0x175deb0, n=1, type=type@entry=-3) at src/os/unix/ngx_process_cycle.c:359
#10 0x000000000044a73f in ngx_master_process_cycle (cycle=cycle@entry=0x175deb0) at src/os/unix/ngx_process_cycle.c:131
#11 0x0000000000420372 in main (argc=<optimized out>, argv=<optimized out>) at src/core/nginx.c:382
(gdb) info locals
cur = 0x4652796c57647873
next = <optimized out>
now = 1547060469
started_count = 7512898459533145189
tried_count = 0
deleted_count = <optimized out>
(gdb) list
1934	    deleted_count++;        
1935	    cur = next;
1936	  }
1937	  
1938	  //any expired messages?
1939	  while(cur != NULL && now > cur->msg->expires) {
1940	    tried_count++;
1941	    next = cur->next;
1942	    chanhead_delete_message(ch, cur);
1943	    cur = next;

(gdb) where
#0  0x00007f5a3fecbc37 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
#1  0x00007f5a3fecf028 in __GI_abort () at abort.c:89
#2  0x00007f5a3fec4bf6 in __assert_fail_base (fmt=0x7f5a40019058 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x55c816 "ch->msg_first != ((void *)0)", 
    file=file@entry=0x55ae00 "/home/deploy/nginx/nchan/src/store/memory/memstore.c", line=line@entry=171, 
    function=function@entry=0x55cbc0 <__PRETTY_FUNCTION__.28319> "memstore_chanhead_reserved_or_in_use") at assert.c:92
#3  0x00007f5a3fec4ca2 in __GI___assert_fail (assertion=assertion@entry=0x55c816 "ch->msg_first != ((void *)0)", 
    file=file@entry=0x55ae00 "/home/deploy/nginx/nchan/src/store/memory/memstore.c", line=line@entry=171, 
    function=function@entry=0x55cbc0 <__PRETTY_FUNCTION__.28319> "memstore_chanhead_reserved_or_in_use") at assert.c:101
#4  0x000000000051859b in memstore_chanhead_reserved_or_in_use (ch=0x2571240) at /home/deploy/nginx/nchan/src/store/memory/memstore.c:171
#5  0x00000000005191c8 in nchan_memstore_chanhead_ready_to_reap (ch=0x2571240, force=<optimized out>) at /home/deploy/nginx/nchan/src/store/memory/memstore.c:203
#6  0x0000000000500034 in its_reaping_time (rp=rp@entry=0x7d47f8 <mdata+408>, force=force@entry=0 '\000') at /home/deploy/nginx/nchan/src/util/nchan_reaper.c:209
#7  0x0000000000500502 in reaper_timer_handler (ev=<optimized out>) at /home/deploy/nginx/nchan/src/util/nchan_reaper.c:308
#8  0x00000000004418c3 in ngx_event_expire_timers () at src/event/ngx_event_timer.c:94
#9  0x000000000044151d in ngx_process_events_and_timers (cycle=cycle@entry=0x22aeeb0) at src/event/ngx_event.c:256
#10 0x0000000000449af5 in ngx_worker_process_cycle (cycle=cycle@entry=0x22aeeb0, data=data@entry=0x0) at src/os/unix/ngx_process_cycle.c:750
#11 0x000000000044811c in ngx_spawn_process (cycle=cycle@entry=0x22aeeb0, proc=0x449ab0 <ngx_worker_process_cycle>, data=0x0, name=0x53b9f3 "worker process", respawn=respawn@entry=0)
    at src/os/unix/ngx_process.c:199
#12 0x000000000044ac76 in ngx_reap_children (cycle=0x22aeeb0) at src/os/unix/ngx_process_cycle.c:622
#13 ngx_master_process_cycle (cycle=cycle@entry=0x22aeeb0) at src/os/unix/ngx_process_cycle.c:175
#14 0x0000000000420372 in main (argc=<optimized out>, argv=<optimized out>) at src/core/nginx.c:382
(gdb) info locals
resultvar = 0
pid = 14700
selftid = 14700
(gdb) list
51	in ../nptl/sysdeps/unix/sysv/linux/raise.c

(gdb) where
#0  chanhead_messages_gc_custom (ch=ch@entry=0x242b3d0, max_messages=58842064) at /home/deploy/nginx/nchan/src/store/memory/memstore.c:1932
#1  0x00000000005190ec in memstore_chanhead_messages_gc (ch=0x242b3d0) at /home/deploy/nginx/nchan/src/store/memory/memstore.c:1952
#2  nchan_memstore_chanhead_ready_to_reap (ch=0x242b3d0, force=<optimized out>) at /home/deploy/nginx/nchan/src/store/memory/memstore.c:186
#3  0x0000000000500034 in its_reaping_time (rp=rp@entry=0x7d47f8 <mdata+408>, force=force@entry=0 '\000') at /home/deploy/nginx/nchan/src/util/nchan_reaper.c:209
#4  0x0000000000500502 in reaper_timer_handler (ev=<optimized out>) at /home/deploy/nginx/nchan/src/util/nchan_reaper.c:308
#5  0x00000000004418c3 in ngx_event_expire_timers () at src/event/ngx_event_timer.c:94
#6  0x000000000044151d in ngx_process_events_and_timers (cycle=cycle@entry=0x22aeeb0) at src/event/ngx_event.c:256
#7  0x0000000000449af5 in ngx_worker_process_cycle (cycle=cycle@entry=0x22aeeb0, data=data@entry=0x0) at src/os/unix/ngx_process_cycle.c:750
#8  0x000000000044811c in ngx_spawn_process (cycle=cycle@entry=0x22aeeb0, proc=0x449ab0 <ngx_worker_process_cycle>, data=0x0, name=0x53b9f3 "worker process", respawn=respawn@entry=0)
    at src/os/unix/ngx_process.c:199
#9  0x000000000044ac76 in ngx_reap_children (cycle=0x22aeeb0) at src/os/unix/ngx_process_cycle.c:622
#10 ngx_master_process_cycle (cycle=cycle@entry=0x22aeeb0) at src/os/unix/ngx_process_cycle.c:175
#11 0x0000000000420372 in main (argc=<optimized out>, argv=<optimized out>) at src/core/nginx.c:382
(gdb) info locals
cur = 0x400000004
next = <optimized out>
now = 1547063887
started_count = 8996295839417663756
tried_count = 0
deleted_count = 0
(gdb) list
1927	  deleted_count = 0;
1928	  
1929	  //is the message queue too big?
1930	  while(cur != NULL && max_messages >= 0 && ch->channel.messages > max_messages) {
1931	    tried_count++;
1932	    next = cur->next;
1933	    chanhead_delete_message(ch, cur);
1934	    deleted_count++;        
1935	    cur = next;
1936	  }
(gdb)

I don't think that 300 subscribers is something that could bring nchan/nginx into crash loop, but that I witnessed just 30 minutes ago ¯_(ツ)_/¯

Jan 09 '19 20:01 ivanovv

Is the first stack trace the initial crash? Anything suspicious in the error logs?

Jan 09 '19 22:01 slact

Also, please add your publisher location config as well.

Jan 09 '19 22:01 slact

Is the first stack trace the initial crash?

Nope, there were around 30 ctashes, these are last 3 of them.

Anything suspicious in the error logs

Not much, I'll post pub endpoint and logs later.

Jan 09 '19 22:01 ivanovv

Can you post the backtrace of the first process that crashed? You should be able to tell by the coredump file creation timestamp.

Jan 09 '19 22:01 slact

Can you post the backtrace of the first process that crashed? You should be able to tell by the coredump file creation timestamp.

I wish I could, but

$> cat /proc/sys/kernel/core_pattern
|/usr/share/apport/apport %p %s %c %P
$> cat /proc/sys/kernel/core_uses_pid
0

and /var/crash has only the last crash dump, named /var/crash/_usr_sbin_nginx.33.crash :(

I'm not sure what exact setting is responsible and how to make it generate crash dumps with PID in the filename.

Jan 10 '19 11:01 ivanovv

Also, please add your publisher location config as well.

Here you go:

server {
  listen   10.xxx.xxx.90:80;
  server_name 10.xxx.xxx.90;

  location ~ /pub/(.*)$ {

    nchan_publisher;
    nchan_channel_id $1;
    nchan_message_buffer_length 0;
    nchan_message_timeout 30s;
  }

  location /nchan_stub_status {
    nchan_stub_status;
  }

  location / {
    root   /var/www/nginx-default;
    index  index.html index.htm;
  }
}

Jan 10 '19 11:01 ivanovv

set /proc/sys/kernel/core_pattern to "core.%h.%e.%t.%s.%p"

echo "core.%h.%e.%t.%s.%p" >  /proc/sys/kernel/core_pattern

set working_directory to somewhere nginx can write to, restart it and test this by sending a worker process a SIGABRT. Make sure this generates a readable core.

Jan 10 '19 20:01 slact

Sure, can do.

Meanwhile, do the stack traces give you any clues what was going on with nginx/nchan pair?

Jan 10 '19 20:01 ivanovv

I also recommend you set the error_log directive at the root level of your nginx config -- that's where you set worker_processes, working_directory, and events. set it to level ``notice`:

error_log /path/to/error.log notice;

Jan 10 '19 20:01 slact

Meanwhile, do the stack traces give you any clues what was going on with nginx/nchan pair?

It's probably got something to do with the publisher configuration

    nchan_message_buffer_length 0;
    nchan_message_timeout 30s;

the 0-length buffer negates the 30sec timeout, which i suspect is triggering some weirdness. Doubly suspicious because the crash happens in message queue trimming code. Don't change this setting though, this is a valid configuration and it should not cause a crash.

Jan 10 '19 20:01 slact

Here are the logs:

2019/01/09 11:39:28 [error] 20349#20349: *9193775 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.198.122.51, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 11:40:12 [error] 20349#20349: *9194043 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.141, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 11:45:25 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 00000000098E7F50 m//webinar~qlC5dTx~private~t838k/webinar~qlC5dTx to chanhead_gc. why?
2019/01/09 11:45:25 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000008CCE030 m//webinar~qlC5dTx~private~rktUh/webinar~qlC5dTx to chanhead_gc. why?
2019/01/09 11:45:25 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 00000000081905D0 m//webinar~u9jJy9C~private~IkjXi/webinar~u9jJy9C to chanhead_gc. why?
2019/01/09 11:45:25 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000008836FB0 m//webinar~qlC5dTx~private~iUQvw/webinar~qlC5dTx to chanhead_gc. why?
2019/01/09 11:45:25 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 000000000753BB70 m//webinar~qlC5dTx~private~zep4D/webinar~qlC5dTx to chanhead_gc. why?
2019/01/09 11:45:25 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000008E4DAD0 m//webinar~qlC5dTx~private~cV1Zu/webinar~qlC5dTx to chanhead_gc. why?
2019/01/09 11:45:53 [error] 20349#20349: *9195906 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.100.140.190, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
27 more errors that basically the same `2: No such file or directory`
2019/01/09 11:59:05 [error] 20349#20349: *9200828 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.145, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 12:00:35 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000008529B90 m//webinar~VunDadP~private~etWld/webinar~VunDadP to chanhead_gc. why?
2019/01/09 12:00:35 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 00000000085A7720 m//webinar~VunDadP~private~MaF7P/webinar~VunDadP to chanhead_gc. why?
2019/01/09 12:00:35 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000006D622C0 m//webinar~VunDadP~private~uTJq_/webinar~VunDadP to chanhead_gc. why?
2019/01/09 12:00:35 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 00000000088345E0 m//webinar~VunDadP~private~n25bL/webinar~VunDadP to chanhead_gc. why?
2019/01/09 12:00:35 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 00000000058A2200 m//webinar~VunDadP~private~sQpNf/webinar~VunDadP to chanhead_gc. why?
2019/01/09 12:00:35 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000004DB83F0 m//webinar~VunDadP~private~Dl0-xRns12U/webinar~VunDadP to chanhead_gc. why?
2019/01/09 12:00:55 [error] 20349#20349: *9201513 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.145, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
12 more errors that basically the same `2: No such file or directory`
2019/01/09 12:15:24 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 00000000075F26F0 m//webinar~Huri65i~private~LaStC/webinar~Huri65i to chanhead_gc. why?
2019/01/09 12:15:24 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000007286C00 m//webinar~Huri65i~private~CUvPz/webinar~Huri65i to chanhead_gc. why?
2019/01/09 12:15:24 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000008575930 m//webinar~Huri65i~private~3GMdQ/webinar~Huri65i to chanhead_gc. why?
2019/01/09 12:15:24 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000004D80F90 m//webinar~Huri65i~private~7_aF9/webinar~Huri65i to chanhead_gc. why?
2019/01/09 12:15:24 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000007556030 m//webinar~Huri65i~private~qJf7l/webinar~Huri65i to chanhead_gc. why?
2019/01/09 12:15:24 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 000000000727EE60 m//webinar~Huri65i~private~eGNKv/webinar~Huri65i to chanhead_gc. why?
2019/01/09 12:15:35 [error] 20349#20349: *9206324 open() "/u/apps/web_production/current/public/uploads/image/898739/basics_logo.png" failed (2: No such file or directory), client: xx.184.229.207, server: *.web.com, request: "GET //uploads/image/898739/basics_logo.png HTTP/1.1", host: "app.web.com"
40 more errors that basically the same `2: No such file or directory`
2019/01/09 13:15:25 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 000000000A532FD0 m//webinar~zFsygBd~private~dQBa2/webinar~zFsygBd to chanhead_gc. why?
2019/01/09 13:15:25 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000008538600 m//webinar~zFsygBd~private~CQHm9/webinar~zFsygBd to chanhead_gc. why?
2019/01/09 13:15:25 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000008C838D0 m//webinar~zFsygBd~private~rjjj5/webinar~zFsygBd to chanhead_gc. why?
2019/01/09 13:15:26 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000002D848F0 m//webinar~zFsygBd~private~mi-YVtz8k4M/webinar~zFsygBd to chanhead_gc. why?
2019/01/09 13:15:26 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000006C0B770 m//webinar~zFsygBd~private~jSFLv/webinar~zFsygBd to chanhead_gc. why?
2019/01/09 13:15:26 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 000000000784B840 m//webinar~zFsygBd~private~FK8_Q/webinar~zFsygBd to chanhead_gc. why?
2019/01/09 13:19:09 [error] 20349#20349: *9221291 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.72.230.5, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
20 more errors that basically the same `2: No such file or directory`
2019/01/09 13:42:55 [error] 20349#20349: *9226004 open() "/u/apps/web_production/current/public/uploads/image/1147556/WhatsApp_Image_2018-10-11_at_13.13.29.jpeg" failed (2: No such file or directory), client: xx.220.149.1, server: *.web.com, request: "GET /uploads/image/1147556/WhatsApp_Image_2018-10-11_at_13.13.29.jpeg HTTP/1.1", host: "app.web.com"
2019/01/09 13:45:24 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 00000000089BF170 m//webinar~5mo0v2I~private~Fo8G2/webinar~5mo0v2I to chanhead_gc. why?
2019/01/09 13:45:24 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000009DA3640 m//webinar~5mo0v2I~private~8sChN/webinar~5mo0v2I to chanhead_gc. why?
2019/01/09 13:45:24 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 000000000972C3F0 m//webinar~5mo0v2I~private~dZiYI/webinar~5mo0v2I to chanhead_gc. why?
2019/01/09 13:45:24 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000008F97E20 m//webinar~5mo0v2I~private~olOpN/webinar~5mo0v2I to chanhead_gc. why?
2019/01/09 13:45:40 [error] 20349#20349: *9226559 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.141, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
45 more errors that basically the same `2: No such file or directory`
2019/01/09 14:30:28 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 000000000948AAA0 m//webinar~rwwhHeW~private~zdseE/webinar~rwwhHeW to chanhead_gc. why?
2019/01/09 14:30:28 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000007C7F0D0 m//webinar~h9RPIZ8~private~5r7ln/webinar~h9RPIZ8 to chanhead_gc. why?
2019/01/09 14:30:48 [error] 20349#20349: *9238861 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.143, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
125 more errors that basically the same `2: No such file or directory`
2019/01/09 16:00:06 [error] 20349#20349: *9260874 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.87.48.132, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com", referrer: "https://outlook.live.com/"
2019/01/09 16:00:30 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000007BCC060 m//webinar~9yi63ag~private~-Au4wEyRbkE/webinar~9yi63ag to chanhead_gc. why?
2019/01/09 16:00:30 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000004D86A90 m//webinar~9yi63ag~private~PYAMg/webinar~9yi63ag to chanhead_gc. why?
2019/01/09 16:00:30 [error] 20349#20349: MEMSTORE:00: tried adding WAITING chanhead 0000000008F05560 m//webinar~9yi63ag~private~_G3mF/webinar~9yi63ag to chanhead_gc. why?
2019/01/09 16:01:42 [error] 20349#20349: *9261180 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.143, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
35 more errors that basically the same `2: No such file or directory`
2019/01/09 16:53:48 [error] 20349#20349: *9270888 upstream prematurely closed connection while reading response header from upstream, client: xx.73.109.3, server: *.web.com, request: "GET /reports/csv_export/17380 HTTP/1.1", upstream: "http://unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock:/reports/csv_export/1780", host: "app.web.com"
2019/01/09 16:54:23 [error] 20349#20349: *9271054 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.198.94.176, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
103 more errors that basically the same `2: No such file or directory`
2019/01/09 19:00:47 [error] 20349#20349: *9308854 open() "/u/apps/web_production/current/public/uploads/image/54785/20180719-1C2A4015__Medium_.jpg" failed (2: No such file or directory), client: xx.220.149.21, server: *.web.com, request: "GET /uploads/image/54785/20180719-1C2A4015__Medium_.jpg HTTP/1.1", host: "app.web.com"
2019/01/09 19:02:12 [alert] 20348#20348: worker process 20349 exited on signal 11 (core dumped)
2019/01/09 19:02:24 [alert] 20348#20348: worker process 27901 exited on signal 11 (core dumped)
2019/01/09 19:03:56 [alert] 20348#20348: worker process 27939 exited on signal 11 (core dumped)
2019/01/09 19:05:37 [error] 28228#28228: *9317221 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.75.59.250, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:05:58 [error] 28228#28228: *9317642 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.205.133.212, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:08:09 [error] 28228#28228: *9320280 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.163.191.56, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:08:23 [error] 28228#28228: *9320502 open() "/u/apps/web_production/current/public/uploads/image/1093472/logo-foto.png" failed (2: No such file or directory), client: xx.249.93.13, server: *.web.com, request: "GET /uploads/image/1093472/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:09:02 [error] 28228#28228: *9321298 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.145, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:10:23 [error] 28228#28228: *9322855 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.87.0.214, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:12:47 [error] 28228#28228: *9325212 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.233.219.104, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:14:07 [error] 28228#28228: *9326649 open() "/u/apps/web_production/current/public/uploads/image/1273208/FL2__1_of_1_.jpg" failed (2: No such file or directory), client: xx.220.149.26, server: *.web.com, request: "GET /uploads/image/1273208/FL2__1_of_1_.jpg HTTP/1.1", host: "app.web.com"
ker process: /home/deploy/nginx/nchan/src/util/nchan_reaper.c:181: reap_ready_thing: Assertion `cur != prev' failed.
2019/01/09 19:14:13 [alert] 20348#20348: worker process 28228 exited on signal 6 (core dumped)
2019/01/09 19:14:25 [alert] 20348#20348: worker process 31493 exited on signal 11 (core dumped)
2019/01/09 19:14:54 [error] 31538#31538: *9329783 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.233.219.104, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:15:55 [error] 31538#31538: *9331165 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.233.219.104, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:16:11 [error] 31538#31538: *9331540 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.60.252.72, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:16:21 [alert] 20348#20348: worker process 31538 exited on signal 11 (core dumped)
2019/01/09 19:16:37 [error] 32543#32543: *9333146 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.144.69.59, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:16:53 [error] 32543#32543: *9333570 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.145, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:17:18 [error] 32543#32543: *9334017 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.206.102.175, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:22:48 [error] 32543#32543: *9340360 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.143, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:24:19 [error] 32543#32543: *9342020 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.119.17.227, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
ker process: /home/deploy/nginx/nchan/src/store/memory/memstore.c:171: memstore_chanhead_reserved_or_in_use: Assertion `ch->msg_first != ((void *)0)' failed.
2019/01/09 19:24:34 [alert] 20348#20348: worker process 32543 exited on signal 6 (core dumped)
2019/01/09 19:24:47 [alert] 20348#20348: worker process 2613 exited on signal 11 (core dumped)
2019/01/09 19:25:00 [alert] 20348#20348: worker process 2628 exited on signal 11 (core dumped)
ker process: /home/deploy/nginx/nchan/src/store/memory/memstore.c:171: memstore_chanhead_reserved_or_in_use: Assertion `ch->msg_first != ((void *)0)' failed.
2019/01/09 19:25:12 [alert] 20348#20348: worker process 2642 exited on signal 6 (core dumped)
2019/01/09 19:25:55 [error] 2900#2900: *9348582 open() "/u/apps/web_production/current/public/uploads/image/1402207/088A6805.jpg" failed (2: No such file or directory), client: xx.252.95.16, server: *.web.com, request: "GET /uploads/image/1402207/088A6805.jpg HTTP/1.1", host: "app.web.com"
32 more errors that basically the same `2: No such file or directory`
2019/01/09 19:47:45 [alert] 20348#20348: worker process 2900 exited on signal 9
another log file started (log rotation occured)
2019/01/09 19:47:42 [notice] 14070#14070: signal process started
2019/01/09 19:47:46 [info] 14081#14081: Using 131072KiB of shared memory for nchan in /etc/nginx/nginx.conf:46
2019/01/09 19:47:47 [error] 14084#14084: *444 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.143, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:47:52 [error] 14084#14084: *1585 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.241.207.171, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
*** Error in `nginx: worker process': munmap_chunk(): invalid pointer: 0x000000000245efe8 ***
2019/01/09 19:48:36 [alert] 14083#14083: worker process 14084 exited on signal 6 (core dumped)
2019/01/09 19:48:48 [alert] 14083#14083: worker process 14475 exited on signal 11 (core dumped)
2019/01/09 19:49:00 [alert] 14083#14083: worker process 14487 exited on signal 11 (core dumped)
2019/01/09 19:49:41 [alert] 14083#14083: worker process 14507 exited on signal 11 (core dumped)
2019/01/09 19:52:27 [error] 14700#14700: *14739 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.80.61.40, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
ker process: /home/deploy/nginx/nchan/src/store/memory/memstore.c:171: memstore_chanhead_reserved_or_in_use: Assertion `ch->msg_first != ((void *)0)' failed.
2019/01/09 19:53:48 [alert] 14083#14083: worker process 14700 exited on signal 6 (core dumped)
2019/01/09 19:53:57 [error] 16112#16112: *18093 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.248.40.69, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:53:58 [error] 16112#16112: *18124 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.141, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:54:00 [alert] 14083#14083: worker process 16112 exited on signal 11 (core dumped)
2019/01/09 19:55:08 [error] 16167#16167: *21152 open() "/u/apps/web_production/current/public/uploads/image/1093658/logo-foto.png" failed (2: No such file or directory), client: xx.109.106.246, server: *.web.com, request: "GET /uploads/image/1093658/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:55:38 [error] 16167#16167: *21966 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.130.35.183, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 19:57:24 [alert] 14083#14083: worker process 16167 exited on signal 11 (core dumped)
2019/01/09 19:57:36 [alert] 14083#14083: worker process 28763 exited on signal 11 (core dumped)
2019/01/09 19:57:54 [alert] 14083#14083: worker process 28809 exited on signal 11 (core dumped)
2019/01/09 19:58:13 [alert] 14083#14083: worker process 28849 exited on signal 11 (core dumped)
2019/01/09 19:58:25 [alert] 14083#14083: worker process 29059 exited on signal 11 (core dumped)
2019/01/09 20:01:34 [error] 29085#29085: *35248 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.143, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 20:03:49 [error] 29085#29085: *37838 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.145, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
ker process: /home/deploy/nginx/nchan/src/store/memory/memstore.c:171: memstore_chanhead_reserved_or_in_use: Assertion `ch->msg_first != ((void *)0)' failed.
2019/01/09 20:04:14 [alert] 14083#14083: worker process 29085 exited on signal 6 (core dumped)
2019/01/09 20:04:33 [info] 31250#31250: Using 131072KiB of shared memory for nchan in /etc/nginx/nginx.conf:46
2019/01/09 20:04:39 [notice] 31257#31257: signal process started
2019/01/09 20:04:39 [alert] 14083#14083: worker process 31195 exited on signal 11 (core dumped)
ker process: /home/deploy/nginx/nchan/src/util/nchan_reaper.c:182: reap_ready_thing: Assertion `cur != next' failed.
2019/01/09 20:04:56 [alert] 14083#14083: worker process 31258 exited on signal 6 (core dumped)
2019/01/09 20:05:04 [error] 31283#31283: *42363 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.81.5.229, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 20:05:08 [alert] 14083#14083: worker process 31283 exited on signal 11 (core dumped)
2019/01/09 20:05:21 [alert] 14083#14083: worker process 31532 exited on signal 11 (core dumped)
2019/01/09 20:05:23 [error] 31588#31588: *43776 connect() to unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: xx.145.201.43, server: *.web.com, request: "GET /service/ajax_log?...", host: "app.web.com", referrer: "https://app.web.com/webinar/admin/0WoBeChG6uYl9qCs/"
2019/01/09 20:05:23 [error] 31588#31588: *44301 connect() to unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: xx.30.232.65, server: *.web.com, request: "GET /service/ajax_log?...", host: "praktijk.web.com", referrer: "https://praktijk.web.com/watch/tle5CDLEf7A/"
2019/01/09 20:05:23 [error] 31588#31588: *44420 connect() to unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: xx.81.233.165, server: *.web.com, request: "GET /service/ajax_log?...", host: "suz.web.com", referrer: "https://suz.web.com/watch/VxhIl6I-kKg/"
2019/01/09 20:05:23 [error] 31588#31588: *44390 connect() to unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: xx.92.88.27, server: *.web.com, request: "GET /service/ajax_log?...", host: "rob-coaching.web.com", referrer: "https://rob-coaching.web.com/watch/LyGqo1HFAIY/"
2019/01/09 20:05:23 [error] 31588#31588: *43776 connect() to unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: xx.145.201.43, server: *.web.com, request: "POST /components/stop/605821 HTTP/1.1", upstream: "http://unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock:/components/stop/605821", host: "app.web.com", referrer: "https://app.web.com/webinar/admin/0WoBeChG6uYl9qCs/"
2019/01/09 20:05:23 [error] 31588#31588: *44457 connect() to unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: xx.57.145.89, server: *.web.com, request: "GET /service/ajax_log?...", host: "praktijk.web.com", referrer: "https://praktijk.web.com/watch/S6icEa6e0lI/"
2019/01/09 20:05:23 [error] 31588#31588: *44460 connect() to unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: xx.62.140.108, server: *.web.com, request: "GET /service/ajax_log?...", host: "praktijk.web.com", referrer: "https://praktijk.web.com/webinar/watch/HqIgBUoKoQg/"
2019/01/09 20:05:23 [error] 31588#31588: *44381 connect() to unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: xx.82.36.226, server: *.web.com, request: "GET /service/ajax_log?...", host: "rob-coaching.web.com", referrer: "https://rob-coaching.web.com/watch/l4PxyvJ2L2s/"
2019/01/09 20:05:23 [error] 31588#31588: *44558 connect() to unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: xx.130.71.219, server: *.web.com, request: "GET /service/ajax_log?...", host: "kotsu.web.com", referrer: "https://kotsu.web.com/watch/v9pByV--MHg/"
2019/01/09 20:05:30 [error] 31588#31588: *44860 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.64.205.160, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 20:05:33 [error] 31588#31588: *44955 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.210.251.165, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
*** Error in `nginx: worker process': corrupted size vs. prev_size: 0x0000000002504470 ***
2019/01/09 20:06:02 [alert] 14083#14083: worker process 31588 exited on signal 6 (core dumped)
2019/01/09 20:06:03 [error] 31874#31874: *45707 open() "/u/apps/web_production/current/public/uploads/image/524546/Friends_Enterprise_LOGO1.png" failed (2: No such file or directory), client: xx.162.59.203, server: *.web.com, request: "GET /uploads/image/524546/Friends_Enterprise_LOGO1.png HTTP/1.1", host: "app.web.com"
ker process: /home/deploy/nginx/nchan/src/store/memory/memstore.c:171: memstore_chanhead_reserved_or_in_use: Assertion `ch->msg_first != ((void *)0)' failed.
2019/01/09 20:06:51 [alert] 14083#14083: worker process 31874 exited on signal 6 (core dumped)
2019/01/09 20:08:06 [alert] 14083#14083: worker process 31978 exited on signal 11 (core dumped)
2019/01/09 20:08:08 [error] 32386#32386: *50844 connect() to unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: xx.146.181.170, server: *.web.com, request: "GET /service/ajax_log?", host: "app.web.com", referrer: "https://app.web.com/webinar/admin/9BDqe5rj/"
2019/01/09 20:08:08 [error] 32386#32386: *50852 connect() to unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: xx.119.17.221, server: *.web.com, request: "GET /service/ajax_log?...", host: "praktijk.web.com", referrer: "https://praktijk.web.com/watch/qeDcUxKY/"
2019/01/09 20:08:08 [error] 32386#32386: *50916 connect() to unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: xx.194.114.141, server: *.web.com, request: "GET /service/ajax_log?..", host: "praktijk.web.com", referrer: "https://praktijk.web.com/webinar/watch/J6sN0g/"
2019/01/09 20:08:19 [alert] 14083#14083: worker process 32386 exited on signal 11 (core dumped)
2019/01/09 20:08:31 [alert] 14083#14083: worker process 32423 exited on signal 11 (core dumped)
2019/01/09 20:09:02 [error] 32469#32469: *53989 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.177.113.200, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
12 more errors that basically the same `2: No such file or directory`
2019/01/09 20:19:59 [notice] 4299#4299: signal process started
2019/01/09 20:20:02 [alert] 14083#14083: worker process 32469 exited on signal 9
2019/01/09 20:20:03 [info] 4618#4618: Using 131072KiB of shared memory for nchan in /etc/nginx/nginx.conf:46
2019/01/09 20:22:50 [error] 4621#4621: *3362 open() "/u/apps/web_production/current/public/uploads/image/421911/Bijgesneden_v2.png" failed (2: No such file or directory), client: xx.220.149.21, server: *.web.com, request: "GET /uploads/image/421911/Bijsneden_v2.png HTTP/1.1", host: "app.web.com"
65 more errors that basically the same `2: No such file or directory`
2019/01/09 21:15:22 [error] 4621#4621: *32116 open() "/u/apps/web_production/current/public/uploads/image/637400/Grapefoodie_logo.png" failed (2: No such file or directory), client: xx.252.127.30, server: *.web.com, request: "GET /uploads/image/637400/Grapefoodie_logo.png HTTP/1.1", host: "app.web.com"
2019/01/09 21:15:24 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001F7EAD0 m//webinar~GmxRedE~private~stxDr/webinar~GmxRedE to chanhead_gc. why?
2019/01/09 21:15:24 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 000000000207A330 m//webinar~GmxRedE~private~BlRFD/webinar~GmxRedE to chanhead_gc. why?
2019/01/09 21:15:24 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001C01210 m//webinar~GmxRedE~private~LerJH/webinar~GmxRedE to chanhead_gc. why?
2019/01/09 21:15:24 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001BDAF20 m//webinar~GmxRedE~private~WFtGj/webinar~GmxRedE to chanhead_gc. why?
2019/01/09 21:15:24 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001DE18A0 m//webinar~GmxRedE~private~la97E/webinar~GmxRedE to chanhead_gc. why?
2019/01/09 21:15:24 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000002714290 m//webinar~GmxRedE~private~21ZWR/webinar~GmxRedE to chanhead_gc. why?
2019/01/09 21:15:24 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001E1C890 m//webinar~GmxRedE~private~sdNCz/webinar~GmxRedE to chanhead_gc. why?
2019/01/09 21:15:24 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000002F0F0D0 m//webinar~GmxRedE~private~cT6fE/webinar~GmxRedE to chanhead_gc. why?
2019/01/09 21:15:31 [error] 4621#4621: *32249 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.143, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
15 more errors that basically the same `2: No such file or directory`
2019/01/09 21:30:28 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000002799830 m//webinar~8sYKSAR~private~Fg60H/webinar~8sYKSAR to chanhead_gc. why?
2019/01/09 21:30:28 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001EE5110 m//webinar~5g1glSj~private~gkTK1/webinar~5g1glSj to chanhead_gc. why?
2019/01/09 21:30:28 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 000000000218E4F0 m//webinar~5g1glSj~private~Y6kQ5/webinar~5g1glSj to chanhead_gc. why?
2019/01/09 21:30:28 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001B238D0 m//webinar~5g1glSj~private~jllJx/webinar~5g1glSj to chanhead_gc. why?
2019/01/09 21:30:36 [error] 4621#4621: *37577 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.145, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 21:32:24 [error] 4621#4621: *38058 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.141, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 21:34:48 [error] 4621#4621: *38709 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.141, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 21:40:12 [error] 4621#4621: *39872 stat() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.141, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 21:45:16 [error] 4621#4621: *41164 open() "/u/apps/web_production/current/public/uploads/image/60353/Logos.PNG" failed (2: No such file or directory), client: xx.16.157.182, server: *.web.com, request: "GET //uploads/image/60353/Logos.PNG HTTP/1.1", host: "app.web.com"
2019/01/09 21:45:23 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000002AF7350 m//webinar~043A4A0~private~Ftf82/webinar~043A4A0 to chanhead_gc. why?
2019/01/09 21:45:23 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 00000000026E4FF0 m//webinar~043A4A0~private~or2sD/webinar~043A4A0 to chanhead_gc. why?
2019/01/09 21:45:23 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 00000000029D4C20 m//webinar~043A4A0zLJQ0SPtkuMDtUrL1b9g~private~-VugeyBuOlk/webinar~043A4A0zLJQ0SPtkuMDtUrL1b9g to chanhead_gc. why?
2019/01/09 21:45:23 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 00000000024F1430 m//webinar~043A4A0~private~LaYiY/webinar~043A4A0 to chanhead_gc. why?
2019/01/09 21:48:45 [error] 4621#4621: *41942 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.249.81.143, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
11 more errors that basically the same `2: No such file or directory`
2019/01/09 21:59:34 [error] 4621#4621: *44464 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.30.176.26, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/09 22:00:32 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000002020020 m//webinar~ogkWfkH~private~zyHGu/webinar~ogkWfkH to chanhead_gc. why?
2019/01/09 22:01:04 [error] 4621#4621: *44838 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.87.252.71, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
19 more errors that basically the same `2: No such file or directory`
2019/01/09 22:25:34 [error] 4621#4621: *49614 open() "/u/apps/web_production/current/public/uploads/image/349450/Logo_final_1.jpg" failed (2: No such file or directory), client: xx.252.127.20, server: *.web.com, request: "GET /uploads/image/349450/Logo_final_1.jpg HTTP/1.1", host: "app.web.com"
2019/01/09 22:30:22 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000002C296E0 m//webinar~P34JUAc~private~vvv2j/webinar~P34JUAc to chanhead_gc. why?
2019/01/09 22:30:22 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 00000000022A6340 m//webinar~P34JUAc~private~HfYbz/webinar~P34JUAc to chanhead_gc. why?
2019/01/09 22:30:22 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001E3AE60 m//webinar~P34JUAc~private~EpGVj/webinar~P34JUAc to chanhead_gc. why?
2019/01/09 22:30:22 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001CCA6A0 m//webinar~P34JUAc~private~agcL6/webinar~P34JUAc to chanhead_gc. why?
2019/01/09 22:30:22 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001C8B620 m//webinar~P34JUAc~private~6e4vU/webinar~P34JUAc to chanhead_gc. why?
2019/01/09 22:30:22 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001ABA8B0 m//webinar~P34JUAc~private~O3FVd/webinar~P34JUAc to chanhead_gc. why?
2019/01/09 22:31:01 [error] 4621#4621: *50566 open() "/u/apps/web_production/current/public/uploads/image/60353/Logos.PNG" failed (2: No such file or directory), client: xx.59.150.181, server: *.web.com, request: "GET //uploads/image/60353/Logos.PNG HTTP/1.1", host: "app.web.com"
303 more errors that basically the same `2: No such file or directory`
2019/01/10 10:14:36 [error] 4621#4621: *171152 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.241.195.217, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/10 10:15:22 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000002725F50 m//webinar~x8l4Ryg~private~TfSBw/webinar~x8l4Ryg to chanhead_gc. why?
2019/01/10 10:15:29 [error] 4621#4621: *171787 open() "/u/apps/web_production/current/public/assets/templates/web_default-eabb13f27de0ea9e5559c710a799a94c.css" failed (2: No such file or directory), client: xx.243.27.146, server: *.web.com, request: "GET /assets/templates/web_default-eabb13f27de0ea9e5559c710a799a94c.css HTTP/1.1", host: "qwesd.web.com", referrer: "https://qwesd.web.com/admin/editor/registration/23517/22082"
15 more errors that basically the same `2: No such file or directory`
2019/01/10 10:28:44 [error] 4621#4621: *183099 open() "/u/apps/web_production/current/public/uploads/image/1402207/088A6805.jpg" failed (2: No such file or directory), client: xx.252.127.9, server: *.web.com, request: "GET /uploads/image/1402207/088A6805.jpg HTTP/1.1", host: "app.web.com"
2019/01/10 10:30:25 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 00000000025D6860 m//webinar~pfnm1BW~private~StGnk/webinar~pfnm1BW to chanhead_gc. why?
2019/01/10 10:30:25 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 00000000024E6CB0 m//webinar~qJzw7zI~private~TKRhm/webinar~qJzw7zI to chanhead_gc. why?
2019/01/10 10:30:25 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001A61FA0 m//webinar~qJzw7zI~private~QJplJ/webinar~qJzw7zI to chanhead_gc. why?
2019/01/10 10:30:25 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000002817910 m//webinar~pfnm1BW~private~jks0l/webinar~pfnm1BW to chanhead_gc. why?
2019/01/10 10:30:26 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 00000000027B9770 m//webinar~pfnm1BW~private~hykwC/webinar~pfnm1BW to chanhead_gc. why?
2019/01/10 10:30:26 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 00000000021DE8E0 m//webinar~qJzw7zI~private~ENsqR/webinar~qJzw7zI to chanhead_gc. why?
2019/01/10 10:31:27 [error] 4621#4621: *185498 stat() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.85.175.160, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
11 more errors that basically the same `2: No such file or directory`
2019/01/10 10:43:48 [error] 4621#4621: *194486 open() "/u/apps/web_production/current/public/uploads/image/181871/__Small_.jpg" failed (2: No such file or directory), client: xx.220.149.12, server: *.web.com, request: "GET /uploads/image/181871/__Small_.jpg HTTP/1.1", host: "app.web.com"
2019/01/10 10:45:22 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001E371D0 m//webinar~eCAkc4M~private~yw6ik/webinar~eCAkc4M to chanhead_gc. why?
2019/01/10 10:45:22 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 000000000198DAD0 m//webinar~0BOz0Ya~private~lq5a8/webinar~0BOz0Ya to chanhead_gc. why?
2019/01/10 10:45:22 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 00000000025D3C70 m//webinar~eCAkc4M~private~pF_yv/webinar~eCAkc4M to chanhead_gc. why?
2019/01/10 10:45:22 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 00000000024DC410 m//webinar~0BOz0Ya~private~HKV0Z/webinar~0BOz0Ya to chanhead_gc. why?
2019/01/10 10:46:10 [error] 4621#4621: *196088 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.106.163.181, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
11 more errors that basically the same `2: No such file or directory`
2019/01/10 10:58:16 [error] 4621#4621: *204179 open() "/u/apps/web_production/current/public/uploads/image/808447/FullSizeRender.jpg" failed (2: No such file or directory), client: xx.252.95.4, server: *.web.com, request: "GET /uploads/image/808447/FullSizeRender.jpg HTTP/1.1", host: "app.web.com"
2019/01/10 11:00:27 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 00000000020CC440 m//webinar~eSNCFTq~private~J7gN_/webinar~eSNCFTq to chanhead_gc. why?
2019/01/10 11:00:27 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000002C17380 m//webinar~eSNCFTq~private~OEE4U/webinar~eSNCFTq to chanhead_gc. why?
2019/01/10 11:00:27 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001EBF6D0 m//webinar~eSNCFTq~private~0Tor0/webinar~eSNCFTq to chanhead_gc. why?
2019/01/10 11:00:27 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000002762BD0 m//webinar~eSNCFTq~private~pmO81/webinar~eSNCFTq to chanhead_gc. why?
2019/01/10 11:01:14 [error] 4621#4621: *206103 open() "/u/apps/web_production/current/public/uploads/image/74318/Schermafbeelding_2016-07-20_om_14.27.19.png" failed (2: No such file or directory), client: xx.13.127.3, server: *.web.com, request: "GET /uploads/image/74318/Schermag_2016-07-20_om_14.27.19.png HTTP/1.1", host: "app.web.com"
18 more errors that basically the same `2: No such file or directory`
2019/01/10 11:14:16 [error] 4621#4621: *212859 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.200.12.80, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"
2019/01/10 11:15:21 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000002036D20 m//webinar~8734JxP~private~_fEbH/webinar~8734JxP to chanhead_gc. why?
2019/01/10 11:15:21 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000003466790 m//webinar~8734JxP0aKBFfrtaF5FdjZ~private~dbAc-ZlfNoc/webinar~8734JxP0aKBFfrtaF5FdjZ to chanhead_gc. why?
2019/01/10 11:15:21 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001D5D9A0 m//webinar~8734JxP~private~_9pom/webinar~8734JxP to chanhead_gc. why?
2019/01/10 11:15:21 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001AD0170 m//webinar~cEeIUkC~private~SO7MD/webinar~cEeIUkC to chanhead_gc. why?
2019/01/10 11:15:21 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 000000000246B910 m//webinar~8734JxP~private~ItO2L/webinar~8734JxP to chanhead_gc. why?
2019/01/10 11:15:21 [error] 4621#4621: MEMSTORE:00: tried adding WAITING chanhead 0000000001B67210 m//webinar~8734JxP~private~ZUHou/webinar~8734JxP to chanhead_gc. why?
2019/01/10 11:16:08 [error] 4621#4621: *213804 open() "/u/apps/web_production/current/public/uploads/image/1386033/foto_A56nne.jpg" failed (2: No such file or directory), client: xx.20.113.128, server: *.web.com, request: "GET /uploads/image/1386033/foto_A56nne.jpg HTTP/1.1", host: "gifted-pony.web.com", referrer: "https://gifted-pony.web.com/webinar/watch/X72va8Q/"
50 more errors that basically the same `2: No such file or directory`
2019/01/10 11:43:14 [error] 4621#4621: *223562 open() "/u/apps/web_production/current/public/uploads/image/445566/logo-foto.png" failed (2: No such file or directory), client: xx.179.94.171, server: *.web.com, request: "GET /uploads/image/445566/logo-foto.png HTTP/1.1", host: "app.web.com"

I removed the lines that were basically the same 2: No such file or directory errors

Jan 10 '19 21:01 ivanovv

Here is full Nginx config (nginx -T) with some info edited

# configuration file /etc/nginx/nginx.conf:
user www-data;
worker_processes  1;
worker_rlimit_nofile 16384;

error_log  /var/log/nginx/error.log notice;
pid        /run/nginx.pid;

events {
  worker_connections  8192;
}

http {

  include       /etc/nginx/mime.types;
  default_type  application/octet-stream;

  log_format nchan '$remote_addr [$time_local] $nchan_channel_id $nchan_subscriber_type $nchan_publisher_type $nchan_message_id';

  access_log	/var/log/nginx/access.log combined;
  server_tokens off;

  sendfile on;
  tcp_nopush on;
  tcp_nodelay on;

  keepalive_timeout  65;

  gzip  on;
  gzip_http_version 1.0;
  gzip_comp_level 2;
  gzip_proxied any;
  gzip_vary off;
  gzip_types text/plain text/css ...;
  gzip_min_length  1000;
  gzip_disable     "MSIE [1-6]\.";

  server_names_hash_bucket_size 64;
  types_hash_max_size 2048;
  types_hash_bucket_size 64;

  client_header_buffer_size 64k;
  large_client_header_buffers 4 64k;

  include /etc/nginx/conf.d/*.conf;
  include /etc/nginx/sites-enabled/*;
}

# configuration file /etc/nginx/mime.types:

types {
    text/html                             html htm shtml;
    text/css                              css;
    ...
    video/x-msvideo                       avi;
}

# configuration file /etc/nginx/sites-enabled/default:
server {
  listen   80;
  server_name  app;

  access_log  /var/log/nginx/localhost.access.log;

  location / {
    root   /var/www/nginx-default;
    index  index.html index.htm;
  }
}

# configuration file /etc/nginx/sites-enabled/nchan.conf:
server {
  listen      10.xxx.xx.90:80;
  server_name 10.xxx.xx.90;

  location ~ /pub/(.*)$ {
    # error_log /var/log/nginx/nchan.pub.debug.log debug;
 
    nchan_publisher;
    nchan_channel_id $1;
    #nchan_channel_group test;
    #nchan_channel_events_channel_id $1;
    nchan_message_buffer_length 0;
    nchan_message_timeout 30s;
  }

  location /nchan_stub_status {
    nchan_stub_status;
  }

  location / {
    root   /var/www/nginx-default;
    index  index.html index.htm;
  }
}

# configuration file /etc/nginx/sites-enabled/something.conf:
server {
  listen 80;
  server_name *.something.com;

  include /etc/nginx/include.default.conf;
  include /etc/nginx/include.http.conf;
}

server {
  listen 443 ssl;
  server_name *.something.com;

  ssl_certificate /something.crt;
  ssl_certificate_key /something.key;

  include /etc/nginx/include.default.conf;
  include /etc/nginx/include.https.conf;
}

# configuration file /etc/nginx/include.default.conf:
root /u/apps/web_production/current/public;

client_max_body_size 128M;

location ~ /sub/(.*)$ {
    nchan_subscriber;
    nchan_channel_id $1;
    nchan_channel_id_split_delimiter ",";
    #nchan_channel_group test;
    nchan_subscriber_first_message oldest;
    nchan_subscribe_request /upstream/sub;
    nchan_unsubscribe_request /upstream/unsub;
}

location = /upstream/unsub {
    proxy_pass http://web_production_main_server/ws/unsub;
    proxy_ignore_client_abort on;  #!!!important!!!!
    proxy_set_header X-Subscriber-Type $nchan_subscriber_type;
    proxy_set_header X-Channel-Id $nchan_channel_id;
    proxy_set_header X-Original-URI $request_uri;
}
location = /upstream/sub {
    proxy_pass http://web_production_main_server/ws/sub;
    proxy_set_header X-Subscriber-Type $nchan_subscriber_type;
    proxy_set_header X-Message-Id $nchan_message_id;
    proxy_set_header X-Channel-Id $nchan_channel_id;
    proxy_set_header X-Original-URI $request_uri;
}

error_page 500 /500.html;
error_page 502 504 /502.html;

# Maintenance page
error_page 503 @503;

# Return a 503 error if the maintenance page exists.
if (-f $document_root/maintenance.html) {
  return 503;
}

location @503 {
  # Serve static assets if found.
  if (-f $request_filename) {
    break;
  }

  rewrite ^(.*)$ /maintenance.html break;
}

# configuration file /etc/nginx/include.http.conf:
location @app {
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header Host $http_host;
    proxy_set_header X-Request-Start "t=${msec}";
    proxy_redirect off;
    proxy_pass http://web_production_main_server;
}

location / {
    try_files $uri @app;
}

# configuration file /etc/nginx/include.https.conf:
location /super_admin/subscriptions {
    try_files $uri @app_main_server;
}

location ~ ^/(assets|uploads)/ {
    gzip_static on;
    expires max;
    add_header Cache-Control public;
    add_header Last-Modified "";
    add_header ETag "";

    open_file_cache max=1000 inactive=500s;
    open_file_cache_valid 600s;
    open_file_cache_errors on;
    break;
}

location ~ ^/(images|javascripts|stylesheets|fonts|icon|lib|sounds|templates|flash)/ {
    gzip_static on;
    expires 1d;
    add_header Cache-Control public;
    add_header Last-Modified "";
    add_header ETag "";

    open_file_cache max=1000 inactive=500s;
    open_file_cache_valid 600s;
    open_file_cache_errors on;
    break;
}

location / {
    try_files $uri @app_farm;
}

location @app_farm {
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header Host $http_host;
    proxy_set_header X-Forwarded-Proto https;
    proxy_set_header X-Request-Start "t=${msec}";
    proxy_redirect off;

    proxy_pass http://web_production_server_farm;
}

location @app_main_server {
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header Host $http_host;
    proxy_set_header X-Forwarded-Proto https;
    proxy_set_header X-Request-Start "t=${msec}";
    proxy_redirect off;

    proxy_pass http://web_production_main_server;
}

# configuration file /etc/nginx/sites-enabled/somethingelse.conf:
server {
  listen 80;
  server_name live.somethingelse.nl;

  include /etc/nginx/include.default.conf;
  include /etc/nginx/include.http.conf;
}

server {
  listen 443 ssl;   
  server_name live.somethingelse.nl;

  ssl_certificate /somethingelse.crt;
  ssl_certificate_key /somethingelse.key;

  include /etc/nginx/include.default.conf;
  include /etc/nginx/include.https.conf;
}

# configuration file /etc/nginx/sites-enabled/web_production.conf:
server {
  listen 80 default_server;
  server_name *.web.com _;

  include /etc/nginx/include.default.conf;
  include /etc/nginx/include.http.conf;
}

server {
  listen 443 ssl;
  server_name *.web.com _;

  ssl_certificate /certificate.crt;
  ssl_certificate_key /certificate.key;

  include /etc/nginx/include.default.conf;
  include /etc/nginx/include.https.conf;
}

include /etc/nginx/include.upstream.conf;

# configuration file /etc/nginx/include.upstream.conf:
upstream web_production_main_server {
    server unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock;
}

upstream web_production_server_farm {
    server unix:/u/apps/web_production/shared/tmp/sockets/unicorn.sock weight=2;
    server 10.xx.xx.xx:9080 weight=3;    
}

Jan 10 '19 21:01 ivanovv

@slact does anything in the logs look suspicious to you?

Jan 16 '19 14:01 ivanovv

not enough to conclusively diagnose the issue. I still need the dump of the first crash after nginx is cleanly stopped and started, ideally with he accompanying error log for that crash. let me know when that happens so i can get this thing fixed.

Jan 16 '19 22:01 slact

Looks like I am having the same problem. One worker crashes multiple times daily with about 1000 connected users. I am willing to run custom debug code, if that helps.

I've emailed @slact a download link for the nginx core dump file.

OS: FreeBSD 12.0 Nginx Version: 1.14.2 with 4 workers Nchan Version: 1.2.3

2019/01/31 19:26:39 [error] 60864#100427: MEMSTORE:00: multimsg 00000008175B3600 timeout!!
2019/01/31 19:26:43 [notice] 60863#100531: signal 20 (SIGCHLD) received from 60867
2019/01/31 19:26:43 [alert] 60863#100531: worker process 60867 exited on signal 11 (core dumped)
2019/01/31 19:26:43 [notice] 60863#100531: start worker process 76012
2019/01/31 19:26:43 [notice] 60863#100531: signal 23 (SIGIO) received
2019/01/31 19:26:43 [notice] 60863#100531: signal 23 (SIGIO) received
2019/01/31 19:26:43 [notice] 60863#100531: signal 23 (SIGIO) received
2019/01/31 19:26:43 [notice] 60863#100531: signal 23 (SIGIO) received
2019/01/31 19:26:43 [notice] 60863#100531: signal 23 (SIGIO) received
2019/01/31 19:26:43 [notice] 60863#100531: signal 23 (SIGIO) received
2019/01/31 19:26:43 [notice] 60863#100531: signal 23 (SIGIO) received
2019/01/31 19:26:43 [notice] 60863#100531: signal 23 (SIGIO) received
2019/01/31 19:26:43 [notice] 60863#100531: signal 23 (SIGIO) received
2019/01/31 19:26:43 [notice] 60863#100531: signal 23 (SIGIO) received
2019/01/31 19:26:43 [error] 76012#101574: nchan: A message from the past has just been published. Unless the system time has been adjusted, this should never happen.
2019/01/31 19:26:43 [error] 76012#101574: nchan: A message from the past has just been published. Unless the system time has been adjusted, this should never happen.
2019/01/31 19:26:44 [notice] 76012#101574: nchan: Received 351 interprocess alerts delayed by 27 sec.
2019/01/31 19:26:48 [error] 60864#100427: MEMSTORE:00: tried adding WAITING chanhead 0000000820A8A800 /user:110740 to chanhead_gc. why?
2019/01/31 19:26:54 [error] 60864#100427: MEMSTORE:00: multimsg 000000081A165600 timeout!!
2019/01/31 19:26:54 [error] 60864#100427: MEMSTORE:00: multimsg 0000000805539000 timeout!!
...
2019/01/31 19:27:03 [error] 76012#101574: MEMSTORE:03: force-reaping msg with refcount 1
2019/01/31 19:27:03 [error] 76012#101574: MEMSTORE:03: force-reaping msg with refcount 1
2019/01/31 19:27:03 [error] 76012#101574: MEMSTORE:03: force-reaping msg with refcount 1
2019/01/31 19:27:03 [error] 76012#101574: MEMSTORE:03: force-reaping msg with refcount 1
2019/01/31 19:27:03 [error] 76012#101574: MEMSTORE:03: force-reaping msg with refcount 1
2019/01/31 19:27:03 [error] 76012#101574: MEMSTORE:03: force-reaping msg with refcount 1
...

GDB Backtrace:

GNU gdb (GDB) 8.2 [GDB v8.2 for FreeBSD]
Copyright (C) 2018 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
This GDB was configured as "x86_64-portbld-freebsd12.0".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
    <http://www.gnu.org/software/gdb/documentation/>.

For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /usr/local/sbin/nginx...done.
[New LWP 100122]
Core was generated by `nginx: worker process'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00000008009548ee in memcpy () from /lib/libc.so.7
(gdb) backtrace full
#0  0x00000008009548ee in memcpy () from /lib/libc.so.7
No symbol table info available.
#1  0x000000000026be8a in ngx_ssl_send_chain (c=0x81582ccf0, in=0x817eab958, limit=2147479551) at src/event/ngx_event_openssl.c:1748
        n = <optimized out>
        buf = <optimized out>
        send = 16937
        flush = 1
        size = -207547238
#2  0x00000000002a942c in ngx_http_v2_send_output_queue (h2c=0x817b58c20) at src/http/v2/ngx_http_v2.c:537
        c = 0x81582ccf0
        wev = 0x81621d148
        frame = 0x822f2bc29
        cl = 0x80133d913
        out = <optimized out>
        fn = <optimized out>
        clcf = <optimized out>
        tcp_nodelay = <optimized out>
#3  0x00000000002a935b in ngx_http_v2_write_handler (wev=0x81621d148) at src/http/v2/ngx_http_v2.c:481
        c = 0x1ffffffffe73747b
        h2c = 0x817b58c20
        rc = <optimized out>
#4  0x00000000002684fa in ngx_kqueue_process_events (cycle=0x801259a90, timer=<optimized out>, flags=1) at src/event/modules/ngx_kqueue_module.c:669
        n = <optimized out>
        tp = <optimized out>
        events = <optimized out>
        err = <optimized out>
        i = <error reading variable i (Cannot access memory at address 0x0)>
        ev = <optimized out>
        instance = <optimized out>
        queue = <optimized out>
        level = <optimized out>
        ts = <optimized out>
#5  0x000000000025ec06 in ngx_process_events_and_timers (cycle=0x801259a90) at src/event/ngx_event.c:242
        flags = 1
        timer = 80
        delta = 1714354905
#6  0x0000000000266c1d in ngx_worker_process_cycle (cycle=0x801259a90, data=<optimized out>) at src/os/unix/ngx_process_cycle.c:750
        worker = <optimized out>
#7  0x0000000000264fbb in ngx_spawn_process (cycle=0x801259a90, proc=0x266bb0 <ngx_worker_process_cycle>, data=0x3, name=0x20a72a "worker process", 
--Type <RET> for more, q to quit, c to continue without paging--c
    respawn=-3) at src/os/unix/ngx_process.c:199
        on = 1
        s = <optimized out>
        pid = 0
#8  0x0000000000265a0d in ngx_start_worker_processes (cycle=0x801259a90, n=<optimized out>, type=-3) at src/os/unix/ngx_process_cycle.c:359
        i = 3
        ch = <optimized out>
#9  ngx_master_process_cycle (cycle=0x801259a90) at src/os/unix/ngx_process_cycle.c:131
        set = {__bits = {0, 0, 0, 0}}
        i = <optimized out>
        size = <optimized out>
        title = <optimized out>
        p = <optimized out>
        ccf = 0x80125b790
        live = <optimized out>
        sigio = <optimized out>
        delay = <optimized out>
        itv = <optimized out>
        n = <optimized out>
        ls = <optimized out>
#10 0x000000000023dd01 in main (argc=<optimized out>, argv=<optimized out>) at src/core/nginx.c:382
        log = 0x2fe498 <ngx_log>
        init_cycle = {conf_ctx = 0x0, pool = 0x801202400, log = 0x2fe498 <ngx_log>, new_log = {log_level = 0, file = 0x0, connection = 0, disk_full_time = 0, handler = 0x0, data = 0x0, writer = 0x0, wdata = 0x0, action = 0x0, next = 0x0}, log_use_stderr = 0, files = 0x0, free_connections = 0x0, free_connection_n = 0, modules = 0x0, modules_n = 0, modules_used = 0, reusable_connections_queue = {prev = 0x0, next = 0x0}, reusable_connections_n = 0, listening = {elts = 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0}, paths = {elts = 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0}, config_dump = {elts = 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0}, config_dump_rbtree = {root = 0x0, sentinel = 0x0, insert = 0x0}, config_dump_sentinel = {key = 0, left = 0x0, right = 0x0, parent = 0x0, color = 0 '\000', data = 0 '\000'}, open_files = {last = 0x0, part = {elts = 0x0, nelts = 0, next = 0x0}, size = 0, nalloc = 0, pool = 0x0}, shared_memory = {last = 0x0, part = {elts = 0x0, nelts = 0, next = 0x0}, size = 0, nalloc = 0, pool = 0x0}, connection_n = 0, files_n = 0, connections = 0x0, read_events = 0x0, write_events = 0x0, old_cycle = 0x0, conf_file = {len = 31, data = 0x20f9e5 "/usr/local/etc/nginx/nginx.conf"}, conf_param = {len = 0, data = 0x0}, conf_prefix = {len = 21, data = 0x20f9e5 "/usr/local/etc/nginx/nginx.conf"}, prefix = {len = 21, data = 0x20c54f "/usr/local/etc/nginx/"}, lock_file = {len = 0, data = 0x0}, hostname = {len = 0, data = 0x0}}
        cycle = 0x801259a90
        i = <optimized out>
        cd = <optimized out>
        b = <optimized out>
        ccf = <optimized out>

Jan 31 '19 11:01 denisu

I've also noticed that subscribers and channels in nchan_stub_status keeps growing over time. It looks like many subscribers are not cleared on disconnect - perhaps related.

Jan 31 '19 18:01 denisu

Similar issue here. I found woker crashed when backend server of nchan_(un)subscribe_request returns non-200 response. Also, with X-Accel feature, workers crashed on 400 response.

I guess there may be some bug in function which controls outgoing request.

Feb 20 '19 06:02 squirrel532

nchan nchan copied to clipboard

Worker crash

nchan
nchan copied to clipboard