passenger
passenger copied to clipboard
phusion
Excessive Apache startup time when running many virtual hosts/rails applications
Question 1: What is the problem?
Startup time of apache instance with many virtual hosts (>2700) and rails apps (>7500) (from apache2ctl start to prompt returning) is taking two minutes or more. (once started, applications spawn and run as expected).
- What is the expected behavior?
On version of passenger <5.2, the startup time is about 3 seconds. Tested on Passenger versions: 5.2.0, 5.2.3, 5.3.3. Runs fine on Passenger (5.1.12 and earlier)
- What is the actual behavior?
On versions of passenger >=5.2 (tested on 5.2.0, 5.2.3, and 5.3.3) the startup time takes approximately 2 minutes with a large number of virtual hosts/applications.
- How can we reproduce it? Please try to provide a sample application (or Virtual Machine) demonstrating the issue. Otherwise, if we can't reproduce it, we might have to ask you a number of followup questions or run certain commands to try and figure out the problem.
Create an apache instance with 2000 virtual hosts each with 1 or more rails applications on a different path. 99% of applications are the same with only url, path, and database.yml being different. These are older rails apps (2.3). I have tried running PassengerDefaultRuby with both older REE 1.8.7 based ruby and newer 2.5.1 ruby with same results.
- Be as detailed as possible in your descriptions, include any logs and stack traces (don't just cut/paste the error, provide some logging before that too).
I have turned up passenger logging to highest level and there is nothing in their that points to the issue. During Startup there is just a big gap between startup steps without indication of the delay. If feels like there is some difference on how the enumeration of Virtual Hosts works from Passenger 5.1 to 5.2 which is much slower. When I tried with just a few virtual hosts things started as expected.
[ D 2018-07-02 10:56:18.7553 30885/T1 apa/Hooks.cpp:1285 ]: Initializing Phusion Passenger...
[ D 2018-07-02 10:57:17.5817 30915/T1 age/Sha/Fun/Initialization.cpp:509 ]: Starting agent: Passenger watchdog
[ D 2018-07-02 10:57:17.9790 30915/T1 age/Sha/Fun/Initialization.cpp:620 ]: Passenger watchdog config: {
"admin_panel_auth_type" : "basic",
"admin_panel_close_timeout" : 10.0,
"admin_panel_connect_timeout" : 30.0,
"admin_panel_data_debug" : false,
"admin_panel_password" : null,
"admin_panel_password_file" : null,
"admin_panel_ping_interval" : 30.0,
"admin_panel_ping_timeout" : 30.0,
"admin_panel_proxy_password" : null,
"admin_panel_proxy_timeout" : 30.0,
"admin_panel_proxy_url" : null,
"admin_panel_proxy_username" : null,
"admin_panel_reconnect_timeout" : 5.0,
"admin_panel_url" : null,
"admin_panel_username" : null,
"admin_panel_websocketpp_debug_access" : false,
"admin_panel_websocketpp_debug_error" : false,
"app_output_log_level" : "notice",
"benchmark_mode" : null,
"config_manifest" : "[OMITTED]",
"controller_accept_burst_count" : 32,
"controller_addresses" : null,
"controller_client_freelist_limit" : 0,
"controller_cpu_affine" : false,
"controller_file_buffered_channel_auto_start_mover" : true,
"controller_file_buffered_channel_auto_truncate_file" : true,
"controller_file_buffered_channel_buffer_dir" : "/tmp",
"controller_file_buffered_channel_delay_in_file_mode_switching" : 0,
"controller_file_buffered_channel_max_disk_chunk_read_size" : 0,
"controller_file_buffered_channel_threshold" : 131072,
"controller_mbuf_block_chunk_size" : 4096,
"controller_min_spare_clients" : 0,
"controller_pid_file" : null,
"controller_request_freelist_limit" : 1024,
"controller_secure_headers_password" : "6OhftHdhSH3k3igjpBguGRpr",
"controller_socket_backlog" : 2048,
"controller_start_reading_after_accept" : true,
"controller_threads" : 16,
"core_api_server_accept_burst_count" : 32,
"core_api_server_addresses" : [],
"core_api_server_authorizations" : [],
"core_api_server_client_freelist_limit" : 0,
"core_api_server_file_buffered_channel_auto_start_mover" : true,
"core_api_server_file_buffered_channel_auto_truncate_file" : true,
"core_api_server_file_buffered_channel_buffer_dir" : "/tmp",
"core_api_server_file_buffered_channel_delay_in_file_mode_switching" : 0,
"core_api_server_file_buffered_channel_max_disk_chunk_read_size" : 0,
"core_api_server_file_buffered_channel_threshold" : 131072,
"core_api_server_mbuf_block_chunk_size" : 4096,
"core_api_server_min_spare_clients" : 0,
"core_api_server_request_freelist_limit" : 1024,
"core_api_server_start_reading_after_accept" : true,
"core_file_descriptor_ulimit" : 0,
"core_pid_file" : null,
"daemonize" : false,
"default_abort_websockets_on_process_shutdown" : true,
"default_app_file_descriptor_ulimit" : null,
"default_environment" : "production",
"default_force_max_concurrent_requests_per_process" : -1,
"default_friendly_error_pages" : "auto",
"default_group" : "nogroup",
"default_load_shell_envvars" : true,
"default_max_preloader_idle_time" : 300,
"default_max_request_queue_size" : 100,
"default_max_requests" : 0,
"default_meteor_app_settings" : null,
"default_min_instances" : 1,
"default_nodejs" : "node",
"default_python" : "python",
"default_ruby" : "/usr/local/rvm/gems/ree-1.8.7-2012.02@tofino-bundler/wrappers/ruby",
"default_server_name" : "127.0.0.1",
"default_server_port" : 3000,
"default_spawn_method" : "smart",
"default_sticky_sessions" : false,
"default_sticky_sessions_cookie_name" : "_passenger_route",
"default_user" : "nobody",
"file_descriptor_log_target" : null,
"graceful_exit" : true,
"hook_after_watchdog_initialization" : null,
"hook_after_watchdog_shutdown" : null,
"hook_before_watchdog_initialization" : null,
"hook_before_watchdog_shutdown" : null,
"instance_registry_dir" : "/var/run/passenger-instreg",
"integration_mode" : "apache",
"log_level" : "debug",
"log_target" : {
"path" : "/var/log/apache2/passenger.log"
},
"max_instances_per_app" : 10,
"max_pool_size" : 190,
"multi_app" : true,
"passenger_root" : "/usr/local/rvm/gems/ree-1.8.7-2012.02@tofino-bundler/gems/passenger-5.3.3",
"pidfiles_to_delete_on_exit" : [],
"pool_idle_time" : 360,
"pool_selfchecks" : false,
"prestart_urls" : [ "https://my.schoolauction.net:443/registrations/reload/" ],
"response_buffer_high_watermark" : 134217728,
"security_update_checker_certificate_path" : null,
"security_update_checker_disabled" : false,
"security_update_checker_interval" : 86400,
"security_update_checker_proxy_url" : null,
"security_update_checker_url" : "https://securitycheck.phusionpassenger.com/v1/check.json",
"server_software" : "Apache/2.4.18 (Ubuntu) OpenSSL/1.0.2g Phusion_Passenger/5.3.3",
"setsid" : false,
"show_version_in_header" : true,
"single_app_mode_app_root" : "/var/www",
"single_app_mode_app_type" : null,
"single_app_mode_startup_file" : null,
"standalone_engine" : null,
"startup_report_file" : null,
"stat_throttle_rate" : 10,
"turbocaching" : true,
"user" : null,
"user_switching" : true,
"vary_turbocache_by_cookie" : null,
"watchdog_api_server_accept_burst_count" : 32,
"watchdog_api_server_addresses" : [],
"watchdog_api_server_authorizations" : [],
"watchdog_api_server_client_freelist_limit" : 0,
"watchdog_api_server_file_buffered_channel_auto_start_mover" : true,
"watchdog_api_server_file_buffered_channel_auto_truncate_file" : true,
"watchdog_api_server_file_buffered_channel_buffer_dir" : "/tmp",
"watchdog_api_server_file_buffered_channel_delay_in_file_mode_switching" : 0,
"watchdog_api_server_file_buffered_channel_max_disk_chunk_read_size" : 0,
"watchdog_api_server_file_buffered_channel_threshold" : 131072,
"watchdog_api_server_mbuf_block_chunk_size" : 4096,
"watchdog_api_server_min_spare_clients" : 0,
"watchdog_api_server_request_freelist_limit" : 1024,
"watchdog_api_server_start_reading_after_accept" : true,
"watchdog_pid_file" : null,
"watchdog_pid_file_autodelete" : true,
"web_server_module_version" : "5.3.3",
"web_server_version" : "2.4.18"
}
[ D 2018-07-02 10:57:23.3430 30915/T1 age/Sha/Fun/Initialization.cpp:621 ]: Passenger watchdog random seed: 1530554179
[ N 2018-07-02 10:57:23.3432 30915/T1 age/Wat/WatchdogMain.cpp:1297 ]: Starting Passenger watchdog...
[ D 2018-07-02 10:57:23.3452 30915/T1 age/Wat/WatchdogMain.cpp:1157 ]: API server will listen on unix:/var/run/passenger-instreg/passenger.DgKKwmI/agents.s/watchdog_api
[ D 2018-07-02 10:57:41.9209 30915/T1 age/Wat/WatchdogMain.cpp:1196 ]: Starting agent: Passenger core
[ D 2018-07-02 10:58:23.3104 31190/T1 age/Sha/Fun/Initialization.cpp:509 ]: Starting agent: Passenger core
[ D 2018-07-02 10:58:23.5591 31190/T1 age/Sha/Fun/Initialization.cpp:620 ]: Passenger core config: {
"admin_panel_auth_type" : "basic",
"admin_panel_close_timeout" : 10.0,
"admin_panel_connect_timeout" : 30.0,
"admin_panel_data_debug" : false,
"admin_panel_password" : null,
"admin_panel_password_file" : null,
"admin_panel_ping_interval" : 30.0,
"admin_panel_ping_timeout" : 30.0,
"admin_panel_proxy_password" : null,
"admin_panel_proxy_timeout" : 30.0,
"admin_panel_proxy_url" : null,
"admin_panel_proxy_username" : null,
"admin_panel_reconnect_timeout" : 5.0,
"admin_panel_url" : null,
"admin_panel_username" : null,
"admin_panel_websocketpp_debug_access" : false,
"admin_panel_websocketpp_debug_error" : false,
"api_server_accept_burst_count" : 32,
"api_server_addresses" : [ "unix:/var/run/passenger-instreg/passenger.DgKKwmI/agents.s/core_api" ],
"api_server_authorizations" : [
{
"level" : "readonly",
"password_file" : "/var/run/passenger-instreg/passenger.DgKKwmI/read_only_admin_password.txt",
"username" : "ro_admin"
},
{
"level" : "full",
"password_file" : "/var/run/passenger-instreg/passenger.DgKKwmI/full_admin_password.txt",
"username" : "admin"
}
],
"api_server_client_freelist_limit" : 0,
"api_server_file_buffered_channel_auto_start_mover" : true,
"api_server_file_buffered_channel_auto_truncate_file" : true,
"api_server_file_buffered_channel_buffer_dir" : "/tmp",
"api_server_file_buffered_channel_delay_in_file_mode_switching" : 0,
"api_server_file_buffered_channel_max_disk_chunk_read_size" : 0,
"api_server_file_buffered_channel_threshold" : 131072,
"api_server_mbuf_block_chunk_size" : 4096,
"api_server_min_spare_clients" : 0,
"api_server_request_freelist_limit" : 1024,
"api_server_start_reading_after_accept" : true,
"app_output_log_level" : "notice",
"benchmark_mode" : null,
"config_manifest" : "[OMITTED]",
"controller_accept_burst_count" : 32,
"controller_addresses" : [ "unix:/var/run/passenger-instreg/passenger.DgKKwmI/agents.s/core" ],
"controller_client_freelist_limit" : 0,
"controller_cpu_affine" : false,
"controller_file_buffered_channel_auto_start_mover" : true,
"controller_file_buffered_channel_auto_truncate_file" : true,
"controller_file_buffered_channel_buffer_dir" : "/tmp",
"controller_file_buffered_channel_delay_in_file_mode_switching" : 0,
"controller_file_buffered_channel_max_disk_chunk_read_size" : 0,
"controller_file_buffered_channel_threshold" : 131072,
"controller_mbuf_block_chunk_size" : 4096,
"controller_min_spare_clients" : 0,
"controller_request_freelist_limit" : 1024,
"controller_secure_headers_password" : "6OhftHdhSH3k3igjpBguGRpr",
"controller_socket_backlog" : 2048,
"controller_start_reading_after_accept" : true,
"controller_threads" : 16,
"default_abort_websockets_on_process_shutdown" : true,
"default_app_file_descriptor_ulimit" : null,
"default_environment" : "production",
"default_force_max_concurrent_requests_per_process" : -1,
"default_friendly_error_pages" : "auto",
"default_group" : "nogroup",
"default_load_shell_envvars" : true,
"default_max_preloader_idle_time" : 300,
"default_max_request_queue_size" : 100,
"default_max_requests" : 0,
"default_meteor_app_settings" : null,
"default_min_instances" : 1,
"default_nodejs" : "node",
"default_python" : "python",
"default_ruby" : "/usr/local/rvm/gems/ree-1.8.7-2012.02@tofino-bundler/wrappers/ruby",
"default_server_name" : "127.0.0.1",
"default_server_port" : 3000,
"default_spawn_method" : "smart",
"default_sticky_sessions" : false,
"default_sticky_sessions_cookie_name" : "_passenger_route",
"default_user" : "nobody",
"file_descriptor_log_target" : null,
"file_descriptor_ulimit" : 0,
"graceful_exit" : true,
"instance_dir" : "/var/run/passenger-instreg/passenger.DgKKwmI",
"integration_mode" : "apache",
"log_level" : "debug",
"log_target" : {
"path" : "/var/log/apache2/passenger.log"
},
"max_instances_per_app" : 10,
"max_pool_size" : 190,
"multi_app" : true,
"passenger_root" : "/usr/local/rvm/gems/ree-1.8.7-2012.02@tofino-bundler/gems/passenger-5.3.3",
"pid_file" : "/var/run/passenger-instreg/passenger.DgKKwmI/core.pid",
"pool_idle_time" : 360,
"pool_selfchecks" : false,
"prestart_urls" : [ "https://my.schoolauction.net:443/registrations/reload/" ],
"response_buffer_high_watermark" : 134217728,
"security_update_checker_certificate_path" : null,
"security_update_checker_disabled" : false,
"security_update_checker_interval" : 86400,
"security_update_checker_proxy_url" : null,
"security_update_checker_url" : "https://securitycheck.phusionpassenger.com/v1/check.json",
"server_software" : "Apache/2.4.18 (Ubuntu) OpenSSL/1.0.2g Phusion_Passenger/5.3.3",
"show_version_in_header" : true,
"single_app_mode_app_root" : "/var/www",
"single_app_mode_app_type" : null,
"single_app_mode_startup_file" : null,
"standalone_engine" : null,
"stat_throttle_rate" : 10,
"turbocaching" : true,
"user_switching" : true,
"vary_turbocache_by_cookie" : null,
"watchdog_fd_passing_password" : "G6qE56Me8Mrh9dR4YqQmtlgA",
"web_server_module_version" : "5.3.3",
"web_server_version" : "2.4.18"
}
[ D 2018-07-02 10:58:24.9068 31190/T1 age/Sha/Fun/Initialization.cpp:621 ]: Passenger core random seed: 1530554262
[ N 2018-07-02 10:58:24.9069 31190/T1 age/Cor/CoreMain.cpp:1202 ]: Starting Passenger core...
[ N 2018-07-02 10:58:24.9069 31190/T1 age/Cor/CoreMain.cpp:252 ]: Passenger core running in multi-application mode.
[ D 2018-07-02 10:58:28.3433 31190/T3 age/Cor/App/Poo/AnalyticsCollection.cpp:116 ]: Analytics collection time...
[ D 2018-07-02 10:58:28.3434 31190/T3 age/Cor/App/Poo/AnalyticsCollection.cpp:143 ]: Collecting process metrics
[ D 2018-07-02 10:58:28.3434 31190/T3 age/Cor/App/Poo/AnalyticsCollection.cpp:151 ]: Collecting system metrics
[ D 2018-07-02 10:58:28.3440 31190/T3 age/Cor/App/Poo/AnalyticsCollection.cpp:69 ]: Analytics collection done; next analytics collection in 1.656 sec
[ D 2018-07-02 10:58:30.0002 31190/T3 age/Cor/App/Poo/AnalyticsCollection.cpp:116 ]: Analytics collection time...
...
(if you are requesting a feature instead of reporting an issue, describe here what you have in mind and how it would help you)
Your answer:
Question 2: Passenger version and integration mode:
- For example: open source 5.0.26 standalone; enterprise 5.0.21/nginx Your answer:
Open Source 5.3.3/apache (also tried 5.2.3, works fine with 5.1.12)
Question 3: OS or Linux distro, platform (including version):
- For example: Debian 8, x86_64 or OS X 10.10 Yosemite, x86_64 Your answer:
Ubuntu 16.04.4 LTS (Xenial Xerus)
Question 4: Passenger installation method:
Your answer:
[ ] RubyGems + Gemfile
[x] RubyGems, no Gemfile
[x] Phusion APT repo
[ ] Phusion YUM repo
[ ] OS X Homebrew
[ ] source tarball
[ ] Other, please specify:
Have tried both methods above with same results
Question 5: Your app's programming language (including any version managers) and framework (including versions):
- For example: Ruby 2.3.0, RVM, Rails 5.0.0; Node.js 4.0.0 with Express 4.13.4
Your answer:
Ruby 1.8.7, RVM, Rails 2.3 (currently we are upgrading to rails 5.2 w/ ruby 2.5.1 but until this is complete we need to run in this environment)
Question 6: Are you using a PaaS and/or containerization? If so which one?
- For example: Heroku, Amazon Container Services, Docker 1.9 with an image based on passenger-docker
Your answer:
**Question 7: Anything else about your setup that we should know?**
Your answer:
Tried lots of combos to fix this issue (different default ruby, removing all restart.txt files that were hanging out, reconfiguring apache so that multiple application on subpaths are not handled by symlinks in the public directory anymore, but use the suggested method of using Aliases and PassengerAppRoot.
---
We strive for quality and appreciate you taking the time to submit a report! Please note that if you want guaranteed response times and priority issue support we encourage you to join our [enterprise customer](https://www.phusionpassenger.com/enterprise) base. They also provide us with the means to continue our high level of open source support!
There's a few things we should check here, can you switch to the user(s) that run the app(s) and then: 1) run env | wc; and 2) make sure the shell startup files do not take a long time (for example add an inter-activeness check at the very beginning).
This issue affects all passenger versions older than 5.2 using with apache. And the problem really becomes noticeable on the servers with a large number of virtual hosts.
Here some research I have done.
As you can see, the longest intervals are between the following entries:
[ D 2018-12-20 11:27:50.6267 110458/T1 apa/Hooks.cpp:1285 ]: Initializing Phusion Passenger...
[ D 2018-12-20 11:28:21.3608 111056/T1 age/Sha/Fun/Initialization.cpp:509 ]: Starting agent: Passenger watchdog
and
[ D 2018-12-20 11:28:32.4019 111056/T1 age/Wat/WatchdogMain.cpp:1196 ]: Starting agent: Passenger core
[ D 2018-12-20 11:28:57.3532 111192/T1 age/Sha/Fun/Initialization.cpp:509 ]: Starting agent: Passenger core
I ran strace and here what I found
STR1----->11:27:50.626880 write(2, "[ D 2018-12-20 11:27:50.6267 110"..., 95) = 95
11:27:50.789268 fstat(2, {st_mode=S_IFREG|0644, st_size=1788, ...}) = 0
11:27:50.789471 open("/etc/apache2/logs/error_log", O_WRONLY|O_APPEND) = 18
11:27:50.789631 fstat(18, {st_mode=S_IFREG|0644, st_size=1788, ...}) = 0
11:27:50.789720 close(18) = 0
11:27:50.789872 stat("/opt/passenger-5.3.3-1.el7.cloudlinux", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
11:27:50.789998 stat("/opt/passenger-5.3.3-1.el7.cloudlinux/buildout/support-binaries/PassengerAgent", {st_mode=S_IFREG|0755, st_size=33921256, ...}) = 0
11:27:50.790326 brk(NULL) = 0x55782ad12000
11:27:50.790474 brk(0x55782ad33000) = 0x55782ad33000
11:27:50.790975 brk(NULL) = 0x55782ad33000
11:27:50.791059 brk(0x55782ad54000) = 0x55782ad54000
11:27:50.791490 brk(NULL) = 0x55782ad54000
... a lot of brk
11:27:51.710146 brk(0x55783751c000) = 0x55783751c000
11:27:51.710315 brk(NULL) = 0x55783751c000
11:27:51.710508 brk(NULL) = 0x55783751c000
11:27:51.710592 brk(NULL) = 0x55783751c000
11:27:51.710669 brk(0x55783751b000) = 0x55783751b000
11:27:51.710759 brk(NULL) = 0x55783751b000
11:27:51.713044 mmap(NULL, 524288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdf63fb1000
11:27:51.713745 munmap(0x7fdf64031000, 262144) = 0
11:27:51.718404 mmap(NULL, 1048576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdf63eb1000
11:27:51.720286 munmap(0x7fdf63fb1000, 524288) = 0
11:27:51.730148 mmap(NULL, 2097152, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdf56ac1000
11:27:51.733226 munmap(0x7fdf63eb1000, 1048576) = 0
11:27:51.752724 mmap(NULL, 4194304, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdf566c1000
11:27:51.757757 munmap(0x7fdf56ac1000, 2097152) = 0
11:27:51.796423 mmap(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdf55ec1000
11:27:51.803777 munmap(0x7fdf566c1000, 4194304) = 0
11:27:51.880031 mmap(NULL, 16777216, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdf54ec1000
11:27:51.894007 munmap(0x7fdf55ec1000, 8388608) = 0
11:27:52.041699 mmap(NULL, 33554432, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdf4e000000
11:27:52.063078 munmap(0x7fdf54ec1000, 16777216) = 0
11:27:52.356317 mmap(NULL, 67108864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdf4a000000
11:27:52.399048 munmap(0x7fdf4e000000, 33554432) = 0
STR2----->11:27:52.874622 writev(18, [{"\3\234\6\365", 4}, {"{\n \"admin_panel_auth_type\" : n"..., 60557045}], 2) = 60557049
11:27:53.208871 munmap(0x7fdf4a000000, 67108864) = 0
11:27:53.213709 read(18, "\0\23", 2) = 2
11:30:03.790005 read(18, "Agents information\0", 19) = 19
11:30:03.790999 read(18, "\0\0\0\267", 4) = 4
STR1 is the place where appears:
[ D 2018-12-20 11:27:50.6267 110458/T1 apa/Hooks.cpp:1285 ]: Initializing Phusion Passenger...
STR2 comes from writeScalarMessage https://github.com/phusion/passenger/blob/23f2d842b12e259820a5971ad061654268bda694/src/cxx_supportlib/WatchdogLauncher.h#L415-L417
In this case, the size of the transmitted config is impressive.
I searched further and saw that all the directives of all the virtual hosts are written to this config.(added in passenger 5.2) https://github.com/phusion/passenger/blob/23f2d842b12e259820a5971ad061654268bda694/src/apache2_module/Utils.h#L134-L137
The question arises whether all these configs are needed to start the Passenger WatchDog and then the Passenger Core?
Because from WatchDog strace we can see:
...
11:27:51.767923 execve("/opt/passenger-5.3.3-1.el7.cloudlinux/buildout/support-binaries/PassengerAgent", ["PassengerAgent", "watchdog", " "...], [/* 10 vars */]) = 0
....
11:27:51.853974 read(3, "\3\234\6\365", 4) = 4
11:27:52.874930 mmap(NULL, 60559360, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f17539f5000
STR3----->11:27:52.875052 read(3, "{\n \"admin_panel_auth_type\" : n"..., 32768) = 32768
11:27:52.875210 read(3, "\n \"source"..., 32768) = 3772
11:27:52.875309 read(3, "default\"\n "..., 28996) = 28996
11:27:52.875490 read(3, "\n \"value_hierar"..., 32768) = 7548
11:27:52.875587 read(3, " \"value\" : -1\n "..., 25220) = 25220
11:27:52.875739 read(3, " \"value\" : 0\n "..., 32768) = 11324
11:27:52.875846 read(3, " }\n ]\n "..., 21444) = 21444
11:27:52.876001 read(3, " \"source\" : {"..., 32768) = 15100
11:27:52.876099 read(3, "lue_hierarchy\" : [\n "..., 17668) = 17668
11:27:52.876254 read(3, " \"value_hierarchy"..., 32768) = 18876
11:27:52.876351 read(3, " \"value_hierarchy\" "..., 13892) = 13892
11:27:52.876567 read(3, " : {\n \"value_hi"..., 32768) = 22652
11:27:52.876667 read(3, " \"source\" : {\n "..., 10116) = 10116
11:27:52.876800 read(3, "PassengerAppEnv\"\n "..., 32768) = 32768
...
11:27:53.208136 read(3, "source\" : {\n "..., 32768) = 32768
11:27:53.208274 read(3, "ce\" : {\n "..., 32768) = 32768
11:27:53.208432 read(3, " \"type\" "..., 32768) = 32768
11:27:53.208574 read(3, " \"path\" : \"/etc/ap"..., 32768) = 32768
11:27:53.208711 read(3, " \"source\" : {\n "..., 32768) = 32768
11:27:53.209792 read(3, " \"source\" : {\n "..., 32768) = 32768
11:27:53.210143 read(3, "\" : {\n "..., 32768) = 32768
11:27:53.210456 read(3, " \"path\" : "..., 32768) = 32768
11:27:53.210698 read(3, " \"source\" : {\n "..., 32768) = 32768
11:27:53.210987 read(3, " \"source\" : {\n "..., 32768) = 32768
11:27:53.211228 read(3, " },\n "..., 1781) = 1781
11:27:53.211539 mmap(NULL, 60559360, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f1750034000
11:27:53.292090 mmap(NULL, 60563456, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f174c672000
11:27:53.399271 brk(NULL) = 0x1eaf000
11:27:53.399498 brk(0x1ed0000) = 0x1ed0000
11:27:53.400398 brk(NULL) = 0x1ed0000
11:27:53.400542 brk(0x1ef1000) = 0x1ef1000
11:27:53.401765 brk(NULL) = 0x1ef1000
11:27:53.401955 brk(0x1f12000) = 0x1f12000
11:27:53.403205 brk(NULL) = 0x1f12000
11:27:53.403382 brk(0x1f33000) = 0x1f33000
11:27:53.404565 brk(NULL) = 0x1f33000
11:27:53.404730 brk(0x1f54000) = 0x1f54000
... again long loop with brk
11:28:01.770455 brk(NULL) = 0x4cc39000
11:28:01.770540 brk(0x4cc5a000) = 0x4cc5a000
11:28:01.771111 brk(NULL) = 0x4cc5a000
11:28:01.771194 brk(0x4cc7b000) = 0x4cc7b000
11:28:01.771768 brk(NULL) = 0x4cc7b000
11:28:01.771867 brk(0x4cc9c000) = 0x4cc9c000
11:28:03.357592 brk(NULL) = 0x4cc9c000
11:28:03.357743 brk(0x4ccbd000) = 0x4ccbd000
11:28:09.496904 open("/sys/devices/system/cpu/online", O_RDONLY|O_CLOEXEC) = 8
11:28:09.497140 read(8, "0-31\n", 8192) = 5
11:28:09.497256 close(8) = 0
11:28:18.154432 munmap(0x7f174c672000, 60563456) = 0
11:28:18.158661 munmap(0x7f17539f5000, 60559360) = 0
11:28:18.163143 stat("/opt/passenger-5.3.3-1.el7.cloudlinux", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
11:28:20.130114 brk(NULL) = 0x4ccbd000
11:28:20.130266 brk(NULL) = 0x4ccbd000
11:28:20.130344 brk(0x275d1000) = 0x275d1000
11:28:20.172041 brk(NULL) = 0x275d1000
11:28:20.618636 open("/sys/devices/system/cpu/online", O_RDONLY|O_CLOEXEC) = 8
11:28:20.618853 read(8, "0-31\n", 8192) = 5
11:28:20.618958 close(8) = 0
11:28:21.360585 dup2(2, 2) = 2
11:28:21.360908 open("/usr/share/zoneinfo/America/New_York", O_RDONLY|O_CLOEXEC) = 8
11:28:21.361029 fstat(8, {st_mode=S_IFREG|0644, st_size=3519, ...}) = 0
11:28:21.361123 fstat(8, {st_mode=S_IFREG|0644, st_size=3519, ...}) = 0
11:28:21.361208 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f175c8a9000
11:28:21.361303 read(8, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\0"..., 4096) = 3519
11:28:21.361450 lseek(8, -2252, SEEK_CUR) = 1267
11:28:21.361534 read(8, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\5\0\0\0\5\0\0\0\0"..., 4096) = 2252
11:28:21.361626 close(8) = 0
11:28:21.361709 munmap(0x7f175c8a9000, 4096) = 0
STR4----->11:28:21.361923 write(2, "[ D 2018-12-20 11:28:21.3608 111"..., 112) = 112
11:28:23.130430 open("/sys/devices/system/cpu/online", O_RDONLY|O_CLOEXEC) = 8
11:28:23.130625 read(8, "0-31\n", 8192) = 5
11:28:23.130726 close(8) = 0
STR4 is the place where appears:
[ D 2018-12-20 11:28:21.3608 111056/T1 age/Sha/Fun/Initialization.cpp:509 ]: Starting agent: Passenger watchdog
The same behavior has Passenger Core startup because read from STR3 runs from https://github.com/phusion/passenger/blob/23f2d842b12e259820a5971ad061654268bda694/src/agent/Shared/Fundamentals/Initialization.cpp#L589
And as I can see initializeAgent used for both Passenger WatchDog and Passenger Core.
Is this part will be reworked in future Passenger versions?
If you need more information on this issue, please feel free to contact me. Thanks!
Coming back to this, I created 1999 VirtualHosts all running a rails app, and started Apache, and by the time I attach gdb (maybe 2 seconds?), Passenger startup is done. Can anyone verify if this is still happening on your end in Passenger 6.0.2?
Tested again aginst Passenger 6.0.4 with the same results. We are currently running 7000+ virtual hosts and any version above 5.1.12 has an over a minute startup time as opposed to seconds.
I can confirm that Passenger 6.0.15 still couldn't cope with 2000+ (or so) virtual hosts.
Currently I'm struggling to upgrade from 4.0.60 to 6.0.15, but that's becoming a problem since I have 20000 virtual hosts (a typical workload for that kind of servers), and Passenger takes around 15 minutes to start.
Since Passenger is taking a long time to start, can you please:
ensure the debug symbols are installed (apt:passenger-dbg, rpm: passenger-debuginfo)
start passenger
attach a debugger to the passenger process using the most CPU while it's slow
print a backtrace of all threads
report it here
These are some traces of Passenger processes at the startup. @CamJN
Watchdog process:
Id Target Id Frame
* 1 Thread 0x7f4d142049c0 (LWP 31671) "PassengerAgent" 0x00007f4d14d6d956 in ?? () from target:/lib/x86_64-linux-gnu/libc.so.6
#0 0x00007f4d14d6d956 in ?? () from target:/lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f4d14d6fc5b in ?? () from target:/lib/x86_64-linux-gnu/libc.so.6
#2 0x00007f4d14d712e2 in malloc () from target:/lib/x86_64-linux-gnu/libc.so.6
#3 0x00007f4d150a9a0c in operator new(unsigned long) () from target:/lib/x86_64-linux-gnu/libstdc++.so.6
#4 0x000055bf4cef7139 in Passenger::StringKeyTable<Passenger::ConfigKit::Store::Entry, Passenger::SKT_DisableMoveSupport>::init (this=0x7ffef0a595e8, initialSize=16, initialStorageSize=240)
at src/cxx_supportlib/DataStructures/StringKeyTable.h:330
#5 0x000055bf4cef5b62 in Passenger::StringKeyTable<Passenger::ConfigKit::Store::Entry, Passenger::SKT_DisableMoveSupport>::StringKeyTable (this=0x7ffef0a595e8, initialSize=16, initialStorageSize=240)
at src/cxx_supportlib/DataStructures/StringKeyTable.h:299
#6 0x000055bf4ceefcf0 in Passenger::ConfigKit::Store::Store (this=0x7ffef0a595e0, _schema=...) at src/cxx_supportlib/ConfigKit/Store.h:290
#7 0x000055bf4cf11a82 in Passenger::ConfigKit::Store::extractDataForSubSchema (this=0x7ffef0a59760, subSchema=..., translator=...) at src/cxx_supportlib/ConfigKit/Store.h:508
#8 0x000055bf4cf11e11 in Passenger::ConfigKit::Schema::validateSubSchema(Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&, Passenger::ConfigKit::Schema const*, Passenger::ConfigKit::Translator const*, boost::function<void (Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&)> const&) (store=..., errors=..., subschema=0x55bf4e0315e0, translator=0x55bf4e031bd0, origValidator=...) at src/cxx_supportlib/ConfigKit/Store.h:662
#9 0x000055bf4cf89301 in boost::_bi::list5<boost::arg<1>, boost::arg<2>, boost::_bi::value<Passenger::ConfigKit::Schema const*>, boost::_bi::value<Passenger::ConfigKit::Translator const*>, boost::_bi::value<boost::function<void (Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&)> > >::operator()<void (*)(Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&, Passenger::ConfigKit::Schema const*, Passenger::ConfigKit::Translator const*, boost::function<void (Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&)> const&), boost::_bi::rrlist2<Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&> >(boost::_bi::type<void>, void (*&)(Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&, Passenger::ConfigKit::Schema const*, Passenger::ConfigKit::Translator const*, boost::function<void (Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&)> const&), boost::_bi::rrlist2<Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&>&, int) (this=0x55bf4e05b8b8,
f=@0x55bf4e05b8b0: 0x55bf4cf11dac <Passenger::ConfigKit::Schema::validateSubSchema(Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&, Passenger::ConfigKit::Schema const*, Passenger::ConfigKit::Translator const*, boost::function<void (Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&)> const&)>, a=...) at src/cxx_supportlib/vendor-modified/boost/bind/bind.hpp:510
#10 0x000055bf4cf7e9fd in boost::_bi::bind_t<void, void (*)(Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&, Passenger::ConfigKit::Schema const*, Passenger::ConfigKit::Translator const*, boost::function<void (Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&)> const&), boost::_bi::list5<boost::arg<1>, boost::arg<2>, boost::_bi::value<Passenger::ConfigKit::Schema const*>, boost::_bi::value<Passenger::ConfigKit::Translator const*>, boost::_bi::value<boost::function<void (Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&)> > > >::operator()<Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&>(Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&) (this=0x55bf4e05b8b0, a1=..., a2=...)
at src/cxx_supportlib/vendor-modified/boost/bind/bind.hpp:1297
#11 0x000055bf4cf71fa9 in boost::detail::function::void_function_obj_invoker2<boost::_bi::bind_t<void, void (*)(Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&, Passenger::ConfigKit::Schema const*, Passenger::ConfigKit::Translator const*, boost::function<void (Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&)> const&), boost::_bi::list5<boost::arg<1>, boost::arg<2>, boost::_bi::value<Passenger::ConfigKit::Schema const*>, boost::_bi::value<Passenger::ConfigKit::Translator const*>, boost::_bi::value<boost::function<void (Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&)> > > >, void, Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&>::invoke(boost::detail::function::function_buffer&, Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&) (function_obj_ptr=..., a0=..., a1=...) at src/cxx_supportlib/vendor-modified/boost/function/function_template.hpp:158
#12 0x000055bf4ce8c81f in boost::function2<void, Passenger::ConfigKit::Store const&, std::vector<Passenger::ConfigKit::Error, std::allocator<Passenger::ConfigKit::Error> >&>::operator() (a1=..., a0=...,
this=0x55bf4e0523e0) at src/cxx_supportlib/vendor-modified/boost/function/function_template.hpp:763
#13 Passenger::ConfigKit::Store::applyCustomValidators (this=this@entry=0x55bf4e033380, updates=..., errors=...) at src/cxx_supportlib/ConfigKit/Store.h:180
#14 0x000055bf4ce8ecb6 in Passenger::ConfigKit::Store::previewUpdate (this=this@entry=0x55bf4e033380, updates=..., errors=..., options=...) at src/cxx_supportlib/ConfigKit/Store.h:453
#15 0x000055bf4ce8fafd in Passenger::ConfigKit::Store::update (this=0x55bf4e033380, updates=..., errors=...) at src/cxx_supportlib/ConfigKit/Store.h:486
#16 0x000055bf4cee5728 in Passenger::Agent::Fundamentals::parseAndCommitConfig (config=..., jsonData=...) at src/agent/Shared/Fundamentals/Initialization.cpp:427
#17 0x000055bf4cee5b19 in Passenger::Agent::Fundamentals::readConfigFromFd (fd=3, config=...) at src/agent/Shared/Fundamentals/Initialization.cpp:446
#18 0x000055bf4cee6c1e in Passenger::Agent::Fundamentals::initializeAgent (argc=3, argv=0x7ffef0a5b660, processName=0x55bf4d276a43 "Passenger watchdog", config=..., loggingKitTranslator=...,
optionParser=0x55bf4cf0397c <parseOptions(int, char const**, Passenger::ConfigKit::Store&)>, loggingKitPreInitFunc=@0x7ffef0a5b670: 0x0, argStartIndex=2)
at src/agent/Shared/Fundamentals/Initialization.cpp:589
#19 0x000055bf4cf058b8 in initializeBareEssentials (argc=3, argv=0x7ffef0a5bea8, wo=...) at src/agent/Watchdog/WatchdogMain.cpp:883
#20 0x000055bf4cf0bfd6 in watchdogMain (argc=3, argv=0x7ffef0a5bea8) at src/agent/Watchdog/WatchdogMain.cpp:1376
#21 0x000055bf4cee4cc2 in dispatchSubcommand (argc=3, argv=0x7ffef0a5bea8) at src/agent/AgentMain.cpp:84
#22 0x000055bf4cee4eb0 in main (argc=3, argv=0x7ffef0a5bea8) at src/agent/AgentMain.cpp:109
Core process:
Id Target Id Frame
* 1 Thread 0x7efe4d6a39c0 (LWP 35339) "PassengerAgent" 0x00007efe4e2859db in brk () from target:/lib/x86_64-linux-gnu/libc.so.6
#0 0x00007efe4e2859db in brk () from target:/lib/x86_64-linux-gnu/libc.so.6
#1 0x00007efe4e285aa4 in sbrk () from target:/lib/x86_64-linux-gnu/libc.so.6
#2 0x00007efe4e20d646 in __default_morecore () from target:/lib/x86_64-linux-gnu/libc.so.6
#3 0x00007efe4e20d6e4 in ?? () from target:/lib/x86_64-linux-gnu/libc.so.6
#4 0x00007efe4e20df49 in ?? () from target:/lib/x86_64-linux-gnu/libc.so.6
#5 0x00007efe4e2104d3 in free () from target:/lib/x86_64-linux-gnu/libc.so.6
#6 0x000055fe362cab35 in Passenger::StringKeyTable<Passenger::ConfigKit::Store::Entry, Passenger::SKT_DisableMoveSupport>::~StringKeyTable (this=0x7ffc324c5288, __in_chrg=<optimized out>)
at src/cxx_supportlib/DataStructures/StringKeyTable.h:307
#7 0x000055fe362cd832 in Passenger::ConfigKit::Store::~Store (this=0x7ffc324c5280, __in_chrg=<optimized out>) at src/cxx_supportlib/ConfigKit/Store.h:58
#8 Passenger::ConfigKit::Store::applyCustomValidators (this=this@entry=0x55fe3f828088, updates=..., errors=...) at src/cxx_supportlib/ConfigKit/Store.h:182
#9 0x000055fe362cfcb6 in Passenger::ConfigKit::Store::previewUpdate (this=this@entry=0x55fe3f828088, updates=..., errors=..., options=...) at src/cxx_supportlib/ConfigKit/Store.h:453
#10 0x000055fe362d0afd in Passenger::ConfigKit::Store::update (this=this@entry=0x55fe3f828088, updates=..., errors=...) at src/cxx_supportlib/ConfigKit/Store.h:486
#11 0x000055fe362bf12a in Passenger::ConfigKit::Store::Store (translator=..., initialValues=..., _schema=..., this=0x55fe3f828088) at src/cxx_supportlib/ConfigKit/Store.h:314
#12 Passenger::LoggingKit::Context::Context (this=0x55fe3f828000, initialConfig=..., translator=...) at src/cxx_supportlib/LoggingKit/Implementation.cpp:529
#13 0x000055fe362bf86a in Passenger::LoggingKit::initialize (initialConfig=..., translator=...) at src/cxx_supportlib/LoggingKit/Implementation.cpp:75
#14 0x000055fe36326d71 in Passenger::Agent::Fundamentals::initializeLoggingKit (processName=0x55fe366db473 "Passenger core", config=..., loggingKitTranslator=...,
loggingKitPreInitFunc=@0x7ffc324c77e0: 0x55fe36417df2 <loggingKitPreInitFunc(Passenger::Json::Value&)>) at src/agent/Shared/Fundamentals/Initialization.cpp:491
#15 0x000055fe36327e76 in Passenger::Agent::Fundamentals::initializeAgent (argc=3, argv=0x7ffc324c77a0, processName=0x55fe366db473 "Passenger core", config=..., loggingKitTranslator=...,
optionParser=0x55fe36417947 <parseOptions(int, char const**, Passenger::ConfigKit::Store&)>, loggingKitPreInitFunc=@0x7ffc324c77e0: 0x55fe36417df2 <loggingKitPreInitFunc(Passenger::Json::Value&)>,
argStartIndex=2) at src/agent/Shared/Fundamentals/Initialization.cpp:609
#16 0x000055fe36417f55 in coreMain (argc=3, argv=0x7ffc324c79a8) at src/agent/Core/CoreMain.cpp:1426
#17 0x000055fe36325cfb in dispatchSubcommand (argc=3, argv=0x7ffc324c79a8) at src/agent/AgentMain.cpp:86
#18 0x000055fe36325eb0 in main (argc=3, argv=0x7ffc324c79a8) at src/agent/AgentMain.cpp:109
The same core process after a few moments:
Id Target Id Frame
* 1 Thread 0x7efe4d6a39c0 (LWP 35339) "PassengerAgent" 0x000055fe362f6362 in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Js
on::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe550ff060, this=<optimized out>)
at /usr/include/c++/11/bits/stl_tree.h:787
2 Thread 0x7efe4d67d640 (LWP 36567) "PassengerAgent" 0x00007efe4e250868 in clock_nanosleep () from target:/lib/x86_64-linux-gnu/libc.so.6
3 Thread 0x7efe4d63c640 (LWP 36568) "PassengerAgent" 0x00007efe4e1fc197 in ?? () from target:/lib/x86_64-linux-gnu/libc.so.6
Thread 3 (Thread 0x7efe4d63c640 (LWP 36568) "PassengerAgent"):
#0 0x00007efe4e1fc197 in ?? () from target:/lib/x86_64-linux-gnu/libc.so.6
#1 0x00007efe4e1fef1b in pthread_cond_timedwait () from target:/lib/x86_64-linux-gnu/libc.so.6
#2 0x000055fe3641a1fd in boost::posix::pthread_cond_timedwait (t=0x7efe4d63b1d0, m=0x55fe4e653530, c=0x55fe4e653558) at src/cxx_supportlib/vendor-modified/boost/thread/pthread/pthread_helpers.hpp:123
#3 boost::condition_variable::do_wait_until (this=0x55fe4e653530, m=..., timeout=...) at src/cxx_supportlib/vendor-modified/boost/thread/pthread/condition_variable.hpp:122
#4 0x000055fe3649722c in boost::condition_variable::timed_wait<boost::date_time::subsecond_duration<boost::posix_time::time_duration, 1000000l> > (this=0x55fe4e653530, m=..., wait_duration=...) at src/cxx_supportlib/vendor-modified/boost/thread/pthread/condition_variable_fwd.hpp:156
#5 0x000055fe365b50ee in Passenger::ApplicationPool2::Pool::garbageCollect (self=...) at src/agent/Core/ApplicationPool/Pool/GarbageCollection.cpp:64
#6 0x000055fe366186cb in boost::_bi::list1<boost::_bi::value<boost::shared_ptr<Passenger::ApplicationPool2::Pool> > >::operator()<void (*)(boost::shared_ptr<Passenger::ApplicationPool2::Pool>), boost::_bi::list0> (this=0x55fe4e6575f0, f=@0x55fe4e6575e8: 0x55fe365b4f42 <Passenger::ApplicationPool2::Pool::garbageCollec
t(boost::shared_ptr<Passenger::ApplicationPool2::Pool>)>, a=...) at src/cxx_supportlib/vendor-modified/boost/bind/bind.hpp:238
#7 0x000055fe36612766 in boost::_bi::bind_t<void, void (*)(boost::shared_ptr<Passenger::ApplicationPool2::Pool>), boost::_bi::list1<boost::_bi::value<boost::shared_ptr<Passenger::ApplicationPool2::Pool> > > >::operator() (this=0x55fe4e6575e8) at src/cxx_supportlib/vendor-modified/boost/bind/bind.hpp:1273
#8 0x000055fe3660c8ae in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, void (*)(boost::shared_ptr<Passenger::ApplicationPool2::Pool>), boost::_bi::list1<boost::_bi::value<boost::shared_ptr<Passenger::ApplicationPool2::Pool> > > >, void>::invoke (function_obj_ptr=...) at src/cxx_supportl
ib/vendor-modified/boost/function/function_template.hpp:158
#9 0x000055fe362e145f in boost::function0<void>::operator() (this=0x55fe4e6575e0) at src/cxx_supportlib/vendor-modified/boost/function/function_template.hpp:763
#10 0x000055fe3641fc7e in oxt::dynamic_thread_group::thread_main (this=0x55fe4e653218, data=0x55fe4e6575e0) at src/cxx_supportlib/oxt/dynamic_thread_group.hpp:131
#11 0x000055fe36534b9d in boost::_mfi::mf1<void, oxt::dynamic_thread_group, oxt::dynamic_thread_group::thread_func_data*>::operator() (this=0x7efe40000b70, p=0x55fe4e653218, a1=0x55fe4e6575e0) at src/cxx_supportlib/vendor-modified/boost/bind/mem_fn_template.hpp:165
#12 0x000055fe36520580 in boost::_bi::list2<boost::_bi::value<oxt::dynamic_thread_group*>, boost::_bi::value<oxt::dynamic_thread_group::thread_func_data*> >::operator()<boost::_mfi::mf1<void, oxt::dynamic_thread_group, oxt::dynamic_thread_group::thread_func_data*>, boost::_bi::list0> (this=0x7efe40000b80, f=..., a=...
) at src/cxx_supportlib/vendor-modified/boost/bind/bind.hpp:298
#13 0x000055fe3650d308 in boost::_bi::bind_t<void, boost::_mfi::mf1<void, oxt::dynamic_thread_group, oxt::dynamic_thread_group::thread_func_data*>, boost::_bi::list2<boost::_bi::value<oxt::dynamic_thread_group*>, boost::_bi::value<oxt::dynamic_thread_group::thread_func_data*> > >::operator() (this=0x7efe40000b70) at s
rc/cxx_supportlib/vendor-modified/boost/bind/bind.hpp:1273
#14 0x000055fe364f3992 in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, oxt::dynamic_thread_group, oxt::dynamic_thread_group::thread_func_data*>, boost::_bi::list2<boost::_bi::value<oxt::dynamic_thread_group*>, boost::_bi::value<oxt::dynamic_thread_group::thread_fu
nc_data*> > >, void>::invoke (function_obj_ptr=...) at src/cxx_supportlib/vendor-modified/boost/function/function_template.hpp:158
#15 0x000055fe362e145f in boost::function0<void>::operator() (this=0x7efe4d63bb40) at src/cxx_supportlib/vendor-modified/boost/function/function_template.hpp:763
#16 0x000055fe3666e084 in oxt::thread::thread_main(boost::function<void ()>, boost::shared_ptr<oxt::thread_local_context>) (func=..., ctx=...) at src/cxx_supportlib/oxt/implementation.cpp:416
#17 0x000055fe362c4279 in boost::_bi::list2<boost::_bi::value<boost::function<void ()> >, boost::_bi::value<boost::shared_ptr<oxt::thread_local_context> > >::operator()<void (*)(boost::function<void ()>, boost::shared_ptr<oxt::thread_local_context>), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(boost::function
<void ()>, boost::shared_ptr<oxt::thread_local_context>), boost::_bi::list0&, int) (a=<synthetic pointer>..., f=<optimized out>, this=<optimized out>) at src/cxx_supportlib/vendor-modified/boost/bind/bind.hpp:298
#18 boost::_bi::bind_t<void, void (*)(boost::function<void ()>, boost::shared_ptr<oxt::thread_local_context>), boost::_bi::list2<boost::_bi::value<boost::function<void ()> >, boost::_bi::value<boost::shared_ptr<oxt::thread_local_context> > > >::operator()() (this=<optimized out>) at src/cxx_supportlib/vendor-modified/
boost/bind/bind.hpp:1273
#19 boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(boost::function<void ()>, boost::shared_ptr<oxt::thread_local_context>), boost::_bi::list2<boost::_bi::value<boost::function<void ()> >, boost::_bi::value<boost::shared_ptr<oxt::thread_local_context> > > > >::run() (this=<optimized out>) at src/cxx_suppo
rtlib/vendor-modified/boost/thread/detail/thread.hpp:120
#20 0x000055fe36661856 in boost::(anonymous namespace)::thread_proxy (param=0x55fe4e657cb0) at src/cxx_supportlib/vendor-modified/boost/libs/thread/src/pthread/thread.cpp:179
#21 0x00007efe4e1ffb43 in ?? () from target:/lib/x86_64-linux-gnu/libc.so.6
#22 0x00007efe4e291a00 in ?? () from target:/lib/x86_64-linux-gnu/libc.so.6
Thread 2 (Thread 0x7efe4d67d640 (LWP 36567) "PassengerAgent"):
#0 0x00007efe4e250868 in clock_nanosleep () from target:/lib/x86_64-linux-gnu/libc.so.6
#1 0x00007efe4e2556e7 in nanosleep () from target:/lib/x86_64-linux-gnu/libc.so.6
#2 0x000055fe36674d5d in oxt::syscalls::nanosleep (req=0x7efe4d67c1c0, rem=0x0) at src/cxx_supportlib/oxt/system_calls.cpp:648
#3 0x000055fe36674cd2 in oxt::syscalls::usleep (usec=4999481) at src/cxx_supportlib/oxt/system_calls.cpp:630
#4 0x000055fe365b3561 in Passenger::ApplicationPool2::Pool::collectAnalytics (self=...) at src/agent/Core/ApplicationPool/Pool/AnalyticsCollection.cpp:71
#5 0x000055fe366186cb in boost::_bi::list1<boost::_bi::value<boost::shared_ptr<Passenger::ApplicationPool2::Pool> > >::operator()<void (*)(boost::shared_ptr<Passenger::ApplicationPool2::Pool>), boost::_bi::list0> (this=0x55fe4e6575c0, f=@0x55fe4e6575b8: 0x55fe365b3310 <Passenger::ApplicationPool2::Pool::collectAnalyt
ics(boost::shared_ptr<Passenger::ApplicationPool2::Pool>)>, a=...) at src/cxx_supportlib/vendor-modified/boost/bind/bind.hpp:238
#6 0x000055fe36612766 in boost::_bi::bind_t<void, void (*)(boost::shared_ptr<Passenger::ApplicationPool2::Pool>), boost::_bi::list1<boost::_bi::value<boost::shared_ptr<Passenger::ApplicationPool2::Pool> > > >::operator() (this=0x55fe4e6575b8) at src/cxx_supportlib/vendor-modified/boost/bind/bind.hpp:1273
#7 0x000055fe3660c8ae in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, void (*)(boost::shared_ptr<Passenger::ApplicationPool2::Pool>), boost::_bi::list1<boost::_bi::value<boost::shared_ptr<Passenger::ApplicationPool2::Pool> > > >, void>::invoke (function_obj_ptr=...) at src/cxx_supportl
ib/vendor-modified/boost/function/function_template.hpp:158
#8 0x000055fe362e145f in boost::function0<void>::operator() (this=0x55fe4e6575b0) at src/cxx_supportlib/vendor-modified/boost/function/function_template.hpp:763
#9 0x000055fe3641fc7e in oxt::dynamic_thread_group::thread_main (this=0x55fe4e653218, data=0x55fe4e6575b0) at src/cxx_supportlib/oxt/dynamic_thread_group.hpp:131
#10 0x000055fe36534b9d in boost::_mfi::mf1<void, oxt::dynamic_thread_group, oxt::dynamic_thread_group::thread_func_data*>::operator() (this=0x7efe48000b70, p=0x55fe4e653218, a1=0x55fe4e6575b0) at src/cxx_supportlib/vendor-modified/boost/bind/mem_fn_template.hpp:165
#11 0x000055fe36520580 in boost::_bi::list2<boost::_bi::value<oxt::dynamic_thread_group*>, boost::_bi::value<oxt::dynamic_thread_group::thread_func_data*> >::operator()<boost::_mfi::mf1<void, oxt::dynamic_thread_group, oxt::dynamic_thread_group::thread_func_data*>, boost::_bi::list0> (this=0x7efe48000b80, f=..., a=...
) at src/cxx_supportlib/vendor-modified/boost/bind/bind.hpp:298
#12 0x000055fe3650d308 in boost::_bi::bind_t<void, boost::_mfi::mf1<void, oxt::dynamic_thread_group, oxt::dynamic_thread_group::thread_func_data*>, boost::_bi::list2<boost::_bi::value<oxt::dynamic_thread_group*>, boost::_bi::value<oxt::dynamic_thread_group::thread_func_data*> > >::operator() (this=0x7efe48000b70) at s
rc/cxx_supportlib/vendor-modified/boost/bind/bind.hpp:1273
#13 0x000055fe364f3992 in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, oxt::dynamic_thread_group, oxt::dynamic_thread_group::thread_func_data*>, boost::_bi::list2<boost::_bi::value<oxt::dynamic_thread_group*>, boost::_bi::value<oxt::dynamic_thread_group::thread_fu
nc_data*> > >, void>::invoke (function_obj_ptr=...) at src/cxx_supportlib/vendor-modified/boost/function/function_template.hpp:158
#14 0x000055fe362e145f in boost::function0<void>::operator() (this=0x7efe4d67cb40) at src/cxx_supportlib/vendor-modified/boost/function/function_template.hpp:763
#15 0x000055fe3666e084 in oxt::thread::thread_main(boost::function<void ()>, boost::shared_ptr<oxt::thread_local_context>) (func=..., ctx=...) at src/cxx_supportlib/oxt/implementation.cpp:416
#16 0x000055fe362c4279 in boost::_bi::list2<boost::_bi::value<boost::function<void ()> >, boost::_bi::value<boost::shared_ptr<oxt::thread_local_context> > >::operator()<void (*)(boost::function<void ()>, boost::shared_ptr<oxt::thread_local_context>), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(boost::function
<void ()>, boost::shared_ptr<oxt::thread_local_context>), boost::_bi::list0&, int) (a=<synthetic pointer>..., f=<optimized out>, this=<optimized out>) at src/cxx_supportlib/vendor-modified/boost/bind/bind.hpp:298
#17 boost::_bi::bind_t<void, void (*)(boost::function<void ()>, boost::shared_ptr<oxt::thread_local_context>), boost::_bi::list2<boost::_bi::value<boost::function<void ()> >, boost::_bi::value<boost::shared_ptr<oxt::thread_local_context> > > >::operator()() (this=<optimized out>) at src/cxx_supportlib/vendor-modified/boost/bind/bind.hpp:1273
#18 boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(boost::function<void ()>, boost::shared_ptr<oxt::thread_local_context>), boost::_bi::list2<boost::_bi::value<boost::function<void ()> >, boost::_bi::value<boost::shared_ptr<oxt::thread_local_context> > > > >::run() (this=<optimized out>) at src/cxx_supportlib/vendor-modified/boost/thread/detail/thread.hpp:120
#19 0x000055fe36661856 in boost::(anonymous namespace)::thread_proxy (param=0x55fe4e6577f0) at src/cxx_supportlib/vendor-modified/boost/libs/thread/src/pthread/thread.cpp:179
#20 0x00007efe4e1ffb43 in ?? () from target:/lib/x86_64-linux-gnu/libc.so.6
#21 0x00007efe4e291a00 in ?? () from target:/lib/x86_64-linux-gnu/libc.so.6
Thread 1 (Thread 0x7efe4d6a39c0 (LWP 35339) "PassengerAgent"):
#0 0x000055fe362f6362 in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passeng
er::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe550ff060, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:787
#1 0x000055fe362f636a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passeng
er::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe550feed0, this=0x55fe550fee90) at /usr/include/c++/11/bits/stl_tree.h:787
#2 0x000055fe362f629a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passeng
er::Json::Value::CZString const, Passenger::Json::Value> > >::~_Rb_tree (this=0x55fe550fee90, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:984
#3 std::map<Passenger::Json::Value::CZString, Passenger::Json::Value, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::~map (this=0x55fe550fee90, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_map.h:302
#4 Passenger::Json::Value::releasePayload (this=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:3445
#5 0x000055fe362f62fc in Passenger::Json::Value::~Value (this=0x55fe550fee50, __in_chrg=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:2865
#6 0x000055fe362f6377 in std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>::~pair (this=0x55fe550fee40, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_pair.h:211
#7 __gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::destroy<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > (__p=0x55fe550fee40, this=<optimized out>) at /usr/include/c++/11/ext/new_allocator.h:168
#8 std::allocator_traits<std::allocator<std::_Rb_tree_node<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > > >::destroy<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > (__p=0x55fe550fee40, __a=...) at /usr/include/c++/11/bits/alloc_traits.h:535
#9 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_destroy_node (this=<optimized out>, __p=0x55fe550fee20) at /usr/include/c++/11/bits/stl_tree.h:623
#10 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_drop_node (__p=0x55fe550fee20, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:631
#11 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZStr
ing const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe550fee20, this=0x55fe550fede0) at /usr/include/c++/11/bits/stl_tree.h:1891
#12 0x000055fe362f629a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::~_Rb_tree (this=0x55fe550fede0, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:984
#13 std::map<Passenger::Json::Value::CZString, Passenger::Json::Value, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::~map (this=0x55fe550fede0, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_map.h:302
#14 Passenger::Json::Value::releasePayload (this=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:3445
#15 0x000055fe362f62fc in Passenger::Json::Value::~Value (this=0x55fe550fed80, __in_chrg=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:2865
#16 0x000055fe362f6377 in std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>::~pair (this=0x55fe550fed70, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_pair.h:211
#17 __gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::destroy<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > (__p=0x55fe550fed70, this=<optimized out>) at /usr/include/c++/11/ext/new_allocator.h:168
#18 std::allocator_traits<std::allocator<std::_Rb_tree_node<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > > >::destroy<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > (__p=0x55fe550fed70, __a=...) at /usr/include/c++/11/bits/alloc_traits.h:535
#19 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_destroy_node (this=<optimized out>, __p=0x55fe550fed50) at /usr/include/c++/11/bits/stl_tree.h:623
#20 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_drop_node (__p=0x55fe550fed50, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:631
#21 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe550fed50, this=0x55fe550fed10) at /usr/include/c++/11/bits/stl_tree.h:1891
#22 0x000055fe362f629a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::~_Rb_tree (this=0x55fe550fed10, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:984
#23 std::map<Passenger::Json::Value::CZString, Passenger::Json::Value, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::~map (this=0x55fe550fed10, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_map.h:302
#24 Passenger::Json::Value::releasePayload (this=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:3445
#25 0x000055fe362f62fc in Passenger::Json::Value::~Value (this=0x55fe550fecb0, __in_chrg=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:2865
#26 0x000055fe362f6377 in std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>::~pair (this=0x55fe550feca0, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_pair.h:211
#27 __gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::destroy<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > (__p=0x55fe550feca0, this=<optimized out>) at /usr/include/c++/11/ext/new_allocator.h:168
#28 std::allocator_traits<std::allocator<std::_Rb_tree_node<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > > >::destroy<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > (__p=0x55fe550feca0, __a=...) at /usr/include/c++/11/bits/alloc_traits.h:535
#29 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_destroy_node (this=<optimized out>, __p=0x55fe550fec80) at /usr/include/c++/11/bits/stl_tree.h:623
#30 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_drop_node (__p=0x55fe550fec80, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:631
#31 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe550fec80, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:1891
#32 0x000055fe362f636a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe550fd610, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:787
#33 0x000055fe362f636a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe550fd190, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:787
#34 0x000055fe362f636a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe550fcd20, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:787
#35 0x000055fe362f636a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passeng
er::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe550fc860, this=0x55fe550fc820) at /usr/include/c++/11/bits/stl_tree.h:787
#36 0x000055fe362f629a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::~_Rb_tree (this=0x55fe550fc820, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:984
#37 std::map<Passenger::Json::Value::CZString, Passenger::Json::Value, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::~map (this=0x55fe550fc820, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_map.h:302
#38 Passenger::Json::Value::releasePayload (this=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:3445
#39 0x000055fe362f62fc in Passenger::Json::Value::~Value (this=0x55fe550fc7c0, __in_chrg=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:2865
#40 0x000055fe362f6377 in std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>::~pair (this=0x55fe550fc7b0, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_pair.h:211
#41 __gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::destroy<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > (__p=0x55fe550fc7b0, this=<optimized out>) at /usr/include/c++/11/ext/new_allocator.h:168
#42 std::allocator_traits<std::allocator<std::_Rb_tree_node<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > > >::destroy<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > (__p=0x55fe550fc7b0, __a=...) at /usr/include/c++/11/bits/alloc_traits.h:535
#43 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_destroy_node (this=<optimized out>, __p=0x55fe550fc790) at /usr/include/c++/11/bits/stl_tree.h:623
#44 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_drop_node (__p=0x55fe550fc790, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:631
#45 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe550fc790, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:1891
#46 0x000055fe362f636a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe550fc6c0, this=0x55fe550fc680) at /usr/include/c++/11/bits/stl_tree.h:787
#47 0x000055fe362f629a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passeng
er::Json::Value::CZString const, Passenger::Json::Value> > >::~_Rb_tree (this=0x55fe550fc680, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:984
#48 std::map<Passenger::Json::Value::CZString, Passenger::Json::Value, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::~map (this=0x55fe550fc680, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_map.h:302
#49 Passenger::Json::Value::releasePayload (this=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:3445
#50 0x000055fe362f62fc in Passenger::Json::Value::~Value (this=0x55fe550fc600, __in_chrg=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:2865
#51 0x000055fe362f6377 in std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>::~pair (this=0x55fe550fc5f0, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_pair.h:211
#52 __gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::destroy<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > (__p=0x55fe550fc5f0, this=<optimized out>) at /usr/include/c++/11/ext/new_allocator.h:168
#53 std::allocator_traits<std::allocator<std::_Rb_tree_node<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > > >::destroy<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > (__p=0x55fe550fc5f0, __a=...) at /usr/include/c++/11/bits/alloc_traits.h:535
#54 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_destroy_node (this=<optimized out>, __p=0x55fe550fc5d0) at /usr/include/c++/11/bits/stl_tree.h:623
#55 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_drop_node (__p=0x55fe550fc5d0, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:631
#56 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe550fc5d0, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:1891
#57 0x000055fe362f636a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe53c459e0, this=0x55fe5ddabde0) at /usr/include/c++/11/bits/stl_tree.h:787
#58 0x000055fe362f629a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::~_Rb_tree (this=0x55fe5ddabde0, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:984
#59 std::map<Passenger::Json::Value::CZString, Passenger::Json::Value, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::~map (this=0x55fe5ddabde0, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_map.h:302
#60 Passenger::Json::Value::releasePayload (this=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:3445
#61 0x000055fe362f62fc in Passenger::Json::Value::~Value (this=0x55fe5ddabd70, __in_chrg=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:2865
#62 0x000055fe362f6377 in std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>::~pair (this=0x55fe5ddabd60, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_pair.h:211
#63 __gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::destroy<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > (__p=0x55fe5ddabd60, this=<optimized out>) at /usr/include/c++/11/ext/new_allocator.h:168
#64 std::allocator_traits<std::allocator<std::_Rb_tree_node<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > > >::destroy<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > (__p=0x55fe5ddabd60, __a=...) at /usr/include/c++/11/bits/alloc_traits.h:535
#65 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_destroy_node (this=<optimized out>, __p=0x55fe5ddabd40) at /usr/include/c++/11/bits/stl_tree.h:623
#66 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_drop_node (__p=0x55fe5ddabd40, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:631
#67 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe5ddabd40, this=0x55fe5dd9ad50) at /usr/include/c++/11/bits/stl_tree.h:1891
#68 0x000055fe362f629a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::~_Rb_tree (this=0x55fe5dd9ad50, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:984
#69 std::map<Passenger::Json::Value::CZString, Passenger::Json::Value, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::~map (this=0x55fe5dd9ad50, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_map.h:302
#70 Passenger::Json::Value::releasePayload (this=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:3445
#71 0x000055fe362f62fc in Passenger::Json::Value::~Value (this=0x55fe5dd9b690, __in_chrg=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:2865
#72 0x000055fe362f6377 in std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>::~pair (this=0x55fe5dd9b680, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_pair.h:211
#73 __gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::destroy<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > (__p=0x55fe5dd9b680, this=<optimized out>) at /usr/include/c++/11/ext/new_allocator.h:168
#74 std::allocator_traits<std::allocator<std::_Rb_tree_node<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > > >::destroy<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > (__p=0x55fe5dd9b680, __a=...) at /usr/include/c++/11/bits/alloc_traits.h:535
#75 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_destroy_node (this=<optimized out>, __p=0x55fe5dd9b660) at /usr/include/c++/11/bits/stl_tree.h:623
#76 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_drop_node (__p=0x55fe5dd9b660, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:631
#77 std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe5dd9b660, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:1891
#78 0x000055fe362f636a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe565e7580, this=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:787
#79 0x000055fe362f636a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::_M_erase (__x=0x55fe5dd9b540, this=0x55fe5dd979a0) at /usr/include/c++/11/bits/stl_tree.h:787
#80 0x000055fe362f629a in std::_Rb_tree<Passenger::Json::Value::CZString, std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value>, std::_Select1st<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> >, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::~_Rb_tree (this=0x55fe5dd979a0, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_tree.h:984
#81 std::map<Passenger::Json::Value::CZString, Passenger::Json::Value, std::less<Passenger::Json::Value::CZString>, std::allocator<std::pair<Passenger::Json::Value::CZString const, Passenger::Json::Value> > >::~map (this=0x55fe5dd979a0, __in_chrg=<optimized out>) at /usr/include/c++/11/bits/stl_map.h:302
#82 Passenger::Json::Value::releasePayload (this=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:3445
#83 0x000055fe362f62fc in Passenger::Json::Value::~Value (this=0x7ffc324c6c80, __in_chrg=<optimized out>) at src/cxx_supportlib/vendor-modified/jsoncpp/jsoncpp.cpp:2865
#84 0x000055fe3635288b in Passenger::ConfigKit::Store::Store (this=0x55fe5dd98f40, _schema=..., initialValues=..., translator=...) at src/cxx_supportlib/ConfigKit/Store.h:314
#85 0x000055fe363591a1 in Passenger::ServerKit::Context::Context (this=0x55fe5dd98f40, schema=..., initialConfig=..., translator=...) at src/cxx_supportlib/ServerKit/Context.h:68
#86 0x000055fe36411695 in initializeNonPrivilegedWorkingObjects () at src/agent/Core/CoreMain.cpp:761
#87 0x000055fe3641754b in runCore () at src/agent/Core/CoreMain.cpp:1351
#88 0x000055fe36417fcb in coreMain (argc=3, argv=0x55fe37ee5770) at src/agent/Core/CoreMain.cpp:1434
#89 0x000055fe36325cfb in dispatchSubcommand (argc=3, argv=0x7ffc324c79a8) at src/agent/AgentMain.cpp:86
#90 0x000055fe36325eb0 in main (argc=3, argv=0x7ffc324c79a8) at src/agent/AgentMain.cpp:109
Looks like it is building up and tearing down a bunch of json, and maybe StringKeyTables too. @FooBarWidget what do you think?