GeoHealthCheck icon indicating copy to clipboard operation
GeoHealthCheck copied to clipboard

Optimization: Reduce number of queries to DB

Open justb4 opened this issue 9 years ago • 0 comments

Using PostgreSQL as backend, logging the DB statements, there seems to be quite a number of queries just to display a single Resource page with a single WMS Resource like GET /?lang=en&resource_type=OGC%3AWMS:

LOG:  statement: BEGIN
LOG:  statement: SELECT t.oid, typarray
	FROM pg_type t JOIN pg_namespace ns
	    ON typnamespace = ns.oid
	WHERE typname = 'hstore';
	
LOG:  statement: ROLLBACK
LOG:  statement: BEGIN
LOG:  statement: select version()
LOG:  statement: select current_schema()
LOG:  statement: show transaction isolation level
LOG:  statement: SELECT CAST('test plain returns' AS VARCHAR(60)) AS anon_1
LOG:  statement: SELECT CAST('test unicode returns' AS VARCHAR(60)) AS anon_1
LOG:  statement: SELECT 'x' AS some_label
LOG:  statement: ROLLBACK
LOG:  statement: BEGIN
LOG:  statement: show standard_conforming_strings
LOG:  statement: SELECT resource.identifier AS resource_identifier, resource.resource_type AS resource_resource_type, resource.title AS resource_title, resource.url AS resource_url, resource.latitude AS resource_latitude, resource.longitude AS resource_longitude, resource.owner_identifier AS resource_owner_identifier 
	FROM resource 
	WHERE resource.resource_type = 'OGC:WMS'
LOG:  statement: SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier ORDER BY run.checked_datetime ASC 
	 LIMIT 1
LOG:  statement: SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier ORDER BY run.checked_datetime ASC 
	 LIMIT 1
LOG:  statement: SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier ORDER BY run.checked_datetime DESC 
	 LIMIT 1
LOG:  statement: SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier ORDER BY run.checked_datetime DESC 
	 LIMIT 1
LOG:  statement: SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier ORDER BY run.checked_datetime DESC 
	 LIMIT 1
LOG:  statement: SELECT count(*) AS count_1 
	FROM (SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier) AS anon_1
LOG:  statement: SELECT count(*) AS count_1 
	FROM (SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier AND run.success = true) AS anon_1
LOG:  statement: SELECT "user".user_id AS user_user_id, "user".username AS user_username, "user".password AS user_password, "user".email AS user_email, "user".role AS user_role, "user".registered_on AS user_registered_on 
	FROM "user" 
	WHERE "user".user_id = 1
LOG:  statement: SELECT count(*) AS count_1 
	FROM (SELECT resource.resource_type AS resource_resource_type 
	FROM resource) AS anon_1
LOG:  statement: SELECT resource.resource_type AS resource_resource_type, count(resource.resource_type) AS count_1 
	FROM resource GROUP BY resource.resource_type
LOG:  statement: SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier ORDER BY run.checked_datetime DESC 
	 LIMIT 1
LOG:  statement: SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier ORDER BY run.checked_datetime DESC 
	 LIMIT 1
LOG:  statement: SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier ORDER BY run.checked_datetime DESC 
	 LIMIT 1
LOG:  statement: SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier
LOG:  statement: SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier
LOG:  statement: SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier
LOG:  statement: SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier
LOG:  statement: SELECT count(*) AS count_1 
	FROM (SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier) AS anon_1
LOG:  statement: SELECT count(*) AS count_1 
	FROM (SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier AND run.success = true) AS anon_1
LOG:  statement: SELECT count(*) AS count_1 
	FROM (SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier) AS anon_1
LOG:  statement: SELECT count(*) AS count_1 
	FROM (SELECT run.identifier AS run_identifier, run.resource_identifier AS run_resource_identifier, run.checked_datetime AS run_checked_datetime, run.success AS run_success, run.response_time AS run_response_time, run.message AS run_message 
	FROM run 
	WHERE 2 = run.resource_identifier AND run.success = true) AS anon_1
127.0.0.1 - - [19/Dec/2016 21:34:41] "GET /?lang=en&resource_type=OGC%3AWMS HTTP/1.1" 200 -

This happens behind the facade of SQLAlchemy I guess. Possibly some statements in https://github.com/geopython/GeoHealthCheck/blob/master/GeoHealthCheck/models.py in combination with the templates may be optimized.

justb4 avatar Dec 19 '16 20:12 justb4