elasticsearch-dsl-py
elasticsearch-dsl-py copied to clipboard
Published
20 hours ago •
elastic
elastic
Searching through objects inside nested documents provides unexpected output.
I am getting unexpected result from Elasticsearch while searching though object property which are inside nested property. I am using elasticsearch-dsl python library for creating document and querying. Is this bug or I am missing something in querying and mapping parts? Below are the elasticsearch json mapping, query, unexpected result and expected result
Mapping:
{
"deal_acls": {
"type": "nested",
"properties": {
"created_at": {
"type": "date"
},
"created_by": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
}
}
},
"permission": {
"properties": {
"CRM": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"deal": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"document": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"external_deal_team": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
},
"internal_deal_team": {
"properties": {
"description": {
"properties": {
"created_by": {
"type": "keyword",
"normalizer": "lowercase"
},
"object": {
"type": "keyword",
"normalizer": "lowercase"
},
"object_id": {
"type": "keyword",
"normalizer": "lowercase"
},
"timestamp": {
"type": "date"
}
}
},
"permission": {
"properties": {
"delete": {
"type": "integer"
},
"edit": {
"type": "integer"
},
"manage": {
"type": "long"
},
"read": {
"type": "integer"
},
"write": {
"type": "integer"
}
}
}
}
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
},
"updated_at": {
"type": "date"
},
"updated_by": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
}
}
},
"user": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"status": {
"type": "keyword",
"normalizer": "lowercase"
}
}
}
}
}
}
Query:
{
"query": {
"bool": {
"must": [
{
"nested": {
"path": "deal_acls",
"query": {
"term": {
"deal_acls.permission.deal.permission.read": 1
}
}
}
},
{
"nested": {
"path": "deal_acls",
"query": {
"terms": {
"deal_acls.user.id": [
"5f7cea05-6562-4bdd-8448-19cfbe11783a"
]
}
}
}
}
]
}
}
}
Unexpected result: Since the deal permission of user with id=5f7cea05-6562-4bdd-8448-19cfbe11783a is 0, it should be returning empty hits.
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 2,
"hits": [
{
"_index": "dev.crecentric.dealvault.deals",
"_type": "_doc",
"_id": "a928838d-3422-41db-b40e-28f5c793f806",
"_score": 2,
"_source": {
"id": "a928838d-3422-41db-b40e-28f5c793f806",
"deal_acls": [
{
"user": {
"id": "5f7cea05-6562-4bdd-8448-19cfbe11783a",
"name": "testerrrrs testesssss",
"status": "active"
},
"permission": {
"deal": {
"permission": {
"edit": 0,
"read": 0,
"write": 0,
"delete": 0,
"manage": 0
},
"description": {
"object": "workspace",
"object_id": "fbc840b1-8727-4945-a070-fa1c105f9550",
"timestamp": "2022-02-10T05:46:05.140867+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"external_deal_team": {
"permission": {
"edit": 0,
"read": 1,
"write": 1,
"delete": 0,
"manage": 0
},
"description": {
"object": "workspace",
"object_id": "fbc840b1-8727-4945-a070-fa1c105f9550",
"timestamp": "2022-02-10T05:46:05.140902+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"internal_deal_team": {
"permission": {
"edit": 0,
"read": 1,
"write": 1,
"delete": 0,
"manage": 0
},
"description": {
"object": "workspace",
"object_id": "fbc840b1-8727-4945-a070-fa1c105f9550",
"timestamp": "2022-02-10T05:46:05.140910+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
}
},
"status": "active",
"created_at": "2022-02-10T05:40:15.727598+05:45",
"updated_at": "2022-02-10T05:46:05.177076+05:45"
},
{
"user": {
"id": "d78411e5-6645-4b95-a98c-db6db8748580",
"name": "Ramesh Pradhan",
"status": "active"
},
"permission": {
"CRM": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"deal": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"document": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"external_deal_team": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
},
"internal_deal_team": {
"permission": {
"edit": 1,
"read": 1,
"write": 1,
"delete": 1,
"manage": 1
},
"description": {
"object": "owner",
"object_id": "d78411e5-6645-4b95-a98c-db6db8748580",
"timestamp": "2022-02-10T05:35:41.453881+00:00",
"created_by": "d78411e5-6645-4b95-a98c-db6db8748580"
}
}
},
"status": "active",
"created_at": "2022-02-10T05:35:41.453913+05:45",
"updated_at": "2022-02-10T05:35:41.462956+05:45"
}
]
}
}
]
}
}
Expected result:
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 2,
"hits": []
}
}
This is definitely a bug and I think that I am encountering the same problem. When you filter on a nested document, results being returned are parent documents whose variants collectively match the nested filters, i.e. in an ecommerce context if I filter products by variants with price 30-40 and color red, I will get parent documents whose combined variants satisfy these properties. But what I want is parents who have a single document with both properties.
Not a bug. Just a confusing DSL spec.
must:
- nested cond1
- nested cond2
translates to
"PARENTS which have a nested document satisfying cond1 AND which have a nested document satisfying cond2"
You probably want
"PARENTS which have a nested document (satisfying cond1 AND satisfying cond2)"
which you'd write as:
nested:
- must:
- nested cond1
- nested cond2
in order to scope the whole must to the deal_acls.
