[BUG] `util:index-keys` can return stale data

[line-o]

Describe the bug

If nothing but the term frequency changes util:index-keys#5 might return stale data. The test results indicate that this happens only if the frequency changes for all terms returned by the current call and the update happens by inserting new nodes into an existing node with update insert into.

So results pages of util:index-keys#5 must be cached at some point. Any form of reindexing will invalidate this cached page and return the correct results. Either by opening the document and saving it unchanged or by calling xmldb:reindex.

Expected behavior

util:index-keys#5 to always return current data.

To Reproduce

xquery version "3.1";

module namespace tuis="http://exist-db.org/xquery/test/util-index-keys";

declare namespace test="http://exist-db.org/xquery/xqsuite";

declare variable $tuis:XML := document {
  <root>
    <child s="1" />
  </root>
};

declare variable $tuis:xconf :=
    <collection xmlns="http://exist-db.org/collection-config/1.0">
        <index xmlns:xs="http://www.w3.org/2001/XMLSchema">
            <range>
                <create qname="@s" type="xs:string"/>
            </range>
        </index>
    </collection>
;

declare variable $tuis:col := 'test-util-index-keys';

declare %private
function tuis:update ($s) {
    let $root := doc('/db/' || $tuis:col || '/test.xml')/root
    return
        update insert <child s='{$s}'/> into $root
};

declare %private
function tuis:term ($term as xs:string, $data as xs:int+) as item()+ {
    $term, $data
};

declare %private
function tuis:get-nodeset () {
    collection('/db/' || $tuis:col)//@s
};

declare %private
function tuis:list-terms (
    $nodes as node()*, $number-of-results as xs:integer
) as item()* {
    util:index-keys($nodes, (), tuis:term#2, $number-of-results, 'range-index')
};

declare
    %test:setUp
function tuis:setup() {
    let $testCol := xmldb:create-collection("/db", $tuis:col)
    let $indexCol := xmldb:create-collection("/db/system/config/db", $tuis:col)
    let $_ :=
        (
            xmldb:store("/db/" || $tuis:col, "test.xml", $tuis:XML),
            xmldb:store("/db/system/config/db/" || $tuis:col,
                "collection.xconf", $tuis:xconf),
            xmldb:reindex("/db/" || $tuis:col)
        )
        
    return
        tuis:list-terms(tuis:get-nodeset(), 1)
};

declare
    %test:tearDown
function tuis:tearDown() {
    xmldb:remove("/db/" || $tuis:col),
    xmldb:remove("/db/system/config/db/" || $tuis:col)
};

declare
    %test:assertEquals('1', 1, 1, 1)
function tuis:test-initial() {
    tuis:list-terms(tuis:get-nodeset(), 1)
};

declare
    %test:assertEquals('1', 2, 1, 1)
function tuis:test-list-after-update() {
    let $_ := tuis:update('1')
    return tuis:list-terms(tuis:get-nodeset(), 1)
};

declare
    %test:assertEquals('1', 2, 1, 1)
function tuis:test-list-with-different-page-size() {
    tuis:list-terms(tuis:get-nodeset(), 2)
};

declare
    %test:assertEquals('1', 2, 1, 1)
function tuis:test-updated-after-reindex() {
    let $doc := xmldb:reindex('/db/' || $tuis:col)
    return tuis:list-terms(tuis:get-nodeset(), 1)
};

Returns

<testsuites>
    <testsuite package="http://exist-db.org/xquery/test/util-index-keys" timestamp="2023-03-17T19:45:39.751Z" tests="4" failures="1" errors="0" pending="0" time="PT0.025S">
        <testcase name="test-initial" class="tuis:test-initial"/>
        <testcase name="test-list-after-update" class="tuis:test-list-after-update">
            <failure message="assertEquals failed." type="failure-error-code-1">1 2 1 1</failure>
            <output>1 1 1 1</output>
        </testcase>
        <testcase name="test-list-with-different-page-size" class="tuis:test-list-with-different-page-size"/>
        <testcase name="test-updated-after-reindex" class="tuis:test-updated-after-reindex"/>
    </testsuite>
</testsuites>

Context (please always complete the following information)

Build: eXist-6.0.1 (51506fc97a9e5bb55fdea71fc76def2c9d402886) Java: 11.0.15 (Debian) OS: Linux 5.10.104-linuxkit (aarch64)

Additional context

• How is eXist-db installed? docker image duncdrum/existdb:6.0.1-debug
• Any custom changes in e.g. conf.xml? none

[adamretter]

I don't believe there is any caching in util:index-keys, this function simply calls either:

1. IndexWorker#scanIndex
2. NativeValueIndex#scanIndexKeys