Introduce multi-version tests

[Gerold103]

Lately Tarantool core gets a lot of care to support smooth upgrade progress. So far many tests are covering schema upgrades, boot from old snaps and xlogs. But not a single test can cover true backward compatibility, when a binary of an older version runs in the same cluster as a binary of a newer version. Or how an older binary could boot from a newer snap + xlogs.

The suggestion is to make it possible to start Tarantool's of publicly available versions. I.e. in server:start() or before that to set the version that the test needs and luatest would fetch the needed binary from some remote storage and cache it locally for reusage.

Then Tarantool could actually cover compatibility tests on a regular basis, in normal tests running in CI. Not just manually, when one another fix gets done and needs testing.

[Totktonada]

My PoC:

local t = require('luatest')
local server = require('luatest.server')
local fio = require('fio')

local g = t.group()

local function setup(required_version)
    local bin_dir = os.getenv('TT_BIN_DIR')
    if bin_dir == nil then
        error('TT_BIN_DIR is not set', 0)
    end
    if bin_dir == '' then
        error('TT_BIN_DIR is empty', 0)
    end
    if not fio.path.exists(bin_dir) then
        error(('TT_BIN_DIR is incorrect: %s does not exists'):format(bin_dir), 0)
    end
    if not fio.path.is_dir(bin_dir) then
        error(('TT_BIN_DIR is incorrect: %s is not a directory'):format(bin_dir), 0)
    end

    local available_versions = {}
    local paths = {}
    for _, path in ipairs(fio.glob(fio.pathjoin(bin_dir, 'tarantool_*'))) do
        local pattern = '^tarantool_([0-9]+%.[0-9]+%.[0-9]+)$'
        local version = fio.basename(path):match(pattern)
        if version ~= nil then
            available_versions[version] = true
            paths[version] = path
        end
    end

    if available_versions[required_version] == nil then
        error(('TT_BIN_DIR is not suitable: %s does not contain tarantool ' ..
            'version %s'):format(bin_dir, required_version), 0)
    end

    local path = paths[required_version]
    if not fio.path.is_file(path) then
        error(('%s is not a file'):format(path), 0)
    end

    return path
end

g.before_all(function(g)
    local required_version = '2.11.4'
    local ok, path = pcall(setup, required_version)
    if not ok then
        t.skip(('Unable to find required tarantool version %s: %s'):format(
            required_version, path))
    end
    g.paths = {
        [required_version] = path,
    }
end)

g.after_each(function(g)
    for k, v in pairs(g) do
        if k:startswith('server') then
            v:stop()
            g[k] = nil
        end
    end
end)

g.test_basic = function(g)
    g.server = server:new({alias = 'head'})
    g.server:start()

    g.server_2_11_4 = server:new({alias = '2.11.4', command = g.paths['2.11.4']})
    g.server_2_11_4.args = {g.server.command}
    g.server_2_11_4.command = g.paths['2.11.4']
    g.server_2_11_4:start({wait_until_ready = true})
    g.server_2_11_4:exec(function()
        t.assert_str_matches(box.info.version, '^2%.11%.4.*')
    end)

    g.server:exec(function(remote_uri)
        local net_box = require('net.box')

        local conn = net_box.connect(remote_uri)
        local box_info = conn:call('box.info')
        t.assert_str_matches(box_info.version, '^2%.11%.4.*')
    end, {g.server_2_11_4.net_box_uri})

    g.server_2_11_4:exec(function(remote_uri, expected_version)
        local net_box = require('net.box')

        local conn = net_box.connect(remote_uri)
        local box_info = conn:call('box.info')
        t.assert_equals(box_info.version, expected_version)
    end, {g.server.net_box_uri, box.info.version})
end

How to run:

$ TT_BIN_DIR=/home/alex/tmp/try-tt-binaries/bin luatest -v test/multiversion-luatest/net_box_test.lua