Flex backend : mark_node ?

[PanierAvide]

Hello,

I'm trying to use the new flex backend in order to retrieve street name from an associatedStreet relation to add it to all nodes being a member of relation. I didn't see any osm2pgsql.mark_node(object.id) so I went for osm2pgsql.mark('n', object.id), but I don't have points anymore in output DB. Is it because nodes aren't going to stage 2, or something related to my script (note that I'm no Lua expert) ?

Best regards.

@joto

-- Objects with any of the following keys will be treated as polygon
local polygon_keys = {
    'boundary',
    'admin_level',
    'leisure',
    'amenity',
    'tourism',
    'landuse',
    'shop',
    'craft',
    'office',
    'emergency',
    'historic',
    'addr:housenumber',
    'internet_access',
    'carpool'
}

-- Objects without any of the following keys will be deleted
local generic_keys = {
    'boundary',
    'admin_level',
    'leisure',
    'amenity',
    'tourism',
    'landuse',
    'shop',
    'craft',
    'office',
    'emergency',
    'historic',
    'highway',
    'cycleway',
    'cycleway:both',
    'cycleway:left',
    'cycleway:right',
    'addr:housenumber',
    'internet_access',
    'carpool',
    'type'
}

-- The following keys will be deleted
local delete_tags = {
    'FIXME',
    'note',
    'source',
    'way',
    'way_area',
    'z_order',
}

-- Array used to specify z_order per key/value combination.
-- Each element has the form {key, value, z_order, is_road}.
local zordering_tags = {{ 'railway', nil, 5, 1}, { 'boundary', 'administrative', 0, 1},
    { 'bridge', 'yes', 10, 0 }, { 'bridge', 'true', 10, 0 }, { 'bridge', 1, 10, 0 },
    { 'tunnel', 'yes', -10, 0}, { 'tunnel', 'true', -10, 0}, { 'tunnel', 1, -10, 0},
    { 'highway', 'minor', 3, 0}, { 'highway', 'road', 3, 0 }, { 'highway', 'unclassified', 3, 0 },
    { 'highway', 'residential', 3, 0 }, { 'highway', 'tertiary_link', 4, 0}, { 'highway', 'tertiary', 4, 0},
    { 'highway', 'secondary_link', 6, 1}, { 'highway', 'secondary', 6, 1},
    { 'highway', 'primary_link', 7, 1}, { 'highway', 'primary', 7, 1},
    { 'highway', 'trunk_link', 8, 1}, { 'highway', 'trunk', 8, 1},
    { 'highway', 'motorway_link', 9, 1}, { 'highway', 'motorway', 9, 1},
}

local tables = {}

tables.point = osm2pgsql.define_table{
    name = 'planet_osm_point',
    ids = { type = 'node', id_column = 'osm_id' },
    columns = {
        { column = 'leisure', type = 'text' },
        { column = 'amenity', type = 'text' },
        { column = 'tourism', type = 'text' },
        { column = 'shop', type = 'text' },
        { column = 'craft', type = 'text' },
        { column = 'office', type = 'text' },
        { column = 'emergency', type = 'text' },
        { column = 'historic', type = 'text' },
        { column = 'highway', type = 'text' },
        { column = 'access', type = 'text' },
        { column = 'wheelchair', type = 'text' },
        { column = 'addr:housenumber', type = 'text' },
        { column = 'internet_access', type = 'text' },
        { column = 'carpool', type = 'text' },
        { column = 'tags', type = 'hstore' },
        { column = 'rel_tags', type = 'hstore' },
        { column = 'way', type = 'point' }
    }
}

tables.line = osm2pgsql.define_table{
    name = 'planet_osm_line',
    ids = { type = 'way', id_column = 'osm_id' },
    columns = {
        { column = 'boundary', type = 'text' },
        { column = 'admin_level', type = 'text' },
        { column = 'leisure', type = 'text' },
        { column = 'amenity', type = 'text' },
        { column = 'tourism', type = 'text' },
        { column = 'emergency', type = 'text' },
        { column = 'historic', type = 'text' },
        { column = 'highway', type = 'text' },
        { column = 'access', type = 'text' },
        { column = 'wheelchair', type = 'text' },
        { column = 'cycleway', type = 'text' },
        { column = 'cycleway:both', type = 'text' },
        { column = 'cycleway:left', type = 'text' },
        { column = 'cycleway:right', type = 'text' },
        { column = 'tags', type = 'hstore' },
        { column = 'rel_tags', type = 'hstore' },
--         { column = 'way_area', type = 'area' },
        { column = 'way', type = 'linestring' },
    }
}

tables.polygon = osm2pgsql.define_table{
    name = 'planet_osm_polygon',
    ids = { type = 'area', id_column = 'osm_id' },
    columns = {
        { column = 'boundary', type = 'text' },
        { column = 'admin_level', type = 'text' },
        { column = 'leisure', type = 'text' },
        { column = 'amenity', type = 'text' },
        { column = 'tourism', type = 'text' },
        { column = 'landuse', type = 'text' },
        { column = 'shop', type = 'text' },
        { column = 'craft', type = 'text' },
        { column = 'office', type = 'text' },
        { column = 'emergency', type = 'text' },
        { column = 'historic', type = 'text' },
        { column = 'highway', type = 'text' },
        { column = 'access', type = 'text' },
        { column = 'wheelchair', type = 'text' },
        { column = 'addr:housenumber', type = 'text' },
        { column = 'internet_access', type = 'text' },
        { column = 'carpool', type = 'text' },
        { column = 'tags', type = 'hstore' },
--         { column = 'way_area', type = 'area' },
        { column = 'way', type = 'geometry' },
    }
}

by_node_id = {}

-- Helper function to check whether a table is empty
function is_empty(some_table)
    return next(some_table) == nil
end

function has_generic_tag(tags)
    for k, v in pairs(tags) do
        for j, k2 in ipairs(generic_keys) do
            if k == k2 then
                return true
            end
        end
    end
    return false
end

function to_row(object)
    local row = { tags = {}, rel_tags = {} }
    for k, v in pairs(object.tags) do
        row[k] = v
        row.tags[k] = v
    end
    for i,k in ipairs(delete_tags) do
        row[k] = nil
    end
    return row
end

function osm2pgsql.process_node(object)
    if is_empty(object.tags) then
        return
    end

    if not has_generic_tag(object.tags) then
        return
    end

    -- In stage 1: Mark all remaining objects so we will see them again in stage 2
    if osm2pgsql.stage == 1 then
        osm2pgsql.mark('n', object.id)
        return
    end

    -- We are now in stage 2
    local row = to_row(object)

    -- If there is any data from relations, add it in
    local d = by_node_id[object.id]
    if d then
        row.rel_tags = d
    end

    tables.point:add_row(row)
end

-- Treat objects with a key in polygon_keys as polygon
function is_polygon(tags)
    for i,k in ipairs(polygon_keys) do
        if tags[k] then
            return true
        end
    end
    return false
end

function osm2pgsql.process_way(object)
    if is_empty(object.tags) then
        return
    end

    local row = to_row(object)

    if not has_generic_tag(object.tags) then
        return
    end

    local polygon = is_polygon(object.tags)
    -- Treat objects tagged as area=yes, area=1, or area=true as polygon,
    -- and treat objects tagged as area=no, area=0, or area=false not as polygon
    local area_tag = object.tags.area
    if area_tag == 'yes' or area_tag == '1' or area_tag == 'true' then
        polygon = true
    elseif area_tag == 'no' or area_tag == '0' or area_tag == 'false' then
        polygon = false
    end

    if polygon then
        row.way = { create = 'area' }
        tables.polygon:add_row(row)
    else
        row.way = { create = 'line', split_at = 100000 }
        tables.line:add_row(row)
    end
end

function osm2pgsql.process_relation(object)
    if is_empty(object.tags) then
        return
    end

    if not has_generic_tag(object.tags) then
        return
    end

    local row = to_row(object)

    -- Go through all the members and store relation tags
    for _, member in ipairs(object.members) do
        if member.type == 'n' then
            if not by_node_id[member.ref] then
                by_node_id[member.ref] = {}
            end
            local d = by_node_id[member.ref]
            for k,v in pairs(object.tags) do
                d[k] = v
            end
        end
    end

    if type == 'multipolygon' then
       row.way = { create = 'area' }
       tables.polygon:add_row(row)
    end
end

[joto]

There is currently no mark_node() and nodes are not processed in stage 2, because to implement it there are some internal changes in osm2pgsql needed that are not related to the flex backend. This is on my list and will be fixed at some point, but it might be a while.

[PanierAvide]

Alright, thanks for your quick reply. So I will wait for this, but as far as I tested it, this Flex backend is quite promising ! :+1:

[PanierAvide]

For solving this street names issues for nodes, I'm using for now another table to store relation tags for every concerned node. Then, in SQL, I add back the tags on planet_osm_point with an UPDATE request. Works quite well, I wrote an article as a feedback and method for this use case.

[lucadelu]

There is currently no mark_node() and nodes are not processed in stage 2, because to implement it there are some internal changes in osm2pgsql needed that are not related to the flex backend. This is on my list and will be fixed at some point, but it might be a while.

This would be really useful to get info from relations to points (like signpost for hiking routes or bus stop for bus routes)

[nextstopsun]

Same issue here. Need to pull admin_level value from relation with marked node. No luck with this yet.

[zdila]

In my case it would be useful for knowing admin_centre nodes in process_relation.