Commit 311e4c03 authored by Jean-Benoist Leger's avatar Jean-Benoist Leger

on line node import from pbf w/out memory repr

parent 50567d58
......@@ -111,8 +111,11 @@ void parseBlock_ways(
void parseBlock_nodes(
osmpbf::PrimitiveBlockInputAdaptor & pbi,
std::unordered_map<nid_t, simple_node_t> & nodes,
std::unordered_map<nid_t, std::vector<wid_t>> & node_ways)
elevation & elevation_db,
nodes_db_t & nodes_db,
std::unordered_map<wid_t, simple_way_t> & ways,
std::unordered_map<nid_t, std::vector<wid_t>> & node_ways,
std::unordered_map<int32_t,std::unordered_map<int32_t,std::list<ncc_t>>> & lookup)
{
if (pbi.nodesSize())
......@@ -122,11 +125,130 @@ void parseBlock_nodes(
// is the node exist in our database
if(node_ways.count(node.id())>0)
{
simple_node_t simple_node;
simple_node.lon = node.lond();
simple_node.lat = node.latd();
double lon = node.lond();
double lat = node.latd();
elevation_value elevation = elevation_db.get_elevation(lon,lat);
nid_t nid = node.id();
ccid_t ccid;
nodes[node.id()] = simple_node;
std::vector<wid_t> & ways_of_nodes = node_ways[nid];
std::list<std::pair<nid_t,way_kind_t>> neighbors_from;
std::list<nid_t> neighbors_to;
for(auto way_it = ways_of_nodes.begin();
way_it != ways_of_nodes.end();
way_it++)
{
simple_way_t & way = ways[*way_it];
ccid = way.ccid;
for(unsigned int i = 0; i<way.nids.size(); i++)
{
if(way.nids[i] == nid)
{
if(i>0)
{
if(way.oneway>=0)
{
// i-1 from
neighbors_from.push_back( std::pair<nid_t,way_kind_t>(way.nids[i-1],way.way_kind) );
}
if(way.oneway<=0)
{
// i-1 to
neighbors_to.push_back( way.nids[i-1] );
}
}
if(i<way.nids.size()-1)
{
if(way.oneway>=0)
{
// i+1 to
neighbors_to.push_back( way.nids[i+1] );
}
if(way.oneway<=0)
{
// i+1 from
neighbors_from.push_back( std::pair<nid_t,way_kind_t>(way.nids[i+1],way.way_kind) );
}
}
}
}
}
std::list<neighbor_t> neighbors;
for(std::list< std::pair<nid_t,way_kind_t> >::iterator neighbors_from_it = neighbors_from.begin();
neighbors_from_it!=neighbors_from.end();
neighbors_from_it++)
{
nid_t nid_from = (*neighbors_from_it).first;
way_kind_t way_kind = (*neighbors_from_it).second;
for(std::list< nid_t >::iterator neighbors_to_it = neighbors_to.begin();
neighbors_to_it!=neighbors_to.end();
neighbors_to_it++)
{
nid_t nid_to = *neighbors_to_it;
if(nid_from!=nid_to)
{
neighbor_t neighbor;
neighbor.from = nid_from;
neighbor.to = nid_to;
neighbor.way_kind = way_kind;
neighbors.push_back(neighbor);
}
}
}
if(neighbors.size()==0)
{
// No-exit way, last node
for(std::list< std::pair<nid_t,way_kind_t> >::iterator neighbors_from_it = neighbors_from.begin();
neighbors_from_it!=neighbors_from.end();
neighbors_from_it++)
{
nid_t nid_from = (*neighbors_from_it).first;
way_kind_t way_kind = (*neighbors_from_it).second;
neighbor_t neighbor;
neighbor.from = nid_from;
neighbor.to = 0;
neighbor.way_kind = way_kind;
neighbors.push_back(neighbor);
}
for(std::list< nid_t >::iterator neighbors_to_it = neighbors_to.begin();
neighbors_to_it!=neighbors_to.end();
neighbors_to_it++)
{
nid_t nid_to = *neighbors_to_it;
neighbor_t neighbor;
neighbor.from = 0;
neighbor.to = nid_to;
neighbor.way_kind = RV_WK_OTHERS;
neighbors.push_back(neighbor);
}
}
nodes_db.write_node(
nid,
lon,
lat,
elevation,
neighbors);
int32_t key_lon = (int32_t)((180+lon)*RV_LOOKUP_PREC);
int32_t key_lat = (int32_t)((180+lat)*RV_LOOKUP_PREC);
ncc_t ncc;
ncc.nid = nid;
ncc.ccid = ccid;
lookup[key_lon][key_lat].push_back(ncc);
}
}
}
......@@ -134,12 +256,34 @@ void parseBlock_nodes(
int main(int argc, char ** argv)
{
if (argc < 4)
{
if (argc < 2) {
std::cout << "Need in file" << std::endl;
fprintf(stderr,"Usage: %s pbf elevation_db route_db\n",argv[0]);
abort();
}
std::unordered_map<nid_t, simple_node_t> nodes;
char zreq[RV_ZCHAR_LENGTH];
char zfilename[RV_ZCHAR_LENGTH];
// cleaning (without checking status, lmdb will check after and fail if this
// is not goot)
snprintf(zfilename,RV_ZCHAR_LENGTH,"%s/%s/data.mdb",argv[3],RV_NODES_DB_RELATIVE_PATH);
remove(zfilename);
snprintf(zfilename,RV_ZCHAR_LENGTH,"%s/%s/lock.mdb",argv[3],RV_NODES_DB_RELATIVE_PATH);
remove(zfilename);
snprintf(zfilename,RV_ZCHAR_LENGTH,"%s/%s/data.mdb",argv[3],RV_LOOKUP_DB_RELATIVE_PATH);
remove(zfilename);
snprintf(zfilename,RV_ZCHAR_LENGTH,"%s/%s/lock.mdb",argv[3],RV_LOOKUP_DB_RELATIVE_PATH);
remove(zfilename);
mkdir(argv[3],0755);
snprintf(zfilename,RV_ZCHAR_LENGTH,"%s/%s",argv[3],RV_NODES_DB_RELATIVE_PATH);
mkdir(zfilename,0755);
snprintf(zfilename,RV_ZCHAR_LENGTH,"%s/%s",argv[3],RV_LOOKUP_DB_RELATIVE_PATH);
mkdir(zfilename,0755);
std::unordered_map<wid_t, simple_way_t> ways;
std::unordered_map<nid_t, std::vector<wid_t>> node_ways;
......@@ -165,28 +309,6 @@ int main(int argc, char ** argv)
}
}
fprintf(stderr,"Parsing nodes from pbf and creating nodes map\n");
// scope
{
std::string inputFileName(argv[1]);
osmpbf::OSMFileIn inFile(inputFileName, false);
if (!inFile.open())
{
std::cout << "Failed to open " << inputFileName << std::endl;
return -1;
}
osmpbf::PrimitiveBlockInputAdaptor pbi;
while (inFile.parseNextBlock(pbi))
{
if (pbi.isNull())
continue;
parseBlock_nodes(pbi, nodes, node_ways);
}
}
fprintf(stderr,"Computing connected components\n");
// now we construct way_way adjacency
{
......@@ -251,220 +373,49 @@ int main(int argc, char ** argv)
if(modified==0)
break;
}
RV_BLANKLINE;
}
/* for(auto ways_it = ways.begin();
ways_it != ways.end();
ways_it++)
{
fprintf(stdout,"%lu -> %lu\n",ways_it->first,ways_it->second.ccid);
}
*/
char zreq[RV_ZCHAR_LENGTH];
char zfilename[RV_ZCHAR_LENGTH];
// cleaning (without checking status, lmdb will check after and fail if this
// is not goot)
snprintf(zfilename,RV_ZCHAR_LENGTH,"%s/%s/data.mdb",argv[3],RV_NODES_DB_RELATIVE_PATH);
remove(zfilename);
snprintf(zfilename,RV_ZCHAR_LENGTH,"%s/%s/lock.mdb",argv[3],RV_NODES_DB_RELATIVE_PATH);
remove(zfilename);
snprintf(zfilename,RV_ZCHAR_LENGTH,"%s/%s/data.mdb",argv[3],RV_LOOKUP_DB_RELATIVE_PATH);
remove(zfilename);
snprintf(zfilename,RV_ZCHAR_LENGTH,"%s/%s/lock.mdb",argv[3],RV_LOOKUP_DB_RELATIVE_PATH);
remove(zfilename);
mkdir(argv[3],0755);
snprintf(zfilename,RV_ZCHAR_LENGTH,"%s/%s",argv[3],RV_NODES_DB_RELATIVE_PATH);
mkdir(zfilename,0755);
snprintf(zfilename,RV_ZCHAR_LENGTH,"%s/%s",argv[3],RV_LOOKUP_DB_RELATIVE_PATH);
mkdir(zfilename,0755);
unsigned int total_number_of_nodes = std::distance(nodes.begin(),nodes.end());
RV_BLANKLINE;
fprintf(stderr,"Computing elevation on nodes\n");
fprintf(stderr,"Parsing nodes from pbf computing elevation and writing nodes preparing also lookup\n");
// scope
std::unordered_map<int32_t,std::unordered_map<int32_t,std::list<ncc_t>>> lookup;
{
// elevation_db (read-only)
elevation elevation_db(argv[2]);
for(auto nodes_it = nodes.begin();
nodes_it != nodes.end();
nodes_it++)
{
nodes_it->second.elevation = elevation_db.get_elevation(
nodes_it->second.lon,
nodes_it->second.lat);
}
}
{
fprintf(stderr,"Exporting nodes on nodes db\n");
// nodes_db (write)
snprintf(zfilename,RV_ZCHAR_LENGTH,"%s/%s",argv[3],RV_NODES_DB_RELATIVE_PATH);
nodes_db_t nodes_db(zfilename,true);
std::string inputFileName(argv[1]);
unsigned int count_nodes = 0;
osmpbf::OSMFileIn inFile(inputFileName, false);
for(auto nodes_it = nodes.begin();
nodes_it != nodes.end();
nodes_it++)
if (!inFile.open())
{
// first we must found neighbors
std::vector<wid_t> & ways_of_nodes = node_ways[nodes_it->first];
std::list<std::pair<nid_t,way_kind_t>> neighbors_from;
std::list<nid_t> neighbors_to;
for(auto way_it = ways_of_nodes.begin();
way_it != ways_of_nodes.end();
way_it++)
{
simple_way_t & way = ways[*way_it];
nodes_it->second.ccid = way.ccid;
for(unsigned int i = 0; i<way.nids.size(); i++)
{
if(way.nids[i] == nodes_it->first)
{
if(i>0)
{
if(way.oneway>=0)
{
// i-1 from
neighbors_from.push_back( std::pair<nid_t,way_kind_t>(way.nids[i-1],way.way_kind) );
}
if(way.oneway<=0)
{
// i-1 to
neighbors_to.push_back( way.nids[i-1] );
}
}
if(i<way.nids.size()-1)
{
if(way.oneway>=0)
{
// i+1 to
neighbors_to.push_back( way.nids[i+1] );
}
if(way.oneway<=0)
{
// i+1 from
neighbors_from.push_back( std::pair<nid_t,way_kind_t>(way.nids[i+1],way.way_kind) );
}
}
}
}
}
std::list<neighbor_t> neighbors;
for(std::list< std::pair<nid_t,way_kind_t> >::iterator neighbors_from_it = neighbors_from.begin();
neighbors_from_it!=neighbors_from.end();
neighbors_from_it++)
{
nid_t nid_from = (*neighbors_from_it).first;
way_kind_t way_kind = (*neighbors_from_it).second;
for(std::list< nid_t >::iterator neighbors_to_it = neighbors_to.begin();
neighbors_to_it!=neighbors_to.end();
neighbors_to_it++)
{
nid_t nid_to = *neighbors_to_it;
if(nid_from!=nid_to)
{
neighbor_t neighbor;
neighbor.from = nid_from;
neighbor.to = nid_to;
neighbor.way_kind = way_kind;
neighbors.push_back(neighbor);
}
}
}
if(neighbors.size()==0)
{
// No-exit way, last node
for(std::list< std::pair<nid_t,way_kind_t> >::iterator neighbors_from_it = neighbors_from.begin();
neighbors_from_it!=neighbors_from.end();
neighbors_from_it++)
{
nid_t nid_from = (*neighbors_from_it).first;
way_kind_t way_kind = (*neighbors_from_it).second;
neighbor_t neighbor;
neighbor.from = nid_from;
neighbor.to = 0;
neighbor.way_kind = way_kind;
neighbors.push_back(neighbor);
}
for(std::list< nid_t >::iterator neighbors_to_it = neighbors_to.begin();
neighbors_to_it!=neighbors_to.end();
neighbors_to_it++)
{
nid_t nid_to = *neighbors_to_it;
neighbor_t neighbor;
neighbor.from = 0;
neighbor.to = nid_to;
neighbor.way_kind = RV_WK_OTHERS;
neighbors.push_back(neighbor);
}
}
nid_t nid = nodes_it->first;
nodes_db.write_node(
nid,
nodes_it->second.lon,
nodes_it->second.lat,
nodes_it->second.elevation,
neighbors);
std::cout << "Failed to open " << inputFileName << std::endl;
return -1;
}
ways.clear();
node_ways.clear();
osmpbf::PrimitiveBlockInputAdaptor pbi;
while (inFile.parseNextBlock(pbi))
{
if (pbi.isNull())
continue;
parseBlock_nodes(pbi, elevation_db, nodes_db, ways, node_ways, lookup);
}
nodes_db.commit();
}
fprintf(stderr,"Lookup db\n");
fprintf(stderr,"-> Computing");
{
// lookup_db (write)
snprintf(zfilename,RV_ZCHAR_LENGTH,"%s/%s",argv[3],RV_LOOKUP_DB_RELATIVE_PATH);
lookup_db_t lookup_db(zfilename,true);
std::unordered_map<int32_t,std::unordered_map<int32_t,std::list<ncc_t>>> lookup;
for(auto nodes_it = nodes.begin();
nodes_it != nodes.end();
nodes_it++)
{
int32_t key_lon = (int32_t)((180+nodes_it->second.lon)*RV_LOOKUP_PREC);
int32_t key_lat = (int32_t)((180+nodes_it->second.lat)*RV_LOOKUP_PREC);
ncc_t ncc;
ncc.nid = nodes_it->first;
ncc.ccid = nodes_it->second.ccid;
lookup[key_lon][key_lat].push_back(ncc);
}
nodes.clear();
RV_BLANKLINE;
fprintf(stderr,"\r-> Writing");
for(auto lit1 = lookup.begin();
lit1 != lookup.end();
lit1++)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment