fly/easy-osm2city
1
0
Fork 0
Code Issues Pull requests Activity

Initial commit

Browse source 
Signed-off-by: merspieler <merspieler@airmail.cc>
This commit is contained in:
merspieler 2018-11-13 16:31:53 +01:00
commit 5332a8bbbe
11 changed files with 539 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
.gitignore vendored Normal file
View file

@ -0,0 +1,21 @@
# vim swap files
*.swp
*.swo
# temporary files from osm2city
*.pkl
# generated directories
venv/
# osm2city files
osm2city*
# pbf files
pbf/*
# add general-settings file since it contains passwords
general-settings
# Ignore the users projects
projects/

65
build Executable file
View file

@ -0,0 +1,65 @@
#! /bin/bash
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
-p|--project)
project="$2"
shift # past argument
shift # past value
;;
-h|--help)
echo "usage: build -p <project> [OPTIONS]"
echo "Builds the tiles with osm2city"
echo ""
echo "OPTIONS"
echo " -p, --project Mandatory, project name which you want to clear of chache files."
echo " -h, --help Shows this help and exit"
echo " -t, --threads Number of threads used for building"
echo " This will overwrite the value from the general-settings file"
exit 0
;;
*)
echo "Unknown option $key"
exit 1
;;
esac
done
if [ -z "$project" ]; then
echo "Option -p <project> is mandatory"
exit 1
fi
source venv/bin/activate > /dev/null
if [ $? == 1 ]; then
echo "Couldn't find the venv. Please run './create-venv' to create one"
exit
fi
source projects/$project/settings > /dev/null
if [ $? == 1 ]; then
echo "Couldn't find the settings file for the given project."
echo "Please check if the project exsists and contains the 'settings' file."
echo "To create a new project run './create-project'"
exit
fi
source general-settings > /dev/null
if [ $? == 1 ]; then
echo "Couldn't find the general settings. Please rename the 'general-settings-template'"
echo "file to 'general-setting' and edit it so the values match your setup."
exit
fi
if [ -z "$man_thread" ]; then
thread=$man_thread
fi
# change to the project dir and build
cd projects/$project
time ( python3 ../../osm2city/build_tiles.py -f params.ini -b "$bounds" -p "$threads" 2>&1 ) 2> exec-time
cat exec-time | grep "real" | sed "s/real\t\([0-9]*\)m\([0-9]*\).*/Building $project took \1 minutes and \2 seconds./"

36
clear-cache-files Executable file
View file

@ -0,0 +1,36 @@
#! /bin/bash
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
-p|--project)
project="$2"
shift # past argument
shift # past value
;;
-h|--help)
echo "usage: clear-cache-files -p <project> [OPTIONS]"
echo "Cleares cache files created by osm2city."
echo ""
echo "OPTIONS"
echo " -p, --project Mandatory, project name which you want to clear of chache files"
echo " -h, --help Shows this help and exit"
exit 0
;;
*)
echo "Unknown option $key"
exit 1
;;
esac
done
if [ -z "$project" ]; then
echo "Option -p <project> is mandatory"
exit 1
fi
rm projects/$project/*.pkl

37
create-db Executable file
View file

@ -0,0 +1,37 @@
#! /bin/bash
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
-d|--database)
database="$2"
shift # past argument
shift # past value
;;
-h|--help)
echo "usage: create-db -d <database> [OPTIONS]"
echo "Creates and prepares the database for the use with osm2city"
echo ""
echo "OPTIONS"
echo " -d, --database Mandatory, database to create"
echo " -h, --help Shows this help and exit"
exit 0
;;
*)
echo "Unknown option $key"
exit 1
;;
esac
done
# TODO read db user from general-settings file and make it overwriteable from the cmd line
sudo -u postgres createdb --encoding=UTF8 --owner=$USER "$database"
sudo -u postgres psql --dbname="$database" -c "CREATE EXTENSION postgis;"
sudo -u postgres psql --dbname="$database" -c "CREATE EXTENSION hstore;"
psql -d "$database" -f sql/pgsnapshot_schema_0.6.sql
psql -d "$database" -f sql/pgsnapshot_schema_0.6_bbox.sql

27
create-venv Executable file
View file

@ -0,0 +1,27 @@
#! /bin/bash
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
-h|--help)
echo "usage: create-venv [OPTIONS]"
echo "Creates a python virtual environment and installs all needed packages in"
echo "the venv."
echo ""
echo "OPTIONS"
echo " -h, --help Shows this help and exit"
exit 0
;;
*)
echo "Unknown option $key"
exit 1
;;
esac
done
python3 -m venv venv/
source venv/bin/activate
pip install descartes matplotlib networkx munpy pillow pyproj scipy shapely psycopg2-binary

13
general-settings.template Normal file
View file

@ -0,0 +1,13 @@
# This file must be used with "source general-settings" *from bash*
# you cannot run it directly
# Absolute path to the osm2city directory
PATHONPATH=/storage/scenery/osm2city/osm2city
export PYTHONPATH
# Absolute path to the fgdata directory
FG_ROOT=/games/fg-linux/install/flightgear/fgdata
export FG_ROOT
# Threads used for osm2city
threads=3

33
index-db Executable file
View file

@ -0,0 +1,33 @@
#! /bin/bash
while [[ "$#" -gt 0 ]]
do
key="$1"
case $key in
-d|--database)
database="$2"
shift # past argument
shift # past value
;;
-h|--help)
echo "usage: index-db -d <database> [OPTIONS]"
echo "Create an index for the database to speed up further operations"
echo ""
echo "OPTIONS"
echo " -d, --database Mandatory, the database to index"
echo " -h, --help Shows this help and exit"
exit 0
;;
*)
echo "Unknown option $key"
exit 1
;;
esac
done
time psql --dbname="$database" -c "CREATE INDEX idx_nodes_tags ON nodes USING gist(tags);"
time psql --dbname="$database" -c "CREATE INDEX idx_ways_tags ON ways USING gist(tags);"
time psql --dbname="$database" -c "CREATE INDEX idx_relations_tags ON relations USING gist(tags);"

76
install Executable file
View file

@ -0,0 +1,76 @@
#! /bin/bash
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
--skip-install)
skip_install=1
shift # past argument
shift # past value
;;
--skip-git)
skip_git=1
shift # past argument
shift # past value
;;
-h|--help)
echo "usage: install [OPTIONS]..."
echo "Installs all needed packages and clones the osm2city and osm2city-data repository"
echo ""
echo "OPTIONS:"
echo " -h, --help Shows this help and exits"
echo " --skip-install Skips install of packages and continues with setup"
echo " --skip-git Skips git clones of the osm2city and osm2city-data repository"
exit
;;
*)
echo "Unknow option $key"
exit 1
;;
esac
done
if [ "$skip_install" != 1 ]; then
which apt-get > /dev/null
if [ $? == 1 ]; then
echo "Currently apt is the only supported package manager but it seems that apt isn't installed."
echo "Please install the following packages manually"
echo "python3-venv"
echo "python-pip"
echo "postgresql-9.6"
echo "postgresql-9.6-postgis-2.3"
echo "postgresql-client-9.6"
echo "postgresql-contrib-9.6"
echo "pgadmin3"
echo "postgis"
echo "python3-psycopg2"
echo "osmosis"
echo "git"
echo "Then rerun this script with skip-install argument"
echo "./install --skip-install"
exit 1
fi
sudo apt-get install -y git osmosis postgis pgadmin3 postgresql-contrib-9.6 postgresql-client-9.6 postgresql-9.6-postgis-2.3 postgresql-9.6 python-pip python3-venv
fi
if [ "$skip_git" != 1 ]; then
which git > /dev/null
if [ $? == 1 ]; then
echo "Unable to find git. Please manually download the following two repositories:"
echo "https://gitlab.com/fg-radi/osm2city"
echo "https://gitlab.com/fg-radi/osm2city-data"
echo "Unzip them in this directory and make sure, the are named osm2city and osm2city-data"
exit 1
fi
git clone https://gitlab.com/fg-radi/osm2city.git
git clone https://gitlab.com/fg-radi/osm2city-data.git
fi
# Create directory for .pbf files
mkdir pbf

40
read-pbf Executable file
View file

@ -0,0 +1,40 @@
#! /bin/bash
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
-d|--database)
database="$2"
shift # past argument
shift # past value
;;
-f|--file)
file="$2"
shift # past argument
shift # past value
;;
-h|--help)
echo "usage: read-pbf -d <database> -f <pbf-file> [OPTIONS]"
echo "Reads the .pbf file and writes it to the database."
echo "NOTE this uses quite some space in the /tmp directory"
echo ""
echo "OPTIONS"
echo " -d, --database Mandatory, database to write to"
echo " -f, --file Mandatory, the .pbf file to read from"
echo " -h, --help Shows this help and exit"
exit 0
*)
echo "Unknown option $key"
exit 1
;;
esac
done
source general-settings
# TODO make password and that stuff overwrite able from cmd line
time osmosis --read-pbf file="$file" --log-progress --write-pgsql database="$database" host="$db_host:$db_port" user="$db_user" password="$db_passwd"

170
sql/pgsnapshot_schema_0.6.sql Normal file
View file

@ -0,0 +1,170 @@
-- Database creation script for the snapshot PostgreSQL schema.
-- Drop all tables if they exist.
DROP TABLE IF EXISTS actions;
DROP TABLE IF EXISTS users;
DROP TABLE IF EXISTS nodes;
DROP TABLE IF EXISTS ways;
DROP TABLE IF EXISTS way_nodes;
DROP TABLE IF EXISTS relations;
DROP TABLE IF EXISTS relation_members;
DROP TABLE IF EXISTS schema_info;
-- Drop all stored procedures if they exist.
DROP FUNCTION IF EXISTS osmosisUpdate();
-- Create a table which will contain a single row defining the current schema version.
CREATE TABLE schema_info (
version integer NOT NULL
);
-- Create a table for users.
CREATE TABLE users (
id int NOT NULL,
name text NOT NULL
);
-- Create a table for nodes.
CREATE TABLE nodes (
id bigint NOT NULL,
version int NOT NULL,
user_id int NOT NULL,
tstamp timestamp without time zone NOT NULL,
changeset_id bigint NOT NULL,
tags hstore
);
-- Add a postgis point column holding the location of the node.
SELECT AddGeometryColumn('nodes', 'geom', 4326, 'POINT', 2);
-- Create a table for ways.
CREATE TABLE ways (
id bigint NOT NULL,
version int NOT NULL,
user_id int NOT NULL,
tstamp timestamp without time zone NOT NULL,
changeset_id bigint NOT NULL,
tags hstore,
nodes bigint[]
);
-- Create a table for representing way to node relationships.
CREATE TABLE way_nodes (
way_id bigint NOT NULL,
node_id bigint NOT NULL,
sequence_id int NOT NULL
);
-- Create a table for relations.
CREATE TABLE relations (
id bigint NOT NULL,
version int NOT NULL,
user_id int NOT NULL,
tstamp timestamp without time zone NOT NULL,
changeset_id bigint NOT NULL,
tags hstore
);
-- Create a table for representing relation member relationships.
CREATE TABLE relation_members (
relation_id bigint NOT NULL,
member_id bigint NOT NULL,
member_type character(1) NOT NULL,
member_role text NOT NULL,
sequence_id int NOT NULL
);
-- Configure the schema version.
INSERT INTO schema_info (version) VALUES (6);
-- Add primary keys to tables.
ALTER TABLE ONLY schema_info ADD CONSTRAINT pk_schema_info PRIMARY KEY (version);
ALTER TABLE ONLY users ADD CONSTRAINT pk_users PRIMARY KEY (id);
ALTER TABLE ONLY nodes ADD CONSTRAINT pk_nodes PRIMARY KEY (id);
ALTER TABLE ONLY ways ADD CONSTRAINT pk_ways PRIMARY KEY (id);
ALTER TABLE ONLY way_nodes ADD CONSTRAINT pk_way_nodes PRIMARY KEY (way_id, sequence_id);
ALTER TABLE ONLY relations ADD CONSTRAINT pk_relations PRIMARY KEY (id);
ALTER TABLE ONLY relation_members ADD CONSTRAINT pk_relation_members PRIMARY KEY (relation_id, sequence_id);
-- Add indexes to tables.
CREATE INDEX idx_nodes_geom ON nodes USING gist (geom);
CREATE INDEX idx_way_nodes_node_id ON way_nodes USING btree (node_id);
CREATE INDEX idx_relation_members_member_id_and_type ON relation_members USING btree (member_id, member_type);
-- Set to cluster nodes by geographical location.
ALTER TABLE ONLY nodes CLUSTER ON idx_nodes_geom;
-- Set to cluster the tables showing relationship by parent ID and sequence
ALTER TABLE ONLY way_nodes CLUSTER ON pk_way_nodes;
ALTER TABLE ONLY relation_members CLUSTER ON pk_relation_members;
-- There are no sensible CLUSTER orders for users or relations.
-- Depending on geometry columns different clustings of ways may be desired.
-- Create the function that provides "unnest" functionality while remaining compatible with 8.3.
CREATE OR REPLACE FUNCTION unnest_bbox_way_nodes() RETURNS void AS $$
DECLARE
previousId ways.id%TYPE;
currentId ways.id%TYPE;
result bigint[];
wayNodeRow way_nodes%ROWTYPE;
wayNodes ways.nodes%TYPE;
BEGIN
FOR wayNodes IN SELECT bw.nodes FROM bbox_ways bw LOOP
FOR i IN 1 .. array_upper(wayNodes, 1) LOOP
INSERT INTO bbox_way_nodes (id) VALUES (wayNodes[i]);
END LOOP;
END LOOP;
END;
$$ LANGUAGE plpgsql;
-- Create customisable hook function that is called within the replication update transaction.
CREATE FUNCTION osmosisUpdate() RETURNS void AS $$
DECLARE
BEGIN
END;
$$ LANGUAGE plpgsql;
-- Manually set statistics for the way_nodes and relation_members table
-- Postgres gets horrible counts of distinct values by sampling random pages
-- and can be off by an 1-2 orders of magnitude
-- Size of the ways table / size of the way_nodes table
ALTER TABLE way_nodes ALTER COLUMN way_id SET (n_distinct = -0.08);
-- Size of the nodes table / size of the way_nodes table * 0.998
-- 0.998 is a factor for nodes not in ways
ALTER TABLE way_nodes ALTER COLUMN node_id SET (n_distinct = -0.83);
-- API allows a maximum of 2000 nodes/way. Unlikely to impact query plans.
ALTER TABLE way_nodes ALTER COLUMN sequence_id SET (n_distinct = 2000);
-- Size of the relations table / size of the relation_members table
ALTER TABLE relation_members ALTER COLUMN relation_id SET (n_distinct = -0.09);
-- Based on June 2013 data
ALTER TABLE relation_members ALTER COLUMN member_id SET (n_distinct = -0.62);
-- Based on June 2013 data. Unlikely to impact query plans.
ALTER TABLE relation_members ALTER COLUMN member_role SET (n_distinct = 6500);
-- Based on June 2013 data. Unlikely to impact query plans.
ALTER TABLE relation_members ALTER COLUMN sequence_id SET (n_distinct = 10000);

21
sql/pgsnapshot_schema_0.6_bbox.sql Normal file
View file

@ -0,0 +1,21 @@
-- Add a postgis GEOMETRY column to the way table for the purpose of indexing the location of the way.
-- This will contain a bounding box surrounding the extremities of the way.
SELECT AddGeometryColumn('ways', 'bbox', 4326, 'GEOMETRY', 2);
-- Add an index to the bbox column.
CREATE INDEX idx_ways_bbox ON ways USING gist (bbox);
-- Cluster table by geographical location.
CLUSTER ways USING idx_ways_bbox;
-- Create an aggregate function that always returns the first non-NULL item. This is required for bbox queries.
CREATE OR REPLACE FUNCTION first_agg (anyelement, anyelement)
RETURNS anyelement AS $$
SELECT CASE WHEN $1 IS NULL THEN $2 ELSE $1 END;
$$ LANGUAGE SQL STABLE;
CREATE AGGREGATE first (
sfunc = first_agg,
basetype = anyelement,
stype = anyelement
);