#! /usr/bin/python3
# Copyright (C) 2018-2020 Merspieler, merspieler _at_ airmail.cc
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
import os
import sys
import math
import re
import json
import time
import subprocess
chunk_size = 5
threads = 5
cont = 0
exclude = []
pbf_path = ""
argc = len(sys.argv)
i = 1
first = 1
while i < argc:
if sys.argv[i] == "-s" or sys.argv[i] == "--chunk-size":
i += 1
chunk_size = sys.argv[i]
elif sys.argv[i] == "-c" or sys.argv[i] == "--continue":
i += 1
cont = int(sys.argv[i])
elif sys.argv[i] == "-t" or sys.argv[i] == "--threads":
i += 1
threads = sys.argv[i]
elif sys.argv[i] == "-e" or sys.argv[i] == "--exclude":
i += 1
if os.path.isfile(sys.argv[i]):
try:
with open(sys.argv[i]) as json_data:
exclude += json.load(json_data)
except ValueError:
print("Exclude file '" + sys.argv[i] + "' is no proper JSON file.")
sys.exit(1)
else:
print("File not found: " + sys.argv[i])
sys.exit(1)
elif sys.argv[i] == "-p" or sys.argv[i] == "--progress":
try:
with open("projects/worldbuild/done") as f:
lines = f.readlines()
for line in lines:
match = re.findall("(-?[0-9]{1,3}\.?[0-9]{0,4})_(-?[0-9]{1,3}\.?[0-9]{0,4})_(-?[0-9]{1,3}\.?[0-9]{0,4})_(-?[0-9]{1,3}\.?[0-9]{0,4})", line)
if match != []:
w = float(match[0][0])
s = float(match[0][1])
e = float(match[0][2])
n = float(match[0][3])
if n == 90:
world = 1
tile = "N/A"
elif n == -80:
world = 2
tile = "N/A"
else:
wm = w % 10
sm = s % 10
em = e % 10
nm = n % 10
wM = (int(w) - wm) / 10
sM = (int(s) - sm) / 10
eM = (int(e) - em) / 10
nM = (int(n) - nm) / 10
cs = n - s
if nm == 0:
nm = 10
tile = (wm % 10) * 10 + cs * cs * (nm / cs)
rows = 0
while sM > -8:
rows += 1
sM -= 1
world = 3 + 36 * rows + wM + 18
print("Current worldbuild tile is " + str(int(world)) + "/578")
print("Current tile " + str(tile) + "% complete")
sys.exit(0)
else:
print("Unable to get progress")
sys.exit(1)
except IOError:
print("Unable to get progress")
sys.exit(1)
elif sys.argv[i] == "-h" or sys.argv[i] == "--help":
print("usage: worldbuild <pbf-path> [OPTIONS]")
print("Builds the world")
print("")
print("OPTIONS")
print(" -s, --chunk-size Sets chunk size, default 5")
print(" -t, --threads Number of threads to run")
print(" -c, --continue Contine build from tile number <n> when building with 'demand' strategy")
print(" -e, --exclude Files containing JSON array naming tiles not to be build")
print(" Can be used multiple times.")
print(" If not given projects/worldbuild/exclude will be used")
print(" -h, --help Shows this help and exit")
print(" -p, --progress Shows progress and exit")
sys.exit(0)
else:
if first == 1:
first = 0
pbf_path = sys.argv[i]
else:
print("Unknown option " + sys.argv[i])
sys.exit(1)
i += 1
if pbf_path == "" and db_strategy == "demand":
print("No pbf-path was given, exiting...")
sys.exit(1)
def run(command):
exit_code = os.system(command)
if exit_code == 0:
return
elif exit_code == 130:
print("Aborted!")
sys.exit(130)
else:
print("Sub process '" + command + "'exited with code " + str(exit_code) + ". Aborting!")
sys.exit(4)
run("mkdir -p projects/worldbuild/output/error")
def build_tile(name, west, south, east, north, chunk_size, threads, cont=False):
global pbf_path
if west < 0:
west = "*" + str(west)
else:
west = str(west)
south = str(south)
east = str(east)
north = str(north)
run("./read-pbf worldbuild " + pbf_path + name + ".osm.pbf")
run('echo "bounds=' + west + "_" + south + "_" + east + "_" + north + '" > projects/worldbuild/settings')
run("./build worldbuild --chunk-size " + str(chunk_size) + " -t " + str(threads))
def after_build(name):
if os.path.isfile("projects/worldbuild/osm2city-exceptions.log"):
run("mv projects/worldbuild/osm2city-exceptions.log projects/worldbuild/output/error/" + name + ".exceptions.log")
# Trigger failed after build script
if os.path.isfile("./scripts/afterbuild-failed"):
os.system("bash -c './scripts/afterbuild-failed " + name + " &'")
else:
# Trigger after build script
if os.path.isfile("./scripts/afterbuild-success"):
os.system("bash -c './scripts/afterbuild-success " + name + " &'")
def prepare():
run("./delete-db worldbuild")
run("./create-db worldbuild")
run("./clear-cache-files worldbuild")
def run_all(name, w, s, e, n, chunk_size, threads, cont=False):
global pbf_path
if os.system("ls -l " + pbf_path + name + ".osm.pbf | grep ' 73 ' > /dev/null") != 0:
if cont == False:
prepare()
build_tile(name, w, s, e, n, chunk_size, threads, cont)
after_build(name)
else:
print("INFO: Skipping " + name + " because pbf file is empty")
def norm(num, length):
num = str(num)
while len(num) < length:
num = "0" + num
return num
def print_build_time(start_time, end_time):
elapsed = end_time - start_time
seconds = elapsed % 60
elapsed = (elapsed - seconds) / 60
minutes = elapsed % 60
elapsed = (elapsed - minutes) / 60
hours = elapsed % 24
days = (elapsed - hours) / 24
time = str(int(hours)) + " Hours, " + str(int(minutes)) + " Minutes and " + str(int(seconds)) + " Seconds"
if days > 0:
time = str(int(days)) + " Days, " + time
print("Running worldbuild took " + time)
# Get exclude file
if os.path.isfile("projects/worldbuild/exclude") and exclude == []:
try:
with open("projects/worldbuild/exclude") as json_data:
exclude = json.load(json_data)
except ValueError:
print("Exclude file 'projects/worldbuild/exclude' is no proper JSON file.")
sys.exit(1)
start_time = time.time()
if cont != 0:
tile = cont - 2
tile_in_row = ((tile - 1) % 36) - 18
ii = (tile - tile_in_row - 19) / 36 - 8
else:
ii = -8
# Build poles first
if not "n-pole" in exclude and cont <= 1:
run_all("n-pole", -180, 80, 180, 90, 360, threads)
if not "s-pole" in exclude and cont <= 2:
run_all("s-pole", -180, -90, 180, -80, 360, threads)
while ii < 8:
i = ii * 10
if cont != 0:
jj = tile_in_row
else:
jj = -18
while jj < 18:
j = jj * 10
if i >= 0:
ns = "n"
else:
ns = "s"
if j >= 0:
ew = "e"
else:
ew = "w"
name = ew + norm(abs(j), 3) + ns + norm(abs(i), 2)
if not name in exclude:
if cont != 0:
run_all(name, j, i, j + 10, i + 10, chunk_size, threads, cont=True)
cont = 0
else:
run_all(name, j, i, j + 10, i + 10, chunk_size, threads, cont=False)
jj += 1
ii += 1
print_build_time(start_time, time.time())