osm2city-scripts/worldbuild.py

#! /usr/bin/python3
# Copyright (C) 2018-2019 Merspieler, merspieler _at_ airmail.cc
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

import os
import sys
import math
import re
import json
import time
import subprocess

chunk_size = 5
threads = 5
cont = 0
exclude = []
pbf_path = ""

argc = len(sys.argv)
i = 1
first = 1
while i < argc:
	if sys.argv[i] == "-s" or sys.argv[i] == "--chunk-size":
		i += 1
		chunk_size = sys.argv[i]
	elif sys.argv[i] == "-c" or sys.argv[i] == "--continue":
		i += 1
		cont = int(sys.argv[i])
	elif sys.argv[i] == "-t" or sys.argv[i] == "--threads":
		i += 1
		threads = sys.argv[i]
	elif sys.argv[i] == "-e" or sys.argv[i] == "--exclude":
		i += 1
		if os.path.isfile(sys.argv[i]):
			try:
				with open(sys.argv[i]) as json_data:
					exclude += json.load(json_data)
			except ValueError:
				print("Exclude file '" + sys.argv[i] + "' is no proper JSON file.")
				sys.exit(1)
		else:
			print("File not found: " + sys.argv[i])
			sys.exit(1)
	elif sys.argv[i] == "-p" or sys.argv[i] == "--progress":
		try:
			with open("projects/worldbuild/done") as f:
				lines = f.readlines()
				for line in lines:
					match = re.findall("(-?[0-9]{1,3}\.?[0-9]{0,4})_(-?[0-9]{1,3}\.?[0-9]{0,4})_(-?[0-9]{1,3}\.?[0-9]{0,4})_(-?[0-9]{1,3}\.?[0-9]{0,4})", line)
					if match != []:

						w = float(match[0][0])
						s = float(match[0][1])
						e = float(match[0][2])
						n = float(match[0][3])

						if n == 90:
							world = 1
							tile = "N/A"
						elif n == -80:
							world = 2
							tile = "N/A"
						else:
							wm = w % 10
							sm = s % 10
							em = e % 10
							nm = n % 10

							wM = (int(w) - wm) / 10
							sM = (int(s) - sm) / 10
							eM = (int(e) - em) / 10
							nM = (int(n) - nm) / 10

							cs = n - s
							if nm == 0:
								nm = 10
							tile = (wm % 10) * 10 + cs * cs * (nm / cs)

							
							rows = 0
							while sM > -8:
								rows += 1
								sM -= 1

							world = 3 + 36 * rows + wM + 18

						print("Current worldbuild tile is " + str(int(world)) + "/578")
						print("Current tile " + str(tile) + "% complete")
						sys.exit(0)
					else:
						print("Unable to get progress")
						sys.exit(1)
		except IOError:
			print("Unable to get progress")
			sys.exit(1)
	elif sys.argv[i] == "-h" or sys.argv[i] == "--help":
		print("usage: worldbuild <pbf-path> [OPTIONS]")
		print("Builds the world")
		print("")
		print("OPTIONS")
		print("  -s, --chunk-size         Sets chunk size,  default 5")
		print("  -t, --threads            Number of threads to run")
		print("  -c, --continue           Contine build from tile number <n> when building with 'demand' strategy")
		print("  -e, --exclude            Files containing JSON array naming tiles not to be build")
		print("                           Can be used multiple times.")
		print("                           If not given projects/worldbuild/exclude will be used")
		print("  -h, --help               Shows this help and exit")
		print("  -p, --progress           Shows progress and exit")
		sys.exit(0)
	else:
		if first == 1:
			first = 0
			pbf_path = sys.argv[i]
		else:
			print("Unknown option " + sys.argv[i])
			sys.exit(1)
	i += 1

if pbf_path == "" and db_strategy == "demand":
	print("No pbf-path was given, exiting...")
	sys.exit(1)

def run(command):
	exit_code = os.system(command)
	if exit_code == 0:
		return
	elif exit_code == 130:
		print("Aborted!")
		sys.exit(130)
	else:
		print("Sub process '" + command + "'exited with code " + str(exit_code) + ". Aborting!")
		sys.exit(4)

run("mkdir -p projects/worldbuild/output/error")

def build_tile(name, west, south, east, north, chunk_size, threads, cont=False):
	global pbf_path
	if west < 0:
		west = "*" + str(west)
	else:
		west = str(west)
	south = str(south)
	east = str(east)
	north = str(north)

	run("./read-pbf worldbuild " + pbf_path + name + ".osm.pbf")
	run('echo "bounds=' + west + "_" + south + "_" + east + "_" + north + '" > projects/worldbuild/settings')
	run("./build worldbuild --chunk-size " + str(chunk_size) + " -t " + str(threads))

def after_build(name):
	if os.path.isfile("projects/worldbuild/osm2city-exceptions.log"):
		run("mv projects/worldbuild/osm2city-exceptions.log projects/worldbuild/output/error/" + name + ".exceptions.log")
		# Trigger failed after build script
		if os.path.isfile("./scripts/afterbuild-failed"):
			os.system("bash -c './scripts/afterbuild-failed " + name + " &'")
	else:
		# Trigger after build script
		if os.path.isfile("./scripts/afterbuild-success"):
			os.system("bash -c './scripts/afterbuild-success " + name + " &'")

def prepare():
	run("./delete-db worldbuild")
	run("./create-db worldbuild")

	run("./clear-cache-files worldbuild")

def run_all(name, w, s, e, n, chunk_size, threads, cont=False):
	global pbf_path
	if os.system("ls -l " + pbf_path + name + ".osm.pbf | grep ' 73 ' > /dev/null") != 0:
		if cont == False:
			prepare()
		build_tile(name, w, s, e, n, chunk_size, threads, cont)
		after_build(name)
	else:
		print("INFO: Skipping " + name + " because pbf file is empty")

def norm(num, length):
	num = str(num)
	while len(num) < length:
		num = "0" + num
	return num

def print_build_time(start_time, end_time):
	elapsed = end_time - start_time
	seconds = elapsed % 60
	elapsed = (elapsed - seconds) / 60
	minutes = elapsed % 60
	elapsed = (elapsed - minutes) / 60
	hours = elapsed % 24
	days = (elapsed - hours) / 24

	time = str(int(hours)) + " Hours, " + str(int(minutes)) + " Minutes and " + str(int(seconds)) + " Seconds"
	if days > 0:
		time = str(int(days)) + " Days, " + time

	print("Running worldbuild took " + time)

# Get exclude file
if os.path.isfile("projects/worldbuild/exclude") and exclude == []:
	try:
		with open("projects/worldbuild/exclude") as json_data:
			exclude = json.load(json_data)
	except ValueError:
		print("Exclude file 'projects/worldbuild/exclude' is no proper JSON file.")
		sys.exit(1)

start_time = time.time()

if cont != 0:
	tile = cont - 2
	tile_in_row = ((tile - 1) % 36) - 18
	ii = (tile - tile_in_row - 19) / 36 - 8
else:
	ii = -8
# Build poles first
if not "n-pole" in exclude and cont <= 1:
	run_all("n-pole", -180, 80, 180, 90, 360, threads)

if not "s-pole" in exclude and cont <= 2:
	run_all("s-pole", -180, -90, 180, -80, 360, threads)

while ii < 8:
	i = ii * 10
	if cont != 0:
		jj = tile_in_row
	else:
		jj = -18
	while jj < 18:
		j = jj * 10
		if i >= 0:
			ns = "n"
		else:
			ns = "s"
		if j >= 0:
			ew = "e"
		else:
			ew = "w"

		name = ew + norm(abs(j), 3) + ns + norm(abs(i), 2)

		if not name in exclude:
			if cont != 0:
				run_all(name, j, i, j + 10, i + 10, chunk_size, threads, cont=True)
				cont = 0
			else:
				run_all(name, j, i, j + 10, i + 10, chunk_size, threads, cont=False)
		jj += 1
	ii += 1

print_build_time(start_time, time.time())