osm2city-scripts/worldbuild-worker.py

#! /usr/bin/python3
# Copyright (C) 2018-2019 Merspieler, merspieler _at_ airmail.cc
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

import os
from subprocess import run, Popen
import sys
import socket
import re
from time import sleep

action = "pending"
host = socket.gethostname()
port = 12345
prefix = ""

argc = len(sys.argv)
i = 1
first = 1
while i < argc:
	if sys.argv[i] == "--port":
		i += 1
		port = sys.argv[i]
	elif sys.argv[i] == "--host":
		i += 1
		host = sys.argv[i]
	elif sys.argv[i] == "-p" or sys.argv[i] == "--prefix":
		i += 1
		prefix = sys.argv[i]
	elif sys.argv[i] == "-a" or sys.argv[i] == "--action":
		i += 1
		if sys.argv[i] == "pending" or sys.argv[i] == "started" or sys.argv[i] == "rebuild" or sys.argv[i] == "skip":
			action = sys.argv[i]
		else:
			print ("ERROR: Unknown action " + sys.argv[i])
			sys.exit(1)
	elif sys.argv[i] == "-h" or sys.argv[i] == "--help":
		print("usage:  build_chunk.py [OPTIONS]")
		print("Runs a single chunk build. Mainly handles status logging")
		print("")
		print("  -p, --prefix   Database prefix to use")
		print("      --host	Logger host")
		print("      --port	Logger port")
		print("  -a, --action   Considered tiles for build. Can be:")
		print("                   pending: Only builds pending tiles. These are always build: default")
		print("                   started: Run tiles marked as started. CAUTION, use with care.")
		print("                   rebuild: Build tiles flaged for rebuild")
		print("                   skip: Force building tile that are marked for being skipped")
		print("  -h, --help	Shows this help and exit")
	else:
		if first == 1:
			first = 0
			name = sys.argv[i]
		else:
			print("Unknown option " + sys.argv[i])
			sys.exit(1)
	i += 1

def norm(num, length):
	num = str(abs(num))
	while len(num) < length:
		num = "0" + num
	return num

def send_status(name, status):
	try:
		sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		sock.connect((host, port))
		sock.send(("set " + name + " " + status).encode())
		sock.close()
	except IOError:
		print("Unable to send status. Aborting...")
		sys.exit(1)

def get_job(action):
	try:
		sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		sock.connect((host, port))
		sock.send(("get " + action).encode())
		msg = sock.recv(128)
		msg = msg.decode()
		match = re.match(r"n-pole|s-pole|[ew]\d{3}[ns]\d{2}|None", msg)
		if match != None:
			ret = match.group(0)
			if ret == "None":
				print("No job got asigned. Exiting...")
				sys.exit(0)
		else:
			print("Recived invalid job. Retrying in 10 seconds...")
			sleep(10)
			ret = get_job(action)
		sock.close()
		return ret
	except IOError:
		print("Unable to get job. Retrying in 10 seconds...")
		sleep(10)
		ret = get_job(action)
		return ret

def execute_build(name, prefix):
	send_status(name, "started")

	run("mkdir -p projects/worldbuild-" + name, shell=True)

	run("cp projects/worldbuild/params.ini projects/worldbuild-" + name + "/", shell=True)

	if name == "n-pole":
		bounds = "bounds=*-180_80_180_90"
		db_name = prefix + "n-pole"
	elif name == "s-pole":
		bounds = "bounds=*-180_-90_180_-80"
		db_name = prefix + "s-pole"
	else:
		match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name)
		if match == None:
			print("ERROR: Invalid tile name")
			sys.exit(1)
		else:
			ew = match.group(1)
			ew_val = int(match.group(2))
			ns = match.group(3)
			ns_val = int(match.group(4))
	
			if ew == "w":
				ew_val *= -1
			if ns == "s":
				ns_val *= -1

			if ew_val < 0:
				bounds = "bounds=*"
			else:
				bounds = "bounds="

			bounds += str(ew_val) + "_" + str(ns_val) + "_" + str(ew_val + 10) + "_" + str(ns_val + 10)
			if ew_val % 10 != 0:
			    ew_val = ew_val - (ew_val % 10)

			if ns_val % 10 != 0:
			    ns_val = ns_val - (ns_val % 10)

			db_name = prefix + ew + norm(ew_val, 3) + ns + norm(ns_val, 2)

	run("sed -i 's/DB_NAME.*/DB_NAME = \"" + db_name + "\"/' projects/worldbuild-" + name + "/params.ini", shell=True)

	run("echo '" + bounds + "' > projects/worldbuild-" + name + "/settings", shell=True)

	build = Popen("./build -S 10 -t 1 worldbuild-" + name, shell=True, start_new_session=True)

	if os.path.isfile("projects/worldbuild-" + name + "/osm2city-exceptions.log"):
		run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + ".exceptions.log", shell=True)

	run("rm -rf projects/worldbuild-" + name, shell=True)

	send_status(name, "done")

run = True:
	while run:
		tile = get_job(action)
		execute_build(tile, prefix)