osm2city-scripts/worldbuild-worker.py

#! /usr/bin/python3
# Copyright (C) 2018-2020 Merspieler, merspieler _at_ airmail.cc
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

import os
from subprocess import run, Popen, STDOUT
import sys
import socket
import re
from time import sleep

action = "pending"
host = socket.gethostname()
port = 12345
prefix = ""

argc = len(sys.argv)
i = 1
first = 1
while i < argc:
	if sys.argv[i] == "--port":
		i += 1
		port = sys.argv[i]
	elif sys.argv[i] == "--host":
		i += 1
		host = sys.argv[i]
	elif sys.argv[i] == "-p" or sys.argv[i] == "--prefix":
		i += 1
		prefix = sys.argv[i]
	elif sys.argv[i] == "-a" or sys.argv[i] == "--action":
		i += 1
		if sys.argv[i] == "pending" or sys.argv[i] == "started" or sys.argv[i] == "rebuild" or sys.argv[i] == "skip":
			action = sys.argv[i]
		else:
			print ("ERROR: Unknown action " + sys.argv[i])
			sys.exit(1)
	elif sys.argv[i] == "-h" or sys.argv[i] == "--help":
		print("usage:worldbuild-worker.py [OPTIONS]")
		print("Retrives jobs from the manager and executes them")
		print("")
		print("  -p, --prefix   Database prefix to use")
		print("      --host     Manager host")
		print("      --port     Manager port")
		print("  -a, --action   Considered tiles for build. Can be:")
		print("                   pending: Only builds pending tiles. These are always build: default")
		print("                   started: Run tiles marked as started. CAUTION, use with care.")
		print("                   rebuild: Build tiles flaged for rebuild")
		print("                   skip: Force building tile that are marked for being skipped")
		print("  -h, --help     Shows this help and exit")
		sys.exit(0)
	else:
		if first == 1:
			first = 0
			name = sys.argv[i]
		else:
			print("Unknown option " + sys.argv[i])
			sys.exit(1)
	i += 1

def norm(num, length):
	num = str(abs(num))
	while len(num) < length:
		num = "0" + num
	return num

def send_status(name, status):
	try:
		sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		sock.connect((host, port))
		sock.send(("set " + name + " " + status).encode())
		sock.close()
	except IOError:
		print("Unable to send status. Aborting...")
		sys.exit(1)

def get_job(action):
	try:
		sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		sock.connect((host, port))
		sock.send(("get " + action).encode())
		msg = sock.recv(128)
		msg = msg.decode()
		match = re.match(r"n-pole|s-pole|[ew]\d{3}[ns]\d{2}|None", msg)
		if match != None:
			ret = match.group(0)
			if ret == "None":
				print("No job got asigned. Exiting...")
				sys.exit(0)
		else:
			print("Recived invalid job. Retrying in 10 seconds...")
			sleep(10)
			ret = get_job(action)
		sock.close()
		return ret
	except IOError:
		print("Unable to get job. Retrying in 10 seconds...")
		sleep(10)
		ret = get_job(action)
		return ret

def cleanup():
	if os.path.isfile("projects/worldbuild-" + name + "/osm2city-exceptions.log"):
		run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + ".exceptions.log", shell=True)

	run("rm -rf projects/worldbuild-" + name, shell=True)


running = True
while running:
	name = get_job(action)

	try:
		send_status(name, "started")
	
		run("mkdir -p projects/worldbuild-" + name, shell=True)
	
		run("cp projects/worldbuild/params.ini projects/worldbuild-" + name + "/", shell=True)
	
		if name == "n-pole":
			bounds = "bounds=*-180_80_180_90"
			db_name = prefix + "n-pole"
		elif name == "s-pole":
			bounds = "bounds=*-180_-90_180_-80"
			db_name = prefix + "s-pole"
		else:
			match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name)
			if match == None:
				print("ERROR: Invalid tile name")
				sys.exit(1)
			else:
				ew = match.group(1)
				ew_val = int(match.group(2))
				ns = match.group(3)
				ns_val = int(match.group(4))
		
				if ew == "w":
					ew_val *= -1
				if ns == "s":
					ns_val *= -1
	
				if ew_val < 0:
					bounds = "bounds=*"
				else:
					bounds = "bounds="
	
				bounds += str(ew_val) + "_" + str(ns_val) + "_" + str(ew_val + 1) + "_" + str(ns_val + 1)
				if ew_val % 10 != 0:
				    ew_val = ew_val - (ew_val % 10)
	
				if ns_val % 10 != 0:
				    ns_val = ns_val - (ns_val % 10)
	
				db_name = prefix + ew + norm(ew_val, 3) + ns + norm(ns_val, 2)
	
		run("sed -i 's/DB_NAME.*/DB_NAME = \"" + db_name + "\"/' projects/worldbuild-" + name + "/params.ini", shell=True)
	
		run("echo '" + bounds + "' > projects/worldbuild-" + name + "/settings", shell=True)

		print("Building " + name)

		with open("projects/worldbuild/output/" + name + ".log", "w") as log_file:
			build = Popen("./build -S 10 -t 1 worldbuild-" + name, stdout=log_file, stderr=STDOUT, shell=True, start_new_session=True)

		build.wait()

		cleanup()
	
		send_status(name, "done")
	except KeyboardInterrupt:
		print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again")
		running = False
		try:
			build.wait()
			cleanup()
			send_status(name, "done")
		except KeyboardInterrupt:
                    #TODO doesn't work
			print("Forcing shutdown...")
			build.terminate()
			sleep(5)
			build.kill()
			sleep(5)
			cleanup()
			sys.exit(0)