#! /usr/bin/python3 # Copyright (C) 2018-2020 Merspieler, merspieler _at_ airmail.cc # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 3 of the # License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. import os from subprocess import run, Popen, STDOUT import sys import socket import re from time import sleep action = "pending" host = socket.gethostname() port = 12345 prefix = "" argc = len(sys.argv) i = 1 first = 1 while i < argc: if sys.argv[i] == "--port": i += 1 port = sys.argv[i] elif sys.argv[i] == "--host": i += 1 host = sys.argv[i] elif sys.argv[i] == "-p" or sys.argv[i] == "--prefix": i += 1 prefix = sys.argv[i] elif sys.argv[i] == "-a" or sys.argv[i] == "--action": i += 1 if sys.argv[i] == "pending" or sys.argv[i] == "started" or sys.argv[i] == "rebuild" or sys.argv[i] == "skip": action = sys.argv[i] else: print ("ERROR: Unknown action " + sys.argv[i]) sys.exit(1) elif sys.argv[i] == "-h" or sys.argv[i] == "--help": print("usage:worldbuild-worker.py [OPTIONS]") print("Retrives jobs from the manager and executes them") print("") print(" -p, --prefix Database prefix to use") print(" --host Manager host") print(" --port Manager port") print(" -a, --action Considered tiles for build. Can be:") print(" pending: Only builds pending tiles. These are always build: default") print(" started: Run tiles marked as started. CAUTION, use with care.") print(" rebuild: Build tiles flaged for rebuild") print(" skip: Force building tile that are marked for being skipped") print(" -h, --help Shows this help and exit") sys.exit(0) else: if first == 1: first = 0 name = sys.argv[i] else: print("Unknown option " + sys.argv[i]) sys.exit(1) i += 1 def norm(num, length): num = str(abs(num)) while len(num) < length: num = "0" + num return num def send_status(name, status): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((host, port)) sock.send(("set " + name + " " + status).encode()) sock.close() except IOError: print("Unable to send status. Aborting...") sys.exit(1) def get_job(action): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((host, port)) sock.send(("get " + action).encode()) msg = sock.recv(128) msg = msg.decode() match = re.match(r"n-pole|s-pole|[ew]\d{3}[ns]\d{2}|None", msg) if match != None: ret = match.group(0) if ret == "None": print("No job got asigned. Exiting...") sys.exit(0) else: print("Recived invalid job. Retrying in 10 seconds...") sleep(10) ret = get_job(action) sock.close() return ret except IOError: print("Unable to get job. Retrying in 10 seconds...") sleep(10) ret = get_job(action) return ret def cleanup(): if os.path.isfile("projects/worldbuild-" + name + "/osm2city-exceptions.log"): run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + ".exceptions.log", shell=True) run("rm -rf projects/worldbuild-" + name, shell=True) running = True while running: name = get_job(action) try: run("mkdir -p projects/worldbuild-" + name, shell=True) run("cp projects/worldbuild/params.ini projects/worldbuild-" + name + "/", shell=True) if name == "n-pole": bounds = "bounds=*-180_80_180_90" db_name = prefix + "n-pole" elif name == "s-pole": bounds = "bounds=*-180_-90_180_-80" db_name = prefix + "s-pole" else: match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name) if match == None: print("ERROR: Invalid tile name") sys.exit(1) else: ew = match.group(1) ew_val = int(match.group(2)) ns = match.group(3) ns_val = int(match.group(4)) if ew == "w": ew_val *= -1 if ns == "s": ns_val *= -1 if ew_val < 0: bounds = "bounds=*" else: bounds = "bounds=" bounds += str(ew_val) + "_" + str(ns_val) + "_" + str(ew_val + 1) + "_" + str(ns_val + 1) if ew_val % 10 != 0: ew_val = ew_val - (ew_val % 10) if ns_val % 10 != 0: ns_val = ns_val - (ns_val % 10) db_name = prefix + ew + norm(ew_val, 3) + ns + norm(ns_val, 2) run("sed -i 's/DB_NAME.*/DB_NAME = \"" + db_name + "\"/' projects/worldbuild-" + name + "/params.ini", shell=True) run("echo '" + bounds + "' > projects/worldbuild-" + name + "/settings", shell=True) print("Building " + name) with open("projects/worldbuild/output/" + name + ".log", "w") as log_file: build = Popen("./build -S 10 -t 1 worldbuild-" + name, stdout=log_file, stderr=STDOUT, shell=True, start_new_session=True) build.wait() cleanup() send_status(name, "done") except KeyboardInterrupt: print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again") running = False try: build.wait() cleanup() send_status(name, "done") except KeyboardInterrupt: #TODO doesn't work print("Forcing shutdown...") build.terminate() sleep(5) build.kill() sleep(5) cleanup() sys.exit(0)