From e08fbea7370c15c2d60c75360b1a45d5f5d57caf Mon Sep 17 00:00:00 2001 From: fly Date: Wed, 15 Jan 2020 11:33:31 +0000 Subject: [PATCH] Bunch of fixes Signed-off-by: fly --- worldbuild-manager.py | 16 +++--- worldbuild-worker.py | 127 +++++++++++++++++++++++------------------- 2 files changed, 79 insertions(+), 64 deletions(-) diff --git a/worldbuild-manager.py b/worldbuild-manager.py index 6ddcb16..361b272 100755 --- a/worldbuild-manager.py +++ b/worldbuild-manager.py @@ -41,9 +41,8 @@ while i < argc: i += 1 sfile = sys.argv[i] elif sys.argv[i] == "-h" or sys.argv[i] == "--help": - print("usage: wb-progress-logger.py [OPTIONS]") - print("Logs progress of the worldbuild when run") - print("with database strategies 'chunk' and 'mono'") + print("usage: worldbuild-manager.py [OPTIONS]") + print("Handles job asignments and keeps track of the status") print("") print("OPTIONS") print(" -f, --file Status file to use") @@ -51,6 +50,7 @@ while i < argc: print(" , --host hostname or interface to bind to") print(" , --port Port to bind to") print(" -h, --help Shows this help and exit") + sys.exit(0) else: print("Unknown option " + sys.argv[i]) sys.exit(1) @@ -62,7 +62,7 @@ def norm(num, length): num = "0" + num return num -def save_state(state) +def save_state(state): try: with open(sfile, 'w') as f: json.dump(state, f, indent=4) @@ -165,9 +165,9 @@ try: elif action == "get": tile = "" # Build poles first - if not "n-pole" in status or ("n-pole" in status and status["n-pole"]["status"] == get): + if not "n-pole" in state or ("n-pole" in state and state["n-pole"]["status"] == get): tile = "n-pole" - elif not "s-pole" in status or ("s-pole" in status and status["s-pole"]["status"] == get): + elif not "s-pole" in state or ("s-pole" in state and state["s-pole"]["status"] == get): tile = "s-pole" else: ii = -8 @@ -188,7 +188,7 @@ try: name = ew + norm(abs(j), 3) + ns + norm(abs(i), 2) - if not name in status or (name in status and status[name]["status"] != "done"): + if not name in state or (name in state and state[name]["status"] != "done"): if ns == "s": ns_step = -1 else: @@ -204,7 +204,7 @@ try: iii = abs(i) for l in range(0, 10): name_minor = ew + norm(j, 3) + ns + norm(iii, 2) - if not name in status or (not name_minor in status[name] or (name_minor in status[name] and (status[name][name_minor]["status"] == get))): + if not name in state or (not name_minor in state[name] or (name_minor in state[name] and (state[name][name_minor]["status"] == get))): tile = name_minor break iii += ns_step diff --git a/worldbuild-worker.py b/worldbuild-worker.py index 8245f94..05dc10a 100755 --- a/worldbuild-worker.py +++ b/worldbuild-worker.py @@ -48,8 +48,8 @@ while i < argc: print ("ERROR: Unknown action " + sys.argv[i]) sys.exit(1) elif sys.argv[i] == "-h" or sys.argv[i] == "--help": - print("usage: build_chunk.py [OPTIONS]") - print("Runs a single chunk build. Mainly handles status logging") + print("usage:worldbuild-worker.py [OPTIONS]") + print("Retrives jobs from the manager and executes them") print("") print(" -p, --prefix Database prefix to use") print(" --host Logger host") @@ -60,6 +60,7 @@ while i < argc: print(" rebuild: Build tiles flaged for rebuild") print(" skip: Force building tile that are marked for being skipped") print(" -h, --help Shows this help and exit") + sys.exit(0) else: if first == 1: first = 0 @@ -110,63 +111,77 @@ def get_job(action): ret = get_job(action) return ret -def execute_build(name, prefix): - send_status(name, "started") - run("mkdir -p projects/worldbuild-" + name, shell=True) +running = True +while running: + name = get_job(action) - run("cp projects/worldbuild/params.ini projects/worldbuild-" + name + "/", shell=True) - - if name == "n-pole": - bounds = "bounds=*-180_80_180_90" - db_name = prefix + "n-pole" - elif name == "s-pole": - bounds = "bounds=*-180_-90_180_-80" - db_name = prefix + "s-pole" - else: - match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name) - if match == None: - print("ERROR: Invalid tile name") - sys.exit(1) - else: - ew = match.group(1) - ew_val = int(match.group(2)) - ns = match.group(3) - ns_val = int(match.group(4)) + try: + send_status(name, "started") - if ew == "w": - ew_val *= -1 - if ns == "s": - ns_val *= -1 - - if ew_val < 0: - bounds = "bounds=*" + run("mkdir -p projects/worldbuild-" + name, shell=True) + + run("cp projects/worldbuild/params.ini projects/worldbuild-" + name + "/", shell=True) + + if name == "n-pole": + bounds = "bounds=*-180_80_180_90" + db_name = prefix + "n-pole" + elif name == "s-pole": + bounds = "bounds=*-180_-90_180_-80" + db_name = prefix + "s-pole" + else: + match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name) + if match == None: + print("ERROR: Invalid tile name") + sys.exit(1) else: - bounds = "bounds=" + ew = match.group(1) + ew_val = int(match.group(2)) + ns = match.group(3) + ns_val = int(match.group(4)) + + if ew == "w": + ew_val *= -1 + if ns == "s": + ns_val *= -1 + + if ew_val < 0: + bounds = "bounds=*" + else: + bounds = "bounds=" + + bounds += str(ew_val) + "_" + str(ns_val) + "_" + str(ew_val + 10) + "_" + str(ns_val + 10) + if ew_val % 10 != 0: + ew_val = ew_val - (ew_val % 10) + + if ns_val % 10 != 0: + ns_val = ns_val - (ns_val % 10) + + db_name = prefix + ew + norm(ew_val, 3) + ns + norm(ns_val, 2) + + run("sed -i 's/DB_NAME.*/DB_NAME = \"" + db_name + "\"/' projects/worldbuild-" + name + "/params.ini", shell=True) + + run("echo '" + bounds + "' > projects/worldbuild-" + name + "/settings", shell=True) - bounds += str(ew_val) + "_" + str(ns_val) + "_" + str(ew_val + 10) + "_" + str(ns_val + 10) - if ew_val % 10 != 0: - ew_val = ew_val - (ew_val % 10) + build = Popen("./build -S 10 -t 1 worldbuild-" + name, shell=True, start_new_session=True) - if ns_val % 10 != 0: - ns_val = ns_val - (ns_val % 10) - - db_name = prefix + ew + norm(ew_val, 3) + ns + norm(ns_val, 2) - - run("sed -i 's/DB_NAME.*/DB_NAME = \"" + db_name + "\"/' projects/worldbuild-" + name + "/params.ini", shell=True) - - run("echo '" + bounds + "' > projects/worldbuild-" + name + "/settings", shell=True) - - build = Popen("./build -S 10 -t 1 worldbuild-" + name, shell=True, start_new_session=True) - - if os.path.isfile("projects/worldbuild-" + name + "/osm2city-exceptions.log"): - run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + ".exceptions.log", shell=True) - - run("rm -rf projects/worldbuild-" + name, shell=True) - - send_status(name, "done") - -run = True: - while run: - tile = get_job(action) - execute_build(tile, prefix) + if os.path.isfile("projects/worldbuild-" + name + "/osm2city-exceptions.log"): + run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + ".exceptions.log", shell=True) + + sleep(10) + run("rm -rf projects/worldbuild-" + name, shell=True) + + send_status(name, "done") + except KeyboardInterrupt: + print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again") + running = False + try: + build.wait() + except KeyboardInterrupt: + #TODO doesn't work + print("Forcing shutdown...") + build.terminate() + sleep(5) + build.kill() + sleep(5) + sys.exit(0)