diff --git a/common.py b/common.py index 4361f29..fe1cff6 100644 --- a/common.py +++ b/common.py @@ -19,6 +19,7 @@ import math import sys import socket import re +from time import sleep # Adds leading 0s def norm(num, length): @@ -57,16 +58,22 @@ def get_tile(lat, lon): return (int(lon + 180) << 14) + (int(lat + 90) << 6) + (y << 3) + x # Returns the area name ie. e145s17 -def get_area_name(n, e): - if n >= 0: +def get_area_name(s, w, major=False): + if s >= 0: ns = "n" else: ns = "s" - if e >= 0: + if w >= 0: ew = "e" else: ew = "w" - return ew + norm(abs(e), 3) + ns + norm(abs(n), 2) + if major: + if w % 10 != 0: + w = w - (w % 10) + + if s % 10 != 0: + s = s - (s % 10) + return ew + norm(abs(w), 3) + ns + norm(abs(s), 2) # Returns latitude of tiles SW corner in area scheme def get_lat(tile): @@ -108,7 +115,7 @@ def send_status(name, status, host, port): sys.exit(1) # Gets new job from manager -def get_job(action): +def get_job(action, host, port, none_exit=True): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((host, port)) @@ -119,22 +126,22 @@ def get_job(action): match = re.match(r"n-pole|s-pole|[ew]\d{3}[ns]\d{2}|None", msg) if match != None: ret = match.group(0) - if ret == "None": + if ret == "None" and none_exit: print("No job got asigned. Exiting...") sys.exit(0) else: print("Recived invalid job. Retrying in 10 seconds...") sleep(10) - ret = get_job(action) + ret = get_job(action, host, port, none_exit) return ret except IOError: print("Unable to get job. Retrying in 10 seconds...") sleep(10) - ret = get_job(action) + ret = get_job(action, host, port, none_exit) return ret # Gets status of a tile -def get_status(name): +def get_status(name, host, port): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((host, port)) diff --git a/db b/db index 9f4a790..1342a66 100644 --- a/db +++ b/db @@ -1,3 +1,5 @@ +Structure of database `worldbuild` + topLevel secondLevel tile -------- ----------- ---- id <-| id <-| id diff --git a/flag-rebuild.py b/flag-rebuild.py index ec2e9b5..8fa9cb6 100755 --- a/flag-rebuild.py +++ b/flag-rebuild.py @@ -65,7 +65,7 @@ try: tiles = [] with open(lfile) as f: for line in f: - match = re.match(r".*[ew]\d{3}[ns]\d{2}/([ew]\d{3}[ns]\d{2}).*\.[bs]tg", line) + match = re.match(r".*[ew]\d{3}[ns]\d{2}/[ew]\d{3}[ns]\d{2}/([0-9]{1,7})\.[bs]tg", line) if match != None: tiles.append(match.group(1)) except IOError: diff --git a/init-db.py b/init-db.py index 71f9911..c438657 100755 --- a/init-db.py +++ b/init-db.py @@ -28,7 +28,7 @@ i = 1 while i < argc: if sys.argv[i] == "-p" or sys.argv[i] == "--password": i += 1 - dbpw = int(sys.argv[i]) + dbpw = sys.argv[i] elif sys.argv[i] == "-u" or sys.argv[i] == "--user": i += 1 dbuser = sys.argv[i] @@ -76,8 +76,8 @@ if force: cursor.execute("DROP TABLE IF EXISTS secondLevel") cursor.execute("DROP TABLE IF EXISTS topLevel") cursor.execute("DROP TABLE IF EXISTS status") - cursor.execute("DROP TRIGGER update_top_level") - cursor.execute("DROP TRIGGER update_second_level") + cursor.execute("DROP TRIGGER IF EXISTS update_top_level") + cursor.execute("DROP TRIGGER IF EXISTS update_second_level") cursor.execute("CREATE TABLE IF NOT EXISTS status (id INT PRIMARY KEY AUTO_INCREMENT, name VARCHAR(15) UNIQUE NOT NULL);") cursor.execute("CREATE TABLE IF NOT EXISTS topLevel (id INT PRIMARY KEY AUTO_INCREMENT, name CHAR(7) UNIQUE NOT NULL, status_id INT NOT NULL, FOREIGN KEY (status_id) REFERENCES status(id));") diff --git a/worldbuild-manager.py b/worldbuild-manager.py index 95ffa61..93a2dc6 100755 --- a/worldbuild-manager.py +++ b/worldbuild-manager.py @@ -139,18 +139,20 @@ try: set_state(name, status) elif action == "get-job": - # TODO handle other types than 'pending' - sql = "SELECT id FROM tile WHERE status_id = (SELECT id FROM status WHERE name = 'pending') LIMIT 1" + additional_type = "" + if get != "pending": + additional_type = "OR status_id = (SELECT id FROM status WHERE name = '" + get + "')" + sql = "SELECT id FROM tile WHERE status_id = (SELECT id FROM status WHERE name = 'pending') " + additional_type + " LIMIT 1" cursor.execute() result = cursor.fetchone() - # TODO handle no tile can be fetched - tile = result[0] - sql = "UPDATE tile SET status_id = (SELECT id FROM status WHERE name = 'started') WHERE id = " + tile - cursor.execute() - db.commit() - - if tile == "": + if result == None: tile = "None" + else: + tile = result[0] + sql = "UPDATE tile SET status_id = (SELECT id FROM status WHERE name = 'started') WHERE id = " + tile + cursor.execute() + db.commit() + if verbose: print("Asigning job on tile " + tile) @@ -158,13 +160,13 @@ try: c.send(tile.encode()) elif action == "get-done": sql = "SELECT id FROM secondLevel WHERE status_id = (SELECT id FROM status WHERE name = 'done') LIMIT 1" - cursor.execute() + cursor.execute(sql) result = cursor.fetchone() - tile = result[0] - # TODO handle no tile can be fetched - - if tile == "": + if result == None: tile = "None" + else: + tile = result[0] + c.send(tile.encode()) elif action == "status": match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name) diff --git a/worldbuild-packager.py b/worldbuild-packager.py index 6d3310f..172a668 100755 --- a/worldbuild-packager.py +++ b/worldbuild-packager.py @@ -127,53 +127,49 @@ def update_index(output, name, name_major): running = True while running: - name = get_job("done") - try: - if name == "n-pole": - pass - # TODO - send_status(name, "packaged") - elif name == "s-pole": - pass - # TODO - send_status(name, "packaged") - else: - match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name) - ew = match.group(1) - ew_val = int(match.group(2)) - ns = match.group(3) - ns_val = int(match.group(4)) + name = "None" + while name == "None": + name = get_job("done", host, port, none_exit=False) + if name == "None": + print("No job got asigned. Trying again in one hour.") + sleep(3600) - ew_val_major = int(ew_val / 10) * 10 - if ew == "w": - if ew_val_major != ew_val: - ew_val_major += 10 + match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name) + ew = match.group(1) + ew_val = int(match.group(2)) + ns = match.group(3) + ns_val = int(match.group(4)) - ns_val_major = int(ns_val / 10) * 10 - if ns == "s": - if ns_val_major != ns_val: - ns_val_major += 10 + ew_val_major = int(ew_val / 10) * 10 + if ew == "w": + if ew_val_major != ew_val: + ew_val_major += 10 - name_major = ew + norm(ew_val_major, 3) + ns + norm(ns_val_major, 2) + ns_val_major = int(ns_val / 10) * 10 + if ns == "s": + if ns_val_major != ns_val: + ns_val_major += 10 - run("mkdir -p " + output + "/" + name_major, shell=True) + name_major = ew + norm(ew_val_major, 3) + ns + norm(ns_val_major, 2) - run("rm -f " + output + "/" + name_major + "/" + name + ".zip", shell=True) + run("mkdir -p " + output + "/" + name_major, shell=True) - print("Packaging " + name + "...") + run("rm -f " + output + "/" + name_major + "/" + name + ".zip", shell=True) - cmd = "bash -c 'cd projects/worldbuild/scenery && zip -rq " + output + "/" + name_major + "/" + name + ".zip {Details,Pylons,Buildings,Roads}/" + name_major + "/" + name + "/*'" - with open(os.devnull, 'w') as fp: - package = Popen(cmd , shell=True, start_new_session=True, stdout=fp) + print("Packaging " + name + "...") - package.wait() + cmd = "bash -c 'cd projects/worldbuild/scenery && zip -rq " + output + "/" + name_major + "/" + name + ".zip {Details,Pylons,Buildings,Roads}/" + name_major + "/" + name + "/*'" + with open(os.devnull, 'w') as fp: + package = Popen(cmd , shell=True, start_new_session=True, stdout=fp) - print("Packaging " + name + " done") + package.wait() - update_index(output, name, name_major) + print("Packaging " + name + " done") - send_status(name, "packaged") + update_index(output, name, name_major) + + send_status(name, "packaged", host, port) except KeyboardInterrupt: print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again") diff --git a/worldbuild-runner.py b/worldbuild-runner.py index 6b101ea..d67e736 100755 --- a/worldbuild-runner.py +++ b/worldbuild-runner.py @@ -23,6 +23,7 @@ import re import sys prefix = "" +global_db = False argc = len(sys.argv) i = 1 @@ -31,13 +32,16 @@ while i < argc: if sys.argv[i] == "-p" or sys.argv[i] == "--prefix": i += 1 prefix = sys.argv[i] + elif sys.argv[i] == "-g" or sys.argv[i] == "--global-db": + global_db = True elif sys.argv[i] == "-h" or sys.argv[i] == "--help": print("usage: worldbuild-runner.py [OPTIONS]") print("Starts workers") print("") - print(" -p, --prefix Database prefix to use") + print(" -p, --prefix Database prefix to use") + print(" -g, --global-db Use global database") # TODO hand through all worker options - print(" -h, --help Shows this help and exit") + print(" -h, --help Shows this help and exit") sys.exit(0) else: print("Unknown option " + sys.argv[i]) @@ -94,7 +98,10 @@ class launcher(threading.Thread): self.run_max = count def start_worker(self): - self.running.append(Popen(["./scripts/worldbuild-worker.py", "-q", "-p", prefix], start_new_session=True)) + cmd = ["./scripts/worldbuild-worker.py", "-q", "-p", prefix] + if global_db: + cmd.append("-g") + self.running.append(Popen(cmd, start_new_session=True)) self.run_count += 1 diff --git a/worldbuild-worker.py b/worldbuild-worker.py index ac42018..424d1ec 100755 --- a/worldbuild-worker.py +++ b/worldbuild-worker.py @@ -20,15 +20,16 @@ from subprocess import run, Popen, STDOUT import sys import socket import re -from time import sleep +from time import sleep, strftime, time -from common import send_status, get_job, norm +from common import send_status, get_job, norm, get_south, get_west, get_east, get_north action = "pending" host = socket.gethostname() port = 12345 prefix = "" quiet = False +global_db = False argc = len(sys.argv) i = 1 @@ -43,6 +44,8 @@ while i < argc: elif sys.argv[i] == "-p" or sys.argv[i] == "--prefix": i += 1 prefix = sys.argv[i] + elif sys.argv[i] == "-g" or sys.argv[i] == "--global-db": + global_db = True elif sys.argv[i] == "-q" or sys.argv[i] == "--quiet": quiet = True elif sys.argv[i] == "-a" or sys.argv[i] == "--action": @@ -56,16 +59,17 @@ while i < argc: print("usage:worldbuild-worker.py [OPTIONS]") print("Retrives jobs from the manager and executes them") print("") - print(" -p, --prefix Database prefix to use") - print(" --host Manager host") - print(" --port Manager port") - print(" -q, --quiet Don't print messages") - print(" -a, --action Considered tiles for build. Can be:") - print(" pending: Only builds pending tiles. These are always build: default") - print(" started: Run tiles marked as started. CAUTION, use with care.") - print(" rebuild: Build tiles flaged for rebuild") - print(" skip: Force building tile that are marked for being skipped") - print(" -h, --help Shows this help and exit") + print(" -p, --prefix Database prefix to use") + print(" -g, --global-db Use global database") + print(" --host Manager host") + print(" --port Manager port") + print(" -q, --quiet Don't print messages") + print(" -a, --action Considered tiles for build. Can be:") + print(" pending: Only builds pending tiles. These are always build: default") + print(" started: Run tiles marked as started. CAUTION, use with care.") + print(" rebuild: Build tiles flaged for rebuild") + print(" skip: Force building tile that are marked for being skipped") + print(" -h, --help Shows this help and exit") sys.exit(0) else: print("Unknown option " + sys.argv[i]) @@ -74,55 +78,41 @@ while i < argc: def cleanup(): if os.path.isfile("projects/worldbuild-" + name + "/osm2city-exceptions.log"): - run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + ".exceptions.log", shell=True) + run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + "-" + strftime("%Y%m%d-%H%M", time()) + ".exceptions.log", shell=True) run("rm -rf projects/worldbuild-" + name, shell=True) running = True while running: - name = get_job(action) + name = get_job(action, host, port) try: run("mkdir -p projects/worldbuild-" + name, shell=True) run("cp projects/worldbuild/params.ini projects/worldbuild-" + name + "/", shell=True) - - if name == "n-pole": - bounds = "bounds=*-180_80_180_90" - db_name = prefix + "n-pole" - elif name == "s-pole": - bounds = "bounds=*-180_-90_180_-80" - db_name = prefix + "s-pole" + + if global_db: + db_name = prefix + "worldbuild" else: - match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name) - if match == None: - print("ERROR: Invalid tile name") - sys.exit(1) + if get_south(name) >= 80: + db_name = prefix + "n-pole" + elif get_south(name) < -80: + db_name = prefix + "s-pole" else: - ew = match.group(1) - ew_val = int(match.group(2)) - ns = match.group(3) - ns_val = int(match.group(4)) - - if ew == "w": - ew_val *= -1 - if ns == "s": - ns_val *= -1 - - if ew_val < 0: - bounds = "bounds=*" + match = re.match(r"[0-9]{1,7}", name) + if match == None: + print("ERROR: Invalid tile name") + sys.exit(1) else: - bounds = "bounds=" + db_name = prefix + get_area_name(name, major=True) + + if get_west(name) < 0: + bounds = "bounds=*" + else: + bounds = "bounds=" - bounds += str(ew_val) + "_" + str(ns_val) + "_" + str(ew_val + 1) + "_" + str(ns_val + 1) - if ew_val % 10 != 0: - ew_val = ew_val - (ew_val % 10) - - if ns_val % 10 != 0: - ns_val = ns_val - (ns_val % 10) - - db_name = prefix + ew + norm(ew_val, 3) + ns + norm(ns_val, 2) + bounds += str(get_west(name)) + "_" + str(get_south(name)) + "_" + str(get_east(name)) + "_" + str(get_north(name)) run("sed -i 's/DB_NAME.*/DB_NAME = \"" + db_name + "\"/' projects/worldbuild-" + name + "/params.ini", shell=True) @@ -131,14 +121,16 @@ while running: if not quiet: print("Building " + name) - with open("projects/worldbuild/output/" + name + ".log", "w") as log_file: + logpath = "projects/worldbuild/output/" + get_area_name(name, major=True) + "/" + get_area_name(name) + "/" + run("mkdir -p " + logpath, shell=True) + with open(logpath + name + "-" + strftime("%Y%m%d-%H%M", time()) + ".log", "w") as log_file: build = Popen("./build -S 10 -t 1 worldbuild-" + name, stdout=log_file, stderr=STDOUT, shell=True, start_new_session=True) build.wait() cleanup() - send_status(name, "done") + send_status(name, "done", host, port) except KeyboardInterrupt: if not quiet: print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again") @@ -146,7 +138,7 @@ while running: try: build.wait() cleanup() - send_status(name, "done") + send_status(name, "done", host, port) except KeyboardInterrupt: #TODO doesn't work print("Forcing shutdown...")