diff --git a/common.py b/common.py index 6f6505b..a322e82 100644 --- a/common.py +++ b/common.py @@ -58,7 +58,7 @@ def get_tile(lat, lon): x = math.floor((lon - base_x) / tile_width) return (int(lon + 180) << 14) + (int(lat + 90) << 6) + (y << 3) + x -# Returns the area name ie. e145s17 +# Returns the area name for a given lat lon ie. e145s17 def get_area_name(s, w, major=False): if s >= 0: ns = "n" @@ -76,6 +76,10 @@ def get_area_name(s, w, major=False): s = s - (s % 10) return ew + norm(abs(math.floor(w)), 3) + ns + norm(abs(math.floor(s)), 2) +# Returns the area name for a given tile +def get_area_name_by_tile(tile, major=False): + return get_area_name(get_south(tile), get_west(tile), major) + # Returns latitude of tiles SW corner in area scheme def get_lat(tile): return ((16320 & int(tile)) >> 6) - 90 @@ -164,51 +168,30 @@ def get_job(action, host, port, none_exit=True): return ret # Gets new job from manager api -def api_get_job(action, api, token, none_exit=True): +def api_get_job(action, new_status, api, token, none_exit=True): try: - # TODO 'if' is temporary until the database is reworked - if action == "done": - response = requests.post(api, data={'auth': token, 'action': 'get-done'}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"}) - if response.ok and response.json()["success"] == True: - match = re.match(r"[ew]\d{3}[ns]\d{2}|None", str(response.json()["job"])) - if match != None: - ret = match.group(0) - if ret == "None" and none_exit: - print("No job got asigned. Exiting...") - sys.exit(0) - else: - print("Recived invalid job. Retrying in 10 seconds...") - sleep(10) - ret = api_get_job(action, api, token, none_exit) - return ret + response = requests.post(api, data={'auth': token, 'action': 'get-job', 'status': action, 'new-status': new_status}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"}) + if response.ok and response.json()["success"] == True: + match = re.match(r"[ew]\d{3}[ns]\d{2}|[0-9]{1,7}|None", str(response.json()["job"])) + if match != None: + ret = match.group(0) + if ret == "None" and none_exit: + print("No job got asigned. Exiting...") + sys.exit(0) else: - print("Unable to get job. Retrying in 60 seconds...") - sleep(60) - ret = api_get_job(action, api, token, none_exit) - return ret + print("Recived invalid job. Retrying in 10 seconds...") + sleep(10) + ret = api_get_job(action, new_status, api, token, none_exit) + return ret else: - response = requests.post(api, data={'auth': token, 'action': 'get-job', 'type': action}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"}) - if response.ok and response.json()["success"] == True: - match = re.match(r"[ew]\d{3}[ns]\d{2}|[0-9]{1,7}|None", str(response.json()["job"])) - if match != None: - ret = match.group(0) - if ret == "None" and none_exit: - print("No job got asigned. Exiting...") - sys.exit(0) - else: - print("Recived invalid job. Retrying in 10 seconds...") - sleep(10) - ret = api_get_job(action, api, token, none_exit) - return ret - else: - print("Unable to get job. Retrying in 60 seconds...") - sleep(60) - ret = api_get_job(action, api, token, none_exit) - return ret + print("Unable to get job. Retrying in 60 seconds...") + sleep(60) + ret = api_get_job(action, new_status, api, token, none_exit) + return ret except (ConnectionError, OSError, IOError): print("Unable to get job. Retrying in 60 seconds...") sleep(60) - ret = api_get_job(action, api, token, none_exit) + ret = api_get_job(action, new_status, api, token, none_exit) return ret # Gets status of a tile diff --git a/worldbuild-packager.py b/worldbuild-packager.py index c0e26f2..434a08e 100755 --- a/worldbuild-packager.py +++ b/worldbuild-packager.py @@ -23,27 +23,26 @@ import socket from subprocess import run, Popen, STDOUT from time import sleep -from common import norm, send_status, get_job, api_get_job, api_send_status +from common import norm, get_area_name_by_tile, api_get_job, api_send_status host = socket.gethostname() port = 12345 api = None api_token = None +input_dir = "" output = "" +tmp_dir = "/tmp/o2c-packager" argc = len(sys.argv) i = 1 first = 1 while i < argc: - if sys.argv[i] == "--port": - i += 1 - port = sys.argv[i] - elif sys.argv[i] == "--host": - i += 1 - host = sys.argv[i] - elif sys.argv[i] == "-o" or sys.argv[i] == "--output": + if sys.argv[i] == "-o" or sys.argv[i] == "--output": i += 1 output = sys.argv[i] + if sys.argv[i] == "-i" or sys.argv[i] == "--input": + i += 1 + input_dir = sys.argv[i] elif sys.argv[i] == "-a" or sys.argv[i] == "--api": i += 1 api = sys.argv[i] @@ -54,9 +53,8 @@ while i < argc: print("usage: worldbuild-packager.py [OPTIONS]") print("Retrives done tiles from the manager and packages them") print("") + print(" -i --input Input directory. Path MUST be absolute") print(" -o, --output Output directory. Path MUST be absolute") - print(" --host Manager host") - print(" --port Manager port") print(" -a, --api Manager api url") print(" -t, --api-token Manager api token") print(" -h, --help Shows this help and exit") @@ -74,6 +72,17 @@ if output == "": print("ERROR: No output directory given") sys.exit(1) +if input_dir == "": + print("ERROR: No input directory given") + sys.exit(1) + +if not api: + print("ERROR: No api given") + sys.exit(1) + +if not api_token: + print("ERROR: No api token given") + def get_sha1(fname): if os.path.isfile(fname): sha1 = hashlib.sha1() @@ -146,64 +155,54 @@ while running: try: name = "None" while name == "None": - if api == None: - name = get_job("done", host, port, none_exit=False) - else: - name = api_get_job("done", api, api_token, none_exit=False) + name = api_get_job("done", "packaging", api, api_token, none_exit=False) if name == "None": print("No job got asigned. Trying again in one hour.") sleep(3600) - match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name) - ew = match.group(1) - ew_val = int(match.group(2)) - ns = match.group(3) - ns_val = int(match.group(4)) - - ew_val_major = int(ew_val / 10) * 10 - if ew == "w": - if ew_val_major != ew_val: - ew_val_major += 10 - - ns_val_major = int(ns_val / 10) * 10 - if ns == "s": - if ns_val_major != ns_val: - ns_val_major += 10 - - name_major = ew + norm(ew_val_major, 3) + ns + norm(ns_val_major, 2) - print("Packaging " + name + "...") - for part in ["Buildings", "Details", "Pylons", "Roads"]: - run("rm -f " + output + "/" + part + "/" + name_major + "/" + name + ".txz", shell=True) - if os.path.exists("projects/worldbuild/scenery/" + part + "/" + name_major + "/" + name): - run("mkdir -p " + output + "/" + part + "/" + name_major, shell=True) + area = get_area_name_by_tile(name) + area_major = get_area_name_by_tile(name, major=True) - cmd = "bash -c 'cd projects/worldbuild/scenery/" + part + "/" + name_major + " && tar -cJf " + output + "/" + part + "/" + name_major + "/" + name + ".txz " + name + "/'" + for part in os.listdir(input_dir): +#run("rm -f " + output + "/" + part + "/" + area_major + "/" + area + ".txz", shell=True) + run("mkdir -p " + tmp_dir + "/" + area) + + # If there is an archive, we need to extract it first and remove old data + if os.path.exists(output + "/" + part + "/" + area_major + "/" + area + ".txz"): + run("bash -c 'cd " + tmp_dir + "/" + area + " && tar -xf " + output + "/" + part + "/" + area_major + "/" + area + ".txz && rm *" + name + "*'") + + # If there's new scenery copy it into the tmp dir + if os.path.exists(input_dir + "/" + part + "/" + area_major + "/" + area + "/" + name + ".stg"): + run("cp " + input_dir + "/" + part + "/" + area_major + "/" + area + "/*" + name + "* " + tmp_dir + "/" + area + "/") + + # If the scenery is not empty, package it + if os.listdir(tmp_dir + "/" + area): + run("mkdir -p " + output + "/" + part + "/" + area_major, shell=True) + + cmd = "bash -c 'cd " + tmp_dir + " && tar -cJf " + output + "/" + part + "/" + area_major + "/" + area + ".txz " + area + "/'" with open(os.devnull, 'w') as fp: package = Popen(cmd , shell=True, start_new_session=True, stdout=fp) package.wait() - update_index(output, part, name, name_major) + update_index(output, part, area, area_major) + + # Clean up + run("rm -r " + tmp_dir + "/" + area) print("Packaging " + name + " done") - if api == None: - send_status(name, "packaged", host, port) - else: - api_send_status(name, "packaged", api, api_token) + api_send_status(name, "packaged", api, api_token) except KeyboardInterrupt: print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again") running = False try: package.wait() - update_index(output, name, name_major) - if api == None: - send_status(name, "packaged", host, port) - else: - api_send_status(name, "packaged", api, api_token) + update_index(output, area, name_major) + api_send_status(name, "packaged", api, api_token) except KeyboardInterrupt: # TODO doesn't work print("Forcing shutdown...") diff --git a/worldbuild-worker.py b/worldbuild-worker.py index 9b108ee..362daee 100755 --- a/worldbuild-worker.py +++ b/worldbuild-worker.py @@ -25,10 +25,8 @@ from time import sleep, strftime, time from common import send_status, get_job, norm, get_south, get_west, get_east, get_north, get_area_name, api_send_status, api_get_job action = "pending" -host = socket.gethostname() api = None api_token = None -port = 12345 prefix = "" quiet = False global_db = False @@ -37,13 +35,7 @@ argc = len(sys.argv) i = 1 first = 1 while i < argc: - if sys.argv[i] == "--port": - i += 1 - port = sys.argv[i] - elif sys.argv[i] == "--host": - i += 1 - host = sys.argv[i] - elif sys.argv[i] == "-a" or sys.argv[i] == "--api": + if sys.argv[i] == "-a" or sys.argv[i] == "--api": i += 1 api = sys.argv[i] elif sys.argv[i] == "-t" or sys.argv[i] == "--api-token": @@ -69,8 +61,6 @@ while i < argc: print("") print(" -p, --prefix Database prefix to use") print(" -g, --global-db Use global database") - print(" --host Manager host") - print(" --port Manager port") print(" -a, --api Manager api url") print(" -t, --api-token Manager api token") print(" -q, --quiet Don't print messages") @@ -86,8 +76,12 @@ while i < argc: sys.exit(1) i += 1 -if api != None and api_token == None: - print("Error: API given but no token") +if api != None: + print("Error: No API url given") + sys.exit(1) + +if api_token == None: + print("Error: No API token given") sys.exit(1) def cleanup(name): @@ -103,10 +97,7 @@ name = None try: running = True while running: - if api != None: - name = api_get_job(action, api, api_token) - else: - name = get_job(action, host, port) + name = api_get_job(action, "started", api, api_token) run("mkdir -p projects/worldbuild-" + name, shell=True) @@ -151,10 +142,7 @@ try: cleanup(name) - if api != None: - api_send_status(name, "done", api, api_token) - else: - send_status(name, "done", host, port) + api_send_status(name, "done", api, api_token) except KeyboardInterrupt: if not quiet: print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again") @@ -163,10 +151,7 @@ except KeyboardInterrupt: if build != None: build.wait() cleanup(name) - if api != None: - api_send_status(name, "done", api, api_token) - else: - send_status(name, "done", host, port) + api_send_status(name, "done", api, api_token) except KeyboardInterrupt: #TODO doesn't work print("Forcing shutdown...")