From 75ff8d8ba1dd727e6b3fd7dca962bfe637fd9faf Mon Sep 17 00:00:00 2001 From: fly Date: Fri, 18 Jun 2021 23:57:57 +0200 Subject: [PATCH] Improve packaging by grouping all tiles within a parent Signed-off-by: fly --- common.py | 12 +++++++----- worldbuild-packager.py | 32 +++++++++++++++++++------------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/common.py b/common.py index 30286e4..4bc2341 100644 --- a/common.py +++ b/common.py @@ -169,9 +169,9 @@ def get_job(action, host, port, none_exit=True): return ret # Gets new job from manager api -def api_get_job(action, new_status, api, token, none_exit=True): +def api_get_job(action, new_status, api, token, none_exit=True, all_in_parent=0): try: - response = requests.post(api, data={'auth': token, 'action': 'get-job', 'status': action, 'new-status': new_status}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"}) + response = requests.post(api, data={'auth': token, 'action': 'get-job', 'status': action, 'new-status': new_status, "all-in-parent": all_in_parent}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"}) if response.ok and response.json()["success"] == True: match = re.match(r"[0-9]{1,7}|None", str(response.json()["job"])) if match != None: @@ -179,20 +179,22 @@ def api_get_job(action, new_status, api, token, none_exit=True): if ret == "None" and none_exit: print("No job got asigned. Exiting...") sys.exit(0) + if ret != "None" and all_in_parent == 1: + ret = response.json()["jobs"] else: print("Recived invalid job. Retrying in 10 seconds...") sleep(10) - ret = api_get_job(action, new_status, api, token, none_exit) + ret = api_get_job(action, new_status, api, token, none_exit, all_in_parent) return ret else: print("Unable to get job. Retrying in 60 seconds...") sleep(60) - ret = api_get_job(action, new_status, api, token, none_exit) + ret = api_get_job(action, new_status, api, token, none_exit, all_in_parent) return ret except (ConnectionError, OSError, IOError, json.decoder.JSONDecodeError): print("Unable to get job. Retrying in 60 seconds...") sleep(60) - ret = api_get_job(action, new_status, api, token, none_exit) + ret = api_get_job(action, new_status, api, token, none_exit, all_in_parent) return ret # Gets status of a tile diff --git a/worldbuild-packager.py b/worldbuild-packager.py index 2371ec4..9f99b90 100755 --- a/worldbuild-packager.py +++ b/worldbuild-packager.py @@ -159,18 +159,17 @@ previous_area = "" running = True while running: try: - name = "None" - while name == "None": - name = api_get_job("done", "packaging", api, api_token, none_exit=False) - if name == "None": + tiles = "None" + while tiles == "None": + tiles = api_get_job("done", "packaging", api, api_token, none_exit=False, all_in_parent=1) + if tiles == "None": print("No job got asigned. Trying again in one hour.") sleep(3600) - print("Packaging " + name + "...") - - area = get_area_name_by_tile(name) - area_major = get_area_name_by_tile(name, major=True) + area = get_area_name_by_tile(tiles[0]) + area_major = get_area_name_by_tile(tiles[0], major=True) + print("Packaging " + area + "...") for part in os.listdir(input_dir): tmp_dir_part = tmp_dir + "/" + part @@ -180,12 +179,17 @@ while running: run("rm -rf " + tmp_dir_part + "/" + area, shell=True) # If there is an archive, we need to extract it first and remove old data if os.path.exists(output + "/" + part + "/" + area_major + "/" + area + ".txz"): - run("bash -c 'cd " + tmp_dir_part + " && tar -xf " + output + "/" + part + "/" + area_major + "/" + area + ".txz && rm -f " + area + "/*" + name + "*'", shell=True) + run("bash -c 'cd " + tmp_dir_part + " && tar -xf " + output + "/" + part + "/" + area_major + "/" + area + ".txz'", shell=True) + for name in tiles: + name = str(name) + run("bash -c 'cd " + tmp_dir_part + " && rm -f " + area + "/*" + name + "*'", shell=True) run("mkdir -p " + tmp_dir_part + "/" + area, shell=True) # If there's new scenery copy it into the tmp dir - if os.path.exists(input_dir + "/" + part + "/" + area_major + "/" + area + "/" + name + ".stg"): - run("cp " + input_dir + "/" + part + "/" + area_major + "/" + area + "/*" + name + "* " + tmp_dir_part + "/" + area + "/", shell=True) + for name in tiles: + name = str(name) + if os.path.exists(input_dir + "/" + part + "/" + area_major + "/" + area + "/" + name + ".stg"): + run("cp " + input_dir + "/" + part + "/" + area_major + "/" + area + "/*" + name + "* " + tmp_dir_part + "/" + area + "/", shell=True) # If the scenery is not empty, package it if os.listdir(tmp_dir_part + "/" + area): @@ -200,10 +204,12 @@ while running: update_index(output, part, area, area_major) - print("Packaging " + name + " done") + print("Packaging " + area + " done") previous_area = area - api_send_status(name, "packaged", api, api_token) + for name in tiles: + name = str(name) + api_send_status(name, "packaged", api, api_token) except KeyboardInterrupt: print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again")