fly/osm2city-scripts
1
0
Fork 0
Code Issues 2 Pull requests Activity

Improve packaging by grouping all tiles within a parent

Browse source 
Signed-off-by: fly <merspieler@airmail.cc>
This commit is contained in:
fly 2021-06-18 23:57:57 +02:00
parent f36c656e0e
commit 75ff8d8ba1
2 changed files with 26 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
common.py
View file

@ -169,9 +169,9 @@ def get_job(action, host, port, none_exit=True):
return ret return ret
# Gets new job from manager api # Gets new job from manager api
def api_get_job(action, new_status, api, token, none_exit=True): def api_get_job(action, new_status, api, token, none_exit=True, all_in_parent=0):
try: try:
response = requests.post(api, data={'auth': token, 'action': 'get-job', 'status': action, 'new-status': new_status}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"}) response = requests.post(api, data={'auth': token, 'action': 'get-job', 'status': action, 'new-status': new_status, "all-in-parent": all_in_parent}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
if response.ok and response.json()["success"] == True: if response.ok and response.json()["success"] == True:
match = re.match(r"[0-9]{1,7}|None", str(response.json()["job"])) match = re.match(r"[0-9]{1,7}|None", str(response.json()["job"]))
if match != None: if match != None:
@ -179,20 +179,22 @@ def api_get_job(action, new_status, api, token, none_exit=True):
if ret == "None" and none_exit: if ret == "None" and none_exit:
print("No job got asigned. Exiting...") print("No job got asigned. Exiting...")
sys.exit(0) sys.exit(0)
if ret != "None" and all_in_parent == 1:
ret = response.json()["jobs"]
else: else:
print("Recived invalid job. Retrying in 10 seconds...") print("Recived invalid job. Retrying in 10 seconds...")
sleep(10) sleep(10)
ret = api_get_job(action, new_status, api, token, none_exit) ret = api_get_job(action, new_status, api, token, none_exit, all_in_parent)
return ret return ret
else: else:
print("Unable to get job. Retrying in 60 seconds...") print("Unable to get job. Retrying in 60 seconds...")
sleep(60) sleep(60)
ret = api_get_job(action, new_status, api, token, none_exit) ret = api_get_job(action, new_status, api, token, none_exit, all_in_parent)
return ret return ret
except (ConnectionError, OSError, IOError, json.decoder.JSONDecodeError): except (ConnectionError, OSError, IOError, json.decoder.JSONDecodeError):
print("Unable to get job. Retrying in 60 seconds...") print("Unable to get job. Retrying in 60 seconds...")
sleep(60) sleep(60)
ret = api_get_job(action, new_status, api, token, none_exit) ret = api_get_job(action, new_status, api, token, none_exit, all_in_parent)
return ret return ret
# Gets status of a tile # Gets status of a tile

26
worldbuild-packager.py
View file

@ -159,18 +159,17 @@ previous_area = ""
running = True running = True
while running: while running:
try: try:
name = "None" tiles = "None"
while name == "None": while tiles == "None":
name = api_get_job("done", "packaging", api, api_token, none_exit=False) tiles = api_get_job("done", "packaging", api, api_token, none_exit=False, all_in_parent=1)
if name == "None": if tiles == "None":
print("No job got asigned. Trying again in one hour.") print("No job got asigned. Trying again in one hour.")
sleep(3600) sleep(3600)
print("Packaging " + name + "...") area = get_area_name_by_tile(tiles[0])
area_major = get_area_name_by_tile(tiles[0], major=True)
area = get_area_name_by_tile(name)
area_major = get_area_name_by_tile(name, major=True)
print("Packaging " + area + "...")
for part in os.listdir(input_dir): for part in os.listdir(input_dir):
tmp_dir_part = tmp_dir + "/" + part tmp_dir_part = tmp_dir + "/" + part
@ -180,10 +179,15 @@ while running:
run("rm -rf " + tmp_dir_part + "/" + area, shell=True) run("rm -rf " + tmp_dir_part + "/" + area, shell=True)
# If there is an archive, we need to extract it first and remove old data # If there is an archive, we need to extract it first and remove old data
if os.path.exists(output + "/" + part + "/" + area_major + "/" + area + ".txz"): if os.path.exists(output + "/" + part + "/" + area_major + "/" + area + ".txz"):
run("bash -c 'cd " + tmp_dir_part + " && tar -xf " + output + "/" + part + "/" + area_major + "/" + area + ".txz && rm -f " + area + "/*" + name + "*'", shell=True) run("bash -c 'cd " + tmp_dir_part + " && tar -xf " + output + "/" + part + "/" + area_major + "/" + area + ".txz'", shell=True)
for name in tiles:
name = str(name)
run("bash -c 'cd " + tmp_dir_part + " && rm -f " + area + "/*" + name + "*'", shell=True)
run("mkdir -p " + tmp_dir_part + "/" + area, shell=True) run("mkdir -p " + tmp_dir_part + "/" + area, shell=True)
# If there's new scenery copy it into the tmp dir # If there's new scenery copy it into the tmp dir
for name in tiles:
name = str(name)
if os.path.exists(input_dir + "/" + part + "/" + area_major + "/" + area + "/" + name + ".stg"): if os.path.exists(input_dir + "/" + part + "/" + area_major + "/" + area + "/" + name + ".stg"):
run("cp " + input_dir + "/" + part + "/" + area_major + "/" + area + "/*" + name + "* " + tmp_dir_part + "/" + area + "/", shell=True) run("cp " + input_dir + "/" + part + "/" + area_major + "/" + area + "/*" + name + "* " + tmp_dir_part + "/" + area + "/", shell=True)
@ -200,9 +204,11 @@ while running:
update_index(output, part, area, area_major) update_index(output, part, area, area_major)
print("Packaging " + name + " done") print("Packaging " + area + " done")
previous_area = area previous_area = area
for name in tiles:
name = str(name)
api_send_status(name, "packaged", api, api_token) api_send_status(name, "packaged", api, api_token)
except KeyboardInterrupt: except KeyboardInterrupt: