Updated packager and worker for api changes
Signed-off-by: fly <merspieler@airmail.cc>
This commit is contained in:
parent
387f4fc521
commit
3f74daf555
3 changed files with 79 additions and 112 deletions
63
common.py
63
common.py
|
|
@ -58,7 +58,7 @@ def get_tile(lat, lon):
|
|||
x = math.floor((lon - base_x) / tile_width)
|
||||
return (int(lon + 180) << 14) + (int(lat + 90) << 6) + (y << 3) + x
|
||||
|
||||
# Returns the area name ie. e145s17
|
||||
# Returns the area name for a given lat lon ie. e145s17
|
||||
def get_area_name(s, w, major=False):
|
||||
if s >= 0:
|
||||
ns = "n"
|
||||
|
|
@ -76,6 +76,10 @@ def get_area_name(s, w, major=False):
|
|||
s = s - (s % 10)
|
||||
return ew + norm(abs(math.floor(w)), 3) + ns + norm(abs(math.floor(s)), 2)
|
||||
|
||||
# Returns the area name for a given tile
|
||||
def get_area_name_by_tile(tile, major=False):
|
||||
return get_area_name(get_south(tile), get_west(tile), major)
|
||||
|
||||
# Returns latitude of tiles SW corner in area scheme
|
||||
def get_lat(tile):
|
||||
return ((16320 & int(tile)) >> 6) - 90
|
||||
|
|
@ -164,51 +168,30 @@ def get_job(action, host, port, none_exit=True):
|
|||
return ret
|
||||
|
||||
# Gets new job from manager api
|
||||
def api_get_job(action, api, token, none_exit=True):
|
||||
def api_get_job(action, new_status, api, token, none_exit=True):
|
||||
try:
|
||||
# TODO 'if' is temporary until the database is reworked
|
||||
if action == "done":
|
||||
response = requests.post(api, data={'auth': token, 'action': 'get-done'}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||
if response.ok and response.json()["success"] == True:
|
||||
match = re.match(r"[ew]\d{3}[ns]\d{2}|None", str(response.json()["job"]))
|
||||
if match != None:
|
||||
ret = match.group(0)
|
||||
if ret == "None" and none_exit:
|
||||
print("No job got asigned. Exiting...")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("Recived invalid job. Retrying in 10 seconds...")
|
||||
sleep(10)
|
||||
ret = api_get_job(action, api, token, none_exit)
|
||||
return ret
|
||||
response = requests.post(api, data={'auth': token, 'action': 'get-job', 'status': action, 'new-status': new_status}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||
if response.ok and response.json()["success"] == True:
|
||||
match = re.match(r"[ew]\d{3}[ns]\d{2}|[0-9]{1,7}|None", str(response.json()["job"]))
|
||||
if match != None:
|
||||
ret = match.group(0)
|
||||
if ret == "None" and none_exit:
|
||||
print("No job got asigned. Exiting...")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("Unable to get job. Retrying in 60 seconds...")
|
||||
sleep(60)
|
||||
ret = api_get_job(action, api, token, none_exit)
|
||||
return ret
|
||||
print("Recived invalid job. Retrying in 10 seconds...")
|
||||
sleep(10)
|
||||
ret = api_get_job(action, new_status, api, token, none_exit)
|
||||
return ret
|
||||
else:
|
||||
response = requests.post(api, data={'auth': token, 'action': 'get-job', 'type': action}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||
if response.ok and response.json()["success"] == True:
|
||||
match = re.match(r"[ew]\d{3}[ns]\d{2}|[0-9]{1,7}|None", str(response.json()["job"]))
|
||||
if match != None:
|
||||
ret = match.group(0)
|
||||
if ret == "None" and none_exit:
|
||||
print("No job got asigned. Exiting...")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("Recived invalid job. Retrying in 10 seconds...")
|
||||
sleep(10)
|
||||
ret = api_get_job(action, api, token, none_exit)
|
||||
return ret
|
||||
else:
|
||||
print("Unable to get job. Retrying in 60 seconds...")
|
||||
sleep(60)
|
||||
ret = api_get_job(action, api, token, none_exit)
|
||||
return ret
|
||||
print("Unable to get job. Retrying in 60 seconds...")
|
||||
sleep(60)
|
||||
ret = api_get_job(action, new_status, api, token, none_exit)
|
||||
return ret
|
||||
except (ConnectionError, OSError, IOError):
|
||||
print("Unable to get job. Retrying in 60 seconds...")
|
||||
sleep(60)
|
||||
ret = api_get_job(action, api, token, none_exit)
|
||||
ret = api_get_job(action, new_status, api, token, none_exit)
|
||||
return ret
|
||||
|
||||
# Gets status of a tile
|
||||
|
|
|
|||
|
|
@ -23,27 +23,26 @@ import socket
|
|||
from subprocess import run, Popen, STDOUT
|
||||
from time import sleep
|
||||
|
||||
from common import norm, send_status, get_job, api_get_job, api_send_status
|
||||
from common import norm, get_area_name_by_tile, api_get_job, api_send_status
|
||||
|
||||
host = socket.gethostname()
|
||||
port = 12345
|
||||
api = None
|
||||
api_token = None
|
||||
input_dir = ""
|
||||
output = ""
|
||||
tmp_dir = "/tmp/o2c-packager"
|
||||
|
||||
argc = len(sys.argv)
|
||||
i = 1
|
||||
first = 1
|
||||
while i < argc:
|
||||
if sys.argv[i] == "--port":
|
||||
i += 1
|
||||
port = sys.argv[i]
|
||||
elif sys.argv[i] == "--host":
|
||||
i += 1
|
||||
host = sys.argv[i]
|
||||
elif sys.argv[i] == "-o" or sys.argv[i] == "--output":
|
||||
if sys.argv[i] == "-o" or sys.argv[i] == "--output":
|
||||
i += 1
|
||||
output = sys.argv[i]
|
||||
if sys.argv[i] == "-i" or sys.argv[i] == "--input":
|
||||
i += 1
|
||||
input_dir = sys.argv[i]
|
||||
elif sys.argv[i] == "-a" or sys.argv[i] == "--api":
|
||||
i += 1
|
||||
api = sys.argv[i]
|
||||
|
|
@ -54,9 +53,8 @@ while i < argc:
|
|||
print("usage: worldbuild-packager.py [OPTIONS]")
|
||||
print("Retrives done tiles from the manager and packages them")
|
||||
print("")
|
||||
print(" -i --input Input directory. Path MUST be absolute")
|
||||
print(" -o, --output Output directory. Path MUST be absolute")
|
||||
print(" --host Manager host")
|
||||
print(" --port Manager port")
|
||||
print(" -a, --api Manager api url")
|
||||
print(" -t, --api-token Manager api token")
|
||||
print(" -h, --help Shows this help and exit")
|
||||
|
|
@ -74,6 +72,17 @@ if output == "":
|
|||
print("ERROR: No output directory given")
|
||||
sys.exit(1)
|
||||
|
||||
if input_dir == "":
|
||||
print("ERROR: No input directory given")
|
||||
sys.exit(1)
|
||||
|
||||
if not api:
|
||||
print("ERROR: No api given")
|
||||
sys.exit(1)
|
||||
|
||||
if not api_token:
|
||||
print("ERROR: No api token given")
|
||||
|
||||
def get_sha1(fname):
|
||||
if os.path.isfile(fname):
|
||||
sha1 = hashlib.sha1()
|
||||
|
|
@ -146,64 +155,54 @@ while running:
|
|||
try:
|
||||
name = "None"
|
||||
while name == "None":
|
||||
if api == None:
|
||||
name = get_job("done", host, port, none_exit=False)
|
||||
else:
|
||||
name = api_get_job("done", api, api_token, none_exit=False)
|
||||
name = api_get_job("done", "packaging", api, api_token, none_exit=False)
|
||||
if name == "None":
|
||||
print("No job got asigned. Trying again in one hour.")
|
||||
sleep(3600)
|
||||
|
||||
match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name)
|
||||
ew = match.group(1)
|
||||
ew_val = int(match.group(2))
|
||||
ns = match.group(3)
|
||||
ns_val = int(match.group(4))
|
||||
|
||||
ew_val_major = int(ew_val / 10) * 10
|
||||
if ew == "w":
|
||||
if ew_val_major != ew_val:
|
||||
ew_val_major += 10
|
||||
|
||||
ns_val_major = int(ns_val / 10) * 10
|
||||
if ns == "s":
|
||||
if ns_val_major != ns_val:
|
||||
ns_val_major += 10
|
||||
|
||||
name_major = ew + norm(ew_val_major, 3) + ns + norm(ns_val_major, 2)
|
||||
|
||||
print("Packaging " + name + "...")
|
||||
|
||||
for part in ["Buildings", "Details", "Pylons", "Roads"]:
|
||||
run("rm -f " + output + "/" + part + "/" + name_major + "/" + name + ".txz", shell=True)
|
||||
if os.path.exists("projects/worldbuild/scenery/" + part + "/" + name_major + "/" + name):
|
||||
run("mkdir -p " + output + "/" + part + "/" + name_major, shell=True)
|
||||
area = get_area_name_by_tile(name)
|
||||
area_major = get_area_name_by_tile(name, major=True)
|
||||
|
||||
cmd = "bash -c 'cd projects/worldbuild/scenery/" + part + "/" + name_major + " && tar -cJf " + output + "/" + part + "/" + name_major + "/" + name + ".txz " + name + "/'"
|
||||
for part in os.listdir(input_dir):
|
||||
#run("rm -f " + output + "/" + part + "/" + area_major + "/" + area + ".txz", shell=True)
|
||||
run("mkdir -p " + tmp_dir + "/" + area)
|
||||
|
||||
# If there is an archive, we need to extract it first and remove old data
|
||||
if os.path.exists(output + "/" + part + "/" + area_major + "/" + area + ".txz"):
|
||||
run("bash -c 'cd " + tmp_dir + "/" + area + " && tar -xf " + output + "/" + part + "/" + area_major + "/" + area + ".txz && rm *" + name + "*'")
|
||||
|
||||
# If there's new scenery copy it into the tmp dir
|
||||
if os.path.exists(input_dir + "/" + part + "/" + area_major + "/" + area + "/" + name + ".stg"):
|
||||
run("cp " + input_dir + "/" + part + "/" + area_major + "/" + area + "/*" + name + "* " + tmp_dir + "/" + area + "/")
|
||||
|
||||
# If the scenery is not empty, package it
|
||||
if os.listdir(tmp_dir + "/" + area):
|
||||
run("mkdir -p " + output + "/" + part + "/" + area_major, shell=True)
|
||||
|
||||
cmd = "bash -c 'cd " + tmp_dir + " && tar -cJf " + output + "/" + part + "/" + area_major + "/" + area + ".txz " + area + "/'"
|
||||
with open(os.devnull, 'w') as fp:
|
||||
package = Popen(cmd , shell=True, start_new_session=True, stdout=fp)
|
||||
|
||||
package.wait()
|
||||
|
||||
update_index(output, part, name, name_major)
|
||||
update_index(output, part, area, area_major)
|
||||
|
||||
# Clean up
|
||||
run("rm -r " + tmp_dir + "/" + area)
|
||||
|
||||
print("Packaging " + name + " done")
|
||||
|
||||
if api == None:
|
||||
send_status(name, "packaged", host, port)
|
||||
else:
|
||||
api_send_status(name, "packaged", api, api_token)
|
||||
api_send_status(name, "packaged", api, api_token)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again")
|
||||
running = False
|
||||
try:
|
||||
package.wait()
|
||||
update_index(output, name, name_major)
|
||||
if api == None:
|
||||
send_status(name, "packaged", host, port)
|
||||
else:
|
||||
api_send_status(name, "packaged", api, api_token)
|
||||
update_index(output, area, name_major)
|
||||
api_send_status(name, "packaged", api, api_token)
|
||||
except KeyboardInterrupt:
|
||||
# TODO doesn't work
|
||||
print("Forcing shutdown...")
|
||||
|
|
|
|||
|
|
@ -25,10 +25,8 @@ from time import sleep, strftime, time
|
|||
from common import send_status, get_job, norm, get_south, get_west, get_east, get_north, get_area_name, api_send_status, api_get_job
|
||||
|
||||
action = "pending"
|
||||
host = socket.gethostname()
|
||||
api = None
|
||||
api_token = None
|
||||
port = 12345
|
||||
prefix = ""
|
||||
quiet = False
|
||||
global_db = False
|
||||
|
|
@ -37,13 +35,7 @@ argc = len(sys.argv)
|
|||
i = 1
|
||||
first = 1
|
||||
while i < argc:
|
||||
if sys.argv[i] == "--port":
|
||||
i += 1
|
||||
port = sys.argv[i]
|
||||
elif sys.argv[i] == "--host":
|
||||
i += 1
|
||||
host = sys.argv[i]
|
||||
elif sys.argv[i] == "-a" or sys.argv[i] == "--api":
|
||||
if sys.argv[i] == "-a" or sys.argv[i] == "--api":
|
||||
i += 1
|
||||
api = sys.argv[i]
|
||||
elif sys.argv[i] == "-t" or sys.argv[i] == "--api-token":
|
||||
|
|
@ -69,8 +61,6 @@ while i < argc:
|
|||
print("")
|
||||
print(" -p, --prefix Database prefix to use")
|
||||
print(" -g, --global-db Use global database")
|
||||
print(" --host Manager host")
|
||||
print(" --port Manager port")
|
||||
print(" -a, --api Manager api url")
|
||||
print(" -t, --api-token Manager api token")
|
||||
print(" -q, --quiet Don't print messages")
|
||||
|
|
@ -86,8 +76,12 @@ while i < argc:
|
|||
sys.exit(1)
|
||||
i += 1
|
||||
|
||||
if api != None and api_token == None:
|
||||
print("Error: API given but no token")
|
||||
if api != None:
|
||||
print("Error: No API url given")
|
||||
sys.exit(1)
|
||||
|
||||
if api_token == None:
|
||||
print("Error: No API token given")
|
||||
sys.exit(1)
|
||||
|
||||
def cleanup(name):
|
||||
|
|
@ -103,10 +97,7 @@ name = None
|
|||
try:
|
||||
running = True
|
||||
while running:
|
||||
if api != None:
|
||||
name = api_get_job(action, api, api_token)
|
||||
else:
|
||||
name = get_job(action, host, port)
|
||||
name = api_get_job(action, "started", api, api_token)
|
||||
|
||||
run("mkdir -p projects/worldbuild-" + name, shell=True)
|
||||
|
||||
|
|
@ -151,10 +142,7 @@ try:
|
|||
|
||||
cleanup(name)
|
||||
|
||||
if api != None:
|
||||
api_send_status(name, "done", api, api_token)
|
||||
else:
|
||||
send_status(name, "done", host, port)
|
||||
api_send_status(name, "done", api, api_token)
|
||||
except KeyboardInterrupt:
|
||||
if not quiet:
|
||||
print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again")
|
||||
|
|
@ -163,10 +151,7 @@ except KeyboardInterrupt:
|
|||
if build != None:
|
||||
build.wait()
|
||||
cleanup(name)
|
||||
if api != None:
|
||||
api_send_status(name, "done", api, api_token)
|
||||
else:
|
||||
send_status(name, "done", host, port)
|
||||
api_send_status(name, "done", api, api_token)
|
||||
except KeyboardInterrupt:
|
||||
#TODO doesn't work
|
||||
print("Forcing shutdown...")
|
||||
|
|
|
|||
Loading…
Reference in a new issue