fly/osm2city-scripts
1
0
Fork 0
Code Issues 2 Pull requests Activity

Bunch of fixes

Browse source 
Signed-off-by: fly <merspieler@airmail.cc>
This commit is contained in:
fly 2020-01-15 11:33:31 +00:00
parent 7116fc64f9
commit e08fbea737
2 changed files with 79 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
worldbuild-manager.py
View file

@ -41,9 +41,8 @@ while i < argc:
i += 1 i += 1
sfile = sys.argv[i] sfile = sys.argv[i]
elif sys.argv[i] == "-h" or sys.argv[i] == "--help": elif sys.argv[i] == "-h" or sys.argv[i] == "--help":
print("usage: wb-progress-logger.py [OPTIONS]") print("usage: worldbuild-manager.py [OPTIONS]")
print("Logs progress of the worldbuild when run") print("Handles job asignments and keeps track of the status")
print("with database strategies 'chunk' and 'mono'")
print("") print("")
print("OPTIONS") print("OPTIONS")
print(" -f, --file Status file to use") print(" -f, --file Status file to use")
@ -51,6 +50,7 @@ while i < argc:
print(" , --host hostname or interface to bind to") print(" , --host hostname or interface to bind to")
print(" , --port Port to bind to") print(" , --port Port to bind to")
print(" -h, --help Shows this help and exit") print(" -h, --help Shows this help and exit")
sys.exit(0)
else: else:
print("Unknown option " + sys.argv[i]) print("Unknown option " + sys.argv[i])
sys.exit(1) sys.exit(1)
@ -62,7 +62,7 @@ def norm(num, length):
num = "0" + num num = "0" + num
return num return num
def save_state(state) def save_state(state):
try: try:
with open(sfile, 'w') as f: with open(sfile, 'w') as f:
json.dump(state, f, indent=4) json.dump(state, f, indent=4)
@ -165,9 +165,9 @@ try:
elif action == "get": elif action == "get":
tile = "" tile = ""
# Build poles first # Build poles first
if not "n-pole" in status or ("n-pole" in status and status["n-pole"]["status"] == get): if not "n-pole" in state or ("n-pole" in state and state["n-pole"]["status"] == get):
tile = "n-pole" tile = "n-pole"
elif not "s-pole" in status or ("s-pole" in status and status["s-pole"]["status"] == get): elif not "s-pole" in state or ("s-pole" in state and state["s-pole"]["status"] == get):
tile = "s-pole" tile = "s-pole"
else: else:
ii = -8 ii = -8
@ -188,7 +188,7 @@ try:
name = ew + norm(abs(j), 3) + ns + norm(abs(i), 2) name = ew + norm(abs(j), 3) + ns + norm(abs(i), 2)
if not name in status or (name in status and status[name]["status"] != "done"): if not name in state or (name in state and state[name]["status"] != "done"):
if ns == "s": if ns == "s":
ns_step = -1 ns_step = -1
else: else:
@ -204,7 +204,7 @@ try:
iii = abs(i) iii = abs(i)
for l in range(0, 10): for l in range(0, 10):
name_minor = ew + norm(j, 3) + ns + norm(iii, 2) name_minor = ew + norm(j, 3) + ns + norm(iii, 2)
if not name in status or (not name_minor in status[name] or (name_minor in status[name] and (status[name][name_minor]["status"] == get))): if not name in state or (not name_minor in state[name] or (name_minor in state[name] and (state[name][name_minor]["status"] == get))):
tile = name_minor tile = name_minor
break break
iii += ns_step iii += ns_step

127
worldbuild-worker.py
View file

@ -48,8 +48,8 @@ while i < argc:
print ("ERROR: Unknown action " + sys.argv[i]) print ("ERROR: Unknown action " + sys.argv[i])
sys.exit(1) sys.exit(1)
elif sys.argv[i] == "-h" or sys.argv[i] == "--help": elif sys.argv[i] == "-h" or sys.argv[i] == "--help":
print("usage: build_chunk.py [OPTIONS]") print("usage:worldbuild-worker.py [OPTIONS]")
print("Runs a single chunk build. Mainly handles status logging") print("Retrives jobs from the manager and executes them")
print("") print("")
print(" -p, --prefix Database prefix to use") print(" -p, --prefix Database prefix to use")
print(" --host Logger host") print(" --host Logger host")
@ -60,6 +60,7 @@ while i < argc:
print(" rebuild: Build tiles flaged for rebuild") print(" rebuild: Build tiles flaged for rebuild")
print(" skip: Force building tile that are marked for being skipped") print(" skip: Force building tile that are marked for being skipped")
print(" -h, --help Shows this help and exit") print(" -h, --help Shows this help and exit")
sys.exit(0)
else: else:
if first == 1: if first == 1:
first = 0 first = 0
@ -110,63 +111,77 @@ def get_job(action):
ret = get_job(action) ret = get_job(action)
return ret return ret
def execute_build(name, prefix):
send_status(name, "started")
run("mkdir -p projects/worldbuild-" + name, shell=True) running = True
while running:
name = get_job(action)
run("cp projects/worldbuild/params.ini projects/worldbuild-" + name + "/", shell=True) try:
send_status(name, "started")
if name == "n-pole":
bounds = "bounds=*-180_80_180_90"
db_name = prefix + "n-pole"
elif name == "s-pole":
bounds = "bounds=*-180_-90_180_-80"
db_name = prefix + "s-pole"
else:
match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name)
if match == None:
print("ERROR: Invalid tile name")
sys.exit(1)
else:
ew = match.group(1)
ew_val = int(match.group(2))
ns = match.group(3)
ns_val = int(match.group(4))
if ew == "w": run("mkdir -p projects/worldbuild-" + name, shell=True)
ew_val *= -1
if ns == "s": run("cp projects/worldbuild/params.ini projects/worldbuild-" + name + "/", shell=True)
ns_val *= -1
if name == "n-pole":
if ew_val < 0: bounds = "bounds=*-180_80_180_90"
bounds = "bounds=*" db_name = prefix + "n-pole"
elif name == "s-pole":
bounds = "bounds=*-180_-90_180_-80"
db_name = prefix + "s-pole"
else:
match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name)
if match == None:
print("ERROR: Invalid tile name")
sys.exit(1)
else: else:
bounds = "bounds=" ew = match.group(1)
ew_val = int(match.group(2))
ns = match.group(3)
ns_val = int(match.group(4))
if ew == "w":
ew_val *= -1
if ns == "s":
ns_val *= -1
if ew_val < 0:
bounds = "bounds=*"
else:
bounds = "bounds="
bounds += str(ew_val) + "_" + str(ns_val) + "_" + str(ew_val + 10) + "_" + str(ns_val + 10)
if ew_val % 10 != 0:
ew_val = ew_val - (ew_val % 10)
if ns_val % 10 != 0:
ns_val = ns_val - (ns_val % 10)
db_name = prefix + ew + norm(ew_val, 3) + ns + norm(ns_val, 2)
run("sed -i 's/DB_NAME.*/DB_NAME = \"" + db_name + "\"/' projects/worldbuild-" + name + "/params.ini", shell=True)
run("echo '" + bounds + "' > projects/worldbuild-" + name + "/settings", shell=True)
bounds += str(ew_val) + "_" + str(ns_val) + "_" + str(ew_val + 10) + "_" + str(ns_val + 10) build = Popen("./build -S 10 -t 1 worldbuild-" + name, shell=True, start_new_session=True)
if ew_val % 10 != 0:
ew_val = ew_val - (ew_val % 10)
if ns_val % 10 != 0: if os.path.isfile("projects/worldbuild-" + name + "/osm2city-exceptions.log"):
ns_val = ns_val - (ns_val % 10) run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + ".exceptions.log", shell=True)
db_name = prefix + ew + norm(ew_val, 3) + ns + norm(ns_val, 2) sleep(10)
run("rm -rf projects/worldbuild-" + name, shell=True)
run("sed -i 's/DB_NAME.*/DB_NAME = \"" + db_name + "\"/' projects/worldbuild-" + name + "/params.ini", shell=True)
send_status(name, "done")
run("echo '" + bounds + "' > projects/worldbuild-" + name + "/settings", shell=True) except KeyboardInterrupt:
print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again")
build = Popen("./build -S 10 -t 1 worldbuild-" + name, shell=True, start_new_session=True) running = False
try:
if os.path.isfile("projects/worldbuild-" + name + "/osm2city-exceptions.log"): build.wait()
run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + ".exceptions.log", shell=True) except KeyboardInterrupt:
#TODO doesn't work
run("rm -rf projects/worldbuild-" + name, shell=True) print("Forcing shutdown...")
build.terminate()
send_status(name, "done") sleep(5)
build.kill()
run = True: sleep(5)
while run: sys.exit(0)
tile = get_job(action)
execute_build(tile, prefix)