fly/osm2city-scripts
1
0
Fork 0
Code Issues Pull requests Activity

Ready?

Browse source 
Signed-off-by: fly <merspieler@airmail.cc>
This commit is contained in:
fly 2020-03-17 17:44:28 +00:00
parent 845f7b6cf9
commit 386e64ddce
8 changed files with 120 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
common.py
View file

@ -19,6 +19,7 @@ import math
import sys import sys
import socket import socket
import re import re
from time import sleep
# Adds leading 0s # Adds leading 0s
def norm(num, length): def norm(num, length):
@ -57,16 +58,22 @@ def get_tile(lat, lon):
return (int(lon + 180) << 14) + (int(lat + 90) << 6) + (y << 3) + x return (int(lon + 180) << 14) + (int(lat + 90) << 6) + (y << 3) + x
# Returns the area name ie. e145s17 # Returns the area name ie. e145s17
def get_area_name(n, e): def get_area_name(s, w, major=False):
if n >= 0: if s >= 0:
ns = "n" ns = "n"
else: else:
ns = "s" ns = "s"
if e >= 0: if w >= 0:
ew = "e" ew = "e"
else: else:
ew = "w" ew = "w"
return ew + norm(abs(e), 3) + ns + norm(abs(n), 2) if major:
if w % 10 != 0:
w = w - (w % 10)
if s % 10 != 0:
s = s - (s % 10)
return ew + norm(abs(w), 3) + ns + norm(abs(s), 2)
# Returns latitude of tiles SW corner in area scheme # Returns latitude of tiles SW corner in area scheme
def get_lat(tile): def get_lat(tile):
@ -108,7 +115,7 @@ def send_status(name, status, host, port):
sys.exit(1) sys.exit(1)
# Gets new job from manager # Gets new job from manager
def get_job(action): def get_job(action, host, port, none_exit=True):
try: try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, port)) sock.connect((host, port))
@ -119,22 +126,22 @@ def get_job(action):
match = re.match(r"n-pole|s-pole|[ew]\d{3}[ns]\d{2}|None", msg) match = re.match(r"n-pole|s-pole|[ew]\d{3}[ns]\d{2}|None", msg)
if match != None: if match != None:
ret = match.group(0) ret = match.group(0)
if ret == "None": if ret == "None" and none_exit:
print("No job got asigned. Exiting...") print("No job got asigned. Exiting...")
sys.exit(0) sys.exit(0)
else: else:
print("Recived invalid job. Retrying in 10 seconds...") print("Recived invalid job. Retrying in 10 seconds...")
sleep(10) sleep(10)
ret = get_job(action) ret = get_job(action, host, port, none_exit)
return ret return ret
except IOError: except IOError:
print("Unable to get job. Retrying in 10 seconds...") print("Unable to get job. Retrying in 10 seconds...")
sleep(10) sleep(10)
ret = get_job(action) ret = get_job(action, host, port, none_exit)
return ret return ret
# Gets status of a tile # Gets status of a tile
def get_status(name): def get_status(name, host, port):
try: try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, port)) sock.connect((host, port))

2
db
View file

@ -1,3 +1,5 @@
Structure of database `worldbuild`
topLevel secondLevel tile topLevel secondLevel tile
-------- ----------- ---- -------- ----------- ----
id <int> <pk> <-| id <int> <pk> <-| id <int> <pk> id <int> <pk> <-| id <int> <pk> <-| id <int> <pk>

2
flag-rebuild.py
View file

@ -65,7 +65,7 @@ try:
tiles = [] tiles = []
with open(lfile) as f: with open(lfile) as f:
for line in f: for line in f:
match = re.match(r".*[ew]\d{3}[ns]\d{2}/([ew]\d{3}[ns]\d{2}).*\.[bs]tg", line) match = re.match(r".*[ew]\d{3}[ns]\d{2}/[ew]\d{3}[ns]\d{2}/([0-9]{1,7})\.[bs]tg", line)
if match != None: if match != None:
tiles.append(match.group(1)) tiles.append(match.group(1))
except IOError: except IOError:

6
init-db.py
View file

@ -28,7 +28,7 @@ i = 1
while i < argc: while i < argc:
if sys.argv[i] == "-p" or sys.argv[i] == "--password": if sys.argv[i] == "-p" or sys.argv[i] == "--password":
i += 1 i += 1
dbpw = int(sys.argv[i]) dbpw = sys.argv[i]
elif sys.argv[i] == "-u" or sys.argv[i] == "--user": elif sys.argv[i] == "-u" or sys.argv[i] == "--user":
i += 1 i += 1
dbuser = sys.argv[i] dbuser = sys.argv[i]
@ -76,8 +76,8 @@ if force:
cursor.execute("DROP TABLE IF EXISTS secondLevel") cursor.execute("DROP TABLE IF EXISTS secondLevel")
cursor.execute("DROP TABLE IF EXISTS topLevel") cursor.execute("DROP TABLE IF EXISTS topLevel")
cursor.execute("DROP TABLE IF EXISTS status") cursor.execute("DROP TABLE IF EXISTS status")
cursor.execute("DROP TRIGGER update_top_level") cursor.execute("DROP TRIGGER IF EXISTS update_top_level")
cursor.execute("DROP TRIGGER update_second_level") cursor.execute("DROP TRIGGER IF EXISTS update_second_level")
cursor.execute("CREATE TABLE IF NOT EXISTS status (id INT PRIMARY KEY AUTO_INCREMENT, name VARCHAR(15) UNIQUE NOT NULL);") cursor.execute("CREATE TABLE IF NOT EXISTS status (id INT PRIMARY KEY AUTO_INCREMENT, name VARCHAR(15) UNIQUE NOT NULL);")
cursor.execute("CREATE TABLE IF NOT EXISTS topLevel (id INT PRIMARY KEY AUTO_INCREMENT, name CHAR(7) UNIQUE NOT NULL, status_id INT NOT NULL, FOREIGN KEY (status_id) REFERENCES status(id));") cursor.execute("CREATE TABLE IF NOT EXISTS topLevel (id INT PRIMARY KEY AUTO_INCREMENT, name CHAR(7) UNIQUE NOT NULL, status_id INT NOT NULL, FOREIGN KEY (status_id) REFERENCES status(id));")

30
worldbuild-manager.py
View file

@ -139,18 +139,20 @@ try:
set_state(name, status) set_state(name, status)
elif action == "get-job": elif action == "get-job":
# TODO handle other types than 'pending' additional_type = ""
sql = "SELECT id FROM tile WHERE status_id = (SELECT id FROM status WHERE name = 'pending') LIMIT 1" if get != "pending":
additional_type = "OR status_id = (SELECT id FROM status WHERE name = '" + get + "')"
sql = "SELECT id FROM tile WHERE status_id = (SELECT id FROM status WHERE name = 'pending') " + additional_type + " LIMIT 1"
cursor.execute() cursor.execute()
result = cursor.fetchone() result = cursor.fetchone()
# TODO handle no tile can be fetched if result == None:
tile = result[0]
sql = "UPDATE tile SET status_id = (SELECT id FROM status WHERE name = 'started') WHERE id = " + tile
cursor.execute()
db.commit()
if tile == "":
tile = "None" tile = "None"
else:
tile = result[0]
sql = "UPDATE tile SET status_id = (SELECT id FROM status WHERE name = 'started') WHERE id = " + tile
cursor.execute()
db.commit()
if verbose: if verbose:
print("Asigning job on tile " + tile) print("Asigning job on tile " + tile)
@ -158,13 +160,13 @@ try:
c.send(tile.encode()) c.send(tile.encode())
elif action == "get-done": elif action == "get-done":
sql = "SELECT id FROM secondLevel WHERE status_id = (SELECT id FROM status WHERE name = 'done') LIMIT 1" sql = "SELECT id FROM secondLevel WHERE status_id = (SELECT id FROM status WHERE name = 'done') LIMIT 1"
cursor.execute() cursor.execute(sql)
result = cursor.fetchone() result = cursor.fetchone()
tile = result[0] if result == None:
# TODO handle no tile can be fetched
if tile == "":
tile = "None" tile = "None"
else:
tile = result[0]
c.send(tile.encode()) c.send(tile.encode())
elif action == "status": elif action == "status":
match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name) match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name)

66
worldbuild-packager.py
View file

@ -127,53 +127,49 @@ def update_index(output, name, name_major):
running = True running = True
while running: while running:
name = get_job("done")
try: try:
if name == "n-pole": name = "None"
pass while name == "None":
# TODO name = get_job("done", host, port, none_exit=False)
send_status(name, "packaged") if name == "None":
elif name == "s-pole": print("No job got asigned. Trying again in one hour.")
pass sleep(3600)
# TODO
send_status(name, "packaged")
else:
match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name)
ew = match.group(1)
ew_val = int(match.group(2))
ns = match.group(3)
ns_val = int(match.group(4))
ew_val_major = int(ew_val / 10) * 10 match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name)
if ew == "w": ew = match.group(1)
if ew_val_major != ew_val: ew_val = int(match.group(2))
ew_val_major += 10 ns = match.group(3)
ns_val = int(match.group(4))
ns_val_major = int(ns_val / 10) * 10 ew_val_major = int(ew_val / 10) * 10
if ns == "s": if ew == "w":
if ns_val_major != ns_val: if ew_val_major != ew_val:
ns_val_major += 10 ew_val_major += 10
name_major = ew + norm(ew_val_major, 3) + ns + norm(ns_val_major, 2) ns_val_major = int(ns_val / 10) * 10
if ns == "s":
if ns_val_major != ns_val:
ns_val_major += 10
run("mkdir -p " + output + "/" + name_major, shell=True) name_major = ew + norm(ew_val_major, 3) + ns + norm(ns_val_major, 2)
run("rm -f " + output + "/" + name_major + "/" + name + ".zip", shell=True) run("mkdir -p " + output + "/" + name_major, shell=True)
print("Packaging " + name + "...") run("rm -f " + output + "/" + name_major + "/" + name + ".zip", shell=True)
cmd = "bash -c 'cd projects/worldbuild/scenery && zip -rq " + output + "/" + name_major + "/" + name + ".zip {Details,Pylons,Buildings,Roads}/" + name_major + "/" + name + "/*'" print("Packaging " + name + "...")
with open(os.devnull, 'w') as fp:
package = Popen(cmd , shell=True, start_new_session=True, stdout=fp)
package.wait() cmd = "bash -c 'cd projects/worldbuild/scenery && zip -rq " + output + "/" + name_major + "/" + name + ".zip {Details,Pylons,Buildings,Roads}/" + name_major + "/" + name + "/*'"
with open(os.devnull, 'w') as fp:
package = Popen(cmd , shell=True, start_new_session=True, stdout=fp)
print("Packaging " + name + " done") package.wait()
update_index(output, name, name_major) print("Packaging " + name + " done")
send_status(name, "packaged") update_index(output, name, name_major)
send_status(name, "packaged", host, port)
except KeyboardInterrupt: except KeyboardInterrupt:
print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again") print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again")

13
worldbuild-runner.py
View file

@ -23,6 +23,7 @@ import re
import sys import sys
prefix = "" prefix = ""
global_db = False
argc = len(sys.argv) argc = len(sys.argv)
i = 1 i = 1
@ -31,13 +32,16 @@ while i < argc:
if sys.argv[i] == "-p" or sys.argv[i] == "--prefix": if sys.argv[i] == "-p" or sys.argv[i] == "--prefix":
i += 1 i += 1
prefix = sys.argv[i] prefix = sys.argv[i]
elif sys.argv[i] == "-g" or sys.argv[i] == "--global-db":
global_db = True
elif sys.argv[i] == "-h" or sys.argv[i] == "--help": elif sys.argv[i] == "-h" or sys.argv[i] == "--help":
print("usage: worldbuild-runner.py [OPTIONS]") print("usage: worldbuild-runner.py [OPTIONS]")
print("Starts workers") print("Starts workers")
print("") print("")
print(" -p, --prefix Database prefix to use") print(" -p, --prefix Database prefix to use")
print(" -g, --global-db Use global database")
# TODO hand through all worker options # TODO hand through all worker options
print(" -h, --help Shows this help and exit") print(" -h, --help Shows this help and exit")
sys.exit(0) sys.exit(0)
else: else:
print("Unknown option " + sys.argv[i]) print("Unknown option " + sys.argv[i])
@ -94,7 +98,10 @@ class launcher(threading.Thread):
self.run_max = count self.run_max = count
def start_worker(self): def start_worker(self):
self.running.append(Popen(["./scripts/worldbuild-worker.py", "-q", "-p", prefix], start_new_session=True)) cmd = ["./scripts/worldbuild-worker.py", "-q", "-p", prefix]
if global_db:
cmd.append("-g")
self.running.append(Popen(cmd, start_new_session=True))
self.run_count += 1 self.run_count += 1

90
worldbuild-worker.py
View file

@ -20,15 +20,16 @@ from subprocess import run, Popen, STDOUT
import sys import sys
import socket import socket
import re import re
from time import sleep from time import sleep, strftime, time
from common import send_status, get_job, norm from common import send_status, get_job, norm, get_south, get_west, get_east, get_north
action = "pending" action = "pending"
host = socket.gethostname() host = socket.gethostname()
port = 12345 port = 12345
prefix = "" prefix = ""
quiet = False quiet = False
global_db = False
argc = len(sys.argv) argc = len(sys.argv)
i = 1 i = 1
@ -43,6 +44,8 @@ while i < argc:
elif sys.argv[i] == "-p" or sys.argv[i] == "--prefix": elif sys.argv[i] == "-p" or sys.argv[i] == "--prefix":
i += 1 i += 1
prefix = sys.argv[i] prefix = sys.argv[i]
elif sys.argv[i] == "-g" or sys.argv[i] == "--global-db":
global_db = True
elif sys.argv[i] == "-q" or sys.argv[i] == "--quiet": elif sys.argv[i] == "-q" or sys.argv[i] == "--quiet":
quiet = True quiet = True
elif sys.argv[i] == "-a" or sys.argv[i] == "--action": elif sys.argv[i] == "-a" or sys.argv[i] == "--action":
@ -56,16 +59,17 @@ while i < argc:
print("usage:worldbuild-worker.py [OPTIONS]") print("usage:worldbuild-worker.py [OPTIONS]")
print("Retrives jobs from the manager and executes them") print("Retrives jobs from the manager and executes them")
print("") print("")
print(" -p, --prefix Database prefix to use") print(" -p, --prefix Database prefix to use")
print(" --host Manager host") print(" -g, --global-db Use global database")
print(" --port Manager port") print(" --host Manager host")
print(" -q, --quiet Don't print messages") print(" --port Manager port")
print(" -a, --action Considered tiles for build. Can be:") print(" -q, --quiet Don't print messages")
print(" pending: Only builds pending tiles. These are always build: default") print(" -a, --action Considered tiles for build. Can be:")
print(" started: Run tiles marked as started. CAUTION, use with care.") print(" pending: Only builds pending tiles. These are always build: default")
print(" rebuild: Build tiles flaged for rebuild") print(" started: Run tiles marked as started. CAUTION, use with care.")
print(" skip: Force building tile that are marked for being skipped") print(" rebuild: Build tiles flaged for rebuild")
print(" -h, --help Shows this help and exit") print(" skip: Force building tile that are marked for being skipped")
print(" -h, --help Shows this help and exit")
sys.exit(0) sys.exit(0)
else: else:
print("Unknown option " + sys.argv[i]) print("Unknown option " + sys.argv[i])
@ -74,55 +78,41 @@ while i < argc:
def cleanup(): def cleanup():
if os.path.isfile("projects/worldbuild-" + name + "/osm2city-exceptions.log"): if os.path.isfile("projects/worldbuild-" + name + "/osm2city-exceptions.log"):
run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + ".exceptions.log", shell=True) run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + "-" + strftime("%Y%m%d-%H%M", time()) + ".exceptions.log", shell=True)
run("rm -rf projects/worldbuild-" + name, shell=True) run("rm -rf projects/worldbuild-" + name, shell=True)
running = True running = True
while running: while running:
name = get_job(action) name = get_job(action, host, port)
try: try:
run("mkdir -p projects/worldbuild-" + name, shell=True) run("mkdir -p projects/worldbuild-" + name, shell=True)
run("cp projects/worldbuild/params.ini projects/worldbuild-" + name + "/", shell=True) run("cp projects/worldbuild/params.ini projects/worldbuild-" + name + "/", shell=True)
if name == "n-pole": if global_db:
bounds = "bounds=*-180_80_180_90" db_name = prefix + "worldbuild"
db_name = prefix + "n-pole"
elif name == "s-pole":
bounds = "bounds=*-180_-90_180_-80"
db_name = prefix + "s-pole"
else: else:
match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name) if get_south(name) >= 80:
if match == None: db_name = prefix + "n-pole"
print("ERROR: Invalid tile name") elif get_south(name) < -80:
sys.exit(1) db_name = prefix + "s-pole"
else: else:
ew = match.group(1) match = re.match(r"[0-9]{1,7}", name)
ew_val = int(match.group(2)) if match == None:
ns = match.group(3) print("ERROR: Invalid tile name")
ns_val = int(match.group(4)) sys.exit(1)
if ew == "w":
ew_val *= -1
if ns == "s":
ns_val *= -1
if ew_val < 0:
bounds = "bounds=*"
else: else:
bounds = "bounds=" db_name = prefix + get_area_name(name, major=True)
if get_west(name) < 0:
bounds = "bounds=*"
else:
bounds = "bounds="
bounds += str(ew_val) + "_" + str(ns_val) + "_" + str(ew_val + 1) + "_" + str(ns_val + 1) bounds += str(get_west(name)) + "_" + str(get_south(name)) + "_" + str(get_east(name)) + "_" + str(get_north(name))
if ew_val % 10 != 0:
ew_val = ew_val - (ew_val % 10)
if ns_val % 10 != 0:
ns_val = ns_val - (ns_val % 10)
db_name = prefix + ew + norm(ew_val, 3) + ns + norm(ns_val, 2)
run("sed -i 's/DB_NAME.*/DB_NAME = \"" + db_name + "\"/' projects/worldbuild-" + name + "/params.ini", shell=True) run("sed -i 's/DB_NAME.*/DB_NAME = \"" + db_name + "\"/' projects/worldbuild-" + name + "/params.ini", shell=True)
@ -131,14 +121,16 @@ while running:
if not quiet: if not quiet:
print("Building " + name) print("Building " + name)
with open("projects/worldbuild/output/" + name + ".log", "w") as log_file: logpath = "projects/worldbuild/output/" + get_area_name(name, major=True) + "/" + get_area_name(name) + "/"
run("mkdir -p " + logpath, shell=True)
with open(logpath + name + "-" + strftime("%Y%m%d-%H%M", time()) + ".log", "w") as log_file:
build = Popen("./build -S 10 -t 1 worldbuild-" + name, stdout=log_file, stderr=STDOUT, shell=True, start_new_session=True) build = Popen("./build -S 10 -t 1 worldbuild-" + name, stdout=log_file, stderr=STDOUT, shell=True, start_new_session=True)
build.wait() build.wait()
cleanup() cleanup()
send_status(name, "done") send_status(name, "done", host, port)
except KeyboardInterrupt: except KeyboardInterrupt:
if not quiet: if not quiet:
print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again") print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again")
@ -146,7 +138,7 @@ while running:
try: try:
build.wait() build.wait()
cleanup() cleanup()
send_status(name, "done") send_status(name, "done", host, port)
except KeyboardInterrupt: except KeyboardInterrupt:
#TODO doesn't work #TODO doesn't work
print("Forcing shutdown...") print("Forcing shutdown...")