Ready?
Signed-off-by: fly <merspieler@airmail.cc>
This commit is contained in:
parent
845f7b6cf9
commit
386e64ddce
8 changed files with 120 additions and 114 deletions
25
common.py
25
common.py
|
|
@ -19,6 +19,7 @@ import math
|
|||
import sys
|
||||
import socket
|
||||
import re
|
||||
from time import sleep
|
||||
|
||||
# Adds leading 0s
|
||||
def norm(num, length):
|
||||
|
|
@ -57,16 +58,22 @@ def get_tile(lat, lon):
|
|||
return (int(lon + 180) << 14) + (int(lat + 90) << 6) + (y << 3) + x
|
||||
|
||||
# Returns the area name ie. e145s17
|
||||
def get_area_name(n, e):
|
||||
if n >= 0:
|
||||
def get_area_name(s, w, major=False):
|
||||
if s >= 0:
|
||||
ns = "n"
|
||||
else:
|
||||
ns = "s"
|
||||
if e >= 0:
|
||||
if w >= 0:
|
||||
ew = "e"
|
||||
else:
|
||||
ew = "w"
|
||||
return ew + norm(abs(e), 3) + ns + norm(abs(n), 2)
|
||||
if major:
|
||||
if w % 10 != 0:
|
||||
w = w - (w % 10)
|
||||
|
||||
if s % 10 != 0:
|
||||
s = s - (s % 10)
|
||||
return ew + norm(abs(w), 3) + ns + norm(abs(s), 2)
|
||||
|
||||
# Returns latitude of tiles SW corner in area scheme
|
||||
def get_lat(tile):
|
||||
|
|
@ -108,7 +115,7 @@ def send_status(name, status, host, port):
|
|||
sys.exit(1)
|
||||
|
||||
# Gets new job from manager
|
||||
def get_job(action):
|
||||
def get_job(action, host, port, none_exit=True):
|
||||
try:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.connect((host, port))
|
||||
|
|
@ -119,22 +126,22 @@ def get_job(action):
|
|||
match = re.match(r"n-pole|s-pole|[ew]\d{3}[ns]\d{2}|None", msg)
|
||||
if match != None:
|
||||
ret = match.group(0)
|
||||
if ret == "None":
|
||||
if ret == "None" and none_exit:
|
||||
print("No job got asigned. Exiting...")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("Recived invalid job. Retrying in 10 seconds...")
|
||||
sleep(10)
|
||||
ret = get_job(action)
|
||||
ret = get_job(action, host, port, none_exit)
|
||||
return ret
|
||||
except IOError:
|
||||
print("Unable to get job. Retrying in 10 seconds...")
|
||||
sleep(10)
|
||||
ret = get_job(action)
|
||||
ret = get_job(action, host, port, none_exit)
|
||||
return ret
|
||||
|
||||
# Gets status of a tile
|
||||
def get_status(name):
|
||||
def get_status(name, host, port):
|
||||
try:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.connect((host, port))
|
||||
|
|
|
|||
2
db
2
db
|
|
@ -1,3 +1,5 @@
|
|||
Structure of database `worldbuild`
|
||||
|
||||
topLevel secondLevel tile
|
||||
-------- ----------- ----
|
||||
id <int> <pk> <-| id <int> <pk> <-| id <int> <pk>
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ try:
|
|||
tiles = []
|
||||
with open(lfile) as f:
|
||||
for line in f:
|
||||
match = re.match(r".*[ew]\d{3}[ns]\d{2}/([ew]\d{3}[ns]\d{2}).*\.[bs]tg", line)
|
||||
match = re.match(r".*[ew]\d{3}[ns]\d{2}/[ew]\d{3}[ns]\d{2}/([0-9]{1,7})\.[bs]tg", line)
|
||||
if match != None:
|
||||
tiles.append(match.group(1))
|
||||
except IOError:
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ i = 1
|
|||
while i < argc:
|
||||
if sys.argv[i] == "-p" or sys.argv[i] == "--password":
|
||||
i += 1
|
||||
dbpw = int(sys.argv[i])
|
||||
dbpw = sys.argv[i]
|
||||
elif sys.argv[i] == "-u" or sys.argv[i] == "--user":
|
||||
i += 1
|
||||
dbuser = sys.argv[i]
|
||||
|
|
@ -76,8 +76,8 @@ if force:
|
|||
cursor.execute("DROP TABLE IF EXISTS secondLevel")
|
||||
cursor.execute("DROP TABLE IF EXISTS topLevel")
|
||||
cursor.execute("DROP TABLE IF EXISTS status")
|
||||
cursor.execute("DROP TRIGGER update_top_level")
|
||||
cursor.execute("DROP TRIGGER update_second_level")
|
||||
cursor.execute("DROP TRIGGER IF EXISTS update_top_level")
|
||||
cursor.execute("DROP TRIGGER IF EXISTS update_second_level")
|
||||
|
||||
cursor.execute("CREATE TABLE IF NOT EXISTS status (id INT PRIMARY KEY AUTO_INCREMENT, name VARCHAR(15) UNIQUE NOT NULL);")
|
||||
cursor.execute("CREATE TABLE IF NOT EXISTS topLevel (id INT PRIMARY KEY AUTO_INCREMENT, name CHAR(7) UNIQUE NOT NULL, status_id INT NOT NULL, FOREIGN KEY (status_id) REFERENCES status(id));")
|
||||
|
|
|
|||
|
|
@ -139,18 +139,20 @@ try:
|
|||
set_state(name, status)
|
||||
|
||||
elif action == "get-job":
|
||||
# TODO handle other types than 'pending'
|
||||
sql = "SELECT id FROM tile WHERE status_id = (SELECT id FROM status WHERE name = 'pending') LIMIT 1"
|
||||
additional_type = ""
|
||||
if get != "pending":
|
||||
additional_type = "OR status_id = (SELECT id FROM status WHERE name = '" + get + "')"
|
||||
sql = "SELECT id FROM tile WHERE status_id = (SELECT id FROM status WHERE name = 'pending') " + additional_type + " LIMIT 1"
|
||||
cursor.execute()
|
||||
result = cursor.fetchone()
|
||||
# TODO handle no tile can be fetched
|
||||
tile = result[0]
|
||||
sql = "UPDATE tile SET status_id = (SELECT id FROM status WHERE name = 'started') WHERE id = " + tile
|
||||
cursor.execute()
|
||||
db.commit()
|
||||
|
||||
if tile == "":
|
||||
if result == None:
|
||||
tile = "None"
|
||||
else:
|
||||
tile = result[0]
|
||||
sql = "UPDATE tile SET status_id = (SELECT id FROM status WHERE name = 'started') WHERE id = " + tile
|
||||
cursor.execute()
|
||||
db.commit()
|
||||
|
||||
|
||||
if verbose:
|
||||
print("Asigning job on tile " + tile)
|
||||
|
|
@ -158,13 +160,13 @@ try:
|
|||
c.send(tile.encode())
|
||||
elif action == "get-done":
|
||||
sql = "SELECT id FROM secondLevel WHERE status_id = (SELECT id FROM status WHERE name = 'done') LIMIT 1"
|
||||
cursor.execute()
|
||||
cursor.execute(sql)
|
||||
result = cursor.fetchone()
|
||||
tile = result[0]
|
||||
# TODO handle no tile can be fetched
|
||||
|
||||
if tile == "":
|
||||
if result == None:
|
||||
tile = "None"
|
||||
else:
|
||||
tile = result[0]
|
||||
|
||||
c.send(tile.encode())
|
||||
elif action == "status":
|
||||
match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name)
|
||||
|
|
|
|||
|
|
@ -127,53 +127,49 @@ def update_index(output, name, name_major):
|
|||
|
||||
running = True
|
||||
while running:
|
||||
name = get_job("done")
|
||||
|
||||
try:
|
||||
if name == "n-pole":
|
||||
pass
|
||||
# TODO
|
||||
send_status(name, "packaged")
|
||||
elif name == "s-pole":
|
||||
pass
|
||||
# TODO
|
||||
send_status(name, "packaged")
|
||||
else:
|
||||
match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name)
|
||||
ew = match.group(1)
|
||||
ew_val = int(match.group(2))
|
||||
ns = match.group(3)
|
||||
ns_val = int(match.group(4))
|
||||
name = "None"
|
||||
while name == "None":
|
||||
name = get_job("done", host, port, none_exit=False)
|
||||
if name == "None":
|
||||
print("No job got asigned. Trying again in one hour.")
|
||||
sleep(3600)
|
||||
|
||||
ew_val_major = int(ew_val / 10) * 10
|
||||
if ew == "w":
|
||||
if ew_val_major != ew_val:
|
||||
ew_val_major += 10
|
||||
match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name)
|
||||
ew = match.group(1)
|
||||
ew_val = int(match.group(2))
|
||||
ns = match.group(3)
|
||||
ns_val = int(match.group(4))
|
||||
|
||||
ns_val_major = int(ns_val / 10) * 10
|
||||
if ns == "s":
|
||||
if ns_val_major != ns_val:
|
||||
ns_val_major += 10
|
||||
ew_val_major = int(ew_val / 10) * 10
|
||||
if ew == "w":
|
||||
if ew_val_major != ew_val:
|
||||
ew_val_major += 10
|
||||
|
||||
name_major = ew + norm(ew_val_major, 3) + ns + norm(ns_val_major, 2)
|
||||
ns_val_major = int(ns_val / 10) * 10
|
||||
if ns == "s":
|
||||
if ns_val_major != ns_val:
|
||||
ns_val_major += 10
|
||||
|
||||
run("mkdir -p " + output + "/" + name_major, shell=True)
|
||||
name_major = ew + norm(ew_val_major, 3) + ns + norm(ns_val_major, 2)
|
||||
|
||||
run("rm -f " + output + "/" + name_major + "/" + name + ".zip", shell=True)
|
||||
run("mkdir -p " + output + "/" + name_major, shell=True)
|
||||
|
||||
print("Packaging " + name + "...")
|
||||
run("rm -f " + output + "/" + name_major + "/" + name + ".zip", shell=True)
|
||||
|
||||
cmd = "bash -c 'cd projects/worldbuild/scenery && zip -rq " + output + "/" + name_major + "/" + name + ".zip {Details,Pylons,Buildings,Roads}/" + name_major + "/" + name + "/*'"
|
||||
with open(os.devnull, 'w') as fp:
|
||||
package = Popen(cmd , shell=True, start_new_session=True, stdout=fp)
|
||||
print("Packaging " + name + "...")
|
||||
|
||||
package.wait()
|
||||
cmd = "bash -c 'cd projects/worldbuild/scenery && zip -rq " + output + "/" + name_major + "/" + name + ".zip {Details,Pylons,Buildings,Roads}/" + name_major + "/" + name + "/*'"
|
||||
with open(os.devnull, 'w') as fp:
|
||||
package = Popen(cmd , shell=True, start_new_session=True, stdout=fp)
|
||||
|
||||
print("Packaging " + name + " done")
|
||||
package.wait()
|
||||
|
||||
update_index(output, name, name_major)
|
||||
print("Packaging " + name + " done")
|
||||
|
||||
send_status(name, "packaged")
|
||||
update_index(output, name, name_major)
|
||||
|
||||
send_status(name, "packaged", host, port)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again")
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import re
|
|||
import sys
|
||||
|
||||
prefix = ""
|
||||
global_db = False
|
||||
|
||||
argc = len(sys.argv)
|
||||
i = 1
|
||||
|
|
@ -31,13 +32,16 @@ while i < argc:
|
|||
if sys.argv[i] == "-p" or sys.argv[i] == "--prefix":
|
||||
i += 1
|
||||
prefix = sys.argv[i]
|
||||
elif sys.argv[i] == "-g" or sys.argv[i] == "--global-db":
|
||||
global_db = True
|
||||
elif sys.argv[i] == "-h" or sys.argv[i] == "--help":
|
||||
print("usage: worldbuild-runner.py [OPTIONS]")
|
||||
print("Starts workers")
|
||||
print("")
|
||||
print(" -p, --prefix Database prefix to use")
|
||||
print(" -p, --prefix Database prefix to use")
|
||||
print(" -g, --global-db Use global database")
|
||||
# TODO hand through all worker options
|
||||
print(" -h, --help Shows this help and exit")
|
||||
print(" -h, --help Shows this help and exit")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("Unknown option " + sys.argv[i])
|
||||
|
|
@ -94,7 +98,10 @@ class launcher(threading.Thread):
|
|||
self.run_max = count
|
||||
|
||||
def start_worker(self):
|
||||
self.running.append(Popen(["./scripts/worldbuild-worker.py", "-q", "-p", prefix], start_new_session=True))
|
||||
cmd = ["./scripts/worldbuild-worker.py", "-q", "-p", prefix]
|
||||
if global_db:
|
||||
cmd.append("-g")
|
||||
self.running.append(Popen(cmd, start_new_session=True))
|
||||
self.run_count += 1
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -20,15 +20,16 @@ from subprocess import run, Popen, STDOUT
|
|||
import sys
|
||||
import socket
|
||||
import re
|
||||
from time import sleep
|
||||
from time import sleep, strftime, time
|
||||
|
||||
from common import send_status, get_job, norm
|
||||
from common import send_status, get_job, norm, get_south, get_west, get_east, get_north
|
||||
|
||||
action = "pending"
|
||||
host = socket.gethostname()
|
||||
port = 12345
|
||||
prefix = ""
|
||||
quiet = False
|
||||
global_db = False
|
||||
|
||||
argc = len(sys.argv)
|
||||
i = 1
|
||||
|
|
@ -43,6 +44,8 @@ while i < argc:
|
|||
elif sys.argv[i] == "-p" or sys.argv[i] == "--prefix":
|
||||
i += 1
|
||||
prefix = sys.argv[i]
|
||||
elif sys.argv[i] == "-g" or sys.argv[i] == "--global-db":
|
||||
global_db = True
|
||||
elif sys.argv[i] == "-q" or sys.argv[i] == "--quiet":
|
||||
quiet = True
|
||||
elif sys.argv[i] == "-a" or sys.argv[i] == "--action":
|
||||
|
|
@ -56,16 +59,17 @@ while i < argc:
|
|||
print("usage:worldbuild-worker.py [OPTIONS]")
|
||||
print("Retrives jobs from the manager and executes them")
|
||||
print("")
|
||||
print(" -p, --prefix Database prefix to use")
|
||||
print(" --host Manager host")
|
||||
print(" --port Manager port")
|
||||
print(" -q, --quiet Don't print messages")
|
||||
print(" -a, --action Considered tiles for build. Can be:")
|
||||
print(" pending: Only builds pending tiles. These are always build: default")
|
||||
print(" started: Run tiles marked as started. CAUTION, use with care.")
|
||||
print(" rebuild: Build tiles flaged for rebuild")
|
||||
print(" skip: Force building tile that are marked for being skipped")
|
||||
print(" -h, --help Shows this help and exit")
|
||||
print(" -p, --prefix Database prefix to use")
|
||||
print(" -g, --global-db Use global database")
|
||||
print(" --host Manager host")
|
||||
print(" --port Manager port")
|
||||
print(" -q, --quiet Don't print messages")
|
||||
print(" -a, --action Considered tiles for build. Can be:")
|
||||
print(" pending: Only builds pending tiles. These are always build: default")
|
||||
print(" started: Run tiles marked as started. CAUTION, use with care.")
|
||||
print(" rebuild: Build tiles flaged for rebuild")
|
||||
print(" skip: Force building tile that are marked for being skipped")
|
||||
print(" -h, --help Shows this help and exit")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("Unknown option " + sys.argv[i])
|
||||
|
|
@ -74,55 +78,41 @@ while i < argc:
|
|||
|
||||
def cleanup():
|
||||
if os.path.isfile("projects/worldbuild-" + name + "/osm2city-exceptions.log"):
|
||||
run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + ".exceptions.log", shell=True)
|
||||
run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + "-" + strftime("%Y%m%d-%H%M", time()) + ".exceptions.log", shell=True)
|
||||
|
||||
run("rm -rf projects/worldbuild-" + name, shell=True)
|
||||
|
||||
|
||||
running = True
|
||||
while running:
|
||||
name = get_job(action)
|
||||
name = get_job(action, host, port)
|
||||
|
||||
try:
|
||||
run("mkdir -p projects/worldbuild-" + name, shell=True)
|
||||
|
||||
run("cp projects/worldbuild/params.ini projects/worldbuild-" + name + "/", shell=True)
|
||||
|
||||
if name == "n-pole":
|
||||
bounds = "bounds=*-180_80_180_90"
|
||||
db_name = prefix + "n-pole"
|
||||
elif name == "s-pole":
|
||||
bounds = "bounds=*-180_-90_180_-80"
|
||||
db_name = prefix + "s-pole"
|
||||
|
||||
if global_db:
|
||||
db_name = prefix + "worldbuild"
|
||||
else:
|
||||
match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name)
|
||||
if match == None:
|
||||
print("ERROR: Invalid tile name")
|
||||
sys.exit(1)
|
||||
if get_south(name) >= 80:
|
||||
db_name = prefix + "n-pole"
|
||||
elif get_south(name) < -80:
|
||||
db_name = prefix + "s-pole"
|
||||
else:
|
||||
ew = match.group(1)
|
||||
ew_val = int(match.group(2))
|
||||
ns = match.group(3)
|
||||
ns_val = int(match.group(4))
|
||||
|
||||
if ew == "w":
|
||||
ew_val *= -1
|
||||
if ns == "s":
|
||||
ns_val *= -1
|
||||
|
||||
if ew_val < 0:
|
||||
bounds = "bounds=*"
|
||||
match = re.match(r"[0-9]{1,7}", name)
|
||||
if match == None:
|
||||
print("ERROR: Invalid tile name")
|
||||
sys.exit(1)
|
||||
else:
|
||||
bounds = "bounds="
|
||||
db_name = prefix + get_area_name(name, major=True)
|
||||
|
||||
if get_west(name) < 0:
|
||||
bounds = "bounds=*"
|
||||
else:
|
||||
bounds = "bounds="
|
||||
|
||||
bounds += str(ew_val) + "_" + str(ns_val) + "_" + str(ew_val + 1) + "_" + str(ns_val + 1)
|
||||
if ew_val % 10 != 0:
|
||||
ew_val = ew_val - (ew_val % 10)
|
||||
|
||||
if ns_val % 10 != 0:
|
||||
ns_val = ns_val - (ns_val % 10)
|
||||
|
||||
db_name = prefix + ew + norm(ew_val, 3) + ns + norm(ns_val, 2)
|
||||
bounds += str(get_west(name)) + "_" + str(get_south(name)) + "_" + str(get_east(name)) + "_" + str(get_north(name))
|
||||
|
||||
run("sed -i 's/DB_NAME.*/DB_NAME = \"" + db_name + "\"/' projects/worldbuild-" + name + "/params.ini", shell=True)
|
||||
|
||||
|
|
@ -131,14 +121,16 @@ while running:
|
|||
if not quiet:
|
||||
print("Building " + name)
|
||||
|
||||
with open("projects/worldbuild/output/" + name + ".log", "w") as log_file:
|
||||
logpath = "projects/worldbuild/output/" + get_area_name(name, major=True) + "/" + get_area_name(name) + "/"
|
||||
run("mkdir -p " + logpath, shell=True)
|
||||
with open(logpath + name + "-" + strftime("%Y%m%d-%H%M", time()) + ".log", "w") as log_file:
|
||||
build = Popen("./build -S 10 -t 1 worldbuild-" + name, stdout=log_file, stderr=STDOUT, shell=True, start_new_session=True)
|
||||
|
||||
build.wait()
|
||||
|
||||
cleanup()
|
||||
|
||||
send_status(name, "done")
|
||||
send_status(name, "done", host, port)
|
||||
except KeyboardInterrupt:
|
||||
if not quiet:
|
||||
print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again")
|
||||
|
|
@ -146,7 +138,7 @@ while running:
|
|||
try:
|
||||
build.wait()
|
||||
cleanup()
|
||||
send_status(name, "done")
|
||||
send_status(name, "done", host, port)
|
||||
except KeyboardInterrupt:
|
||||
#TODO doesn't work
|
||||
print("Forcing shutdown...")
|
||||
|
|
|
|||
Loading…
Reference in a new issue