203 lines
6.8 KiB
Python
Executable file
203 lines
6.8 KiB
Python
Executable file
#! /usr/bin/python3
|
|
# Copyright (C) 2018-2020 Merspieler, merspieler _at_ airmail.cc
|
|
#
|
|
# This program is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU General Public License as
|
|
# published by the Free Software Foundation; either version 3 of the
|
|
# License, or (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful, but
|
|
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
# General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
|
|
import os
|
|
from subprocess import run, Popen, STDOUT
|
|
import sys
|
|
import socket
|
|
import re
|
|
from time import sleep, strftime, time
|
|
|
|
from common import send_status, get_job, norm, get_south, get_west, get_east, get_north, get_area_name, api_send_status, api_get_job
|
|
|
|
action = "pending"
|
|
api = None
|
|
api_token = None
|
|
prefix = ""
|
|
quiet = False
|
|
global_db = False
|
|
setup_db = False
|
|
pbf_path = None
|
|
|
|
argc = len(sys.argv)
|
|
i = 1
|
|
first = 1
|
|
while i < argc:
|
|
if sys.argv[i] == "-a" or sys.argv[i] == "--api":
|
|
i += 1
|
|
api = sys.argv[i]
|
|
elif sys.argv[i] == "-t" or sys.argv[i] == "--api-token":
|
|
i += 1
|
|
api_token = sys.argv[i]
|
|
elif sys.argv[i] == "-p" or sys.argv[i] == "--prefix":
|
|
i += 1
|
|
prefix = sys.argv[i]
|
|
elif sys.argv[i] == "-g" or sys.argv[i] == "--global-db":
|
|
global_db = True
|
|
elif sys.argv[i] == "-q" or sys.argv[i] == "--quiet":
|
|
quiet = True
|
|
elif sys.argv[i] == "--action":
|
|
i += 1
|
|
if sys.argv[i] == "pending" or sys.argv[i] == "started" or sys.argv[i] == "rebuild" or sys.argv[i] == "skip":
|
|
action = sys.argv[i]
|
|
else:
|
|
print ("ERROR: Unknown action " + sys.argv[i])
|
|
sys.exit(1)
|
|
elif sys.argv[i] == "--setup-db":
|
|
setup_db = True
|
|
elif sys.argv[i] == "--pbf-path":
|
|
i += 1
|
|
pbf_path = sys.argv[i]
|
|
elif sys.argv[i] == "-h" or sys.argv[i] == "--help":
|
|
print("usage:worldbuild-worker.py [OPTIONS]")
|
|
print("Retrives jobs from the manager and executes them")
|
|
print("")
|
|
print(" -p, --prefix Database prefix to use")
|
|
print(" -g, --global-db Use global database")
|
|
print(" -a, --api Manager api url")
|
|
print(" -t, --api-token Manager api token")
|
|
print(" -q, --quiet Don't print messages")
|
|
print(" --action Considered tiles for build. Can be:")
|
|
print(" pending: Only builds pending tiles. These are always build: default")
|
|
print(" started: Run tiles marked as started. CAUTION, use with care.")
|
|
print(" rebuild: Build tiles flaged for rebuild")
|
|
print(" skip: Force building tile that are marked for being skipped")
|
|
print(" Set the following options, if the worker should manage the databases")
|
|
print(" --setup-db Enables managment of databases")
|
|
print(" --pbf-path Path to the dir with the 10x10 degrees .pbf files")
|
|
print("")
|
|
print(" -h, --help Shows this help and exit")
|
|
sys.exit(0)
|
|
else:
|
|
print("Unknown option " + sys.argv[i])
|
|
sys.exit(1)
|
|
i += 1
|
|
|
|
if api == None:
|
|
print("Error: No API url given")
|
|
sys.exit(1)
|
|
|
|
if api_token == None:
|
|
print("Error: No API token given")
|
|
sys.exit(1)
|
|
|
|
if setup_db and pbf_path == None:
|
|
print("Error: Database managment requested but not pbf path given")
|
|
sys.exit(1)
|
|
|
|
def cleanup(name):
|
|
if name != None:
|
|
if os.path.isfile("projects/worldbuild-" + name + "/osm2city-exceptions.log"):
|
|
run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + "-" + strftime("%Y%m%d-%H%M") + ".exceptions.log", shell=True)
|
|
|
|
run("rm -rf projects/worldbuild-" + name, shell=True)
|
|
name = None
|
|
if setup_db:
|
|
run("./delete-db " + db_name, shell=True)
|
|
|
|
build = None
|
|
name = None
|
|
try:
|
|
running = True
|
|
while running:
|
|
name = api_get_job(action, "started", api, api_token)
|
|
|
|
run("mkdir -p projects/worldbuild-" + name, shell=True)
|
|
|
|
run("cp projects/worldbuild/params.ini projects/worldbuild-" + name + "/", shell=True)
|
|
|
|
if setup_db:
|
|
if not quiet:
|
|
print("Cutting .pbf chunk")
|
|
if get_south(name) >= 80:
|
|
source_pbf = pbf_path + "/" + "n-pole.osm.pbf"
|
|
elif get_south(name) < -80:
|
|
source_pbf = pbf_path + "/" + "s-pole.osm.pbf"
|
|
else:
|
|
match = re.match(r"[0-9]{1,7}", name)
|
|
if match == None:
|
|
print("ERROR: Invalid tile name")
|
|
sys.exit(1)
|
|
else:
|
|
source_pbf = pbf_path + "/" + get_area_name(get_south(name), get_west(name), major=True) + ".osm.pbf"
|
|
db_name = prefix + name
|
|
run("osmium extract " + source_pbf + " --overwrite --bbox " + str(get_west(name)) + "," + str(get_south(name)) + "," + str(get_east(name)) + "," + str(get_north(name)) + " -o projects/worldbuild-" + name + "/tile.osm.pbf", shell=True)
|
|
if not quiet:
|
|
print("Creating database")
|
|
run("./creat-db " + db_name, shell=True)
|
|
if not quiet:
|
|
print("Loading .pbf into database")
|
|
run("./read-pbf " + db_name + " projects/worldbuild-" + name + "/tile.osm.pbf", shell=True)
|
|
else:
|
|
if global_db:
|
|
db_name = prefix + "worldbuild"
|
|
else:
|
|
if get_south(name) >= 80:
|
|
db_name = prefix + "n-pole"
|
|
elif get_south(name) < -80:
|
|
db_name = prefix + "s-pole"
|
|
else:
|
|
match = re.match(r"[0-9]{1,7}", name)
|
|
if match == None:
|
|
print("ERROR: Invalid tile name")
|
|
sys.exit(1)
|
|
else:
|
|
db_name = prefix + get_area_name(get_south(name), get_west(name), major=True)
|
|
|
|
if get_west(name) < 0:
|
|
bounds = "bounds=*"
|
|
else:
|
|
bounds = "bounds="
|
|
|
|
bounds += str(get_west(name)) + "_" + str(get_south(name)) + "_" + str(get_east(name)) + "_" + str(get_north(name))
|
|
|
|
run("sed -i 's/DB_NAME.*/DB_NAME = \"" + db_name + "\"/' projects/worldbuild-" + name + "/params.ini", shell=True)
|
|
|
|
run("echo '" + bounds + "' > projects/worldbuild-" + name + "/settings", shell=True)
|
|
|
|
if not quiet:
|
|
print("Building " + name)
|
|
|
|
logpath = "projects/worldbuild/output/" + get_area_name(get_south(name), get_west(name), major=True) + "/" + get_area_name(get_south(name), get_west(name)) + "/"
|
|
run("mkdir -p " + logpath, shell=True)
|
|
with open(logpath + name + "-" + strftime("%Y%m%d-%H%M") + ".log", "w") as log_file:
|
|
build = Popen("./build -S 10 -t 1 worldbuild-" + name, stdout=log_file, stderr=STDOUT, shell=True, start_new_session=True)
|
|
|
|
build.wait()
|
|
build = None
|
|
|
|
cleanup(name)
|
|
|
|
api_send_status(name, "done", api, api_token)
|
|
except KeyboardInterrupt:
|
|
if not quiet:
|
|
print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again")
|
|
running = False
|
|
try:
|
|
if build != None:
|
|
build.wait()
|
|
cleanup(name)
|
|
api_send_status(name, "done", api, api_token)
|
|
except KeyboardInterrupt:
|
|
#TODO doesn't work
|
|
print("Forcing shutdown...")
|
|
build.terminate()
|
|
sleep(5)
|
|
build.kill()
|
|
sleep(5)
|
|
cleanup(name)
|
|
sys.exit(0)
|