Initial commit
Signed-off-by: fly <merspieler@airmail.cc>
This commit is contained in:
commit
dfae55128e
4 changed files with 397 additions and 0 deletions
5
.gitignore
vendored
Normal file
5
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
__pycache__
|
||||||
|
config.py
|
||||||
|
logs
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
242
common.py
Normal file
242
common.py
Normal file
|
|
@ -0,0 +1,242 @@
|
||||||
|
#! /usr/bin/python3
|
||||||
|
# Copyright (C) 2020 Merspieler, merspieler _at_ airmail.cc
|
||||||
|
#
|
||||||
|
# This program is free software; you can redistribute it and/or
|
||||||
|
# modify it under the terms of the GNU General Public License as
|
||||||
|
# published by the Free Software Foundation; either version 3 of the
|
||||||
|
# License, or (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful, but
|
||||||
|
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
# General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program; if not, write to the Free Software
|
||||||
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||||
|
|
||||||
|
import math
|
||||||
|
import sys
|
||||||
|
import socket
|
||||||
|
import re
|
||||||
|
import requests
|
||||||
|
import json
|
||||||
|
from time import sleep
|
||||||
|
|
||||||
|
# Adds leading 0s
|
||||||
|
def norm(num, length):
|
||||||
|
num = str(num)
|
||||||
|
while len(num) < length:
|
||||||
|
num = "0" + num
|
||||||
|
return num
|
||||||
|
|
||||||
|
# Returns tile width depending on how north/south the tile is
|
||||||
|
def get_tile_width(lat):
|
||||||
|
if abs(lat) >= 89:
|
||||||
|
tile_width = 12
|
||||||
|
elif abs(lat) >= 86:
|
||||||
|
tile_width = 4
|
||||||
|
elif abs(lat) >= 83:
|
||||||
|
tile_width = 2
|
||||||
|
elif abs(lat) >= 76:
|
||||||
|
tile_width = 1
|
||||||
|
elif abs(lat) >= 62:
|
||||||
|
tile_width = 0.5
|
||||||
|
elif abs(lat) >= 22:
|
||||||
|
tile_width = 0.25
|
||||||
|
else:
|
||||||
|
tile_width = 0.125
|
||||||
|
return tile_width
|
||||||
|
|
||||||
|
# Returns the tile name for the given coordinates
|
||||||
|
def get_tile(lat, lon):
|
||||||
|
|
||||||
|
tile_width = get_tile_width(lat)
|
||||||
|
|
||||||
|
base_y = math.floor(lat)
|
||||||
|
y = math.trunc((lat - base_y) * 8)
|
||||||
|
base_x = math.floor(math.floor(lon / tile_width) * tile_width)
|
||||||
|
x = math.floor((lon - base_x) / tile_width)
|
||||||
|
return (int(lon + 180) << 14) + (int(lat + 90) << 6) + (y << 3) + x
|
||||||
|
|
||||||
|
# Returns the area name for a given lat lon ie. e145s17
|
||||||
|
def get_area_name(s, w, major=False):
|
||||||
|
if s >= 0:
|
||||||
|
ns = "n"
|
||||||
|
else:
|
||||||
|
ns = "s"
|
||||||
|
if w >= 0:
|
||||||
|
ew = "e"
|
||||||
|
else:
|
||||||
|
ew = "w"
|
||||||
|
if major:
|
||||||
|
if w % 10 != 0:
|
||||||
|
w = w - (w % 10)
|
||||||
|
|
||||||
|
if s % 10 != 0:
|
||||||
|
s = s - (s % 10)
|
||||||
|
return ew + norm(abs(math.floor(w)), 3) + ns + norm(abs(math.floor(s)), 2)
|
||||||
|
|
||||||
|
# Returns the area name for a given tile
|
||||||
|
def get_area_name_by_tile(tile, major=False):
|
||||||
|
return get_area_name(get_south(tile), get_west(tile), major)
|
||||||
|
|
||||||
|
# Returns latitude of tiles SW corner in area scheme
|
||||||
|
def get_lat(tile):
|
||||||
|
return ((16320 & int(tile)) >> 6) - 90
|
||||||
|
|
||||||
|
# Returns longditude of tiles SW corner in area scheme
|
||||||
|
def get_lon(tile):
|
||||||
|
return ((16760832 & int(tile)) >> 14) - 180
|
||||||
|
|
||||||
|
# Returns precise west boundary of tile
|
||||||
|
def get_west(tile):
|
||||||
|
lon = ((16760832 & int(tile)) >> 14) - 180
|
||||||
|
y = (7 & int(tile)) / (1 / get_tile_width(get_south(tile)))
|
||||||
|
return lon + y
|
||||||
|
|
||||||
|
# Returns east boundary of tile
|
||||||
|
def get_east(tile):
|
||||||
|
return get_west(tile) + get_tile_width(get_lat(tile))
|
||||||
|
|
||||||
|
# Returns precise south boundary of tile
|
||||||
|
def get_south(tile):
|
||||||
|
lat = ((16320 & int(tile)) >> 6) - 90
|
||||||
|
x = ((56 & int(tile)) >> 3) / 8
|
||||||
|
return lat + x
|
||||||
|
|
||||||
|
# Returns north boundary of tile
|
||||||
|
def get_north(tile):
|
||||||
|
return get_south(tile) + 0.125
|
||||||
|
|
||||||
|
# Sends status to the manager
|
||||||
|
def send_status(name, status, host, port):
|
||||||
|
try:
|
||||||
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
|
sock.connect((host, port))
|
||||||
|
sock.send(("set " + name + " " + status).encode())
|
||||||
|
sock.close()
|
||||||
|
except IOError:
|
||||||
|
print("Unable to send status " + status + " for tile " + name + ". Aborting...")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Sends status to the manager api
|
||||||
|
def api_send_status(name, status, api, token):
|
||||||
|
success = False
|
||||||
|
while not success:
|
||||||
|
try:
|
||||||
|
match = re.match(r"[0-9]{1,7}", name)
|
||||||
|
if match != None:
|
||||||
|
response = requests.post(api, data={'auth': token, 'action': 'set', 'tile': name, 'status': status}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||||
|
else:
|
||||||
|
response = requests.post(api, data={'auth': token, 'action': 'set', 'area': name, 'status': status}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||||
|
if response.ok:
|
||||||
|
success = response.json()["success"]
|
||||||
|
if not success:
|
||||||
|
print("Warning: Unable to send status '" + status + "' for tile '" + name + "'. Trying again in 60 seconds")
|
||||||
|
sleep(60)
|
||||||
|
else:
|
||||||
|
print("Warning: Unable to send status '" + status + "' for tile '" + name + "'. Trying again in 60 seconds")
|
||||||
|
sleep(60)
|
||||||
|
except (ConnectionError, OSError, IOError, json.JSONDecodeError):
|
||||||
|
print("Warning: Unable to send status " + status + " for tile " + name + ". Trying again in 60 seconds")
|
||||||
|
sleep(60)
|
||||||
|
|
||||||
|
# Gets new job from manager
|
||||||
|
def get_job(action, host, port, none_exit=True):
|
||||||
|
try:
|
||||||
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
|
sock.connect((host, port))
|
||||||
|
sock.send(("get " + action).encode())
|
||||||
|
msg = sock.recv(128)
|
||||||
|
sock.close()
|
||||||
|
msg = msg.decode()
|
||||||
|
match = re.match(r"[ew]\d{3}[ns]\d{2}|[0-9]{1,7}|None", msg)
|
||||||
|
if match != None:
|
||||||
|
ret = match.group(0)
|
||||||
|
if ret == "None" and none_exit:
|
||||||
|
print("No job got asigned. Exiting...")
|
||||||
|
sys.exit(0)
|
||||||
|
else:
|
||||||
|
print("Recived invalid job. Retrying in 10 seconds...")
|
||||||
|
sleep(10)
|
||||||
|
ret = get_job(action, host, port, none_exit)
|
||||||
|
return ret
|
||||||
|
except IOError:
|
||||||
|
print("Unable to get job. Retrying in 10 seconds...")
|
||||||
|
sleep(10)
|
||||||
|
ret = get_job(action, host, port, none_exit)
|
||||||
|
return ret
|
||||||
|
|
||||||
|
# Gets new job from manager api
|
||||||
|
def api_get_job(action, new_status, api, token, level, none_exit=True, all_in_parent=0):
|
||||||
|
try:
|
||||||
|
response = requests.post(api, data={'auth': token, 'action': 'get-job', 'status': action, 'new-status': new_status, "all-in-parent": all_in_parent, "level": level}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||||
|
if response.ok and response.json()["success"] == True:
|
||||||
|
match = re.match(r"[0-9]{1,7}|[we]\d{3}[sn]\d{2}", str(response.json()["job"]))
|
||||||
|
if match != None:
|
||||||
|
ret = match.group(0)
|
||||||
|
if ret != "None" and all_in_parent == 1:
|
||||||
|
return response.json()["jobs"]
|
||||||
|
return ret
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
except (ConnectionError, OSError, IOError, json.JSONDecodeError):
|
||||||
|
print(response.text)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Gets status of a tile
|
||||||
|
def get_status(name, host, port):
|
||||||
|
try:
|
||||||
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
|
sock.connect((host, port))
|
||||||
|
sock.send(("status " + name).encode())
|
||||||
|
msg = sock.recv(128)
|
||||||
|
sock.close()
|
||||||
|
msg = msg.decode()
|
||||||
|
match = re.match(r"pending|done|rebuild|skip|started|packaged", msg)
|
||||||
|
if match != None:
|
||||||
|
return match.group(0)
|
||||||
|
else:
|
||||||
|
print("ERROR: Recived invalid state for tile " + name)
|
||||||
|
sys.exit(1)
|
||||||
|
except IOError:
|
||||||
|
print("ERROR: Unable to get status.")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Gets status of a tile from api
|
||||||
|
def api_get_status(name, api, api_token):
|
||||||
|
try:
|
||||||
|
match = re.match(r"[ew]\d{3}[ns]\d{2}", name)
|
||||||
|
if match != None:
|
||||||
|
response = requests.post(api, data={'auth': api_token, 'action': 'status', 'area': name}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||||
|
else:
|
||||||
|
response = requests.post(api, data={'auth': api_token, 'action': 'status', 'tile': name}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||||
|
if response.ok and response.json()["success"] == True:
|
||||||
|
match = re.match(r"pending|done|rebuild|skip|started|packaged", response.json()["status"])
|
||||||
|
if match != None:
|
||||||
|
return match.group(0)
|
||||||
|
else:
|
||||||
|
print("ERROR: Recived invalid state for " + name)
|
||||||
|
sys.exit(1)
|
||||||
|
else:
|
||||||
|
print("ERROR: Unable to get status.")
|
||||||
|
sys.exit(1)
|
||||||
|
except (ConnectionError, OSError, IOError, json.JSONDecodeError):
|
||||||
|
print("ERROR: Unable to get status.")
|
||||||
|
sys.exit(1)
|
||||||
|
def api_get_options(name, api, api_token):
|
||||||
|
try:
|
||||||
|
match = re.match(r"[ew]\d{3}[ns]\d{2}", name)
|
||||||
|
if match != None:
|
||||||
|
response = requests.post(api, data={'auth': api_token, 'action': 'get-options', 'area': name}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||||
|
else:
|
||||||
|
response = requests.post(api, data={'auth': api_token, 'action': 'get-options', 'tile': name}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||||
|
if response.ok and response.json()["success"] == True:
|
||||||
|
return response.json()["options"]
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
except (ConnectionError, OSError, IOError, json.JSONDecodeError):
|
||||||
|
return None
|
||||||
40
config.py.example
Normal file
40
config.py.example
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
import re
|
||||||
|
import json
|
||||||
|
|
||||||
|
api = "" # fill in the api url and token
|
||||||
|
api_token = ""
|
||||||
|
level = "area" # either area or tile,
|
||||||
|
process_job_status = "rebuild" # build tiles makred as this
|
||||||
|
process_status_started = "started" # set tiles we've got a job for as this
|
||||||
|
|
||||||
|
# called pre-build
|
||||||
|
def get_command(name, options):
|
||||||
|
match = re.match("^([we])(\d{3})([sn])(\d{2})", name)
|
||||||
|
if match == None:
|
||||||
|
return (None, "failed")
|
||||||
|
if match.group(1) == "w":
|
||||||
|
w = -1 * int(match.group(2))
|
||||||
|
e = str(w + 1)
|
||||||
|
w = str(w)
|
||||||
|
else:
|
||||||
|
w = int(match.group(2))
|
||||||
|
e = str(w + 1)
|
||||||
|
w = str(w)
|
||||||
|
if match.group(3) == "s":
|
||||||
|
s = -1 * int(match.group(4))
|
||||||
|
n = str(s + 1)
|
||||||
|
s = str(s)
|
||||||
|
else:
|
||||||
|
s = int(match.group(4))
|
||||||
|
n = str(s + 1)
|
||||||
|
s = str(s)
|
||||||
|
|
||||||
|
cmd = "" # command to run, fill in the coordinates
|
||||||
|
|
||||||
|
return (cmd, "started")
|
||||||
|
|
||||||
|
# called post-run, gets the return code
|
||||||
|
def check_result(return_code):
|
||||||
|
if return_code == 0:
|
||||||
|
return "done"
|
||||||
|
return "failed"
|
||||||
110
worldbuild.py
Executable file
110
worldbuild.py
Executable file
|
|
@ -0,0 +1,110 @@
|
||||||
|
#! /usr/bin/python3
|
||||||
|
# Copyright (C) 2018-2023 Merspieler, merspieler _at_ airmail.cc
|
||||||
|
#
|
||||||
|
# This program is free software; you can redistribute it and/or
|
||||||
|
# modify it under the terms of the GNU General Public License as
|
||||||
|
# published by the Free Software Foundation; either version 3 of the
|
||||||
|
# License, or (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful, but
|
||||||
|
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
# General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program; if not, write to the Free Software
|
||||||
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||||
|
|
||||||
|
import os
|
||||||
|
from subprocess import run, Popen, STDOUT
|
||||||
|
import sys
|
||||||
|
import socket
|
||||||
|
import re
|
||||||
|
from time import sleep, strftime, time
|
||||||
|
import threading
|
||||||
|
import signal
|
||||||
|
|
||||||
|
from common import send_status, norm, get_south, get_west, get_east, get_north, get_area_name, api_send_status, api_get_job, api_get_options
|
||||||
|
import config
|
||||||
|
|
||||||
|
threads = 1
|
||||||
|
quiet = False
|
||||||
|
running = True
|
||||||
|
|
||||||
|
argc = len(sys.argv)
|
||||||
|
i = 1
|
||||||
|
first = 1
|
||||||
|
while i < argc:
|
||||||
|
if sys.argv[i] == "-T" or sys.argv[i] == "--threads":
|
||||||
|
i += 1
|
||||||
|
threads = int(sys.argv[i])
|
||||||
|
elif sys.argv[i] == "-q" or sys.argv[i] == "--quiet":
|
||||||
|
quiet = True
|
||||||
|
elif sys.argv[i] == "-h" or sys.argv[i] == "--help":
|
||||||
|
print("usage:worldbuild.py [OPTIONS]")
|
||||||
|
print("TODO")
|
||||||
|
print("")
|
||||||
|
print(" -T, --threads n Number of concurent threads")
|
||||||
|
print(" -q, --quiet Don't print messages")
|
||||||
|
print("")
|
||||||
|
print(" -h, --help Shows this help and exit")
|
||||||
|
sys.exit(0)
|
||||||
|
else:
|
||||||
|
print("Unknown option " + sys.argv[i])
|
||||||
|
sys.exit(1)
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
if config.api == None:
|
||||||
|
print("Error: No API url given")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if config.api_token == None:
|
||||||
|
print("Error: No API token given")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if threads < 1:
|
||||||
|
print("Error: Invalid number of threads given")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
run("mkdir -p logs", shell=True)
|
||||||
|
|
||||||
|
def processThread():
|
||||||
|
global running
|
||||||
|
while running:
|
||||||
|
name = api_get_job(config.process_job_status, config.process_status_started, config.api, config.api_token, config.level)
|
||||||
|
if name == None and running:
|
||||||
|
sleep(60)
|
||||||
|
continue
|
||||||
|
options = api_get_options(name, config.api, config.api_token)
|
||||||
|
cmd, status = config.get_command(name, options)
|
||||||
|
# If config tells us a different status than we expect
|
||||||
|
if status != config.process_status_started:
|
||||||
|
api_send_status(name, status, config.api, config.api_token)
|
||||||
|
if cmd != None:
|
||||||
|
logpath = "logs/" + name + "/"
|
||||||
|
run("mkdir -p " + logpath, shell=True)
|
||||||
|
with open(logpath + name + "-" + strftime("%Y%m%d-%H%M") + ".log", "w") as log_file:
|
||||||
|
build = Popen(cmd, stdout=log_file, stderr=STDOUT, shell=True, start_new_session=True)
|
||||||
|
|
||||||
|
ret = build.wait()
|
||||||
|
status = config.check_result(ret)
|
||||||
|
api_send_status(name, status, config.api, config.api_token)
|
||||||
|
|
||||||
|
|
||||||
|
def exit(signal, frame):
|
||||||
|
global running
|
||||||
|
print("Stopping threads...")
|
||||||
|
running = False
|
||||||
|
for t in threadObjs:
|
||||||
|
t.join()
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
signal.signal(signal.SIGINT, exit)
|
||||||
|
threadObjs = []
|
||||||
|
for i in range(0, threads):
|
||||||
|
t = threading.Thread(target=processThread)
|
||||||
|
t.start()
|
||||||
|
threadObjs.append(t)
|
||||||
|
|
||||||
|
while True:
|
||||||
|
sleep(315360000)
|
||||||
Loading…
Reference in a new issue