Initial commit
Signed-off-by: fly <merspieler@airmail.cc>
This commit is contained in:
commit
dfae55128e
4 changed files with 397 additions and 0 deletions
5
.gitignore
vendored
Normal file
5
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
__pycache__
|
||||
config.py
|
||||
logs
|
||||
*.swp
|
||||
*.swo
|
||||
242
common.py
Normal file
242
common.py
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
#! /usr/bin/python3
|
||||
# Copyright (C) 2020 Merspieler, merspieler _at_ airmail.cc
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License as
|
||||
# published by the Free Software Foundation; either version 3 of the
|
||||
# License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
|
||||
import math
|
||||
import sys
|
||||
import socket
|
||||
import re
|
||||
import requests
|
||||
import json
|
||||
from time import sleep
|
||||
|
||||
# Adds leading 0s
|
||||
def norm(num, length):
|
||||
num = str(num)
|
||||
while len(num) < length:
|
||||
num = "0" + num
|
||||
return num
|
||||
|
||||
# Returns tile width depending on how north/south the tile is
|
||||
def get_tile_width(lat):
|
||||
if abs(lat) >= 89:
|
||||
tile_width = 12
|
||||
elif abs(lat) >= 86:
|
||||
tile_width = 4
|
||||
elif abs(lat) >= 83:
|
||||
tile_width = 2
|
||||
elif abs(lat) >= 76:
|
||||
tile_width = 1
|
||||
elif abs(lat) >= 62:
|
||||
tile_width = 0.5
|
||||
elif abs(lat) >= 22:
|
||||
tile_width = 0.25
|
||||
else:
|
||||
tile_width = 0.125
|
||||
return tile_width
|
||||
|
||||
# Returns the tile name for the given coordinates
|
||||
def get_tile(lat, lon):
|
||||
|
||||
tile_width = get_tile_width(lat)
|
||||
|
||||
base_y = math.floor(lat)
|
||||
y = math.trunc((lat - base_y) * 8)
|
||||
base_x = math.floor(math.floor(lon / tile_width) * tile_width)
|
||||
x = math.floor((lon - base_x) / tile_width)
|
||||
return (int(lon + 180) << 14) + (int(lat + 90) << 6) + (y << 3) + x
|
||||
|
||||
# Returns the area name for a given lat lon ie. e145s17
|
||||
def get_area_name(s, w, major=False):
|
||||
if s >= 0:
|
||||
ns = "n"
|
||||
else:
|
||||
ns = "s"
|
||||
if w >= 0:
|
||||
ew = "e"
|
||||
else:
|
||||
ew = "w"
|
||||
if major:
|
||||
if w % 10 != 0:
|
||||
w = w - (w % 10)
|
||||
|
||||
if s % 10 != 0:
|
||||
s = s - (s % 10)
|
||||
return ew + norm(abs(math.floor(w)), 3) + ns + norm(abs(math.floor(s)), 2)
|
||||
|
||||
# Returns the area name for a given tile
|
||||
def get_area_name_by_tile(tile, major=False):
|
||||
return get_area_name(get_south(tile), get_west(tile), major)
|
||||
|
||||
# Returns latitude of tiles SW corner in area scheme
|
||||
def get_lat(tile):
|
||||
return ((16320 & int(tile)) >> 6) - 90
|
||||
|
||||
# Returns longditude of tiles SW corner in area scheme
|
||||
def get_lon(tile):
|
||||
return ((16760832 & int(tile)) >> 14) - 180
|
||||
|
||||
# Returns precise west boundary of tile
|
||||
def get_west(tile):
|
||||
lon = ((16760832 & int(tile)) >> 14) - 180
|
||||
y = (7 & int(tile)) / (1 / get_tile_width(get_south(tile)))
|
||||
return lon + y
|
||||
|
||||
# Returns east boundary of tile
|
||||
def get_east(tile):
|
||||
return get_west(tile) + get_tile_width(get_lat(tile))
|
||||
|
||||
# Returns precise south boundary of tile
|
||||
def get_south(tile):
|
||||
lat = ((16320 & int(tile)) >> 6) - 90
|
||||
x = ((56 & int(tile)) >> 3) / 8
|
||||
return lat + x
|
||||
|
||||
# Returns north boundary of tile
|
||||
def get_north(tile):
|
||||
return get_south(tile) + 0.125
|
||||
|
||||
# Sends status to the manager
|
||||
def send_status(name, status, host, port):
|
||||
try:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.connect((host, port))
|
||||
sock.send(("set " + name + " " + status).encode())
|
||||
sock.close()
|
||||
except IOError:
|
||||
print("Unable to send status " + status + " for tile " + name + ". Aborting...")
|
||||
sys.exit(1)
|
||||
|
||||
# Sends status to the manager api
|
||||
def api_send_status(name, status, api, token):
|
||||
success = False
|
||||
while not success:
|
||||
try:
|
||||
match = re.match(r"[0-9]{1,7}", name)
|
||||
if match != None:
|
||||
response = requests.post(api, data={'auth': token, 'action': 'set', 'tile': name, 'status': status}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||
else:
|
||||
response = requests.post(api, data={'auth': token, 'action': 'set', 'area': name, 'status': status}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||
if response.ok:
|
||||
success = response.json()["success"]
|
||||
if not success:
|
||||
print("Warning: Unable to send status '" + status + "' for tile '" + name + "'. Trying again in 60 seconds")
|
||||
sleep(60)
|
||||
else:
|
||||
print("Warning: Unable to send status '" + status + "' for tile '" + name + "'. Trying again in 60 seconds")
|
||||
sleep(60)
|
||||
except (ConnectionError, OSError, IOError, json.JSONDecodeError):
|
||||
print("Warning: Unable to send status " + status + " for tile " + name + ". Trying again in 60 seconds")
|
||||
sleep(60)
|
||||
|
||||
# Gets new job from manager
|
||||
def get_job(action, host, port, none_exit=True):
|
||||
try:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.connect((host, port))
|
||||
sock.send(("get " + action).encode())
|
||||
msg = sock.recv(128)
|
||||
sock.close()
|
||||
msg = msg.decode()
|
||||
match = re.match(r"[ew]\d{3}[ns]\d{2}|[0-9]{1,7}|None", msg)
|
||||
if match != None:
|
||||
ret = match.group(0)
|
||||
if ret == "None" and none_exit:
|
||||
print("No job got asigned. Exiting...")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("Recived invalid job. Retrying in 10 seconds...")
|
||||
sleep(10)
|
||||
ret = get_job(action, host, port, none_exit)
|
||||
return ret
|
||||
except IOError:
|
||||
print("Unable to get job. Retrying in 10 seconds...")
|
||||
sleep(10)
|
||||
ret = get_job(action, host, port, none_exit)
|
||||
return ret
|
||||
|
||||
# Gets new job from manager api
|
||||
def api_get_job(action, new_status, api, token, level, none_exit=True, all_in_parent=0):
|
||||
try:
|
||||
response = requests.post(api, data={'auth': token, 'action': 'get-job', 'status': action, 'new-status': new_status, "all-in-parent": all_in_parent, "level": level}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||
if response.ok and response.json()["success"] == True:
|
||||
match = re.match(r"[0-9]{1,7}|[we]\d{3}[sn]\d{2}", str(response.json()["job"]))
|
||||
if match != None:
|
||||
ret = match.group(0)
|
||||
if ret != "None" and all_in_parent == 1:
|
||||
return response.json()["jobs"]
|
||||
return ret
|
||||
else:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
except (ConnectionError, OSError, IOError, json.JSONDecodeError):
|
||||
print(response.text)
|
||||
return None
|
||||
|
||||
# Gets status of a tile
|
||||
def get_status(name, host, port):
|
||||
try:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.connect((host, port))
|
||||
sock.send(("status " + name).encode())
|
||||
msg = sock.recv(128)
|
||||
sock.close()
|
||||
msg = msg.decode()
|
||||
match = re.match(r"pending|done|rebuild|skip|started|packaged", msg)
|
||||
if match != None:
|
||||
return match.group(0)
|
||||
else:
|
||||
print("ERROR: Recived invalid state for tile " + name)
|
||||
sys.exit(1)
|
||||
except IOError:
|
||||
print("ERROR: Unable to get status.")
|
||||
sys.exit(1)
|
||||
|
||||
# Gets status of a tile from api
|
||||
def api_get_status(name, api, api_token):
|
||||
try:
|
||||
match = re.match(r"[ew]\d{3}[ns]\d{2}", name)
|
||||
if match != None:
|
||||
response = requests.post(api, data={'auth': api_token, 'action': 'status', 'area': name}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||
else:
|
||||
response = requests.post(api, data={'auth': api_token, 'action': 'status', 'tile': name}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||
if response.ok and response.json()["success"] == True:
|
||||
match = re.match(r"pending|done|rebuild|skip|started|packaged", response.json()["status"])
|
||||
if match != None:
|
||||
return match.group(0)
|
||||
else:
|
||||
print("ERROR: Recived invalid state for " + name)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("ERROR: Unable to get status.")
|
||||
sys.exit(1)
|
||||
except (ConnectionError, OSError, IOError, json.JSONDecodeError):
|
||||
print("ERROR: Unable to get status.")
|
||||
sys.exit(1)
|
||||
def api_get_options(name, api, api_token):
|
||||
try:
|
||||
match = re.match(r"[ew]\d{3}[ns]\d{2}", name)
|
||||
if match != None:
|
||||
response = requests.post(api, data={'auth': api_token, 'action': 'get-options', 'area': name}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||
else:
|
||||
response = requests.post(api, data={'auth': api_token, 'action': 'get-options', 'tile': name}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
|
||||
if response.ok and response.json()["success"] == True:
|
||||
return response.json()["options"]
|
||||
else:
|
||||
return None
|
||||
except (ConnectionError, OSError, IOError, json.JSONDecodeError):
|
||||
return None
|
||||
40
config.py.example
Normal file
40
config.py.example
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
import re
|
||||
import json
|
||||
|
||||
api = "" # fill in the api url and token
|
||||
api_token = ""
|
||||
level = "area" # either area or tile,
|
||||
process_job_status = "rebuild" # build tiles makred as this
|
||||
process_status_started = "started" # set tiles we've got a job for as this
|
||||
|
||||
# called pre-build
|
||||
def get_command(name, options):
|
||||
match = re.match("^([we])(\d{3})([sn])(\d{2})", name)
|
||||
if match == None:
|
||||
return (None, "failed")
|
||||
if match.group(1) == "w":
|
||||
w = -1 * int(match.group(2))
|
||||
e = str(w + 1)
|
||||
w = str(w)
|
||||
else:
|
||||
w = int(match.group(2))
|
||||
e = str(w + 1)
|
||||
w = str(w)
|
||||
if match.group(3) == "s":
|
||||
s = -1 * int(match.group(4))
|
||||
n = str(s + 1)
|
||||
s = str(s)
|
||||
else:
|
||||
s = int(match.group(4))
|
||||
n = str(s + 1)
|
||||
s = str(s)
|
||||
|
||||
cmd = "" # command to run, fill in the coordinates
|
||||
|
||||
return (cmd, "started")
|
||||
|
||||
# called post-run, gets the return code
|
||||
def check_result(return_code):
|
||||
if return_code == 0:
|
||||
return "done"
|
||||
return "failed"
|
||||
110
worldbuild.py
Executable file
110
worldbuild.py
Executable file
|
|
@ -0,0 +1,110 @@
|
|||
#! /usr/bin/python3
|
||||
# Copyright (C) 2018-2023 Merspieler, merspieler _at_ airmail.cc
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License as
|
||||
# published by the Free Software Foundation; either version 3 of the
|
||||
# License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
|
||||
import os
|
||||
from subprocess import run, Popen, STDOUT
|
||||
import sys
|
||||
import socket
|
||||
import re
|
||||
from time import sleep, strftime, time
|
||||
import threading
|
||||
import signal
|
||||
|
||||
from common import send_status, norm, get_south, get_west, get_east, get_north, get_area_name, api_send_status, api_get_job, api_get_options
|
||||
import config
|
||||
|
||||
threads = 1
|
||||
quiet = False
|
||||
running = True
|
||||
|
||||
argc = len(sys.argv)
|
||||
i = 1
|
||||
first = 1
|
||||
while i < argc:
|
||||
if sys.argv[i] == "-T" or sys.argv[i] == "--threads":
|
||||
i += 1
|
||||
threads = int(sys.argv[i])
|
||||
elif sys.argv[i] == "-q" or sys.argv[i] == "--quiet":
|
||||
quiet = True
|
||||
elif sys.argv[i] == "-h" or sys.argv[i] == "--help":
|
||||
print("usage:worldbuild.py [OPTIONS]")
|
||||
print("TODO")
|
||||
print("")
|
||||
print(" -T, --threads n Number of concurent threads")
|
||||
print(" -q, --quiet Don't print messages")
|
||||
print("")
|
||||
print(" -h, --help Shows this help and exit")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("Unknown option " + sys.argv[i])
|
||||
sys.exit(1)
|
||||
i += 1
|
||||
|
||||
if config.api == None:
|
||||
print("Error: No API url given")
|
||||
sys.exit(1)
|
||||
|
||||
if config.api_token == None:
|
||||
print("Error: No API token given")
|
||||
sys.exit(1)
|
||||
|
||||
if threads < 1:
|
||||
print("Error: Invalid number of threads given")
|
||||
sys.exit(1)
|
||||
|
||||
run("mkdir -p logs", shell=True)
|
||||
|
||||
def processThread():
|
||||
global running
|
||||
while running:
|
||||
name = api_get_job(config.process_job_status, config.process_status_started, config.api, config.api_token, config.level)
|
||||
if name == None and running:
|
||||
sleep(60)
|
||||
continue
|
||||
options = api_get_options(name, config.api, config.api_token)
|
||||
cmd, status = config.get_command(name, options)
|
||||
# If config tells us a different status than we expect
|
||||
if status != config.process_status_started:
|
||||
api_send_status(name, status, config.api, config.api_token)
|
||||
if cmd != None:
|
||||
logpath = "logs/" + name + "/"
|
||||
run("mkdir -p " + logpath, shell=True)
|
||||
with open(logpath + name + "-" + strftime("%Y%m%d-%H%M") + ".log", "w") as log_file:
|
||||
build = Popen(cmd, stdout=log_file, stderr=STDOUT, shell=True, start_new_session=True)
|
||||
|
||||
ret = build.wait()
|
||||
status = config.check_result(ret)
|
||||
api_send_status(name, status, config.api, config.api_token)
|
||||
|
||||
|
||||
def exit(signal, frame):
|
||||
global running
|
||||
print("Stopping threads...")
|
||||
running = False
|
||||
for t in threadObjs:
|
||||
t.join()
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGINT, exit)
|
||||
threadObjs = []
|
||||
for i in range(0, threads):
|
||||
t = threading.Thread(target=processThread)
|
||||
t.start()
|
||||
threadObjs.append(t)
|
||||
|
||||
while True:
|
||||
sleep(315360000)
|
||||
Loading…
Reference in a new issue