From dfae55128eb99cf01926d77f0827b2ef79047527 Mon Sep 17 00:00:00 2001 From: fly Date: Tue, 24 Jan 2023 19:56:57 +0000 Subject: [PATCH] Initial commit Signed-off-by: fly --- .gitignore | 5 + common.py | 242 ++++++++++++++++++++++++++++++++++++++++++++++ config.py.example | 40 ++++++++ worldbuild.py | 110 +++++++++++++++++++++ 4 files changed, 397 insertions(+) create mode 100644 .gitignore create mode 100644 common.py create mode 100644 config.py.example create mode 100755 worldbuild.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..37452ec --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__ +config.py +logs +*.swp +*.swo diff --git a/common.py b/common.py new file mode 100644 index 0000000..3be89d9 --- /dev/null +++ b/common.py @@ -0,0 +1,242 @@ +#! /usr/bin/python3 +# Copyright (C) 2020 Merspieler, merspieler _at_ airmail.cc +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +import math +import sys +import socket +import re +import requests +import json +from time import sleep + +# Adds leading 0s +def norm(num, length): + num = str(num) + while len(num) < length: + num = "0" + num + return num + +# Returns tile width depending on how north/south the tile is +def get_tile_width(lat): + if abs(lat) >= 89: + tile_width = 12 + elif abs(lat) >= 86: + tile_width = 4 + elif abs(lat) >= 83: + tile_width = 2 + elif abs(lat) >= 76: + tile_width = 1 + elif abs(lat) >= 62: + tile_width = 0.5 + elif abs(lat) >= 22: + tile_width = 0.25 + else: + tile_width = 0.125 + return tile_width + +# Returns the tile name for the given coordinates +def get_tile(lat, lon): + + tile_width = get_tile_width(lat) + + base_y = math.floor(lat) + y = math.trunc((lat - base_y) * 8) + base_x = math.floor(math.floor(lon / tile_width) * tile_width) + x = math.floor((lon - base_x) / tile_width) + return (int(lon + 180) << 14) + (int(lat + 90) << 6) + (y << 3) + x + +# Returns the area name for a given lat lon ie. e145s17 +def get_area_name(s, w, major=False): + if s >= 0: + ns = "n" + else: + ns = "s" + if w >= 0: + ew = "e" + else: + ew = "w" + if major: + if w % 10 != 0: + w = w - (w % 10) + + if s % 10 != 0: + s = s - (s % 10) + return ew + norm(abs(math.floor(w)), 3) + ns + norm(abs(math.floor(s)), 2) + +# Returns the area name for a given tile +def get_area_name_by_tile(tile, major=False): + return get_area_name(get_south(tile), get_west(tile), major) + +# Returns latitude of tiles SW corner in area scheme +def get_lat(tile): + return ((16320 & int(tile)) >> 6) - 90 + +# Returns longditude of tiles SW corner in area scheme +def get_lon(tile): + return ((16760832 & int(tile)) >> 14) - 180 + +# Returns precise west boundary of tile +def get_west(tile): + lon = ((16760832 & int(tile)) >> 14) - 180 + y = (7 & int(tile)) / (1 / get_tile_width(get_south(tile))) + return lon + y + +# Returns east boundary of tile +def get_east(tile): + return get_west(tile) + get_tile_width(get_lat(tile)) + +# Returns precise south boundary of tile +def get_south(tile): + lat = ((16320 & int(tile)) >> 6) - 90 + x = ((56 & int(tile)) >> 3) / 8 + return lat + x + +# Returns north boundary of tile +def get_north(tile): + return get_south(tile) + 0.125 + +# Sends status to the manager +def send_status(name, status, host, port): + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((host, port)) + sock.send(("set " + name + " " + status).encode()) + sock.close() + except IOError: + print("Unable to send status " + status + " for tile " + name + ". Aborting...") + sys.exit(1) + +# Sends status to the manager api +def api_send_status(name, status, api, token): + success = False + while not success: + try: + match = re.match(r"[0-9]{1,7}", name) + if match != None: + response = requests.post(api, data={'auth': token, 'action': 'set', 'tile': name, 'status': status}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"}) + else: + response = requests.post(api, data={'auth': token, 'action': 'set', 'area': name, 'status': status}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"}) + if response.ok: + success = response.json()["success"] + if not success: + print("Warning: Unable to send status '" + status + "' for tile '" + name + "'. Trying again in 60 seconds") + sleep(60) + else: + print("Warning: Unable to send status '" + status + "' for tile '" + name + "'. Trying again in 60 seconds") + sleep(60) + except (ConnectionError, OSError, IOError, json.JSONDecodeError): + print("Warning: Unable to send status " + status + " for tile " + name + ". Trying again in 60 seconds") + sleep(60) + +# Gets new job from manager +def get_job(action, host, port, none_exit=True): + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((host, port)) + sock.send(("get " + action).encode()) + msg = sock.recv(128) + sock.close() + msg = msg.decode() + match = re.match(r"[ew]\d{3}[ns]\d{2}|[0-9]{1,7}|None", msg) + if match != None: + ret = match.group(0) + if ret == "None" and none_exit: + print("No job got asigned. Exiting...") + sys.exit(0) + else: + print("Recived invalid job. Retrying in 10 seconds...") + sleep(10) + ret = get_job(action, host, port, none_exit) + return ret + except IOError: + print("Unable to get job. Retrying in 10 seconds...") + sleep(10) + ret = get_job(action, host, port, none_exit) + return ret + +# Gets new job from manager api +def api_get_job(action, new_status, api, token, level, none_exit=True, all_in_parent=0): + try: + response = requests.post(api, data={'auth': token, 'action': 'get-job', 'status': action, 'new-status': new_status, "all-in-parent": all_in_parent, "level": level}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"}) + if response.ok and response.json()["success"] == True: + match = re.match(r"[0-9]{1,7}|[we]\d{3}[sn]\d{2}", str(response.json()["job"])) + if match != None: + ret = match.group(0) + if ret != "None" and all_in_parent == 1: + return response.json()["jobs"] + return ret + else: + return None + else: + return None + except (ConnectionError, OSError, IOError, json.JSONDecodeError): + print(response.text) + return None + +# Gets status of a tile +def get_status(name, host, port): + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((host, port)) + sock.send(("status " + name).encode()) + msg = sock.recv(128) + sock.close() + msg = msg.decode() + match = re.match(r"pending|done|rebuild|skip|started|packaged", msg) + if match != None: + return match.group(0) + else: + print("ERROR: Recived invalid state for tile " + name) + sys.exit(1) + except IOError: + print("ERROR: Unable to get status.") + sys.exit(1) + +# Gets status of a tile from api +def api_get_status(name, api, api_token): + try: + match = re.match(r"[ew]\d{3}[ns]\d{2}", name) + if match != None: + response = requests.post(api, data={'auth': api_token, 'action': 'status', 'area': name}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"}) + else: + response = requests.post(api, data={'auth': api_token, 'action': 'status', 'tile': name}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"}) + if response.ok and response.json()["success"] == True: + match = re.match(r"pending|done|rebuild|skip|started|packaged", response.json()["status"]) + if match != None: + return match.group(0) + else: + print("ERROR: Recived invalid state for " + name) + sys.exit(1) + else: + print("ERROR: Unable to get status.") + sys.exit(1) + except (ConnectionError, OSError, IOError, json.JSONDecodeError): + print("ERROR: Unable to get status.") + sys.exit(1) +def api_get_options(name, api, api_token): + try: + match = re.match(r"[ew]\d{3}[ns]\d{2}", name) + if match != None: + response = requests.post(api, data={'auth': api_token, 'action': 'get-options', 'area': name}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"}) + else: + response = requests.post(api, data={'auth': api_token, 'action': 'get-options', 'tile': name}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"}) + if response.ok and response.json()["success"] == True: + return response.json()["options"] + else: + return None + except (ConnectionError, OSError, IOError, json.JSONDecodeError): + return None diff --git a/config.py.example b/config.py.example new file mode 100644 index 0000000..4deb5a9 --- /dev/null +++ b/config.py.example @@ -0,0 +1,40 @@ +import re +import json + +api = "" # fill in the api url and token +api_token = "" +level = "area" # either area or tile, +process_job_status = "rebuild" # build tiles makred as this +process_status_started = "started" # set tiles we've got a job for as this + +# called pre-build +def get_command(name, options): + match = re.match("^([we])(\d{3})([sn])(\d{2})", name) + if match == None: + return (None, "failed") + if match.group(1) == "w": + w = -1 * int(match.group(2)) + e = str(w + 1) + w = str(w) + else: + w = int(match.group(2)) + e = str(w + 1) + w = str(w) + if match.group(3) == "s": + s = -1 * int(match.group(4)) + n = str(s + 1) + s = str(s) + else: + s = int(match.group(4)) + n = str(s + 1) + s = str(s) + + cmd = "" # command to run, fill in the coordinates + + return (cmd, "started") + +# called post-run, gets the return code +def check_result(return_code): + if return_code == 0: + return "done" + return "failed" diff --git a/worldbuild.py b/worldbuild.py new file mode 100755 index 0000000..40d5e46 --- /dev/null +++ b/worldbuild.py @@ -0,0 +1,110 @@ +#! /usr/bin/python3 +# Copyright (C) 2018-2023 Merspieler, merspieler _at_ airmail.cc +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +import os +from subprocess import run, Popen, STDOUT +import sys +import socket +import re +from time import sleep, strftime, time +import threading +import signal + +from common import send_status, norm, get_south, get_west, get_east, get_north, get_area_name, api_send_status, api_get_job, api_get_options +import config + +threads = 1 +quiet = False +running = True + +argc = len(sys.argv) +i = 1 +first = 1 +while i < argc: + if sys.argv[i] == "-T" or sys.argv[i] == "--threads": + i += 1 + threads = int(sys.argv[i]) + elif sys.argv[i] == "-q" or sys.argv[i] == "--quiet": + quiet = True + elif sys.argv[i] == "-h" or sys.argv[i] == "--help": + print("usage:worldbuild.py [OPTIONS]") + print("TODO") + print("") + print(" -T, --threads n Number of concurent threads") + print(" -q, --quiet Don't print messages") + print("") + print(" -h, --help Shows this help and exit") + sys.exit(0) + else: + print("Unknown option " + sys.argv[i]) + sys.exit(1) + i += 1 + +if config.api == None: + print("Error: No API url given") + sys.exit(1) + +if config.api_token == None: + print("Error: No API token given") + sys.exit(1) + +if threads < 1: + print("Error: Invalid number of threads given") + sys.exit(1) + +run("mkdir -p logs", shell=True) + +def processThread(): + global running + while running: + name = api_get_job(config.process_job_status, config.process_status_started, config.api, config.api_token, config.level) + if name == None and running: + sleep(60) + continue + options = api_get_options(name, config.api, config.api_token) + cmd, status = config.get_command(name, options) + # If config tells us a different status than we expect + if status != config.process_status_started: + api_send_status(name, status, config.api, config.api_token) + if cmd != None: + logpath = "logs/" + name + "/" + run("mkdir -p " + logpath, shell=True) + with open(logpath + name + "-" + strftime("%Y%m%d-%H%M") + ".log", "w") as log_file: + build = Popen(cmd, stdout=log_file, stderr=STDOUT, shell=True, start_new_session=True) + + ret = build.wait() + status = config.check_result(ret) + api_send_status(name, status, config.api, config.api_token) + + +def exit(signal, frame): + global running + print("Stopping threads...") + running = False + for t in threadObjs: + t.join() + sys.exit(0) + +signal.signal(signal.SIGINT, exit) +threadObjs = [] +for i in range(0, threads): + t = threading.Thread(target=processThread) + t.start() + threadObjs.append(t) + +while True: + sleep(315360000)