From f6b92b31c01b1e691d9707727a9ffc9568070ea2 Mon Sep 17 00:00:00 2001 From: fly Date: Thu, 5 Mar 2020 19:04:43 +0000 Subject: [PATCH] Move common functions to own file Signed-off-by: fly --- common.py | 153 +++++++++++++++++++++++++++++++++++++++++ flag-rebuild.py | 31 +-------- worldbuild-manager.py | 8 +-- worldbuild-packager.py | 44 +----------- worldbuild-worker.py | 43 +----------- 5 files changed, 161 insertions(+), 118 deletions(-) create mode 100644 common.py diff --git a/common.py b/common.py new file mode 100644 index 0000000..5030a3e --- /dev/null +++ b/common.py @@ -0,0 +1,153 @@ +#! /usr/bin/python3 +# Copyright (C) 2020 Merspieler, merspieler _at_ airmail.cc +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +import math +import sys +import socket +import re + +# Adds leading 0s +def norm(num, length): + num = str(num) + while len(num) < length: + num = "0" + num + return num + +# Returns tile width depending on how north/south the tile is +def get_tile_width(lat): + if abs(lat) >= 89: + tile_width = 12 + elif abs(lat) >= 86: + tile_width = 4 + elif abs(lat) >= 83: + tile_width = 2 + elif abs(lat) >= 76: + tile_width = 1 + elif abs(lat) >= 62: + tile_width = 0.5 + elif abs(lat) >= 22: + tile_width = 0.25 + else: + tile_width = 0.125 + return tile_width + +# Returns the tile name for the given coordinates +def get_tile(lat, lon): + + tile_width = get_tile_width(lat) + + base_y = math.floor(lat) + y = math.trunc((lat - base_y) * 8) + base_x = math.floor(math.floor(lon / tile_width) * tile_width) + x = math.floor((lon - base_x) / tile_width) + return (int(lon + 180) << 14) + (int(lat + 90) << 6) + (y << 3) + x + +# Returns the area name ie. e145s17 +def get_area_name(n, e): + if n >= 0: + ns = "n" + else: + ns = "s" + if e >= 0: + ew = "e" + else: + ew = "w" + return ew + norm(abs(e), 3) + ns + norm(abs(n), 2) + +# Returns latitude of tiles SW corner in area scheme +def get_lat(tile): + return ((16320 & tile) >> 6) - 90 + +# Returns longditude of tiles SW corner in area scheme +def get_lon(tile): + return ((16760832 & tile) >> 14) - 180 + +# Returns precise west boundary of tile +def get_west(tile): + lon = ((16760832 & tile) >> 14) - 180 + y = (7 & tile) / (1 / get_tile_width(get_lat(tile))) + return lon + y + +# Returns east boundary of tile +def get_east(tile): + return get_west(tile) + get_tile_width(get_lat(tile)) + +# Returns precise south boundary of tile +def get_south(tile): + lat = ((16320 & tile) >> 6) - 90 + x = ((56 & tile) >> 3) / 8 + return lat + x + +# Returns north boundary of tile +def get_north(tile): + return get_south(tile) + 0.125 + +# Sends status to the manager +def send_status(name, status): + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((host, port)) + sock.send(("set " + name + " " + status).encode()) + sock.close() + except IOError: + print("Unable to send status " + status + " for tile " + name + ". Aborting...") + sys.exit(1) + +# Gets new job from manager +def get_job(action): + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((host, port)) + sock.send(("get " + action).encode()) + msg = sock.recv(128) + sock.close() + msg = msg.decode() + match = re.match(r"n-pole|s-pole|[ew]\d{3}[ns]\d{2}|None", msg) + if match != None: + ret = match.group(0) + if ret == "None": + print("No job got asigned. Exiting...") + sys.exit(0) + else: + print("Recived invalid job. Retrying in 10 seconds...") + sleep(10) + ret = get_job(action) + return ret + except IOError: + print("Unable to get job. Retrying in 10 seconds...") + sleep(10) + ret = get_job(action) + return ret + +# Gets status of a tile +def get_status(name): + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((host, port)) + sock.send(("status " + name).encode()) + msg = sock.recv(128) + sock.close() + msg = msg.decode() + match = re.match(r"pending|done|rebuild|skip|started|packaged", msg) + if match != None: + return match.group(0) + else: + print("ERROR: Recived invalid state for tile " + name) + sys.exit(1) + except IOError: + print("ERROR: Unable to send status.") + sys.exit(1) diff --git a/flag-rebuild.py b/flag-rebuild.py index eb3b3ce..ec2e9b5 100755 --- a/flag-rebuild.py +++ b/flag-rebuild.py @@ -18,6 +18,7 @@ import socket import re import sys +from common import send_status, get_status host = socket.gethostname() port = 12345 @@ -60,34 +61,6 @@ if lfile == "": print("ERROR: No file given") sys.exit(1) -def get_status(name): - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) - sock.send(("status " + name).encode()) - msg = sock.recv(128) - sock.close() - msg = msg.decode() - match = re.match(r"pending|done|rebuild|skip|started|packaged", msg) - if match != None: - return match.group(0) - else: - print("ERROR: Recived invalid state for tile " + name) - sys.exit(1) - except IOError: - print("ERROR: Unable to send status.") - sys.exit(1) - -def flag_tile(name): - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) - sock.send(("set " + name + " rebuild").encode()) - sock.close() - except IOError: - print("Unable to send status. Aborting...") - sys.exit(1) - try: tiles = [] with open(lfile) as f: @@ -106,6 +79,6 @@ for tile in tiles: if status == "done" or status == "packaged": if verbose: print("Flagging " + tile) - flag_tile(tile) + send_status(tile, "rebuild") elif verbose: print("Skipping " + tile + ". Not yet build") diff --git a/worldbuild-manager.py b/worldbuild-manager.py index c6ff0a7..267510e 100755 --- a/worldbuild-manager.py +++ b/worldbuild-manager.py @@ -21,6 +21,8 @@ import json import os import re +from common import norm + host = socket.gethostname() port = 12345 verbose = False @@ -56,12 +58,6 @@ while i < argc: sys.exit(1) i += 1 -def norm(num, length): - num = str(num) - while len(num) < length: - num = "0" + num - return num - def save_state(state): try: with open(sfile, 'w') as f: diff --git a/worldbuild-packager.py b/worldbuild-packager.py index 8fb42a1..6d3310f 100755 --- a/worldbuild-packager.py +++ b/worldbuild-packager.py @@ -23,6 +23,8 @@ import socket from subprocess import run, Popen, STDOUT from time import sleep +from common import norm, send_status, get_job + host = socket.gethostname() port = 12345 output = "" @@ -62,48 +64,6 @@ if output == "": print("ERROR: No output directory given") sys.exit(1) -def norm(num, length): - num = str(abs(num)) - while len(num) < length: - num = "0" + num - return num - -def send_status(name, status): - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) - sock.send(("set " + name + " " + status).encode()) - sock.close() - except IOError: - print("Unable to send status. Aborting...") - sys.exit(1) - -def get_job(action): - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) - sock.send(("get " + action).encode()) - msg = sock.recv(128) - sock.close() - msg = msg.decode() - match = re.match(r"n-pole|s-pole|[ew]\d{3}[ns]\d{2}|None", msg) - if match != None: - ret = match.group(0) - if ret == "None": - print("No job got asigned. Trying again in one hour.") - sleep(3600) - ret = get_job(action) - else: - print("Recived invalid job. Retrying in 10 seconds...") - sleep(10) - ret = get_job(action) - return ret - except IOError: - print("Unable to get job. Retrying in 10 seconds...") - sleep(10) - ret = get_job(action) - return ret - def get_sha1(fname): if os.path.isfile(fname): sha1 = hashlib.sha1() diff --git a/worldbuild-worker.py b/worldbuild-worker.py index f1ace51..ac42018 100755 --- a/worldbuild-worker.py +++ b/worldbuild-worker.py @@ -22,6 +22,8 @@ import socket import re from time import sleep +from common import send_status, get_job, norm + action = "pending" host = socket.gethostname() port = 12345 @@ -70,47 +72,6 @@ while i < argc: sys.exit(1) i += 1 -def norm(num, length): - num = str(abs(num)) - while len(num) < length: - num = "0" + num - return num - -def send_status(name, status): - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) - sock.send(("set " + name + " " + status).encode()) - sock.close() - except IOError: - print("Unable to send status " + status + " for tile " + name + ". Aborting...") - sys.exit(1) - -def get_job(action): - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) - sock.send(("get " + action).encode()) - msg = sock.recv(128) - sock.close() - msg = msg.decode() - match = re.match(r"n-pole|s-pole|[ew]\d{3}[ns]\d{2}|None", msg) - if match != None: - ret = match.group(0) - if ret == "None": - print("No job got asigned. Exiting...") - sys.exit(0) - else: - print("Recived invalid job. Retrying in 10 seconds...") - sleep(10) - ret = get_job(action) - return ret - except IOError: - print("Unable to get job. Retrying in 10 seconds...") - sleep(10) - ret = get_job(action) - return ret - def cleanup(): if os.path.isfile("projects/worldbuild-" + name + "/osm2city-exceptions.log"): run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + ".exceptions.log", shell=True)