fly/osm2city-scripts
1
0
Fork 0
Code Issues 2 Pull requests Activity

Move common functions to own file

Browse source 
Signed-off-by: fly <merspieler@airmail.cc>
This commit is contained in:
fly 2020-03-05 19:04:43 +00:00
parent b339adaa02
commit f6b92b31c0
5 changed files with 161 additions and 118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

153
common.py Normal file
View file

@ -0,0 +1,153 @@
#! /usr/bin/python3
# Copyright (C) 2020 Merspieler, merspieler _at_ airmail.cc
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
import math
import sys
import socket
import re
# Adds leading 0s
def norm(num, length):
num = str(num)
while len(num) < length:
num = "0" + num
return num
# Returns tile width depending on how north/south the tile is
def get_tile_width(lat):
if abs(lat) >= 89:
tile_width = 12
elif abs(lat) >= 86:
tile_width = 4
elif abs(lat) >= 83:
tile_width = 2
elif abs(lat) >= 76:
tile_width = 1
elif abs(lat) >= 62:
tile_width = 0.5
elif abs(lat) >= 22:
tile_width = 0.25
else:
tile_width = 0.125
return tile_width
# Returns the tile name for the given coordinates
def get_tile(lat, lon):
tile_width = get_tile_width(lat)
base_y = math.floor(lat)
y = math.trunc((lat - base_y) * 8)
base_x = math.floor(math.floor(lon / tile_width) * tile_width)
x = math.floor((lon - base_x) / tile_width)
return (int(lon + 180) << 14) + (int(lat + 90) << 6) + (y << 3) + x
# Returns the area name ie. e145s17
def get_area_name(n, e):
if n >= 0:
ns = "n"
else:
ns = "s"
if e >= 0:
ew = "e"
else:
ew = "w"
return ew + norm(abs(e), 3) + ns + norm(abs(n), 2)
# Returns latitude of tiles SW corner in area scheme
def get_lat(tile):
return ((16320 & tile) >> 6) - 90
# Returns longditude of tiles SW corner in area scheme
def get_lon(tile):
return ((16760832 & tile) >> 14) - 180
# Returns precise west boundary of tile
def get_west(tile):
lon = ((16760832 & tile) >> 14) - 180
y = (7 & tile) / (1 / get_tile_width(get_lat(tile)))
return lon + y
# Returns east boundary of tile
def get_east(tile):
return get_west(tile) + get_tile_width(get_lat(tile))
# Returns precise south boundary of tile
def get_south(tile):
lat = ((16320 & tile) >> 6) - 90
x = ((56 & tile) >> 3) / 8
return lat + x
# Returns north boundary of tile
def get_north(tile):
return get_south(tile) + 0.125
# Sends status to the manager
def send_status(name, status):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, port))
sock.send(("set " + name + " " + status).encode())
sock.close()
except IOError:
print("Unable to send status " + status + " for tile " + name + ". Aborting...")
sys.exit(1)
# Gets new job from manager
def get_job(action):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, port))
sock.send(("get " + action).encode())
msg = sock.recv(128)
sock.close()
msg = msg.decode()
match = re.match(r"n-pole|s-pole|[ew]\d{3}[ns]\d{2}|None", msg)
if match != None:
ret = match.group(0)
if ret == "None":
print("No job got asigned. Exiting...")
sys.exit(0)
else:
print("Recived invalid job. Retrying in 10 seconds...")
sleep(10)
ret = get_job(action)
return ret
except IOError:
print("Unable to get job. Retrying in 10 seconds...")
sleep(10)
ret = get_job(action)
return ret
# Gets status of a tile
def get_status(name):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, port))
sock.send(("status " + name).encode())
msg = sock.recv(128)
sock.close()
msg = msg.decode()
match = re.match(r"pending|done|rebuild|skip|started|packaged", msg)
if match != None:
return match.group(0)
else:
print("ERROR: Recived invalid state for tile " + name)
sys.exit(1)
except IOError:
print("ERROR: Unable to send status.")
sys.exit(1)

31
flag-rebuild.py
View file

@ -18,6 +18,7 @@
import socket
import re
import sys
from common import send_status, get_status
host = socket.gethostname()
port = 12345
@ -60,34 +61,6 @@ if lfile == "":
print("ERROR: No file given")
sys.exit(1)
def get_status(name):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, port))
sock.send(("status " + name).encode())
msg = sock.recv(128)
sock.close()
msg = msg.decode()
match = re.match(r"pending|done|rebuild|skip|started|packaged", msg)
if match != None:
return match.group(0)
else:
print("ERROR: Recived invalid state for tile " + name)
sys.exit(1)
except IOError:
print("ERROR: Unable to send status.")
sys.exit(1)
def flag_tile(name):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, port))
sock.send(("set " + name + " rebuild").encode())
sock.close()
except IOError:
print("Unable to send status. Aborting...")
sys.exit(1)
try:
tiles = []
with open(lfile) as f:
@ -106,6 +79,6 @@ for tile in tiles:
if status == "done" or status == "packaged":
if verbose:
print("Flagging " + tile)
flag_tile(tile)
send_status(tile, "rebuild")
elif verbose:
print("Skipping " + tile + ". Not yet build")

8
worldbuild-manager.py
View file

@ -21,6 +21,8 @@ import json
import os
import re
from common import norm
host = socket.gethostname()
port = 12345
verbose = False
@ -56,12 +58,6 @@ while i < argc:
sys.exit(1)
i += 1
def norm(num, length):
num = str(num)
while len(num) < length:
num = "0" + num
return num
def save_state(state):
try:
with open(sfile, 'w') as f:

44
worldbuild-packager.py
View file

@ -23,6 +23,8 @@ import socket
from subprocess import run, Popen, STDOUT
from time import sleep
from common import norm, send_status, get_job
host = socket.gethostname()
port = 12345
output = ""
@ -62,48 +64,6 @@ if output == "":
print("ERROR: No output directory given")
sys.exit(1)
def norm(num, length):
num = str(abs(num))
while len(num) < length:
num = "0" + num
return num
def send_status(name, status):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, port))
sock.send(("set " + name + " " + status).encode())
sock.close()
except IOError:
print("Unable to send status. Aborting...")
sys.exit(1)
def get_job(action):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, port))
sock.send(("get " + action).encode())
msg = sock.recv(128)
sock.close()
msg = msg.decode()
match = re.match(r"n-pole|s-pole|[ew]\d{3}[ns]\d{2}|None", msg)
if match != None:
ret = match.group(0)
if ret == "None":
print("No job got asigned. Trying again in one hour.")
sleep(3600)
ret = get_job(action)
else:
print("Recived invalid job. Retrying in 10 seconds...")
sleep(10)
ret = get_job(action)
return ret
except IOError:
print("Unable to get job. Retrying in 10 seconds...")
sleep(10)
ret = get_job(action)
return ret
def get_sha1(fname):
if os.path.isfile(fname):
sha1 = hashlib.sha1()

43
worldbuild-worker.py
View file

@ -22,6 +22,8 @@ import socket
import re
from time import sleep
from common import send_status, get_job, norm
action = "pending"
host = socket.gethostname()
port = 12345
@ -70,47 +72,6 @@ while i < argc:
sys.exit(1)
i += 1
def norm(num, length):
num = str(abs(num))
while len(num) < length:
num = "0" + num
return num
def send_status(name, status):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, port))
sock.send(("set " + name + " " + status).encode())
sock.close()
except IOError:
print("Unable to send status " + status + " for tile " + name + ". Aborting...")
sys.exit(1)
def get_job(action):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, port))
sock.send(("get " + action).encode())
msg = sock.recv(128)
sock.close()
msg = msg.decode()
match = re.match(r"n-pole|s-pole|[ew]\d{3}[ns]\d{2}|None", msg)
if match != None:
ret = match.group(0)
if ret == "None":
print("No job got asigned. Exiting...")
sys.exit(0)
else:
print("Recived invalid job. Retrying in 10 seconds...")
sleep(10)
ret = get_job(action)
return ret
except IOError:
print("Unable to get job. Retrying in 10 seconds...")
sleep(10)
ret = get_job(action)
return ret
def cleanup():
if os.path.isfile("projects/worldbuild-" + name + "/osm2city-exceptions.log"):
run("mv projects/worldbuild-" + name + "/osm2city-exceptions.log projects/worldbuild/output/error/" + name + ".exceptions.log", shell=True)