#! /usr/bin/python3 # Copyright (C) 2018-2020 Merspieler, merspieler _at_ airmail.cc # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 3 of the # License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. import os import sys import hashlib import re import socket from subprocess import run, Popen, STDOUT from time import sleep from common import norm, send_status, get_job host = socket.gethostname() port = 12345 api = None api_token = None output = "" argc = len(sys.argv) i = 1 first = 1 while i < argc: if sys.argv[i] == "--port": i += 1 port = sys.argv[i] elif sys.argv[i] == "--host": i += 1 host = sys.argv[i] elif sys.argv[i] == "-o" or sys.argv[i] == "--output": i += 1 output = sys.argv[i] elif sys.argv[i] == "-a" or sys.argv[i] == "--api": i += 1 api = sys.argv[i] elif sys.argv[i] == "-t" or sys.argv[i] == "--api-token": i += 1 api_token = sys.argv[i] elif sys.argv[i] == "-h" or sys.argv[i] == "--help": print("usage: worldbuild-packager.py [OPTIONS]") print("Retrives done tiles from the manager and packages them") print("") print(" -o, --output Output directory. Path MUST be absolute") print(" --host Manager host") print(" --port Manager port") print(" -a, --api Manager api url") print(" -t, --api-token Manager api token") print(" -h, --help Shows this help and exit") sys.exit(0) else: if first == 1: first = 0 name = sys.argv[i] else: print("Unknown option " + sys.argv[i]) sys.exit(1) i += 1 if output == "": print("ERROR: No output directory given") sys.exit(1) def get_sha1(fname): if os.path.isfile(fname): sha1 = hashlib.sha1() with open(fname, 'rb') as f: while True: data = f.read(65536) if not data: break sha1.update(data) return sha1.hexdigest() else: return None def read_index(fname): index = {} with open(fname) as f: for line in f: match = re.match("(version|time|path|d:.*):(.*)", line) if match != None: index[match.group(1)] = match.group(2) else: match = re.match("(.*:.*):(.*:.*)", line) if match != None: index[match.group(1)] = match.group(2) return index def write_index(fname, index): with open(fname, 'w') as f: for element in index: f.write(element + ":" + str(index[element]) + "\n") def update_parent_index(base, path, changed_dir): if os.path.isfile(base + "/" + path + "/.dirindex"): index = read_index(base + "/" + path + "/.dirindex") else: index = {} index["version"] = 1 if path != "": index["path"] = path else: index["path"] = "" sha = get_sha1(base + "/" + path + "/" + changed_dir + "/.dirindex") if sha != None: index["d:" + changed_dir] = sha write_index(base + "/" + path + "/.dirindex", index) def update_index(output, part, name, name_major): if os.path.isfile(output + "/" + part + "/" + name_major + "/.dirindex"): index = read_index(output + "/" + part + "/" + name_major + "/.dirindex") else: index = {} index["version"] = 1 index["path"] = part + "/" + name_major sha = get_sha1(output + "/" + part + "/" + name_major + "/" + name + ".txz") if sha != None: index["t:" + name + ".txz"] = sha + ":" + str(os.path.getsize(output + "/" + part + "/" + name_major + "/" + name + ".txz")) write_index(output + "/" + part + "/" + name_major + "/.dirindex", index) update_parent_index(output, part, name_major) update_parent_index(output, "", part) running = True while running: try: name = "None" while name == "None": if api == None: name = get_job("done", host, port, none_exit=False) else: name = api_get_job("done", api, token, none_exit=False) if name == "None": print("No job got asigned. Trying again in one hour.") sleep(3600) match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name) ew = match.group(1) ew_val = int(match.group(2)) ns = match.group(3) ns_val = int(match.group(4)) ew_val_major = int(ew_val / 10) * 10 if ew == "w": if ew_val_major != ew_val: ew_val_major += 10 ns_val_major = int(ns_val / 10) * 10 if ns == "s": if ns_val_major != ns_val: ns_val_major += 10 name_major = ew + norm(ew_val_major, 3) + ns + norm(ns_val_major, 2) print("Packaging " + name + "...") for part in ["Buildings", "Details", "Pylons", "Roads"]: run("rm -f " + output + "/" + part + "/" + name_major + "/" + name + ".txz", shell=True) if os.path.exists("projects/worldbuild/scenery/" + part + "/" + name_major + "/" + name): run("mkdir -p " + output + "/" + part + "/" + name_major, shell=True) cmd = "bash -c 'cd projects/worldbuild/scenery/" + part + "/" + name_major + " && tar -cJf " + output + "/" + part + "/" + name_major + "/" + name + ".txz " + name + "/'" with open(os.devnull, 'w') as fp: package = Popen(cmd , shell=True, start_new_session=True, stdout=fp) package.wait() update_index(output, part, name, name_major) print("Packaging " + name + " done") if api == None: send_status(name, "packaged", host, port) else: api_send_status(name, "packaged", api, token) except KeyboardInterrupt: print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again") running = False try: package.wait() update_index(output, name, name_major) if api == None: send_status(name, "packaged", host, port) else: api_send_status(name, "packaged", api, token) except KeyboardInterrupt: # TODO doesn't work print("Forcing shutdown...") package.terminate() sleep(5) package.kill() sys.exit(0)