Updated packager and worker for api changes

Signed-off-by: fly <merspieler@airmail.cc>

common.py

@@ -58,7 +58,7 @@ def get_tile(lat, lon):
 	x = math.floor((lon - base_x) / tile_width)
 	return (int(lon + 180) << 14) + (int(lat + 90) << 6) + (y << 3) + x
 
-# Returns the area name ie. e145s17
+# Returns the area name for a given lat lon ie. e145s17
 def get_area_name(s, w, major=False):
 	if s >= 0:
 		ns = "n"
@@ -76,6 +76,10 @@ def get_area_name(s, w, major=False):
 		    s = s - (s % 10)
 	return ew + norm(abs(math.floor(w)), 3) + ns + norm(abs(math.floor(s)), 2)
 
+# Returns the area name for a given tile
+def get_area_name_by_tile(tile, major=False):
+	return get_area_name(get_south(tile), get_west(tile), major)
+
 # Returns latitude of tiles SW corner in area scheme
 def get_lat(tile):
 	return ((16320 & int(tile)) >> 6) - 90
@@ -164,30 +168,9 @@ def get_job(action, host, port, none_exit=True):
 		return ret
 
 # Gets new job from manager api
-def api_get_job(action, api, token, none_exit=True):
+def api_get_job(action, new_status, api, token, none_exit=True):
 	try:
-		# TODO 'if' is temporary until the database is reworked
+		response = requests.post(api, data={'auth': token, 'action': 'get-job', 'status': action, 'new-status': new_status}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
-		if action == "done":
-			response = requests.post(api, data={'auth': token, 'action': 'get-done'}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
-			if response.ok and response.json()["success"] == True:
-				match = re.match(r"[ew]\d{3}[ns]\d{2}|None", str(response.json()["job"]))
-				if match != None:
-					ret = match.group(0)
-					if ret == "None" and none_exit:
-						print("No job got asigned. Exiting...")
-						sys.exit(0)
-				else:
-					print("Recived invalid job. Retrying in 10 seconds...")
-					sleep(10)
-					ret = api_get_job(action, api, token, none_exit)
-				return ret
-			else:
-				print("Unable to get job. Retrying in 60 seconds...")
-				sleep(60)
-				ret = api_get_job(action, api, token, none_exit)
-				return ret
-		else:
-			response = requests.post(api, data={'auth': token, 'action': 'get-job', 'type': action}, headers={"lAccept": "application/json", "Content-Type": "application/x-www-form-urlencoded"})
 		if response.ok and response.json()["success"] == True:
 			match = re.match(r"[ew]\d{3}[ns]\d{2}|[0-9]{1,7}|None", str(response.json()["job"]))
 			if match != None:
@@ -198,17 +181,17 @@ def api_get_job(action, api, token, none_exit=True):
 			else:
 				print("Recived invalid job. Retrying in 10 seconds...")
 				sleep(10)
-					ret = api_get_job(action, api, token, none_exit)
+				ret = api_get_job(action, new_status, api, token, none_exit)
 			return ret
 		else:
 			print("Unable to get job. Retrying in 60 seconds...")
 			sleep(60)
-				ret = api_get_job(action, api, token, none_exit)
+			ret = api_get_job(action, new_status, api, token, none_exit)
 			return ret
 	except (ConnectionError, OSError, IOError):
 		print("Unable to get job. Retrying in 60 seconds...")
 		sleep(60)
-		ret = api_get_job(action, api, token, none_exit)
+		ret = api_get_job(action, new_status, api, token, none_exit)
 		return ret
 
 # Gets status of a tile

worldbuild-packager.py

@@ -23,27 +23,26 @@ import socket
 from subprocess import run, Popen, STDOUT
 from time import sleep
 
-from common import norm, send_status, get_job, api_get_job, api_send_status
+from common import norm, get_area_name_by_tile, api_get_job, api_send_status
 
 host = socket.gethostname()
 port = 12345
 api = None
 api_token = None
+input_dir = ""
 output = ""
+tmp_dir = "/tmp/o2c-packager"
 
 argc = len(sys.argv)
 i = 1
 first = 1
 while i < argc:
-	if sys.argv[i] == "--port":
+	if sys.argv[i] == "-o" or sys.argv[i] == "--output":
-		i += 1
-		port = sys.argv[i]
-	elif sys.argv[i] == "--host":
-		i += 1
-		host = sys.argv[i]
-	elif sys.argv[i] == "-o" or sys.argv[i] == "--output":
 		i += 1
 		output = sys.argv[i]
+	if sys.argv[i] == "-i" or sys.argv[i] == "--input":
+		i += 1
+		input_dir = sys.argv[i]
 	elif sys.argv[i] == "-a" or sys.argv[i] == "--api":
 		i += 1
 		api = sys.argv[i]
@@ -54,9 +53,8 @@ while i < argc:
 		print("usage: worldbuild-packager.py [OPTIONS]")
 		print("Retrives done tiles from the manager and packages them")
 		print("")
+		print("  -i  --input      Input directory. Path MUST be absolute")
 		print("  -o, --output     Output directory. Path MUST be absolute")
-		print("      --host       Manager host")
-		print("      --port       Manager port")
 		print("  -a, --api        Manager api url")
 		print("  -t, --api-token  Manager api token")
 		print("  -h, --help       Shows this help and exit")
@@ -74,6 +72,17 @@ if output == "":
 	print("ERROR: No output directory given")
 	sys.exit(1)
 
+if input_dir == "":
+	print("ERROR: No input directory given")
+	sys.exit(1)
+
+if not api:
+	print("ERROR: No api given")
+	sys.exit(1)
+
+if not api_token:
+	print("ERROR: No api token given")
+
 def get_sha1(fname):
 	if os.path.isfile(fname):
 		sha1 = hashlib.sha1()
@@ -146,52 +155,45 @@ while running:
 	try:
 		name = "None"
 		while name == "None":
-			if api == None:
+			name = api_get_job("done", "packaging", api, api_token, none_exit=False)
-				name = get_job("done", host, port, none_exit=False)
-			else:
-				name = api_get_job("done", api, api_token, none_exit=False)
 			if name == "None":
 				print("No job got asigned. Trying again in one hour.")
 				sleep(3600)
 
-		match = re.match(r"([ew])(\d{3})([ns])(\d{2})", name)
-		ew = match.group(1)
-		ew_val = int(match.group(2))
-		ns = match.group(3)
-		ns_val = int(match.group(4))
-
-		ew_val_major = int(ew_val / 10) * 10
-		if ew == "w":
-			if ew_val_major != ew_val:
-				ew_val_major += 10
-
-		ns_val_major = int(ns_val / 10) * 10
-		if ns == "s":
-			if ns_val_major != ns_val:
-				ns_val_major += 10
-
-		name_major = ew + norm(ew_val_major, 3) + ns + norm(ns_val_major, 2)
-
 		print("Packaging " + name + "...")
 
-		for part in ["Buildings", "Details", "Pylons", "Roads"]:
+		area = get_area_name_by_tile(name)
-			run("rm -f " + output + "/" + part + "/" + name_major + "/" + name + ".txz", shell=True)
+		area_major = get_area_name_by_tile(name, major=True)
-			if os.path.exists("projects/worldbuild/scenery/" + part + "/" + name_major + "/" + name):
-				run("mkdir -p " + output + "/" + part + "/" + name_major, shell=True)
 
-				cmd = "bash -c 'cd projects/worldbuild/scenery/" + part + "/" + name_major + " && tar -cJf " + output + "/" + part + "/" + name_major + "/" + name + ".txz " + name + "/'"
+		for part in os.listdir(input_dir):
+#run("rm -f " + output + "/" + part + "/" + area_major + "/" + area + ".txz", shell=True)
+			run("mkdir -p " + tmp_dir + "/" + area)
+
+			# If there is an archive, we need to extract it first and remove old data
+			if os.path.exists(output + "/" + part + "/" + area_major + "/" + area + ".txz"):
+				run("bash -c 'cd " + tmp_dir + "/" + area + " && tar -xf " + output + "/" + part + "/" + area_major + "/" + area + ".txz && rm *" + name + "*'")
+
+			# If there's new scenery copy it into the tmp dir
+			if os.path.exists(input_dir + "/" + part + "/" + area_major + "/" + area + "/" + name + ".stg"):
+				run("cp " + input_dir + "/" + part + "/" + area_major + "/" + area + "/*" + name + "* " + tmp_dir + "/" + area + "/")
+
+			# If the scenery is not empty, package it
+			if os.listdir(tmp_dir + "/" + area):
+				run("mkdir -p " + output + "/" + part + "/" + area_major, shell=True)
+
+				cmd = "bash -c 'cd " + tmp_dir + " && tar -cJf " + output + "/" + part + "/" + area_major + "/" + area + ".txz " + area + "/'"
 				with open(os.devnull, 'w') as fp:
 					package = Popen(cmd , shell=True, start_new_session=True, stdout=fp)
 
 				package.wait()
 
-				update_index(output, part, name, name_major)
+				update_index(output, part, area, area_major)
+
+			# Clean up
+			run("rm -r " + tmp_dir + "/" + area)
 
 		print("Packaging " + name + " done")
 
-		if api == None:
-			send_status(name, "packaged", host, port)
-		else:
 		api_send_status(name, "packaged", api, api_token)
 
 	except KeyboardInterrupt:
@@ -199,10 +201,7 @@ while running:
 		running = False
 		try:
 			package.wait()
-			update_index(output, name, name_major)
+			update_index(output, area, name_major)
-			if api == None:
-				send_status(name, "packaged", host, port)
-			else:
 			api_send_status(name, "packaged", api, api_token)
 		except KeyboardInterrupt:
 			# TODO doesn't work

worldbuild-worker.py

@@ -25,10 +25,8 @@ from time import sleep, strftime, time
 from common import send_status, get_job, norm, get_south, get_west, get_east, get_north, get_area_name, api_send_status, api_get_job
 
 action = "pending"
-host = socket.gethostname()
 api = None
 api_token = None
-port = 12345
 prefix = ""
 quiet = False
 global_db = False
@@ -37,13 +35,7 @@ argc = len(sys.argv)
 i = 1
 first = 1
 while i < argc:
-	if sys.argv[i] == "--port":
+	if sys.argv[i] == "-a" or sys.argv[i] == "--api":
-		i += 1
-		port = sys.argv[i]
-	elif sys.argv[i] == "--host":
-		i += 1
-		host = sys.argv[i]
-	elif sys.argv[i] == "-a" or sys.argv[i] == "--api":
 		i += 1
 		api = sys.argv[i]
 	elif sys.argv[i] == "-t" or sys.argv[i] == "--api-token":
@@ -69,8 +61,6 @@ while i < argc:
 		print("")
 		print("  -p, --prefix     Database prefix to use")
 		print("  -g, --global-db  Use global database")
-		print("      --host       Manager host")
-		print("      --port       Manager port")
 		print("  -a, --api        Manager api url")
 		print("  -t, --api-token  Manager api token")
 		print("  -q, --quiet      Don't print messages")
@@ -86,8 +76,12 @@ while i < argc:
 		sys.exit(1)
 	i += 1
 
-if api != None and api_token == None:
+if api != None:
-	print("Error: API given but no token")
+	print("Error: No API url given")
+	sys.exit(1)
+
+if api_token == None:
+	print("Error: No API token given")
 	sys.exit(1)
 
 def cleanup(name):
@@ -103,10 +97,7 @@ name = None
 try:
 	running = True
 	while running:
-		if api != None:
+		name = api_get_job(action, "started", api, api_token)
-			name = api_get_job(action, api, api_token)
-		else:
-			name = get_job(action, host, port)
 	
 		run("mkdir -p projects/worldbuild-" + name, shell=True)
 		
@@ -151,10 +142,7 @@ try:
 	
 		cleanup(name)
 
-		if api != None:
 		api_send_status(name, "done", api, api_token)
-		else:
-			send_status(name, "done", host, port)
 except KeyboardInterrupt:
 	if not quiet:
 		print("Graceful shutdown triggered. To force immedate stop, press Ctrl+C again")
@@ -163,10 +151,7 @@ except KeyboardInterrupt:
 		if build != None:
 			build.wait()
 			cleanup(name)
-			if api != None:
 			api_send_status(name, "done", api, api_token)
-			else:
-				send_status(name, "done", host, port)
 	except KeyboardInterrupt:
             #TODO doesn't work
 		print("Forcing shutdown...")