Update-catalog does SGProps parsing of -set.xml
This restores the ability to use includes in -set.xml files visible to the catalog code, and also exposes some problems / validation issues in our -set.xml files. (Which can of course be fixed)
This commit is contained in:
James Turner
parent
0409f339ae
commit
21a53b3537
3 changed files with 337 additions and 31 deletions
|
|
@ -11,6 +11,9 @@
|
|||
<skip>c172</skip>
|
||||
<skip>tu134</skip>
|
||||
</scm>
|
||||
<include-dir>/home/curt/Projects/FlightGear/flightgear-fgdata</include-dir>
|
||||
<include-dir>/home/curt/Projects/FlightGear/flightgear-fgaddon</include-dir>
|
||||
|
||||
<!-- <scm>
|
||||
<type>git</type>
|
||||
<update type="bool">false</update>
|
||||
|
|
|
|||
277
catalog/sgprops.py
Normal file
277
catalog/sgprops.py
Normal file
|
|
@ -0,0 +1,277 @@
|
|||
# SAX for parsing
|
||||
from xml.sax import make_parser, handler, expatreader
|
||||
|
||||
# ElementTree for writing
|
||||
import xml.etree.cElementTree as ET
|
||||
|
||||
import re, os
|
||||
|
||||
class Node(object):
|
||||
def __init__(self, name = '', index = 0, parent = None):
|
||||
self._parent = parent
|
||||
self._name = name
|
||||
self._value = None
|
||||
self._index = index
|
||||
self._children = []
|
||||
|
||||
@property
|
||||
def value(self):
|
||||
return self._value
|
||||
|
||||
@value.setter
|
||||
def value(self, v):
|
||||
self._value = v
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self._name
|
||||
|
||||
@property
|
||||
def index(self):
|
||||
return self._index
|
||||
|
||||
@property
|
||||
def parent(self):
|
||||
return self._parent
|
||||
|
||||
def getChild(self, n, i=None, create = False):
|
||||
|
||||
if i is None:
|
||||
i = 0
|
||||
# parse name as foo[999] if necessary
|
||||
m = re.match(R"(\w+)\[(\d+)\]", n)
|
||||
if m is not None:
|
||||
n = m.group(1)
|
||||
i = int(m.group(2))
|
||||
|
||||
for c in self._children:
|
||||
if (c.name == n) and (c.index == i):
|
||||
return c
|
||||
|
||||
if create:
|
||||
c = Node(n, i, self)
|
||||
self._children.append(c)
|
||||
return c
|
||||
else:
|
||||
raise IndexError("no such child:" + str(n) + " index=" + str(i))
|
||||
|
||||
def addChild(self, n):
|
||||
# adding an existing instance
|
||||
if isinstance(n, Node):
|
||||
n._parent = self
|
||||
n._index = self.firstUnusedIndex(n.name)
|
||||
self._children.append(n)
|
||||
return n
|
||||
|
||||
i = self.firstUnusedIndex(n)
|
||||
# create it via getChild
|
||||
return self.getChild(n, i, create=True)
|
||||
|
||||
def firstUnusedIndex(self, n):
|
||||
usedIndices = frozenset(c.index for c in self.getChildren(n))
|
||||
i = 0
|
||||
while i < 1000:
|
||||
if i not in usedIndices:
|
||||
return i
|
||||
i += 1
|
||||
raise RuntimeException("too many children with name:" + n)
|
||||
|
||||
def hasChild(self, nm):
|
||||
for c in self._children:
|
||||
if (c.name == nm):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def getChildren(self, n = None):
|
||||
if n is None:
|
||||
return self._children
|
||||
|
||||
return [c for c in self._children if c.name == n]
|
||||
|
||||
def getNode(self, path, cr = False):
|
||||
axes = path.split('/')
|
||||
nd = self
|
||||
for ax in axes:
|
||||
nd = nd.getChild(ax, create = cr)
|
||||
|
||||
return nd
|
||||
|
||||
def getValue(self, path, default = None):
|
||||
try:
|
||||
nd = self.getNode(path)
|
||||
return nd.value
|
||||
except:
|
||||
return default
|
||||
|
||||
def write(self, path):
|
||||
root = self._createXMLElement('PropertyList')
|
||||
t = ET.ElementTree(root)
|
||||
t.write(path, 'utf-8', xml_declaration = True)
|
||||
|
||||
def _createXMLElement(self, nm = None):
|
||||
if nm is None:
|
||||
nm = self.name
|
||||
|
||||
n = ET.Element(nm)
|
||||
|
||||
# value and type specification
|
||||
try:
|
||||
if self._value is not None:
|
||||
if isinstance(self._value, basestring):
|
||||
# don't call str() on strings, breaks the
|
||||
# encoding
|
||||
n.text = self._value
|
||||
else:
|
||||
# use str() to turn non-string types into text
|
||||
n.text = str(self._value)
|
||||
if isinstance(self._value, int):
|
||||
n.set('type', 'int')
|
||||
elif isinstance(self._value, float):
|
||||
n.set('type', 'double')
|
||||
elif isinstance(self._value, bool):
|
||||
n.set('type', "bool")
|
||||
except UnicodeEncodeError:
|
||||
print "Encoding error with", self._value, type(self._value)
|
||||
|
||||
# index in parent
|
||||
if (self.index != 0):
|
||||
n.set('n', str(self.index))
|
||||
|
||||
# children
|
||||
for c in self._children:
|
||||
n.append(c._createXMLElement())
|
||||
|
||||
return n;
|
||||
|
||||
|
||||
class PropsHandler(handler.ContentHandler):
|
||||
def __init__(self, root = None, path = None, includePaths = []):
|
||||
self._root = root
|
||||
self._path = path
|
||||
self._basePath = os.path.dirname(path)
|
||||
self._includes = includePaths
|
||||
self._locator = None
|
||||
|
||||
if root is None:
|
||||
# make a nameless root node
|
||||
self._root = Node("", 0)
|
||||
self._current = self._root
|
||||
|
||||
def setDocumentLocator(self, loc):
|
||||
self._locator = loc
|
||||
|
||||
def startElement(self, name, attrs):
|
||||
self._content = None
|
||||
if (name == 'PropertyList'):
|
||||
return
|
||||
|
||||
if 'n' in attrs.keys():
|
||||
try:
|
||||
index = int(attrs['n'])
|
||||
except:
|
||||
print "Invalid index at line:", self._locator.getLineNumber(), "of", self._path
|
||||
self._current = self._current.addChild(name)
|
||||
return
|
||||
|
||||
self._current = self._current.getChild(name, index, create=True)
|
||||
else:
|
||||
self._current = self._current.addChild(name)
|
||||
|
||||
|
||||
if 'include' in attrs.keys():
|
||||
self.handleInclude(attrs['include'])
|
||||
|
||||
self._currentTy = None;
|
||||
if 'type' in attrs.keys():
|
||||
self._currentTy = attrs['type']
|
||||
|
||||
def handleInclude(self, includePath):
|
||||
if includePath.startswith('/'):
|
||||
includePath = includePath[1:]
|
||||
|
||||
p = os.path.join(self._basePath, includePath)
|
||||
if not os.path.exists(p):
|
||||
found = False
|
||||
for i in self._includes:
|
||||
p = os.path.join(i, includePath)
|
||||
if os.path.exists(p):
|
||||
found = True
|
||||
break
|
||||
|
||||
if not found:
|
||||
raise RuntimeError("include file not found", includePath, "at line", self._locator.getLineNumber())
|
||||
|
||||
readProps(p, self._current, self._includes)
|
||||
|
||||
def endElement(self, name):
|
||||
if (name == 'PropertyList'):
|
||||
return
|
||||
|
||||
try:
|
||||
# convert and store value
|
||||
self._current.value = self._content
|
||||
if self._currentTy == "int":
|
||||
self._current.value = int(self._content) if self._content is not None else 0
|
||||
if self._currentTy == "bool":
|
||||
self._current.value = self.parsePropsBool(self._content)
|
||||
if self._currentTy == "double":
|
||||
if self._content is None:
|
||||
self._current.value = 0.0
|
||||
else:
|
||||
if self._content.endswith('f'):
|
||||
self._content = self._content[:-1]
|
||||
self._current.value = float(self._content)
|
||||
except:
|
||||
print "Parse error for value:", self._content, "at line:", self._locator.getLineNumber(), "of:", self._path
|
||||
|
||||
self._current = self._current.parent
|
||||
self._content = None
|
||||
self._currentTy = None
|
||||
|
||||
|
||||
def parsePropsBool(self, content):
|
||||
if content == "True" or content == "true":
|
||||
return True
|
||||
|
||||
if content == "False" or content == "false":
|
||||
return False
|
||||
|
||||
try:
|
||||
icontent = int(content)
|
||||
if icontent is not None:
|
||||
if icontent == 0:
|
||||
return False
|
||||
else:
|
||||
return True;
|
||||
except:
|
||||
return False
|
||||
|
||||
def characters(self, content):
|
||||
if self._content is None:
|
||||
self._content = ''
|
||||
self._content += content
|
||||
|
||||
def endDocument(self):
|
||||
pass
|
||||
|
||||
@property
|
||||
def root(self):
|
||||
return self._root
|
||||
|
||||
def readProps(path, root = None, includePaths = []):
|
||||
parser = make_parser()
|
||||
locator = expatreader.ExpatLocator( parser )
|
||||
h = PropsHandler(root, path, includePaths)
|
||||
h.setDocumentLocator(locator)
|
||||
parser.setContentHandler(h)
|
||||
parser.parse(path)
|
||||
return h.root
|
||||
|
||||
def copy(src, dest):
|
||||
dest.value = src.value
|
||||
|
||||
# recurse over children
|
||||
for c in src.getChildren() :
|
||||
dc = dest.getChild(c.name, i = c.index, create = True)
|
||||
copy(c, dc)
|
||||
|
|
@ -9,6 +9,7 @@ import re
|
|||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import sgprops
|
||||
|
||||
CATALOG_VERSION = 4
|
||||
|
||||
|
|
@ -23,6 +24,8 @@ parser.add_argument("--clean", help="Force regeneration of all zip files",
|
|||
parser.add_argument("dir", help="Catalog directory")
|
||||
args = parser.parse_args()
|
||||
|
||||
includes = []
|
||||
|
||||
# xml node (robust) get text helper
|
||||
def get_xml_text(e):
|
||||
if e != None and e.text != None:
|
||||
|
|
@ -44,29 +47,39 @@ def make_xml_leaf(name, text):
|
|||
|
||||
# return all available aircraft information from the set file as a
|
||||
# dict
|
||||
def scan_set_file(set_file):
|
||||
def scan_set_file(aircraft_dir, set_file):
|
||||
global includes
|
||||
|
||||
base_file = os.path.basename(set_file)
|
||||
base_id = base_file[:-8]
|
||||
#print ' scanning:', base_file
|
||||
parser = ET.XMLParser(remove_blank_text=True)
|
||||
set_xml = ET.parse(set_file, parser)
|
||||
set_node = set_xml.getroot()
|
||||
sim_node = set_node.find('sim')
|
||||
set_path = os.path.join(aircraft_dir, set_file)
|
||||
|
||||
local_includes = includes
|
||||
local_includes.append(aircraft_dir)
|
||||
root_node = sgprops.readProps(set_path, includePaths = local_includes)
|
||||
|
||||
if not root_node.hasChild("sim"):
|
||||
return None
|
||||
|
||||
sim_node = root_node.getChild("sim")
|
||||
if sim_node == None:
|
||||
return None
|
||||
|
||||
variant = {}
|
||||
variant['name'] = get_xml_text(sim_node.find('description'))
|
||||
variant['status'] = get_xml_text(sim_node.find('status'))
|
||||
variant['author'] = get_xml_text(sim_node.find('author'))
|
||||
variant['description'] = get_xml_text(sim_node.find('long-description'))
|
||||
variant['name'] = sim_node.getValue("description", None)
|
||||
variant['status'] = sim_node.getValue("status", None)
|
||||
variant['author'] = sim_node.getValue("author", None)
|
||||
variant['description'] = sim_node.getValue("long-description", None)
|
||||
variant['id'] = base_id
|
||||
rating_node = sim_node.find('rating')
|
||||
if rating_node != None:
|
||||
variant['rating_FDM'] = int(get_xml_text(rating_node.find('FDM')))
|
||||
variant['rating_systems'] = int(get_xml_text(rating_node.find('systems')))
|
||||
variant['rating_cockpit'] = int(get_xml_text(rating_node.find('cockpit')))
|
||||
variant['rating_model'] = int(get_xml_text(rating_node.find('model')))
|
||||
variant['variant-of'] = get_xml_text(sim_node.find('variant-of'))
|
||||
|
||||
if sim_node.hasChild('rating'):
|
||||
rating_node = sim_node.getChild("rating")
|
||||
variant['rating_FDM'] = rating_node.getValue("FDM", 0)
|
||||
variant['rating_systems'] = rating_node.getValue("systems", 0)
|
||||
variant['rating_cockpit'] = rating_node.getValue("cockpit", 0)
|
||||
variant['rating_model'] = rating_node.getValue("model", 0)
|
||||
|
||||
variant['variant-of'] = sim_node.getValue("variant-of", None)
|
||||
#print ' ', variant
|
||||
return variant
|
||||
|
||||
|
|
@ -79,20 +92,24 @@ def scan_aircraft_dir(aircraft_dir):
|
|||
files = os.listdir(aircraft_dir)
|
||||
for file in sorted(files, key=lambda s: s.lower()):
|
||||
if file.endswith('-set.xml'):
|
||||
variant = scan_set_file(os.path.join(aircraft_dir, file))
|
||||
if variant == None:
|
||||
continue
|
||||
if package == None:
|
||||
# just in case no one claims to be master, the first
|
||||
# variant defaults to master, but we will overwrite
|
||||
# this if we find something better.
|
||||
package = variant
|
||||
if not found_master and variant['variant-of'] == '':
|
||||
found_master = True
|
||||
package = variant
|
||||
else:
|
||||
variants.append( {'id': variant['id'],
|
||||
'name': variant['name'] } )
|
||||
try:
|
||||
variant = scan_set_file(aircraft_dir, file)
|
||||
if variant == None:
|
||||
continue
|
||||
if package == None:
|
||||
# just in case no one claims to be master, the first
|
||||
# variant defaults to master, but we will overwrite
|
||||
# this if we find something better.
|
||||
package = variant
|
||||
if not found_master and variant['variant-of'] == '':
|
||||
found_master = True
|
||||
package = variant
|
||||
else:
|
||||
variants.append( {'id': variant['id'],
|
||||
'name': variant['name'] } )
|
||||
except:
|
||||
print "Scanning aircraft -set.xml failed", os.path.join(aircraft_dir, file)
|
||||
|
||||
return (package, variants)
|
||||
|
||||
# use svn commands to report the last change date within dir
|
||||
|
|
@ -186,6 +203,13 @@ if not os.path.isdir(thumbnail_dir):
|
|||
tmp = os.path.join(args.dir, 'zip-excludes.lst')
|
||||
zip_excludes = os.path.realpath(tmp)
|
||||
|
||||
for i in config_node.findall("include-dir"):
|
||||
path = get_xml_text(i)
|
||||
if not os.path.exists(path):
|
||||
print "Skipping missing include path:", path
|
||||
continue
|
||||
includes.append(path)
|
||||
|
||||
# freshen repositories
|
||||
if args.no_update:
|
||||
print 'Skipping repository updates.'
|
||||
|
|
@ -194,6 +218,8 @@ else:
|
|||
for scm in scm_list:
|
||||
repo_type = get_xml_text(scm.find('type'))
|
||||
repo_path = get_xml_text(scm.find('path'))
|
||||
includes.append(repo_path)
|
||||
|
||||
if repo_type == 'svn':
|
||||
print 'SVN update:', repo_path
|
||||
subprocess.call(['svn', 'update', repo_path])
|
||||
|
|
|
|||
Loading…
Reference in a new issue