Update-catalog does SGProps parsing of -set.xml

This restores the ability to use includes in -set.xml files visible
to the catalog code, and also exposes some problems / validation
issues in our -set.xml files. (Which can of course be fixed)

catalog/fgaddon-catalog/catalog.config.xml

@@ -11,6 +11,9 @@
     <skip>c172</skip>
     <skip>tu134</skip>
   </scm>
+    <include-dir>/home/curt/Projects/FlightGear/flightgear-fgdata</include-dir>
+    <include-dir>/home/curt/Projects/FlightGear/flightgear-fgaddon</include-dir>
+    
   <!-- <scm>
        <type>git</type>
        <update type="bool">false</update>

catalog/sgprops.py

@@ -0,0 +1,277 @@
+# SAX for parsing
+from xml.sax import make_parser, handler, expatreader
+
+# ElementTree for writing
+import xml.etree.cElementTree as ET
+
+import re, os
+
+class Node(object):
+    def __init__(self, name = '', index = 0, parent = None):
+        self._parent = parent
+        self._name = name
+        self._value = None
+        self._index = index
+        self._children = []
+
+    @property
+    def value(self):
+        return self._value
+
+    @value.setter
+    def value(self, v):
+        self._value = v
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def index(self):
+        return self._index
+
+    @property
+    def parent(self):
+        return self._parent
+
+    def getChild(self, n, i=None, create = False):
+
+        if i is None:
+            i = 0
+            # parse name as foo[999] if necessary
+            m = re.match(R"(\w+)\[(\d+)\]", n)
+            if m is not None:
+                n = m.group(1)
+                i = int(m.group(2))
+
+        for c in self._children:
+            if (c.name == n) and (c.index == i):
+                return c
+
+        if create:
+            c = Node(n, i, self)
+            self._children.append(c)
+            return c
+        else:
+            raise IndexError("no such child:" + str(n) + " index=" + str(i))
+
+    def addChild(self, n):
+        # adding an existing instance
+        if isinstance(n, Node):
+            n._parent = self
+            n._index = self.firstUnusedIndex(n.name)
+            self._children.append(n)
+            return n
+
+        i = self.firstUnusedIndex(n)
+        # create it via getChild
+        return self.getChild(n, i, create=True)
+
+    def firstUnusedIndex(self, n):
+        usedIndices = frozenset(c.index for c in self.getChildren(n))
+        i = 0
+        while i < 1000:
+            if i not in usedIndices:
+                 return i
+            i += 1
+        raise RuntimeException("too many children with name:" + n)
+
+    def hasChild(self, nm):
+        for c in self._children:
+            if (c.name == nm):
+                return True
+
+        return False
+
+    def getChildren(self, n = None):
+        if n is None:
+            return self._children
+
+        return [c for c in self._children if c.name == n]
+
+    def getNode(self, path, cr = False):
+        axes = path.split('/')
+        nd = self
+        for ax in axes:
+            nd = nd.getChild(ax, create = cr)
+
+        return nd
+
+    def getValue(self, path, default = None):
+        try:
+            nd = self.getNode(path)
+            return nd.value
+        except:
+            return default
+
+    def write(self, path):
+        root = self._createXMLElement('PropertyList')
+        t = ET.ElementTree(root)
+        t.write(path, 'utf-8', xml_declaration = True)
+
+    def _createXMLElement(self, nm = None):
+        if nm is None:
+            nm = self.name
+
+        n = ET.Element(nm)
+
+        # value and type specification
+        try:
+            if self._value is not None:
+                if isinstance(self._value, basestring):
+                    # don't call str() on strings, breaks the
+                    # encoding
+                    n.text = self._value
+                else:
+                    # use str() to turn non-string types into text
+                    n.text = str(self._value)
+                    if isinstance(self._value, int):
+                        n.set('type', 'int')
+                    elif isinstance(self._value, float):
+                        n.set('type', 'double')
+                    elif isinstance(self._value, bool):
+                        n.set('type', "bool")
+        except UnicodeEncodeError:
+            print "Encoding error with", self._value, type(self._value)
+
+        # index in parent
+        if (self.index != 0):
+            n.set('n', str(self.index))
+
+        # children
+        for c in self._children:
+            n.append(c._createXMLElement())
+
+        return n;
+
+
+class PropsHandler(handler.ContentHandler):
+    def __init__(self, root = None, path = None, includePaths = []):
+        self._root = root
+        self._path = path
+        self._basePath = os.path.dirname(path)
+        self._includes = includePaths
+        self._locator = None
+
+        if root is None:
+            # make a nameless root node
+            self._root = Node("", 0)
+        self._current = self._root
+
+    def setDocumentLocator(self, loc):
+        self._locator = loc
+
+    def startElement(self, name, attrs):
+        self._content = None
+        if (name == 'PropertyList'):
+            return
+
+        if 'n' in attrs.keys():
+            try:
+                index = int(attrs['n'])
+            except:
+                print "Invalid index at line:", self._locator.getLineNumber(), "of", self._path
+                self._current = self._current.addChild(name)
+                return
+
+            self._current = self._current.getChild(name, index, create=True)
+        else:
+            self._current = self._current.addChild(name)
+
+
+        if 'include' in attrs.keys():
+            self.handleInclude(attrs['include'])
+
+        self._currentTy = None;
+        if 'type' in attrs.keys():
+            self._currentTy = attrs['type']
+
+    def handleInclude(self, includePath):
+        if includePath.startswith('/'):
+            includePath = includePath[1:]
+
+        p = os.path.join(self._basePath, includePath)
+        if not os.path.exists(p):
+            found = False
+            for i in self._includes:
+                p = os.path.join(i, includePath)
+                if os.path.exists(p):
+                    found = True
+                    break
+
+            if not found:
+                raise RuntimeError("include file not found", includePath, "at line", self._locator.getLineNumber())
+
+        readProps(p, self._current, self._includes)
+
+    def endElement(self, name):
+        if (name == 'PropertyList'):
+            return
+
+        try:
+            # convert and store value
+            self._current.value = self._content
+            if self._currentTy == "int":
+                self._current.value = int(self._content) if self._content is not None else 0
+            if self._currentTy == "bool":
+                self._current.value = self.parsePropsBool(self._content)
+            if self._currentTy == "double":
+                if self._content is None:
+                    self._current.value = 0.0
+                else:
+                    if self._content.endswith('f'):
+                        self._content = self._content[:-1]
+                    self._current.value = float(self._content)
+        except:
+            print "Parse error for value:", self._content, "at line:", self._locator.getLineNumber(), "of:", self._path
+
+        self._current = self._current.parent
+        self._content = None
+        self._currentTy = None
+
+
+    def parsePropsBool(self, content):
+        if content == "True" or content == "true":
+            return True
+
+        if content == "False" or content == "false":
+            return False
+
+        try:
+            icontent = int(content)
+            if icontent is not None:
+                if icontent == 0:
+                    return False
+                else:
+                    return True;
+        except:
+            return False
+
+    def characters(self, content):
+        if self._content is None:
+            self._content = ''
+        self._content += content
+
+    def endDocument(self):
+        pass
+
+    @property
+    def root(self):
+        return self._root
+
+def readProps(path, root = None, includePaths = []):
+    parser = make_parser()
+    locator = expatreader.ExpatLocator( parser )
+    h = PropsHandler(root, path, includePaths)
+    h.setDocumentLocator(locator)
+    parser.setContentHandler(h)
+    parser.parse(path)
+    return h.root
+
+def copy(src, dest):
+    dest.value = src.value
+
+    # recurse over children
+    for c in src.getChildren() :
+        dc = dest.getChild(c.name, i = c.index, create = True)
+        copy(c, dc)

catalog/update-catalog.py

@@ -9,6 +9,7 @@ import re
 import shutil
 import subprocess
 import time
+import sgprops
 
 CATALOG_VERSION = 4
 
@@ -23,6 +24,8 @@ parser.add_argument("--clean", help="Force regeneration of all zip files",
 parser.add_argument("dir", help="Catalog directory")
 args = parser.parse_args()
 
+includes = []
+
 # xml node (robust) get text helper
 def get_xml_text(e):
     if e != None and e.text != None:
@@ -44,29 +47,39 @@ def make_xml_leaf(name, text):
 
 # return all available aircraft information from the set file as a
 # dict
-def scan_set_file(set_file):
+def scan_set_file(aircraft_dir, set_file):
+    global includes
+
     base_file = os.path.basename(set_file)
     base_id = base_file[:-8]
-    #print '  scanning:', base_file
+    set_path = os.path.join(aircraft_dir, set_file)
-    parser = ET.XMLParser(remove_blank_text=True)
+
-    set_xml = ET.parse(set_file, parser)
+    local_includes = includes
-    set_node = set_xml.getroot()
+    local_includes.append(aircraft_dir)
-    sim_node = set_node.find('sim')
+    root_node = sgprops.readProps(set_path, includePaths = local_includes)
+
+    if not root_node.hasChild("sim"):
+        return None
+
+    sim_node = root_node.getChild("sim")
     if sim_node == None:
         return None
+
     variant = {}
-    variant['name'] = get_xml_text(sim_node.find('description'))
+    variant['name'] = sim_node.getValue("description", None)
-    variant['status'] = get_xml_text(sim_node.find('status'))
+    variant['status'] = sim_node.getValue("status", None)
-    variant['author'] = get_xml_text(sim_node.find('author'))
+    variant['author'] = sim_node.getValue("author", None)
-    variant['description'] = get_xml_text(sim_node.find('long-description'))
+    variant['description'] = sim_node.getValue("long-description", None)
     variant['id'] = base_id
-    rating_node = sim_node.find('rating')
+
-    if rating_node != None:
+    if sim_node.hasChild('rating'):
-        variant['rating_FDM'] = int(get_xml_text(rating_node.find('FDM')))
+        rating_node = sim_node.getChild("rating")
-        variant['rating_systems'] = int(get_xml_text(rating_node.find('systems')))
+        variant['rating_FDM'] = rating_node.getValue("FDM", 0)
-        variant['rating_cockpit'] = int(get_xml_text(rating_node.find('cockpit')))
+        variant['rating_systems'] = rating_node.getValue("systems", 0)
-        variant['rating_model'] = int(get_xml_text(rating_node.find('model')))
+        variant['rating_cockpit'] = rating_node.getValue("cockpit", 0)
-    variant['variant-of'] = get_xml_text(sim_node.find('variant-of'))
+        variant['rating_model'] = rating_node.getValue("model", 0)
+
+    variant['variant-of'] = sim_node.getValue("variant-of", None)
     #print '    ', variant
     return variant
 
@@ -79,7 +92,8 @@ def scan_aircraft_dir(aircraft_dir):
     files = os.listdir(aircraft_dir)
     for file in sorted(files, key=lambda s: s.lower()):
         if file.endswith('-set.xml'):
-            variant = scan_set_file(os.path.join(aircraft_dir, file))
+            try:
+                variant = scan_set_file(aircraft_dir, file)
                 if variant == None:
                     continue
                 if package == None:
@@ -93,6 +107,9 @@ def scan_aircraft_dir(aircraft_dir):
                 else:
                     variants.append( {'id': variant['id'],
                                       'name': variant['name'] } )
+            except:
+                print "Scanning aircraft -set.xml failed", os.path.join(aircraft_dir, file)
+
     return (package, variants)
 
 # use svn commands to report the last change date within dir
@@ -186,6 +203,13 @@ if not os.path.isdir(thumbnail_dir):
 tmp = os.path.join(args.dir, 'zip-excludes.lst')
 zip_excludes = os.path.realpath(tmp)
 
+for i in config_node.findall("include-dir"):
+    path = get_xml_text(i)
+    if not os.path.exists(path):
+        print "Skipping missing include path:", path
+        continue
+    includes.append(path)
+
 # freshen repositories
 if args.no_update:
     print 'Skipping repository updates.'
@@ -194,6 +218,8 @@ else:
     for scm in scm_list:
         repo_type = get_xml_text(scm.find('type'))
         repo_path = get_xml_text(scm.find('path'))
+        includes.append(repo_path)
+
         if repo_type == 'svn':
             print 'SVN update:', repo_path
             subprocess.call(['svn', 'update', repo_path])