Catalog creation work

- XML scanning works
- thumbnail extraction progressing

catalogTags.py

@@ -1,12 +1,13 @@
 aircraftTypeTags = [
-    "ga", "fighter", "helicopter", "glider", "spaceship", "bomber", "groundvehicle", 
+    "ga", "fighter", "helicopter", "glider", "spaceship", "bomber", "groundvehicle",
     "tanker", "cargo", "transport", "bizjet", "trainer", "airship", "balloon"
 ]
 
 manufacturerTags = [
-    "boeing", "cessna", "diamond", "douglas", "bell", "piper", 
+    "boeing", "cessna", "diamond", "douglas", "bell", "piper",
     "airbus", "vickers", "lockheed", "fokker",
-    "embrarer", "bombardier", "pilatus", "robin"
+    "embrarer", "bombardier", "pilatus", "robin",
+    "eurocopter"
 ]
 
 eraTags = [
@@ -21,6 +22,8 @@ eraTags = [
     "1960s",
     "1970s",
     "1980s",
+    "1990s",
+    "2000s",
     "gulfwar1",
     "gulfwar2"
 ]
@@ -69,7 +72,8 @@ propulsionTags = [
 
 simFeatureTags = [
     "tow",
-    "dual-controls"
+    "dual-controls",
+    "rembrandt"
 ]
 
 tags = aircraftTypeTags + manufacturerTags + eraTags + simFeatureTags + propulsionTags + featureTags

maintain_catalog.py

@@ -3,7 +3,7 @@
 import os, sys, re, glob
 import hashlib # for MD5
 import subprocess
-
+import shutil # for copy2
 import catalogTags
 import sgprops
 
@@ -11,10 +11,12 @@ import svn_catalog_repository
 import git_catalog_repository
 import git_discrete_repository
 
-# TODO
+standardTagSet = frozenset(catalogTags.tags)
-# uploading / rsyncing
+def isNonstandardTag(t):
+    return t not in standardTagSet
 
 thumbnailNames = ["thumbnail.png", "thumbnail.jpg"]
+includePaths = []
 
 class VariantData:
     def __init__(self, path, node):
@@ -28,8 +30,8 @@ class VariantData:
 
     @property
     def catalogNode(self):
-        n = Node("variant")
+        n = sgprops.Node("variant")
-        n.addChild("id").value = path
+        n.addChild("id").value = self._path
         n.addChild("name").value = self._name
 
 class PackageData:
@@ -85,15 +87,19 @@ class PackageData:
 
         return True
 
-    def scanSetXmlFiles(self):
+    def scanSetXmlFiles(self, includes):
         foundPrimary = False
+        foundMultiple = False
 
         for f in os.listdir(self._path):
             if not f.endswith("-set.xml"):
                 continue
 
             p = os.path.join(self._path, f)
-            node = sgprops.readProps(p)
+            node = sgprops.readProps(p, includePaths = includes)
+            if not node.hasChild("sim"):
+                continue
+
             simNode = node.getChild("sim")
             if (simNode.getValue("exclude", False)):
                 continue
@@ -106,7 +112,9 @@ class PackageData:
                 continue
 
             if foundPrimary:
-                print "Multiple primary -set.xml files at:" + self._path
+                if not foundMultiple:
+                    print "Multiple primary -set.xml files at:" + self._path
+                    foundMultiple = True
                 continue
             else:
                 foundPrimary = True;
@@ -147,11 +155,12 @@ class PackageData:
         if sim.hasChild('tags'):
             for c in sim.getChild('tags').getChildren('tag'):
                 if isNonstandardTag(c.value):
-                    print "Skipping non-standard tag:", c.value
+                    print "Skipping non-standard tag:", c.value, self.path
                 else:
                     self._node.addChild('tag').value = c.value
 
-        self._thumbnails.append(t.value for t in sim.getChildren("thumbnail"))
+        for t in sim.getChildren("thumbnail"):
+            self._thumbnails.append(t.value)
 
     def validate(self):
         for t in self._thumbnails:
@@ -199,7 +208,7 @@ class PackageData:
     def extractThumbnails(self, thumbnailDir):
         for t in self._thumbnails:
             fullName = self.id + "_" + t
-            os.file.copy(os.path.join(self._path, t),
+            shutil.copy2(os.path.join(self._path, t),
                          os.path.join(thumbnailDir, fullName)
                          )
             # TODO : verify image format, size and so on
@@ -209,6 +218,11 @@ def scanPackages(globPath):
     print "Scanning", globPath
     print os.getcwd()
     for d in glob.glob(globPath):
+        # check dir contains at least one -set.xml file
+        if len(glob.glob(os.path.join(d, "*-set.xml"))) == 0:
+            print "no -set.xml in", d
+            continue
+
         result.append(PackageData(d))
 
     return result
@@ -278,10 +292,19 @@ if not os.path.exists(outPath):
 print "Output path is:" + outPath
 
 thumbnailPath = os.path.join(outPath, config.getValue('thumbnail-dir', "thumbnails"))
+if not os.path.exists(thumbnailPath):
+    os.mkdir(thumbnailPath)
+
 thumbnailUrl = config.getValue('thumbnail-url')
 
 print "Thumbnail url is:", thumbnailUrl
 
+for i in config.getChildren("include-dir"):
+    if not os.path.exists(i.value):
+        print "Skipping missing include path:", i.value
+        continue
+    includePaths.append(i.value)
+
 mirrorUrls = []
 
 # contains existing catalog
@@ -318,7 +341,7 @@ mirrorUrls = (m.value for m in config.getChildren("mirror"))
 
 packagesToGenerate = []
 for p in packages.values():
-    p.scanSetXmlFiles()
+    p.scanSetXmlFiles(includePaths)
 
     if (p.isSourceModified(scmRepo)):
         packagesToGenerate.append(p)

sgprops.py

@@ -146,11 +146,11 @@ class Node(object):
 
 
 class PropsHandler(handler.ContentHandler):
-    def __init__(self, root = None, path = None, dataDirPath = None):
+    def __init__(self, root = None, path = None, includePaths = []):
         self._root = root
         self._path = path
         self._basePath = os.path.dirname(path)
-        self._dataDirPath = dataDirPath
+        self._includes = includePaths
         self._locator = None
 
         if root is None:
@@ -167,7 +167,13 @@ class PropsHandler(handler.ContentHandler):
             return
 
         if 'n' in attrs.keys():
-            index = int(attrs['n'])
+            try:
+                index = int(attrs['n'])
+            except:
+                print "Invalid index at line:", self._locator.getLineNumber(), "of", self._path
+                self._current = self._current.addChild(name)
+                return
+
             self._current = self._current.getChild(name, index, create=True)
         else:
             self._current = self._current.addChild(name)
@@ -186,11 +192,17 @@ class PropsHandler(handler.ContentHandler):
 
         p = os.path.join(self._basePath, includePath)
         if not os.path.exists(p):
-            p = os.path.join(self._dataDirPath, includePath)
+            found = False
-            if not os.path.exists(p):
+            for i in self._includes:
+                p = os.path.join(i, includePath)
+                if os.path.exists(p):
+                    found = True
+                    break
+
+            if not found:
                 raise RuntimeError("include file not found", includePath, "at line", self._locator.getLineNumber())
 
-        readProps(p, self._current, self._dataDirPath)
+        readProps(p, self._current, self._includes)
 
     def endElement(self, name):
         if (name == 'PropertyList'):
@@ -200,16 +212,23 @@ class PropsHandler(handler.ContentHandler):
             # convert and store value
             self._current.value = self._content
             if self._currentTy == "int":
-                self._current.value = int(self._content)
+                self._current.value = int(self._content) if self._content is not None else 0
             if self._currentTy == "bool":
                 self._current.value = self.parsePropsBool(self._content)
             if self._currentTy == "double":
-                self._current.value = float(self._content)
+                if self._content is None:
+                    self._current.value = 0.0
+                else:
+                    if self._content.endswith('f'):
+                        self._content = self._content[:-1]
+                    self._current.value = float(self._content)
         except:
             print "Parse error for value:", self._content, "at line:", self._locator.getLineNumber(), "of:", self._path
 
         self._current = self._current.parent
         self._content = None
+        self._currentTy = None
+
 
     def parsePropsBool(self, content):
         if content == "True" or content == "true":
@@ -240,11 +259,10 @@ class PropsHandler(handler.ContentHandler):
     def root(self):
         return self._root
 
-
+def readProps(path, root = None, includePaths = []):
-def readProps(path, root = None, dataDirPath = None):
     parser = make_parser()
     locator = expatreader.ExpatLocator( parser )
-    h = PropsHandler(root, path, dataDirPath)
+    h = PropsHandler(root, path, includePaths)
     h.setDocumentLocator(locator)
     parser.setContentHandler(h)
     parser.parse(path)