Maintain catalog script starting to work.

git_catalog_repository.py

@@ -7,7 +7,7 @@ class GITCatalogRepository:
     def __init__(self, path, usesSubmodules = False, singleAircraft = False):
         self._path = path
         
-        if !os.path.exists(os.path.join(path, ".git")):
+        if not os.path.exists(os.path.join(path, ".git")):
             raise RuntimeError("not a Git directory:" + path)
         
         self._usesSubmodules = usesSubmodules

git_discrete_repository.py

@@ -4,7 +4,7 @@ import subprocess
 import os
 import sgprops
 
-import GITCatalogRepository
+import git_catalog_repository
 
 class GitDiscreteSCM:
     def __init__(self, node):

maintain_catalog.py

@@ -12,120 +12,131 @@ import git_catalog_repository
 import git_discrete_repository
 
 # TODO
-# uploading / rsyncing 
+# uploading / rsyncing
 
 class VariantData:
     def __init__(self, primary, path, node):
         self._primary = primary
         self._path = path
         self._name = node.getValue("sim/description")
-        
+
         # ratings
-        
+
         # seperate thumbnails
-        
+
     @property
     def catalogNode(self):
         n = Node("variant")
-        n.addChild("id") = path
+        n.addChild("id").value = path
-        m.addChild("name") = self._name
+        n.addChild("name").value = self._name
-        
+
 class PackageData:
-    def __init__(path):
+    def __init__(self, path):
         self._path = path
         self._previousSCMRevision = None
         self._previousRevision = 0
         self._thumbnails = []
         self._variants = {}
-        
+        self._revision = 0
-        self._node = sgprops.Node()
+        self._md5 = None
+        self._fileSize = 0
+
+        self._node = sgprops.Node("package")
         self._node.addChild("id").value = self.id
-        
+
-    def setPreviousData(node):
+    def setPreviousData(self, node):
         self._previousRevision = node.getValue("revision")
         self._previousMD5 = node.getValue("md5")
         self._previousSCMRevision = node.getValue("scm-revision")
-    
+        self._fileSize = int(node.getValue("file-size-bytes"))
+
     @property
     def id(self):
         return os.path.basename(self._path)
-    
+
     @property
     def thumbnails(self):
         return self._thumbnails
-        
+
-    def isSourceModified(self, scmRepo):
+    @property
-        if (self._previousSCMRevision == None):
+    def path(self):
-            return True
+        return self._path
-            
+
+    @property
+    def scmRevision(self):
         currentRev = scmRepo.scmRevisionForPath(self._path)
         if (currentRev is None):
             raise RuntimeError("Unable to query SCM revision of files")
-            
+
-        if (self._previousSCMRevision == currentRev):
+        return currentRev
-            self._scm = self._previousSCMRevision
+
+    def isSourceModified(self, scmRepo):
+        if (self._previousSCMRevision == None):
+            return True
+
+        if (self._previousSCMRevision == self.scmRevision):
             return False
-            
+
-        self._scm = currentRev
         return True
-        
+
     def scanSetXmlFiles(self):
         foundPrimary = False
-        
+
         for f in os.listdir(self._path):
-            if !f.endswith("-set.xml"):
+            if not f.endswith("-set.xml"):
                 continue
-                
+
             p = os.path.join(self._path, f)
             node = readProps(p)
             simNode = node.getChild("sim")
             if (simNode.getValue("exclude")):
                 continue
-                
+
-            if primary = simNode.getValue("variant-of", None):
+            primary = simNode.getValue("variant-of", None)
+            if primary:
                 if not primary in variants:
                     self._variants[primary] = []
                 self._variants[primary].append(VariantData(self, node))
                 continue
-            
+
             if foundPrimary:
                 print "Multiple primary -set.xml files at:" + self._path
                 continue
             else:
                 foundPrimary = True;
-                
+
             parsePrimarySetNode(simNode)
-            
+
             if os.path.exists(os.path.join(self._path, "thumbnail.png")):
                 self._thumbnails.append("thumbnail.png")
-            
+
         if not foundPrimary:
             raise RuntimeError("No primary -set.xml found at:" + self._path)
-            
+
-        
+
-        
+
     def parsePrimarySetNode(self, sim):
-        
+
         # basic / mandatory values
         self._node.addChild('id').value = d
         self._node.addChild('name').value = sim.getValue('description')
-        
+
         longDesc = sim.getValue('long-description')
         if longDesc is not None:
             self._node.addChild('description').value = longDesc
-            
+
         # copy all the standard values
         for p in ['status', 'author', 'license']:
             v = sim.getValue(p)
             if v is not None:
                 self._node.addChild(p).value = v
-            
+
         # ratings
         if sim.hasChild('rating'):
             pkgRatings = self._node.addChild('rating')
             for r in ['FDM', 'systems', 'cockpit', 'model']:
                 pkgRatings.addChild(r).value = sim.getValue('rating/' + r, 0)
-            
+
         # copy tags
         if sim.hasChild('tags'):
             for c in sim.getChild('tags').getChildren('tag'):
@@ -133,110 +144,137 @@ class PackageData:
                     print "Skipping non-standard tag:", c.value
                 else:
                     self._node.addChild('tag').value = c.value
-                    
+
         self._thumbnails = (t.value for t in self.getChildren("thumbnail"))
-        
+
     def validate(self):
         for t in self._thumbnails:
             if not os.path.exists(os.path.join(self._path, t)):
                 raise RuntimeError("missing thumbnail:" + t);
-        
+
     def generateZip(self, outDir):
         self._revision = self._previousRevision + 1
-        
+
-        zipName = self.id
+        zipName = self.id + ".zip"
         zipFilePath = os.path.join(outDir, zipName)
-        
+
+        os.chdir(os.path.dirname(self.path))
+
+        print "Creating zip", zipFilePath
         # TODO: exclude certain files
-        subprocess.call(['zip', '-r', self.path, zipFilePath])
+        # anything we can do to make this faster?
-        
+        subprocess.call(['zip', '--quiet', '-r', zipFilePath, self.id])
-        zipFile = open(zipFilePath + ".zip", 'r')
+
+        zipFile = open(zipFilePath, 'r')
         self._md5 = hashlib.md5(zipFile.read()).hexdigest()
-        self._fileSize = os.path.getsize(zipFile)
+        self._fileSize = os.path.getsize(zipFilePath)
-        
+
-    @property
+    def useExistingCatalogData(self):
-    def catalogNode(self, mirrorUrls, thumbnailUrl):
+        self._md5 = self._previousMD5
+
+    def packageNode(self, mirrorUrls, thumbnailUrl):
         self._node.getChild("md5", create = True).value = self._md5
         self._node.getChild("file-size-bytes", create = True).value = self._fileSize
-        self._node.addChild("revision", create = True).value = self._revision
+        self._node.getChild("revision", create = True).value = int(self._revision)
-        self._node.addChild("scm-revision", create = True).value = self._scm
+        self._node.getChild("scm-revision", create = True).value = self.scmRevision
-        
+
         for m in mirrorUrls:
             self._node.addChild("url", m + "/" + self.id + ".zip")
-            
+
         for t in self._thumbnails:
             self._node.addChild("thumbnail", thumbnailUrl + "/" + self.id + "_" + t)
-        
+
         for pr in self._variants:
             for vr in self._variants[pr]:
                 self._node.addChild(vr.catalogNode)
-        
+
         return self._node
-        
+
-    def extractThumnbails(self, thumbnailDir):
+    def extractThumbnails(self, thumbnailDir):
         for t in self._thumbnails:
             fullName = self.id + "_" + t
             os.file.copy(os.path.join(self._path, t),
                          os.path.join(thumbnailDir, fullName)
                          )
             # TODO : verify image format, size and so on
-        
+
 def scanPackages(globPath):
     result = []
-    for d = in glob.glob(globPath):
+    print "Scanning", globPath
+    print os.getcwd()
+    for d in glob.glob(globPath):
         result.append(PackageData(d))
-    
+
     return result
 
 def initScmRepository(node):
     scmType = node.getValue("type")
     if (scmType == "svn"):
         svnPath = node.getValue("path")
-        return SVNCatalogRepository(svnPath)
+        return svn_catalog_repository.SVNCatalogRepository(svnPath)
-    else if (scmType == "git"):
+    elif (scmType == "git"):
         gitPath = node.getValue("path")
         usesSubmodules = node.getValue("uses-submodules", False)
-        return GitCatalogRepository(gitPath, usesSubmodules)
+        return git_catalog_repository.GitCatalogRepository(gitPath, usesSubmodules)
-    else if (scmType == "git-discrete")
+    elif (scmType == "git-discrete"):
-        return GitDiscreteSCM(node)
+        return git_discrete_repository.GitDiscreteSCM(node)
-    else if (scmType == None):
+    elif (scmType == None):
         raise RuntimeError("No scm/type defined in catalog configuration")
     else:
         raise RuntimeError("Unspported SCM type:" + scmType)
-    
+
 def processUpload(node, outputPath):
+    print "Enabled value is:", node.getValue("enabled")
+    if not node.getValue("enabled", True):
+        print "Upload disabled"
+        return
+
     uploadType = node.getValue("type")
-    if (type == "rsync"):
+    if (uploadType == "rsync"):
-        subprocess.call(["rsync", node.getValue("args", "-az"), ".", 
+        subprocess.call(["rsync", node.getValue("args", "-az"), ".",
             node.getValue("remote")],
         cwd = outputPath)
-    else if (type == "scp"):
+    elif (uploadType == "scp"):
         subprocess.call(["scp", node.getValue("args", "-r"), outputPath,
             node.getValue("remote")])
     else:
         raise RuntimeError("Unsupported upload type:" + uploadType)
 
-# dictionary    
+# dictionary
 packages = {}
 
+if len(sys.argv) < 2:
+    raise RuntimeError("no root dir specified")
+
 rootDir = sys.argv[1]
-os.path.chdir(rootDir)
+if not os.path.isabs(rootDir):
+    rootDir = os.path.abspath(rootDir)
+os.chdir(rootDir)
+print "Root path is:", rootDir
 
 configPath = 'catalog.config.xml'
-if !os.path.exists(configPath):
+if not os.path.exists(configPath):
     raise RuntimeError("no config file found at:" + configPath)
-    
+
-config = readProps(configPath)
+config = sgprops.readProps(configPath)
 
 # out path
 outPath = config.getValue('output-dir')
 if outPath is None:
     # default out path
-    outPath = "output"
+    outPath = os.path.join(rootDir, "output")
-    
+elif not os.path.isabs(outPath):
+    outPath = os.path.join(rootDir, "output")
+
+if not os.path.exists(outPath):
+    os.mkdir(outPath)
+
 print "Output path is:" + outPath
 
 thumbnailPath = os.path.join(outPath, config.getValue('thumbnail-dir', "thumbnails"))
+thumbnailUrl = config.getValue('thumbnail-url')
+
+mirrorUrls = []
 
 # contains existing catalog
 existingCatalogPath = os.path.join(outPath, 'catalog.xml')
@@ -244,33 +282,48 @@ existingCatalogPath = os.path.join(outPath, 'catalog.xml')
 scmRepo = initScmRepository(config.getChild('scm'))
 
 # scan the directories in the aircraft paths
-for g in config.getChildren("aircraft-dir"):     
+for g in config.getChildren("aircraft-dir"):
-    for p in scanPackages(g):
+    for p in scanPackages(g.value):
         packages[p.id] = p
 
-previousCatalog = readProps(existingCatalogPath)
+if os.path.exists(existingCatalogPath):
-for p in previousCatalog.getChildren("package"):
+    try:
-    pkgId = p.getValue("id")
+        previousCatalog = sgprops.readProps(existingCatalogPath)
-    if !packages.contains(pkgId):
+    except:
-        print "Orphaned old package:", pkgId
+        print "Previous catalog is malformed"
-        continue
+        previousCatalog = sgprops.Node()
-        
-    packages[pkgId].setPreviousData(p)
-    
 
-catalogNode = sgprops.Node()
+    for p in previousCatalog.getChildren("package"):
+        pkgId = p.getValue("id")
+        if not pkgId in packages.keys():
+            print "Orphaned old package:", pkgId
+            continue
 
+        packages[pkgId].setPreviousData(p)
+else:
+    print "No previous catalog"
+
+catalogNode = sgprops.Node("catalog")
 sgprops.copy(config.getChild("template"), catalogNode)
 
+
 packagesToGenerate = []
-for p in packages:
+for p in packages.values():
     if (p.isSourceModified(scmRepo)):
         packagesToGenerate.append(p)
-    
+    else:
+        p.useExistingCatalogData()
+
 for p in packagesToGenerate:
     p.generateZip(outPath)
     p.extractThumbnails(thumbnailPath)
-    catalogNode.addChild(p.catalogNode)
+
-    
+print "Creating catalog"
+for p in packages.values():
+    catalogNode.addChild(p.packageNode(mirrorUrls, thumbnailUrl))
+
+catalogNode.write(os.path.join(outPath, "catalog.xml"))
+
+print "Uploading"
 if config.hasChild("upload"):
-    processUpload(config.getChild("upload"), outPath)
+    processUpload(config.getChild("upload"), outPath)

sgprops.py

@@ -13,7 +13,7 @@ class Node(object):
         self._value = None
         self._index = 0
         self._children = []
-        
+
     @property
     def value(self):
         return self._value
@@ -21,21 +21,21 @@ class Node(object):
     @value.setter
     def value(self, v):
         self._value = v
-    
+
     @property
     def name(self):
         return self._name
-        
+
     @property
     def index(self):
         return self._index
-        
+
-    @property 
+    @property
     def parent(self):
         return self._parent
-        
+
     def getChild(self, n, i=None, create = False):
-     
+
         if i is None:
             i = 0
             # parse name as foo[999] if necessary
@@ -43,18 +43,18 @@ class Node(object):
             if m is not None:
                 n = m.group(1)
                 i = int(m.group(2))
-                
+
         for c in self._children:
             if (c.name == n) and (c.index == i):
                 return c
-                
+
         if create:
             c = Node(n, i, self)
             self._children.append(c)
             return c
         else:
             raise IndexError("no such child:" + str(n) + " index=" + str(i))
-            
+
     def addChild(self, n):
         # adding an existing instance
         if isinstance(n, Node):
@@ -62,60 +62,64 @@ class Node(object):
             n._index = self.firstUnusedIndex(n.name)
             self._children.append(n)
             return n
-            
+
-        i = self.firstUnusedIndex(n)        
+        i = self.firstUnusedIndex(n)
         # create it via getChild
         return self.getChild(n, i, create=True)
-            
+
     def firstUnusedIndex(self, n):
         usedIndices = frozenset(c.index for c in self.getChildren(n))
+        i = 0
         while i < 1000:
             if i not in usedIndices:
                  return i
             i += 1
         raise RuntimeException("too many children with name:" + n)
-            
+
     def hasChild(self, nm):
         for c in self._children:
             if (c.name == nm):
                 return True
-                
+
         return False
-            
+
     def getChildren(self, n = None):
         if n is None:
             return self._children
-        
+
         return [c for c in self._children if c.name == n]
-            
+
     def getNode(self, path, cr = False):
         axes = path.split('/')
         nd = self
         for ax in axes:
             nd = nd.getChild(ax, create = cr)
-            
+
         return nd
-        
+
     def getValue(self, path, default = None):
         try:
             nd = self.getNode(path)
             return nd.value
         except:
             return default
-            
+
     def write(self, path):
         root = self._createXMLElement('PropertyList')
-        
+
         t = ET.ElementTree(root)
-        t.write(path, 'utf-8')
+
-        
+        ET.dump(root)
+
+        t.write(path, 'utf-8', xml_declaration = True)
+
     def _createXMLElement(self, nm = None):
         if nm is None:
             nm = self.name
-            
+
         n = ET.Element(nm)
-        
+
-        # value and type specification 
+        # value and type specification
         try:
             if self._value is not None:
                 if isinstance(self._value, basestring):
@@ -133,17 +137,17 @@ class Node(object):
                         n.set('type', "bool")
         except UnicodeEncodeError:
             print "Encoding error with", self._value, type(self._value)
-            
+
         # index in parent
         if (self.index != 0):
-            n.set('n', self.index)
+            n.set('n', str(self.index))
-            
+
         # children
         for c in self._children:
             n.append(c._createXMLElement())
-            
+
         return n;
-        
+
 
 class PropsHandler(handler.ContentHandler):
     def __init__(self, root = None, path = None, dataDirPath = None):
@@ -152,50 +156,50 @@ class PropsHandler(handler.ContentHandler):
         self._basePath = os.path.dirname(path)
         self._dataDirPath = dataDirPath
         self._locator = None
-        
+
         if root is None:
             # make a nameless root node
             self._root = Node("", 0)
         self._current = self._root
-            
+
     def setDocumentLocator(self, loc):
-        self._locator = loc    
+        self._locator = loc
-        
+
-    def startElement(self, name, attrs):    
+    def startElement(self, name, attrs):
-        self._content = ''
+        self._content = None
         if (name == 'PropertyList'):
             return
-            
+
         if 'n' in attrs.keys():
             index = int(attrs['n'])
             self._current = self._current.getChild(name, index, create=True)
         else:
             self._current = self._current.addChild(name)
-            
+
-            
+
         if 'include' in attrs.keys():
             self.handleInclude(attrs['include'])
-            
+
         self._currentTy = None;
         if 'type' in attrs.keys():
             self._currentTy = attrs['type']
-                     
+
     def handleInclude(self, includePath):
         if includePath.startswith('/'):
             includePath = includePath[1:]
-   
+
         p = os.path.join(self._basePath, includePath)
         if not os.path.exists(p):
             p = os.path.join(self._dataDirPath, includePath)
             if not os.path.exists(p):
                 raise RuntimeError("include file not found", includePath, "at line", self._locator.getLineNumber())
-                
+
         readProps(p, self._current, self._dataDirPath)
-            
+
     def endElement(self, name):
         if (name == 'PropertyList'):
             return
-            
+
         try:
             # convert and store value
             self._current.value = self._content
@@ -207,20 +211,23 @@ class PropsHandler(handler.ContentHandler):
                 self._current.value = float(self._content)
         except:
             print "Parse error for value:", self._content, "at line:", self._locator.getLineNumber(), "of:", self._path
-            
+
         self._current = self._current.parent
-            
+        self._content = None
+
     def characters(self, content):
+        if self._content is None:
+            self._content = ''
         self._content += content
-            
+
     def endDocument(self):
         pass
-        
+
     @property
     def root(self):
         return self._root
 
-    
+
 def readProps(path, root = None, dataDirPath = None):
     parser = make_parser()
     locator = expatreader.ExpatLocator( parser )
@@ -229,11 +236,11 @@ def readProps(path, root = None, dataDirPath = None):
     parser.setContentHandler(h)
     parser.parse(path)
     return h.root
-    
+
 def copy(src, dest):
     dest.value = src.value
-    
+
     # recurse over children
-    for c in src.children:
+    for c in src.getChildren() :
         dc = dest.getChild(c.name, i = c.index, create = True)
         copy(c, dc)

svn_catalog_repository.py

@@ -7,7 +7,8 @@ class SVNCatalogRepository:
         self._path = path
         xml = subprocess.check_output(["svn", "info", "--xml", path])
         root = ET.fromstring(xml)
-        if (root.find("repository/root") == None):
+        
+        if (root.find(".//repository/root") == None):
             raise RuntimeError("Not an SVN repository:" + path)
     
     def hasPathChanged(self, path, oldRevision):
@@ -16,7 +17,7 @@ class SVNCatalogRepository:
     def scmRevisionForPath(self, path):
         xml = subprocess.check_output(["svn", "info", "--xml", path])
         root = ET.fromstring(xml)
-        commit = root.find("entry/commit")
+        commit = root.find(".//entry/commit")
         return commit.get('revision', 0)
         
     def update(self):