Maintain catalog script starting to work.

git_catalog_repository.py

@@ -7,7 +7,7 @@ class GITCatalogRepository:
     def __init__(self, path, usesSubmodules = False, singleAircraft = False):
         self._path = path
         
-        if !os.path.exists(os.path.join(path, ".git")):
+        if not os.path.exists(os.path.join(path, ".git")):
             raise RuntimeError("not a Git directory:" + path)
         
         self._usesSubmodules = usesSubmodules

git_discrete_repository.py

@@ -4,7 +4,7 @@ import subprocess
 import os
 import sgprops
 
-import GITCatalogRepository
+import git_catalog_repository
 
 class GitDiscreteSCM:
     def __init__(self, node):

maintain_catalog.py

@@ -27,24 +27,28 @@ class VariantData:
     @property
     def catalogNode(self):
         n = Node("variant")
-        n.addChild("id") = path
+        n.addChild("id").value = path
-        m.addChild("name") = self._name
+        n.addChild("name").value = self._name
 
 class PackageData:
-    def __init__(path):
+    def __init__(self, path):
         self._path = path
         self._previousSCMRevision = None
         self._previousRevision = 0
         self._thumbnails = []
         self._variants = {}
+        self._revision = 0
+        self._md5 = None
+        self._fileSize = 0
 
-        self._node = sgprops.Node()
+        self._node = sgprops.Node("package")
         self._node.addChild("id").value = self.id
 
-    def setPreviousData(node):
+    def setPreviousData(self, node):
         self._previousRevision = node.getValue("revision")
         self._previousMD5 = node.getValue("md5")
         self._previousSCMRevision = node.getValue("scm-revision")
+        self._fileSize = int(node.getValue("file-size-bytes"))
 
     @property
     def id(self):
@@ -54,26 +58,32 @@ class PackageData:
     def thumbnails(self):
         return self._thumbnails
 
-    def isSourceModified(self, scmRepo):
+    @property
-        if (self._previousSCMRevision == None):
+    def path(self):
-            return True
+        return self._path
 
+    @property
+    def scmRevision(self):
         currentRev = scmRepo.scmRevisionForPath(self._path)
         if (currentRev is None):
             raise RuntimeError("Unable to query SCM revision of files")
 
-        if (self._previousSCMRevision == currentRev):
+        return currentRev
-            self._scm = self._previousSCMRevision
+
+    def isSourceModified(self, scmRepo):
+        if (self._previousSCMRevision == None):
+            return True
+
+        if (self._previousSCMRevision == self.scmRevision):
             return False
 
-        self._scm = currentRev
         return True
 
     def scanSetXmlFiles(self):
         foundPrimary = False
 
         for f in os.listdir(self._path):
-            if !f.endswith("-set.xml"):
+            if not f.endswith("-set.xml"):
                 continue
 
             p = os.path.join(self._path, f)
@@ -82,7 +92,8 @@ class PackageData:
             if (simNode.getValue("exclude")):
                 continue
 
-            if primary = simNode.getValue("variant-of", None):
+            primary = simNode.getValue("variant-of", None)
+            if primary:
                 if not primary in variants:
                     self._variants[primary] = []
                 self._variants[primary].append(VariantData(self, node))
@@ -144,22 +155,28 @@ class PackageData:
     def generateZip(self, outDir):
         self._revision = self._previousRevision + 1
 
-        zipName = self.id
+        zipName = self.id + ".zip"
         zipFilePath = os.path.join(outDir, zipName)
 
+        os.chdir(os.path.dirname(self.path))
+
+        print "Creating zip", zipFilePath
         # TODO: exclude certain files
-        subprocess.call(['zip', '-r', self.path, zipFilePath])
+        # anything we can do to make this faster?
+        subprocess.call(['zip', '--quiet', '-r', zipFilePath, self.id])
 
-        zipFile = open(zipFilePath + ".zip", 'r')
+        zipFile = open(zipFilePath, 'r')
         self._md5 = hashlib.md5(zipFile.read()).hexdigest()
-        self._fileSize = os.path.getsize(zipFile)
+        self._fileSize = os.path.getsize(zipFilePath)
 
-    @property
+    def useExistingCatalogData(self):
-    def catalogNode(self, mirrorUrls, thumbnailUrl):
+        self._md5 = self._previousMD5
+
+    def packageNode(self, mirrorUrls, thumbnailUrl):
         self._node.getChild("md5", create = True).value = self._md5
         self._node.getChild("file-size-bytes", create = True).value = self._fileSize
-        self._node.addChild("revision", create = True).value = self._revision
+        self._node.getChild("revision", create = True).value = int(self._revision)
-        self._node.addChild("scm-revision", create = True).value = self._scm
+        self._node.getChild("scm-revision", create = True).value = self.scmRevision
 
         for m in mirrorUrls:
             self._node.addChild("url", m + "/" + self.id + ".zip")
@@ -173,7 +190,7 @@ class PackageData:
 
         return self._node
 
-    def extractThumnbails(self, thumbnailDir):
+    def extractThumbnails(self, thumbnailDir):
         for t in self._thumbnails:
             fullName = self.id + "_" + t
             os.file.copy(os.path.join(self._path, t),
@@ -183,7 +200,9 @@ class PackageData:
 
 def scanPackages(globPath):
     result = []
-    for d = in glob.glob(globPath):
+    print "Scanning", globPath
+    print os.getcwd()
+    for d in glob.glob(globPath):
         result.append(PackageData(d))
 
     return result
@@ -192,25 +211,30 @@ def initScmRepository(node):
     scmType = node.getValue("type")
     if (scmType == "svn"):
         svnPath = node.getValue("path")
-        return SVNCatalogRepository(svnPath)
+        return svn_catalog_repository.SVNCatalogRepository(svnPath)
-    else if (scmType == "git"):
+    elif (scmType == "git"):
         gitPath = node.getValue("path")
         usesSubmodules = node.getValue("uses-submodules", False)
-        return GitCatalogRepository(gitPath, usesSubmodules)
+        return git_catalog_repository.GitCatalogRepository(gitPath, usesSubmodules)
-    else if (scmType == "git-discrete")
+    elif (scmType == "git-discrete"):
-        return GitDiscreteSCM(node)
+        return git_discrete_repository.GitDiscreteSCM(node)
-    else if (scmType == None):
+    elif (scmType == None):
         raise RuntimeError("No scm/type defined in catalog configuration")
     else:
         raise RuntimeError("Unspported SCM type:" + scmType)
 
 def processUpload(node, outputPath):
+    print "Enabled value is:", node.getValue("enabled")
+    if not node.getValue("enabled", True):
+        print "Upload disabled"
+        return
+
     uploadType = node.getValue("type")
-    if (type == "rsync"):
+    if (uploadType == "rsync"):
         subprocess.call(["rsync", node.getValue("args", "-az"), ".",
             node.getValue("remote")],
         cwd = outputPath)
-    else if (type == "scp"):
+    elif (uploadType == "scp"):
         subprocess.call(["scp", node.getValue("args", "-r"), outputPath,
             node.getValue("remote")])
     else:
@@ -219,24 +243,38 @@ def processUpload(node, outputPath):
 # dictionary
 packages = {}
 
+if len(sys.argv) < 2:
+    raise RuntimeError("no root dir specified")
+
 rootDir = sys.argv[1]
-os.path.chdir(rootDir)
+if not os.path.isabs(rootDir):
+    rootDir = os.path.abspath(rootDir)
+os.chdir(rootDir)
+print "Root path is:", rootDir
 
 configPath = 'catalog.config.xml'
-if !os.path.exists(configPath):
+if not os.path.exists(configPath):
     raise RuntimeError("no config file found at:" + configPath)
 
-config = readProps(configPath)
+config = sgprops.readProps(configPath)
 
 # out path
 outPath = config.getValue('output-dir')
 if outPath is None:
     # default out path
-    outPath = "output"
+    outPath = os.path.join(rootDir, "output")
+elif not os.path.isabs(outPath):
+    outPath = os.path.join(rootDir, "output")
+
+if not os.path.exists(outPath):
+    os.mkdir(outPath)
 
 print "Output path is:" + outPath
 
 thumbnailPath = os.path.join(outPath, config.getValue('thumbnail-dir', "thumbnails"))
+thumbnailUrl = config.getValue('thumbnail-url')
+
+mirrorUrls = []
 
 # contains existing catalog
 existingCatalogPath = os.path.join(outPath, 'catalog.xml')
@@ -245,32 +283,47 @@ scmRepo = initScmRepository(config.getChild('scm'))
 
 # scan the directories in the aircraft paths
 for g in config.getChildren("aircraft-dir"):
-    for p in scanPackages(g):
+    for p in scanPackages(g.value):
         packages[p.id] = p
 
-previousCatalog = readProps(existingCatalogPath)
+if os.path.exists(existingCatalogPath):
+    try:
+        previousCatalog = sgprops.readProps(existingCatalogPath)
+    except:
+        print "Previous catalog is malformed"
+        previousCatalog = sgprops.Node()
+
     for p in previousCatalog.getChildren("package"):
         pkgId = p.getValue("id")
-    if !packages.contains(pkgId):
+        if not pkgId in packages.keys():
             print "Orphaned old package:", pkgId
             continue
 
         packages[pkgId].setPreviousData(p)
+else:
+    print "No previous catalog"
 
-
+catalogNode = sgprops.Node("catalog")
-catalogNode = sgprops.Node()
-
 sgprops.copy(config.getChild("template"), catalogNode)
 
+
 packagesToGenerate = []
-for p in packages:
+for p in packages.values():
     if (p.isSourceModified(scmRepo)):
         packagesToGenerate.append(p)
+    else:
+        p.useExistingCatalogData()
 
 for p in packagesToGenerate:
     p.generateZip(outPath)
     p.extractThumbnails(thumbnailPath)
-    catalogNode.addChild(p.catalogNode)
 
+print "Creating catalog"
+for p in packages.values():
+    catalogNode.addChild(p.packageNode(mirrorUrls, thumbnailUrl))
+
+catalogNode.write(os.path.join(outPath, "catalog.xml"))
+
+print "Uploading"
 if config.hasChild("upload"):
     processUpload(config.getChild("upload"), outPath)

sgprops.py

@@ -69,6 +69,7 @@ class Node(object):
 
     def firstUnusedIndex(self, n):
         usedIndices = frozenset(c.index for c in self.getChildren(n))
+        i = 0
         while i < 1000:
             if i not in usedIndices:
                  return i
@@ -107,7 +108,10 @@ class Node(object):
         root = self._createXMLElement('PropertyList')
 
         t = ET.ElementTree(root)
-        t.write(path, 'utf-8')
+
+        ET.dump(root)
+
+        t.write(path, 'utf-8', xml_declaration = True)
 
     def _createXMLElement(self, nm = None):
         if nm is None:
@@ -136,7 +140,7 @@ class Node(object):
 
         # index in parent
         if (self.index != 0):
-            n.set('n', self.index)
+            n.set('n', str(self.index))
 
         # children
         for c in self._children:
@@ -162,7 +166,7 @@ class PropsHandler(handler.ContentHandler):
         self._locator = loc
 
     def startElement(self, name, attrs):
-        self._content = ''
+        self._content = None
         if (name == 'PropertyList'):
             return
 
@@ -209,8 +213,11 @@ class PropsHandler(handler.ContentHandler):
             print "Parse error for value:", self._content, "at line:", self._locator.getLineNumber(), "of:", self._path
 
         self._current = self._current.parent
+        self._content = None
 
     def characters(self, content):
+        if self._content is None:
+            self._content = ''
         self._content += content
 
     def endDocument(self):
@@ -234,6 +241,6 @@ def copy(src, dest):
     dest.value = src.value
 
     # recurse over children
-    for c in src.children:
+    for c in src.getChildren() :
         dc = dest.getChild(c.name, i = c.index, create = True)
         copy(c, dc)

svn_catalog_repository.py

@@ -7,7 +7,8 @@ class SVNCatalogRepository:
         self._path = path
         xml = subprocess.check_output(["svn", "info", "--xml", path])
         root = ET.fromstring(xml)
-        if (root.find("repository/root") == None):
+        
+        if (root.find(".//repository/root") == None):
             raise RuntimeError("Not an SVN repository:" + path)
     
     def hasPathChanged(self, path, oldRevision):
@@ -16,7 +17,7 @@ class SVNCatalogRepository:
     def scmRevisionForPath(self, path):
         xml = subprocess.check_output(["svn", "info", "--xml", path])
         root = ET.fromstring(xml)
-        commit = root.find("entry/commit")
+        commit = root.find(".//entry/commit")
         return commit.get('revision', 0)
         
     def update(self):