Drive scan from SCM repos.

- reduces repetition in the config XML.

git_catalog_repository.py

@@ -1,40 +1,50 @@
 # git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/
- 
+
 import subprocess
-import os
+import os, sgprops
 
 class GITCatalogRepository:
-    def __init__(self, path, usesSubmodules = False, singleAircraft = False):
+    def __init__(self, node, singleAircraft = False):
-        self._path = path
+        self._path = node.getValue("path")
-        
+
-        if not os.path.exists(os.path.join(path, ".git")):
+        if not os.path.exists(os.path.join(self._path , ".git")):
-            raise RuntimeError("not a Git directory:" + path)
+            raise RuntimeError("not a Git directory:" + self._path )
-        
+
-        self._usesSubmodules = usesSubmodules
+        self._usesSubmodules = node.getValue("uses-submodules", False)
         self._singleAircraft = singleAircraft
-        
+
-        self._currentRevision = subprocess.catch_output(["git", "rev-parse", "HEAD"],
+        self._currentRevision = subprocess.check_output(["git", "rev-parse", "HEAD"],
             cwd = self._path)
 
+        self._aircraftPath = None
+        if node.hasChild("scan-suffix"):
+            self._aircraftPath = os.path.join(path, node.getValue("scan-suffix"))
+
+    @property
+    def path(self):
+        return self._path
+
+    @property
+    def aircraftPath(self):
+        return self._aircraftPath
+
     def hasPathChanged(self, path, oldRev):
         diffArgs = ["git", "diff", "--quiet", oldRev, "--"]
         if not (self._usesSubmodules and self._singleAircraft):
             diffArgs.append(path)
-            
+
         return subprocess.call(diffArgs, cwd = self._path)
 
     def update(self):
         subprocess.call(["git", "pull"])
-        self._currentRevision = subprocess.catch_output(["git", "rev-parse", "HEAD"],
+        self._currentRevision = subprocess.check_output(["git", "rev-parse", "HEAD"],
                 cwd = self._path)
-        
+
         if self._usesSubmodules:
-            subprocess.call(["git", "submodule", "update"], cwd = self._path)    
+            subprocess.call(["git", "submodule", "update"], cwd = self._path)
 
     def scmRevisionForPath(self, path):
         if self._usesSubmodules:
-            return subprocess.catch_output(["git", "rev-parse", "HEAD"], cwd = self._path)
+            return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd = self._path)
-            
-        return self._currentRevision
 
- 
+        return self._currentRevision

maintain_catalog.py

@@ -25,8 +25,12 @@ args = parser.parse_args()
 
 includePaths = []
 
-def scanPackages(globPath):
+def scanPackages(scmRepo):
     result = []
+    globPath = scmRepo.aircraftPath
+    if globPath is None:
+        return result
+
     print "Scanning", globPath
     print os.getcwd()
     for d in glob.glob(globPath):
@@ -35,19 +39,16 @@ def scanPackages(globPath):
             print "no -set.xml in", d
             continue
 
-        result.append(pkg.PackageData(d))
+        result.append(pkg.PackageData(d, scmRepo))
 
     return result
 
 def initScmRepository(node):
     scmType = node.getValue("type")
     if (scmType == "svn"):
-        svnPath = node.getValue("path")
+        return svn_catalog_repository.SVNCatalogRepository(node)
-        return svn_catalog_repository.SVNCatalogRepository(svnPath)
     elif (scmType == "git"):
-        gitPath = node.getValue("path")
+        return git_catalog_repository.GITCatalogRepository(node)
-        usesSubmodules = node.getValue("uses-submodules", False)
-        return git_catalog_repository.GitCatalogRepository(gitPath, usesSubmodules)
     elif (scmType == "git-discrete"):
         return git_discrete_repository.GitDiscreteSCM(node)
     elif (scmType == None):
@@ -66,6 +67,7 @@ def processUpload(node, outputPath):
             node.getValue("remote")],
         cwd = outputPath)
     elif (uploadType == "rsync-ssh"):
+        print "Doing rsync upload to:", node.getValue("remote")
         subprocess.call(["rsync", node.getValue("args", "-azve"),
             "ssh", ".",
             node.getValue("remote")],
@@ -121,14 +123,15 @@ for i in config.getChildren("include-dir"):
 # contains existing catalog
 existingCatalogPath = os.path.join(outPath, 'catalog.xml')
 
-scmProps = config.getChild('scm')
+for scm in config.getChildren("scm"):
-scmRepo = initScmRepository(scmProps)
+    scmRepo = initScmRepository(scm)
-if args.update or (not args.noupdate and scmProps.getValue("update")):
+    if args.update or (not args.noupdate and scm.getValue("update")):
-    scmRepo.update()
+        scmRepo.update()
+    # presumably include repos in parse path
+    # TODO: make this configurable
+    includePaths.append(scmRepo.path)
 
-# scan the directories in the aircraft paths
+    for p in scanPackages(scmRepo):
-for g in config.getChildren("aircraft-dir"):
-    for p in scanPackages(g.value):
         packages[p.id] = p
 
 if not os.path.exists(existingCatalogPath):
@@ -170,7 +173,7 @@ packagesToGenerate = []
 for p in packages.values():
     p.scanSetXmlFiles(includePaths)
 
-    if (p.isSourceModified(scmRepo)):
+    if p.isSourceModified:
         packagesToGenerate.append(p)
     else:
         p.useExistingCatalogData()
@@ -190,10 +193,9 @@ for p in packagesToGenerate:
 
 print "Creating catalog"
 for p in packages.values():
-    catalogNode.addChild(p.packageNode(scmRepo, mirrorUrls, thumbnailUrls[0]))
+    catalogNode.addChild(p.packageNode(mirrorUrls, thumbnailUrls[0]))
 
 catalogNode.write(os.path.join(outPath, "catalog.xml"))
 
-print "Uploading"
 for up in config.getChildren("upload"):
     processUpload(up, outPath)

package.py

@@ -27,8 +27,9 @@ class VariantData:
         n.addChild("name").value = self._name
 
 class PackageData:
-    def __init__(self, path):
+    def __init__(self, path, scmRepo):
         self._path = path
+        self._scm = scmRepo
         self._previousSCMRevision = None
         self._previousRevision = 0
         self._thumbnails = []
@@ -62,18 +63,20 @@ class PackageData:
     def variants(self):
         return self._variants
 
-    def scmRevision(self, repo):
+    @property
-        currentRev = repo.scmRevisionForPath(self._path)
+    def scmRevision(self):
+        currentRev = self._scm.scmRevisionForPath(self._path)
         if (currentRev is None):
             raise RuntimeError("Unable to query SCM revision of files")
 
         return currentRev
 
-    def isSourceModified(self, scmRepo):
+    @property
+    def isSourceModified(self):
         if (self._previousSCMRevision == None):
             return True
 
-        if (self._previousSCMRevision == self.scmRevision(scmRepo)):
+        if (self._previousSCMRevision == self.scmRevision):
             return False
 
         return True
@@ -178,11 +181,11 @@ class PackageData:
     def useExistingCatalogData(self):
         self._md5 = self._previousMD5
 
-    def packageNode(self, repo, mirrorUrls, thumbnailUrl):
+    def packageNode(self, mirrorUrls, thumbnailUrl):
         self._node.getChild("md5", create = True).value = self._md5
         self._node.getChild("file-size-bytes", create = True).value = self._fileSize
         self._node.getChild("revision", create = True).value = int(self._revision)
-        self._node.getChild("scm-revision", create = True).value = self.scmRevision(repo)
+        self._node.getChild("scm-revision", create = True).value = self.scmRevision
 
         for m in mirrorUrls:
             self._node.addChild("url").value = m + "/" + self.id + ".zip"

svn_catalog_repository.py

@@ -1,9 +1,10 @@
 
-import subprocess, os
+import subprocess, os, sgprops
 import xml.etree.cElementTree as ET
 
 class SVNCatalogRepository:
-    def __init__(self, path):
+    def __init__(self, node):
+        path = node.getValue("path")
         if not os.path.exists(path):
             raise RuntimeError("No directory at:" + path)
 
@@ -14,6 +15,18 @@ class SVNCatalogRepository:
         if (root.find(".//repository/root") == None):
             raise RuntimeError("Not an SVN repository:" + path)
 
+        self._aircraftPath = None
+        if node.hasChild("scan-suffix"):
+            self._aircraftPath = os.path.join(path, node.getValue("scan-suffix"))
+
+    @property
+    def path(self):
+        return self._path
+
+    @property
+    def aircraftPath(self):
+        return self._aircraftPath
+
     def hasPathChanged(self, path, oldRevision):
         return self.scmRevisionForPath(path) != oldRevision