From bd453e4a3e3dae47079d86965f24e3c4a2d53517 Mon Sep 17 00:00:00 2001 From: James Turner Date: Tue, 28 Jul 2015 15:26:00 -0500 Subject: [PATCH] Drive scan from SCM repos. - reduces repetition in the config XML. --- git_catalog_repository.py | 48 +++++++++++++++++++++++---------------- maintain_catalog.py | 36 +++++++++++++++-------------- package.py | 17 ++++++++------ svn_catalog_repository.py | 17 ++++++++++++-- 4 files changed, 73 insertions(+), 45 deletions(-) diff --git a/git_catalog_repository.py b/git_catalog_repository.py index 2ac085b..836e8e2 100644 --- a/git_catalog_repository.py +++ b/git_catalog_repository.py @@ -1,40 +1,50 @@ # git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/ - + import subprocess -import os +import os, sgprops class GITCatalogRepository: - def __init__(self, path, usesSubmodules = False, singleAircraft = False): - self._path = path - - if not os.path.exists(os.path.join(path, ".git")): - raise RuntimeError("not a Git directory:" + path) - - self._usesSubmodules = usesSubmodules + def __init__(self, node, singleAircraft = False): + self._path = node.getValue("path") + + if not os.path.exists(os.path.join(self._path , ".git")): + raise RuntimeError("not a Git directory:" + self._path ) + + self._usesSubmodules = node.getValue("uses-submodules", False) self._singleAircraft = singleAircraft - - self._currentRevision = subprocess.catch_output(["git", "rev-parse", "HEAD"], + + self._currentRevision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd = self._path) + self._aircraftPath = None + if node.hasChild("scan-suffix"): + self._aircraftPath = os.path.join(path, node.getValue("scan-suffix")) + + @property + def path(self): + return self._path + + @property + def aircraftPath(self): + return self._aircraftPath + def hasPathChanged(self, path, oldRev): diffArgs = ["git", "diff", "--quiet", oldRev, "--"] if not (self._usesSubmodules and self._singleAircraft): diffArgs.append(path) - + return subprocess.call(diffArgs, cwd = self._path) def update(self): subprocess.call(["git", "pull"]) - self._currentRevision = subprocess.catch_output(["git", "rev-parse", "HEAD"], + self._currentRevision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd = self._path) - + if self._usesSubmodules: - subprocess.call(["git", "submodule", "update"], cwd = self._path) + subprocess.call(["git", "submodule", "update"], cwd = self._path) def scmRevisionForPath(self, path): if self._usesSubmodules: - return subprocess.catch_output(["git", "rev-parse", "HEAD"], cwd = self._path) - - return self._currentRevision + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd = self._path) - \ No newline at end of file + return self._currentRevision diff --git a/maintain_catalog.py b/maintain_catalog.py index 7a0cc0b..d2443f4 100755 --- a/maintain_catalog.py +++ b/maintain_catalog.py @@ -25,8 +25,12 @@ args = parser.parse_args() includePaths = [] -def scanPackages(globPath): +def scanPackages(scmRepo): result = [] + globPath = scmRepo.aircraftPath + if globPath is None: + return result + print "Scanning", globPath print os.getcwd() for d in glob.glob(globPath): @@ -35,19 +39,16 @@ def scanPackages(globPath): print "no -set.xml in", d continue - result.append(pkg.PackageData(d)) + result.append(pkg.PackageData(d, scmRepo)) return result def initScmRepository(node): scmType = node.getValue("type") if (scmType == "svn"): - svnPath = node.getValue("path") - return svn_catalog_repository.SVNCatalogRepository(svnPath) + return svn_catalog_repository.SVNCatalogRepository(node) elif (scmType == "git"): - gitPath = node.getValue("path") - usesSubmodules = node.getValue("uses-submodules", False) - return git_catalog_repository.GitCatalogRepository(gitPath, usesSubmodules) + return git_catalog_repository.GITCatalogRepository(node) elif (scmType == "git-discrete"): return git_discrete_repository.GitDiscreteSCM(node) elif (scmType == None): @@ -66,6 +67,7 @@ def processUpload(node, outputPath): node.getValue("remote")], cwd = outputPath) elif (uploadType == "rsync-ssh"): + print "Doing rsync upload to:", node.getValue("remote") subprocess.call(["rsync", node.getValue("args", "-azve"), "ssh", ".", node.getValue("remote")], @@ -121,14 +123,15 @@ for i in config.getChildren("include-dir"): # contains existing catalog existingCatalogPath = os.path.join(outPath, 'catalog.xml') -scmProps = config.getChild('scm') -scmRepo = initScmRepository(scmProps) -if args.update or (not args.noupdate and scmProps.getValue("update")): - scmRepo.update() +for scm in config.getChildren("scm"): + scmRepo = initScmRepository(scm) + if args.update or (not args.noupdate and scm.getValue("update")): + scmRepo.update() + # presumably include repos in parse path + # TODO: make this configurable + includePaths.append(scmRepo.path) -# scan the directories in the aircraft paths -for g in config.getChildren("aircraft-dir"): - for p in scanPackages(g.value): + for p in scanPackages(scmRepo): packages[p.id] = p if not os.path.exists(existingCatalogPath): @@ -170,7 +173,7 @@ packagesToGenerate = [] for p in packages.values(): p.scanSetXmlFiles(includePaths) - if (p.isSourceModified(scmRepo)): + if p.isSourceModified: packagesToGenerate.append(p) else: p.useExistingCatalogData() @@ -190,10 +193,9 @@ for p in packagesToGenerate: print "Creating catalog" for p in packages.values(): - catalogNode.addChild(p.packageNode(scmRepo, mirrorUrls, thumbnailUrls[0])) + catalogNode.addChild(p.packageNode(mirrorUrls, thumbnailUrls[0])) catalogNode.write(os.path.join(outPath, "catalog.xml")) -print "Uploading" for up in config.getChildren("upload"): processUpload(up, outPath) diff --git a/package.py b/package.py index 14eb911..68f0a3a 100644 --- a/package.py +++ b/package.py @@ -27,8 +27,9 @@ class VariantData: n.addChild("name").value = self._name class PackageData: - def __init__(self, path): + def __init__(self, path, scmRepo): self._path = path + self._scm = scmRepo self._previousSCMRevision = None self._previousRevision = 0 self._thumbnails = [] @@ -62,18 +63,20 @@ class PackageData: def variants(self): return self._variants - def scmRevision(self, repo): - currentRev = repo.scmRevisionForPath(self._path) + @property + def scmRevision(self): + currentRev = self._scm.scmRevisionForPath(self._path) if (currentRev is None): raise RuntimeError("Unable to query SCM revision of files") return currentRev - def isSourceModified(self, scmRepo): + @property + def isSourceModified(self): if (self._previousSCMRevision == None): return True - if (self._previousSCMRevision == self.scmRevision(scmRepo)): + if (self._previousSCMRevision == self.scmRevision): return False return True @@ -178,11 +181,11 @@ class PackageData: def useExistingCatalogData(self): self._md5 = self._previousMD5 - def packageNode(self, repo, mirrorUrls, thumbnailUrl): + def packageNode(self, mirrorUrls, thumbnailUrl): self._node.getChild("md5", create = True).value = self._md5 self._node.getChild("file-size-bytes", create = True).value = self._fileSize self._node.getChild("revision", create = True).value = int(self._revision) - self._node.getChild("scm-revision", create = True).value = self.scmRevision(repo) + self._node.getChild("scm-revision", create = True).value = self.scmRevision for m in mirrorUrls: self._node.addChild("url").value = m + "/" + self.id + ".zip" diff --git a/svn_catalog_repository.py b/svn_catalog_repository.py index 6d09127..da182cc 100644 --- a/svn_catalog_repository.py +++ b/svn_catalog_repository.py @@ -1,9 +1,10 @@ -import subprocess, os +import subprocess, os, sgprops import xml.etree.cElementTree as ET class SVNCatalogRepository: - def __init__(self, path): + def __init__(self, node): + path = node.getValue("path") if not os.path.exists(path): raise RuntimeError("No directory at:" + path) @@ -14,6 +15,18 @@ class SVNCatalogRepository: if (root.find(".//repository/root") == None): raise RuntimeError("Not an SVN repository:" + path) + self._aircraftPath = None + if node.hasChild("scan-suffix"): + self._aircraftPath = os.path.join(path, node.getValue("scan-suffix")) + + @property + def path(self): + return self._path + + @property + def aircraftPath(self): + return self._aircraftPath + def hasPathChanged(self, path, oldRevision): return self.scmRevisionForPath(path) != oldRevision