1
0
Fork 0

Drive scan from SCM repos.

- reduces repetition in the config XML.
This commit is contained in:
James Turner 2015-07-28 15:26:00 -05:00
parent 80f7a7f2d3
commit bd453e4a3e
4 changed files with 73 additions and 45 deletions

View file

@ -1,40 +1,50 @@
# git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/ # git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/
import subprocess import subprocess
import os import os, sgprops
class GITCatalogRepository: class GITCatalogRepository:
def __init__(self, path, usesSubmodules = False, singleAircraft = False): def __init__(self, node, singleAircraft = False):
self._path = path self._path = node.getValue("path")
if not os.path.exists(os.path.join(path, ".git")): if not os.path.exists(os.path.join(self._path , ".git")):
raise RuntimeError("not a Git directory:" + path) raise RuntimeError("not a Git directory:" + self._path )
self._usesSubmodules = usesSubmodules self._usesSubmodules = node.getValue("uses-submodules", False)
self._singleAircraft = singleAircraft self._singleAircraft = singleAircraft
self._currentRevision = subprocess.catch_output(["git", "rev-parse", "HEAD"], self._currentRevision = subprocess.check_output(["git", "rev-parse", "HEAD"],
cwd = self._path) cwd = self._path)
self._aircraftPath = None
if node.hasChild("scan-suffix"):
self._aircraftPath = os.path.join(path, node.getValue("scan-suffix"))
@property
def path(self):
return self._path
@property
def aircraftPath(self):
return self._aircraftPath
def hasPathChanged(self, path, oldRev): def hasPathChanged(self, path, oldRev):
diffArgs = ["git", "diff", "--quiet", oldRev, "--"] diffArgs = ["git", "diff", "--quiet", oldRev, "--"]
if not (self._usesSubmodules and self._singleAircraft): if not (self._usesSubmodules and self._singleAircraft):
diffArgs.append(path) diffArgs.append(path)
return subprocess.call(diffArgs, cwd = self._path) return subprocess.call(diffArgs, cwd = self._path)
def update(self): def update(self):
subprocess.call(["git", "pull"]) subprocess.call(["git", "pull"])
self._currentRevision = subprocess.catch_output(["git", "rev-parse", "HEAD"], self._currentRevision = subprocess.check_output(["git", "rev-parse", "HEAD"],
cwd = self._path) cwd = self._path)
if self._usesSubmodules: if self._usesSubmodules:
subprocess.call(["git", "submodule", "update"], cwd = self._path) subprocess.call(["git", "submodule", "update"], cwd = self._path)
def scmRevisionForPath(self, path): def scmRevisionForPath(self, path):
if self._usesSubmodules: if self._usesSubmodules:
return subprocess.catch_output(["git", "rev-parse", "HEAD"], cwd = self._path) return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd = self._path)
return self._currentRevision
return self._currentRevision

View file

@ -25,8 +25,12 @@ args = parser.parse_args()
includePaths = [] includePaths = []
def scanPackages(globPath): def scanPackages(scmRepo):
result = [] result = []
globPath = scmRepo.aircraftPath
if globPath is None:
return result
print "Scanning", globPath print "Scanning", globPath
print os.getcwd() print os.getcwd()
for d in glob.glob(globPath): for d in glob.glob(globPath):
@ -35,19 +39,16 @@ def scanPackages(globPath):
print "no -set.xml in", d print "no -set.xml in", d
continue continue
result.append(pkg.PackageData(d)) result.append(pkg.PackageData(d, scmRepo))
return result return result
def initScmRepository(node): def initScmRepository(node):
scmType = node.getValue("type") scmType = node.getValue("type")
if (scmType == "svn"): if (scmType == "svn"):
svnPath = node.getValue("path") return svn_catalog_repository.SVNCatalogRepository(node)
return svn_catalog_repository.SVNCatalogRepository(svnPath)
elif (scmType == "git"): elif (scmType == "git"):
gitPath = node.getValue("path") return git_catalog_repository.GITCatalogRepository(node)
usesSubmodules = node.getValue("uses-submodules", False)
return git_catalog_repository.GitCatalogRepository(gitPath, usesSubmodules)
elif (scmType == "git-discrete"): elif (scmType == "git-discrete"):
return git_discrete_repository.GitDiscreteSCM(node) return git_discrete_repository.GitDiscreteSCM(node)
elif (scmType == None): elif (scmType == None):
@ -66,6 +67,7 @@ def processUpload(node, outputPath):
node.getValue("remote")], node.getValue("remote")],
cwd = outputPath) cwd = outputPath)
elif (uploadType == "rsync-ssh"): elif (uploadType == "rsync-ssh"):
print "Doing rsync upload to:", node.getValue("remote")
subprocess.call(["rsync", node.getValue("args", "-azve"), subprocess.call(["rsync", node.getValue("args", "-azve"),
"ssh", ".", "ssh", ".",
node.getValue("remote")], node.getValue("remote")],
@ -121,14 +123,15 @@ for i in config.getChildren("include-dir"):
# contains existing catalog # contains existing catalog
existingCatalogPath = os.path.join(outPath, 'catalog.xml') existingCatalogPath = os.path.join(outPath, 'catalog.xml')
scmProps = config.getChild('scm') for scm in config.getChildren("scm"):
scmRepo = initScmRepository(scmProps) scmRepo = initScmRepository(scm)
if args.update or (not args.noupdate and scmProps.getValue("update")): if args.update or (not args.noupdate and scm.getValue("update")):
scmRepo.update() scmRepo.update()
# presumably include repos in parse path
# TODO: make this configurable
includePaths.append(scmRepo.path)
# scan the directories in the aircraft paths for p in scanPackages(scmRepo):
for g in config.getChildren("aircraft-dir"):
for p in scanPackages(g.value):
packages[p.id] = p packages[p.id] = p
if not os.path.exists(existingCatalogPath): if not os.path.exists(existingCatalogPath):
@ -170,7 +173,7 @@ packagesToGenerate = []
for p in packages.values(): for p in packages.values():
p.scanSetXmlFiles(includePaths) p.scanSetXmlFiles(includePaths)
if (p.isSourceModified(scmRepo)): if p.isSourceModified:
packagesToGenerate.append(p) packagesToGenerate.append(p)
else: else:
p.useExistingCatalogData() p.useExistingCatalogData()
@ -190,10 +193,9 @@ for p in packagesToGenerate:
print "Creating catalog" print "Creating catalog"
for p in packages.values(): for p in packages.values():
catalogNode.addChild(p.packageNode(scmRepo, mirrorUrls, thumbnailUrls[0])) catalogNode.addChild(p.packageNode(mirrorUrls, thumbnailUrls[0]))
catalogNode.write(os.path.join(outPath, "catalog.xml")) catalogNode.write(os.path.join(outPath, "catalog.xml"))
print "Uploading"
for up in config.getChildren("upload"): for up in config.getChildren("upload"):
processUpload(up, outPath) processUpload(up, outPath)

View file

@ -27,8 +27,9 @@ class VariantData:
n.addChild("name").value = self._name n.addChild("name").value = self._name
class PackageData: class PackageData:
def __init__(self, path): def __init__(self, path, scmRepo):
self._path = path self._path = path
self._scm = scmRepo
self._previousSCMRevision = None self._previousSCMRevision = None
self._previousRevision = 0 self._previousRevision = 0
self._thumbnails = [] self._thumbnails = []
@ -62,18 +63,20 @@ class PackageData:
def variants(self): def variants(self):
return self._variants return self._variants
def scmRevision(self, repo): @property
currentRev = repo.scmRevisionForPath(self._path) def scmRevision(self):
currentRev = self._scm.scmRevisionForPath(self._path)
if (currentRev is None): if (currentRev is None):
raise RuntimeError("Unable to query SCM revision of files") raise RuntimeError("Unable to query SCM revision of files")
return currentRev return currentRev
def isSourceModified(self, scmRepo): @property
def isSourceModified(self):
if (self._previousSCMRevision == None): if (self._previousSCMRevision == None):
return True return True
if (self._previousSCMRevision == self.scmRevision(scmRepo)): if (self._previousSCMRevision == self.scmRevision):
return False return False
return True return True
@ -178,11 +181,11 @@ class PackageData:
def useExistingCatalogData(self): def useExistingCatalogData(self):
self._md5 = self._previousMD5 self._md5 = self._previousMD5
def packageNode(self, repo, mirrorUrls, thumbnailUrl): def packageNode(self, mirrorUrls, thumbnailUrl):
self._node.getChild("md5", create = True).value = self._md5 self._node.getChild("md5", create = True).value = self._md5
self._node.getChild("file-size-bytes", create = True).value = self._fileSize self._node.getChild("file-size-bytes", create = True).value = self._fileSize
self._node.getChild("revision", create = True).value = int(self._revision) self._node.getChild("revision", create = True).value = int(self._revision)
self._node.getChild("scm-revision", create = True).value = self.scmRevision(repo) self._node.getChild("scm-revision", create = True).value = self.scmRevision
for m in mirrorUrls: for m in mirrorUrls:
self._node.addChild("url").value = m + "/" + self.id + ".zip" self._node.addChild("url").value = m + "/" + self.id + ".zip"

View file

@ -1,9 +1,10 @@
import subprocess, os import subprocess, os, sgprops
import xml.etree.cElementTree as ET import xml.etree.cElementTree as ET
class SVNCatalogRepository: class SVNCatalogRepository:
def __init__(self, path): def __init__(self, node):
path = node.getValue("path")
if not os.path.exists(path): if not os.path.exists(path):
raise RuntimeError("No directory at:" + path) raise RuntimeError("No directory at:" + path)
@ -14,6 +15,18 @@ class SVNCatalogRepository:
if (root.find(".//repository/root") == None): if (root.find(".//repository/root") == None):
raise RuntimeError("Not an SVN repository:" + path) raise RuntimeError("Not an SVN repository:" + path)
self._aircraftPath = None
if node.hasChild("scan-suffix"):
self._aircraftPath = os.path.join(path, node.getValue("scan-suffix"))
@property
def path(self):
return self._path
@property
def aircraftPath(self):
return self._aircraftPath
def hasPathChanged(self, path, oldRevision): def hasPathChanged(self, path, oldRevision):
return self.scmRevisionForPath(path) != oldRevision return self.scmRevisionForPath(path) != oldRevision