Drive scan from SCM repos.
- reduces repetition in the config XML.
This commit is contained in:
parent
80f7a7f2d3
commit
bd453e4a3e
4 changed files with 73 additions and 45 deletions
|
@ -1,40 +1,50 @@
|
||||||
# git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/
|
# git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/
|
||||||
|
|
||||||
import subprocess
|
import subprocess
|
||||||
import os
|
import os, sgprops
|
||||||
|
|
||||||
class GITCatalogRepository:
|
class GITCatalogRepository:
|
||||||
def __init__(self, path, usesSubmodules = False, singleAircraft = False):
|
def __init__(self, node, singleAircraft = False):
|
||||||
self._path = path
|
self._path = node.getValue("path")
|
||||||
|
|
||||||
if not os.path.exists(os.path.join(path, ".git")):
|
if not os.path.exists(os.path.join(self._path , ".git")):
|
||||||
raise RuntimeError("not a Git directory:" + path)
|
raise RuntimeError("not a Git directory:" + self._path )
|
||||||
|
|
||||||
self._usesSubmodules = usesSubmodules
|
self._usesSubmodules = node.getValue("uses-submodules", False)
|
||||||
self._singleAircraft = singleAircraft
|
self._singleAircraft = singleAircraft
|
||||||
|
|
||||||
self._currentRevision = subprocess.catch_output(["git", "rev-parse", "HEAD"],
|
self._currentRevision = subprocess.check_output(["git", "rev-parse", "HEAD"],
|
||||||
cwd = self._path)
|
cwd = self._path)
|
||||||
|
|
||||||
|
self._aircraftPath = None
|
||||||
|
if node.hasChild("scan-suffix"):
|
||||||
|
self._aircraftPath = os.path.join(path, node.getValue("scan-suffix"))
|
||||||
|
|
||||||
|
@property
|
||||||
|
def path(self):
|
||||||
|
return self._path
|
||||||
|
|
||||||
|
@property
|
||||||
|
def aircraftPath(self):
|
||||||
|
return self._aircraftPath
|
||||||
|
|
||||||
def hasPathChanged(self, path, oldRev):
|
def hasPathChanged(self, path, oldRev):
|
||||||
diffArgs = ["git", "diff", "--quiet", oldRev, "--"]
|
diffArgs = ["git", "diff", "--quiet", oldRev, "--"]
|
||||||
if not (self._usesSubmodules and self._singleAircraft):
|
if not (self._usesSubmodules and self._singleAircraft):
|
||||||
diffArgs.append(path)
|
diffArgs.append(path)
|
||||||
|
|
||||||
return subprocess.call(diffArgs, cwd = self._path)
|
return subprocess.call(diffArgs, cwd = self._path)
|
||||||
|
|
||||||
def update(self):
|
def update(self):
|
||||||
subprocess.call(["git", "pull"])
|
subprocess.call(["git", "pull"])
|
||||||
self._currentRevision = subprocess.catch_output(["git", "rev-parse", "HEAD"],
|
self._currentRevision = subprocess.check_output(["git", "rev-parse", "HEAD"],
|
||||||
cwd = self._path)
|
cwd = self._path)
|
||||||
|
|
||||||
if self._usesSubmodules:
|
if self._usesSubmodules:
|
||||||
subprocess.call(["git", "submodule", "update"], cwd = self._path)
|
subprocess.call(["git", "submodule", "update"], cwd = self._path)
|
||||||
|
|
||||||
def scmRevisionForPath(self, path):
|
def scmRevisionForPath(self, path):
|
||||||
if self._usesSubmodules:
|
if self._usesSubmodules:
|
||||||
return subprocess.catch_output(["git", "rev-parse", "HEAD"], cwd = self._path)
|
return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd = self._path)
|
||||||
|
|
||||||
return self._currentRevision
|
|
||||||
|
|
||||||
|
return self._currentRevision
|
||||||
|
|
|
@ -25,8 +25,12 @@ args = parser.parse_args()
|
||||||
|
|
||||||
includePaths = []
|
includePaths = []
|
||||||
|
|
||||||
def scanPackages(globPath):
|
def scanPackages(scmRepo):
|
||||||
result = []
|
result = []
|
||||||
|
globPath = scmRepo.aircraftPath
|
||||||
|
if globPath is None:
|
||||||
|
return result
|
||||||
|
|
||||||
print "Scanning", globPath
|
print "Scanning", globPath
|
||||||
print os.getcwd()
|
print os.getcwd()
|
||||||
for d in glob.glob(globPath):
|
for d in glob.glob(globPath):
|
||||||
|
@ -35,19 +39,16 @@ def scanPackages(globPath):
|
||||||
print "no -set.xml in", d
|
print "no -set.xml in", d
|
||||||
continue
|
continue
|
||||||
|
|
||||||
result.append(pkg.PackageData(d))
|
result.append(pkg.PackageData(d, scmRepo))
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def initScmRepository(node):
|
def initScmRepository(node):
|
||||||
scmType = node.getValue("type")
|
scmType = node.getValue("type")
|
||||||
if (scmType == "svn"):
|
if (scmType == "svn"):
|
||||||
svnPath = node.getValue("path")
|
return svn_catalog_repository.SVNCatalogRepository(node)
|
||||||
return svn_catalog_repository.SVNCatalogRepository(svnPath)
|
|
||||||
elif (scmType == "git"):
|
elif (scmType == "git"):
|
||||||
gitPath = node.getValue("path")
|
return git_catalog_repository.GITCatalogRepository(node)
|
||||||
usesSubmodules = node.getValue("uses-submodules", False)
|
|
||||||
return git_catalog_repository.GitCatalogRepository(gitPath, usesSubmodules)
|
|
||||||
elif (scmType == "git-discrete"):
|
elif (scmType == "git-discrete"):
|
||||||
return git_discrete_repository.GitDiscreteSCM(node)
|
return git_discrete_repository.GitDiscreteSCM(node)
|
||||||
elif (scmType == None):
|
elif (scmType == None):
|
||||||
|
@ -66,6 +67,7 @@ def processUpload(node, outputPath):
|
||||||
node.getValue("remote")],
|
node.getValue("remote")],
|
||||||
cwd = outputPath)
|
cwd = outputPath)
|
||||||
elif (uploadType == "rsync-ssh"):
|
elif (uploadType == "rsync-ssh"):
|
||||||
|
print "Doing rsync upload to:", node.getValue("remote")
|
||||||
subprocess.call(["rsync", node.getValue("args", "-azve"),
|
subprocess.call(["rsync", node.getValue("args", "-azve"),
|
||||||
"ssh", ".",
|
"ssh", ".",
|
||||||
node.getValue("remote")],
|
node.getValue("remote")],
|
||||||
|
@ -121,14 +123,15 @@ for i in config.getChildren("include-dir"):
|
||||||
# contains existing catalog
|
# contains existing catalog
|
||||||
existingCatalogPath = os.path.join(outPath, 'catalog.xml')
|
existingCatalogPath = os.path.join(outPath, 'catalog.xml')
|
||||||
|
|
||||||
scmProps = config.getChild('scm')
|
for scm in config.getChildren("scm"):
|
||||||
scmRepo = initScmRepository(scmProps)
|
scmRepo = initScmRepository(scm)
|
||||||
if args.update or (not args.noupdate and scmProps.getValue("update")):
|
if args.update or (not args.noupdate and scm.getValue("update")):
|
||||||
scmRepo.update()
|
scmRepo.update()
|
||||||
|
# presumably include repos in parse path
|
||||||
|
# TODO: make this configurable
|
||||||
|
includePaths.append(scmRepo.path)
|
||||||
|
|
||||||
# scan the directories in the aircraft paths
|
for p in scanPackages(scmRepo):
|
||||||
for g in config.getChildren("aircraft-dir"):
|
|
||||||
for p in scanPackages(g.value):
|
|
||||||
packages[p.id] = p
|
packages[p.id] = p
|
||||||
|
|
||||||
if not os.path.exists(existingCatalogPath):
|
if not os.path.exists(existingCatalogPath):
|
||||||
|
@ -170,7 +173,7 @@ packagesToGenerate = []
|
||||||
for p in packages.values():
|
for p in packages.values():
|
||||||
p.scanSetXmlFiles(includePaths)
|
p.scanSetXmlFiles(includePaths)
|
||||||
|
|
||||||
if (p.isSourceModified(scmRepo)):
|
if p.isSourceModified:
|
||||||
packagesToGenerate.append(p)
|
packagesToGenerate.append(p)
|
||||||
else:
|
else:
|
||||||
p.useExistingCatalogData()
|
p.useExistingCatalogData()
|
||||||
|
@ -190,10 +193,9 @@ for p in packagesToGenerate:
|
||||||
|
|
||||||
print "Creating catalog"
|
print "Creating catalog"
|
||||||
for p in packages.values():
|
for p in packages.values():
|
||||||
catalogNode.addChild(p.packageNode(scmRepo, mirrorUrls, thumbnailUrls[0]))
|
catalogNode.addChild(p.packageNode(mirrorUrls, thumbnailUrls[0]))
|
||||||
|
|
||||||
catalogNode.write(os.path.join(outPath, "catalog.xml"))
|
catalogNode.write(os.path.join(outPath, "catalog.xml"))
|
||||||
|
|
||||||
print "Uploading"
|
|
||||||
for up in config.getChildren("upload"):
|
for up in config.getChildren("upload"):
|
||||||
processUpload(up, outPath)
|
processUpload(up, outPath)
|
||||||
|
|
17
package.py
17
package.py
|
@ -27,8 +27,9 @@ class VariantData:
|
||||||
n.addChild("name").value = self._name
|
n.addChild("name").value = self._name
|
||||||
|
|
||||||
class PackageData:
|
class PackageData:
|
||||||
def __init__(self, path):
|
def __init__(self, path, scmRepo):
|
||||||
self._path = path
|
self._path = path
|
||||||
|
self._scm = scmRepo
|
||||||
self._previousSCMRevision = None
|
self._previousSCMRevision = None
|
||||||
self._previousRevision = 0
|
self._previousRevision = 0
|
||||||
self._thumbnails = []
|
self._thumbnails = []
|
||||||
|
@ -62,18 +63,20 @@ class PackageData:
|
||||||
def variants(self):
|
def variants(self):
|
||||||
return self._variants
|
return self._variants
|
||||||
|
|
||||||
def scmRevision(self, repo):
|
@property
|
||||||
currentRev = repo.scmRevisionForPath(self._path)
|
def scmRevision(self):
|
||||||
|
currentRev = self._scm.scmRevisionForPath(self._path)
|
||||||
if (currentRev is None):
|
if (currentRev is None):
|
||||||
raise RuntimeError("Unable to query SCM revision of files")
|
raise RuntimeError("Unable to query SCM revision of files")
|
||||||
|
|
||||||
return currentRev
|
return currentRev
|
||||||
|
|
||||||
def isSourceModified(self, scmRepo):
|
@property
|
||||||
|
def isSourceModified(self):
|
||||||
if (self._previousSCMRevision == None):
|
if (self._previousSCMRevision == None):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
if (self._previousSCMRevision == self.scmRevision(scmRepo)):
|
if (self._previousSCMRevision == self.scmRevision):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
@ -178,11 +181,11 @@ class PackageData:
|
||||||
def useExistingCatalogData(self):
|
def useExistingCatalogData(self):
|
||||||
self._md5 = self._previousMD5
|
self._md5 = self._previousMD5
|
||||||
|
|
||||||
def packageNode(self, repo, mirrorUrls, thumbnailUrl):
|
def packageNode(self, mirrorUrls, thumbnailUrl):
|
||||||
self._node.getChild("md5", create = True).value = self._md5
|
self._node.getChild("md5", create = True).value = self._md5
|
||||||
self._node.getChild("file-size-bytes", create = True).value = self._fileSize
|
self._node.getChild("file-size-bytes", create = True).value = self._fileSize
|
||||||
self._node.getChild("revision", create = True).value = int(self._revision)
|
self._node.getChild("revision", create = True).value = int(self._revision)
|
||||||
self._node.getChild("scm-revision", create = True).value = self.scmRevision(repo)
|
self._node.getChild("scm-revision", create = True).value = self.scmRevision
|
||||||
|
|
||||||
for m in mirrorUrls:
|
for m in mirrorUrls:
|
||||||
self._node.addChild("url").value = m + "/" + self.id + ".zip"
|
self._node.addChild("url").value = m + "/" + self.id + ".zip"
|
||||||
|
|
|
@ -1,9 +1,10 @@
|
||||||
|
|
||||||
import subprocess, os
|
import subprocess, os, sgprops
|
||||||
import xml.etree.cElementTree as ET
|
import xml.etree.cElementTree as ET
|
||||||
|
|
||||||
class SVNCatalogRepository:
|
class SVNCatalogRepository:
|
||||||
def __init__(self, path):
|
def __init__(self, node):
|
||||||
|
path = node.getValue("path")
|
||||||
if not os.path.exists(path):
|
if not os.path.exists(path):
|
||||||
raise RuntimeError("No directory at:" + path)
|
raise RuntimeError("No directory at:" + path)
|
||||||
|
|
||||||
|
@ -14,6 +15,18 @@ class SVNCatalogRepository:
|
||||||
if (root.find(".//repository/root") == None):
|
if (root.find(".//repository/root") == None):
|
||||||
raise RuntimeError("Not an SVN repository:" + path)
|
raise RuntimeError("Not an SVN repository:" + path)
|
||||||
|
|
||||||
|
self._aircraftPath = None
|
||||||
|
if node.hasChild("scan-suffix"):
|
||||||
|
self._aircraftPath = os.path.join(path, node.getValue("scan-suffix"))
|
||||||
|
|
||||||
|
@property
|
||||||
|
def path(self):
|
||||||
|
return self._path
|
||||||
|
|
||||||
|
@property
|
||||||
|
def aircraftPath(self):
|
||||||
|
return self._aircraftPath
|
||||||
|
|
||||||
def hasPathChanged(self, path, oldRevision):
|
def hasPathChanged(self, path, oldRevision):
|
||||||
return self.scmRevisionForPath(path) != oldRevision
|
return self.scmRevisionForPath(path) != oldRevision
|
||||||
|
|
||||||
|
|
Loading…
Add table
Reference in a new issue