Maintain catalog script starting to work.
This commit is contained in:
James Turner
parent
287380d98d
commit
fbda7475f0
5 changed files with 220 additions and 159 deletions
|
|
@ -7,7 +7,7 @@ class GITCatalogRepository:
|
|||
def __init__(self, path, usesSubmodules = False, singleAircraft = False):
|
||||
self._path = path
|
||||
|
||||
if !os.path.exists(os.path.join(path, ".git")):
|
||||
if not os.path.exists(os.path.join(path, ".git")):
|
||||
raise RuntimeError("not a Git directory:" + path)
|
||||
|
||||
self._usesSubmodules = usesSubmodules
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import subprocess
|
|||
import os
|
||||
import sgprops
|
||||
|
||||
import GITCatalogRepository
|
||||
import git_catalog_repository
|
||||
|
||||
class GitDiscreteSCM:
|
||||
def __init__(self, node):
|
||||
|
|
|
|||
|
|
@ -27,24 +27,28 @@ class VariantData:
|
|||
@property
|
||||
def catalogNode(self):
|
||||
n = Node("variant")
|
||||
n.addChild("id") = path
|
||||
m.addChild("name") = self._name
|
||||
n.addChild("id").value = path
|
||||
n.addChild("name").value = self._name
|
||||
|
||||
class PackageData:
|
||||
def __init__(path):
|
||||
def __init__(self, path):
|
||||
self._path = path
|
||||
self._previousSCMRevision = None
|
||||
self._previousRevision = 0
|
||||
self._thumbnails = []
|
||||
self._variants = {}
|
||||
self._revision = 0
|
||||
self._md5 = None
|
||||
self._fileSize = 0
|
||||
|
||||
self._node = sgprops.Node()
|
||||
self._node = sgprops.Node("package")
|
||||
self._node.addChild("id").value = self.id
|
||||
|
||||
def setPreviousData(node):
|
||||
def setPreviousData(self, node):
|
||||
self._previousRevision = node.getValue("revision")
|
||||
self._previousMD5 = node.getValue("md5")
|
||||
self._previousSCMRevision = node.getValue("scm-revision")
|
||||
self._fileSize = int(node.getValue("file-size-bytes"))
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
|
|
@ -54,26 +58,32 @@ class PackageData:
|
|||
def thumbnails(self):
|
||||
return self._thumbnails
|
||||
|
||||
def isSourceModified(self, scmRepo):
|
||||
if (self._previousSCMRevision == None):
|
||||
return True
|
||||
@property
|
||||
def path(self):
|
||||
return self._path
|
||||
|
||||
@property
|
||||
def scmRevision(self):
|
||||
currentRev = scmRepo.scmRevisionForPath(self._path)
|
||||
if (currentRev is None):
|
||||
raise RuntimeError("Unable to query SCM revision of files")
|
||||
|
||||
if (self._previousSCMRevision == currentRev):
|
||||
self._scm = self._previousSCMRevision
|
||||
return currentRev
|
||||
|
||||
def isSourceModified(self, scmRepo):
|
||||
if (self._previousSCMRevision == None):
|
||||
return True
|
||||
|
||||
if (self._previousSCMRevision == self.scmRevision):
|
||||
return False
|
||||
|
||||
self._scm = currentRev
|
||||
return True
|
||||
|
||||
def scanSetXmlFiles(self):
|
||||
foundPrimary = False
|
||||
|
||||
for f in os.listdir(self._path):
|
||||
if !f.endswith("-set.xml"):
|
||||
if not f.endswith("-set.xml"):
|
||||
continue
|
||||
|
||||
p = os.path.join(self._path, f)
|
||||
|
|
@ -82,7 +92,8 @@ class PackageData:
|
|||
if (simNode.getValue("exclude")):
|
||||
continue
|
||||
|
||||
if primary = simNode.getValue("variant-of", None):
|
||||
primary = simNode.getValue("variant-of", None)
|
||||
if primary:
|
||||
if not primary in variants:
|
||||
self._variants[primary] = []
|
||||
self._variants[primary].append(VariantData(self, node))
|
||||
|
|
@ -144,22 +155,28 @@ class PackageData:
|
|||
def generateZip(self, outDir):
|
||||
self._revision = self._previousRevision + 1
|
||||
|
||||
zipName = self.id
|
||||
zipName = self.id + ".zip"
|
||||
zipFilePath = os.path.join(outDir, zipName)
|
||||
|
||||
os.chdir(os.path.dirname(self.path))
|
||||
|
||||
print "Creating zip", zipFilePath
|
||||
# TODO: exclude certain files
|
||||
subprocess.call(['zip', '-r', self.path, zipFilePath])
|
||||
# anything we can do to make this faster?
|
||||
subprocess.call(['zip', '--quiet', '-r', zipFilePath, self.id])
|
||||
|
||||
zipFile = open(zipFilePath + ".zip", 'r')
|
||||
zipFile = open(zipFilePath, 'r')
|
||||
self._md5 = hashlib.md5(zipFile.read()).hexdigest()
|
||||
self._fileSize = os.path.getsize(zipFile)
|
||||
self._fileSize = os.path.getsize(zipFilePath)
|
||||
|
||||
@property
|
||||
def catalogNode(self, mirrorUrls, thumbnailUrl):
|
||||
def useExistingCatalogData(self):
|
||||
self._md5 = self._previousMD5
|
||||
|
||||
def packageNode(self, mirrorUrls, thumbnailUrl):
|
||||
self._node.getChild("md5", create = True).value = self._md5
|
||||
self._node.getChild("file-size-bytes", create = True).value = self._fileSize
|
||||
self._node.addChild("revision", create = True).value = self._revision
|
||||
self._node.addChild("scm-revision", create = True).value = self._scm
|
||||
self._node.getChild("revision", create = True).value = int(self._revision)
|
||||
self._node.getChild("scm-revision", create = True).value = self.scmRevision
|
||||
|
||||
for m in mirrorUrls:
|
||||
self._node.addChild("url", m + "/" + self.id + ".zip")
|
||||
|
|
@ -173,7 +190,7 @@ class PackageData:
|
|||
|
||||
return self._node
|
||||
|
||||
def extractThumnbails(self, thumbnailDir):
|
||||
def extractThumbnails(self, thumbnailDir):
|
||||
for t in self._thumbnails:
|
||||
fullName = self.id + "_" + t
|
||||
os.file.copy(os.path.join(self._path, t),
|
||||
|
|
@ -183,7 +200,9 @@ class PackageData:
|
|||
|
||||
def scanPackages(globPath):
|
||||
result = []
|
||||
for d = in glob.glob(globPath):
|
||||
print "Scanning", globPath
|
||||
print os.getcwd()
|
||||
for d in glob.glob(globPath):
|
||||
result.append(PackageData(d))
|
||||
|
||||
return result
|
||||
|
|
@ -192,25 +211,30 @@ def initScmRepository(node):
|
|||
scmType = node.getValue("type")
|
||||
if (scmType == "svn"):
|
||||
svnPath = node.getValue("path")
|
||||
return SVNCatalogRepository(svnPath)
|
||||
else if (scmType == "git"):
|
||||
return svn_catalog_repository.SVNCatalogRepository(svnPath)
|
||||
elif (scmType == "git"):
|
||||
gitPath = node.getValue("path")
|
||||
usesSubmodules = node.getValue("uses-submodules", False)
|
||||
return GitCatalogRepository(gitPath, usesSubmodules)
|
||||
else if (scmType == "git-discrete")
|
||||
return GitDiscreteSCM(node)
|
||||
else if (scmType == None):
|
||||
return git_catalog_repository.GitCatalogRepository(gitPath, usesSubmodules)
|
||||
elif (scmType == "git-discrete"):
|
||||
return git_discrete_repository.GitDiscreteSCM(node)
|
||||
elif (scmType == None):
|
||||
raise RuntimeError("No scm/type defined in catalog configuration")
|
||||
else:
|
||||
raise RuntimeError("Unspported SCM type:" + scmType)
|
||||
|
||||
def processUpload(node, outputPath):
|
||||
print "Enabled value is:", node.getValue("enabled")
|
||||
if not node.getValue("enabled", True):
|
||||
print "Upload disabled"
|
||||
return
|
||||
|
||||
uploadType = node.getValue("type")
|
||||
if (type == "rsync"):
|
||||
if (uploadType == "rsync"):
|
||||
subprocess.call(["rsync", node.getValue("args", "-az"), ".",
|
||||
node.getValue("remote")],
|
||||
cwd = outputPath)
|
||||
else if (type == "scp"):
|
||||
elif (uploadType == "scp"):
|
||||
subprocess.call(["scp", node.getValue("args", "-r"), outputPath,
|
||||
node.getValue("remote")])
|
||||
else:
|
||||
|
|
@ -219,24 +243,38 @@ def processUpload(node, outputPath):
|
|||
# dictionary
|
||||
packages = {}
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
raise RuntimeError("no root dir specified")
|
||||
|
||||
rootDir = sys.argv[1]
|
||||
os.path.chdir(rootDir)
|
||||
if not os.path.isabs(rootDir):
|
||||
rootDir = os.path.abspath(rootDir)
|
||||
os.chdir(rootDir)
|
||||
print "Root path is:", rootDir
|
||||
|
||||
configPath = 'catalog.config.xml'
|
||||
if !os.path.exists(configPath):
|
||||
if not os.path.exists(configPath):
|
||||
raise RuntimeError("no config file found at:" + configPath)
|
||||
|
||||
config = readProps(configPath)
|
||||
config = sgprops.readProps(configPath)
|
||||
|
||||
# out path
|
||||
outPath = config.getValue('output-dir')
|
||||
if outPath is None:
|
||||
# default out path
|
||||
outPath = "output"
|
||||
outPath = os.path.join(rootDir, "output")
|
||||
elif not os.path.isabs(outPath):
|
||||
outPath = os.path.join(rootDir, "output")
|
||||
|
||||
if not os.path.exists(outPath):
|
||||
os.mkdir(outPath)
|
||||
|
||||
print "Output path is:" + outPath
|
||||
|
||||
thumbnailPath = os.path.join(outPath, config.getValue('thumbnail-dir', "thumbnails"))
|
||||
thumbnailUrl = config.getValue('thumbnail-url')
|
||||
|
||||
mirrorUrls = []
|
||||
|
||||
# contains existing catalog
|
||||
existingCatalogPath = os.path.join(outPath, 'catalog.xml')
|
||||
|
|
@ -245,32 +283,47 @@ scmRepo = initScmRepository(config.getChild('scm'))
|
|||
|
||||
# scan the directories in the aircraft paths
|
||||
for g in config.getChildren("aircraft-dir"):
|
||||
for p in scanPackages(g):
|
||||
for p in scanPackages(g.value):
|
||||
packages[p.id] = p
|
||||
|
||||
previousCatalog = readProps(existingCatalogPath)
|
||||
for p in previousCatalog.getChildren("package"):
|
||||
if os.path.exists(existingCatalogPath):
|
||||
try:
|
||||
previousCatalog = sgprops.readProps(existingCatalogPath)
|
||||
except:
|
||||
print "Previous catalog is malformed"
|
||||
previousCatalog = sgprops.Node()
|
||||
|
||||
for p in previousCatalog.getChildren("package"):
|
||||
pkgId = p.getValue("id")
|
||||
if !packages.contains(pkgId):
|
||||
if not pkgId in packages.keys():
|
||||
print "Orphaned old package:", pkgId
|
||||
continue
|
||||
|
||||
packages[pkgId].setPreviousData(p)
|
||||
else:
|
||||
print "No previous catalog"
|
||||
|
||||
|
||||
catalogNode = sgprops.Node()
|
||||
|
||||
catalogNode = sgprops.Node("catalog")
|
||||
sgprops.copy(config.getChild("template"), catalogNode)
|
||||
|
||||
|
||||
packagesToGenerate = []
|
||||
for p in packages:
|
||||
for p in packages.values():
|
||||
if (p.isSourceModified(scmRepo)):
|
||||
packagesToGenerate.append(p)
|
||||
else:
|
||||
p.useExistingCatalogData()
|
||||
|
||||
for p in packagesToGenerate:
|
||||
p.generateZip(outPath)
|
||||
p.extractThumbnails(thumbnailPath)
|
||||
catalogNode.addChild(p.catalogNode)
|
||||
|
||||
print "Creating catalog"
|
||||
for p in packages.values():
|
||||
catalogNode.addChild(p.packageNode(mirrorUrls, thumbnailUrl))
|
||||
|
||||
catalogNode.write(os.path.join(outPath, "catalog.xml"))
|
||||
|
||||
print "Uploading"
|
||||
if config.hasChild("upload"):
|
||||
processUpload(config.getChild("upload"), outPath)
|
||||
15
sgprops.py
15
sgprops.py
|
|
@ -69,6 +69,7 @@ class Node(object):
|
|||
|
||||
def firstUnusedIndex(self, n):
|
||||
usedIndices = frozenset(c.index for c in self.getChildren(n))
|
||||
i = 0
|
||||
while i < 1000:
|
||||
if i not in usedIndices:
|
||||
return i
|
||||
|
|
@ -107,7 +108,10 @@ class Node(object):
|
|||
root = self._createXMLElement('PropertyList')
|
||||
|
||||
t = ET.ElementTree(root)
|
||||
t.write(path, 'utf-8')
|
||||
|
||||
ET.dump(root)
|
||||
|
||||
t.write(path, 'utf-8', xml_declaration = True)
|
||||
|
||||
def _createXMLElement(self, nm = None):
|
||||
if nm is None:
|
||||
|
|
@ -136,7 +140,7 @@ class Node(object):
|
|||
|
||||
# index in parent
|
||||
if (self.index != 0):
|
||||
n.set('n', self.index)
|
||||
n.set('n', str(self.index))
|
||||
|
||||
# children
|
||||
for c in self._children:
|
||||
|
|
@ -162,7 +166,7 @@ class PropsHandler(handler.ContentHandler):
|
|||
self._locator = loc
|
||||
|
||||
def startElement(self, name, attrs):
|
||||
self._content = ''
|
||||
self._content = None
|
||||
if (name == 'PropertyList'):
|
||||
return
|
||||
|
||||
|
|
@ -209,8 +213,11 @@ class PropsHandler(handler.ContentHandler):
|
|||
print "Parse error for value:", self._content, "at line:", self._locator.getLineNumber(), "of:", self._path
|
||||
|
||||
self._current = self._current.parent
|
||||
self._content = None
|
||||
|
||||
def characters(self, content):
|
||||
if self._content is None:
|
||||
self._content = ''
|
||||
self._content += content
|
||||
|
||||
def endDocument(self):
|
||||
|
|
@ -234,6 +241,6 @@ def copy(src, dest):
|
|||
dest.value = src.value
|
||||
|
||||
# recurse over children
|
||||
for c in src.children:
|
||||
for c in src.getChildren() :
|
||||
dc = dest.getChild(c.name, i = c.index, create = True)
|
||||
copy(c, dc)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ class SVNCatalogRepository:
|
|||
self._path = path
|
||||
xml = subprocess.check_output(["svn", "info", "--xml", path])
|
||||
root = ET.fromstring(xml)
|
||||
if (root.find("repository/root") == None):
|
||||
|
||||
if (root.find(".//repository/root") == None):
|
||||
raise RuntimeError("Not an SVN repository:" + path)
|
||||
|
||||
def hasPathChanged(self, path, oldRevision):
|
||||
|
|
@ -16,7 +17,7 @@ class SVNCatalogRepository:
|
|||
def scmRevisionForPath(self, path):
|
||||
xml = subprocess.check_output(["svn", "info", "--xml", path])
|
||||
root = ET.fromstring(xml)
|
||||
commit = root.find("entry/commit")
|
||||
commit = root.find(".//entry/commit")
|
||||
return commit.get('revision', 0)
|
||||
|
||||
def update(self):
|
||||
|
|
|
|||
Loading…
Reference in a new issue