fly/fgmeta
1
0
Fork 0
Code Activity

Maintain catalog script starting to work.

Browse source
This commit is contained in:
James Turner 2015-07-11 21:47:13 +01:00
parent 287380d98d
commit fbda7475f0
5 changed files with 220 additions and 159 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
git_catalog_repository.py
View file

@ -7,7 +7,7 @@ class GITCatalogRepository:
def __init__(self, path, usesSubmodules = False, singleAircraft = False):
self._path = path
if !os.path.exists(os.path.join(path, ".git")):
if not os.path.exists(os.path.join(path, ".git")):
raise RuntimeError("not a Git directory:" + path)
self._usesSubmodules = usesSubmodules

2
git_discrete_repository.py
View file

@ -4,7 +4,7 @@ import subprocess
import os
import sgprops
import GITCatalogRepository
import git_catalog_repository
class GitDiscreteSCM:
def __init__(self, node):

147
maintain_catalog.py
View file

@ -27,24 +27,28 @@ class VariantData:
@property
def catalogNode(self):
n = Node("variant")
n.addChild("id") = path
m.addChild("name") = self._name
n.addChild("id").value = path
n.addChild("name").value = self._name
class PackageData:
def __init__(path):
def __init__(self, path):
self._path = path
self._previousSCMRevision = None
self._previousRevision = 0
self._thumbnails = []
self._variants = {}
self._revision = 0
self._md5 = None
self._fileSize = 0
self._node = sgprops.Node()
self._node = sgprops.Node("package")
self._node.addChild("id").value = self.id
def setPreviousData(node):
def setPreviousData(self, node):
self._previousRevision = node.getValue("revision")
self._previousMD5 = node.getValue("md5")
self._previousSCMRevision = node.getValue("scm-revision")
self._fileSize = int(node.getValue("file-size-bytes"))
@property
def id(self):
@ -54,26 +58,32 @@ class PackageData:
def thumbnails(self):
return self._thumbnails
def isSourceModified(self, scmRepo):
if (self._previousSCMRevision == None):
return True
@property
def path(self):
return self._path
@property
def scmRevision(self):
currentRev = scmRepo.scmRevisionForPath(self._path)
if (currentRev is None):
raise RuntimeError("Unable to query SCM revision of files")
if (self._previousSCMRevision == currentRev):
self._scm = self._previousSCMRevision
return currentRev
def isSourceModified(self, scmRepo):
if (self._previousSCMRevision == None):
return True
if (self._previousSCMRevision == self.scmRevision):
return False
self._scm = currentRev
return True
def scanSetXmlFiles(self):
foundPrimary = False
for f in os.listdir(self._path):
if !f.endswith("-set.xml"):
if not f.endswith("-set.xml"):
continue
p = os.path.join(self._path, f)
@ -82,7 +92,8 @@ class PackageData:
if (simNode.getValue("exclude")):
continue
if primary = simNode.getValue("variant-of", None):
primary = simNode.getValue("variant-of", None)
if primary:
if not primary in variants:
self._variants[primary] = []
self._variants[primary].append(VariantData(self, node))
@ -144,22 +155,28 @@ class PackageData:
def generateZip(self, outDir):
self._revision = self._previousRevision + 1
zipName = self.id
zipName = self.id + ".zip"
zipFilePath = os.path.join(outDir, zipName)
os.chdir(os.path.dirname(self.path))
print "Creating zip", zipFilePath
# TODO: exclude certain files
subprocess.call(['zip', '-r', self.path, zipFilePath])
# anything we can do to make this faster?
subprocess.call(['zip', '--quiet', '-r', zipFilePath, self.id])
zipFile = open(zipFilePath + ".zip", 'r')
zipFile = open(zipFilePath, 'r')
self._md5 = hashlib.md5(zipFile.read()).hexdigest()
self._fileSize = os.path.getsize(zipFile)
self._fileSize = os.path.getsize(zipFilePath)
@property
def catalogNode(self, mirrorUrls, thumbnailUrl):
def useExistingCatalogData(self):
self._md5 = self._previousMD5
def packageNode(self, mirrorUrls, thumbnailUrl):
self._node.getChild("md5", create = True).value = self._md5
self._node.getChild("file-size-bytes", create = True).value = self._fileSize
self._node.addChild("revision", create = True).value = self._revision
self._node.addChild("scm-revision", create = True).value = self._scm
self._node.getChild("revision", create = True).value = int(self._revision)
self._node.getChild("scm-revision", create = True).value = self.scmRevision
for m in mirrorUrls:
self._node.addChild("url", m + "/" + self.id + ".zip")
@ -173,7 +190,7 @@ class PackageData:
return self._node
def extractThumnbails(self, thumbnailDir):
def extractThumbnails(self, thumbnailDir):
for t in self._thumbnails:
fullName = self.id + "_" + t
os.file.copy(os.path.join(self._path, t),
@ -183,7 +200,9 @@ class PackageData:
def scanPackages(globPath):
result = []
for d = in glob.glob(globPath):
print "Scanning", globPath
print os.getcwd()
for d in glob.glob(globPath):
result.append(PackageData(d))
return result
@ -192,25 +211,30 @@ def initScmRepository(node):
scmType = node.getValue("type")
if (scmType == "svn"):
svnPath = node.getValue("path")
return SVNCatalogRepository(svnPath)
else if (scmType == "git"):
return svn_catalog_repository.SVNCatalogRepository(svnPath)
elif (scmType == "git"):
gitPath = node.getValue("path")
usesSubmodules = node.getValue("uses-submodules", False)
return GitCatalogRepository(gitPath, usesSubmodules)
else if (scmType == "git-discrete")
return GitDiscreteSCM(node)
else if (scmType == None):
return git_catalog_repository.GitCatalogRepository(gitPath, usesSubmodules)
elif (scmType == "git-discrete"):
return git_discrete_repository.GitDiscreteSCM(node)
elif (scmType == None):
raise RuntimeError("No scm/type defined in catalog configuration")
else:
raise RuntimeError("Unspported SCM type:" + scmType)
def processUpload(node, outputPath):
print "Enabled value is:", node.getValue("enabled")
if not node.getValue("enabled", True):
print "Upload disabled"
return
uploadType = node.getValue("type")
if (type == "rsync"):
if (uploadType == "rsync"):
subprocess.call(["rsync", node.getValue("args", "-az"), ".",
node.getValue("remote")],
cwd = outputPath)
else if (type == "scp"):
elif (uploadType == "scp"):
subprocess.call(["scp", node.getValue("args", "-r"), outputPath,
node.getValue("remote")])
else:
@ -219,24 +243,38 @@ def processUpload(node, outputPath):
# dictionary
packages = {}
if len(sys.argv) < 2:
raise RuntimeError("no root dir specified")
rootDir = sys.argv[1]
os.path.chdir(rootDir)
if not os.path.isabs(rootDir):
rootDir = os.path.abspath(rootDir)
os.chdir(rootDir)
print "Root path is:", rootDir
configPath = 'catalog.config.xml'
if !os.path.exists(configPath):
if not os.path.exists(configPath):
raise RuntimeError("no config file found at:" + configPath)
config = readProps(configPath)
config = sgprops.readProps(configPath)
# out path
outPath = config.getValue('output-dir')
if outPath is None:
# default out path
outPath = "output"
outPath = os.path.join(rootDir, "output")
elif not os.path.isabs(outPath):
outPath = os.path.join(rootDir, "output")
if not os.path.exists(outPath):
os.mkdir(outPath)
print "Output path is:" + outPath
thumbnailPath = os.path.join(outPath, config.getValue('thumbnail-dir', "thumbnails"))
thumbnailUrl = config.getValue('thumbnail-url')
mirrorUrls = []
# contains existing catalog
existingCatalogPath = os.path.join(outPath, 'catalog.xml')
@ -245,32 +283,47 @@ scmRepo = initScmRepository(config.getChild('scm'))
# scan the directories in the aircraft paths
for g in config.getChildren("aircraft-dir"):
for p in scanPackages(g):
for p in scanPackages(g.value):
packages[p.id] = p
previousCatalog = readProps(existingCatalogPath)
for p in previousCatalog.getChildren("package"):
pkgId = p.getValue("id")
if !packages.contains(pkgId):
print "Orphaned old package:", pkgId
continue
if os.path.exists(existingCatalogPath):
try:
previousCatalog = sgprops.readProps(existingCatalogPath)
except:
print "Previous catalog is malformed"
previousCatalog = sgprops.Node()
packages[pkgId].setPreviousData(p)
for p in previousCatalog.getChildren("package"):
pkgId = p.getValue("id")
if not pkgId in packages.keys():
print "Orphaned old package:", pkgId
continue
packages[pkgId].setPreviousData(p)
else:
print "No previous catalog"
catalogNode = sgprops.Node()
catalogNode = sgprops.Node("catalog")
sgprops.copy(config.getChild("template"), catalogNode)
packagesToGenerate = []
for p in packages:
for p in packages.values():
if (p.isSourceModified(scmRepo)):
packagesToGenerate.append(p)
else:
p.useExistingCatalogData()
for p in packagesToGenerate:
p.generateZip(outPath)
p.extractThumbnails(thumbnailPath)
catalogNode.addChild(p.catalogNode)
print "Creating catalog"
for p in packages.values():
catalogNode.addChild(p.packageNode(mirrorUrls, thumbnailUrl))
catalogNode.write(os.path.join(outPath, "catalog.xml"))
print "Uploading"
if config.hasChild("upload"):
processUpload(config.getChild("upload"), outPath)

15
sgprops.py
View file

@ -69,6 +69,7 @@ class Node(object):
def firstUnusedIndex(self, n):
usedIndices = frozenset(c.index for c in self.getChildren(n))
i = 0
while i < 1000:
if i not in usedIndices:
return i
@ -107,7 +108,10 @@ class Node(object):
root = self._createXMLElement('PropertyList')
t = ET.ElementTree(root)
t.write(path, 'utf-8')
ET.dump(root)
t.write(path, 'utf-8', xml_declaration = True)
def _createXMLElement(self, nm = None):
if nm is None:
@ -136,7 +140,7 @@ class Node(object):
# index in parent
if (self.index != 0):
n.set('n', self.index)
n.set('n', str(self.index))
# children
for c in self._children:
@ -162,7 +166,7 @@ class PropsHandler(handler.ContentHandler):
self._locator = loc
def startElement(self, name, attrs):
self._content = ''
self._content = None
if (name == 'PropertyList'):
return
@ -209,8 +213,11 @@ class PropsHandler(handler.ContentHandler):
print "Parse error for value:", self._content, "at line:", self._locator.getLineNumber(), "of:", self._path
self._current = self._current.parent
self._content = None
def characters(self, content):
if self._content is None:
self._content = ''
self._content += content
def endDocument(self):
@ -234,6 +241,6 @@ def copy(src, dest):
dest.value = src.value
# recurse over children
for c in src.children:
for c in src.getChildren() :
dc = dest.getChild(c.name, i = c.index, create = True)
copy(c, dc)

5
svn_catalog_repository.py
View file

@ -7,7 +7,8 @@ class SVNCatalogRepository:
self._path = path
xml = subprocess.check_output(["svn", "info", "--xml", path])
root = ET.fromstring(xml)
if (root.find("repository/root") == None):
if (root.find(".//repository/root") == None):
raise RuntimeError("Not an SVN repository:" + path)
def hasPathChanged(self, path, oldRevision):
@ -16,7 +17,7 @@ class SVNCatalogRepository:
def scmRevisionForPath(self, path):
xml = subprocess.check_output(["svn", "info", "--xml", path])
root = ET.fromstring(xml)
commit = root.find("entry/commit")
commit = root.find(".//entry/commit")
return commit.get('revision', 0)
def update(self):