1
0
Fork 0

Maintain catalog script starting to work.

This commit is contained in:
James Turner 2015-07-11 21:47:13 +01:00
parent 287380d98d
commit fbda7475f0
5 changed files with 220 additions and 159 deletions

View file

@ -7,7 +7,7 @@ class GITCatalogRepository:
def __init__(self, path, usesSubmodules = False, singleAircraft = False):
self._path = path
if !os.path.exists(os.path.join(path, ".git")):
if not os.path.exists(os.path.join(path, ".git")):
raise RuntimeError("not a Git directory:" + path)
self._usesSubmodules = usesSubmodules

View file

@ -4,7 +4,7 @@ import subprocess
import os
import sgprops
import GITCatalogRepository
import git_catalog_repository
class GitDiscreteSCM:
def __init__(self, node):

View file

@ -12,120 +12,131 @@ import git_catalog_repository
import git_discrete_repository
# TODO
# uploading / rsyncing
# uploading / rsyncing
class VariantData:
def __init__(self, primary, path, node):
self._primary = primary
self._path = path
self._name = node.getValue("sim/description")
# ratings
# seperate thumbnails
@property
def catalogNode(self):
n = Node("variant")
n.addChild("id") = path
m.addChild("name") = self._name
n.addChild("id").value = path
n.addChild("name").value = self._name
class PackageData:
def __init__(path):
def __init__(self, path):
self._path = path
self._previousSCMRevision = None
self._previousRevision = 0
self._thumbnails = []
self._variants = {}
self._node = sgprops.Node()
self._revision = 0
self._md5 = None
self._fileSize = 0
self._node = sgprops.Node("package")
self._node.addChild("id").value = self.id
def setPreviousData(node):
def setPreviousData(self, node):
self._previousRevision = node.getValue("revision")
self._previousMD5 = node.getValue("md5")
self._previousSCMRevision = node.getValue("scm-revision")
self._fileSize = int(node.getValue("file-size-bytes"))
@property
def id(self):
return os.path.basename(self._path)
@property
def thumbnails(self):
return self._thumbnails
def isSourceModified(self, scmRepo):
if (self._previousSCMRevision == None):
return True
@property
def path(self):
return self._path
@property
def scmRevision(self):
currentRev = scmRepo.scmRevisionForPath(self._path)
if (currentRev is None):
raise RuntimeError("Unable to query SCM revision of files")
if (self._previousSCMRevision == currentRev):
self._scm = self._previousSCMRevision
return currentRev
def isSourceModified(self, scmRepo):
if (self._previousSCMRevision == None):
return True
if (self._previousSCMRevision == self.scmRevision):
return False
self._scm = currentRev
return True
def scanSetXmlFiles(self):
foundPrimary = False
for f in os.listdir(self._path):
if !f.endswith("-set.xml"):
if not f.endswith("-set.xml"):
continue
p = os.path.join(self._path, f)
node = readProps(p)
simNode = node.getChild("sim")
if (simNode.getValue("exclude")):
continue
if primary = simNode.getValue("variant-of", None):
primary = simNode.getValue("variant-of", None)
if primary:
if not primary in variants:
self._variants[primary] = []
self._variants[primary].append(VariantData(self, node))
continue
if foundPrimary:
print "Multiple primary -set.xml files at:" + self._path
continue
else:
foundPrimary = True;
parsePrimarySetNode(simNode)
if os.path.exists(os.path.join(self._path, "thumbnail.png")):
self._thumbnails.append("thumbnail.png")
if not foundPrimary:
raise RuntimeError("No primary -set.xml found at:" + self._path)
def parsePrimarySetNode(self, sim):
# basic / mandatory values
self._node.addChild('id').value = d
self._node.addChild('name').value = sim.getValue('description')
longDesc = sim.getValue('long-description')
if longDesc is not None:
self._node.addChild('description').value = longDesc
# copy all the standard values
for p in ['status', 'author', 'license']:
v = sim.getValue(p)
if v is not None:
self._node.addChild(p).value = v
# ratings
if sim.hasChild('rating'):
pkgRatings = self._node.addChild('rating')
for r in ['FDM', 'systems', 'cockpit', 'model']:
pkgRatings.addChild(r).value = sim.getValue('rating/' + r, 0)
# copy tags
if sim.hasChild('tags'):
for c in sim.getChild('tags').getChildren('tag'):
@ -133,110 +144,137 @@ class PackageData:
print "Skipping non-standard tag:", c.value
else:
self._node.addChild('tag').value = c.value
self._thumbnails = (t.value for t in self.getChildren("thumbnail"))
def validate(self):
for t in self._thumbnails:
if not os.path.exists(os.path.join(self._path, t)):
raise RuntimeError("missing thumbnail:" + t);
def generateZip(self, outDir):
self._revision = self._previousRevision + 1
zipName = self.id
zipName = self.id + ".zip"
zipFilePath = os.path.join(outDir, zipName)
os.chdir(os.path.dirname(self.path))
print "Creating zip", zipFilePath
# TODO: exclude certain files
subprocess.call(['zip', '-r', self.path, zipFilePath])
zipFile = open(zipFilePath + ".zip", 'r')
# anything we can do to make this faster?
subprocess.call(['zip', '--quiet', '-r', zipFilePath, self.id])
zipFile = open(zipFilePath, 'r')
self._md5 = hashlib.md5(zipFile.read()).hexdigest()
self._fileSize = os.path.getsize(zipFile)
@property
def catalogNode(self, mirrorUrls, thumbnailUrl):
self._fileSize = os.path.getsize(zipFilePath)
def useExistingCatalogData(self):
self._md5 = self._previousMD5
def packageNode(self, mirrorUrls, thumbnailUrl):
self._node.getChild("md5", create = True).value = self._md5
self._node.getChild("file-size-bytes", create = True).value = self._fileSize
self._node.addChild("revision", create = True).value = self._revision
self._node.addChild("scm-revision", create = True).value = self._scm
self._node.getChild("revision", create = True).value = int(self._revision)
self._node.getChild("scm-revision", create = True).value = self.scmRevision
for m in mirrorUrls:
self._node.addChild("url", m + "/" + self.id + ".zip")
for t in self._thumbnails:
self._node.addChild("thumbnail", thumbnailUrl + "/" + self.id + "_" + t)
for pr in self._variants:
for vr in self._variants[pr]:
self._node.addChild(vr.catalogNode)
return self._node
def extractThumnbails(self, thumbnailDir):
def extractThumbnails(self, thumbnailDir):
for t in self._thumbnails:
fullName = self.id + "_" + t
os.file.copy(os.path.join(self._path, t),
os.path.join(thumbnailDir, fullName)
)
# TODO : verify image format, size and so on
def scanPackages(globPath):
result = []
for d = in glob.glob(globPath):
print "Scanning", globPath
print os.getcwd()
for d in glob.glob(globPath):
result.append(PackageData(d))
return result
def initScmRepository(node):
scmType = node.getValue("type")
if (scmType == "svn"):
svnPath = node.getValue("path")
return SVNCatalogRepository(svnPath)
else if (scmType == "git"):
return svn_catalog_repository.SVNCatalogRepository(svnPath)
elif (scmType == "git"):
gitPath = node.getValue("path")
usesSubmodules = node.getValue("uses-submodules", False)
return GitCatalogRepository(gitPath, usesSubmodules)
else if (scmType == "git-discrete")
return GitDiscreteSCM(node)
else if (scmType == None):
return git_catalog_repository.GitCatalogRepository(gitPath, usesSubmodules)
elif (scmType == "git-discrete"):
return git_discrete_repository.GitDiscreteSCM(node)
elif (scmType == None):
raise RuntimeError("No scm/type defined in catalog configuration")
else:
raise RuntimeError("Unspported SCM type:" + scmType)
def processUpload(node, outputPath):
print "Enabled value is:", node.getValue("enabled")
if not node.getValue("enabled", True):
print "Upload disabled"
return
uploadType = node.getValue("type")
if (type == "rsync"):
subprocess.call(["rsync", node.getValue("args", "-az"), ".",
if (uploadType == "rsync"):
subprocess.call(["rsync", node.getValue("args", "-az"), ".",
node.getValue("remote")],
cwd = outputPath)
else if (type == "scp"):
elif (uploadType == "scp"):
subprocess.call(["scp", node.getValue("args", "-r"), outputPath,
node.getValue("remote")])
else:
raise RuntimeError("Unsupported upload type:" + uploadType)
# dictionary
# dictionary
packages = {}
if len(sys.argv) < 2:
raise RuntimeError("no root dir specified")
rootDir = sys.argv[1]
os.path.chdir(rootDir)
if not os.path.isabs(rootDir):
rootDir = os.path.abspath(rootDir)
os.chdir(rootDir)
print "Root path is:", rootDir
configPath = 'catalog.config.xml'
if !os.path.exists(configPath):
if not os.path.exists(configPath):
raise RuntimeError("no config file found at:" + configPath)
config = readProps(configPath)
config = sgprops.readProps(configPath)
# out path
outPath = config.getValue('output-dir')
if outPath is None:
# default out path
outPath = "output"
outPath = os.path.join(rootDir, "output")
elif not os.path.isabs(outPath):
outPath = os.path.join(rootDir, "output")
if not os.path.exists(outPath):
os.mkdir(outPath)
print "Output path is:" + outPath
thumbnailPath = os.path.join(outPath, config.getValue('thumbnail-dir', "thumbnails"))
thumbnailUrl = config.getValue('thumbnail-url')
mirrorUrls = []
# contains existing catalog
existingCatalogPath = os.path.join(outPath, 'catalog.xml')
@ -244,33 +282,48 @@ existingCatalogPath = os.path.join(outPath, 'catalog.xml')
scmRepo = initScmRepository(config.getChild('scm'))
# scan the directories in the aircraft paths
for g in config.getChildren("aircraft-dir"):
for p in scanPackages(g):
for g in config.getChildren("aircraft-dir"):
for p in scanPackages(g.value):
packages[p.id] = p
previousCatalog = readProps(existingCatalogPath)
for p in previousCatalog.getChildren("package"):
pkgId = p.getValue("id")
if !packages.contains(pkgId):
print "Orphaned old package:", pkgId
continue
packages[pkgId].setPreviousData(p)
if os.path.exists(existingCatalogPath):
try:
previousCatalog = sgprops.readProps(existingCatalogPath)
except:
print "Previous catalog is malformed"
previousCatalog = sgprops.Node()
catalogNode = sgprops.Node()
for p in previousCatalog.getChildren("package"):
pkgId = p.getValue("id")
if not pkgId in packages.keys():
print "Orphaned old package:", pkgId
continue
packages[pkgId].setPreviousData(p)
else:
print "No previous catalog"
catalogNode = sgprops.Node("catalog")
sgprops.copy(config.getChild("template"), catalogNode)
packagesToGenerate = []
for p in packages:
for p in packages.values():
if (p.isSourceModified(scmRepo)):
packagesToGenerate.append(p)
else:
p.useExistingCatalogData()
for p in packagesToGenerate:
p.generateZip(outPath)
p.extractThumbnails(thumbnailPath)
catalogNode.addChild(p.catalogNode)
print "Creating catalog"
for p in packages.values():
catalogNode.addChild(p.packageNode(mirrorUrls, thumbnailUrl))
catalogNode.write(os.path.join(outPath, "catalog.xml"))
print "Uploading"
if config.hasChild("upload"):
processUpload(config.getChild("upload"), outPath)
processUpload(config.getChild("upload"), outPath)

View file

@ -13,7 +13,7 @@ class Node(object):
self._value = None
self._index = 0
self._children = []
@property
def value(self):
return self._value
@ -21,21 +21,21 @@ class Node(object):
@value.setter
def value(self, v):
self._value = v
@property
def name(self):
return self._name
@property
def index(self):
return self._index
@property
@property
def parent(self):
return self._parent
def getChild(self, n, i=None, create = False):
if i is None:
i = 0
# parse name as foo[999] if necessary
@ -43,18 +43,18 @@ class Node(object):
if m is not None:
n = m.group(1)
i = int(m.group(2))
for c in self._children:
if (c.name == n) and (c.index == i):
return c
if create:
c = Node(n, i, self)
self._children.append(c)
return c
else:
raise IndexError("no such child:" + str(n) + " index=" + str(i))
def addChild(self, n):
# adding an existing instance
if isinstance(n, Node):
@ -62,60 +62,64 @@ class Node(object):
n._index = self.firstUnusedIndex(n.name)
self._children.append(n)
return n
i = self.firstUnusedIndex(n)
i = self.firstUnusedIndex(n)
# create it via getChild
return self.getChild(n, i, create=True)
def firstUnusedIndex(self, n):
usedIndices = frozenset(c.index for c in self.getChildren(n))
i = 0
while i < 1000:
if i not in usedIndices:
return i
i += 1
raise RuntimeException("too many children with name:" + n)
def hasChild(self, nm):
for c in self._children:
if (c.name == nm):
return True
return False
def getChildren(self, n = None):
if n is None:
return self._children
return [c for c in self._children if c.name == n]
def getNode(self, path, cr = False):
axes = path.split('/')
nd = self
for ax in axes:
nd = nd.getChild(ax, create = cr)
return nd
def getValue(self, path, default = None):
try:
nd = self.getNode(path)
return nd.value
except:
return default
def write(self, path):
root = self._createXMLElement('PropertyList')
t = ET.ElementTree(root)
t.write(path, 'utf-8')
ET.dump(root)
t.write(path, 'utf-8', xml_declaration = True)
def _createXMLElement(self, nm = None):
if nm is None:
nm = self.name
n = ET.Element(nm)
# value and type specification
# value and type specification
try:
if self._value is not None:
if isinstance(self._value, basestring):
@ -133,17 +137,17 @@ class Node(object):
n.set('type', "bool")
except UnicodeEncodeError:
print "Encoding error with", self._value, type(self._value)
# index in parent
if (self.index != 0):
n.set('n', self.index)
n.set('n', str(self.index))
# children
for c in self._children:
n.append(c._createXMLElement())
return n;
class PropsHandler(handler.ContentHandler):
def __init__(self, root = None, path = None, dataDirPath = None):
@ -152,50 +156,50 @@ class PropsHandler(handler.ContentHandler):
self._basePath = os.path.dirname(path)
self._dataDirPath = dataDirPath
self._locator = None
if root is None:
# make a nameless root node
self._root = Node("", 0)
self._current = self._root
def setDocumentLocator(self, loc):
self._locator = loc
def startElement(self, name, attrs):
self._content = ''
self._locator = loc
def startElement(self, name, attrs):
self._content = None
if (name == 'PropertyList'):
return
if 'n' in attrs.keys():
index = int(attrs['n'])
self._current = self._current.getChild(name, index, create=True)
else:
self._current = self._current.addChild(name)
if 'include' in attrs.keys():
self.handleInclude(attrs['include'])
self._currentTy = None;
if 'type' in attrs.keys():
self._currentTy = attrs['type']
def handleInclude(self, includePath):
if includePath.startswith('/'):
includePath = includePath[1:]
p = os.path.join(self._basePath, includePath)
if not os.path.exists(p):
p = os.path.join(self._dataDirPath, includePath)
if not os.path.exists(p):
raise RuntimeError("include file not found", includePath, "at line", self._locator.getLineNumber())
readProps(p, self._current, self._dataDirPath)
def endElement(self, name):
if (name == 'PropertyList'):
return
try:
# convert and store value
self._current.value = self._content
@ -207,20 +211,23 @@ class PropsHandler(handler.ContentHandler):
self._current.value = float(self._content)
except:
print "Parse error for value:", self._content, "at line:", self._locator.getLineNumber(), "of:", self._path
self._current = self._current.parent
self._content = None
def characters(self, content):
if self._content is None:
self._content = ''
self._content += content
def endDocument(self):
pass
@property
def root(self):
return self._root
def readProps(path, root = None, dataDirPath = None):
parser = make_parser()
locator = expatreader.ExpatLocator( parser )
@ -229,11 +236,11 @@ def readProps(path, root = None, dataDirPath = None):
parser.setContentHandler(h)
parser.parse(path)
return h.root
def copy(src, dest):
dest.value = src.value
# recurse over children
for c in src.children:
for c in src.getChildren() :
dc = dest.getChild(c.name, i = c.index, create = True)
copy(c, dc)

View file

@ -7,7 +7,8 @@ class SVNCatalogRepository:
self._path = path
xml = subprocess.check_output(["svn", "info", "--xml", path])
root = ET.fromstring(xml)
if (root.find("repository/root") == None):
if (root.find(".//repository/root") == None):
raise RuntimeError("Not an SVN repository:" + path)
def hasPathChanged(self, path, oldRevision):
@ -16,7 +17,7 @@ class SVNCatalogRepository:
def scmRevisionForPath(self, path):
xml = subprocess.check_output(["svn", "info", "--xml", path])
root = ET.fromstring(xml)
commit = root.find("entry/commit")
commit = root.find(".//entry/commit")
return commit.get('revision', 0)
def update(self):