Hacking on catalog-creation script.
Only appropriate for official FlightGear aircraft right now, needs additional work to be useful for other hangars.
This commit is contained in:
James Turner
parent
c25db033d6
commit
1ab162d4d6
2 changed files with 278 additions and 153 deletions
|
|
@ -1,171 +1,82 @@
|
||||||
#!/usr/bin/python
|
#!/usr/bin/python
|
||||||
|
|
||||||
import os
|
import os, sys, re
|
||||||
import sys
|
|
||||||
import fnmatch
|
|
||||||
import tarfile
|
|
||||||
import hashlib
|
|
||||||
import shutil
|
|
||||||
|
|
||||||
import xml.etree.cElementTree as ET
|
import sgprops
|
||||||
|
|
||||||
rootPath = sys.argv[1]
|
fgRoot = sys.argv[1]
|
||||||
outputDir = sys.argv[2]
|
aircraftDir = os.path.join(fgRoot, 'Aircraft')
|
||||||
|
|
||||||
existingCatalogPath = os.path.join(outputDir, 'catalog.xml')
|
catalogProps = sgprops.Node()
|
||||||
existingCatalog = None
|
catalogProps.addChild('version').value = '3.1.0'
|
||||||
print 'existing ctalog path:' + existingCatalogPath
|
catalogProps.addChild('id').value = 'org.flightgear.default'
|
||||||
if os.path.exists(existingCatalogPath):
|
catalogProps.addChild('license').value = 'GPL'
|
||||||
existingCatalog = ET.parse(existingCatalogPath)
|
catalogProps.addChild('url').value = "http://fgfs.goneabitbursar.com/pkg/3.1.0/default-catalog.xml"
|
||||||
|
|
||||||
for file in os.listdir(outputDir):
|
catalogProps.addChild('description').value = "Aircraft developed and maintained by the FlightGear project"
|
||||||
if fnmatch.fnmatch(file, '*.tar.gz'):
|
|
||||||
os.remove(os.path.join(outputDir, file));
|
|
||||||
|
|
||||||
|
de = catalogProps.addChild('de')
|
||||||
|
# de.addChild('description').value = "<German translation of catalog description>"
|
||||||
|
|
||||||
thumbsDir = os.path.join(outputDir, 'thumbs')
|
fr = catalogProps.addChild('fr')
|
||||||
shutil.rmtree(thumbsDir)
|
|
||||||
os.makedirs(thumbsDir)
|
|
||||||
|
|
||||||
def setProperty(node, id, value):
|
urls = [
|
||||||
s = node.find(id) # check for existing
|
"http://flightgear.wo0t.de/Aircraft-3.0/{acft}_20140116.zip",
|
||||||
if s is None:
|
"http://ftp.icm.edu.pl/packages/flightgear/Aircraft-3.0/{acft}_20140216.zip",
|
||||||
s = ET.SubElement(node, id)
|
"http://mirrors.ibiblio.org/pub/mirrors/flightgear/ftp/Aircraft-3.0/{acft}_20140216.zip",
|
||||||
s.text = value
|
"http://ftp.igh.cnrs.fr/pub/flightgear/ftp/Aircraft-3.0/{acft}_20140116.zip",
|
||||||
|
"http://ftp.linux.kiev.ua/pub/fgfs/Aircraft-3.0/{acft}_20140116.zip",
|
||||||
|
"http://fgfs.physra.net/ftp/Aircraft-3.0/{acft}_20130225.zip"
|
||||||
|
]
|
||||||
|
|
||||||
def clearChildren(node, tag):
|
thumbs = [
|
||||||
for c in node.findall(tag):
|
"http://www.flightgear.org/thumbs/v3.0/{acft}.jpg"
|
||||||
node.remove(c)
|
]
|
||||||
|
|
||||||
def parse_setXml(path):
|
for d in os.listdir(aircraftDir):
|
||||||
tree = ET.parse(path)
|
acftDirPath = os.path.join(aircraftDir, d)
|
||||||
|
if not os.path.isdir(acftDirPath):
|
||||||
desc = tree.find('sim/description')
|
continue
|
||||||
ratings = tree.find('sim/rating')
|
|
||||||
if (ratings is not None):
|
|
||||||
for rating in list(ratings):
|
|
||||||
if rating.tag == 'status':
|
|
||||||
continue
|
|
||||||
|
|
||||||
rvalue = int(rating.text)
|
|
||||||
if rvalue < 2:
|
|
||||||
return None
|
|
||||||
else:
|
|
||||||
return None
|
|
||||||
|
|
||||||
d = {}
|
|
||||||
|
|
||||||
d['desc'] = desc
|
|
||||||
d['ratings'] = ratings;
|
|
||||||
d['status'] = tree.find('sim/status')
|
|
||||||
d['authors'] = tree.findall('sim/author')
|
|
||||||
|
|
||||||
return d
|
|
||||||
|
|
||||||
def process_aircraft(acft, path):
|
|
||||||
print '===' + acft + '==='
|
|
||||||
setFiles = []
|
|
||||||
thumbs = []
|
|
||||||
|
|
||||||
for file in os.listdir(path):
|
|
||||||
if fnmatch.fnmatch(file, '*-set.xml'):
|
|
||||||
setFiles.append(file);
|
|
||||||
|
|
||||||
if fnmatch.fnmatch(file, 'thumbnail*'):
|
setFilePath = None
|
||||||
thumbs.append(file)
|
|
||||||
|
# find the first set file
|
||||||
aircraft = []
|
# FIXME - way to designate the primary file
|
||||||
for s in setFiles:
|
for f in os.listdir(acftDirPath):
|
||||||
d = parse_setXml(os.path.join(path, s))
|
if f.endswith("-set.xml"):
|
||||||
if d is None:
|
setFilePath = os.path.join(acftDirPath, f)
|
||||||
continue
|
break
|
||||||
|
|
||||||
d['set'] = s[0:-8]
|
|
||||||
aircraft.append(d)
|
|
||||||
|
|
||||||
thumbnailNames = []
|
|
||||||
# copy thumbnails
|
|
||||||
for t in thumbs:
|
|
||||||
outThumb = os.path.join(thumbsDir, acft + "-" + t)
|
|
||||||
thumbnailNames.append(acft + "-" + t)
|
|
||||||
shutil.copyfile(os.path.join(path, t), outThumb)
|
|
||||||
|
|
||||||
if len(aircraft) == 0:
|
if setFilePath is None:
|
||||||
print "no aircraft profiles for " + acft
|
print "No -set.xml file found in",acftDirPath,"will be skipped"
|
||||||
return
|
continue
|
||||||
|
|
||||||
# tarball creation
|
|
||||||
outTar = os.path.join(outputDir, acft + ".tar.gz")
|
|
||||||
tar = tarfile.open(outTar, "w:gz")
|
|
||||||
tar.add(path, acft)
|
|
||||||
tar.close()
|
|
||||||
|
|
||||||
digest = hashlib.md5(open(outTar, 'r').read()).hexdigest()
|
|
||||||
revision = 1
|
|
||||||
|
|
||||||
# revision check
|
try:
|
||||||
if acft in existingPackages:
|
props = sgprops.readProps(setFilePath, dataDirPath = fgRoot)
|
||||||
previousMd5 = existingPackages[acft].find('md5').text
|
sim = props.getNode("sim")
|
||||||
previousRevsion = int(existingPackages[acft].find('revision').text)
|
|
||||||
if digest != previousMd5:
|
pkgNode = catalogProps.addChild('package')
|
||||||
print acft + ": MD5 has changed"
|
pkgNode.addChild('id').value = d
|
||||||
revision = previousRevsion + 1
|
pkgNode.addChild('name').value = sim.getValue('description')
|
||||||
else:
|
|
||||||
print acft + ": MD5 is unchanged"
|
|
||||||
else:
|
|
||||||
existingPackages[acft] = ET.Element('package')
|
|
||||||
setProperty(existingPackages[acft], 'id', acft)
|
|
||||||
|
|
||||||
setProperty(existingPackages[acft], 'revision', str(revision))
|
|
||||||
setProperty(existingPackages[acft], 'md5', digest)
|
|
||||||
setProperty(existingPackages[acft], 'description', aircraft[0]['desc'])
|
|
||||||
|
|
||||||
clearChildren(existingPackages[acft], 'thumbnail')
|
longDesc = sim.getValue('long-description')
|
||||||
for t in thumbnailNames:
|
if longDesc is not None:
|
||||||
tn = ET.SubElement(existingPackages[acft], 'thumbnail')
|
pkgNode.addChild('description').value = longDesc
|
||||||
tn.text = 'thumbs/' + t
|
|
||||||
|
|
||||||
clearChildren(existingPackages[acft], 'rating')
|
|
||||||
existingPackages[acft].append(aircraft[0]['ratings'])
|
|
||||||
|
|
||||||
print "wrote tarfile, digest is " + digest
|
|
||||||
|
|
||||||
root = ET.Element('PropertyList')
|
|
||||||
catalogTree = ET.ElementTree(root)
|
|
||||||
|
|
||||||
existingPackages = dict()
|
|
||||||
|
|
||||||
if (existingCatalog is not None):
|
|
||||||
print 'have existing catalog data'
|
|
||||||
|
|
||||||
root.append(existingCatalog.find('license'))
|
|
||||||
root.append(existingCatalog.find('url'))
|
|
||||||
root.append(existingCatalog.find('description'))
|
|
||||||
root.append(existingCatalog.find('id'))
|
|
||||||
|
|
||||||
# existing data (for revision incrementing)
|
|
||||||
for n in existingCatalog.findall('package'):
|
|
||||||
idNode = n.find('id')
|
|
||||||
if idNode is None:
|
|
||||||
print 'Missing <id> tag on package'
|
|
||||||
continue
|
|
||||||
|
|
||||||
existingPackages[idNode.text] = n;
|
# copy tags
|
||||||
|
if sim.hasChild('tags'):
|
||||||
|
for c in sim.getChild('tags').getChildren('tag'):
|
||||||
|
pkgNode.addChild('tag').value = c.value
|
||||||
|
|
||||||
|
# create download and thumbnail URLs
|
||||||
|
for u in urls:
|
||||||
|
pkgNode.addChild("url").value = u.format(acft=d)
|
||||||
|
|
||||||
|
for t in thumbs:
|
||||||
|
pkgNode.addChild("thumbnail").value = t.format(acft=d)
|
||||||
|
|
||||||
|
except:
|
||||||
|
print "Failure processing:", setFilePath
|
||||||
|
|
||||||
#licenseElement = ET.SubElement(root, 'license')
|
|
||||||
#licenseElement.text = 'gpl'
|
|
||||||
|
|
||||||
#urlElement = ET.SubElement(root, 'url')
|
|
||||||
#urlElement.text = 'http://catalog.xml'
|
|
||||||
|
|
||||||
for acft in os.listdir(rootPath):
|
|
||||||
path = os.path.join(rootPath, acft);
|
|
||||||
if (os.path.isdir(path)):
|
|
||||||
process_aircraft(acft, path)
|
|
||||||
|
|
||||||
|
|
||||||
for ep in existingPackages:
|
|
||||||
root.append(existingPackages[ep])
|
|
||||||
|
|
||||||
catalogTree.write(os.path.join(outputDir, 'catalog.xml'), 'UTF-8')
|
|
||||||
|
|
||||||
|
catalogProps.write("catalog.xml")
|
||||||
214
sgprops.py
Normal file
214
sgprops.py
Normal file
|
|
@ -0,0 +1,214 @@
|
||||||
|
# SAX for parsing
|
||||||
|
from xml.sax import make_parser, handler, expatreader
|
||||||
|
|
||||||
|
# ElementTree for writing
|
||||||
|
import xml.etree.cElementTree as ET
|
||||||
|
|
||||||
|
import re, os
|
||||||
|
|
||||||
|
class Node(object):
|
||||||
|
def __init__(self, name = '', index = 0, parent = None):
|
||||||
|
self._parent = parent
|
||||||
|
self._name = name
|
||||||
|
self._value = None
|
||||||
|
self._index = 0
|
||||||
|
self._children = []
|
||||||
|
|
||||||
|
@property
|
||||||
|
def value(self):
|
||||||
|
return self._value
|
||||||
|
|
||||||
|
@value.setter
|
||||||
|
def value(self, v):
|
||||||
|
self._value = v
|
||||||
|
|
||||||
|
@property
|
||||||
|
def name(self):
|
||||||
|
return self._name
|
||||||
|
|
||||||
|
@property
|
||||||
|
def index(self):
|
||||||
|
return self._index
|
||||||
|
|
||||||
|
@property
|
||||||
|
def parent(self):
|
||||||
|
return self._parent
|
||||||
|
|
||||||
|
def getChild(self, n, i=None, create = False):
|
||||||
|
|
||||||
|
if i is None:
|
||||||
|
i = 0
|
||||||
|
# parse name as foo[999] if necessary
|
||||||
|
m = re.match(R"(\w+)\[(\d+)\]", n)
|
||||||
|
if m is not None:
|
||||||
|
n = m.group(1)
|
||||||
|
i = int(m.group(2))
|
||||||
|
|
||||||
|
for c in self._children:
|
||||||
|
if (c.name == n) and (c.index == i):
|
||||||
|
return c
|
||||||
|
|
||||||
|
if create:
|
||||||
|
c = Node(n, i, self)
|
||||||
|
self._children.append(c)
|
||||||
|
return c
|
||||||
|
else:
|
||||||
|
raise IndexError("no such child:" + str(n) + " index=" + str(i))
|
||||||
|
|
||||||
|
def addChild(self, n):
|
||||||
|
i = 0
|
||||||
|
|
||||||
|
# find first free index
|
||||||
|
usedIndices = frozenset(c.index for c in self.getChildren(n))
|
||||||
|
while i < 1000:
|
||||||
|
if i not in usedIndices:
|
||||||
|
break
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
# create it via getChild
|
||||||
|
return self.getChild(n, i, create=True)
|
||||||
|
|
||||||
|
def hasChild(self, nm):
|
||||||
|
for c in self._children:
|
||||||
|
if (c.name == nm):
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
def getChildren(self, n = None):
|
||||||
|
if n is None:
|
||||||
|
return self._children
|
||||||
|
|
||||||
|
return [c for c in self._children if c.name == n]
|
||||||
|
|
||||||
|
def getNode(self, path, cr = False):
|
||||||
|
axes = path.split('/')
|
||||||
|
nd = self
|
||||||
|
for ax in axes:
|
||||||
|
nd = nd.getChild(ax, create = cr)
|
||||||
|
|
||||||
|
return nd
|
||||||
|
|
||||||
|
def getValue(self, path, default = None):
|
||||||
|
try:
|
||||||
|
nd = self.getNode(path)
|
||||||
|
return nd.value
|
||||||
|
except:
|
||||||
|
return default
|
||||||
|
|
||||||
|
def write(self, path):
|
||||||
|
root = self._createXMLElement('PropertyList')
|
||||||
|
|
||||||
|
t = ET.ElementTree(root)
|
||||||
|
t.write(path, 'UTF-8')
|
||||||
|
|
||||||
|
def _createXMLElement(self, nm = None):
|
||||||
|
if nm is None:
|
||||||
|
nm = self.name
|
||||||
|
|
||||||
|
n = ET.Element(nm)
|
||||||
|
|
||||||
|
# value and type specification
|
||||||
|
if self._value is not None:
|
||||||
|
n.text = str(self._value)
|
||||||
|
if isinstance(self._value, int):
|
||||||
|
n.set('type', 'int')
|
||||||
|
elif isinstance(self._value, float):
|
||||||
|
n.set('type', 'double')
|
||||||
|
elif isinstance(self._value, bool):
|
||||||
|
n.set('type', "bool")
|
||||||
|
|
||||||
|
# index in parent
|
||||||
|
if (self.index != 0):
|
||||||
|
n.set('n', self.index)
|
||||||
|
|
||||||
|
# children
|
||||||
|
for c in self._children:
|
||||||
|
n.append(c._createXMLElement())
|
||||||
|
|
||||||
|
return n;
|
||||||
|
|
||||||
|
|
||||||
|
class PropsHandler(handler.ContentHandler):
|
||||||
|
def __init__(self, root = None, path = None, dataDirPath = None):
|
||||||
|
self._root = root
|
||||||
|
self._path = path
|
||||||
|
self._basePath = os.path.dirname(path)
|
||||||
|
self._dataDirPath = dataDirPath
|
||||||
|
self._locator = None
|
||||||
|
|
||||||
|
if root is None:
|
||||||
|
# make a nameless root node
|
||||||
|
self._root = Node("", 0)
|
||||||
|
self._current = self._root
|
||||||
|
|
||||||
|
def setDocumentLocator(self, loc):
|
||||||
|
self._locator = loc
|
||||||
|
|
||||||
|
def startElement(self, name, attrs):
|
||||||
|
self._content = ''
|
||||||
|
if (name == 'PropertyList'):
|
||||||
|
return
|
||||||
|
|
||||||
|
index = 0
|
||||||
|
if 'n' in attrs.keys():
|
||||||
|
index = int(attrs['n'])
|
||||||
|
|
||||||
|
self._current = self._current.getChild(name, index, create=True)
|
||||||
|
|
||||||
|
if 'include' in attrs.keys():
|
||||||
|
self.handleInclude(attrs['include'])
|
||||||
|
|
||||||
|
self._currentTy = None;
|
||||||
|
if 'type' in attrs.keys():
|
||||||
|
self._currentTy = attrs['type']
|
||||||
|
|
||||||
|
def handleInclude(self, includePath):
|
||||||
|
if includePath.startswith('/'):
|
||||||
|
includePath = includePath[1:]
|
||||||
|
|
||||||
|
p = os.path.join(self._basePath, includePath)
|
||||||
|
if not os.path.exists(p):
|
||||||
|
p = os.path.join(self._dataDirPath, includePath)
|
||||||
|
if not os.path.exists(p):
|
||||||
|
raise RuntimeError("include file not found", includePath, "at line", self._locator.getLineNumber())
|
||||||
|
|
||||||
|
readProps(p, self._current, self._dataDirPath)
|
||||||
|
|
||||||
|
def endElement(self, name):
|
||||||
|
if (name == 'PropertyList'):
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
# convert and store value
|
||||||
|
self._current.value = self._content
|
||||||
|
if self._currentTy == "int":
|
||||||
|
self._current.value = int(self._content)
|
||||||
|
if self._currentTy is "bool":
|
||||||
|
self._current.value = bool(self._content)
|
||||||
|
if self._currentTy is "double":
|
||||||
|
self._current.value = float(self._content)
|
||||||
|
except:
|
||||||
|
print "Parse error for value:", self._content, "at line:", self._locator.getLineNumber(), "of:", self._path
|
||||||
|
|
||||||
|
self._current = self._current.parent
|
||||||
|
|
||||||
|
def characters(self, content):
|
||||||
|
self._content += content
|
||||||
|
|
||||||
|
def endDocument(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
@property
|
||||||
|
def root(self):
|
||||||
|
return self._root
|
||||||
|
|
||||||
|
|
||||||
|
def readProps(path, root = None, dataDirPath = None):
|
||||||
|
parser = make_parser()
|
||||||
|
locator = expatreader.ExpatLocator( parser )
|
||||||
|
h = PropsHandler(root, path, dataDirPath)
|
||||||
|
h.setDocumentLocator(locator)
|
||||||
|
parser.setContentHandler(h)
|
||||||
|
parser.parse(path)
|
||||||
|
return h.root
|
||||||
Loading…
Reference in a new issue