diff --git a/create_catalog.py b/create_catalog.py index 7af7f4f..e572746 100755 --- a/create_catalog.py +++ b/create_catalog.py @@ -1,171 +1,82 @@ #!/usr/bin/python -import os -import sys -import fnmatch -import tarfile -import hashlib -import shutil +import os, sys, re -import xml.etree.cElementTree as ET +import sgprops -rootPath = sys.argv[1] -outputDir = sys.argv[2] +fgRoot = sys.argv[1] +aircraftDir = os.path.join(fgRoot, 'Aircraft') -existingCatalogPath = os.path.join(outputDir, 'catalog.xml') -existingCatalog = None -print 'existing ctalog path:' + existingCatalogPath -if os.path.exists(existingCatalogPath): - existingCatalog = ET.parse(existingCatalogPath) +catalogProps = sgprops.Node() +catalogProps.addChild('version').value = '3.1.0' +catalogProps.addChild('id').value = 'org.flightgear.default' +catalogProps.addChild('license').value = 'GPL' +catalogProps.addChild('url').value = "http://fgfs.goneabitbursar.com/pkg/3.1.0/default-catalog.xml" -for file in os.listdir(outputDir): - if fnmatch.fnmatch(file, '*.tar.gz'): - os.remove(os.path.join(outputDir, file)); +catalogProps.addChild('description').value = "Aircraft developed and maintained by the FlightGear project" +de = catalogProps.addChild('de') +# de.addChild('description').value = "" -thumbsDir = os.path.join(outputDir, 'thumbs') -shutil.rmtree(thumbsDir) -os.makedirs(thumbsDir) +fr = catalogProps.addChild('fr') -def setProperty(node, id, value): - s = node.find(id) # check for existing - if s is None: - s = ET.SubElement(node, id) - s.text = value +urls = [ + "http://flightgear.wo0t.de/Aircraft-3.0/{acft}_20140116.zip", + "http://ftp.icm.edu.pl/packages/flightgear/Aircraft-3.0/{acft}_20140216.zip", + "http://mirrors.ibiblio.org/pub/mirrors/flightgear/ftp/Aircraft-3.0/{acft}_20140216.zip", + "http://ftp.igh.cnrs.fr/pub/flightgear/ftp/Aircraft-3.0/{acft}_20140116.zip", + "http://ftp.linux.kiev.ua/pub/fgfs/Aircraft-3.0/{acft}_20140116.zip", + "http://fgfs.physra.net/ftp/Aircraft-3.0/{acft}_20130225.zip" +] -def clearChildren(node, tag): - for c in node.findall(tag): - node.remove(c) +thumbs = [ + "http://www.flightgear.org/thumbs/v3.0/{acft}.jpg" +] -def parse_setXml(path): - tree = ET.parse(path) - - desc = tree.find('sim/description') - ratings = tree.find('sim/rating') - if (ratings is not None): - for rating in list(ratings): - if rating.tag == 'status': - continue - - rvalue = int(rating.text) - if rvalue < 2: - return None - else: - return None - - d = {} - - d['desc'] = desc - d['ratings'] = ratings; - d['status'] = tree.find('sim/status') - d['authors'] = tree.findall('sim/author') - - return d - -def process_aircraft(acft, path): - print '===' + acft + '===' - setFiles = [] - thumbs = [] - - for file in os.listdir(path): - if fnmatch.fnmatch(file, '*-set.xml'): - setFiles.append(file); +for d in os.listdir(aircraftDir): + acftDirPath = os.path.join(aircraftDir, d) + if not os.path.isdir(acftDirPath): + continue - if fnmatch.fnmatch(file, 'thumbnail*'): - thumbs.append(file) - - aircraft = [] - for s in setFiles: - d = parse_setXml(os.path.join(path, s)) - if d is None: - continue - - d['set'] = s[0:-8] - aircraft.append(d) - - thumbnailNames = [] - # copy thumbnails - for t in thumbs: - outThumb = os.path.join(thumbsDir, acft + "-" + t) - thumbnailNames.append(acft + "-" + t) - shutil.copyfile(os.path.join(path, t), outThumb) + setFilePath = None + + # find the first set file + # FIXME - way to designate the primary file + for f in os.listdir(acftDirPath): + if f.endswith("-set.xml"): + setFilePath = os.path.join(acftDirPath, f) + break - if len(aircraft) == 0: - print "no aircraft profiles for " + acft - return - - # tarball creation - outTar = os.path.join(outputDir, acft + ".tar.gz") - tar = tarfile.open(outTar, "w:gz") - tar.add(path, acft) - tar.close() - - digest = hashlib.md5(open(outTar, 'r').read()).hexdigest() - revision = 1 + if setFilePath is None: + print "No -set.xml file found in",acftDirPath,"will be skipped" + continue - # revision check - if acft in existingPackages: - previousMd5 = existingPackages[acft].find('md5').text - previousRevsion = int(existingPackages[acft].find('revision').text) - if digest != previousMd5: - print acft + ": MD5 has changed" - revision = previousRevsion + 1 - else: - print acft + ": MD5 is unchanged" - else: - existingPackages[acft] = ET.Element('package') - setProperty(existingPackages[acft], 'id', acft) - - setProperty(existingPackages[acft], 'revision', str(revision)) - setProperty(existingPackages[acft], 'md5', digest) - setProperty(existingPackages[acft], 'description', aircraft[0]['desc']) + try: + props = sgprops.readProps(setFilePath, dataDirPath = fgRoot) + sim = props.getNode("sim") + + pkgNode = catalogProps.addChild('package') + pkgNode.addChild('id').value = d + pkgNode.addChild('name').value = sim.getValue('description') - clearChildren(existingPackages[acft], 'thumbnail') - for t in thumbnailNames: - tn = ET.SubElement(existingPackages[acft], 'thumbnail') - tn.text = 'thumbs/' + t - - clearChildren(existingPackages[acft], 'rating') - existingPackages[acft].append(aircraft[0]['ratings']) - - print "wrote tarfile, digest is " + digest - -root = ET.Element('PropertyList') -catalogTree = ET.ElementTree(root) - -existingPackages = dict() - -if (existingCatalog is not None): - print 'have existing catalog data' - - root.append(existingCatalog.find('license')) - root.append(existingCatalog.find('url')) - root.append(existingCatalog.find('description')) - root.append(existingCatalog.find('id')) - - # existing data (for revision incrementing) - for n in existingCatalog.findall('package'): - idNode = n.find('id') - if idNode is None: - print 'Missing tag on package' - continue + longDesc = sim.getValue('long-description') + if longDesc is not None: + pkgNode.addChild('description').value = longDesc - existingPackages[idNode.text] = n; + # copy tags + if sim.hasChild('tags'): + for c in sim.getChild('tags').getChildren('tag'): + pkgNode.addChild('tag').value = c.value + + # create download and thumbnail URLs + for u in urls: + pkgNode.addChild("url").value = u.format(acft=d) + + for t in thumbs: + pkgNode.addChild("thumbnail").value = t.format(acft=d) + + except: + print "Failure processing:", setFilePath -#licenseElement = ET.SubElement(root, 'license') -#licenseElement.text = 'gpl' - -#urlElement = ET.SubElement(root, 'url') -#urlElement.text = 'http://catalog.xml' - -for acft in os.listdir(rootPath): - path = os.path.join(rootPath, acft); - if (os.path.isdir(path)): - process_aircraft(acft, path) - - -for ep in existingPackages: - root.append(existingPackages[ep]) - -catalogTree.write(os.path.join(outputDir, 'catalog.xml'), 'UTF-8') +catalogProps.write("catalog.xml") \ No newline at end of file diff --git a/sgprops.py b/sgprops.py new file mode 100644 index 0000000..1b4a210 --- /dev/null +++ b/sgprops.py @@ -0,0 +1,214 @@ +# SAX for parsing +from xml.sax import make_parser, handler, expatreader + +# ElementTree for writing +import xml.etree.cElementTree as ET + +import re, os + +class Node(object): + def __init__(self, name = '', index = 0, parent = None): + self._parent = parent + self._name = name + self._value = None + self._index = 0 + self._children = [] + + @property + def value(self): + return self._value + + @value.setter + def value(self, v): + self._value = v + + @property + def name(self): + return self._name + + @property + def index(self): + return self._index + + @property + def parent(self): + return self._parent + + def getChild(self, n, i=None, create = False): + + if i is None: + i = 0 + # parse name as foo[999] if necessary + m = re.match(R"(\w+)\[(\d+)\]", n) + if m is not None: + n = m.group(1) + i = int(m.group(2)) + + for c in self._children: + if (c.name == n) and (c.index == i): + return c + + if create: + c = Node(n, i, self) + self._children.append(c) + return c + else: + raise IndexError("no such child:" + str(n) + " index=" + str(i)) + + def addChild(self, n): + i = 0 + + # find first free index + usedIndices = frozenset(c.index for c in self.getChildren(n)) + while i < 1000: + if i not in usedIndices: + break + i += 1 + + # create it via getChild + return self.getChild(n, i, create=True) + + def hasChild(self, nm): + for c in self._children: + if (c.name == nm): + return True + + return False + + def getChildren(self, n = None): + if n is None: + return self._children + + return [c for c in self._children if c.name == n] + + def getNode(self, path, cr = False): + axes = path.split('/') + nd = self + for ax in axes: + nd = nd.getChild(ax, create = cr) + + return nd + + def getValue(self, path, default = None): + try: + nd = self.getNode(path) + return nd.value + except: + return default + + def write(self, path): + root = self._createXMLElement('PropertyList') + + t = ET.ElementTree(root) + t.write(path, 'UTF-8') + + def _createXMLElement(self, nm = None): + if nm is None: + nm = self.name + + n = ET.Element(nm) + + # value and type specification + if self._value is not None: + n.text = str(self._value) + if isinstance(self._value, int): + n.set('type', 'int') + elif isinstance(self._value, float): + n.set('type', 'double') + elif isinstance(self._value, bool): + n.set('type', "bool") + + # index in parent + if (self.index != 0): + n.set('n', self.index) + + # children + for c in self._children: + n.append(c._createXMLElement()) + + return n; + + +class PropsHandler(handler.ContentHandler): + def __init__(self, root = None, path = None, dataDirPath = None): + self._root = root + self._path = path + self._basePath = os.path.dirname(path) + self._dataDirPath = dataDirPath + self._locator = None + + if root is None: + # make a nameless root node + self._root = Node("", 0) + self._current = self._root + + def setDocumentLocator(self, loc): + self._locator = loc + + def startElement(self, name, attrs): + self._content = '' + if (name == 'PropertyList'): + return + + index = 0 + if 'n' in attrs.keys(): + index = int(attrs['n']) + + self._current = self._current.getChild(name, index, create=True) + + if 'include' in attrs.keys(): + self.handleInclude(attrs['include']) + + self._currentTy = None; + if 'type' in attrs.keys(): + self._currentTy = attrs['type'] + + def handleInclude(self, includePath): + if includePath.startswith('/'): + includePath = includePath[1:] + + p = os.path.join(self._basePath, includePath) + if not os.path.exists(p): + p = os.path.join(self._dataDirPath, includePath) + if not os.path.exists(p): + raise RuntimeError("include file not found", includePath, "at line", self._locator.getLineNumber()) + + readProps(p, self._current, self._dataDirPath) + + def endElement(self, name): + if (name == 'PropertyList'): + return + + try: + # convert and store value + self._current.value = self._content + if self._currentTy == "int": + self._current.value = int(self._content) + if self._currentTy is "bool": + self._current.value = bool(self._content) + if self._currentTy is "double": + self._current.value = float(self._content) + except: + print "Parse error for value:", self._content, "at line:", self._locator.getLineNumber(), "of:", self._path + + self._current = self._current.parent + + def characters(self, content): + self._content += content + + def endDocument(self): + pass + + @property + def root(self): + return self._root + + +def readProps(path, root = None, dataDirPath = None): + parser = make_parser() + locator = expatreader.ExpatLocator( parser ) + h = PropsHandler(root, path, dataDirPath) + h.setDocumentLocator(locator) + parser.setContentHandler(h) + parser.parse(path) + return h.root \ No newline at end of file