diff --git a/catalogTags.py b/catalog/catalogTags.py similarity index 100% rename from catalogTags.py rename to catalog/catalogTags.py diff --git a/git_catalog_repository.py b/git_catalog_repository.py deleted file mode 100644 index 836e8e2..0000000 --- a/git_catalog_repository.py +++ /dev/null @@ -1,50 +0,0 @@ -# git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/ - -import subprocess -import os, sgprops - -class GITCatalogRepository: - def __init__(self, node, singleAircraft = False): - self._path = node.getValue("path") - - if not os.path.exists(os.path.join(self._path , ".git")): - raise RuntimeError("not a Git directory:" + self._path ) - - self._usesSubmodules = node.getValue("uses-submodules", False) - self._singleAircraft = singleAircraft - - self._currentRevision = subprocess.check_output(["git", "rev-parse", "HEAD"], - cwd = self._path) - - self._aircraftPath = None - if node.hasChild("scan-suffix"): - self._aircraftPath = os.path.join(path, node.getValue("scan-suffix")) - - @property - def path(self): - return self._path - - @property - def aircraftPath(self): - return self._aircraftPath - - def hasPathChanged(self, path, oldRev): - diffArgs = ["git", "diff", "--quiet", oldRev, "--"] - if not (self._usesSubmodules and self._singleAircraft): - diffArgs.append(path) - - return subprocess.call(diffArgs, cwd = self._path) - - def update(self): - subprocess.call(["git", "pull"]) - self._currentRevision = subprocess.check_output(["git", "rev-parse", "HEAD"], - cwd = self._path) - - if self._usesSubmodules: - subprocess.call(["git", "submodule", "update"], cwd = self._path) - - def scmRevisionForPath(self, path): - if self._usesSubmodules: - return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd = self._path) - - return self._currentRevision diff --git a/git_discrete_repository.py b/git_discrete_repository.py deleted file mode 100644 index c44f15c..0000000 --- a/git_discrete_repository.py +++ /dev/null @@ -1,34 +0,0 @@ -# git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/ - -import subprocess -import os -import sgprops - -import git_catalog_repository - -class GitDiscreteSCM: - def __init__(self, node): - - configNode = node.parent - - self._repos = {} - - # iterate over aicraft paths finding repositories - for g in config.getChildren("aircraft-dir"): - repo = GITCatalogRepository(g, useSubmodules = False, - singleAircraft = True) - - - def hasPathChanged(self, path, oldRev): - - return self._repos[path].hasPathChanged(path, oldRev) - - def update(self): - for r in self._repos: - r.update() - - def scmRevisionForPath(self, path): - return self._repos[path].scmRevisionForPath(path) - - - \ No newline at end of file diff --git a/maintain_catalog.py b/maintain_catalog.py deleted file mode 100755 index d317455..0000000 --- a/maintain_catalog.py +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/python - -import os, sys, re, glob, shutil -import subprocess -import sgprops -import argparse -import urllib2 -import package as pkg - -import svn_catalog_repository -import git_catalog_repository -import git_discrete_repository - -parser = argparse.ArgumentParser() -parser.add_argument("--clean", help="Regenerate every package", - action="store_true") -parser.add_argument("--update", help="Update/pull SCM source", - action="store_true") -parser.add_argument("--force-dirty", dest="forcedirty", - help="Mark every package as dirty", action="store_true") -parser.add_argument("--no-update", - dest = "noupdate", - help="Disable updating from SCM source", - action="store_true") -parser.add_argument("--no-upload", - dest = "noupload", - help="Disable uploading to destination server", - action="store_true") -parser.add_argument("dir", help="Catalog directory") -args = parser.parse_args() - -CATALOG_VERSION = 4 -includePaths = [] -packages = {} - -def scanPackages(scmRepo): - result = [] - globPath = scmRepo.aircraftPath - if globPath is None: - return result - - print "Scanning", globPath - print os.getcwd() - for d in glob.glob(globPath): - # check dir contains at least one -set.xml file - if len(glob.glob(os.path.join(d, "*-set.xml"))) == 0: - print "no -set.xml in", d - continue - - result.append(pkg.PackageData(d, scmRepo)) - - return result - -def initScmRepository(node): - scmType = node.getValue("type") - if (scmType == "svn"): - return svn_catalog_repository.SVNCatalogRepository(node) - elif (scmType == "git"): - return git_catalog_repository.GITCatalogRepository(node) - elif (scmType == "git-discrete"): - return git_discrete_repository.GitDiscreteSCM(node) - elif (scmType == None): - raise RuntimeError("No scm/type defined in catalog configuration") - else: - raise RuntimeError("Unspported SCM type:" + scmType) - -def initRepositories(): - repositories = [] - - for scm in config.getChildren("scm"): - scmRepo = initScmRepository(scm) - if args.update or (not args.noupdate and scm.getValue("update")): - scmRepo.update() - # presumably include repos in parse path - # TODO: make this configurable - includePaths.append(scmRepo.path) - repositories.append(scmRepo) - - return repositories - -def processUpload(node, outputPath): - if args.noupload or not node.getValue("enabled", True): - print "Upload disabled" - return - - uploadType = node.getValue("type") - if (uploadType == "rsync"): - subprocess.call(["rsync", node.getValue("args", "-az"), ".", - node.getValue("remote")], - cwd = outputPath) - elif (uploadType == "rsync-ssh"): - print "Doing rsync upload to:", node.getValue("remote") - subprocess.call(["rsync", node.getValue("args", "-azve"), - "ssh", ".", - node.getValue("remote")], - cwd = outputPath) - elif (uploadType == "scp"): - subprocess.call(["scp", node.getValue("args", "-r"), ".", - node.getValue("remote")], - cwd = outputPath) - else: - raise RuntimeError("Unsupported upload type:" + uploadType) - -def parseExistingCatalog(): - global existingCatalogPath - global previousCatalog - - # contains existing catalog - existingCatalogPath = os.path.join(outPath, 'catalog.xml') - - if not os.path.exists(existingCatalogPath): - url = config.getValue("template/url") - print "Attempting downloading from", url - try: - # can happen on new or from clean, try to pull current - # catalog from the upload location - response = urllib2.urlopen(url, timeout = 5) - content = response.read() - f = open(existingCatalogPath, 'w' ) - f.write( content ) - f.close() - print "...worked" - except urllib2.URLError as e: - print "Downloading current catalog failed", e, "from", url - -rootDir = args.dir -if not os.path.isabs(rootDir): - rootDir = os.path.abspath(rootDir) -os.chdir(rootDir) - -configPath = 'catalog.config.xml' -if not os.path.exists(configPath): - raise RuntimeError("no config file found at:" + configPath) - -config = sgprops.readProps(configPath) - -# out path -outPath = config.getValue('output-dir') -if outPath is None: - # default out path - outPath = os.path.join(rootDir, "output") -elif not os.path.isabs(outPath): - outPath = os.path.join(rootDir, "output") - -if args.clean: - print "Cleaning output" - shutil.rmtree(outPath) - -if not os.path.exists(outPath): - os.mkdir(outPath) - -thumbnailPath = os.path.join(outPath, config.getValue('thumbnail-dir', "thumbnails")) -if not os.path.exists(thumbnailPath): - os.mkdir(thumbnailPath) - -thumbnailUrls = list(t.value for t in config.getChildren("thumbnail-url")) - -for i in config.getChildren("include-dir"): - if not os.path.exists(i.value): - print "Skipping missing include path:", i.value - continue - includePaths.append(i.value) - -parseExistingCatalog() -repositories = initRepositories() - -for scm in repositories: - for p in scanPackages(scm): - try: - p.scanSetXmlFiles(includePaths) - packages[p.id] = p - except: - print "Skipping SCM package due to exception:", p.path - -if os.path.exists(existingCatalogPath): - try: - previousCatalog = sgprops.readProps(existingCatalogPath) - except: - print "Previous catalog is malformed" - previousCatalog = sgprops.Node() - - for p in previousCatalog.getChildren("package"): - pkgId = p.getValue("id") - if not pkgId in packages.keys(): - print "Orphaned old package:", pkgId - continue - - packages[pkgId].setPreviousData(p) -else: - print "No previous catalog" - -catalogNode = sgprops.Node("catalog") -sgprops.copy(config.getChild("template"), catalogNode) - -catalogNode.getChild("catalog-version", create = True).value = CATALOG_VERSION -mirrorUrls = list(m.value for m in config.getChildren("mirror")) - -packagesToGenerate = [] -for p in packages.values(): - if p.isSourceModified or args.forcedirty: - packagesToGenerate.append(p) - else: - p.useExistingCatalogData() - -excludeFilePath = os.path.join(rootDir, "zip-excludes.lst") - -# def f(x): -# x.generateZip(outPath) -# x.extractThumbnails(thumbnailPath) -# return True -# -# p = Pool(8) -# print(p.map(f,packagesToGenerate)) - -for p in packagesToGenerate: - p.generateZip(outPath, excludeFilePath) - p.extractThumbnails(thumbnailPath) - -print "Creating catalog" -for p in packages.values(): - catalogNode.addChild(p.packageNode(mirrorUrls, thumbnailUrls[0])) - -catalogNode.write(os.path.join(outPath, "catalog.xml")) - -for up in config.getChildren("upload"): - processUpload(up, outPath) diff --git a/package.py b/package.py deleted file mode 100644 index 757106c..0000000 --- a/package.py +++ /dev/null @@ -1,239 +0,0 @@ -import os, subprocess -import sgprops -import hashlib # for MD5 -import shutil # for copy2 -import catalogTags - -standardTagSet = frozenset(catalogTags.tags) -def isNonstandardTag(t): - return t not in standardTagSet - -thumbnailNames = ["thumbnail.png", "thumbnail.jpg"] - -class VariantData: - def __init__(self, path, node): - #self._primary = primary - self._path = path - self._name = node.getValue("sim/description") - if (not self._name): - print "Missing description for " + path - self._name = "Missing description:" + self.id - - # ratings - - # seperate thumbnails - - @property - def name(self): - return self._name - - @property - def id(self): - return self._path[:-8] # "remove -set.xml" (8 chars) - - @property - def catalogNode(self): - n = sgprops.Node("variant") - n.addChild("id").value = self.id - n.addChild("name").value = self.name - return n - -class PackageData: - def __init__(self, path, scmRepo): - self._path = path - self._scm = scmRepo - self._previousSCMRevision = None - self._previousRevision = 0 - self._thumbnails = [] - self._variants = {} - self._revision = 0 - self._md5 = None - self._fileSize = 0 - self._primarySetXmlPath = None - self._node = sgprops.Node("package") - - def setPreviousData(self, node): - self._previousRevision = node.getValue("revision") - self._previousMD5 = node.getValue("md5") - self._previousSCMRevision = node.getValue("scm-revision") - self._fileSize = int(node.getValue("file-size-bytes")) - - @property - def id(self): - return self._primarySetXmlPath - - @property - def thumbnails(self): - return self._thumbnails - - @property - def path(self): - return self._path - - @property - def variants(self): - return self._variants - - @property - def scmRevision(self): - currentRev = self._scm.scmRevisionForPath(self.path) - if (currentRev is None): - raise RuntimeError("Unable to query SCM revision of files") - - return currentRev - - @property - def isSourceModified(self): - if (self._previousSCMRevision == None): - return True - - if (self._previousSCMRevision == self.scmRevision): - return False - - return True - - def scanSetXmlFiles(self, includes): - foundPrimary = False - foundMultiple = False - - for f in os.listdir(self.path): - if not f.endswith("-set.xml"): - continue - - p = os.path.join(self.path, f) - node = sgprops.readProps(p, includePaths = includes) - if not node.hasChild("sim"): - continue - - simNode = node.getChild("sim") - # honour variosu exclusion flags - if (simNode.getValue("exclude-from-catalog", False) or simNode.getValue("exclude-from-gui", False)): - continue - - primary = simNode.getValue("variant-of", None) - if primary: - if not primary in self.variants: - self._variants[primary] = [] - self._variants[primary].append(VariantData(f, node)) - continue - - if foundPrimary: - if not foundMultiple: - print "Multiple primary -set.xml files at:" + self.path - print "\t" + p - foundMultiple = True - continue - else: - foundPrimary = True; - self._primarySetXmlPath = f[:-8] # trim -set.xml - - self.parsePrimarySetNode(simNode) - - for n in thumbnailNames: - if os.path.exists(os.path.join(self.path, n)): - self._thumbnails.append(n) - - if not foundPrimary: - raise RuntimeError("No primary -set.xml found at:" + self.path) - - - - def parsePrimarySetNode(self, sim): - - # basic / mandatory values - self._node.addChild('name').value = sim.getValue('description') - - longDesc = sim.getValue('long-description') - if longDesc is not None: - self._node.addChild('description').value = longDesc - - # copy all the standard values - for p in ['status', 'author', 'license']: - v = sim.getValue(p) - if v is not None: - self._node.addChild(p).value = v - - # ratings - if sim.hasChild('rating'): - pkgRatings = self._node.addChild('rating') - for r in ['FDM', 'systems', 'cockpit', 'model']: - pkgRatings.addChild(r).value = sim.getValue('rating/' + r, 0) - - # copy tags - if sim.hasChild('tags'): - for c in sim.getChild('tags').getChildren('tag'): - if isNonstandardTag(c.value): - print "Skipping non-standard tag:", c.value, self.path - else: - self._node.addChild('tag').value = c.value - - for t in sim.getChildren("thumbnail"): - self._thumbnails.append(t.value) - - def validate(self): - for t in self._thumbnails: - if not os.path.exists(os.path.join(self.path, t)): - raise RuntimeError("missing thumbnail:" + t); - - def generateZip(self, outDir, globalExcludePath): - self._revision = self._previousRevision + 1 - - baseName = os.path.basename(self.path) - zipName = baseName + ".zip" - zipFilePath = os.path.join(outDir, zipName) - - os.chdir(os.path.dirname(self.path)) - - print "Creating zip", zipFilePath - # anything we can do to make this faster? - - zipArgs = ['zip', '--quiet', '-r'] - if os.path.exists(globalExcludePath): - zipArgs += [ "-x@" + globalExcludePath] - - excludePath = os.path.join(self.path, 'package-exclude.lst') - if (os.path.exists(excludePath)): - print self.id, "has zip exclude list" - zipArgs += ["-x@" + excludePath] - - zipArgs += [zipFilePath, baseName] - subprocess.call(zipArgs) - - zipFile = open(zipFilePath, 'r') - self._md5 = hashlib.md5(zipFile.read()).hexdigest() - self._fileSize = os.path.getsize(zipFilePath) - - def useExistingCatalogData(self): - self._md5 = self._previousMD5 - - def packageNode(self, mirrorUrls, thumbnailUrl): - self._node.addChild("id").value = self.id - self._node.getChild("md5", create = True).value = self._md5 - self._node.getChild("file-size-bytes", create = True).value = self._fileSize - self._node.getChild("revision", create = True).value = int(self._revision) - self._node.getChild("scm-revision", create = True).value = self.scmRevision - - baseName = os.path.basename(self.path) - self._node.getChild("dir", create = True).value = baseName - zipName = baseName + ".zip" - - for m in mirrorUrls: - self._node.addChild("url").value = m + "/" + zipName - - for t in self._thumbnails: - self._node.addChild("thumbnail-path").value = t - self._node.addChild("thumbnail").value = thumbnailUrl + "/" + self.id + "_" + t - - for pr in self._variants: - for vr in self._variants[pr]: - self._node.addChild(vr.catalogNode) - - return self._node - - def extractThumbnails(self, thumbnailDir): - for t in self._thumbnails: - fullName = self.id + "_" + t - shutil.copy2(os.path.join(self._path, t), - os.path.join(thumbnailDir, fullName) - ) - # TODO : verify image format, size and so on diff --git a/sgprops.py b/sgprops.py deleted file mode 100644 index 1ea6d69..0000000 --- a/sgprops.py +++ /dev/null @@ -1,277 +0,0 @@ -# SAX for parsing -from xml.sax import make_parser, handler, expatreader - -# ElementTree for writing -import xml.etree.cElementTree as ET - -import re, os - -class Node(object): - def __init__(self, name = '', index = 0, parent = None): - self._parent = parent - self._name = name - self._value = None - self._index = index - self._children = [] - - @property - def value(self): - return self._value - - @value.setter - def value(self, v): - self._value = v - - @property - def name(self): - return self._name - - @property - def index(self): - return self._index - - @property - def parent(self): - return self._parent - - def getChild(self, n, i=None, create = False): - - if i is None: - i = 0 - # parse name as foo[999] if necessary - m = re.match(R"(\w+)\[(\d+)\]", n) - if m is not None: - n = m.group(1) - i = int(m.group(2)) - - for c in self._children: - if (c.name == n) and (c.index == i): - return c - - if create: - c = Node(n, i, self) - self._children.append(c) - return c - else: - raise IndexError("no such child:" + str(n) + " index=" + str(i)) - - def addChild(self, n): - # adding an existing instance - if isinstance(n, Node): - n._parent = self - n._index = self.firstUnusedIndex(n.name) - self._children.append(n) - return n - - i = self.firstUnusedIndex(n) - # create it via getChild - return self.getChild(n, i, create=True) - - def firstUnusedIndex(self, n): - usedIndices = frozenset(c.index for c in self.getChildren(n)) - i = 0 - while i < 1000: - if i not in usedIndices: - return i - i += 1 - raise RuntimeException("too many children with name:" + n) - - def hasChild(self, nm): - for c in self._children: - if (c.name == nm): - return True - - return False - - def getChildren(self, n = None): - if n is None: - return self._children - - return [c for c in self._children if c.name == n] - - def getNode(self, path, cr = False): - axes = path.split('/') - nd = self - for ax in axes: - nd = nd.getChild(ax, create = cr) - - return nd - - def getValue(self, path, default = None): - try: - nd = self.getNode(path) - return nd.value - except: - return default - - def write(self, path): - root = self._createXMLElement('PropertyList') - t = ET.ElementTree(root) - t.write(path, 'utf-8', xml_declaration = True) - - def _createXMLElement(self, nm = None): - if nm is None: - nm = self.name - - n = ET.Element(nm) - - # value and type specification - try: - if self._value is not None: - if isinstance(self._value, basestring): - # don't call str() on strings, breaks the - # encoding - n.text = self._value - else: - # use str() to turn non-string types into text - n.text = str(self._value) - if isinstance(self._value, int): - n.set('type', 'int') - elif isinstance(self._value, float): - n.set('type', 'double') - elif isinstance(self._value, bool): - n.set('type', "bool") - except UnicodeEncodeError: - print "Encoding error with", self._value, type(self._value) - - # index in parent - if (self.index != 0): - n.set('n', str(self.index)) - - # children - for c in self._children: - n.append(c._createXMLElement()) - - return n; - - -class PropsHandler(handler.ContentHandler): - def __init__(self, root = None, path = None, includePaths = []): - self._root = root - self._path = path - self._basePath = os.path.dirname(path) - self._includes = includePaths - self._locator = None - - if root is None: - # make a nameless root node - self._root = Node("", 0) - self._current = self._root - - def setDocumentLocator(self, loc): - self._locator = loc - - def startElement(self, name, attrs): - self._content = None - if (name == 'PropertyList'): - return - - if 'n' in attrs.keys(): - try: - index = int(attrs['n']) - except: - print "Invalid index at line:", self._locator.getLineNumber(), "of", self._path - self._current = self._current.addChild(name) - return - - self._current = self._current.getChild(name, index, create=True) - else: - self._current = self._current.addChild(name) - - - if 'include' in attrs.keys(): - self.handleInclude(attrs['include']) - - self._currentTy = None; - if 'type' in attrs.keys(): - self._currentTy = attrs['type'] - - def handleInclude(self, includePath): - if includePath.startswith('/'): - includePath = includePath[1:] - - p = os.path.join(self._basePath, includePath) - if not os.path.exists(p): - found = False - for i in self._includes: - p = os.path.join(i, includePath) - if os.path.exists(p): - found = True - break - - if not found: - raise RuntimeError("include file not found", includePath, "at line", self._locator.getLineNumber()) - - readProps(p, self._current, self._includes) - - def endElement(self, name): - if (name == 'PropertyList'): - return - - try: - # convert and store value - self._current.value = self._content - if self._currentTy == "int": - self._current.value = int(self._content) if self._content is not None else 0 - if self._currentTy == "bool": - self._current.value = self.parsePropsBool(self._content) - if self._currentTy == "double": - if self._content is None: - self._current.value = 0.0 - else: - if self._content.endswith('f'): - self._content = self._content[:-1] - self._current.value = float(self._content) - except: - print "Parse error for value:", self._content, "at line:", self._locator.getLineNumber(), "of:", self._path - - self._current = self._current.parent - self._content = None - self._currentTy = None - - - def parsePropsBool(self, content): - if content == "True" or content == "true": - return True - - if content == "False" or content == "false": - return False - - try: - icontent = int(content) - if icontent is not None: - if icontent == 0: - return False - else: - return True; - except: - return False - - def characters(self, content): - if self._content is None: - self._content = '' - self._content += content - - def endDocument(self): - pass - - @property - def root(self): - return self._root - -def readProps(path, root = None, includePaths = []): - parser = make_parser() - locator = expatreader.ExpatLocator( parser ) - h = PropsHandler(root, path, includePaths) - h.setDocumentLocator(locator) - parser.setContentHandler(h) - parser.parse(path) - return h.root - -def copy(src, dest): - dest.value = src.value - - # recurse over children - for c in src.getChildren() : - dc = dest.getChild(c.name, i = c.index, create = True) - copy(c, dc) diff --git a/svn_catalog_repository.py b/svn_catalog_repository.py deleted file mode 100644 index da182cc..0000000 --- a/svn_catalog_repository.py +++ /dev/null @@ -1,41 +0,0 @@ - -import subprocess, os, sgprops -import xml.etree.cElementTree as ET - -class SVNCatalogRepository: - def __init__(self, node): - path = node.getValue("path") - if not os.path.exists(path): - raise RuntimeError("No directory at:" + path) - - self._path = path - xml = subprocess.check_output(["svn", "info", "--xml", path]) - root = ET.fromstring(xml) - - if (root.find(".//repository/root") == None): - raise RuntimeError("Not an SVN repository:" + path) - - self._aircraftPath = None - if node.hasChild("scan-suffix"): - self._aircraftPath = os.path.join(path, node.getValue("scan-suffix")) - - @property - def path(self): - return self._path - - @property - def aircraftPath(self): - return self._aircraftPath - - def hasPathChanged(self, path, oldRevision): - return self.scmRevisionForPath(path) != oldRevision - - def scmRevisionForPath(self, path): - xml = subprocess.check_output(["svn", "info", "--xml", path]) - root = ET.fromstring(xml) - commit = root.find(".//entry/commit") - return commit.get('revision', 0) - - def update(self): - print "SVN update of", self._path - subprocess.call(["svn", "update", self._path]) diff --git a/template_catalog.xml b/template_catalog.xml deleted file mode 100644 index ed315fe..0000000 --- a/template_catalog.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - - 3.4.* - 3.5.* - 3.6.* - - org.myorganisation.hangar - GPL - http://some.stable.url.com/foo/bar/catalog.xml - - A collection of interesting aircraft with some features - - - - Au Deutsch - - - - Francais - - - http://some.url/ - - - - http://some.url/images - - git://some.git.repo/ - - Aircraft - - - - \ No newline at end of file