diff --git a/git_catalog_repository.py b/git_catalog_repository.py index 789c468..2ac085b 100644 --- a/git_catalog_repository.py +++ b/git_catalog_repository.py @@ -7,7 +7,7 @@ class GITCatalogRepository: def __init__(self, path, usesSubmodules = False, singleAircraft = False): self._path = path - if !os.path.exists(os.path.join(path, ".git")): + if not os.path.exists(os.path.join(path, ".git")): raise RuntimeError("not a Git directory:" + path) self._usesSubmodules = usesSubmodules diff --git a/git_discrete_repository.py b/git_discrete_repository.py index 33dc496..c44f15c 100644 --- a/git_discrete_repository.py +++ b/git_discrete_repository.py @@ -4,7 +4,7 @@ import subprocess import os import sgprops -import GITCatalogRepository +import git_catalog_repository class GitDiscreteSCM: def __init__(self, node): diff --git a/maintain_catalog.py b/maintain_catalog.py index e32ce4f..319e50f 100755 --- a/maintain_catalog.py +++ b/maintain_catalog.py @@ -12,120 +12,131 @@ import git_catalog_repository import git_discrete_repository # TODO -# uploading / rsyncing +# uploading / rsyncing class VariantData: def __init__(self, primary, path, node): self._primary = primary self._path = path self._name = node.getValue("sim/description") - + # ratings - + # seperate thumbnails - + @property def catalogNode(self): n = Node("variant") - n.addChild("id") = path - m.addChild("name") = self._name - + n.addChild("id").value = path + n.addChild("name").value = self._name + class PackageData: - def __init__(path): + def __init__(self, path): self._path = path self._previousSCMRevision = None self._previousRevision = 0 self._thumbnails = [] self._variants = {} - - self._node = sgprops.Node() + self._revision = 0 + self._md5 = None + self._fileSize = 0 + + self._node = sgprops.Node("package") self._node.addChild("id").value = self.id - - def setPreviousData(node): + + def setPreviousData(self, node): self._previousRevision = node.getValue("revision") self._previousMD5 = node.getValue("md5") self._previousSCMRevision = node.getValue("scm-revision") - + self._fileSize = int(node.getValue("file-size-bytes")) + @property def id(self): return os.path.basename(self._path) - + @property def thumbnails(self): return self._thumbnails - - def isSourceModified(self, scmRepo): - if (self._previousSCMRevision == None): - return True - + + @property + def path(self): + return self._path + + @property + def scmRevision(self): currentRev = scmRepo.scmRevisionForPath(self._path) if (currentRev is None): raise RuntimeError("Unable to query SCM revision of files") - - if (self._previousSCMRevision == currentRev): - self._scm = self._previousSCMRevision + + return currentRev + + def isSourceModified(self, scmRepo): + if (self._previousSCMRevision == None): + return True + + if (self._previousSCMRevision == self.scmRevision): return False - - self._scm = currentRev + return True - + def scanSetXmlFiles(self): foundPrimary = False - + for f in os.listdir(self._path): - if !f.endswith("-set.xml"): + if not f.endswith("-set.xml"): continue - + p = os.path.join(self._path, f) node = readProps(p) simNode = node.getChild("sim") if (simNode.getValue("exclude")): continue - - if primary = simNode.getValue("variant-of", None): + + primary = simNode.getValue("variant-of", None) + if primary: if not primary in variants: self._variants[primary] = [] self._variants[primary].append(VariantData(self, node)) continue - + if foundPrimary: print "Multiple primary -set.xml files at:" + self._path continue else: foundPrimary = True; - + parsePrimarySetNode(simNode) - + if os.path.exists(os.path.join(self._path, "thumbnail.png")): self._thumbnails.append("thumbnail.png") - + if not foundPrimary: raise RuntimeError("No primary -set.xml found at:" + self._path) - - - + + + def parsePrimarySetNode(self, sim): - + # basic / mandatory values self._node.addChild('id').value = d self._node.addChild('name').value = sim.getValue('description') - + longDesc = sim.getValue('long-description') if longDesc is not None: self._node.addChild('description').value = longDesc - + # copy all the standard values for p in ['status', 'author', 'license']: v = sim.getValue(p) if v is not None: self._node.addChild(p).value = v - + # ratings if sim.hasChild('rating'): pkgRatings = self._node.addChild('rating') for r in ['FDM', 'systems', 'cockpit', 'model']: pkgRatings.addChild(r).value = sim.getValue('rating/' + r, 0) - + # copy tags if sim.hasChild('tags'): for c in sim.getChild('tags').getChildren('tag'): @@ -133,110 +144,137 @@ class PackageData: print "Skipping non-standard tag:", c.value else: self._node.addChild('tag').value = c.value - + self._thumbnails = (t.value for t in self.getChildren("thumbnail")) - + def validate(self): for t in self._thumbnails: if not os.path.exists(os.path.join(self._path, t)): raise RuntimeError("missing thumbnail:" + t); - + def generateZip(self, outDir): self._revision = self._previousRevision + 1 - - zipName = self.id + + zipName = self.id + ".zip" zipFilePath = os.path.join(outDir, zipName) - + + os.chdir(os.path.dirname(self.path)) + + print "Creating zip", zipFilePath # TODO: exclude certain files - subprocess.call(['zip', '-r', self.path, zipFilePath]) - - zipFile = open(zipFilePath + ".zip", 'r') + # anything we can do to make this faster? + subprocess.call(['zip', '--quiet', '-r', zipFilePath, self.id]) + + zipFile = open(zipFilePath, 'r') self._md5 = hashlib.md5(zipFile.read()).hexdigest() - self._fileSize = os.path.getsize(zipFile) - - @property - def catalogNode(self, mirrorUrls, thumbnailUrl): + self._fileSize = os.path.getsize(zipFilePath) + + def useExistingCatalogData(self): + self._md5 = self._previousMD5 + + def packageNode(self, mirrorUrls, thumbnailUrl): self._node.getChild("md5", create = True).value = self._md5 self._node.getChild("file-size-bytes", create = True).value = self._fileSize - self._node.addChild("revision", create = True).value = self._revision - self._node.addChild("scm-revision", create = True).value = self._scm - + self._node.getChild("revision", create = True).value = int(self._revision) + self._node.getChild("scm-revision", create = True).value = self.scmRevision + for m in mirrorUrls: self._node.addChild("url", m + "/" + self.id + ".zip") - + for t in self._thumbnails: self._node.addChild("thumbnail", thumbnailUrl + "/" + self.id + "_" + t) - + for pr in self._variants: for vr in self._variants[pr]: self._node.addChild(vr.catalogNode) - + return self._node - - def extractThumnbails(self, thumbnailDir): + + def extractThumbnails(self, thumbnailDir): for t in self._thumbnails: fullName = self.id + "_" + t os.file.copy(os.path.join(self._path, t), os.path.join(thumbnailDir, fullName) ) # TODO : verify image format, size and so on - + def scanPackages(globPath): result = [] - for d = in glob.glob(globPath): + print "Scanning", globPath + print os.getcwd() + for d in glob.glob(globPath): result.append(PackageData(d)) - + return result def initScmRepository(node): scmType = node.getValue("type") if (scmType == "svn"): svnPath = node.getValue("path") - return SVNCatalogRepository(svnPath) - else if (scmType == "git"): + return svn_catalog_repository.SVNCatalogRepository(svnPath) + elif (scmType == "git"): gitPath = node.getValue("path") usesSubmodules = node.getValue("uses-submodules", False) - return GitCatalogRepository(gitPath, usesSubmodules) - else if (scmType == "git-discrete") - return GitDiscreteSCM(node) - else if (scmType == None): + return git_catalog_repository.GitCatalogRepository(gitPath, usesSubmodules) + elif (scmType == "git-discrete"): + return git_discrete_repository.GitDiscreteSCM(node) + elif (scmType == None): raise RuntimeError("No scm/type defined in catalog configuration") else: raise RuntimeError("Unspported SCM type:" + scmType) - + def processUpload(node, outputPath): + print "Enabled value is:", node.getValue("enabled") + if not node.getValue("enabled", True): + print "Upload disabled" + return + uploadType = node.getValue("type") - if (type == "rsync"): - subprocess.call(["rsync", node.getValue("args", "-az"), ".", + if (uploadType == "rsync"): + subprocess.call(["rsync", node.getValue("args", "-az"), ".", node.getValue("remote")], cwd = outputPath) - else if (type == "scp"): + elif (uploadType == "scp"): subprocess.call(["scp", node.getValue("args", "-r"), outputPath, node.getValue("remote")]) else: raise RuntimeError("Unsupported upload type:" + uploadType) -# dictionary +# dictionary packages = {} +if len(sys.argv) < 2: + raise RuntimeError("no root dir specified") + rootDir = sys.argv[1] -os.path.chdir(rootDir) +if not os.path.isabs(rootDir): + rootDir = os.path.abspath(rootDir) +os.chdir(rootDir) +print "Root path is:", rootDir configPath = 'catalog.config.xml' -if !os.path.exists(configPath): +if not os.path.exists(configPath): raise RuntimeError("no config file found at:" + configPath) - -config = readProps(configPath) + +config = sgprops.readProps(configPath) # out path outPath = config.getValue('output-dir') if outPath is None: # default out path - outPath = "output" - + outPath = os.path.join(rootDir, "output") +elif not os.path.isabs(outPath): + outPath = os.path.join(rootDir, "output") + +if not os.path.exists(outPath): + os.mkdir(outPath) + print "Output path is:" + outPath thumbnailPath = os.path.join(outPath, config.getValue('thumbnail-dir', "thumbnails")) +thumbnailUrl = config.getValue('thumbnail-url') + +mirrorUrls = [] # contains existing catalog existingCatalogPath = os.path.join(outPath, 'catalog.xml') @@ -244,33 +282,48 @@ existingCatalogPath = os.path.join(outPath, 'catalog.xml') scmRepo = initScmRepository(config.getChild('scm')) # scan the directories in the aircraft paths -for g in config.getChildren("aircraft-dir"): - for p in scanPackages(g): +for g in config.getChildren("aircraft-dir"): + for p in scanPackages(g.value): packages[p.id] = p -previousCatalog = readProps(existingCatalogPath) -for p in previousCatalog.getChildren("package"): - pkgId = p.getValue("id") - if !packages.contains(pkgId): - print "Orphaned old package:", pkgId - continue - - packages[pkgId].setPreviousData(p) - +if os.path.exists(existingCatalogPath): + try: + previousCatalog = sgprops.readProps(existingCatalogPath) + except: + print "Previous catalog is malformed" + previousCatalog = sgprops.Node() -catalogNode = sgprops.Node() + for p in previousCatalog.getChildren("package"): + pkgId = p.getValue("id") + if not pkgId in packages.keys(): + print "Orphaned old package:", pkgId + continue + packages[pkgId].setPreviousData(p) +else: + print "No previous catalog" + +catalogNode = sgprops.Node("catalog") sgprops.copy(config.getChild("template"), catalogNode) + packagesToGenerate = [] -for p in packages: +for p in packages.values(): if (p.isSourceModified(scmRepo)): packagesToGenerate.append(p) - + else: + p.useExistingCatalogData() + for p in packagesToGenerate: p.generateZip(outPath) p.extractThumbnails(thumbnailPath) - catalogNode.addChild(p.catalogNode) - + +print "Creating catalog" +for p in packages.values(): + catalogNode.addChild(p.packageNode(mirrorUrls, thumbnailUrl)) + +catalogNode.write(os.path.join(outPath, "catalog.xml")) + +print "Uploading" if config.hasChild("upload"): - processUpload(config.getChild("upload"), outPath) \ No newline at end of file + processUpload(config.getChild("upload"), outPath) diff --git a/sgprops.py b/sgprops.py index b528b4a..bcab4e9 100644 --- a/sgprops.py +++ b/sgprops.py @@ -13,7 +13,7 @@ class Node(object): self._value = None self._index = 0 self._children = [] - + @property def value(self): return self._value @@ -21,21 +21,21 @@ class Node(object): @value.setter def value(self, v): self._value = v - + @property def name(self): return self._name - + @property def index(self): return self._index - - @property + + @property def parent(self): return self._parent - + def getChild(self, n, i=None, create = False): - + if i is None: i = 0 # parse name as foo[999] if necessary @@ -43,18 +43,18 @@ class Node(object): if m is not None: n = m.group(1) i = int(m.group(2)) - + for c in self._children: if (c.name == n) and (c.index == i): return c - + if create: c = Node(n, i, self) self._children.append(c) return c else: raise IndexError("no such child:" + str(n) + " index=" + str(i)) - + def addChild(self, n): # adding an existing instance if isinstance(n, Node): @@ -62,60 +62,64 @@ class Node(object): n._index = self.firstUnusedIndex(n.name) self._children.append(n) return n - - i = self.firstUnusedIndex(n) + + i = self.firstUnusedIndex(n) # create it via getChild return self.getChild(n, i, create=True) - + def firstUnusedIndex(self, n): usedIndices = frozenset(c.index for c in self.getChildren(n)) + i = 0 while i < 1000: if i not in usedIndices: return i i += 1 raise RuntimeException("too many children with name:" + n) - + def hasChild(self, nm): for c in self._children: if (c.name == nm): return True - + return False - + def getChildren(self, n = None): if n is None: return self._children - + return [c for c in self._children if c.name == n] - + def getNode(self, path, cr = False): axes = path.split('/') nd = self for ax in axes: nd = nd.getChild(ax, create = cr) - + return nd - + def getValue(self, path, default = None): try: nd = self.getNode(path) return nd.value except: return default - + def write(self, path): root = self._createXMLElement('PropertyList') - + t = ET.ElementTree(root) - t.write(path, 'utf-8') - + + ET.dump(root) + + t.write(path, 'utf-8', xml_declaration = True) + def _createXMLElement(self, nm = None): if nm is None: nm = self.name - + n = ET.Element(nm) - - # value and type specification + + # value and type specification try: if self._value is not None: if isinstance(self._value, basestring): @@ -133,17 +137,17 @@ class Node(object): n.set('type', "bool") except UnicodeEncodeError: print "Encoding error with", self._value, type(self._value) - + # index in parent if (self.index != 0): - n.set('n', self.index) - + n.set('n', str(self.index)) + # children for c in self._children: n.append(c._createXMLElement()) - + return n; - + class PropsHandler(handler.ContentHandler): def __init__(self, root = None, path = None, dataDirPath = None): @@ -152,50 +156,50 @@ class PropsHandler(handler.ContentHandler): self._basePath = os.path.dirname(path) self._dataDirPath = dataDirPath self._locator = None - + if root is None: # make a nameless root node self._root = Node("", 0) self._current = self._root - + def setDocumentLocator(self, loc): - self._locator = loc - - def startElement(self, name, attrs): - self._content = '' + self._locator = loc + + def startElement(self, name, attrs): + self._content = None if (name == 'PropertyList'): return - + if 'n' in attrs.keys(): index = int(attrs['n']) self._current = self._current.getChild(name, index, create=True) else: self._current = self._current.addChild(name) - - + + if 'include' in attrs.keys(): self.handleInclude(attrs['include']) - + self._currentTy = None; if 'type' in attrs.keys(): self._currentTy = attrs['type'] - + def handleInclude(self, includePath): if includePath.startswith('/'): includePath = includePath[1:] - + p = os.path.join(self._basePath, includePath) if not os.path.exists(p): p = os.path.join(self._dataDirPath, includePath) if not os.path.exists(p): raise RuntimeError("include file not found", includePath, "at line", self._locator.getLineNumber()) - + readProps(p, self._current, self._dataDirPath) - + def endElement(self, name): if (name == 'PropertyList'): return - + try: # convert and store value self._current.value = self._content @@ -207,20 +211,23 @@ class PropsHandler(handler.ContentHandler): self._current.value = float(self._content) except: print "Parse error for value:", self._content, "at line:", self._locator.getLineNumber(), "of:", self._path - + self._current = self._current.parent - + self._content = None + def characters(self, content): + if self._content is None: + self._content = '' self._content += content - + def endDocument(self): pass - + @property def root(self): return self._root - + def readProps(path, root = None, dataDirPath = None): parser = make_parser() locator = expatreader.ExpatLocator( parser ) @@ -229,11 +236,11 @@ def readProps(path, root = None, dataDirPath = None): parser.setContentHandler(h) parser.parse(path) return h.root - + def copy(src, dest): dest.value = src.value - + # recurse over children - for c in src.children: + for c in src.getChildren() : dc = dest.getChild(c.name, i = c.index, create = True) copy(c, dc) diff --git a/svn_catalog_repository.py b/svn_catalog_repository.py index 5e5f56d..7fc11eb 100644 --- a/svn_catalog_repository.py +++ b/svn_catalog_repository.py @@ -7,7 +7,8 @@ class SVNCatalogRepository: self._path = path xml = subprocess.check_output(["svn", "info", "--xml", path]) root = ET.fromstring(xml) - if (root.find("repository/root") == None): + + if (root.find(".//repository/root") == None): raise RuntimeError("Not an SVN repository:" + path) def hasPathChanged(self, path, oldRevision): @@ -16,7 +17,7 @@ class SVNCatalogRepository: def scmRevisionForPath(self, path): xml = subprocess.check_output(["svn", "info", "--xml", path]) root = ET.fromstring(xml) - commit = root.find("entry/commit") + commit = root.find(".//entry/commit") return commit.get('revision', 0) def update(self):