1
0
Fork 0

Move PackageData to its own file.

This commit is contained in:
James Turner 2015-07-25 22:45:01 -05:00
parent 651482a0df
commit 242c0f9172
3 changed files with 216 additions and 212 deletions

View file

@ -1,14 +1,11 @@
#!/usr/bin/python
import os, sys, re, glob
import hashlib # for MD5
import subprocess
import shutil # for copy2
import catalogTags
import sgprops
#from multiprocessing import Pool
import argparse
import urllib2
import package as pkg
import svn_catalog_repository
import git_catalog_repository
@ -19,208 +16,8 @@ parser.add_argument("--clean", help="Regenerate every package", type=bool)
parser.add_argument("dir", help="Catalog directory")
args = parser.parse_args()
standardTagSet = frozenset(catalogTags.tags)
def isNonstandardTag(t):
return t not in standardTagSet
thumbnailNames = ["thumbnail.png", "thumbnail.jpg"]
includePaths = []
class VariantData:
def __init__(self, path, node):
#self._primary = primary
self._path = path
self._name = node.getValue("sim/description")
# ratings
# seperate thumbnails
@property
def catalogNode(self):
n = sgprops.Node("variant")
n.addChild("id").value = self._path
n.addChild("name").value = self._name
class PackageData:
def __init__(self, path):
self._path = path
self._previousSCMRevision = None
self._previousRevision = 0
self._thumbnails = []
self._variants = {}
self._revision = 0
self._md5 = None
self._fileSize = 0
self._node = sgprops.Node("package")
self._node.addChild("id").value = self.id
def setPreviousData(self, node):
self._previousRevision = node.getValue("revision")
self._previousMD5 = node.getValue("md5")
self._previousSCMRevision = node.getValue("scm-revision")
self._fileSize = int(node.getValue("file-size-bytes"))
@property
def id(self):
return os.path.basename(self._path)
@property
def thumbnails(self):
return self._thumbnails
@property
def path(self):
return self._path
@property
def variants(self):
return self._variants
@property
def scmRevision(self):
currentRev = scmRepo.scmRevisionForPath(self._path)
if (currentRev is None):
raise RuntimeError("Unable to query SCM revision of files")
return currentRev
def isSourceModified(self, scmRepo):
if (self._previousSCMRevision == None):
return True
if (self._previousSCMRevision == self.scmRevision):
return False
return True
def scanSetXmlFiles(self, includes):
foundPrimary = False
foundMultiple = False
for f in os.listdir(self._path):
if not f.endswith("-set.xml"):
continue
p = os.path.join(self._path, f)
node = sgprops.readProps(p, includePaths = includes)
if not node.hasChild("sim"):
continue
simNode = node.getChild("sim")
if (simNode.getValue("exclude", False)):
continue
primary = simNode.getValue("variant-of", None)
if primary:
if not primary in self.variants:
self._variants[primary] = []
self._variants[primary].append(VariantData(self, node))
continue
if foundPrimary:
if not foundMultiple:
print "Multiple primary -set.xml files at:" + self._path
foundMultiple = True
continue
else:
foundPrimary = True;
self.parsePrimarySetNode(simNode)
for n in thumbnailNames:
if os.path.exists(os.path.join(self._path, n)):
self._thumbnails.append(n)
if not foundPrimary:
raise RuntimeError("No primary -set.xml found at:" + self._path)
def parsePrimarySetNode(self, sim):
# basic / mandatory values
self._node.addChild('name').value = sim.getValue('description')
longDesc = sim.getValue('long-description')
if longDesc is not None:
self._node.addChild('description').value = longDesc
# copy all the standard values
for p in ['status', 'author', 'license']:
v = sim.getValue(p)
if v is not None:
self._node.addChild(p).value = v
# ratings
if sim.hasChild('rating'):
pkgRatings = self._node.addChild('rating')
for r in ['FDM', 'systems', 'cockpit', 'model']:
pkgRatings.addChild(r).value = sim.getValue('rating/' + r, 0)
# copy tags
if sim.hasChild('tags'):
for c in sim.getChild('tags').getChildren('tag'):
if isNonstandardTag(c.value):
print "Skipping non-standard tag:", c.value, self.path
else:
self._node.addChild('tag').value = c.value
for t in sim.getChildren("thumbnail"):
self._thumbnails.append(t.value)
def validate(self):
for t in self._thumbnails:
if not os.path.exists(os.path.join(self._path, t)):
raise RuntimeError("missing thumbnail:" + t);
def generateZip(self, outDir):
self._revision = self._previousRevision + 1
zipName = self.id + ".zip"
zipFilePath = os.path.join(outDir, zipName)
os.chdir(os.path.dirname(self.path))
print "Creating zip", zipFilePath
# TODO: exclude certain files
# anything we can do to make this faster?
subprocess.call(['zip', '--quiet', '-r', zipFilePath, self.id])
zipFile = open(zipFilePath, 'r')
self._md5 = hashlib.md5(zipFile.read()).hexdigest()
self._fileSize = os.path.getsize(zipFilePath)
def useExistingCatalogData(self):
self._md5 = self._previousMD5
def packageNode(self, mirrorUrls, thumbnailUrl):
self._node.getChild("md5", create = True).value = self._md5
self._node.getChild("file-size-bytes", create = True).value = self._fileSize
self._node.getChild("revision", create = True).value = int(self._revision)
self._node.getChild("scm-revision", create = True).value = self.scmRevision
for m in mirrorUrls:
self._node.addChild("url").value = m + "/" + self.id + ".zip"
for t in self._thumbnails:
self._node.addChild("thumbnail").value = thumbnailUrl + "/" + self.id + "_" + t
for pr in self._variants:
for vr in self._variants[pr]:
self._node.addChild(vr.catalogNode)
return self._node
def extractThumbnails(self, thumbnailDir):
for t in self._thumbnails:
fullName = self.id + "_" + t
shutil.copy2(os.path.join(self._path, t),
os.path.join(thumbnailDir, fullName)
)
# TODO : verify image format, size and so on
def scanPackages(globPath):
result = []
print "Scanning", globPath
@ -231,7 +28,7 @@ def scanPackages(globPath):
print "no -set.xml in", d
continue
result.append(PackageData(d))
result.append(pkg.PackageData(d))
return result
@ -385,7 +182,7 @@ for p in packagesToGenerate:
print "Creating catalog"
for p in packages.values():
catalogNode.addChild(p.packageNode(mirrorUrls, thumbnailUrl))
catalogNode.addChild(p.packageNode(scmRepo, mirrorUrls, thumbnailUrl))
catalogNode.write(os.path.join(outPath, "catalog.xml"))

205
package.py Normal file
View file

@ -0,0 +1,205 @@
import os, subprocess
import sgprops
import hashlib # for MD5
import shutil # for copy2
import catalogTags
standardTagSet = frozenset(catalogTags.tags)
def isNonstandardTag(t):
return t not in standardTagSet
thumbnailNames = ["thumbnail.png", "thumbnail.jpg"]
class VariantData:
def __init__(self, path, node):
#self._primary = primary
self._path = path
self._name = node.getValue("sim/description")
# ratings
# seperate thumbnails
@property
def catalogNode(self):
n = sgprops.Node("variant")
n.addChild("id").value = self._path
n.addChild("name").value = self._name
class PackageData:
def __init__(self, path):
self._path = path
self._previousSCMRevision = None
self._previousRevision = 0
self._thumbnails = []
self._variants = {}
self._revision = 0
self._md5 = None
self._fileSize = 0
self._node = sgprops.Node("package")
self._node.addChild("id").value = self.id
def setPreviousData(self, node):
self._previousRevision = node.getValue("revision")
self._previousMD5 = node.getValue("md5")
self._previousSCMRevision = node.getValue("scm-revision")
self._fileSize = int(node.getValue("file-size-bytes"))
@property
def id(self):
return os.path.basename(self._path)
@property
def thumbnails(self):
return self._thumbnails
@property
def path(self):
return self._path
@property
def variants(self):
return self._variants
def scmRevision(self, repo):
currentRev = repo.scmRevisionForPath(self._path)
if (currentRev is None):
raise RuntimeError("Unable to query SCM revision of files")
return currentRev
def isSourceModified(self, scmRepo):
if (self._previousSCMRevision == None):
return True
if (self._previousSCMRevision == self.scmRevision(scmRepo)):
return False
return True
def scanSetXmlFiles(self, includes):
foundPrimary = False
foundMultiple = False
for f in os.listdir(self._path):
if not f.endswith("-set.xml"):
continue
p = os.path.join(self._path, f)
node = sgprops.readProps(p, includePaths = includes)
if not node.hasChild("sim"):
continue
simNode = node.getChild("sim")
if (simNode.getValue("exclude", False)):
continue
primary = simNode.getValue("variant-of", None)
if primary:
if not primary in self.variants:
self._variants[primary] = []
self._variants[primary].append(VariantData(self, node))
continue
if foundPrimary:
if not foundMultiple:
print "Multiple primary -set.xml files at:" + self._path
foundMultiple = True
continue
else:
foundPrimary = True;
self.parsePrimarySetNode(simNode)
for n in thumbnailNames:
if os.path.exists(os.path.join(self._path, n)):
self._thumbnails.append(n)
if not foundPrimary:
raise RuntimeError("No primary -set.xml found at:" + self._path)
def parsePrimarySetNode(self, sim):
# basic / mandatory values
self._node.addChild('name').value = sim.getValue('description')
longDesc = sim.getValue('long-description')
if longDesc is not None:
self._node.addChild('description').value = longDesc
# copy all the standard values
for p in ['status', 'author', 'license']:
v = sim.getValue(p)
if v is not None:
self._node.addChild(p).value = v
# ratings
if sim.hasChild('rating'):
pkgRatings = self._node.addChild('rating')
for r in ['FDM', 'systems', 'cockpit', 'model']:
pkgRatings.addChild(r).value = sim.getValue('rating/' + r, 0)
# copy tags
if sim.hasChild('tags'):
for c in sim.getChild('tags').getChildren('tag'):
if isNonstandardTag(c.value):
print "Skipping non-standard tag:", c.value, self.path
else:
self._node.addChild('tag').value = c.value
for t in sim.getChildren("thumbnail"):
self._thumbnails.append(t.value)
def validate(self):
for t in self._thumbnails:
if not os.path.exists(os.path.join(self._path, t)):
raise RuntimeError("missing thumbnail:" + t);
def generateZip(self, outDir):
self._revision = self._previousRevision + 1
zipName = self.id + ".zip"
zipFilePath = os.path.join(outDir, zipName)
os.chdir(os.path.dirname(self.path))
print "Creating zip", zipFilePath
# TODO: exclude certain files
# anything we can do to make this faster?
subprocess.call(['zip', '--quiet', '-r', zipFilePath, self.id])
zipFile = open(zipFilePath, 'r')
self._md5 = hashlib.md5(zipFile.read()).hexdigest()
self._fileSize = os.path.getsize(zipFilePath)
def useExistingCatalogData(self):
self._md5 = self._previousMD5
def packageNode(self, repo, mirrorUrls, thumbnailUrl):
self._node.getChild("md5", create = True).value = self._md5
self._node.getChild("file-size-bytes", create = True).value = self._fileSize
self._node.getChild("revision", create = True).value = int(self._revision)
self._node.getChild("scm-revision", create = True).value = self.scmRevision(repo)
for m in mirrorUrls:
self._node.addChild("url").value = m + "/" + self.id + ".zip"
for t in self._thumbnails:
self._node.addChild("thumbnail").value = thumbnailUrl + "/" + self.id + "_" + t
for pr in self._variants:
for vr in self._variants[pr]:
self._node.addChild(vr.catalogNode)
return self._node
def extractThumbnails(self, thumbnailDir):
for t in self._thumbnails:
fullName = self.id + "_" + t
shutil.copy2(os.path.join(self._path, t),
os.path.join(thumbnailDir, fullName)
)
# TODO : verify image format, size and so on

View file

@ -1,25 +1,27 @@
import subprocess
import subprocess, os
import xml.etree.cElementTree as ET
class SVNCatalogRepository:
def __init__(self, path):
if not os.path.exists(path):
raise RuntimeError("No directory at:" + path)
self._path = path
xml = subprocess.check_output(["svn", "info", "--xml", path])
root = ET.fromstring(xml)
if (root.find(".//repository/root") == None):
raise RuntimeError("Not an SVN repository:" + path)
def hasPathChanged(self, path, oldRevision):
return self.scmRevisionForPath(path) != oldRevision
def scmRevisionForPath(self, path):
xml = subprocess.check_output(["svn", "info", "--xml", path])
root = ET.fromstring(xml)
commit = root.find(".//entry/commit")
return commit.get('revision', 0)
def update(self):
subprocess.call(["svn", "update"])