From 8e52a6a247dc9c1e031a7694ede83e6cbf770900 Mon Sep 17 00:00:00 2001
From: James Turner <zakalawe@mac.com>
Date: Thu, 12 Jun 2014 10:34:31 +0100
Subject: [PATCH] Catalog creation tweaks:

- add file-size-bytes
- cache zips locally for MD5 generation
- audit tags against known list
---
 catalogTags.py    | 73 +++++++++++++++++++++++++++++++++++++++++++++++
 create_catalog.py | 41 ++++++++++++++++++++------
 2 files changed, 106 insertions(+), 8 deletions(-)
 create mode 100644 catalogTags.py

diff --git a/catalogTags.py b/catalogTags.py
new file mode 100644
index 0000000..d99fd2d
--- /dev/null
+++ b/catalogTags.py
@@ -0,0 +1,73 @@
+aircraftTypeTags = [
+    "ga", "fighter", "helicopter", "glider", "spaceship", "bomber", "groundvehicle", 
+    "tanker", "cargo", "passenger", "bizjet", "trainer", "airship", "balloon"
+]
+
+manufacturerTags = [
+    "boeing", "cessna", "diamond", "douglas", "bell", "piper", 
+    "airbus", "vickers", "lockheed", "fokker",
+    "embrarer", "bombardier", "pilatus"
+]
+
+eraTags = [
+    "early-pioneers",
+    "ww1",
+    "1920s",
+    "1930s",
+    "golden-age",
+    "ww2",
+    "coldwar", "vietnam",
+    "1950s",
+    "1960s",
+    "1970s",
+    "1980s",
+    "gulfwar1",
+    "gulfwar2"
+]
+
+featureTags = [
+    "ifr",
+    "retractable-gear",
+    "fixed-gear",
+    "tail-dragger",
+    "seaplane",
+    "vtol",
+    "stol",
+    "experimental",
+    "prototype",
+    "fictional",
+    "biplane",
+    "triplane",
+    "supersonic",
+    "t-tail",
+    "v-tail",
+    "high-wing",
+    "cannard",
+    "tail-hook",
+    "refuel",
+    "delta",
+    "variable-geometry",
+    "glass-cockpit",
+    "hud",
+    "etops",
+    "floats",
+    "amphibious",
+    "supersonic"
+]
+
+propulsionTags = [
+    "piston", "radial",
+    "diesel",
+    "variable-pitch",
+    "supercharged",
+    "turboprop",
+    "jet", "afterburner", "rocket",
+    "electric"
+]
+
+simFeatureTags = [
+    "tow",
+    "dual-controls"
+]
+
+tags = aircraftTypeTags + manufacturerTags + eraTags + simFeatureTags + propulsionTags + featureTags
diff --git a/create_catalog.py b/create_catalog.py
index a120608..16598b8 100755
--- a/create_catalog.py
+++ b/create_catalog.py
@@ -1,10 +1,11 @@
 #!/usr/bin/python
 
 import os, sys, re
-import urllib2
+import urllib
 import hashlib # for MD5
 
 import catalogFilenames
+import catalogTags
 import sgprops
 
 fgRoot = sys.argv[1]
@@ -36,6 +37,18 @@ thumbs = [
     "http://www.flightgear.org/thumbs/v3.0/{acft}.jpg"
 ]
     
+standardTagSet = frozenset(catalogTags.tags)
+def isNonstandardTag(t):
+    return t not in standardTagSet
+    
+# create the download cache dir if require
+
+cacheDir = '.catalog_cache'
+if not os.path.isdir(cacheDir):
+    print "Creating catalog cache dir"
+    os.mkdir(cacheDir)
+    
+    
 for d in os.listdir(aircraftDir):
     acftDirPath = os.path.join(aircraftDir, d)
     if not os.path.isdir(acftDirPath):
@@ -83,7 +96,10 @@ for d in os.listdir(aircraftDir):
         # copy tags
         if sim.hasChild('tags'):
             for c in sim.getChild('tags').getChildren('tag'):
-                pkgNode.addChild('tag').value = c.value
+                if isNonstandardTag(c.value):
+                    print "Skipping non-standard tag:", c.value
+                else:
+                    pkgNode.addChild('tag').value = c.value
                 
         # create download and thumbnail URLs
         s = "{url}Aircraft-3.0/"
@@ -93,17 +109,26 @@ for d in os.listdir(aircraftDir):
         s += catalogFilenames.aircraft[d]
         
         for u in urls:
-            pkgNode.addChild("url").value = s.format(url=u,filename=f)
+            pkgNode.addChild("url").value = s.format(url=u)
         
         for t in thumbs:
             pkgNode.addChild("thumbnail").value = t.format(acft=d)
         
-        # download and compute MD5 sum
-        dl = urllib2.urlopen(s.format(url=urls[0],filename=f))
-        digest = hashlib.md5(dl.read()).hexdigest()
-        pkgNode.addChild("md5").value = digest
+        cachedZip = os.path.join(cacheDir, catalogFilenames.aircraft[d])
+        if not os.path.exists(cachedZip):
+            # download the zip
+            url = s.format(url=urls[0])
+            print "Downloading ", url
+            urllib.urlretrieve(url, cachedZip)
+        #else:
+        #    print "Using cached zip for", d
+            
+        zipFile = open(cachedZip, 'r')
         
+        digest = hashlib.md5(zipFile.read()).hexdigest()
+        pkgNode.addChild("md5").value = digest
+        pkgNode.addChild("file-size-bytes").value = os.path.getsize(cachedZip)
     except:
         print "Failure processing:", setFilePath
-
+        
 catalogProps.write("catalog.xml")        
\ No newline at end of file