From b8c05410e35e9e3f40745713999b41eb258df7d1 Mon Sep 17 00:00:00 2001 From: Edward d'Auvergne Date: Wed, 13 Nov 2019 10:59:38 +0100 Subject: [PATCH] Catalogs: Rewrite of the craft zip creation using the Python zipfile module. If a 'zip-excludes.lst' file is found in the craft's base directory, that will overwrite the global catalog exclusion list. Three new unit tests have been created to test the functionality. --- catalog/catalog.py | 109 +++++++++++++++++- .../Aircraft/c150/Resources/crazy_20Gb_file | 0 .../testData/Aircraft/c150/Resources/x.xcf.gz | 0 catalog/testData/Aircraft/c150/gimp.xcf.gz | 0 .../testData/Aircraft/c150/zip-excludes.lst | 1 + catalog/testData/Aircraft/dc3/.svn/entries | 1 + .../dc3/Resources/BluePrints/test.png | 0 .../Aircraft/dc3/Resources/massive_10Gb_file | 0 catalog/testData/Aircraft/dc3/gimp.xcf | 0 catalog/test_catalog.py | 85 ++++++++++++++ catalog/update-catalog.py | 21 +--- 11 files changed, 199 insertions(+), 18 deletions(-) create mode 100644 catalog/testData/Aircraft/c150/Resources/crazy_20Gb_file create mode 100644 catalog/testData/Aircraft/c150/Resources/x.xcf.gz create mode 100644 catalog/testData/Aircraft/c150/gimp.xcf.gz create mode 100644 catalog/testData/Aircraft/c150/zip-excludes.lst create mode 100644 catalog/testData/Aircraft/dc3/.svn/entries create mode 100644 catalog/testData/Aircraft/dc3/Resources/BluePrints/test.png create mode 100644 catalog/testData/Aircraft/dc3/Resources/massive_10Gb_file create mode 100644 catalog/testData/Aircraft/dc3/gimp.xcf diff --git a/catalog/catalog.py b/catalog/catalog.py index c6641ce..2fea611 100644 --- a/catalog/catalog.py +++ b/catalog/catalog.py @@ -2,13 +2,16 @@ import argparse import datetime -#import xml.etree.cElementTree as ET +from fnmatch import fnmatch, translate import lxml.etree as ET import os +from os.path import exists, join, relpath +from os import walk import re import sgprops import sys import catalogTags +import zipfile CATALOG_VERSION = 4 quiet = False @@ -286,3 +289,107 @@ def make_aircraft_node(aircraftDirName, package, variants, downloadBase, mirrors package_node.append(package['urls']._createXMLElement()) return package_node + + +def make_aircraft_zip(repo_path, craft_name, zip_file, global_zip_excludes, verbose=True): + """Create a zip archive of the given aircraft.""" + + # Printout. + if verbose: + print("Zip file creation: %s.zip" % craft_name) + + # Go to the directory of crafts to catalog. + savedir = os.getcwd() + os.chdir(repo_path) + + # Clear out the old file. + if exists(zip_file): + os.remove(zip_file) + + # Use the Python zipfile module to create the zip file. + zip_handle = zipfile.ZipFile(zip_file, 'w', zipfile.ZIP_DEFLATED) + + # Find a per-craft exclude list. + craft_path = join(repo_path, craft_name) + exclude_file = join(craft_path, 'zip-excludes.lst') + if exists(exclude_file): + if verbose: + print("Found the craft specific exclusion list '%s'" % exclude_file) + + # Otherwise use the catalog default exclusion list. + else: + exclude_file = global_zip_excludes + + # Process the exclusion list and find all matching file names. + blacklist = fetch_zip_exclude_list(craft_name, craft_path, exclude_file) + + # Walk over all craft files. + print_format = " %-30s '%s'" + for root, dirs, files in walk(craft_path): + # Loop over the files. + for file in files: + # The directory and relative and absolute paths. + dir = relpath(root, start=repo_path) + full_path = join(root, file) + rel_path = relpath(full_path, start=repo_path) + + # Skip blacklist files or directories. + skip = False + if file == 'zip-excludes.lst': + if verbose: + print(print_format % ("Skipping the file:", join(dir, 'zip-excludes.lst'))) + skip = True + if dir in blacklist: + if verbose: + print(print_format % ("Skipping the file:", join(dir, file))) + skip = True + for name in blacklist: + if fnmatch(rel_path, name): + if verbose: + print(print_format % ("Skipping the file:", rel_path)) + skip = True + break + if skip: + continue + + # Otherwise add the file. + zip_handle.write(rel_path) + + # Clean up. + os.chdir(savedir) + zip_handle.close() + + +def fetch_zip_exclude_list(name, path, exclude_path): + """Use Unix style path regular expression to find all files to exclude.""" + + # Init. + blacklist = [] + file = open(exclude_path) + exclude_list = file.readlines() + file.close() + old_path = os.getcwd() + os.chdir(path) + + # Process each exclusion path or regular expression, converting to Python RE objects. + reobj_list = [] + for i in range(len(exclude_list)): + reobj_list.append(re.compile(translate(exclude_list[i].strip()))) + + # Recursively loop over all files, finding the ones to exclude. + for root, dirs, files in walk(path): + for file in files: + full_path = join(root, file) + rel_path = join(name, relpath(full_path, start=path)) + + # Skip Unix shell-style wildcard matches + for i in range(len(reobj_list)): + if reobj_list[i].match(rel_path): + blacklist.append(rel_path) + break + + # Return to the original path. + os.chdir(old_path) + + # Return the list. + return blacklist diff --git a/catalog/testData/Aircraft/c150/Resources/crazy_20Gb_file b/catalog/testData/Aircraft/c150/Resources/crazy_20Gb_file new file mode 100644 index 0000000..e69de29 diff --git a/catalog/testData/Aircraft/c150/Resources/x.xcf.gz b/catalog/testData/Aircraft/c150/Resources/x.xcf.gz new file mode 100644 index 0000000..e69de29 diff --git a/catalog/testData/Aircraft/c150/gimp.xcf.gz b/catalog/testData/Aircraft/c150/gimp.xcf.gz new file mode 100644 index 0000000..e69de29 diff --git a/catalog/testData/Aircraft/c150/zip-excludes.lst b/catalog/testData/Aircraft/c150/zip-excludes.lst new file mode 100644 index 0000000..e505aae --- /dev/null +++ b/catalog/testData/Aircraft/c150/zip-excludes.lst @@ -0,0 +1 @@ +*.xcf* diff --git a/catalog/testData/Aircraft/dc3/.svn/entries b/catalog/testData/Aircraft/dc3/.svn/entries new file mode 100644 index 0000000..48082f7 --- /dev/null +++ b/catalog/testData/Aircraft/dc3/.svn/entries @@ -0,0 +1 @@ +12 diff --git a/catalog/testData/Aircraft/dc3/Resources/BluePrints/test.png b/catalog/testData/Aircraft/dc3/Resources/BluePrints/test.png new file mode 100644 index 0000000..e69de29 diff --git a/catalog/testData/Aircraft/dc3/Resources/massive_10Gb_file b/catalog/testData/Aircraft/dc3/Resources/massive_10Gb_file new file mode 100644 index 0000000..e69de29 diff --git a/catalog/testData/Aircraft/dc3/gimp.xcf b/catalog/testData/Aircraft/dc3/gimp.xcf new file mode 100644 index 0000000..e69de29 diff --git a/catalog/test_catalog.py b/catalog/test_catalog.py index 0e2fd79..db84f94 100755 --- a/catalog/test_catalog.py +++ b/catalog/test_catalog.py @@ -3,8 +3,13 @@ import unittest import sgprops import os +from os.path import join import catalog import lxml.etree as ET +from shutil import rmtree +from tempfile import mkdtemp +import zipfile + catalog.quiet = True @@ -234,5 +239,85 @@ class UpdateCatalogTests(unittest.TestCase): self.assertFalse(parsedPkgNode.hasChild('minimum-fg-version')); self.assertFalse(parsedPkgNode.hasChild('variant')); + +class ZipTests(unittest.TestCase): + """Specific craft zip file creation tests.""" + + def check_zip(self, file_name, expected_content=None): + """General checks for the zip file.""" + + # Check for file existence. + self.assert_(os.access(file_name, os.F_OK)) + + # Check the contents. + file = zipfile.ZipFile(file_name) + zip_contents = file.namelist() + if len(zip_contents) != len(expected_content): + print("Zip contents:\n %s" % zip_contents) + print("Expected contents:\n %s" % expected_content) + self.assertEqual(len(zip_contents), len(expected_content)) + for i in range(len(zip_contents)): + self.assertEqual(zip_contents[i], expected_content[i]) + + + def setUp(self): + """Common set up for these system tests.""" + + # Store the current directory. + self._cwd = os.getcwd() + + # Create a temporary directory for dumping files. + self.tmpdir = mkdtemp() + + + def tearDown(self): + """Delete temp files.""" + + # Force return to the correct directory. + os.chdir(self._cwd) + + # Remove temporary file (if there is a deletion failure, continue to allow the test suite to survive). + try: + rmtree(self.tmpdir) + except: + pass + + # Remove the variable. + del self.tmpdir + + + def test_zip_creation(self): + """Test the creation of a basic craft zip archive.""" + + # Create a basic zip file. + name = "c172" + catalog.make_aircraft_zip(join(os.getcwd(), "testData/Aircraft"), name, join(self.tmpdir, name+'.zip'), join(os.getcwd(), 'fgaddon-catalog/zip-excludes.lst'), verbose=False) + + # Checks. + self.check_zip(join(self.tmpdir, name+'.zip'), expected_content=['c172/c172-set.xml']) + + + def test_zip_exclusion_global(self): + """Test file exclusion in a craft zip archive using the global catalog exclusion list.""" + + # Create a basic zip file. + name = "dc3" + catalog.make_aircraft_zip(join(os.getcwd(), "testData/Aircraft"), name, join(self.tmpdir, name+'.zip'), join(os.getcwd(), 'fgaddon-catalog/zip-excludes.lst'), verbose=False) + + # Checks. + self.check_zip(join(self.tmpdir, name+'.zip'), expected_content=['dc3/dc3-set.xml']) + + + def test_zip_exclusion_local(self): + """Test file exclusion in a craft zip archive using a local catalog exclusion list.""" + + # Create a basic zip file. + name = "c150" + catalog.make_aircraft_zip(join(os.getcwd(), "testData/Aircraft"), name, join(self.tmpdir, name+'.zip'), join(os.getcwd(), 'testData/Aircraft/c150/zip-excludes.lst'), verbose=False) + + # Checks. + self.check_zip(join(self.tmpdir, name+'.zip'), expected_content=['c150/c150-set.xml', 'c150/Resources/crazy_20Gb_file']) + + if __name__ == '__main__': unittest.main() diff --git a/catalog/update-catalog.py b/catalog/update-catalog.py index bbcf4fb..a263b70 100755 --- a/catalog/update-catalog.py +++ b/catalog/update-catalog.py @@ -13,6 +13,8 @@ import sgprops import sys import catalogTags import catalog +from catalog import make_aircraft_node, make_aircraft_zip + CATALOG_VERSION = 4 @@ -66,21 +68,6 @@ def scan_dir_for_change_date_mtime(path): maxsec = mtime return maxsec -def make_aircraft_zip(repo_path, name, zip_file): - if (not args.quiet): - print "Updating:", name + '.zip' - savedir = os.getcwd() - os.chdir(repo_path) - if os.path.exists(zip_file): - os.remove(zip_file) - command = ['zip', '-rq', '-9'] - if os.path.exists(zip_excludes): - command += ['-x@' + zip_excludes] - else: - print "warning: no zip-excludes.lst file provided", zip_excludes - command += [zip_file, name] - subprocess.call(command) - os.chdir(savedir) def get_md5sum(file): f = open(file, 'r') @@ -148,7 +135,7 @@ def process_aircraft_dir(name, repo_path): if not args.quiet: print "%s:" % name, - package_node = catalog.make_aircraft_node(name, package, variants, download_base, mirrors) + package_node = make_aircraft_node(name, package, variants, download_base, mirrors) download_url = download_base + name + '.zip' if 'thumbnail' in package: @@ -177,7 +164,7 @@ def process_aircraft_dir(name, repo_path): # rebuild zip file if not args.quiet: print "updating:", zipfile - make_aircraft_zip(repo_path, name, zipfile) + make_aircraft_zip(repo_path, name, zipfile, zip_excludes, verbose=not args.quiet) md5sum = get_md5sum(zipfile) else: if not args.quiet: