fgdata_checkers: various improvements

create_reduced_fgdata: add parallelism,
add options to use or overwrite existing .dds,
don't include alpha when not needed
fgdata_size: output more statistics, improve formatting
find_unused_textures: add used_dds comparison strip

fgdata_checkers.py

@@ -8,17 +8,20 @@ import os.path
 import re
 from collections import defaultdict
 import subprocess
+import multiprocessing
 import math
 import tarfile
+import tempfile
 import gzip
 import shutil
+import time
 try:
     devnull=subprocess.DEVNULL#hide annoying nvcompress messages
 except (AttributeError,NameError):#pre-3.3 Python
     devnull=None
 def path_join(*args):
-    """Unlike plain os.path.join, this doesn't add a trailing / if the last component is empty"""
+    """Unlike plain os.path.join, this always uses forward slashes, and doesn't add a trailing / if the last component is empty"""
-    return os.path.normpath(os.path.join(*args))
+    return os.path.normpath(os.path.join(*args)).replace('\\','/')
 def rfilelist(path,exclude_dirs=[]):
     """Dict of files/sizes in path, including those in any subdirectories (as relative paths)"""
     files=defaultdict(int)
@@ -72,16 +75,16 @@ def files_used(pattern,path,exclude_dirs=[],filelist=None,filetypes=None,relativ
             tex=texfind.search(line)
             if tex:
                 if relative_path:
-                    textures.append(os.path.normpath(path_join(os.path.dirname(file),tex.group(1).replace('\\','/'))))
+                    textures.append(os.path.normpath(path_join(os.path.dirname(file),tex.group(1).replace('\\','/'))).replace('\\','/'))
                 else:
-                    textures.append(os.path.normpath(tex.group(1).replace('\\','/')))
+                    textures.append(os.path.normpath(tex.group(1).replace('\\','/')).replace('\\','/'))
     return textures
 def find_unused_textures(basedir,output_lists=True,grep_check=False,output_rsync_rules=False,output_comparison_strips=False, output_removal_commands=False,return_used_noregions=False):
     """Checks if any textures are unused (wasting space), and if any textures are only available as .dds (not recommended in the source repository, as it is a lossy-compressed format)
 
 Set basedir to your fg-root, and enable the kind(s) of output you want:
 output_lists prints lists of unused textures, and of dds-only textures
-grep_check checks for possible use outside the normal directories; requires Unix shell
+grep_check checks for possible use outside the normal directories; requires Unix shell and assumes side-by-side fgdata,flightgear,simgear
 output_rsync_rules prints rsync rules for excluding unused textures from the release flightgear-data.  Warning: if you use this, re-run this script regularly, in case they start being used
 output_comparison_strips creates thumbnail strips, unused_duplicate.png/unused_dds.png/high_low.png, for visually checking whether same-name textures are the same (remove the unused one entirely) or different (move it to Unused); requires imagemagick or graphicsmagick
 output_removal_commands creates another script, delete_unused_textures.sh, which will remove unused textures when run in a Unix shell"""
@@ -124,6 +127,9 @@ output_removal_commands creates another script, delete_unused_textures.sh, which
     low_unneeded_nondup=low_unneeded&set(known_highlow_mismatch)
     def image_check_strip(basedir,index_fname,ilist1,ilist2=None,size=128):
         """Generate two rows of thumbnails, for easy visual comparison (between the two lists given, or if a single list is given, between low and high resolution)"""
+        if not ilist1:
+            print(index_fname," empty, skipping")
+            return
         if ilist2 is None:
             ipairs=[[path_join(basedir,'Textures',f),path_join(basedir,'Textures.high',f)] for f in ilist1]
         else:
@@ -169,6 +175,12 @@ output_removal_commands creates another script, delete_unused_textures.sh, which
     if output_comparison_strips:
         image_check_strip(basedir,"unused_duplicate.png",unused_duplicate,["Terrain"+f[14:] for f in unused_duplicate])
         image_check_strip(basedir,"unused_dds.png",unused_dds,[f[:-4]+".png" for f in unused_dds])
+        dds_skip=set(['Runway/rwy-normalmap.dds','Water/perlin-noise-nm.dds','Water/water_sine_nmap.dds','Water/waves-ver10-nm.dds'])
+        used_dds_withpng=set(f for f in (high_textures|low_textures) if (f[-4:]==".dds" and f[:-4]+".png" in (high_textures|low_textures)))-unused_dds-dds_skip
+        print(".dds omitted from comparison strip (normal maps etc): ",dds_skip)
+        #used_dds_withpng different: p{a,c}_taxiway.dds and possibly more runway markings (darker),sand{4,5,6}.dds(less high-freq noise),water.dds(more high-freq noise),grass_rwy.dds (has stripes),cropwood.dds,irrcrop.dds,shrub.dds,herbtundra.dds,cropgrass.dds(unrelated)
+        #water-reflection.{png,dds} really are plain white (so do match), not an alpha map
+        image_check_strip(basedir,"used_dds.png",used_dds_withpng,[f[:-4]+".png" for f in used_dds_withpng])
         image_check_strip(basedir,"high_low.png",high_textures&low_textures)
         #image_check_strip(basedir,"high_low2.png",[f for f in high_textures&low_textures if (f[0:14]=="Terrain.winter" or "_taxiway." in f or "lava" in f or "sand" in f)],size=512)#closer look at the doubtful cases
     if output_lists:
@@ -178,11 +190,11 @@ output_removal_commands creates another script, delete_unused_textures.sh, which
         unused_f=[os.path.basename(f) for f in unused]
         all_f=[os.path.basename(f) for f in (high_textures|low_textures)]
         print("\n\nPossible use outside main search:")#used to set false_positives
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(unused)+")","/home/rnpalmer/fs_dev/git/fgdata","/home/rnpalmer/fs_dev/git/flightgear","/home/rnpalmer/fs_dev/git/simgear"])#everywhere using full names
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(unused)+")",basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])#everywhere using full names
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e","("+")|(".join(all_f)+")","/home/rnpalmer/fs_dev/git/fgdata","/home/rnpalmer/fs_dev/git/flightgear","/home/rnpalmer/fs_dev/git/simgear"])#restricted (to avoid false positives from Terrain.winter vs Terrain) using filenames
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e","("+")|(".join(all_f)+")",basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])#restricted (to avoid false positives from Terrain.winter vs Terrain) using filenames
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e",'[."\']dds',"/home/rnpalmer/fs_dev/git/fgdata","/home/rnpalmer/fs_dev/git/flightgear","/home/rnpalmer/fs_dev/git/simgear"])#check for programmatic .png -> .dds swap; none found
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e",'[."\']dds',basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])#check for programmatic .png -> .dds swap; none found
         print("\n\nUse of sourceless textures:")
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(sourceless)+")","/home/rnpalmer/fs_dev/git/fgdata","/home/rnpalmer/fs_dev/git/flightgear","/home/rnpalmer/fs_dev/git/simgear"])
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(sourceless)+")",basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])
     if output_rsync_rules:
         print("\n\nFull flightgear-data:\n")
         rsync_rules(basedir,unused)
@@ -247,51 +259,168 @@ def find_locally_unused_models(basedir):
     missing=set(f for f in (used_models|used_textures) if ((f.startswith('Models') and f not in models_allfiles.keys()) or not os.path.isfile(path_join(basedir,f))))
     print("used\n",sorted(used_models),"\nsize=",t_size(used_models),"\n\n",sorted(used_textures),"\nsize=",t_size(used_textures),"\n\nunused\n",sorted(unused),"\nsize=",t_size(unused),"\n\nmissing\n",sorted(missing),"\nsize=",t_size(missing))
 
-def size_by_type(path,exclude_dirs=[]):
+class FilesetSizes:
-    """Dict of total file size by file extension"""
+    def __init__(self):
+        self.count=0
+        self.ncsize=0
+        self.csize=0
+    @property
+    def size(self):
+        if self.csize>0:
+            return self.csize
+        else:
+            return self.ncsize
+    def __str__(self):
+        if self.csize>0:
+            return "{0:10,} {1:15,} {2:15,}".format(self.count,self.ncsize,self.csize)
+        else:
+            return "{0:10,} {1:15,}".format(self.count,self.ncsize)
+    def __add__(self,other):
+        result=FilesetSizes()
+        result.count=self.count+other.count
+        result.ncsize=self.ncsize+other.ncsize
+        result.csize=self.csize+other.csize
+        return result
+def size_by_category(path,exclude_dirs,keyfn,compressed_size=False):
+    """Total size of files, in each category returned from keyfn"""
     files=rfilelist(path,exclude_dirs)
-    size_totals=defaultdict(int)
+    result=defaultdict(FilesetSizes)
-    for filename,size in files.items():
+    gzdir=tempfile.TemporaryDirectory()
-        file_ext=os.path.splitext(filename)[1]
+    gzpath=gzdir.name
-        if file_ext==".gz":
+    gzcount=0
-            file_ext=os.path.splitext(os.path.splitext(filename)[0])[1]+file_ext
+    for file,size in files.items():
-        size_totals[file_ext]=size_totals[file_ext]+size
+        cat=keyfn(file,size)
-    return size_totals
+        if cat is None:
-def size_by_size(path,exclude_dirs=[],exts=[".png",".dds",".rgb"]):
+            continue
-    """Dict of total file size by individual file size range, of given extensions (empty list for all files)"""
+        result[cat].count=result[cat].count+1
-    files=rfilelist(path,exclude_dirs)
+        result[cat].ncsize=result[cat].ncsize+size
-    size_totals=defaultdict(int)
+        if compressed_size:
-    for filename,size in files.items():
+            try:
-        file_ext=os.path.splitext(filename)[1]
+                result[cat].targz.add(path_join(path,file))
-        if (not exts) or (file_ext in exts):
+            except AttributeError:
-            size_totals[2**math.frexp(size)[1]]=size_totals[2**math.frexp(size)[1]]+size
+                result[cat].targzname=path_join(gzpath,str(gzcount)+".tar.gz")
-    return size_totals
+                result[cat].targz=tarfile.open(result[cat].targzname,mode="w:gz")
-def fgdata_size(path,dirs_to_list=["AI/Aircraft","AI/Traffic","Aircraft","Models","Scenery","Textures","Textures.high"],exclude_dirs=None,compressed_size=False,num_types=3):
+                gzcount=gzcount+1
-    if dirs_to_list is None:
+                result[cat].targz.add(path_join(path,file))
-        dirs_to_list=[d for d in os.listdir(path) if os.path.isdir(path_join(path,d))]
+    if compressed_size:
+        for cat in result:
+            result[cat].targz.close()
+            result[cat].csize=os.path.getsize(result[cat].targzname)
+    return result
+def fgdata_size(path,by_size=False,exclude_dirs=None,compressed_size=False,min_size=1e6,include_aircraft=['UIUC','777','777-200','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'],sections={"Aircraft/c172p":"Aircraft-base","Aircraft/Generic":"Aircraft-base","Aircraft/Instruments":"Aircraft-base","Aircraft/Instruments-3d":"Aircraft-base","Aircraft/ufo":"Aircraft-base","Aircraft/777":"Aircraft-777","Aircraft/777-200":"Aircraft-777","Aircraft":"Aircraft-other","Textures":"Textures","Textures.high":"Textures","AI/Aircraft":"AI-aircraft","AI":"AI-other","Scenery":"scenery","Models":"models"}):
+    if any('\\' in s for s in sections):
+        raise ValueError("sections always uses forward slashes")
+    if "" not in sections:
+        sections[""]="Other"
     if exclude_dirs is None:
         if os.path.exists(path_join(path,".git")):
-            exclude_dirs=[".git","Aircraft"]
+            exclude_dirs=[".git","Textures/Unused"]+[path_join("Aircraft",d) for d in os.listdir(path_join(path,"Aircraft")) if d not in ["c172p","ufo","Generic","Instruments","Instruments-3d"]+include_aircraft]
         else:
             exclude_dirs=[]
-    total_compressed_size=0
+    if by_size:
-    exclude_list=[[]]*len(dirs_to_list)+[dirs_to_list+exclude_dirs]+[exclude_dirs]
+        keyfn=lambda file,size: ("texture",math.frexp(size)[1]) if os.path.splitext(file)[1] in [".png",".dds",".rgb",".jpg"] else ("nontexture",math.frexp(size)[1])
-    names_list=dirs_to_list+["other","all"]
+    else:
-    for n,dir1 in enumerate(dirs_to_list+["",""]):
+        def keyfn(file,size):
-        size_totals=size_by_type(path_join(path,dir1),exclude_list[n])
+            file_ext=os.path.splitext(file)[1]
-        print(names_list[n],sorted(size_totals.items(),key=lambda x:-x[1])[:num_types],"total",sum(size_totals.values()))
+            if file_ext==".gz":
-        if compressed_size:
+                file_ext=os.path.splitext(os.path.splitext(file)[0])[1]+file_ext
-            if names_list[n]=="all":
+            file0=file.replace('\\','/')
-                print("compressed size",total_compressed_size)
+            section=sections[max((s for s in sections if file0.startswith(s)),key=len)]
-                continue
+            return (section,file_ext)
-            targz=tarfile.open("fgdata_sizetest_temp.tar.gz",mode="w:gz")
+    result=size_by_category(path,exclude_dirs,keyfn,compressed_size)
-            for file in rfilelist(path_join(path,dir1),exclude_list[n]):
+    #print(result)
-                targz.add(path_join(path,dir1,file))
+    totals1=defaultdict(FilesetSizes)
-            targz.close()
+    totals2=defaultdict(FilesetSizes)
-            print("compressed size",os.path.getsize("fgdata_sizetest_temp.tar.gz"))
+    other=defaultdict(FilesetSizes)
-            total_compressed_size=total_compressed_size+os.path.getsize("fgdata_sizetest_temp.tar.gz")
+    for cat in result:
+        totals1[cat[0]]=totals1[cat[0]]+result[cat]
+        totals2[cat[1]]=totals2[cat[1]]+result[cat]
+        if (not by_size) and (result[cat].size<min_size):
+            other[cat[0]]=other[cat[0]]+result[cat]
+    print("{0:>10} {1:>15} {2:>15}".format("Count","Size","Compr.size"))
+    for cat0,r0 in sorted(totals1.items(),key=lambda p:-p[1].size):
+        print(cat0)
+        for cat,r in sorted([(c[1],r) for (c,r) in result.items() if (c[0]==cat0 and (by_size or r.size>min_size))],key=(lambda p:p[0]) if by_size else (lambda p:-p[1].size)):
+            print(r,cat)
+        if not by_size:
+            print(other[cat0],"other")
+        print(totals1[cat0],"all")
+    print("All")
+    for cat,r in sorted([(c,r) for (c,r) in totals2.items() if (by_size or r.size>min_size)],key=(lambda p:p[0]) if by_size else (lambda p:-p[1].size)):
+        print(r,cat)
+    if not by_size:
+        print(sum((r for r in totals2.values() if r.size<min_size),FilesetSizes()),"other")
+    print(sum(totals1.values(),FilesetSizes()),"all")
+def create_reduced_file(input_path,output_path,temp_path,cdir,file,downsample_this,compress_this,compress_names_find, compressed_format,texture_filetypes,binary_types,fclass):
+    """Process a single file in create_reduced_fgdata.  (Separate function to allow parallel processing)"""
+    retcode=0
+    if downsample_this or compress_this:
+        image_type=texture_filetypes[os.path.splitext(file)[1]]
+        output_image_type=compressed_format if compress_this else os.path.splitext(file)[1]
+        output_file=os.path.splitext(file)[0]+output_image_type
+        output_image_type=texture_filetypes[output_image_type]
+        if "{0}" in output_path and fclass=="base-textures":#downsampled in base-textures, full resolution in extra-textures
+            shutil.copy(path_join(input_path,cdir,file),path_join(output_path.format("extra-textures"),cdir,file))
+        if output_image_type=="DDS":# in Ubuntu, neither imagemagick nor graphicsmagick can write .dds
+            #doesn't work (for dds -> smaller dds): subprocess.call(["nvzoom","-s","0.5","-f","box",path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file)])
+            if subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file)]+(["-flip"] if ((image_type=="DDS")!=(output_image_type=="DDS")) else [])+(["-sample","50%"] if downsample_this else [])+[path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")]):#fails on some DDS formats, so just copy them
+                shutil.copy(path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file))
+                print(path_join(cdir,file),"unsupported type, probably normal map")
+                if compress_this:
+                    raise TypeError#copy will have the wrong name
+            else:
+                try:
+                    image_properties=subprocess.check_output(["identify","-verbose",path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")])
+                except subprocess.CalledProcessError as err:
+                    print("identify error on",path_join(cdir,file),"after ",["convert",image_type+":"+path_join(input_path,cdir,file)]+(["-flip"] if ((image_type=="DDS")!=(output_image_type=="DDS")) else [])+(["-sample","50%"] if downsample_this else [])+[path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")],["identify","-verbose",path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")],err)
+                    raise
+                has_alpha=b"Alpha" in image_properties
+                needs_alpha=has_alpha
+                if has_alpha and re.search(rb"Alpha:\s*min: 255 \(1\)",image_properties):
+                    print(path_join(cdir,file),"has always-255 alpha")
+                    needs_alpha=False
+                retcode=subprocess.call(["nvcompress","-bc3" if needs_alpha else "-bc1",path_join(temp_path,cdir,os.path.splitext(file)[0]+".png"),path_join(output_path.format(fclass),cdir,output_file)],stdout=devnull)
+        else:
+            retcode=subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file)]+(["-sample","50%"] if downsample_this else [])+[output_image_type+":"+path_join(output_path.format(fclass),cdir,output_file)])#we use sample rather than an averaging filter to not break mask/rotation/... maps
+    else:#not to be downsampled/compressed
+        if os.path.splitext(file)[1] in binary_types:#just copy
+            shutil.copy(path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file))
+        else:#texture name replacement
+            file_in=open(path_join(input_path,cdir,file),'rb')
+            file_out=open(path_join(output_path.format(fclass),cdir,file),'wb')
+            file_str=file_in.read(None)
+            file_in.close()
+            (file_strout,num_matches)=compress_names_find.subn(lambda mf: os.path.splitext(mf.group(0))[0]+(compressed_format.encode('utf-8')),file_str)
+            file_out.write(file_strout)
+            file_out.close()
+            #if ((os.path.splitext(file)[1] not in textureuser_types) and num_matches>0):
+                #print("Warning: ",num_matches," unexpected use(s) in ",path_join(cdir,file))
+            #if compress_names_find0.search(file_strout):
+                #print("Warning: unreplaced match(es) in ",path_join(cdir,file),compress_names_find0.search(file_strout).group(0))
+            """Warning: unreplaced match(es) in... correct rejections of match within a filename:
+                Aircraft/Instruments-3d/AN-APS-13.ac b'panel.png'
+                Aircraft/Instruments-3d/magneto-switch/mag_switch.ac b'black.png'
+                Nasal/canvas/map/Images/chart_symbols.svg b'wash.png'
+                Models/Airport/blast-deflector49m.ac b'generic.png'
+                Models/Airport/blast-deflector63m.ac b'generic.png'
+                Models/Industrial/oilrig09.ac b'yellow.png'
+                Models/Industrial/oilrig10.ac b'yellow.png'
+                Models/Industrial/oilrig09.ac.before-color-change b'yellow.png'
+                Models/Industrial/oilrig10.ac.before-color-change b'yellow.png'
+                Models/Maritime/Civilian/Tanker.ac b'black.png'
+                Models/Transport/flatcar.xml b'evergreen.png'
+                Models/Commercial/tower-grey-black.ac b'black.png'
+                Materials/base/materials-base.xml b'yellow.png'
+                
+                Warning: unexpected use(s) in...
+                Docs/README.local_weather.html (the only one that looke like an actual problem; hence, Docs is now skipped)
+                Nasal/canvas/map/Images/chart_symbols.svg (probably inkscape:export-filename, which are creator-specific absolute paths anyway, but now skipped)
+                oilrig09.ac.before-color-change,oilrig10.ac.before-color-change,stbd_coaming_panel.ac.bak (presumably backup files)
+                """
+    if retcode:
+        print("Error ",retcode," in ",path_join(cdir,file))
 
-def create_reduced_fgdata(input_path,output_path,split_textures=True,exclude_parts=[],include_aircraft=['UIUC','777','777-200','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'],dirs_to_downsample=(),dirs_to_compress=(),compressed_format=".dds",downsample_min_filesize=30000):
+def create_reduced_fgdata(input_path,output_path,reject_positional_args=None,split_textures=False,exclude_parts=[],include_aircraft=['UIUC','777','777-200','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'],dirs_to_downsample=(),dirs_to_compress=(),compressed_format=".dds",downsample_min_filesize=1e5,compress_min_filesize=3e4,use_ready_compressed=True):
     """Create a smaller, reduced-quality flightgear-data package
 Can downsample textures 50%, change texture format, and/or omit sections (region-specific textures, aircraft, AI traffic)
 Downsampling and format change require imagemagick or graphicsmagick (for convert) and libnvtt-bin (for nvcompress)
@@ -303,7 +432,8 @@ extra-textures (requires split_textures=True): no region-specific textures
 The c172p and ufo are always included; other aircraft are added by include_aircraft
 
 Texture downsampling: textures in dirs_to_downsample and larger than downsample_min_filesize downsampled 50%
-Texture format conversion: textures in dirs_to_compress and larger than downsample_min_filesize converted to compressed_format
+Texture format conversion: textures in dirs_to_compress and larger than compress_min_filesize converted to compressed_format
+use_ready_compressed determines what happens if a same-basename file in compressed_format already exists: True uses the already-compressed one, False uses the uncompressed one, None keeps both
 Suggested dirs_to_downsample:
 3.2: ('Textures.high/Terrain','Textures.high/Trees','Textures.high/Terrain.winter','AI/Aircraft','Models')
 3.3: ('Textures/Terrain','Textures/Trees','Textures/Terrain.winter','AI/Aircraft','Models')
@@ -313,7 +443,14 @@ To put each section in its own directory (e.g. for building a Debian-style fligh
 python3 -c "import fgdata_checkers; fgdata_checkers.create_reduced_fgdata(input_path='/home/rnpalmer/fs_dev/git/fgdata',output_path='/home/rnpalmer/fs_dev/flightgear/data_split/debian/flightgear-data-{0}/usr/share/games/flightgear',include_aircraft=['UIUC','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'])"
 This creates separate preferences-regions.xml and preferences-noregions.xml files for with and without regional textures; you need to handle symlinking preferences.xml to the correct one
 """
+    start_time=time.time()
+    if reject_positional_args is not None:
+        raise TypeError("Keyword arguments only please: this is not a stable API")
+    if use_ready_compressed not in (True,False,None):
+        raise TypeError("invalid use_ready_compressed setting")
     texture_filetypes={".png":"PNG",".dds":"DDS",".jpg":"JPEG"}#,".rgb":"SGI" loses cloud transparency
+    if compressed_format not in texture_filetypes:
+        raise ValueError("Invalid compressed_format (include the .)")
     textureuser_types={".eff",".xml",".ac",".nas"}
     binary_types={".png",".dds",".rgb",".RGB",".jpg",".wav",".WAV",".btg.gz",".zip",".tar.gz"}#don't search these for texture name replacement
     """Textures named directly in flightgear/simgear code:
@@ -326,7 +463,7 @@ unknown.rgb (probably Textures/ or Textures/Terrain/, neither exists)
 Aircraft/Instruments/Textures/nd-symbols.png (doesn't actually exist),Aircraft/Instruments/Textures/compass-ribbon.rgb,Aircraft/Instruments/Textures/od_wxradar.rgb,Aircraft/Instruments/Textures/od_wxradar.rgb,Aircraft/Instruments/Textures/wxecho.rgb,Aircraft/Instruments/Textures/od_groundradar.rgb (doesn't actually exist)
 also, Aircraft/{Instruments,Instruments-3d,Generic} may be used by downloaded aircraft, and Docs images are used in .html
 Nasal (Canvas map) probably wouldn't break anything, but guessing it's a bad idea visually"""
-    no_compress_pattern=re.compile(r'mask|light|relief|nmap|nm\.|normal|^Splash[0-9].png$|^buildings.png$|^buildings-lightmap.png$|^world.topo.bathy.200407.3x4096x2048.png$')#edge blurring from lossy compression may break masks, and this script doesn't know how to create DDS normal maps
+    no_compress_pattern=re.compile(r'mask|light|relief|nmap|nm\.|normal|dudv|^Splash[0-9].png$|^buildings.png$|^buildings-lightmap.png$|^world.topo.bathy.200407.3x4096x2048.png$')#edge blurring from lossy compression may break masks, and this script doesn't know how to create DDS normal maps
     no_compress_dirs=("gui","Docs","webgui","Nasal","Textures/Sky","Aircraft/Instruments","Aircraft/Instruments-3d","Aircraft/Generic")
     exclude_dirs=[".git","Textures/Unused"]
     exclude_unnamed_subdirs=["Aircraft"]#these are a separate mechanism from subtree_class/exclude_parts mostly to save time (subtree_class still fully scans excluded directories because the class may change again further down the tree, e.g. AI/Aircraft ai -> performancedb.xml base; these don't)
@@ -359,17 +496,21 @@ Nasal (Canvas map) probably wouldn't break anything, but guessing it's a bad ide
                     if (path_join(cdir,file) not in exclude_dirs) and (cdir not in exclude_unnamed_subdirs or path_join(cdir,file) in subtree_class):
                         dirs[path_join(cdir,file)]=fclass
                 else:#file
-                    compress_this=cdir.startswith(dirs_to_compress) and (os.path.splitext(file)[1] in texture_filetypes) and (os.path.getsize(path_join(input_path,cdir,file))>downsample_min_filesize) and not no_compress_pattern.search(file) and not cdir.startswith(no_compress_dirs) and (file not in no_compress_names) and (not os.path.exists(path_join(input_path,cdir,os.path.splitext(file)[0]+compressed_format)))
+                    compress_this=cdir.startswith(dirs_to_compress) and (os.path.splitext(file)[1] in texture_filetypes) and (os.path.splitext(file)[1]!=compressed_format) and (os.path.getsize(path_join(input_path,cdir,file))>compress_min_filesize) and not no_compress_pattern.search(file) and not cdir.startswith(no_compress_dirs) and (file not in no_compress_names) and ((use_ready_compressed is not None) or not os.path.exists(path_join(input_path,cdir,os.path.splitext(file)[0]+compressed_format)))
                     if compress_this:
                         compress_names.add(file)
                     else:
                         no_compress_names.add(file)
                         compress_names.discard(file)#if there are two with the same name in different directories, compress both or neither, to simplify name replacement
     compress_names_find=re.compile(('(?<=["\'>/\\\\ \\n])('+'|'.join(re.escape(f) for f in compress_names)+')($|(?=["\'< \\n]))').encode('utf-8'))
-    compress_names_replace=lambda mf: os.path.splitext(mf.group(0))[0]+(compressed_format.encode('utf-8'))
     compress_names_find0=re.compile(('|'.join(re.escape(f) for f in compress_names)).encode('utf-8'))
     #print(compress_names,"\n\n",no_compress_names,"\n\n",'(?<=["\'>/\\\\ \\n])('+'|'.join(re.escape(f) for f in compress_names)+')($|(?=["\'< \\n]))',"\n\n",'|'.join(re.escape(f) for f in compress_names),"\n\n")
+    print("Starting conversion...",len(compress_names),"files to change format, runtime so far=",int(time.time()-start_time),"sec")
     dirs={"":"base"}
+    subprocess_pool=multiprocessing.Pool(processes=8)
+    subprocess_list=[]
+    temp_dir=tempfile.TemporaryDirectory()
+    temp_path=temp_dir.name
     while dirs:
         cdir,cclass=dirs.popitem()
         cdirfiles=os.listdir(path_join(input_path,cdir))
@@ -387,71 +528,29 @@ Nasal (Canvas map) probably wouldn't break anything, but guessing it's a bad ide
                 if fclass in exclude_parts:
                     continue
                 if not os.path.exists(path_join(output_path.format(fclass),cdir)):
-                    os.mkdirs(path_join(output_path.format(fclass),cdir))
+                    os.makedirs(path_join(output_path.format(fclass),cdir))#errors out if the directory does exist, so calling it in the per-file subprocess would be a race condition
+                if "{0}" in output_path and fclass=="base-textures" and not os.path.exists(path_join(output_path.format("extra-textures"),cdir)):#downsampled in base-textures, full resolution in extra-textures
+                    os.makedirs(path_join(output_path.format("extra-textures"),cdir))
+                if not os.path.exists(path_join(temp_path,cdir)):
+                    os.makedirs(path_join(temp_path,cdir))
                 downsample_this=(cdir.startswith(dirs_to_downsample)) and (os.path.splitext(file)[1] in texture_filetypes) and (os.path.getsize(path_join(input_path,cdir,file))>downsample_min_filesize)
                 compress_this=(file in compress_names)
-                if downsample_this or compress_this:
+                if compress_this and use_ready_compressed==True and os.path.exists(path_join(input_path,cdir,os.path.splitext(file)[0]+compressed_format)):
-                    image_type=texture_filetypes[os.path.splitext(file)[1]]
+                    continue
-                    output_image_type=compressed_format if compress_this else os.path.splitext(file)[1]
+                if (not compress_this) and use_ready_compressed==False and os.path.splitext(file)[1]==compressed_format and any((os.path.exists(path_join(input_path,cdir,os.path.splitext(file)[0]+f)) and os.path.splitext(file)[0]+f in compress_names) for f in texture_filetypes if f!=os.path.splitext(file)[1]):
-                    output_file=os.path.splitext(file)[0]+output_image_type
+                    continue
-                    output_image_type=texture_filetypes[output_image_type]
+                subprocess_list.append(subprocess_pool.apply_async(create_reduced_file,args=(input_path,output_path,temp_path,cdir,file,downsample_this,compress_this,compress_names_find, compressed_format,texture_filetypes,binary_types,fclass)))
-                    if "{0}" in output_path and fclass=="base-textures":#downsampled in base-textures, full resolution in extra-textures
+    print(len(subprocess_list),"file tasks started...runtime so far=",int(time.time()-start_time),"sec\n(Not a linear progress indicator: they are different lengths.)")
-                        if not os.path.exists(path_join(output_path.format("extra-textures"),cdir)):
+    last_report=time.time()-100#print first report immediately
-                            os.mkdirs(path_join(output_path.format("extra-textures"),cdir))
+    for s0 in subprocess_list:
-                        shutil.copy(path_join(input_path,cdir,file),path_join(output_path.format("extra-textures"),cdir,file))
+        if time.time()>last_report+60:
-                    if output_image_type=="DDS":# in Ubuntu, neither imagemagick nor graphicsmagick can write .dds
+            print("Waiting for",len([s for s in subprocess_list if not s.ready()]),"file tasks...runtime so far=",int(time.time()-start_time),"sec")
-                        #doesn't work subprocess.call(["nvzoom","-s","0.5","-f","box",path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file)])
+            last_report=time.time()
-                        if subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file)]+(["-flip"] if ((image_type=="DDS")!=(output_image_type=="DDS")) else [])+(["-sample","50%"] if downsample_this else [])+["temp_reduced_size.png"]):#fails on DDS normal maps, so just copy them
+        s0.get()
-                            retcode=None
+    subprocess_pool.close()
-                            shutil.copy(path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file))
+    subprocess_pool.join()
-                            if compress_this:
-                                print("Error:")
-                            print(path_join(cdir,file)," probably normal map")
-                        else:
-                            retcode=subprocess.call(["nvcompress","-bc3","temp_reduced_size.png",path_join(output_path.format(fclass),cdir,output_file)],stdout=devnull)
-                    else:
-                        retcode=subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file)]+(["-sample","50%"] if downsample_this else [])+[output_image_type+":"+path_join(output_path.format(fclass),cdir,output_file)])#we use sample rather than an averaging filter to not break mask/rotation/... maps
-                else:#not to be downsampled/compressed
-                    if os.path.splitext(file)[1] in binary_types:#just copy
-                        retcode=None
-                        shutil.copy(path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file))
-                    else:#texture name replacement
-                        retcode=None
-                        file_in=open(path_join(input_path,cdir,file),'rb')
-                        file_out=open(path_join(output_path.format(fclass),cdir,file),'wb')
-                        file_str=file_in.read(None)
-                        file_in.close()
-                        (file_strout,num_matches)=compress_names_find.subn(compress_names_replace,file_str)
-                        file_out.write(file_strout)
-                        file_out.close()
-                        #if ((os.path.splitext(file)[1] not in textureuser_types) and num_matches>0):
-                            #print("Warning: ",num_matches," unexpected use(s) in ",path_join(cdir,file))
-                        #if compress_names_find0.search(file_strout):
-                            #print("Warning: unreplaced match(es) in ",path_join(cdir,file),compress_names_find0.search(file_strout).group(0))
-                            """Warning: unreplaced match(es) in... correct rejections of match within a filename:
-                            Aircraft/Instruments-3d/AN-APS-13.ac b'panel.png'
-                            Aircraft/Instruments-3d/magneto-switch/mag_switch.ac b'black.png'
-                            Nasal/canvas/map/Images/chart_symbols.svg b'wash.png'
-                            Models/Airport/blast-deflector49m.ac b'generic.png'
-                            Models/Airport/blast-deflector63m.ac b'generic.png'
-                            Models/Industrial/oilrig09.ac b'yellow.png'
-                            Models/Industrial/oilrig10.ac b'yellow.png'
-                            Models/Industrial/oilrig09.ac.before-color-change b'yellow.png'
-                            Models/Industrial/oilrig10.ac.before-color-change b'yellow.png'
-                            Models/Maritime/Civilian/Tanker.ac b'black.png'
-                            Models/Transport/flatcar.xml b'evergreen.png'
-                            Models/Commercial/tower-grey-black.ac b'black.png'
-                            Materials/base/materials-base.xml b'yellow.png'
-                            
-                            Warning: unexpected use(s) in...
-                            Docs/README.local_weather.html (the only one that looke like an actual problem; hence, Docs is now skipped)
-                            Nasal/canvas/map/Images/chart_symbols.svg (probably inkscape:export-filename, which are creator-specific absolute paths anyway, but now skipped)
-                            oilrig09.ac.before-color-change,oilrig10.ac.before-color-chang,stbd_coaming_panel.ac.bak (presumably backup files)
-                            """
-                if retcode:
-                    print("Error ",retcode," on ",path_join(cdir,file))
     if "{0}" in output_path:
-        subprocess.call(["mv",path_join(output_path.format("base"),"preferences.xml"),path_join(output_path.format("base"),"preferences-regions.xml")])
+        os.rename(path_join(output_path.format("base"),"preferences.xml"),path_join(output_path.format("base"),"preferences-regions.xml"))
     if "extra-textures" in exclude_parts or "{0}" in output_path:
         prefs_in=open(path_join(input_path,"preferences.xml"),'r')
         prefs_out=open(path_join(output_path.format("base"),"preferences-noregions.xml" if "{0}" in output_path else "preferences.xml"),'w')
@@ -460,6 +559,7 @@ Nasal (Canvas map) probably wouldn't break anything, but guessing it's a bad ide
         prefs_str=prefs_str.replace("Materials/regions/materials.xml","Materials/default/materials.xml")#turn off regional textures
         prefs_out.write(prefs_str)
         prefs_out.close()
+    print("Total runtime=",int(time.time()-start_time),"sec")
 def check_text_encoding(path,filelist=None,binary_types=(".png",".dds",".rgb",".RGB",".jpg",".wav",".WAV",".btg.gz",".xcf.gz",".xcf",".XCF","Thumbs.db",".blend",".bmp",".gif", ".3ds",".3DS",".pdf",".ttf",".txf",".htsvoice",".odt",".ods",".xls",".mp3",".zip",".tar.gz"),exclude_dirs=[".git","Timezone"]):
     """filelist is intended for quick testing: see fgdata_nonascii_filelist.py"""
     def err_context(err):