fly/fgmeta
1
0
Fork 0
Code Activity

fgdata_checkers: rename, add more functions

Browse source 
Previously textures_used_checker, now also has:
size calculators
(not-yet-working) minimal fgdata generator
This commit is contained in:
Rebecca Palmer 2014-06-12 23:39:02 +01:00
parent 8e52a6a247
commit 75c90bfb3f
2 changed files with 230 additions and 172 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

230
fgdata_checkers.py Normal file
View file

@ -0,0 +1,230 @@
#!/usr/bin/env python3
from __future__ import print_function#defaults to Python 3, but should also work in 2.7
import os
import os.path
import re
from collections import defaultdict
import subprocess
import math
def rfilelist(path,exclude_dirs=[]):
"""Dict of files/sizes in path, including those in any subdirectories (as relative paths)"""
files=defaultdict(int)
dirs=[""]
while dirs:
cdir=dirs.pop()
cdirfiles=os.listdir(os.path.join(path,cdir))
for file in cdirfiles:
if os.path.isdir(os.path.join(path,cdir,file)):
if os.path.join(cdir,file) not in exclude_dirs:
dirs.append(os.path.join(cdir,file))
else:
files[os.path.join(cdir,file)]=os.path.getsize(os.path.join(path,cdir,file))
return files
def textures_used(path,exclude_dirs=[],pattern=r'<(?:texture|object-mask|tree-texture).*?>(\S+?)</(texture|object-mask|tree-texture)'):
"""Textures used by a file in paths; default pattern matches a <texture> (possibly with number), <tree-texture> or <object-mask> element (for materials files)"""
textures=[]
matfiles=rfilelist(path,exclude_dirs).keys()
texfind=re.compile(pattern)
for file in matfiles:
f=open(os.path.join(path,file),'r')
for line in f:
tex=texfind.search(line)
if tex:
textures.append(tex.group(1))
return textures
def find_unused_textures(basedir,output_lists=True,grep_check=False,output_rsync_rules=False,output_comparison_strips=False,output_removal_commands=False):
"""Checks if any textures are unused (wasting space), and if any textures are only available as .dds (not recommended in the source repository, as it is a lossy-compressed format)
Set basedir to your fg-root, and enable the kind(s) of output you want:
output_lists prints lists of unused textures, and of dds-only textures
grep_check checks for possible use outside the normal directories; requires Unix shell
output_rsync_rules prints rsync rules for excluding unused textures from the release flightgear-data. Warning: if you use this, re-run this script regularly, in case they start being used
output_comparison_strips creates thumbnail strips, unused_duplicate.png/unused_dds.png/high_low.png, for visually checking whether same-name textures are the same (remove the unused one entirely) or different (move it to Unused); requires ImageMagick/graphicsmagick
output_removal_commands creates another script, delete_unused_textures.sh, which will remove unused textures when run in a Unix shell"""
false_positives=set(['buildings-lightmap.png','buildings.png','Credits','Globe/00README.txt', 'Globe/01READMEocean_depth_1png.txt', 'Globe/world.topo.bathy.200407.3x4096x2048.png','Trees/convert.pl'])#these either aren't textures, or are used where we don't check
used_textures=set(textures_used(os.path.join(basedir,'Materials')))|false_positives
used_textures_noregions=set(textures_used(os.path.join(basedir,'Materials'),exclude_dirs=['regions']))|false_positives
used_effectslow=set(textures_used(os.path.join(basedir,'Effects'),pattern=r'image.*?>[\\/]?Textures[\\/](\S+?)</.*?image'))|set(textures_used(os.path.join(basedir,'Materials'),pattern=r'<building-(?:texture|lightmap).*?>Textures[\\/](\S+?)</building-(?:texture|lightmap)'))#Effects (<image>), and Materials <building-texture>/<building-lightmap>, explicitly includes the Textures/ or Textures.high/
used_effectshigh=set(textures_used(os.path.join(basedir,'Effects'),pattern=r'image.*?>[\\/]?Textures.high[\\/](\S+?)</.*?image'))|set(textures_used(os.path.join(basedir,'Materials'),pattern=r'<building-(?:texture|lightmap).*?>Textures.high[\\/](\S+?)</building-(?:texture|lightmap)'))
high_tsizes=rfilelist(os.path.join(basedir,'Textures.high'))
high_textures=set(high_tsizes.keys())
low_tsizes=rfilelist(os.path.join(basedir,'Textures'),exclude_dirs=['Sky','Unused'])#sky textures are used where we don't check
low_textures=set(low_tsizes.keys())
only_high=high_textures-low_textures
used_noreg_onlyhigh=(only_high&used_textures_noregions)|used_effectshigh
used_noreg_onlyhighsize=sum(high_tsizes[t] for t in used_noreg_onlyhigh)
used_noreg_low=(low_textures&used_textures_noregions)|used_effectslow
used_noreg_lowsize=sum(low_tsizes[t] for t in used_noreg_low)
used_noreg_defsize=sum(low_tsizes[t] for t in (used_textures_noregions-high_textures)|used_effectslow)+sum(high_tsizes[t] for t in used_textures_noregions|used_effectshigh)
used_defsize=sum(low_tsizes[t] for t in (used_textures-high_textures)|used_effectslow)+sum(high_tsizes[t] for t in used_textures|used_effectshigh)
unused=(high_textures|low_textures)-used_textures-used_effectslow-used_effectshigh
t_size=lambda tset: sum(high_tsizes[t] for t in tset)+sum(low_tsizes[t] for t in tset)
missing=(used_textures-(high_textures|low_textures))|(used_effectslow-low_textures)|(used_effectshigh-high_textures)
if missing:
raise ValueError("Some used textures not found: "+repr(missing))
sourceless=[f for f in (high_textures|low_textures) if (f[-4:]==".dds" and f[:-4]+".png" not in high_textures and (f in high_textures or f[:-4]+".png" not in low_textures) )]+['Terrain/airport.dds']#airport.dds isn't the same as airport.png; crop-colors.dds/cropgrass-colors.dds/rock-colors.dds/forest-colors.dds also differ but only in strip width, which doesn't matter as they are 1D color strips
sourceless_used=set(sourceless)-unused
needed_as_source=[f for f in unused if (f[-4:]!=".dds" and f[:-4]+".png" in (used_textures|used_effectslow|used_effectshigh) or f[:-4]+".dds" in (used_textures|used_effectslow|used_effectshigh))]+['Runway/designation_letters.svg']
known_non_duplicates=['deciduous.png','drycrop.png','irrcrop.png','marsh1.png','gravel.png','Town.png','grass.png','mixedcrop.png','resgrid.png']+['glacier.png','rock.png','cropgrass.png']#first group real winter textures, second group unrelated textures
unused_duplicate=[f for f in unused if (f[0:14]=="Terrain.winter" and "Terrain"+f[14:] in (high_textures|low_textures) and f[15:] not in known_non_duplicates)]
unused_dds=set(f for f in (unused-set(unused_duplicate)) if (f[-4:]==".dds" and f[:-4]+".png" in (high_textures|low_textures) and f!='Terrain/airport.dds'))#airport.dds isn't the same as airport.png; crop-colors.dds/cropgrass-colors.dds/rock-colors.dds/forest-colors.dds also differ but only in strip width, which doesn't matter as they are 1D color strips
unused_other=unused-set(unused_duplicate)-set(unused_dds)-set(needed_as_source)
known_highlow_mismatch=set(['Terrain.winter/mixedcrop4.png','Terrain.winter/cropgrass3.png','Terrain.winter/drycrop4.png','Terrain.winter/irrcrop2.png','Terrain.winter/drycrop1.png','Terrain.winter/drycrop3.png','Terrain.winter/mixedcrop1.png','Terrain.winter/ mixedforest2.png','Terrain.winter/cropgrass2.png','Terrain.winter/cropgrass1.png','Terrain.winter/tundra.png','Terrain.winter/mixedforest3.png','Terrain.winter/shrub2.png','Terrain.winter/drycrop2.png','Terrain.winter/deciduous1.png','Terrain.winter/ mixedcrop3.png','Terrain.winter/naturalcrop1.png']+['Terrain.winter/tundra3.png','Terrain.winter/forest1c.png','Terrain.winter/herbtundra.png']+['Terrain/grass_rwy.dds','Terrain/cropwood.dds','Terrain/herbtundra.dds','Terrain/irrcrop.dds','Terrain/shrub.dds','Terrain.winter/mixedforest.png','Runway/pa_taxiway.png','Runway/pc_taxiway.png'])#first group are different degrees of snow cover on the same base texture, last group unrelated textures, middle group hard to tell; p{a,c}_taxiway (only low-res has side lines) are also mismatched in .dds, but as each .dds matches its size .png, only the .png needs to be kept in Unused
lowres_maybe_source=['Terrain/lava1.png','Terrain/lava2.png','Terrain/lava3.png','Terrain/sand4.png','Terrain/sand5.png','Terrain/sand6.png']#these are clearly related, but the high-res version has unnatural-looking high-frequency noise, suggesting that the low-res version might be the original: keep it
unused_dds_matchhigh=set(f for f in (unused_dds&known_highlow_mismatch) if f[:-4]+".png" not in low_textures)
unused_dds_matchlow=set(f for f in (unused_dds&known_highlow_mismatch) if f[:-4]+".png" not in high_textures)
low_unneeded=(high_textures&low_textures)-used_effectslow-unused-set(lowres_maybe_source)
low_unneeded_duplicate=low_unneeded-set(known_highlow_mismatch)
low_unneeded_nondup=low_unneeded&set(known_highlow_mismatch)
def image_check_strip(basedir,index_fname,ilist1,ilist2=None,size=128):
"""Generate two rows of thumbnails, for easy visual comparison (between the two lists given, or if a single list is given, between low and high resolution)"""
if ilist2 is None:
ipairs=[[os.path.join(basedir,'Textures',f),os.path.join(basedir,'Textures.high',f)] for f in ilist1]
else:
ipairs=[]
for f1,f2 in zip(ilist1,ilist2):
if f1 in low_textures:
ipairs.append([os.path.join(basedir,'Textures',f1),os.path.join(basedir,'Textures',f2) if f2 in low_textures else os.path.join(basedir,'Textures.high',f2)])
if f1 in high_textures:
ipairs.append([os.path.join(basedir,'Textures.high',f1),os.path.join(basedir,'Textures.high',f2) if f2 in high_textures else os.path.join(basedir,'Textures',f2)])
ilist_f=[f[0] for f in ipairs]+[f[1] for f in ipairs]
subprocess.call(['montage','-label',"'%f'"]+ilist_f+['-tile','x2','-geometry',str(size)+'x'+str(size)]+[index_fname])
def rsync_rules(basedir,flist,include=False,high=None):
"""Output rsync rules to exclude/include the specified textures from high/low/both (high=True/False/None) resolutions"""
for f in flist:
if high!=True and f in low_textures:
print("+" if include else "-",os.path.join('/fgdata/Textures',f))
if high!=False and f in high_textures:
print("+" if include else "-",os.path.join('/fgdata/Textures.high',f))
def removal_command(basedir,flist,high=None):
"""Return command to delete the specified textures from high/low/both (high=True/False/None) resolutions"""
if not flist:
return ""
a="rm"
for f in flist:
if high!=True and f in low_textures:
a=a+" "+os.path.join('Textures',f)
if high!=False and f in high_textures:
a=a+" "+os.path.join('Textures.high',f)
a=a+"\n"
return a
def move_command(basedir,flist,high=None,comment=False):
"""Return command to move the specified textures to Unused from high/low/both (high=True/False/None) resolutions"""
if not flist:
return ""
dirset_low=set() if high==True else set(os.path.dirname(f) for f in set(flist)&low_textures)
dirset_high=set() if high==False else set(os.path.dirname(f) for f in set(flist)&high_textures)
a=""
for d in dirset_low:
a=a+("#" if comment else "")+"mv --target-directory="+os.path.join("Textures/Unused",d)+" "+(" ".join(os.path.join("Textures",f) for f in flist if (os.path.dirname(f)==d and f in low_textures)))+"\n"
for d in dirset_high:
a=a+("#" if comment else "")+"mv --target-directory="+os.path.join("Textures/Unused",d+".high")+" "+(" ".join(os.path.join("Textures.high",f) for f in flist if (os.path.dirname(f)==d and f in high_textures)))+"\n"
return a
if output_comparison_strips:
image_check_strip(basedir,"unused_duplicate.png",unused_duplicate,["Terrain"+f[14:] for f in unused_duplicate])
image_check_strip(basedir,"unused_dds.png",unused_dds,[f[:-4]+".png" for f in unused_dds])
image_check_strip(basedir,"high_low.png",high_textures&low_textures)
#image_check_strip(basedir,"high_low2.png",[f for f in high_textures&low_textures if (f[0:14]=="Terrain.winter" or "_taxiway." in f or "lava" in f or "sand" in f)],size=512)#closer look at the doubtful cases
if output_lists:
print("\n\nunused-winter same as normal:",sorted(unused_duplicate),"\nsize=",t_size(unused_duplicate),"\n\nunused-dds with matching png:",sorted(unused_dds),"\nsize=",t_size(unused_dds),"\n\nunused-unique:",sorted(unused_other),"\nsize=",t_size(unused_other),"\n\nnot directly used but keep as source:",sorted(needed_as_source),"\nsize=",t_size(needed_as_source),"\n\nunused low, matches high:",sorted(low_unneeded_duplicate),"\nsize=",sum(low_tsizes[f] for f in low_unneeded_duplicate),"\n\nunused low, unique:",sorted(low_unneeded_nondup),"\nsize=",sum(low_tsizes[f] for f in low_unneeded_nondup),"\n\nall non-sky textures size=",sum(high_tsizes.values())+sum(low_tsizes.values()),"used size=",used_defsize,"used no-regions size=",used_noreg_defsize,"\n\nnot found:",sorted(missing),"\n\n.dds only/highest-res:",sorted(sourceless),"\n\n.dds only/highest-res, used:",sorted(sourceless_used))
#not really meaningful after removing low-res duplicates: ,"\n\nused high-only, not regions:",sorted(used_noreg_onlyhigh),"\nsize=",used_noreg_onlyhighsize,"these+used low (i.e. minimal flightgear-data) size=",used_noreg_onlyhighsize+used_noreg_lowsize
if grep_check:
unused_f=[os.path.basename(f) for f in unused]
all_f=[os.path.basename(f) for f in (high_textures|low_textures)]
print("\n\nPossible use outside main search:")#used to set false_positives
subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(unused)+")","/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])#everywhere using full names
subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e","("+")|(".join(all_f)+")","/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])#restricted (to avoid false positives from Terrain.winter vs Terrain) using filenames
subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e",'[."\']dds',"/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])#check for programmatic .png -> .dds swap; none found
print("\n\nUse of sourceless textures:")
subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(sourceless)+")","/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])
if output_rsync_rules:
print("\n\nFull flightgear-data:\n")
rsync_rules(basedir,unused)
rsync_rules(basedir,low_unneeded,high=False)
print("\n\nMinimal flightgear-data:\n")
rsync_rules(basedir,low_textures-used_noreg_low,high=False)
rsync_rules(basedir,high_textures-used_noreg_onlyhigh,high=True)
if output_removal_commands:
r_script=open('delete_unused_textures.sh','w')
r_script.write("cd "+basedir+"\n")
r_script.write("#Unused duplicates\n")
r_script.write(removal_command(basedir,unused_duplicate))
r_script.write("#Unused .dds versions\n")
r_script.write(removal_command(basedir,unused_dds-unused_dds_matchhigh,high=False))
r_script.write(removal_command(basedir,unused_dds-unused_dds_matchlow,high=True))
r_script.write("#Unused reduced-resolution versions\n")
r_script.write(removal_command(basedir,low_unneeded_duplicate|(unused_other&high_textures&low_textures)-set(lowres_maybe_source),high=False))
r_script.write("#Unused unique .png (move to Unused)\n")
r_script.write("\n".join(["mkdir -p Textures/Unused/"+d for d in ['Terrain','Terrain.winter','Trees','Terrain.high','Terrain.winter.high','Trees.high','Runway','Water']])+"\n")
r_script.write(move_command(basedir,[f for f in unused_other&high_textures if (f[-4:]!=".dds" and f[:5]!="Signs" and f[:6]!="Runway")],high=True))
r_script.write(move_command(basedir,[f for f in (unused_other-high_textures)|low_unneeded_nondup if (f[-4:]!=".dds" and f[:5]!="Signs" and f[:6]!="Runway")],high=False))
r_script.write("#Unused unique .dds\n")
r_script.write("#It is my opinion that these should go, but if you'd prefer to move them to Unused I won't argue further\n")
r_script.write(removal_command(basedir,[f for f in (unused_other&high_textures)|unused_dds_matchlow if (f[-4:]==".dds" and f[:5]!="Signs" and f[:6]!="Runway")],high=True))
r_script.write(removal_command(basedir,[f for f in (unused_other-high_textures)|low_unneeded_nondup|unused_dds_matchhigh if f[-4:]==".dds"],high=False))
r_script.write(move_command(basedir,[f for f in (unused_other&high_textures)|unused_dds_matchlow if (f[-4:]==".dds" and f[:5]!="Signs" and f[:6]!="Runway")],high=True,comment=True))
r_script.write(move_command(basedir,[f for f in (unused_other-high_textures)|low_unneeded_nondup|unused_dds_matchhigh if (f[-4:]==".dds" and f[:5]!="Signs" and f[:6]!="Runway")],high=False,comment=True))
r_script.close()
def size_by_type(path,exclude_dirs=[]):
"""Dict of total file size by file extension"""
files=rfilelist(path,exclude_dirs)
size_totals=defaultdict(int)
for filename,size in files.items():
file_ext=os.path.splitext(filename)[1]
if file_ext==".gz":
file_ext=os.path.splitext(os.path.splitext(filename)[0])[1]+file_ext
size_totals[file_ext]=size_totals[file_ext]+size
return size_totals
def size_by_size(path,exclude_dirs=[],exts=[".png",".dds",".rgb"]):
"""Dict of total file size by individual file size range, of given extensions (empty list for all files)"""
files=rfilelist(path,exclude_dirs)
size_totals=defaultdict(int)
for filename,size in files.items():
file_ext=os.path.splitext(filename)[1]
if (not exts) or (file_ext in exts):
size_totals[2**math.frexp(size)[1]]=size_totals[2**math.frexp(size)[1]]+size
return size_totals
def fgdata_size(path,dirs_to_list=["AI/Aircraft","AI/Traffic","Models","Scenery","Textures","Textures.high"],exclude_dirs=[".git","Aircraft"]):
exclude_list=[[]]*len(dirs_to_list)+[dirs_to_list+exclude_dirs]+[exclude_dirs]
names_list=dirs_to_list+["other","all"]
for n,dir1 in enumerate(dirs_to_list+["",""]):
size_totals=size_by_type(os.path.join(path,dir1),exclude_list[n])
print(names_list[n],sorted(size_totals.items(),key=lambda x:-x[1]),"total",sum(size_totals.values()))
def create_reduced_fgdata(input_path,output_path,exclude_ai=False):
"""Create a smaller, reduced-quality flightgear-data package
Requires imagemagick (not Ubuntu graphicsmagick, it can't write .dds)"""
raise ValueError("The package this generates currently crashes FlightGear; not sure why yet")
texture_filetypes=[".png",".rgb",".dds"]
downsample_min_filesize=30000
dirs_to_downsample=("Textures.high/Terrain","Textures.high/Trees","Textures.high/Terrain.winter","AI/Aircraft","Models")
exclude_dirs=[".git"]
exclude_unnamed_subdirs=["Aircraft"]
include_subdirs=["Aircraft/c172p","Aircraft/Generic","Aircraft/Instruments","Aircraft/Instruments-3d","Aircraft/ufo"]
if exclude_ai:
exclude_unnamed_subdirs.extend(["AI/Aircraft","AI/Traffic"])
subprocess.call(["mkdir","-p",output_path])
if os.path.exists(os.path.join(input_path,".git")):
print(input_path,"appears to be a git clone; this will work, but the result will be slightly larger than starting from a standard flightgear-data package.\nTo create this use (adjusting paths as necessary) rsync -av --filter=\"merge /home/palmer/fs_dev/git/fgmeta/base-package.rules\" ~/fs_dev/git/fgdata ~/fs_dev/flightgear/data_full")
if os.listdir(output_path):
print("output path",output_path,"non-empty, aborting to avoid data loss\nIf you did want to lose its previous contents, run:\nrm -r",output_path,"\nthen re-run this script")
return
dirs=[""]
while dirs:
cdir=dirs.pop()
cdirfiles=os.listdir(os.path.join(input_path,cdir))
for file in cdirfiles:
if os.path.isdir(os.path.join(input_path,cdir,file)):
if (os.path.join(cdir,file) not in exclude_dirs) and (cdir not in exclude_unnamed_subdirs or os.path.join(cdir,file) in include_subdirs):
subprocess.call(["mkdir","-p",os.path.join(output_path,cdir,file)])
dirs.append(os.path.join(cdir,file))
else:
if (cdir.startswith(dirs_to_downsample)) and (os.path.splitext(file)[1] in texture_filetypes) and (os.path.getsize(os.path.join(input_path,cdir,file))>downsample_min_filesize):
subprocess.call(["convert",os.path.join(input_path,cdir,file),"-sample","50%",os.path.join(output_path,cdir,file)])
else:
subprocess.call(["cp",os.path.join(input_path,cdir,file),os.path.join(output_path,cdir,file)])

172
textures_used_checker.py
View file

@ -1,172 +0,0 @@
#!/usr/bin/env python3
from __future__ import print_function#defaults to Python 3, but should also work in 2.7
"""Script for checking if any textures are unused (wasting space), and if any textures are only available as .dds (not recommended in the source repository, as it is a lossy-compressed format)
Set basedir to your fg-root, and enable the kind(s) of output you want below:"""
#basedir='/usr/share/games/flightgear/'#Debian/Ubuntu installed default
output_lists=True#prints lists of unused textures, and of dds-only textures
grep_check=False#checks for possible use outside the normal directories; requires Unix shell
output_rsync_rules=False#prints rsync rules for excluding unused textures from the release flightgear-data. Warning: if you use this, re-run this script regularly, in case they start being used
output_comparison_strips=False#creates thumbnail strips, unused_duplicate.png/unused_dds.png/high_low.png, for visually checking whether same-name textures are the same (remove the unused one entirely) or different (move it to Unused); requires ImageMagick/graphicsmagick
output_removal_commands=False#creates another script, delete_unused_textures.sh, which will remove unused textures when run in a Unix shell
import os
import os.path
import re
from collections import defaultdict
import subprocess
try:
basedir
except NameError:
print("basedir not set, please set it")
def rfilelist(path,exclude_dirs=[]):
"""Dict of files/sizes in path, including those in any subdirectories (as relative paths)"""
files=defaultdict(int)
dirs=[""]
while dirs:
cdir=dirs.pop()
cdirfiles=os.listdir(os.path.join(path,cdir))
for file in cdirfiles:
if os.path.isdir(os.path.join(path,cdir,file)):
if os.path.join(cdir,file) not in exclude_dirs:
dirs.append(os.path.join(cdir,file))
else:
files[os.path.join(cdir,file)]=os.path.getsize(os.path.join(path,cdir,file))
return files
def textures_used(path,exclude_dirs=[],pattern=r'<(?:texture|object-mask|tree-texture).*?>(\S+?)</(texture|object-mask|tree-texture)'):
"""Textures used by a file in paths; default pattern matches a <texture> (possibly with number), <tree-texture> or <object-mask> element (for materials files)"""
textures=[]
matfiles=rfilelist(path,exclude_dirs).keys()
texfind=re.compile(pattern)
for file in matfiles:
f=open(os.path.join(path,file),'r')
for line in f:
tex=texfind.search(line)
if tex:
textures.append(tex.group(1))
return textures
def image_check_strip(index_fname,ilist1,ilist2=None,size=128):
"""Generate two rows of thumbnails, for easy visual comparison (between the two lists given, or if a single list is given, between low and high resolution)"""
if ilist2 is None:
ipairs=[[os.path.join(basedir,'Textures',f),os.path.join(basedir,'Textures.high',f)] for f in ilist1]
else:
ipairs=[]
for f1,f2 in zip(ilist1,ilist2):
if f1 in low_textures:
ipairs.append([os.path.join(basedir,'Textures',f1),os.path.join(basedir,'Textures',f2) if f2 in low_textures else os.path.join(basedir,'Textures.high',f2)])
if f1 in high_textures:
ipairs.append([os.path.join(basedir,'Textures.high',f1),os.path.join(basedir,'Textures.high',f2) if f2 in high_textures else os.path.join(basedir,'Textures',f2)])
ilist_f=[f[0] for f in ipairs]+[f[1] for f in ipairs]
subprocess.call(['montage','-label',"'%f'"]+ilist_f+['-tile','x2','-geometry',str(size)+'x'+str(size)]+[index_fname])
def rsync_rules(flist,include=False,high=None):
"""Output rsync rules to exclude/include the specified textures from high/low/both (high=True/False/None) resolutions"""
for f in flist:
if high!=True and f in low_textures:
print("+" if include else "-",os.path.join('/fgdata/Textures',f))
if high!=False and f in high_textures:
print("+" if include else "-",os.path.join('/fgdata/Textures.high',f))
def removal_command(flist,high=None):
"""Return command to delete the specified textures from high/low/both (high=True/False/None) resolutions"""
a="rm"
for f in flist:
if high!=True and f in low_textures:
a=a+" "+os.path.join('Textures',f)
if high!=False and f in high_textures:
a=a+" "+os.path.join('Textures.high',f)
a=a+"\n"
return a
def move_command(flist,high=None,comment=False):
"""Return command to move the specified textures to Unused from high/low/both (high=True/False/None) resolutions"""
dirset_low=set() if high==True else set(os.path.dirname(f) for f in set(flist)&low_textures)
dirset_high=set() if high==False else set(os.path.dirname(f) for f in set(flist)&high_textures)
a=""
for d in dirset_low:
a=a+("#" if comment else "")+"mv --target-directory="+os.path.join("Textures/Unused",d)+" "+(" ".join(os.path.join("Textures",f) for f in flist if (os.path.dirname(f)==d and f in low_textures)))+"\n"
for d in dirset_high:
a=a+("#" if comment else "")+"mv --target-directory="+os.path.join("Textures/Unused",d+".high")+" "+(" ".join(os.path.join("Textures.high",f) for f in flist if (os.path.dirname(f)==d and f in high_textures)))+"\n"
return a
false_positives=set(['buildings-lightmap.png','buildings.png','Credits','Globe/00README.txt', 'Globe/01READMEocean_depth_1png.txt', 'Globe/world.topo.bathy.200407.3x4096x2048.png','Trees/convert.pl'])#these either aren't textures, or are used where we don't check
used_textures=set(textures_used(basedir+'Materials'))|false_positives
used_textures_noregions=set(textures_used(basedir+'Materials',exclude_dirs=['regions']))|false_positives
used_effectslow=set(textures_used(basedir+'Effects',pattern=r'image.*?>[\\/]?Textures[\\/](\S+?)</.*?image'))|set(textures_used(basedir+'Materials',pattern=r'<building-(?:texture|lightmap).*?>Textures[\\/](\S+?)</building-(?:texture|lightmap)'))#Effects (<image>), and Materials <building-texture>/<building-lightmap>, explicitly includes the Textures/ or Textures.high/
used_effectshigh=set(textures_used(basedir+'Effects',pattern=r'image.*?>[\\/]?Textures.high[\\/](\S+?)</.*?image'))|set(textures_used(basedir+'Materials',pattern=r'<building-(?:texture|lightmap).*?>Textures.high[\\/](\S+?)</building-(?:texture|lightmap)'))
high_tsizes=rfilelist(basedir+'Textures.high')
high_textures=set(high_tsizes.keys())
low_tsizes=rfilelist(basedir+'Textures',exclude_dirs=['Sky','Unused'])#sky textures are used where we don't check
low_textures=set(low_tsizes.keys())
only_high=high_textures-low_textures
used_noreg_onlyhigh=(only_high&used_textures_noregions)|used_effectshigh
used_noreg_onlyhighsize=sum(high_tsizes[t] for t in used_noreg_onlyhigh)
used_noreg_low=(low_textures&used_textures_noregions)|used_effectslow
used_noreg_lowsize=sum(low_tsizes[t] for t in used_noreg_low)
used_noreg_defsize=sum(low_tsizes[t] for t in (used_textures_noregions-high_textures)|used_effectslow)+sum(high_tsizes[t] for t in used_textures_noregions|used_effectshigh)
used_defsize=sum(low_tsizes[t] for t in (used_textures-high_textures)|used_effectslow)+sum(high_tsizes[t] for t in used_textures|used_effectshigh)
unused=(high_textures|low_textures)-used_textures-used_effectslow-used_effectshigh
t_size=lambda tset: sum(high_tsizes[t] for t in tset)+sum(low_tsizes[t] for t in tset)
missing=(used_textures-(high_textures|low_textures))|(used_effectslow-low_textures)|(used_effectshigh-high_textures)
if missing:
raise ValueError("Some used textures not found: "+repr(missing))
sourceless=[f for f in (high_textures|low_textures) if (f[-4:]==".dds" and f[:-4]+".png" not in high_textures and (f in high_textures or f[:-4]+".png" not in low_textures) )]+['Terrain/airport.dds']#airport.dds isn't the same as airport.png; crop-colors.dds/cropgrass-colors.dds/rock-colors.dds/forest-colors.dds also differ but only in strip width, which doesn't matter as they are 1D color strips
sourceless_used=set(sourceless)-unused
needed_as_source=[f for f in unused if (f[-4:]!=".dds" and f[:-4]+".png" in (used_textures|used_effectslow|used_effectshigh) or f[:-4]+".dds" in (used_textures|used_effectslow|used_effectshigh))]+['Runway/designation_letters.svg']
known_non_duplicates=['deciduous.png','drycrop.png','irrcrop.png','marsh1.png','gravel.png','Town.png','grass.png','mixedcrop.png','resgrid.png']+['glacier.png','rock.png','cropgrass.png']#first group real winter textures, second group unrelated textures
unused_duplicate=[f for f in unused if (f[0:14]=="Terrain.winter" and "Terrain"+f[14:] in (high_textures|low_textures) and f[15:] not in known_non_duplicates)]
unused_dds=set(f for f in (unused-set(unused_duplicate)) if (f[-4:]==".dds" and f[:-4]+".png" in (high_textures|low_textures) and f!='Terrain/airport.dds'))#airport.dds isn't the same as airport.png; crop-colors.dds/cropgrass-colors.dds/rock-colors.dds/forest-colors.dds also differ but only in strip width, which doesn't matter as they are 1D color strips
unused_other=unused-set(unused_duplicate)-set(unused_dds)-set(needed_as_source)
known_highlow_mismatch=set(['Terrain.winter/mixedcrop4.png','Terrain.winter/cropgrass3.png','Terrain.winter/drycrop4.png','Terrain.winter/irrcrop2.png','Terrain.winter/drycrop1.png','Terrain.winter/drycrop3.png','Terrain.winter/mixedcrop1.png','Terrain.winter/ mixedforest2.png','Terrain.winter/cropgrass2.png','Terrain.winter/cropgrass1.png','Terrain.winter/tundra.png','Terrain.winter/mixedforest3.png','Terrain.winter/shrub2.png','Terrain.winter/drycrop2.png','Terrain.winter/deciduous1.png','Terrain.winter/ mixedcrop3.png','Terrain.winter/naturalcrop1.png']+['Terrain.winter/tundra3.png','Terrain.winter/forest1c.png','Terrain.winter/herbtundra.png']+['Terrain/grass_rwy.dds','Terrain/cropwood.dds','Terrain/herbtundra.dds','Terrain/irrcrop.dds','Terrain/shrub.dds','Terrain.winter/mixedforest.png','Runway/pa_taxiway.png','Runway/pc_taxiway.png'])#first group are different degrees of snow cover on the same base texture, last group unrelated textures, middle group hard to tell; p{a,c}_taxiway (only low-res has side lines) are also mismatched in .dds, but as each .dds matches its size .png, only the .png needs to be kept in Unused
lowres_maybe_source=['Terrain/lava1.png','Terrain/lava2.png','Terrain/lava3.png','Terrain/sand4.png','Terrain/sand5.png','Terrain/sand6.png']#these are clearly related, but the high-res version has unnatural-looking high-frequency noise, suggesting that the low-res version might be the original: keep it
unused_dds_matchhigh=set(f for f in (unused_dds&known_highlow_mismatch) if f[:-4]+".png" not in low_textures)
unused_dds_matchlow=set(f for f in (unused_dds&known_highlow_mismatch) if f[:-4]+".png" not in high_textures)
low_unneeded=(high_textures&low_textures)-used_effectslow-unused-set(lowres_maybe_source)
low_unneeded_duplicate=low_unneeded-set(known_highlow_mismatch)
low_unneeded_nondup=low_unneeded&set(known_highlow_mismatch)
if output_comparison_strips:
image_check_strip("unused_duplicate.png",unused_duplicate,["Terrain"+f[14:] for f in unused_duplicate])
image_check_strip("unused_dds.png",unused_dds,[f[:-4]+".png" for f in unused_dds])
image_check_strip("high_low.png",high_textures&low_textures)
#image_check_strip("high_low2.png",[f for f in high_textures&low_textures if (f[0:14]=="Terrain.winter" or "_taxiway." in f or "lava" in f or "sand" in f)],size=512)#closer look at the doubtful cases
if output_lists:
print("\n\nunused-winter same as normal:",sorted(unused_duplicate),"\nsize=",t_size(unused_duplicate),"\n\nunused-dds with matching png:",sorted(unused_dds),"\nsize=",t_size(unused_dds),"\n\nunused-unique:",sorted(unused_other),"\nsize=",t_size(unused_other),"\n\nnot directly used but keep as source:",sorted(needed_as_source),"\nsize=",t_size(needed_as_source),"\n\nunused low, matches high:",sorted(low_unneeded_duplicate),"\nsize=",sum(low_tsizes[f] for f in low_unneeded_duplicate),"\n\nunused low, unique:",sorted(low_unneeded_nondup),"\nsize=",sum(low_tsizes[f] for f in low_unneeded_nondup),"\n\nall non-sky textures size=",sum(high_tsizes.values())+sum(low_tsizes.values()),"used size=",used_defsize,"used no-regions size=",used_noreg_defsize,"\n\nnot found:",sorted(missing),"\n\n.dds only/highest-res:",sorted(sourceless),"\n\n.dds only/highest-res, used:",sorted(sourceless_used))
#not really meaningful after removing low-res duplicates: ,"\n\nused high-only, not regions:",sorted(used_noreg_onlyhigh),"\nsize=",used_noreg_onlyhighsize,"these+used low (i.e. minimal flightgear-data) size=",used_noreg_onlyhighsize+used_noreg_lowsize
if grep_check:
unused_f=[os.path.basename(f) for f in unused]
all_f=[os.path.basename(f) for f in (high_textures|low_textures)]
print("\n\nPossible use outside main search:")#used to set false_positives
subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(unused)+")","/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])#everywhere using full names
subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e","("+")|(".join(all_f)+")","/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])#restricted (to avoid false positives from Terrain.winter vs Terrain) using filenames
subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e",'[."\']dds',"/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])#check for programmatic .png -> .dds swap; none found
print("\n\nUse of sourceless textures:")
subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(sourceless)+")","/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])
if output_rsync_rules:
print("\n\nFull flightgear-data:\n")
rsync_rules(unused)
rsync_rules(low_unneeded,high=False)
print("\n\nMinimal flightgear-data:\n")
rsync_rules(low_textures-used_noreg_low,high=False)
rsync_rules(high_textures-used_noreg_onlyhigh,high=True)
if output_removal_commands:
r_script=open('delete_unused_textures.sh','w')
r_script.write("cd "+basedir+"\n")
r_script.write("#Unused duplicates\n")
r_script.write(removal_command(unused_duplicate))
r_script.write("#Unused .dds versions\n")
r_script.write(removal_command(unused_dds-unused_dds_matchhigh,high=False))
r_script.write(removal_command(unused_dds-unused_dds_matchlow,high=True))
r_script.write("#Unused reduced-resolution versions\n")
r_script.write(removal_command(low_unneeded_duplicate|(unused_other&high_textures&low_textures)-set(lowres_maybe_source),high=False))
r_script.write("#Unused unique .png (move to Unused)\n")
r_script.write("\n".join(["mkdir -p Textures/Unused/"+d for d in ['Terrain','Terrain.winter','Trees','Terrain.high','Terrain.winter.high','Trees.high','Runway','Water']])+"\n")
r_script.write(move_command([f for f in unused_other&high_textures if (f[-4:]!=".dds" and f[:5]!="Signs" and f[:6]!="Runway")],high=True))
r_script.write(move_command([f for f in (unused_other-high_textures)|low_unneeded_nondup if (f[-4:]!=".dds" and f[:5]!="Signs" and f[:6]!="Runway")],high=False))
r_script.write("#Unused unique .dds\n")
r_script.write("#It is my opinion that these should go, but if you'd prefer to move them to Unused I won't argue further\n")
r_script.write(removal_command([f for f in (unused_other&high_textures)|unused_dds_matchlow if (f[-4:]==".dds" and f[:5]!="Signs" and f[:6]!="Runway")],high=True))
r_script.write(removal_command([f for f in low_unneeded_nondup|unused_dds_matchhigh if f[-4:]==".dds"],high=False))
r_script.write(move_command([f for f in (unused_other&high_textures)|unused_dds_matchlow if (f[-4:]==".dds" and f[:5]!="Signs" and f[:6]!="Runway")],high=True,comment=True))
r_script.write(move_command([f for f in (unused_other-high_textures)|low_unneeded_nondup|unused_dds_matchhigh if (f[-4:]==".dds" and f[:5]!="Signs" and f[:6]!="Runway")],high=False,comment=True))
r_script.close()