c27ae92c73
- New generic exception class TerraSyncPyException. - Add subclass NetworkError of TerraSyncPyException. - Raise a NetworkError exception when the HTTP return code is not 200. - hash_of_file() does not silently ignore errors anymore; exceptions should be dealt with wherever appropriate by the callers. Whenever hash_of_file() returns, its return value is now the SHA-1 hash of the specified file. This is less error-prone IMHO than returning None. Otherwise, calling code could erroneously conclude that there is a matching hash when the file to check is actually missing. For a concrete example, see the 'dirIndexHash' parameter of TerraSync.updateDirectory(), which so far is used precisely with the value None to express that "we are just starting the recursion and have no hash from the server to compare to".
362 lines
13 KiB
Python
Executable file
362 lines
13 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# Copyright (C) 2016 Torsten Dreyer
|
|
#
|
|
# This program is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU General Public License as
|
|
# published by the Free Software Foundation; either version 2 of the
|
|
# License, or (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful, but
|
|
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
# General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
#
|
|
# terrasync.py - synchronize terrascenery data to your local disk
|
|
# needs dnspython (pip install dnspython)
|
|
|
|
import urllib, os, hashlib
|
|
from urllib.parse import urlparse
|
|
from http.client import HTTPConnection, _CS_IDLE, HTTPException
|
|
from os import listdir
|
|
from os.path import isfile, isdir, join
|
|
import re
|
|
import argparse
|
|
import shutil
|
|
|
|
|
|
# Generic exception class for terrasync.py, to be subclassed for each specific
|
|
# kind exception.
|
|
class TerraSyncPyException(Exception):
|
|
def __init__(self, message=None, *, mayCapitalizeMsg=True):
|
|
"""Initialize a TerraSyncPyException instance.
|
|
|
|
Except in cases where 'message' starts with a proper noun or
|
|
something like that, its first character should be given in
|
|
lower case. Automated treatments of this exception may print the
|
|
message with its first character changed to upper case, unless
|
|
'mayCapitalizeMsg' is False. In other words, if the case of the
|
|
first character of 'message' must not be changed under any
|
|
circumstances, set 'mayCapitalizeMsg' to False.
|
|
|
|
"""
|
|
self.message = message
|
|
self.mayCapitalizeMsg = mayCapitalizeMsg
|
|
|
|
def __str__(self):
|
|
return self.completeMessage()
|
|
|
|
def __repr__(self):
|
|
return "{}.{}({!r})".format(__name__, type(self).__name__, self.message)
|
|
|
|
# Typically overridden by subclasses with a custom constructor
|
|
def detail(self):
|
|
return self.message
|
|
|
|
def completeMessage(self):
|
|
if self.message:
|
|
return "{shortDesc}: {detail}".format(
|
|
shortDesc=self.ExceptionShortDescription,
|
|
detail=self.detail())
|
|
else:
|
|
return self.ExceptionShortDescription
|
|
|
|
ExceptionShortDescription = "terrasync.py generic exception"
|
|
|
|
|
|
class NetworkError(TerraSyncPyException):
|
|
"""Exception raised when getting a network error even after retrying."""
|
|
ExceptionShortDescription = "Network error"
|
|
|
|
|
|
#################################################################################################################################
|
|
class HTTPGetCallback:
|
|
def __init__(self, src, callback):
|
|
self.callback = callback
|
|
self.src = src
|
|
|
|
class HTTPGetter:
|
|
def __init__(self, baseUrl, maxPending=10):
|
|
self.baseUrl = baseUrl
|
|
self.parsedBaseUrl = urlparse(baseUrl)
|
|
self.maxPending = maxPending
|
|
self.requests = []
|
|
self.pendingRequests = []
|
|
self.httpConnection = HTTPConnection(self.parsedBaseUrl.netloc)
|
|
self.httpRequestHeaders = headers = {'Host':self.parsedBaseUrl.netloc,'Content-Length':0,'Connection':'Keep-Alive','User-Agent':'FlightGear terrasync.py'}
|
|
|
|
def doGet(self, httpGetCallback):
|
|
conn = self.httpConnection
|
|
url = self.parsedBaseUrl.path + httpGetCallback.src
|
|
self.httpConnection.request("GET", url, None, self.httpRequestHeaders)
|
|
httpResponse = self.httpConnection.getresponse()
|
|
|
|
# 'httpResponse' is an http.client.HTTPResponse instance
|
|
return httpGetCallback.callback(url, httpResponse)
|
|
|
|
def get(self, httpGetCallback):
|
|
try:
|
|
res = self.doGet(httpGetCallback)
|
|
except HTTPException:
|
|
# try to reconnect once
|
|
#print("reconnect")
|
|
self.httpConnection.close()
|
|
self.httpConnection.connect()
|
|
res = self.doGet(httpGetCallback)
|
|
|
|
return res
|
|
|
|
|
|
#################################################################################################################################
|
|
class DirIndex:
|
|
|
|
def __init__(self, dirIndexFile):
|
|
self.d = []
|
|
self.f = []
|
|
self.version = 0
|
|
self.path = ""
|
|
|
|
with open(dirIndexFile) as f:
|
|
self.readFrom(f)
|
|
|
|
def readFrom(self, readable):
|
|
for line in readable:
|
|
line = line.strip()
|
|
if line.startswith('#'):
|
|
continue
|
|
|
|
tokens = line.split(':')
|
|
if len(tokens) == 0:
|
|
continue
|
|
|
|
if tokens[0] == "version":
|
|
self.version = int(tokens[1])
|
|
|
|
elif tokens[0] == "path":
|
|
self.path = tokens[1]
|
|
|
|
elif tokens[0] == "d":
|
|
self.d.append({ 'name': tokens[1], 'hash': tokens[2] })
|
|
|
|
elif tokens[0] == "f":
|
|
self.f.append({ 'name': tokens[1], 'hash': tokens[2], 'size': tokens[3] })
|
|
|
|
def getVersion(self):
|
|
return self.version
|
|
|
|
def getPath(self):
|
|
return self.path
|
|
|
|
def getDirectories(self):
|
|
return self.d
|
|
|
|
def getFiles(self):
|
|
return self.f
|
|
|
|
#################################################################################################################################
|
|
class HTTPDownloadRequest(HTTPGetCallback):
|
|
def __init__(self, terrasync, src, dst, callback = None ):
|
|
super().__init__(src, self.callback)
|
|
self.terrasync = terrasync
|
|
self.dst = dst
|
|
self.mycallback = callback
|
|
|
|
# 'httpResponse' is an http.client.HTTPResponse instance
|
|
def callback(self, url, httpResponse):
|
|
# I suspect this doesn't handle HTTP redirects and things like that. As
|
|
# mentioned at <https://docs.python.org/3/library/http.client.html>,
|
|
# http.client is a low-level interface that should normally not be used
|
|
# directly!
|
|
if httpResponse.status != 200:
|
|
raise NetworkError("HTTP callback got status {status} for URL {url}"
|
|
.format(status=httpResponse.status, url=url))
|
|
|
|
with open(self.dst, 'wb') as f:
|
|
f.write(httpResponse.read())
|
|
|
|
if self.mycallback != None:
|
|
self.mycallback(self)
|
|
|
|
#################################################################################################################################
|
|
|
|
def hash_of_file(fname):
|
|
hash = hashlib.sha1()
|
|
|
|
with open(fname, "rb") as f:
|
|
for chunk in iter(lambda: f.read(4096), b""):
|
|
hash.update(chunk)
|
|
|
|
return hash.hexdigest()
|
|
|
|
#################################################################################################################################
|
|
|
|
class Coordinate:
|
|
def __init__(self, lat, lon):
|
|
self.lat = lat
|
|
self.lon = lon
|
|
|
|
class DownloadBoundaries:
|
|
def __init__(self, top, left, bottom, right):
|
|
if top < bottom:
|
|
raise ValueError("top cannot be less than bottom")
|
|
if right < left:
|
|
raise ValueError("right cannot be less than left")
|
|
|
|
if top > 90 or bottom < -90:
|
|
raise ValueError("top and bottom must be a valid latitude")
|
|
if left < -180 or right > 180:
|
|
raise ValueError("left and right must be a valid longitude")
|
|
self.top = top
|
|
self.left = left
|
|
self.bottom = bottom
|
|
self.right = right
|
|
|
|
def is_coordinate_inside_boundaries(self, coordinate):
|
|
if coordinate.lat < self.bottom or coordinate.lat > self.top:
|
|
return False
|
|
if coordinate.lon < self.left or coordinate.lon > self.right:
|
|
return False
|
|
return True
|
|
|
|
def parse_terrasync_coordinate(coordinate):
|
|
matches = re.match("(w|e)(\d{3})(n|s)(\d{2})", coordinate)
|
|
if not matches:
|
|
return None
|
|
lon = int(matches.group(2))
|
|
if matches.group(1) == "w":
|
|
lon *= -1
|
|
lat = int(matches.group(4))
|
|
if matches.group(3) == "s":
|
|
lat *= -1
|
|
return Coordinate(lat, lon)
|
|
|
|
class TerraSync:
|
|
|
|
def __init__(self, url, target, quick, removeOrphan, downloadBoundaries):
|
|
self.setUrl(url).setTarget(target)
|
|
self.quick = quick
|
|
self.removeOrphan = removeOrphan
|
|
self.httpGetter = None
|
|
self.downloadBoundaries = downloadBoundaries
|
|
|
|
def setUrl(self, url):
|
|
self.url = url.rstrip('/').strip()
|
|
return self
|
|
|
|
def setTarget(self, target):
|
|
self.target = target.rstrip('/').strip()
|
|
return self
|
|
|
|
def start(self):
|
|
self.httpGetter = HTTPGetter(self.url)
|
|
self.updateDirectory("", "", None )
|
|
|
|
def updateFile(self, serverPath, localPath, fileHash ):
|
|
localFullPath = join(self.target, localPath)
|
|
if fileHash != None and hash_of_file(localFullPath) == fileHash:
|
|
#print("hash of file matches, not downloading")
|
|
return
|
|
|
|
print("Downloading '{}'".format(serverPath))
|
|
|
|
request = HTTPDownloadRequest(self, serverPath, localFullPath )
|
|
self.httpGetter.get(request)
|
|
|
|
|
|
def updateDirectory(self, serverPath, localPath, dirIndexHash):
|
|
print("Processing '{}'...".format(serverPath))
|
|
|
|
if serverPath:
|
|
serverFolderName = os.path.basename(serverPath)
|
|
coordinate = parse_terrasync_coordinate(serverFolderName)
|
|
if coordinate and not self.downloadBoundaries.is_coordinate_inside_boundaries(coordinate):
|
|
return
|
|
|
|
localFullPath = join(self.target, localPath)
|
|
if not os.path.exists( localFullPath ):
|
|
os.makedirs( localFullPath )
|
|
|
|
localDirIndex = join(localFullPath, ".dirindex")
|
|
if dirIndexHash != None and hash_of_file(localDirIndex) == dirIndexHash:
|
|
# print("hash of dirindex matches, not downloading")
|
|
if not self.quick:
|
|
self.handleDirindexFile( localDirIndex )
|
|
else:
|
|
request = HTTPDownloadRequest(self,
|
|
serverPath + "/.dirindex",
|
|
localDirIndex,
|
|
self.handleDirindexRequest)
|
|
self.httpGetter.get(request)
|
|
|
|
def handleDirindexRequest(self, dirindexRequest):
|
|
self.handleDirindexFile(dirindexRequest.dst)
|
|
|
|
def handleDirindexFile(self, dirindexFile):
|
|
dirIndex = DirIndex(dirindexFile)
|
|
serverFiles = []
|
|
serverDirs = []
|
|
|
|
for file in dirIndex.getFiles():
|
|
f = file['name']
|
|
h = file['hash']
|
|
self.updateFile("/" + dirIndex.getPath() + "/" + f,
|
|
join(dirIndex.getPath(), f),
|
|
h)
|
|
serverFiles.append(f)
|
|
|
|
for subdir in dirIndex.getDirectories():
|
|
d = subdir['name']
|
|
h = subdir['hash']
|
|
self.updateDirectory("/" + dirIndex.getPath() + "/" + d,
|
|
join(dirIndex.getPath(), d),
|
|
h)
|
|
serverDirs.append(d)
|
|
|
|
if self.removeOrphan:
|
|
localFullPath = join(self.target, dirIndex.getPath())
|
|
localFiles = [ f for f in listdir(localFullPath)
|
|
if isfile(join(localFullPath, f)) ]
|
|
for f in localFiles:
|
|
if f != ".dirindex" and not f in serverFiles:
|
|
#print("removing orphan file", join(localFullPath,f) )
|
|
os.remove( join(localFullPath,f) )
|
|
localDirs = [ f for f in listdir(localFullPath)
|
|
if isdir(join(localFullPath, f)) ]
|
|
for f in localDirs:
|
|
if not f in serverDirs:
|
|
#print ("removing orphan dir",f)
|
|
shutil.rmtree( join(localFullPath,f) )
|
|
|
|
#################################################################################################################################
|
|
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("-u", "--url", dest="url", metavar="URL",
|
|
default="http://flightgear.sourceforge.net/scenery", help="Server URL [default: %(default)s]")
|
|
parser.add_argument("-t", "--target", dest="target", metavar="DIR",
|
|
default=".", help="Directory to store the files [default: current directory]")
|
|
parser.add_argument("-q", "--quick", dest="quick", action="store_true",
|
|
default=False, help="Quick")
|
|
parser.add_argument("-r", "--remove-orphan", dest="removeOrphan", action="store_true",
|
|
default=False, help="Remove old scenery files")
|
|
|
|
parser.add_argument("--top", dest="top", type=int,
|
|
default=90, help="Maximum latitude to include in download [default: %(default)d]")
|
|
parser.add_argument("--bottom", dest="bottom", type=int,
|
|
default=-90, help="Minimum latitude to include in download [default: %(default)d]")
|
|
parser.add_argument("--left", dest="left", type=int,
|
|
default=-180, help="Minimum longitude to include in download [default: %(default)d]")
|
|
parser.add_argument("--right", dest="right", type=int,
|
|
default=180, help="Maximum longitude to include in download [default: %(default)d]")
|
|
|
|
args = parser.parse_args()
|
|
|
|
terraSync = TerraSync(args.url, args.target, args.quick, args.removeOrphan,
|
|
DownloadBoundaries(args.top, args.left, args.bottom, args.right))
|
|
|
|
terraSync.start()
|