19209a71f0
When called, the callback passed to HTTPGetter.get() is now explicitly passed the URL and the http.client.HTTPResponse instance. Remove the HTTPGetCallback.result attribute (not needed anymore, leaves more freedom when implementating HTTPGetCallback subclasses...).
317 lines
11 KiB
Python
Executable file
317 lines
11 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# Copyright (C) 2016 Torsten Dreyer
|
|
#
|
|
# This program is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU General Public License as
|
|
# published by the Free Software Foundation; either version 2 of the
|
|
# License, or (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful, but
|
|
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
# General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
#
|
|
# terrasync.py - synchronize terrascenery data to your local disk
|
|
# needs dnspython (pip install dnspython)
|
|
|
|
import urllib, os, hashlib
|
|
from urllib.parse import urlparse
|
|
from http.client import HTTPConnection, _CS_IDLE, HTTPException
|
|
from os import listdir
|
|
from os.path import isfile, isdir, join
|
|
import re
|
|
import argparse
|
|
import shutil
|
|
|
|
#################################################################################################################################
|
|
class HTTPGetCallback:
|
|
def __init__(self, src, callback):
|
|
self.callback = callback
|
|
self.src = src
|
|
|
|
class HTTPGetter:
|
|
def __init__(self, baseUrl, maxPending=10):
|
|
self.baseUrl = baseUrl
|
|
self.parsedBaseUrl = urlparse(baseUrl)
|
|
self.maxPending = maxPending
|
|
self.requests = []
|
|
self.pendingRequests = []
|
|
self.httpConnection = HTTPConnection(self.parsedBaseUrl.netloc)
|
|
self.httpRequestHeaders = headers = {'Host':self.parsedBaseUrl.netloc,'Content-Length':0,'Connection':'Keep-Alive','User-Agent':'FlightGear terrasync.py'}
|
|
|
|
def doGet(self, httpGetCallback):
|
|
conn = self.httpConnection
|
|
url = self.parsedBaseUrl.path + httpGetCallback.src
|
|
self.httpConnection.request("GET", url, None, self.httpRequestHeaders)
|
|
httpResponse = self.httpConnection.getresponse()
|
|
|
|
# 'httpResponse' is an http.client.HTTPResponse instance
|
|
return httpGetCallback.callback(url, httpResponse)
|
|
|
|
def get(self, httpGetCallback):
|
|
try:
|
|
res = self.doGet(httpGetCallback)
|
|
except HTTPException:
|
|
# try to reconnect once
|
|
#print("reconnect")
|
|
self.httpConnection.close()
|
|
self.httpConnection.connect()
|
|
res = self.doGet(httpGetCallback)
|
|
|
|
return res
|
|
|
|
|
|
#################################################################################################################################
|
|
class DirIndex:
|
|
|
|
def __init__(self, dirIndexFile):
|
|
self.d = []
|
|
self.f = []
|
|
self.version = 0
|
|
self.path = ""
|
|
|
|
with open(dirIndexFile) as f:
|
|
self.readFrom(f)
|
|
|
|
def readFrom(self, readable):
|
|
for line in readable:
|
|
line = line.strip()
|
|
if line.startswith('#'):
|
|
continue
|
|
|
|
tokens = line.split(':')
|
|
if len(tokens) == 0:
|
|
continue
|
|
|
|
if tokens[0] == "version":
|
|
self.version = int(tokens[1])
|
|
|
|
elif tokens[0] == "path":
|
|
self.path = tokens[1]
|
|
|
|
elif tokens[0] == "d":
|
|
self.d.append({ 'name': tokens[1], 'hash': tokens[2] })
|
|
|
|
elif tokens[0] == "f":
|
|
self.f.append({ 'name': tokens[1], 'hash': tokens[2], 'size': tokens[3] })
|
|
|
|
def getVersion(self):
|
|
return self.version
|
|
|
|
def getPath(self):
|
|
return self.path
|
|
|
|
def getDirectories(self):
|
|
return self.d
|
|
|
|
def getFiles(self):
|
|
return self.f
|
|
|
|
#################################################################################################################################
|
|
class HTTPDownloadRequest(HTTPGetCallback):
|
|
def __init__(self, terrasync, src, dst, callback = None ):
|
|
super().__init__(src, self.callback)
|
|
self.terrasync = terrasync
|
|
self.dst = dst
|
|
self.mycallback = callback
|
|
|
|
# 'httpResponse' is an http.client.HTTPResponse instance
|
|
def callback(self, url, httpResponse):
|
|
if httpResponse.status != 200:
|
|
return
|
|
|
|
with open(self.dst, 'wb') as f:
|
|
f.write(httpResponse.read())
|
|
|
|
if self.mycallback != None:
|
|
self.mycallback(self)
|
|
|
|
#################################################################################################################################
|
|
|
|
def hash_of_file(fname):
|
|
if not os.path.exists( fname ):
|
|
return None
|
|
|
|
hash = hashlib.sha1()
|
|
try:
|
|
with open(fname, "rb") as f:
|
|
for chunk in iter(lambda: f.read(4096), b""):
|
|
hash.update(chunk)
|
|
except:
|
|
pass
|
|
|
|
return hash.hexdigest()
|
|
|
|
#################################################################################################################################
|
|
|
|
class Coordinate:
|
|
def __init__(self, lat, lon):
|
|
self.lat = lat
|
|
self.lon = lon
|
|
|
|
class DownloadBoundaries:
|
|
def __init__(self, top, left, bottom, right):
|
|
if top < bottom:
|
|
raise ValueError("top cannot be less than bottom")
|
|
if right < left:
|
|
raise ValueError("right cannot be less than left")
|
|
|
|
if top > 90 or bottom < -90:
|
|
raise ValueError("top and bottom must be a valid latitude")
|
|
if left < -180 or right > 180:
|
|
raise ValueError("left and right must be a valid longitude")
|
|
self.top = top
|
|
self.left = left
|
|
self.bottom = bottom
|
|
self.right = right
|
|
|
|
def is_coordinate_inside_boundaries(self, coordinate):
|
|
if coordinate.lat < self.bottom or coordinate.lat > self.top:
|
|
return False
|
|
if coordinate.lon < self.left or coordinate.lon > self.right:
|
|
return False
|
|
return True
|
|
|
|
def parse_terrasync_coordinate(coordinate):
|
|
matches = re.match("(w|e)(\d{3})(n|s)(\d{2})", coordinate)
|
|
if not matches:
|
|
return None
|
|
lon = int(matches.group(2))
|
|
if matches.group(1) == "w":
|
|
lon *= -1
|
|
lat = int(matches.group(4))
|
|
if matches.group(3) == "s":
|
|
lat *= -1
|
|
return Coordinate(lat, lon)
|
|
|
|
class TerraSync:
|
|
|
|
def __init__(self, url, target, quick, removeOrphan, downloadBoundaries):
|
|
self.setUrl(url).setTarget(target)
|
|
self.quick = quick
|
|
self.removeOrphan = removeOrphan
|
|
self.httpGetter = None
|
|
self.downloadBoundaries = downloadBoundaries
|
|
|
|
def setUrl(self, url):
|
|
self.url = url.rstrip('/').strip()
|
|
return self
|
|
|
|
def setTarget(self, target):
|
|
self.target = target.rstrip('/').strip()
|
|
return self
|
|
|
|
def start(self):
|
|
self.httpGetter = HTTPGetter(self.url)
|
|
self.updateDirectory("", "", None )
|
|
|
|
def updateFile(self, serverPath, localPath, fileHash ):
|
|
localFullPath = join(self.target, localPath)
|
|
if fileHash != None and hash_of_file(localFullPath) == fileHash:
|
|
#print("hash of file matches, not downloading")
|
|
return
|
|
|
|
print("Downloading '{}'".format(serverPath))
|
|
|
|
request = HTTPDownloadRequest(self, serverPath, localFullPath )
|
|
self.httpGetter.get(request)
|
|
|
|
|
|
def updateDirectory(self, serverPath, localPath, dirIndexHash):
|
|
print("Processing '{}'...".format(serverPath))
|
|
|
|
if serverPath:
|
|
serverFolderName = os.path.basename(serverPath)
|
|
coordinate = parse_terrasync_coordinate(serverFolderName)
|
|
if coordinate and not self.downloadBoundaries.is_coordinate_inside_boundaries(coordinate):
|
|
return
|
|
|
|
localFullPath = join(self.target, localPath)
|
|
if not os.path.exists( localFullPath ):
|
|
os.makedirs( localFullPath )
|
|
|
|
localDirIndex = join(localFullPath, ".dirindex")
|
|
if dirIndexHash != None and hash_of_file(localDirIndex) == dirIndexHash:
|
|
# print("hash of dirindex matches, not downloading")
|
|
if not self.quick:
|
|
self.handleDirindexFile( localDirIndex )
|
|
else:
|
|
request = HTTPDownloadRequest(self,
|
|
serverPath + "/.dirindex",
|
|
localDirIndex,
|
|
self.handleDirindexRequest)
|
|
self.httpGetter.get(request)
|
|
|
|
def handleDirindexRequest(self, dirindexRequest):
|
|
self.handleDirindexFile(dirindexRequest.dst)
|
|
|
|
def handleDirindexFile(self, dirindexFile):
|
|
dirIndex = DirIndex(dirindexFile)
|
|
serverFiles = []
|
|
serverDirs = []
|
|
|
|
for file in dirIndex.getFiles():
|
|
f = file['name']
|
|
h = file['hash']
|
|
self.updateFile("/" + dirIndex.getPath() + "/" + f,
|
|
join(dirIndex.getPath(), f),
|
|
h)
|
|
serverFiles.append(f)
|
|
|
|
for subdir in dirIndex.getDirectories():
|
|
d = subdir['name']
|
|
h = subdir['hash']
|
|
self.updateDirectory("/" + dirIndex.getPath() + "/" + d,
|
|
join(dirIndex.getPath(), d),
|
|
h)
|
|
serverDirs.append(d)
|
|
|
|
if self.removeOrphan:
|
|
localFullPath = join(self.target, dirIndex.getPath())
|
|
localFiles = [ f for f in listdir(localFullPath)
|
|
if isfile(join(localFullPath, f)) ]
|
|
for f in localFiles:
|
|
if f != ".dirindex" and not f in serverFiles:
|
|
#print("removing orphan file", join(localFullPath,f) )
|
|
os.remove( join(localFullPath,f) )
|
|
localDirs = [ f for f in listdir(localFullPath)
|
|
if isdir(join(localFullPath, f)) ]
|
|
for f in localDirs:
|
|
if not f in serverDirs:
|
|
#print ("removing orphan dir",f)
|
|
shutil.rmtree( join(localFullPath,f) )
|
|
|
|
#################################################################################################################################
|
|
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("-u", "--url", dest="url", metavar="URL",
|
|
default="http://flightgear.sourceforge.net/scenery", help="Server URL [default: %(default)s]")
|
|
parser.add_argument("-t", "--target", dest="target", metavar="DIR",
|
|
default=".", help="Directory to store the files [default: current directory]")
|
|
parser.add_argument("-q", "--quick", dest="quick", action="store_true",
|
|
default=False, help="Quick")
|
|
parser.add_argument("-r", "--remove-orphan", dest="removeOrphan", action="store_true",
|
|
default=False, help="Remove old scenery files")
|
|
|
|
parser.add_argument("--top", dest="top", type=int,
|
|
default=90, help="Maximum latitude to include in download [default: %(default)d]")
|
|
parser.add_argument("--bottom", dest="bottom", type=int,
|
|
default=-90, help="Minimum latitude to include in download [default: %(default)d]")
|
|
parser.add_argument("--left", dest="left", type=int,
|
|
default=-180, help="Minimum longitude to include in download [default: %(default)d]")
|
|
parser.add_argument("--right", dest="right", type=int,
|
|
default=180, help="Maximum longitude to include in download [default: %(default)d]")
|
|
|
|
args = parser.parse_args()
|
|
|
|
terraSync = TerraSync(args.url, args.target, args.quick, args.removeOrphan,
|
|
DownloadBoundaries(args.top, args.left, args.bottom, args.right))
|
|
|
|
terraSync.start()
|