2c5429c589
If the response to the HTTP request isn't 200 (success), then don't save the response, and don't call the callback. Additionally, only retry in the case of HTTPException. This allows using Ctrl-C to work correctly (and easily).
252 lines
8.4 KiB
Python
Executable file
252 lines
8.4 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# Copyright (C) 2016 Torsten Dreyer
|
|
#
|
|
# This program is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU General Public License as
|
|
# published by the Free Software Foundation; either version 2 of the
|
|
# License, or (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful, but
|
|
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
# General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
#
|
|
# terrasync.py - synchronize terrascenery data to your local disk
|
|
# needs dnspython (pip install dnspython)
|
|
|
|
import urllib, os, hashlib
|
|
from urllib.parse import urlparse
|
|
from http.client import HTTPConnection, _CS_IDLE, HTTPException
|
|
from os import listdir
|
|
from os.path import isfile, join
|
|
|
|
#################################################################################################################################
|
|
class HTTPGetCallback:
|
|
def __init__(self, src, callback):
|
|
self.callback = callback
|
|
self.src = src
|
|
self.result = None
|
|
|
|
class HTTPGetter:
|
|
def __init__(self, baseUrl, maxPending=10):
|
|
self.baseUrl = baseUrl
|
|
self.parsedBaseUrl = urlparse(baseUrl)
|
|
self.maxPending = maxPending
|
|
self.requests = []
|
|
self.pendingRequests = []
|
|
self.httpConnection = HTTPConnection(self.parsedBaseUrl.netloc)
|
|
self.httpRequestHeaders = headers = {'Host':self.parsedBaseUrl.netloc,'Content-Length':0,'Connection':'Keep-Alive','User-Agent':'FlightGear terrasync.py'}
|
|
|
|
def doGet(self, httpGetCallback):
|
|
conn = self.httpConnection
|
|
request = httpGetCallback
|
|
self.httpConnection.request("GET", self.parsedBaseUrl.path + request.src, None, self.httpRequestHeaders)
|
|
httpGetCallback.result = self.httpConnection.getresponse()
|
|
httpGetCallback.callback()
|
|
|
|
def get(self, httpGetCallback):
|
|
|
|
try:
|
|
self.doGet(httpGetCallback)
|
|
except HTTPException:
|
|
# try to reconnect once
|
|
#print("reconnect")
|
|
self.httpConnection.close()
|
|
self.httpConnection.connect()
|
|
self.doGet(httpGetCallback)
|
|
|
|
|
|
#################################################################################################################################
|
|
class DirIndex:
|
|
|
|
def __init__(self, dirIndexFile):
|
|
self.d = []
|
|
self.f = []
|
|
self.version = 0
|
|
self.path = ""
|
|
|
|
with open(dirIndexFile) as f:
|
|
self.readFrom(f)
|
|
|
|
def readFrom(self, readable):
|
|
for line in readable:
|
|
line = line.strip()
|
|
if line.startswith('#'):
|
|
continue
|
|
|
|
tokens = line.split(':')
|
|
if len(tokens) == 0:
|
|
continue
|
|
|
|
if tokens[0] == "version":
|
|
self.version = int(tokens[1])
|
|
|
|
elif tokens[0] == "path":
|
|
self.path = tokens[1]
|
|
|
|
elif tokens[0] == "d":
|
|
self.d.append({ 'name': tokens[1], 'hash': tokens[2] })
|
|
|
|
elif tokens[0] == "f":
|
|
self.f.append({ 'name': tokens[1], 'hash': tokens[2], 'size': tokens[3] })
|
|
|
|
def getVersion(self):
|
|
return self.version
|
|
|
|
def getPath(self):
|
|
return self.path
|
|
|
|
def getDirectories(self):
|
|
return self.d
|
|
|
|
def getFiles(self):
|
|
return self.f
|
|
|
|
#################################################################################################################################
|
|
class HTTPDownloadRequest(HTTPGetCallback):
|
|
def __init__(self, terrasync, src, dst, callback = None ):
|
|
super().__init__(src, self.callback)
|
|
self.terrasync = terrasync
|
|
self.dst = dst
|
|
self.mycallback = callback
|
|
|
|
def callback(self):
|
|
if self.result.status != 200:
|
|
return
|
|
|
|
with open(self.dst, 'wb') as f:
|
|
f.write(self.result.read())
|
|
|
|
if self.mycallback != None:
|
|
self.mycallback(self)
|
|
|
|
#################################################################################################################################
|
|
|
|
def hash_of_file(fname):
|
|
if not os.path.exists( fname ):
|
|
return None
|
|
|
|
hash = hashlib.sha1()
|
|
try:
|
|
with open(fname, "rb") as f:
|
|
for chunk in iter(lambda: f.read(4096), b""):
|
|
hash.update(chunk)
|
|
except:
|
|
pass
|
|
|
|
return hash.hexdigest()
|
|
|
|
#################################################################################################################################
|
|
class TerraSync:
|
|
|
|
def __init__(self, url="http://flightgear.sourceforge.net/scenery", target=".", quick=False, removeOrphan=False):
|
|
self.setUrl(url).setTarget(target)
|
|
self.quick = quick
|
|
self.removeOrphan = removeOrphan
|
|
self.httpGetter = None
|
|
|
|
def setUrl(self, url):
|
|
self.url = url.rstrip('/').strip()
|
|
return self
|
|
|
|
def setTarget(self, target):
|
|
self.target = target.rstrip('/').strip()
|
|
return self
|
|
|
|
def start(self):
|
|
self.httpGetter = HTTPGetter(self.url)
|
|
self.updateDirectory("", "", None )
|
|
|
|
def updateFile(self, serverPath, localPath, fileHash ):
|
|
localFullPath = join(self.target, localPath)
|
|
if fileHash != None and hash_of_file(localFullPath) == fileHash:
|
|
#print("hash of file matches, not downloading")
|
|
return
|
|
|
|
print("downloading ", serverPath )
|
|
|
|
request = HTTPDownloadRequest(self, serverPath, localFullPath )
|
|
self.httpGetter.get(request)
|
|
|
|
|
|
def updateDirectory(self, serverPath, localPath, dirIndexHash):
|
|
print("processing ", serverPath)
|
|
|
|
localFullPath = join(self.target, localPath)
|
|
if not os.path.exists( localFullPath ):
|
|
os.makedirs( localFullPath )
|
|
|
|
localDirIndex = join(localFullPath, ".dirindex")
|
|
if dirIndexHash != None and hash_of_file(localDirIndex) == dirIndexHash:
|
|
# print("hash of dirindex matches, not downloading")
|
|
if not self.quick:
|
|
self.handleDirindexFile( localDirIndex )
|
|
else:
|
|
request = HTTPDownloadRequest(self, serverPath + "/.dirindex", localDirIndex, self.handleDirindexRequest )
|
|
self.httpGetter.get(request)
|
|
|
|
def handleDirindexRequest(self, dirindexRequest):
|
|
self.handleDirindexFile(dirindexRequest.dst)
|
|
|
|
def handleDirindexFile(self, dirindexFile):
|
|
dirIndex = DirIndex(dirindexFile)
|
|
serverFiles = []
|
|
|
|
for file in dirIndex.getFiles():
|
|
f = file['name']
|
|
h = file['hash']
|
|
self.updateFile( "/" + dirIndex.getPath() + "/" + f, join(dirIndex.getPath(),f), h )
|
|
serverFiles.append(f)
|
|
|
|
for subdir in dirIndex.getDirectories():
|
|
d = subdir['name']
|
|
h = subdir['hash']
|
|
self.updateDirectory( "/" + dirIndex.getPath() + "/" + d, join(dirIndex.getPath(),d), h )
|
|
|
|
if self.removeOrphan:
|
|
localFullPath = join(self.target, dirIndex.getPath())
|
|
localFiles = [f for f in listdir(localFullPath) if isfile(join(localFullPath, f))]
|
|
for f in localFiles:
|
|
if f != ".dirindex" and not f in serverFiles:
|
|
#print("removing orphan", join(localFullPath,f) )
|
|
os.remove( join(localFullPath,f) )
|
|
|
|
|
|
def isReady(self):
|
|
return self.httpGetter and self.httpGetter.isReady()
|
|
return False
|
|
|
|
def update(self):
|
|
if self.httpGetter:
|
|
self.httpGetter.update()
|
|
|
|
#################################################################################################################################
|
|
import getopt, sys
|
|
try:
|
|
opts, args = getopt.getopt(sys.argv[1:], "u:t:qr", [ "url=", "target=", "quick", "remove-orphan" ])
|
|
|
|
except getopt.GetoptError:
|
|
print("terrasync.py [--url=http://some.server.org/scenery] [--target=/some/path] [-q|--quick] [-r|--remove-orphan]")
|
|
sys.exit(2)
|
|
|
|
terraSync = TerraSync()
|
|
for opt, arg in opts:
|
|
if opt in("-u", "--url"):
|
|
terraSync.url = arg
|
|
|
|
elif opt in ("-t", "--target"):
|
|
terraSync.target = arg
|
|
|
|
elif opt in ("-q", "--quick"):
|
|
terraSync.quick = True
|
|
|
|
elif opt in ("-r", "--remove-orphan"):
|
|
terraSync.removeOrphan = True
|
|
|
|
terraSync.start()
|