#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright (C) 2016 Torsten Dreyer # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of the # License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # terrasync.py - synchronize terrascenery data to your local disk # needs dnspython (pip install dnspython) import urllib, os, hashlib from urllib.parse import urlparse from http.client import HTTPConnection, _CS_IDLE from os import listdir from os.path import isfile, join ################################################################################################################################# class HTTPGetCallback: def __init__(self, src, callback): self.callback = callback self.src = src self.result = None class HTTPGetter: def __init__(self, baseUrl, maxPending=10): self.baseUrl = baseUrl self.parsedBaseUrl = urlparse(baseUrl) self.maxPending = maxPending self.requests = [] self.pendingRequests = [] self.httpConnection = HTTPConnection(self.parsedBaseUrl.netloc,80, True) self.httpRequestHeaders = headers = {'Host':self.parsedBaseUrl.netloc,'Content-Length':0,'Connection':'Keep-Alive','User-Agent':'FlightGear terrasync.py'} def get(self, httpGetCallback): #self.requests.append(httpGetCallback) conn = self.httpConnection request = httpGetCallback conn.request("GET", self.parsedBaseUrl.path + request.src, None, self.httpRequestHeaders) try: httpGetCallback.result = conn.getresponse() except: # try to reconnect once #print("reconnect") conn.close() conn.connect() conn.request("GET", self.parsedBaseUrl.path + request.src, None, self.httpRequestHeaders) httpGetCallback.result = conn.getresponse() httpGetCallback.callback() #self.requests.remove(httpGetCallback) ################################################################################################################################# class DirIndex: def __init__(self, dirIndexFile): self.d = [] self.f = [] self.version = 0 self.path = "" with open(dirIndexFile) as f: self.readFrom(f) def readFrom(self, readable): for line in readable: line = line.strip() if line.startswith('#'): continue tokens = line.split(':') if len(tokens) == 0: continue if tokens[0] == "version": self.version = int(tokens[1]) elif tokens[0] == "path": self.path = tokens[1] elif tokens[0] == "d": self.d.append({ 'name': tokens[1], 'hash': tokens[2] }) elif tokens[0] == "f": self.f.append({ 'name': tokens[1], 'hash': tokens[2], 'size': tokens[3] }) def getVersion(self): return self.version def getPath(self): return self.path def getDirectories(self): return self.d def getFiles(self): return self.f ################################################################################################################################# class HTTPDownloadRequest(HTTPGetCallback): def __init__(self, terrasync, src, dst, callback = None ): super().__init__(src, self.callback) self.terrasync = terrasync self.dst = dst self.mycallback = callback def callback(self): with open(self.dst, 'wb') as f: f.write(self.result.read()) if self.mycallback != None: self.mycallback(self) ################################################################################################################################# def hash_of_file(fname): if not os.path.exists( fname ): return None hash = hashlib.sha1() try: with open(fname, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): hash.update(chunk) except: pass return hash.hexdigest() ################################################################################################################################# class TerraSync: def __init__(self, url="http://flightgear.sourceforge.net/scenery", target=".", quick=False, removeOrphan=False): self.setUrl(url).setTarget(target) self.quick = quick self.removeOrphan = removeOrphan self.httpGetter = None def setUrl(self, url): self.url = url.rstrip('/').strip() return self def setTarget(self, target): self.target = target.rstrip('/').strip() return self def start(self): self.httpGetter = HTTPGetter(self.url) self.updateDirectory("", "", None ) def updateFile(self, serverPath, localPath, fileHash ): localFullPath = join(self.target, localPath) if fileHash != None and hash_of_file(localFullPath) == fileHash: #print("hash of file matches, not downloading") return print("downloading ", serverPath ) request = HTTPDownloadRequest(self, serverPath, localFullPath ) self.httpGetter.get(request) def updateDirectory(self, serverPath, localPath, dirIndexHash): print("processing ", serverPath) localFullPath = join(self.target, localPath) if not os.path.exists( localFullPath ): os.makedirs( localFullPath ) localDirIndex = join(localFullPath, ".dirindex") if dirIndexHash != None and hash_of_file(localDirIndex) == dirIndexHash: # print("hash of dirindex matches, not downloading") if not self.quick: self.handleDirindexFile( localDirIndex ) else: request = HTTPDownloadRequest(self, serverPath + "/.dirindex", localDirIndex, self.handleDirindexRequest ) self.httpGetter.get(request) def handleDirindexRequest(self, dirindexRequest): self.handleDirindexFile(dirindexRequest.dst) def handleDirindexFile(self, dirindexFile): dirIndex = DirIndex(dirindexFile) serverFiles = [] for file in dirIndex.getFiles(): f = file['name'] h = file['hash'] self.updateFile( "/" + dirIndex.getPath() + "/" + f, join(dirIndex.getPath(),f), h ) serverFiles.append(f) for subdir in dirIndex.getDirectories(): d = subdir['name'] h = subdir['hash'] self.updateDirectory( "/" + dirIndex.getPath() + "/" + d, join(dirIndex.getPath(),d), h ) if self.removeOrphan: localFullPath = join(self.target, dirIndex.getPath()) localFiles = [f for f in listdir(localFullPath) if isfile(join(localFullPath, f))] for f in localFiles: if f != ".dirindex" and not f in serverFiles: #print("removing orphan", join(localFullPath,f) ) os.remove( join(localFullPath,f) ) def isReady(self): return self.httpGetter and self.httpGetter.isReady() return False def update(self): if self.httpGetter: self.httpGetter.update() ################################################################################################################################# import getopt, sys try: opts, args = getopt.getopt(sys.argv[1:], "u:t:qr", [ "url=", "target=", "quick", "remove-orphan" ]) except getopt.GetoptError: print("terrasync.py [--url=http://some.server.org/scenery] [--target=/some/path] [-q|--quick] [-r|--remove-orphan]") sys.exit(2) terraSync = TerraSync() for opt, arg in opts: if opt in("-u", "--url"): terraSync.url = arg elif opt in ("-t", "--target"): terraSync.target = arg elif opt in ("-q", "--quick"): terraSync.quick = True elif opt in ("-r", "--remove-orphan"): terraSync.removeOrphan = True terraSync.start()