flightgear/scripts/python/terrasync.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2016  Torsten Dreyer
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#
# terrasync.py - synchronize terrascenery data to your local disk
# needs dnspython (pip install dnspython)

import urllib, os, hashlib
from urllib.parse import urlparse
from http.client import HTTPConnection, _CS_IDLE, HTTPException
from os import listdir
from os.path import isfile, isdir, join
import re
import argparse
import shutil

#################################################################################################################################
class HTTPGetCallback:
    def __init__(self, src, callback):
        self.callback = callback
        self.src = src
        self.result = None

class HTTPGetter:
    def __init__(self, baseUrl, maxPending=10):
        self.baseUrl = baseUrl
        self.parsedBaseUrl = urlparse(baseUrl)
        self.maxPending = maxPending
        self.requests = []
        self.pendingRequests = []
        self.httpConnection = HTTPConnection(self.parsedBaseUrl.netloc)
        self.httpRequestHeaders = headers = {'Host':self.parsedBaseUrl.netloc,'Content-Length':0,'Connection':'Keep-Alive','User-Agent':'FlightGear terrasync.py'}

    def doGet(self, httpGetCallback):
        conn = self.httpConnection
        request = httpGetCallback
        self.httpConnection.request("GET", self.parsedBaseUrl.path + request.src, None, self.httpRequestHeaders)
        httpGetCallback.result = self.httpConnection.getresponse()
        httpGetCallback.callback()

    def get(self, httpGetCallback):

        try:
            self.doGet(httpGetCallback)
        except HTTPException:
            # try to reconnect once
            #print("reconnect")
            self.httpConnection.close()
            self.httpConnection.connect()
            self.doGet(httpGetCallback)


#################################################################################################################################
class DirIndex:

    def __init__(self, dirIndexFile):
        self.d = []
        self.f = []
        self.version = 0
        self.path = ""

        with open(dirIndexFile) as f:
            self.readFrom(f)

    def readFrom(self, readable):
        for line in readable:
            line = line.strip()
            if line.startswith('#'):
                continue

            tokens = line.split(':')
            if len(tokens) == 0:
                continue

            if tokens[0] == "version":
                self.version = int(tokens[1])

            elif tokens[0] == "path":
                self.path = tokens[1]

            elif tokens[0] == "d":
                self.d.append({ 'name': tokens[1], 'hash': tokens[2] })

            elif tokens[0] == "f":
                self.f.append({ 'name': tokens[1], 'hash': tokens[2], 'size': tokens[3] })

    def getVersion(self):
        return self.version

    def getPath(self):
        return self.path

    def getDirectories(self):
        return self.d

    def getFiles(self):
        return self.f

#################################################################################################################################
class HTTPDownloadRequest(HTTPGetCallback):
    def __init__(self, terrasync, src, dst, callback = None ):
        super().__init__(src, self.callback)
        self.terrasync = terrasync
        self.dst = dst
        self.mycallback = callback

    def callback(self):
        if self.result.status != 200:
            return

        with open(self.dst, 'wb') as f:
            f.write(self.result.read())

        if self.mycallback != None:
            self.mycallback(self)

#################################################################################################################################

def hash_of_file(fname):
    if not os.path.exists( fname ):
      return None

    hash = hashlib.sha1()
    try:
        with open(fname, "rb") as f:
            for chunk in iter(lambda: f.read(4096), b""):
                hash.update(chunk)
    except:
        pass

    return hash.hexdigest()

#################################################################################################################################

class Coordinate:
    def __init__(self, lat, lon):
        self.lat = lat
        self.lon = lon

class DownloadBoundaries:
    def __init__(self, top, left, bottom, right):
        if top < bottom:
            raise ValueError("top cannot be less than bottom")
        if right < left:
            raise ValueError("right cannot be less than left")

        if top > 90 or bottom < -90:
            raise ValueError("top and bottom must be a valid latitude")
        if left < -180 or right > 180:
            raise ValueError("left and right must be a valid longitude")
        self.top = top
        self.left = left
        self.bottom = bottom
        self.right = right

    def is_coordinate_inside_boundaries(self, coordinate):
        if coordinate.lat < self.bottom or coordinate.lat > self.top:
            return False
        if coordinate.lon < self.left or coordinate.lon > self.right:
            return False
        return True

def parse_terrasync_coordinate(coordinate):
    matches = re.match("(w|e)(\d{3})(n|s)(\d{2})", coordinate)
    if not matches:
        return None
    lon = int(matches.group(2))
    if matches.group(1) == "w":
        lon *= -1
    lat = int(matches.group(4))
    if matches.group(3) == "s":
        lat *= -1
    return Coordinate(lat, lon)

class TerraSync:

    def __init__(self, url, target, quick, removeOrphan, downloadBoundaries):
        self.setUrl(url).setTarget(target)
        self.quick = quick
        self.removeOrphan = removeOrphan
        self.httpGetter = None
        self.downloadBoundaries = downloadBoundaries

    def setUrl(self, url):
        self.url = url.rstrip('/').strip()
        return self

    def setTarget(self, target):
        self.target = target.rstrip('/').strip()
        return self

    def start(self):
        self.httpGetter = HTTPGetter(self.url)
        self.updateDirectory("", "", None )

    def updateFile(self, serverPath, localPath, fileHash ):
        localFullPath = join(self.target, localPath)
        if fileHash != None and hash_of_file(localFullPath) == fileHash:
            #print("hash of file matches, not downloading")
            return

        print("Downloading '{}'".format(serverPath))

        request = HTTPDownloadRequest(self, serverPath, localFullPath )
        self.httpGetter.get(request)


    def updateDirectory(self, serverPath, localPath, dirIndexHash):
        print("Processing '{}'...".format(serverPath))

        if serverPath:
            serverFolderName = os.path.basename(serverPath)
            coordinate = parse_terrasync_coordinate(serverFolderName)
            if coordinate and not self.downloadBoundaries.is_coordinate_inside_boundaries(coordinate):
                return

        localFullPath = join(self.target, localPath)
        if not os.path.exists( localFullPath ):
          os.makedirs( localFullPath )

        localDirIndex = join(localFullPath, ".dirindex")
        if dirIndexHash != None and  hash_of_file(localDirIndex) == dirIndexHash:
            # print("hash of dirindex matches, not downloading")
            if not self.quick:
                self.handleDirindexFile( localDirIndex )
        else:
            request = HTTPDownloadRequest(self,
                                          serverPath + "/.dirindex",
                                          localDirIndex,
                                          self.handleDirindexRequest)
            self.httpGetter.get(request)

    def handleDirindexRequest(self, dirindexRequest):
        self.handleDirindexFile(dirindexRequest.dst)

    def handleDirindexFile(self, dirindexFile):
        dirIndex = DirIndex(dirindexFile)
        serverFiles = []
        serverDirs = []

        for file in dirIndex.getFiles():
            f = file['name']
            h = file['hash']
            self.updateFile("/" + dirIndex.getPath() + "/" + f,
                            join(dirIndex.getPath(), f),
                            h)
            serverFiles.append(f)

        for subdir in dirIndex.getDirectories():
            d = subdir['name']
            h = subdir['hash']
            self.updateDirectory("/" + dirIndex.getPath() + "/" + d,
                                 join(dirIndex.getPath(), d),
                                 h)
            serverDirs.append(d)

        if self.removeOrphan:
            localFullPath = join(self.target, dirIndex.getPath())
            localFiles = [ f for f in listdir(localFullPath)
                           if isfile(join(localFullPath, f)) ]
            for f in localFiles:
                if f != ".dirindex" and not f in serverFiles:
                    #print("removing orphan file", join(localFullPath,f) )
                    os.remove( join(localFullPath,f) )
            localDirs = [ f for f in listdir(localFullPath)
                          if isdir(join(localFullPath, f)) ]
            for f in localDirs:
                if not f in serverDirs:
                    #print ("removing orphan dir",f)
                    shutil.rmtree( join(localFullPath,f) )

#################################################################################################################################


parser = argparse.ArgumentParser()
parser.add_argument("-u", "--url", dest="url", metavar="URL",
        default="http://flightgear.sourceforge.net/scenery", help="Server URL [default: %(default)s]")
parser.add_argument("-t", "--target", dest="target", metavar="DIR",
        default=".", help="Directory to store the files [default: current directory]")
parser.add_argument("-q", "--quick", dest="quick", action="store_true",
        default=False, help="Quick")
parser.add_argument("-r", "--remove-orphan", dest="removeOrphan", action="store_true",
        default=False, help="Remove old scenery files")

parser.add_argument("--top", dest="top", type=int,
        default=90, help="Maximum latitude to include in download [default: %(default)d]")
parser.add_argument("--bottom", dest="bottom", type=int,
        default=-90, help="Minimum latitude to include in download [default: %(default)d]")
parser.add_argument("--left", dest="left", type=int,
        default=-180, help="Minimum longitude to include in download [default: %(default)d]")
parser.add_argument("--right", dest="right", type=int,
        default=180, help="Maximum longitude to include in download [default: %(default)d]")

args = parser.parse_args()

terraSync = TerraSync(args.url, args.target, args.quick, args.removeOrphan,
        DownloadBoundaries(args.top, args.left, args.bottom, args.right))

terraSync.start()