flightgear/scripts/python/terrasync.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2016  Torsten Dreyer
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#
# terrasync.py - synchronize terrascenery data to your local disk
# needs dnspython (pip install dnspython)

import urllib, os, hashlib
from urllib.parse import urlparse
from http.client import HTTPConnection, _CS_IDLE
from os import listdir
from os.path import isfile, join

#################################################################################################################################
class HTTPGetCallback:
    def __init__(self, src, callback):
        self.callback = callback
        self.src = src
        self.result = None

class HTTPGetter:
    def __init__(self, baseUrl, maxPending=10):
        self.baseUrl = baseUrl
        self.parsedBaseUrl = urlparse(baseUrl)
        self.maxPending = maxPending
        self.requests = []
        self.pendingRequests = []
        self.httpConnection = HTTPConnection(self.parsedBaseUrl.netloc)
        self.httpRequestHeaders = headers = {'Host':self.parsedBaseUrl.netloc,'Content-Length':0,'Connection':'Keep-Alive','User-Agent':'FlightGear terrasync.py'}

    def doGet(self, httpGetCallback):
        conn = self.httpConnection
        request = httpGetCallback
        self.httpConnection.request("GET", self.parsedBaseUrl.path + request.src, None, self.httpRequestHeaders)
        httpGetCallback.result = self.httpConnection.getresponse()
        httpGetCallback.callback()

    def get(self, httpGetCallback):

        try:
            self.doGet(httpGetCallback)
        except:
            # try to reconnect once
            #print("reconnect")
            self.httpConnection.close()
            self.httpConnection.connect()
            self.doGet(httpGetCallback)


#################################################################################################################################
class DirIndex:

    def __init__(self, dirIndexFile):
        self.d = []
        self.f = []
        self.version = 0
        self.path = ""

        with open(dirIndexFile) as f:
            self.readFrom(f)

    def readFrom(self, readable):
        for line in readable:
            line = line.strip()
            if line.startswith('#'):
                continue

            tokens = line.split(':')
            if len(tokens) == 0:
                continue

            if tokens[0] == "version":
                self.version = int(tokens[1])

            elif tokens[0] == "path":
                self.path = tokens[1]

            elif tokens[0] == "d":
                self.d.append({ 'name': tokens[1], 'hash': tokens[2] })

            elif tokens[0] == "f":
                self.f.append({ 'name': tokens[1], 'hash': tokens[2], 'size': tokens[3] })

    def getVersion(self):
        return self.version

    def getPath(self):
        return self.path

    def getDirectories(self):
        return self.d

    def getFiles(self):
        return self.f

#################################################################################################################################
class HTTPDownloadRequest(HTTPGetCallback):
    def __init__(self, terrasync, src, dst, callback = None ):
        super().__init__(src, self.callback)
        self.terrasync = terrasync
        self.dst = dst
        self.mycallback = callback

    def callback(self):
        with open(self.dst, 'wb') as f:
            f.write(self.result.read())

        if self.mycallback != None:
            self.mycallback(self)

#################################################################################################################################

def hash_of_file(fname):
    if not os.path.exists( fname ):
      return None

    hash = hashlib.sha1()
    try:
        with open(fname, "rb") as f:
            for chunk in iter(lambda: f.read(4096), b""):
                hash.update(chunk)
    except:
        pass

    return hash.hexdigest()

#################################################################################################################################
class TerraSync:

    def __init__(self, url="http://flightgear.sourceforge.net/scenery", target=".", quick=False, removeOrphan=False):
        self.setUrl(url).setTarget(target)
        self.quick = quick
        self.removeOrphan = removeOrphan
        self.httpGetter = None

    def setUrl(self, url):
        self.url = url.rstrip('/').strip()
        return self

    def setTarget(self, target):
        self.target = target.rstrip('/').strip()
        return self

    def start(self):
        self.httpGetter = HTTPGetter(self.url)
        self.updateDirectory("", "", None )

    def updateFile(self, serverPath, localPath, fileHash ):
        localFullPath = join(self.target, localPath)
        if fileHash != None and hash_of_file(localFullPath) == fileHash:
            #print("hash of file matches, not downloading")
            return

        print("downloading ", serverPath )

        request = HTTPDownloadRequest(self, serverPath, localFullPath )
        self.httpGetter.get(request)


    def updateDirectory(self, serverPath, localPath, dirIndexHash):
        print("processing ", serverPath)

        localFullPath = join(self.target, localPath)
        if not os.path.exists( localFullPath ):
          os.makedirs( localFullPath )

        localDirIndex = join(localFullPath, ".dirindex")
        if dirIndexHash != None and  hash_of_file(localDirIndex) == dirIndexHash:
            # print("hash of dirindex matches, not downloading")
            if not self.quick:
                self.handleDirindexFile( localDirIndex )
        else:
            request = HTTPDownloadRequest(self, serverPath + "/.dirindex", localDirIndex, self.handleDirindexRequest )
            self.httpGetter.get(request)

    def handleDirindexRequest(self, dirindexRequest):
        self.handleDirindexFile(dirindexRequest.dst)

    def handleDirindexFile(self, dirindexFile):
        dirIndex = DirIndex(dirindexFile)
        serverFiles = []

        for file in dirIndex.getFiles():
            f = file['name']
            h = file['hash']
            self.updateFile( "/" + dirIndex.getPath() + "/" + f, join(dirIndex.getPath(),f), h )
            serverFiles.append(f)

        for subdir in dirIndex.getDirectories():
            d = subdir['name']
            h = subdir['hash']
            self.updateDirectory( "/" + dirIndex.getPath() + "/" + d, join(dirIndex.getPath(),d), h )

        if self.removeOrphan:
            localFullPath = join(self.target, dirIndex.getPath())
            localFiles = [f for f in listdir(localFullPath) if isfile(join(localFullPath, f))]
            for f in localFiles:
                if f != ".dirindex" and not f in serverFiles:
                    #print("removing orphan", join(localFullPath,f) )
                    os.remove( join(localFullPath,f) )


    def isReady(self):
        return self.httpGetter and self.httpGetter.isReady()
        return False

    def update(self):
        if self.httpGetter:
            self.httpGetter.update()

#################################################################################################################################
import getopt, sys
try:
  opts, args = getopt.getopt(sys.argv[1:], "u:t:qr", [ "url=", "target=", "quick", "remove-orphan" ])

except getopt.GetoptError:
  print("terrasync.py [--url=http://some.server.org/scenery] [--target=/some/path] [-q|--quick] [-r|--remove-orphan]")
  sys.exit(2)

terraSync = TerraSync()
for opt, arg in opts:
  if opt in("-u", "--url"):
    terraSync.url = arg

  elif opt in ("-t", "--target"):
    terraSync.target = arg

  elif opt in ("-q", "--quick"):
    terraSync.quick = True

  elif opt in ("-r", "--remove-orphan"):
    terraSync.removeOrphan = True

terraSync.start()