terrasync.py: more thorough checking of .dirindex contents

- only accept ASCII-encoded .dirindex files (this is guaranteed to work
  fine "everywhere");

- reject .dirindex files with a 'path' entry that contains a backslash
  or starts with a slash;

- reject .dirindex files with a 'path' entry that contains a '..'
  component;

- reject .dirindex files with an 'f', 'd' or 't' entry whose name field
  contains a slash or a backslash;

- reject .dirindex files with an 'f', 'd' or 't' entry whose name field
  is '..';

- add comment lines (starting with '#') in the sample good .dirindex
  file used by unit tests.

scripts/python/TerraSync/terrasync/dirindex.py

@@ -38,11 +38,29 @@ class DirIndex:
         # attribute. This is useful for troubleshooting.
         self._rawContents = None
 
-        with open(dirIndexFile, "r", encoding="utf-8") as f:
+        with open(dirIndexFile, "r", encoding="ascii") as f:
             self.readFrom(f)
 
         self._sanityCheck()
 
+    @classmethod
+    def checkForBackslashOrLeadingSlash(cls, line, path):
+        if '\\' in path or path.startswith('/'):
+            raise InvalidDirIndexFile(
+                r"invalid '\' or leading '/' in path field from line {!r}"
+                .format(line))
+
+    @classmethod
+    def checkForSlashBackslashOrDoubleColon(cls, line, name):
+        if '/' in name or '\\' in name:
+            raise InvalidDirIndexFile(
+                r"invalid '\' or '/' in name field from line {!r}"
+                .format(line))
+
+        if name == "..":
+            raise InvalidDirIndexFile(
+                r"invalid name field equal to '..' in line {!r}".format(line))
+
     def readFrom(self, readable):
         self._rawContents = readable.read()
 
@@ -57,14 +75,23 @@ class DirIndex:
             elif tokens[0] == "version":
                 self.version = int(tokens[1])
             elif tokens[0] == "path":
+                self.checkForBackslashOrLeadingSlash(line, tokens[1])
                 # This is relative to the repository root
                 self.path = VirtualPath(tokens[1])
+
+                if ".." in self.path.parts:
+                    raise InvalidDirIndexFile(
+                        "'..' component found in 'path' entry {!r}"
+                        .format(self.path))
             elif tokens[0] == "d":
+                self.checkForSlashBackslashOrDoubleColon(line, tokens[1])
                 self.directories.append({'name': tokens[1], 'hash': tokens[2]})
             elif tokens[0] == "f":
+                self.checkForSlashBackslashOrDoubleColon(line, tokens[1])
                 self.files.append({'name': tokens[1],
                                    'hash': tokens[2], 'size': int(tokens[3])})
             elif tokens[0] == "t":
+                self.checkForSlashBackslashOrDoubleColon(line, tokens[1])
                 self.tarballs.append({'name': tokens[1], 'hash': tokens[2],
                                       'size': int(tokens[3])})
 

scripts/python/TerraSync/tests/data/dirindex/bad/bad_dirindex_backslash_in_directory_name

@@ -0,0 +1,3 @@
+version:1
+path:some/path
+d:some\illegal directory name with a backslash:378b3dd58ce3058f2992b70aa5ecf8947a4d7f9e

scripts/python/TerraSync/tests/data/dirindex/bad/bad_dirindex_backslash_in_file_name

@@ -0,0 +1,3 @@
+version:1
+path:some/path
+f:some\illegal file name with a backslash:4cbf3d1746a1249bff7809e4b079dd80cfce594c:123

scripts/python/TerraSync/tests/data/dirindex/bad/bad_dirindex_backslash_in_tarball_name

@@ -0,0 +1,3 @@
+version:1
+path:some/path
+t:some\illegal tarball name with a backslash.tgz:b63a067d82824f158d6bde66f9e76654274277fe:1234567

scripts/python/TerraSync/tests/data/dirindex/bad/bad_dirindex_directory_name_is_double_colon

@@ -0,0 +1,3 @@
+version:1
+path:some/path
+d:..:378b3dd58ce3058f2992b70aa5ecf8947a4d7f9e

scripts/python/TerraSync/tests/data/dirindex/bad/bad_dirindex_dotdot_in_path

@@ -0,0 +1,2 @@
+version:1
+path:some/path/with/a/../component

scripts/python/TerraSync/tests/data/dirindex/bad/bad_dirindex_encoding

@@ -0,0 +1,2 @@
+version:1
+path:some/path/non-ASCII chars like é, ê, €, Œ, Ÿ, etc./foo/bar

scripts/python/TerraSync/tests/data/dirindex/bad/bad_dirindex_file_name_is_double_colon

@@ -0,0 +1,3 @@
+version:1
+path:some/path
+f:..:4cbf3d1746a1249bff7809e4b079dd80cfce594c:123

scripts/python/TerraSync/tests/data/dirindex/bad/bad_dirindex_path_contains_a_backslash

@@ -0,0 +1,2 @@
+version:1
+path:some/path/that/contains \ a/backslash

scripts/python/TerraSync/tests/data/dirindex/bad/bad_dirindex_path_starts_with_slash

@@ -0,0 +1,2 @@
+version:1
+path:/some/path/that/starts/with/a/slash

scripts/python/TerraSync/tests/data/dirindex/bad/bad_dirindex_slash_in_directory_name

@@ -0,0 +1,3 @@
+version:1
+path:some/path
+d:some/illegal directory name with a slash:378b3dd58ce3058f2992b70aa5ecf8947a4d7f9e

scripts/python/TerraSync/tests/data/dirindex/bad/bad_dirindex_slash_in_file_name

@@ -0,0 +1,3 @@
+version:1
+path:some/path
+f:some/illegal file name with a slash:4cbf3d1746a1249bff7809e4b079dd80cfce594c:123

scripts/python/TerraSync/tests/data/dirindex/bad/bad_dirindex_slash_in_tarball_name

@@ -0,0 +1,3 @@
+version:1
+path:some/path
+t:some/illegal tarball name with a slash.tgz:b63a067d82824f158d6bde66f9e76654274277fe:1234567

scripts/python/TerraSync/tests/data/dirindex/bad/bad_dirindex_tarball_name_is_double_colon

@@ -0,0 +1,3 @@
+version:1
+path:some/path
+t:..:b63a067d82824f158d6bde66f9e76654274277fe:1234567

scripts/python/TerraSync/tests/data/dirindex/good/sample_dirindex_1

@@ -1,9 +1,11 @@
+# Comment line
 version:1
 path:some/path
 time:20200926-10:38Z
 d:Airports:8a93b5d8a2b04d2fb8de4ef58ad02f9e8819d314
 d:Models:bee221c9d2621dc9b69cd9e0ad7dd0605f6ea928
 d:Objects:10ae32c986470fa55b56b8eefbc6ed565cce0642
+# Other comment line
 d:Terrain:e934024dc0f959f9a433e47c646d256630052c2e
 d:Buildings:19060725efc2a301fa6844991e2922d42d8de5e2
 d:Pylons:378b3dd58ce3058f2992b70aa5ecf8947a4d7f9e

scripts/python/TerraSync/tests/test_dirindex.py

@@ -28,6 +28,7 @@
 import os
 import unittest
 from terrasync.dirindex import DirIndex
+from terrasync.exceptions import InvalidDirIndexFile
 from terrasync.virtual_path import VirtualPath
 
 
@@ -66,10 +67,28 @@ tarballs_in_sample_dirindex_1 = [
 class TestDirIndex(unittest.TestCase):
     """Unit tests for the DirIndex class."""
 
-    def test_readFrom(self):
+    def test_constructor(self):
-        d = DirIndex(testData("sample_dirindex_1"))
+        d = DirIndex(testData("good", "sample_dirindex_1"))
         self.assertEqual(d.version, 1)
         self.assertEqual(d.path, VirtualPath("some/path"))
         self.assertEqual(d.directories, directories_in_sample_dirindex_1)
         self.assertEqual(d.files, files_in_sample_dirindex_1)
         self.assertEqual(d.tarballs, tarballs_in_sample_dirindex_1)
+
+        for stem in ("path_starts_with_slash",
+                     "path_contains_a_backslash",
+                     "dotdot_in_path",
+                     "slash_in_directory_name",
+                     "slash_in_file_name",
+                     "slash_in_tarball_name",
+                     "backslash_in_directory_name",
+                     "backslash_in_file_name",
+                     "backslash_in_tarball_name",
+                     "directory_name_is_double_colon",
+                     "file_name_is_double_colon",
+                     "tarball_name_is_double_colon"):
+            with self.assertRaises(InvalidDirIndexFile):
+                DirIndex(testData("bad", "bad_dirindex_" + stem))
+
+        with self.assertRaises(UnicodeDecodeError):
+            d = DirIndex(testData("bad", "bad_dirindex_encoding"))