TGArray: convert to binary arr files, clean up memory leaks in terrafit

Binary array patch by James Turner

src/Lib/Array/array.cxx

@@ -30,6 +30,7 @@
 #include <simgear/debug/logstream.hxx>
 #include <simgear/misc/strutils.hxx>
 #include <simgear/misc/sg_path.hxx>
+#include <simgear/io/lowlevel.hxx>
 
 #include "array.hxx"
 
@@ -38,22 +39,18 @@ using std::string;
 
 TGArray::TGArray( void ):
   array_in(NULL),
-  fitted_in(NULL)
+  fitted_in(NULL),
+      in_data(NULL)
 {
-    SG_LOG(SG_GENERAL, SG_DEBUG, "class TGArray CONstructor called." );
+
-    in_data = new int[ARRAY_SIZE_1 * ARRAY_SIZE_1];
 }
 
 
 TGArray::TGArray( const string &file ):
   array_in(NULL),
-  fitted_in(NULL)
+  fitted_in(NULL),
+      in_data(NULL)
 {
-    SG_LOG(SG_GENERAL, SG_DEBUG, "class TGArray CONstructor called." );
-    in_data = new int[ARRAY_SIZE_1 * ARRAY_SIZE_1];
-
-    SG_LOG(SG_GENERAL, SG_ALERT, "ps TGArray CONstructor called." );
-
     TGArray::open(file);
 }
 
@@ -62,13 +59,10 @@ TGArray::TGArray( const string &file ):
 bool TGArray::open( const string& file_base ) {
     // open array data file
     string array_name = file_base + ".arr.gz";
-    array_in = new sg_gzifstream( array_name );
+
-    if ( !array_in->is_open() ) {
+    array_in = gzopen( array_name.c_str(), "rb" );
-        SG_LOG(SG_GENERAL, SG_DEBUG, "  ps: Cannot open " << array_name );
+    if (array_in == NULL) {
-        delete array_in;
+        return false;
-        array_in = NULL;
-    } else {
-        SG_LOG(SG_GENERAL, SG_DEBUG, "  Opening array data file: " << array_name );
     }
 
     // open fitted data file
@@ -79,7 +73,7 @@ bool TGArray::open( const string& file_base ) {
         // can do a really stupid/crude fit on the fly, but it will
         // not be nearly as nice as what the offline terrafit utility
         // would have produced.
-        SG_LOG(SG_GENERAL, SG_DEBUG, "  ps: Cannot open " << fitted_name );
+        SG_LOG(SG_GENERAL, SG_DEBUG, "  Cannot open " << fitted_name );
         delete fitted_in;
         fitted_in = NULL;
     } else {
@@ -94,8 +88,7 @@ bool TGArray::open( const string& file_base ) {
 bool
 TGArray::close() {
     if (array_in) {
-        array_in->close();
+        gzclose(array_in);
-        delete array_in;
         array_in = NULL;
     }
 
@@ -114,23 +107,8 @@ TGArray::close() {
 bool
 TGArray::parse( SGBucket& b ) {
     // Parse/load the array data file
-    if ( array_in && array_in->is_open() ) {
+    if ( array_in ) {
-        // file open, parse
+        parse_bin();
-        *array_in >> originx >> originy;
-        *array_in >> cols >> col_step;
-        *array_in >> rows >> row_step;
-
-        SG_LOG(SG_GENERAL, SG_DEBUG, "    origin  = " << originx << "  " << originy );
-        SG_LOG(SG_GENERAL, SG_DEBUG, "    cols = " << cols << "  rows = " << rows );
-        SG_LOG(SG_GENERAL, SG_DEBUG, "    col_step = " << col_step << "  row_step = " << row_step );
-
-        for ( int i = 0; i < cols; i++ ) {
-            for ( int j = 0; j < rows; j++ ) {
-                *array_in >> in_data[(i * ARRAY_SIZE_1) + j];
-            }
-        }
-
-        SG_LOG(SG_GENERAL, SG_DEBUG, "    Done parsing" );
     } else {
         // file not open (not found?), fill with zero'd data
 
@@ -149,7 +127,9 @@ TGArray::parse( SGBucket& b ) {
         SG_LOG(SG_GENERAL, SG_DEBUG, "    cols = " << cols << "  rows = " << rows );
         SG_LOG(SG_GENERAL, SG_DEBUG, "    col_step = " << col_step << "  row_step = " << row_step );
 
-        memset(in_data, 0, sizeof(int) * cols * rows);
+
+        in_data = new short[cols * rows];
+        memset(in_data, 0, sizeof(short) * cols * rows);
         SG_LOG(SG_GENERAL, SG_DEBUG, "    File not open, so using zero'd data" );
     }
 
@@ -168,6 +148,33 @@ TGArray::parse( SGBucket& b ) {
     return true;
 }
 
+void TGArray::parse_bin()
+{
+    int32_t header;
+    sgReadLong(array_in, &header);
+    if (header != 0x54474152) {
+        SG_LOG(SG_GENERAL, SG_ALERT, "\nThe .arr file is not in the correct binary format."
+        << "\nPlease rebuild it using the latest TerraGear HGT tools.");
+        exit(1);
+    }
+
+    int minX, minY, intColStep, intRowStep;
+    sgReadInt(array_in, &minX);
+    sgReadInt(array_in, &minY);
+    originx = minX;
+    originy = minY;
+
+    sgReadInt(array_in, &cols);
+    sgReadInt(array_in, &intColStep);
+    sgReadInt(array_in, &rows);
+    sgReadInt(array_in, &intRowStep);
+
+    col_step = intColStep;
+    row_step = intRowStep;
+
+    in_data = new short[cols * rows];
+    sgReadShort(array_in, cols * rows, in_data);
+}
 
 // write an Array file
 bool TGArray::write( const string root_dir, SGBucket& b ) {
@@ -287,26 +294,6 @@ void TGArray::remove_voids( ) {
 }
 
 
-// add a node to the output corner node list
-void TGArray::add_corner_node( int i, int j, double val ) {
-
-    double x = (originx + i * col_step) / 3600.0;
-    double y = (originy + j * row_step) / 3600.0;
-    SG_LOG(SG_GENERAL, SG_DEBUG, "originx = " << originx << "  originy = " << originy );
-    SG_LOG(SG_GENERAL, SG_DEBUG, "corner = " << Point3D(x, y, val) );
-    corner_list.push_back( Point3D(x, y, val) );
-}
-
-
-// add a node to the output fitted node list
-void TGArray::add_fit_node( int i, int j, double val ) {
-    double x = (originx + i * col_step) / 3600.0;
-    double y = (originy + j * row_step) / 3600.0;
-    SG_LOG(SG_GENERAL, SG_DEBUG, Point3D(x, y, val) );
-    fitted_list.push_back( Point3D(x, y, val) );
-}
-
-
 // Return the elevation of the closest non-void grid point to lon, lat
 double TGArray::closest_nonvoid_elev( double lon, double lat ) const {
     double mindist = 99999999999.9;
@@ -335,8 +322,8 @@ double TGArray::closest_nonvoid_elev( double lon, double lat ) const {
 }
 
 
-// return the current altitude based on grid data.  We should rewrite
+// return the current altitude based on grid data.
-// this to interpolate exact values, but for now this is good enough
+// TODO: We should rewrite this to interpolate exact values, but for now this is good enough
 double TGArray::altitude_from_grid( double lon, double lat ) const {
     // we expect incoming (lon,lat) to be in arcsec for now
 
@@ -445,15 +432,24 @@ double TGArray::altitude_from_grid( double lon, double lat ) const {
 
 TGArray::~TGArray( void )
 {
-    delete in_data;
+    delete[] in_data;
 }
 
 int TGArray::get_array_elev( int col, int row ) const
 {
-    return in_data[(col * ARRAY_SIZE_1) + row];
+    return in_data[(col * rows) + row];
 }
 
 void TGArray::set_array_elev( int col, int row, int val )
 {
-    in_data[(col * ARRAY_SIZE_1) + row] = val;
+    in_data[(col * rows) + row] = val;
+}
+
+bool TGArray::is_open() const
+{
+  if ( array_in != NULL ) {
+      return true;
+  } else {
+      return false;
+  }
 }

src/Lib/Array/array.hxx

@@ -23,31 +23,21 @@
 #ifndef _ARRAY_HXX
 #define _ARRAY_HXX
 
-
-#ifndef __cplusplus
-# error This library requires C++
-#endif
-
 #ifdef HAVE_CONFIG_H
 #  include <config.h>
 #endif
 
 #include <simgear/compiler.h>
-
 #include <simgear/bucket/newbucket.hxx>
-#include <Geometry/point3d.hxx>
 #include <simgear/math/sg_types.hxx>
 #include <simgear/misc/sgstream.hxx>
 
-#define ARRAY_SIZE_1 20000
+#include <Lib/Geometry/point3d.hxx>
-
 
 class TGArray {
 
 private:
-
+    gzFile array_in;
-    // array file pointer
-    sg_gzifstream *array_in;
 
     // fitted file pointer
     sg_gzifstream *fitted_in;
@@ -62,13 +52,13 @@ private:
     double col_step, row_step;
 
     // pointers to the actual grid data allocated here
-    int *in_data;
+    short *in_data;
-    // float (*out_data)[ARRAY_SIZE_1];
 
     // output nodes
     point_list corner_list;
     point_list fitted_list;
 
+    void parse_bin();
 public:
 
     // Constructor
@@ -82,13 +72,7 @@ public:
     bool open ( const std::string& file_base );
 
     // return if array was successfully opened or not
-    inline bool is_open() {
+    bool is_open() const;
-      if ( array_in != NULL ) {
-	return array_in->is_open();
-      } else {
-	return false;
-      }
-    }
 
     // close a Array file
     bool close();
@@ -103,12 +87,6 @@ public:
     // neighbor.
     void remove_voids();
 
-    // add a node to the output corner node list
-    void add_corner_node( int i, int j, double val );
-
-    // add a node to the output fitted node list
-    void add_fit_node( int i, int j, double val );
-
     // Return the elevation of the closest non-void grid point to lon, lat
     double closest_nonvoid_elev( double lon, double lat ) const;
 
@@ -130,14 +108,7 @@ public:
 
     int get_array_elev( int col, int row ) const;
     void set_array_elev( int col, int row, int val );
-
-
-    inline Point3D get_fitted_pt( int i ) {
-        return fitted_list[i];
-    }
 };
 
 
 #endif // _ARRAY_HXX
-
-

src/Lib/Geometry/TNT/tnt_array1d.h

@@ -258,7 +258,7 @@ inline int Array1D<T>::ref_count() const
 template <class T>
 inline Array1D<T> Array1D<T>::subarray(int i0, int i1)
 {
-	if ((i0 >= 0) && (i1 < n_) || (i0 <= i1))
+	if (((i0 >= 0) && (i1 < n_)) || (i0 <= i1))
 	{
 		Array1D<T> X(*this);  /* create a new instance of this array. */
 		X.n_ = i1-i0+1;

src/Lib/HGT/CMakeLists.txt

@@ -1,13 +1,4 @@
-
+add_library(HGT STATIC
-
-add_library(HGT STATIC 
     hgt.cxx hgt.hxx
     srtmbase.cxx srtmbase.hxx
 )
-
-add_executable(test_hgt testhgt.cxx)
-
-target_link_libraries(test_hgt 
-    HGT
-	${SIMGEAR_CORE_LIBRARIES}
-	${SIMGEAR_CORE_LIBRARY_DEPENDENCIES})

src/Lib/HGT/srtmbase.cxx

@@ -25,12 +25,13 @@
 #  include <config.h>
 #endif
 
-#include <simgear/compiler.h>
-
 #include <iostream>
 #include <stdlib.h>
 #include <zlib.h>
 
+#include <simgear/compiler.h>
+#include <simgear/io/lowlevel.hxx>
+
 #include "srtmbase.hxx"
 
 using std::cout;
@@ -57,7 +58,7 @@ TGSrtmBase::write_area( const string& root, SGBucket& b ) {
     double max_y = ( b.get_center_lat() + 0.5 * b.get_height() ) * 3600.0;
 
     cout << b << endl;
-    cout << "width = " << b.get_width() << " height = " << b.get_height() 
+    cout << "width = " << b.get_width() << " height = " << b.get_height()
 	 << endl;
     cout << "min = " << min_x << "," << min_y
          << "  max = " << max_x << "," << max_y << endl;
@@ -99,51 +100,38 @@ TGSrtmBase::write_area( const string& root, SGBucket& b ) {
     string array_file = path + "/" + b.gen_index_str() + ".arr.gz";
     cout << "array_file = " << array_file << endl;
 
-    // write the file
+    write_area_bin(array_file, start_x, start_y, min_x, min_y,
+        span_x, span_y, col_step, row_step);
+
+    return true;
+}
+
+bool TGSrtmBase::write_area_bin(const SGPath& aPath, int start_x, int start_y,
+    int min_x, int min_y,
+    int span_x, int span_y, int col_step, int row_step)
+{
     gzFile fp;
-    if ( (fp = gzopen( array_file.c_str(), "wb9" )) == NULL ) {
+    if ( (fp = gzopen( aPath.c_str(), "wb9" )) == NULL ) {
-	cout << "ERROR:  cannot open " << array_file << " for writing!" << endl;
+	    cout << "ERROR:  cannot open " << aPath.str() << " for writing!" << endl;
-	exit(-1);
+	    return false;
     }
 
-    gzprintf( fp, "%d %d\n", (int)min_x, (int)min_y );
+    int32_t header = 0x54474152; // 'TGAR'
-    gzprintf( fp, "%d %d %d %d\n", span_x + 1, (int)col_step, 
+    sgWriteLong(fp, header);
-              span_y + 1, (int)row_step );
+    sgWriteInt(fp, min_x); sgWriteInt(fp, min_y);
+    sgWriteInt(fp, span_x + 1); sgWriteInt(fp, col_step);
+    sgWriteInt(fp, span_y + 1); sgWriteInt(fp, row_step);
+
     for ( int i = start_x; i <= start_x + span_x; ++i ) {
-	for ( int j = start_y; j <= start_y + span_y; ++j ) {
+	    for ( int j = start_y; j <= start_y + span_y; ++j ) {
-	    gzprintf( fp, "%d ", (int)height(i,j) );
+            sgWriteShort(fp, height(i,j));
-	}
+	    }
-	gzprintf( fp, "\n" );
     }
-    gzclose(fp);
 
+    gzclose(fp);
     return true;
 }
 
-
-// write the entire area out in a simple ascii format
-bool TGSrtmBase::write_whole_ascii( const string& file ) {
-    cout << "writing to " << file << endl;
-    // write the file
-    gzFile fp;
-    if ( (fp = gzopen( file.c_str(), "wb9" )) == NULL ) {
-	cout << "ERROR:  cannot open " << file << " for writing!" << endl;
-	exit(-1);
-    }
-
-    gzprintf( fp, "%d\n%d\n", rows, cols );
-    for ( int row = rows - 1; row >= 0; row-- ) {
-        for ( int col = 0; col < cols; col++ ) {
-            gzprintf( fp, "%d\n", (int)height(col,row) );
-        }
-    }
-    gzclose(fp);
-
-    return true;
-}
-
-
-
 bool
 TGSrtmBase::has_non_zero_elev (int start_x, int span_x,
                           int start_y, int span_y) const

src/Lib/HGT/srtmbase.hxx

@@ -38,15 +38,15 @@ class TGSrtmBase {
 protected:
     TGSrtmBase() : remove_tmp_file(false)
     {}
-    
+
     ~TGSrtmBase();
 
     // coordinates (in arc seconds) of south west corner
     double originx, originy;
-    
+
     // number of columns and rows
     int cols, rows;
-    
+
     // Distance between column and row data points (in arc seconds)
     double col_step, row_step;
 
@@ -60,8 +60,9 @@ public:
     // hand corner.
     bool write_area( const std::string& root, SGBucket& b );
 
-    // write the entire area out in a simple ascii format
+    bool write_area_bin(const SGPath& aPath,
-    bool write_whole_ascii( const std::string& file );
+        int start_x, int start_y, int min_x, int min_y,
+    int span_x, int span_y, int col_step, int row_step);
 
     // Informational methods
     inline double get_originx() const { return originx; }

src/Lib/HGT/testhgt.cxx

@@ -1,18 +0,0 @@
-#include <simgear/misc/sg_path.hxx>
-
-#include "hgt.hxx"
-
-int main() {
-    // SGPath path3( "/stage/fgfs03/SRTM/United_States/N38W124/3arcsec/N38W124.hgt.gz" );
-    SGPath path3( "/home/curt/trash/N38W124.hgt.gz" );
-
-    // TGHgt hgt1( 1, path1 );
-    TGHgt hgt3( 3, path3 );
-
-    // hgt1.load();
-    hgt3.load();
-
-    // SGBucket b(-79.5, 33.5);
-    // hgt1.write_area( ".", b );
-    hgt3.write_whole_ascii( "w124n38.txt.gz" );
-}

src/Prep/TerraFit/terrafit.cc

@@ -27,7 +27,7 @@
 #include <stdio.h>
 #include <errno.h>
 #include <stdlib.h>
-     
+
 #ifndef _MSC_VER
 #  include <unistd.h>
 #  ifdef __APPLE__
@@ -41,7 +41,7 @@
 #  include <Prep/Terra/getopt.h>
 #  define sleep(x) Sleep(x*1000)
 #endif
-     
+
 #include <zlib.h>
 #include <boost/foreach.hpp>
 
@@ -52,7 +52,7 @@
 #include <simgear/structure/exception.hxx>
 #include <simgear/threads/SGQueue.hxx>
 #include <simgear/threads/SGThread.hxx>
-     
+
 #include <Array/array.hxx>
 #include <Include/version.h>
 #include <Prep/Terra/GreedyInsert.h>
@@ -81,7 +81,7 @@ public:
                 min=30000;
                 max=-30000;
                 for (int i=0;i<width;i++) {
-                        for (int j=0;j<width;j++) {
+                        for (int j=0;j<height;j++) {
                                 Terra::real v=eval(i,j);
                                 if (v<min)
                                         min=v;
@@ -167,7 +167,7 @@ void fit_file(const SGPath& path) {
     if ( outPath.exists() ) {
         unlink( outPath.c_str() );
     }
-    
+
     SGBucket bucket(0,0); // dummy bucket
     TGArray inarray(path.dir() + "/" + path.file_base());
     inarray.parse(bucket);
@@ -200,7 +200,7 @@ void fit_file(const SGPath& path) {
             gzprintf(fp,"%+03.8f %+02.8f %0.2f\n",vx,vy,vz);
         }
     }
-    
+
     delete mesh;
     delete DEM;
 
@@ -218,7 +218,7 @@ void queue_fit_file(const SGPath& path)
             return;
         }
     }
-    
+
     global_workQueue.push(path);
 }
 
@@ -237,14 +237,14 @@ public:
 };
 
 void walk_path(const SGPath& path) {
-    
+
     if (!path.exists()) {
         SG_LOG(SG_GENERAL, SG_ALERT ,"ERROR: Unable to stat '" << path.str() << "':" << strerror(errno));
         return;
     }
-    
+
     if ((path.lower_extension() == "arr") || (path.complete_lower_extension() == "arr.gz")) {
-        SG_LOG(SG_GENERAL, SG_INFO, "will queue " << path);
+        SG_LOG(SG_GENERAL, SG_DEBUG, "will queue " << path);
         queue_fit_file(path);
     } else if (path.isDir()) {
         Dir d(path);
@@ -261,12 +261,12 @@ void usage(char* progname, const std::string& msg) {
     }
 
     SG_LOG(SG_GENERAL,SG_INFO, "Usage: " << progname << " [options] <file | path to walk>");
-    SG_LOG(SG_GENERAL,SG_INFO, "\t -h | --help ");
+    SG_LOG(SG_GENERAL,SG_INFO, "\t -h | --help");
     SG_LOG(SG_GENERAL,SG_INFO, "\t -m | --minnodes 50");
     SG_LOG(SG_GENERAL,SG_INFO, "\t -x | --maxnodes 1000");
     SG_LOG(SG_GENERAL,SG_INFO, "\t -e | --maxerror 40");
     SG_LOG(SG_GENERAL,SG_INFO, "\t -f | --force");
-    SG_LOG(SG_GENERAL,SG_INFO, "\t -j | --threads");
+    SG_LOG(SG_GENERAL,SG_INFO, "\t -j | --threads <number>");
     SG_LOG(SG_GENERAL,SG_INFO, "\t -v | --version");
     SG_LOG(SG_GENERAL,SG_INFO, "");
     SG_LOG(SG_GENERAL,SG_INFO, "Algorithm will produce at least <minnodes> fitted nodes, but no");
@@ -303,10 +303,10 @@ struct option options[]={
 };
 
 int main(int argc, char** argv) {
-    sglog().setLogLevels( SG_ALL, SG_DEBUG );
+    sglog().setLogLevels( SG_ALL, SG_INFO );
     int option;
 
-    while ((option=getopt_long(argc,argv,"h:m:x:e:f:v:j:",options,NULL))!=-1) {
+    while ((option=getopt_long(argc,argv,"hm:x:e:fvj:",options,NULL))!=-1) {
         switch (option) {
             case 'h':
                 usage(argv[0],"");
@@ -336,7 +336,7 @@ int main(int argc, char** argv) {
         }
     }
 
-    SG_LOG(SG_GENERAL, SG_INFO, "TerraFit version " << getTGVersion());
+    SG_LOG(SG_GENERAL, SG_INFO, "TerraFit version " << getTGVersion() << " using " << num_threads << " threads");
     SG_LOG(SG_GENERAL, SG_INFO, "Min points = " << min_points);
     SG_LOG(SG_GENERAL, SG_INFO, "Max points = " << point_limit);
     SG_LOG(SG_GENERAL, SG_INFO, "Max error  = " << error_threshold);
@@ -350,21 +350,21 @@ int main(int argc, char** argv) {
         SG_LOG(SG_GENERAL, SG_INFO, "Use 'terrafit --help' for commands");
         exit(1);
     }
-    
+
     std::vector<FitThread*> threads;
     for (unsigned int t=0; t<num_threads; ++t) {
         FitThread* thread = new FitThread;
         thread->start();
         threads.push_back(thread);
     }
-    
+
     while (!global_workQueue.empty()) {
         sleep(1);
     }
-    
+
     for (unsigned int t=0; t<num_threads; ++t) {
         threads[t]->join();
     }
-    
+
     SG_LOG(SG_GENERAL, SG_INFO, "Work queue is empty\n");
 }