Better error catching and recovery when our input sends the triangulator

into never, never land.

Tools/Construct/Main/main.cxx

@@ -24,6 +24,10 @@
 #include <sys/types.h>  // for directory reading
 #include <dirent.h>     // for directory reading
 
+#include <sys/time.h>		// set mem allocation limit
+#include <sys/resource.h>	// set mem allocation limit
+#include <unistd.h>		// set mem allocation limit
+
 #include <Bucket/newbucket.hxx>
 #include <Include/fg_constants.h>
 #include <Math/mat3.h>
@@ -393,23 +397,34 @@ void construct_tile( FGConstruct& c ) {
 		 << endl;
 	}
 
-	if ( (count > c.get_max_nodes()) && (error <= 1000.0) ) {
-	    // increase error tolerance until number of points drops below
-	    // the maximum threshold
-	    cout << "produced too many nodes ..." << endl;
+	if ( count > c.get_max_nodes() ) {
+	    if ( error <= 1000.0 ) {
+		// increase error tolerance until number of points drops below
+		// the maximum threshold
+		cout << "produced too many nodes ..." << endl;
 	    
-	    acceptable = false;
-	    shrinking = true;
+		acceptable = false;
+		shrinking = true;
 
-	    if ( growing ) {
-		error *= 1.25;
-		growing = false;
+		if ( growing ) {
+		    error *= 1.25;
+		    growing = false;
+		} else {
+		    error *= 1.5;
+		}
+
+		cout << "Setting error to " << error << " and retrying fit." 
+		     << endl;
 	    } else {
-		error *= 1.5;
+		// we tried, but can't seem to get down to a
+		// reasonable number of points even with a huge error
+		// tolerance.  This could be related to the triangle()
+		// call which might be having trouble with our input
+		// set.  Let's just die hope that our parent can try
+		// again with a smaller interior triangle angle.
+		cout << "Error:  Too many nodes." << endl;
+		exit(-1);
 	    }
-
-	    cout << "Setting error to " << error << " and retrying fit." 
-		 << endl;
 	}
     }
 
@@ -478,6 +493,30 @@ main(int argc, char **argv) {
 	usage( argv[0] );
     }
 
+    // set mem allocation limit.  Reason: occasionally the triangle()
+    // routine can blow up and allocate memory forever.  We'd like
+    // this process to die before things get out of hand so we can try
+    // again with a smaller interior angle limit.
+    int result;
+    struct rlimit limit;
+    limit.rlim_cur = 20000000;
+    limit.rlim_max = 20000000;
+    result = setrlimit( RLIMIT_DATA, &limit );
+    cout << "result of setting mem limit = " << result << endl;
+    result = setrlimit( RLIMIT_STACK, &limit );
+    cout << "result of setting mem limit = " << result << endl;
+    result = setrlimit( RLIMIT_CORE, &limit );
+    cout << "result of setting mem limit = " << result << endl;
+    result = setrlimit( RLIMIT_RSS, &limit );
+    cout << "result of setting mem limit = " << result << endl;
+
+    // cpu time limit since occassionally the triangulator can go into
+    // and infinite loop.
+    limit.rlim_cur = 120;
+    limit.rlim_max = 120;
+    result = setrlimit( RLIMIT_CPU, &limit );
+    cout << "result of setting mem limit = " << result << endl;
+   
     // main construction data management class
     FGConstruct c;
 

Tools/Construct/Parallel/client.cxx

@@ -134,7 +134,10 @@ bool construct_tile( const string& work_base, const string& output_base,
 	    if ( line_str == "[Finished successfully]" ) {
 		fclose(fp);
 		return true;
-	    } else if ( line_str == "Error:  Ran out of precision at" ) {
+	    } else if 
+		( (line_str.substr(0, 31) == "Error:  Ran out of precision at")
+		  || (line_str.substr(0, 22) == "Error:  Out of memory.")
+		  || (line_str.substr(0, 23) == "Error:  Too many nodes.") ) {
 		if ( angle > 9.0 ) {
 		    angle = 5.0;
 		    still_trying = true;
@@ -143,8 +146,16 @@ bool construct_tile( const string& work_base, const string& output_base,
 		    still_trying = true;
 		}
 	    }
+
 	}
 	fclose(fp);
+
+	if ( !still_trying && ( angle > 0.0 ) ) {
+	    // build died for some reason ... lets try one last time
+	    // with an interior angle restriction of 0
+	    angle = 0.0;
+	    still_trying = true;
+	}
     }
     return false;
 }

Tools/Construct/Parallel/server.cxx

@@ -133,11 +133,13 @@ long int get_next_tile( const string& work_base )
 
 	// reset lat
 	// lat = -89.0 + (shift_up*dy) - (dy*0.5);
+	// lat = 27.0 + (0*dy) + (dy*0.5);
 	lat = 15.0 + (shift_up*dy) + (dy*0.5);
 
 	// reset lon
 	FGBucket tmp( 0.0, lat );
 	double dx = tmp.get_width();
+	// lon = -82 + (shift_over*dx) + (dx*0.5);
 	lon = -180 + (shift_over*dx) + (dx*0.5);
 
 	cout << "starting pass = " << pass