Refine threading and reduce kernel calls (Trying to get docker container to run same everwhere) (#4215)

* Adding GreenLightning

* fix missing quote

create object for json production

* turn off telemetry for default test

* allow any external domain or ip

* Update to next version and template construction

* Revert "Update to next version and template construction"

This reverts commit da4fbc3b085f6e3b3bb50283d4d6e79efffdd3bb.

* Revert "Revert "Update to next version and template construction""

This reverts commit eb05517a192554caec7c5f690eeaaab171a549b6.

* Revert "Update to next version and template construction"

This reverts commit da4fbc3b085f6e3b3bb50283d4d6e79efffdd3bb.

* fix host ip to 0.0.0.0 to take load

added database read tests

* response large enough for multi db

remove epoll, it was not helping

added required headers

* update list of which tests we have implmented

* removed unused dependency

* remove unrequired 16G memory grab to get past travis check.

* Update to next GL version to fix overload issue past 1.5M rps

Refine template method to remove dead argument

* simplify arg parse

fixed issue with multi db under heavy load

* updat to atomic int

* Removed support for MultiTest, not stable at this time.

* added clean for safety

* remove dead code

* remove old comments

* [ci fw-only Java/greenlightning]

added comment

* [ci fw-only Java/greenlightning]

Added tests for remaining multi, update and fortunes

* [ci fw-only Java/greenlightning]

disable update test, seems to be missing some writes.

* [ci fw-only Java/greenlightning]

re-test of DBUpdate

* [ci fw-only Java/greenlightning]

disabled multi and update while tracking issue

* fixed muti paylod response JSON dups.

* [ci fw-only Java/greenlightning]

narrow building

* [ci skip] update readme

* [ci fw-only Java/greenlightning]

Update to version supporting 0.0.0.0 wildcard host

* [ci fw-only Java/greenlightning]

explicit memory min/max setting instead of default

update to new maven for build

* lower required low end to 8G

* [ci fw-only Java/greenlightning]

dropped min to 6G

* [ci fw-only Java/greenlightning]

memory reduction, was using as much as 20G, paging may have slowed test

reduce pipe memory allocations

combined behaviors to reduce memory

reduce db inflight collection to reduce memory

* Revert "[ci fw-only Java/greenlightning]"

This reverts commit 47c351db1ddeb64e65801f15e3c4398d5eca84ad.

* [ci fw-only Java/greenlightning]

Was using 16-20G which may cause paging and slow results

Reduced Pipe lenghts

Reduced concurrent connections

Combined behaviors to reduce Pipe counts for more memory.

Lowered limit to 16G to know it is enforced.

* removed minimum required memory in docker file

* [ci fw-only Java/greenlightning]  Ready for merge

* [ci fw-only Java/greenlightning]

Lowering memory usage to 7G from 14G

Clean up design to be easier to read

* [ci fw-only Java/greenlightning]

Investigating pef issue, returned threads to normal priority

Reduce the new and ongoing network kernel calls

Tread Executor using hardcoded count instead of asking docker

* [ci fw-only Java/greenlightning]

remove dead code

frameworks/Java/greenlightning/pom.xml

@@ -14,7 +14,7 @@
 		<dependency>
 			<groupId>com.ociweb</groupId>
 			<artifactId>greenlightning</artifactId>
-			<version>1.0.14</version>
+			<version>1.0.15</version>
 		</dependency>
 		<dependency>
 			<groupId>org.slf4j</groupId>

frameworks/Java/greenlightning/src/main/java/com/ociweb/gl/benchmark/DBRest.java

@@ -14,7 +14,7 @@ public class DBRest implements RestMethodListener, PubSubMethodListener, TickLis
 	private final ProcessUpdate processUpdate;
 	private final ProcessFortune processFortune;
 	private final ProcessQuery processQuery;
-	private final PoolManager pm;
+	private final transient PoolManager pm;
 	
 	public DBRest(GreenRuntime runtime, PgPoolOptions options, int pipelineBits, int maxResponseCount, int maxResponseSize) {
 		

frameworks/Java/greenlightning/src/main/java/com/ociweb/gl/benchmark/FrameworkTest.java

@@ -11,15 +11,10 @@ import com.ociweb.gl.api.GreenApp;
  */
 import com.ociweb.gl.api.GreenFramework;
 import com.ociweb.gl.api.GreenRuntime;
-import com.ociweb.gl.api.HTTPResponseService;
-import com.ociweb.pronghorn.network.ServerSocketReaderStage;
 import com.ociweb.pronghorn.network.ServerSocketWriterStage;
-import com.ociweb.pronghorn.network.http.HTTP1xRouterStage;
-import com.ociweb.pronghorn.pipe.ObjectPipe;
 
 import io.reactiverse.pgclient.PgClient;
 import io.reactiverse.pgclient.PgPoolOptions;
-import io.reactiverse.reactivex.pgclient.PgConnection;
 
 public class FrameworkTest implements GreenApp {
 
@@ -57,8 +52,8 @@ public class FrameworkTest implements GreenApp {
     	this(System.getProperty("host","0.0.0.0"), 
     		 8080,    //default port for test 
     		 10,      //default concurrency, 5 to support 140 channels on 14 core boxes
-    		 2*1024,  //default max rest requests allowed to queue in wait
-    		 1<<20,   //default network buffer per input socket connection
+    		 8*1024,  //default max rest requests allowed to queue in wait
+    		 1<<19,   //default network buffer per input socket connection
     		 Integer.parseInt(System.getProperty("telemetry.port", "-1")),
     		 "tfb-database", // jdbc:postgresql://tfb-database:5432/hello_world
     		 "hello_world",
@@ -86,7 +81,7 @@ public class FrameworkTest implements GreenApp {
     	this.pipelineBits = 17;//max concurrent in flight database requests 1<<pipelineBits
 
     	this.dbCallMaxResponseCount = 1<<4;
-    	this.jsonMaxResponseCount = 1<<11;
+    	this.jsonMaxResponseCount = 1<<16;
     	
     	this.dbCallMaxResponseSize = 20_000; //for 500 mult db call in JSON format
     	this.jsonMaxResponseSize = 1<<9;
@@ -140,6 +135,8 @@ public class FrameworkTest implements GreenApp {
 	@Override
     public void declareConfiguration(GreenFramework framework) {
 		
+		framework.setDefaultRate(20_000);
+		
 		//for 14 cores this is expected to use less than 16G
 		framework.useHTTP1xServer(bindPort, this::parallelBehavior) //standard auto-scale
     			 .setHost(host)
@@ -149,7 +146,7 @@ public class FrameworkTest implements GreenApp {
     			 .setMaxQueueIn(queueLengthOfPendingRequests)
     			 
     			 .setMinimumInputPipeMemory(minMemoryOfInputPipes)
-    			 .setMaxQueueOut(32)
+    			 .setMaxQueueOut(64)
     			 .setMaxResponseSize(dbCallMaxResponseSize) //big enough for large mult db response
     	         .useInsecureServer(); //turn off TLS
 

frameworks/Java/greenlightning/src/main/java/com/ociweb/gl/benchmark/GreenLightning.java

@@ -5,10 +5,7 @@ import com.ociweb.gl.api.GreenRuntime;
 public class GreenLightning {
 
 	public static void main(String[] args) {
-		
-		//PipeConfig.showConfigsCreatedLargerThan = 1<<23;
-		
-		//System.setProperty("pronghorn.processors", "28"); //TODO: could also lower memory usage by shrinking the stack space...
+
 		GreenRuntime.run(new FrameworkTest(),args);
 	
 	}

frameworks/Java/greenlightning/src/main/java/com/ociweb/gl/benchmark/ProcessFortune.java

@@ -14,7 +14,7 @@ public class ProcessFortune {
 	private transient ObjectPipe<FortunesObject> fortuneInFlight;
 	private int htmlFortunePos=0;
 	private final transient AppendableBuilder htmlFortuneBuffer = new AppendableBuilder();
-	private final PoolManager pm;
+	private final transient PoolManager pm;
 	private final HTTPResponseService service;
 	
 	public ProcessFortune(int pipelineBits, HTTPResponseService service, PoolManager pm) {

frameworks/Java/greenlightning/src/main/java/com/ociweb/gl/benchmark/ProcessQuery.java

@@ -14,12 +14,12 @@ import io.reactiverse.pgclient.Tuple;
 
 public class ProcessQuery {
 	
-	private final ObjectPipe<ResultObject> DBRestInFlight;
+	private final transient ObjectPipe<ResultObject> DBRestInFlight;
 	private boolean collectionPendingDBRest = false;
 	private transient final List<ResultObject> collectorDBRest = new ArrayList<ResultObject>();
 	private final HTTPResponseService service;
 	private transient final PoolManager pm;
-	private final ThreadLocalRandom localRandom = ThreadLocalRandom.current();
+	private final transient ThreadLocalRandom localRandom = ThreadLocalRandom.current();
 
 	public ProcessQuery(int pipelineBits, HTTPResponseService service, PoolManager pm) {
 		

frameworks/Java/greenlightning/src/main/java/com/ociweb/gl/benchmark/ProcessUpdate.java

@@ -14,12 +14,12 @@ import io.reactiverse.pgclient.Tuple;
 
 public class ProcessUpdate {
 	
-	private ObjectPipe<ResultObject> DBUpdateInFlight;	
+	private transient ObjectPipe<ResultObject> DBUpdateInFlight;	
 	private boolean collectionPendingDBUpdate = false;	
-	private final List<ResultObject> collectorDBUpdate = new ArrayList<ResultObject>();
-	private final ThreadLocalRandom localRandom = ThreadLocalRandom.current();
+	private final transient List<ResultObject> collectorDBUpdate = new ArrayList<ResultObject>();
+	private final transient ThreadLocalRandom localRandom = ThreadLocalRandom.current();
 	private final HTTPResponseService service;
-	private final PoolManager pm;
+	private final transient PoolManager pm;
 	
 	public ProcessUpdate(int pipelineBits, HTTPResponseService service, PoolManager pm) {
 		this.DBUpdateInFlight = new ObjectPipe<ResultObject>(pipelineBits, ResultObject.class,	ResultObject::new);