More test stuff... make it more granular and less batch based.

tests/http/agent.js

@@ -3,21 +3,23 @@
 // ---------------------------------------------------------------------------
 // Customizable parameters:
 
-// Maximum test duration in milliseconds
-var TEST_DURATION = (30 * 1000);
+// Maximum interval between test attempts
+//var TEST_INTERVAL_MAX = (60 * 1 * 1000);
+var TEST_INTERVAL_MAX = 1000;
 
-// Interval between tests (should be several times longer than TEST_DURATION)
-var TEST_INTERVAL = (60 * 2 * 1000);
+// Test timeout in ms
+var TEST_TIMEOUT = 30000;
 
 // Where should I contact to register and query a list of other test agents?
-var SERVER_HOST = '104.238.141.145';
+var SERVER_HOST = '127.0.0.1';
+//var SERVER_HOST = '104.238.141.145';
 var SERVER_PORT = 18080;
 
 // Which port should agents use for their HTTP?
 var AGENT_PORT = 18888;
 
 // Payload size in bytes
-var PAYLOAD_SIZE = 100000;
+var PAYLOAD_SIZE = 10000;
 
 // ---------------------------------------------------------------------------
 
@@ -72,9 +74,6 @@ for(var xx=0;xx<PAYLOAD_SIZE;++xx) {
 	payload.writeUInt8(Math.round(Math.random() * 255.0),xx);
 }
 
-// Incremented with each test
-var testNumber = 0;
-
 function agentIdToIp(agentId)
 {
 	var ip = '';
@@ -96,32 +95,93 @@ function agentIdToIp(agentId)
 	return ip;
 };
 
-function registerAndGetPeers(callback)
+var lastTestResult = null;
+var allOtherAgents = [];
+
+function doTest()
 {
-	http.get({
+	var submit = http.request({
 		host: SERVER_HOST,
 		port: SERVER_PORT,
-		path: '/'+thisAgentId
+		path: '/'+thisAgentId,
+		method: 'POST'
 	},function(res) {
 		var body = '';
 		res.on('data',function(chunk) { body += chunk.toString(); });
 		res.on('end',function() {
-			if (!body)
-				return callback(null,[]);
-			try {
-				var peers = JSON.parse(body);
-				if (Array.isArray(peers))
-					return callback(null,peers);
-				else return callback(new Error('invalid JSON response from server'),null);
-			} catch (e) {
-				return callback(new Error('invalid JSON response from server: '+e.toString()),null);
+
+			if (body) {
+				try {
+					var peers = JSON.parse(body);
+					if (Array.isArray(peers))
+						allOtherAgents = peers;
+				} catch (e) {}
+			}
+
+			if (allOtherAgents.length > 0) {
+
+				var target = allOtherAgents[Math.floor(Math.random() * allOtherAgents.length)];
+
+				var testRequest = null;
+				var timeoutId = null;
+				timeoutId = setTimeout(function() {
+					if (testRequest !== null)
+						testRequest.abort();
+					timeoutId = null;
+				});
+				var startTime = Date.now();
+
+				testRequest = http.get({
+					host: agentIdToIp(target),
+					port: AGENT_PORT,
+					path: '/'
+				},function(res) {
+					var bytes = 0;
+					res.on('data',function(chunk) { bytes += chunk.length; });
+					res.on('end',function() {
+						lastTestResult = {
+							source: thisAgentId,
+							target: target,
+							time: (Date.now() - startTime),
+							bytes: bytes,
+							timedOut: (timeoutId === null),
+							error: null
+						};
+						if (timeoutId !== null)
+							clearTimeout(timeoutId);
+						return setTimeout(doTest,Math.round(Math.random() * TEST_INTERVAL_MAX) + 1);
+					});
+				}).on('error',function(e) {
+					lastTestResult = {
+						source: thisAgentId,
+						target: target,
+						time: (Date.now() - startTime),
+						bytes: 0,
+						timedOut: (timeoutId === null),
+						error: e.toString()
+					};
+					if (timeoutId !== null)
+						clearTimeout(timeoutId);
+					return setTimeout(doTest,Math.round(Math.random() * TEST_INTERVAL_MAX) + 1);
+				});
+
+			} else {
+				return setTimeout(doTest,Math.round(Math.random() * TEST_INTERVAL_MAX) + 1);
 			}
+
 		});
 	}).on('error',function(e) {
-		return callback(e,null);
+		console.log('POST failed: '+e.toString());
+		return setTimeout(doTest,1000);
 	});
+	if (lastTestResult !== null) {
+		submit.write(JSON.stringify(lastTestResult));
+		lastTestResult = null;
+	}
+	submit.end();
 };
 
+/*
 function performTestOnAllPeers(peers,callback)
 {
 	var allResults = {};
@@ -191,11 +251,10 @@ function doTestsAndReport()
 			console.error('WARNING: skipping test: unable to contact or query server: '+err.toString());
 		} else {
 			performTestOnAllPeers(peers,function(results) {
-				++testNumber;
 				var submit = http.request({
 					host: SERVER_HOST,
 					port: SERVER_PORT,
-					path: '/'+testNumber+'/'+thisAgentId,
+					path: '/'+thisAgentId,
 					method: 'POST'
 				},function(res) {
 				}).on('error',function(e) {
@@ -207,29 +266,12 @@ function doTestsAndReport()
 		}
 	});
 };
+*/
 
 // Agents just serve up a test payload
-app.get('/',function(req,res) {
-	return res.status(200).send(payload);
-});
+app.get('/',function(req,res) { return res.status(200).send(payload); });
 
 var expressServer = app.listen(AGENT_PORT,function () {
-	var serverUp = false;
-	async.whilst(
-		function() { return (!serverUp); },
-		function(nextTry) {
-			registerAndGetPeers(function(err,peers) {
-				if ((err)||(!peers)) {
-					setTimeout(nextTry,1000);
-				} else {
-					serverUp = true;
-					return nextTry(null);
-				}
-			});
-		},
-		function(err) {
-			console.log('Server up, starting!');
-			doTestsAndReport();
-			setInterval(doTestsAndReport,TEST_INTERVAL);
-		});
+	// Start timeout-based loop
+	doTest();
 });

tests/http/big-test-kill.sh

@@ -13,6 +13,6 @@ CONTAINER_IMAGE=zerotier/http-test
 
 export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 
-pssh -h big-test-hosts -i -t 128 -p 256 "docker ps -q | xargs -r docker kill && docker ps -aq | xargs -r docker rm"
+pssh -h big-test-hosts -i -t 128 -p 256 "docker ps -aq | xargs -r docker rm -f"
 
 exit 0

tests/http/big-test-out/[email protected]

@@ -1,67 +0,0 @@
-5fc1676570b10196c5e261dac6ff28998c4f66c706b131d6423532a38b8b7a15
-894bbfa612f772aa0170d5c5ddb9362a9edb0f5fbe22c81591b50dfa0a0eadeb
-bae9bf7c37806adbbdfe9e01b4e62034953d01ecae55e0cd450a21b1415f5c00
-26e83eccbd6083f2c7a8532786fd1ec5c96d6a51bb3508ec4fd1919a1883630a
-90e95ac0cd3b4ba6ffa33fe946a1f12a2cdba61168770af9fca7b42df45c9530
-88a11bbc89c0b3579de7faf4db4dbe3f0a5a073aa49fd7a5482eff35c93fab4e
-50be80765f09000085d6c0895958e6d3794f672f562ff5bc27634b15c77b7583
-99630786ef067f4c053f7bca96e2869f14c9ceb9f1d07b7635945c143700b950
-8422d4cb09ed44e34488163a96910ece8f1dd39c280daf3d369d17ce2ac70766
-e0a0af014418076adc351827fabbd9ef6da7404f2d6184f20d601736c4685154
-387cd6dbbd8e8b17d9ccf943fd13dcaddf27c178834a9630e357407a42733fb8
-b06b75cb0c5748321ab4501e1055dd5ff8457e81502548954702ead843011388
-59d874f5e36f15b62e8540b86d552950c72f086d90667780b6d82b5595326fce
-69dc1347f8671ba4bbe1da12f26e8f67b0980f1e2ad73bc0b77cc06e5cbf1b06
-68ef4b43d3f5559b5b0c82ba2f396a5b6dcb6001a67efd3a3b3b2a415c2b7b61
-3ecc9f45151f95a194d8274a88f433f83540f5397523de7a86db714cd9155bb2
-3b24b66b1dcd5e8ce1fa24d33cb2eb3cc55f3a157602a09bca4942089e25790f
-ba34465f6cedd2f306022cc9259ef4e43819959f51f980a8cb94ea33a29c7e34
-4e9a701f18a0ac42ee24f3bf2dd6dc442706a0bf4b6288336447b03752640852
-5865df2182165576a0825644d9b7537314c9fd4323cb2023260382acc2f9d7fd
-f1498fda2dd6d1f1fdfc95d539d6ee511ca8baad65b1a1b44d76309e84015550
-7d9232ecda50856096523dc2d0c1fae46481053067d8bfe024e1dfccf8f9c0fc
-173b47f327a3c21187e25d2d02dbde49760182a40e00cf4a64746c00537a3ff6
-9221fdb92693a29f720a5c41f09791c35f2b220459a2e4bdcc24fd5e3af4ad85
-ef8def0260ba3ff21aa292bb482bedf94b17fd9306e529f6feae5cab04e1bdc2
-7392179b9d51cad626b6cf78b00c355b280ddccccd7013e11a8cecbdd2db1c15
-fc31766af9265f2c496ee3847971ee0f5249aca8ad0cd214620643c7347436b2
-7f8bd6d1fde6948a09b132b4b2c3919ca11c932bfdf8a279c4cbe140daf24287
-ce9596c1d8dd42929820e4d3a56e8b1f8eb2d8b67474c525bbed1b888143cc14
-67c9de4f3d88daa89796e8a03306d2c5f2bcc4c409eef89e67c6ffc6a6282060
-705c1973e4a9ef77ffe6319375e585aec1d762a031c8b93ba9883af98e377590
-c7fdcc10eed007a30da7c7bbb8767d0b6a7287fadd67d935651f0bb265a71e1b
-5102e854fe72cdcd91b228d520a9380a36a273df7949de054355e8f99bd14e95
-77c37e503ccaa1350bf2fe16daaef486ec223348bc8f678b6f6d6bb477ae10c1
-fd97792d7bd61449163fa4e953212446e11cf02b27a45a2073a1de5148da63e8
-7d5b84b290f727713ad02f0817856c9b891996bdcc6ad4d0994005608cf9bbb7
-104c41c9cce7934f0e205cfc90b65f5ba89ca696cab41771a12b4384af7f6805
-75e1751a7e290a34faddea54c98e870b4a1cd5bb37c810cafabda8d3ba1faa5f
-2293a17e82ff54e04218c5aafd904079d15d71e47496249dc125f90f0960039f
-3f3563782349fddb61cdc638cf5f54030f726d9759cb104253f5b8b04ae8e2b0
-2328da71617e1abc0e2335e5974a70f45f8abaedba641292ad4d87e2f27a6b83
-00e79b478925c7b866f7d669ee73af1a7b377fb8bf22a04f2bd5356f256f59ba
-2960f757cca294c32abd51f072798f5457a1552de52bd42ca9233cd075da086f
-efbc248c9a3ce7b7a52724b67f14e27b02f5f37da295bc905d4f8fabe847cd00
-a068d735478e065236e840e50697a078d77aad9b82f906555279e5dee074db5d
-c4a366870dd1f3d1fb776b25f009d9079a6f7d0d83b03cd178b237b412d8dce6
-20340c7bd3bd9d32d6ecb7a451c377fb239bfe2d2c976886e2bd59746ed180ee
-8c522ecbabb9580528794d94f82bdef2982f0458b677598f236acc14e819e480
-8dfcc3407b050a39d82af263eec6e332bd69cb848ffa660344644ea10e3c2221
-74e06d9deba29982f9dccdb841163715dd419dee6a54eaf3ac987cd3458a1a2a
-725d990ca2ea34dc3e9acc02480cc8009a9c2016414c9c9cca3c7b135bd384f4
-120d7eac6a5bd761ee6acfb751f48bf7075c0316123326ce6bbb6ce3fe05c3a7
-91aa44d2650ff308877d9c81357619c51c0c0d05dc9f4c899df9ce460673b2aa
-3c203cd73c6be2606397357154789a94fbfe8670271825f75bbfc6c97fa0e048
-f9018c8390a472798c7a39bfe834cd01bb62bb4b0882dfe1108bf43334a3bd0e
-cdf8afae641e0423b2a7a1ff92cca80c7db478ff8deb1d81032808ac84415921
-9a5364df1df5460f6a9ed15e020b8bd283c47464556a7c5896707cee00c01a14
-df9365b8816e9d67f484adf94eef53aab236a92a588a1f3f650fd36b8073f7bb
-bd0172f67fa20716502e2bee82387de7f426e3cdb19d5ac6ce9dcc177f919cde
-90dd259c03b11625c09b8db614f45759e67edf07fe350681d273bfd988b45443
-9f69d376248b6851aa7837a7d09a1b9eda917601049e5942796815069d09a80f
-1a065bee20e8f4c6a91bb92ec9cca6ca16e8eb434798ed433a5248c48d91f596
-fb5a6a9397546a97ccdc4252603d5e774d8430195b06ec74926c48cf372b9906
-51cf0867773bb298140eb09c88d69587aae3a6da3e63275d32f3d32d98a737d7
-70574f3e616413ce90b045e0e9fd92353766e216eabf8139556fa61efea9c3b4
-ccd21257994f9eaf309d06b2fc5652b14ea80796a79face304fcc8fd1da53423
-7d8417be656f17fbfb779a9803b2de045e2f496f75ea5f1ea69e223572bde2a5

tests/http/big-test-out/[email protected]

@@ -1,75 +0,0 @@
-5ea6049a0b92070494f40a5ccecccaf788a5aafccee7c2eada9b9eb8731bc002
-798f8beecd2e3fbb50df49b7ef57cdd1e8e00c0680046b3c2d53a3554f956fda
-dd20da6340210e1b7612d8922aaa4b045e84da32f264add073a65a15f676a9e4
-0479f14d0aa68e835c07dc5ca413febd9da19b6554fd8bdba7e319e5f4661f80
-df6747868f90cfa069f5f9f954626b7392cd99026e43e6d6c83ab7c16d5cbdc6
-c24799f74233b1bc7d7d936d57699b955000c640531f3db38be8196a87eb262c
-46b00a65d527738c0bfad924051bd2117563e0c6ad74b803b662e74720d8d085
-dc14b9428032771388d30c6002bb5cba05131972cab53360f088c51769786c47
-e7dc364aeccf60bafa5a42787cc6de231612782252f57b9f03ebae3a309b2352
-4e8e578a8948ab384525646a17c2e0cb9f2b9ef67fd0c489ad6aa2bffbfabddc
-cc626d978e32dfe14782011218ba265ae4e69886f44335a2c402001dc0c4c3c7
-36a148254d34f954906a810ba4a8644a4433e8847d3cc30e091b1f63723f0590
-1bdd06b691fa06b3af77d2868b78f2a01b91026b4f483fb278cb8872a9150987
-d6d5709039708a515b295519a4007c3b49361ef67465ddca2dfba9a473b9c37c
-f8a3a0b3dd5ce42bd87cdba28df7469071f948fb28151955cfa75abe0455a000
-f7eb37fecb17571091f05f1bfc66f6fba731ff934988a529813f4751451401af
-1c2f2c77c429095b1dad53032684df672f351489ed6b7e00e1097f7dc1c0ba97
-361dd9cc6764facd2aca0b462e64a2400f6b41e124c4d9be71466d801763270a
-3f5f49ce854439bc672c4b0cf4c1e1b9a978e8f2db14709977ddcfd39b7d6bb3
-5221bec1aec2c9f08ddec548a24b0700e3d0c0568d10caa753564c914b35c95f
-ebdcc27c264d326619262f82b5d7dfcbf102d720ad9bab4428b9725118bf627a
-bc01c863316a29c8f119da7cce1c891185c43d385521d46b06186a89cb6cff9d
-47fb8b119e3d098f22cc6f4a7867f2016f244cc8b114aa66630dadbd4bfb2a0c
-4ca2402d762adbd7ce860c3f3d072e948bb33afb7c2830ad51ae9d3fb2c714df
-328e4fda6dd1befcfda64d5c89c458fcc9386d88375218cb5197d479d3d292d0
-099636c67de66af06ee8493ccd55b588cf8bbacf67352877f37a077a2166117a
-624e0c032e4b8a78af1387c0199c5b02da68e0795e9b6397ba8bdd5ebcf7daa4
-265e038f17e4bc3c99737bf4ac98364c98a12d9d22a28b6a9302ce5a83c7d4f8
-de198984870126801aa20f25c459ea8f89bb7a4782614e91659b820c42a33c93
-ad48788c3f91a1292fb28dde84750f94e27ef150b2ebb52b7807f3d0c7986c4b
-34ab2f912eae5e75bbde977b8bb2952e552e36c83f29c2cd64d7ac3165aa9726
-951efa9c57da7eb2a1f6578927f809ab0c9feac2aa4326ea42f182e3ad74e600
-8b8193d9ebd89c36a728a3fb89282854bfb89b27172d93952747126c22ea6a97
-79a78a9f96f2c961b2d06dfd484a495ddc3809228243c608ecd51715a228e528
-4301dfc330137391585d36ddc4b54971999a1da96b1c12aa43221dd92cc32e79
-f956660447a893adbc0fc4382fe67ad2a7bb8591a647a130faa9e17aa380328a
-0cee22f18107559eff4f4eba20320c349d70d82b72197ae380fc514e6241730c
-d74601767fec3cff13062ad1393fad9a88277bbc6710c4fce6d78c21d001bfa4
-c1c46fd958b7806bf0c22c77c997317b7e4dffd7243d0a5919d4922fafd841a2
-a07e0f4e1cac4a84e3ca922b8c59a37ed8049096d9154be76e6d2d9094ba3714
-6861c27e7584e83a792ce710d004f7dc22213cc732b23f0a2025606cc5e9e325
-ee334626da5143ddf49561522483eb689663d031ba6cf9891204b709b279b28b
-3be351d25381f07b85c7f5c2bb08b80b5bbed80c348515187d727ccbf293b13b
-268da41127aa3b7768bbf6075baad5397f0af4f0ff16f5b8dff2cad9c3019750
-74cc47af92d6cf6b2315f62e0261d4461c6297c5ffc19b50a97784cd6271acfc
-8e52beef1bd61be4d223c460b589d32d1a64d5525406ab179d1962fccd734309
-9471871b2f1ec2331c1f9855b408d212ba868965f99aeee91ddd0b7dd76b2985
-16eb9e7e24b61ade0939345118d5d14032ae496de3b5fc702c0c1356662b8a80
-f503549eb9c03c8dc7e54559bf076e1f7eaf7c8599c84722062851597a4c91c1
-d9dce304b504003c97ffdf9d076ff348343d7d0ce50070038d049b71239bc70e
-2ed388e1730860b7605527213d7d61bbc8f29703a2f586d127b37e7ed8eca708
-d20988195a901d19597bf4ed2b136c3d2aed2d169093409a5d3ea8daf1f983af
-a9be22365634b68f0ff5ba9228550eba9b3923319eff07f5cf5785eec85fa11e
-99b512fe14b21569c81d358b161976c9dce45c608cd1a03da6f6063dbabb6c41
-88f76143cb289e8daf8edbc183a4b760c0d86efd1cc9da3933e59603bf92c539
-62054c1a23221461f1c06b94148614754f7da0cd7145eb85290bb11bdf7b9af8
-8dffffd5b63672f8749160dbff55c489f1f1f6c41ce01e086f2d9aa8bc6150ad
-f86a0512be3ec599065dec982545ecb96bdb0d82dda7285c58b3c3b7666988c9
-773786ea8094e7f013702ef721a2dde7657dcb6c44927f41f5acb32161957d83
-ef981b569d0c98e79bfc5bb387cf00457c4254e13f1ec625d0149b06ef92d5e1
-77841624d9fbba73cf722327f531ea36cfa42be59b4709c3b4ca4e7e453ac7fa
-f90ada865c766ef7e0e47d0a677b64183aa1e612f14f3f1f996d411b7197ea3e
-11c278763c1f2ff8d9b2ee5e3100a40538adcc1b74f6ca7d9668fcc8c1ab2f9a
-4a72c59eda21b43136cb5f308298aba39235bbd227668c2c9f3830fbab7b4a34
-f12a3d9308eb7800a1436b3258d899aae3b643f2c78648258f126266f9707032
-ce651585240975fc5c954c526203a048c6e4326e2d16feb083ca3a3ff4ad682a
-43aa6c22cb636983ff1b6e169057b0d7a70ad75754ef1b3a3bc7f49461c84cdb
-7e71c166a6f285c4c501b2125713e698575d1987f1819b76d4fabbbf246eae6c
-d38478eec0109a5b76da1a6e3de982382cf01bbc871b651d3e258330642cbe27
-c4c6ce60a0b1c4f2553ecb0f551de2c7df2e2d2cf2101e80af48b29e03cfefd5
-38905fb0f59281f1d4aa80894654e56df76653f3d3545a883e37c80053e72977
-1a6ee044aa753748035975b12885e73504ffde1bf129a7ff992f012d9cff111b
-f0d37e0a5ccb7871a30b143fad68a4e07624aa2a153e295022868e68e34ff770
-ee9d0ef6b557bd5e7c8fba9e087a428a98ee5350ec86785205db6ea10493b21c
-53ca280c12ba4d5be4ea78144fb2d411ecd9910f5105d04537d4bec362865c40

tests/http/big-test-start.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
-NUM_CONTAINERS=64
+NUM_CONTAINERS=100
 CONTAINER_IMAGE=zerotier/http-test
 
 #
@@ -25,6 +25,6 @@ export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 #	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
 #done
 
-pssh -o big-test-out -h big-test-hosts -i -t 128 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; done"
+pssh -h big-test-hosts -i -t 128 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done"
 
 exit 0

tests/http/server.js

@@ -20,42 +20,22 @@ app.use(function(req,res,next) {
 
 var knownAgents = {};
 
-app.get('/:agentId',function(req,res) {
-	var agentId = req.params.agentId;
-	if ((!agentId)||(agentId.length !== 32))
-		return res.status(404).send('');
-	knownAgents[agentId] = Date.now();
-	return res.status(200).send(JSON.stringify(Object.keys(knownAgents)));
-});
-
-app.post('/:testNumber/:agentId',function(req,res) {
-	var testNumber = req.params.testNumber;
+app.post('/:agentId',function(req,res) {
 	var agentId = req.params.agentId;
 	if ((!agentId)||(agentId.length !== 32))
 		return res.status(404).send('');
 
-	var receiveTime = Date.now();
-	var resultData = null;
-	try {
-		resultData = JSON.parse(req.rawBody);
-	} catch (e) {
-		resultData = req.rawBody;
+	if (req.rawBody) {
+		var receiveTime = Date.now();
+		var resultData = null;
+		try {
+			resultData = JSON.parse(req.rawBody);
+			console.log(resultData.source+','+resultData.target+','+resultData.time+','+resultData.bytes+','+resultData.timedOut+',"'+((resultData.error) ? resultData.error : '')+'"');
+		} catch (e) {}
 	}
-	result = {
-		agentId: agentId,
-		testNumber: parseInt(testNumber),
-		receiveTime: receiveTime,
-		results: resultData
-	};
-
-	testNumber = testNumber.toString();
-	while (testNumber.length < 10)
-		testNumber = '0' + testNumber;
-	fs.writeFile('result_'+testNumber+'_'+agentId,JSON.stringify(result),function(err) {
-		console.log(result);
-	});
 
-	return res.status(200).send('');
+	knownAgents[agentId] = Date.now();
+	return res.status(200).send(JSON.stringify(Object.keys(knownAgents)));
 });
 
 var expressServer = app.listen(SERVER_PORT,function () {