Dump huffman, doesnt add much and complicates porting to other languages. Also fix compile error in idtool.

ext/huffandpuff/huffman.c

@@ -1,252 +0,0 @@
-/*
- * Huffandpuff minimal Huffman coder
- *
- * (c)2013 Adam Ierymenko <[email protected]>
- * This code is in the public domain and is distributed with NO WARRANTY.
- */
-
-#include "huffman.h"
-
-struct _huffman_node
-{
-	struct _huffman_node *lr[2];
-	struct _huffman_node *qprev,*qnext;
-	double prob;
-	unsigned long c;
-};
-
-struct _huffman_encode_table
-{
-	unsigned long code;
-	unsigned long bits;
-};
-
-static void _huffman_write_tree_and_make_encode_table(unsigned char *out,unsigned long *outbitctr,unsigned long outlen,struct _huffman_encode_table *et,unsigned long code,unsigned int bits,struct _huffman_node *t)
-{
-	struct _huffman_encode_table *eti;
-	unsigned int i;
-	unsigned long byte_index;
-
-	byte_index = (*outbitctr)++ >> 3;
-	byte_index *= (byte_index < outlen);
-	if (t->lr[0]) {
-		out[byte_index] <<= 1;
-		_huffman_write_tree_and_make_encode_table(out,outbitctr,outlen,et,code,bits + 1,t->lr[0]);
-		_huffman_write_tree_and_make_encode_table(out,outbitctr,outlen,et,code | (1 << bits),bits + 1,t->lr[1]);
-	} else {
-		out[byte_index] = (out[byte_index] << 1) | 1;
-		for(i=0;i<9;++i) {
-			byte_index = (*outbitctr)++ >> 3;
-			if (byte_index >= outlen) return;
-			out[byte_index] = (out[byte_index] << 1) | ((unsigned char)((t->c >> i) & 1));
-		}
-		eti = &(et[t->c]);
-		eti->code = code;
-		eti->bits = bits;
-	}
-}
-
-static struct _huffman_node *_huffman_read_tree(const unsigned char *in,unsigned long *inbitctr,unsigned long inlen,unsigned char **heapptr,unsigned char *heapend)
-{
-	struct _huffman_node *n;
-	unsigned int i;
-	unsigned long byte_index;
-
-	n = (struct _huffman_node *)(*heapptr);
-	*heapptr += sizeof(struct _huffman_node);
-	if (*heapptr > heapend) return (struct _huffman_node *)0;
-
-	byte_index = *inbitctr >> 3;
-	byte_index *= (byte_index < inlen);
-	if (((in[byte_index] >> (~((*inbitctr)++) & 7)) & 1)) {
-		n->lr[0] = (struct _huffman_node *)0;
-		n->lr[1] = (struct _huffman_node *)0;
-		n->c = 0;
-		for(i=0;i<9;++i) {
-			byte_index = *inbitctr >> 3;
-			if (byte_index >= inlen) return (struct _huffman_node *)0;
-			n->c |= (((unsigned int)(in[byte_index] >> (~((*inbitctr)++) & 7))) & 1) << i;
-		}
-	} else {
-		n->lr[0] = _huffman_read_tree(in,inbitctr,inlen,heapptr,heapend);
-		n->lr[1] = _huffman_read_tree(in,inbitctr,inlen,heapptr,heapend);
-		if (!((n->lr[0])&&(n->lr[1])))
-			return (struct _huffman_node *)0;
-	}
-
-	return n;
-}
-
-unsigned long huffman_compress(const unsigned char *in,unsigned long inlen,unsigned char *out,unsigned long outlen,void *huffheap)
-{
-	struct _huffman_encode_table *et,*eti;
-	struct _huffman_node *t,*n;
-	struct _huffman_node *pair[2];
-	unsigned char *heapptr = (unsigned char *)huffheap;
-	unsigned long i,code,byte_index,outbitctr;
-	unsigned int bits,b;
-	double *counts,lowest_prob,total_symbols;
-
-	counts = (double *)heapptr;
-	heapptr += (sizeof(double) * 257);
-	for(i=0;i<256;++i)
-		counts[i] = 0.0;
-	counts[256] = 1.0; /* one stop code at end */
-	for(i=0;i<inlen;++i)
-		counts[(unsigned long)in[i]] += 1.0;
-
-	t = (struct _huffman_node *)0;
-	total_symbols = (double)(inlen + 1);
-	for(i=0;i<=256;++i) {
-		if (counts[i] > 0.0) {
-			n = (struct _huffman_node *)heapptr;
-			heapptr += sizeof(struct _huffman_node);
-			if (t)
-				t->qprev = n;
-			n->qprev = (struct _huffman_node *)0;
-			n->qnext = t;
-			n->lr[0] = (struct _huffman_node *)0;
-			n->lr[1] = (struct _huffman_node *)0;
-			n->prob = counts[i] / total_symbols;
-			n->c = (unsigned int)i;
-			t = n;
-		}
-	}
-
-	while (t->qnext) {
-		for(i=0;i<2;++i) {
-			lowest_prob = 1.0;
-			pair[i] = (struct _huffman_node *)0;
-			n = t;
-			while (n) {
-				if (n->prob <= lowest_prob) {
-					lowest_prob = n->prob;
-					pair[i] = n;
-				}
-				n = n->qnext;
-			}
-			if (pair[i]->qprev)
-				pair[i]->qprev->qnext = pair[i]->qnext;
-			else t = pair[i]->qnext;
-			if (pair[i]->qnext)
-				pair[i]->qnext->qprev = pair[i]->qprev;
-		}
-		n = (struct _huffman_node *)heapptr;
-		heapptr += sizeof(struct _huffman_node);
-		n->lr[0] = pair[0];
-		n->lr[1] = pair[1];
-		n->prob = pair[0]->prob + pair[1]->prob;
-		if (t)
-			t->qprev = n;
-		n->qprev = (struct _huffman_node *)0;
-		n->qnext = t;
-		t = n;
-	}
-
-	et = (struct _huffman_encode_table *)heapptr;
-	heapptr += (sizeof(struct _huffman_encode_table) * 257);
-	outbitctr = 0;
-	_huffman_write_tree_and_make_encode_table(out,&outbitctr,outlen,et,0,0,t);
-
-	for(i=0;i<inlen;++i) {
-		eti = &(et[(unsigned long)in[i]]);
-		code = eti->code;
-		bits = eti->bits;
-		for(b=0;b<bits;++b) {
-			byte_index = outbitctr++ >> 3;
-			if (byte_index >= outlen) return 0;
-			out[byte_index] = (out[byte_index] << 1) | (unsigned char)(code & 1);
-			code >>= 1;
-		}
-	}
-	code = et[256].code;
-	bits = et[256].bits;
-	for(b=0;b<bits;++b) {
-		byte_index = outbitctr++ >> 3;
-		if (byte_index >= outlen) return 0;
-		out[byte_index] = (out[byte_index] << 1) | (unsigned char)(code & 1);
-		code >>= 1;
-	}
-
-	if (outbitctr > (outlen << 3))
-		return 0;
-	else if ((outbitctr & 7)) {
-		out[i = (outbitctr >> 3)] <<= 8 - (outbitctr & 7);
-		return (i + 1);
-	} else return (outbitctr >> 3);
-}
-
-unsigned long huffman_decompress(const unsigned char *in,unsigned long inlen,unsigned char *out,unsigned long outlen,void *huffheap)
-{
-	struct _huffman_node *t,*n;
-	unsigned char *heapptr = (unsigned char *)huffheap;
-	unsigned long inbitctr,outptr,byte_index = 0;
-
-	inbitctr = 0;
-	t = _huffman_read_tree(in,&inbitctr,inlen,&heapptr,heapptr + HUFFHEAP_SIZE);
-	if (!t) return 0;
-	outptr = 0;
-	for(;;) {
-		n = t;
-		while (n->lr[0]) {
-			byte_index = inbitctr >> 3;
-			if (byte_index >= inlen) return 0;
-			n = n->lr[((unsigned long)(in[byte_index] >> (~(inbitctr++) & 7))) & 1];
-		}
-		if (n->c == 256) return outptr;
-		if (outptr == outlen) return 0;
-		out[outptr++] = (unsigned char)n->c;
-	}
-}
-
-#ifdef HUFFANDPUFF_TEST
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-
-#define HUFFANDPUFF_TEST_MAXLEN 1048576
-#define HUFFANDPUFF_TEST_ITER 1024
-
-static unsigned char testin[HUFFANDPUFF_TEST_MAXLEN];
-static unsigned char testout[HUFFANDPUFF_TEST_MAXLEN * 2];
-static unsigned char testver[HUFFANDPUFF_TEST_MAXLEN];
-static unsigned char huffbuf[HUFFHEAP_SIZE];
-
-int main(int argc,char **argv)
-{
-	unsigned long i,k,l,cl,dcl;
-	int v;
-	unsigned char mask;
-
-	srand(time(0));
-
-	for(k=0;k<HUFFANDPUFF_TEST_ITER;++k) {
-		l = (rand() % HUFFANDPUFF_TEST_MAXLEN) + 1;
-		mask = (rand() & 0xff);
-		for(i=0;i<l;++i)
-			testin[i] = (unsigned char)(rand() & 0xff) & mask;
-		cl = huffman_compress(testin,l,testout,sizeof(testout),huffbuf);
-		if (cl) {
-			memset(testver,0,sizeof(testver));
-			dcl = huffman_decompress(testout,cl,testver,sizeof(testver),huffbuf);
-			v = ((dcl)&&(!memcmp(testver,testin,l)));
-			printf("[%d] in: %d, out: %d, verified: %s\n",(int)k,(int)l,(int)cl,(v) ? "OK" : "FAIL");
-		} else printf("[%d] in: %d, out: FAIL\n",(int)k,(int)l);
-	}
-
-	printf("\nFuzzing decompress function...\n");
-	for(;;) {
-		l = (rand() % HUFFANDPUFF_TEST_MAXLEN) + 1;
-		mask = (rand() & 0xff);
-		for(i=0;i<l;++i)
-			testin[i] = (unsigned char)(rand() & 0xff) & mask;
-		huffman_decompress(testin,l,testver,sizeof(testver),huffbuf);
-		printf("."); fflush(stdout);
-	}
-
-	return 0;
-}
-
-#endif

ext/huffandpuff/huffman.h

@@ -1,52 +0,0 @@
-/*
- * Huffandpuff minimal Huffman coder
- *
- * (c)2013 Adam Ierymenko <[email protected]>
- * This code is in the public domain and is distributed with NO WARRANTY.
- */
-
-#ifndef ____HUFFMAN_H
-#define ____HUFFMAN_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/**
- * Required size of huffheap parameter to compress and decompress
- *
- * Note: if you change any of the data types in the _huffman_node
- * or _huffman_encode_table structs in huffman.c, this also must be
- * changed.
- */
-#define HUFFHEAP_SIZE ((sizeof(double) * 257) + (((sizeof(void *) * 4) + sizeof(double) + sizeof(unsigned long)) * (257 * 3)) + ((sizeof(unsigned long) + sizeof(unsigned long)) * 257))
-
-/**
- * Huffman encode a block of data
- *
- * @param in Input data
- * @param inlen Input data length
- * @param out Output buffer
- * @param outlen Output buffer length
- * @param huffheap Heap memory to use for compression (must be HUFFHEAP_SIZE in size)
- * @return Size of encoded result or 0 on out buffer overrun
- */
-extern unsigned long huffman_compress(const unsigned char *in,unsigned long inlen,unsigned char *out,unsigned long outlen,void *huffheap);
-
-/**
- * Huffman decode a block of data
- *
- * @param in Input data
- * @param inlen Length of input data
- * @param out Output buffer
- * @param outlen Length of output buffer
- * @param huffheap Heap memory to use for decompression (must be HUFFHEAP_SIZE in size)
- * @return Size of decoded result or 0 on out buffer overrun or corrupt input data
- */
-extern unsigned long huffman_decompress(const unsigned char *in,unsigned long inlen,unsigned char *out,unsigned long outlen,void *huffheap);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

idtool.cpp

@@ -159,7 +159,7 @@ int main(int argc,char **argv)
 		}
 
 		std::string signature(Utils::base64Decode(argv[4],strlen(argv[4])));
-		if ((signature.length() > ZEROTIER_ADDRESS_LENGTH)&&(id.verifySignature(inf.data(),inf.length(),signature))) {
+		if ((signature.length() > ZT_ADDRESS_LENGTH)&&(id.verifySignature(inf.data(),inf.length(),signature.data(),signature.length()))) {
 			std::cout << argv[3] << " signature valid" << std::endl;
 		} else {
 			std::cerr << argv[3] << " signature check FAILED" << std::endl;

node/Utils.hpp

@@ -42,7 +42,6 @@
 
 #include "../ext/lz4/lz4.h"
 #include "../ext/lz4/lz4hc.h"
-#include "../ext/huffandpuff/huffman.h"
 
 #include "Constants.hpp"
 
@@ -176,7 +175,6 @@ public:
 	template<typename I,typename O>
 	static inline void compress(I begin,I end,O out)
 	{
-		char huffheap[HUFFHEAP_SIZE];
 		unsigned int bufLen = LZ4_compressBound(ZT_COMPRESSION_BLOCK_SIZE);
 		char *buf = new char[bufLen * 2];
 		char *buf2 = buf + bufLen;
@@ -210,16 +208,9 @@ public:
 					continue;
 				}
 
-				unsigned long huffCompressedLen = huffman_compress((const unsigned char *)buf2,lz4CompressedLen,(unsigned char *)buf,bufLen,huffheap);
-				if ((!huffCompressedLen)||((int)huffCompressedLen >= lz4CompressedLen)) {
-					l = hton((uint32_t)lz4CompressedLen); // lz4 only
-					out((const void *)&l,4);
-					out((const void *)buf2,(unsigned int)lz4CompressedLen);
-				} else {
-					l = hton((uint32_t)0x80000000 | (uint32_t)huffCompressedLen); // lz4 with huffman
-					out((const void *)&l,4);
-					out((const void *)buf,(unsigned int)huffCompressedLen);
-				}
+				l = hton((uint32_t)lz4CompressedLen); // lz4 only
+				out((const void *)&l,4);
+				out((const void *)buf2,(unsigned int)lz4CompressedLen);
 			}
 
 			delete [] buf;
@@ -242,7 +233,6 @@ public:
 	template<typename I,typename O>
 	static inline bool decompress(I begin,I end,O out)
 	{
-		char huffheap[HUFFHEAP_SIZE];
 		volatile char i32c[4];
 		void *const i32cp = (void *)i32c;
 		unsigned int bufLen = LZ4_compressBound(ZT_COMPRESSION_BLOCK_SIZE);
@@ -279,23 +269,10 @@ public:
 						return false;
 					}
 
-					if ((_compressedSize & 0x80000000)) { // lz4 and huffman
-						unsigned long lz4CompressedSize = huffman_decompress((const unsigned char *)buf,compressedSize,(unsigned char *)buf2,bufLen,huffheap);
-						if (lz4CompressedSize) {
-							if (LZ4_uncompress_unknownOutputSize(buf2,buf,lz4CompressedSize,bufLen) != (int)originalSize) {
-								delete [] buf;
-								return false;
-							} else out((const void *)buf,(unsigned int)originalSize);
-						} else {
-							delete [] buf;
-							return false;
-						}
-					} else { // lz4 only
-						if (LZ4_uncompress_unknownOutputSize(buf,buf2,compressedSize,bufLen) != (int)originalSize) {
-							delete [] buf;
-							return false;
-						} else out((const void *)buf2,(unsigned int)originalSize);
-					}
+					if (LZ4_uncompress_unknownOutputSize(buf,buf2,compressedSize,bufLen) != (int)originalSize) {
+						delete [] buf;
+						return false;
+					} else out((const void *)buf2,(unsigned int)originalSize);
 				} else { // stored
 					if (originalSize > bufLen) {
 						delete [] buf;

objects.mk

@@ -1,5 +1,4 @@
 OBJS=\
-	ext/huffandpuff/huffman.o \
 	ext/kissdb/kissdb.o \
 	ext/lz4/lz4hc.o \
 	ext/lz4/lz4.o \

selftest.cpp

@@ -43,7 +43,6 @@
 #include "node/HMAC.hpp"
 #include "node/MAC.hpp"
 #include "node/Peer.hpp"
-#include "node/Http.hpp"
 #include "node/Condition.hpp"
 #include "node/NodeConfig.hpp"
 
@@ -302,39 +301,12 @@ static int testOther()
 	return 0;
 }
 
-static Condition testHttpDoneCondition;
-
-static bool testHttpHandler(Http::Request *req,void *arg,const std::string &url,int code,const std::map<std::string,std::string> &headers,const std::string &body)
-{
-	if (code)
-		std::cout << "[net] " << url << " " << code << " bytes: " << body.length() << std::endl;
-	else std::cout << "[net] " << url << " FAILED: " << body << std::endl;
-	testHttpDoneCondition.signal();
-	return false;
-}
-
-static int testNet()
-{
-	std::cout << "[net] GET http://www.uc.edu/" << std::endl;
-	new Http::Request(Http::HTTP_METHOD_GET,"http://www.uc.edu/",Http::EMPTY_HEADERS,std::string(),&testHttpHandler,(void *)0);
-	testHttpDoneCondition.wait();
-	std::cout << "[net] GET http://zerotier.com/" << std::endl;
-	new Http::Request(Http::HTTP_METHOD_GET,"http://zerotier.com/",Http::EMPTY_HEADERS,std::string(),&testHttpHandler,(void *)0);
-	testHttpDoneCondition.wait();
-	std::cout << "[net] GET http://www.google.com/" << std::endl;
-	new Http::Request(Http::HTTP_METHOD_GET,"http://www.google.com/",Http::EMPTY_HEADERS,std::string(),&testHttpHandler,(void *)0);
-	testHttpDoneCondition.wait();
-
-	return 0;
-}
-
 int main(int argc,char **argv)
 {
 	int r = 0;
 
 	srand(time(0));
 
-	r |= testNet();
 	r |= testCrypto();
 	r |= testPacket();
 	r |= testOther();