new network code. not perfect. connectionless packets is missing so no server discovery is possible. reduced network traffic by A LOT

16 files changed, 1154 insertions, 515 deletions

diff --git a/src/engine/client/ec_client.c b/src/engine/client/ec_client.c
index 4e139b75..c2143121 100644
--- a/src/engine/client/ec_client.c
+++ b/src/engine/client/ec_client.c
@@ -21,6 +21,8 @@
 #include <engine/e_console.h>
 #include <engine/e_ringbuffer.h>
 
+#include <engine/e_huffman.h>
+
 #include <mastersrv/mastersrv.h>
 
 const int prediction_margin = 7; /* magic network prediction value */
@@ -280,19 +282,21 @@ float client_localtime() { return (time_get()-local_start_time)/(float)(time_fre
 int client_send_msg()
 {
 	const MSG_INFO *info = msg_get_info();
-	NETPACKET packet;
+	NETCHUNK packet;
 	
 	if(!info)
 		return -1;
 		
-	mem_zero(&packet, sizeof(NETPACKET));
+	mem_zero(&packet, sizeof(NETCHUNK));
 	
 	packet.client_id = 0;
 	packet.data = info->data;
 	packet.data_size = info->size;
 
-	if(info->flags&MSGFLAG_VITAL)	
-		packet.flags = PACKETFLAG_VITAL;
+	if(info->flags&MSGFLAG_VITAL)
+		packet.flags = NETSENDFLAG_VITAL;
+	if(info->flags&MSGFLAG_FLUSH)
+		packet.flags = NETSENDFLAG_FLUSH;
 	
 	netclient_send(net, &packet);
 	return 0;
@@ -380,7 +384,11 @@ static void client_send_input()
 	/* fetch input */
 	size = modc_snap_input(inputs[current_input].data);
 	
-	msg_pack_start_system(NETMSG_INPUT, 0);
+	if(!size)
+		return;
+	
+	/* pack input */
+	msg_pack_start_system(NETMSG_INPUT, MSGFLAG_FLUSH);
 	msg_pack_int(ack_game_tick);
 	msg_pack_int(current_predtick);
 	msg_pack_int(size);
@@ -539,18 +547,20 @@ static void client_debug_render()
 	gfx_texture_set(debug_font);
 	gfx_mapscreen(0,0,gfx_screenwidth(),gfx_screenheight());
 	
-	if(time_get()-last_snap > time_freq()/10)
+	if(time_get()-last_snap > time_freq())
 	{
 		last_snap = time_get();
 		prev = current;
-		netclient_stats(net, &current);
+		net_stats(&current);
 	}
 	
 	frametime_avg = frametime_avg*0.9f + frametime*0.1f;
-	str_format(buffer, sizeof(buffer), "ticks: %8d %8d send: %6d recv: %6d snaploss: %d  mem %dk   gfxmem: %dk  fps: %3d",
+	str_format(buffer, sizeof(buffer), "ticks: %8d %8d send: %5d/%3d recv: %5d/%3d snaploss: %d  mem %dk   gfxmem: %dk  fps: %3d",
 		current_tick, current_predtick,
-		(current.send_bytes-prev.send_bytes)*10,
-		(current.recv_bytes-prev.recv_bytes)*10,
+		(current.sent_bytes-prev.sent_bytes),
+		(current.sent_packets-prev.sent_packets),
+		(current.recv_bytes-prev.recv_bytes),
+		(current.recv_packets-prev.recv_packets),
 		snaploss,
 		mem_allocated()/1024,
 		gfx_memory_usage()/1024,
@@ -653,7 +663,6 @@ static const char *client_load_map_search(const char *mapname, int wanted_crc)
 	return error;
 }
 
-
 static int player_score_comp(const void *a, const void *b)
 {
 	SERVER_INFO_PLAYER *p0 = (SERVER_INFO_PLAYER *)a;
@@ -665,7 +674,7 @@ static int player_score_comp(const void *a, const void *b)
 	return -1;
 }
 
-static void client_process_packet(NETPACKET *packet)
+static void client_process_packet(NETCHUNK *packet)
 {
 	if(packet->client_id == -1)
 	{
@@ -955,7 +964,7 @@ static void client_process_packet(NETPACKET *packet)
 						int purgetick;
 						void *deltadata;
 						int deltasize;
-						unsigned char tmpbuffer[MAX_SNAPSHOT_SIZE];
+						/*unsigned char tmpbuffer[MAX_SNAPSHOT_SIZE];*/
 						unsigned char tmpbuffer2[MAX_SNAPSHOT_SIZE];
 						unsigned char tmpbuffer3[MAX_SNAPSHOT_SIZE];
 						int snapsize;
@@ -995,12 +1004,13 @@ static void client_process_packet(NETPACKET *packet)
 						if(complete_size)
 						{	
 							int intsize;
+							/*
 							int compsize = zerobit_decompress(snapshot_incomming_data, complete_size, tmpbuffer);
 							
-							if(compsize < 0) /* failure during decompression, bail */
-								return;
+							if(compsize < 0)  failure during decompression, bail 
+								return;*/
 								
-							intsize = intpack_decompress(tmpbuffer, compsize, tmpbuffer2);
+							intsize = intpack_decompress(snapshot_incomming_data, complete_size, tmpbuffer2);
 
 							if(intsize < 0) /* failure during decompression, bail */
 								return;
@@ -1096,7 +1106,7 @@ int client_mapdownload_totalsize() { return mapdownload_totalsize; }
 
 static void client_pump_network()
 {
-	NETPACKET packet;
+	NETCHUNK packet;
 
 	netclient_update(net);
 
@@ -1513,7 +1523,9 @@ int main(int argc, char **argv)
 	dbg_msg("client", "starting...");
 	engine_init("Teeworlds");
 
-/*	test_parser();
+/*
+	return huffman_test();
+	test_parser();
 	return 0;*/
 	
 	/* register all console commands */
diff --git a/src/engine/client/ec_srvbrowse.c b/src/engine/client/ec_srvbrowse.c
index 3ce8166c..d6f045f1 100644
--- a/src/engine/client/ec_srvbrowse.c
+++ b/src/engine/client/ec_srvbrowse.c
@@ -358,7 +358,7 @@ void client_serverbrowse_refresh(int lan)
 	
 	if(serverlist_lan)
 	{
-		NETPACKET packet;
+		NETCHUNK packet;
 		packet.client_id = -1;
 		mem_zero(&packet, sizeof(packet));
 		packet.address.ip[0] = 255;
@@ -366,7 +366,7 @@ void client_serverbrowse_refresh(int lan)
 		packet.address.ip[2] = 255;
 		packet.address.ip[3] = 255;
 		packet.address.port = 8303;
-		packet.flags = PACKETFLAG_CONNLESS;
+		packet.flags = NETSENDFLAG_CONNLESS;
 		packet.data_size = sizeof(SERVERBROWSE_GETINFO_LAN);
 		packet.data = SERVERBROWSE_GETINFO_LAN;
 		broadcast_time = time_get();
@@ -378,14 +378,14 @@ void client_serverbrowse_refresh(int lan)
 	else
 	{
 		NETADDR4 addr;
-		NETPACKET p;
+		NETCHUNK p;
 		int i;
 		
 		/*net_host_lookup(config.masterserver, MASTERSERVER_PORT, &master_server);*/
 
 		mem_zero(&p, sizeof(p));
 		p.client_id = -1;
-		p.flags = PACKETFLAG_CONNLESS;
+		p.flags = NETSENDFLAG_CONNLESS;
 		p.data_size = sizeof(SERVERBROWSE_GETLIST);
 		p.data = SERVERBROWSE_GETLIST;
 		
@@ -406,7 +406,7 @@ void client_serverbrowse_refresh(int lan)
 
 static void client_serverbrowse_request(SERVERENTRY *entry)
 {
-	NETPACKET p;
+	NETCHUNK p;
 
 	if(config.debug)
 	{
@@ -417,7 +417,7 @@ static void client_serverbrowse_request(SERVERENTRY *entry)
 	
 	p.client_id = -1;
 	p.address = entry->addr;
-	p.flags = PACKETFLAG_CONNLESS;
+	p.flags = NETSENDFLAG_CONNLESS;
 	p.data_size = sizeof(SERVERBROWSE_GETINFO);
 	p.data = SERVERBROWSE_GETINFO;
 	netclient_send(net, &p);
diff --git a/src/engine/e_engine.c b/src/engine/e_engine.c
index d356fd4b..6e1efa95 100644
--- a/src/engine/e_engine.c
+++ b/src/engine/e_engine.c
@@ -9,6 +9,7 @@
 #include <engine/e_config.h>
 #include <engine/e_console.h>
 #include <engine/e_engine.h>
+#include <engine/e_network.h>
 #include "e_linereader.h"
 
 static void con_dbg_dumpmem(void *result, void *user_data)
@@ -42,6 +43,7 @@ void engine_init(const char *appname)
 
 	/* init the network */
 	net_init();
+	netcommon_init();
 	
 	/* create storage location */
 	{
diff --git a/src/engine/e_huffman.c b/src/engine/e_huffman.c
new file mode 100644
index 00000000..f2e58f0c
--- /dev/null
+++ b/src/engine/e_huffman.c
@@ -0,0 +1,380 @@
+#include <stdlib.h>
+#include <string.h>
+#include <engine/e_system.h>
+#include <engine/e_huffman.h>
+
+void huffman_init(HUFFSTATE *huff)
+{
+	mem_zero(huff, sizeof(huff));
+	huff->nodes[0].frequency = 1;
+	huff->nodes[0].symbol_size = -1;
+	huff->num_symbols++;
+	huff->num_nodes++;
+}
+
+void huffman_add_symbol(HUFFSTATE *huff, int frequency, int size, unsigned char *symbol)
+{
+	huff->nodes[huff->num_nodes].frequency = frequency;
+	huff->nodes[huff->num_nodes].symbol_size = size;
+	mem_copy(huff->nodes[huff->num_nodes].symbol, symbol, size);
+	huff->num_nodes++;
+	huff->num_symbols++;
+}
+
+
+static int sort_func(const void *a, const void *b)
+{
+	if((*(HUFFNODE **)a)->frequency > (*(HUFFNODE **)b)->frequency)
+		return -1;
+	if((*(HUFFNODE **)a)->frequency < (*(HUFFNODE **)b)->frequency)
+		return 1;
+	return 0;
+}
+
+void huffman_setbits_r(HUFFNODE *node, int bits, int depth)
+{
+	if(node->one)
+		huffman_setbits_r(node->one, (bits<<1)|1, depth+1);
+	if(node->zero)
+		huffman_setbits_r(node->zero, (bits<<1), depth+1);
+		
+	if(node->symbol_size)
+	{
+		node->bits = bits;
+		node->num_bits = depth;
+	}
+}
+	
+void huffman_checktree_r(HUFFNODE *node, int bits, int depth)
+{
+	if(node->one)
+		huffman_checktree_r(node->one, (bits<<1)|1, depth+1);
+	if(node->zero)
+		huffman_checktree_r(node->zero, (bits<<1), depth+1);
+		
+	if(node->symbol_size)
+	{
+		/*dbg_msg("", "%p %p %d %d %d", node->one, node->zero, node->symbol[0], node->bits, node->num_bits);*/
+		
+		if(node->bits != bits || node->num_bits != depth)
+		{
+			dbg_msg("", "crap!   %d %d=%d   %d!=%d", node->bits>>1, node->bits, bits, node->num_bits , depth);
+			/*dbg_msg("", "%p %p %d", node->one, node->zero, node->symbol[0]);*/
+		}
+	}
+}
+
+void huffman_construct_tree(HUFFSTATE *huff)
+{
+	HUFFNODE *nodes_left[MAX_NODES];
+	int num_nodes_left = huff->num_nodes;
+	int i, k;
+
+	for(i = 0; i < num_nodes_left; i++)
+		nodes_left[i] = &huff->nodes[i];
+	
+	/* construct the table */
+	while(num_nodes_left > 1)
+	{
+		qsort(nodes_left, num_nodes_left, sizeof(HUFFNODE *), sort_func);
+		
+		huff->nodes[huff->num_nodes].symbol_size = 0;
+		huff->nodes[huff->num_nodes].frequency = nodes_left[num_nodes_left-1]->frequency + nodes_left[num_nodes_left-2]->frequency;
+		huff->nodes[huff->num_nodes].zero = nodes_left[num_nodes_left-1];
+		huff->nodes[huff->num_nodes].one = nodes_left[num_nodes_left-2];
+		nodes_left[num_nodes_left-1]->parent = &huff->nodes[huff->num_nodes];
+		nodes_left[num_nodes_left-2]->parent = &huff->nodes[huff->num_nodes];
+		nodes_left[num_nodes_left-2] = &huff->nodes[huff->num_nodes];
+		huff->num_nodes++;
+		num_nodes_left--;
+	}
+
+	dbg_msg("", "%d", huff->num_nodes);
+	for(i = 0; i < huff->num_nodes; i++)
+	{
+		if(huff->nodes[i].symbol_size && (huff->nodes[i].one || huff->nodes[i].zero))
+			dbg_msg("", "tree strangeness");
+			
+		if(!huff->nodes[i].parent)
+		{
+			dbg_msg("", "root %p %p", huff->nodes[i].one, huff->nodes[i].zero);
+			huff->start_node = &huff->nodes[i];
+		}
+	}
+	
+	huffman_setbits_r(huff->start_node, 0, 0);
+	
+	/*
+	for(i = 0; i < huff->num_symbols; i++)
+	{
+		unsigned bits = 0;
+		int num_bits = 0;
+		HUFFNODE *n = &huff->nodes[i];
+		HUFFNODE *p = n;
+		HUFFNODE *c = n->parent;
+		
+		while(c)
+		{
+			num_bits++;
+			if(c->one == p)
+				bits |= 1;
+			bits <<= 1;
+			p = c;
+			c = c->parent;
+		}
+
+		n->bits = bits;
+		n->num_bits = num_bits;
+	}*/
+	
+	huffman_checktree_r(huff->start_node, 0, 0);
+
+	for(i = 0; i < huff->num_symbols; i++)
+	{
+		for(k = 0; k < huff->num_symbols; k++)
+		{
+			if(k == i)
+				continue;
+			if(huff->nodes[i].num_bits == huff->nodes[k].num_bits && huff->nodes[i].bits == huff->nodes[k].bits)
+				dbg_msg("", "tree error %d %d %d", i, k, huff->nodes[i].num_bits);
+		}
+	}
+
+}
+
+typedef struct
+{
+	unsigned char *data;
+	unsigned char current_bits;
+	int num;
+} HUFFBITIO;
+
+int debug_count = 0;
+
+static void bitio_init(HUFFBITIO *bitio, unsigned char *data)
+{
+	bitio->data = data;
+	bitio->num = 0;
+	bitio->current_bits = 0;
+}
+
+static void bitio_flush(HUFFBITIO *bitio)
+{
+	*bitio->data = bitio->current_bits;
+	bitio->data++;
+	bitio->num = 0;
+	bitio->current_bits = 0;
+}
+
+static void bitio_write(HUFFBITIO *bitio, int bit)
+{
+	bitio->current_bits = (bitio->current_bits<<1)|bit;
+	bitio->num++;
+	if(bitio->num == 8)
+		bitio_flush(bitio);
+		
+	if(debug_count)
+	{
+		debug_count--;
+		dbg_msg("", "out %d", bit);
+	}
+}
+
+static int bitio_read(HUFFBITIO *bitio)
+{
+	int bit;
+	
+	if(!bitio->num)
+	{
+		bitio->current_bits = *bitio->data;
+		bitio->data++;
+		bitio->num = 8;
+	}
+	
+	bitio->num--;
+	bit = (bitio->current_bits>>bitio->num)&1;
+
+	if(debug_count)
+	{
+		debug_count--;
+		dbg_msg("", "in %d", bit);
+	}
+	return bit;
+}
+
+int huffman_compress(HUFFSTATE *huff, const void *input, int input_size, void *output, int output_size)
+{
+	const unsigned char *src = (const unsigned char *)input;
+	int ret = 0;
+	int quit = 0;
+	HUFFBITIO io;
+	
+	bitio_init(&io, (unsigned char *)output);
+	
+	while(!quit)
+	{
+		int i;
+		int best_match = -1;
+		int best_match_size = 0;
+		
+		if(input_size)
+		{
+			for(i = 0; i < huff->num_symbols; i++)
+			{
+				if(huff->nodes[i].symbol_size <= input_size && huff->nodes[i].symbol_size > best_match_size)
+				{
+					if(memcmp(src, huff->nodes[i].symbol, huff->nodes[i].symbol_size) == 0)
+					{
+						best_match = i;
+						best_match_size = huff->nodes[i].symbol_size;
+					}
+				}
+			}
+		}
+		else
+		{
+			best_match = 0;
+			best_match_size = 0;
+			quit = 1;
+		}
+		
+		
+		if(best_match == -1)
+		{
+			dbg_msg("huffman", "couldn't find symbol! %d left", input_size);
+			return -1;
+		}
+		
+		i = huff->nodes[best_match].num_bits;
+		for(i = huff->nodes[best_match].num_bits-1; i >= 0; i--)
+			bitio_write(&io, (huff->nodes[best_match].bits>>i)&1);
+
+		if(debug_count)		
+			dbg_msg("", "--");
+		
+		ret += huff->nodes[best_match].num_bits;
+		input_size -= best_match_size;
+		src += best_match_size;
+	}
+	
+	bitio_flush(&io);
+	return ret;
+}
+
+int huffman_decompress(HUFFSTATE *huff, const void *input, int input_size, void *output, int output_size)
+{
+	unsigned char *dst = (unsigned char *)output;
+	HUFFBITIO io;
+	int size = 0;
+	
+	bitio_init(&io, (unsigned char *)input);
+	
+	while(size < 1401)
+	{
+		HUFFNODE *node = huff->start_node;
+		int i;
+		
+		while(node)
+		{
+			if(node->symbol_size)
+				break;
+				
+			if(bitio_read(&io))
+				node = node->one;
+			else
+				node = node->zero;
+		}
+
+		if(debug_count)		
+			dbg_msg("", "-- %d %d", node->bits, node->num_bits);
+		
+		/* check for eof */
+		if(node == &huff->nodes[0])
+			break;
+		
+		for(i = 0; i < node->symbol_size; i++)
+		{
+			*dst++ = node->symbol[i];
+			size++;
+		}
+	}
+	
+	return size;
+}
+
+unsigned char test_data[1024*64];
+int test_data_size;
+
+unsigned char compressed_data[1024*64];
+int compressed_data_size;
+
+unsigned char output_data[1024*64];
+int output_data_size;
+
+HUFFSTATE state;
+
+int huffman_test()
+{
+	huffman_init(&state);
+	
+	dbg_msg("", "test test");
+	
+	/* bitio testing */
+	{
+		char bits[] = {1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1,1,1,1,1,1,1,0};
+		unsigned char buf[64];
+		int i;
+		HUFFBITIO io;
+		
+		bitio_init(&io, buf);
+		for(i = 0; i < sizeof(bits); i++)
+			bitio_write(&io, bits[i]);
+			
+		bitio_flush(&io);
+		bitio_init(&io, buf);
+		for(i = 0; i < sizeof(bits); i++)
+		{
+			if(bitio_read(&io) != bits[i])
+				dbg_msg("", "bitio failed at %d", i);
+		}
+	}
+	
+	/* read test data */
+	{
+		IOHANDLE io = io_open("license.txt", IOFLAG_READ);
+		test_data_size = io_read(io, test_data, sizeof(test_data));
+		io_close(io);
+	}
+	
+	/* add symbols */
+	{
+		int counts[256] = {0};
+		int i;
+		for(i = 0; i < test_data_size; i++)
+			counts[test_data[i]]++;
+		
+		for(i = 0; i < 256; i++)
+		{
+			unsigned char symbol = (unsigned char )i;
+			if(counts[i])
+				huffman_add_symbol(&state, counts[i], 1, &symbol);
+		}
+	}
+	
+	huffman_construct_tree(&state);
+	/*debug_count = 20;*/
+	compressed_data_size = huffman_compress(&state, test_data, test_data_size, compressed_data, sizeof(compressed_data));
+	/*debug_count = 20;*/
+	output_data_size = huffman_decompress(&state, compressed_data, compressed_data_size, output_data, sizeof(output_data));
+	
+	dbg_msg("huffman", "%d -> %d -> %d", test_data_size, compressed_data_size/8, output_data_size);
+	
+	/* write test data */
+	{
+		IOHANDLE io = io_open("out.txt", IOFLAG_WRITE);
+		io_write(io, output_data, output_data_size);
+		io_close(io);
+	}	
+	
+	return 0;
+}
diff --git a/src/engine/e_huffman.h b/src/engine/e_huffman.h
new file mode 100644
index 00000000..84c71e60
--- /dev/null
+++ b/src/engine/e_huffman.h
@@ -0,0 +1,43 @@
+
+enum
+{
+	MAX_SYMBOL_SIZE=8,
+	MAX_NODES=1024*8
+};
+
+typedef struct
+{
+	int i;
+} HUFFSYMBOL;
+
+typedef struct HUFFNODE_t
+{
+	int frequency;
+	
+	int symbol_size;
+	unsigned char symbol[MAX_SYMBOL_SIZE];
+	
+	int num_bits;
+	unsigned bits;
+
+	struct HUFFNODE_t *parent;
+	struct HUFFNODE_t *zero;
+	struct HUFFNODE_t *one;
+} HUFFNODE;
+
+typedef struct
+{
+	HUFFNODE nodes[MAX_NODES];
+	HUFFNODE *start_node;
+	int num_symbols;
+	int num_nodes;
+} HUFFSTATE;
+
+
+void huffman_add_symbol(HUFFSTATE *huff, int frequency, int size, unsigned char *symbol);
+void huffman_init(HUFFSTATE *huff);
+void huffman_construct_tree(HUFFSTATE *huff);
+int huffman_compress(HUFFSTATE *huff, const void *input, int input_size, void *output, int output_size);
+int huffman_decompress(HUFFSTATE *huff, const void *input, int input_size, void *output, int output_size);
+
+int huffman_test();
diff --git a/src/engine/e_if_other.h b/src/engine/e_if_other.h
index 66e2d3a8..e02a7fd1 100644
--- a/src/engine/e_if_other.h
+++ b/src/engine/e_if_other.h
@@ -284,7 +284,8 @@ void snap_input(void *data, int size);
 /* message packing */
 enum
 {
-	MSGFLAG_VITAL=1
+	MSGFLAG_VITAL=1,
+	MSGFLAG_FLUSH=2
 };
 
 /* message sending */
diff --git a/src/engine/e_network.c b/src/engine/e_network.c
index 682ecf2f..b523e73b 100644
--- a/src/engine/e_network.c
+++ b/src/engine/e_network.c
@@ -5,69 +5,92 @@
 #include "e_system.h"
 #include "e_config.h"
 #include "e_network.h"
+#include "e_huffman.h"
 
 /*
-	header (6 bytes)
-		unsigned char flags;		1
-		unsigned char seq_ack[3];	4
-		unsigned char token[2];		6
+
+CURRENT:
+	packet header: 4 bytes
+		unsigned char flags;
+		unsigned char ack[2];
+		unsigned char num_chunks;
+		
+	chunk header: 3-5 bytes
+		unsigned char flags;
+		unsigned char size[2];
+		(unsigned char seq[2]);
+
+FINAL:
+
+	packet header: 3 bytes
+		unsigned char flags_ack[2]; // 6bit flags, 10bit ack
+		unsigned char num_chunks;
+		
+	chunk header: 2-3 bytes
+		unsigned char flags_seq_ack[2]; // 2bit flag, 4bit seq, 10bit size
+		(unsigned char seq;) // if vital flag is set
+		
 */
 
 enum
 {
-	NETWORK_VERSION = 1,
-	
-	NETWORK_HEADER_SIZE = 6,
-	NETWORK_MAX_PAYLOAD = 1024,
-	NETWORK_MAX_PACKET_SIZE = NETWORK_HEADER_SIZE+NETWORK_MAX_PAYLOAD,
-	NETWORK_MAX_CLIENTS = 16,
+	NET_VERSION = 2,
+
+	NET_MAX_CHUNKSIZE = 1024,
+	NET_MAX_PAYLOAD = NET_MAX_CHUNKSIZE+16,
+	NET_MAX_PACKETSIZE = NET_MAX_PAYLOAD+16,
+	NET_MAX_CHUNKHEADERSIZE = 5,
+	NET_PACKETHEADERSIZE = 4,
+	NET_MAX_CLIENTS = 16,
+	NET_MAX_SEQUENCE = 1<<16,
+
+	NET_CONNSTATE_OFFLINE=0,
+	NET_CONNSTATE_CONNECT=1,
+	NET_CONNSTATE_CONNECTACCEPTED=2,
+	NET_CONNSTATE_ONLINE=3,
+	NET_CONNSTATE_ERROR=4,
+
+	NET_PACKETFLAG_CONTROL=1,
+	NET_PACKETFLAG_RESEND=2,
+	NET_PACKETFLAG_CONNLESS=4,
+
+	NET_CHUNKFLAG_VITAL=1,
+	NET_CHUNKFLAG_RESEND=2,
 	
-	NETWORK_CONNSTATE_OFFLINE=0,
-	NETWORK_CONNSTATE_CONNECT=1,
-	NETWORK_CONNSTATE_CONNECTACCEPTED=2,
-	NETWORK_CONNSTATE_ONLINE=3,
-	NETWORK_CONNSTATE_ERROR=4,
+	NET_CTRLMSG_KEEPALIVE=0,
+	NET_CTRLMSG_CONNECT=1,
+	NET_CTRLMSG_CONNECTACCEPT=2,
+	NET_CTRLMSG_ACCEPT=3,
+	NET_CTRLMSG_CLOSE=4,
 	
-	NETWORK_PACKETFLAG_CONNECT=0x01,
-	NETWORK_PACKETFLAG_ACCEPT=0x02,
-	NETWORK_PACKETFLAG_CLOSE=0x04,
-	NETWORK_PACKETFLAG_VITAL=0x08,
-	NETWORK_PACKETFLAG_RESEND=0x10,
-	NETWORK_PACKETFLAG_CONNLESS=0x20,
-	
-	NETWORK_MAX_SEQACK=0x1000
+	NET_ENUM_TERMINATOR
 };
 
-static int current_token = 1;
+typedef struct
+{
+	int flags;
+	int ack;
+	int num_chunks;
+	int data_size;
+	unsigned char chunk_data[NET_MAX_PAYLOAD];
+} NETPACKETCONSTRUCT;
 
 typedef struct
 {
-	unsigned char ID[2];
-	unsigned char version;
-	unsigned char flags;
-	unsigned short seq;
-	unsigned short ack;
-	unsigned crc;
-	int token;
-	unsigned data_size;
-	int64 first_send_time;
-	unsigned char *data;
-} NETPACKETDATA;
+	int flags;
+	int size;
+	int sequence;
+} NETCHUNKHEADER;
 
-static void send_packet(NETSOCKET socket, NETADDR4 *addr, NETPACKETDATA *packet)
+typedef struct
 {
-	unsigned char buffer[NETWORK_MAX_PACKET_SIZE];
-	int send_size = NETWORK_HEADER_SIZE+packet->data_size;
+	int flags;
+	int data_size;
+	unsigned char *data;
 
-	buffer[0] = packet->flags;
-	buffer[1] = ((packet->seq>>4)&0xf0) | ((packet->ack>>8)&0x0f);
-	buffer[2] = packet->seq;
-	buffer[3] = packet->ack;
-	buffer[4] = packet->token>>8;
-	buffer[5] = packet->token&0xff;
-	mem_copy(buffer+NETWORK_HEADER_SIZE, packet->data, packet->data_size);
-	net_udp4_send(socket, addr, buffer, send_size);
-}
+	int sequence;
+	int64 first_send_time;
+} NETCHUNKDATA;
 
 typedef struct RINGBUFFER_ITEM_t
 {
@@ -155,6 +178,8 @@ typedef struct
 	
 	char error_string[256];
 	
+	NETPACKETCONSTRUCT construct;
+	
 	NETADDR4 peeraddr;
 	NETSOCKET socket;
 	NETSTATS stats;
@@ -165,26 +190,97 @@ typedef struct
 	NETCONNECTION conn;
 } NETSLOT;
 
+typedef struct
+{
+	NETADDR4 addr;
+	NETCONNECTION *conn;
+	int current_chunk;
+	int client_id;
+	int valid;
+	NETPACKETCONSTRUCT data;
+	unsigned char buffer[NET_MAX_PACKETSIZE];
+} NETRECVINFO;
+
 struct NETSERVER_t
 {
 	NETSOCKET socket;
-	NETSLOT slots[NETWORK_MAX_CLIENTS];
+	NETSLOT slots[NET_MAX_CLIENTS];
 	int max_clients;
 	NETFUNC_NEWCLIENT new_client;
 	NETFUNC_NEWCLIENT del_client;
 	void *user_ptr;
-	unsigned char recv_buffer[NETWORK_MAX_PACKET_SIZE];
+	
+	NETRECVINFO recv;
 } ;
 
 struct NETCLIENT_t
 {
 	NETADDR4 server_addr;
 	NETSOCKET socket;
-	unsigned char recv_buffer[NETWORK_MAX_PACKET_SIZE];
 	
+	NETRECVINFO recv;
 	NETCONNECTION conn;
 };
 
+static IOHANDLE datalog = 0;
+static HUFFSTATE huffmanstate;
+
+/* packs the data tight and sends it */
+static void send_packet(NETSOCKET socket, NETADDR4 *addr, NETPACKETCONSTRUCT *packet)
+{
+	unsigned char buffer[NET_MAX_PACKETSIZE];
+	buffer[0] = packet->flags;
+	buffer[1] = (packet->ack>>8)&0xff;
+	buffer[2] = packet->ack&0xff;
+	buffer[3] = packet->num_chunks;
+	if(datalog)
+	{
+		io_write(datalog, &packet->data_size, sizeof(packet->data_size));
+		io_write(datalog, &packet->chunk_data, packet->data_size);
+	}
+	
+	if(1)
+	{
+		int compressed_size = huffman_compress(&huffmanstate, packet->chunk_data, packet->data_size, &buffer[4], NET_MAX_PACKETSIZE-4);
+		net_udp4_send(socket, addr, buffer, 4+(compressed_size+7)/8);
+	}
+	else
+	{
+		mem_copy(&buffer[4], packet->chunk_data, packet->data_size);
+		net_udp4_send(socket, addr, buffer, 4+packet->data_size);
+	}
+}
+
+static unsigned char *unpack_chunk_header(unsigned char *data, NETCHUNKHEADER *header)
+{
+	int i = 0;
+	header->flags = data[i++];
+	header->size = data[i++]<<8;
+	header->size |= data[i++];
+	header->sequence = -1;
+	if(header->flags&NET_CHUNKFLAG_VITAL)
+	{
+		header->sequence = data[i++]<<8;
+		header->sequence |= data[i++];
+	}
+	return &data[i];
+}
+
+/* TODO: change the arguments of this function */
+static unsigned char *pack_chunk_header(unsigned char *data, int flags, int size, int sequence)
+{
+	int i = 0;
+	data[i++] = flags;
+	data[i++] = (size>>8)&0xff;
+	data[i++] = size&0xff;
+	if(flags&NET_CHUNKFLAG_VITAL)
+	{
+		data[i++] = (sequence>>8)&0xff;
+		data[i++] = sequence&0xff;
+	}
+	return &data[i];
+}
+
 static void conn_reset_stats(NETCONNECTION *conn)
 {
 	mem_zero(&conn->stats, sizeof(conn->stats));
@@ -196,13 +292,13 @@ static void conn_reset(NETCONNECTION *conn)
 	conn->ack = 0;
 	conn->remote_closed = 0;
 	
-	if(conn->state == NETWORK_CONNSTATE_ONLINE ||
-		conn->state == NETWORK_CONNSTATE_ERROR)
+	if(conn->state == NET_CONNSTATE_ONLINE ||
+		conn->state == NET_CONNSTATE_ERROR)
 	{
 		conn->disconnected++;
 	}
 		
-	conn->state = NETWORK_CONNSTATE_OFFLINE;
+	conn->state = NET_CONNSTATE_OFFLINE;
 	conn->last_send_time = 0;
 	conn->last_recv_time = 0;
 	conn->last_update_time = 0;
@@ -223,12 +319,6 @@ static void conn_set_error(NETCONNECTION *conn, const char *str)
 	str_copy(conn->error_string, str, sizeof(conn->error_string));
 }
 
-/*
-static int conn_state(NETCONNECTION *conn)
-{
-	return conn->state;
-}*/
-
 static void conn_init(NETCONNECTION *conn, NETSOCKET socket)
 {
 	conn_reset(conn);
@@ -240,88 +330,121 @@ static void conn_init(NETCONNECTION *conn, NETSOCKET socket)
 	mem_zero(conn->error_string, sizeof(conn->error_string));
 }
 
+
 static void conn_ack(NETCONNECTION *conn, int ack)
 {
 	while(1)
 	{
 		RINGBUFFER_ITEM *item = conn->buffer.first;
-		NETPACKETDATA *resend;
+		NETCHUNKDATA *resend;
 		if(!item)
 			break;
 			
-		resend = (NETPACKETDATA *)rb_item_data(item);
-		if(resend->seq <= ack || (ack < NETWORK_MAX_SEQACK/3 && resend->seq > NETWORK_MAX_SEQACK/2))
+		resend = (NETCHUNKDATA *)rb_item_data(item);
+		if(resend->sequence <= ack || (ack < NET_MAX_SEQUENCE/3 && resend->sequence > NET_MAX_SEQUENCE/2))
 			rb_pop_first(&conn->buffer);
 		else
 			break;
 	}
 }
 
-static void conn_send_raw(NETCONNECTION *conn, NETPACKETDATA *data)
+static void conn_want_resend(NETCONNECTION *conn)
 {
+	conn->construct.flags |= NET_PACKETFLAG_RESEND;
+}
+
+static int conn_flush(NETCONNECTION *conn)
+{
+	if(!conn->construct.num_chunks && !conn->construct.flags)
+		return 0;
+	
+	conn->construct.ack = conn->ack;
+	send_packet(conn->socket, &conn->peeraddr, &conn->construct);
 	conn->last_send_time = time_get();
-	conn->stats.send_packets++;
-	conn->stats.send_bytes += data->data_size + NETWORK_HEADER_SIZE;
-	send_packet(conn->socket, &conn->peeraddr, data);
+	
+	/* clear construct so we can start building a new package */
+	mem_zero(&conn->construct, sizeof(conn->construct));
+	return 1;
 }
 
-static void conn_resend(NETCONNECTION *conn)
+/*NETCHUNKDATA *data*/
+
+static void conn_queue_chunk(NETCONNECTION *conn, int flags, int data_size, const void *data)
 {
-	RINGBUFFER_ITEM *item = conn->buffer.first;
-	while(item)
+	unsigned char *chunk_data;
+	/* check if we have space for it, if not, flush the connection */
+	if(conn->construct.data_size + data_size + NET_MAX_CHUNKHEADERSIZE > sizeof(conn->construct.chunk_data))
+		conn_flush(conn);
+
+	if(flags&NET_CHUNKFLAG_VITAL && !(flags&NET_CHUNKFLAG_RESEND))
+		conn->seq = (conn->seq+1)%NET_MAX_SEQUENCE;
+
+	/* pack all the data */
+	chunk_data = &conn->construct.chunk_data[conn->construct.data_size];
+	chunk_data = pack_chunk_header(chunk_data, flags, data_size, conn->seq);
+	mem_copy(chunk_data, data, data_size);
+	chunk_data += data_size;
+
+	/* */
+	conn->construct.num_chunks++;
+	conn->construct.data_size = (int)(chunk_data-conn->construct.chunk_data);
+	
+	/* set packet flags aswell */
+	
+	if(flags&NET_CHUNKFLAG_VITAL && !(flags&NET_CHUNKFLAG_RESEND))
 	{
-		NETPACKETDATA *resend = (NETPACKETDATA *)rb_item_data(item);
-		conn->stats.resend_packets++;
-		conn->stats.resend_bytes += resend->data_size + NETWORK_HEADER_SIZE;
-		conn_send_raw(conn, resend);
-		item = item->next;
+		/* save packet if we need to resend */
+		NETCHUNKDATA *resend = (NETCHUNKDATA *)rb_alloc(&conn->buffer, sizeof(NETCHUNKDATA)+data_size);
+		resend->sequence = conn->seq;
+		resend->flags = flags;
+		resend->data_size = data_size;
+		resend->data = (unsigned char *)(resend+1);
+		resend->first_send_time = time_get();
+		mem_copy(resend->data, data, data_size);
 	}
 }
 
-static void conn_send(NETCONNECTION *conn, int flags, int data_size, const void *data)
+static void conn_send_control(NETCONNECTION *conn, int controlmsg, const void *extra, int extra_size)
 {
-	NETPACKETDATA p;
-
-	if(flags&NETWORK_PACKETFLAG_VITAL)
-		conn->seq = (conn->seq+1)%NETWORK_MAX_SEQACK;
-
-	p.ID[0] = 'T';
-	p.ID[1] = 'W';
-	p.version = NETWORK_VERSION;
-	p.flags = flags;
-	p.seq = conn->seq;
-	p.ack = conn->ack;
-	p.crc = 0;
-	p.token = conn->token;
-	p.data_size = data_size;
-	p.data = (unsigned char *)data;
-	p.first_send_time = time_get();
+	NETPACKETCONSTRUCT construct;
+	construct.flags = NET_PACKETFLAG_CONTROL;
+	construct.ack = conn->ack;
+	construct.num_chunks = 0;
+	construct.data_size = 1+extra_size;
+	construct.chunk_data[0] = controlmsg;
+	mem_copy(&construct.chunk_data[1], extra, extra_size);
+
+	/* send the control message */
+	send_packet(conn->socket, &conn->peeraddr, &construct);
+	conn->last_send_time = time_get();
+}
 
-	if(flags&NETWORK_PACKETFLAG_VITAL)
+static void conn_resend(NETCONNECTION *conn)
+{
+	int max = 10;
+	RINGBUFFER_ITEM *item = conn->buffer.first;
+	while(item)
 	{
-		/* save packet if we need to resend */
-		NETPACKETDATA *resend = (NETPACKETDATA *)rb_alloc(&conn->buffer, sizeof(NETPACKETDATA)+data_size);
-		*resend = p;
-		resend->data = (unsigned char *)(resend+1);
-		mem_copy(resend->data, p.data, p.data_size);
+		NETCHUNKDATA *resend = (NETCHUNKDATA *)rb_item_data(item);
+		conn_queue_chunk(conn, resend->flags|NET_CHUNKFLAG_RESEND, resend->data_size, resend->data);
+		item = item->next;
+		max--;
+		if(!max)
+			break;
 	}
-	
-	/* TODO: calc crc */
-	conn_send_raw(conn, &p);
 }
 
 static int conn_connect(NETCONNECTION *conn, NETADDR4 *addr)
 {
-	if(conn->state != NETWORK_CONNSTATE_OFFLINE)
+	if(conn->state != NET_CONNSTATE_OFFLINE)
 		return -1;
 	
 	/* init connection */
 	conn_reset(conn);
 	conn->peeraddr = *addr;
-	conn->token = current_token++;
 	mem_zero(conn->error_string, sizeof(conn->error_string));
-	conn->state = NETWORK_CONNSTATE_CONNECT;
-	conn_send(conn, NETWORK_PACKETFLAG_CONNECT, 0, 0);
+	conn->state = NET_CONNSTATE_CONNECT;
+	conn_send_control(conn, NET_CTRLMSG_CONNECT, 0, 0);
 	return 0;
 }
 
@@ -330,9 +453,9 @@ static void conn_disconnect(NETCONNECTION *conn, const char *reason)
 	if(conn->remote_closed == 0)
 	{
 		if(reason)
-			conn_send(conn, NETWORK_PACKETFLAG_CLOSE, strlen(reason)+1, reason);
+			conn_send_control(conn, NET_CTRLMSG_CLOSE, reason, strlen(reason)+1);
 		else
-			conn_send(conn, NETWORK_PACKETFLAG_CLOSE, 0, 0);
+			conn_send_control(conn, NET_CTRLMSG_CLOSE, 0, 0);
 
 		conn->error_string[0] = 0;
 		if(reason)
@@ -342,145 +465,109 @@ static void conn_disconnect(NETCONNECTION *conn, const char *reason)
 	conn_reset(conn);
 }
 
-static int conn_feed(NETCONNECTION *conn, NETPACKETDATA *p, NETADDR4 *addr)
+static int conn_feed(NETCONNECTION *conn, NETPACKETCONSTRUCT *packet, NETADDR4 *addr)
 {
 	int64 now = time_get();
 	conn->last_recv_time = now;
-	conn->stats.recv_packets++;
-	conn->stats.recv_bytes += p->data_size + NETWORK_HEADER_SIZE;
 	
-	if(p->flags&NETWORK_PACKETFLAG_CLOSE)
+	/* check if resend is requested */
+	if(packet->flags&NET_PACKETFLAG_RESEND)
+		conn_resend(conn);
+
+	/* */									
+	if(packet->flags&NET_PACKETFLAG_CONTROL)
 	{
-		conn->state = NETWORK_CONNSTATE_ERROR;
-		conn->remote_closed = 1;
+		int ctrlmsg = packet->chunk_data[0];
 		
-		if(p->data_size)
-		{
-			/* make sure to sanitize the error string form the other party*/
-			char str[128];
-			if(p->data_size < 128)
-				str_copy(str, (char *)p->data, p->data_size);
-			else
-				str_copy(str, (char *)p->data, 128);
-			str_sanitize_strong(str);
-			
-			/* set the error string */
-			conn_set_error(conn, str);
-		}
-		else
-			conn_set_error(conn, "no reason given");
-		if(config.debug)
-			dbg_msg("conn", "closed reason='%s'", conn_error(conn));
-		return 0;
-	}
-	
-	if(conn->state == NETWORK_CONNSTATE_OFFLINE)
-	{
-		if(p->flags == NETWORK_PACKETFLAG_CONNECT)
-		{
-			/* send response and init connection */
-			conn_reset(conn);
-			conn->state = NETWORK_CONNSTATE_ONLINE;
-			conn->connected++;
-			conn->peeraddr = *addr;
-			conn->token = p->token;
-			conn->last_send_time = now;
-			conn->last_recv_time = now;
-			conn->last_update_time = now;
-			conn_send(conn, NETWORK_PACKETFLAG_CONNECT|NETWORK_PACKETFLAG_ACCEPT, 0, 0);
-			if(config.debug)
-				dbg_msg("connection", "got connection, sending connect+accept");
-		}
-	}
-	else if(net_addr4_cmp(&conn->peeraddr, addr) == 0)
-	{
-		if(p->token != conn->token)
-			return 0;
+		dbg_msg("connection", "\tgot control message %d", ctrlmsg);
 
-		if(conn->state == NETWORK_CONNSTATE_ONLINE)
+		if(ctrlmsg == NET_CTRLMSG_CLOSE)
 		{
-			/* remove packages that are acked */
-			conn_ack(conn, p->ack);
+			conn->state = NET_CONNSTATE_ERROR;
+			conn->remote_closed = 1;
 			
-			/* check if resend is requested */
-			if(p->flags&NETWORK_PACKETFLAG_RESEND)
-				conn_resend(conn);
-				
-			if(p->flags&NETWORK_PACKETFLAG_VITAL)
+			if(packet->data_size)
 			{
-				if(p->seq == (conn->ack+1)%NETWORK_MAX_SEQACK)
-				{
-					/* in sequence */
-					conn->ack = (conn->ack+1)%NETWORK_MAX_SEQACK;
-				}
+				/* make sure to sanitize the error string form the other party*/
+				char str[128];
+				if(packet->data_size < 128)
+					str_copy(str, (char *)packet->chunk_data, packet->data_size);
 				else
-				{
-					/* out of sequence, request resend */
-					if(config.debug)
-						dbg_msg("conn", "asking for resend %d %d", p->seq, (conn->ack+1)%NETWORK_MAX_SEQACK);
-					conn_send(conn, NETWORK_PACKETFLAG_RESEND, 0, 0);
-					return 0;
-				}
+					str_copy(str, (char *)packet->chunk_data, 128);
+				str_sanitize_strong(str);
+				
+				/* set the error string */
+				conn_set_error(conn, str);
 			}
 			else
-			{
-				if(p->seq > conn->ack)
-					conn_send(conn, NETWORK_PACKETFLAG_RESEND, 0, 0);
-			}
-			
-			if(p->data_size == 0)
-				return 0;
+				conn_set_error(conn, "no reason given");
 				
-			return 1;
+			if(config.debug)
+				dbg_msg("conn", "closed reason='%s'", conn_error(conn));
+			return 0;			
 		}
-		else if(conn->state == NETWORK_CONNSTATE_CONNECT)
+		else
 		{
-			/* connection made */
-			if(p->flags == (NETWORK_PACKETFLAG_CONNECT|NETWORK_PACKETFLAG_ACCEPT))
+			if(conn->state == NET_CONNSTATE_OFFLINE)
+			{
+				if(ctrlmsg == NET_CTRLMSG_CONNECT)
+				{
+					/* send response and init connection */
+					conn_reset(conn);
+					conn->state = NET_CONNSTATE_ONLINE;
+					conn->connected++;
+					conn->peeraddr = *addr;
+					conn->last_send_time = now;
+					conn->last_recv_time = now;
+					conn->last_update_time = now;
+					conn_send_control(conn, NET_CTRLMSG_CONNECTACCEPT, 0, 0);
+					if(config.debug)
+						dbg_msg("connection", "got connection, sending connect+accept");			
+				}
+			}
+			else if(conn->state == NET_CONNSTATE_CONNECT)
 			{
-				conn_send(conn, NETWORK_PACKETFLAG_ACCEPT, 0, 0);
-				conn->state = NETWORK_CONNSTATE_ONLINE;
-				conn->connected++;
-				if(config.debug)
-					dbg_msg("connection", "got connect+accept, sending accept. connection online");
+				/* connection made */
+				if(ctrlmsg == NET_CTRLMSG_CONNECTACCEPT)
+				{
+					conn_send_control(conn, NET_CTRLMSG_ACCEPT, 0, 0);
+					conn->state = NET_CONNSTATE_ONLINE;
+					conn->connected++;
+					if(config.debug)
+						dbg_msg("connection", "got connect+accept, sending accept. connection online");
+				}
 			}
-		}
-		/*
-		else if(conn->state == NETWORK_CONNSTATE_CONNECTACCEPTED)
-		{
-			// connection made
-			if(p->flags == NETWORK_PACKETFLAG_ACCEPT)
+			else if(conn->state == NET_CONNSTATE_ONLINE)
 			{
-				conn->state = NETWORK_CONNSTATE_ONLINE;
-				dbg_msg("connection", "got accept. connection online");
+				/* connection made */
+				/*
+				if(ctrlmsg == NET_CTRLMSG_CONNECTACCEPT)
+				{
+					
+				}*/
 			}
-		}*/
-		else
-		{
-			/* strange packet, wrong state */
-			conn->state = NETWORK_CONNSTATE_ERROR;
-			conn_set_error(conn, "strange state and packet");
 		}
 	}
-	else
+	
+	if(conn->state == NET_CONNSTATE_ONLINE)
 	{
-		/* strange packet, not ment for me */
+		conn_ack(conn, packet->ack);
 	}
 	
-	return 0;
+	return 1;
 }
 
-
-
 static int conn_update(NETCONNECTION *conn)
 {
 	int64 now = time_get();
 
-	if(conn->state == NETWORK_CONNSTATE_OFFLINE || conn->state == NETWORK_CONNSTATE_ERROR)
+	if(conn->state == NET_CONNSTATE_OFFLINE || conn->state == NET_CONNSTATE_ERROR)
 		return 0;
 
 	/* watch out for major hitches */
 	{
+		/* TODO: fix this */
+		/*
 		int64 delta = now-conn->last_update_time;
 		if(conn->last_update_time && delta > time_freq()/2)
 		{
@@ -498,87 +585,66 @@ static int conn_update(NETCONNECTION *conn)
 		}
 
 		conn->last_update_time = now;
+		*/
 	}
 		
 	
 	/* check for timeout */
-	if(conn->state != NETWORK_CONNSTATE_OFFLINE &&
-		conn->state != NETWORK_CONNSTATE_CONNECT &&
+	if(conn->state != NET_CONNSTATE_OFFLINE &&
+		conn->state != NET_CONNSTATE_CONNECT &&
 		(now-conn->last_recv_time) > time_freq()*10)
 	{
-		conn->state = NETWORK_CONNSTATE_ERROR;
+		conn->state = NET_CONNSTATE_ERROR;
 		conn_set_error(conn, "timeout");
 	}
 	
 	/* check for large buffer errors */
 	if(conn->buffer.buffer_size > 1024*64)
 	{
-		conn->state = NETWORK_CONNSTATE_ERROR;
+		conn->state = NET_CONNSTATE_ERROR;
 		conn_set_error(conn, "too weak connection (out of buffer)");
 	}
 	
 	if(conn->buffer.first)
 	{
-		NETPACKETDATA *resend = (NETPACKETDATA *)(conn->buffer.first+1);
+		/* TODO: fix this */
+		NETCHUNKDATA *resend = (NETCHUNKDATA *)(conn->buffer.first+1);
 		if(now-resend->first_send_time > time_freq()*10)
 		{
-			conn->state = NETWORK_CONNSTATE_ERROR;
+			conn->state = NET_CONNSTATE_ERROR;
 			conn_set_error(conn, "too weak connection (not acked for 10 seconds)");
 		}
 	}
 	
-	/* send keep alives if nothing has happend for 250ms */
-	if(conn->state == NETWORK_CONNSTATE_ONLINE)
+	/* send keep alives if nothing has happend for 1000ms */
+	if(conn->state == NET_CONNSTATE_ONLINE)
 	{
-		if(time_get()-conn->last_send_time> time_freq()/4)
-			conn_send(conn, NETWORK_PACKETFLAG_VITAL, 0, 0);
+		if(time_get()-conn->last_send_time > time_freq()/2) /* flush connection after 250ms if needed */
+			if(conn_flush(conn))
+			{
+				dbg_msg("connection", "flushed connection due to timeout");
+			}
+			
+		if(time_get()-conn->last_send_time > time_freq())
+			conn_send_control(conn, NET_CTRLMSG_KEEPALIVE, 0, 0);
 	}
-	else if(conn->state == NETWORK_CONNSTATE_CONNECT)
+	else if(conn->state == NET_CONNSTATE_CONNECT)
 	{
 		if(time_get()-conn->last_send_time > time_freq()/2) /* send a new connect every 500ms */
-			conn_send(conn, NETWORK_PACKETFLAG_CONNECT, 0, 0);
+			conn_send_control(conn, NET_CTRLMSG_CONNECT, 0, 0);
+			/*conn_send(conn, NETWORK_PACKETFLAG_CONNECT, 0, 0);*/
 	}
-	else if(conn->state == NETWORK_CONNSTATE_CONNECTACCEPTED)
+	else if(conn->state == NET_CONNSTATE_CONNECTACCEPTED)
 	{
+
 		if(time_get()-conn->last_send_time > time_freq()/2) /* send a new connect/accept every 500ms */
-			conn_send(conn, NETWORK_PACKETFLAG_CONNECT|NETWORK_PACKETFLAG_ACCEPT, 0, 0);
+			conn_send_control(conn, NET_CTRLMSG_CONNECTACCEPT, 0, 0);
+			/*conn_send(conn, NETWORK_PACKETFLAG_CONNECT|NETWORK_PACKETFLAG_ACCEPT, 0, 0);*/
 	}
 	
 	return 0;
 }
 
-
-static int check_packet(unsigned char *buffer, int size, NETPACKETDATA *packet)
-{
-	/* check the size */
-	if(size < NETWORK_HEADER_SIZE || size > NETWORK_MAX_PACKET_SIZE)
-		return -1;
-	
-	/* read the packet */
-	packet->ID[0] = 'T';
-	packet->ID[1] = 'W';
-	packet->version = NETWORK_VERSION;
-	packet->flags = buffer[0];
-	packet->seq = ((buffer[1]&0xf0)<<4)|buffer[2];
-	packet->ack = ((buffer[1]&0x0f)<<8)|buffer[3];
-	packet->crc = 0;
-	packet->token = (buffer[4]<<8)|buffer[5];
-	packet->data_size = size - NETWORK_HEADER_SIZE;
-	packet->data = buffer+NETWORK_HEADER_SIZE;
-	
-	/* check the packet */
-	if(packet->ID[0] != 'T' || packet->ID[1] != 'W')
-		return 1;
-	
-	if(packet->version != NETWORK_VERSION)
-		return 1;
-
-	/* TODO: perform crc check */
-	
-	/* return success */
-	return 0;
-}
-
 NETSERVER *netserver_open(NETADDR4 bindaddr, int max_clients, int flags)
 {
 	int i;
@@ -591,12 +657,12 @@ NETSERVER *netserver_open(NETADDR4 bindaddr, int max_clients, int flags)
 	mem_zero(server, sizeof(NETSERVER));
 	server->socket = socket;
 	server->max_clients = max_clients;
-	if(server->max_clients > NETWORK_MAX_CLIENTS)
-		server->max_clients = NETWORK_MAX_CLIENTS;
+	if(server->max_clients > NET_MAX_CLIENTS)
+		server->max_clients = NET_MAX_CLIENTS;
 	if(server->max_clients < 1)
 		server->max_clients = 1;
 	
-	for(i = 0; i < NETWORK_MAX_CLIENTS; i++)
+	for(i = 0; i < NET_MAX_CLIENTS; i++)
 		conn_init(&server->slots[i].conn, server->socket);
 
 	return server;
@@ -639,163 +705,209 @@ int netserver_update(NETSERVER *s)
 	for(i = 0; i < s->max_clients; i++)
 	{
 		conn_update(&s->slots[i].conn);
-		if(s->slots[i].conn.state == NETWORK_CONNSTATE_ERROR)
+		if(s->slots[i].conn.state == NET_CONNSTATE_ERROR)
 			netserver_drop(s, i, conn_error(&s->slots[i].conn));
 	}
 	return 0;
 }
 
-int netserver_recv(NETSERVER *s, NETPACKET *packet)
+/* TODO: rename this function */
+static int unpack_packet(unsigned char *buffer, int size, NETPACKETCONSTRUCT *packet)
 {
-	NETPACKETDATA data;
-	int i, r, bytes, found;
-	NETADDR4 addr;
+	/* check the size */
+	if(size < NET_PACKETHEADERSIZE || size > NET_MAX_PACKETSIZE)
+		return -1;
+	
+	/* read the packet */
+	packet->flags = buffer[0];
+	packet->ack = (buffer[1]<<8) | buffer[2];
+	packet->num_chunks = buffer[3];
+	packet->data_size = size - NET_PACKETHEADERSIZE;
+	
+	if(1)
+		huffman_decompress(&huffmanstate, &buffer[4], packet->data_size, packet->chunk_data, sizeof(packet->chunk_data));
+	else
+		mem_copy(packet->chunk_data, &buffer[4], packet->data_size);
+	
+	/* return success */
+	return 0;
+}
+
+static void recvinfo_clear(NETRECVINFO *info)
+{
+	info->valid = 0;
+}
+
+static void recvinfo_start(NETRECVINFO *info, NETADDR4 *addr, NETCONNECTION *conn, int cid)
+{
+	info->addr = *addr;
+	info->conn = conn;
+	info->client_id = cid;
+	info->current_chunk = 0;
+	info->valid = 1;
+	if(info->data.num_chunks > 1)
+		dbg_msg("connection", "%d chunks", info->data.num_chunks);
+}
+
+/* TODO: rename this function */
+static int recvinfo_fetch_chunk(NETRECVINFO *info, NETCHUNK *chunk)
+{
+	NETCHUNKHEADER header;
+	unsigned char *data = info->data.chunk_data;
+	int i;
 	
 	while(1)
 	{
-		bytes = net_udp4_recv(s->socket, &addr, s->recv_buffer, NETWORK_MAX_PACKET_SIZE);
+		/* check for old data to unpack */
+		if(!info->valid || info->current_chunk >= info->data.num_chunks)
+		{
+			recvinfo_clear(info);
+			return 0;
+		}
+		
+		/* TODO: add checking here so we don't read too far */
+		for(i = 0; i < info->current_chunk; i++)
+		{
+			data = unpack_chunk_header(data, &header);
+			data += header.size;
+		}
+		
+		/* unpack the header */	
+		data = unpack_chunk_header(data, &header);
+		info->current_chunk++;
+		
+		/* handle sequence stuff */
+		if(info->conn && (header.flags&NET_CHUNKFLAG_VITAL))
+		{
+			if(header.sequence == (info->conn->ack+1)%NET_MAX_SEQUENCE)
+			{
+				/* in sequence */
+				info->conn->ack = (info->conn->ack+1)%NET_MAX_SEQUENCE;
+			}
+			else
+			{
+				/* out of sequence, request resend */
+				dbg_msg("conn", "asking for resend %d %d", header.sequence, (info->conn->ack+1)%NET_MAX_SEQUENCE);
+				conn_want_resend(info->conn);
+				continue; /* take the next chunk in the packet */
+			}
+		}
+		
+		/* fill in the info */
+		chunk->client_id = info->client_id;
+		chunk->address = info->addr;
+		chunk->flags = 0;
+		chunk->data_size = header.size;
+		chunk->data = data;
+		return 1;
+	}
+}
+
+int netserver_recv(NETSERVER *s, NETCHUNK *chunk)
+{
+	while(1)
+	{
+		NETADDR4 addr;
+		int i, bytes, found;
+			
+		/* check for a chunk */
+		if(recvinfo_fetch_chunk(&s->recv, chunk))
+			return 1;
+		
+		/* TODO: empty the recvinfo */
+		bytes = net_udp4_recv(s->socket, &addr, s->recv.buffer, NET_MAX_PACKETSIZE);
 
 		/* no more packets for now */
 		if(bytes <= 0)
 			break;
 		
-		r = check_packet(s->recv_buffer, bytes, &data);
-		if(r == 0)
+		if(unpack_packet(s->recv.buffer, bytes, &s->recv.data) == 0)
 		{
-			if(data.flags&NETWORK_PACKETFLAG_CONNLESS)
+			/* TODO: handle connection less packets here */
+			/* TODO: check size here */
+			if(s->recv.data.flags&NET_PACKETFLAG_CONTROL && s->recv.data.chunk_data[0] == NET_CTRLMSG_CONNECT)
 			{
-				/* connection less packets */
-				packet->client_id = -1;
-				packet->address = addr;
-				packet->flags = PACKETFLAG_CONNLESS;
-				packet->data_size = data.data_size;
-				packet->data = data.data;
-				return 1;		
-			}
-			else
-			{
-				/* ok packet, process it */
-				if(data.flags == NETWORK_PACKETFLAG_CONNECT)
+				found = 0;
+				
+				/* check if we already got this client */
+				for(i = 0; i < s->max_clients; i++)
+				{
+					if(s->slots[i].conn.state != NET_CONNSTATE_OFFLINE &&
+						net_addr4_cmp(&s->slots[i].conn.peeraddr, &addr) == 0)
+					{
+						found = 1; /* silent ignore.. we got this client already */
+						break;
+					}
+				}
+				
+				/* client that wants to connect */
+				if(!found)
 				{
 					found = 0;
 					
-					/* check if we already got this client */
 					for(i = 0; i < s->max_clients; i++)
 					{
-						if(s->slots[i].conn.state != NETWORK_CONNSTATE_OFFLINE &&
-							net_addr4_cmp(&s->slots[i].conn.peeraddr, &addr) == 0)
+						if(s->slots[i].conn.state == NET_CONNSTATE_OFFLINE)
 						{
-							found = 1; /* silent ignore.. we got this client already */
+							found = 1;
+							conn_feed(&s->slots[i].conn, &s->recv.data, &addr);
+							if(s->new_client)
+								s->new_client(i, s->user_ptr);
 							break;
 						}
 					}
 					
-					/* client that wants to connect */
 					if(!found)
 					{
-						for(i = 0; i < s->max_clients; i++)
-						{
-							if(s->slots[i].conn.state == NETWORK_CONNSTATE_OFFLINE)
-							{
-								found = 1;
-								conn_feed(&s->slots[i].conn, &data, &addr);
-								if(s->new_client)
-									s->new_client(i, s->user_ptr);
-								break;
-							}
-						}
-					}
-					
-					if(!found)
-					{
-						/* send connectionless packet */
-						const char errstring[] = "server full";
-						NETPACKETDATA p;
-						p.ID[0] = 'T';
-						p.ID[1] = 'W';
-						p.version = NETWORK_VERSION;
-						p.flags = NETWORK_PACKETFLAG_CLOSE;
-						p.seq = 0;
-						p.ack = 0;
-						p.crc = 0;
-						p.token = data.token;
-						p.data_size = sizeof(errstring);
-						p.data = (unsigned char *)errstring;
-						send_packet(s->socket, &addr, &p);
+						/* TODO: send server full emssage */
 					}
 				}
-				else
+			}
+			else
+			{
+				/* normal packet, find matching slot */
+				for(i = 0; i < s->max_clients; i++)
 				{
-					/* find matching slot */
-					for(i = 0; i < s->max_clients; i++)
+					if(net_addr4_cmp(&s->slots[i].conn.peeraddr, &addr) == 0)
 					{
-						/* must be in some sort of online state */
-						if(s->slots[i].conn.state == NETWORK_CONNSTATE_OFFLINE)
-							continue;
-							
-						if(net_addr4_cmp(&s->slots[i].conn.peeraddr, &addr) == 0)
+						if(conn_feed(&s->slots[i].conn, &s->recv.data, &addr))
 						{
-							if(conn_feed(&s->slots[i].conn, &data, &addr))
-							{
-								if(data.data_size)
-								{
-									packet->client_id = i;	
-									packet->address = addr;
-									packet->flags = 0;
-									packet->data_size = data.data_size;
-									packet->data = data.data;
-									return 1;
-								}
-							}
+							if(s->recv.data.data_size)
+								recvinfo_start(&s->recv, &addr, &s->slots[i].conn, i);
 						}
 					}
 				}
 			}
 		}
-		else
-		{
-			/* errornous packet, drop it */
-			/* dbg_msg("server", "crazy packet"); */
-		}
-		
-		/* read header */
-		/* do checksum */
-	}	
-	
+	}
 	return 0;
 }
 
-int netserver_send(NETSERVER *s, NETPACKET *packet)
+int netserver_send(NETSERVER *s, NETCHUNK *chunk)
 {
-	if(packet->data_size >= NETWORK_MAX_PAYLOAD)
+	if(chunk->data_size >= NET_MAX_PAYLOAD)
 	{
-		dbg_msg("netserver", "packet payload too big. %d. dropping packet", packet->data_size);
+		dbg_msg("netserver", "packet payload too big. %d. dropping packet", chunk->data_size);
 		return -1;
 	}
 	
-	if(packet->flags&PACKETFLAG_CONNLESS)
+	if(chunk->flags&NETSENDFLAG_CONNLESS)
 	{
-		/* send connectionless packet */
-		NETPACKETDATA p;
-		p.ID[0] = 'T';
-		p.ID[1] = 'W';
-		p.version = NETWORK_VERSION;
-		p.flags = NETWORK_PACKETFLAG_CONNLESS;
-		p.seq = 0;
-		p.ack = 0;
-		p.crc = 0;
-		p.data_size = packet->data_size;
-		p.data = (unsigned char *)packet->data;
-		send_packet(s->socket, &packet->address, &p);
+		/* TODO: handle connectionless */		
 	}
 	else
 	{
-		int flags  = 0;
-		dbg_assert(packet->client_id >= 0, "errornous client id");
-		dbg_assert(packet->client_id < s->max_clients, "errornous client id");
-		if(packet->flags&PACKETFLAG_VITAL)
-			flags |= NETWORK_PACKETFLAG_VITAL;
-		conn_send(&s->slots[packet->client_id].conn, flags, packet->data_size, packet->data);
+		int f = 0;
+		dbg_assert(chunk->client_id >= 0, "errornous client id");
+		dbg_assert(chunk->client_id < s->max_clients, "errornous client id");
+		
+		if(chunk->flags&NETSENDFLAG_VITAL)
+			f = NET_CHUNKFLAG_VITAL;
+		
+		conn_queue_chunk(&s->slots[chunk->client_id].conn, f, chunk->data_size, chunk->data);
+
+		if(chunk->flags&NETSENDFLAG_FLUSH)
+			conn_flush(&s->slots[chunk->client_id].conn);
 	}
 	return 0;
 }
@@ -810,7 +922,7 @@ void netserver_stats(NETSERVER *s, NETSTATS *stats)
 	
 	for(c = 0; c < s->max_clients; c++)
 	{
-		if(s->slots[c].conn.state != NETWORK_CONNSTATE_OFFLINE)
+		if(s->slots[c].conn.state != NET_CONNSTATE_OFFLINE)
 		{
 			int *sstats = (int *)(&(s->slots[c].conn.stats));
 			for(i = 0; i < num_stats; i++)
@@ -848,7 +960,7 @@ int netclient_close(NETCLIENT *c)
 int netclient_update(NETCLIENT *c)
 {
 	conn_update(&c->conn);
-	if(c->conn.state == NETWORK_CONNSTATE_ERROR)
+	if(c->conn.state == NET_CONNSTATE_ERROR)
 		netclient_disconnect(c, conn_error(&c->conn));
 	return 0;
 }
@@ -866,96 +978,68 @@ int netclient_connect(NETCLIENT *c, NETADDR4 *addr)
 	return 0;
 }
 
-int netclient_recv(NETCLIENT *c, NETPACKET *packet)
+int netclient_recv(NETCLIENT *c, NETCHUNK *chunk)
 {
 	while(1)
 	{
 		NETADDR4 addr;
-		NETPACKETDATA data;
-		int r;
-		int bytes = net_udp4_recv(c->socket, &addr, c->recv_buffer, NETWORK_MAX_PACKET_SIZE);
+		int bytes;
+			
+		/* check for a chunk */
+		if(recvinfo_fetch_chunk(&c->recv, chunk))
+			return 1;
+		
+		/* TODO: empty the recvinfo */
+		bytes = net_udp4_recv(c->socket, &addr, c->recv.buffer, NET_MAX_PACKETSIZE);
 
 		/* no more packets for now */
 		if(bytes <= 0)
 			break;
-		
-		r = check_packet(c->recv_buffer, bytes, &data);
-		
-		if(r == 0)
+
+		if(unpack_packet(c->recv.buffer, bytes, &c->recv.data) == 0)
 		{
-			if(data.flags&NETWORK_PACKETFLAG_CONNLESS)
-			{
-				/* connection less packets */
-				packet->client_id = -1;
-				packet->address = addr;
-				packet->flags = PACKETFLAG_CONNLESS;
-				packet->data_size = data.data_size;
-				packet->data = data.data;
-				return 1;		
-			}
-			else
-			{
-				if(conn_feed(&c->conn, &data, &addr))
-				{
-					/* fill in packet */
-					packet->client_id = 0;
-					packet->address = addr;
-					packet->flags = 0;
-					packet->data_size = data.data_size;
-					packet->data = data.data;
-					return 1;
-				}
-				else
-				{
-					/* errornous packet, drop it */
-				}
-			}			
+			/* TODO: handle connection less packets here */
+			if(conn_feed(&c->conn, &c->recv.data, &addr))
+				recvinfo_start(&c->recv, &addr, &c->conn, 0);
 		}
 	}
-	
 	return 0;
 }
 
-int netclient_send(NETCLIENT *c, NETPACKET *packet)
+int netclient_send(NETCLIENT *c, NETCHUNK *chunk)
 {
-	if(packet->data_size >= NETWORK_MAX_PAYLOAD)
+	if(chunk->data_size >= NET_MAX_PAYLOAD)
 	{
-		dbg_msg("netclient", "packet payload too big. %d. dropping packet", packet->data_size);
+		dbg_msg("netclient", "chunk payload too big. %d. dropping chunk", chunk->data_size);
 		return -1;
 	}
 	
-	if(packet->flags&PACKETFLAG_CONNLESS)
+	if(chunk->flags&NETSENDFLAG_CONNLESS)
 	{
 		/* send connectionless packet */
-		NETPACKETDATA p;
-		p.ID[0] = 'T';
-		p.ID[1] = 'W';
-		p.version = NETWORK_VERSION;
-		p.flags = NETWORK_PACKETFLAG_CONNLESS;
-		p.seq = 0;
-		p.ack = 0;
-		p.crc = 0;
-		p.token = 0;
-		p.data_size = packet->data_size;
-		p.data = (unsigned char *)packet->data;
-		send_packet(c->socket, &packet->address, &p);
+		/* TODO: fix connectionless packets */
 	}
 	else
 	{
-		int flags = 0;		
-		dbg_assert(packet->client_id == 0, "errornous client id");
-		if(packet->flags&PACKETFLAG_VITAL)
-			flags |= NETWORK_PACKETFLAG_VITAL;
-		conn_send(&c->conn, flags, packet->data_size, packet->data);
+		int f = 0;
+		dbg_assert(chunk->client_id == 0, "errornous client id");
+		
+		if(chunk->flags&NETSENDFLAG_VITAL)
+			f = NET_CHUNKFLAG_VITAL;
+		
+		conn_queue_chunk(&c->conn, f, chunk->data_size, chunk->data);
+
+		if(chunk->flags&NETSENDFLAG_FLUSH)
+			conn_flush(&c->conn);
 	}
 	return 0;
 }
 
 int netclient_state(NETCLIENT *c)
 {
-	if(c->conn.state == NETWORK_CONNSTATE_ONLINE)
+	if(c->conn.state == NET_CONNSTATE_ONLINE)
 		return NETSTATE_ONLINE;
-	if(c->conn.state == NETWORK_CONNSTATE_OFFLINE)
+	if(c->conn.state == NET_CONNSTATE_OFFLINE)
 		return NETSTATE_OFFLINE;
 	return NETSTATE_CONNECTING;
 }
@@ -976,3 +1060,37 @@ const char *netclient_error_string(NETCLIENT *c)
 {
 	return conn_error(&c->conn);
 }
+
+void netcommon_openlog(const char *filename)
+{
+	datalog = io_open(filename, IOFLAG_WRITE);
+}
+
+
+static const int freq_table[256+1] = {
+31230,4545,2657,431,1950,919,444,482,2244,617,838,542,715,1814,304,240,754,212,647,186,
+283,131,146,166,543,164,167,136,179,859,363,113,157,154,204,108,137,180,202,176,
+872,404,168,134,151,111,113,109,120,126,129,100,41,20,16,22,18,18,17,19,
+16,37,13,21,362,166,99,78,95,88,81,70,83,284,91,187,77,68,52,68,
+59,66,61,638,71,157,50,46,69,43,11,24,13,19,10,12,12,20,14,9,
+20,20,10,10,15,15,12,12,7,19,15,14,13,18,35,19,17,14,8,5,
+15,17,9,15,14,18,8,10,2173,134,157,68,188,60,170,60,194,62,175,71,
+148,67,167,78,211,67,156,69,1674,90,174,53,147,89,181,51,174,63,163,80,
+167,94,128,122,223,153,218,77,200,110,190,73,174,69,145,66,277,143,141,60,
+136,53,180,57,142,57,158,61,166,112,152,92,26,22,21,28,20,26,30,21,
+32,27,20,17,23,21,30,22,22,21,27,25,17,27,23,18,39,26,15,21,
+12,18,18,27,20,18,15,19,11,17,33,12,18,15,19,18,16,26,17,18,
+9,10,25,22,22,17,20,16,6,16,15,20,14,18,24,335,1517 };
+
+void netcommon_init()
+{
+	int i;
+	huffman_init(&huffmanstate);
+	for(i = 0; i < 256; i++)
+	{
+		unsigned char sym = (unsigned char)i;
+		huffman_add_symbol(&huffmanstate, freq_table[i], 1, &sym);
+	}
+	
+	huffman_construct_tree(&huffmanstate);
+}
diff --git a/src/engine/e_network.h b/src/engine/e_network.h
index 29843849..cfa59d84 100644
--- a/src/engine/e_network.h
+++ b/src/engine/e_network.h
@@ -9,9 +9,9 @@ typedef struct
 	int flags;
 	int data_size;
 	const void *data;
-} NETPACKET;
+} NETCHUNK;
 
-typedef struct
+/*typedef struct
 {
 	int send_bytes;
 	int recv_bytes;
@@ -20,7 +20,7 @@ typedef struct
 	
 	int resend_packets;
 	int resend_bytes;
-} NETSTATS;
+} NETSTATS;*/
 
 typedef struct NETSERVER_t NETSERVER;
 typedef struct NETCLIENT_t NETCLIENT;
@@ -28,8 +28,9 @@ typedef struct NETCLIENT_t NETCLIENT;
 enum
 {
 	NETFLAG_ALLOWSTATELESS=1,
-	PACKETFLAG_VITAL=1,
-	PACKETFLAG_CONNLESS=2,
+	NETSENDFLAG_VITAL=1,
+	NETSENDFLAG_CONNLESS=2,
+	NETSENDFLAG_FLUSH=4,
 	
 	NETSTATE_OFFLINE=0,
 	NETSTATE_CONNECTING,
@@ -39,30 +40,34 @@ enum
 typedef int (*NETFUNC_DELCLIENT)(int cid, void *user);
 typedef int (*NETFUNC_NEWCLIENT)(int cid, void *user);
 
+/* both */
+void netcommon_openlog(const char *filename);
+void netcommon_init();
+
 /* server side */
 NETSERVER *netserver_open(NETADDR4 bindaddr, int max_clients, int flags);
 int netserver_set_callbacks(NETSERVER *s, NETFUNC_NEWCLIENT new_client, NETFUNC_DELCLIENT del_client, void *user);
-int netserver_recv(NETSERVER *s, NETPACKET *packet);
-int netserver_send(NETSERVER *s, NETPACKET *packet);
+int netserver_recv(NETSERVER *s, NETCHUNK *chunk);
+int netserver_send(NETSERVER *s, NETCHUNK *chunk);
 int netserver_close(NETSERVER *s);
 int netserver_update(NETSERVER *s);
 NETSOCKET netserver_socket(NETSERVER *s);
 int netserver_drop(NETSERVER *s, int client_id, const char *reason);
 int netserver_client_addr(NETSERVER *s, int client_id, NETADDR4 *addr);
 int netserver_max_clients(NETSERVER *s);
-void netserver_stats(NETSERVER *s, NETSTATS *stats);
+/*void netserver_stats(NETSERVER *s, NETSTATS *stats);*/
 
 /* client side */
 NETCLIENT *netclient_open(NETADDR4 bindaddr, int flags);
 int netclient_disconnect(NETCLIENT *c, const char *reason);
 int netclient_connect(NETCLIENT *c, NETADDR4 *addr);
-int netclient_recv(NETCLIENT *c, NETPACKET *packet);
-int netclient_send(NETCLIENT *c, NETPACKET *packet);
+int netclient_recv(NETCLIENT *c, NETCHUNK *chunk);
+int netclient_send(NETCLIENT *c, NETCHUNK *chunk);
 int netclient_close(NETCLIENT *c);
 int netclient_update(NETCLIENT *c);
 int netclient_state(NETCLIENT *c);
 int netclient_gotproblems(NETCLIENT *c);
-void netclient_stats(NETCLIENT *c, NETSTATS *stats);
+/*void netclient_stats(NETCLIENT *c, NETSTATS *stats);*/
 const char *netclient_error_string(NETCLIENT *c);
 
 #ifdef __cplusplus
@@ -79,14 +84,14 @@ public:
 	int set_callbacks(NETFUNC_NEWCLIENT new_client, NETFUNC_DELCLIENT del_client, void *user)
 	{ return netserver_set_callbacks(ptr, new_client, del_client, user); }
 	
-	int recv(NETPACKET *packet) { return netserver_recv(ptr, packet); }
-	int send(NETPACKET *packet) { return netserver_send(ptr, packet); }
+	int recv(NETCHUNK *chunk) { return netserver_recv(ptr, chunk); }
+	int send(NETCHUNK *chunk) { return netserver_send(ptr, chunk); }
 	int update() { return netserver_update(ptr); }
 	
 	int drop(int client_id, const char *reason) { return netserver_drop(ptr, client_id, reason); } 
 
 	int max_clients() { return netserver_max_clients(ptr); }
-	void stats(NETSTATS *stats) { netserver_stats(ptr, stats); }
+	/*void stats(NETSTATS *stats) { netserver_stats(ptr, stats); }*/
 };
 
 
@@ -103,13 +108,14 @@ public:
 	int connect(NETADDR4 *addr) { return netclient_connect(ptr, addr); }
 	int disconnect(const char *reason) { return netclient_disconnect(ptr, reason); }
 	
-	int recv(NETPACKET *packet) { return netclient_recv(ptr, packet); }
-	int send(NETPACKET *packet) { return netclient_send(ptr, packet); }
+	int recv(NETCHUNK *chunk) { return netclient_recv(ptr, chunk); }
+	int send(NETCHUNK *chunk) { return netclient_send(ptr, chunk); }
 	int update() { return netclient_update(ptr); }
 	
 	const char *error_string() { return netclient_error_string(ptr); }
 	
 	int state() { return netclient_state(ptr); }
-	void stats(NETSTATS *stats) { netclient_stats(ptr, stats); }
+	/*void stats(NETSTATS *stats) { netclient_stats(ptr, stats); }*/
 };
 #endif
+
diff --git a/src/engine/e_system.c b/src/engine/e_system.c
index 9545369d..f87fd6f3 100644
--- a/src/engine/e_system.c
+++ b/src/engine/e_system.c
@@ -54,6 +54,8 @@ IOHANDLE io_stderr() { return (IOHANDLE)stderr; }
 static DBG_LOGGER loggers[16];
 static int num_loggers = 0;
 
+static NETSTATS network_stats = {0};
+
 void dbg_logger(DBG_LOGGER logger)
 {
 	loggers[num_loggers++] = logger;
@@ -557,6 +559,8 @@ int net_udp4_send(NETSOCKET sock, const NETADDR4 *addr, const void *data, int si
 	d = sendto((int)sock, (const char*)data, size, 0, &sa, sizeof(sa));
 	if(d < 0)
 		dbg_msg("net", "sendto error %d %x", d, d);
+	network_stats.sent_bytes += size;
+	network_stats.sent_packets++;
 	return d;
 }
 
@@ -569,6 +573,8 @@ int net_udp4_recv(NETSOCKET sock, NETADDR4 *addr, void *data, int maxsize)
 	if(bytes > 0)
 	{
 		sockaddr_to_netaddr4(&from, addr);
+		network_stats.recv_bytes += bytes;
+		network_stats.recv_packets++;
 		return bytes;
 	}
 	else if(bytes == 0)
@@ -940,6 +946,25 @@ const char *str_find_nocase(const char *haystack, const char *needle)
 	return 0;
 }
 
+void str_hex(char *dst, int dst_size, const void *data, int data_size)
+{
+	static const char hex[] = "0123456789ABCDEF";
+	int b;
+
+	for(b = 0; b < data_size && b < dst_size/4-4; b++)
+	{
+		dst[b*3] = hex[((const unsigned char *)data)[b]>>4];
+		dst[b*3+1] = hex[((const unsigned char *)data)[b]&0xf];
+		dst[b*3+2] = ' ';
+		dst[b*3+3] = 0;
+	}
+}
+
+void net_stats(NETSTATS *stats_inout)
+{
+	*stats_inout = network_stats;
+}
+
 #if defined(__cplusplus)
 }
 #endif
diff --git a/src/engine/e_system.h b/src/engine/e_system.h
index 9902cb24..38c2b076 100644
--- a/src/engine/e_system.h
+++ b/src/engine/e_system.h
@@ -541,6 +541,7 @@ void str_sanitize_strong(char *str);
 void str_sanitize(char *str);
 int str_comp_nocase(const char *a, const char *b);
 const char *str_find_nocase(const char *haystack, const char *needle);
+void str_hex(char *dst, int dst_size, const void *data, int data_size);
 
 typedef void (*DBG_LOGGER)(const char *line);
 void dbg_logger(DBG_LOGGER logger);
@@ -552,6 +553,16 @@ IOHANDLE io_stdin();
 IOHANDLE io_stdout();
 IOHANDLE io_stderr();
 
+typedef struct
+{
+	int sent_packets;
+	int sent_bytes;
+	int recv_packets;
+	int recv_bytes;
+} NETSTATS;
+
+void net_stats(NETSTATS *stats);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/engine/server/es_register.c b/src/engine/server/es_register.c
index fe8cd959..cec0fa19 100644
--- a/src/engine/server/es_register.c
+++ b/src/engine/server/es_register.c
@@ -31,10 +31,10 @@ static void register_new_state(int state)
 
 static void register_send_fwcheckresponse(NETADDR4 *addr)
 {
-	NETPACKET packet;
+	NETCHUNK packet;
 	packet.client_id = -1;
 	packet.address = *addr;
-	packet.flags = PACKETFLAG_CONNLESS;
+	packet.flags = NETSENDFLAG_CONNLESS;
 	packet.data_size = sizeof(SERVERBROWSE_FWRESPONSE);
 	packet.data = SERVERBROWSE_FWRESPONSE;
 	netserver_send(net, &packet);
@@ -44,13 +44,13 @@ static void register_send_heartbeat(NETADDR4 addr)
 {
 	static unsigned char data[sizeof(SERVERBROWSE_HEARTBEAT) + 2];
 	unsigned short port = config.sv_port;
-	NETPACKET packet;
+	NETCHUNK packet;
 	
 	mem_copy(data, SERVERBROWSE_HEARTBEAT, sizeof(SERVERBROWSE_HEARTBEAT));
 	
 	packet.client_id = -1;
 	packet.address = addr;
-	packet.flags = PACKETFLAG_CONNLESS;
+	packet.flags = NETSENDFLAG_CONNLESS;
 	packet.data_size = sizeof(SERVERBROWSE_HEARTBEAT) + 2;
 	packet.data = &data;
 
@@ -64,10 +64,10 @@ static void register_send_heartbeat(NETADDR4 addr)
 
 static void register_send_count_request(NETADDR4 addr)
 {
-	NETPACKET packet;
+	NETCHUNK packet;
 	packet.client_id = -1;
 	packet.address = addr;
-	packet.flags = PACKETFLAG_CONNLESS;
+	packet.flags = NETSENDFLAG_CONNLESS;
 	packet.data_size = sizeof(SERVERBROWSE_GETCOUNT);
 	packet.data = SERVERBROWSE_GETCOUNT;
 	netserver_send(net, &packet);
@@ -221,7 +221,7 @@ void register_update()
 	}
 }
 
-static void register_got_count(NETPACKET *p)
+static void register_got_count(NETCHUNK *p)
 {
 	unsigned char *data = (unsigned char *)p->data;
 	int count = (data[sizeof(SERVERBROWSE_COUNT)]<<8) | data[sizeof(SERVERBROWSE_COUNT)+1];
@@ -237,7 +237,7 @@ static void register_got_count(NETPACKET *p)
 	}
 }
 
-int register_process_packet(NETPACKET *packet)
+int register_process_packet(NETCHUNK *packet)
 {
 	if(packet->data_size == sizeof(SERVERBROWSE_FWCHECK) &&
 		memcmp(packet->data, SERVERBROWSE_FWCHECK, sizeof(SERVERBROWSE_FWCHECK)) == 0)
diff --git a/src/engine/server/es_server.c b/src/engine/server/es_server.c
index a0c35772..a64bbd46 100644
--- a/src/engine/server/es_server.c
+++ b/src/engine/server/es_server.c
@@ -310,22 +310,23 @@ int server_getclientinfo(int client_id, CLIENT_INFO *info)
 	return 0;
 }
 
-
 int server_send_msg(int client_id)
 {
 	const MSG_INFO *info = msg_get_info();
-	NETPACKET packet;
+	NETCHUNK packet;
 	if(!info)
 		return -1;
 		
-	mem_zero(&packet, sizeof(NETPACKET));
+	mem_zero(&packet, sizeof(NETCHUNK));
 	
 	packet.client_id = client_id;
 	packet.data = info->data;
 	packet.data_size = info->size;
 
-	if(info->flags&MSGFLAG_VITAL)	
-		packet.flags = PACKETFLAG_VITAL;
+	if(info->flags&MSGFLAG_VITAL)
+		packet.flags |= NETSENDFLAG_VITAL;
+	if(info->flags&MSGFLAG_FLUSH)
+		packet.flags |= NETSENDFLAG_FLUSH;
 			
 	if(client_id == -1)
 	{
@@ -442,8 +443,8 @@ static void server_do_snap()
 			if(deltasize)
 			{
 				/* compress it */
-				unsigned char intdata[MAX_SNAPSHOT_SIZE];
-				int intsize;
+				/*unsigned char intdata[MAX_SNAPSHOT_SIZE];
+				int intsize;*/
 				int snapshot_size;
 				const int max_size = MAX_SNAPSHOT_PACKSIZE;
 				int numpackets;
@@ -456,16 +457,17 @@ static void server_do_snap()
 					{
 						static PERFORMACE_INFO scope = {"int", 0};
 						perf_start(&scope);
-						intsize = intpack_compress(deltadata, deltasize, intdata);
+						snapshot_size = intpack_compress(deltadata, deltasize, compdata);
 						perf_end();
 					}
 					
+					/*
 					{
 						static PERFORMACE_INFO scope = {"zero", 0};
 						perf_start(&scope);
 						snapshot_size = zerobit_compress(intdata, intsize, compdata);
 						perf_end();
-					}
+					}*/
 					perf_end();
 				}
 				
@@ -480,9 +482,9 @@ static void server_do_snap()
 					left -= chunk;
 
 					if(numpackets == 1)
-						msg_pack_start_system(NETMSG_SNAPSINGLE, 0);
+						msg_pack_start_system(NETMSG_SNAPSINGLE, MSGFLAG_FLUSH);
 					else
-						msg_pack_start_system(NETMSG_SNAP, 0);
+						msg_pack_start_system(NETMSG_SNAP, MSGFLAG_FLUSH);
 						
 					msg_pack_int(current_tick);
 					msg_pack_int(current_tick-delta_tick); /* compressed with */
@@ -504,7 +506,7 @@ static void server_do_snap()
 			}
 			else
 			{
-				msg_pack_start_system(NETMSG_SNAPEMPTY, 0);
+				msg_pack_start_system(NETMSG_SNAPEMPTY, MSGFLAG_FLUSH);
 				msg_pack_int(current_tick);
 				msg_pack_int(current_tick-delta_tick); /* compressed with */
 				msg_pack_int(input_predtick);
@@ -597,7 +599,7 @@ static void server_send_rcon_line_authed(const char *line)
 	reentry_guard--;
 }
 
-static void server_process_client_packet(NETPACKET *packet)
+static void server_process_client_packet(NETCHUNK *packet)
 {
 	int cid = packet->client_id;
 	int sys;
@@ -763,7 +765,21 @@ static void server_process_client_packet(NETPACKET *packet)
 		}
 		else
 		{
+			char hex[] = "0123456789ABCDEF";
+			char buf[512];
+			int b;
+
+			for(b = 0; b < packet->data_size && b < 32; b++)
+			{
+				buf[b*3] = hex[((const unsigned char *)packet->data)[b]>>4];
+				buf[b*3+1] = hex[((const unsigned char *)packet->data)[b]&0xf];
+				buf[b*3+2] = ' ';
+				buf[b*3+3] = 0;
+			}
+
 			dbg_msg("server", "strange message cid=%d msg=%d data_size=%d", cid, msg, packet->data_size);
+			dbg_msg("server", "%s", buf);
+			
 		}
 	}
 	else
@@ -775,7 +791,7 @@ static void server_process_client_packet(NETPACKET *packet)
 
 static void server_send_serverinfo(NETADDR4 *addr, int lan)
 {
-	NETPACKET packet;
+	NETCHUNK packet;
 	PACKER p;
 	char buf[128];
 
@@ -827,18 +843,18 @@ static void server_send_serverinfo(NETADDR4 *addr, int lan)
 	
 	packet.client_id = -1;
 	packet.address = *addr;
-	packet.flags = PACKETFLAG_CONNLESS;
+	packet.flags = NETSENDFLAG_CONNLESS;
 	packet.data_size = packer_size(&p);
 	packet.data = packer_data(&p);
 	netserver_send(net, &packet);
 }
 
-extern int register_process_packet(NETPACKET *packet);
+extern int register_process_packet(NETCHUNK *packet);
 extern int register_update();
 
 static void server_pump_network()
 {
-	NETPACKET packet;
+	NETCHUNK packet;
 
 	netserver_update(net);
 	
@@ -1063,6 +1079,7 @@ static int server_run()
 			{
 				if(config.debug)
 				{
+					/*
 					static NETSTATS prev_stats;
 					NETSTATS stats;
 					netserver_stats(net, &stats);
@@ -1077,6 +1094,7 @@ static int server_run()
 						(stats.recv_bytes - prev_stats.recv_bytes)/reportinterval);
 						
 					prev_stats = stats;
+					*/
 				}
 	
 				reporttime += time_freq()*reportinterval;
@@ -1147,6 +1165,8 @@ int main(int argc, char **argv)
 	dbg_msg("server", "starting...");
 	engine_init("Teeworlds");
 	
+	netcommon_openlog("output.dat");
+	
 	/* register all console commands */
 	server_register_commands();
 	mods_console_init();
diff --git a/src/game/client/gc_hooks.cpp b/src/game/client/gc_hooks.cpp
index 33b9ba91..72c94abc 100644
--- a/src/game/client/gc_hooks.cpp
+++ b/src/game/client/gc_hooks.cpp
@@ -361,6 +361,7 @@ extern "C" void modc_rcon_line(const char *line)
 extern "C" int modc_snap_input(int *data)
 {
 	static NETOBJ_PLAYER_INPUT last_data = {0};
+	static int64 last_send_time = 0;
 	
 	// update player state
 	if(chat_mode != CHATMODE_NONE)
@@ -414,8 +415,28 @@ extern "C" int modc_snap_input(int *data)
 		input_data.target_y = (int)(cosf(t*3)*100.0f);
 	}
 
-	// copy and return size	
+	// check if we need to send input
+	bool send = false;
+	if(input_data.left != last_data.left) send = true;
+	else if(input_data.left != last_data.left) send = true;
+	else if(input_data.right != last_data.right) send = true;
+	else if(input_data.jump != last_data.jump) send = true;
+	else if(input_data.fire != last_data.fire) send = true;
+	else if(input_data.hook != last_data.hook) send = true;
+	else if(input_data.player_state != last_data.player_state) send = true;
+	else if(input_data.wanted_weapon != last_data.wanted_weapon) send = true;
+	else if(input_data.next_weapon != last_data.next_weapon) send = true;
+	else if(input_data.prev_weapon != last_data.prev_weapon) send = true;
+
+	if(time_get() > last_send_time + time_freq()/5)
+		send = true;
+
 	last_data = input_data;
+	if(!send)
+		return 0;
+		
+	// copy and return size	
+	last_send_time = time_get();
 	mem_copy(data, &input_data, sizeof(input_data));
 	return sizeof(input_data);
 }
diff --git a/src/game/g_version.h b/src/game/g_version.h
index e577316a..7472c783 100644
--- a/src/game/g_version.h
+++ b/src/game/g_version.h
@@ -1,4 +1,4 @@
 /* copyright (c) 2007 magnus auvinen, see licence.txt for more info */
 #include "generated/nethash.c"
-#define GAME_VERSION "0.4.2"
-#define GAME_NETVERSION "0.4 " GAME_NETVERSION_HASH
+#define GAME_VERSION "0.5.0-dev"
+#define GAME_NETVERSION "0.5 " GAME_NETVERSION_HASH
diff --git a/src/mastersrv/mastersrv.cpp b/src/mastersrv/mastersrv.cpp
index b83330a5..c9022328 100644
--- a/src/mastersrv/mastersrv.cpp
+++ b/src/mastersrv/mastersrv.cpp
@@ -43,10 +43,10 @@ static net_client net_op; // main
 
 void send_ok(NETADDR4 *addr)
 {
-	NETPACKET p;
+	NETCHUNK p;
 	p.client_id = -1;
 	p.address = *addr;
-	p.flags = PACKETFLAG_CONNLESS;
+	p.flags = NETSENDFLAG_CONNLESS;
 	p.data_size = sizeof(SERVERBROWSE_FWOK);
 	p.data = SERVERBROWSE_FWOK;
 	
@@ -57,10 +57,10 @@ void send_ok(NETADDR4 *addr)
 
 void send_error(NETADDR4 *addr)
 {
-	NETPACKET p;
+	NETCHUNK p;
 	p.client_id = -1;
 	p.address = *addr;
-	p.flags = PACKETFLAG_CONNLESS;
+	p.flags = NETSENDFLAG_CONNLESS;
 	p.data_size = sizeof(SERVERBROWSE_FWERROR);
 	p.data = SERVERBROWSE_FWERROR;
 	net_op.send(&p);
@@ -68,10 +68,10 @@ void send_error(NETADDR4 *addr)
 
 void send_check(NETADDR4 *addr)
 {
-	NETPACKET p;
+	NETCHUNK p;
 	p.client_id = -1;
 	p.address = *addr;
-	p.flags = PACKETFLAG_CONNLESS;
+	p.flags = NETSENDFLAG_CONNLESS;
 	p.data_size = sizeof(SERVERBROWSE_FWCHECK);
 	p.data = SERVERBROWSE_FWCHECK;
 	net_checker.send(&p);
@@ -204,7 +204,7 @@ int main(int argc, char **argv)
 		net_checker.update();
 		
 		// process packets
-		NETPACKET packet;
+		NETCHUNK packet;
 		while(net_op.recv(&packet))
 		{
 			if(packet.data_size == sizeof(SERVERBROWSE_HEARTBEAT)+2 &&
@@ -225,10 +225,10 @@ int main(int argc, char **argv)
 			{
 				dbg_msg("mastersrv", "count requested, responding with %d", num_servers);
 				
-				NETPACKET p;
+				NETCHUNK p;
 				p.client_id = -1;
 				p.address = packet.address;
-				p.flags = PACKETFLAG_CONNLESS;
+				p.flags = NETSENDFLAG_CONNLESS;
 				p.data_size = sizeof(count_data);
 				p.data = &count_data;
 				count_data.high = (num_servers>>8)&0xff;
@@ -240,10 +240,10 @@ int main(int argc, char **argv)
 			{
 				// someone requested the list
 				dbg_msg("mastersrv", "requested, responding with %d servers", num_servers);
-				NETPACKET p;
+				NETCHUNK p;
 				p.client_id = -1;
 				p.address = packet.address;
-				p.flags = PACKETFLAG_CONNLESS;
+				p.flags = NETSENDFLAG_CONNLESS;
 				p.data_size = num_servers*sizeof(NETADDR4)+sizeof(SERVERBROWSE_LIST);
 				p.data = &data;
 				net_op.send(&p);
diff --git a/src/tools/fake_server.c b/src/tools/fake_server.c
index 00fe20f8..9f588e37 100644
--- a/src/tools/fake_server.c
+++ b/src/tools/fake_server.c
@@ -50,13 +50,13 @@ int max_players = 0;
 static void send_heartbeats()
 {
 	static unsigned char data[sizeof(SERVERBROWSE_HEARTBEAT) + 2];
-	NETPACKET packet;
+	NETCHUNK packet;
 	int i;
 	
 	mem_copy(data, SERVERBROWSE_HEARTBEAT, sizeof(SERVERBROWSE_HEARTBEAT));
 	
 	packet.client_id = -1;
-	packet.flags = PACKETFLAG_CONNLESS;
+	packet.flags = NETSENDFLAG_CONNLESS;
 	packet.data_size = sizeof(SERVERBROWSE_HEARTBEAT) + 2;
 	packet.data = &data;
 
@@ -111,10 +111,10 @@ static void build_infomessage()
 
 static void send_serverinfo(NETADDR4 *addr)
 {
-	NETPACKET p;
+	NETCHUNK p;
 	p.client_id = -1;
 	p.address = *addr;
-	p.flags = PACKETFLAG_CONNLESS;
+	p.flags = NETSENDFLAG_CONNLESS;
 	p.data_size = infomsg_size;
 	p.data = infomsg;
 	netserver_send(net, &p);
@@ -122,10 +122,10 @@ static void send_serverinfo(NETADDR4 *addr)
 
 static void send_fwcheckresponse(NETADDR4 *addr)
 {
-	NETPACKET p;
+	NETCHUNK p;
 	p.client_id = -1;
 	p.address = *addr;
-	p.flags = PACKETFLAG_CONNLESS;
+	p.flags = NETSENDFLAG_CONNLESS;
 	p.data_size = sizeof(SERVERBROWSE_FWRESPONSE);
 	p.data = SERVERBROWSE_FWRESPONSE;
 	netserver_send(net, &p);
@@ -141,7 +141,7 @@ static int run()
 	
 	while(1)
 	{
-		NETPACKET p;
+		NETCHUNK p;
 		netserver_update(net);
 		while(netserver_recv(net, &p))
 		{