1 files changed, 630 insertions, 0 deletions

diff --git a/src/engine/e_datafile.c b/src/engine/e_datafile.c
new file mode 100644
index 00000000..14138ed9
--- /dev/null
+++ b/src/engine/e_datafile.c
@@ -0,0 +1,630 @@
+/* copyright (c) 2007 magnus auvinen, see licence.txt for more info */
+#include "e_system.h"
+#include "e_datafile.h"
+#include "external/zlib/zlib.h"
+
+static const int DEBUG=0;
+
+typedef struct 
+{
+	int type;
+	int start;
+	int num;
+} DATAFILE_ITEM_TYPE;
+
+typedef struct 
+{
+	int type_and_id;
+	int size;
+} DATAFILE_ITEM;
+
+typedef struct
+{
+	char id[4];
+	int version;
+	int size;
+	int swaplen;
+	int num_item_types;
+	int num_items;
+	int num_raw_data;
+	int item_size;
+	int data_size;
+} DATAFILE_HEADER;
+
+typedef struct
+{
+	int num_item_types;
+	int num_items;
+	int num_raw_data;
+	int item_size;
+	int data_size;
+	char start[4];
+} DATAFILE_DATA;
+
+typedef struct
+{
+	DATAFILE_ITEM_TYPE *item_types;
+	int *item_offsets;
+	int *data_offsets;
+	int *data_sizes;
+
+	char *item_start;
+	char *data_start;
+} DATAFILE_INFO;
+
+struct DATAFILE_t
+{
+	IOHANDLE file;
+	DATAFILE_INFO info;
+	DATAFILE_HEADER header;
+	int data_start_offset;
+	char **data_ptrs;
+	char *data;
+};
+
+DATAFILE *datafile_load(const char *filename)
+{
+	DATAFILE *df;
+	IOHANDLE file;
+	DATAFILE_HEADER header;
+	unsigned readsize;
+	
+	unsigned *dst;
+	unsigned char *src;
+	unsigned j;
+	int size = 0;
+	int allocsize = 0;
+	
+	(void)dst;
+	(void)src;
+	(void)j;
+	
+	
+	dbg_msg("datafile", "datafile loading. filename='%s'", filename);
+
+	file = io_open(filename, IOFLAG_READ);
+	if(!file)
+		return 0;
+	
+	/* TODO: change this header */
+	io_read(file, &header, sizeof(header));
+	if(header.id[0] != 'A' || header.id[1] != 'T' || header.id[2] != 'A' || header.id[3] != 'D')
+	{
+		if(header.id[0] != 'D' || header.id[1] != 'A' || header.id[2] != 'T' || header.id[3] != 'A')
+		{
+			dbg_msg("datafile", "wrong signature. %x %x %x %x", header.id[0], header.id[1], header.id[2], header.id[3]);
+			return 0;
+		}
+	}
+
+#if defined(CONF_ARCH_ENDIAN_BIG)
+	swap_endian(&header, sizeof(int), sizeof(header)/sizeof(int));	
+#endif
+	if(header.version != 3 && header.version != 4)
+	{
+		dbg_msg("datafile", "wrong version. version=%x", header.version);
+		return 0;
+	}
+	
+	/* read in the rest except the data */
+	size = 0;
+	size += header.num_item_types*sizeof(DATAFILE_ITEM_TYPE);
+	size += (header.num_items+header.num_raw_data)*sizeof(int);
+	if(header.version == 4)
+		size += header.num_raw_data*sizeof(int); /* v4 has uncompressed data sizes aswell */
+	size += header.item_size;
+	
+	allocsize = size;
+	allocsize += sizeof(DATAFILE); /* add space for info structure */
+	allocsize += header.num_raw_data*sizeof(void*); /* add space for data pointers */
+
+	df = (DATAFILE*)mem_alloc(allocsize, 1);
+	df->header = header;
+	df->data_start_offset = sizeof(DATAFILE_HEADER) + size;
+	df->data_ptrs = (char**)(df+1);
+	df->data = (char *)(df+1)+header.num_raw_data*sizeof(char *);
+	df->file = file;
+	
+	/* clear the data pointers */
+	mem_zero(df->data_ptrs, header.num_raw_data*sizeof(void*));
+	
+	/* read types, offsets, sizes and item data */
+	readsize = io_read(file, df->data, size);
+	if(readsize != size)
+	{
+		dbg_msg("datafile", "couldn't load the whole thing, wanted=%d got=%d", size, readsize);
+		return 0;
+	}
+
+#if defined(CONF_ARCH_ENDIAN_BIG)
+	swap_endian(df->data, sizeof(int), header.swaplen);
+#endif
+	
+	if(DEBUG)
+		dbg_msg("datafile", "item_size=%d", df->header.item_size);
+	
+	df->info.item_types = (DATAFILE_ITEM_TYPE *)df->data;
+	df->info.item_offsets = (int *)&df->info.item_types[df->header.num_item_types];
+	df->info.data_offsets = (int *)&df->info.item_offsets[df->header.num_items];
+	df->info.data_sizes = (int *)&df->info.data_offsets[df->header.num_raw_data];
+	
+	if(header.version == 4)
+		df->info.item_start = (char *)&df->info.data_sizes[df->header.num_raw_data];
+	else
+		df->info.item_start = (char *)&df->info.data_offsets[df->header.num_raw_data];
+	df->info.data_start = df->info.item_start + df->header.item_size;
+
+	if(DEBUG)
+		dbg_msg("datafile", "datafile loading done. datafile='%s'", filename);
+
+	if(DEBUG)
+	{
+		/*
+		for(int i = 0; i < df->data.num_raw_data; i++)
+		{
+			void *p = datafile_get_data(df, i);
+			dbg_msg("datafile", "%d %d", (int)((char*)p - (char*)(&df->data)), size);
+		}
+			
+		for(int i = 0; i < datafile_num_items(df); i++)
+		{
+			int type, id;
+			void *data = datafile_get_item(df, i, &type, &id);
+			dbg_msg("map", "\t%d: type=%x id=%x p=%p offset=%d", i, type, id, data, df->info.item_offsets[i]);
+			int *idata = (int*)data;
+			for(int k = 0; k < 3; k++)
+				dbg_msg("datafile", "\t\t%d=%d (%x)", k, idata[k], idata[k]);
+		}
+
+		for(int i = 0; i < df->data.num_item_types; i++)
+		{
+			dbg_msg("map", "\t%d: type=%x start=%d num=%d", i,
+				df->info.item_types[i].type,
+				df->info.item_types[i].start,
+				df->info.item_types[i].num);
+			for(int k = 0; k < df->info.item_types[i].num; k++)
+			{
+				int type, id;
+				datafile_get_item(df, df->info.item_types[i].start+k, &type, &id);
+				if(type != df->info.item_types[i].type)
+					dbg_msg("map", "\tERROR");
+			}
+		}
+		*/
+	}
+		
+	return df;
+}
+
+int datafile_num_data(DATAFILE *df)
+{
+	return df->header.num_raw_data;
+}
+
+/* always returns the size in the file */
+int datafile_get_datasize(DATAFILE *df, int index)
+{
+	if(index == df->header.num_raw_data-1)
+		return df->header.data_size-df->info.data_offsets[index];
+	return  df->info.data_offsets[index+1]-df->info.data_offsets[index];
+}
+
+void *datafile_get_data(DATAFILE *df, int index)
+{
+	/* load it if needed */
+	if(!df->data_ptrs[index])
+	{
+		/* fetch the data size */
+		int datasize = datafile_get_datasize(df, index);
+		
+		if(df->header.version == 4)
+		{
+			/* v4 has compressed data */
+			void *temp = (char *)mem_alloc(datasize, 1);
+			unsigned long uncompressed_size = df->info.data_sizes[index];
+			unsigned long s;
+
+			dbg_msg("datafile", "loading data index=%d size=%d", index, datasize);
+			df->data_ptrs[index] = (char *)mem_alloc(uncompressed_size, 1);
+			
+			/* read the compressed data */
+			io_seek(df->file, df->data_start_offset+df->info.data_offsets[index], IOSEEK_START);
+			io_read(df->file, temp, datasize);
+			
+			/* decompress the data, TODO: check for errors */
+			s = uncompressed_size;
+			uncompress((Bytef*)df->data_ptrs[index], &s, (Bytef*)temp, datasize);
+			
+			/* clean up the temporary buffers */
+			mem_free(temp);
+		}
+		else
+		{
+			/* load the data */
+			dbg_msg("datafile", "loading data index=%d size=%d", index, datasize);
+			df->data_ptrs[index] = (char *)mem_alloc(datasize, 1);
+			io_seek(df->file, df->data_start_offset+df->info.data_offsets[index], IOSEEK_START);
+			io_read(df->file, df->data_ptrs[index], datasize);
+		}
+	}
+	
+	return df->data_ptrs[index];
+}
+
+void datafile_unload_data(DATAFILE *df, int index)
+{
+	/* */
+	mem_free(df->data_ptrs[index]);
+	df->data_ptrs[index] = 0x0;
+}
+
+int datafile_get_itemsize(DATAFILE *df, int index)
+{
+	if(index == df->header.num_items-1)
+		return df->header.item_size-df->info.item_offsets[index];
+	return  df->info.item_offsets[index+1]-df->info.item_offsets[index];
+}
+
+void *datafile_get_item(DATAFILE *df, int index, int *type, int *id)
+{
+	DATAFILE_ITEM *i = (DATAFILE_ITEM *)(df->info.item_start+df->info.item_offsets[index]);
+	if(type)
+		*type = (i->type_and_id>>16)&0xffff; /* remove sign extention */
+	if(id)
+		*id = i->type_and_id&0xffff;
+	return (void *)(i+1);
+}
+
+void datafile_get_type(DATAFILE *df, int type, int *start, int *num)
+{
+	int i;
+	for(i = 0; i < df->header.num_item_types; i++)
+	{
+		if(df->info.item_types[i].type == type)
+		{
+			*start = df->info.item_types[i].start;
+			*num = df->info.item_types[i].num;
+			return;
+		}
+	}
+	
+	*start = 0;
+	*num = 0;
+}
+
+void *datafile_find_item(DATAFILE *df, int type, int id)
+{
+	int start, num,i ;
+	int item_id;
+	void *item;
+	
+	datafile_get_type(df, type, &start, &num);
+	for(i = 0; i < num; i++)
+	{
+		item = datafile_get_item(df, start+i,0, &item_id);
+		if(id == item_id)
+			return item;
+	}
+	return 0;
+}
+
+int datafile_num_items(DATAFILE *df)
+{
+	return df->header.num_items;
+}
+
+void datafile_unload(DATAFILE *df)
+{
+	if(df)
+	{
+		/* free the data that is loaded */
+		int i;
+		for(i = 0; i < df->header.num_raw_data; i++)
+			mem_free(df->data_ptrs[i]);
+		
+		io_close(df->file);
+		mem_free(df);
+	}
+}
+
+/* DATAFILE output */
+typedef struct
+{
+	int uncompressed_size;
+	int compressed_size;
+	void *compressed_data;
+} DATA_INFO;
+
+typedef struct
+{
+	int type;
+	int id;
+	int size;
+	int next;
+	int prev;
+	void *data;
+} ITEM_INFO;
+
+typedef struct
+{
+	int num;
+	int first;
+	int last;
+} ITEMTYPE_INFO;
+
+/* */
+struct DATAFILE_OUT_t
+{
+	IOHANDLE file;
+	int num_items;
+	int num_datas;
+	int num_item_types;
+	ITEMTYPE_INFO item_types[0xffff];
+	ITEM_INFO items[1024];
+	DATA_INFO datas[1024];
+};
+
+DATAFILE_OUT *datafile_create(const char *filename)
+{
+	int i;
+	DATAFILE_OUT *df = (DATAFILE_OUT*)mem_alloc(sizeof(DATAFILE_OUT), 1);
+	df->file = io_open(filename, IOFLAG_WRITE);
+	if(!df->file)
+	{
+		mem_free(df);
+		return 0;
+	}
+	
+	df->num_items = 0;
+	df->num_datas = 0;
+	df->num_item_types = 0;
+	mem_zero(&df->item_types, sizeof(df->item_types));
+
+	for(i = 0; i < 0xffff; i++)
+	{
+		df->item_types[i].first = -1;
+		df->item_types[i].last = -1;
+	}
+	
+	return df;
+}
+
+int datafile_add_item(DATAFILE_OUT *df, int type, int id, int size, void *data)
+{
+	df->items[df->num_items].type = type;
+	df->items[df->num_items].id = id;
+	df->items[df->num_items].size = size;
+	
+	/*
+	dbg_msg("datafile", "added item type=%d id=%d size=%d", type, id, size);
+	int i;
+	for(i = 0; i < size/4; i++)
+		dbg_msg("datafile", "\t%d: %08x %d", i, ((int*)data)[i], ((int*)data)[i]);
+	*/
+	
+	/* copy data */
+	df->items[df->num_items].data = mem_alloc(size, 1);
+	mem_copy(df->items[df->num_items].data, data, size);
+
+	if(!df->item_types[type].num) /* count item types */
+		df->num_item_types++;
+
+	/* link */
+	df->items[df->num_items].prev = df->item_types[type].last;
+	df->items[df->num_items].next = -1;
+	
+	if(df->item_types[type].last != -1)
+		df->items[df->item_types[type].last].next = df->num_items;
+	df->item_types[type].last = df->num_items;
+	
+	if(df->item_types[type].first == -1)
+		df->item_types[type].first = df->num_items;
+	
+	df->item_types[type].num++;
+		
+	df->num_items++;
+	return df->num_items-1;
+}
+
+int datafile_add_data(DATAFILE_OUT *df, int size, void *data)
+{
+	DATA_INFO *info = &df->datas[df->num_datas];
+	void *compdata = mem_alloc(size, 1); /* temporary buffer that we use duing compression */
+	unsigned long s = size;
+
+	info->uncompressed_size = size;
+	if(compress((Bytef*)compdata, &s, (Bytef*)data, size) != Z_OK)
+		dbg_assert(0, "zlib error");
+	info->compressed_size = (int)s;
+	info->compressed_data = mem_alloc(info->compressed_size, 1);
+	mem_copy(info->compressed_data, compdata, info->compressed_size);
+	mem_free(compdata);
+
+	df->num_datas++;
+	return df->num_datas-1;
+}
+
+int datafile_finish(DATAFILE_OUT *df)
+{
+	int itemsize = 0;
+	int i, count, offset;
+	int typessize, headersize, offsetsize, filesize, swapsize;
+	int datasize = 0;
+	DATAFILE_ITEM_TYPE info;
+	DATAFILE_ITEM itm;
+	DATAFILE_HEADER header;
+
+	/* we should now write this file! */
+	if(DEBUG)
+		dbg_msg("datafile", "writing");
+
+	/* calculate sizes */
+	for(i = 0; i < df->num_items; i++)
+	{
+		if(DEBUG)
+			dbg_msg("datafile", "item=%d size=%d (%d)", i, df->items[i].size, df->items[i].size+sizeof(DATAFILE_ITEM));
+		itemsize += df->items[i].size + sizeof(DATAFILE_ITEM);
+	}
+	
+	
+	for(i = 0; i < df->num_datas; i++)
+		datasize += df->datas[i].compressed_size;
+	
+	/* calculate the complete size */
+	typessize = df->num_item_types*sizeof(DATAFILE_ITEM_TYPE);
+	headersize = sizeof(DATAFILE_HEADER);
+	offsetsize = df->num_items*sizeof(int) + df->num_datas*sizeof(int);
+	filesize = headersize + typessize + offsetsize + itemsize + datasize;
+	swapsize = filesize - datasize;
+	
+	(void)swapsize;
+	
+	if(DEBUG)
+		dbg_msg("datafile", "num_item_types=%d typessize=%d itemsize=%d datasize=%d", df->num_item_types, typessize, itemsize, datasize);
+	
+	/* construct header */
+	{
+		header.id[0] = 'D';
+		header.id[1] = 'A';
+		header.id[2] = 'T';
+		header.id[3] = 'A';
+		header.version = 4;
+		header.size = filesize - 16;
+		header.swaplen = swapsize - 16;
+		header.num_item_types = df->num_item_types;
+		header.num_items = df->num_items;
+		header.num_raw_data = df->num_datas;
+		header.item_size = itemsize;
+		header.data_size = datasize;
+		
+		/* TODO: apply swapping */
+		/* write header */
+		if(DEBUG)
+			dbg_msg("datafile", "headersize=%d", sizeof(header));
+		io_write(df->file, &header, sizeof(header));
+	}
+	
+	/* write types */
+	for(i = 0, count = 0; i < 0xffff; i++)
+	{
+		if(df->item_types[i].num)
+		{
+			/* write info */
+			info.type = i;
+			info.start = count;
+			info.num = df->item_types[i].num;
+			if(DEBUG)
+				dbg_msg("datafile", "writing type=%x start=%d num=%d", info.type, info.start, info.num);
+			io_write(df->file, &info, sizeof(info));
+			count += df->item_types[i].num;
+		}
+	}
+	
+	/* write item offsets */
+	for(i = 0, offset = 0; i < 0xffff; i++)
+	{
+		if(df->item_types[i].num)
+		{
+			/* write all items in of this type */
+			int k = df->item_types[i].first;
+			while(k != -1)
+			{
+				if(DEBUG)
+					dbg_msg("datafile", "writing item offset num=%d offset=%d", k, offset);
+				io_write(df->file, &offset, sizeof(offset));
+				offset += df->items[k].size + sizeof(DATAFILE_ITEM);
+				
+				/* next */
+				k = df->items[k].next;
+			}
+		}
+	}
+	
+	/* write data offsets */
+	for(i = 0, offset = 0; i < df->num_datas; i++)
+	{
+		if(DEBUG)
+			dbg_msg("datafile", "writing data offset num=%d offset=%d", i, offset);
+		io_write(df->file, &offset, sizeof(offset));
+		offset += df->datas[i].compressed_size;
+	}
+
+	/* write data uncompressed sizes */
+	for(i = 0, offset = 0; i < df->num_datas; i++)
+	{
+		/*
+		if(DEBUG)
+			dbg_msg("datafile", "writing data offset num=%d offset=%d", i, offset);
+		*/
+		io_write(df->file, &df->datas[i].uncompressed_size, sizeof(int));
+	}
+	
+	/* write items */
+	for(i = 0; i < 0xffff; i++)
+	{
+		if(df->item_types[i].num)
+		{
+			/* write all items in of this type */
+			int k = df->item_types[i].first;
+			while(k != -1)
+			{
+				itm.type_and_id = (i<<16)|df->items[k].id;
+				itm.size = df->items[k].size;
+				if(DEBUG)
+					dbg_msg("datafile", "writing item type=%x idx=%d id=%d size=%d", i, k, df->items[k].id, df->items[k].size);
+				
+				io_write(df->file, &itm, sizeof(itm));
+				io_write(df->file, df->items[k].data, df->items[k].size);
+				
+				/* next */
+				k = df->items[k].next;
+			}
+		}
+	}
+	
+	/* write data */
+	for(i = 0; i < df->num_datas; i++)
+	{
+		if(DEBUG)
+			dbg_msg("datafile", "writing data id=%d size=%d", i, df->datas[i].compressed_size);
+		io_write(df->file, df->datas[i].compressed_data, df->datas[i].compressed_size);
+	}
+
+	/* free data */
+	for(i = 0; i < df->num_items; i++)
+		mem_free(df->items[i].data);
+
+	
+	io_close(df->file);
+	mem_free(df);
+
+	if(DEBUG)
+		dbg_msg("datafile", "done");
+	return 0;
+}
+
+#define BUFFER_SIZE 64*1024
+
+int datafile_crc(const char *filename)
+{
+	unsigned char buffer[BUFFER_SIZE];
+	IOHANDLE file;
+	int crc = 0;
+	unsigned bytes = 0;
+	
+	file = io_open(filename, IOFLAG_READ);
+	if(!file)
+		return 0;
+		
+	while(1)
+	{
+		bytes = io_read(file, buffer, BUFFER_SIZE);
+		if(bytes <= 0)
+			break;
+		crc = crc32(crc, buffer, bytes);
+	}
+	
+	io_close(file);
+
+	return crc;	
+}