add lzs comp & decomp tools

Makefile

@@ -1,11 +1,14 @@
 CFLAGS := -O2 -pipe -Wall
 
-OBJS := configfile.o event.o logging.o
-OBJS += context.o serial.o statemachine.o xmodem.o
+all: zyxel-revert compress decompress
 
-all: zyxel-revert
+zyxel-revert: configfile.o event.o logging.o context.o serial.o statemachine.o xmodem.o zyxel-revert.o
+	$(CC) $(CFLAGS) $^ -o $@
 
-zyxel-revert: $(OBJS) zyxel-revert.o
+compress: lzsc.o filedata.o compress.o
+	$(CC) $(CFLAGS) $^ -o $@
+
+decompress: lzsd.o filedata.o decompress.o
 	$(CC) $(CFLAGS) $^ -o $@
 
 %.o: %.c
@@ -15,7 +18,7 @@ zyxel-revert: $(OBJS) zyxel-revert.o
 	$(CC) $(CFLAGS) -MM -c $< -o $@
 
 clean:
-	rm -f zyxel-revert *.d *.o *.log
+	rm -f zyxel-revert compress decompress *.d *.o *.log
 
 DEPS := $(wildcard *.c)
 -include $(DEPS:.c=.d)

compress.c

@@ -0,0 +1,68 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <stdint.h>
+
+#include <arpa/inet.h>
+
+#include "lzsc.h"
+#include "filedata.h"
+
+struct rom0file {
+	uint16_t version;
+	uint16_t size;
+	uint16_t offset;
+
+	char name[14];
+} __attribute__((packed));
+
+int parse_rom0(struct filedata *filedata, const char *infile)
+{
+	struct rom0file file;
+
+	/* zweite page enthält config */
+	uint32_t offset = 0x2000;
+	while (1) {
+		memcpy(&file, (void *)(filedata->data) + offset, sizeof(file));
+
+		if (strcmp(file.name, "autoexec.net") == 0) {
+			printf("found autoexec.net: 0x%04x - 0x%04x (%d bytes)\n",
+				0x2000 + htons(file.offset), 0x2000 + htons(file.offset) + htons(file.size), htons(file.size));
+
+			/* 16 byte header */
+			void *buf = (void *)(filedata->data) + 0x2000 + htons(file.offset) +12;
+
+			/* little cleanup */
+			memset(buf, 0, htons(file.size));
+
+			struct filedata *indata = get_filedata(infile);
+			int size = lzs_pack(indata->data, indata->size, buf, 0xC00);
+
+			file.size = htons(size +12);
+
+			memcpy((void *)(filedata->data) + offset, &file, sizeof(file));
+			printf("new autoexec.net: 0x%04x - 0x%04x (%d bytes)\n",
+				0x2000 + htons(file.offset), 0x2000 + htons(file.offset) + htons(file.size), htons(file.size));
+
+			put_filedata("350LI2C1.rom.own", filedata);
+
+			free(indata);
+			return 0;
+		}
+
+		if (file.name[0] == 0 || file.name[0] == -1)
+			return -1;
+
+		offset += sizeof(file);
+	}
+}
+
+int main(int argc, char *argv[])
+{
+	struct filedata *rom0 = get_filedata("350LI2C1.rom");
+	parse_rom0(rom0, "350LI2C1.rom.own_decomp");
+
+	free(rom0);
+	return 0;
+}

decompress.c

@@ -0,0 +1,65 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <stdint.h>
+
+#include <arpa/inet.h>
+
+#include "lzsd.h"
+#include "filedata.h"
+
+struct rom0file {
+	uint16_t version;
+	uint16_t size;
+	uint16_t offset;
+
+	char name[14];
+} __attribute__((packed));
+
+int parse_rom0(struct filedata *filedata, const char *outfile)
+{
+	struct rom0file file;
+
+	/* zweite page enthält config */
+	uint32_t offset = 0x2000;
+	while (1) {
+		memcpy(&file, (void *)(filedata->data) + offset, sizeof(file));
+
+		if (strcmp(file.name, "autoexec.net") == 0) {
+			printf("found autoexec.net: 0x%04x - 0x%04x (%d bytes)\n",
+				0x2000 + htons(file.offset), 0x2000 + htons(file.offset) + htons(file.size), htons(file.size));
+
+			/* 64kb sollten reichen */
+			struct filedata *out = alloc_filedata(65535);
+
+			/* 16 byte header */
+			void *buf = (void *)(filedata->data) + 0x2000 + htons(file.offset) +12;
+
+			out->size = lzs_unpack(buf, htons(file.size) -12, out->data, out->size);
+			put_filedata(outfile, out);
+			free(out);
+
+			return 0;
+		}
+
+		if (file.name[0] == 0 || file.name[0] == -1)
+			return -1;
+
+		offset += sizeof(file);
+	}
+}
+
+int main(int argc, char *argv[])
+{
+	struct filedata *in = get_filedata(argv[1]);
+
+	char outname[64];
+	strncpy(outname, argv[1], sizeof(outname));
+	strcat(outname, ".own_decomp");
+
+	parse_rom0(in, outname);
+
+	free(in);
+	return 0;
+}

filedata.c

@@ -0,0 +1,75 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#include "filedata.h"
+
+struct filedata * get_filedata(const char *filename)
+{
+	int fd = open(filename, O_RDONLY);
+	if (fd < 0) {
+		perror("get_filedata(): open()");
+		return NULL;
+	}
+
+	struct stat filestat;
+	if (fstat(fd, &filestat) < 0) {
+		perror("get_filedata(): fstat()");
+		close(fd);
+		return NULL;
+	}
+
+	struct filedata *filedata = malloc(sizeof(struct filedata) + filestat.st_size);
+	if (filedata == NULL) {
+		perror("get_filedata(): malloc()");
+		close(fd);
+		return NULL;
+	}
+
+	filedata->size = filestat.st_size;
+
+	int readsize = read(fd, filedata->data, filedata->size);
+	if (readsize != filedata->size) {
+		perror("get_filedata(): read()");
+		free(filedata);
+		close(fd);
+		return NULL;
+	}
+
+	close(fd);
+	return filedata;
+}
+
+struct filedata * alloc_filedata(int size)
+{
+	struct filedata *retval = malloc(sizeof(struct filedata) + size);
+	if (retval == NULL) {
+		perror("alloc_filedata(): malloc()");
+		return NULL;
+	}
+	retval->size = size;
+	return retval;
+}
+
+int put_filedata(const char *filename, struct filedata *filedata)
+{
+	int fd = open(filename, O_RDWR | O_CREAT | O_TRUNC, 0644);
+	if (fd < 0) {
+		perror("put_filedata(): open()");
+		return -1;
+	}
+
+	int writesize = write(fd, filedata->data, filedata->size);
+	if (writesize != filedata->size) {
+		perror("put_filedata(): write()");
+		close(fd);
+		return -1;
+	}
+
+	close(fd);
+	return 0;
+}

filedata.h

@@ -0,0 +1,13 @@
+#ifndef _FILEDATA_H_
+#define _FILEDATA_H_
+
+struct filedata {
+	int size;
+	void *data[0];
+};
+
+struct filedata * get_filedata(const char *filename);
+struct filedata * alloc_filedata(int size);
+int put_filedata(const char *filename, struct filedata *filedata);
+
+#endif /* _FILEDATA_H_ */

lzsc.c

@@ -0,0 +1,293 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "list.h"
+
+#define HASH_BUCKETS 127
+
+#define MIN(a,b) ((a) < (b) ? (a) : (b))
+
+struct lzs_state {
+	uint8_t *srcblkstart;
+	uint8_t *src;
+	uint32_t srcsize;
+
+	uint8_t *dstblkstart;
+	uint8_t *dst;
+	uint32_t dstsize;
+
+	uint32_t bitbuf;
+	uint32_t bitcnt;
+
+	struct list_head *hash;
+};
+
+struct lzs_hash_entry {
+	struct list_head list;
+	uint8_t *pos;
+};
+
+/* TODO: check dstsize */
+static int put_bits(struct lzs_state *state, uint32_t bits, uint32_t len)
+{
+	state->bitbuf <<= len;
+	state->bitbuf |= bits;
+	state->bitcnt += len;
+
+	while (state->bitcnt >= 8) {
+		state->bitcnt -= 8;
+		*(state->dst)++ = (state->bitbuf >> (state->bitcnt)) & 0xFF;
+//		printf("    wrote byte: 0x%02x\n", *(state->dst -1));
+	}
+
+	return 0;
+}
+
+/* TODO: check dstsize */
+static int put_literal_byte(struct lzs_state *state, uint8_t byte)
+{
+	printf("  put_literal_byte: 0x%02x\n", byte);
+	return put_bits(state, (0 << 8) | byte, 1+8);
+}
+
+/* TODO: check dstsize */
+static int put_compressed_string(struct lzs_state *state, uint32_t offset, uint32_t len)
+{
+	printf("  put_compressed_string: offset=0x%03x len=0x%03x\n", offset, len);
+
+	if (offset > 0x7ff || len > 0x800)
+		printf("    ERROR\n");
+
+	if (offset < 128)
+		put_bits(state, (1 << 8) | (1 << 7) | offset, 1+1+7);
+	else
+		put_bits(state, (1 << 12) | (0 << 11) | offset, 1+1+11);
+
+	if (len <= 4) {
+		/*
+		 * 00 - 2
+		 * 01 - 3
+		 * 10 - 4
+		 */
+		put_bits(state, len - 2, 2);
+
+	} else if (len < 8) {
+		/*
+		 * 1100 - 5
+		 * 1101 - 6
+		 * 1110 - 7
+		 */
+		put_bits(state, len + 7, 4);
+
+	} else if (len >= 8) {
+		/*
+		 * 1111 0000 - 8
+		 * 1111 0001 - 9
+		 * ...
+		 * 1111 1110 - 22
+		 * 1111 1111 0000 - 23
+		 * 1111 1111 0001 - 24
+		 * ...
+		 */
+		len -= 8;
+		put_bits(state, 15, 4);
+		while (len >= 15) {
+			put_bits(state, 15, 4);
+			len -= 15;
+		}
+		put_bits(state, len, 4);
+	}
+	return 0;
+}
+
+/* TODO: check dstsize */
+static int put_blockend(struct lzs_state *state)
+{
+	/* 7bit offset = 0 -> end code */
+	put_bits(state, (1 << 8) | (1 << 7) | 0, 1+1+7);
+
+	/* align to bytes */
+	if (state->bitcnt)
+		put_bits(state, 0, 8 - state->bitcnt);
+
+	printf(" =============== BLOCK END =============== \n");
+	return 0;
+}
+
+static int put_zyxel_header(struct lzs_state *state)
+{
+	uint16_t len = state->src - state->srcblkstart;
+	uint16_t lenc = state->dst - state->dstblkstart;
+
+	/* remove own header size */
+	lenc -= 4;
+
+	printf("header of previous block: 0x%04x%04x\n", len, lenc);
+
+	uint8_t *p = state->dstblkstart;
+	p[0] = (len >> 8) & 0xFF;
+	p[1] = len & 0xFF;
+
+	p[2] = (lenc >> 8) & 0xFF;
+	p[3] = lenc & 0xFF;
+
+	return 0;
+}
+
+static int alloc_hash(struct lzs_state *state)
+{
+	state->hash = malloc(sizeof(struct lzs_hash_entry) * HASH_BUCKETS);
+	if (state->hash == NULL) {
+		perror("alloc_hashtable(): malloc()");
+		return -1;
+	}
+
+	int i;
+	for (i = 0; i < HASH_BUCKETS; i++)
+		INIT_LIST_HEAD(&state->hash[i]);
+
+	return 0;
+}
+
+static int free_hash(struct lzs_state *state)
+{
+	int i;
+	for (i = 0; i < HASH_BUCKETS; i++) {
+		struct lzs_hash_entry *entry, *tmp;
+		list_for_each_entry_safe(entry, tmp, &state->hash[i], list) {
+			list_del(&entry->list);
+			free(entry);
+		}
+	}
+
+	return 0;
+}
+
+static int hash_key_calc(uint8_t *data)
+{
+	int key = 0x456789AB;
+
+	key = (key << 5) ^ (key >> 27) ^ data[0];
+	key = (key << 5) ^ (key >> 27) ^ data[1];
+
+	return key % HASH_BUCKETS;
+}
+
+static int hash_add(struct lzs_state *state, uint8_t *data)
+{
+	struct lzs_hash_entry *entry = malloc(sizeof(struct lzs_hash_entry));
+	if (entry == NULL) {
+		perror("hash_add_bytes(): malloc()");
+		return -1;
+	}
+
+	entry->pos = data;
+
+	list_add(&entry->list, &state->hash[hash_key_calc(data)]);
+	return 0;
+}
+
+static int getMatchLen(uint8_t *a, uint8_t *b, uint32_t maxlen)
+{
+	/* shortcut, first 2 bytes *must* match */
+	if ((a[0] ^ b[0]) | (a[1] ^ b[1]))
+		return 0;
+
+	a += 2;
+	b += 2;
+	maxlen -= 2;
+
+	int retval = 2;
+	while ((*a++ == *b++) && maxlen--)
+		retval++;
+
+	return retval;
+}
+
+int lzs_pack(uint8_t *srcbuf, int srcsize, uint8_t *dstbuf, int dstsize)
+{
+	struct lzs_state state = {
+		.srcblkstart = srcbuf,
+		.src = srcbuf,
+		.srcsize = srcsize,
+
+		.dstblkstart = dstbuf,
+		.dst = dstbuf,
+		.dstsize = dstsize,
+
+		.bitbuf = 0,
+		.bitcnt = 0,
+	};
+
+	alloc_hash(&state);
+
+	/* at least 2 bytes in input */
+	while (state.src +2 <= srcbuf + srcsize) {
+
+		/* new dst block: insert dummy header */
+		if (state.dstblkstart == state.dst)
+			state.dst += 4;
+
+		int key = hash_key_calc(state.src);
+		int maxlen = MIN(state.srcblkstart + 2048, srcbuf + srcsize) - state.src;
+
+		printf("searching for 0x%02x%02x abs=0x%04x key=0x%02x maxlen=%d\n",
+			state.src[0], state.src[1], state.src - srcbuf, key, maxlen);
+
+		int bestmatchlen = 0;
+		struct lzs_hash_entry *search, *tmp, *bestmatch = NULL;
+		list_for_each_entry_safe(search, tmp, &state.hash[key], list) {
+			/* hash entry too old, discard it */
+			if (search->pos + 2048 <= state.src) {
+				list_del(&search->list);
+				free(search);
+				continue;
+			}
+
+			/* get length of match (min. 2, 0 if collision) */
+			int matchlen = getMatchLen(search->pos, state.src, maxlen);
+			if (matchlen > bestmatchlen) {
+				bestmatchlen = matchlen;
+				bestmatch = search;
+			}
+		}
+
+		/* found something? */
+		if (bestmatch != NULL) {
+			put_compressed_string(&state, state.src - bestmatch->pos, bestmatchlen);
+			/* add bytes to history hash */
+			while (bestmatchlen--)
+				hash_add(&state, state.src++);
+
+		} else {
+			put_literal_byte(&state, *state.src);
+			hash_add(&state, state.src++);
+		}
+
+		/* block full? */
+		if (state.src - state.srcblkstart >= 2048) {
+			put_blockend(&state);
+
+			put_zyxel_header(&state);
+			state.srcblkstart = state.src;
+			state.dstblkstart = state.dst;
+		}
+	}
+
+	/* add remaining bytes (== last one) as literal */
+	if (state.src < srcbuf + srcsize)
+		put_literal_byte(&state, *state.src++);
+
+	put_blockend(&state);
+	put_zyxel_header(&state);
+
+	free_hash(&state);
+
+	printf("lzs_pack: packed %d (%d) bytes to %d (%d) bytes\n",
+		state.src - srcbuf, srcsize, state.dst - dstbuf, dstsize);
+
+	return state.dst - dstbuf;
+}

lzsc.h

@@ -0,0 +1,6 @@
+#ifndef _LZSC_H_
+#define _LZSC_H_
+
+int lzs_pack(void *src, int srcsize, void *dst, int dstsize);
+
+#endif /* _LZSC_H_ */

lzsd.c

@@ -0,0 +1,138 @@
+#include <stdio.h>
+#include <stdint.h>
+
+struct lzs_state {
+	uint8_t *srcblkstart;
+	uint8_t *src;
+	uint32_t srcsize;
+
+	uint8_t *dstblkstart;
+	uint8_t *dst;
+	uint32_t dstsize;
+
+	uint32_t bitbuf;
+	uint32_t bitcnt;
+};
+
+static uint32_t get_bits(struct lzs_state *state, int num)
+{
+	while (state->bitcnt < num) {
+		state->bitbuf = (state->bitbuf << 8) | *(state->src)++;
+		state->bitcnt += 8;
+	}
+
+	state->bitcnt -= num;
+	return (state->bitbuf >> state->bitcnt) & ((1 << num) -1);
+}
+
+static uint32_t get_len(struct lzs_state *state)
+{
+	uint32_t bits;
+	uint32_t length = 2;
+
+	do {
+		bits = get_bits(state, 2);
+		length += bits;
+	} while ((bits == 3) && (length < 8));
+
+	if (length == 8) {
+		do {
+			bits = get_bits(state, 4);
+			length += bits;
+		} while (bits == 15);
+	}
+
+	return length;
+}
+
+static int get_zyxel_header(struct lzs_state *state)
+{
+	uint8_t *p = state->srcblkstart;
+	uint32_t a = (p[0] << 24) | (p[1] << 16) | (p[2] << 8) | p[3];
+
+	uint32_t b = ((state->dst - state->dstblkstart) << 16) & 0xFFFF0000;
+	b |= ((state->src - state->srcblkstart) & 0xFFFF);
+
+	/* remove own header */
+	b -= 4;
+
+	printf("header of previous block is=0x%08x expected=0x%08x %s\n", a, b, ((a == b) ? "OK" : "ERROR"));
+	return 0;
+}
+
+/*
+ * TODO: check src/dst sizes
+ */
+uint32_t lzs_unpack(uint8_t *srcbuf, uint32_t srcsize, uint8_t *dstbuf, uint32_t dstsize)
+{
+	struct lzs_state state = {
+		.srcblkstart = srcbuf,
+		.src = srcbuf,
+		.srcsize = srcsize,
+
+		.dstblkstart = dstbuf,
+		.dst = dstbuf,
+		.dstsize = dstsize,
+
+		.bitbuf = 0,
+		.bitcnt = 0,
+	};
+
+	while (1) {
+		/* jump over header */
+		if (state.srcblkstart == state.src)
+			state.src += 4;
+
+		uint32_t tag = get_bits(&state, 1);
+
+		/* Uncompressed byte */
+		if (tag == 0) {
+			*(state.dst)++ = get_bits(&state, 8);
+			printf("uncompressed byte: 0x%02x\n", *(state.dst -1));
+
+		/* Compressed string */
+		} else {
+			/* read 7 or 11 bit offset */
+			tag = get_bits(&state, 1);
+			uint32_t offset = get_bits(&state, (tag == 1) ? 7 : 11);
+
+			/* end condition (7bit offset == 0x00) */
+			if (tag == 1 && offset == 0) {
+				/* align src to next byte */
+				if (state.bitcnt > 7)
+					printf("ERROR: alignment?\n");
+
+				state.bitcnt = 0;
+
+				printf("=== BLOCK END === \n");
+				get_zyxel_header(&state);
+				state.srcblkstart = state.src;
+				state.dstblkstart = state.dst;
+
+				/* all src bytes used? */
+				if (state.src >= srcbuf + srcsize)
+					break;
+
+				continue;
+			}
+
+			uint8_t *dict = state.dst - offset;
+			if (dict < dstbuf) {
+				printf("lzs_unpack: invalid dict: %p < %p (tag=%d, offset=0x%x)\n",
+					dict, dstbuf, tag, offset);
+				break;
+			}
+
+			uint32_t len = get_len(&state);
+			printf("compressed string, offset(%d)=0x%03x len=0x%04x\n", tag, offset, len);
+
+			while (len--)
+				*(state.dst)++ = *dict++;
+		}
+	}
+
+	printf("lzs_unpack: decompressed %d (%d) bytes to %d (%d) bytes\n",
+		(state.src - srcbuf), srcsize, (state.dst - dstbuf), dstsize);
+
+	return state.dst - dstbuf;
+}

lzsd.h

@@ -0,0 +1,6 @@
+#ifndef _LZSD_H_
+#define _LZSD_H_
+
+uint32_t lzs_unpack(void *src, uint32_t srcsize, void *dst, uint32_t dstsize);
+
+#endif /* _LZSD_H_ */

zyxel-revert.conf

@@ -1,5 +1,5 @@
 [global]
-configdata 350LI2C1.rom
+configdata 350LI2C1.rom.own
 
 [ports]
 serial /dev/ttyUSB0