From 59aab392521734b7d99a9fa7305b32ffc5707b89 Mon Sep 17 00:00:00 2001 From: Olaf Rempel Date: Sun, 29 Apr 2007 19:08:12 +0200 Subject: [PATCH] add lzs comp & decomp tools --- Makefile | 13 +- compress.c | 68 +++++++++++ decompress.c | 65 ++++++++++ filedata.c | 75 ++++++++++++ filedata.h | 13 ++ lzsc.c | 293 ++++++++++++++++++++++++++++++++++++++++++++++ lzsc.h | 6 + lzsd.c | 138 ++++++++++++++++++++++ lzsd.h | 6 + zyxel-revert.conf | 2 +- 10 files changed, 673 insertions(+), 6 deletions(-) create mode 100644 compress.c create mode 100644 decompress.c create mode 100644 filedata.c create mode 100644 filedata.h create mode 100644 lzsc.c create mode 100644 lzsc.h create mode 100644 lzsd.c create mode 100644 lzsd.h diff --git a/Makefile b/Makefile index ecf25e1..61c9d8c 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,14 @@ CFLAGS := -O2 -pipe -Wall -OBJS := configfile.o event.o logging.o -OBJS += context.o serial.o statemachine.o xmodem.o +all: zyxel-revert compress decompress -all: zyxel-revert +zyxel-revert: configfile.o event.o logging.o context.o serial.o statemachine.o xmodem.o zyxel-revert.o + $(CC) $(CFLAGS) $^ -o $@ -zyxel-revert: $(OBJS) zyxel-revert.o +compress: lzsc.o filedata.o compress.o + $(CC) $(CFLAGS) $^ -o $@ + +decompress: lzsd.o filedata.o decompress.o $(CC) $(CFLAGS) $^ -o $@ %.o: %.c @@ -15,7 +18,7 @@ zyxel-revert: $(OBJS) zyxel-revert.o $(CC) $(CFLAGS) -MM -c $< -o $@ clean: - rm -f zyxel-revert *.d *.o *.log + rm -f zyxel-revert compress decompress *.d *.o *.log DEPS := $(wildcard *.c) -include $(DEPS:.c=.d) diff --git a/compress.c b/compress.c new file mode 100644 index 0000000..717cccf --- /dev/null +++ b/compress.c @@ -0,0 +1,68 @@ +#include +#include +#include +#include +#include + +#include + +#include "lzsc.h" +#include "filedata.h" + +struct rom0file { + uint16_t version; + uint16_t size; + uint16_t offset; + + char name[14]; +} __attribute__((packed)); + +int parse_rom0(struct filedata *filedata, const char *infile) +{ + struct rom0file file; + + /* zweite page enthält config */ + uint32_t offset = 0x2000; + while (1) { + memcpy(&file, (void *)(filedata->data) + offset, sizeof(file)); + + if (strcmp(file.name, "autoexec.net") == 0) { + printf("found autoexec.net: 0x%04x - 0x%04x (%d bytes)\n", + 0x2000 + htons(file.offset), 0x2000 + htons(file.offset) + htons(file.size), htons(file.size)); + + /* 16 byte header */ + void *buf = (void *)(filedata->data) + 0x2000 + htons(file.offset) +12; + + /* little cleanup */ + memset(buf, 0, htons(file.size)); + + struct filedata *indata = get_filedata(infile); + int size = lzs_pack(indata->data, indata->size, buf, 0xC00); + + file.size = htons(size +12); + + memcpy((void *)(filedata->data) + offset, &file, sizeof(file)); + printf("new autoexec.net: 0x%04x - 0x%04x (%d bytes)\n", + 0x2000 + htons(file.offset), 0x2000 + htons(file.offset) + htons(file.size), htons(file.size)); + + put_filedata("350LI2C1.rom.own", filedata); + + free(indata); + return 0; + } + + if (file.name[0] == 0 || file.name[0] == -1) + return -1; + + offset += sizeof(file); + } +} + +int main(int argc, char *argv[]) +{ + struct filedata *rom0 = get_filedata("350LI2C1.rom"); + parse_rom0(rom0, "350LI2C1.rom.own_decomp"); + + free(rom0); + return 0; +} diff --git a/decompress.c b/decompress.c new file mode 100644 index 0000000..91f89c8 --- /dev/null +++ b/decompress.c @@ -0,0 +1,65 @@ +#include +#include +#include +#include +#include + +#include + +#include "lzsd.h" +#include "filedata.h" + +struct rom0file { + uint16_t version; + uint16_t size; + uint16_t offset; + + char name[14]; +} __attribute__((packed)); + +int parse_rom0(struct filedata *filedata, const char *outfile) +{ + struct rom0file file; + + /* zweite page enthält config */ + uint32_t offset = 0x2000; + while (1) { + memcpy(&file, (void *)(filedata->data) + offset, sizeof(file)); + + if (strcmp(file.name, "autoexec.net") == 0) { + printf("found autoexec.net: 0x%04x - 0x%04x (%d bytes)\n", + 0x2000 + htons(file.offset), 0x2000 + htons(file.offset) + htons(file.size), htons(file.size)); + + /* 64kb sollten reichen */ + struct filedata *out = alloc_filedata(65535); + + /* 16 byte header */ + void *buf = (void *)(filedata->data) + 0x2000 + htons(file.offset) +12; + + out->size = lzs_unpack(buf, htons(file.size) -12, out->data, out->size); + put_filedata(outfile, out); + free(out); + + return 0; + } + + if (file.name[0] == 0 || file.name[0] == -1) + return -1; + + offset += sizeof(file); + } +} + +int main(int argc, char *argv[]) +{ + struct filedata *in = get_filedata(argv[1]); + + char outname[64]; + strncpy(outname, argv[1], sizeof(outname)); + strcat(outname, ".own_decomp"); + + parse_rom0(in, outname); + + free(in); + return 0; +} diff --git a/filedata.c b/filedata.c new file mode 100644 index 0000000..385a0b4 --- /dev/null +++ b/filedata.c @@ -0,0 +1,75 @@ +#include +#include +#include + +#include +#include +#include + +#include "filedata.h" + +struct filedata * get_filedata(const char *filename) +{ + int fd = open(filename, O_RDONLY); + if (fd < 0) { + perror("get_filedata(): open()"); + return NULL; + } + + struct stat filestat; + if (fstat(fd, &filestat) < 0) { + perror("get_filedata(): fstat()"); + close(fd); + return NULL; + } + + struct filedata *filedata = malloc(sizeof(struct filedata) + filestat.st_size); + if (filedata == NULL) { + perror("get_filedata(): malloc()"); + close(fd); + return NULL; + } + + filedata->size = filestat.st_size; + + int readsize = read(fd, filedata->data, filedata->size); + if (readsize != filedata->size) { + perror("get_filedata(): read()"); + free(filedata); + close(fd); + return NULL; + } + + close(fd); + return filedata; +} + +struct filedata * alloc_filedata(int size) +{ + struct filedata *retval = malloc(sizeof(struct filedata) + size); + if (retval == NULL) { + perror("alloc_filedata(): malloc()"); + return NULL; + } + retval->size = size; + return retval; +} + +int put_filedata(const char *filename, struct filedata *filedata) +{ + int fd = open(filename, O_RDWR | O_CREAT | O_TRUNC, 0644); + if (fd < 0) { + perror("put_filedata(): open()"); + return -1; + } + + int writesize = write(fd, filedata->data, filedata->size); + if (writesize != filedata->size) { + perror("put_filedata(): write()"); + close(fd); + return -1; + } + + close(fd); + return 0; +} diff --git a/filedata.h b/filedata.h new file mode 100644 index 0000000..2774be4 --- /dev/null +++ b/filedata.h @@ -0,0 +1,13 @@ +#ifndef _FILEDATA_H_ +#define _FILEDATA_H_ + +struct filedata { + int size; + void *data[0]; +}; + +struct filedata * get_filedata(const char *filename); +struct filedata * alloc_filedata(int size); +int put_filedata(const char *filename, struct filedata *filedata); + +#endif /* _FILEDATA_H_ */ diff --git a/lzsc.c b/lzsc.c new file mode 100644 index 0000000..df28bc5 --- /dev/null +++ b/lzsc.c @@ -0,0 +1,293 @@ +#include +#include +#include +#include +#include + +#include "list.h" + +#define HASH_BUCKETS 127 + +#define MIN(a,b) ((a) < (b) ? (a) : (b)) + +struct lzs_state { + uint8_t *srcblkstart; + uint8_t *src; + uint32_t srcsize; + + uint8_t *dstblkstart; + uint8_t *dst; + uint32_t dstsize; + + uint32_t bitbuf; + uint32_t bitcnt; + + struct list_head *hash; +}; + +struct lzs_hash_entry { + struct list_head list; + uint8_t *pos; +}; + +/* TODO: check dstsize */ +static int put_bits(struct lzs_state *state, uint32_t bits, uint32_t len) +{ + state->bitbuf <<= len; + state->bitbuf |= bits; + state->bitcnt += len; + + while (state->bitcnt >= 8) { + state->bitcnt -= 8; + *(state->dst)++ = (state->bitbuf >> (state->bitcnt)) & 0xFF; +// printf(" wrote byte: 0x%02x\n", *(state->dst -1)); + } + + return 0; +} + +/* TODO: check dstsize */ +static int put_literal_byte(struct lzs_state *state, uint8_t byte) +{ + printf(" put_literal_byte: 0x%02x\n", byte); + return put_bits(state, (0 << 8) | byte, 1+8); +} + +/* TODO: check dstsize */ +static int put_compressed_string(struct lzs_state *state, uint32_t offset, uint32_t len) +{ + printf(" put_compressed_string: offset=0x%03x len=0x%03x\n", offset, len); + + if (offset > 0x7ff || len > 0x800) + printf(" ERROR\n"); + + if (offset < 128) + put_bits(state, (1 << 8) | (1 << 7) | offset, 1+1+7); + else + put_bits(state, (1 << 12) | (0 << 11) | offset, 1+1+11); + + if (len <= 4) { + /* + * 00 - 2 + * 01 - 3 + * 10 - 4 + */ + put_bits(state, len - 2, 2); + + } else if (len < 8) { + /* + * 1100 - 5 + * 1101 - 6 + * 1110 - 7 + */ + put_bits(state, len + 7, 4); + + } else if (len >= 8) { + /* + * 1111 0000 - 8 + * 1111 0001 - 9 + * ... + * 1111 1110 - 22 + * 1111 1111 0000 - 23 + * 1111 1111 0001 - 24 + * ... + */ + len -= 8; + put_bits(state, 15, 4); + while (len >= 15) { + put_bits(state, 15, 4); + len -= 15; + } + put_bits(state, len, 4); + } + return 0; +} + +/* TODO: check dstsize */ +static int put_blockend(struct lzs_state *state) +{ + /* 7bit offset = 0 -> end code */ + put_bits(state, (1 << 8) | (1 << 7) | 0, 1+1+7); + + /* align to bytes */ + if (state->bitcnt) + put_bits(state, 0, 8 - state->bitcnt); + + printf(" =============== BLOCK END =============== \n"); + return 0; +} + +static int put_zyxel_header(struct lzs_state *state) +{ + uint16_t len = state->src - state->srcblkstart; + uint16_t lenc = state->dst - state->dstblkstart; + + /* remove own header size */ + lenc -= 4; + + printf("header of previous block: 0x%04x%04x\n", len, lenc); + + uint8_t *p = state->dstblkstart; + p[0] = (len >> 8) & 0xFF; + p[1] = len & 0xFF; + + p[2] = (lenc >> 8) & 0xFF; + p[3] = lenc & 0xFF; + + return 0; +} + +static int alloc_hash(struct lzs_state *state) +{ + state->hash = malloc(sizeof(struct lzs_hash_entry) * HASH_BUCKETS); + if (state->hash == NULL) { + perror("alloc_hashtable(): malloc()"); + return -1; + } + + int i; + for (i = 0; i < HASH_BUCKETS; i++) + INIT_LIST_HEAD(&state->hash[i]); + + return 0; +} + +static int free_hash(struct lzs_state *state) +{ + int i; + for (i = 0; i < HASH_BUCKETS; i++) { + struct lzs_hash_entry *entry, *tmp; + list_for_each_entry_safe(entry, tmp, &state->hash[i], list) { + list_del(&entry->list); + free(entry); + } + } + + return 0; +} + +static int hash_key_calc(uint8_t *data) +{ + int key = 0x456789AB; + + key = (key << 5) ^ (key >> 27) ^ data[0]; + key = (key << 5) ^ (key >> 27) ^ data[1]; + + return key % HASH_BUCKETS; +} + +static int hash_add(struct lzs_state *state, uint8_t *data) +{ + struct lzs_hash_entry *entry = malloc(sizeof(struct lzs_hash_entry)); + if (entry == NULL) { + perror("hash_add_bytes(): malloc()"); + return -1; + } + + entry->pos = data; + + list_add(&entry->list, &state->hash[hash_key_calc(data)]); + return 0; +} + +static int getMatchLen(uint8_t *a, uint8_t *b, uint32_t maxlen) +{ + /* shortcut, first 2 bytes *must* match */ + if ((a[0] ^ b[0]) | (a[1] ^ b[1])) + return 0; + + a += 2; + b += 2; + maxlen -= 2; + + int retval = 2; + while ((*a++ == *b++) && maxlen--) + retval++; + + return retval; +} + +int lzs_pack(uint8_t *srcbuf, int srcsize, uint8_t *dstbuf, int dstsize) +{ + struct lzs_state state = { + .srcblkstart = srcbuf, + .src = srcbuf, + .srcsize = srcsize, + + .dstblkstart = dstbuf, + .dst = dstbuf, + .dstsize = dstsize, + + .bitbuf = 0, + .bitcnt = 0, + }; + + alloc_hash(&state); + + /* at least 2 bytes in input */ + while (state.src +2 <= srcbuf + srcsize) { + + /* new dst block: insert dummy header */ + if (state.dstblkstart == state.dst) + state.dst += 4; + + int key = hash_key_calc(state.src); + int maxlen = MIN(state.srcblkstart + 2048, srcbuf + srcsize) - state.src; + + printf("searching for 0x%02x%02x abs=0x%04x key=0x%02x maxlen=%d\n", + state.src[0], state.src[1], state.src - srcbuf, key, maxlen); + + int bestmatchlen = 0; + struct lzs_hash_entry *search, *tmp, *bestmatch = NULL; + list_for_each_entry_safe(search, tmp, &state.hash[key], list) { + /* hash entry too old, discard it */ + if (search->pos + 2048 <= state.src) { + list_del(&search->list); + free(search); + continue; + } + + /* get length of match (min. 2, 0 if collision) */ + int matchlen = getMatchLen(search->pos, state.src, maxlen); + if (matchlen > bestmatchlen) { + bestmatchlen = matchlen; + bestmatch = search; + } + } + + /* found something? */ + if (bestmatch != NULL) { + put_compressed_string(&state, state.src - bestmatch->pos, bestmatchlen); + /* add bytes to history hash */ + while (bestmatchlen--) + hash_add(&state, state.src++); + + } else { + put_literal_byte(&state, *state.src); + hash_add(&state, state.src++); + } + + /* block full? */ + if (state.src - state.srcblkstart >= 2048) { + put_blockend(&state); + + put_zyxel_header(&state); + state.srcblkstart = state.src; + state.dstblkstart = state.dst; + } + } + + /* add remaining bytes (== last one) as literal */ + if (state.src < srcbuf + srcsize) + put_literal_byte(&state, *state.src++); + + put_blockend(&state); + put_zyxel_header(&state); + + free_hash(&state); + + printf("lzs_pack: packed %d (%d) bytes to %d (%d) bytes\n", + state.src - srcbuf, srcsize, state.dst - dstbuf, dstsize); + + return state.dst - dstbuf; +} diff --git a/lzsc.h b/lzsc.h new file mode 100644 index 0000000..d6607f2 --- /dev/null +++ b/lzsc.h @@ -0,0 +1,6 @@ +#ifndef _LZSC_H_ +#define _LZSC_H_ + +int lzs_pack(void *src, int srcsize, void *dst, int dstsize); + +#endif /* _LZSC_H_ */ diff --git a/lzsd.c b/lzsd.c new file mode 100644 index 0000000..ea98997 --- /dev/null +++ b/lzsd.c @@ -0,0 +1,138 @@ +#include +#include + +struct lzs_state { + uint8_t *srcblkstart; + uint8_t *src; + uint32_t srcsize; + + uint8_t *dstblkstart; + uint8_t *dst; + uint32_t dstsize; + + uint32_t bitbuf; + uint32_t bitcnt; +}; + +static uint32_t get_bits(struct lzs_state *state, int num) +{ + while (state->bitcnt < num) { + state->bitbuf = (state->bitbuf << 8) | *(state->src)++; + state->bitcnt += 8; + } + + state->bitcnt -= num; + return (state->bitbuf >> state->bitcnt) & ((1 << num) -1); +} + +static uint32_t get_len(struct lzs_state *state) +{ + uint32_t bits; + uint32_t length = 2; + + do { + bits = get_bits(state, 2); + length += bits; + } while ((bits == 3) && (length < 8)); + + if (length == 8) { + do { + bits = get_bits(state, 4); + length += bits; + } while (bits == 15); + } + + return length; +} + +static int get_zyxel_header(struct lzs_state *state) +{ + uint8_t *p = state->srcblkstart; + uint32_t a = (p[0] << 24) | (p[1] << 16) | (p[2] << 8) | p[3]; + + uint32_t b = ((state->dst - state->dstblkstart) << 16) & 0xFFFF0000; + b |= ((state->src - state->srcblkstart) & 0xFFFF); + + /* remove own header */ + b -= 4; + + printf("header of previous block is=0x%08x expected=0x%08x %s\n", a, b, ((a == b) ? "OK" : "ERROR")); + return 0; +} + +/* + * TODO: check src/dst sizes + */ +uint32_t lzs_unpack(uint8_t *srcbuf, uint32_t srcsize, uint8_t *dstbuf, uint32_t dstsize) +{ + struct lzs_state state = { + .srcblkstart = srcbuf, + .src = srcbuf, + .srcsize = srcsize, + + .dstblkstart = dstbuf, + .dst = dstbuf, + .dstsize = dstsize, + + .bitbuf = 0, + .bitcnt = 0, + }; + + while (1) { + /* jump over header */ + if (state.srcblkstart == state.src) + state.src += 4; + + uint32_t tag = get_bits(&state, 1); + + /* Uncompressed byte */ + if (tag == 0) { + *(state.dst)++ = get_bits(&state, 8); + printf("uncompressed byte: 0x%02x\n", *(state.dst -1)); + + /* Compressed string */ + } else { + /* read 7 or 11 bit offset */ + tag = get_bits(&state, 1); + uint32_t offset = get_bits(&state, (tag == 1) ? 7 : 11); + + /* end condition (7bit offset == 0x00) */ + if (tag == 1 && offset == 0) { + /* align src to next byte */ + if (state.bitcnt > 7) + printf("ERROR: alignment?\n"); + + state.bitcnt = 0; + + printf("=== BLOCK END === \n"); + get_zyxel_header(&state); + state.srcblkstart = state.src; + state.dstblkstart = state.dst; + + /* all src bytes used? */ + if (state.src >= srcbuf + srcsize) + break; + + continue; + } + + uint8_t *dict = state.dst - offset; + if (dict < dstbuf) { + printf("lzs_unpack: invalid dict: %p < %p (tag=%d, offset=0x%x)\n", + dict, dstbuf, tag, offset); + break; + } + + uint32_t len = get_len(&state); + printf("compressed string, offset(%d)=0x%03x len=0x%04x\n", tag, offset, len); + + while (len--) + *(state.dst)++ = *dict++; + } + } + + printf("lzs_unpack: decompressed %d (%d) bytes to %d (%d) bytes\n", + (state.src - srcbuf), srcsize, (state.dst - dstbuf), dstsize); + + return state.dst - dstbuf; +} diff --git a/lzsd.h b/lzsd.h new file mode 100644 index 0000000..9295a42 --- /dev/null +++ b/lzsd.h @@ -0,0 +1,6 @@ +#ifndef _LZSD_H_ +#define _LZSD_H_ + +uint32_t lzs_unpack(void *src, uint32_t srcsize, void *dst, uint32_t dstsize); + +#endif /* _LZSD_H_ */ diff --git a/zyxel-revert.conf b/zyxel-revert.conf index 7f584ad..d1cadd4 100644 --- a/zyxel-revert.conf +++ b/zyxel-revert.conf @@ -1,5 +1,5 @@ [global] -configdata 350LI2C1.rom +configdata 350LI2C1.rom.own [ports] serial /dev/ttyUSB0