Allow GZIP compressed flash updates

Modified the bootloader to be able to take stored updates in compressed
GZIP format (i.e. the output of "gzip -9 xxx.bin") and decompress them
on-the-fly to their final destination.  This can work for apps and for
filesystems (when used with the 2-step update option).

Allow eboot to be built using -Os/2 optimizations by fixing some portions
which failed when any optimizations were used.  Add -Wall and use data
and function sections to reduce size.  Use -Os to minimize size.

Remove obsolete esptool-ck calls to build a .ROM image, we don't use it.

Move all uninitted variables to RAM from IRAM, allowing 8-bit access.

Hook in @d-a-v and @pfalcon's uzlib port to actually do the
decompression.  Do not use any CRC checking which saves space.  Since we
have overwritten all of flash by the time we know id the CRC matches,
there's nothing we could have done anyway.

Adjust the Updater class to support GZIP files and not attempt to patch
them.

Bootloader builds to 0xd90 out of 0xfff bytes.

.gitmodules

@@ -19,3 +19,6 @@
 [submodule "tools/esptool"]
 	path = tools/esptool
 	url = https://github.com/espressif/esptool.git
+[submodule "tools/sdk/uzlib"]
+	path = tools/sdk/uzlib
+	url = https://github.com/earlephilhower/uzlib.git

bootloaders/eboot/Makefile

@@ -6,44 +6,51 @@ TARGET_DIR := ./
 
 TARGET_OBJ_FILES := \
 	eboot.o \
-	eboot_command.o \
-
+	eboot_command.o
 
 TARGET_OBJ_PATHS := $(addprefix $(TARGET_DIR)/,$(TARGET_OBJ_FILES))
 
+UZLIB_PATH := ../../tools/sdk/uzlib/src
+UZLIB_FLAGS := -DRUNTIME_BITS_TABLES
+
 CC := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-gcc
 CXX := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-g++
 AR := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-ar
 LD := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-gcc
 OBJDUMP := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-objdump
 
-INC += -I../../tools/sdk/include
+INC += -I../../tools/sdk/include -I../../tools/sdk/uzlib/src
+
 CFLAGS += -std=gnu99
 
-CFLAGS += -O0 -g -Wpointer-arith -Wno-implicit-function-declaration -Wl,-EL -fno-inline-functions -nostdlib -mlongcalls -mno-text-section-literals
+CFLAGS += -Os -g -Wall -Wpointer-arith -Wno-implicit-function-declaration -Wl,-EL -fno-inline-functions -nostdlib -mlongcalls -mno-text-section-literals -ffunction-sections -fdata-sections
 
 CFLAGS += $(INC)
 
-LDFLAGS	+= -nostdlib -Wl,--no-check-sections -umain
+CFLAGS += $(UZLIB_FLAGS)
+
+LDFLAGS	+= -nostdlib -Wl,--no-check-sections -Wl,--gc-sections -umain
 
 LD_SCRIPT := -Teboot.ld
 
 APP_OUT:= eboot.elf
 APP_AR := eboot.a
 APP_FW := eboot.bin
 
-all: $(APP_FW)
 
-$(APP_AR): $(TARGET_OBJ_PATHS)
-	$(AR) cru $@ $^
+all: $(APP_OUT)
 
+tinflate.o: $(UZLIB_PATH)/tinflate.c $(UZLIB_PATH)/uzlib.h $(UZLIB_PATH)/uzlib_conf.h
+	$(CC) $(CFLAGS) -c -o tinflate.o $(UZLIB_PATH)/tinflate.c
 
-$(APP_OUT): $(APP_AR)
-	$(LD) $(LD_SCRIPT) $(LDFLAGS) -Wl,--start-group -Wl,--whole-archive $(APP_AR) -Wl,--end-group -o $@
+tinfgzip.o: $(UZLIB_PATH)/tinfgzip.c $(UZLIB_PATH)/uzlib.h $(UZLIB_PATH)/uzlib_conf.h
+	$(CC) $(CFLAGS) -c -o tinfgzip.o $(UZLIB_PATH)/tinfgzip.c
 
-$(APP_FW): $(APP_OUT)
-	$(ESPTOOL) -vvv -eo $(APP_OUT) -bo $@ -bs .text -bs .data -bs .rodata -bc -ec || true
+$(APP_AR): $(TARGET_OBJ_PATHS) tinflate.o tinfgzip.o
+	$(AR) cru $@ $^
 
+$(APP_OUT): $(APP_AR)
+	$(LD) $(LD_SCRIPT) $(LDFLAGS) -Wl,--start-group -Wl,--whole-archive $(APP_AR) -Wl,--end-group -o $@
 
 clean:
 	rm -f *.o

bootloaders/eboot/eboot.c

@@ -12,6 +12,9 @@
 #include <string.h>
 #include "flash.h"
 #include "eboot_command.h"
+#include <uzlib.h>
+
+extern unsigned char _gzip_dict;
 
 #define SWRST do { (*((volatile uint32_t*) 0x60000700)) |= 0x80000000; } while(0);
 
@@ -24,10 +27,14 @@ int print_version(const uint32_t flash_addr)
     if (SPIRead(flash_addr + APP_START_OFFSET + sizeof(image_header_t) + sizeof(section_header_t), &ver, sizeof(ver))) {
         return 1;
     }
-    const char* __attribute__ ((aligned (4))) fmtt = "v%08x\n\0\0";
-    uint32_t fmt[2];
-    fmt[0] = ((uint32_t*) fmtt)[0];
-    fmt[1] = ((uint32_t*) fmtt)[1];
+    char fmt[16];
+    fmt[0] = 'v';
+    fmt[1] = '%';
+    fmt[2] = '0';
+    fmt[3] = '8';
+    fmt[4] = 'x';
+    fmt[5] = '\n';
+    fmt[6] = '0';
     ets_printf((const char*) fmt, ver);
     return 0;
 }
@@ -80,37 +87,96 @@ int load_app_from_flash_raw(const uint32_t flash_addr)
         pos += section_header.size;
     }
 
-    register uint32_t sp asm("a1") = 0x3ffffff0;
-    register uint32_t pc asm("a3") = image_header.entry;
-    __asm__  __volatile__ ("jx a3");
+    asm volatile("" ::: "memory");
+    asm volatile ("mov.n a1, %0\n"
+        "mov.n a3, %1\n"
+        "jx a3\n" : : "r" (0x3ffffff0), "r" (image_header.entry) );
 
+    __builtin_unreachable(); // Save a few bytes by letting GCC know no need to pop regs/return
     return 0;
 }
 
+uint8_t read_flash_byte(const uint32_t addr)
+{
+    uint8_t __attribute__((aligned(4))) buff[4];
+    SPIRead(addr & ~3, buff, 4);
+    return buff[addr & 3];
+}
+unsigned char __attribute__((aligned(4))) uzlib_flash_read_cb_buff[4096];
+uint32_t uzlib_flash_read_cb_addr;
+int uzlib_flash_read_cb(struct uzlib_uncomp *m)
+{
+    m->source = uzlib_flash_read_cb_buff;
+    m->source_limit = uzlib_flash_read_cb_buff + sizeof(uzlib_flash_read_cb_buff);
+    SPIRead(uzlib_flash_read_cb_addr, uzlib_flash_read_cb_buff, sizeof(uzlib_flash_read_cb_buff));
+    uzlib_flash_read_cb_addr += sizeof(uzlib_flash_read_cb_buff);
+    return *(m->source++);
+}
 
+unsigned char gzip_dict[32768];
 
 int copy_raw(const uint32_t src_addr,
              const uint32_t dst_addr,
              const uint32_t size)
 {
     // require regions to be aligned
-    if (src_addr & 0xfff != 0 ||
-        dst_addr & 0xfff != 0) {
+    if ((src_addr & 0xfff) != 0 ||
+        (dst_addr & 0xfff) != 0) {
         return 1;
     }
 
     const uint32_t buffer_size = FLASH_SECTOR_SIZE;
     uint8_t buffer[buffer_size];
-    uint32_t left = ((size+buffer_size-1) & ~(buffer_size-1));
+    int32_t left = ((size+buffer_size-1) & ~(buffer_size-1));
     uint32_t saddr = src_addr;
     uint32_t daddr = dst_addr;
-
-    while (left) {
+    struct uzlib_uncomp m_uncomp;
+    bool gzip = false;
+
+    // Check if we are uncompressing a GZIP upload or not
+    if ((read_flash_byte(saddr) == 0x1f) && (read_flash_byte(saddr + 1) == 0x8b)) {
+        // GZIP signature matched.  Find real size as encoded at the end
+        left = read_flash_byte(saddr + size - 4);
+        left += read_flash_byte(saddr + size - 3)<<8;
+        left += read_flash_byte(saddr + size - 2)<<16;
+        left += read_flash_byte(saddr + size - 1)<<24;
+
+        uzlib_init();
+
+        /* all 3 fields below must be initialized by user */
+        m_uncomp.source = NULL;
+        m_uncomp.source_limit = NULL;
+        uzlib_flash_read_cb_addr = src_addr;
+        m_uncomp.source_read_cb = uzlib_flash_read_cb;
+        uzlib_uncompress_init(&m_uncomp, gzip_dict, sizeof(gzip_dict));
+
+        int res = uzlib_gzip_parse_header(&m_uncomp);
+        if (res != TINF_OK) {
+            return 5; // Error uncompress header read
+        }
+	gzip = true;
+    }
+    while (left > 0) {
         if (SPIEraseSector(daddr/buffer_size)) {
             return 2;
         }
-        if (SPIRead(saddr, buffer, buffer_size)) {
-            return 3;
+        if (!gzip) {
+            if (SPIRead(saddr, buffer, buffer_size)) {
+                return 3;
+            }
+        } else {
+            m_uncomp.dest_start = buffer;
+            m_uncomp.dest = buffer;
+            int to_read = (left > buffer_size) ? buffer_size : left;
+            m_uncomp.dest_limit = buffer + to_read;
+            int res = uzlib_uncompress(&m_uncomp);
+            if ((res != TINF_DONE) && (res != TINF_OK)) {
+                return 6;
+            }
+            // Fill any remaining with 0xff
+            for (int i = to_read; i < buffer_size; i++) {
+                buffer[i] = 0xff;
+            }
         }
         if (SPIWrite(daddr, buffer, buffer_size)) {
             return 4;
@@ -124,12 +190,11 @@ int copy_raw(const uint32_t src_addr,
 }
 
 
-
-void main()
+int main()
 {
     int res = 9;
     struct eboot_command cmd;
-    
+
     print_version(0);
 
     if (eboot_command_read(&cmd) == 0) {
@@ -167,4 +232,7 @@ void main()
     }
 
     while(true){}
+
+    __builtin_unreachable();
+    return 0;
 }

bootloaders/eboot/eboot.ld

@@ -68,6 +68,8 @@ SECTIONS
 
   .data : ALIGN(4)
   {
+    *(COMMON)  /* Global vars */
+    . = ALIGN(4);
     _heap_start = ABSOLUTE(.);
 /*    _stack_sentry = ALIGN(0x8); */
   } >dram0_0_seg :dram0_0_bss_phdr
@@ -150,7 +152,6 @@ SECTIONS
     *(.bss)
     *(.bss.*)
     *(.gnu.linkonce.b.*)
-    *(COMMON)
     . = ALIGN (8);
     _bss_end = ABSOLUTE(.);
   } >iram1_0_seg :iram1_0_phdr

bootloaders/eboot/eboot_command.c

@@ -37,7 +37,7 @@ int eboot_command_read(struct eboot_command* cmd)
     }
 
     uint32_t crc32 = eboot_command_calculate_crc32(cmd);
-    if (cmd->magic & EBOOT_MAGIC_MASK != EBOOT_MAGIC || 
+    if ((cmd->magic & EBOOT_MAGIC_MASK) != EBOOT_MAGIC ||
         cmd->crc32 != crc32) {
         return 1;
     }

cores/esp8266/Updater.cpp

@@ -329,7 +329,8 @@ bool UpdaterClass::_writeBuffer(){
   bool modifyFlashMode = false;
   FlashMode_t flashMode = FM_QIO;
   FlashMode_t bufferFlashMode = FM_QIO;
-  if (_currentAddress == _startAddress + FLASH_MODE_PAGE) {
+  //TODO - GZIP can't do this
+  if ((_currentAddress == _startAddress + FLASH_MODE_PAGE) && (_buffer[0] != 0x1f)) {
     flashMode = ESP.getFlashChipMode();
     #ifdef DEBUG_UPDATER
       DEBUG_UPDATER.printf_P(PSTR("Header: 0x%1X %1X %1X %1X\n"), _buffer[0], _buffer[1], _buffer[2], _buffer[3]);
@@ -411,7 +412,7 @@ size_t UpdaterClass::write(uint8_t *data, size_t len) {
 bool UpdaterClass::_verifyHeader(uint8_t data) {
     if(_command == U_FLASH) {
         // check for valid first magic byte (is always 0xE9)
-        if(data != 0xE9) {
+        if ((data != 0xE9) && (data != 0x1f)) {
             _currentAddress = (_startAddress + _size);
             _setError(UPDATE_ERROR_MAGIC_BYTE);
             return false;
@@ -435,7 +436,12 @@ bool UpdaterClass::_verifyEnd() {
         }
 
         // check for valid first magic byte
-        if(buf[0] != 0xE9) {
+	//
+	// TODO: GZIP compresses the chipsize flags, so can't do check here
+	if ((buf[0] == 0x1f) && (buf[1] == 0x8b)) {
+            // GZIP, just assume OK
+            return true;
+        } else if (buf[0] != 0xE9) {
             _currentAddress = (_startAddress);
             _setError(UPDATE_ERROR_MAGIC_BYTE);            
             return false;