blob: 70ffdbecf3747d3df2a263396091a0c3729dba97 [file] [log] [blame]
/*
* Copyright (c) 2018 MediaTek Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files
* (the "Software"), to deal in the Software without restriction,
* including without limitation the rights to use, copy, modify, merge,
* publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include <aee.h>
#include <assert.h>
#include <kdump.h>
#include <lib/mempool.h>
#include <lib/zlib.h>
#include <lib/zutil.h>
#include <malloc.h>
#include <platform.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include "lib/cksum.h"
#ifdef MTK_3LEVEL_PAGETABLE
#include <stdlib.h>
#include <err.h>
#endif
#define KZIP_DEBUG(x...)
#define KDUMP_SIZE_REPORT 0x1000000
#define KDUMP_TICK_WDT 0x10000
#ifdef MRDUMP_USB_DUMP_NO_COMPRESSION
#define KDUMP_ZLIB_LEVEL Z_NO_COMPRESSION
#else
#define KDUMP_ZLIB_LEVEL Z_BEST_SPEED
#endif
#define ZIPVERSION_NEEDED 45UL
static void putvalue(void *dest, uint64_t x, int nbByte)
{
uint8_t *buf = (uint8_t *)dest;
int n;
for (n = 0; n < nbByte; n++) {
buf[n] = (unsigned char)(x & 0xff);
x >>= 8;
}
#if 0
if (x != 0) {
/* data overflow - hack for ZIP64 */
for (n = 0; n < nbByte; n++) {
buf[n] = 0xff;
}
}
#endif
}
static int put_zip64_eoc_directory_record(uint8_t *buf, int record_num, uint64_t zip_centralheader_offset, int zip_centralheader_size)
{
uint8_t *oldbuf = buf;
putvalue(buf, ZIP64ENDOFCENTRALDIRMAGIC, 4);
buf += 4;
putvalue(buf, 44, 8); /* zip64 eoc size */
buf += 8;
putvalue(buf, ZIPVERSION_NEEDED, 2); /* version made by */
buf += 2;
putvalue(buf, ZIPVERSION_NEEDED, 2); /* version needed to extract */
buf += 2;
putvalue(buf, 0, 4); /* number of this disk */
buf += 4;
putvalue(buf, 0, 4); /* number of the disk with the start of the central directory */
buf += 4;
putvalue(buf, record_num, 8); /* total number of entries in the central directory on this disk */
buf += 8;
putvalue(buf, record_num, 8); /* total number of entries in the central directory */
buf += 8;
putvalue(buf, zip_centralheader_size, 8); /* size of the central directory */
buf += 8;
putvalue(buf, zip_centralheader_offset, 8); /* offset of start of central directory with respect to the starting disk number */
buf += 8;
return buf - oldbuf;
}
static int put_zip64_eoc_directory_locator(uint8_t *buf, uint64_t zip64_eoc_offset)
{
uint8_t *oldbuf = buf;
putvalue(buf, ZIP64ENDOFCENTRALDIRLOCATORMAGIC, 4);
buf += 4;
putvalue(buf, 0, 4); /* number of the disk with the start of the zip64 end of central directory */
buf += 4;
putvalue(buf, zip64_eoc_offset, 8); /* relative offset of the zip64 end of central directory record */
buf += 8;
putvalue(buf, 1, 4); /* total number of disks */
buf += 4;
return buf - oldbuf;
}
static int put_eoc_directory_record(uint8_t *buf, int record_num, uint64_t zip_centralheader_offset, int zip_centralheader_size)
{
uint8_t *oldbuf = buf;
putvalue(buf, ENDOFCENTRALDIRMAGIC, 4);
buf += 4;
putvalue(buf, 0, 2); /* Number of this disk */
buf += 2;
putvalue(buf, 0, 2); /* Disk where central directory starts */
buf += 2;
putvalue(buf, record_num, 2); /* Number of central directory records on this disk */
buf += 2;
putvalue(buf, record_num, 2); /* Total number of central directory records */
buf += 2;
putvalue(buf, zip_centralheader_size, 4);
buf += 4;
putvalue(buf, 0xffffffff, 4);
buf += 4;
putvalue(buf, 0, 2); /* Comment length (n) */
buf += 2;
return buf - oldbuf;
}
static int put_localheader(uint8_t *buf, const char *filename, int level)
{
uint8_t *oldbuf = buf;
putvalue(buf, LOCALHEADERMAGIC, 4);
buf += 4;
putvalue(buf, ZIPVERSION_NEEDED, 2);
buf += 2;
uint16_t flag = 0;
if ((level==8) || (level == 9))
flag |= 2;
if (level == 2)
flag |= 4;
if (level == 1)
flag |= 6;
#if 0
if (password != NULL)
flag |= 1;
#endif
putvalue(buf, flag | 0x8, 2);
buf += 2;
putvalue(buf, Z_DEFLATED, 2);
buf += 2;
putvalue(buf,0UL, 4);
buf += 4;
// CRC / Compressed size / Uncompressed size will be filled in later and rewritten later
putvalue(buf, 0UL, 4); /* crc 32, unknown */
buf += 4;
putvalue(buf, 0xFFFFFFFFUL, 4); /* compressed size, unknown */
buf += 4;
putvalue(buf, 0xFFFFFFFFUL, 4); /* uncompressed size, unknown */
buf += 4;
putvalue(buf, strlen(filename), 2); /* size of filename */
buf += 2;
putvalue(buf, 4 + 8 + 8, 2); /* size of extra field */
buf += 2;
memcpy(buf, filename, strlen(filename));
buf += strlen(filename);
/* ZIP64 extension */
putvalue(buf, 0x0001, 2); /* ZIP64_EXTENSION */
buf += 2;
putvalue(buf, 8 + 8, 2);
buf += 2;
putvalue(buf, 0UL, 8);
buf += 8;
putvalue(buf, 0UL, 8);
buf += 8;
return buf - oldbuf;
}
static int put_centralheader(uint8_t *buf, const char *filename, int level, uint64_t zip_localheader_offset, uint64_t size, uint64_t uncomp_size, uint32_t crc32_value)
{
uint8_t *oldbuf = buf;
putvalue(buf, CENTRALHEADERMAGIC, 4);
buf += 4;
putvalue(buf, ZIPVERSION_NEEDED, 2);
buf += 2;
putvalue(buf, ZIPVERSION_NEEDED, 2);
buf += 2;
uint16_t flag = 0;
if ((level==8) || (level == 9))
flag |= 2;
if (level == 2)
flag |= 4;
if (level == 1)
flag |= 6;
#if 0
if (password != NULL)
flag |= 1;
#endif
putvalue(buf, flag | 0x8, 2);
buf += 2;
putvalue(buf, Z_DEFLATED, 2);
buf += 2;
putvalue(buf,0UL, 4);
buf += 4;
// CRC / Compressed size / Uncompressed size will be filled in later and rewritten later
putvalue(buf, crc32_value, 4); /* crc 32 */
buf += 4;
putvalue(buf, 0xffffffffUL, 4); /* compressed size */
buf += 4;
putvalue(buf, 0xffffffffUL, 4); /* uncompressed size */
buf += 4;
putvalue(buf, strlen(filename), 2);
buf += 2;
putvalue(buf, 4 + 8 + 8 + 8, 2); /* size of extra field */
buf += 2;
putvalue(buf, 0UL, 2); /* size of comment field */
buf += 2;
putvalue(buf, 0UL, 2); /* disk number */
buf += 2;
putvalue(buf, 0UL, 2); /* internal attributes */
buf += 2;
putvalue(buf, 0UL, 4); /* external file attributes */
buf += 4;
putvalue(buf, 0xffffffffUL, 4); /* Relative offset */
buf += 4;
memcpy(buf, filename, strlen(filename));
buf += strlen(filename);
/* ZIP64 extension */
putvalue(buf, 0x0001, 2); /* ZIP64_EXTENSION */
buf += 2;
putvalue(buf, 8 + 8 + 8, 2);
buf += 2;
putvalue(buf, uncomp_size, 8);
buf += 8;
putvalue(buf, size, 8);
buf += 8;
putvalue(buf, zip_localheader_offset, 8);
buf += 8;
return buf - oldbuf;
}
static int kzip_write_current(struct kzip_file *zfile, void *buf, int len)
{
KZIP_DEBUG("%s: write_cb %p len %d\n", __func__, zfile->write_cb, len);
int retval = zfile->write_cb(zfile->handle, buf, len);
if (retval > 0) {
zfile->current_size += retval;
}
if ((zfile->current_size - zfile->reported_size) >= KDUMP_SIZE_REPORT) {
vo_show_progress(zfile->current_size / 0x100000);
zfile->reported_size = zfile->current_size;
}
if ((zfile->current_size - zfile->wdk_kick_size) >= KDUMP_TICK_WDT) {
mtk_wdt_restart();
zfile->wdk_kick_size = zfile->current_size;
}
return retval;
}
struct kzip_file *kzip_open(void *handle, int (*write_cb)(void *handle, void *p, int size))
{
struct kzip_file *zf = mempool_alloc(sizeof(struct kzip_file), MEMPOOL_ANY);
memset(zf, 0, sizeof(struct kzip_file));
zf->handle = handle;
zf->write_cb = write_cb;
KZIP_DEBUG("%s: handle zf %p %p write_cb %p\n", __func__, zf, zf->handle, zf->write_cb);
vo_show_progress(0);
return zf;
}
bool kzip_close(struct kzip_file *zf)
{
uint64_t central_header_offset = zf->current_size;
struct kzip_entry *zentries = zf->zentries;
int num = zf->entries_num;
int i, hsize = 0, local_hsize;
uint8_t databuf[128];
for (i = 0; i < num; i++) {
local_hsize = put_centralheader(databuf, zentries[i].filename, zentries[i].level,
zentries[i].localheader_offset,
zentries[i].comp_size,
zentries[i].uncomp_size,
zentries[i].crc32);
if (kzip_write_current(zf, databuf, local_hsize) != local_hsize) {
return false;
}
hsize += local_hsize;
}
voprintf_debug("%s: current_size %lld hoffset %lld\n", __func__, zf->current_size, central_header_offset);
uint64_t zip64_eoc_offset = zf->current_size;
local_hsize = put_zip64_eoc_directory_record(databuf, num, central_header_offset, hsize);
if ((local_hsize > 0) && (kzip_write_current(zf, databuf, local_hsize) != local_hsize)) {
return false;
}
local_hsize = put_zip64_eoc_directory_locator(databuf, zip64_eoc_offset);
if ((local_hsize > 0) && (kzip_write_current(zf, databuf, local_hsize) != local_hsize)) {
return false;
}
local_hsize = put_eoc_directory_record(databuf, num, central_header_offset, hsize);
if (kzip_write_current(zf, databuf, local_hsize) != local_hsize) {
return false;
}
mempool_free(zf);
return true;
}
#define CHUNK 65536
/* Compress from file source to file dest until EOF on source.
def() returns Z_OK on success, Z_MEM_ERROR if memory could not be
allocated for processing, Z_STREAM_ERROR if an invalid compression
level is supplied, Z_VERSION_ERROR if the version of zlib.h and the
version of the library linked do not match, or Z_ERRNO if there is
an error reading or writing the files. */
static bool kzip_add_file_no_mapped(struct kzip_file *zfile, const struct kzip_addlist *memlist,
int *flush, int *ret, uint8_t *in, uint8_t *out,
struct kzip_entry *zentry, z_stream *strm, struct aee_timer *zip_time)
{
lk_time_t t = current_time();
uint64_t mem_size = memlist->size;
uint8_t *memsrc = (uint8_t *)(uintptr_t)memlist->address;
*flush = (memlist->size == 0) ? Z_FINISH : Z_NO_FLUSH;
voprintf_debug("-- Compress memory %llx, size %llu\n",
memlist->address, memlist->size);
do {
int in_size = mem_size > CHUNK ? CHUNK : mem_size;
if ((current_time() - t)/1000 > 10) {
voprintf_debug("-- I am working(interval to 10s) --\n");
t = current_time();
}
memcpy(in, memsrc, in_size);
zentry->crc32 = crc32_no_comp(zentry->crc32, in, in_size);
memsrc += in_size;
mem_size -= in_size;
strm->avail_in = in_size;
strm->next_in = in;
/* run deflate() on input until output buffer not full, finish
compression if all of source has been read in */
do {
strm->avail_out = CHUNK;
strm->next_out = out;
*ret = deflate(strm, *flush); /* no bad return value */
assert(*ret != Z_STREAM_ERROR); /* state not clobbered */
int have = CHUNK - strm->avail_out;
if (have > 0) {
aee_timer_stop(zip_time);
if (kzip_write_current(zfile, out, have) != have) {
voprintf_debug("-- Compress memory %llx, error. surplus size: %u\n",
memlist->address, mem_size);
return false;
}
aee_timer_start(zip_time);
}
} while (strm->avail_out == 0);
assert(strm->avail_in == 0); /* all input will be used */
} while (mem_size > 0);
voprintf_debug("-- Compress memory %llx, done. surplus size: %u\n",
memlist->address, mem_size);
return true;
}
#ifndef MTK_3LEVEL_PAGETABLE
/* full mapping at LK, only one function is needed. kzip_add_file_no_mapped() */
#define kzip_add_file_do_mapped kzip_add_file_no_mapped
#else
/* LPAE, map before accessing the address of memory */
#define MAX_MAP_CNT (SECTION_SIZE/CHUNK)
#define mapflags (MMU_MEMORY_TYPE_NORMAL_WRITE_BACK | MMU_MEMORY_AP_P_RW_U_NA)
static bool kzip_add_file_do_mapped(struct kzip_file *zfile, const struct kzip_addlist *memlist,
int *flush, int *ret, uint8_t *in, uint8_t *out,
struct kzip_entry *zentry, z_stream *strm, struct aee_timer *zip_time)
{
/* multiple physical address mapped onto static 2MB address space. (scratch_addr) */
int cnt = 0;
uint64_t paddr = memlist->address;
vaddr_t vaddr = ROUNDUP(scratch_addr(), SECTION_SIZE);
voprintf_debug("-- map: paddr=0x%016llx, vaddr=0x%08x\n", paddr, vaddr);
uint64_t mem_size = memlist->size;
uint8_t *memsrc;
*flush = (memlist->size == 0) ? Z_FINISH : Z_NO_FLUSH;
voprintf_debug("-- Compress memory %llx, size %llu\n", memlist->address, memlist->size);
do {
/* arch_mmu_map by each 2MB size */
if (cnt == 0) {
int map_ok = arch_mmu_map(paddr, vaddr, mapflags, SECTION_SIZE);
if (map_ok != NO_ERROR) {
voprintf_debug("-- arch_mmu_map map error: map_ok=%d, paddr=0x%016llx, vaddr=0x%08x\n",
map_ok, paddr, vaddr);
return false;
}
memsrc = (uint8_t *)vaddr;
}
int in_size = mem_size > CHUNK ? CHUNK : mem_size;
memcpy(in, memsrc, in_size);
zentry->crc32 = crc32_no_comp(zentry->crc32, in, in_size);
memsrc += in_size;
mem_size -= in_size;
strm->avail_in = in_size;
strm->next_in = in;
/* run deflate() on input until output buffer not full, finish
compression if all of source has been read in */
do {
strm->avail_out = CHUNK;
strm->next_out = out;
*ret = deflate(strm, *flush); /* no bad return value */
assert(*ret != Z_STREAM_ERROR); /* state not clobbered */
int have = CHUNK - strm->avail_out;
if (have > 0) {
aee_timer_stop(zip_time);
if (kzip_write_current(zfile, out, have) != have) {
voprintf_debug("-- Compress memory %llx, error. surplus size: %u\n",
memlist->address, mem_size);
return false;
}
aee_timer_start(zip_time);
}
} while (strm->avail_out == 0);
assert(strm->avail_in == 0); /* all input will be used */
/* mod by 2MB (SECTION_SIZE) */
cnt++;
cnt %= MAX_MAP_CNT;
if (cnt == 0)
paddr += SECTION_SIZE;
} while (mem_size > 0);
voprintf_debug("-- Compress memory %llx, done. surplus size: %u\n", memlist->address, mem_size);
return true;
}
#endif
static bool kzip_add_file_from_expdb(struct kzip_file *zfile, const struct kzip_addlist *addlist,
int *flush, int *ret, uint8_t *in, uint8_t *out,
struct kzip_entry *zentry, z_stream *strm, struct aee_timer *zip_time)
{
uint64_t offset_src = addlist->address;
int64_t mem_size = addlist->size;
*flush = (addlist->size == 0) ? Z_FINISH : Z_NO_FLUSH;
voprintf_debug("-- Compress expdb offset %llx, size %llu\n", offset_src, mem_size);
do {
/* read from expdb to memsrc */
if (mem_size > CHUNK) {
//mrdump_read_expdb(in, CHUNK, offset_src);
} else {
//mrdump_read_expdb(in, (int)mem_size, offset_src);
}
int in_size = mem_size > CHUNK ? CHUNK : mem_size;
zentry->crc32 = crc32_no_comp(zentry->crc32, in, in_size);
mem_size -= in_size;
strm->avail_in = in_size;
strm->next_in = in;
/* run deflate() on input until output buffer not full, finish
compression if all of source has been read in */
do {
strm->avail_out = CHUNK;
strm->next_out = out;
*ret = deflate(strm, *flush); /* no bad return value */
assert(*ret != Z_STREAM_ERROR); /* state not clobbered */
int have = CHUNK - strm->avail_out;
if (have > 0) {
aee_timer_stop(zip_time);
if (kzip_write_current(zfile, out, have) != have) {
voprintf_debug("-- Compress expdb offset %llx, error. surplus size: %u\n",
offset_src, mem_size);
return false;
}
aee_timer_start(zip_time);
}
} while (strm->avail_out == 0);
assert(strm->avail_in == 0); /* all input will be used */
offset_src += in_size;
} while (mem_size > 0);
voprintf_debug("-- Compress expdb offset %llx, done. surplus size: %u\n", offset_src, mem_size);
return true;
}
bool kzip_add_file(struct kzip_file *zfile, const struct kzip_addlist *addlist, const char *zfilename)
{
int ret, flush;
z_stream strm;
struct aee_timer zip_time;
if (zfile->entries_num >= KZIP_ENTRY_MAX) {
voprintf_error("Too many zip entry %d\n", zfile->entries_num);
return false;
}
voprintf_debug("%s: zf %p(%p) %s\n", __func__, zfile, zfile->write_cb, zfilename);
struct kzip_entry *zentry = &zfile->zentries[zfile->entries_num++];
zentry->filename = strdup(zfilename);
zentry->localheader_offset = zfile->current_size;
zentry->level = KDUMP_ZLIB_LEVEL;
zentry->crc32 = 0xffffffffUL;
KZIP_DEBUG("%s: write local header\n", __func__);
uint8_t zip_localheader[128];
int hsize = put_localheader(zip_localheader, zfilename, zentry->level);
if (kzip_write_current(zfile, zip_localheader, hsize) != hsize) {
return false;
}
/* allocate deflate state */
memset(&strm, 0, sizeof(z_stream));
ret = deflateInit2(&strm, zentry->level, Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
if (ret != Z_OK) {
voprintf_error("zlib compress init failed\n");
return false;
}
uint8_t *out = mempool_alloc(CHUNK, MEMPOOL_ANY);
uint8_t *in = mempool_alloc(CHUNK, MEMPOOL_ANY);
if ((out == NULL) || (in == NULL)) {
voprintf_error("%s: malloc failed.\n", __func__);
goto error;
}
aee_timer_init(&zip_time);
aee_timer_start(&zip_time);
uint64_t start = zfile->current_size, uncomp_size = 0;
do {
switch (addlist->type) {
case MEM_NO_MAP:
if (kzip_add_file_no_mapped(zfile, addlist, &flush, &ret, in, out, zentry, &strm, &zip_time) == false)
goto error;
break;
case MEM_DO_MAP:
if (kzip_add_file_do_mapped(zfile, addlist, &flush, &ret, in, out, zentry, &strm, &zip_time) == false)
goto error;
break;
case EXPDB_FILE:
if (kzip_add_file_from_expdb(zfile, addlist, &flush, &ret, in, out, zentry, &strm, &zip_time) == false)
goto error;
break;
default:
goto error;
break;
}
uncomp_size += addlist->size;
addlist++;
/* done when last data in file processed */
} while (flush != Z_FINISH);
assert(ret == Z_STREAM_END); /* stream will be complete */
deflateEnd(&strm);
mempool_free(out);
mempool_free(in);
aee_timer_stop(&zip_time);
voprintf_info("Zip time: %d sec\n", zip_time.acc_ms / 1000);
zentry->comp_size = zfile->current_size - start;
zentry->uncomp_size = uncomp_size;
zentry->crc32 = zentry->crc32 ^ 0xffffffffUL;
return true;
error:
mempool_free(out);
mempool_free(in);
deflateEnd(&strm);
return false;
}