|
|
@@ -11,7 +11,7 @@
|
|
|
#define MAXDCODES 30
|
|
|
#define MAXCODES (MAXLCODES + MAXDCODES)
|
|
|
#define FIXLCODES 288
|
|
|
-#define FASTBITS 9
|
|
|
+#define FASTBITS 10
|
|
|
|
|
|
static unsigned int crc_table[256];
|
|
|
static int crc_table_ready;
|
|
|
@@ -580,82 +580,399 @@ int inflate_gzip(struct inflate *z, unsigned int *out_len)
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
-/* ---- whole-buffer helpers ------------------------------------------- */
|
|
|
+/* ---- whole-buffer fast path ----------------------------------------
|
|
|
+ *
|
|
|
+ * The streaming decoder above reads one bit per Huffman step and stages
|
|
|
+ * every byte in the 32 KB ring. When the whole input and output are in
|
|
|
+ * memory (the boot rootfs, PNG/HTTP bodies decoded into one buffer) that
|
|
|
+ * is needlessly slow: here the output buffer itself is the LZ77 history,
|
|
|
+ * the bit buffer is refilled a byte at a time up to 32 bits, and codes of
|
|
|
+ * up to FASTBITS are resolved with one table lookup. Longer codes fall
|
|
|
+ * back to the canonical walk on the bits already in the buffer. Only the
|
|
|
+ * checksum the container needs is computed, once, over the finished
|
|
|
+ * output. The tables live in the caller's struct-inflate workspace. */
|
|
|
+
|
|
|
+#define FAST_SIZE (1u << FASTBITS)
|
|
|
+
|
|
|
+struct fhuff {
|
|
|
+ struct inflate_huff h;
|
|
|
+ unsigned short fast[FAST_SIZE]; /* (sym << 4) | len, 0 = slow */
|
|
|
+};
|
|
|
+
|
|
|
+struct fz {
|
|
|
+ const unsigned char *in, *in_end;
|
|
|
+ unsigned int bitbuf;
|
|
|
+ int bitcnt;
|
|
|
+ unsigned int over; /* zero bytes padded past the end */
|
|
|
+ unsigned char *out, *out_start, *out_end;
|
|
|
+};
|
|
|
|
|
|
-struct mem_sink {
|
|
|
- unsigned char *dst;
|
|
|
- unsigned int cap, pos;
|
|
|
+struct fz_ws {
|
|
|
+ struct fhuff lit, dist;
|
|
|
};
|
|
|
|
|
|
-static int mem_write(void *ctx, const unsigned char *data, unsigned int len)
|
|
|
+static void fz_refill(struct fz *f)
|
|
|
+{
|
|
|
+ while (f->bitcnt <= 24) {
|
|
|
+ unsigned int b = 0;
|
|
|
+
|
|
|
+ if (f->in < f->in_end)
|
|
|
+ b = *f->in++;
|
|
|
+ else
|
|
|
+ f->over++;
|
|
|
+ f->bitbuf |= b << f->bitcnt;
|
|
|
+ f->bitcnt += 8;
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+static int fz_bits(struct fz *f, int n)
|
|
|
+{
|
|
|
+ unsigned int v;
|
|
|
+
|
|
|
+ if (f->bitcnt < n)
|
|
|
+ fz_refill(f);
|
|
|
+ v = f->bitbuf & ((1u << n) - 1u);
|
|
|
+ f->bitbuf >>= n;
|
|
|
+ f->bitcnt -= n;
|
|
|
+ return (int)v;
|
|
|
+}
|
|
|
+
|
|
|
+static int fhuff_build(struct fhuff *fh, const unsigned char *lengths, int n)
|
|
|
{
|
|
|
- struct mem_sink *m = (struct mem_sink *)ctx;
|
|
|
+ unsigned int next[MAXBITS + 2];
|
|
|
+ int r, len, sym, i;
|
|
|
|
|
|
- if (len > m->cap - m->pos)
|
|
|
+ r = huff_build(&fh->h, lengths, n);
|
|
|
+ if (r < 0)
|
|
|
return -1;
|
|
|
+ for (i = 0; i < (int)FAST_SIZE; i++)
|
|
|
+ fh->fast[i] = 0;
|
|
|
{
|
|
|
- unsigned int i;
|
|
|
+ /* next[len] = first canonical code of length len. */
|
|
|
+ unsigned int c = 0;
|
|
|
+
|
|
|
+ next[1] = 0;
|
|
|
+ for (len = 1; len < MAXBITS; len++) {
|
|
|
+ c = (c + fh->h.count[len]) << 1;
|
|
|
+ next[len + 1] = c;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ for (sym = 0; sym < n; sym++) {
|
|
|
+ unsigned int c, rev = 0;
|
|
|
+ int l = lengths[sym], k;
|
|
|
+
|
|
|
+ if (!l)
|
|
|
+ continue;
|
|
|
+ c = next[l]++;
|
|
|
+ if (l > FASTBITS)
|
|
|
+ continue;
|
|
|
+ for (k = 0; k < l; k++)
|
|
|
+ rev |= ((c >> k) & 1u) << (l - 1 - k);
|
|
|
+ for (; rev < FAST_SIZE; rev += 1u << l)
|
|
|
+ fh->fast[rev] = (unsigned short)((sym << 4) | l);
|
|
|
+ }
|
|
|
+ return r;
|
|
|
+}
|
|
|
+
|
|
|
+static int fz_decode(struct fz *f, const struct fhuff *fh)
|
|
|
+{
|
|
|
+ unsigned int e;
|
|
|
+ int len, code = 0, first = 0, index = 0;
|
|
|
+
|
|
|
+ if (f->bitcnt < MAXBITS)
|
|
|
+ fz_refill(f);
|
|
|
+ e = fh->fast[f->bitbuf & (FAST_SIZE - 1u)];
|
|
|
+ if (e) {
|
|
|
+ f->bitbuf >>= e & 15u;
|
|
|
+ f->bitcnt -= (int)(e & 15u);
|
|
|
+ return (int)(e >> 4);
|
|
|
+ }
|
|
|
+ for (len = 1; len <= MAXBITS; len++) {
|
|
|
+ int count;
|
|
|
+
|
|
|
+ code |= (int)(f->bitbuf & 1u);
|
|
|
+ f->bitbuf >>= 1;
|
|
|
+ f->bitcnt--;
|
|
|
+ count = fh->h.count[len];
|
|
|
+ if (code - count < first)
|
|
|
+ return fh->h.symbol[index + (code - first)];
|
|
|
+ index += count;
|
|
|
+ first += count;
|
|
|
+ first <<= 1;
|
|
|
+ code <<= 1;
|
|
|
+ }
|
|
|
+ return -1;
|
|
|
+}
|
|
|
+
|
|
|
+static int fz_codes(struct fz *f, const struct fhuff *lit,
|
|
|
+ const struct fhuff *dist)
|
|
|
+{
|
|
|
+ for (;;) {
|
|
|
+ int sym = fz_decode(f, lit);
|
|
|
+ unsigned int len, d;
|
|
|
+ unsigned char *out, *src;
|
|
|
+
|
|
|
+ if (sym < 256) {
|
|
|
+ if (sym < 0 || f->out >= f->out_end)
|
|
|
+ return -1;
|
|
|
+ *f->out++ = (unsigned char)sym;
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if (sym == 256)
|
|
|
+ return f->over > 4 ? -1 : 0;
|
|
|
+ sym -= 257;
|
|
|
+ if (sym >= 29)
|
|
|
+ return -1;
|
|
|
+ len = len_base[sym];
|
|
|
+ if (len_ext[sym])
|
|
|
+ len += (unsigned int)fz_bits(f, len_ext[sym]);
|
|
|
+ sym = fz_decode(f, dist);
|
|
|
+ if (sym < 0 || sym >= 30)
|
|
|
+ return -1;
|
|
|
+ d = dist_base[sym];
|
|
|
+ if (dist_ext[sym])
|
|
|
+ d += (unsigned int)fz_bits(f, dist_ext[sym]);
|
|
|
+ out = f->out;
|
|
|
+ if (d > (unsigned int)(out - f->out_start) ||
|
|
|
+ len > (unsigned int)(f->out_end - out))
|
|
|
+ return -1;
|
|
|
+ src = out - d;
|
|
|
+ if (d >= 4) {
|
|
|
+ while (len >= 4) {
|
|
|
+ out[0] = src[0]; out[1] = src[1];
|
|
|
+ out[2] = src[2]; out[3] = src[3];
|
|
|
+ out += 4; src += 4; len -= 4;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ while (len--)
|
|
|
+ *out++ = *src++;
|
|
|
+ f->out = out;
|
|
|
+ }
|
|
|
+}
|
|
|
|
|
|
- for (i = 0; i < len; i++)
|
|
|
- m->dst[m->pos + i] = data[i];
|
|
|
+static int fz_blocks(struct fz *f, struct fz_ws *ws)
|
|
|
+{
|
|
|
+ static const unsigned char order[19] = {
|
|
|
+ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
|
|
|
+ };
|
|
|
+ unsigned char lengths[FIXLCODES + MAXDCODES];
|
|
|
+ int last, type, i;
|
|
|
+
|
|
|
+ do {
|
|
|
+ last = fz_bits(f, 1);
|
|
|
+ type = fz_bits(f, 2);
|
|
|
+ if (type == 0) {
|
|
|
+ unsigned int len, nlen;
|
|
|
+
|
|
|
+ fz_bits(f, f->bitcnt & 7);
|
|
|
+ len = (unsigned int)fz_bits(f, 16);
|
|
|
+ nlen = (unsigned int)fz_bits(f, 16);
|
|
|
+ if ((len ^ nlen) != 0xFFFFu ||
|
|
|
+ len > (unsigned int)(f->out_end - f->out))
|
|
|
+ return -1;
|
|
|
+ /* Return the whole buffered bytes to the input and
|
|
|
+ * copy the stored data straight from it. */
|
|
|
+ {
|
|
|
+ unsigned int back = (unsigned int)f->bitcnt >> 3;
|
|
|
+
|
|
|
+ if (back < f->over)
|
|
|
+ return -1;
|
|
|
+ f->in -= back - f->over;
|
|
|
+ f->bitbuf = 0;
|
|
|
+ f->bitcnt = 0;
|
|
|
+ f->over = 0;
|
|
|
+ }
|
|
|
+ if (len > (unsigned int)(f->in_end - f->in))
|
|
|
+ return -1;
|
|
|
+ while (len--)
|
|
|
+ *f->out++ = *f->in++;
|
|
|
+ } else if (type == 1) {
|
|
|
+ for (i = 0; i < 144; i++)
|
|
|
+ lengths[i] = 8;
|
|
|
+ for (; i < 256; i++)
|
|
|
+ lengths[i] = 9;
|
|
|
+ for (; i < 280; i++)
|
|
|
+ lengths[i] = 7;
|
|
|
+ for (; i < 288; i++)
|
|
|
+ lengths[i] = 8;
|
|
|
+ for (i = 0; i < 30; i++)
|
|
|
+ lengths[288 + i] = 5;
|
|
|
+ if (fhuff_build(&ws->lit, lengths, 288) < 0 ||
|
|
|
+ fhuff_build(&ws->dist, lengths + 288, 30) < 0 ||
|
|
|
+ fz_codes(f, &ws->lit, &ws->dist))
|
|
|
+ return -1;
|
|
|
+ } else if (type == 2) {
|
|
|
+ int hlit = fz_bits(f, 5) + 257;
|
|
|
+ int hdist = fz_bits(f, 5) + 1;
|
|
|
+ int hclen = fz_bits(f, 4) + 4;
|
|
|
+ int total = hlit + hdist;
|
|
|
+
|
|
|
+ if (hlit > MAXLCODES || hdist > MAXDCODES)
|
|
|
+ return -1;
|
|
|
+ for (i = 0; i < 19; i++)
|
|
|
+ lengths[i] = 0;
|
|
|
+ for (i = 0; i < hclen; i++)
|
|
|
+ lengths[order[i]] = (unsigned char)fz_bits(f, 3);
|
|
|
+ if (fhuff_build(&ws->lit, lengths, 19) < 0)
|
|
|
+ return -1;
|
|
|
+ for (i = 0; i < total; ) {
|
|
|
+ int sym = fz_decode(f, &ws->lit), rep, val;
|
|
|
+
|
|
|
+ if (sym < 0)
|
|
|
+ return -1;
|
|
|
+ if (sym < 16) {
|
|
|
+ lengths[i++] = (unsigned char)sym;
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if (sym == 16) {
|
|
|
+ if (i == 0)
|
|
|
+ return -1;
|
|
|
+ rep = 3 + fz_bits(f, 2);
|
|
|
+ val = lengths[i - 1];
|
|
|
+ } else if (sym == 17) {
|
|
|
+ rep = 3 + fz_bits(f, 3);
|
|
|
+ val = 0;
|
|
|
+ } else {
|
|
|
+ rep = 11 + fz_bits(f, 7);
|
|
|
+ val = 0;
|
|
|
+ }
|
|
|
+ if (i + rep > total)
|
|
|
+ return -1;
|
|
|
+ while (rep--)
|
|
|
+ lengths[i++] = (unsigned char)val;
|
|
|
+ }
|
|
|
+ if (fhuff_build(&ws->lit, lengths, hlit) < 0 ||
|
|
|
+ fhuff_build(&ws->dist, lengths + hlit, hdist) < 0 ||
|
|
|
+ fz_codes(f, &ws->lit, &ws->dist))
|
|
|
+ return -1;
|
|
|
+ } else {
|
|
|
+ return -1;
|
|
|
+ }
|
|
|
+ if (f->over > 4)
|
|
|
+ return -1;
|
|
|
+ } while (!last);
|
|
|
+
|
|
|
+ /* Byte-align and hand the whole bytes still buffered back to the
|
|
|
+ * input, so the container trailer is read from memory. */
|
|
|
+ fz_bits(f, f->bitcnt & 7);
|
|
|
+ {
|
|
|
+ unsigned int back = (unsigned int)f->bitcnt >> 3;
|
|
|
+
|
|
|
+ if (back < f->over)
|
|
|
+ return -1; /* consumed padding: truncated */
|
|
|
+ f->in -= back - f->over;
|
|
|
+ f->bitbuf = 0;
|
|
|
+ f->bitcnt = 0;
|
|
|
+ f->over = 0;
|
|
|
}
|
|
|
- m->pos += len;
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
-static int mem_inflate(const void *src, unsigned int len, void *dst,
|
|
|
- unsigned int cap, unsigned int *out_len, void *ws,
|
|
|
- int gzip)
|
|
|
-{
|
|
|
- struct inflate *z = (struct inflate *)ws;
|
|
|
- struct inflate_src s;
|
|
|
- struct inflate_sink sink;
|
|
|
- struct mem_sink m;
|
|
|
-
|
|
|
- s.p = (const unsigned char *)src;
|
|
|
- s.len = len;
|
|
|
- s.more = 0;
|
|
|
- s.ctx = 0;
|
|
|
- m.dst = (unsigned char *)dst;
|
|
|
- m.cap = cap;
|
|
|
- m.pos = 0;
|
|
|
- sink.write = mem_write;
|
|
|
- sink.ctx = &m;
|
|
|
- inflate_init(z, &s, &sink);
|
|
|
- if (gzip ? inflate_gzip(z, out_len) : inflate_zlib(z, out_len))
|
|
|
+/* container: 0 raw, 1 zlib, 2 gzip */
|
|
|
+static int fz_inflate(const void *src, unsigned int len, void *dst,
|
|
|
+ unsigned int cap, unsigned int *out_len, void *ws,
|
|
|
+ int container)
|
|
|
+{
|
|
|
+ struct fz f;
|
|
|
+ const unsigned char *p = (const unsigned char *)src;
|
|
|
+ const unsigned char *end = p + len;
|
|
|
+ unsigned int n;
|
|
|
+
|
|
|
+ if (container == 2) {
|
|
|
+ unsigned int flg;
|
|
|
+
|
|
|
+ if (len < 18 || p[0] != 0x1F || p[1] != 0x8B || p[2] != 8)
|
|
|
+ return -1;
|
|
|
+ flg = p[3];
|
|
|
+ p += 10;
|
|
|
+ if (flg & 0x04) {
|
|
|
+ if (end - p < 2)
|
|
|
+ return -1;
|
|
|
+ n = p[0] | ((unsigned int)p[1] << 8);
|
|
|
+ p += 2;
|
|
|
+ if ((unsigned int)(end - p) < n)
|
|
|
+ return -1;
|
|
|
+ p += n;
|
|
|
+ }
|
|
|
+ if (flg & 0x08) {
|
|
|
+ while (p < end && *p)
|
|
|
+ p++;
|
|
|
+ p++;
|
|
|
+ }
|
|
|
+ if (flg & 0x10) {
|
|
|
+ while (p < end && *p)
|
|
|
+ p++;
|
|
|
+ p++;
|
|
|
+ }
|
|
|
+ if (flg & 0x02)
|
|
|
+ p += 2;
|
|
|
+ if (p >= end)
|
|
|
+ return -1;
|
|
|
+ } else if (container == 1) {
|
|
|
+ if (len < 6 || (p[0] & 0x0F) != 8 || (p[0] >> 4) > 7 ||
|
|
|
+ (((unsigned int)p[0] << 8) | p[1]) % 31 != 0 ||
|
|
|
+ (p[1] & 0x20))
|
|
|
+ return -1;
|
|
|
+ p += 2;
|
|
|
+ }
|
|
|
+
|
|
|
+ f.in = p;
|
|
|
+ f.in_end = end;
|
|
|
+ f.bitbuf = 0;
|
|
|
+ f.bitcnt = 0;
|
|
|
+ f.over = 0;
|
|
|
+ f.out = f.out_start = (unsigned char *)dst;
|
|
|
+ f.out_end = f.out_start + cap;
|
|
|
+ if (fz_blocks(&f, (struct fz_ws *)ws))
|
|
|
return -1;
|
|
|
+ n = (unsigned int)(f.out - f.out_start);
|
|
|
+
|
|
|
+ if (container == 2) {
|
|
|
+ unsigned int crc, size;
|
|
|
+
|
|
|
+ if (f.in_end - f.in < 8)
|
|
|
+ return -1;
|
|
|
+ crc_table_init();
|
|
|
+ crc = crc_update(0xFFFFFFFFu, f.out_start, n) ^ 0xFFFFFFFFu;
|
|
|
+ if (crc != (f.in[0] | ((unsigned int)f.in[1] << 8) |
|
|
|
+ ((unsigned int)f.in[2] << 16) |
|
|
|
+ ((unsigned int)f.in[3] << 24)))
|
|
|
+ return -1;
|
|
|
+ size = f.in[4] | ((unsigned int)f.in[5] << 8) |
|
|
|
+ ((unsigned int)f.in[6] << 16) |
|
|
|
+ ((unsigned int)f.in[7] << 24);
|
|
|
+ if (size != n)
|
|
|
+ return -1;
|
|
|
+ } else if (container == 1) {
|
|
|
+ unsigned int a = 1, b = 0;
|
|
|
+
|
|
|
+ if (f.in_end - f.in < 4)
|
|
|
+ return -1;
|
|
|
+ adler_update(&a, &b, f.out_start, n);
|
|
|
+ if (((b << 16) | a) != (((unsigned int)f.in[0] << 24) |
|
|
|
+ ((unsigned int)f.in[1] << 16) |
|
|
|
+ ((unsigned int)f.in[2] << 8) | f.in[3]))
|
|
|
+ return -1;
|
|
|
+ }
|
|
|
+ if (out_len)
|
|
|
+ *out_len = n;
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
int inflate_zlib_mem(const void *src, unsigned int len, void *dst,
|
|
|
unsigned int cap, unsigned int *out_len, void *ws)
|
|
|
{
|
|
|
- return mem_inflate(src, len, dst, cap, out_len, ws, 0);
|
|
|
+ return fz_inflate(src, len, dst, cap, out_len, ws, 1);
|
|
|
}
|
|
|
|
|
|
int inflate_gzip_mem(const void *src, unsigned int len, void *dst,
|
|
|
unsigned int cap, unsigned int *out_len, void *ws)
|
|
|
{
|
|
|
- return mem_inflate(src, len, dst, cap, out_len, ws, 1);
|
|
|
+ return fz_inflate(src, len, dst, cap, out_len, ws, 2);
|
|
|
}
|
|
|
|
|
|
int inflate_raw_mem(const void *src, unsigned int len, void *dst,
|
|
|
unsigned int cap, unsigned int *out_len, void *ws)
|
|
|
{
|
|
|
- struct inflate *z = (struct inflate *)ws;
|
|
|
- struct inflate_src s;
|
|
|
- struct inflate_sink sink;
|
|
|
- struct mem_sink m;
|
|
|
-
|
|
|
- s.p = (const unsigned char *)src;
|
|
|
- s.len = len;
|
|
|
- s.more = 0;
|
|
|
- s.ctx = 0;
|
|
|
- m.dst = (unsigned char *)dst;
|
|
|
- m.cap = cap;
|
|
|
- m.pos = 0;
|
|
|
- sink.write = mem_write;
|
|
|
- sink.ctx = &m;
|
|
|
- inflate_init(z, &s, &sink);
|
|
|
- return inflate_raw(z, out_len);
|
|
|
+ return fz_inflate(src, len, dst, cap, out_len, ws, 0);
|
|
|
}
|