From 247147654fe5cd11cf15d8dff91440405ea57040 Mon Sep 17 00:00:00 2001
From: Simon Hosie <simon.hosie@arm.com>
Date: Wed, 12 Apr 2017 15:44:21 -0700
Subject: [PATCH 2/2] Inflate using wider loads and stores

In inflate_fast() the output pointer always has plenty of room to write. This
means that so long as the target is capable, wide un-aligned loads and stores
can be used to transfer several bytes at once. When the reference distance is
too short simply unroll the data a little to increase the distance.

Change-Id: I59854eb25d2b1e43561c8a2afaf9175bf10cf674
---
 contrib/arm/chunkcopy.h | 279 ++++++++++++++++++++++++++++++++++++++++++++++++
 contrib/arm/inffast.c   |  96 +++++++----------
 contrib/arm/inflate.c   |  22 ++--
 3 files changed, 335 insertions(+), 62 deletions(-)
 create mode 100644 contrib/arm/chunkcopy.h

--- /dev/null
+++ b/contrib/arm/chunkcopy.h
@@ -0,0 +1,279 @@
+/* chunkcopy.h -- fast copies and sets
+ * Copyright (C) 2017 ARM, Inc.
+ * For conditions of distribution and use, see copyright notice in zlib.h
+ */
+
+#ifndef CHUNKCOPY_H
+#define CHUNKCOPY_H
+
+#include "zutil.h"
+#include <arm_neon.h>
+
+#if __STDC_VERSION__ >= 199901L
+#define Z_RESTRICT restrict
+#else
+#define Z_RESTRICT
+#endif
+
+typedef uint8x16_t chunkcopy_chunk_t;
+#define CHUNKCOPY_CHUNK_SIZE sizeof(chunkcopy_chunk_t)
+
+/*
+   Ask the compiler to perform a wide, unaligned load with an machine
+   instruction appropriate for the chunkcopy_chunk_t type.
+ */
+static inline chunkcopy_chunk_t loadchunk(const unsigned char FAR *s) {
+    chunkcopy_chunk_t c;
+    __builtin_memcpy(&c, s, sizeof(c));
+    return c;
+}
+
+/*
+   Ask the compiler to perform a wide, unaligned store with an machine
+   instruction appropriate for the chunkcopy_chunk_t type.
+ */
+static inline void storechunk(unsigned char FAR *d, chunkcopy_chunk_t c) {
+    __builtin_memcpy(d, &c, sizeof(c));
+}
+
+/*
+   Perform a memcpy-like operation, but assume that length is non-zero and that
+   it's OK to overwrite at least CHUNKCOPY_CHUNK_SIZE bytes of output even if
+   the length is shorter than this.
+
+   It also guarantees that it will properly unroll the data if the distance
+   between `out` and `from` is at least CHUNKCOPY_CHUNK_SIZE, which we rely on
+   in chunkcopy_relaxed().
+
+   Aside from better memory bus utilisation, this means that short copies
+   (CHUNKCOPY_CHUNK_SIZE bytes or fewer) will fall straight through the loop
+   without iteration, which will hopefully make the branch prediction more
+   reliable.
+ */
+static inline unsigned char FAR *chunkcopy_core(unsigned char FAR *out,
+                                                const unsigned char FAR *from,
+                                                unsigned len) {
+    int bump = (--len % CHUNKCOPY_CHUNK_SIZE) + 1;
+    storechunk(out, loadchunk(from));
+    out += bump;
+    from += bump;
+    len /= CHUNKCOPY_CHUNK_SIZE;
+    while (len-- > 0) {
+        storechunk(out, loadchunk(from));
+        out += CHUNKCOPY_CHUNK_SIZE;
+        from += CHUNKCOPY_CHUNK_SIZE;
+    }
+    return out;
+}
+
+/*
+   Like chunkcopy_core, but avoid writing beyond of legal output.
+
+   Accepts an additional pointer to the end of safe output.  A generic safe
+   copy would use (out + len), but it's normally the case that the end of the
+   output buffer is beyond the end of the current copy, and this can still be
+   exploited.
+ */
+static inline unsigned char FAR *chunkcopy_core_safe(unsigned char FAR *out,
+                                                     const unsigned char FAR * from,
+                                                     unsigned len,
+                                                     unsigned char FAR *limit) {
+    Assert(out + len <= limit, "chunk copy exceeds safety limit");
+    if (limit - out < CHUNKCOPY_CHUNK_SIZE) {
+        const unsigned char FAR * Z_RESTRICT rfrom = from;
+        if (len & 8) { __builtin_memcpy(out, rfrom, 8); out += 8; rfrom += 8; }
+        if (len & 4) { __builtin_memcpy(out, rfrom, 4); out += 4; rfrom += 4; }
+        if (len & 2) { __builtin_memcpy(out, rfrom, 2); out += 2; rfrom += 2; }
+        if (len & 1) { *out++ = *rfrom++; }
+        return out;
+    }
+    return chunkcopy_core(out, from, len);
+}
+
+/*
+   Perform short copies until distance can be rewritten as being at least
+   CHUNKCOPY_CHUNK_SIZE.
+
+   This assumes that it's OK to overwrite at least the first
+   2*CHUNKCOPY_CHUNK_SIZE bytes of output even if the copy is shorter than
+   this.  This assumption holds within inflate_fast() which starts every
+   iteration with at least 258 bytes of output space available (258 being the
+   maximum length output from a single token; see inffast.c).
+ */
+static inline unsigned char FAR *chunkunroll_relaxed(unsigned char FAR *out,
+                                                     unsigned FAR *dist,
+                                                     unsigned FAR *len) {
+    const unsigned char FAR *from = out - *dist;
+    while (*dist < *len && *dist < CHUNKCOPY_CHUNK_SIZE) {
+        storechunk(out, loadchunk(from));
+        out += *dist;
+        *len -= *dist;
+        *dist += *dist;
+    }
+    return out;
+}
+
+
+static inline uint8x16_t chunkset_vld1q_dup_u8x8(const unsigned char FAR * Z_RESTRICT from) {
+#if defined(__clang__) || defined(__aarch64__)
+    return vreinterpretq_u8_u64(vld1q_dup_u64((void *)from));
+#else
+    /* 32-bit GCC uses an alignment hint for vld1q_dup_u64, even when given a
+     * void pointer, so here's an alternate implementation.
+     */
+    uint8x8_t h = vld1_u8(from);
+    return vcombine_u8(h, h);
+#endif
+}
+
+/*
+   Perform an overlapping copy which behaves as a memset() operation, but
+   supporting periods other than one, and assume that length is non-zero and
+   that it's OK to overwrite at least CHUNKCOPY_CHUNK_SIZE*3 bytes of output
+   even if the length is shorter than this.
+ */
+static inline unsigned char FAR *chunkset_core(unsigned char FAR *out,
+                                               unsigned period,
+                                               unsigned len) {
+    uint8x16_t f;
+    int bump = ((len - 1) % sizeof(f)) + 1;
+
+    switch (period) {
+    case 1:
+        f = vld1q_dup_u8(out - 1);
+        vst1q_u8(out, f);
+        out += bump;
+        len -= bump;
+        while (len > 0) {
+            vst1q_u8(out, f);
+            out += sizeof(f);
+            len -= sizeof(f);
+        }
+        return out;
+    case 2:
+        f = vreinterpretq_u8_u16(vld1q_dup_u16((void *)(out - 2)));
+        vst1q_u8(out, f);
+        out += bump;
+        len -= bump;
+        if (len > 0) {
+            f = vreinterpretq_u8_u16(vld1q_dup_u16((void *)(out - 2)));
+            do {
+                vst1q_u8(out, f);
+                out += sizeof(f);
+                len -= sizeof(f);
+            } while (len > 0);
+        }
+        return out;
+    case 4:
+        f = vreinterpretq_u8_u32(vld1q_dup_u32((void *)(out - 4)));
+        vst1q_u8(out, f);
+        out += bump;
+        len -= bump;
+        if (len > 0) {
+            f = vreinterpretq_u8_u32(vld1q_dup_u32((void *)(out - 4)));
+            do {
+                vst1q_u8(out, f);
+                out += sizeof(f);
+                len -= sizeof(f);
+            } while (len > 0);
+        }
+        return out;
+    case 8:
+        f = chunkset_vld1q_dup_u8x8(out - 8);
+        vst1q_u8(out, f);
+        out += bump;
+        len -= bump;
+        if (len > 0) {
+            f = chunkset_vld1q_dup_u8x8(out - 8);
+            do {
+                vst1q_u8(out, f);
+                out += sizeof(f);
+                len -= sizeof(f);
+            } while (len > 0);
+        }
+        return out;
+    }
+    out = chunkunroll_relaxed(out, &period, &len);
+    return chunkcopy_core(out, out - period, len);
+}
+
+/*
+   Perform a memcpy-like operation, but assume that length is non-zero and that
+   it's OK to overwrite at least CHUNKCOPY_CHUNK_SIZE bytes of output even if
+   the length is shorter than this.
+
+   Unlike chunkcopy_core() above, no guarantee is made regarding the behaviour
+   of overlapping buffers, regardless of the distance between the pointers.
+   This is reflected in the `restrict`-qualified pointers, allowing the
+   compiler to reorder loads and stores.
+ */
+static inline unsigned char FAR *chunkcopy_relaxed(unsigned char FAR * Z_RESTRICT out,
+                                                   const unsigned char FAR * Z_RESTRICT from,
+                                                   unsigned len) {
+    return chunkcopy_core(out, from, len);
+}
+
+/*
+   Like chunkcopy_relaxed, but avoid writing beyond of legal output.
+
+   Unlike chunkcopy_core_safe() above, no guarantee is made regarding the
+   behaviour of overlapping buffers, regardless of the distance between the
+   pointers.  This is reflected in the `restrict`-qualified pointers, allowing
+   the compiler to reorder loads and stores.
+
+   Accepts an additional pointer to the end of safe output.  A generic safe
+   copy would use (out + len), but it's normally the case that the end of the
+   output buffer is beyond the end of the current copy, and this can still be
+   exploited.
+ */
+static inline unsigned char FAR *chunkcopy_safe(unsigned char FAR *out,
+                                                const unsigned char FAR * Z_RESTRICT from,
+                                                unsigned len,
+                                                unsigned char FAR *limit) {
+    Assert(out + len <= limit, "chunk copy exceeds safety limit");
+    return chunkcopy_core_safe(out, from, len, limit);
+}
+
+/*
+   Perform chunky copy within the same buffer, where the source and destination
+   may potentially overlap.
+
+   Assumes that len > 0 on entry, and that it's safe to write at least
+   CHUNKCOPY_CHUNK_SIZE*3 bytes to the output.
+ */
+static inline unsigned char FAR *chunkcopy_lapped_relaxed(unsigned char FAR *out,
+                                                          unsigned dist,
+                                                          unsigned len) {
+    if (dist < len && dist < CHUNKCOPY_CHUNK_SIZE) {
+        return chunkset_core(out, dist, len);
+    }
+    return chunkcopy_core(out, out - dist, len);
+}
+
+/*
+   Behave like chunkcopy_lapped_relaxed, but avoid writing beyond of legal output.
+
+   Accepts an additional pointer to the end of safe output.  A generic safe
+   copy would use (out + len), but it's normally the case that the end of the
+   output buffer is beyond the end of the current copy, and this can still be
+   exploited.
+ */
+static inline unsigned char FAR *chunkcopy_lapped_safe(unsigned char FAR *out,
+                                                       unsigned dist,
+                                                       unsigned len,
+                                                       unsigned char FAR *limit) {
+    Assert(out + len <= limit, "chunk copy exceeds safety limit");
+    if (limit - out < CHUNKCOPY_CHUNK_SIZE * 3) {
+        /* TODO: try harder to optimise this */
+        while (len-- > 0) {
+            *out = *(out - dist);
+            out++;
+        }
+        return out;
+    }
+    return chunkcopy_lapped_relaxed(out, dist, len);
+}
+
+#undef Z_RESTRICT
+
+#endif /* CHUNKCOPY_H */
--- a/contrib/arm/inffast.c
+++ b/contrib/arm/inffast.c
@@ -7,6 +7,7 @@
 #include "inftrees.h"
 #include "inflate.h"
 #include "inffast.h"
+#include "chunkcopy.h"
 
 #ifdef ASMINF
 #  pragma message("Assembler code may have bugs -- use at your own risk")
@@ -57,6 +58,7 @@ unsigned start;         /* inflate()'s s
     unsigned char FAR *out;     /* local strm->next_out */
     unsigned char FAR *beg;     /* inflate()'s initial strm->next_out */
     unsigned char FAR *end;     /* while out < end, enough space available */
+    unsigned char FAR *limit;   /* safety limit for chunky copies */
 #ifdef INFLATE_STRICT
     unsigned dmax;              /* maximum distance from zlib header */
 #endif
@@ -84,12 +86,13 @@ unsigned start;         /* inflate()'s s
     out = strm->next_out;
     beg = out - (start - strm->avail_out);
     end = out + (strm->avail_out - 257);
+    limit = out + strm->avail_out;
 #ifdef INFLATE_STRICT
     dmax = state->dmax;
 #endif
     wsize = state->wsize;
     whave = state->whave;
-    wnext = state->wnext;
+    wnext = (state->wnext == 0 && whave >= wsize) ? wsize : state->wnext;
     window = state->window;
     hold = state->hold;
     bits = state->bits;
@@ -197,70 +200,51 @@ unsigned start;         /* inflate()'s s
 #endif
                     }
                     from = window;
-                    if (wnext == 0) {           /* very common case */
-                        from += wsize - op;
-                        if (op < len) {         /* some from window */
-                            len -= op;
-                            do {
-                                *out++ = *from++;
-                            } while (--op);
-                            from = out - dist;  /* rest from output */
-                        }
+                    if (wnext >= op) {          /* contiguous in window */
+                        from += wnext - op;
                     }
-                    else if (wnext < op) {      /* wrap around window */
-                        from += wsize + wnext - op;
+                    else {                      /* wrap around window */
                         op -= wnext;
+                        from += wsize - op;
                         if (op < len) {         /* some from end of window */
                             len -= op;
-                            do {
-                                *out++ = *from++;
-                            } while (--op);
-                            from = window;
-                            if (wnext < len) {  /* some from start of window */
-                                op = wnext;
-                                len -= op;
-                                do {
-                                    *out++ = *from++;
-                                } while (--op);
-                                from = out - dist;      /* rest from output */
-                            }
-                        }
-                    }
-                    else {                      /* contiguous in window */
-                        from += wnext - op;
-                        if (op < len) {         /* some from window */
-                            len -= op;
-                            do {
-                                *out++ = *from++;
-                            } while (--op);
-                            from = out - dist;  /* rest from output */
-                        }
-                    }
-                    while (len > 2) {
-                        *out++ = *from++;
-                        *out++ = *from++;
-                        *out++ = *from++;
-                        len -= 3;
-                    }
-                    if (len) {
-                        *out++ = *from++;
-                        if (len > 1)
-                            *out++ = *from++;
+                            out = chunkcopy_safe(out, from, op, limit);
+                            from = window;      /* more from start of window */
+                            op = wnext;
+                            /* This (rare) case can create a situation where
+                               the first chunkcopy below must be checked.
+                             */
+                        }
+                    }
+                    if (op < len) {             /* still need some from output */
+                        out = chunkcopy_safe(out, from, op, limit);
+                        len -= op;
+                        /* When dist is small the amount of data that can be
+                           copied from the window is also small, and progress
+                           towards the dangerous end of the output buffer is
+                           also small.  This means that for trivial memsets and
+                           for chunkunroll_relaxed() a safety check is
+                           unnecessary.  However, these conditions may not be
+                           entered at all, and in that case it's possible that
+                           the main copy is near the end.
+                          */
+                        out = chunkunroll_relaxed(out, &dist, &len);
+                        out = chunkcopy_safe(out, out - dist, len, limit);
+                    } else {
+                        /* from points to window, so there is no risk of
+                           overlapping pointers requiring memset-like behaviour
+                         */
+                        out = chunkcopy_safe(out, from, len, limit);
                     }
                 }
                 else {
-                    from = out - dist;          /* copy direct from output */
-                    do {                        /* minimum length is three */
-                        *out++ = *from++;
-                        *out++ = *from++;
-                        *out++ = *from++;
-                        len -= 3;
-                    } while (len > 2);
-                    if (len) {
-                        *out++ = *from++;
-                        if (len > 1)
-                            *out++ = *from++;
-                    }
+                    /* Whole reference is in range of current output.  No
+                       range checks are necessary because we start with room
+                       for at least 258 bytes of output, so unroll and roundoff
+                       operations can write beyond `out+len` so long as they
+                       stay within 258 bytes of `out`.
+                     */
+                    out = chunkcopy_lapped_relaxed(out, dist, len);
                 }
             }
             else if ((op & 64) == 0) {          /* 2nd level distance code */
--- a/contrib/arm/inflate.c
+++ b/contrib/arm/inflate.c
@@ -84,6 +84,7 @@
 #include "inftrees.h"
 #include "inflate.h"
 #include "inffast.h"
+#include "contrib/arm/chunkcopy.h"
 
 #ifdef MAKEFIXED
 #  ifndef BUILDFIXED
@@ -405,10 +406,20 @@ unsigned copy;
 
     /* if it hasn't been done already, allocate space for the window */
     if (state->window == Z_NULL) {
+        unsigned wsize = 1U << state->wbits;
         state->window = (unsigned char FAR *)
-                        ZALLOC(strm, 1U << state->wbits,
+                        ZALLOC(strm, wsize + CHUNKCOPY_CHUNK_SIZE,
                                sizeof(unsigned char));
         if (state->window == Z_NULL) return 1;
+#ifdef INFLATE_CLEAR_UNUSED_UNDEFINED
+        /* Copies from the overflow portion of this buffer are undefined and
+           may cause analysis tools to raise a warning if we don't initialize
+           it.  However, this undefined data overwrites other undefined data
+           and is subsequently either overwritten or left deliberately
+           undefined at the end of decode; so there's really no point.
+         */
+        memset(state->window + wsize, 0, CHUNKCOPY_CHUNK_SIZE);
+#endif
     }
 
     /* if window not in use yet, initialize */
@@ -1175,17 +1186,16 @@ int flush;
                 else
                     from = state->window + (state->wnext - copy);
                 if (copy > state->length) copy = state->length;
+                if (copy > left) copy = left;
+                put = chunkcopy_safe(put, from, copy, put + left);
             }
             else {                              /* copy from output */
-                from = put - state->offset;
                 copy = state->length;
+                if (copy > left) copy = left;
+                put = chunkcopy_lapped_safe(put, state->offset, copy, put + left);
             }
-            if (copy > left) copy = left;
             left -= copy;
             state->length -= copy;
-            do {
-                *put++ = *from++;
-            } while (--copy);
             if (state->length == 0) state->mode = LEN;
             break;
         case LIT: