diff --git a/ap/lib/libpng/pnggccrd.c b/ap/lib/libpng/pnggccrd.c
new file mode 100755
index 0000000..8d81c31
--- /dev/null
+++ b/ap/lib/libpng/pnggccrd.c
@@ -0,0 +1,5397 @@
+/* pnggccrd.c - mixed C/assembler version of utilities to read a PNG file
+ *
+ * For Intel x86 CPU (Pentium-MMX or later) and GNU C compiler.
+ *
+ *     See http://www.intel.com/drg/pentiumII/appnotes/916/916.htm
+ *     and http://www.intel.com/drg/pentiumII/appnotes/923/923.htm
+ *     for Intel's performance analysis of the MMX vs. non-MMX code.
+ *
+ * libpng version 1.2.5 - October 3, 2002
+ * For conditions of distribution and use, see copyright notice in png.h
+ * Copyright (c) 1998-2002 Glenn Randers-Pehrson
+ * Copyright (c) 1998, Intel Corporation
+ *
+ * Based on MSVC code contributed by Nirav Chhatrapati, Intel Corp., 1998.
+ * Interface to libpng contributed by Gilles Vollant, 1999.
+ * GNU C port by Greg Roelofs, 1999-2001.
+ *
+ * Lines 2350-4300 converted in place with intel2gas 1.3.1:
+ *
+ *   intel2gas -mdI pnggccrd.c.partially-msvc -o pnggccrd.c
+ *
+ * and then cleaned up by hand.  See http://hermes.terminal.at/intel2gas/ .
+ *
+ * NOTE:  A sufficiently recent version of GNU as (or as.exe under DOS/Windows)
+ *        is required to assemble the newer MMX instructions such as movq.
+ *        For djgpp, see
+ *
+ *           ftp://ftp.simtel.net/pub/simtelnet/gnu/djgpp/v2gnu/bnu281b.zip
+ *
+ *        (or a later version in the same directory).  For Linux, check your
+ *        distribution's web site(s) or try these links:
+ *
+ *           http://rufus.w3.org/linux/RPM/binutils.html
+ *           http://www.debian.org/Packages/stable/devel/binutils.html
+ *           ftp://ftp.slackware.com/pub/linux/slackware/slackware/slakware/d1/
+ *             binutils.tgz
+ *
+ *        For other platforms, see the main GNU site:
+ *
+ *           ftp://ftp.gnu.org/pub/gnu/binutils/
+ *
+ *        Version 2.5.2l.15 is definitely too old...
+ */
+
+/*
+ * TEMPORARY PORTING NOTES AND CHANGELOG (mostly by Greg Roelofs)
+ * =====================================
+ *
+ * 19991006:
+ *  - fixed sign error in post-MMX cleanup code (16- & 32-bit cases)
+ *
+ * 19991007:
+ *  - additional optimizations (possible or definite):
+ *     x [DONE] write MMX code for 64-bit case (pixel_bytes == 8) [not tested]
+ *     - write MMX code for 48-bit case (pixel_bytes == 6)
+ *     - figure out what's up with 24-bit case (pixel_bytes == 3):
+ *        why subtract 8 from width_mmx in the pass 4/5 case?
+ *        (only width_mmx case) (near line 1606)
+ *     x [DONE] replace pixel_bytes within each block with the true
+ *        constant value (or are compilers smart enough to do that?)
+ *     - rewrite all MMX interlacing code so it's aligned with
+ *        the *beginning* of the row buffer, not the end.  This
+ *        would not only allow one to eliminate half of the memory
+ *        writes for odd passes (that is, pass == odd), it may also
+ *        eliminate some unaligned-data-access exceptions (assuming
+ *        there's a penalty for not aligning 64-bit accesses on
+ *        64-bit boundaries).  The only catch is that the "leftover"
+ *        pixel(s) at the end of the row would have to be saved,
+ *        but there are enough unused MMX registers in every case,
+ *        so this is not a problem.  A further benefit is that the
+ *        post-MMX cleanup code (C code) in at least some of the
+ *        cases could be done within the assembler block.
+ *  x [DONE] the "v3 v2 v1 v0 v7 v6 v5 v4" comments are confusing,
+ *     inconsistent, and don't match the MMX Programmer's Reference
+ *     Manual conventions anyway.  They should be changed to
+ *     "b7 b6 b5 b4 b3 b2 b1 b0," where b0 indicates the byte that
+ *     was lowest in memory (e.g., corresponding to a left pixel)
+ *     and b7 is the byte that was highest (e.g., a right pixel).
+ *
+ * 19991016:
+ *  - Brennan's Guide notwithstanding, gcc under Linux does *not*
+ *     want globals prefixed by underscores when referencing them--
+ *     i.e., if the variable is const4, then refer to it as const4,
+ *     not _const4.  This seems to be a djgpp-specific requirement.
+ *     Also, such variables apparently *must* be declared outside
+ *     of functions; neither static nor automatic variables work if
+ *     defined within the scope of a single function, but both
+ *     static and truly global (multi-module) variables work fine.
+ *
+ * 19991023:
+ *  - fixed png_combine_row() non-MMX replication bug (odd passes only?)
+ *  - switched from string-concatenation-with-macros to cleaner method of
+ *     renaming global variables for djgpp--i.e., always use prefixes in
+ *     inlined assembler code (== strings) and conditionally rename the
+ *     variables, not the other way around.  Hence _const4, _mask8_0, etc.
+ *
+ * 19991024:
+ *  - fixed mmxsupport()/png_do_read_interlace() first-row bug
+ *     This one was severely weird:  even though mmxsupport() doesn't touch
+ *     ebx (where "row" pointer was stored), it nevertheless managed to zero
+ *     the register (even in static/non-fPIC code--see below), which in turn
+ *     caused png_do_read_interlace() to return prematurely on the first row of
+ *     interlaced images (i.e., without expanding the interlaced pixels).
+ *     Inspection of the generated assembly code didn't turn up any clues,
+ *     although it did point at a minor optimization (i.e., get rid of
+ *     mmx_supported_local variable and just use eax).  Possibly the CPUID
+ *     instruction is more destructive than it looks?  (Not yet checked.)
+ *  - "info gcc" was next to useless, so compared fPIC and non-fPIC assembly
+ *     listings...  Apparently register spillage has to do with ebx, since
+ *     it's used to index the global offset table.  Commenting it out of the
+ *     input-reg lists in png_combine_row() eliminated compiler barfage, so
+ *     ifdef'd with __PIC__ macro:  if defined, use a global for unmask
+ *
+ * 19991107:
+ *  - verified CPUID clobberage:  12-char string constant ("GenuineIntel",
+ *     "AuthenticAMD", etc.) placed in ebx:ecx:edx.  Still need to polish.
+ *
+ * 19991120:
+ *  - made "diff" variable (now "_dif") global to simplify conversion of
+ *     filtering routines (running out of regs, sigh).  "diff" is still used
+ *     in interlacing routines, however.
+ *  - fixed up both versions of mmxsupport() (ORIG_THAT_USED_TO_CLOBBER_EBX
+ *     macro determines which is used); original not yet tested.
+ *
+ * 20000213:
+ *  - when compiling with gcc, be sure to use  -fomit-frame-pointer
+ *
+ * 20000319:
+ *  - fixed a register-name typo in png_do_read_interlace(), default (MMX) case,
+ *     pass == 4 or 5, that caused visible corruption of interlaced images
+ *
+ * 20000623:
+ *  - Various problems were reported with gcc 2.95.2 in the Cygwin environment,
+ *     many of the form "forbidden register 0 (ax) was spilled for class AREG."
+ *     This is explained at http://gcc.gnu.org/fom_serv/cache/23.html, and
+ *     Chuck Wilson supplied a patch involving dummy output registers.  See
+ *     http://sourceforge.net/bugs/?func=detailbug&bug_id=108741&group_id=5624
+ *     for the original (anonymous) SourceForge bug report.
+ *
+ * 20000706:
+ *  - Chuck Wilson passed along these remaining gcc 2.95.2 errors:
+ *       pnggccrd.c: In function `png_combine_row':
+ *       pnggccrd.c:525: more than 10 operands in `asm'
+ *       pnggccrd.c:669: more than 10 operands in `asm'
+ *       pnggccrd.c:828: more than 10 operands in `asm'
+ *       pnggccrd.c:994: more than 10 operands in `asm'
+ *       pnggccrd.c:1177: more than 10 operands in `asm'
+ *     They are all the same problem and can be worked around by using the
+ *     global _unmask variable unconditionally, not just in the -fPIC case.
+ *     Reportedly earlier versions of gcc also have the problem with more than
+ *     10 operands; they just don't report it.  Much strangeness ensues, etc.
+ *
+ * 20000729:
+ *  - enabled png_read_filter_row_mmx_up() (shortest remaining unconverted
+ *     MMX routine); began converting png_read_filter_row_mmx_sub()
+ *  - to finish remaining sections:
+ *     - clean up indentation and comments
+ *     - preload local variables
+ *     - add output and input regs (order of former determines numerical
+ *        mapping of latter)
+ *     - avoid all usage of ebx (including bx, bh, bl) register [20000823]
+ *     - remove "$" from addressing of Shift and Mask variables [20000823]
+ *
+ * 20000731:
+ *  - global union vars causing segfaults in png_read_filter_row_mmx_sub()?
+ *
+ * 20000822:
+ *  - ARGH, stupid png_read_filter_row_mmx_sub() segfault only happens with
+ *     shared-library (-fPIC) version!  Code works just fine as part of static
+ *     library.  Damn damn damn damn damn, should have tested that sooner.
+ *     ebx is getting clobbered again (explicitly this time); need to save it
+ *     on stack or rewrite asm code to avoid using it altogether.  Blargh!
+ *
+ * 20000823:
+ *  - first section was trickiest; all remaining sections have ebx -> edx now.
+ *     (-fPIC works again.)  Also added missing underscores to various Shift*
+ *     and *Mask* globals and got rid of leading "$" signs.
+ *
+ * 20000826:
+ *  - added visual separators to help navigate microscopic printed copies
+ *     (http://pobox.com/~newt/code/gpr-latest.zip, mode 10); started working
+ *     on png_read_filter_row_mmx_avg()
+ *
+ * 20000828:
+ *  - finished png_read_filter_row_mmx_avg():  only Paeth left! (930 lines...)
+ *     What the hell, did png_read_filter_row_mmx_paeth(), too.  Comments not
+ *     cleaned up/shortened in either routine, but functionality is complete
+ *     and seems to be working fine.
+ *
+ * 20000829:
+ *  - ahhh, figured out last(?) bit of gcc/gas asm-fu:  if register is listed
+ *     as an input reg (with dummy output variables, etc.), then it *cannot*
+ *     also appear in the clobber list or gcc 2.95.2 will barf.  The solution
+ *     is simple enough...
+ *
+ * 20000914:
+ *  - bug in png_read_filter_row_mmx_avg():  16-bit grayscale not handled
+ *     correctly (but 48-bit RGB just fine)
+ *
+ * 20000916:
+ *  - fixed bug in png_read_filter_row_mmx_avg(), bpp == 2 case; three errors:
+ *     - "_ShiftBpp.use = 24;"      should have been   "_ShiftBpp.use = 16;"
+ *     - "_ShiftRem.use = 40;"      should have been   "_ShiftRem.use = 48;"
+ *     - "psllq _ShiftRem, %%mm2"   should have been   "psrlq _ShiftRem, %%mm2"
+ *
+ * 20010101:
+ *  - added new png_init_mmx_flags() function (here only because it needs to
+ *     call mmxsupport(), which should probably become global png_mmxsupport());
+ *     modified other MMX routines to run conditionally (png_ptr->asm_flags)
+ *
+ * 20010103:
+ *  - renamed mmxsupport() to png_mmx_support(), with auto-set of mmx_supported,
+ *     and made it public; moved png_init_mmx_flags() to png.c as internal func
+ *
+ * 20010104:
+ *  - removed dependency on png_read_filter_row_c() (C code already duplicated
+ *     within MMX version of png_read_filter_row()) so no longer necessary to
+ *     compile it into pngrutil.o
+ *
+ * 20010310:
+ *  - fixed buffer-overrun bug in png_combine_row() C code (non-MMX)
+ *
+ * 20020304:
+ *  - eliminated incorrect use of width_mmx in pixel_bytes == 8 case
+ *
+ * STILL TO DO:
+ *     - test png_do_read_interlace() 64-bit case (pixel_bytes == 8)
+ *     - write MMX code for 48-bit case (pixel_bytes == 6)
+ *     - figure out what's up with 24-bit case (pixel_bytes == 3):
+ *        why subtract 8 from width_mmx in the pass 4/5 case?
+ *        (only width_mmx case) (near line 1606)
+ *     - rewrite all MMX interlacing code so it's aligned with beginning
+ *        of the row buffer, not the end (see 19991007 for details)
+ *     x pick one version of mmxsupport() and get rid of the other
+ *     - add error messages to any remaining bogus default cases
+ *     - enable pixel_depth == 8 cases in png_read_filter_row()? (test speed)
+ *     x add support for runtime enable/disable/query of various MMX routines
+ */
+
+#define PNG_INTERNAL
+#include "png.h"
+
+#if defined(PNG_USE_PNGGCCRD)
+
+int PNGAPI png_mmx_support(void);
+
+#ifdef PNG_USE_LOCAL_ARRAYS
+static const int FARDATA png_pass_start[7] = {0, 4, 0, 2, 0, 1, 0};
+static const int FARDATA png_pass_inc[7]   = {8, 8, 4, 4, 2, 2, 1};
+static const int FARDATA png_pass_width[7] = {8, 4, 4, 2, 2, 1, 1};
+#endif
+
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
+/* djgpp, Win32, and Cygwin add their own underscores to global variables,
+ * so define them without: */
+#if defined(__DJGPP__) || defined(WIN32) || defined(__CYGWIN__)
+#  define _mmx_supported  mmx_supported
+#  define _const4         const4
+#  define _const6         const6
+#  define _mask8_0        mask8_0
+#  define _mask16_1       mask16_1
+#  define _mask16_0       mask16_0
+#  define _mask24_2       mask24_2
+#  define _mask24_1       mask24_1
+#  define _mask24_0       mask24_0
+#  define _mask32_3       mask32_3
+#  define _mask32_2       mask32_2
+#  define _mask32_1       mask32_1
+#  define _mask32_0       mask32_0
+#  define _mask48_5       mask48_5
+#  define _mask48_4       mask48_4
+#  define _mask48_3       mask48_3
+#  define _mask48_2       mask48_2
+#  define _mask48_1       mask48_1
+#  define _mask48_0       mask48_0
+#  define _LBCarryMask    LBCarryMask
+#  define _HBClearMask    HBClearMask
+#  define _ActiveMask     ActiveMask
+#  define _ActiveMask2    ActiveMask2
+#  define _ActiveMaskEnd  ActiveMaskEnd
+#  define _ShiftBpp       ShiftBpp
+#  define _ShiftRem       ShiftRem
+#ifdef PNG_THREAD_UNSAFE_OK
+#  define _unmask         unmask
+#  define _FullLength     FullLength
+#  define _MMXLength      MMXLength
+#  define _dif            dif
+#  define _patemp         patemp
+#  define _pbtemp         pbtemp
+#  define _pctemp         pctemp
+#endif
+#endif
+
+
+/* These constants are used in the inlined MMX assembly code.
+   Ignore gcc's "At top level: defined but not used" warnings. */
+
+/* GRR 20000706:  originally _unmask was needed only when compiling with -fPIC,
+ *  since that case uses the %ebx register for indexing the Global Offset Table
+ *  and there were no other registers available.  But gcc 2.95 and later emit
+ *  "more than 10 operands in `asm'" errors when %ebx is used to preload unmask
+ *  in the non-PIC case, so we'll just use the global unconditionally now.
+ */
+#ifdef PNG_THREAD_UNSAFE_OK
+static int _unmask;
+#endif
+
+static unsigned long long _mask8_0  = 0x0102040810204080LL;
+
+static unsigned long long _mask16_1 = 0x0101020204040808LL;
+static unsigned long long _mask16_0 = 0x1010202040408080LL;
+
+static unsigned long long _mask24_2 = 0x0101010202020404LL;
+static unsigned long long _mask24_1 = 0x0408080810101020LL;
+static unsigned long long _mask24_0 = 0x2020404040808080LL;
+
+static unsigned long long _mask32_3 = 0x0101010102020202LL;
+static unsigned long long _mask32_2 = 0x0404040408080808LL;
+static unsigned long long _mask32_1 = 0x1010101020202020LL;
+static unsigned long long _mask32_0 = 0x4040404080808080LL;
+
+static unsigned long long _mask48_5 = 0x0101010101010202LL;
+static unsigned long long _mask48_4 = 0x0202020204040404LL;
+static unsigned long long _mask48_3 = 0x0404080808080808LL;
+static unsigned long long _mask48_2 = 0x1010101010102020LL;
+static unsigned long long _mask48_1 = 0x2020202040404040LL;
+static unsigned long long _mask48_0 = 0x4040808080808080LL;
+
+static unsigned long long _const4   = 0x0000000000FFFFFFLL;
+//static unsigned long long _const5 = 0x000000FFFFFF0000LL;     // NOT USED
+static unsigned long long _const6   = 0x00000000000000FFLL;
+
+// These are used in the row-filter routines and should/would be local
+//  variables if not for gcc addressing limitations.
+// WARNING: Their presence probably defeats the thread safety of libpng.
+
+#ifdef PNG_THREAD_UNSAFE_OK
+static png_uint_32  _FullLength;
+static png_uint_32  _MMXLength;
+static int          _dif;
+static int          _patemp; // temp variables for Paeth routine
+static int          _pbtemp;
+static int          _pctemp;
+#endif
+
+void /* PRIVATE */
+png_squelch_warnings(void)
+{
+#ifdef PNG_THREAD_UNSAFE_OK
+   _dif = _dif;
+   _patemp = _patemp;
+   _pbtemp = _pbtemp;
+   _pctemp = _pctemp;
+   _MMXLength = _MMXLength;
+#endif
+   _const4  = _const4;
+   _const6  = _const6;
+   _mask8_0  = _mask8_0;
+   _mask16_1 = _mask16_1;
+   _mask16_0 = _mask16_0;
+   _mask24_2 = _mask24_2;
+   _mask24_1 = _mask24_1;
+   _mask24_0 = _mask24_0;
+   _mask32_3 = _mask32_3;
+   _mask32_2 = _mask32_2;
+   _mask32_1 = _mask32_1;
+   _mask32_0 = _mask32_0;
+   _mask48_5 = _mask48_5;
+   _mask48_4 = _mask48_4;
+   _mask48_3 = _mask48_3;
+   _mask48_2 = _mask48_2;
+   _mask48_1 = _mask48_1;
+   _mask48_0 = _mask48_0;
+}
+#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
+
+
+static int _mmx_supported = 2;
+
+/*===========================================================================*/
+/*                                                                           */
+/*                       P N G _ C O M B I N E _ R O W                       */
+/*                                                                           */
+/*===========================================================================*/
+
+#if defined(PNG_HAVE_ASSEMBLER_COMBINE_ROW)
+
+#define BPP2  2
+#define BPP3  3 /* bytes per pixel (a.k.a. pixel_bytes) */
+#define BPP4  4
+#define BPP6  6 /* (defined only to help avoid cut-and-paste errors) */
+#define BPP8  8
+
+/* Combines the row recently read in with the previous row.
+   This routine takes care of alpha and transparency if requested.
+   This routine also handles the two methods of progressive display
+   of interlaced images, depending on the mask value.
+   The mask value describes which pixels are to be combined with
+   the row.  The pattern always repeats every 8 pixels, so just 8
+   bits are needed.  A one indicates the pixel is to be combined; a
+   zero indicates the pixel is to be skipped.  This is in addition
+   to any alpha or transparency value associated with the pixel.
+   If you want all pixels to be combined, pass 0xff (255) in mask. */
+
+/* Use this routine for the x86 platform - it uses a faster MMX routine
+   if the machine supports MMX. */
+
+void /* PRIVATE */
+png_combine_row(png_structp png_ptr, png_bytep row, int mask)
+{
+   png_debug(1, "in png_combine_row (pnggccrd.c)\n");
+
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
+   if (_mmx_supported == 2) {
+       /* this should have happened in png_init_mmx_flags() already */
+       png_warning(png_ptr, "asm_flags may not have been initialized");
+       png_mmx_support();
+   }
+#endif
+
+   if (mask == 0xff)
+   {
+      png_debug(2,"mask == 0xff:  doing single png_memcpy()\n");
+      png_memcpy(row, png_ptr->row_buf + 1,
+       (png_size_t)((png_ptr->width * png_ptr->row_info.pixel_depth + 7) >> 3));
+   }
+   else   /* (png_combine_row() is never called with mask == 0) */
+   {
+      switch (png_ptr->row_info.pixel_depth)
+      {
+         case 1:        /* png_ptr->row_info.pixel_depth */
+         {
+            png_bytep sp;
+            png_bytep dp;
+            int s_inc, s_start, s_end;
+            int m;
+            int shift;
+            png_uint_32 i;
+
+            sp = png_ptr->row_buf + 1;
+            dp = row;
+            m = 0x80;
+#if defined(PNG_READ_PACKSWAP_SUPPORTED)
+            if (png_ptr->transformations & PNG_PACKSWAP)
+            {
+                s_start = 0;
+                s_end = 7;
+                s_inc = 1;
+            }
+            else
+#endif
+            {
+                s_start = 7;
+                s_end = 0;
+                s_inc = -1;
+            }
+
+            shift = s_start;
+
+            for (i = 0; i < png_ptr->width; i++)
+            {
+               if (m & mask)
+               {
+                  int value;
+
+                  value = (*sp >> shift) & 0x1;
+                  *dp &= (png_byte)((0x7f7f >> (7 - shift)) & 0xff);
+                  *dp |= (png_byte)(value << shift);
+               }
+
+               if (shift == s_end)
+               {
+                  shift = s_start;
+                  sp++;
+                  dp++;
+               }
+               else
+                  shift += s_inc;
+
+               if (m == 1)
+                  m = 0x80;
+               else
+                  m >>= 1;
+            }
+            break;
+         }
+
+         case 2:        /* png_ptr->row_info.pixel_depth */
+         {
+            png_bytep sp;
+            png_bytep dp;
+            int s_start, s_end, s_inc;
+            int m;
+            int shift;
+            png_uint_32 i;
+            int value;
+
+            sp = png_ptr->row_buf + 1;
+            dp = row;
+            m = 0x80;
+#if defined(PNG_READ_PACKSWAP_SUPPORTED)
+            if (png_ptr->transformations & PNG_PACKSWAP)
+            {
+               s_start = 0;
+               s_end = 6;
+               s_inc = 2;
+            }
+            else
+#endif
+            {
+               s_start = 6;
+               s_end = 0;
+               s_inc = -2;
+            }
+
+            shift = s_start;
+
+            for (i = 0; i < png_ptr->width; i++)
+            {
+               if (m & mask)
+               {
+                  value = (*sp >> shift) & 0x3;
+                  *dp &= (png_byte)((0x3f3f >> (6 - shift)) & 0xff);
+                  *dp |= (png_byte)(value << shift);
+               }
+
+               if (shift == s_end)
+               {
+                  shift = s_start;
+                  sp++;
+                  dp++;
+               }
+               else
+                  shift += s_inc;
+               if (m == 1)
+                  m = 0x80;
+               else
+                  m >>= 1;
+            }
+            break;
+         }
+
+         case 4:        /* png_ptr->row_info.pixel_depth */
+         {
+            png_bytep sp;
+            png_bytep dp;
+            int s_start, s_end, s_inc;
+            int m;
+            int shift;
+            png_uint_32 i;
+            int value;
+
+            sp = png_ptr->row_buf + 1;
+            dp = row;
+            m = 0x80;
+#if defined(PNG_READ_PACKSWAP_SUPPORTED)
+            if (png_ptr->transformations & PNG_PACKSWAP)
+            {
+               s_start = 0;
+               s_end = 4;
+               s_inc = 4;
+            }
+            else
+#endif
+            {
+               s_start = 4;
+               s_end = 0;
+               s_inc = -4;
+            }
+            shift = s_start;
+
+            for (i = 0; i < png_ptr->width; i++)
+            {
+               if (m & mask)
+               {
+                  value = (*sp >> shift) & 0xf;
+                  *dp &= (png_byte)((0xf0f >> (4 - shift)) & 0xff);
+                  *dp |= (png_byte)(value << shift);
+               }
+
+               if (shift == s_end)
+               {
+                  shift = s_start;
+                  sp++;
+                  dp++;
+               }
+               else
+                  shift += s_inc;
+               if (m == 1)
+                  m = 0x80;
+               else
+                  m >>= 1;
+            }
+            break;
+         }
+
+         case 8:        /* png_ptr->row_info.pixel_depth */
+         {
+            png_bytep srcptr;
+            png_bytep dstptr;
+
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
+#if !defined(PNG_1_0_X)
+            if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
+                /* && _mmx_supported */ )
+#else
+            if (_mmx_supported)
+#endif
+            {
+               png_uint_32 len;
+               int diff;
+               int dummy_value_a;   // fix 'forbidden register spilled' error
+               int dummy_value_d;
+               int dummy_value_c;
+               int dummy_value_S;
+               int dummy_value_D;
+               _unmask = ~mask;            // global variable for -fPIC version
+               srcptr = png_ptr->row_buf + 1;
+               dstptr = row;
+               len  = png_ptr->width &~7;  // reduce to multiple of 8
+               diff = (int) (png_ptr->width & 7);  // amount lost
+
+               __asm__ __volatile__ (
+                  "movd      _unmask, %%mm7  \n\t" // load bit pattern
+                  "psubb     %%mm6, %%mm6    \n\t" // zero mm6
+                  "punpcklbw %%mm7, %%mm7    \n\t"
+                  "punpcklwd %%mm7, %%mm7    \n\t"
+                  "punpckldq %%mm7, %%mm7    \n\t" // fill reg with 8 masks
+
+                  "movq      _mask8_0, %%mm0 \n\t"
+                  "pand      %%mm7, %%mm0    \n\t" // nonzero if keep byte
+                  "pcmpeqb   %%mm6, %%mm0    \n\t" // zeros->1s, v versa
+
+// preload        "movl      len, %%ecx      \n\t" // load length of line
+// preload        "movl      srcptr, %%esi   \n\t" // load source
+// preload        "movl      dstptr, %%edi   \n\t" // load dest
+
+                  "cmpl      $0, %%ecx       \n\t" // len == 0 ?
+                  "je        mainloop8end    \n\t"
+
+                "mainloop8:                  \n\t"
+                  "movq      (%%esi), %%mm4  \n\t" // *srcptr
+                  "pand      %%mm0, %%mm4    \n\t"
+                  "movq      %%mm0, %%mm6    \n\t"
+                  "pandn     (%%edi), %%mm6  \n\t" // *dstptr
+                  "por       %%mm6, %%mm4    \n\t"
+                  "movq      %%mm4, (%%edi)  \n\t"
+                  "addl      $8, %%esi       \n\t" // inc by 8 bytes processed
+                  "addl      $8, %%edi       \n\t"
+                  "subl      $8, %%ecx       \n\t" // dec by 8 pixels processed
+                  "ja        mainloop8       \n\t"
+
+                "mainloop8end:               \n\t"
+// preload        "movl      diff, %%ecx     \n\t" // (diff is in eax)
+                  "movl      %%eax, %%ecx    \n\t"
+                  "cmpl      $0, %%ecx       \n\t"
+                  "jz        end8            \n\t"
+// preload        "movl      mask, %%edx     \n\t"
+                  "sall      $24, %%edx      \n\t" // make low byte, high byte
+
+                "secondloop8:                \n\t"
+                  "sall      %%edx           \n\t" // move high bit to CF
+                  "jnc       skip8           \n\t" // if CF = 0
+                  "movb      (%%esi), %%al   \n\t"
+                  "movb      %%al, (%%edi)   \n\t"
+
+                "skip8:                      \n\t"
+                  "incl      %%esi           \n\t"
+                  "incl      %%edi           \n\t"
+                  "decl      %%ecx           \n\t"
+                  "jnz       secondloop8     \n\t"
+
+                "end8:                       \n\t"
+                  "EMMS                      \n\t"  // DONE
+
+                  : "=a" (dummy_value_a),           // output regs (dummy)
+                    "=d" (dummy_value_d),
+                    "=c" (dummy_value_c),
+                    "=S" (dummy_value_S),
+                    "=D" (dummy_value_D)
+
+                  : "3" (srcptr),      // esi       // input regs
+                    "4" (dstptr),      // edi
+                    "0" (diff),        // eax
+// was (unmask)     "b"    RESERVED    // ebx       // Global Offset Table idx
+                    "2" (len),         // ecx
+                    "1" (mask)         // edx
+
+#if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
+                  : "%mm0", "%mm4", "%mm6", "%mm7"  // clobber list
+#endif
+               );
+            }
+            else /* mmx _not supported - Use modified C routine */
+#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
+            {
+               register png_uint_32 i;
+               png_uint_32 initial_val = png_pass_start[png_ptr->pass];
+                 /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
+               register int stride = png_pass_inc[png_ptr->pass];
+                 /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
+               register int rep_bytes = png_pass_width[png_ptr->pass];
+                 /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
+               png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
+               int diff = (int) (png_ptr->width & 7); /* amount lost */
+               register png_uint_32 final_val = len;  /* GRR bugfix */
+
+               srcptr = png_ptr->row_buf + 1 + initial_val;
+               dstptr = row + initial_val;
+
+               for (i = initial_val; i < final_val; i += stride)
+               {
+                  png_memcpy(dstptr, srcptr, rep_bytes);
+                  srcptr += stride;
+                  dstptr += stride;
+               }
+               if (diff)  /* number of leftover pixels:  3 for pngtest */
+               {
+                  final_val+=diff /* *BPP1 */ ;
+                  for (; i < final_val; i += stride)
+                  {
+                     if (rep_bytes > (int)(final_val-i))
+                        rep_bytes = (int)(final_val-i);
+                     png_memcpy(dstptr, srcptr, rep_bytes);
+                     srcptr += stride;
+                     dstptr += stride;
+                  }
+               }
+
+            } /* end of else (_mmx_supported) */
+
+            break;
+         }       /* end 8 bpp */
+
+         case 16:       /* png_ptr->row_info.pixel_depth */
+         {
+            png_bytep srcptr;
+            png_bytep dstptr;
+
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
+#if !defined(PNG_1_0_X)
+            if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
+                /* && _mmx_supported */ )
+#else
+            if (_mmx_supported)
+#endif
+            {
+               png_uint_32 len;
+               int diff;
+               int dummy_value_a;   // fix 'forbidden register spilled' error
+               int dummy_value_d;
+               int dummy_value_c;
+               int dummy_value_S;
+               int dummy_value_D;
+               _unmask = ~mask;            // global variable for -fPIC version
+               srcptr = png_ptr->row_buf + 1;
+               dstptr = row;
+               len  = png_ptr->width &~7;  // reduce to multiple of 8
+               diff = (int) (png_ptr->width & 7); // amount lost //
+
+               __asm__ __volatile__ (
+                  "movd      _unmask, %%mm7   \n\t" // load bit pattern
+                  "psubb     %%mm6, %%mm6     \n\t" // zero mm6
+                  "punpcklbw %%mm7, %%mm7     \n\t"
+                  "punpcklwd %%mm7, %%mm7     \n\t"
+                  "punpckldq %%mm7, %%mm7     \n\t" // fill reg with 8 masks
+
+                  "movq      _mask16_0, %%mm0 \n\t"
+                  "movq      _mask16_1, %%mm1 \n\t"
+
+                  "pand      %%mm7, %%mm0     \n\t"
+                  "pand      %%mm7, %%mm1     \n\t"
+
+                  "pcmpeqb   %%mm6, %%mm0     \n\t"
+                  "pcmpeqb   %%mm6, %%mm1     \n\t"
+
+// preload        "movl      len, %%ecx       \n\t" // load length of line
+// preload        "movl      srcptr, %%esi    \n\t" // load source
+// preload        "movl      dstptr, %%edi    \n\t" // load dest
+
+                  "cmpl      $0, %%ecx        \n\t"
+                  "jz        mainloop16end    \n\t"
+
+                "mainloop16:                  \n\t"
+                  "movq      (%%esi), %%mm4   \n\t"
+                  "pand      %%mm0, %%mm4     \n\t"
+                  "movq      %%mm0, %%mm6     \n\t"
+                  "movq      (%%edi), %%mm7   \n\t"
+                  "pandn     %%mm7, %%mm6     \n\t"
+                  "por       %%mm6, %%mm4     \n\t"
+                  "movq      %%mm4, (%%edi)   \n\t"
+
+                  "movq      8(%%esi), %%mm5  \n\t"
+                  "pand      %%mm1, %%mm5     \n\t"
+                  "movq      %%mm1, %%mm7     \n\t"
+                  "movq      8(%%edi), %%mm6  \n\t"
+                  "pandn     %%mm6, %%mm7     \n\t"
+                  "por       %%mm7, %%mm5     \n\t"
+                  "movq      %%mm5, 8(%%edi)  \n\t"
+
+                  "addl      $16, %%esi       \n\t" // inc by 16 bytes processed
+                  "addl      $16, %%edi       \n\t"
+                  "subl      $8, %%ecx        \n\t" // dec by 8 pixels processed
+                  "ja        mainloop16       \n\t"
+
+                "mainloop16end:               \n\t"
+// preload        "movl      diff, %%ecx      \n\t" // (diff is in eax)
+                  "movl      %%eax, %%ecx     \n\t"
+                  "cmpl      $0, %%ecx        \n\t"
+                  "jz        end16            \n\t"
+// preload        "movl      mask, %%edx      \n\t"
+                  "sall      $24, %%edx       \n\t" // make low byte, high byte
+
+                "secondloop16:                \n\t"
+                  "sall      %%edx            \n\t" // move high bit to CF
+                  "jnc       skip16           \n\t" // if CF = 0
+                  "movw      (%%esi), %%ax    \n\t"
+                  "movw      %%ax, (%%edi)    \n\t"
+
+                "skip16:                      \n\t"
+                  "addl      $2, %%esi        \n\t"
+                  "addl      $2, %%edi        \n\t"
+                  "decl      %%ecx            \n\t"
+                  "jnz       secondloop16     \n\t"
+
+                "end16:                       \n\t"
+                  "EMMS                       \n\t" // DONE
+
+                  : "=a" (dummy_value_a),           // output regs (dummy)
+                    "=c" (dummy_value_c),
+                    "=d" (dummy_value_d),
+                    "=S" (dummy_value_S),
+                    "=D" (dummy_value_D)
+
+                  : "0" (diff),        // eax       // input regs
+// was (unmask)     " "    RESERVED    // ebx       // Global Offset Table idx
+                    "1" (len),         // ecx
+                    "2" (mask),        // edx
+                    "3" (srcptr),      // esi
+                    "4" (dstptr)       // edi
+
+#if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
+                  : "%mm0", "%mm1", "%mm4"          // clobber list
+                  , "%mm5", "%mm6", "%mm7"
+#endif
+               );
+            }
+            else /* mmx _not supported - Use modified C routine */
+#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
+            {
+               register png_uint_32 i;
+               png_uint_32 initial_val = BPP2 * png_pass_start[png_ptr->pass];
+                 /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
+               register int stride = BPP2 * png_pass_inc[png_ptr->pass];
+                 /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
+               register int rep_bytes = BPP2 * png_pass_width[png_ptr->pass];
+                 /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
+               png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
+               int diff = (int) (png_ptr->width & 7); /* amount lost */
+               register png_uint_32 final_val = BPP2 * len;   /* GRR bugfix */
+
+               srcptr = png_ptr->row_buf + 1 + initial_val;
+               dstptr = row + initial_val;
+
+               for (i = initial_val; i < final_val; i += stride)
+               {
+                  png_memcpy(dstptr, srcptr, rep_bytes);
+                  srcptr += stride;
+                  dstptr += stride;
+               }
+               if (diff)  /* number of leftover pixels:  3 for pngtest */
+               {
+                  final_val+=diff*BPP2;
+                  for (; i < final_val; i += stride)
+                  {
+                     if (rep_bytes > (int)(final_val-i))
+                        rep_bytes = (int)(final_val-i);
+                     png_memcpy(dstptr, srcptr, rep_bytes);
+                     srcptr += stride;
+                     dstptr += stride;
+                  }
+               }
+            } /* end of else (_mmx_supported) */
+
+            break;
+         }       /* end 16 bpp */
+
+         case 24:       /* png_ptr->row_info.pixel_depth */
+         {
+            png_bytep srcptr;
+            png_bytep dstptr;
+
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
+#if !defined(PNG_1_0_X)
+            if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
+                /* && _mmx_supported */ )
+#else
+            if (_mmx_supported)
+#endif
+            {
+               png_uint_32 len;
+               int diff;
+               int dummy_value_a;   // fix 'forbidden register spilled' error
+               int dummy_value_d;
+               int dummy_value_c;
+               int dummy_value_S;
+               int dummy_value_D;
+               _unmask = ~mask;            // global variable for -fPIC version
+               srcptr = png_ptr->row_buf + 1;
+               dstptr = row;
+               len  = png_ptr->width &~7;  // reduce to multiple of 8
+               diff = (int) (png_ptr->width & 7); // amount lost //
+
+               __asm__ __volatile__ (
+                  "movd      _unmask, %%mm7   \n\t" // load bit pattern
+                  "psubb     %%mm6, %%mm6     \n\t" // zero mm6
+                  "punpcklbw %%mm7, %%mm7     \n\t"
+                  "punpcklwd %%mm7, %%mm7     \n\t"
+                  "punpckldq %%mm7, %%mm7     \n\t" // fill reg with 8 masks
+
+                  "movq      _mask24_0, %%mm0 \n\t"
+                  "movq      _mask24_1, %%mm1 \n\t"
+                  "movq      _mask24_2, %%mm2 \n\t"
+
+                  "pand      %%mm7, %%mm0     \n\t"
+                  "pand      %%mm7, %%mm1     \n\t"
+                  "pand      %%mm7, %%mm2     \n\t"
+
+                  "pcmpeqb   %%mm6, %%mm0     \n\t"
+                  "pcmpeqb   %%mm6, %%mm1     \n\t"
+                  "pcmpeqb   %%mm6, %%mm2     \n\t"
+
+// preload        "movl      len, %%ecx       \n\t" // load length of line
+// preload        "movl      srcptr, %%esi    \n\t" // load source
+// preload        "movl      dstptr, %%edi    \n\t" // load dest
+
+                  "cmpl      $0, %%ecx        \n\t"
+                  "jz        mainloop24end    \n\t"
+
+                "mainloop24:                  \n\t"
+                  "movq      (%%esi), %%mm4   \n\t"
+                  "pand      %%mm0, %%mm4     \n\t"
+                  "movq      %%mm0, %%mm6     \n\t"
+                  "movq      (%%edi), %%mm7   \n\t"
+                  "pandn     %%mm7, %%mm6     \n\t"
+                  "por       %%mm6, %%mm4     \n\t"
+                  "movq      %%mm4, (%%edi)   \n\t"
+
+                  "movq      8(%%esi), %%mm5  \n\t"
+                  "pand      %%mm1, %%mm5     \n\t"
+                  "movq      %%mm1, %%mm7     \n\t"
+                  "movq      8(%%edi), %%mm6  \n\t"
+                  "pandn     %%mm6, %%mm7     \n\t"
+                  "por       %%mm7, %%mm5     \n\t"
+                  "movq      %%mm5, 8(%%edi)  \n\t"
+
+                  "movq      16(%%esi), %%mm6 \n\t"
+                  "pand      %%mm2, %%mm6     \n\t"
+                  "movq      %%mm2, %%mm4     \n\t"
+                  "movq      16(%%edi), %%mm7 \n\t"
+                  "pandn     %%mm7, %%mm4     \n\t"
+                  "por       %%mm4, %%mm6     \n\t"
+                  "movq      %%mm6, 16(%%edi) \n\t"
+
+                  "addl      $24, %%esi       \n\t" // inc by 24 bytes processed
+                  "addl      $24, %%edi       \n\t"
+                  "subl      $8, %%ecx        \n\t" // dec by 8 pixels processed
+
+                  "ja        mainloop24       \n\t"
+
+                "mainloop24end:               \n\t"
+// preload        "movl      diff, %%ecx      \n\t" // (diff is in eax)
+                  "movl      %%eax, %%ecx     \n\t"
+                  "cmpl      $0, %%ecx        \n\t"
+                  "jz        end24            \n\t"
+// preload        "movl      mask, %%edx      \n\t"
+                  "sall      $24, %%edx       \n\t" // make low byte, high byte
+
+                "secondloop24:                \n\t"
+                  "sall      %%edx            \n\t" // move high bit to CF
+                  "jnc       skip24           \n\t" // if CF = 0
+                  "movw      (%%esi), %%ax    \n\t"
+                  "movw      %%ax, (%%edi)    \n\t"
+                  "xorl      %%eax, %%eax     \n\t"
+                  "movb      2(%%esi), %%al   \n\t"
+                  "movb      %%al, 2(%%edi)   \n\t"
+
+                "skip24:                      \n\t"
+                  "addl      $3, %%esi        \n\t"
+                  "addl      $3, %%edi        \n\t"
+                  "decl      %%ecx            \n\t"
+                  "jnz       secondloop24     \n\t"
+
+                "end24:                       \n\t"
+                  "EMMS                       \n\t" // DONE
+
+                  : "=a" (dummy_value_a),           // output regs (dummy)
+                    "=d" (dummy_value_d),
+                    "=c" (dummy_value_c),
+                    "=S" (dummy_value_S),
+                    "=D" (dummy_value_D)
+
+                  : "3" (srcptr),      // esi       // input regs
+                    "4" (dstptr),      // edi
+                    "0" (diff),        // eax
+// was (unmask)     "b"    RESERVED    // ebx       // Global Offset Table idx
+                    "2" (len),         // ecx
+                    "1" (mask)         // edx
+
+#if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
+                  : "%mm0", "%mm1", "%mm2"          // clobber list
+                  , "%mm4", "%mm5", "%mm6", "%mm7"
+#endif
+               );
+            }
+            else /* mmx _not supported - Use modified C routine */
+#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
+            {
+               register png_uint_32 i;
+               png_uint_32 initial_val = BPP3 * png_pass_start[png_ptr->pass];
+                 /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
+               register int stride = BPP3 * png_pass_inc[png_ptr->pass];
+                 /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
+               register int rep_bytes = BPP3 * png_pass_width[png_ptr->pass];
+                 /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
+               png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
+               int diff = (int) (png_ptr->width & 7); /* amount lost */
+               register png_uint_32 final_val = BPP3 * len;   /* GRR bugfix */
+
+               srcptr = png_ptr->row_buf + 1 + initial_val;
+               dstptr = row + initial_val;
+
+               for (i = initial_val; i < final_val; i += stride)
+               {
+                  png_memcpy(dstptr, srcptr, rep_bytes);
+                  srcptr += stride;
+                  dstptr += stride;
+               }
+               if (diff)  /* number of leftover pixels:  3 for pngtest */
+               {
+                  final_val+=diff*BPP3;
+                  for (; i < final_val; i += stride)
+                  {
+                     if (rep_bytes > (int)(final_val-i))
+                        rep_bytes = (int)(final_val-i);
+                     png_memcpy(dstptr, srcptr, rep_bytes);
+                     srcptr += stride;
+                     dstptr += stride;
+                  }
+               }
+            } /* end of else (_mmx_supported) */
+
+            break;
+         }       /* end 24 bpp */
+
+         case 32:       /* png_ptr->row_info.pixel_depth */
+         {
+            png_bytep srcptr;
+            png_bytep dstptr;
+
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
+#if !defined(PNG_1_0_X)
+            if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
+                /* && _mmx_supported */ )
+#else
+            if (_mmx_supported)
+#endif
+            {
+               png_uint_32 len;
+               int diff;
+               int dummy_value_a;   // fix 'forbidden register spilled' error
+               int dummy_value_d;
+               int dummy_value_c;
+               int dummy_value_S;
+               int dummy_value_D;
+               _unmask = ~mask;            // global variable for -fPIC version
+               srcptr = png_ptr->row_buf + 1;
+               dstptr = row;
+               len  = png_ptr->width &~7;  // reduce to multiple of 8
+               diff = (int) (png_ptr->width & 7); // amount lost //
+
+               __asm__ __volatile__ (
+                  "movd      _unmask, %%mm7   \n\t" // load bit pattern
+                  "psubb     %%mm6, %%mm6     \n\t" // zero mm6
+                  "punpcklbw %%mm7, %%mm7     \n\t"
+                  "punpcklwd %%mm7, %%mm7     \n\t"
+                  "punpckldq %%mm7, %%mm7     \n\t" // fill reg with 8 masks
+
+                  "movq      _mask32_0, %%mm0 \n\t"
+                  "movq      _mask32_1, %%mm1 \n\t"
+                  "movq      _mask32_2, %%mm2 \n\t"
+                  "movq      _mask32_3, %%mm3 \n\t"
+
+                  "pand      %%mm7, %%mm0     \n\t"
+                  "pand      %%mm7, %%mm1     \n\t"
+                  "pand      %%mm7, %%mm2     \n\t"
+                  "pand      %%mm7, %%mm3     \n\t"
+
+                  "pcmpeqb   %%mm6, %%mm0     \n\t"
+                  "pcmpeqb   %%mm6, %%mm1     \n\t"
+                  "pcmpeqb   %%mm6, %%mm2     \n\t"
+                  "pcmpeqb   %%mm6, %%mm3     \n\t"
+
+// preload        "movl      len, %%ecx       \n\t" // load length of line
+// preload        "movl      srcptr, %%esi    \n\t" // load source
+// preload        "movl      dstptr, %%edi    \n\t" // load dest
+
+                  "cmpl      $0, %%ecx        \n\t" // lcr
+                  "jz        mainloop32end    \n\t"
+
+                "mainloop32:                  \n\t"
+                  "movq      (%%esi), %%mm4   \n\t"
+                  "pand      %%mm0, %%mm4     \n\t"
+                  "movq      %%mm0, %%mm6     \n\t"
+                  "movq      (%%edi), %%mm7   \n\t"
+                  "pandn     %%mm7, %%mm6     \n\t"
+                  "por       %%mm6, %%mm4     \n\t"
+                  "movq      %%mm4, (%%edi)   \n\t"
+
+                  "movq      8(%%esi), %%mm5  \n\t"
+                  "pand      %%mm1, %%mm5     \n\t"
+                  "movq      %%mm1, %%mm7     \n\t"
+                  "movq      8(%%edi), %%mm6  \n\t"
+                  "pandn     %%mm6, %%mm7     \n\t"
+                  "por       %%mm7, %%mm5     \n\t"
+                  "movq      %%mm5, 8(%%edi)  \n\t"
+
+                  "movq      16(%%esi), %%mm6 \n\t"
+                  "pand      %%mm2, %%mm6     \n\t"
+                  "movq      %%mm2, %%mm4     \n\t"
+                  "movq      16(%%edi), %%mm7 \n\t"
+                  "pandn     %%mm7, %%mm4     \n\t"
+                  "por       %%mm4, %%mm6     \n\t"
+                  "movq      %%mm6, 16(%%edi) \n\t"
+
+                  "movq      24(%%esi), %%mm7 \n\t"
+                  "pand      %%mm3, %%mm7     \n\t"
+                  "movq      %%mm3, %%mm5     \n\t"
+                  "movq      24(%%edi), %%mm4 \n\t"
+                  "pandn     %%mm4, %%mm5     \n\t"
+                  "por       %%mm5, %%mm7     \n\t"
+                  "movq      %%mm7, 24(%%edi) \n\t"
+
+                  "addl      $32, %%esi       \n\t" // inc by 32 bytes processed
+                  "addl      $32, %%edi       \n\t"
+                  "subl      $8, %%ecx        \n\t" // dec by 8 pixels processed
+                  "ja        mainloop32       \n\t"
+
+                "mainloop32end:               \n\t"
+// preload        "movl      diff, %%ecx      \n\t" // (diff is in eax)
+                  "movl      %%eax, %%ecx     \n\t"
+                  "cmpl      $0, %%ecx        \n\t"
+                  "jz        end32            \n\t"
+// preload        "movl      mask, %%edx      \n\t"
+                  "sall      $24, %%edx       \n\t" // low byte => high byte
+
+                "secondloop32:                \n\t"
+                  "sall      %%edx            \n\t" // move high bit to CF
+                  "jnc       skip32           \n\t" // if CF = 0
+                  "movl      (%%esi), %%eax   \n\t"
+                  "movl      %%eax, (%%edi)   \n\t"
+
+                "skip32:                      \n\t"
+                  "addl      $4, %%esi        \n\t"
+                  "addl      $4, %%edi        \n\t"
+                  "decl      %%ecx            \n\t"
+                  "jnz       secondloop32     \n\t"
+
+                "end32:                       \n\t"
+                  "EMMS                       \n\t" // DONE
+
+                  : "=a" (dummy_value_a),           // output regs (dummy)
+                    "=d" (dummy_value_d),
+                    "=c" (dummy_value_c),
+                    "=S" (dummy_value_S),
+                    "=D" (dummy_value_D)
+
+                  : "3" (srcptr),      // esi       // input regs
+                    "4" (dstptr),      // edi
+                    "0" (diff),        // eax
+// was (unmask)     "b"    RESERVED    // ebx       // Global Offset Table idx
+                    "2" (len),         // ecx
+                    "1" (mask)         // edx
+
+#if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
+                  : "%mm0", "%mm1", "%mm2", "%mm3"  // clobber list
+                  , "%mm4", "%mm5", "%mm6", "%mm7"
+#endif
+               );
+            }
+            else /* mmx _not supported - Use modified C routine */
+#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
+            {
+               register png_uint_32 i;
+               png_uint_32 initial_val = BPP4 * png_pass_start[png_ptr->pass];
+                 /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
+               register int stride = BPP4 * png_pass_inc[png_ptr->pass];
+                 /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
+               register int rep_bytes = BPP4 * png_pass_width[png_ptr->pass];
+                 /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
+               png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
+               int diff = (int) (png_ptr->width & 7); /* amount lost */
+               register png_uint_32 final_val = BPP4 * len;   /* GRR bugfix */
+
+               srcptr = png_ptr->row_buf + 1 + initial_val;
+               dstptr = row + initial_val;
+
+               for (i = initial_val; i < final_val; i += stride)
+               {
+                  png_memcpy(dstptr, srcptr, rep_bytes);
+                  srcptr += stride;
+                  dstptr += stride;
+               }
+               if (diff)  /* number of leftover pixels:  3 for pngtest */
+               {
+                  final_val+=diff*BPP4;
+                  for (; i < final_val; i += stride)
+                  {
+                     if (rep_bytes > (int)(final_val-i))
+                        rep_bytes = (int)(final_val-i);
+                     png_memcpy(dstptr, srcptr, rep_bytes);
+                     srcptr += stride;
+                     dstptr += stride;
+                  }
+               }
+            } /* end of else (_mmx_supported) */
+
+            break;
+         }       /* end 32 bpp */
+
+         case 48:       /* png_ptr->row_info.pixel_depth */
+         {
+            png_bytep srcptr;
+            png_bytep dstptr;
+
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
+#if !defined(PNG_1_0_X)
+            if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
+                /* && _mmx_supported */ )
+#else
+            if (_mmx_supported)
+#endif
+            {
+               png_uint_32 len;
+               int diff;
+               int dummy_value_a;   // fix 'forbidden register spilled' error
+               int dummy_value_d;
+               int dummy_value_c;
+               int dummy_value_S;
+               int dummy_value_D;
+               _unmask = ~mask;            // global variable for -fPIC version
+               srcptr = png_ptr->row_buf + 1;
+               dstptr = row;
+               len  = png_ptr->width &~7;  // reduce to multiple of 8
+               diff = (int) (png_ptr->width & 7); // amount lost //
+
+               __asm__ __volatile__ (
+                  "movd      _unmask, %%mm7   \n\t" // load bit pattern
+                  "psubb     %%mm6, %%mm6     \n\t" // zero mm6
+                  "punpcklbw %%mm7, %%mm7     \n\t"
+                  "punpcklwd %%mm7, %%mm7     \n\t"
+                  "punpckldq %%mm7, %%mm7     \n\t" // fill reg with 8 masks
+
+                  "movq      _mask48_0, %%mm0 \n\t"
+                  "movq      _mask48_1, %%mm1 \n\t"
+                  "movq      _mask48_2, %%mm2 \n\t"
+                  "movq      _mask48_3, %%mm3 \n\t"
+                  "movq      _mask48_4, %%mm4 \n\t"
+                  "movq      _mask48_5, %%mm5 \n\t"
+
+                  "pand      %%mm7, %%mm0     \n\t"
+                  "pand      %%mm7, %%mm1     \n\t"
+                  "pand      %%mm7, %%mm2     \n\t"
+                  "pand      %%mm7, %%mm3     \n\t"
+                  "pand      %%mm7, %%mm4     \n\t"
+                  "pand      %%mm7, %%mm5     \n\t"
+
+                  "pcmpeqb   %%mm6, %%mm0     \n\t"
+                  "pcmpeqb   %%mm6, %%mm1     \n\t"
+                  "pcmpeqb   %%mm6, %%mm2     \n\t"
+                  "pcmpeqb   %%mm6, %%mm3     \n\t"
+                  "pcmpeqb   %%mm6, %%mm4     \n\t"
+                  "pcmpeqb   %%mm6, %%mm5     \n\t"
+
+// preload        "movl      len, %%ecx       \n\t" // load length of line
+// preload        "movl      srcptr, %%esi    \n\t" // load source
+// preload        "movl      dstptr, %%edi    \n\t" // load dest
+
+                  "cmpl      $0, %%ecx        \n\t"
+                  "jz        mainloop48end    \n\t"
+
+                "mainloop48:                  \n\t"
+                  "movq      (%%esi), %%mm7   \n\t"
+                  "pand      %%mm0, %%mm7     \n\t"
+                  "movq      %%mm0, %%mm6     \n\t"
+                  "pandn     (%%edi), %%mm6   \n\t"
+                  "por       %%mm6, %%mm7     \n\t"
+                  "movq      %%mm7, (%%edi)   \n\t"
+
+                  "movq      8(%%esi), %%mm6  \n\t"
+                  "pand      %%mm1, %%mm6     \n\t"
+                  "movq      %%mm1, %%mm7     \n\t"
+                  "pandn     8(%%edi), %%mm7  \n\t"
+                  "por       %%mm7, %%mm6     \n\t"
+                  "movq      %%mm6, 8(%%edi)  \n\t"
+
+                  "movq      16(%%esi), %%mm6 \n\t"
+                  "pand      %%mm2, %%mm6     \n\t"
+                  "movq      %%mm2, %%mm7     \n\t"
+                  "pandn     16(%%edi), %%mm7 \n\t"
+                  "por       %%mm7, %%mm6     \n\t"
+                  "movq      %%mm6, 16(%%edi) \n\t"
+
+                  "movq      24(%%esi), %%mm7 \n\t"
+                  "pand      %%mm3, %%mm7     \n\t"
+                  "movq      %%mm3, %%mm6     \n\t"
+                  "pandn     24(%%edi), %%mm6 \n\t"
+                  "por       %%mm6, %%mm7     \n\t"
+                  "movq      %%mm7, 24(%%edi) \n\t"
+
+                  "movq      32(%%esi), %%mm6 \n\t"
+                  "pand      %%mm4, %%mm6     \n\t"
+                  "movq      %%mm4, %%mm7     \n\t"
+                  "pandn     32(%%edi), %%mm7 \n\t"
+                  "por       %%mm7, %%mm6     \n\t"
+                  "movq      %%mm6, 32(%%edi) \n\t"
+
+                  "movq      40(%%esi), %%mm7 \n\t"
+                  "pand      %%mm5, %%mm7     \n\t"
+                  "movq      %%mm5, %%mm6     \n\t"
+                  "pandn     40(%%edi), %%mm6 \n\t"
+                  "por       %%mm6, %%mm7     \n\t"
+                  "movq      %%mm7, 40(%%edi) \n\t"
+
+                  "addl      $48, %%esi       \n\t" // inc by 48 bytes processed
+                  "addl      $48, %%edi       \n\t"
+                  "subl      $8, %%ecx        \n\t" // dec by 8 pixels processed
+
+                  "ja        mainloop48       \n\t"
+
+                "mainloop48end:               \n\t"
+// preload        "movl      diff, %%ecx      \n\t" // (diff is in eax)
+                  "movl      %%eax, %%ecx     \n\t"
+                  "cmpl      $0, %%ecx        \n\t"
+                  "jz        end48            \n\t"
+// preload        "movl      mask, %%edx      \n\t"
+                  "sall      $24, %%edx       \n\t" // make low byte, high byte
+
+                "secondloop48:                \n\t"
+                  "sall      %%edx            \n\t" // move high bit to CF
+                  "jnc       skip48           \n\t" // if CF = 0
+                  "movl      (%%esi), %%eax   \n\t"
+                  "movl      %%eax, (%%edi)   \n\t"
+
+                "skip48:                      \n\t"
+                  "addl      $4, %%esi        \n\t"
+                  "addl      $4, %%edi        \n\t"
+                  "decl      %%ecx            \n\t"
+                  "jnz       secondloop48     \n\t"
+
+                "end48:                       \n\t"
+                  "EMMS                       \n\t" // DONE
+
+                  : "=a" (dummy_value_a),           // output regs (dummy)
+                    "=d" (dummy_value_d),
+                    "=c" (dummy_value_c),
+                    "=S" (dummy_value_S),
+                    "=D" (dummy_value_D)
+
+                  : "3" (srcptr),      // esi       // input regs
+                    "4" (dstptr),      // edi
+                    "0" (diff),        // eax
+// was (unmask)     "b"    RESERVED    // ebx       // Global Offset Table idx
+                    "2" (len),         // ecx
+                    "1" (mask)         // edx
+
+#if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
+                  : "%mm0", "%mm1", "%mm2", "%mm3"  // clobber list
+                  , "%mm4", "%mm5", "%mm6", "%mm7"
+#endif
+               );
+            }
+            else /* mmx _not supported - Use modified C routine */
+#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
+            {
+               register png_uint_32 i;
+               png_uint_32 initial_val = BPP6 * png_pass_start[png_ptr->pass];
+                 /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
+               register int stride = BPP6 * png_pass_inc[png_ptr->pass];
+                 /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
+               register int rep_bytes = BPP6 * png_pass_width[png_ptr->pass];
+                 /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
+               png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
+               int diff = (int) (png_ptr->width & 7); /* amount lost */
+               register png_uint_32 final_val = BPP6 * len;   /* GRR bugfix */
+
+               srcptr = png_ptr->row_buf + 1 + initial_val;
+               dstptr = row + initial_val;
+
+               for (i = initial_val; i < final_val; i += stride)
+               {
+                  png_memcpy(dstptr, srcptr, rep_bytes);
+                  srcptr += stride;
+                  dstptr += stride;
+               }
+               if (diff)  /* number of leftover pixels:  3 for pngtest */
+               {
+                  final_val+=diff*BPP6;
+                  for (; i < final_val; i += stride)
+                  {
+                     if (rep_bytes > (int)(final_val-i))
+                        rep_bytes = (int)(final_val-i);
+                     png_memcpy(dstptr, srcptr, rep_bytes);
+                     srcptr += stride;
+                     dstptr += stride;
+                  }
+               }
+            } /* end of else (_mmx_supported) */
+
+            break;
+         }       /* end 48 bpp */
+
+         case 64:       /* png_ptr->row_info.pixel_depth */
+         {
+            png_bytep srcptr;
+            png_bytep dstptr;
+            register png_uint_32 i;
+            png_uint_32 initial_val = BPP8 * png_pass_start[png_ptr->pass];
+              /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
+            register int stride = BPP8 * png_pass_inc[png_ptr->pass];
+              /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
+            register int rep_bytes = BPP8 * png_pass_width[png_ptr->pass];
+              /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
+            png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
+            int diff = (int) (png_ptr->width & 7); /* amount lost */
+            register png_uint_32 final_val = BPP8 * len;   /* GRR bugfix */
+
+            srcptr = png_ptr->row_buf + 1 + initial_val;
+            dstptr = row + initial_val;
+
+            for (i = initial_val; i < final_val; i += stride)
+            {
+               png_memcpy(dstptr, srcptr, rep_bytes);
+               srcptr += stride;
+               dstptr += stride;
+            }
+            if (diff)  /* number of leftover pixels:  3 for pngtest */
+            {
+               final_val+=diff*BPP8;
+               for (; i < final_val; i += stride)
+               {
+                  if (rep_bytes > (int)(final_val-i))
+                     rep_bytes = (int)(final_val-i);
+                  png_memcpy(dstptr, srcptr, rep_bytes);
+                  srcptr += stride;
+                  dstptr += stride;
+               }
+            }
+
+            break;
+         }       /* end 64 bpp */
+
+         default: /* png_ptr->row_info.pixel_depth != 1,2,4,8,16,24,32,48,64 */
+         {
+            /* this should never happen */
+            png_warning(png_ptr, "Invalid row_info.pixel_depth in pnggccrd");
+            break;
+         }
+      } /* end switch (png_ptr->row_info.pixel_depth) */
+
+   } /* end if (non-trivial mask) */
+
+} /* end png_combine_row() */
+
+#endif /* PNG_HAVE_ASSEMBLER_COMBINE_ROW */
+
+
+
+
+/*===========================================================================*/
+/*                                                                           */
+/*                 P N G _ D O _ R E A D _ I N T E R L A C E                 */
+/*                                                                           */
+/*===========================================================================*/
+
+#if defined(PNG_READ_INTERLACING_SUPPORTED)
+#if defined(PNG_HAVE_ASSEMBLER_READ_INTERLACE)
+
+/* png_do_read_interlace() is called after any 16-bit to 8-bit conversion
+ * has taken place.  [GRR: what other steps come before and/or after?]
+ */
+
+void /* PRIVATE */
+png_do_read_interlace(png_structp png_ptr)
+{
+   png_row_infop row_info = &(png_ptr->row_info);
+   png_bytep row = png_ptr->row_buf + 1;
+   int pass = png_ptr->pass;
+#if defined(PNG_READ_PACKSWAP_SUPPORTED)
+   png_uint_32 transformations = png_ptr->transformations;
+#endif
+
+   png_debug(1, "in png_do_read_interlace (pnggccrd.c)\n");
+
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
+   if (_mmx_supported == 2) {
+#if !defined(PNG_1_0_X)
+       /* this should have happened in png_init_mmx_flags() already */
+       png_warning(png_ptr, "asm_flags may not have been initialized");
+#endif
+       png_mmx_support();
+   }
+#endif
+
+   if (row != NULL && row_info != NULL)
+   {
+      png_uint_32 final_width;
+
+      final_width = row_info->width * png_pass_inc[pass];
+
+      switch (row_info->pixel_depth)
+      {
+         case 1:
+         {
+            png_bytep sp, dp;
+            int sshift, dshift;
+            int s_start, s_end, s_inc;
+            png_byte v;
+            png_uint_32 i;
+            int j;
+
+            sp = row + (png_size_t)((row_info->width - 1) >> 3);
+            dp = row + (png_size_t)((final_width - 1) >> 3);
+#if defined(PNG_READ_PACKSWAP_SUPPORTED)
+            if (transformations & PNG_PACKSWAP)
+            {
+               sshift = (int)((row_info->width + 7) & 7);
+               dshift = (int)((final_width + 7) & 7);
+               s_start = 7;
+               s_end = 0;
+               s_inc = -1;
+            }
+            else
+#endif
+            {
+               sshift = 7 - (int)((row_info->width + 7) & 7);
+               dshift = 7 - (int)((final_width + 7) & 7);
+               s_start = 0;
+               s_end = 7;
+               s_inc = 1;
+            }
+
+            for (i = row_info->width; i; i--)
+            {
+               v = (png_byte)((*sp >> sshift) & 0x1);
+               for (j = 0; j < png_pass_inc[pass]; j++)
+               {
+                  *dp &= (png_byte)((0x7f7f >> (7 - dshift)) & 0xff);
+                  *dp |= (png_byte)(v << dshift);
+                  if (dshift == s_end)
+                  {
+                     dshift = s_start;
+                     dp--;
+                  }
+                  else
+                     dshift += s_inc;
+               }
+               if (sshift == s_end)
+               {
+                  sshift = s_start;
+                  sp--;
+               }
+               else
+                  sshift += s_inc;
+            }
+            break;
+         }
+
+         case 2:
+         {
+            png_bytep sp, dp;
+            int sshift, dshift;
+            int s_start, s_end, s_inc;
+            png_uint_32 i;
+
+            sp = row + (png_size_t)((row_info->width - 1) >> 2);
+            dp = row + (png_size_t)((final_width - 1) >> 2);
+#if defined(PNG_READ_PACKSWAP_SUPPORTED)
+            if (transformations & PNG_PACKSWAP)
+            {
+               sshift = (png_size_t)(((row_info->width + 3) & 3) << 1);
+               dshift = (png_size_t)(((final_width + 3) & 3) << 1);
+               s_start = 6;
+               s_end = 0;
+               s_inc = -2;
+            }
+            else
+#endif
+            {
+               sshift = (png_size_t)((3 - ((row_info->width + 3) & 3)) << 1);
+               dshift = (png_size_t)((3 - ((final_width + 3) & 3)) << 1);
+               s_start = 0;
+               s_end = 6;
+               s_inc = 2;
+            }
+
+            for (i = row_info->width; i; i--)
+            {
+               png_byte v;
+               int j;
+
+               v = (png_byte)((*sp >> sshift) & 0x3);
+               for (j = 0; j < png_pass_inc[pass]; j++)
+               {
+                  *dp &= (png_byte)((0x3f3f >> (6 - dshift)) & 0xff);
+                  *dp |= (png_byte)(v << dshift);
+                  if (dshift == s_end)
+                  {
+                     dshift = s_start;
+                     dp--;
+                  }
+                  else
+                     dshift += s_inc;
+               }
+               if (sshift == s_end)
+               {
+                  sshift = s_start;
+                  sp--;
+               }
+               else
+                  sshift += s_inc;
+            }
+            break;
+         }
+
+         case 4:
+         {
+            png_bytep sp, dp;
+            int sshift, dshift;
+            int s_start, s_end, s_inc;
+            png_uint_32 i;
+
+            sp = row + (png_size_t)((row_info->width - 1) >> 1);
+            dp = row + (png_size_t)((final_width - 1) >> 1);
+#if defined(PNG_READ_PACKSWAP_SUPPORTED)
+            if (transformations & PNG_PACKSWAP)
+            {
+               sshift = (png_size_t)(((row_info->width + 1) & 1) << 2);
+               dshift = (png_size_t)(((final_width + 1) & 1) << 2);
+               s_start = 4;
+               s_end = 0;
+               s_inc = -4;
+            }
+            else
+#endif
+            {
+               sshift = (png_size_t)((1 - ((row_info->width + 1) & 1)) << 2);
+               dshift = (png_size_t)((1 - ((final_width + 1) & 1)) << 2);
+               s_start = 0;
+               s_end = 4;
+               s_inc = 4;
+            }
+
+            for (i = row_info->width; i; i--)
+            {
+               png_byte v;
+               int j;
+
+               v = (png_byte)((*sp >> sshift) & 0xf);
+               for (j = 0; j < png_pass_inc[pass]; j++)
+               {
+                  *dp &= (png_byte)((0xf0f >> (4 - dshift)) & 0xff);
+                  *dp |= (png_byte)(v << dshift);
+                  if (dshift == s_end)
+                  {
+                     dshift = s_start;
+                     dp--;
+                  }
+                  else
+                     dshift += s_inc;
+               }
+               if (sshift == s_end)
+               {
+                  sshift = s_start;
+                  sp--;
+               }
+               else
+                  sshift += s_inc;
+            }
+            break;
+         }
+
+       /*====================================================================*/
+
+         default: /* 8-bit or larger (this is where the routine is modified) */
+         {
+#if 0
+//          static unsigned long long _const4 = 0x0000000000FFFFFFLL;  no good
+//          static unsigned long long const4 = 0x0000000000FFFFFFLL;   no good
+//          unsigned long long _const4 = 0x0000000000FFFFFFLL;         no good
+//          unsigned long long const4 = 0x0000000000FFFFFFLL;          no good
+#endif
+            png_bytep sptr, dp;
+            png_uint_32 i;
+            png_size_t pixel_bytes;
+            int width = (int)row_info->width;
+
+            pixel_bytes = (row_info->pixel_depth >> 3);
+
+            /* point sptr at the last pixel in the pre-expanded row: */
+            sptr = row + (width - 1) * pixel_bytes;
+
+            /* point dp at the last pixel position in the expanded row: */
+            dp = row + (final_width - 1) * pixel_bytes;
+
+            /* New code by Nirav Chhatrapati - Intel Corporation */
+
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
+#if !defined(PNG_1_0_X)
+            if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_INTERLACE)
+                /* && _mmx_supported */ )
+#else
+            if (_mmx_supported)
+#endif
+            {
+               //--------------------------------------------------------------
+               if (pixel_bytes == 3)
+               {
+                  if (((pass == 0) || (pass == 1)) && width)
+                  {
+                     int dummy_value_c;   // fix 'forbidden register spilled'
+                     int dummy_value_S;
+                     int dummy_value_D;
+
+                     __asm__ __volatile__ (
+                        "subl $21, %%edi         \n\t"
+                                     // (png_pass_inc[pass] - 1)*pixel_bytes
+
+                     ".loop3_pass0:              \n\t"
+                        "movd (%%esi), %%mm0     \n\t" // x x x x x 2 1 0
+                        "pand _const4, %%mm0     \n\t" // z z z z z 2 1 0
+                        "movq %%mm0, %%mm1       \n\t" // z z z z z 2 1 0
+                        "psllq $16, %%mm0        \n\t" // z z z 2 1 0 z z
+                        "movq %%mm0, %%mm2       \n\t" // z z z 2 1 0 z z
+                        "psllq $24, %%mm0        \n\t" // 2 1 0 z z z z z
+                        "psrlq $8, %%mm1         \n\t" // z z z z z z 2 1
+                        "por %%mm2, %%mm0        \n\t" // 2 1 0 2 1 0 z z
+                        "por %%mm1, %%mm0        \n\t" // 2 1 0 2 1 0 2 1
+                        "movq %%mm0, %%mm3       \n\t" // 2 1 0 2 1 0 2 1
+                        "psllq $16, %%mm0        \n\t" // 0 2 1 0 2 1 z z
+                        "movq %%mm3, %%mm4       \n\t" // 2 1 0 2 1 0 2 1
+                        "punpckhdq %%mm0, %%mm3  \n\t" // 0 2 1 0 2 1 0 2
+                        "movq %%mm4, 16(%%edi)   \n\t"
+                        "psrlq $32, %%mm0        \n\t" // z z z z 0 2 1 0
+                        "movq %%mm3, 8(%%edi)    \n\t"
+                        "punpckldq %%mm4, %%mm0  \n\t" // 1 0 2 1 0 2 1 0
+                        "subl $3, %%esi          \n\t"
+                        "movq %%mm0, (%%edi)     \n\t"
+                        "subl $24, %%edi         \n\t"
+                        "decl %%ecx              \n\t"
+                        "jnz .loop3_pass0        \n\t"
+                        "EMMS                    \n\t" // DONE
+
+                        : "=c" (dummy_value_c),        // output regs (dummy)
+                          "=S" (dummy_value_S),
+                          "=D" (dummy_value_D)
+
+                        : "1" (sptr),      // esi      // input regs
+                          "2" (dp),        // edi
+                          "0" (width)      // ecx
+// doesn't work           "i" (0x0000000000FFFFFFLL)   // %1 (a.k.a. _const4)
+
+#if 0  /* %mm0, ..., %mm4 not supported by gcc 2.7.2.3 or egcs 1.1 */
+                        : "%mm0", "%mm1", "%mm2"       // clobber list
+                        , "%mm3", "%mm4"
+#endif
+                     );
+                  }
+                  else if (((pass == 2) || (pass == 3)) && width)
+                  {
+                     int dummy_value_c;   // fix 'forbidden register spilled'
+                     int dummy_value_S;
+                     int dummy_value_D;
+
+                     __asm__ __volatile__ (
+                        "subl $9, %%edi          \n\t"
+                                     // (png_pass_inc[pass] - 1)*pixel_bytes
+
+                     ".loop3_pass2:              \n\t"
+                        "movd (%%esi), %%mm0     \n\t" // x x x x x 2 1 0
+                        "pand _const4, %%mm0     \n\t" // z z z z z 2 1 0
+                        "movq %%mm0, %%mm1       \n\t" // z z z z z 2 1 0
+                        "psllq $16, %%mm0        \n\t" // z z z 2 1 0 z z
+                        "movq %%mm0, %%mm2       \n\t" // z z z 2 1 0 z z
+                        "psllq $24, %%mm0        \n\t" // 2 1 0 z z z z z
+                        "psrlq $8, %%mm1         \n\t" // z z z z z z 2 1
+                        "por %%mm2, %%mm0        \n\t" // 2 1 0 2 1 0 z z
+                        "por %%mm1, %%mm0        \n\t" // 2 1 0 2 1 0 2 1
+                        "movq %%mm0, 4(%%edi)    \n\t"
+                        "psrlq $16, %%mm0        \n\t" // z z 2 1 0 2 1 0
+                        "subl $3, %%esi          \n\t"
+                        "movd %%mm0, (%%edi)     \n\t"
+                        "subl $12, %%edi         \n\t"
+                        "decl %%ecx              \n\t"
+                        "jnz .loop3_pass2        \n\t"
+                        "EMMS                    \n\t" // DONE
+
+                        : "=c" (dummy_value_c),        // output regs (dummy)
+                          "=S" (dummy_value_S),
+                          "=D" (dummy_value_D)
+
+                        : "1" (sptr),      // esi      // input regs
+                          "2" (dp),        // edi
+                          "0" (width)      // ecx
+
+#if 0  /* %mm0, ..., %mm2 not supported by gcc 2.7.2.3 or egcs 1.1 */
+                        : "%mm0", "%mm1", "%mm2"       // clobber list
+#endif
+                     );
+                  }
+                  else if (width) /* && ((pass == 4) || (pass == 5)) */
+                  {
+                     int width_mmx = ((width >> 1) << 1) - 8;   // GRR:  huh?
+                     if (width_mmx < 0)
+                         width_mmx = 0;
+                     width -= width_mmx;        // 8 or 9 pix, 24 or 27 bytes
+                     if (width_mmx)
+                     {
+                        // png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1};
+                        // sptr points at last pixel in pre-expanded row
+                        // dp points at last pixel position in expanded row
+                        int dummy_value_c;  // fix 'forbidden register spilled'
+                        int dummy_value_S;
+                        int dummy_value_D;
+
+                        __asm__ __volatile__ (
+                           "subl $3, %%esi          \n\t"
+                           "subl $9, %%edi          \n\t"
+                                        // (png_pass_inc[pass] + 1)*pixel_bytes
+
+                        ".loop3_pass4:              \n\t"
+                           "movq (%%esi), %%mm0     \n\t" // x x 5 4 3 2 1 0
+                           "movq %%mm0, %%mm1       \n\t" // x x 5 4 3 2 1 0
+                           "movq %%mm0, %%mm2       \n\t" // x x 5 4 3 2 1 0
+                           "psllq $24, %%mm0        \n\t" // 4 3 2 1 0 z z z
+                           "pand _const4, %%mm1     \n\t" // z z z z z 2 1 0
+                           "psrlq $24, %%mm2        \n\t" // z z z x x 5 4 3
+                           "por %%mm1, %%mm0        \n\t" // 4 3 2 1 0 2 1 0
+                           "movq %%mm2, %%mm3       \n\t" // z z z x x 5 4 3
+                           "psllq $8, %%mm2         \n\t" // z z x x 5 4 3 z
+                           "movq %%mm0, (%%edi)     \n\t"
+                           "psrlq $16, %%mm3        \n\t" // z z z z z x x 5
+                           "pand _const6, %%mm3     \n\t" // z z z z z z z 5
+                           "por %%mm3, %%mm2        \n\t" // z z x x 5 4 3 5
+                           "subl $6, %%esi          \n\t"
+                           "movd %%mm2, 8(%%edi)    \n\t"
+                           "subl $12, %%edi         \n\t"
+                           "subl $2, %%ecx          \n\t"
+                           "jnz .loop3_pass4        \n\t"
+                           "EMMS                    \n\t" // DONE
+
+                           : "=c" (dummy_value_c),        // output regs (dummy)
+                             "=S" (dummy_value_S),
+                             "=D" (dummy_value_D)
+
+                           : "1" (sptr),      // esi      // input regs
+                             "2" (dp),        // edi
+                             "0" (width_mmx)  // ecx
+
+#if 0  /* %mm0, ..., %mm3 not supported by gcc 2.7.2.3 or egcs 1.1 */
+                           : "%mm0", "%mm1"               // clobber list
+                           , "%mm2", "%mm3"
+#endif
+                        );
+                     }
+
+                     sptr -= width_mmx*3;
+                     dp -= width_mmx*6;
+                     for (i = width; i; i--)
+                     {
+                        png_byte v[8];
+                        int j;
+
+                        png_memcpy(v, sptr, 3);
+                        for (j = 0; j < png_pass_inc[pass]; j++)
+                        {
+                           png_memcpy(dp, v, 3);
+                           dp -= 3;
+                        }
+                        sptr -= 3;
+                     }
+                  }
+               } /* end of pixel_bytes == 3 */
+
+               //--------------------------------------------------------------
+               else if (pixel_bytes == 1)
+               {
+                  if (((pass == 0) || (pass == 1)) && width)
+                  {
+                     int width_mmx = ((width >> 2) << 2);
+                     width -= width_mmx;        // 0-3 pixels => 0-3 bytes
+                     if (width_mmx)
+                     {
+                        int dummy_value_c;  // fix 'forbidden register spilled'
+                        int dummy_value_S;
+                        int dummy_value_D;
+
+                        __asm__ __volatile__ (
+                           "subl $3, %%esi          \n\t"
+                           "subl $31, %%edi         \n\t"
+
+                        ".loop1_pass0:              \n\t"
+                           "movd (%%esi), %%mm0     \n\t" // x x x x 3 2 1 0
+                           "movq %%mm0, %%mm1       \n\t" // x x x x 3 2 1 0
+                           "punpcklbw %%mm0, %%mm0  \n\t" // 3 3 2 2 1 1 0 0
+                           "movq %%mm0, %%mm2       \n\t" // 3 3 2 2 1 1 0 0
+                           "punpcklwd %%mm0, %%mm0  \n\t" // 1 1 1 1 0 0 0 0
+                           "movq %%mm0, %%mm3       \n\t" // 1 1 1 1 0 0 0 0
+                           "punpckldq %%mm0, %%mm0  \n\t" // 0 0 0 0 0 0 0 0
+                           "punpckhdq %%mm3, %%mm3  \n\t" // 1 1 1 1 1 1 1 1
+                           "movq %%mm0, (%%edi)     \n\t"
+                           "punpckhwd %%mm2, %%mm2  \n\t" // 3 3 3 3 2 2 2 2
+                           "movq %%mm3, 8(%%edi)    \n\t"
+                           "movq %%mm2, %%mm4       \n\t" // 3 3 3 3 2 2 2 2
+                           "punpckldq %%mm2, %%mm2  \n\t" // 2 2 2 2 2 2 2 2
+                           "punpckhdq %%mm4, %%mm4  \n\t" // 3 3 3 3 3 3 3 3
+                           "movq %%mm2, 16(%%edi)   \n\t"
+                           "subl $4, %%esi          \n\t"
+                           "movq %%mm4, 24(%%edi)   \n\t"
+                           "subl $32, %%edi         \n\t"
+                           "subl $4, %%ecx          \n\t"
+                           "jnz .loop1_pass0        \n\t"
+                           "EMMS                    \n\t" // DONE
+
+                           : "=c" (dummy_value_c),        // output regs (dummy)
+                             "=S" (dummy_value_S),
+                             "=D" (dummy_value_D)
+
+                           : "1" (sptr),      // esi      // input regs
+                             "2" (dp),        // edi
+                             "0" (width_mmx)  // ecx
+
+#if 0  /* %mm0, ..., %mm4 not supported by gcc 2.7.2.3 or egcs 1.1 */
+                           : "%mm0", "%mm1", "%mm2"       // clobber list
+                           , "%mm3", "%mm4"
+#endif
+                        );
+                     }
+
+                     sptr -= width_mmx;
+                     dp -= width_mmx*8;
+                     for (i = width; i; i--)
+                     {
+                        int j;
+
+                       /* I simplified this part in version 1.0.4e
+                        * here and in several other instances where
+                        * pixel_bytes == 1  -- GR-P
+                        *
+                        * Original code:
+                        *
+                        * png_byte v[8];
+                        * png_memcpy(v, sptr, pixel_bytes);
+                        * for (j = 0; j < png_pass_inc[pass]; j++)
+                        * {
+                        *    png_memcpy(dp, v, pixel_bytes);
+                        *    dp -= pixel_bytes;
+                        * }
+                        * sptr -= pixel_bytes;
+                        *
+                        * Replacement code is in the next three lines:
+                        */
+
+                        for (j = 0; j < png_pass_inc[pass]; j++)
+                        {
+                           *dp-- = *sptr;
+                        }
+                        --sptr;
+                     }
+                  }
+                  else if (((pass == 2) || (pass == 3)) && width)
+                  {
+                     int width_mmx = ((width >> 2) << 2);
+                     width -= width_mmx;        // 0-3 pixels => 0-3 bytes
+                     if (width_mmx)
+                     {
+                        int dummy_value_c;  // fix 'forbidden register spilled'
+                        int dummy_value_S;
+                        int dummy_value_D;
+
+                        __asm__ __volatile__ (
+                           "subl $3, %%esi          \n\t"
+                           "subl $15, %%edi         \n\t"
+
+                        ".loop1_pass2:              \n\t"
+                           "movd (%%esi), %%mm0     \n\t" // x x x x 3 2 1 0
+                           "punpcklbw %%mm0, %%mm0  \n\t" // 3 3 2 2 1 1 0 0
+                           "movq %%mm0, %%mm1       \n\t" // 3 3 2 2 1 1 0 0
+                           "punpcklwd %%mm0, %%mm0  \n\t" // 1 1 1 1 0 0 0 0
+                           "punpckhwd %%mm1, %%mm1  \n\t" // 3 3 3 3 2 2 2 2
+                           "movq %%mm0, (%%edi)     \n\t"
+                           "subl $4, %%esi          \n\t"
+                           "movq %%mm1, 8(%%edi)    \n\t"
+                           "subl $16, %%edi         \n\t"
+                           "subl $4, %%ecx          \n\t"
+                           "jnz .loop1_pass2        \n\t"
+                           "EMMS                    \n\t" // DONE
+
+                           : "=c" (dummy_value_c),        // output regs (dummy)
+                             "=S" (dummy_value_S),
+                             "=D" (dummy_value_D)
+
+                           : "1" (sptr),      // esi      // input regs
+                             "2" (dp),        // edi
+                             "0" (width_mmx)  // ecx
+
+#if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
+                           : "%mm0", "%mm1"               // clobber list
+#endif
+                        );
+                     }
+
+                     sptr -= width_mmx;
+                     dp -= width_mmx*4;
+                     for (i = width; i; i--)
+                     {
+                        int j;
+
+                        for (j = 0; j < png_pass_inc[pass]; j++)
+                        {
+                           *dp-- = *sptr;
+                        }
+                        --sptr;
+                     }
+                  }
+                  else if (width)  /* && ((pass == 4) || (pass == 5)) */
+                  {
+                     int width_mmx = ((width >> 3) << 3);
+                     width -= width_mmx;        // 0-3 pixels => 0-3 bytes
+                     if (width_mmx)
+                     {
+                        int dummy_value_c;  // fix 'forbidden register spilled'
+                        int dummy_value_S;
+                        int dummy_value_D;
+
+                        __asm__ __volatile__ (
+                           "subl $7, %%esi          \n\t"
+                           "subl $15, %%edi         \n\t"
+
+                        ".loop1_pass4:              \n\t"
+                           "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
+                           "movq %%mm0, %%mm1       \n\t" // 7 6 5 4 3 2 1 0
+                           "punpcklbw %%mm0, %%mm0  \n\t" // 3 3 2 2 1 1 0 0
+                           "punpckhbw %%mm1, %%mm1  \n\t" // 7 7 6 6 5 5 4 4
+                           "movq %%mm1, 8(%%edi)    \n\t"
+                           "subl $8, %%esi          \n\t"
+                           "movq %%mm0, (%%edi)     \n\t"
+                           "subl $16, %%edi         \n\t"
+                           "subl $8, %%ecx          \n\t"
+                           "jnz .loop1_pass4        \n\t"
+                           "EMMS                    \n\t" // DONE
+
+                           : "=c" (dummy_value_c),        // output regs (none)
+                             "=S" (dummy_value_S),
+                             "=D" (dummy_value_D)
+
+                           : "1" (sptr),      // esi      // input regs
+                             "2" (dp),        // edi
+                             "0" (width_mmx)  // ecx
+
+#if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
+                           : "%mm0", "%mm1"               // clobber list
+#endif
+                        );
+                     }
+
+                     sptr -= width_mmx;
+                     dp -= width_mmx*2;
+                     for (i = width; i; i--)
+                     {
+                        int j;
+
+                        for (j = 0; j < png_pass_inc[pass]; j++)
+                        {
+                           *dp-- = *sptr;
+                        }
+                        --sptr;
+                     }
+                  }
+               } /* end of pixel_bytes == 1 */
+
+               //--------------------------------------------------------------
+               else if (pixel_bytes == 2)
+               {
+                  if (((pass == 0) || (pass == 1)) && width)
+                  {
+                     int width_mmx = ((width >> 1) << 1);
+                     width -= width_mmx;        // 0,1 pixels => 0,2 bytes
+                     if (width_mmx)
+                     {
+                        int dummy_value_c;  // fix 'forbidden register spilled'
+                        int dummy_value_S;
+                        int dummy_value_D;
+
+                        __asm__ __volatile__ (
+                           "subl $2, %%esi          \n\t"
+                           "subl $30, %%edi         \n\t"
+
+                        ".loop2_pass0:              \n\t"
+                           "movd (%%esi), %%mm0     \n\t" // x x x x 3 2 1 0
+                           "punpcklwd %%mm0, %%mm0  \n\t" // 3 2 3 2 1 0 1 0
+                           "movq %%mm0, %%mm1       \n\t" // 3 2 3 2 1 0 1 0
+                           "punpckldq %%mm0, %%mm0  \n\t" // 1 0 1 0 1 0 1 0
+                           "punpckhdq %%mm1, %%mm1  \n\t" // 3 2 3 2 3 2 3 2
+                           "movq %%mm0, (%%edi)     \n\t"
+                           "movq %%mm0, 8(%%edi)    \n\t"
+                           "movq %%mm1, 16(%%edi)   \n\t"
+                           "subl $4, %%esi          \n\t"
+                           "movq %%mm1, 24(%%edi)   \n\t"
+                           "subl $32, %%edi         \n\t"
+                           "subl $2, %%ecx          \n\t"
+                           "jnz .loop2_pass0        \n\t"
+                           "EMMS                    \n\t" // DONE
+
+                           : "=c" (dummy_value_c),        // output regs (dummy)
+                             "=S" (dummy_value_S),
+                             "=D" (dummy_value_D)
+
+                           : "1" (sptr),      // esi      // input regs
+                             "2" (dp),        // edi
+                             "0" (width_mmx)  // ecx
+
+#if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
+                           : "%mm0", "%mm1"               // clobber list
+#endif
+                        );
+                     }
+
+                     sptr -= (width_mmx*2 - 2); // sign fixed
+                     dp -= (width_mmx*16 - 2);  // sign fixed
+                     for (i = width; i; i--)
+                     {
+                        png_byte v[8];
+                        int j;
+                        sptr -= 2;
+                        png_memcpy(v, sptr, 2);
+                        for (j = 0; j < png_pass_inc[pass]; j++)
+                        {
+                           dp -= 2;
+                           png_memcpy(dp, v, 2);
+                        }
+                     }
+                  }
+                  else if (((pass == 2) || (pass == 3)) && width)
+                  {
+                     int width_mmx = ((width >> 1) << 1) ;
+                     width -= width_mmx;        // 0,1 pixels => 0,2 bytes
+                     if (width_mmx)
+                     {
+                        int dummy_value_c;  // fix 'forbidden register spilled'
+                        int dummy_value_S;
+                        int dummy_value_D;
+
+                        __asm__ __volatile__ (
+                           "subl $2, %%esi          \n\t"
+                           "subl $14, %%edi         \n\t"
+
+                        ".loop2_pass2:              \n\t"
+                           "movd (%%esi), %%mm0     \n\t" // x x x x 3 2 1 0
+                           "punpcklwd %%mm0, %%mm0  \n\t" // 3 2 3 2 1 0 1 0
+                           "movq %%mm0, %%mm1       \n\t" // 3 2 3 2 1 0 1 0
+                           "punpckldq %%mm0, %%mm0  \n\t" // 1 0 1 0 1 0 1 0
+                           "punpckhdq %%mm1, %%mm1  \n\t" // 3 2 3 2 3 2 3 2
+                           "movq %%mm0, (%%edi)     \n\t"
+                           "subl $4, %%esi          \n\t"
+                           "movq %%mm1, 8(%%edi)    \n\t"
+                           "subl $16, %%edi         \n\t"
+                           "subl $2, %%ecx          \n\t"
+                           "jnz .loop2_pass2        \n\t"
+                           "EMMS                    \n\t" // DONE
+
+                           : "=c" (dummy_value_c),        // output regs (dummy)
+                             "=S" (dummy_value_S),
+                             "=D" (dummy_value_D)
+
+                           : "1" (sptr),      // esi      // input regs
+                             "2" (dp),        // edi
+                             "0" (width_mmx)  // ecx
+
+#if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
+                           : "%mm0", "%mm1"               // clobber list
+#endif
+                        );
+                     }
+
+                     sptr -= (width_mmx*2 - 2); // sign fixed
+                     dp -= (width_mmx*8 - 2);   // sign fixed
+                     for (i = width; i; i--)
+                     {
+                        png_byte v[8];
+                        int j;
+                        sptr -= 2;
+                        png_memcpy(v, sptr, 2);
+                        for (j = 0; j < png_pass_inc[pass]; j++)
+                        {
+                           dp -= 2;
+                           png_memcpy(dp, v, 2);
+                        }
+                     }
+                  }
+                  else if (width)  // pass == 4 or 5
+                  {
+                     int width_mmx = ((width >> 1) << 1) ;
+                     width -= width_mmx;        // 0,1 pixels => 0,2 bytes
+                     if (width_mmx)
+                     {
+                        int dummy_value_c;  // fix 'forbidden register spilled'
+                        int dummy_value_S;
+                        int dummy_value_D;
+
+                        __asm__ __volatile__ (
+                           "subl $2, %%esi          \n\t"
+                           "subl $6, %%edi          \n\t"
+
+                        ".loop2_pass4:              \n\t"
+                           "movd (%%esi), %%mm0     \n\t" // x x x x 3 2 1 0
+                           "punpcklwd %%mm0, %%mm0  \n\t" // 3 2 3 2 1 0 1 0
+                           "subl $4, %%esi          \n\t"
+                           "movq %%mm0, (%%edi)     \n\t"
+                           "subl $8, %%edi          \n\t"
+                           "subl $2, %%ecx          \n\t"
+                           "jnz .loop2_pass4        \n\t"
+                           "EMMS                    \n\t" // DONE
+
+                           : "=c" (dummy_value_c),        // output regs (dummy)
+                             "=S" (dummy_value_S),
+                             "=D" (dummy_value_D)
+
+                           : "1" (sptr),      // esi      // input regs
+                             "2" (dp),        // edi
+                             "0" (width_mmx)  // ecx
+
+#if 0  /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
+                           : "%mm0"                       // clobber list
+#endif
+                        );
+                     }
+
+                     sptr -= (width_mmx*2 - 2); // sign fixed
+                     dp -= (width_mmx*4 - 2);   // sign fixed
+                     for (i = width; i; i--)
+                     {
+                        png_byte v[8];
+                        int j;
+                        sptr -= 2;
+                        png_memcpy(v, sptr, 2);
+                        for (j = 0; j < png_pass_inc[pass]; j++)
+                        {
+                           dp -= 2;
+                           png_memcpy(dp, v, 2);
+                        }
+                     }
+                  }
+               } /* end of pixel_bytes == 2 */
+
+               //--------------------------------------------------------------
+               else if (pixel_bytes == 4)
+               {
+                  if (((pass == 0) || (pass == 1)) && width)
+                  {
+                     int width_mmx = ((width >> 1) << 1);
+                     width -= width_mmx;        // 0,1 pixels => 0,4 bytes
+                     if (width_mmx)
+                     {
+                        int dummy_value_c;  // fix 'forbidden register spilled'
+                        int dummy_value_S;
+                        int dummy_value_D;
+
+                        __asm__ __volatile__ (
+                           "subl $4, %%esi          \n\t"
+                           "subl $60, %%edi         \n\t"
+
+                        ".loop4_pass0:              \n\t"
+                           "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
+                           "movq %%mm0, %%mm1       \n\t" // 7 6 5 4 3 2 1 0
+                           "punpckldq %%mm0, %%mm0  \n\t" // 3 2 1 0 3 2 1 0
+                           "punpckhdq %%mm1, %%mm1  \n\t" // 7 6 5 4 7 6 5 4
+                           "movq %%mm0, (%%edi)     \n\t"
+                           "movq %%mm0, 8(%%edi)    \n\t"
+                           "movq %%mm0, 16(%%edi)   \n\t"
+                           "movq %%mm0, 24(%%edi)   \n\t"
+                           "movq %%mm1, 32(%%edi)   \n\t"
+                           "movq %%mm1, 40(%%edi)   \n\t"
+                           "movq %%mm1, 48(%%edi)   \n\t"
+                           "subl $8, %%esi          \n\t"
+                           "movq %%mm1, 56(%%edi)   \n\t"
+                           "subl $64, %%edi         \n\t"
+                           "subl $2, %%ecx          \n\t"
+                           "jnz .loop4_pass0        \n\t"
+                           "EMMS                    \n\t" // DONE
+
+                           : "=c" (dummy_value_c),        // output regs (dummy)
+                             "=S" (dummy_value_S),
+                             "=D" (dummy_value_D)
+
+                           : "1" (sptr),      // esi      // input regs
+                             "2" (dp),        // edi
+                             "0" (width_mmx)  // ecx
+
+#if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
+                           : "%mm0", "%mm1"               // clobber list
+#endif
+                        );
+                     }
+
+                     sptr -= (width_mmx*4 - 4); // sign fixed
+                     dp -= (width_mmx*32 - 4);  // sign fixed
+                     for (i = width; i; i--)
+                     {
+                        png_byte v[8];
+                        int j;
+                        sptr -= 4;
+                        png_memcpy(v, sptr, 4);
+                        for (j = 0; j < png_pass_inc[pass]; j++)
+                        {
+                           dp -= 4;
+                           png_memcpy(dp, v, 4);
+                        }
+                     }
+                  }
+                  else if (((pass == 2) || (pass == 3)) && width)
+                  {
+                     int width_mmx = ((width >> 1) << 1);
+                     width -= width_mmx;        // 0,1 pixels => 0,4 bytes
+                     if (width_mmx)
+                     {
+                        int dummy_value_c;  // fix 'forbidden register spilled'
+                        int dummy_value_S;
+                        int dummy_value_D;
+
+                        __asm__ __volatile__ (
+                           "subl $4, %%esi          \n\t"
+                           "subl $28, %%edi         \n\t"
+
+                        ".loop4_pass2:              \n\t"
+                           "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
+                           "movq %%mm0, %%mm1       \n\t" // 7 6 5 4 3 2 1 0
+                           "punpckldq %%mm0, %%mm0  \n\t" // 3 2 1 0 3 2 1 0
+                           "punpckhdq %%mm1, %%mm1  \n\t" // 7 6 5 4 7 6 5 4
+                           "movq %%mm0, (%%edi)     \n\t"
+                           "movq %%mm0, 8(%%edi)    \n\t"
+                           "movq %%mm1, 16(%%edi)   \n\t"
+                           "movq %%mm1, 24(%%edi)   \n\t"
+                           "subl $8, %%esi          \n\t"
+                           "subl $32, %%edi         \n\t"
+                           "subl $2, %%ecx          \n\t"
+                           "jnz .loop4_pass2        \n\t"
+                           "EMMS                    \n\t" // DONE
+
+                           : "=c" (dummy_value_c),        // output regs (dummy)
+                             "=S" (dummy_value_S),
+                             "=D" (dummy_value_D)
+
+                           : "1" (sptr),      // esi      // input regs
+                             "2" (dp),        // edi
+                             "0" (width_mmx)  // ecx
+
+#if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
+                           : "%mm0", "%mm1"               // clobber list
+#endif
+                        );
+                     }
+
+                     sptr -= (width_mmx*4 - 4); // sign fixed
+                     dp -= (width_mmx*16 - 4);  // sign fixed
+                     for (i = width; i; i--)
+                     {
+                        png_byte v[8];
+                        int j;
+                        sptr -= 4;
+                        png_memcpy(v, sptr, 4);
+                        for (j = 0; j < png_pass_inc[pass]; j++)
+                        {
+                           dp -= 4;
+                           png_memcpy(dp, v, 4);
+                        }
+                     }
+                  }
+                  else if (width)  // pass == 4 or 5
+                  {
+                     int width_mmx = ((width >> 1) << 1) ;
+                     width -= width_mmx;        // 0,1 pixels => 0,4 bytes
+                     if (width_mmx)
+                     {
+                        int dummy_value_c;  // fix 'forbidden register spilled'
+                        int dummy_value_S;
+                        int dummy_value_D;
+
+                        __asm__ __volatile__ (
+                           "subl $4, %%esi          \n\t"
+                           "subl $12, %%edi         \n\t"
+
+                        ".loop4_pass4:              \n\t"
+                           "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
+                           "movq %%mm0, %%mm1       \n\t" // 7 6 5 4 3 2 1 0
+                           "punpckldq %%mm0, %%mm0  \n\t" // 3 2 1 0 3 2 1 0
+                           "punpckhdq %%mm1, %%mm1  \n\t" // 7 6 5 4 7 6 5 4
+                           "movq %%mm0, (%%edi)     \n\t"
+                           "subl $8, %%esi          \n\t"
+                           "movq %%mm1, 8(%%edi)    \n\t"
+                           "subl $16, %%edi         \n\t"
+                           "subl $2, %%ecx          \n\t"
+                           "jnz .loop4_pass4        \n\t"
+                           "EMMS                    \n\t" // DONE
+
+                           : "=c" (dummy_value_c),        // output regs (dummy)
+                             "=S" (dummy_value_S),
+                             "=D" (dummy_value_D)
+
+                           : "1" (sptr),      // esi      // input regs
+                             "2" (dp),        // edi
+                             "0" (width_mmx)  // ecx
+
+#if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
+                           : "%mm0", "%mm1"               // clobber list
+#endif
+                        );
+                     }
+
+                     sptr -= (width_mmx*4 - 4); // sign fixed
+                     dp -= (width_mmx*8 - 4);   // sign fixed
+                     for (i = width; i; i--)
+                     {
+                        png_byte v[8];
+                        int j;
+                        sptr -= 4;
+                        png_memcpy(v, sptr, 4);
+                        for (j = 0; j < png_pass_inc[pass]; j++)
+                        {
+                           dp -= 4;
+                           png_memcpy(dp, v, 4);
+                        }
+                     }
+                  }
+               } /* end of pixel_bytes == 4 */
+
+               //--------------------------------------------------------------
+               else if (pixel_bytes == 8)
+               {
+// GRR TEST:  should work, but needs testing (special 64-bit version of rpng2?)
+                  // GRR NOTE:  no need to combine passes here!
+                  if (((pass == 0) || (pass == 1)) && width)
+                  {
+                     int dummy_value_c;  // fix 'forbidden register spilled'
+                     int dummy_value_S;
+                     int dummy_value_D;
+
+                     // source is 8-byte RRGGBBAA
+                     // dest is 64-byte RRGGBBAA RRGGBBAA RRGGBBAA RRGGBBAA ...
+                     __asm__ __volatile__ (
+                        "subl $56, %%edi         \n\t" // start of last block
+
+                     ".loop8_pass0:              \n\t"
+                        "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
+                        "movq %%mm0, (%%edi)     \n\t"
+                        "movq %%mm0, 8(%%edi)    \n\t"
+                        "movq %%mm0, 16(%%edi)   \n\t"
+                        "movq %%mm0, 24(%%edi)   \n\t"
+                        "movq %%mm0, 32(%%edi)   \n\t"
+                        "movq %%mm0, 40(%%edi)   \n\t"
+                        "movq %%mm0, 48(%%edi)   \n\t"
+                        "subl $8, %%esi          \n\t"
+                        "movq %%mm0, 56(%%edi)   \n\t"
+                        "subl $64, %%edi         \n\t"
+                        "decl %%ecx              \n\t"
+                        "jnz .loop8_pass0        \n\t"
+                        "EMMS                    \n\t" // DONE
+
+                        : "=c" (dummy_value_c),        // output regs (dummy)
+                          "=S" (dummy_value_S),
+                          "=D" (dummy_value_D)
+
+                        : "1" (sptr),      // esi      // input regs
+                          "2" (dp),        // edi
+                          "0" (width)      // ecx
+
+#if 0  /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
+                        : "%mm0"                       // clobber list
+#endif
+                     );
+                  }
+                  else if (((pass == 2) || (pass == 3)) && width)
+                  {
+                     // source is 8-byte RRGGBBAA
+                     // dest is 32-byte RRGGBBAA RRGGBBAA RRGGBBAA RRGGBBAA
+                     // (recall that expansion is _in place_:  sptr and dp
+                     //  both point at locations within same row buffer)
+                     {
+                        int dummy_value_c;  // fix 'forbidden register spilled'
+                        int dummy_value_S;
+                        int dummy_value_D;
+
+                        __asm__ __volatile__ (
+                           "subl $24, %%edi         \n\t" // start of last block
+
+                        ".loop8_pass2:              \n\t"
+                           "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
+                           "movq %%mm0, (%%edi)     \n\t"
+                           "movq %%mm0, 8(%%edi)    \n\t"
+                           "movq %%mm0, 16(%%edi)   \n\t"
+                           "subl $8, %%esi          \n\t"
+                           "movq %%mm0, 24(%%edi)   \n\t"
+                           "subl $32, %%edi         \n\t"
+                           "decl %%ecx              \n\t"
+                           "jnz .loop8_pass2        \n\t"
+                           "EMMS                    \n\t" // DONE
+
+                           : "=c" (dummy_value_c),        // output regs (dummy)
+                             "=S" (dummy_value_S),
+                             "=D" (dummy_value_D)
+
+                           : "1" (sptr),      // esi      // input regs
+                             "2" (dp),        // edi
+                             "0" (width)      // ecx
+
+#if 0  /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
+                           : "%mm0"                       // clobber list
+#endif
+                        );
+                     }
+                  }
+                  else if (width)  // pass == 4 or 5
+                  {
+                     // source is 8-byte RRGGBBAA
+                     // dest is 16-byte RRGGBBAA RRGGBBAA
+                     {
+                        int dummy_value_c;  // fix 'forbidden register spilled'
+                        int dummy_value_S;
+                        int dummy_value_D;
+
+                        __asm__ __volatile__ (
+                           "subl $8, %%edi          \n\t" // start of last block
+
+                        ".loop8_pass4:              \n\t"
+                           "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
+                           "movq %%mm0, (%%edi)     \n\t"
+                           "subl $8, %%esi          \n\t"
+                           "movq %%mm0, 8(%%edi)    \n\t"
+                           "subl $16, %%edi         \n\t"
+                           "decl %%ecx              \n\t"
+                           "jnz .loop8_pass4        \n\t"
+                           "EMMS                    \n\t" // DONE
+
+                           : "=c" (dummy_value_c),        // output regs (dummy)
+                             "=S" (dummy_value_S),
+                             "=D" (dummy_value_D)
+
+                           : "1" (sptr),      // esi      // input regs
+                             "2" (dp),        // edi
+                             "0" (width)      // ecx
+
+#if 0  /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
+                           : "%mm0"                       // clobber list
+#endif
+                        );
+                     }
+                  }
+
+               } /* end of pixel_bytes == 8 */
+
+               //--------------------------------------------------------------
+               else if (pixel_bytes == 6)
+               {
+                  for (i = width; i; i--)
+                  {
+                     png_byte v[8];
+                     int j;
+                     png_memcpy(v, sptr, 6);
+                     for (j = 0; j < png_pass_inc[pass]; j++)
+                     {
+                        png_memcpy(dp, v, 6);
+                        dp -= 6;
+                     }
+                     sptr -= 6;
+                  }
+               } /* end of pixel_bytes == 6 */
+
+               //--------------------------------------------------------------
+               else
+               {
+                  for (i = width; i; i--)
+                  {
+                     png_byte v[8];
+                     int j;
+                     png_memcpy(v, sptr, pixel_bytes);
+                     for (j = 0; j < png_pass_inc[pass]; j++)
+                     {
+                        png_memcpy(dp, v, pixel_bytes);
+                        dp -= pixel_bytes;
+                     }
+                     sptr-= pixel_bytes;
+                  }
+               }
+            } // end of _mmx_supported ========================================
+
+            else /* MMX not supported:  use modified C code - takes advantage
+                  *   of inlining of png_memcpy for a constant */
+                 /* GRR 19991007:  does it?  or should pixel_bytes in each
+                  *   block be replaced with immediate value (e.g., 1)? */
+                 /* GRR 19991017:  replaced with constants in each case */
+#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
+            {
+               if (pixel_bytes == 1)
+               {
+                  for (i = width; i; i--)
+                  {
+                     int j;
+                     for (j = 0; j < png_pass_inc[pass]; j++)
+                     {
+                        *dp-- = *sptr;
+                     }
+                     --sptr;
+                  }
+               }
+               else if (pixel_bytes == 3)
+               {
+                  for (i = width; i; i--)
+                  {
+                     png_byte v[8];
+                     int j;
+                     png_memcpy(v, sptr, 3);
+                     for (j = 0; j < png_pass_inc[pass]; j++)
+                     {
+                        png_memcpy(dp, v, 3);
+                        dp -= 3;
+                     }
+                     sptr -= 3;
+                  }
+               }
+               else if (pixel_bytes == 2)
+               {
+                  for (i = width; i; i--)
+                  {
+                     png_byte v[8];
+                     int j;
+                     png_memcpy(v, sptr, 2);
+                     for (j = 0; j < png_pass_inc[pass]; j++)
+                     {
+                        png_memcpy(dp, v, 2);
+                        dp -= 2;
+                     }
+                     sptr -= 2;
+                  }
+               }
+               else if (pixel_bytes == 4)
+               {
+                  for (i = width; i; i--)
+                  {
+                     png_byte v[8];
+                     int j;
+                     png_memcpy(v, sptr, 4);
+                     for (j = 0; j < png_pass_inc[pass]; j++)
+                     {
+#ifdef PNG_DEBUG
+                        if (dp < row || dp+3 > row+png_ptr->row_buf_size)
+                        {
+                           printf("dp out of bounds: row=%d, dp=%d, rp=%d\n",
+                             row, dp, row+png_ptr->row_buf_size);
+                           printf("row_buf=%d\n",png_ptr->row_buf_size);
+                        }
+#endif
+                        png_memcpy(dp, v, 4);
+                        dp -= 4;
+                     }
+                     sptr -= 4;
+                  }
+               }
+               else if (pixel_bytes == 6)
+               {
+                  for (i = width; i; i--)
+                  {
+                     png_byte v[8];
+                     int j;
+                     png_memcpy(v, sptr, 6);
+                     for (j = 0; j < png_pass_inc[pass]; j++)
+                     {
+                        png_memcpy(dp, v, 6);
+                        dp -= 6;
+                     }
+                     sptr -= 6;
+                  }
+               }
+               else if (pixel_bytes == 8)
+               {
+                  for (i = width; i; i--)
+                  {
+                     png_byte v[8];
+                     int j;
+                     png_memcpy(v, sptr, 8);
+                     for (j = 0; j < png_pass_inc[pass]; j++)
+                     {
+                        png_memcpy(dp, v, 8);
+                        dp -= 8;
+                     }
+                     sptr -= 8;
+                  }
+               }
+               else     /* GRR:  should never be reached */
+               {
+                  for (i = width; i; i--)
+                  {
+                     png_byte v[8];
+                     int j;
+                     png_memcpy(v, sptr, pixel_bytes);
+                     for (j = 0; j < png_pass_inc[pass]; j++)
+                     {
+                        png_memcpy(dp, v, pixel_bytes);
+                        dp -= pixel_bytes;
+                     }
+                     sptr -= pixel_bytes;
+                  }
+               }
+
+            } /* end if (MMX not supported) */
+            break;
+         }
+      } /* end switch (row_info->pixel_depth) */
+
+      row_info->width = final_width;
+      row_info->rowbytes = ((final_width *
+         (png_uint_32)row_info->pixel_depth + 7) >> 3);
+   }
+
+} /* end png_do_read_interlace() */
+
+#endif /* PNG_HAVE_ASSEMBLER_READ_INTERLACE */
+#endif /* PNG_READ_INTERLACING_SUPPORTED */
+
+
+
+#if defined(PNG_HAVE_ASSEMBLER_READ_FILTER_ROW)
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
+
+// These variables are utilized in the functions below.  They are declared
+// globally here to ensure alignment on 8-byte boundaries.
+
+union uAll {
+   long long use;
+   double  align;
+} _LBCarryMask = {0x0101010101010101LL},
+  _HBClearMask = {0x7f7f7f7f7f7f7f7fLL},
+  _ActiveMask, _ActiveMask2, _ActiveMaskEnd, _ShiftBpp, _ShiftRem;
+
+#ifdef PNG_THREAD_UNSAFE_OK
+//===========================================================================//
+//                                                                           //
+//           P N G _ R E A D _ F I L T E R _ R O W _ M M X _ A V G           //
+//                                                                           //
+//===========================================================================//
+
+// Optimized code for PNG Average filter decoder
+
+static void /* PRIVATE */
+png_read_filter_row_mmx_avg(png_row_infop row_info, png_bytep row,
+                            png_bytep prev_row)
+{
+   int bpp;
+   int dummy_value_c;   // fix 'forbidden register 2 (cx) was spilled' error
+   int dummy_value_S;
+   int dummy_value_D;
+
+   bpp = (row_info->pixel_depth + 7) >> 3;  // get # bytes per pixel
+   _FullLength  = row_info->rowbytes;       // # of bytes to filter
+
+   __asm__ __volatile__ (
+      // initialize address pointers and offset
+#ifdef __PIC__
+      "pushl %%ebx                 \n\t" // save index to Global Offset Table
+#endif
+//pre "movl row, %%edi             \n\t" // edi:  Avg(x)
+      "xorl %%ebx, %%ebx           \n\t" // ebx:  x
+      "movl %%edi, %%edx           \n\t"
+//pre "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
+//pre "subl bpp, %%edx             \n\t" // (bpp is preloaded into ecx)
+      "subl %%ecx, %%edx           \n\t" // edx:  Raw(x-bpp)
+
+      "xorl %%eax,%%eax            \n\t"
+
+      // Compute the Raw value for the first bpp bytes
+      //    Raw(x) = Avg(x) + (Prior(x)/2)
+   "avg_rlp:                       \n\t"
+      "movb (%%esi,%%ebx,),%%al    \n\t" // load al with Prior(x)
+      "incl %%ebx                  \n\t"
+      "shrb %%al                   \n\t" // divide by 2
+      "addb -1(%%edi,%%ebx,),%%al  \n\t" // add Avg(x); -1 to offset inc ebx
+//pre "cmpl bpp, %%ebx             \n\t" // (bpp is preloaded into ecx)
+      "cmpl %%ecx, %%ebx           \n\t"
+      "movb %%al,-1(%%edi,%%ebx,)  \n\t" // write Raw(x); -1 to offset inc ebx
+      "jb avg_rlp                  \n\t" // mov does not affect flags
+
+      // get # of bytes to alignment
+      "movl %%edi, _dif            \n\t" // take start of row
+      "addl %%ebx, _dif            \n\t" // add bpp
+      "addl $0xf, _dif             \n\t" // add 7+8 to incr past alignment bdry
+      "andl $0xfffffff8, _dif      \n\t" // mask to alignment boundary
+      "subl %%edi, _dif            \n\t" // subtract from start => value ebx at
+      "jz avg_go                   \n\t" //  alignment
+
+      // fix alignment
+      // Compute the Raw value for the bytes up to the alignment boundary
+      //    Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2)
+      "xorl %%ecx, %%ecx           \n\t"
+
+   "avg_lp1:                       \n\t"
+      "xorl %%eax, %%eax           \n\t"
+      "movb (%%esi,%%ebx,), %%cl   \n\t" // load cl with Prior(x)
+      "movb (%%edx,%%ebx,), %%al   \n\t" // load al with Raw(x-bpp)
+      "addw %%cx, %%ax             \n\t"
+      "incl %%ebx                  \n\t"
+      "shrw %%ax                   \n\t" // divide by 2
+      "addb -1(%%edi,%%ebx,), %%al \n\t" // add Avg(x); -1 to offset inc ebx
+      "cmpl _dif, %%ebx            \n\t" // check if at alignment boundary
+      "movb %%al, -1(%%edi,%%ebx,) \n\t" // write Raw(x); -1 to offset inc ebx
+      "jb avg_lp1                  \n\t" // repeat until at alignment boundary
+
+   "avg_go:                        \n\t"
+      "movl _FullLength, %%eax     \n\t"
+      "movl %%eax, %%ecx           \n\t"
+      "subl %%ebx, %%eax           \n\t" // subtract alignment fix
+      "andl $0x00000007, %%eax     \n\t" // calc bytes over mult of 8
+      "subl %%eax, %%ecx           \n\t" // drop over bytes from original length
+      "movl %%ecx, _MMXLength      \n\t"
+#ifdef __PIC__
+      "popl %%ebx                  \n\t" // restore index to Global Offset Table
+#endif
+
+      : "=c" (dummy_value_c),            // output regs (dummy)
+        "=S" (dummy_value_S),
+        "=D" (dummy_value_D)
+
+      : "0" (bpp),       // ecx          // input regs
+        "1" (prev_row),  // esi
+        "2" (row)        // edi
+
+      : "%eax", "%edx"                   // clobber list
+#ifndef __PIC__
+      , "%ebx"
+#endif
+      // GRR: INCLUDE "memory" as clobbered? (_dif, _MMXLength)
+      // (seems to work fine without...)
+   );
+
+   // now do the math for the rest of the row
+   switch (bpp)
+   {
+      case 3:
+      {
+         _ActiveMask.use  = 0x0000000000ffffffLL;
+         _ShiftBpp.use = 24;    // == 3 * 8
+         _ShiftRem.use = 40;    // == 64 - 24
+
+         __asm__ __volatile__ (
+            // re-init address pointers and offset
+            "movq _ActiveMask, %%mm7      \n\t"
+            "movl _dif, %%ecx             \n\t" // ecx:  x = offset to
+            "movq _LBCarryMask, %%mm5     \n\t" //  alignment boundary
+// preload  "movl row, %%edi              \n\t" // edi:  Avg(x)
+            "movq _HBClearMask, %%mm4     \n\t"
+// preload  "movl prev_row, %%esi         \n\t" // esi:  Prior(x)
+
+            // prime the pump:  load the first Raw(x-bpp) data set
+            "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
+                                                // (correct pos. in loop below)
+         "avg_3lp:                        \n\t"
+            "movq (%%edi,%%ecx,), %%mm0   \n\t" // load mm0 with Avg(x)
+            "movq %%mm5, %%mm3            \n\t"
+            "psrlq _ShiftRem, %%mm2       \n\t" // correct position Raw(x-bpp)
+                                                // data
+            "movq (%%esi,%%ecx,), %%mm1   \n\t" // load mm1 with Prior(x)
+            "movq %%mm7, %%mm6            \n\t"
+            "pand %%mm1, %%mm3            \n\t" // get lsb for each prev_row byte
+            "psrlq $1, %%mm1              \n\t" // divide prev_row bytes by 2
+            "pand  %%mm4, %%mm1           \n\t" // clear invalid bit 7 of each
+                                                // byte
+            "paddb %%mm1, %%mm0           \n\t" // add (Prev_row/2) to Avg for
+                                                // each byte
+            // add 1st active group (Raw(x-bpp)/2) to average with LBCarry
+            "movq %%mm3, %%mm1            \n\t" // now use mm1 for getting
+                                                // LBCarrys
+            "pand %%mm2, %%mm1            \n\t" // get LBCarrys for each byte
+                                                // where both
+                               // lsb's were == 1 (only valid for active group)
+            "psrlq $1, %%mm2              \n\t" // divide raw bytes by 2
+            "pand  %%mm4, %%mm2           \n\t" // clear invalid bit 7 of each
+                                                // byte
+            "paddb %%mm1, %%mm2           \n\t" // add LBCarrys to (Raw(x-bpp)/2)
+                                                // for each byte
+            "pand %%mm6, %%mm2            \n\t" // leave only Active Group 1
+                                                // bytes to add to Avg
+            "paddb %%mm2, %%mm0           \n\t" // add (Raw/2) + LBCarrys to
+                                                // Avg for each Active
+                               //  byte
+            // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
+            "psllq _ShiftBpp, %%mm6       \n\t" // shift the mm6 mask to cover
+                                                // bytes 3-5
+            "movq %%mm0, %%mm2            \n\t" // mov updated Raws to mm2
+            "psllq _ShiftBpp, %%mm2       \n\t" // shift data to pos. correctly
+            "movq %%mm3, %%mm1            \n\t" // now use mm1 for getting
+                                                // LBCarrys
+            "pand %%mm2, %%mm1            \n\t" // get LBCarrys for each byte
+                                                // where both
+                               // lsb's were == 1 (only valid for active group)
+            "psrlq $1, %%mm2              \n\t" // divide raw bytes by 2
+            "pand  %%mm4, %%mm2           \n\t" // clear invalid bit 7 of each
+                                                // byte
+            "paddb %%mm1, %%mm2           \n\t" // add LBCarrys to (Raw(x-bpp)/2)
+                                                // for each byte
+            "pand %%mm6, %%mm2            \n\t" // leave only Active Group 2
+                                                // bytes to add to Avg
+            "paddb %%mm2, %%mm0           \n\t" // add (Raw/2) + LBCarrys to
+                                                // Avg for each Active
+                               //  byte
+
+            // add 3rd active group (Raw(x-bpp)/2) to average with _LBCarry
+            "psllq _ShiftBpp, %%mm6       \n\t" // shift mm6 mask to cover last
+                                                // two
+                                 // bytes
+            "movq %%mm0, %%mm2            \n\t" // mov updated Raws to mm2
+            "psllq _ShiftBpp, %%mm2       \n\t" // shift data to pos. correctly
+                              // Data only needs to be shifted once here to
+                              // get the correct x-bpp offset.
+            "movq %%mm3, %%mm1            \n\t" // now use mm1 for getting
+                                                // LBCarrys
+            "pand %%mm2, %%mm1            \n\t" // get LBCarrys for each byte
+                                                // where both
+                              // lsb's were == 1 (only valid for active group)
+            "psrlq $1, %%mm2              \n\t" // divide raw bytes by 2
+            "pand  %%mm4, %%mm2           \n\t" // clear invalid bit 7 of each
+                                                // byte
+            "paddb %%mm1, %%mm2           \n\t" // add LBCarrys to (Raw(x-bpp)/2)
+                                                // for each byte
+            "pand %%mm6, %%mm2            \n\t" // leave only Active Group 2
+                                                // bytes to add to Avg
+            "addl $8, %%ecx               \n\t"
+            "paddb %%mm2, %%mm0           \n\t" // add (Raw/2) + LBCarrys to
+                                                // Avg for each Active
+                                                // byte
+            // now ready to write back to memory
+            "movq %%mm0, -8(%%edi,%%ecx,) \n\t"
+            // move updated Raw(x) to use as Raw(x-bpp) for next loop
+            "cmpl _MMXLength, %%ecx       \n\t"
+            "movq %%mm0, %%mm2            \n\t" // mov updated Raw(x) to mm2
+            "jb avg_3lp                   \n\t"
+
+            : "=S" (dummy_value_S),             // output regs (dummy)
+              "=D" (dummy_value_D)
+
+            : "0" (prev_row),  // esi           // input regs
+              "1" (row)        // edi
+
+            : "%ecx"                            // clobber list
+#if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
+            , "%mm0", "%mm1", "%mm2", "%mm3"
+            , "%mm4", "%mm5", "%mm6", "%mm7"
+#endif
+         );
+      }
+      break;  // end 3 bpp
+
+      case 6:
+      case 4:
+      //case 7:   // who wrote this?  PNG doesn't support 5 or 7 bytes/pixel
+      //case 5:   // GRR BOGUS
+      {
+         _ActiveMask.use  = 0xffffffffffffffffLL; // use shift below to clear
+                                                  // appropriate inactive bytes
+         _ShiftBpp.use = bpp << 3;
+         _ShiftRem.use = 64 - _ShiftBpp.use;
+
+         __asm__ __volatile__ (
+            "movq _HBClearMask, %%mm4    \n\t"
+
+            // re-init address pointers and offset
+            "movl _dif, %%ecx            \n\t" // ecx:  x = offset to
+                                               // alignment boundary
+
+            // load _ActiveMask and clear all bytes except for 1st active group
+            "movq _ActiveMask, %%mm7     \n\t"
+// preload  "movl row, %%edi             \n\t" // edi:  Avg(x)
+            "psrlq _ShiftRem, %%mm7      \n\t"
+// preload  "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
+            "movq %%mm7, %%mm6           \n\t"
+            "movq _LBCarryMask, %%mm5    \n\t"
+            "psllq _ShiftBpp, %%mm6      \n\t" // create mask for 2nd active
+                                               // group
+
+            // prime the pump:  load the first Raw(x-bpp) data set
+            "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
+                                          // (we correct pos. in loop below)
+         "avg_4lp:                       \n\t"
+            "movq (%%edi,%%ecx,), %%mm0  \n\t"
+            "psrlq _ShiftRem, %%mm2      \n\t" // shift data to pos. correctly
+            "movq (%%esi,%%ecx,), %%mm1  \n\t"
+            // add (Prev_row/2) to average
+            "movq %%mm5, %%mm3           \n\t"
+            "pand %%mm1, %%mm3           \n\t" // get lsb for each prev_row byte
+            "psrlq $1, %%mm1             \n\t" // divide prev_row bytes by 2
+            "pand  %%mm4, %%mm1          \n\t" // clear invalid bit 7 of each
+                                               // byte
+            "paddb %%mm1, %%mm0          \n\t" // add (Prev_row/2) to Avg for
+                                               // each byte
+            // add 1st active group (Raw(x-bpp)/2) to average with _LBCarry
+            "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
+                                               // LBCarrys
+            "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
+                                               // where both
+                              // lsb's were == 1 (only valid for active group)
+            "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
+            "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
+                                               // byte
+            "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
+                                               // for each byte
+            "pand %%mm7, %%mm2           \n\t" // leave only Active Group 1
+                                               // bytes to add to Avg
+            "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to Avg
+                                               // for each Active
+                              // byte
+            // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
+            "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
+            "psllq _ShiftBpp, %%mm2      \n\t" // shift data to pos. correctly
+            "addl $8, %%ecx              \n\t"
+            "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
+                                               // LBCarrys
+            "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
+                                               // where both
+                              // lsb's were == 1 (only valid for active group)
+            "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
+            "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
+                                               // byte
+            "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
+                                               // for each byte
+            "pand %%mm6, %%mm2           \n\t" // leave only Active Group 2
+                                               // bytes to add to Avg
+            "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to
+                                               // Avg for each Active
+                              // byte
+            "cmpl _MMXLength, %%ecx      \n\t"
+            // now ready to write back to memory
+            "movq %%mm0, -8(%%edi,%%ecx,) \n\t"
+            // prep Raw(x-bpp) for next loop
+            "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
+            "jb avg_4lp                  \n\t"
+
+            : "=S" (dummy_value_S),            // output regs (dummy)
+              "=D" (dummy_value_D)
+
+            : "0" (prev_row),  // esi          // input regs
+              "1" (row)        // edi
+
+            : "%ecx"                           // clobber list
+#if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
+            , "%mm0", "%mm1", "%mm2", "%mm3"
+            , "%mm4", "%mm5", "%mm6", "%mm7"
+#endif
+         );
+      }
+      break;  // end 4,6 bpp
+
+      case 2:
+      {
+         _ActiveMask.use  = 0x000000000000ffffLL;
+         _ShiftBpp.use = 16;   // == 2 * 8
+         _ShiftRem.use = 48;   // == 64 - 16
+
+         __asm__ __volatile__ (
+            // load _ActiveMask
+            "movq _ActiveMask, %%mm7     \n\t"
+            // re-init address pointers and offset
+            "movl _dif, %%ecx            \n\t" // ecx:  x = offset to alignment
+                                               // boundary
+            "movq _LBCarryMask, %%mm5    \n\t"
+// preload  "movl row, %%edi             \n\t" // edi:  Avg(x)
+            "movq _HBClearMask, %%mm4    \n\t"
+// preload  "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
+
+            // prime the pump:  load the first Raw(x-bpp) data set
+            "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
+                              // (we correct pos. in loop below)
+         "avg_2lp:                       \n\t"
+            "movq (%%edi,%%ecx,), %%mm0  \n\t"
+            "psrlq _ShiftRem, %%mm2      \n\t" // shift data to pos. correctly
+            "movq (%%esi,%%ecx,), %%mm1  \n\t" //  (GRR BUGFIX:  was psllq)
+            // add (Prev_row/2) to average
+            "movq %%mm5, %%mm3           \n\t"
+            "pand %%mm1, %%mm3           \n\t" // get lsb for each prev_row byte
+            "psrlq $1, %%mm1             \n\t" // divide prev_row bytes by 2
+            "pand  %%mm4, %%mm1          \n\t" // clear invalid bit 7 of each
+                                               // byte
+            "movq %%mm7, %%mm6           \n\t"
+            "paddb %%mm1, %%mm0          \n\t" // add (Prev_row/2) to Avg for
+                                               // each byte
+
+            // add 1st active group (Raw(x-bpp)/2) to average with _LBCarry
+            "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
+                                               // LBCarrys
+            "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
+                                               // where both
+                                               // lsb's were == 1 (only valid
+                                               // for active group)
+            "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
+            "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
+                                               // byte
+            "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
+                                               // for each byte
+            "pand %%mm6, %%mm2           \n\t" // leave only Active Group 1
+                                               // bytes to add to Avg
+            "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to Avg
+                                               // for each Active byte
+
+            // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
+            "psllq _ShiftBpp, %%mm6      \n\t" // shift the mm6 mask to cover
+                                               // bytes 2 & 3
+            "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
+            "psllq _ShiftBpp, %%mm2      \n\t" // shift data to pos. correctly
+            "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
+                                               // LBCarrys
+            "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
+                                               // where both
+                                               // lsb's were == 1 (only valid
+                                               // for active group)
+            "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
+            "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
+                                               // byte
+            "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
+                                               // for each byte
+            "pand %%mm6, %%mm2           \n\t" // leave only Active Group 2
+                                               // bytes to add to Avg
+            "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to
+                                               // Avg for each Active byte
+
+            // add 3rd active group (Raw(x-bpp)/2) to average with _LBCarry
+            "psllq _ShiftBpp, %%mm6      \n\t" // shift the mm6 mask to cover
+                                               // bytes 4 & 5
+            "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
+            "psllq _ShiftBpp, %%mm2      \n\t" // shift data to pos. correctly
+            "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
+                                               // LBCarrys
+            "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
+                                               // where both lsb's were == 1
+                                               // (only valid for active group)
+            "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
+            "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
+                                               // byte
+            "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
+                                               // for each byte
+            "pand %%mm6, %%mm2           \n\t" // leave only Active Group 2
+                                               // bytes to add to Avg
+            "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to
+                                               // Avg for each Active byte
+
+            // add 4th active group (Raw(x-bpp)/2) to average with _LBCarry
+            "psllq _ShiftBpp, %%mm6      \n\t" // shift the mm6 mask to cover
+                                               // bytes 6 & 7
+            "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
+            "psllq _ShiftBpp, %%mm2      \n\t" // shift data to pos. correctly
+            "addl $8, %%ecx              \n\t"
+            "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
+                                               // LBCarrys
+            "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
+                                               // where both
+                                               // lsb's were == 1 (only valid
+                                               // for active group)
+            "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
+            "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
+                                               // byte
+            "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
+                                               // for each byte
+            "pand %%mm6, %%mm2           \n\t" // leave only Active Group 2
+                                               // bytes to add to Avg
+            "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to
+                                               // Avg for each Active byte
+
+            "cmpl _MMXLength, %%ecx      \n\t"
+            // now ready to write back to memory
+            "movq %%mm0, -8(%%edi,%%ecx,) \n\t"
+            // prep Raw(x-bpp) for next loop
+            "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
+            "jb avg_2lp                  \n\t"
+
+            : "=S" (dummy_value_S),            // output regs (dummy)
+              "=D" (dummy_value_D)
+
+            : "0" (prev_row),  // esi          // input regs
+              "1" (row)        // edi
+
+            : "%ecx"                           // clobber list
+#if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
+            , "%mm0", "%mm1", "%mm2", "%mm3"
+            , "%mm4", "%mm5", "%mm6", "%mm7"
+#endif
+         );
+      }
+      break;  // end 2 bpp
+
+      case 1:
+      {
+         __asm__ __volatile__ (
+            // re-init address pointers and offset
+#ifdef __PIC__
+            "pushl %%ebx                 \n\t" // save Global Offset Table index
+#endif
+            "movl _dif, %%ebx            \n\t" // ebx:  x = offset to alignment
+                                               // boundary
+// preload  "movl row, %%edi             \n\t" // edi:  Avg(x)
+            "cmpl _FullLength, %%ebx     \n\t" // test if offset at end of array
+            "jnb avg_1end                \n\t"
+            // do Paeth decode for remaining bytes
+// preload  "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
+            "movl %%edi, %%edx           \n\t"
+// preload  "subl bpp, %%edx             \n\t" // (bpp is preloaded into ecx)
+            "subl %%ecx, %%edx           \n\t" // edx:  Raw(x-bpp)
+            "xorl %%ecx, %%ecx           \n\t" // zero ecx before using cl & cx
+                                               //  in loop below
+         "avg_1lp:                       \n\t"
+            // Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2)
+            "xorl %%eax, %%eax           \n\t"
+            "movb (%%esi,%%ebx,), %%cl   \n\t" // load cl with Prior(x)
+            "movb (%%edx,%%ebx,), %%al   \n\t" // load al with Raw(x-bpp)
+            "addw %%cx, %%ax             \n\t"
+            "incl %%ebx                  \n\t"
+            "shrw %%ax                   \n\t" // divide by 2
+            "addb -1(%%edi,%%ebx,), %%al \n\t" // add Avg(x); -1 to offset
+                                               // inc ebx
+            "cmpl _FullLength, %%ebx     \n\t" // check if at end of array
+            "movb %%al, -1(%%edi,%%ebx,) \n\t" // write back Raw(x);
+                         // mov does not affect flags; -1 to offset inc ebx
+            "jb avg_1lp                  \n\t"
+
+         "avg_1end:                      \n\t"
+#ifdef __PIC__
+            "popl %%ebx                  \n\t" // Global Offset Table index
+#endif
+
+            : "=c" (dummy_value_c),            // output regs (dummy)
+              "=S" (dummy_value_S),
+              "=D" (dummy_value_D)
+
+            : "0" (bpp),       // ecx          // input regs
+              "1" (prev_row),  // esi
+              "2" (row)        // edi
+
+            : "%eax", "%edx"                   // clobber list
+#ifndef __PIC__
+            , "%ebx"
+#endif
+         );
+      }
+      return;  // end 1 bpp
+
+      case 8:
+      {
+         __asm__ __volatile__ (
+            // re-init address pointers and offset
+            "movl _dif, %%ecx            \n\t" // ecx:  x == offset to alignment
+            "movq _LBCarryMask, %%mm5    \n\t" //            boundary
+// preload  "movl row, %%edi             \n\t" // edi:  Avg(x)
+            "movq _HBClearMask, %%mm4    \n\t"
+// preload  "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
+
+            // prime the pump:  load the first Raw(x-bpp) data set
+            "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
+                                      // (NO NEED to correct pos. in loop below)
+
+         "avg_8lp:                       \n\t"
+            "movq (%%edi,%%ecx,), %%mm0  \n\t"
+            "movq %%mm5, %%mm3           \n\t"
+            "movq (%%esi,%%ecx,), %%mm1  \n\t"
+            "addl $8, %%ecx              \n\t"
+            "pand %%mm1, %%mm3           \n\t" // get lsb for each prev_row byte
+            "psrlq $1, %%mm1             \n\t" // divide prev_row bytes by 2
+            "pand %%mm2, %%mm3           \n\t" // get LBCarrys for each byte
+                                               //  where both lsb's were == 1
+            "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
+            "pand  %%mm4, %%mm1          \n\t" // clear invalid bit 7, each byte
+            "paddb %%mm3, %%mm0          \n\t" // add LBCarrys to Avg, each byte
+            "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7, each byte
+            "paddb %%mm1, %%mm0          \n\t" // add (Prev_row/2) to Avg, each
+            "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) to Avg for each
+            "cmpl _MMXLength, %%ecx      \n\t"
+            "movq %%mm0, -8(%%edi,%%ecx,) \n\t"
+            "movq %%mm0, %%mm2           \n\t" // reuse as Raw(x-bpp)
+            "jb avg_8lp                  \n\t"
+
+            : "=S" (dummy_value_S),            // output regs (dummy)
+              "=D" (dummy_value_D)
+
+            : "0" (prev_row),  // esi          // input regs
+              "1" (row)        // edi
+
+            : "%ecx"                           // clobber list
+#if 0  /* %mm0, ..., %mm5 not supported by gcc 2.7.2.3 or egcs 1.1 */
+            , "%mm0", "%mm1", "%mm2"
+            , "%mm3", "%mm4", "%mm5"
+#endif
+         );
+      }
+      break;  // end 8 bpp
+
+      default:                  // bpp greater than 8 (!= 1,2,3,4,[5],6,[7],8)
+      {
+
+#ifdef PNG_DEBUG
+         // GRR:  PRINT ERROR HERE:  SHOULD NEVER BE REACHED
+        png_debug(1,
+        "Internal logic error in pnggccrd (png_read_filter_row_mmx_avg())\n");
+#endif
+
+#if 0
+        __asm__ __volatile__ (
+            "movq _LBCarryMask, %%mm5    \n\t"
+            // re-init address pointers and offset
+            "movl _dif, %%ebx            \n\t" // ebx:  x = offset to
+                                               // alignment boundary
+            "movl row, %%edi             \n\t" // edi:  Avg(x)
+            "movq _HBClearMask, %%mm4    \n\t"
+            "movl %%edi, %%edx           \n\t"
+            "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
+            "subl bpp, %%edx             \n\t" // edx:  Raw(x-bpp)
+         "avg_Alp:                       \n\t"
+            "movq (%%edi,%%ebx,), %%mm0  \n\t"
+            "movq %%mm5, %%mm3           \n\t"
+            "movq (%%esi,%%ebx,), %%mm1  \n\t"
+            "pand %%mm1, %%mm3           \n\t" // get lsb for each prev_row byte
+            "movq (%%edx,%%ebx,), %%mm2  \n\t"
+            "psrlq $1, %%mm1             \n\t" // divide prev_row bytes by 2
+            "pand %%mm2, %%mm3           \n\t" // get LBCarrys for each byte
+                                               // where both lsb's were == 1
+            "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
+            "pand  %%mm4, %%mm1          \n\t" // clear invalid bit 7 of each
+                                               // byte
+            "paddb %%mm3, %%mm0          \n\t" // add LBCarrys to Avg for each
+                                               // byte
+            "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
+                                               // byte
+            "paddb %%mm1, %%mm0          \n\t" // add (Prev_row/2) to Avg for
+                                               // each byte
+            "addl $8, %%ebx              \n\t"
+            "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) to Avg for each
+                                               // byte
+            "cmpl _MMXLength, %%ebx      \n\t"
+            "movq %%mm0, -8(%%edi,%%ebx,) \n\t"
+            "jb avg_Alp                  \n\t"
+
+            : // FIXASM: output regs/vars go here, e.g.:  "=m" (memory_var)
+
+            : // FIXASM: input regs, e.g.:  "c" (count), "S" (src), "D" (dest)
+
+            : "%ebx", "%edx", "%edi", "%esi" // CHECKASM: clobber list
+         );
+#endif /* 0 - NEVER REACHED */
+      }
+      break;
+
+   } // end switch (bpp)
+
+   __asm__ __volatile__ (
+      // MMX acceleration complete; now do clean-up
+      // check if any remaining bytes left to decode
+#ifdef __PIC__
+      "pushl %%ebx                 \n\t" // save index to Global Offset Table
+#endif
+      "movl _MMXLength, %%ebx      \n\t" // ebx:  x == offset bytes after MMX
+//pre "movl row, %%edi             \n\t" // edi:  Avg(x)
+      "cmpl _FullLength, %%ebx     \n\t" // test if offset at end of array
+      "jnb avg_end                 \n\t"
+
+      // do Avg decode for remaining bytes
+//pre "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
+      "movl %%edi, %%edx           \n\t"
+//pre "subl bpp, %%edx             \n\t" // (bpp is preloaded into ecx)
+      "subl %%ecx, %%edx           \n\t" // edx:  Raw(x-bpp)
+      "xorl %%ecx, %%ecx           \n\t" // zero ecx before using cl & cx below
+
+   "avg_lp2:                       \n\t"
+      // Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2)
+      "xorl %%eax, %%eax           \n\t"
+      "movb (%%esi,%%ebx,), %%cl   \n\t" // load cl with Prior(x)
+      "movb (%%edx,%%ebx,), %%al   \n\t" // load al with Raw(x-bpp)
+      "addw %%cx, %%ax             \n\t"
+      "incl %%ebx                  \n\t"
+      "shrw %%ax                   \n\t" // divide by 2
+      "addb -1(%%edi,%%ebx,), %%al \n\t" // add Avg(x); -1 to offset inc ebx
+      "cmpl _FullLength, %%ebx     \n\t" // check if at end of array
+      "movb %%al, -1(%%edi,%%ebx,) \n\t" // write back Raw(x) [mov does not
+      "jb avg_lp2                  \n\t" //  affect flags; -1 to offset inc ebx]
+
+   "avg_end:                       \n\t"
+      "EMMS                        \n\t" // end MMX; prep for poss. FP instrs.
+#ifdef __PIC__
+      "popl %%ebx                  \n\t" // restore index to Global Offset Table
+#endif
+
+      : "=c" (dummy_value_c),            // output regs (dummy)
+        "=S" (dummy_value_S),
+        "=D" (dummy_value_D)
+
+      : "0" (bpp),       // ecx          // input regs
+        "1" (prev_row),  // esi
+        "2" (row)        // edi
+
+      : "%eax", "%edx"                   // clobber list
+#ifndef __PIC__
+      , "%ebx"
+#endif
+   );
+
+} /* end png_read_filter_row_mmx_avg() */
+#endif
+
+
+
+#ifdef PNG_THREAD_UNSAFE_OK
+//===========================================================================//
+//                                                                           //
+//         P N G _ R E A D _ F I L T E R _ R O W _ M M X _ P A E T H         //
+//                                                                           //
+//===========================================================================//
+
+// Optimized code for PNG Paeth filter decoder
+
+static void /* PRIVATE */
+png_read_filter_row_mmx_paeth(png_row_infop row_info, png_bytep row,
+                              png_bytep prev_row)
+{
+   int bpp;
+   int dummy_value_c;   // fix 'forbidden register 2 (cx) was spilled' error
+   int dummy_value_S;
+   int dummy_value_D;
+
+   bpp = (row_info->pixel_depth + 7) >> 3; // Get # bytes per pixel
+   _FullLength  = row_info->rowbytes; // # of bytes to filter
+
+   __asm__ __volatile__ (
+#ifdef __PIC__
+      "pushl %%ebx                 \n\t" // save index to Global Offset Table
+#endif
+      "xorl %%ebx, %%ebx           \n\t" // ebx:  x offset
+//pre "movl row, %%edi             \n\t"
+      "xorl %%edx, %%edx           \n\t" // edx:  x-bpp offset
+//pre "movl prev_row, %%esi        \n\t"
+      "xorl %%eax, %%eax           \n\t"
+
+      // Compute the Raw value for the first bpp bytes
+      // Note: the formula works out to be always
+      //   Paeth(x) = Raw(x) + Prior(x)      where x < bpp
+   "paeth_rlp:                     \n\t"
+      "movb (%%edi,%%ebx,), %%al   \n\t"
+      "addb (%%esi,%%ebx,), %%al   \n\t"
+      "incl %%ebx                  \n\t"
+//pre "cmpl bpp, %%ebx             \n\t" (bpp is preloaded into ecx)
+      "cmpl %%ecx, %%ebx           \n\t"
+      "movb %%al, -1(%%edi,%%ebx,) \n\t"
+      "jb paeth_rlp                \n\t"
+      // get # of bytes to alignment
+      "movl %%edi, _dif            \n\t" // take start of row
+      "addl %%ebx, _dif            \n\t" // add bpp
+      "xorl %%ecx, %%ecx           \n\t"
+      "addl $0xf, _dif             \n\t" // add 7 + 8 to incr past alignment
+                                         // boundary
+      "andl $0xfffffff8, _dif      \n\t" // mask to alignment boundary
+      "subl %%edi, _dif            \n\t" // subtract from start ==> value ebx
+                                         // at alignment
+      "jz paeth_go                 \n\t"
+      // fix alignment
+
+   "paeth_lp1:                     \n\t"
+      "xorl %%eax, %%eax           \n\t"
+      // pav = p - a = (a + b - c) - a = b - c
+      "movb (%%esi,%%ebx,), %%al   \n\t" // load Prior(x) into al
+      "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
+      "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
+      "movl %%eax, _patemp         \n\t" // Save pav for later use
+      "xorl %%eax, %%eax           \n\t"
+      // pbv = p - b = (a + b - c) - b = a - c
+      "movb (%%edi,%%edx,), %%al   \n\t" // load Raw(x-bpp) into al
+      "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
+      "movl %%eax, %%ecx           \n\t"
+      // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
+      "addl _patemp, %%eax         \n\t" // pcv = pav + pbv
+      // pc = abs(pcv)
+      "testl $0x80000000, %%eax    \n\t"
+      "jz paeth_pca                \n\t"
+      "negl %%eax                  \n\t" // reverse sign of neg values
+
+   "paeth_pca:                     \n\t"
+      "movl %%eax, _pctemp         \n\t" // save pc for later use
+      // pb = abs(pbv)
+      "testl $0x80000000, %%ecx    \n\t"
+      "jz paeth_pba                \n\t"
+      "negl %%ecx                  \n\t" // reverse sign of neg values
+
+   "paeth_pba:                     \n\t"
+      "movl %%ecx, _pbtemp         \n\t" // save pb for later use
+      // pa = abs(pav)
+      "movl _patemp, %%eax         \n\t"
+      "testl $0x80000000, %%eax    \n\t"
+      "jz paeth_paa                \n\t"
+      "negl %%eax                  \n\t" // reverse sign of neg values
+
+   "paeth_paa:                     \n\t"
+      "movl %%eax, _patemp         \n\t" // save pa for later use
+      // test if pa <= pb
+      "cmpl %%ecx, %%eax           \n\t"
+      "jna paeth_abb               \n\t"
+      // pa > pb; now test if pb <= pc
+      "cmpl _pctemp, %%ecx         \n\t"
+      "jna paeth_bbc               \n\t"
+      // pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
+      "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
+      "jmp paeth_paeth             \n\t"
+
+   "paeth_bbc:                     \n\t"
+      // pb <= pc; Raw(x) = Paeth(x) + Prior(x)
+      "movb (%%esi,%%ebx,), %%cl   \n\t" // load Prior(x) into cl
+      "jmp paeth_paeth             \n\t"
+
+   "paeth_abb:                     \n\t"
+      // pa <= pb; now test if pa <= pc
+      "cmpl _pctemp, %%eax         \n\t"
+      "jna paeth_abc               \n\t"
+      // pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
+      "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
+      "jmp paeth_paeth             \n\t"
+
+   "paeth_abc:                     \n\t"
+      // pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp)
+      "movb (%%edi,%%edx,), %%cl   \n\t" // load Raw(x-bpp) into cl
+
+   "paeth_paeth:                   \n\t"
+      "incl %%ebx                  \n\t"
+      "incl %%edx                  \n\t"
+      // Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256
+      "addb %%cl, -1(%%edi,%%ebx,) \n\t"
+      "cmpl _dif, %%ebx            \n\t"
+      "jb paeth_lp1                \n\t"
+
+   "paeth_go:                      \n\t"
+      "movl _FullLength, %%ecx     \n\t"
+      "movl %%ecx, %%eax           \n\t"
+      "subl %%ebx, %%eax           \n\t" // subtract alignment fix
+      "andl $0x00000007, %%eax     \n\t" // calc bytes over mult of 8
+      "subl %%eax, %%ecx           \n\t" // drop over bytes from original length
+      "movl %%ecx, _MMXLength      \n\t"
+#ifdef __PIC__
+      "popl %%ebx                  \n\t" // restore index to Global Offset Table
+#endif
+
+      : "=c" (dummy_value_c),            // output regs (dummy)
+        "=S" (dummy_value_S),
+        "=D" (dummy_value_D)
+
+      : "0" (bpp),       // ecx          // input regs
+        "1" (prev_row),  // esi
+        "2" (row)        // edi
+
+      : "%eax", "%edx"                   // clobber list
+#ifndef __PIC__
+      , "%ebx"
+#endif
+   );
+
+   // now do the math for the rest of the row
+   switch (bpp)
+   {
+      case 3:
+      {
+         _ActiveMask.use = 0x0000000000ffffffLL;
+         _ActiveMaskEnd.use = 0xffff000000000000LL;
+         _ShiftBpp.use = 24;    // == bpp(3) * 8
+         _ShiftRem.use = 40;    // == 64 - 24
+
+         __asm__ __volatile__ (
+            "movl _dif, %%ecx            \n\t"
+// preload  "movl row, %%edi             \n\t"
+// preload  "movl prev_row, %%esi        \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            // prime the pump:  load the first Raw(x-bpp) data set
+            "movq -8(%%edi,%%ecx,), %%mm1 \n\t"
+         "paeth_3lp:                     \n\t"
+            "psrlq _ShiftRem, %%mm1      \n\t" // shift last 3 bytes to 1st
+                                               // 3 bytes
+            "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
+            "punpcklbw %%mm0, %%mm1      \n\t" // unpack High bytes of a
+            "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // prep c=Prior(x-bpp) bytes
+            "punpcklbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
+            "psrlq _ShiftRem, %%mm3      \n\t" // shift last 3 bytes to 1st
+                                               // 3 bytes
+            // pav = p - a = (a + b - c) - a = b - c
+            "movq %%mm2, %%mm4           \n\t"
+            "punpcklbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
+            // pbv = p - b = (a + b - c) - b = a - c
+            "movq %%mm1, %%mm5           \n\t"
+            "psubw %%mm3, %%mm4          \n\t"
+            "pxor %%mm7, %%mm7           \n\t"
+            // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
+            "movq %%mm4, %%mm6           \n\t"
+            "psubw %%mm3, %%mm5          \n\t"
+
+            // pa = abs(p-a) = abs(pav)
+            // pb = abs(p-b) = abs(pbv)
+            // pc = abs(p-c) = abs(pcv)
+            "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
+            "paddw %%mm5, %%mm6          \n\t"
+            "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
+            "psubw %%mm0, %%mm4          \n\t"
+            "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
+            "psubw %%mm0, %%mm4          \n\t"
+            "psubw %%mm7, %%mm5          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
+            "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "psubw %%mm7, %%mm5          \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            //  test pa <= pb
+            "movq %%mm4, %%mm7           \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
+            "movq %%mm7, %%mm0           \n\t"
+            // use mm7 mask to merge pa & pb
+            "pand %%mm7, %%mm5           \n\t"
+            // use mm0 mask copy to merge a & b
+            "pand %%mm0, %%mm2           \n\t"
+            "pandn %%mm4, %%mm7          \n\t"
+            "pandn %%mm1, %%mm0          \n\t"
+            "paddw %%mm5, %%mm7          \n\t"
+            "paddw %%mm2, %%mm0          \n\t"
+            //  test  ((pa <= pb)? pa:pb) <= pc
+            "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
+            "pxor %%mm1, %%mm1           \n\t"
+            "pand %%mm7, %%mm3           \n\t"
+            "pandn %%mm0, %%mm7          \n\t"
+            "paddw %%mm3, %%mm7          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            "packuswb %%mm1, %%mm7       \n\t"
+            "movq (%%esi,%%ecx,), %%mm3  \n\t" // load c=Prior(x-bpp)
+            "pand _ActiveMask, %%mm7     \n\t"
+            "movq %%mm3, %%mm2           \n\t" // load b=Prior(x) step 1
+            "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
+            "punpcklbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
+            "movq %%mm7, (%%edi,%%ecx,)  \n\t" // write back updated value
+            "movq %%mm7, %%mm1           \n\t" // now mm1 will be used as
+                                               // Raw(x-bpp)
+            // now do Paeth for 2nd set of bytes (3-5)
+            "psrlq _ShiftBpp, %%mm2      \n\t" // load b=Prior(x) step 2
+            "punpcklbw %%mm0, %%mm1      \n\t" // unpack High bytes of a
+            "pxor %%mm7, %%mm7           \n\t"
+            "punpcklbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
+            // pbv = p - b = (a + b - c) - b = a - c
+            "movq %%mm1, %%mm5           \n\t"
+            // pav = p - a = (a + b - c) - a = b - c
+            "movq %%mm2, %%mm4           \n\t"
+            "psubw %%mm3, %%mm5          \n\t"
+            "psubw %%mm3, %%mm4          \n\t"
+            // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) =
+            //       pav + pbv = pbv + pav
+            "movq %%mm5, %%mm6           \n\t"
+            "paddw %%mm4, %%mm6          \n\t"
+
+            // pa = abs(p-a) = abs(pav)
+            // pb = abs(p-b) = abs(pbv)
+            // pc = abs(p-c) = abs(pcv)
+            "pcmpgtw %%mm5, %%mm0        \n\t" // create mask pbv bytes < 0
+            "pcmpgtw %%mm4, %%mm7        \n\t" // create mask pav bytes < 0
+            "pand %%mm5, %%mm0           \n\t" // only pbv bytes < 0 in mm0
+            "pand %%mm4, %%mm7           \n\t" // only pav bytes < 0 in mm7
+            "psubw %%mm0, %%mm5          \n\t"
+            "psubw %%mm7, %%mm4          \n\t"
+            "psubw %%mm0, %%mm5          \n\t"
+            "psubw %%mm7, %%mm4          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
+            "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "psubw %%mm0, %%mm6          \n\t"
+            //  test pa <= pb
+            "movq %%mm4, %%mm7           \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
+            "movq %%mm7, %%mm0           \n\t"
+            // use mm7 mask to merge pa & pb
+            "pand %%mm7, %%mm5           \n\t"
+            // use mm0 mask copy to merge a & b
+            "pand %%mm0, %%mm2           \n\t"
+            "pandn %%mm4, %%mm7          \n\t"
+            "pandn %%mm1, %%mm0          \n\t"
+            "paddw %%mm5, %%mm7          \n\t"
+            "paddw %%mm2, %%mm0          \n\t"
+            //  test  ((pa <= pb)? pa:pb) <= pc
+            "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
+            "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
+            "pand %%mm7, %%mm3           \n\t"
+            "pandn %%mm0, %%mm7          \n\t"
+            "pxor %%mm1, %%mm1           \n\t"
+            "paddw %%mm3, %%mm7          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            "packuswb %%mm1, %%mm7       \n\t"
+            "movq %%mm2, %%mm3           \n\t" // load c=Prior(x-bpp) step 1
+            "pand _ActiveMask, %%mm7     \n\t"
+            "punpckhbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
+            "psllq _ShiftBpp, %%mm7      \n\t" // shift bytes to 2nd group of
+                                               // 3 bytes
+             // pav = p - a = (a + b - c) - a = b - c
+            "movq %%mm2, %%mm4           \n\t"
+            "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
+            "psllq _ShiftBpp, %%mm3      \n\t" // load c=Prior(x-bpp) step 2
+            "movq %%mm7, (%%edi,%%ecx,)  \n\t" // write back updated value
+            "movq %%mm7, %%mm1           \n\t"
+            "punpckhbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
+            "psllq _ShiftBpp, %%mm1      \n\t" // shift bytes
+                                    // now mm1 will be used as Raw(x-bpp)
+            // now do Paeth for 3rd, and final, set of bytes (6-7)
+            "pxor %%mm7, %%mm7           \n\t"
+            "punpckhbw %%mm0, %%mm1      \n\t" // unpack High bytes of a
+            "psubw %%mm3, %%mm4          \n\t"
+            // pbv = p - b = (a + b - c) - b = a - c
+            "movq %%mm1, %%mm5           \n\t"
+            // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
+            "movq %%mm4, %%mm6           \n\t"
+            "psubw %%mm3, %%mm5          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            "paddw %%mm5, %%mm6          \n\t"
+
+            // pa = abs(p-a) = abs(pav)
+            // pb = abs(p-b) = abs(pbv)
+            // pc = abs(p-c) = abs(pcv)
+            "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
+            "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
+            "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
+            "psubw %%mm0, %%mm4          \n\t"
+            "psubw %%mm7, %%mm5          \n\t"
+            "psubw %%mm0, %%mm4          \n\t"
+            "psubw %%mm7, %%mm5          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
+            "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "psubw %%mm0, %%mm6          \n\t"
+            //  test pa <= pb
+            "movq %%mm4, %%mm7           \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
+            "movq %%mm7, %%mm0           \n\t"
+            // use mm0 mask copy to merge a & b
+            "pand %%mm0, %%mm2           \n\t"
+            // use mm7 mask to merge pa & pb
+            "pand %%mm7, %%mm5           \n\t"
+            "pandn %%mm1, %%mm0          \n\t"
+            "pandn %%mm4, %%mm7          \n\t"
+            "paddw %%mm2, %%mm0          \n\t"
+            "paddw %%mm5, %%mm7          \n\t"
+            //  test  ((pa <= pb)? pa:pb) <= pc
+            "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
+            "pand %%mm7, %%mm3           \n\t"
+            "pandn %%mm0, %%mm7          \n\t"
+            "paddw %%mm3, %%mm7          \n\t"
+            "pxor %%mm1, %%mm1           \n\t"
+            "packuswb %%mm7, %%mm1       \n\t"
+            // step ecx to next set of 8 bytes and repeat loop til done
+            "addl $8, %%ecx              \n\t"
+            "pand _ActiveMaskEnd, %%mm1  \n\t"
+            "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add Paeth predictor with
+                                                 // Raw(x)
+
+            "cmpl _MMXLength, %%ecx      \n\t"
+            "pxor %%mm0, %%mm0           \n\t" // pxor does not affect flags
+            "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
+                                 // mm1 will be used as Raw(x-bpp) next loop
+                           // mm3 ready to be used as Prior(x-bpp) next loop
+            "jb paeth_3lp                \n\t"
+
+            : "=S" (dummy_value_S),             // output regs (dummy)
+              "=D" (dummy_value_D)
+
+            : "0" (prev_row),  // esi           // input regs
+              "1" (row)        // edi
+
+            : "%ecx"                            // clobber list
+#if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
+            , "%mm0", "%mm1", "%mm2", "%mm3"
+            , "%mm4", "%mm5", "%mm6", "%mm7"
+#endif
+         );
+      }
+      break;  // end 3 bpp
+
+      case 6:
+      //case 7:   // GRR BOGUS
+      //case 5:   // GRR BOGUS
+      {
+         _ActiveMask.use  = 0x00000000ffffffffLL;
+         _ActiveMask2.use = 0xffffffff00000000LL;
+         _ShiftBpp.use = bpp << 3;    // == bpp * 8
+         _ShiftRem.use = 64 - _ShiftBpp.use;
+
+         __asm__ __volatile__ (
+            "movl _dif, %%ecx            \n\t"
+// preload  "movl row, %%edi             \n\t"
+// preload  "movl prev_row, %%esi        \n\t"
+            // prime the pump:  load the first Raw(x-bpp) data set
+            "movq -8(%%edi,%%ecx,), %%mm1 \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+
+         "paeth_6lp:                     \n\t"
+            // must shift to position Raw(x-bpp) data
+            "psrlq _ShiftRem, %%mm1      \n\t"
+            // do first set of 4 bytes
+            "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
+            "punpcklbw %%mm0, %%mm1      \n\t" // unpack Low bytes of a
+            "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
+            "punpcklbw %%mm0, %%mm2      \n\t" // unpack Low bytes of b
+            // must shift to position Prior(x-bpp) data
+            "psrlq _ShiftRem, %%mm3      \n\t"
+            // pav = p - a = (a + b - c) - a = b - c
+            "movq %%mm2, %%mm4           \n\t"
+            "punpcklbw %%mm0, %%mm3      \n\t" // unpack Low bytes of c
+            // pbv = p - b = (a + b - c) - b = a - c
+            "movq %%mm1, %%mm5           \n\t"
+            "psubw %%mm3, %%mm4          \n\t"
+            "pxor %%mm7, %%mm7           \n\t"
+            // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
+            "movq %%mm4, %%mm6           \n\t"
+            "psubw %%mm3, %%mm5          \n\t"
+            // pa = abs(p-a) = abs(pav)
+            // pb = abs(p-b) = abs(pbv)
+            // pc = abs(p-c) = abs(pcv)
+            "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
+            "paddw %%mm5, %%mm6          \n\t"
+            "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
+            "psubw %%mm0, %%mm4          \n\t"
+            "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
+            "psubw %%mm0, %%mm4          \n\t"
+            "psubw %%mm7, %%mm5          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
+            "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "psubw %%mm7, %%mm5          \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            //  test pa <= pb
+            "movq %%mm4, %%mm7           \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
+            "movq %%mm7, %%mm0           \n\t"
+            // use mm7 mask to merge pa & pb
+            "pand %%mm7, %%mm5           \n\t"
+            // use mm0 mask copy to merge a & b
+            "pand %%mm0, %%mm2           \n\t"
+            "pandn %%mm4, %%mm7          \n\t"
+            "pandn %%mm1, %%mm0          \n\t"
+            "paddw %%mm5, %%mm7          \n\t"
+            "paddw %%mm2, %%mm0          \n\t"
+            //  test  ((pa <= pb)? pa:pb) <= pc
+            "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
+            "pxor %%mm1, %%mm1           \n\t"
+            "pand %%mm7, %%mm3           \n\t"
+            "pandn %%mm0, %%mm7          \n\t"
+            "paddw %%mm3, %%mm7          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            "packuswb %%mm1, %%mm7       \n\t"
+            "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // load c=Prior(x-bpp)
+            "pand _ActiveMask, %%mm7     \n\t"
+            "psrlq _ShiftRem, %%mm3      \n\t"
+            "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x) step 1
+            "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor and Raw(x)
+            "movq %%mm2, %%mm6           \n\t"
+            "movq %%mm7, (%%edi,%%ecx,)  \n\t" // write back updated value
+            "movq -8(%%edi,%%ecx,), %%mm1 \n\t"
+            "psllq _ShiftBpp, %%mm6      \n\t"
+            "movq %%mm7, %%mm5           \n\t"
+            "psrlq _ShiftRem, %%mm1      \n\t"
+            "por %%mm6, %%mm3            \n\t"
+            "psllq _ShiftBpp, %%mm5      \n\t"
+            "punpckhbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
+            "por %%mm5, %%mm1            \n\t"
+            // do second set of 4 bytes
+            "punpckhbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
+            "punpckhbw %%mm0, %%mm1      \n\t" // unpack High bytes of a
+            // pav = p - a = (a + b - c) - a = b - c
+            "movq %%mm2, %%mm4           \n\t"
+            // pbv = p - b = (a + b - c) - b = a - c
+            "movq %%mm1, %%mm5           \n\t"
+            "psubw %%mm3, %%mm4          \n\t"
+            "pxor %%mm7, %%mm7           \n\t"
+            // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
+            "movq %%mm4, %%mm6           \n\t"
+            "psubw %%mm3, %%mm5          \n\t"
+            // pa = abs(p-a) = abs(pav)
+            // pb = abs(p-b) = abs(pbv)
+            // pc = abs(p-c) = abs(pcv)
+            "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
+            "paddw %%mm5, %%mm6          \n\t"
+            "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
+            "psubw %%mm0, %%mm4          \n\t"
+            "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
+            "psubw %%mm0, %%mm4          \n\t"
+            "psubw %%mm7, %%mm5          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
+            "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "psubw %%mm7, %%mm5          \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            //  test pa <= pb
+            "movq %%mm4, %%mm7           \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
+            "movq %%mm7, %%mm0           \n\t"
+            // use mm7 mask to merge pa & pb
+            "pand %%mm7, %%mm5           \n\t"
+            // use mm0 mask copy to merge a & b
+            "pand %%mm0, %%mm2           \n\t"
+            "pandn %%mm4, %%mm7          \n\t"
+            "pandn %%mm1, %%mm0          \n\t"
+            "paddw %%mm5, %%mm7          \n\t"
+            "paddw %%mm2, %%mm0          \n\t"
+            //  test  ((pa <= pb)? pa:pb) <= pc
+            "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
+            "pxor %%mm1, %%mm1           \n\t"
+            "pand %%mm7, %%mm3           \n\t"
+            "pandn %%mm0, %%mm7          \n\t"
+            "pxor %%mm1, %%mm1           \n\t"
+            "paddw %%mm3, %%mm7          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            // step ecx to next set of 8 bytes and repeat loop til done
+            "addl $8, %%ecx              \n\t"
+            "packuswb %%mm7, %%mm1       \n\t"
+            "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add Paeth predictor with Raw(x)
+            "cmpl _MMXLength, %%ecx      \n\t"
+            "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
+                                // mm1 will be used as Raw(x-bpp) next loop
+            "jb paeth_6lp                \n\t"
+
+            : "=S" (dummy_value_S),             // output regs (dummy)
+              "=D" (dummy_value_D)
+
+            : "0" (prev_row),  // esi           // input regs
+              "1" (row)        // edi
+
+            : "%ecx"                            // clobber list
+#if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
+            , "%mm0", "%mm1", "%mm2", "%mm3"
+            , "%mm4", "%mm5", "%mm6", "%mm7"
+#endif
+         );
+      }
+      break;  // end 6 bpp
+
+      case 4:
+      {
+         _ActiveMask.use  = 0x00000000ffffffffLL;
+
+         __asm__ __volatile__ (
+            "movl _dif, %%ecx            \n\t"
+// preload  "movl row, %%edi             \n\t"
+// preload  "movl prev_row, %%esi        \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            // prime the pump:  load the first Raw(x-bpp) data set
+            "movq -8(%%edi,%%ecx,), %%mm1 \n\t" // only time should need to read
+                                     //  a=Raw(x-bpp) bytes
+         "paeth_4lp:                     \n\t"
+            // do first set of 4 bytes
+            "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
+            "punpckhbw %%mm0, %%mm1      \n\t" // unpack Low bytes of a
+            "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
+            "punpcklbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
+            // pav = p - a = (a + b - c) - a = b - c
+            "movq %%mm2, %%mm4           \n\t"
+            "punpckhbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
+            // pbv = p - b = (a + b - c) - b = a - c
+            "movq %%mm1, %%mm5           \n\t"
+            "psubw %%mm3, %%mm4          \n\t"
+            "pxor %%mm7, %%mm7           \n\t"
+            // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
+            "movq %%mm4, %%mm6           \n\t"
+            "psubw %%mm3, %%mm5          \n\t"
+            // pa = abs(p-a) = abs(pav)
+            // pb = abs(p-b) = abs(pbv)
+            // pc = abs(p-c) = abs(pcv)
+            "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
+            "paddw %%mm5, %%mm6          \n\t"
+            "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
+            "psubw %%mm0, %%mm4          \n\t"
+            "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
+            "psubw %%mm0, %%mm4          \n\t"
+            "psubw %%mm7, %%mm5          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
+            "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "psubw %%mm7, %%mm5          \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            //  test pa <= pb
+            "movq %%mm4, %%mm7           \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
+            "movq %%mm7, %%mm0           \n\t"
+            // use mm7 mask to merge pa & pb
+            "pand %%mm7, %%mm5           \n\t"
+            // use mm0 mask copy to merge a & b
+            "pand %%mm0, %%mm2           \n\t"
+            "pandn %%mm4, %%mm7          \n\t"
+            "pandn %%mm1, %%mm0          \n\t"
+            "paddw %%mm5, %%mm7          \n\t"
+            "paddw %%mm2, %%mm0          \n\t"
+            //  test  ((pa <= pb)? pa:pb) <= pc
+            "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
+            "pxor %%mm1, %%mm1           \n\t"
+            "pand %%mm7, %%mm3           \n\t"
+            "pandn %%mm0, %%mm7          \n\t"
+            "paddw %%mm3, %%mm7          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            "packuswb %%mm1, %%mm7       \n\t"
+            "movq (%%esi,%%ecx,), %%mm3  \n\t" // load c=Prior(x-bpp)
+            "pand _ActiveMask, %%mm7     \n\t"
+            "movq %%mm3, %%mm2           \n\t" // load b=Prior(x) step 1
+            "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
+            "punpcklbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
+            "movq %%mm7, (%%edi,%%ecx,)  \n\t" // write back updated value
+            "movq %%mm7, %%mm1           \n\t" // now mm1 will be used as Raw(x-bpp)
+            // do second set of 4 bytes
+            "punpckhbw %%mm0, %%mm2      \n\t" // unpack Low bytes of b
+            "punpcklbw %%mm0, %%mm1      \n\t" // unpack Low bytes of a
+            // pav = p - a = (a + b - c) - a = b - c
+            "movq %%mm2, %%mm4           \n\t"
+            // pbv = p - b = (a + b - c) - b = a - c
+            "movq %%mm1, %%mm5           \n\t"
+            "psubw %%mm3, %%mm4          \n\t"
+            "pxor %%mm7, %%mm7           \n\t"
+            // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
+            "movq %%mm4, %%mm6           \n\t"
+            "psubw %%mm3, %%mm5          \n\t"
+            // pa = abs(p-a) = abs(pav)
+            // pb = abs(p-b) = abs(pbv)
+            // pc = abs(p-c) = abs(pcv)
+            "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
+            "paddw %%mm5, %%mm6          \n\t"
+            "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
+            "psubw %%mm0, %%mm4          \n\t"
+            "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
+            "psubw %%mm0, %%mm4          \n\t"
+            "psubw %%mm7, %%mm5          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
+            "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "psubw %%mm7, %%mm5          \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            //  test pa <= pb
+            "movq %%mm4, %%mm7           \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
+            "movq %%mm7, %%mm0           \n\t"
+            // use mm7 mask to merge pa & pb
+            "pand %%mm7, %%mm5           \n\t"
+            // use mm0 mask copy to merge a & b
+            "pand %%mm0, %%mm2           \n\t"
+            "pandn %%mm4, %%mm7          \n\t"
+            "pandn %%mm1, %%mm0          \n\t"
+            "paddw %%mm5, %%mm7          \n\t"
+            "paddw %%mm2, %%mm0          \n\t"
+            //  test  ((pa <= pb)? pa:pb) <= pc
+            "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
+            "pxor %%mm1, %%mm1           \n\t"
+            "pand %%mm7, %%mm3           \n\t"
+            "pandn %%mm0, %%mm7          \n\t"
+            "pxor %%mm1, %%mm1           \n\t"
+            "paddw %%mm3, %%mm7          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            // step ecx to next set of 8 bytes and repeat loop til done
+            "addl $8, %%ecx              \n\t"
+            "packuswb %%mm7, %%mm1       \n\t"
+            "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add predictor with Raw(x)
+            "cmpl _MMXLength, %%ecx      \n\t"
+            "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
+                                // mm1 will be used as Raw(x-bpp) next loop
+            "jb paeth_4lp                \n\t"
+
+            : "=S" (dummy_value_S),             // output regs (dummy)
+              "=D" (dummy_value_D)
+
+            : "0" (prev_row),  // esi           // input regs
+              "1" (row)        // edi
+
+            : "%ecx"                            // clobber list
+#if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
+            , "%mm0", "%mm1", "%mm2", "%mm3"
+            , "%mm4", "%mm5", "%mm6", "%mm7"
+#endif
+         );
+      }
+      break;  // end 4 bpp
+
+      case 8:                          // bpp == 8
+      {
+         _ActiveMask.use  = 0x00000000ffffffffLL;
+
+         __asm__ __volatile__ (
+            "movl _dif, %%ecx            \n\t"
+// preload  "movl row, %%edi             \n\t"
+// preload  "movl prev_row, %%esi        \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            // prime the pump:  load the first Raw(x-bpp) data set
+            "movq -8(%%edi,%%ecx,), %%mm1 \n\t" // only time should need to read
+                                       //  a=Raw(x-bpp) bytes
+         "paeth_8lp:                     \n\t"
+            // do first set of 4 bytes
+            "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
+            "punpcklbw %%mm0, %%mm1      \n\t" // unpack Low bytes of a
+            "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
+            "punpcklbw %%mm0, %%mm2      \n\t" // unpack Low bytes of b
+            // pav = p - a = (a + b - c) - a = b - c
+            "movq %%mm2, %%mm4           \n\t"
+            "punpcklbw %%mm0, %%mm3      \n\t" // unpack Low bytes of c
+            // pbv = p - b = (a + b - c) - b = a - c
+            "movq %%mm1, %%mm5           \n\t"
+            "psubw %%mm3, %%mm4          \n\t"
+            "pxor %%mm7, %%mm7           \n\t"
+            // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
+            "movq %%mm4, %%mm6           \n\t"
+            "psubw %%mm3, %%mm5          \n\t"
+            // pa = abs(p-a) = abs(pav)
+            // pb = abs(p-b) = abs(pbv)
+            // pc = abs(p-c) = abs(pcv)
+            "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
+            "paddw %%mm5, %%mm6          \n\t"
+            "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
+            "psubw %%mm0, %%mm4          \n\t"
+            "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
+            "psubw %%mm0, %%mm4          \n\t"
+            "psubw %%mm7, %%mm5          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
+            "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "psubw %%mm7, %%mm5          \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            //  test pa <= pb
+            "movq %%mm4, %%mm7           \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
+            "movq %%mm7, %%mm0           \n\t"
+            // use mm7 mask to merge pa & pb
+            "pand %%mm7, %%mm5           \n\t"
+            // use mm0 mask copy to merge a & b
+            "pand %%mm0, %%mm2           \n\t"
+            "pandn %%mm4, %%mm7          \n\t"
+            "pandn %%mm1, %%mm0          \n\t"
+            "paddw %%mm5, %%mm7          \n\t"
+            "paddw %%mm2, %%mm0          \n\t"
+            //  test  ((pa <= pb)? pa:pb) <= pc
+            "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
+            "pxor %%mm1, %%mm1           \n\t"
+            "pand %%mm7, %%mm3           \n\t"
+            "pandn %%mm0, %%mm7          \n\t"
+            "paddw %%mm3, %%mm7          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            "packuswb %%mm1, %%mm7       \n\t"
+            "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
+            "pand _ActiveMask, %%mm7     \n\t"
+            "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
+            "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
+            "punpckhbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
+            "movq %%mm7, (%%edi,%%ecx,)  \n\t" // write back updated value
+            "movq -8(%%edi,%%ecx,), %%mm1 \n\t" // read a=Raw(x-bpp) bytes
+
+            // do second set of 4 bytes
+            "punpckhbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
+            "punpckhbw %%mm0, %%mm1      \n\t" // unpack High bytes of a
+            // pav = p - a = (a + b - c) - a = b - c
+            "movq %%mm2, %%mm4           \n\t"
+            // pbv = p - b = (a + b - c) - b = a - c
+            "movq %%mm1, %%mm5           \n\t"
+            "psubw %%mm3, %%mm4          \n\t"
+            "pxor %%mm7, %%mm7           \n\t"
+            // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
+            "movq %%mm4, %%mm6           \n\t"
+            "psubw %%mm3, %%mm5          \n\t"
+            // pa = abs(p-a) = abs(pav)
+            // pb = abs(p-b) = abs(pbv)
+            // pc = abs(p-c) = abs(pcv)
+            "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
+            "paddw %%mm5, %%mm6          \n\t"
+            "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
+            "psubw %%mm0, %%mm4          \n\t"
+            "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
+            "psubw %%mm0, %%mm4          \n\t"
+            "psubw %%mm7, %%mm5          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
+            "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
+            "psubw %%mm7, %%mm5          \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            //  test pa <= pb
+            "movq %%mm4, %%mm7           \n\t"
+            "psubw %%mm0, %%mm6          \n\t"
+            "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
+            "movq %%mm7, %%mm0           \n\t"
+            // use mm7 mask to merge pa & pb
+            "pand %%mm7, %%mm5           \n\t"
+            // use mm0 mask copy to merge a & b
+            "pand %%mm0, %%mm2           \n\t"
+            "pandn %%mm4, %%mm7          \n\t"
+            "pandn %%mm1, %%mm0          \n\t"
+            "paddw %%mm5, %%mm7          \n\t"
+            "paddw %%mm2, %%mm0          \n\t"
+            //  test  ((pa <= pb)? pa:pb) <= pc
+            "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
+            "pxor %%mm1, %%mm1           \n\t"
+            "pand %%mm7, %%mm3           \n\t"
+            "pandn %%mm0, %%mm7          \n\t"
+            "pxor %%mm1, %%mm1           \n\t"
+            "paddw %%mm3, %%mm7          \n\t"
+            "pxor %%mm0, %%mm0           \n\t"
+            // step ecx to next set of 8 bytes and repeat loop til done
+            "addl $8, %%ecx              \n\t"
+            "packuswb %%mm7, %%mm1       \n\t"
+            "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add Paeth predictor with Raw(x)
+            "cmpl _MMXLength, %%ecx      \n\t"
+            "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
+                            // mm1 will be used as Raw(x-bpp) next loop
+            "jb paeth_8lp                \n\t"
+
+            : "=S" (dummy_value_S),             // output regs (dummy)
+              "=D" (dummy_value_D)
+
+            : "0" (prev_row),  // esi           // input regs
+              "1" (row)        // edi
+
+            : "%ecx"                            // clobber list
+#if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
+            , "%mm0", "%mm1", "%mm2", "%mm3"
+            , "%mm4", "%mm5", "%mm6", "%mm7"
+#endif
+         );
+      }
+      break;  // end 8 bpp
+
+      case 1:                // bpp = 1
+      case 2:                // bpp = 2
+      default:               // bpp > 8
+      {
+         __asm__ __volatile__ (
+#ifdef __PIC__
+            "pushl %%ebx                 \n\t" // save Global Offset Table index
+#endif
+            "movl _dif, %%ebx            \n\t"
+            "cmpl _FullLength, %%ebx     \n\t"
+            "jnb paeth_dend              \n\t"
+
+// preload  "movl row, %%edi             \n\t"
+// preload  "movl prev_row, %%esi        \n\t"
+            // do Paeth decode for remaining bytes
+            "movl %%ebx, %%edx           \n\t"
+// preload  "subl bpp, %%edx             \n\t" // (bpp is preloaded into ecx)
+            "subl %%ecx, %%edx           \n\t" // edx = ebx - bpp
+            "xorl %%ecx, %%ecx           \n\t" // zero ecx before using cl & cx
+
+         "paeth_dlp:                     \n\t"
+            "xorl %%eax, %%eax           \n\t"
+            // pav = p - a = (a + b - c) - a = b - c
+            "movb (%%esi,%%ebx,), %%al   \n\t" // load Prior(x) into al
+            "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
+            "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
+            "movl %%eax, _patemp         \n\t" // Save pav for later use
+            "xorl %%eax, %%eax           \n\t"
+            // pbv = p - b = (a + b - c) - b = a - c
+            "movb (%%edi,%%edx,), %%al   \n\t" // load Raw(x-bpp) into al
+            "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
+            "movl %%eax, %%ecx           \n\t"
+            // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
+            "addl _patemp, %%eax         \n\t" // pcv = pav + pbv
+            // pc = abs(pcv)
+            "testl $0x80000000, %%eax    \n\t"
+            "jz paeth_dpca               \n\t"
+            "negl %%eax                  \n\t" // reverse sign of neg values
+
+         "paeth_dpca:                    \n\t"
+            "movl %%eax, _pctemp         \n\t" // save pc for later use
+            // pb = abs(pbv)
+            "testl $0x80000000, %%ecx    \n\t"
+            "jz paeth_dpba               \n\t"
+            "negl %%ecx                  \n\t" // reverse sign of neg values
+
+         "paeth_dpba:                    \n\t"
+            "movl %%ecx, _pbtemp         \n\t" // save pb for later use
+            // pa = abs(pav)
+            "movl _patemp, %%eax         \n\t"
+            "testl $0x80000000, %%eax    \n\t"
+            "jz paeth_dpaa               \n\t"
+            "negl %%eax                  \n\t" // reverse sign of neg values
+
+         "paeth_dpaa:                    \n\t"
+            "movl %%eax, _patemp         \n\t" // save pa for later use
+            // test if pa <= pb
+            "cmpl %%ecx, %%eax           \n\t"
+            "jna paeth_dabb              \n\t"
+            // pa > pb; now test if pb <= pc
+            "cmpl _pctemp, %%ecx         \n\t"
+            "jna paeth_dbbc              \n\t"
+            // pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
+            "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
+            "jmp paeth_dpaeth            \n\t"
+
+         "paeth_dbbc:                    \n\t"
+            // pb <= pc; Raw(x) = Paeth(x) + Prior(x)
+            "movb (%%esi,%%ebx,), %%cl   \n\t" // load Prior(x) into cl
+            "jmp paeth_dpaeth            \n\t"
+
+         "paeth_dabb:                    \n\t"
+            // pa <= pb; now test if pa <= pc
+            "cmpl _pctemp, %%eax         \n\t"
+            "jna paeth_dabc              \n\t"
+            // pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
+            "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
+            "jmp paeth_dpaeth            \n\t"
+
+         "paeth_dabc:                    \n\t"
+            // pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp)
+            "movb (%%edi,%%edx,), %%cl   \n\t" // load Raw(x-bpp) into cl
+
+         "paeth_dpaeth:                  \n\t"
+            "incl %%ebx                  \n\t"
+            "incl %%edx                  \n\t"
+            // Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256
+            "addb %%cl, -1(%%edi,%%ebx,) \n\t"
+            "cmpl _FullLength, %%ebx     \n\t"
+            "jb paeth_dlp                \n\t"
+
+         "paeth_dend:                    \n\t"
+#ifdef __PIC__
+            "popl %%ebx                  \n\t" // index to Global Offset Table
+#endif
+
+            : "=c" (dummy_value_c),            // output regs (dummy)
+              "=S" (dummy_value_S),
+              "=D" (dummy_value_D)
+
+            : "0" (bpp),       // ecx          // input regs
+              "1" (prev_row),  // esi
+              "2" (row)        // edi
+
+            : "%eax", "%edx"                   // clobber list
+#ifndef __PIC__
+            , "%ebx"
+#endif
+         );
+      }
+      return;                   // No need to go further with this one
+
+   } // end switch (bpp)
+
+   __asm__ __volatile__ (
+      // MMX acceleration complete; now do clean-up
+      // check if any remaining bytes left to decode
+#ifdef __PIC__
+      "pushl %%ebx                 \n\t" // save index to Global Offset Table
+#endif
+      "movl _MMXLength, %%ebx      \n\t"
+      "cmpl _FullLength, %%ebx     \n\t"
+      "jnb paeth_end               \n\t"
+//pre "movl row, %%edi             \n\t"
+//pre "movl prev_row, %%esi        \n\t"
+      // do Paeth decode for remaining bytes
+      "movl %%ebx, %%edx           \n\t"
+//pre "subl bpp, %%edx             \n\t" // (bpp is preloaded into ecx)
+      "subl %%ecx, %%edx           \n\t" // edx = ebx - bpp
+      "xorl %%ecx, %%ecx           \n\t" // zero ecx before using cl & cx below
+
+   "paeth_lp2:                     \n\t"
+      "xorl %%eax, %%eax           \n\t"
+      // pav = p - a = (a + b - c) - a = b - c
+      "movb (%%esi,%%ebx,), %%al   \n\t" // load Prior(x) into al
+      "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
+      "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
+      "movl %%eax, _patemp         \n\t" // Save pav for later use
+      "xorl %%eax, %%eax           \n\t"
+      // pbv = p - b = (a + b - c) - b = a - c
+      "movb (%%edi,%%edx,), %%al   \n\t" // load Raw(x-bpp) into al
+      "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
+      "movl %%eax, %%ecx           \n\t"
+      // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
+      "addl _patemp, %%eax         \n\t" // pcv = pav + pbv
+      // pc = abs(pcv)
+      "testl $0x80000000, %%eax    \n\t"
+      "jz paeth_pca2               \n\t"
+      "negl %%eax                  \n\t" // reverse sign of neg values
+
+   "paeth_pca2:                    \n\t"
+      "movl %%eax, _pctemp         \n\t" // save pc for later use
+      // pb = abs(pbv)
+      "testl $0x80000000, %%ecx    \n\t"
+      "jz paeth_pba2               \n\t"
+      "negl %%ecx                  \n\t" // reverse sign of neg values
+
+   "paeth_pba2:                    \n\t"
+      "movl %%ecx, _pbtemp         \n\t" // save pb for later use
+      // pa = abs(pav)
+      "movl _patemp, %%eax         \n\t"
+      "testl $0x80000000, %%eax    \n\t"
+      "jz paeth_paa2               \n\t"
+      "negl %%eax                  \n\t" // reverse sign of neg values
+
+   "paeth_paa2:                    \n\t"
+      "movl %%eax, _patemp         \n\t" // save pa for later use
+      // test if pa <= pb
+      "cmpl %%ecx, %%eax           \n\t"
+      "jna paeth_abb2              \n\t"
+      // pa > pb; now test if pb <= pc
+      "cmpl _pctemp, %%ecx         \n\t"
+      "jna paeth_bbc2              \n\t"
+      // pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
+      "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
+      "jmp paeth_paeth2            \n\t"
+
+   "paeth_bbc2:                    \n\t"
+      // pb <= pc; Raw(x) = Paeth(x) + Prior(x)
+      "movb (%%esi,%%ebx,), %%cl   \n\t" // load Prior(x) into cl
+      "jmp paeth_paeth2            \n\t"
+
+   "paeth_abb2:                    \n\t"
+      // pa <= pb; now test if pa <= pc
+      "cmpl _pctemp, %%eax         \n\t"
+      "jna paeth_abc2              \n\t"
+      // pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
+      "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
+      "jmp paeth_paeth2            \n\t"
+
+   "paeth_abc2:                    \n\t"
+      // pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp)
+      "movb (%%edi,%%edx,), %%cl   \n\t" // load Raw(x-bpp) into cl
+
+   "paeth_paeth2:                  \n\t"
+      "incl %%ebx                  \n\t"
+      "incl %%edx                  \n\t"
+      // Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256
+      "addb %%cl, -1(%%edi,%%ebx,) \n\t"
+      "cmpl _FullLength, %%ebx     \n\t"
+      "jb paeth_lp2                \n\t"
+
+   "paeth_end:                     \n\t"
+      "EMMS                        \n\t" // end MMX; prep for poss. FP instrs.
+#ifdef __PIC__
+      "popl %%ebx                  \n\t" // restore index to Global Offset Table
+#endif
+
+      : "=c" (dummy_value_c),            // output regs (dummy)
+        "=S" (dummy_value_S),
+        "=D" (dummy_value_D)
+
+      : "0" (bpp),       // ecx          // input regs
+        "1" (prev_row),  // esi
+        "2" (row)        // edi
+
+      : "%eax", "%edx"                   // clobber list (no input regs!)
+#ifndef __PIC__
+      , "%ebx"
+#endif
+   );
+
+} /* end png_read_filter_row_mmx_paeth() */
+#endif
+
+
+
+
+#ifdef PNG_THREAD_UNSAFE_OK
+//===========================================================================//
+//                                                                           //
+//           P N G _ R E A D _ F I L T E R _ R O W _ M M X _ S U B           //
+//                                                                           //
+//===========================================================================//
+
+// Optimized code for PNG Sub filter decoder
+
+static void /* PRIVATE */
+png_read_filter_row_mmx_sub(png_row_infop row_info, png_bytep row)
+{
+   int bpp;
+   int dummy_value_a;
+   int dummy_value_D;
+
+   bpp = (row_info->pixel_depth + 7) >> 3;   // calc number of bytes per pixel
+   _FullLength = row_info->rowbytes - bpp;   // number of bytes to filter
+
+   __asm__ __volatile__ (
+//pre "movl row, %%edi             \n\t"
+      "movl %%edi, %%esi           \n\t" // lp = row
+//pre "movl bpp, %%eax             \n\t"
+      "addl %%eax, %%edi           \n\t" // rp = row + bpp
+//irr "xorl %%eax, %%eax           \n\t"
+      // get # of bytes to alignment
+      "movl %%edi, _dif            \n\t" // take start of row
+      "addl $0xf, _dif             \n\t" // add 7 + 8 to incr past
+                                         //  alignment boundary
+      "xorl %%ecx, %%ecx           \n\t"
+      "andl $0xfffffff8, _dif      \n\t" // mask to alignment boundary
+      "subl %%edi, _dif            \n\t" // subtract from start ==> value
+      "jz sub_go                   \n\t" //  ecx at alignment
+
+   "sub_lp1:                       \n\t" // fix alignment
+      "movb (%%esi,%%ecx,), %%al   \n\t"
+      "addb %%al, (%%edi,%%ecx,)   \n\t"
+      "incl %%ecx                  \n\t"
+      "cmpl _dif, %%ecx            \n\t"
+      "jb sub_lp1                  \n\t"
+
+   "sub_go:                        \n\t"
+      "movl _FullLength, %%eax     \n\t"
+      "movl %%eax, %%edx           \n\t"
+      "subl %%ecx, %%edx           \n\t" // subtract alignment fix
+      "andl $0x00000007, %%edx     \n\t" // calc bytes over mult of 8
+      "subl %%edx, %%eax           \n\t" // drop over bytes from length
+      "movl %%eax, _MMXLength      \n\t"
+
+      : "=a" (dummy_value_a),   // 0      // output regs (dummy)
+        "=D" (dummy_value_D)    // 1
+
+      : "0" (bpp),              // eax    // input regs
+        "1" (row)               // edi
+
+      : "%ebx", "%ecx", "%edx"            // clobber list
+      , "%esi"
+
+#if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
+      , "%mm0", "%mm1", "%mm2", "%mm3"
+      , "%mm4", "%mm5", "%mm6", "%mm7"
+#endif
+   );
+
+   // now do the math for the rest of the row
+   switch (bpp)
+   {
+      case 3:
+      {
+         _ActiveMask.use  = 0x0000ffffff000000LL;
+         _ShiftBpp.use = 24;       // == 3 * 8
+         _ShiftRem.use  = 40;      // == 64 - 24
+
+         __asm__ __volatile__ (
+// preload  "movl row, %%edi              \n\t"
+            "movq _ActiveMask, %%mm7       \n\t" // load _ActiveMask for 2nd
+                                                //  active byte group
+            "movl %%edi, %%esi            \n\t" // lp = row
+// preload  "movl bpp, %%eax              \n\t"
+            "addl %%eax, %%edi            \n\t" // rp = row + bpp
+            "movq %%mm7, %%mm6            \n\t"
+            "movl _dif, %%edx             \n\t"
+            "psllq _ShiftBpp, %%mm6       \n\t" // move mask in mm6 to cover
+                                                //  3rd active byte group
+            // prime the pump:  load the first Raw(x-bpp) data set
+            "movq -8(%%edi,%%edx,), %%mm1 \n\t"
+
+         "sub_3lp:                        \n\t" // shift data for adding first
+            "psrlq _ShiftRem, %%mm1       \n\t" //  bpp bytes (no need for mask;
+                                                //  shift clears inactive bytes)
+            // add 1st active group
+            "movq (%%edi,%%edx,), %%mm0   \n\t"
+            "paddb %%mm1, %%mm0           \n\t"
+
+            // add 2nd active group
+            "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
+            "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
+            "pand %%mm7, %%mm1            \n\t" // mask to use 2nd active group
+            "paddb %%mm1, %%mm0           \n\t"
+
+            // add 3rd active group
+            "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
+            "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
+            "pand %%mm6, %%mm1            \n\t" // mask to use 3rd active group
+            "addl $8, %%edx               \n\t"
+            "paddb %%mm1, %%mm0           \n\t"
+
+            "cmpl _MMXLength, %%edx       \n\t"
+            "movq %%mm0, -8(%%edi,%%edx,) \n\t" // write updated Raws to array
+            "movq %%mm0, %%mm1            \n\t" // prep 1st add at top of loop
+            "jb sub_3lp                   \n\t"
+
+            : "=a" (dummy_value_a),   // 0      // output regs (dummy)
+              "=D" (dummy_value_D)    // 1
+
+            : "0" (bpp),              // eax    // input regs
+              "1" (row)               // edi
+
+            : "%edx", "%esi"                    // clobber list
+#if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
+            , "%mm0", "%mm1", "%mm6", "%mm7"
+#endif
+         );
+      }
+      break;
+
+      case 1:
+      {
+         __asm__ __volatile__ (
+            "movl _dif, %%edx            \n\t"
+// preload  "movl row, %%edi             \n\t"
+            "cmpl _FullLength, %%edx     \n\t"
+            "jnb sub_1end                \n\t"
+            "movl %%edi, %%esi           \n\t" // lp = row
+            "xorl %%eax, %%eax           \n\t"
+// preload  "movl bpp, %%eax             \n\t"
+            "addl %%eax, %%edi           \n\t" // rp = row + bpp
+
+         "sub_1lp:                       \n\t"
+            "movb (%%esi,%%edx,), %%al   \n\t"
+            "addb %%al, (%%edi,%%edx,)   \n\t"
+            "incl %%edx                  \n\t"
+            "cmpl _FullLength, %%edx     \n\t"
+            "jb sub_1lp                  \n\t"
+
+         "sub_1end:                      \n\t"
+
+            : "=a" (dummy_value_a),   // 0      // output regs (dummy)
+              "=D" (dummy_value_D)    // 1
+
+            : "0" (bpp),              // eax    // input regs
+              "1" (row)               // edi
+
+            : "%edx", "%esi"                    // clobber list
+         );
+      }
+      return;
+
+      case 6:
+      case 4:
+      //case 7:   // GRR BOGUS
+      //case 5:   // GRR BOGUS
+      {
+         _ShiftBpp.use = bpp << 3;
+         _ShiftRem.use = 64 - _ShiftBpp.use;
+
+         __asm__ __volatile__ (
+// preload  "movl row, %%edi              \n\t"
+            "movl _dif, %%edx             \n\t"
+            "movl %%edi, %%esi            \n\t" // lp = row
+// preload  "movl bpp, %%eax              \n\t"
+            "addl %%eax, %%edi            \n\t" // rp = row + bpp
+
+            // prime the pump:  load the first Raw(x-bpp) data set
+            "movq -8(%%edi,%%edx,), %%mm1 \n\t"
+
+         "sub_4lp:                        \n\t" // shift data for adding first
+            "psrlq _ShiftRem, %%mm1       \n\t" //  bpp bytes (no need for mask;
+                                                //  shift clears inactive bytes)
+            "movq (%%edi,%%edx,), %%mm0   \n\t"
+            "paddb %%mm1, %%mm0           \n\t"
+
+            // add 2nd active group
+            "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
+            "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
+            "addl $8, %%edx               \n\t"
+            "paddb %%mm1, %%mm0           \n\t"
+
+            "cmpl _MMXLength, %%edx       \n\t"
+            "movq %%mm0, -8(%%edi,%%edx,) \n\t"
+            "movq %%mm0, %%mm1            \n\t" // prep 1st add at top of loop
+            "jb sub_4lp                   \n\t"
+
+            : "=a" (dummy_value_a),   // 0      // output regs (dummy)
+              "=D" (dummy_value_D)    // 1
+
+            : "0" (bpp),              // eax    // input regs
+              "1" (row)               // edi
+
+            : "%edx", "%esi"                    // clobber list
+#if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
+            , "%mm0", "%mm1"
+#endif
+         );
+      }
+      break;
+
+      case 2:
+      {
+         _ActiveMask.use = 0x00000000ffff0000LL;
+         _ShiftBpp.use = 16;       // == 2 * 8
+         _ShiftRem.use = 48;       // == 64 - 16
+
+         __asm__ __volatile__ (
+            "movq _ActiveMask, %%mm7      \n\t" // load _ActiveMask for 2nd
+                                                //  active byte group
+            "movl _dif, %%edx             \n\t"
+            "movq %%mm7, %%mm6            \n\t"
+// preload  "movl row, %%edi              \n\t"
+            "psllq _ShiftBpp, %%mm6       \n\t" // move mask in mm6 to cover
+                                                //  3rd active byte group
+            "movl %%edi, %%esi            \n\t" // lp = row
+            "movq %%mm6, %%mm5            \n\t"
+// preload  "movl bpp, %%eax              \n\t"
+            "addl %%eax, %%edi            \n\t" // rp = row + bpp
+            "psllq _ShiftBpp, %%mm5       \n\t" // move mask in mm5 to cover
+                                                //  4th active byte group
+            // prime the pump:  load the first Raw(x-bpp) data set
+            "movq -8(%%edi,%%edx,), %%mm1 \n\t"
+
+         "sub_2lp:                        \n\t" // shift data for adding first
+            "psrlq _ShiftRem, %%mm1       \n\t" //  bpp bytes (no need for mask;
+                                                //  shift clears inactive bytes)
+            // add 1st active group
+            "movq (%%edi,%%edx,), %%mm0   \n\t"
+            "paddb %%mm1, %%mm0           \n\t"
+
+            // add 2nd active group
+            "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
+            "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
+            "pand %%mm7, %%mm1            \n\t" // mask to use 2nd active group
+            "paddb %%mm1, %%mm0           \n\t"
+
+            // add 3rd active group
+            "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
+            "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
+            "pand %%mm6, %%mm1            \n\t" // mask to use 3rd active group
+            "paddb %%mm1, %%mm0           \n\t"
+
+            // add 4th active group
+            "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
+            "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
+            "pand %%mm5, %%mm1            \n\t" // mask to use 4th active group
+            "addl $8, %%edx               \n\t"
+            "paddb %%mm1, %%mm0           \n\t"
+            "cmpl _MMXLength, %%edx       \n\t"
+            "movq %%mm0, -8(%%edi,%%edx,) \n\t" // write updated Raws to array
+            "movq %%mm0, %%mm1            \n\t" // prep 1st add at top of loop
+            "jb sub_2lp                   \n\t"
+
+            : "=a" (dummy_value_a),   // 0      // output regs (dummy)
+              "=D" (dummy_value_D)    // 1
+
+            : "0" (bpp),              // eax    // input regs
+              "1" (row)               // edi
+
+            : "%edx", "%esi"                    // clobber list
+#if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
+            , "%mm0", "%mm1", "%mm5", "%mm6", "%mm7"
+#endif
+         );
+      }
+      break;
+
+      case 8:
+      {
+         __asm__ __volatile__ (
+// preload  "movl row, %%edi              \n\t"
+            "movl _dif, %%edx             \n\t"
+            "movl %%edi, %%esi            \n\t" // lp = row
+// preload  "movl bpp, %%eax              \n\t"
+            "addl %%eax, %%edi            \n\t" // rp = row + bpp
+            "movl _MMXLength, %%ecx       \n\t"
+
+            // prime the pump:  load the first Raw(x-bpp) data set
+            "movq -8(%%edi,%%edx,), %%mm7 \n\t"
+            "andl $0x0000003f, %%ecx      \n\t" // calc bytes over mult of 64
+
+         "sub_8lp:                        \n\t"
+            "movq (%%edi,%%edx,), %%mm0   \n\t" // load Sub(x) for 1st 8 bytes
+            "paddb %%mm7, %%mm0           \n\t"
+            "movq 8(%%edi,%%edx,), %%mm1  \n\t" // load Sub(x) for 2nd 8 bytes
+            "movq %%mm0, (%%edi,%%edx,)   \n\t" // write Raw(x) for 1st 8 bytes
+
+            // Now mm0 will be used as Raw(x-bpp) for the 2nd group of 8 bytes.
+            // This will be repeated for each group of 8 bytes with the 8th
+            // group being used as the Raw(x-bpp) for the 1st group of the
+            // next loop.
+
+            "paddb %%mm0, %%mm1           \n\t"
+            "movq 16(%%edi,%%edx,), %%mm2 \n\t" // load Sub(x) for 3rd 8 bytes
+            "movq %%mm1, 8(%%edi,%%edx,)  \n\t" // write Raw(x) for 2nd 8 bytes
+            "paddb %%mm1, %%mm2           \n\t"
+            "movq 24(%%edi,%%edx,), %%mm3 \n\t" // load Sub(x) for 4th 8 bytes
+            "movq %%mm2, 16(%%edi,%%edx,) \n\t" // write Raw(x) for 3rd 8 bytes
+            "paddb %%mm2, %%mm3           \n\t"
+            "movq 32(%%edi,%%edx,), %%mm4 \n\t" // load Sub(x) for 5th 8 bytes
+            "movq %%mm3, 24(%%edi,%%edx,) \n\t" // write Raw(x) for 4th 8 bytes
+            "paddb %%mm3, %%mm4           \n\t"
+            "movq 40(%%edi,%%edx,), %%mm5 \n\t" // load Sub(x) for 6th 8 bytes
+            "movq %%mm4, 32(%%edi,%%edx,) \n\t" // write Raw(x) for 5th 8 bytes
+            "paddb %%mm4, %%mm5           \n\t"
+            "movq 48(%%edi,%%edx,), %%mm6 \n\t" // load Sub(x) for 7th 8 bytes
+            "movq %%mm5, 40(%%edi,%%edx,) \n\t" // write Raw(x) for 6th 8 bytes
+            "paddb %%mm5, %%mm6           \n\t"
+            "movq 56(%%edi,%%edx,), %%mm7 \n\t" // load Sub(x) for 8th 8 bytes
+            "movq %%mm6, 48(%%edi,%%edx,) \n\t" // write Raw(x) for 7th 8 bytes
+            "addl $64, %%edx              \n\t"
+            "paddb %%mm6, %%mm7           \n\t"
+            "cmpl %%ecx, %%edx            \n\t"
+            "movq %%mm7, -8(%%edi,%%edx,) \n\t" // write Raw(x) for 8th 8 bytes
+            "jb sub_8lp                   \n\t"
+
+            "cmpl _MMXLength, %%edx       \n\t"
+            "jnb sub_8lt8                 \n\t"
+
+         "sub_8lpA:                       \n\t"
+            "movq (%%edi,%%edx,), %%mm0   \n\t"
+            "addl $8, %%edx               \n\t"
+            "paddb %%mm7, %%mm0           \n\t"
+            "cmpl _MMXLength, %%edx       \n\t"
+            "movq %%mm0, -8(%%edi,%%edx,) \n\t" // -8 to offset early addl edx
+            "movq %%mm0, %%mm7            \n\t" // move calculated Raw(x) data
+                                                //  to mm1 to be new Raw(x-bpp)
+                                                //  for next loop
+            "jb sub_8lpA                  \n\t"
+
+         "sub_8lt8:                       \n\t"
+
+            : "=a" (dummy_value_a),   // 0      // output regs (dummy)
+              "=D" (dummy_value_D)    // 1
+
+            : "0" (bpp),              // eax    // input regs
+              "1" (row)               // edi
+
+            : "%ecx", "%edx", "%esi"            // clobber list
+#if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
+            , "%mm0", "%mm1", "%mm2", "%mm3", "%mm4", "%mm5", "%mm6", "%mm7"
+#endif
+         );
+      }
+      break;
+
+      default:                // bpp greater than 8 bytes   GRR BOGUS
+      {
+         __asm__ __volatile__ (
+            "movl _dif, %%edx             \n\t"
+// preload  "movl row, %%edi              \n\t"
+            "movl %%edi, %%esi            \n\t" // lp = row
+// preload  "movl bpp, %%eax              \n\t"
+            "addl %%eax, %%edi            \n\t" // rp = row + bpp
+
+         "sub_Alp:                        \n\t"
+            "movq (%%edi,%%edx,), %%mm0   \n\t"
+            "movq (%%esi,%%edx,), %%mm1   \n\t"
+            "addl $8, %%edx               \n\t"
+            "paddb %%mm1, %%mm0           \n\t"
+            "cmpl _MMXLength, %%edx       \n\t"
+            "movq %%mm0, -8(%%edi,%%edx,) \n\t" // mov does not affect flags;
+                                                //  -8 to offset addl edx
+            "jb sub_Alp                   \n\t"
+
+            : "=a" (dummy_value_a),   // 0      // output regs (dummy)
+              "=D" (dummy_value_D)    // 1
+
+            : "0" (bpp),              // eax    // input regs
+              "1" (row)               // edi
+
+            : "%edx", "%esi"                    // clobber list
+#if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
+            , "%mm0", "%mm1"
+#endif
+         );
+      }
+      break;
+
+   } // end switch (bpp)
+
+   __asm__ __volatile__ (
+      "movl _MMXLength, %%edx       \n\t"
+//pre "movl row, %%edi              \n\t"
+      "cmpl _FullLength, %%edx      \n\t"
+      "jnb sub_end                  \n\t"
+
+      "movl %%edi, %%esi            \n\t" // lp = row
+//pre "movl bpp, %%eax              \n\t"
+      "addl %%eax, %%edi            \n\t" // rp = row + bpp
+      "xorl %%eax, %%eax            \n\t"
+
+   "sub_lp2:                        \n\t"
+      "movb (%%esi,%%edx,), %%al    \n\t"
+      "addb %%al, (%%edi,%%edx,)    \n\t"
+      "incl %%edx                   \n\t"
+      "cmpl _FullLength, %%edx      \n\t"
+      "jb sub_lp2                   \n\t"
+
+   "sub_end:                        \n\t"
+      "EMMS                         \n\t" // end MMX instructions
+
+      : "=a" (dummy_value_a),   // 0      // output regs (dummy)
+        "=D" (dummy_value_D)    // 1
+
+      : "0" (bpp),              // eax    // input regs
+        "1" (row)               // edi
+
+      : "%edx", "%esi"                    // clobber list
+   );
+
+} // end of png_read_filter_row_mmx_sub()
+#endif
+
+
+
+
+//===========================================================================//
+//                                                                           //
+//            P N G _ R E A D _ F I L T E R _ R O W _ M M X _ U P            //
+//                                                                           //
+//===========================================================================//
+
+// Optimized code for PNG Up filter decoder
+
+static void /* PRIVATE */
+png_read_filter_row_mmx_up(png_row_infop row_info, png_bytep row,
+                           png_bytep prev_row)
+{
+   png_uint_32 len;
+   int dummy_value_d;   // fix 'forbidden register 3 (dx) was spilled' error
+   int dummy_value_S;
+   int dummy_value_D;
+
+   len = row_info->rowbytes;              // number of bytes to filter
+
+   __asm__ __volatile__ (
+//pre "movl row, %%edi              \n\t"
+      // get # of bytes to alignment
+#ifdef __PIC__
+      "pushl %%ebx                  \n\t"
+#endif
+      "movl %%edi, %%ecx            \n\t"
+      "xorl %%ebx, %%ebx            \n\t"
+      "addl $0x7, %%ecx             \n\t"
+      "xorl %%eax, %%eax            \n\t"
+      "andl $0xfffffff8, %%ecx      \n\t"
+//pre "movl prev_row, %%esi         \n\t"
+      "subl %%edi, %%ecx            \n\t"
+      "jz up_go                     \n\t"
+
+   "up_lp1:                         \n\t" // fix alignment
+      "movb (%%edi,%%ebx,), %%al    \n\t"
+      "addb (%%esi,%%ebx,), %%al    \n\t"
+      "incl %%ebx                   \n\t"
+      "cmpl %%ecx, %%ebx            \n\t"
+      "movb %%al, -1(%%edi,%%ebx,)  \n\t" // mov does not affect flags; -1 to
+      "jb up_lp1                    \n\t" //  offset incl ebx
+
+   "up_go:                          \n\t"
+//pre "movl len, %%edx              \n\t"
+      "movl %%edx, %%ecx            \n\t"
+      "subl %%ebx, %%edx            \n\t" // subtract alignment fix
+      "andl $0x0000003f, %%edx      \n\t" // calc bytes over mult of 64
+      "subl %%edx, %%ecx            \n\t" // drop over bytes from length
+
+      // unrolled loop - use all MMX registers and interleave to reduce
+      // number of branch instructions (loops) and reduce partial stalls
+   "up_loop:                        \n\t"
+      "movq (%%esi,%%ebx,), %%mm1   \n\t"
+      "movq (%%edi,%%ebx,), %%mm0   \n\t"
+      "movq 8(%%esi,%%ebx,), %%mm3  \n\t"
+      "paddb %%mm1, %%mm0           \n\t"
+      "movq 8(%%edi,%%ebx,), %%mm2  \n\t"
+      "movq %%mm0, (%%edi,%%ebx,)   \n\t"
+      "paddb %%mm3, %%mm2           \n\t"
+      "movq 16(%%esi,%%ebx,), %%mm5 \n\t"
+      "movq %%mm2, 8(%%edi,%%ebx,)  \n\t"
+      "movq 16(%%edi,%%ebx,), %%mm4 \n\t"
+      "movq 24(%%esi,%%ebx,), %%mm7 \n\t"
+      "paddb %%mm5, %%mm4           \n\t"
+      "movq 24(%%edi,%%ebx,), %%mm6 \n\t"
+      "movq %%mm4, 16(%%edi,%%ebx,) \n\t"
+      "paddb %%mm7, %%mm6           \n\t"
+      "movq 32(%%esi,%%ebx,), %%mm1 \n\t"
+      "movq %%mm6, 24(%%edi,%%ebx,) \n\t"
+      "movq 32(%%edi,%%ebx,), %%mm0 \n\t"
+      "movq 40(%%esi,%%ebx,), %%mm3 \n\t"
+      "paddb %%mm1, %%mm0           \n\t"
+      "movq 40(%%edi,%%ebx,), %%mm2 \n\t"
+      "movq %%mm0, 32(%%edi,%%ebx,) \n\t"
+      "paddb %%mm3, %%mm2           \n\t"
+      "movq 48(%%esi,%%ebx,), %%mm5 \n\t"
+      "movq %%mm2, 40(%%edi,%%ebx,) \n\t"
+      "movq 48(%%edi,%%ebx,), %%mm4 \n\t"
+      "movq 56(%%esi,%%ebx,), %%mm7 \n\t"
+      "paddb %%mm5, %%mm4           \n\t"
+      "movq 56(%%edi,%%ebx,), %%mm6 \n\t"
+      "movq %%mm4, 48(%%edi,%%ebx,) \n\t"
+      "addl $64, %%ebx              \n\t"
+      "paddb %%mm7, %%mm6           \n\t"
+      "cmpl %%ecx, %%ebx            \n\t"
+      "movq %%mm6, -8(%%edi,%%ebx,) \n\t" // (+56)movq does not affect flags;
+      "jb up_loop                   \n\t" //  -8 to offset addl ebx
+
+      "cmpl $0, %%edx               \n\t" // test for bytes over mult of 64
+      "jz up_end                    \n\t"
+
+      "cmpl $8, %%edx               \n\t" // test for less than 8 bytes
+      "jb up_lt8                    \n\t" //  [added by lcreeve@netins.net]
+
+      "addl %%edx, %%ecx            \n\t"
+      "andl $0x00000007, %%edx      \n\t" // calc bytes over mult of 8
+      "subl %%edx, %%ecx            \n\t" // drop over bytes from length
+      "jz up_lt8                    \n\t"
+
+   "up_lpA:                         \n\t" // use MMX regs to update 8 bytes sim.
+      "movq (%%esi,%%ebx,), %%mm1   \n\t"
+      "movq (%%edi,%%ebx,), %%mm0   \n\t"
+      "addl $8, %%ebx               \n\t"
+      "paddb %%mm1, %%mm0           \n\t"
+      "cmpl %%ecx, %%ebx            \n\t"
+      "movq %%mm0, -8(%%edi,%%ebx,) \n\t" // movq does not affect flags; -8 to
+      "jb up_lpA                    \n\t" //  offset add ebx
+      "cmpl $0, %%edx               \n\t" // test for bytes over mult of 8
+      "jz up_end                    \n\t"
+
+   "up_lt8:                         \n\t"
+      "xorl %%eax, %%eax            \n\t"
+      "addl %%edx, %%ecx            \n\t" // move over byte count into counter
+
+   "up_lp2:                         \n\t" // use x86 regs for remaining bytes
+      "movb (%%edi,%%ebx,), %%al    \n\t"
+      "addb (%%esi,%%ebx,), %%al    \n\t"
+      "incl %%ebx                   \n\t"
+      "cmpl %%ecx, %%ebx            \n\t"
+      "movb %%al, -1(%%edi,%%ebx,)  \n\t" // mov does not affect flags; -1 to
+      "jb up_lp2                    \n\t" //  offset inc ebx
+
+   "up_end:                         \n\t"
+      "EMMS                         \n\t" // conversion of filtered row complete
+#ifdef __PIC__
+      "popl %%ebx                   \n\t"
+#endif
+
+      : "=d" (dummy_value_d),   // 0      // output regs (dummy)
+        "=S" (dummy_value_S),   // 1
+        "=D" (dummy_value_D)    // 2
+
+      : "0" (len),              // edx    // input regs
+        "1" (prev_row),         // esi
+        "2" (row)               // edi
+
+      : "%eax", "%ebx", "%ecx"            // clobber list (no input regs!)
+
+#if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
+      , "%mm0", "%mm1", "%mm2", "%mm3"
+      , "%mm4", "%mm5", "%mm6", "%mm7"
+#endif
+   );
+
+} // end of png_read_filter_row_mmx_up()
+
+#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
+
+
+
+
+/*===========================================================================*/
+/*                                                                           */
+/*                   P N G _ R E A D _ F I L T E R _ R O W                   */
+/*                                                                           */
+/*===========================================================================*/
+
+
+/* Optimized png_read_filter_row routines */
+
+void /* PRIVATE */
+png_read_filter_row(png_structp png_ptr, png_row_infop row_info, png_bytep
+   row, png_bytep prev_row, int filter)
+{
+#ifdef PNG_DEBUG
+   char filnm[10];
+#endif
+
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
+/* GRR:  these are superseded by png_ptr->asm_flags: */
+#define UseMMX_sub    1   // GRR:  converted 20000730
+#define UseMMX_up     1   // GRR:  converted 20000729
+#define UseMMX_avg    1   // GRR:  converted 20000828 (+ 16-bit bugfix 20000916)
+#define UseMMX_paeth  1   // GRR:  converted 20000828
+
+   if (_mmx_supported == 2) {
+       /* this should have happened in png_init_mmx_flags() already */
+#if !defined(PNG_1_0_X)
+       png_warning(png_ptr, "asm_flags may not have been initialized");
+#endif
+       png_mmx_support();
+   }
+#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
+
+#ifdef PNG_DEBUG
+   png_debug(1, "in png_read_filter_row (pnggccrd.c)\n");
+   switch (filter)
+   {
+      case 0: sprintf(filnm, "none");
+         break;
+      case 1: sprintf(filnm, "sub-%s",
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
+#if !defined(PNG_1_0_X)
+        (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_SUB)? "MMX" : 
+#endif
+#endif
+"x86");
+         break;
+      case 2: sprintf(filnm, "up-%s",
+#ifdef PNG_ASSEMBLER_CODE_SUPPORTED
+#if !defined(PNG_1_0_X)
+        (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_UP)? "MMX" :
+#endif
+#endif
+ "x86");
+         break;
+      case 3: sprintf(filnm, "avg-%s",
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
+#if !defined(PNG_1_0_X)
+        (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_AVG)? "MMX" :
+#endif
+#endif
+ "x86");
+         break;
+      case 4: sprintf(filnm, "Paeth-%s",
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
+#if !defined(PNG_1_0_X)
+        (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_PAETH)? "MMX":
+#endif
+#endif
+"x86");
+         break;
+      default: sprintf(filnm, "unknw");
+         break;
+   }
+   png_debug2(0, "row_number=%5ld, %5s, ", png_ptr->row_number, filnm);
+   png_debug1(0, "row=0x%08lx, ", (unsigned long)row);
+   png_debug2(0, "pixdepth=%2d, bytes=%d, ", (int)row_info->pixel_depth,
+      (int)((row_info->pixel_depth + 7) >> 3));
+   png_debug1(0,"rowbytes=%8ld\n", row_info->rowbytes);
+#endif /* PNG_DEBUG */
+
+   switch (filter)
+   {
+      case PNG_FILTER_VALUE_NONE:
+         break;
+
+      case PNG_FILTER_VALUE_SUB:
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
+#if !defined(PNG_1_0_X)
+         if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_SUB) &&
+             (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
+             (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
+#else
+         if (_mmx_supported)
+#endif
+         {
+            png_read_filter_row_mmx_sub(row_info, row);
+         }
+         else
+#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
+         {
+            png_uint_32 i;
+            png_uint_32 istop = row_info->rowbytes;
+            png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3;
+            png_bytep rp = row + bpp;
+            png_bytep lp = row;
+
+            for (i = bpp; i < istop; i++)
+            {
+               *rp = (png_byte)(((int)(*rp) + (int)(*lp++)) & 0xff);
+               rp++;
+            }
+         }  /* end !UseMMX_sub */
+         break;
+
+      case PNG_FILTER_VALUE_UP:
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
+#if !defined(PNG_1_0_X)
+         if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_UP) &&
+             (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
+             (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
+#else
+         if (_mmx_supported)
+#endif
+         {
+            png_read_filter_row_mmx_up(row_info, row, prev_row);
+         }
+          else
+#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
+         {
+            png_uint_32 i;
+            png_uint_32 istop = row_info->rowbytes;
+            png_bytep rp = row;
+            png_bytep pp = prev_row;
+
+            for (i = 0; i < istop; ++i)
+            {
+               *rp = (png_byte)(((int)(*rp) + (int)(*pp++)) & 0xff);
+               rp++;
+            }
+         }  /* end !UseMMX_up */
+         break;
+
+      case PNG_FILTER_VALUE_AVG:
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
+#if !defined(PNG_1_0_X)
+         if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_AVG) &&
+             (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
+             (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
+#else
+         if (_mmx_supported)
+#endif
+         {
+            png_read_filter_row_mmx_avg(row_info, row, prev_row);
+         }
+         else
+#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
+         {
+            png_uint_32 i;
+            png_bytep rp = row;
+            png_bytep pp = prev_row;
+            png_bytep lp = row;
+            png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3;
+            png_uint_32 istop = row_info->rowbytes - bpp;
+
+            for (i = 0; i < bpp; i++)
+            {
+               *rp = (png_byte)(((int)(*rp) +
+                  ((int)(*pp++) >> 1)) & 0xff);
+               rp++;
+            }
+
+            for (i = 0; i < istop; i++)
+            {
+               *rp = (png_byte)(((int)(*rp) +
+                  ((int)(*pp++ + *lp++) >> 1)) & 0xff);
+               rp++;
+            }
+         }  /* end !UseMMX_avg */
+         break;
+
+      case PNG_FILTER_VALUE_PAETH:
+#if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
+#if !defined(PNG_1_0_X)
+         if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_PAETH) &&
+             (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
+             (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
+#else
+         if (_mmx_supported)
+#endif
+         {
+            png_read_filter_row_mmx_paeth(row_info, row, prev_row);
+         }
+         else
+#endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
+         {
+            png_uint_32 i;
+            png_bytep rp = row;
+            png_bytep pp = prev_row;
+            png_bytep lp = row;
+            png_bytep cp = prev_row;
+            png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3;
+            png_uint_32 istop = row_info->rowbytes - bpp;
+
+            for (i = 0; i < bpp; i++)
+            {
+               *rp = (png_byte)(((int)(*rp) + (int)(*pp++)) & 0xff);
+               rp++;
+            }
+
+            for (i = 0; i < istop; i++)   /* use leftover rp,pp */
+            {
+               int a, b, c, pa, pb, pc, p;
+
+               a = *lp++;
+               b = *pp++;
+               c = *cp++;
+
+               p = b - c;
+               pc = a - c;
+
+#ifdef PNG_USE_ABS
+               pa = abs(p);
+               pb = abs(pc);
+               pc = abs(p + pc);
+#else
+               pa = p < 0 ? -p : p;
+               pb = pc < 0 ? -pc : pc;
+               pc = (p + pc) < 0 ? -(p + pc) : p + pc;
+#endif
+
+               /*
+                  if (pa <= pb && pa <= pc)
+                     p = a;
+                  else if (pb <= pc)
+                     p = b;
+                  else
+                     p = c;
+                */
+
+               p = (pa <= pb && pa <= pc) ? a : (pb <= pc) ? b : c;
+
+               *rp = (png_byte)(((int)(*rp) + p) & 0xff);
+               rp++;
+            }
+         }  /* end !UseMMX_paeth */
+         break;
+
+      default:
+         png_warning(png_ptr, "Ignoring bad row-filter type");
+         *row=0;
+         break;
+   }
+}
+
+#endif /* PNG_HAVE_ASSEMBLER_READ_FILTER_ROW */
+
+
+/*===========================================================================*/
+/*                                                                           */
+/*                      P N G _ M M X _ S U P P O R T                        */
+/*                                                                           */
+/*===========================================================================*/
+
+/* GRR NOTES:  (1) the following code assumes 386 or better (pushfl/popfl)
+ *             (2) all instructions compile with gcc 2.7.2.3 and later
+ *             (3) the function is moved down here to prevent gcc from
+ *                  inlining it in multiple places and then barfing be-
+ *                  cause the ".NOT_SUPPORTED" label is multiply defined
+ *             [is there a way to signal that a *single* function should
+ *              not be inlined?  is there a way to modify the label for
+ *              each inlined instance, e.g., by appending _1, _2, etc.?
+ *              maybe if don't use leading "." in label name? (nope...sigh)]
+ */
+
+int PNGAPI
+png_mmx_support(void)
+{
+#if defined(PNG_MMX_CODE_SUPPORTED)
+    __asm__ __volatile__ (
+        "pushl %%ebx          \n\t"  // ebx gets clobbered by CPUID instruction
+        "pushl %%ecx          \n\t"  // so does ecx...
+        "pushl %%edx          \n\t"  // ...and edx (but ecx & edx safe on Linux)
+//      ".byte  0x66          \n\t"  // convert 16-bit pushf to 32-bit pushfd
+//      "pushf                \n\t"  // 16-bit pushf
+        "pushfl               \n\t"  // save Eflag to stack
+        "popl %%eax           \n\t"  // get Eflag from stack into eax
+        "movl %%eax, %%ecx    \n\t"  // make another copy of Eflag in ecx
+        "xorl $0x200000, %%eax \n\t" // toggle ID bit in Eflag (i.e., bit 21)
+        "pushl %%eax          \n\t"  // save modified Eflag back to stack
+//      ".byte  0x66          \n\t"  // convert 16-bit popf to 32-bit popfd
+//      "popf                 \n\t"  // 16-bit popf
+        "popfl                \n\t"  // restore modified value to Eflag reg
+        "pushfl               \n\t"  // save Eflag to stack
+        "popl %%eax           \n\t"  // get Eflag from stack
+        "pushl %%ecx          \n\t"  // save original Eflag to stack
+        "popfl                \n\t"  // restore original Eflag
+        "xorl %%ecx, %%eax    \n\t"  // compare new Eflag with original Eflag
+        "jz 0f                \n\t"  // if same, CPUID instr. is not supported
+
+        "xorl %%eax, %%eax    \n\t"  // set eax to zero
+//      ".byte  0x0f, 0xa2    \n\t"  // CPUID instruction (two-byte opcode)
+        "cpuid                \n\t"  // get the CPU identification info
+        "cmpl $1, %%eax       \n\t"  // make sure eax return non-zero value
+        "jl 0f                \n\t"  // if eax is zero, MMX is not supported
+
+        "xorl %%eax, %%eax    \n\t"  // set eax to zero and...
+        "incl %%eax           \n\t"  // ...increment eax to 1.  This pair is
+                                     // faster than the instruction "mov eax, 1"
+        "cpuid                \n\t"  // get the CPU identification info again
+        "andl $0x800000, %%edx \n\t" // mask out all bits but MMX bit (23)
+        "cmpl $0, %%edx       \n\t"  // 0 = MMX not supported
+        "jz 0f                \n\t"  // non-zero = yes, MMX IS supported
+
+        "movl $1, %%eax       \n\t"  // set return value to 1
+        "jmp  1f              \n\t"  // DONE:  have MMX support
+
+    "0:                       \n\t"  // .NOT_SUPPORTED: target label for jump instructions
+        "movl $0, %%eax       \n\t"  // set return value to 0
+    "1:                       \n\t"  // .RETURN: target label for jump instructions
+        "movl %%eax, _mmx_supported \n\t" // save in global static variable, too
+        "popl %%edx           \n\t"  // restore edx
+        "popl %%ecx           \n\t"  // restore ecx
+        "popl %%ebx           \n\t"  // restore ebx
+
+//      "ret                  \n\t"  // DONE:  no MMX support
+                                     // (fall through to standard C "ret")
+
+        :                            // output list (none)
+
+        :                            // any variables used on input (none)
+
+        : "%eax"                     // clobber list
+//      , "%ebx", "%ecx", "%edx"     // GRR:  we handle these manually
+//      , "memory"   // if write to a variable gcc thought was in a reg
+//      , "cc"       // "condition codes" (flag bits)
+    );
+#else     
+    _mmx_supported = 0;
+#endif /* PNG_MMX_CODE_SUPPORTED */
+
+    return _mmx_supported;
+}
+
+
+#endif /* PNG_USE_PNGGCCRD */