diff options
| author | Rémi Verschelde <rverschelde@gmail.com> | 2016-10-11 23:35:58 +0200 |
|---|---|---|
| committer | Rémi Verschelde <rverschelde@gmail.com> | 2016-10-15 11:50:39 +0200 |
| commit | ee3cf211c6fd4d1e30617467cdbbe945798a68b3 (patch) | |
| tree | d770150c48c806df4daca66770cde8d5b665a3ff /thirdparty/libwebp/dsp | |
| parent | b1e8889d969f5f88539c47c2afac6c9ea2a2dc11 (diff) | |
webp: Make it a module and unbundle libwebp thirdparty files
Note that there are two Godot-specific changes made to libwebp
for the javascript/HTML5 platform. They are documented in the
README.md.
Diffstat (limited to 'thirdparty/libwebp/dsp')
61 files changed, 25712 insertions, 0 deletions
diff --git a/thirdparty/libwebp/dsp/alpha_processing.c b/thirdparty/libwebp/dsp/alpha_processing.c new file mode 100644 index 0000000000..1716cace8d --- /dev/null +++ b/thirdparty/libwebp/dsp/alpha_processing.c @@ -0,0 +1,383 @@ +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Utilities for processing transparent channel. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include <assert.h> +#include "./dsp.h" + +// Tables can be faster on some platform but incur some extra binary size (~2k). +// #define USE_TABLES_FOR_ALPHA_MULT + +// ----------------------------------------------------------------------------- + +#define MFIX 24 // 24bit fixed-point arithmetic +#define HALF ((1u << MFIX) >> 1) +#define KINV_255 ((1u << MFIX) / 255u) + +static uint32_t Mult(uint8_t x, uint32_t mult) { + const uint32_t v = (x * mult + HALF) >> MFIX; + assert(v <= 255); // <- 24bit precision is enough to ensure that. + return v; +} + +#ifdef USE_TABLES_FOR_ALPHA_MULT + +static const uint32_t kMultTables[2][256] = { + { // (255u << MFIX) / alpha + 0x00000000, 0xff000000, 0x7f800000, 0x55000000, 0x3fc00000, 0x33000000, + 0x2a800000, 0x246db6db, 0x1fe00000, 0x1c555555, 0x19800000, 0x172e8ba2, + 0x15400000, 0x139d89d8, 0x1236db6d, 0x11000000, 0x0ff00000, 0x0f000000, + 0x0e2aaaaa, 0x0d6bca1a, 0x0cc00000, 0x0c249249, 0x0b9745d1, 0x0b1642c8, + 0x0aa00000, 0x0a333333, 0x09cec4ec, 0x0971c71c, 0x091b6db6, 0x08cb08d3, + 0x08800000, 0x0839ce73, 0x07f80000, 0x07ba2e8b, 0x07800000, 0x07492492, + 0x07155555, 0x06e45306, 0x06b5e50d, 0x0689d89d, 0x06600000, 0x063831f3, + 0x06124924, 0x05ee23b8, 0x05cba2e8, 0x05aaaaaa, 0x058b2164, 0x056cefa8, + 0x05500000, 0x05343eb1, 0x05199999, 0x05000000, 0x04e76276, 0x04cfb2b7, + 0x04b8e38e, 0x04a2e8ba, 0x048db6db, 0x0479435e, 0x04658469, 0x045270d0, + 0x04400000, 0x042e29f7, 0x041ce739, 0x040c30c3, 0x03fc0000, 0x03ec4ec4, + 0x03dd1745, 0x03ce540f, 0x03c00000, 0x03b21642, 0x03a49249, 0x03976fc6, + 0x038aaaaa, 0x037e3f1f, 0x03722983, 0x03666666, 0x035af286, 0x034fcace, + 0x0344ec4e, 0x033a5440, 0x03300000, 0x0325ed09, 0x031c18f9, 0x0312818a, + 0x03092492, 0x03000000, 0x02f711dc, 0x02ee5846, 0x02e5d174, 0x02dd7baf, + 0x02d55555, 0x02cd5cd5, 0x02c590b2, 0x02bdef7b, 0x02b677d4, 0x02af286b, + 0x02a80000, 0x02a0fd5c, 0x029a1f58, 0x029364d9, 0x028ccccc, 0x0286562d, + 0x02800000, 0x0279c952, 0x0273b13b, 0x026db6db, 0x0267d95b, 0x026217ec, + 0x025c71c7, 0x0256e62a, 0x0251745d, 0x024c1bac, 0x0246db6d, 0x0241b2f9, + 0x023ca1af, 0x0237a6f4, 0x0232c234, 0x022df2df, 0x02293868, 0x02249249, + 0x02200000, 0x021b810e, 0x021714fb, 0x0212bb51, 0x020e739c, 0x020a3d70, + 0x02061861, 0x02020408, 0x01fe0000, 0x01fa0be8, 0x01f62762, 0x01f25213, + 0x01ee8ba2, 0x01ead3ba, 0x01e72a07, 0x01e38e38, 0x01e00000, 0x01dc7f10, + 0x01d90b21, 0x01d5a3e9, 0x01d24924, 0x01cefa8d, 0x01cbb7e3, 0x01c880e5, + 0x01c55555, 0x01c234f7, 0x01bf1f8f, 0x01bc14e5, 0x01b914c1, 0x01b61eed, + 0x01b33333, 0x01b05160, 0x01ad7943, 0x01aaaaaa, 0x01a7e567, 0x01a5294a, + 0x01a27627, 0x019fcbd2, 0x019d2a20, 0x019a90e7, 0x01980000, 0x01957741, + 0x0192f684, 0x01907da4, 0x018e0c7c, 0x018ba2e8, 0x018940c5, 0x0186e5f0, + 0x01849249, 0x018245ae, 0x01800000, 0x017dc11f, 0x017b88ee, 0x0179574e, + 0x01772c23, 0x01750750, 0x0172e8ba, 0x0170d045, 0x016ebdd7, 0x016cb157, + 0x016aaaaa, 0x0168a9b9, 0x0166ae6a, 0x0164b8a7, 0x0162c859, 0x0160dd67, + 0x015ef7bd, 0x015d1745, 0x015b3bea, 0x01596596, 0x01579435, 0x0155c7b4, + 0x01540000, 0x01523d03, 0x01507eae, 0x014ec4ec, 0x014d0fac, 0x014b5edc, + 0x0149b26c, 0x01480a4a, 0x01466666, 0x0144c6af, 0x01432b16, 0x0141938b, + 0x01400000, 0x013e7063, 0x013ce4a9, 0x013b5cc0, 0x0139d89d, 0x01385830, + 0x0136db6d, 0x01356246, 0x0133ecad, 0x01327a97, 0x01310bf6, 0x012fa0be, + 0x012e38e3, 0x012cd459, 0x012b7315, 0x012a150a, 0x0128ba2e, 0x01276276, + 0x01260dd6, 0x0124bc44, 0x01236db6, 0x01222222, 0x0120d97c, 0x011f93bc, + 0x011e50d7, 0x011d10c4, 0x011bd37a, 0x011a98ef, 0x0119611a, 0x01182bf2, + 0x0116f96f, 0x0115c988, 0x01149c34, 0x0113716a, 0x01124924, 0x01112358, + 0x01100000, 0x010edf12, 0x010dc087, 0x010ca458, 0x010b8a7d, 0x010a72f0, + 0x01095da8, 0x01084a9f, 0x010739ce, 0x01062b2e, 0x01051eb8, 0x01041465, + 0x01030c30, 0x01020612, 0x01010204, 0x01000000 }, + { // alpha * KINV_255 + 0x00000000, 0x00010101, 0x00020202, 0x00030303, 0x00040404, 0x00050505, + 0x00060606, 0x00070707, 0x00080808, 0x00090909, 0x000a0a0a, 0x000b0b0b, + 0x000c0c0c, 0x000d0d0d, 0x000e0e0e, 0x000f0f0f, 0x00101010, 0x00111111, + 0x00121212, 0x00131313, 0x00141414, 0x00151515, 0x00161616, 0x00171717, + 0x00181818, 0x00191919, 0x001a1a1a, 0x001b1b1b, 0x001c1c1c, 0x001d1d1d, + 0x001e1e1e, 0x001f1f1f, 0x00202020, 0x00212121, 0x00222222, 0x00232323, + 0x00242424, 0x00252525, 0x00262626, 0x00272727, 0x00282828, 0x00292929, + 0x002a2a2a, 0x002b2b2b, 0x002c2c2c, 0x002d2d2d, 0x002e2e2e, 0x002f2f2f, + 0x00303030, 0x00313131, 0x00323232, 0x00333333, 0x00343434, 0x00353535, + 0x00363636, 0x00373737, 0x00383838, 0x00393939, 0x003a3a3a, 0x003b3b3b, + 0x003c3c3c, 0x003d3d3d, 0x003e3e3e, 0x003f3f3f, 0x00404040, 0x00414141, + 0x00424242, 0x00434343, 0x00444444, 0x00454545, 0x00464646, 0x00474747, + 0x00484848, 0x00494949, 0x004a4a4a, 0x004b4b4b, 0x004c4c4c, 0x004d4d4d, + 0x004e4e4e, 0x004f4f4f, 0x00505050, 0x00515151, 0x00525252, 0x00535353, + 0x00545454, 0x00555555, 0x00565656, 0x00575757, 0x00585858, 0x00595959, + 0x005a5a5a, 0x005b5b5b, 0x005c5c5c, 0x005d5d5d, 0x005e5e5e, 0x005f5f5f, + 0x00606060, 0x00616161, 0x00626262, 0x00636363, 0x00646464, 0x00656565, + 0x00666666, 0x00676767, 0x00686868, 0x00696969, 0x006a6a6a, 0x006b6b6b, + 0x006c6c6c, 0x006d6d6d, 0x006e6e6e, 0x006f6f6f, 0x00707070, 0x00717171, + 0x00727272, 0x00737373, 0x00747474, 0x00757575, 0x00767676, 0x00777777, + 0x00787878, 0x00797979, 0x007a7a7a, 0x007b7b7b, 0x007c7c7c, 0x007d7d7d, + 0x007e7e7e, 0x007f7f7f, 0x00808080, 0x00818181, 0x00828282, 0x00838383, + 0x00848484, 0x00858585, 0x00868686, 0x00878787, 0x00888888, 0x00898989, + 0x008a8a8a, 0x008b8b8b, 0x008c8c8c, 0x008d8d8d, 0x008e8e8e, 0x008f8f8f, + 0x00909090, 0x00919191, 0x00929292, 0x00939393, 0x00949494, 0x00959595, + 0x00969696, 0x00979797, 0x00989898, 0x00999999, 0x009a9a9a, 0x009b9b9b, + 0x009c9c9c, 0x009d9d9d, 0x009e9e9e, 0x009f9f9f, 0x00a0a0a0, 0x00a1a1a1, + 0x00a2a2a2, 0x00a3a3a3, 0x00a4a4a4, 0x00a5a5a5, 0x00a6a6a6, 0x00a7a7a7, + 0x00a8a8a8, 0x00a9a9a9, 0x00aaaaaa, 0x00ababab, 0x00acacac, 0x00adadad, + 0x00aeaeae, 0x00afafaf, 0x00b0b0b0, 0x00b1b1b1, 0x00b2b2b2, 0x00b3b3b3, + 0x00b4b4b4, 0x00b5b5b5, 0x00b6b6b6, 0x00b7b7b7, 0x00b8b8b8, 0x00b9b9b9, + 0x00bababa, 0x00bbbbbb, 0x00bcbcbc, 0x00bdbdbd, 0x00bebebe, 0x00bfbfbf, + 0x00c0c0c0, 0x00c1c1c1, 0x00c2c2c2, 0x00c3c3c3, 0x00c4c4c4, 0x00c5c5c5, + 0x00c6c6c6, 0x00c7c7c7, 0x00c8c8c8, 0x00c9c9c9, 0x00cacaca, 0x00cbcbcb, + 0x00cccccc, 0x00cdcdcd, 0x00cecece, 0x00cfcfcf, 0x00d0d0d0, 0x00d1d1d1, + 0x00d2d2d2, 0x00d3d3d3, 0x00d4d4d4, 0x00d5d5d5, 0x00d6d6d6, 0x00d7d7d7, + 0x00d8d8d8, 0x00d9d9d9, 0x00dadada, 0x00dbdbdb, 0x00dcdcdc, 0x00dddddd, + 0x00dedede, 0x00dfdfdf, 0x00e0e0e0, 0x00e1e1e1, 0x00e2e2e2, 0x00e3e3e3, + 0x00e4e4e4, 0x00e5e5e5, 0x00e6e6e6, 0x00e7e7e7, 0x00e8e8e8, 0x00e9e9e9, + 0x00eaeaea, 0x00ebebeb, 0x00ececec, 0x00ededed, 0x00eeeeee, 0x00efefef, + 0x00f0f0f0, 0x00f1f1f1, 0x00f2f2f2, 0x00f3f3f3, 0x00f4f4f4, 0x00f5f5f5, + 0x00f6f6f6, 0x00f7f7f7, 0x00f8f8f8, 0x00f9f9f9, 0x00fafafa, 0x00fbfbfb, + 0x00fcfcfc, 0x00fdfdfd, 0x00fefefe, 0x00ffffff } +}; + +static WEBP_INLINE uint32_t GetScale(uint32_t a, int inverse) { + return kMultTables[!inverse][a]; +} + +#else + +static WEBP_INLINE uint32_t GetScale(uint32_t a, int inverse) { + return inverse ? (255u << MFIX) / a : a * KINV_255; +} + +#endif // USE_TABLES_FOR_ALPHA_MULT + +void WebPMultARGBRowC(uint32_t* const ptr, int width, int inverse) { + int x; + for (x = 0; x < width; ++x) { + const uint32_t argb = ptr[x]; + if (argb < 0xff000000u) { // alpha < 255 + if (argb <= 0x00ffffffu) { // alpha == 0 + ptr[x] = 0; + } else { + const uint32_t alpha = (argb >> 24) & 0xff; + const uint32_t scale = GetScale(alpha, inverse); + uint32_t out = argb & 0xff000000u; + out |= Mult(argb >> 0, scale) << 0; + out |= Mult(argb >> 8, scale) << 8; + out |= Mult(argb >> 16, scale) << 16; + ptr[x] = out; + } + } + } +} + +void WebPMultRowC(uint8_t* const ptr, const uint8_t* const alpha, + int width, int inverse) { + int x; + for (x = 0; x < width; ++x) { + const uint32_t a = alpha[x]; + if (a != 255) { + if (a == 0) { + ptr[x] = 0; + } else { + const uint32_t scale = GetScale(a, inverse); + ptr[x] = Mult(ptr[x], scale); + } + } + } +} + +#undef KINV_255 +#undef HALF +#undef MFIX + +void (*WebPMultARGBRow)(uint32_t* const ptr, int width, int inverse); +void (*WebPMultRow)(uint8_t* const ptr, const uint8_t* const alpha, + int width, int inverse); + +//------------------------------------------------------------------------------ +// Generic per-plane calls + +void WebPMultARGBRows(uint8_t* ptr, int stride, int width, int num_rows, + int inverse) { + int n; + for (n = 0; n < num_rows; ++n) { + WebPMultARGBRow((uint32_t*)ptr, width, inverse); + ptr += stride; + } +} + +void WebPMultRows(uint8_t* ptr, int stride, + const uint8_t* alpha, int alpha_stride, + int width, int num_rows, int inverse) { + int n; + for (n = 0; n < num_rows; ++n) { + WebPMultRow(ptr, alpha, width, inverse); + ptr += stride; + alpha += alpha_stride; + } +} + +//------------------------------------------------------------------------------ +// Premultiplied modes + +// non dithered-modes + +// (x * a * 32897) >> 23 is bit-wise equivalent to (int)(x * a / 255.) +// for all 8bit x or a. For bit-wise equivalence to (int)(x * a / 255. + .5), +// one can use instead: (x * a * 65793 + (1 << 23)) >> 24 +#if 1 // (int)(x * a / 255.) +#define MULTIPLIER(a) ((a) * 32897U) +#define PREMULTIPLY(x, m) (((x) * (m)) >> 23) +#else // (int)(x * a / 255. + .5) +#define MULTIPLIER(a) ((a) * 65793U) +#define PREMULTIPLY(x, m) (((x) * (m) + (1U << 23)) >> 24) +#endif + +static void ApplyAlphaMultiply(uint8_t* rgba, int alpha_first, + int w, int h, int stride) { + while (h-- > 0) { + uint8_t* const rgb = rgba + (alpha_first ? 1 : 0); + const uint8_t* const alpha = rgba + (alpha_first ? 0 : 3); + int i; + for (i = 0; i < w; ++i) { + const uint32_t a = alpha[4 * i]; + if (a != 0xff) { + const uint32_t mult = MULTIPLIER(a); + rgb[4 * i + 0] = PREMULTIPLY(rgb[4 * i + 0], mult); + rgb[4 * i + 1] = PREMULTIPLY(rgb[4 * i + 1], mult); + rgb[4 * i + 2] = PREMULTIPLY(rgb[4 * i + 2], mult); + } + } + rgba += stride; + } +} +#undef MULTIPLIER +#undef PREMULTIPLY + +// rgbA4444 + +#define MULTIPLIER(a) ((a) * 0x1111) // 0x1111 ~= (1 << 16) / 15 + +static WEBP_INLINE uint8_t dither_hi(uint8_t x) { + return (x & 0xf0) | (x >> 4); +} + +static WEBP_INLINE uint8_t dither_lo(uint8_t x) { + return (x & 0x0f) | (x << 4); +} + +static WEBP_INLINE uint8_t multiply(uint8_t x, uint32_t m) { + return (x * m) >> 16; +} + +static WEBP_INLINE void ApplyAlphaMultiply4444(uint8_t* rgba4444, + int w, int h, int stride, + int rg_byte_pos /* 0 or 1 */) { + while (h-- > 0) { + int i; + for (i = 0; i < w; ++i) { + const uint32_t rg = rgba4444[2 * i + rg_byte_pos]; + const uint32_t ba = rgba4444[2 * i + (rg_byte_pos ^ 1)]; + const uint8_t a = ba & 0x0f; + const uint32_t mult = MULTIPLIER(a); + const uint8_t r = multiply(dither_hi(rg), mult); + const uint8_t g = multiply(dither_lo(rg), mult); + const uint8_t b = multiply(dither_hi(ba), mult); + rgba4444[2 * i + rg_byte_pos] = (r & 0xf0) | ((g >> 4) & 0x0f); + rgba4444[2 * i + (rg_byte_pos ^ 1)] = (b & 0xf0) | a; + } + rgba4444 += stride; + } +} +#undef MULTIPLIER + +static void ApplyAlphaMultiply_16b(uint8_t* rgba4444, + int w, int h, int stride) { +#ifdef WEBP_SWAP_16BIT_CSP + ApplyAlphaMultiply4444(rgba4444, w, h, stride, 1); +#else + ApplyAlphaMultiply4444(rgba4444, w, h, stride, 0); +#endif +} + +static int DispatchAlpha(const uint8_t* alpha, int alpha_stride, + int width, int height, + uint8_t* dst, int dst_stride) { + uint32_t alpha_mask = 0xff; + int i, j; + + for (j = 0; j < height; ++j) { + for (i = 0; i < width; ++i) { + const uint32_t alpha_value = alpha[i]; + dst[4 * i] = alpha_value; + alpha_mask &= alpha_value; + } + alpha += alpha_stride; + dst += dst_stride; + } + + return (alpha_mask != 0xff); +} + +static void DispatchAlphaToGreen(const uint8_t* alpha, int alpha_stride, + int width, int height, + uint32_t* dst, int dst_stride) { + int i, j; + for (j = 0; j < height; ++j) { + for (i = 0; i < width; ++i) { + dst[i] = alpha[i] << 8; // leave A/R/B channels zero'd. + } + alpha += alpha_stride; + dst += dst_stride; + } +} + +static int ExtractAlpha(const uint8_t* argb, int argb_stride, + int width, int height, + uint8_t* alpha, int alpha_stride) { + uint8_t alpha_mask = 0xff; + int i, j; + + for (j = 0; j < height; ++j) { + for (i = 0; i < width; ++i) { + const uint8_t alpha_value = argb[4 * i]; + alpha[i] = alpha_value; + alpha_mask &= alpha_value; + } + argb += argb_stride; + alpha += alpha_stride; + } + return (alpha_mask == 0xff); +} + +void (*WebPApplyAlphaMultiply)(uint8_t*, int, int, int, int); +void (*WebPApplyAlphaMultiply4444)(uint8_t*, int, int, int); +int (*WebPDispatchAlpha)(const uint8_t*, int, int, int, uint8_t*, int); +void (*WebPDispatchAlphaToGreen)(const uint8_t*, int, int, int, uint32_t*, int); +int (*WebPExtractAlpha)(const uint8_t*, int, int, int, uint8_t*, int); + +//------------------------------------------------------------------------------ +// Init function + +extern void WebPInitAlphaProcessingMIPSdspR2(void); +extern void WebPInitAlphaProcessingSSE2(void); +extern void WebPInitAlphaProcessingSSE41(void); + +static volatile VP8CPUInfo alpha_processing_last_cpuinfo_used = + (VP8CPUInfo)&alpha_processing_last_cpuinfo_used; + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitAlphaProcessing(void) { + if (alpha_processing_last_cpuinfo_used == VP8GetCPUInfo) return; + + WebPMultARGBRow = WebPMultARGBRowC; + WebPMultRow = WebPMultRowC; + WebPApplyAlphaMultiply = ApplyAlphaMultiply; + WebPApplyAlphaMultiply4444 = ApplyAlphaMultiply_16b; + WebPDispatchAlpha = DispatchAlpha; + WebPDispatchAlphaToGreen = DispatchAlphaToGreen; + WebPExtractAlpha = ExtractAlpha; + + // If defined, use CPUInfo() to overwrite some pointers with faster versions. + if (VP8GetCPUInfo != NULL) { +#if defined(WEBP_USE_SSE2) + if (VP8GetCPUInfo(kSSE2)) { + WebPInitAlphaProcessingSSE2(); +#if defined(WEBP_USE_SSE41) + if (VP8GetCPUInfo(kSSE4_1)) { + WebPInitAlphaProcessingSSE41(); + } +#endif + } +#endif +#if defined(WEBP_USE_MIPS_DSP_R2) + if (VP8GetCPUInfo(kMIPSdspR2)) { + WebPInitAlphaProcessingMIPSdspR2(); + } +#endif + } + alpha_processing_last_cpuinfo_used = VP8GetCPUInfo; +} diff --git a/thirdparty/libwebp/dsp/alpha_processing_mips_dsp_r2.c b/thirdparty/libwebp/dsp/alpha_processing_mips_dsp_r2.c new file mode 100644 index 0000000000..c631d78905 --- /dev/null +++ b/thirdparty/libwebp/dsp/alpha_processing_mips_dsp_r2.c @@ -0,0 +1,141 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Utilities for processing transparent channel. +// +// Author(s): Branimir Vasic (branimir.vasic@imgtec.com) +// Djordje Pesut (djordje.pesut@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS_DSP_R2) + +static int DispatchAlpha(const uint8_t* alpha, int alpha_stride, + int width, int height, + uint8_t* dst, int dst_stride) { + uint32_t alpha_mask = 0xffffffff; + int i, j, temp0; + + for (j = 0; j < height; ++j) { + uint8_t* pdst = dst; + const uint8_t* palpha = alpha; + for (i = 0; i < (width >> 2); ++i) { + int temp1, temp2, temp3; + + __asm__ volatile ( + "ulw %[temp0], 0(%[palpha]) \n\t" + "addiu %[palpha], %[palpha], 4 \n\t" + "addiu %[pdst], %[pdst], 16 \n\t" + "srl %[temp1], %[temp0], 8 \n\t" + "srl %[temp2], %[temp0], 16 \n\t" + "srl %[temp3], %[temp0], 24 \n\t" + "and %[alpha_mask], %[alpha_mask], %[temp0] \n\t" + "sb %[temp0], -16(%[pdst]) \n\t" + "sb %[temp1], -12(%[pdst]) \n\t" + "sb %[temp2], -8(%[pdst]) \n\t" + "sb %[temp3], -4(%[pdst]) \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [palpha]"+r"(palpha), [pdst]"+r"(pdst), + [alpha_mask]"+r"(alpha_mask) + : + : "memory" + ); + } + + for (i = 0; i < (width & 3); ++i) { + __asm__ volatile ( + "lbu %[temp0], 0(%[palpha]) \n\t" + "addiu %[palpha], %[palpha], 1 \n\t" + "sb %[temp0], 0(%[pdst]) \n\t" + "and %[alpha_mask], %[alpha_mask], %[temp0] \n\t" + "addiu %[pdst], %[pdst], 4 \n\t" + : [temp0]"=&r"(temp0), [palpha]"+r"(palpha), [pdst]"+r"(pdst), + [alpha_mask]"+r"(alpha_mask) + : + : "memory" + ); + } + alpha += alpha_stride; + dst += dst_stride; + } + + __asm__ volatile ( + "ext %[temp0], %[alpha_mask], 0, 16 \n\t" + "srl %[alpha_mask], %[alpha_mask], 16 \n\t" + "and %[alpha_mask], %[alpha_mask], %[temp0] \n\t" + "ext %[temp0], %[alpha_mask], 0, 8 \n\t" + "srl %[alpha_mask], %[alpha_mask], 8 \n\t" + "and %[alpha_mask], %[alpha_mask], %[temp0] \n\t" + : [temp0]"=&r"(temp0), [alpha_mask]"+r"(alpha_mask) + : + ); + + return (alpha_mask != 0xff); +} + +static void MultARGBRow(uint32_t* const ptr, int width, int inverse) { + int x; + const uint32_t c_00ffffff = 0x00ffffffu; + const uint32_t c_ff000000 = 0xff000000u; + const uint32_t c_8000000 = 0x00800000u; + const uint32_t c_8000080 = 0x00800080u; + for (x = 0; x < width; ++x) { + const uint32_t argb = ptr[x]; + if (argb < 0xff000000u) { // alpha < 255 + if (argb <= 0x00ffffffu) { // alpha == 0 + ptr[x] = 0; + } else { + int temp0, temp1, temp2, temp3, alpha; + __asm__ volatile ( + "srl %[alpha], %[argb], 24 \n\t" + "replv.qb %[temp0], %[alpha] \n\t" + "and %[temp0], %[temp0], %[c_00ffffff] \n\t" + "beqz %[inverse], 0f \n\t" + "divu $zero, %[c_ff000000], %[alpha] \n\t" + "mflo %[temp0] \n\t" + "0: \n\t" + "andi %[temp1], %[argb], 0xff \n\t" + "ext %[temp2], %[argb], 8, 8 \n\t" + "ext %[temp3], %[argb], 16, 8 \n\t" + "mul %[temp1], %[temp1], %[temp0] \n\t" + "mul %[temp2], %[temp2], %[temp0] \n\t" + "mul %[temp3], %[temp3], %[temp0] \n\t" + "precrq.ph.w %[temp1], %[temp2], %[temp1] \n\t" + "addu %[temp3], %[temp3], %[c_8000000] \n\t" + "addu %[temp1], %[temp1], %[c_8000080] \n\t" + "precrq.ph.w %[temp3], %[argb], %[temp3] \n\t" + "precrq.qb.ph %[temp1], %[temp3], %[temp1] \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [alpha]"=&r"(alpha) + : [inverse]"r"(inverse), [c_00ffffff]"r"(c_00ffffff), + [c_8000000]"r"(c_8000000), [c_8000080]"r"(c_8000080), + [c_ff000000]"r"(c_ff000000), [argb]"r"(argb) + : "memory", "hi", "lo" + ); + ptr[x] = temp1; + } + } + } +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void WebPInitAlphaProcessingMIPSdspR2(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitAlphaProcessingMIPSdspR2(void) { + WebPDispatchAlpha = DispatchAlpha; + WebPMultARGBRow = MultARGBRow; +} + +#else // !WEBP_USE_MIPS_DSP_R2 + +WEBP_DSP_INIT_STUB(WebPInitAlphaProcessingMIPSdspR2) + +#endif // WEBP_USE_MIPS_DSP_R2 diff --git a/thirdparty/libwebp/dsp/alpha_processing_sse2.c b/thirdparty/libwebp/dsp/alpha_processing_sse2.c new file mode 100644 index 0000000000..5acb481dcd --- /dev/null +++ b/thirdparty/libwebp/dsp/alpha_processing_sse2.c @@ -0,0 +1,298 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Utilities for processing transparent channel. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE2) +#include <emmintrin.h> + +//------------------------------------------------------------------------------ + +static int DispatchAlpha(const uint8_t* alpha, int alpha_stride, + int width, int height, + uint8_t* dst, int dst_stride) { + // alpha_and stores an 'and' operation of all the alpha[] values. The final + // value is not 0xff if any of the alpha[] is not equal to 0xff. + uint32_t alpha_and = 0xff; + int i, j; + const __m128i zero = _mm_setzero_si128(); + const __m128i rgb_mask = _mm_set1_epi32(0xffffff00u); // to preserve RGB + const __m128i all_0xff = _mm_set_epi32(0, 0, ~0u, ~0u); + __m128i all_alphas = all_0xff; + + // We must be able to access 3 extra bytes after the last written byte + // 'dst[4 * width - 4]', because we don't know if alpha is the first or the + // last byte of the quadruplet. + const int limit = (width - 1) & ~7; + + for (j = 0; j < height; ++j) { + __m128i* out = (__m128i*)dst; + for (i = 0; i < limit; i += 8) { + // load 8 alpha bytes + const __m128i a0 = _mm_loadl_epi64((const __m128i*)&alpha[i]); + const __m128i a1 = _mm_unpacklo_epi8(a0, zero); + const __m128i a2_lo = _mm_unpacklo_epi16(a1, zero); + const __m128i a2_hi = _mm_unpackhi_epi16(a1, zero); + // load 8 dst pixels (32 bytes) + const __m128i b0_lo = _mm_loadu_si128(out + 0); + const __m128i b0_hi = _mm_loadu_si128(out + 1); + // mask dst alpha values + const __m128i b1_lo = _mm_and_si128(b0_lo, rgb_mask); + const __m128i b1_hi = _mm_and_si128(b0_hi, rgb_mask); + // combine + const __m128i b2_lo = _mm_or_si128(b1_lo, a2_lo); + const __m128i b2_hi = _mm_or_si128(b1_hi, a2_hi); + // store + _mm_storeu_si128(out + 0, b2_lo); + _mm_storeu_si128(out + 1, b2_hi); + // accumulate eight alpha 'and' in parallel + all_alphas = _mm_and_si128(all_alphas, a0); + out += 2; + } + for (; i < width; ++i) { + const uint32_t alpha_value = alpha[i]; + dst[4 * i] = alpha_value; + alpha_and &= alpha_value; + } + alpha += alpha_stride; + dst += dst_stride; + } + // Combine the eight alpha 'and' into a 8-bit mask. + alpha_and &= _mm_movemask_epi8(_mm_cmpeq_epi8(all_alphas, all_0xff)); + return (alpha_and != 0xff); +} + +static void DispatchAlphaToGreen(const uint8_t* alpha, int alpha_stride, + int width, int height, + uint32_t* dst, int dst_stride) { + int i, j; + const __m128i zero = _mm_setzero_si128(); + const int limit = width & ~15; + for (j = 0; j < height; ++j) { + for (i = 0; i < limit; i += 16) { // process 16 alpha bytes + const __m128i a0 = _mm_loadu_si128((const __m128i*)&alpha[i]); + const __m128i a1 = _mm_unpacklo_epi8(zero, a0); // note the 'zero' first! + const __m128i b1 = _mm_unpackhi_epi8(zero, a0); + const __m128i a2_lo = _mm_unpacklo_epi16(a1, zero); + const __m128i b2_lo = _mm_unpacklo_epi16(b1, zero); + const __m128i a2_hi = _mm_unpackhi_epi16(a1, zero); + const __m128i b2_hi = _mm_unpackhi_epi16(b1, zero); + _mm_storeu_si128((__m128i*)&dst[i + 0], a2_lo); + _mm_storeu_si128((__m128i*)&dst[i + 4], a2_hi); + _mm_storeu_si128((__m128i*)&dst[i + 8], b2_lo); + _mm_storeu_si128((__m128i*)&dst[i + 12], b2_hi); + } + for (; i < width; ++i) dst[i] = alpha[i] << 8; + alpha += alpha_stride; + dst += dst_stride; + } +} + +static int ExtractAlpha(const uint8_t* argb, int argb_stride, + int width, int height, + uint8_t* alpha, int alpha_stride) { + // alpha_and stores an 'and' operation of all the alpha[] values. The final + // value is not 0xff if any of the alpha[] is not equal to 0xff. + uint32_t alpha_and = 0xff; + int i, j; + const __m128i a_mask = _mm_set1_epi32(0xffu); // to preserve alpha + const __m128i all_0xff = _mm_set_epi32(0, 0, ~0u, ~0u); + __m128i all_alphas = all_0xff; + + // We must be able to access 3 extra bytes after the last written byte + // 'src[4 * width - 4]', because we don't know if alpha is the first or the + // last byte of the quadruplet. + const int limit = (width - 1) & ~7; + + for (j = 0; j < height; ++j) { + const __m128i* src = (const __m128i*)argb; + for (i = 0; i < limit; i += 8) { + // load 32 argb bytes + const __m128i a0 = _mm_loadu_si128(src + 0); + const __m128i a1 = _mm_loadu_si128(src + 1); + const __m128i b0 = _mm_and_si128(a0, a_mask); + const __m128i b1 = _mm_and_si128(a1, a_mask); + const __m128i c0 = _mm_packs_epi32(b0, b1); + const __m128i d0 = _mm_packus_epi16(c0, c0); + // store + _mm_storel_epi64((__m128i*)&alpha[i], d0); + // accumulate eight alpha 'and' in parallel + all_alphas = _mm_and_si128(all_alphas, d0); + src += 2; + } + for (; i < width; ++i) { + const uint32_t alpha_value = argb[4 * i]; + alpha[i] = alpha_value; + alpha_and &= alpha_value; + } + argb += argb_stride; + alpha += alpha_stride; + } + // Combine the eight alpha 'and' into a 8-bit mask. + alpha_and &= _mm_movemask_epi8(_mm_cmpeq_epi8(all_alphas, all_0xff)); + return (alpha_and == 0xff); +} + +//------------------------------------------------------------------------------ +// Non-dither premultiplied modes + +#define MULTIPLIER(a) ((a) * 0x8081) +#define PREMULTIPLY(x, m) (((x) * (m)) >> 23) + +// We can't use a 'const int' for the SHUFFLE value, because it has to be an +// immediate in the _mm_shufflexx_epi16() instruction. We really a macro here. +#define APPLY_ALPHA(RGBX, SHUFFLE, MASK, MULT) do { \ + const __m128i argb0 = _mm_loadl_epi64((__m128i*)&(RGBX)); \ + const __m128i argb1 = _mm_unpacklo_epi8(argb0, zero); \ + const __m128i alpha0 = _mm_and_si128(argb1, MASK); \ + const __m128i alpha1 = _mm_shufflelo_epi16(alpha0, SHUFFLE); \ + const __m128i alpha2 = _mm_shufflehi_epi16(alpha1, SHUFFLE); \ + /* alpha2 = [0 a0 a0 a0][0 a1 a1 a1] */ \ + const __m128i scale0 = _mm_mullo_epi16(alpha2, MULT); \ + const __m128i scale1 = _mm_mulhi_epu16(alpha2, MULT); \ + const __m128i argb2 = _mm_mulhi_epu16(argb1, scale0); \ + const __m128i argb3 = _mm_mullo_epi16(argb1, scale1); \ + const __m128i argb4 = _mm_adds_epu16(argb2, argb3); \ + const __m128i argb5 = _mm_srli_epi16(argb4, 7); \ + const __m128i argb6 = _mm_or_si128(argb5, alpha0); \ + const __m128i argb7 = _mm_packus_epi16(argb6, zero); \ + _mm_storel_epi64((__m128i*)&(RGBX), argb7); \ +} while (0) + +static void ApplyAlphaMultiply(uint8_t* rgba, int alpha_first, + int w, int h, int stride) { + const __m128i zero = _mm_setzero_si128(); + const int kSpan = 2; + const int w2 = w & ~(kSpan - 1); + while (h-- > 0) { + uint32_t* const rgbx = (uint32_t*)rgba; + int i; + if (!alpha_first) { + const __m128i kMask = _mm_set_epi16(0xff, 0, 0, 0, 0xff, 0, 0, 0); + const __m128i kMult = + _mm_set_epi16(0, 0x8081, 0x8081, 0x8081, 0, 0x8081, 0x8081, 0x8081); + for (i = 0; i < w2; i += kSpan) { + APPLY_ALPHA(rgbx[i], _MM_SHUFFLE(0, 3, 3, 3), kMask, kMult); + } + } else { + const __m128i kMask = _mm_set_epi16(0, 0, 0, 0xff, 0, 0, 0, 0xff); + const __m128i kMult = + _mm_set_epi16(0x8081, 0x8081, 0x8081, 0, 0x8081, 0x8081, 0x8081, 0); + for (i = 0; i < w2; i += kSpan) { + APPLY_ALPHA(rgbx[i], _MM_SHUFFLE(0, 0, 0, 3), kMask, kMult); + } + } + // Finish with left-overs. + for (; i < w; ++i) { + uint8_t* const rgb = rgba + (alpha_first ? 1 : 0); + const uint8_t* const alpha = rgba + (alpha_first ? 0 : 3); + const uint32_t a = alpha[4 * i]; + if (a != 0xff) { + const uint32_t mult = MULTIPLIER(a); + rgb[4 * i + 0] = PREMULTIPLY(rgb[4 * i + 0], mult); + rgb[4 * i + 1] = PREMULTIPLY(rgb[4 * i + 1], mult); + rgb[4 * i + 2] = PREMULTIPLY(rgb[4 * i + 2], mult); + } + } + rgba += stride; + } +} +#undef MULTIPLIER +#undef PREMULTIPLY + +// ----------------------------------------------------------------------------- +// Apply alpha value to rows + +// We use: kINV255 = (1 << 24) / 255 = 0x010101 +// So: a * kINV255 = (a << 16) | [(a << 8) | a] +// -> _mm_mulhi_epu16() takes care of the (a<<16) part, +// and _mm_mullo_epu16(a * 0x0101,...) takes care of the "(a << 8) | a" one. + +static void MultARGBRow(uint32_t* const ptr, int width, int inverse) { + int x = 0; + if (!inverse) { + const int kSpan = 2; + const __m128i zero = _mm_setzero_si128(); + const __m128i kRound = + _mm_set_epi16(0, 1 << 7, 1 << 7, 1 << 7, 0, 1 << 7, 1 << 7, 1 << 7); + const __m128i kMult = + _mm_set_epi16(0, 0x0101, 0x0101, 0x0101, 0, 0x0101, 0x0101, 0x0101); + const __m128i kOne64 = _mm_set_epi16(1u << 8, 0, 0, 0, 1u << 8, 0, 0, 0); + const int w2 = width & ~(kSpan - 1); + for (x = 0; x < w2; x += kSpan) { + const __m128i argb0 = _mm_loadl_epi64((__m128i*)&ptr[x]); + const __m128i argb1 = _mm_unpacklo_epi8(argb0, zero); + const __m128i tmp0 = _mm_shufflelo_epi16(argb1, _MM_SHUFFLE(3, 3, 3, 3)); + const __m128i tmp1 = _mm_shufflehi_epi16(tmp0, _MM_SHUFFLE(3, 3, 3, 3)); + const __m128i tmp2 = _mm_srli_epi64(tmp1, 16); + const __m128i scale0 = _mm_mullo_epi16(tmp1, kMult); + const __m128i scale1 = _mm_or_si128(tmp2, kOne64); + const __m128i argb2 = _mm_mulhi_epu16(argb1, scale0); + const __m128i argb3 = _mm_mullo_epi16(argb1, scale1); + const __m128i argb4 = _mm_adds_epu16(argb2, argb3); + const __m128i argb5 = _mm_adds_epu16(argb4, kRound); + const __m128i argb6 = _mm_srli_epi16(argb5, 8); + const __m128i argb7 = _mm_packus_epi16(argb6, zero); + _mm_storel_epi64((__m128i*)&ptr[x], argb7); + } + } + width -= x; + if (width > 0) WebPMultARGBRowC(ptr + x, width, inverse); +} + +static void MultRow(uint8_t* const ptr, const uint8_t* const alpha, + int width, int inverse) { + int x = 0; + if (!inverse) { + const int kSpan = 8; + const __m128i zero = _mm_setzero_si128(); + const __m128i kRound = _mm_set1_epi16(1 << 7); + const int w2 = width & ~(kSpan - 1); + for (x = 0; x < w2; x += kSpan) { + const __m128i v0 = _mm_loadl_epi64((__m128i*)&ptr[x]); + const __m128i v1 = _mm_unpacklo_epi8(v0, zero); + const __m128i alpha0 = _mm_loadl_epi64((const __m128i*)&alpha[x]); + const __m128i alpha1 = _mm_unpacklo_epi8(alpha0, zero); + const __m128i alpha2 = _mm_unpacklo_epi8(alpha0, alpha0); + const __m128i v2 = _mm_mulhi_epu16(v1, alpha2); + const __m128i v3 = _mm_mullo_epi16(v1, alpha1); + const __m128i v4 = _mm_adds_epu16(v2, v3); + const __m128i v5 = _mm_adds_epu16(v4, kRound); + const __m128i v6 = _mm_srli_epi16(v5, 8); + const __m128i v7 = _mm_packus_epi16(v6, zero); + _mm_storel_epi64((__m128i*)&ptr[x], v7); + } + } + width -= x; + if (width > 0) WebPMultRowC(ptr + x, alpha + x, width, inverse); +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void WebPInitAlphaProcessingSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitAlphaProcessingSSE2(void) { + WebPMultARGBRow = MultARGBRow; + WebPMultRow = MultRow; + WebPApplyAlphaMultiply = ApplyAlphaMultiply; + WebPDispatchAlpha = DispatchAlpha; + WebPDispatchAlphaToGreen = DispatchAlphaToGreen; + WebPExtractAlpha = ExtractAlpha; +} + +#else // !WEBP_USE_SSE2 + +WEBP_DSP_INIT_STUB(WebPInitAlphaProcessingSSE2) + +#endif // WEBP_USE_SSE2 diff --git a/thirdparty/libwebp/dsp/alpha_processing_sse41.c b/thirdparty/libwebp/dsp/alpha_processing_sse41.c new file mode 100644 index 0000000000..986fde94ed --- /dev/null +++ b/thirdparty/libwebp/dsp/alpha_processing_sse41.c @@ -0,0 +1,92 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Utilities for processing transparent channel, SSE4.1 variant. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE41) + +#include <smmintrin.h> + +//------------------------------------------------------------------------------ + +static int ExtractAlpha(const uint8_t* argb, int argb_stride, + int width, int height, + uint8_t* alpha, int alpha_stride) { + // alpha_and stores an 'and' operation of all the alpha[] values. The final + // value is not 0xff if any of the alpha[] is not equal to 0xff. + uint32_t alpha_and = 0xff; + int i, j; + const __m128i all_0xff = _mm_set1_epi32(~0u); + __m128i all_alphas = all_0xff; + + // We must be able to access 3 extra bytes after the last written byte + // 'src[4 * width - 4]', because we don't know if alpha is the first or the + // last byte of the quadruplet. + const int limit = (width - 1) & ~15; + const __m128i kCstAlpha0 = _mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 12, 8, 4, 0); + const __m128i kCstAlpha1 = _mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, + 12, 8, 4, 0, -1, -1, -1, -1); + const __m128i kCstAlpha2 = _mm_set_epi8(-1, -1, -1, -1, 12, 8, 4, 0, + -1, -1, -1, -1, -1, -1, -1, -1); + const __m128i kCstAlpha3 = _mm_set_epi8(12, 8, 4, 0, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1); + for (j = 0; j < height; ++j) { + const __m128i* src = (const __m128i*)argb; + for (i = 0; i < limit; i += 16) { + // load 64 argb bytes + const __m128i a0 = _mm_loadu_si128(src + 0); + const __m128i a1 = _mm_loadu_si128(src + 1); + const __m128i a2 = _mm_loadu_si128(src + 2); + const __m128i a3 = _mm_loadu_si128(src + 3); + const __m128i b0 = _mm_shuffle_epi8(a0, kCstAlpha0); + const __m128i b1 = _mm_shuffle_epi8(a1, kCstAlpha1); + const __m128i b2 = _mm_shuffle_epi8(a2, kCstAlpha2); + const __m128i b3 = _mm_shuffle_epi8(a3, kCstAlpha3); + const __m128i c0 = _mm_or_si128(b0, b1); + const __m128i c1 = _mm_or_si128(b2, b3); + const __m128i d0 = _mm_or_si128(c0, c1); + // store + _mm_storeu_si128((__m128i*)&alpha[i], d0); + // accumulate sixteen alpha 'and' in parallel + all_alphas = _mm_and_si128(all_alphas, d0); + src += 4; + } + for (; i < width; ++i) { + const uint32_t alpha_value = argb[4 * i]; + alpha[i] = alpha_value; + alpha_and &= alpha_value; + } + argb += argb_stride; + alpha += alpha_stride; + } + // Combine the sixteen alpha 'and' into an 8-bit mask. + alpha_and |= 0xff00u; // pretend the upper bits [8..15] were tested ok. + alpha_and &= _mm_movemask_epi8(_mm_cmpeq_epi8(all_alphas, all_0xff)); + return (alpha_and == 0xffffu); +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void WebPInitAlphaProcessingSSE41(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitAlphaProcessingSSE41(void) { + WebPExtractAlpha = ExtractAlpha; +} + +#else // !WEBP_USE_SSE41 + +WEBP_DSP_INIT_STUB(WebPInitAlphaProcessingSSE41) + +#endif // WEBP_USE_SSE41 diff --git a/thirdparty/libwebp/dsp/argb.c b/thirdparty/libwebp/dsp/argb.c new file mode 100644 index 0000000000..cc1f9a96c3 --- /dev/null +++ b/thirdparty/libwebp/dsp/argb.c @@ -0,0 +1,68 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// ARGB making functions. +// +// Author: Djordje Pesut (djordje.pesut@imgtec.com) + +#include "./dsp.h" + +static WEBP_INLINE uint32_t MakeARGB32(int a, int r, int g, int b) { + return (((uint32_t)a << 24) | (r << 16) | (g << 8) | b); +} + +static void PackARGB(const uint8_t* a, const uint8_t* r, const uint8_t* g, + const uint8_t* b, int len, uint32_t* out) { + int i; + for (i = 0; i < len; ++i) { + out[i] = MakeARGB32(a[4 * i], r[4 * i], g[4 * i], b[4 * i]); + } +} + +static void PackRGB(const uint8_t* r, const uint8_t* g, const uint8_t* b, + int len, int step, uint32_t* out) { + int i, offset = 0; + for (i = 0; i < len; ++i) { + out[i] = MakeARGB32(0xff, r[offset], g[offset], b[offset]); + offset += step; + } +} + +void (*VP8PackARGB)(const uint8_t*, const uint8_t*, const uint8_t*, + const uint8_t*, int, uint32_t*); +void (*VP8PackRGB)(const uint8_t*, const uint8_t*, const uint8_t*, + int, int, uint32_t*); + +extern void VP8EncDspARGBInitMIPSdspR2(void); +extern void VP8EncDspARGBInitSSE2(void); + +static volatile VP8CPUInfo argb_last_cpuinfo_used = + (VP8CPUInfo)&argb_last_cpuinfo_used; + +WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspARGBInit(void) { + if (argb_last_cpuinfo_used == VP8GetCPUInfo) return; + + VP8PackARGB = PackARGB; + VP8PackRGB = PackRGB; + + // If defined, use CPUInfo() to overwrite some pointers with faster versions. + if (VP8GetCPUInfo != NULL) { +#if defined(WEBP_USE_SSE2) + if (VP8GetCPUInfo(kSSE2)) { + VP8EncDspARGBInitSSE2(); + } +#endif +#if defined(WEBP_USE_MIPS_DSP_R2) + if (VP8GetCPUInfo(kMIPSdspR2)) { + VP8EncDspARGBInitMIPSdspR2(); + } +#endif + } + argb_last_cpuinfo_used = VP8GetCPUInfo; +} diff --git a/thirdparty/libwebp/dsp/argb_mips_dsp_r2.c b/thirdparty/libwebp/dsp/argb_mips_dsp_r2.c new file mode 100644 index 0000000000..af65acb8ff --- /dev/null +++ b/thirdparty/libwebp/dsp/argb_mips_dsp_r2.c @@ -0,0 +1,110 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// ARGB making functions (mips version). +// +// Author: Djordje Pesut (djordje.pesut@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS_DSP_R2) + +static void PackARGB(const uint8_t* a, const uint8_t* r, const uint8_t* g, + const uint8_t* b, int len, uint32_t* out) { + int temp0, temp1, temp2, temp3, offset; + const int rest = len & 1; + const uint32_t* const loop_end = out + len - rest; + const int step = 4; + __asm__ volatile ( + "xor %[offset], %[offset], %[offset] \n\t" + "beq %[loop_end], %[out], 0f \n\t" + "2: \n\t" + "lbux %[temp0], %[offset](%[a]) \n\t" + "lbux %[temp1], %[offset](%[r]) \n\t" + "lbux %[temp2], %[offset](%[g]) \n\t" + "lbux %[temp3], %[offset](%[b]) \n\t" + "ins %[temp1], %[temp0], 16, 16 \n\t" + "ins %[temp3], %[temp2], 16, 16 \n\t" + "addiu %[out], %[out], 4 \n\t" + "precr.qb.ph %[temp0], %[temp1], %[temp3] \n\t" + "sw %[temp0], -4(%[out]) \n\t" + "addu %[offset], %[offset], %[step] \n\t" + "bne %[loop_end], %[out], 2b \n\t" + "0: \n\t" + "beq %[rest], $zero, 1f \n\t" + "lbux %[temp0], %[offset](%[a]) \n\t" + "lbux %[temp1], %[offset](%[r]) \n\t" + "lbux %[temp2], %[offset](%[g]) \n\t" + "lbux %[temp3], %[offset](%[b]) \n\t" + "ins %[temp1], %[temp0], 16, 16 \n\t" + "ins %[temp3], %[temp2], 16, 16 \n\t" + "precr.qb.ph %[temp0], %[temp1], %[temp3] \n\t" + "sw %[temp0], 0(%[out]) \n\t" + "1: \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [offset]"=&r"(offset), [out]"+&r"(out) + : [a]"r"(a), [r]"r"(r), [g]"r"(g), [b]"r"(b), [step]"r"(step), + [loop_end]"r"(loop_end), [rest]"r"(rest) + : "memory" + ); +} + +static void PackRGB(const uint8_t* r, const uint8_t* g, const uint8_t* b, + int len, int step, uint32_t* out) { + int temp0, temp1, temp2, offset; + const int rest = len & 1; + const int a = 0xff; + const uint32_t* const loop_end = out + len - rest; + __asm__ volatile ( + "xor %[offset], %[offset], %[offset] \n\t" + "beq %[loop_end], %[out], 0f \n\t" + "2: \n\t" + "lbux %[temp0], %[offset](%[r]) \n\t" + "lbux %[temp1], %[offset](%[g]) \n\t" + "lbux %[temp2], %[offset](%[b]) \n\t" + "ins %[temp0], %[a], 16, 16 \n\t" + "ins %[temp2], %[temp1], 16, 16 \n\t" + "addiu %[out], %[out], 4 \n\t" + "precr.qb.ph %[temp0], %[temp0], %[temp2] \n\t" + "sw %[temp0], -4(%[out]) \n\t" + "addu %[offset], %[offset], %[step] \n\t" + "bne %[loop_end], %[out], 2b \n\t" + "0: \n\t" + "beq %[rest], $zero, 1f \n\t" + "lbux %[temp0], %[offset](%[r]) \n\t" + "lbux %[temp1], %[offset](%[g]) \n\t" + "lbux %[temp2], %[offset](%[b]) \n\t" + "ins %[temp0], %[a], 16, 16 \n\t" + "ins %[temp2], %[temp1], 16, 16 \n\t" + "precr.qb.ph %[temp0], %[temp0], %[temp2] \n\t" + "sw %[temp0], 0(%[out]) \n\t" + "1: \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [offset]"=&r"(offset), [out]"+&r"(out) + : [a]"r"(a), [r]"r"(r), [g]"r"(g), [b]"r"(b), [step]"r"(step), + [loop_end]"r"(loop_end), [rest]"r"(rest) + : "memory" + ); +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8EncDspARGBInitMIPSdspR2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspARGBInitMIPSdspR2(void) { + VP8PackARGB = PackARGB; + VP8PackRGB = PackRGB; +} + +#else // !WEBP_USE_MIPS_DSP_R2 + +WEBP_DSP_INIT_STUB(VP8EncDspARGBInitMIPSdspR2) + +#endif // WEBP_USE_MIPS_DSP_R2 diff --git a/thirdparty/libwebp/dsp/argb_sse2.c b/thirdparty/libwebp/dsp/argb_sse2.c new file mode 100644 index 0000000000..afcb1957e7 --- /dev/null +++ b/thirdparty/libwebp/dsp/argb_sse2.c @@ -0,0 +1,67 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// ARGB making functions (SSE2 version). +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE2) + +#include <assert.h> +#include <emmintrin.h> +#include <string.h> + +static WEBP_INLINE uint32_t MakeARGB32(int a, int r, int g, int b) { + return (((uint32_t)a << 24) | (r << 16) | (g << 8) | b); +} + +static void PackARGB(const uint8_t* a, const uint8_t* r, const uint8_t* g, + const uint8_t* b, int len, uint32_t* out) { + if (g == r + 1) { // RGBA input order. Need to swap R and B. + int i = 0; + const int len_max = len & ~3; // max length processed in main loop + const __m128i red_blue_mask = _mm_set1_epi32(0x00ff00ffu); + assert(b == r + 2); + assert(a == r + 3); + for (; i < len_max; i += 4) { + const __m128i A = _mm_loadu_si128((const __m128i*)(r + 4 * i)); + const __m128i B = _mm_and_si128(A, red_blue_mask); // R 0 B 0 + const __m128i C = _mm_andnot_si128(red_blue_mask, A); // 0 G 0 A + const __m128i D = _mm_shufflelo_epi16(B, _MM_SHUFFLE(2, 3, 0, 1)); + const __m128i E = _mm_shufflehi_epi16(D, _MM_SHUFFLE(2, 3, 0, 1)); + const __m128i F = _mm_or_si128(E, C); + _mm_storeu_si128((__m128i*)(out + i), F); + } + for (; i < len; ++i) { + out[i] = MakeARGB32(a[4 * i], r[4 * i], g[4 * i], b[4 * i]); + } + } else { + assert(g == b + 1); + assert(r == b + 2); + assert(a == b + 3); + memcpy(out, b, len * 4); + } +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8EncDspARGBInitSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspARGBInitSSE2(void) { + VP8PackARGB = PackARGB; +} + +#else // !WEBP_USE_SSE2 + +WEBP_DSP_INIT_STUB(VP8EncDspARGBInitSSE2) + +#endif // WEBP_USE_SSE2 diff --git a/thirdparty/libwebp/dsp/common_sse2.h b/thirdparty/libwebp/dsp/common_sse2.h new file mode 100644 index 0000000000..7cea13fb3c --- /dev/null +++ b/thirdparty/libwebp/dsp/common_sse2.h @@ -0,0 +1,109 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE2 code common to several files. +// +// Author: Vincent Rabaud (vrabaud@google.com) + +#ifndef WEBP_DSP_COMMON_SSE2_H_ +#define WEBP_DSP_COMMON_SSE2_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(WEBP_USE_SSE2) + +#include <emmintrin.h> + +//------------------------------------------------------------------------------ +// Quite useful macro for debugging. Left here for convenience. + +#if 0 +#include <stdio.h> +static WEBP_INLINE void PrintReg(const __m128i r, const char* const name, + int size) { + int n; + union { + __m128i r; + uint8_t i8[16]; + uint16_t i16[8]; + uint32_t i32[4]; + uint64_t i64[2]; + } tmp; + tmp.r = r; + fprintf(stderr, "%s\t: ", name); + if (size == 8) { + for (n = 0; n < 16; ++n) fprintf(stderr, "%.2x ", tmp.i8[n]); + } else if (size == 16) { + for (n = 0; n < 8; ++n) fprintf(stderr, "%.4x ", tmp.i16[n]); + } else if (size == 32) { + for (n = 0; n < 4; ++n) fprintf(stderr, "%.8x ", tmp.i32[n]); + } else { + for (n = 0; n < 2; ++n) fprintf(stderr, "%.16lx ", tmp.i64[n]); + } + fprintf(stderr, "\n"); +} +#endif + +//------------------------------------------------------------------------------ +// Math functions. + +// Return the sum of all the 8b in the register. +static WEBP_INLINE int VP8HorizontalAdd8b(const __m128i* const a) { + const __m128i zero = _mm_setzero_si128(); + const __m128i sad8x2 = _mm_sad_epu8(*a, zero); + // sum the two sads: sad8x2[0:1] + sad8x2[8:9] + const __m128i sum = _mm_add_epi32(sad8x2, _mm_shuffle_epi32(sad8x2, 2)); + return _mm_cvtsi128_si32(sum); +} + +// Transpose two 4x4 16b matrices horizontally stored in registers. +static WEBP_INLINE void VP8Transpose_2_4x4_16b( + const __m128i* const in0, const __m128i* const in1, + const __m128i* const in2, const __m128i* const in3, __m128i* const out0, + __m128i* const out1, __m128i* const out2, __m128i* const out3) { + // Transpose the two 4x4. + // a00 a01 a02 a03 b00 b01 b02 b03 + // a10 a11 a12 a13 b10 b11 b12 b13 + // a20 a21 a22 a23 b20 b21 b22 b23 + // a30 a31 a32 a33 b30 b31 b32 b33 + const __m128i transpose0_0 = _mm_unpacklo_epi16(*in0, *in1); + const __m128i transpose0_1 = _mm_unpacklo_epi16(*in2, *in3); + const __m128i transpose0_2 = _mm_unpackhi_epi16(*in0, *in1); + const __m128i transpose0_3 = _mm_unpackhi_epi16(*in2, *in3); + // a00 a10 a01 a11 a02 a12 a03 a13 + // a20 a30 a21 a31 a22 a32 a23 a33 + // b00 b10 b01 b11 b02 b12 b03 b13 + // b20 b30 b21 b31 b22 b32 b23 b33 + const __m128i transpose1_0 = _mm_unpacklo_epi32(transpose0_0, transpose0_1); + const __m128i transpose1_1 = _mm_unpacklo_epi32(transpose0_2, transpose0_3); + const __m128i transpose1_2 = _mm_unpackhi_epi32(transpose0_0, transpose0_1); + const __m128i transpose1_3 = _mm_unpackhi_epi32(transpose0_2, transpose0_3); + // a00 a10 a20 a30 a01 a11 a21 a31 + // b00 b10 b20 b30 b01 b11 b21 b31 + // a02 a12 a22 a32 a03 a13 a23 a33 + // b02 b12 a22 b32 b03 b13 b23 b33 + *out0 = _mm_unpacklo_epi64(transpose1_0, transpose1_1); + *out1 = _mm_unpackhi_epi64(transpose1_0, transpose1_1); + *out2 = _mm_unpacklo_epi64(transpose1_2, transpose1_3); + *out3 = _mm_unpackhi_epi64(transpose1_2, transpose1_3); + // a00 a10 a20 a30 b00 b10 b20 b30 + // a01 a11 a21 a31 b01 b11 b21 b31 + // a02 a12 a22 a32 b02 b12 b22 b32 + // a03 a13 a23 a33 b03 b13 b23 b33 +} + +#endif // WEBP_USE_SSE2 + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // WEBP_DSP_COMMON_SSE2_H_ diff --git a/thirdparty/libwebp/dsp/cost.c b/thirdparty/libwebp/dsp/cost.c new file mode 100644 index 0000000000..fe72d26e79 --- /dev/null +++ b/thirdparty/libwebp/dsp/cost.c @@ -0,0 +1,412 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" +#include "../enc/cost.h" + +//------------------------------------------------------------------------------ +// Boolean-cost cost table + +const uint16_t VP8EntropyCost[256] = { + 1792, 1792, 1792, 1536, 1536, 1408, 1366, 1280, 1280, 1216, + 1178, 1152, 1110, 1076, 1061, 1024, 1024, 992, 968, 951, + 939, 911, 896, 878, 871, 854, 838, 820, 811, 794, + 786, 768, 768, 752, 740, 732, 720, 709, 704, 690, + 683, 672, 666, 655, 647, 640, 631, 622, 615, 607, + 598, 592, 586, 576, 572, 564, 559, 555, 547, 541, + 534, 528, 522, 512, 512, 504, 500, 494, 488, 483, + 477, 473, 467, 461, 458, 452, 448, 443, 438, 434, + 427, 424, 419, 415, 410, 406, 403, 399, 394, 390, + 384, 384, 377, 374, 370, 366, 362, 359, 355, 351, + 347, 342, 342, 336, 333, 330, 326, 323, 320, 316, + 312, 308, 305, 302, 299, 296, 293, 288, 287, 283, + 280, 277, 274, 272, 268, 266, 262, 256, 256, 256, + 251, 248, 245, 242, 240, 237, 234, 232, 228, 226, + 223, 221, 218, 216, 214, 211, 208, 205, 203, 201, + 198, 196, 192, 191, 188, 187, 183, 181, 179, 176, + 175, 171, 171, 168, 165, 163, 160, 159, 156, 154, + 152, 150, 148, 146, 144, 142, 139, 138, 135, 133, + 131, 128, 128, 125, 123, 121, 119, 117, 115, 113, + 111, 110, 107, 105, 103, 102, 100, 98, 96, 94, + 92, 91, 89, 86, 86, 83, 82, 80, 77, 76, + 74, 73, 71, 69, 67, 66, 64, 63, 61, 59, + 57, 55, 54, 52, 51, 49, 47, 46, 44, 43, + 41, 40, 38, 36, 35, 33, 32, 30, 29, 27, + 25, 24, 22, 21, 19, 18, 16, 15, 13, 12, + 10, 9, 7, 6, 4, 3 +}; + +//------------------------------------------------------------------------------ +// Level cost tables + +// fixed costs for coding levels, deduce from the coding tree. +// This is only the part that doesn't depend on the probability state. +const uint16_t VP8LevelFixedCosts[MAX_LEVEL + 1] = { + 0, 256, 256, 256, 256, 432, 618, 630, + 731, 640, 640, 828, 901, 948, 1021, 1101, + 1174, 1221, 1294, 1042, 1085, 1115, 1158, 1202, + 1245, 1275, 1318, 1337, 1380, 1410, 1453, 1497, + 1540, 1570, 1613, 1280, 1295, 1317, 1332, 1358, + 1373, 1395, 1410, 1454, 1469, 1491, 1506, 1532, + 1547, 1569, 1584, 1601, 1616, 1638, 1653, 1679, + 1694, 1716, 1731, 1775, 1790, 1812, 1827, 1853, + 1868, 1890, 1905, 1727, 1733, 1742, 1748, 1759, + 1765, 1774, 1780, 1800, 1806, 1815, 1821, 1832, + 1838, 1847, 1853, 1878, 1884, 1893, 1899, 1910, + 1916, 1925, 1931, 1951, 1957, 1966, 1972, 1983, + 1989, 1998, 2004, 2027, 2033, 2042, 2048, 2059, + 2065, 2074, 2080, 2100, 2106, 2115, 2121, 2132, + 2138, 2147, 2153, 2178, 2184, 2193, 2199, 2210, + 2216, 2225, 2231, 2251, 2257, 2266, 2272, 2283, + 2289, 2298, 2304, 2168, 2174, 2183, 2189, 2200, + 2206, 2215, 2221, 2241, 2247, 2256, 2262, 2273, + 2279, 2288, 2294, 2319, 2325, 2334, 2340, 2351, + 2357, 2366, 2372, 2392, 2398, 2407, 2413, 2424, + 2430, 2439, 2445, 2468, 2474, 2483, 2489, 2500, + 2506, 2515, 2521, 2541, 2547, 2556, 2562, 2573, + 2579, 2588, 2594, 2619, 2625, 2634, 2640, 2651, + 2657, 2666, 2672, 2692, 2698, 2707, 2713, 2724, + 2730, 2739, 2745, 2540, 2546, 2555, 2561, 2572, + 2578, 2587, 2593, 2613, 2619, 2628, 2634, 2645, + 2651, 2660, 2666, 2691, 2697, 2706, 2712, 2723, + 2729, 2738, 2744, 2764, 2770, 2779, 2785, 2796, + 2802, 2811, 2817, 2840, 2846, 2855, 2861, 2872, + 2878, 2887, 2893, 2913, 2919, 2928, 2934, 2945, + 2951, 2960, 2966, 2991, 2997, 3006, 3012, 3023, + 3029, 3038, 3044, 3064, 3070, 3079, 3085, 3096, + 3102, 3111, 3117, 2981, 2987, 2996, 3002, 3013, + 3019, 3028, 3034, 3054, 3060, 3069, 3075, 3086, + 3092, 3101, 3107, 3132, 3138, 3147, 3153, 3164, + 3170, 3179, 3185, 3205, 3211, 3220, 3226, 3237, + 3243, 3252, 3258, 3281, 3287, 3296, 3302, 3313, + 3319, 3328, 3334, 3354, 3360, 3369, 3375, 3386, + 3392, 3401, 3407, 3432, 3438, 3447, 3453, 3464, + 3470, 3479, 3485, 3505, 3511, 3520, 3526, 3537, + 3543, 3552, 3558, 2816, 2822, 2831, 2837, 2848, + 2854, 2863, 2869, 2889, 2895, 2904, 2910, 2921, + 2927, 2936, 2942, 2967, 2973, 2982, 2988, 2999, + 3005, 3014, 3020, 3040, 3046, 3055, 3061, 3072, + 3078, 3087, 3093, 3116, 3122, 3131, 3137, 3148, + 3154, 3163, 3169, 3189, 3195, 3204, 3210, 3221, + 3227, 3236, 3242, 3267, 3273, 3282, 3288, 3299, + 3305, 3314, 3320, 3340, 3346, 3355, 3361, 3372, + 3378, 3387, 3393, 3257, 3263, 3272, 3278, 3289, + 3295, 3304, 3310, 3330, 3336, 3345, 3351, 3362, + 3368, 3377, 3383, 3408, 3414, 3423, 3429, 3440, + 3446, 3455, 3461, 3481, 3487, 3496, 3502, 3513, + 3519, 3528, 3534, 3557, 3563, 3572, 3578, 3589, + 3595, 3604, 3610, 3630, 3636, 3645, 3651, 3662, + 3668, 3677, 3683, 3708, 3714, 3723, 3729, 3740, + 3746, 3755, 3761, 3781, 3787, 3796, 3802, 3813, + 3819, 3828, 3834, 3629, 3635, 3644, 3650, 3661, + 3667, 3676, 3682, 3702, 3708, 3717, 3723, 3734, + 3740, 3749, 3755, 3780, 3786, 3795, 3801, 3812, + 3818, 3827, 3833, 3853, 3859, 3868, 3874, 3885, + 3891, 3900, 3906, 3929, 3935, 3944, 3950, 3961, + 3967, 3976, 3982, 4002, 4008, 4017, 4023, 4034, + 4040, 4049, 4055, 4080, 4086, 4095, 4101, 4112, + 4118, 4127, 4133, 4153, 4159, 4168, 4174, 4185, + 4191, 4200, 4206, 4070, 4076, 4085, 4091, 4102, + 4108, 4117, 4123, 4143, 4149, 4158, 4164, 4175, + 4181, 4190, 4196, 4221, 4227, 4236, 4242, 4253, + 4259, 4268, 4274, 4294, 4300, 4309, 4315, 4326, + 4332, 4341, 4347, 4370, 4376, 4385, 4391, 4402, + 4408, 4417, 4423, 4443, 4449, 4458, 4464, 4475, + 4481, 4490, 4496, 4521, 4527, 4536, 4542, 4553, + 4559, 4568, 4574, 4594, 4600, 4609, 4615, 4626, + 4632, 4641, 4647, 3515, 3521, 3530, 3536, 3547, + 3553, 3562, 3568, 3588, 3594, 3603, 3609, 3620, + 3626, 3635, 3641, 3666, 3672, 3681, 3687, 3698, + 3704, 3713, 3719, 3739, 3745, 3754, 3760, 3771, + 3777, 3786, 3792, 3815, 3821, 3830, 3836, 3847, + 3853, 3862, 3868, 3888, 3894, 3903, 3909, 3920, + 3926, 3935, 3941, 3966, 3972, 3981, 3987, 3998, + 4004, 4013, 4019, 4039, 4045, 4054, 4060, 4071, + 4077, 4086, 4092, 3956, 3962, 3971, 3977, 3988, + 3994, 4003, 4009, 4029, 4035, 4044, 4050, 4061, + 4067, 4076, 4082, 4107, 4113, 4122, 4128, 4139, + 4145, 4154, 4160, 4180, 4186, 4195, 4201, 4212, + 4218, 4227, 4233, 4256, 4262, 4271, 4277, 4288, + 4294, 4303, 4309, 4329, 4335, 4344, 4350, 4361, + 4367, 4376, 4382, 4407, 4413, 4422, 4428, 4439, + 4445, 4454, 4460, 4480, 4486, 4495, 4501, 4512, + 4518, 4527, 4533, 4328, 4334, 4343, 4349, 4360, + 4366, 4375, 4381, 4401, 4407, 4416, 4422, 4433, + 4439, 4448, 4454, 4479, 4485, 4494, 4500, 4511, + 4517, 4526, 4532, 4552, 4558, 4567, 4573, 4584, + 4590, 4599, 4605, 4628, 4634, 4643, 4649, 4660, + 4666, 4675, 4681, 4701, 4707, 4716, 4722, 4733, + 4739, 4748, 4754, 4779, 4785, 4794, 4800, 4811, + 4817, 4826, 4832, 4852, 4858, 4867, 4873, 4884, + 4890, 4899, 4905, 4769, 4775, 4784, 4790, 4801, + 4807, 4816, 4822, 4842, 4848, 4857, 4863, 4874, + 4880, 4889, 4895, 4920, 4926, 4935, 4941, 4952, + 4958, 4967, 4973, 4993, 4999, 5008, 5014, 5025, + 5031, 5040, 5046, 5069, 5075, 5084, 5090, 5101, + 5107, 5116, 5122, 5142, 5148, 5157, 5163, 5174, + 5180, 5189, 5195, 5220, 5226, 5235, 5241, 5252, + 5258, 5267, 5273, 5293, 5299, 5308, 5314, 5325, + 5331, 5340, 5346, 4604, 4610, 4619, 4625, 4636, + 4642, 4651, 4657, 4677, 4683, 4692, 4698, 4709, + 4715, 4724, 4730, 4755, 4761, 4770, 4776, 4787, + 4793, 4802, 4808, 4828, 4834, 4843, 4849, 4860, + 4866, 4875, 4881, 4904, 4910, 4919, 4925, 4936, + 4942, 4951, 4957, 4977, 4983, 4992, 4998, 5009, + 5015, 5024, 5030, 5055, 5061, 5070, 5076, 5087, + 5093, 5102, 5108, 5128, 5134, 5143, 5149, 5160, + 5166, 5175, 5181, 5045, 5051, 5060, 5066, 5077, + 5083, 5092, 5098, 5118, 5124, 5133, 5139, 5150, + 5156, 5165, 5171, 5196, 5202, 5211, 5217, 5228, + 5234, 5243, 5249, 5269, 5275, 5284, 5290, 5301, + 5307, 5316, 5322, 5345, 5351, 5360, 5366, 5377, + 5383, 5392, 5398, 5418, 5424, 5433, 5439, 5450, + 5456, 5465, 5471, 5496, 5502, 5511, 5517, 5528, + 5534, 5543, 5549, 5569, 5575, 5584, 5590, 5601, + 5607, 5616, 5622, 5417, 5423, 5432, 5438, 5449, + 5455, 5464, 5470, 5490, 5496, 5505, 5511, 5522, + 5528, 5537, 5543, 5568, 5574, 5583, 5589, 5600, + 5606, 5615, 5621, 5641, 5647, 5656, 5662, 5673, + 5679, 5688, 5694, 5717, 5723, 5732, 5738, 5749, + 5755, 5764, 5770, 5790, 5796, 5805, 5811, 5822, + 5828, 5837, 5843, 5868, 5874, 5883, 5889, 5900, + 5906, 5915, 5921, 5941, 5947, 5956, 5962, 5973, + 5979, 5988, 5994, 5858, 5864, 5873, 5879, 5890, + 5896, 5905, 5911, 5931, 5937, 5946, 5952, 5963, + 5969, 5978, 5984, 6009, 6015, 6024, 6030, 6041, + 6047, 6056, 6062, 6082, 6088, 6097, 6103, 6114, + 6120, 6129, 6135, 6158, 6164, 6173, 6179, 6190, + 6196, 6205, 6211, 6231, 6237, 6246, 6252, 6263, + 6269, 6278, 6284, 6309, 6315, 6324, 6330, 6341, + 6347, 6356, 6362, 6382, 6388, 6397, 6403, 6414, + 6420, 6429, 6435, 3515, 3521, 3530, 3536, 3547, + 3553, 3562, 3568, 3588, 3594, 3603, 3609, 3620, + 3626, 3635, 3641, 3666, 3672, 3681, 3687, 3698, + 3704, 3713, 3719, 3739, 3745, 3754, 3760, 3771, + 3777, 3786, 3792, 3815, 3821, 3830, 3836, 3847, + 3853, 3862, 3868, 3888, 3894, 3903, 3909, 3920, + 3926, 3935, 3941, 3966, 3972, 3981, 3987, 3998, + 4004, 4013, 4019, 4039, 4045, 4054, 4060, 4071, + 4077, 4086, 4092, 3956, 3962, 3971, 3977, 3988, + 3994, 4003, 4009, 4029, 4035, 4044, 4050, 4061, + 4067, 4076, 4082, 4107, 4113, 4122, 4128, 4139, + 4145, 4154, 4160, 4180, 4186, 4195, 4201, 4212, + 4218, 4227, 4233, 4256, 4262, 4271, 4277, 4288, + 4294, 4303, 4309, 4329, 4335, 4344, 4350, 4361, + 4367, 4376, 4382, 4407, 4413, 4422, 4428, 4439, + 4445, 4454, 4460, 4480, 4486, 4495, 4501, 4512, + 4518, 4527, 4533, 4328, 4334, 4343, 4349, 4360, + 4366, 4375, 4381, 4401, 4407, 4416, 4422, 4433, + 4439, 4448, 4454, 4479, 4485, 4494, 4500, 4511, + 4517, 4526, 4532, 4552, 4558, 4567, 4573, 4584, + 4590, 4599, 4605, 4628, 4634, 4643, 4649, 4660, + 4666, 4675, 4681, 4701, 4707, 4716, 4722, 4733, + 4739, 4748, 4754, 4779, 4785, 4794, 4800, 4811, + 4817, 4826, 4832, 4852, 4858, 4867, 4873, 4884, + 4890, 4899, 4905, 4769, 4775, 4784, 4790, 4801, + 4807, 4816, 4822, 4842, 4848, 4857, 4863, 4874, + 4880, 4889, 4895, 4920, 4926, 4935, 4941, 4952, + 4958, 4967, 4973, 4993, 4999, 5008, 5014, 5025, + 5031, 5040, 5046, 5069, 5075, 5084, 5090, 5101, + 5107, 5116, 5122, 5142, 5148, 5157, 5163, 5174, + 5180, 5189, 5195, 5220, 5226, 5235, 5241, 5252, + 5258, 5267, 5273, 5293, 5299, 5308, 5314, 5325, + 5331, 5340, 5346, 4604, 4610, 4619, 4625, 4636, + 4642, 4651, 4657, 4677, 4683, 4692, 4698, 4709, + 4715, 4724, 4730, 4755, 4761, 4770, 4776, 4787, + 4793, 4802, 4808, 4828, 4834, 4843, 4849, 4860, + 4866, 4875, 4881, 4904, 4910, 4919, 4925, 4936, + 4942, 4951, 4957, 4977, 4983, 4992, 4998, 5009, + 5015, 5024, 5030, 5055, 5061, 5070, 5076, 5087, + 5093, 5102, 5108, 5128, 5134, 5143, 5149, 5160, + 5166, 5175, 5181, 5045, 5051, 5060, 5066, 5077, + 5083, 5092, 5098, 5118, 5124, 5133, 5139, 5150, + 5156, 5165, 5171, 5196, 5202, 5211, 5217, 5228, + 5234, 5243, 5249, 5269, 5275, 5284, 5290, 5301, + 5307, 5316, 5322, 5345, 5351, 5360, 5366, 5377, + 5383, 5392, 5398, 5418, 5424, 5433, 5439, 5450, + 5456, 5465, 5471, 5496, 5502, 5511, 5517, 5528, + 5534, 5543, 5549, 5569, 5575, 5584, 5590, 5601, + 5607, 5616, 5622, 5417, 5423, 5432, 5438, 5449, + 5455, 5464, 5470, 5490, 5496, 5505, 5511, 5522, + 5528, 5537, 5543, 5568, 5574, 5583, 5589, 5600, + 5606, 5615, 5621, 5641, 5647, 5656, 5662, 5673, + 5679, 5688, 5694, 5717, 5723, 5732, 5738, 5749, + 5755, 5764, 5770, 5790, 5796, 5805, 5811, 5822, + 5828, 5837, 5843, 5868, 5874, 5883, 5889, 5900, + 5906, 5915, 5921, 5941, 5947, 5956, 5962, 5973, + 5979, 5988, 5994, 5858, 5864, 5873, 5879, 5890, + 5896, 5905, 5911, 5931, 5937, 5946, 5952, 5963, + 5969, 5978, 5984, 6009, 6015, 6024, 6030, 6041, + 6047, 6056, 6062, 6082, 6088, 6097, 6103, 6114, + 6120, 6129, 6135, 6158, 6164, 6173, 6179, 6190, + 6196, 6205, 6211, 6231, 6237, 6246, 6252, 6263, + 6269, 6278, 6284, 6309, 6315, 6324, 6330, 6341, + 6347, 6356, 6362, 6382, 6388, 6397, 6403, 6414, + 6420, 6429, 6435, 5303, 5309, 5318, 5324, 5335, + 5341, 5350, 5356, 5376, 5382, 5391, 5397, 5408, + 5414, 5423, 5429, 5454, 5460, 5469, 5475, 5486, + 5492, 5501, 5507, 5527, 5533, 5542, 5548, 5559, + 5565, 5574, 5580, 5603, 5609, 5618, 5624, 5635, + 5641, 5650, 5656, 5676, 5682, 5691, 5697, 5708, + 5714, 5723, 5729, 5754, 5760, 5769, 5775, 5786, + 5792, 5801, 5807, 5827, 5833, 5842, 5848, 5859, + 5865, 5874, 5880, 5744, 5750, 5759, 5765, 5776, + 5782, 5791, 5797, 5817, 5823, 5832, 5838, 5849, + 5855, 5864, 5870, 5895, 5901, 5910, 5916, 5927, + 5933, 5942, 5948, 5968, 5974, 5983, 5989, 6000, + 6006, 6015, 6021, 6044, 6050, 6059, 6065, 6076, + 6082, 6091, 6097, 6117, 6123, 6132, 6138, 6149, + 6155, 6164, 6170, 6195, 6201, 6210, 6216, 6227, + 6233, 6242, 6248, 6268, 6274, 6283, 6289, 6300, + 6306, 6315, 6321, 6116, 6122, 6131, 6137, 6148, + 6154, 6163, 6169, 6189, 6195, 6204, 6210, 6221, + 6227, 6236, 6242, 6267, 6273, 6282, 6288, 6299, + 6305, 6314, 6320, 6340, 6346, 6355, 6361, 6372, + 6378, 6387, 6393, 6416, 6422, 6431, 6437, 6448, + 6454, 6463, 6469, 6489, 6495, 6504, 6510, 6521, + 6527, 6536, 6542, 6567, 6573, 6582, 6588, 6599, + 6605, 6614, 6620, 6640, 6646, 6655, 6661, 6672, + 6678, 6687, 6693, 6557, 6563, 6572, 6578, 6589, + 6595, 6604, 6610, 6630, 6636, 6645, 6651, 6662, + 6668, 6677, 6683, 6708, 6714, 6723, 6729, 6740, + 6746, 6755, 6761, 6781, 6787, 6796, 6802, 6813, + 6819, 6828, 6834, 6857, 6863, 6872, 6878, 6889, + 6895, 6904, 6910, 6930, 6936, 6945, 6951, 6962, + 6968, 6977, 6983, 7008, 7014, 7023, 7029, 7040, + 7046, 7055, 7061, 7081, 7087, 7096, 7102, 7113, + 7119, 7128, 7134, 6392, 6398, 6407, 6413, 6424, + 6430, 6439, 6445, 6465, 6471, 6480, 6486, 6497, + 6503, 6512, 6518, 6543, 6549, 6558, 6564, 6575, + 6581, 6590, 6596, 6616, 6622, 6631, 6637, 6648, + 6654, 6663, 6669, 6692, 6698, 6707, 6713, 6724, + 6730, 6739, 6745, 6765, 6771, 6780, 6786, 6797, + 6803, 6812, 6818, 6843, 6849, 6858, 6864, 6875, + 6881, 6890, 6896, 6916, 6922, 6931, 6937, 6948, + 6954, 6963, 6969, 6833, 6839, 6848, 6854, 6865, + 6871, 6880, 6886, 6906, 6912, 6921, 6927, 6938, + 6944, 6953, 6959, 6984, 6990, 6999, 7005, 7016, + 7022, 7031, 7037, 7057, 7063, 7072, 7078, 7089, + 7095, 7104, 7110, 7133, 7139, 7148, 7154, 7165, + 7171, 7180, 7186, 7206, 7212, 7221, 7227, 7238, + 7244, 7253, 7259, 7284, 7290, 7299, 7305, 7316, + 7322, 7331, 7337, 7357, 7363, 7372, 7378, 7389, + 7395, 7404, 7410, 7205, 7211, 7220, 7226, 7237, + 7243, 7252, 7258, 7278, 7284, 7293, 7299, 7310, + 7316, 7325, 7331, 7356, 7362, 7371, 7377, 7388, + 7394, 7403, 7409, 7429, 7435, 7444, 7450, 7461, + 7467, 7476, 7482, 7505, 7511, 7520, 7526, 7537, + 7543, 7552, 7558, 7578, 7584, 7593, 7599, 7610, + 7616, 7625, 7631, 7656, 7662, 7671, 7677, 7688, + 7694, 7703, 7709, 7729, 7735, 7744, 7750, 7761 +}; + +//------------------------------------------------------------------------------ +// Tables for level coding + +const uint8_t VP8EncBands[16 + 1] = { + 0, 1, 2, 3, 6, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, + 0 // sentinel +}; + +//------------------------------------------------------------------------------ +// Mode costs + +static int GetResidualCost(int ctx0, const VP8Residual* const res) { + int n = res->first; + // should be prob[VP8EncBands[n]], but it's equivalent for n=0 or 1 + const int p0 = res->prob[n][ctx0][0]; + CostArrayPtr const costs = res->costs; + const uint16_t* t = costs[n][ctx0]; + // bit_cost(1, p0) is already incorporated in t[] tables, but only if ctx != 0 + // (as required by the syntax). For ctx0 == 0, we need to add it here or it'll + // be missing during the loop. + int cost = (ctx0 == 0) ? VP8BitCost(1, p0) : 0; + + if (res->last < 0) { + return VP8BitCost(0, p0); + } + for (; n < res->last; ++n) { + const int v = abs(res->coeffs[n]); + const int ctx = (v >= 2) ? 2 : v; + cost += VP8LevelCost(t, v); + t = costs[n + 1][ctx]; + } + // Last coefficient is always non-zero + { + const int v = abs(res->coeffs[n]); + assert(v != 0); + cost += VP8LevelCost(t, v); + if (n < 15) { + const int b = VP8EncBands[n + 1]; + const int ctx = (v == 1) ? 1 : 2; + const int last_p0 = res->prob[b][ctx][0]; + cost += VP8BitCost(0, last_p0); + } + } + return cost; +} + +static void SetResidualCoeffs(const int16_t* const coeffs, + VP8Residual* const res) { + int n; + res->last = -1; + assert(res->first == 0 || coeffs[0] == 0); + for (n = 15; n >= 0; --n) { + if (coeffs[n]) { + res->last = n; + break; + } + } + res->coeffs = coeffs; +} + +//------------------------------------------------------------------------------ +// init function + +VP8GetResidualCostFunc VP8GetResidualCost; +VP8SetResidualCoeffsFunc VP8SetResidualCoeffs; + +extern void VP8EncDspCostInitMIPS32(void); +extern void VP8EncDspCostInitMIPSdspR2(void); +extern void VP8EncDspCostInitSSE2(void); + +static volatile VP8CPUInfo cost_last_cpuinfo_used = + (VP8CPUInfo)&cost_last_cpuinfo_used; + +WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspCostInit(void) { + if (cost_last_cpuinfo_used == VP8GetCPUInfo) return; + + VP8GetResidualCost = GetResidualCost; + VP8SetResidualCoeffs = SetResidualCoeffs; + + // If defined, use CPUInfo() to overwrite some pointers with faster versions. + if (VP8GetCPUInfo != NULL) { +#if defined(WEBP_USE_MIPS32) + if (VP8GetCPUInfo(kMIPS32)) { + VP8EncDspCostInitMIPS32(); + } +#endif +#if defined(WEBP_USE_MIPS_DSP_R2) + if (VP8GetCPUInfo(kMIPSdspR2)) { + VP8EncDspCostInitMIPSdspR2(); + } +#endif +#if defined(WEBP_USE_SSE2) + if (VP8GetCPUInfo(kSSE2)) { + VP8EncDspCostInitSSE2(); + } +#endif + } + + cost_last_cpuinfo_used = VP8GetCPUInfo; +} + +//------------------------------------------------------------------------------ diff --git a/thirdparty/libwebp/dsp/cost_mips32.c b/thirdparty/libwebp/dsp/cost_mips32.c new file mode 100644 index 0000000000..d1e240e191 --- /dev/null +++ b/thirdparty/libwebp/dsp/cost_mips32.c @@ -0,0 +1,154 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Author: Djordje Pesut (djordje.pesut@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS32) + +#include "../enc/cost.h" + +static int GetResidualCost(int ctx0, const VP8Residual* const res) { + int temp0, temp1; + int v_reg, ctx_reg; + int n = res->first; + // should be prob[VP8EncBands[n]], but it's equivalent for n=0 or 1 + int p0 = res->prob[n][ctx0][0]; + CostArrayPtr const costs = res->costs; + const uint16_t* t = costs[n][ctx0]; + // bit_cost(1, p0) is already incorporated in t[] tables, but only if ctx != 0 + // (as required by the syntax). For ctx0 == 0, we need to add it here or it'll + // be missing during the loop. + int cost = (ctx0 == 0) ? VP8BitCost(1, p0) : 0; + const int16_t* res_coeffs = res->coeffs; + const int res_last = res->last; + const int const_max_level = MAX_VARIABLE_LEVEL; + const int const_2 = 2; + const uint16_t** p_costs = &costs[n][0]; + const size_t inc_p_costs = NUM_CTX * sizeof(*p_costs); + + if (res->last < 0) { + return VP8BitCost(0, p0); + } + + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "subu %[temp1], %[res_last], %[n] \n\t" + "sll %[temp0], %[n], 1 \n\t" + "blez %[temp1], 2f \n\t" + " addu %[res_coeffs], %[res_coeffs], %[temp0] \n\t" + "1: \n\t" + "lh %[v_reg], 0(%[res_coeffs]) \n\t" + "addiu %[n], %[n], 1 \n\t" + "negu %[temp0], %[v_reg] \n\t" + "slti %[temp1], %[v_reg], 0 \n\t" + "movn %[v_reg], %[temp0], %[temp1] \n\t" + "sltiu %[temp0], %[v_reg], 2 \n\t" + "move %[ctx_reg], %[v_reg] \n\t" + "movz %[ctx_reg], %[const_2], %[temp0] \n\t" + "sll %[temp1], %[v_reg], 1 \n\t" + "addu %[temp1], %[temp1], %[VP8LevelFixedCosts] \n\t" + "lhu %[temp1], 0(%[temp1]) \n\t" + "slt %[temp0], %[v_reg], %[const_max_level] \n\t" + "movz %[v_reg], %[const_max_level], %[temp0] \n\t" + "addu %[cost], %[cost], %[temp1] \n\t" + "sll %[v_reg], %[v_reg], 1 \n\t" + "sll %[ctx_reg], %[ctx_reg], 2 \n\t" + "addu %[v_reg], %[v_reg], %[t] \n\t" + "lhu %[temp0], 0(%[v_reg]) \n\t" + "addu %[p_costs], %[p_costs], %[inc_p_costs] \n\t" + "addu %[t], %[p_costs], %[ctx_reg] \n\t" + "addu %[cost], %[cost], %[temp0] \n\t" + "addiu %[res_coeffs], %[res_coeffs], 2 \n\t" + "bne %[n], %[res_last], 1b \n\t" + " lw %[t], 0(%[t]) \n\t" + "2: \n\t" + ".set pop \n\t" + : [cost]"+&r"(cost), [t]"+&r"(t), [n]"+&r"(n), [v_reg]"=&r"(v_reg), + [ctx_reg]"=&r"(ctx_reg), [p_costs]"+&r"(p_costs), [temp0]"=&r"(temp0), + [temp1]"=&r"(temp1), [res_coeffs]"+&r"(res_coeffs) + : [const_2]"r"(const_2), [const_max_level]"r"(const_max_level), + [VP8LevelFixedCosts]"r"(VP8LevelFixedCosts), [res_last]"r"(res_last), + [inc_p_costs]"r"(inc_p_costs) + : "memory" + ); + + // Last coefficient is always non-zero + { + const int v = abs(res->coeffs[n]); + assert(v != 0); + cost += VP8LevelCost(t, v); + if (n < 15) { + const int b = VP8EncBands[n + 1]; + const int ctx = (v == 1) ? 1 : 2; + const int last_p0 = res->prob[b][ctx][0]; + cost += VP8BitCost(0, last_p0); + } + } + return cost; +} + +static void SetResidualCoeffs(const int16_t* const coeffs, + VP8Residual* const res) { + const int16_t* p_coeffs = (int16_t*)coeffs; + int temp0, temp1, temp2, n, n1; + assert(res->first == 0 || coeffs[0] == 0); + + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "addiu %[p_coeffs], %[p_coeffs], 28 \n\t" + "li %[n], 15 \n\t" + "li %[temp2], -1 \n\t" + "0: \n\t" + "ulw %[temp0], 0(%[p_coeffs]) \n\t" + "beqz %[temp0], 1f \n\t" +#if defined(WORDS_BIGENDIAN) + " sll %[temp1], %[temp0], 16 \n\t" +#else + " srl %[temp1], %[temp0], 16 \n\t" +#endif + "addiu %[n1], %[n], -1 \n\t" + "movz %[temp0], %[n1], %[temp1] \n\t" + "movn %[temp0], %[n], %[temp1] \n\t" + "j 2f \n\t" + " addiu %[temp2], %[temp0], 0 \n\t" + "1: \n\t" + "addiu %[n], %[n], -2 \n\t" + "bgtz %[n], 0b \n\t" + " addiu %[p_coeffs], %[p_coeffs], -4 \n\t" + "2: \n\t" + ".set pop \n\t" + : [p_coeffs]"+&r"(p_coeffs), [temp0]"=&r"(temp0), + [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [n]"=&r"(n), [n1]"=&r"(n1) + : + : "memory" + ); + res->last = temp2; + res->coeffs = coeffs; +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8EncDspCostInitMIPS32(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspCostInitMIPS32(void) { + VP8GetResidualCost = GetResidualCost; + VP8SetResidualCoeffs = SetResidualCoeffs; +} + +#else // !WEBP_USE_MIPS32 + +WEBP_DSP_INIT_STUB(VP8EncDspCostInitMIPS32) + +#endif // WEBP_USE_MIPS32 diff --git a/thirdparty/libwebp/dsp/cost_mips_dsp_r2.c b/thirdparty/libwebp/dsp/cost_mips_dsp_r2.c new file mode 100644 index 0000000000..ce64067756 --- /dev/null +++ b/thirdparty/libwebp/dsp/cost_mips_dsp_r2.c @@ -0,0 +1,107 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Author: Djordje Pesut (djordje.pesut@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS_DSP_R2) + +#include "../enc/cost.h" + +static int GetResidualCost(int ctx0, const VP8Residual* const res) { + int temp0, temp1; + int v_reg, ctx_reg; + int n = res->first; + // should be prob[VP8EncBands[n]], but it's equivalent for n=0 or 1 + int p0 = res->prob[n][ctx0][0]; + CostArrayPtr const costs = res->costs; + const uint16_t* t = costs[n][ctx0]; + // bit_cost(1, p0) is already incorporated in t[] tables, but only if ctx != 0 + // (as required by the syntax). For ctx0 == 0, we need to add it here or it'll + // be missing during the loop. + int cost = (ctx0 == 0) ? VP8BitCost(1, p0) : 0; + const int16_t* res_coeffs = res->coeffs; + const int res_last = res->last; + const int const_max_level = MAX_VARIABLE_LEVEL; + const int const_2 = 2; + const uint16_t** p_costs = &costs[n][0]; + const size_t inc_p_costs = NUM_CTX * sizeof(*p_costs); + + if (res->last < 0) { + return VP8BitCost(0, p0); + } + + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "subu %[temp1], %[res_last], %[n] \n\t" + "blez %[temp1], 2f \n\t" + " nop \n\t" + "1: \n\t" + "sll %[temp0], %[n], 1 \n\t" + "lhx %[v_reg], %[temp0](%[res_coeffs]) \n\t" + "addiu %[n], %[n], 1 \n\t" + "absq_s.w %[v_reg], %[v_reg] \n\t" + "sltiu %[temp0], %[v_reg], 2 \n\t" + "move %[ctx_reg], %[v_reg] \n\t" + "movz %[ctx_reg], %[const_2], %[temp0] \n\t" + "sll %[temp1], %[v_reg], 1 \n\t" + "lhx %[temp1], %[temp1](%[VP8LevelFixedCosts]) \n\t" + "slt %[temp0], %[v_reg], %[const_max_level] \n\t" + "movz %[v_reg], %[const_max_level], %[temp0] \n\t" + "addu %[cost], %[cost], %[temp1] \n\t" + "sll %[v_reg], %[v_reg], 1 \n\t" + "sll %[ctx_reg], %[ctx_reg], 2 \n\t" + "lhx %[temp0], %[v_reg](%[t]) \n\t" + "addu %[p_costs], %[p_costs], %[inc_p_costs] \n\t" + "addu %[t], %[p_costs], %[ctx_reg] \n\t" + "addu %[cost], %[cost], %[temp0] \n\t" + "bne %[n], %[res_last], 1b \n\t" + " lw %[t], 0(%[t]) \n\t" + "2: \n\t" + ".set pop \n\t" + : [cost]"+&r"(cost), [t]"+&r"(t), [n]"+&r"(n), [v_reg]"=&r"(v_reg), + [ctx_reg]"=&r"(ctx_reg), [p_costs]"+&r"(p_costs), [temp0]"=&r"(temp0), + [temp1]"=&r"(temp1) + : [const_2]"r"(const_2), [const_max_level]"r"(const_max_level), + [VP8LevelFixedCosts]"r"(VP8LevelFixedCosts), [res_last]"r"(res_last), + [res_coeffs]"r"(res_coeffs), [inc_p_costs]"r"(inc_p_costs) + : "memory" + ); + + // Last coefficient is always non-zero + { + const int v = abs(res->coeffs[n]); + assert(v != 0); + cost += VP8LevelCost(t, v); + if (n < 15) { + const int b = VP8EncBands[n + 1]; + const int ctx = (v == 1) ? 1 : 2; + const int last_p0 = res->prob[b][ctx][0]; + cost += VP8BitCost(0, last_p0); + } + } + return cost; +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8EncDspCostInitMIPSdspR2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspCostInitMIPSdspR2(void) { + VP8GetResidualCost = GetResidualCost; +} + +#else // !WEBP_USE_MIPS_DSP_R2 + +WEBP_DSP_INIT_STUB(VP8EncDspCostInitMIPSdspR2) + +#endif // WEBP_USE_MIPS_DSP_R2 diff --git a/thirdparty/libwebp/dsp/cost_sse2.c b/thirdparty/libwebp/dsp/cost_sse2.c new file mode 100644 index 0000000000..0cb1c1fa04 --- /dev/null +++ b/thirdparty/libwebp/dsp/cost_sse2.c @@ -0,0 +1,119 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE2 version of cost functions +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE2) +#include <emmintrin.h> + +#include "../enc/cost.h" +#include "../enc/vp8enci.h" +#include "../utils/utils.h" + +//------------------------------------------------------------------------------ + +static void SetResidualCoeffsSSE2(const int16_t* const coeffs, + VP8Residual* const res) { + const __m128i c0 = _mm_loadu_si128((const __m128i*)(coeffs + 0)); + const __m128i c1 = _mm_loadu_si128((const __m128i*)(coeffs + 8)); + // Use SSE2 to compare 16 values with a single instruction. + const __m128i zero = _mm_setzero_si128(); + const __m128i m0 = _mm_packs_epi16(c0, c1); + const __m128i m1 = _mm_cmpeq_epi8(m0, zero); + // Get the comparison results as a bitmask into 16bits. Negate the mask to get + // the position of entries that are not equal to zero. We don't need to mask + // out least significant bits according to res->first, since coeffs[0] is 0 + // if res->first > 0. + const uint32_t mask = 0x0000ffffu ^ (uint32_t)_mm_movemask_epi8(m1); + // The position of the most significant non-zero bit indicates the position of + // the last non-zero value. + assert(res->first == 0 || coeffs[0] == 0); + res->last = mask ? BitsLog2Floor(mask) : -1; + res->coeffs = coeffs; +} + +static int GetResidualCostSSE2(int ctx0, const VP8Residual* const res) { + uint8_t levels[16], ctxs[16]; + uint16_t abs_levels[16]; + int n = res->first; + // should be prob[VP8EncBands[n]], but it's equivalent for n=0 or 1 + const int p0 = res->prob[n][ctx0][0]; + CostArrayPtr const costs = res->costs; + const uint16_t* t = costs[n][ctx0]; + // bit_cost(1, p0) is already incorporated in t[] tables, but only if ctx != 0 + // (as required by the syntax). For ctx0 == 0, we need to add it here or it'll + // be missing during the loop. + int cost = (ctx0 == 0) ? VP8BitCost(1, p0) : 0; + + if (res->last < 0) { + return VP8BitCost(0, p0); + } + + { // precompute clamped levels and contexts, packed to 8b. + const __m128i zero = _mm_setzero_si128(); + const __m128i kCst2 = _mm_set1_epi8(2); + const __m128i kCst67 = _mm_set1_epi8(MAX_VARIABLE_LEVEL); + const __m128i c0 = _mm_loadu_si128((const __m128i*)&res->coeffs[0]); + const __m128i c1 = _mm_loadu_si128((const __m128i*)&res->coeffs[8]); + const __m128i D0 = _mm_sub_epi16(zero, c0); + const __m128i D1 = _mm_sub_epi16(zero, c1); + const __m128i E0 = _mm_max_epi16(c0, D0); // abs(v), 16b + const __m128i E1 = _mm_max_epi16(c1, D1); + const __m128i F = _mm_packs_epi16(E0, E1); + const __m128i G = _mm_min_epu8(F, kCst2); // context = 0,1,2 + const __m128i H = _mm_min_epu8(F, kCst67); // clamp_level in [0..67] + + _mm_storeu_si128((__m128i*)&ctxs[0], G); + _mm_storeu_si128((__m128i*)&levels[0], H); + + _mm_storeu_si128((__m128i*)&abs_levels[0], E0); + _mm_storeu_si128((__m128i*)&abs_levels[8], E1); + } + for (; n < res->last; ++n) { + const int ctx = ctxs[n]; + const int level = levels[n]; + const int flevel = abs_levels[n]; // full level + cost += VP8LevelFixedCosts[flevel] + t[level]; // simplified VP8LevelCost() + t = costs[n + 1][ctx]; + } + // Last coefficient is always non-zero + { + const int level = levels[n]; + const int flevel = abs_levels[n]; + assert(flevel != 0); + cost += VP8LevelFixedCosts[flevel] + t[level]; + if (n < 15) { + const int b = VP8EncBands[n + 1]; + const int ctx = ctxs[n]; + const int last_p0 = res->prob[b][ctx][0]; + cost += VP8BitCost(0, last_p0); + } + } + return cost; +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8EncDspCostInitSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspCostInitSSE2(void) { + VP8SetResidualCoeffs = SetResidualCoeffsSSE2; + VP8GetResidualCost = GetResidualCostSSE2; +} + +#else // !WEBP_USE_SSE2 + +WEBP_DSP_INIT_STUB(VP8EncDspCostInitSSE2) + +#endif // WEBP_USE_SSE2 diff --git a/thirdparty/libwebp/dsp/cpu.c b/thirdparty/libwebp/dsp/cpu.c new file mode 100644 index 0000000000..cbb08db90a --- /dev/null +++ b/thirdparty/libwebp/dsp/cpu.c @@ -0,0 +1,187 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// CPU detection +// +// Author: Christian Duvivier (cduvivier@google.com) + +#include "./dsp.h" + +#if defined(WEBP_HAVE_NEON_RTCD) +#include <stdio.h> +#include <string.h> +#endif + +#if defined(WEBP_ANDROID_NEON) +#include <cpu-features.h> +#endif + +//------------------------------------------------------------------------------ +// SSE2 detection. +// + +// apple/darwin gcc-4.0.1 defines __PIC__, but not __pic__ with -fPIC. +#if (defined(__pic__) || defined(__PIC__)) && defined(__i386__) +static WEBP_INLINE void GetCPUInfo(int cpu_info[4], int info_type) { + __asm__ volatile ( + "mov %%ebx, %%edi\n" + "cpuid\n" + "xchg %%edi, %%ebx\n" + : "=a"(cpu_info[0]), "=D"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3]) + : "a"(info_type), "c"(0)); +} +#elif defined(__x86_64__) && \ + (defined(__code_model_medium__) || defined(__code_model_large__)) && \ + defined(__PIC__) +static WEBP_INLINE void GetCPUInfo(int cpu_info[4], int info_type) { + __asm__ volatile ( + "xchg{q}\t{%%rbx}, %q1\n" + "cpuid\n" + "xchg{q}\t{%%rbx}, %q1\n" + : "=a"(cpu_info[0]), "=&r"(cpu_info[1]), "=c"(cpu_info[2]), + "=d"(cpu_info[3]) + : "a"(info_type), "c"(0)); +} +#elif defined(__i386__) || defined(__x86_64__) +static WEBP_INLINE void GetCPUInfo(int cpu_info[4], int info_type) { + __asm__ volatile ( + "cpuid\n" + : "=a"(cpu_info[0]), "=b"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3]) + : "a"(info_type), "c"(0)); +} +#elif (defined(_M_X64) || defined(_M_IX86)) && \ + defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 150030729 // >= VS2008 SP1 +#include <intrin.h> +#define GetCPUInfo(info, type) __cpuidex(info, type, 0) // set ecx=0 +#elif defined(WEBP_MSC_SSE2) +#define GetCPUInfo __cpuid +#endif + +// NaCl has no support for xgetbv or the raw opcode. +#if !defined(__native_client__) && (defined(__i386__) || defined(__x86_64__)) +static WEBP_INLINE uint64_t xgetbv(void) { + const uint32_t ecx = 0; + uint32_t eax, edx; + // Use the raw opcode for xgetbv for compatibility with older toolchains. + __asm__ volatile ( + ".byte 0x0f, 0x01, 0xd0\n" + : "=a"(eax), "=d"(edx) : "c" (ecx)); + return ((uint64_t)edx << 32) | eax; +} +#elif (defined(_M_X64) || defined(_M_IX86)) && \ + defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 160040219 // >= VS2010 SP1 +#include <immintrin.h> +#define xgetbv() _xgetbv(0) +#elif defined(_MSC_VER) && defined(_M_IX86) +static WEBP_INLINE uint64_t xgetbv(void) { + uint32_t eax_, edx_; + __asm { + xor ecx, ecx // ecx = 0 + // Use the raw opcode for xgetbv for compatibility with older toolchains. + __asm _emit 0x0f __asm _emit 0x01 __asm _emit 0xd0 + mov eax_, eax + mov edx_, edx + } + return ((uint64_t)edx_ << 32) | eax_; +} +#else +#define xgetbv() 0U // no AVX for older x64 or unrecognized toolchains. +#endif + +#if defined(__i386__) || defined(__x86_64__) || defined(WEBP_MSC_SSE2) +static int x86CPUInfo(CPUFeature feature) { + int max_cpuid_value; + int cpu_info[4]; + + // get the highest feature value cpuid supports + GetCPUInfo(cpu_info, 0); + max_cpuid_value = cpu_info[0]; + if (max_cpuid_value < 1) { + return 0; + } + + GetCPUInfo(cpu_info, 1); + if (feature == kSSE2) { + return 0 != (cpu_info[3] & 0x04000000); + } + if (feature == kSSE3) { + return 0 != (cpu_info[2] & 0x00000001); + } + if (feature == kSSE4_1) { + return 0 != (cpu_info[2] & 0x00080000); + } + if (feature == kAVX) { + // bits 27 (OSXSAVE) & 28 (256-bit AVX) + if ((cpu_info[2] & 0x18000000) == 0x18000000) { + // XMM state and YMM state enabled by the OS. + return (xgetbv() & 0x6) == 0x6; + } + } + if (feature == kAVX2) { + if (x86CPUInfo(kAVX) && max_cpuid_value >= 7) { + GetCPUInfo(cpu_info, 7); + return ((cpu_info[1] & 0x00000020) == 0x00000020); + } + } + return 0; +} +VP8CPUInfo VP8GetCPUInfo = x86CPUInfo; +#elif defined(WEBP_ANDROID_NEON) // NB: needs to be before generic NEON test. +static int AndroidCPUInfo(CPUFeature feature) { + const AndroidCpuFamily cpu_family = android_getCpuFamily(); + const uint64_t cpu_features = android_getCpuFeatures(); + if (feature == kNEON) { + return (cpu_family == ANDROID_CPU_FAMILY_ARM && + 0 != (cpu_features & ANDROID_CPU_ARM_FEATURE_NEON)); + } + return 0; +} +VP8CPUInfo VP8GetCPUInfo = AndroidCPUInfo; +#elif defined(WEBP_USE_NEON) +// define a dummy function to enable turning off NEON at runtime by setting +// VP8DecGetCPUInfo = NULL +static int armCPUInfo(CPUFeature feature) { + if (feature != kNEON) return 0; +#if defined(__linux__) && defined(WEBP_HAVE_NEON_RTCD) + { + int has_neon = 0; + char line[200]; + FILE* const cpuinfo = fopen("/proc/cpuinfo", "r"); + if (cpuinfo == NULL) return 0; + while (fgets(line, sizeof(line), cpuinfo)) { + if (!strncmp(line, "Features", 8)) { + if (strstr(line, " neon ") != NULL) { + has_neon = 1; + break; + } + } + } + fclose(cpuinfo); + return has_neon; + } +#else + return 1; +#endif +} +VP8CPUInfo VP8GetCPUInfo = armCPUInfo; +#elif defined(WEBP_USE_MIPS32) || defined(WEBP_USE_MIPS_DSP_R2) || \ + defined(WEBP_USE_MSA) +static int mipsCPUInfo(CPUFeature feature) { + if ((feature == kMIPS32) || (feature == kMIPSdspR2) || (feature == kMSA)) { + return 1; + } else { + return 0; + } + +} +VP8CPUInfo VP8GetCPUInfo = mipsCPUInfo; +#else +VP8CPUInfo VP8GetCPUInfo = NULL; +#endif + diff --git a/thirdparty/libwebp/dsp/dec.c b/thirdparty/libwebp/dsp/dec.c new file mode 100644 index 0000000000..e92d693362 --- /dev/null +++ b/thirdparty/libwebp/dsp/dec.c @@ -0,0 +1,795 @@ +// Copyright 2010 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Speed-critical decoding functions, default plain-C implementations. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" +#include "../dec/vp8i.h" +#include "../utils/utils.h" + +//------------------------------------------------------------------------------ + +static WEBP_INLINE uint8_t clip_8b(int v) { + return (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255; +} + +//------------------------------------------------------------------------------ +// Transforms (Paragraph 14.4) + +#define STORE(x, y, v) \ + dst[x + y * BPS] = clip_8b(dst[x + y * BPS] + ((v) >> 3)) + +#define STORE2(y, dc, d, c) do { \ + const int DC = (dc); \ + STORE(0, y, DC + (d)); \ + STORE(1, y, DC + (c)); \ + STORE(2, y, DC - (c)); \ + STORE(3, y, DC - (d)); \ +} while (0) + +#define MUL1(a) ((((a) * 20091) >> 16) + (a)) +#define MUL2(a) (((a) * 35468) >> 16) + +static void TransformOne(const int16_t* in, uint8_t* dst) { + int C[4 * 4], *tmp; + int i; + tmp = C; + for (i = 0; i < 4; ++i) { // vertical pass + const int a = in[0] + in[8]; // [-4096, 4094] + const int b = in[0] - in[8]; // [-4095, 4095] + const int c = MUL2(in[4]) - MUL1(in[12]); // [-3783, 3783] + const int d = MUL1(in[4]) + MUL2(in[12]); // [-3785, 3781] + tmp[0] = a + d; // [-7881, 7875] + tmp[1] = b + c; // [-7878, 7878] + tmp[2] = b - c; // [-7878, 7878] + tmp[3] = a - d; // [-7877, 7879] + tmp += 4; + in++; + } + // Each pass is expanding the dynamic range by ~3.85 (upper bound). + // The exact value is (2. + (20091 + 35468) / 65536). + // After the second pass, maximum interval is [-3794, 3794], assuming + // an input in [-2048, 2047] interval. We then need to add a dst value + // in the [0, 255] range. + // In the worst case scenario, the input to clip_8b() can be as large as + // [-60713, 60968]. + tmp = C; + for (i = 0; i < 4; ++i) { // horizontal pass + const int dc = tmp[0] + 4; + const int a = dc + tmp[8]; + const int b = dc - tmp[8]; + const int c = MUL2(tmp[4]) - MUL1(tmp[12]); + const int d = MUL1(tmp[4]) + MUL2(tmp[12]); + STORE(0, 0, a + d); + STORE(1, 0, b + c); + STORE(2, 0, b - c); + STORE(3, 0, a - d); + tmp++; + dst += BPS; + } +} + +// Simplified transform when only in[0], in[1] and in[4] are non-zero +static void TransformAC3(const int16_t* in, uint8_t* dst) { + const int a = in[0] + 4; + const int c4 = MUL2(in[4]); + const int d4 = MUL1(in[4]); + const int c1 = MUL2(in[1]); + const int d1 = MUL1(in[1]); + STORE2(0, a + d4, d1, c1); + STORE2(1, a + c4, d1, c1); + STORE2(2, a - c4, d1, c1); + STORE2(3, a - d4, d1, c1); +} +#undef MUL1 +#undef MUL2 +#undef STORE2 + +static void TransformTwo(const int16_t* in, uint8_t* dst, int do_two) { + TransformOne(in, dst); + if (do_two) { + TransformOne(in + 16, dst + 4); + } +} + +static void TransformUV(const int16_t* in, uint8_t* dst) { + VP8Transform(in + 0 * 16, dst, 1); + VP8Transform(in + 2 * 16, dst + 4 * BPS, 1); +} + +static void TransformDC(const int16_t* in, uint8_t* dst) { + const int DC = in[0] + 4; + int i, j; + for (j = 0; j < 4; ++j) { + for (i = 0; i < 4; ++i) { + STORE(i, j, DC); + } + } +} + +static void TransformDCUV(const int16_t* in, uint8_t* dst) { + if (in[0 * 16]) VP8TransformDC(in + 0 * 16, dst); + if (in[1 * 16]) VP8TransformDC(in + 1 * 16, dst + 4); + if (in[2 * 16]) VP8TransformDC(in + 2 * 16, dst + 4 * BPS); + if (in[3 * 16]) VP8TransformDC(in + 3 * 16, dst + 4 * BPS + 4); +} + +#undef STORE + +//------------------------------------------------------------------------------ +// Paragraph 14.3 + +static void TransformWHT(const int16_t* in, int16_t* out) { + int tmp[16]; + int i; + for (i = 0; i < 4; ++i) { + const int a0 = in[0 + i] + in[12 + i]; + const int a1 = in[4 + i] + in[ 8 + i]; + const int a2 = in[4 + i] - in[ 8 + i]; + const int a3 = in[0 + i] - in[12 + i]; + tmp[0 + i] = a0 + a1; + tmp[8 + i] = a0 - a1; + tmp[4 + i] = a3 + a2; + tmp[12 + i] = a3 - a2; + } + for (i = 0; i < 4; ++i) { + const int dc = tmp[0 + i * 4] + 3; // w/ rounder + const int a0 = dc + tmp[3 + i * 4]; + const int a1 = tmp[1 + i * 4] + tmp[2 + i * 4]; + const int a2 = tmp[1 + i * 4] - tmp[2 + i * 4]; + const int a3 = dc - tmp[3 + i * 4]; + out[ 0] = (a0 + a1) >> 3; + out[16] = (a3 + a2) >> 3; + out[32] = (a0 - a1) >> 3; + out[48] = (a3 - a2) >> 3; + out += 64; + } +} + +void (*VP8TransformWHT)(const int16_t* in, int16_t* out); + +//------------------------------------------------------------------------------ +// Intra predictions + +#define DST(x, y) dst[(x) + (y) * BPS] + +static WEBP_INLINE void TrueMotion(uint8_t* dst, int size) { + const uint8_t* top = dst - BPS; + const uint8_t* const clip0 = VP8kclip1 - top[-1]; + int y; + for (y = 0; y < size; ++y) { + const uint8_t* const clip = clip0 + dst[-1]; + int x; + for (x = 0; x < size; ++x) { + dst[x] = clip[top[x]]; + } + dst += BPS; + } +} +static void TM4(uint8_t* dst) { TrueMotion(dst, 4); } +static void TM8uv(uint8_t* dst) { TrueMotion(dst, 8); } +static void TM16(uint8_t* dst) { TrueMotion(dst, 16); } + +//------------------------------------------------------------------------------ +// 16x16 + +static void VE16(uint8_t* dst) { // vertical + int j; + for (j = 0; j < 16; ++j) { + memcpy(dst + j * BPS, dst - BPS, 16); + } +} + +static void HE16(uint8_t* dst) { // horizontal + int j; + for (j = 16; j > 0; --j) { + memset(dst, dst[-1], 16); + dst += BPS; + } +} + +static WEBP_INLINE void Put16(int v, uint8_t* dst) { + int j; + for (j = 0; j < 16; ++j) { + memset(dst + j * BPS, v, 16); + } +} + +static void DC16(uint8_t* dst) { // DC + int DC = 16; + int j; + for (j = 0; j < 16; ++j) { + DC += dst[-1 + j * BPS] + dst[j - BPS]; + } + Put16(DC >> 5, dst); +} + +static void DC16NoTop(uint8_t* dst) { // DC with top samples not available + int DC = 8; + int j; + for (j = 0; j < 16; ++j) { + DC += dst[-1 + j * BPS]; + } + Put16(DC >> 4, dst); +} + +static void DC16NoLeft(uint8_t* dst) { // DC with left samples not available + int DC = 8; + int i; + for (i = 0; i < 16; ++i) { + DC += dst[i - BPS]; + } + Put16(DC >> 4, dst); +} + +static void DC16NoTopLeft(uint8_t* dst) { // DC with no top and left samples + Put16(0x80, dst); +} + +VP8PredFunc VP8PredLuma16[NUM_B_DC_MODES]; + +//------------------------------------------------------------------------------ +// 4x4 + +#define AVG3(a, b, c) (((a) + 2 * (b) + (c) + 2) >> 2) +#define AVG2(a, b) (((a) + (b) + 1) >> 1) + +static void VE4(uint8_t* dst) { // vertical + const uint8_t* top = dst - BPS; + const uint8_t vals[4] = { + AVG3(top[-1], top[0], top[1]), + AVG3(top[ 0], top[1], top[2]), + AVG3(top[ 1], top[2], top[3]), + AVG3(top[ 2], top[3], top[4]) + }; + int i; + for (i = 0; i < 4; ++i) { + memcpy(dst + i * BPS, vals, sizeof(vals)); + } +} + +static void HE4(uint8_t* dst) { // horizontal + const int A = dst[-1 - BPS]; + const int B = dst[-1]; + const int C = dst[-1 + BPS]; + const int D = dst[-1 + 2 * BPS]; + const int E = dst[-1 + 3 * BPS]; + WebPUint32ToMem(dst + 0 * BPS, 0x01010101U * AVG3(A, B, C)); + WebPUint32ToMem(dst + 1 * BPS, 0x01010101U * AVG3(B, C, D)); + WebPUint32ToMem(dst + 2 * BPS, 0x01010101U * AVG3(C, D, E)); + WebPUint32ToMem(dst + 3 * BPS, 0x01010101U * AVG3(D, E, E)); +} + +static void DC4(uint8_t* dst) { // DC + uint32_t dc = 4; + int i; + for (i = 0; i < 4; ++i) dc += dst[i - BPS] + dst[-1 + i * BPS]; + dc >>= 3; + for (i = 0; i < 4; ++i) memset(dst + i * BPS, dc, 4); +} + +static void RD4(uint8_t* dst) { // Down-right + const int I = dst[-1 + 0 * BPS]; + const int J = dst[-1 + 1 * BPS]; + const int K = dst[-1 + 2 * BPS]; + const int L = dst[-1 + 3 * BPS]; + const int X = dst[-1 - BPS]; + const int A = dst[0 - BPS]; + const int B = dst[1 - BPS]; + const int C = dst[2 - BPS]; + const int D = dst[3 - BPS]; + DST(0, 3) = AVG3(J, K, L); + DST(1, 3) = DST(0, 2) = AVG3(I, J, K); + DST(2, 3) = DST(1, 2) = DST(0, 1) = AVG3(X, I, J); + DST(3, 3) = DST(2, 2) = DST(1, 1) = DST(0, 0) = AVG3(A, X, I); + DST(3, 2) = DST(2, 1) = DST(1, 0) = AVG3(B, A, X); + DST(3, 1) = DST(2, 0) = AVG3(C, B, A); + DST(3, 0) = AVG3(D, C, B); +} + +static void LD4(uint8_t* dst) { // Down-Left + const int A = dst[0 - BPS]; + const int B = dst[1 - BPS]; + const int C = dst[2 - BPS]; + const int D = dst[3 - BPS]; + const int E = dst[4 - BPS]; + const int F = dst[5 - BPS]; + const int G = dst[6 - BPS]; + const int H = dst[7 - BPS]; + DST(0, 0) = AVG3(A, B, C); + DST(1, 0) = DST(0, 1) = AVG3(B, C, D); + DST(2, 0) = DST(1, 1) = DST(0, 2) = AVG3(C, D, E); + DST(3, 0) = DST(2, 1) = DST(1, 2) = DST(0, 3) = AVG3(D, E, F); + DST(3, 1) = DST(2, 2) = DST(1, 3) = AVG3(E, F, G); + DST(3, 2) = DST(2, 3) = AVG3(F, G, H); + DST(3, 3) = AVG3(G, H, H); +} + +static void VR4(uint8_t* dst) { // Vertical-Right + const int I = dst[-1 + 0 * BPS]; + const int J = dst[-1 + 1 * BPS]; + const int K = dst[-1 + 2 * BPS]; + const int X = dst[-1 - BPS]; + const int A = dst[0 - BPS]; + const int B = dst[1 - BPS]; + const int C = dst[2 - BPS]; + const int D = dst[3 - BPS]; + DST(0, 0) = DST(1, 2) = AVG2(X, A); + DST(1, 0) = DST(2, 2) = AVG2(A, B); + DST(2, 0) = DST(3, 2) = AVG2(B, C); + DST(3, 0) = AVG2(C, D); + + DST(0, 3) = AVG3(K, J, I); + DST(0, 2) = AVG3(J, I, X); + DST(0, 1) = DST(1, 3) = AVG3(I, X, A); + DST(1, 1) = DST(2, 3) = AVG3(X, A, B); + DST(2, 1) = DST(3, 3) = AVG3(A, B, C); + DST(3, 1) = AVG3(B, C, D); +} + +static void VL4(uint8_t* dst) { // Vertical-Left + const int A = dst[0 - BPS]; + const int B = dst[1 - BPS]; + const int C = dst[2 - BPS]; + const int D = dst[3 - BPS]; + const int E = dst[4 - BPS]; + const int F = dst[5 - BPS]; + const int G = dst[6 - BPS]; + const int H = dst[7 - BPS]; + DST(0, 0) = AVG2(A, B); + DST(1, 0) = DST(0, 2) = AVG2(B, C); + DST(2, 0) = DST(1, 2) = AVG2(C, D); + DST(3, 0) = DST(2, 2) = AVG2(D, E); + + DST(0, 1) = AVG3(A, B, C); + DST(1, 1) = DST(0, 3) = AVG3(B, C, D); + DST(2, 1) = DST(1, 3) = AVG3(C, D, E); + DST(3, 1) = DST(2, 3) = AVG3(D, E, F); + DST(3, 2) = AVG3(E, F, G); + DST(3, 3) = AVG3(F, G, H); +} + +static void HU4(uint8_t* dst) { // Horizontal-Up + const int I = dst[-1 + 0 * BPS]; + const int J = dst[-1 + 1 * BPS]; + const int K = dst[-1 + 2 * BPS]; + const int L = dst[-1 + 3 * BPS]; + DST(0, 0) = AVG2(I, J); + DST(2, 0) = DST(0, 1) = AVG2(J, K); + DST(2, 1) = DST(0, 2) = AVG2(K, L); + DST(1, 0) = AVG3(I, J, K); + DST(3, 0) = DST(1, 1) = AVG3(J, K, L); + DST(3, 1) = DST(1, 2) = AVG3(K, L, L); + DST(3, 2) = DST(2, 2) = + DST(0, 3) = DST(1, 3) = DST(2, 3) = DST(3, 3) = L; +} + +static void HD4(uint8_t* dst) { // Horizontal-Down + const int I = dst[-1 + 0 * BPS]; + const int J = dst[-1 + 1 * BPS]; + const int K = dst[-1 + 2 * BPS]; + const int L = dst[-1 + 3 * BPS]; + const int X = dst[-1 - BPS]; + const int A = dst[0 - BPS]; + const int B = dst[1 - BPS]; + const int C = dst[2 - BPS]; + + DST(0, 0) = DST(2, 1) = AVG2(I, X); + DST(0, 1) = DST(2, 2) = AVG2(J, I); + DST(0, 2) = DST(2, 3) = AVG2(K, J); + DST(0, 3) = AVG2(L, K); + + DST(3, 0) = AVG3(A, B, C); + DST(2, 0) = AVG3(X, A, B); + DST(1, 0) = DST(3, 1) = AVG3(I, X, A); + DST(1, 1) = DST(3, 2) = AVG3(J, I, X); + DST(1, 2) = DST(3, 3) = AVG3(K, J, I); + DST(1, 3) = AVG3(L, K, J); +} + +#undef DST +#undef AVG3 +#undef AVG2 + +VP8PredFunc VP8PredLuma4[NUM_BMODES]; + +//------------------------------------------------------------------------------ +// Chroma + +static void VE8uv(uint8_t* dst) { // vertical + int j; + for (j = 0; j < 8; ++j) { + memcpy(dst + j * BPS, dst - BPS, 8); + } +} + +static void HE8uv(uint8_t* dst) { // horizontal + int j; + for (j = 0; j < 8; ++j) { + memset(dst, dst[-1], 8); + dst += BPS; + } +} + +// helper for chroma-DC predictions +static WEBP_INLINE void Put8x8uv(uint8_t value, uint8_t* dst) { + int j; + for (j = 0; j < 8; ++j) { + memset(dst + j * BPS, value, 8); + } +} + +static void DC8uv(uint8_t* dst) { // DC + int dc0 = 8; + int i; + for (i = 0; i < 8; ++i) { + dc0 += dst[i - BPS] + dst[-1 + i * BPS]; + } + Put8x8uv(dc0 >> 4, dst); +} + +static void DC8uvNoLeft(uint8_t* dst) { // DC with no left samples + int dc0 = 4; + int i; + for (i = 0; i < 8; ++i) { + dc0 += dst[i - BPS]; + } + Put8x8uv(dc0 >> 3, dst); +} + +static void DC8uvNoTop(uint8_t* dst) { // DC with no top samples + int dc0 = 4; + int i; + for (i = 0; i < 8; ++i) { + dc0 += dst[-1 + i * BPS]; + } + Put8x8uv(dc0 >> 3, dst); +} + +static void DC8uvNoTopLeft(uint8_t* dst) { // DC with nothing + Put8x8uv(0x80, dst); +} + +VP8PredFunc VP8PredChroma8[NUM_B_DC_MODES]; + +//------------------------------------------------------------------------------ +// Edge filtering functions + +// 4 pixels in, 2 pixels out +static WEBP_INLINE void do_filter2(uint8_t* p, int step) { + const int p1 = p[-2*step], p0 = p[-step], q0 = p[0], q1 = p[step]; + const int a = 3 * (q0 - p0) + VP8ksclip1[p1 - q1]; // in [-893,892] + const int a1 = VP8ksclip2[(a + 4) >> 3]; // in [-16,15] + const int a2 = VP8ksclip2[(a + 3) >> 3]; + p[-step] = VP8kclip1[p0 + a2]; + p[ 0] = VP8kclip1[q0 - a1]; +} + +// 4 pixels in, 4 pixels out +static WEBP_INLINE void do_filter4(uint8_t* p, int step) { + const int p1 = p[-2*step], p0 = p[-step], q0 = p[0], q1 = p[step]; + const int a = 3 * (q0 - p0); + const int a1 = VP8ksclip2[(a + 4) >> 3]; + const int a2 = VP8ksclip2[(a + 3) >> 3]; + const int a3 = (a1 + 1) >> 1; + p[-2*step] = VP8kclip1[p1 + a3]; + p[- step] = VP8kclip1[p0 + a2]; + p[ 0] = VP8kclip1[q0 - a1]; + p[ step] = VP8kclip1[q1 - a3]; +} + +// 6 pixels in, 6 pixels out +static WEBP_INLINE void do_filter6(uint8_t* p, int step) { + const int p2 = p[-3*step], p1 = p[-2*step], p0 = p[-step]; + const int q0 = p[0], q1 = p[step], q2 = p[2*step]; + const int a = VP8ksclip1[3 * (q0 - p0) + VP8ksclip1[p1 - q1]]; + // a is in [-128,127], a1 in [-27,27], a2 in [-18,18] and a3 in [-9,9] + const int a1 = (27 * a + 63) >> 7; // eq. to ((3 * a + 7) * 9) >> 7 + const int a2 = (18 * a + 63) >> 7; // eq. to ((2 * a + 7) * 9) >> 7 + const int a3 = (9 * a + 63) >> 7; // eq. to ((1 * a + 7) * 9) >> 7 + p[-3*step] = VP8kclip1[p2 + a3]; + p[-2*step] = VP8kclip1[p1 + a2]; + p[- step] = VP8kclip1[p0 + a1]; + p[ 0] = VP8kclip1[q0 - a1]; + p[ step] = VP8kclip1[q1 - a2]; + p[ 2*step] = VP8kclip1[q2 - a3]; +} + +static WEBP_INLINE int hev(const uint8_t* p, int step, int thresh) { + const int p1 = p[-2*step], p0 = p[-step], q0 = p[0], q1 = p[step]; + return (VP8kabs0[p1 - p0] > thresh) || (VP8kabs0[q1 - q0] > thresh); +} + +static WEBP_INLINE int needs_filter(const uint8_t* p, int step, int t) { + const int p1 = p[-2 * step], p0 = p[-step], q0 = p[0], q1 = p[step]; + return ((4 * VP8kabs0[p0 - q0] + VP8kabs0[p1 - q1]) <= t); +} + +static WEBP_INLINE int needs_filter2(const uint8_t* p, + int step, int t, int it) { + const int p3 = p[-4 * step], p2 = p[-3 * step], p1 = p[-2 * step]; + const int p0 = p[-step], q0 = p[0]; + const int q1 = p[step], q2 = p[2 * step], q3 = p[3 * step]; + if ((4 * VP8kabs0[p0 - q0] + VP8kabs0[p1 - q1]) > t) return 0; + return VP8kabs0[p3 - p2] <= it && VP8kabs0[p2 - p1] <= it && + VP8kabs0[p1 - p0] <= it && VP8kabs0[q3 - q2] <= it && + VP8kabs0[q2 - q1] <= it && VP8kabs0[q1 - q0] <= it; +} + +//------------------------------------------------------------------------------ +// Simple In-loop filtering (Paragraph 15.2) + +static void SimpleVFilter16(uint8_t* p, int stride, int thresh) { + int i; + const int thresh2 = 2 * thresh + 1; + for (i = 0; i < 16; ++i) { + if (needs_filter(p + i, stride, thresh2)) { + do_filter2(p + i, stride); + } + } +} + +static void SimpleHFilter16(uint8_t* p, int stride, int thresh) { + int i; + const int thresh2 = 2 * thresh + 1; + for (i = 0; i < 16; ++i) { + if (needs_filter(p + i * stride, 1, thresh2)) { + do_filter2(p + i * stride, 1); + } + } +} + +static void SimpleVFilter16i(uint8_t* p, int stride, int thresh) { + int k; + for (k = 3; k > 0; --k) { + p += 4 * stride; + SimpleVFilter16(p, stride, thresh); + } +} + +static void SimpleHFilter16i(uint8_t* p, int stride, int thresh) { + int k; + for (k = 3; k > 0; --k) { + p += 4; + SimpleHFilter16(p, stride, thresh); + } +} + +//------------------------------------------------------------------------------ +// Complex In-loop filtering (Paragraph 15.3) + +static WEBP_INLINE void FilterLoop26(uint8_t* p, + int hstride, int vstride, int size, + int thresh, int ithresh, int hev_thresh) { + const int thresh2 = 2 * thresh + 1; + while (size-- > 0) { + if (needs_filter2(p, hstride, thresh2, ithresh)) { + if (hev(p, hstride, hev_thresh)) { + do_filter2(p, hstride); + } else { + do_filter6(p, hstride); + } + } + p += vstride; + } +} + +static WEBP_INLINE void FilterLoop24(uint8_t* p, + int hstride, int vstride, int size, + int thresh, int ithresh, int hev_thresh) { + const int thresh2 = 2 * thresh + 1; + while (size-- > 0) { + if (needs_filter2(p, hstride, thresh2, ithresh)) { + if (hev(p, hstride, hev_thresh)) { + do_filter2(p, hstride); + } else { + do_filter4(p, hstride); + } + } + p += vstride; + } +} + +// on macroblock edges +static void VFilter16(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop26(p, stride, 1, 16, thresh, ithresh, hev_thresh); +} + +static void HFilter16(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop26(p, 1, stride, 16, thresh, ithresh, hev_thresh); +} + +// on three inner edges +static void VFilter16i(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + int k; + for (k = 3; k > 0; --k) { + p += 4 * stride; + FilterLoop24(p, stride, 1, 16, thresh, ithresh, hev_thresh); + } +} + +static void HFilter16i(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + int k; + for (k = 3; k > 0; --k) { + p += 4; + FilterLoop24(p, 1, stride, 16, thresh, ithresh, hev_thresh); + } +} + +// 8-pixels wide variant, for chroma filtering +static void VFilter8(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop26(u, stride, 1, 8, thresh, ithresh, hev_thresh); + FilterLoop26(v, stride, 1, 8, thresh, ithresh, hev_thresh); +} + +static void HFilter8(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop26(u, 1, stride, 8, thresh, ithresh, hev_thresh); + FilterLoop26(v, 1, stride, 8, thresh, ithresh, hev_thresh); +} + +static void VFilter8i(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop24(u + 4 * stride, stride, 1, 8, thresh, ithresh, hev_thresh); + FilterLoop24(v + 4 * stride, stride, 1, 8, thresh, ithresh, hev_thresh); +} + +static void HFilter8i(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop24(u + 4, 1, stride, 8, thresh, ithresh, hev_thresh); + FilterLoop24(v + 4, 1, stride, 8, thresh, ithresh, hev_thresh); +} + +//------------------------------------------------------------------------------ + +static void DitherCombine8x8(const uint8_t* dither, uint8_t* dst, + int dst_stride) { + int i, j; + for (j = 0; j < 8; ++j) { + for (i = 0; i < 8; ++i) { + const int delta0 = dither[i] - VP8_DITHER_AMP_CENTER; + const int delta1 = + (delta0 + VP8_DITHER_DESCALE_ROUNDER) >> VP8_DITHER_DESCALE; + dst[i] = clip_8b((int)dst[i] + delta1); + } + dst += dst_stride; + dither += 8; + } +} + +//------------------------------------------------------------------------------ + +VP8DecIdct2 VP8Transform; +VP8DecIdct VP8TransformAC3; +VP8DecIdct VP8TransformUV; +VP8DecIdct VP8TransformDC; +VP8DecIdct VP8TransformDCUV; + +VP8LumaFilterFunc VP8VFilter16; +VP8LumaFilterFunc VP8HFilter16; +VP8ChromaFilterFunc VP8VFilter8; +VP8ChromaFilterFunc VP8HFilter8; +VP8LumaFilterFunc VP8VFilter16i; +VP8LumaFilterFunc VP8HFilter16i; +VP8ChromaFilterFunc VP8VFilter8i; +VP8ChromaFilterFunc VP8HFilter8i; +VP8SimpleFilterFunc VP8SimpleVFilter16; +VP8SimpleFilterFunc VP8SimpleHFilter16; +VP8SimpleFilterFunc VP8SimpleVFilter16i; +VP8SimpleFilterFunc VP8SimpleHFilter16i; + +void (*VP8DitherCombine8x8)(const uint8_t* dither, uint8_t* dst, + int dst_stride); + +extern void VP8DspInitSSE2(void); +extern void VP8DspInitSSE41(void); +extern void VP8DspInitNEON(void); +extern void VP8DspInitMIPS32(void); +extern void VP8DspInitMIPSdspR2(void); +extern void VP8DspInitMSA(void); + +static volatile VP8CPUInfo dec_last_cpuinfo_used = + (VP8CPUInfo)&dec_last_cpuinfo_used; + +WEBP_TSAN_IGNORE_FUNCTION void VP8DspInit(void) { + if (dec_last_cpuinfo_used == VP8GetCPUInfo) return; + + VP8InitClipTables(); + + VP8TransformWHT = TransformWHT; + VP8Transform = TransformTwo; + VP8TransformUV = TransformUV; + VP8TransformDC = TransformDC; + VP8TransformDCUV = TransformDCUV; + VP8TransformAC3 = TransformAC3; + + VP8VFilter16 = VFilter16; + VP8HFilter16 = HFilter16; + VP8VFilter8 = VFilter8; + VP8HFilter8 = HFilter8; + VP8VFilter16i = VFilter16i; + VP8HFilter16i = HFilter16i; + VP8VFilter8i = VFilter8i; + VP8HFilter8i = HFilter8i; + VP8SimpleVFilter16 = SimpleVFilter16; + VP8SimpleHFilter16 = SimpleHFilter16; + VP8SimpleVFilter16i = SimpleVFilter16i; + VP8SimpleHFilter16i = SimpleHFilter16i; + + VP8PredLuma4[0] = DC4; + VP8PredLuma4[1] = TM4; + VP8PredLuma4[2] = VE4; + VP8PredLuma4[3] = HE4; + VP8PredLuma4[4] = RD4; + VP8PredLuma4[5] = VR4; + VP8PredLuma4[6] = LD4; + VP8PredLuma4[7] = VL4; + VP8PredLuma4[8] = HD4; + VP8PredLuma4[9] = HU4; + + VP8PredLuma16[0] = DC16; + VP8PredLuma16[1] = TM16; + VP8PredLuma16[2] = VE16; + VP8PredLuma16[3] = HE16; + VP8PredLuma16[4] = DC16NoTop; + VP8PredLuma16[5] = DC16NoLeft; + VP8PredLuma16[6] = DC16NoTopLeft; + + VP8PredChroma8[0] = DC8uv; + VP8PredChroma8[1] = TM8uv; + VP8PredChroma8[2] = VE8uv; + VP8PredChroma8[3] = HE8uv; + VP8PredChroma8[4] = DC8uvNoTop; + VP8PredChroma8[5] = DC8uvNoLeft; + VP8PredChroma8[6] = DC8uvNoTopLeft; + + VP8DitherCombine8x8 = DitherCombine8x8; + + // If defined, use CPUInfo() to overwrite some pointers with faster versions. + if (VP8GetCPUInfo != NULL) { +#if defined(WEBP_USE_SSE2) + if (VP8GetCPUInfo(kSSE2)) { + VP8DspInitSSE2(); +#if defined(WEBP_USE_SSE41) + if (VP8GetCPUInfo(kSSE4_1)) { + VP8DspInitSSE41(); + } +#endif + } +#endif +#if defined(WEBP_USE_NEON) + if (VP8GetCPUInfo(kNEON)) { + VP8DspInitNEON(); + } +#endif +#if defined(WEBP_USE_MIPS32) + if (VP8GetCPUInfo(kMIPS32)) { + VP8DspInitMIPS32(); + } +#endif +#if defined(WEBP_USE_MIPS_DSP_R2) + if (VP8GetCPUInfo(kMIPSdspR2)) { + VP8DspInitMIPSdspR2(); + } +#endif +#if defined(WEBP_USE_MSA) + if (VP8GetCPUInfo(kMSA)) { + VP8DspInitMSA(); + } +#endif + } + dec_last_cpuinfo_used = VP8GetCPUInfo; +} diff --git a/thirdparty/libwebp/dsp/dec_clip_tables.c b/thirdparty/libwebp/dsp/dec_clip_tables.c new file mode 100644 index 0000000000..3b6dde86ba --- /dev/null +++ b/thirdparty/libwebp/dsp/dec_clip_tables.c @@ -0,0 +1,366 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Clipping tables for filtering +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#define USE_STATIC_TABLES // undefine to have run-time table initialization + +#ifdef USE_STATIC_TABLES + +static const uint8_t abs0[255 + 255 + 1] = { + 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, 0xf7, 0xf6, 0xf5, 0xf4, + 0xf3, 0xf2, 0xf1, 0xf0, 0xef, 0xee, 0xed, 0xec, 0xeb, 0xea, 0xe9, 0xe8, + 0xe7, 0xe6, 0xe5, 0xe4, 0xe3, 0xe2, 0xe1, 0xe0, 0xdf, 0xde, 0xdd, 0xdc, + 0xdb, 0xda, 0xd9, 0xd8, 0xd7, 0xd6, 0xd5, 0xd4, 0xd3, 0xd2, 0xd1, 0xd0, + 0xcf, 0xce, 0xcd, 0xcc, 0xcb, 0xca, 0xc9, 0xc8, 0xc7, 0xc6, 0xc5, 0xc4, + 0xc3, 0xc2, 0xc1, 0xc0, 0xbf, 0xbe, 0xbd, 0xbc, 0xbb, 0xba, 0xb9, 0xb8, + 0xb7, 0xb6, 0xb5, 0xb4, 0xb3, 0xb2, 0xb1, 0xb0, 0xaf, 0xae, 0xad, 0xac, + 0xab, 0xaa, 0xa9, 0xa8, 0xa7, 0xa6, 0xa5, 0xa4, 0xa3, 0xa2, 0xa1, 0xa0, + 0x9f, 0x9e, 0x9d, 0x9c, 0x9b, 0x9a, 0x99, 0x98, 0x97, 0x96, 0x95, 0x94, + 0x93, 0x92, 0x91, 0x90, 0x8f, 0x8e, 0x8d, 0x8c, 0x8b, 0x8a, 0x89, 0x88, + 0x87, 0x86, 0x85, 0x84, 0x83, 0x82, 0x81, 0x80, 0x7f, 0x7e, 0x7d, 0x7c, + 0x7b, 0x7a, 0x79, 0x78, 0x77, 0x76, 0x75, 0x74, 0x73, 0x72, 0x71, 0x70, + 0x6f, 0x6e, 0x6d, 0x6c, 0x6b, 0x6a, 0x69, 0x68, 0x67, 0x66, 0x65, 0x64, + 0x63, 0x62, 0x61, 0x60, 0x5f, 0x5e, 0x5d, 0x5c, 0x5b, 0x5a, 0x59, 0x58, + 0x57, 0x56, 0x55, 0x54, 0x53, 0x52, 0x51, 0x50, 0x4f, 0x4e, 0x4d, 0x4c, + 0x4b, 0x4a, 0x49, 0x48, 0x47, 0x46, 0x45, 0x44, 0x43, 0x42, 0x41, 0x40, + 0x3f, 0x3e, 0x3d, 0x3c, 0x3b, 0x3a, 0x39, 0x38, 0x37, 0x36, 0x35, 0x34, + 0x33, 0x32, 0x31, 0x30, 0x2f, 0x2e, 0x2d, 0x2c, 0x2b, 0x2a, 0x29, 0x28, + 0x27, 0x26, 0x25, 0x24, 0x23, 0x22, 0x21, 0x20, 0x1f, 0x1e, 0x1d, 0x1c, + 0x1b, 0x1a, 0x19, 0x18, 0x17, 0x16, 0x15, 0x14, 0x13, 0x12, 0x11, 0x10, + 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08, 0x07, 0x06, 0x05, 0x04, + 0x03, 0x02, 0x01, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, + 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, + 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, + 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, + 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, + 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, + 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, + 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, + 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, + 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, + 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, + 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, + 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, + 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, + 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, + 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, + 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, + 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, + 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff +}; + +static const int8_t sclip1[1020 + 1020 + 1] = { + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, + 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, + 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, + 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, + 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, + 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, + 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, + 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, + 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f +}; + +static const int8_t sclip2[112 + 112 + 1] = { + 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, + 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, + 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, + 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, + 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, + 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, + 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, + 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, + 0xfc, 0xfd, 0xfe, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f +}; + +static const uint8_t clip1[255 + 511 + 1] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, + 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, + 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, + 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, + 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, + 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, + 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, + 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, + 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, + 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, + 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, + 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, + 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, + 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, + 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, + 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, + 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, + 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, + 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; + +#else + +// uninitialized tables +static uint8_t abs0[255 + 255 + 1]; +static int8_t sclip1[1020 + 1020 + 1]; +static int8_t sclip2[112 + 112 + 1]; +static uint8_t clip1[255 + 511 + 1]; + +// We declare this variable 'volatile' to prevent instruction reordering +// and make sure it's set to true _last_ (so as to be thread-safe) +static volatile int tables_ok = 0; + +#endif + +const int8_t* const VP8ksclip1 = &sclip1[1020]; +const int8_t* const VP8ksclip2 = &sclip2[112]; +const uint8_t* const VP8kclip1 = &clip1[255]; +const uint8_t* const VP8kabs0 = &abs0[255]; + +WEBP_TSAN_IGNORE_FUNCTION void VP8InitClipTables(void) { +#if !defined(USE_STATIC_TABLES) + int i; + if (!tables_ok) { + for (i = -255; i <= 255; ++i) { + abs0[255 + i] = (i < 0) ? -i : i; + } + for (i = -1020; i <= 1020; ++i) { + sclip1[1020 + i] = (i < -128) ? -128 : (i > 127) ? 127 : i; + } + for (i = -112; i <= 112; ++i) { + sclip2[112 + i] = (i < -16) ? -16 : (i > 15) ? 15 : i; + } + for (i = -255; i <= 255 + 255; ++i) { + clip1[255 + i] = (i < 0) ? 0 : (i > 255) ? 255 : i; + } + tables_ok = 1; + } +#endif // USE_STATIC_TABLES +} diff --git a/thirdparty/libwebp/dsp/dec_mips32.c b/thirdparty/libwebp/dsp/dec_mips32.c new file mode 100644 index 0000000000..4e9ef42605 --- /dev/null +++ b/thirdparty/libwebp/dsp/dec_mips32.c @@ -0,0 +1,587 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MIPS version of dsp functions +// +// Author(s): Djordje Pesut (djordje.pesut@imgtec.com) +// Jovan Zelincevic (jovan.zelincevic@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS32) + +#include "./mips_macro.h" + +static const int kC1 = 20091 + (1 << 16); +static const int kC2 = 35468; + +static WEBP_INLINE int abs_mips32(int x) { + const int sign = x >> 31; + return (x ^ sign) - sign; +} + +// 4 pixels in, 2 pixels out +static WEBP_INLINE void do_filter2(uint8_t* p, int step) { + const int p1 = p[-2 * step], p0 = p[-step], q0 = p[0], q1 = p[step]; + const int a = 3 * (q0 - p0) + VP8ksclip1[p1 - q1]; + const int a1 = VP8ksclip2[(a + 4) >> 3]; + const int a2 = VP8ksclip2[(a + 3) >> 3]; + p[-step] = VP8kclip1[p0 + a2]; + p[ 0] = VP8kclip1[q0 - a1]; +} + +// 4 pixels in, 4 pixels out +static WEBP_INLINE void do_filter4(uint8_t* p, int step) { + const int p1 = p[-2 * step], p0 = p[-step], q0 = p[0], q1 = p[step]; + const int a = 3 * (q0 - p0); + const int a1 = VP8ksclip2[(a + 4) >> 3]; + const int a2 = VP8ksclip2[(a + 3) >> 3]; + const int a3 = (a1 + 1) >> 1; + p[-2 * step] = VP8kclip1[p1 + a3]; + p[- step] = VP8kclip1[p0 + a2]; + p[ 0] = VP8kclip1[q0 - a1]; + p[ step] = VP8kclip1[q1 - a3]; +} + +// 6 pixels in, 6 pixels out +static WEBP_INLINE void do_filter6(uint8_t* p, int step) { + const int p2 = p[-3 * step], p1 = p[-2 * step], p0 = p[-step]; + const int q0 = p[0], q1 = p[step], q2 = p[2 * step]; + const int a = VP8ksclip1[3 * (q0 - p0) + VP8ksclip1[p1 - q1]]; + // a is in [-128,127], a1 in [-27,27], a2 in [-18,18] and a3 in [-9,9] + const int a1 = (27 * a + 63) >> 7; // eq. to ((3 * a + 7) * 9) >> 7 + const int a2 = (18 * a + 63) >> 7; // eq. to ((2 * a + 7) * 9) >> 7 + const int a3 = (9 * a + 63) >> 7; // eq. to ((1 * a + 7) * 9) >> 7 + p[-3 * step] = VP8kclip1[p2 + a3]; + p[-2 * step] = VP8kclip1[p1 + a2]; + p[- step] = VP8kclip1[p0 + a1]; + p[ 0] = VP8kclip1[q0 - a1]; + p[ step] = VP8kclip1[q1 - a2]; + p[ 2 * step] = VP8kclip1[q2 - a3]; +} + +static WEBP_INLINE int hev(const uint8_t* p, int step, int thresh) { + const int p1 = p[-2 * step], p0 = p[-step], q0 = p[0], q1 = p[step]; + return (abs_mips32(p1 - p0) > thresh) || (abs_mips32(q1 - q0) > thresh); +} + +static WEBP_INLINE int needs_filter(const uint8_t* p, int step, int t) { + const int p1 = p[-2 * step], p0 = p[-step], q0 = p[0], q1 = p[step]; + return ((4 * abs_mips32(p0 - q0) + abs_mips32(p1 - q1)) <= t); +} + +static WEBP_INLINE int needs_filter2(const uint8_t* p, + int step, int t, int it) { + const int p3 = p[-4 * step], p2 = p[-3 * step]; + const int p1 = p[-2 * step], p0 = p[-step]; + const int q0 = p[0], q1 = p[step], q2 = p[2 * step], q3 = p[3 * step]; + if ((4 * abs_mips32(p0 - q0) + abs_mips32(p1 - q1)) > t) { + return 0; + } + return abs_mips32(p3 - p2) <= it && abs_mips32(p2 - p1) <= it && + abs_mips32(p1 - p0) <= it && abs_mips32(q3 - q2) <= it && + abs_mips32(q2 - q1) <= it && abs_mips32(q1 - q0) <= it; +} + +static WEBP_INLINE void FilterLoop26(uint8_t* p, + int hstride, int vstride, int size, + int thresh, int ithresh, int hev_thresh) { + const int thresh2 = 2 * thresh + 1; + while (size-- > 0) { + if (needs_filter2(p, hstride, thresh2, ithresh)) { + if (hev(p, hstride, hev_thresh)) { + do_filter2(p, hstride); + } else { + do_filter6(p, hstride); + } + } + p += vstride; + } +} + +static WEBP_INLINE void FilterLoop24(uint8_t* p, + int hstride, int vstride, int size, + int thresh, int ithresh, int hev_thresh) { + const int thresh2 = 2 * thresh + 1; + while (size-- > 0) { + if (needs_filter2(p, hstride, thresh2, ithresh)) { + if (hev(p, hstride, hev_thresh)) { + do_filter2(p, hstride); + } else { + do_filter4(p, hstride); + } + } + p += vstride; + } +} + +// on macroblock edges +static void VFilter16(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop26(p, stride, 1, 16, thresh, ithresh, hev_thresh); +} + +static void HFilter16(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop26(p, 1, stride, 16, thresh, ithresh, hev_thresh); +} + +// 8-pixels wide variant, for chroma filtering +static void VFilter8(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop26(u, stride, 1, 8, thresh, ithresh, hev_thresh); + FilterLoop26(v, stride, 1, 8, thresh, ithresh, hev_thresh); +} + +static void HFilter8(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop26(u, 1, stride, 8, thresh, ithresh, hev_thresh); + FilterLoop26(v, 1, stride, 8, thresh, ithresh, hev_thresh); +} + +static void VFilter8i(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop24(u + 4 * stride, stride, 1, 8, thresh, ithresh, hev_thresh); + FilterLoop24(v + 4 * stride, stride, 1, 8, thresh, ithresh, hev_thresh); +} + +static void HFilter8i(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop24(u + 4, 1, stride, 8, thresh, ithresh, hev_thresh); + FilterLoop24(v + 4, 1, stride, 8, thresh, ithresh, hev_thresh); +} + +// on three inner edges +static void VFilter16i(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + int k; + for (k = 3; k > 0; --k) { + p += 4 * stride; + FilterLoop24(p, stride, 1, 16, thresh, ithresh, hev_thresh); + } +} + +static void HFilter16i(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + int k; + for (k = 3; k > 0; --k) { + p += 4; + FilterLoop24(p, 1, stride, 16, thresh, ithresh, hev_thresh); + } +} + +//------------------------------------------------------------------------------ +// Simple In-loop filtering (Paragraph 15.2) + +static void SimpleVFilter16(uint8_t* p, int stride, int thresh) { + int i; + const int thresh2 = 2 * thresh + 1; + for (i = 0; i < 16; ++i) { + if (needs_filter(p + i, stride, thresh2)) { + do_filter2(p + i, stride); + } + } +} + +static void SimpleHFilter16(uint8_t* p, int stride, int thresh) { + int i; + const int thresh2 = 2 * thresh + 1; + for (i = 0; i < 16; ++i) { + if (needs_filter(p + i * stride, 1, thresh2)) { + do_filter2(p + i * stride, 1); + } + } +} + +static void SimpleVFilter16i(uint8_t* p, int stride, int thresh) { + int k; + for (k = 3; k > 0; --k) { + p += 4 * stride; + SimpleVFilter16(p, stride, thresh); + } +} + +static void SimpleHFilter16i(uint8_t* p, int stride, int thresh) { + int k; + for (k = 3; k > 0; --k) { + p += 4; + SimpleHFilter16(p, stride, thresh); + } +} + +static void TransformOne(const int16_t* in, uint8_t* dst) { + int temp0, temp1, temp2, temp3, temp4; + int temp5, temp6, temp7, temp8, temp9; + int temp10, temp11, temp12, temp13, temp14; + int temp15, temp16, temp17, temp18; + int16_t* p_in = (int16_t*)in; + + // loops unrolled and merged to avoid usage of tmp buffer + // and to reduce number of stalls. MUL macro is written + // in assembler and inlined + __asm__ volatile( + "lh %[temp0], 0(%[in]) \n\t" + "lh %[temp8], 16(%[in]) \n\t" + "lh %[temp4], 8(%[in]) \n\t" + "lh %[temp12], 24(%[in]) \n\t" + "addu %[temp16], %[temp0], %[temp8] \n\t" + "subu %[temp0], %[temp0], %[temp8] \n\t" + "mul %[temp8], %[temp4], %[kC2] \n\t" + "mul %[temp17], %[temp12], %[kC1] \n\t" + "mul %[temp4], %[temp4], %[kC1] \n\t" + "mul %[temp12], %[temp12], %[kC2] \n\t" + "lh %[temp1], 2(%[in]) \n\t" + "lh %[temp5], 10(%[in]) \n\t" + "lh %[temp9], 18(%[in]) \n\t" + "lh %[temp13], 26(%[in]) \n\t" + "sra %[temp8], %[temp8], 16 \n\t" + "sra %[temp17], %[temp17], 16 \n\t" + "sra %[temp4], %[temp4], 16 \n\t" + "sra %[temp12], %[temp12], 16 \n\t" + "lh %[temp2], 4(%[in]) \n\t" + "lh %[temp6], 12(%[in]) \n\t" + "lh %[temp10], 20(%[in]) \n\t" + "lh %[temp14], 28(%[in]) \n\t" + "subu %[temp17], %[temp8], %[temp17] \n\t" + "addu %[temp4], %[temp4], %[temp12] \n\t" + "addu %[temp8], %[temp16], %[temp4] \n\t" + "subu %[temp4], %[temp16], %[temp4] \n\t" + "addu %[temp16], %[temp1], %[temp9] \n\t" + "subu %[temp1], %[temp1], %[temp9] \n\t" + "lh %[temp3], 6(%[in]) \n\t" + "lh %[temp7], 14(%[in]) \n\t" + "lh %[temp11], 22(%[in]) \n\t" + "lh %[temp15], 30(%[in]) \n\t" + "addu %[temp12], %[temp0], %[temp17] \n\t" + "subu %[temp0], %[temp0], %[temp17] \n\t" + "mul %[temp9], %[temp5], %[kC2] \n\t" + "mul %[temp17], %[temp13], %[kC1] \n\t" + "mul %[temp5], %[temp5], %[kC1] \n\t" + "mul %[temp13], %[temp13], %[kC2] \n\t" + "sra %[temp9], %[temp9], 16 \n\t" + "sra %[temp17], %[temp17], 16 \n\t" + "subu %[temp17], %[temp9], %[temp17] \n\t" + "sra %[temp5], %[temp5], 16 \n\t" + "sra %[temp13], %[temp13], 16 \n\t" + "addu %[temp5], %[temp5], %[temp13] \n\t" + "addu %[temp13], %[temp1], %[temp17] \n\t" + "subu %[temp1], %[temp1], %[temp17] \n\t" + "mul %[temp17], %[temp14], %[kC1] \n\t" + "mul %[temp14], %[temp14], %[kC2] \n\t" + "addu %[temp9], %[temp16], %[temp5] \n\t" + "subu %[temp5], %[temp16], %[temp5] \n\t" + "addu %[temp16], %[temp2], %[temp10] \n\t" + "subu %[temp2], %[temp2], %[temp10] \n\t" + "mul %[temp10], %[temp6], %[kC2] \n\t" + "mul %[temp6], %[temp6], %[kC1] \n\t" + "sra %[temp17], %[temp17], 16 \n\t" + "sra %[temp14], %[temp14], 16 \n\t" + "sra %[temp10], %[temp10], 16 \n\t" + "sra %[temp6], %[temp6], 16 \n\t" + "subu %[temp17], %[temp10], %[temp17] \n\t" + "addu %[temp6], %[temp6], %[temp14] \n\t" + "addu %[temp10], %[temp16], %[temp6] \n\t" + "subu %[temp6], %[temp16], %[temp6] \n\t" + "addu %[temp14], %[temp2], %[temp17] \n\t" + "subu %[temp2], %[temp2], %[temp17] \n\t" + "mul %[temp17], %[temp15], %[kC1] \n\t" + "mul %[temp15], %[temp15], %[kC2] \n\t" + "addu %[temp16], %[temp3], %[temp11] \n\t" + "subu %[temp3], %[temp3], %[temp11] \n\t" + "mul %[temp11], %[temp7], %[kC2] \n\t" + "mul %[temp7], %[temp7], %[kC1] \n\t" + "addiu %[temp8], %[temp8], 4 \n\t" + "addiu %[temp12], %[temp12], 4 \n\t" + "addiu %[temp0], %[temp0], 4 \n\t" + "addiu %[temp4], %[temp4], 4 \n\t" + "sra %[temp17], %[temp17], 16 \n\t" + "sra %[temp15], %[temp15], 16 \n\t" + "sra %[temp11], %[temp11], 16 \n\t" + "sra %[temp7], %[temp7], 16 \n\t" + "subu %[temp17], %[temp11], %[temp17] \n\t" + "addu %[temp7], %[temp7], %[temp15] \n\t" + "addu %[temp15], %[temp3], %[temp17] \n\t" + "subu %[temp3], %[temp3], %[temp17] \n\t" + "addu %[temp11], %[temp16], %[temp7] \n\t" + "subu %[temp7], %[temp16], %[temp7] \n\t" + "addu %[temp16], %[temp8], %[temp10] \n\t" + "subu %[temp8], %[temp8], %[temp10] \n\t" + "mul %[temp10], %[temp9], %[kC2] \n\t" + "mul %[temp17], %[temp11], %[kC1] \n\t" + "mul %[temp9], %[temp9], %[kC1] \n\t" + "mul %[temp11], %[temp11], %[kC2] \n\t" + "sra %[temp10], %[temp10], 16 \n\t" + "sra %[temp17], %[temp17], 16 \n\t" + "sra %[temp9], %[temp9], 16 \n\t" + "sra %[temp11], %[temp11], 16 \n\t" + "subu %[temp17], %[temp10], %[temp17] \n\t" + "addu %[temp11], %[temp9], %[temp11] \n\t" + "addu %[temp10], %[temp12], %[temp14] \n\t" + "subu %[temp12], %[temp12], %[temp14] \n\t" + "mul %[temp14], %[temp13], %[kC2] \n\t" + "mul %[temp9], %[temp15], %[kC1] \n\t" + "mul %[temp13], %[temp13], %[kC1] \n\t" + "mul %[temp15], %[temp15], %[kC2] \n\t" + "sra %[temp14], %[temp14], 16 \n\t" + "sra %[temp9], %[temp9], 16 \n\t" + "sra %[temp13], %[temp13], 16 \n\t" + "sra %[temp15], %[temp15], 16 \n\t" + "subu %[temp9], %[temp14], %[temp9] \n\t" + "addu %[temp15], %[temp13], %[temp15] \n\t" + "addu %[temp14], %[temp0], %[temp2] \n\t" + "subu %[temp0], %[temp0], %[temp2] \n\t" + "mul %[temp2], %[temp1], %[kC2] \n\t" + "mul %[temp13], %[temp3], %[kC1] \n\t" + "mul %[temp1], %[temp1], %[kC1] \n\t" + "mul %[temp3], %[temp3], %[kC2] \n\t" + "sra %[temp2], %[temp2], 16 \n\t" + "sra %[temp13], %[temp13], 16 \n\t" + "sra %[temp1], %[temp1], 16 \n\t" + "sra %[temp3], %[temp3], 16 \n\t" + "subu %[temp13], %[temp2], %[temp13] \n\t" + "addu %[temp3], %[temp1], %[temp3] \n\t" + "addu %[temp2], %[temp4], %[temp6] \n\t" + "subu %[temp4], %[temp4], %[temp6] \n\t" + "mul %[temp6], %[temp5], %[kC2] \n\t" + "mul %[temp1], %[temp7], %[kC1] \n\t" + "mul %[temp5], %[temp5], %[kC1] \n\t" + "mul %[temp7], %[temp7], %[kC2] \n\t" + "sra %[temp6], %[temp6], 16 \n\t" + "sra %[temp1], %[temp1], 16 \n\t" + "sra %[temp5], %[temp5], 16 \n\t" + "sra %[temp7], %[temp7], 16 \n\t" + "subu %[temp1], %[temp6], %[temp1] \n\t" + "addu %[temp7], %[temp5], %[temp7] \n\t" + "addu %[temp5], %[temp16], %[temp11] \n\t" + "subu %[temp16], %[temp16], %[temp11] \n\t" + "addu %[temp11], %[temp8], %[temp17] \n\t" + "subu %[temp8], %[temp8], %[temp17] \n\t" + "sra %[temp5], %[temp5], 3 \n\t" + "sra %[temp16], %[temp16], 3 \n\t" + "sra %[temp11], %[temp11], 3 \n\t" + "sra %[temp8], %[temp8], 3 \n\t" + "addu %[temp17], %[temp10], %[temp15] \n\t" + "subu %[temp10], %[temp10], %[temp15] \n\t" + "addu %[temp15], %[temp12], %[temp9] \n\t" + "subu %[temp12], %[temp12], %[temp9] \n\t" + "sra %[temp17], %[temp17], 3 \n\t" + "sra %[temp10], %[temp10], 3 \n\t" + "sra %[temp15], %[temp15], 3 \n\t" + "sra %[temp12], %[temp12], 3 \n\t" + "addu %[temp9], %[temp14], %[temp3] \n\t" + "subu %[temp14], %[temp14], %[temp3] \n\t" + "addu %[temp3], %[temp0], %[temp13] \n\t" + "subu %[temp0], %[temp0], %[temp13] \n\t" + "sra %[temp9], %[temp9], 3 \n\t" + "sra %[temp14], %[temp14], 3 \n\t" + "sra %[temp3], %[temp3], 3 \n\t" + "sra %[temp0], %[temp0], 3 \n\t" + "addu %[temp13], %[temp2], %[temp7] \n\t" + "subu %[temp2], %[temp2], %[temp7] \n\t" + "addu %[temp7], %[temp4], %[temp1] \n\t" + "subu %[temp4], %[temp4], %[temp1] \n\t" + "sra %[temp13], %[temp13], 3 \n\t" + "sra %[temp2], %[temp2], 3 \n\t" + "sra %[temp7], %[temp7], 3 \n\t" + "sra %[temp4], %[temp4], 3 \n\t" + "addiu %[temp6], $zero, 255 \n\t" + "lbu %[temp1], 0+0*" XSTR(BPS) "(%[dst]) \n\t" + "addu %[temp1], %[temp1], %[temp5] \n\t" + "sra %[temp5], %[temp1], 8 \n\t" + "sra %[temp18], %[temp1], 31 \n\t" + "beqz %[temp5], 1f \n\t" + "xor %[temp1], %[temp1], %[temp1] \n\t" + "movz %[temp1], %[temp6], %[temp18] \n\t" + "1: \n\t" + "lbu %[temp18], 1+0*" XSTR(BPS) "(%[dst]) \n\t" + "sb %[temp1], 0+0*" XSTR(BPS) "(%[dst]) \n\t" + "addu %[temp18], %[temp18], %[temp11] \n\t" + "sra %[temp11], %[temp18], 8 \n\t" + "sra %[temp1], %[temp18], 31 \n\t" + "beqz %[temp11], 2f \n\t" + "xor %[temp18], %[temp18], %[temp18] \n\t" + "movz %[temp18], %[temp6], %[temp1] \n\t" + "2: \n\t" + "lbu %[temp1], 2+0*" XSTR(BPS) "(%[dst]) \n\t" + "sb %[temp18], 1+0*" XSTR(BPS) "(%[dst]) \n\t" + "addu %[temp1], %[temp1], %[temp8] \n\t" + "sra %[temp8], %[temp1], 8 \n\t" + "sra %[temp18], %[temp1], 31 \n\t" + "beqz %[temp8], 3f \n\t" + "xor %[temp1], %[temp1], %[temp1] \n\t" + "movz %[temp1], %[temp6], %[temp18] \n\t" + "3: \n\t" + "lbu %[temp18], 3+0*" XSTR(BPS) "(%[dst]) \n\t" + "sb %[temp1], 2+0*" XSTR(BPS) "(%[dst]) \n\t" + "addu %[temp18], %[temp18], %[temp16] \n\t" + "sra %[temp16], %[temp18], 8 \n\t" + "sra %[temp1], %[temp18], 31 \n\t" + "beqz %[temp16], 4f \n\t" + "xor %[temp18], %[temp18], %[temp18] \n\t" + "movz %[temp18], %[temp6], %[temp1] \n\t" + "4: \n\t" + "sb %[temp18], 3+0*" XSTR(BPS) "(%[dst]) \n\t" + "lbu %[temp5], 0+1*" XSTR(BPS) "(%[dst]) \n\t" + "lbu %[temp8], 1+1*" XSTR(BPS) "(%[dst]) \n\t" + "lbu %[temp11], 2+1*" XSTR(BPS) "(%[dst]) \n\t" + "lbu %[temp16], 3+1*" XSTR(BPS) "(%[dst]) \n\t" + "addu %[temp5], %[temp5], %[temp17] \n\t" + "addu %[temp8], %[temp8], %[temp15] \n\t" + "addu %[temp11], %[temp11], %[temp12] \n\t" + "addu %[temp16], %[temp16], %[temp10] \n\t" + "sra %[temp18], %[temp5], 8 \n\t" + "sra %[temp1], %[temp5], 31 \n\t" + "beqz %[temp18], 5f \n\t" + "xor %[temp5], %[temp5], %[temp5] \n\t" + "movz %[temp5], %[temp6], %[temp1] \n\t" + "5: \n\t" + "sra %[temp18], %[temp8], 8 \n\t" + "sra %[temp1], %[temp8], 31 \n\t" + "beqz %[temp18], 6f \n\t" + "xor %[temp8], %[temp8], %[temp8] \n\t" + "movz %[temp8], %[temp6], %[temp1] \n\t" + "6: \n\t" + "sra %[temp18], %[temp11], 8 \n\t" + "sra %[temp1], %[temp11], 31 \n\t" + "sra %[temp17], %[temp16], 8 \n\t" + "sra %[temp15], %[temp16], 31 \n\t" + "beqz %[temp18], 7f \n\t" + "xor %[temp11], %[temp11], %[temp11] \n\t" + "movz %[temp11], %[temp6], %[temp1] \n\t" + "7: \n\t" + "beqz %[temp17], 8f \n\t" + "xor %[temp16], %[temp16], %[temp16] \n\t" + "movz %[temp16], %[temp6], %[temp15] \n\t" + "8: \n\t" + "sb %[temp5], 0+1*" XSTR(BPS) "(%[dst]) \n\t" + "sb %[temp8], 1+1*" XSTR(BPS) "(%[dst]) \n\t" + "sb %[temp11], 2+1*" XSTR(BPS) "(%[dst]) \n\t" + "sb %[temp16], 3+1*" XSTR(BPS) "(%[dst]) \n\t" + "lbu %[temp5], 0+2*" XSTR(BPS) "(%[dst]) \n\t" + "lbu %[temp8], 1+2*" XSTR(BPS) "(%[dst]) \n\t" + "lbu %[temp11], 2+2*" XSTR(BPS) "(%[dst]) \n\t" + "lbu %[temp16], 3+2*" XSTR(BPS) "(%[dst]) \n\t" + "addu %[temp5], %[temp5], %[temp9] \n\t" + "addu %[temp8], %[temp8], %[temp3] \n\t" + "addu %[temp11], %[temp11], %[temp0] \n\t" + "addu %[temp16], %[temp16], %[temp14] \n\t" + "sra %[temp18], %[temp5], 8 \n\t" + "sra %[temp1], %[temp5], 31 \n\t" + "sra %[temp17], %[temp8], 8 \n\t" + "sra %[temp15], %[temp8], 31 \n\t" + "sra %[temp12], %[temp11], 8 \n\t" + "sra %[temp10], %[temp11], 31 \n\t" + "sra %[temp9], %[temp16], 8 \n\t" + "sra %[temp3], %[temp16], 31 \n\t" + "beqz %[temp18], 9f \n\t" + "xor %[temp5], %[temp5], %[temp5] \n\t" + "movz %[temp5], %[temp6], %[temp1] \n\t" + "9: \n\t" + "beqz %[temp17], 10f \n\t" + "xor %[temp8], %[temp8], %[temp8] \n\t" + "movz %[temp8], %[temp6], %[temp15] \n\t" + "10: \n\t" + "beqz %[temp12], 11f \n\t" + "xor %[temp11], %[temp11], %[temp11] \n\t" + "movz %[temp11], %[temp6], %[temp10] \n\t" + "11: \n\t" + "beqz %[temp9], 12f \n\t" + "xor %[temp16], %[temp16], %[temp16] \n\t" + "movz %[temp16], %[temp6], %[temp3] \n\t" + "12: \n\t" + "sb %[temp5], 0+2*" XSTR(BPS) "(%[dst]) \n\t" + "sb %[temp8], 1+2*" XSTR(BPS) "(%[dst]) \n\t" + "sb %[temp11], 2+2*" XSTR(BPS) "(%[dst]) \n\t" + "sb %[temp16], 3+2*" XSTR(BPS) "(%[dst]) \n\t" + "lbu %[temp5], 0+3*" XSTR(BPS) "(%[dst]) \n\t" + "lbu %[temp8], 1+3*" XSTR(BPS) "(%[dst]) \n\t" + "lbu %[temp11], 2+3*" XSTR(BPS) "(%[dst]) \n\t" + "lbu %[temp16], 3+3*" XSTR(BPS) "(%[dst]) \n\t" + "addu %[temp5], %[temp5], %[temp13] \n\t" + "addu %[temp8], %[temp8], %[temp7] \n\t" + "addu %[temp11], %[temp11], %[temp4] \n\t" + "addu %[temp16], %[temp16], %[temp2] \n\t" + "sra %[temp18], %[temp5], 8 \n\t" + "sra %[temp1], %[temp5], 31 \n\t" + "sra %[temp17], %[temp8], 8 \n\t" + "sra %[temp15], %[temp8], 31 \n\t" + "sra %[temp12], %[temp11], 8 \n\t" + "sra %[temp10], %[temp11], 31 \n\t" + "sra %[temp9], %[temp16], 8 \n\t" + "sra %[temp3], %[temp16], 31 \n\t" + "beqz %[temp18], 13f \n\t" + "xor %[temp5], %[temp5], %[temp5] \n\t" + "movz %[temp5], %[temp6], %[temp1] \n\t" + "13: \n\t" + "beqz %[temp17], 14f \n\t" + "xor %[temp8], %[temp8], %[temp8] \n\t" + "movz %[temp8], %[temp6], %[temp15] \n\t" + "14: \n\t" + "beqz %[temp12], 15f \n\t" + "xor %[temp11], %[temp11], %[temp11] \n\t" + "movz %[temp11], %[temp6], %[temp10] \n\t" + "15: \n\t" + "beqz %[temp9], 16f \n\t" + "xor %[temp16], %[temp16], %[temp16] \n\t" + "movz %[temp16], %[temp6], %[temp3] \n\t" + "16: \n\t" + "sb %[temp5], 0+3*" XSTR(BPS) "(%[dst]) \n\t" + "sb %[temp8], 1+3*" XSTR(BPS) "(%[dst]) \n\t" + "sb %[temp11], 2+3*" XSTR(BPS) "(%[dst]) \n\t" + "sb %[temp16], 3+3*" XSTR(BPS) "(%[dst]) \n\t" + + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8), + [temp9]"=&r"(temp9), [temp10]"=&r"(temp10), [temp11]"=&r"(temp11), + [temp12]"=&r"(temp12), [temp13]"=&r"(temp13), [temp14]"=&r"(temp14), + [temp15]"=&r"(temp15), [temp16]"=&r"(temp16), [temp17]"=&r"(temp17), + [temp18]"=&r"(temp18) + : [in]"r"(p_in), [kC1]"r"(kC1), [kC2]"r"(kC2), [dst]"r"(dst) + : "memory", "hi", "lo" + ); +} + +static void TransformTwo(const int16_t* in, uint8_t* dst, int do_two) { + TransformOne(in, dst); + if (do_two) { + TransformOne(in + 16, dst + 4); + } +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8DspInitMIPS32(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8DspInitMIPS32(void) { + VP8InitClipTables(); + + VP8Transform = TransformTwo; + + VP8VFilter16 = VFilter16; + VP8HFilter16 = HFilter16; + VP8VFilter8 = VFilter8; + VP8HFilter8 = HFilter8; + VP8VFilter16i = VFilter16i; + VP8HFilter16i = HFilter16i; + VP8VFilter8i = VFilter8i; + VP8HFilter8i = HFilter8i; + + VP8SimpleVFilter16 = SimpleVFilter16; + VP8SimpleHFilter16 = SimpleHFilter16; + VP8SimpleVFilter16i = SimpleVFilter16i; + VP8SimpleHFilter16i = SimpleHFilter16i; +} + +#else // !WEBP_USE_MIPS32 + +WEBP_DSP_INIT_STUB(VP8DspInitMIPS32) + +#endif // WEBP_USE_MIPS32 diff --git a/thirdparty/libwebp/dsp/dec_mips_dsp_r2.c b/thirdparty/libwebp/dsp/dec_mips_dsp_r2.c new file mode 100644 index 0000000000..db5c657228 --- /dev/null +++ b/thirdparty/libwebp/dsp/dec_mips_dsp_r2.c @@ -0,0 +1,994 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MIPS version of dsp functions +// +// Author(s): Djordje Pesut (djordje.pesut@imgtec.com) +// Jovan Zelincevic (jovan.zelincevic@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS_DSP_R2) + +#include "./mips_macro.h" + +static const int kC1 = 20091 + (1 << 16); +static const int kC2 = 35468; + +#define MUL(a, b) (((a) * (b)) >> 16) + +static void TransformDC(const int16_t* in, uint8_t* dst) { + int temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10; + + __asm__ volatile ( + LOAD_WITH_OFFSET_X4(temp1, temp2, temp3, temp4, dst, + 0, 0, 0, 0, + 0, 1, 2, 3, + BPS) + "lh %[temp5], 0(%[in]) \n\t" + "addiu %[temp5], %[temp5], 4 \n\t" + "ins %[temp5], %[temp5], 16, 16 \n\t" + "shra.ph %[temp5], %[temp5], 3 \n\t" + CONVERT_2_BYTES_TO_HALF(temp6, temp7, temp8, temp9, temp10, temp1, temp2, + temp3, temp1, temp2, temp3, temp4) + STORE_SAT_SUM_X2(temp6, temp7, temp8, temp9, temp10, temp1, temp2, temp3, + temp5, temp5, temp5, temp5, temp5, temp5, temp5, temp5, + dst, 0, 1, 2, 3, BPS) + + OUTPUT_EARLY_CLOBBER_REGS_10() + : [in]"r"(in), [dst]"r"(dst) + : "memory" + ); +} + +static void TransformAC3(const int16_t* in, uint8_t* dst) { + const int a = in[0] + 4; + int c4 = MUL(in[4], kC2); + const int d4 = MUL(in[4], kC1); + const int c1 = MUL(in[1], kC2); + const int d1 = MUL(in[1], kC1); + int temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9; + int temp10, temp11, temp12, temp13, temp14, temp15, temp16, temp17, temp18; + + __asm__ volatile ( + "ins %[c4], %[d4], 16, 16 \n\t" + "replv.ph %[temp1], %[a] \n\t" + "replv.ph %[temp4], %[d1] \n\t" + ADD_SUB_HALVES(temp2, temp3, temp1, c4) + "replv.ph %[temp5], %[c1] \n\t" + SHIFT_R_SUM_X2(temp1, temp6, temp7, temp8, temp2, temp9, temp10, temp4, + temp2, temp2, temp3, temp3, temp4, temp5, temp4, temp5) + LOAD_WITH_OFFSET_X4(temp3, temp5, temp11, temp12, dst, + 0, 0, 0, 0, + 0, 1, 2, 3, + BPS) + CONVERT_2_BYTES_TO_HALF(temp13, temp14, temp3, temp15, temp5, temp16, + temp11, temp17, temp3, temp5, temp11, temp12) + PACK_2_HALVES_TO_WORD(temp12, temp18, temp7, temp6, temp1, temp8, temp2, + temp4, temp7, temp6, temp10, temp9) + STORE_SAT_SUM_X2(temp13, temp14, temp3, temp15, temp5, temp16, temp11, + temp17, temp12, temp18, temp1, temp8, temp2, temp4, + temp7, temp6, dst, 0, 1, 2, 3, BPS) + + OUTPUT_EARLY_CLOBBER_REGS_18(), + [c4]"+&r"(c4) + : [dst]"r"(dst), [a]"r"(a), [d1]"r"(d1), [d4]"r"(d4), [c1]"r"(c1) + : "memory" + ); +} + +static void TransformOne(const int16_t* in, uint8_t* dst) { + int temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9; + int temp10, temp11, temp12, temp13, temp14, temp15, temp16, temp17, temp18; + + __asm__ volatile ( + "ulw %[temp1], 0(%[in]) \n\t" + "ulw %[temp2], 16(%[in]) \n\t" + LOAD_IN_X2(temp5, temp6, 24, 26) + ADD_SUB_HALVES(temp3, temp4, temp1, temp2) + LOAD_IN_X2(temp1, temp2, 8, 10) + MUL_SHIFT_SUM(temp7, temp8, temp9, temp10, temp11, temp12, temp13, temp14, + temp10, temp8, temp9, temp7, temp1, temp2, temp5, temp6, + temp13, temp11, temp14, temp12) + INSERT_HALF_X2(temp8, temp7, temp10, temp9) + "ulw %[temp17], 4(%[in]) \n\t" + "ulw %[temp18], 20(%[in]) \n\t" + ADD_SUB_HALVES(temp1, temp2, temp3, temp8) + ADD_SUB_HALVES(temp5, temp6, temp4, temp7) + ADD_SUB_HALVES(temp7, temp8, temp17, temp18) + LOAD_IN_X2(temp17, temp18, 12, 14) + LOAD_IN_X2(temp9, temp10, 28, 30) + MUL_SHIFT_SUM(temp11, temp12, temp13, temp14, temp15, temp16, temp4, temp17, + temp12, temp14, temp11, temp13, temp17, temp18, temp9, temp10, + temp15, temp4, temp16, temp17) + INSERT_HALF_X2(temp11, temp12, temp13, temp14) + ADD_SUB_HALVES(temp17, temp8, temp8, temp11) + ADD_SUB_HALVES(temp3, temp4, temp7, temp12) + + // horizontal + SRA_16(temp9, temp10, temp11, temp12, temp1, temp2, temp5, temp6) + INSERT_HALF_X2(temp1, temp6, temp5, temp2) + SRA_16(temp13, temp14, temp15, temp16, temp3, temp4, temp17, temp8) + "repl.ph %[temp2], 0x4 \n\t" + INSERT_HALF_X2(temp3, temp8, temp17, temp4) + "addq.ph %[temp1], %[temp1], %[temp2] \n\t" + "addq.ph %[temp6], %[temp6], %[temp2] \n\t" + ADD_SUB_HALVES(temp2, temp4, temp1, temp3) + ADD_SUB_HALVES(temp5, temp7, temp6, temp8) + MUL_SHIFT_SUM(temp1, temp3, temp6, temp8, temp9, temp13, temp17, temp18, + temp3, temp13, temp1, temp9, temp9, temp13, temp11, temp15, + temp6, temp17, temp8, temp18) + MUL_SHIFT_SUM(temp6, temp8, temp18, temp17, temp11, temp15, temp12, temp16, + temp8, temp15, temp6, temp11, temp12, temp16, temp10, temp14, + temp18, temp12, temp17, temp16) + INSERT_HALF_X2(temp1, temp3, temp9, temp13) + INSERT_HALF_X2(temp6, temp8, temp11, temp15) + SHIFT_R_SUM_X2(temp9, temp10, temp11, temp12, temp13, temp14, temp15, + temp16, temp2, temp4, temp5, temp7, temp3, temp1, temp8, + temp6) + PACK_2_HALVES_TO_WORD(temp1, temp2, temp3, temp4, temp9, temp12, temp13, + temp16, temp11, temp10, temp15, temp14) + LOAD_WITH_OFFSET_X4(temp10, temp11, temp14, temp15, dst, + 0, 0, 0, 0, + 0, 1, 2, 3, + BPS) + CONVERT_2_BYTES_TO_HALF(temp5, temp6, temp7, temp8, temp17, temp18, temp10, + temp11, temp10, temp11, temp14, temp15) + STORE_SAT_SUM_X2(temp5, temp6, temp7, temp8, temp17, temp18, temp10, temp11, + temp9, temp12, temp1, temp2, temp13, temp16, temp3, temp4, + dst, 0, 1, 2, 3, BPS) + + OUTPUT_EARLY_CLOBBER_REGS_18() + : [dst]"r"(dst), [in]"r"(in), [kC1]"r"(kC1), [kC2]"r"(kC2) + : "memory", "hi", "lo" + ); +} + +static void TransformTwo(const int16_t* in, uint8_t* dst, int do_two) { + TransformOne(in, dst); + if (do_two) { + TransformOne(in + 16, dst + 4); + } +} + +static WEBP_INLINE void FilterLoop26(uint8_t* p, + int hstride, int vstride, int size, + int thresh, int ithresh, int hev_thresh) { + const int thresh2 = 2 * thresh + 1; + int temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9; + int temp10, temp11, temp12, temp13, temp14, temp15; + + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "1: \n\t" + "negu %[temp1], %[hstride] \n\t" + "addiu %[size], %[size], -1 \n\t" + "sll %[temp2], %[hstride], 1 \n\t" + "sll %[temp3], %[temp1], 1 \n\t" + "addu %[temp4], %[temp2], %[hstride] \n\t" + "addu %[temp5], %[temp3], %[temp1] \n\t" + "lbu %[temp7], 0(%[p]) \n\t" + "sll %[temp6], %[temp3], 1 \n\t" + "lbux %[temp8], %[temp5](%[p]) \n\t" + "lbux %[temp9], %[temp3](%[p]) \n\t" + "lbux %[temp10], %[temp1](%[p]) \n\t" + "lbux %[temp11], %[temp6](%[p]) \n\t" + "lbux %[temp12], %[hstride](%[p]) \n\t" + "lbux %[temp13], %[temp2](%[p]) \n\t" + "lbux %[temp14], %[temp4](%[p]) \n\t" + "subu %[temp1], %[temp10], %[temp7] \n\t" + "subu %[temp2], %[temp9], %[temp12] \n\t" + "absq_s.w %[temp3], %[temp1] \n\t" + "absq_s.w %[temp4], %[temp2] \n\t" + "negu %[temp1], %[temp1] \n\t" + "sll %[temp3], %[temp3], 2 \n\t" + "addu %[temp15], %[temp3], %[temp4] \n\t" + "subu %[temp3], %[temp15], %[thresh2] \n\t" + "sll %[temp6], %[temp1], 1 \n\t" + "bgtz %[temp3], 3f \n\t" + " subu %[temp4], %[temp11], %[temp8] \n\t" + "absq_s.w %[temp4], %[temp4] \n\t" + "shll_s.w %[temp2], %[temp2], 24 \n\t" + "subu %[temp4], %[temp4], %[ithresh] \n\t" + "bgtz %[temp4], 3f \n\t" + " subu %[temp3], %[temp8], %[temp9] \n\t" + "absq_s.w %[temp3], %[temp3] \n\t" + "subu %[temp3], %[temp3], %[ithresh] \n\t" + "bgtz %[temp3], 3f \n\t" + " subu %[temp5], %[temp9], %[temp10] \n\t" + "absq_s.w %[temp3], %[temp5] \n\t" + "absq_s.w %[temp5], %[temp5] \n\t" + "subu %[temp3], %[temp3], %[ithresh] \n\t" + "bgtz %[temp3], 3f \n\t" + " subu %[temp3], %[temp14], %[temp13] \n\t" + "absq_s.w %[temp3], %[temp3] \n\t" + "slt %[temp5], %[hev_thresh], %[temp5] \n\t" + "subu %[temp3], %[temp3], %[ithresh] \n\t" + "bgtz %[temp3], 3f \n\t" + " subu %[temp3], %[temp13], %[temp12] \n\t" + "absq_s.w %[temp3], %[temp3] \n\t" + "sra %[temp4], %[temp2], 24 \n\t" + "subu %[temp3], %[temp3], %[ithresh] \n\t" + "bgtz %[temp3], 3f \n\t" + " subu %[temp15], %[temp12], %[temp7] \n\t" + "absq_s.w %[temp3], %[temp15] \n\t" + "absq_s.w %[temp15], %[temp15] \n\t" + "subu %[temp3], %[temp3], %[ithresh] \n\t" + "bgtz %[temp3], 3f \n\t" + " slt %[temp15], %[hev_thresh], %[temp15] \n\t" + "addu %[temp3], %[temp6], %[temp1] \n\t" + "or %[temp2], %[temp5], %[temp15] \n\t" + "addu %[temp5], %[temp4], %[temp3] \n\t" + "beqz %[temp2], 4f \n\t" + " shra_r.w %[temp1], %[temp5], 3 \n\t" + "addiu %[temp2], %[temp5], 3 \n\t" + "sra %[temp2], %[temp2], 3 \n\t" + "shll_s.w %[temp1], %[temp1], 27 \n\t" + "shll_s.w %[temp2], %[temp2], 27 \n\t" + "subu %[temp3], %[p], %[hstride] \n\t" + "sra %[temp1], %[temp1], 27 \n\t" + "sra %[temp2], %[temp2], 27 \n\t" + "subu %[temp1], %[temp7], %[temp1] \n\t" + "addu %[temp2], %[temp10], %[temp2] \n\t" + "lbux %[temp2], %[temp2](%[VP8kclip1]) \n\t" + "lbux %[temp1], %[temp1](%[VP8kclip1]) \n\t" + "sb %[temp2], 0(%[temp3]) \n\t" + "j 3f \n\t" + " sb %[temp1], 0(%[p]) \n\t" + "4: \n\t" + "shll_s.w %[temp5], %[temp5], 24 \n\t" + "subu %[temp14], %[p], %[hstride] \n\t" + "subu %[temp11], %[temp14], %[hstride] \n\t" + "sra %[temp6], %[temp5], 24 \n\t" + "sll %[temp1], %[temp6], 3 \n\t" + "subu %[temp15], %[temp11], %[hstride] \n\t" + "addu %[temp2], %[temp6], %[temp1] \n\t" + "sll %[temp3], %[temp2], 1 \n\t" + "addu %[temp4], %[temp3], %[temp2] \n\t" + "addiu %[temp2], %[temp2], 63 \n\t" + "addiu %[temp3], %[temp3], 63 \n\t" + "addiu %[temp4], %[temp4], 63 \n\t" + "sra %[temp2], %[temp2], 7 \n\t" + "sra %[temp3], %[temp3], 7 \n\t" + "sra %[temp4], %[temp4], 7 \n\t" + "addu %[temp1], %[temp8], %[temp2] \n\t" + "addu %[temp5], %[temp9], %[temp3] \n\t" + "addu %[temp6], %[temp10], %[temp4] \n\t" + "subu %[temp8], %[temp7], %[temp4] \n\t" + "subu %[temp7], %[temp12], %[temp3] \n\t" + "addu %[temp10], %[p], %[hstride] \n\t" + "subu %[temp9], %[temp13], %[temp2] \n\t" + "addu %[temp12], %[temp10], %[hstride] \n\t" + "lbux %[temp2], %[temp1](%[VP8kclip1]) \n\t" + "lbux %[temp3], %[temp5](%[VP8kclip1]) \n\t" + "lbux %[temp4], %[temp6](%[VP8kclip1]) \n\t" + "lbux %[temp5], %[temp8](%[VP8kclip1]) \n\t" + "lbux %[temp6], %[temp7](%[VP8kclip1]) \n\t" + "lbux %[temp8], %[temp9](%[VP8kclip1]) \n\t" + "sb %[temp2], 0(%[temp15]) \n\t" + "sb %[temp3], 0(%[temp11]) \n\t" + "sb %[temp4], 0(%[temp14]) \n\t" + "sb %[temp5], 0(%[p]) \n\t" + "sb %[temp6], 0(%[temp10]) \n\t" + "sb %[temp8], 0(%[temp12]) \n\t" + "3: \n\t" + "bgtz %[size], 1b \n\t" + " addu %[p], %[p], %[vstride] \n\t" + ".set pop \n\t" + : [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),[temp3]"=&r"(temp3), + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [temp6]"=&r"(temp6), + [temp7]"=&r"(temp7),[temp8]"=&r"(temp8),[temp9]"=&r"(temp9), + [temp10]"=&r"(temp10),[temp11]"=&r"(temp11),[temp12]"=&r"(temp12), + [temp13]"=&r"(temp13),[temp14]"=&r"(temp14),[temp15]"=&r"(temp15), + [size]"+&r"(size), [p]"+&r"(p) + : [hstride]"r"(hstride), [thresh2]"r"(thresh2), + [ithresh]"r"(ithresh),[vstride]"r"(vstride), [hev_thresh]"r"(hev_thresh), + [VP8kclip1]"r"(VP8kclip1) + : "memory" + ); +} + +static WEBP_INLINE void FilterLoop24(uint8_t* p, + int hstride, int vstride, int size, + int thresh, int ithresh, int hev_thresh) { + int p0, q0, p1, q1, p2, q2, p3, q3; + int step1, step2, temp1, temp2, temp3, temp4; + uint8_t* pTemp0; + uint8_t* pTemp1; + const int thresh2 = 2 * thresh + 1; + + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "bltz %[size], 3f \n\t" + " nop \n\t" + "2: \n\t" + "negu %[step1], %[hstride] \n\t" + "lbu %[q0], 0(%[p]) \n\t" + "lbux %[p0], %[step1](%[p]) \n\t" + "subu %[step1], %[step1], %[hstride] \n\t" + "lbux %[q1], %[hstride](%[p]) \n\t" + "subu %[temp1], %[p0], %[q0] \n\t" + "lbux %[p1], %[step1](%[p]) \n\t" + "addu %[step2], %[hstride], %[hstride] \n\t" + "absq_s.w %[temp2], %[temp1] \n\t" + "subu %[temp3], %[p1], %[q1] \n\t" + "absq_s.w %[temp4], %[temp3] \n\t" + "sll %[temp2], %[temp2], 2 \n\t" + "addu %[temp2], %[temp2], %[temp4] \n\t" + "subu %[temp4], %[temp2], %[thresh2] \n\t" + "subu %[step1], %[step1], %[hstride] \n\t" + "bgtz %[temp4], 0f \n\t" + " lbux %[p2], %[step1](%[p]) \n\t" + "subu %[step1], %[step1], %[hstride] \n\t" + "lbux %[q2], %[step2](%[p]) \n\t" + "lbux %[p3], %[step1](%[p]) \n\t" + "subu %[temp4], %[p2], %[p1] \n\t" + "addu %[step2], %[step2], %[hstride] \n\t" + "subu %[temp2], %[p3], %[p2] \n\t" + "absq_s.w %[temp4], %[temp4] \n\t" + "absq_s.w %[temp2], %[temp2] \n\t" + "lbux %[q3], %[step2](%[p]) \n\t" + "subu %[temp4], %[temp4], %[ithresh] \n\t" + "negu %[temp1], %[temp1] \n\t" + "bgtz %[temp4], 0f \n\t" + " subu %[temp2], %[temp2], %[ithresh] \n\t" + "subu %[p3], %[p1], %[p0] \n\t" + "bgtz %[temp2], 0f \n\t" + " absq_s.w %[p3], %[p3] \n\t" + "subu %[temp4], %[q3], %[q2] \n\t" + "subu %[pTemp0], %[p], %[hstride] \n\t" + "absq_s.w %[temp4], %[temp4] \n\t" + "subu %[temp2], %[p3], %[ithresh] \n\t" + "sll %[step1], %[temp1], 1 \n\t" + "bgtz %[temp2], 0f \n\t" + " subu %[temp4], %[temp4], %[ithresh] \n\t" + "subu %[temp2], %[q2], %[q1] \n\t" + "bgtz %[temp4], 0f \n\t" + " absq_s.w %[temp2], %[temp2] \n\t" + "subu %[q3], %[q1], %[q0] \n\t" + "absq_s.w %[q3], %[q3] \n\t" + "subu %[temp2], %[temp2], %[ithresh] \n\t" + "addu %[temp1], %[temp1], %[step1] \n\t" + "bgtz %[temp2], 0f \n\t" + " subu %[temp4], %[q3], %[ithresh] \n\t" + "slt %[p3], %[hev_thresh], %[p3] \n\t" + "bgtz %[temp4], 0f \n\t" + " slt %[q3], %[hev_thresh], %[q3] \n\t" + "or %[q3], %[q3], %[p3] \n\t" + "bgtz %[q3], 1f \n\t" + " shra_r.w %[temp2], %[temp1], 3 \n\t" + "addiu %[temp1], %[temp1], 3 \n\t" + "sra %[temp1], %[temp1], 3 \n\t" + "shll_s.w %[temp2], %[temp2], 27 \n\t" + "shll_s.w %[temp1], %[temp1], 27 \n\t" + "addu %[pTemp1], %[p], %[hstride] \n\t" + "sra %[temp2], %[temp2], 27 \n\t" + "sra %[temp1], %[temp1], 27 \n\t" + "addiu %[step1], %[temp2], 1 \n\t" + "sra %[step1], %[step1], 1 \n\t" + "addu %[p0], %[p0], %[temp1] \n\t" + "addu %[p1], %[p1], %[step1] \n\t" + "subu %[q0], %[q0], %[temp2] \n\t" + "subu %[q1], %[q1], %[step1] \n\t" + "lbux %[temp2], %[p0](%[VP8kclip1]) \n\t" + "lbux %[temp3], %[q0](%[VP8kclip1]) \n\t" + "lbux %[temp4], %[q1](%[VP8kclip1]) \n\t" + "sb %[temp2], 0(%[pTemp0]) \n\t" + "lbux %[temp1], %[p1](%[VP8kclip1]) \n\t" + "subu %[pTemp0], %[pTemp0], %[hstride] \n\t" + "sb %[temp3], 0(%[p]) \n\t" + "sb %[temp4], 0(%[pTemp1]) \n\t" + "j 0f \n\t" + " sb %[temp1], 0(%[pTemp0]) \n\t" + "1: \n\t" + "shll_s.w %[temp3], %[temp3], 24 \n\t" + "sra %[temp3], %[temp3], 24 \n\t" + "addu %[temp1], %[temp1], %[temp3] \n\t" + "shra_r.w %[temp2], %[temp1], 3 \n\t" + "addiu %[temp1], %[temp1], 3 \n\t" + "shll_s.w %[temp2], %[temp2], 27 \n\t" + "sra %[temp1], %[temp1], 3 \n\t" + "shll_s.w %[temp1], %[temp1], 27 \n\t" + "sra %[temp2], %[temp2], 27 \n\t" + "sra %[temp1], %[temp1], 27 \n\t" + "addu %[p0], %[p0], %[temp1] \n\t" + "subu %[q0], %[q0], %[temp2] \n\t" + "lbux %[temp1], %[p0](%[VP8kclip1]) \n\t" + "lbux %[temp2], %[q0](%[VP8kclip1]) \n\t" + "sb %[temp2], 0(%[p]) \n\t" + "sb %[temp1], 0(%[pTemp0]) \n\t" + "0: \n\t" + "subu %[size], %[size], 1 \n\t" + "bgtz %[size], 2b \n\t" + " addu %[p], %[p], %[vstride] \n\t" + "3: \n\t" + ".set pop \n\t" + : [p0]"=&r"(p0), [q0]"=&r"(q0), [p1]"=&r"(p1), [q1]"=&r"(q1), + [p2]"=&r"(p2), [q2]"=&r"(q2), [p3]"=&r"(p3), [q3]"=&r"(q3), + [step2]"=&r"(step2), [step1]"=&r"(step1), [temp1]"=&r"(temp1), + [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), + [pTemp0]"=&r"(pTemp0), [pTemp1]"=&r"(pTemp1), [p]"+&r"(p), + [size]"+&r"(size) + : [vstride]"r"(vstride), [ithresh]"r"(ithresh), + [hev_thresh]"r"(hev_thresh), [hstride]"r"(hstride), + [VP8kclip1]"r"(VP8kclip1), [thresh2]"r"(thresh2) + : "memory" + ); +} + +// on macroblock edges +static void VFilter16(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop26(p, stride, 1, 16, thresh, ithresh, hev_thresh); +} + +static void HFilter16(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop26(p, 1, stride, 16, thresh, ithresh, hev_thresh); +} + +// 8-pixels wide variant, for chroma filtering +static void VFilter8(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop26(u, stride, 1, 8, thresh, ithresh, hev_thresh); + FilterLoop26(v, stride, 1, 8, thresh, ithresh, hev_thresh); +} + +static void HFilter8(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop26(u, 1, stride, 8, thresh, ithresh, hev_thresh); + FilterLoop26(v, 1, stride, 8, thresh, ithresh, hev_thresh); +} + +// on three inner edges +static void VFilter16i(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + int k; + for (k = 3; k > 0; --k) { + p += 4 * stride; + FilterLoop24(p, stride, 1, 16, thresh, ithresh, hev_thresh); + } +} + +static void HFilter16i(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + int k; + for (k = 3; k > 0; --k) { + p += 4; + FilterLoop24(p, 1, stride, 16, thresh, ithresh, hev_thresh); + } +} + +static void VFilter8i(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop24(u + 4 * stride, stride, 1, 8, thresh, ithresh, hev_thresh); + FilterLoop24(v + 4 * stride, stride, 1, 8, thresh, ithresh, hev_thresh); +} + +static void HFilter8i(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + FilterLoop24(u + 4, 1, stride, 8, thresh, ithresh, hev_thresh); + FilterLoop24(v + 4, 1, stride, 8, thresh, ithresh, hev_thresh); +} + +#undef MUL + +//------------------------------------------------------------------------------ +// Simple In-loop filtering (Paragraph 15.2) + +static void SimpleVFilter16(uint8_t* p, int stride, int thresh) { + int i; + const int thresh2 = 2 * thresh + 1; + int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8; + uint8_t* p1 = p - stride; + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "li %[i], 16 \n\t" + "0: \n\t" + "negu %[temp4], %[stride] \n\t" + "sll %[temp5], %[temp4], 1 \n\t" + "lbu %[temp2], 0(%[p]) \n\t" + "lbux %[temp3], %[stride](%[p]) \n\t" + "lbux %[temp1], %[temp4](%[p]) \n\t" + "lbux %[temp0], %[temp5](%[p]) \n\t" + "subu %[temp7], %[temp1], %[temp2] \n\t" + "subu %[temp6], %[temp0], %[temp3] \n\t" + "absq_s.w %[temp4], %[temp7] \n\t" + "absq_s.w %[temp5], %[temp6] \n\t" + "sll %[temp4], %[temp4], 2 \n\t" + "subu %[temp5], %[temp5], %[thresh2] \n\t" + "addu %[temp5], %[temp4], %[temp5] \n\t" + "negu %[temp8], %[temp7] \n\t" + "bgtz %[temp5], 1f \n\t" + " addiu %[i], %[i], -1 \n\t" + "sll %[temp4], %[temp8], 1 \n\t" + "shll_s.w %[temp5], %[temp6], 24 \n\t" + "addu %[temp3], %[temp4], %[temp8] \n\t" + "sra %[temp5], %[temp5], 24 \n\t" + "addu %[temp3], %[temp3], %[temp5] \n\t" + "addiu %[temp7], %[temp3], 3 \n\t" + "sra %[temp7], %[temp7], 3 \n\t" + "shra_r.w %[temp8], %[temp3], 3 \n\t" + "shll_s.w %[temp0], %[temp7], 27 \n\t" + "shll_s.w %[temp4], %[temp8], 27 \n\t" + "sra %[temp0], %[temp0], 27 \n\t" + "sra %[temp4], %[temp4], 27 \n\t" + "addu %[temp7], %[temp1], %[temp0] \n\t" + "subu %[temp2], %[temp2], %[temp4] \n\t" + "lbux %[temp3], %[temp7](%[VP8kclip1]) \n\t" + "lbux %[temp4], %[temp2](%[VP8kclip1]) \n\t" + "sb %[temp3], 0(%[p1]) \n\t" + "sb %[temp4], 0(%[p]) \n\t" + "1: \n\t" + "addiu %[p1], %[p1], 1 \n\t" + "bgtz %[i], 0b \n\t" + " addiu %[p], %[p], 1 \n\t" + " .set pop \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8), + [p]"+&r"(p), [i]"=&r"(i), [p1]"+&r"(p1) + : [stride]"r"(stride), [VP8kclip1]"r"(VP8kclip1), [thresh2]"r"(thresh2) + : "memory" + ); +} + +// TEMP0 = SRC[A + A1 * BPS] +// TEMP1 = SRC[B + B1 * BPS] +// TEMP2 = SRC[C + C1 * BPS] +// TEMP3 = SRC[D + D1 * BPS] +#define LOAD_4_BYTES(TEMP0, TEMP1, TEMP2, TEMP3, \ + A, A1, B, B1, C, C1, D, D1, SRC) \ + "lbu %[" #TEMP0 "], " #A "+" #A1 "*" XSTR(BPS) "(%[" #SRC "]) \n\t" \ + "lbu %[" #TEMP1 "], " #B "+" #B1 "*" XSTR(BPS) "(%[" #SRC "]) \n\t" \ + "lbu %[" #TEMP2 "], " #C "+" #C1 "*" XSTR(BPS) "(%[" #SRC "]) \n\t" \ + "lbu %[" #TEMP3 "], " #D "+" #D1 "*" XSTR(BPS) "(%[" #SRC "]) \n\t" \ + +static void SimpleHFilter16(uint8_t* p, int stride, int thresh) { + int i; + const int thresh2 = 2 * thresh + 1; + int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8; + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "li %[i], 16 \n\t" + "0: \n\t" + LOAD_4_BYTES(temp0, temp1, temp2, temp3, -2, 0, -1, 0, 0, 0, 1, 0, p) + "subu %[temp7], %[temp1], %[temp2] \n\t" + "subu %[temp6], %[temp0], %[temp3] \n\t" + "absq_s.w %[temp4], %[temp7] \n\t" + "absq_s.w %[temp5], %[temp6] \n\t" + "sll %[temp4], %[temp4], 2 \n\t" + "addu %[temp5], %[temp4], %[temp5] \n\t" + "subu %[temp5], %[temp5], %[thresh2] \n\t" + "negu %[temp8], %[temp7] \n\t" + "bgtz %[temp5], 1f \n\t" + " addiu %[i], %[i], -1 \n\t" + "sll %[temp4], %[temp8], 1 \n\t" + "shll_s.w %[temp5], %[temp6], 24 \n\t" + "addu %[temp3], %[temp4], %[temp8] \n\t" + "sra %[temp5], %[temp5], 24 \n\t" + "addu %[temp3], %[temp3], %[temp5] \n\t" + "addiu %[temp7], %[temp3], 3 \n\t" + "sra %[temp7], %[temp7], 3 \n\t" + "shra_r.w %[temp8], %[temp3], 3 \n\t" + "shll_s.w %[temp0], %[temp7], 27 \n\t" + "shll_s.w %[temp4], %[temp8], 27 \n\t" + "sra %[temp0], %[temp0], 27 \n\t" + "sra %[temp4], %[temp4], 27 \n\t" + "addu %[temp7], %[temp1], %[temp0] \n\t" + "subu %[temp2], %[temp2], %[temp4] \n\t" + "lbux %[temp3], %[temp7](%[VP8kclip1]) \n\t" + "lbux %[temp4], %[temp2](%[VP8kclip1]) \n\t" + "sb %[temp3], -1(%[p]) \n\t" + "sb %[temp4], 0(%[p]) \n\t" + "1: \n\t" + "bgtz %[i], 0b \n\t" + " addu %[p], %[p], %[stride] \n\t" + ".set pop \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8), + [p]"+&r"(p), [i]"=&r"(i) + : [stride]"r"(stride), [VP8kclip1]"r"(VP8kclip1), [thresh2]"r"(thresh2) + : "memory" + ); +} + +static void SimpleVFilter16i(uint8_t* p, int stride, int thresh) { + int k; + for (k = 3; k > 0; --k) { + p += 4 * stride; + SimpleVFilter16(p, stride, thresh); + } +} + +static void SimpleHFilter16i(uint8_t* p, int stride, int thresh) { + int k; + for (k = 3; k > 0; --k) { + p += 4; + SimpleHFilter16(p, stride, thresh); + } +} + +// DST[A * BPS] = TEMP0 +// DST[B + C * BPS] = TEMP1 +#define STORE_8_BYTES(TEMP0, TEMP1, A, B, C, DST) \ + "usw %[" #TEMP0 "], " #A "*" XSTR(BPS) "(%[" #DST "]) \n\t" \ + "usw %[" #TEMP1 "], " #B "+" #C "*" XSTR(BPS) "(%[" #DST "]) \n\t" + +static void VE4(uint8_t* dst) { // vertical + const uint8_t* top = dst - BPS; + int temp0, temp1, temp2, temp3, temp4, temp5, temp6; + __asm__ volatile ( + "ulw %[temp0], -1(%[top]) \n\t" + "ulh %[temp1], 3(%[top]) \n\t" + "preceu.ph.qbr %[temp2], %[temp0] \n\t" + "preceu.ph.qbl %[temp3], %[temp0] \n\t" + "preceu.ph.qbr %[temp4], %[temp1] \n\t" + "packrl.ph %[temp5], %[temp3], %[temp2] \n\t" + "packrl.ph %[temp6], %[temp4], %[temp3] \n\t" + "shll.ph %[temp5], %[temp5], 1 \n\t" + "shll.ph %[temp6], %[temp6], 1 \n\t" + "addq.ph %[temp2], %[temp5], %[temp2] \n\t" + "addq.ph %[temp6], %[temp6], %[temp4] \n\t" + "addq.ph %[temp2], %[temp2], %[temp3] \n\t" + "addq.ph %[temp6], %[temp6], %[temp3] \n\t" + "shra_r.ph %[temp2], %[temp2], 2 \n\t" + "shra_r.ph %[temp6], %[temp6], 2 \n\t" + "precr.qb.ph %[temp4], %[temp6], %[temp2] \n\t" + STORE_8_BYTES(temp4, temp4, 0, 0, 1, dst) + STORE_8_BYTES(temp4, temp4, 2, 0, 3, dst) + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6) + : [top]"r"(top), [dst]"r"(dst) + : "memory" + ); +} + +static void DC4(uint8_t* dst) { // DC + int temp0, temp1, temp2, temp3, temp4; + __asm__ volatile ( + "ulw %[temp0], -1*" XSTR(BPS) "(%[dst]) \n\t" + LOAD_4_BYTES(temp1, temp2, temp3, temp4, -1, 0, -1, 1, -1, 2, -1, 3, dst) + "ins %[temp1], %[temp2], 8, 8 \n\t" + "ins %[temp1], %[temp3], 16, 8 \n\t" + "ins %[temp1], %[temp4], 24, 8 \n\t" + "raddu.w.qb %[temp0], %[temp0] \n\t" + "raddu.w.qb %[temp1], %[temp1] \n\t" + "addu %[temp0], %[temp0], %[temp1] \n\t" + "shra_r.w %[temp0], %[temp0], 3 \n\t" + "replv.qb %[temp0], %[temp0] \n\t" + STORE_8_BYTES(temp0, temp0, 0, 0, 1, dst) + STORE_8_BYTES(temp0, temp0, 2, 0, 3, dst) + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4) + : [dst]"r"(dst) + : "memory" + ); +} + +static void RD4(uint8_t* dst) { // Down-right + int temp0, temp1, temp2, temp3, temp4; + int temp5, temp6, temp7, temp8; + __asm__ volatile ( + LOAD_4_BYTES(temp0, temp1, temp2, temp3, -1, 0, -1, 1, -1, 2, -1, 3, dst) + "ulw %[temp7], -1-" XSTR(BPS) "(%[dst]) \n\t" + "ins %[temp1], %[temp0], 16, 16 \n\t" + "preceu.ph.qbr %[temp5], %[temp7] \n\t" + "ins %[temp2], %[temp1], 16, 16 \n\t" + "preceu.ph.qbl %[temp4], %[temp7] \n\t" + "ins %[temp3], %[temp2], 16, 16 \n\t" + "shll.ph %[temp2], %[temp2], 1 \n\t" + "addq.ph %[temp3], %[temp3], %[temp1] \n\t" + "packrl.ph %[temp6], %[temp5], %[temp1] \n\t" + "addq.ph %[temp3], %[temp3], %[temp2] \n\t" + "addq.ph %[temp1], %[temp1], %[temp5] \n\t" + "shll.ph %[temp6], %[temp6], 1 \n\t" + "addq.ph %[temp1], %[temp1], %[temp6] \n\t" + "packrl.ph %[temp0], %[temp4], %[temp5] \n\t" + "addq.ph %[temp8], %[temp5], %[temp4] \n\t" + "shra_r.ph %[temp3], %[temp3], 2 \n\t" + "shll.ph %[temp0], %[temp0], 1 \n\t" + "shra_r.ph %[temp1], %[temp1], 2 \n\t" + "addq.ph %[temp8], %[temp0], %[temp8] \n\t" + "lbu %[temp5], 3-" XSTR(BPS) "(%[dst]) \n\t" + "precrq.ph.w %[temp7], %[temp7], %[temp7] \n\t" + "shra_r.ph %[temp8], %[temp8], 2 \n\t" + "ins %[temp7], %[temp5], 0, 8 \n\t" + "precr.qb.ph %[temp2], %[temp1], %[temp3] \n\t" + "raddu.w.qb %[temp4], %[temp7] \n\t" + "precr.qb.ph %[temp6], %[temp8], %[temp1] \n\t" + "shra_r.w %[temp4], %[temp4], 2 \n\t" + STORE_8_BYTES(temp2, temp6, 3, 0, 1, dst) + "prepend %[temp2], %[temp8], 8 \n\t" + "prepend %[temp6], %[temp4], 8 \n\t" + STORE_8_BYTES(temp2, temp6, 2, 0, 0, dst) + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8) + : [dst]"r"(dst) + : "memory" + ); +} + +// TEMP0 = SRC[A * BPS] +// TEMP1 = SRC[B + C * BPS] +#define LOAD_8_BYTES(TEMP0, TEMP1, A, B, C, SRC) \ + "ulw %[" #TEMP0 "], " #A "*" XSTR(BPS) "(%[" #SRC "]) \n\t" \ + "ulw %[" #TEMP1 "], " #B "+" #C "*" XSTR(BPS) "(%[" #SRC "]) \n\t" + +static void LD4(uint8_t* dst) { // Down-Left + int temp0, temp1, temp2, temp3, temp4; + int temp5, temp6, temp7, temp8, temp9; + __asm__ volatile ( + LOAD_8_BYTES(temp0, temp1, -1, 4, -1, dst) + "preceu.ph.qbl %[temp2], %[temp0] \n\t" + "preceu.ph.qbr %[temp3], %[temp0] \n\t" + "preceu.ph.qbr %[temp4], %[temp1] \n\t" + "preceu.ph.qbl %[temp5], %[temp1] \n\t" + "packrl.ph %[temp6], %[temp2], %[temp3] \n\t" + "packrl.ph %[temp7], %[temp4], %[temp2] \n\t" + "packrl.ph %[temp8], %[temp5], %[temp4] \n\t" + "shll.ph %[temp6], %[temp6], 1 \n\t" + "addq.ph %[temp9], %[temp2], %[temp6] \n\t" + "shll.ph %[temp7], %[temp7], 1 \n\t" + "addq.ph %[temp9], %[temp9], %[temp3] \n\t" + "shll.ph %[temp8], %[temp8], 1 \n\t" + "shra_r.ph %[temp9], %[temp9], 2 \n\t" + "addq.ph %[temp3], %[temp4], %[temp7] \n\t" + "addq.ph %[temp0], %[temp5], %[temp8] \n\t" + "addq.ph %[temp3], %[temp3], %[temp2] \n\t" + "addq.ph %[temp0], %[temp0], %[temp4] \n\t" + "shra_r.ph %[temp3], %[temp3], 2 \n\t" + "shra_r.ph %[temp0], %[temp0], 2 \n\t" + "srl %[temp1], %[temp1], 24 \n\t" + "sll %[temp1], %[temp1], 1 \n\t" + "raddu.w.qb %[temp5], %[temp5] \n\t" + "precr.qb.ph %[temp9], %[temp3], %[temp9] \n\t" + "precr.qb.ph %[temp3], %[temp0], %[temp3] \n\t" + "addu %[temp1], %[temp1], %[temp5] \n\t" + "shra_r.w %[temp1], %[temp1], 2 \n\t" + STORE_8_BYTES(temp9, temp3, 0, 0, 2, dst) + "prepend %[temp9], %[temp0], 8 \n\t" + "prepend %[temp3], %[temp1], 8 \n\t" + STORE_8_BYTES(temp9, temp3, 1, 0, 3, dst) + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8), + [temp9]"=&r"(temp9) + : [dst]"r"(dst) + : "memory" + ); +} + +//------------------------------------------------------------------------------ +// Chroma + +static void DC8uv(uint8_t* dst) { // DC + int temp0, temp1, temp2, temp3, temp4; + int temp5, temp6, temp7, temp8, temp9; + __asm__ volatile ( + LOAD_8_BYTES(temp0, temp1, -1, 4, -1, dst) + LOAD_4_BYTES(temp2, temp3, temp4, temp5, -1, 0, -1, 1, -1, 2, -1, 3, dst) + LOAD_4_BYTES(temp6, temp7, temp8, temp9, -1, 4, -1, 5, -1, 6, -1, 7, dst) + "raddu.w.qb %[temp0], %[temp0] \n\t" + "raddu.w.qb %[temp1], %[temp1] \n\t" + "addu %[temp2], %[temp2], %[temp3] \n\t" + "addu %[temp4], %[temp4], %[temp5] \n\t" + "addu %[temp6], %[temp6], %[temp7] \n\t" + "addu %[temp8], %[temp8], %[temp9] \n\t" + "addu %[temp0], %[temp0], %[temp1] \n\t" + "addu %[temp2], %[temp2], %[temp4] \n\t" + "addu %[temp6], %[temp6], %[temp8] \n\t" + "addu %[temp0], %[temp0], %[temp2] \n\t" + "addu %[temp0], %[temp0], %[temp6] \n\t" + "shra_r.w %[temp0], %[temp0], 4 \n\t" + "replv.qb %[temp0], %[temp0] \n\t" + STORE_8_BYTES(temp0, temp0, 0, 4, 0, dst) + STORE_8_BYTES(temp0, temp0, 1, 4, 1, dst) + STORE_8_BYTES(temp0, temp0, 2, 4, 2, dst) + STORE_8_BYTES(temp0, temp0, 3, 4, 3, dst) + STORE_8_BYTES(temp0, temp0, 4, 4, 4, dst) + STORE_8_BYTES(temp0, temp0, 5, 4, 5, dst) + STORE_8_BYTES(temp0, temp0, 6, 4, 6, dst) + STORE_8_BYTES(temp0, temp0, 7, 4, 7, dst) + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8), + [temp9]"=&r"(temp9) + : [dst]"r"(dst) + : "memory" + ); +} + +static void DC8uvNoLeft(uint8_t* dst) { // DC with no left samples + int temp0, temp1; + __asm__ volatile ( + LOAD_8_BYTES(temp0, temp1, -1, 4, -1, dst) + "raddu.w.qb %[temp0], %[temp0] \n\t" + "raddu.w.qb %[temp1], %[temp1] \n\t" + "addu %[temp0], %[temp0], %[temp1] \n\t" + "shra_r.w %[temp0], %[temp0], 3 \n\t" + "replv.qb %[temp0], %[temp0] \n\t" + STORE_8_BYTES(temp0, temp0, 0, 4, 0, dst) + STORE_8_BYTES(temp0, temp0, 1, 4, 1, dst) + STORE_8_BYTES(temp0, temp0, 2, 4, 2, dst) + STORE_8_BYTES(temp0, temp0, 3, 4, 3, dst) + STORE_8_BYTES(temp0, temp0, 4, 4, 4, dst) + STORE_8_BYTES(temp0, temp0, 5, 4, 5, dst) + STORE_8_BYTES(temp0, temp0, 6, 4, 6, dst) + STORE_8_BYTES(temp0, temp0, 7, 4, 7, dst) + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1) + : [dst]"r"(dst) + : "memory" + ); +} + +static void DC8uvNoTop(uint8_t* dst) { // DC with no top samples + int temp0, temp1, temp2, temp3, temp4; + int temp5, temp6, temp7, temp8; + __asm__ volatile ( + LOAD_4_BYTES(temp2, temp3, temp4, temp5, -1, 0, -1, 1, -1, 2, -1, 3, dst) + LOAD_4_BYTES(temp6, temp7, temp8, temp1, -1, 4, -1, 5, -1, 6, -1, 7, dst) + "addu %[temp2], %[temp2], %[temp3] \n\t" + "addu %[temp4], %[temp4], %[temp5] \n\t" + "addu %[temp6], %[temp6], %[temp7] \n\t" + "addu %[temp8], %[temp8], %[temp1] \n\t" + "addu %[temp2], %[temp2], %[temp4] \n\t" + "addu %[temp6], %[temp6], %[temp8] \n\t" + "addu %[temp0], %[temp6], %[temp2] \n\t" + "shra_r.w %[temp0], %[temp0], 3 \n\t" + "replv.qb %[temp0], %[temp0] \n\t" + STORE_8_BYTES(temp0, temp0, 0, 4, 0, dst) + STORE_8_BYTES(temp0, temp0, 1, 4, 1, dst) + STORE_8_BYTES(temp0, temp0, 2, 4, 2, dst) + STORE_8_BYTES(temp0, temp0, 3, 4, 3, dst) + STORE_8_BYTES(temp0, temp0, 4, 4, 4, dst) + STORE_8_BYTES(temp0, temp0, 5, 4, 5, dst) + STORE_8_BYTES(temp0, temp0, 6, 4, 6, dst) + STORE_8_BYTES(temp0, temp0, 7, 4, 7, dst) + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8) + : [dst]"r"(dst) + : "memory" + ); +} + +#undef LOAD_8_BYTES +#undef STORE_8_BYTES +#undef LOAD_4_BYTES + +#define CLIPPING(SIZE) \ + "preceu.ph.qbl %[temp2], %[temp0] \n\t" \ + "preceu.ph.qbr %[temp0], %[temp0] \n\t" \ +".if " #SIZE " == 8 \n\t" \ + "preceu.ph.qbl %[temp3], %[temp1] \n\t" \ + "preceu.ph.qbr %[temp1], %[temp1] \n\t" \ +".endif \n\t" \ + "addu.ph %[temp2], %[temp2], %[dst_1] \n\t" \ + "addu.ph %[temp0], %[temp0], %[dst_1] \n\t" \ +".if " #SIZE " == 8 \n\t" \ + "addu.ph %[temp3], %[temp3], %[dst_1] \n\t" \ + "addu.ph %[temp1], %[temp1], %[dst_1] \n\t" \ +".endif \n\t" \ + "shll_s.ph %[temp2], %[temp2], 7 \n\t" \ + "shll_s.ph %[temp0], %[temp0], 7 \n\t" \ +".if " #SIZE " == 8 \n\t" \ + "shll_s.ph %[temp3], %[temp3], 7 \n\t" \ + "shll_s.ph %[temp1], %[temp1], 7 \n\t" \ +".endif \n\t" \ + "precrqu_s.qb.ph %[temp0], %[temp2], %[temp0] \n\t" \ +".if " #SIZE " == 8 \n\t" \ + "precrqu_s.qb.ph %[temp1], %[temp3], %[temp1] \n\t" \ +".endif \n\t" + + +#define CLIP_8B_TO_DST(DST, TOP, SIZE) do { \ + int dst_1 = ((int)(DST)[-1] << 16) + (DST)[-1]; \ + int temp0, temp1, temp2, temp3; \ + __asm__ volatile ( \ + ".if " #SIZE " < 8 \n\t" \ + "ulw %[temp0], 0(%[top]) \n\t" \ + "subu.ph %[dst_1], %[dst_1], %[top_1] \n\t" \ + CLIPPING(4) \ + "usw %[temp0], 0(%[dst]) \n\t" \ + ".else \n\t" \ + "ulw %[temp0], 0(%[top]) \n\t" \ + "ulw %[temp1], 4(%[top]) \n\t" \ + "subu.ph %[dst_1], %[dst_1], %[top_1] \n\t" \ + CLIPPING(8) \ + "usw %[temp0], 0(%[dst]) \n\t" \ + "usw %[temp1], 4(%[dst]) \n\t" \ + ".if " #SIZE " == 16 \n\t" \ + "ulw %[temp0], 8(%[top]) \n\t" \ + "ulw %[temp1], 12(%[top]) \n\t" \ + CLIPPING(8) \ + "usw %[temp0], 8(%[dst]) \n\t" \ + "usw %[temp1], 12(%[dst]) \n\t" \ + ".endif \n\t" \ + ".endif \n\t" \ + : [dst_1]"+&r"(dst_1), [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), \ + [temp2]"=&r"(temp2), [temp3]"=&r"(temp3) \ + : [top_1]"r"(top_1), [top]"r"((TOP)), [dst]"r"((DST)) \ + : "memory" \ + ); \ +} while (0) + +#define CLIP_TO_DST(DST, SIZE) do { \ + int y; \ + const uint8_t* top = (DST) - BPS; \ + const int top_1 = ((int)top[-1] << 16) + top[-1]; \ + for (y = 0; y < (SIZE); ++y) { \ + CLIP_8B_TO_DST((DST), top, (SIZE)); \ + (DST) += BPS; \ + } \ +} while (0) + +#define TRUE_MOTION(DST, SIZE) \ +static void TrueMotion##SIZE(uint8_t* (DST)) { \ + CLIP_TO_DST((DST), (SIZE)); \ +} + +TRUE_MOTION(dst, 4) +TRUE_MOTION(dst, 8) +TRUE_MOTION(dst, 16) + +#undef TRUE_MOTION +#undef CLIP_TO_DST +#undef CLIP_8B_TO_DST +#undef CLIPPING + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8DspInitMIPSdspR2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8DspInitMIPSdspR2(void) { + VP8TransformDC = TransformDC; + VP8TransformAC3 = TransformAC3; + VP8Transform = TransformTwo; + + VP8VFilter16 = VFilter16; + VP8HFilter16 = HFilter16; + VP8VFilter8 = VFilter8; + VP8HFilter8 = HFilter8; + VP8VFilter16i = VFilter16i; + VP8HFilter16i = HFilter16i; + VP8VFilter8i = VFilter8i; + VP8HFilter8i = HFilter8i; + VP8SimpleVFilter16 = SimpleVFilter16; + VP8SimpleHFilter16 = SimpleHFilter16; + VP8SimpleVFilter16i = SimpleVFilter16i; + VP8SimpleHFilter16i = SimpleHFilter16i; + + VP8PredLuma4[0] = DC4; + VP8PredLuma4[1] = TrueMotion4; + VP8PredLuma4[2] = VE4; + VP8PredLuma4[4] = RD4; + VP8PredLuma4[6] = LD4; + + VP8PredChroma8[0] = DC8uv; + VP8PredChroma8[1] = TrueMotion8; + VP8PredChroma8[4] = DC8uvNoTop; + VP8PredChroma8[5] = DC8uvNoLeft; + + VP8PredLuma16[1] = TrueMotion16; +} + +#else // !WEBP_USE_MIPS_DSP_R2 + +WEBP_DSP_INIT_STUB(VP8DspInitMIPSdspR2) + +#endif // WEBP_USE_MIPS_DSP_R2 diff --git a/thirdparty/libwebp/dsp/dec_msa.c b/thirdparty/libwebp/dsp/dec_msa.c new file mode 100644 index 0000000000..f76055cab0 --- /dev/null +++ b/thirdparty/libwebp/dsp/dec_msa.c @@ -0,0 +1,172 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MSA version of dsp functions +// +// Author(s): Prashant Patil (prashant.patil@imgtec.com) + + +#include "./dsp.h" + +#if defined(WEBP_USE_MSA) + +#include "./msa_macro.h" + +//------------------------------------------------------------------------------ +// Transforms + +#define IDCT_1D_W(in0, in1, in2, in3, out0, out1, out2, out3) { \ + v4i32 a1_m, b1_m, c1_m, d1_m; \ + v4i32 c_tmp1_m, c_tmp2_m, d_tmp1_m, d_tmp2_m; \ + const v4i32 cospi8sqrt2minus1 = __msa_fill_w(20091); \ + const v4i32 sinpi8sqrt2 = __msa_fill_w(35468); \ + \ + a1_m = in0 + in2; \ + b1_m = in0 - in2; \ + c_tmp1_m = (in1 * sinpi8sqrt2) >> 16; \ + c_tmp2_m = in3 + ((in3 * cospi8sqrt2minus1) >> 16); \ + c1_m = c_tmp1_m - c_tmp2_m; \ + d_tmp1_m = in1 + ((in1 * cospi8sqrt2minus1) >> 16); \ + d_tmp2_m = (in3 * sinpi8sqrt2) >> 16; \ + d1_m = d_tmp1_m + d_tmp2_m; \ + BUTTERFLY_4(a1_m, b1_m, c1_m, d1_m, out0, out1, out2, out3); \ +} +#define MULT1(a) ((((a) * 20091) >> 16) + (a)) +#define MULT2(a) (((a) * 35468) >> 16) + +static void TransformOne(const int16_t* in, uint8_t* dst) { + v8i16 input0, input1; + v4i32 in0, in1, in2, in3, hz0, hz1, hz2, hz3, vt0, vt1, vt2, vt3; + v4i32 res0, res1, res2, res3; + const v16i8 zero = { 0 }; + v16i8 dest0, dest1, dest2, dest3; + + LD_SH2(in, 8, input0, input1); + UNPCK_SH_SW(input0, in0, in1); + UNPCK_SH_SW(input1, in2, in3); + IDCT_1D_W(in0, in1, in2, in3, hz0, hz1, hz2, hz3); + TRANSPOSE4x4_SW_SW(hz0, hz1, hz2, hz3, hz0, hz1, hz2, hz3); + IDCT_1D_W(hz0, hz1, hz2, hz3, vt0, vt1, vt2, vt3); + SRARI_W4_SW(vt0, vt1, vt2, vt3, 3); + TRANSPOSE4x4_SW_SW(vt0, vt1, vt2, vt3, vt0, vt1, vt2, vt3); + LD_SB4(dst, BPS, dest0, dest1, dest2, dest3); + ILVR_B4_SW(zero, dest0, zero, dest1, zero, dest2, zero, dest3, + res0, res1, res2, res3); + ILVR_H4_SW(zero, res0, zero, res1, zero, res2, zero, res3, + res0, res1, res2, res3); + ADD4(res0, vt0, res1, vt1, res2, vt2, res3, vt3, res0, res1, res2, res3); + CLIP_SW4_0_255(res0, res1, res2, res3); + PCKEV_B2_SW(res0, res1, res2, res3, vt0, vt1); + res0 = (v4i32)__msa_pckev_b((v16i8)vt0, (v16i8)vt1); + ST4x4_UB(res0, res0, 3, 2, 1, 0, dst, BPS); +} + +static void TransformTwo(const int16_t* in, uint8_t* dst, int do_two) { + TransformOne(in, dst); + if (do_two) { + TransformOne(in + 16, dst + 4); + } +} + +static void TransformWHT(const int16_t* in, int16_t* out) { + v8i16 input0, input1; + const v8i16 mask0 = { 0, 1, 2, 3, 8, 9, 10, 11 }; + const v8i16 mask1 = { 4, 5, 6, 7, 12, 13, 14, 15 }; + const v8i16 mask2 = { 0, 4, 8, 12, 1, 5, 9, 13 }; + const v8i16 mask3 = { 3, 7, 11, 15, 2, 6, 10, 14 }; + v8i16 tmp0, tmp1, tmp2, tmp3; + v8i16 out0, out1; + + LD_SH2(in, 8, input0, input1); + input1 = SLDI_SH(input1, input1, 8); + tmp0 = input0 + input1; + tmp1 = input0 - input1; + VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask0, mask1, tmp2, tmp3); + out0 = tmp2 + tmp3; + out1 = tmp2 - tmp3; + VSHF_H2_SH(out0, out1, out0, out1, mask2, mask3, input0, input1); + tmp0 = input0 + input1; + tmp1 = input0 - input1; + VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask0, mask1, tmp2, tmp3); + tmp0 = tmp2 + tmp3; + tmp1 = tmp2 - tmp3; + ADDVI_H2_SH(tmp0, 3, tmp1, 3, out0, out1); + SRAI_H2_SH(out0, out1, 3); + out[0] = __msa_copy_s_h(out0, 0); + out[16] = __msa_copy_s_h(out0, 4); + out[32] = __msa_copy_s_h(out1, 0); + out[48] = __msa_copy_s_h(out1, 4); + out[64] = __msa_copy_s_h(out0, 1); + out[80] = __msa_copy_s_h(out0, 5); + out[96] = __msa_copy_s_h(out1, 1); + out[112] = __msa_copy_s_h(out1, 5); + out[128] = __msa_copy_s_h(out0, 2); + out[144] = __msa_copy_s_h(out0, 6); + out[160] = __msa_copy_s_h(out1, 2); + out[176] = __msa_copy_s_h(out1, 6); + out[192] = __msa_copy_s_h(out0, 3); + out[208] = __msa_copy_s_h(out0, 7); + out[224] = __msa_copy_s_h(out1, 3); + out[240] = __msa_copy_s_h(out1, 7); +} + +static void TransformDC(const int16_t* in, uint8_t* dst) { + const int DC = (in[0] + 4) >> 3; + const v8i16 tmp0 = __msa_fill_h(DC); + ADDBLK_ST4x4_UB(tmp0, tmp0, tmp0, tmp0, dst, BPS); +} + +static void TransformAC3(const int16_t* in, uint8_t* dst) { + const int a = in[0] + 4; + const int c4 = MULT2(in[4]); + const int d4 = MULT1(in[4]); + const int in2 = MULT2(in[1]); + const int in3 = MULT1(in[1]); + v4i32 tmp0 = { 0 }; + v4i32 out0 = __msa_fill_w(a + d4); + v4i32 out1 = __msa_fill_w(a + c4); + v4i32 out2 = __msa_fill_w(a - c4); + v4i32 out3 = __msa_fill_w(a - d4); + v4i32 res0, res1, res2, res3; + const v4i32 zero = { 0 }; + v16u8 dest0, dest1, dest2, dest3; + + INSERT_W4_SW(in3, in2, -in2, -in3, tmp0); + ADD4(out0, tmp0, out1, tmp0, out2, tmp0, out3, tmp0, + out0, out1, out2, out3); + SRAI_W4_SW(out0, out1, out2, out3, 3); + LD_UB4(dst, BPS, dest0, dest1, dest2, dest3); + ILVR_B4_SW(zero, dest0, zero, dest1, zero, dest2, zero, dest3, + res0, res1, res2, res3); + ILVR_H4_SW(zero, res0, zero, res1, zero, res2, zero, res3, + res0, res1, res2, res3); + ADD4(res0, out0, res1, out1, res2, out2, res3, out3, res0, res1, res2, res3); + CLIP_SW4_0_255(res0, res1, res2, res3); + PCKEV_B2_SW(res0, res1, res2, res3, out0, out1); + res0 = (v4i32)__msa_pckev_b((v16i8)out0, (v16i8)out1); + ST4x4_UB(res0, res0, 3, 2, 1, 0, dst, BPS); +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8DspInitMSA(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8DspInitMSA(void) { + VP8TransformWHT = TransformWHT; + VP8Transform = TransformTwo; + VP8TransformDC = TransformDC; + VP8TransformAC3 = TransformAC3; +} + +#else // !WEBP_USE_MSA + +WEBP_DSP_INIT_STUB(VP8DspInitMSA) + +#endif // WEBP_USE_MSA diff --git a/thirdparty/libwebp/dsp/dec_neon.c b/thirdparty/libwebp/dsp/dec_neon.c new file mode 100644 index 0000000000..a63f43fe17 --- /dev/null +++ b/thirdparty/libwebp/dsp/dec_neon.c @@ -0,0 +1,1639 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// ARM NEON version of dsp functions and loop filtering. +// +// Authors: Somnath Banerjee (somnath@google.com) +// Johann Koenig (johannkoenig@google.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_NEON) + +#include "./neon.h" +#include "../dec/vp8i.h" + +//------------------------------------------------------------------------------ +// NxM Loading functions + +// Load/Store vertical edge +#define LOAD8x4(c1, c2, c3, c4, b1, b2, stride) \ + "vld4.8 {" #c1 "[0]," #c2 "[0]," #c3 "[0]," #c4 "[0]}," #b1 "," #stride "\n" \ + "vld4.8 {" #c1 "[1]," #c2 "[1]," #c3 "[1]," #c4 "[1]}," #b2 "," #stride "\n" \ + "vld4.8 {" #c1 "[2]," #c2 "[2]," #c3 "[2]," #c4 "[2]}," #b1 "," #stride "\n" \ + "vld4.8 {" #c1 "[3]," #c2 "[3]," #c3 "[3]," #c4 "[3]}," #b2 "," #stride "\n" \ + "vld4.8 {" #c1 "[4]," #c2 "[4]," #c3 "[4]," #c4 "[4]}," #b1 "," #stride "\n" \ + "vld4.8 {" #c1 "[5]," #c2 "[5]," #c3 "[5]," #c4 "[5]}," #b2 "," #stride "\n" \ + "vld4.8 {" #c1 "[6]," #c2 "[6]," #c3 "[6]," #c4 "[6]}," #b1 "," #stride "\n" \ + "vld4.8 {" #c1 "[7]," #c2 "[7]," #c3 "[7]," #c4 "[7]}," #b2 "," #stride "\n" + +#define STORE8x2(c1, c2, p, stride) \ + "vst2.8 {" #c1 "[0], " #c2 "[0]}," #p "," #stride " \n" \ + "vst2.8 {" #c1 "[1], " #c2 "[1]}," #p "," #stride " \n" \ + "vst2.8 {" #c1 "[2], " #c2 "[2]}," #p "," #stride " \n" \ + "vst2.8 {" #c1 "[3], " #c2 "[3]}," #p "," #stride " \n" \ + "vst2.8 {" #c1 "[4], " #c2 "[4]}," #p "," #stride " \n" \ + "vst2.8 {" #c1 "[5], " #c2 "[5]}," #p "," #stride " \n" \ + "vst2.8 {" #c1 "[6], " #c2 "[6]}," #p "," #stride " \n" \ + "vst2.8 {" #c1 "[7], " #c2 "[7]}," #p "," #stride " \n" + +#if !defined(WORK_AROUND_GCC) + +// This intrinsics version makes gcc-4.6.3 crash during Load4x??() compilation +// (register alloc, probably). The variants somewhat mitigate the problem, but +// not quite. HFilter16i() remains problematic. +static WEBP_INLINE uint8x8x4_t Load4x8(const uint8_t* const src, int stride) { + const uint8x8_t zero = vdup_n_u8(0); + uint8x8x4_t out; + INIT_VECTOR4(out, zero, zero, zero, zero); + out = vld4_lane_u8(src + 0 * stride, out, 0); + out = vld4_lane_u8(src + 1 * stride, out, 1); + out = vld4_lane_u8(src + 2 * stride, out, 2); + out = vld4_lane_u8(src + 3 * stride, out, 3); + out = vld4_lane_u8(src + 4 * stride, out, 4); + out = vld4_lane_u8(src + 5 * stride, out, 5); + out = vld4_lane_u8(src + 6 * stride, out, 6); + out = vld4_lane_u8(src + 7 * stride, out, 7); + return out; +} + +static WEBP_INLINE void Load4x16(const uint8_t* const src, int stride, + uint8x16_t* const p1, uint8x16_t* const p0, + uint8x16_t* const q0, uint8x16_t* const q1) { + // row0 = p1[0..7]|p0[0..7]|q0[0..7]|q1[0..7] + // row8 = p1[8..15]|p0[8..15]|q0[8..15]|q1[8..15] + const uint8x8x4_t row0 = Load4x8(src - 2 + 0 * stride, stride); + const uint8x8x4_t row8 = Load4x8(src - 2 + 8 * stride, stride); + *p1 = vcombine_u8(row0.val[0], row8.val[0]); + *p0 = vcombine_u8(row0.val[1], row8.val[1]); + *q0 = vcombine_u8(row0.val[2], row8.val[2]); + *q1 = vcombine_u8(row0.val[3], row8.val[3]); +} + +#else // WORK_AROUND_GCC + +#define LOADQ_LANE_32b(VALUE, LANE) do { \ + (VALUE) = vld1q_lane_u32((const uint32_t*)src, (VALUE), (LANE)); \ + src += stride; \ +} while (0) + +static WEBP_INLINE void Load4x16(const uint8_t* src, int stride, + uint8x16_t* const p1, uint8x16_t* const p0, + uint8x16_t* const q0, uint8x16_t* const q1) { + const uint32x4_t zero = vdupq_n_u32(0); + uint32x4x4_t in; + INIT_VECTOR4(in, zero, zero, zero, zero); + src -= 2; + LOADQ_LANE_32b(in.val[0], 0); + LOADQ_LANE_32b(in.val[1], 0); + LOADQ_LANE_32b(in.val[2], 0); + LOADQ_LANE_32b(in.val[3], 0); + LOADQ_LANE_32b(in.val[0], 1); + LOADQ_LANE_32b(in.val[1], 1); + LOADQ_LANE_32b(in.val[2], 1); + LOADQ_LANE_32b(in.val[3], 1); + LOADQ_LANE_32b(in.val[0], 2); + LOADQ_LANE_32b(in.val[1], 2); + LOADQ_LANE_32b(in.val[2], 2); + LOADQ_LANE_32b(in.val[3], 2); + LOADQ_LANE_32b(in.val[0], 3); + LOADQ_LANE_32b(in.val[1], 3); + LOADQ_LANE_32b(in.val[2], 3); + LOADQ_LANE_32b(in.val[3], 3); + // Transpose four 4x4 parts: + { + const uint8x16x2_t row01 = vtrnq_u8(vreinterpretq_u8_u32(in.val[0]), + vreinterpretq_u8_u32(in.val[1])); + const uint8x16x2_t row23 = vtrnq_u8(vreinterpretq_u8_u32(in.val[2]), + vreinterpretq_u8_u32(in.val[3])); + const uint16x8x2_t row02 = vtrnq_u16(vreinterpretq_u16_u8(row01.val[0]), + vreinterpretq_u16_u8(row23.val[0])); + const uint16x8x2_t row13 = vtrnq_u16(vreinterpretq_u16_u8(row01.val[1]), + vreinterpretq_u16_u8(row23.val[1])); + *p1 = vreinterpretq_u8_u16(row02.val[0]); + *p0 = vreinterpretq_u8_u16(row13.val[0]); + *q0 = vreinterpretq_u8_u16(row02.val[1]); + *q1 = vreinterpretq_u8_u16(row13.val[1]); + } +} +#undef LOADQ_LANE_32b + +#endif // !WORK_AROUND_GCC + +static WEBP_INLINE void Load8x16(const uint8_t* const src, int stride, + uint8x16_t* const p3, uint8x16_t* const p2, + uint8x16_t* const p1, uint8x16_t* const p0, + uint8x16_t* const q0, uint8x16_t* const q1, + uint8x16_t* const q2, uint8x16_t* const q3) { + Load4x16(src - 2, stride, p3, p2, p1, p0); + Load4x16(src + 2, stride, q0, q1, q2, q3); +} + +static WEBP_INLINE void Load16x4(const uint8_t* const src, int stride, + uint8x16_t* const p1, uint8x16_t* const p0, + uint8x16_t* const q0, uint8x16_t* const q1) { + *p1 = vld1q_u8(src - 2 * stride); + *p0 = vld1q_u8(src - 1 * stride); + *q0 = vld1q_u8(src + 0 * stride); + *q1 = vld1q_u8(src + 1 * stride); +} + +static WEBP_INLINE void Load16x8(const uint8_t* const src, int stride, + uint8x16_t* const p3, uint8x16_t* const p2, + uint8x16_t* const p1, uint8x16_t* const p0, + uint8x16_t* const q0, uint8x16_t* const q1, + uint8x16_t* const q2, uint8x16_t* const q3) { + Load16x4(src - 2 * stride, stride, p3, p2, p1, p0); + Load16x4(src + 2 * stride, stride, q0, q1, q2, q3); +} + +static WEBP_INLINE void Load8x8x2(const uint8_t* const u, + const uint8_t* const v, + int stride, + uint8x16_t* const p3, uint8x16_t* const p2, + uint8x16_t* const p1, uint8x16_t* const p0, + uint8x16_t* const q0, uint8x16_t* const q1, + uint8x16_t* const q2, uint8x16_t* const q3) { + // We pack the 8x8 u-samples in the lower half of the uint8x16_t destination + // and the v-samples on the higher half. + *p3 = vcombine_u8(vld1_u8(u - 4 * stride), vld1_u8(v - 4 * stride)); + *p2 = vcombine_u8(vld1_u8(u - 3 * stride), vld1_u8(v - 3 * stride)); + *p1 = vcombine_u8(vld1_u8(u - 2 * stride), vld1_u8(v - 2 * stride)); + *p0 = vcombine_u8(vld1_u8(u - 1 * stride), vld1_u8(v - 1 * stride)); + *q0 = vcombine_u8(vld1_u8(u + 0 * stride), vld1_u8(v + 0 * stride)); + *q1 = vcombine_u8(vld1_u8(u + 1 * stride), vld1_u8(v + 1 * stride)); + *q2 = vcombine_u8(vld1_u8(u + 2 * stride), vld1_u8(v + 2 * stride)); + *q3 = vcombine_u8(vld1_u8(u + 3 * stride), vld1_u8(v + 3 * stride)); +} + +#if !defined(WORK_AROUND_GCC) + +#define LOAD_UV_8(ROW) \ + vcombine_u8(vld1_u8(u - 4 + (ROW) * stride), vld1_u8(v - 4 + (ROW) * stride)) + +static WEBP_INLINE void Load8x8x2T(const uint8_t* const u, + const uint8_t* const v, + int stride, + uint8x16_t* const p3, uint8x16_t* const p2, + uint8x16_t* const p1, uint8x16_t* const p0, + uint8x16_t* const q0, uint8x16_t* const q1, + uint8x16_t* const q2, uint8x16_t* const q3) { + // We pack the 8x8 u-samples in the lower half of the uint8x16_t destination + // and the v-samples on the higher half. + const uint8x16_t row0 = LOAD_UV_8(0); + const uint8x16_t row1 = LOAD_UV_8(1); + const uint8x16_t row2 = LOAD_UV_8(2); + const uint8x16_t row3 = LOAD_UV_8(3); + const uint8x16_t row4 = LOAD_UV_8(4); + const uint8x16_t row5 = LOAD_UV_8(5); + const uint8x16_t row6 = LOAD_UV_8(6); + const uint8x16_t row7 = LOAD_UV_8(7); + // Perform two side-by-side 8x8 transposes + // u00 u01 u02 u03 u04 u05 u06 u07 | v00 v01 v02 v03 v04 v05 v06 v07 + // u10 u11 u12 u13 u14 u15 u16 u17 | v10 v11 v12 ... + // u20 u21 u22 u23 u24 u25 u26 u27 | v20 v21 ... + // u30 u31 u32 u33 u34 u35 u36 u37 | ... + // u40 u41 u42 u43 u44 u45 u46 u47 | ... + // u50 u51 u52 u53 u54 u55 u56 u57 | ... + // u60 u61 u62 u63 u64 u65 u66 u67 | v60 ... + // u70 u71 u72 u73 u74 u75 u76 u77 | v70 v71 v72 ... + const uint8x16x2_t row01 = vtrnq_u8(row0, row1); // u00 u10 u02 u12 ... + // u01 u11 u03 u13 ... + const uint8x16x2_t row23 = vtrnq_u8(row2, row3); // u20 u30 u22 u32 ... + // u21 u31 u23 u33 ... + const uint8x16x2_t row45 = vtrnq_u8(row4, row5); // ... + const uint8x16x2_t row67 = vtrnq_u8(row6, row7); // ... + const uint16x8x2_t row02 = vtrnq_u16(vreinterpretq_u16_u8(row01.val[0]), + vreinterpretq_u16_u8(row23.val[0])); + const uint16x8x2_t row13 = vtrnq_u16(vreinterpretq_u16_u8(row01.val[1]), + vreinterpretq_u16_u8(row23.val[1])); + const uint16x8x2_t row46 = vtrnq_u16(vreinterpretq_u16_u8(row45.val[0]), + vreinterpretq_u16_u8(row67.val[0])); + const uint16x8x2_t row57 = vtrnq_u16(vreinterpretq_u16_u8(row45.val[1]), + vreinterpretq_u16_u8(row67.val[1])); + const uint32x4x2_t row04 = vtrnq_u32(vreinterpretq_u32_u16(row02.val[0]), + vreinterpretq_u32_u16(row46.val[0])); + const uint32x4x2_t row26 = vtrnq_u32(vreinterpretq_u32_u16(row02.val[1]), + vreinterpretq_u32_u16(row46.val[1])); + const uint32x4x2_t row15 = vtrnq_u32(vreinterpretq_u32_u16(row13.val[0]), + vreinterpretq_u32_u16(row57.val[0])); + const uint32x4x2_t row37 = vtrnq_u32(vreinterpretq_u32_u16(row13.val[1]), + vreinterpretq_u32_u16(row57.val[1])); + *p3 = vreinterpretq_u8_u32(row04.val[0]); + *p2 = vreinterpretq_u8_u32(row15.val[0]); + *p1 = vreinterpretq_u8_u32(row26.val[0]); + *p0 = vreinterpretq_u8_u32(row37.val[0]); + *q0 = vreinterpretq_u8_u32(row04.val[1]); + *q1 = vreinterpretq_u8_u32(row15.val[1]); + *q2 = vreinterpretq_u8_u32(row26.val[1]); + *q3 = vreinterpretq_u8_u32(row37.val[1]); +} +#undef LOAD_UV_8 + +#endif // !WORK_AROUND_GCC + +static WEBP_INLINE void Store2x8(const uint8x8x2_t v, + uint8_t* const dst, int stride) { + vst2_lane_u8(dst + 0 * stride, v, 0); + vst2_lane_u8(dst + 1 * stride, v, 1); + vst2_lane_u8(dst + 2 * stride, v, 2); + vst2_lane_u8(dst + 3 * stride, v, 3); + vst2_lane_u8(dst + 4 * stride, v, 4); + vst2_lane_u8(dst + 5 * stride, v, 5); + vst2_lane_u8(dst + 6 * stride, v, 6); + vst2_lane_u8(dst + 7 * stride, v, 7); +} + +static WEBP_INLINE void Store2x16(const uint8x16_t p0, const uint8x16_t q0, + uint8_t* const dst, int stride) { + uint8x8x2_t lo, hi; + lo.val[0] = vget_low_u8(p0); + lo.val[1] = vget_low_u8(q0); + hi.val[0] = vget_high_u8(p0); + hi.val[1] = vget_high_u8(q0); + Store2x8(lo, dst - 1 + 0 * stride, stride); + Store2x8(hi, dst - 1 + 8 * stride, stride); +} + +#if !defined(WORK_AROUND_GCC) +static WEBP_INLINE void Store4x8(const uint8x8x4_t v, + uint8_t* const dst, int stride) { + vst4_lane_u8(dst + 0 * stride, v, 0); + vst4_lane_u8(dst + 1 * stride, v, 1); + vst4_lane_u8(dst + 2 * stride, v, 2); + vst4_lane_u8(dst + 3 * stride, v, 3); + vst4_lane_u8(dst + 4 * stride, v, 4); + vst4_lane_u8(dst + 5 * stride, v, 5); + vst4_lane_u8(dst + 6 * stride, v, 6); + vst4_lane_u8(dst + 7 * stride, v, 7); +} + +static WEBP_INLINE void Store4x16(const uint8x16_t p1, const uint8x16_t p0, + const uint8x16_t q0, const uint8x16_t q1, + uint8_t* const dst, int stride) { + uint8x8x4_t lo, hi; + INIT_VECTOR4(lo, + vget_low_u8(p1), vget_low_u8(p0), + vget_low_u8(q0), vget_low_u8(q1)); + INIT_VECTOR4(hi, + vget_high_u8(p1), vget_high_u8(p0), + vget_high_u8(q0), vget_high_u8(q1)); + Store4x8(lo, dst - 2 + 0 * stride, stride); + Store4x8(hi, dst - 2 + 8 * stride, stride); +} +#endif // !WORK_AROUND_GCC + +static WEBP_INLINE void Store16x2(const uint8x16_t p0, const uint8x16_t q0, + uint8_t* const dst, int stride) { + vst1q_u8(dst - stride, p0); + vst1q_u8(dst, q0); +} + +static WEBP_INLINE void Store16x4(const uint8x16_t p1, const uint8x16_t p0, + const uint8x16_t q0, const uint8x16_t q1, + uint8_t* const dst, int stride) { + Store16x2(p1, p0, dst - stride, stride); + Store16x2(q0, q1, dst + stride, stride); +} + +static WEBP_INLINE void Store8x2x2(const uint8x16_t p0, const uint8x16_t q0, + uint8_t* const u, uint8_t* const v, + int stride) { + // p0 and q0 contain the u+v samples packed in low/high halves. + vst1_u8(u - stride, vget_low_u8(p0)); + vst1_u8(u, vget_low_u8(q0)); + vst1_u8(v - stride, vget_high_u8(p0)); + vst1_u8(v, vget_high_u8(q0)); +} + +static WEBP_INLINE void Store8x4x2(const uint8x16_t p1, const uint8x16_t p0, + const uint8x16_t q0, const uint8x16_t q1, + uint8_t* const u, uint8_t* const v, + int stride) { + // The p1...q1 registers contain the u+v samples packed in low/high halves. + Store8x2x2(p1, p0, u - stride, v - stride, stride); + Store8x2x2(q0, q1, u + stride, v + stride, stride); +} + +#if !defined(WORK_AROUND_GCC) + +#define STORE6_LANE(DST, VAL0, VAL1, LANE) do { \ + vst3_lane_u8((DST) - 3, (VAL0), (LANE)); \ + vst3_lane_u8((DST) + 0, (VAL1), (LANE)); \ + (DST) += stride; \ +} while (0) + +static WEBP_INLINE void Store6x8x2(const uint8x16_t p2, const uint8x16_t p1, + const uint8x16_t p0, const uint8x16_t q0, + const uint8x16_t q1, const uint8x16_t q2, + uint8_t* u, uint8_t* v, + int stride) { + uint8x8x3_t u0, u1, v0, v1; + INIT_VECTOR3(u0, vget_low_u8(p2), vget_low_u8(p1), vget_low_u8(p0)); + INIT_VECTOR3(u1, vget_low_u8(q0), vget_low_u8(q1), vget_low_u8(q2)); + INIT_VECTOR3(v0, vget_high_u8(p2), vget_high_u8(p1), vget_high_u8(p0)); + INIT_VECTOR3(v1, vget_high_u8(q0), vget_high_u8(q1), vget_high_u8(q2)); + STORE6_LANE(u, u0, u1, 0); + STORE6_LANE(u, u0, u1, 1); + STORE6_LANE(u, u0, u1, 2); + STORE6_LANE(u, u0, u1, 3); + STORE6_LANE(u, u0, u1, 4); + STORE6_LANE(u, u0, u1, 5); + STORE6_LANE(u, u0, u1, 6); + STORE6_LANE(u, u0, u1, 7); + STORE6_LANE(v, v0, v1, 0); + STORE6_LANE(v, v0, v1, 1); + STORE6_LANE(v, v0, v1, 2); + STORE6_LANE(v, v0, v1, 3); + STORE6_LANE(v, v0, v1, 4); + STORE6_LANE(v, v0, v1, 5); + STORE6_LANE(v, v0, v1, 6); + STORE6_LANE(v, v0, v1, 7); +} +#undef STORE6_LANE + +static WEBP_INLINE void Store4x8x2(const uint8x16_t p1, const uint8x16_t p0, + const uint8x16_t q0, const uint8x16_t q1, + uint8_t* const u, uint8_t* const v, + int stride) { + uint8x8x4_t u0, v0; + INIT_VECTOR4(u0, + vget_low_u8(p1), vget_low_u8(p0), + vget_low_u8(q0), vget_low_u8(q1)); + INIT_VECTOR4(v0, + vget_high_u8(p1), vget_high_u8(p0), + vget_high_u8(q0), vget_high_u8(q1)); + vst4_lane_u8(u - 2 + 0 * stride, u0, 0); + vst4_lane_u8(u - 2 + 1 * stride, u0, 1); + vst4_lane_u8(u - 2 + 2 * stride, u0, 2); + vst4_lane_u8(u - 2 + 3 * stride, u0, 3); + vst4_lane_u8(u - 2 + 4 * stride, u0, 4); + vst4_lane_u8(u - 2 + 5 * stride, u0, 5); + vst4_lane_u8(u - 2 + 6 * stride, u0, 6); + vst4_lane_u8(u - 2 + 7 * stride, u0, 7); + vst4_lane_u8(v - 2 + 0 * stride, v0, 0); + vst4_lane_u8(v - 2 + 1 * stride, v0, 1); + vst4_lane_u8(v - 2 + 2 * stride, v0, 2); + vst4_lane_u8(v - 2 + 3 * stride, v0, 3); + vst4_lane_u8(v - 2 + 4 * stride, v0, 4); + vst4_lane_u8(v - 2 + 5 * stride, v0, 5); + vst4_lane_u8(v - 2 + 6 * stride, v0, 6); + vst4_lane_u8(v - 2 + 7 * stride, v0, 7); +} + +#endif // !WORK_AROUND_GCC + +// Zero extend 'v' to an int16x8_t. +static WEBP_INLINE int16x8_t ConvertU8ToS16(uint8x8_t v) { + return vreinterpretq_s16_u16(vmovl_u8(v)); +} + +// Performs unsigned 8b saturation on 'dst01' and 'dst23' storing the result +// to the corresponding rows of 'dst'. +static WEBP_INLINE void SaturateAndStore4x4(uint8_t* const dst, + const int16x8_t dst01, + const int16x8_t dst23) { + // Unsigned saturate to 8b. + const uint8x8_t dst01_u8 = vqmovun_s16(dst01); + const uint8x8_t dst23_u8 = vqmovun_s16(dst23); + + // Store the results. + vst1_lane_u32((uint32_t*)(dst + 0 * BPS), vreinterpret_u32_u8(dst01_u8), 0); + vst1_lane_u32((uint32_t*)(dst + 1 * BPS), vreinterpret_u32_u8(dst01_u8), 1); + vst1_lane_u32((uint32_t*)(dst + 2 * BPS), vreinterpret_u32_u8(dst23_u8), 0); + vst1_lane_u32((uint32_t*)(dst + 3 * BPS), vreinterpret_u32_u8(dst23_u8), 1); +} + +static WEBP_INLINE void Add4x4(const int16x8_t row01, const int16x8_t row23, + uint8_t* const dst) { + uint32x2_t dst01 = vdup_n_u32(0); + uint32x2_t dst23 = vdup_n_u32(0); + + // Load the source pixels. + dst01 = vld1_lane_u32((uint32_t*)(dst + 0 * BPS), dst01, 0); + dst23 = vld1_lane_u32((uint32_t*)(dst + 2 * BPS), dst23, 0); + dst01 = vld1_lane_u32((uint32_t*)(dst + 1 * BPS), dst01, 1); + dst23 = vld1_lane_u32((uint32_t*)(dst + 3 * BPS), dst23, 1); + + { + // Convert to 16b. + const int16x8_t dst01_s16 = ConvertU8ToS16(vreinterpret_u8_u32(dst01)); + const int16x8_t dst23_s16 = ConvertU8ToS16(vreinterpret_u8_u32(dst23)); + + // Descale with rounding. + const int16x8_t out01 = vrsraq_n_s16(dst01_s16, row01, 3); + const int16x8_t out23 = vrsraq_n_s16(dst23_s16, row23, 3); + // Add the inverse transform. + SaturateAndStore4x4(dst, out01, out23); + } +} + +//----------------------------------------------------------------------------- +// Simple In-loop filtering (Paragraph 15.2) + +static uint8x16_t NeedsFilter(const uint8x16_t p1, const uint8x16_t p0, + const uint8x16_t q0, const uint8x16_t q1, + int thresh) { + const uint8x16_t thresh_v = vdupq_n_u8((uint8_t)thresh); + const uint8x16_t a_p0_q0 = vabdq_u8(p0, q0); // abs(p0-q0) + const uint8x16_t a_p1_q1 = vabdq_u8(p1, q1); // abs(p1-q1) + const uint8x16_t a_p0_q0_2 = vqaddq_u8(a_p0_q0, a_p0_q0); // 2 * abs(p0-q0) + const uint8x16_t a_p1_q1_2 = vshrq_n_u8(a_p1_q1, 1); // abs(p1-q1) / 2 + const uint8x16_t sum = vqaddq_u8(a_p0_q0_2, a_p1_q1_2); + const uint8x16_t mask = vcgeq_u8(thresh_v, sum); + return mask; +} + +static int8x16_t FlipSign(const uint8x16_t v) { + const uint8x16_t sign_bit = vdupq_n_u8(0x80); + return vreinterpretq_s8_u8(veorq_u8(v, sign_bit)); +} + +static uint8x16_t FlipSignBack(const int8x16_t v) { + const int8x16_t sign_bit = vdupq_n_s8(0x80); + return vreinterpretq_u8_s8(veorq_s8(v, sign_bit)); +} + +static int8x16_t GetBaseDelta(const int8x16_t p1, const int8x16_t p0, + const int8x16_t q0, const int8x16_t q1) { + const int8x16_t q0_p0 = vqsubq_s8(q0, p0); // (q0-p0) + const int8x16_t p1_q1 = vqsubq_s8(p1, q1); // (p1-q1) + const int8x16_t s1 = vqaddq_s8(p1_q1, q0_p0); // (p1-q1) + 1 * (q0 - p0) + const int8x16_t s2 = vqaddq_s8(q0_p0, s1); // (p1-q1) + 2 * (q0 - p0) + const int8x16_t s3 = vqaddq_s8(q0_p0, s2); // (p1-q1) + 3 * (q0 - p0) + return s3; +} + +static int8x16_t GetBaseDelta0(const int8x16_t p0, const int8x16_t q0) { + const int8x16_t q0_p0 = vqsubq_s8(q0, p0); // (q0-p0) + const int8x16_t s1 = vqaddq_s8(q0_p0, q0_p0); // 2 * (q0 - p0) + const int8x16_t s2 = vqaddq_s8(q0_p0, s1); // 3 * (q0 - p0) + return s2; +} + +//------------------------------------------------------------------------------ + +static void ApplyFilter2NoFlip(const int8x16_t p0s, const int8x16_t q0s, + const int8x16_t delta, + int8x16_t* const op0, int8x16_t* const oq0) { + const int8x16_t kCst3 = vdupq_n_s8(0x03); + const int8x16_t kCst4 = vdupq_n_s8(0x04); + const int8x16_t delta_p3 = vqaddq_s8(delta, kCst3); + const int8x16_t delta_p4 = vqaddq_s8(delta, kCst4); + const int8x16_t delta3 = vshrq_n_s8(delta_p3, 3); + const int8x16_t delta4 = vshrq_n_s8(delta_p4, 3); + *op0 = vqaddq_s8(p0s, delta3); + *oq0 = vqsubq_s8(q0s, delta4); +} + +#if defined(WEBP_USE_INTRINSICS) + +static void ApplyFilter2(const int8x16_t p0s, const int8x16_t q0s, + const int8x16_t delta, + uint8x16_t* const op0, uint8x16_t* const oq0) { + const int8x16_t kCst3 = vdupq_n_s8(0x03); + const int8x16_t kCst4 = vdupq_n_s8(0x04); + const int8x16_t delta_p3 = vqaddq_s8(delta, kCst3); + const int8x16_t delta_p4 = vqaddq_s8(delta, kCst4); + const int8x16_t delta3 = vshrq_n_s8(delta_p3, 3); + const int8x16_t delta4 = vshrq_n_s8(delta_p4, 3); + const int8x16_t sp0 = vqaddq_s8(p0s, delta3); + const int8x16_t sq0 = vqsubq_s8(q0s, delta4); + *op0 = FlipSignBack(sp0); + *oq0 = FlipSignBack(sq0); +} + +static void DoFilter2(const uint8x16_t p1, const uint8x16_t p0, + const uint8x16_t q0, const uint8x16_t q1, + const uint8x16_t mask, + uint8x16_t* const op0, uint8x16_t* const oq0) { + const int8x16_t p1s = FlipSign(p1); + const int8x16_t p0s = FlipSign(p0); + const int8x16_t q0s = FlipSign(q0); + const int8x16_t q1s = FlipSign(q1); + const int8x16_t delta0 = GetBaseDelta(p1s, p0s, q0s, q1s); + const int8x16_t delta1 = vandq_s8(delta0, vreinterpretq_s8_u8(mask)); + ApplyFilter2(p0s, q0s, delta1, op0, oq0); +} + +static void SimpleVFilter16(uint8_t* p, int stride, int thresh) { + uint8x16_t p1, p0, q0, q1, op0, oq0; + Load16x4(p, stride, &p1, &p0, &q0, &q1); + { + const uint8x16_t mask = NeedsFilter(p1, p0, q0, q1, thresh); + DoFilter2(p1, p0, q0, q1, mask, &op0, &oq0); + } + Store16x2(op0, oq0, p, stride); +} + +static void SimpleHFilter16(uint8_t* p, int stride, int thresh) { + uint8x16_t p1, p0, q0, q1, oq0, op0; + Load4x16(p, stride, &p1, &p0, &q0, &q1); + { + const uint8x16_t mask = NeedsFilter(p1, p0, q0, q1, thresh); + DoFilter2(p1, p0, q0, q1, mask, &op0, &oq0); + } + Store2x16(op0, oq0, p, stride); +} + +#else + +#define QRegs "q0", "q1", "q2", "q3", \ + "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" + +#define FLIP_SIGN_BIT2(a, b, s) \ + "veor " #a "," #a "," #s " \n" \ + "veor " #b "," #b "," #s " \n" \ + +#define FLIP_SIGN_BIT4(a, b, c, d, s) \ + FLIP_SIGN_BIT2(a, b, s) \ + FLIP_SIGN_BIT2(c, d, s) \ + +#define NEEDS_FILTER(p1, p0, q0, q1, thresh, mask) \ + "vabd.u8 q15," #p0 "," #q0 " \n" /* abs(p0 - q0) */ \ + "vabd.u8 q14," #p1 "," #q1 " \n" /* abs(p1 - q1) */ \ + "vqadd.u8 q15, q15, q15 \n" /* abs(p0 - q0) * 2 */ \ + "vshr.u8 q14, q14, #1 \n" /* abs(p1 - q1) / 2 */ \ + "vqadd.u8 q15, q15, q14 \n" /* abs(p0 - q0) * 2 + abs(p1 - q1) / 2 */ \ + "vdup.8 q14, " #thresh " \n" \ + "vcge.u8 " #mask ", q14, q15 \n" /* mask <= thresh */ + +#define GET_BASE_DELTA(p1, p0, q0, q1, o) \ + "vqsub.s8 q15," #q0 "," #p0 " \n" /* (q0 - p0) */ \ + "vqsub.s8 " #o "," #p1 "," #q1 " \n" /* (p1 - q1) */ \ + "vqadd.s8 " #o "," #o ", q15 \n" /* (p1 - q1) + 1 * (p0 - q0) */ \ + "vqadd.s8 " #o "," #o ", q15 \n" /* (p1 - q1) + 2 * (p0 - q0) */ \ + "vqadd.s8 " #o "," #o ", q15 \n" /* (p1 - q1) + 3 * (p0 - q0) */ + +#define DO_SIMPLE_FILTER(p0, q0, fl) \ + "vmov.i8 q15, #0x03 \n" \ + "vqadd.s8 q15, q15, " #fl " \n" /* filter1 = filter + 3 */ \ + "vshr.s8 q15, q15, #3 \n" /* filter1 >> 3 */ \ + "vqadd.s8 " #p0 "," #p0 ", q15 \n" /* p0 += filter1 */ \ + \ + "vmov.i8 q15, #0x04 \n" \ + "vqadd.s8 q15, q15, " #fl " \n" /* filter1 = filter + 4 */ \ + "vshr.s8 q15, q15, #3 \n" /* filter2 >> 3 */ \ + "vqsub.s8 " #q0 "," #q0 ", q15 \n" /* q0 -= filter2 */ + +// Applies filter on 2 pixels (p0 and q0) +#define DO_FILTER2(p1, p0, q0, q1, thresh) \ + NEEDS_FILTER(p1, p0, q0, q1, thresh, q9) /* filter mask in q9 */ \ + "vmov.i8 q10, #0x80 \n" /* sign bit */ \ + FLIP_SIGN_BIT4(p1, p0, q0, q1, q10) /* convert to signed value */ \ + GET_BASE_DELTA(p1, p0, q0, q1, q11) /* get filter level */ \ + "vand q9, q9, q11 \n" /* apply filter mask */ \ + DO_SIMPLE_FILTER(p0, q0, q9) /* apply filter */ \ + FLIP_SIGN_BIT2(p0, q0, q10) + +static void SimpleVFilter16(uint8_t* p, int stride, int thresh) { + __asm__ volatile ( + "sub %[p], %[p], %[stride], lsl #1 \n" // p -= 2 * stride + + "vld1.u8 {q1}, [%[p]], %[stride] \n" // p1 + "vld1.u8 {q2}, [%[p]], %[stride] \n" // p0 + "vld1.u8 {q3}, [%[p]], %[stride] \n" // q0 + "vld1.u8 {q12}, [%[p]] \n" // q1 + + DO_FILTER2(q1, q2, q3, q12, %[thresh]) + + "sub %[p], %[p], %[stride], lsl #1 \n" // p -= 2 * stride + + "vst1.u8 {q2}, [%[p]], %[stride] \n" // store op0 + "vst1.u8 {q3}, [%[p]] \n" // store oq0 + : [p] "+r"(p) + : [stride] "r"(stride), [thresh] "r"(thresh) + : "memory", QRegs + ); +} + +static void SimpleHFilter16(uint8_t* p, int stride, int thresh) { + __asm__ volatile ( + "sub r4, %[p], #2 \n" // base1 = p - 2 + "lsl r6, %[stride], #1 \n" // r6 = 2 * stride + "add r5, r4, %[stride] \n" // base2 = base1 + stride + + LOAD8x4(d2, d3, d4, d5, [r4], [r5], r6) + LOAD8x4(d24, d25, d26, d27, [r4], [r5], r6) + "vswp d3, d24 \n" // p1:q1 p0:q3 + "vswp d5, d26 \n" // q0:q2 q1:q4 + "vswp q2, q12 \n" // p1:q1 p0:q2 q0:q3 q1:q4 + + DO_FILTER2(q1, q2, q12, q13, %[thresh]) + + "sub %[p], %[p], #1 \n" // p - 1 + + "vswp d5, d24 \n" + STORE8x2(d4, d5, [%[p]], %[stride]) + STORE8x2(d24, d25, [%[p]], %[stride]) + + : [p] "+r"(p) + : [stride] "r"(stride), [thresh] "r"(thresh) + : "memory", "r4", "r5", "r6", QRegs + ); +} + +#endif // WEBP_USE_INTRINSICS + +static void SimpleVFilter16i(uint8_t* p, int stride, int thresh) { + uint32_t k; + for (k = 3; k != 0; --k) { + p += 4 * stride; + SimpleVFilter16(p, stride, thresh); + } +} + +static void SimpleHFilter16i(uint8_t* p, int stride, int thresh) { + uint32_t k; + for (k = 3; k != 0; --k) { + p += 4; + SimpleHFilter16(p, stride, thresh); + } +} + +//------------------------------------------------------------------------------ +// Complex In-loop filtering (Paragraph 15.3) + +static uint8x16_t NeedsHev(const uint8x16_t p1, const uint8x16_t p0, + const uint8x16_t q0, const uint8x16_t q1, + int hev_thresh) { + const uint8x16_t hev_thresh_v = vdupq_n_u8((uint8_t)hev_thresh); + const uint8x16_t a_p1_p0 = vabdq_u8(p1, p0); // abs(p1 - p0) + const uint8x16_t a_q1_q0 = vabdq_u8(q1, q0); // abs(q1 - q0) + const uint8x16_t mask1 = vcgtq_u8(a_p1_p0, hev_thresh_v); + const uint8x16_t mask2 = vcgtq_u8(a_q1_q0, hev_thresh_v); + const uint8x16_t mask = vorrq_u8(mask1, mask2); + return mask; +} + +static uint8x16_t NeedsFilter2(const uint8x16_t p3, const uint8x16_t p2, + const uint8x16_t p1, const uint8x16_t p0, + const uint8x16_t q0, const uint8x16_t q1, + const uint8x16_t q2, const uint8x16_t q3, + int ithresh, int thresh) { + const uint8x16_t ithresh_v = vdupq_n_u8((uint8_t)ithresh); + const uint8x16_t a_p3_p2 = vabdq_u8(p3, p2); // abs(p3 - p2) + const uint8x16_t a_p2_p1 = vabdq_u8(p2, p1); // abs(p2 - p1) + const uint8x16_t a_p1_p0 = vabdq_u8(p1, p0); // abs(p1 - p0) + const uint8x16_t a_q3_q2 = vabdq_u8(q3, q2); // abs(q3 - q2) + const uint8x16_t a_q2_q1 = vabdq_u8(q2, q1); // abs(q2 - q1) + const uint8x16_t a_q1_q0 = vabdq_u8(q1, q0); // abs(q1 - q0) + const uint8x16_t max1 = vmaxq_u8(a_p3_p2, a_p2_p1); + const uint8x16_t max2 = vmaxq_u8(a_p1_p0, a_q3_q2); + const uint8x16_t max3 = vmaxq_u8(a_q2_q1, a_q1_q0); + const uint8x16_t max12 = vmaxq_u8(max1, max2); + const uint8x16_t max123 = vmaxq_u8(max12, max3); + const uint8x16_t mask2 = vcgeq_u8(ithresh_v, max123); + const uint8x16_t mask1 = NeedsFilter(p1, p0, q0, q1, thresh); + const uint8x16_t mask = vandq_u8(mask1, mask2); + return mask; +} + +// 4-points filter + +static void ApplyFilter4( + const int8x16_t p1, const int8x16_t p0, + const int8x16_t q0, const int8x16_t q1, + const int8x16_t delta0, + uint8x16_t* const op1, uint8x16_t* const op0, + uint8x16_t* const oq0, uint8x16_t* const oq1) { + const int8x16_t kCst3 = vdupq_n_s8(0x03); + const int8x16_t kCst4 = vdupq_n_s8(0x04); + const int8x16_t delta1 = vqaddq_s8(delta0, kCst4); + const int8x16_t delta2 = vqaddq_s8(delta0, kCst3); + const int8x16_t a1 = vshrq_n_s8(delta1, 3); + const int8x16_t a2 = vshrq_n_s8(delta2, 3); + const int8x16_t a3 = vrshrq_n_s8(a1, 1); // a3 = (a1 + 1) >> 1 + *op0 = FlipSignBack(vqaddq_s8(p0, a2)); // clip(p0 + a2) + *oq0 = FlipSignBack(vqsubq_s8(q0, a1)); // clip(q0 - a1) + *op1 = FlipSignBack(vqaddq_s8(p1, a3)); // clip(p1 + a3) + *oq1 = FlipSignBack(vqsubq_s8(q1, a3)); // clip(q1 - a3) +} + +static void DoFilter4( + const uint8x16_t p1, const uint8x16_t p0, + const uint8x16_t q0, const uint8x16_t q1, + const uint8x16_t mask, const uint8x16_t hev_mask, + uint8x16_t* const op1, uint8x16_t* const op0, + uint8x16_t* const oq0, uint8x16_t* const oq1) { + // This is a fused version of DoFilter2() calling ApplyFilter2 directly + const int8x16_t p1s = FlipSign(p1); + int8x16_t p0s = FlipSign(p0); + int8x16_t q0s = FlipSign(q0); + const int8x16_t q1s = FlipSign(q1); + const uint8x16_t simple_lf_mask = vandq_u8(mask, hev_mask); + + // do_filter2 part (simple loopfilter on pixels with hev) + { + const int8x16_t delta = GetBaseDelta(p1s, p0s, q0s, q1s); + const int8x16_t simple_lf_delta = + vandq_s8(delta, vreinterpretq_s8_u8(simple_lf_mask)); + ApplyFilter2NoFlip(p0s, q0s, simple_lf_delta, &p0s, &q0s); + } + + // do_filter4 part (complex loopfilter on pixels without hev) + { + const int8x16_t delta0 = GetBaseDelta0(p0s, q0s); + // we use: (mask & hev_mask) ^ mask = mask & !hev_mask + const uint8x16_t complex_lf_mask = veorq_u8(simple_lf_mask, mask); + const int8x16_t complex_lf_delta = + vandq_s8(delta0, vreinterpretq_s8_u8(complex_lf_mask)); + ApplyFilter4(p1s, p0s, q0s, q1s, complex_lf_delta, op1, op0, oq0, oq1); + } +} + +// 6-points filter + +static void ApplyFilter6( + const int8x16_t p2, const int8x16_t p1, const int8x16_t p0, + const int8x16_t q0, const int8x16_t q1, const int8x16_t q2, + const int8x16_t delta, + uint8x16_t* const op2, uint8x16_t* const op1, uint8x16_t* const op0, + uint8x16_t* const oq0, uint8x16_t* const oq1, uint8x16_t* const oq2) { + const int16x8_t kCst63 = vdupq_n_s16(63); + const int8x8_t kCst27 = vdup_n_s8(27); + const int8x8_t kCst18 = vdup_n_s8(18); + const int8x8_t kCst9 = vdup_n_s8(9); + const int8x8_t delta_lo = vget_low_s8(delta); + const int8x8_t delta_hi = vget_high_s8(delta); + const int16x8_t s1_lo = vmlal_s8(kCst63, kCst27, delta_lo); // 63 + 27 * a + const int16x8_t s1_hi = vmlal_s8(kCst63, kCst27, delta_hi); // 63 + 27 * a + const int16x8_t s2_lo = vmlal_s8(kCst63, kCst18, delta_lo); // 63 + 18 * a + const int16x8_t s2_hi = vmlal_s8(kCst63, kCst18, delta_hi); // 63 + 18 * a + const int16x8_t s3_lo = vmlal_s8(kCst63, kCst9, delta_lo); // 63 + 9 * a + const int16x8_t s3_hi = vmlal_s8(kCst63, kCst9, delta_hi); // 63 + 9 * a + const int8x8_t a1_lo = vqshrn_n_s16(s1_lo, 7); + const int8x8_t a1_hi = vqshrn_n_s16(s1_hi, 7); + const int8x8_t a2_lo = vqshrn_n_s16(s2_lo, 7); + const int8x8_t a2_hi = vqshrn_n_s16(s2_hi, 7); + const int8x8_t a3_lo = vqshrn_n_s16(s3_lo, 7); + const int8x8_t a3_hi = vqshrn_n_s16(s3_hi, 7); + const int8x16_t a1 = vcombine_s8(a1_lo, a1_hi); + const int8x16_t a2 = vcombine_s8(a2_lo, a2_hi); + const int8x16_t a3 = vcombine_s8(a3_lo, a3_hi); + + *op0 = FlipSignBack(vqaddq_s8(p0, a1)); // clip(p0 + a1) + *oq0 = FlipSignBack(vqsubq_s8(q0, a1)); // clip(q0 - q1) + *oq1 = FlipSignBack(vqsubq_s8(q1, a2)); // clip(q1 - a2) + *op1 = FlipSignBack(vqaddq_s8(p1, a2)); // clip(p1 + a2) + *oq2 = FlipSignBack(vqsubq_s8(q2, a3)); // clip(q2 - a3) + *op2 = FlipSignBack(vqaddq_s8(p2, a3)); // clip(p2 + a3) +} + +static void DoFilter6( + const uint8x16_t p2, const uint8x16_t p1, const uint8x16_t p0, + const uint8x16_t q0, const uint8x16_t q1, const uint8x16_t q2, + const uint8x16_t mask, const uint8x16_t hev_mask, + uint8x16_t* const op2, uint8x16_t* const op1, uint8x16_t* const op0, + uint8x16_t* const oq0, uint8x16_t* const oq1, uint8x16_t* const oq2) { + // This is a fused version of DoFilter2() calling ApplyFilter2 directly + const int8x16_t p2s = FlipSign(p2); + const int8x16_t p1s = FlipSign(p1); + int8x16_t p0s = FlipSign(p0); + int8x16_t q0s = FlipSign(q0); + const int8x16_t q1s = FlipSign(q1); + const int8x16_t q2s = FlipSign(q2); + const uint8x16_t simple_lf_mask = vandq_u8(mask, hev_mask); + const int8x16_t delta0 = GetBaseDelta(p1s, p0s, q0s, q1s); + + // do_filter2 part (simple loopfilter on pixels with hev) + { + const int8x16_t simple_lf_delta = + vandq_s8(delta0, vreinterpretq_s8_u8(simple_lf_mask)); + ApplyFilter2NoFlip(p0s, q0s, simple_lf_delta, &p0s, &q0s); + } + + // do_filter6 part (complex loopfilter on pixels without hev) + { + // we use: (mask & hev_mask) ^ mask = mask & !hev_mask + const uint8x16_t complex_lf_mask = veorq_u8(simple_lf_mask, mask); + const int8x16_t complex_lf_delta = + vandq_s8(delta0, vreinterpretq_s8_u8(complex_lf_mask)); + ApplyFilter6(p2s, p1s, p0s, q0s, q1s, q2s, complex_lf_delta, + op2, op1, op0, oq0, oq1, oq2); + } +} + +// on macroblock edges + +static void VFilter16(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + uint8x16_t p3, p2, p1, p0, q0, q1, q2, q3; + Load16x8(p, stride, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3); + { + const uint8x16_t mask = NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, + ithresh, thresh); + const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); + uint8x16_t op2, op1, op0, oq0, oq1, oq2; + DoFilter6(p2, p1, p0, q0, q1, q2, mask, hev_mask, + &op2, &op1, &op0, &oq0, &oq1, &oq2); + Store16x2(op2, op1, p - 2 * stride, stride); + Store16x2(op0, oq0, p + 0 * stride, stride); + Store16x2(oq1, oq2, p + 2 * stride, stride); + } +} + +static void HFilter16(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + uint8x16_t p3, p2, p1, p0, q0, q1, q2, q3; + Load8x16(p, stride, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3); + { + const uint8x16_t mask = NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, + ithresh, thresh); + const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); + uint8x16_t op2, op1, op0, oq0, oq1, oq2; + DoFilter6(p2, p1, p0, q0, q1, q2, mask, hev_mask, + &op2, &op1, &op0, &oq0, &oq1, &oq2); + Store2x16(op2, op1, p - 2, stride); + Store2x16(op0, oq0, p + 0, stride); + Store2x16(oq1, oq2, p + 2, stride); + } +} + +// on three inner edges +static void VFilter16i(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + uint32_t k; + uint8x16_t p3, p2, p1, p0; + Load16x4(p + 2 * stride, stride, &p3, &p2, &p1, &p0); + for (k = 3; k != 0; --k) { + uint8x16_t q0, q1, q2, q3; + p += 4 * stride; + Load16x4(p + 2 * stride, stride, &q0, &q1, &q2, &q3); + { + const uint8x16_t mask = + NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, ithresh, thresh); + const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); + // p3 and p2 are not just temporary variables here: they will be + // re-used for next span. And q2/q3 will become p1/p0 accordingly. + DoFilter4(p1, p0, q0, q1, mask, hev_mask, &p1, &p0, &p3, &p2); + Store16x4(p1, p0, p3, p2, p, stride); + p1 = q2; + p0 = q3; + } + } +} + +#if !defined(WORK_AROUND_GCC) +static void HFilter16i(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + uint32_t k; + uint8x16_t p3, p2, p1, p0; + Load4x16(p + 2, stride, &p3, &p2, &p1, &p0); + for (k = 3; k != 0; --k) { + uint8x16_t q0, q1, q2, q3; + p += 4; + Load4x16(p + 2, stride, &q0, &q1, &q2, &q3); + { + const uint8x16_t mask = + NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, ithresh, thresh); + const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); + DoFilter4(p1, p0, q0, q1, mask, hev_mask, &p1, &p0, &p3, &p2); + Store4x16(p1, p0, p3, p2, p, stride); + p1 = q2; + p0 = q3; + } + } +} +#endif // !WORK_AROUND_GCC + +// 8-pixels wide variant, for chroma filtering +static void VFilter8(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + uint8x16_t p3, p2, p1, p0, q0, q1, q2, q3; + Load8x8x2(u, v, stride, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3); + { + const uint8x16_t mask = NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, + ithresh, thresh); + const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); + uint8x16_t op2, op1, op0, oq0, oq1, oq2; + DoFilter6(p2, p1, p0, q0, q1, q2, mask, hev_mask, + &op2, &op1, &op0, &oq0, &oq1, &oq2); + Store8x2x2(op2, op1, u - 2 * stride, v - 2 * stride, stride); + Store8x2x2(op0, oq0, u + 0 * stride, v + 0 * stride, stride); + Store8x2x2(oq1, oq2, u + 2 * stride, v + 2 * stride, stride); + } +} +static void VFilter8i(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + uint8x16_t p3, p2, p1, p0, q0, q1, q2, q3; + u += 4 * stride; + v += 4 * stride; + Load8x8x2(u, v, stride, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3); + { + const uint8x16_t mask = NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, + ithresh, thresh); + const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); + uint8x16_t op1, op0, oq0, oq1; + DoFilter4(p1, p0, q0, q1, mask, hev_mask, &op1, &op0, &oq0, &oq1); + Store8x4x2(op1, op0, oq0, oq1, u, v, stride); + } +} + +#if !defined(WORK_AROUND_GCC) +static void HFilter8(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + uint8x16_t p3, p2, p1, p0, q0, q1, q2, q3; + Load8x8x2T(u, v, stride, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3); + { + const uint8x16_t mask = NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, + ithresh, thresh); + const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); + uint8x16_t op2, op1, op0, oq0, oq1, oq2; + DoFilter6(p2, p1, p0, q0, q1, q2, mask, hev_mask, + &op2, &op1, &op0, &oq0, &oq1, &oq2); + Store6x8x2(op2, op1, op0, oq0, oq1, oq2, u, v, stride); + } +} + +static void HFilter8i(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + uint8x16_t p3, p2, p1, p0, q0, q1, q2, q3; + u += 4; + v += 4; + Load8x8x2T(u, v, stride, &p3, &p2, &p1, &p0, &q0, &q1, &q2, &q3); + { + const uint8x16_t mask = NeedsFilter2(p3, p2, p1, p0, q0, q1, q2, q3, + ithresh, thresh); + const uint8x16_t hev_mask = NeedsHev(p1, p0, q0, q1, hev_thresh); + uint8x16_t op1, op0, oq0, oq1; + DoFilter4(p1, p0, q0, q1, mask, hev_mask, &op1, &op0, &oq0, &oq1); + Store4x8x2(op1, op0, oq0, oq1, u, v, stride); + } +} +#endif // !WORK_AROUND_GCC + +//----------------------------------------------------------------------------- +// Inverse transforms (Paragraph 14.4) + +// Technically these are unsigned but vqdmulh is only available in signed. +// vqdmulh returns high half (effectively >> 16) but also doubles the value, +// changing the >> 16 to >> 15 and requiring an additional >> 1. +// We use this to our advantage with kC2. The canonical value is 35468. +// However, the high bit is set so treating it as signed will give incorrect +// results. We avoid this by down shifting by 1 here to clear the highest bit. +// Combined with the doubling effect of vqdmulh we get >> 16. +// This can not be applied to kC1 because the lowest bit is set. Down shifting +// the constant would reduce precision. + +// libwebp uses a trick to avoid some extra addition that libvpx does. +// Instead of: +// temp2 = ip[12] + ((ip[12] * cospi8sqrt2minus1) >> 16); +// libwebp adds 1 << 16 to cospi8sqrt2minus1 (kC1). However, this causes the +// same issue with kC1 and vqdmulh that we work around by down shifting kC2 + +static const int16_t kC1 = 20091; +static const int16_t kC2 = 17734; // half of kC2, actually. See comment above. + +#if defined(WEBP_USE_INTRINSICS) +static WEBP_INLINE void Transpose8x2(const int16x8_t in0, const int16x8_t in1, + int16x8x2_t* const out) { + // a0 a1 a2 a3 | b0 b1 b2 b3 => a0 b0 c0 d0 | a1 b1 c1 d1 + // c0 c1 c2 c3 | d0 d1 d2 d3 a2 b2 c2 d2 | a3 b3 c3 d3 + const int16x8x2_t tmp0 = vzipq_s16(in0, in1); // a0 c0 a1 c1 a2 c2 ... + // b0 d0 b1 d1 b2 d2 ... + *out = vzipq_s16(tmp0.val[0], tmp0.val[1]); +} + +static WEBP_INLINE void TransformPass(int16x8x2_t* const rows) { + // {rows} = in0 | in4 + // in8 | in12 + // B1 = in4 | in12 + const int16x8_t B1 = + vcombine_s16(vget_high_s16(rows->val[0]), vget_high_s16(rows->val[1])); + // C0 = kC1 * in4 | kC1 * in12 + // C1 = kC2 * in4 | kC2 * in12 + const int16x8_t C0 = vsraq_n_s16(B1, vqdmulhq_n_s16(B1, kC1), 1); + const int16x8_t C1 = vqdmulhq_n_s16(B1, kC2); + const int16x4_t a = vqadd_s16(vget_low_s16(rows->val[0]), + vget_low_s16(rows->val[1])); // in0 + in8 + const int16x4_t b = vqsub_s16(vget_low_s16(rows->val[0]), + vget_low_s16(rows->val[1])); // in0 - in8 + // c = kC2 * in4 - kC1 * in12 + // d = kC1 * in4 + kC2 * in12 + const int16x4_t c = vqsub_s16(vget_low_s16(C1), vget_high_s16(C0)); + const int16x4_t d = vqadd_s16(vget_low_s16(C0), vget_high_s16(C1)); + const int16x8_t D0 = vcombine_s16(a, b); // D0 = a | b + const int16x8_t D1 = vcombine_s16(d, c); // D1 = d | c + const int16x8_t E0 = vqaddq_s16(D0, D1); // a+d | b+c + const int16x8_t E_tmp = vqsubq_s16(D0, D1); // a-d | b-c + const int16x8_t E1 = vcombine_s16(vget_high_s16(E_tmp), vget_low_s16(E_tmp)); + Transpose8x2(E0, E1, rows); +} + +static void TransformOne(const int16_t* in, uint8_t* dst) { + int16x8x2_t rows; + INIT_VECTOR2(rows, vld1q_s16(in + 0), vld1q_s16(in + 8)); + TransformPass(&rows); + TransformPass(&rows); + Add4x4(rows.val[0], rows.val[1], dst); +} + +#else + +static void TransformOne(const int16_t* in, uint8_t* dst) { + const int kBPS = BPS; + // kC1, kC2. Padded because vld1.16 loads 8 bytes + const int16_t constants[4] = { kC1, kC2, 0, 0 }; + /* Adapted from libvpx: vp8/common/arm/neon/shortidct4x4llm_neon.asm */ + __asm__ volatile ( + "vld1.16 {q1, q2}, [%[in]] \n" + "vld1.16 {d0}, [%[constants]] \n" + + /* d2: in[0] + * d3: in[8] + * d4: in[4] + * d5: in[12] + */ + "vswp d3, d4 \n" + + /* q8 = {in[4], in[12]} * kC1 * 2 >> 16 + * q9 = {in[4], in[12]} * kC2 >> 16 + */ + "vqdmulh.s16 q8, q2, d0[0] \n" + "vqdmulh.s16 q9, q2, d0[1] \n" + + /* d22 = a = in[0] + in[8] + * d23 = b = in[0] - in[8] + */ + "vqadd.s16 d22, d2, d3 \n" + "vqsub.s16 d23, d2, d3 \n" + + /* The multiplication should be x * kC1 >> 16 + * However, with vqdmulh we get x * kC1 * 2 >> 16 + * (multiply, double, return high half) + * We avoided this in kC2 by pre-shifting the constant. + * q8 = in[4]/[12] * kC1 >> 16 + */ + "vshr.s16 q8, q8, #1 \n" + + /* Add {in[4], in[12]} back after the multiplication. This is handled by + * adding 1 << 16 to kC1 in the libwebp C code. + */ + "vqadd.s16 q8, q2, q8 \n" + + /* d20 = c = in[4]*kC2 - in[12]*kC1 + * d21 = d = in[4]*kC1 + in[12]*kC2 + */ + "vqsub.s16 d20, d18, d17 \n" + "vqadd.s16 d21, d19, d16 \n" + + /* d2 = tmp[0] = a + d + * d3 = tmp[1] = b + c + * d4 = tmp[2] = b - c + * d5 = tmp[3] = a - d + */ + "vqadd.s16 d2, d22, d21 \n" + "vqadd.s16 d3, d23, d20 \n" + "vqsub.s16 d4, d23, d20 \n" + "vqsub.s16 d5, d22, d21 \n" + + "vzip.16 q1, q2 \n" + "vzip.16 q1, q2 \n" + + "vswp d3, d4 \n" + + /* q8 = {tmp[4], tmp[12]} * kC1 * 2 >> 16 + * q9 = {tmp[4], tmp[12]} * kC2 >> 16 + */ + "vqdmulh.s16 q8, q2, d0[0] \n" + "vqdmulh.s16 q9, q2, d0[1] \n" + + /* d22 = a = tmp[0] + tmp[8] + * d23 = b = tmp[0] - tmp[8] + */ + "vqadd.s16 d22, d2, d3 \n" + "vqsub.s16 d23, d2, d3 \n" + + /* See long winded explanations prior */ + "vshr.s16 q8, q8, #1 \n" + "vqadd.s16 q8, q2, q8 \n" + + /* d20 = c = in[4]*kC2 - in[12]*kC1 + * d21 = d = in[4]*kC1 + in[12]*kC2 + */ + "vqsub.s16 d20, d18, d17 \n" + "vqadd.s16 d21, d19, d16 \n" + + /* d2 = tmp[0] = a + d + * d3 = tmp[1] = b + c + * d4 = tmp[2] = b - c + * d5 = tmp[3] = a - d + */ + "vqadd.s16 d2, d22, d21 \n" + "vqadd.s16 d3, d23, d20 \n" + "vqsub.s16 d4, d23, d20 \n" + "vqsub.s16 d5, d22, d21 \n" + + "vld1.32 d6[0], [%[dst]], %[kBPS] \n" + "vld1.32 d6[1], [%[dst]], %[kBPS] \n" + "vld1.32 d7[0], [%[dst]], %[kBPS] \n" + "vld1.32 d7[1], [%[dst]], %[kBPS] \n" + + "sub %[dst], %[dst], %[kBPS], lsl #2 \n" + + /* (val) + 4 >> 3 */ + "vrshr.s16 d2, d2, #3 \n" + "vrshr.s16 d3, d3, #3 \n" + "vrshr.s16 d4, d4, #3 \n" + "vrshr.s16 d5, d5, #3 \n" + + "vzip.16 q1, q2 \n" + "vzip.16 q1, q2 \n" + + /* Must accumulate before saturating */ + "vmovl.u8 q8, d6 \n" + "vmovl.u8 q9, d7 \n" + + "vqadd.s16 q1, q1, q8 \n" + "vqadd.s16 q2, q2, q9 \n" + + "vqmovun.s16 d0, q1 \n" + "vqmovun.s16 d1, q2 \n" + + "vst1.32 d0[0], [%[dst]], %[kBPS] \n" + "vst1.32 d0[1], [%[dst]], %[kBPS] \n" + "vst1.32 d1[0], [%[dst]], %[kBPS] \n" + "vst1.32 d1[1], [%[dst]] \n" + + : [in] "+r"(in), [dst] "+r"(dst) /* modified registers */ + : [kBPS] "r"(kBPS), [constants] "r"(constants) /* constants */ + : "memory", "q0", "q1", "q2", "q8", "q9", "q10", "q11" /* clobbered */ + ); +} + +#endif // WEBP_USE_INTRINSICS + +static void TransformTwo(const int16_t* in, uint8_t* dst, int do_two) { + TransformOne(in, dst); + if (do_two) { + TransformOne(in + 16, dst + 4); + } +} + +static void TransformDC(const int16_t* in, uint8_t* dst) { + const int16x8_t DC = vdupq_n_s16(in[0]); + Add4x4(DC, DC, dst); +} + +//------------------------------------------------------------------------------ + +#define STORE_WHT(dst, col, rows) do { \ + *dst = vgetq_lane_s32(rows.val[0], col); (dst) += 16; \ + *dst = vgetq_lane_s32(rows.val[1], col); (dst) += 16; \ + *dst = vgetq_lane_s32(rows.val[2], col); (dst) += 16; \ + *dst = vgetq_lane_s32(rows.val[3], col); (dst) += 16; \ +} while (0) + +static void TransformWHT(const int16_t* in, int16_t* out) { + int32x4x4_t tmp; + + { + // Load the source. + const int16x4_t in00_03 = vld1_s16(in + 0); + const int16x4_t in04_07 = vld1_s16(in + 4); + const int16x4_t in08_11 = vld1_s16(in + 8); + const int16x4_t in12_15 = vld1_s16(in + 12); + const int32x4_t a0 = vaddl_s16(in00_03, in12_15); // in[0..3] + in[12..15] + const int32x4_t a1 = vaddl_s16(in04_07, in08_11); // in[4..7] + in[8..11] + const int32x4_t a2 = vsubl_s16(in04_07, in08_11); // in[4..7] - in[8..11] + const int32x4_t a3 = vsubl_s16(in00_03, in12_15); // in[0..3] - in[12..15] + tmp.val[0] = vaddq_s32(a0, a1); + tmp.val[1] = vaddq_s32(a3, a2); + tmp.val[2] = vsubq_s32(a0, a1); + tmp.val[3] = vsubq_s32(a3, a2); + // Arrange the temporary results column-wise. + tmp = Transpose4x4(tmp); + } + + { + const int32x4_t kCst3 = vdupq_n_s32(3); + const int32x4_t dc = vaddq_s32(tmp.val[0], kCst3); // add rounder + const int32x4_t a0 = vaddq_s32(dc, tmp.val[3]); + const int32x4_t a1 = vaddq_s32(tmp.val[1], tmp.val[2]); + const int32x4_t a2 = vsubq_s32(tmp.val[1], tmp.val[2]); + const int32x4_t a3 = vsubq_s32(dc, tmp.val[3]); + + tmp.val[0] = vaddq_s32(a0, a1); + tmp.val[1] = vaddq_s32(a3, a2); + tmp.val[2] = vsubq_s32(a0, a1); + tmp.val[3] = vsubq_s32(a3, a2); + + // right shift the results by 3. + tmp.val[0] = vshrq_n_s32(tmp.val[0], 3); + tmp.val[1] = vshrq_n_s32(tmp.val[1], 3); + tmp.val[2] = vshrq_n_s32(tmp.val[2], 3); + tmp.val[3] = vshrq_n_s32(tmp.val[3], 3); + + STORE_WHT(out, 0, tmp); + STORE_WHT(out, 1, tmp); + STORE_WHT(out, 2, tmp); + STORE_WHT(out, 3, tmp); + } +} + +#undef STORE_WHT + +//------------------------------------------------------------------------------ + +#define MUL(a, b) (((a) * (b)) >> 16) +static void TransformAC3(const int16_t* in, uint8_t* dst) { + static const int kC1_full = 20091 + (1 << 16); + static const int kC2_full = 35468; + const int16x4_t A = vld1_dup_s16(in); + const int16x4_t c4 = vdup_n_s16(MUL(in[4], kC2_full)); + const int16x4_t d4 = vdup_n_s16(MUL(in[4], kC1_full)); + const int c1 = MUL(in[1], kC2_full); + const int d1 = MUL(in[1], kC1_full); + const uint64_t cd = (uint64_t)( d1 & 0xffff) << 0 | + (uint64_t)( c1 & 0xffff) << 16 | + (uint64_t)(-c1 & 0xffff) << 32 | + (uint64_t)(-d1 & 0xffff) << 48; + const int16x4_t CD = vcreate_s16(cd); + const int16x4_t B = vqadd_s16(A, CD); + const int16x8_t m0_m1 = vcombine_s16(vqadd_s16(B, d4), vqadd_s16(B, c4)); + const int16x8_t m2_m3 = vcombine_s16(vqsub_s16(B, c4), vqsub_s16(B, d4)); + Add4x4(m0_m1, m2_m3, dst); +} +#undef MUL + +//------------------------------------------------------------------------------ +// 4x4 + +static void DC4(uint8_t* dst) { // DC + const uint8x8_t A = vld1_u8(dst - BPS); // top row + const uint16x4_t p0 = vpaddl_u8(A); // cascading summation of the top + const uint16x4_t p1 = vpadd_u16(p0, p0); + const uint16x8_t L0 = vmovl_u8(vld1_u8(dst + 0 * BPS - 1)); + const uint16x8_t L1 = vmovl_u8(vld1_u8(dst + 1 * BPS - 1)); + const uint16x8_t L2 = vmovl_u8(vld1_u8(dst + 2 * BPS - 1)); + const uint16x8_t L3 = vmovl_u8(vld1_u8(dst + 3 * BPS - 1)); + const uint16x8_t s0 = vaddq_u16(L0, L1); + const uint16x8_t s1 = vaddq_u16(L2, L3); + const uint16x8_t s01 = vaddq_u16(s0, s1); + const uint16x8_t sum = vaddq_u16(s01, vcombine_u16(p1, p1)); + const uint8x8_t dc0 = vrshrn_n_u16(sum, 3); // (sum + 4) >> 3 + const uint8x8_t dc = vdup_lane_u8(dc0, 0); + int i; + for (i = 0; i < 4; ++i) { + vst1_lane_u32((uint32_t*)(dst + i * BPS), vreinterpret_u32_u8(dc), 0); + } +} + +// TrueMotion (4x4 + 8x8) +static WEBP_INLINE void TrueMotion(uint8_t* dst, int size) { + const uint8x8_t TL = vld1_dup_u8(dst - BPS - 1); // top-left pixel 'A[-1]' + const uint8x8_t T = vld1_u8(dst - BPS); // top row 'A[0..3]' + const int16x8_t d = vreinterpretq_s16_u16(vsubl_u8(T, TL)); // A[c] - A[-1] + int y; + for (y = 0; y < size; y += 4) { + // left edge + const int16x8_t L0 = ConvertU8ToS16(vld1_dup_u8(dst + 0 * BPS - 1)); + const int16x8_t L1 = ConvertU8ToS16(vld1_dup_u8(dst + 1 * BPS - 1)); + const int16x8_t L2 = ConvertU8ToS16(vld1_dup_u8(dst + 2 * BPS - 1)); + const int16x8_t L3 = ConvertU8ToS16(vld1_dup_u8(dst + 3 * BPS - 1)); + const int16x8_t r0 = vaddq_s16(L0, d); // L[r] + A[c] - A[-1] + const int16x8_t r1 = vaddq_s16(L1, d); + const int16x8_t r2 = vaddq_s16(L2, d); + const int16x8_t r3 = vaddq_s16(L3, d); + // Saturate and store the result. + const uint32x2_t r0_u32 = vreinterpret_u32_u8(vqmovun_s16(r0)); + const uint32x2_t r1_u32 = vreinterpret_u32_u8(vqmovun_s16(r1)); + const uint32x2_t r2_u32 = vreinterpret_u32_u8(vqmovun_s16(r2)); + const uint32x2_t r3_u32 = vreinterpret_u32_u8(vqmovun_s16(r3)); + if (size == 4) { + vst1_lane_u32((uint32_t*)(dst + 0 * BPS), r0_u32, 0); + vst1_lane_u32((uint32_t*)(dst + 1 * BPS), r1_u32, 0); + vst1_lane_u32((uint32_t*)(dst + 2 * BPS), r2_u32, 0); + vst1_lane_u32((uint32_t*)(dst + 3 * BPS), r3_u32, 0); + } else { + vst1_u32((uint32_t*)(dst + 0 * BPS), r0_u32); + vst1_u32((uint32_t*)(dst + 1 * BPS), r1_u32); + vst1_u32((uint32_t*)(dst + 2 * BPS), r2_u32); + vst1_u32((uint32_t*)(dst + 3 * BPS), r3_u32); + } + dst += 4 * BPS; + } +} + +static void TM4(uint8_t* dst) { TrueMotion(dst, 4); } + +static void VE4(uint8_t* dst) { // vertical + // NB: avoid vld1_u64 here as an alignment hint may be added -> SIGBUS. + const uint64x1_t A0 = vreinterpret_u64_u8(vld1_u8(dst - BPS - 1)); // top row + const uint64x1_t A1 = vshr_n_u64(A0, 8); + const uint64x1_t A2 = vshr_n_u64(A0, 16); + const uint8x8_t ABCDEFGH = vreinterpret_u8_u64(A0); + const uint8x8_t BCDEFGH0 = vreinterpret_u8_u64(A1); + const uint8x8_t CDEFGH00 = vreinterpret_u8_u64(A2); + const uint8x8_t b = vhadd_u8(ABCDEFGH, CDEFGH00); + const uint8x8_t avg = vrhadd_u8(b, BCDEFGH0); + int i; + for (i = 0; i < 4; ++i) { + vst1_lane_u32((uint32_t*)(dst + i * BPS), vreinterpret_u32_u8(avg), 0); + } +} + +static void RD4(uint8_t* dst) { // Down-right + const uint8x8_t XABCD_u8 = vld1_u8(dst - BPS - 1); + const uint64x1_t XABCD = vreinterpret_u64_u8(XABCD_u8); + const uint64x1_t ____XABC = vshl_n_u64(XABCD, 32); + const uint32_t I = dst[-1 + 0 * BPS]; + const uint32_t J = dst[-1 + 1 * BPS]; + const uint32_t K = dst[-1 + 2 * BPS]; + const uint32_t L = dst[-1 + 3 * BPS]; + const uint64x1_t LKJI____ = vcreate_u64(L | (K << 8) | (J << 16) | (I << 24)); + const uint64x1_t LKJIXABC = vorr_u64(LKJI____, ____XABC); + const uint8x8_t KJIXABC_ = vreinterpret_u8_u64(vshr_n_u64(LKJIXABC, 8)); + const uint8x8_t JIXABC__ = vreinterpret_u8_u64(vshr_n_u64(LKJIXABC, 16)); + const uint8_t D = vget_lane_u8(XABCD_u8, 4); + const uint8x8_t JIXABCD_ = vset_lane_u8(D, JIXABC__, 6); + const uint8x8_t LKJIXABC_u8 = vreinterpret_u8_u64(LKJIXABC); + const uint8x8_t avg1 = vhadd_u8(JIXABCD_, LKJIXABC_u8); + const uint8x8_t avg2 = vrhadd_u8(avg1, KJIXABC_); + const uint64x1_t avg2_u64 = vreinterpret_u64_u8(avg2); + const uint32x2_t r3 = vreinterpret_u32_u8(avg2); + const uint32x2_t r2 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 8)); + const uint32x2_t r1 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 16)); + const uint32x2_t r0 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 24)); + vst1_lane_u32((uint32_t*)(dst + 0 * BPS), r0, 0); + vst1_lane_u32((uint32_t*)(dst + 1 * BPS), r1, 0); + vst1_lane_u32((uint32_t*)(dst + 2 * BPS), r2, 0); + vst1_lane_u32((uint32_t*)(dst + 3 * BPS), r3, 0); +} + +static void LD4(uint8_t* dst) { // Down-left + // Note using the same shift trick as VE4() is slower here. + const uint8x8_t ABCDEFGH = vld1_u8(dst - BPS + 0); + const uint8x8_t BCDEFGH0 = vld1_u8(dst - BPS + 1); + const uint8x8_t CDEFGH00 = vld1_u8(dst - BPS + 2); + const uint8x8_t CDEFGHH0 = vset_lane_u8(dst[-BPS + 7], CDEFGH00, 6); + const uint8x8_t avg1 = vhadd_u8(ABCDEFGH, CDEFGHH0); + const uint8x8_t avg2 = vrhadd_u8(avg1, BCDEFGH0); + const uint64x1_t avg2_u64 = vreinterpret_u64_u8(avg2); + const uint32x2_t r0 = vreinterpret_u32_u8(avg2); + const uint32x2_t r1 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 8)); + const uint32x2_t r2 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 16)); + const uint32x2_t r3 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 24)); + vst1_lane_u32((uint32_t*)(dst + 0 * BPS), r0, 0); + vst1_lane_u32((uint32_t*)(dst + 1 * BPS), r1, 0); + vst1_lane_u32((uint32_t*)(dst + 2 * BPS), r2, 0); + vst1_lane_u32((uint32_t*)(dst + 3 * BPS), r3, 0); +} + +//------------------------------------------------------------------------------ +// Chroma + +static void VE8uv(uint8_t* dst) { // vertical + const uint8x8_t top = vld1_u8(dst - BPS); + int j; + for (j = 0; j < 8; ++j) { + vst1_u8(dst + j * BPS, top); + } +} + +static void HE8uv(uint8_t* dst) { // horizontal + int j; + for (j = 0; j < 8; ++j) { + const uint8x8_t left = vld1_dup_u8(dst - 1); + vst1_u8(dst, left); + dst += BPS; + } +} + +static WEBP_INLINE void DC8(uint8_t* dst, int do_top, int do_left) { + uint16x8_t sum_top; + uint16x8_t sum_left; + uint8x8_t dc0; + + if (do_top) { + const uint8x8_t A = vld1_u8(dst - BPS); // top row + const uint16x4_t p0 = vpaddl_u8(A); // cascading summation of the top + const uint16x4_t p1 = vpadd_u16(p0, p0); + const uint16x4_t p2 = vpadd_u16(p1, p1); + sum_top = vcombine_u16(p2, p2); + } + + if (do_left) { + const uint16x8_t L0 = vmovl_u8(vld1_u8(dst + 0 * BPS - 1)); + const uint16x8_t L1 = vmovl_u8(vld1_u8(dst + 1 * BPS - 1)); + const uint16x8_t L2 = vmovl_u8(vld1_u8(dst + 2 * BPS - 1)); + const uint16x8_t L3 = vmovl_u8(vld1_u8(dst + 3 * BPS - 1)); + const uint16x8_t L4 = vmovl_u8(vld1_u8(dst + 4 * BPS - 1)); + const uint16x8_t L5 = vmovl_u8(vld1_u8(dst + 5 * BPS - 1)); + const uint16x8_t L6 = vmovl_u8(vld1_u8(dst + 6 * BPS - 1)); + const uint16x8_t L7 = vmovl_u8(vld1_u8(dst + 7 * BPS - 1)); + const uint16x8_t s0 = vaddq_u16(L0, L1); + const uint16x8_t s1 = vaddq_u16(L2, L3); + const uint16x8_t s2 = vaddq_u16(L4, L5); + const uint16x8_t s3 = vaddq_u16(L6, L7); + const uint16x8_t s01 = vaddq_u16(s0, s1); + const uint16x8_t s23 = vaddq_u16(s2, s3); + sum_left = vaddq_u16(s01, s23); + } + + if (do_top && do_left) { + const uint16x8_t sum = vaddq_u16(sum_left, sum_top); + dc0 = vrshrn_n_u16(sum, 4); + } else if (do_top) { + dc0 = vrshrn_n_u16(sum_top, 3); + } else if (do_left) { + dc0 = vrshrn_n_u16(sum_left, 3); + } else { + dc0 = vdup_n_u8(0x80); + } + + { + const uint8x8_t dc = vdup_lane_u8(dc0, 0); + int i; + for (i = 0; i < 8; ++i) { + vst1_u32((uint32_t*)(dst + i * BPS), vreinterpret_u32_u8(dc)); + } + } +} + +static void DC8uv(uint8_t* dst) { DC8(dst, 1, 1); } +static void DC8uvNoTop(uint8_t* dst) { DC8(dst, 0, 1); } +static void DC8uvNoLeft(uint8_t* dst) { DC8(dst, 1, 0); } +static void DC8uvNoTopLeft(uint8_t* dst) { DC8(dst, 0, 0); } + +static void TM8uv(uint8_t* dst) { TrueMotion(dst, 8); } + +//------------------------------------------------------------------------------ +// 16x16 + +static void VE16(uint8_t* dst) { // vertical + const uint8x16_t top = vld1q_u8(dst - BPS); + int j; + for (j = 0; j < 16; ++j) { + vst1q_u8(dst + j * BPS, top); + } +} + +static void HE16(uint8_t* dst) { // horizontal + int j; + for (j = 0; j < 16; ++j) { + const uint8x16_t left = vld1q_dup_u8(dst - 1); + vst1q_u8(dst, left); + dst += BPS; + } +} + +static WEBP_INLINE void DC16(uint8_t* dst, int do_top, int do_left) { + uint16x8_t sum_top; + uint16x8_t sum_left; + uint8x8_t dc0; + + if (do_top) { + const uint8x16_t A = vld1q_u8(dst - BPS); // top row + const uint16x8_t p0 = vpaddlq_u8(A); // cascading summation of the top + const uint16x4_t p1 = vadd_u16(vget_low_u16(p0), vget_high_u16(p0)); + const uint16x4_t p2 = vpadd_u16(p1, p1); + const uint16x4_t p3 = vpadd_u16(p2, p2); + sum_top = vcombine_u16(p3, p3); + } + + if (do_left) { + int i; + sum_left = vdupq_n_u16(0); + for (i = 0; i < 16; i += 8) { + const uint16x8_t L0 = vmovl_u8(vld1_u8(dst + (i + 0) * BPS - 1)); + const uint16x8_t L1 = vmovl_u8(vld1_u8(dst + (i + 1) * BPS - 1)); + const uint16x8_t L2 = vmovl_u8(vld1_u8(dst + (i + 2) * BPS - 1)); + const uint16x8_t L3 = vmovl_u8(vld1_u8(dst + (i + 3) * BPS - 1)); + const uint16x8_t L4 = vmovl_u8(vld1_u8(dst + (i + 4) * BPS - 1)); + const uint16x8_t L5 = vmovl_u8(vld1_u8(dst + (i + 5) * BPS - 1)); + const uint16x8_t L6 = vmovl_u8(vld1_u8(dst + (i + 6) * BPS - 1)); + const uint16x8_t L7 = vmovl_u8(vld1_u8(dst + (i + 7) * BPS - 1)); + const uint16x8_t s0 = vaddq_u16(L0, L1); + const uint16x8_t s1 = vaddq_u16(L2, L3); + const uint16x8_t s2 = vaddq_u16(L4, L5); + const uint16x8_t s3 = vaddq_u16(L6, L7); + const uint16x8_t s01 = vaddq_u16(s0, s1); + const uint16x8_t s23 = vaddq_u16(s2, s3); + const uint16x8_t sum = vaddq_u16(s01, s23); + sum_left = vaddq_u16(sum_left, sum); + } + } + + if (do_top && do_left) { + const uint16x8_t sum = vaddq_u16(sum_left, sum_top); + dc0 = vrshrn_n_u16(sum, 5); + } else if (do_top) { + dc0 = vrshrn_n_u16(sum_top, 4); + } else if (do_left) { + dc0 = vrshrn_n_u16(sum_left, 4); + } else { + dc0 = vdup_n_u8(0x80); + } + + { + const uint8x16_t dc = vdupq_lane_u8(dc0, 0); + int i; + for (i = 0; i < 16; ++i) { + vst1q_u8(dst + i * BPS, dc); + } + } +} + +static void DC16TopLeft(uint8_t* dst) { DC16(dst, 1, 1); } +static void DC16NoTop(uint8_t* dst) { DC16(dst, 0, 1); } +static void DC16NoLeft(uint8_t* dst) { DC16(dst, 1, 0); } +static void DC16NoTopLeft(uint8_t* dst) { DC16(dst, 0, 0); } + +static void TM16(uint8_t* dst) { + const uint8x8_t TL = vld1_dup_u8(dst - BPS - 1); // top-left pixel 'A[-1]' + const uint8x16_t T = vld1q_u8(dst - BPS); // top row 'A[0..15]' + // A[c] - A[-1] + const int16x8_t d_lo = vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(T), TL)); + const int16x8_t d_hi = vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(T), TL)); + int y; + for (y = 0; y < 16; y += 4) { + // left edge + const int16x8_t L0 = ConvertU8ToS16(vld1_dup_u8(dst + 0 * BPS - 1)); + const int16x8_t L1 = ConvertU8ToS16(vld1_dup_u8(dst + 1 * BPS - 1)); + const int16x8_t L2 = ConvertU8ToS16(vld1_dup_u8(dst + 2 * BPS - 1)); + const int16x8_t L3 = ConvertU8ToS16(vld1_dup_u8(dst + 3 * BPS - 1)); + const int16x8_t r0_lo = vaddq_s16(L0, d_lo); // L[r] + A[c] - A[-1] + const int16x8_t r1_lo = vaddq_s16(L1, d_lo); + const int16x8_t r2_lo = vaddq_s16(L2, d_lo); + const int16x8_t r3_lo = vaddq_s16(L3, d_lo); + const int16x8_t r0_hi = vaddq_s16(L0, d_hi); + const int16x8_t r1_hi = vaddq_s16(L1, d_hi); + const int16x8_t r2_hi = vaddq_s16(L2, d_hi); + const int16x8_t r3_hi = vaddq_s16(L3, d_hi); + // Saturate and store the result. + const uint8x16_t row0 = vcombine_u8(vqmovun_s16(r0_lo), vqmovun_s16(r0_hi)); + const uint8x16_t row1 = vcombine_u8(vqmovun_s16(r1_lo), vqmovun_s16(r1_hi)); + const uint8x16_t row2 = vcombine_u8(vqmovun_s16(r2_lo), vqmovun_s16(r2_hi)); + const uint8x16_t row3 = vcombine_u8(vqmovun_s16(r3_lo), vqmovun_s16(r3_hi)); + vst1q_u8(dst + 0 * BPS, row0); + vst1q_u8(dst + 1 * BPS, row1); + vst1q_u8(dst + 2 * BPS, row2); + vst1q_u8(dst + 3 * BPS, row3); + dst += 4 * BPS; + } +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8DspInitNEON(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8DspInitNEON(void) { + VP8Transform = TransformTwo; + VP8TransformAC3 = TransformAC3; + VP8TransformDC = TransformDC; + VP8TransformWHT = TransformWHT; + + VP8VFilter16 = VFilter16; + VP8VFilter16i = VFilter16i; + VP8HFilter16 = HFilter16; +#if !defined(WORK_AROUND_GCC) + VP8HFilter16i = HFilter16i; +#endif + VP8VFilter8 = VFilter8; + VP8VFilter8i = VFilter8i; +#if !defined(WORK_AROUND_GCC) + VP8HFilter8 = HFilter8; + VP8HFilter8i = HFilter8i; +#endif + VP8SimpleVFilter16 = SimpleVFilter16; + VP8SimpleHFilter16 = SimpleHFilter16; + VP8SimpleVFilter16i = SimpleVFilter16i; + VP8SimpleHFilter16i = SimpleHFilter16i; + + VP8PredLuma4[0] = DC4; + VP8PredLuma4[1] = TM4; + VP8PredLuma4[2] = VE4; + VP8PredLuma4[4] = RD4; + VP8PredLuma4[6] = LD4; + + VP8PredLuma16[0] = DC16TopLeft; + VP8PredLuma16[1] = TM16; + VP8PredLuma16[2] = VE16; + VP8PredLuma16[3] = HE16; + VP8PredLuma16[4] = DC16NoTop; + VP8PredLuma16[5] = DC16NoLeft; + VP8PredLuma16[6] = DC16NoTopLeft; + + VP8PredChroma8[0] = DC8uv; + VP8PredChroma8[1] = TM8uv; + VP8PredChroma8[2] = VE8uv; + VP8PredChroma8[3] = HE8uv; + VP8PredChroma8[4] = DC8uvNoTop; + VP8PredChroma8[5] = DC8uvNoLeft; + VP8PredChroma8[6] = DC8uvNoTopLeft; +} + +#else // !WEBP_USE_NEON + +WEBP_DSP_INIT_STUB(VP8DspInitNEON) + +#endif // WEBP_USE_NEON diff --git a/thirdparty/libwebp/dsp/dec_sse2.c b/thirdparty/libwebp/dsp/dec_sse2.c new file mode 100644 index 0000000000..f0a8ddcaf3 --- /dev/null +++ b/thirdparty/libwebp/dsp/dec_sse2.c @@ -0,0 +1,1231 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE2 version of some decoding functions (idct, loop filtering). +// +// Author: somnath@google.com (Somnath Banerjee) +// cduvivier@google.com (Christian Duvivier) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE2) + +// The 3-coeff sparse transform in SSE2 is not really faster than the plain-C +// one it seems => disable it by default. Uncomment the following to enable: +// #define USE_TRANSFORM_AC3 + +#include <emmintrin.h> +#include "./common_sse2.h" +#include "../dec/vp8i.h" +#include "../utils/utils.h" + +//------------------------------------------------------------------------------ +// Transforms (Paragraph 14.4) + +static void Transform(const int16_t* in, uint8_t* dst, int do_two) { + // This implementation makes use of 16-bit fixed point versions of two + // multiply constants: + // K1 = sqrt(2) * cos (pi/8) ~= 85627 / 2^16 + // K2 = sqrt(2) * sin (pi/8) ~= 35468 / 2^16 + // + // To be able to use signed 16-bit integers, we use the following trick to + // have constants within range: + // - Associated constants are obtained by subtracting the 16-bit fixed point + // version of one: + // k = K - (1 << 16) => K = k + (1 << 16) + // K1 = 85267 => k1 = 20091 + // K2 = 35468 => k2 = -30068 + // - The multiplication of a variable by a constant become the sum of the + // variable and the multiplication of that variable by the associated + // constant: + // (x * K) >> 16 = (x * (k + (1 << 16))) >> 16 = ((x * k ) >> 16) + x + const __m128i k1 = _mm_set1_epi16(20091); + const __m128i k2 = _mm_set1_epi16(-30068); + __m128i T0, T1, T2, T3; + + // Load and concatenate the transform coefficients (we'll do two transforms + // in parallel). In the case of only one transform, the second half of the + // vectors will just contain random value we'll never use nor store. + __m128i in0, in1, in2, in3; + { + in0 = _mm_loadl_epi64((const __m128i*)&in[0]); + in1 = _mm_loadl_epi64((const __m128i*)&in[4]); + in2 = _mm_loadl_epi64((const __m128i*)&in[8]); + in3 = _mm_loadl_epi64((const __m128i*)&in[12]); + // a00 a10 a20 a30 x x x x + // a01 a11 a21 a31 x x x x + // a02 a12 a22 a32 x x x x + // a03 a13 a23 a33 x x x x + if (do_two) { + const __m128i inB0 = _mm_loadl_epi64((const __m128i*)&in[16]); + const __m128i inB1 = _mm_loadl_epi64((const __m128i*)&in[20]); + const __m128i inB2 = _mm_loadl_epi64((const __m128i*)&in[24]); + const __m128i inB3 = _mm_loadl_epi64((const __m128i*)&in[28]); + in0 = _mm_unpacklo_epi64(in0, inB0); + in1 = _mm_unpacklo_epi64(in1, inB1); + in2 = _mm_unpacklo_epi64(in2, inB2); + in3 = _mm_unpacklo_epi64(in3, inB3); + // a00 a10 a20 a30 b00 b10 b20 b30 + // a01 a11 a21 a31 b01 b11 b21 b31 + // a02 a12 a22 a32 b02 b12 b22 b32 + // a03 a13 a23 a33 b03 b13 b23 b33 + } + } + + // Vertical pass and subsequent transpose. + { + // First pass, c and d calculations are longer because of the "trick" + // multiplications. + const __m128i a = _mm_add_epi16(in0, in2); + const __m128i b = _mm_sub_epi16(in0, in2); + // c = MUL(in1, K2) - MUL(in3, K1) = MUL(in1, k2) - MUL(in3, k1) + in1 - in3 + const __m128i c1 = _mm_mulhi_epi16(in1, k2); + const __m128i c2 = _mm_mulhi_epi16(in3, k1); + const __m128i c3 = _mm_sub_epi16(in1, in3); + const __m128i c4 = _mm_sub_epi16(c1, c2); + const __m128i c = _mm_add_epi16(c3, c4); + // d = MUL(in1, K1) + MUL(in3, K2) = MUL(in1, k1) + MUL(in3, k2) + in1 + in3 + const __m128i d1 = _mm_mulhi_epi16(in1, k1); + const __m128i d2 = _mm_mulhi_epi16(in3, k2); + const __m128i d3 = _mm_add_epi16(in1, in3); + const __m128i d4 = _mm_add_epi16(d1, d2); + const __m128i d = _mm_add_epi16(d3, d4); + + // Second pass. + const __m128i tmp0 = _mm_add_epi16(a, d); + const __m128i tmp1 = _mm_add_epi16(b, c); + const __m128i tmp2 = _mm_sub_epi16(b, c); + const __m128i tmp3 = _mm_sub_epi16(a, d); + + // Transpose the two 4x4. + VP8Transpose_2_4x4_16b(&tmp0, &tmp1, &tmp2, &tmp3, &T0, &T1, &T2, &T3); + } + + // Horizontal pass and subsequent transpose. + { + // First pass, c and d calculations are longer because of the "trick" + // multiplications. + const __m128i four = _mm_set1_epi16(4); + const __m128i dc = _mm_add_epi16(T0, four); + const __m128i a = _mm_add_epi16(dc, T2); + const __m128i b = _mm_sub_epi16(dc, T2); + // c = MUL(T1, K2) - MUL(T3, K1) = MUL(T1, k2) - MUL(T3, k1) + T1 - T3 + const __m128i c1 = _mm_mulhi_epi16(T1, k2); + const __m128i c2 = _mm_mulhi_epi16(T3, k1); + const __m128i c3 = _mm_sub_epi16(T1, T3); + const __m128i c4 = _mm_sub_epi16(c1, c2); + const __m128i c = _mm_add_epi16(c3, c4); + // d = MUL(T1, K1) + MUL(T3, K2) = MUL(T1, k1) + MUL(T3, k2) + T1 + T3 + const __m128i d1 = _mm_mulhi_epi16(T1, k1); + const __m128i d2 = _mm_mulhi_epi16(T3, k2); + const __m128i d3 = _mm_add_epi16(T1, T3); + const __m128i d4 = _mm_add_epi16(d1, d2); + const __m128i d = _mm_add_epi16(d3, d4); + + // Second pass. + const __m128i tmp0 = _mm_add_epi16(a, d); + const __m128i tmp1 = _mm_add_epi16(b, c); + const __m128i tmp2 = _mm_sub_epi16(b, c); + const __m128i tmp3 = _mm_sub_epi16(a, d); + const __m128i shifted0 = _mm_srai_epi16(tmp0, 3); + const __m128i shifted1 = _mm_srai_epi16(tmp1, 3); + const __m128i shifted2 = _mm_srai_epi16(tmp2, 3); + const __m128i shifted3 = _mm_srai_epi16(tmp3, 3); + + // Transpose the two 4x4. + VP8Transpose_2_4x4_16b(&shifted0, &shifted1, &shifted2, &shifted3, &T0, &T1, + &T2, &T3); + } + + // Add inverse transform to 'dst' and store. + { + const __m128i zero = _mm_setzero_si128(); + // Load the reference(s). + __m128i dst0, dst1, dst2, dst3; + if (do_two) { + // Load eight bytes/pixels per line. + dst0 = _mm_loadl_epi64((__m128i*)(dst + 0 * BPS)); + dst1 = _mm_loadl_epi64((__m128i*)(dst + 1 * BPS)); + dst2 = _mm_loadl_epi64((__m128i*)(dst + 2 * BPS)); + dst3 = _mm_loadl_epi64((__m128i*)(dst + 3 * BPS)); + } else { + // Load four bytes/pixels per line. + dst0 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 0 * BPS)); + dst1 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 1 * BPS)); + dst2 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 2 * BPS)); + dst3 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 3 * BPS)); + } + // Convert to 16b. + dst0 = _mm_unpacklo_epi8(dst0, zero); + dst1 = _mm_unpacklo_epi8(dst1, zero); + dst2 = _mm_unpacklo_epi8(dst2, zero); + dst3 = _mm_unpacklo_epi8(dst3, zero); + // Add the inverse transform(s). + dst0 = _mm_add_epi16(dst0, T0); + dst1 = _mm_add_epi16(dst1, T1); + dst2 = _mm_add_epi16(dst2, T2); + dst3 = _mm_add_epi16(dst3, T3); + // Unsigned saturate to 8b. + dst0 = _mm_packus_epi16(dst0, dst0); + dst1 = _mm_packus_epi16(dst1, dst1); + dst2 = _mm_packus_epi16(dst2, dst2); + dst3 = _mm_packus_epi16(dst3, dst3); + // Store the results. + if (do_two) { + // Store eight bytes/pixels per line. + _mm_storel_epi64((__m128i*)(dst + 0 * BPS), dst0); + _mm_storel_epi64((__m128i*)(dst + 1 * BPS), dst1); + _mm_storel_epi64((__m128i*)(dst + 2 * BPS), dst2); + _mm_storel_epi64((__m128i*)(dst + 3 * BPS), dst3); + } else { + // Store four bytes/pixels per line. + WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(dst0)); + WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(dst1)); + WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(dst2)); + WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(dst3)); + } + } +} + +#if defined(USE_TRANSFORM_AC3) +#define MUL(a, b) (((a) * (b)) >> 16) +static void TransformAC3(const int16_t* in, uint8_t* dst) { + static const int kC1 = 20091 + (1 << 16); + static const int kC2 = 35468; + const __m128i A = _mm_set1_epi16(in[0] + 4); + const __m128i c4 = _mm_set1_epi16(MUL(in[4], kC2)); + const __m128i d4 = _mm_set1_epi16(MUL(in[4], kC1)); + const int c1 = MUL(in[1], kC2); + const int d1 = MUL(in[1], kC1); + const __m128i CD = _mm_set_epi16(0, 0, 0, 0, -d1, -c1, c1, d1); + const __m128i B = _mm_adds_epi16(A, CD); + const __m128i m0 = _mm_adds_epi16(B, d4); + const __m128i m1 = _mm_adds_epi16(B, c4); + const __m128i m2 = _mm_subs_epi16(B, c4); + const __m128i m3 = _mm_subs_epi16(B, d4); + const __m128i zero = _mm_setzero_si128(); + // Load the source pixels. + __m128i dst0 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 0 * BPS)); + __m128i dst1 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 1 * BPS)); + __m128i dst2 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 2 * BPS)); + __m128i dst3 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 3 * BPS)); + // Convert to 16b. + dst0 = _mm_unpacklo_epi8(dst0, zero); + dst1 = _mm_unpacklo_epi8(dst1, zero); + dst2 = _mm_unpacklo_epi8(dst2, zero); + dst3 = _mm_unpacklo_epi8(dst3, zero); + // Add the inverse transform. + dst0 = _mm_adds_epi16(dst0, _mm_srai_epi16(m0, 3)); + dst1 = _mm_adds_epi16(dst1, _mm_srai_epi16(m1, 3)); + dst2 = _mm_adds_epi16(dst2, _mm_srai_epi16(m2, 3)); + dst3 = _mm_adds_epi16(dst3, _mm_srai_epi16(m3, 3)); + // Unsigned saturate to 8b. + dst0 = _mm_packus_epi16(dst0, dst0); + dst1 = _mm_packus_epi16(dst1, dst1); + dst2 = _mm_packus_epi16(dst2, dst2); + dst3 = _mm_packus_epi16(dst3, dst3); + // Store the results. + WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(dst0)); + WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(dst1)); + WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(dst2)); + WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(dst3)); +} +#undef MUL +#endif // USE_TRANSFORM_AC3 + +//------------------------------------------------------------------------------ +// Loop Filter (Paragraph 15) + +// Compute abs(p - q) = subs(p - q) OR subs(q - p) +#define MM_ABS(p, q) _mm_or_si128( \ + _mm_subs_epu8((q), (p)), \ + _mm_subs_epu8((p), (q))) + +// Shift each byte of "x" by 3 bits while preserving by the sign bit. +static WEBP_INLINE void SignedShift8b(__m128i* const x) { + const __m128i zero = _mm_setzero_si128(); + const __m128i lo_0 = _mm_unpacklo_epi8(zero, *x); + const __m128i hi_0 = _mm_unpackhi_epi8(zero, *x); + const __m128i lo_1 = _mm_srai_epi16(lo_0, 3 + 8); + const __m128i hi_1 = _mm_srai_epi16(hi_0, 3 + 8); + *x = _mm_packs_epi16(lo_1, hi_1); +} + +#define FLIP_SIGN_BIT2(a, b) { \ + a = _mm_xor_si128(a, sign_bit); \ + b = _mm_xor_si128(b, sign_bit); \ +} + +#define FLIP_SIGN_BIT4(a, b, c, d) { \ + FLIP_SIGN_BIT2(a, b); \ + FLIP_SIGN_BIT2(c, d); \ +} + +// input/output is uint8_t +static WEBP_INLINE void GetNotHEV(const __m128i* const p1, + const __m128i* const p0, + const __m128i* const q0, + const __m128i* const q1, + int hev_thresh, __m128i* const not_hev) { + const __m128i zero = _mm_setzero_si128(); + const __m128i t_1 = MM_ABS(*p1, *p0); + const __m128i t_2 = MM_ABS(*q1, *q0); + + const __m128i h = _mm_set1_epi8(hev_thresh); + const __m128i t_max = _mm_max_epu8(t_1, t_2); + + const __m128i t_max_h = _mm_subs_epu8(t_max, h); + *not_hev = _mm_cmpeq_epi8(t_max_h, zero); // not_hev <= t1 && not_hev <= t2 +} + +// input pixels are int8_t +static WEBP_INLINE void GetBaseDelta(const __m128i* const p1, + const __m128i* const p0, + const __m128i* const q0, + const __m128i* const q1, + __m128i* const delta) { + // beware of addition order, for saturation! + const __m128i p1_q1 = _mm_subs_epi8(*p1, *q1); // p1 - q1 + const __m128i q0_p0 = _mm_subs_epi8(*q0, *p0); // q0 - p0 + const __m128i s1 = _mm_adds_epi8(p1_q1, q0_p0); // p1 - q1 + 1 * (q0 - p0) + const __m128i s2 = _mm_adds_epi8(q0_p0, s1); // p1 - q1 + 2 * (q0 - p0) + const __m128i s3 = _mm_adds_epi8(q0_p0, s2); // p1 - q1 + 3 * (q0 - p0) + *delta = s3; +} + +// input and output are int8_t +static WEBP_INLINE void DoSimpleFilter(__m128i* const p0, __m128i* const q0, + const __m128i* const fl) { + const __m128i k3 = _mm_set1_epi8(3); + const __m128i k4 = _mm_set1_epi8(4); + __m128i v3 = _mm_adds_epi8(*fl, k3); + __m128i v4 = _mm_adds_epi8(*fl, k4); + + SignedShift8b(&v4); // v4 >> 3 + SignedShift8b(&v3); // v3 >> 3 + *q0 = _mm_subs_epi8(*q0, v4); // q0 -= v4 + *p0 = _mm_adds_epi8(*p0, v3); // p0 += v3 +} + +// Updates values of 2 pixels at MB edge during complex filtering. +// Update operations: +// q = q - delta and p = p + delta; where delta = [(a_hi >> 7), (a_lo >> 7)] +// Pixels 'pi' and 'qi' are int8_t on input, uint8_t on output (sign flip). +static WEBP_INLINE void Update2Pixels(__m128i* const pi, __m128i* const qi, + const __m128i* const a0_lo, + const __m128i* const a0_hi) { + const __m128i a1_lo = _mm_srai_epi16(*a0_lo, 7); + const __m128i a1_hi = _mm_srai_epi16(*a0_hi, 7); + const __m128i delta = _mm_packs_epi16(a1_lo, a1_hi); + const __m128i sign_bit = _mm_set1_epi8(0x80); + *pi = _mm_adds_epi8(*pi, delta); + *qi = _mm_subs_epi8(*qi, delta); + FLIP_SIGN_BIT2(*pi, *qi); +} + +// input pixels are uint8_t +static WEBP_INLINE void NeedsFilter(const __m128i* const p1, + const __m128i* const p0, + const __m128i* const q0, + const __m128i* const q1, + int thresh, __m128i* const mask) { + const __m128i m_thresh = _mm_set1_epi8(thresh); + const __m128i t1 = MM_ABS(*p1, *q1); // abs(p1 - q1) + const __m128i kFE = _mm_set1_epi8(0xFE); + const __m128i t2 = _mm_and_si128(t1, kFE); // set lsb of each byte to zero + const __m128i t3 = _mm_srli_epi16(t2, 1); // abs(p1 - q1) / 2 + + const __m128i t4 = MM_ABS(*p0, *q0); // abs(p0 - q0) + const __m128i t5 = _mm_adds_epu8(t4, t4); // abs(p0 - q0) * 2 + const __m128i t6 = _mm_adds_epu8(t5, t3); // abs(p0-q0)*2 + abs(p1-q1)/2 + + const __m128i t7 = _mm_subs_epu8(t6, m_thresh); // mask <= m_thresh + *mask = _mm_cmpeq_epi8(t7, _mm_setzero_si128()); +} + +//------------------------------------------------------------------------------ +// Edge filtering functions + +// Applies filter on 2 pixels (p0 and q0) +static WEBP_INLINE void DoFilter2(__m128i* const p1, __m128i* const p0, + __m128i* const q0, __m128i* const q1, + int thresh) { + __m128i a, mask; + const __m128i sign_bit = _mm_set1_epi8(0x80); + // convert p1/q1 to int8_t (for GetBaseDelta) + const __m128i p1s = _mm_xor_si128(*p1, sign_bit); + const __m128i q1s = _mm_xor_si128(*q1, sign_bit); + + NeedsFilter(p1, p0, q0, q1, thresh, &mask); + + FLIP_SIGN_BIT2(*p0, *q0); + GetBaseDelta(&p1s, p0, q0, &q1s, &a); + a = _mm_and_si128(a, mask); // mask filter values we don't care about + DoSimpleFilter(p0, q0, &a); + FLIP_SIGN_BIT2(*p0, *q0); +} + +// Applies filter on 4 pixels (p1, p0, q0 and q1) +static WEBP_INLINE void DoFilter4(__m128i* const p1, __m128i* const p0, + __m128i* const q0, __m128i* const q1, + const __m128i* const mask, int hev_thresh) { + const __m128i zero = _mm_setzero_si128(); + const __m128i sign_bit = _mm_set1_epi8(0x80); + const __m128i k64 = _mm_set1_epi8(64); + const __m128i k3 = _mm_set1_epi8(3); + const __m128i k4 = _mm_set1_epi8(4); + __m128i not_hev; + __m128i t1, t2, t3; + + // compute hev mask + GetNotHEV(p1, p0, q0, q1, hev_thresh, ¬_hev); + + // convert to signed values + FLIP_SIGN_BIT4(*p1, *p0, *q0, *q1); + + t1 = _mm_subs_epi8(*p1, *q1); // p1 - q1 + t1 = _mm_andnot_si128(not_hev, t1); // hev(p1 - q1) + t2 = _mm_subs_epi8(*q0, *p0); // q0 - p0 + t1 = _mm_adds_epi8(t1, t2); // hev(p1 - q1) + 1 * (q0 - p0) + t1 = _mm_adds_epi8(t1, t2); // hev(p1 - q1) + 2 * (q0 - p0) + t1 = _mm_adds_epi8(t1, t2); // hev(p1 - q1) + 3 * (q0 - p0) + t1 = _mm_and_si128(t1, *mask); // mask filter values we don't care about + + t2 = _mm_adds_epi8(t1, k3); // 3 * (q0 - p0) + hev(p1 - q1) + 3 + t3 = _mm_adds_epi8(t1, k4); // 3 * (q0 - p0) + hev(p1 - q1) + 4 + SignedShift8b(&t2); // (3 * (q0 - p0) + hev(p1 - q1) + 3) >> 3 + SignedShift8b(&t3); // (3 * (q0 - p0) + hev(p1 - q1) + 4) >> 3 + *p0 = _mm_adds_epi8(*p0, t2); // p0 += t2 + *q0 = _mm_subs_epi8(*q0, t3); // q0 -= t3 + FLIP_SIGN_BIT2(*p0, *q0); + + // this is equivalent to signed (a + 1) >> 1 calculation + t2 = _mm_add_epi8(t3, sign_bit); + t3 = _mm_avg_epu8(t2, zero); + t3 = _mm_sub_epi8(t3, k64); + + t3 = _mm_and_si128(not_hev, t3); // if !hev + *q1 = _mm_subs_epi8(*q1, t3); // q1 -= t3 + *p1 = _mm_adds_epi8(*p1, t3); // p1 += t3 + FLIP_SIGN_BIT2(*p1, *q1); +} + +// Applies filter on 6 pixels (p2, p1, p0, q0, q1 and q2) +static WEBP_INLINE void DoFilter6(__m128i* const p2, __m128i* const p1, + __m128i* const p0, __m128i* const q0, + __m128i* const q1, __m128i* const q2, + const __m128i* const mask, int hev_thresh) { + const __m128i zero = _mm_setzero_si128(); + const __m128i sign_bit = _mm_set1_epi8(0x80); + __m128i a, not_hev; + + // compute hev mask + GetNotHEV(p1, p0, q0, q1, hev_thresh, ¬_hev); + + FLIP_SIGN_BIT4(*p1, *p0, *q0, *q1); + FLIP_SIGN_BIT2(*p2, *q2); + GetBaseDelta(p1, p0, q0, q1, &a); + + { // do simple filter on pixels with hev + const __m128i m = _mm_andnot_si128(not_hev, *mask); + const __m128i f = _mm_and_si128(a, m); + DoSimpleFilter(p0, q0, &f); + } + + { // do strong filter on pixels with not hev + const __m128i k9 = _mm_set1_epi16(0x0900); + const __m128i k63 = _mm_set1_epi16(63); + + const __m128i m = _mm_and_si128(not_hev, *mask); + const __m128i f = _mm_and_si128(a, m); + + const __m128i f_lo = _mm_unpacklo_epi8(zero, f); + const __m128i f_hi = _mm_unpackhi_epi8(zero, f); + + const __m128i f9_lo = _mm_mulhi_epi16(f_lo, k9); // Filter (lo) * 9 + const __m128i f9_hi = _mm_mulhi_epi16(f_hi, k9); // Filter (hi) * 9 + + const __m128i a2_lo = _mm_add_epi16(f9_lo, k63); // Filter * 9 + 63 + const __m128i a2_hi = _mm_add_epi16(f9_hi, k63); // Filter * 9 + 63 + + const __m128i a1_lo = _mm_add_epi16(a2_lo, f9_lo); // Filter * 18 + 63 + const __m128i a1_hi = _mm_add_epi16(a2_hi, f9_hi); // Filter * 18 + 63 + + const __m128i a0_lo = _mm_add_epi16(a1_lo, f9_lo); // Filter * 27 + 63 + const __m128i a0_hi = _mm_add_epi16(a1_hi, f9_hi); // Filter * 27 + 63 + + Update2Pixels(p2, q2, &a2_lo, &a2_hi); + Update2Pixels(p1, q1, &a1_lo, &a1_hi); + Update2Pixels(p0, q0, &a0_lo, &a0_hi); + } +} + +// reads 8 rows across a vertical edge. +static WEBP_INLINE void Load8x4(const uint8_t* const b, int stride, + __m128i* const p, __m128i* const q) { + // A0 = 63 62 61 60 23 22 21 20 43 42 41 40 03 02 01 00 + // A1 = 73 72 71 70 33 32 31 30 53 52 51 50 13 12 11 10 + const __m128i A0 = _mm_set_epi32( + WebPMemToUint32(&b[6 * stride]), WebPMemToUint32(&b[2 * stride]), + WebPMemToUint32(&b[4 * stride]), WebPMemToUint32(&b[0 * stride])); + const __m128i A1 = _mm_set_epi32( + WebPMemToUint32(&b[7 * stride]), WebPMemToUint32(&b[3 * stride]), + WebPMemToUint32(&b[5 * stride]), WebPMemToUint32(&b[1 * stride])); + + // B0 = 53 43 52 42 51 41 50 40 13 03 12 02 11 01 10 00 + // B1 = 73 63 72 62 71 61 70 60 33 23 32 22 31 21 30 20 + const __m128i B0 = _mm_unpacklo_epi8(A0, A1); + const __m128i B1 = _mm_unpackhi_epi8(A0, A1); + + // C0 = 33 23 13 03 32 22 12 02 31 21 11 01 30 20 10 00 + // C1 = 73 63 53 43 72 62 52 42 71 61 51 41 70 60 50 40 + const __m128i C0 = _mm_unpacklo_epi16(B0, B1); + const __m128i C1 = _mm_unpackhi_epi16(B0, B1); + + // *p = 71 61 51 41 31 21 11 01 70 60 50 40 30 20 10 00 + // *q = 73 63 53 43 33 23 13 03 72 62 52 42 32 22 12 02 + *p = _mm_unpacklo_epi32(C0, C1); + *q = _mm_unpackhi_epi32(C0, C1); +} + +static WEBP_INLINE void Load16x4(const uint8_t* const r0, + const uint8_t* const r8, + int stride, + __m128i* const p1, __m128i* const p0, + __m128i* const q0, __m128i* const q1) { + // Assume the pixels around the edge (|) are numbered as follows + // 00 01 | 02 03 + // 10 11 | 12 13 + // ... | ... + // e0 e1 | e2 e3 + // f0 f1 | f2 f3 + // + // r0 is pointing to the 0th row (00) + // r8 is pointing to the 8th row (80) + + // Load + // p1 = 71 61 51 41 31 21 11 01 70 60 50 40 30 20 10 00 + // q0 = 73 63 53 43 33 23 13 03 72 62 52 42 32 22 12 02 + // p0 = f1 e1 d1 c1 b1 a1 91 81 f0 e0 d0 c0 b0 a0 90 80 + // q1 = f3 e3 d3 c3 b3 a3 93 83 f2 e2 d2 c2 b2 a2 92 82 + Load8x4(r0, stride, p1, q0); + Load8x4(r8, stride, p0, q1); + + { + // p1 = f0 e0 d0 c0 b0 a0 90 80 70 60 50 40 30 20 10 00 + // p0 = f1 e1 d1 c1 b1 a1 91 81 71 61 51 41 31 21 11 01 + // q0 = f2 e2 d2 c2 b2 a2 92 82 72 62 52 42 32 22 12 02 + // q1 = f3 e3 d3 c3 b3 a3 93 83 73 63 53 43 33 23 13 03 + const __m128i t1 = *p1; + const __m128i t2 = *q0; + *p1 = _mm_unpacklo_epi64(t1, *p0); + *p0 = _mm_unpackhi_epi64(t1, *p0); + *q0 = _mm_unpacklo_epi64(t2, *q1); + *q1 = _mm_unpackhi_epi64(t2, *q1); + } +} + +static WEBP_INLINE void Store4x4(__m128i* const x, uint8_t* dst, int stride) { + int i; + for (i = 0; i < 4; ++i, dst += stride) { + WebPUint32ToMem(dst, _mm_cvtsi128_si32(*x)); + *x = _mm_srli_si128(*x, 4); + } +} + +// Transpose back and store +static WEBP_INLINE void Store16x4(const __m128i* const p1, + const __m128i* const p0, + const __m128i* const q0, + const __m128i* const q1, + uint8_t* r0, uint8_t* r8, + int stride) { + __m128i t1, p1_s, p0_s, q0_s, q1_s; + + // p0 = 71 70 61 60 51 50 41 40 31 30 21 20 11 10 01 00 + // p1 = f1 f0 e1 e0 d1 d0 c1 c0 b1 b0 a1 a0 91 90 81 80 + t1 = *p0; + p0_s = _mm_unpacklo_epi8(*p1, t1); + p1_s = _mm_unpackhi_epi8(*p1, t1); + + // q0 = 73 72 63 62 53 52 43 42 33 32 23 22 13 12 03 02 + // q1 = f3 f2 e3 e2 d3 d2 c3 c2 b3 b2 a3 a2 93 92 83 82 + t1 = *q0; + q0_s = _mm_unpacklo_epi8(t1, *q1); + q1_s = _mm_unpackhi_epi8(t1, *q1); + + // p0 = 33 32 31 30 23 22 21 20 13 12 11 10 03 02 01 00 + // q0 = 73 72 71 70 63 62 61 60 53 52 51 50 43 42 41 40 + t1 = p0_s; + p0_s = _mm_unpacklo_epi16(t1, q0_s); + q0_s = _mm_unpackhi_epi16(t1, q0_s); + + // p1 = b3 b2 b1 b0 a3 a2 a1 a0 93 92 91 90 83 82 81 80 + // q1 = f3 f2 f1 f0 e3 e2 e1 e0 d3 d2 d1 d0 c3 c2 c1 c0 + t1 = p1_s; + p1_s = _mm_unpacklo_epi16(t1, q1_s); + q1_s = _mm_unpackhi_epi16(t1, q1_s); + + Store4x4(&p0_s, r0, stride); + r0 += 4 * stride; + Store4x4(&q0_s, r0, stride); + + Store4x4(&p1_s, r8, stride); + r8 += 4 * stride; + Store4x4(&q1_s, r8, stride); +} + +//------------------------------------------------------------------------------ +// Simple In-loop filtering (Paragraph 15.2) + +static void SimpleVFilter16(uint8_t* p, int stride, int thresh) { + // Load + __m128i p1 = _mm_loadu_si128((__m128i*)&p[-2 * stride]); + __m128i p0 = _mm_loadu_si128((__m128i*)&p[-stride]); + __m128i q0 = _mm_loadu_si128((__m128i*)&p[0]); + __m128i q1 = _mm_loadu_si128((__m128i*)&p[stride]); + + DoFilter2(&p1, &p0, &q0, &q1, thresh); + + // Store + _mm_storeu_si128((__m128i*)&p[-stride], p0); + _mm_storeu_si128((__m128i*)&p[0], q0); +} + +static void SimpleHFilter16(uint8_t* p, int stride, int thresh) { + __m128i p1, p0, q0, q1; + + p -= 2; // beginning of p1 + + Load16x4(p, p + 8 * stride, stride, &p1, &p0, &q0, &q1); + DoFilter2(&p1, &p0, &q0, &q1, thresh); + Store16x4(&p1, &p0, &q0, &q1, p, p + 8 * stride, stride); +} + +static void SimpleVFilter16i(uint8_t* p, int stride, int thresh) { + int k; + for (k = 3; k > 0; --k) { + p += 4 * stride; + SimpleVFilter16(p, stride, thresh); + } +} + +static void SimpleHFilter16i(uint8_t* p, int stride, int thresh) { + int k; + for (k = 3; k > 0; --k) { + p += 4; + SimpleHFilter16(p, stride, thresh); + } +} + +//------------------------------------------------------------------------------ +// Complex In-loop filtering (Paragraph 15.3) + +#define MAX_DIFF1(p3, p2, p1, p0, m) do { \ + m = MM_ABS(p1, p0); \ + m = _mm_max_epu8(m, MM_ABS(p3, p2)); \ + m = _mm_max_epu8(m, MM_ABS(p2, p1)); \ +} while (0) + +#define MAX_DIFF2(p3, p2, p1, p0, m) do { \ + m = _mm_max_epu8(m, MM_ABS(p1, p0)); \ + m = _mm_max_epu8(m, MM_ABS(p3, p2)); \ + m = _mm_max_epu8(m, MM_ABS(p2, p1)); \ +} while (0) + +#define LOAD_H_EDGES4(p, stride, e1, e2, e3, e4) { \ + e1 = _mm_loadu_si128((__m128i*)&(p)[0 * stride]); \ + e2 = _mm_loadu_si128((__m128i*)&(p)[1 * stride]); \ + e3 = _mm_loadu_si128((__m128i*)&(p)[2 * stride]); \ + e4 = _mm_loadu_si128((__m128i*)&(p)[3 * stride]); \ +} + +#define LOADUV_H_EDGE(p, u, v, stride) do { \ + const __m128i U = _mm_loadl_epi64((__m128i*)&(u)[(stride)]); \ + const __m128i V = _mm_loadl_epi64((__m128i*)&(v)[(stride)]); \ + p = _mm_unpacklo_epi64(U, V); \ +} while (0) + +#define LOADUV_H_EDGES4(u, v, stride, e1, e2, e3, e4) { \ + LOADUV_H_EDGE(e1, u, v, 0 * stride); \ + LOADUV_H_EDGE(e2, u, v, 1 * stride); \ + LOADUV_H_EDGE(e3, u, v, 2 * stride); \ + LOADUV_H_EDGE(e4, u, v, 3 * stride); \ +} + +#define STOREUV(p, u, v, stride) { \ + _mm_storel_epi64((__m128i*)&u[(stride)], p); \ + p = _mm_srli_si128(p, 8); \ + _mm_storel_epi64((__m128i*)&v[(stride)], p); \ +} + +static WEBP_INLINE void ComplexMask(const __m128i* const p1, + const __m128i* const p0, + const __m128i* const q0, + const __m128i* const q1, + int thresh, int ithresh, + __m128i* const mask) { + const __m128i it = _mm_set1_epi8(ithresh); + const __m128i diff = _mm_subs_epu8(*mask, it); + const __m128i thresh_mask = _mm_cmpeq_epi8(diff, _mm_setzero_si128()); + __m128i filter_mask; + NeedsFilter(p1, p0, q0, q1, thresh, &filter_mask); + *mask = _mm_and_si128(thresh_mask, filter_mask); +} + +// on macroblock edges +static void VFilter16(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + __m128i t1; + __m128i mask; + __m128i p2, p1, p0, q0, q1, q2; + + // Load p3, p2, p1, p0 + LOAD_H_EDGES4(p - 4 * stride, stride, t1, p2, p1, p0); + MAX_DIFF1(t1, p2, p1, p0, mask); + + // Load q0, q1, q2, q3 + LOAD_H_EDGES4(p, stride, q0, q1, q2, t1); + MAX_DIFF2(t1, q2, q1, q0, mask); + + ComplexMask(&p1, &p0, &q0, &q1, thresh, ithresh, &mask); + DoFilter6(&p2, &p1, &p0, &q0, &q1, &q2, &mask, hev_thresh); + + // Store + _mm_storeu_si128((__m128i*)&p[-3 * stride], p2); + _mm_storeu_si128((__m128i*)&p[-2 * stride], p1); + _mm_storeu_si128((__m128i*)&p[-1 * stride], p0); + _mm_storeu_si128((__m128i*)&p[+0 * stride], q0); + _mm_storeu_si128((__m128i*)&p[+1 * stride], q1); + _mm_storeu_si128((__m128i*)&p[+2 * stride], q2); +} + +static void HFilter16(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + __m128i mask; + __m128i p3, p2, p1, p0, q0, q1, q2, q3; + + uint8_t* const b = p - 4; + Load16x4(b, b + 8 * stride, stride, &p3, &p2, &p1, &p0); // p3, p2, p1, p0 + MAX_DIFF1(p3, p2, p1, p0, mask); + + Load16x4(p, p + 8 * stride, stride, &q0, &q1, &q2, &q3); // q0, q1, q2, q3 + MAX_DIFF2(q3, q2, q1, q0, mask); + + ComplexMask(&p1, &p0, &q0, &q1, thresh, ithresh, &mask); + DoFilter6(&p2, &p1, &p0, &q0, &q1, &q2, &mask, hev_thresh); + + Store16x4(&p3, &p2, &p1, &p0, b, b + 8 * stride, stride); + Store16x4(&q0, &q1, &q2, &q3, p, p + 8 * stride, stride); +} + +// on three inner edges +static void VFilter16i(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + int k; + __m128i p3, p2, p1, p0; // loop invariants + + LOAD_H_EDGES4(p, stride, p3, p2, p1, p0); // prologue + + for (k = 3; k > 0; --k) { + __m128i mask, tmp1, tmp2; + uint8_t* const b = p + 2 * stride; // beginning of p1 + p += 4 * stride; + + MAX_DIFF1(p3, p2, p1, p0, mask); // compute partial mask + LOAD_H_EDGES4(p, stride, p3, p2, tmp1, tmp2); + MAX_DIFF2(p3, p2, tmp1, tmp2, mask); + + // p3 and p2 are not just temporary variables here: they will be + // re-used for next span. And q2/q3 will become p1/p0 accordingly. + ComplexMask(&p1, &p0, &p3, &p2, thresh, ithresh, &mask); + DoFilter4(&p1, &p0, &p3, &p2, &mask, hev_thresh); + + // Store + _mm_storeu_si128((__m128i*)&b[0 * stride], p1); + _mm_storeu_si128((__m128i*)&b[1 * stride], p0); + _mm_storeu_si128((__m128i*)&b[2 * stride], p3); + _mm_storeu_si128((__m128i*)&b[3 * stride], p2); + + // rotate samples + p1 = tmp1; + p0 = tmp2; + } +} + +static void HFilter16i(uint8_t* p, int stride, + int thresh, int ithresh, int hev_thresh) { + int k; + __m128i p3, p2, p1, p0; // loop invariants + + Load16x4(p, p + 8 * stride, stride, &p3, &p2, &p1, &p0); // prologue + + for (k = 3; k > 0; --k) { + __m128i mask, tmp1, tmp2; + uint8_t* const b = p + 2; // beginning of p1 + + p += 4; // beginning of q0 (and next span) + + MAX_DIFF1(p3, p2, p1, p0, mask); // compute partial mask + Load16x4(p, p + 8 * stride, stride, &p3, &p2, &tmp1, &tmp2); + MAX_DIFF2(p3, p2, tmp1, tmp2, mask); + + ComplexMask(&p1, &p0, &p3, &p2, thresh, ithresh, &mask); + DoFilter4(&p1, &p0, &p3, &p2, &mask, hev_thresh); + + Store16x4(&p1, &p0, &p3, &p2, b, b + 8 * stride, stride); + + // rotate samples + p1 = tmp1; + p0 = tmp2; + } +} + +// 8-pixels wide variant, for chroma filtering +static void VFilter8(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + __m128i mask; + __m128i t1, p2, p1, p0, q0, q1, q2; + + // Load p3, p2, p1, p0 + LOADUV_H_EDGES4(u - 4 * stride, v - 4 * stride, stride, t1, p2, p1, p0); + MAX_DIFF1(t1, p2, p1, p0, mask); + + // Load q0, q1, q2, q3 + LOADUV_H_EDGES4(u, v, stride, q0, q1, q2, t1); + MAX_DIFF2(t1, q2, q1, q0, mask); + + ComplexMask(&p1, &p0, &q0, &q1, thresh, ithresh, &mask); + DoFilter6(&p2, &p1, &p0, &q0, &q1, &q2, &mask, hev_thresh); + + // Store + STOREUV(p2, u, v, -3 * stride); + STOREUV(p1, u, v, -2 * stride); + STOREUV(p0, u, v, -1 * stride); + STOREUV(q0, u, v, 0 * stride); + STOREUV(q1, u, v, 1 * stride); + STOREUV(q2, u, v, 2 * stride); +} + +static void HFilter8(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + __m128i mask; + __m128i p3, p2, p1, p0, q0, q1, q2, q3; + + uint8_t* const tu = u - 4; + uint8_t* const tv = v - 4; + Load16x4(tu, tv, stride, &p3, &p2, &p1, &p0); // p3, p2, p1, p0 + MAX_DIFF1(p3, p2, p1, p0, mask); + + Load16x4(u, v, stride, &q0, &q1, &q2, &q3); // q0, q1, q2, q3 + MAX_DIFF2(q3, q2, q1, q0, mask); + + ComplexMask(&p1, &p0, &q0, &q1, thresh, ithresh, &mask); + DoFilter6(&p2, &p1, &p0, &q0, &q1, &q2, &mask, hev_thresh); + + Store16x4(&p3, &p2, &p1, &p0, tu, tv, stride); + Store16x4(&q0, &q1, &q2, &q3, u, v, stride); +} + +static void VFilter8i(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + __m128i mask; + __m128i t1, t2, p1, p0, q0, q1; + + // Load p3, p2, p1, p0 + LOADUV_H_EDGES4(u, v, stride, t2, t1, p1, p0); + MAX_DIFF1(t2, t1, p1, p0, mask); + + u += 4 * stride; + v += 4 * stride; + + // Load q0, q1, q2, q3 + LOADUV_H_EDGES4(u, v, stride, q0, q1, t1, t2); + MAX_DIFF2(t2, t1, q1, q0, mask); + + ComplexMask(&p1, &p0, &q0, &q1, thresh, ithresh, &mask); + DoFilter4(&p1, &p0, &q0, &q1, &mask, hev_thresh); + + // Store + STOREUV(p1, u, v, -2 * stride); + STOREUV(p0, u, v, -1 * stride); + STOREUV(q0, u, v, 0 * stride); + STOREUV(q1, u, v, 1 * stride); +} + +static void HFilter8i(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_thresh) { + __m128i mask; + __m128i t1, t2, p1, p0, q0, q1; + Load16x4(u, v, stride, &t2, &t1, &p1, &p0); // p3, p2, p1, p0 + MAX_DIFF1(t2, t1, p1, p0, mask); + + u += 4; // beginning of q0 + v += 4; + Load16x4(u, v, stride, &q0, &q1, &t1, &t2); // q0, q1, q2, q3 + MAX_DIFF2(t2, t1, q1, q0, mask); + + ComplexMask(&p1, &p0, &q0, &q1, thresh, ithresh, &mask); + DoFilter4(&p1, &p0, &q0, &q1, &mask, hev_thresh); + + u -= 2; // beginning of p1 + v -= 2; + Store16x4(&p1, &p0, &q0, &q1, u, v, stride); +} + +//------------------------------------------------------------------------------ +// 4x4 predictions + +#define DST(x, y) dst[(x) + (y) * BPS] +#define AVG3(a, b, c) (((a) + 2 * (b) + (c) + 2) >> 2) + +// We use the following 8b-arithmetic tricks: +// (a + 2 * b + c + 2) >> 2 = (AC + b + 1) >> 1 +// where: AC = (a + c) >> 1 = [(a + c + 1) >> 1] - [(a^c) & 1] +// and: +// (a + 2 * b + c + 2) >> 2 = (AB + BC + 1) >> 1 - (ab|bc)&lsb +// where: AC = (a + b + 1) >> 1, BC = (b + c + 1) >> 1 +// and ab = a ^ b, bc = b ^ c, lsb = (AC^BC)&1 + +static void VE4(uint8_t* dst) { // vertical + const __m128i one = _mm_set1_epi8(1); + const __m128i ABCDEFGH = _mm_loadl_epi64((__m128i*)(dst - BPS - 1)); + const __m128i BCDEFGH0 = _mm_srli_si128(ABCDEFGH, 1); + const __m128i CDEFGH00 = _mm_srli_si128(ABCDEFGH, 2); + const __m128i a = _mm_avg_epu8(ABCDEFGH, CDEFGH00); + const __m128i lsb = _mm_and_si128(_mm_xor_si128(ABCDEFGH, CDEFGH00), one); + const __m128i b = _mm_subs_epu8(a, lsb); + const __m128i avg = _mm_avg_epu8(b, BCDEFGH0); + const uint32_t vals = _mm_cvtsi128_si32(avg); + int i; + for (i = 0; i < 4; ++i) { + WebPUint32ToMem(dst + i * BPS, vals); + } +} + +static void LD4(uint8_t* dst) { // Down-Left + const __m128i one = _mm_set1_epi8(1); + const __m128i ABCDEFGH = _mm_loadl_epi64((__m128i*)(dst - BPS)); + const __m128i BCDEFGH0 = _mm_srli_si128(ABCDEFGH, 1); + const __m128i CDEFGH00 = _mm_srli_si128(ABCDEFGH, 2); + const __m128i CDEFGHH0 = _mm_insert_epi16(CDEFGH00, dst[-BPS + 7], 3); + const __m128i avg1 = _mm_avg_epu8(ABCDEFGH, CDEFGHH0); + const __m128i lsb = _mm_and_si128(_mm_xor_si128(ABCDEFGH, CDEFGHH0), one); + const __m128i avg2 = _mm_subs_epu8(avg1, lsb); + const __m128i abcdefg = _mm_avg_epu8(avg2, BCDEFGH0); + WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcdefg )); + WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); + WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); + WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); +} + +static void VR4(uint8_t* dst) { // Vertical-Right + const __m128i one = _mm_set1_epi8(1); + const int I = dst[-1 + 0 * BPS]; + const int J = dst[-1 + 1 * BPS]; + const int K = dst[-1 + 2 * BPS]; + const int X = dst[-1 - BPS]; + const __m128i XABCD = _mm_loadl_epi64((__m128i*)(dst - BPS - 1)); + const __m128i ABCD0 = _mm_srli_si128(XABCD, 1); + const __m128i abcd = _mm_avg_epu8(XABCD, ABCD0); + const __m128i _XABCD = _mm_slli_si128(XABCD, 1); + const __m128i IXABCD = _mm_insert_epi16(_XABCD, I | (X << 8), 0); + const __m128i avg1 = _mm_avg_epu8(IXABCD, ABCD0); + const __m128i lsb = _mm_and_si128(_mm_xor_si128(IXABCD, ABCD0), one); + const __m128i avg2 = _mm_subs_epu8(avg1, lsb); + const __m128i efgh = _mm_avg_epu8(avg2, XABCD); + WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcd )); + WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( efgh )); + WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(abcd, 1))); + WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(efgh, 1))); + + // these two are hard to implement in SSE2, so we keep the C-version: + DST(0, 2) = AVG3(J, I, X); + DST(0, 3) = AVG3(K, J, I); +} + +static void VL4(uint8_t* dst) { // Vertical-Left + const __m128i one = _mm_set1_epi8(1); + const __m128i ABCDEFGH = _mm_loadl_epi64((__m128i*)(dst - BPS)); + const __m128i BCDEFGH_ = _mm_srli_si128(ABCDEFGH, 1); + const __m128i CDEFGH__ = _mm_srli_si128(ABCDEFGH, 2); + const __m128i avg1 = _mm_avg_epu8(ABCDEFGH, BCDEFGH_); + const __m128i avg2 = _mm_avg_epu8(CDEFGH__, BCDEFGH_); + const __m128i avg3 = _mm_avg_epu8(avg1, avg2); + const __m128i lsb1 = _mm_and_si128(_mm_xor_si128(avg1, avg2), one); + const __m128i ab = _mm_xor_si128(ABCDEFGH, BCDEFGH_); + const __m128i bc = _mm_xor_si128(CDEFGH__, BCDEFGH_); + const __m128i abbc = _mm_or_si128(ab, bc); + const __m128i lsb2 = _mm_and_si128(abbc, lsb1); + const __m128i avg4 = _mm_subs_epu8(avg3, lsb2); + const uint32_t extra_out = _mm_cvtsi128_si32(_mm_srli_si128(avg4, 4)); + WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( avg1 )); + WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( avg4 )); + WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg1, 1))); + WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg4, 1))); + + // these two are hard to get and irregular + DST(3, 2) = (extra_out >> 0) & 0xff; + DST(3, 3) = (extra_out >> 8) & 0xff; +} + +static void RD4(uint8_t* dst) { // Down-right + const __m128i one = _mm_set1_epi8(1); + const __m128i XABCD = _mm_loadl_epi64((__m128i*)(dst - BPS - 1)); + const __m128i ____XABCD = _mm_slli_si128(XABCD, 4); + const uint32_t I = dst[-1 + 0 * BPS]; + const uint32_t J = dst[-1 + 1 * BPS]; + const uint32_t K = dst[-1 + 2 * BPS]; + const uint32_t L = dst[-1 + 3 * BPS]; + const __m128i LKJI_____ = + _mm_cvtsi32_si128(L | (K << 8) | (J << 16) | (I << 24)); + const __m128i LKJIXABCD = _mm_or_si128(LKJI_____, ____XABCD); + const __m128i KJIXABCD_ = _mm_srli_si128(LKJIXABCD, 1); + const __m128i JIXABCD__ = _mm_srli_si128(LKJIXABCD, 2); + const __m128i avg1 = _mm_avg_epu8(JIXABCD__, LKJIXABCD); + const __m128i lsb = _mm_and_si128(_mm_xor_si128(JIXABCD__, LKJIXABCD), one); + const __m128i avg2 = _mm_subs_epu8(avg1, lsb); + const __m128i abcdefg = _mm_avg_epu8(avg2, KJIXABCD_); + WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32( abcdefg )); + WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); + WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); + WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); +} + +#undef DST +#undef AVG3 + +//------------------------------------------------------------------------------ +// Luma 16x16 + +static WEBP_INLINE void TrueMotion(uint8_t* dst, int size) { + const uint8_t* top = dst - BPS; + const __m128i zero = _mm_setzero_si128(); + int y; + if (size == 4) { + const __m128i top_values = _mm_cvtsi32_si128(WebPMemToUint32(top)); + const __m128i top_base = _mm_unpacklo_epi8(top_values, zero); + for (y = 0; y < 4; ++y, dst += BPS) { + const int val = dst[-1] - top[-1]; + const __m128i base = _mm_set1_epi16(val); + const __m128i out = _mm_packus_epi16(_mm_add_epi16(base, top_base), zero); + WebPUint32ToMem(dst, _mm_cvtsi128_si32(out)); + } + } else if (size == 8) { + const __m128i top_values = _mm_loadl_epi64((const __m128i*)top); + const __m128i top_base = _mm_unpacklo_epi8(top_values, zero); + for (y = 0; y < 8; ++y, dst += BPS) { + const int val = dst[-1] - top[-1]; + const __m128i base = _mm_set1_epi16(val); + const __m128i out = _mm_packus_epi16(_mm_add_epi16(base, top_base), zero); + _mm_storel_epi64((__m128i*)dst, out); + } + } else { + const __m128i top_values = _mm_loadu_si128((const __m128i*)top); + const __m128i top_base_0 = _mm_unpacklo_epi8(top_values, zero); + const __m128i top_base_1 = _mm_unpackhi_epi8(top_values, zero); + for (y = 0; y < 16; ++y, dst += BPS) { + const int val = dst[-1] - top[-1]; + const __m128i base = _mm_set1_epi16(val); + const __m128i out_0 = _mm_add_epi16(base, top_base_0); + const __m128i out_1 = _mm_add_epi16(base, top_base_1); + const __m128i out = _mm_packus_epi16(out_0, out_1); + _mm_storeu_si128((__m128i*)dst, out); + } + } +} + +static void TM4(uint8_t* dst) { TrueMotion(dst, 4); } +static void TM8uv(uint8_t* dst) { TrueMotion(dst, 8); } +static void TM16(uint8_t* dst) { TrueMotion(dst, 16); } + +static void VE16(uint8_t* dst) { + const __m128i top = _mm_loadu_si128((const __m128i*)(dst - BPS)); + int j; + for (j = 0; j < 16; ++j) { + _mm_storeu_si128((__m128i*)(dst + j * BPS), top); + } +} + +static void HE16(uint8_t* dst) { // horizontal + int j; + for (j = 16; j > 0; --j) { + const __m128i values = _mm_set1_epi8(dst[-1]); + _mm_storeu_si128((__m128i*)dst, values); + dst += BPS; + } +} + +static WEBP_INLINE void Put16(uint8_t v, uint8_t* dst) { + int j; + const __m128i values = _mm_set1_epi8(v); + for (j = 0; j < 16; ++j) { + _mm_storeu_si128((__m128i*)(dst + j * BPS), values); + } +} + +static void DC16(uint8_t* dst) { // DC + const __m128i zero = _mm_setzero_si128(); + const __m128i top = _mm_loadu_si128((const __m128i*)(dst - BPS)); + const __m128i sad8x2 = _mm_sad_epu8(top, zero); + // sum the two sads: sad8x2[0:1] + sad8x2[8:9] + const __m128i sum = _mm_add_epi16(sad8x2, _mm_shuffle_epi32(sad8x2, 2)); + int left = 0; + int j; + for (j = 0; j < 16; ++j) { + left += dst[-1 + j * BPS]; + } + { + const int DC = _mm_cvtsi128_si32(sum) + left + 16; + Put16(DC >> 5, dst); + } +} + +static void DC16NoTop(uint8_t* dst) { // DC with top samples not available + int DC = 8; + int j; + for (j = 0; j < 16; ++j) { + DC += dst[-1 + j * BPS]; + } + Put16(DC >> 4, dst); +} + +static void DC16NoLeft(uint8_t* dst) { // DC with left samples not available + const __m128i zero = _mm_setzero_si128(); + const __m128i top = _mm_loadu_si128((const __m128i*)(dst - BPS)); + const __m128i sad8x2 = _mm_sad_epu8(top, zero); + // sum the two sads: sad8x2[0:1] + sad8x2[8:9] + const __m128i sum = _mm_add_epi16(sad8x2, _mm_shuffle_epi32(sad8x2, 2)); + const int DC = _mm_cvtsi128_si32(sum) + 8; + Put16(DC >> 4, dst); +} + +static void DC16NoTopLeft(uint8_t* dst) { // DC with no top and left samples + Put16(0x80, dst); +} + +//------------------------------------------------------------------------------ +// Chroma + +static void VE8uv(uint8_t* dst) { // vertical + int j; + const __m128i top = _mm_loadl_epi64((const __m128i*)(dst - BPS)); + for (j = 0; j < 8; ++j) { + _mm_storel_epi64((__m128i*)(dst + j * BPS), top); + } +} + +static void HE8uv(uint8_t* dst) { // horizontal + int j; + for (j = 0; j < 8; ++j) { + const __m128i values = _mm_set1_epi8(dst[-1]); + _mm_storel_epi64((__m128i*)dst, values); + dst += BPS; + } +} + +// helper for chroma-DC predictions +static WEBP_INLINE void Put8x8uv(uint8_t v, uint8_t* dst) { + int j; + const __m128i values = _mm_set1_epi8(v); + for (j = 0; j < 8; ++j) { + _mm_storel_epi64((__m128i*)(dst + j * BPS), values); + } +} + +static void DC8uv(uint8_t* dst) { // DC + const __m128i zero = _mm_setzero_si128(); + const __m128i top = _mm_loadl_epi64((const __m128i*)(dst - BPS)); + const __m128i sum = _mm_sad_epu8(top, zero); + int left = 0; + int j; + for (j = 0; j < 8; ++j) { + left += dst[-1 + j * BPS]; + } + { + const int DC = _mm_cvtsi128_si32(sum) + left + 8; + Put8x8uv(DC >> 4, dst); + } +} + +static void DC8uvNoLeft(uint8_t* dst) { // DC with no left samples + const __m128i zero = _mm_setzero_si128(); + const __m128i top = _mm_loadl_epi64((const __m128i*)(dst - BPS)); + const __m128i sum = _mm_sad_epu8(top, zero); + const int DC = _mm_cvtsi128_si32(sum) + 4; + Put8x8uv(DC >> 3, dst); +} + +static void DC8uvNoTop(uint8_t* dst) { // DC with no top samples + int dc0 = 4; + int i; + for (i = 0; i < 8; ++i) { + dc0 += dst[-1 + i * BPS]; + } + Put8x8uv(dc0 >> 3, dst); +} + +static void DC8uvNoTopLeft(uint8_t* dst) { // DC with nothing + Put8x8uv(0x80, dst); +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8DspInitSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8DspInitSSE2(void) { + VP8Transform = Transform; +#if defined(USE_TRANSFORM_AC3) + VP8TransformAC3 = TransformAC3; +#endif + + VP8VFilter16 = VFilter16; + VP8HFilter16 = HFilter16; + VP8VFilter8 = VFilter8; + VP8HFilter8 = HFilter8; + VP8VFilter16i = VFilter16i; + VP8HFilter16i = HFilter16i; + VP8VFilter8i = VFilter8i; + VP8HFilter8i = HFilter8i; + + VP8SimpleVFilter16 = SimpleVFilter16; + VP8SimpleHFilter16 = SimpleHFilter16; + VP8SimpleVFilter16i = SimpleVFilter16i; + VP8SimpleHFilter16i = SimpleHFilter16i; + + VP8PredLuma4[1] = TM4; + VP8PredLuma4[2] = VE4; + VP8PredLuma4[4] = RD4; + VP8PredLuma4[5] = VR4; + VP8PredLuma4[6] = LD4; + VP8PredLuma4[7] = VL4; + + VP8PredLuma16[0] = DC16; + VP8PredLuma16[1] = TM16; + VP8PredLuma16[2] = VE16; + VP8PredLuma16[3] = HE16; + VP8PredLuma16[4] = DC16NoTop; + VP8PredLuma16[5] = DC16NoLeft; + VP8PredLuma16[6] = DC16NoTopLeft; + + VP8PredChroma8[0] = DC8uv; + VP8PredChroma8[1] = TM8uv; + VP8PredChroma8[2] = VE8uv; + VP8PredChroma8[3] = HE8uv; + VP8PredChroma8[4] = DC8uvNoTop; + VP8PredChroma8[5] = DC8uvNoLeft; + VP8PredChroma8[6] = DC8uvNoTopLeft; +} + +#else // !WEBP_USE_SSE2 + +WEBP_DSP_INIT_STUB(VP8DspInitSSE2) + +#endif // WEBP_USE_SSE2 diff --git a/thirdparty/libwebp/dsp/dec_sse41.c b/thirdparty/libwebp/dsp/dec_sse41.c new file mode 100644 index 0000000000..8d6aed13e6 --- /dev/null +++ b/thirdparty/libwebp/dsp/dec_sse41.c @@ -0,0 +1,46 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE4 version of some decoding functions. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE41) + +#include <smmintrin.h> +#include "../dec/vp8i.h" +#include "../utils/utils.h" + +static void HE16(uint8_t* dst) { // horizontal + int j; + const __m128i kShuffle3 = _mm_set1_epi8(3); + for (j = 16; j > 0; --j) { + const __m128i in = _mm_cvtsi32_si128(WebPMemToUint32(dst - 4)); + const __m128i values = _mm_shuffle_epi8(in, kShuffle3); + _mm_storeu_si128((__m128i*)dst, values); + dst += BPS; + } +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8DspInitSSE41(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8DspInitSSE41(void) { + VP8PredLuma16[3] = HE16; +} + +#else // !WEBP_USE_SSE41 + +WEBP_DSP_INIT_STUB(VP8DspInitSSE41) + +#endif // WEBP_USE_SSE41 diff --git a/thirdparty/libwebp/dsp/dsp.h b/thirdparty/libwebp/dsp/dsp.h new file mode 100644 index 0000000000..1faac27b2b --- /dev/null +++ b/thirdparty/libwebp/dsp/dsp.h @@ -0,0 +1,569 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Speed-critical functions. +// +// Author: Skal (pascal.massimino@gmail.com) + +#ifndef WEBP_DSP_DSP_H_ +#define WEBP_DSP_DSP_H_ + +#ifdef HAVE_CONFIG_H +#include "../webp/config.h" +#endif + +#include "../webp/types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define BPS 32 // this is the common stride for enc/dec + +//------------------------------------------------------------------------------ +// CPU detection + +#if defined(__GNUC__) +# define LOCAL_GCC_VERSION ((__GNUC__ << 8) | __GNUC_MINOR__) +# define LOCAL_GCC_PREREQ(maj, min) \ + (LOCAL_GCC_VERSION >= (((maj) << 8) | (min))) +#else +# define LOCAL_GCC_VERSION 0 +# define LOCAL_GCC_PREREQ(maj, min) 0 +#endif + +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +#if defined(_MSC_VER) && _MSC_VER > 1310 && \ + (defined(_M_X64) || defined(_M_IX86)) +#define WEBP_MSC_SSE2 // Visual C++ SSE2 targets +#endif + +#if defined(_MSC_VER) && _MSC_VER >= 1500 && \ + (defined(_M_X64) || defined(_M_IX86)) +#define WEBP_MSC_SSE41 // Visual C++ SSE4.1 targets +#endif + +// WEBP_HAVE_* are used to indicate the presence of the instruction set in dsp +// files without intrinsics, allowing the corresponding Init() to be called. +// Files containing intrinsics will need to be built targeting the instruction +// set so should succeed on one of the earlier tests. +#if defined(__SSE2__) || defined(WEBP_MSC_SSE2) || defined(WEBP_HAVE_SSE2) +#define WEBP_USE_SSE2 +#endif + +#if defined(__SSE4_1__) || defined(WEBP_MSC_SSE41) || defined(WEBP_HAVE_SSE41) +#define WEBP_USE_SSE41 +#endif + +#if defined(__AVX2__) || defined(WEBP_HAVE_AVX2) +#define WEBP_USE_AVX2 +#endif + +#if defined(__ANDROID__) && defined(__ARM_ARCH_7A__) +#define WEBP_ANDROID_NEON // Android targets that might support NEON +#endif + +// The intrinsics currently cause compiler errors with arm-nacl-gcc and the +// inline assembly would need to be modified for use with Native Client. +#if (defined(__ARM_NEON__) || defined(WEBP_ANDROID_NEON) || \ + defined(__aarch64__) || defined(WEBP_HAVE_NEON)) && \ + !defined(__native_client__) +#define WEBP_USE_NEON +#endif + +#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_M_ARM) +#define WEBP_USE_NEON +#define WEBP_USE_INTRINSICS +#endif + +#if defined(__mips__) && !defined(__mips64) && \ + defined(__mips_isa_rev) && (__mips_isa_rev >= 1) && (__mips_isa_rev < 6) +#define WEBP_USE_MIPS32 +#if (__mips_isa_rev >= 2) +#define WEBP_USE_MIPS32_R2 +#if defined(__mips_dspr2) || (__mips_dsp_rev >= 2) +#define WEBP_USE_MIPS_DSP_R2 +#endif +#endif +#endif + +#if defined(__mips_msa) && defined(__mips_isa_rev) && (__mips_isa_rev >= 5) +#define WEBP_USE_MSA +#endif + +// This macro prevents thread_sanitizer from reporting known concurrent writes. +#define WEBP_TSAN_IGNORE_FUNCTION +#if defined(__has_feature) +#if __has_feature(thread_sanitizer) +#undef WEBP_TSAN_IGNORE_FUNCTION +#define WEBP_TSAN_IGNORE_FUNCTION __attribute__((no_sanitize_thread)) +#endif +#endif + +#define WEBP_UBSAN_IGNORE_UNDEF +#define WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW +#if !defined(WEBP_FORCE_ALIGNED) && defined(__clang__) && \ + defined(__has_attribute) +#if __has_attribute(no_sanitize) +// This macro prevents the undefined behavior sanitizer from reporting +// failures. This is only meant to silence unaligned loads on platforms that +// are known to support them. +#undef WEBP_UBSAN_IGNORE_UNDEF +#define WEBP_UBSAN_IGNORE_UNDEF \ + __attribute__((no_sanitize("undefined"))) + +// This macro prevents the undefined behavior sanitizer from reporting +// failures related to unsigned integer overflows. This is only meant to +// silence cases where this well defined behavior is expected. +#undef WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW +#define WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW \ + __attribute__((no_sanitize("unsigned-integer-overflow"))) +#endif +#endif + +typedef enum { + kSSE2, + kSSE3, + kSSE4_1, + kAVX, + kAVX2, + kNEON, + kMIPS32, + kMIPSdspR2, + kMSA +} CPUFeature; +// returns true if the CPU supports the feature. +typedef int (*VP8CPUInfo)(CPUFeature feature); +WEBP_EXTERN(VP8CPUInfo) VP8GetCPUInfo; + +//------------------------------------------------------------------------------ +// Init stub generator + +// Defines an init function stub to ensure each module exposes a symbol, +// avoiding a compiler warning. +#define WEBP_DSP_INIT_STUB(func) \ + extern void func(void); \ + WEBP_TSAN_IGNORE_FUNCTION void func(void) {} + +//------------------------------------------------------------------------------ +// Encoding + +// Transforms +// VP8Idct: Does one of two inverse transforms. If do_two is set, the transforms +// will be done for (ref, in, dst) and (ref + 4, in + 16, dst + 4). +typedef void (*VP8Idct)(const uint8_t* ref, const int16_t* in, uint8_t* dst, + int do_two); +typedef void (*VP8Fdct)(const uint8_t* src, const uint8_t* ref, int16_t* out); +typedef void (*VP8WHT)(const int16_t* in, int16_t* out); +extern VP8Idct VP8ITransform; +extern VP8Fdct VP8FTransform; +extern VP8Fdct VP8FTransform2; // performs two transforms at a time +extern VP8WHT VP8FTransformWHT; +// Predictions +// *dst is the destination block. *top and *left can be NULL. +typedef void (*VP8IntraPreds)(uint8_t *dst, const uint8_t* left, + const uint8_t* top); +typedef void (*VP8Intra4Preds)(uint8_t *dst, const uint8_t* top); +extern VP8Intra4Preds VP8EncPredLuma4; +extern VP8IntraPreds VP8EncPredLuma16; +extern VP8IntraPreds VP8EncPredChroma8; + +typedef int (*VP8Metric)(const uint8_t* pix, const uint8_t* ref); +extern VP8Metric VP8SSE16x16, VP8SSE16x8, VP8SSE8x8, VP8SSE4x4; +typedef int (*VP8WMetric)(const uint8_t* pix, const uint8_t* ref, + const uint16_t* const weights); +// The weights for VP8TDisto4x4 and VP8TDisto16x16 contain a row-major +// 4 by 4 symmetric matrix. +extern VP8WMetric VP8TDisto4x4, VP8TDisto16x16; + +typedef void (*VP8BlockCopy)(const uint8_t* src, uint8_t* dst); +extern VP8BlockCopy VP8Copy4x4; +extern VP8BlockCopy VP8Copy16x8; +// Quantization +struct VP8Matrix; // forward declaration +typedef int (*VP8QuantizeBlock)(int16_t in[16], int16_t out[16], + const struct VP8Matrix* const mtx); +// Same as VP8QuantizeBlock, but quantizes two consecutive blocks. +typedef int (*VP8Quantize2Blocks)(int16_t in[32], int16_t out[32], + const struct VP8Matrix* const mtx); + +extern VP8QuantizeBlock VP8EncQuantizeBlock; +extern VP8Quantize2Blocks VP8EncQuantize2Blocks; + +// specific to 2nd transform: +typedef int (*VP8QuantizeBlockWHT)(int16_t in[16], int16_t out[16], + const struct VP8Matrix* const mtx); +extern VP8QuantizeBlockWHT VP8EncQuantizeBlockWHT; + +extern const int VP8DspScan[16 + 4 + 4]; + +// Collect histogram for susceptibility calculation. +#define MAX_COEFF_THRESH 31 // size of histogram used by CollectHistogram. +typedef struct { + // We only need to store max_value and last_non_zero, not the distribution. + int max_value; + int last_non_zero; +} VP8Histogram; +typedef void (*VP8CHisto)(const uint8_t* ref, const uint8_t* pred, + int start_block, int end_block, + VP8Histogram* const histo); +extern VP8CHisto VP8CollectHistogram; +// General-purpose util function to help VP8CollectHistogram(). +void VP8SetHistogramData(const int distribution[MAX_COEFF_THRESH + 1], + VP8Histogram* const histo); + +// must be called before using any of the above +void VP8EncDspInit(void); + +//------------------------------------------------------------------------------ +// cost functions (encoding) + +extern const uint16_t VP8EntropyCost[256]; // 8bit fixed-point log(p) +// approximate cost per level: +extern const uint16_t VP8LevelFixedCosts[2047 /*MAX_LEVEL*/ + 1]; +extern const uint8_t VP8EncBands[16 + 1]; + +struct VP8Residual; +typedef void (*VP8SetResidualCoeffsFunc)(const int16_t* const coeffs, + struct VP8Residual* const res); +extern VP8SetResidualCoeffsFunc VP8SetResidualCoeffs; + +// Cost calculation function. +typedef int (*VP8GetResidualCostFunc)(int ctx0, + const struct VP8Residual* const res); +extern VP8GetResidualCostFunc VP8GetResidualCost; + +// must be called before anything using the above +void VP8EncDspCostInit(void); + +//------------------------------------------------------------------------------ +// SSIM utils + +// struct for accumulating statistical moments +typedef struct { + double w; // sum(w_i) : sum of weights + double xm, ym; // sum(w_i * x_i), sum(w_i * y_i) + double xxm, xym, yym; // sum(w_i * x_i * x_i), etc. +} VP8DistoStats; + +#define VP8_SSIM_KERNEL 3 // total size of the kernel: 2 * VP8_SSIM_KERNEL + 1 +typedef void (*VP8SSIMAccumulateClippedFunc)(const uint8_t* src1, int stride1, + const uint8_t* src2, int stride2, + int xo, int yo, // center position + int W, int H, // plane dimension + VP8DistoStats* const stats); + +// This version is called with the guarantee that you can load 8 bytes and +// 8 rows at offset src1 and src2 +typedef void (*VP8SSIMAccumulateFunc)(const uint8_t* src1, int stride1, + const uint8_t* src2, int stride2, + VP8DistoStats* const stats); + +extern VP8SSIMAccumulateFunc VP8SSIMAccumulate; // unclipped / unchecked +extern VP8SSIMAccumulateClippedFunc VP8SSIMAccumulateClipped; // with clipping + +// must be called before using any of the above directly +void VP8SSIMDspInit(void); + +//------------------------------------------------------------------------------ +// Decoding + +typedef void (*VP8DecIdct)(const int16_t* coeffs, uint8_t* dst); +// when doing two transforms, coeffs is actually int16_t[2][16]. +typedef void (*VP8DecIdct2)(const int16_t* coeffs, uint8_t* dst, int do_two); +extern VP8DecIdct2 VP8Transform; +extern VP8DecIdct VP8TransformAC3; +extern VP8DecIdct VP8TransformUV; +extern VP8DecIdct VP8TransformDC; +extern VP8DecIdct VP8TransformDCUV; +extern VP8WHT VP8TransformWHT; + +// *dst is the destination block, with stride BPS. Boundary samples are +// assumed accessible when needed. +typedef void (*VP8PredFunc)(uint8_t* dst); +extern VP8PredFunc VP8PredLuma16[/* NUM_B_DC_MODES */]; +extern VP8PredFunc VP8PredChroma8[/* NUM_B_DC_MODES */]; +extern VP8PredFunc VP8PredLuma4[/* NUM_BMODES */]; + +// clipping tables (for filtering) +extern const int8_t* const VP8ksclip1; // clips [-1020, 1020] to [-128, 127] +extern const int8_t* const VP8ksclip2; // clips [-112, 112] to [-16, 15] +extern const uint8_t* const VP8kclip1; // clips [-255,511] to [0,255] +extern const uint8_t* const VP8kabs0; // abs(x) for x in [-255,255] +// must be called first +void VP8InitClipTables(void); + +// simple filter (only for luma) +typedef void (*VP8SimpleFilterFunc)(uint8_t* p, int stride, int thresh); +extern VP8SimpleFilterFunc VP8SimpleVFilter16; +extern VP8SimpleFilterFunc VP8SimpleHFilter16; +extern VP8SimpleFilterFunc VP8SimpleVFilter16i; // filter 3 inner edges +extern VP8SimpleFilterFunc VP8SimpleHFilter16i; + +// regular filter (on both macroblock edges and inner edges) +typedef void (*VP8LumaFilterFunc)(uint8_t* luma, int stride, + int thresh, int ithresh, int hev_t); +typedef void (*VP8ChromaFilterFunc)(uint8_t* u, uint8_t* v, int stride, + int thresh, int ithresh, int hev_t); +// on outer edge +extern VP8LumaFilterFunc VP8VFilter16; +extern VP8LumaFilterFunc VP8HFilter16; +extern VP8ChromaFilterFunc VP8VFilter8; +extern VP8ChromaFilterFunc VP8HFilter8; + +// on inner edge +extern VP8LumaFilterFunc VP8VFilter16i; // filtering 3 inner edges altogether +extern VP8LumaFilterFunc VP8HFilter16i; +extern VP8ChromaFilterFunc VP8VFilter8i; // filtering u and v altogether +extern VP8ChromaFilterFunc VP8HFilter8i; + +// Dithering. Combines dithering values (centered around 128) with dst[], +// according to: dst[] = clip(dst[] + (((dither[]-128) + 8) >> 4) +#define VP8_DITHER_DESCALE 4 +#define VP8_DITHER_DESCALE_ROUNDER (1 << (VP8_DITHER_DESCALE - 1)) +#define VP8_DITHER_AMP_BITS 7 +#define VP8_DITHER_AMP_CENTER (1 << VP8_DITHER_AMP_BITS) +extern void (*VP8DitherCombine8x8)(const uint8_t* dither, uint8_t* dst, + int dst_stride); + +// must be called before anything using the above +void VP8DspInit(void); + +//------------------------------------------------------------------------------ +// WebP I/O + +#define FANCY_UPSAMPLING // undefined to remove fancy upsampling support + +// Convert a pair of y/u/v lines together to the output rgb/a colorspace. +// bottom_y can be NULL if only one line of output is needed (at top/bottom). +typedef void (*WebPUpsampleLinePairFunc)( + const uint8_t* top_y, const uint8_t* bottom_y, + const uint8_t* top_u, const uint8_t* top_v, + const uint8_t* cur_u, const uint8_t* cur_v, + uint8_t* top_dst, uint8_t* bottom_dst, int len); + +#ifdef FANCY_UPSAMPLING + +// Fancy upsampling functions to convert YUV to RGB(A) modes +extern WebPUpsampleLinePairFunc WebPUpsamplers[/* MODE_LAST */]; + +#endif // FANCY_UPSAMPLING + +// Per-row point-sampling methods. +typedef void (*WebPSamplerRowFunc)(const uint8_t* y, + const uint8_t* u, const uint8_t* v, + uint8_t* dst, int len); +// Generic function to apply 'WebPSamplerRowFunc' to the whole plane: +void WebPSamplerProcessPlane(const uint8_t* y, int y_stride, + const uint8_t* u, const uint8_t* v, int uv_stride, + uint8_t* dst, int dst_stride, + int width, int height, WebPSamplerRowFunc func); + +// Sampling functions to convert rows of YUV to RGB(A) +extern WebPSamplerRowFunc WebPSamplers[/* MODE_LAST */]; + +// General function for converting two lines of ARGB or RGBA. +// 'alpha_is_last' should be true if 0xff000000 is stored in memory as +// as 0x00, 0x00, 0x00, 0xff (little endian). +WebPUpsampleLinePairFunc WebPGetLinePairConverter(int alpha_is_last); + +// YUV444->RGB converters +typedef void (*WebPYUV444Converter)(const uint8_t* y, + const uint8_t* u, const uint8_t* v, + uint8_t* dst, int len); + +extern WebPYUV444Converter WebPYUV444Converters[/* MODE_LAST */]; + +// Must be called before using the WebPUpsamplers[] (and for premultiplied +// colorspaces like rgbA, rgbA4444, etc) +void WebPInitUpsamplers(void); +// Must be called before using WebPSamplers[] +void WebPInitSamplers(void); +// Must be called before using WebPYUV444Converters[] +void WebPInitYUV444Converters(void); + +//------------------------------------------------------------------------------ +// ARGB -> YUV converters + +// Convert ARGB samples to luma Y. +extern void (*WebPConvertARGBToY)(const uint32_t* argb, uint8_t* y, int width); +// Convert ARGB samples to U/V with downsampling. do_store should be '1' for +// even lines and '0' for odd ones. 'src_width' is the original width, not +// the U/V one. +extern void (*WebPConvertARGBToUV)(const uint32_t* argb, uint8_t* u, uint8_t* v, + int src_width, int do_store); + +// Convert a row of accumulated (four-values) of rgba32 toward U/V +extern void (*WebPConvertRGBA32ToUV)(const uint16_t* rgb, + uint8_t* u, uint8_t* v, int width); + +// Convert RGB or BGR to Y +extern void (*WebPConvertRGB24ToY)(const uint8_t* rgb, uint8_t* y, int width); +extern void (*WebPConvertBGR24ToY)(const uint8_t* bgr, uint8_t* y, int width); + +// used for plain-C fallback. +extern void WebPConvertARGBToUV_C(const uint32_t* argb, uint8_t* u, uint8_t* v, + int src_width, int do_store); +extern void WebPConvertRGBA32ToUV_C(const uint16_t* rgb, + uint8_t* u, uint8_t* v, int width); + +// Must be called before using the above. +void WebPInitConvertARGBToYUV(void); + +//------------------------------------------------------------------------------ +// Rescaler + +struct WebPRescaler; + +// Import a row of data and save its contribution in the rescaler. +// 'channel' denotes the channel number to be imported. 'Expand' corresponds to +// the wrk->x_expand case. Otherwise, 'Shrink' is to be used. +typedef void (*WebPRescalerImportRowFunc)(struct WebPRescaler* const wrk, + const uint8_t* src); + +extern WebPRescalerImportRowFunc WebPRescalerImportRowExpand; +extern WebPRescalerImportRowFunc WebPRescalerImportRowShrink; + +// Export one row (starting at x_out position) from rescaler. +// 'Expand' corresponds to the wrk->y_expand case. +// Otherwise 'Shrink' is to be used +typedef void (*WebPRescalerExportRowFunc)(struct WebPRescaler* const wrk); +extern WebPRescalerExportRowFunc WebPRescalerExportRowExpand; +extern WebPRescalerExportRowFunc WebPRescalerExportRowShrink; + +// Plain-C implementation, as fall-back. +extern void WebPRescalerImportRowExpandC(struct WebPRescaler* const wrk, + const uint8_t* src); +extern void WebPRescalerImportRowShrinkC(struct WebPRescaler* const wrk, + const uint8_t* src); +extern void WebPRescalerExportRowExpandC(struct WebPRescaler* const wrk); +extern void WebPRescalerExportRowShrinkC(struct WebPRescaler* const wrk); + +// Main entry calls: +extern void WebPRescalerImportRow(struct WebPRescaler* const wrk, + const uint8_t* src); +// Export one row (starting at x_out position) from rescaler. +extern void WebPRescalerExportRow(struct WebPRescaler* const wrk); + +// Must be called first before using the above. +void WebPRescalerDspInit(void); + +//------------------------------------------------------------------------------ +// Utilities for processing transparent channel. + +// Apply alpha pre-multiply on an rgba, bgra or argb plane of size w * h. +// alpha_first should be 0 for argb, 1 for rgba or bgra (where alpha is last). +extern void (*WebPApplyAlphaMultiply)( + uint8_t* rgba, int alpha_first, int w, int h, int stride); + +// Same, buf specifically for RGBA4444 format +extern void (*WebPApplyAlphaMultiply4444)( + uint8_t* rgba4444, int w, int h, int stride); + +// Dispatch the values from alpha[] plane to the ARGB destination 'dst'. +// Returns true if alpha[] plane has non-trivial values different from 0xff. +extern int (*WebPDispatchAlpha)(const uint8_t* alpha, int alpha_stride, + int width, int height, + uint8_t* dst, int dst_stride); + +// Transfer packed 8b alpha[] values to green channel in dst[], zero'ing the +// A/R/B values. 'dst_stride' is the stride for dst[] in uint32_t units. +extern void (*WebPDispatchAlphaToGreen)(const uint8_t* alpha, int alpha_stride, + int width, int height, + uint32_t* dst, int dst_stride); + +// Extract the alpha values from 32b values in argb[] and pack them into alpha[] +// (this is the opposite of WebPDispatchAlpha). +// Returns true if there's only trivial 0xff alpha values. +extern int (*WebPExtractAlpha)(const uint8_t* argb, int argb_stride, + int width, int height, + uint8_t* alpha, int alpha_stride); + +// Pre-Multiply operation transforms x into x * A / 255 (where x=Y,R,G or B). +// Un-Multiply operation transforms x into x * 255 / A. + +// Pre-Multiply or Un-Multiply (if 'inverse' is true) argb values in a row. +extern void (*WebPMultARGBRow)(uint32_t* const ptr, int width, int inverse); + +// Same a WebPMultARGBRow(), but for several rows. +void WebPMultARGBRows(uint8_t* ptr, int stride, int width, int num_rows, + int inverse); + +// Same for a row of single values, with side alpha values. +extern void (*WebPMultRow)(uint8_t* const ptr, const uint8_t* const alpha, + int width, int inverse); + +// Same a WebPMultRow(), but for several 'num_rows' rows. +void WebPMultRows(uint8_t* ptr, int stride, + const uint8_t* alpha, int alpha_stride, + int width, int num_rows, int inverse); + +// Plain-C versions, used as fallback by some implementations. +void WebPMultRowC(uint8_t* const ptr, const uint8_t* const alpha, + int width, int inverse); +void WebPMultARGBRowC(uint32_t* const ptr, int width, int inverse); + +// To be called first before using the above. +void WebPInitAlphaProcessing(void); + +// ARGB packing function: a/r/g/b input is rgba or bgra order. +extern void (*VP8PackARGB)(const uint8_t* a, const uint8_t* r, + const uint8_t* g, const uint8_t* b, int len, + uint32_t* out); + +// RGB packing function. 'step' can be 3 or 4. r/g/b input is rgb or bgr order. +extern void (*VP8PackRGB)(const uint8_t* r, const uint8_t* g, const uint8_t* b, + int len, int step, uint32_t* out); + +// To be called first before using the above. +void VP8EncDspARGBInit(void); + +//------------------------------------------------------------------------------ +// Filter functions + +typedef enum { // Filter types. + WEBP_FILTER_NONE = 0, + WEBP_FILTER_HORIZONTAL, + WEBP_FILTER_VERTICAL, + WEBP_FILTER_GRADIENT, + WEBP_FILTER_LAST = WEBP_FILTER_GRADIENT + 1, // end marker + WEBP_FILTER_BEST, // meta-types + WEBP_FILTER_FAST +} WEBP_FILTER_TYPE; + +typedef void (*WebPFilterFunc)(const uint8_t* in, int width, int height, + int stride, uint8_t* out); +// In-place un-filtering. +// Warning! 'prev_line' pointer can be equal to 'cur_line' or 'preds'. +typedef void (*WebPUnfilterFunc)(const uint8_t* prev_line, const uint8_t* preds, + uint8_t* cur_line, int width); + +// Filter the given data using the given predictor. +// 'in' corresponds to a 2-dimensional pixel array of size (stride * height) +// in raster order. +// 'stride' is number of bytes per scan line (with possible padding). +// 'out' should be pre-allocated. +extern WebPFilterFunc WebPFilters[WEBP_FILTER_LAST]; + +// In-place reconstruct the original data from the given filtered data. +// The reconstruction will be done for 'num_rows' rows starting from 'row' +// (assuming rows upto 'row - 1' are already reconstructed). +extern WebPUnfilterFunc WebPUnfilters[WEBP_FILTER_LAST]; + +// To be called first before using the above. +void VP8FiltersInit(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* WEBP_DSP_DSP_H_ */ diff --git a/thirdparty/libwebp/dsp/enc.c b/thirdparty/libwebp/dsp/enc.c new file mode 100644 index 0000000000..f639f5570c --- /dev/null +++ b/thirdparty/libwebp/dsp/enc.c @@ -0,0 +1,863 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Speed-critical encoding functions. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include <assert.h> +#include <stdlib.h> // for abs() + +#include "./dsp.h" +#include "../enc/vp8enci.h" + +static WEBP_INLINE uint8_t clip_8b(int v) { + return (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255; +} + +static WEBP_INLINE int clip_max(int v, int max) { + return (v > max) ? max : v; +} + +//------------------------------------------------------------------------------ +// Compute susceptibility based on DCT-coeff histograms: +// the higher, the "easier" the macroblock is to compress. + +const int VP8DspScan[16 + 4 + 4] = { + // Luma + 0 + 0 * BPS, 4 + 0 * BPS, 8 + 0 * BPS, 12 + 0 * BPS, + 0 + 4 * BPS, 4 + 4 * BPS, 8 + 4 * BPS, 12 + 4 * BPS, + 0 + 8 * BPS, 4 + 8 * BPS, 8 + 8 * BPS, 12 + 8 * BPS, + 0 + 12 * BPS, 4 + 12 * BPS, 8 + 12 * BPS, 12 + 12 * BPS, + + 0 + 0 * BPS, 4 + 0 * BPS, 0 + 4 * BPS, 4 + 4 * BPS, // U + 8 + 0 * BPS, 12 + 0 * BPS, 8 + 4 * BPS, 12 + 4 * BPS // V +}; + +// general-purpose util function +void VP8SetHistogramData(const int distribution[MAX_COEFF_THRESH + 1], + VP8Histogram* const histo) { + int max_value = 0, last_non_zero = 1; + int k; + for (k = 0; k <= MAX_COEFF_THRESH; ++k) { + const int value = distribution[k]; + if (value > 0) { + if (value > max_value) max_value = value; + last_non_zero = k; + } + } + histo->max_value = max_value; + histo->last_non_zero = last_non_zero; +} + +static void CollectHistogram(const uint8_t* ref, const uint8_t* pred, + int start_block, int end_block, + VP8Histogram* const histo) { + int j; + int distribution[MAX_COEFF_THRESH + 1] = { 0 }; + for (j = start_block; j < end_block; ++j) { + int k; + int16_t out[16]; + + VP8FTransform(ref + VP8DspScan[j], pred + VP8DspScan[j], out); + + // Convert coefficients to bin. + for (k = 0; k < 16; ++k) { + const int v = abs(out[k]) >> 3; + const int clipped_value = clip_max(v, MAX_COEFF_THRESH); + ++distribution[clipped_value]; + } + } + VP8SetHistogramData(distribution, histo); +} + +//------------------------------------------------------------------------------ +// run-time tables (~4k) + +static uint8_t clip1[255 + 510 + 1]; // clips [-255,510] to [0,255] + +// We declare this variable 'volatile' to prevent instruction reordering +// and make sure it's set to true _last_ (so as to be thread-safe) +static volatile int tables_ok = 0; + +static WEBP_TSAN_IGNORE_FUNCTION void InitTables(void) { + if (!tables_ok) { + int i; + for (i = -255; i <= 255 + 255; ++i) { + clip1[255 + i] = clip_8b(i); + } + tables_ok = 1; + } +} + + +//------------------------------------------------------------------------------ +// Transforms (Paragraph 14.4) + +#define STORE(x, y, v) \ + dst[(x) + (y) * BPS] = clip_8b(ref[(x) + (y) * BPS] + ((v) >> 3)) + +static const int kC1 = 20091 + (1 << 16); +static const int kC2 = 35468; +#define MUL(a, b) (((a) * (b)) >> 16) + +static WEBP_INLINE void ITransformOne(const uint8_t* ref, const int16_t* in, + uint8_t* dst) { + int C[4 * 4], *tmp; + int i; + tmp = C; + for (i = 0; i < 4; ++i) { // vertical pass + const int a = in[0] + in[8]; + const int b = in[0] - in[8]; + const int c = MUL(in[4], kC2) - MUL(in[12], kC1); + const int d = MUL(in[4], kC1) + MUL(in[12], kC2); + tmp[0] = a + d; + tmp[1] = b + c; + tmp[2] = b - c; + tmp[3] = a - d; + tmp += 4; + in++; + } + + tmp = C; + for (i = 0; i < 4; ++i) { // horizontal pass + const int dc = tmp[0] + 4; + const int a = dc + tmp[8]; + const int b = dc - tmp[8]; + const int c = MUL(tmp[4], kC2) - MUL(tmp[12], kC1); + const int d = MUL(tmp[4], kC1) + MUL(tmp[12], kC2); + STORE(0, i, a + d); + STORE(1, i, b + c); + STORE(2, i, b - c); + STORE(3, i, a - d); + tmp++; + } +} + +static void ITransform(const uint8_t* ref, const int16_t* in, uint8_t* dst, + int do_two) { + ITransformOne(ref, in, dst); + if (do_two) { + ITransformOne(ref + 4, in + 16, dst + 4); + } +} + +static void FTransform(const uint8_t* src, const uint8_t* ref, int16_t* out) { + int i; + int tmp[16]; + for (i = 0; i < 4; ++i, src += BPS, ref += BPS) { + const int d0 = src[0] - ref[0]; // 9bit dynamic range ([-255,255]) + const int d1 = src[1] - ref[1]; + const int d2 = src[2] - ref[2]; + const int d3 = src[3] - ref[3]; + const int a0 = (d0 + d3); // 10b [-510,510] + const int a1 = (d1 + d2); + const int a2 = (d1 - d2); + const int a3 = (d0 - d3); + tmp[0 + i * 4] = (a0 + a1) * 8; // 14b [-8160,8160] + tmp[1 + i * 4] = (a2 * 2217 + a3 * 5352 + 1812) >> 9; // [-7536,7542] + tmp[2 + i * 4] = (a0 - a1) * 8; + tmp[3 + i * 4] = (a3 * 2217 - a2 * 5352 + 937) >> 9; + } + for (i = 0; i < 4; ++i) { + const int a0 = (tmp[0 + i] + tmp[12 + i]); // 15b + const int a1 = (tmp[4 + i] + tmp[ 8 + i]); + const int a2 = (tmp[4 + i] - tmp[ 8 + i]); + const int a3 = (tmp[0 + i] - tmp[12 + i]); + out[0 + i] = (a0 + a1 + 7) >> 4; // 12b + out[4 + i] = ((a2 * 2217 + a3 * 5352 + 12000) >> 16) + (a3 != 0); + out[8 + i] = (a0 - a1 + 7) >> 4; + out[12+ i] = ((a3 * 2217 - a2 * 5352 + 51000) >> 16); + } +} + +static void FTransform2(const uint8_t* src, const uint8_t* ref, int16_t* out) { + VP8FTransform(src, ref, out); + VP8FTransform(src + 4, ref + 4, out + 16); +} + +static void FTransformWHT(const int16_t* in, int16_t* out) { + // input is 12b signed + int32_t tmp[16]; + int i; + for (i = 0; i < 4; ++i, in += 64) { + const int a0 = (in[0 * 16] + in[2 * 16]); // 13b + const int a1 = (in[1 * 16] + in[3 * 16]); + const int a2 = (in[1 * 16] - in[3 * 16]); + const int a3 = (in[0 * 16] - in[2 * 16]); + tmp[0 + i * 4] = a0 + a1; // 14b + tmp[1 + i * 4] = a3 + a2; + tmp[2 + i * 4] = a3 - a2; + tmp[3 + i * 4] = a0 - a1; + } + for (i = 0; i < 4; ++i) { + const int a0 = (tmp[0 + i] + tmp[8 + i]); // 15b + const int a1 = (tmp[4 + i] + tmp[12+ i]); + const int a2 = (tmp[4 + i] - tmp[12+ i]); + const int a3 = (tmp[0 + i] - tmp[8 + i]); + const int b0 = a0 + a1; // 16b + const int b1 = a3 + a2; + const int b2 = a3 - a2; + const int b3 = a0 - a1; + out[ 0 + i] = b0 >> 1; // 15b + out[ 4 + i] = b1 >> 1; + out[ 8 + i] = b2 >> 1; + out[12 + i] = b3 >> 1; + } +} + +#undef MUL +#undef STORE + +//------------------------------------------------------------------------------ +// Intra predictions + +static WEBP_INLINE void Fill(uint8_t* dst, int value, int size) { + int j; + for (j = 0; j < size; ++j) { + memset(dst + j * BPS, value, size); + } +} + +static WEBP_INLINE void VerticalPred(uint8_t* dst, + const uint8_t* top, int size) { + int j; + if (top != NULL) { + for (j = 0; j < size; ++j) memcpy(dst + j * BPS, top, size); + } else { + Fill(dst, 127, size); + } +} + +static WEBP_INLINE void HorizontalPred(uint8_t* dst, + const uint8_t* left, int size) { + if (left != NULL) { + int j; + for (j = 0; j < size; ++j) { + memset(dst + j * BPS, left[j], size); + } + } else { + Fill(dst, 129, size); + } +} + +static WEBP_INLINE void TrueMotion(uint8_t* dst, const uint8_t* left, + const uint8_t* top, int size) { + int y; + if (left != NULL) { + if (top != NULL) { + const uint8_t* const clip = clip1 + 255 - left[-1]; + for (y = 0; y < size; ++y) { + const uint8_t* const clip_table = clip + left[y]; + int x; + for (x = 0; x < size; ++x) { + dst[x] = clip_table[top[x]]; + } + dst += BPS; + } + } else { + HorizontalPred(dst, left, size); + } + } else { + // true motion without left samples (hence: with default 129 value) + // is equivalent to VE prediction where you just copy the top samples. + // Note that if top samples are not available, the default value is + // then 129, and not 127 as in the VerticalPred case. + if (top != NULL) { + VerticalPred(dst, top, size); + } else { + Fill(dst, 129, size); + } + } +} + +static WEBP_INLINE void DCMode(uint8_t* dst, const uint8_t* left, + const uint8_t* top, + int size, int round, int shift) { + int DC = 0; + int j; + if (top != NULL) { + for (j = 0; j < size; ++j) DC += top[j]; + if (left != NULL) { // top and left present + for (j = 0; j < size; ++j) DC += left[j]; + } else { // top, but no left + DC += DC; + } + DC = (DC + round) >> shift; + } else if (left != NULL) { // left but no top + for (j = 0; j < size; ++j) DC += left[j]; + DC += DC; + DC = (DC + round) >> shift; + } else { // no top, no left, nothing. + DC = 0x80; + } + Fill(dst, DC, size); +} + +//------------------------------------------------------------------------------ +// Chroma 8x8 prediction (paragraph 12.2) + +static void IntraChromaPreds(uint8_t* dst, const uint8_t* left, + const uint8_t* top) { + // U block + DCMode(C8DC8 + dst, left, top, 8, 8, 4); + VerticalPred(C8VE8 + dst, top, 8); + HorizontalPred(C8HE8 + dst, left, 8); + TrueMotion(C8TM8 + dst, left, top, 8); + // V block + dst += 8; + if (top != NULL) top += 8; + if (left != NULL) left += 16; + DCMode(C8DC8 + dst, left, top, 8, 8, 4); + VerticalPred(C8VE8 + dst, top, 8); + HorizontalPred(C8HE8 + dst, left, 8); + TrueMotion(C8TM8 + dst, left, top, 8); +} + +//------------------------------------------------------------------------------ +// luma 16x16 prediction (paragraph 12.3) + +static void Intra16Preds(uint8_t* dst, + const uint8_t* left, const uint8_t* top) { + DCMode(I16DC16 + dst, left, top, 16, 16, 5); + VerticalPred(I16VE16 + dst, top, 16); + HorizontalPred(I16HE16 + dst, left, 16); + TrueMotion(I16TM16 + dst, left, top, 16); +} + +//------------------------------------------------------------------------------ +// luma 4x4 prediction + +#define DST(x, y) dst[(x) + (y) * BPS] +#define AVG3(a, b, c) (((a) + 2 * (b) + (c) + 2) >> 2) +#define AVG2(a, b) (((a) + (b) + 1) >> 1) + +static void VE4(uint8_t* dst, const uint8_t* top) { // vertical + const uint8_t vals[4] = { + AVG3(top[-1], top[0], top[1]), + AVG3(top[ 0], top[1], top[2]), + AVG3(top[ 1], top[2], top[3]), + AVG3(top[ 2], top[3], top[4]) + }; + int i; + for (i = 0; i < 4; ++i) { + memcpy(dst + i * BPS, vals, 4); + } +} + +static void HE4(uint8_t* dst, const uint8_t* top) { // horizontal + const int X = top[-1]; + const int I = top[-2]; + const int J = top[-3]; + const int K = top[-4]; + const int L = top[-5]; + WebPUint32ToMem(dst + 0 * BPS, 0x01010101U * AVG3(X, I, J)); + WebPUint32ToMem(dst + 1 * BPS, 0x01010101U * AVG3(I, J, K)); + WebPUint32ToMem(dst + 2 * BPS, 0x01010101U * AVG3(J, K, L)); + WebPUint32ToMem(dst + 3 * BPS, 0x01010101U * AVG3(K, L, L)); +} + +static void DC4(uint8_t* dst, const uint8_t* top) { + uint32_t dc = 4; + int i; + for (i = 0; i < 4; ++i) dc += top[i] + top[-5 + i]; + Fill(dst, dc >> 3, 4); +} + +static void RD4(uint8_t* dst, const uint8_t* top) { + const int X = top[-1]; + const int I = top[-2]; + const int J = top[-3]; + const int K = top[-4]; + const int L = top[-5]; + const int A = top[0]; + const int B = top[1]; + const int C = top[2]; + const int D = top[3]; + DST(0, 3) = AVG3(J, K, L); + DST(0, 2) = DST(1, 3) = AVG3(I, J, K); + DST(0, 1) = DST(1, 2) = DST(2, 3) = AVG3(X, I, J); + DST(0, 0) = DST(1, 1) = DST(2, 2) = DST(3, 3) = AVG3(A, X, I); + DST(1, 0) = DST(2, 1) = DST(3, 2) = AVG3(B, A, X); + DST(2, 0) = DST(3, 1) = AVG3(C, B, A); + DST(3, 0) = AVG3(D, C, B); +} + +static void LD4(uint8_t* dst, const uint8_t* top) { + const int A = top[0]; + const int B = top[1]; + const int C = top[2]; + const int D = top[3]; + const int E = top[4]; + const int F = top[5]; + const int G = top[6]; + const int H = top[7]; + DST(0, 0) = AVG3(A, B, C); + DST(1, 0) = DST(0, 1) = AVG3(B, C, D); + DST(2, 0) = DST(1, 1) = DST(0, 2) = AVG3(C, D, E); + DST(3, 0) = DST(2, 1) = DST(1, 2) = DST(0, 3) = AVG3(D, E, F); + DST(3, 1) = DST(2, 2) = DST(1, 3) = AVG3(E, F, G); + DST(3, 2) = DST(2, 3) = AVG3(F, G, H); + DST(3, 3) = AVG3(G, H, H); +} + +static void VR4(uint8_t* dst, const uint8_t* top) { + const int X = top[-1]; + const int I = top[-2]; + const int J = top[-3]; + const int K = top[-4]; + const int A = top[0]; + const int B = top[1]; + const int C = top[2]; + const int D = top[3]; + DST(0, 0) = DST(1, 2) = AVG2(X, A); + DST(1, 0) = DST(2, 2) = AVG2(A, B); + DST(2, 0) = DST(3, 2) = AVG2(B, C); + DST(3, 0) = AVG2(C, D); + + DST(0, 3) = AVG3(K, J, I); + DST(0, 2) = AVG3(J, I, X); + DST(0, 1) = DST(1, 3) = AVG3(I, X, A); + DST(1, 1) = DST(2, 3) = AVG3(X, A, B); + DST(2, 1) = DST(3, 3) = AVG3(A, B, C); + DST(3, 1) = AVG3(B, C, D); +} + +static void VL4(uint8_t* dst, const uint8_t* top) { + const int A = top[0]; + const int B = top[1]; + const int C = top[2]; + const int D = top[3]; + const int E = top[4]; + const int F = top[5]; + const int G = top[6]; + const int H = top[7]; + DST(0, 0) = AVG2(A, B); + DST(1, 0) = DST(0, 2) = AVG2(B, C); + DST(2, 0) = DST(1, 2) = AVG2(C, D); + DST(3, 0) = DST(2, 2) = AVG2(D, E); + + DST(0, 1) = AVG3(A, B, C); + DST(1, 1) = DST(0, 3) = AVG3(B, C, D); + DST(2, 1) = DST(1, 3) = AVG3(C, D, E); + DST(3, 1) = DST(2, 3) = AVG3(D, E, F); + DST(3, 2) = AVG3(E, F, G); + DST(3, 3) = AVG3(F, G, H); +} + +static void HU4(uint8_t* dst, const uint8_t* top) { + const int I = top[-2]; + const int J = top[-3]; + const int K = top[-4]; + const int L = top[-5]; + DST(0, 0) = AVG2(I, J); + DST(2, 0) = DST(0, 1) = AVG2(J, K); + DST(2, 1) = DST(0, 2) = AVG2(K, L); + DST(1, 0) = AVG3(I, J, K); + DST(3, 0) = DST(1, 1) = AVG3(J, K, L); + DST(3, 1) = DST(1, 2) = AVG3(K, L, L); + DST(3, 2) = DST(2, 2) = + DST(0, 3) = DST(1, 3) = DST(2, 3) = DST(3, 3) = L; +} + +static void HD4(uint8_t* dst, const uint8_t* top) { + const int X = top[-1]; + const int I = top[-2]; + const int J = top[-3]; + const int K = top[-4]; + const int L = top[-5]; + const int A = top[0]; + const int B = top[1]; + const int C = top[2]; + + DST(0, 0) = DST(2, 1) = AVG2(I, X); + DST(0, 1) = DST(2, 2) = AVG2(J, I); + DST(0, 2) = DST(2, 3) = AVG2(K, J); + DST(0, 3) = AVG2(L, K); + + DST(3, 0) = AVG3(A, B, C); + DST(2, 0) = AVG3(X, A, B); + DST(1, 0) = DST(3, 1) = AVG3(I, X, A); + DST(1, 1) = DST(3, 2) = AVG3(J, I, X); + DST(1, 2) = DST(3, 3) = AVG3(K, J, I); + DST(1, 3) = AVG3(L, K, J); +} + +static void TM4(uint8_t* dst, const uint8_t* top) { + int x, y; + const uint8_t* const clip = clip1 + 255 - top[-1]; + for (y = 0; y < 4; ++y) { + const uint8_t* const clip_table = clip + top[-2 - y]; + for (x = 0; x < 4; ++x) { + dst[x] = clip_table[top[x]]; + } + dst += BPS; + } +} + +#undef DST +#undef AVG3 +#undef AVG2 + +// Left samples are top[-5 .. -2], top_left is top[-1], top are +// located at top[0..3], and top right is top[4..7] +static void Intra4Preds(uint8_t* dst, const uint8_t* top) { + DC4(I4DC4 + dst, top); + TM4(I4TM4 + dst, top); + VE4(I4VE4 + dst, top); + HE4(I4HE4 + dst, top); + RD4(I4RD4 + dst, top); + VR4(I4VR4 + dst, top); + LD4(I4LD4 + dst, top); + VL4(I4VL4 + dst, top); + HD4(I4HD4 + dst, top); + HU4(I4HU4 + dst, top); +} + +//------------------------------------------------------------------------------ +// Metric + +static WEBP_INLINE int GetSSE(const uint8_t* a, const uint8_t* b, + int w, int h) { + int count = 0; + int y, x; + for (y = 0; y < h; ++y) { + for (x = 0; x < w; ++x) { + const int diff = (int)a[x] - b[x]; + count += diff * diff; + } + a += BPS; + b += BPS; + } + return count; +} + +static int SSE16x16(const uint8_t* a, const uint8_t* b) { + return GetSSE(a, b, 16, 16); +} +static int SSE16x8(const uint8_t* a, const uint8_t* b) { + return GetSSE(a, b, 16, 8); +} +static int SSE8x8(const uint8_t* a, const uint8_t* b) { + return GetSSE(a, b, 8, 8); +} +static int SSE4x4(const uint8_t* a, const uint8_t* b) { + return GetSSE(a, b, 4, 4); +} + +//------------------------------------------------------------------------------ +// Texture distortion +// +// We try to match the spectral content (weighted) between source and +// reconstructed samples. + +// Hadamard transform +// Returns the weighted sum of the absolute value of transformed coefficients. +// w[] contains a row-major 4 by 4 symmetric matrix. +static int TTransform(const uint8_t* in, const uint16_t* w) { + int sum = 0; + int tmp[16]; + int i; + // horizontal pass + for (i = 0; i < 4; ++i, in += BPS) { + const int a0 = in[0] + in[2]; + const int a1 = in[1] + in[3]; + const int a2 = in[1] - in[3]; + const int a3 = in[0] - in[2]; + tmp[0 + i * 4] = a0 + a1; + tmp[1 + i * 4] = a3 + a2; + tmp[2 + i * 4] = a3 - a2; + tmp[3 + i * 4] = a0 - a1; + } + // vertical pass + for (i = 0; i < 4; ++i, ++w) { + const int a0 = tmp[0 + i] + tmp[8 + i]; + const int a1 = tmp[4 + i] + tmp[12+ i]; + const int a2 = tmp[4 + i] - tmp[12+ i]; + const int a3 = tmp[0 + i] - tmp[8 + i]; + const int b0 = a0 + a1; + const int b1 = a3 + a2; + const int b2 = a3 - a2; + const int b3 = a0 - a1; + + sum += w[ 0] * abs(b0); + sum += w[ 4] * abs(b1); + sum += w[ 8] * abs(b2); + sum += w[12] * abs(b3); + } + return sum; +} + +static int Disto4x4(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + const int sum1 = TTransform(a, w); + const int sum2 = TTransform(b, w); + return abs(sum2 - sum1) >> 5; +} + +static int Disto16x16(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + int D = 0; + int x, y; + for (y = 0; y < 16 * BPS; y += 4 * BPS) { + for (x = 0; x < 16; x += 4) { + D += Disto4x4(a + x + y, b + x + y, w); + } + } + return D; +} + +//------------------------------------------------------------------------------ +// Quantization +// + +static const uint8_t kZigzag[16] = { + 0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15 +}; + +// Simple quantization +static int QuantizeBlock(int16_t in[16], int16_t out[16], + const VP8Matrix* const mtx) { + int last = -1; + int n; + for (n = 0; n < 16; ++n) { + const int j = kZigzag[n]; + const int sign = (in[j] < 0); + const uint32_t coeff = (sign ? -in[j] : in[j]) + mtx->sharpen_[j]; + if (coeff > mtx->zthresh_[j]) { + const uint32_t Q = mtx->q_[j]; + const uint32_t iQ = mtx->iq_[j]; + const uint32_t B = mtx->bias_[j]; + int level = QUANTDIV(coeff, iQ, B); + if (level > MAX_LEVEL) level = MAX_LEVEL; + if (sign) level = -level; + in[j] = level * (int)Q; + out[n] = level; + if (level) last = n; + } else { + out[n] = 0; + in[j] = 0; + } + } + return (last >= 0); +} + +static int Quantize2Blocks(int16_t in[32], int16_t out[32], + const VP8Matrix* const mtx) { + int nz; + nz = VP8EncQuantizeBlock(in + 0 * 16, out + 0 * 16, mtx) << 0; + nz |= VP8EncQuantizeBlock(in + 1 * 16, out + 1 * 16, mtx) << 1; + return nz; +} + +static int QuantizeBlockWHT(int16_t in[16], int16_t out[16], + const VP8Matrix* const mtx) { + int n, last = -1; + for (n = 0; n < 16; ++n) { + const int j = kZigzag[n]; + const int sign = (in[j] < 0); + const uint32_t coeff = sign ? -in[j] : in[j]; + assert(mtx->sharpen_[j] == 0); + if (coeff > mtx->zthresh_[j]) { + const uint32_t Q = mtx->q_[j]; + const uint32_t iQ = mtx->iq_[j]; + const uint32_t B = mtx->bias_[j]; + int level = QUANTDIV(coeff, iQ, B); + if (level > MAX_LEVEL) level = MAX_LEVEL; + if (sign) level = -level; + in[j] = level * (int)Q; + out[n] = level; + if (level) last = n; + } else { + out[n] = 0; + in[j] = 0; + } + } + return (last >= 0); +} + +//------------------------------------------------------------------------------ +// Block copy + +static WEBP_INLINE void Copy(const uint8_t* src, uint8_t* dst, int w, int h) { + int y; + for (y = 0; y < h; ++y) { + memcpy(dst, src, w); + src += BPS; + dst += BPS; + } +} + +static void Copy4x4(const uint8_t* src, uint8_t* dst) { + Copy(src, dst, 4, 4); +} + +static void Copy16x8(const uint8_t* src, uint8_t* dst) { + Copy(src, dst, 16, 8); +} + +//------------------------------------------------------------------------------ + +static void SSIMAccumulateClipped(const uint8_t* src1, int stride1, + const uint8_t* src2, int stride2, + int xo, int yo, int W, int H, + VP8DistoStats* const stats) { + const int ymin = (yo - VP8_SSIM_KERNEL < 0) ? 0 : yo - VP8_SSIM_KERNEL; + const int ymax = (yo + VP8_SSIM_KERNEL > H - 1) ? H - 1 + : yo + VP8_SSIM_KERNEL; + const int xmin = (xo - VP8_SSIM_KERNEL < 0) ? 0 : xo - VP8_SSIM_KERNEL; + const int xmax = (xo + VP8_SSIM_KERNEL > W - 1) ? W - 1 + : xo + VP8_SSIM_KERNEL; + int x, y; + src1 += ymin * stride1; + src2 += ymin * stride2; + for (y = ymin; y <= ymax; ++y, src1 += stride1, src2 += stride2) { + for (x = xmin; x <= xmax; ++x) { + const int s1 = src1[x]; + const int s2 = src2[x]; + stats->w += 1; + stats->xm += s1; + stats->ym += s2; + stats->xxm += s1 * s1; + stats->xym += s1 * s2; + stats->yym += s2 * s2; + } + } +} + +static void SSIMAccumulate(const uint8_t* src1, int stride1, + const uint8_t* src2, int stride2, + VP8DistoStats* const stats) { + int x, y; + for (y = 0; y <= 2 * VP8_SSIM_KERNEL; ++y, src1 += stride1, src2 += stride2) { + for (x = 0; x <= 2 * VP8_SSIM_KERNEL; ++x) { + const int s1 = src1[x]; + const int s2 = src2[x]; + stats->w += 1; + stats->xm += s1; + stats->ym += s2; + stats->xxm += s1 * s1; + stats->xym += s1 * s2; + stats->yym += s2 * s2; + } + } +} + +VP8SSIMAccumulateFunc VP8SSIMAccumulate; +VP8SSIMAccumulateClippedFunc VP8SSIMAccumulateClipped; + +static volatile VP8CPUInfo ssim_last_cpuinfo_used = + (VP8CPUInfo)&ssim_last_cpuinfo_used; + +WEBP_TSAN_IGNORE_FUNCTION void VP8SSIMDspInit(void) { + if (ssim_last_cpuinfo_used == VP8GetCPUInfo) return; + + VP8SSIMAccumulate = SSIMAccumulate; + VP8SSIMAccumulateClipped = SSIMAccumulateClipped; + + ssim_last_cpuinfo_used = VP8GetCPUInfo; +} + +//------------------------------------------------------------------------------ +// Initialization + +// Speed-critical function pointers. We have to initialize them to the default +// implementations within VP8EncDspInit(). +VP8CHisto VP8CollectHistogram; +VP8Idct VP8ITransform; +VP8Fdct VP8FTransform; +VP8Fdct VP8FTransform2; +VP8WHT VP8FTransformWHT; +VP8Intra4Preds VP8EncPredLuma4; +VP8IntraPreds VP8EncPredLuma16; +VP8IntraPreds VP8EncPredChroma8; +VP8Metric VP8SSE16x16; +VP8Metric VP8SSE8x8; +VP8Metric VP8SSE16x8; +VP8Metric VP8SSE4x4; +VP8WMetric VP8TDisto4x4; +VP8WMetric VP8TDisto16x16; +VP8QuantizeBlock VP8EncQuantizeBlock; +VP8Quantize2Blocks VP8EncQuantize2Blocks; +VP8QuantizeBlockWHT VP8EncQuantizeBlockWHT; +VP8BlockCopy VP8Copy4x4; +VP8BlockCopy VP8Copy16x8; + +extern void VP8EncDspInitSSE2(void); +extern void VP8EncDspInitSSE41(void); +extern void VP8EncDspInitAVX2(void); +extern void VP8EncDspInitNEON(void); +extern void VP8EncDspInitMIPS32(void); +extern void VP8EncDspInitMIPSdspR2(void); + +static volatile VP8CPUInfo enc_last_cpuinfo_used = + (VP8CPUInfo)&enc_last_cpuinfo_used; + +WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInit(void) { + if (enc_last_cpuinfo_used == VP8GetCPUInfo) return; + + VP8DspInit(); // common inverse transforms + InitTables(); + + // default C implementations + VP8CollectHistogram = CollectHistogram; + VP8ITransform = ITransform; + VP8FTransform = FTransform; + VP8FTransform2 = FTransform2; + VP8FTransformWHT = FTransformWHT; + VP8EncPredLuma4 = Intra4Preds; + VP8EncPredLuma16 = Intra16Preds; + VP8EncPredChroma8 = IntraChromaPreds; + VP8SSE16x16 = SSE16x16; + VP8SSE8x8 = SSE8x8; + VP8SSE16x8 = SSE16x8; + VP8SSE4x4 = SSE4x4; + VP8TDisto4x4 = Disto4x4; + VP8TDisto16x16 = Disto16x16; + VP8EncQuantizeBlock = QuantizeBlock; + VP8EncQuantize2Blocks = Quantize2Blocks; + VP8EncQuantizeBlockWHT = QuantizeBlockWHT; + VP8Copy4x4 = Copy4x4; + VP8Copy16x8 = Copy16x8; + + // If defined, use CPUInfo() to overwrite some pointers with faster versions. + if (VP8GetCPUInfo != NULL) { +#if defined(WEBP_USE_SSE2) + if (VP8GetCPUInfo(kSSE2)) { + VP8EncDspInitSSE2(); +#if defined(WEBP_USE_SSE41) + if (VP8GetCPUInfo(kSSE4_1)) { + VP8EncDspInitSSE41(); + } +#endif + } +#endif +#if defined(WEBP_USE_AVX2) + if (VP8GetCPUInfo(kAVX2)) { + VP8EncDspInitAVX2(); + } +#endif +#if defined(WEBP_USE_NEON) + if (VP8GetCPUInfo(kNEON)) { + VP8EncDspInitNEON(); + } +#endif +#if defined(WEBP_USE_MIPS32) + if (VP8GetCPUInfo(kMIPS32)) { + VP8EncDspInitMIPS32(); + } +#endif +#if defined(WEBP_USE_MIPS_DSP_R2) + if (VP8GetCPUInfo(kMIPSdspR2)) { + VP8EncDspInitMIPSdspR2(); + } +#endif + } + enc_last_cpuinfo_used = VP8GetCPUInfo; +} diff --git a/thirdparty/libwebp/dsp/enc_avx2.c b/thirdparty/libwebp/dsp/enc_avx2.c new file mode 100644 index 0000000000..93efb30b10 --- /dev/null +++ b/thirdparty/libwebp/dsp/enc_avx2.c @@ -0,0 +1,21 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// AVX2 version of speed-critical encoding functions. + +#include "./dsp.h" + +#if defined(WEBP_USE_AVX2) + +#endif // WEBP_USE_AVX2 + +//------------------------------------------------------------------------------ +// Entry point + +WEBP_DSP_INIT_STUB(VP8EncDspInitAVX2) diff --git a/thirdparty/libwebp/dsp/enc_mips32.c b/thirdparty/libwebp/dsp/enc_mips32.c new file mode 100644 index 0000000000..fd10143de9 --- /dev/null +++ b/thirdparty/libwebp/dsp/enc_mips32.c @@ -0,0 +1,672 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MIPS version of speed-critical encoding functions. +// +// Author(s): Djordje Pesut (djordje.pesut@imgtec.com) +// Jovan Zelincevic (jovan.zelincevic@imgtec.com) +// Slobodan Prijic (slobodan.prijic@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS32) + +#include "./mips_macro.h" +#include "../enc/vp8enci.h" +#include "../enc/cost.h" + +static const int kC1 = 20091 + (1 << 16); +static const int kC2 = 35468; + +// macro for one vertical pass in ITransformOne +// MUL macro inlined +// temp0..temp15 holds tmp[0]..tmp[15] +// A..D - offsets in bytes to load from in buffer +// TEMP0..TEMP3 - registers for corresponding tmp elements +// TEMP4..TEMP5 - temporary registers +#define VERTICAL_PASS(A, B, C, D, TEMP4, TEMP0, TEMP1, TEMP2, TEMP3) \ + "lh %[temp16], " #A "(%[temp20]) \n\t" \ + "lh %[temp18], " #B "(%[temp20]) \n\t" \ + "lh %[temp17], " #C "(%[temp20]) \n\t" \ + "lh %[temp19], " #D "(%[temp20]) \n\t" \ + "addu %[" #TEMP4 "], %[temp16], %[temp18] \n\t" \ + "subu %[temp16], %[temp16], %[temp18] \n\t" \ + "mul %[" #TEMP0 "], %[temp17], %[kC2] \n\t" \ + "mul %[temp18], %[temp19], %[kC1] \n\t" \ + "mul %[temp17], %[temp17], %[kC1] \n\t" \ + "mul %[temp19], %[temp19], %[kC2] \n\t" \ + "sra %[" #TEMP0 "], %[" #TEMP0 "], 16 \n\n" \ + "sra %[temp18], %[temp18], 16 \n\n" \ + "sra %[temp17], %[temp17], 16 \n\n" \ + "sra %[temp19], %[temp19], 16 \n\n" \ + "subu %[" #TEMP2 "], %[" #TEMP0 "], %[temp18] \n\t" \ + "addu %[" #TEMP3 "], %[temp17], %[temp19] \n\t" \ + "addu %[" #TEMP0 "], %[" #TEMP4 "], %[" #TEMP3 "] \n\t" \ + "addu %[" #TEMP1 "], %[temp16], %[" #TEMP2 "] \n\t" \ + "subu %[" #TEMP2 "], %[temp16], %[" #TEMP2 "] \n\t" \ + "subu %[" #TEMP3 "], %[" #TEMP4 "], %[" #TEMP3 "] \n\t" + +// macro for one horizontal pass in ITransformOne +// MUL and STORE macros inlined +// a = clip_8b(a) is replaced with: a = max(a, 0); a = min(a, 255) +// temp0..temp15 holds tmp[0]..tmp[15] +// A - offset in bytes to load from ref and store to dst buffer +// TEMP0, TEMP4, TEMP8 and TEMP12 - registers for corresponding tmp elements +#define HORIZONTAL_PASS(A, TEMP0, TEMP4, TEMP8, TEMP12) \ + "addiu %[" #TEMP0 "], %[" #TEMP0 "], 4 \n\t" \ + "addu %[temp16], %[" #TEMP0 "], %[" #TEMP8 "] \n\t" \ + "subu %[temp17], %[" #TEMP0 "], %[" #TEMP8 "] \n\t" \ + "mul %[" #TEMP0 "], %[" #TEMP4 "], %[kC2] \n\t" \ + "mul %[" #TEMP8 "], %[" #TEMP12 "], %[kC1] \n\t" \ + "mul %[" #TEMP4 "], %[" #TEMP4 "], %[kC1] \n\t" \ + "mul %[" #TEMP12 "], %[" #TEMP12 "], %[kC2] \n\t" \ + "sra %[" #TEMP0 "], %[" #TEMP0 "], 16 \n\t" \ + "sra %[" #TEMP8 "], %[" #TEMP8 "], 16 \n\t" \ + "sra %[" #TEMP4 "], %[" #TEMP4 "], 16 \n\t" \ + "sra %[" #TEMP12 "], %[" #TEMP12 "], 16 \n\t" \ + "subu %[temp18], %[" #TEMP0 "], %[" #TEMP8 "] \n\t" \ + "addu %[temp19], %[" #TEMP4 "], %[" #TEMP12 "] \n\t" \ + "addu %[" #TEMP0 "], %[temp16], %[temp19] \n\t" \ + "addu %[" #TEMP4 "], %[temp17], %[temp18] \n\t" \ + "subu %[" #TEMP8 "], %[temp17], %[temp18] \n\t" \ + "subu %[" #TEMP12 "], %[temp16], %[temp19] \n\t" \ + "lw %[temp20], 0(%[args]) \n\t" \ + "sra %[" #TEMP0 "], %[" #TEMP0 "], 3 \n\t" \ + "sra %[" #TEMP4 "], %[" #TEMP4 "], 3 \n\t" \ + "sra %[" #TEMP8 "], %[" #TEMP8 "], 3 \n\t" \ + "sra %[" #TEMP12 "], %[" #TEMP12 "], 3 \n\t" \ + "lbu %[temp16], 0+" XSTR(BPS) "*" #A "(%[temp20]) \n\t" \ + "lbu %[temp17], 1+" XSTR(BPS) "*" #A "(%[temp20]) \n\t" \ + "lbu %[temp18], 2+" XSTR(BPS) "*" #A "(%[temp20]) \n\t" \ + "lbu %[temp19], 3+" XSTR(BPS) "*" #A "(%[temp20]) \n\t" \ + "addu %[" #TEMP0 "], %[temp16], %[" #TEMP0 "] \n\t" \ + "addu %[" #TEMP4 "], %[temp17], %[" #TEMP4 "] \n\t" \ + "addu %[" #TEMP8 "], %[temp18], %[" #TEMP8 "] \n\t" \ + "addu %[" #TEMP12 "], %[temp19], %[" #TEMP12 "] \n\t" \ + "slt %[temp16], %[" #TEMP0 "], $zero \n\t" \ + "slt %[temp17], %[" #TEMP4 "], $zero \n\t" \ + "slt %[temp18], %[" #TEMP8 "], $zero \n\t" \ + "slt %[temp19], %[" #TEMP12 "], $zero \n\t" \ + "movn %[" #TEMP0 "], $zero, %[temp16] \n\t" \ + "movn %[" #TEMP4 "], $zero, %[temp17] \n\t" \ + "movn %[" #TEMP8 "], $zero, %[temp18] \n\t" \ + "movn %[" #TEMP12 "], $zero, %[temp19] \n\t" \ + "addiu %[temp20], $zero, 255 \n\t" \ + "slt %[temp16], %[" #TEMP0 "], %[temp20] \n\t" \ + "slt %[temp17], %[" #TEMP4 "], %[temp20] \n\t" \ + "slt %[temp18], %[" #TEMP8 "], %[temp20] \n\t" \ + "slt %[temp19], %[" #TEMP12 "], %[temp20] \n\t" \ + "movz %[" #TEMP0 "], %[temp20], %[temp16] \n\t" \ + "movz %[" #TEMP4 "], %[temp20], %[temp17] \n\t" \ + "lw %[temp16], 8(%[args]) \n\t" \ + "movz %[" #TEMP8 "], %[temp20], %[temp18] \n\t" \ + "movz %[" #TEMP12 "], %[temp20], %[temp19] \n\t" \ + "sb %[" #TEMP0 "], 0+" XSTR(BPS) "*" #A "(%[temp16]) \n\t" \ + "sb %[" #TEMP4 "], 1+" XSTR(BPS) "*" #A "(%[temp16]) \n\t" \ + "sb %[" #TEMP8 "], 2+" XSTR(BPS) "*" #A "(%[temp16]) \n\t" \ + "sb %[" #TEMP12 "], 3+" XSTR(BPS) "*" #A "(%[temp16]) \n\t" + +// Does one or two inverse transforms. +static WEBP_INLINE void ITransformOne(const uint8_t* ref, const int16_t* in, + uint8_t* dst) { + int temp0, temp1, temp2, temp3, temp4, temp5, temp6; + int temp7, temp8, temp9, temp10, temp11, temp12, temp13; + int temp14, temp15, temp16, temp17, temp18, temp19, temp20; + const int* args[3] = {(const int*)ref, (const int*)in, (const int*)dst}; + + __asm__ volatile( + "lw %[temp20], 4(%[args]) \n\t" + VERTICAL_PASS(0, 16, 8, 24, temp4, temp0, temp1, temp2, temp3) + VERTICAL_PASS(2, 18, 10, 26, temp8, temp4, temp5, temp6, temp7) + VERTICAL_PASS(4, 20, 12, 28, temp12, temp8, temp9, temp10, temp11) + VERTICAL_PASS(6, 22, 14, 30, temp20, temp12, temp13, temp14, temp15) + + HORIZONTAL_PASS(0, temp0, temp4, temp8, temp12) + HORIZONTAL_PASS(1, temp1, temp5, temp9, temp13) + HORIZONTAL_PASS(2, temp2, temp6, temp10, temp14) + HORIZONTAL_PASS(3, temp3, temp7, temp11, temp15) + + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8), + [temp9]"=&r"(temp9), [temp10]"=&r"(temp10), [temp11]"=&r"(temp11), + [temp12]"=&r"(temp12), [temp13]"=&r"(temp13), [temp14]"=&r"(temp14), + [temp15]"=&r"(temp15), [temp16]"=&r"(temp16), [temp17]"=&r"(temp17), + [temp18]"=&r"(temp18), [temp19]"=&r"(temp19), [temp20]"=&r"(temp20) + : [args]"r"(args), [kC1]"r"(kC1), [kC2]"r"(kC2) + : "memory", "hi", "lo" + ); +} + +static void ITransform(const uint8_t* ref, const int16_t* in, + uint8_t* dst, int do_two) { + ITransformOne(ref, in, dst); + if (do_two) { + ITransformOne(ref + 4, in + 16, dst + 4); + } +} + +#undef VERTICAL_PASS +#undef HORIZONTAL_PASS + +// macro for one pass through for loop in QuantizeBlock +// QUANTDIV macro inlined +// J - offset in bytes (kZigzag[n] * 2) +// K - offset in bytes (kZigzag[n] * 4) +// N - offset in bytes (n * 2) +#define QUANTIZE_ONE(J, K, N) \ + "lh %[temp0], " #J "(%[ppin]) \n\t" \ + "lhu %[temp1], " #J "(%[ppsharpen]) \n\t" \ + "lw %[temp2], " #K "(%[ppzthresh]) \n\t" \ + "sra %[sign], %[temp0], 15 \n\t" \ + "xor %[coeff], %[temp0], %[sign] \n\t" \ + "subu %[coeff], %[coeff], %[sign] \n\t" \ + "addu %[coeff], %[coeff], %[temp1] \n\t" \ + "slt %[temp4], %[temp2], %[coeff] \n\t" \ + "addiu %[temp5], $zero, 0 \n\t" \ + "addiu %[level], $zero, 0 \n\t" \ + "beqz %[temp4], 2f \n\t" \ + "lhu %[temp1], " #J "(%[ppiq]) \n\t" \ + "lw %[temp2], " #K "(%[ppbias]) \n\t" \ + "lhu %[temp3], " #J "(%[ppq]) \n\t" \ + "mul %[level], %[coeff], %[temp1] \n\t" \ + "addu %[level], %[level], %[temp2] \n\t" \ + "sra %[level], %[level], 17 \n\t" \ + "slt %[temp4], %[max_level], %[level] \n\t" \ + "movn %[level], %[max_level], %[temp4] \n\t" \ + "xor %[level], %[level], %[sign] \n\t" \ + "subu %[level], %[level], %[sign] \n\t" \ + "mul %[temp5], %[level], %[temp3] \n\t" \ +"2: \n\t" \ + "sh %[temp5], " #J "(%[ppin]) \n\t" \ + "sh %[level], " #N "(%[pout]) \n\t" + +static int QuantizeBlock(int16_t in[16], int16_t out[16], + const VP8Matrix* const mtx) { + int temp0, temp1, temp2, temp3, temp4, temp5; + int sign, coeff, level, i; + int max_level = MAX_LEVEL; + + int16_t* ppin = &in[0]; + int16_t* pout = &out[0]; + const uint16_t* ppsharpen = &mtx->sharpen_[0]; + const uint32_t* ppzthresh = &mtx->zthresh_[0]; + const uint16_t* ppq = &mtx->q_[0]; + const uint16_t* ppiq = &mtx->iq_[0]; + const uint32_t* ppbias = &mtx->bias_[0]; + + __asm__ volatile( + QUANTIZE_ONE( 0, 0, 0) + QUANTIZE_ONE( 2, 4, 2) + QUANTIZE_ONE( 8, 16, 4) + QUANTIZE_ONE(16, 32, 6) + QUANTIZE_ONE(10, 20, 8) + QUANTIZE_ONE( 4, 8, 10) + QUANTIZE_ONE( 6, 12, 12) + QUANTIZE_ONE(12, 24, 14) + QUANTIZE_ONE(18, 36, 16) + QUANTIZE_ONE(24, 48, 18) + QUANTIZE_ONE(26, 52, 20) + QUANTIZE_ONE(20, 40, 22) + QUANTIZE_ONE(14, 28, 24) + QUANTIZE_ONE(22, 44, 26) + QUANTIZE_ONE(28, 56, 28) + QUANTIZE_ONE(30, 60, 30) + + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), + [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [sign]"=&r"(sign), [coeff]"=&r"(coeff), + [level]"=&r"(level) + : [pout]"r"(pout), [ppin]"r"(ppin), + [ppiq]"r"(ppiq), [max_level]"r"(max_level), + [ppbias]"r"(ppbias), [ppzthresh]"r"(ppzthresh), + [ppsharpen]"r"(ppsharpen), [ppq]"r"(ppq) + : "memory", "hi", "lo" + ); + + // moved out from macro to increase possibility for earlier breaking + for (i = 15; i >= 0; i--) { + if (out[i]) return 1; + } + return 0; +} + +static int Quantize2Blocks(int16_t in[32], int16_t out[32], + const VP8Matrix* const mtx) { + int nz; + nz = QuantizeBlock(in + 0 * 16, out + 0 * 16, mtx) << 0; + nz |= QuantizeBlock(in + 1 * 16, out + 1 * 16, mtx) << 1; + return nz; +} + +#undef QUANTIZE_ONE + +// macro for one horizontal pass in Disto4x4 (TTransform) +// two calls of function TTransform are merged into single one +// A - offset in bytes to load from a and b buffers +// E..H - offsets in bytes to store first results to tmp buffer +// E1..H1 - offsets in bytes to store second results to tmp buffer +#define HORIZONTAL_PASS(A, E, F, G, H, E1, F1, G1, H1) \ + "lbu %[temp0], 0+" XSTR(BPS) "*" #A "(%[a]) \n\t" \ + "lbu %[temp1], 1+" XSTR(BPS) "*" #A "(%[a]) \n\t" \ + "lbu %[temp2], 2+" XSTR(BPS) "*" #A "(%[a]) \n\t" \ + "lbu %[temp3], 3+" XSTR(BPS) "*" #A "(%[a]) \n\t" \ + "lbu %[temp4], 0+" XSTR(BPS) "*" #A "(%[b]) \n\t" \ + "lbu %[temp5], 1+" XSTR(BPS) "*" #A "(%[b]) \n\t" \ + "lbu %[temp6], 2+" XSTR(BPS) "*" #A "(%[b]) \n\t" \ + "lbu %[temp7], 3+" XSTR(BPS) "*" #A "(%[b]) \n\t" \ + "addu %[temp8], %[temp0], %[temp2] \n\t" \ + "subu %[temp0], %[temp0], %[temp2] \n\t" \ + "addu %[temp2], %[temp1], %[temp3] \n\t" \ + "subu %[temp1], %[temp1], %[temp3] \n\t" \ + "addu %[temp3], %[temp4], %[temp6] \n\t" \ + "subu %[temp4], %[temp4], %[temp6] \n\t" \ + "addu %[temp6], %[temp5], %[temp7] \n\t" \ + "subu %[temp5], %[temp5], %[temp7] \n\t" \ + "addu %[temp7], %[temp8], %[temp2] \n\t" \ + "subu %[temp2], %[temp8], %[temp2] \n\t" \ + "addu %[temp8], %[temp0], %[temp1] \n\t" \ + "subu %[temp0], %[temp0], %[temp1] \n\t" \ + "addu %[temp1], %[temp3], %[temp6] \n\t" \ + "subu %[temp3], %[temp3], %[temp6] \n\t" \ + "addu %[temp6], %[temp4], %[temp5] \n\t" \ + "subu %[temp4], %[temp4], %[temp5] \n\t" \ + "sw %[temp7], " #E "(%[tmp]) \n\t" \ + "sw %[temp2], " #H "(%[tmp]) \n\t" \ + "sw %[temp8], " #F "(%[tmp]) \n\t" \ + "sw %[temp0], " #G "(%[tmp]) \n\t" \ + "sw %[temp1], " #E1 "(%[tmp]) \n\t" \ + "sw %[temp3], " #H1 "(%[tmp]) \n\t" \ + "sw %[temp6], " #F1 "(%[tmp]) \n\t" \ + "sw %[temp4], " #G1 "(%[tmp]) \n\t" + +// macro for one vertical pass in Disto4x4 (TTransform) +// two calls of function TTransform are merged into single one +// since only one accu is available in mips32r1 instruction set +// first is done second call of function TTransform and after +// that first one. +// const int sum1 = TTransform(a, w); +// const int sum2 = TTransform(b, w); +// return abs(sum2 - sum1) >> 5; +// (sum2 - sum1) is calculated with madds (sub2) and msubs (sub1) +// A..D - offsets in bytes to load first results from tmp buffer +// A1..D1 - offsets in bytes to load second results from tmp buffer +// E..H - offsets in bytes to load from w buffer +#define VERTICAL_PASS(A, B, C, D, A1, B1, C1, D1, E, F, G, H) \ + "lw %[temp0], " #A1 "(%[tmp]) \n\t" \ + "lw %[temp1], " #C1 "(%[tmp]) \n\t" \ + "lw %[temp2], " #B1 "(%[tmp]) \n\t" \ + "lw %[temp3], " #D1 "(%[tmp]) \n\t" \ + "addu %[temp8], %[temp0], %[temp1] \n\t" \ + "subu %[temp0], %[temp0], %[temp1] \n\t" \ + "addu %[temp1], %[temp2], %[temp3] \n\t" \ + "subu %[temp2], %[temp2], %[temp3] \n\t" \ + "addu %[temp3], %[temp8], %[temp1] \n\t" \ + "subu %[temp8], %[temp8], %[temp1] \n\t" \ + "addu %[temp1], %[temp0], %[temp2] \n\t" \ + "subu %[temp0], %[temp0], %[temp2] \n\t" \ + "sra %[temp4], %[temp3], 31 \n\t" \ + "sra %[temp5], %[temp1], 31 \n\t" \ + "sra %[temp6], %[temp0], 31 \n\t" \ + "sra %[temp7], %[temp8], 31 \n\t" \ + "xor %[temp3], %[temp3], %[temp4] \n\t" \ + "xor %[temp1], %[temp1], %[temp5] \n\t" \ + "xor %[temp0], %[temp0], %[temp6] \n\t" \ + "xor %[temp8], %[temp8], %[temp7] \n\t" \ + "subu %[temp3], %[temp3], %[temp4] \n\t" \ + "subu %[temp1], %[temp1], %[temp5] \n\t" \ + "subu %[temp0], %[temp0], %[temp6] \n\t" \ + "subu %[temp8], %[temp8], %[temp7] \n\t" \ + "lhu %[temp4], " #E "(%[w]) \n\t" \ + "lhu %[temp5], " #F "(%[w]) \n\t" \ + "lhu %[temp6], " #G "(%[w]) \n\t" \ + "lhu %[temp7], " #H "(%[w]) \n\t" \ + "madd %[temp4], %[temp3] \n\t" \ + "madd %[temp5], %[temp1] \n\t" \ + "madd %[temp6], %[temp0] \n\t" \ + "madd %[temp7], %[temp8] \n\t" \ + "lw %[temp0], " #A "(%[tmp]) \n\t" \ + "lw %[temp1], " #C "(%[tmp]) \n\t" \ + "lw %[temp2], " #B "(%[tmp]) \n\t" \ + "lw %[temp3], " #D "(%[tmp]) \n\t" \ + "addu %[temp8], %[temp0], %[temp1] \n\t" \ + "subu %[temp0], %[temp0], %[temp1] \n\t" \ + "addu %[temp1], %[temp2], %[temp3] \n\t" \ + "subu %[temp2], %[temp2], %[temp3] \n\t" \ + "addu %[temp3], %[temp8], %[temp1] \n\t" \ + "subu %[temp1], %[temp8], %[temp1] \n\t" \ + "addu %[temp8], %[temp0], %[temp2] \n\t" \ + "subu %[temp0], %[temp0], %[temp2] \n\t" \ + "sra %[temp2], %[temp3], 31 \n\t" \ + "xor %[temp3], %[temp3], %[temp2] \n\t" \ + "subu %[temp3], %[temp3], %[temp2] \n\t" \ + "msub %[temp4], %[temp3] \n\t" \ + "sra %[temp2], %[temp8], 31 \n\t" \ + "sra %[temp3], %[temp0], 31 \n\t" \ + "sra %[temp4], %[temp1], 31 \n\t" \ + "xor %[temp8], %[temp8], %[temp2] \n\t" \ + "xor %[temp0], %[temp0], %[temp3] \n\t" \ + "xor %[temp1], %[temp1], %[temp4] \n\t" \ + "subu %[temp8], %[temp8], %[temp2] \n\t" \ + "subu %[temp0], %[temp0], %[temp3] \n\t" \ + "subu %[temp1], %[temp1], %[temp4] \n\t" \ + "msub %[temp5], %[temp8] \n\t" \ + "msub %[temp6], %[temp0] \n\t" \ + "msub %[temp7], %[temp1] \n\t" + +static int Disto4x4(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + int tmp[32]; + int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8; + + __asm__ volatile( + HORIZONTAL_PASS(0, 0, 4, 8, 12, 64, 68, 72, 76) + HORIZONTAL_PASS(1, 16, 20, 24, 28, 80, 84, 88, 92) + HORIZONTAL_PASS(2, 32, 36, 40, 44, 96, 100, 104, 108) + HORIZONTAL_PASS(3, 48, 52, 56, 60, 112, 116, 120, 124) + "mthi $zero \n\t" + "mtlo $zero \n\t" + VERTICAL_PASS( 0, 16, 32, 48, 64, 80, 96, 112, 0, 8, 16, 24) + VERTICAL_PASS( 4, 20, 36, 52, 68, 84, 100, 116, 2, 10, 18, 26) + VERTICAL_PASS( 8, 24, 40, 56, 72, 88, 104, 120, 4, 12, 20, 28) + VERTICAL_PASS(12, 28, 44, 60, 76, 92, 108, 124, 6, 14, 22, 30) + "mflo %[temp0] \n\t" + "sra %[temp1], %[temp0], 31 \n\t" + "xor %[temp0], %[temp0], %[temp1] \n\t" + "subu %[temp0], %[temp0], %[temp1] \n\t" + "sra %[temp0], %[temp0], 5 \n\t" + + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8) + : [a]"r"(a), [b]"r"(b), [w]"r"(w), [tmp]"r"(tmp) + : "memory", "hi", "lo" + ); + + return temp0; +} + +#undef VERTICAL_PASS +#undef HORIZONTAL_PASS + +static int Disto16x16(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + int D = 0; + int x, y; + for (y = 0; y < 16 * BPS; y += 4 * BPS) { + for (x = 0; x < 16; x += 4) { + D += Disto4x4(a + x + y, b + x + y, w); + } + } + return D; +} + +// macro for one horizontal pass in FTransform +// temp0..temp15 holds tmp[0]..tmp[15] +// A - offset in bytes to load from src and ref buffers +// TEMP0..TEMP3 - registers for corresponding tmp elements +#define HORIZONTAL_PASS(A, TEMP0, TEMP1, TEMP2, TEMP3) \ + "lw %[" #TEMP1 "], 0(%[args]) \n\t" \ + "lw %[" #TEMP2 "], 4(%[args]) \n\t" \ + "lbu %[temp16], 0+" XSTR(BPS) "*" #A "(%[" #TEMP1 "]) \n\t" \ + "lbu %[temp17], 0+" XSTR(BPS) "*" #A "(%[" #TEMP2 "]) \n\t" \ + "lbu %[temp18], 1+" XSTR(BPS) "*" #A "(%[" #TEMP1 "]) \n\t" \ + "lbu %[temp19], 1+" XSTR(BPS) "*" #A "(%[" #TEMP2 "]) \n\t" \ + "subu %[temp20], %[temp16], %[temp17] \n\t" \ + "lbu %[temp16], 2+" XSTR(BPS) "*" #A "(%[" #TEMP1 "]) \n\t" \ + "lbu %[temp17], 2+" XSTR(BPS) "*" #A "(%[" #TEMP2 "]) \n\t" \ + "subu %[" #TEMP0 "], %[temp18], %[temp19] \n\t" \ + "lbu %[temp18], 3+" XSTR(BPS) "*" #A "(%[" #TEMP1 "]) \n\t" \ + "lbu %[temp19], 3+" XSTR(BPS) "*" #A "(%[" #TEMP2 "]) \n\t" \ + "subu %[" #TEMP1 "], %[temp16], %[temp17] \n\t" \ + "subu %[" #TEMP2 "], %[temp18], %[temp19] \n\t" \ + "addu %[" #TEMP3 "], %[temp20], %[" #TEMP2 "] \n\t" \ + "subu %[" #TEMP2 "], %[temp20], %[" #TEMP2 "] \n\t" \ + "addu %[temp20], %[" #TEMP0 "], %[" #TEMP1 "] \n\t" \ + "subu %[" #TEMP0 "], %[" #TEMP0 "], %[" #TEMP1 "] \n\t" \ + "mul %[temp16], %[" #TEMP2 "], %[c5352] \n\t" \ + "mul %[temp17], %[" #TEMP2 "], %[c2217] \n\t" \ + "mul %[temp18], %[" #TEMP0 "], %[c5352] \n\t" \ + "mul %[temp19], %[" #TEMP0 "], %[c2217] \n\t" \ + "addu %[" #TEMP1 "], %[" #TEMP3 "], %[temp20] \n\t" \ + "subu %[temp20], %[" #TEMP3 "], %[temp20] \n\t" \ + "sll %[" #TEMP0 "], %[" #TEMP1 "], 3 \n\t" \ + "sll %[" #TEMP2 "], %[temp20], 3 \n\t" \ + "addiu %[temp16], %[temp16], 1812 \n\t" \ + "addiu %[temp17], %[temp17], 937 \n\t" \ + "addu %[temp16], %[temp16], %[temp19] \n\t" \ + "subu %[temp17], %[temp17], %[temp18] \n\t" \ + "sra %[" #TEMP1 "], %[temp16], 9 \n\t" \ + "sra %[" #TEMP3 "], %[temp17], 9 \n\t" + +// macro for one vertical pass in FTransform +// temp0..temp15 holds tmp[0]..tmp[15] +// A..D - offsets in bytes to store to out buffer +// TEMP0, TEMP4, TEMP8 and TEMP12 - registers for corresponding tmp elements +#define VERTICAL_PASS(A, B, C, D, TEMP0, TEMP4, TEMP8, TEMP12) \ + "addu %[temp16], %[" #TEMP0 "], %[" #TEMP12 "] \n\t" \ + "subu %[temp19], %[" #TEMP0 "], %[" #TEMP12 "] \n\t" \ + "addu %[temp17], %[" #TEMP4 "], %[" #TEMP8 "] \n\t" \ + "subu %[temp18], %[" #TEMP4 "], %[" #TEMP8 "] \n\t" \ + "mul %[" #TEMP8 "], %[temp19], %[c2217] \n\t" \ + "mul %[" #TEMP12 "], %[temp18], %[c2217] \n\t" \ + "mul %[" #TEMP4 "], %[temp19], %[c5352] \n\t" \ + "mul %[temp18], %[temp18], %[c5352] \n\t" \ + "addiu %[temp16], %[temp16], 7 \n\t" \ + "addu %[" #TEMP0 "], %[temp16], %[temp17] \n\t" \ + "sra %[" #TEMP0 "], %[" #TEMP0 "], 4 \n\t" \ + "addu %[" #TEMP12 "], %[" #TEMP12 "], %[" #TEMP4 "] \n\t" \ + "subu %[" #TEMP4 "], %[temp16], %[temp17] \n\t" \ + "sra %[" #TEMP4 "], %[" #TEMP4 "], 4 \n\t" \ + "addiu %[" #TEMP8 "], %[" #TEMP8 "], 30000 \n\t" \ + "addiu %[" #TEMP12 "], %[" #TEMP12 "], 12000 \n\t" \ + "addiu %[" #TEMP8 "], %[" #TEMP8 "], 21000 \n\t" \ + "subu %[" #TEMP8 "], %[" #TEMP8 "], %[temp18] \n\t" \ + "sra %[" #TEMP12 "], %[" #TEMP12 "], 16 \n\t" \ + "sra %[" #TEMP8 "], %[" #TEMP8 "], 16 \n\t" \ + "addiu %[temp16], %[" #TEMP12 "], 1 \n\t" \ + "movn %[" #TEMP12 "], %[temp16], %[temp19] \n\t" \ + "sh %[" #TEMP0 "], " #A "(%[temp20]) \n\t" \ + "sh %[" #TEMP4 "], " #C "(%[temp20]) \n\t" \ + "sh %[" #TEMP8 "], " #D "(%[temp20]) \n\t" \ + "sh %[" #TEMP12 "], " #B "(%[temp20]) \n\t" + +static void FTransform(const uint8_t* src, const uint8_t* ref, int16_t* out) { + int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8; + int temp9, temp10, temp11, temp12, temp13, temp14, temp15, temp16; + int temp17, temp18, temp19, temp20; + const int c2217 = 2217; + const int c5352 = 5352; + const int* const args[3] = + { (const int*)src, (const int*)ref, (const int*)out }; + + __asm__ volatile( + HORIZONTAL_PASS(0, temp0, temp1, temp2, temp3) + HORIZONTAL_PASS(1, temp4, temp5, temp6, temp7) + HORIZONTAL_PASS(2, temp8, temp9, temp10, temp11) + HORIZONTAL_PASS(3, temp12, temp13, temp14, temp15) + "lw %[temp20], 8(%[args]) \n\t" + VERTICAL_PASS(0, 8, 16, 24, temp0, temp4, temp8, temp12) + VERTICAL_PASS(2, 10, 18, 26, temp1, temp5, temp9, temp13) + VERTICAL_PASS(4, 12, 20, 28, temp2, temp6, temp10, temp14) + VERTICAL_PASS(6, 14, 22, 30, temp3, temp7, temp11, temp15) + + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8), + [temp9]"=&r"(temp9), [temp10]"=&r"(temp10), [temp11]"=&r"(temp11), + [temp12]"=&r"(temp12), [temp13]"=&r"(temp13), [temp14]"=&r"(temp14), + [temp15]"=&r"(temp15), [temp16]"=&r"(temp16), [temp17]"=&r"(temp17), + [temp18]"=&r"(temp18), [temp19]"=&r"(temp19), [temp20]"=&r"(temp20) + : [args]"r"(args), [c2217]"r"(c2217), [c5352]"r"(c5352) + : "memory", "hi", "lo" + ); +} + +#undef VERTICAL_PASS +#undef HORIZONTAL_PASS + +#if !defined(WORK_AROUND_GCC) + +#define GET_SSE_INNER(A, B, C, D) \ + "lbu %[temp0], " #A "(%[a]) \n\t" \ + "lbu %[temp1], " #A "(%[b]) \n\t" \ + "lbu %[temp2], " #B "(%[a]) \n\t" \ + "lbu %[temp3], " #B "(%[b]) \n\t" \ + "lbu %[temp4], " #C "(%[a]) \n\t" \ + "lbu %[temp5], " #C "(%[b]) \n\t" \ + "lbu %[temp6], " #D "(%[a]) \n\t" \ + "lbu %[temp7], " #D "(%[b]) \n\t" \ + "subu %[temp0], %[temp0], %[temp1] \n\t" \ + "subu %[temp2], %[temp2], %[temp3] \n\t" \ + "subu %[temp4], %[temp4], %[temp5] \n\t" \ + "subu %[temp6], %[temp6], %[temp7] \n\t" \ + "madd %[temp0], %[temp0] \n\t" \ + "madd %[temp2], %[temp2] \n\t" \ + "madd %[temp4], %[temp4] \n\t" \ + "madd %[temp6], %[temp6] \n\t" + +#define GET_SSE(A, B, C, D) \ + GET_SSE_INNER(A, A + 1, A + 2, A + 3) \ + GET_SSE_INNER(B, B + 1, B + 2, B + 3) \ + GET_SSE_INNER(C, C + 1, C + 2, C + 3) \ + GET_SSE_INNER(D, D + 1, D + 2, D + 3) + +static int SSE16x16(const uint8_t* a, const uint8_t* b) { + int count; + int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; + + __asm__ volatile( + "mult $zero, $zero \n\t" + + GET_SSE( 0 * BPS, 4 + 0 * BPS, 8 + 0 * BPS, 12 + 0 * BPS) + GET_SSE( 1 * BPS, 4 + 1 * BPS, 8 + 1 * BPS, 12 + 1 * BPS) + GET_SSE( 2 * BPS, 4 + 2 * BPS, 8 + 2 * BPS, 12 + 2 * BPS) + GET_SSE( 3 * BPS, 4 + 3 * BPS, 8 + 3 * BPS, 12 + 3 * BPS) + GET_SSE( 4 * BPS, 4 + 4 * BPS, 8 + 4 * BPS, 12 + 4 * BPS) + GET_SSE( 5 * BPS, 4 + 5 * BPS, 8 + 5 * BPS, 12 + 5 * BPS) + GET_SSE( 6 * BPS, 4 + 6 * BPS, 8 + 6 * BPS, 12 + 6 * BPS) + GET_SSE( 7 * BPS, 4 + 7 * BPS, 8 + 7 * BPS, 12 + 7 * BPS) + GET_SSE( 8 * BPS, 4 + 8 * BPS, 8 + 8 * BPS, 12 + 8 * BPS) + GET_SSE( 9 * BPS, 4 + 9 * BPS, 8 + 9 * BPS, 12 + 9 * BPS) + GET_SSE(10 * BPS, 4 + 10 * BPS, 8 + 10 * BPS, 12 + 10 * BPS) + GET_SSE(11 * BPS, 4 + 11 * BPS, 8 + 11 * BPS, 12 + 11 * BPS) + GET_SSE(12 * BPS, 4 + 12 * BPS, 8 + 12 * BPS, 12 + 12 * BPS) + GET_SSE(13 * BPS, 4 + 13 * BPS, 8 + 13 * BPS, 12 + 13 * BPS) + GET_SSE(14 * BPS, 4 + 14 * BPS, 8 + 14 * BPS, 12 + 14 * BPS) + GET_SSE(15 * BPS, 4 + 15 * BPS, 8 + 15 * BPS, 12 + 15 * BPS) + + "mflo %[count] \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [count]"=&r"(count) + : [a]"r"(a), [b]"r"(b) + : "memory", "hi", "lo" + ); + return count; +} + +static int SSE16x8(const uint8_t* a, const uint8_t* b) { + int count; + int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; + + __asm__ volatile( + "mult $zero, $zero \n\t" + + GET_SSE( 0 * BPS, 4 + 0 * BPS, 8 + 0 * BPS, 12 + 0 * BPS) + GET_SSE( 1 * BPS, 4 + 1 * BPS, 8 + 1 * BPS, 12 + 1 * BPS) + GET_SSE( 2 * BPS, 4 + 2 * BPS, 8 + 2 * BPS, 12 + 2 * BPS) + GET_SSE( 3 * BPS, 4 + 3 * BPS, 8 + 3 * BPS, 12 + 3 * BPS) + GET_SSE( 4 * BPS, 4 + 4 * BPS, 8 + 4 * BPS, 12 + 4 * BPS) + GET_SSE( 5 * BPS, 4 + 5 * BPS, 8 + 5 * BPS, 12 + 5 * BPS) + GET_SSE( 6 * BPS, 4 + 6 * BPS, 8 + 6 * BPS, 12 + 6 * BPS) + GET_SSE( 7 * BPS, 4 + 7 * BPS, 8 + 7 * BPS, 12 + 7 * BPS) + + "mflo %[count] \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [count]"=&r"(count) + : [a]"r"(a), [b]"r"(b) + : "memory", "hi", "lo" + ); + return count; +} + +static int SSE8x8(const uint8_t* a, const uint8_t* b) { + int count; + int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; + + __asm__ volatile( + "mult $zero, $zero \n\t" + + GET_SSE(0 * BPS, 4 + 0 * BPS, 1 * BPS, 4 + 1 * BPS) + GET_SSE(2 * BPS, 4 + 2 * BPS, 3 * BPS, 4 + 3 * BPS) + GET_SSE(4 * BPS, 4 + 4 * BPS, 5 * BPS, 4 + 5 * BPS) + GET_SSE(6 * BPS, 4 + 6 * BPS, 7 * BPS, 4 + 7 * BPS) + + "mflo %[count] \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [count]"=&r"(count) + : [a]"r"(a), [b]"r"(b) + : "memory", "hi", "lo" + ); + return count; +} + +static int SSE4x4(const uint8_t* a, const uint8_t* b) { + int count; + int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; + + __asm__ volatile( + "mult $zero, $zero \n\t" + + GET_SSE(0 * BPS, 1 * BPS, 2 * BPS, 3 * BPS) + + "mflo %[count] \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [count]"=&r"(count) + : [a]"r"(a), [b]"r"(b) + : "memory", "hi", "lo" + ); + return count; +} + +#undef GET_SSE +#undef GET_SSE_INNER + +#endif // !WORK_AROUND_GCC + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8EncDspInitMIPS32(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitMIPS32(void) { + VP8ITransform = ITransform; + VP8FTransform = FTransform; + VP8EncQuantizeBlock = QuantizeBlock; + VP8EncQuantize2Blocks = Quantize2Blocks; + VP8TDisto4x4 = Disto4x4; + VP8TDisto16x16 = Disto16x16; +#if !defined(WORK_AROUND_GCC) + VP8SSE16x16 = SSE16x16; + VP8SSE8x8 = SSE8x8; + VP8SSE16x8 = SSE16x8; + VP8SSE4x4 = SSE4x4; +#endif +} + +#else // !WEBP_USE_MIPS32 + +WEBP_DSP_INIT_STUB(VP8EncDspInitMIPS32) + +#endif // WEBP_USE_MIPS32 diff --git a/thirdparty/libwebp/dsp/enc_mips_dsp_r2.c b/thirdparty/libwebp/dsp/enc_mips_dsp_r2.c new file mode 100644 index 0000000000..7ab96f6800 --- /dev/null +++ b/thirdparty/libwebp/dsp/enc_mips_dsp_r2.c @@ -0,0 +1,1510 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MIPS version of speed-critical encoding functions. +// +// Author(s): Darko Laus (darko.laus@imgtec.com) +// Mirko Raus (mirko.raus@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS_DSP_R2) + +#include "./mips_macro.h" +#include "../enc/cost.h" +#include "../enc/vp8enci.h" + +static const int kC1 = 20091 + (1 << 16); +static const int kC2 = 35468; + +// O - output +// I - input (macro doesn't change it) +#define ADD_SUB_HALVES_X4(O0, O1, O2, O3, O4, O5, O6, O7, \ + I0, I1, I2, I3, I4, I5, I6, I7) \ + "addq.ph %[" #O0 "], %[" #I0 "], %[" #I1 "] \n\t" \ + "subq.ph %[" #O1 "], %[" #I0 "], %[" #I1 "] \n\t" \ + "addq.ph %[" #O2 "], %[" #I2 "], %[" #I3 "] \n\t" \ + "subq.ph %[" #O3 "], %[" #I2 "], %[" #I3 "] \n\t" \ + "addq.ph %[" #O4 "], %[" #I4 "], %[" #I5 "] \n\t" \ + "subq.ph %[" #O5 "], %[" #I4 "], %[" #I5 "] \n\t" \ + "addq.ph %[" #O6 "], %[" #I6 "], %[" #I7 "] \n\t" \ + "subq.ph %[" #O7 "], %[" #I6 "], %[" #I7 "] \n\t" + +// IO - input/output +#define ABS_X8(IO0, IO1, IO2, IO3, IO4, IO5, IO6, IO7) \ + "absq_s.ph %[" #IO0 "], %[" #IO0 "] \n\t" \ + "absq_s.ph %[" #IO1 "], %[" #IO1 "] \n\t" \ + "absq_s.ph %[" #IO2 "], %[" #IO2 "] \n\t" \ + "absq_s.ph %[" #IO3 "], %[" #IO3 "] \n\t" \ + "absq_s.ph %[" #IO4 "], %[" #IO4 "] \n\t" \ + "absq_s.ph %[" #IO5 "], %[" #IO5 "] \n\t" \ + "absq_s.ph %[" #IO6 "], %[" #IO6 "] \n\t" \ + "absq_s.ph %[" #IO7 "], %[" #IO7 "] \n\t" + +// dpa.w.ph $ac0 temp0 ,temp1 +// $ac += temp0[31..16] * temp1[31..16] + temp0[15..0] * temp1[15..0] +// dpax.w.ph $ac0 temp0 ,temp1 +// $ac += temp0[31..16] * temp1[15..0] + temp0[15..0] * temp1[31..16] +// O - output +// I - input (macro doesn't change it) +#define MUL_HALF(O0, I0, I1, I2, I3, I4, I5, I6, I7, \ + I8, I9, I10, I11, I12, I13, I14, I15) \ + "mult $ac0, $zero, $zero \n\t" \ + "dpa.w.ph $ac0, %[" #I2 "], %[" #I0 "] \n\t" \ + "dpax.w.ph $ac0, %[" #I5 "], %[" #I6 "] \n\t" \ + "dpa.w.ph $ac0, %[" #I8 "], %[" #I9 "] \n\t" \ + "dpax.w.ph $ac0, %[" #I11 "], %[" #I4 "] \n\t" \ + "dpa.w.ph $ac0, %[" #I12 "], %[" #I7 "] \n\t" \ + "dpax.w.ph $ac0, %[" #I13 "], %[" #I1 "] \n\t" \ + "dpa.w.ph $ac0, %[" #I14 "], %[" #I3 "] \n\t" \ + "dpax.w.ph $ac0, %[" #I15 "], %[" #I10 "] \n\t" \ + "mflo %[" #O0 "], $ac0 \n\t" + +#define OUTPUT_EARLY_CLOBBER_REGS_17() \ + OUTPUT_EARLY_CLOBBER_REGS_10(), \ + [temp11]"=&r"(temp11), [temp12]"=&r"(temp12), [temp13]"=&r"(temp13), \ + [temp14]"=&r"(temp14), [temp15]"=&r"(temp15), [temp16]"=&r"(temp16), \ + [temp17]"=&r"(temp17) + +// macro for one horizontal pass in FTransform +// temp0..temp15 holds tmp[0]..tmp[15] +// A - offset in bytes to load from src and ref buffers +// TEMP0..TEMP3 - registers for corresponding tmp elements +#define HORIZONTAL_PASS(A, TEMP0, TEMP1, TEMP2, TEMP3) \ + "lw %[" #TEMP0 "], 0(%[args]) \n\t" \ + "lw %[" #TEMP1 "], 4(%[args]) \n\t" \ + "lw %[" #TEMP2 "], " XSTR(BPS) "*" #A "(%[" #TEMP0 "]) \n\t" \ + "lw %[" #TEMP3 "], " XSTR(BPS) "*" #A "(%[" #TEMP1 "]) \n\t" \ + "preceu.ph.qbl %[" #TEMP0 "], %[" #TEMP2 "] \n\t" \ + "preceu.ph.qbl %[" #TEMP1 "], %[" #TEMP3 "] \n\t" \ + "preceu.ph.qbr %[" #TEMP2 "], %[" #TEMP2 "] \n\t" \ + "preceu.ph.qbr %[" #TEMP3 "], %[" #TEMP3 "] \n\t" \ + "subq.ph %[" #TEMP0 "], %[" #TEMP0 "], %[" #TEMP1 "] \n\t" \ + "subq.ph %[" #TEMP2 "], %[" #TEMP2 "], %[" #TEMP3 "] \n\t" \ + "rotr %[" #TEMP0 "], %[" #TEMP0 "], 16 \n\t" \ + "addq.ph %[" #TEMP1 "], %[" #TEMP2 "], %[" #TEMP0 "] \n\t" \ + "subq.ph %[" #TEMP3 "], %[" #TEMP2 "], %[" #TEMP0 "] \n\t" \ + "seh %[" #TEMP0 "], %[" #TEMP1 "] \n\t" \ + "sra %[temp16], %[" #TEMP1 "], 16 \n\t" \ + "seh %[temp19], %[" #TEMP3 "] \n\t" \ + "sra %[" #TEMP3 "], %[" #TEMP3 "], 16 \n\t" \ + "subu %[" #TEMP2 "], %[" #TEMP0 "], %[temp16] \n\t" \ + "addu %[" #TEMP0 "], %[" #TEMP0 "], %[temp16] \n\t" \ + "mul %[temp17], %[temp19], %[c2217] \n\t" \ + "mul %[temp18], %[" #TEMP3 "], %[c5352] \n\t" \ + "mul %[" #TEMP1 "], %[temp19], %[c5352] \n\t" \ + "mul %[temp16], %[" #TEMP3 "], %[c2217] \n\t" \ + "sll %[" #TEMP2 "], %[" #TEMP2 "], 3 \n\t" \ + "sll %[" #TEMP0 "], %[" #TEMP0 "], 3 \n\t" \ + "subu %[" #TEMP3 "], %[temp17], %[temp18] \n\t" \ + "addu %[" #TEMP1 "], %[temp16], %[" #TEMP1 "] \n\t" \ + "addiu %[" #TEMP3 "], %[" #TEMP3 "], 937 \n\t" \ + "addiu %[" #TEMP1 "], %[" #TEMP1 "], 1812 \n\t" \ + "sra %[" #TEMP3 "], %[" #TEMP3 "], 9 \n\t" \ + "sra %[" #TEMP1 "], %[" #TEMP1 "], 9 \n\t" + +// macro for one vertical pass in FTransform +// temp0..temp15 holds tmp[0]..tmp[15] +// A..D - offsets in bytes to store to out buffer +// TEMP0, TEMP4, TEMP8 and TEMP12 - registers for corresponding tmp elements +#define VERTICAL_PASS(A, B, C, D, TEMP0, TEMP4, TEMP8, TEMP12) \ + "addu %[temp16], %[" #TEMP0 "], %[" #TEMP12 "] \n\t" \ + "subu %[temp19], %[" #TEMP0 "], %[" #TEMP12 "] \n\t" \ + "addu %[temp17], %[" #TEMP4 "], %[" #TEMP8 "] \n\t" \ + "subu %[temp18], %[" #TEMP4 "], %[" #TEMP8 "] \n\t" \ + "mul %[" #TEMP8 "], %[temp19], %[c2217] \n\t" \ + "mul %[" #TEMP12 "], %[temp18], %[c2217] \n\t" \ + "mul %[" #TEMP4 "], %[temp19], %[c5352] \n\t" \ + "mul %[temp18], %[temp18], %[c5352] \n\t" \ + "addiu %[temp16], %[temp16], 7 \n\t" \ + "addu %[" #TEMP0 "], %[temp16], %[temp17] \n\t" \ + "sra %[" #TEMP0 "], %[" #TEMP0 "], 4 \n\t" \ + "addu %[" #TEMP12 "], %[" #TEMP12 "], %[" #TEMP4 "] \n\t" \ + "subu %[" #TEMP4 "], %[temp16], %[temp17] \n\t" \ + "sra %[" #TEMP4 "], %[" #TEMP4 "], 4 \n\t" \ + "addiu %[" #TEMP8 "], %[" #TEMP8 "], 30000 \n\t" \ + "addiu %[" #TEMP12 "], %[" #TEMP12 "], 12000 \n\t" \ + "addiu %[" #TEMP8 "], %[" #TEMP8 "], 21000 \n\t" \ + "subu %[" #TEMP8 "], %[" #TEMP8 "], %[temp18] \n\t" \ + "sra %[" #TEMP12 "], %[" #TEMP12 "], 16 \n\t" \ + "sra %[" #TEMP8 "], %[" #TEMP8 "], 16 \n\t" \ + "addiu %[temp16], %[" #TEMP12 "], 1 \n\t" \ + "movn %[" #TEMP12 "], %[temp16], %[temp19] \n\t" \ + "sh %[" #TEMP0 "], " #A "(%[temp20]) \n\t" \ + "sh %[" #TEMP4 "], " #C "(%[temp20]) \n\t" \ + "sh %[" #TEMP8 "], " #D "(%[temp20]) \n\t" \ + "sh %[" #TEMP12 "], " #B "(%[temp20]) \n\t" + +static void FTransform(const uint8_t* src, const uint8_t* ref, int16_t* out) { + const int c2217 = 2217; + const int c5352 = 5352; + int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8; + int temp9, temp10, temp11, temp12, temp13, temp14, temp15, temp16; + int temp17, temp18, temp19, temp20; + const int* const args[3] = + { (const int*)src, (const int*)ref, (const int*)out }; + + __asm__ volatile ( + HORIZONTAL_PASS(0, temp0, temp1, temp2, temp3) + HORIZONTAL_PASS(1, temp4, temp5, temp6, temp7) + HORIZONTAL_PASS(2, temp8, temp9, temp10, temp11) + HORIZONTAL_PASS(3, temp12, temp13, temp14, temp15) + "lw %[temp20], 8(%[args]) \n\t" + VERTICAL_PASS(0, 8, 16, 24, temp0, temp4, temp8, temp12) + VERTICAL_PASS(2, 10, 18, 26, temp1, temp5, temp9, temp13) + VERTICAL_PASS(4, 12, 20, 28, temp2, temp6, temp10, temp14) + VERTICAL_PASS(6, 14, 22, 30, temp3, temp7, temp11, temp15) + OUTPUT_EARLY_CLOBBER_REGS_18(), + [temp0]"=&r"(temp0), [temp19]"=&r"(temp19), [temp20]"=&r"(temp20) + : [args]"r"(args), [c2217]"r"(c2217), [c5352]"r"(c5352) + : "memory", "hi", "lo" + ); +} + +#undef VERTICAL_PASS +#undef HORIZONTAL_PASS + +static WEBP_INLINE void ITransformOne(const uint8_t* ref, const int16_t* in, + uint8_t* dst) { + int temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9; + int temp10, temp11, temp12, temp13, temp14, temp15, temp16, temp17, temp18; + + __asm__ volatile ( + "ulw %[temp1], 0(%[in]) \n\t" + "ulw %[temp2], 16(%[in]) \n\t" + LOAD_IN_X2(temp5, temp6, 24, 26) + ADD_SUB_HALVES(temp3, temp4, temp1, temp2) + LOAD_IN_X2(temp1, temp2, 8, 10) + MUL_SHIFT_SUM(temp7, temp8, temp9, temp10, temp11, temp12, temp13, temp14, + temp10, temp8, temp9, temp7, temp1, temp2, temp5, temp6, + temp13, temp11, temp14, temp12) + INSERT_HALF_X2(temp8, temp7, temp10, temp9) + "ulw %[temp17], 4(%[in]) \n\t" + "ulw %[temp18], 20(%[in]) \n\t" + ADD_SUB_HALVES(temp1, temp2, temp3, temp8) + ADD_SUB_HALVES(temp5, temp6, temp4, temp7) + ADD_SUB_HALVES(temp7, temp8, temp17, temp18) + LOAD_IN_X2(temp17, temp18, 12, 14) + LOAD_IN_X2(temp9, temp10, 28, 30) + MUL_SHIFT_SUM(temp11, temp12, temp13, temp14, temp15, temp16, temp4, temp17, + temp12, temp14, temp11, temp13, temp17, temp18, temp9, temp10, + temp15, temp4, temp16, temp17) + INSERT_HALF_X2(temp11, temp12, temp13, temp14) + ADD_SUB_HALVES(temp17, temp8, temp8, temp11) + ADD_SUB_HALVES(temp3, temp4, temp7, temp12) + + // horizontal + SRA_16(temp9, temp10, temp11, temp12, temp1, temp2, temp5, temp6) + INSERT_HALF_X2(temp1, temp6, temp5, temp2) + SRA_16(temp13, temp14, temp15, temp16, temp3, temp4, temp17, temp8) + "repl.ph %[temp2], 0x4 \n\t" + INSERT_HALF_X2(temp3, temp8, temp17, temp4) + "addq.ph %[temp1], %[temp1], %[temp2] \n\t" + "addq.ph %[temp6], %[temp6], %[temp2] \n\t" + ADD_SUB_HALVES(temp2, temp4, temp1, temp3) + ADD_SUB_HALVES(temp5, temp7, temp6, temp8) + MUL_SHIFT_SUM(temp1, temp3, temp6, temp8, temp9, temp13, temp17, temp18, + temp3, temp13, temp1, temp9, temp9, temp13, temp11, temp15, + temp6, temp17, temp8, temp18) + MUL_SHIFT_SUM(temp6, temp8, temp18, temp17, temp11, temp15, temp12, temp16, + temp8, temp15, temp6, temp11, temp12, temp16, temp10, temp14, + temp18, temp12, temp17, temp16) + INSERT_HALF_X2(temp1, temp3, temp9, temp13) + INSERT_HALF_X2(temp6, temp8, temp11, temp15) + SHIFT_R_SUM_X2(temp9, temp10, temp11, temp12, temp13, temp14, temp15, + temp16, temp2, temp4, temp5, temp7, temp3, temp1, temp8, + temp6) + PACK_2_HALVES_TO_WORD(temp1, temp2, temp3, temp4, temp9, temp12, temp13, + temp16, temp11, temp10, temp15, temp14) + LOAD_WITH_OFFSET_X4(temp10, temp11, temp14, temp15, ref, + 0, 0, 0, 0, + 0, 1, 2, 3, + BPS) + CONVERT_2_BYTES_TO_HALF(temp5, temp6, temp7, temp8, temp17, temp18, temp10, + temp11, temp10, temp11, temp14, temp15) + STORE_SAT_SUM_X2(temp5, temp6, temp7, temp8, temp17, temp18, temp10, temp11, + temp9, temp12, temp1, temp2, temp13, temp16, temp3, temp4, + dst, 0, 1, 2, 3, BPS) + + OUTPUT_EARLY_CLOBBER_REGS_18() + : [dst]"r"(dst), [in]"r"(in), [kC1]"r"(kC1), [kC2]"r"(kC2), [ref]"r"(ref) + : "memory", "hi", "lo" + ); +} + +static void ITransform(const uint8_t* ref, const int16_t* in, uint8_t* dst, + int do_two) { + ITransformOne(ref, in, dst); + if (do_two) { + ITransformOne(ref + 4, in + 16, dst + 4); + } +} + +static int Disto4x4(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + int temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9; + int temp10, temp11, temp12, temp13, temp14, temp15, temp16, temp17; + + __asm__ volatile ( + LOAD_WITH_OFFSET_X4(temp1, temp2, temp3, temp4, a, + 0, 0, 0, 0, + 0, 1, 2, 3, + BPS) + CONVERT_2_BYTES_TO_HALF(temp5, temp6, temp7, temp8, temp9,temp10, temp11, + temp12, temp1, temp2, temp3, temp4) + ADD_SUB_HALVES_X4(temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, + temp5, temp6, temp7, temp8, temp9, temp10, temp11, temp12) + PACK_2_HALVES_TO_WORD(temp9, temp10, temp11, temp12, temp1, temp3, temp5, + temp7, temp2, temp4, temp6, temp8) + ADD_SUB_HALVES_X4(temp2, temp4, temp6, temp8, temp9, temp1, temp3, temp10, + temp1, temp9, temp3, temp10, temp5, temp11, temp7, temp12) + ADD_SUB_HALVES_X4(temp5, temp11, temp7, temp2, temp9, temp3, temp6, temp12, + temp2, temp9, temp6, temp3, temp4, temp1, temp8, temp10) + ADD_SUB_HALVES_X4(temp1, temp4, temp10, temp8, temp7, temp11, temp5, temp2, + temp5, temp7, temp11, temp2, temp9, temp6, temp3, temp12) + ABS_X8(temp1, temp4, temp10, temp8, temp7, temp11, temp5, temp2) + LOAD_WITH_OFFSET_X4(temp3, temp6, temp9, temp12, w, + 0, 4, 8, 12, + 0, 0, 0, 0, + 0) + LOAD_WITH_OFFSET_X4(temp13, temp14, temp15, temp16, w, + 0, 4, 8, 12, + 1, 1, 1, 1, + 16) + MUL_HALF(temp17, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, + temp9, temp10, temp11, temp12, temp13, temp14, temp15, temp16) + LOAD_WITH_OFFSET_X4(temp1, temp2, temp3, temp4, b, + 0, 0, 0, 0, + 0, 1, 2, 3, + BPS) + CONVERT_2_BYTES_TO_HALF(temp5,temp6, temp7, temp8, temp9,temp10, temp11, + temp12, temp1, temp2, temp3, temp4) + ADD_SUB_HALVES_X4(temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, + temp5, temp6, temp7, temp8, temp9, temp10, temp11, temp12) + PACK_2_HALVES_TO_WORD(temp9, temp10, temp11, temp12, temp1, temp3, temp5, + temp7, temp2, temp4, temp6, temp8) + ADD_SUB_HALVES_X4(temp2, temp4, temp6, temp8, temp9, temp1, temp3, temp10, + temp1, temp9, temp3, temp10, temp5, temp11, temp7, temp12) + ADD_SUB_HALVES_X4(temp5, temp11, temp7, temp2, temp9, temp3, temp6, temp12, + temp2, temp9, temp6, temp3, temp4, temp1, temp8, temp10) + ADD_SUB_HALVES_X4(temp1, temp4, temp10, temp8, temp7, temp11, temp5, temp2, + temp5, temp7, temp11, temp2, temp9, temp6, temp3, temp12) + ABS_X8(temp1, temp4, temp10, temp8, temp7, temp11, temp5, temp2) + LOAD_WITH_OFFSET_X4(temp3, temp6, temp9, temp12, w, + 0, 4, 8, 12, + 0, 0, 0, 0, + 0) + LOAD_WITH_OFFSET_X4(temp13, temp14, temp15, temp16, w, + 0, 4, 8, 12, + 1, 1, 1, 1, + 16) + MUL_HALF(temp3, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, + temp9, temp10, temp11, temp12, temp13, temp14, temp15, temp16) + OUTPUT_EARLY_CLOBBER_REGS_17() + : [a]"r"(a), [b]"r"(b), [w]"r"(w) + : "memory", "hi", "lo" + ); + return abs(temp3 - temp17) >> 5; +} + +static int Disto16x16(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + int D = 0; + int x, y; + for (y = 0; y < 16 * BPS; y += 4 * BPS) { + for (x = 0; x < 16; x += 4) { + D += Disto4x4(a + x + y, b + x + y, w); + } + } + return D; +} + +//------------------------------------------------------------------------------ +// Intra predictions + +#define FILL_PART(J, SIZE) \ + "usw %[value], 0+" #J "*" XSTR(BPS) "(%[dst]) \n\t" \ + "usw %[value], 4+" #J "*" XSTR(BPS) "(%[dst]) \n\t" \ + ".if " #SIZE " == 16 \n\t" \ + "usw %[value], 8+" #J "*" XSTR(BPS) "(%[dst]) \n\t" \ + "usw %[value], 12+" #J "*" XSTR(BPS) "(%[dst]) \n\t" \ + ".endif \n\t" + +#define FILL_8_OR_16(DST, VALUE, SIZE) do { \ + int value = (VALUE); \ + __asm__ volatile ( \ + "replv.qb %[value], %[value] \n\t" \ + FILL_PART( 0, SIZE) \ + FILL_PART( 1, SIZE) \ + FILL_PART( 2, SIZE) \ + FILL_PART( 3, SIZE) \ + FILL_PART( 4, SIZE) \ + FILL_PART( 5, SIZE) \ + FILL_PART( 6, SIZE) \ + FILL_PART( 7, SIZE) \ + ".if " #SIZE " == 16 \n\t" \ + FILL_PART( 8, 16) \ + FILL_PART( 9, 16) \ + FILL_PART(10, 16) \ + FILL_PART(11, 16) \ + FILL_PART(12, 16) \ + FILL_PART(13, 16) \ + FILL_PART(14, 16) \ + FILL_PART(15, 16) \ + ".endif \n\t" \ + : [value]"+&r"(value) \ + : [dst]"r"((DST)) \ + : "memory" \ + ); \ +} while (0) + +#define VERTICAL_PRED(DST, TOP, SIZE) \ +static WEBP_INLINE void VerticalPred##SIZE(uint8_t* (DST), \ + const uint8_t* (TOP)) { \ + int j; \ + if ((TOP)) { \ + for (j = 0; j < (SIZE); ++j) memcpy((DST) + j * BPS, (TOP), (SIZE)); \ + } else { \ + FILL_8_OR_16((DST), 127, (SIZE)); \ + } \ +} + +VERTICAL_PRED(dst, top, 8) +VERTICAL_PRED(dst, top, 16) + +#undef VERTICAL_PRED + +#define HORIZONTAL_PRED(DST, LEFT, SIZE) \ +static WEBP_INLINE void HorizontalPred##SIZE(uint8_t* (DST), \ + const uint8_t* (LEFT)) { \ + if (LEFT) { \ + int j; \ + for (j = 0; j < (SIZE); ++j) { \ + memset((DST) + j * BPS, (LEFT)[j], (SIZE)); \ + } \ + } else { \ + FILL_8_OR_16((DST), 129, (SIZE)); \ + } \ +} + +HORIZONTAL_PRED(dst, left, 8) +HORIZONTAL_PRED(dst, left, 16) + +#undef HORIZONTAL_PRED + +#define CLIPPING() \ + "preceu.ph.qbl %[temp2], %[temp0] \n\t" \ + "preceu.ph.qbr %[temp0], %[temp0] \n\t" \ + "preceu.ph.qbl %[temp3], %[temp1] \n\t" \ + "preceu.ph.qbr %[temp1], %[temp1] \n\t" \ + "addu.ph %[temp2], %[temp2], %[leftY_1] \n\t" \ + "addu.ph %[temp0], %[temp0], %[leftY_1] \n\t" \ + "addu.ph %[temp3], %[temp3], %[leftY_1] \n\t" \ + "addu.ph %[temp1], %[temp1], %[leftY_1] \n\t" \ + "shll_s.ph %[temp2], %[temp2], 7 \n\t" \ + "shll_s.ph %[temp0], %[temp0], 7 \n\t" \ + "shll_s.ph %[temp3], %[temp3], 7 \n\t" \ + "shll_s.ph %[temp1], %[temp1], 7 \n\t" \ + "precrqu_s.qb.ph %[temp0], %[temp2], %[temp0] \n\t" \ + "precrqu_s.qb.ph %[temp1], %[temp3], %[temp1] \n\t" + +#define CLIP_8B_TO_DST(DST, LEFT, TOP, SIZE) do { \ + int leftY_1 = ((int)(LEFT)[y] << 16) + (LEFT)[y]; \ + int temp0, temp1, temp2, temp3; \ + __asm__ volatile ( \ + "replv.ph %[leftY_1], %[leftY_1] \n\t" \ + "ulw %[temp0], 0(%[top]) \n\t" \ + "ulw %[temp1], 4(%[top]) \n\t" \ + "subu.ph %[leftY_1], %[leftY_1], %[left_1] \n\t" \ + CLIPPING() \ + "usw %[temp0], 0(%[dst]) \n\t" \ + "usw %[temp1], 4(%[dst]) \n\t" \ + ".if " #SIZE " == 16 \n\t" \ + "ulw %[temp0], 8(%[top]) \n\t" \ + "ulw %[temp1], 12(%[top]) \n\t" \ + CLIPPING() \ + "usw %[temp0], 8(%[dst]) \n\t" \ + "usw %[temp1], 12(%[dst]) \n\t" \ + ".endif \n\t" \ + : [leftY_1]"+&r"(leftY_1), [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), \ + [temp2]"=&r"(temp2), [temp3]"=&r"(temp3) \ + : [left_1]"r"(left_1), [top]"r"((TOP)), [dst]"r"((DST)) \ + : "memory" \ + ); \ +} while (0) + +#define CLIP_TO_DST(DST, LEFT, TOP, SIZE) do { \ + int y; \ + const int left_1 = ((int)(LEFT)[-1] << 16) + (LEFT)[-1]; \ + for (y = 0; y < (SIZE); ++y) { \ + CLIP_8B_TO_DST((DST), (LEFT), (TOP), (SIZE)); \ + (DST) += BPS; \ + } \ +} while (0) + +#define TRUE_MOTION(DST, LEFT, TOP, SIZE) \ +static WEBP_INLINE void TrueMotion##SIZE(uint8_t* (DST), const uint8_t* (LEFT),\ + const uint8_t* (TOP)) { \ + if ((LEFT) != NULL) { \ + if ((TOP) != NULL) { \ + CLIP_TO_DST((DST), (LEFT), (TOP), (SIZE)); \ + } else { \ + HorizontalPred##SIZE((DST), (LEFT)); \ + } \ + } else { \ + /* true motion without left samples (hence: with default 129 value) */ \ + /* is equivalent to VE prediction where you just copy the top samples. */ \ + /* Note that if top samples are not available, the default value is */ \ + /* then 129, and not 127 as in the VerticalPred case. */ \ + if ((TOP) != NULL) { \ + VerticalPred##SIZE((DST), (TOP)); \ + } else { \ + FILL_8_OR_16((DST), 129, (SIZE)); \ + } \ + } \ +} + +TRUE_MOTION(dst, left, top, 8) +TRUE_MOTION(dst, left, top, 16) + +#undef TRUE_MOTION +#undef CLIP_TO_DST +#undef CLIP_8B_TO_DST +#undef CLIPPING + +static WEBP_INLINE void DCMode16(uint8_t* dst, const uint8_t* left, + const uint8_t* top) { + int DC, DC1; + int temp0, temp1, temp2, temp3; + + __asm__ volatile( + "beqz %[top], 2f \n\t" + LOAD_WITH_OFFSET_X4(temp0, temp1, temp2, temp3, top, + 0, 4, 8, 12, + 0, 0, 0, 0, + 0) + "raddu.w.qb %[temp0], %[temp0] \n\t" + "raddu.w.qb %[temp1], %[temp1] \n\t" + "raddu.w.qb %[temp2], %[temp2] \n\t" + "raddu.w.qb %[temp3], %[temp3] \n\t" + "addu %[temp0], %[temp0], %[temp1] \n\t" + "addu %[temp2], %[temp2], %[temp3] \n\t" + "addu %[DC], %[temp0], %[temp2] \n\t" + "move %[DC1], %[DC] \n\t" + "beqz %[left], 1f \n\t" + LOAD_WITH_OFFSET_X4(temp0, temp1, temp2, temp3, left, + 0, 4, 8, 12, + 0, 0, 0, 0, + 0) + "raddu.w.qb %[temp0], %[temp0] \n\t" + "raddu.w.qb %[temp1], %[temp1] \n\t" + "raddu.w.qb %[temp2], %[temp2] \n\t" + "raddu.w.qb %[temp3], %[temp3] \n\t" + "addu %[temp0], %[temp0], %[temp1] \n\t" + "addu %[temp2], %[temp2], %[temp3] \n\t" + "addu %[DC1], %[temp0], %[temp2] \n\t" + "1: \n\t" + "addu %[DC], %[DC], %[DC1] \n\t" + "j 3f \n\t" + "2: \n\t" + "beqz %[left], 4f \n\t" + LOAD_WITH_OFFSET_X4(temp0, temp1, temp2, temp3, left, + 0, 4, 8, 12, + 0, 0, 0, 0, + 0) + "raddu.w.qb %[temp0], %[temp0] \n\t" + "raddu.w.qb %[temp1], %[temp1] \n\t" + "raddu.w.qb %[temp2], %[temp2] \n\t" + "raddu.w.qb %[temp3], %[temp3] \n\t" + "addu %[temp0], %[temp0], %[temp1] \n\t" + "addu %[temp2], %[temp2], %[temp3] \n\t" + "addu %[DC], %[temp0], %[temp2] \n\t" + "addu %[DC], %[DC], %[DC] \n\t" + "3: \n\t" + "shra_r.w %[DC], %[DC], 5 \n\t" + "j 5f \n\t" + "4: \n\t" + "li %[DC], 0x80 \n\t" + "5: \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [DC]"=&r"(DC), + [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), [DC1]"=&r"(DC1) + : [left]"r"(left), [top]"r"(top) + : "memory" + ); + + FILL_8_OR_16(dst, DC, 16); +} + +static WEBP_INLINE void DCMode8(uint8_t* dst, const uint8_t* left, + const uint8_t* top) { + int DC, DC1; + int temp0, temp1, temp2, temp3; + + __asm__ volatile( + "beqz %[top], 2f \n\t" + "ulw %[temp0], 0(%[top]) \n\t" + "ulw %[temp1], 4(%[top]) \n\t" + "raddu.w.qb %[temp0], %[temp0] \n\t" + "raddu.w.qb %[temp1], %[temp1] \n\t" + "addu %[DC], %[temp0], %[temp1] \n\t" + "move %[DC1], %[DC] \n\t" + "beqz %[left], 1f \n\t" + "ulw %[temp2], 0(%[left]) \n\t" + "ulw %[temp3], 4(%[left]) \n\t" + "raddu.w.qb %[temp2], %[temp2] \n\t" + "raddu.w.qb %[temp3], %[temp3] \n\t" + "addu %[DC1], %[temp2], %[temp3] \n\t" + "1: \n\t" + "addu %[DC], %[DC], %[DC1] \n\t" + "j 3f \n\t" + "2: \n\t" + "beqz %[left], 4f \n\t" + "ulw %[temp2], 0(%[left]) \n\t" + "ulw %[temp3], 4(%[left]) \n\t" + "raddu.w.qb %[temp2], %[temp2] \n\t" + "raddu.w.qb %[temp3], %[temp3] \n\t" + "addu %[DC], %[temp2], %[temp3] \n\t" + "addu %[DC], %[DC], %[DC] \n\t" + "3: \n\t" + "shra_r.w %[DC], %[DC], 4 \n\t" + "j 5f \n\t" + "4: \n\t" + "li %[DC], 0x80 \n\t" + "5: \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [DC]"=&r"(DC), + [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), [DC1]"=&r"(DC1) + : [left]"r"(left), [top]"r"(top) + : "memory" + ); + + FILL_8_OR_16(dst, DC, 8); +} + +static void DC4(uint8_t* dst, const uint8_t* top) { + int temp0, temp1; + __asm__ volatile( + "ulw %[temp0], 0(%[top]) \n\t" + "ulw %[temp1], -5(%[top]) \n\t" + "raddu.w.qb %[temp0], %[temp0] \n\t" + "raddu.w.qb %[temp1], %[temp1] \n\t" + "addu %[temp0], %[temp0], %[temp1] \n\t" + "addiu %[temp0], %[temp0], 4 \n\t" + "srl %[temp0], %[temp0], 3 \n\t" + "replv.qb %[temp0], %[temp0] \n\t" + "usw %[temp0], 0*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp0], 1*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp0], 2*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp0], 3*" XSTR(BPS) "(%[dst]) \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1) + : [top]"r"(top), [dst]"r"(dst) + : "memory" + ); +} + +static void TM4(uint8_t* dst, const uint8_t* top) { + int a10, a32, temp0, temp1, temp2, temp3, temp4, temp5; + const int c35 = 0xff00ff; + __asm__ volatile ( + "lbu %[temp1], 0(%[top]) \n\t" + "lbu %[a10], 1(%[top]) \n\t" + "lbu %[temp2], 2(%[top]) \n\t" + "lbu %[a32], 3(%[top]) \n\t" + "ulw %[temp0], -5(%[top]) \n\t" + "lbu %[temp4], -1(%[top]) \n\t" + "append %[a10], %[temp1], 16 \n\t" + "append %[a32], %[temp2], 16 \n\t" + "replv.ph %[temp4], %[temp4] \n\t" + "shrl.ph %[temp1], %[temp0], 8 \n\t" + "and %[temp0], %[temp0], %[c35] \n\t" + "subu.ph %[temp1], %[temp1], %[temp4] \n\t" + "subu.ph %[temp0], %[temp0], %[temp4] \n\t" + "srl %[temp2], %[temp1], 16 \n\t" + "srl %[temp3], %[temp0], 16 \n\t" + "replv.ph %[temp2], %[temp2] \n\t" + "replv.ph %[temp3], %[temp3] \n\t" + "replv.ph %[temp4], %[temp1] \n\t" + "replv.ph %[temp5], %[temp0] \n\t" + "addu.ph %[temp0], %[temp3], %[a10] \n\t" + "addu.ph %[temp1], %[temp3], %[a32] \n\t" + "addu.ph %[temp3], %[temp2], %[a10] \n\t" + "addu.ph %[temp2], %[temp2], %[a32] \n\t" + "shll_s.ph %[temp0], %[temp0], 7 \n\t" + "shll_s.ph %[temp1], %[temp1], 7 \n\t" + "shll_s.ph %[temp3], %[temp3], 7 \n\t" + "shll_s.ph %[temp2], %[temp2], 7 \n\t" + "precrqu_s.qb.ph %[temp0], %[temp1], %[temp0] \n\t" + "precrqu_s.qb.ph %[temp1], %[temp2], %[temp3] \n\t" + "addu.ph %[temp2], %[temp5], %[a10] \n\t" + "addu.ph %[temp3], %[temp5], %[a32] \n\t" + "addu.ph %[temp5], %[temp4], %[a10] \n\t" + "addu.ph %[temp4], %[temp4], %[a32] \n\t" + "shll_s.ph %[temp2], %[temp2], 7 \n\t" + "shll_s.ph %[temp3], %[temp3], 7 \n\t" + "shll_s.ph %[temp4], %[temp4], 7 \n\t" + "shll_s.ph %[temp5], %[temp5], 7 \n\t" + "precrqu_s.qb.ph %[temp2], %[temp3], %[temp2] \n\t" + "precrqu_s.qb.ph %[temp3], %[temp4], %[temp5] \n\t" + "usw %[temp1], 0*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp0], 1*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp3], 2*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp2], 3*" XSTR(BPS) "(%[dst]) \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [a10]"=&r"(a10), [a32]"=&r"(a32) + : [c35]"r"(c35), [top]"r"(top), [dst]"r"(dst) + : "memory" + ); +} + +static void VE4(uint8_t* dst, const uint8_t* top) { + int temp0, temp1, temp2, temp3, temp4, temp5, temp6; + __asm__ volatile( + "ulw %[temp0], -1(%[top]) \n\t" + "ulh %[temp1], 3(%[top]) \n\t" + "preceu.ph.qbr %[temp2], %[temp0] \n\t" + "preceu.ph.qbl %[temp3], %[temp0] \n\t" + "preceu.ph.qbr %[temp4], %[temp1] \n\t" + "packrl.ph %[temp5], %[temp3], %[temp2] \n\t" + "packrl.ph %[temp6], %[temp4], %[temp3] \n\t" + "shll.ph %[temp5], %[temp5], 1 \n\t" + "shll.ph %[temp6], %[temp6], 1 \n\t" + "addq.ph %[temp2], %[temp5], %[temp2] \n\t" + "addq.ph %[temp6], %[temp6], %[temp4] \n\t" + "addq.ph %[temp2], %[temp2], %[temp3] \n\t" + "addq.ph %[temp6], %[temp6], %[temp3] \n\t" + "shra_r.ph %[temp2], %[temp2], 2 \n\t" + "shra_r.ph %[temp6], %[temp6], 2 \n\t" + "precr.qb.ph %[temp4], %[temp6], %[temp2] \n\t" + "usw %[temp4], 0*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp4], 1*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp4], 2*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp4], 3*" XSTR(BPS) "(%[dst]) \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6) + : [top]"r"(top), [dst]"r"(dst) + : "memory" + ); +} + +static void HE4(uint8_t* dst, const uint8_t* top) { + int temp0, temp1, temp2, temp3, temp4, temp5, temp6; + __asm__ volatile( + "ulw %[temp0], -4(%[top]) \n\t" + "lbu %[temp1], -5(%[top]) \n\t" + "preceu.ph.qbr %[temp2], %[temp0] \n\t" + "preceu.ph.qbl %[temp3], %[temp0] \n\t" + "replv.ph %[temp4], %[temp1] \n\t" + "packrl.ph %[temp5], %[temp3], %[temp2] \n\t" + "packrl.ph %[temp6], %[temp2], %[temp4] \n\t" + "shll.ph %[temp5], %[temp5], 1 \n\t" + "shll.ph %[temp6], %[temp6], 1 \n\t" + "addq.ph %[temp3], %[temp3], %[temp5] \n\t" + "addq.ph %[temp3], %[temp3], %[temp2] \n\t" + "addq.ph %[temp2], %[temp2], %[temp6] \n\t" + "addq.ph %[temp2], %[temp2], %[temp4] \n\t" + "shra_r.ph %[temp3], %[temp3], 2 \n\t" + "shra_r.ph %[temp2], %[temp2], 2 \n\t" + "replv.qb %[temp0], %[temp3] \n\t" + "replv.qb %[temp1], %[temp2] \n\t" + "srl %[temp3], %[temp3], 16 \n\t" + "srl %[temp2], %[temp2], 16 \n\t" + "replv.qb %[temp3], %[temp3] \n\t" + "replv.qb %[temp2], %[temp2] \n\t" + "usw %[temp3], 0*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp0], 1*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp2], 2*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp1], 3*" XSTR(BPS) "(%[dst]) \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6) + : [top]"r"(top), [dst]"r"(dst) + : "memory" + ); +} + +static void RD4(uint8_t* dst, const uint8_t* top) { + int temp0, temp1, temp2, temp3, temp4, temp5; + int temp6, temp7, temp8, temp9, temp10, temp11; + __asm__ volatile( + "ulw %[temp0], -5(%[top]) \n\t" + "ulw %[temp1], -1(%[top]) \n\t" + "preceu.ph.qbl %[temp2], %[temp0] \n\t" + "preceu.ph.qbr %[temp3], %[temp0] \n\t" + "preceu.ph.qbr %[temp4], %[temp1] \n\t" + "preceu.ph.qbl %[temp5], %[temp1] \n\t" + "packrl.ph %[temp6], %[temp2], %[temp3] \n\t" + "packrl.ph %[temp7], %[temp4], %[temp2] \n\t" + "packrl.ph %[temp8], %[temp5], %[temp4] \n\t" + "shll.ph %[temp6], %[temp6], 1 \n\t" + "addq.ph %[temp9], %[temp2], %[temp6] \n\t" + "shll.ph %[temp7], %[temp7], 1 \n\t" + "addq.ph %[temp9], %[temp9], %[temp3] \n\t" + "shll.ph %[temp8], %[temp8], 1 \n\t" + "shra_r.ph %[temp9], %[temp9], 2 \n\t" + "addq.ph %[temp10], %[temp4], %[temp7] \n\t" + "addq.ph %[temp11], %[temp5], %[temp8] \n\t" + "addq.ph %[temp10], %[temp10], %[temp2] \n\t" + "addq.ph %[temp11], %[temp11], %[temp4] \n\t" + "shra_r.ph %[temp10], %[temp10], 2 \n\t" + "shra_r.ph %[temp11], %[temp11], 2 \n\t" + "lbu %[temp0], 3(%[top]) \n\t" + "lbu %[temp1], 2(%[top]) \n\t" + "lbu %[temp2], 1(%[top]) \n\t" + "sll %[temp1], %[temp1], 1 \n\t" + "addu %[temp0], %[temp0], %[temp1] \n\t" + "addu %[temp0], %[temp0], %[temp2] \n\t" + "precr.qb.ph %[temp9], %[temp10], %[temp9] \n\t" + "shra_r.w %[temp0], %[temp0], 2 \n\t" + "precr.qb.ph %[temp10], %[temp11], %[temp10] \n\t" + "usw %[temp9], 3*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp10], 1*" XSTR(BPS) "(%[dst]) \n\t" + "prepend %[temp9], %[temp11], 8 \n\t" + "prepend %[temp10], %[temp0], 8 \n\t" + "usw %[temp9], 2*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp10], 0*" XSTR(BPS) "(%[dst]) \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8), + [temp9]"=&r"(temp9), [temp10]"=&r"(temp10), [temp11]"=&r"(temp11) + : [top]"r"(top), [dst]"r"(dst) + : "memory" + ); +} + +static void VR4(uint8_t* dst, const uint8_t* top) { + int temp0, temp1, temp2, temp3, temp4; + int temp5, temp6, temp7, temp8, temp9; + __asm__ volatile ( + "ulw %[temp0], -4(%[top]) \n\t" + "ulw %[temp1], 0(%[top]) \n\t" + "preceu.ph.qbl %[temp2], %[temp0] \n\t" + "preceu.ph.qbr %[temp0], %[temp0] \n\t" + "preceu.ph.qbla %[temp3], %[temp1] \n\t" + "preceu.ph.qbra %[temp1], %[temp1] \n\t" + "packrl.ph %[temp7], %[temp3], %[temp2] \n\t" + "addqh_r.ph %[temp4], %[temp1], %[temp3] \n\t" + "move %[temp6], %[temp1] \n\t" + "append %[temp1], %[temp2], 16 \n\t" + "shll.ph %[temp9], %[temp6], 1 \n\t" + "addqh_r.ph %[temp5], %[temp7], %[temp6] \n\t" + "shll.ph %[temp8], %[temp7], 1 \n\t" + "addu.ph %[temp3], %[temp7], %[temp3] \n\t" + "addu.ph %[temp1], %[temp1], %[temp6] \n\t" + "packrl.ph %[temp7], %[temp2], %[temp0] \n\t" + "addu.ph %[temp6], %[temp0], %[temp2] \n\t" + "addu.ph %[temp3], %[temp3], %[temp9] \n\t" + "addu.ph %[temp1], %[temp1], %[temp8] \n\t" + "shll.ph %[temp7], %[temp7], 1 \n\t" + "shra_r.ph %[temp3], %[temp3], 2 \n\t" + "shra_r.ph %[temp1], %[temp1], 2 \n\t" + "addu.ph %[temp6], %[temp6], %[temp7] \n\t" + "shra_r.ph %[temp6], %[temp6], 2 \n\t" + "precrq.ph.w %[temp8], %[temp4], %[temp5] \n\t" + "append %[temp4], %[temp5], 16 \n\t" + "precrq.ph.w %[temp2], %[temp3], %[temp1] \n\t" + "append %[temp3], %[temp1], 16 \n\t" + "precr.qb.ph %[temp8], %[temp8], %[temp4] \n\t" + "precr.qb.ph %[temp3], %[temp2], %[temp3] \n\t" + "usw %[temp8], 0*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp3], 1*" XSTR(BPS) "(%[dst]) \n\t" + "append %[temp3], %[temp6], 8 \n\t" + "srl %[temp6], %[temp6], 16 \n\t" + "append %[temp8], %[temp6], 8 \n\t" + "usw %[temp3], 3*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp8], 2*" XSTR(BPS) "(%[dst]) \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8), + [temp9]"=&r"(temp9) + : [top]"r"(top), [dst]"r"(dst) + : "memory" + ); +} + +static void LD4(uint8_t* dst, const uint8_t* top) { + int temp0, temp1, temp2, temp3, temp4, temp5; + int temp6, temp7, temp8, temp9, temp10, temp11; + __asm__ volatile( + "ulw %[temp0], 0(%[top]) \n\t" + "ulw %[temp1], 4(%[top]) \n\t" + "preceu.ph.qbl %[temp2], %[temp0] \n\t" + "preceu.ph.qbr %[temp3], %[temp0] \n\t" + "preceu.ph.qbr %[temp4], %[temp1] \n\t" + "preceu.ph.qbl %[temp5], %[temp1] \n\t" + "packrl.ph %[temp6], %[temp2], %[temp3] \n\t" + "packrl.ph %[temp7], %[temp4], %[temp2] \n\t" + "packrl.ph %[temp8], %[temp5], %[temp4] \n\t" + "shll.ph %[temp6], %[temp6], 1 \n\t" + "addq.ph %[temp9], %[temp2], %[temp6] \n\t" + "shll.ph %[temp7], %[temp7], 1 \n\t" + "addq.ph %[temp9], %[temp9], %[temp3] \n\t" + "shll.ph %[temp8], %[temp8], 1 \n\t" + "shra_r.ph %[temp9], %[temp9], 2 \n\t" + "addq.ph %[temp10], %[temp4], %[temp7] \n\t" + "addq.ph %[temp11], %[temp5], %[temp8] \n\t" + "addq.ph %[temp10], %[temp10], %[temp2] \n\t" + "addq.ph %[temp11], %[temp11], %[temp4] \n\t" + "shra_r.ph %[temp10], %[temp10], 2 \n\t" + "shra_r.ph %[temp11], %[temp11], 2 \n\t" + "srl %[temp1], %[temp1], 24 \n\t" + "sll %[temp1], %[temp1], 1 \n\t" + "raddu.w.qb %[temp5], %[temp5] \n\t" + "precr.qb.ph %[temp9], %[temp10], %[temp9] \n\t" + "precr.qb.ph %[temp10], %[temp11], %[temp10] \n\t" + "addu %[temp1], %[temp1], %[temp5] \n\t" + "shra_r.w %[temp1], %[temp1], 2 \n\t" + "usw %[temp9], 0*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp10], 2*" XSTR(BPS) "(%[dst]) \n\t" + "prepend %[temp9], %[temp11], 8 \n\t" + "prepend %[temp10], %[temp1], 8 \n\t" + "usw %[temp9], 1*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp10], 3*" XSTR(BPS) "(%[dst]) \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8), + [temp9]"=&r"(temp9), [temp10]"=&r"(temp10), [temp11]"=&r"(temp11) + : [top]"r"(top), [dst]"r"(dst) + : "memory" + ); +} + +static void VL4(uint8_t* dst, const uint8_t* top) { + int temp0, temp1, temp2, temp3, temp4; + int temp5, temp6, temp7, temp8, temp9; + __asm__ volatile ( + "ulw %[temp0], 0(%[top]) \n\t" + "ulw %[temp1], 4(%[top]) \n\t" + "preceu.ph.qbla %[temp2], %[temp0] \n\t" + "preceu.ph.qbra %[temp0], %[temp0] \n\t" + "preceu.ph.qbl %[temp3], %[temp1] \n\t" + "preceu.ph.qbr %[temp1], %[temp1] \n\t" + "addqh_r.ph %[temp4], %[temp0], %[temp2] \n\t" + "packrl.ph %[temp7], %[temp1], %[temp0] \n\t" + "precrq.ph.w %[temp6], %[temp1], %[temp2] \n\t" + "shll.ph %[temp9], %[temp2], 1 \n\t" + "addqh_r.ph %[temp5], %[temp7], %[temp2] \n\t" + "shll.ph %[temp8], %[temp7], 1 \n\t" + "addu.ph %[temp2], %[temp2], %[temp6] \n\t" + "addu.ph %[temp0], %[temp0], %[temp7] \n\t" + "packrl.ph %[temp7], %[temp3], %[temp1] \n\t" + "addu.ph %[temp6], %[temp1], %[temp3] \n\t" + "addu.ph %[temp2], %[temp2], %[temp8] \n\t" + "addu.ph %[temp0], %[temp0], %[temp9] \n\t" + "shll.ph %[temp7], %[temp7], 1 \n\t" + "shra_r.ph %[temp2], %[temp2], 2 \n\t" + "shra_r.ph %[temp0], %[temp0], 2 \n\t" + "addu.ph %[temp6], %[temp6], %[temp7] \n\t" + "shra_r.ph %[temp6], %[temp6], 2 \n\t" + "precrq.ph.w %[temp8], %[temp5], %[temp4] \n\t" + "append %[temp5], %[temp4], 16 \n\t" + "precrq.ph.w %[temp3], %[temp2], %[temp0] \n\t" + "append %[temp2], %[temp0], 16 \n\t" + "precr.qb.ph %[temp8], %[temp8], %[temp5] \n\t" + "precr.qb.ph %[temp3], %[temp3], %[temp2] \n\t" + "usw %[temp8], 0*" XSTR(BPS) "(%[dst]) \n\t" + "prepend %[temp8], %[temp6], 8 \n\t" + "usw %[temp3], 1*" XSTR(BPS) "(%[dst]) \n\t" + "srl %[temp6], %[temp6], 16 \n\t" + "prepend %[temp3], %[temp6], 8 \n\t" + "usw %[temp8], 2*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp3], 3*" XSTR(BPS) "(%[dst]) \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8), + [temp9]"=&r"(temp9) + : [top]"r"(top), [dst]"r"(dst) + : "memory" + ); +} + +static void HD4(uint8_t* dst, const uint8_t* top) { + int temp0, temp1, temp2, temp3, temp4; + int temp5, temp6, temp7, temp8, temp9; + __asm__ volatile ( + "ulw %[temp0], -5(%[top]) \n\t" + "ulw %[temp1], -1(%[top]) \n\t" + "preceu.ph.qbla %[temp2], %[temp0] \n\t" + "preceu.ph.qbra %[temp0], %[temp0] \n\t" + "preceu.ph.qbl %[temp3], %[temp1] \n\t" + "preceu.ph.qbr %[temp1], %[temp1] \n\t" + "addqh_r.ph %[temp4], %[temp0], %[temp2] \n\t" + "packrl.ph %[temp7], %[temp1], %[temp0] \n\t" + "precrq.ph.w %[temp6], %[temp1], %[temp2] \n\t" + "shll.ph %[temp9], %[temp2], 1 \n\t" + "addqh_r.ph %[temp5], %[temp7], %[temp2] \n\t" + "shll.ph %[temp8], %[temp7], 1 \n\t" + "addu.ph %[temp2], %[temp2], %[temp6] \n\t" + "addu.ph %[temp0], %[temp0], %[temp7] \n\t" + "packrl.ph %[temp7], %[temp3], %[temp1] \n\t" + "addu.ph %[temp6], %[temp1], %[temp3] \n\t" + "addu.ph %[temp2], %[temp2], %[temp8] \n\t" + "addu.ph %[temp0], %[temp0], %[temp9] \n\t" + "shll.ph %[temp7], %[temp7], 1 \n\t" + "shra_r.ph %[temp2], %[temp2], 2 \n\t" + "shra_r.ph %[temp0], %[temp0], 2 \n\t" + "addu.ph %[temp6], %[temp6], %[temp7] \n\t" + "shra_r.ph %[temp6], %[temp6], 2 \n\t" + "precrq.ph.w %[temp1], %[temp2], %[temp5] \n\t" + "precrq.ph.w %[temp3], %[temp0], %[temp4] \n\t" + "precr.qb.ph %[temp7], %[temp6], %[temp1] \n\t" + "precr.qb.ph %[temp6], %[temp1], %[temp3] \n\t" + "usw %[temp7], 0*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp6], 1*" XSTR(BPS) "(%[dst]) \n\t" + "append %[temp2], %[temp5], 16 \n\t" + "append %[temp0], %[temp4], 16 \n\t" + "precr.qb.ph %[temp5], %[temp3], %[temp2] \n\t" + "precr.qb.ph %[temp4], %[temp2], %[temp0] \n\t" + "usw %[temp5], 2*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp4], 3*" XSTR(BPS) "(%[dst]) \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8), + [temp9]"=&r"(temp9) + : [top]"r"(top), [dst]"r"(dst) + : "memory" + ); +} + +static void HU4(uint8_t* dst, const uint8_t* top) { + int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; + __asm__ volatile ( + "ulw %[temp0], -5(%[top]) \n\t" + "preceu.ph.qbl %[temp1], %[temp0] \n\t" + "preceu.ph.qbr %[temp2], %[temp0] \n\t" + "packrl.ph %[temp3], %[temp1], %[temp2] \n\t" + "replv.qb %[temp7], %[temp2] \n\t" + "addqh_r.ph %[temp4], %[temp1], %[temp3] \n\t" + "addqh_r.ph %[temp5], %[temp3], %[temp2] \n\t" + "shll.ph %[temp6], %[temp3], 1 \n\t" + "addu.ph %[temp3], %[temp2], %[temp3] \n\t" + "addu.ph %[temp6], %[temp1], %[temp6] \n\t" + "shll.ph %[temp0], %[temp2], 1 \n\t" + "addu.ph %[temp6], %[temp6], %[temp2] \n\t" + "addu.ph %[temp0], %[temp3], %[temp0] \n\t" + "shra_r.ph %[temp6], %[temp6], 2 \n\t" + "shra_r.ph %[temp0], %[temp0], 2 \n\t" + "packrl.ph %[temp3], %[temp6], %[temp5] \n\t" + "precrq.ph.w %[temp2], %[temp6], %[temp4] \n\t" + "append %[temp0], %[temp5], 16 \n\t" + "precr.qb.ph %[temp3], %[temp3], %[temp2] \n\t" + "usw %[temp3], 0*" XSTR(BPS) "(%[dst]) \n\t" + "precr.qb.ph %[temp1], %[temp7], %[temp0] \n\t" + "usw %[temp7], 3*" XSTR(BPS) "(%[dst]) \n\t" + "packrl.ph %[temp2], %[temp1], %[temp3] \n\t" + "usw %[temp1], 2*" XSTR(BPS) "(%[dst]) \n\t" + "usw %[temp2], 1*" XSTR(BPS) "(%[dst]) \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7) + : [top]"r"(top), [dst]"r"(dst) + : "memory" + ); +} + +//------------------------------------------------------------------------------ +// Chroma 8x8 prediction (paragraph 12.2) + +static void IntraChromaPreds(uint8_t* dst, const uint8_t* left, + const uint8_t* top) { + // U block + DCMode8(C8DC8 + dst, left, top); + VerticalPred8(C8VE8 + dst, top); + HorizontalPred8(C8HE8 + dst, left); + TrueMotion8(C8TM8 + dst, left, top); + // V block + dst += 8; + if (top) top += 8; + if (left) left += 16; + DCMode8(C8DC8 + dst, left, top); + VerticalPred8(C8VE8 + dst, top); + HorizontalPred8(C8HE8 + dst, left); + TrueMotion8(C8TM8 + dst, left, top); +} + +//------------------------------------------------------------------------------ +// luma 16x16 prediction (paragraph 12.3) + +static void Intra16Preds(uint8_t* dst, + const uint8_t* left, const uint8_t* top) { + DCMode16(I16DC16 + dst, left, top); + VerticalPred16(I16VE16 + dst, top); + HorizontalPred16(I16HE16 + dst, left); + TrueMotion16(I16TM16 + dst, left, top); +} + +// Left samples are top[-5 .. -2], top_left is top[-1], top are +// located at top[0..3], and top right is top[4..7] +static void Intra4Preds(uint8_t* dst, const uint8_t* top) { + DC4(I4DC4 + dst, top); + TM4(I4TM4 + dst, top); + VE4(I4VE4 + dst, top); + HE4(I4HE4 + dst, top); + RD4(I4RD4 + dst, top); + VR4(I4VR4 + dst, top); + LD4(I4LD4 + dst, top); + VL4(I4VL4 + dst, top); + HD4(I4HD4 + dst, top); + HU4(I4HU4 + dst, top); +} + +//------------------------------------------------------------------------------ +// Metric + +#if !defined(WORK_AROUND_GCC) + +#define GET_SSE_INNER(A) \ + "lw %[temp0], " #A "(%[a]) \n\t" \ + "lw %[temp1], " #A "(%[b]) \n\t" \ + "preceu.ph.qbr %[temp2], %[temp0] \n\t" \ + "preceu.ph.qbl %[temp0], %[temp0] \n\t" \ + "preceu.ph.qbr %[temp3], %[temp1] \n\t" \ + "preceu.ph.qbl %[temp1], %[temp1] \n\t" \ + "subq.ph %[temp2], %[temp2], %[temp3] \n\t" \ + "subq.ph %[temp0], %[temp0], %[temp1] \n\t" \ + "dpa.w.ph $ac0, %[temp2], %[temp2] \n\t" \ + "dpa.w.ph $ac0, %[temp0], %[temp0] \n\t" + +#define GET_SSE(A, B, C, D) \ + GET_SSE_INNER(A) \ + GET_SSE_INNER(B) \ + GET_SSE_INNER(C) \ + GET_SSE_INNER(D) + +static int SSE16x16(const uint8_t* a, const uint8_t* b) { + int count; + int temp0, temp1, temp2, temp3; + __asm__ volatile ( + "mult $zero, $zero \n\t" + GET_SSE( 0 * BPS, 4 + 0 * BPS, 8 + 0 * BPS, 12 + 0 * BPS) + GET_SSE( 1 * BPS, 4 + 1 * BPS, 8 + 1 * BPS, 12 + 1 * BPS) + GET_SSE( 2 * BPS, 4 + 2 * BPS, 8 + 2 * BPS, 12 + 2 * BPS) + GET_SSE( 3 * BPS, 4 + 3 * BPS, 8 + 3 * BPS, 12 + 3 * BPS) + GET_SSE( 4 * BPS, 4 + 4 * BPS, 8 + 4 * BPS, 12 + 4 * BPS) + GET_SSE( 5 * BPS, 4 + 5 * BPS, 8 + 5 * BPS, 12 + 5 * BPS) + GET_SSE( 6 * BPS, 4 + 6 * BPS, 8 + 6 * BPS, 12 + 6 * BPS) + GET_SSE( 7 * BPS, 4 + 7 * BPS, 8 + 7 * BPS, 12 + 7 * BPS) + GET_SSE( 8 * BPS, 4 + 8 * BPS, 8 + 8 * BPS, 12 + 8 * BPS) + GET_SSE( 9 * BPS, 4 + 9 * BPS, 8 + 9 * BPS, 12 + 9 * BPS) + GET_SSE(10 * BPS, 4 + 10 * BPS, 8 + 10 * BPS, 12 + 10 * BPS) + GET_SSE(11 * BPS, 4 + 11 * BPS, 8 + 11 * BPS, 12 + 11 * BPS) + GET_SSE(12 * BPS, 4 + 12 * BPS, 8 + 12 * BPS, 12 + 12 * BPS) + GET_SSE(13 * BPS, 4 + 13 * BPS, 8 + 13 * BPS, 12 + 13 * BPS) + GET_SSE(14 * BPS, 4 + 14 * BPS, 8 + 14 * BPS, 12 + 14 * BPS) + GET_SSE(15 * BPS, 4 + 15 * BPS, 8 + 15 * BPS, 12 + 15 * BPS) + "mflo %[count] \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [count]"=&r"(count) + : [a]"r"(a), [b]"r"(b) + : "memory", "hi", "lo" + ); + return count; +} + +static int SSE16x8(const uint8_t* a, const uint8_t* b) { + int count; + int temp0, temp1, temp2, temp3; + __asm__ volatile ( + "mult $zero, $zero \n\t" + GET_SSE( 0 * BPS, 4 + 0 * BPS, 8 + 0 * BPS, 12 + 0 * BPS) + GET_SSE( 1 * BPS, 4 + 1 * BPS, 8 + 1 * BPS, 12 + 1 * BPS) + GET_SSE( 2 * BPS, 4 + 2 * BPS, 8 + 2 * BPS, 12 + 2 * BPS) + GET_SSE( 3 * BPS, 4 + 3 * BPS, 8 + 3 * BPS, 12 + 3 * BPS) + GET_SSE( 4 * BPS, 4 + 4 * BPS, 8 + 4 * BPS, 12 + 4 * BPS) + GET_SSE( 5 * BPS, 4 + 5 * BPS, 8 + 5 * BPS, 12 + 5 * BPS) + GET_SSE( 6 * BPS, 4 + 6 * BPS, 8 + 6 * BPS, 12 + 6 * BPS) + GET_SSE( 7 * BPS, 4 + 7 * BPS, 8 + 7 * BPS, 12 + 7 * BPS) + "mflo %[count] \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [count]"=&r"(count) + : [a]"r"(a), [b]"r"(b) + : "memory", "hi", "lo" + ); + return count; +} + +static int SSE8x8(const uint8_t* a, const uint8_t* b) { + int count; + int temp0, temp1, temp2, temp3; + __asm__ volatile ( + "mult $zero, $zero \n\t" + GET_SSE(0 * BPS, 4 + 0 * BPS, 1 * BPS, 4 + 1 * BPS) + GET_SSE(2 * BPS, 4 + 2 * BPS, 3 * BPS, 4 + 3 * BPS) + GET_SSE(4 * BPS, 4 + 4 * BPS, 5 * BPS, 4 + 5 * BPS) + GET_SSE(6 * BPS, 4 + 6 * BPS, 7 * BPS, 4 + 7 * BPS) + "mflo %[count] \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [count]"=&r"(count) + : [a]"r"(a), [b]"r"(b) + : "memory", "hi", "lo" + ); + return count; +} + +static int SSE4x4(const uint8_t* a, const uint8_t* b) { + int count; + int temp0, temp1, temp2, temp3; + __asm__ volatile ( + "mult $zero, $zero \n\t" + GET_SSE(0 * BPS, 1 * BPS, 2 * BPS, 3 * BPS) + "mflo %[count] \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [count]"=&r"(count) + : [a]"r"(a), [b]"r"(b) + : "memory", "hi", "lo" + ); + return count; +} + +#undef GET_SSE +#undef GET_SSE_INNER + +#endif // !WORK_AROUND_GCC + +#undef FILL_8_OR_16 +#undef FILL_PART +#undef OUTPUT_EARLY_CLOBBER_REGS_17 +#undef MUL_HALF +#undef ABS_X8 +#undef ADD_SUB_HALVES_X4 + +//------------------------------------------------------------------------------ +// Quantization +// + +// macro for one pass through for loop in QuantizeBlock reading 2 values at time +// QUANTDIV macro inlined +// J - offset in bytes (kZigzag[n] * 2) +// K - offset in bytes (kZigzag[n] * 4) +// N - offset in bytes (n * 2) +// N1 - offset in bytes ((n + 1) * 2) +#define QUANTIZE_ONE(J, K, N, N1) \ + "ulw %[temp1], " #J "(%[ppin]) \n\t" \ + "ulw %[temp2], " #J "(%[ppsharpen]) \n\t" \ + "lhu %[temp3], " #K "(%[ppzthresh]) \n\t" \ + "lhu %[temp6], " #K "+4(%[ppzthresh]) \n\t" \ + "absq_s.ph %[temp4], %[temp1] \n\t" \ + "ins %[temp3], %[temp6], 16, 16 \n\t" \ + "addu.ph %[coeff], %[temp4], %[temp2] \n\t" \ + "shra.ph %[sign], %[temp1], 15 \n\t" \ + "li %[level], 0x10001 \n\t" \ + "cmp.lt.ph %[temp3], %[coeff] \n\t" \ + "lhu %[temp1], " #J "(%[ppiq]) \n\t" \ + "pick.ph %[temp5], %[level], $0 \n\t" \ + "lw %[temp2], " #K "(%[ppbias]) \n\t" \ + "beqz %[temp5], 0f \n\t" \ + "lhu %[temp3], " #J "(%[ppq]) \n\t" \ + "beq %[temp5], %[level], 1f \n\t" \ + "andi %[temp5], %[temp5], 0x1 \n\t" \ + "andi %[temp4], %[coeff], 0xffff \n\t" \ + "beqz %[temp5], 2f \n\t" \ + "mul %[level], %[temp4], %[temp1] \n\t" \ + "sh $0, " #J "+2(%[ppin]) \n\t" \ + "sh $0, " #N1 "(%[pout]) \n\t" \ + "addu %[level], %[level], %[temp2] \n\t" \ + "sra %[level], %[level], 17 \n\t" \ + "slt %[temp4], %[max_level], %[level] \n\t" \ + "movn %[level], %[max_level], %[temp4] \n\t" \ + "andi %[temp6], %[sign], 0xffff \n\t" \ + "xor %[level], %[level], %[temp6] \n\t" \ + "subu %[level], %[level], %[temp6] \n\t" \ + "mul %[temp5], %[level], %[temp3] \n\t" \ + "or %[ret], %[ret], %[level] \n\t" \ + "sh %[level], " #N "(%[pout]) \n\t" \ + "sh %[temp5], " #J "(%[ppin]) \n\t" \ + "j 3f \n\t" \ +"2: \n\t" \ + "lhu %[temp1], " #J "+2(%[ppiq]) \n\t" \ + "srl %[temp5], %[coeff], 16 \n\t" \ + "mul %[level], %[temp5], %[temp1] \n\t" \ + "lw %[temp2], " #K "+4(%[ppbias]) \n\t" \ + "lhu %[temp3], " #J "+2(%[ppq]) \n\t" \ + "addu %[level], %[level], %[temp2] \n\t" \ + "sra %[level], %[level], 17 \n\t" \ + "srl %[temp6], %[sign], 16 \n\t" \ + "slt %[temp4], %[max_level], %[level] \n\t" \ + "movn %[level], %[max_level], %[temp4] \n\t" \ + "xor %[level], %[level], %[temp6] \n\t" \ + "subu %[level], %[level], %[temp6] \n\t" \ + "mul %[temp5], %[level], %[temp3] \n\t" \ + "sh $0, " #J "(%[ppin]) \n\t" \ + "sh $0, " #N "(%[pout]) \n\t" \ + "or %[ret], %[ret], %[level] \n\t" \ + "sh %[temp5], " #J "+2(%[ppin]) \n\t" \ + "sh %[level], " #N1 "(%[pout]) \n\t" \ + "j 3f \n\t" \ +"1: \n\t" \ + "lhu %[temp1], " #J "(%[ppiq]) \n\t" \ + "lw %[temp2], " #K "(%[ppbias]) \n\t" \ + "ulw %[temp3], " #J "(%[ppq]) \n\t" \ + "andi %[temp5], %[coeff], 0xffff \n\t" \ + "srl %[temp0], %[coeff], 16 \n\t" \ + "lhu %[temp6], " #J "+2(%[ppiq]) \n\t" \ + "lw %[coeff], " #K "+4(%[ppbias]) \n\t" \ + "mul %[level], %[temp5], %[temp1] \n\t" \ + "mul %[temp4], %[temp0], %[temp6] \n\t" \ + "addu %[level], %[level], %[temp2] \n\t" \ + "addu %[temp4], %[temp4], %[coeff] \n\t" \ + "precrq.ph.w %[level], %[temp4], %[level] \n\t" \ + "shra.ph %[level], %[level], 1 \n\t" \ + "cmp.lt.ph %[max_level1],%[level] \n\t" \ + "pick.ph %[level], %[max_level], %[level] \n\t" \ + "xor %[level], %[level], %[sign] \n\t" \ + "subu.ph %[level], %[level], %[sign] \n\t" \ + "mul.ph %[temp3], %[level], %[temp3] \n\t" \ + "or %[ret], %[ret], %[level] \n\t" \ + "sh %[level], " #N "(%[pout]) \n\t" \ + "srl %[level], %[level], 16 \n\t" \ + "sh %[level], " #N1 "(%[pout]) \n\t" \ + "usw %[temp3], " #J "(%[ppin]) \n\t" \ + "j 3f \n\t" \ +"0: \n\t" \ + "sh $0, " #N "(%[pout]) \n\t" \ + "sh $0, " #N1 "(%[pout]) \n\t" \ + "usw $0, " #J "(%[ppin]) \n\t" \ +"3: \n\t" + +static int QuantizeBlock(int16_t in[16], int16_t out[16], + const VP8Matrix* const mtx) { + int temp0, temp1, temp2, temp3, temp4, temp5,temp6; + int sign, coeff, level; + int max_level = MAX_LEVEL; + int max_level1 = max_level << 16 | max_level; + int ret = 0; + + int16_t* ppin = &in[0]; + int16_t* pout = &out[0]; + const uint16_t* ppsharpen = &mtx->sharpen_[0]; + const uint32_t* ppzthresh = &mtx->zthresh_[0]; + const uint16_t* ppq = &mtx->q_[0]; + const uint16_t* ppiq = &mtx->iq_[0]; + const uint32_t* ppbias = &mtx->bias_[0]; + + __asm__ volatile ( + QUANTIZE_ONE( 0, 0, 0, 2) + QUANTIZE_ONE( 4, 8, 10, 12) + QUANTIZE_ONE( 8, 16, 4, 8) + QUANTIZE_ONE(12, 24, 14, 24) + QUANTIZE_ONE(16, 32, 6, 16) + QUANTIZE_ONE(20, 40, 22, 26) + QUANTIZE_ONE(24, 48, 18, 20) + QUANTIZE_ONE(28, 56, 28, 30) + + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), + [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [sign]"=&r"(sign), [coeff]"=&r"(coeff), + [level]"=&r"(level), [temp6]"=&r"(temp6), [ret]"+&r"(ret) + : [ppin]"r"(ppin), [pout]"r"(pout), [max_level1]"r"(max_level1), + [ppiq]"r"(ppiq), [max_level]"r"(max_level), + [ppbias]"r"(ppbias), [ppzthresh]"r"(ppzthresh), + [ppsharpen]"r"(ppsharpen), [ppq]"r"(ppq) + : "memory", "hi", "lo" + ); + + return (ret != 0); +} + +static int Quantize2Blocks(int16_t in[32], int16_t out[32], + const VP8Matrix* const mtx) { + int nz; + nz = QuantizeBlock(in + 0 * 16, out + 0 * 16, mtx) << 0; + nz |= QuantizeBlock(in + 1 * 16, out + 1 * 16, mtx) << 1; + return nz; +} + +#undef QUANTIZE_ONE + +// macro for one horizontal pass in FTransformWHT +// temp0..temp7 holds tmp[0]..tmp[15] +// A, B, C, D - offset in bytes to load from in buffer +// TEMP0, TEMP1 - registers for corresponding tmp elements +#define HORIZONTAL_PASS_WHT(A, B, C, D, TEMP0, TEMP1) \ + "lh %[" #TEMP0 "], " #A "(%[in]) \n\t" \ + "lh %[" #TEMP1 "], " #B "(%[in]) \n\t" \ + "lh %[temp8], " #C "(%[in]) \n\t" \ + "lh %[temp9], " #D "(%[in]) \n\t" \ + "ins %[" #TEMP1 "], %[" #TEMP0 "], 16, 16 \n\t" \ + "ins %[temp9], %[temp8], 16, 16 \n\t" \ + "subq.ph %[temp8], %[" #TEMP1 "], %[temp9] \n\t" \ + "addq.ph %[temp9], %[" #TEMP1 "], %[temp9] \n\t" \ + "precrq.ph.w %[" #TEMP0 "], %[temp8], %[temp9] \n\t" \ + "append %[temp8], %[temp9], 16 \n\t" \ + "subq.ph %[" #TEMP1 "], %[" #TEMP0 "], %[temp8] \n\t" \ + "addq.ph %[" #TEMP0 "], %[" #TEMP0 "], %[temp8] \n\t" \ + "rotr %[" #TEMP1 "], %[" #TEMP1 "], 16 \n\t" + +// macro for one vertical pass in FTransformWHT +// temp0..temp7 holds tmp[0]..tmp[15] +// A, B, C, D - offsets in bytes to store to out buffer +// TEMP0, TEMP2, TEMP4 and TEMP6 - registers for corresponding tmp elements +#define VERTICAL_PASS_WHT(A, B, C, D, TEMP0, TEMP2, TEMP4, TEMP6) \ + "addq.ph %[temp8], %[" #TEMP0 "], %[" #TEMP4 "] \n\t" \ + "addq.ph %[temp9], %[" #TEMP2 "], %[" #TEMP6 "] \n\t" \ + "subq.ph %[" #TEMP2 "], %[" #TEMP2 "], %[" #TEMP6 "] \n\t" \ + "subq.ph %[" #TEMP6 "], %[" #TEMP0 "], %[" #TEMP4 "] \n\t" \ + "addqh.ph %[" #TEMP0 "], %[temp8], %[temp9] \n\t" \ + "subqh.ph %[" #TEMP4 "], %[" #TEMP6 "], %[" #TEMP2 "] \n\t" \ + "addqh.ph %[" #TEMP2 "], %[" #TEMP2 "], %[" #TEMP6 "] \n\t" \ + "subqh.ph %[" #TEMP6 "], %[temp8], %[temp9] \n\t" \ + "usw %[" #TEMP0 "], " #A "(%[out]) \n\t" \ + "usw %[" #TEMP2 "], " #B "(%[out]) \n\t" \ + "usw %[" #TEMP4 "], " #C "(%[out]) \n\t" \ + "usw %[" #TEMP6 "], " #D "(%[out]) \n\t" + +static void FTransformWHT(const int16_t* in, int16_t* out) { + int temp0, temp1, temp2, temp3, temp4; + int temp5, temp6, temp7, temp8, temp9; + + __asm__ volatile ( + HORIZONTAL_PASS_WHT( 0, 32, 64, 96, temp0, temp1) + HORIZONTAL_PASS_WHT(128, 160, 192, 224, temp2, temp3) + HORIZONTAL_PASS_WHT(256, 288, 320, 352, temp4, temp5) + HORIZONTAL_PASS_WHT(384, 416, 448, 480, temp6, temp7) + VERTICAL_PASS_WHT(0, 8, 16, 24, temp0, temp2, temp4, temp6) + VERTICAL_PASS_WHT(4, 12, 20, 28, temp1, temp3, temp5, temp7) + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8), + [temp9]"=&r"(temp9) + : [in]"r"(in), [out]"r"(out) + : "memory" + ); +} + +#undef VERTICAL_PASS_WHT +#undef HORIZONTAL_PASS_WHT + +// macro for converting coefficients to bin +// convert 8 coeffs at time +// A, B, C, D - offsets in bytes to load from out buffer +#define CONVERT_COEFFS_TO_BIN(A, B, C, D) \ + "ulw %[temp0], " #A "(%[out]) \n\t" \ + "ulw %[temp1], " #B "(%[out]) \n\t" \ + "ulw %[temp2], " #C "(%[out]) \n\t" \ + "ulw %[temp3], " #D "(%[out]) \n\t" \ + "absq_s.ph %[temp0], %[temp0] \n\t" \ + "absq_s.ph %[temp1], %[temp1] \n\t" \ + "absq_s.ph %[temp2], %[temp2] \n\t" \ + "absq_s.ph %[temp3], %[temp3] \n\t" \ + "shra.ph %[temp0], %[temp0], 3 \n\t" \ + "shra.ph %[temp1], %[temp1], 3 \n\t" \ + "shra.ph %[temp2], %[temp2], 3 \n\t" \ + "shra.ph %[temp3], %[temp3], 3 \n\t" \ + "shll_s.ph %[temp0], %[temp0], 10 \n\t" \ + "shll_s.ph %[temp1], %[temp1], 10 \n\t" \ + "shll_s.ph %[temp2], %[temp2], 10 \n\t" \ + "shll_s.ph %[temp3], %[temp3], 10 \n\t" \ + "shrl.ph %[temp0], %[temp0], 10 \n\t" \ + "shrl.ph %[temp1], %[temp1], 10 \n\t" \ + "shrl.ph %[temp2], %[temp2], 10 \n\t" \ + "shrl.ph %[temp3], %[temp3], 10 \n\t" \ + "shll.ph %[temp0], %[temp0], 2 \n\t" \ + "shll.ph %[temp1], %[temp1], 2 \n\t" \ + "shll.ph %[temp2], %[temp2], 2 \n\t" \ + "shll.ph %[temp3], %[temp3], 2 \n\t" \ + "ext %[temp4], %[temp0], 0, 16 \n\t" \ + "ext %[temp0], %[temp0], 16, 16 \n\t" \ + "addu %[temp4], %[temp4], %[dist] \n\t" \ + "addu %[temp0], %[temp0], %[dist] \n\t" \ + "ext %[temp5], %[temp1], 0, 16 \n\t" \ + "lw %[temp8], 0(%[temp4]) \n\t" \ + "ext %[temp1], %[temp1], 16, 16 \n\t" \ + "addu %[temp5], %[temp5], %[dist] \n\t" \ + "addiu %[temp8], %[temp8], 1 \n\t" \ + "sw %[temp8], 0(%[temp4]) \n\t" \ + "lw %[temp8], 0(%[temp0]) \n\t" \ + "addu %[temp1], %[temp1], %[dist] \n\t" \ + "ext %[temp6], %[temp2], 0, 16 \n\t" \ + "addiu %[temp8], %[temp8], 1 \n\t" \ + "sw %[temp8], 0(%[temp0]) \n\t" \ + "lw %[temp8], 0(%[temp5]) \n\t" \ + "ext %[temp2], %[temp2], 16, 16 \n\t" \ + "addu %[temp6], %[temp6], %[dist] \n\t" \ + "addiu %[temp8], %[temp8], 1 \n\t" \ + "sw %[temp8], 0(%[temp5]) \n\t" \ + "lw %[temp8], 0(%[temp1]) \n\t" \ + "addu %[temp2], %[temp2], %[dist] \n\t" \ + "ext %[temp7], %[temp3], 0, 16 \n\t" \ + "addiu %[temp8], %[temp8], 1 \n\t" \ + "sw %[temp8], 0(%[temp1]) \n\t" \ + "lw %[temp8], 0(%[temp6]) \n\t" \ + "ext %[temp3], %[temp3], 16, 16 \n\t" \ + "addu %[temp7], %[temp7], %[dist] \n\t" \ + "addiu %[temp8], %[temp8], 1 \n\t" \ + "sw %[temp8], 0(%[temp6]) \n\t" \ + "lw %[temp8], 0(%[temp2]) \n\t" \ + "addu %[temp3], %[temp3], %[dist] \n\t" \ + "addiu %[temp8], %[temp8], 1 \n\t" \ + "sw %[temp8], 0(%[temp2]) \n\t" \ + "lw %[temp8], 0(%[temp7]) \n\t" \ + "addiu %[temp8], %[temp8], 1 \n\t" \ + "sw %[temp8], 0(%[temp7]) \n\t" \ + "lw %[temp8], 0(%[temp3]) \n\t" \ + "addiu %[temp8], %[temp8], 1 \n\t" \ + "sw %[temp8], 0(%[temp3]) \n\t" + +static void CollectHistogram(const uint8_t* ref, const uint8_t* pred, + int start_block, int end_block, + VP8Histogram* const histo) { + int j; + int distribution[MAX_COEFF_THRESH + 1] = { 0 }; + const int max_coeff = (MAX_COEFF_THRESH << 16) + MAX_COEFF_THRESH; + for (j = start_block; j < end_block; ++j) { + int16_t out[16]; + int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8; + + VP8FTransform(ref + VP8DspScan[j], pred + VP8DspScan[j], out); + + // Convert coefficients to bin. + __asm__ volatile ( + CONVERT_COEFFS_TO_BIN( 0, 4, 8, 12) + CONVERT_COEFFS_TO_BIN(16, 20, 24, 28) + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8) + : [dist]"r"(distribution), [out]"r"(out), [max_coeff]"r"(max_coeff) + : "memory" + ); + } + VP8SetHistogramData(distribution, histo); +} + +#undef CONVERT_COEFFS_TO_BIN + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8EncDspInitMIPSdspR2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitMIPSdspR2(void) { + VP8FTransform = FTransform; + VP8ITransform = ITransform; + VP8TDisto4x4 = Disto4x4; + VP8TDisto16x16 = Disto16x16; + VP8EncPredLuma16 = Intra16Preds; + VP8EncPredChroma8 = IntraChromaPreds; + VP8EncPredLuma4 = Intra4Preds; +#if !defined(WORK_AROUND_GCC) + VP8SSE16x16 = SSE16x16; + VP8SSE8x8 = SSE8x8; + VP8SSE16x8 = SSE16x8; + VP8SSE4x4 = SSE4x4; +#endif + VP8EncQuantizeBlock = QuantizeBlock; + VP8EncQuantize2Blocks = Quantize2Blocks; + VP8FTransformWHT = FTransformWHT; + VP8CollectHistogram = CollectHistogram; +} + +#else // !WEBP_USE_MIPS_DSP_R2 + +WEBP_DSP_INIT_STUB(VP8EncDspInitMIPSdspR2) + +#endif // WEBP_USE_MIPS_DSP_R2 diff --git a/thirdparty/libwebp/dsp/enc_neon.c b/thirdparty/libwebp/dsp/enc_neon.c new file mode 100644 index 0000000000..46f6bf9a33 --- /dev/null +++ b/thirdparty/libwebp/dsp/enc_neon.c @@ -0,0 +1,920 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// ARM NEON version of speed-critical encoding functions. +// +// adapted from libvpx (http://www.webmproject.org/code/) + +#include "./dsp.h" + +#if defined(WEBP_USE_NEON) + +#include <assert.h> + +#include "./neon.h" +#include "../enc/vp8enci.h" + +//------------------------------------------------------------------------------ +// Transforms (Paragraph 14.4) + +// Inverse transform. +// This code is pretty much the same as TransformOne in the dec_neon.c, except +// for subtraction to *ref. See the comments there for algorithmic explanations. + +static const int16_t kC1 = 20091; +static const int16_t kC2 = 17734; // half of kC2, actually. See comment above. + +// This code works but is *slower* than the inlined-asm version below +// (with gcc-4.6). So we disable it for now. Later, it'll be conditional to +// WEBP_USE_INTRINSICS define. +// With gcc-4.8, it's a little faster speed than inlined-assembly. +#if defined(WEBP_USE_INTRINSICS) + +// Treats 'v' as an uint8x8_t and zero extends to an int16x8_t. +static WEBP_INLINE int16x8_t ConvertU8ToS16(uint32x2_t v) { + return vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(v))); +} + +// Performs unsigned 8b saturation on 'dst01' and 'dst23' storing the result +// to the corresponding rows of 'dst'. +static WEBP_INLINE void SaturateAndStore4x4(uint8_t* const dst, + const int16x8_t dst01, + const int16x8_t dst23) { + // Unsigned saturate to 8b. + const uint8x8_t dst01_u8 = vqmovun_s16(dst01); + const uint8x8_t dst23_u8 = vqmovun_s16(dst23); + + // Store the results. + vst1_lane_u32((uint32_t*)(dst + 0 * BPS), vreinterpret_u32_u8(dst01_u8), 0); + vst1_lane_u32((uint32_t*)(dst + 1 * BPS), vreinterpret_u32_u8(dst01_u8), 1); + vst1_lane_u32((uint32_t*)(dst + 2 * BPS), vreinterpret_u32_u8(dst23_u8), 0); + vst1_lane_u32((uint32_t*)(dst + 3 * BPS), vreinterpret_u32_u8(dst23_u8), 1); +} + +static WEBP_INLINE void Add4x4(const int16x8_t row01, const int16x8_t row23, + const uint8_t* const ref, uint8_t* const dst) { + uint32x2_t dst01 = vdup_n_u32(0); + uint32x2_t dst23 = vdup_n_u32(0); + + // Load the source pixels. + dst01 = vld1_lane_u32((uint32_t*)(ref + 0 * BPS), dst01, 0); + dst23 = vld1_lane_u32((uint32_t*)(ref + 2 * BPS), dst23, 0); + dst01 = vld1_lane_u32((uint32_t*)(ref + 1 * BPS), dst01, 1); + dst23 = vld1_lane_u32((uint32_t*)(ref + 3 * BPS), dst23, 1); + + { + // Convert to 16b. + const int16x8_t dst01_s16 = ConvertU8ToS16(dst01); + const int16x8_t dst23_s16 = ConvertU8ToS16(dst23); + + // Descale with rounding. + const int16x8_t out01 = vrsraq_n_s16(dst01_s16, row01, 3); + const int16x8_t out23 = vrsraq_n_s16(dst23_s16, row23, 3); + // Add the inverse transform. + SaturateAndStore4x4(dst, out01, out23); + } +} + +static WEBP_INLINE void Transpose8x2(const int16x8_t in0, const int16x8_t in1, + int16x8x2_t* const out) { + // a0 a1 a2 a3 | b0 b1 b2 b3 => a0 b0 c0 d0 | a1 b1 c1 d1 + // c0 c1 c2 c3 | d0 d1 d2 d3 a2 b2 c2 d2 | a3 b3 c3 d3 + const int16x8x2_t tmp0 = vzipq_s16(in0, in1); // a0 c0 a1 c1 a2 c2 ... + // b0 d0 b1 d1 b2 d2 ... + *out = vzipq_s16(tmp0.val[0], tmp0.val[1]); +} + +static WEBP_INLINE void TransformPass(int16x8x2_t* const rows) { + // {rows} = in0 | in4 + // in8 | in12 + // B1 = in4 | in12 + const int16x8_t B1 = + vcombine_s16(vget_high_s16(rows->val[0]), vget_high_s16(rows->val[1])); + // C0 = kC1 * in4 | kC1 * in12 + // C1 = kC2 * in4 | kC2 * in12 + const int16x8_t C0 = vsraq_n_s16(B1, vqdmulhq_n_s16(B1, kC1), 1); + const int16x8_t C1 = vqdmulhq_n_s16(B1, kC2); + const int16x4_t a = vqadd_s16(vget_low_s16(rows->val[0]), + vget_low_s16(rows->val[1])); // in0 + in8 + const int16x4_t b = vqsub_s16(vget_low_s16(rows->val[0]), + vget_low_s16(rows->val[1])); // in0 - in8 + // c = kC2 * in4 - kC1 * in12 + // d = kC1 * in4 + kC2 * in12 + const int16x4_t c = vqsub_s16(vget_low_s16(C1), vget_high_s16(C0)); + const int16x4_t d = vqadd_s16(vget_low_s16(C0), vget_high_s16(C1)); + const int16x8_t D0 = vcombine_s16(a, b); // D0 = a | b + const int16x8_t D1 = vcombine_s16(d, c); // D1 = d | c + const int16x8_t E0 = vqaddq_s16(D0, D1); // a+d | b+c + const int16x8_t E_tmp = vqsubq_s16(D0, D1); // a-d | b-c + const int16x8_t E1 = vcombine_s16(vget_high_s16(E_tmp), vget_low_s16(E_tmp)); + Transpose8x2(E0, E1, rows); +} + +static void ITransformOne(const uint8_t* ref, + const int16_t* in, uint8_t* dst) { + int16x8x2_t rows; + INIT_VECTOR2(rows, vld1q_s16(in + 0), vld1q_s16(in + 8)); + TransformPass(&rows); + TransformPass(&rows); + Add4x4(rows.val[0], rows.val[1], ref, dst); +} + +#else + +static void ITransformOne(const uint8_t* ref, + const int16_t* in, uint8_t* dst) { + const int kBPS = BPS; + const int16_t kC1C2[] = { kC1, kC2, 0, 0 }; + + __asm__ volatile ( + "vld1.16 {q1, q2}, [%[in]] \n" + "vld1.16 {d0}, [%[kC1C2]] \n" + + // d2: in[0] + // d3: in[8] + // d4: in[4] + // d5: in[12] + "vswp d3, d4 \n" + + // q8 = {in[4], in[12]} * kC1 * 2 >> 16 + // q9 = {in[4], in[12]} * kC2 >> 16 + "vqdmulh.s16 q8, q2, d0[0] \n" + "vqdmulh.s16 q9, q2, d0[1] \n" + + // d22 = a = in[0] + in[8] + // d23 = b = in[0] - in[8] + "vqadd.s16 d22, d2, d3 \n" + "vqsub.s16 d23, d2, d3 \n" + + // q8 = in[4]/[12] * kC1 >> 16 + "vshr.s16 q8, q8, #1 \n" + + // Add {in[4], in[12]} back after the multiplication. + "vqadd.s16 q8, q2, q8 \n" + + // d20 = c = in[4]*kC2 - in[12]*kC1 + // d21 = d = in[4]*kC1 + in[12]*kC2 + "vqsub.s16 d20, d18, d17 \n" + "vqadd.s16 d21, d19, d16 \n" + + // d2 = tmp[0] = a + d + // d3 = tmp[1] = b + c + // d4 = tmp[2] = b - c + // d5 = tmp[3] = a - d + "vqadd.s16 d2, d22, d21 \n" + "vqadd.s16 d3, d23, d20 \n" + "vqsub.s16 d4, d23, d20 \n" + "vqsub.s16 d5, d22, d21 \n" + + "vzip.16 q1, q2 \n" + "vzip.16 q1, q2 \n" + + "vswp d3, d4 \n" + + // q8 = {tmp[4], tmp[12]} * kC1 * 2 >> 16 + // q9 = {tmp[4], tmp[12]} * kC2 >> 16 + "vqdmulh.s16 q8, q2, d0[0] \n" + "vqdmulh.s16 q9, q2, d0[1] \n" + + // d22 = a = tmp[0] + tmp[8] + // d23 = b = tmp[0] - tmp[8] + "vqadd.s16 d22, d2, d3 \n" + "vqsub.s16 d23, d2, d3 \n" + + "vshr.s16 q8, q8, #1 \n" + "vqadd.s16 q8, q2, q8 \n" + + // d20 = c = in[4]*kC2 - in[12]*kC1 + // d21 = d = in[4]*kC1 + in[12]*kC2 + "vqsub.s16 d20, d18, d17 \n" + "vqadd.s16 d21, d19, d16 \n" + + // d2 = tmp[0] = a + d + // d3 = tmp[1] = b + c + // d4 = tmp[2] = b - c + // d5 = tmp[3] = a - d + "vqadd.s16 d2, d22, d21 \n" + "vqadd.s16 d3, d23, d20 \n" + "vqsub.s16 d4, d23, d20 \n" + "vqsub.s16 d5, d22, d21 \n" + + "vld1.32 d6[0], [%[ref]], %[kBPS] \n" + "vld1.32 d6[1], [%[ref]], %[kBPS] \n" + "vld1.32 d7[0], [%[ref]], %[kBPS] \n" + "vld1.32 d7[1], [%[ref]], %[kBPS] \n" + + "sub %[ref], %[ref], %[kBPS], lsl #2 \n" + + // (val) + 4 >> 3 + "vrshr.s16 d2, d2, #3 \n" + "vrshr.s16 d3, d3, #3 \n" + "vrshr.s16 d4, d4, #3 \n" + "vrshr.s16 d5, d5, #3 \n" + + "vzip.16 q1, q2 \n" + "vzip.16 q1, q2 \n" + + // Must accumulate before saturating + "vmovl.u8 q8, d6 \n" + "vmovl.u8 q9, d7 \n" + + "vqadd.s16 q1, q1, q8 \n" + "vqadd.s16 q2, q2, q9 \n" + + "vqmovun.s16 d0, q1 \n" + "vqmovun.s16 d1, q2 \n" + + "vst1.32 d0[0], [%[dst]], %[kBPS] \n" + "vst1.32 d0[1], [%[dst]], %[kBPS] \n" + "vst1.32 d1[0], [%[dst]], %[kBPS] \n" + "vst1.32 d1[1], [%[dst]] \n" + + : [in] "+r"(in), [dst] "+r"(dst) // modified registers + : [kBPS] "r"(kBPS), [kC1C2] "r"(kC1C2), [ref] "r"(ref) // constants + : "memory", "q0", "q1", "q2", "q8", "q9", "q10", "q11" // clobbered + ); +} + +#endif // WEBP_USE_INTRINSICS + +static void ITransform(const uint8_t* ref, + const int16_t* in, uint8_t* dst, int do_two) { + ITransformOne(ref, in, dst); + if (do_two) { + ITransformOne(ref + 4, in + 16, dst + 4); + } +} + +// Load all 4x4 pixels into a single uint8x16_t variable. +static uint8x16_t Load4x4(const uint8_t* src) { + uint32x4_t out = vdupq_n_u32(0); + out = vld1q_lane_u32((const uint32_t*)(src + 0 * BPS), out, 0); + out = vld1q_lane_u32((const uint32_t*)(src + 1 * BPS), out, 1); + out = vld1q_lane_u32((const uint32_t*)(src + 2 * BPS), out, 2); + out = vld1q_lane_u32((const uint32_t*)(src + 3 * BPS), out, 3); + return vreinterpretq_u8_u32(out); +} + +// Forward transform. + +#if defined(WEBP_USE_INTRINSICS) + +static WEBP_INLINE void Transpose4x4_S16(const int16x4_t A, const int16x4_t B, + const int16x4_t C, const int16x4_t D, + int16x8_t* const out01, + int16x8_t* const out32) { + const int16x4x2_t AB = vtrn_s16(A, B); + const int16x4x2_t CD = vtrn_s16(C, D); + const int32x2x2_t tmp02 = vtrn_s32(vreinterpret_s32_s16(AB.val[0]), + vreinterpret_s32_s16(CD.val[0])); + const int32x2x2_t tmp13 = vtrn_s32(vreinterpret_s32_s16(AB.val[1]), + vreinterpret_s32_s16(CD.val[1])); + *out01 = vreinterpretq_s16_s64( + vcombine_s64(vreinterpret_s64_s32(tmp02.val[0]), + vreinterpret_s64_s32(tmp13.val[0]))); + *out32 = vreinterpretq_s16_s64( + vcombine_s64(vreinterpret_s64_s32(tmp13.val[1]), + vreinterpret_s64_s32(tmp02.val[1]))); +} + +static WEBP_INLINE int16x8_t DiffU8ToS16(const uint8x8_t a, + const uint8x8_t b) { + return vreinterpretq_s16_u16(vsubl_u8(a, b)); +} + +static void FTransform(const uint8_t* src, const uint8_t* ref, + int16_t* out) { + int16x8_t d0d1, d3d2; // working 4x4 int16 variables + { + const uint8x16_t S0 = Load4x4(src); + const uint8x16_t R0 = Load4x4(ref); + const int16x8_t D0D1 = DiffU8ToS16(vget_low_u8(S0), vget_low_u8(R0)); + const int16x8_t D2D3 = DiffU8ToS16(vget_high_u8(S0), vget_high_u8(R0)); + const int16x4_t D0 = vget_low_s16(D0D1); + const int16x4_t D1 = vget_high_s16(D0D1); + const int16x4_t D2 = vget_low_s16(D2D3); + const int16x4_t D3 = vget_high_s16(D2D3); + Transpose4x4_S16(D0, D1, D2, D3, &d0d1, &d3d2); + } + { // 1rst pass + const int32x4_t kCst937 = vdupq_n_s32(937); + const int32x4_t kCst1812 = vdupq_n_s32(1812); + const int16x8_t a0a1 = vaddq_s16(d0d1, d3d2); // d0+d3 | d1+d2 (=a0|a1) + const int16x8_t a3a2 = vsubq_s16(d0d1, d3d2); // d0-d3 | d1-d2 (=a3|a2) + const int16x8_t a0a1_2 = vshlq_n_s16(a0a1, 3); + const int16x4_t tmp0 = vadd_s16(vget_low_s16(a0a1_2), + vget_high_s16(a0a1_2)); + const int16x4_t tmp2 = vsub_s16(vget_low_s16(a0a1_2), + vget_high_s16(a0a1_2)); + const int32x4_t a3_2217 = vmull_n_s16(vget_low_s16(a3a2), 2217); + const int32x4_t a2_2217 = vmull_n_s16(vget_high_s16(a3a2), 2217); + const int32x4_t a2_p_a3 = vmlal_n_s16(a2_2217, vget_low_s16(a3a2), 5352); + const int32x4_t a3_m_a2 = vmlsl_n_s16(a3_2217, vget_high_s16(a3a2), 5352); + const int16x4_t tmp1 = vshrn_n_s32(vaddq_s32(a2_p_a3, kCst1812), 9); + const int16x4_t tmp3 = vshrn_n_s32(vaddq_s32(a3_m_a2, kCst937), 9); + Transpose4x4_S16(tmp0, tmp1, tmp2, tmp3, &d0d1, &d3d2); + } + { // 2nd pass + // the (1<<16) addition is for the replacement: a3!=0 <-> 1-(a3==0) + const int32x4_t kCst12000 = vdupq_n_s32(12000 + (1 << 16)); + const int32x4_t kCst51000 = vdupq_n_s32(51000); + const int16x8_t a0a1 = vaddq_s16(d0d1, d3d2); // d0+d3 | d1+d2 (=a0|a1) + const int16x8_t a3a2 = vsubq_s16(d0d1, d3d2); // d0-d3 | d1-d2 (=a3|a2) + const int16x4_t a0_k7 = vadd_s16(vget_low_s16(a0a1), vdup_n_s16(7)); + const int16x4_t out0 = vshr_n_s16(vadd_s16(a0_k7, vget_high_s16(a0a1)), 4); + const int16x4_t out2 = vshr_n_s16(vsub_s16(a0_k7, vget_high_s16(a0a1)), 4); + const int32x4_t a3_2217 = vmull_n_s16(vget_low_s16(a3a2), 2217); + const int32x4_t a2_2217 = vmull_n_s16(vget_high_s16(a3a2), 2217); + const int32x4_t a2_p_a3 = vmlal_n_s16(a2_2217, vget_low_s16(a3a2), 5352); + const int32x4_t a3_m_a2 = vmlsl_n_s16(a3_2217, vget_high_s16(a3a2), 5352); + const int16x4_t tmp1 = vaddhn_s32(a2_p_a3, kCst12000); + const int16x4_t out3 = vaddhn_s32(a3_m_a2, kCst51000); + const int16x4_t a3_eq_0 = + vreinterpret_s16_u16(vceq_s16(vget_low_s16(a3a2), vdup_n_s16(0))); + const int16x4_t out1 = vadd_s16(tmp1, a3_eq_0); + vst1_s16(out + 0, out0); + vst1_s16(out + 4, out1); + vst1_s16(out + 8, out2); + vst1_s16(out + 12, out3); + } +} + +#else + +// adapted from vp8/encoder/arm/neon/shortfdct_neon.asm +static const int16_t kCoeff16[] = { + 5352, 5352, 5352, 5352, 2217, 2217, 2217, 2217 +}; +static const int32_t kCoeff32[] = { + 1812, 1812, 1812, 1812, + 937, 937, 937, 937, + 12000, 12000, 12000, 12000, + 51000, 51000, 51000, 51000 +}; + +static void FTransform(const uint8_t* src, const uint8_t* ref, + int16_t* out) { + const int kBPS = BPS; + const uint8_t* src_ptr = src; + const uint8_t* ref_ptr = ref; + const int16_t* coeff16 = kCoeff16; + const int32_t* coeff32 = kCoeff32; + + __asm__ volatile ( + // load src into q4, q5 in high half + "vld1.8 {d8}, [%[src_ptr]], %[kBPS] \n" + "vld1.8 {d10}, [%[src_ptr]], %[kBPS] \n" + "vld1.8 {d9}, [%[src_ptr]], %[kBPS] \n" + "vld1.8 {d11}, [%[src_ptr]] \n" + + // load ref into q6, q7 in high half + "vld1.8 {d12}, [%[ref_ptr]], %[kBPS] \n" + "vld1.8 {d14}, [%[ref_ptr]], %[kBPS] \n" + "vld1.8 {d13}, [%[ref_ptr]], %[kBPS] \n" + "vld1.8 {d15}, [%[ref_ptr]] \n" + + // Pack the high values in to q4 and q6 + "vtrn.32 q4, q5 \n" + "vtrn.32 q6, q7 \n" + + // d[0-3] = src - ref + "vsubl.u8 q0, d8, d12 \n" + "vsubl.u8 q1, d9, d13 \n" + + // load coeff16 into q8(d16=5352, d17=2217) + "vld1.16 {q8}, [%[coeff16]] \n" + + // load coeff32 high half into q9 = 1812, q10 = 937 + "vld1.32 {q9, q10}, [%[coeff32]]! \n" + + // load coeff32 low half into q11=12000, q12=51000 + "vld1.32 {q11,q12}, [%[coeff32]] \n" + + // part 1 + // Transpose. Register dN is the same as dN in C + "vtrn.32 d0, d2 \n" + "vtrn.32 d1, d3 \n" + "vtrn.16 d0, d1 \n" + "vtrn.16 d2, d3 \n" + + "vadd.s16 d4, d0, d3 \n" // a0 = d0 + d3 + "vadd.s16 d5, d1, d2 \n" // a1 = d1 + d2 + "vsub.s16 d6, d1, d2 \n" // a2 = d1 - d2 + "vsub.s16 d7, d0, d3 \n" // a3 = d0 - d3 + + "vadd.s16 d0, d4, d5 \n" // a0 + a1 + "vshl.s16 d0, d0, #3 \n" // temp[0+i*4] = (a0+a1) << 3 + "vsub.s16 d2, d4, d5 \n" // a0 - a1 + "vshl.s16 d2, d2, #3 \n" // (temp[2+i*4] = (a0-a1) << 3 + + "vmlal.s16 q9, d7, d16 \n" // a3*5352 + 1812 + "vmlal.s16 q10, d7, d17 \n" // a3*2217 + 937 + "vmlal.s16 q9, d6, d17 \n" // a2*2217 + a3*5352 + 1812 + "vmlsl.s16 q10, d6, d16 \n" // a3*2217 + 937 - a2*5352 + + // temp[1+i*4] = (d2*2217 + d3*5352 + 1812) >> 9 + // temp[3+i*4] = (d3*2217 + 937 - d2*5352) >> 9 + "vshrn.s32 d1, q9, #9 \n" + "vshrn.s32 d3, q10, #9 \n" + + // part 2 + // transpose d0=ip[0], d1=ip[4], d2=ip[8], d3=ip[12] + "vtrn.32 d0, d2 \n" + "vtrn.32 d1, d3 \n" + "vtrn.16 d0, d1 \n" + "vtrn.16 d2, d3 \n" + + "vmov.s16 d26, #7 \n" + + "vadd.s16 d4, d0, d3 \n" // a1 = ip[0] + ip[12] + "vadd.s16 d5, d1, d2 \n" // b1 = ip[4] + ip[8] + "vsub.s16 d6, d1, d2 \n" // c1 = ip[4] - ip[8] + "vadd.s16 d4, d4, d26 \n" // a1 + 7 + "vsub.s16 d7, d0, d3 \n" // d1 = ip[0] - ip[12] + + "vadd.s16 d0, d4, d5 \n" // op[0] = a1 + b1 + 7 + "vsub.s16 d2, d4, d5 \n" // op[8] = a1 - b1 + 7 + + "vmlal.s16 q11, d7, d16 \n" // d1*5352 + 12000 + "vmlal.s16 q12, d7, d17 \n" // d1*2217 + 51000 + + "vceq.s16 d4, d7, #0 \n" + + "vshr.s16 d0, d0, #4 \n" + "vshr.s16 d2, d2, #4 \n" + + "vmlal.s16 q11, d6, d17 \n" // c1*2217 + d1*5352 + 12000 + "vmlsl.s16 q12, d6, d16 \n" // d1*2217 - c1*5352 + 51000 + + "vmvn d4, d4 \n" // !(d1 == 0) + // op[4] = (c1*2217 + d1*5352 + 12000)>>16 + "vshrn.s32 d1, q11, #16 \n" + // op[4] += (d1!=0) + "vsub.s16 d1, d1, d4 \n" + // op[12]= (d1*2217 - c1*5352 + 51000)>>16 + "vshrn.s32 d3, q12, #16 \n" + + // set result to out array + "vst1.16 {q0, q1}, [%[out]] \n" + : [src_ptr] "+r"(src_ptr), [ref_ptr] "+r"(ref_ptr), + [coeff32] "+r"(coeff32) // modified registers + : [kBPS] "r"(kBPS), [coeff16] "r"(coeff16), + [out] "r"(out) // constants + : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", + "q10", "q11", "q12", "q13" // clobbered + ); +} + +#endif + +#define LOAD_LANE_16b(VALUE, LANE) do { \ + (VALUE) = vld1_lane_s16(src, (VALUE), (LANE)); \ + src += stride; \ +} while (0) + +static void FTransformWHT(const int16_t* src, int16_t* out) { + const int stride = 16; + const int16x4_t zero = vdup_n_s16(0); + int32x4x4_t tmp0; + int16x4x4_t in; + INIT_VECTOR4(in, zero, zero, zero, zero); + LOAD_LANE_16b(in.val[0], 0); + LOAD_LANE_16b(in.val[1], 0); + LOAD_LANE_16b(in.val[2], 0); + LOAD_LANE_16b(in.val[3], 0); + LOAD_LANE_16b(in.val[0], 1); + LOAD_LANE_16b(in.val[1], 1); + LOAD_LANE_16b(in.val[2], 1); + LOAD_LANE_16b(in.val[3], 1); + LOAD_LANE_16b(in.val[0], 2); + LOAD_LANE_16b(in.val[1], 2); + LOAD_LANE_16b(in.val[2], 2); + LOAD_LANE_16b(in.val[3], 2); + LOAD_LANE_16b(in.val[0], 3); + LOAD_LANE_16b(in.val[1], 3); + LOAD_LANE_16b(in.val[2], 3); + LOAD_LANE_16b(in.val[3], 3); + + { + // a0 = in[0 * 16] + in[2 * 16] + // a1 = in[1 * 16] + in[3 * 16] + // a2 = in[1 * 16] - in[3 * 16] + // a3 = in[0 * 16] - in[2 * 16] + const int32x4_t a0 = vaddl_s16(in.val[0], in.val[2]); + const int32x4_t a1 = vaddl_s16(in.val[1], in.val[3]); + const int32x4_t a2 = vsubl_s16(in.val[1], in.val[3]); + const int32x4_t a3 = vsubl_s16(in.val[0], in.val[2]); + tmp0.val[0] = vaddq_s32(a0, a1); + tmp0.val[1] = vaddq_s32(a3, a2); + tmp0.val[2] = vsubq_s32(a3, a2); + tmp0.val[3] = vsubq_s32(a0, a1); + } + { + const int32x4x4_t tmp1 = Transpose4x4(tmp0); + // a0 = tmp[0 + i] + tmp[ 8 + i] + // a1 = tmp[4 + i] + tmp[12 + i] + // a2 = tmp[4 + i] - tmp[12 + i] + // a3 = tmp[0 + i] - tmp[ 8 + i] + const int32x4_t a0 = vaddq_s32(tmp1.val[0], tmp1.val[2]); + const int32x4_t a1 = vaddq_s32(tmp1.val[1], tmp1.val[3]); + const int32x4_t a2 = vsubq_s32(tmp1.val[1], tmp1.val[3]); + const int32x4_t a3 = vsubq_s32(tmp1.val[0], tmp1.val[2]); + const int32x4_t b0 = vhaddq_s32(a0, a1); // (a0 + a1) >> 1 + const int32x4_t b1 = vhaddq_s32(a3, a2); // (a3 + a2) >> 1 + const int32x4_t b2 = vhsubq_s32(a3, a2); // (a3 - a2) >> 1 + const int32x4_t b3 = vhsubq_s32(a0, a1); // (a0 - a1) >> 1 + const int16x4_t out0 = vmovn_s32(b0); + const int16x4_t out1 = vmovn_s32(b1); + const int16x4_t out2 = vmovn_s32(b2); + const int16x4_t out3 = vmovn_s32(b3); + + vst1_s16(out + 0, out0); + vst1_s16(out + 4, out1); + vst1_s16(out + 8, out2); + vst1_s16(out + 12, out3); + } +} +#undef LOAD_LANE_16b + +//------------------------------------------------------------------------------ +// Texture distortion +// +// We try to match the spectral content (weighted) between source and +// reconstructed samples. + +// a 0123, b 0123 +// a 4567, b 4567 +// a 89ab, b 89ab +// a cdef, b cdef +// +// transpose +// +// a 048c, b 048c +// a 159d, b 159d +// a 26ae, b 26ae +// a 37bf, b 37bf +// +static WEBP_INLINE int16x8x4_t DistoTranspose4x4S16(int16x8x4_t q4_in) { + const int16x8x2_t q2_tmp0 = vtrnq_s16(q4_in.val[0], q4_in.val[1]); + const int16x8x2_t q2_tmp1 = vtrnq_s16(q4_in.val[2], q4_in.val[3]); + const int32x4x2_t q2_tmp2 = vtrnq_s32(vreinterpretq_s32_s16(q2_tmp0.val[0]), + vreinterpretq_s32_s16(q2_tmp1.val[0])); + const int32x4x2_t q2_tmp3 = vtrnq_s32(vreinterpretq_s32_s16(q2_tmp0.val[1]), + vreinterpretq_s32_s16(q2_tmp1.val[1])); + q4_in.val[0] = vreinterpretq_s16_s32(q2_tmp2.val[0]); + q4_in.val[2] = vreinterpretq_s16_s32(q2_tmp2.val[1]); + q4_in.val[1] = vreinterpretq_s16_s32(q2_tmp3.val[0]); + q4_in.val[3] = vreinterpretq_s16_s32(q2_tmp3.val[1]); + return q4_in; +} + +static WEBP_INLINE int16x8x4_t DistoHorizontalPass(const int16x8x4_t q4_in) { + // {a0, a1} = {in[0] + in[2], in[1] + in[3]} + // {a3, a2} = {in[0] - in[2], in[1] - in[3]} + const int16x8_t q_a0 = vaddq_s16(q4_in.val[0], q4_in.val[2]); + const int16x8_t q_a1 = vaddq_s16(q4_in.val[1], q4_in.val[3]); + const int16x8_t q_a3 = vsubq_s16(q4_in.val[0], q4_in.val[2]); + const int16x8_t q_a2 = vsubq_s16(q4_in.val[1], q4_in.val[3]); + int16x8x4_t q4_out; + // tmp[0] = a0 + a1 + // tmp[1] = a3 + a2 + // tmp[2] = a3 - a2 + // tmp[3] = a0 - a1 + INIT_VECTOR4(q4_out, + vabsq_s16(vaddq_s16(q_a0, q_a1)), + vabsq_s16(vaddq_s16(q_a3, q_a2)), + vabdq_s16(q_a3, q_a2), vabdq_s16(q_a0, q_a1)); + return q4_out; +} + +static WEBP_INLINE int16x8x4_t DistoVerticalPass(const uint8x8x4_t q4_in) { + const int16x8_t q_a0 = vreinterpretq_s16_u16(vaddl_u8(q4_in.val[0], + q4_in.val[2])); + const int16x8_t q_a1 = vreinterpretq_s16_u16(vaddl_u8(q4_in.val[1], + q4_in.val[3])); + const int16x8_t q_a2 = vreinterpretq_s16_u16(vsubl_u8(q4_in.val[1], + q4_in.val[3])); + const int16x8_t q_a3 = vreinterpretq_s16_u16(vsubl_u8(q4_in.val[0], + q4_in.val[2])); + int16x8x4_t q4_out; + + INIT_VECTOR4(q4_out, + vaddq_s16(q_a0, q_a1), vaddq_s16(q_a3, q_a2), + vsubq_s16(q_a3, q_a2), vsubq_s16(q_a0, q_a1)); + return q4_out; +} + +static WEBP_INLINE int16x4x4_t DistoLoadW(const uint16_t* w) { + const uint16x8_t q_w07 = vld1q_u16(&w[0]); + const uint16x8_t q_w8f = vld1q_u16(&w[8]); + int16x4x4_t d4_w; + INIT_VECTOR4(d4_w, + vget_low_s16(vreinterpretq_s16_u16(q_w07)), + vget_high_s16(vreinterpretq_s16_u16(q_w07)), + vget_low_s16(vreinterpretq_s16_u16(q_w8f)), + vget_high_s16(vreinterpretq_s16_u16(q_w8f))); + return d4_w; +} + +static WEBP_INLINE int32x2_t DistoSum(const int16x8x4_t q4_in, + const int16x4x4_t d4_w) { + int32x2_t d_sum; + // sum += w[ 0] * abs(b0); + // sum += w[ 4] * abs(b1); + // sum += w[ 8] * abs(b2); + // sum += w[12] * abs(b3); + int32x4_t q_sum0 = vmull_s16(d4_w.val[0], vget_low_s16(q4_in.val[0])); + int32x4_t q_sum1 = vmull_s16(d4_w.val[1], vget_low_s16(q4_in.val[1])); + int32x4_t q_sum2 = vmull_s16(d4_w.val[2], vget_low_s16(q4_in.val[2])); + int32x4_t q_sum3 = vmull_s16(d4_w.val[3], vget_low_s16(q4_in.val[3])); + q_sum0 = vmlsl_s16(q_sum0, d4_w.val[0], vget_high_s16(q4_in.val[0])); + q_sum1 = vmlsl_s16(q_sum1, d4_w.val[1], vget_high_s16(q4_in.val[1])); + q_sum2 = vmlsl_s16(q_sum2, d4_w.val[2], vget_high_s16(q4_in.val[2])); + q_sum3 = vmlsl_s16(q_sum3, d4_w.val[3], vget_high_s16(q4_in.val[3])); + + q_sum0 = vaddq_s32(q_sum0, q_sum1); + q_sum2 = vaddq_s32(q_sum2, q_sum3); + q_sum2 = vaddq_s32(q_sum0, q_sum2); + d_sum = vpadd_s32(vget_low_s32(q_sum2), vget_high_s32(q_sum2)); + d_sum = vpadd_s32(d_sum, d_sum); + return d_sum; +} + +#define LOAD_LANE_32b(src, VALUE, LANE) \ + (VALUE) = vld1_lane_u32((const uint32_t*)(src), (VALUE), (LANE)) + +// Hadamard transform +// Returns the weighted sum of the absolute value of transformed coefficients. +// w[] contains a row-major 4 by 4 symmetric matrix. +static int Disto4x4(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + uint32x2_t d_in_ab_0123 = vdup_n_u32(0); + uint32x2_t d_in_ab_4567 = vdup_n_u32(0); + uint32x2_t d_in_ab_89ab = vdup_n_u32(0); + uint32x2_t d_in_ab_cdef = vdup_n_u32(0); + uint8x8x4_t d4_in; + + // load data a, b + LOAD_LANE_32b(a + 0 * BPS, d_in_ab_0123, 0); + LOAD_LANE_32b(a + 1 * BPS, d_in_ab_4567, 0); + LOAD_LANE_32b(a + 2 * BPS, d_in_ab_89ab, 0); + LOAD_LANE_32b(a + 3 * BPS, d_in_ab_cdef, 0); + LOAD_LANE_32b(b + 0 * BPS, d_in_ab_0123, 1); + LOAD_LANE_32b(b + 1 * BPS, d_in_ab_4567, 1); + LOAD_LANE_32b(b + 2 * BPS, d_in_ab_89ab, 1); + LOAD_LANE_32b(b + 3 * BPS, d_in_ab_cdef, 1); + INIT_VECTOR4(d4_in, + vreinterpret_u8_u32(d_in_ab_0123), + vreinterpret_u8_u32(d_in_ab_4567), + vreinterpret_u8_u32(d_in_ab_89ab), + vreinterpret_u8_u32(d_in_ab_cdef)); + + { + // Vertical pass first to avoid a transpose (vertical and horizontal passes + // are commutative because w/kWeightY is symmetric) and subsequent + // transpose. + const int16x8x4_t q4_v = DistoVerticalPass(d4_in); + const int16x4x4_t d4_w = DistoLoadW(w); + // horizontal pass + const int16x8x4_t q4_t = DistoTranspose4x4S16(q4_v); + const int16x8x4_t q4_h = DistoHorizontalPass(q4_t); + int32x2_t d_sum = DistoSum(q4_h, d4_w); + + // abs(sum2 - sum1) >> 5 + d_sum = vabs_s32(d_sum); + d_sum = vshr_n_s32(d_sum, 5); + return vget_lane_s32(d_sum, 0); + } +} +#undef LOAD_LANE_32b + +static int Disto16x16(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + int D = 0; + int x, y; + for (y = 0; y < 16 * BPS; y += 4 * BPS) { + for (x = 0; x < 16; x += 4) { + D += Disto4x4(a + x + y, b + x + y, w); + } + } + return D; +} + +//------------------------------------------------------------------------------ + +static void CollectHistogram(const uint8_t* ref, const uint8_t* pred, + int start_block, int end_block, + VP8Histogram* const histo) { + const uint16x8_t max_coeff_thresh = vdupq_n_u16(MAX_COEFF_THRESH); + int j; + int distribution[MAX_COEFF_THRESH + 1] = { 0 }; + for (j = start_block; j < end_block; ++j) { + int16_t out[16]; + FTransform(ref + VP8DspScan[j], pred + VP8DspScan[j], out); + { + int k; + const int16x8_t a0 = vld1q_s16(out + 0); + const int16x8_t b0 = vld1q_s16(out + 8); + const uint16x8_t a1 = vreinterpretq_u16_s16(vabsq_s16(a0)); + const uint16x8_t b1 = vreinterpretq_u16_s16(vabsq_s16(b0)); + const uint16x8_t a2 = vshrq_n_u16(a1, 3); + const uint16x8_t b2 = vshrq_n_u16(b1, 3); + const uint16x8_t a3 = vminq_u16(a2, max_coeff_thresh); + const uint16x8_t b3 = vminq_u16(b2, max_coeff_thresh); + vst1q_s16(out + 0, vreinterpretq_s16_u16(a3)); + vst1q_s16(out + 8, vreinterpretq_s16_u16(b3)); + // Convert coefficients to bin. + for (k = 0; k < 16; ++k) { + ++distribution[out[k]]; + } + } + } + VP8SetHistogramData(distribution, histo); +} + +//------------------------------------------------------------------------------ + +static WEBP_INLINE void AccumulateSSE16(const uint8_t* const a, + const uint8_t* const b, + uint32x4_t* const sum) { + const uint8x16_t a0 = vld1q_u8(a); + const uint8x16_t b0 = vld1q_u8(b); + const uint8x16_t abs_diff = vabdq_u8(a0, b0); + uint16x8_t prod = vmull_u8(vget_low_u8(abs_diff), vget_low_u8(abs_diff)); + prod = vmlal_u8(prod, vget_high_u8(abs_diff), vget_high_u8(abs_diff)); + *sum = vpadalq_u16(*sum, prod); // pair-wise add and accumulate +} + +// Horizontal sum of all four uint32_t values in 'sum'. +static int SumToInt(uint32x4_t sum) { + const uint64x2_t sum2 = vpaddlq_u32(sum); + const uint64_t sum3 = vgetq_lane_u64(sum2, 0) + vgetq_lane_u64(sum2, 1); + return (int)sum3; +} + +static int SSE16x16(const uint8_t* a, const uint8_t* b) { + uint32x4_t sum = vdupq_n_u32(0); + int y; + for (y = 0; y < 16; ++y) { + AccumulateSSE16(a + y * BPS, b + y * BPS, &sum); + } + return SumToInt(sum); +} + +static int SSE16x8(const uint8_t* a, const uint8_t* b) { + uint32x4_t sum = vdupq_n_u32(0); + int y; + for (y = 0; y < 8; ++y) { + AccumulateSSE16(a + y * BPS, b + y * BPS, &sum); + } + return SumToInt(sum); +} + +static int SSE8x8(const uint8_t* a, const uint8_t* b) { + uint32x4_t sum = vdupq_n_u32(0); + int y; + for (y = 0; y < 8; ++y) { + const uint8x8_t a0 = vld1_u8(a + y * BPS); + const uint8x8_t b0 = vld1_u8(b + y * BPS); + const uint8x8_t abs_diff = vabd_u8(a0, b0); + const uint16x8_t prod = vmull_u8(abs_diff, abs_diff); + sum = vpadalq_u16(sum, prod); + } + return SumToInt(sum); +} + +static int SSE4x4(const uint8_t* a, const uint8_t* b) { + const uint8x16_t a0 = Load4x4(a); + const uint8x16_t b0 = Load4x4(b); + const uint8x16_t abs_diff = vabdq_u8(a0, b0); + uint16x8_t prod = vmull_u8(vget_low_u8(abs_diff), vget_low_u8(abs_diff)); + prod = vmlal_u8(prod, vget_high_u8(abs_diff), vget_high_u8(abs_diff)); + return SumToInt(vpaddlq_u16(prod)); +} + +//------------------------------------------------------------------------------ + +// Compilation with gcc-4.6.x is problematic for now. +#if !defined(WORK_AROUND_GCC) + +static int16x8_t Quantize(int16_t* const in, + const VP8Matrix* const mtx, int offset) { + const uint16x8_t sharp = vld1q_u16(&mtx->sharpen_[offset]); + const uint16x8_t q = vld1q_u16(&mtx->q_[offset]); + const uint16x8_t iq = vld1q_u16(&mtx->iq_[offset]); + const uint32x4_t bias0 = vld1q_u32(&mtx->bias_[offset + 0]); + const uint32x4_t bias1 = vld1q_u32(&mtx->bias_[offset + 4]); + + const int16x8_t a = vld1q_s16(in + offset); // in + const uint16x8_t b = vreinterpretq_u16_s16(vabsq_s16(a)); // coeff = abs(in) + const int16x8_t sign = vshrq_n_s16(a, 15); // sign + const uint16x8_t c = vaddq_u16(b, sharp); // + sharpen + const uint32x4_t m0 = vmull_u16(vget_low_u16(c), vget_low_u16(iq)); + const uint32x4_t m1 = vmull_u16(vget_high_u16(c), vget_high_u16(iq)); + const uint32x4_t m2 = vhaddq_u32(m0, bias0); + const uint32x4_t m3 = vhaddq_u32(m1, bias1); // (coeff * iQ + bias) >> 1 + const uint16x8_t c0 = vcombine_u16(vshrn_n_u32(m2, 16), + vshrn_n_u32(m3, 16)); // QFIX=17 = 16+1 + const uint16x8_t c1 = vminq_u16(c0, vdupq_n_u16(MAX_LEVEL)); + const int16x8_t c2 = veorq_s16(vreinterpretq_s16_u16(c1), sign); + const int16x8_t c3 = vsubq_s16(c2, sign); // restore sign + const int16x8_t c4 = vmulq_s16(c3, vreinterpretq_s16_u16(q)); + vst1q_s16(in + offset, c4); + assert(QFIX == 17); // this function can't work as is if QFIX != 16+1 + return c3; +} + +static const uint8_t kShuffles[4][8] = { + { 0, 1, 2, 3, 8, 9, 16, 17 }, + { 10, 11, 4, 5, 6, 7, 12, 13 }, + { 18, 19, 24, 25, 26, 27, 20, 21 }, + { 14, 15, 22, 23, 28, 29, 30, 31 } +}; + +static int QuantizeBlock(int16_t in[16], int16_t out[16], + const VP8Matrix* const mtx) { + const int16x8_t out0 = Quantize(in, mtx, 0); + const int16x8_t out1 = Quantize(in, mtx, 8); + uint8x8x4_t shuffles; + // vtbl?_u8 are marked unavailable for iOS arm64 with Xcode < 6.3, use + // non-standard versions there. +#if defined(__APPLE__) && defined(__aarch64__) && \ + defined(__apple_build_version__) && (__apple_build_version__< 6020037) + uint8x16x2_t all_out; + INIT_VECTOR2(all_out, vreinterpretq_u8_s16(out0), vreinterpretq_u8_s16(out1)); + INIT_VECTOR4(shuffles, + vtbl2q_u8(all_out, vld1_u8(kShuffles[0])), + vtbl2q_u8(all_out, vld1_u8(kShuffles[1])), + vtbl2q_u8(all_out, vld1_u8(kShuffles[2])), + vtbl2q_u8(all_out, vld1_u8(kShuffles[3]))); +#else + uint8x8x4_t all_out; + INIT_VECTOR4(all_out, + vreinterpret_u8_s16(vget_low_s16(out0)), + vreinterpret_u8_s16(vget_high_s16(out0)), + vreinterpret_u8_s16(vget_low_s16(out1)), + vreinterpret_u8_s16(vget_high_s16(out1))); + INIT_VECTOR4(shuffles, + vtbl4_u8(all_out, vld1_u8(kShuffles[0])), + vtbl4_u8(all_out, vld1_u8(kShuffles[1])), + vtbl4_u8(all_out, vld1_u8(kShuffles[2])), + vtbl4_u8(all_out, vld1_u8(kShuffles[3]))); +#endif + // Zigzag reordering + vst1_u8((uint8_t*)(out + 0), shuffles.val[0]); + vst1_u8((uint8_t*)(out + 4), shuffles.val[1]); + vst1_u8((uint8_t*)(out + 8), shuffles.val[2]); + vst1_u8((uint8_t*)(out + 12), shuffles.val[3]); + // test zeros + if (*(uint64_t*)(out + 0) != 0) return 1; + if (*(uint64_t*)(out + 4) != 0) return 1; + if (*(uint64_t*)(out + 8) != 0) return 1; + if (*(uint64_t*)(out + 12) != 0) return 1; + return 0; +} + +static int Quantize2Blocks(int16_t in[32], int16_t out[32], + const VP8Matrix* const mtx) { + int nz; + nz = QuantizeBlock(in + 0 * 16, out + 0 * 16, mtx) << 0; + nz |= QuantizeBlock(in + 1 * 16, out + 1 * 16, mtx) << 1; + return nz; +} + +#endif // !WORK_AROUND_GCC + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8EncDspInitNEON(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitNEON(void) { + VP8ITransform = ITransform; + VP8FTransform = FTransform; + + VP8FTransformWHT = FTransformWHT; + + VP8TDisto4x4 = Disto4x4; + VP8TDisto16x16 = Disto16x16; + VP8CollectHistogram = CollectHistogram; + VP8SSE16x16 = SSE16x16; + VP8SSE16x8 = SSE16x8; + VP8SSE8x8 = SSE8x8; + VP8SSE4x4 = SSE4x4; +#if !defined(WORK_AROUND_GCC) + VP8EncQuantizeBlock = QuantizeBlock; + VP8EncQuantize2Blocks = Quantize2Blocks; +#endif +} + +#else // !WEBP_USE_NEON + +WEBP_DSP_INIT_STUB(VP8EncDspInitNEON) + +#endif // WEBP_USE_NEON diff --git a/thirdparty/libwebp/dsp/enc_sse2.c b/thirdparty/libwebp/dsp/enc_sse2.c new file mode 100644 index 0000000000..4a2e3ce14f --- /dev/null +++ b/thirdparty/libwebp/dsp/enc_sse2.c @@ -0,0 +1,1340 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE2 version of speed-critical encoding functions. +// +// Author: Christian Duvivier (cduvivier@google.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE2) +#include <stdlib.h> // for abs() +#include <emmintrin.h> + +#include "./common_sse2.h" +#include "../enc/cost.h" +#include "../enc/vp8enci.h" + +//------------------------------------------------------------------------------ +// Transforms (Paragraph 14.4) + +// Does one or two inverse transforms. +static void ITransform(const uint8_t* ref, const int16_t* in, uint8_t* dst, + int do_two) { + // This implementation makes use of 16-bit fixed point versions of two + // multiply constants: + // K1 = sqrt(2) * cos (pi/8) ~= 85627 / 2^16 + // K2 = sqrt(2) * sin (pi/8) ~= 35468 / 2^16 + // + // To be able to use signed 16-bit integers, we use the following trick to + // have constants within range: + // - Associated constants are obtained by subtracting the 16-bit fixed point + // version of one: + // k = K - (1 << 16) => K = k + (1 << 16) + // K1 = 85267 => k1 = 20091 + // K2 = 35468 => k2 = -30068 + // - The multiplication of a variable by a constant become the sum of the + // variable and the multiplication of that variable by the associated + // constant: + // (x * K) >> 16 = (x * (k + (1 << 16))) >> 16 = ((x * k ) >> 16) + x + const __m128i k1 = _mm_set1_epi16(20091); + const __m128i k2 = _mm_set1_epi16(-30068); + __m128i T0, T1, T2, T3; + + // Load and concatenate the transform coefficients (we'll do two inverse + // transforms in parallel). In the case of only one inverse transform, the + // second half of the vectors will just contain random value we'll never + // use nor store. + __m128i in0, in1, in2, in3; + { + in0 = _mm_loadl_epi64((const __m128i*)&in[0]); + in1 = _mm_loadl_epi64((const __m128i*)&in[4]); + in2 = _mm_loadl_epi64((const __m128i*)&in[8]); + in3 = _mm_loadl_epi64((const __m128i*)&in[12]); + // a00 a10 a20 a30 x x x x + // a01 a11 a21 a31 x x x x + // a02 a12 a22 a32 x x x x + // a03 a13 a23 a33 x x x x + if (do_two) { + const __m128i inB0 = _mm_loadl_epi64((const __m128i*)&in[16]); + const __m128i inB1 = _mm_loadl_epi64((const __m128i*)&in[20]); + const __m128i inB2 = _mm_loadl_epi64((const __m128i*)&in[24]); + const __m128i inB3 = _mm_loadl_epi64((const __m128i*)&in[28]); + in0 = _mm_unpacklo_epi64(in0, inB0); + in1 = _mm_unpacklo_epi64(in1, inB1); + in2 = _mm_unpacklo_epi64(in2, inB2); + in3 = _mm_unpacklo_epi64(in3, inB3); + // a00 a10 a20 a30 b00 b10 b20 b30 + // a01 a11 a21 a31 b01 b11 b21 b31 + // a02 a12 a22 a32 b02 b12 b22 b32 + // a03 a13 a23 a33 b03 b13 b23 b33 + } + } + + // Vertical pass and subsequent transpose. + { + // First pass, c and d calculations are longer because of the "trick" + // multiplications. + const __m128i a = _mm_add_epi16(in0, in2); + const __m128i b = _mm_sub_epi16(in0, in2); + // c = MUL(in1, K2) - MUL(in3, K1) = MUL(in1, k2) - MUL(in3, k1) + in1 - in3 + const __m128i c1 = _mm_mulhi_epi16(in1, k2); + const __m128i c2 = _mm_mulhi_epi16(in3, k1); + const __m128i c3 = _mm_sub_epi16(in1, in3); + const __m128i c4 = _mm_sub_epi16(c1, c2); + const __m128i c = _mm_add_epi16(c3, c4); + // d = MUL(in1, K1) + MUL(in3, K2) = MUL(in1, k1) + MUL(in3, k2) + in1 + in3 + const __m128i d1 = _mm_mulhi_epi16(in1, k1); + const __m128i d2 = _mm_mulhi_epi16(in3, k2); + const __m128i d3 = _mm_add_epi16(in1, in3); + const __m128i d4 = _mm_add_epi16(d1, d2); + const __m128i d = _mm_add_epi16(d3, d4); + + // Second pass. + const __m128i tmp0 = _mm_add_epi16(a, d); + const __m128i tmp1 = _mm_add_epi16(b, c); + const __m128i tmp2 = _mm_sub_epi16(b, c); + const __m128i tmp3 = _mm_sub_epi16(a, d); + + // Transpose the two 4x4. + VP8Transpose_2_4x4_16b(&tmp0, &tmp1, &tmp2, &tmp3, &T0, &T1, &T2, &T3); + } + + // Horizontal pass and subsequent transpose. + { + // First pass, c and d calculations are longer because of the "trick" + // multiplications. + const __m128i four = _mm_set1_epi16(4); + const __m128i dc = _mm_add_epi16(T0, four); + const __m128i a = _mm_add_epi16(dc, T2); + const __m128i b = _mm_sub_epi16(dc, T2); + // c = MUL(T1, K2) - MUL(T3, K1) = MUL(T1, k2) - MUL(T3, k1) + T1 - T3 + const __m128i c1 = _mm_mulhi_epi16(T1, k2); + const __m128i c2 = _mm_mulhi_epi16(T3, k1); + const __m128i c3 = _mm_sub_epi16(T1, T3); + const __m128i c4 = _mm_sub_epi16(c1, c2); + const __m128i c = _mm_add_epi16(c3, c4); + // d = MUL(T1, K1) + MUL(T3, K2) = MUL(T1, k1) + MUL(T3, k2) + T1 + T3 + const __m128i d1 = _mm_mulhi_epi16(T1, k1); + const __m128i d2 = _mm_mulhi_epi16(T3, k2); + const __m128i d3 = _mm_add_epi16(T1, T3); + const __m128i d4 = _mm_add_epi16(d1, d2); + const __m128i d = _mm_add_epi16(d3, d4); + + // Second pass. + const __m128i tmp0 = _mm_add_epi16(a, d); + const __m128i tmp1 = _mm_add_epi16(b, c); + const __m128i tmp2 = _mm_sub_epi16(b, c); + const __m128i tmp3 = _mm_sub_epi16(a, d); + const __m128i shifted0 = _mm_srai_epi16(tmp0, 3); + const __m128i shifted1 = _mm_srai_epi16(tmp1, 3); + const __m128i shifted2 = _mm_srai_epi16(tmp2, 3); + const __m128i shifted3 = _mm_srai_epi16(tmp3, 3); + + // Transpose the two 4x4. + VP8Transpose_2_4x4_16b(&shifted0, &shifted1, &shifted2, &shifted3, &T0, &T1, + &T2, &T3); + } + + // Add inverse transform to 'ref' and store. + { + const __m128i zero = _mm_setzero_si128(); + // Load the reference(s). + __m128i ref0, ref1, ref2, ref3; + if (do_two) { + // Load eight bytes/pixels per line. + ref0 = _mm_loadl_epi64((const __m128i*)&ref[0 * BPS]); + ref1 = _mm_loadl_epi64((const __m128i*)&ref[1 * BPS]); + ref2 = _mm_loadl_epi64((const __m128i*)&ref[2 * BPS]); + ref3 = _mm_loadl_epi64((const __m128i*)&ref[3 * BPS]); + } else { + // Load four bytes/pixels per line. + ref0 = _mm_cvtsi32_si128(WebPMemToUint32(&ref[0 * BPS])); + ref1 = _mm_cvtsi32_si128(WebPMemToUint32(&ref[1 * BPS])); + ref2 = _mm_cvtsi32_si128(WebPMemToUint32(&ref[2 * BPS])); + ref3 = _mm_cvtsi32_si128(WebPMemToUint32(&ref[3 * BPS])); + } + // Convert to 16b. + ref0 = _mm_unpacklo_epi8(ref0, zero); + ref1 = _mm_unpacklo_epi8(ref1, zero); + ref2 = _mm_unpacklo_epi8(ref2, zero); + ref3 = _mm_unpacklo_epi8(ref3, zero); + // Add the inverse transform(s). + ref0 = _mm_add_epi16(ref0, T0); + ref1 = _mm_add_epi16(ref1, T1); + ref2 = _mm_add_epi16(ref2, T2); + ref3 = _mm_add_epi16(ref3, T3); + // Unsigned saturate to 8b. + ref0 = _mm_packus_epi16(ref0, ref0); + ref1 = _mm_packus_epi16(ref1, ref1); + ref2 = _mm_packus_epi16(ref2, ref2); + ref3 = _mm_packus_epi16(ref3, ref3); + // Store the results. + if (do_two) { + // Store eight bytes/pixels per line. + _mm_storel_epi64((__m128i*)&dst[0 * BPS], ref0); + _mm_storel_epi64((__m128i*)&dst[1 * BPS], ref1); + _mm_storel_epi64((__m128i*)&dst[2 * BPS], ref2); + _mm_storel_epi64((__m128i*)&dst[3 * BPS], ref3); + } else { + // Store four bytes/pixels per line. + WebPUint32ToMem(&dst[0 * BPS], _mm_cvtsi128_si32(ref0)); + WebPUint32ToMem(&dst[1 * BPS], _mm_cvtsi128_si32(ref1)); + WebPUint32ToMem(&dst[2 * BPS], _mm_cvtsi128_si32(ref2)); + WebPUint32ToMem(&dst[3 * BPS], _mm_cvtsi128_si32(ref3)); + } + } +} + +static void FTransformPass1(const __m128i* const in01, + const __m128i* const in23, + __m128i* const out01, + __m128i* const out32) { + const __m128i k937 = _mm_set1_epi32(937); + const __m128i k1812 = _mm_set1_epi32(1812); + + const __m128i k88p = _mm_set_epi16(8, 8, 8, 8, 8, 8, 8, 8); + const __m128i k88m = _mm_set_epi16(-8, 8, -8, 8, -8, 8, -8, 8); + const __m128i k5352_2217p = _mm_set_epi16(2217, 5352, 2217, 5352, + 2217, 5352, 2217, 5352); + const __m128i k5352_2217m = _mm_set_epi16(-5352, 2217, -5352, 2217, + -5352, 2217, -5352, 2217); + + // *in01 = 00 01 10 11 02 03 12 13 + // *in23 = 20 21 30 31 22 23 32 33 + const __m128i shuf01_p = _mm_shufflehi_epi16(*in01, _MM_SHUFFLE(2, 3, 0, 1)); + const __m128i shuf23_p = _mm_shufflehi_epi16(*in23, _MM_SHUFFLE(2, 3, 0, 1)); + // 00 01 10 11 03 02 13 12 + // 20 21 30 31 23 22 33 32 + const __m128i s01 = _mm_unpacklo_epi64(shuf01_p, shuf23_p); + const __m128i s32 = _mm_unpackhi_epi64(shuf01_p, shuf23_p); + // 00 01 10 11 20 21 30 31 + // 03 02 13 12 23 22 33 32 + const __m128i a01 = _mm_add_epi16(s01, s32); + const __m128i a32 = _mm_sub_epi16(s01, s32); + // [d0 + d3 | d1 + d2 | ...] = [a0 a1 | a0' a1' | ... ] + // [d0 - d3 | d1 - d2 | ...] = [a3 a2 | a3' a2' | ... ] + + const __m128i tmp0 = _mm_madd_epi16(a01, k88p); // [ (a0 + a1) << 3, ... ] + const __m128i tmp2 = _mm_madd_epi16(a01, k88m); // [ (a0 - a1) << 3, ... ] + const __m128i tmp1_1 = _mm_madd_epi16(a32, k5352_2217p); + const __m128i tmp3_1 = _mm_madd_epi16(a32, k5352_2217m); + const __m128i tmp1_2 = _mm_add_epi32(tmp1_1, k1812); + const __m128i tmp3_2 = _mm_add_epi32(tmp3_1, k937); + const __m128i tmp1 = _mm_srai_epi32(tmp1_2, 9); + const __m128i tmp3 = _mm_srai_epi32(tmp3_2, 9); + const __m128i s03 = _mm_packs_epi32(tmp0, tmp2); + const __m128i s12 = _mm_packs_epi32(tmp1, tmp3); + const __m128i s_lo = _mm_unpacklo_epi16(s03, s12); // 0 1 0 1 0 1... + const __m128i s_hi = _mm_unpackhi_epi16(s03, s12); // 2 3 2 3 2 3 + const __m128i v23 = _mm_unpackhi_epi32(s_lo, s_hi); + *out01 = _mm_unpacklo_epi32(s_lo, s_hi); + *out32 = _mm_shuffle_epi32(v23, _MM_SHUFFLE(1, 0, 3, 2)); // 3 2 3 2 3 2.. +} + +static void FTransformPass2(const __m128i* const v01, const __m128i* const v32, + int16_t* out) { + const __m128i zero = _mm_setzero_si128(); + const __m128i seven = _mm_set1_epi16(7); + const __m128i k5352_2217 = _mm_set_epi16(5352, 2217, 5352, 2217, + 5352, 2217, 5352, 2217); + const __m128i k2217_5352 = _mm_set_epi16(2217, -5352, 2217, -5352, + 2217, -5352, 2217, -5352); + const __m128i k12000_plus_one = _mm_set1_epi32(12000 + (1 << 16)); + const __m128i k51000 = _mm_set1_epi32(51000); + + // Same operations are done on the (0,3) and (1,2) pairs. + // a0 = v0 + v3 + // a1 = v1 + v2 + // a3 = v0 - v3 + // a2 = v1 - v2 + const __m128i a01 = _mm_add_epi16(*v01, *v32); + const __m128i a32 = _mm_sub_epi16(*v01, *v32); + const __m128i a11 = _mm_unpackhi_epi64(a01, a01); + const __m128i a22 = _mm_unpackhi_epi64(a32, a32); + const __m128i a01_plus_7 = _mm_add_epi16(a01, seven); + + // d0 = (a0 + a1 + 7) >> 4; + // d2 = (a0 - a1 + 7) >> 4; + const __m128i c0 = _mm_add_epi16(a01_plus_7, a11); + const __m128i c2 = _mm_sub_epi16(a01_plus_7, a11); + const __m128i d0 = _mm_srai_epi16(c0, 4); + const __m128i d2 = _mm_srai_epi16(c2, 4); + + // f1 = ((b3 * 5352 + b2 * 2217 + 12000) >> 16) + // f3 = ((b3 * 2217 - b2 * 5352 + 51000) >> 16) + const __m128i b23 = _mm_unpacklo_epi16(a22, a32); + const __m128i c1 = _mm_madd_epi16(b23, k5352_2217); + const __m128i c3 = _mm_madd_epi16(b23, k2217_5352); + const __m128i d1 = _mm_add_epi32(c1, k12000_plus_one); + const __m128i d3 = _mm_add_epi32(c3, k51000); + const __m128i e1 = _mm_srai_epi32(d1, 16); + const __m128i e3 = _mm_srai_epi32(d3, 16); + const __m128i f1 = _mm_packs_epi32(e1, e1); + const __m128i f3 = _mm_packs_epi32(e3, e3); + // f1 = f1 + (a3 != 0); + // The compare will return (0xffff, 0) for (==0, !=0). To turn that into the + // desired (0, 1), we add one earlier through k12000_plus_one. + // -> f1 = f1 + 1 - (a3 == 0) + const __m128i g1 = _mm_add_epi16(f1, _mm_cmpeq_epi16(a32, zero)); + + const __m128i d0_g1 = _mm_unpacklo_epi64(d0, g1); + const __m128i d2_f3 = _mm_unpacklo_epi64(d2, f3); + _mm_storeu_si128((__m128i*)&out[0], d0_g1); + _mm_storeu_si128((__m128i*)&out[8], d2_f3); +} + +static void FTransform(const uint8_t* src, const uint8_t* ref, int16_t* out) { + const __m128i zero = _mm_setzero_si128(); + // Load src. + const __m128i src0 = _mm_loadl_epi64((const __m128i*)&src[0 * BPS]); + const __m128i src1 = _mm_loadl_epi64((const __m128i*)&src[1 * BPS]); + const __m128i src2 = _mm_loadl_epi64((const __m128i*)&src[2 * BPS]); + const __m128i src3 = _mm_loadl_epi64((const __m128i*)&src[3 * BPS]); + // 00 01 02 03 * + // 10 11 12 13 * + // 20 21 22 23 * + // 30 31 32 33 * + // Shuffle. + const __m128i src_0 = _mm_unpacklo_epi16(src0, src1); + const __m128i src_1 = _mm_unpacklo_epi16(src2, src3); + // 00 01 10 11 02 03 12 13 * * ... + // 20 21 30 31 22 22 32 33 * * ... + + // Load ref. + const __m128i ref0 = _mm_loadl_epi64((const __m128i*)&ref[0 * BPS]); + const __m128i ref1 = _mm_loadl_epi64((const __m128i*)&ref[1 * BPS]); + const __m128i ref2 = _mm_loadl_epi64((const __m128i*)&ref[2 * BPS]); + const __m128i ref3 = _mm_loadl_epi64((const __m128i*)&ref[3 * BPS]); + const __m128i ref_0 = _mm_unpacklo_epi16(ref0, ref1); + const __m128i ref_1 = _mm_unpacklo_epi16(ref2, ref3); + + // Convert both to 16 bit. + const __m128i src_0_16b = _mm_unpacklo_epi8(src_0, zero); + const __m128i src_1_16b = _mm_unpacklo_epi8(src_1, zero); + const __m128i ref_0_16b = _mm_unpacklo_epi8(ref_0, zero); + const __m128i ref_1_16b = _mm_unpacklo_epi8(ref_1, zero); + + // Compute the difference. + const __m128i row01 = _mm_sub_epi16(src_0_16b, ref_0_16b); + const __m128i row23 = _mm_sub_epi16(src_1_16b, ref_1_16b); + __m128i v01, v32; + + // First pass + FTransformPass1(&row01, &row23, &v01, &v32); + + // Second pass + FTransformPass2(&v01, &v32, out); +} + +static void FTransform2(const uint8_t* src, const uint8_t* ref, int16_t* out) { + const __m128i zero = _mm_setzero_si128(); + + // Load src and convert to 16b. + const __m128i src0 = _mm_loadl_epi64((const __m128i*)&src[0 * BPS]); + const __m128i src1 = _mm_loadl_epi64((const __m128i*)&src[1 * BPS]); + const __m128i src2 = _mm_loadl_epi64((const __m128i*)&src[2 * BPS]); + const __m128i src3 = _mm_loadl_epi64((const __m128i*)&src[3 * BPS]); + const __m128i src_0 = _mm_unpacklo_epi8(src0, zero); + const __m128i src_1 = _mm_unpacklo_epi8(src1, zero); + const __m128i src_2 = _mm_unpacklo_epi8(src2, zero); + const __m128i src_3 = _mm_unpacklo_epi8(src3, zero); + // Load ref and convert to 16b. + const __m128i ref0 = _mm_loadl_epi64((const __m128i*)&ref[0 * BPS]); + const __m128i ref1 = _mm_loadl_epi64((const __m128i*)&ref[1 * BPS]); + const __m128i ref2 = _mm_loadl_epi64((const __m128i*)&ref[2 * BPS]); + const __m128i ref3 = _mm_loadl_epi64((const __m128i*)&ref[3 * BPS]); + const __m128i ref_0 = _mm_unpacklo_epi8(ref0, zero); + const __m128i ref_1 = _mm_unpacklo_epi8(ref1, zero); + const __m128i ref_2 = _mm_unpacklo_epi8(ref2, zero); + const __m128i ref_3 = _mm_unpacklo_epi8(ref3, zero); + // Compute difference. -> 00 01 02 03 00' 01' 02' 03' + const __m128i diff0 = _mm_sub_epi16(src_0, ref_0); + const __m128i diff1 = _mm_sub_epi16(src_1, ref_1); + const __m128i diff2 = _mm_sub_epi16(src_2, ref_2); + const __m128i diff3 = _mm_sub_epi16(src_3, ref_3); + + // Unpack and shuffle + // 00 01 02 03 0 0 0 0 + // 10 11 12 13 0 0 0 0 + // 20 21 22 23 0 0 0 0 + // 30 31 32 33 0 0 0 0 + const __m128i shuf01l = _mm_unpacklo_epi32(diff0, diff1); + const __m128i shuf23l = _mm_unpacklo_epi32(diff2, diff3); + const __m128i shuf01h = _mm_unpackhi_epi32(diff0, diff1); + const __m128i shuf23h = _mm_unpackhi_epi32(diff2, diff3); + __m128i v01l, v32l; + __m128i v01h, v32h; + + // First pass + FTransformPass1(&shuf01l, &shuf23l, &v01l, &v32l); + FTransformPass1(&shuf01h, &shuf23h, &v01h, &v32h); + + // Second pass + FTransformPass2(&v01l, &v32l, out + 0); + FTransformPass2(&v01h, &v32h, out + 16); +} + +static void FTransformWHTRow(const int16_t* const in, __m128i* const out) { + const __m128i kMult = _mm_set_epi16(-1, 1, -1, 1, 1, 1, 1, 1); + const __m128i src0 = _mm_loadl_epi64((__m128i*)&in[0 * 16]); + const __m128i src1 = _mm_loadl_epi64((__m128i*)&in[1 * 16]); + const __m128i src2 = _mm_loadl_epi64((__m128i*)&in[2 * 16]); + const __m128i src3 = _mm_loadl_epi64((__m128i*)&in[3 * 16]); + const __m128i A01 = _mm_unpacklo_epi16(src0, src1); // A0 A1 | ... + const __m128i A23 = _mm_unpacklo_epi16(src2, src3); // A2 A3 | ... + const __m128i B0 = _mm_adds_epi16(A01, A23); // a0 | a1 | ... + const __m128i B1 = _mm_subs_epi16(A01, A23); // a3 | a2 | ... + const __m128i C0 = _mm_unpacklo_epi32(B0, B1); // a0 | a1 | a3 | a2 | ... + const __m128i C1 = _mm_unpacklo_epi32(B1, B0); // a3 | a2 | a0 | a1 | ... + const __m128i D = _mm_unpacklo_epi64(C0, C1); // a0 a1 a3 a2 a3 a2 a0 a1 + *out = _mm_madd_epi16(D, kMult); +} + +static void FTransformWHT(const int16_t* in, int16_t* out) { + // Input is 12b signed. + __m128i row0, row1, row2, row3; + // Rows are 14b signed. + FTransformWHTRow(in + 0 * 64, &row0); + FTransformWHTRow(in + 1 * 64, &row1); + FTransformWHTRow(in + 2 * 64, &row2); + FTransformWHTRow(in + 3 * 64, &row3); + + { + // The a* are 15b signed. + const __m128i a0 = _mm_add_epi32(row0, row2); + const __m128i a1 = _mm_add_epi32(row1, row3); + const __m128i a2 = _mm_sub_epi32(row1, row3); + const __m128i a3 = _mm_sub_epi32(row0, row2); + const __m128i a0a3 = _mm_packs_epi32(a0, a3); + const __m128i a1a2 = _mm_packs_epi32(a1, a2); + + // The b* are 16b signed. + const __m128i b0b1 = _mm_add_epi16(a0a3, a1a2); + const __m128i b3b2 = _mm_sub_epi16(a0a3, a1a2); + const __m128i tmp_b2b3 = _mm_unpackhi_epi64(b3b2, b3b2); + const __m128i b2b3 = _mm_unpacklo_epi64(tmp_b2b3, b3b2); + + _mm_storeu_si128((__m128i*)&out[0], _mm_srai_epi16(b0b1, 1)); + _mm_storeu_si128((__m128i*)&out[8], _mm_srai_epi16(b2b3, 1)); + } +} + +//------------------------------------------------------------------------------ +// Compute susceptibility based on DCT-coeff histograms: +// the higher, the "easier" the macroblock is to compress. + +static void CollectHistogram(const uint8_t* ref, const uint8_t* pred, + int start_block, int end_block, + VP8Histogram* const histo) { + const __m128i zero = _mm_setzero_si128(); + const __m128i max_coeff_thresh = _mm_set1_epi16(MAX_COEFF_THRESH); + int j; + int distribution[MAX_COEFF_THRESH + 1] = { 0 }; + for (j = start_block; j < end_block; ++j) { + int16_t out[16]; + int k; + + FTransform(ref + VP8DspScan[j], pred + VP8DspScan[j], out); + + // Convert coefficients to bin (within out[]). + { + // Load. + const __m128i out0 = _mm_loadu_si128((__m128i*)&out[0]); + const __m128i out1 = _mm_loadu_si128((__m128i*)&out[8]); + const __m128i d0 = _mm_sub_epi16(zero, out0); + const __m128i d1 = _mm_sub_epi16(zero, out1); + const __m128i abs0 = _mm_max_epi16(out0, d0); // abs(v), 16b + const __m128i abs1 = _mm_max_epi16(out1, d1); + // v = abs(out) >> 3 + const __m128i v0 = _mm_srai_epi16(abs0, 3); + const __m128i v1 = _mm_srai_epi16(abs1, 3); + // bin = min(v, MAX_COEFF_THRESH) + const __m128i bin0 = _mm_min_epi16(v0, max_coeff_thresh); + const __m128i bin1 = _mm_min_epi16(v1, max_coeff_thresh); + // Store. + _mm_storeu_si128((__m128i*)&out[0], bin0); + _mm_storeu_si128((__m128i*)&out[8], bin1); + } + + // Convert coefficients to bin. + for (k = 0; k < 16; ++k) { + ++distribution[out[k]]; + } + } + VP8SetHistogramData(distribution, histo); +} + +//------------------------------------------------------------------------------ +// Intra predictions + +// helper for chroma-DC predictions +static WEBP_INLINE void Put8x8uv(uint8_t v, uint8_t* dst) { + int j; + const __m128i values = _mm_set1_epi8(v); + for (j = 0; j < 8; ++j) { + _mm_storel_epi64((__m128i*)(dst + j * BPS), values); + } +} + +static WEBP_INLINE void Put16(uint8_t v, uint8_t* dst) { + int j; + const __m128i values = _mm_set1_epi8(v); + for (j = 0; j < 16; ++j) { + _mm_store_si128((__m128i*)(dst + j * BPS), values); + } +} + +static WEBP_INLINE void Fill(uint8_t* dst, int value, int size) { + if (size == 4) { + int j; + for (j = 0; j < 4; ++j) { + memset(dst + j * BPS, value, 4); + } + } else if (size == 8) { + Put8x8uv(value, dst); + } else { + Put16(value, dst); + } +} + +static WEBP_INLINE void VE8uv(uint8_t* dst, const uint8_t* top) { + int j; + const __m128i top_values = _mm_loadl_epi64((const __m128i*)top); + for (j = 0; j < 8; ++j) { + _mm_storel_epi64((__m128i*)(dst + j * BPS), top_values); + } +} + +static WEBP_INLINE void VE16(uint8_t* dst, const uint8_t* top) { + const __m128i top_values = _mm_load_si128((const __m128i*)top); + int j; + for (j = 0; j < 16; ++j) { + _mm_store_si128((__m128i*)(dst + j * BPS), top_values); + } +} + +static WEBP_INLINE void VerticalPred(uint8_t* dst, + const uint8_t* top, int size) { + if (top != NULL) { + if (size == 8) { + VE8uv(dst, top); + } else { + VE16(dst, top); + } + } else { + Fill(dst, 127, size); + } +} + +static WEBP_INLINE void HE8uv(uint8_t* dst, const uint8_t* left) { + int j; + for (j = 0; j < 8; ++j) { + const __m128i values = _mm_set1_epi8(left[j]); + _mm_storel_epi64((__m128i*)dst, values); + dst += BPS; + } +} + +static WEBP_INLINE void HE16(uint8_t* dst, const uint8_t* left) { + int j; + for (j = 0; j < 16; ++j) { + const __m128i values = _mm_set1_epi8(left[j]); + _mm_store_si128((__m128i*)dst, values); + dst += BPS; + } +} + +static WEBP_INLINE void HorizontalPred(uint8_t* dst, + const uint8_t* left, int size) { + if (left != NULL) { + if (size == 8) { + HE8uv(dst, left); + } else { + HE16(dst, left); + } + } else { + Fill(dst, 129, size); + } +} + +static WEBP_INLINE void TM(uint8_t* dst, const uint8_t* left, + const uint8_t* top, int size) { + const __m128i zero = _mm_setzero_si128(); + int y; + if (size == 8) { + const __m128i top_values = _mm_loadl_epi64((const __m128i*)top); + const __m128i top_base = _mm_unpacklo_epi8(top_values, zero); + for (y = 0; y < 8; ++y, dst += BPS) { + const int val = left[y] - left[-1]; + const __m128i base = _mm_set1_epi16(val); + const __m128i out = _mm_packus_epi16(_mm_add_epi16(base, top_base), zero); + _mm_storel_epi64((__m128i*)dst, out); + } + } else { + const __m128i top_values = _mm_load_si128((const __m128i*)top); + const __m128i top_base_0 = _mm_unpacklo_epi8(top_values, zero); + const __m128i top_base_1 = _mm_unpackhi_epi8(top_values, zero); + for (y = 0; y < 16; ++y, dst += BPS) { + const int val = left[y] - left[-1]; + const __m128i base = _mm_set1_epi16(val); + const __m128i out_0 = _mm_add_epi16(base, top_base_0); + const __m128i out_1 = _mm_add_epi16(base, top_base_1); + const __m128i out = _mm_packus_epi16(out_0, out_1); + _mm_store_si128((__m128i*)dst, out); + } + } +} + +static WEBP_INLINE void TrueMotion(uint8_t* dst, const uint8_t* left, + const uint8_t* top, int size) { + if (left != NULL) { + if (top != NULL) { + TM(dst, left, top, size); + } else { + HorizontalPred(dst, left, size); + } + } else { + // true motion without left samples (hence: with default 129 value) + // is equivalent to VE prediction where you just copy the top samples. + // Note that if top samples are not available, the default value is + // then 129, and not 127 as in the VerticalPred case. + if (top != NULL) { + VerticalPred(dst, top, size); + } else { + Fill(dst, 129, size); + } + } +} + +static WEBP_INLINE void DC8uv(uint8_t* dst, const uint8_t* left, + const uint8_t* top) { + const __m128i top_values = _mm_loadl_epi64((const __m128i*)top); + const __m128i left_values = _mm_loadl_epi64((const __m128i*)left); + const __m128i combined = _mm_unpacklo_epi64(top_values, left_values); + const int DC = VP8HorizontalAdd8b(&combined) + 8; + Put8x8uv(DC >> 4, dst); +} + +static WEBP_INLINE void DC8uvNoLeft(uint8_t* dst, const uint8_t* top) { + const __m128i zero = _mm_setzero_si128(); + const __m128i top_values = _mm_loadl_epi64((const __m128i*)top); + const __m128i sum = _mm_sad_epu8(top_values, zero); + const int DC = _mm_cvtsi128_si32(sum) + 4; + Put8x8uv(DC >> 3, dst); +} + +static WEBP_INLINE void DC8uvNoTop(uint8_t* dst, const uint8_t* left) { + // 'left' is contiguous so we can reuse the top summation. + DC8uvNoLeft(dst, left); +} + +static WEBP_INLINE void DC8uvNoTopLeft(uint8_t* dst) { + Put8x8uv(0x80, dst); +} + +static WEBP_INLINE void DC8uvMode(uint8_t* dst, const uint8_t* left, + const uint8_t* top) { + if (top != NULL) { + if (left != NULL) { // top and left present + DC8uv(dst, left, top); + } else { // top, but no left + DC8uvNoLeft(dst, top); + } + } else if (left != NULL) { // left but no top + DC8uvNoTop(dst, left); + } else { // no top, no left, nothing. + DC8uvNoTopLeft(dst); + } +} + +static WEBP_INLINE void DC16(uint8_t* dst, const uint8_t* left, + const uint8_t* top) { + const __m128i top_row = _mm_load_si128((const __m128i*)top); + const __m128i left_row = _mm_load_si128((const __m128i*)left); + const int DC = + VP8HorizontalAdd8b(&top_row) + VP8HorizontalAdd8b(&left_row) + 16; + Put16(DC >> 5, dst); +} + +static WEBP_INLINE void DC16NoLeft(uint8_t* dst, const uint8_t* top) { + const __m128i top_row = _mm_load_si128((const __m128i*)top); + const int DC = VP8HorizontalAdd8b(&top_row) + 8; + Put16(DC >> 4, dst); +} + +static WEBP_INLINE void DC16NoTop(uint8_t* dst, const uint8_t* left) { + // 'left' is contiguous so we can reuse the top summation. + DC16NoLeft(dst, left); +} + +static WEBP_INLINE void DC16NoTopLeft(uint8_t* dst) { + Put16(0x80, dst); +} + +static WEBP_INLINE void DC16Mode(uint8_t* dst, const uint8_t* left, + const uint8_t* top) { + if (top != NULL) { + if (left != NULL) { // top and left present + DC16(dst, left, top); + } else { // top, but no left + DC16NoLeft(dst, top); + } + } else if (left != NULL) { // left but no top + DC16NoTop(dst, left); + } else { // no top, no left, nothing. + DC16NoTopLeft(dst); + } +} + +//------------------------------------------------------------------------------ +// 4x4 predictions + +#define DST(x, y) dst[(x) + (y) * BPS] +#define AVG3(a, b, c) (((a) + 2 * (b) + (c) + 2) >> 2) +#define AVG2(a, b) (((a) + (b) + 1) >> 1) + +// We use the following 8b-arithmetic tricks: +// (a + 2 * b + c + 2) >> 2 = (AC + b + 1) >> 1 +// where: AC = (a + c) >> 1 = [(a + c + 1) >> 1] - [(a^c) & 1] +// and: +// (a + 2 * b + c + 2) >> 2 = (AB + BC + 1) >> 1 - (ab|bc)&lsb +// where: AC = (a + b + 1) >> 1, BC = (b + c + 1) >> 1 +// and ab = a ^ b, bc = b ^ c, lsb = (AC^BC)&1 + +static WEBP_INLINE void VE4(uint8_t* dst, const uint8_t* top) { // vertical + const __m128i one = _mm_set1_epi8(1); + const __m128i ABCDEFGH = _mm_loadl_epi64((__m128i*)(top - 1)); + const __m128i BCDEFGH0 = _mm_srli_si128(ABCDEFGH, 1); + const __m128i CDEFGH00 = _mm_srli_si128(ABCDEFGH, 2); + const __m128i a = _mm_avg_epu8(ABCDEFGH, CDEFGH00); + const __m128i lsb = _mm_and_si128(_mm_xor_si128(ABCDEFGH, CDEFGH00), one); + const __m128i b = _mm_subs_epu8(a, lsb); + const __m128i avg = _mm_avg_epu8(b, BCDEFGH0); + const uint32_t vals = _mm_cvtsi128_si32(avg); + int i; + for (i = 0; i < 4; ++i) { + WebPUint32ToMem(dst + i * BPS, vals); + } +} + +static WEBP_INLINE void HE4(uint8_t* dst, const uint8_t* top) { // horizontal + const int X = top[-1]; + const int I = top[-2]; + const int J = top[-3]; + const int K = top[-4]; + const int L = top[-5]; + WebPUint32ToMem(dst + 0 * BPS, 0x01010101U * AVG3(X, I, J)); + WebPUint32ToMem(dst + 1 * BPS, 0x01010101U * AVG3(I, J, K)); + WebPUint32ToMem(dst + 2 * BPS, 0x01010101U * AVG3(J, K, L)); + WebPUint32ToMem(dst + 3 * BPS, 0x01010101U * AVG3(K, L, L)); +} + +static WEBP_INLINE void DC4(uint8_t* dst, const uint8_t* top) { + uint32_t dc = 4; + int i; + for (i = 0; i < 4; ++i) dc += top[i] + top[-5 + i]; + Fill(dst, dc >> 3, 4); +} + +static WEBP_INLINE void LD4(uint8_t* dst, const uint8_t* top) { // Down-Left + const __m128i one = _mm_set1_epi8(1); + const __m128i ABCDEFGH = _mm_loadl_epi64((const __m128i*)top); + const __m128i BCDEFGH0 = _mm_srli_si128(ABCDEFGH, 1); + const __m128i CDEFGH00 = _mm_srli_si128(ABCDEFGH, 2); + const __m128i CDEFGHH0 = _mm_insert_epi16(CDEFGH00, top[7], 3); + const __m128i avg1 = _mm_avg_epu8(ABCDEFGH, CDEFGHH0); + const __m128i lsb = _mm_and_si128(_mm_xor_si128(ABCDEFGH, CDEFGHH0), one); + const __m128i avg2 = _mm_subs_epu8(avg1, lsb); + const __m128i abcdefg = _mm_avg_epu8(avg2, BCDEFGH0); + WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcdefg )); + WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); + WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); + WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); +} + +static WEBP_INLINE void VR4(uint8_t* dst, + const uint8_t* top) { // Vertical-Right + const __m128i one = _mm_set1_epi8(1); + const int I = top[-2]; + const int J = top[-3]; + const int K = top[-4]; + const int X = top[-1]; + const __m128i XABCD = _mm_loadl_epi64((const __m128i*)(top - 1)); + const __m128i ABCD0 = _mm_srli_si128(XABCD, 1); + const __m128i abcd = _mm_avg_epu8(XABCD, ABCD0); + const __m128i _XABCD = _mm_slli_si128(XABCD, 1); + const __m128i IXABCD = _mm_insert_epi16(_XABCD, I | (X << 8), 0); + const __m128i avg1 = _mm_avg_epu8(IXABCD, ABCD0); + const __m128i lsb = _mm_and_si128(_mm_xor_si128(IXABCD, ABCD0), one); + const __m128i avg2 = _mm_subs_epu8(avg1, lsb); + const __m128i efgh = _mm_avg_epu8(avg2, XABCD); + WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcd )); + WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( efgh )); + WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(abcd, 1))); + WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(efgh, 1))); + + // these two are hard to implement in SSE2, so we keep the C-version: + DST(0, 2) = AVG3(J, I, X); + DST(0, 3) = AVG3(K, J, I); +} + +static WEBP_INLINE void VL4(uint8_t* dst, + const uint8_t* top) { // Vertical-Left + const __m128i one = _mm_set1_epi8(1); + const __m128i ABCDEFGH = _mm_loadl_epi64((const __m128i*)top); + const __m128i BCDEFGH_ = _mm_srli_si128(ABCDEFGH, 1); + const __m128i CDEFGH__ = _mm_srli_si128(ABCDEFGH, 2); + const __m128i avg1 = _mm_avg_epu8(ABCDEFGH, BCDEFGH_); + const __m128i avg2 = _mm_avg_epu8(CDEFGH__, BCDEFGH_); + const __m128i avg3 = _mm_avg_epu8(avg1, avg2); + const __m128i lsb1 = _mm_and_si128(_mm_xor_si128(avg1, avg2), one); + const __m128i ab = _mm_xor_si128(ABCDEFGH, BCDEFGH_); + const __m128i bc = _mm_xor_si128(CDEFGH__, BCDEFGH_); + const __m128i abbc = _mm_or_si128(ab, bc); + const __m128i lsb2 = _mm_and_si128(abbc, lsb1); + const __m128i avg4 = _mm_subs_epu8(avg3, lsb2); + const uint32_t extra_out = _mm_cvtsi128_si32(_mm_srli_si128(avg4, 4)); + WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( avg1 )); + WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( avg4 )); + WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg1, 1))); + WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg4, 1))); + + // these two are hard to get and irregular + DST(3, 2) = (extra_out >> 0) & 0xff; + DST(3, 3) = (extra_out >> 8) & 0xff; +} + +static WEBP_INLINE void RD4(uint8_t* dst, const uint8_t* top) { // Down-right + const __m128i one = _mm_set1_epi8(1); + const __m128i LKJIXABC = _mm_loadl_epi64((const __m128i*)(top - 5)); + const __m128i LKJIXABCD = _mm_insert_epi16(LKJIXABC, top[3], 4); + const __m128i KJIXABCD_ = _mm_srli_si128(LKJIXABCD, 1); + const __m128i JIXABCD__ = _mm_srli_si128(LKJIXABCD, 2); + const __m128i avg1 = _mm_avg_epu8(JIXABCD__, LKJIXABCD); + const __m128i lsb = _mm_and_si128(_mm_xor_si128(JIXABCD__, LKJIXABCD), one); + const __m128i avg2 = _mm_subs_epu8(avg1, lsb); + const __m128i abcdefg = _mm_avg_epu8(avg2, KJIXABCD_); + WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32( abcdefg )); + WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); + WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); + WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); +} + +static WEBP_INLINE void HU4(uint8_t* dst, const uint8_t* top) { + const int I = top[-2]; + const int J = top[-3]; + const int K = top[-4]; + const int L = top[-5]; + DST(0, 0) = AVG2(I, J); + DST(2, 0) = DST(0, 1) = AVG2(J, K); + DST(2, 1) = DST(0, 2) = AVG2(K, L); + DST(1, 0) = AVG3(I, J, K); + DST(3, 0) = DST(1, 1) = AVG3(J, K, L); + DST(3, 1) = DST(1, 2) = AVG3(K, L, L); + DST(3, 2) = DST(2, 2) = + DST(0, 3) = DST(1, 3) = DST(2, 3) = DST(3, 3) = L; +} + +static WEBP_INLINE void HD4(uint8_t* dst, const uint8_t* top) { + const int X = top[-1]; + const int I = top[-2]; + const int J = top[-3]; + const int K = top[-4]; + const int L = top[-5]; + const int A = top[0]; + const int B = top[1]; + const int C = top[2]; + + DST(0, 0) = DST(2, 1) = AVG2(I, X); + DST(0, 1) = DST(2, 2) = AVG2(J, I); + DST(0, 2) = DST(2, 3) = AVG2(K, J); + DST(0, 3) = AVG2(L, K); + + DST(3, 0) = AVG3(A, B, C); + DST(2, 0) = AVG3(X, A, B); + DST(1, 0) = DST(3, 1) = AVG3(I, X, A); + DST(1, 1) = DST(3, 2) = AVG3(J, I, X); + DST(1, 2) = DST(3, 3) = AVG3(K, J, I); + DST(1, 3) = AVG3(L, K, J); +} + +static WEBP_INLINE void TM4(uint8_t* dst, const uint8_t* top) { + const __m128i zero = _mm_setzero_si128(); + const __m128i top_values = _mm_cvtsi32_si128(WebPMemToUint32(top)); + const __m128i top_base = _mm_unpacklo_epi8(top_values, zero); + int y; + for (y = 0; y < 4; ++y, dst += BPS) { + const int val = top[-2 - y] - top[-1]; + const __m128i base = _mm_set1_epi16(val); + const __m128i out = _mm_packus_epi16(_mm_add_epi16(base, top_base), zero); + WebPUint32ToMem(dst, _mm_cvtsi128_si32(out)); + } +} + +#undef DST +#undef AVG3 +#undef AVG2 + +//------------------------------------------------------------------------------ +// luma 4x4 prediction + +// Left samples are top[-5 .. -2], top_left is top[-1], top are +// located at top[0..3], and top right is top[4..7] +static void Intra4Preds(uint8_t* dst, const uint8_t* top) { + DC4(I4DC4 + dst, top); + TM4(I4TM4 + dst, top); + VE4(I4VE4 + dst, top); + HE4(I4HE4 + dst, top); + RD4(I4RD4 + dst, top); + VR4(I4VR4 + dst, top); + LD4(I4LD4 + dst, top); + VL4(I4VL4 + dst, top); + HD4(I4HD4 + dst, top); + HU4(I4HU4 + dst, top); +} + +//------------------------------------------------------------------------------ +// Chroma 8x8 prediction (paragraph 12.2) + +static void IntraChromaPreds(uint8_t* dst, const uint8_t* left, + const uint8_t* top) { + // U block + DC8uvMode(C8DC8 + dst, left, top); + VerticalPred(C8VE8 + dst, top, 8); + HorizontalPred(C8HE8 + dst, left, 8); + TrueMotion(C8TM8 + dst, left, top, 8); + // V block + dst += 8; + if (top != NULL) top += 8; + if (left != NULL) left += 16; + DC8uvMode(C8DC8 + dst, left, top); + VerticalPred(C8VE8 + dst, top, 8); + HorizontalPred(C8HE8 + dst, left, 8); + TrueMotion(C8TM8 + dst, left, top, 8); +} + +//------------------------------------------------------------------------------ +// luma 16x16 prediction (paragraph 12.3) + +static void Intra16Preds(uint8_t* dst, + const uint8_t* left, const uint8_t* top) { + DC16Mode(I16DC16 + dst, left, top); + VerticalPred(I16VE16 + dst, top, 16); + HorizontalPred(I16HE16 + dst, left, 16); + TrueMotion(I16TM16 + dst, left, top, 16); +} + +//------------------------------------------------------------------------------ +// Metric + +static WEBP_INLINE void SubtractAndAccumulate(const __m128i a, const __m128i b, + __m128i* const sum) { + // take abs(a-b) in 8b + const __m128i a_b = _mm_subs_epu8(a, b); + const __m128i b_a = _mm_subs_epu8(b, a); + const __m128i abs_a_b = _mm_or_si128(a_b, b_a); + // zero-extend to 16b + const __m128i zero = _mm_setzero_si128(); + const __m128i C0 = _mm_unpacklo_epi8(abs_a_b, zero); + const __m128i C1 = _mm_unpackhi_epi8(abs_a_b, zero); + // multiply with self + const __m128i sum1 = _mm_madd_epi16(C0, C0); + const __m128i sum2 = _mm_madd_epi16(C1, C1); + *sum = _mm_add_epi32(sum1, sum2); +} + +static WEBP_INLINE int SSE_16xN(const uint8_t* a, const uint8_t* b, + int num_pairs) { + __m128i sum = _mm_setzero_si128(); + int32_t tmp[4]; + int i; + + for (i = 0; i < num_pairs; ++i) { + const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[BPS * 0]); + const __m128i b0 = _mm_loadu_si128((const __m128i*)&b[BPS * 0]); + const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[BPS * 1]); + const __m128i b1 = _mm_loadu_si128((const __m128i*)&b[BPS * 1]); + __m128i sum1, sum2; + SubtractAndAccumulate(a0, b0, &sum1); + SubtractAndAccumulate(a1, b1, &sum2); + sum = _mm_add_epi32(sum, _mm_add_epi32(sum1, sum2)); + a += 2 * BPS; + b += 2 * BPS; + } + _mm_storeu_si128((__m128i*)tmp, sum); + return (tmp[3] + tmp[2] + tmp[1] + tmp[0]); +} + +static int SSE16x16(const uint8_t* a, const uint8_t* b) { + return SSE_16xN(a, b, 8); +} + +static int SSE16x8(const uint8_t* a, const uint8_t* b) { + return SSE_16xN(a, b, 4); +} + +#define LOAD_8x16b(ptr) \ + _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i*)(ptr)), zero) + +static int SSE8x8(const uint8_t* a, const uint8_t* b) { + const __m128i zero = _mm_setzero_si128(); + int num_pairs = 4; + __m128i sum = zero; + int32_t tmp[4]; + while (num_pairs-- > 0) { + const __m128i a0 = LOAD_8x16b(&a[BPS * 0]); + const __m128i a1 = LOAD_8x16b(&a[BPS * 1]); + const __m128i b0 = LOAD_8x16b(&b[BPS * 0]); + const __m128i b1 = LOAD_8x16b(&b[BPS * 1]); + // subtract + const __m128i c0 = _mm_subs_epi16(a0, b0); + const __m128i c1 = _mm_subs_epi16(a1, b1); + // multiply/accumulate with self + const __m128i d0 = _mm_madd_epi16(c0, c0); + const __m128i d1 = _mm_madd_epi16(c1, c1); + // collect + const __m128i sum01 = _mm_add_epi32(d0, d1); + sum = _mm_add_epi32(sum, sum01); + a += 2 * BPS; + b += 2 * BPS; + } + _mm_storeu_si128((__m128i*)tmp, sum); + return (tmp[3] + tmp[2] + tmp[1] + tmp[0]); +} +#undef LOAD_8x16b + +static int SSE4x4(const uint8_t* a, const uint8_t* b) { + const __m128i zero = _mm_setzero_si128(); + + // Load values. Note that we read 8 pixels instead of 4, + // but the a/b buffers are over-allocated to that effect. + const __m128i a0 = _mm_loadl_epi64((const __m128i*)&a[BPS * 0]); + const __m128i a1 = _mm_loadl_epi64((const __m128i*)&a[BPS * 1]); + const __m128i a2 = _mm_loadl_epi64((const __m128i*)&a[BPS * 2]); + const __m128i a3 = _mm_loadl_epi64((const __m128i*)&a[BPS * 3]); + const __m128i b0 = _mm_loadl_epi64((const __m128i*)&b[BPS * 0]); + const __m128i b1 = _mm_loadl_epi64((const __m128i*)&b[BPS * 1]); + const __m128i b2 = _mm_loadl_epi64((const __m128i*)&b[BPS * 2]); + const __m128i b3 = _mm_loadl_epi64((const __m128i*)&b[BPS * 3]); + // Combine pair of lines. + const __m128i a01 = _mm_unpacklo_epi32(a0, a1); + const __m128i a23 = _mm_unpacklo_epi32(a2, a3); + const __m128i b01 = _mm_unpacklo_epi32(b0, b1); + const __m128i b23 = _mm_unpacklo_epi32(b2, b3); + // Convert to 16b. + const __m128i a01s = _mm_unpacklo_epi8(a01, zero); + const __m128i a23s = _mm_unpacklo_epi8(a23, zero); + const __m128i b01s = _mm_unpacklo_epi8(b01, zero); + const __m128i b23s = _mm_unpacklo_epi8(b23, zero); + // subtract, square and accumulate + const __m128i d0 = _mm_subs_epi16(a01s, b01s); + const __m128i d1 = _mm_subs_epi16(a23s, b23s); + const __m128i e0 = _mm_madd_epi16(d0, d0); + const __m128i e1 = _mm_madd_epi16(d1, d1); + const __m128i sum = _mm_add_epi32(e0, e1); + + int32_t tmp[4]; + _mm_storeu_si128((__m128i*)tmp, sum); + return (tmp[3] + tmp[2] + tmp[1] + tmp[0]); +} + +//------------------------------------------------------------------------------ +// Texture distortion +// +// We try to match the spectral content (weighted) between source and +// reconstructed samples. + +// Hadamard transform +// Returns the weighted sum of the absolute value of transformed coefficients. +// w[] contains a row-major 4 by 4 symmetric matrix. +static int TTransform(const uint8_t* inA, const uint8_t* inB, + const uint16_t* const w) { + int32_t sum[4]; + __m128i tmp_0, tmp_1, tmp_2, tmp_3; + const __m128i zero = _mm_setzero_si128(); + + // Load and combine inputs. + { + const __m128i inA_0 = _mm_loadl_epi64((const __m128i*)&inA[BPS * 0]); + const __m128i inA_1 = _mm_loadl_epi64((const __m128i*)&inA[BPS * 1]); + const __m128i inA_2 = _mm_loadl_epi64((const __m128i*)&inA[BPS * 2]); + const __m128i inA_3 = _mm_loadl_epi64((const __m128i*)&inA[BPS * 3]); + const __m128i inB_0 = _mm_loadl_epi64((const __m128i*)&inB[BPS * 0]); + const __m128i inB_1 = _mm_loadl_epi64((const __m128i*)&inB[BPS * 1]); + const __m128i inB_2 = _mm_loadl_epi64((const __m128i*)&inB[BPS * 2]); + const __m128i inB_3 = _mm_loadl_epi64((const __m128i*)&inB[BPS * 3]); + + // Combine inA and inB (we'll do two transforms in parallel). + const __m128i inAB_0 = _mm_unpacklo_epi32(inA_0, inB_0); + const __m128i inAB_1 = _mm_unpacklo_epi32(inA_1, inB_1); + const __m128i inAB_2 = _mm_unpacklo_epi32(inA_2, inB_2); + const __m128i inAB_3 = _mm_unpacklo_epi32(inA_3, inB_3); + tmp_0 = _mm_unpacklo_epi8(inAB_0, zero); + tmp_1 = _mm_unpacklo_epi8(inAB_1, zero); + tmp_2 = _mm_unpacklo_epi8(inAB_2, zero); + tmp_3 = _mm_unpacklo_epi8(inAB_3, zero); + // a00 a01 a02 a03 b00 b01 b02 b03 + // a10 a11 a12 a13 b10 b11 b12 b13 + // a20 a21 a22 a23 b20 b21 b22 b23 + // a30 a31 a32 a33 b30 b31 b32 b33 + } + + // Vertical pass first to avoid a transpose (vertical and horizontal passes + // are commutative because w/kWeightY is symmetric) and subsequent transpose. + { + // Calculate a and b (two 4x4 at once). + const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2); + const __m128i a1 = _mm_add_epi16(tmp_1, tmp_3); + const __m128i a2 = _mm_sub_epi16(tmp_1, tmp_3); + const __m128i a3 = _mm_sub_epi16(tmp_0, tmp_2); + const __m128i b0 = _mm_add_epi16(a0, a1); + const __m128i b1 = _mm_add_epi16(a3, a2); + const __m128i b2 = _mm_sub_epi16(a3, a2); + const __m128i b3 = _mm_sub_epi16(a0, a1); + // a00 a01 a02 a03 b00 b01 b02 b03 + // a10 a11 a12 a13 b10 b11 b12 b13 + // a20 a21 a22 a23 b20 b21 b22 b23 + // a30 a31 a32 a33 b30 b31 b32 b33 + + // Transpose the two 4x4. + VP8Transpose_2_4x4_16b(&b0, &b1, &b2, &b3, &tmp_0, &tmp_1, &tmp_2, &tmp_3); + } + + // Horizontal pass and difference of weighted sums. + { + // Load all inputs. + const __m128i w_0 = _mm_loadu_si128((const __m128i*)&w[0]); + const __m128i w_8 = _mm_loadu_si128((const __m128i*)&w[8]); + + // Calculate a and b (two 4x4 at once). + const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2); + const __m128i a1 = _mm_add_epi16(tmp_1, tmp_3); + const __m128i a2 = _mm_sub_epi16(tmp_1, tmp_3); + const __m128i a3 = _mm_sub_epi16(tmp_0, tmp_2); + const __m128i b0 = _mm_add_epi16(a0, a1); + const __m128i b1 = _mm_add_epi16(a3, a2); + const __m128i b2 = _mm_sub_epi16(a3, a2); + const __m128i b3 = _mm_sub_epi16(a0, a1); + + // Separate the transforms of inA and inB. + __m128i A_b0 = _mm_unpacklo_epi64(b0, b1); + __m128i A_b2 = _mm_unpacklo_epi64(b2, b3); + __m128i B_b0 = _mm_unpackhi_epi64(b0, b1); + __m128i B_b2 = _mm_unpackhi_epi64(b2, b3); + + { + const __m128i d0 = _mm_sub_epi16(zero, A_b0); + const __m128i d1 = _mm_sub_epi16(zero, A_b2); + const __m128i d2 = _mm_sub_epi16(zero, B_b0); + const __m128i d3 = _mm_sub_epi16(zero, B_b2); + A_b0 = _mm_max_epi16(A_b0, d0); // abs(v), 16b + A_b2 = _mm_max_epi16(A_b2, d1); + B_b0 = _mm_max_epi16(B_b0, d2); + B_b2 = _mm_max_epi16(B_b2, d3); + } + + // weighted sums + A_b0 = _mm_madd_epi16(A_b0, w_0); + A_b2 = _mm_madd_epi16(A_b2, w_8); + B_b0 = _mm_madd_epi16(B_b0, w_0); + B_b2 = _mm_madd_epi16(B_b2, w_8); + A_b0 = _mm_add_epi32(A_b0, A_b2); + B_b0 = _mm_add_epi32(B_b0, B_b2); + + // difference of weighted sums + A_b0 = _mm_sub_epi32(A_b0, B_b0); + _mm_storeu_si128((__m128i*)&sum[0], A_b0); + } + return sum[0] + sum[1] + sum[2] + sum[3]; +} + +static int Disto4x4(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + const int diff_sum = TTransform(a, b, w); + return abs(diff_sum) >> 5; +} + +static int Disto16x16(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + int D = 0; + int x, y; + for (y = 0; y < 16 * BPS; y += 4 * BPS) { + for (x = 0; x < 16; x += 4) { + D += Disto4x4(a + x + y, b + x + y, w); + } + } + return D; +} + +//------------------------------------------------------------------------------ +// Quantization +// + +static WEBP_INLINE int DoQuantizeBlock(int16_t in[16], int16_t out[16], + const uint16_t* const sharpen, + const VP8Matrix* const mtx) { + const __m128i max_coeff_2047 = _mm_set1_epi16(MAX_LEVEL); + const __m128i zero = _mm_setzero_si128(); + __m128i coeff0, coeff8; + __m128i out0, out8; + __m128i packed_out; + + // Load all inputs. + __m128i in0 = _mm_loadu_si128((__m128i*)&in[0]); + __m128i in8 = _mm_loadu_si128((__m128i*)&in[8]); + const __m128i iq0 = _mm_loadu_si128((const __m128i*)&mtx->iq_[0]); + const __m128i iq8 = _mm_loadu_si128((const __m128i*)&mtx->iq_[8]); + const __m128i q0 = _mm_loadu_si128((const __m128i*)&mtx->q_[0]); + const __m128i q8 = _mm_loadu_si128((const __m128i*)&mtx->q_[8]); + + // extract sign(in) (0x0000 if positive, 0xffff if negative) + const __m128i sign0 = _mm_cmpgt_epi16(zero, in0); + const __m128i sign8 = _mm_cmpgt_epi16(zero, in8); + + // coeff = abs(in) = (in ^ sign) - sign + coeff0 = _mm_xor_si128(in0, sign0); + coeff8 = _mm_xor_si128(in8, sign8); + coeff0 = _mm_sub_epi16(coeff0, sign0); + coeff8 = _mm_sub_epi16(coeff8, sign8); + + // coeff = abs(in) + sharpen + if (sharpen != NULL) { + const __m128i sharpen0 = _mm_loadu_si128((const __m128i*)&sharpen[0]); + const __m128i sharpen8 = _mm_loadu_si128((const __m128i*)&sharpen[8]); + coeff0 = _mm_add_epi16(coeff0, sharpen0); + coeff8 = _mm_add_epi16(coeff8, sharpen8); + } + + // out = (coeff * iQ + B) >> QFIX + { + // doing calculations with 32b precision (QFIX=17) + // out = (coeff * iQ) + const __m128i coeff_iQ0H = _mm_mulhi_epu16(coeff0, iq0); + const __m128i coeff_iQ0L = _mm_mullo_epi16(coeff0, iq0); + const __m128i coeff_iQ8H = _mm_mulhi_epu16(coeff8, iq8); + const __m128i coeff_iQ8L = _mm_mullo_epi16(coeff8, iq8); + __m128i out_00 = _mm_unpacklo_epi16(coeff_iQ0L, coeff_iQ0H); + __m128i out_04 = _mm_unpackhi_epi16(coeff_iQ0L, coeff_iQ0H); + __m128i out_08 = _mm_unpacklo_epi16(coeff_iQ8L, coeff_iQ8H); + __m128i out_12 = _mm_unpackhi_epi16(coeff_iQ8L, coeff_iQ8H); + // out = (coeff * iQ + B) + const __m128i bias_00 = _mm_loadu_si128((const __m128i*)&mtx->bias_[0]); + const __m128i bias_04 = _mm_loadu_si128((const __m128i*)&mtx->bias_[4]); + const __m128i bias_08 = _mm_loadu_si128((const __m128i*)&mtx->bias_[8]); + const __m128i bias_12 = _mm_loadu_si128((const __m128i*)&mtx->bias_[12]); + out_00 = _mm_add_epi32(out_00, bias_00); + out_04 = _mm_add_epi32(out_04, bias_04); + out_08 = _mm_add_epi32(out_08, bias_08); + out_12 = _mm_add_epi32(out_12, bias_12); + // out = QUANTDIV(coeff, iQ, B, QFIX) + out_00 = _mm_srai_epi32(out_00, QFIX); + out_04 = _mm_srai_epi32(out_04, QFIX); + out_08 = _mm_srai_epi32(out_08, QFIX); + out_12 = _mm_srai_epi32(out_12, QFIX); + + // pack result as 16b + out0 = _mm_packs_epi32(out_00, out_04); + out8 = _mm_packs_epi32(out_08, out_12); + + // if (coeff > 2047) coeff = 2047 + out0 = _mm_min_epi16(out0, max_coeff_2047); + out8 = _mm_min_epi16(out8, max_coeff_2047); + } + + // get sign back (if (sign[j]) out_n = -out_n) + out0 = _mm_xor_si128(out0, sign0); + out8 = _mm_xor_si128(out8, sign8); + out0 = _mm_sub_epi16(out0, sign0); + out8 = _mm_sub_epi16(out8, sign8); + + // in = out * Q + in0 = _mm_mullo_epi16(out0, q0); + in8 = _mm_mullo_epi16(out8, q8); + + _mm_storeu_si128((__m128i*)&in[0], in0); + _mm_storeu_si128((__m128i*)&in[8], in8); + + // zigzag the output before storing it. + // + // The zigzag pattern can almost be reproduced with a small sequence of + // shuffles. After it, we only need to swap the 7th (ending up in third + // position instead of twelfth) and 8th values. + { + __m128i outZ0, outZ8; + outZ0 = _mm_shufflehi_epi16(out0, _MM_SHUFFLE(2, 1, 3, 0)); + outZ0 = _mm_shuffle_epi32 (outZ0, _MM_SHUFFLE(3, 1, 2, 0)); + outZ0 = _mm_shufflehi_epi16(outZ0, _MM_SHUFFLE(3, 1, 0, 2)); + outZ8 = _mm_shufflelo_epi16(out8, _MM_SHUFFLE(3, 0, 2, 1)); + outZ8 = _mm_shuffle_epi32 (outZ8, _MM_SHUFFLE(3, 1, 2, 0)); + outZ8 = _mm_shufflelo_epi16(outZ8, _MM_SHUFFLE(1, 3, 2, 0)); + _mm_storeu_si128((__m128i*)&out[0], outZ0); + _mm_storeu_si128((__m128i*)&out[8], outZ8); + packed_out = _mm_packs_epi16(outZ0, outZ8); + } + { + const int16_t outZ_12 = out[12]; + const int16_t outZ_3 = out[3]; + out[3] = outZ_12; + out[12] = outZ_3; + } + + // detect if all 'out' values are zeroes or not + return (_mm_movemask_epi8(_mm_cmpeq_epi8(packed_out, zero)) != 0xffff); +} + +static int QuantizeBlock(int16_t in[16], int16_t out[16], + const VP8Matrix* const mtx) { + return DoQuantizeBlock(in, out, &mtx->sharpen_[0], mtx); +} + +static int QuantizeBlockWHT(int16_t in[16], int16_t out[16], + const VP8Matrix* const mtx) { + return DoQuantizeBlock(in, out, NULL, mtx); +} + +static int Quantize2Blocks(int16_t in[32], int16_t out[32], + const VP8Matrix* const mtx) { + int nz; + const uint16_t* const sharpen = &mtx->sharpen_[0]; + nz = DoQuantizeBlock(in + 0 * 16, out + 0 * 16, sharpen, mtx) << 0; + nz |= DoQuantizeBlock(in + 1 * 16, out + 1 * 16, sharpen, mtx) << 1; + return nz; +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8EncDspInitSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitSSE2(void) { + VP8CollectHistogram = CollectHistogram; + VP8EncPredLuma16 = Intra16Preds; + VP8EncPredChroma8 = IntraChromaPreds; + VP8EncPredLuma4 = Intra4Preds; + VP8EncQuantizeBlock = QuantizeBlock; + VP8EncQuantize2Blocks = Quantize2Blocks; + VP8EncQuantizeBlockWHT = QuantizeBlockWHT; + VP8ITransform = ITransform; + VP8FTransform = FTransform; + VP8FTransform2 = FTransform2; + VP8FTransformWHT = FTransformWHT; + VP8SSE16x16 = SSE16x16; + VP8SSE16x8 = SSE16x8; + VP8SSE8x8 = SSE8x8; + VP8SSE4x4 = SSE4x4; + VP8TDisto4x4 = Disto4x4; + VP8TDisto16x16 = Disto16x16; +} + +#else // !WEBP_USE_SSE2 + +WEBP_DSP_INIT_STUB(VP8EncDspInitSSE2) + +#endif // WEBP_USE_SSE2 diff --git a/thirdparty/libwebp/dsp/enc_sse41.c b/thirdparty/libwebp/dsp/enc_sse41.c new file mode 100644 index 0000000000..a1783901a6 --- /dev/null +++ b/thirdparty/libwebp/dsp/enc_sse41.c @@ -0,0 +1,339 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE4 version of some encoding functions. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE41) +#include <smmintrin.h> +#include <stdlib.h> // for abs() + +#include "./common_sse2.h" +#include "../enc/vp8enci.h" + +//------------------------------------------------------------------------------ +// Compute susceptibility based on DCT-coeff histograms. + +static void CollectHistogram(const uint8_t* ref, const uint8_t* pred, + int start_block, int end_block, + VP8Histogram* const histo) { + const __m128i max_coeff_thresh = _mm_set1_epi16(MAX_COEFF_THRESH); + int j; + int distribution[MAX_COEFF_THRESH + 1] = { 0 }; + for (j = start_block; j < end_block; ++j) { + int16_t out[16]; + int k; + + VP8FTransform(ref + VP8DspScan[j], pred + VP8DspScan[j], out); + + // Convert coefficients to bin (within out[]). + { + // Load. + const __m128i out0 = _mm_loadu_si128((__m128i*)&out[0]); + const __m128i out1 = _mm_loadu_si128((__m128i*)&out[8]); + // v = abs(out) >> 3 + const __m128i abs0 = _mm_abs_epi16(out0); + const __m128i abs1 = _mm_abs_epi16(out1); + const __m128i v0 = _mm_srai_epi16(abs0, 3); + const __m128i v1 = _mm_srai_epi16(abs1, 3); + // bin = min(v, MAX_COEFF_THRESH) + const __m128i bin0 = _mm_min_epi16(v0, max_coeff_thresh); + const __m128i bin1 = _mm_min_epi16(v1, max_coeff_thresh); + // Store. + _mm_storeu_si128((__m128i*)&out[0], bin0); + _mm_storeu_si128((__m128i*)&out[8], bin1); + } + + // Convert coefficients to bin. + for (k = 0; k < 16; ++k) { + ++distribution[out[k]]; + } + } + VP8SetHistogramData(distribution, histo); +} + +//------------------------------------------------------------------------------ +// Texture distortion +// +// We try to match the spectral content (weighted) between source and +// reconstructed samples. + +// Hadamard transform +// Returns the weighted sum of the absolute value of transformed coefficients. +// w[] contains a row-major 4 by 4 symmetric matrix. +static int TTransform(const uint8_t* inA, const uint8_t* inB, + const uint16_t* const w) { + int32_t sum[4]; + __m128i tmp_0, tmp_1, tmp_2, tmp_3; + + // Load and combine inputs. + { + const __m128i inA_0 = _mm_loadu_si128((const __m128i*)&inA[BPS * 0]); + const __m128i inA_1 = _mm_loadu_si128((const __m128i*)&inA[BPS * 1]); + const __m128i inA_2 = _mm_loadu_si128((const __m128i*)&inA[BPS * 2]); + // In SSE4.1, with gcc 4.8 at least (maybe other versions), + // _mm_loadu_si128 is faster than _mm_loadl_epi64. But for the last lump + // of inA and inB, _mm_loadl_epi64 is still used not to have an out of + // bound read. + const __m128i inA_3 = _mm_loadl_epi64((const __m128i*)&inA[BPS * 3]); + const __m128i inB_0 = _mm_loadu_si128((const __m128i*)&inB[BPS * 0]); + const __m128i inB_1 = _mm_loadu_si128((const __m128i*)&inB[BPS * 1]); + const __m128i inB_2 = _mm_loadu_si128((const __m128i*)&inB[BPS * 2]); + const __m128i inB_3 = _mm_loadl_epi64((const __m128i*)&inB[BPS * 3]); + + // Combine inA and inB (we'll do two transforms in parallel). + const __m128i inAB_0 = _mm_unpacklo_epi32(inA_0, inB_0); + const __m128i inAB_1 = _mm_unpacklo_epi32(inA_1, inB_1); + const __m128i inAB_2 = _mm_unpacklo_epi32(inA_2, inB_2); + const __m128i inAB_3 = _mm_unpacklo_epi32(inA_3, inB_3); + tmp_0 = _mm_cvtepu8_epi16(inAB_0); + tmp_1 = _mm_cvtepu8_epi16(inAB_1); + tmp_2 = _mm_cvtepu8_epi16(inAB_2); + tmp_3 = _mm_cvtepu8_epi16(inAB_3); + // a00 a01 a02 a03 b00 b01 b02 b03 + // a10 a11 a12 a13 b10 b11 b12 b13 + // a20 a21 a22 a23 b20 b21 b22 b23 + // a30 a31 a32 a33 b30 b31 b32 b33 + } + + // Vertical pass first to avoid a transpose (vertical and horizontal passes + // are commutative because w/kWeightY is symmetric) and subsequent transpose. + { + // Calculate a and b (two 4x4 at once). + const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2); + const __m128i a1 = _mm_add_epi16(tmp_1, tmp_3); + const __m128i a2 = _mm_sub_epi16(tmp_1, tmp_3); + const __m128i a3 = _mm_sub_epi16(tmp_0, tmp_2); + const __m128i b0 = _mm_add_epi16(a0, a1); + const __m128i b1 = _mm_add_epi16(a3, a2); + const __m128i b2 = _mm_sub_epi16(a3, a2); + const __m128i b3 = _mm_sub_epi16(a0, a1); + // a00 a01 a02 a03 b00 b01 b02 b03 + // a10 a11 a12 a13 b10 b11 b12 b13 + // a20 a21 a22 a23 b20 b21 b22 b23 + // a30 a31 a32 a33 b30 b31 b32 b33 + + // Transpose the two 4x4. + VP8Transpose_2_4x4_16b(&b0, &b1, &b2, &b3, &tmp_0, &tmp_1, &tmp_2, &tmp_3); + } + + // Horizontal pass and difference of weighted sums. + { + // Load all inputs. + const __m128i w_0 = _mm_loadu_si128((const __m128i*)&w[0]); + const __m128i w_8 = _mm_loadu_si128((const __m128i*)&w[8]); + + // Calculate a and b (two 4x4 at once). + const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2); + const __m128i a1 = _mm_add_epi16(tmp_1, tmp_3); + const __m128i a2 = _mm_sub_epi16(tmp_1, tmp_3); + const __m128i a3 = _mm_sub_epi16(tmp_0, tmp_2); + const __m128i b0 = _mm_add_epi16(a0, a1); + const __m128i b1 = _mm_add_epi16(a3, a2); + const __m128i b2 = _mm_sub_epi16(a3, a2); + const __m128i b3 = _mm_sub_epi16(a0, a1); + + // Separate the transforms of inA and inB. + __m128i A_b0 = _mm_unpacklo_epi64(b0, b1); + __m128i A_b2 = _mm_unpacklo_epi64(b2, b3); + __m128i B_b0 = _mm_unpackhi_epi64(b0, b1); + __m128i B_b2 = _mm_unpackhi_epi64(b2, b3); + + A_b0 = _mm_abs_epi16(A_b0); + A_b2 = _mm_abs_epi16(A_b2); + B_b0 = _mm_abs_epi16(B_b0); + B_b2 = _mm_abs_epi16(B_b2); + + // weighted sums + A_b0 = _mm_madd_epi16(A_b0, w_0); + A_b2 = _mm_madd_epi16(A_b2, w_8); + B_b0 = _mm_madd_epi16(B_b0, w_0); + B_b2 = _mm_madd_epi16(B_b2, w_8); + A_b0 = _mm_add_epi32(A_b0, A_b2); + B_b0 = _mm_add_epi32(B_b0, B_b2); + + // difference of weighted sums + A_b2 = _mm_sub_epi32(A_b0, B_b0); + _mm_storeu_si128((__m128i*)&sum[0], A_b2); + } + return sum[0] + sum[1] + sum[2] + sum[3]; +} + +static int Disto4x4(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + const int diff_sum = TTransform(a, b, w); + return abs(diff_sum) >> 5; +} + +static int Disto16x16(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + int D = 0; + int x, y; + for (y = 0; y < 16 * BPS; y += 4 * BPS) { + for (x = 0; x < 16; x += 4) { + D += Disto4x4(a + x + y, b + x + y, w); + } + } + return D; +} + +//------------------------------------------------------------------------------ +// Quantization +// + +// Generates a pshufb constant for shuffling 16b words. +#define PSHUFB_CST(A,B,C,D,E,F,G,H) \ + _mm_set_epi8(2 * (H) + 1, 2 * (H) + 0, 2 * (G) + 1, 2 * (G) + 0, \ + 2 * (F) + 1, 2 * (F) + 0, 2 * (E) + 1, 2 * (E) + 0, \ + 2 * (D) + 1, 2 * (D) + 0, 2 * (C) + 1, 2 * (C) + 0, \ + 2 * (B) + 1, 2 * (B) + 0, 2 * (A) + 1, 2 * (A) + 0) + +static WEBP_INLINE int DoQuantizeBlock(int16_t in[16], int16_t out[16], + const uint16_t* const sharpen, + const VP8Matrix* const mtx) { + const __m128i max_coeff_2047 = _mm_set1_epi16(MAX_LEVEL); + const __m128i zero = _mm_setzero_si128(); + __m128i out0, out8; + __m128i packed_out; + + // Load all inputs. + __m128i in0 = _mm_loadu_si128((__m128i*)&in[0]); + __m128i in8 = _mm_loadu_si128((__m128i*)&in[8]); + const __m128i iq0 = _mm_loadu_si128((const __m128i*)&mtx->iq_[0]); + const __m128i iq8 = _mm_loadu_si128((const __m128i*)&mtx->iq_[8]); + const __m128i q0 = _mm_loadu_si128((const __m128i*)&mtx->q_[0]); + const __m128i q8 = _mm_loadu_si128((const __m128i*)&mtx->q_[8]); + + // coeff = abs(in) + __m128i coeff0 = _mm_abs_epi16(in0); + __m128i coeff8 = _mm_abs_epi16(in8); + + // coeff = abs(in) + sharpen + if (sharpen != NULL) { + const __m128i sharpen0 = _mm_loadu_si128((const __m128i*)&sharpen[0]); + const __m128i sharpen8 = _mm_loadu_si128((const __m128i*)&sharpen[8]); + coeff0 = _mm_add_epi16(coeff0, sharpen0); + coeff8 = _mm_add_epi16(coeff8, sharpen8); + } + + // out = (coeff * iQ + B) >> QFIX + { + // doing calculations with 32b precision (QFIX=17) + // out = (coeff * iQ) + const __m128i coeff_iQ0H = _mm_mulhi_epu16(coeff0, iq0); + const __m128i coeff_iQ0L = _mm_mullo_epi16(coeff0, iq0); + const __m128i coeff_iQ8H = _mm_mulhi_epu16(coeff8, iq8); + const __m128i coeff_iQ8L = _mm_mullo_epi16(coeff8, iq8); + __m128i out_00 = _mm_unpacklo_epi16(coeff_iQ0L, coeff_iQ0H); + __m128i out_04 = _mm_unpackhi_epi16(coeff_iQ0L, coeff_iQ0H); + __m128i out_08 = _mm_unpacklo_epi16(coeff_iQ8L, coeff_iQ8H); + __m128i out_12 = _mm_unpackhi_epi16(coeff_iQ8L, coeff_iQ8H); + // out = (coeff * iQ + B) + const __m128i bias_00 = _mm_loadu_si128((const __m128i*)&mtx->bias_[0]); + const __m128i bias_04 = _mm_loadu_si128((const __m128i*)&mtx->bias_[4]); + const __m128i bias_08 = _mm_loadu_si128((const __m128i*)&mtx->bias_[8]); + const __m128i bias_12 = _mm_loadu_si128((const __m128i*)&mtx->bias_[12]); + out_00 = _mm_add_epi32(out_00, bias_00); + out_04 = _mm_add_epi32(out_04, bias_04); + out_08 = _mm_add_epi32(out_08, bias_08); + out_12 = _mm_add_epi32(out_12, bias_12); + // out = QUANTDIV(coeff, iQ, B, QFIX) + out_00 = _mm_srai_epi32(out_00, QFIX); + out_04 = _mm_srai_epi32(out_04, QFIX); + out_08 = _mm_srai_epi32(out_08, QFIX); + out_12 = _mm_srai_epi32(out_12, QFIX); + + // pack result as 16b + out0 = _mm_packs_epi32(out_00, out_04); + out8 = _mm_packs_epi32(out_08, out_12); + + // if (coeff > 2047) coeff = 2047 + out0 = _mm_min_epi16(out0, max_coeff_2047); + out8 = _mm_min_epi16(out8, max_coeff_2047); + } + + // put sign back + out0 = _mm_sign_epi16(out0, in0); + out8 = _mm_sign_epi16(out8, in8); + + // in = out * Q + in0 = _mm_mullo_epi16(out0, q0); + in8 = _mm_mullo_epi16(out8, q8); + + _mm_storeu_si128((__m128i*)&in[0], in0); + _mm_storeu_si128((__m128i*)&in[8], in8); + + // zigzag the output before storing it. The re-ordering is: + // 0 1 2 3 4 5 6 7 | 8 9 10 11 12 13 14 15 + // -> 0 1 4[8]5 2 3 6 | 9 12 13 10 [7]11 14 15 + // There's only two misplaced entries ([8] and [7]) that are crossing the + // reg's boundaries. + // We use pshufb instead of pshuflo/pshufhi. + { + const __m128i kCst_lo = PSHUFB_CST(0, 1, 4, -1, 5, 2, 3, 6); + const __m128i kCst_7 = PSHUFB_CST(-1, -1, -1, -1, 7, -1, -1, -1); + const __m128i tmp_lo = _mm_shuffle_epi8(out0, kCst_lo); + const __m128i tmp_7 = _mm_shuffle_epi8(out0, kCst_7); // extract #7 + const __m128i kCst_hi = PSHUFB_CST(1, 4, 5, 2, -1, 3, 6, 7); + const __m128i kCst_8 = PSHUFB_CST(-1, -1, -1, 0, -1, -1, -1, -1); + const __m128i tmp_hi = _mm_shuffle_epi8(out8, kCst_hi); + const __m128i tmp_8 = _mm_shuffle_epi8(out8, kCst_8); // extract #8 + const __m128i out_z0 = _mm_or_si128(tmp_lo, tmp_8); + const __m128i out_z8 = _mm_or_si128(tmp_hi, tmp_7); + _mm_storeu_si128((__m128i*)&out[0], out_z0); + _mm_storeu_si128((__m128i*)&out[8], out_z8); + packed_out = _mm_packs_epi16(out_z0, out_z8); + } + + // detect if all 'out' values are zeroes or not + return (_mm_movemask_epi8(_mm_cmpeq_epi8(packed_out, zero)) != 0xffff); +} + +#undef PSHUFB_CST + +static int QuantizeBlock(int16_t in[16], int16_t out[16], + const VP8Matrix* const mtx) { + return DoQuantizeBlock(in, out, &mtx->sharpen_[0], mtx); +} + +static int QuantizeBlockWHT(int16_t in[16], int16_t out[16], + const VP8Matrix* const mtx) { + return DoQuantizeBlock(in, out, NULL, mtx); +} + +static int Quantize2Blocks(int16_t in[32], int16_t out[32], + const VP8Matrix* const mtx) { + int nz; + const uint16_t* const sharpen = &mtx->sharpen_[0]; + nz = DoQuantizeBlock(in + 0 * 16, out + 0 * 16, sharpen, mtx) << 0; + nz |= DoQuantizeBlock(in + 1 * 16, out + 1 * 16, sharpen, mtx) << 1; + return nz; +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8EncDspInitSSE41(void); +WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitSSE41(void) { + VP8CollectHistogram = CollectHistogram; + VP8EncQuantizeBlock = QuantizeBlock; + VP8EncQuantize2Blocks = Quantize2Blocks; + VP8EncQuantizeBlockWHT = QuantizeBlockWHT; + VP8TDisto4x4 = Disto4x4; + VP8TDisto16x16 = Disto16x16; +} + +#else // !WEBP_USE_SSE41 + +WEBP_DSP_INIT_STUB(VP8EncDspInitSSE41) + +#endif // WEBP_USE_SSE41 diff --git a/thirdparty/libwebp/dsp/filters.c b/thirdparty/libwebp/dsp/filters.c new file mode 100644 index 0000000000..9f04faf0cb --- /dev/null +++ b/thirdparty/libwebp/dsp/filters.c @@ -0,0 +1,261 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Spatial prediction using various filters +// +// Author: Urvang (urvang@google.com) + +#include "./dsp.h" +#include <assert.h> +#include <stdlib.h> +#include <string.h> + +//------------------------------------------------------------------------------ +// Helpful macro. + +# define SANITY_CHECK(in, out) \ + assert(in != NULL); \ + assert(out != NULL); \ + assert(width > 0); \ + assert(height > 0); \ + assert(stride >= width); \ + assert(row >= 0 && num_rows > 0 && row + num_rows <= height); \ + (void)height; // Silence unused warning. + +static WEBP_INLINE void PredictLine(const uint8_t* src, const uint8_t* pred, + uint8_t* dst, int length, int inverse) { + int i; + if (inverse) { + for (i = 0; i < length; ++i) dst[i] = src[i] + pred[i]; + } else { + for (i = 0; i < length; ++i) dst[i] = src[i] - pred[i]; + } +} + +//------------------------------------------------------------------------------ +// Horizontal filter. + +static WEBP_INLINE void DoHorizontalFilter(const uint8_t* in, + int width, int height, int stride, + int row, int num_rows, + int inverse, uint8_t* out) { + const uint8_t* preds; + const size_t start_offset = row * stride; + const int last_row = row + num_rows; + SANITY_CHECK(in, out); + in += start_offset; + out += start_offset; + preds = inverse ? out : in; + + if (row == 0) { + // Leftmost pixel is the same as input for topmost scanline. + out[0] = in[0]; + PredictLine(in + 1, preds, out + 1, width - 1, inverse); + row = 1; + preds += stride; + in += stride; + out += stride; + } + + // Filter line-by-line. + while (row < last_row) { + // Leftmost pixel is predicted from above. + PredictLine(in, preds - stride, out, 1, inverse); + PredictLine(in + 1, preds, out + 1, width - 1, inverse); + ++row; + preds += stride; + in += stride; + out += stride; + } +} + +//------------------------------------------------------------------------------ +// Vertical filter. + +static WEBP_INLINE void DoVerticalFilter(const uint8_t* in, + int width, int height, int stride, + int row, int num_rows, + int inverse, uint8_t* out) { + const uint8_t* preds; + const size_t start_offset = row * stride; + const int last_row = row + num_rows; + SANITY_CHECK(in, out); + in += start_offset; + out += start_offset; + preds = inverse ? out : in; + + if (row == 0) { + // Very first top-left pixel is copied. + out[0] = in[0]; + // Rest of top scan-line is left-predicted. + PredictLine(in + 1, preds, out + 1, width - 1, inverse); + row = 1; + in += stride; + out += stride; + } else { + // We are starting from in-between. Make sure 'preds' points to prev row. + preds -= stride; + } + + // Filter line-by-line. + while (row < last_row) { + PredictLine(in, preds, out, width, inverse); + ++row; + preds += stride; + in += stride; + out += stride; + } +} + +//------------------------------------------------------------------------------ +// Gradient filter. + +static WEBP_INLINE int GradientPredictor(uint8_t a, uint8_t b, uint8_t c) { + const int g = a + b - c; + return ((g & ~0xff) == 0) ? g : (g < 0) ? 0 : 255; // clip to 8bit +} + +static WEBP_INLINE void DoGradientFilter(const uint8_t* in, + int width, int height, int stride, + int row, int num_rows, + int inverse, uint8_t* out) { + const uint8_t* preds; + const size_t start_offset = row * stride; + const int last_row = row + num_rows; + SANITY_CHECK(in, out); + in += start_offset; + out += start_offset; + preds = inverse ? out : in; + + // left prediction for top scan-line + if (row == 0) { + out[0] = in[0]; + PredictLine(in + 1, preds, out + 1, width - 1, inverse); + row = 1; + preds += stride; + in += stride; + out += stride; + } + + // Filter line-by-line. + while (row < last_row) { + int w; + // leftmost pixel: predict from above. + PredictLine(in, preds - stride, out, 1, inverse); + for (w = 1; w < width; ++w) { + const int pred = GradientPredictor(preds[w - 1], + preds[w - stride], + preds[w - stride - 1]); + out[w] = in[w] + (inverse ? pred : -pred); + } + ++row; + preds += stride; + in += stride; + out += stride; + } +} + +#undef SANITY_CHECK + +//------------------------------------------------------------------------------ + +static void HorizontalFilter(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + DoHorizontalFilter(data, width, height, stride, 0, height, 0, filtered_data); +} + +static void VerticalFilter(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + DoVerticalFilter(data, width, height, stride, 0, height, 0, filtered_data); +} + + +static void GradientFilter(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + DoGradientFilter(data, width, height, stride, 0, height, 0, filtered_data); +} + + +//------------------------------------------------------------------------------ + +static void HorizontalUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + uint8_t pred = (prev == NULL) ? 0 : prev[0]; + int i; + for (i = 0; i < width; ++i) { + out[i] = pred + in[i]; + pred = out[i]; + } +} + +static void VerticalUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + if (prev == NULL) { + HorizontalUnfilter(NULL, in, out, width); + } else { + int i; + for (i = 0; i < width; ++i) out[i] = prev[i] + in[i]; + } +} + +static void GradientUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + if (prev == NULL) { + HorizontalUnfilter(NULL, in, out, width); + } else { + uint8_t top = prev[0], top_left = top, left = top; + int i; + for (i = 0; i < width; ++i) { + top = prev[i]; // need to read this first, in case prev==out + left = in[i] + GradientPredictor(left, top, top_left); + top_left = top; + out[i] = left; + } + } +} + +//------------------------------------------------------------------------------ +// Init function + +WebPFilterFunc WebPFilters[WEBP_FILTER_LAST]; +WebPUnfilterFunc WebPUnfilters[WEBP_FILTER_LAST]; + +extern void VP8FiltersInitMIPSdspR2(void); +extern void VP8FiltersInitSSE2(void); + +static volatile VP8CPUInfo filters_last_cpuinfo_used = + (VP8CPUInfo)&filters_last_cpuinfo_used; + +WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInit(void) { + if (filters_last_cpuinfo_used == VP8GetCPUInfo) return; + + WebPUnfilters[WEBP_FILTER_NONE] = NULL; + WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter; + WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter; + WebPUnfilters[WEBP_FILTER_GRADIENT] = GradientUnfilter; + + WebPFilters[WEBP_FILTER_NONE] = NULL; + WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter; + WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter; + WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter; + + if (VP8GetCPUInfo != NULL) { +#if defined(WEBP_USE_SSE2) + if (VP8GetCPUInfo(kSSE2)) { + VP8FiltersInitSSE2(); + } +#endif +#if defined(WEBP_USE_MIPS_DSP_R2) + if (VP8GetCPUInfo(kMIPSdspR2)) { + VP8FiltersInitMIPSdspR2(); + } +#endif + } + filters_last_cpuinfo_used = VP8GetCPUInfo; +} diff --git a/thirdparty/libwebp/dsp/filters_mips_dsp_r2.c b/thirdparty/libwebp/dsp/filters_mips_dsp_r2.c new file mode 100644 index 0000000000..1d82e3c2e1 --- /dev/null +++ b/thirdparty/libwebp/dsp/filters_mips_dsp_r2.c @@ -0,0 +1,395 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Spatial prediction using various filters +// +// Author(s): Branimir Vasic (branimir.vasic@imgtec.com) +// Djordje Pesut (djordje.pesut@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS_DSP_R2) + +#include "../dsp/dsp.h" +#include <assert.h> +#include <stdlib.h> +#include <string.h> + +//------------------------------------------------------------------------------ +// Helpful macro. + +# define SANITY_CHECK(in, out) \ + assert(in != NULL); \ + assert(out != NULL); \ + assert(width > 0); \ + assert(height > 0); \ + assert(stride >= width); \ + assert(row >= 0 && num_rows > 0 && row + num_rows <= height); \ + (void)height; // Silence unused warning. + +#define DO_PREDICT_LINE(SRC, DST, LENGTH, INVERSE) do { \ + const uint8_t* psrc = (uint8_t*)(SRC); \ + uint8_t* pdst = (uint8_t*)(DST); \ + const int ilength = (int)(LENGTH); \ + int temp0, temp1, temp2, temp3, temp4, temp5, temp6; \ + __asm__ volatile ( \ + ".set push \n\t" \ + ".set noreorder \n\t" \ + "srl %[temp0], %[length], 2 \n\t" \ + "beqz %[temp0], 4f \n\t" \ + " andi %[temp6], %[length], 3 \n\t" \ + ".if " #INVERSE " \n\t" \ + "1: \n\t" \ + "lbu %[temp1], -1(%[dst]) \n\t" \ + "lbu %[temp2], 0(%[src]) \n\t" \ + "lbu %[temp3], 1(%[src]) \n\t" \ + "lbu %[temp4], 2(%[src]) \n\t" \ + "lbu %[temp5], 3(%[src]) \n\t" \ + "addu %[temp1], %[temp1], %[temp2] \n\t" \ + "addu %[temp2], %[temp1], %[temp3] \n\t" \ + "addu %[temp3], %[temp2], %[temp4] \n\t" \ + "addu %[temp4], %[temp3], %[temp5] \n\t" \ + "sb %[temp1], 0(%[dst]) \n\t" \ + "sb %[temp2], 1(%[dst]) \n\t" \ + "sb %[temp3], 2(%[dst]) \n\t" \ + "sb %[temp4], 3(%[dst]) \n\t" \ + "addiu %[src], %[src], 4 \n\t" \ + "addiu %[temp0], %[temp0], -1 \n\t" \ + "bnez %[temp0], 1b \n\t" \ + " addiu %[dst], %[dst], 4 \n\t" \ + ".else \n\t" \ + "1: \n\t" \ + "ulw %[temp1], -1(%[src]) \n\t" \ + "ulw %[temp2], 0(%[src]) \n\t" \ + "addiu %[src], %[src], 4 \n\t" \ + "addiu %[temp0], %[temp0], -1 \n\t" \ + "subu.qb %[temp3], %[temp2], %[temp1] \n\t" \ + "usw %[temp3], 0(%[dst]) \n\t" \ + "bnez %[temp0], 1b \n\t" \ + " addiu %[dst], %[dst], 4 \n\t" \ + ".endif \n\t" \ + "4: \n\t" \ + "beqz %[temp6], 3f \n\t" \ + " nop \n\t" \ + "2: \n\t" \ + "lbu %[temp2], 0(%[src]) \n\t" \ + ".if " #INVERSE " \n\t" \ + "lbu %[temp1], -1(%[dst]) \n\t" \ + "addu %[temp3], %[temp1], %[temp2] \n\t" \ + ".else \n\t" \ + "lbu %[temp1], -1(%[src]) \n\t" \ + "subu %[temp3], %[temp1], %[temp2] \n\t" \ + ".endif \n\t" \ + "addiu %[src], %[src], 1 \n\t" \ + "sb %[temp3], 0(%[dst]) \n\t" \ + "addiu %[temp6], %[temp6], -1 \n\t" \ + "bnez %[temp6], 2b \n\t" \ + " addiu %[dst], %[dst], 1 \n\t" \ + "3: \n\t" \ + ".set pop \n\t" \ + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), \ + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), \ + [temp6]"=&r"(temp6), [dst]"+&r"(pdst), [src]"+&r"(psrc) \ + : [length]"r"(ilength) \ + : "memory" \ + ); \ + } while (0) + +static WEBP_INLINE void PredictLine(const uint8_t* src, uint8_t* dst, + int length) { + DO_PREDICT_LINE(src, dst, length, 0); +} + +#define DO_PREDICT_LINE_VERTICAL(SRC, PRED, DST, LENGTH, INVERSE) do { \ + const uint8_t* psrc = (uint8_t*)(SRC); \ + const uint8_t* ppred = (uint8_t*)(PRED); \ + uint8_t* pdst = (uint8_t*)(DST); \ + const int ilength = (int)(LENGTH); \ + int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; \ + __asm__ volatile ( \ + ".set push \n\t" \ + ".set noreorder \n\t" \ + "srl %[temp0], %[length], 0x3 \n\t" \ + "beqz %[temp0], 4f \n\t" \ + " andi %[temp7], %[length], 0x7 \n\t" \ + "1: \n\t" \ + "ulw %[temp1], 0(%[src]) \n\t" \ + "ulw %[temp2], 0(%[pred]) \n\t" \ + "ulw %[temp3], 4(%[src]) \n\t" \ + "ulw %[temp4], 4(%[pred]) \n\t" \ + "addiu %[src], %[src], 8 \n\t" \ + ".if " #INVERSE " \n\t" \ + "addu.qb %[temp5], %[temp1], %[temp2] \n\t" \ + "addu.qb %[temp6], %[temp3], %[temp4] \n\t" \ + ".else \n\t" \ + "subu.qb %[temp5], %[temp1], %[temp2] \n\t" \ + "subu.qb %[temp6], %[temp3], %[temp4] \n\t" \ + ".endif \n\t" \ + "addiu %[pred], %[pred], 8 \n\t" \ + "usw %[temp5], 0(%[dst]) \n\t" \ + "usw %[temp6], 4(%[dst]) \n\t" \ + "addiu %[temp0], %[temp0], -1 \n\t" \ + "bnez %[temp0], 1b \n\t" \ + " addiu %[dst], %[dst], 8 \n\t" \ + "4: \n\t" \ + "beqz %[temp7], 3f \n\t" \ + " nop \n\t" \ + "2: \n\t" \ + "lbu %[temp1], 0(%[src]) \n\t" \ + "lbu %[temp2], 0(%[pred]) \n\t" \ + "addiu %[src], %[src], 1 \n\t" \ + "addiu %[pred], %[pred], 1 \n\t" \ + ".if " #INVERSE " \n\t" \ + "addu %[temp3], %[temp1], %[temp2] \n\t" \ + ".else \n\t" \ + "subu %[temp3], %[temp1], %[temp2] \n\t" \ + ".endif \n\t" \ + "sb %[temp3], 0(%[dst]) \n\t" \ + "addiu %[temp7], %[temp7], -1 \n\t" \ + "bnez %[temp7], 2b \n\t" \ + " addiu %[dst], %[dst], 1 \n\t" \ + "3: \n\t" \ + ".set pop \n\t" \ + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), \ + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), \ + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [pred]"+&r"(ppred), \ + [dst]"+&r"(pdst), [src]"+&r"(psrc) \ + : [length]"r"(ilength) \ + : "memory" \ + ); \ + } while (0) + +#define PREDICT_LINE_ONE_PASS(SRC, PRED, DST) do { \ + int temp1, temp2, temp3; \ + __asm__ volatile ( \ + "lbu %[temp1], 0(%[src]) \n\t" \ + "lbu %[temp2], 0(%[pred]) \n\t" \ + "subu %[temp3], %[temp1], %[temp2] \n\t" \ + "sb %[temp3], 0(%[dst]) \n\t" \ + : [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), [temp3]"=&r"(temp3) \ + : [pred]"r"((PRED)), [dst]"r"((DST)), [src]"r"((SRC)) \ + : "memory" \ + ); \ + } while (0) + +//------------------------------------------------------------------------------ +// Horizontal filter. + +#define FILTER_LINE_BY_LINE do { \ + while (row < last_row) { \ + PREDICT_LINE_ONE_PASS(in, preds - stride, out); \ + DO_PREDICT_LINE(in + 1, out + 1, width - 1, 0); \ + ++row; \ + preds += stride; \ + in += stride; \ + out += stride; \ + } \ + } while (0) + +static WEBP_INLINE void DoHorizontalFilter(const uint8_t* in, + int width, int height, int stride, + int row, int num_rows, + uint8_t* out) { + const uint8_t* preds; + const size_t start_offset = row * stride; + const int last_row = row + num_rows; + SANITY_CHECK(in, out); + in += start_offset; + out += start_offset; + preds = in; + + if (row == 0) { + // Leftmost pixel is the same as input for topmost scanline. + out[0] = in[0]; + PredictLine(in + 1, out + 1, width - 1); + row = 1; + preds += stride; + in += stride; + out += stride; + } + + // Filter line-by-line. + FILTER_LINE_BY_LINE; +} +#undef FILTER_LINE_BY_LINE + +static void HorizontalFilter(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + DoHorizontalFilter(data, width, height, stride, 0, height, filtered_data); +} + +//------------------------------------------------------------------------------ +// Vertical filter. + +#define FILTER_LINE_BY_LINE do { \ + while (row < last_row) { \ + DO_PREDICT_LINE_VERTICAL(in, preds, out, width, 0); \ + ++row; \ + preds += stride; \ + in += stride; \ + out += stride; \ + } \ + } while (0) + +static WEBP_INLINE void DoVerticalFilter(const uint8_t* in, + int width, int height, int stride, + int row, int num_rows, uint8_t* out) { + const uint8_t* preds; + const size_t start_offset = row * stride; + const int last_row = row + num_rows; + SANITY_CHECK(in, out); + in += start_offset; + out += start_offset; + preds = in; + + if (row == 0) { + // Very first top-left pixel is copied. + out[0] = in[0]; + // Rest of top scan-line is left-predicted. + PredictLine(in + 1, out + 1, width - 1); + row = 1; + in += stride; + out += stride; + } else { + // We are starting from in-between. Make sure 'preds' points to prev row. + preds -= stride; + } + + // Filter line-by-line. + FILTER_LINE_BY_LINE; +} +#undef FILTER_LINE_BY_LINE + +static void VerticalFilter(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + DoVerticalFilter(data, width, height, stride, 0, height, filtered_data); +} + +//------------------------------------------------------------------------------ +// Gradient filter. + +static WEBP_INLINE int GradientPredictor(uint8_t a, uint8_t b, uint8_t c) { + int temp0; + __asm__ volatile ( + "addu %[temp0], %[a], %[b] \n\t" + "subu %[temp0], %[temp0], %[c] \n\t" + "shll_s.w %[temp0], %[temp0], 23 \n\t" + "precrqu_s.qb.ph %[temp0], %[temp0], $zero \n\t" + "srl %[temp0], %[temp0], 24 \n\t" + : [temp0]"=&r"(temp0) + : [a]"r"(a),[b]"r"(b),[c]"r"(c) + ); + return temp0; +} + +#define FILTER_LINE_BY_LINE(PREDS, OPERATION) do { \ + while (row < last_row) { \ + int w; \ + PREDICT_LINE_ONE_PASS(in, PREDS - stride, out); \ + for (w = 1; w < width; ++w) { \ + const int pred = GradientPredictor(PREDS[w - 1], \ + PREDS[w - stride], \ + PREDS[w - stride - 1]); \ + out[w] = in[w] OPERATION pred; \ + } \ + ++row; \ + in += stride; \ + out += stride; \ + } \ + } while (0) + +static WEBP_INLINE void DoGradientFilter(const uint8_t* in, + int width, int height, int stride, + int row, int num_rows, uint8_t* out) { + const uint8_t* preds; + const size_t start_offset = row * stride; + const int last_row = row + num_rows; + SANITY_CHECK(in, out); + in += start_offset; + out += start_offset; + preds = in; + + // left prediction for top scan-line + if (row == 0) { + out[0] = in[0]; + PredictLine(in + 1, out + 1, width - 1); + row = 1; + preds += stride; + in += stride; + out += stride; + } + + // Filter line-by-line. + FILTER_LINE_BY_LINE(in, -); +} +#undef FILTER_LINE_BY_LINE + +static void GradientFilter(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + DoGradientFilter(data, width, height, stride, 0, height, filtered_data); +} + +//------------------------------------------------------------------------------ + +static void HorizontalUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + out[0] = in[0] + (prev == NULL ? 0 : prev[0]); + DO_PREDICT_LINE(in + 1, out + 1, width - 1, 1); +} + +static void VerticalUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + if (prev == NULL) { + HorizontalUnfilter(NULL, in, out, width); + } else { + DO_PREDICT_LINE_VERTICAL(in, prev, out, width, 1); + } +} + +static void GradientUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + if (prev == NULL) { + HorizontalUnfilter(NULL, in, out, width); + } else { + uint8_t top = prev[0], top_left = top, left = top; + int i; + for (i = 0; i < width; ++i) { + top = prev[i]; // need to read this first, in case prev==dst + left = in[i] + GradientPredictor(left, top, top_left); + top_left = top; + out[i] = left; + } + } +} + +#undef DO_PREDICT_LINE_VERTICAL +#undef PREDICT_LINE_ONE_PASS +#undef DO_PREDICT_LINE +#undef SANITY_CHECK + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8FiltersInitMIPSdspR2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInitMIPSdspR2(void) { + WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter; + WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter; + WebPUnfilters[WEBP_FILTER_GRADIENT] = GradientUnfilter; + + WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter; + WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter; + WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter; +} + +#else // !WEBP_USE_MIPS_DSP_R2 + +WEBP_DSP_INIT_STUB(VP8FiltersInitMIPSdspR2) + +#endif // WEBP_USE_MIPS_DSP_R2 diff --git a/thirdparty/libwebp/dsp/filters_sse2.c b/thirdparty/libwebp/dsp/filters_sse2.c new file mode 100644 index 0000000000..67f77999e6 --- /dev/null +++ b/thirdparty/libwebp/dsp/filters_sse2.c @@ -0,0 +1,330 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE2 variant of alpha filters +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE2) + +#include <assert.h> +#include <emmintrin.h> +#include <stdlib.h> +#include <string.h> + +//------------------------------------------------------------------------------ +// Helpful macro. + +# define SANITY_CHECK(in, out) \ + assert(in != NULL); \ + assert(out != NULL); \ + assert(width > 0); \ + assert(height > 0); \ + assert(stride >= width); \ + assert(row >= 0 && num_rows > 0 && row + num_rows <= height); \ + (void)height; // Silence unused warning. + +static void PredictLineTop(const uint8_t* src, const uint8_t* pred, + uint8_t* dst, int length) { + int i; + const int max_pos = length & ~31; + assert(length >= 0); + for (i = 0; i < max_pos; i += 32) { + const __m128i A0 = _mm_loadu_si128((const __m128i*)&src[i + 0]); + const __m128i A1 = _mm_loadu_si128((const __m128i*)&src[i + 16]); + const __m128i B0 = _mm_loadu_si128((const __m128i*)&pred[i + 0]); + const __m128i B1 = _mm_loadu_si128((const __m128i*)&pred[i + 16]); + const __m128i C0 = _mm_sub_epi8(A0, B0); + const __m128i C1 = _mm_sub_epi8(A1, B1); + _mm_storeu_si128((__m128i*)&dst[i + 0], C0); + _mm_storeu_si128((__m128i*)&dst[i + 16], C1); + } + for (; i < length; ++i) dst[i] = src[i] - pred[i]; +} + +// Special case for left-based prediction (when preds==dst-1 or preds==src-1). +static void PredictLineLeft(const uint8_t* src, uint8_t* dst, int length) { + int i; + const int max_pos = length & ~31; + assert(length >= 0); + for (i = 0; i < max_pos; i += 32) { + const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + i + 0 )); + const __m128i B0 = _mm_loadu_si128((const __m128i*)(src + i + 0 - 1)); + const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + i + 16 )); + const __m128i B1 = _mm_loadu_si128((const __m128i*)(src + i + 16 - 1)); + const __m128i C0 = _mm_sub_epi8(A0, B0); + const __m128i C1 = _mm_sub_epi8(A1, B1); + _mm_storeu_si128((__m128i*)(dst + i + 0), C0); + _mm_storeu_si128((__m128i*)(dst + i + 16), C1); + } + for (; i < length; ++i) dst[i] = src[i] - src[i - 1]; +} + +//------------------------------------------------------------------------------ +// Horizontal filter. + +static WEBP_INLINE void DoHorizontalFilter(const uint8_t* in, + int width, int height, int stride, + int row, int num_rows, + uint8_t* out) { + const size_t start_offset = row * stride; + const int last_row = row + num_rows; + SANITY_CHECK(in, out); + in += start_offset; + out += start_offset; + + if (row == 0) { + // Leftmost pixel is the same as input for topmost scanline. + out[0] = in[0]; + PredictLineLeft(in + 1, out + 1, width - 1); + row = 1; + in += stride; + out += stride; + } + + // Filter line-by-line. + while (row < last_row) { + // Leftmost pixel is predicted from above. + out[0] = in[0] - in[-stride]; + PredictLineLeft(in + 1, out + 1, width - 1); + ++row; + in += stride; + out += stride; + } +} + +//------------------------------------------------------------------------------ +// Vertical filter. + +static WEBP_INLINE void DoVerticalFilter(const uint8_t* in, + int width, int height, int stride, + int row, int num_rows, uint8_t* out) { + const size_t start_offset = row * stride; + const int last_row = row + num_rows; + SANITY_CHECK(in, out); + in += start_offset; + out += start_offset; + + if (row == 0) { + // Very first top-left pixel is copied. + out[0] = in[0]; + // Rest of top scan-line is left-predicted. + PredictLineLeft(in + 1, out + 1, width - 1); + row = 1; + in += stride; + out += stride; + } + + // Filter line-by-line. + while (row < last_row) { + PredictLineTop(in, in - stride, out, width); + ++row; + in += stride; + out += stride; + } +} + +//------------------------------------------------------------------------------ +// Gradient filter. + +static WEBP_INLINE int GradientPredictorC(uint8_t a, uint8_t b, uint8_t c) { + const int g = a + b - c; + return ((g & ~0xff) == 0) ? g : (g < 0) ? 0 : 255; // clip to 8bit +} + +static void GradientPredictDirect(const uint8_t* const row, + const uint8_t* const top, + uint8_t* const out, int length) { + const int max_pos = length & ~7; + int i; + const __m128i zero = _mm_setzero_si128(); + for (i = 0; i < max_pos; i += 8) { + const __m128i A0 = _mm_loadl_epi64((const __m128i*)&row[i - 1]); + const __m128i B0 = _mm_loadl_epi64((const __m128i*)&top[i]); + const __m128i C0 = _mm_loadl_epi64((const __m128i*)&top[i - 1]); + const __m128i D = _mm_loadl_epi64((const __m128i*)&row[i]); + const __m128i A1 = _mm_unpacklo_epi8(A0, zero); + const __m128i B1 = _mm_unpacklo_epi8(B0, zero); + const __m128i C1 = _mm_unpacklo_epi8(C0, zero); + const __m128i E = _mm_add_epi16(A1, B1); + const __m128i F = _mm_sub_epi16(E, C1); + const __m128i G = _mm_packus_epi16(F, zero); + const __m128i H = _mm_sub_epi8(D, G); + _mm_storel_epi64((__m128i*)(out + i), H); + } + for (; i < length; ++i) { + out[i] = row[i] - GradientPredictorC(row[i - 1], top[i], top[i - 1]); + } +} + +static WEBP_INLINE void DoGradientFilter(const uint8_t* in, + int width, int height, int stride, + int row, int num_rows, + uint8_t* out) { + const size_t start_offset = row * stride; + const int last_row = row + num_rows; + SANITY_CHECK(in, out); + in += start_offset; + out += start_offset; + + // left prediction for top scan-line + if (row == 0) { + out[0] = in[0]; + PredictLineLeft(in + 1, out + 1, width - 1); + row = 1; + in += stride; + out += stride; + } + + // Filter line-by-line. + while (row < last_row) { + out[0] = in[0] - in[-stride]; + GradientPredictDirect(in + 1, in + 1 - stride, out + 1, width - 1); + ++row; + in += stride; + out += stride; + } +} + +#undef SANITY_CHECK + +//------------------------------------------------------------------------------ + +static void HorizontalFilter(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + DoHorizontalFilter(data, width, height, stride, 0, height, filtered_data); +} + +static void VerticalFilter(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + DoVerticalFilter(data, width, height, stride, 0, height, filtered_data); +} + +static void GradientFilter(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + DoGradientFilter(data, width, height, stride, 0, height, filtered_data); +} + +//------------------------------------------------------------------------------ +// Inverse transforms + +static void HorizontalUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + int i; + __m128i last; + out[0] = in[0] + (prev == NULL ? 0 : prev[0]); + if (width <= 1) return; + last = _mm_set_epi32(0, 0, 0, out[0]); + for (i = 1; i + 8 <= width; i += 8) { + const __m128i A0 = _mm_loadl_epi64((const __m128i*)(in + i)); + const __m128i A1 = _mm_add_epi8(A0, last); + const __m128i A2 = _mm_slli_si128(A1, 1); + const __m128i A3 = _mm_add_epi8(A1, A2); + const __m128i A4 = _mm_slli_si128(A3, 2); + const __m128i A5 = _mm_add_epi8(A3, A4); + const __m128i A6 = _mm_slli_si128(A5, 4); + const __m128i A7 = _mm_add_epi8(A5, A6); + _mm_storel_epi64((__m128i*)(out + i), A7); + last = _mm_srli_epi64(A7, 56); + } + for (; i < width; ++i) out[i] = in[i] + out[i - 1]; +} + +static void VerticalUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + if (prev == NULL) { + HorizontalUnfilter(NULL, in, out, width); + } else { + int i; + const int max_pos = width & ~31; + assert(width >= 0); + for (i = 0; i < max_pos; i += 32) { + const __m128i A0 = _mm_loadu_si128((const __m128i*)&in[i + 0]); + const __m128i A1 = _mm_loadu_si128((const __m128i*)&in[i + 16]); + const __m128i B0 = _mm_loadu_si128((const __m128i*)&prev[i + 0]); + const __m128i B1 = _mm_loadu_si128((const __m128i*)&prev[i + 16]); + const __m128i C0 = _mm_add_epi8(A0, B0); + const __m128i C1 = _mm_add_epi8(A1, B1); + _mm_storeu_si128((__m128i*)&out[i + 0], C0); + _mm_storeu_si128((__m128i*)&out[i + 16], C1); + } + for (; i < width; ++i) out[i] = in[i] + prev[i]; + } +} + +static void GradientPredictInverse(const uint8_t* const in, + const uint8_t* const top, + uint8_t* const row, int length) { + if (length > 0) { + int i; + const int max_pos = length & ~7; + const __m128i zero = _mm_setzero_si128(); + __m128i A = _mm_set_epi32(0, 0, 0, row[-1]); // left sample + for (i = 0; i < max_pos; i += 8) { + const __m128i tmp0 = _mm_loadl_epi64((const __m128i*)&top[i]); + const __m128i tmp1 = _mm_loadl_epi64((const __m128i*)&top[i - 1]); + const __m128i B = _mm_unpacklo_epi8(tmp0, zero); + const __m128i C = _mm_unpacklo_epi8(tmp1, zero); + const __m128i D = _mm_loadl_epi64((const __m128i*)&in[i]); // base input + const __m128i E = _mm_sub_epi16(B, C); // unclipped gradient basis B - C + __m128i out = zero; // accumulator for output + __m128i mask_hi = _mm_set_epi32(0, 0, 0, 0xff); + int k = 8; + while (1) { + const __m128i tmp3 = _mm_add_epi16(A, E); // delta = A + B - C + const __m128i tmp4 = _mm_packus_epi16(tmp3, zero); // saturate delta + const __m128i tmp5 = _mm_add_epi8(tmp4, D); // add to in[] + A = _mm_and_si128(tmp5, mask_hi); // 1-complement clip + out = _mm_or_si128(out, A); // accumulate output + if (--k == 0) break; + A = _mm_slli_si128(A, 1); // rotate left sample + mask_hi = _mm_slli_si128(mask_hi, 1); // rotate mask + A = _mm_unpacklo_epi8(A, zero); // convert 8b->16b + } + A = _mm_srli_si128(A, 7); // prepare left sample for next iteration + _mm_storel_epi64((__m128i*)&row[i], out); + } + for (; i < length; ++i) { + row[i] = in[i] + GradientPredictorC(row[i - 1], top[i], top[i - 1]); + } + } +} + +static void GradientUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + if (prev == NULL) { + HorizontalUnfilter(NULL, in, out, width); + } else { + out[0] = in[0] + prev[0]; // predict from above + GradientPredictInverse(in + 1, prev + 1, out + 1, width - 1); + } +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8FiltersInitSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInitSSE2(void) { + WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter; + WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter; + WebPUnfilters[WEBP_FILTER_GRADIENT] = GradientUnfilter; + + WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter; + WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter; + WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter; +} + +#else // !WEBP_USE_SSE2 + +WEBP_DSP_INIT_STUB(VP8FiltersInitSSE2) + +#endif // WEBP_USE_SSE2 diff --git a/thirdparty/libwebp/dsp/lossless.c b/thirdparty/libwebp/dsp/lossless.c new file mode 100644 index 0000000000..af913efccb --- /dev/null +++ b/thirdparty/libwebp/dsp/lossless.c @@ -0,0 +1,633 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Image transforms and color space conversion methods for lossless decoder. +// +// Authors: Vikas Arora (vikaas.arora@gmail.com) +// Jyrki Alakuijala (jyrki@google.com) +// Urvang Joshi (urvang@google.com) + +#include "./dsp.h" + +#include <math.h> +#include <stdlib.h> +#include "../dec/vp8li.h" +#include "../utils/endian_inl.h" +#include "./lossless.h" + +#define MAX_DIFF_COST (1e30f) + +//------------------------------------------------------------------------------ +// Image transforms. + +// In-place sum of each component with mod 256. +static WEBP_INLINE void AddPixelsEq(uint32_t* a, uint32_t b) { + *a = VP8LAddPixels(*a, b); +} + +static WEBP_INLINE uint32_t Average2(uint32_t a0, uint32_t a1) { + return (((a0 ^ a1) & 0xfefefefeu) >> 1) + (a0 & a1); +} + +static WEBP_INLINE uint32_t Average3(uint32_t a0, uint32_t a1, uint32_t a2) { + return Average2(Average2(a0, a2), a1); +} + +static WEBP_INLINE uint32_t Average4(uint32_t a0, uint32_t a1, + uint32_t a2, uint32_t a3) { + return Average2(Average2(a0, a1), Average2(a2, a3)); +} + +static WEBP_INLINE uint32_t Clip255(uint32_t a) { + if (a < 256) { + return a; + } + // return 0, when a is a negative integer. + // return 255, when a is positive. + return ~a >> 24; +} + +static WEBP_INLINE int AddSubtractComponentFull(int a, int b, int c) { + return Clip255(a + b - c); +} + +static WEBP_INLINE uint32_t ClampedAddSubtractFull(uint32_t c0, uint32_t c1, + uint32_t c2) { + const int a = AddSubtractComponentFull(c0 >> 24, c1 >> 24, c2 >> 24); + const int r = AddSubtractComponentFull((c0 >> 16) & 0xff, + (c1 >> 16) & 0xff, + (c2 >> 16) & 0xff); + const int g = AddSubtractComponentFull((c0 >> 8) & 0xff, + (c1 >> 8) & 0xff, + (c2 >> 8) & 0xff); + const int b = AddSubtractComponentFull(c0 & 0xff, c1 & 0xff, c2 & 0xff); + return ((uint32_t)a << 24) | (r << 16) | (g << 8) | b; +} + +static WEBP_INLINE int AddSubtractComponentHalf(int a, int b) { + return Clip255(a + (a - b) / 2); +} + +static WEBP_INLINE uint32_t ClampedAddSubtractHalf(uint32_t c0, uint32_t c1, + uint32_t c2) { + const uint32_t ave = Average2(c0, c1); + const int a = AddSubtractComponentHalf(ave >> 24, c2 >> 24); + const int r = AddSubtractComponentHalf((ave >> 16) & 0xff, (c2 >> 16) & 0xff); + const int g = AddSubtractComponentHalf((ave >> 8) & 0xff, (c2 >> 8) & 0xff); + const int b = AddSubtractComponentHalf((ave >> 0) & 0xff, (c2 >> 0) & 0xff); + return ((uint32_t)a << 24) | (r << 16) | (g << 8) | b; +} + +// gcc-4.9 on ARM generates incorrect code in Select() when Sub3() is inlined. +#if defined(__arm__) && LOCAL_GCC_VERSION == 0x409 +# define LOCAL_INLINE __attribute__ ((noinline)) +#else +# define LOCAL_INLINE WEBP_INLINE +#endif + +static LOCAL_INLINE int Sub3(int a, int b, int c) { + const int pb = b - c; + const int pa = a - c; + return abs(pb) - abs(pa); +} + +#undef LOCAL_INLINE + +static WEBP_INLINE uint32_t Select(uint32_t a, uint32_t b, uint32_t c) { + const int pa_minus_pb = + Sub3((a >> 24) , (b >> 24) , (c >> 24) ) + + Sub3((a >> 16) & 0xff, (b >> 16) & 0xff, (c >> 16) & 0xff) + + Sub3((a >> 8) & 0xff, (b >> 8) & 0xff, (c >> 8) & 0xff) + + Sub3((a ) & 0xff, (b ) & 0xff, (c ) & 0xff); + return (pa_minus_pb <= 0) ? a : b; +} + +//------------------------------------------------------------------------------ +// Predictors + +static uint32_t Predictor0(uint32_t left, const uint32_t* const top) { + (void)top; + (void)left; + return ARGB_BLACK; +} +static uint32_t Predictor1(uint32_t left, const uint32_t* const top) { + (void)top; + return left; +} +static uint32_t Predictor2(uint32_t left, const uint32_t* const top) { + (void)left; + return top[0]; +} +static uint32_t Predictor3(uint32_t left, const uint32_t* const top) { + (void)left; + return top[1]; +} +static uint32_t Predictor4(uint32_t left, const uint32_t* const top) { + (void)left; + return top[-1]; +} +static uint32_t Predictor5(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average3(left, top[0], top[1]); + return pred; +} +static uint32_t Predictor6(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average2(left, top[-1]); + return pred; +} +static uint32_t Predictor7(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average2(left, top[0]); + return pred; +} +static uint32_t Predictor8(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average2(top[-1], top[0]); + (void)left; + return pred; +} +static uint32_t Predictor9(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average2(top[0], top[1]); + (void)left; + return pred; +} +static uint32_t Predictor10(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average4(left, top[-1], top[0], top[1]); + return pred; +} +static uint32_t Predictor11(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Select(top[0], left, top[-1]); + return pred; +} +static uint32_t Predictor12(uint32_t left, const uint32_t* const top) { + const uint32_t pred = ClampedAddSubtractFull(left, top[0], top[-1]); + return pred; +} +static uint32_t Predictor13(uint32_t left, const uint32_t* const top) { + const uint32_t pred = ClampedAddSubtractHalf(left, top[0], top[-1]); + return pred; +} + +//------------------------------------------------------------------------------ + +// Inverse prediction. +static void PredictorInverseTransform(const VP8LTransform* const transform, + int y_start, int y_end, uint32_t* data) { + const int width = transform->xsize_; + if (y_start == 0) { // First Row follows the L (mode=1) mode. + int x; + const uint32_t pred0 = Predictor0(data[-1], NULL); + AddPixelsEq(data, pred0); + for (x = 1; x < width; ++x) { + const uint32_t pred1 = Predictor1(data[x - 1], NULL); + AddPixelsEq(data + x, pred1); + } + data += width; + ++y_start; + } + + { + int y = y_start; + const int tile_width = 1 << transform->bits_; + const int mask = tile_width - 1; + const int safe_width = width & ~mask; + const int tiles_per_row = VP8LSubSampleSize(width, transform->bits_); + const uint32_t* pred_mode_base = + transform->data_ + (y >> transform->bits_) * tiles_per_row; + + while (y < y_end) { + const uint32_t pred2 = Predictor2(data[-1], data - width); + const uint32_t* pred_mode_src = pred_mode_base; + VP8LPredictorFunc pred_func; + int x = 1; + int t = 1; + // First pixel follows the T (mode=2) mode. + AddPixelsEq(data, pred2); + // .. the rest: + while (x < safe_width) { + pred_func = VP8LPredictors[((*pred_mode_src++) >> 8) & 0xf]; + for (; t < tile_width; ++t, ++x) { + const uint32_t pred = pred_func(data[x - 1], data + x - width); + AddPixelsEq(data + x, pred); + } + t = 0; + } + if (x < width) { + pred_func = VP8LPredictors[((*pred_mode_src++) >> 8) & 0xf]; + for (; x < width; ++x) { + const uint32_t pred = pred_func(data[x - 1], data + x - width); + AddPixelsEq(data + x, pred); + } + } + data += width; + ++y; + if ((y & mask) == 0) { // Use the same mask, since tiles are squares. + pred_mode_base += tiles_per_row; + } + } + } +} + +// Add green to blue and red channels (i.e. perform the inverse transform of +// 'subtract green'). +void VP8LAddGreenToBlueAndRed_C(uint32_t* data, int num_pixels) { + int i; + for (i = 0; i < num_pixels; ++i) { + const uint32_t argb = data[i]; + const uint32_t green = ((argb >> 8) & 0xff); + uint32_t red_blue = (argb & 0x00ff00ffu); + red_blue += (green << 16) | green; + red_blue &= 0x00ff00ffu; + data[i] = (argb & 0xff00ff00u) | red_blue; + } +} + +static WEBP_INLINE uint32_t ColorTransformDelta(int8_t color_pred, + int8_t color) { + return (uint32_t)((int)(color_pred) * color) >> 5; +} + +static WEBP_INLINE void ColorCodeToMultipliers(uint32_t color_code, + VP8LMultipliers* const m) { + m->green_to_red_ = (color_code >> 0) & 0xff; + m->green_to_blue_ = (color_code >> 8) & 0xff; + m->red_to_blue_ = (color_code >> 16) & 0xff; +} + +void VP8LTransformColorInverse_C(const VP8LMultipliers* const m, uint32_t* data, + int num_pixels) { + int i; + for (i = 0; i < num_pixels; ++i) { + const uint32_t argb = data[i]; + const uint32_t green = argb >> 8; + const uint32_t red = argb >> 16; + uint32_t new_red = red; + uint32_t new_blue = argb; + new_red += ColorTransformDelta(m->green_to_red_, green); + new_red &= 0xff; + new_blue += ColorTransformDelta(m->green_to_blue_, green); + new_blue += ColorTransformDelta(m->red_to_blue_, new_red); + new_blue &= 0xff; + data[i] = (argb & 0xff00ff00u) | (new_red << 16) | (new_blue); + } +} + +// Color space inverse transform. +static void ColorSpaceInverseTransform(const VP8LTransform* const transform, + int y_start, int y_end, uint32_t* data) { + const int width = transform->xsize_; + const int tile_width = 1 << transform->bits_; + const int mask = tile_width - 1; + const int safe_width = width & ~mask; + const int remaining_width = width - safe_width; + const int tiles_per_row = VP8LSubSampleSize(width, transform->bits_); + int y = y_start; + const uint32_t* pred_row = + transform->data_ + (y >> transform->bits_) * tiles_per_row; + + while (y < y_end) { + const uint32_t* pred = pred_row; + VP8LMultipliers m = { 0, 0, 0 }; + const uint32_t* const data_safe_end = data + safe_width; + const uint32_t* const data_end = data + width; + while (data < data_safe_end) { + ColorCodeToMultipliers(*pred++, &m); + VP8LTransformColorInverse(&m, data, tile_width); + data += tile_width; + } + if (data < data_end) { // Left-overs using C-version. + ColorCodeToMultipliers(*pred++, &m); + VP8LTransformColorInverse(&m, data, remaining_width); + data += remaining_width; + } + ++y; + if ((y & mask) == 0) pred_row += tiles_per_row; + } +} + +// Separate out pixels packed together using pixel-bundling. +// We define two methods for ARGB data (uint32_t) and alpha-only data (uint8_t). +#define COLOR_INDEX_INVERSE(FUNC_NAME, F_NAME, STATIC_DECL, TYPE, BIT_SUFFIX, \ + GET_INDEX, GET_VALUE) \ +static void F_NAME(const TYPE* src, const uint32_t* const color_map, \ + TYPE* dst, int y_start, int y_end, int width) { \ + int y; \ + for (y = y_start; y < y_end; ++y) { \ + int x; \ + for (x = 0; x < width; ++x) { \ + *dst++ = GET_VALUE(color_map[GET_INDEX(*src++)]); \ + } \ + } \ +} \ +STATIC_DECL void FUNC_NAME(const VP8LTransform* const transform, \ + int y_start, int y_end, const TYPE* src, \ + TYPE* dst) { \ + int y; \ + const int bits_per_pixel = 8 >> transform->bits_; \ + const int width = transform->xsize_; \ + const uint32_t* const color_map = transform->data_; \ + if (bits_per_pixel < 8) { \ + const int pixels_per_byte = 1 << transform->bits_; \ + const int count_mask = pixels_per_byte - 1; \ + const uint32_t bit_mask = (1 << bits_per_pixel) - 1; \ + for (y = y_start; y < y_end; ++y) { \ + uint32_t packed_pixels = 0; \ + int x; \ + for (x = 0; x < width; ++x) { \ + /* We need to load fresh 'packed_pixels' once every */ \ + /* 'pixels_per_byte' increments of x. Fortunately, pixels_per_byte */ \ + /* is a power of 2, so can just use a mask for that, instead of */ \ + /* decrementing a counter. */ \ + if ((x & count_mask) == 0) packed_pixels = GET_INDEX(*src++); \ + *dst++ = GET_VALUE(color_map[packed_pixels & bit_mask]); \ + packed_pixels >>= bits_per_pixel; \ + } \ + } \ + } else { \ + VP8LMapColor##BIT_SUFFIX(src, color_map, dst, y_start, y_end, width); \ + } \ +} + +COLOR_INDEX_INVERSE(ColorIndexInverseTransform, MapARGB, static, uint32_t, 32b, + VP8GetARGBIndex, VP8GetARGBValue) +COLOR_INDEX_INVERSE(VP8LColorIndexInverseTransformAlpha, MapAlpha, , uint8_t, + 8b, VP8GetAlphaIndex, VP8GetAlphaValue) + +#undef COLOR_INDEX_INVERSE + +void VP8LInverseTransform(const VP8LTransform* const transform, + int row_start, int row_end, + const uint32_t* const in, uint32_t* const out) { + const int width = transform->xsize_; + assert(row_start < row_end); + assert(row_end <= transform->ysize_); + switch (transform->type_) { + case SUBTRACT_GREEN: + VP8LAddGreenToBlueAndRed(out, (row_end - row_start) * width); + break; + case PREDICTOR_TRANSFORM: + PredictorInverseTransform(transform, row_start, row_end, out); + if (row_end != transform->ysize_) { + // The last predicted row in this iteration will be the top-pred row + // for the first row in next iteration. + memcpy(out - width, out + (row_end - row_start - 1) * width, + width * sizeof(*out)); + } + break; + case CROSS_COLOR_TRANSFORM: + ColorSpaceInverseTransform(transform, row_start, row_end, out); + break; + case COLOR_INDEXING_TRANSFORM: + if (in == out && transform->bits_ > 0) { + // Move packed pixels to the end of unpacked region, so that unpacking + // can occur seamlessly. + // Also, note that this is the only transform that applies on + // the effective width of VP8LSubSampleSize(xsize_, bits_). All other + // transforms work on effective width of xsize_. + const int out_stride = (row_end - row_start) * width; + const int in_stride = (row_end - row_start) * + VP8LSubSampleSize(transform->xsize_, transform->bits_); + uint32_t* const src = out + out_stride - in_stride; + memmove(src, out, in_stride * sizeof(*src)); + ColorIndexInverseTransform(transform, row_start, row_end, src, out); + } else { + ColorIndexInverseTransform(transform, row_start, row_end, in, out); + } + break; + } +} + +//------------------------------------------------------------------------------ +// Color space conversion. + +static int is_big_endian(void) { + static const union { + uint16_t w; + uint8_t b[2]; + } tmp = { 1 }; + return (tmp.b[0] != 1); +} + +void VP8LConvertBGRAToRGB_C(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const uint32_t* const src_end = src + num_pixels; + while (src < src_end) { + const uint32_t argb = *src++; + *dst++ = (argb >> 16) & 0xff; + *dst++ = (argb >> 8) & 0xff; + *dst++ = (argb >> 0) & 0xff; + } +} + +void VP8LConvertBGRAToRGBA_C(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const uint32_t* const src_end = src + num_pixels; + while (src < src_end) { + const uint32_t argb = *src++; + *dst++ = (argb >> 16) & 0xff; + *dst++ = (argb >> 8) & 0xff; + *dst++ = (argb >> 0) & 0xff; + *dst++ = (argb >> 24) & 0xff; + } +} + +void VP8LConvertBGRAToRGBA4444_C(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const uint32_t* const src_end = src + num_pixels; + while (src < src_end) { + const uint32_t argb = *src++; + const uint8_t rg = ((argb >> 16) & 0xf0) | ((argb >> 12) & 0xf); + const uint8_t ba = ((argb >> 0) & 0xf0) | ((argb >> 28) & 0xf); +#ifdef WEBP_SWAP_16BIT_CSP + *dst++ = ba; + *dst++ = rg; +#else + *dst++ = rg; + *dst++ = ba; +#endif + } +} + +void VP8LConvertBGRAToRGB565_C(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const uint32_t* const src_end = src + num_pixels; + while (src < src_end) { + const uint32_t argb = *src++; + const uint8_t rg = ((argb >> 16) & 0xf8) | ((argb >> 13) & 0x7); + const uint8_t gb = ((argb >> 5) & 0xe0) | ((argb >> 3) & 0x1f); +#ifdef WEBP_SWAP_16BIT_CSP + *dst++ = gb; + *dst++ = rg; +#else + *dst++ = rg; + *dst++ = gb; +#endif + } +} + +void VP8LConvertBGRAToBGR_C(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const uint32_t* const src_end = src + num_pixels; + while (src < src_end) { + const uint32_t argb = *src++; + *dst++ = (argb >> 0) & 0xff; + *dst++ = (argb >> 8) & 0xff; + *dst++ = (argb >> 16) & 0xff; + } +} + +static void CopyOrSwap(const uint32_t* src, int num_pixels, uint8_t* dst, + int swap_on_big_endian) { + if (is_big_endian() == swap_on_big_endian) { + const uint32_t* const src_end = src + num_pixels; + while (src < src_end) { + const uint32_t argb = *src++; + +#if !defined(WORDS_BIGENDIAN) +#if !defined(WEBP_REFERENCE_IMPLEMENTATION) + WebPUint32ToMem(dst, BSwap32(argb)); +#else // WEBP_REFERENCE_IMPLEMENTATION + dst[0] = (argb >> 24) & 0xff; + dst[1] = (argb >> 16) & 0xff; + dst[2] = (argb >> 8) & 0xff; + dst[3] = (argb >> 0) & 0xff; +#endif +#else // WORDS_BIGENDIAN + dst[0] = (argb >> 0) & 0xff; + dst[1] = (argb >> 8) & 0xff; + dst[2] = (argb >> 16) & 0xff; + dst[3] = (argb >> 24) & 0xff; +#endif + dst += sizeof(argb); + } + } else { + memcpy(dst, src, num_pixels * sizeof(*src)); + } +} + +void VP8LConvertFromBGRA(const uint32_t* const in_data, int num_pixels, + WEBP_CSP_MODE out_colorspace, uint8_t* const rgba) { + switch (out_colorspace) { + case MODE_RGB: + VP8LConvertBGRAToRGB(in_data, num_pixels, rgba); + break; + case MODE_RGBA: + VP8LConvertBGRAToRGBA(in_data, num_pixels, rgba); + break; + case MODE_rgbA: + VP8LConvertBGRAToRGBA(in_data, num_pixels, rgba); + WebPApplyAlphaMultiply(rgba, 0, num_pixels, 1, 0); + break; + case MODE_BGR: + VP8LConvertBGRAToBGR(in_data, num_pixels, rgba); + break; + case MODE_BGRA: + CopyOrSwap(in_data, num_pixels, rgba, 1); + break; + case MODE_bgrA: + CopyOrSwap(in_data, num_pixels, rgba, 1); + WebPApplyAlphaMultiply(rgba, 0, num_pixels, 1, 0); + break; + case MODE_ARGB: + CopyOrSwap(in_data, num_pixels, rgba, 0); + break; + case MODE_Argb: + CopyOrSwap(in_data, num_pixels, rgba, 0); + WebPApplyAlphaMultiply(rgba, 1, num_pixels, 1, 0); + break; + case MODE_RGBA_4444: + VP8LConvertBGRAToRGBA4444(in_data, num_pixels, rgba); + break; + case MODE_rgbA_4444: + VP8LConvertBGRAToRGBA4444(in_data, num_pixels, rgba); + WebPApplyAlphaMultiply4444(rgba, num_pixels, 1, 0); + break; + case MODE_RGB_565: + VP8LConvertBGRAToRGB565(in_data, num_pixels, rgba); + break; + default: + assert(0); // Code flow should not reach here. + } +} + +//------------------------------------------------------------------------------ + +VP8LProcessBlueAndRedFunc VP8LAddGreenToBlueAndRed; +VP8LPredictorFunc VP8LPredictors[16]; + +VP8LTransformColorFunc VP8LTransformColorInverse; + +VP8LConvertFunc VP8LConvertBGRAToRGB; +VP8LConvertFunc VP8LConvertBGRAToRGBA; +VP8LConvertFunc VP8LConvertBGRAToRGBA4444; +VP8LConvertFunc VP8LConvertBGRAToRGB565; +VP8LConvertFunc VP8LConvertBGRAToBGR; + +VP8LMapARGBFunc VP8LMapColor32b; +VP8LMapAlphaFunc VP8LMapColor8b; + +extern void VP8LDspInitSSE2(void); +extern void VP8LDspInitNEON(void); +extern void VP8LDspInitMIPSdspR2(void); + +static volatile VP8CPUInfo lossless_last_cpuinfo_used = + (VP8CPUInfo)&lossless_last_cpuinfo_used; + +WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInit(void) { + if (lossless_last_cpuinfo_used == VP8GetCPUInfo) return; + + VP8LPredictors[0] = Predictor0; + VP8LPredictors[1] = Predictor1; + VP8LPredictors[2] = Predictor2; + VP8LPredictors[3] = Predictor3; + VP8LPredictors[4] = Predictor4; + VP8LPredictors[5] = Predictor5; + VP8LPredictors[6] = Predictor6; + VP8LPredictors[7] = Predictor7; + VP8LPredictors[8] = Predictor8; + VP8LPredictors[9] = Predictor9; + VP8LPredictors[10] = Predictor10; + VP8LPredictors[11] = Predictor11; + VP8LPredictors[12] = Predictor12; + VP8LPredictors[13] = Predictor13; + VP8LPredictors[14] = Predictor0; // <- padding security sentinels + VP8LPredictors[15] = Predictor0; + + VP8LAddGreenToBlueAndRed = VP8LAddGreenToBlueAndRed_C; + + VP8LTransformColorInverse = VP8LTransformColorInverse_C; + + VP8LConvertBGRAToRGB = VP8LConvertBGRAToRGB_C; + VP8LConvertBGRAToRGBA = VP8LConvertBGRAToRGBA_C; + VP8LConvertBGRAToRGBA4444 = VP8LConvertBGRAToRGBA4444_C; + VP8LConvertBGRAToRGB565 = VP8LConvertBGRAToRGB565_C; + VP8LConvertBGRAToBGR = VP8LConvertBGRAToBGR_C; + + VP8LMapColor32b = MapARGB; + VP8LMapColor8b = MapAlpha; + + // If defined, use CPUInfo() to overwrite some pointers with faster versions. + if (VP8GetCPUInfo != NULL) { +#if defined(WEBP_USE_SSE2) + if (VP8GetCPUInfo(kSSE2)) { + VP8LDspInitSSE2(); + } +#endif +#if defined(WEBP_USE_NEON) + if (VP8GetCPUInfo(kNEON)) { + VP8LDspInitNEON(); + } +#endif +#if defined(WEBP_USE_MIPS_DSP_R2) + if (VP8GetCPUInfo(kMIPSdspR2)) { + VP8LDspInitMIPSdspR2(); + } +#endif + } + lossless_last_cpuinfo_used = VP8GetCPUInfo; +} + +//------------------------------------------------------------------------------ diff --git a/thirdparty/libwebp/dsp/lossless.h b/thirdparty/libwebp/dsp/lossless.h new file mode 100644 index 0000000000..9f0d7a25b7 --- /dev/null +++ b/thirdparty/libwebp/dsp/lossless.h @@ -0,0 +1,372 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Image transforms and color space conversion methods for lossless decoder. +// +// Authors: Vikas Arora (vikaas.arora@gmail.com) +// Jyrki Alakuijala (jyrki@google.com) + +#ifndef WEBP_DSP_LOSSLESS_H_ +#define WEBP_DSP_LOSSLESS_H_ + +#include "../webp/types.h" +#include "../webp/decode.h" + +#include "../enc/histogram.h" +#include "../utils/utils.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef WEBP_EXPERIMENTAL_FEATURES +#include "../enc/delta_palettization.h" +#endif // WEBP_EXPERIMENTAL_FEATURES + +//------------------------------------------------------------------------------ +// Decoding + +typedef uint32_t (*VP8LPredictorFunc)(uint32_t left, const uint32_t* const top); +extern VP8LPredictorFunc VP8LPredictors[16]; + +typedef void (*VP8LProcessBlueAndRedFunc)(uint32_t* argb_data, int num_pixels); +extern VP8LProcessBlueAndRedFunc VP8LAddGreenToBlueAndRed; + +typedef struct { + // Note: the members are uint8_t, so that any negative values are + // automatically converted to "mod 256" values. + uint8_t green_to_red_; + uint8_t green_to_blue_; + uint8_t red_to_blue_; +} VP8LMultipliers; +typedef void (*VP8LTransformColorFunc)(const VP8LMultipliers* const m, + uint32_t* argb_data, int num_pixels); +extern VP8LTransformColorFunc VP8LTransformColorInverse; + +struct VP8LTransform; // Defined in dec/vp8li.h. + +// Performs inverse transform of data given transform information, start and end +// rows. Transform will be applied to rows [row_start, row_end[. +// The *in and *out pointers refer to source and destination data respectively +// corresponding to the intermediate row (row_start). +void VP8LInverseTransform(const struct VP8LTransform* const transform, + int row_start, int row_end, + const uint32_t* const in, uint32_t* const out); + +// Color space conversion. +typedef void (*VP8LConvertFunc)(const uint32_t* src, int num_pixels, + uint8_t* dst); +extern VP8LConvertFunc VP8LConvertBGRAToRGB; +extern VP8LConvertFunc VP8LConvertBGRAToRGBA; +extern VP8LConvertFunc VP8LConvertBGRAToRGBA4444; +extern VP8LConvertFunc VP8LConvertBGRAToRGB565; +extern VP8LConvertFunc VP8LConvertBGRAToBGR; + +// Converts from BGRA to other color spaces. +void VP8LConvertFromBGRA(const uint32_t* const in_data, int num_pixels, + WEBP_CSP_MODE out_colorspace, uint8_t* const rgba); + +// color mapping related functions. +static WEBP_INLINE uint32_t VP8GetARGBIndex(uint32_t idx) { + return (idx >> 8) & 0xff; +} + +static WEBP_INLINE uint8_t VP8GetAlphaIndex(uint8_t idx) { + return idx; +} + +static WEBP_INLINE uint32_t VP8GetARGBValue(uint32_t val) { + return val; +} + +static WEBP_INLINE uint8_t VP8GetAlphaValue(uint32_t val) { + return (val >> 8) & 0xff; +} + +typedef void (*VP8LMapARGBFunc)(const uint32_t* src, + const uint32_t* const color_map, + uint32_t* dst, int y_start, + int y_end, int width); +typedef void (*VP8LMapAlphaFunc)(const uint8_t* src, + const uint32_t* const color_map, + uint8_t* dst, int y_start, + int y_end, int width); + +extern VP8LMapARGBFunc VP8LMapColor32b; +extern VP8LMapAlphaFunc VP8LMapColor8b; + +// Similar to the static method ColorIndexInverseTransform() that is part of +// lossless.c, but used only for alpha decoding. It takes uint8_t (rather than +// uint32_t) arguments for 'src' and 'dst'. +void VP8LColorIndexInverseTransformAlpha( + const struct VP8LTransform* const transform, int y_start, int y_end, + const uint8_t* src, uint8_t* dst); + +// Expose some C-only fallback functions +void VP8LTransformColorInverse_C(const VP8LMultipliers* const m, + uint32_t* data, int num_pixels); + +void VP8LConvertBGRAToRGB_C(const uint32_t* src, int num_pixels, uint8_t* dst); +void VP8LConvertBGRAToRGBA_C(const uint32_t* src, int num_pixels, uint8_t* dst); +void VP8LConvertBGRAToRGBA4444_C(const uint32_t* src, + int num_pixels, uint8_t* dst); +void VP8LConvertBGRAToRGB565_C(const uint32_t* src, + int num_pixels, uint8_t* dst); +void VP8LConvertBGRAToBGR_C(const uint32_t* src, int num_pixels, uint8_t* dst); +void VP8LAddGreenToBlueAndRed_C(uint32_t* data, int num_pixels); + +// Must be called before calling any of the above methods. +void VP8LDspInit(void); + +//------------------------------------------------------------------------------ +// Encoding + +extern VP8LProcessBlueAndRedFunc VP8LSubtractGreenFromBlueAndRed; +extern VP8LTransformColorFunc VP8LTransformColor; +typedef void (*VP8LCollectColorBlueTransformsFunc)( + const uint32_t* argb, int stride, + int tile_width, int tile_height, + int green_to_blue, int red_to_blue, int histo[]); +extern VP8LCollectColorBlueTransformsFunc VP8LCollectColorBlueTransforms; + +typedef void (*VP8LCollectColorRedTransformsFunc)( + const uint32_t* argb, int stride, + int tile_width, int tile_height, + int green_to_red, int histo[]); +extern VP8LCollectColorRedTransformsFunc VP8LCollectColorRedTransforms; + +// Expose some C-only fallback functions +void VP8LTransformColor_C(const VP8LMultipliers* const m, + uint32_t* data, int num_pixels); +void VP8LSubtractGreenFromBlueAndRed_C(uint32_t* argb_data, int num_pixels); +void VP8LCollectColorRedTransforms_C(const uint32_t* argb, int stride, + int tile_width, int tile_height, + int green_to_red, int histo[]); +void VP8LCollectColorBlueTransforms_C(const uint32_t* argb, int stride, + int tile_width, int tile_height, + int green_to_blue, int red_to_blue, + int histo[]); + +//------------------------------------------------------------------------------ +// Image transforms. + +void VP8LResidualImage(int width, int height, int bits, int low_effort, + uint32_t* const argb, uint32_t* const argb_scratch, + uint32_t* const image, int near_lossless, int exact, + int used_subtract_green); + +void VP8LColorSpaceTransform(int width, int height, int bits, int quality, + uint32_t* const argb, uint32_t* image); + +//------------------------------------------------------------------------------ +// Misc methods. + +// Computes sampled size of 'size' when sampling using 'sampling bits'. +static WEBP_INLINE uint32_t VP8LSubSampleSize(uint32_t size, + uint32_t sampling_bits) { + return (size + (1 << sampling_bits) - 1) >> sampling_bits; +} + +// Converts near lossless quality into max number of bits shaved off. +static WEBP_INLINE int VP8LNearLosslessBits(int near_lossless_quality) { + // 100 -> 0 + // 80..99 -> 1 + // 60..79 -> 2 + // 40..59 -> 3 + // 20..39 -> 4 + // 0..19 -> 5 + return 5 - near_lossless_quality / 20; +} + +// ----------------------------------------------------------------------------- +// Faster logarithm for integers. Small values use a look-up table. + +// The threshold till approximate version of log_2 can be used. +// Practically, we can get rid of the call to log() as the two values match to +// very high degree (the ratio of these two is 0.99999x). +// Keeping a high threshold for now. +#define APPROX_LOG_WITH_CORRECTION_MAX 65536 +#define APPROX_LOG_MAX 4096 +#define LOG_2_RECIPROCAL 1.44269504088896338700465094007086 +#define LOG_LOOKUP_IDX_MAX 256 +extern const float kLog2Table[LOG_LOOKUP_IDX_MAX]; +extern const float kSLog2Table[LOG_LOOKUP_IDX_MAX]; +typedef float (*VP8LFastLog2SlowFunc)(uint32_t v); + +extern VP8LFastLog2SlowFunc VP8LFastLog2Slow; +extern VP8LFastLog2SlowFunc VP8LFastSLog2Slow; + +static WEBP_INLINE float VP8LFastLog2(uint32_t v) { + return (v < LOG_LOOKUP_IDX_MAX) ? kLog2Table[v] : VP8LFastLog2Slow(v); +} +// Fast calculation of v * log2(v) for integer input. +static WEBP_INLINE float VP8LFastSLog2(uint32_t v) { + return (v < LOG_LOOKUP_IDX_MAX) ? kSLog2Table[v] : VP8LFastSLog2Slow(v); +} + +// ----------------------------------------------------------------------------- +// Huffman-cost related functions. + +typedef double (*VP8LCostFunc)(const uint32_t* population, int length); +typedef double (*VP8LCostCombinedFunc)(const uint32_t* X, const uint32_t* Y, + int length); +typedef float (*VP8LCombinedShannonEntropyFunc)(const int X[256], + const int Y[256]); + +extern VP8LCostFunc VP8LExtraCost; +extern VP8LCostCombinedFunc VP8LExtraCostCombined; +extern VP8LCombinedShannonEntropyFunc VP8LCombinedShannonEntropy; + +typedef struct { // small struct to hold counters + int counts[2]; // index: 0=zero steak, 1=non-zero streak + int streaks[2][2]; // [zero/non-zero][streak<3 / streak>=3] +} VP8LStreaks; + +typedef VP8LStreaks (*VP8LCostCombinedCountFunc)(const uint32_t* X, + const uint32_t* Y, int length); + +extern VP8LCostCombinedCountFunc VP8LHuffmanCostCombinedCount; + +typedef struct { // small struct to hold bit entropy results + double entropy; // entropy + uint32_t sum; // sum of the population + int nonzeros; // number of non-zero elements in the population + uint32_t max_val; // maximum value in the population + uint32_t nonzero_code; // index of the last non-zero in the population +} VP8LBitEntropy; + +void VP8LBitEntropyInit(VP8LBitEntropy* const entropy); + +// Get the combined symbol bit entropy and Huffman cost stats for the +// distributions 'X' and 'Y'. Those results can then be refined according to +// codec specific heuristics. +void VP8LGetCombinedEntropyUnrefined(const uint32_t* const X, + const uint32_t* const Y, int length, + VP8LBitEntropy* const bit_entropy, + VP8LStreaks* const stats); +// Get the entropy for the distribution 'X'. +void VP8LGetEntropyUnrefined(const uint32_t* const X, int length, + VP8LBitEntropy* const bit_entropy, + VP8LStreaks* const stats); + +void VP8LBitsEntropyUnrefined(const uint32_t* const array, int n, + VP8LBitEntropy* const entropy); + +typedef void (*GetEntropyUnrefinedHelperFunc)(uint32_t val, int i, + uint32_t* const val_prev, + int* const i_prev, + VP8LBitEntropy* const bit_entropy, + VP8LStreaks* const stats); +// Internal function used by VP8LGet*EntropyUnrefined. +extern GetEntropyUnrefinedHelperFunc VP8LGetEntropyUnrefinedHelper; + +typedef void (*VP8LHistogramAddFunc)(const VP8LHistogram* const a, + const VP8LHistogram* const b, + VP8LHistogram* const out); +extern VP8LHistogramAddFunc VP8LHistogramAdd; + +// ----------------------------------------------------------------------------- +// PrefixEncode() + +typedef int (*VP8LVectorMismatchFunc)(const uint32_t* const array1, + const uint32_t* const array2, int length); +// Returns the first index where array1 and array2 are different. +extern VP8LVectorMismatchFunc VP8LVectorMismatch; + +static WEBP_INLINE int VP8LBitsLog2Ceiling(uint32_t n) { + const int log_floor = BitsLog2Floor(n); + if (n == (n & ~(n - 1))) // zero or a power of two. + return log_floor; + else + return log_floor + 1; +} + +// Splitting of distance and length codes into prefixes and +// extra bits. The prefixes are encoded with an entropy code +// while the extra bits are stored just as normal bits. +static WEBP_INLINE void VP8LPrefixEncodeBitsNoLUT(int distance, int* const code, + int* const extra_bits) { + const int highest_bit = BitsLog2Floor(--distance); + const int second_highest_bit = (distance >> (highest_bit - 1)) & 1; + *extra_bits = highest_bit - 1; + *code = 2 * highest_bit + second_highest_bit; +} + +static WEBP_INLINE void VP8LPrefixEncodeNoLUT(int distance, int* const code, + int* const extra_bits, + int* const extra_bits_value) { + const int highest_bit = BitsLog2Floor(--distance); + const int second_highest_bit = (distance >> (highest_bit - 1)) & 1; + *extra_bits = highest_bit - 1; + *extra_bits_value = distance & ((1 << *extra_bits) - 1); + *code = 2 * highest_bit + second_highest_bit; +} + +#define PREFIX_LOOKUP_IDX_MAX 512 +typedef struct { + int8_t code_; + int8_t extra_bits_; +} VP8LPrefixCode; + +// These tables are derived using VP8LPrefixEncodeNoLUT. +extern const VP8LPrefixCode kPrefixEncodeCode[PREFIX_LOOKUP_IDX_MAX]; +extern const uint8_t kPrefixEncodeExtraBitsValue[PREFIX_LOOKUP_IDX_MAX]; +static WEBP_INLINE void VP8LPrefixEncodeBits(int distance, int* const code, + int* const extra_bits) { + if (distance < PREFIX_LOOKUP_IDX_MAX) { + const VP8LPrefixCode prefix_code = kPrefixEncodeCode[distance]; + *code = prefix_code.code_; + *extra_bits = prefix_code.extra_bits_; + } else { + VP8LPrefixEncodeBitsNoLUT(distance, code, extra_bits); + } +} + +static WEBP_INLINE void VP8LPrefixEncode(int distance, int* const code, + int* const extra_bits, + int* const extra_bits_value) { + if (distance < PREFIX_LOOKUP_IDX_MAX) { + const VP8LPrefixCode prefix_code = kPrefixEncodeCode[distance]; + *code = prefix_code.code_; + *extra_bits = prefix_code.extra_bits_; + *extra_bits_value = kPrefixEncodeExtraBitsValue[distance]; + } else { + VP8LPrefixEncodeNoLUT(distance, code, extra_bits, extra_bits_value); + } +} + +// Sum of each component, mod 256. +static WEBP_INLINE uint32_t VP8LAddPixels(uint32_t a, uint32_t b) { + const uint32_t alpha_and_green = (a & 0xff00ff00u) + (b & 0xff00ff00u); + const uint32_t red_and_blue = (a & 0x00ff00ffu) + (b & 0x00ff00ffu); + return (alpha_and_green & 0xff00ff00u) | (red_and_blue & 0x00ff00ffu); +} + +// Difference of each component, mod 256. +static WEBP_INLINE uint32_t VP8LSubPixels(uint32_t a, uint32_t b) { + const uint32_t alpha_and_green = + 0x00ff00ffu + (a & 0xff00ff00u) - (b & 0xff00ff00u); + const uint32_t red_and_blue = + 0xff00ff00u + (a & 0x00ff00ffu) - (b & 0x00ff00ffu); + return (alpha_and_green & 0xff00ff00u) | (red_and_blue & 0x00ff00ffu); +} + +void VP8LBundleColorMap(const uint8_t* const row, int width, + int xbits, uint32_t* const dst); + +// Must be called before calling any of the above methods. +void VP8LEncDspInit(void); + +//------------------------------------------------------------------------------ + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // WEBP_DSP_LOSSLESS_H_ diff --git a/thirdparty/libwebp/dsp/lossless_enc.c b/thirdparty/libwebp/dsp/lossless_enc.c new file mode 100644 index 0000000000..256f6f5f8b --- /dev/null +++ b/thirdparty/libwebp/dsp/lossless_enc.c @@ -0,0 +1,1439 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Image transform methods for lossless encoder. +// +// Authors: Vikas Arora (vikaas.arora@gmail.com) +// Jyrki Alakuijala (jyrki@google.com) +// Urvang Joshi (urvang@google.com) + +#include "./dsp.h" + +#include <math.h> +#include <stdlib.h> +#include "../dec/vp8li.h" +#include "../utils/endian_inl.h" +#include "./lossless.h" +#include "./yuv.h" + +#define MAX_DIFF_COST (1e30f) + +static const int kPredLowEffort = 11; +static const uint32_t kMaskAlpha = 0xff000000; + +// lookup table for small values of log2(int) +const float kLog2Table[LOG_LOOKUP_IDX_MAX] = { + 0.0000000000000000f, 0.0000000000000000f, + 1.0000000000000000f, 1.5849625007211560f, + 2.0000000000000000f, 2.3219280948873621f, + 2.5849625007211560f, 2.8073549220576041f, + 3.0000000000000000f, 3.1699250014423121f, + 3.3219280948873621f, 3.4594316186372973f, + 3.5849625007211560f, 3.7004397181410921f, + 3.8073549220576041f, 3.9068905956085187f, + 4.0000000000000000f, 4.0874628412503390f, + 4.1699250014423121f, 4.2479275134435852f, + 4.3219280948873626f, 4.3923174227787606f, + 4.4594316186372973f, 4.5235619560570130f, + 4.5849625007211560f, 4.6438561897747243f, + 4.7004397181410917f, 4.7548875021634682f, + 4.8073549220576037f, 4.8579809951275718f, + 4.9068905956085187f, 4.9541963103868749f, + 5.0000000000000000f, 5.0443941193584533f, + 5.0874628412503390f, 5.1292830169449663f, + 5.1699250014423121f, 5.2094533656289501f, + 5.2479275134435852f, 5.2854022188622487f, + 5.3219280948873626f, 5.3575520046180837f, + 5.3923174227787606f, 5.4262647547020979f, + 5.4594316186372973f, 5.4918530963296747f, + 5.5235619560570130f, 5.5545888516776376f, + 5.5849625007211560f, 5.6147098441152083f, + 5.6438561897747243f, 5.6724253419714951f, + 5.7004397181410917f, 5.7279204545631987f, + 5.7548875021634682f, 5.7813597135246599f, + 5.8073549220576037f, 5.8328900141647412f, + 5.8579809951275718f, 5.8826430493618415f, + 5.9068905956085187f, 5.9307373375628866f, + 5.9541963103868749f, 5.9772799234999167f, + 6.0000000000000000f, 6.0223678130284543f, + 6.0443941193584533f, 6.0660891904577720f, + 6.0874628412503390f, 6.1085244567781691f, + 6.1292830169449663f, 6.1497471195046822f, + 6.1699250014423121f, 6.1898245588800175f, + 6.2094533656289501f, 6.2288186904958804f, + 6.2479275134435852f, 6.2667865406949010f, + 6.2854022188622487f, 6.3037807481771030f, + 6.3219280948873626f, 6.3398500028846243f, + 6.3575520046180837f, 6.3750394313469245f, + 6.3923174227787606f, 6.4093909361377017f, + 6.4262647547020979f, 6.4429434958487279f, + 6.4594316186372973f, 6.4757334309663976f, + 6.4918530963296747f, 6.5077946401986963f, + 6.5235619560570130f, 6.5391588111080309f, + 6.5545888516776376f, 6.5698556083309478f, + 6.5849625007211560f, 6.5999128421871278f, + 6.6147098441152083f, 6.6293566200796094f, + 6.6438561897747243f, 6.6582114827517946f, + 6.6724253419714951f, 6.6865005271832185f, + 6.7004397181410917f, 6.7142455176661224f, + 6.7279204545631987f, 6.7414669864011464f, + 6.7548875021634682f, 6.7681843247769259f, + 6.7813597135246599f, 6.7944158663501061f, + 6.8073549220576037f, 6.8201789624151878f, + 6.8328900141647412f, 6.8454900509443747f, + 6.8579809951275718f, 6.8703647195834047f, + 6.8826430493618415f, 6.8948177633079437f, + 6.9068905956085187f, 6.9188632372745946f, + 6.9307373375628866f, 6.9425145053392398f, + 6.9541963103868749f, 6.9657842846620869f, + 6.9772799234999167f, 6.9886846867721654f, + 7.0000000000000000f, 7.0112272554232539f, + 7.0223678130284543f, 7.0334230015374501f, + 7.0443941193584533f, 7.0552824355011898f, + 7.0660891904577720f, 7.0768155970508308f, + 7.0874628412503390f, 7.0980320829605263f, + 7.1085244567781691f, 7.1189410727235076f, + 7.1292830169449663f, 7.1395513523987936f, + 7.1497471195046822f, 7.1598713367783890f, + 7.1699250014423121f, 7.1799090900149344f, + 7.1898245588800175f, 7.1996723448363644f, + 7.2094533656289501f, 7.2191685204621611f, + 7.2288186904958804f, 7.2384047393250785f, + 7.2479275134435852f, 7.2573878426926521f, + 7.2667865406949010f, 7.2761244052742375f, + 7.2854022188622487f, 7.2946207488916270f, + 7.3037807481771030f, 7.3128829552843557f, + 7.3219280948873626f, 7.3309168781146167f, + 7.3398500028846243f, 7.3487281542310771f, + 7.3575520046180837f, 7.3663222142458160f, + 7.3750394313469245f, 7.3837042924740519f, + 7.3923174227787606f, 7.4008794362821843f, + 7.4093909361377017f, 7.4178525148858982f, + 7.4262647547020979f, 7.4346282276367245f, + 7.4429434958487279f, 7.4512111118323289f, + 7.4594316186372973f, 7.4676055500829976f, + 7.4757334309663976f, 7.4838157772642563f, + 7.4918530963296747f, 7.4998458870832056f, + 7.5077946401986963f, 7.5156998382840427f, + 7.5235619560570130f, 7.5313814605163118f, + 7.5391588111080309f, 7.5468944598876364f, + 7.5545888516776376f, 7.5622424242210728f, + 7.5698556083309478f, 7.5774288280357486f, + 7.5849625007211560f, 7.5924570372680806f, + 7.5999128421871278f, 7.6073303137496104f, + 7.6147098441152083f, 7.6220518194563764f, + 7.6293566200796094f, 7.6366246205436487f, + 7.6438561897747243f, 7.6510516911789281f, + 7.6582114827517946f, 7.6653359171851764f, + 7.6724253419714951f, 7.6794800995054464f, + 7.6865005271832185f, 7.6934869574993252f, + 7.7004397181410917f, 7.7073591320808825f, + 7.7142455176661224f, 7.7210991887071855f, + 7.7279204545631987f, 7.7347096202258383f, + 7.7414669864011464f, 7.7481928495894605f, + 7.7548875021634682f, 7.7615512324444795f, + 7.7681843247769259f, 7.7747870596011736f, + 7.7813597135246599f, 7.7879025593914317f, + 7.7944158663501061f, 7.8008998999203047f, + 7.8073549220576037f, 7.8137811912170374f, + 7.8201789624151878f, 7.8265484872909150f, + 7.8328900141647412f, 7.8392037880969436f, + 7.8454900509443747f, 7.8517490414160571f, + 7.8579809951275718f, 7.8641861446542797f, + 7.8703647195834047f, 7.8765169465649993f, + 7.8826430493618415f, 7.8887432488982591f, + 7.8948177633079437f, 7.9008668079807486f, + 7.9068905956085187f, 7.9128893362299619f, + 7.9188632372745946f, 7.9248125036057812f, + 7.9307373375628866f, 7.9366379390025709f, + 7.9425145053392398f, 7.9483672315846778f, + 7.9541963103868749f, 7.9600019320680805f, + 7.9657842846620869f, 7.9715435539507719f, + 7.9772799234999167f, 7.9829935746943103f, + 7.9886846867721654f, 7.9943534368588577f +}; + +const float kSLog2Table[LOG_LOOKUP_IDX_MAX] = { + 0.00000000f, 0.00000000f, 2.00000000f, 4.75488750f, + 8.00000000f, 11.60964047f, 15.50977500f, 19.65148445f, + 24.00000000f, 28.52932501f, 33.21928095f, 38.05374781f, + 43.01955001f, 48.10571634f, 53.30296891f, 58.60335893f, + 64.00000000f, 69.48686830f, 75.05865003f, 80.71062276f, + 86.43856190f, 92.23866588f, 98.10749561f, 104.04192499f, + 110.03910002f, 116.09640474f, 122.21143267f, 128.38196256f, + 134.60593782f, 140.88144886f, 147.20671787f, 153.58008562f, + 160.00000000f, 166.46500594f, 172.97373660f, 179.52490559f, + 186.11730005f, 192.74977453f, 199.42124551f, 206.13068654f, + 212.87712380f, 219.65963219f, 226.47733176f, 233.32938445f, + 240.21499122f, 247.13338933f, 254.08384998f, 261.06567603f, + 268.07820003f, 275.12078236f, 282.19280949f, 289.29369244f, + 296.42286534f, 303.57978409f, 310.76392512f, 317.97478424f, + 325.21187564f, 332.47473081f, 339.76289772f, 347.07593991f, + 354.41343574f, 361.77497759f, 369.16017124f, 376.56863518f, + 384.00000000f, 391.45390785f, 398.93001188f, 406.42797576f, + 413.94747321f, 421.48818752f, 429.04981119f, 436.63204548f, + 444.23460010f, 451.85719280f, 459.49954906f, 467.16140179f, + 474.84249102f, 482.54256363f, 490.26137307f, 497.99867911f, + 505.75424759f, 513.52785023f, 521.31926438f, 529.12827280f, + 536.95466351f, 544.79822957f, 552.65876890f, 560.53608414f, + 568.42998244f, 576.34027536f, 584.26677867f, 592.20931226f, + 600.16769996f, 608.14176943f, 616.13135206f, 624.13628279f, + 632.15640007f, 640.19154569f, 648.24156472f, 656.30630539f, + 664.38561898f, 672.47935976f, 680.58738488f, 688.70955430f, + 696.84573069f, 704.99577935f, 713.15956818f, 721.33696754f, + 729.52785023f, 737.73209140f, 745.94956849f, 754.18016116f, + 762.42375127f, 770.68022275f, 778.94946161f, 787.23135586f, + 795.52579543f, 803.83267219f, 812.15187982f, 820.48331383f, + 828.82687147f, 837.18245171f, 845.54995518f, 853.92928416f, + 862.32034249f, 870.72303558f, 879.13727036f, 887.56295522f, + 896.00000000f, 904.44831595f, 912.90781569f, 921.37841320f, + 929.86002376f, 938.35256392f, 946.85595152f, 955.37010560f, + 963.89494641f, 972.43039537f, 980.97637504f, 989.53280911f, + 998.09962237f, 1006.67674069f, 1015.26409097f, 1023.86160116f, + 1032.46920021f, 1041.08681805f, 1049.71438560f, 1058.35183469f, + 1066.99909811f, 1075.65610955f, 1084.32280357f, 1092.99911564f, + 1101.68498204f, 1110.38033993f, 1119.08512727f, 1127.79928282f, + 1136.52274614f, 1145.25545758f, 1153.99735821f, 1162.74838989f, + 1171.50849518f, 1180.27761738f, 1189.05570047f, 1197.84268914f, + 1206.63852876f, 1215.44316535f, 1224.25654560f, 1233.07861684f, + 1241.90932703f, 1250.74862473f, 1259.59645914f, 1268.45278005f, + 1277.31753781f, 1286.19068338f, 1295.07216828f, 1303.96194457f, + 1312.85996488f, 1321.76618236f, 1330.68055071f, 1339.60302413f, + 1348.53355734f, 1357.47210556f, 1366.41862452f, 1375.37307041f, + 1384.33539991f, 1393.30557020f, 1402.28353887f, 1411.26926400f, + 1420.26270412f, 1429.26381818f, 1438.27256558f, 1447.28890615f, + 1456.31280014f, 1465.34420819f, 1474.38309138f, 1483.42941118f, + 1492.48312945f, 1501.54420843f, 1510.61261078f, 1519.68829949f, + 1528.77123795f, 1537.86138993f, 1546.95871952f, 1556.06319119f, + 1565.17476976f, 1574.29342040f, 1583.41910860f, 1592.55180020f, + 1601.69146137f, 1610.83805860f, 1619.99155871f, 1629.15192882f, + 1638.31913637f, 1647.49314911f, 1656.67393509f, 1665.86146266f, + 1675.05570047f, 1684.25661744f, 1693.46418280f, 1702.67836605f, + 1711.89913698f, 1721.12646563f, 1730.36032233f, 1739.60067768f, + 1748.84750254f, 1758.10076802f, 1767.36044551f, 1776.62650662f, + 1785.89892323f, 1795.17766747f, 1804.46271172f, 1813.75402857f, + 1823.05159087f, 1832.35537170f, 1841.66534438f, 1850.98148244f, + 1860.30375965f, 1869.63214999f, 1878.96662767f, 1888.30716711f, + 1897.65374295f, 1907.00633003f, 1916.36490342f, 1925.72943838f, + 1935.09991037f, 1944.47629506f, 1953.85856831f, 1963.24670620f, + 1972.64068498f, 1982.04048108f, 1991.44607117f, 2000.85743204f, + 2010.27454072f, 2019.69737440f, 2029.12591044f, 2038.56012640f +}; + +const VP8LPrefixCode kPrefixEncodeCode[PREFIX_LOOKUP_IDX_MAX] = { + { 0, 0}, { 0, 0}, { 1, 0}, { 2, 0}, { 3, 0}, { 4, 1}, { 4, 1}, { 5, 1}, + { 5, 1}, { 6, 2}, { 6, 2}, { 6, 2}, { 6, 2}, { 7, 2}, { 7, 2}, { 7, 2}, + { 7, 2}, { 8, 3}, { 8, 3}, { 8, 3}, { 8, 3}, { 8, 3}, { 8, 3}, { 8, 3}, + { 8, 3}, { 9, 3}, { 9, 3}, { 9, 3}, { 9, 3}, { 9, 3}, { 9, 3}, { 9, 3}, + { 9, 3}, {10, 4}, {10, 4}, {10, 4}, {10, 4}, {10, 4}, {10, 4}, {10, 4}, + {10, 4}, {10, 4}, {10, 4}, {10, 4}, {10, 4}, {10, 4}, {10, 4}, {10, 4}, + {10, 4}, {11, 4}, {11, 4}, {11, 4}, {11, 4}, {11, 4}, {11, 4}, {11, 4}, + {11, 4}, {11, 4}, {11, 4}, {11, 4}, {11, 4}, {11, 4}, {11, 4}, {11, 4}, + {11, 4}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, + {12, 5}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, + {12, 5}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, + {12, 5}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, {12, 5}, + {12, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, + {13, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, + {13, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, + {13, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, {13, 5}, + {13, 5}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, + {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, + {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, + {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, + {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, + {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, + {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, + {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, {14, 6}, + {14, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, + {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, + {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, + {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, + {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, + {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, + {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, + {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, {15, 6}, + {15, 6}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, {16, 7}, + {16, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, + {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, + {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, + {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, + {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, + {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, + {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, + {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, + {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, + {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, + {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, + {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, + {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, + {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, + {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, + {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, {17, 7}, +}; + +const uint8_t kPrefixEncodeExtraBitsValue[PREFIX_LOOKUP_IDX_MAX] = { + 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, + 127, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126 +}; + +static float FastSLog2Slow(uint32_t v) { + assert(v >= LOG_LOOKUP_IDX_MAX); + if (v < APPROX_LOG_WITH_CORRECTION_MAX) { + int log_cnt = 0; + uint32_t y = 1; + int correction = 0; + const float v_f = (float)v; + const uint32_t orig_v = v; + do { + ++log_cnt; + v = v >> 1; + y = y << 1; + } while (v >= LOG_LOOKUP_IDX_MAX); + // vf = (2^log_cnt) * Xf; where y = 2^log_cnt and Xf < 256 + // Xf = floor(Xf) * (1 + (v % y) / v) + // log2(Xf) = log2(floor(Xf)) + log2(1 + (v % y) / v) + // The correction factor: log(1 + d) ~ d; for very small d values, so + // log2(1 + (v % y) / v) ~ LOG_2_RECIPROCAL * (v % y)/v + // LOG_2_RECIPROCAL ~ 23/16 + correction = (23 * (orig_v & (y - 1))) >> 4; + return v_f * (kLog2Table[v] + log_cnt) + correction; + } else { + return (float)(LOG_2_RECIPROCAL * v * log((double)v)); + } +} + +static float FastLog2Slow(uint32_t v) { + assert(v >= LOG_LOOKUP_IDX_MAX); + if (v < APPROX_LOG_WITH_CORRECTION_MAX) { + int log_cnt = 0; + uint32_t y = 1; + const uint32_t orig_v = v; + double log_2; + do { + ++log_cnt; + v = v >> 1; + y = y << 1; + } while (v >= LOG_LOOKUP_IDX_MAX); + log_2 = kLog2Table[v] + log_cnt; + if (orig_v >= APPROX_LOG_MAX) { + // Since the division is still expensive, add this correction factor only + // for large values of 'v'. + const int correction = (23 * (orig_v & (y - 1))) >> 4; + log_2 += (double)correction / orig_v; + } + return (float)log_2; + } else { + return (float)(LOG_2_RECIPROCAL * log((double)v)); + } +} + +// Mostly used to reduce code size + readability +static WEBP_INLINE int GetMin(int a, int b) { return (a > b) ? b : a; } +static WEBP_INLINE int GetMax(int a, int b) { return (a < b) ? b : a; } + +//------------------------------------------------------------------------------ +// Methods to calculate Entropy (Shannon). + +static float PredictionCostSpatial(const int counts[256], int weight_0, + double exp_val) { + const int significant_symbols = 256 >> 4; + const double exp_decay_factor = 0.6; + double bits = weight_0 * counts[0]; + int i; + for (i = 1; i < significant_symbols; ++i) { + bits += exp_val * (counts[i] + counts[256 - i]); + exp_val *= exp_decay_factor; + } + return (float)(-0.1 * bits); +} + +// Compute the combined Shanon's entropy for distribution {X} and {X+Y} +static float CombinedShannonEntropy(const int X[256], const int Y[256]) { + int i; + double retval = 0.; + int sumX = 0, sumXY = 0; + for (i = 0; i < 256; ++i) { + const int x = X[i]; + if (x != 0) { + const int xy = x + Y[i]; + sumX += x; + retval -= VP8LFastSLog2(x); + sumXY += xy; + retval -= VP8LFastSLog2(xy); + } else if (Y[i] != 0) { + sumXY += Y[i]; + retval -= VP8LFastSLog2(Y[i]); + } + } + retval += VP8LFastSLog2(sumX) + VP8LFastSLog2(sumXY); + return (float)retval; +} + +static float PredictionCostSpatialHistogram(const int accumulated[4][256], + const int tile[4][256]) { + int i; + double retval = 0; + for (i = 0; i < 4; ++i) { + const double kExpValue = 0.94; + retval += PredictionCostSpatial(tile[i], 1, kExpValue); + retval += VP8LCombinedShannonEntropy(tile[i], accumulated[i]); + } + return (float)retval; +} + +void VP8LBitEntropyInit(VP8LBitEntropy* const entropy) { + entropy->entropy = 0.; + entropy->sum = 0; + entropy->nonzeros = 0; + entropy->max_val = 0; + entropy->nonzero_code = VP8L_NON_TRIVIAL_SYM; +} + +void VP8LBitsEntropyUnrefined(const uint32_t* const array, int n, + VP8LBitEntropy* const entropy) { + int i; + + VP8LBitEntropyInit(entropy); + + for (i = 0; i < n; ++i) { + if (array[i] != 0) { + entropy->sum += array[i]; + entropy->nonzero_code = i; + ++entropy->nonzeros; + entropy->entropy -= VP8LFastSLog2(array[i]); + if (entropy->max_val < array[i]) { + entropy->max_val = array[i]; + } + } + } + entropy->entropy += VP8LFastSLog2(entropy->sum); +} + +static WEBP_INLINE void GetEntropyUnrefinedHelper( + uint32_t val, int i, uint32_t* const val_prev, int* const i_prev, + VP8LBitEntropy* const bit_entropy, VP8LStreaks* const stats) { + const int streak = i - *i_prev; + + // Gather info for the bit entropy. + if (*val_prev != 0) { + bit_entropy->sum += (*val_prev) * streak; + bit_entropy->nonzeros += streak; + bit_entropy->nonzero_code = *i_prev; + bit_entropy->entropy -= VP8LFastSLog2(*val_prev) * streak; + if (bit_entropy->max_val < *val_prev) { + bit_entropy->max_val = *val_prev; + } + } + + // Gather info for the Huffman cost. + stats->counts[*val_prev != 0] += (streak > 3); + stats->streaks[*val_prev != 0][(streak > 3)] += streak; + + *val_prev = val; + *i_prev = i; +} + +void VP8LGetEntropyUnrefined(const uint32_t* const X, int length, + VP8LBitEntropy* const bit_entropy, + VP8LStreaks* const stats) { + int i; + int i_prev = 0; + uint32_t x_prev = X[0]; + + memset(stats, 0, sizeof(*stats)); + VP8LBitEntropyInit(bit_entropy); + + for (i = 1; i < length; ++i) { + const uint32_t x = X[i]; + if (x != x_prev) { + VP8LGetEntropyUnrefinedHelper(x, i, &x_prev, &i_prev, bit_entropy, stats); + } + } + VP8LGetEntropyUnrefinedHelper(0, i, &x_prev, &i_prev, bit_entropy, stats); + + bit_entropy->entropy += VP8LFastSLog2(bit_entropy->sum); +} + +void VP8LGetCombinedEntropyUnrefined(const uint32_t* const X, + const uint32_t* const Y, int length, + VP8LBitEntropy* const bit_entropy, + VP8LStreaks* const stats) { + int i = 1; + int i_prev = 0; + uint32_t xy_prev = X[0] + Y[0]; + + memset(stats, 0, sizeof(*stats)); + VP8LBitEntropyInit(bit_entropy); + + for (i = 1; i < length; ++i) { + const uint32_t xy = X[i] + Y[i]; + if (xy != xy_prev) { + VP8LGetEntropyUnrefinedHelper(xy, i, &xy_prev, &i_prev, bit_entropy, + stats); + } + } + VP8LGetEntropyUnrefinedHelper(0, i, &xy_prev, &i_prev, bit_entropy, stats); + + bit_entropy->entropy += VP8LFastSLog2(bit_entropy->sum); +} + +static WEBP_INLINE void UpdateHisto(int histo_argb[4][256], uint32_t argb) { + ++histo_argb[0][argb >> 24]; + ++histo_argb[1][(argb >> 16) & 0xff]; + ++histo_argb[2][(argb >> 8) & 0xff]; + ++histo_argb[3][argb & 0xff]; +} + +//------------------------------------------------------------------------------ + +static WEBP_INLINE uint32_t Predict(VP8LPredictorFunc pred_func, + int x, int y, + const uint32_t* current_row, + const uint32_t* upper_row) { + if (y == 0) { + return (x == 0) ? ARGB_BLACK : current_row[x - 1]; // Left. + } else if (x == 0) { + return upper_row[x]; // Top. + } else { + return pred_func(current_row[x - 1], upper_row + x); + } +} + +static int MaxDiffBetweenPixels(uint32_t p1, uint32_t p2) { + const int diff_a = abs((int)(p1 >> 24) - (int)(p2 >> 24)); + const int diff_r = abs((int)((p1 >> 16) & 0xff) - (int)((p2 >> 16) & 0xff)); + const int diff_g = abs((int)((p1 >> 8) & 0xff) - (int)((p2 >> 8) & 0xff)); + const int diff_b = abs((int)(p1 & 0xff) - (int)(p2 & 0xff)); + return GetMax(GetMax(diff_a, diff_r), GetMax(diff_g, diff_b)); +} + +static int MaxDiffAroundPixel(uint32_t current, uint32_t up, uint32_t down, + uint32_t left, uint32_t right) { + const int diff_up = MaxDiffBetweenPixels(current, up); + const int diff_down = MaxDiffBetweenPixels(current, down); + const int diff_left = MaxDiffBetweenPixels(current, left); + const int diff_right = MaxDiffBetweenPixels(current, right); + return GetMax(GetMax(diff_up, diff_down), GetMax(diff_left, diff_right)); +} + +static uint32_t AddGreenToBlueAndRed(uint32_t argb) { + const uint32_t green = (argb >> 8) & 0xff; + uint32_t red_blue = argb & 0x00ff00ffu; + red_blue += (green << 16) | green; + red_blue &= 0x00ff00ffu; + return (argb & 0xff00ff00u) | red_blue; +} + +static void MaxDiffsForRow(int width, int stride, const uint32_t* const argb, + uint8_t* const max_diffs, int used_subtract_green) { + uint32_t current, up, down, left, right; + int x; + if (width <= 2) return; + current = argb[0]; + right = argb[1]; + if (used_subtract_green) { + current = AddGreenToBlueAndRed(current); + right = AddGreenToBlueAndRed(right); + } + // max_diffs[0] and max_diffs[width - 1] are never used. + for (x = 1; x < width - 1; ++x) { + up = argb[-stride + x]; + down = argb[stride + x]; + left = current; + current = right; + right = argb[x + 1]; + if (used_subtract_green) { + up = AddGreenToBlueAndRed(up); + down = AddGreenToBlueAndRed(down); + right = AddGreenToBlueAndRed(right); + } + max_diffs[x] = MaxDiffAroundPixel(current, up, down, left, right); + } +} + +// Quantize the difference between the actual component value and its prediction +// to a multiple of quantization, working modulo 256, taking care not to cross +// a boundary (inclusive upper limit). +static uint8_t NearLosslessComponent(uint8_t value, uint8_t predict, + uint8_t boundary, int quantization) { + const int residual = (value - predict) & 0xff; + const int boundary_residual = (boundary - predict) & 0xff; + const int lower = residual & ~(quantization - 1); + const int upper = lower + quantization; + // Resolve ties towards a value closer to the prediction (i.e. towards lower + // if value comes after prediction and towards upper otherwise). + const int bias = ((boundary - value) & 0xff) < boundary_residual; + if (residual - lower < upper - residual + bias) { + // lower is closer to residual than upper. + if (residual > boundary_residual && lower <= boundary_residual) { + // Halve quantization step to avoid crossing boundary. This midpoint is + // on the same side of boundary as residual because midpoint >= residual + // (since lower is closer than upper) and residual is above the boundary. + return lower + (quantization >> 1); + } + return lower; + } else { + // upper is closer to residual than lower. + if (residual <= boundary_residual && upper > boundary_residual) { + // Halve quantization step to avoid crossing boundary. This midpoint is + // on the same side of boundary as residual because midpoint <= residual + // (since upper is closer than lower) and residual is below the boundary. + return lower + (quantization >> 1); + } + return upper & 0xff; + } +} + +// Quantize every component of the difference between the actual pixel value and +// its prediction to a multiple of a quantization (a power of 2, not larger than +// max_quantization which is a power of 2, smaller than max_diff). Take care if +// value and predict have undergone subtract green, which means that red and +// blue are represented as offsets from green. +static uint32_t NearLossless(uint32_t value, uint32_t predict, + int max_quantization, int max_diff, + int used_subtract_green) { + int quantization; + uint8_t new_green = 0; + uint8_t green_diff = 0; + uint8_t a, r, g, b; + if (max_diff <= 2) { + return VP8LSubPixels(value, predict); + } + quantization = max_quantization; + while (quantization >= max_diff) { + quantization >>= 1; + } + if ((value >> 24) == 0 || (value >> 24) == 0xff) { + // Preserve transparency of fully transparent or fully opaque pixels. + a = ((value >> 24) - (predict >> 24)) & 0xff; + } else { + a = NearLosslessComponent(value >> 24, predict >> 24, 0xff, quantization); + } + g = NearLosslessComponent((value >> 8) & 0xff, (predict >> 8) & 0xff, 0xff, + quantization); + if (used_subtract_green) { + // The green offset will be added to red and blue components during decoding + // to obtain the actual red and blue values. + new_green = ((predict >> 8) + g) & 0xff; + // The amount by which green has been adjusted during quantization. It is + // subtracted from red and blue for compensation, to avoid accumulating two + // quantization errors in them. + green_diff = (new_green - (value >> 8)) & 0xff; + } + r = NearLosslessComponent(((value >> 16) - green_diff) & 0xff, + (predict >> 16) & 0xff, 0xff - new_green, + quantization); + b = NearLosslessComponent((value - green_diff) & 0xff, predict & 0xff, + 0xff - new_green, quantization); + return ((uint32_t)a << 24) | ((uint32_t)r << 16) | ((uint32_t)g << 8) | b; +} + +// Returns the difference between the pixel and its prediction. In case of a +// lossy encoding, updates the source image to avoid propagating the deviation +// further to pixels which depend on the current pixel for their predictions. +static WEBP_INLINE uint32_t GetResidual(int width, int height, + uint32_t* const upper_row, + uint32_t* const current_row, + const uint8_t* const max_diffs, + int mode, VP8LPredictorFunc pred_func, + int x, int y, int max_quantization, + int exact, int used_subtract_green) { + const uint32_t predict = Predict(pred_func, x, y, current_row, upper_row); + uint32_t residual; + if (max_quantization == 1 || mode == 0 || y == 0 || y == height - 1 || + x == 0 || x == width - 1) { + residual = VP8LSubPixels(current_row[x], predict); + } else { + residual = NearLossless(current_row[x], predict, max_quantization, + max_diffs[x], used_subtract_green); + // Update the source image. + current_row[x] = VP8LAddPixels(predict, residual); + // x is never 0 here so we do not need to update upper_row like below. + } + if (!exact && (current_row[x] & kMaskAlpha) == 0) { + // If alpha is 0, cleanup RGB. We can choose the RGB values of the residual + // for best compression. The prediction of alpha itself can be non-zero and + // must be kept though. We choose RGB of the residual to be 0. + residual &= kMaskAlpha; + // Update the source image. + current_row[x] = predict & ~kMaskAlpha; + // The prediction for the rightmost pixel in a row uses the leftmost pixel + // in that row as its top-right context pixel. Hence if we change the + // leftmost pixel of current_row, the corresponding change must be applied + // to upper_row as well where top-right context is being read from. + if (x == 0 && y != 0) upper_row[width] = current_row[0]; + } + return residual; +} + +// Returns best predictor and updates the accumulated histogram. +// If max_quantization > 1, assumes that near lossless processing will be +// applied, quantizing residuals to multiples of quantization levels up to +// max_quantization (the actual quantization level depends on smoothness near +// the given pixel). +static int GetBestPredictorForTile(int width, int height, + int tile_x, int tile_y, int bits, + int accumulated[4][256], + uint32_t* const argb_scratch, + const uint32_t* const argb, + int max_quantization, + int exact, int used_subtract_green) { + const int kNumPredModes = 14; + const int start_x = tile_x << bits; + const int start_y = tile_y << bits; + const int tile_size = 1 << bits; + const int max_y = GetMin(tile_size, height - start_y); + const int max_x = GetMin(tile_size, width - start_x); + // Whether there exist columns just outside the tile. + const int have_left = (start_x > 0); + const int have_right = (max_x < width - start_x); + // Position and size of the strip covering the tile and adjacent columns if + // they exist. + const int context_start_x = start_x - have_left; + const int context_width = max_x + have_left + have_right; + // The width of upper_row and current_row is one pixel larger than image width + // to allow the top right pixel to point to the leftmost pixel of the next row + // when at the right edge. + uint32_t* upper_row = argb_scratch; + uint32_t* current_row = upper_row + width + 1; + uint8_t* const max_diffs = (uint8_t*)(current_row + width + 1); + float best_diff = MAX_DIFF_COST; + int best_mode = 0; + int mode; + int histo_stack_1[4][256]; + int histo_stack_2[4][256]; + // Need pointers to be able to swap arrays. + int (*histo_argb)[256] = histo_stack_1; + int (*best_histo)[256] = histo_stack_2; + int i, j; + + for (mode = 0; mode < kNumPredModes; ++mode) { + const VP8LPredictorFunc pred_func = VP8LPredictors[mode]; + float cur_diff; + int relative_y; + memset(histo_argb, 0, sizeof(histo_stack_1)); + if (start_y > 0) { + // Read the row above the tile which will become the first upper_row. + // Include a pixel to the left if it exists; include a pixel to the right + // in all cases (wrapping to the leftmost pixel of the next row if it does + // not exist). + memcpy(current_row + context_start_x, + argb + (start_y - 1) * width + context_start_x, + sizeof(*argb) * (max_x + have_left + 1)); + } + for (relative_y = 0; relative_y < max_y; ++relative_y) { + const int y = start_y + relative_y; + int relative_x; + uint32_t* tmp = upper_row; + upper_row = current_row; + current_row = tmp; + // Read current_row. Include a pixel to the left if it exists; include a + // pixel to the right in all cases except at the bottom right corner of + // the image (wrapping to the leftmost pixel of the next row if it does + // not exist in the current row). + memcpy(current_row + context_start_x, + argb + y * width + context_start_x, + sizeof(*argb) * (max_x + have_left + (y + 1 < height))); + if (max_quantization > 1 && y >= 1 && y + 1 < height) { + MaxDiffsForRow(context_width, width, argb + y * width + context_start_x, + max_diffs + context_start_x, used_subtract_green); + } + + for (relative_x = 0; relative_x < max_x; ++relative_x) { + const int x = start_x + relative_x; + UpdateHisto(histo_argb, + GetResidual(width, height, upper_row, current_row, + max_diffs, mode, pred_func, x, y, + max_quantization, exact, used_subtract_green)); + } + } + cur_diff = PredictionCostSpatialHistogram( + (const int (*)[256])accumulated, (const int (*)[256])histo_argb); + if (cur_diff < best_diff) { + int (*tmp)[256] = histo_argb; + histo_argb = best_histo; + best_histo = tmp; + best_diff = cur_diff; + best_mode = mode; + } + } + + for (i = 0; i < 4; i++) { + for (j = 0; j < 256; j++) { + accumulated[i][j] += best_histo[i][j]; + } + } + + return best_mode; +} + +// Converts pixels of the image to residuals with respect to predictions. +// If max_quantization > 1, applies near lossless processing, quantizing +// residuals to multiples of quantization levels up to max_quantization +// (the actual quantization level depends on smoothness near the given pixel). +static void CopyImageWithPrediction(int width, int height, + int bits, uint32_t* const modes, + uint32_t* const argb_scratch, + uint32_t* const argb, + int low_effort, int max_quantization, + int exact, int used_subtract_green) { + const int tiles_per_row = VP8LSubSampleSize(width, bits); + const int mask = (1 << bits) - 1; + // The width of upper_row and current_row is one pixel larger than image width + // to allow the top right pixel to point to the leftmost pixel of the next row + // when at the right edge. + uint32_t* upper_row = argb_scratch; + uint32_t* current_row = upper_row + width + 1; + uint8_t* current_max_diffs = (uint8_t*)(current_row + width + 1); + uint8_t* lower_max_diffs = current_max_diffs + width; + int y; + int mode = 0; + VP8LPredictorFunc pred_func = NULL; + + for (y = 0; y < height; ++y) { + int x; + uint32_t* const tmp32 = upper_row; + upper_row = current_row; + current_row = tmp32; + memcpy(current_row, argb + y * width, + sizeof(*argb) * (width + (y + 1 < height))); + + if (low_effort) { + for (x = 0; x < width; ++x) { + const uint32_t predict = Predict(VP8LPredictors[kPredLowEffort], x, y, + current_row, upper_row); + argb[y * width + x] = VP8LSubPixels(current_row[x], predict); + } + } else { + if (max_quantization > 1) { + // Compute max_diffs for the lower row now, because that needs the + // contents of argb for the current row, which we will overwrite with + // residuals before proceeding with the next row. + uint8_t* const tmp8 = current_max_diffs; + current_max_diffs = lower_max_diffs; + lower_max_diffs = tmp8; + if (y + 2 < height) { + MaxDiffsForRow(width, width, argb + (y + 1) * width, lower_max_diffs, + used_subtract_green); + } + } + for (x = 0; x < width; ++x) { + if ((x & mask) == 0) { + mode = (modes[(y >> bits) * tiles_per_row + (x >> bits)] >> 8) & 0xff; + pred_func = VP8LPredictors[mode]; + } + argb[y * width + x] = GetResidual( + width, height, upper_row, current_row, current_max_diffs, mode, + pred_func, x, y, max_quantization, exact, used_subtract_green); + } + } + } +} + +// Finds the best predictor for each tile, and converts the image to residuals +// with respect to predictions. If near_lossless_quality < 100, applies +// near lossless processing, shaving off more bits of residuals for lower +// qualities. +void VP8LResidualImage(int width, int height, int bits, int low_effort, + uint32_t* const argb, uint32_t* const argb_scratch, + uint32_t* const image, int near_lossless_quality, + int exact, int used_subtract_green) { + const int tiles_per_row = VP8LSubSampleSize(width, bits); + const int tiles_per_col = VP8LSubSampleSize(height, bits); + int tile_y; + int histo[4][256]; + const int max_quantization = 1 << VP8LNearLosslessBits(near_lossless_quality); + if (low_effort) { + int i; + for (i = 0; i < tiles_per_row * tiles_per_col; ++i) { + image[i] = ARGB_BLACK | (kPredLowEffort << 8); + } + } else { + memset(histo, 0, sizeof(histo)); + for (tile_y = 0; tile_y < tiles_per_col; ++tile_y) { + int tile_x; + for (tile_x = 0; tile_x < tiles_per_row; ++tile_x) { + const int pred = GetBestPredictorForTile(width, height, tile_x, tile_y, + bits, histo, argb_scratch, argb, max_quantization, exact, + used_subtract_green); + image[tile_y * tiles_per_row + tile_x] = ARGB_BLACK | (pred << 8); + } + } + } + + CopyImageWithPrediction(width, height, bits, image, argb_scratch, argb, + low_effort, max_quantization, exact, + used_subtract_green); +} + +void VP8LSubtractGreenFromBlueAndRed_C(uint32_t* argb_data, int num_pixels) { + int i; + for (i = 0; i < num_pixels; ++i) { + const uint32_t argb = argb_data[i]; + const uint32_t green = (argb >> 8) & 0xff; + const uint32_t new_r = (((argb >> 16) & 0xff) - green) & 0xff; + const uint32_t new_b = ((argb & 0xff) - green) & 0xff; + argb_data[i] = (argb & 0xff00ff00) | (new_r << 16) | new_b; + } +} + +static WEBP_INLINE void MultipliersClear(VP8LMultipliers* const m) { + m->green_to_red_ = 0; + m->green_to_blue_ = 0; + m->red_to_blue_ = 0; +} + +static WEBP_INLINE uint32_t ColorTransformDelta(int8_t color_pred, + int8_t color) { + return (uint32_t)((int)(color_pred) * color) >> 5; +} + +static WEBP_INLINE void ColorCodeToMultipliers(uint32_t color_code, + VP8LMultipliers* const m) { + m->green_to_red_ = (color_code >> 0) & 0xff; + m->green_to_blue_ = (color_code >> 8) & 0xff; + m->red_to_blue_ = (color_code >> 16) & 0xff; +} + +static WEBP_INLINE uint32_t MultipliersToColorCode( + const VP8LMultipliers* const m) { + return 0xff000000u | + ((uint32_t)(m->red_to_blue_) << 16) | + ((uint32_t)(m->green_to_blue_) << 8) | + m->green_to_red_; +} + +void VP8LTransformColor_C(const VP8LMultipliers* const m, uint32_t* data, + int num_pixels) { + int i; + for (i = 0; i < num_pixels; ++i) { + const uint32_t argb = data[i]; + const uint32_t green = argb >> 8; + const uint32_t red = argb >> 16; + uint32_t new_red = red; + uint32_t new_blue = argb; + new_red -= ColorTransformDelta(m->green_to_red_, green); + new_red &= 0xff; + new_blue -= ColorTransformDelta(m->green_to_blue_, green); + new_blue -= ColorTransformDelta(m->red_to_blue_, red); + new_blue &= 0xff; + data[i] = (argb & 0xff00ff00u) | (new_red << 16) | (new_blue); + } +} + +static WEBP_INLINE uint8_t TransformColorRed(uint8_t green_to_red, + uint32_t argb) { + const uint32_t green = argb >> 8; + uint32_t new_red = argb >> 16; + new_red -= ColorTransformDelta(green_to_red, green); + return (new_red & 0xff); +} + +static WEBP_INLINE uint8_t TransformColorBlue(uint8_t green_to_blue, + uint8_t red_to_blue, + uint32_t argb) { + const uint32_t green = argb >> 8; + const uint32_t red = argb >> 16; + uint8_t new_blue = argb; + new_blue -= ColorTransformDelta(green_to_blue, green); + new_blue -= ColorTransformDelta(red_to_blue, red); + return (new_blue & 0xff); +} + +static float PredictionCostCrossColor(const int accumulated[256], + const int counts[256]) { + // Favor low entropy, locally and globally. + // Favor small absolute values for PredictionCostSpatial + static const double kExpValue = 2.4; + return VP8LCombinedShannonEntropy(counts, accumulated) + + PredictionCostSpatial(counts, 3, kExpValue); +} + +void VP8LCollectColorRedTransforms_C(const uint32_t* argb, int stride, + int tile_width, int tile_height, + int green_to_red, int histo[]) { + while (tile_height-- > 0) { + int x; + for (x = 0; x < tile_width; ++x) { + ++histo[TransformColorRed(green_to_red, argb[x])]; + } + argb += stride; + } +} + +static float GetPredictionCostCrossColorRed( + const uint32_t* argb, int stride, int tile_width, int tile_height, + VP8LMultipliers prev_x, VP8LMultipliers prev_y, int green_to_red, + const int accumulated_red_histo[256]) { + int histo[256] = { 0 }; + float cur_diff; + + VP8LCollectColorRedTransforms(argb, stride, tile_width, tile_height, + green_to_red, histo); + + cur_diff = PredictionCostCrossColor(accumulated_red_histo, histo); + if ((uint8_t)green_to_red == prev_x.green_to_red_) { + cur_diff -= 3; // favor keeping the areas locally similar + } + if ((uint8_t)green_to_red == prev_y.green_to_red_) { + cur_diff -= 3; // favor keeping the areas locally similar + } + if (green_to_red == 0) { + cur_diff -= 3; + } + return cur_diff; +} + +static void GetBestGreenToRed( + const uint32_t* argb, int stride, int tile_width, int tile_height, + VP8LMultipliers prev_x, VP8LMultipliers prev_y, int quality, + const int accumulated_red_histo[256], VP8LMultipliers* const best_tx) { + const int kMaxIters = 4 + ((7 * quality) >> 8); // in range [4..6] + int green_to_red_best = 0; + int iter, offset; + float best_diff = GetPredictionCostCrossColorRed( + argb, stride, tile_width, tile_height, prev_x, prev_y, + green_to_red_best, accumulated_red_histo); + for (iter = 0; iter < kMaxIters; ++iter) { + // ColorTransformDelta is a 3.5 bit fixed point, so 32 is equal to + // one in color computation. Having initial delta here as 1 is sufficient + // to explore the range of (-2, 2). + const int delta = 32 >> iter; + // Try a negative and a positive delta from the best known value. + for (offset = -delta; offset <= delta; offset += 2 * delta) { + const int green_to_red_cur = offset + green_to_red_best; + const float cur_diff = GetPredictionCostCrossColorRed( + argb, stride, tile_width, tile_height, prev_x, prev_y, + green_to_red_cur, accumulated_red_histo); + if (cur_diff < best_diff) { + best_diff = cur_diff; + green_to_red_best = green_to_red_cur; + } + } + } + best_tx->green_to_red_ = green_to_red_best; +} + +void VP8LCollectColorBlueTransforms_C(const uint32_t* argb, int stride, + int tile_width, int tile_height, + int green_to_blue, int red_to_blue, + int histo[]) { + while (tile_height-- > 0) { + int x; + for (x = 0; x < tile_width; ++x) { + ++histo[TransformColorBlue(green_to_blue, red_to_blue, argb[x])]; + } + argb += stride; + } +} + +static float GetPredictionCostCrossColorBlue( + const uint32_t* argb, int stride, int tile_width, int tile_height, + VP8LMultipliers prev_x, VP8LMultipliers prev_y, + int green_to_blue, int red_to_blue, const int accumulated_blue_histo[256]) { + int histo[256] = { 0 }; + float cur_diff; + + VP8LCollectColorBlueTransforms(argb, stride, tile_width, tile_height, + green_to_blue, red_to_blue, histo); + + cur_diff = PredictionCostCrossColor(accumulated_blue_histo, histo); + if ((uint8_t)green_to_blue == prev_x.green_to_blue_) { + cur_diff -= 3; // favor keeping the areas locally similar + } + if ((uint8_t)green_to_blue == prev_y.green_to_blue_) { + cur_diff -= 3; // favor keeping the areas locally similar + } + if ((uint8_t)red_to_blue == prev_x.red_to_blue_) { + cur_diff -= 3; // favor keeping the areas locally similar + } + if ((uint8_t)red_to_blue == prev_y.red_to_blue_) { + cur_diff -= 3; // favor keeping the areas locally similar + } + if (green_to_blue == 0) { + cur_diff -= 3; + } + if (red_to_blue == 0) { + cur_diff -= 3; + } + return cur_diff; +} + +#define kGreenRedToBlueNumAxis 8 +#define kGreenRedToBlueMaxIters 7 +static void GetBestGreenRedToBlue( + const uint32_t* argb, int stride, int tile_width, int tile_height, + VP8LMultipliers prev_x, VP8LMultipliers prev_y, int quality, + const int accumulated_blue_histo[256], + VP8LMultipliers* const best_tx) { + const int8_t offset[kGreenRedToBlueNumAxis][2] = + {{0, -1}, {0, 1}, {-1, 0}, {1, 0}, {-1, -1}, {-1, 1}, {1, -1}, {1, 1}}; + const int8_t delta_lut[kGreenRedToBlueMaxIters] = { 16, 16, 8, 4, 2, 2, 2 }; + const int iters = + (quality < 25) ? 1 : (quality > 50) ? kGreenRedToBlueMaxIters : 4; + int green_to_blue_best = 0; + int red_to_blue_best = 0; + int iter; + // Initial value at origin: + float best_diff = GetPredictionCostCrossColorBlue( + argb, stride, tile_width, tile_height, prev_x, prev_y, + green_to_blue_best, red_to_blue_best, accumulated_blue_histo); + for (iter = 0; iter < iters; ++iter) { + const int delta = delta_lut[iter]; + int axis; + for (axis = 0; axis < kGreenRedToBlueNumAxis; ++axis) { + const int green_to_blue_cur = + offset[axis][0] * delta + green_to_blue_best; + const int red_to_blue_cur = offset[axis][1] * delta + red_to_blue_best; + const float cur_diff = GetPredictionCostCrossColorBlue( + argb, stride, tile_width, tile_height, prev_x, prev_y, + green_to_blue_cur, red_to_blue_cur, accumulated_blue_histo); + if (cur_diff < best_diff) { + best_diff = cur_diff; + green_to_blue_best = green_to_blue_cur; + red_to_blue_best = red_to_blue_cur; + } + if (quality < 25 && iter == 4) { + // Only axis aligned diffs for lower quality. + break; // next iter. + } + } + if (delta == 2 && green_to_blue_best == 0 && red_to_blue_best == 0) { + // Further iterations would not help. + break; // out of iter-loop. + } + } + best_tx->green_to_blue_ = green_to_blue_best; + best_tx->red_to_blue_ = red_to_blue_best; +} +#undef kGreenRedToBlueMaxIters +#undef kGreenRedToBlueNumAxis + +static VP8LMultipliers GetBestColorTransformForTile( + int tile_x, int tile_y, int bits, + VP8LMultipliers prev_x, + VP8LMultipliers prev_y, + int quality, int xsize, int ysize, + const int accumulated_red_histo[256], + const int accumulated_blue_histo[256], + const uint32_t* const argb) { + const int max_tile_size = 1 << bits; + const int tile_y_offset = tile_y * max_tile_size; + const int tile_x_offset = tile_x * max_tile_size; + const int all_x_max = GetMin(tile_x_offset + max_tile_size, xsize); + const int all_y_max = GetMin(tile_y_offset + max_tile_size, ysize); + const int tile_width = all_x_max - tile_x_offset; + const int tile_height = all_y_max - tile_y_offset; + const uint32_t* const tile_argb = argb + tile_y_offset * xsize + + tile_x_offset; + VP8LMultipliers best_tx; + MultipliersClear(&best_tx); + + GetBestGreenToRed(tile_argb, xsize, tile_width, tile_height, + prev_x, prev_y, quality, accumulated_red_histo, &best_tx); + GetBestGreenRedToBlue(tile_argb, xsize, tile_width, tile_height, + prev_x, prev_y, quality, accumulated_blue_histo, + &best_tx); + return best_tx; +} + +static void CopyTileWithColorTransform(int xsize, int ysize, + int tile_x, int tile_y, + int max_tile_size, + VP8LMultipliers color_transform, + uint32_t* argb) { + const int xscan = GetMin(max_tile_size, xsize - tile_x); + int yscan = GetMin(max_tile_size, ysize - tile_y); + argb += tile_y * xsize + tile_x; + while (yscan-- > 0) { + VP8LTransformColor(&color_transform, argb, xscan); + argb += xsize; + } +} + +void VP8LColorSpaceTransform(int width, int height, int bits, int quality, + uint32_t* const argb, uint32_t* image) { + const int max_tile_size = 1 << bits; + const int tile_xsize = VP8LSubSampleSize(width, bits); + const int tile_ysize = VP8LSubSampleSize(height, bits); + int accumulated_red_histo[256] = { 0 }; + int accumulated_blue_histo[256] = { 0 }; + int tile_x, tile_y; + VP8LMultipliers prev_x, prev_y; + MultipliersClear(&prev_y); + MultipliersClear(&prev_x); + for (tile_y = 0; tile_y < tile_ysize; ++tile_y) { + for (tile_x = 0; tile_x < tile_xsize; ++tile_x) { + int y; + const int tile_x_offset = tile_x * max_tile_size; + const int tile_y_offset = tile_y * max_tile_size; + const int all_x_max = GetMin(tile_x_offset + max_tile_size, width); + const int all_y_max = GetMin(tile_y_offset + max_tile_size, height); + const int offset = tile_y * tile_xsize + tile_x; + if (tile_y != 0) { + ColorCodeToMultipliers(image[offset - tile_xsize], &prev_y); + } + prev_x = GetBestColorTransformForTile(tile_x, tile_y, bits, + prev_x, prev_y, + quality, width, height, + accumulated_red_histo, + accumulated_blue_histo, + argb); + image[offset] = MultipliersToColorCode(&prev_x); + CopyTileWithColorTransform(width, height, tile_x_offset, tile_y_offset, + max_tile_size, prev_x, argb); + + // Gather accumulated histogram data. + for (y = tile_y_offset; y < all_y_max; ++y) { + int ix = y * width + tile_x_offset; + const int ix_end = ix + all_x_max - tile_x_offset; + for (; ix < ix_end; ++ix) { + const uint32_t pix = argb[ix]; + if (ix >= 2 && + pix == argb[ix - 2] && + pix == argb[ix - 1]) { + continue; // repeated pixels are handled by backward references + } + if (ix >= width + 2 && + argb[ix - 2] == argb[ix - width - 2] && + argb[ix - 1] == argb[ix - width - 1] && + pix == argb[ix - width]) { + continue; // repeated pixels are handled by backward references + } + ++accumulated_red_histo[(pix >> 16) & 0xff]; + ++accumulated_blue_histo[(pix >> 0) & 0xff]; + } + } + } + } +} + +//------------------------------------------------------------------------------ + +static int VectorMismatch(const uint32_t* const array1, + const uint32_t* const array2, int length) { + int match_len = 0; + + while (match_len < length && array1[match_len] == array2[match_len]) { + ++match_len; + } + return match_len; +} + +// Bundles multiple (1, 2, 4 or 8) pixels into a single pixel. +void VP8LBundleColorMap(const uint8_t* const row, int width, + int xbits, uint32_t* const dst) { + int x; + if (xbits > 0) { + const int bit_depth = 1 << (3 - xbits); + const int mask = (1 << xbits) - 1; + uint32_t code = 0xff000000; + for (x = 0; x < width; ++x) { + const int xsub = x & mask; + if (xsub == 0) { + code = 0xff000000; + } + code |= row[x] << (8 + bit_depth * xsub); + dst[x >> xbits] = code; + } + } else { + for (x = 0; x < width; ++x) dst[x] = 0xff000000 | (row[x] << 8); + } +} + +//------------------------------------------------------------------------------ + +static double ExtraCost(const uint32_t* population, int length) { + int i; + double cost = 0.; + for (i = 2; i < length - 2; ++i) cost += (i >> 1) * population[i + 2]; + return cost; +} + +static double ExtraCostCombined(const uint32_t* X, const uint32_t* Y, + int length) { + int i; + double cost = 0.; + for (i = 2; i < length - 2; ++i) { + const int xy = X[i + 2] + Y[i + 2]; + cost += (i >> 1) * xy; + } + return cost; +} + +//------------------------------------------------------------------------------ + +static void HistogramAdd(const VP8LHistogram* const a, + const VP8LHistogram* const b, + VP8LHistogram* const out) { + int i; + const int literal_size = VP8LHistogramNumCodes(a->palette_code_bits_); + assert(a->palette_code_bits_ == b->palette_code_bits_); + if (b != out) { + for (i = 0; i < literal_size; ++i) { + out->literal_[i] = a->literal_[i] + b->literal_[i]; + } + for (i = 0; i < NUM_DISTANCE_CODES; ++i) { + out->distance_[i] = a->distance_[i] + b->distance_[i]; + } + for (i = 0; i < NUM_LITERAL_CODES; ++i) { + out->red_[i] = a->red_[i] + b->red_[i]; + out->blue_[i] = a->blue_[i] + b->blue_[i]; + out->alpha_[i] = a->alpha_[i] + b->alpha_[i]; + } + } else { + for (i = 0; i < literal_size; ++i) { + out->literal_[i] += a->literal_[i]; + } + for (i = 0; i < NUM_DISTANCE_CODES; ++i) { + out->distance_[i] += a->distance_[i]; + } + for (i = 0; i < NUM_LITERAL_CODES; ++i) { + out->red_[i] += a->red_[i]; + out->blue_[i] += a->blue_[i]; + out->alpha_[i] += a->alpha_[i]; + } + } +} + +//------------------------------------------------------------------------------ + +VP8LProcessBlueAndRedFunc VP8LSubtractGreenFromBlueAndRed; + +VP8LTransformColorFunc VP8LTransformColor; + +VP8LCollectColorBlueTransformsFunc VP8LCollectColorBlueTransforms; +VP8LCollectColorRedTransformsFunc VP8LCollectColorRedTransforms; + +VP8LFastLog2SlowFunc VP8LFastLog2Slow; +VP8LFastLog2SlowFunc VP8LFastSLog2Slow; + +VP8LCostFunc VP8LExtraCost; +VP8LCostCombinedFunc VP8LExtraCostCombined; +VP8LCombinedShannonEntropyFunc VP8LCombinedShannonEntropy; + +GetEntropyUnrefinedHelperFunc VP8LGetEntropyUnrefinedHelper; + +VP8LHistogramAddFunc VP8LHistogramAdd; + +VP8LVectorMismatchFunc VP8LVectorMismatch; + +extern void VP8LEncDspInitSSE2(void); +extern void VP8LEncDspInitSSE41(void); +extern void VP8LEncDspInitNEON(void); +extern void VP8LEncDspInitMIPS32(void); +extern void VP8LEncDspInitMIPSdspR2(void); + +static volatile VP8CPUInfo lossless_enc_last_cpuinfo_used = + (VP8CPUInfo)&lossless_enc_last_cpuinfo_used; + +WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInit(void) { + if (lossless_enc_last_cpuinfo_used == VP8GetCPUInfo) return; + + VP8LDspInit(); + + VP8LSubtractGreenFromBlueAndRed = VP8LSubtractGreenFromBlueAndRed_C; + + VP8LTransformColor = VP8LTransformColor_C; + + VP8LCollectColorBlueTransforms = VP8LCollectColorBlueTransforms_C; + VP8LCollectColorRedTransforms = VP8LCollectColorRedTransforms_C; + + VP8LFastLog2Slow = FastLog2Slow; + VP8LFastSLog2Slow = FastSLog2Slow; + + VP8LExtraCost = ExtraCost; + VP8LExtraCostCombined = ExtraCostCombined; + VP8LCombinedShannonEntropy = CombinedShannonEntropy; + + VP8LGetEntropyUnrefinedHelper = GetEntropyUnrefinedHelper; + + VP8LHistogramAdd = HistogramAdd; + + VP8LVectorMismatch = VectorMismatch; + + // If defined, use CPUInfo() to overwrite some pointers with faster versions. + if (VP8GetCPUInfo != NULL) { +#if defined(WEBP_USE_SSE2) + if (VP8GetCPUInfo(kSSE2)) { + VP8LEncDspInitSSE2(); +#if defined(WEBP_USE_SSE41) + if (VP8GetCPUInfo(kSSE4_1)) { + VP8LEncDspInitSSE41(); + } +#endif + } +#endif +#if defined(WEBP_USE_NEON) + if (VP8GetCPUInfo(kNEON)) { + VP8LEncDspInitNEON(); + } +#endif +#if defined(WEBP_USE_MIPS32) + if (VP8GetCPUInfo(kMIPS32)) { + VP8LEncDspInitMIPS32(); + } +#endif +#if defined(WEBP_USE_MIPS_DSP_R2) + if (VP8GetCPUInfo(kMIPSdspR2)) { + VP8LEncDspInitMIPSdspR2(); + } +#endif + } + lossless_enc_last_cpuinfo_used = VP8GetCPUInfo; +} + +//------------------------------------------------------------------------------ diff --git a/thirdparty/libwebp/dsp/lossless_enc_mips32.c b/thirdparty/libwebp/dsp/lossless_enc_mips32.c new file mode 100644 index 0000000000..49c666d4fd --- /dev/null +++ b/thirdparty/libwebp/dsp/lossless_enc_mips32.c @@ -0,0 +1,386 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MIPS version of lossless functions +// +// Author(s): Djordje Pesut (djordje.pesut@imgtec.com) +// Jovan Zelincevic (jovan.zelincevic@imgtec.com) + +#include "./dsp.h" +#include "./lossless.h" + +#if defined(WEBP_USE_MIPS32) + +#include <assert.h> +#include <math.h> +#include <stdlib.h> +#include <string.h> + +static float FastSLog2Slow(uint32_t v) { + assert(v >= LOG_LOOKUP_IDX_MAX); + if (v < APPROX_LOG_WITH_CORRECTION_MAX) { + uint32_t log_cnt, y, correction; + const int c24 = 24; + const float v_f = (float)v; + uint32_t temp; + + // Xf = 256 = 2^8 + // log_cnt is index of leading one in upper 24 bits + __asm__ volatile( + "clz %[log_cnt], %[v] \n\t" + "addiu %[y], $zero, 1 \n\t" + "subu %[log_cnt], %[c24], %[log_cnt] \n\t" + "sllv %[y], %[y], %[log_cnt] \n\t" + "srlv %[temp], %[v], %[log_cnt] \n\t" + : [log_cnt]"=&r"(log_cnt), [y]"=&r"(y), + [temp]"=r"(temp) + : [c24]"r"(c24), [v]"r"(v) + ); + + // vf = (2^log_cnt) * Xf; where y = 2^log_cnt and Xf < 256 + // Xf = floor(Xf) * (1 + (v % y) / v) + // log2(Xf) = log2(floor(Xf)) + log2(1 + (v % y) / v) + // The correction factor: log(1 + d) ~ d; for very small d values, so + // log2(1 + (v % y) / v) ~ LOG_2_RECIPROCAL * (v % y)/v + // LOG_2_RECIPROCAL ~ 23/16 + + // (v % y) = (v % 2^log_cnt) = v & (2^log_cnt - 1) + correction = (23 * (v & (y - 1))) >> 4; + return v_f * (kLog2Table[temp] + log_cnt) + correction; + } else { + return (float)(LOG_2_RECIPROCAL * v * log((double)v)); + } +} + +static float FastLog2Slow(uint32_t v) { + assert(v >= LOG_LOOKUP_IDX_MAX); + if (v < APPROX_LOG_WITH_CORRECTION_MAX) { + uint32_t log_cnt, y; + const int c24 = 24; + double log_2; + uint32_t temp; + + __asm__ volatile( + "clz %[log_cnt], %[v] \n\t" + "addiu %[y], $zero, 1 \n\t" + "subu %[log_cnt], %[c24], %[log_cnt] \n\t" + "sllv %[y], %[y], %[log_cnt] \n\t" + "srlv %[temp], %[v], %[log_cnt] \n\t" + : [log_cnt]"=&r"(log_cnt), [y]"=&r"(y), + [temp]"=r"(temp) + : [c24]"r"(c24), [v]"r"(v) + ); + + log_2 = kLog2Table[temp] + log_cnt; + if (v >= APPROX_LOG_MAX) { + // Since the division is still expensive, add this correction factor only + // for large values of 'v'. + + const uint32_t correction = (23 * (v & (y - 1))) >> 4; + log_2 += (double)correction / v; + } + return (float)log_2; + } else { + return (float)(LOG_2_RECIPROCAL * log((double)v)); + } +} + +// C version of this function: +// int i = 0; +// int64_t cost = 0; +// const uint32_t* pop = &population[4]; +// const uint32_t* LoopEnd = &population[length]; +// while (pop != LoopEnd) { +// ++i; +// cost += i * *pop; +// cost += i * *(pop + 1); +// pop += 2; +// } +// return (double)cost; +static double ExtraCost(const uint32_t* const population, int length) { + int i, temp0, temp1; + const uint32_t* pop = &population[4]; + const uint32_t* const LoopEnd = &population[length]; + + __asm__ volatile( + "mult $zero, $zero \n\t" + "xor %[i], %[i], %[i] \n\t" + "beq %[pop], %[LoopEnd], 2f \n\t" + "1: \n\t" + "lw %[temp0], 0(%[pop]) \n\t" + "lw %[temp1], 4(%[pop]) \n\t" + "addiu %[i], %[i], 1 \n\t" + "addiu %[pop], %[pop], 8 \n\t" + "madd %[i], %[temp0] \n\t" + "madd %[i], %[temp1] \n\t" + "bne %[pop], %[LoopEnd], 1b \n\t" + "2: \n\t" + "mfhi %[temp0] \n\t" + "mflo %[temp1] \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), + [i]"=&r"(i), [pop]"+r"(pop) + : [LoopEnd]"r"(LoopEnd) + : "memory", "hi", "lo" + ); + + return (double)((int64_t)temp0 << 32 | temp1); +} + +// C version of this function: +// int i = 0; +// int64_t cost = 0; +// const uint32_t* pX = &X[4]; +// const uint32_t* pY = &Y[4]; +// const uint32_t* LoopEnd = &X[length]; +// while (pX != LoopEnd) { +// const uint32_t xy0 = *pX + *pY; +// const uint32_t xy1 = *(pX + 1) + *(pY + 1); +// ++i; +// cost += i * xy0; +// cost += i * xy1; +// pX += 2; +// pY += 2; +// } +// return (double)cost; +static double ExtraCostCombined(const uint32_t* const X, + const uint32_t* const Y, int length) { + int i, temp0, temp1, temp2, temp3; + const uint32_t* pX = &X[4]; + const uint32_t* pY = &Y[4]; + const uint32_t* const LoopEnd = &X[length]; + + __asm__ volatile( + "mult $zero, $zero \n\t" + "xor %[i], %[i], %[i] \n\t" + "beq %[pX], %[LoopEnd], 2f \n\t" + "1: \n\t" + "lw %[temp0], 0(%[pX]) \n\t" + "lw %[temp1], 0(%[pY]) \n\t" + "lw %[temp2], 4(%[pX]) \n\t" + "lw %[temp3], 4(%[pY]) \n\t" + "addiu %[i], %[i], 1 \n\t" + "addu %[temp0], %[temp0], %[temp1] \n\t" + "addu %[temp2], %[temp2], %[temp3] \n\t" + "addiu %[pX], %[pX], 8 \n\t" + "addiu %[pY], %[pY], 8 \n\t" + "madd %[i], %[temp0] \n\t" + "madd %[i], %[temp2] \n\t" + "bne %[pX], %[LoopEnd], 1b \n\t" + "2: \n\t" + "mfhi %[temp0] \n\t" + "mflo %[temp1] \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), + [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), + [i]"=&r"(i), [pX]"+r"(pX), [pY]"+r"(pY) + : [LoopEnd]"r"(LoopEnd) + : "memory", "hi", "lo" + ); + + return (double)((int64_t)temp0 << 32 | temp1); +} + +#define HUFFMAN_COST_PASS \ + __asm__ volatile( \ + "sll %[temp1], %[temp0], 3 \n\t" \ + "addiu %[temp3], %[streak], -3 \n\t" \ + "addu %[temp2], %[pstreaks], %[temp1] \n\t" \ + "blez %[temp3], 1f \n\t" \ + "srl %[temp1], %[temp1], 1 \n\t" \ + "addu %[temp3], %[pcnts], %[temp1] \n\t" \ + "lw %[temp0], 4(%[temp2]) \n\t" \ + "lw %[temp1], 0(%[temp3]) \n\t" \ + "addu %[temp0], %[temp0], %[streak] \n\t" \ + "addiu %[temp1], %[temp1], 1 \n\t" \ + "sw %[temp0], 4(%[temp2]) \n\t" \ + "sw %[temp1], 0(%[temp3]) \n\t" \ + "b 2f \n\t" \ + "1: \n\t" \ + "lw %[temp0], 0(%[temp2]) \n\t" \ + "addu %[temp0], %[temp0], %[streak] \n\t" \ + "sw %[temp0], 0(%[temp2]) \n\t" \ + "2: \n\t" \ + : [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), \ + [temp3]"=&r"(temp3), [temp0]"+r"(temp0) \ + : [pstreaks]"r"(pstreaks), [pcnts]"r"(pcnts), \ + [streak]"r"(streak) \ + : "memory" \ + ); + +// Returns the various RLE counts +static WEBP_INLINE void GetEntropyUnrefinedHelper( + uint32_t val, int i, uint32_t* const val_prev, int* const i_prev, + VP8LBitEntropy* const bit_entropy, VP8LStreaks* const stats) { + int* const pstreaks = &stats->streaks[0][0]; + int* const pcnts = &stats->counts[0]; + int temp0, temp1, temp2, temp3; + const int streak = i - *i_prev; + + // Gather info for the bit entropy. + if (*val_prev != 0) { + bit_entropy->sum += (*val_prev) * streak; + bit_entropy->nonzeros += streak; + bit_entropy->nonzero_code = *i_prev; + bit_entropy->entropy -= VP8LFastSLog2(*val_prev) * streak; + if (bit_entropy->max_val < *val_prev) { + bit_entropy->max_val = *val_prev; + } + } + + // Gather info for the Huffman cost. + temp0 = (*val_prev != 0); + HUFFMAN_COST_PASS + + *val_prev = val; + *i_prev = i; +} + +#define ASM_START \ + __asm__ volatile( \ + ".set push \n\t" \ + ".set at \n\t" \ + ".set macro \n\t" \ + "1: \n\t" + +// P2 = P0 + P1 +// A..D - offsets +// E - temp variable to tell macro +// if pointer should be incremented +// literal_ and successive histograms could be unaligned +// so we must use ulw and usw +#define ADD_TO_OUT(A, B, C, D, E, P0, P1, P2) \ + "ulw %[temp0], " #A "(%[" #P0 "]) \n\t" \ + "ulw %[temp1], " #B "(%[" #P0 "]) \n\t" \ + "ulw %[temp2], " #C "(%[" #P0 "]) \n\t" \ + "ulw %[temp3], " #D "(%[" #P0 "]) \n\t" \ + "ulw %[temp4], " #A "(%[" #P1 "]) \n\t" \ + "ulw %[temp5], " #B "(%[" #P1 "]) \n\t" \ + "ulw %[temp6], " #C "(%[" #P1 "]) \n\t" \ + "ulw %[temp7], " #D "(%[" #P1 "]) \n\t" \ + "addu %[temp4], %[temp4], %[temp0] \n\t" \ + "addu %[temp5], %[temp5], %[temp1] \n\t" \ + "addu %[temp6], %[temp6], %[temp2] \n\t" \ + "addu %[temp7], %[temp7], %[temp3] \n\t" \ + "addiu %[" #P0 "], %[" #P0 "], 16 \n\t" \ + ".if " #E " == 1 \n\t" \ + "addiu %[" #P1 "], %[" #P1 "], 16 \n\t" \ + ".endif \n\t" \ + "usw %[temp4], " #A "(%[" #P2 "]) \n\t" \ + "usw %[temp5], " #B "(%[" #P2 "]) \n\t" \ + "usw %[temp6], " #C "(%[" #P2 "]) \n\t" \ + "usw %[temp7], " #D "(%[" #P2 "]) \n\t" \ + "addiu %[" #P2 "], %[" #P2 "], 16 \n\t" \ + "bne %[" #P0 "], %[LoopEnd], 1b \n\t" \ + ".set pop \n\t" \ + +#define ASM_END_COMMON_0 \ + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), \ + [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), \ + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), \ + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), \ + [pa]"+r"(pa), [pout]"+r"(pout) + +#define ASM_END_COMMON_1 \ + : [LoopEnd]"r"(LoopEnd) \ + : "memory", "at" \ + ); + +#define ASM_END_0 \ + ASM_END_COMMON_0 \ + , [pb]"+r"(pb) \ + ASM_END_COMMON_1 + +#define ASM_END_1 \ + ASM_END_COMMON_0 \ + ASM_END_COMMON_1 + +#define ADD_VECTOR(A, B, OUT, SIZE, EXTRA_SIZE) do { \ + const uint32_t* pa = (const uint32_t*)(A); \ + const uint32_t* pb = (const uint32_t*)(B); \ + uint32_t* pout = (uint32_t*)(OUT); \ + const uint32_t* const LoopEnd = pa + (SIZE); \ + assert((SIZE) % 4 == 0); \ + ASM_START \ + ADD_TO_OUT(0, 4, 8, 12, 1, pa, pb, pout) \ + ASM_END_0 \ + if ((EXTRA_SIZE) > 0) { \ + const int last = (EXTRA_SIZE); \ + int i; \ + for (i = 0; i < last; ++i) pout[i] = pa[i] + pb[i]; \ + } \ +} while (0) + +#define ADD_VECTOR_EQ(A, OUT, SIZE, EXTRA_SIZE) do { \ + const uint32_t* pa = (const uint32_t*)(A); \ + uint32_t* pout = (uint32_t*)(OUT); \ + const uint32_t* const LoopEnd = pa + (SIZE); \ + assert((SIZE) % 4 == 0); \ + ASM_START \ + ADD_TO_OUT(0, 4, 8, 12, 0, pa, pout, pout) \ + ASM_END_1 \ + if ((EXTRA_SIZE) > 0) { \ + const int last = (EXTRA_SIZE); \ + int i; \ + for (i = 0; i < last; ++i) pout[i] += pa[i]; \ + } \ +} while (0) + +static void HistogramAdd(const VP8LHistogram* const a, + const VP8LHistogram* const b, + VP8LHistogram* const out) { + uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; + const int extra_cache_size = VP8LHistogramNumCodes(a->palette_code_bits_) + - (NUM_LITERAL_CODES + NUM_LENGTH_CODES); + assert(a->palette_code_bits_ == b->palette_code_bits_); + + if (b != out) { + ADD_VECTOR(a->literal_, b->literal_, out->literal_, + NUM_LITERAL_CODES + NUM_LENGTH_CODES, extra_cache_size); + ADD_VECTOR(a->distance_, b->distance_, out->distance_, + NUM_DISTANCE_CODES, 0); + ADD_VECTOR(a->red_, b->red_, out->red_, NUM_LITERAL_CODES, 0); + ADD_VECTOR(a->blue_, b->blue_, out->blue_, NUM_LITERAL_CODES, 0); + ADD_VECTOR(a->alpha_, b->alpha_, out->alpha_, NUM_LITERAL_CODES, 0); + } else { + ADD_VECTOR_EQ(a->literal_, out->literal_, + NUM_LITERAL_CODES + NUM_LENGTH_CODES, extra_cache_size); + ADD_VECTOR_EQ(a->distance_, out->distance_, NUM_DISTANCE_CODES, 0); + ADD_VECTOR_EQ(a->red_, out->red_, NUM_LITERAL_CODES, 0); + ADD_VECTOR_EQ(a->blue_, out->blue_, NUM_LITERAL_CODES, 0); + ADD_VECTOR_EQ(a->alpha_, out->alpha_, NUM_LITERAL_CODES, 0); + } +} + +#undef ADD_VECTOR_EQ +#undef ADD_VECTOR +#undef ASM_END_1 +#undef ASM_END_0 +#undef ASM_END_COMMON_1 +#undef ASM_END_COMMON_0 +#undef ADD_TO_OUT +#undef ASM_START + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8LEncDspInitMIPS32(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitMIPS32(void) { + VP8LFastSLog2Slow = FastSLog2Slow; + VP8LFastLog2Slow = FastLog2Slow; + VP8LExtraCost = ExtraCost; + VP8LExtraCostCombined = ExtraCostCombined; + VP8LGetEntropyUnrefinedHelper = GetEntropyUnrefinedHelper; + VP8LHistogramAdd = HistogramAdd; +} + +#else // !WEBP_USE_MIPS32 + +WEBP_DSP_INIT_STUB(VP8LEncDspInitMIPS32) + +#endif // WEBP_USE_MIPS32 diff --git a/thirdparty/libwebp/dsp/lossless_enc_mips_dsp_r2.c b/thirdparty/libwebp/dsp/lossless_enc_mips_dsp_r2.c new file mode 100644 index 0000000000..0abf3c4f36 --- /dev/null +++ b/thirdparty/libwebp/dsp/lossless_enc_mips_dsp_r2.c @@ -0,0 +1,275 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Image transform methods for lossless encoder. +// +// Author(s): Djordje Pesut (djordje.pesut@imgtec.com) +// Jovan Zelincevic (jovan.zelincevic@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS_DSP_R2) + +#include "./lossless.h" + +static void SubtractGreenFromBlueAndRed(uint32_t* argb_data, + int num_pixels) { + uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; + uint32_t* const p_loop1_end = argb_data + (num_pixels & ~3); + uint32_t* const p_loop2_end = p_loop1_end + (num_pixels & 3); + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "beq %[argb_data], %[p_loop1_end], 3f \n\t" + " nop \n\t" + "0: \n\t" + "lw %[temp0], 0(%[argb_data]) \n\t" + "lw %[temp1], 4(%[argb_data]) \n\t" + "lw %[temp2], 8(%[argb_data]) \n\t" + "lw %[temp3], 12(%[argb_data]) \n\t" + "ext %[temp4], %[temp0], 8, 8 \n\t" + "ext %[temp5], %[temp1], 8, 8 \n\t" + "ext %[temp6], %[temp2], 8, 8 \n\t" + "ext %[temp7], %[temp3], 8, 8 \n\t" + "addiu %[argb_data], %[argb_data], 16 \n\t" + "replv.ph %[temp4], %[temp4] \n\t" + "replv.ph %[temp5], %[temp5] \n\t" + "replv.ph %[temp6], %[temp6] \n\t" + "replv.ph %[temp7], %[temp7] \n\t" + "subu.qb %[temp0], %[temp0], %[temp4] \n\t" + "subu.qb %[temp1], %[temp1], %[temp5] \n\t" + "subu.qb %[temp2], %[temp2], %[temp6] \n\t" + "subu.qb %[temp3], %[temp3], %[temp7] \n\t" + "sw %[temp0], -16(%[argb_data]) \n\t" + "sw %[temp1], -12(%[argb_data]) \n\t" + "sw %[temp2], -8(%[argb_data]) \n\t" + "bne %[argb_data], %[p_loop1_end], 0b \n\t" + " sw %[temp3], -4(%[argb_data]) \n\t" + "3: \n\t" + "beq %[argb_data], %[p_loop2_end], 2f \n\t" + " nop \n\t" + "1: \n\t" + "lw %[temp0], 0(%[argb_data]) \n\t" + "addiu %[argb_data], %[argb_data], 4 \n\t" + "ext %[temp4], %[temp0], 8, 8 \n\t" + "replv.ph %[temp4], %[temp4] \n\t" + "subu.qb %[temp0], %[temp0], %[temp4] \n\t" + "bne %[argb_data], %[p_loop2_end], 1b \n\t" + " sw %[temp0], -4(%[argb_data]) \n\t" + "2: \n\t" + ".set pop \n\t" + : [argb_data]"+&r"(argb_data), [temp0]"=&r"(temp0), + [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [temp6]"=&r"(temp6), + [temp7]"=&r"(temp7) + : [p_loop1_end]"r"(p_loop1_end), [p_loop2_end]"r"(p_loop2_end) + : "memory" + ); +} + +static WEBP_INLINE uint32_t ColorTransformDelta(int8_t color_pred, + int8_t color) { + return (uint32_t)((int)(color_pred) * color) >> 5; +} + +static void TransformColor(const VP8LMultipliers* const m, uint32_t* data, + int num_pixels) { + int temp0, temp1, temp2, temp3, temp4, temp5; + uint32_t argb, argb1, new_red, new_red1; + const uint32_t G_to_R = m->green_to_red_; + const uint32_t G_to_B = m->green_to_blue_; + const uint32_t R_to_B = m->red_to_blue_; + uint32_t* const p_loop_end = data + (num_pixels & ~1); + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "beq %[data], %[p_loop_end], 1f \n\t" + " nop \n\t" + "replv.ph %[temp0], %[G_to_R] \n\t" + "replv.ph %[temp1], %[G_to_B] \n\t" + "replv.ph %[temp2], %[R_to_B] \n\t" + "shll.ph %[temp0], %[temp0], 8 \n\t" + "shll.ph %[temp1], %[temp1], 8 \n\t" + "shll.ph %[temp2], %[temp2], 8 \n\t" + "shra.ph %[temp0], %[temp0], 8 \n\t" + "shra.ph %[temp1], %[temp1], 8 \n\t" + "shra.ph %[temp2], %[temp2], 8 \n\t" + "0: \n\t" + "lw %[argb], 0(%[data]) \n\t" + "lw %[argb1], 4(%[data]) \n\t" + "lhu %[new_red], 2(%[data]) \n\t" + "lhu %[new_red1], 6(%[data]) \n\t" + "precrq.qb.ph %[temp3], %[argb], %[argb1] \n\t" + "precr.qb.ph %[temp4], %[argb], %[argb1] \n\t" + "preceu.ph.qbra %[temp3], %[temp3] \n\t" + "preceu.ph.qbla %[temp4], %[temp4] \n\t" + "shll.ph %[temp3], %[temp3], 8 \n\t" + "shll.ph %[temp4], %[temp4], 8 \n\t" + "shra.ph %[temp3], %[temp3], 8 \n\t" + "shra.ph %[temp4], %[temp4], 8 \n\t" + "mul.ph %[temp5], %[temp3], %[temp0] \n\t" + "mul.ph %[temp3], %[temp3], %[temp1] \n\t" + "mul.ph %[temp4], %[temp4], %[temp2] \n\t" + "addiu %[data], %[data], 8 \n\t" + "ins %[new_red1], %[new_red], 16, 16 \n\t" + "ins %[argb1], %[argb], 16, 16 \n\t" + "shra.ph %[temp5], %[temp5], 5 \n\t" + "shra.ph %[temp3], %[temp3], 5 \n\t" + "shra.ph %[temp4], %[temp4], 5 \n\t" + "subu.ph %[new_red1], %[new_red1], %[temp5] \n\t" + "subu.ph %[argb1], %[argb1], %[temp3] \n\t" + "preceu.ph.qbra %[temp5], %[new_red1] \n\t" + "subu.ph %[argb1], %[argb1], %[temp4] \n\t" + "preceu.ph.qbra %[temp3], %[argb1] \n\t" + "sb %[temp5], -2(%[data]) \n\t" + "sb %[temp3], -4(%[data]) \n\t" + "sra %[temp5], %[temp5], 16 \n\t" + "sra %[temp3], %[temp3], 16 \n\t" + "sb %[temp5], -6(%[data]) \n\t" + "bne %[data], %[p_loop_end], 0b \n\t" + " sb %[temp3], -8(%[data]) \n\t" + "1: \n\t" + ".set pop \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [new_red1]"=&r"(new_red1), [new_red]"=&r"(new_red), + [argb]"=&r"(argb), [argb1]"=&r"(argb1), [data]"+&r"(data) + : [G_to_R]"r"(G_to_R), [R_to_B]"r"(R_to_B), + [G_to_B]"r"(G_to_B), [p_loop_end]"r"(p_loop_end) + : "memory", "hi", "lo" + ); + + if (num_pixels & 1) { + const uint32_t argb_ = data[0]; + const uint32_t green = argb_ >> 8; + const uint32_t red = argb_ >> 16; + uint32_t new_blue = argb_; + new_red = red; + new_red -= ColorTransformDelta(m->green_to_red_, green); + new_red &= 0xff; + new_blue -= ColorTransformDelta(m->green_to_blue_, green); + new_blue -= ColorTransformDelta(m->red_to_blue_, red); + new_blue &= 0xff; + data[0] = (argb_ & 0xff00ff00u) | (new_red << 16) | (new_blue); + } +} + +static WEBP_INLINE uint8_t TransformColorBlue(uint8_t green_to_blue, + uint8_t red_to_blue, + uint32_t argb) { + const uint32_t green = argb >> 8; + const uint32_t red = argb >> 16; + uint8_t new_blue = argb; + new_blue -= ColorTransformDelta(green_to_blue, green); + new_blue -= ColorTransformDelta(red_to_blue, red); + return (new_blue & 0xff); +} + +static void CollectColorBlueTransforms(const uint32_t* argb, int stride, + int tile_width, int tile_height, + int green_to_blue, int red_to_blue, + int histo[]) { + const int rtb = (red_to_blue << 16) | (red_to_blue & 0xffff); + const int gtb = (green_to_blue << 16) | (green_to_blue & 0xffff); + const uint32_t mask = 0xff00ffu; + while (tile_height-- > 0) { + int x; + const uint32_t* p_argb = argb; + argb += stride; + for (x = 0; x < (tile_width >> 1); ++x) { + int temp0, temp1, temp2, temp3, temp4, temp5, temp6; + __asm__ volatile ( + "lw %[temp0], 0(%[p_argb]) \n\t" + "lw %[temp1], 4(%[p_argb]) \n\t" + "precr.qb.ph %[temp2], %[temp0], %[temp1] \n\t" + "ins %[temp1], %[temp0], 16, 16 \n\t" + "shra.ph %[temp2], %[temp2], 8 \n\t" + "shra.ph %[temp3], %[temp1], 8 \n\t" + "mul.ph %[temp5], %[temp2], %[rtb] \n\t" + "mul.ph %[temp6], %[temp3], %[gtb] \n\t" + "and %[temp4], %[temp1], %[mask] \n\t" + "addiu %[p_argb], %[p_argb], 8 \n\t" + "shra.ph %[temp5], %[temp5], 5 \n\t" + "shra.ph %[temp6], %[temp6], 5 \n\t" + "subu.qb %[temp2], %[temp4], %[temp5] \n\t" + "subu.qb %[temp2], %[temp2], %[temp6] \n\t" + : [p_argb]"+&r"(p_argb), [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), + [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), + [temp5]"=&r"(temp5), [temp6]"=&r"(temp6) + : [rtb]"r"(rtb), [gtb]"r"(gtb), [mask]"r"(mask) + : "memory", "hi", "lo" + ); + ++histo[(uint8_t)(temp2 >> 16)]; + ++histo[(uint8_t)temp2]; + } + if (tile_width & 1) { + ++histo[TransformColorBlue(green_to_blue, red_to_blue, *p_argb)]; + } + } +} + +static WEBP_INLINE uint8_t TransformColorRed(uint8_t green_to_red, + uint32_t argb) { + const uint32_t green = argb >> 8; + uint32_t new_red = argb >> 16; + new_red -= ColorTransformDelta(green_to_red, green); + return (new_red & 0xff); +} + +static void CollectColorRedTransforms(const uint32_t* argb, int stride, + int tile_width, int tile_height, + int green_to_red, int histo[]) { + const int gtr = (green_to_red << 16) | (green_to_red & 0xffff); + while (tile_height-- > 0) { + int x; + const uint32_t* p_argb = argb; + argb += stride; + for (x = 0; x < (tile_width >> 1); ++x) { + int temp0, temp1, temp2, temp3, temp4; + __asm__ volatile ( + "lw %[temp0], 0(%[p_argb]) \n\t" + "lw %[temp1], 4(%[p_argb]) \n\t" + "precrq.ph.w %[temp4], %[temp0], %[temp1] \n\t" + "ins %[temp1], %[temp0], 16, 16 \n\t" + "shra.ph %[temp3], %[temp1], 8 \n\t" + "mul.ph %[temp2], %[temp3], %[gtr] \n\t" + "addiu %[p_argb], %[p_argb], 8 \n\t" + "shra.ph %[temp2], %[temp2], 5 \n\t" + "subu.qb %[temp2], %[temp4], %[temp2] \n\t" + : [p_argb]"+&r"(p_argb), [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), + [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), [temp4]"=&r"(temp4) + : [gtr]"r"(gtr) + : "memory", "hi", "lo" + ); + ++histo[(uint8_t)(temp2 >> 16)]; + ++histo[(uint8_t)temp2]; + } + if (tile_width & 1) { + ++histo[TransformColorRed(green_to_red, *p_argb)]; + } + } +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8LEncDspInitMIPSdspR2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitMIPSdspR2(void) { + VP8LSubtractGreenFromBlueAndRed = SubtractGreenFromBlueAndRed; + VP8LTransformColor = TransformColor; + VP8LCollectColorBlueTransforms = CollectColorBlueTransforms; + VP8LCollectColorRedTransforms = CollectColorRedTransforms; +} + +#else // !WEBP_USE_MIPS_DSP_R2 + +WEBP_DSP_INIT_STUB(VP8LEncDspInitMIPSdspR2) + +#endif // WEBP_USE_MIPS_DSP_R2 diff --git a/thirdparty/libwebp/dsp/lossless_enc_neon.c b/thirdparty/libwebp/dsp/lossless_enc_neon.c new file mode 100644 index 0000000000..4c56f2594b --- /dev/null +++ b/thirdparty/libwebp/dsp/lossless_enc_neon.c @@ -0,0 +1,143 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// NEON variant of methods for lossless encoder +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_NEON) + +#include <arm_neon.h> + +#include "./lossless.h" +#include "./neon.h" + +//------------------------------------------------------------------------------ +// Subtract-Green Transform + +// vtbl?_u8 are marked unavailable for iOS arm64 with Xcode < 6.3, use +// non-standard versions there. +#if defined(__APPLE__) && defined(__aarch64__) && \ + defined(__apple_build_version__) && (__apple_build_version__< 6020037) +#define USE_VTBLQ +#endif + +#ifdef USE_VTBLQ +// 255 = byte will be zeroed +static const uint8_t kGreenShuffle[16] = { + 1, 255, 1, 255, 5, 255, 5, 255, 9, 255, 9, 255, 13, 255, 13, 255 +}; + +static WEBP_INLINE uint8x16_t DoGreenShuffle(const uint8x16_t argb, + const uint8x16_t shuffle) { + return vcombine_u8(vtbl1q_u8(argb, vget_low_u8(shuffle)), + vtbl1q_u8(argb, vget_high_u8(shuffle))); +} +#else // !USE_VTBLQ +// 255 = byte will be zeroed +static const uint8_t kGreenShuffle[8] = { 1, 255, 1, 255, 5, 255, 5, 255 }; + +static WEBP_INLINE uint8x16_t DoGreenShuffle(const uint8x16_t argb, + const uint8x8_t shuffle) { + return vcombine_u8(vtbl1_u8(vget_low_u8(argb), shuffle), + vtbl1_u8(vget_high_u8(argb), shuffle)); +} +#endif // USE_VTBLQ + +static void SubtractGreenFromBlueAndRed(uint32_t* argb_data, int num_pixels) { + const uint32_t* const end = argb_data + (num_pixels & ~3); +#ifdef USE_VTBLQ + const uint8x16_t shuffle = vld1q_u8(kGreenShuffle); +#else + const uint8x8_t shuffle = vld1_u8(kGreenShuffle); +#endif + for (; argb_data < end; argb_data += 4) { + const uint8x16_t argb = vld1q_u8((uint8_t*)argb_data); + const uint8x16_t greens = DoGreenShuffle(argb, shuffle); + vst1q_u8((uint8_t*)argb_data, vsubq_u8(argb, greens)); + } + // fallthrough and finish off with plain-C + VP8LSubtractGreenFromBlueAndRed_C(argb_data, num_pixels & 3); +} + +//------------------------------------------------------------------------------ +// Color Transform + +static void TransformColor(const VP8LMultipliers* const m, + uint32_t* argb_data, int num_pixels) { + // sign-extended multiplying constants, pre-shifted by 6. +#define CST(X) (((int16_t)(m->X << 8)) >> 6) + const int16_t rb[8] = { + CST(green_to_blue_), CST(green_to_red_), + CST(green_to_blue_), CST(green_to_red_), + CST(green_to_blue_), CST(green_to_red_), + CST(green_to_blue_), CST(green_to_red_) + }; + const int16x8_t mults_rb = vld1q_s16(rb); + const int16_t b2[8] = { + 0, CST(red_to_blue_), 0, CST(red_to_blue_), + 0, CST(red_to_blue_), 0, CST(red_to_blue_), + }; + const int16x8_t mults_b2 = vld1q_s16(b2); +#undef CST +#ifdef USE_VTBLQ + static const uint8_t kg0g0[16] = { + 255, 1, 255, 1, 255, 5, 255, 5, 255, 9, 255, 9, 255, 13, 255, 13 + }; + const uint8x16_t shuffle = vld1q_u8(kg0g0); +#else + static const uint8_t k0g0g[8] = { 255, 1, 255, 1, 255, 5, 255, 5 }; + const uint8x8_t shuffle = vld1_u8(k0g0g); +#endif + const uint32x4_t mask_rb = vdupq_n_u32(0x00ff00ffu); // red-blue masks + int i; + for (i = 0; i + 4 <= num_pixels; i += 4) { + const uint8x16_t in = vld1q_u8((uint8_t*)(argb_data + i)); + // 0 g 0 g + const uint8x16_t greens = DoGreenShuffle(in, shuffle); + // x dr x db1 + const int16x8_t A = vqdmulhq_s16(vreinterpretq_s16_u8(greens), mults_rb); + // r 0 b 0 + const int16x8_t B = vshlq_n_s16(vreinterpretq_s16_u8(in), 8); + // x db2 0 0 + const int16x8_t C = vqdmulhq_s16(B, mults_b2); + // 0 0 x db2 + const uint32x4_t D = vshrq_n_u32(vreinterpretq_u32_s16(C), 16); + // x dr x db + const int8x16_t E = vaddq_s8(vreinterpretq_s8_u32(D), + vreinterpretq_s8_s16(A)); + // 0 dr 0 db + const uint32x4_t F = vandq_u32(vreinterpretq_u32_s8(E), mask_rb); + const int8x16_t out = vsubq_s8(vreinterpretq_s8_u8(in), + vreinterpretq_s8_u32(F)); + vst1q_s8((int8_t*)(argb_data + i), out); + } + // fallthrough and finish off with plain-C + VP8LTransformColor_C(m, argb_data + i, num_pixels - i); +} + +#undef USE_VTBLQ + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8LEncDspInitNEON(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitNEON(void) { + VP8LSubtractGreenFromBlueAndRed = SubtractGreenFromBlueAndRed; + VP8LTransformColor = TransformColor; +} + +#else // !WEBP_USE_NEON + +WEBP_DSP_INIT_STUB(VP8LEncDspInitNEON) + +#endif // WEBP_USE_NEON diff --git a/thirdparty/libwebp/dsp/lossless_enc_sse2.c b/thirdparty/libwebp/dsp/lossless_enc_sse2.c new file mode 100644 index 0000000000..7c894e7ca4 --- /dev/null +++ b/thirdparty/libwebp/dsp/lossless_enc_sse2.c @@ -0,0 +1,397 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE2 variant of methods for lossless encoder +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE2) +#include <assert.h> +#include <emmintrin.h> +#include "./lossless.h" + +// For sign-extended multiplying constants, pre-shifted by 5: +#define CST_5b(X) (((int16_t)((uint16_t)X << 8)) >> 5) + +//------------------------------------------------------------------------------ +// Subtract-Green Transform + +static void SubtractGreenFromBlueAndRed(uint32_t* argb_data, int num_pixels) { + int i; + for (i = 0; i + 4 <= num_pixels; i += 4) { + const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb + const __m128i A = _mm_srli_epi16(in, 8); // 0 a 0 g + const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0)); + const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // 0g0g + const __m128i out = _mm_sub_epi8(in, C); + _mm_storeu_si128((__m128i*)&argb_data[i], out); + } + // fallthrough and finish off with plain-C + VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i); +} + +//------------------------------------------------------------------------------ +// Color Transform + +static void TransformColor(const VP8LMultipliers* const m, + uint32_t* argb_data, int num_pixels) { + const __m128i mults_rb = _mm_set_epi16( + CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_), + CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_), + CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_), + CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_)); + const __m128i mults_b2 = _mm_set_epi16( + CST_5b(m->red_to_blue_), 0, CST_5b(m->red_to_blue_), 0, + CST_5b(m->red_to_blue_), 0, CST_5b(m->red_to_blue_), 0); + const __m128i mask_ag = _mm_set1_epi32(0xff00ff00); // alpha-green masks + const __m128i mask_rb = _mm_set1_epi32(0x00ff00ff); // red-blue masks + int i; + for (i = 0; i + 4 <= num_pixels; i += 4) { + const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb + const __m128i A = _mm_and_si128(in, mask_ag); // a 0 g 0 + const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0)); + const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // g0g0 + const __m128i D = _mm_mulhi_epi16(C, mults_rb); // x dr x db1 + const __m128i E = _mm_slli_epi16(in, 8); // r 0 b 0 + const __m128i F = _mm_mulhi_epi16(E, mults_b2); // x db2 0 0 + const __m128i G = _mm_srli_epi32(F, 16); // 0 0 x db2 + const __m128i H = _mm_add_epi8(G, D); // x dr x db + const __m128i I = _mm_and_si128(H, mask_rb); // 0 dr 0 db + const __m128i out = _mm_sub_epi8(in, I); + _mm_storeu_si128((__m128i*)&argb_data[i], out); + } + // fallthrough and finish off with plain-C + VP8LTransformColor_C(m, argb_data + i, num_pixels - i); +} + +//------------------------------------------------------------------------------ +#define SPAN 8 +static void CollectColorBlueTransforms(const uint32_t* argb, int stride, + int tile_width, int tile_height, + int green_to_blue, int red_to_blue, + int histo[]) { + const __m128i mults_r = _mm_set_epi16( + CST_5b(red_to_blue), 0, CST_5b(red_to_blue), 0, + CST_5b(red_to_blue), 0, CST_5b(red_to_blue), 0); + const __m128i mults_g = _mm_set_epi16( + 0, CST_5b(green_to_blue), 0, CST_5b(green_to_blue), + 0, CST_5b(green_to_blue), 0, CST_5b(green_to_blue)); + const __m128i mask_g = _mm_set1_epi32(0x00ff00); // green mask + const __m128i mask_b = _mm_set1_epi32(0x0000ff); // blue mask + int y; + for (y = 0; y < tile_height; ++y) { + const uint32_t* const src = argb + y * stride; + int i, x; + for (x = 0; x + SPAN <= tile_width; x += SPAN) { + uint16_t values[SPAN]; + const __m128i in0 = _mm_loadu_si128((__m128i*)&src[x + 0]); + const __m128i in1 = _mm_loadu_si128((__m128i*)&src[x + SPAN / 2]); + const __m128i A0 = _mm_slli_epi16(in0, 8); // r 0 | b 0 + const __m128i A1 = _mm_slli_epi16(in1, 8); + const __m128i B0 = _mm_and_si128(in0, mask_g); // 0 0 | g 0 + const __m128i B1 = _mm_and_si128(in1, mask_g); + const __m128i C0 = _mm_mulhi_epi16(A0, mults_r); // x db | 0 0 + const __m128i C1 = _mm_mulhi_epi16(A1, mults_r); + const __m128i D0 = _mm_mulhi_epi16(B0, mults_g); // 0 0 | x db + const __m128i D1 = _mm_mulhi_epi16(B1, mults_g); + const __m128i E0 = _mm_sub_epi8(in0, D0); // x x | x b' + const __m128i E1 = _mm_sub_epi8(in1, D1); + const __m128i F0 = _mm_srli_epi32(C0, 16); // 0 0 | x db + const __m128i F1 = _mm_srli_epi32(C1, 16); + const __m128i G0 = _mm_sub_epi8(E0, F0); // 0 0 | x b' + const __m128i G1 = _mm_sub_epi8(E1, F1); + const __m128i H0 = _mm_and_si128(G0, mask_b); // 0 0 | 0 b + const __m128i H1 = _mm_and_si128(G1, mask_b); + const __m128i I = _mm_packs_epi32(H0, H1); // 0 b' | 0 b' + _mm_storeu_si128((__m128i*)values, I); + for (i = 0; i < SPAN; ++i) ++histo[values[i]]; + } + } + { + const int left_over = tile_width & (SPAN - 1); + if (left_over > 0) { + VP8LCollectColorBlueTransforms_C(argb + tile_width - left_over, stride, + left_over, tile_height, + green_to_blue, red_to_blue, histo); + } + } +} + +static void CollectColorRedTransforms(const uint32_t* argb, int stride, + int tile_width, int tile_height, + int green_to_red, int histo[]) { + const __m128i mults_g = _mm_set_epi16( + 0, CST_5b(green_to_red), 0, CST_5b(green_to_red), + 0, CST_5b(green_to_red), 0, CST_5b(green_to_red)); + const __m128i mask_g = _mm_set1_epi32(0x00ff00); // green mask + const __m128i mask = _mm_set1_epi32(0xff); + + int y; + for (y = 0; y < tile_height; ++y) { + const uint32_t* const src = argb + y * stride; + int i, x; + for (x = 0; x + SPAN <= tile_width; x += SPAN) { + uint16_t values[SPAN]; + const __m128i in0 = _mm_loadu_si128((__m128i*)&src[x + 0]); + const __m128i in1 = _mm_loadu_si128((__m128i*)&src[x + SPAN / 2]); + const __m128i A0 = _mm_and_si128(in0, mask_g); // 0 0 | g 0 + const __m128i A1 = _mm_and_si128(in1, mask_g); + const __m128i B0 = _mm_srli_epi32(in0, 16); // 0 0 | x r + const __m128i B1 = _mm_srli_epi32(in1, 16); + const __m128i C0 = _mm_mulhi_epi16(A0, mults_g); // 0 0 | x dr + const __m128i C1 = _mm_mulhi_epi16(A1, mults_g); + const __m128i E0 = _mm_sub_epi8(B0, C0); // x x | x r' + const __m128i E1 = _mm_sub_epi8(B1, C1); + const __m128i F0 = _mm_and_si128(E0, mask); // 0 0 | 0 r' + const __m128i F1 = _mm_and_si128(E1, mask); + const __m128i I = _mm_packs_epi32(F0, F1); + _mm_storeu_si128((__m128i*)values, I); + for (i = 0; i < SPAN; ++i) ++histo[values[i]]; + } + } + { + const int left_over = tile_width & (SPAN - 1); + if (left_over > 0) { + VP8LCollectColorRedTransforms_C(argb + tile_width - left_over, stride, + left_over, tile_height, + green_to_red, histo); + } + } +} +#undef SPAN + +//------------------------------------------------------------------------------ + +#define LINE_SIZE 16 // 8 or 16 +static void AddVector(const uint32_t* a, const uint32_t* b, uint32_t* out, + int size) { + int i; + assert(size % LINE_SIZE == 0); + for (i = 0; i < size; i += LINE_SIZE) { + const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i + 0]); + const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]); +#if (LINE_SIZE == 16) + const __m128i a2 = _mm_loadu_si128((const __m128i*)&a[i + 8]); + const __m128i a3 = _mm_loadu_si128((const __m128i*)&a[i + 12]); +#endif + const __m128i b0 = _mm_loadu_si128((const __m128i*)&b[i + 0]); + const __m128i b1 = _mm_loadu_si128((const __m128i*)&b[i + 4]); +#if (LINE_SIZE == 16) + const __m128i b2 = _mm_loadu_si128((const __m128i*)&b[i + 8]); + const __m128i b3 = _mm_loadu_si128((const __m128i*)&b[i + 12]); +#endif + _mm_storeu_si128((__m128i*)&out[i + 0], _mm_add_epi32(a0, b0)); + _mm_storeu_si128((__m128i*)&out[i + 4], _mm_add_epi32(a1, b1)); +#if (LINE_SIZE == 16) + _mm_storeu_si128((__m128i*)&out[i + 8], _mm_add_epi32(a2, b2)); + _mm_storeu_si128((__m128i*)&out[i + 12], _mm_add_epi32(a3, b3)); +#endif + } +} + +static void AddVectorEq(const uint32_t* a, uint32_t* out, int size) { + int i; + assert(size % LINE_SIZE == 0); + for (i = 0; i < size; i += LINE_SIZE) { + const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i + 0]); + const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]); +#if (LINE_SIZE == 16) + const __m128i a2 = _mm_loadu_si128((const __m128i*)&a[i + 8]); + const __m128i a3 = _mm_loadu_si128((const __m128i*)&a[i + 12]); +#endif + const __m128i b0 = _mm_loadu_si128((const __m128i*)&out[i + 0]); + const __m128i b1 = _mm_loadu_si128((const __m128i*)&out[i + 4]); +#if (LINE_SIZE == 16) + const __m128i b2 = _mm_loadu_si128((const __m128i*)&out[i + 8]); + const __m128i b3 = _mm_loadu_si128((const __m128i*)&out[i + 12]); +#endif + _mm_storeu_si128((__m128i*)&out[i + 0], _mm_add_epi32(a0, b0)); + _mm_storeu_si128((__m128i*)&out[i + 4], _mm_add_epi32(a1, b1)); +#if (LINE_SIZE == 16) + _mm_storeu_si128((__m128i*)&out[i + 8], _mm_add_epi32(a2, b2)); + _mm_storeu_si128((__m128i*)&out[i + 12], _mm_add_epi32(a3, b3)); +#endif + } +} +#undef LINE_SIZE + +// Note we are adding uint32_t's as *signed* int32's (using _mm_add_epi32). But +// that's ok since the histogram values are less than 1<<28 (max picture size). +static void HistogramAdd(const VP8LHistogram* const a, + const VP8LHistogram* const b, + VP8LHistogram* const out) { + int i; + const int literal_size = VP8LHistogramNumCodes(a->palette_code_bits_); + assert(a->palette_code_bits_ == b->palette_code_bits_); + if (b != out) { + AddVector(a->literal_, b->literal_, out->literal_, NUM_LITERAL_CODES); + AddVector(a->red_, b->red_, out->red_, NUM_LITERAL_CODES); + AddVector(a->blue_, b->blue_, out->blue_, NUM_LITERAL_CODES); + AddVector(a->alpha_, b->alpha_, out->alpha_, NUM_LITERAL_CODES); + } else { + AddVectorEq(a->literal_, out->literal_, NUM_LITERAL_CODES); + AddVectorEq(a->red_, out->red_, NUM_LITERAL_CODES); + AddVectorEq(a->blue_, out->blue_, NUM_LITERAL_CODES); + AddVectorEq(a->alpha_, out->alpha_, NUM_LITERAL_CODES); + } + for (i = NUM_LITERAL_CODES; i < literal_size; ++i) { + out->literal_[i] = a->literal_[i] + b->literal_[i]; + } + for (i = 0; i < NUM_DISTANCE_CODES; ++i) { + out->distance_[i] = a->distance_[i] + b->distance_[i]; + } +} + +//------------------------------------------------------------------------------ +// Entropy + +// Checks whether the X or Y contribution is worth computing and adding. +// Used in loop unrolling. +#define ANALYZE_X_OR_Y(x_or_y, j) \ + do { \ + if (x_or_y[i + j] != 0) retval -= VP8LFastSLog2(x_or_y[i + j]); \ + } while (0) + +// Checks whether the X + Y contribution is worth computing and adding. +// Used in loop unrolling. +#define ANALYZE_XY(j) \ + do { \ + if (tmp[j] != 0) { \ + retval -= VP8LFastSLog2(tmp[j]); \ + ANALYZE_X_OR_Y(X, j); \ + } \ + } while (0) + +static float CombinedShannonEntropy(const int X[256], const int Y[256]) { + int i; + double retval = 0.; + int sumX, sumXY; + int32_t tmp[4]; + __m128i zero = _mm_setzero_si128(); + // Sums up X + Y, 4 ints at a time (and will merge it at the end for sumXY). + __m128i sumXY_128 = zero; + __m128i sumX_128 = zero; + + for (i = 0; i < 256; i += 4) { + const __m128i x = _mm_loadu_si128((const __m128i*)(X + i)); + const __m128i y = _mm_loadu_si128((const __m128i*)(Y + i)); + + // Check if any X is non-zero: this actually provides a speedup as X is + // usually sparse. + if (_mm_movemask_epi8(_mm_cmpeq_epi32(x, zero)) != 0xFFFF) { + const __m128i xy_128 = _mm_add_epi32(x, y); + sumXY_128 = _mm_add_epi32(sumXY_128, xy_128); + + sumX_128 = _mm_add_epi32(sumX_128, x); + + // Analyze the different X + Y. + _mm_storeu_si128((__m128i*)tmp, xy_128); + + ANALYZE_XY(0); + ANALYZE_XY(1); + ANALYZE_XY(2); + ANALYZE_XY(3); + } else { + // X is fully 0, so only deal with Y. + sumXY_128 = _mm_add_epi32(sumXY_128, y); + + ANALYZE_X_OR_Y(Y, 0); + ANALYZE_X_OR_Y(Y, 1); + ANALYZE_X_OR_Y(Y, 2); + ANALYZE_X_OR_Y(Y, 3); + } + } + + // Sum up sumX_128 to get sumX. + _mm_storeu_si128((__m128i*)tmp, sumX_128); + sumX = tmp[3] + tmp[2] + tmp[1] + tmp[0]; + + // Sum up sumXY_128 to get sumXY. + _mm_storeu_si128((__m128i*)tmp, sumXY_128); + sumXY = tmp[3] + tmp[2] + tmp[1] + tmp[0]; + + retval += VP8LFastSLog2(sumX) + VP8LFastSLog2(sumXY); + return (float)retval; +} +#undef ANALYZE_X_OR_Y +#undef ANALYZE_XY + +//------------------------------------------------------------------------------ + +static int VectorMismatch(const uint32_t* const array1, + const uint32_t* const array2, int length) { + int match_len; + + if (length >= 12) { + __m128i A0 = _mm_loadu_si128((const __m128i*)&array1[0]); + __m128i A1 = _mm_loadu_si128((const __m128i*)&array2[0]); + match_len = 0; + do { + // Loop unrolling and early load both provide a speedup of 10% for the + // current function. Also, max_limit can be MAX_LENGTH=4096 at most. + const __m128i cmpA = _mm_cmpeq_epi32(A0, A1); + const __m128i B0 = + _mm_loadu_si128((const __m128i*)&array1[match_len + 4]); + const __m128i B1 = + _mm_loadu_si128((const __m128i*)&array2[match_len + 4]); + if (_mm_movemask_epi8(cmpA) != 0xffff) break; + match_len += 4; + + { + const __m128i cmpB = _mm_cmpeq_epi32(B0, B1); + A0 = _mm_loadu_si128((const __m128i*)&array1[match_len + 4]); + A1 = _mm_loadu_si128((const __m128i*)&array2[match_len + 4]); + if (_mm_movemask_epi8(cmpB) != 0xffff) break; + match_len += 4; + } + } while (match_len + 12 < length); + } else { + match_len = 0; + // Unroll the potential first two loops. + if (length >= 4 && + _mm_movemask_epi8(_mm_cmpeq_epi32( + _mm_loadu_si128((const __m128i*)&array1[0]), + _mm_loadu_si128((const __m128i*)&array2[0]))) == 0xffff) { + match_len = 4; + if (length >= 8 && + _mm_movemask_epi8(_mm_cmpeq_epi32( + _mm_loadu_si128((const __m128i*)&array1[4]), + _mm_loadu_si128((const __m128i*)&array2[4]))) == 0xffff) + match_len = 8; + } + } + + while (match_len < length && array1[match_len] == array2[match_len]) { + ++match_len; + } + return match_len; +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8LEncDspInitSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitSSE2(void) { + VP8LSubtractGreenFromBlueAndRed = SubtractGreenFromBlueAndRed; + VP8LTransformColor = TransformColor; + VP8LCollectColorBlueTransforms = CollectColorBlueTransforms; + VP8LCollectColorRedTransforms = CollectColorRedTransforms; + VP8LHistogramAdd = HistogramAdd; + VP8LCombinedShannonEntropy = CombinedShannonEntropy; + VP8LVectorMismatch = VectorMismatch; +} + +#else // !WEBP_USE_SSE2 + +WEBP_DSP_INIT_STUB(VP8LEncDspInitSSE2) + +#endif // WEBP_USE_SSE2 diff --git a/thirdparty/libwebp/dsp/lossless_enc_sse41.c b/thirdparty/libwebp/dsp/lossless_enc_sse41.c new file mode 100644 index 0000000000..3e493198db --- /dev/null +++ b/thirdparty/libwebp/dsp/lossless_enc_sse41.c @@ -0,0 +1,51 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE4.1 variant of methods for lossless encoder +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE41) +#include <assert.h> +#include <smmintrin.h> +#include "./lossless.h" + +//------------------------------------------------------------------------------ +// Subtract-Green Transform + +static void SubtractGreenFromBlueAndRed(uint32_t* argb_data, int num_pixels) { + int i; + const __m128i kCstShuffle = _mm_set_epi8(-1, 13, -1, 13, -1, 9, -1, 9, + -1, 5, -1, 5, -1, 1, -1, 1); + for (i = 0; i + 4 <= num_pixels; i += 4) { + const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); + const __m128i in_0g0g = _mm_shuffle_epi8(in, kCstShuffle); + const __m128i out = _mm_sub_epi8(in, in_0g0g); + _mm_storeu_si128((__m128i*)&argb_data[i], out); + } + // fallthrough and finish off with plain-C + VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i); +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8LEncDspInitSSE41(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitSSE41(void) { + VP8LSubtractGreenFromBlueAndRed = SubtractGreenFromBlueAndRed; +} + +#else // !WEBP_USE_SSE41 + +WEBP_DSP_INIT_STUB(VP8LEncDspInitSSE41) + +#endif // WEBP_USE_SSE41 diff --git a/thirdparty/libwebp/dsp/lossless_mips_dsp_r2.c b/thirdparty/libwebp/dsp/lossless_mips_dsp_r2.c new file mode 100644 index 0000000000..90aed7f151 --- /dev/null +++ b/thirdparty/libwebp/dsp/lossless_mips_dsp_r2.c @@ -0,0 +1,680 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Image transforms and color space conversion methods for lossless decoder. +// +// Author(s): Djordje Pesut (djordje.pesut@imgtec.com) +// Jovan Zelincevic (jovan.zelincevic@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS_DSP_R2) + +#include "./lossless.h" + +#define MAP_COLOR_FUNCS(FUNC_NAME, TYPE, GET_INDEX, GET_VALUE) \ +static void FUNC_NAME(const TYPE* src, \ + const uint32_t* const color_map, \ + TYPE* dst, int y_start, int y_end, \ + int width) { \ + int y; \ + for (y = y_start; y < y_end; ++y) { \ + int x; \ + for (x = 0; x < (width >> 2); ++x) { \ + int tmp1, tmp2, tmp3, tmp4; \ + __asm__ volatile ( \ + ".ifc " #TYPE ", uint8_t \n\t" \ + "lbu %[tmp1], 0(%[src]) \n\t" \ + "lbu %[tmp2], 1(%[src]) \n\t" \ + "lbu %[tmp3], 2(%[src]) \n\t" \ + "lbu %[tmp4], 3(%[src]) \n\t" \ + "addiu %[src], %[src], 4 \n\t" \ + ".endif \n\t" \ + ".ifc " #TYPE ", uint32_t \n\t" \ + "lw %[tmp1], 0(%[src]) \n\t" \ + "lw %[tmp2], 4(%[src]) \n\t" \ + "lw %[tmp3], 8(%[src]) \n\t" \ + "lw %[tmp4], 12(%[src]) \n\t" \ + "ext %[tmp1], %[tmp1], 8, 8 \n\t" \ + "ext %[tmp2], %[tmp2], 8, 8 \n\t" \ + "ext %[tmp3], %[tmp3], 8, 8 \n\t" \ + "ext %[tmp4], %[tmp4], 8, 8 \n\t" \ + "addiu %[src], %[src], 16 \n\t" \ + ".endif \n\t" \ + "sll %[tmp1], %[tmp1], 2 \n\t" \ + "sll %[tmp2], %[tmp2], 2 \n\t" \ + "sll %[tmp3], %[tmp3], 2 \n\t" \ + "sll %[tmp4], %[tmp4], 2 \n\t" \ + "lwx %[tmp1], %[tmp1](%[color_map]) \n\t" \ + "lwx %[tmp2], %[tmp2](%[color_map]) \n\t" \ + "lwx %[tmp3], %[tmp3](%[color_map]) \n\t" \ + "lwx %[tmp4], %[tmp4](%[color_map]) \n\t" \ + ".ifc " #TYPE ", uint8_t \n\t" \ + "ext %[tmp1], %[tmp1], 8, 8 \n\t" \ + "ext %[tmp2], %[tmp2], 8, 8 \n\t" \ + "ext %[tmp3], %[tmp3], 8, 8 \n\t" \ + "ext %[tmp4], %[tmp4], 8, 8 \n\t" \ + "sb %[tmp1], 0(%[dst]) \n\t" \ + "sb %[tmp2], 1(%[dst]) \n\t" \ + "sb %[tmp3], 2(%[dst]) \n\t" \ + "sb %[tmp4], 3(%[dst]) \n\t" \ + "addiu %[dst], %[dst], 4 \n\t" \ + ".endif \n\t" \ + ".ifc " #TYPE ", uint32_t \n\t" \ + "sw %[tmp1], 0(%[dst]) \n\t" \ + "sw %[tmp2], 4(%[dst]) \n\t" \ + "sw %[tmp3], 8(%[dst]) \n\t" \ + "sw %[tmp4], 12(%[dst]) \n\t" \ + "addiu %[dst], %[dst], 16 \n\t" \ + ".endif \n\t" \ + : [tmp1]"=&r"(tmp1), [tmp2]"=&r"(tmp2), [tmp3]"=&r"(tmp3), \ + [tmp4]"=&r"(tmp4), [src]"+&r"(src), [dst]"+r"(dst) \ + : [color_map]"r"(color_map) \ + : "memory" \ + ); \ + } \ + for (x = 0; x < (width & 3); ++x) { \ + *dst++ = GET_VALUE(color_map[GET_INDEX(*src++)]); \ + } \ + } \ +} + +MAP_COLOR_FUNCS(MapARGB, uint32_t, VP8GetARGBIndex, VP8GetARGBValue) +MAP_COLOR_FUNCS(MapAlpha, uint8_t, VP8GetAlphaIndex, VP8GetAlphaValue) + +#undef MAP_COLOR_FUNCS + +static WEBP_INLINE uint32_t ClampedAddSubtractFull(uint32_t c0, uint32_t c1, + uint32_t c2) { + int temp0, temp1, temp2, temp3, temp4, temp5; + __asm__ volatile ( + "preceu.ph.qbr %[temp1], %[c0] \n\t" + "preceu.ph.qbl %[temp2], %[c0] \n\t" + "preceu.ph.qbr %[temp3], %[c1] \n\t" + "preceu.ph.qbl %[temp4], %[c1] \n\t" + "preceu.ph.qbr %[temp5], %[c2] \n\t" + "preceu.ph.qbl %[temp0], %[c2] \n\t" + "subq.ph %[temp3], %[temp3], %[temp5] \n\t" + "subq.ph %[temp4], %[temp4], %[temp0] \n\t" + "addq.ph %[temp1], %[temp1], %[temp3] \n\t" + "addq.ph %[temp2], %[temp2], %[temp4] \n\t" + "shll_s.ph %[temp1], %[temp1], 7 \n\t" + "shll_s.ph %[temp2], %[temp2], 7 \n\t" + "precrqu_s.qb.ph %[temp2], %[temp2], %[temp1] \n\t" + : [temp0]"=r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5) + : [c0]"r"(c0), [c1]"r"(c1), [c2]"r"(c2) + : "memory" + ); + return temp2; +} + +static WEBP_INLINE uint32_t ClampedAddSubtractHalf(uint32_t c0, uint32_t c1, + uint32_t c2) { + int temp0, temp1, temp2, temp3, temp4, temp5; + __asm__ volatile ( + "adduh.qb %[temp5], %[c0], %[c1] \n\t" + "preceu.ph.qbr %[temp3], %[c2] \n\t" + "preceu.ph.qbr %[temp1], %[temp5] \n\t" + "preceu.ph.qbl %[temp2], %[temp5] \n\t" + "preceu.ph.qbl %[temp4], %[c2] \n\t" + "subq.ph %[temp3], %[temp1], %[temp3] \n\t" + "subq.ph %[temp4], %[temp2], %[temp4] \n\t" + "shrl.ph %[temp5], %[temp3], 15 \n\t" + "shrl.ph %[temp0], %[temp4], 15 \n\t" + "addq.ph %[temp3], %[temp3], %[temp5] \n\t" + "addq.ph %[temp4], %[temp0], %[temp4] \n\t" + "shra.ph %[temp3], %[temp3], 1 \n\t" + "shra.ph %[temp4], %[temp4], 1 \n\t" + "addq.ph %[temp1], %[temp1], %[temp3] \n\t" + "addq.ph %[temp2], %[temp2], %[temp4] \n\t" + "shll_s.ph %[temp1], %[temp1], 7 \n\t" + "shll_s.ph %[temp2], %[temp2], 7 \n\t" + "precrqu_s.qb.ph %[temp1], %[temp2], %[temp1] \n\t" + : [temp0]"=r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=r"(temp4), [temp5]"=&r"(temp5) + : [c0]"r"(c0), [c1]"r"(c1), [c2]"r"(c2) + : "memory" + ); + return temp1; +} + +static WEBP_INLINE uint32_t Select(uint32_t a, uint32_t b, uint32_t c) { + int temp0, temp1, temp2, temp3, temp4, temp5; + __asm__ volatile ( + "cmpgdu.lt.qb %[temp1], %[c], %[b] \n\t" + "pick.qb %[temp1], %[b], %[c] \n\t" + "pick.qb %[temp2], %[c], %[b] \n\t" + "cmpgdu.lt.qb %[temp4], %[c], %[a] \n\t" + "pick.qb %[temp4], %[a], %[c] \n\t" + "pick.qb %[temp5], %[c], %[a] \n\t" + "subu.qb %[temp3], %[temp1], %[temp2] \n\t" + "subu.qb %[temp0], %[temp4], %[temp5] \n\t" + "raddu.w.qb %[temp3], %[temp3] \n\t" + "raddu.w.qb %[temp0], %[temp0] \n\t" + "subu %[temp3], %[temp3], %[temp0] \n\t" + "slti %[temp0], %[temp3], 0x1 \n\t" + "movz %[a], %[b], %[temp0] \n\t" + : [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [temp0]"=&r"(temp0), + [a]"+&r"(a) + : [b]"r"(b), [c]"r"(c) + ); + return a; +} + +static WEBP_INLINE uint32_t Average2(uint32_t a0, uint32_t a1) { + __asm__ volatile ( + "adduh.qb %[a0], %[a0], %[a1] \n\t" + : [a0]"+r"(a0) + : [a1]"r"(a1) + ); + return a0; +} + +static WEBP_INLINE uint32_t Average3(uint32_t a0, uint32_t a1, uint32_t a2) { + return Average2(Average2(a0, a2), a1); +} + +static WEBP_INLINE uint32_t Average4(uint32_t a0, uint32_t a1, + uint32_t a2, uint32_t a3) { + return Average2(Average2(a0, a1), Average2(a2, a3)); +} + +static uint32_t Predictor5(uint32_t left, const uint32_t* const top) { + return Average3(left, top[0], top[1]); +} + +static uint32_t Predictor6(uint32_t left, const uint32_t* const top) { + return Average2(left, top[-1]); +} + +static uint32_t Predictor7(uint32_t left, const uint32_t* const top) { + return Average2(left, top[0]); +} + +static uint32_t Predictor8(uint32_t left, const uint32_t* const top) { + (void)left; + return Average2(top[-1], top[0]); +} + +static uint32_t Predictor9(uint32_t left, const uint32_t* const top) { + (void)left; + return Average2(top[0], top[1]); +} + +static uint32_t Predictor10(uint32_t left, const uint32_t* const top) { + return Average4(left, top[-1], top[0], top[1]); +} + +static uint32_t Predictor11(uint32_t left, const uint32_t* const top) { + return Select(top[0], left, top[-1]); +} + +static uint32_t Predictor12(uint32_t left, const uint32_t* const top) { + return ClampedAddSubtractFull(left, top[0], top[-1]); +} + +static uint32_t Predictor13(uint32_t left, const uint32_t* const top) { + return ClampedAddSubtractHalf(left, top[0], top[-1]); +} + +// Add green to blue and red channels (i.e. perform the inverse transform of +// 'subtract green'). +static void AddGreenToBlueAndRed(uint32_t* data, int num_pixels) { + uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; + uint32_t* const p_loop1_end = data + (num_pixels & ~3); + uint32_t* const p_loop2_end = data + num_pixels; + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "beq %[data], %[p_loop1_end], 3f \n\t" + " nop \n\t" + "0: \n\t" + "lw %[temp0], 0(%[data]) \n\t" + "lw %[temp1], 4(%[data]) \n\t" + "lw %[temp2], 8(%[data]) \n\t" + "lw %[temp3], 12(%[data]) \n\t" + "ext %[temp4], %[temp0], 8, 8 \n\t" + "ext %[temp5], %[temp1], 8, 8 \n\t" + "ext %[temp6], %[temp2], 8, 8 \n\t" + "ext %[temp7], %[temp3], 8, 8 \n\t" + "addiu %[data], %[data], 16 \n\t" + "replv.ph %[temp4], %[temp4] \n\t" + "replv.ph %[temp5], %[temp5] \n\t" + "replv.ph %[temp6], %[temp6] \n\t" + "replv.ph %[temp7], %[temp7] \n\t" + "addu.qb %[temp0], %[temp0], %[temp4] \n\t" + "addu.qb %[temp1], %[temp1], %[temp5] \n\t" + "addu.qb %[temp2], %[temp2], %[temp6] \n\t" + "addu.qb %[temp3], %[temp3], %[temp7] \n\t" + "sw %[temp0], -16(%[data]) \n\t" + "sw %[temp1], -12(%[data]) \n\t" + "sw %[temp2], -8(%[data]) \n\t" + "bne %[data], %[p_loop1_end], 0b \n\t" + " sw %[temp3], -4(%[data]) \n\t" + "3: \n\t" + "beq %[data], %[p_loop2_end], 2f \n\t" + " nop \n\t" + "1: \n\t" + "lw %[temp0], 0(%[data]) \n\t" + "addiu %[data], %[data], 4 \n\t" + "ext %[temp4], %[temp0], 8, 8 \n\t" + "replv.ph %[temp4], %[temp4] \n\t" + "addu.qb %[temp0], %[temp0], %[temp4] \n\t" + "bne %[data], %[p_loop2_end], 1b \n\t" + " sw %[temp0], -4(%[data]) \n\t" + "2: \n\t" + ".set pop \n\t" + : [data]"+&r"(data), [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), + [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), + [temp5]"=&r"(temp5), [temp6]"=&r"(temp6), [temp7]"=&r"(temp7) + : [p_loop1_end]"r"(p_loop1_end), [p_loop2_end]"r"(p_loop2_end) + : "memory" + ); +} + +static void TransformColorInverse(const VP8LMultipliers* const m, + uint32_t* data, int num_pixels) { + int temp0, temp1, temp2, temp3, temp4, temp5; + uint32_t argb, argb1, new_red; + const uint32_t G_to_R = m->green_to_red_; + const uint32_t G_to_B = m->green_to_blue_; + const uint32_t R_to_B = m->red_to_blue_; + uint32_t* const p_loop_end = data + (num_pixels & ~1); + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "beq %[data], %[p_loop_end], 1f \n\t" + " nop \n\t" + "replv.ph %[temp0], %[G_to_R] \n\t" + "replv.ph %[temp1], %[G_to_B] \n\t" + "replv.ph %[temp2], %[R_to_B] \n\t" + "shll.ph %[temp0], %[temp0], 8 \n\t" + "shll.ph %[temp1], %[temp1], 8 \n\t" + "shll.ph %[temp2], %[temp2], 8 \n\t" + "shra.ph %[temp0], %[temp0], 8 \n\t" + "shra.ph %[temp1], %[temp1], 8 \n\t" + "shra.ph %[temp2], %[temp2], 8 \n\t" + "0: \n\t" + "lw %[argb], 0(%[data]) \n\t" + "lw %[argb1], 4(%[data]) \n\t" + "addiu %[data], %[data], 8 \n\t" + "precrq.qb.ph %[temp3], %[argb], %[argb1] \n\t" + "preceu.ph.qbra %[temp3], %[temp3] \n\t" + "shll.ph %[temp3], %[temp3], 8 \n\t" + "shra.ph %[temp3], %[temp3], 8 \n\t" + "mul.ph %[temp5], %[temp3], %[temp0] \n\t" + "mul.ph %[temp3], %[temp3], %[temp1] \n\t" + "precrq.ph.w %[new_red], %[argb], %[argb1] \n\t" + "ins %[argb1], %[argb], 16, 16 \n\t" + "shra.ph %[temp5], %[temp5], 5 \n\t" + "shra.ph %[temp3], %[temp3], 5 \n\t" + "addu.ph %[new_red], %[new_red], %[temp5] \n\t" + "addu.ph %[argb1], %[argb1], %[temp3] \n\t" + "preceu.ph.qbra %[temp5], %[new_red] \n\t" + "shll.ph %[temp4], %[temp5], 8 \n\t" + "shra.ph %[temp4], %[temp4], 8 \n\t" + "mul.ph %[temp4], %[temp4], %[temp2] \n\t" + "sb %[temp5], -2(%[data]) \n\t" + "sra %[temp5], %[temp5], 16 \n\t" + "shra.ph %[temp4], %[temp4], 5 \n\t" + "addu.ph %[argb1], %[argb1], %[temp4] \n\t" + "preceu.ph.qbra %[temp3], %[argb1] \n\t" + "sb %[temp5], -6(%[data]) \n\t" + "sb %[temp3], -4(%[data]) \n\t" + "sra %[temp3], %[temp3], 16 \n\t" + "bne %[data], %[p_loop_end], 0b \n\t" + " sb %[temp3], -8(%[data]) \n\t" + "1: \n\t" + ".set pop \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [new_red]"=&r"(new_red), [argb]"=&r"(argb), + [argb1]"=&r"(argb1), [data]"+&r"(data) + : [G_to_R]"r"(G_to_R), [R_to_B]"r"(R_to_B), + [G_to_B]"r"(G_to_B), [p_loop_end]"r"(p_loop_end) + : "memory", "hi", "lo" + ); + + // Fall-back to C-version for left-overs. + if (num_pixels & 1) VP8LTransformColorInverse_C(m, data, 1); +} + +static void ConvertBGRAToRGB(const uint32_t* src, + int num_pixels, uint8_t* dst) { + int temp0, temp1, temp2, temp3; + const uint32_t* const p_loop1_end = src + (num_pixels & ~3); + const uint32_t* const p_loop2_end = src + num_pixels; + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "beq %[src], %[p_loop1_end], 3f \n\t" + " nop \n\t" + "0: \n\t" + "lw %[temp3], 12(%[src]) \n\t" + "lw %[temp2], 8(%[src]) \n\t" + "lw %[temp1], 4(%[src]) \n\t" + "lw %[temp0], 0(%[src]) \n\t" + "ins %[temp3], %[temp2], 24, 8 \n\t" + "sll %[temp2], %[temp2], 8 \n\t" + "rotr %[temp3], %[temp3], 16 \n\t" + "ins %[temp2], %[temp1], 0, 16 \n\t" + "sll %[temp1], %[temp1], 8 \n\t" + "wsbh %[temp3], %[temp3] \n\t" + "balign %[temp0], %[temp1], 1 \n\t" + "wsbh %[temp2], %[temp2] \n\t" + "wsbh %[temp0], %[temp0] \n\t" + "usw %[temp3], 8(%[dst]) \n\t" + "rotr %[temp0], %[temp0], 16 \n\t" + "usw %[temp2], 4(%[dst]) \n\t" + "addiu %[src], %[src], 16 \n\t" + "usw %[temp0], 0(%[dst]) \n\t" + "bne %[src], %[p_loop1_end], 0b \n\t" + " addiu %[dst], %[dst], 12 \n\t" + "3: \n\t" + "beq %[src], %[p_loop2_end], 2f \n\t" + " nop \n\t" + "1: \n\t" + "lw %[temp0], 0(%[src]) \n\t" + "addiu %[src], %[src], 4 \n\t" + "wsbh %[temp1], %[temp0] \n\t" + "addiu %[dst], %[dst], 3 \n\t" + "ush %[temp1], -2(%[dst]) \n\t" + "sra %[temp0], %[temp0], 16 \n\t" + "bne %[src], %[p_loop2_end], 1b \n\t" + " sb %[temp0], -3(%[dst]) \n\t" + "2: \n\t" + ".set pop \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [dst]"+&r"(dst), [src]"+&r"(src) + : [p_loop1_end]"r"(p_loop1_end), [p_loop2_end]"r"(p_loop2_end) + : "memory" + ); +} + +static void ConvertBGRAToRGBA(const uint32_t* src, + int num_pixels, uint8_t* dst) { + int temp0, temp1, temp2, temp3; + const uint32_t* const p_loop1_end = src + (num_pixels & ~3); + const uint32_t* const p_loop2_end = src + num_pixels; + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "beq %[src], %[p_loop1_end], 3f \n\t" + " nop \n\t" + "0: \n\t" + "lw %[temp0], 0(%[src]) \n\t" + "lw %[temp1], 4(%[src]) \n\t" + "lw %[temp2], 8(%[src]) \n\t" + "lw %[temp3], 12(%[src]) \n\t" + "wsbh %[temp0], %[temp0] \n\t" + "wsbh %[temp1], %[temp1] \n\t" + "wsbh %[temp2], %[temp2] \n\t" + "wsbh %[temp3], %[temp3] \n\t" + "addiu %[src], %[src], 16 \n\t" + "balign %[temp0], %[temp0], 1 \n\t" + "balign %[temp1], %[temp1], 1 \n\t" + "balign %[temp2], %[temp2], 1 \n\t" + "balign %[temp3], %[temp3], 1 \n\t" + "usw %[temp0], 0(%[dst]) \n\t" + "usw %[temp1], 4(%[dst]) \n\t" + "usw %[temp2], 8(%[dst]) \n\t" + "usw %[temp3], 12(%[dst]) \n\t" + "bne %[src], %[p_loop1_end], 0b \n\t" + " addiu %[dst], %[dst], 16 \n\t" + "3: \n\t" + "beq %[src], %[p_loop2_end], 2f \n\t" + " nop \n\t" + "1: \n\t" + "lw %[temp0], 0(%[src]) \n\t" + "wsbh %[temp0], %[temp0] \n\t" + "addiu %[src], %[src], 4 \n\t" + "balign %[temp0], %[temp0], 1 \n\t" + "usw %[temp0], 0(%[dst]) \n\t" + "bne %[src], %[p_loop2_end], 1b \n\t" + " addiu %[dst], %[dst], 4 \n\t" + "2: \n\t" + ".set pop \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [dst]"+&r"(dst), [src]"+&r"(src) + : [p_loop1_end]"r"(p_loop1_end), [p_loop2_end]"r"(p_loop2_end) + : "memory" + ); +} + +static void ConvertBGRAToRGBA4444(const uint32_t* src, + int num_pixels, uint8_t* dst) { + int temp0, temp1, temp2, temp3, temp4, temp5; + const uint32_t* const p_loop1_end = src + (num_pixels & ~3); + const uint32_t* const p_loop2_end = src + num_pixels; + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "beq %[src], %[p_loop1_end], 3f \n\t" + " nop \n\t" + "0: \n\t" + "lw %[temp0], 0(%[src]) \n\t" + "lw %[temp1], 4(%[src]) \n\t" + "lw %[temp2], 8(%[src]) \n\t" + "lw %[temp3], 12(%[src]) \n\t" + "ext %[temp4], %[temp0], 28, 4 \n\t" + "ext %[temp5], %[temp0], 12, 4 \n\t" + "ins %[temp0], %[temp4], 0, 4 \n\t" + "ext %[temp4], %[temp1], 28, 4 \n\t" + "ins %[temp0], %[temp5], 16, 4 \n\t" + "ext %[temp5], %[temp1], 12, 4 \n\t" + "ins %[temp1], %[temp4], 0, 4 \n\t" + "ext %[temp4], %[temp2], 28, 4 \n\t" + "ins %[temp1], %[temp5], 16, 4 \n\t" + "ext %[temp5], %[temp2], 12, 4 \n\t" + "ins %[temp2], %[temp4], 0, 4 \n\t" + "ext %[temp4], %[temp3], 28, 4 \n\t" + "ins %[temp2], %[temp5], 16, 4 \n\t" + "ext %[temp5], %[temp3], 12, 4 \n\t" + "ins %[temp3], %[temp4], 0, 4 \n\t" + "precr.qb.ph %[temp1], %[temp1], %[temp0] \n\t" + "ins %[temp3], %[temp5], 16, 4 \n\t" + "addiu %[src], %[src], 16 \n\t" + "precr.qb.ph %[temp3], %[temp3], %[temp2] \n\t" +#ifdef WEBP_SWAP_16BIT_CSP + "usw %[temp1], 0(%[dst]) \n\t" + "usw %[temp3], 4(%[dst]) \n\t" +#else + "wsbh %[temp1], %[temp1] \n\t" + "wsbh %[temp3], %[temp3] \n\t" + "usw %[temp1], 0(%[dst]) \n\t" + "usw %[temp3], 4(%[dst]) \n\t" +#endif + "bne %[src], %[p_loop1_end], 0b \n\t" + " addiu %[dst], %[dst], 8 \n\t" + "3: \n\t" + "beq %[src], %[p_loop2_end], 2f \n\t" + " nop \n\t" + "1: \n\t" + "lw %[temp0], 0(%[src]) \n\t" + "ext %[temp4], %[temp0], 28, 4 \n\t" + "ext %[temp5], %[temp0], 12, 4 \n\t" + "ins %[temp0], %[temp4], 0, 4 \n\t" + "ins %[temp0], %[temp5], 16, 4 \n\t" + "addiu %[src], %[src], 4 \n\t" + "precr.qb.ph %[temp0], %[temp0], %[temp0] \n\t" +#ifdef WEBP_SWAP_16BIT_CSP + "ush %[temp0], 0(%[dst]) \n\t" +#else + "wsbh %[temp0], %[temp0] \n\t" + "ush %[temp0], 0(%[dst]) \n\t" +#endif + "bne %[src], %[p_loop2_end], 1b \n\t" + " addiu %[dst], %[dst], 2 \n\t" + "2: \n\t" + ".set pop \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [dst]"+&r"(dst), [src]"+&r"(src) + : [p_loop1_end]"r"(p_loop1_end), [p_loop2_end]"r"(p_loop2_end) + : "memory" + ); +} + +static void ConvertBGRAToRGB565(const uint32_t* src, + int num_pixels, uint8_t* dst) { + int temp0, temp1, temp2, temp3, temp4, temp5; + const uint32_t* const p_loop1_end = src + (num_pixels & ~3); + const uint32_t* const p_loop2_end = src + num_pixels; + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "beq %[src], %[p_loop1_end], 3f \n\t" + " nop \n\t" + "0: \n\t" + "lw %[temp0], 0(%[src]) \n\t" + "lw %[temp1], 4(%[src]) \n\t" + "lw %[temp2], 8(%[src]) \n\t" + "lw %[temp3], 12(%[src]) \n\t" + "ext %[temp4], %[temp0], 8, 16 \n\t" + "ext %[temp5], %[temp0], 5, 11 \n\t" + "ext %[temp0], %[temp0], 3, 5 \n\t" + "ins %[temp4], %[temp5], 0, 11 \n\t" + "ext %[temp5], %[temp1], 5, 11 \n\t" + "ins %[temp4], %[temp0], 0, 5 \n\t" + "ext %[temp0], %[temp1], 8, 16 \n\t" + "ext %[temp1], %[temp1], 3, 5 \n\t" + "ins %[temp0], %[temp5], 0, 11 \n\t" + "ext %[temp5], %[temp2], 5, 11 \n\t" + "ins %[temp0], %[temp1], 0, 5 \n\t" + "ext %[temp1], %[temp2], 8, 16 \n\t" + "ext %[temp2], %[temp2], 3, 5 \n\t" + "ins %[temp1], %[temp5], 0, 11 \n\t" + "ext %[temp5], %[temp3], 5, 11 \n\t" + "ins %[temp1], %[temp2], 0, 5 \n\t" + "ext %[temp2], %[temp3], 8, 16 \n\t" + "ext %[temp3], %[temp3], 3, 5 \n\t" + "ins %[temp2], %[temp5], 0, 11 \n\t" + "append %[temp0], %[temp4], 16 \n\t" + "ins %[temp2], %[temp3], 0, 5 \n\t" + "addiu %[src], %[src], 16 \n\t" + "append %[temp2], %[temp1], 16 \n\t" +#ifdef WEBP_SWAP_16BIT_CSP + "usw %[temp0], 0(%[dst]) \n\t" + "usw %[temp2], 4(%[dst]) \n\t" +#else + "wsbh %[temp0], %[temp0] \n\t" + "wsbh %[temp2], %[temp2] \n\t" + "usw %[temp0], 0(%[dst]) \n\t" + "usw %[temp2], 4(%[dst]) \n\t" +#endif + "bne %[src], %[p_loop1_end], 0b \n\t" + " addiu %[dst], %[dst], 8 \n\t" + "3: \n\t" + "beq %[src], %[p_loop2_end], 2f \n\t" + " nop \n\t" + "1: \n\t" + "lw %[temp0], 0(%[src]) \n\t" + "ext %[temp4], %[temp0], 8, 16 \n\t" + "ext %[temp5], %[temp0], 5, 11 \n\t" + "ext %[temp0], %[temp0], 3, 5 \n\t" + "ins %[temp4], %[temp5], 0, 11 \n\t" + "addiu %[src], %[src], 4 \n\t" + "ins %[temp4], %[temp0], 0, 5 \n\t" +#ifdef WEBP_SWAP_16BIT_CSP + "ush %[temp4], 0(%[dst]) \n\t" +#else + "wsbh %[temp4], %[temp4] \n\t" + "ush %[temp4], 0(%[dst]) \n\t" +#endif + "bne %[src], %[p_loop2_end], 1b \n\t" + " addiu %[dst], %[dst], 2 \n\t" + "2: \n\t" + ".set pop \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), + [dst]"+&r"(dst), [src]"+&r"(src) + : [p_loop1_end]"r"(p_loop1_end), [p_loop2_end]"r"(p_loop2_end) + : "memory" + ); +} + +static void ConvertBGRAToBGR(const uint32_t* src, + int num_pixels, uint8_t* dst) { + int temp0, temp1, temp2, temp3; + const uint32_t* const p_loop1_end = src + (num_pixels & ~3); + const uint32_t* const p_loop2_end = src + num_pixels; + __asm__ volatile ( + ".set push \n\t" + ".set noreorder \n\t" + "beq %[src], %[p_loop1_end], 3f \n\t" + " nop \n\t" + "0: \n\t" + "lw %[temp0], 0(%[src]) \n\t" + "lw %[temp1], 4(%[src]) \n\t" + "lw %[temp2], 8(%[src]) \n\t" + "lw %[temp3], 12(%[src]) \n\t" + "ins %[temp0], %[temp1], 24, 8 \n\t" + "sra %[temp1], %[temp1], 8 \n\t" + "ins %[temp1], %[temp2], 16, 16 \n\t" + "sll %[temp2], %[temp2], 8 \n\t" + "balign %[temp3], %[temp2], 1 \n\t" + "addiu %[src], %[src], 16 \n\t" + "usw %[temp0], 0(%[dst]) \n\t" + "usw %[temp1], 4(%[dst]) \n\t" + "usw %[temp3], 8(%[dst]) \n\t" + "bne %[src], %[p_loop1_end], 0b \n\t" + " addiu %[dst], %[dst], 12 \n\t" + "3: \n\t" + "beq %[src], %[p_loop2_end], 2f \n\t" + " nop \n\t" + "1: \n\t" + "lw %[temp0], 0(%[src]) \n\t" + "addiu %[src], %[src], 4 \n\t" + "addiu %[dst], %[dst], 3 \n\t" + "ush %[temp0], -3(%[dst]) \n\t" + "sra %[temp0], %[temp0], 16 \n\t" + "bne %[src], %[p_loop2_end], 1b \n\t" + " sb %[temp0], -1(%[dst]) \n\t" + "2: \n\t" + ".set pop \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), + [temp3]"=&r"(temp3), [dst]"+&r"(dst), [src]"+&r"(src) + : [p_loop1_end]"r"(p_loop1_end), [p_loop2_end]"r"(p_loop2_end) + : "memory" + ); +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8LDspInitMIPSdspR2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInitMIPSdspR2(void) { + VP8LMapColor32b = MapARGB; + VP8LMapColor8b = MapAlpha; + VP8LPredictors[5] = Predictor5; + VP8LPredictors[6] = Predictor6; + VP8LPredictors[7] = Predictor7; + VP8LPredictors[8] = Predictor8; + VP8LPredictors[9] = Predictor9; + VP8LPredictors[10] = Predictor10; + VP8LPredictors[11] = Predictor11; + VP8LPredictors[12] = Predictor12; + VP8LPredictors[13] = Predictor13; + VP8LAddGreenToBlueAndRed = AddGreenToBlueAndRed; + VP8LTransformColorInverse = TransformColorInverse; + VP8LConvertBGRAToRGB = ConvertBGRAToRGB; + VP8LConvertBGRAToRGBA = ConvertBGRAToRGBA; + VP8LConvertBGRAToRGBA4444 = ConvertBGRAToRGBA4444; + VP8LConvertBGRAToRGB565 = ConvertBGRAToRGB565; + VP8LConvertBGRAToBGR = ConvertBGRAToBGR; +} + +#else // !WEBP_USE_MIPS_DSP_R2 + +WEBP_DSP_INIT_STUB(VP8LDspInitMIPSdspR2) + +#endif // WEBP_USE_MIPS_DSP_R2 diff --git a/thirdparty/libwebp/dsp/lossless_neon.c b/thirdparty/libwebp/dsp/lossless_neon.c new file mode 100644 index 0000000000..6faccb8f97 --- /dev/null +++ b/thirdparty/libwebp/dsp/lossless_neon.c @@ -0,0 +1,269 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// NEON variant of methods for lossless decoder +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_NEON) + +#include <arm_neon.h> + +#include "./lossless.h" +#include "./neon.h" + +//------------------------------------------------------------------------------ +// Colorspace conversion functions + +#if !defined(WORK_AROUND_GCC) +// gcc 4.6.0 had some trouble (NDK-r9) with this code. We only use it for +// gcc-4.8.x at least. +static void ConvertBGRAToRGBA(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const uint32_t* const end = src + (num_pixels & ~15); + for (; src < end; src += 16) { + uint8x16x4_t pixel = vld4q_u8((uint8_t*)src); + // swap B and R. (VSWP d0,d2 has no intrinsics equivalent!) + const uint8x16_t tmp = pixel.val[0]; + pixel.val[0] = pixel.val[2]; + pixel.val[2] = tmp; + vst4q_u8(dst, pixel); + dst += 64; + } + VP8LConvertBGRAToRGBA_C(src, num_pixels & 15, dst); // left-overs +} + +static void ConvertBGRAToBGR(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const uint32_t* const end = src + (num_pixels & ~15); + for (; src < end; src += 16) { + const uint8x16x4_t pixel = vld4q_u8((uint8_t*)src); + const uint8x16x3_t tmp = { { pixel.val[0], pixel.val[1], pixel.val[2] } }; + vst3q_u8(dst, tmp); + dst += 48; + } + VP8LConvertBGRAToBGR_C(src, num_pixels & 15, dst); // left-overs +} + +static void ConvertBGRAToRGB(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const uint32_t* const end = src + (num_pixels & ~15); + for (; src < end; src += 16) { + const uint8x16x4_t pixel = vld4q_u8((uint8_t*)src); + const uint8x16x3_t tmp = { { pixel.val[2], pixel.val[1], pixel.val[0] } }; + vst3q_u8(dst, tmp); + dst += 48; + } + VP8LConvertBGRAToRGB_C(src, num_pixels & 15, dst); // left-overs +} + +#else // WORK_AROUND_GCC + +// gcc-4.6.0 fallback + +static const uint8_t kRGBAShuffle[8] = { 2, 1, 0, 3, 6, 5, 4, 7 }; + +static void ConvertBGRAToRGBA(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const uint32_t* const end = src + (num_pixels & ~1); + const uint8x8_t shuffle = vld1_u8(kRGBAShuffle); + for (; src < end; src += 2) { + const uint8x8_t pixels = vld1_u8((uint8_t*)src); + vst1_u8(dst, vtbl1_u8(pixels, shuffle)); + dst += 8; + } + VP8LConvertBGRAToRGBA_C(src, num_pixels & 1, dst); // left-overs +} + +static const uint8_t kBGRShuffle[3][8] = { + { 0, 1, 2, 4, 5, 6, 8, 9 }, + { 10, 12, 13, 14, 16, 17, 18, 20 }, + { 21, 22, 24, 25, 26, 28, 29, 30 } +}; + +static void ConvertBGRAToBGR(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const uint32_t* const end = src + (num_pixels & ~7); + const uint8x8_t shuffle0 = vld1_u8(kBGRShuffle[0]); + const uint8x8_t shuffle1 = vld1_u8(kBGRShuffle[1]); + const uint8x8_t shuffle2 = vld1_u8(kBGRShuffle[2]); + for (; src < end; src += 8) { + uint8x8x4_t pixels; + INIT_VECTOR4(pixels, + vld1_u8((const uint8_t*)(src + 0)), + vld1_u8((const uint8_t*)(src + 2)), + vld1_u8((const uint8_t*)(src + 4)), + vld1_u8((const uint8_t*)(src + 6))); + vst1_u8(dst + 0, vtbl4_u8(pixels, shuffle0)); + vst1_u8(dst + 8, vtbl4_u8(pixels, shuffle1)); + vst1_u8(dst + 16, vtbl4_u8(pixels, shuffle2)); + dst += 8 * 3; + } + VP8LConvertBGRAToBGR_C(src, num_pixels & 7, dst); // left-overs +} + +static const uint8_t kRGBShuffle[3][8] = { + { 2, 1, 0, 6, 5, 4, 10, 9 }, + { 8, 14, 13, 12, 18, 17, 16, 22 }, + { 21, 20, 26, 25, 24, 30, 29, 28 } +}; + +static void ConvertBGRAToRGB(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const uint32_t* const end = src + (num_pixels & ~7); + const uint8x8_t shuffle0 = vld1_u8(kRGBShuffle[0]); + const uint8x8_t shuffle1 = vld1_u8(kRGBShuffle[1]); + const uint8x8_t shuffle2 = vld1_u8(kRGBShuffle[2]); + for (; src < end; src += 8) { + uint8x8x4_t pixels; + INIT_VECTOR4(pixels, + vld1_u8((const uint8_t*)(src + 0)), + vld1_u8((const uint8_t*)(src + 2)), + vld1_u8((const uint8_t*)(src + 4)), + vld1_u8((const uint8_t*)(src + 6))); + vst1_u8(dst + 0, vtbl4_u8(pixels, shuffle0)); + vst1_u8(dst + 8, vtbl4_u8(pixels, shuffle1)); + vst1_u8(dst + 16, vtbl4_u8(pixels, shuffle2)); + dst += 8 * 3; + } + VP8LConvertBGRAToRGB_C(src, num_pixels & 7, dst); // left-overs +} + +#endif // !WORK_AROUND_GCC + +//------------------------------------------------------------------------------ +// Subtract-Green Transform + +// vtbl?_u8 are marked unavailable for iOS arm64 with Xcode < 6.3, use +// non-standard versions there. +#if defined(__APPLE__) && defined(__aarch64__) && \ + defined(__apple_build_version__) && (__apple_build_version__< 6020037) +#define USE_VTBLQ +#endif + +#ifdef USE_VTBLQ +// 255 = byte will be zeroed +static const uint8_t kGreenShuffle[16] = { + 1, 255, 1, 255, 5, 255, 5, 255, 9, 255, 9, 255, 13, 255, 13, 255 +}; + +static WEBP_INLINE uint8x16_t DoGreenShuffle(const uint8x16_t argb, + const uint8x16_t shuffle) { + return vcombine_u8(vtbl1q_u8(argb, vget_low_u8(shuffle)), + vtbl1q_u8(argb, vget_high_u8(shuffle))); +} +#else // !USE_VTBLQ +// 255 = byte will be zeroed +static const uint8_t kGreenShuffle[8] = { 1, 255, 1, 255, 5, 255, 5, 255 }; + +static WEBP_INLINE uint8x16_t DoGreenShuffle(const uint8x16_t argb, + const uint8x8_t shuffle) { + return vcombine_u8(vtbl1_u8(vget_low_u8(argb), shuffle), + vtbl1_u8(vget_high_u8(argb), shuffle)); +} +#endif // USE_VTBLQ + +static void AddGreenToBlueAndRed(uint32_t* argb_data, int num_pixels) { + const uint32_t* const end = argb_data + (num_pixels & ~3); +#ifdef USE_VTBLQ + const uint8x16_t shuffle = vld1q_u8(kGreenShuffle); +#else + const uint8x8_t shuffle = vld1_u8(kGreenShuffle); +#endif + for (; argb_data < end; argb_data += 4) { + const uint8x16_t argb = vld1q_u8((uint8_t*)argb_data); + const uint8x16_t greens = DoGreenShuffle(argb, shuffle); + vst1q_u8((uint8_t*)argb_data, vaddq_u8(argb, greens)); + } + // fallthrough and finish off with plain-C + VP8LAddGreenToBlueAndRed_C(argb_data, num_pixels & 3); +} + +//------------------------------------------------------------------------------ +// Color Transform + +static void TransformColorInverse(const VP8LMultipliers* const m, + uint32_t* argb_data, int num_pixels) { + // sign-extended multiplying constants, pre-shifted by 6. +#define CST(X) (((int16_t)(m->X << 8)) >> 6) + const int16_t rb[8] = { + CST(green_to_blue_), CST(green_to_red_), + CST(green_to_blue_), CST(green_to_red_), + CST(green_to_blue_), CST(green_to_red_), + CST(green_to_blue_), CST(green_to_red_) + }; + const int16x8_t mults_rb = vld1q_s16(rb); + const int16_t b2[8] = { + 0, CST(red_to_blue_), 0, CST(red_to_blue_), + 0, CST(red_to_blue_), 0, CST(red_to_blue_), + }; + const int16x8_t mults_b2 = vld1q_s16(b2); +#undef CST +#ifdef USE_VTBLQ + static const uint8_t kg0g0[16] = { + 255, 1, 255, 1, 255, 5, 255, 5, 255, 9, 255, 9, 255, 13, 255, 13 + }; + const uint8x16_t shuffle = vld1q_u8(kg0g0); +#else + static const uint8_t k0g0g[8] = { 255, 1, 255, 1, 255, 5, 255, 5 }; + const uint8x8_t shuffle = vld1_u8(k0g0g); +#endif + const uint32x4_t mask_ag = vdupq_n_u32(0xff00ff00u); + int i; + for (i = 0; i + 4 <= num_pixels; i += 4) { + const uint8x16_t in = vld1q_u8((uint8_t*)(argb_data + i)); + const uint32x4_t a0g0 = vandq_u32(vreinterpretq_u32_u8(in), mask_ag); + // 0 g 0 g + const uint8x16_t greens = DoGreenShuffle(in, shuffle); + // x dr x db1 + const int16x8_t A = vqdmulhq_s16(vreinterpretq_s16_u8(greens), mults_rb); + // x r' x b' + const int8x16_t B = vaddq_s8(vreinterpretq_s8_u8(in), + vreinterpretq_s8_s16(A)); + // r' 0 b' 0 + const int16x8_t C = vshlq_n_s16(vreinterpretq_s16_s8(B), 8); + // x db2 0 0 + const int16x8_t D = vqdmulhq_s16(C, mults_b2); + // 0 x db2 0 + const uint32x4_t E = vshrq_n_u32(vreinterpretq_u32_s16(D), 8); + // r' x b'' 0 + const int8x16_t F = vaddq_s8(vreinterpretq_s8_u32(E), + vreinterpretq_s8_s16(C)); + // 0 r' 0 b'' + const uint16x8_t G = vshrq_n_u16(vreinterpretq_u16_s8(F), 8); + const uint32x4_t out = vorrq_u32(vreinterpretq_u32_u16(G), a0g0); + vst1q_u32(argb_data + i, out); + } + // Fall-back to C-version for left-overs. + VP8LTransformColorInverse_C(m, argb_data + i, num_pixels - i); +} + +#undef USE_VTBLQ + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8LDspInitNEON(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInitNEON(void) { + VP8LConvertBGRAToRGBA = ConvertBGRAToRGBA; + VP8LConvertBGRAToBGR = ConvertBGRAToBGR; + VP8LConvertBGRAToRGB = ConvertBGRAToRGB; + + VP8LAddGreenToBlueAndRed = AddGreenToBlueAndRed; + VP8LTransformColorInverse = TransformColorInverse; +} + +#else // !WEBP_USE_NEON + +WEBP_DSP_INIT_STUB(VP8LDspInitNEON) + +#endif // WEBP_USE_NEON diff --git a/thirdparty/libwebp/dsp/lossless_sse2.c b/thirdparty/libwebp/dsp/lossless_sse2.c new file mode 100644 index 0000000000..2d016c2911 --- /dev/null +++ b/thirdparty/libwebp/dsp/lossless_sse2.c @@ -0,0 +1,372 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE2 variant of methods for lossless decoder +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE2) +#include <assert.h> +#include <emmintrin.h> +#include "./lossless.h" + +//------------------------------------------------------------------------------ +// Predictor Transform + +static WEBP_INLINE uint32_t ClampedAddSubtractFull(uint32_t c0, uint32_t c1, + uint32_t c2) { + const __m128i zero = _mm_setzero_si128(); + const __m128i C0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c0), zero); + const __m128i C1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c1), zero); + const __m128i C2 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c2), zero); + const __m128i V1 = _mm_add_epi16(C0, C1); + const __m128i V2 = _mm_sub_epi16(V1, C2); + const __m128i b = _mm_packus_epi16(V2, V2); + const uint32_t output = _mm_cvtsi128_si32(b); + return output; +} + +static WEBP_INLINE uint32_t ClampedAddSubtractHalf(uint32_t c0, uint32_t c1, + uint32_t c2) { + const __m128i zero = _mm_setzero_si128(); + const __m128i C0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c0), zero); + const __m128i C1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c1), zero); + const __m128i B0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c2), zero); + const __m128i avg = _mm_add_epi16(C1, C0); + const __m128i A0 = _mm_srli_epi16(avg, 1); + const __m128i A1 = _mm_sub_epi16(A0, B0); + const __m128i BgtA = _mm_cmpgt_epi16(B0, A0); + const __m128i A2 = _mm_sub_epi16(A1, BgtA); + const __m128i A3 = _mm_srai_epi16(A2, 1); + const __m128i A4 = _mm_add_epi16(A0, A3); + const __m128i A5 = _mm_packus_epi16(A4, A4); + const uint32_t output = _mm_cvtsi128_si32(A5); + return output; +} + +static WEBP_INLINE uint32_t Select(uint32_t a, uint32_t b, uint32_t c) { + int pa_minus_pb; + const __m128i zero = _mm_setzero_si128(); + const __m128i A0 = _mm_cvtsi32_si128(a); + const __m128i B0 = _mm_cvtsi32_si128(b); + const __m128i C0 = _mm_cvtsi32_si128(c); + const __m128i AC0 = _mm_subs_epu8(A0, C0); + const __m128i CA0 = _mm_subs_epu8(C0, A0); + const __m128i BC0 = _mm_subs_epu8(B0, C0); + const __m128i CB0 = _mm_subs_epu8(C0, B0); + const __m128i AC = _mm_or_si128(AC0, CA0); + const __m128i BC = _mm_or_si128(BC0, CB0); + const __m128i pa = _mm_unpacklo_epi8(AC, zero); // |a - c| + const __m128i pb = _mm_unpacklo_epi8(BC, zero); // |b - c| + const __m128i diff = _mm_sub_epi16(pb, pa); + { + int16_t out[8]; + _mm_storeu_si128((__m128i*)out, diff); + pa_minus_pb = out[0] + out[1] + out[2] + out[3]; + } + return (pa_minus_pb <= 0) ? a : b; +} + +static WEBP_INLINE __m128i Average2_128i(uint32_t a0, uint32_t a1) { + const __m128i zero = _mm_setzero_si128(); + const __m128i A0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(a0), zero); + const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(a1), zero); + const __m128i sum = _mm_add_epi16(A1, A0); + const __m128i avg = _mm_srli_epi16(sum, 1); + return avg; +} + +static WEBP_INLINE uint32_t Average2(uint32_t a0, uint32_t a1) { + const __m128i avg = Average2_128i(a0, a1); + const __m128i A2 = _mm_packus_epi16(avg, avg); + const uint32_t output = _mm_cvtsi128_si32(A2); + return output; +} + +static WEBP_INLINE uint32_t Average3(uint32_t a0, uint32_t a1, uint32_t a2) { + const __m128i zero = _mm_setzero_si128(); + const __m128i avg1 = Average2_128i(a0, a2); + const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(a1), zero); + const __m128i sum = _mm_add_epi16(avg1, A1); + const __m128i avg2 = _mm_srli_epi16(sum, 1); + const __m128i A2 = _mm_packus_epi16(avg2, avg2); + const uint32_t output = _mm_cvtsi128_si32(A2); + return output; +} + +static WEBP_INLINE uint32_t Average4(uint32_t a0, uint32_t a1, + uint32_t a2, uint32_t a3) { + const __m128i avg1 = Average2_128i(a0, a1); + const __m128i avg2 = Average2_128i(a2, a3); + const __m128i sum = _mm_add_epi16(avg2, avg1); + const __m128i avg3 = _mm_srli_epi16(sum, 1); + const __m128i A0 = _mm_packus_epi16(avg3, avg3); + const uint32_t output = _mm_cvtsi128_si32(A0); + return output; +} + +static uint32_t Predictor5(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average3(left, top[0], top[1]); + return pred; +} +static uint32_t Predictor6(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average2(left, top[-1]); + return pred; +} +static uint32_t Predictor7(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average2(left, top[0]); + return pred; +} +static uint32_t Predictor8(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average2(top[-1], top[0]); + (void)left; + return pred; +} +static uint32_t Predictor9(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average2(top[0], top[1]); + (void)left; + return pred; +} +static uint32_t Predictor10(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average4(left, top[-1], top[0], top[1]); + return pred; +} +static uint32_t Predictor11(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Select(top[0], left, top[-1]); + return pred; +} +static uint32_t Predictor12(uint32_t left, const uint32_t* const top) { + const uint32_t pred = ClampedAddSubtractFull(left, top[0], top[-1]); + return pred; +} +static uint32_t Predictor13(uint32_t left, const uint32_t* const top) { + const uint32_t pred = ClampedAddSubtractHalf(left, top[0], top[-1]); + return pred; +} + +//------------------------------------------------------------------------------ +// Subtract-Green Transform + +static void AddGreenToBlueAndRed(uint32_t* argb_data, int num_pixels) { + int i; + for (i = 0; i + 4 <= num_pixels; i += 4) { + const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb + const __m128i A = _mm_srli_epi16(in, 8); // 0 a 0 g + const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0)); + const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // 0g0g + const __m128i out = _mm_add_epi8(in, C); + _mm_storeu_si128((__m128i*)&argb_data[i], out); + } + // fallthrough and finish off with plain-C + VP8LAddGreenToBlueAndRed_C(argb_data + i, num_pixels - i); +} + +//------------------------------------------------------------------------------ +// Color Transform + +static void TransformColorInverse(const VP8LMultipliers* const m, + uint32_t* argb_data, int num_pixels) { + // sign-extended multiplying constants, pre-shifted by 5. +#define CST(X) (((int16_t)(m->X << 8)) >> 5) // sign-extend + const __m128i mults_rb = _mm_set_epi16( + CST(green_to_red_), CST(green_to_blue_), + CST(green_to_red_), CST(green_to_blue_), + CST(green_to_red_), CST(green_to_blue_), + CST(green_to_red_), CST(green_to_blue_)); + const __m128i mults_b2 = _mm_set_epi16( + CST(red_to_blue_), 0, CST(red_to_blue_), 0, + CST(red_to_blue_), 0, CST(red_to_blue_), 0); +#undef CST + const __m128i mask_ag = _mm_set1_epi32(0xff00ff00); // alpha-green masks + int i; + for (i = 0; i + 4 <= num_pixels; i += 4) { + const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb + const __m128i A = _mm_and_si128(in, mask_ag); // a 0 g 0 + const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0)); + const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // g0g0 + const __m128i D = _mm_mulhi_epi16(C, mults_rb); // x dr x db1 + const __m128i E = _mm_add_epi8(in, D); // x r' x b' + const __m128i F = _mm_slli_epi16(E, 8); // r' 0 b' 0 + const __m128i G = _mm_mulhi_epi16(F, mults_b2); // x db2 0 0 + const __m128i H = _mm_srli_epi32(G, 8); // 0 x db2 0 + const __m128i I = _mm_add_epi8(H, F); // r' x b'' 0 + const __m128i J = _mm_srli_epi16(I, 8); // 0 r' 0 b'' + const __m128i out = _mm_or_si128(J, A); + _mm_storeu_si128((__m128i*)&argb_data[i], out); + } + // Fall-back to C-version for left-overs. + VP8LTransformColorInverse_C(m, argb_data + i, num_pixels - i); +} + +//------------------------------------------------------------------------------ +// Color-space conversion functions + +static void ConvertBGRAToRGBA(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const __m128i* in = (const __m128i*)src; + __m128i* out = (__m128i*)dst; + while (num_pixels >= 8) { + const __m128i bgra0 = _mm_loadu_si128(in++); // bgra0|bgra1|bgra2|bgra3 + const __m128i bgra4 = _mm_loadu_si128(in++); // bgra4|bgra5|bgra6|bgra7 + const __m128i v0l = _mm_unpacklo_epi8(bgra0, bgra4); // b0b4g0g4r0r4a0a4... + const __m128i v0h = _mm_unpackhi_epi8(bgra0, bgra4); // b2b6g2g6r2r6a2a6... + const __m128i v1l = _mm_unpacklo_epi8(v0l, v0h); // b0b2b4b6g0g2g4g6... + const __m128i v1h = _mm_unpackhi_epi8(v0l, v0h); // b1b3b5b7g1g3g5g7... + const __m128i v2l = _mm_unpacklo_epi8(v1l, v1h); // b0...b7 | g0...g7 + const __m128i v2h = _mm_unpackhi_epi8(v1l, v1h); // r0...r7 | a0...a7 + const __m128i ga0 = _mm_unpackhi_epi64(v2l, v2h); // g0...g7 | a0...a7 + const __m128i rb0 = _mm_unpacklo_epi64(v2h, v2l); // r0...r7 | b0...b7 + const __m128i rg0 = _mm_unpacklo_epi8(rb0, ga0); // r0g0r1g1 ... r6g6r7g7 + const __m128i ba0 = _mm_unpackhi_epi8(rb0, ga0); // b0a0b1a1 ... b6a6b7a7 + const __m128i rgba0 = _mm_unpacklo_epi16(rg0, ba0); // rgba0|rgba1... + const __m128i rgba4 = _mm_unpackhi_epi16(rg0, ba0); // rgba4|rgba5... + _mm_storeu_si128(out++, rgba0); + _mm_storeu_si128(out++, rgba4); + num_pixels -= 8; + } + // left-overs + VP8LConvertBGRAToRGBA_C((const uint32_t*)in, num_pixels, (uint8_t*)out); +} + +static void ConvertBGRAToRGBA4444(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const __m128i mask_0x0f = _mm_set1_epi8(0x0f); + const __m128i mask_0xf0 = _mm_set1_epi8(0xf0); + const __m128i* in = (const __m128i*)src; + __m128i* out = (__m128i*)dst; + while (num_pixels >= 8) { + const __m128i bgra0 = _mm_loadu_si128(in++); // bgra0|bgra1|bgra2|bgra3 + const __m128i bgra4 = _mm_loadu_si128(in++); // bgra4|bgra5|bgra6|bgra7 + const __m128i v0l = _mm_unpacklo_epi8(bgra0, bgra4); // b0b4g0g4r0r4a0a4... + const __m128i v0h = _mm_unpackhi_epi8(bgra0, bgra4); // b2b6g2g6r2r6a2a6... + const __m128i v1l = _mm_unpacklo_epi8(v0l, v0h); // b0b2b4b6g0g2g4g6... + const __m128i v1h = _mm_unpackhi_epi8(v0l, v0h); // b1b3b5b7g1g3g5g7... + const __m128i v2l = _mm_unpacklo_epi8(v1l, v1h); // b0...b7 | g0...g7 + const __m128i v2h = _mm_unpackhi_epi8(v1l, v1h); // r0...r7 | a0...a7 + const __m128i ga0 = _mm_unpackhi_epi64(v2l, v2h); // g0...g7 | a0...a7 + const __m128i rb0 = _mm_unpacklo_epi64(v2h, v2l); // r0...r7 | b0...b7 + const __m128i ga1 = _mm_srli_epi16(ga0, 4); // g0-|g1-|...|a6-|a7- + const __m128i rb1 = _mm_and_si128(rb0, mask_0xf0); // -r0|-r1|...|-b6|-a7 + const __m128i ga2 = _mm_and_si128(ga1, mask_0x0f); // g0-|g1-|...|a6-|a7- + const __m128i rgba0 = _mm_or_si128(ga2, rb1); // rg0..rg7 | ba0..ba7 + const __m128i rgba1 = _mm_srli_si128(rgba0, 8); // ba0..ba7 | 0 +#ifdef WEBP_SWAP_16BIT_CSP + const __m128i rgba = _mm_unpacklo_epi8(rgba1, rgba0); // barg0...barg7 +#else + const __m128i rgba = _mm_unpacklo_epi8(rgba0, rgba1); // rgba0...rgba7 +#endif + _mm_storeu_si128(out++, rgba); + num_pixels -= 8; + } + // left-overs + VP8LConvertBGRAToRGBA4444_C((const uint32_t*)in, num_pixels, (uint8_t*)out); +} + +static void ConvertBGRAToRGB565(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const __m128i mask_0xe0 = _mm_set1_epi8(0xe0); + const __m128i mask_0xf8 = _mm_set1_epi8(0xf8); + const __m128i mask_0x07 = _mm_set1_epi8(0x07); + const __m128i* in = (const __m128i*)src; + __m128i* out = (__m128i*)dst; + while (num_pixels >= 8) { + const __m128i bgra0 = _mm_loadu_si128(in++); // bgra0|bgra1|bgra2|bgra3 + const __m128i bgra4 = _mm_loadu_si128(in++); // bgra4|bgra5|bgra6|bgra7 + const __m128i v0l = _mm_unpacklo_epi8(bgra0, bgra4); // b0b4g0g4r0r4a0a4... + const __m128i v0h = _mm_unpackhi_epi8(bgra0, bgra4); // b2b6g2g6r2r6a2a6... + const __m128i v1l = _mm_unpacklo_epi8(v0l, v0h); // b0b2b4b6g0g2g4g6... + const __m128i v1h = _mm_unpackhi_epi8(v0l, v0h); // b1b3b5b7g1g3g5g7... + const __m128i v2l = _mm_unpacklo_epi8(v1l, v1h); // b0...b7 | g0...g7 + const __m128i v2h = _mm_unpackhi_epi8(v1l, v1h); // r0...r7 | a0...a7 + const __m128i ga0 = _mm_unpackhi_epi64(v2l, v2h); // g0...g7 | a0...a7 + const __m128i rb0 = _mm_unpacklo_epi64(v2h, v2l); // r0...r7 | b0...b7 + const __m128i rb1 = _mm_and_si128(rb0, mask_0xf8); // -r0..-r7|-b0..-b7 + const __m128i g_lo1 = _mm_srli_epi16(ga0, 5); + const __m128i g_lo2 = _mm_and_si128(g_lo1, mask_0x07); // g0-...g7-|xx (3b) + const __m128i g_hi1 = _mm_slli_epi16(ga0, 3); + const __m128i g_hi2 = _mm_and_si128(g_hi1, mask_0xe0); // -g0...-g7|xx (3b) + const __m128i b0 = _mm_srli_si128(rb1, 8); // -b0...-b7|0 + const __m128i rg1 = _mm_or_si128(rb1, g_lo2); // gr0...gr7|xx + const __m128i b1 = _mm_srli_epi16(b0, 3); + const __m128i gb1 = _mm_or_si128(b1, g_hi2); // bg0...bg7|xx +#ifdef WEBP_SWAP_16BIT_CSP + const __m128i rgba = _mm_unpacklo_epi8(gb1, rg1); // rggb0...rggb7 +#else + const __m128i rgba = _mm_unpacklo_epi8(rg1, gb1); // bgrb0...bgrb7 +#endif + _mm_storeu_si128(out++, rgba); + num_pixels -= 8; + } + // left-overs + VP8LConvertBGRAToRGB565_C((const uint32_t*)in, num_pixels, (uint8_t*)out); +} + +static void ConvertBGRAToBGR(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const __m128i mask_l = _mm_set_epi32(0, 0x00ffffff, 0, 0x00ffffff); + const __m128i mask_h = _mm_set_epi32(0x00ffffff, 0, 0x00ffffff, 0); + const __m128i* in = (const __m128i*)src; + const uint8_t* const end = dst + num_pixels * 3; + // the last storel_epi64 below writes 8 bytes starting at offset 18 + while (dst + 26 <= end) { + const __m128i bgra0 = _mm_loadu_si128(in++); // bgra0|bgra1|bgra2|bgra3 + const __m128i bgra4 = _mm_loadu_si128(in++); // bgra4|bgra5|bgra6|bgra7 + const __m128i a0l = _mm_and_si128(bgra0, mask_l); // bgr0|0|bgr0|0 + const __m128i a4l = _mm_and_si128(bgra4, mask_l); // bgr0|0|bgr0|0 + const __m128i a0h = _mm_and_si128(bgra0, mask_h); // 0|bgr0|0|bgr0 + const __m128i a4h = _mm_and_si128(bgra4, mask_h); // 0|bgr0|0|bgr0 + const __m128i b0h = _mm_srli_epi64(a0h, 8); // 000b|gr00|000b|gr00 + const __m128i b4h = _mm_srli_epi64(a4h, 8); // 000b|gr00|000b|gr00 + const __m128i c0 = _mm_or_si128(a0l, b0h); // rgbrgb00|rgbrgb00 + const __m128i c4 = _mm_or_si128(a4l, b4h); // rgbrgb00|rgbrgb00 + const __m128i c2 = _mm_srli_si128(c0, 8); + const __m128i c6 = _mm_srli_si128(c4, 8); + _mm_storel_epi64((__m128i*)(dst + 0), c0); + _mm_storel_epi64((__m128i*)(dst + 6), c2); + _mm_storel_epi64((__m128i*)(dst + 12), c4); + _mm_storel_epi64((__m128i*)(dst + 18), c6); + dst += 24; + num_pixels -= 8; + } + // left-overs + VP8LConvertBGRAToBGR_C((const uint32_t*)in, num_pixels, dst); +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8LDspInitSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInitSSE2(void) { + VP8LPredictors[5] = Predictor5; + VP8LPredictors[6] = Predictor6; + VP8LPredictors[7] = Predictor7; + VP8LPredictors[8] = Predictor8; + VP8LPredictors[9] = Predictor9; + VP8LPredictors[10] = Predictor10; + VP8LPredictors[11] = Predictor11; + VP8LPredictors[12] = Predictor12; + VP8LPredictors[13] = Predictor13; + + VP8LAddGreenToBlueAndRed = AddGreenToBlueAndRed; + VP8LTransformColorInverse = TransformColorInverse; + + VP8LConvertBGRAToRGBA = ConvertBGRAToRGBA; + VP8LConvertBGRAToRGBA4444 = ConvertBGRAToRGBA4444; + VP8LConvertBGRAToRGB565 = ConvertBGRAToRGB565; + VP8LConvertBGRAToBGR = ConvertBGRAToBGR; +} + +#else // !WEBP_USE_SSE2 + +WEBP_DSP_INIT_STUB(VP8LDspInitSSE2) + +#endif // WEBP_USE_SSE2 diff --git a/thirdparty/libwebp/dsp/mips_macro.h b/thirdparty/libwebp/dsp/mips_macro.h new file mode 100644 index 0000000000..44aba9b71d --- /dev/null +++ b/thirdparty/libwebp/dsp/mips_macro.h @@ -0,0 +1,200 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MIPS common macros + +#ifndef WEBP_DSP_MIPS_MACRO_H_ +#define WEBP_DSP_MIPS_MACRO_H_ + +#if defined(__GNUC__) && defined(__ANDROID__) && LOCAL_GCC_VERSION == 0x409 +#define WORK_AROUND_GCC +#endif + +#define STR(s) #s +#define XSTR(s) STR(s) + +// O0[31..16 | 15..0] = I0[31..16 | 15..0] + I1[31..16 | 15..0] +// O1[31..16 | 15..0] = I0[31..16 | 15..0] - I1[31..16 | 15..0] +// O - output +// I - input (macro doesn't change it) +#define ADD_SUB_HALVES(O0, O1, \ + I0, I1) \ + "addq.ph %[" #O0 "], %[" #I0 "], %[" #I1 "] \n\t" \ + "subq.ph %[" #O1 "], %[" #I0 "], %[" #I1 "] \n\t" + +// O - output +// I - input (macro doesn't change it) +// I[0/1] - offset in bytes +#define LOAD_IN_X2(O0, O1, \ + I0, I1) \ + "lh %[" #O0 "], " #I0 "(%[in]) \n\t" \ + "lh %[" #O1 "], " #I1 "(%[in]) \n\t" + +// I0 - location +// I1..I9 - offsets in bytes +#define LOAD_WITH_OFFSET_X4(O0, O1, O2, O3, \ + I0, I1, I2, I3, I4, I5, I6, I7, I8, I9) \ + "ulw %[" #O0 "], " #I1 "+" XSTR(I9) "*" #I5 "(%[" #I0 "]) \n\t" \ + "ulw %[" #O1 "], " #I2 "+" XSTR(I9) "*" #I6 "(%[" #I0 "]) \n\t" \ + "ulw %[" #O2 "], " #I3 "+" XSTR(I9) "*" #I7 "(%[" #I0 "]) \n\t" \ + "ulw %[" #O3 "], " #I4 "+" XSTR(I9) "*" #I8 "(%[" #I0 "]) \n\t" + +// O - output +// IO - input/output +// I - input (macro doesn't change it) +#define MUL_SHIFT_SUM(O0, O1, O2, O3, O4, O5, O6, O7, \ + IO0, IO1, IO2, IO3, \ + I0, I1, I2, I3, I4, I5, I6, I7) \ + "mul %[" #O0 "], %[" #I0 "], %[kC2] \n\t" \ + "mul %[" #O1 "], %[" #I0 "], %[kC1] \n\t" \ + "mul %[" #O2 "], %[" #I1 "], %[kC2] \n\t" \ + "mul %[" #O3 "], %[" #I1 "], %[kC1] \n\t" \ + "mul %[" #O4 "], %[" #I2 "], %[kC2] \n\t" \ + "mul %[" #O5 "], %[" #I2 "], %[kC1] \n\t" \ + "mul %[" #O6 "], %[" #I3 "], %[kC2] \n\t" \ + "mul %[" #O7 "], %[" #I3 "], %[kC1] \n\t" \ + "sra %[" #O0 "], %[" #O0 "], 16 \n\t" \ + "sra %[" #O1 "], %[" #O1 "], 16 \n\t" \ + "sra %[" #O2 "], %[" #O2 "], 16 \n\t" \ + "sra %[" #O3 "], %[" #O3 "], 16 \n\t" \ + "sra %[" #O4 "], %[" #O4 "], 16 \n\t" \ + "sra %[" #O5 "], %[" #O5 "], 16 \n\t" \ + "sra %[" #O6 "], %[" #O6 "], 16 \n\t" \ + "sra %[" #O7 "], %[" #O7 "], 16 \n\t" \ + "addu %[" #IO0 "], %[" #IO0 "], %[" #I4 "] \n\t" \ + "addu %[" #IO1 "], %[" #IO1 "], %[" #I5 "] \n\t" \ + "subu %[" #IO2 "], %[" #IO2 "], %[" #I6 "] \n\t" \ + "subu %[" #IO3 "], %[" #IO3 "], %[" #I7 "] \n\t" + +// O - output +// I - input (macro doesn't change it) +#define INSERT_HALF_X2(O0, O1, \ + I0, I1) \ + "ins %[" #O0 "], %[" #I0 "], 16, 16 \n\t" \ + "ins %[" #O1 "], %[" #I1 "], 16, 16 \n\t" + +// O - output +// I - input (macro doesn't change it) +#define SRA_16(O0, O1, O2, O3, \ + I0, I1, I2, I3) \ + "sra %[" #O0 "], %[" #I0 "], 16 \n\t" \ + "sra %[" #O1 "], %[" #I1 "], 16 \n\t" \ + "sra %[" #O2 "], %[" #I2 "], 16 \n\t" \ + "sra %[" #O3 "], %[" #I3 "], 16 \n\t" + +// temp0[31..16 | 15..0] = temp8[31..16 | 15..0] + temp12[31..16 | 15..0] +// temp1[31..16 | 15..0] = temp8[31..16 | 15..0] - temp12[31..16 | 15..0] +// temp0[31..16 | 15..0] = temp0[31..16 >> 3 | 15..0 >> 3] +// temp1[31..16 | 15..0] = temp1[31..16 >> 3 | 15..0 >> 3] +// O - output +// I - input (macro doesn't change it) +#define SHIFT_R_SUM_X2(O0, O1, O2, O3, O4, O5, O6, O7, \ + I0, I1, I2, I3, I4, I5, I6, I7) \ + "addq.ph %[" #O0 "], %[" #I0 "], %[" #I4 "] \n\t" \ + "subq.ph %[" #O1 "], %[" #I0 "], %[" #I4 "] \n\t" \ + "addq.ph %[" #O2 "], %[" #I1 "], %[" #I5 "] \n\t" \ + "subq.ph %[" #O3 "], %[" #I1 "], %[" #I5 "] \n\t" \ + "addq.ph %[" #O4 "], %[" #I2 "], %[" #I6 "] \n\t" \ + "subq.ph %[" #O5 "], %[" #I2 "], %[" #I6 "] \n\t" \ + "addq.ph %[" #O6 "], %[" #I3 "], %[" #I7 "] \n\t" \ + "subq.ph %[" #O7 "], %[" #I3 "], %[" #I7 "] \n\t" \ + "shra.ph %[" #O0 "], %[" #O0 "], 3 \n\t" \ + "shra.ph %[" #O1 "], %[" #O1 "], 3 \n\t" \ + "shra.ph %[" #O2 "], %[" #O2 "], 3 \n\t" \ + "shra.ph %[" #O3 "], %[" #O3 "], 3 \n\t" \ + "shra.ph %[" #O4 "], %[" #O4 "], 3 \n\t" \ + "shra.ph %[" #O5 "], %[" #O5 "], 3 \n\t" \ + "shra.ph %[" #O6 "], %[" #O6 "], 3 \n\t" \ + "shra.ph %[" #O7 "], %[" #O7 "], 3 \n\t" + +// precrq.ph.w temp0, temp8, temp2 +// temp0 = temp8[31..16] | temp2[31..16] +// ins temp2, temp8, 16, 16 +// temp2 = temp8[31..16] | temp2[15..0] +// O - output +// IO - input/output +// I - input (macro doesn't change it) +#define PACK_2_HALVES_TO_WORD(O0, O1, O2, O3, \ + IO0, IO1, IO2, IO3, \ + I0, I1, I2, I3) \ + "precrq.ph.w %[" #O0 "], %[" #I0 "], %[" #IO0 "] \n\t" \ + "precrq.ph.w %[" #O1 "], %[" #I1 "], %[" #IO1 "] \n\t" \ + "ins %[" #IO0 "], %[" #I0 "], 16, 16 \n\t" \ + "ins %[" #IO1 "], %[" #I1 "], 16, 16 \n\t" \ + "precrq.ph.w %[" #O2 "], %[" #I2 "], %[" #IO2 "] \n\t" \ + "precrq.ph.w %[" #O3 "], %[" #I3 "], %[" #IO3 "] \n\t" \ + "ins %[" #IO2 "], %[" #I2 "], 16, 16 \n\t" \ + "ins %[" #IO3 "], %[" #I3 "], 16, 16 \n\t" + +// preceu.ph.qbr temp0, temp8 +// temp0 = 0 | 0 | temp8[23..16] | temp8[7..0] +// preceu.ph.qbl temp1, temp8 +// temp1 = temp8[23..16] | temp8[7..0] | 0 | 0 +// O - output +// I - input (macro doesn't change it) +#define CONVERT_2_BYTES_TO_HALF(O0, O1, O2, O3, O4, O5, O6, O7, \ + I0, I1, I2, I3) \ + "preceu.ph.qbr %[" #O0 "], %[" #I0 "] \n\t" \ + "preceu.ph.qbl %[" #O1 "], %[" #I0 "] \n\t" \ + "preceu.ph.qbr %[" #O2 "], %[" #I1 "] \n\t" \ + "preceu.ph.qbl %[" #O3 "], %[" #I1 "] \n\t" \ + "preceu.ph.qbr %[" #O4 "], %[" #I2 "] \n\t" \ + "preceu.ph.qbl %[" #O5 "], %[" #I2 "] \n\t" \ + "preceu.ph.qbr %[" #O6 "], %[" #I3 "] \n\t" \ + "preceu.ph.qbl %[" #O7 "], %[" #I3 "] \n\t" + +// temp0[31..16 | 15..0] = temp0[31..16 | 15..0] + temp8[31..16 | 15..0] +// temp0[31..16 | 15..0] = temp0[31..16 <<(s) 7 | 15..0 <<(s) 7] +// temp1..temp7 same as temp0 +// precrqu_s.qb.ph temp0, temp1, temp0: +// temp0 = temp1[31..24] | temp1[15..8] | temp0[31..24] | temp0[15..8] +// store temp0 to dst +// IO - input/output +// I - input (macro doesn't change it) +#define STORE_SAT_SUM_X2(IO0, IO1, IO2, IO3, IO4, IO5, IO6, IO7, \ + I0, I1, I2, I3, I4, I5, I6, I7, \ + I8, I9, I10, I11, I12, I13) \ + "addq.ph %[" #IO0 "], %[" #IO0 "], %[" #I0 "] \n\t" \ + "addq.ph %[" #IO1 "], %[" #IO1 "], %[" #I1 "] \n\t" \ + "addq.ph %[" #IO2 "], %[" #IO2 "], %[" #I2 "] \n\t" \ + "addq.ph %[" #IO3 "], %[" #IO3 "], %[" #I3 "] \n\t" \ + "addq.ph %[" #IO4 "], %[" #IO4 "], %[" #I4 "] \n\t" \ + "addq.ph %[" #IO5 "], %[" #IO5 "], %[" #I5 "] \n\t" \ + "addq.ph %[" #IO6 "], %[" #IO6 "], %[" #I6 "] \n\t" \ + "addq.ph %[" #IO7 "], %[" #IO7 "], %[" #I7 "] \n\t" \ + "shll_s.ph %[" #IO0 "], %[" #IO0 "], 7 \n\t" \ + "shll_s.ph %[" #IO1 "], %[" #IO1 "], 7 \n\t" \ + "shll_s.ph %[" #IO2 "], %[" #IO2 "], 7 \n\t" \ + "shll_s.ph %[" #IO3 "], %[" #IO3 "], 7 \n\t" \ + "shll_s.ph %[" #IO4 "], %[" #IO4 "], 7 \n\t" \ + "shll_s.ph %[" #IO5 "], %[" #IO5 "], 7 \n\t" \ + "shll_s.ph %[" #IO6 "], %[" #IO6 "], 7 \n\t" \ + "shll_s.ph %[" #IO7 "], %[" #IO7 "], 7 \n\t" \ + "precrqu_s.qb.ph %[" #IO0 "], %[" #IO1 "], %[" #IO0 "] \n\t" \ + "precrqu_s.qb.ph %[" #IO2 "], %[" #IO3 "], %[" #IO2 "] \n\t" \ + "precrqu_s.qb.ph %[" #IO4 "], %[" #IO5 "], %[" #IO4 "] \n\t" \ + "precrqu_s.qb.ph %[" #IO6 "], %[" #IO7 "], %[" #IO6 "] \n\t" \ + "usw %[" #IO0 "], " XSTR(I13) "*" #I9 "(%[" #I8 "]) \n\t" \ + "usw %[" #IO2 "], " XSTR(I13) "*" #I10 "(%[" #I8 "]) \n\t" \ + "usw %[" #IO4 "], " XSTR(I13) "*" #I11 "(%[" #I8 "]) \n\t" \ + "usw %[" #IO6 "], " XSTR(I13) "*" #I12 "(%[" #I8 "]) \n\t" + +#define OUTPUT_EARLY_CLOBBER_REGS_10() \ + : [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), \ + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [temp6]"=&r"(temp6), \ + [temp7]"=&r"(temp7), [temp8]"=&r"(temp8), [temp9]"=&r"(temp9), \ + [temp10]"=&r"(temp10) + +#define OUTPUT_EARLY_CLOBBER_REGS_18() \ + OUTPUT_EARLY_CLOBBER_REGS_10(), \ + [temp11]"=&r"(temp11), [temp12]"=&r"(temp12), [temp13]"=&r"(temp13), \ + [temp14]"=&r"(temp14), [temp15]"=&r"(temp15), [temp16]"=&r"(temp16), \ + [temp17]"=&r"(temp17), [temp18]"=&r"(temp18) + +#endif // WEBP_DSP_MIPS_MACRO_H_ diff --git a/thirdparty/libwebp/dsp/msa_macro.h b/thirdparty/libwebp/dsp/msa_macro.h new file mode 100644 index 0000000000..5c707f476a --- /dev/null +++ b/thirdparty/libwebp/dsp/msa_macro.h @@ -0,0 +1,555 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MSA common macros +// +// Author(s): Prashant Patil (prashant.patil@imgtec.com) + +#ifndef WEBP_DSP_MSA_MACRO_H_ +#define WEBP_DSP_MSA_MACRO_H_ + +#include <stdint.h> +#include <msa.h> + +#if defined(__clang__) + #define CLANG_BUILD +#endif + +#ifdef CLANG_BUILD + #define ADDVI_H(a, b) __msa_addvi_h((v8i16)a, b) + #define SRAI_H(a, b) __msa_srai_h((v8i16)a, b) + #define SRAI_W(a, b) __msa_srai_w((v4i32)a, b) +#else + #define ADDVI_H(a, b) (a + b) + #define SRAI_H(a, b) (a >> b) + #define SRAI_W(a, b) (a >> b) +#endif + +#define LD_B(RTYPE, psrc) *((RTYPE*)(psrc)) +#define LD_UB(...) LD_B(v16u8, __VA_ARGS__) +#define LD_SB(...) LD_B(v16i8, __VA_ARGS__) + +#define LD_H(RTYPE, psrc) *((RTYPE*)(psrc)) +#define LD_UH(...) LD_H(v8u16, __VA_ARGS__) +#define LD_SH(...) LD_H(v8i16, __VA_ARGS__) + +#define LD_W(RTYPE, psrc) *((RTYPE*)(psrc)) +#define LD_UW(...) LD_W(v4u32, __VA_ARGS__) +#define LD_SW(...) LD_W(v4i32, __VA_ARGS__) + +#define ST_B(RTYPE, in, pdst) *((RTYPE*)(pdst)) = in +#define ST_UB(...) ST_B(v16u8, __VA_ARGS__) +#define ST_SB(...) ST_B(v16i8, __VA_ARGS__) + +#define ST_H(RTYPE, in, pdst) *((RTYPE*)(pdst)) = in +#define ST_UH(...) ST_H(v8u16, __VA_ARGS__) +#define ST_SH(...) ST_H(v8i16, __VA_ARGS__) + +#define ST_W(RTYPE, in, pdst) *((RTYPE*)(pdst)) = in +#define ST_UW(...) ST_W(v4u32, __VA_ARGS__) +#define ST_SW(...) ST_W(v4i32, __VA_ARGS__) + +#define MSA_LOAD_FUNC(TYPE, INSTR, FUNC_NAME) \ + static inline TYPE FUNC_NAME(const void* const psrc) { \ + const uint8_t* const psrc_m = (const uint8_t*)psrc; \ + TYPE val_m; \ + asm volatile ( \ + "" #INSTR " %[val_m], %[psrc_m] \n\t" \ + : [val_m] "=r" (val_m) \ + : [psrc_m] "m" (*psrc_m)); \ + return val_m; \ + } + +#define MSA_LOAD(psrc, FUNC_NAME) FUNC_NAME(psrc) + +#define MSA_STORE_FUNC(TYPE, INSTR, FUNC_NAME) \ + static inline void FUNC_NAME(TYPE val, void* const pdst) { \ + uint8_t* const pdst_m = (uint8_t*)pdst; \ + TYPE val_m = val; \ + asm volatile ( \ + " " #INSTR " %[val_m], %[pdst_m] \n\t" \ + : [pdst_m] "=m" (*pdst_m) \ + : [val_m] "r" (val_m)); \ + } + +#define MSA_STORE(val, pdst, FUNC_NAME) FUNC_NAME(val, pdst) + +#if (__mips_isa_rev >= 6) + MSA_LOAD_FUNC(uint16_t, lh, msa_lh); + #define LH(psrc) MSA_LOAD(psrc, msa_lh) + MSA_LOAD_FUNC(uint32_t, lw, msa_lw); + #define LW(psrc) MSA_LOAD(psrc, msa_lw) + #if (__mips == 64) + MSA_LOAD_FUNC(uint64_t, ld, msa_ld); + #define LD(psrc) MSA_LOAD(psrc, msa_ld) + #else // !(__mips == 64) + #define LD(psrc) ((((uint64_t)MSA_LOAD(psrc + 4, msa_lw)) << 32) | \ + MSA_LOAD(psrc, msa_lw)) + #endif // (__mips == 64) + + MSA_STORE_FUNC(uint16_t, sh, msa_sh); + #define SH(val, pdst) MSA_STORE(val, pdst, msa_sh) + MSA_STORE_FUNC(uint32_t, sw, msa_sw); + #define SW(val, pdst) MSA_STORE(val, pdst, msa_sw) + MSA_STORE_FUNC(uint64_t, sd, msa_sd); + #define SD(val, pdst) MSA_STORE(val, pdst, msa_sd) +#else // !(__mips_isa_rev >= 6) + MSA_LOAD_FUNC(uint16_t, ulh, msa_ulh); + #define LH(psrc) MSA_LOAD(psrc, msa_ulh) + MSA_LOAD_FUNC(uint32_t, ulw, msa_ulw); + #define LW(psrc) MSA_LOAD(psrc, msa_ulw) + #if (__mips == 64) + MSA_LOAD_FUNC(uint64_t, uld, msa_uld); + #define LD(psrc) MSA_LOAD(psrc, msa_uld) + #else // !(__mips == 64) + #define LD(psrc) ((((uint64_t)MSA_LOAD(psrc + 4, msa_ulw)) << 32) | \ + MSA_LOAD(psrc, msa_ulw)) + #endif // (__mips == 64) + + MSA_STORE_FUNC(uint16_t, ush, msa_ush); + #define SH(val, pdst) MSA_STORE(val, pdst, msa_ush) + MSA_STORE_FUNC(uint32_t, usw, msa_usw); + #define SW(val, pdst) MSA_STORE(val, pdst, msa_usw) + #define SD(val, pdst) { \ + uint8_t* const pdst_sd_m = (uint8_t*)(pdst); \ + const uint32_t val0_m = (uint32_t)(val & 0x00000000FFFFFFFF); \ + const uint32_t val1_m = (uint32_t)((val >> 32) & 0x00000000FFFFFFFF); \ + SW(val0_m, pdst_sd_m); \ + SW(val1_m, pdst_sd_m + 4); \ + } +#endif // (__mips_isa_rev >= 6) + +/* Description : Load 4 words with stride + * Arguments : Inputs - psrc, stride + * Outputs - out0, out1, out2, out3 + * Details : Load word in 'out0' from (psrc) + * Load word in 'out1' from (psrc + stride) + * Load word in 'out2' from (psrc + 2 * stride) + * Load word in 'out3' from (psrc + 3 * stride) + */ +#define LW4(psrc, stride, out0, out1, out2, out3) { \ + const uint8_t* ptmp = (const uint8_t*)psrc; \ + out0 = LW(ptmp); \ + ptmp += stride; \ + out1 = LW(ptmp); \ + ptmp += stride; \ + out2 = LW(ptmp); \ + ptmp += stride; \ + out3 = LW(ptmp); \ +} + +/* Description : Store 4 words with stride + * Arguments : Inputs - in0, in1, in2, in3, pdst, stride + * Details : Store word from 'in0' to (pdst) + * Store word from 'in1' to (pdst + stride) + * Store word from 'in2' to (pdst + 2 * stride) + * Store word from 'in3' to (pdst + 3 * stride) + */ +#define SW4(in0, in1, in2, in3, pdst, stride) { \ + uint8_t* ptmp = (uint8_t*)pdst; \ + SW(in0, ptmp); \ + ptmp += stride; \ + SW(in1, ptmp); \ + ptmp += stride; \ + SW(in2, ptmp); \ + ptmp += stride; \ + SW(in3, ptmp); \ +} + +/* Description : Load vectors with 16 byte elements with stride + * Arguments : Inputs - psrc, stride + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Load 16 byte elements in 'out0' from (psrc) + * Load 16 byte elements in 'out1' from (psrc + stride) + */ +#define LD_B2(RTYPE, psrc, stride, out0, out1) { \ + out0 = LD_B(RTYPE, psrc); \ + out1 = LD_B(RTYPE, psrc + stride); \ +} +#define LD_UB2(...) LD_B2(v16u8, __VA_ARGS__) +#define LD_SB2(...) LD_B2(v16i8, __VA_ARGS__) + +#define LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3) { \ + LD_B2(RTYPE, psrc, stride, out0, out1); \ + LD_B2(RTYPE, psrc + 2 * stride , stride, out2, out3); \ +} +#define LD_UB4(...) LD_B4(v16u8, __VA_ARGS__) +#define LD_SB4(...) LD_B4(v16i8, __VA_ARGS__) + +/* Description : Load vectors with 8 halfword elements with stride + * Arguments : Inputs - psrc, stride + * Outputs - out0, out1 + * Details : Load 8 halfword elements in 'out0' from (psrc) + * Load 8 halfword elements in 'out1' from (psrc + stride) + */ +#define LD_H2(RTYPE, psrc, stride, out0, out1) { \ + out0 = LD_H(RTYPE, psrc); \ + out1 = LD_H(RTYPE, psrc + stride); \ +} +#define LD_UH2(...) LD_H2(v8u16, __VA_ARGS__) +#define LD_SH2(...) LD_H2(v8i16, __VA_ARGS__) + +/* Description : Store 4x4 byte block to destination memory from input vector + * Arguments : Inputs - in0, in1, pdst, stride + * Details : 'Idx0' word element from input vector 'in0' is copied to the + * GP register and stored to (pdst) + * 'Idx1' word element from input vector 'in0' is copied to the + * GP register and stored to (pdst + stride) + * 'Idx2' word element from input vector 'in0' is copied to the + * GP register and stored to (pdst + 2 * stride) + * 'Idx3' word element from input vector 'in0' is copied to the + * GP register and stored to (pdst + 3 * stride) + */ +#define ST4x4_UB(in0, in1, idx0, idx1, idx2, idx3, pdst, stride) { \ + uint8_t* const pblk_4x4_m = (uint8_t*)pdst; \ + const uint32_t out0_m = __msa_copy_s_w((v4i32)in0, idx0); \ + const uint32_t out1_m = __msa_copy_s_w((v4i32)in0, idx1); \ + const uint32_t out2_m = __msa_copy_s_w((v4i32)in1, idx2); \ + const uint32_t out3_m = __msa_copy_s_w((v4i32)in1, idx3); \ + SW4(out0_m, out1_m, out2_m, out3_m, pblk_4x4_m, stride); \ +} + +/* Description : Immediate number of elements to slide + * Arguments : Inputs - in0, in1, slide_val + * Outputs - out + * Return Type - as per RTYPE + * Details : Byte elements from 'in1' vector are slid into 'in0' by + * value specified in the 'slide_val' + */ +#define SLDI_B(RTYPE, in0, in1, slide_val) \ + (RTYPE)__msa_sldi_b((v16i8)in0, (v16i8)in1, slide_val) \ + +#define SLDI_UB(...) SLDI_B(v16u8, __VA_ARGS__) +#define SLDI_SB(...) SLDI_B(v16i8, __VA_ARGS__) +#define SLDI_SH(...) SLDI_B(v8i16, __VA_ARGS__) + +/* Description : Shuffle halfword vector elements as per mask vector + * Arguments : Inputs - in0, in1, in2, in3, mask0, mask1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : halfword elements from 'in0' & 'in1' are copied selectively to + * 'out0' as per control vector 'mask0' + */ +#define VSHF_H2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) { \ + out0 = (RTYPE)__msa_vshf_h((v8i16)mask0, (v8i16)in1, (v8i16)in0); \ + out1 = (RTYPE)__msa_vshf_h((v8i16)mask1, (v8i16)in3, (v8i16)in2); \ +} +#define VSHF_H2_UH(...) VSHF_H2(v8u16, __VA_ARGS__) +#define VSHF_H2_SH(...) VSHF_H2(v8i16, __VA_ARGS__) + +/* Description : Clips all signed halfword elements of input vector + * between 0 & 255 + * Arguments : Input/output - val + * Return Type - signed halfword + */ +#define CLIP_SH_0_255(val) { \ + const v8i16 max_m = __msa_ldi_h(255); \ + val = __msa_maxi_s_h((v8i16)val, 0); \ + val = __msa_min_s_h(max_m, (v8i16)val); \ +} +#define CLIP_SH2_0_255(in0, in1) { \ + CLIP_SH_0_255(in0); \ + CLIP_SH_0_255(in1); \ +} + +/* Description : Clips all signed word elements of input vector + * between 0 & 255 + * Arguments : Input/output - val + * Return Type - signed word + */ +#define CLIP_SW_0_255(val) { \ + const v4i32 max_m = __msa_ldi_w(255); \ + val = __msa_maxi_s_w((v4i32)val, 0); \ + val = __msa_min_s_w(max_m, (v4i32)val); \ +} +#define CLIP_SW4_0_255(in0, in1, in2, in3) { \ + CLIP_SW_0_255(in0); \ + CLIP_SW_0_255(in1); \ + CLIP_SW_0_255(in2); \ + CLIP_SW_0_255(in3); \ +} + +/* Description : Set element n input vector to GPR value + * Arguments : Inputs - in0, in1, in2, in3 + * Output - out + * Return Type - as per RTYPE + * Details : Set element 0 in vector 'out' to value specified in 'in0' + */ +#define INSERT_W2(RTYPE, in0, in1, out) { \ + out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0); \ + out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1); \ +} +#define INSERT_W2_UB(...) INSERT_W2(v16u8, __VA_ARGS__) +#define INSERT_W2_SB(...) INSERT_W2(v16i8, __VA_ARGS__) + +#define INSERT_W4(RTYPE, in0, in1, in2, in3, out) { \ + out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0); \ + out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1); \ + out = (RTYPE)__msa_insert_w((v4i32)out, 2, in2); \ + out = (RTYPE)__msa_insert_w((v4i32)out, 3, in3); \ +} +#define INSERT_W4_UB(...) INSERT_W4(v16u8, __VA_ARGS__) +#define INSERT_W4_SB(...) INSERT_W4(v16i8, __VA_ARGS__) +#define INSERT_W4_SW(...) INSERT_W4(v4i32, __VA_ARGS__) + +/* Description : Interleave right half of byte elements from vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Right half of byte elements of 'in0' and 'in1' are interleaved + * and written to out0. + */ +#define ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1) { \ + out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1); \ + out1 = (RTYPE)__msa_ilvr_b((v16i8)in2, (v16i8)in3); \ +} +#define ILVR_B2_UB(...) ILVR_B2(v16u8, __VA_ARGS__) +#define ILVR_B2_SB(...) ILVR_B2(v16i8, __VA_ARGS__) +#define ILVR_B2_UH(...) ILVR_B2(v8u16, __VA_ARGS__) +#define ILVR_B2_SH(...) ILVR_B2(v8i16, __VA_ARGS__) +#define ILVR_B2_SW(...) ILVR_B2(v4i32, __VA_ARGS__) + +#define ILVR_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3) { \ + ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ + ILVR_B2(RTYPE, in4, in5, in6, in7, out2, out3); \ +} +#define ILVR_B4_UB(...) ILVR_B4(v16u8, __VA_ARGS__) +#define ILVR_B4_SB(...) ILVR_B4(v16i8, __VA_ARGS__) +#define ILVR_B4_UH(...) ILVR_B4(v8u16, __VA_ARGS__) +#define ILVR_B4_SH(...) ILVR_B4(v8i16, __VA_ARGS__) +#define ILVR_B4_SW(...) ILVR_B4(v4i32, __VA_ARGS__) + +/* Description : Interleave right half of halfword elements from vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Right half of halfword elements of 'in0' and 'in1' are + * interleaved and written to 'out0'. + */ +#define ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1) { \ + out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1); \ + out1 = (RTYPE)__msa_ilvr_h((v8i16)in2, (v8i16)in3); \ +} +#define ILVR_H2_UB(...) ILVR_H2(v16u8, __VA_ARGS__) +#define ILVR_H2_SH(...) ILVR_H2(v8i16, __VA_ARGS__) +#define ILVR_H2_SW(...) ILVR_H2(v4i32, __VA_ARGS__) + +#define ILVR_H4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3) { \ + ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1); \ + ILVR_H2(RTYPE, in4, in5, in6, in7, out2, out3); \ +} +#define ILVR_H4_UB(...) ILVR_H4(v16u8, __VA_ARGS__) +#define ILVR_H4_SH(...) ILVR_H4(v8i16, __VA_ARGS__) +#define ILVR_H4_SW(...) ILVR_H4(v4i32, __VA_ARGS__) + +/* Description : Interleave right half of double word elements from vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Right half of double word elements of 'in0' and 'in1' are + * interleaved and written to 'out0'. + */ +#define ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1) { \ + out0 = (RTYPE)__msa_ilvr_d((v2i64)in0, (v2i64)in1); \ + out1 = (RTYPE)__msa_ilvr_d((v2i64)in2, (v2i64)in3); \ +} +#define ILVR_D2_UB(...) ILVR_D2(v16u8, __VA_ARGS__) +#define ILVR_D2_SB(...) ILVR_D2(v16i8, __VA_ARGS__) +#define ILVR_D2_SH(...) ILVR_D2(v8i16, __VA_ARGS__) + +#define ILVRL_H2(RTYPE, in0, in1, out0, out1) { \ + out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1); \ + out1 = (RTYPE)__msa_ilvl_h((v8i16)in0, (v8i16)in1); \ +} +#define ILVRL_H2_UB(...) ILVRL_H2(v16u8, __VA_ARGS__) +#define ILVRL_H2_SB(...) ILVRL_H2(v16i8, __VA_ARGS__) +#define ILVRL_H2_SH(...) ILVRL_H2(v8i16, __VA_ARGS__) +#define ILVRL_H2_SW(...) ILVRL_H2(v4i32, __VA_ARGS__) +#define ILVRL_H2_UW(...) ILVRL_H2(v4u32, __VA_ARGS__) + +#define ILVRL_W2(RTYPE, in0, in1, out0, out1) { \ + out0 = (RTYPE)__msa_ilvr_w((v4i32)in0, (v4i32)in1); \ + out1 = (RTYPE)__msa_ilvl_w((v4i32)in0, (v4i32)in1); \ +} +#define ILVRL_W2_UB(...) ILVRL_W2(v16u8, __VA_ARGS__) +#define ILVRL_W2_SH(...) ILVRL_W2(v8i16, __VA_ARGS__) +#define ILVRL_W2_SW(...) ILVRL_W2(v4i32, __VA_ARGS__) + +/* Description : Pack even byte elements of vector pairs + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Even byte elements of 'in0' are copied to the left half of + * 'out0' & even byte elements of 'in1' are copied to the right + * half of 'out0'. + */ +#define PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) { \ + out0 = (RTYPE)__msa_pckev_b((v16i8)in0, (v16i8)in1); \ + out1 = (RTYPE)__msa_pckev_b((v16i8)in2, (v16i8)in3); \ +} +#define PCKEV_B2_SB(...) PCKEV_B2(v16i8, __VA_ARGS__) +#define PCKEV_B2_UB(...) PCKEV_B2(v16u8, __VA_ARGS__) +#define PCKEV_B2_SH(...) PCKEV_B2(v8i16, __VA_ARGS__) +#define PCKEV_B2_SW(...) PCKEV_B2(v4i32, __VA_ARGS__) + +/* Description : Arithmetic immediate shift right all elements of word vector + * Arguments : Inputs - in0, in1, shift + * Outputs - in place operation + * Return Type - as per input vector RTYPE + * Details : Each element of vector 'in0' is right shifted by 'shift' and + * the result is written in-place. 'shift' is a GP variable. + */ +#define SRAI_W2(RTYPE, in0, in1, shift_val) { \ + in0 = (RTYPE)SRAI_W(in0, shift_val); \ + in1 = (RTYPE)SRAI_W(in1, shift_val); \ +} +#define SRAI_W2_SW(...) SRAI_W2(v4i32, __VA_ARGS__) +#define SRAI_W2_UW(...) SRAI_W2(v4u32, __VA_ARGS__) + +#define SRAI_W4(RTYPE, in0, in1, in2, in3, shift_val) { \ + SRAI_W2(RTYPE, in0, in1, shift_val); \ + SRAI_W2(RTYPE, in2, in3, shift_val); \ +} +#define SRAI_W4_SW(...) SRAI_W4(v4i32, __VA_ARGS__) +#define SRAI_W4_UW(...) SRAI_W4(v4u32, __VA_ARGS__) + +/* Description : Arithmetic shift right all elements of half-word vector + * Arguments : Inputs - in0, in1, shift + * Outputs - in place operation + * Return Type - as per input vector RTYPE + * Details : Each element of vector 'in0' is right shifted by 'shift' and + * the result is written in-place. 'shift' is a GP variable. + */ +#define SRAI_H2(RTYPE, in0, in1, shift_val) { \ + in0 = (RTYPE)SRAI_H(in0, shift_val); \ + in1 = (RTYPE)SRAI_H(in1, shift_val); \ +} +#define SRAI_H2_SH(...) SRAI_H2(v8i16, __VA_ARGS__) +#define SRAI_H2_UH(...) SRAI_H2(v8u16, __VA_ARGS__) + +/* Description : Arithmetic rounded shift right all elements of word vector + * Arguments : Inputs - in0, in1, shift + * Outputs - in place operation + * Return Type - as per input vector RTYPE + * Details : Each element of vector 'in0' is right shifted by 'shift' and + * the result is written in-place. 'shift' is a GP variable. + */ +#define SRARI_W2(RTYPE, in0, in1, shift) { \ + in0 = (RTYPE)__msa_srari_w((v4i32)in0, shift); \ + in1 = (RTYPE)__msa_srari_w((v4i32)in1, shift); \ +} +#define SRARI_W2_SW(...) SRARI_W2(v4i32, __VA_ARGS__) + +#define SRARI_W4(RTYPE, in0, in1, in2, in3, shift) { \ + SRARI_W2(RTYPE, in0, in1, shift); \ + SRARI_W2(RTYPE, in2, in3, shift); \ +} +#define SRARI_W4_SH(...) SRARI_W4(v8i16, __VA_ARGS__) +#define SRARI_W4_UW(...) SRARI_W4(v4u32, __VA_ARGS__) +#define SRARI_W4_SW(...) SRARI_W4(v4i32, __VA_ARGS__) + +/* Description : Addition of 2 pairs of half-word vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Details : Each element in 'in0' is added to 'in1' and result is written + * to 'out0'. + */ +#define ADDVI_H2(RTYPE, in0, in1, in2, in3, out0, out1) { \ + out0 = (RTYPE)ADDVI_H(in0, in1); \ + out1 = (RTYPE)ADDVI_H(in2, in3); \ +} +#define ADDVI_H2_SH(...) ADDVI_H2(v8i16, __VA_ARGS__) +#define ADDVI_H2_UH(...) ADDVI_H2(v8u16, __VA_ARGS__) + +/* Description : Addition of 2 pairs of vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Details : Each element in 'in0' is added to 'in1' and result is written + * to 'out0'. + */ +#define ADD2(in0, in1, in2, in3, out0, out1) { \ + out0 = in0 + in1; \ + out1 = in2 + in3; \ +} +#define ADD4(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3) { \ + ADD2(in0, in1, in2, in3, out0, out1); \ + ADD2(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : Sign extend halfword elements from input vector and return + * the result in pair of vectors + * Arguments : Input - in (halfword vector) + * Outputs - out0, out1 (sign extended word vectors) + * Return Type - signed word + * Details : Sign bit of halfword elements from input vector 'in' is + * extracted and interleaved right with same vector 'in0' to + * generate 4 signed word elements in 'out0' + * Then interleaved left with same vector 'in0' to + * generate 4 signed word elements in 'out1' + */ +#define UNPCK_SH_SW(in, out0, out1) { \ + const v8i16 tmp_m = __msa_clti_s_h((v8i16)in, 0); \ + ILVRL_H2_SW(tmp_m, in, out0, out1); \ +} + +/* Description : Butterfly of 4 input vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1, out2, out3 + * Details : Butterfly operation + */ +#define BUTTERFLY_4(in0, in1, in2, in3, out0, out1, out2, out3) { \ + out0 = in0 + in3; \ + out1 = in1 + in2; \ + out2 = in1 - in2; \ + out3 = in0 - in3; \ +} + +/* Description : Transpose 4x4 block with word elements in vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1, out2, out3 + * Return Type - as per RTYPE + */ +#define TRANSPOSE4x4_W(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) { \ + v4i32 s0_m, s1_m, s2_m, s3_m; \ + ILVRL_W2_SW(in1, in0, s0_m, s1_m); \ + ILVRL_W2_SW(in3, in2, s2_m, s3_m); \ + out0 = (RTYPE)__msa_ilvr_d((v2i64)s2_m, (v2i64)s0_m); \ + out1 = (RTYPE)__msa_ilvl_d((v2i64)s2_m, (v2i64)s0_m); \ + out2 = (RTYPE)__msa_ilvr_d((v2i64)s3_m, (v2i64)s1_m); \ + out3 = (RTYPE)__msa_ilvl_d((v2i64)s3_m, (v2i64)s1_m); \ +} +#define TRANSPOSE4x4_SW_SW(...) TRANSPOSE4x4_W(v4i32, __VA_ARGS__) + +/* Description : Add block 4x4 + * Arguments : Inputs - in0, in1, in2, in3, pdst, stride + * Details : Least significant 4 bytes from each input vector are added to + * the destination bytes, clipped between 0-255 and stored. + */ +#define ADDBLK_ST4x4_UB(in0, in1, in2, in3, pdst, stride) { \ + uint32_t src0_m, src1_m, src2_m, src3_m; \ + v8i16 inp0_m, inp1_m, res0_m, res1_m; \ + v16i8 dst0_m = { 0 }; \ + v16i8 dst1_m = { 0 }; \ + const v16i8 zero_m = { 0 }; \ + ILVR_D2_SH(in1, in0, in3, in2, inp0_m, inp1_m); \ + LW4(pdst, stride, src0_m, src1_m, src2_m, src3_m); \ + INSERT_W2_SB(src0_m, src1_m, dst0_m); \ + INSERT_W2_SB(src2_m, src3_m, dst1_m); \ + ILVR_B2_SH(zero_m, dst0_m, zero_m, dst1_m, res0_m, res1_m); \ + ADD2(res0_m, inp0_m, res1_m, inp1_m, res0_m, res1_m); \ + CLIP_SH2_0_255(res0_m, res1_m); \ + PCKEV_B2_SB(res0_m, res0_m, res1_m, res1_m, dst0_m, dst1_m); \ + ST4x4_UB(dst0_m, dst1_m, 0, 1, 0, 1, pdst, stride); \ +} + +#endif /* WEBP_DSP_MSA_MACRO_H_ */ diff --git a/thirdparty/libwebp/dsp/neon.h b/thirdparty/libwebp/dsp/neon.h new file mode 100644 index 0000000000..0a06266848 --- /dev/null +++ b/thirdparty/libwebp/dsp/neon.h @@ -0,0 +1,82 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// NEON common code. + +#ifndef WEBP_DSP_NEON_H_ +#define WEBP_DSP_NEON_H_ + +#include <arm_neon.h> + +#include "./dsp.h" + +// Right now, some intrinsics functions seem slower, so we disable them +// everywhere except aarch64 where the inline assembly is incompatible. +#if defined(__aarch64__) +#define WEBP_USE_INTRINSICS // use intrinsics when possible +#endif + +#define INIT_VECTOR2(v, a, b) do { \ + v.val[0] = a; \ + v.val[1] = b; \ +} while (0) + +#define INIT_VECTOR3(v, a, b, c) do { \ + v.val[0] = a; \ + v.val[1] = b; \ + v.val[2] = c; \ +} while (0) + +#define INIT_VECTOR4(v, a, b, c, d) do { \ + v.val[0] = a; \ + v.val[1] = b; \ + v.val[2] = c; \ + v.val[3] = d; \ +} while (0) + +// if using intrinsics, this flag avoids some functions that make gcc-4.6.3 +// crash ("internal compiler error: in immed_double_const, at emit-rtl."). +// (probably similar to gcc.gnu.org/bugzilla/show_bug.cgi?id=48183) +#if !(LOCAL_GCC_PREREQ(4,8) || defined(__aarch64__)) +#define WORK_AROUND_GCC +#endif + +static WEBP_INLINE int32x4x4_t Transpose4x4(const int32x4x4_t rows) { + uint64x2x2_t row01, row23; + + row01.val[0] = vreinterpretq_u64_s32(rows.val[0]); + row01.val[1] = vreinterpretq_u64_s32(rows.val[1]); + row23.val[0] = vreinterpretq_u64_s32(rows.val[2]); + row23.val[1] = vreinterpretq_u64_s32(rows.val[3]); + // Transpose 64-bit values (there's no vswp equivalent) + { + const uint64x1_t row0h = vget_high_u64(row01.val[0]); + const uint64x1_t row2l = vget_low_u64(row23.val[0]); + const uint64x1_t row1h = vget_high_u64(row01.val[1]); + const uint64x1_t row3l = vget_low_u64(row23.val[1]); + row01.val[0] = vcombine_u64(vget_low_u64(row01.val[0]), row2l); + row23.val[0] = vcombine_u64(row0h, vget_high_u64(row23.val[0])); + row01.val[1] = vcombine_u64(vget_low_u64(row01.val[1]), row3l); + row23.val[1] = vcombine_u64(row1h, vget_high_u64(row23.val[1])); + } + { + const int32x4x2_t out01 = vtrnq_s32(vreinterpretq_s32_u64(row01.val[0]), + vreinterpretq_s32_u64(row01.val[1])); + const int32x4x2_t out23 = vtrnq_s32(vreinterpretq_s32_u64(row23.val[0]), + vreinterpretq_s32_u64(row23.val[1])); + int32x4x4_t out; + out.val[0] = out01.val[0]; + out.val[1] = out01.val[1]; + out.val[2] = out23.val[0]; + out.val[3] = out23.val[1]; + return out; + } +} + +#endif // WEBP_DSP_NEON_H_ diff --git a/thirdparty/libwebp/dsp/rescaler.c b/thirdparty/libwebp/dsp/rescaler.c new file mode 100644 index 0000000000..bc743d5dc5 --- /dev/null +++ b/thirdparty/libwebp/dsp/rescaler.c @@ -0,0 +1,238 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Rescaling functions +// +// Author: Skal (pascal.massimino@gmail.com) + +#include <assert.h> + +#include "./dsp.h" +#include "../utils/rescaler.h" + +//------------------------------------------------------------------------------ +// Implementations of critical functions ImportRow / ExportRow + +#define ROUNDER (WEBP_RESCALER_ONE >> 1) +#define MULT_FIX(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX) + +//------------------------------------------------------------------------------ +// Row import + +void WebPRescalerImportRowExpandC(WebPRescaler* const wrk, const uint8_t* src) { + const int x_stride = wrk->num_channels; + const int x_out_max = wrk->dst_width * wrk->num_channels; + int channel; + assert(!WebPRescalerInputDone(wrk)); + assert(wrk->x_expand); + for (channel = 0; channel < x_stride; ++channel) { + int x_in = channel; + int x_out = channel; + // simple bilinear interpolation + int accum = wrk->x_add; + int left = src[x_in]; + int right = (wrk->src_width > 1) ? src[x_in + x_stride] : left; + x_in += x_stride; + while (1) { + wrk->frow[x_out] = right * wrk->x_add + (left - right) * accum; + x_out += x_stride; + if (x_out >= x_out_max) break; + accum -= wrk->x_sub; + if (accum < 0) { + left = right; + x_in += x_stride; + assert(x_in < wrk->src_width * x_stride); + right = src[x_in]; + accum += wrk->x_add; + } + } + assert(wrk->x_sub == 0 /* <- special case for src_width=1 */ || accum == 0); + } +} + +void WebPRescalerImportRowShrinkC(WebPRescaler* const wrk, const uint8_t* src) { + const int x_stride = wrk->num_channels; + const int x_out_max = wrk->dst_width * wrk->num_channels; + int channel; + assert(!WebPRescalerInputDone(wrk)); + assert(!wrk->x_expand); + for (channel = 0; channel < x_stride; ++channel) { + int x_in = channel; + int x_out = channel; + uint32_t sum = 0; + int accum = 0; + while (x_out < x_out_max) { + uint32_t base = 0; + accum += wrk->x_add; + while (accum > 0) { + accum -= wrk->x_sub; + assert(x_in < wrk->src_width * x_stride); + base = src[x_in]; + sum += base; + x_in += x_stride; + } + { // Emit next horizontal pixel. + const rescaler_t frac = base * (-accum); + wrk->frow[x_out] = sum * wrk->x_sub - frac; + // fresh fractional start for next pixel + sum = (int)MULT_FIX(frac, wrk->fx_scale); + } + x_out += x_stride; + } + assert(accum == 0); + } +} + +//------------------------------------------------------------------------------ +// Row export + +void WebPRescalerExportRowExpandC(WebPRescaler* const wrk) { + int x_out; + uint8_t* const dst = wrk->dst; + rescaler_t* const irow = wrk->irow; + const int x_out_max = wrk->dst_width * wrk->num_channels; + const rescaler_t* const frow = wrk->frow; + assert(!WebPRescalerOutputDone(wrk)); + assert(wrk->y_accum <= 0); + assert(wrk->y_expand); + assert(wrk->y_sub != 0); + if (wrk->y_accum == 0) { + for (x_out = 0; x_out < x_out_max; ++x_out) { + const uint32_t J = frow[x_out]; + const int v = (int)MULT_FIX(J, wrk->fy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + } + } else { + const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); + const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B); + for (x_out = 0; x_out < x_out_max; ++x_out) { + const uint64_t I = (uint64_t)A * frow[x_out] + + (uint64_t)B * irow[x_out]; + const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); + const int v = (int)MULT_FIX(J, wrk->fy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + } + } +} + +void WebPRescalerExportRowShrinkC(WebPRescaler* const wrk) { + int x_out; + uint8_t* const dst = wrk->dst; + rescaler_t* const irow = wrk->irow; + const int x_out_max = wrk->dst_width * wrk->num_channels; + const rescaler_t* const frow = wrk->frow; + const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum); + assert(!WebPRescalerOutputDone(wrk)); + assert(wrk->y_accum <= 0); + assert(!wrk->y_expand); + if (yscale) { + for (x_out = 0; x_out < x_out_max; ++x_out) { + const uint32_t frac = (uint32_t)MULT_FIX(frow[x_out], yscale); + const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + irow[x_out] = frac; // new fractional start + } + } else { + for (x_out = 0; x_out < x_out_max; ++x_out) { + const int v = (int)MULT_FIX(irow[x_out], wrk->fxy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + irow[x_out] = 0; + } + } +} + +#undef MULT_FIX +#undef ROUNDER + +//------------------------------------------------------------------------------ +// Main entry calls + +void WebPRescalerImportRow(WebPRescaler* const wrk, const uint8_t* src) { + assert(!WebPRescalerInputDone(wrk)); + if (!wrk->x_expand) { + WebPRescalerImportRowShrink(wrk, src); + } else { + WebPRescalerImportRowExpand(wrk, src); + } +} + +void WebPRescalerExportRow(WebPRescaler* const wrk) { + if (wrk->y_accum <= 0) { + assert(!WebPRescalerOutputDone(wrk)); + if (wrk->y_expand) { + WebPRescalerExportRowExpand(wrk); + } else if (wrk->fxy_scale) { + WebPRescalerExportRowShrink(wrk); + } else { // very special case for src = dst = 1x1 + int i; + assert(wrk->src_width == 1 && wrk->dst_width <= 2); + assert(wrk->src_height == 1 && wrk->dst_height == 1); + for (i = 0; i < wrk->num_channels * wrk->dst_width; ++i) { + wrk->dst[i] = wrk->irow[i]; + wrk->irow[i] = 0; + } + } + wrk->y_accum += wrk->y_add; + wrk->dst += wrk->dst_stride; + ++wrk->dst_y; + } +} + +//------------------------------------------------------------------------------ + +WebPRescalerImportRowFunc WebPRescalerImportRowExpand; +WebPRescalerImportRowFunc WebPRescalerImportRowShrink; + +WebPRescalerExportRowFunc WebPRescalerExportRowExpand; +WebPRescalerExportRowFunc WebPRescalerExportRowShrink; + +extern void WebPRescalerDspInitSSE2(void); +extern void WebPRescalerDspInitMIPS32(void); +extern void WebPRescalerDspInitMIPSdspR2(void); +extern void WebPRescalerDspInitNEON(void); + +static volatile VP8CPUInfo rescaler_last_cpuinfo_used = + (VP8CPUInfo)&rescaler_last_cpuinfo_used; + +WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInit(void) { + if (rescaler_last_cpuinfo_used == VP8GetCPUInfo) return; + + WebPRescalerImportRowExpand = WebPRescalerImportRowExpandC; + WebPRescalerImportRowShrink = WebPRescalerImportRowShrinkC; + WebPRescalerExportRowExpand = WebPRescalerExportRowExpandC; + WebPRescalerExportRowShrink = WebPRescalerExportRowShrinkC; + + if (VP8GetCPUInfo != NULL) { +#if defined(WEBP_USE_SSE2) + if (VP8GetCPUInfo(kSSE2)) { + WebPRescalerDspInitSSE2(); + } +#endif +#if defined(WEBP_USE_NEON) + if (VP8GetCPUInfo(kNEON)) { + WebPRescalerDspInitNEON(); + } +#endif +#if defined(WEBP_USE_MIPS32) + if (VP8GetCPUInfo(kMIPS32)) { + WebPRescalerDspInitMIPS32(); + } +#endif +#if defined(WEBP_USE_MIPS_DSP_R2) + if (VP8GetCPUInfo(kMIPSdspR2)) { + WebPRescalerDspInitMIPSdspR2(); + } +#endif + } + rescaler_last_cpuinfo_used = VP8GetCPUInfo; +} diff --git a/thirdparty/libwebp/dsp/rescaler_mips32.c b/thirdparty/libwebp/dsp/rescaler_mips32.c new file mode 100644 index 0000000000..ddaa391336 --- /dev/null +++ b/thirdparty/libwebp/dsp/rescaler_mips32.c @@ -0,0 +1,291 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MIPS version of rescaling functions +// +// Author(s): Djordje Pesut (djordje.pesut@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS32) + +#include <assert.h> +#include "../utils/rescaler.h" + +//------------------------------------------------------------------------------ +// Row import + +static void ImportRowShrink(WebPRescaler* const wrk, const uint8_t* src) { + const int x_stride = wrk->num_channels; + const int x_out_max = wrk->dst_width * wrk->num_channels; + const int fx_scale = wrk->fx_scale; + const int x_add = wrk->x_add; + const int x_sub = wrk->x_sub; + const int x_stride1 = x_stride << 2; + int channel; + assert(!wrk->x_expand); + assert(!WebPRescalerInputDone(wrk)); + + for (channel = 0; channel < x_stride; ++channel) { + const uint8_t* src1 = src + channel; + rescaler_t* frow = wrk->frow + channel; + int temp1, temp2, temp3; + int base, frac, sum; + int accum, accum1; + int loop_c = x_out_max - channel; + + __asm__ volatile ( + "li %[temp1], 0x8000 \n\t" + "li %[temp2], 0x10000 \n\t" + "li %[sum], 0 \n\t" + "li %[accum], 0 \n\t" + "1: \n\t" + "addu %[accum], %[accum], %[x_add] \n\t" + "li %[base], 0 \n\t" + "blez %[accum], 3f \n\t" + "2: \n\t" + "lbu %[base], 0(%[src1]) \n\t" + "subu %[accum], %[accum], %[x_sub] \n\t" + "addu %[src1], %[src1], %[x_stride] \n\t" + "addu %[sum], %[sum], %[base] \n\t" + "bgtz %[accum], 2b \n\t" + "3: \n\t" + "negu %[accum1], %[accum] \n\t" + "mul %[frac], %[base], %[accum1] \n\t" + "mul %[temp3], %[sum], %[x_sub] \n\t" + "subu %[loop_c], %[loop_c], %[x_stride] \n\t" + "mult %[temp1], %[temp2] \n\t" + "maddu %[frac], %[fx_scale] \n\t" + "mfhi %[sum] \n\t" + "subu %[temp3], %[temp3], %[frac] \n\t" + "sw %[temp3], 0(%[frow]) \n\t" + "addu %[frow], %[frow], %[x_stride1] \n\t" + "bgtz %[loop_c], 1b \n\t" + : [accum]"=&r"(accum), [src1]"+r"(src1), [temp3]"=&r"(temp3), + [sum]"=&r"(sum), [base]"=&r"(base), [frac]"=&r"(frac), + [frow]"+r"(frow), [accum1]"=&r"(accum1), + [temp2]"=&r"(temp2), [temp1]"=&r"(temp1) + : [x_stride]"r"(x_stride), [fx_scale]"r"(fx_scale), + [x_sub]"r"(x_sub), [x_add]"r"(x_add), + [loop_c]"r"(loop_c), [x_stride1]"r"(x_stride1) + : "memory", "hi", "lo" + ); + assert(accum == 0); + } +} + +static void ImportRowExpand(WebPRescaler* const wrk, const uint8_t* src) { + const int x_stride = wrk->num_channels; + const int x_out_max = wrk->dst_width * wrk->num_channels; + const int x_add = wrk->x_add; + const int x_sub = wrk->x_sub; + const int src_width = wrk->src_width; + const int x_stride1 = x_stride << 2; + int channel; + assert(wrk->x_expand); + assert(!WebPRescalerInputDone(wrk)); + + for (channel = 0; channel < x_stride; ++channel) { + const uint8_t* src1 = src + channel; + rescaler_t* frow = wrk->frow + channel; + int temp1, temp2, temp3, temp4; + int frac; + int accum; + int x_out = channel; + + __asm__ volatile ( + "addiu %[temp3], %[src_width], -1 \n\t" + "lbu %[temp2], 0(%[src1]) \n\t" + "addu %[src1], %[src1], %[x_stride] \n\t" + "bgtz %[temp3], 0f \n\t" + "addiu %[temp1], %[temp2], 0 \n\t" + "b 3f \n\t" + "0: \n\t" + "lbu %[temp1], 0(%[src1]) \n\t" + "3: \n\t" + "addiu %[accum], %[x_add], 0 \n\t" + "1: \n\t" + "subu %[temp3], %[temp2], %[temp1] \n\t" + "mul %[temp3], %[temp3], %[accum] \n\t" + "mul %[temp4], %[temp1], %[x_add] \n\t" + "addu %[temp3], %[temp4], %[temp3] \n\t" + "sw %[temp3], 0(%[frow]) \n\t" + "addu %[frow], %[frow], %[x_stride1] \n\t" + "addu %[x_out], %[x_out], %[x_stride] \n\t" + "subu %[temp3], %[x_out], %[x_out_max] \n\t" + "bgez %[temp3], 2f \n\t" + "subu %[accum], %[accum], %[x_sub] \n\t" + "bgez %[accum], 4f \n\t" + "addiu %[temp2], %[temp1], 0 \n\t" + "addu %[src1], %[src1], %[x_stride] \n\t" + "lbu %[temp1], 0(%[src1]) \n\t" + "addu %[accum], %[accum], %[x_add] \n\t" + "4: \n\t" + "b 1b \n\t" + "2: \n\t" + : [src1]"+r"(src1), [accum]"=&r"(accum), [temp1]"=&r"(temp1), + [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), + [x_out]"+r"(x_out), [frac]"=&r"(frac), [frow]"+r"(frow) + : [x_stride]"r"(x_stride), [x_add]"r"(x_add), [x_sub]"r"(x_sub), + [x_stride1]"r"(x_stride1), [src_width]"r"(src_width), + [x_out_max]"r"(x_out_max) + : "memory", "hi", "lo" + ); + assert(wrk->x_sub == 0 /* <- special case for src_width=1 */ || accum == 0); + } +} + +//------------------------------------------------------------------------------ +// Row export + +static void ExportRowExpand(WebPRescaler* const wrk) { + uint8_t* dst = wrk->dst; + rescaler_t* irow = wrk->irow; + const int x_out_max = wrk->dst_width * wrk->num_channels; + const rescaler_t* frow = wrk->frow; + int temp0, temp1, temp3, temp4, temp5, loop_end; + const int temp2 = (int)wrk->fy_scale; + const int temp6 = x_out_max << 2; + assert(!WebPRescalerOutputDone(wrk)); + assert(wrk->y_accum <= 0); + assert(wrk->y_expand); + assert(wrk->y_sub != 0); + if (wrk->y_accum == 0) { + __asm__ volatile ( + "li %[temp3], 0x10000 \n\t" + "li %[temp4], 0x8000 \n\t" + "addu %[loop_end], %[frow], %[temp6] \n\t" + "1: \n\t" + "lw %[temp0], 0(%[frow]) \n\t" + "addiu %[dst], %[dst], 1 \n\t" + "addiu %[frow], %[frow], 4 \n\t" + "mult %[temp3], %[temp4] \n\t" + "maddu %[temp0], %[temp2] \n\t" + "mfhi %[temp5] \n\t" + "sb %[temp5], -1(%[dst]) \n\t" + "bne %[frow], %[loop_end], 1b \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp3]"=&r"(temp3), + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [frow]"+r"(frow), + [dst]"+r"(dst), [loop_end]"=&r"(loop_end) + : [temp2]"r"(temp2), [temp6]"r"(temp6) + : "memory", "hi", "lo" + ); + } else { + const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); + const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B); + __asm__ volatile ( + "li %[temp3], 0x10000 \n\t" + "li %[temp4], 0x8000 \n\t" + "addu %[loop_end], %[frow], %[temp6] \n\t" + "1: \n\t" + "lw %[temp0], 0(%[frow]) \n\t" + "lw %[temp1], 0(%[irow]) \n\t" + "addiu %[dst], %[dst], 1 \n\t" + "mult %[temp3], %[temp4] \n\t" + "maddu %[A], %[temp0] \n\t" + "maddu %[B], %[temp1] \n\t" + "addiu %[frow], %[frow], 4 \n\t" + "addiu %[irow], %[irow], 4 \n\t" + "mfhi %[temp5] \n\t" + "mult %[temp3], %[temp4] \n\t" + "maddu %[temp5], %[temp2] \n\t" + "mfhi %[temp5] \n\t" + "sb %[temp5], -1(%[dst]) \n\t" + "bne %[frow], %[loop_end], 1b \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp3]"=&r"(temp3), + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [frow]"+r"(frow), + [irow]"+r"(irow), [dst]"+r"(dst), [loop_end]"=&r"(loop_end) + : [temp2]"r"(temp2), [temp6]"r"(temp6), [A]"r"(A), [B]"r"(B) + : "memory", "hi", "lo" + ); + } +} + +static void ExportRowShrink(WebPRescaler* const wrk) { + const int x_out_max = wrk->dst_width * wrk->num_channels; + uint8_t* dst = wrk->dst; + rescaler_t* irow = wrk->irow; + const rescaler_t* frow = wrk->frow; + const int yscale = wrk->fy_scale * (-wrk->y_accum); + int temp0, temp1, temp3, temp4, temp5, loop_end; + const int temp2 = (int)wrk->fxy_scale; + const int temp6 = x_out_max << 2; + + assert(!WebPRescalerOutputDone(wrk)); + assert(wrk->y_accum <= 0); + assert(!wrk->y_expand); + assert(wrk->fxy_scale != 0); + if (yscale) { + __asm__ volatile ( + "li %[temp3], 0x10000 \n\t" + "li %[temp4], 0x8000 \n\t" + "addu %[loop_end], %[frow], %[temp6] \n\t" + "1: \n\t" + "lw %[temp0], 0(%[frow]) \n\t" + "mult %[temp3], %[temp4] \n\t" + "addiu %[frow], %[frow], 4 \n\t" + "maddu %[temp0], %[yscale] \n\t" + "mfhi %[temp1] \n\t" + "lw %[temp0], 0(%[irow]) \n\t" + "addiu %[dst], %[dst], 1 \n\t" + "addiu %[irow], %[irow], 4 \n\t" + "subu %[temp0], %[temp0], %[temp1] \n\t" + "mult %[temp3], %[temp4] \n\t" + "maddu %[temp0], %[temp2] \n\t" + "mfhi %[temp5] \n\t" + "sw %[temp1], -4(%[irow]) \n\t" + "sb %[temp5], -1(%[dst]) \n\t" + "bne %[frow], %[loop_end], 1b \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp3]"=&r"(temp3), + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [frow]"+r"(frow), + [irow]"+r"(irow), [dst]"+r"(dst), [loop_end]"=&r"(loop_end) + : [temp2]"r"(temp2), [yscale]"r"(yscale), [temp6]"r"(temp6) + : "memory", "hi", "lo" + ); + } else { + __asm__ volatile ( + "li %[temp3], 0x10000 \n\t" + "li %[temp4], 0x8000 \n\t" + "addu %[loop_end], %[irow], %[temp6] \n\t" + "1: \n\t" + "lw %[temp0], 0(%[irow]) \n\t" + "addiu %[dst], %[dst], 1 \n\t" + "addiu %[irow], %[irow], 4 \n\t" + "mult %[temp3], %[temp4] \n\t" + "maddu %[temp0], %[temp2] \n\t" + "mfhi %[temp5] \n\t" + "sw $zero, -4(%[irow]) \n\t" + "sb %[temp5], -1(%[dst]) \n\t" + "bne %[irow], %[loop_end], 1b \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp3]"=&r"(temp3), + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [irow]"+r"(irow), + [dst]"+r"(dst), [loop_end]"=&r"(loop_end) + : [temp2]"r"(temp2), [temp6]"r"(temp6) + : "memory", "hi", "lo" + ); + } +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void WebPRescalerDspInitMIPS32(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitMIPS32(void) { + WebPRescalerImportRowExpand = ImportRowExpand; + WebPRescalerImportRowShrink = ImportRowShrink; + WebPRescalerExportRowExpand = ExportRowExpand; + WebPRescalerExportRowShrink = ExportRowShrink; +} + +#else // !WEBP_USE_MIPS32 + +WEBP_DSP_INIT_STUB(WebPRescalerDspInitMIPS32) + +#endif // WEBP_USE_MIPS32 diff --git a/thirdparty/libwebp/dsp/rescaler_mips_dsp_r2.c b/thirdparty/libwebp/dsp/rescaler_mips_dsp_r2.c new file mode 100644 index 0000000000..b457d0a30a --- /dev/null +++ b/thirdparty/libwebp/dsp/rescaler_mips_dsp_r2.c @@ -0,0 +1,314 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MIPS version of rescaling functions +// +// Author(s): Djordje Pesut (djordje.pesut@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS_DSP_R2) + +#include <assert.h> +#include "../utils/rescaler.h" + +#define ROUNDER (WEBP_RESCALER_ONE >> 1) +#define MULT_FIX(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX) + +//------------------------------------------------------------------------------ +// Row export + +static void ExportRowShrink(WebPRescaler* const wrk) { + int i; + const int x_out_max = wrk->dst_width * wrk->num_channels; + uint8_t* dst = wrk->dst; + rescaler_t* irow = wrk->irow; + const rescaler_t* frow = wrk->frow; + const int yscale = wrk->fy_scale * (-wrk->y_accum); + int temp0, temp1, temp2, temp3, temp4, temp5, loop_end; + const int temp7 = (int)wrk->fxy_scale; + const int temp6 = (x_out_max & ~0x3) << 2; + assert(!WebPRescalerOutputDone(wrk)); + assert(wrk->y_accum <= 0); + assert(!wrk->y_expand); + assert(wrk->fxy_scale != 0); + if (yscale) { + if (x_out_max >= 4) { + int temp8, temp9, temp10, temp11; + __asm__ volatile ( + "li %[temp3], 0x10000 \n\t" + "li %[temp4], 0x8000 \n\t" + "addu %[loop_end], %[frow], %[temp6] \n\t" + "1: \n\t" + "lw %[temp0], 0(%[frow]) \n\t" + "lw %[temp1], 4(%[frow]) \n\t" + "lw %[temp2], 8(%[frow]) \n\t" + "lw %[temp5], 12(%[frow]) \n\t" + "mult $ac0, %[temp3], %[temp4] \n\t" + "maddu $ac0, %[temp0], %[yscale] \n\t" + "mult $ac1, %[temp3], %[temp4] \n\t" + "maddu $ac1, %[temp1], %[yscale] \n\t" + "mult $ac2, %[temp3], %[temp4] \n\t" + "maddu $ac2, %[temp2], %[yscale] \n\t" + "mult $ac3, %[temp3], %[temp4] \n\t" + "maddu $ac3, %[temp5], %[yscale] \n\t" + "addiu %[frow], %[frow], 16 \n\t" + "mfhi %[temp0], $ac0 \n\t" + "mfhi %[temp1], $ac1 \n\t" + "mfhi %[temp2], $ac2 \n\t" + "mfhi %[temp5], $ac3 \n\t" + "lw %[temp8], 0(%[irow]) \n\t" + "lw %[temp9], 4(%[irow]) \n\t" + "lw %[temp10], 8(%[irow]) \n\t" + "lw %[temp11], 12(%[irow]) \n\t" + "addiu %[dst], %[dst], 4 \n\t" + "addiu %[irow], %[irow], 16 \n\t" + "subu %[temp8], %[temp8], %[temp0] \n\t" + "subu %[temp9], %[temp9], %[temp1] \n\t" + "subu %[temp10], %[temp10], %[temp2] \n\t" + "subu %[temp11], %[temp11], %[temp5] \n\t" + "mult $ac0, %[temp3], %[temp4] \n\t" + "maddu $ac0, %[temp8], %[temp7] \n\t" + "mult $ac1, %[temp3], %[temp4] \n\t" + "maddu $ac1, %[temp9], %[temp7] \n\t" + "mult $ac2, %[temp3], %[temp4] \n\t" + "maddu $ac2, %[temp10], %[temp7] \n\t" + "mult $ac3, %[temp3], %[temp4] \n\t" + "maddu $ac3, %[temp11], %[temp7] \n\t" + "mfhi %[temp8], $ac0 \n\t" + "mfhi %[temp9], $ac1 \n\t" + "mfhi %[temp10], $ac2 \n\t" + "mfhi %[temp11], $ac3 \n\t" + "sw %[temp0], -16(%[irow]) \n\t" + "sw %[temp1], -12(%[irow]) \n\t" + "sw %[temp2], -8(%[irow]) \n\t" + "sw %[temp5], -4(%[irow]) \n\t" + "sb %[temp8], -4(%[dst]) \n\t" + "sb %[temp9], -3(%[dst]) \n\t" + "sb %[temp10], -2(%[dst]) \n\t" + "sb %[temp11], -1(%[dst]) \n\t" + "bne %[frow], %[loop_end], 1b \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp3]"=&r"(temp3), + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [frow]"+r"(frow), + [irow]"+r"(irow), [dst]"+r"(dst), [loop_end]"=&r"(loop_end), + [temp8]"=&r"(temp8), [temp9]"=&r"(temp9), [temp10]"=&r"(temp10), + [temp11]"=&r"(temp11), [temp2]"=&r"(temp2) + : [temp7]"r"(temp7), [yscale]"r"(yscale), [temp6]"r"(temp6) + : "memory", "hi", "lo", "$ac1hi", "$ac1lo", + "$ac2hi", "$ac2lo", "$ac3hi", "$ac3lo" + ); + } + for (i = 0; i < (x_out_max & 0x3); ++i) { + const uint32_t frac = (uint32_t)MULT_FIX(*frow++, yscale); + const int v = (int)MULT_FIX(*irow - frac, wrk->fxy_scale); + assert(v >= 0 && v <= 255); + *dst++ = v; + *irow++ = frac; // new fractional start + } + } else { + if (x_out_max >= 4) { + __asm__ volatile ( + "li %[temp3], 0x10000 \n\t" + "li %[temp4], 0x8000 \n\t" + "addu %[loop_end], %[irow], %[temp6] \n\t" + "1: \n\t" + "lw %[temp0], 0(%[irow]) \n\t" + "lw %[temp1], 4(%[irow]) \n\t" + "lw %[temp2], 8(%[irow]) \n\t" + "lw %[temp5], 12(%[irow]) \n\t" + "addiu %[dst], %[dst], 4 \n\t" + "addiu %[irow], %[irow], 16 \n\t" + "mult $ac0, %[temp3], %[temp4] \n\t" + "maddu $ac0, %[temp0], %[temp7] \n\t" + "mult $ac1, %[temp3], %[temp4] \n\t" + "maddu $ac1, %[temp1], %[temp7] \n\t" + "mult $ac2, %[temp3], %[temp4] \n\t" + "maddu $ac2, %[temp2], %[temp7] \n\t" + "mult $ac3, %[temp3], %[temp4] \n\t" + "maddu $ac3, %[temp5], %[temp7] \n\t" + "mfhi %[temp0], $ac0 \n\t" + "mfhi %[temp1], $ac1 \n\t" + "mfhi %[temp2], $ac2 \n\t" + "mfhi %[temp5], $ac3 \n\t" + "sw $zero, -16(%[irow]) \n\t" + "sw $zero, -12(%[irow]) \n\t" + "sw $zero, -8(%[irow]) \n\t" + "sw $zero, -4(%[irow]) \n\t" + "sb %[temp0], -4(%[dst]) \n\t" + "sb %[temp1], -3(%[dst]) \n\t" + "sb %[temp2], -2(%[dst]) \n\t" + "sb %[temp5], -1(%[dst]) \n\t" + "bne %[irow], %[loop_end], 1b \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp3]"=&r"(temp3), + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [irow]"+r"(irow), + [dst]"+r"(dst), [loop_end]"=&r"(loop_end), [temp2]"=&r"(temp2) + : [temp7]"r"(temp7), [temp6]"r"(temp6) + : "memory", "hi", "lo", "$ac1hi", "$ac1lo", + "$ac2hi", "$ac2lo", "$ac3hi", "$ac3lo" + ); + } + for (i = 0; i < (x_out_max & 0x3); ++i) { + const int v = (int)MULT_FIX(*irow, wrk->fxy_scale); + assert(v >= 0 && v <= 255); + *dst++ = v; + *irow++ = 0; + } + } +} + +static void ExportRowExpand(WebPRescaler* const wrk) { + int i; + uint8_t* dst = wrk->dst; + rescaler_t* irow = wrk->irow; + const int x_out_max = wrk->dst_width * wrk->num_channels; + const rescaler_t* frow = wrk->frow; + int temp0, temp1, temp2, temp3, temp4, temp5, loop_end; + const int temp6 = (x_out_max & ~0x3) << 2; + const int temp7 = (int)wrk->fy_scale; + assert(!WebPRescalerOutputDone(wrk)); + assert(wrk->y_accum <= 0); + assert(wrk->y_expand); + assert(wrk->y_sub != 0); + if (wrk->y_accum == 0) { + if (x_out_max >= 4) { + __asm__ volatile ( + "li %[temp4], 0x10000 \n\t" + "li %[temp5], 0x8000 \n\t" + "addu %[loop_end], %[frow], %[temp6] \n\t" + "1: \n\t" + "lw %[temp0], 0(%[frow]) \n\t" + "lw %[temp1], 4(%[frow]) \n\t" + "lw %[temp2], 8(%[frow]) \n\t" + "lw %[temp3], 12(%[frow]) \n\t" + "addiu %[dst], %[dst], 4 \n\t" + "addiu %[frow], %[frow], 16 \n\t" + "mult $ac0, %[temp4], %[temp5] \n\t" + "maddu $ac0, %[temp0], %[temp7] \n\t" + "mult $ac1, %[temp4], %[temp5] \n\t" + "maddu $ac1, %[temp1], %[temp7] \n\t" + "mult $ac2, %[temp4], %[temp5] \n\t" + "maddu $ac2, %[temp2], %[temp7] \n\t" + "mult $ac3, %[temp4], %[temp5] \n\t" + "maddu $ac3, %[temp3], %[temp7] \n\t" + "mfhi %[temp0], $ac0 \n\t" + "mfhi %[temp1], $ac1 \n\t" + "mfhi %[temp2], $ac2 \n\t" + "mfhi %[temp3], $ac3 \n\t" + "sb %[temp0], -4(%[dst]) \n\t" + "sb %[temp1], -3(%[dst]) \n\t" + "sb %[temp2], -2(%[dst]) \n\t" + "sb %[temp3], -1(%[dst]) \n\t" + "bne %[frow], %[loop_end], 1b \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp3]"=&r"(temp3), + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [frow]"+r"(frow), + [dst]"+r"(dst), [loop_end]"=&r"(loop_end), [temp2]"=&r"(temp2) + : [temp7]"r"(temp7), [temp6]"r"(temp6) + : "memory", "hi", "lo", "$ac1hi", "$ac1lo", + "$ac2hi", "$ac2lo", "$ac3hi", "$ac3lo" + ); + } + for (i = 0; i < (x_out_max & 0x3); ++i) { + const uint32_t J = *frow++; + const int v = (int)MULT_FIX(J, wrk->fy_scale); + assert(v >= 0 && v <= 255); + *dst++ = v; + } + } else { + const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); + const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B); + if (x_out_max >= 4) { + int temp8, temp9, temp10, temp11; + __asm__ volatile ( + "li %[temp8], 0x10000 \n\t" + "li %[temp9], 0x8000 \n\t" + "addu %[loop_end], %[frow], %[temp6] \n\t" + "1: \n\t" + "lw %[temp0], 0(%[frow]) \n\t" + "lw %[temp1], 4(%[frow]) \n\t" + "lw %[temp2], 8(%[frow]) \n\t" + "lw %[temp3], 12(%[frow]) \n\t" + "lw %[temp4], 0(%[irow]) \n\t" + "lw %[temp5], 4(%[irow]) \n\t" + "lw %[temp10], 8(%[irow]) \n\t" + "lw %[temp11], 12(%[irow]) \n\t" + "addiu %[dst], %[dst], 4 \n\t" + "mult $ac0, %[temp8], %[temp9] \n\t" + "maddu $ac0, %[A], %[temp0] \n\t" + "maddu $ac0, %[B], %[temp4] \n\t" + "mult $ac1, %[temp8], %[temp9] \n\t" + "maddu $ac1, %[A], %[temp1] \n\t" + "maddu $ac1, %[B], %[temp5] \n\t" + "mult $ac2, %[temp8], %[temp9] \n\t" + "maddu $ac2, %[A], %[temp2] \n\t" + "maddu $ac2, %[B], %[temp10] \n\t" + "mult $ac3, %[temp8], %[temp9] \n\t" + "maddu $ac3, %[A], %[temp3] \n\t" + "maddu $ac3, %[B], %[temp11] \n\t" + "addiu %[frow], %[frow], 16 \n\t" + "addiu %[irow], %[irow], 16 \n\t" + "mfhi %[temp0], $ac0 \n\t" + "mfhi %[temp1], $ac1 \n\t" + "mfhi %[temp2], $ac2 \n\t" + "mfhi %[temp3], $ac3 \n\t" + "mult $ac0, %[temp8], %[temp9] \n\t" + "maddu $ac0, %[temp0], %[temp7] \n\t" + "mult $ac1, %[temp8], %[temp9] \n\t" + "maddu $ac1, %[temp1], %[temp7] \n\t" + "mult $ac2, %[temp8], %[temp9] \n\t" + "maddu $ac2, %[temp2], %[temp7] \n\t" + "mult $ac3, %[temp8], %[temp9] \n\t" + "maddu $ac3, %[temp3], %[temp7] \n\t" + "mfhi %[temp0], $ac0 \n\t" + "mfhi %[temp1], $ac1 \n\t" + "mfhi %[temp2], $ac2 \n\t" + "mfhi %[temp3], $ac3 \n\t" + "sb %[temp0], -4(%[dst]) \n\t" + "sb %[temp1], -3(%[dst]) \n\t" + "sb %[temp2], -2(%[dst]) \n\t" + "sb %[temp3], -1(%[dst]) \n\t" + "bne %[frow], %[loop_end], 1b \n\t" + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp3]"=&r"(temp3), + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [frow]"+r"(frow), + [irow]"+r"(irow), [dst]"+r"(dst), [loop_end]"=&r"(loop_end), + [temp8]"=&r"(temp8), [temp9]"=&r"(temp9), [temp10]"=&r"(temp10), + [temp11]"=&r"(temp11), [temp2]"=&r"(temp2) + : [temp7]"r"(temp7), [temp6]"r"(temp6), [A]"r"(A), [B]"r"(B) + : "memory", "hi", "lo", "$ac1hi", "$ac1lo", + "$ac2hi", "$ac2lo", "$ac3hi", "$ac3lo" + ); + } + for (i = 0; i < (x_out_max & 0x3); ++i) { + const uint64_t I = (uint64_t)A * *frow++ + + (uint64_t)B * *irow++; + const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); + const int v = (int)MULT_FIX(J, wrk->fy_scale); + assert(v >= 0 && v <= 255); + *dst++ = v; + } + } +} + +#undef MULT_FIX +#undef ROUNDER + +//------------------------------------------------------------------------------ +// Entry point + +extern void WebPRescalerDspInitMIPSdspR2(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitMIPSdspR2(void) { + WebPRescalerExportRowExpand = ExportRowExpand; + WebPRescalerExportRowShrink = ExportRowShrink; +} + +#else // !WEBP_USE_MIPS_DSP_R2 + +WEBP_DSP_INIT_STUB(WebPRescalerDspInitMIPSdspR2) + +#endif // WEBP_USE_MIPS_DSP_R2 diff --git a/thirdparty/libwebp/dsp/rescaler_neon.c b/thirdparty/libwebp/dsp/rescaler_neon.c new file mode 100644 index 0000000000..16fd450ea3 --- /dev/null +++ b/thirdparty/libwebp/dsp/rescaler_neon.c @@ -0,0 +1,186 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// NEON version of rescaling functions +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_NEON) + +#include <arm_neon.h> +#include <assert.h> +#include "./neon.h" +#include "../utils/rescaler.h" + +#define ROUNDER (WEBP_RESCALER_ONE >> 1) +#define MULT_FIX_C(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX) + +#define LOAD_32x4(SRC, DST) const uint32x4_t DST = vld1q_u32((SRC)) +#define LOAD_32x8(SRC, DST0, DST1) \ + LOAD_32x4(SRC + 0, DST0); \ + LOAD_32x4(SRC + 4, DST1) + +#define STORE_32x8(SRC0, SRC1, DST) do { \ + vst1q_u32((DST) + 0, SRC0); \ + vst1q_u32((DST) + 4, SRC1); \ +} while (0); + +#if (WEBP_RESCALER_RFIX == 32) +#define MAKE_HALF_CST(C) vdupq_n_s32((int32_t)((C) >> 1)) +#define MULT_FIX(A, B) /* note: B is actualy scale>>1. See MAKE_HALF_CST */ \ + vreinterpretq_u32_s32(vqrdmulhq_s32(vreinterpretq_s32_u32((A)), (B))) +#else +#error "MULT_FIX/WEBP_RESCALER_RFIX need some more work" +#endif + +static uint32x4_t Interpolate(const rescaler_t* const frow, + const rescaler_t* const irow, + uint32_t A, uint32_t B) { + LOAD_32x4(frow, A0); + LOAD_32x4(irow, B0); + const uint64x2_t C0 = vmull_n_u32(vget_low_u32(A0), A); + const uint64x2_t C1 = vmull_n_u32(vget_high_u32(A0), A); + const uint64x2_t D0 = vmlal_n_u32(C0, vget_low_u32(B0), B); + const uint64x2_t D1 = vmlal_n_u32(C1, vget_high_u32(B0), B); + const uint32x4_t E = vcombine_u32( + vrshrn_n_u64(D0, WEBP_RESCALER_RFIX), + vrshrn_n_u64(D1, WEBP_RESCALER_RFIX)); + return E; +} + +static void RescalerExportRowExpand(WebPRescaler* const wrk) { + int x_out; + uint8_t* const dst = wrk->dst; + rescaler_t* const irow = wrk->irow; + const int x_out_max = wrk->dst_width * wrk->num_channels; + const int max_span = x_out_max & ~7; + const rescaler_t* const frow = wrk->frow; + const uint32_t fy_scale = wrk->fy_scale; + const int32x4_t fy_scale_half = MAKE_HALF_CST(fy_scale); + assert(!WebPRescalerOutputDone(wrk)); + assert(wrk->y_accum <= 0); + assert(wrk->y_expand); + assert(wrk->y_sub != 0); + if (wrk->y_accum == 0) { + for (x_out = 0; x_out < max_span; x_out += 8) { + LOAD_32x4(frow + x_out + 0, A0); + LOAD_32x4(frow + x_out + 4, A1); + const uint32x4_t B0 = MULT_FIX(A0, fy_scale_half); + const uint32x4_t B1 = MULT_FIX(A1, fy_scale_half); + const uint16x4_t C0 = vmovn_u32(B0); + const uint16x4_t C1 = vmovn_u32(B1); + const uint8x8_t D = vmovn_u16(vcombine_u16(C0, C1)); + vst1_u8(dst + x_out, D); + } + for (; x_out < x_out_max; ++x_out) { + const uint32_t J = frow[x_out]; + const int v = (int)MULT_FIX_C(J, fy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + } + } else { + const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); + const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B); + for (x_out = 0; x_out < max_span; x_out += 8) { + const uint32x4_t C0 = + Interpolate(frow + x_out + 0, irow + x_out + 0, A, B); + const uint32x4_t C1 = + Interpolate(frow + x_out + 4, irow + x_out + 4, A, B); + const uint32x4_t D0 = MULT_FIX(C0, fy_scale_half); + const uint32x4_t D1 = MULT_FIX(C1, fy_scale_half); + const uint16x4_t E0 = vmovn_u32(D0); + const uint16x4_t E1 = vmovn_u32(D1); + const uint8x8_t F = vmovn_u16(vcombine_u16(E0, E1)); + vst1_u8(dst + x_out, F); + } + for (; x_out < x_out_max; ++x_out) { + const uint64_t I = (uint64_t)A * frow[x_out] + + (uint64_t)B * irow[x_out]; + const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); + const int v = (int)MULT_FIX_C(J, fy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + } + } +} + +static void RescalerExportRowShrink(WebPRescaler* const wrk) { + int x_out; + uint8_t* const dst = wrk->dst; + rescaler_t* const irow = wrk->irow; + const int x_out_max = wrk->dst_width * wrk->num_channels; + const int max_span = x_out_max & ~7; + const rescaler_t* const frow = wrk->frow; + const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum); + const uint32_t fxy_scale = wrk->fxy_scale; + const uint32x4_t zero = vdupq_n_u32(0); + const int32x4_t yscale_half = MAKE_HALF_CST(yscale); + const int32x4_t fxy_scale_half = MAKE_HALF_CST(fxy_scale); + assert(!WebPRescalerOutputDone(wrk)); + assert(wrk->y_accum <= 0); + assert(!wrk->y_expand); + if (yscale) { + for (x_out = 0; x_out < max_span; x_out += 8) { + LOAD_32x8(frow + x_out, in0, in1); + LOAD_32x8(irow + x_out, in2, in3); + const uint32x4_t A0 = MULT_FIX(in0, yscale_half); + const uint32x4_t A1 = MULT_FIX(in1, yscale_half); + const uint32x4_t B0 = vqsubq_u32(in2, A0); + const uint32x4_t B1 = vqsubq_u32(in3, A1); + const uint32x4_t C0 = MULT_FIX(B0, fxy_scale_half); + const uint32x4_t C1 = MULT_FIX(B1, fxy_scale_half); + const uint16x4_t D0 = vmovn_u32(C0); + const uint16x4_t D1 = vmovn_u32(C1); + const uint8x8_t E = vmovn_u16(vcombine_u16(D0, D1)); + vst1_u8(dst + x_out, E); + STORE_32x8(A0, A1, irow + x_out); + } + for (; x_out < x_out_max; ++x_out) { + const uint32_t frac = (uint32_t)MULT_FIX_C(frow[x_out], yscale); + const int v = (int)MULT_FIX_C(irow[x_out] - frac, wrk->fxy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + irow[x_out] = frac; // new fractional start + } + } else { + for (x_out = 0; x_out < max_span; x_out += 8) { + LOAD_32x8(irow + x_out, in0, in1); + const uint32x4_t A0 = MULT_FIX(in0, fxy_scale_half); + const uint32x4_t A1 = MULT_FIX(in1, fxy_scale_half); + const uint16x4_t B0 = vmovn_u32(A0); + const uint16x4_t B1 = vmovn_u32(A1); + const uint8x8_t C = vmovn_u16(vcombine_u16(B0, B1)); + vst1_u8(dst + x_out, C); + STORE_32x8(zero, zero, irow + x_out); + } + for (; x_out < x_out_max; ++x_out) { + const int v = (int)MULT_FIX_C(irow[x_out], fxy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + irow[x_out] = 0; + } + } +} + +//------------------------------------------------------------------------------ + +extern void WebPRescalerDspInitNEON(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitNEON(void) { + WebPRescalerExportRowExpand = RescalerExportRowExpand; + WebPRescalerExportRowShrink = RescalerExportRowShrink; +} + +#else // !WEBP_USE_NEON + +WEBP_DSP_INIT_STUB(WebPRescalerDspInitNEON) + +#endif // WEBP_USE_NEON diff --git a/thirdparty/libwebp/dsp/rescaler_sse2.c b/thirdparty/libwebp/dsp/rescaler_sse2.c new file mode 100644 index 0000000000..5b9702817c --- /dev/null +++ b/thirdparty/libwebp/dsp/rescaler_sse2.c @@ -0,0 +1,375 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE2 Rescaling functions +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE2) +#include <emmintrin.h> + +#include <assert.h> +#include "../utils/rescaler.h" +#include "../utils/utils.h" + +//------------------------------------------------------------------------------ +// Implementations of critical functions ImportRow / ExportRow + +#define ROUNDER (WEBP_RESCALER_ONE >> 1) +#define MULT_FIX(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX) + +// input: 8 bytes ABCDEFGH -> output: A0E0B0F0C0G0D0H0 +static void LoadTwoPixels(const uint8_t* const src, __m128i* out) { + const __m128i zero = _mm_setzero_si128(); + const __m128i A = _mm_loadl_epi64((const __m128i*)(src)); // ABCDEFGH + const __m128i B = _mm_unpacklo_epi8(A, zero); // A0B0C0D0E0F0G0H0 + const __m128i C = _mm_srli_si128(B, 8); // E0F0G0H0 + *out = _mm_unpacklo_epi16(B, C); +} + +// input: 8 bytes ABCDEFGH -> output: A0B0C0D0E0F0G0H0 +static void LoadHeightPixels(const uint8_t* const src, __m128i* out) { + const __m128i zero = _mm_setzero_si128(); + const __m128i A = _mm_loadl_epi64((const __m128i*)(src)); // ABCDEFGH + *out = _mm_unpacklo_epi8(A, zero); +} + +static void RescalerImportRowExpandSSE2(WebPRescaler* const wrk, + const uint8_t* src) { + rescaler_t* frow = wrk->frow; + const rescaler_t* const frow_end = frow + wrk->dst_width * wrk->num_channels; + const int x_add = wrk->x_add; + int accum = x_add; + __m128i cur_pixels; + + assert(!WebPRescalerInputDone(wrk)); + assert(wrk->x_expand); + if (wrk->num_channels == 4) { + if (wrk->src_width < 2) { + WebPRescalerImportRowExpandC(wrk, src); + return; + } + LoadTwoPixels(src, &cur_pixels); + src += 4; + while (1) { + const __m128i mult = _mm_set1_epi32(((x_add - accum) << 16) | accum); + const __m128i out = _mm_madd_epi16(cur_pixels, mult); + _mm_storeu_si128((__m128i*)frow, out); + frow += 4; + if (frow >= frow_end) break; + accum -= wrk->x_sub; + if (accum < 0) { + LoadTwoPixels(src, &cur_pixels); + src += 4; + accum += x_add; + } + } + } else { + int left; + const uint8_t* const src_limit = src + wrk->src_width - 8; + if (wrk->src_width < 8) { + WebPRescalerImportRowExpandC(wrk, src); + return; + } + LoadHeightPixels(src, &cur_pixels); + src += 7; + left = 7; + while (1) { + const __m128i mult = _mm_cvtsi32_si128(((x_add - accum) << 16) | accum); + const __m128i out = _mm_madd_epi16(cur_pixels, mult); + assert(sizeof(*frow) == sizeof(uint32_t)); + WebPUint32ToMem((uint8_t*)frow, _mm_cvtsi128_si32(out)); + frow += 1; + if (frow >= frow_end) break; + accum -= wrk->x_sub; + if (accum < 0) { + if (--left) { + cur_pixels = _mm_srli_si128(cur_pixels, 2); + } else if (src <= src_limit) { + LoadHeightPixels(src, &cur_pixels); + src += 7; + left = 7; + } else { // tail + cur_pixels = _mm_srli_si128(cur_pixels, 2); + cur_pixels = _mm_insert_epi16(cur_pixels, src[1], 1); + src += 1; + left = 1; + } + accum += x_add; + } + } + } + assert(accum == 0); +} + +static void RescalerImportRowShrinkSSE2(WebPRescaler* const wrk, + const uint8_t* src) { + const int x_sub = wrk->x_sub; + int accum = 0; + const __m128i zero = _mm_setzero_si128(); + const __m128i mult0 = _mm_set1_epi16(x_sub); + const __m128i mult1 = _mm_set1_epi32(wrk->fx_scale); + const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER); + __m128i sum = zero; + rescaler_t* frow = wrk->frow; + const rescaler_t* const frow_end = wrk->frow + 4 * wrk->dst_width; + + if (wrk->num_channels != 4 || wrk->x_add > (x_sub << 7)) { + WebPRescalerImportRowShrinkC(wrk, src); + return; + } + assert(!WebPRescalerInputDone(wrk)); + assert(!wrk->x_expand); + + for (; frow < frow_end; frow += 4) { + __m128i base = zero; + accum += wrk->x_add; + while (accum > 0) { + const __m128i A = _mm_cvtsi32_si128(WebPMemToUint32(src)); + src += 4; + base = _mm_unpacklo_epi8(A, zero); + // To avoid overflow, we need: base * x_add / x_sub < 32768 + // => x_add < x_sub << 7. That's a 1/128 reduction ratio limit. + sum = _mm_add_epi16(sum, base); + accum -= x_sub; + } + { // Emit next horizontal pixel. + const __m128i mult = _mm_set1_epi16(-accum); + const __m128i frac0 = _mm_mullo_epi16(base, mult); // 16b x 16b -> 32b + const __m128i frac1 = _mm_mulhi_epu16(base, mult); + const __m128i frac = _mm_unpacklo_epi16(frac0, frac1); // frac is 32b + const __m128i A0 = _mm_mullo_epi16(sum, mult0); + const __m128i A1 = _mm_mulhi_epu16(sum, mult0); + const __m128i B0 = _mm_unpacklo_epi16(A0, A1); // sum * x_sub + const __m128i frow_out = _mm_sub_epi32(B0, frac); // sum * x_sub - frac + const __m128i D0 = _mm_srli_epi64(frac, 32); + const __m128i D1 = _mm_mul_epu32(frac, mult1); // 32b x 16b -> 64b + const __m128i D2 = _mm_mul_epu32(D0, mult1); + const __m128i E1 = _mm_add_epi64(D1, rounder); + const __m128i E2 = _mm_add_epi64(D2, rounder); + const __m128i F1 = _mm_shuffle_epi32(E1, 1 | (3 << 2)); + const __m128i F2 = _mm_shuffle_epi32(E2, 1 | (3 << 2)); + const __m128i G = _mm_unpacklo_epi32(F1, F2); + sum = _mm_packs_epi32(G, zero); + _mm_storeu_si128((__m128i*)frow, frow_out); + } + } + assert(accum == 0); +} + +//------------------------------------------------------------------------------ +// Row export + +// load *src as epi64, multiply by mult and store result in [out0 ... out3] +static WEBP_INLINE void LoadDispatchAndMult(const rescaler_t* const src, + const __m128i* const mult, + __m128i* const out0, + __m128i* const out1, + __m128i* const out2, + __m128i* const out3) { + const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + 0)); + const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + 4)); + const __m128i A2 = _mm_srli_epi64(A0, 32); + const __m128i A3 = _mm_srli_epi64(A1, 32); + if (mult != NULL) { + *out0 = _mm_mul_epu32(A0, *mult); + *out1 = _mm_mul_epu32(A1, *mult); + *out2 = _mm_mul_epu32(A2, *mult); + *out3 = _mm_mul_epu32(A3, *mult); + } else { + *out0 = A0; + *out1 = A1; + *out2 = A2; + *out3 = A3; + } +} + +static WEBP_INLINE void ProcessRow(const __m128i* const A0, + const __m128i* const A1, + const __m128i* const A2, + const __m128i* const A3, + const __m128i* const mult, + uint8_t* const dst) { + const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER); + const __m128i mask = _mm_set_epi32(0xffffffffu, 0, 0xffffffffu, 0); + const __m128i B0 = _mm_mul_epu32(*A0, *mult); + const __m128i B1 = _mm_mul_epu32(*A1, *mult); + const __m128i B2 = _mm_mul_epu32(*A2, *mult); + const __m128i B3 = _mm_mul_epu32(*A3, *mult); + const __m128i C0 = _mm_add_epi64(B0, rounder); + const __m128i C1 = _mm_add_epi64(B1, rounder); + const __m128i C2 = _mm_add_epi64(B2, rounder); + const __m128i C3 = _mm_add_epi64(B3, rounder); + const __m128i D0 = _mm_srli_epi64(C0, WEBP_RESCALER_RFIX); + const __m128i D1 = _mm_srli_epi64(C1, WEBP_RESCALER_RFIX); +#if (WEBP_RESCALER_FIX < 32) + const __m128i D2 = + _mm_and_si128(_mm_slli_epi64(C2, 32 - WEBP_RESCALER_RFIX), mask); + const __m128i D3 = + _mm_and_si128(_mm_slli_epi64(C3, 32 - WEBP_RESCALER_RFIX), mask); +#else + const __m128i D2 = _mm_and_si128(C2, mask); + const __m128i D3 = _mm_and_si128(C3, mask); +#endif + const __m128i E0 = _mm_or_si128(D0, D2); + const __m128i E1 = _mm_or_si128(D1, D3); + const __m128i F = _mm_packs_epi32(E0, E1); + const __m128i G = _mm_packus_epi16(F, F); + _mm_storel_epi64((__m128i*)dst, G); +} + +static void RescalerExportRowExpandSSE2(WebPRescaler* const wrk) { + int x_out; + uint8_t* const dst = wrk->dst; + rescaler_t* const irow = wrk->irow; + const int x_out_max = wrk->dst_width * wrk->num_channels; + const rescaler_t* const frow = wrk->frow; + const __m128i mult = _mm_set_epi32(0, wrk->fy_scale, 0, wrk->fy_scale); + + assert(!WebPRescalerOutputDone(wrk)); + assert(wrk->y_accum <= 0 && wrk->y_sub + wrk->y_accum >= 0); + assert(wrk->y_expand); + if (wrk->y_accum == 0) { + for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { + __m128i A0, A1, A2, A3; + LoadDispatchAndMult(frow + x_out, NULL, &A0, &A1, &A2, &A3); + ProcessRow(&A0, &A1, &A2, &A3, &mult, dst + x_out); + } + for (; x_out < x_out_max; ++x_out) { + const uint32_t J = frow[x_out]; + const int v = (int)MULT_FIX(J, wrk->fy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + } + } else { + const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); + const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B); + const __m128i mA = _mm_set_epi32(0, A, 0, A); + const __m128i mB = _mm_set_epi32(0, B, 0, B); + const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER); + for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { + __m128i A0, A1, A2, A3, B0, B1, B2, B3; + LoadDispatchAndMult(frow + x_out, &mA, &A0, &A1, &A2, &A3); + LoadDispatchAndMult(irow + x_out, &mB, &B0, &B1, &B2, &B3); + { + const __m128i C0 = _mm_add_epi64(A0, B0); + const __m128i C1 = _mm_add_epi64(A1, B1); + const __m128i C2 = _mm_add_epi64(A2, B2); + const __m128i C3 = _mm_add_epi64(A3, B3); + const __m128i D0 = _mm_add_epi64(C0, rounder); + const __m128i D1 = _mm_add_epi64(C1, rounder); + const __m128i D2 = _mm_add_epi64(C2, rounder); + const __m128i D3 = _mm_add_epi64(C3, rounder); + const __m128i E0 = _mm_srli_epi64(D0, WEBP_RESCALER_RFIX); + const __m128i E1 = _mm_srli_epi64(D1, WEBP_RESCALER_RFIX); + const __m128i E2 = _mm_srli_epi64(D2, WEBP_RESCALER_RFIX); + const __m128i E3 = _mm_srli_epi64(D3, WEBP_RESCALER_RFIX); + ProcessRow(&E0, &E1, &E2, &E3, &mult, dst + x_out); + } + } + for (; x_out < x_out_max; ++x_out) { + const uint64_t I = (uint64_t)A * frow[x_out] + + (uint64_t)B * irow[x_out]; + const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); + const int v = (int)MULT_FIX(J, wrk->fy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + } + } +} + +static void RescalerExportRowShrinkSSE2(WebPRescaler* const wrk) { + int x_out; + uint8_t* const dst = wrk->dst; + rescaler_t* const irow = wrk->irow; + const int x_out_max = wrk->dst_width * wrk->num_channels; + const rescaler_t* const frow = wrk->frow; + const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum); + assert(!WebPRescalerOutputDone(wrk)); + assert(wrk->y_accum <= 0); + assert(!wrk->y_expand); + if (yscale) { + const int scale_xy = wrk->fxy_scale; + const __m128i mult_xy = _mm_set_epi32(0, scale_xy, 0, scale_xy); + const __m128i mult_y = _mm_set_epi32(0, yscale, 0, yscale); + const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER); + for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { + __m128i A0, A1, A2, A3, B0, B1, B2, B3; + LoadDispatchAndMult(irow + x_out, NULL, &A0, &A1, &A2, &A3); + LoadDispatchAndMult(frow + x_out, &mult_y, &B0, &B1, &B2, &B3); + { + const __m128i C0 = _mm_add_epi64(B0, rounder); + const __m128i C1 = _mm_add_epi64(B1, rounder); + const __m128i C2 = _mm_add_epi64(B2, rounder); + const __m128i C3 = _mm_add_epi64(B3, rounder); + const __m128i D0 = _mm_srli_epi64(C0, WEBP_RESCALER_RFIX); // = frac + const __m128i D1 = _mm_srli_epi64(C1, WEBP_RESCALER_RFIX); + const __m128i D2 = _mm_srli_epi64(C2, WEBP_RESCALER_RFIX); + const __m128i D3 = _mm_srli_epi64(C3, WEBP_RESCALER_RFIX); + const __m128i E0 = _mm_sub_epi64(A0, D0); // irow[x] - frac + const __m128i E1 = _mm_sub_epi64(A1, D1); + const __m128i E2 = _mm_sub_epi64(A2, D2); + const __m128i E3 = _mm_sub_epi64(A3, D3); + const __m128i F2 = _mm_slli_epi64(D2, 32); + const __m128i F3 = _mm_slli_epi64(D3, 32); + const __m128i G0 = _mm_or_si128(D0, F2); + const __m128i G1 = _mm_or_si128(D1, F3); + _mm_storeu_si128((__m128i*)(irow + x_out + 0), G0); + _mm_storeu_si128((__m128i*)(irow + x_out + 4), G1); + ProcessRow(&E0, &E1, &E2, &E3, &mult_xy, dst + x_out); + } + } + for (; x_out < x_out_max; ++x_out) { + const uint32_t frac = (int)MULT_FIX(frow[x_out], yscale); + const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + irow[x_out] = frac; // new fractional start + } + } else { + const uint32_t scale = wrk->fxy_scale; + const __m128i mult = _mm_set_epi32(0, scale, 0, scale); + const __m128i zero = _mm_setzero_si128(); + for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { + __m128i A0, A1, A2, A3; + LoadDispatchAndMult(irow + x_out, NULL, &A0, &A1, &A2, &A3); + _mm_storeu_si128((__m128i*)(irow + x_out + 0), zero); + _mm_storeu_si128((__m128i*)(irow + x_out + 4), zero); + ProcessRow(&A0, &A1, &A2, &A3, &mult, dst + x_out); + } + for (; x_out < x_out_max; ++x_out) { + const int v = (int)MULT_FIX(irow[x_out], scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + irow[x_out] = 0; + } + } +} + +#undef MULT_FIX +#undef ROUNDER + +//------------------------------------------------------------------------------ + +extern void WebPRescalerDspInitSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitSSE2(void) { + WebPRescalerImportRowExpand = RescalerImportRowExpandSSE2; + WebPRescalerImportRowShrink = RescalerImportRowShrinkSSE2; + WebPRescalerExportRowExpand = RescalerExportRowExpandSSE2; + WebPRescalerExportRowShrink = RescalerExportRowShrinkSSE2; +} + +#else // !WEBP_USE_SSE2 + +WEBP_DSP_INIT_STUB(WebPRescalerDspInitSSE2) + +#endif // WEBP_USE_SSE2 diff --git a/thirdparty/libwebp/dsp/upsampling.c b/thirdparty/libwebp/dsp/upsampling.c new file mode 100644 index 0000000000..651274fcee --- /dev/null +++ b/thirdparty/libwebp/dsp/upsampling.c @@ -0,0 +1,260 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// YUV to RGB upsampling functions. +// +// Author: somnath@google.com (Somnath Banerjee) + +#include "./dsp.h" +#include "./yuv.h" + +#include <assert.h> + +//------------------------------------------------------------------------------ +// Fancy upsampler + +#ifdef FANCY_UPSAMPLING + +// Fancy upsampling functions to convert YUV to RGB +WebPUpsampleLinePairFunc WebPUpsamplers[MODE_LAST]; + +// Given samples laid out in a square as: +// [a b] +// [c d] +// we interpolate u/v as: +// ([9*a + 3*b + 3*c + d 3*a + 9*b + 3*c + d] + [8 8]) / 16 +// ([3*a + b + 9*c + 3*d a + 3*b + 3*c + 9*d] [8 8]) / 16 + +// We process u and v together stashed into 32bit (16bit each). +#define LOAD_UV(u, v) ((u) | ((v) << 16)) + +#define UPSAMPLE_FUNC(FUNC_NAME, FUNC, XSTEP) \ +static void FUNC_NAME(const uint8_t* top_y, const uint8_t* bottom_y, \ + const uint8_t* top_u, const uint8_t* top_v, \ + const uint8_t* cur_u, const uint8_t* cur_v, \ + uint8_t* top_dst, uint8_t* bottom_dst, int len) { \ + int x; \ + const int last_pixel_pair = (len - 1) >> 1; \ + uint32_t tl_uv = LOAD_UV(top_u[0], top_v[0]); /* top-left sample */ \ + uint32_t l_uv = LOAD_UV(cur_u[0], cur_v[0]); /* left-sample */ \ + assert(top_y != NULL); \ + { \ + const uint32_t uv0 = (3 * tl_uv + l_uv + 0x00020002u) >> 2; \ + FUNC(top_y[0], uv0 & 0xff, (uv0 >> 16), top_dst); \ + } \ + if (bottom_y != NULL) { \ + const uint32_t uv0 = (3 * l_uv + tl_uv + 0x00020002u) >> 2; \ + FUNC(bottom_y[0], uv0 & 0xff, (uv0 >> 16), bottom_dst); \ + } \ + for (x = 1; x <= last_pixel_pair; ++x) { \ + const uint32_t t_uv = LOAD_UV(top_u[x], top_v[x]); /* top sample */ \ + const uint32_t uv = LOAD_UV(cur_u[x], cur_v[x]); /* sample */ \ + /* precompute invariant values associated with first and second diagonals*/\ + const uint32_t avg = tl_uv + t_uv + l_uv + uv + 0x00080008u; \ + const uint32_t diag_12 = (avg + 2 * (t_uv + l_uv)) >> 3; \ + const uint32_t diag_03 = (avg + 2 * (tl_uv + uv)) >> 3; \ + { \ + const uint32_t uv0 = (diag_12 + tl_uv) >> 1; \ + const uint32_t uv1 = (diag_03 + t_uv) >> 1; \ + FUNC(top_y[2 * x - 1], uv0 & 0xff, (uv0 >> 16), \ + top_dst + (2 * x - 1) * XSTEP); \ + FUNC(top_y[2 * x - 0], uv1 & 0xff, (uv1 >> 16), \ + top_dst + (2 * x - 0) * XSTEP); \ + } \ + if (bottom_y != NULL) { \ + const uint32_t uv0 = (diag_03 + l_uv) >> 1; \ + const uint32_t uv1 = (diag_12 + uv) >> 1; \ + FUNC(bottom_y[2 * x - 1], uv0 & 0xff, (uv0 >> 16), \ + bottom_dst + (2 * x - 1) * XSTEP); \ + FUNC(bottom_y[2 * x + 0], uv1 & 0xff, (uv1 >> 16), \ + bottom_dst + (2 * x + 0) * XSTEP); \ + } \ + tl_uv = t_uv; \ + l_uv = uv; \ + } \ + if (!(len & 1)) { \ + { \ + const uint32_t uv0 = (3 * tl_uv + l_uv + 0x00020002u) >> 2; \ + FUNC(top_y[len - 1], uv0 & 0xff, (uv0 >> 16), \ + top_dst + (len - 1) * XSTEP); \ + } \ + if (bottom_y != NULL) { \ + const uint32_t uv0 = (3 * l_uv + tl_uv + 0x00020002u) >> 2; \ + FUNC(bottom_y[len - 1], uv0 & 0xff, (uv0 >> 16), \ + bottom_dst + (len - 1) * XSTEP); \ + } \ + } \ +} + +// All variants implemented. +UPSAMPLE_FUNC(UpsampleRgbLinePair, VP8YuvToRgb, 3) +UPSAMPLE_FUNC(UpsampleBgrLinePair, VP8YuvToBgr, 3) +UPSAMPLE_FUNC(UpsampleRgbaLinePair, VP8YuvToRgba, 4) +UPSAMPLE_FUNC(UpsampleBgraLinePair, VP8YuvToBgra, 4) +UPSAMPLE_FUNC(UpsampleArgbLinePair, VP8YuvToArgb, 4) +UPSAMPLE_FUNC(UpsampleRgba4444LinePair, VP8YuvToRgba4444, 2) +UPSAMPLE_FUNC(UpsampleRgb565LinePair, VP8YuvToRgb565, 2) + +#undef LOAD_UV +#undef UPSAMPLE_FUNC + +#endif // FANCY_UPSAMPLING + +//------------------------------------------------------------------------------ + +#if !defined(FANCY_UPSAMPLING) +#define DUAL_SAMPLE_FUNC(FUNC_NAME, FUNC) \ +static void FUNC_NAME(const uint8_t* top_y, const uint8_t* bot_y, \ + const uint8_t* top_u, const uint8_t* top_v, \ + const uint8_t* bot_u, const uint8_t* bot_v, \ + uint8_t* top_dst, uint8_t* bot_dst, int len) { \ + const int half_len = len >> 1; \ + int x; \ + assert(top_dst != NULL); \ + { \ + for (x = 0; x < half_len; ++x) { \ + FUNC(top_y[2 * x + 0], top_u[x], top_v[x], top_dst + 8 * x + 0); \ + FUNC(top_y[2 * x + 1], top_u[x], top_v[x], top_dst + 8 * x + 4); \ + } \ + if (len & 1) FUNC(top_y[2 * x + 0], top_u[x], top_v[x], top_dst + 8 * x); \ + } \ + if (bot_dst != NULL) { \ + for (x = 0; x < half_len; ++x) { \ + FUNC(bot_y[2 * x + 0], bot_u[x], bot_v[x], bot_dst + 8 * x + 0); \ + FUNC(bot_y[2 * x + 1], bot_u[x], bot_v[x], bot_dst + 8 * x + 4); \ + } \ + if (len & 1) FUNC(bot_y[2 * x + 0], bot_u[x], bot_v[x], bot_dst + 8 * x); \ + } \ +} + +DUAL_SAMPLE_FUNC(DualLineSamplerBGRA, VP8YuvToBgra) +DUAL_SAMPLE_FUNC(DualLineSamplerARGB, VP8YuvToArgb) +#undef DUAL_SAMPLE_FUNC + +#endif // !FANCY_UPSAMPLING + +WebPUpsampleLinePairFunc WebPGetLinePairConverter(int alpha_is_last) { + WebPInitUpsamplers(); + VP8YUVInit(); +#ifdef FANCY_UPSAMPLING + return WebPUpsamplers[alpha_is_last ? MODE_BGRA : MODE_ARGB]; +#else + return (alpha_is_last ? DualLineSamplerBGRA : DualLineSamplerARGB); +#endif +} + +//------------------------------------------------------------------------------ +// YUV444 converter + +#define YUV444_FUNC(FUNC_NAME, FUNC, XSTEP) \ +extern void FUNC_NAME(const uint8_t* y, const uint8_t* u, const uint8_t* v, \ + uint8_t* dst, int len); \ +void FUNC_NAME(const uint8_t* y, const uint8_t* u, const uint8_t* v, \ + uint8_t* dst, int len) { \ + int i; \ + for (i = 0; i < len; ++i) FUNC(y[i], u[i], v[i], &dst[i * XSTEP]); \ +} + +YUV444_FUNC(WebPYuv444ToRgbC, VP8YuvToRgb, 3) +YUV444_FUNC(WebPYuv444ToBgrC, VP8YuvToBgr, 3) +YUV444_FUNC(WebPYuv444ToRgbaC, VP8YuvToRgba, 4) +YUV444_FUNC(WebPYuv444ToBgraC, VP8YuvToBgra, 4) +YUV444_FUNC(WebPYuv444ToArgbC, VP8YuvToArgb, 4) +YUV444_FUNC(WebPYuv444ToRgba4444C, VP8YuvToRgba4444, 2) +YUV444_FUNC(WebPYuv444ToRgb565C, VP8YuvToRgb565, 2) + +#undef YUV444_FUNC + +WebPYUV444Converter WebPYUV444Converters[MODE_LAST]; + +extern void WebPInitYUV444ConvertersMIPSdspR2(void); +extern void WebPInitYUV444ConvertersSSE2(void); + +static volatile VP8CPUInfo upsampling_last_cpuinfo_used1 = + (VP8CPUInfo)&upsampling_last_cpuinfo_used1; + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitYUV444Converters(void) { + if (upsampling_last_cpuinfo_used1 == VP8GetCPUInfo) return; + + WebPYUV444Converters[MODE_RGB] = WebPYuv444ToRgbC; + WebPYUV444Converters[MODE_RGBA] = WebPYuv444ToRgbaC; + WebPYUV444Converters[MODE_BGR] = WebPYuv444ToBgrC; + WebPYUV444Converters[MODE_BGRA] = WebPYuv444ToBgraC; + WebPYUV444Converters[MODE_ARGB] = WebPYuv444ToArgbC; + WebPYUV444Converters[MODE_RGBA_4444] = WebPYuv444ToRgba4444C; + WebPYUV444Converters[MODE_RGB_565] = WebPYuv444ToRgb565C; + WebPYUV444Converters[MODE_rgbA] = WebPYuv444ToRgbaC; + WebPYUV444Converters[MODE_bgrA] = WebPYuv444ToBgraC; + WebPYUV444Converters[MODE_Argb] = WebPYuv444ToArgbC; + WebPYUV444Converters[MODE_rgbA_4444] = WebPYuv444ToRgba4444C; + + if (VP8GetCPUInfo != NULL) { +#if defined(WEBP_USE_SSE2) + if (VP8GetCPUInfo(kSSE2)) { + WebPInitYUV444ConvertersSSE2(); + } +#endif +#if defined(WEBP_USE_MIPS_DSP_R2) + if (VP8GetCPUInfo(kMIPSdspR2)) { + WebPInitYUV444ConvertersMIPSdspR2(); + } +#endif + } + upsampling_last_cpuinfo_used1 = VP8GetCPUInfo; +} + +//------------------------------------------------------------------------------ +// Main calls + +extern void WebPInitUpsamplersSSE2(void); +extern void WebPInitUpsamplersNEON(void); +extern void WebPInitUpsamplersMIPSdspR2(void); + +static volatile VP8CPUInfo upsampling_last_cpuinfo_used2 = + (VP8CPUInfo)&upsampling_last_cpuinfo_used2; + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitUpsamplers(void) { + if (upsampling_last_cpuinfo_used2 == VP8GetCPUInfo) return; + +#ifdef FANCY_UPSAMPLING + WebPUpsamplers[MODE_RGB] = UpsampleRgbLinePair; + WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePair; + WebPUpsamplers[MODE_BGR] = UpsampleBgrLinePair; + WebPUpsamplers[MODE_BGRA] = UpsampleBgraLinePair; + WebPUpsamplers[MODE_ARGB] = UpsampleArgbLinePair; + WebPUpsamplers[MODE_RGBA_4444] = UpsampleRgba4444LinePair; + WebPUpsamplers[MODE_RGB_565] = UpsampleRgb565LinePair; + WebPUpsamplers[MODE_rgbA] = UpsampleRgbaLinePair; + WebPUpsamplers[MODE_bgrA] = UpsampleBgraLinePair; + WebPUpsamplers[MODE_Argb] = UpsampleArgbLinePair; + WebPUpsamplers[MODE_rgbA_4444] = UpsampleRgba4444LinePair; + + // If defined, use CPUInfo() to overwrite some pointers with faster versions. + if (VP8GetCPUInfo != NULL) { +#if defined(WEBP_USE_SSE2) + if (VP8GetCPUInfo(kSSE2)) { + WebPInitUpsamplersSSE2(); + } +#endif +#if defined(WEBP_USE_NEON) + if (VP8GetCPUInfo(kNEON)) { + WebPInitUpsamplersNEON(); + } +#endif +#if defined(WEBP_USE_MIPS_DSP_R2) + if (VP8GetCPUInfo(kMIPSdspR2)) { + WebPInitUpsamplersMIPSdspR2(); + } +#endif + } +#endif // FANCY_UPSAMPLING + upsampling_last_cpuinfo_used2 = VP8GetCPUInfo; +} + +//------------------------------------------------------------------------------ diff --git a/thirdparty/libwebp/dsp/upsampling_mips_dsp_r2.c b/thirdparty/libwebp/dsp/upsampling_mips_dsp_r2.c new file mode 100644 index 0000000000..ed2eb74825 --- /dev/null +++ b/thirdparty/libwebp/dsp/upsampling_mips_dsp_r2.c @@ -0,0 +1,282 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// YUV to RGB upsampling functions. +// +// Author(s): Branimir Vasic (branimir.vasic@imgtec.com) +// Djordje Pesut (djordje.pesut@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS_DSP_R2) + +#include <assert.h> +#include "./yuv.h" + +#if !defined(WEBP_YUV_USE_TABLE) + +#define YUV_TO_RGB(Y, U, V, R, G, B) do { \ + const int t1 = MultHi(Y, 19077); \ + const int t2 = MultHi(V, 13320); \ + R = MultHi(V, 26149); \ + G = MultHi(U, 6419); \ + B = MultHi(U, 33050); \ + R = t1 + R; \ + G = t1 - G; \ + B = t1 + B; \ + R = R - 14234; \ + G = G - t2 + 8708; \ + B = B - 17685; \ + __asm__ volatile ( \ + "shll_s.w %[" #R "], %[" #R "], 17 \n\t" \ + "shll_s.w %[" #G "], %[" #G "], 17 \n\t" \ + "shll_s.w %[" #B "], %[" #B "], 17 \n\t" \ + "precrqu_s.qb.ph %[" #R "], %[" #R "], $zero \n\t" \ + "precrqu_s.qb.ph %[" #G "], %[" #G "], $zero \n\t" \ + "precrqu_s.qb.ph %[" #B "], %[" #B "], $zero \n\t" \ + "srl %[" #R "], %[" #R "], 24 \n\t" \ + "srl %[" #G "], %[" #G "], 24 \n\t" \ + "srl %[" #B "], %[" #B "], 24 \n\t" \ + : [R]"+r"(R), [G]"+r"(G), [B]"+r"(B) \ + : \ + ); \ + } while (0) + +static WEBP_INLINE void YuvToRgb(int y, int u, int v, uint8_t* const rgb) { + int r, g, b; + YUV_TO_RGB(y, u, v, r, g, b); + rgb[0] = r; + rgb[1] = g; + rgb[2] = b; +} +static WEBP_INLINE void YuvToBgr(int y, int u, int v, uint8_t* const bgr) { + int r, g, b; + YUV_TO_RGB(y, u, v, r, g, b); + bgr[0] = b; + bgr[1] = g; + bgr[2] = r; +} +static WEBP_INLINE void YuvToRgb565(int y, int u, int v, uint8_t* const rgb) { + int r, g, b; + YUV_TO_RGB(y, u, v, r, g, b); + { + const int rg = (r & 0xf8) | (g >> 5); + const int gb = ((g << 3) & 0xe0) | (b >> 3); +#ifdef WEBP_SWAP_16BIT_CSP + rgb[0] = gb; + rgb[1] = rg; +#else + rgb[0] = rg; + rgb[1] = gb; +#endif + } +} +static WEBP_INLINE void YuvToRgba4444(int y, int u, int v, + uint8_t* const argb) { + int r, g, b; + YUV_TO_RGB(y, u, v, r, g, b); + { + const int rg = (r & 0xf0) | (g >> 4); + const int ba = (b & 0xf0) | 0x0f; // overwrite the lower 4 bits +#ifdef WEBP_SWAP_16BIT_CSP + argb[0] = ba; + argb[1] = rg; +#else + argb[0] = rg; + argb[1] = ba; +#endif + } +} +#endif // WEBP_YUV_USE_TABLE + +//----------------------------------------------------------------------------- +// Alpha handling variants + +static WEBP_INLINE void YuvToArgb(uint8_t y, uint8_t u, uint8_t v, + uint8_t* const argb) { + int r, g, b; + YUV_TO_RGB(y, u, v, r, g, b); + argb[0] = 0xff; + argb[1] = r; + argb[2] = g; + argb[3] = b; +} +static WEBP_INLINE void YuvToBgra(uint8_t y, uint8_t u, uint8_t v, + uint8_t* const bgra) { + int r, g, b; + YUV_TO_RGB(y, u, v, r, g, b); + bgra[0] = b; + bgra[1] = g; + bgra[2] = r; + bgra[3] = 0xff; +} +static WEBP_INLINE void YuvToRgba(uint8_t y, uint8_t u, uint8_t v, + uint8_t* const rgba) { + int r, g, b; + YUV_TO_RGB(y, u, v, r, g, b); + rgba[0] = r; + rgba[1] = g; + rgba[2] = b; + rgba[3] = 0xff; +} + +//------------------------------------------------------------------------------ +// Fancy upsampler + +#ifdef FANCY_UPSAMPLING + +// Given samples laid out in a square as: +// [a b] +// [c d] +// we interpolate u/v as: +// ([9*a + 3*b + 3*c + d 3*a + 9*b + 3*c + d] + [8 8]) / 16 +// ([3*a + b + 9*c + 3*d a + 3*b + 3*c + 9*d] [8 8]) / 16 + +// We process u and v together stashed into 32bit (16bit each). +#define LOAD_UV(u, v) ((u) | ((v) << 16)) + +#define UPSAMPLE_FUNC(FUNC_NAME, FUNC, XSTEP) \ +static void FUNC_NAME(const uint8_t* top_y, const uint8_t* bottom_y, \ + const uint8_t* top_u, const uint8_t* top_v, \ + const uint8_t* cur_u, const uint8_t* cur_v, \ + uint8_t* top_dst, uint8_t* bottom_dst, int len) { \ + int x; \ + const int last_pixel_pair = (len - 1) >> 1; \ + uint32_t tl_uv = LOAD_UV(top_u[0], top_v[0]); /* top-left sample */ \ + uint32_t l_uv = LOAD_UV(cur_u[0], cur_v[0]); /* left-sample */ \ + assert(top_y != NULL); \ + { \ + const uint32_t uv0 = (3 * tl_uv + l_uv + 0x00020002u) >> 2; \ + FUNC(top_y[0], uv0 & 0xff, (uv0 >> 16), top_dst); \ + } \ + if (bottom_y != NULL) { \ + const uint32_t uv0 = (3 * l_uv + tl_uv + 0x00020002u) >> 2; \ + FUNC(bottom_y[0], uv0 & 0xff, (uv0 >> 16), bottom_dst); \ + } \ + for (x = 1; x <= last_pixel_pair; ++x) { \ + const uint32_t t_uv = LOAD_UV(top_u[x], top_v[x]); /* top sample */ \ + const uint32_t uv = LOAD_UV(cur_u[x], cur_v[x]); /* sample */ \ + /* precompute invariant values associated with first and second diagonals*/\ + const uint32_t avg = tl_uv + t_uv + l_uv + uv + 0x00080008u; \ + const uint32_t diag_12 = (avg + 2 * (t_uv + l_uv)) >> 3; \ + const uint32_t diag_03 = (avg + 2 * (tl_uv + uv)) >> 3; \ + { \ + const uint32_t uv0 = (diag_12 + tl_uv) >> 1; \ + const uint32_t uv1 = (diag_03 + t_uv) >> 1; \ + FUNC(top_y[2 * x - 1], uv0 & 0xff, (uv0 >> 16), \ + top_dst + (2 * x - 1) * XSTEP); \ + FUNC(top_y[2 * x - 0], uv1 & 0xff, (uv1 >> 16), \ + top_dst + (2 * x - 0) * XSTEP); \ + } \ + if (bottom_y != NULL) { \ + const uint32_t uv0 = (diag_03 + l_uv) >> 1; \ + const uint32_t uv1 = (diag_12 + uv) >> 1; \ + FUNC(bottom_y[2 * x - 1], uv0 & 0xff, (uv0 >> 16), \ + bottom_dst + (2 * x - 1) * XSTEP); \ + FUNC(bottom_y[2 * x + 0], uv1 & 0xff, (uv1 >> 16), \ + bottom_dst + (2 * x + 0) * XSTEP); \ + } \ + tl_uv = t_uv; \ + l_uv = uv; \ + } \ + if (!(len & 1)) { \ + { \ + const uint32_t uv0 = (3 * tl_uv + l_uv + 0x00020002u) >> 2; \ + FUNC(top_y[len - 1], uv0 & 0xff, (uv0 >> 16), \ + top_dst + (len - 1) * XSTEP); \ + } \ + if (bottom_y != NULL) { \ + const uint32_t uv0 = (3 * l_uv + tl_uv + 0x00020002u) >> 2; \ + FUNC(bottom_y[len - 1], uv0 & 0xff, (uv0 >> 16), \ + bottom_dst + (len - 1) * XSTEP); \ + } \ + } \ +} + +// All variants implemented. +UPSAMPLE_FUNC(UpsampleRgbLinePair, YuvToRgb, 3) +UPSAMPLE_FUNC(UpsampleBgrLinePair, YuvToBgr, 3) +UPSAMPLE_FUNC(UpsampleRgbaLinePair, YuvToRgba, 4) +UPSAMPLE_FUNC(UpsampleBgraLinePair, YuvToBgra, 4) +UPSAMPLE_FUNC(UpsampleArgbLinePair, YuvToArgb, 4) +UPSAMPLE_FUNC(UpsampleRgba4444LinePair, YuvToRgba4444, 2) +UPSAMPLE_FUNC(UpsampleRgb565LinePair, YuvToRgb565, 2) + +#undef LOAD_UV +#undef UPSAMPLE_FUNC + +//------------------------------------------------------------------------------ +// Entry point + +extern void WebPInitUpsamplersMIPSdspR2(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitUpsamplersMIPSdspR2(void) { + WebPUpsamplers[MODE_RGB] = UpsampleRgbLinePair; + WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePair; + WebPUpsamplers[MODE_BGR] = UpsampleBgrLinePair; + WebPUpsamplers[MODE_BGRA] = UpsampleBgraLinePair; + WebPUpsamplers[MODE_ARGB] = UpsampleArgbLinePair; + WebPUpsamplers[MODE_RGBA_4444] = UpsampleRgba4444LinePair; + WebPUpsamplers[MODE_RGB_565] = UpsampleRgb565LinePair; + WebPUpsamplers[MODE_rgbA] = UpsampleRgbaLinePair; + WebPUpsamplers[MODE_bgrA] = UpsampleBgraLinePair; + WebPUpsamplers[MODE_Argb] = UpsampleArgbLinePair; + WebPUpsamplers[MODE_rgbA_4444] = UpsampleRgba4444LinePair; +} + +#endif // FANCY_UPSAMPLING + +//------------------------------------------------------------------------------ +// YUV444 converter + +#define YUV444_FUNC(FUNC_NAME, FUNC, XSTEP) \ +static void FUNC_NAME(const uint8_t* y, const uint8_t* u, const uint8_t* v, \ + uint8_t* dst, int len) { \ + int i; \ + for (i = 0; i < len; ++i) FUNC(y[i], u[i], v[i], &dst[i * XSTEP]); \ +} + +YUV444_FUNC(Yuv444ToRgb, YuvToRgb, 3) +YUV444_FUNC(Yuv444ToBgr, YuvToBgr, 3) +YUV444_FUNC(Yuv444ToRgba, YuvToRgba, 4) +YUV444_FUNC(Yuv444ToBgra, YuvToBgra, 4) +YUV444_FUNC(Yuv444ToArgb, YuvToArgb, 4) +YUV444_FUNC(Yuv444ToRgba4444, YuvToRgba4444, 2) +YUV444_FUNC(Yuv444ToRgb565, YuvToRgb565, 2) + +#undef YUV444_FUNC + +//------------------------------------------------------------------------------ +// Entry point + +extern void WebPInitYUV444ConvertersMIPSdspR2(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitYUV444ConvertersMIPSdspR2(void) { + WebPYUV444Converters[MODE_RGB] = Yuv444ToRgb; + WebPYUV444Converters[MODE_RGBA] = Yuv444ToRgba; + WebPYUV444Converters[MODE_BGR] = Yuv444ToBgr; + WebPYUV444Converters[MODE_BGRA] = Yuv444ToBgra; + WebPYUV444Converters[MODE_ARGB] = Yuv444ToArgb; + WebPYUV444Converters[MODE_RGBA_4444] = Yuv444ToRgba4444; + WebPYUV444Converters[MODE_RGB_565] = Yuv444ToRgb565; + WebPYUV444Converters[MODE_rgbA] = Yuv444ToRgba; + WebPYUV444Converters[MODE_bgrA] = Yuv444ToBgra; + WebPYUV444Converters[MODE_Argb] = Yuv444ToArgb; + WebPYUV444Converters[MODE_rgbA_4444] = Yuv444ToRgba4444; +} + +#else // !WEBP_USE_MIPS_DSP_R2 + +WEBP_DSP_INIT_STUB(WebPInitYUV444ConvertersMIPSdspR2) + +#endif // WEBP_USE_MIPS_DSP_R2 + +#if !(defined(FANCY_UPSAMPLING) && defined(WEBP_USE_MIPS_DSP_R2)) +WEBP_DSP_INIT_STUB(WebPInitUpsamplersMIPSdspR2) +#endif diff --git a/thirdparty/libwebp/dsp/upsampling_neon.c b/thirdparty/libwebp/dsp/upsampling_neon.c new file mode 100644 index 0000000000..2b0c99bddb --- /dev/null +++ b/thirdparty/libwebp/dsp/upsampling_neon.c @@ -0,0 +1,300 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// NEON version of YUV to RGB upsampling functions. +// +// Author: mans@mansr.com (Mans Rullgard) +// Based on SSE code by: somnath@google.com (Somnath Banerjee) + +#include "./dsp.h" + +#if defined(WEBP_USE_NEON) + +#include <assert.h> +#include <arm_neon.h> +#include <string.h> +#include "./neon.h" +#include "./yuv.h" + +#ifdef FANCY_UPSAMPLING + +//----------------------------------------------------------------------------- +// U/V upsampling + +// Loads 9 pixels each from rows r1 and r2 and generates 16 pixels. +#define UPSAMPLE_16PIXELS(r1, r2, out) { \ + uint8x8_t a = vld1_u8(r1); \ + uint8x8_t b = vld1_u8(r1 + 1); \ + uint8x8_t c = vld1_u8(r2); \ + uint8x8_t d = vld1_u8(r2 + 1); \ + \ + uint16x8_t al = vshll_n_u8(a, 1); \ + uint16x8_t bl = vshll_n_u8(b, 1); \ + uint16x8_t cl = vshll_n_u8(c, 1); \ + uint16x8_t dl = vshll_n_u8(d, 1); \ + \ + uint8x8_t diag1, diag2; \ + uint16x8_t sl; \ + \ + /* a + b + c + d */ \ + sl = vaddl_u8(a, b); \ + sl = vaddw_u8(sl, c); \ + sl = vaddw_u8(sl, d); \ + \ + al = vaddq_u16(sl, al); /* 3a + b + c + d */ \ + bl = vaddq_u16(sl, bl); /* a + 3b + c + d */ \ + \ + al = vaddq_u16(al, dl); /* 3a + b + c + 3d */ \ + bl = vaddq_u16(bl, cl); /* a + 3b + 3c + d */ \ + \ + diag2 = vshrn_n_u16(al, 3); \ + diag1 = vshrn_n_u16(bl, 3); \ + \ + a = vrhadd_u8(a, diag1); \ + b = vrhadd_u8(b, diag2); \ + c = vrhadd_u8(c, diag2); \ + d = vrhadd_u8(d, diag1); \ + \ + { \ + uint8x8x2_t a_b, c_d; \ + INIT_VECTOR2(a_b, a, b); \ + INIT_VECTOR2(c_d, c, d); \ + vst2_u8(out, a_b); \ + vst2_u8(out + 32, c_d); \ + } \ +} + +// Turn the macro into a function for reducing code-size when non-critical +static void Upsample16Pixels(const uint8_t *r1, const uint8_t *r2, + uint8_t *out) { + UPSAMPLE_16PIXELS(r1, r2, out); +} + +#define UPSAMPLE_LAST_BLOCK(tb, bb, num_pixels, out) { \ + uint8_t r1[9], r2[9]; \ + memcpy(r1, (tb), (num_pixels)); \ + memcpy(r2, (bb), (num_pixels)); \ + /* replicate last byte */ \ + memset(r1 + (num_pixels), r1[(num_pixels) - 1], 9 - (num_pixels)); \ + memset(r2 + (num_pixels), r2[(num_pixels) - 1], 9 - (num_pixels)); \ + Upsample16Pixels(r1, r2, out); \ +} + +//----------------------------------------------------------------------------- +// YUV->RGB conversion + +// note: we represent the 33050 large constant as 32768 + 282 +static const int16_t kCoeffs1[4] = { 19077, 26149, 6419, 13320 }; + +#define v255 vdup_n_u8(255) +#define v_0x0f vdup_n_u8(15) + +#define STORE_Rgb(out, r, g, b) do { \ + uint8x8x3_t r_g_b; \ + INIT_VECTOR3(r_g_b, r, g, b); \ + vst3_u8(out, r_g_b); \ +} while (0) + +#define STORE_Bgr(out, r, g, b) do { \ + uint8x8x3_t b_g_r; \ + INIT_VECTOR3(b_g_r, b, g, r); \ + vst3_u8(out, b_g_r); \ +} while (0) + +#define STORE_Rgba(out, r, g, b) do { \ + uint8x8x4_t r_g_b_v255; \ + INIT_VECTOR4(r_g_b_v255, r, g, b, v255); \ + vst4_u8(out, r_g_b_v255); \ +} while (0) + +#define STORE_Bgra(out, r, g, b) do { \ + uint8x8x4_t b_g_r_v255; \ + INIT_VECTOR4(b_g_r_v255, b, g, r, v255); \ + vst4_u8(out, b_g_r_v255); \ +} while (0) + +#define STORE_Argb(out, r, g, b) do { \ + uint8x8x4_t v255_r_g_b; \ + INIT_VECTOR4(v255_r_g_b, v255, r, g, b); \ + vst4_u8(out, v255_r_g_b); \ +} while (0) + +#if !defined(WEBP_SWAP_16BIT_CSP) +#define ZIP_U8(lo, hi) vzip_u8((lo), (hi)) +#else +#define ZIP_U8(lo, hi) vzip_u8((hi), (lo)) +#endif + +#define STORE_Rgba4444(out, r, g, b) do { \ + const uint8x8_t r1 = vshl_n_u8(vshr_n_u8(r, 4), 4); /* 4bits */ \ + const uint8x8_t g1 = vshr_n_u8(g, 4); \ + const uint8x8_t ba = vorr_u8(b, v_0x0f); \ + const uint8x8_t rg = vorr_u8(r1, g1); \ + const uint8x8x2_t rgba4444 = ZIP_U8(rg, ba); \ + vst1q_u8(out, vcombine_u8(rgba4444.val[0], rgba4444.val[1])); \ +} while (0) + +#define STORE_Rgb565(out, r, g, b) do { \ + const uint8x8_t r1 = vshl_n_u8(vshr_n_u8(r, 3), 3); /* 5bits */ \ + const uint8x8_t g1 = vshr_n_u8(g, 5); /* upper 3bits */\ + const uint8x8_t g2 = vshl_n_u8(vshr_n_u8(g, 2), 5); /* lower 3bits */\ + const uint8x8_t b1 = vshr_n_u8(b, 3); /* 5bits */ \ + const uint8x8_t rg = vorr_u8(r1, g1); \ + const uint8x8_t gb = vorr_u8(g2, b1); \ + const uint8x8x2_t rgb565 = ZIP_U8(rg, gb); \ + vst1q_u8(out, vcombine_u8(rgb565.val[0], rgb565.val[1])); \ +} while (0) + +#define CONVERT8(FMT, XSTEP, N, src_y, src_uv, out, cur_x) do { \ + int i; \ + for (i = 0; i < N; i += 8) { \ + const int off = ((cur_x) + i) * XSTEP; \ + const uint8x8_t y = vld1_u8((src_y) + (cur_x) + i); \ + const uint8x8_t u = vld1_u8((src_uv) + i + 0); \ + const uint8x8_t v = vld1_u8((src_uv) + i + 16); \ + const int16x8_t Y0 = vreinterpretq_s16_u16(vshll_n_u8(y, 7)); \ + const int16x8_t U0 = vreinterpretq_s16_u16(vshll_n_u8(u, 7)); \ + const int16x8_t V0 = vreinterpretq_s16_u16(vshll_n_u8(v, 7)); \ + const int16x8_t Y1 = vqdmulhq_lane_s16(Y0, coeff1, 0); \ + const int16x8_t R0 = vqdmulhq_lane_s16(V0, coeff1, 1); \ + const int16x8_t G0 = vqdmulhq_lane_s16(U0, coeff1, 2); \ + const int16x8_t G1 = vqdmulhq_lane_s16(V0, coeff1, 3); \ + const int16x8_t B0 = vqdmulhq_n_s16(U0, 282); \ + const int16x8_t R1 = vqaddq_s16(Y1, R_Rounder); \ + const int16x8_t G2 = vqaddq_s16(Y1, G_Rounder); \ + const int16x8_t B1 = vqaddq_s16(Y1, B_Rounder); \ + const int16x8_t R2 = vqaddq_s16(R0, R1); \ + const int16x8_t G3 = vqaddq_s16(G0, G1); \ + const int16x8_t B2 = vqaddq_s16(B0, B1); \ + const int16x8_t G4 = vqsubq_s16(G2, G3); \ + const int16x8_t B3 = vqaddq_s16(B2, U0); \ + const uint8x8_t R = vqshrun_n_s16(R2, YUV_FIX2); \ + const uint8x8_t G = vqshrun_n_s16(G4, YUV_FIX2); \ + const uint8x8_t B = vqshrun_n_s16(B3, YUV_FIX2); \ + STORE_ ## FMT(out + off, R, G, B); \ + } \ +} while (0) + +#define CONVERT1(FUNC, XSTEP, N, src_y, src_uv, rgb, cur_x) { \ + int i; \ + for (i = 0; i < N; i++) { \ + const int off = ((cur_x) + i) * XSTEP; \ + const int y = src_y[(cur_x) + i]; \ + const int u = (src_uv)[i]; \ + const int v = (src_uv)[i + 16]; \ + FUNC(y, u, v, rgb + off); \ + } \ +} + +#define CONVERT2RGB_8(FMT, XSTEP, top_y, bottom_y, uv, \ + top_dst, bottom_dst, cur_x, len) { \ + CONVERT8(FMT, XSTEP, len, top_y, uv, top_dst, cur_x); \ + if (bottom_y != NULL) { \ + CONVERT8(FMT, XSTEP, len, bottom_y, (uv) + 32, bottom_dst, cur_x); \ + } \ +} + +#define CONVERT2RGB_1(FUNC, XSTEP, top_y, bottom_y, uv, \ + top_dst, bottom_dst, cur_x, len) { \ + CONVERT1(FUNC, XSTEP, len, top_y, uv, top_dst, cur_x); \ + if (bottom_y != NULL) { \ + CONVERT1(FUNC, XSTEP, len, bottom_y, (uv) + 32, bottom_dst, cur_x); \ + } \ +} + +#define NEON_UPSAMPLE_FUNC(FUNC_NAME, FMT, XSTEP) \ +static void FUNC_NAME(const uint8_t *top_y, const uint8_t *bottom_y, \ + const uint8_t *top_u, const uint8_t *top_v, \ + const uint8_t *cur_u, const uint8_t *cur_v, \ + uint8_t *top_dst, uint8_t *bottom_dst, int len) { \ + int block; \ + /* 16 byte aligned array to cache reconstructed u and v */ \ + uint8_t uv_buf[2 * 32 + 15]; \ + uint8_t *const r_uv = (uint8_t*)((uintptr_t)(uv_buf + 15) & ~15); \ + const int uv_len = (len + 1) >> 1; \ + /* 9 pixels must be read-able for each block */ \ + const int num_blocks = (uv_len - 1) >> 3; \ + const int leftover = uv_len - num_blocks * 8; \ + const int last_pos = 1 + 16 * num_blocks; \ + \ + const int u_diag = ((top_u[0] + cur_u[0]) >> 1) + 1; \ + const int v_diag = ((top_v[0] + cur_v[0]) >> 1) + 1; \ + \ + const int16x4_t coeff1 = vld1_s16(kCoeffs1); \ + const int16x8_t R_Rounder = vdupq_n_s16(-14234); \ + const int16x8_t G_Rounder = vdupq_n_s16(8708); \ + const int16x8_t B_Rounder = vdupq_n_s16(-17685); \ + \ + /* Treat the first pixel in regular way */ \ + assert(top_y != NULL); \ + { \ + const int u0 = (top_u[0] + u_diag) >> 1; \ + const int v0 = (top_v[0] + v_diag) >> 1; \ + VP8YuvTo ## FMT(top_y[0], u0, v0, top_dst); \ + } \ + if (bottom_y != NULL) { \ + const int u0 = (cur_u[0] + u_diag) >> 1; \ + const int v0 = (cur_v[0] + v_diag) >> 1; \ + VP8YuvTo ## FMT(bottom_y[0], u0, v0, bottom_dst); \ + } \ + \ + for (block = 0; block < num_blocks; ++block) { \ + UPSAMPLE_16PIXELS(top_u, cur_u, r_uv); \ + UPSAMPLE_16PIXELS(top_v, cur_v, r_uv + 16); \ + CONVERT2RGB_8(FMT, XSTEP, top_y, bottom_y, r_uv, \ + top_dst, bottom_dst, 16 * block + 1, 16); \ + top_u += 8; \ + cur_u += 8; \ + top_v += 8; \ + cur_v += 8; \ + } \ + \ + UPSAMPLE_LAST_BLOCK(top_u, cur_u, leftover, r_uv); \ + UPSAMPLE_LAST_BLOCK(top_v, cur_v, leftover, r_uv + 16); \ + CONVERT2RGB_1(VP8YuvTo ## FMT, XSTEP, top_y, bottom_y, r_uv, \ + top_dst, bottom_dst, last_pos, len - last_pos); \ +} + +// NEON variants of the fancy upsampler. +NEON_UPSAMPLE_FUNC(UpsampleRgbLinePair, Rgb, 3) +NEON_UPSAMPLE_FUNC(UpsampleBgrLinePair, Bgr, 3) +NEON_UPSAMPLE_FUNC(UpsampleRgbaLinePair, Rgba, 4) +NEON_UPSAMPLE_FUNC(UpsampleBgraLinePair, Bgra, 4) +NEON_UPSAMPLE_FUNC(UpsampleArgbLinePair, Argb, 4) +NEON_UPSAMPLE_FUNC(UpsampleRgba4444LinePair, Rgba4444, 2) +NEON_UPSAMPLE_FUNC(UpsampleRgb565LinePair, Rgb565, 2) + +//------------------------------------------------------------------------------ +// Entry point + +extern WebPUpsampleLinePairFunc WebPUpsamplers[/* MODE_LAST */]; + +extern void WebPInitUpsamplersNEON(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitUpsamplersNEON(void) { + WebPUpsamplers[MODE_RGB] = UpsampleRgbLinePair; + WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePair; + WebPUpsamplers[MODE_BGR] = UpsampleBgrLinePair; + WebPUpsamplers[MODE_BGRA] = UpsampleBgraLinePair; + WebPUpsamplers[MODE_ARGB] = UpsampleArgbLinePair; + WebPUpsamplers[MODE_rgbA] = UpsampleRgbaLinePair; + WebPUpsamplers[MODE_bgrA] = UpsampleBgraLinePair; + WebPUpsamplers[MODE_Argb] = UpsampleArgbLinePair; + WebPUpsamplers[MODE_RGB_565] = UpsampleRgb565LinePair; + WebPUpsamplers[MODE_RGBA_4444] = UpsampleRgba4444LinePair; + WebPUpsamplers[MODE_rgbA_4444] = UpsampleRgba4444LinePair; +} + +#endif // FANCY_UPSAMPLING + +#endif // WEBP_USE_NEON + +#if !(defined(FANCY_UPSAMPLING) && defined(WEBP_USE_NEON)) +WEBP_DSP_INIT_STUB(WebPInitUpsamplersNEON) +#endif diff --git a/thirdparty/libwebp/dsp/upsampling_sse2.c b/thirdparty/libwebp/dsp/upsampling_sse2.c new file mode 100644 index 0000000000..b5b668900f --- /dev/null +++ b/thirdparty/libwebp/dsp/upsampling_sse2.c @@ -0,0 +1,249 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE2 version of YUV to RGB upsampling functions. +// +// Author: somnath@google.com (Somnath Banerjee) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE2) + +#include <assert.h> +#include <emmintrin.h> +#include <string.h> +#include "./yuv.h" + +#ifdef FANCY_UPSAMPLING + +// We compute (9*a + 3*b + 3*c + d + 8) / 16 as follows +// u = (9*a + 3*b + 3*c + d + 8) / 16 +// = (a + (a + 3*b + 3*c + d) / 8 + 1) / 2 +// = (a + m + 1) / 2 +// where m = (a + 3*b + 3*c + d) / 8 +// = ((a + b + c + d) / 2 + b + c) / 4 +// +// Let's say k = (a + b + c + d) / 4. +// We can compute k as +// k = (s + t + 1) / 2 - ((a^d) | (b^c) | (s^t)) & 1 +// where s = (a + d + 1) / 2 and t = (b + c + 1) / 2 +// +// Then m can be written as +// m = (k + t + 1) / 2 - (((b^c) & (s^t)) | (k^t)) & 1 + +// Computes out = (k + in + 1) / 2 - ((ij & (s^t)) | (k^in)) & 1 +#define GET_M(ij, in, out) do { \ + const __m128i tmp0 = _mm_avg_epu8(k, (in)); /* (k + in + 1) / 2 */ \ + const __m128i tmp1 = _mm_and_si128((ij), st); /* (ij) & (s^t) */ \ + const __m128i tmp2 = _mm_xor_si128(k, (in)); /* (k^in) */ \ + const __m128i tmp3 = _mm_or_si128(tmp1, tmp2); /* ((ij) & (s^t)) | (k^in) */\ + const __m128i tmp4 = _mm_and_si128(tmp3, one); /* & 1 -> lsb_correction */ \ + (out) = _mm_sub_epi8(tmp0, tmp4); /* (k + in + 1) / 2 - lsb_correction */ \ +} while (0) + +// pack and store two alternating pixel rows +#define PACK_AND_STORE(a, b, da, db, out) do { \ + const __m128i t_a = _mm_avg_epu8(a, da); /* (9a + 3b + 3c + d + 8) / 16 */ \ + const __m128i t_b = _mm_avg_epu8(b, db); /* (3a + 9b + c + 3d + 8) / 16 */ \ + const __m128i t_1 = _mm_unpacklo_epi8(t_a, t_b); \ + const __m128i t_2 = _mm_unpackhi_epi8(t_a, t_b); \ + _mm_store_si128(((__m128i*)(out)) + 0, t_1); \ + _mm_store_si128(((__m128i*)(out)) + 1, t_2); \ +} while (0) + +// Loads 17 pixels each from rows r1 and r2 and generates 32 pixels. +#define UPSAMPLE_32PIXELS(r1, r2, out) { \ + const __m128i one = _mm_set1_epi8(1); \ + const __m128i a = _mm_loadu_si128((const __m128i*)&(r1)[0]); \ + const __m128i b = _mm_loadu_si128((const __m128i*)&(r1)[1]); \ + const __m128i c = _mm_loadu_si128((const __m128i*)&(r2)[0]); \ + const __m128i d = _mm_loadu_si128((const __m128i*)&(r2)[1]); \ + \ + const __m128i s = _mm_avg_epu8(a, d); /* s = (a + d + 1) / 2 */ \ + const __m128i t = _mm_avg_epu8(b, c); /* t = (b + c + 1) / 2 */ \ + const __m128i st = _mm_xor_si128(s, t); /* st = s^t */ \ + \ + const __m128i ad = _mm_xor_si128(a, d); /* ad = a^d */ \ + const __m128i bc = _mm_xor_si128(b, c); /* bc = b^c */ \ + \ + const __m128i t1 = _mm_or_si128(ad, bc); /* (a^d) | (b^c) */ \ + const __m128i t2 = _mm_or_si128(t1, st); /* (a^d) | (b^c) | (s^t) */ \ + const __m128i t3 = _mm_and_si128(t2, one); /* (a^d) | (b^c) | (s^t) & 1 */ \ + const __m128i t4 = _mm_avg_epu8(s, t); \ + const __m128i k = _mm_sub_epi8(t4, t3); /* k = (a + b + c + d) / 4 */ \ + __m128i diag1, diag2; \ + \ + GET_M(bc, t, diag1); /* diag1 = (a + 3b + 3c + d) / 8 */ \ + GET_M(ad, s, diag2); /* diag2 = (3a + b + c + 3d) / 8 */ \ + \ + /* pack the alternate pixels */ \ + PACK_AND_STORE(a, b, diag1, diag2, out + 0); /* store top */ \ + PACK_AND_STORE(c, d, diag2, diag1, out + 2 * 32); /* store bottom */ \ +} + +// Turn the macro into a function for reducing code-size when non-critical +static void Upsample32Pixels(const uint8_t r1[], const uint8_t r2[], + uint8_t* const out) { + UPSAMPLE_32PIXELS(r1, r2, out); +} + +#define UPSAMPLE_LAST_BLOCK(tb, bb, num_pixels, out) { \ + uint8_t r1[17], r2[17]; \ + memcpy(r1, (tb), (num_pixels)); \ + memcpy(r2, (bb), (num_pixels)); \ + /* replicate last byte */ \ + memset(r1 + (num_pixels), r1[(num_pixels) - 1], 17 - (num_pixels)); \ + memset(r2 + (num_pixels), r2[(num_pixels) - 1], 17 - (num_pixels)); \ + /* using the shared function instead of the macro saves ~3k code size */ \ + Upsample32Pixels(r1, r2, out); \ +} + +#define CONVERT2RGB(FUNC, XSTEP, top_y, bottom_y, \ + top_dst, bottom_dst, cur_x, num_pixels) { \ + int n; \ + for (n = 0; n < (num_pixels); ++n) { \ + FUNC(top_y[(cur_x) + n], r_u[n], r_v[n], \ + top_dst + ((cur_x) + n) * XSTEP); \ + } \ + if (bottom_y != NULL) { \ + for (n = 0; n < (num_pixels); ++n) { \ + FUNC(bottom_y[(cur_x) + n], r_u[64 + n], r_v[64 + n], \ + bottom_dst + ((cur_x) + n) * XSTEP); \ + } \ + } \ +} + +#define CONVERT2RGB_32(FUNC, XSTEP, top_y, bottom_y, \ + top_dst, bottom_dst, cur_x) do { \ + FUNC##32(top_y + (cur_x), r_u, r_v, top_dst + (cur_x) * XSTEP); \ + if (bottom_y != NULL) { \ + FUNC##32(bottom_y + (cur_x), r_u + 64, r_v + 64, \ + bottom_dst + (cur_x) * XSTEP); \ + } \ +} while (0) + +#define SSE2_UPSAMPLE_FUNC(FUNC_NAME, FUNC, XSTEP) \ +static void FUNC_NAME(const uint8_t* top_y, const uint8_t* bottom_y, \ + const uint8_t* top_u, const uint8_t* top_v, \ + const uint8_t* cur_u, const uint8_t* cur_v, \ + uint8_t* top_dst, uint8_t* bottom_dst, int len) { \ + int uv_pos, pos; \ + /* 16byte-aligned array to cache reconstructed u and v */ \ + uint8_t uv_buf[4 * 32 + 15]; \ + uint8_t* const r_u = (uint8_t*)((uintptr_t)(uv_buf + 15) & ~15); \ + uint8_t* const r_v = r_u + 32; \ + \ + assert(top_y != NULL); \ + { /* Treat the first pixel in regular way */ \ + const int u_diag = ((top_u[0] + cur_u[0]) >> 1) + 1; \ + const int v_diag = ((top_v[0] + cur_v[0]) >> 1) + 1; \ + const int u0_t = (top_u[0] + u_diag) >> 1; \ + const int v0_t = (top_v[0] + v_diag) >> 1; \ + FUNC(top_y[0], u0_t, v0_t, top_dst); \ + if (bottom_y != NULL) { \ + const int u0_b = (cur_u[0] + u_diag) >> 1; \ + const int v0_b = (cur_v[0] + v_diag) >> 1; \ + FUNC(bottom_y[0], u0_b, v0_b, bottom_dst); \ + } \ + } \ + /* For UPSAMPLE_32PIXELS, 17 u/v values must be read-able for each block */ \ + for (pos = 1, uv_pos = 0; pos + 32 + 1 <= len; pos += 32, uv_pos += 16) { \ + UPSAMPLE_32PIXELS(top_u + uv_pos, cur_u + uv_pos, r_u); \ + UPSAMPLE_32PIXELS(top_v + uv_pos, cur_v + uv_pos, r_v); \ + CONVERT2RGB_32(FUNC, XSTEP, top_y, bottom_y, top_dst, bottom_dst, pos); \ + } \ + if (len > 1) { \ + const int left_over = ((len + 1) >> 1) - (pos >> 1); \ + assert(left_over > 0); \ + UPSAMPLE_LAST_BLOCK(top_u + uv_pos, cur_u + uv_pos, left_over, r_u); \ + UPSAMPLE_LAST_BLOCK(top_v + uv_pos, cur_v + uv_pos, left_over, r_v); \ + CONVERT2RGB(FUNC, XSTEP, top_y, bottom_y, top_dst, bottom_dst, \ + pos, len - pos); \ + } \ +} + +// SSE2 variants of the fancy upsampler. +SSE2_UPSAMPLE_FUNC(UpsampleRgbLinePair, VP8YuvToRgb, 3) +SSE2_UPSAMPLE_FUNC(UpsampleBgrLinePair, VP8YuvToBgr, 3) +SSE2_UPSAMPLE_FUNC(UpsampleRgbaLinePair, VP8YuvToRgba, 4) +SSE2_UPSAMPLE_FUNC(UpsampleBgraLinePair, VP8YuvToBgra, 4) +SSE2_UPSAMPLE_FUNC(UpsampleArgbLinePair, VP8YuvToArgb, 4) +SSE2_UPSAMPLE_FUNC(UpsampleRgba4444LinePair, VP8YuvToRgba4444, 2) +SSE2_UPSAMPLE_FUNC(UpsampleRgb565LinePair, VP8YuvToRgb565, 2) + +#undef GET_M +#undef PACK_AND_STORE +#undef UPSAMPLE_32PIXELS +#undef UPSAMPLE_LAST_BLOCK +#undef CONVERT2RGB +#undef CONVERT2RGB_32 +#undef SSE2_UPSAMPLE_FUNC + +//------------------------------------------------------------------------------ +// Entry point + +extern WebPUpsampleLinePairFunc WebPUpsamplers[/* MODE_LAST */]; + +extern void WebPInitUpsamplersSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitUpsamplersSSE2(void) { + WebPUpsamplers[MODE_RGB] = UpsampleRgbLinePair; + WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePair; + WebPUpsamplers[MODE_BGR] = UpsampleBgrLinePair; + WebPUpsamplers[MODE_BGRA] = UpsampleBgraLinePair; + WebPUpsamplers[MODE_ARGB] = UpsampleArgbLinePair; + WebPUpsamplers[MODE_rgbA] = UpsampleRgbaLinePair; + WebPUpsamplers[MODE_bgrA] = UpsampleBgraLinePair; + WebPUpsamplers[MODE_Argb] = UpsampleArgbLinePair; + WebPUpsamplers[MODE_RGB_565] = UpsampleRgb565LinePair; + WebPUpsamplers[MODE_RGBA_4444] = UpsampleRgba4444LinePair; + WebPUpsamplers[MODE_rgbA_4444] = UpsampleRgba4444LinePair; +} + +#endif // FANCY_UPSAMPLING + +//------------------------------------------------------------------------------ + +extern WebPYUV444Converter WebPYUV444Converters[/* MODE_LAST */]; +extern void WebPInitYUV444ConvertersSSE2(void); + +#define YUV444_FUNC(FUNC_NAME, CALL, XSTEP) \ +extern void WebP##FUNC_NAME##C(const uint8_t* y, const uint8_t* u, \ + const uint8_t* v, uint8_t* dst, int len); \ +static void FUNC_NAME(const uint8_t* y, const uint8_t* u, const uint8_t* v, \ + uint8_t* dst, int len) { \ + int i; \ + const int max_len = len & ~31; \ + for (i = 0; i < max_len; i += 32) CALL(y + i, u + i, v + i, dst + i * XSTEP);\ + if (i < len) { /* C-fallback */ \ + WebP##FUNC_NAME##C(y + i, u + i, v + i, dst + i * XSTEP, len - i); \ + } \ +} + +YUV444_FUNC(Yuv444ToRgba, VP8YuvToRgba32, 4); +YUV444_FUNC(Yuv444ToBgra, VP8YuvToBgra32, 4); +YUV444_FUNC(Yuv444ToRgb, VP8YuvToRgb32, 3); +YUV444_FUNC(Yuv444ToBgr, VP8YuvToBgr32, 3); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitYUV444ConvertersSSE2(void) { + WebPYUV444Converters[MODE_RGBA] = Yuv444ToRgba; + WebPYUV444Converters[MODE_BGRA] = Yuv444ToBgra; + WebPYUV444Converters[MODE_RGB] = Yuv444ToRgb; + WebPYUV444Converters[MODE_BGR] = Yuv444ToBgr; +} + +#else + +WEBP_DSP_INIT_STUB(WebPInitYUV444ConvertersSSE2) + +#endif // WEBP_USE_SSE2 + +#if !(defined(FANCY_UPSAMPLING) && defined(WEBP_USE_SSE2)) +WEBP_DSP_INIT_STUB(WebPInitUpsamplersSSE2) +#endif diff --git a/thirdparty/libwebp/dsp/yuv.c b/thirdparty/libwebp/dsp/yuv.c new file mode 100644 index 0000000000..f50a253168 --- /dev/null +++ b/thirdparty/libwebp/dsp/yuv.c @@ -0,0 +1,280 @@ +// Copyright 2010 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// YUV->RGB conversion functions +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./yuv.h" + +#if defined(WEBP_YUV_USE_TABLE) + +static int done = 0; + +static WEBP_INLINE uint8_t clip(int v, int max_value) { + return v < 0 ? 0 : v > max_value ? max_value : v; +} + +int16_t VP8kVToR[256], VP8kUToB[256]; +int32_t VP8kVToG[256], VP8kUToG[256]; +uint8_t VP8kClip[YUV_RANGE_MAX - YUV_RANGE_MIN]; +uint8_t VP8kClip4Bits[YUV_RANGE_MAX - YUV_RANGE_MIN]; + +WEBP_TSAN_IGNORE_FUNCTION void VP8YUVInit(void) { + int i; + if (done) { + return; + } +#ifndef USE_YUVj + for (i = 0; i < 256; ++i) { + VP8kVToR[i] = (89858 * (i - 128) + YUV_HALF) >> YUV_FIX; + VP8kUToG[i] = -22014 * (i - 128) + YUV_HALF; + VP8kVToG[i] = -45773 * (i - 128); + VP8kUToB[i] = (113618 * (i - 128) + YUV_HALF) >> YUV_FIX; + } + for (i = YUV_RANGE_MIN; i < YUV_RANGE_MAX; ++i) { + const int k = ((i - 16) * 76283 + YUV_HALF) >> YUV_FIX; + VP8kClip[i - YUV_RANGE_MIN] = clip(k, 255); + VP8kClip4Bits[i - YUV_RANGE_MIN] = clip((k + 8) >> 4, 15); + } +#else + for (i = 0; i < 256; ++i) { + VP8kVToR[i] = (91881 * (i - 128) + YUV_HALF) >> YUV_FIX; + VP8kUToG[i] = -22554 * (i - 128) + YUV_HALF; + VP8kVToG[i] = -46802 * (i - 128); + VP8kUToB[i] = (116130 * (i - 128) + YUV_HALF) >> YUV_FIX; + } + for (i = YUV_RANGE_MIN; i < YUV_RANGE_MAX; ++i) { + const int k = i; + VP8kClip[i - YUV_RANGE_MIN] = clip(k, 255); + VP8kClip4Bits[i - YUV_RANGE_MIN] = clip((k + 8) >> 4, 15); + } +#endif + + done = 1; +} + +#else + +WEBP_TSAN_IGNORE_FUNCTION void VP8YUVInit(void) {} + +#endif // WEBP_YUV_USE_TABLE + +//----------------------------------------------------------------------------- +// Plain-C version + +#define ROW_FUNC(FUNC_NAME, FUNC, XSTEP) \ +static void FUNC_NAME(const uint8_t* y, \ + const uint8_t* u, const uint8_t* v, \ + uint8_t* dst, int len) { \ + const uint8_t* const end = dst + (len & ~1) * XSTEP; \ + while (dst != end) { \ + FUNC(y[0], u[0], v[0], dst); \ + FUNC(y[1], u[0], v[0], dst + XSTEP); \ + y += 2; \ + ++u; \ + ++v; \ + dst += 2 * XSTEP; \ + } \ + if (len & 1) { \ + FUNC(y[0], u[0], v[0], dst); \ + } \ +} \ + +// All variants implemented. +ROW_FUNC(YuvToRgbRow, VP8YuvToRgb, 3) +ROW_FUNC(YuvToBgrRow, VP8YuvToBgr, 3) +ROW_FUNC(YuvToRgbaRow, VP8YuvToRgba, 4) +ROW_FUNC(YuvToBgraRow, VP8YuvToBgra, 4) +ROW_FUNC(YuvToArgbRow, VP8YuvToArgb, 4) +ROW_FUNC(YuvToRgba4444Row, VP8YuvToRgba4444, 2) +ROW_FUNC(YuvToRgb565Row, VP8YuvToRgb565, 2) + +#undef ROW_FUNC + +// Main call for processing a plane with a WebPSamplerRowFunc function: +void WebPSamplerProcessPlane(const uint8_t* y, int y_stride, + const uint8_t* u, const uint8_t* v, int uv_stride, + uint8_t* dst, int dst_stride, + int width, int height, WebPSamplerRowFunc func) { + int j; + for (j = 0; j < height; ++j) { + func(y, u, v, dst, width); + y += y_stride; + if (j & 1) { + u += uv_stride; + v += uv_stride; + } + dst += dst_stride; + } +} + +//----------------------------------------------------------------------------- +// Main call + +WebPSamplerRowFunc WebPSamplers[MODE_LAST]; + +extern void WebPInitSamplersSSE2(void); +extern void WebPInitSamplersMIPS32(void); +extern void WebPInitSamplersMIPSdspR2(void); + +static volatile VP8CPUInfo yuv_last_cpuinfo_used = + (VP8CPUInfo)&yuv_last_cpuinfo_used; + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitSamplers(void) { + if (yuv_last_cpuinfo_used == VP8GetCPUInfo) return; + + WebPSamplers[MODE_RGB] = YuvToRgbRow; + WebPSamplers[MODE_RGBA] = YuvToRgbaRow; + WebPSamplers[MODE_BGR] = YuvToBgrRow; + WebPSamplers[MODE_BGRA] = YuvToBgraRow; + WebPSamplers[MODE_ARGB] = YuvToArgbRow; + WebPSamplers[MODE_RGBA_4444] = YuvToRgba4444Row; + WebPSamplers[MODE_RGB_565] = YuvToRgb565Row; + WebPSamplers[MODE_rgbA] = YuvToRgbaRow; + WebPSamplers[MODE_bgrA] = YuvToBgraRow; + WebPSamplers[MODE_Argb] = YuvToArgbRow; + WebPSamplers[MODE_rgbA_4444] = YuvToRgba4444Row; + + // If defined, use CPUInfo() to overwrite some pointers with faster versions. + if (VP8GetCPUInfo != NULL) { +#if defined(WEBP_USE_SSE2) + if (VP8GetCPUInfo(kSSE2)) { + WebPInitSamplersSSE2(); + } +#endif // WEBP_USE_SSE2 +#if defined(WEBP_USE_MIPS32) + if (VP8GetCPUInfo(kMIPS32)) { + WebPInitSamplersMIPS32(); + } +#endif // WEBP_USE_MIPS32 +#if defined(WEBP_USE_MIPS_DSP_R2) + if (VP8GetCPUInfo(kMIPSdspR2)) { + WebPInitSamplersMIPSdspR2(); + } +#endif // WEBP_USE_MIPS_DSP_R2 + } + yuv_last_cpuinfo_used = VP8GetCPUInfo; +} + +//----------------------------------------------------------------------------- +// ARGB -> YUV converters + +static void ConvertARGBToY(const uint32_t* argb, uint8_t* y, int width) { + int i; + for (i = 0; i < width; ++i) { + const uint32_t p = argb[i]; + y[i] = VP8RGBToY((p >> 16) & 0xff, (p >> 8) & 0xff, (p >> 0) & 0xff, + YUV_HALF); + } +} + +void WebPConvertARGBToUV_C(const uint32_t* argb, uint8_t* u, uint8_t* v, + int src_width, int do_store) { + // No rounding. Last pixel is dealt with separately. + const int uv_width = src_width >> 1; + int i; + for (i = 0; i < uv_width; ++i) { + const uint32_t v0 = argb[2 * i + 0]; + const uint32_t v1 = argb[2 * i + 1]; + // VP8RGBToU/V expects four accumulated pixels. Hence we need to + // scale r/g/b value by a factor 2. We just shift v0/v1 one bit less. + const int r = ((v0 >> 15) & 0x1fe) + ((v1 >> 15) & 0x1fe); + const int g = ((v0 >> 7) & 0x1fe) + ((v1 >> 7) & 0x1fe); + const int b = ((v0 << 1) & 0x1fe) + ((v1 << 1) & 0x1fe); + const int tmp_u = VP8RGBToU(r, g, b, YUV_HALF << 2); + const int tmp_v = VP8RGBToV(r, g, b, YUV_HALF << 2); + if (do_store) { + u[i] = tmp_u; + v[i] = tmp_v; + } else { + // Approximated average-of-four. But it's an acceptable diff. + u[i] = (u[i] + tmp_u + 1) >> 1; + v[i] = (v[i] + tmp_v + 1) >> 1; + } + } + if (src_width & 1) { // last pixel + const uint32_t v0 = argb[2 * i + 0]; + const int r = (v0 >> 14) & 0x3fc; + const int g = (v0 >> 6) & 0x3fc; + const int b = (v0 << 2) & 0x3fc; + const int tmp_u = VP8RGBToU(r, g, b, YUV_HALF << 2); + const int tmp_v = VP8RGBToV(r, g, b, YUV_HALF << 2); + if (do_store) { + u[i] = tmp_u; + v[i] = tmp_v; + } else { + u[i] = (u[i] + tmp_u + 1) >> 1; + v[i] = (v[i] + tmp_v + 1) >> 1; + } + } +} + +//----------------------------------------------------------------------------- + +static void ConvertRGB24ToY(const uint8_t* rgb, uint8_t* y, int width) { + int i; + for (i = 0; i < width; ++i, rgb += 3) { + y[i] = VP8RGBToY(rgb[0], rgb[1], rgb[2], YUV_HALF); + } +} + +static void ConvertBGR24ToY(const uint8_t* bgr, uint8_t* y, int width) { + int i; + for (i = 0; i < width; ++i, bgr += 3) { + y[i] = VP8RGBToY(bgr[2], bgr[1], bgr[0], YUV_HALF); + } +} + +void WebPConvertRGBA32ToUV_C(const uint16_t* rgb, + uint8_t* u, uint8_t* v, int width) { + int i; + for (i = 0; i < width; i += 1, rgb += 4) { + const int r = rgb[0], g = rgb[1], b = rgb[2]; + u[i] = VP8RGBToU(r, g, b, YUV_HALF << 2); + v[i] = VP8RGBToV(r, g, b, YUV_HALF << 2); + } +} + +//----------------------------------------------------------------------------- + +void (*WebPConvertRGB24ToY)(const uint8_t* rgb, uint8_t* y, int width); +void (*WebPConvertBGR24ToY)(const uint8_t* bgr, uint8_t* y, int width); +void (*WebPConvertRGBA32ToUV)(const uint16_t* rgb, + uint8_t* u, uint8_t* v, int width); + +void (*WebPConvertARGBToY)(const uint32_t* argb, uint8_t* y, int width); +void (*WebPConvertARGBToUV)(const uint32_t* argb, uint8_t* u, uint8_t* v, + int src_width, int do_store); + +static volatile VP8CPUInfo rgba_to_yuv_last_cpuinfo_used = + (VP8CPUInfo)&rgba_to_yuv_last_cpuinfo_used; + +extern void WebPInitConvertARGBToYUVSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitConvertARGBToYUV(void) { + if (rgba_to_yuv_last_cpuinfo_used == VP8GetCPUInfo) return; + + WebPConvertARGBToY = ConvertARGBToY; + WebPConvertARGBToUV = WebPConvertARGBToUV_C; + + WebPConvertRGB24ToY = ConvertRGB24ToY; + WebPConvertBGR24ToY = ConvertBGR24ToY; + + WebPConvertRGBA32ToUV = WebPConvertRGBA32ToUV_C; + + if (VP8GetCPUInfo != NULL) { +#if defined(WEBP_USE_SSE2) + if (VP8GetCPUInfo(kSSE2)) { + WebPInitConvertARGBToYUVSSE2(); + } +#endif // WEBP_USE_SSE2 + } + rgba_to_yuv_last_cpuinfo_used = VP8GetCPUInfo; +} diff --git a/thirdparty/libwebp/dsp/yuv.h b/thirdparty/libwebp/dsp/yuv.h new file mode 100644 index 0000000000..01c40fcb84 --- /dev/null +++ b/thirdparty/libwebp/dsp/yuv.h @@ -0,0 +1,238 @@ +// Copyright 2010 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// inline YUV<->RGB conversion function +// +// The exact naming is Y'CbCr, following the ITU-R BT.601 standard. +// More information at: http://en.wikipedia.org/wiki/YCbCr +// Y = 0.2569 * R + 0.5044 * G + 0.0979 * B + 16 +// U = -0.1483 * R - 0.2911 * G + 0.4394 * B + 128 +// V = 0.4394 * R - 0.3679 * G - 0.0715 * B + 128 +// We use 16bit fixed point operations for RGB->YUV conversion (YUV_FIX). +// +// For the Y'CbCr to RGB conversion, the BT.601 specification reads: +// R = 1.164 * (Y-16) + 1.596 * (V-128) +// G = 1.164 * (Y-16) - 0.813 * (V-128) - 0.391 * (U-128) +// B = 1.164 * (Y-16) + 2.018 * (U-128) +// where Y is in the [16,235] range, and U/V in the [16,240] range. +// +// The fixed-point implementation used here is: +// R = (19077 . y + 26149 . v - 14234) >> 6 +// G = (19077 . y - 6419 . u - 13320 . v + 8708) >> 6 +// B = (19077 . y + 33050 . u - 17685) >> 6 +// where the '.' operator is the mulhi_epu16 variant: +// a . b = ((a << 8) * b) >> 16 +// that preserves 8 bits of fractional precision before final descaling. + +// Author: Skal (pascal.massimino@gmail.com) + +#ifndef WEBP_DSP_YUV_H_ +#define WEBP_DSP_YUV_H_ + +#include "./dsp.h" +#include "../dec/decode_vp8.h" + +#if defined(WEBP_EXPERIMENTAL_FEATURES) +// Do NOT activate this feature for real compression. This is only experimental! +// This flag is for comparison purpose against JPEG's "YUVj" natural colorspace. +// This colorspace is close to Rec.601's Y'CbCr model with the notable +// difference of allowing larger range for luma/chroma. +// See http://en.wikipedia.org/wiki/YCbCr#JPEG_conversion paragraph, and its +// difference with http://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion +// #define USE_YUVj +#endif + +//------------------------------------------------------------------------------ +// YUV -> RGB conversion + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + YUV_FIX = 16, // fixed-point precision for RGB->YUV + YUV_HALF = 1 << (YUV_FIX - 1), + YUV_MASK = (256 << YUV_FIX) - 1, + YUV_RANGE_MIN = -227, // min value of r/g/b output + YUV_RANGE_MAX = 256 + 226, // max value of r/g/b output + + YUV_FIX2 = 6, // fixed-point precision for YUV->RGB + YUV_HALF2 = 1 << YUV_FIX2 >> 1, + YUV_MASK2 = (256 << YUV_FIX2) - 1 +}; + +//------------------------------------------------------------------------------ +// slower on x86 by ~7-8%, but bit-exact with the SSE2/NEON version + +static WEBP_INLINE int MultHi(int v, int coeff) { // _mm_mulhi_epu16 emulation + return (v * coeff) >> 8; +} + +static WEBP_INLINE int VP8Clip8(int v) { + return ((v & ~YUV_MASK2) == 0) ? (v >> YUV_FIX2) : (v < 0) ? 0 : 255; +} + +static WEBP_INLINE int VP8YUVToR(int y, int v) { + return VP8Clip8(MultHi(y, 19077) + MultHi(v, 26149) - 14234); +} + +static WEBP_INLINE int VP8YUVToG(int y, int u, int v) { + return VP8Clip8(MultHi(y, 19077) - MultHi(u, 6419) - MultHi(v, 13320) + 8708); +} + +static WEBP_INLINE int VP8YUVToB(int y, int u) { + return VP8Clip8(MultHi(y, 19077) + MultHi(u, 33050) - 17685); +} + +static WEBP_INLINE void VP8YuvToRgb(int y, int u, int v, + uint8_t* const rgb) { + rgb[0] = VP8YUVToR(y, v); + rgb[1] = VP8YUVToG(y, u, v); + rgb[2] = VP8YUVToB(y, u); +} + +static WEBP_INLINE void VP8YuvToBgr(int y, int u, int v, + uint8_t* const bgr) { + bgr[0] = VP8YUVToB(y, u); + bgr[1] = VP8YUVToG(y, u, v); + bgr[2] = VP8YUVToR(y, v); +} + +static WEBP_INLINE void VP8YuvToRgb565(int y, int u, int v, + uint8_t* const rgb) { + const int r = VP8YUVToR(y, v); // 5 usable bits + const int g = VP8YUVToG(y, u, v); // 6 usable bits + const int b = VP8YUVToB(y, u); // 5 usable bits + const int rg = (r & 0xf8) | (g >> 5); + const int gb = ((g << 3) & 0xe0) | (b >> 3); +#ifdef WEBP_SWAP_16BIT_CSP + rgb[0] = gb; + rgb[1] = rg; +#else + rgb[0] = rg; + rgb[1] = gb; +#endif +} + +static WEBP_INLINE void VP8YuvToRgba4444(int y, int u, int v, + uint8_t* const argb) { + const int r = VP8YUVToR(y, v); // 4 usable bits + const int g = VP8YUVToG(y, u, v); // 4 usable bits + const int b = VP8YUVToB(y, u); // 4 usable bits + const int rg = (r & 0xf0) | (g >> 4); + const int ba = (b & 0xf0) | 0x0f; // overwrite the lower 4 bits +#ifdef WEBP_SWAP_16BIT_CSP + argb[0] = ba; + argb[1] = rg; +#else + argb[0] = rg; + argb[1] = ba; +#endif +} + +//----------------------------------------------------------------------------- +// Alpha handling variants + +static WEBP_INLINE void VP8YuvToArgb(uint8_t y, uint8_t u, uint8_t v, + uint8_t* const argb) { + argb[0] = 0xff; + VP8YuvToRgb(y, u, v, argb + 1); +} + +static WEBP_INLINE void VP8YuvToBgra(uint8_t y, uint8_t u, uint8_t v, + uint8_t* const bgra) { + VP8YuvToBgr(y, u, v, bgra); + bgra[3] = 0xff; +} + +static WEBP_INLINE void VP8YuvToRgba(uint8_t y, uint8_t u, uint8_t v, + uint8_t* const rgba) { + VP8YuvToRgb(y, u, v, rgba); + rgba[3] = 0xff; +} + +// Must be called before everything, to initialize the tables. +void VP8YUVInit(void); + +//----------------------------------------------------------------------------- +// SSE2 extra functions (mostly for upsampling_sse2.c) + +#if defined(WEBP_USE_SSE2) + +// Process 32 pixels and store the result (16b, 24b or 32b per pixel) in *dst. +void VP8YuvToRgba32(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst); +void VP8YuvToRgb32(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst); +void VP8YuvToBgra32(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst); +void VP8YuvToBgr32(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst); +void VP8YuvToArgb32(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst); +void VP8YuvToRgba444432(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst); +void VP8YuvToRgb56532(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst); + +#endif // WEBP_USE_SSE2 + +//------------------------------------------------------------------------------ +// RGB -> YUV conversion + +// Stub functions that can be called with various rounding values: +static WEBP_INLINE int VP8ClipUV(int uv, int rounding) { + uv = (uv + rounding + (128 << (YUV_FIX + 2))) >> (YUV_FIX + 2); + return ((uv & ~0xff) == 0) ? uv : (uv < 0) ? 0 : 255; +} + +#ifndef USE_YUVj + +static WEBP_INLINE int VP8RGBToY(int r, int g, int b, int rounding) { + const int luma = 16839 * r + 33059 * g + 6420 * b; + return (luma + rounding + (16 << YUV_FIX)) >> YUV_FIX; // no need to clip +} + +static WEBP_INLINE int VP8RGBToU(int r, int g, int b, int rounding) { + const int u = -9719 * r - 19081 * g + 28800 * b; + return VP8ClipUV(u, rounding); +} + +static WEBP_INLINE int VP8RGBToV(int r, int g, int b, int rounding) { + const int v = +28800 * r - 24116 * g - 4684 * b; + return VP8ClipUV(v, rounding); +} + +#else + +// This JPEG-YUV colorspace, only for comparison! +// These are also 16bit precision coefficients from Rec.601, but with full +// [0..255] output range. +static WEBP_INLINE int VP8RGBToY(int r, int g, int b, int rounding) { + const int luma = 19595 * r + 38470 * g + 7471 * b; + return (luma + rounding) >> YUV_FIX; // no need to clip +} + +static WEBP_INLINE int VP8RGBToU(int r, int g, int b, int rounding) { + const int u = -11058 * r - 21710 * g + 32768 * b; + return VP8ClipUV(u, rounding); +} + +static WEBP_INLINE int VP8RGBToV(int r, int g, int b, int rounding) { + const int v = 32768 * r - 27439 * g - 5329 * b; + return VP8ClipUV(v, rounding); +} + +#endif // USE_YUVj + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* WEBP_DSP_YUV_H_ */ diff --git a/thirdparty/libwebp/dsp/yuv_mips32.c b/thirdparty/libwebp/dsp/yuv_mips32.c new file mode 100644 index 0000000000..e61aac571f --- /dev/null +++ b/thirdparty/libwebp/dsp/yuv_mips32.c @@ -0,0 +1,103 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MIPS version of YUV to RGB upsampling functions. +// +// Author(s): Djordje Pesut (djordje.pesut@imgtec.com) +// Jovan Zelincevic (jovan.zelincevic@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS32) + +#include "./yuv.h" + +//------------------------------------------------------------------------------ +// simple point-sampling + +#define ROW_FUNC(FUNC_NAME, XSTEP, R, G, B, A) \ +static void FUNC_NAME(const uint8_t* y, \ + const uint8_t* u, const uint8_t* v, \ + uint8_t* dst, int len) { \ + int i, r, g, b; \ + int temp0, temp1, temp2, temp3, temp4; \ + for (i = 0; i < (len >> 1); i++) { \ + temp1 = MultHi(v[0], 26149); \ + temp3 = MultHi(v[0], 13320); \ + temp2 = MultHi(u[0], 6419); \ + temp4 = MultHi(u[0], 33050); \ + temp0 = MultHi(y[0], 19077); \ + temp1 -= 14234; \ + temp3 -= 8708; \ + temp2 += temp3; \ + temp4 -= 17685; \ + r = VP8Clip8(temp0 + temp1); \ + g = VP8Clip8(temp0 - temp2); \ + b = VP8Clip8(temp0 + temp4); \ + temp0 = MultHi(y[1], 19077); \ + dst[R] = r; \ + dst[G] = g; \ + dst[B] = b; \ + if (A) dst[A] = 0xff; \ + r = VP8Clip8(temp0 + temp1); \ + g = VP8Clip8(temp0 - temp2); \ + b = VP8Clip8(temp0 + temp4); \ + dst[R + XSTEP] = r; \ + dst[G + XSTEP] = g; \ + dst[B + XSTEP] = b; \ + if (A) dst[A + XSTEP] = 0xff; \ + y += 2; \ + ++u; \ + ++v; \ + dst += 2 * XSTEP; \ + } \ + if (len & 1) { \ + temp1 = MultHi(v[0], 26149); \ + temp3 = MultHi(v[0], 13320); \ + temp2 = MultHi(u[0], 6419); \ + temp4 = MultHi(u[0], 33050); \ + temp0 = MultHi(y[0], 19077); \ + temp1 -= 14234; \ + temp3 -= 8708; \ + temp2 += temp3; \ + temp4 -= 17685; \ + r = VP8Clip8(temp0 + temp1); \ + g = VP8Clip8(temp0 - temp2); \ + b = VP8Clip8(temp0 + temp4); \ + dst[R] = r; \ + dst[G] = g; \ + dst[B] = b; \ + if (A) dst[A] = 0xff; \ + } \ +} + +ROW_FUNC(YuvToRgbRow, 3, 0, 1, 2, 0) +ROW_FUNC(YuvToRgbaRow, 4, 0, 1, 2, 3) +ROW_FUNC(YuvToBgrRow, 3, 2, 1, 0, 0) +ROW_FUNC(YuvToBgraRow, 4, 2, 1, 0, 3) + +#undef ROW_FUNC + +//------------------------------------------------------------------------------ +// Entry point + +extern void WebPInitSamplersMIPS32(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitSamplersMIPS32(void) { + WebPSamplers[MODE_RGB] = YuvToRgbRow; + WebPSamplers[MODE_RGBA] = YuvToRgbaRow; + WebPSamplers[MODE_BGR] = YuvToBgrRow; + WebPSamplers[MODE_BGRA] = YuvToBgraRow; +} + +#else // !WEBP_USE_MIPS32 + +WEBP_DSP_INIT_STUB(WebPInitSamplersMIPS32) + +#endif // WEBP_USE_MIPS32 diff --git a/thirdparty/libwebp/dsp/yuv_mips_dsp_r2.c b/thirdparty/libwebp/dsp/yuv_mips_dsp_r2.c new file mode 100644 index 0000000000..1720d4190f --- /dev/null +++ b/thirdparty/libwebp/dsp/yuv_mips_dsp_r2.c @@ -0,0 +1,134 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MIPS DSPr2 version of YUV to RGB upsampling functions. +// +// Author(s): Branimir Vasic (branimir.vasic@imgtec.com) +// Djordje Pesut (djordje.pesut@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MIPS_DSP_R2) + +#include "./yuv.h" + +//------------------------------------------------------------------------------ +// simple point-sampling + +#define ROW_FUNC_PART_1() \ + "lbu %[temp3], 0(%[v]) \n\t" \ + "lbu %[temp4], 0(%[u]) \n\t" \ + "lbu %[temp0], 0(%[y]) \n\t" \ + "mul %[temp1], %[t_con_1], %[temp3] \n\t" \ + "mul %[temp3], %[t_con_2], %[temp3] \n\t" \ + "mul %[temp2], %[t_con_3], %[temp4] \n\t" \ + "mul %[temp4], %[t_con_4], %[temp4] \n\t" \ + "mul %[temp0], %[t_con_5], %[temp0] \n\t" \ + "subu %[temp1], %[temp1], %[t_con_6] \n\t" \ + "subu %[temp3], %[temp3], %[t_con_7] \n\t" \ + "addu %[temp2], %[temp2], %[temp3] \n\t" \ + "subu %[temp4], %[temp4], %[t_con_8] \n\t" \ + +#define ROW_FUNC_PART_2(R, G, B, K) \ + "addu %[temp5], %[temp0], %[temp1] \n\t" \ + "subu %[temp6], %[temp0], %[temp2] \n\t" \ + "addu %[temp7], %[temp0], %[temp4] \n\t" \ +".if " #K " \n\t" \ + "lbu %[temp0], 1(%[y]) \n\t" \ +".endif \n\t" \ + "shll_s.w %[temp5], %[temp5], 17 \n\t" \ + "shll_s.w %[temp6], %[temp6], 17 \n\t" \ +".if " #K " \n\t" \ + "mul %[temp0], %[t_con_5], %[temp0] \n\t" \ +".endif \n\t" \ + "shll_s.w %[temp7], %[temp7], 17 \n\t" \ + "precrqu_s.qb.ph %[temp5], %[temp5], $zero \n\t" \ + "precrqu_s.qb.ph %[temp6], %[temp6], $zero \n\t" \ + "precrqu_s.qb.ph %[temp7], %[temp7], $zero \n\t" \ + "srl %[temp5], %[temp5], 24 \n\t" \ + "srl %[temp6], %[temp6], 24 \n\t" \ + "srl %[temp7], %[temp7], 24 \n\t" \ + "sb %[temp5], " #R "(%[dst]) \n\t" \ + "sb %[temp6], " #G "(%[dst]) \n\t" \ + "sb %[temp7], " #B "(%[dst]) \n\t" \ + +#define ASM_CLOBBER_LIST() \ + : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), \ + [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), \ + [temp6]"=&r"(temp6), [temp7]"=&r"(temp7) \ + : [t_con_1]"r"(t_con_1), [t_con_2]"r"(t_con_2), [t_con_3]"r"(t_con_3), \ + [t_con_4]"r"(t_con_4), [t_con_5]"r"(t_con_5), [t_con_6]"r"(t_con_6), \ + [u]"r"(u), [v]"r"(v), [y]"r"(y), [dst]"r"(dst), \ + [t_con_7]"r"(t_con_7), [t_con_8]"r"(t_con_8) \ + : "memory", "hi", "lo" \ + +#define ROW_FUNC(FUNC_NAME, XSTEP, R, G, B, A) \ +static void FUNC_NAME(const uint8_t* y, \ + const uint8_t* u, const uint8_t* v, \ + uint8_t* dst, int len) { \ + int i; \ + uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; \ + const int t_con_1 = 26149; \ + const int t_con_2 = 13320; \ + const int t_con_3 = 6419; \ + const int t_con_4 = 33050; \ + const int t_con_5 = 19077; \ + const int t_con_6 = 14234; \ + const int t_con_7 = 8708; \ + const int t_con_8 = 17685; \ + for (i = 0; i < (len >> 1); i++) { \ + __asm__ volatile ( \ + ROW_FUNC_PART_1() \ + ROW_FUNC_PART_2(R, G, B, 1) \ + ROW_FUNC_PART_2(R + XSTEP, G + XSTEP, B + XSTEP, 0) \ + ASM_CLOBBER_LIST() \ + ); \ + if (A) dst[A] = dst[A + XSTEP] = 0xff; \ + y += 2; \ + ++u; \ + ++v; \ + dst += 2 * XSTEP; \ + } \ + if (len & 1) { \ + __asm__ volatile ( \ + ROW_FUNC_PART_1() \ + ROW_FUNC_PART_2(R, G, B, 0) \ + ASM_CLOBBER_LIST() \ + ); \ + if (A) dst[A] = 0xff; \ + } \ +} + +ROW_FUNC(YuvToRgbRow, 3, 0, 1, 2, 0) +ROW_FUNC(YuvToRgbaRow, 4, 0, 1, 2, 3) +ROW_FUNC(YuvToBgrRow, 3, 2, 1, 0, 0) +ROW_FUNC(YuvToBgraRow, 4, 2, 1, 0, 3) + +#undef ROW_FUNC +#undef ASM_CLOBBER_LIST +#undef ROW_FUNC_PART_2 +#undef ROW_FUNC_PART_1 + +//------------------------------------------------------------------------------ +// Entry point + +extern void WebPInitSamplersMIPSdspR2(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitSamplersMIPSdspR2(void) { + WebPSamplers[MODE_RGB] = YuvToRgbRow; + WebPSamplers[MODE_RGBA] = YuvToRgbaRow; + WebPSamplers[MODE_BGR] = YuvToBgrRow; + WebPSamplers[MODE_BGRA] = YuvToBgraRow; +} + +#else // !WEBP_USE_MIPS_DSP_R2 + +WEBP_DSP_INIT_STUB(WebPInitSamplersMIPSdspR2) + +#endif // WEBP_USE_MIPS_DSP_R2 diff --git a/thirdparty/libwebp/dsp/yuv_sse2.c b/thirdparty/libwebp/dsp/yuv_sse2.c new file mode 100644 index 0000000000..e19bddff6c --- /dev/null +++ b/thirdparty/libwebp/dsp/yuv_sse2.c @@ -0,0 +1,775 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// YUV->RGB conversion functions +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./yuv.h" + +#if defined(WEBP_USE_SSE2) + +#include <emmintrin.h> + +//----------------------------------------------------------------------------- +// Convert spans of 32 pixels to various RGB formats for the fancy upsampler. + +// These constants are 14b fixed-point version of ITU-R BT.601 constants. +// R = (19077 * y + 26149 * v - 14234) >> 6 +// G = (19077 * y - 6419 * u - 13320 * v + 8708) >> 6 +// B = (19077 * y + 33050 * u - 17685) >> 6 +static void ConvertYUV444ToRGB(const __m128i* const Y0, + const __m128i* const U0, + const __m128i* const V0, + __m128i* const R, + __m128i* const G, + __m128i* const B) { + const __m128i k19077 = _mm_set1_epi16(19077); + const __m128i k26149 = _mm_set1_epi16(26149); + const __m128i k14234 = _mm_set1_epi16(14234); + // 33050 doesn't fit in a signed short: only use this with unsigned arithmetic + const __m128i k33050 = _mm_set1_epi16((short)33050); + const __m128i k17685 = _mm_set1_epi16(17685); + const __m128i k6419 = _mm_set1_epi16(6419); + const __m128i k13320 = _mm_set1_epi16(13320); + const __m128i k8708 = _mm_set1_epi16(8708); + + const __m128i Y1 = _mm_mulhi_epu16(*Y0, k19077); + + const __m128i R0 = _mm_mulhi_epu16(*V0, k26149); + const __m128i R1 = _mm_sub_epi16(Y1, k14234); + const __m128i R2 = _mm_add_epi16(R1, R0); + + const __m128i G0 = _mm_mulhi_epu16(*U0, k6419); + const __m128i G1 = _mm_mulhi_epu16(*V0, k13320); + const __m128i G2 = _mm_add_epi16(Y1, k8708); + const __m128i G3 = _mm_add_epi16(G0, G1); + const __m128i G4 = _mm_sub_epi16(G2, G3); + + // be careful with the saturated *unsigned* arithmetic here! + const __m128i B0 = _mm_mulhi_epu16(*U0, k33050); + const __m128i B1 = _mm_adds_epu16(B0, Y1); + const __m128i B2 = _mm_subs_epu16(B1, k17685); + + // use logical shift for B2, which can be larger than 32767 + *R = _mm_srai_epi16(R2, 6); // range: [-14234, 30815] + *G = _mm_srai_epi16(G4, 6); // range: [-10953, 27710] + *B = _mm_srli_epi16(B2, 6); // range: [0, 34238] +} + +// Load the bytes into the *upper* part of 16b words. That's "<< 8", basically. +static WEBP_INLINE __m128i Load_HI_16(const uint8_t* src) { + const __m128i zero = _mm_setzero_si128(); + return _mm_unpacklo_epi8(zero, _mm_loadl_epi64((const __m128i*)src)); +} + +// Load and replicate the U/V samples +static WEBP_INLINE __m128i Load_UV_HI_8(const uint8_t* src) { + const __m128i zero = _mm_setzero_si128(); + const __m128i tmp0 = _mm_cvtsi32_si128(*(const uint32_t*)src); + const __m128i tmp1 = _mm_unpacklo_epi8(zero, tmp0); + return _mm_unpacklo_epi16(tmp1, tmp1); // replicate samples +} + +// Convert 32 samples of YUV444 to R/G/B +static void YUV444ToRGB(const uint8_t* const y, + const uint8_t* const u, + const uint8_t* const v, + __m128i* const R, __m128i* const G, __m128i* const B) { + const __m128i Y0 = Load_HI_16(y), U0 = Load_HI_16(u), V0 = Load_HI_16(v); + ConvertYUV444ToRGB(&Y0, &U0, &V0, R, G, B); +} + +// Convert 32 samples of YUV420 to R/G/B +static void YUV420ToRGB(const uint8_t* const y, + const uint8_t* const u, + const uint8_t* const v, + __m128i* const R, __m128i* const G, __m128i* const B) { + const __m128i Y0 = Load_HI_16(y), U0 = Load_UV_HI_8(u), V0 = Load_UV_HI_8(v); + ConvertYUV444ToRGB(&Y0, &U0, &V0, R, G, B); +} + +// Pack R/G/B/A results into 32b output. +static WEBP_INLINE void PackAndStore4(const __m128i* const R, + const __m128i* const G, + const __m128i* const B, + const __m128i* const A, + uint8_t* const dst) { + const __m128i rb = _mm_packus_epi16(*R, *B); + const __m128i ga = _mm_packus_epi16(*G, *A); + const __m128i rg = _mm_unpacklo_epi8(rb, ga); + const __m128i ba = _mm_unpackhi_epi8(rb, ga); + const __m128i RGBA_lo = _mm_unpacklo_epi16(rg, ba); + const __m128i RGBA_hi = _mm_unpackhi_epi16(rg, ba); + _mm_storeu_si128((__m128i*)(dst + 0), RGBA_lo); + _mm_storeu_si128((__m128i*)(dst + 16), RGBA_hi); +} + +// Pack R/G/B/A results into 16b output. +static WEBP_INLINE void PackAndStore4444(const __m128i* const R, + const __m128i* const G, + const __m128i* const B, + const __m128i* const A, + uint8_t* const dst) { +#if !defined(WEBP_SWAP_16BIT_CSP) + const __m128i rg0 = _mm_packus_epi16(*R, *G); + const __m128i ba0 = _mm_packus_epi16(*B, *A); +#else + const __m128i rg0 = _mm_packus_epi16(*B, *A); + const __m128i ba0 = _mm_packus_epi16(*R, *G); +#endif + const __m128i mask_0xf0 = _mm_set1_epi8(0xf0); + const __m128i rb1 = _mm_unpacklo_epi8(rg0, ba0); // rbrbrbrbrb... + const __m128i ga1 = _mm_unpackhi_epi8(rg0, ba0); // gagagagaga... + const __m128i rb2 = _mm_and_si128(rb1, mask_0xf0); + const __m128i ga2 = _mm_srli_epi16(_mm_and_si128(ga1, mask_0xf0), 4); + const __m128i rgba4444 = _mm_or_si128(rb2, ga2); + _mm_storeu_si128((__m128i*)dst, rgba4444); +} + +// Pack R/G/B results into 16b output. +static WEBP_INLINE void PackAndStore565(const __m128i* const R, + const __m128i* const G, + const __m128i* const B, + uint8_t* const dst) { + const __m128i r0 = _mm_packus_epi16(*R, *R); + const __m128i g0 = _mm_packus_epi16(*G, *G); + const __m128i b0 = _mm_packus_epi16(*B, *B); + const __m128i r1 = _mm_and_si128(r0, _mm_set1_epi8(0xf8)); + const __m128i b1 = _mm_and_si128(_mm_srli_epi16(b0, 3), _mm_set1_epi8(0x1f)); + const __m128i g1 = _mm_srli_epi16(_mm_and_si128(g0, _mm_set1_epi8(0xe0)), 5); + const __m128i g2 = _mm_slli_epi16(_mm_and_si128(g0, _mm_set1_epi8(0x1c)), 3); + const __m128i rg = _mm_or_si128(r1, g1); + const __m128i gb = _mm_or_si128(g2, b1); +#if !defined(WEBP_SWAP_16BIT_CSP) + const __m128i rgb565 = _mm_unpacklo_epi8(rg, gb); +#else + const __m128i rgb565 = _mm_unpacklo_epi8(gb, rg); +#endif + _mm_storeu_si128((__m128i*)dst, rgb565); +} + +// Function used several times in PlanarTo24b. +// It samples the in buffer as follows: one every two unsigned char is stored +// at the beginning of the buffer, while the other half is stored at the end. +static WEBP_INLINE void PlanarTo24bHelper(const __m128i* const in /*in[6]*/, + __m128i* const out /*out[6]*/) { + const __m128i v_mask = _mm_set1_epi16(0x00ff); + + // Take one every two upper 8b values. + out[0] = _mm_packus_epi16(_mm_and_si128(in[0], v_mask), + _mm_and_si128(in[1], v_mask)); + out[1] = _mm_packus_epi16(_mm_and_si128(in[2], v_mask), + _mm_and_si128(in[3], v_mask)); + out[2] = _mm_packus_epi16(_mm_and_si128(in[4], v_mask), + _mm_and_si128(in[5], v_mask)); + // Take one every two lower 8b values. + out[3] = _mm_packus_epi16(_mm_srli_epi16(in[0], 8), _mm_srli_epi16(in[1], 8)); + out[4] = _mm_packus_epi16(_mm_srli_epi16(in[2], 8), _mm_srli_epi16(in[3], 8)); + out[5] = _mm_packus_epi16(_mm_srli_epi16(in[4], 8), _mm_srli_epi16(in[5], 8)); +} + +// Pack the planar buffers +// rrrr... rrrr... gggg... gggg... bbbb... bbbb.... +// triplet by triplet in the output buffer rgb as rgbrgbrgbrgb ... +static WEBP_INLINE void PlanarTo24b(__m128i* const in /*in[6]*/, uint8_t* rgb) { + // The input is 6 registers of sixteen 8b but for the sake of explanation, + // let's take 6 registers of four 8b values. + // To pack, we will keep taking one every two 8b integer and move it + // around as follows: + // Input: + // r0r1r2r3 | r4r5r6r7 | g0g1g2g3 | g4g5g6g7 | b0b1b2b3 | b4b5b6b7 + // Split the 6 registers in two sets of 3 registers: the first set as the even + // 8b bytes, the second the odd ones: + // r0r2r4r6 | g0g2g4g6 | b0b2b4b6 | r1r3r5r7 | g1g3g5g7 | b1b3b5b7 + // Repeat the same permutations twice more: + // r0r4g0g4 | b0b4r1r5 | g1g5b1b5 | r2r6g2g6 | b2b6r3r7 | g3g7b3b7 + // r0g0b0r1 | g1b1r2g2 | b2r3g3b3 | r4g4b4r5 | g5b5r6g6 | b6r7g7b7 + __m128i tmp[6]; + PlanarTo24bHelper(in, tmp); + PlanarTo24bHelper(tmp, in); + PlanarTo24bHelper(in, tmp); + // We need to do it two more times than the example as we have sixteen bytes. + PlanarTo24bHelper(tmp, in); + PlanarTo24bHelper(in, tmp); + + _mm_storeu_si128((__m128i*)(rgb + 0), tmp[0]); + _mm_storeu_si128((__m128i*)(rgb + 16), tmp[1]); + _mm_storeu_si128((__m128i*)(rgb + 32), tmp[2]); + _mm_storeu_si128((__m128i*)(rgb + 48), tmp[3]); + _mm_storeu_si128((__m128i*)(rgb + 64), tmp[4]); + _mm_storeu_si128((__m128i*)(rgb + 80), tmp[5]); +} +#undef MK_UINT32 + +void VP8YuvToRgba32(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst) { + const __m128i kAlpha = _mm_set1_epi16(255); + int n; + for (n = 0; n < 32; n += 8, dst += 32) { + __m128i R, G, B; + YUV444ToRGB(y + n, u + n, v + n, &R, &G, &B); + PackAndStore4(&R, &G, &B, &kAlpha, dst); + } +} + +void VP8YuvToBgra32(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst) { + const __m128i kAlpha = _mm_set1_epi16(255); + int n; + for (n = 0; n < 32; n += 8, dst += 32) { + __m128i R, G, B; + YUV444ToRGB(y + n, u + n, v + n, &R, &G, &B); + PackAndStore4(&B, &G, &R, &kAlpha, dst); + } +} + +void VP8YuvToArgb32(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst) { + const __m128i kAlpha = _mm_set1_epi16(255); + int n; + for (n = 0; n < 32; n += 8, dst += 32) { + __m128i R, G, B; + YUV444ToRGB(y + n, u + n, v + n, &R, &G, &B); + PackAndStore4(&kAlpha, &R, &G, &B, dst); + } +} + +void VP8YuvToRgba444432(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst) { + const __m128i kAlpha = _mm_set1_epi16(255); + int n; + for (n = 0; n < 32; n += 8, dst += 16) { + __m128i R, G, B; + YUV444ToRGB(y + n, u + n, v + n, &R, &G, &B); + PackAndStore4444(&R, &G, &B, &kAlpha, dst); + } +} + +void VP8YuvToRgb56532(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst) { + int n; + for (n = 0; n < 32; n += 8, dst += 16) { + __m128i R, G, B; + YUV444ToRGB(y + n, u + n, v + n, &R, &G, &B); + PackAndStore565(&R, &G, &B, dst); + } +} + +void VP8YuvToRgb32(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst) { + __m128i R0, R1, R2, R3, G0, G1, G2, G3, B0, B1, B2, B3; + __m128i rgb[6]; + + YUV444ToRGB(y + 0, u + 0, v + 0, &R0, &G0, &B0); + YUV444ToRGB(y + 8, u + 8, v + 8, &R1, &G1, &B1); + YUV444ToRGB(y + 16, u + 16, v + 16, &R2, &G2, &B2); + YUV444ToRGB(y + 24, u + 24, v + 24, &R3, &G3, &B3); + + // Cast to 8b and store as RRRRGGGGBBBB. + rgb[0] = _mm_packus_epi16(R0, R1); + rgb[1] = _mm_packus_epi16(R2, R3); + rgb[2] = _mm_packus_epi16(G0, G1); + rgb[3] = _mm_packus_epi16(G2, G3); + rgb[4] = _mm_packus_epi16(B0, B1); + rgb[5] = _mm_packus_epi16(B2, B3); + + // Pack as RGBRGBRGBRGB. + PlanarTo24b(rgb, dst); +} + +void VP8YuvToBgr32(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst) { + __m128i R0, R1, R2, R3, G0, G1, G2, G3, B0, B1, B2, B3; + __m128i bgr[6]; + + YUV444ToRGB(y + 0, u + 0, v + 0, &R0, &G0, &B0); + YUV444ToRGB(y + 8, u + 8, v + 8, &R1, &G1, &B1); + YUV444ToRGB(y + 16, u + 16, v + 16, &R2, &G2, &B2); + YUV444ToRGB(y + 24, u + 24, v + 24, &R3, &G3, &B3); + + // Cast to 8b and store as BBBBGGGGRRRR. + bgr[0] = _mm_packus_epi16(B0, B1); + bgr[1] = _mm_packus_epi16(B2, B3); + bgr[2] = _mm_packus_epi16(G0, G1); + bgr[3] = _mm_packus_epi16(G2, G3); + bgr[4] = _mm_packus_epi16(R0, R1); + bgr[5] = _mm_packus_epi16(R2, R3); + + // Pack as BGRBGRBGRBGR. + PlanarTo24b(bgr, dst); +} + +//----------------------------------------------------------------------------- +// Arbitrary-length row conversion functions + +static void YuvToRgbaRow(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst, int len) { + const __m128i kAlpha = _mm_set1_epi16(255); + int n; + for (n = 0; n + 8 <= len; n += 8, dst += 32) { + __m128i R, G, B; + YUV420ToRGB(y, u, v, &R, &G, &B); + PackAndStore4(&R, &G, &B, &kAlpha, dst); + y += 8; + u += 4; + v += 4; + } + for (; n < len; ++n) { // Finish off + VP8YuvToRgba(y[0], u[0], v[0], dst); + dst += 4; + y += 1; + u += (n & 1); + v += (n & 1); + } +} + +static void YuvToBgraRow(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst, int len) { + const __m128i kAlpha = _mm_set1_epi16(255); + int n; + for (n = 0; n + 8 <= len; n += 8, dst += 32) { + __m128i R, G, B; + YUV420ToRGB(y, u, v, &R, &G, &B); + PackAndStore4(&B, &G, &R, &kAlpha, dst); + y += 8; + u += 4; + v += 4; + } + for (; n < len; ++n) { // Finish off + VP8YuvToBgra(y[0], u[0], v[0], dst); + dst += 4; + y += 1; + u += (n & 1); + v += (n & 1); + } +} + +static void YuvToArgbRow(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst, int len) { + const __m128i kAlpha = _mm_set1_epi16(255); + int n; + for (n = 0; n + 8 <= len; n += 8, dst += 32) { + __m128i R, G, B; + YUV420ToRGB(y, u, v, &R, &G, &B); + PackAndStore4(&kAlpha, &R, &G, &B, dst); + y += 8; + u += 4; + v += 4; + } + for (; n < len; ++n) { // Finish off + VP8YuvToArgb(y[0], u[0], v[0], dst); + dst += 4; + y += 1; + u += (n & 1); + v += (n & 1); + } +} + +static void YuvToRgbRow(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst, int len) { + int n; + for (n = 0; n + 32 <= len; n += 32, dst += 32 * 3) { + __m128i R0, R1, R2, R3, G0, G1, G2, G3, B0, B1, B2, B3; + __m128i rgb[6]; + + YUV420ToRGB(y + 0, u + 0, v + 0, &R0, &G0, &B0); + YUV420ToRGB(y + 8, u + 4, v + 4, &R1, &G1, &B1); + YUV420ToRGB(y + 16, u + 8, v + 8, &R2, &G2, &B2); + YUV420ToRGB(y + 24, u + 12, v + 12, &R3, &G3, &B3); + + // Cast to 8b and store as RRRRGGGGBBBB. + rgb[0] = _mm_packus_epi16(R0, R1); + rgb[1] = _mm_packus_epi16(R2, R3); + rgb[2] = _mm_packus_epi16(G0, G1); + rgb[3] = _mm_packus_epi16(G2, G3); + rgb[4] = _mm_packus_epi16(B0, B1); + rgb[5] = _mm_packus_epi16(B2, B3); + + // Pack as RGBRGBRGBRGB. + PlanarTo24b(rgb, dst); + + y += 32; + u += 16; + v += 16; + } + for (; n < len; ++n) { // Finish off + VP8YuvToRgb(y[0], u[0], v[0], dst); + dst += 3; + y += 1; + u += (n & 1); + v += (n & 1); + } +} + +static void YuvToBgrRow(const uint8_t* y, const uint8_t* u, const uint8_t* v, + uint8_t* dst, int len) { + int n; + for (n = 0; n + 32 <= len; n += 32, dst += 32 * 3) { + __m128i R0, R1, R2, R3, G0, G1, G2, G3, B0, B1, B2, B3; + __m128i bgr[6]; + + YUV420ToRGB(y + 0, u + 0, v + 0, &R0, &G0, &B0); + YUV420ToRGB(y + 8, u + 4, v + 4, &R1, &G1, &B1); + YUV420ToRGB(y + 16, u + 8, v + 8, &R2, &G2, &B2); + YUV420ToRGB(y + 24, u + 12, v + 12, &R3, &G3, &B3); + + // Cast to 8b and store as BBBBGGGGRRRR. + bgr[0] = _mm_packus_epi16(B0, B1); + bgr[1] = _mm_packus_epi16(B2, B3); + bgr[2] = _mm_packus_epi16(G0, G1); + bgr[3] = _mm_packus_epi16(G2, G3); + bgr[4] = _mm_packus_epi16(R0, R1); + bgr[5] = _mm_packus_epi16(R2, R3); + + // Pack as BGRBGRBGRBGR. + PlanarTo24b(bgr, dst); + + y += 32; + u += 16; + v += 16; + } + for (; n < len; ++n) { // Finish off + VP8YuvToBgr(y[0], u[0], v[0], dst); + dst += 3; + y += 1; + u += (n & 1); + v += (n & 1); + } +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void WebPInitSamplersSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitSamplersSSE2(void) { + WebPSamplers[MODE_RGB] = YuvToRgbRow; + WebPSamplers[MODE_RGBA] = YuvToRgbaRow; + WebPSamplers[MODE_BGR] = YuvToBgrRow; + WebPSamplers[MODE_BGRA] = YuvToBgraRow; + WebPSamplers[MODE_ARGB] = YuvToArgbRow; +} + +//------------------------------------------------------------------------------ +// RGB24/32 -> YUV converters + +// Load eight 16b-words from *src. +#define LOAD_16(src) _mm_loadu_si128((const __m128i*)(src)) +// Store either 16b-words into *dst +#define STORE_16(V, dst) _mm_storeu_si128((__m128i*)(dst), (V)) + +// Function that inserts a value of the second half of the in buffer in between +// every two char of the first half. +static WEBP_INLINE void RGB24PackedToPlanarHelper( + const __m128i* const in /*in[6]*/, __m128i* const out /*out[6]*/) { + out[0] = _mm_unpacklo_epi8(in[0], in[3]); + out[1] = _mm_unpackhi_epi8(in[0], in[3]); + out[2] = _mm_unpacklo_epi8(in[1], in[4]); + out[3] = _mm_unpackhi_epi8(in[1], in[4]); + out[4] = _mm_unpacklo_epi8(in[2], in[5]); + out[5] = _mm_unpackhi_epi8(in[2], in[5]); +} + +// Unpack the 8b input rgbrgbrgbrgb ... as contiguous registers: +// rrrr... rrrr... gggg... gggg... bbbb... bbbb.... +// Similar to PlanarTo24bHelper(), but in reverse order. +static WEBP_INLINE void RGB24PackedToPlanar(const uint8_t* const rgb, + __m128i* const out /*out[6]*/) { + __m128i tmp[6]; + tmp[0] = _mm_loadu_si128((const __m128i*)(rgb + 0)); + tmp[1] = _mm_loadu_si128((const __m128i*)(rgb + 16)); + tmp[2] = _mm_loadu_si128((const __m128i*)(rgb + 32)); + tmp[3] = _mm_loadu_si128((const __m128i*)(rgb + 48)); + tmp[4] = _mm_loadu_si128((const __m128i*)(rgb + 64)); + tmp[5] = _mm_loadu_si128((const __m128i*)(rgb + 80)); + + RGB24PackedToPlanarHelper(tmp, out); + RGB24PackedToPlanarHelper(out, tmp); + RGB24PackedToPlanarHelper(tmp, out); + RGB24PackedToPlanarHelper(out, tmp); + RGB24PackedToPlanarHelper(tmp, out); +} + +// Convert 8 packed ARGB to r[], g[], b[] +static WEBP_INLINE void RGB32PackedToPlanar(const uint32_t* const argb, + __m128i* const r, + __m128i* const g, + __m128i* const b) { + const __m128i zero = _mm_setzero_si128(); + const __m128i in0 = LOAD_16(argb + 0); // argb3 | argb2 | argb1 | argb0 + const __m128i in1 = LOAD_16(argb + 4); // argb7 | argb6 | argb5 | argb4 + // column-wise transpose + const __m128i A0 = _mm_unpacklo_epi8(in0, in1); + const __m128i A1 = _mm_unpackhi_epi8(in0, in1); + const __m128i B0 = _mm_unpacklo_epi8(A0, A1); + const __m128i B1 = _mm_unpackhi_epi8(A0, A1); + // C0 = g7 g6 ... g1 g0 | b7 b6 ... b1 b0 + // C1 = a7 a6 ... a1 a0 | r7 r6 ... r1 r0 + const __m128i C0 = _mm_unpacklo_epi8(B0, B1); + const __m128i C1 = _mm_unpackhi_epi8(B0, B1); + // store 16b + *r = _mm_unpacklo_epi8(C1, zero); + *g = _mm_unpackhi_epi8(C0, zero); + *b = _mm_unpacklo_epi8(C0, zero); +} + +// This macro computes (RG * MULT_RG + GB * MULT_GB + ROUNDER) >> DESCALE_FIX +// It's a macro and not a function because we need to use immediate values with +// srai_epi32, e.g. +#define TRANSFORM(RG_LO, RG_HI, GB_LO, GB_HI, MULT_RG, MULT_GB, \ + ROUNDER, DESCALE_FIX, OUT) do { \ + const __m128i V0_lo = _mm_madd_epi16(RG_LO, MULT_RG); \ + const __m128i V0_hi = _mm_madd_epi16(RG_HI, MULT_RG); \ + const __m128i V1_lo = _mm_madd_epi16(GB_LO, MULT_GB); \ + const __m128i V1_hi = _mm_madd_epi16(GB_HI, MULT_GB); \ + const __m128i V2_lo = _mm_add_epi32(V0_lo, V1_lo); \ + const __m128i V2_hi = _mm_add_epi32(V0_hi, V1_hi); \ + const __m128i V3_lo = _mm_add_epi32(V2_lo, ROUNDER); \ + const __m128i V3_hi = _mm_add_epi32(V2_hi, ROUNDER); \ + const __m128i V5_lo = _mm_srai_epi32(V3_lo, DESCALE_FIX); \ + const __m128i V5_hi = _mm_srai_epi32(V3_hi, DESCALE_FIX); \ + (OUT) = _mm_packs_epi32(V5_lo, V5_hi); \ +} while (0) + +#define MK_CST_16(A, B) _mm_set_epi16((B), (A), (B), (A), (B), (A), (B), (A)) +static WEBP_INLINE void ConvertRGBToY(const __m128i* const R, + const __m128i* const G, + const __m128i* const B, + __m128i* const Y) { + const __m128i kRG_y = MK_CST_16(16839, 33059 - 16384); + const __m128i kGB_y = MK_CST_16(16384, 6420); + const __m128i kHALF_Y = _mm_set1_epi32((16 << YUV_FIX) + YUV_HALF); + + const __m128i RG_lo = _mm_unpacklo_epi16(*R, *G); + const __m128i RG_hi = _mm_unpackhi_epi16(*R, *G); + const __m128i GB_lo = _mm_unpacklo_epi16(*G, *B); + const __m128i GB_hi = _mm_unpackhi_epi16(*G, *B); + TRANSFORM(RG_lo, RG_hi, GB_lo, GB_hi, kRG_y, kGB_y, kHALF_Y, YUV_FIX, *Y); +} + +static WEBP_INLINE void ConvertRGBToUV(const __m128i* const R, + const __m128i* const G, + const __m128i* const B, + __m128i* const U, __m128i* const V) { + const __m128i kRG_u = MK_CST_16(-9719, -19081); + const __m128i kGB_u = MK_CST_16(0, 28800); + const __m128i kRG_v = MK_CST_16(28800, 0); + const __m128i kGB_v = MK_CST_16(-24116, -4684); + const __m128i kHALF_UV = _mm_set1_epi32(((128 << YUV_FIX) + YUV_HALF) << 2); + + const __m128i RG_lo = _mm_unpacklo_epi16(*R, *G); + const __m128i RG_hi = _mm_unpackhi_epi16(*R, *G); + const __m128i GB_lo = _mm_unpacklo_epi16(*G, *B); + const __m128i GB_hi = _mm_unpackhi_epi16(*G, *B); + TRANSFORM(RG_lo, RG_hi, GB_lo, GB_hi, kRG_u, kGB_u, + kHALF_UV, YUV_FIX + 2, *U); + TRANSFORM(RG_lo, RG_hi, GB_lo, GB_hi, kRG_v, kGB_v, + kHALF_UV, YUV_FIX + 2, *V); +} + +#undef MK_CST_16 +#undef TRANSFORM + +static void ConvertRGB24ToY(const uint8_t* rgb, uint8_t* y, int width) { + const int max_width = width & ~31; + int i; + for (i = 0; i < max_width; rgb += 3 * 16 * 2) { + __m128i rgb_plane[6]; + int j; + + RGB24PackedToPlanar(rgb, rgb_plane); + + for (j = 0; j < 2; ++j, i += 16) { + const __m128i zero = _mm_setzero_si128(); + __m128i r, g, b, Y0, Y1; + + // Convert to 16-bit Y. + r = _mm_unpacklo_epi8(rgb_plane[0 + j], zero); + g = _mm_unpacklo_epi8(rgb_plane[2 + j], zero); + b = _mm_unpacklo_epi8(rgb_plane[4 + j], zero); + ConvertRGBToY(&r, &g, &b, &Y0); + + // Convert to 16-bit Y. + r = _mm_unpackhi_epi8(rgb_plane[0 + j], zero); + g = _mm_unpackhi_epi8(rgb_plane[2 + j], zero); + b = _mm_unpackhi_epi8(rgb_plane[4 + j], zero); + ConvertRGBToY(&r, &g, &b, &Y1); + + // Cast to 8-bit and store. + STORE_16(_mm_packus_epi16(Y0, Y1), y + i); + } + } + for (; i < width; ++i, rgb += 3) { // left-over + y[i] = VP8RGBToY(rgb[0], rgb[1], rgb[2], YUV_HALF); + } +} + +static void ConvertBGR24ToY(const uint8_t* bgr, uint8_t* y, int width) { + const int max_width = width & ~31; + int i; + for (i = 0; i < max_width; bgr += 3 * 16 * 2) { + __m128i bgr_plane[6]; + int j; + + RGB24PackedToPlanar(bgr, bgr_plane); + + for (j = 0; j < 2; ++j, i += 16) { + const __m128i zero = _mm_setzero_si128(); + __m128i r, g, b, Y0, Y1; + + // Convert to 16-bit Y. + b = _mm_unpacklo_epi8(bgr_plane[0 + j], zero); + g = _mm_unpacklo_epi8(bgr_plane[2 + j], zero); + r = _mm_unpacklo_epi8(bgr_plane[4 + j], zero); + ConvertRGBToY(&r, &g, &b, &Y0); + + // Convert to 16-bit Y. + b = _mm_unpackhi_epi8(bgr_plane[0 + j], zero); + g = _mm_unpackhi_epi8(bgr_plane[2 + j], zero); + r = _mm_unpackhi_epi8(bgr_plane[4 + j], zero); + ConvertRGBToY(&r, &g, &b, &Y1); + + // Cast to 8-bit and store. + STORE_16(_mm_packus_epi16(Y0, Y1), y + i); + } + } + for (; i < width; ++i, bgr += 3) { // left-over + y[i] = VP8RGBToY(bgr[2], bgr[1], bgr[0], YUV_HALF); + } +} + +static void ConvertARGBToY(const uint32_t* argb, uint8_t* y, int width) { + const int max_width = width & ~15; + int i; + for (i = 0; i < max_width; i += 16) { + __m128i r, g, b, Y0, Y1; + RGB32PackedToPlanar(&argb[i + 0], &r, &g, &b); + ConvertRGBToY(&r, &g, &b, &Y0); + RGB32PackedToPlanar(&argb[i + 8], &r, &g, &b); + ConvertRGBToY(&r, &g, &b, &Y1); + STORE_16(_mm_packus_epi16(Y0, Y1), y + i); + } + for (; i < width; ++i) { // left-over + const uint32_t p = argb[i]; + y[i] = VP8RGBToY((p >> 16) & 0xff, (p >> 8) & 0xff, (p >> 0) & 0xff, + YUV_HALF); + } +} + +// Horizontal add (doubled) of two 16b values, result is 16b. +// in: A | B | C | D | ... -> out: 2*(A+B) | 2*(C+D) | ... +static void HorizontalAddPack(const __m128i* const A, const __m128i* const B, + __m128i* const out) { + const __m128i k2 = _mm_set1_epi16(2); + const __m128i C = _mm_madd_epi16(*A, k2); + const __m128i D = _mm_madd_epi16(*B, k2); + *out = _mm_packs_epi32(C, D); +} + +static void ConvertARGBToUV(const uint32_t* argb, uint8_t* u, uint8_t* v, + int src_width, int do_store) { + const int max_width = src_width & ~31; + int i; + for (i = 0; i < max_width; i += 32, u += 16, v += 16) { + __m128i r0, g0, b0, r1, g1, b1, U0, V0, U1, V1; + RGB32PackedToPlanar(&argb[i + 0], &r0, &g0, &b0); + RGB32PackedToPlanar(&argb[i + 8], &r1, &g1, &b1); + HorizontalAddPack(&r0, &r1, &r0); + HorizontalAddPack(&g0, &g1, &g0); + HorizontalAddPack(&b0, &b1, &b0); + ConvertRGBToUV(&r0, &g0, &b0, &U0, &V0); + + RGB32PackedToPlanar(&argb[i + 16], &r0, &g0, &b0); + RGB32PackedToPlanar(&argb[i + 24], &r1, &g1, &b1); + HorizontalAddPack(&r0, &r1, &r0); + HorizontalAddPack(&g0, &g1, &g0); + HorizontalAddPack(&b0, &b1, &b0); + ConvertRGBToUV(&r0, &g0, &b0, &U1, &V1); + + U0 = _mm_packus_epi16(U0, U1); + V0 = _mm_packus_epi16(V0, V1); + if (!do_store) { + const __m128i prev_u = LOAD_16(u); + const __m128i prev_v = LOAD_16(v); + U0 = _mm_avg_epu8(U0, prev_u); + V0 = _mm_avg_epu8(V0, prev_v); + } + STORE_16(U0, u); + STORE_16(V0, v); + } + if (i < src_width) { // left-over + WebPConvertARGBToUV_C(argb + i, u, v, src_width - i, do_store); + } +} + +// Convert 16 packed ARGB 16b-values to r[], g[], b[] +static WEBP_INLINE void RGBA32PackedToPlanar_16b(const uint16_t* const rgbx, + __m128i* const r, + __m128i* const g, + __m128i* const b) { + const __m128i in0 = LOAD_16(rgbx + 0); // r0 | g0 | b0 |x| r1 | g1 | b1 |x + const __m128i in1 = LOAD_16(rgbx + 8); // r2 | g2 | b2 |x| r3 | g3 | b3 |x + const __m128i in2 = LOAD_16(rgbx + 16); // r4 | ... + const __m128i in3 = LOAD_16(rgbx + 24); // r6 | ... + // column-wise transpose + const __m128i A0 = _mm_unpacklo_epi16(in0, in1); + const __m128i A1 = _mm_unpackhi_epi16(in0, in1); + const __m128i A2 = _mm_unpacklo_epi16(in2, in3); + const __m128i A3 = _mm_unpackhi_epi16(in2, in3); + const __m128i B0 = _mm_unpacklo_epi16(A0, A1); // r0 r1 r2 r3 | g0 g1 .. + const __m128i B1 = _mm_unpackhi_epi16(A0, A1); // b0 b1 b2 b3 | x x x x + const __m128i B2 = _mm_unpacklo_epi16(A2, A3); // r4 r5 r6 r7 | g4 g5 .. + const __m128i B3 = _mm_unpackhi_epi16(A2, A3); // b4 b5 b6 b7 | x x x x + *r = _mm_unpacklo_epi64(B0, B2); + *g = _mm_unpackhi_epi64(B0, B2); + *b = _mm_unpacklo_epi64(B1, B3); +} + +static void ConvertRGBA32ToUV(const uint16_t* rgb, + uint8_t* u, uint8_t* v, int width) { + const int max_width = width & ~15; + const uint16_t* const last_rgb = rgb + 4 * max_width; + while (rgb < last_rgb) { + __m128i r, g, b, U0, V0, U1, V1; + RGBA32PackedToPlanar_16b(rgb + 0, &r, &g, &b); + ConvertRGBToUV(&r, &g, &b, &U0, &V0); + RGBA32PackedToPlanar_16b(rgb + 32, &r, &g, &b); + ConvertRGBToUV(&r, &g, &b, &U1, &V1); + STORE_16(_mm_packus_epi16(U0, U1), u); + STORE_16(_mm_packus_epi16(V0, V1), v); + u += 16; + v += 16; + rgb += 2 * 32; + } + if (max_width < width) { // left-over + WebPConvertRGBA32ToUV_C(rgb, u, v, width - max_width); + } +} + +//------------------------------------------------------------------------------ + +extern void WebPInitConvertARGBToYUVSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitConvertARGBToYUVSSE2(void) { + WebPConvertARGBToY = ConvertARGBToY; + WebPConvertARGBToUV = ConvertARGBToUV; + + WebPConvertRGB24ToY = ConvertRGB24ToY; + WebPConvertBGR24ToY = ConvertBGR24ToY; + + WebPConvertRGBA32ToUV = ConvertRGBA32ToUV; +} + +#else // !WEBP_USE_SSE2 + +WEBP_DSP_INIT_STUB(WebPInitSamplersSSE2) +WEBP_DSP_INIT_STUB(WebPInitConvertARGBToYUVSSE2) + +#endif // WEBP_USE_SSE2 diff --git a/thirdparty/libwebp/dsp/yuv_tables_sse2.h b/thirdparty/libwebp/dsp/yuv_tables_sse2.h new file mode 100644 index 0000000000..2b0f057518 --- /dev/null +++ b/thirdparty/libwebp/dsp/yuv_tables_sse2.h @@ -0,0 +1,536 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE2 tables for YUV->RGB conversion (12kB overall) +// +// Author: Skal (pascal.massimino@gmail.com) + +// This file is not compiled, but #include'd directly from yuv.c +// Only used if WEBP_YUV_USE_SSE2_TABLES is defined. + +static const VP8kCstSSE2 VP8kYtoRGBA[256] = { + {{0xfffb77b0, 0xfffb77b0, 0xfffb77b0, 0x003fc000}}, + {{0xfffbc235, 0xfffbc235, 0xfffbc235, 0x003fc000}}, + {{0xfffc0cba, 0xfffc0cba, 0xfffc0cba, 0x003fc000}}, + {{0xfffc573f, 0xfffc573f, 0xfffc573f, 0x003fc000}}, + {{0xfffca1c4, 0xfffca1c4, 0xfffca1c4, 0x003fc000}}, + {{0xfffcec49, 0xfffcec49, 0xfffcec49, 0x003fc000}}, + {{0xfffd36ce, 0xfffd36ce, 0xfffd36ce, 0x003fc000}}, + {{0xfffd8153, 0xfffd8153, 0xfffd8153, 0x003fc000}}, + {{0xfffdcbd8, 0xfffdcbd8, 0xfffdcbd8, 0x003fc000}}, + {{0xfffe165d, 0xfffe165d, 0xfffe165d, 0x003fc000}}, + {{0xfffe60e2, 0xfffe60e2, 0xfffe60e2, 0x003fc000}}, + {{0xfffeab67, 0xfffeab67, 0xfffeab67, 0x003fc000}}, + {{0xfffef5ec, 0xfffef5ec, 0xfffef5ec, 0x003fc000}}, + {{0xffff4071, 0xffff4071, 0xffff4071, 0x003fc000}}, + {{0xffff8af6, 0xffff8af6, 0xffff8af6, 0x003fc000}}, + {{0xffffd57b, 0xffffd57b, 0xffffd57b, 0x003fc000}}, + {{0x00002000, 0x00002000, 0x00002000, 0x003fc000}}, + {{0x00006a85, 0x00006a85, 0x00006a85, 0x003fc000}}, + {{0x0000b50a, 0x0000b50a, 0x0000b50a, 0x003fc000}}, + {{0x0000ff8f, 0x0000ff8f, 0x0000ff8f, 0x003fc000}}, + {{0x00014a14, 0x00014a14, 0x00014a14, 0x003fc000}}, + {{0x00019499, 0x00019499, 0x00019499, 0x003fc000}}, + {{0x0001df1e, 0x0001df1e, 0x0001df1e, 0x003fc000}}, + {{0x000229a3, 0x000229a3, 0x000229a3, 0x003fc000}}, + {{0x00027428, 0x00027428, 0x00027428, 0x003fc000}}, + {{0x0002bead, 0x0002bead, 0x0002bead, 0x003fc000}}, + {{0x00030932, 0x00030932, 0x00030932, 0x003fc000}}, + {{0x000353b7, 0x000353b7, 0x000353b7, 0x003fc000}}, + {{0x00039e3c, 0x00039e3c, 0x00039e3c, 0x003fc000}}, + {{0x0003e8c1, 0x0003e8c1, 0x0003e8c1, 0x003fc000}}, + {{0x00043346, 0x00043346, 0x00043346, 0x003fc000}}, + {{0x00047dcb, 0x00047dcb, 0x00047dcb, 0x003fc000}}, + {{0x0004c850, 0x0004c850, 0x0004c850, 0x003fc000}}, + {{0x000512d5, 0x000512d5, 0x000512d5, 0x003fc000}}, + {{0x00055d5a, 0x00055d5a, 0x00055d5a, 0x003fc000}}, + {{0x0005a7df, 0x0005a7df, 0x0005a7df, 0x003fc000}}, + {{0x0005f264, 0x0005f264, 0x0005f264, 0x003fc000}}, + {{0x00063ce9, 0x00063ce9, 0x00063ce9, 0x003fc000}}, + {{0x0006876e, 0x0006876e, 0x0006876e, 0x003fc000}}, + {{0x0006d1f3, 0x0006d1f3, 0x0006d1f3, 0x003fc000}}, + {{0x00071c78, 0x00071c78, 0x00071c78, 0x003fc000}}, + {{0x000766fd, 0x000766fd, 0x000766fd, 0x003fc000}}, + {{0x0007b182, 0x0007b182, 0x0007b182, 0x003fc000}}, + {{0x0007fc07, 0x0007fc07, 0x0007fc07, 0x003fc000}}, + {{0x0008468c, 0x0008468c, 0x0008468c, 0x003fc000}}, + {{0x00089111, 0x00089111, 0x00089111, 0x003fc000}}, + {{0x0008db96, 0x0008db96, 0x0008db96, 0x003fc000}}, + {{0x0009261b, 0x0009261b, 0x0009261b, 0x003fc000}}, + {{0x000970a0, 0x000970a0, 0x000970a0, 0x003fc000}}, + {{0x0009bb25, 0x0009bb25, 0x0009bb25, 0x003fc000}}, + {{0x000a05aa, 0x000a05aa, 0x000a05aa, 0x003fc000}}, + {{0x000a502f, 0x000a502f, 0x000a502f, 0x003fc000}}, + {{0x000a9ab4, 0x000a9ab4, 0x000a9ab4, 0x003fc000}}, + {{0x000ae539, 0x000ae539, 0x000ae539, 0x003fc000}}, + {{0x000b2fbe, 0x000b2fbe, 0x000b2fbe, 0x003fc000}}, + {{0x000b7a43, 0x000b7a43, 0x000b7a43, 0x003fc000}}, + {{0x000bc4c8, 0x000bc4c8, 0x000bc4c8, 0x003fc000}}, + {{0x000c0f4d, 0x000c0f4d, 0x000c0f4d, 0x003fc000}}, + {{0x000c59d2, 0x000c59d2, 0x000c59d2, 0x003fc000}}, + {{0x000ca457, 0x000ca457, 0x000ca457, 0x003fc000}}, + {{0x000ceedc, 0x000ceedc, 0x000ceedc, 0x003fc000}}, + {{0x000d3961, 0x000d3961, 0x000d3961, 0x003fc000}}, + {{0x000d83e6, 0x000d83e6, 0x000d83e6, 0x003fc000}}, + {{0x000dce6b, 0x000dce6b, 0x000dce6b, 0x003fc000}}, + {{0x000e18f0, 0x000e18f0, 0x000e18f0, 0x003fc000}}, + {{0x000e6375, 0x000e6375, 0x000e6375, 0x003fc000}}, + {{0x000eadfa, 0x000eadfa, 0x000eadfa, 0x003fc000}}, + {{0x000ef87f, 0x000ef87f, 0x000ef87f, 0x003fc000}}, + {{0x000f4304, 0x000f4304, 0x000f4304, 0x003fc000}}, + {{0x000f8d89, 0x000f8d89, 0x000f8d89, 0x003fc000}}, + {{0x000fd80e, 0x000fd80e, 0x000fd80e, 0x003fc000}}, + {{0x00102293, 0x00102293, 0x00102293, 0x003fc000}}, + {{0x00106d18, 0x00106d18, 0x00106d18, 0x003fc000}}, + {{0x0010b79d, 0x0010b79d, 0x0010b79d, 0x003fc000}}, + {{0x00110222, 0x00110222, 0x00110222, 0x003fc000}}, + {{0x00114ca7, 0x00114ca7, 0x00114ca7, 0x003fc000}}, + {{0x0011972c, 0x0011972c, 0x0011972c, 0x003fc000}}, + {{0x0011e1b1, 0x0011e1b1, 0x0011e1b1, 0x003fc000}}, + {{0x00122c36, 0x00122c36, 0x00122c36, 0x003fc000}}, + {{0x001276bb, 0x001276bb, 0x001276bb, 0x003fc000}}, + {{0x0012c140, 0x0012c140, 0x0012c140, 0x003fc000}}, + {{0x00130bc5, 0x00130bc5, 0x00130bc5, 0x003fc000}}, + {{0x0013564a, 0x0013564a, 0x0013564a, 0x003fc000}}, + {{0x0013a0cf, 0x0013a0cf, 0x0013a0cf, 0x003fc000}}, + {{0x0013eb54, 0x0013eb54, 0x0013eb54, 0x003fc000}}, + {{0x001435d9, 0x001435d9, 0x001435d9, 0x003fc000}}, + {{0x0014805e, 0x0014805e, 0x0014805e, 0x003fc000}}, + {{0x0014cae3, 0x0014cae3, 0x0014cae3, 0x003fc000}}, + {{0x00151568, 0x00151568, 0x00151568, 0x003fc000}}, + {{0x00155fed, 0x00155fed, 0x00155fed, 0x003fc000}}, + {{0x0015aa72, 0x0015aa72, 0x0015aa72, 0x003fc000}}, + {{0x0015f4f7, 0x0015f4f7, 0x0015f4f7, 0x003fc000}}, + {{0x00163f7c, 0x00163f7c, 0x00163f7c, 0x003fc000}}, + {{0x00168a01, 0x00168a01, 0x00168a01, 0x003fc000}}, + {{0x0016d486, 0x0016d486, 0x0016d486, 0x003fc000}}, + {{0x00171f0b, 0x00171f0b, 0x00171f0b, 0x003fc000}}, + {{0x00176990, 0x00176990, 0x00176990, 0x003fc000}}, + {{0x0017b415, 0x0017b415, 0x0017b415, 0x003fc000}}, + {{0x0017fe9a, 0x0017fe9a, 0x0017fe9a, 0x003fc000}}, + {{0x0018491f, 0x0018491f, 0x0018491f, 0x003fc000}}, + {{0x001893a4, 0x001893a4, 0x001893a4, 0x003fc000}}, + {{0x0018de29, 0x0018de29, 0x0018de29, 0x003fc000}}, + {{0x001928ae, 0x001928ae, 0x001928ae, 0x003fc000}}, + {{0x00197333, 0x00197333, 0x00197333, 0x003fc000}}, + {{0x0019bdb8, 0x0019bdb8, 0x0019bdb8, 0x003fc000}}, + {{0x001a083d, 0x001a083d, 0x001a083d, 0x003fc000}}, + {{0x001a52c2, 0x001a52c2, 0x001a52c2, 0x003fc000}}, + {{0x001a9d47, 0x001a9d47, 0x001a9d47, 0x003fc000}}, + {{0x001ae7cc, 0x001ae7cc, 0x001ae7cc, 0x003fc000}}, + {{0x001b3251, 0x001b3251, 0x001b3251, 0x003fc000}}, + {{0x001b7cd6, 0x001b7cd6, 0x001b7cd6, 0x003fc000}}, + {{0x001bc75b, 0x001bc75b, 0x001bc75b, 0x003fc000}}, + {{0x001c11e0, 0x001c11e0, 0x001c11e0, 0x003fc000}}, + {{0x001c5c65, 0x001c5c65, 0x001c5c65, 0x003fc000}}, + {{0x001ca6ea, 0x001ca6ea, 0x001ca6ea, 0x003fc000}}, + {{0x001cf16f, 0x001cf16f, 0x001cf16f, 0x003fc000}}, + {{0x001d3bf4, 0x001d3bf4, 0x001d3bf4, 0x003fc000}}, + {{0x001d8679, 0x001d8679, 0x001d8679, 0x003fc000}}, + {{0x001dd0fe, 0x001dd0fe, 0x001dd0fe, 0x003fc000}}, + {{0x001e1b83, 0x001e1b83, 0x001e1b83, 0x003fc000}}, + {{0x001e6608, 0x001e6608, 0x001e6608, 0x003fc000}}, + {{0x001eb08d, 0x001eb08d, 0x001eb08d, 0x003fc000}}, + {{0x001efb12, 0x001efb12, 0x001efb12, 0x003fc000}}, + {{0x001f4597, 0x001f4597, 0x001f4597, 0x003fc000}}, + {{0x001f901c, 0x001f901c, 0x001f901c, 0x003fc000}}, + {{0x001fdaa1, 0x001fdaa1, 0x001fdaa1, 0x003fc000}}, + {{0x00202526, 0x00202526, 0x00202526, 0x003fc000}}, + {{0x00206fab, 0x00206fab, 0x00206fab, 0x003fc000}}, + {{0x0020ba30, 0x0020ba30, 0x0020ba30, 0x003fc000}}, + {{0x002104b5, 0x002104b5, 0x002104b5, 0x003fc000}}, + {{0x00214f3a, 0x00214f3a, 0x00214f3a, 0x003fc000}}, + {{0x002199bf, 0x002199bf, 0x002199bf, 0x003fc000}}, + {{0x0021e444, 0x0021e444, 0x0021e444, 0x003fc000}}, + {{0x00222ec9, 0x00222ec9, 0x00222ec9, 0x003fc000}}, + {{0x0022794e, 0x0022794e, 0x0022794e, 0x003fc000}}, + {{0x0022c3d3, 0x0022c3d3, 0x0022c3d3, 0x003fc000}}, + {{0x00230e58, 0x00230e58, 0x00230e58, 0x003fc000}}, + {{0x002358dd, 0x002358dd, 0x002358dd, 0x003fc000}}, + {{0x0023a362, 0x0023a362, 0x0023a362, 0x003fc000}}, + {{0x0023ede7, 0x0023ede7, 0x0023ede7, 0x003fc000}}, + {{0x0024386c, 0x0024386c, 0x0024386c, 0x003fc000}}, + {{0x002482f1, 0x002482f1, 0x002482f1, 0x003fc000}}, + {{0x0024cd76, 0x0024cd76, 0x0024cd76, 0x003fc000}}, + {{0x002517fb, 0x002517fb, 0x002517fb, 0x003fc000}}, + {{0x00256280, 0x00256280, 0x00256280, 0x003fc000}}, + {{0x0025ad05, 0x0025ad05, 0x0025ad05, 0x003fc000}}, + {{0x0025f78a, 0x0025f78a, 0x0025f78a, 0x003fc000}}, + {{0x0026420f, 0x0026420f, 0x0026420f, 0x003fc000}}, + {{0x00268c94, 0x00268c94, 0x00268c94, 0x003fc000}}, + {{0x0026d719, 0x0026d719, 0x0026d719, 0x003fc000}}, + {{0x0027219e, 0x0027219e, 0x0027219e, 0x003fc000}}, + {{0x00276c23, 0x00276c23, 0x00276c23, 0x003fc000}}, + {{0x0027b6a8, 0x0027b6a8, 0x0027b6a8, 0x003fc000}}, + {{0x0028012d, 0x0028012d, 0x0028012d, 0x003fc000}}, + {{0x00284bb2, 0x00284bb2, 0x00284bb2, 0x003fc000}}, + {{0x00289637, 0x00289637, 0x00289637, 0x003fc000}}, + {{0x0028e0bc, 0x0028e0bc, 0x0028e0bc, 0x003fc000}}, + {{0x00292b41, 0x00292b41, 0x00292b41, 0x003fc000}}, + {{0x002975c6, 0x002975c6, 0x002975c6, 0x003fc000}}, + {{0x0029c04b, 0x0029c04b, 0x0029c04b, 0x003fc000}}, + {{0x002a0ad0, 0x002a0ad0, 0x002a0ad0, 0x003fc000}}, + {{0x002a5555, 0x002a5555, 0x002a5555, 0x003fc000}}, + {{0x002a9fda, 0x002a9fda, 0x002a9fda, 0x003fc000}}, + {{0x002aea5f, 0x002aea5f, 0x002aea5f, 0x003fc000}}, + {{0x002b34e4, 0x002b34e4, 0x002b34e4, 0x003fc000}}, + {{0x002b7f69, 0x002b7f69, 0x002b7f69, 0x003fc000}}, + {{0x002bc9ee, 0x002bc9ee, 0x002bc9ee, 0x003fc000}}, + {{0x002c1473, 0x002c1473, 0x002c1473, 0x003fc000}}, + {{0x002c5ef8, 0x002c5ef8, 0x002c5ef8, 0x003fc000}}, + {{0x002ca97d, 0x002ca97d, 0x002ca97d, 0x003fc000}}, + {{0x002cf402, 0x002cf402, 0x002cf402, 0x003fc000}}, + {{0x002d3e87, 0x002d3e87, 0x002d3e87, 0x003fc000}}, + {{0x002d890c, 0x002d890c, 0x002d890c, 0x003fc000}}, + {{0x002dd391, 0x002dd391, 0x002dd391, 0x003fc000}}, + {{0x002e1e16, 0x002e1e16, 0x002e1e16, 0x003fc000}}, + {{0x002e689b, 0x002e689b, 0x002e689b, 0x003fc000}}, + {{0x002eb320, 0x002eb320, 0x002eb320, 0x003fc000}}, + {{0x002efda5, 0x002efda5, 0x002efda5, 0x003fc000}}, + {{0x002f482a, 0x002f482a, 0x002f482a, 0x003fc000}}, + {{0x002f92af, 0x002f92af, 0x002f92af, 0x003fc000}}, + {{0x002fdd34, 0x002fdd34, 0x002fdd34, 0x003fc000}}, + {{0x003027b9, 0x003027b9, 0x003027b9, 0x003fc000}}, + {{0x0030723e, 0x0030723e, 0x0030723e, 0x003fc000}}, + {{0x0030bcc3, 0x0030bcc3, 0x0030bcc3, 0x003fc000}}, + {{0x00310748, 0x00310748, 0x00310748, 0x003fc000}}, + {{0x003151cd, 0x003151cd, 0x003151cd, 0x003fc000}}, + {{0x00319c52, 0x00319c52, 0x00319c52, 0x003fc000}}, + {{0x0031e6d7, 0x0031e6d7, 0x0031e6d7, 0x003fc000}}, + {{0x0032315c, 0x0032315c, 0x0032315c, 0x003fc000}}, + {{0x00327be1, 0x00327be1, 0x00327be1, 0x003fc000}}, + {{0x0032c666, 0x0032c666, 0x0032c666, 0x003fc000}}, + {{0x003310eb, 0x003310eb, 0x003310eb, 0x003fc000}}, + {{0x00335b70, 0x00335b70, 0x00335b70, 0x003fc000}}, + {{0x0033a5f5, 0x0033a5f5, 0x0033a5f5, 0x003fc000}}, + {{0x0033f07a, 0x0033f07a, 0x0033f07a, 0x003fc000}}, + {{0x00343aff, 0x00343aff, 0x00343aff, 0x003fc000}}, + {{0x00348584, 0x00348584, 0x00348584, 0x003fc000}}, + {{0x0034d009, 0x0034d009, 0x0034d009, 0x003fc000}}, + {{0x00351a8e, 0x00351a8e, 0x00351a8e, 0x003fc000}}, + {{0x00356513, 0x00356513, 0x00356513, 0x003fc000}}, + {{0x0035af98, 0x0035af98, 0x0035af98, 0x003fc000}}, + {{0x0035fa1d, 0x0035fa1d, 0x0035fa1d, 0x003fc000}}, + {{0x003644a2, 0x003644a2, 0x003644a2, 0x003fc000}}, + {{0x00368f27, 0x00368f27, 0x00368f27, 0x003fc000}}, + {{0x0036d9ac, 0x0036d9ac, 0x0036d9ac, 0x003fc000}}, + {{0x00372431, 0x00372431, 0x00372431, 0x003fc000}}, + {{0x00376eb6, 0x00376eb6, 0x00376eb6, 0x003fc000}}, + {{0x0037b93b, 0x0037b93b, 0x0037b93b, 0x003fc000}}, + {{0x003803c0, 0x003803c0, 0x003803c0, 0x003fc000}}, + {{0x00384e45, 0x00384e45, 0x00384e45, 0x003fc000}}, + {{0x003898ca, 0x003898ca, 0x003898ca, 0x003fc000}}, + {{0x0038e34f, 0x0038e34f, 0x0038e34f, 0x003fc000}}, + {{0x00392dd4, 0x00392dd4, 0x00392dd4, 0x003fc000}}, + {{0x00397859, 0x00397859, 0x00397859, 0x003fc000}}, + {{0x0039c2de, 0x0039c2de, 0x0039c2de, 0x003fc000}}, + {{0x003a0d63, 0x003a0d63, 0x003a0d63, 0x003fc000}}, + {{0x003a57e8, 0x003a57e8, 0x003a57e8, 0x003fc000}}, + {{0x003aa26d, 0x003aa26d, 0x003aa26d, 0x003fc000}}, + {{0x003aecf2, 0x003aecf2, 0x003aecf2, 0x003fc000}}, + {{0x003b3777, 0x003b3777, 0x003b3777, 0x003fc000}}, + {{0x003b81fc, 0x003b81fc, 0x003b81fc, 0x003fc000}}, + {{0x003bcc81, 0x003bcc81, 0x003bcc81, 0x003fc000}}, + {{0x003c1706, 0x003c1706, 0x003c1706, 0x003fc000}}, + {{0x003c618b, 0x003c618b, 0x003c618b, 0x003fc000}}, + {{0x003cac10, 0x003cac10, 0x003cac10, 0x003fc000}}, + {{0x003cf695, 0x003cf695, 0x003cf695, 0x003fc000}}, + {{0x003d411a, 0x003d411a, 0x003d411a, 0x003fc000}}, + {{0x003d8b9f, 0x003d8b9f, 0x003d8b9f, 0x003fc000}}, + {{0x003dd624, 0x003dd624, 0x003dd624, 0x003fc000}}, + {{0x003e20a9, 0x003e20a9, 0x003e20a9, 0x003fc000}}, + {{0x003e6b2e, 0x003e6b2e, 0x003e6b2e, 0x003fc000}}, + {{0x003eb5b3, 0x003eb5b3, 0x003eb5b3, 0x003fc000}}, + {{0x003f0038, 0x003f0038, 0x003f0038, 0x003fc000}}, + {{0x003f4abd, 0x003f4abd, 0x003f4abd, 0x003fc000}}, + {{0x003f9542, 0x003f9542, 0x003f9542, 0x003fc000}}, + {{0x003fdfc7, 0x003fdfc7, 0x003fdfc7, 0x003fc000}}, + {{0x00402a4c, 0x00402a4c, 0x00402a4c, 0x003fc000}}, + {{0x004074d1, 0x004074d1, 0x004074d1, 0x003fc000}}, + {{0x0040bf56, 0x0040bf56, 0x0040bf56, 0x003fc000}}, + {{0x004109db, 0x004109db, 0x004109db, 0x003fc000}}, + {{0x00415460, 0x00415460, 0x00415460, 0x003fc000}}, + {{0x00419ee5, 0x00419ee5, 0x00419ee5, 0x003fc000}}, + {{0x0041e96a, 0x0041e96a, 0x0041e96a, 0x003fc000}}, + {{0x004233ef, 0x004233ef, 0x004233ef, 0x003fc000}}, + {{0x00427e74, 0x00427e74, 0x00427e74, 0x003fc000}}, + {{0x0042c8f9, 0x0042c8f9, 0x0042c8f9, 0x003fc000}}, + {{0x0043137e, 0x0043137e, 0x0043137e, 0x003fc000}}, + {{0x00435e03, 0x00435e03, 0x00435e03, 0x003fc000}}, + {{0x0043a888, 0x0043a888, 0x0043a888, 0x003fc000}}, + {{0x0043f30d, 0x0043f30d, 0x0043f30d, 0x003fc000}}, + {{0x00443d92, 0x00443d92, 0x00443d92, 0x003fc000}}, + {{0x00448817, 0x00448817, 0x00448817, 0x003fc000}}, + {{0x0044d29c, 0x0044d29c, 0x0044d29c, 0x003fc000}}, + {{0x00451d21, 0x00451d21, 0x00451d21, 0x003fc000}}, + {{0x004567a6, 0x004567a6, 0x004567a6, 0x003fc000}}, + {{0x0045b22b, 0x0045b22b, 0x0045b22b, 0x003fc000}} +}; + +static const VP8kCstSSE2 VP8kUtoRGBA[256] = { + {{0, 0x000c8980, 0xffbf7300, 0}}, {{0, 0x000c706d, 0xffbff41a, 0}}, + {{0, 0x000c575a, 0xffc07534, 0}}, {{0, 0x000c3e47, 0xffc0f64e, 0}}, + {{0, 0x000c2534, 0xffc17768, 0}}, {{0, 0x000c0c21, 0xffc1f882, 0}}, + {{0, 0x000bf30e, 0xffc2799c, 0}}, {{0, 0x000bd9fb, 0xffc2fab6, 0}}, + {{0, 0x000bc0e8, 0xffc37bd0, 0}}, {{0, 0x000ba7d5, 0xffc3fcea, 0}}, + {{0, 0x000b8ec2, 0xffc47e04, 0}}, {{0, 0x000b75af, 0xffc4ff1e, 0}}, + {{0, 0x000b5c9c, 0xffc58038, 0}}, {{0, 0x000b4389, 0xffc60152, 0}}, + {{0, 0x000b2a76, 0xffc6826c, 0}}, {{0, 0x000b1163, 0xffc70386, 0}}, + {{0, 0x000af850, 0xffc784a0, 0}}, {{0, 0x000adf3d, 0xffc805ba, 0}}, + {{0, 0x000ac62a, 0xffc886d4, 0}}, {{0, 0x000aad17, 0xffc907ee, 0}}, + {{0, 0x000a9404, 0xffc98908, 0}}, {{0, 0x000a7af1, 0xffca0a22, 0}}, + {{0, 0x000a61de, 0xffca8b3c, 0}}, {{0, 0x000a48cb, 0xffcb0c56, 0}}, + {{0, 0x000a2fb8, 0xffcb8d70, 0}}, {{0, 0x000a16a5, 0xffcc0e8a, 0}}, + {{0, 0x0009fd92, 0xffcc8fa4, 0}}, {{0, 0x0009e47f, 0xffcd10be, 0}}, + {{0, 0x0009cb6c, 0xffcd91d8, 0}}, {{0, 0x0009b259, 0xffce12f2, 0}}, + {{0, 0x00099946, 0xffce940c, 0}}, {{0, 0x00098033, 0xffcf1526, 0}}, + {{0, 0x00096720, 0xffcf9640, 0}}, {{0, 0x00094e0d, 0xffd0175a, 0}}, + {{0, 0x000934fa, 0xffd09874, 0}}, {{0, 0x00091be7, 0xffd1198e, 0}}, + {{0, 0x000902d4, 0xffd19aa8, 0}}, {{0, 0x0008e9c1, 0xffd21bc2, 0}}, + {{0, 0x0008d0ae, 0xffd29cdc, 0}}, {{0, 0x0008b79b, 0xffd31df6, 0}}, + {{0, 0x00089e88, 0xffd39f10, 0}}, {{0, 0x00088575, 0xffd4202a, 0}}, + {{0, 0x00086c62, 0xffd4a144, 0}}, {{0, 0x0008534f, 0xffd5225e, 0}}, + {{0, 0x00083a3c, 0xffd5a378, 0}}, {{0, 0x00082129, 0xffd62492, 0}}, + {{0, 0x00080816, 0xffd6a5ac, 0}}, {{0, 0x0007ef03, 0xffd726c6, 0}}, + {{0, 0x0007d5f0, 0xffd7a7e0, 0}}, {{0, 0x0007bcdd, 0xffd828fa, 0}}, + {{0, 0x0007a3ca, 0xffd8aa14, 0}}, {{0, 0x00078ab7, 0xffd92b2e, 0}}, + {{0, 0x000771a4, 0xffd9ac48, 0}}, {{0, 0x00075891, 0xffda2d62, 0}}, + {{0, 0x00073f7e, 0xffdaae7c, 0}}, {{0, 0x0007266b, 0xffdb2f96, 0}}, + {{0, 0x00070d58, 0xffdbb0b0, 0}}, {{0, 0x0006f445, 0xffdc31ca, 0}}, + {{0, 0x0006db32, 0xffdcb2e4, 0}}, {{0, 0x0006c21f, 0xffdd33fe, 0}}, + {{0, 0x0006a90c, 0xffddb518, 0}}, {{0, 0x00068ff9, 0xffde3632, 0}}, + {{0, 0x000676e6, 0xffdeb74c, 0}}, {{0, 0x00065dd3, 0xffdf3866, 0}}, + {{0, 0x000644c0, 0xffdfb980, 0}}, {{0, 0x00062bad, 0xffe03a9a, 0}}, + {{0, 0x0006129a, 0xffe0bbb4, 0}}, {{0, 0x0005f987, 0xffe13cce, 0}}, + {{0, 0x0005e074, 0xffe1bde8, 0}}, {{0, 0x0005c761, 0xffe23f02, 0}}, + {{0, 0x0005ae4e, 0xffe2c01c, 0}}, {{0, 0x0005953b, 0xffe34136, 0}}, + {{0, 0x00057c28, 0xffe3c250, 0}}, {{0, 0x00056315, 0xffe4436a, 0}}, + {{0, 0x00054a02, 0xffe4c484, 0}}, {{0, 0x000530ef, 0xffe5459e, 0}}, + {{0, 0x000517dc, 0xffe5c6b8, 0}}, {{0, 0x0004fec9, 0xffe647d2, 0}}, + {{0, 0x0004e5b6, 0xffe6c8ec, 0}}, {{0, 0x0004cca3, 0xffe74a06, 0}}, + {{0, 0x0004b390, 0xffe7cb20, 0}}, {{0, 0x00049a7d, 0xffe84c3a, 0}}, + {{0, 0x0004816a, 0xffe8cd54, 0}}, {{0, 0x00046857, 0xffe94e6e, 0}}, + {{0, 0x00044f44, 0xffe9cf88, 0}}, {{0, 0x00043631, 0xffea50a2, 0}}, + {{0, 0x00041d1e, 0xffead1bc, 0}}, {{0, 0x0004040b, 0xffeb52d6, 0}}, + {{0, 0x0003eaf8, 0xffebd3f0, 0}}, {{0, 0x0003d1e5, 0xffec550a, 0}}, + {{0, 0x0003b8d2, 0xffecd624, 0}}, {{0, 0x00039fbf, 0xffed573e, 0}}, + {{0, 0x000386ac, 0xffedd858, 0}}, {{0, 0x00036d99, 0xffee5972, 0}}, + {{0, 0x00035486, 0xffeeda8c, 0}}, {{0, 0x00033b73, 0xffef5ba6, 0}}, + {{0, 0x00032260, 0xffefdcc0, 0}}, {{0, 0x0003094d, 0xfff05dda, 0}}, + {{0, 0x0002f03a, 0xfff0def4, 0}}, {{0, 0x0002d727, 0xfff1600e, 0}}, + {{0, 0x0002be14, 0xfff1e128, 0}}, {{0, 0x0002a501, 0xfff26242, 0}}, + {{0, 0x00028bee, 0xfff2e35c, 0}}, {{0, 0x000272db, 0xfff36476, 0}}, + {{0, 0x000259c8, 0xfff3e590, 0}}, {{0, 0x000240b5, 0xfff466aa, 0}}, + {{0, 0x000227a2, 0xfff4e7c4, 0}}, {{0, 0x00020e8f, 0xfff568de, 0}}, + {{0, 0x0001f57c, 0xfff5e9f8, 0}}, {{0, 0x0001dc69, 0xfff66b12, 0}}, + {{0, 0x0001c356, 0xfff6ec2c, 0}}, {{0, 0x0001aa43, 0xfff76d46, 0}}, + {{0, 0x00019130, 0xfff7ee60, 0}}, {{0, 0x0001781d, 0xfff86f7a, 0}}, + {{0, 0x00015f0a, 0xfff8f094, 0}}, {{0, 0x000145f7, 0xfff971ae, 0}}, + {{0, 0x00012ce4, 0xfff9f2c8, 0}}, {{0, 0x000113d1, 0xfffa73e2, 0}}, + {{0, 0x0000fabe, 0xfffaf4fc, 0}}, {{0, 0x0000e1ab, 0xfffb7616, 0}}, + {{0, 0x0000c898, 0xfffbf730, 0}}, {{0, 0x0000af85, 0xfffc784a, 0}}, + {{0, 0x00009672, 0xfffcf964, 0}}, {{0, 0x00007d5f, 0xfffd7a7e, 0}}, + {{0, 0x0000644c, 0xfffdfb98, 0}}, {{0, 0x00004b39, 0xfffe7cb2, 0}}, + {{0, 0x00003226, 0xfffefdcc, 0}}, {{0, 0x00001913, 0xffff7ee6, 0}}, + {{0, 0x00000000, 0x00000000, 0}}, {{0, 0xffffe6ed, 0x0000811a, 0}}, + {{0, 0xffffcdda, 0x00010234, 0}}, {{0, 0xffffb4c7, 0x0001834e, 0}}, + {{0, 0xffff9bb4, 0x00020468, 0}}, {{0, 0xffff82a1, 0x00028582, 0}}, + {{0, 0xffff698e, 0x0003069c, 0}}, {{0, 0xffff507b, 0x000387b6, 0}}, + {{0, 0xffff3768, 0x000408d0, 0}}, {{0, 0xffff1e55, 0x000489ea, 0}}, + {{0, 0xffff0542, 0x00050b04, 0}}, {{0, 0xfffeec2f, 0x00058c1e, 0}}, + {{0, 0xfffed31c, 0x00060d38, 0}}, {{0, 0xfffeba09, 0x00068e52, 0}}, + {{0, 0xfffea0f6, 0x00070f6c, 0}}, {{0, 0xfffe87e3, 0x00079086, 0}}, + {{0, 0xfffe6ed0, 0x000811a0, 0}}, {{0, 0xfffe55bd, 0x000892ba, 0}}, + {{0, 0xfffe3caa, 0x000913d4, 0}}, {{0, 0xfffe2397, 0x000994ee, 0}}, + {{0, 0xfffe0a84, 0x000a1608, 0}}, {{0, 0xfffdf171, 0x000a9722, 0}}, + {{0, 0xfffdd85e, 0x000b183c, 0}}, {{0, 0xfffdbf4b, 0x000b9956, 0}}, + {{0, 0xfffda638, 0x000c1a70, 0}}, {{0, 0xfffd8d25, 0x000c9b8a, 0}}, + {{0, 0xfffd7412, 0x000d1ca4, 0}}, {{0, 0xfffd5aff, 0x000d9dbe, 0}}, + {{0, 0xfffd41ec, 0x000e1ed8, 0}}, {{0, 0xfffd28d9, 0x000e9ff2, 0}}, + {{0, 0xfffd0fc6, 0x000f210c, 0}}, {{0, 0xfffcf6b3, 0x000fa226, 0}}, + {{0, 0xfffcdda0, 0x00102340, 0}}, {{0, 0xfffcc48d, 0x0010a45a, 0}}, + {{0, 0xfffcab7a, 0x00112574, 0}}, {{0, 0xfffc9267, 0x0011a68e, 0}}, + {{0, 0xfffc7954, 0x001227a8, 0}}, {{0, 0xfffc6041, 0x0012a8c2, 0}}, + {{0, 0xfffc472e, 0x001329dc, 0}}, {{0, 0xfffc2e1b, 0x0013aaf6, 0}}, + {{0, 0xfffc1508, 0x00142c10, 0}}, {{0, 0xfffbfbf5, 0x0014ad2a, 0}}, + {{0, 0xfffbe2e2, 0x00152e44, 0}}, {{0, 0xfffbc9cf, 0x0015af5e, 0}}, + {{0, 0xfffbb0bc, 0x00163078, 0}}, {{0, 0xfffb97a9, 0x0016b192, 0}}, + {{0, 0xfffb7e96, 0x001732ac, 0}}, {{0, 0xfffb6583, 0x0017b3c6, 0}}, + {{0, 0xfffb4c70, 0x001834e0, 0}}, {{0, 0xfffb335d, 0x0018b5fa, 0}}, + {{0, 0xfffb1a4a, 0x00193714, 0}}, {{0, 0xfffb0137, 0x0019b82e, 0}}, + {{0, 0xfffae824, 0x001a3948, 0}}, {{0, 0xfffacf11, 0x001aba62, 0}}, + {{0, 0xfffab5fe, 0x001b3b7c, 0}}, {{0, 0xfffa9ceb, 0x001bbc96, 0}}, + {{0, 0xfffa83d8, 0x001c3db0, 0}}, {{0, 0xfffa6ac5, 0x001cbeca, 0}}, + {{0, 0xfffa51b2, 0x001d3fe4, 0}}, {{0, 0xfffa389f, 0x001dc0fe, 0}}, + {{0, 0xfffa1f8c, 0x001e4218, 0}}, {{0, 0xfffa0679, 0x001ec332, 0}}, + {{0, 0xfff9ed66, 0x001f444c, 0}}, {{0, 0xfff9d453, 0x001fc566, 0}}, + {{0, 0xfff9bb40, 0x00204680, 0}}, {{0, 0xfff9a22d, 0x0020c79a, 0}}, + {{0, 0xfff9891a, 0x002148b4, 0}}, {{0, 0xfff97007, 0x0021c9ce, 0}}, + {{0, 0xfff956f4, 0x00224ae8, 0}}, {{0, 0xfff93de1, 0x0022cc02, 0}}, + {{0, 0xfff924ce, 0x00234d1c, 0}}, {{0, 0xfff90bbb, 0x0023ce36, 0}}, + {{0, 0xfff8f2a8, 0x00244f50, 0}}, {{0, 0xfff8d995, 0x0024d06a, 0}}, + {{0, 0xfff8c082, 0x00255184, 0}}, {{0, 0xfff8a76f, 0x0025d29e, 0}}, + {{0, 0xfff88e5c, 0x002653b8, 0}}, {{0, 0xfff87549, 0x0026d4d2, 0}}, + {{0, 0xfff85c36, 0x002755ec, 0}}, {{0, 0xfff84323, 0x0027d706, 0}}, + {{0, 0xfff82a10, 0x00285820, 0}}, {{0, 0xfff810fd, 0x0028d93a, 0}}, + {{0, 0xfff7f7ea, 0x00295a54, 0}}, {{0, 0xfff7ded7, 0x0029db6e, 0}}, + {{0, 0xfff7c5c4, 0x002a5c88, 0}}, {{0, 0xfff7acb1, 0x002adda2, 0}}, + {{0, 0xfff7939e, 0x002b5ebc, 0}}, {{0, 0xfff77a8b, 0x002bdfd6, 0}}, + {{0, 0xfff76178, 0x002c60f0, 0}}, {{0, 0xfff74865, 0x002ce20a, 0}}, + {{0, 0xfff72f52, 0x002d6324, 0}}, {{0, 0xfff7163f, 0x002de43e, 0}}, + {{0, 0xfff6fd2c, 0x002e6558, 0}}, {{0, 0xfff6e419, 0x002ee672, 0}}, + {{0, 0xfff6cb06, 0x002f678c, 0}}, {{0, 0xfff6b1f3, 0x002fe8a6, 0}}, + {{0, 0xfff698e0, 0x003069c0, 0}}, {{0, 0xfff67fcd, 0x0030eada, 0}}, + {{0, 0xfff666ba, 0x00316bf4, 0}}, {{0, 0xfff64da7, 0x0031ed0e, 0}}, + {{0, 0xfff63494, 0x00326e28, 0}}, {{0, 0xfff61b81, 0x0032ef42, 0}}, + {{0, 0xfff6026e, 0x0033705c, 0}}, {{0, 0xfff5e95b, 0x0033f176, 0}}, + {{0, 0xfff5d048, 0x00347290, 0}}, {{0, 0xfff5b735, 0x0034f3aa, 0}}, + {{0, 0xfff59e22, 0x003574c4, 0}}, {{0, 0xfff5850f, 0x0035f5de, 0}}, + {{0, 0xfff56bfc, 0x003676f8, 0}}, {{0, 0xfff552e9, 0x0036f812, 0}}, + {{0, 0xfff539d6, 0x0037792c, 0}}, {{0, 0xfff520c3, 0x0037fa46, 0}}, + {{0, 0xfff507b0, 0x00387b60, 0}}, {{0, 0xfff4ee9d, 0x0038fc7a, 0}}, + {{0, 0xfff4d58a, 0x00397d94, 0}}, {{0, 0xfff4bc77, 0x0039feae, 0}}, + {{0, 0xfff4a364, 0x003a7fc8, 0}}, {{0, 0xfff48a51, 0x003b00e2, 0}}, + {{0, 0xfff4713e, 0x003b81fc, 0}}, {{0, 0xfff4582b, 0x003c0316, 0}}, + {{0, 0xfff43f18, 0x003c8430, 0}}, {{0, 0xfff42605, 0x003d054a, 0}}, + {{0, 0xfff40cf2, 0x003d8664, 0}}, {{0, 0xfff3f3df, 0x003e077e, 0}}, + {{0, 0xfff3dacc, 0x003e8898, 0}}, {{0, 0xfff3c1b9, 0x003f09b2, 0}}, + {{0, 0xfff3a8a6, 0x003f8acc, 0}}, {{0, 0xfff38f93, 0x00400be6, 0}} +}; + +static VP8kCstSSE2 VP8kVtoRGBA[256] = { + {{0xffcced80, 0x001a0400, 0, 0}}, {{0xffcd53a5, 0x0019cff8, 0, 0}}, + {{0xffcdb9ca, 0x00199bf0, 0, 0}}, {{0xffce1fef, 0x001967e8, 0, 0}}, + {{0xffce8614, 0x001933e0, 0, 0}}, {{0xffceec39, 0x0018ffd8, 0, 0}}, + {{0xffcf525e, 0x0018cbd0, 0, 0}}, {{0xffcfb883, 0x001897c8, 0, 0}}, + {{0xffd01ea8, 0x001863c0, 0, 0}}, {{0xffd084cd, 0x00182fb8, 0, 0}}, + {{0xffd0eaf2, 0x0017fbb0, 0, 0}}, {{0xffd15117, 0x0017c7a8, 0, 0}}, + {{0xffd1b73c, 0x001793a0, 0, 0}}, {{0xffd21d61, 0x00175f98, 0, 0}}, + {{0xffd28386, 0x00172b90, 0, 0}}, {{0xffd2e9ab, 0x0016f788, 0, 0}}, + {{0xffd34fd0, 0x0016c380, 0, 0}}, {{0xffd3b5f5, 0x00168f78, 0, 0}}, + {{0xffd41c1a, 0x00165b70, 0, 0}}, {{0xffd4823f, 0x00162768, 0, 0}}, + {{0xffd4e864, 0x0015f360, 0, 0}}, {{0xffd54e89, 0x0015bf58, 0, 0}}, + {{0xffd5b4ae, 0x00158b50, 0, 0}}, {{0xffd61ad3, 0x00155748, 0, 0}}, + {{0xffd680f8, 0x00152340, 0, 0}}, {{0xffd6e71d, 0x0014ef38, 0, 0}}, + {{0xffd74d42, 0x0014bb30, 0, 0}}, {{0xffd7b367, 0x00148728, 0, 0}}, + {{0xffd8198c, 0x00145320, 0, 0}}, {{0xffd87fb1, 0x00141f18, 0, 0}}, + {{0xffd8e5d6, 0x0013eb10, 0, 0}}, {{0xffd94bfb, 0x0013b708, 0, 0}}, + {{0xffd9b220, 0x00138300, 0, 0}}, {{0xffda1845, 0x00134ef8, 0, 0}}, + {{0xffda7e6a, 0x00131af0, 0, 0}}, {{0xffdae48f, 0x0012e6e8, 0, 0}}, + {{0xffdb4ab4, 0x0012b2e0, 0, 0}}, {{0xffdbb0d9, 0x00127ed8, 0, 0}}, + {{0xffdc16fe, 0x00124ad0, 0, 0}}, {{0xffdc7d23, 0x001216c8, 0, 0}}, + {{0xffdce348, 0x0011e2c0, 0, 0}}, {{0xffdd496d, 0x0011aeb8, 0, 0}}, + {{0xffddaf92, 0x00117ab0, 0, 0}}, {{0xffde15b7, 0x001146a8, 0, 0}}, + {{0xffde7bdc, 0x001112a0, 0, 0}}, {{0xffdee201, 0x0010de98, 0, 0}}, + {{0xffdf4826, 0x0010aa90, 0, 0}}, {{0xffdfae4b, 0x00107688, 0, 0}}, + {{0xffe01470, 0x00104280, 0, 0}}, {{0xffe07a95, 0x00100e78, 0, 0}}, + {{0xffe0e0ba, 0x000fda70, 0, 0}}, {{0xffe146df, 0x000fa668, 0, 0}}, + {{0xffe1ad04, 0x000f7260, 0, 0}}, {{0xffe21329, 0x000f3e58, 0, 0}}, + {{0xffe2794e, 0x000f0a50, 0, 0}}, {{0xffe2df73, 0x000ed648, 0, 0}}, + {{0xffe34598, 0x000ea240, 0, 0}}, {{0xffe3abbd, 0x000e6e38, 0, 0}}, + {{0xffe411e2, 0x000e3a30, 0, 0}}, {{0xffe47807, 0x000e0628, 0, 0}}, + {{0xffe4de2c, 0x000dd220, 0, 0}}, {{0xffe54451, 0x000d9e18, 0, 0}}, + {{0xffe5aa76, 0x000d6a10, 0, 0}}, {{0xffe6109b, 0x000d3608, 0, 0}}, + {{0xffe676c0, 0x000d0200, 0, 0}}, {{0xffe6dce5, 0x000ccdf8, 0, 0}}, + {{0xffe7430a, 0x000c99f0, 0, 0}}, {{0xffe7a92f, 0x000c65e8, 0, 0}}, + {{0xffe80f54, 0x000c31e0, 0, 0}}, {{0xffe87579, 0x000bfdd8, 0, 0}}, + {{0xffe8db9e, 0x000bc9d0, 0, 0}}, {{0xffe941c3, 0x000b95c8, 0, 0}}, + {{0xffe9a7e8, 0x000b61c0, 0, 0}}, {{0xffea0e0d, 0x000b2db8, 0, 0}}, + {{0xffea7432, 0x000af9b0, 0, 0}}, {{0xffeada57, 0x000ac5a8, 0, 0}}, + {{0xffeb407c, 0x000a91a0, 0, 0}}, {{0xffeba6a1, 0x000a5d98, 0, 0}}, + {{0xffec0cc6, 0x000a2990, 0, 0}}, {{0xffec72eb, 0x0009f588, 0, 0}}, + {{0xffecd910, 0x0009c180, 0, 0}}, {{0xffed3f35, 0x00098d78, 0, 0}}, + {{0xffeda55a, 0x00095970, 0, 0}}, {{0xffee0b7f, 0x00092568, 0, 0}}, + {{0xffee71a4, 0x0008f160, 0, 0}}, {{0xffeed7c9, 0x0008bd58, 0, 0}}, + {{0xffef3dee, 0x00088950, 0, 0}}, {{0xffefa413, 0x00085548, 0, 0}}, + {{0xfff00a38, 0x00082140, 0, 0}}, {{0xfff0705d, 0x0007ed38, 0, 0}}, + {{0xfff0d682, 0x0007b930, 0, 0}}, {{0xfff13ca7, 0x00078528, 0, 0}}, + {{0xfff1a2cc, 0x00075120, 0, 0}}, {{0xfff208f1, 0x00071d18, 0, 0}}, + {{0xfff26f16, 0x0006e910, 0, 0}}, {{0xfff2d53b, 0x0006b508, 0, 0}}, + {{0xfff33b60, 0x00068100, 0, 0}}, {{0xfff3a185, 0x00064cf8, 0, 0}}, + {{0xfff407aa, 0x000618f0, 0, 0}}, {{0xfff46dcf, 0x0005e4e8, 0, 0}}, + {{0xfff4d3f4, 0x0005b0e0, 0, 0}}, {{0xfff53a19, 0x00057cd8, 0, 0}}, + {{0xfff5a03e, 0x000548d0, 0, 0}}, {{0xfff60663, 0x000514c8, 0, 0}}, + {{0xfff66c88, 0x0004e0c0, 0, 0}}, {{0xfff6d2ad, 0x0004acb8, 0, 0}}, + {{0xfff738d2, 0x000478b0, 0, 0}}, {{0xfff79ef7, 0x000444a8, 0, 0}}, + {{0xfff8051c, 0x000410a0, 0, 0}}, {{0xfff86b41, 0x0003dc98, 0, 0}}, + {{0xfff8d166, 0x0003a890, 0, 0}}, {{0xfff9378b, 0x00037488, 0, 0}}, + {{0xfff99db0, 0x00034080, 0, 0}}, {{0xfffa03d5, 0x00030c78, 0, 0}}, + {{0xfffa69fa, 0x0002d870, 0, 0}}, {{0xfffad01f, 0x0002a468, 0, 0}}, + {{0xfffb3644, 0x00027060, 0, 0}}, {{0xfffb9c69, 0x00023c58, 0, 0}}, + {{0xfffc028e, 0x00020850, 0, 0}}, {{0xfffc68b3, 0x0001d448, 0, 0}}, + {{0xfffcced8, 0x0001a040, 0, 0}}, {{0xfffd34fd, 0x00016c38, 0, 0}}, + {{0xfffd9b22, 0x00013830, 0, 0}}, {{0xfffe0147, 0x00010428, 0, 0}}, + {{0xfffe676c, 0x0000d020, 0, 0}}, {{0xfffecd91, 0x00009c18, 0, 0}}, + {{0xffff33b6, 0x00006810, 0, 0}}, {{0xffff99db, 0x00003408, 0, 0}}, + {{0x00000000, 0x00000000, 0, 0}}, {{0x00006625, 0xffffcbf8, 0, 0}}, + {{0x0000cc4a, 0xffff97f0, 0, 0}}, {{0x0001326f, 0xffff63e8, 0, 0}}, + {{0x00019894, 0xffff2fe0, 0, 0}}, {{0x0001feb9, 0xfffefbd8, 0, 0}}, + {{0x000264de, 0xfffec7d0, 0, 0}}, {{0x0002cb03, 0xfffe93c8, 0, 0}}, + {{0x00033128, 0xfffe5fc0, 0, 0}}, {{0x0003974d, 0xfffe2bb8, 0, 0}}, + {{0x0003fd72, 0xfffdf7b0, 0, 0}}, {{0x00046397, 0xfffdc3a8, 0, 0}}, + {{0x0004c9bc, 0xfffd8fa0, 0, 0}}, {{0x00052fe1, 0xfffd5b98, 0, 0}}, + {{0x00059606, 0xfffd2790, 0, 0}}, {{0x0005fc2b, 0xfffcf388, 0, 0}}, + {{0x00066250, 0xfffcbf80, 0, 0}}, {{0x0006c875, 0xfffc8b78, 0, 0}}, + {{0x00072e9a, 0xfffc5770, 0, 0}}, {{0x000794bf, 0xfffc2368, 0, 0}}, + {{0x0007fae4, 0xfffbef60, 0, 0}}, {{0x00086109, 0xfffbbb58, 0, 0}}, + {{0x0008c72e, 0xfffb8750, 0, 0}}, {{0x00092d53, 0xfffb5348, 0, 0}}, + {{0x00099378, 0xfffb1f40, 0, 0}}, {{0x0009f99d, 0xfffaeb38, 0, 0}}, + {{0x000a5fc2, 0xfffab730, 0, 0}}, {{0x000ac5e7, 0xfffa8328, 0, 0}}, + {{0x000b2c0c, 0xfffa4f20, 0, 0}}, {{0x000b9231, 0xfffa1b18, 0, 0}}, + {{0x000bf856, 0xfff9e710, 0, 0}}, {{0x000c5e7b, 0xfff9b308, 0, 0}}, + {{0x000cc4a0, 0xfff97f00, 0, 0}}, {{0x000d2ac5, 0xfff94af8, 0, 0}}, + {{0x000d90ea, 0xfff916f0, 0, 0}}, {{0x000df70f, 0xfff8e2e8, 0, 0}}, + {{0x000e5d34, 0xfff8aee0, 0, 0}}, {{0x000ec359, 0xfff87ad8, 0, 0}}, + {{0x000f297e, 0xfff846d0, 0, 0}}, {{0x000f8fa3, 0xfff812c8, 0, 0}}, + {{0x000ff5c8, 0xfff7dec0, 0, 0}}, {{0x00105bed, 0xfff7aab8, 0, 0}}, + {{0x0010c212, 0xfff776b0, 0, 0}}, {{0x00112837, 0xfff742a8, 0, 0}}, + {{0x00118e5c, 0xfff70ea0, 0, 0}}, {{0x0011f481, 0xfff6da98, 0, 0}}, + {{0x00125aa6, 0xfff6a690, 0, 0}}, {{0x0012c0cb, 0xfff67288, 0, 0}}, + {{0x001326f0, 0xfff63e80, 0, 0}}, {{0x00138d15, 0xfff60a78, 0, 0}}, + {{0x0013f33a, 0xfff5d670, 0, 0}}, {{0x0014595f, 0xfff5a268, 0, 0}}, + {{0x0014bf84, 0xfff56e60, 0, 0}}, {{0x001525a9, 0xfff53a58, 0, 0}}, + {{0x00158bce, 0xfff50650, 0, 0}}, {{0x0015f1f3, 0xfff4d248, 0, 0}}, + {{0x00165818, 0xfff49e40, 0, 0}}, {{0x0016be3d, 0xfff46a38, 0, 0}}, + {{0x00172462, 0xfff43630, 0, 0}}, {{0x00178a87, 0xfff40228, 0, 0}}, + {{0x0017f0ac, 0xfff3ce20, 0, 0}}, {{0x001856d1, 0xfff39a18, 0, 0}}, + {{0x0018bcf6, 0xfff36610, 0, 0}}, {{0x0019231b, 0xfff33208, 0, 0}}, + {{0x00198940, 0xfff2fe00, 0, 0}}, {{0x0019ef65, 0xfff2c9f8, 0, 0}}, + {{0x001a558a, 0xfff295f0, 0, 0}}, {{0x001abbaf, 0xfff261e8, 0, 0}}, + {{0x001b21d4, 0xfff22de0, 0, 0}}, {{0x001b87f9, 0xfff1f9d8, 0, 0}}, + {{0x001bee1e, 0xfff1c5d0, 0, 0}}, {{0x001c5443, 0xfff191c8, 0, 0}}, + {{0x001cba68, 0xfff15dc0, 0, 0}}, {{0x001d208d, 0xfff129b8, 0, 0}}, + {{0x001d86b2, 0xfff0f5b0, 0, 0}}, {{0x001decd7, 0xfff0c1a8, 0, 0}}, + {{0x001e52fc, 0xfff08da0, 0, 0}}, {{0x001eb921, 0xfff05998, 0, 0}}, + {{0x001f1f46, 0xfff02590, 0, 0}}, {{0x001f856b, 0xffeff188, 0, 0}}, + {{0x001feb90, 0xffefbd80, 0, 0}}, {{0x002051b5, 0xffef8978, 0, 0}}, + {{0x0020b7da, 0xffef5570, 0, 0}}, {{0x00211dff, 0xffef2168, 0, 0}}, + {{0x00218424, 0xffeeed60, 0, 0}}, {{0x0021ea49, 0xffeeb958, 0, 0}}, + {{0x0022506e, 0xffee8550, 0, 0}}, {{0x0022b693, 0xffee5148, 0, 0}}, + {{0x00231cb8, 0xffee1d40, 0, 0}}, {{0x002382dd, 0xffede938, 0, 0}}, + {{0x0023e902, 0xffedb530, 0, 0}}, {{0x00244f27, 0xffed8128, 0, 0}}, + {{0x0024b54c, 0xffed4d20, 0, 0}}, {{0x00251b71, 0xffed1918, 0, 0}}, + {{0x00258196, 0xffece510, 0, 0}}, {{0x0025e7bb, 0xffecb108, 0, 0}}, + {{0x00264de0, 0xffec7d00, 0, 0}}, {{0x0026b405, 0xffec48f8, 0, 0}}, + {{0x00271a2a, 0xffec14f0, 0, 0}}, {{0x0027804f, 0xffebe0e8, 0, 0}}, + {{0x0027e674, 0xffebace0, 0, 0}}, {{0x00284c99, 0xffeb78d8, 0, 0}}, + {{0x0028b2be, 0xffeb44d0, 0, 0}}, {{0x002918e3, 0xffeb10c8, 0, 0}}, + {{0x00297f08, 0xffeadcc0, 0, 0}}, {{0x0029e52d, 0xffeaa8b8, 0, 0}}, + {{0x002a4b52, 0xffea74b0, 0, 0}}, {{0x002ab177, 0xffea40a8, 0, 0}}, + {{0x002b179c, 0xffea0ca0, 0, 0}}, {{0x002b7dc1, 0xffe9d898, 0, 0}}, + {{0x002be3e6, 0xffe9a490, 0, 0}}, {{0x002c4a0b, 0xffe97088, 0, 0}}, + {{0x002cb030, 0xffe93c80, 0, 0}}, {{0x002d1655, 0xffe90878, 0, 0}}, + {{0x002d7c7a, 0xffe8d470, 0, 0}}, {{0x002de29f, 0xffe8a068, 0, 0}}, + {{0x002e48c4, 0xffe86c60, 0, 0}}, {{0x002eaee9, 0xffe83858, 0, 0}}, + {{0x002f150e, 0xffe80450, 0, 0}}, {{0x002f7b33, 0xffe7d048, 0, 0}}, + {{0x002fe158, 0xffe79c40, 0, 0}}, {{0x0030477d, 0xffe76838, 0, 0}}, + {{0x0030ada2, 0xffe73430, 0, 0}}, {{0x003113c7, 0xffe70028, 0, 0}}, + {{0x003179ec, 0xffe6cc20, 0, 0}}, {{0x0031e011, 0xffe69818, 0, 0}}, + {{0x00324636, 0xffe66410, 0, 0}}, {{0x0032ac5b, 0xffe63008, 0, 0}} +}; |