diff options
Diffstat (limited to 'src/3rdparty/libwebp/src/dsp')
52 files changed, 7729 insertions, 1603 deletions
diff --git a/src/3rdparty/libwebp/src/dsp/alpha_processing.c b/src/3rdparty/libwebp/src/dsp/alpha_processing.c index 1716cac..4b60e09 100644 --- a/src/3rdparty/libwebp/src/dsp/alpha_processing.c +++ b/src/3rdparty/libwebp/src/dsp/alpha_processing.c @@ -284,9 +284,9 @@ static void ApplyAlphaMultiply_16b(uint8_t* rgba4444, #endif } -static int DispatchAlpha(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint8_t* dst, int dst_stride) { +static int DispatchAlpha_C(const uint8_t* alpha, int alpha_stride, + int width, int height, + uint8_t* dst, int dst_stride) { uint32_t alpha_mask = 0xff; int i, j; @@ -303,9 +303,9 @@ static int DispatchAlpha(const uint8_t* alpha, int alpha_stride, return (alpha_mask != 0xff); } -static void DispatchAlphaToGreen(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint32_t* dst, int dst_stride) { +static void DispatchAlphaToGreen_C(const uint8_t* alpha, int alpha_stride, + int width, int height, + uint32_t* dst, int dst_stride) { int i, j; for (j = 0; j < height; ++j) { for (i = 0; i < width; ++i) { @@ -316,9 +316,9 @@ static void DispatchAlphaToGreen(const uint8_t* alpha, int alpha_stride, } } -static int ExtractAlpha(const uint8_t* argb, int argb_stride, - int width, int height, - uint8_t* alpha, int alpha_stride) { +static int ExtractAlpha_C(const uint8_t* argb, int argb_stride, + int width, int height, + uint8_t* alpha, int alpha_stride) { uint8_t alpha_mask = 0xff; int i, j; @@ -334,11 +334,17 @@ static int ExtractAlpha(const uint8_t* argb, int argb_stride, return (alpha_mask == 0xff); } +static void ExtractGreen_C(const uint32_t* argb, uint8_t* alpha, int size) { + int i; + for (i = 0; i < size; ++i) alpha[i] = argb[i] >> 8; +} + void (*WebPApplyAlphaMultiply)(uint8_t*, int, int, int, int); void (*WebPApplyAlphaMultiply4444)(uint8_t*, int, int, int); int (*WebPDispatchAlpha)(const uint8_t*, int, int, int, uint8_t*, int); void (*WebPDispatchAlphaToGreen)(const uint8_t*, int, int, int, uint32_t*, int); int (*WebPExtractAlpha)(const uint8_t*, int, int, int, uint8_t*, int); +void (*WebPExtractGreen)(const uint32_t* argb, uint8_t* alpha, int size); //------------------------------------------------------------------------------ // Init function @@ -346,6 +352,7 @@ int (*WebPExtractAlpha)(const uint8_t*, int, int, int, uint8_t*, int); extern void WebPInitAlphaProcessingMIPSdspR2(void); extern void WebPInitAlphaProcessingSSE2(void); extern void WebPInitAlphaProcessingSSE41(void); +extern void WebPInitAlphaProcessingNEON(void); static volatile VP8CPUInfo alpha_processing_last_cpuinfo_used = (VP8CPUInfo)&alpha_processing_last_cpuinfo_used; @@ -357,9 +364,11 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitAlphaProcessing(void) { WebPMultRow = WebPMultRowC; WebPApplyAlphaMultiply = ApplyAlphaMultiply; WebPApplyAlphaMultiply4444 = ApplyAlphaMultiply_16b; - WebPDispatchAlpha = DispatchAlpha; - WebPDispatchAlphaToGreen = DispatchAlphaToGreen; - WebPExtractAlpha = ExtractAlpha; + + WebPDispatchAlpha = DispatchAlpha_C; + WebPDispatchAlphaToGreen = DispatchAlphaToGreen_C; + WebPExtractAlpha = ExtractAlpha_C; + WebPExtractGreen = ExtractGreen_C; // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { @@ -373,6 +382,11 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitAlphaProcessing(void) { #endif } #endif +#if defined(WEBP_USE_NEON) + if (VP8GetCPUInfo(kNEON)) { + WebPInitAlphaProcessingNEON(); + } +#endif #if defined(WEBP_USE_MIPS_DSP_R2) if (VP8GetCPUInfo(kMIPSdspR2)) { WebPInitAlphaProcessingMIPSdspR2(); diff --git a/src/3rdparty/libwebp/src/dsp/alpha_processing_neon.c b/src/3rdparty/libwebp/src/dsp/alpha_processing_neon.c new file mode 100644 index 0000000..606a401 --- /dev/null +++ b/src/3rdparty/libwebp/src/dsp/alpha_processing_neon.c @@ -0,0 +1,191 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Utilities for processing transparent channel, NEON version. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_NEON) + +#include "./neon.h" + +//------------------------------------------------------------------------------ + +#define MULTIPLIER(a) ((a) * 0x8081) +#define PREMULTIPLY(x, m) (((x) * (m)) >> 23) + +#define MULTIPLY_BY_ALPHA(V, ALPHA, OTHER) do { \ + const uint8x8_t alpha = (V).val[(ALPHA)]; \ + const uint16x8_t r1 = vmull_u8((V).val[1], alpha); \ + const uint16x8_t g1 = vmull_u8((V).val[2], alpha); \ + const uint16x8_t b1 = vmull_u8((V).val[(OTHER)], alpha); \ + /* we use: v / 255 = (v + 1 + (v >> 8)) >> 8 */ \ + const uint16x8_t r2 = vsraq_n_u16(r1, r1, 8); \ + const uint16x8_t g2 = vsraq_n_u16(g1, g1, 8); \ + const uint16x8_t b2 = vsraq_n_u16(b1, b1, 8); \ + const uint16x8_t r3 = vaddq_u16(r2, kOne); \ + const uint16x8_t g3 = vaddq_u16(g2, kOne); \ + const uint16x8_t b3 = vaddq_u16(b2, kOne); \ + (V).val[1] = vshrn_n_u16(r3, 8); \ + (V).val[2] = vshrn_n_u16(g3, 8); \ + (V).val[(OTHER)] = vshrn_n_u16(b3, 8); \ +} while (0) + +static void ApplyAlphaMultiply_NEON(uint8_t* rgba, int alpha_first, + int w, int h, int stride) { + const uint16x8_t kOne = vdupq_n_u16(1u); + while (h-- > 0) { + uint32_t* const rgbx = (uint32_t*)rgba; + int i = 0; + if (alpha_first) { + for (; i + 8 <= w; i += 8) { + // load aaaa...|rrrr...|gggg...|bbbb... + uint8x8x4_t RGBX = vld4_u8((const uint8_t*)(rgbx + i)); + MULTIPLY_BY_ALPHA(RGBX, 0, 3); + vst4_u8((uint8_t*)(rgbx + i), RGBX); + } + } else { + for (; i + 8 <= w; i += 8) { + uint8x8x4_t RGBX = vld4_u8((const uint8_t*)(rgbx + i)); + MULTIPLY_BY_ALPHA(RGBX, 3, 0); + vst4_u8((uint8_t*)(rgbx + i), RGBX); + } + } + // Finish with left-overs. + for (; i < w; ++i) { + uint8_t* const rgb = rgba + (alpha_first ? 1 : 0); + const uint8_t* const alpha = rgba + (alpha_first ? 0 : 3); + const uint32_t a = alpha[4 * i]; + if (a != 0xff) { + const uint32_t mult = MULTIPLIER(a); + rgb[4 * i + 0] = PREMULTIPLY(rgb[4 * i + 0], mult); + rgb[4 * i + 1] = PREMULTIPLY(rgb[4 * i + 1], mult); + rgb[4 * i + 2] = PREMULTIPLY(rgb[4 * i + 2], mult); + } + } + rgba += stride; + } +} +#undef MULTIPLY_BY_ALPHA +#undef MULTIPLIER +#undef PREMULTIPLY + +//------------------------------------------------------------------------------ + +static int DispatchAlpha_NEON(const uint8_t* alpha, int alpha_stride, + int width, int height, + uint8_t* dst, int dst_stride) { + uint32_t alpha_mask = 0xffffffffu; + uint8x8_t mask8 = vdup_n_u8(0xff); + uint32_t tmp[2]; + int i, j; + for (j = 0; j < height; ++j) { + // We don't know if alpha is first or last in dst[] (depending on rgbA/Argb + // mode). So we must be sure dst[4*i + 8 - 1] is writable for the store. + // Hence the test with 'width - 1' instead of just 'width'. + for (i = 0; i + 8 <= width - 1; i += 8) { + uint8x8x4_t rgbX = vld4_u8((const uint8_t*)(dst + 4 * i)); + const uint8x8_t alphas = vld1_u8(alpha + i); + rgbX.val[0] = alphas; + vst4_u8((uint8_t*)(dst + 4 * i), rgbX); + mask8 = vand_u8(mask8, alphas); + } + for (; i < width; ++i) { + const uint32_t alpha_value = alpha[i]; + dst[4 * i] = alpha_value; + alpha_mask &= alpha_value; + } + alpha += alpha_stride; + dst += dst_stride; + } + vst1_u8((uint8_t*)tmp, mask8); + alpha_mask &= tmp[0]; + alpha_mask &= tmp[1]; + return (alpha_mask != 0xffffffffu); +} + +static void DispatchAlphaToGreen_NEON(const uint8_t* alpha, int alpha_stride, + int width, int height, + uint32_t* dst, int dst_stride) { + int i, j; + uint8x8x4_t greens; // leave A/R/B channels zero'd. + greens.val[0] = vdup_n_u8(0); + greens.val[2] = vdup_n_u8(0); + greens.val[3] = vdup_n_u8(0); + for (j = 0; j < height; ++j) { + for (i = 0; i + 8 <= width; i += 8) { + greens.val[1] = vld1_u8(alpha + i); + vst4_u8((uint8_t*)(dst + i), greens); + } + for (; i < width; ++i) dst[i] = alpha[i] << 8; + alpha += alpha_stride; + dst += dst_stride; + } +} + +static int ExtractAlpha_NEON(const uint8_t* argb, int argb_stride, + int width, int height, + uint8_t* alpha, int alpha_stride) { + uint32_t alpha_mask = 0xffffffffu; + uint8x8_t mask8 = vdup_n_u8(0xff); + uint32_t tmp[2]; + int i, j; + for (j = 0; j < height; ++j) { + // We don't know if alpha is first or last in dst[] (depending on rgbA/Argb + // mode). So we must be sure dst[4*i + 8 - 1] is writable for the store. + // Hence the test with 'width - 1' instead of just 'width'. + for (i = 0; i + 8 <= width - 1; i += 8) { + const uint8x8x4_t rgbX = vld4_u8((const uint8_t*)(argb + 4 * i)); + const uint8x8_t alphas = rgbX.val[0]; + vst1_u8((uint8_t*)(alpha + i), alphas); + mask8 = vand_u8(mask8, alphas); + } + for (; i < width; ++i) { + alpha[i] = argb[4 * i]; + alpha_mask &= alpha[i]; + } + argb += argb_stride; + alpha += alpha_stride; + } + vst1_u8((uint8_t*)tmp, mask8); + alpha_mask &= tmp[0]; + alpha_mask &= tmp[1]; + return (alpha_mask == 0xffffffffu); +} + +static void ExtractGreen_NEON(const uint32_t* argb, + uint8_t* alpha, int size) { + int i; + for (i = 0; i + 16 <= size; i += 16) { + const uint8x16x4_t rgbX = vld4q_u8((const uint8_t*)(argb + i)); + const uint8x16_t greens = rgbX.val[1]; + vst1q_u8(alpha + i, greens); + } + for (; i < size; ++i) alpha[i] = (argb[i] >> 8) & 0xff; +} + +//------------------------------------------------------------------------------ + +extern void WebPInitAlphaProcessingNEON(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitAlphaProcessingNEON(void) { + WebPApplyAlphaMultiply = ApplyAlphaMultiply_NEON; + WebPDispatchAlpha = DispatchAlpha_NEON; + WebPDispatchAlphaToGreen = DispatchAlphaToGreen_NEON; + WebPExtractAlpha = ExtractAlpha_NEON; + WebPExtractGreen = ExtractGreen_NEON; +} + +#else // !WEBP_USE_NEON + +WEBP_DSP_INIT_STUB(WebPInitAlphaProcessingNEON) + +#endif // WEBP_USE_NEON diff --git a/src/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c b/src/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c index 5acb481..83dc559 100644 --- a/src/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c +++ b/src/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c @@ -150,46 +150,46 @@ static int ExtractAlpha(const uint8_t* argb, int argb_stride, #define PREMULTIPLY(x, m) (((x) * (m)) >> 23) // We can't use a 'const int' for the SHUFFLE value, because it has to be an -// immediate in the _mm_shufflexx_epi16() instruction. We really a macro here. -#define APPLY_ALPHA(RGBX, SHUFFLE, MASK, MULT) do { \ - const __m128i argb0 = _mm_loadl_epi64((__m128i*)&(RGBX)); \ - const __m128i argb1 = _mm_unpacklo_epi8(argb0, zero); \ - const __m128i alpha0 = _mm_and_si128(argb1, MASK); \ - const __m128i alpha1 = _mm_shufflelo_epi16(alpha0, SHUFFLE); \ - const __m128i alpha2 = _mm_shufflehi_epi16(alpha1, SHUFFLE); \ - /* alpha2 = [0 a0 a0 a0][0 a1 a1 a1] */ \ - const __m128i scale0 = _mm_mullo_epi16(alpha2, MULT); \ - const __m128i scale1 = _mm_mulhi_epu16(alpha2, MULT); \ - const __m128i argb2 = _mm_mulhi_epu16(argb1, scale0); \ - const __m128i argb3 = _mm_mullo_epi16(argb1, scale1); \ - const __m128i argb4 = _mm_adds_epu16(argb2, argb3); \ - const __m128i argb5 = _mm_srli_epi16(argb4, 7); \ - const __m128i argb6 = _mm_or_si128(argb5, alpha0); \ - const __m128i argb7 = _mm_packus_epi16(argb6, zero); \ - _mm_storel_epi64((__m128i*)&(RGBX), argb7); \ +// immediate in the _mm_shufflexx_epi16() instruction. We really need a macro. +// We use: v / 255 = (v * 0x8081) >> 23, where v = alpha * {r,g,b} is a 16bit +// value. +#define APPLY_ALPHA(RGBX, SHUFFLE) do { \ + const __m128i argb0 = _mm_loadu_si128((const __m128i*)&(RGBX)); \ + const __m128i argb1_lo = _mm_unpacklo_epi8(argb0, zero); \ + const __m128i argb1_hi = _mm_unpackhi_epi8(argb0, zero); \ + const __m128i alpha0_lo = _mm_or_si128(argb1_lo, kMask); \ + const __m128i alpha0_hi = _mm_or_si128(argb1_hi, kMask); \ + const __m128i alpha1_lo = _mm_shufflelo_epi16(alpha0_lo, SHUFFLE); \ + const __m128i alpha1_hi = _mm_shufflelo_epi16(alpha0_hi, SHUFFLE); \ + const __m128i alpha2_lo = _mm_shufflehi_epi16(alpha1_lo, SHUFFLE); \ + const __m128i alpha2_hi = _mm_shufflehi_epi16(alpha1_hi, SHUFFLE); \ + /* alpha2 = [ff a0 a0 a0][ff a1 a1 a1] */ \ + const __m128i A0_lo = _mm_mullo_epi16(alpha2_lo, argb1_lo); \ + const __m128i A0_hi = _mm_mullo_epi16(alpha2_hi, argb1_hi); \ + const __m128i A1_lo = _mm_mulhi_epu16(A0_lo, kMult); \ + const __m128i A1_hi = _mm_mulhi_epu16(A0_hi, kMult); \ + const __m128i A2_lo = _mm_srli_epi16(A1_lo, 7); \ + const __m128i A2_hi = _mm_srli_epi16(A1_hi, 7); \ + const __m128i A3 = _mm_packus_epi16(A2_lo, A2_hi); \ + _mm_storeu_si128((__m128i*)&(RGBX), A3); \ } while (0) -static void ApplyAlphaMultiply(uint8_t* rgba, int alpha_first, - int w, int h, int stride) { +static void ApplyAlphaMultiply_SSE2(uint8_t* rgba, int alpha_first, + int w, int h, int stride) { const __m128i zero = _mm_setzero_si128(); - const int kSpan = 2; - const int w2 = w & ~(kSpan - 1); + const __m128i kMult = _mm_set1_epi16(0x8081u); + const __m128i kMask = _mm_set_epi16(0, 0xff, 0xff, 0, 0, 0xff, 0xff, 0); + const int kSpan = 4; while (h-- > 0) { uint32_t* const rgbx = (uint32_t*)rgba; int i; if (!alpha_first) { - const __m128i kMask = _mm_set_epi16(0xff, 0, 0, 0, 0xff, 0, 0, 0); - const __m128i kMult = - _mm_set_epi16(0, 0x8081, 0x8081, 0x8081, 0, 0x8081, 0x8081, 0x8081); - for (i = 0; i < w2; i += kSpan) { - APPLY_ALPHA(rgbx[i], _MM_SHUFFLE(0, 3, 3, 3), kMask, kMult); + for (i = 0; i + kSpan <= w; i += kSpan) { + APPLY_ALPHA(rgbx[i], _MM_SHUFFLE(2, 3, 3, 3)); } } else { - const __m128i kMask = _mm_set_epi16(0, 0, 0, 0xff, 0, 0, 0, 0xff); - const __m128i kMult = - _mm_set_epi16(0x8081, 0x8081, 0x8081, 0, 0x8081, 0x8081, 0x8081, 0); - for (i = 0; i < w2; i += kSpan) { - APPLY_ALPHA(rgbx[i], _MM_SHUFFLE(0, 0, 0, 3), kMask, kMult); + for (i = 0; i + kSpan <= w; i += kSpan) { + APPLY_ALPHA(rgbx[i], _MM_SHUFFLE(0, 0, 0, 1)); } } // Finish with left-overs. @@ -213,64 +213,51 @@ static void ApplyAlphaMultiply(uint8_t* rgba, int alpha_first, // ----------------------------------------------------------------------------- // Apply alpha value to rows -// We use: kINV255 = (1 << 24) / 255 = 0x010101 -// So: a * kINV255 = (a << 16) | [(a << 8) | a] -// -> _mm_mulhi_epu16() takes care of the (a<<16) part, -// and _mm_mullo_epu16(a * 0x0101,...) takes care of the "(a << 8) | a" one. - -static void MultARGBRow(uint32_t* const ptr, int width, int inverse) { +static void MultARGBRow_SSE2(uint32_t* const ptr, int width, int inverse) { int x = 0; if (!inverse) { const int kSpan = 2; const __m128i zero = _mm_setzero_si128(); - const __m128i kRound = - _mm_set_epi16(0, 1 << 7, 1 << 7, 1 << 7, 0, 1 << 7, 1 << 7, 1 << 7); - const __m128i kMult = - _mm_set_epi16(0, 0x0101, 0x0101, 0x0101, 0, 0x0101, 0x0101, 0x0101); - const __m128i kOne64 = _mm_set_epi16(1u << 8, 0, 0, 0, 1u << 8, 0, 0, 0); - const int w2 = width & ~(kSpan - 1); - for (x = 0; x < w2; x += kSpan) { - const __m128i argb0 = _mm_loadl_epi64((__m128i*)&ptr[x]); - const __m128i argb1 = _mm_unpacklo_epi8(argb0, zero); - const __m128i tmp0 = _mm_shufflelo_epi16(argb1, _MM_SHUFFLE(3, 3, 3, 3)); - const __m128i tmp1 = _mm_shufflehi_epi16(tmp0, _MM_SHUFFLE(3, 3, 3, 3)); - const __m128i tmp2 = _mm_srli_epi64(tmp1, 16); - const __m128i scale0 = _mm_mullo_epi16(tmp1, kMult); - const __m128i scale1 = _mm_or_si128(tmp2, kOne64); - const __m128i argb2 = _mm_mulhi_epu16(argb1, scale0); - const __m128i argb3 = _mm_mullo_epi16(argb1, scale1); - const __m128i argb4 = _mm_adds_epu16(argb2, argb3); - const __m128i argb5 = _mm_adds_epu16(argb4, kRound); - const __m128i argb6 = _mm_srli_epi16(argb5, 8); - const __m128i argb7 = _mm_packus_epi16(argb6, zero); - _mm_storel_epi64((__m128i*)&ptr[x], argb7); + const __m128i k128 = _mm_set1_epi16(128); + const __m128i kMult = _mm_set1_epi16(0x0101); + const __m128i kMask = _mm_set_epi16(0, 0xff, 0, 0, 0, 0xff, 0, 0); + for (x = 0; x + kSpan <= width; x += kSpan) { + // To compute 'result = (int)(a * x / 255. + .5)', we use: + // tmp = a * v + 128, result = (tmp * 0x0101u) >> 16 + const __m128i A0 = _mm_loadl_epi64((const __m128i*)&ptr[x]); + const __m128i A1 = _mm_unpacklo_epi8(A0, zero); + const __m128i A2 = _mm_or_si128(A1, kMask); + const __m128i A3 = _mm_shufflelo_epi16(A2, _MM_SHUFFLE(2, 3, 3, 3)); + const __m128i A4 = _mm_shufflehi_epi16(A3, _MM_SHUFFLE(2, 3, 3, 3)); + // here, A4 = [ff a0 a0 a0][ff a1 a1 a1] + const __m128i A5 = _mm_mullo_epi16(A4, A1); + const __m128i A6 = _mm_add_epi16(A5, k128); + const __m128i A7 = _mm_mulhi_epu16(A6, kMult); + const __m128i A10 = _mm_packus_epi16(A7, zero); + _mm_storel_epi64((__m128i*)&ptr[x], A10); } } width -= x; if (width > 0) WebPMultARGBRowC(ptr + x, width, inverse); } -static void MultRow(uint8_t* const ptr, const uint8_t* const alpha, - int width, int inverse) { +static void MultRow_SSE2(uint8_t* const ptr, const uint8_t* const alpha, + int width, int inverse) { int x = 0; if (!inverse) { - const int kSpan = 8; const __m128i zero = _mm_setzero_si128(); - const __m128i kRound = _mm_set1_epi16(1 << 7); - const int w2 = width & ~(kSpan - 1); - for (x = 0; x < w2; x += kSpan) { + const __m128i k128 = _mm_set1_epi16(128); + const __m128i kMult = _mm_set1_epi16(0x0101); + for (x = 0; x + 8 <= width; x += 8) { const __m128i v0 = _mm_loadl_epi64((__m128i*)&ptr[x]); + const __m128i a0 = _mm_loadl_epi64((const __m128i*)&alpha[x]); const __m128i v1 = _mm_unpacklo_epi8(v0, zero); - const __m128i alpha0 = _mm_loadl_epi64((const __m128i*)&alpha[x]); - const __m128i alpha1 = _mm_unpacklo_epi8(alpha0, zero); - const __m128i alpha2 = _mm_unpacklo_epi8(alpha0, alpha0); - const __m128i v2 = _mm_mulhi_epu16(v1, alpha2); - const __m128i v3 = _mm_mullo_epi16(v1, alpha1); - const __m128i v4 = _mm_adds_epu16(v2, v3); - const __m128i v5 = _mm_adds_epu16(v4, kRound); - const __m128i v6 = _mm_srli_epi16(v5, 8); - const __m128i v7 = _mm_packus_epi16(v6, zero); - _mm_storel_epi64((__m128i*)&ptr[x], v7); + const __m128i a1 = _mm_unpacklo_epi8(a0, zero); + const __m128i v2 = _mm_mullo_epi16(v1, a1); + const __m128i v3 = _mm_add_epi16(v2, k128); + const __m128i v4 = _mm_mulhi_epu16(v3, kMult); + const __m128i v5 = _mm_packus_epi16(v4, zero); + _mm_storel_epi64((__m128i*)&ptr[x], v5); } } width -= x; @@ -283,9 +270,9 @@ static void MultRow(uint8_t* const ptr, const uint8_t* const alpha, extern void WebPInitAlphaProcessingSSE2(void); WEBP_TSAN_IGNORE_FUNCTION void WebPInitAlphaProcessingSSE2(void) { - WebPMultARGBRow = MultARGBRow; - WebPMultRow = MultRow; - WebPApplyAlphaMultiply = ApplyAlphaMultiply; + WebPMultARGBRow = MultARGBRow_SSE2; + WebPMultRow = MultRow_SSE2; + WebPApplyAlphaMultiply = ApplyAlphaMultiply_SSE2; WebPDispatchAlpha = DispatchAlpha; WebPDispatchAlphaToGreen = DispatchAlphaToGreen; WebPExtractAlpha = ExtractAlpha; diff --git a/src/3rdparty/libwebp/src/dsp/common_sse2.h b/src/3rdparty/libwebp/src/dsp/common_sse2.h index 7cea13f..995d7cf 100644 --- a/src/3rdparty/libwebp/src/dsp/common_sse2.h +++ b/src/3rdparty/libwebp/src/dsp/common_sse2.h @@ -100,6 +100,91 @@ static WEBP_INLINE void VP8Transpose_2_4x4_16b( // a03 a13 a23 a33 b03 b13 b23 b33 } +//------------------------------------------------------------------------------ +// Channel mixing. + +// Function used several times in VP8PlanarTo24b. +// It samples the in buffer as follows: one every two unsigned char is stored +// at the beginning of the buffer, while the other half is stored at the end. +#define VP8PlanarTo24bHelper(IN, OUT) \ + do { \ + const __m128i v_mask = _mm_set1_epi16(0x00ff); \ + /* Take one every two upper 8b values.*/ \ + (OUT##0) = _mm_packus_epi16(_mm_and_si128((IN##0), v_mask), \ + _mm_and_si128((IN##1), v_mask)); \ + (OUT##1) = _mm_packus_epi16(_mm_and_si128((IN##2), v_mask), \ + _mm_and_si128((IN##3), v_mask)); \ + (OUT##2) = _mm_packus_epi16(_mm_and_si128((IN##4), v_mask), \ + _mm_and_si128((IN##5), v_mask)); \ + /* Take one every two lower 8b values.*/ \ + (OUT##3) = _mm_packus_epi16(_mm_srli_epi16((IN##0), 8), \ + _mm_srli_epi16((IN##1), 8)); \ + (OUT##4) = _mm_packus_epi16(_mm_srli_epi16((IN##2), 8), \ + _mm_srli_epi16((IN##3), 8)); \ + (OUT##5) = _mm_packus_epi16(_mm_srli_epi16((IN##4), 8), \ + _mm_srli_epi16((IN##5), 8)); \ + } while (0) + +// Pack the planar buffers +// rrrr... rrrr... gggg... gggg... bbbb... bbbb.... +// triplet by triplet in the output buffer rgb as rgbrgbrgbrgb ... +static WEBP_INLINE void VP8PlanarTo24b(__m128i* const in0, __m128i* const in1, + __m128i* const in2, __m128i* const in3, + __m128i* const in4, __m128i* const in5) { + // The input is 6 registers of sixteen 8b but for the sake of explanation, + // let's take 6 registers of four 8b values. + // To pack, we will keep taking one every two 8b integer and move it + // around as follows: + // Input: + // r0r1r2r3 | r4r5r6r7 | g0g1g2g3 | g4g5g6g7 | b0b1b2b3 | b4b5b6b7 + // Split the 6 registers in two sets of 3 registers: the first set as the even + // 8b bytes, the second the odd ones: + // r0r2r4r6 | g0g2g4g6 | b0b2b4b6 | r1r3r5r7 | g1g3g5g7 | b1b3b5b7 + // Repeat the same permutations twice more: + // r0r4g0g4 | b0b4r1r5 | g1g5b1b5 | r2r6g2g6 | b2b6r3r7 | g3g7b3b7 + // r0g0b0r1 | g1b1r2g2 | b2r3g3b3 | r4g4b4r5 | g5b5r6g6 | b6r7g7b7 + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + VP8PlanarTo24bHelper(*in, tmp); + VP8PlanarTo24bHelper(tmp, *in); + VP8PlanarTo24bHelper(*in, tmp); + // We need to do it two more times than the example as we have sixteen bytes. + { + __m128i out0, out1, out2, out3, out4, out5; + VP8PlanarTo24bHelper(tmp, out); + VP8PlanarTo24bHelper(out, *in); + } +} + +#undef VP8PlanarTo24bHelper + +// Convert four packed four-channel buffers like argbargbargbargb... into the +// split channels aaaaa ... rrrr ... gggg .... bbbbb ...... +static WEBP_INLINE void VP8L32bToPlanar(__m128i* const in0, + __m128i* const in1, + __m128i* const in2, + __m128i* const in3) { + // Column-wise transpose. + const __m128i A0 = _mm_unpacklo_epi8(*in0, *in1); + const __m128i A1 = _mm_unpackhi_epi8(*in0, *in1); + const __m128i A2 = _mm_unpacklo_epi8(*in2, *in3); + const __m128i A3 = _mm_unpackhi_epi8(*in2, *in3); + const __m128i B0 = _mm_unpacklo_epi8(A0, A1); + const __m128i B1 = _mm_unpackhi_epi8(A0, A1); + const __m128i B2 = _mm_unpacklo_epi8(A2, A3); + const __m128i B3 = _mm_unpackhi_epi8(A2, A3); + // C0 = g7 g6 ... g1 g0 | b7 b6 ... b1 b0 + // C1 = a7 a6 ... a1 a0 | r7 r6 ... r1 r0 + const __m128i C0 = _mm_unpacklo_epi8(B0, B1); + const __m128i C1 = _mm_unpackhi_epi8(B0, B1); + const __m128i C2 = _mm_unpacklo_epi8(B2, B3); + const __m128i C3 = _mm_unpackhi_epi8(B2, B3); + // Gather the channels. + *in0 = _mm_unpackhi_epi64(C1, C3); + *in1 = _mm_unpacklo_epi64(C1, C3); + *in2 = _mm_unpackhi_epi64(C0, C2); + *in3 = _mm_unpacklo_epi64(C0, C2); +} + #endif // WEBP_USE_SSE2 #ifdef __cplusplus diff --git a/src/3rdparty/libwebp/src/dsp/cost.c b/src/3rdparty/libwebp/src/dsp/cost.c index fe72d26..58ddea7 100644 --- a/src/3rdparty/libwebp/src/dsp/cost.c +++ b/src/3rdparty/libwebp/src/dsp/cost.c @@ -10,7 +10,7 @@ // Author: Skal (pascal.massimino@gmail.com) #include "./dsp.h" -#include "../enc/cost.h" +#include "../enc/cost_enc.h" //------------------------------------------------------------------------------ // Boolean-cost cost table diff --git a/src/3rdparty/libwebp/src/dsp/cost_mips32.c b/src/3rdparty/libwebp/src/dsp/cost_mips32.c index d1e240e..3102da8 100644 --- a/src/3rdparty/libwebp/src/dsp/cost_mips32.c +++ b/src/3rdparty/libwebp/src/dsp/cost_mips32.c @@ -13,7 +13,7 @@ #if defined(WEBP_USE_MIPS32) -#include "../enc/cost.h" +#include "../enc/cost_enc.h" static int GetResidualCost(int ctx0, const VP8Residual* const res) { int temp0, temp1; diff --git a/src/3rdparty/libwebp/src/dsp/cost_mips_dsp_r2.c b/src/3rdparty/libwebp/src/dsp/cost_mips_dsp_r2.c index ce64067..6ec8aeb 100644 --- a/src/3rdparty/libwebp/src/dsp/cost_mips_dsp_r2.c +++ b/src/3rdparty/libwebp/src/dsp/cost_mips_dsp_r2.c @@ -13,7 +13,7 @@ #if defined(WEBP_USE_MIPS_DSP_R2) -#include "../enc/cost.h" +#include "../enc/cost_enc.h" static int GetResidualCost(int ctx0, const VP8Residual* const res) { int temp0, temp1; diff --git a/src/3rdparty/libwebp/src/dsp/cost_sse2.c b/src/3rdparty/libwebp/src/dsp/cost_sse2.c index 0cb1c1f..421d51f 100644 --- a/src/3rdparty/libwebp/src/dsp/cost_sse2.c +++ b/src/3rdparty/libwebp/src/dsp/cost_sse2.c @@ -16,8 +16,8 @@ #if defined(WEBP_USE_SSE2) #include <emmintrin.h> -#include "../enc/cost.h" -#include "../enc/vp8enci.h" +#include "../enc/cost_enc.h" +#include "../enc/vp8i_enc.h" #include "../utils/utils.h" //------------------------------------------------------------------------------ diff --git a/src/3rdparty/libwebp/src/dsp/cpu.c b/src/3rdparty/libwebp/src/dsp/cpu.c index cbb08db..b5583b6 100644 --- a/src/3rdparty/libwebp/src/dsp/cpu.c +++ b/src/3rdparty/libwebp/src/dsp/cpu.c @@ -95,26 +95,62 @@ static WEBP_INLINE uint64_t xgetbv(void) { #endif #if defined(__i386__) || defined(__x86_64__) || defined(WEBP_MSC_SSE2) + +// helper function for run-time detection of slow SSSE3 platforms +static int CheckSlowModel(int info) { + // Table listing display models with longer latencies for the bsr instruction + // (ie 2 cycles vs 10/16 cycles) and some SSSE3 instructions like pshufb. + // Refer to Intel 64 and IA-32 Architectures Optimization Reference Manual. + static const uint8_t kSlowModels[] = { + 0x37, 0x4a, 0x4d, // Silvermont Microarchitecture + 0x1c, 0x26, 0x27 // Atom Microarchitecture + }; + const uint32_t model = ((info & 0xf0000) >> 12) | ((info >> 4) & 0xf); + const uint32_t family = (info >> 8) & 0xf; + if (family == 0x06) { + size_t i; + for (i = 0; i < sizeof(kSlowModels) / sizeof(kSlowModels[0]); ++i) { + if (model == kSlowModels[i]) return 1; + } + } + return 0; +} + static int x86CPUInfo(CPUFeature feature) { int max_cpuid_value; int cpu_info[4]; + int is_intel = 0; // get the highest feature value cpuid supports GetCPUInfo(cpu_info, 0); max_cpuid_value = cpu_info[0]; if (max_cpuid_value < 1) { return 0; + } else { + const int VENDOR_ID_INTEL_EBX = 0x756e6547; // uneG + const int VENDOR_ID_INTEL_EDX = 0x49656e69; // Ieni + const int VENDOR_ID_INTEL_ECX = 0x6c65746e; // letn + is_intel = (cpu_info[1] == VENDOR_ID_INTEL_EBX && + cpu_info[2] == VENDOR_ID_INTEL_ECX && + cpu_info[3] == VENDOR_ID_INTEL_EDX); // genuine Intel? } GetCPUInfo(cpu_info, 1); if (feature == kSSE2) { - return 0 != (cpu_info[3] & 0x04000000); + return !!(cpu_info[3] & (1 << 26)); } if (feature == kSSE3) { - return 0 != (cpu_info[2] & 0x00000001); + return !!(cpu_info[2] & (1 << 0)); + } + if (feature == kSlowSSSE3) { + if (is_intel && (cpu_info[2] & (1 << 0))) { // SSSE3? + return CheckSlowModel(cpu_info[0]); + } + return 0; } + if (feature == kSSE4_1) { - return 0 != (cpu_info[2] & 0x00080000); + return !!(cpu_info[2] & (1 << 19)); } if (feature == kAVX) { // bits 27 (OSXSAVE) & 28 (256-bit AVX) @@ -126,7 +162,7 @@ static int x86CPUInfo(CPUFeature feature) { if (feature == kAVX2) { if (x86CPUInfo(kAVX) && max_cpuid_value >= 7) { GetCPUInfo(cpu_info, 7); - return ((cpu_info[1] & 0x00000020) == 0x00000020); + return !!(cpu_info[1] & (1 << 5)); } } return 0; @@ -184,4 +220,3 @@ VP8CPUInfo VP8GetCPUInfo = mipsCPUInfo; #else VP8CPUInfo VP8GetCPUInfo = NULL; #endif - diff --git a/src/3rdparty/libwebp/src/dsp/dec.c b/src/3rdparty/libwebp/src/dsp/dec.c index e92d693..007e985 100644 --- a/src/3rdparty/libwebp/src/dsp/dec.c +++ b/src/3rdparty/libwebp/src/dsp/dec.c @@ -12,7 +12,7 @@ // Author: Skal (pascal.massimino@gmail.com) #include "./dsp.h" -#include "../dec/vp8i.h" +#include "../dec/vp8i_dec.h" #include "../utils/utils.h" //------------------------------------------------------------------------------ @@ -239,7 +239,7 @@ VP8PredFunc VP8PredLuma16[NUM_B_DC_MODES]; //------------------------------------------------------------------------------ // 4x4 -#define AVG3(a, b, c) (((a) + 2 * (b) + (c) + 2) >> 2) +#define AVG3(a, b, c) ((uint8_t)(((a) + 2 * (b) + (c) + 2) >> 2)) #define AVG2(a, b) (((a) + (b) + 1) >> 1) static void VE4(uint8_t* dst) { // vertical diff --git a/src/3rdparty/libwebp/src/dsp/dec_clip_tables.c b/src/3rdparty/libwebp/src/dsp/dec_clip_tables.c index 3b6dde8..74ba34c 100644 --- a/src/3rdparty/libwebp/src/dsp/dec_clip_tables.c +++ b/src/3rdparty/libwebp/src/dsp/dec_clip_tables.c @@ -63,7 +63,7 @@ static const uint8_t abs0[255 + 255 + 1] = { 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff }; -static const int8_t sclip1[1020 + 1020 + 1] = { +static const uint8_t sclip1[1020 + 1020 + 1] = { 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, @@ -236,7 +236,7 @@ static const int8_t sclip1[1020 + 1020 + 1] = { 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f }; -static const int8_t sclip2[112 + 112 + 1] = { +static const uint8_t sclip2[112 + 112 + 1] = { 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, @@ -339,8 +339,8 @@ static volatile int tables_ok = 0; #endif -const int8_t* const VP8ksclip1 = &sclip1[1020]; -const int8_t* const VP8ksclip2 = &sclip2[112]; +const int8_t* const VP8ksclip1 = (const int8_t*)&sclip1[1020]; +const int8_t* const VP8ksclip2 = (const int8_t*)&sclip2[112]; const uint8_t* const VP8kclip1 = &clip1[255]; const uint8_t* const VP8kabs0 = &abs0[255]; diff --git a/src/3rdparty/libwebp/src/dsp/dec_msa.c b/src/3rdparty/libwebp/src/dsp/dec_msa.c index f76055c..8d9c98c 100644 --- a/src/3rdparty/libwebp/src/dsp/dec_msa.c +++ b/src/3rdparty/libwebp/src/dsp/dec_msa.c @@ -154,6 +154,820 @@ static void TransformAC3(const int16_t* in, uint8_t* dst) { } //------------------------------------------------------------------------------ +// Edge filtering functions + +#define FLIP_SIGN2(in0, in1, out0, out1) { \ + out0 = (v16i8)__msa_xori_b(in0, 0x80); \ + out1 = (v16i8)__msa_xori_b(in1, 0x80); \ +} + +#define FLIP_SIGN4(in0, in1, in2, in3, out0, out1, out2, out3) { \ + FLIP_SIGN2(in0, in1, out0, out1); \ + FLIP_SIGN2(in2, in3, out2, out3); \ +} + +#define FILT_VAL(q0_m, p0_m, mask, filt) do { \ + v16i8 q0_sub_p0; \ + q0_sub_p0 = __msa_subs_s_b(q0_m, p0_m); \ + filt = __msa_adds_s_b(filt, q0_sub_p0); \ + filt = __msa_adds_s_b(filt, q0_sub_p0); \ + filt = __msa_adds_s_b(filt, q0_sub_p0); \ + filt = filt & mask; \ +} while (0) + +#define FILT2(q_m, p_m, q, p) do { \ + u_r = SRAI_H(temp1, 7); \ + u_r = __msa_sat_s_h(u_r, 7); \ + u_l = SRAI_H(temp3, 7); \ + u_l = __msa_sat_s_h(u_l, 7); \ + u = __msa_pckev_b((v16i8)u_l, (v16i8)u_r); \ + q_m = __msa_subs_s_b(q_m, u); \ + p_m = __msa_adds_s_b(p_m, u); \ + q = __msa_xori_b((v16u8)q_m, 0x80); \ + p = __msa_xori_b((v16u8)p_m, 0x80); \ +} while (0) + +#define LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev) do { \ + v16i8 p1_m, p0_m, q0_m, q1_m; \ + v16i8 filt, t1, t2; \ + const v16i8 cnst4b = __msa_ldi_b(4); \ + const v16i8 cnst3b = __msa_ldi_b(3); \ + \ + FLIP_SIGN4(p1, p0, q0, q1, p1_m, p0_m, q0_m, q1_m); \ + filt = __msa_subs_s_b(p1_m, q1_m); \ + filt = filt & hev; \ + FILT_VAL(q0_m, p0_m, mask, filt); \ + t1 = __msa_adds_s_b(filt, cnst4b); \ + t1 = SRAI_B(t1, 3); \ + t2 = __msa_adds_s_b(filt, cnst3b); \ + t2 = SRAI_B(t2, 3); \ + q0_m = __msa_subs_s_b(q0_m, t1); \ + q0 = __msa_xori_b((v16u8)q0_m, 0x80); \ + p0_m = __msa_adds_s_b(p0_m, t2); \ + p0 = __msa_xori_b((v16u8)p0_m, 0x80); \ + filt = __msa_srari_b(t1, 1); \ + hev = __msa_xori_b(hev, 0xff); \ + filt = filt & hev; \ + q1_m = __msa_subs_s_b(q1_m, filt); \ + q1 = __msa_xori_b((v16u8)q1_m, 0x80); \ + p1_m = __msa_adds_s_b(p1_m, filt); \ + p1 = __msa_xori_b((v16u8)p1_m, 0x80); \ +} while (0) + +#define LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev) do { \ + v16i8 p2_m, p1_m, p0_m, q2_m, q1_m, q0_m; \ + v16i8 u, filt, t1, t2, filt_sign; \ + v8i16 filt_r, filt_l, u_r, u_l; \ + v8i16 temp0, temp1, temp2, temp3; \ + const v16i8 cnst4b = __msa_ldi_b(4); \ + const v16i8 cnst3b = __msa_ldi_b(3); \ + const v8i16 cnst9h = __msa_ldi_h(9); \ + \ + FLIP_SIGN4(p1, p0, q0, q1, p1_m, p0_m, q0_m, q1_m); \ + filt = __msa_subs_s_b(p1_m, q1_m); \ + FILT_VAL(q0_m, p0_m, mask, filt); \ + FLIP_SIGN2(p2, q2, p2_m, q2_m); \ + t2 = filt & hev; \ + /* filt_val &= ~hev */ \ + hev = __msa_xori_b(hev, 0xff); \ + filt = filt & hev; \ + t1 = __msa_adds_s_b(t2, cnst4b); \ + t1 = SRAI_B(t1, 3); \ + t2 = __msa_adds_s_b(t2, cnst3b); \ + t2 = SRAI_B(t2, 3); \ + q0_m = __msa_subs_s_b(q0_m, t1); \ + p0_m = __msa_adds_s_b(p0_m, t2); \ + filt_sign = __msa_clti_s_b(filt, 0); \ + ILVRL_B2_SH(filt_sign, filt, filt_r, filt_l); \ + /* update q2/p2 */ \ + temp0 = filt_r * cnst9h; \ + temp1 = ADDVI_H(temp0, 63); \ + temp2 = filt_l * cnst9h; \ + temp3 = ADDVI_H(temp2, 63); \ + FILT2(q2_m, p2_m, q2, p2); \ + /* update q1/p1 */ \ + temp1 = temp1 + temp0; \ + temp3 = temp3 + temp2; \ + FILT2(q1_m, p1_m, q1, p1); \ + /* update q0/p0 */ \ + temp1 = temp1 + temp0; \ + temp3 = temp3 + temp2; \ + FILT2(q0_m, p0_m, q0, p0); \ +} while (0) + +#define LPF_MASK_HEV(p3_in, p2_in, p1_in, p0_in, \ + q0_in, q1_in, q2_in, q3_in, \ + limit_in, b_limit_in, thresh_in, \ + hev_out, mask_out) do { \ + v16u8 p3_asub_p2_m, p2_asub_p1_m, p1_asub_p0_m, q1_asub_q0_m; \ + v16u8 p1_asub_q1_m, p0_asub_q0_m, q3_asub_q2_m, q2_asub_q1_m; \ + v16u8 flat_out; \ + \ + /* absolute subtraction of pixel values */ \ + p3_asub_p2_m = __msa_asub_u_b(p3_in, p2_in); \ + p2_asub_p1_m = __msa_asub_u_b(p2_in, p1_in); \ + p1_asub_p0_m = __msa_asub_u_b(p1_in, p0_in); \ + q1_asub_q0_m = __msa_asub_u_b(q1_in, q0_in); \ + q2_asub_q1_m = __msa_asub_u_b(q2_in, q1_in); \ + q3_asub_q2_m = __msa_asub_u_b(q3_in, q2_in); \ + p0_asub_q0_m = __msa_asub_u_b(p0_in, q0_in); \ + p1_asub_q1_m = __msa_asub_u_b(p1_in, q1_in); \ + /* calculation of hev */ \ + flat_out = __msa_max_u_b(p1_asub_p0_m, q1_asub_q0_m); \ + hev_out = (thresh_in < flat_out); \ + /* calculation of mask */ \ + p0_asub_q0_m = __msa_adds_u_b(p0_asub_q0_m, p0_asub_q0_m); \ + p1_asub_q1_m = SRAI_B(p1_asub_q1_m, 1); \ + p0_asub_q0_m = __msa_adds_u_b(p0_asub_q0_m, p1_asub_q1_m); \ + mask_out = (b_limit_in < p0_asub_q0_m); \ + mask_out = __msa_max_u_b(flat_out, mask_out); \ + p3_asub_p2_m = __msa_max_u_b(p3_asub_p2_m, p2_asub_p1_m); \ + mask_out = __msa_max_u_b(p3_asub_p2_m, mask_out); \ + q2_asub_q1_m = __msa_max_u_b(q2_asub_q1_m, q3_asub_q2_m); \ + mask_out = __msa_max_u_b(q2_asub_q1_m, mask_out); \ + mask_out = (limit_in < mask_out); \ + mask_out = __msa_xori_b(mask_out, 0xff); \ +} while (0) + +#define ST6x1_UB(in0, in0_idx, in1, in1_idx, pdst, stride) do { \ + const uint16_t tmp0_h = __msa_copy_s_h((v8i16)in1, in1_idx); \ + const uint32_t tmp0_w = __msa_copy_s_w((v4i32)in0, in0_idx); \ + SW(tmp0_w, pdst); \ + SH(tmp0_h, pdst + stride); \ +} while (0) + +#define ST6x4_UB(in0, start_in0_idx, in1, start_in1_idx, pdst, stride) do { \ + uint8_t* ptmp1 = (uint8_t*)pdst; \ + ST6x1_UB(in0, start_in0_idx, in1, start_in1_idx, ptmp1, 4); \ + ptmp1 += stride; \ + ST6x1_UB(in0, start_in0_idx + 1, in1, start_in1_idx + 1, ptmp1, 4); \ + ptmp1 += stride; \ + ST6x1_UB(in0, start_in0_idx + 2, in1, start_in1_idx + 2, ptmp1, 4); \ + ptmp1 += stride; \ + ST6x1_UB(in0, start_in0_idx + 3, in1, start_in1_idx + 3, ptmp1, 4); \ +} while (0) + +#define LPF_SIMPLE_FILT(p1_in, p0_in, q0_in, q1_in, mask) do { \ + v16i8 p1_m, p0_m, q0_m, q1_m, filt, filt1, filt2; \ + const v16i8 cnst4b = __msa_ldi_b(4); \ + const v16i8 cnst3b = __msa_ldi_b(3); \ + \ + FLIP_SIGN4(p1_in, p0_in, q0_in, q1_in, p1_m, p0_m, q0_m, q1_m); \ + filt = __msa_subs_s_b(p1_m, q1_m); \ + FILT_VAL(q0_m, p0_m, mask, filt); \ + filt1 = __msa_adds_s_b(filt, cnst4b); \ + filt1 = SRAI_B(filt1, 3); \ + filt2 = __msa_adds_s_b(filt, cnst3b); \ + filt2 = SRAI_B(filt2, 3); \ + q0_m = __msa_subs_s_b(q0_m, filt1); \ + p0_m = __msa_adds_s_b(p0_m, filt2); \ + q0_in = __msa_xori_b((v16u8)q0_m, 0x80); \ + p0_in = __msa_xori_b((v16u8)p0_m, 0x80); \ +} while (0) + +#define LPF_SIMPLE_MASK(p1, p0, q0, q1, b_limit, mask) do { \ + v16u8 p1_a_sub_q1, p0_a_sub_q0; \ + \ + p0_a_sub_q0 = __msa_asub_u_b(p0, q0); \ + p1_a_sub_q1 = __msa_asub_u_b(p1, q1); \ + p1_a_sub_q1 = (v16u8)__msa_srli_b((v16i8)p1_a_sub_q1, 1); \ + p0_a_sub_q0 = __msa_adds_u_b(p0_a_sub_q0, p0_a_sub_q0); \ + mask = __msa_adds_u_b(p0_a_sub_q0, p1_a_sub_q1); \ + mask = (mask <= b_limit); \ +} while (0) + +static void VFilter16(uint8_t* src, int stride, + int b_limit_in, int limit_in, int thresh_in) { + uint8_t* ptemp = src - 4 * stride; + v16u8 p3, p2, p1, p0, q3, q2, q1, q0; + v16u8 mask, hev; + const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in); + const v16u8 limit = (v16u8)__msa_fill_b(limit_in); + const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); + + LD_UB8(ptemp, stride, p3, p2, p1, p0, q0, q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask); + LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + ptemp = src - 3 * stride; + ST_UB4(p2, p1, p0, q0, ptemp, stride); + ptemp += (4 * stride); + ST_UB2(q1, q2, ptemp, stride); +} + +static void HFilter16(uint8_t* src, int stride, + int b_limit_in, int limit_in, int thresh_in) { + uint8_t* ptmp = src - 4; + v16u8 p3, p2, p1, p0, q3, q2, q1, q0; + v16u8 mask, hev; + v16u8 row0, row1, row2, row3, row4, row5, row6, row7, row8; + v16u8 row9, row10, row11, row12, row13, row14, row15; + v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); + const v16u8 limit = (v16u8)__msa_fill_b(limit_in); + const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in); + + LD_UB8(ptmp, stride, row0, row1, row2, row3, row4, row5, row6, row7); + ptmp += (8 * stride); + LD_UB8(ptmp, stride, row8, row9, row10, row11, row12, row13, row14, row15); + TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7, + row8, row9, row10, row11, row12, row13, row14, row15, + p3, p2, p1, p0, q0, q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask); + LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + ILVR_B2_SH(p1, p2, q0, p0, tmp0, tmp1); + ILVRL_H2_SH(tmp1, tmp0, tmp3, tmp4); + ILVL_B2_SH(p1, p2, q0, p0, tmp0, tmp1); + ILVRL_H2_SH(tmp1, tmp0, tmp6, tmp7); + ILVRL_B2_SH(q2, q1, tmp2, tmp5); + ptmp = src - 3; + ST6x1_UB(tmp3, 0, tmp2, 0, ptmp, 4); + ptmp += stride; + ST6x1_UB(tmp3, 1, tmp2, 1, ptmp, 4); + ptmp += stride; + ST6x1_UB(tmp3, 2, tmp2, 2, ptmp, 4); + ptmp += stride; + ST6x1_UB(tmp3, 3, tmp2, 3, ptmp, 4); + ptmp += stride; + ST6x1_UB(tmp4, 0, tmp2, 4, ptmp, 4); + ptmp += stride; + ST6x1_UB(tmp4, 1, tmp2, 5, ptmp, 4); + ptmp += stride; + ST6x1_UB(tmp4, 2, tmp2, 6, ptmp, 4); + ptmp += stride; + ST6x1_UB(tmp4, 3, tmp2, 7, ptmp, 4); + ptmp += stride; + ST6x1_UB(tmp6, 0, tmp5, 0, ptmp, 4); + ptmp += stride; + ST6x1_UB(tmp6, 1, tmp5, 1, ptmp, 4); + ptmp += stride; + ST6x1_UB(tmp6, 2, tmp5, 2, ptmp, 4); + ptmp += stride; + ST6x1_UB(tmp6, 3, tmp5, 3, ptmp, 4); + ptmp += stride; + ST6x1_UB(tmp7, 0, tmp5, 4, ptmp, 4); + ptmp += stride; + ST6x1_UB(tmp7, 1, tmp5, 5, ptmp, 4); + ptmp += stride; + ST6x1_UB(tmp7, 2, tmp5, 6, ptmp, 4); + ptmp += stride; + ST6x1_UB(tmp7, 3, tmp5, 7, ptmp, 4); +} + +// on three inner edges +static void VFilterHorEdge16i(uint8_t* src, int stride, + int b_limit, int limit, int thresh) { + v16u8 mask, hev; + v16u8 p3, p2, p1, p0, q3, q2, q1, q0; + const v16u8 thresh0 = (v16u8)__msa_fill_b(thresh); + const v16u8 b_limit0 = (v16u8)__msa_fill_b(b_limit); + const v16u8 limit0 = (v16u8)__msa_fill_b(limit); + + LD_UB8((src - 4 * stride), stride, p3, p2, p1, p0, q0, q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, + hev, mask); + LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); + ST_UB4(p1, p0, q0, q1, (src - 2 * stride), stride); +} + +static void VFilter16i(uint8_t* src_y, int stride, + int b_limit, int limit, int thresh) { + VFilterHorEdge16i(src_y + 4 * stride, stride, b_limit, limit, thresh); + VFilterHorEdge16i(src_y + 8 * stride, stride, b_limit, limit, thresh); + VFilterHorEdge16i(src_y + 12 * stride, stride, b_limit, limit, thresh); +} + +static void HFilterVertEdge16i(uint8_t* src, int stride, + int b_limit, int limit, int thresh) { + v16u8 mask, hev; + v16u8 p3, p2, p1, p0, q3, q2, q1, q0; + v16u8 row0, row1, row2, row3, row4, row5, row6, row7; + v16u8 row8, row9, row10, row11, row12, row13, row14, row15; + v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + const v16u8 thresh0 = (v16u8)__msa_fill_b(thresh); + const v16u8 b_limit0 = (v16u8)__msa_fill_b(b_limit); + const v16u8 limit0 = (v16u8)__msa_fill_b(limit); + + LD_UB8(src - 4, stride, row0, row1, row2, row3, row4, row5, row6, row7); + LD_UB8(src - 4 + (8 * stride), stride, + row8, row9, row10, row11, row12, row13, row14, row15); + TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7, + row8, row9, row10, row11, row12, row13, row14, row15, + p3, p2, p1, p0, q0, q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, + hev, mask); + LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); + ILVR_B2_SH(p0, p1, q1, q0, tmp0, tmp1); + ILVRL_H2_SH(tmp1, tmp0, tmp2, tmp3); + ILVL_B2_SH(p0, p1, q1, q0, tmp0, tmp1); + ILVRL_H2_SH(tmp1, tmp0, tmp4, tmp5); + src -= 2; + ST4x8_UB(tmp2, tmp3, src, stride); + src += (8 * stride); + ST4x8_UB(tmp4, tmp5, src, stride); +} + +static void HFilter16i(uint8_t* src_y, int stride, + int b_limit, int limit, int thresh) { + HFilterVertEdge16i(src_y + 4, stride, b_limit, limit, thresh); + HFilterVertEdge16i(src_y + 8, stride, b_limit, limit, thresh); + HFilterVertEdge16i(src_y + 12, stride, b_limit, limit, thresh); +} + +// 8-pixels wide variants, for chroma filtering +static void VFilter8(uint8_t* src_u, uint8_t* src_v, int stride, + int b_limit_in, int limit_in, int thresh_in) { + uint8_t* ptmp_src_u = src_u - 4 * stride; + uint8_t* ptmp_src_v = src_v - 4 * stride; + uint64_t p2_d, p1_d, p0_d, q0_d, q1_d, q2_d; + v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev; + v16u8 p3_u, p2_u, p1_u, p0_u, q3_u, q2_u, q1_u, q0_u; + v16u8 p3_v, p2_v, p1_v, p0_v, q3_v, q2_v, q1_v, q0_v; + const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); + const v16u8 limit = (v16u8)__msa_fill_b(limit_in); + const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in); + + LD_UB8(ptmp_src_u, stride, p3_u, p2_u, p1_u, p0_u, q0_u, q1_u, q2_u, q3_u); + LD_UB8(ptmp_src_v, stride, p3_v, p2_v, p1_v, p0_v, q0_v, q1_v, q2_v, q3_v); + ILVR_D4_UB(p3_v, p3_u, p2_v, p2_u, p1_v, p1_u, p0_v, p0_u, p3, p2, p1, p0); + ILVR_D4_UB(q0_v, q0_u, q1_v, q1_u, q2_v, q2_u, q3_v, q3_u, q0, q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask); + LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + p2_d = __msa_copy_s_d((v2i64)p2, 0); + p1_d = __msa_copy_s_d((v2i64)p1, 0); + p0_d = __msa_copy_s_d((v2i64)p0, 0); + q0_d = __msa_copy_s_d((v2i64)q0, 0); + q1_d = __msa_copy_s_d((v2i64)q1, 0); + q2_d = __msa_copy_s_d((v2i64)q2, 0); + ptmp_src_u += stride; + SD4(p2_d, p1_d, p0_d, q0_d, ptmp_src_u, stride); + ptmp_src_u += (4 * stride); + SD(q1_d, ptmp_src_u); + ptmp_src_u += stride; + SD(q2_d, ptmp_src_u); + p2_d = __msa_copy_s_d((v2i64)p2, 1); + p1_d = __msa_copy_s_d((v2i64)p1, 1); + p0_d = __msa_copy_s_d((v2i64)p0, 1); + q0_d = __msa_copy_s_d((v2i64)q0, 1); + q1_d = __msa_copy_s_d((v2i64)q1, 1); + q2_d = __msa_copy_s_d((v2i64)q2, 1); + ptmp_src_v += stride; + SD4(p2_d, p1_d, p0_d, q0_d, ptmp_src_v, stride); + ptmp_src_v += (4 * stride); + SD(q1_d, ptmp_src_v); + ptmp_src_v += stride; + SD(q2_d, ptmp_src_v); +} + +static void HFilter8(uint8_t* src_u, uint8_t* src_v, int stride, + int b_limit_in, int limit_in, int thresh_in) { + uint8_t* ptmp_src_u = src_u - 4; + uint8_t* ptmp_src_v = src_v - 4; + v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev; + v16u8 row0, row1, row2, row3, row4, row5, row6, row7, row8; + v16u8 row9, row10, row11, row12, row13, row14, row15; + v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); + const v16u8 limit = (v16u8)__msa_fill_b(limit_in); + const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in); + + LD_UB8(ptmp_src_u, stride, row0, row1, row2, row3, row4, row5, row6, row7); + LD_UB8(ptmp_src_v, stride, + row8, row9, row10, row11, row12, row13, row14, row15); + TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7, + row8, row9, row10, row11, row12, row13, row14, row15, + p3, p2, p1, p0, q0, q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask); + LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + ILVR_B2_SH(p1, p2, q0, p0, tmp0, tmp1); + ILVRL_H2_SH(tmp1, tmp0, tmp3, tmp4); + ILVL_B2_SH(p1, p2, q0, p0, tmp0, tmp1); + ILVRL_H2_SH(tmp1, tmp0, tmp6, tmp7); + ILVRL_B2_SH(q2, q1, tmp2, tmp5); + ptmp_src_u += 1; + ST6x4_UB(tmp3, 0, tmp2, 0, ptmp_src_u, stride); + ptmp_src_u += 4 * stride; + ST6x4_UB(tmp4, 0, tmp2, 4, ptmp_src_u, stride); + ptmp_src_v += 1; + ST6x4_UB(tmp6, 0, tmp5, 0, ptmp_src_v, stride); + ptmp_src_v += 4 * stride; + ST6x4_UB(tmp7, 0, tmp5, 4, ptmp_src_v, stride); +} + +static void VFilter8i(uint8_t* src_u, uint8_t* src_v, int stride, + int b_limit_in, int limit_in, int thresh_in) { + uint64_t p1_d, p0_d, q0_d, q1_d; + v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev; + v16u8 p3_u, p2_u, p1_u, p0_u, q3_u, q2_u, q1_u, q0_u; + v16u8 p3_v, p2_v, p1_v, p0_v, q3_v, q2_v, q1_v, q0_v; + const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in); + const v16u8 limit = (v16u8)__msa_fill_b(limit_in); + const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); + + LD_UB8(src_u, stride, p3_u, p2_u, p1_u, p0_u, q0_u, q1_u, q2_u, q3_u); + src_u += (5 * stride); + LD_UB8(src_v, stride, p3_v, p2_v, p1_v, p0_v, q0_v, q1_v, q2_v, q3_v); + src_v += (5 * stride); + ILVR_D4_UB(p3_v, p3_u, p2_v, p2_u, p1_v, p1_u, p0_v, p0_u, p3, p2, p1, p0); + ILVR_D4_UB(q0_v, q0_u, q1_v, q1_u, q2_v, q2_u, q3_v, q3_u, q0, q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask); + LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); + p1_d = __msa_copy_s_d((v2i64)p1, 0); + p0_d = __msa_copy_s_d((v2i64)p0, 0); + q0_d = __msa_copy_s_d((v2i64)q0, 0); + q1_d = __msa_copy_s_d((v2i64)q1, 0); + SD4(q1_d, q0_d, p0_d, p1_d, src_u, -stride); + p1_d = __msa_copy_s_d((v2i64)p1, 1); + p0_d = __msa_copy_s_d((v2i64)p0, 1); + q0_d = __msa_copy_s_d((v2i64)q0, 1); + q1_d = __msa_copy_s_d((v2i64)q1, 1); + SD4(q1_d, q0_d, p0_d, p1_d, src_v, -stride); +} + +static void HFilter8i(uint8_t* src_u, uint8_t* src_v, int stride, + int b_limit_in, int limit_in, int thresh_in) { + v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev; + v16u8 row0, row1, row2, row3, row4, row5, row6, row7, row8; + v16u8 row9, row10, row11, row12, row13, row14, row15; + v4i32 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in); + const v16u8 limit = (v16u8)__msa_fill_b(limit_in); + const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); + + LD_UB8(src_u, stride, row0, row1, row2, row3, row4, row5, row6, row7); + LD_UB8(src_v, stride, + row8, row9, row10, row11, row12, row13, row14, row15); + TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7, + row8, row9, row10, row11, row12, row13, row14, row15, + p3, p2, p1, p0, q0, q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask); + LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); + ILVR_B2_SW(p0, p1, q1, q0, tmp0, tmp1); + ILVRL_H2_SW(tmp1, tmp0, tmp2, tmp3); + ILVL_B2_SW(p0, p1, q1, q0, tmp0, tmp1); + ILVRL_H2_SW(tmp1, tmp0, tmp4, tmp5); + src_u += 2; + ST4x4_UB(tmp2, tmp2, 0, 1, 2, 3, src_u, stride); + src_u += 4 * stride; + ST4x4_UB(tmp3, tmp3, 0, 1, 2, 3, src_u, stride); + src_v += 2; + ST4x4_UB(tmp4, tmp4, 0, 1, 2, 3, src_v, stride); + src_v += 4 * stride; + ST4x4_UB(tmp5, tmp5, 0, 1, 2, 3, src_v, stride); +} + +static void SimpleVFilter16(uint8_t* src, int stride, int b_limit_in) { + v16u8 p1, p0, q1, q0, mask; + const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); + + LD_UB4(src - 2 * stride, stride, p1, p0, q0, q1); + LPF_SIMPLE_MASK(p1, p0, q0, q1, b_limit, mask); + LPF_SIMPLE_FILT(p1, p0, q0, q1, mask); + ST_UB2(p0, q0, src - stride, stride); +} + +static void SimpleHFilter16(uint8_t* src, int stride, int b_limit_in) { + v16u8 p1, p0, q1, q0, mask, row0, row1, row2, row3, row4, row5, row6, row7; + v16u8 row8, row9, row10, row11, row12, row13, row14, row15; + v8i16 tmp0, tmp1; + const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); + uint8_t* ptemp_src = src - 2; + + LD_UB8(ptemp_src, stride, row0, row1, row2, row3, row4, row5, row6, row7); + LD_UB8(ptemp_src + 8 * stride, stride, + row8, row9, row10, row11, row12, row13, row14, row15); + TRANSPOSE16x4_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7, + row8, row9, row10, row11, row12, row13, row14, row15, + p1, p0, q0, q1); + LPF_SIMPLE_MASK(p1, p0, q0, q1, b_limit, mask); + LPF_SIMPLE_FILT(p1, p0, q0, q1, mask); + ILVRL_B2_SH(q0, p0, tmp1, tmp0); + ptemp_src += 1; + ST2x4_UB(tmp1, 0, ptemp_src, stride); + ptemp_src += 4 * stride; + ST2x4_UB(tmp1, 4, ptemp_src, stride); + ptemp_src += 4 * stride; + ST2x4_UB(tmp0, 0, ptemp_src, stride); + ptemp_src += 4 * stride; + ST2x4_UB(tmp0, 4, ptemp_src, stride); + ptemp_src += 4 * stride; +} + +static void SimpleVFilter16i(uint8_t* src_y, int stride, int b_limit_in) { + SimpleVFilter16(src_y + 4 * stride, stride, b_limit_in); + SimpleVFilter16(src_y + 8 * stride, stride, b_limit_in); + SimpleVFilter16(src_y + 12 * stride, stride, b_limit_in); +} + +static void SimpleHFilter16i(uint8_t* src_y, int stride, int b_limit_in) { + SimpleHFilter16(src_y + 4, stride, b_limit_in); + SimpleHFilter16(src_y + 8, stride, b_limit_in); + SimpleHFilter16(src_y + 12, stride, b_limit_in); +} + +//------------------------------------------------------------------------------ +// Intra predictions +//------------------------------------------------------------------------------ + +// 4x4 + +static void DC4(uint8_t* dst) { // DC + uint32_t dc = 4; + int i; + for (i = 0; i < 4; ++i) dc += dst[i - BPS] + dst[-1 + i * BPS]; + dc >>= 3; + dc = dc | (dc << 8) | (dc << 16) | (dc << 24); + SW4(dc, dc, dc, dc, dst, BPS); +} + +static void TM4(uint8_t* dst) { + const uint8_t* const ptemp = dst - BPS - 1; + v8i16 T, d, r0, r1, r2, r3; + const v16i8 zero = { 0 }; + const v8i16 TL = (v8i16)__msa_fill_h(ptemp[0 * BPS]); + const v8i16 L0 = (v8i16)__msa_fill_h(ptemp[1 * BPS]); + const v8i16 L1 = (v8i16)__msa_fill_h(ptemp[2 * BPS]); + const v8i16 L2 = (v8i16)__msa_fill_h(ptemp[3 * BPS]); + const v8i16 L3 = (v8i16)__msa_fill_h(ptemp[4 * BPS]); + const v16u8 T1 = LD_UB(ptemp + 1); + + T = (v8i16)__msa_ilvr_b(zero, (v16i8)T1); + d = T - TL; + ADD4(d, L0, d, L1, d, L2, d, L3, r0, r1, r2, r3); + CLIP_SH4_0_255(r0, r1, r2, r3); + PCKEV_ST4x4_UB(r0, r1, r2, r3, dst, BPS); +} + +static void VE4(uint8_t* dst) { // vertical + const uint8_t* const ptop = dst - BPS - 1; + const uint32_t val0 = LW(ptop + 0); + const uint32_t val1 = LW(ptop + 4); + uint32_t out; + v16u8 A, B, C, AC, B2, R; + + INSERT_W2_UB(val0, val1, A); + B = SLDI_UB(A, A, 1); + C = SLDI_UB(A, A, 2); + AC = __msa_ave_u_b(A, C); + B2 = __msa_ave_u_b(B, B); + R = __msa_aver_u_b(AC, B2); + out = __msa_copy_s_w((v4i32)R, 0); + SW4(out, out, out, out, dst, BPS); +} + +static void RD4(uint8_t* dst) { // Down-right + const uint8_t* const ptop = dst - 1 - BPS; + uint32_t val0 = LW(ptop + 0); + uint32_t val1 = LW(ptop + 4); + uint32_t val2, val3; + v16u8 A, B, C, AC, B2, R, A1; + + INSERT_W2_UB(val0, val1, A1); + A = SLDI_UB(A1, A1, 12); + A = (v16u8)__msa_insert_b((v16i8)A, 3, ptop[1 * BPS]); + A = (v16u8)__msa_insert_b((v16i8)A, 2, ptop[2 * BPS]); + A = (v16u8)__msa_insert_b((v16i8)A, 1, ptop[3 * BPS]); + A = (v16u8)__msa_insert_b((v16i8)A, 0, ptop[4 * BPS]); + B = SLDI_UB(A, A, 1); + C = SLDI_UB(A, A, 2); + AC = __msa_ave_u_b(A, C); + B2 = __msa_ave_u_b(B, B); + R = __msa_aver_u_b(AC, B2); + val3 = __msa_copy_s_w((v4i32)R, 0); + R = SLDI_UB(R, R, 1); + val2 = __msa_copy_s_w((v4i32)R, 0); + R = SLDI_UB(R, R, 1); + val1 = __msa_copy_s_w((v4i32)R, 0); + R = SLDI_UB(R, R, 1); + val0 = __msa_copy_s_w((v4i32)R, 0); + SW4(val0, val1, val2, val3, dst, BPS); +} + +static void LD4(uint8_t* dst) { // Down-Left + const uint8_t* const ptop = dst - BPS; + uint32_t val0 = LW(ptop + 0); + uint32_t val1 = LW(ptop + 4); + uint32_t val2, val3; + v16u8 A, B, C, AC, B2, R; + + INSERT_W2_UB(val0, val1, A); + B = SLDI_UB(A, A, 1); + C = SLDI_UB(A, A, 2); + C = (v16u8)__msa_insert_b((v16i8)C, 6, ptop[7]); + AC = __msa_ave_u_b(A, C); + B2 = __msa_ave_u_b(B, B); + R = __msa_aver_u_b(AC, B2); + val0 = __msa_copy_s_w((v4i32)R, 0); + R = SLDI_UB(R, R, 1); + val1 = __msa_copy_s_w((v4i32)R, 0); + R = SLDI_UB(R, R, 1); + val2 = __msa_copy_s_w((v4i32)R, 0); + R = SLDI_UB(R, R, 1); + val3 = __msa_copy_s_w((v4i32)R, 0); + SW4(val0, val1, val2, val3, dst, BPS); +} + +// 16x16 + +static void DC16(uint8_t* dst) { // DC + uint32_t dc = 16; + int i; + const v16u8 rtop = LD_UB(dst - BPS); + const v8u16 dctop = __msa_hadd_u_h(rtop, rtop); + v16u8 out; + + for (i = 0; i < 16; ++i) { + dc += dst[-1 + i * BPS]; + } + dc += HADD_UH_U32(dctop); + out = (v16u8)__msa_fill_b(dc >> 5); + ST_UB8(out, out, out, out, out, out, out, out, dst, BPS); + ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS); +} + +static void TM16(uint8_t* dst) { + int j; + v8i16 d1, d2; + const v16i8 zero = { 0 }; + const v8i16 TL = (v8i16)__msa_fill_h(dst[-1 - BPS]); + const v16i8 T = LD_SB(dst - BPS); + + ILVRL_B2_SH(zero, T, d1, d2); + SUB2(d1, TL, d2, TL, d1, d2); + for (j = 0; j < 16; j += 4) { + v16i8 t0, t1, t2, t3; + v8i16 r0, r1, r2, r3, r4, r5, r6, r7; + const v8i16 L0 = (v8i16)__msa_fill_h(dst[-1 + 0 * BPS]); + const v8i16 L1 = (v8i16)__msa_fill_h(dst[-1 + 1 * BPS]); + const v8i16 L2 = (v8i16)__msa_fill_h(dst[-1 + 2 * BPS]); + const v8i16 L3 = (v8i16)__msa_fill_h(dst[-1 + 3 * BPS]); + ADD4(d1, L0, d1, L1, d1, L2, d1, L3, r0, r1, r2, r3); + ADD4(d2, L0, d2, L1, d2, L2, d2, L3, r4, r5, r6, r7); + CLIP_SH4_0_255(r0, r1, r2, r3); + CLIP_SH4_0_255(r4, r5, r6, r7); + PCKEV_B4_SB(r4, r0, r5, r1, r6, r2, r7, r3, t0, t1, t2, t3); + ST_SB4(t0, t1, t2, t3, dst, BPS); + dst += 4 * BPS; + } +} + +static void VE16(uint8_t* dst) { // vertical + const v16u8 rtop = LD_UB(dst - BPS); + ST_UB8(rtop, rtop, rtop, rtop, rtop, rtop, rtop, rtop, dst, BPS); + ST_UB8(rtop, rtop, rtop, rtop, rtop, rtop, rtop, rtop, dst + 8 * BPS, BPS); +} + +static void HE16(uint8_t* dst) { // horizontal + int j; + for (j = 16; j > 0; j -= 4) { + const v16u8 L0 = (v16u8)__msa_fill_b(dst[-1 + 0 * BPS]); + const v16u8 L1 = (v16u8)__msa_fill_b(dst[-1 + 1 * BPS]); + const v16u8 L2 = (v16u8)__msa_fill_b(dst[-1 + 2 * BPS]); + const v16u8 L3 = (v16u8)__msa_fill_b(dst[-1 + 3 * BPS]); + ST_UB4(L0, L1, L2, L3, dst, BPS); + dst += 4 * BPS; + } +} + +static void DC16NoTop(uint8_t* dst) { // DC with top samples not available + int j; + uint32_t dc = 8; + v16u8 out; + + for (j = 0; j < 16; ++j) { + dc += dst[-1 + j * BPS]; + } + out = (v16u8)__msa_fill_b(dc >> 4); + ST_UB8(out, out, out, out, out, out, out, out, dst, BPS); + ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS); +} + +static void DC16NoLeft(uint8_t* dst) { // DC with left samples not available + uint32_t dc = 8; + const v16u8 rtop = LD_UB(dst - BPS); + const v8u16 dctop = __msa_hadd_u_h(rtop, rtop); + v16u8 out; + + dc += HADD_UH_U32(dctop); + out = (v16u8)__msa_fill_b(dc >> 4); + ST_UB8(out, out, out, out, out, out, out, out, dst, BPS); + ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS); +} + +static void DC16NoTopLeft(uint8_t* dst) { // DC with nothing + const v16u8 out = (v16u8)__msa_fill_b(0x80); + ST_UB8(out, out, out, out, out, out, out, out, dst, BPS); + ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS); +} + +// Chroma + +#define STORE8x8(out, dst) do { \ + SD4(out, out, out, out, dst + 0 * BPS, BPS); \ + SD4(out, out, out, out, dst + 4 * BPS, BPS); \ +} while (0) + +static void DC8uv(uint8_t* dst) { // DC + uint32_t dc = 8; + int i; + uint64_t out; + const v16u8 rtop = LD_UB(dst - BPS); + const v8u16 temp0 = __msa_hadd_u_h(rtop, rtop); + const v4u32 temp1 = __msa_hadd_u_w(temp0, temp0); + const v2u64 temp2 = __msa_hadd_u_d(temp1, temp1); + v16u8 dctemp; + + for (i = 0; i < 8; ++i) { + dc += dst[-1 + i * BPS]; + } + dc += __msa_copy_s_w((v4i32)temp2, 0); + dctemp = (v16u8)__msa_fill_b(dc >> 4); + out = __msa_copy_s_d((v2i64)dctemp, 0); + STORE8x8(out, dst); +} + +static void TM8uv(uint8_t* dst) { + int j; + const v16i8 T1 = LD_SB(dst - BPS); + const v16i8 zero = { 0 }; + const v8i16 T = (v8i16)__msa_ilvr_b(zero, T1); + const v8i16 TL = (v8i16)__msa_fill_h(dst[-1 - BPS]); + const v8i16 d = T - TL; + + for (j = 0; j < 8; j += 4) { + v16i8 t0, t1; + v8i16 r0 = (v8i16)__msa_fill_h(dst[-1 + 0 * BPS]); + v8i16 r1 = (v8i16)__msa_fill_h(dst[-1 + 1 * BPS]); + v8i16 r2 = (v8i16)__msa_fill_h(dst[-1 + 2 * BPS]); + v8i16 r3 = (v8i16)__msa_fill_h(dst[-1 + 3 * BPS]); + ADD4(d, r0, d, r1, d, r2, d, r3, r0, r1, r2, r3); + CLIP_SH4_0_255(r0, r1, r2, r3); + PCKEV_B2_SB(r1, r0, r3, r2, t0, t1); + ST4x4_UB(t0, t1, 0, 2, 0, 2, dst, BPS); + ST4x4_UB(t0, t1, 1, 3, 1, 3, dst + 4, BPS); + dst += 4 * BPS; + } +} + +static void VE8uv(uint8_t* dst) { // vertical + const v16u8 rtop = LD_UB(dst - BPS); + const uint64_t out = __msa_copy_s_d((v2i64)rtop, 0); + STORE8x8(out, dst); +} + +static void HE8uv(uint8_t* dst) { // horizontal + int j; + for (j = 0; j < 8; j += 4) { + const v16u8 L0 = (v16u8)__msa_fill_b(dst[-1 + 0 * BPS]); + const v16u8 L1 = (v16u8)__msa_fill_b(dst[-1 + 1 * BPS]); + const v16u8 L2 = (v16u8)__msa_fill_b(dst[-1 + 2 * BPS]); + const v16u8 L3 = (v16u8)__msa_fill_b(dst[-1 + 3 * BPS]); + const uint64_t out0 = __msa_copy_s_d((v2i64)L0, 0); + const uint64_t out1 = __msa_copy_s_d((v2i64)L1, 0); + const uint64_t out2 = __msa_copy_s_d((v2i64)L2, 0); + const uint64_t out3 = __msa_copy_s_d((v2i64)L3, 0); + SD4(out0, out1, out2, out3, dst, BPS); + dst += 4 * BPS; + } +} + +static void DC8uvNoLeft(uint8_t* dst) { // DC with no left samples + const uint32_t dc = 4; + const v16u8 rtop = LD_UB(dst - BPS); + const v8u16 temp0 = __msa_hadd_u_h(rtop, rtop); + const v4u32 temp1 = __msa_hadd_u_w(temp0, temp0); + const v2u64 temp2 = __msa_hadd_u_d(temp1, temp1); + const uint32_t sum_m = __msa_copy_s_w((v4i32)temp2, 0); + const v16u8 dcval = (v16u8)__msa_fill_b((dc + sum_m) >> 3); + const uint64_t out = __msa_copy_s_d((v2i64)dcval, 0); + STORE8x8(out, dst); +} + +static void DC8uvNoTop(uint8_t* dst) { // DC with no top samples + uint32_t dc = 4; + int i; + uint64_t out; + v16u8 dctemp; + + for (i = 0; i < 8; ++i) { + dc += dst[-1 + i * BPS]; + } + dctemp = (v16u8)__msa_fill_b(dc >> 3); + out = __msa_copy_s_d((v2i64)dctemp, 0); + STORE8x8(out, dst); +} + +static void DC8uvNoTopLeft(uint8_t* dst) { // DC with nothing + const uint64_t out = 0x8080808080808080ULL; + STORE8x8(out, dst); +} + +//------------------------------------------------------------------------------ // Entry point extern void VP8DspInitMSA(void); @@ -163,6 +977,39 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8DspInitMSA(void) { VP8Transform = TransformTwo; VP8TransformDC = TransformDC; VP8TransformAC3 = TransformAC3; + + VP8VFilter16 = VFilter16; + VP8HFilter16 = HFilter16; + VP8VFilter16i = VFilter16i; + VP8HFilter16i = HFilter16i; + VP8VFilter8 = VFilter8; + VP8HFilter8 = HFilter8; + VP8VFilter8i = VFilter8i; + VP8HFilter8i = HFilter8i; + VP8SimpleVFilter16 = SimpleVFilter16; + VP8SimpleHFilter16 = SimpleHFilter16; + VP8SimpleVFilter16i = SimpleVFilter16i; + VP8SimpleHFilter16i = SimpleHFilter16i; + + VP8PredLuma4[0] = DC4; + VP8PredLuma4[1] = TM4; + VP8PredLuma4[2] = VE4; + VP8PredLuma4[4] = RD4; + VP8PredLuma4[6] = LD4; + VP8PredLuma16[0] = DC16; + VP8PredLuma16[1] = TM16; + VP8PredLuma16[2] = VE16; + VP8PredLuma16[3] = HE16; + VP8PredLuma16[4] = DC16NoTop; + VP8PredLuma16[5] = DC16NoLeft; + VP8PredLuma16[6] = DC16NoTopLeft; + VP8PredChroma8[0] = DC8uv; + VP8PredChroma8[1] = TM8uv; + VP8PredChroma8[2] = VE8uv; + VP8PredChroma8[3] = HE8uv; + VP8PredChroma8[4] = DC8uvNoTop; + VP8PredChroma8[5] = DC8uvNoLeft; + VP8PredChroma8[6] = DC8uvNoTopLeft; } #else // !WEBP_USE_MSA diff --git a/src/3rdparty/libwebp/src/dsp/dec_neon.c b/src/3rdparty/libwebp/src/dsp/dec_neon.c index a63f43f..34796cf 100644 --- a/src/3rdparty/libwebp/src/dsp/dec_neon.c +++ b/src/3rdparty/libwebp/src/dsp/dec_neon.c @@ -17,7 +17,7 @@ #if defined(WEBP_USE_NEON) #include "./neon.h" -#include "../dec/vp8i.h" +#include "../dec/vp8i_dec.h" //------------------------------------------------------------------------------ // NxM Loading functions @@ -666,9 +666,8 @@ static uint8x16_t NeedsHev(const uint8x16_t p1, const uint8x16_t p0, const uint8x16_t hev_thresh_v = vdupq_n_u8((uint8_t)hev_thresh); const uint8x16_t a_p1_p0 = vabdq_u8(p1, p0); // abs(p1 - p0) const uint8x16_t a_q1_q0 = vabdq_u8(q1, q0); // abs(q1 - q0) - const uint8x16_t mask1 = vcgtq_u8(a_p1_p0, hev_thresh_v); - const uint8x16_t mask2 = vcgtq_u8(a_q1_q0, hev_thresh_v); - const uint8x16_t mask = vorrq_u8(mask1, mask2); + const uint8x16_t a_max = vmaxq_u8(a_p1_p0, a_q1_q0); + const uint8x16_t mask = vcgtq_u8(a_max, hev_thresh_v); return mask; } @@ -756,24 +755,25 @@ static void ApplyFilter6( const int8x16_t delta, uint8x16_t* const op2, uint8x16_t* const op1, uint8x16_t* const op0, uint8x16_t* const oq0, uint8x16_t* const oq1, uint8x16_t* const oq2) { - const int16x8_t kCst63 = vdupq_n_s16(63); - const int8x8_t kCst27 = vdup_n_s8(27); - const int8x8_t kCst18 = vdup_n_s8(18); - const int8x8_t kCst9 = vdup_n_s8(9); + // We have to compute: X = (9*a+63) >> 7, Y = (18*a+63)>>7, Z = (27*a+63) >> 7 + // Turns out, there's a common sub-expression S=9 * a - 1 that can be used + // with the special vqrshrn_n_s16 rounding-shift-and-narrow instruction: + // X = (S + 64) >> 7, Y = (S + 32) >> 6, Z = (18 * a + S + 64) >> 7 const int8x8_t delta_lo = vget_low_s8(delta); const int8x8_t delta_hi = vget_high_s8(delta); - const int16x8_t s1_lo = vmlal_s8(kCst63, kCst27, delta_lo); // 63 + 27 * a - const int16x8_t s1_hi = vmlal_s8(kCst63, kCst27, delta_hi); // 63 + 27 * a - const int16x8_t s2_lo = vmlal_s8(kCst63, kCst18, delta_lo); // 63 + 18 * a - const int16x8_t s2_hi = vmlal_s8(kCst63, kCst18, delta_hi); // 63 + 18 * a - const int16x8_t s3_lo = vmlal_s8(kCst63, kCst9, delta_lo); // 63 + 9 * a - const int16x8_t s3_hi = vmlal_s8(kCst63, kCst9, delta_hi); // 63 + 9 * a - const int8x8_t a1_lo = vqshrn_n_s16(s1_lo, 7); - const int8x8_t a1_hi = vqshrn_n_s16(s1_hi, 7); - const int8x8_t a2_lo = vqshrn_n_s16(s2_lo, 7); - const int8x8_t a2_hi = vqshrn_n_s16(s2_hi, 7); - const int8x8_t a3_lo = vqshrn_n_s16(s3_lo, 7); - const int8x8_t a3_hi = vqshrn_n_s16(s3_hi, 7); + const int8x8_t kCst9 = vdup_n_s8(9); + const int16x8_t kCstm1 = vdupq_n_s16(-1); + const int8x8_t kCst18 = vdup_n_s8(18); + const int16x8_t S_lo = vmlal_s8(kCstm1, kCst9, delta_lo); // S = 9 * a - 1 + const int16x8_t S_hi = vmlal_s8(kCstm1, kCst9, delta_hi); + const int16x8_t Z_lo = vmlal_s8(S_lo, kCst18, delta_lo); // S + 18 * a + const int16x8_t Z_hi = vmlal_s8(S_hi, kCst18, delta_hi); + const int8x8_t a3_lo = vqrshrn_n_s16(S_lo, 7); // (9 * a + 63) >> 7 + const int8x8_t a3_hi = vqrshrn_n_s16(S_hi, 7); + const int8x8_t a2_lo = vqrshrn_n_s16(S_lo, 6); // (9 * a + 31) >> 6 + const int8x8_t a2_hi = vqrshrn_n_s16(S_hi, 6); + const int8x8_t a1_lo = vqrshrn_n_s16(Z_lo, 7); // (27 * a + 63) >> 7 + const int8x8_t a1_hi = vqrshrn_n_s16(Z_hi, 7); const int8x16_t a1 = vcombine_s8(a1_lo, a1_hi); const int8x16_t a2 = vcombine_s8(a2_lo, a2_hi); const int8x16_t a3 = vcombine_s8(a3_lo, a3_hi); diff --git a/src/3rdparty/libwebp/src/dsp/dec_sse2.c b/src/3rdparty/libwebp/src/dsp/dec_sse2.c index f0a8ddc..411fb02 100644 --- a/src/3rdparty/libwebp/src/dsp/dec_sse2.c +++ b/src/3rdparty/libwebp/src/dsp/dec_sse2.c @@ -22,7 +22,7 @@ #include <emmintrin.h> #include "./common_sse2.h" -#include "../dec/vp8i.h" +#include "../dec/vp8i_dec.h" #include "../utils/utils.h" //------------------------------------------------------------------------------ @@ -140,7 +140,7 @@ static void Transform(const int16_t* in, uint8_t* dst, int do_two) { // Transpose the two 4x4. VP8Transpose_2_4x4_16b(&shifted0, &shifted1, &shifted2, &shifted3, &T0, &T1, - &T2, &T3); + &T2, &T3); } // Add inverse transform to 'dst' and store. diff --git a/src/3rdparty/libwebp/src/dsp/dec_sse41.c b/src/3rdparty/libwebp/src/dsp/dec_sse41.c index 8d6aed1..4e81ec4 100644 --- a/src/3rdparty/libwebp/src/dsp/dec_sse41.c +++ b/src/3rdparty/libwebp/src/dsp/dec_sse41.c @@ -16,7 +16,7 @@ #if defined(WEBP_USE_SSE41) #include <smmintrin.h> -#include "../dec/vp8i.h" +#include "../dec/vp8i_dec.h" #include "../utils/utils.h" static void HE16(uint8_t* dst) { // horizontal diff --git a/src/3rdparty/libwebp/src/dsp/dsp.h b/src/3rdparty/libwebp/src/dsp/dsp.h index 1faac27..813fed4 100644 --- a/src/3rdparty/libwebp/src/dsp/dsp.h +++ b/src/3rdparty/libwebp/src/dsp/dsp.h @@ -111,8 +111,7 @@ extern "C" { #define WEBP_UBSAN_IGNORE_UNDEF #define WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW -#if !defined(WEBP_FORCE_ALIGNED) && defined(__clang__) && \ - defined(__has_attribute) +#if defined(__clang__) && defined(__has_attribute) #if __has_attribute(no_sanitize) // This macro prevents the undefined behavior sanitizer from reporting // failures. This is only meant to silence unaligned loads on platforms that @@ -133,6 +132,7 @@ extern "C" { typedef enum { kSSE2, kSSE3, + kSlowSSSE3, // special feature for slow SSSE3 architectures kSSE4_1, kAVX, kAVX2, @@ -185,6 +185,11 @@ typedef int (*VP8WMetric)(const uint8_t* pix, const uint8_t* ref, // 4 by 4 symmetric matrix. extern VP8WMetric VP8TDisto4x4, VP8TDisto16x16; +// Compute the average (DC) of four 4x4 blocks. +// Each sub-4x4 block #i sum is stored in dc[i]. +typedef void (*VP8MeanMetric)(const uint8_t* ref, uint32_t dc[4]); +extern VP8MeanMetric VP8Mean16x4; + typedef void (*VP8BlockCopy)(const uint8_t* src, uint8_t* dst); extern VP8BlockCopy VP8Copy4x4; extern VP8BlockCopy VP8Copy16x8; @@ -246,30 +251,37 @@ extern VP8GetResidualCostFunc VP8GetResidualCost; void VP8EncDspCostInit(void); //------------------------------------------------------------------------------ -// SSIM utils +// SSIM / PSNR utils // struct for accumulating statistical moments typedef struct { - double w; // sum(w_i) : sum of weights - double xm, ym; // sum(w_i * x_i), sum(w_i * y_i) - double xxm, xym, yym; // sum(w_i * x_i * x_i), etc. + uint32_t w; // sum(w_i) : sum of weights + uint32_t xm, ym; // sum(w_i * x_i), sum(w_i * y_i) + uint32_t xxm, xym, yym; // sum(w_i * x_i * x_i), etc. } VP8DistoStats; +// Compute the final SSIM value +// The non-clipped version assumes stats->w = (2 * VP8_SSIM_KERNEL + 1)^2. +double VP8SSIMFromStats(const VP8DistoStats* const stats); +double VP8SSIMFromStatsClipped(const VP8DistoStats* const stats); + #define VP8_SSIM_KERNEL 3 // total size of the kernel: 2 * VP8_SSIM_KERNEL + 1 -typedef void (*VP8SSIMAccumulateClippedFunc)(const uint8_t* src1, int stride1, - const uint8_t* src2, int stride2, - int xo, int yo, // center position - int W, int H, // plane dimension - VP8DistoStats* const stats); +typedef double (*VP8SSIMGetClippedFunc)(const uint8_t* src1, int stride1, + const uint8_t* src2, int stride2, + int xo, int yo, // center position + int W, int H); // plane dimension // This version is called with the guarantee that you can load 8 bytes and // 8 rows at offset src1 and src2 -typedef void (*VP8SSIMAccumulateFunc)(const uint8_t* src1, int stride1, - const uint8_t* src2, int stride2, - VP8DistoStats* const stats); +typedef double (*VP8SSIMGetFunc)(const uint8_t* src1, int stride1, + const uint8_t* src2, int stride2); + +extern VP8SSIMGetFunc VP8SSIMGet; // unclipped / unchecked +extern VP8SSIMGetClippedFunc VP8SSIMGetClipped; // with clipping -extern VP8SSIMAccumulateFunc VP8SSIMAccumulate; // unclipped / unchecked -extern VP8SSIMAccumulateClippedFunc VP8SSIMAccumulateClipped; // with clipping +typedef uint32_t (*VP8AccumulateSSEFunc)(const uint8_t* src1, + const uint8_t* src2, int len); +extern VP8AccumulateSSEFunc VP8AccumulateSSE; // must be called before using any of the above directly void VP8SSIMDspInit(void); @@ -416,6 +428,15 @@ extern void WebPConvertARGBToUV_C(const uint32_t* argb, uint8_t* u, uint8_t* v, extern void WebPConvertRGBA32ToUV_C(const uint16_t* rgb, uint8_t* u, uint8_t* v, int width); +// utilities for accurate RGB->YUV conversion +extern uint64_t (*WebPSharpYUVUpdateY)(const uint16_t* src, const uint16_t* ref, + uint16_t* dst, int len); +extern void (*WebPSharpYUVUpdateRGB)(const int16_t* src, const int16_t* ref, + int16_t* dst, int len); +extern void (*WebPSharpYUVFilterRow)(const int16_t* A, const int16_t* B, + int len, + const uint16_t* best_y, uint16_t* out); + // Must be called before using the above. void WebPInitConvertARGBToYUV(void); @@ -488,6 +509,10 @@ extern int (*WebPExtractAlpha)(const uint8_t* argb, int argb_stride, int width, int height, uint8_t* alpha, int alpha_stride); +// Extract the green values from 32b values in argb[] and pack them into alpha[] +// (this is the opposite of WebPDispatchAlphaToGreen). +extern void (*WebPExtractGreen)(const uint32_t* argb, uint8_t* alpha, int size); + // Pre-Multiply operation transforms x into x * A / 255 (where x=Y,R,G or B). // Un-Multiply operation transforms x into x * 255 / A. diff --git a/src/3rdparty/libwebp/src/dsp/enc.c b/src/3rdparty/libwebp/src/dsp/enc.c index f639f55..f31bc6d 100644 --- a/src/3rdparty/libwebp/src/dsp/enc.c +++ b/src/3rdparty/libwebp/src/dsp/enc.c @@ -15,7 +15,7 @@ #include <stdlib.h> // for abs() #include "./dsp.h" -#include "../enc/vp8enci.h" +#include "../enc/vp8i_enc.h" static WEBP_INLINE uint8_t clip_8b(int v) { return (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255; @@ -335,7 +335,7 @@ static void Intra16Preds(uint8_t* dst, // luma 4x4 prediction #define DST(x, y) dst[(x) + (y) * BPS] -#define AVG3(a, b, c) (((a) + 2 * (b) + (c) + 2) >> 2) +#define AVG3(a, b, c) ((uint8_t)(((a) + 2 * (b) + (c) + 2) >> 2)) #define AVG2(a, b) (((a) + (b) + 1) >> 1) static void VE4(uint8_t* dst, const uint8_t* top) { // vertical @@ -551,6 +551,20 @@ static int SSE4x4(const uint8_t* a, const uint8_t* b) { return GetSSE(a, b, 4, 4); } +static void Mean16x4(const uint8_t* ref, uint32_t dc[4]) { + int k, x, y; + for (k = 0; k < 4; ++k) { + uint32_t avg = 0; + for (y = 0; y < 4; ++y) { + for (x = 0; x < 4; ++x) { + avg += ref[x + y * BPS]; + } + } + dc[k] = avg; + ref += 4; // go to next 4x4 block. + } +} + //------------------------------------------------------------------------------ // Texture distortion // @@ -656,32 +670,6 @@ static int Quantize2Blocks(int16_t in[32], int16_t out[32], return nz; } -static int QuantizeBlockWHT(int16_t in[16], int16_t out[16], - const VP8Matrix* const mtx) { - int n, last = -1; - for (n = 0; n < 16; ++n) { - const int j = kZigzag[n]; - const int sign = (in[j] < 0); - const uint32_t coeff = sign ? -in[j] : in[j]; - assert(mtx->sharpen_[j] == 0); - if (coeff > mtx->zthresh_[j]) { - const uint32_t Q = mtx->q_[j]; - const uint32_t iQ = mtx->iq_[j]; - const uint32_t B = mtx->bias_[j]; - int level = QUANTDIV(coeff, iQ, B); - if (level > MAX_LEVEL) level = MAX_LEVEL; - if (sign) level = -level; - in[j] = level * (int)Q; - out[n] = level; - if (level) last = n; - } else { - out[n] = 0; - in[j] = 0; - } - } - return (last >= 0); -} - //------------------------------------------------------------------------------ // Block copy @@ -703,11 +691,51 @@ static void Copy16x8(const uint8_t* src, uint8_t* dst) { } //------------------------------------------------------------------------------ +// SSIM / PSNR -static void SSIMAccumulateClipped(const uint8_t* src1, int stride1, - const uint8_t* src2, int stride2, - int xo, int yo, int W, int H, - VP8DistoStats* const stats) { +// hat-shaped filter. Sum of coefficients is equal to 16. +static const uint32_t kWeight[2 * VP8_SSIM_KERNEL + 1] = { + 1, 2, 3, 4, 3, 2, 1 +}; +static const uint32_t kWeightSum = 16 * 16; // sum{kWeight}^2 + +static WEBP_INLINE double SSIMCalculation( + const VP8DistoStats* const stats, uint32_t N /*num samples*/) { + const uint32_t w2 = N * N; + const uint32_t C1 = 20 * w2; + const uint32_t C2 = 60 * w2; + const uint32_t C3 = 8 * 8 * w2; // 'dark' limit ~= 6 + const uint64_t xmxm = (uint64_t)stats->xm * stats->xm; + const uint64_t ymym = (uint64_t)stats->ym * stats->ym; + if (xmxm + ymym >= C3) { + const int64_t xmym = (int64_t)stats->xm * stats->ym; + const int64_t sxy = (int64_t)stats->xym * N - xmym; // can be negative + const uint64_t sxx = (uint64_t)stats->xxm * N - xmxm; + const uint64_t syy = (uint64_t)stats->yym * N - ymym; + // we descale by 8 to prevent overflow during the fnum/fden multiply. + const uint64_t num_S = (2 * (uint64_t)(sxy < 0 ? 0 : sxy) + C2) >> 8; + const uint64_t den_S = (sxx + syy + C2) >> 8; + const uint64_t fnum = (2 * xmym + C1) * num_S; + const uint64_t fden = (xmxm + ymym + C1) * den_S; + const double r = (double)fnum / fden; + assert(r >= 0. && r <= 1.0); + return r; + } + return 1.; // area is too dark to contribute meaningfully +} + +double VP8SSIMFromStats(const VP8DistoStats* const stats) { + return SSIMCalculation(stats, kWeightSum); +} + +double VP8SSIMFromStatsClipped(const VP8DistoStats* const stats) { + return SSIMCalculation(stats, stats->w); +} + +static double SSIMGetClipped_C(const uint8_t* src1, int stride1, + const uint8_t* src2, int stride2, + int xo, int yo, int W, int H) { + VP8DistoStats stats = { 0, 0, 0, 0, 0, 0 }; const int ymin = (yo - VP8_SSIM_KERNEL < 0) ? 0 : yo - VP8_SSIM_KERNEL; const int ymax = (yo + VP8_SSIM_KERNEL > H - 1) ? H - 1 : yo + VP8_SSIM_KERNEL; @@ -719,38 +747,61 @@ static void SSIMAccumulateClipped(const uint8_t* src1, int stride1, src2 += ymin * stride2; for (y = ymin; y <= ymax; ++y, src1 += stride1, src2 += stride2) { for (x = xmin; x <= xmax; ++x) { - const int s1 = src1[x]; - const int s2 = src2[x]; - stats->w += 1; - stats->xm += s1; - stats->ym += s2; - stats->xxm += s1 * s1; - stats->xym += s1 * s2; - stats->yym += s2 * s2; + const uint32_t w = kWeight[VP8_SSIM_KERNEL + x - xo] + * kWeight[VP8_SSIM_KERNEL + y - yo]; + const uint32_t s1 = src1[x]; + const uint32_t s2 = src2[x]; + stats.w += w; + stats.xm += w * s1; + stats.ym += w * s2; + stats.xxm += w * s1 * s1; + stats.xym += w * s1 * s2; + stats.yym += w * s2 * s2; } } + return VP8SSIMFromStatsClipped(&stats); } -static void SSIMAccumulate(const uint8_t* src1, int stride1, - const uint8_t* src2, int stride2, - VP8DistoStats* const stats) { +static double SSIMGet_C(const uint8_t* src1, int stride1, + const uint8_t* src2, int stride2) { + VP8DistoStats stats = { 0, 0, 0, 0, 0, 0 }; int x, y; for (y = 0; y <= 2 * VP8_SSIM_KERNEL; ++y, src1 += stride1, src2 += stride2) { for (x = 0; x <= 2 * VP8_SSIM_KERNEL; ++x) { - const int s1 = src1[x]; - const int s2 = src2[x]; - stats->w += 1; - stats->xm += s1; - stats->ym += s2; - stats->xxm += s1 * s1; - stats->xym += s1 * s2; - stats->yym += s2 * s2; + const uint32_t w = kWeight[x] * kWeight[y]; + const uint32_t s1 = src1[x]; + const uint32_t s2 = src2[x]; + stats.xm += w * s1; + stats.ym += w * s2; + stats.xxm += w * s1 * s1; + stats.xym += w * s1 * s2; + stats.yym += w * s2 * s2; } } + return VP8SSIMFromStats(&stats); +} + +//------------------------------------------------------------------------------ + +static uint32_t AccumulateSSE(const uint8_t* src1, + const uint8_t* src2, int len) { + int i; + uint32_t sse2 = 0; + assert(len <= 65535); // to ensure that accumulation fits within uint32_t + for (i = 0; i < len; ++i) { + const int32_t diff = src1[i] - src2[i]; + sse2 += diff * diff; + } + return sse2; } -VP8SSIMAccumulateFunc VP8SSIMAccumulate; -VP8SSIMAccumulateClippedFunc VP8SSIMAccumulateClipped; +//------------------------------------------------------------------------------ + +VP8SSIMGetFunc VP8SSIMGet; +VP8SSIMGetClippedFunc VP8SSIMGetClipped; +VP8AccumulateSSEFunc VP8AccumulateSSE; + +extern void VP8SSIMDspInitSSE2(void); static volatile VP8CPUInfo ssim_last_cpuinfo_used = (VP8CPUInfo)&ssim_last_cpuinfo_used; @@ -758,8 +809,17 @@ static volatile VP8CPUInfo ssim_last_cpuinfo_used = WEBP_TSAN_IGNORE_FUNCTION void VP8SSIMDspInit(void) { if (ssim_last_cpuinfo_used == VP8GetCPUInfo) return; - VP8SSIMAccumulate = SSIMAccumulate; - VP8SSIMAccumulateClipped = SSIMAccumulateClipped; + VP8SSIMGetClipped = SSIMGetClipped_C; + VP8SSIMGet = SSIMGet_C; + + VP8AccumulateSSE = AccumulateSSE; + if (VP8GetCPUInfo != NULL) { +#if defined(WEBP_USE_SSE2) + if (VP8GetCPUInfo(kSSE2)) { + VP8SSIMDspInitSSE2(); + } +#endif + } ssim_last_cpuinfo_used = VP8GetCPUInfo; } @@ -783,6 +843,7 @@ VP8Metric VP8SSE16x8; VP8Metric VP8SSE4x4; VP8WMetric VP8TDisto4x4; VP8WMetric VP8TDisto16x16; +VP8MeanMetric VP8Mean16x4; VP8QuantizeBlock VP8EncQuantizeBlock; VP8Quantize2Blocks VP8EncQuantize2Blocks; VP8QuantizeBlockWHT VP8EncQuantizeBlockWHT; @@ -795,6 +856,7 @@ extern void VP8EncDspInitAVX2(void); extern void VP8EncDspInitNEON(void); extern void VP8EncDspInitMIPS32(void); extern void VP8EncDspInitMIPSdspR2(void); +extern void VP8EncDspInitMSA(void); static volatile VP8CPUInfo enc_last_cpuinfo_used = (VP8CPUInfo)&enc_last_cpuinfo_used; @@ -820,9 +882,10 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInit(void) { VP8SSE4x4 = SSE4x4; VP8TDisto4x4 = Disto4x4; VP8TDisto16x16 = Disto16x16; + VP8Mean16x4 = Mean16x4; VP8EncQuantizeBlock = QuantizeBlock; VP8EncQuantize2Blocks = Quantize2Blocks; - VP8EncQuantizeBlockWHT = QuantizeBlockWHT; + VP8EncQuantizeBlockWHT = QuantizeBlock; VP8Copy4x4 = Copy4x4; VP8Copy16x8 = Copy16x8; @@ -858,6 +921,11 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInit(void) { VP8EncDspInitMIPSdspR2(); } #endif +#if defined(WEBP_USE_MSA) + if (VP8GetCPUInfo(kMSA)) { + VP8EncDspInitMSA(); + } +#endif } enc_last_cpuinfo_used = VP8GetCPUInfo; } diff --git a/src/3rdparty/libwebp/src/dsp/enc_mips32.c b/src/3rdparty/libwebp/src/dsp/enc_mips32.c index fd10143..752b14d 100644 --- a/src/3rdparty/libwebp/src/dsp/enc_mips32.c +++ b/src/3rdparty/libwebp/src/dsp/enc_mips32.c @@ -18,8 +18,8 @@ #if defined(WEBP_USE_MIPS32) #include "./mips_macro.h" -#include "../enc/vp8enci.h" -#include "../enc/cost.h" +#include "../enc/vp8i_enc.h" +#include "../enc/cost_enc.h" static const int kC1 = 20091 + (1 << 16); static const int kC2 = 35468; diff --git a/src/3rdparty/libwebp/src/dsp/enc_mips_dsp_r2.c b/src/3rdparty/libwebp/src/dsp/enc_mips_dsp_r2.c index 7ab96f6..6c8c1c6 100644 --- a/src/3rdparty/libwebp/src/dsp/enc_mips_dsp_r2.c +++ b/src/3rdparty/libwebp/src/dsp/enc_mips_dsp_r2.c @@ -17,8 +17,8 @@ #if defined(WEBP_USE_MIPS_DSP_R2) #include "./mips_macro.h" -#include "../enc/cost.h" -#include "../enc/vp8enci.h" +#include "../enc/cost_enc.h" +#include "../enc/vp8i_enc.h" static const int kC1 = 20091 + (1 << 16); static const int kC2 = 35468; diff --git a/src/3rdparty/libwebp/src/dsp/enc_msa.c b/src/3rdparty/libwebp/src/dsp/enc_msa.c new file mode 100644 index 0000000..909b46d --- /dev/null +++ b/src/3rdparty/libwebp/src/dsp/enc_msa.c @@ -0,0 +1,892 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MSA version of encoder dsp functions. +// +// Author: Prashant Patil (prashant.patil@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MSA) + +#include <stdlib.h> +#include "./msa_macro.h" +#include "../enc/vp8i_enc.h" + +//------------------------------------------------------------------------------ +// Transforms + +#define IDCT_1D_W(in0, in1, in2, in3, out0, out1, out2, out3) do { \ + v4i32 a1_m, b1_m, c1_m, d1_m; \ + const v4i32 cospi8sqrt2minus1 = __msa_fill_w(20091); \ + const v4i32 sinpi8sqrt2 = __msa_fill_w(35468); \ + v4i32 c_tmp1_m = in1 * sinpi8sqrt2; \ + v4i32 c_tmp2_m = in3 * cospi8sqrt2minus1; \ + v4i32 d_tmp1_m = in1 * cospi8sqrt2minus1; \ + v4i32 d_tmp2_m = in3 * sinpi8sqrt2; \ + \ + ADDSUB2(in0, in2, a1_m, b1_m); \ + SRAI_W2_SW(c_tmp1_m, c_tmp2_m, 16); \ + c_tmp2_m = c_tmp2_m + in3; \ + c1_m = c_tmp1_m - c_tmp2_m; \ + SRAI_W2_SW(d_tmp1_m, d_tmp2_m, 16); \ + d_tmp1_m = d_tmp1_m + in1; \ + d1_m = d_tmp1_m + d_tmp2_m; \ + BUTTERFLY_4(a1_m, b1_m, c1_m, d1_m, out0, out1, out2, out3); \ +} while (0) + +static WEBP_INLINE void ITransformOne(const uint8_t* ref, const int16_t* in, + uint8_t* dst) { + v8i16 input0, input1; + v4i32 in0, in1, in2, in3, hz0, hz1, hz2, hz3, vt0, vt1, vt2, vt3; + v4i32 res0, res1, res2, res3; + v16i8 dest0, dest1, dest2, dest3; + const v16i8 zero = { 0 }; + + LD_SH2(in, 8, input0, input1); + UNPCK_SH_SW(input0, in0, in1); + UNPCK_SH_SW(input1, in2, in3); + IDCT_1D_W(in0, in1, in2, in3, hz0, hz1, hz2, hz3); + TRANSPOSE4x4_SW_SW(hz0, hz1, hz2, hz3, hz0, hz1, hz2, hz3); + IDCT_1D_W(hz0, hz1, hz2, hz3, vt0, vt1, vt2, vt3); + SRARI_W4_SW(vt0, vt1, vt2, vt3, 3); + TRANSPOSE4x4_SW_SW(vt0, vt1, vt2, vt3, vt0, vt1, vt2, vt3); + LD_SB4(ref, BPS, dest0, dest1, dest2, dest3); + ILVR_B4_SW(zero, dest0, zero, dest1, zero, dest2, zero, dest3, + res0, res1, res2, res3); + ILVR_H4_SW(zero, res0, zero, res1, zero, res2, zero, res3, + res0, res1, res2, res3); + ADD4(res0, vt0, res1, vt1, res2, vt2, res3, vt3, res0, res1, res2, res3); + CLIP_SW4_0_255(res0, res1, res2, res3); + PCKEV_B2_SW(res0, res1, res2, res3, vt0, vt1); + res0 = (v4i32)__msa_pckev_b((v16i8)vt0, (v16i8)vt1); + ST4x4_UB(res0, res0, 3, 2, 1, 0, dst, BPS); +} + +static void ITransform(const uint8_t* ref, const int16_t* in, uint8_t* dst, + int do_two) { + ITransformOne(ref, in, dst); + if (do_two) { + ITransformOne(ref + 4, in + 16, dst + 4); + } +} + +static void FTransform(const uint8_t* src, const uint8_t* ref, int16_t* out) { + uint64_t out0, out1, out2, out3; + uint32_t in0, in1, in2, in3; + v4i32 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + v8i16 t0, t1, t2, t3; + v16u8 srcl0, srcl1, src0, src1; + const v8i16 mask0 = { 0, 4, 8, 12, 1, 5, 9, 13 }; + const v8i16 mask1 = { 3, 7, 11, 15, 2, 6, 10, 14 }; + const v8i16 mask2 = { 4, 0, 5, 1, 6, 2, 7, 3 }; + const v8i16 mask3 = { 0, 4, 1, 5, 2, 6, 3, 7 }; + const v8i16 cnst0 = { 2217, -5352, 2217, -5352, 2217, -5352, 2217, -5352 }; + const v8i16 cnst1 = { 5352, 2217, 5352, 2217, 5352, 2217, 5352, 2217 }; + + LW4(src, BPS, in0, in1, in2, in3); + INSERT_W4_UB(in0, in1, in2, in3, src0); + LW4(ref, BPS, in0, in1, in2, in3); + INSERT_W4_UB(in0, in1, in2, in3, src1); + ILVRL_B2_UB(src0, src1, srcl0, srcl1); + HSUB_UB2_SH(srcl0, srcl1, t0, t1); + VSHF_H2_SH(t0, t1, t0, t1, mask0, mask1, t2, t3); + ADDSUB2(t2, t3, t0, t1); + t0 = SRLI_H(t0, 3); + VSHF_H2_SH(t0, t0, t1, t1, mask2, mask3, t3, t2); + tmp0 = __msa_hadd_s_w(t3, t3); + tmp2 = __msa_hsub_s_w(t3, t3); + FILL_W2_SW(1812, 937, tmp1, tmp3); + DPADD_SH2_SW(t2, t2, cnst0, cnst1, tmp3, tmp1); + SRAI_W2_SW(tmp1, tmp3, 9); + PCKEV_H2_SH(tmp1, tmp0, tmp3, tmp2, t0, t1); + VSHF_H2_SH(t0, t1, t0, t1, mask0, mask1, t2, t3); + ADDSUB2(t2, t3, t0, t1); + VSHF_H2_SH(t0, t0, t1, t1, mask2, mask3, t3, t2); + tmp0 = __msa_hadd_s_w(t3, t3); + tmp2 = __msa_hsub_s_w(t3, t3); + ADDVI_W2_SW(tmp0, 7, tmp2, 7, tmp0, tmp2); + SRAI_W2_SW(tmp0, tmp2, 4); + FILL_W2_SW(12000, 51000, tmp1, tmp3); + DPADD_SH2_SW(t2, t2, cnst0, cnst1, tmp3, tmp1); + SRAI_W2_SW(tmp1, tmp3, 16); + UNPCK_R_SH_SW(t1, tmp4); + tmp5 = __msa_ceqi_w(tmp4, 0); + tmp4 = (v4i32)__msa_nor_v((v16u8)tmp5, (v16u8)tmp5); + tmp5 = __msa_fill_w(1); + tmp5 = (v4i32)__msa_and_v((v16u8)tmp5, (v16u8)tmp4); + tmp1 += tmp5; + PCKEV_H2_SH(tmp1, tmp0, tmp3, tmp2, t0, t1); + out0 = __msa_copy_s_d((v2i64)t0, 0); + out1 = __msa_copy_s_d((v2i64)t0, 1); + out2 = __msa_copy_s_d((v2i64)t1, 0); + out3 = __msa_copy_s_d((v2i64)t1, 1); + SD4(out0, out1, out2, out3, out, 8); +} + +static void FTransformWHT(const int16_t* in, int16_t* out) { + v8i16 in0 = { 0 }; + v8i16 in1 = { 0 }; + v8i16 tmp0, tmp1, tmp2, tmp3; + v8i16 out0, out1; + const v8i16 mask0 = { 0, 1, 2, 3, 8, 9, 10, 11 }; + const v8i16 mask1 = { 4, 5, 6, 7, 12, 13, 14, 15 }; + const v8i16 mask2 = { 0, 4, 8, 12, 1, 5, 9, 13 }; + const v8i16 mask3 = { 3, 7, 11, 15, 2, 6, 10, 14 }; + + in0 = __msa_insert_h(in0, 0, in[ 0]); + in0 = __msa_insert_h(in0, 1, in[ 64]); + in0 = __msa_insert_h(in0, 2, in[128]); + in0 = __msa_insert_h(in0, 3, in[192]); + in0 = __msa_insert_h(in0, 4, in[ 16]); + in0 = __msa_insert_h(in0, 5, in[ 80]); + in0 = __msa_insert_h(in0, 6, in[144]); + in0 = __msa_insert_h(in0, 7, in[208]); + in1 = __msa_insert_h(in1, 0, in[ 48]); + in1 = __msa_insert_h(in1, 1, in[112]); + in1 = __msa_insert_h(in1, 2, in[176]); + in1 = __msa_insert_h(in1, 3, in[240]); + in1 = __msa_insert_h(in1, 4, in[ 32]); + in1 = __msa_insert_h(in1, 5, in[ 96]); + in1 = __msa_insert_h(in1, 6, in[160]); + in1 = __msa_insert_h(in1, 7, in[224]); + ADDSUB2(in0, in1, tmp0, tmp1); + VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask0, mask1, tmp2, tmp3); + ADDSUB2(tmp2, tmp3, tmp0, tmp1); + VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask2, mask3, in0, in1); + ADDSUB2(in0, in1, tmp0, tmp1); + VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask0, mask1, tmp2, tmp3); + ADDSUB2(tmp2, tmp3, out0, out1); + SRAI_H2_SH(out0, out1, 1); + ST_SH2(out0, out1, out, 8); +} + +static int TTransform(const uint8_t* in, const uint16_t* w) { + int sum; + uint32_t in0_m, in1_m, in2_m, in3_m; + v16i8 src0; + v8i16 in0, in1, tmp0, tmp1, tmp2, tmp3; + v4i32 dst0, dst1; + const v16i8 zero = { 0 }; + const v8i16 mask0 = { 0, 1, 2, 3, 8, 9, 10, 11 }; + const v8i16 mask1 = { 4, 5, 6, 7, 12, 13, 14, 15 }; + const v8i16 mask2 = { 0, 4, 8, 12, 1, 5, 9, 13 }; + const v8i16 mask3 = { 3, 7, 11, 15, 2, 6, 10, 14 }; + + LW4(in, BPS, in0_m, in1_m, in2_m, in3_m); + INSERT_W4_SB(in0_m, in1_m, in2_m, in3_m, src0); + ILVRL_B2_SH(zero, src0, tmp0, tmp1); + VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask2, mask3, in0, in1); + ADDSUB2(in0, in1, tmp0, tmp1); + VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask0, mask1, tmp2, tmp3); + ADDSUB2(tmp2, tmp3, tmp0, tmp1); + VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask2, mask3, in0, in1); + ADDSUB2(in0, in1, tmp0, tmp1); + VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask0, mask1, tmp2, tmp3); + ADDSUB2(tmp2, tmp3, tmp0, tmp1); + tmp0 = __msa_add_a_h(tmp0, (v8i16)zero); + tmp1 = __msa_add_a_h(tmp1, (v8i16)zero); + LD_SH2(w, 8, tmp2, tmp3); + DOTP_SH2_SW(tmp0, tmp1, tmp2, tmp3, dst0, dst1); + dst0 = dst0 + dst1; + sum = HADD_SW_S32(dst0); + return sum; +} + +static int Disto4x4(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + const int sum1 = TTransform(a, w); + const int sum2 = TTransform(b, w); + return abs(sum2 - sum1) >> 5; +} + +static int Disto16x16(const uint8_t* const a, const uint8_t* const b, + const uint16_t* const w) { + int D = 0; + int x, y; + for (y = 0; y < 16 * BPS; y += 4 * BPS) { + for (x = 0; x < 16; x += 4) { + D += Disto4x4(a + x + y, b + x + y, w); + } + } + return D; +} + +//------------------------------------------------------------------------------ +// Histogram + +static void CollectHistogram(const uint8_t* ref, const uint8_t* pred, + int start_block, int end_block, + VP8Histogram* const histo) { + int j; + int distribution[MAX_COEFF_THRESH + 1] = { 0 }; + for (j = start_block; j < end_block; ++j) { + int16_t out[16]; + VP8FTransform(ref + VP8DspScan[j], pred + VP8DspScan[j], out); + { + int k; + v8i16 coeff0, coeff1; + const v8i16 zero = { 0 }; + const v8i16 max_coeff_thr = __msa_ldi_h(MAX_COEFF_THRESH); + LD_SH2(&out[0], 8, coeff0, coeff1); + coeff0 = __msa_add_a_h(coeff0, zero); + coeff1 = __msa_add_a_h(coeff1, zero); + SRAI_H2_SH(coeff0, coeff1, 3); + coeff0 = __msa_min_s_h(coeff0, max_coeff_thr); + coeff1 = __msa_min_s_h(coeff1, max_coeff_thr); + ST_SH2(coeff0, coeff1, &out[0], 8); + for (k = 0; k < 16; ++k) { + ++distribution[out[k]]; + } + } + } + VP8SetHistogramData(distribution, histo); +} + +//------------------------------------------------------------------------------ +// Intra predictions + +// luma 4x4 prediction + +#define DST(x, y) dst[(x) + (y) * BPS] +#define AVG3(a, b, c) (((a) + 2 * (b) + (c) + 2) >> 2) +#define AVG2(a, b) (((a) + (b) + 1) >> 1) + +static WEBP_INLINE void VE4(uint8_t* dst, const uint8_t* top) { // vertical + const uint64_t val_m = LD(top - 1); + const v16u8 A = (v16u8)__msa_insert_d((v2i64)A, 0, val_m); + const v16u8 B = SLDI_UB(A, A, 1); + const v16u8 C = SLDI_UB(A, A, 2); + const v16u8 AC = __msa_ave_u_b(A, C); + const v16u8 B2 = __msa_ave_u_b(B, B); + const v16u8 R = __msa_aver_u_b(AC, B2); + const uint32_t out = __msa_copy_s_w((v4i32)R, 0); + SW4(out, out, out, out, dst, BPS); +} + +static WEBP_INLINE void HE4(uint8_t* dst, const uint8_t* top) { // horizontal + const int X = top[-1]; + const int I = top[-2]; + const int J = top[-3]; + const int K = top[-4]; + const int L = top[-5]; + WebPUint32ToMem(dst + 0 * BPS, 0x01010101U * AVG3(X, I, J)); + WebPUint32ToMem(dst + 1 * BPS, 0x01010101U * AVG3(I, J, K)); + WebPUint32ToMem(dst + 2 * BPS, 0x01010101U * AVG3(J, K, L)); + WebPUint32ToMem(dst + 3 * BPS, 0x01010101U * AVG3(K, L, L)); +} + +static WEBP_INLINE void DC4(uint8_t* dst, const uint8_t* top) { + uint32_t dc = 4; + int i; + for (i = 0; i < 4; ++i) dc += top[i] + top[-5 + i]; + dc >>= 3; + dc = dc | (dc << 8) | (dc << 16) | (dc << 24); + SW4(dc, dc, dc, dc, dst, BPS); +} + +static WEBP_INLINE void RD4(uint8_t* dst, const uint8_t* top) { + const uint64_t val_m = LD(top - 5); + const v16u8 A1 = (v16u8)__msa_insert_d((v2i64)A1, 0, val_m); + const v16u8 A = (v16u8)__msa_insert_b((v16i8)A1, 8, top[3]); + const v16u8 B = SLDI_UB(A, A, 1); + const v16u8 C = SLDI_UB(A, A, 2); + const v16u8 AC = __msa_ave_u_b(A, C); + const v16u8 B2 = __msa_ave_u_b(B, B); + const v16u8 R0 = __msa_aver_u_b(AC, B2); + const v16u8 R1 = SLDI_UB(R0, R0, 1); + const v16u8 R2 = SLDI_UB(R1, R1, 1); + const v16u8 R3 = SLDI_UB(R2, R2, 1); + const uint32_t val0 = __msa_copy_s_w((v4i32)R0, 0); + const uint32_t val1 = __msa_copy_s_w((v4i32)R1, 0); + const uint32_t val2 = __msa_copy_s_w((v4i32)R2, 0); + const uint32_t val3 = __msa_copy_s_w((v4i32)R3, 0); + SW4(val3, val2, val1, val0, dst, BPS); +} + +static WEBP_INLINE void LD4(uint8_t* dst, const uint8_t* top) { + const uint64_t val_m = LD(top); + const v16u8 A = (v16u8)__msa_insert_d((v2i64)A, 0, val_m); + const v16u8 B = SLDI_UB(A, A, 1); + const v16u8 C1 = SLDI_UB(A, A, 2); + const v16u8 C = (v16u8)__msa_insert_b((v16i8)C1, 6, top[7]); + const v16u8 AC = __msa_ave_u_b(A, C); + const v16u8 B2 = __msa_ave_u_b(B, B); + const v16u8 R0 = __msa_aver_u_b(AC, B2); + const v16u8 R1 = SLDI_UB(R0, R0, 1); + const v16u8 R2 = SLDI_UB(R1, R1, 1); + const v16u8 R3 = SLDI_UB(R2, R2, 1); + const uint32_t val0 = __msa_copy_s_w((v4i32)R0, 0); + const uint32_t val1 = __msa_copy_s_w((v4i32)R1, 0); + const uint32_t val2 = __msa_copy_s_w((v4i32)R2, 0); + const uint32_t val3 = __msa_copy_s_w((v4i32)R3, 0); + SW4(val0, val1, val2, val3, dst, BPS); +} + +static WEBP_INLINE void VR4(uint8_t* dst, const uint8_t* top) { + const int X = top[-1]; + const int I = top[-2]; + const int J = top[-3]; + const int K = top[-4]; + const int A = top[0]; + const int B = top[1]; + const int C = top[2]; + const int D = top[3]; + DST(0, 0) = DST(1, 2) = AVG2(X, A); + DST(1, 0) = DST(2, 2) = AVG2(A, B); + DST(2, 0) = DST(3, 2) = AVG2(B, C); + DST(3, 0) = AVG2(C, D); + DST(0, 3) = AVG3(K, J, I); + DST(0, 2) = AVG3(J, I, X); + DST(0, 1) = DST(1, 3) = AVG3(I, X, A); + DST(1, 1) = DST(2, 3) = AVG3(X, A, B); + DST(2, 1) = DST(3, 3) = AVG3(A, B, C); + DST(3, 1) = AVG3(B, C, D); +} + +static WEBP_INLINE void VL4(uint8_t* dst, const uint8_t* top) { + const int A = top[0]; + const int B = top[1]; + const int C = top[2]; + const int D = top[3]; + const int E = top[4]; + const int F = top[5]; + const int G = top[6]; + const int H = top[7]; + DST(0, 0) = AVG2(A, B); + DST(1, 0) = DST(0, 2) = AVG2(B, C); + DST(2, 0) = DST(1, 2) = AVG2(C, D); + DST(3, 0) = DST(2, 2) = AVG2(D, E); + DST(0, 1) = AVG3(A, B, C); + DST(1, 1) = DST(0, 3) = AVG3(B, C, D); + DST(2, 1) = DST(1, 3) = AVG3(C, D, E); + DST(3, 1) = DST(2, 3) = AVG3(D, E, F); + DST(3, 2) = AVG3(E, F, G); + DST(3, 3) = AVG3(F, G, H); +} + +static WEBP_INLINE void HU4(uint8_t* dst, const uint8_t* top) { + const int I = top[-2]; + const int J = top[-3]; + const int K = top[-4]; + const int L = top[-5]; + DST(0, 0) = AVG2(I, J); + DST(2, 0) = DST(0, 1) = AVG2(J, K); + DST(2, 1) = DST(0, 2) = AVG2(K, L); + DST(1, 0) = AVG3(I, J, K); + DST(3, 0) = DST(1, 1) = AVG3(J, K, L); + DST(3, 1) = DST(1, 2) = AVG3(K, L, L); + DST(3, 2) = DST(2, 2) = + DST(0, 3) = DST(1, 3) = DST(2, 3) = DST(3, 3) = L; +} + +static WEBP_INLINE void HD4(uint8_t* dst, const uint8_t* top) { + const int X = top[-1]; + const int I = top[-2]; + const int J = top[-3]; + const int K = top[-4]; + const int L = top[-5]; + const int A = top[0]; + const int B = top[1]; + const int C = top[2]; + DST(0, 0) = DST(2, 1) = AVG2(I, X); + DST(0, 1) = DST(2, 2) = AVG2(J, I); + DST(0, 2) = DST(2, 3) = AVG2(K, J); + DST(0, 3) = AVG2(L, K); + DST(3, 0) = AVG3(A, B, C); + DST(2, 0) = AVG3(X, A, B); + DST(1, 0) = DST(3, 1) = AVG3(I, X, A); + DST(1, 1) = DST(3, 2) = AVG3(J, I, X); + DST(1, 2) = DST(3, 3) = AVG3(K, J, I); + DST(1, 3) = AVG3(L, K, J); +} + +static WEBP_INLINE void TM4(uint8_t* dst, const uint8_t* top) { + const v16i8 zero = { 0 }; + const v8i16 TL = (v8i16)__msa_fill_h(top[-1]); + const v8i16 L0 = (v8i16)__msa_fill_h(top[-2]); + const v8i16 L1 = (v8i16)__msa_fill_h(top[-3]); + const v8i16 L2 = (v8i16)__msa_fill_h(top[-4]); + const v8i16 L3 = (v8i16)__msa_fill_h(top[-5]); + const v16u8 T1 = LD_UB(top); + const v8i16 T = (v8i16)__msa_ilvr_b(zero, (v16i8)T1); + const v8i16 d = T - TL; + v8i16 r0, r1, r2, r3; + ADD4(d, L0, d, L1, d, L2, d, L3, r0, r1, r2, r3); + CLIP_SH4_0_255(r0, r1, r2, r3); + PCKEV_ST4x4_UB(r0, r1, r2, r3, dst, BPS); +} + +#undef DST +#undef AVG3 +#undef AVG2 + +static void Intra4Preds(uint8_t* dst, const uint8_t* top) { + DC4(I4DC4 + dst, top); + TM4(I4TM4 + dst, top); + VE4(I4VE4 + dst, top); + HE4(I4HE4 + dst, top); + RD4(I4RD4 + dst, top); + VR4(I4VR4 + dst, top); + LD4(I4LD4 + dst, top); + VL4(I4VL4 + dst, top); + HD4(I4HD4 + dst, top); + HU4(I4HU4 + dst, top); +} + +// luma 16x16 prediction + +#define STORE16x16(out, dst) do { \ + ST_UB8(out, out, out, out, out, out, out, out, dst + 0 * BPS, BPS); \ + ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS); \ +} while (0) + +static WEBP_INLINE void VerticalPred16x16(uint8_t* dst, const uint8_t* top) { + if (top != NULL) { + const v16u8 out = LD_UB(top); + STORE16x16(out, dst); + } else { + const v16u8 out = (v16u8)__msa_fill_b(0x7f); + STORE16x16(out, dst); + } +} + +static WEBP_INLINE void HorizontalPred16x16(uint8_t* dst, + const uint8_t* left) { + if (left != NULL) { + int j; + for (j = 0; j < 16; j += 4) { + const v16u8 L0 = (v16u8)__msa_fill_b(left[0]); + const v16u8 L1 = (v16u8)__msa_fill_b(left[1]); + const v16u8 L2 = (v16u8)__msa_fill_b(left[2]); + const v16u8 L3 = (v16u8)__msa_fill_b(left[3]); + ST_UB4(L0, L1, L2, L3, dst, BPS); + dst += 4 * BPS; + left += 4; + } + } else { + const v16u8 out = (v16u8)__msa_fill_b(0x81); + STORE16x16(out, dst); + } +} + +static WEBP_INLINE void TrueMotion16x16(uint8_t* dst, const uint8_t* left, + const uint8_t* top) { + if (left != NULL) { + if (top != NULL) { + int j; + v8i16 d1, d2; + const v16i8 zero = { 0 }; + const v8i16 TL = (v8i16)__msa_fill_h(left[-1]); + const v16u8 T = LD_UB(top); + ILVRL_B2_SH(zero, T, d1, d2); + SUB2(d1, TL, d2, TL, d1, d2); + for (j = 0; j < 16; j += 4) { + v16i8 t0, t1, t2, t3; + v8i16 r0, r1, r2, r3, r4, r5, r6, r7; + const v8i16 L0 = (v8i16)__msa_fill_h(left[j + 0]); + const v8i16 L1 = (v8i16)__msa_fill_h(left[j + 1]); + const v8i16 L2 = (v8i16)__msa_fill_h(left[j + 2]); + const v8i16 L3 = (v8i16)__msa_fill_h(left[j + 3]); + ADD4(d1, L0, d1, L1, d1, L2, d1, L3, r0, r1, r2, r3); + ADD4(d2, L0, d2, L1, d2, L2, d2, L3, r4, r5, r6, r7); + CLIP_SH4_0_255(r0, r1, r2, r3); + CLIP_SH4_0_255(r4, r5, r6, r7); + PCKEV_B4_SB(r4, r0, r5, r1, r6, r2, r7, r3, t0, t1, t2, t3); + ST_SB4(t0, t1, t2, t3, dst, BPS); + dst += 4 * BPS; + } + } else { + HorizontalPred16x16(dst, left); + } + } else { + if (top != NULL) { + VerticalPred16x16(dst, top); + } else { + const v16u8 out = (v16u8)__msa_fill_b(0x81); + STORE16x16(out, dst); + } + } +} + +static WEBP_INLINE void DCMode16x16(uint8_t* dst, const uint8_t* left, + const uint8_t* top) { + int DC; + v16u8 out; + if (top != NULL && left != NULL) { + const v16u8 rtop = LD_UB(top); + const v8u16 dctop = __msa_hadd_u_h(rtop, rtop); + const v16u8 rleft = LD_UB(left); + const v8u16 dcleft = __msa_hadd_u_h(rleft, rleft); + const v8u16 dctemp = dctop + dcleft; + DC = HADD_UH_U32(dctemp); + DC = (DC + 16) >> 5; + } else if (left != NULL) { // left but no top + const v16u8 rleft = LD_UB(left); + const v8u16 dcleft = __msa_hadd_u_h(rleft, rleft); + DC = HADD_UH_U32(dcleft); + DC = (DC + DC + 16) >> 5; + } else if (top != NULL) { // top but no left + const v16u8 rtop = LD_UB(top); + const v8u16 dctop = __msa_hadd_u_h(rtop, rtop); + DC = HADD_UH_U32(dctop); + DC = (DC + DC + 16) >> 5; + } else { // no top, no left, nothing. + DC = 0x80; + } + out = (v16u8)__msa_fill_b(DC); + STORE16x16(out, dst); +} + +static void Intra16Preds(uint8_t* dst, + const uint8_t* left, const uint8_t* top) { + DCMode16x16(I16DC16 + dst, left, top); + VerticalPred16x16(I16VE16 + dst, top); + HorizontalPred16x16(I16HE16 + dst, left); + TrueMotion16x16(I16TM16 + dst, left, top); +} + +// Chroma 8x8 prediction + +#define CALC_DC8(in, out) do { \ + const v8u16 temp0 = __msa_hadd_u_h(in, in); \ + const v4u32 temp1 = __msa_hadd_u_w(temp0, temp0); \ + const v2i64 temp2 = (v2i64)__msa_hadd_u_d(temp1, temp1); \ + const v2i64 temp3 = __msa_splati_d(temp2, 1); \ + const v2i64 temp4 = temp3 + temp2; \ + const v16i8 temp5 = (v16i8)__msa_srari_d(temp4, 4); \ + const v2i64 temp6 = (v2i64)__msa_splati_b(temp5, 0); \ + out = __msa_copy_s_d(temp6, 0); \ +} while (0) + +#define STORE8x8(out, dst) do { \ + SD4(out, out, out, out, dst + 0 * BPS, BPS); \ + SD4(out, out, out, out, dst + 4 * BPS, BPS); \ +} while (0) + +static WEBP_INLINE void VerticalPred8x8(uint8_t* dst, const uint8_t* top) { + if (top != NULL) { + const uint64_t out = LD(top); + STORE8x8(out, dst); + } else { + const uint64_t out = 0x7f7f7f7f7f7f7f7fULL; + STORE8x8(out, dst); + } +} + +static WEBP_INLINE void HorizontalPred8x8(uint8_t* dst, const uint8_t* left) { + if (left != NULL) { + int j; + for (j = 0; j < 8; j += 4) { + const v16u8 L0 = (v16u8)__msa_fill_b(left[0]); + const v16u8 L1 = (v16u8)__msa_fill_b(left[1]); + const v16u8 L2 = (v16u8)__msa_fill_b(left[2]); + const v16u8 L3 = (v16u8)__msa_fill_b(left[3]); + const uint64_t out0 = __msa_copy_s_d((v2i64)L0, 0); + const uint64_t out1 = __msa_copy_s_d((v2i64)L1, 0); + const uint64_t out2 = __msa_copy_s_d((v2i64)L2, 0); + const uint64_t out3 = __msa_copy_s_d((v2i64)L3, 0); + SD4(out0, out1, out2, out3, dst, BPS); + dst += 4 * BPS; + left += 4; + } + } else { + const uint64_t out = 0x8181818181818181ULL; + STORE8x8(out, dst); + } +} + +static WEBP_INLINE void TrueMotion8x8(uint8_t* dst, const uint8_t* left, + const uint8_t* top) { + if (left != NULL) { + if (top != NULL) { + int j; + const v8i16 TL = (v8i16)__msa_fill_h(left[-1]); + const v16u8 T1 = LD_UB(top); + const v16i8 zero = { 0 }; + const v8i16 T = (v8i16)__msa_ilvr_b(zero, (v16i8)T1); + const v8i16 d = T - TL; + for (j = 0; j < 8; j += 4) { + uint64_t out0, out1, out2, out3; + v16i8 t0, t1; + v8i16 r0 = (v8i16)__msa_fill_h(left[j + 0]); + v8i16 r1 = (v8i16)__msa_fill_h(left[j + 1]); + v8i16 r2 = (v8i16)__msa_fill_h(left[j + 2]); + v8i16 r3 = (v8i16)__msa_fill_h(left[j + 3]); + ADD4(d, r0, d, r1, d, r2, d, r3, r0, r1, r2, r3); + CLIP_SH4_0_255(r0, r1, r2, r3); + PCKEV_B2_SB(r1, r0, r3, r2, t0, t1); + out0 = __msa_copy_s_d((v2i64)t0, 0); + out1 = __msa_copy_s_d((v2i64)t0, 1); + out2 = __msa_copy_s_d((v2i64)t1, 0); + out3 = __msa_copy_s_d((v2i64)t1, 1); + SD4(out0, out1, out2, out3, dst, BPS); + dst += 4 * BPS; + } + } else { + HorizontalPred8x8(dst, left); + } + } else { + if (top != NULL) { + VerticalPred8x8(dst, top); + } else { + const uint64_t out = 0x8181818181818181ULL; + STORE8x8(out, dst); + } + } +} + +static WEBP_INLINE void DCMode8x8(uint8_t* dst, const uint8_t* left, + const uint8_t* top) { + uint64_t out; + v16u8 src; + if (top != NULL && left != NULL) { + const uint64_t left_m = LD(left); + const uint64_t top_m = LD(top); + INSERT_D2_UB(left_m, top_m, src); + CALC_DC8(src, out); + } else if (left != NULL) { // left but no top + const uint64_t left_m = LD(left); + INSERT_D2_UB(left_m, left_m, src); + CALC_DC8(src, out); + } else if (top != NULL) { // top but no left + const uint64_t top_m = LD(top); + INSERT_D2_UB(top_m, top_m, src); + CALC_DC8(src, out); + } else { // no top, no left, nothing. + src = (v16u8)__msa_fill_b(0x80); + out = __msa_copy_s_d((v2i64)src, 0); + } + STORE8x8(out, dst); +} + +static void IntraChromaPreds(uint8_t* dst, const uint8_t* left, + const uint8_t* top) { + // U block + DCMode8x8(C8DC8 + dst, left, top); + VerticalPred8x8(C8VE8 + dst, top); + HorizontalPred8x8(C8HE8 + dst, left); + TrueMotion8x8(C8TM8 + dst, left, top); + // V block + dst += 8; + if (top != NULL) top += 8; + if (left != NULL) left += 16; + DCMode8x8(C8DC8 + dst, left, top); + VerticalPred8x8(C8VE8 + dst, top); + HorizontalPred8x8(C8HE8 + dst, left); + TrueMotion8x8(C8TM8 + dst, left, top); +} + +//------------------------------------------------------------------------------ +// Metric + +#define PACK_DOTP_UB4_SW(in0, in1, in2, in3, out0, out1, out2, out3) do { \ + v16u8 tmp0, tmp1; \ + v8i16 tmp2, tmp3; \ + ILVRL_B2_UB(in0, in1, tmp0, tmp1); \ + HSUB_UB2_SH(tmp0, tmp1, tmp2, tmp3); \ + DOTP_SH2_SW(tmp2, tmp3, tmp2, tmp3, out0, out1); \ + ILVRL_B2_UB(in2, in3, tmp0, tmp1); \ + HSUB_UB2_SH(tmp0, tmp1, tmp2, tmp3); \ + DOTP_SH2_SW(tmp2, tmp3, tmp2, tmp3, out2, out3); \ +} while (0) + +#define PACK_DPADD_UB4_SW(in0, in1, in2, in3, out0, out1, out2, out3) do { \ + v16u8 tmp0, tmp1; \ + v8i16 tmp2, tmp3; \ + ILVRL_B2_UB(in0, in1, tmp0, tmp1); \ + HSUB_UB2_SH(tmp0, tmp1, tmp2, tmp3); \ + DPADD_SH2_SW(tmp2, tmp3, tmp2, tmp3, out0, out1); \ + ILVRL_B2_UB(in2, in3, tmp0, tmp1); \ + HSUB_UB2_SH(tmp0, tmp1, tmp2, tmp3); \ + DPADD_SH2_SW(tmp2, tmp3, tmp2, tmp3, out2, out3); \ +} while (0) + +static int SSE16x16(const uint8_t* a, const uint8_t* b) { + uint32_t sum; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7; + v16u8 ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7; + v4i32 out0, out1, out2, out3; + + LD_UB8(a, BPS, src0, src1, src2, src3, src4, src5, src6, src7); + LD_UB8(b, BPS, ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7); + PACK_DOTP_UB4_SW(src0, ref0, src1, ref1, out0, out1, out2, out3); + PACK_DPADD_UB4_SW(src2, ref2, src3, ref3, out0, out1, out2, out3); + PACK_DPADD_UB4_SW(src4, ref4, src5, ref5, out0, out1, out2, out3); + PACK_DPADD_UB4_SW(src6, ref6, src7, ref7, out0, out1, out2, out3); + a += 8 * BPS; + b += 8 * BPS; + LD_UB8(a, BPS, src0, src1, src2, src3, src4, src5, src6, src7); + LD_UB8(b, BPS, ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7); + PACK_DPADD_UB4_SW(src0, ref0, src1, ref1, out0, out1, out2, out3); + PACK_DPADD_UB4_SW(src2, ref2, src3, ref3, out0, out1, out2, out3); + PACK_DPADD_UB4_SW(src4, ref4, src5, ref5, out0, out1, out2, out3); + PACK_DPADD_UB4_SW(src6, ref6, src7, ref7, out0, out1, out2, out3); + out0 += out1; + out2 += out3; + out0 += out2; + sum = HADD_SW_S32(out0); + return sum; +} + +static int SSE16x8(const uint8_t* a, const uint8_t* b) { + uint32_t sum; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7; + v16u8 ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7; + v4i32 out0, out1, out2, out3; + + LD_UB8(a, BPS, src0, src1, src2, src3, src4, src5, src6, src7); + LD_UB8(b, BPS, ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7); + PACK_DOTP_UB4_SW(src0, ref0, src1, ref1, out0, out1, out2, out3); + PACK_DPADD_UB4_SW(src2, ref2, src3, ref3, out0, out1, out2, out3); + PACK_DPADD_UB4_SW(src4, ref4, src5, ref5, out0, out1, out2, out3); + PACK_DPADD_UB4_SW(src6, ref6, src7, ref7, out0, out1, out2, out3); + out0 += out1; + out2 += out3; + out0 += out2; + sum = HADD_SW_S32(out0); + return sum; +} + +static int SSE8x8(const uint8_t* a, const uint8_t* b) { + uint32_t sum; + v16u8 src0, src1, src2, src3, src4, src5, src6, src7; + v16u8 ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7; + v16u8 t0, t1, t2, t3; + v4i32 out0, out1, out2, out3; + + LD_UB8(a, BPS, src0, src1, src2, src3, src4, src5, src6, src7); + LD_UB8(b, BPS, ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7); + ILVR_B4_UB(src0, src1, src2, src3, ref0, ref1, ref2, ref3, t0, t1, t2, t3); + PACK_DOTP_UB4_SW(t0, t2, t1, t3, out0, out1, out2, out3); + ILVR_B4_UB(src4, src5, src6, src7, ref4, ref5, ref6, ref7, t0, t1, t2, t3); + PACK_DPADD_UB4_SW(t0, t2, t1, t3, out0, out1, out2, out3); + out0 += out1; + out2 += out3; + out0 += out2; + sum = HADD_SW_S32(out0); + return sum; +} + +static int SSE4x4(const uint8_t* a, const uint8_t* b) { + uint32_t sum = 0; + uint32_t src0, src1, src2, src3, ref0, ref1, ref2, ref3; + v16u8 src, ref, tmp0, tmp1; + v8i16 diff0, diff1; + v4i32 out0, out1; + + LW4(a, BPS, src0, src1, src2, src3); + LW4(b, BPS, ref0, ref1, ref2, ref3); + INSERT_W4_UB(src0, src1, src2, src3, src); + INSERT_W4_UB(ref0, ref1, ref2, ref3, ref); + ILVRL_B2_UB(src, ref, tmp0, tmp1); + HSUB_UB2_SH(tmp0, tmp1, diff0, diff1); + DOTP_SH2_SW(diff0, diff1, diff0, diff1, out0, out1); + out0 += out1; + sum = HADD_SW_S32(out0); + return sum; +} + +//------------------------------------------------------------------------------ +// Quantization + +static int QuantizeBlock(int16_t in[16], int16_t out[16], + const VP8Matrix* const mtx) { + int sum; + v8i16 in0, in1, sh0, sh1, out0, out1; + v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, sign0, sign1; + v4i32 s0, s1, s2, s3, b0, b1, b2, b3, t0, t1, t2, t3; + const v8i16 zero = { 0 }; + const v8i16 zigzag0 = { 0, 1, 4, 8, 5, 2, 3, 6 }; + const v8i16 zigzag1 = { 9, 12, 13, 10, 7, 11, 14, 15 }; + const v8i16 maxlevel = __msa_fill_h(MAX_LEVEL); + + LD_SH2(&in[0], 8, in0, in1); + LD_SH2(&mtx->sharpen_[0], 8, sh0, sh1); + tmp4 = __msa_add_a_h(in0, zero); + tmp5 = __msa_add_a_h(in1, zero); + ILVRL_H2_SH(sh0, tmp4, tmp0, tmp1); + ILVRL_H2_SH(sh1, tmp5, tmp2, tmp3); + HADD_SH4_SW(tmp0, tmp1, tmp2, tmp3, s0, s1, s2, s3); + sign0 = (in0 < zero); + sign1 = (in1 < zero); // sign + LD_SH2(&mtx->iq_[0], 8, tmp0, tmp1); // iq + ILVRL_H2_SW(zero, tmp0, t0, t1); + ILVRL_H2_SW(zero, tmp1, t2, t3); + LD_SW4(&mtx->bias_[0], 4, b0, b1, b2, b3); // bias + MUL4(t0, s0, t1, s1, t2, s2, t3, s3, t0, t1, t2, t3); + ADD4(b0, t0, b1, t1, b2, t2, b3, t3, b0, b1, b2, b3); + SRAI_W4_SW(b0, b1, b2, b3, 17); + PCKEV_H2_SH(b1, b0, b3, b2, tmp2, tmp3); + tmp0 = (tmp2 > maxlevel); + tmp1 = (tmp3 > maxlevel); + tmp2 = (v8i16)__msa_bmnz_v((v16u8)tmp2, (v16u8)maxlevel, (v16u8)tmp0); + tmp3 = (v8i16)__msa_bmnz_v((v16u8)tmp3, (v16u8)maxlevel, (v16u8)tmp1); + SUB2(0, tmp2, 0, tmp3, tmp0, tmp1); + tmp2 = (v8i16)__msa_bmnz_v((v16u8)tmp2, (v16u8)tmp0, (v16u8)sign0); + tmp3 = (v8i16)__msa_bmnz_v((v16u8)tmp3, (v16u8)tmp1, (v16u8)sign1); + LD_SW4(&mtx->zthresh_[0], 4, t0, t1, t2, t3); // zthresh + t0 = (s0 > t0); + t1 = (s1 > t1); + t2 = (s2 > t2); + t3 = (s3 > t3); + PCKEV_H2_SH(t1, t0, t3, t2, tmp0, tmp1); + tmp4 = (v8i16)__msa_bmnz_v((v16u8)zero, (v16u8)tmp2, (v16u8)tmp0); + tmp5 = (v8i16)__msa_bmnz_v((v16u8)zero, (v16u8)tmp3, (v16u8)tmp1); + LD_SH2(&mtx->q_[0], 8, tmp0, tmp1); + MUL2(tmp4, tmp0, tmp5, tmp1, in0, in1); + VSHF_H2_SH(tmp4, tmp5, tmp4, tmp5, zigzag0, zigzag1, out0, out1); + ST_SH2(in0, in1, &in[0], 8); + ST_SH2(out0, out1, &out[0], 8); + out0 = __msa_add_a_h(out0, out1); + sum = HADD_SH_S32(out0); + return (sum > 0); +} + +static int Quantize2Blocks(int16_t in[32], int16_t out[32], + const VP8Matrix* const mtx) { + int nz; + nz = VP8EncQuantizeBlock(in + 0 * 16, out + 0 * 16, mtx) << 0; + nz |= VP8EncQuantizeBlock(in + 1 * 16, out + 1 * 16, mtx) << 1; + return nz; +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8EncDspInitMSA(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitMSA(void) { + VP8ITransform = ITransform; + VP8FTransform = FTransform; + VP8FTransformWHT = FTransformWHT; + + VP8TDisto4x4 = Disto4x4; + VP8TDisto16x16 = Disto16x16; + VP8CollectHistogram = CollectHistogram; + + VP8EncPredLuma4 = Intra4Preds; + VP8EncPredLuma16 = Intra16Preds; + VP8EncPredChroma8 = IntraChromaPreds; + + VP8SSE16x16 = SSE16x16; + VP8SSE16x8 = SSE16x8; + VP8SSE8x8 = SSE8x8; + VP8SSE4x4 = SSE4x4; + + VP8EncQuantizeBlock = QuantizeBlock; + VP8EncQuantize2Blocks = Quantize2Blocks; + VP8EncQuantizeBlockWHT = QuantizeBlock; +} + +#else // !WEBP_USE_MSA + +WEBP_DSP_INIT_STUB(VP8EncDspInitMSA) + +#endif // WEBP_USE_MSA diff --git a/src/3rdparty/libwebp/src/dsp/enc_neon.c b/src/3rdparty/libwebp/src/dsp/enc_neon.c index 46f6bf9..6a078d6 100644 --- a/src/3rdparty/libwebp/src/dsp/enc_neon.c +++ b/src/3rdparty/libwebp/src/dsp/enc_neon.c @@ -18,7 +18,7 @@ #include <assert.h> #include "./neon.h" -#include "../enc/vp8enci.h" +#include "../enc/vp8i_enc.h" //------------------------------------------------------------------------------ // Transforms (Paragraph 14.4) @@ -746,9 +746,14 @@ static WEBP_INLINE void AccumulateSSE16(const uint8_t* const a, const uint8x16_t a0 = vld1q_u8(a); const uint8x16_t b0 = vld1q_u8(b); const uint8x16_t abs_diff = vabdq_u8(a0, b0); - uint16x8_t prod = vmull_u8(vget_low_u8(abs_diff), vget_low_u8(abs_diff)); - prod = vmlal_u8(prod, vget_high_u8(abs_diff), vget_high_u8(abs_diff)); - *sum = vpadalq_u16(*sum, prod); // pair-wise add and accumulate + const uint16x8_t prod1 = vmull_u8(vget_low_u8(abs_diff), + vget_low_u8(abs_diff)); + const uint16x8_t prod2 = vmull_u8(vget_high_u8(abs_diff), + vget_high_u8(abs_diff)); + /* pair-wise adds and widen */ + const uint32x4_t sum1 = vpaddlq_u16(prod1); + const uint32x4_t sum2 = vpaddlq_u16(prod2); + *sum = vaddq_u32(*sum, vaddq_u32(sum1, sum2)); } // Horizontal sum of all four uint32_t values in 'sum'. @@ -758,7 +763,7 @@ static int SumToInt(uint32x4_t sum) { return (int)sum3; } -static int SSE16x16(const uint8_t* a, const uint8_t* b) { +static int SSE16x16_NEON(const uint8_t* a, const uint8_t* b) { uint32x4_t sum = vdupq_n_u32(0); int y; for (y = 0; y < 16; ++y) { @@ -767,7 +772,7 @@ static int SSE16x16(const uint8_t* a, const uint8_t* b) { return SumToInt(sum); } -static int SSE16x8(const uint8_t* a, const uint8_t* b) { +static int SSE16x8_NEON(const uint8_t* a, const uint8_t* b) { uint32x4_t sum = vdupq_n_u32(0); int y; for (y = 0; y < 8; ++y) { @@ -776,7 +781,7 @@ static int SSE16x8(const uint8_t* a, const uint8_t* b) { return SumToInt(sum); } -static int SSE8x8(const uint8_t* a, const uint8_t* b) { +static int SSE8x8_NEON(const uint8_t* a, const uint8_t* b) { uint32x4_t sum = vdupq_n_u32(0); int y; for (y = 0; y < 8; ++y) { @@ -789,13 +794,18 @@ static int SSE8x8(const uint8_t* a, const uint8_t* b) { return SumToInt(sum); } -static int SSE4x4(const uint8_t* a, const uint8_t* b) { +static int SSE4x4_NEON(const uint8_t* a, const uint8_t* b) { const uint8x16_t a0 = Load4x4(a); const uint8x16_t b0 = Load4x4(b); const uint8x16_t abs_diff = vabdq_u8(a0, b0); - uint16x8_t prod = vmull_u8(vget_low_u8(abs_diff), vget_low_u8(abs_diff)); - prod = vmlal_u8(prod, vget_high_u8(abs_diff), vget_high_u8(abs_diff)); - return SumToInt(vpaddlq_u16(prod)); + const uint16x8_t prod1 = vmull_u8(vget_low_u8(abs_diff), + vget_low_u8(abs_diff)); + const uint16x8_t prod2 = vmull_u8(vget_high_u8(abs_diff), + vget_high_u8(abs_diff)); + /* pair-wise adds and widen */ + const uint32x4_t sum1 = vpaddlq_u16(prod1); + const uint32x4_t sum2 = vpaddlq_u16(prod2); + return SumToInt(vaddq_u32(sum1, sum2)); } //------------------------------------------------------------------------------ @@ -903,10 +913,12 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitNEON(void) { VP8TDisto4x4 = Disto4x4; VP8TDisto16x16 = Disto16x16; VP8CollectHistogram = CollectHistogram; - VP8SSE16x16 = SSE16x16; - VP8SSE16x8 = SSE16x8; - VP8SSE8x8 = SSE8x8; - VP8SSE4x4 = SSE4x4; + + VP8SSE16x16 = SSE16x16_NEON; + VP8SSE16x8 = SSE16x8_NEON; + VP8SSE8x8 = SSE8x8_NEON; + VP8SSE4x4 = SSE4x4_NEON; + #if !defined(WORK_AROUND_GCC) VP8EncQuantizeBlock = QuantizeBlock; VP8EncQuantize2Blocks = Quantize2Blocks; diff --git a/src/3rdparty/libwebp/src/dsp/enc_sse2.c b/src/3rdparty/libwebp/src/dsp/enc_sse2.c index 4a2e3ce..2026a74 100644 --- a/src/3rdparty/libwebp/src/dsp/enc_sse2.c +++ b/src/3rdparty/libwebp/src/dsp/enc_sse2.c @@ -14,12 +14,13 @@ #include "./dsp.h" #if defined(WEBP_USE_SSE2) +#include <assert.h> #include <stdlib.h> // for abs() #include <emmintrin.h> #include "./common_sse2.h" -#include "../enc/cost.h" -#include "../enc/vp8enci.h" +#include "../enc/cost_enc.h" +#include "../enc/vp8i_enc.h" //------------------------------------------------------------------------------ // Transforms (Paragraph 14.4) @@ -139,7 +140,7 @@ static void ITransform(const uint8_t* ref, const int16_t* in, uint8_t* dst, // Transpose the two 4x4. VP8Transpose_2_4x4_16b(&shifted0, &shifted1, &shifted2, &shifted3, &T0, &T1, - &T2, &T3); + &T2, &T3); } // Add inverse transform to 'ref' and store. @@ -250,25 +251,11 @@ static void FTransformPass2(const __m128i* const v01, const __m128i* const v32, const __m128i k51000 = _mm_set1_epi32(51000); // Same operations are done on the (0,3) and (1,2) pairs. - // a0 = v0 + v3 - // a1 = v1 + v2 // a3 = v0 - v3 // a2 = v1 - v2 - const __m128i a01 = _mm_add_epi16(*v01, *v32); const __m128i a32 = _mm_sub_epi16(*v01, *v32); - const __m128i a11 = _mm_unpackhi_epi64(a01, a01); const __m128i a22 = _mm_unpackhi_epi64(a32, a32); - const __m128i a01_plus_7 = _mm_add_epi16(a01, seven); - // d0 = (a0 + a1 + 7) >> 4; - // d2 = (a0 - a1 + 7) >> 4; - const __m128i c0 = _mm_add_epi16(a01_plus_7, a11); - const __m128i c2 = _mm_sub_epi16(a01_plus_7, a11); - const __m128i d0 = _mm_srai_epi16(c0, 4); - const __m128i d2 = _mm_srai_epi16(c2, 4); - - // f1 = ((b3 * 5352 + b2 * 2217 + 12000) >> 16) - // f3 = ((b3 * 2217 - b2 * 5352 + 51000) >> 16) const __m128i b23 = _mm_unpacklo_epi16(a22, a32); const __m128i c1 = _mm_madd_epi16(b23, k5352_2217); const __m128i c3 = _mm_madd_epi16(b23, k2217_5352); @@ -276,14 +263,28 @@ static void FTransformPass2(const __m128i* const v01, const __m128i* const v32, const __m128i d3 = _mm_add_epi32(c3, k51000); const __m128i e1 = _mm_srai_epi32(d1, 16); const __m128i e3 = _mm_srai_epi32(d3, 16); + // f1 = ((b3 * 5352 + b2 * 2217 + 12000) >> 16) + // f3 = ((b3 * 2217 - b2 * 5352 + 51000) >> 16) const __m128i f1 = _mm_packs_epi32(e1, e1); const __m128i f3 = _mm_packs_epi32(e3, e3); - // f1 = f1 + (a3 != 0); + // g1 = f1 + (a3 != 0); // The compare will return (0xffff, 0) for (==0, !=0). To turn that into the // desired (0, 1), we add one earlier through k12000_plus_one. - // -> f1 = f1 + 1 - (a3 == 0) + // -> g1 = f1 + 1 - (a3 == 0) const __m128i g1 = _mm_add_epi16(f1, _mm_cmpeq_epi16(a32, zero)); + // a0 = v0 + v3 + // a1 = v1 + v2 + const __m128i a01 = _mm_add_epi16(*v01, *v32); + const __m128i a01_plus_7 = _mm_add_epi16(a01, seven); + const __m128i a11 = _mm_unpackhi_epi64(a01, a01); + const __m128i c0 = _mm_add_epi16(a01_plus_7, a11); + const __m128i c2 = _mm_sub_epi16(a01_plus_7, a11); + // d0 = (a0 + a1 + 7) >> 4; + // d2 = (a0 - a1 + 7) >> 4; + const __m128i d0 = _mm_srai_epi16(c0, 4); + const __m128i d2 = _mm_srai_epi16(c2, 4); + const __m128i d0_g1 = _mm_unpacklo_epi64(d0, g1); const __m128i d2_f3 = _mm_unpacklo_epi64(d2, f3); _mm_storeu_si128((__m128i*)&out[0], d0_g1); @@ -1046,6 +1047,37 @@ static int SSE4x4(const uint8_t* a, const uint8_t* b) { } //------------------------------------------------------------------------------ + +static void Mean16x4(const uint8_t* ref, uint32_t dc[4]) { + const __m128i mask = _mm_set1_epi16(0x00ff); + const __m128i a0 = _mm_loadu_si128((const __m128i*)&ref[BPS * 0]); + const __m128i a1 = _mm_loadu_si128((const __m128i*)&ref[BPS * 1]); + const __m128i a2 = _mm_loadu_si128((const __m128i*)&ref[BPS * 2]); + const __m128i a3 = _mm_loadu_si128((const __m128i*)&ref[BPS * 3]); + const __m128i b0 = _mm_srli_epi16(a0, 8); // hi byte + const __m128i b1 = _mm_srli_epi16(a1, 8); + const __m128i b2 = _mm_srli_epi16(a2, 8); + const __m128i b3 = _mm_srli_epi16(a3, 8); + const __m128i c0 = _mm_and_si128(a0, mask); // lo byte + const __m128i c1 = _mm_and_si128(a1, mask); + const __m128i c2 = _mm_and_si128(a2, mask); + const __m128i c3 = _mm_and_si128(a3, mask); + const __m128i d0 = _mm_add_epi32(b0, c0); + const __m128i d1 = _mm_add_epi32(b1, c1); + const __m128i d2 = _mm_add_epi32(b2, c2); + const __m128i d3 = _mm_add_epi32(b3, c3); + const __m128i e0 = _mm_add_epi32(d0, d1); + const __m128i e1 = _mm_add_epi32(d2, d3); + const __m128i f0 = _mm_add_epi32(e0, e1); + uint16_t tmp[8]; + _mm_storeu_si128((__m128i*)tmp, f0); + dc[0] = tmp[0] + tmp[1]; + dc[1] = tmp[2] + tmp[3]; + dc[2] = tmp[4] + tmp[5]; + dc[3] = tmp[6] + tmp[7]; +} + +//------------------------------------------------------------------------------ // Texture distortion // // We try to match the spectral content (weighted) between source and @@ -1331,10 +1363,122 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitSSE2(void) { VP8SSE4x4 = SSE4x4; VP8TDisto4x4 = Disto4x4; VP8TDisto16x16 = Disto16x16; + VP8Mean16x4 = Mean16x4; +} + +//------------------------------------------------------------------------------ +// SSIM / PSNR entry point (TODO(skal): move to its own file later) + +static uint32_t AccumulateSSE_SSE2(const uint8_t* src1, + const uint8_t* src2, int len) { + int i = 0; + uint32_t sse2 = 0; + if (len >= 16) { + const int limit = len - 32; + int32_t tmp[4]; + __m128i sum1; + __m128i sum = _mm_setzero_si128(); + __m128i a0 = _mm_loadu_si128((const __m128i*)&src1[i]); + __m128i b0 = _mm_loadu_si128((const __m128i*)&src2[i]); + i += 16; + while (i <= limit) { + const __m128i a1 = _mm_loadu_si128((const __m128i*)&src1[i]); + const __m128i b1 = _mm_loadu_si128((const __m128i*)&src2[i]); + __m128i sum2; + i += 16; + SubtractAndAccumulate(a0, b0, &sum1); + sum = _mm_add_epi32(sum, sum1); + a0 = _mm_loadu_si128((const __m128i*)&src1[i]); + b0 = _mm_loadu_si128((const __m128i*)&src2[i]); + i += 16; + SubtractAndAccumulate(a1, b1, &sum2); + sum = _mm_add_epi32(sum, sum2); + } + SubtractAndAccumulate(a0, b0, &sum1); + sum = _mm_add_epi32(sum, sum1); + _mm_storeu_si128((__m128i*)tmp, sum); + sse2 += (tmp[3] + tmp[2] + tmp[1] + tmp[0]); + } + + for (; i < len; ++i) { + const int32_t diff = src1[i] - src2[i]; + sse2 += diff * diff; + } + return sse2; +} + +static uint32_t HorizontalAdd16b(const __m128i* const m) { + uint16_t tmp[8]; + const __m128i a = _mm_srli_si128(*m, 8); + const __m128i b = _mm_add_epi16(*m, a); + _mm_storeu_si128((__m128i*)tmp, b); + return (uint32_t)tmp[3] + tmp[2] + tmp[1] + tmp[0]; +} + +static uint32_t HorizontalAdd32b(const __m128i* const m) { + const __m128i a = _mm_srli_si128(*m, 8); + const __m128i b = _mm_add_epi32(*m, a); + const __m128i c = _mm_add_epi32(b, _mm_srli_si128(b, 4)); + return (uint32_t)_mm_cvtsi128_si32(c); +} + +static const uint16_t kWeight[] = { 1, 2, 3, 4, 3, 2, 1, 0 }; + +#define ACCUMULATE_ROW(WEIGHT) do { \ + /* compute row weight (Wx * Wy) */ \ + const __m128i Wy = _mm_set1_epi16((WEIGHT)); \ + const __m128i W = _mm_mullo_epi16(Wx, Wy); \ + /* process 8 bytes at a time (7 bytes, actually) */ \ + const __m128i a0 = _mm_loadl_epi64((const __m128i*)src1); \ + const __m128i b0 = _mm_loadl_epi64((const __m128i*)src2); \ + /* convert to 16b and multiply by weight */ \ + const __m128i a1 = _mm_unpacklo_epi8(a0, zero); \ + const __m128i b1 = _mm_unpacklo_epi8(b0, zero); \ + const __m128i wa1 = _mm_mullo_epi16(a1, W); \ + const __m128i wb1 = _mm_mullo_epi16(b1, W); \ + /* accumulate */ \ + xm = _mm_add_epi16(xm, wa1); \ + ym = _mm_add_epi16(ym, wb1); \ + xxm = _mm_add_epi32(xxm, _mm_madd_epi16(a1, wa1)); \ + xym = _mm_add_epi32(xym, _mm_madd_epi16(a1, wb1)); \ + yym = _mm_add_epi32(yym, _mm_madd_epi16(b1, wb1)); \ + src1 += stride1; \ + src2 += stride2; \ +} while (0) + +static double SSIMGet_SSE2(const uint8_t* src1, int stride1, + const uint8_t* src2, int stride2) { + VP8DistoStats stats; + const __m128i zero = _mm_setzero_si128(); + __m128i xm = zero, ym = zero; // 16b accums + __m128i xxm = zero, yym = zero, xym = zero; // 32b accum + const __m128i Wx = _mm_loadu_si128((const __m128i*)kWeight); + assert(2 * VP8_SSIM_KERNEL + 1 == 7); + ACCUMULATE_ROW(1); + ACCUMULATE_ROW(2); + ACCUMULATE_ROW(3); + ACCUMULATE_ROW(4); + ACCUMULATE_ROW(3); + ACCUMULATE_ROW(2); + ACCUMULATE_ROW(1); + stats.xm = HorizontalAdd16b(&xm); + stats.ym = HorizontalAdd16b(&ym); + stats.xxm = HorizontalAdd32b(&xxm); + stats.xym = HorizontalAdd32b(&xym); + stats.yym = HorizontalAdd32b(&yym); + return VP8SSIMFromStats(&stats); +} + +extern void VP8SSIMDspInitSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8SSIMDspInitSSE2(void) { + VP8AccumulateSSE = AccumulateSSE_SSE2; + VP8SSIMGet = SSIMGet_SSE2; } #else // !WEBP_USE_SSE2 WEBP_DSP_INIT_STUB(VP8EncDspInitSSE2) +WEBP_DSP_INIT_STUB(VP8SSIMDspInitSSE2) #endif // WEBP_USE_SSE2 diff --git a/src/3rdparty/libwebp/src/dsp/enc_sse41.c b/src/3rdparty/libwebp/src/dsp/enc_sse41.c index a178390..e32086d 100644 --- a/src/3rdparty/libwebp/src/dsp/enc_sse41.c +++ b/src/3rdparty/libwebp/src/dsp/enc_sse41.c @@ -18,7 +18,7 @@ #include <stdlib.h> // for abs() #include "./common_sse2.h" -#include "../enc/vp8enci.h" +#include "../enc/vp8i_enc.h" //------------------------------------------------------------------------------ // Compute susceptibility based on DCT-coeff histograms. diff --git a/src/3rdparty/libwebp/src/dsp/filters.c b/src/3rdparty/libwebp/src/dsp/filters.c index 9f04faf..65f34aa 100644 --- a/src/3rdparty/libwebp/src/dsp/filters.c +++ b/src/3rdparty/libwebp/src/dsp/filters.c @@ -227,6 +227,8 @@ WebPFilterFunc WebPFilters[WEBP_FILTER_LAST]; WebPUnfilterFunc WebPUnfilters[WEBP_FILTER_LAST]; extern void VP8FiltersInitMIPSdspR2(void); +extern void VP8FiltersInitMSA(void); +extern void VP8FiltersInitNEON(void); extern void VP8FiltersInitSSE2(void); static volatile VP8CPUInfo filters_last_cpuinfo_used = @@ -251,11 +253,21 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInit(void) { VP8FiltersInitSSE2(); } #endif +#if defined(WEBP_USE_NEON) + if (VP8GetCPUInfo(kNEON)) { + VP8FiltersInitNEON(); + } +#endif #if defined(WEBP_USE_MIPS_DSP_R2) if (VP8GetCPUInfo(kMIPSdspR2)) { VP8FiltersInitMIPSdspR2(); } #endif +#if defined(WEBP_USE_MSA) + if (VP8GetCPUInfo(kMSA)) { + VP8FiltersInitMSA(); + } +#endif } filters_last_cpuinfo_used = VP8GetCPUInfo; } diff --git a/src/3rdparty/libwebp/src/dsp/filters_msa.c b/src/3rdparty/libwebp/src/dsp/filters_msa.c new file mode 100644 index 0000000..4b8922d --- /dev/null +++ b/src/3rdparty/libwebp/src/dsp/filters_msa.c @@ -0,0 +1,202 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MSA variant of alpha filters +// +// Author: Prashant Patil (prashant.patil@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MSA) + +#include "./msa_macro.h" + +#include <assert.h> + +static WEBP_INLINE void PredictLineInverse0(const uint8_t* src, + const uint8_t* pred, + uint8_t* dst, int length) { + v16u8 src0, pred0, dst0; + assert(length >= 0); + while (length >= 32) { + v16u8 src1, pred1, dst1; + LD_UB2(src, 16, src0, src1); + LD_UB2(pred, 16, pred0, pred1); + SUB2(src0, pred0, src1, pred1, dst0, dst1); + ST_UB2(dst0, dst1, dst, 16); + src += 32; + pred += 32; + dst += 32; + length -= 32; + } + if (length > 0) { + int i; + if (length >= 16) { + src0 = LD_UB(src); + pred0 = LD_UB(pred); + dst0 = src0 - pred0; + ST_UB(dst0, dst); + src += 16; + pred += 16; + dst += 16; + length -= 16; + } + for (i = 0; i < length; i++) { + dst[i] = src[i] - pred[i]; + } + } +} + +//------------------------------------------------------------------------------ +// Helpful macro. + +#define SANITY_CHECK(in, out) \ + assert(in != NULL); \ + assert(out != NULL); \ + assert(width > 0); \ + assert(height > 0); \ + assert(stride >= width); + +//------------------------------------------------------------------------------ +// Horrizontal filter + +static void HorizontalFilter(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + const uint8_t* preds = data; + const uint8_t* in = data; + uint8_t* out = filtered_data; + int row = 1; + SANITY_CHECK(in, out); + + // Leftmost pixel is the same as input for topmost scanline. + out[0] = in[0]; + PredictLineInverse0(in + 1, preds, out + 1, width - 1); + preds += stride; + in += stride; + out += stride; + // Filter line-by-line. + while (row < height) { + // Leftmost pixel is predicted from above. + PredictLineInverse0(in, preds - stride, out, 1); + PredictLineInverse0(in + 1, preds, out + 1, width - 1); + ++row; + preds += stride; + in += stride; + out += stride; + } +} + +//------------------------------------------------------------------------------ +// Gradient filter + +static WEBP_INLINE void PredictLineGradient(const uint8_t* pinput, + const uint8_t* ppred, + uint8_t* poutput, int stride, + int size) { + int w; + const v16i8 zero = { 0 }; + while (size >= 16) { + v16u8 pred0, dst0; + v8i16 a0, a1, b0, b1, c0, c1; + const v16u8 tmp0 = LD_UB(ppred - 1); + const v16u8 tmp1 = LD_UB(ppred - stride); + const v16u8 tmp2 = LD_UB(ppred - stride - 1); + const v16u8 src0 = LD_UB(pinput); + ILVRL_B2_SH(zero, tmp0, a0, a1); + ILVRL_B2_SH(zero, tmp1, b0, b1); + ILVRL_B2_SH(zero, tmp2, c0, c1); + ADD2(a0, b0, a1, b1, a0, a1); + SUB2(a0, c0, a1, c1, a0, a1); + CLIP_SH2_0_255(a0, a1); + pred0 = (v16u8)__msa_pckev_b((v16i8)a1, (v16i8)a0); + dst0 = src0 - pred0; + ST_UB(dst0, poutput); + ppred += 16; + pinput += 16; + poutput += 16; + size -= 16; + } + for (w = 0; w < size; ++w) { + const int pred = ppred[w - 1] + ppred[w - stride] - ppred[w - stride - 1]; + poutput[w] = pinput[w] - (pred < 0 ? 0 : pred > 255 ? 255 : pred); + } +} + + +static void GradientFilter(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + const uint8_t* in = data; + const uint8_t* preds = data; + uint8_t* out = filtered_data; + int row = 1; + SANITY_CHECK(in, out); + + // left prediction for top scan-line + out[0] = in[0]; + PredictLineInverse0(in + 1, preds, out + 1, width - 1); + preds += stride; + in += stride; + out += stride; + // Filter line-by-line. + while (row < height) { + out[0] = in[0] - preds[- stride]; + PredictLineGradient(preds + 1, in + 1, out + 1, stride, width - 1); + ++row; + preds += stride; + in += stride; + out += stride; + } +} + +//------------------------------------------------------------------------------ +// Vertical filter + +static void VerticalFilter(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + const uint8_t* in = data; + const uint8_t* preds = data; + uint8_t* out = filtered_data; + int row = 1; + SANITY_CHECK(in, out); + + // Very first top-left pixel is copied. + out[0] = in[0]; + // Rest of top scan-line is left-predicted. + PredictLineInverse0(in + 1, preds, out + 1, width - 1); + in += stride; + out += stride; + + // Filter line-by-line. + while (row < height) { + PredictLineInverse0(in, preds, out, width); + ++row; + preds += stride; + in += stride; + out += stride; + } +} + +#undef SANITY_CHECK + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8FiltersInitMSA(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInitMSA(void) { + WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter; + WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter; + WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter; +} + +#else // !WEBP_USE_MSA + +WEBP_DSP_INIT_STUB(VP8FiltersInitMSA) + +#endif // WEBP_USE_MSA diff --git a/src/3rdparty/libwebp/src/dsp/filters_neon.c b/src/3rdparty/libwebp/src/dsp/filters_neon.c new file mode 100644 index 0000000..4d6e50c --- /dev/null +++ b/src/3rdparty/libwebp/src/dsp/filters_neon.c @@ -0,0 +1,327 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// NEON variant of alpha filters +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_NEON) + +#include <assert.h> +#include "./neon.h" + +//------------------------------------------------------------------------------ +// Helpful macros. + +# define SANITY_CHECK(in, out) \ + assert(in != NULL); \ + assert(out != NULL); \ + assert(width > 0); \ + assert(height > 0); \ + assert(stride >= width); \ + assert(row >= 0 && num_rows > 0 && row + num_rows <= height); \ + (void)height; // Silence unused warning. + +// load eight u8 and widen to s16 +#define U8_TO_S16(A) vreinterpretq_s16_u16(vmovl_u8(A)) +#define LOAD_U8_TO_S16(A) U8_TO_S16(vld1_u8(A)) + +// shift left or right by N byte, inserting zeros +#define SHIFT_RIGHT_N_Q(A, N) vextq_u8((A), zero, (N)) +#define SHIFT_LEFT_N_Q(A, N) vextq_u8(zero, (A), (16 - (N)) % 16) + +// rotate left by N bytes +#define ROTATE_LEFT_N(A, N) vext_u8((A), (A), (N)) +// rotate right by N bytes +#define ROTATE_RIGHT_N(A, N) vext_u8((A), (A), (8 - (N)) % 8) + +static void PredictLine_NEON(const uint8_t* src, const uint8_t* pred, + uint8_t* dst, int length) { + int i; + assert(length >= 0); + for (i = 0; i + 16 <= length; i += 16) { + const uint8x16_t A = vld1q_u8(&src[i]); + const uint8x16_t B = vld1q_u8(&pred[i]); + const uint8x16_t C = vsubq_u8(A, B); + vst1q_u8(&dst[i], C); + } + for (; i < length; ++i) dst[i] = src[i] - pred[i]; +} + +// Special case for left-based prediction (when preds==dst-1 or preds==src-1). +static void PredictLineLeft_NEON(const uint8_t* src, uint8_t* dst, int length) { + PredictLine_NEON(src, src - 1, dst, length); +} + +//------------------------------------------------------------------------------ +// Horizontal filter. + +static WEBP_INLINE void DoHorizontalFilter_NEON(const uint8_t* in, + int width, int height, + int stride, + int row, int num_rows, + uint8_t* out) { + const size_t start_offset = row * stride; + const int last_row = row + num_rows; + SANITY_CHECK(in, out); + in += start_offset; + out += start_offset; + + if (row == 0) { + // Leftmost pixel is the same as input for topmost scanline. + out[0] = in[0]; + PredictLineLeft_NEON(in + 1, out + 1, width - 1); + row = 1; + in += stride; + out += stride; + } + + // Filter line-by-line. + while (row < last_row) { + // Leftmost pixel is predicted from above. + out[0] = in[0] - in[-stride]; + PredictLineLeft_NEON(in + 1, out + 1, width - 1); + ++row; + in += stride; + out += stride; + } +} + +static void HorizontalFilter_NEON(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + DoHorizontalFilter_NEON(data, width, height, stride, 0, height, + filtered_data); +} + +//------------------------------------------------------------------------------ +// Vertical filter. + +static WEBP_INLINE void DoVerticalFilter_NEON(const uint8_t* in, + int width, int height, int stride, + int row, int num_rows, + uint8_t* out) { + const size_t start_offset = row * stride; + const int last_row = row + num_rows; + SANITY_CHECK(in, out); + in += start_offset; + out += start_offset; + + if (row == 0) { + // Very first top-left pixel is copied. + out[0] = in[0]; + // Rest of top scan-line is left-predicted. + PredictLineLeft_NEON(in + 1, out + 1, width - 1); + row = 1; + in += stride; + out += stride; + } + + // Filter line-by-line. + while (row < last_row) { + PredictLine_NEON(in, in - stride, out, width); + ++row; + in += stride; + out += stride; + } +} + +static void VerticalFilter_NEON(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + DoVerticalFilter_NEON(data, width, height, stride, 0, height, + filtered_data); +} + +//------------------------------------------------------------------------------ +// Gradient filter. + +static WEBP_INLINE int GradientPredictor_C(uint8_t a, uint8_t b, uint8_t c) { + const int g = a + b - c; + return ((g & ~0xff) == 0) ? g : (g < 0) ? 0 : 255; // clip to 8bit +} + +static void GradientPredictDirect_NEON(const uint8_t* const row, + const uint8_t* const top, + uint8_t* const out, int length) { + int i; + for (i = 0; i + 8 <= length; i += 8) { + const uint8x8_t A = vld1_u8(&row[i - 1]); + const uint8x8_t B = vld1_u8(&top[i + 0]); + const int16x8_t C = vreinterpretq_s16_u16(vaddl_u8(A, B)); + const int16x8_t D = LOAD_U8_TO_S16(&top[i - 1]); + const uint8x8_t E = vqmovun_s16(vsubq_s16(C, D)); + const uint8x8_t F = vld1_u8(&row[i + 0]); + vst1_u8(&out[i], vsub_u8(F, E)); + } + for (; i < length; ++i) { + out[i] = row[i] - GradientPredictor_C(row[i - 1], top[i], top[i - 1]); + } +} + +static WEBP_INLINE void DoGradientFilter_NEON(const uint8_t* in, + int width, int height, + int stride, + int row, int num_rows, + uint8_t* out) { + const size_t start_offset = row * stride; + const int last_row = row + num_rows; + SANITY_CHECK(in, out); + in += start_offset; + out += start_offset; + + // left prediction for top scan-line + if (row == 0) { + out[0] = in[0]; + PredictLineLeft_NEON(in + 1, out + 1, width - 1); + row = 1; + in += stride; + out += stride; + } + + // Filter line-by-line. + while (row < last_row) { + out[0] = in[0] - in[-stride]; + GradientPredictDirect_NEON(in + 1, in + 1 - stride, out + 1, width - 1); + ++row; + in += stride; + out += stride; + } +} + +static void GradientFilter_NEON(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + DoGradientFilter_NEON(data, width, height, stride, 0, height, + filtered_data); +} + +#undef SANITY_CHECK + +//------------------------------------------------------------------------------ +// Inverse transforms + +static void HorizontalUnfilter_NEON(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + int i; + const uint8x16_t zero = vdupq_n_u8(0); + uint8x16_t last; + out[0] = in[0] + (prev == NULL ? 0 : prev[0]); + if (width <= 1) return; + last = vsetq_lane_u8(out[0], zero, 0); + for (i = 1; i + 16 <= width; i += 16) { + const uint8x16_t A0 = vld1q_u8(&in[i]); + const uint8x16_t A1 = vaddq_u8(A0, last); + const uint8x16_t A2 = SHIFT_LEFT_N_Q(A1, 1); + const uint8x16_t A3 = vaddq_u8(A1, A2); + const uint8x16_t A4 = SHIFT_LEFT_N_Q(A3, 2); + const uint8x16_t A5 = vaddq_u8(A3, A4); + const uint8x16_t A6 = SHIFT_LEFT_N_Q(A5, 4); + const uint8x16_t A7 = vaddq_u8(A5, A6); + const uint8x16_t A8 = SHIFT_LEFT_N_Q(A7, 8); + const uint8x16_t A9 = vaddq_u8(A7, A8); + vst1q_u8(&out[i], A9); + last = SHIFT_RIGHT_N_Q(A9, 15); + } + for (; i < width; ++i) out[i] = in[i] + out[i - 1]; +} + +static void VerticalUnfilter_NEON(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + if (prev == NULL) { + HorizontalUnfilter_NEON(NULL, in, out, width); + } else { + int i; + assert(width >= 0); + for (i = 0; i + 16 <= width; i += 16) { + const uint8x16_t A = vld1q_u8(&in[i]); + const uint8x16_t B = vld1q_u8(&prev[i]); + const uint8x16_t C = vaddq_u8(A, B); + vst1q_u8(&out[i], C); + } + for (; i < width; ++i) out[i] = in[i] + prev[i]; + } +} + +// GradientUnfilter_NEON is correct but slower than the C-version, +// at least on ARM64. For armv7, it's a wash. +// So best is to disable it for now, but keep the idea around... +// #define USE_GRADIENT_UNFILTER + +#if defined(USE_GRADIENT_UNFILTER) +#define GRAD_PROCESS_LANE(L) do { \ + const uint8x8_t tmp1 = ROTATE_RIGHT_N(pred, 1); /* rotate predictor in */ \ + const int16x8_t tmp2 = vaddq_s16(BC, U8_TO_S16(tmp1)); \ + const uint8x8_t delta = vqmovun_s16(tmp2); \ + pred = vadd_u8(D, delta); \ + out = vext_u8(out, ROTATE_LEFT_N(pred, (L)), 1); \ +} while (0) + +static void GradientPredictInverse_NEON(const uint8_t* const in, + const uint8_t* const top, + uint8_t* const row, int length) { + if (length > 0) { + int i; + uint8x8_t pred = vdup_n_u8(row[-1]); // left sample + uint8x8_t out = vdup_n_u8(0); + for (i = 0; i + 8 <= length; i += 8) { + const int16x8_t B = LOAD_U8_TO_S16(&top[i + 0]); + const int16x8_t C = LOAD_U8_TO_S16(&top[i - 1]); + const int16x8_t BC = vsubq_s16(B, C); // unclipped gradient basis B - C + const uint8x8_t D = vld1_u8(&in[i]); // base input + GRAD_PROCESS_LANE(0); + GRAD_PROCESS_LANE(1); + GRAD_PROCESS_LANE(2); + GRAD_PROCESS_LANE(3); + GRAD_PROCESS_LANE(4); + GRAD_PROCESS_LANE(5); + GRAD_PROCESS_LANE(6); + GRAD_PROCESS_LANE(7); + vst1_u8(&row[i], out); + } + for (; i < length; ++i) { + row[i] = in[i] + GradientPredictor_C(row[i - 1], top[i], top[i - 1]); + } + } +} +#undef GRAD_PROCESS_LANE + +static void GradientUnfilter_NEON(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + if (prev == NULL) { + HorizontalUnfilter_NEON(NULL, in, out, width); + } else { + out[0] = in[0] + prev[0]; // predict from above + GradientPredictInverse_NEON(in + 1, prev + 1, out + 1, width - 1); + } +} + +#endif // USE_GRADIENT_UNFILTER + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8FiltersInitNEON(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInitNEON(void) { + WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter_NEON; + WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter_NEON; +#if defined(USE_GRADIENT_UNFILTER) + WebPUnfilters[WEBP_FILTER_GRADIENT] = GradientUnfilter_NEON; +#endif + + WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter_NEON; + WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter_NEON; + WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter_NEON; +} + +#else // !WEBP_USE_NEON + +WEBP_DSP_INIT_STUB(VP8FiltersInitNEON) + +#endif // WEBP_USE_NEON diff --git a/src/3rdparty/libwebp/src/dsp/lossless.c b/src/3rdparty/libwebp/src/dsp/lossless.c index af913ef..20d18f6 100644 --- a/src/3rdparty/libwebp/src/dsp/lossless.c +++ b/src/3rdparty/libwebp/src/dsp/lossless.c @@ -17,20 +17,16 @@ #include <math.h> #include <stdlib.h> -#include "../dec/vp8li.h" -#include "../utils/endian_inl.h" +#include "../dec/vp8li_dec.h" +#include "../utils/endian_inl_utils.h" #include "./lossless.h" +#include "./lossless_common.h" #define MAX_DIFF_COST (1e30f) //------------------------------------------------------------------------------ // Image transforms. -// In-place sum of each component with mod 256. -static WEBP_INLINE void AddPixelsEq(uint32_t* a, uint32_t b) { - *a = VP8LAddPixels(*a, b); -} - static WEBP_INLINE uint32_t Average2(uint32_t a0, uint32_t a1) { return (((a0 ^ a1) & 0xfefefefeu) >> 1) + (a0 & a1); } @@ -171,21 +167,41 @@ static uint32_t Predictor13(uint32_t left, const uint32_t* const top) { return pred; } +GENERATE_PREDICTOR_ADD(Predictor0, PredictorAdd0) +static void PredictorAdd1(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + uint32_t left = out[-1]; + for (i = 0; i < num_pixels; ++i) { + out[i] = left = VP8LAddPixels(in[i], left); + } + (void)upper; +} +GENERATE_PREDICTOR_ADD(Predictor2, PredictorAdd2) +GENERATE_PREDICTOR_ADD(Predictor3, PredictorAdd3) +GENERATE_PREDICTOR_ADD(Predictor4, PredictorAdd4) +GENERATE_PREDICTOR_ADD(Predictor5, PredictorAdd5) +GENERATE_PREDICTOR_ADD(Predictor6, PredictorAdd6) +GENERATE_PREDICTOR_ADD(Predictor7, PredictorAdd7) +GENERATE_PREDICTOR_ADD(Predictor8, PredictorAdd8) +GENERATE_PREDICTOR_ADD(Predictor9, PredictorAdd9) +GENERATE_PREDICTOR_ADD(Predictor10, PredictorAdd10) +GENERATE_PREDICTOR_ADD(Predictor11, PredictorAdd11) +GENERATE_PREDICTOR_ADD(Predictor12, PredictorAdd12) +GENERATE_PREDICTOR_ADD(Predictor13, PredictorAdd13) + //------------------------------------------------------------------------------ // Inverse prediction. static void PredictorInverseTransform(const VP8LTransform* const transform, - int y_start, int y_end, uint32_t* data) { + int y_start, int y_end, + const uint32_t* in, uint32_t* out) { const int width = transform->xsize_; if (y_start == 0) { // First Row follows the L (mode=1) mode. - int x; - const uint32_t pred0 = Predictor0(data[-1], NULL); - AddPixelsEq(data, pred0); - for (x = 1; x < width; ++x) { - const uint32_t pred1 = Predictor1(data[x - 1], NULL); - AddPixelsEq(data + x, pred1); - } - data += width; + PredictorAdd0(in, NULL, 1, out); + PredictorAdd1(in + 1, NULL, width - 1, out + 1); + in += width; + out += width; ++y_start; } @@ -193,36 +209,26 @@ static void PredictorInverseTransform(const VP8LTransform* const transform, int y = y_start; const int tile_width = 1 << transform->bits_; const int mask = tile_width - 1; - const int safe_width = width & ~mask; const int tiles_per_row = VP8LSubSampleSize(width, transform->bits_); const uint32_t* pred_mode_base = transform->data_ + (y >> transform->bits_) * tiles_per_row; while (y < y_end) { - const uint32_t pred2 = Predictor2(data[-1], data - width); const uint32_t* pred_mode_src = pred_mode_base; - VP8LPredictorFunc pred_func; int x = 1; - int t = 1; // First pixel follows the T (mode=2) mode. - AddPixelsEq(data, pred2); + PredictorAdd2(in, out - width, 1, out); // .. the rest: - while (x < safe_width) { - pred_func = VP8LPredictors[((*pred_mode_src++) >> 8) & 0xf]; - for (; t < tile_width; ++t, ++x) { - const uint32_t pred = pred_func(data[x - 1], data + x - width); - AddPixelsEq(data + x, pred); - } - t = 0; - } - if (x < width) { - pred_func = VP8LPredictors[((*pred_mode_src++) >> 8) & 0xf]; - for (; x < width; ++x) { - const uint32_t pred = pred_func(data[x - 1], data + x - width); - AddPixelsEq(data + x, pred); - } + while (x < width) { + const VP8LPredictorAddSubFunc pred_func = + VP8LPredictorsAdd[((*pred_mode_src++) >> 8) & 0xf]; + int x_end = (x & ~mask) + tile_width; + if (x_end > width) x_end = width; + pred_func(in + x, out + x - width, x_end - x, out + x); + x = x_end; } - data += width; + in += width; + out += width; ++y; if ((y & mask) == 0) { // Use the same mask, since tiles are squares. pred_mode_base += tiles_per_row; @@ -233,21 +239,22 @@ static void PredictorInverseTransform(const VP8LTransform* const transform, // Add green to blue and red channels (i.e. perform the inverse transform of // 'subtract green'). -void VP8LAddGreenToBlueAndRed_C(uint32_t* data, int num_pixels) { +void VP8LAddGreenToBlueAndRed_C(const uint32_t* src, int num_pixels, + uint32_t* dst) { int i; for (i = 0; i < num_pixels; ++i) { - const uint32_t argb = data[i]; + const uint32_t argb = src[i]; const uint32_t green = ((argb >> 8) & 0xff); uint32_t red_blue = (argb & 0x00ff00ffu); red_blue += (green << 16) | green; red_blue &= 0x00ff00ffu; - data[i] = (argb & 0xff00ff00u) | red_blue; + dst[i] = (argb & 0xff00ff00u) | red_blue; } } -static WEBP_INLINE uint32_t ColorTransformDelta(int8_t color_pred, - int8_t color) { - return (uint32_t)((int)(color_pred) * color) >> 5; +static WEBP_INLINE int ColorTransformDelta(int8_t color_pred, + int8_t color) { + return ((int)color_pred * color) >> 5; } static WEBP_INLINE void ColorCodeToMultipliers(uint32_t color_code, @@ -257,27 +264,29 @@ static WEBP_INLINE void ColorCodeToMultipliers(uint32_t color_code, m->red_to_blue_ = (color_code >> 16) & 0xff; } -void VP8LTransformColorInverse_C(const VP8LMultipliers* const m, uint32_t* data, - int num_pixels) { +void VP8LTransformColorInverse_C(const VP8LMultipliers* const m, + const uint32_t* src, int num_pixels, + uint32_t* dst) { int i; for (i = 0; i < num_pixels; ++i) { - const uint32_t argb = data[i]; + const uint32_t argb = src[i]; const uint32_t green = argb >> 8; const uint32_t red = argb >> 16; - uint32_t new_red = red; - uint32_t new_blue = argb; + int new_red = red; + int new_blue = argb; new_red += ColorTransformDelta(m->green_to_red_, green); new_red &= 0xff; new_blue += ColorTransformDelta(m->green_to_blue_, green); new_blue += ColorTransformDelta(m->red_to_blue_, new_red); new_blue &= 0xff; - data[i] = (argb & 0xff00ff00u) | (new_red << 16) | (new_blue); + dst[i] = (argb & 0xff00ff00u) | (new_red << 16) | (new_blue); } } // Color space inverse transform. static void ColorSpaceInverseTransform(const VP8LTransform* const transform, - int y_start, int y_end, uint32_t* data) { + int y_start, int y_end, + const uint32_t* src, uint32_t* dst) { const int width = transform->xsize_; const int tile_width = 1 << transform->bits_; const int mask = tile_width - 1; @@ -291,17 +300,19 @@ static void ColorSpaceInverseTransform(const VP8LTransform* const transform, while (y < y_end) { const uint32_t* pred = pred_row; VP8LMultipliers m = { 0, 0, 0 }; - const uint32_t* const data_safe_end = data + safe_width; - const uint32_t* const data_end = data + width; - while (data < data_safe_end) { + const uint32_t* const src_safe_end = src + safe_width; + const uint32_t* const src_end = src + width; + while (src < src_safe_end) { ColorCodeToMultipliers(*pred++, &m); - VP8LTransformColorInverse(&m, data, tile_width); - data += tile_width; + VP8LTransformColorInverse(&m, src, tile_width, dst); + src += tile_width; + dst += tile_width; } - if (data < data_end) { // Left-overs using C-version. + if (src < src_end) { // Left-overs using C-version. ColorCodeToMultipliers(*pred++, &m); - VP8LTransformColorInverse(&m, data, remaining_width); - data += remaining_width; + VP8LTransformColorInverse(&m, src, remaining_width, dst); + src += remaining_width; + dst += remaining_width; } ++y; if ((y & mask) == 0) pred_row += tiles_per_row; @@ -366,10 +377,10 @@ void VP8LInverseTransform(const VP8LTransform* const transform, assert(row_end <= transform->ysize_); switch (transform->type_) { case SUBTRACT_GREEN: - VP8LAddGreenToBlueAndRed(out, (row_end - row_start) * width); + VP8LAddGreenToBlueAndRed(in, (row_end - row_start) * width, out); break; case PREDICTOR_TRANSFORM: - PredictorInverseTransform(transform, row_start, row_end, out); + PredictorInverseTransform(transform, row_start, row_end, in, out); if (row_end != transform->ysize_) { // The last predicted row in this iteration will be the top-pred row // for the first row in next iteration. @@ -378,7 +389,7 @@ void VP8LInverseTransform(const VP8LTransform* const transform, } break; case CROSS_COLOR_TRANSFORM: - ColorSpaceInverseTransform(transform, row_start, row_end, out); + ColorSpaceInverseTransform(transform, row_start, row_end, in, out); break; case COLOR_INDEXING_TRANSFORM: if (in == out && transform->bits_ > 0) { @@ -555,10 +566,15 @@ void VP8LConvertFromBGRA(const uint32_t* const in_data, int num_pixels, //------------------------------------------------------------------------------ -VP8LProcessBlueAndRedFunc VP8LAddGreenToBlueAndRed; +VP8LProcessDecBlueAndRedFunc VP8LAddGreenToBlueAndRed; +VP8LPredictorAddSubFunc VP8LPredictorsAdd[16]; VP8LPredictorFunc VP8LPredictors[16]; -VP8LTransformColorFunc VP8LTransformColorInverse; +// exposed plain-C implementations +VP8LPredictorAddSubFunc VP8LPredictorsAdd_C[16]; +VP8LPredictorFunc VP8LPredictors_C[16]; + +VP8LTransformColorInverseFunc VP8LTransformColorInverse; VP8LConvertFunc VP8LConvertBGRAToRGB; VP8LConvertFunc VP8LConvertBGRAToRGBA; @@ -572,29 +588,37 @@ VP8LMapAlphaFunc VP8LMapColor8b; extern void VP8LDspInitSSE2(void); extern void VP8LDspInitNEON(void); extern void VP8LDspInitMIPSdspR2(void); +extern void VP8LDspInitMSA(void); static volatile VP8CPUInfo lossless_last_cpuinfo_used = (VP8CPUInfo)&lossless_last_cpuinfo_used; +#define COPY_PREDICTOR_ARRAY(IN, OUT) do { \ + (OUT)[0] = IN##0; \ + (OUT)[1] = IN##1; \ + (OUT)[2] = IN##2; \ + (OUT)[3] = IN##3; \ + (OUT)[4] = IN##4; \ + (OUT)[5] = IN##5; \ + (OUT)[6] = IN##6; \ + (OUT)[7] = IN##7; \ + (OUT)[8] = IN##8; \ + (OUT)[9] = IN##9; \ + (OUT)[10] = IN##10; \ + (OUT)[11] = IN##11; \ + (OUT)[12] = IN##12; \ + (OUT)[13] = IN##13; \ + (OUT)[14] = IN##0; /* <- padding security sentinels*/ \ + (OUT)[15] = IN##0; \ +} while (0); + WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInit(void) { if (lossless_last_cpuinfo_used == VP8GetCPUInfo) return; - VP8LPredictors[0] = Predictor0; - VP8LPredictors[1] = Predictor1; - VP8LPredictors[2] = Predictor2; - VP8LPredictors[3] = Predictor3; - VP8LPredictors[4] = Predictor4; - VP8LPredictors[5] = Predictor5; - VP8LPredictors[6] = Predictor6; - VP8LPredictors[7] = Predictor7; - VP8LPredictors[8] = Predictor8; - VP8LPredictors[9] = Predictor9; - VP8LPredictors[10] = Predictor10; - VP8LPredictors[11] = Predictor11; - VP8LPredictors[12] = Predictor12; - VP8LPredictors[13] = Predictor13; - VP8LPredictors[14] = Predictor0; // <- padding security sentinels - VP8LPredictors[15] = Predictor0; + COPY_PREDICTOR_ARRAY(Predictor, VP8LPredictors) + COPY_PREDICTOR_ARRAY(Predictor, VP8LPredictors_C) + COPY_PREDICTOR_ARRAY(PredictorAdd, VP8LPredictorsAdd) + COPY_PREDICTOR_ARRAY(PredictorAdd, VP8LPredictorsAdd_C) VP8LAddGreenToBlueAndRed = VP8LAddGreenToBlueAndRed_C; @@ -626,8 +650,14 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInit(void) { VP8LDspInitMIPSdspR2(); } #endif +#if defined(WEBP_USE_MSA) + if (VP8GetCPUInfo(kMSA)) { + VP8LDspInitMSA(); + } +#endif } lossless_last_cpuinfo_used = VP8GetCPUInfo; } +#undef COPY_PREDICTOR_ARRAY //------------------------------------------------------------------------------ diff --git a/src/3rdparty/libwebp/src/dsp/lossless.h b/src/3rdparty/libwebp/src/dsp/lossless.h index 9f0d7a2..352a54e 100644 --- a/src/3rdparty/libwebp/src/dsp/lossless.h +++ b/src/3rdparty/libwebp/src/dsp/lossless.h @@ -18,7 +18,7 @@ #include "../webp/types.h" #include "../webp/decode.h" -#include "../enc/histogram.h" +#include "../enc/histogram_enc.h" #include "../utils/utils.h" #ifdef __cplusplus @@ -26,7 +26,7 @@ extern "C" { #endif #ifdef WEBP_EXPERIMENTAL_FEATURES -#include "../enc/delta_palettization.h" +#include "../enc/delta_palettization_enc.h" #endif // WEBP_EXPERIMENTAL_FEATURES //------------------------------------------------------------------------------ @@ -34,9 +34,17 @@ extern "C" { typedef uint32_t (*VP8LPredictorFunc)(uint32_t left, const uint32_t* const top); extern VP8LPredictorFunc VP8LPredictors[16]; - -typedef void (*VP8LProcessBlueAndRedFunc)(uint32_t* argb_data, int num_pixels); -extern VP8LProcessBlueAndRedFunc VP8LAddGreenToBlueAndRed; +extern VP8LPredictorFunc VP8LPredictors_C[16]; +// These Add/Sub function expects upper[-1] and out[-1] to be readable. +typedef void (*VP8LPredictorAddSubFunc)(const uint32_t* in, + const uint32_t* upper, int num_pixels, + uint32_t* out); +extern VP8LPredictorAddSubFunc VP8LPredictorsAdd[16]; +extern VP8LPredictorAddSubFunc VP8LPredictorsAdd_C[16]; + +typedef void (*VP8LProcessDecBlueAndRedFunc)(const uint32_t* src, + int num_pixels, uint32_t* dst); +extern VP8LProcessDecBlueAndRedFunc VP8LAddGreenToBlueAndRed; typedef struct { // Note: the members are uint8_t, so that any negative values are @@ -45,9 +53,10 @@ typedef struct { uint8_t green_to_blue_; uint8_t red_to_blue_; } VP8LMultipliers; -typedef void (*VP8LTransformColorFunc)(const VP8LMultipliers* const m, - uint32_t* argb_data, int num_pixels); -extern VP8LTransformColorFunc VP8LTransformColorInverse; +typedef void (*VP8LTransformColorInverseFunc)(const VP8LMultipliers* const m, + const uint32_t* src, + int num_pixels, uint32_t* dst); +extern VP8LTransformColorInverseFunc VP8LTransformColorInverse; struct VP8LTransform; // Defined in dec/vp8li.h. @@ -72,23 +81,6 @@ extern VP8LConvertFunc VP8LConvertBGRAToBGR; void VP8LConvertFromBGRA(const uint32_t* const in_data, int num_pixels, WEBP_CSP_MODE out_colorspace, uint8_t* const rgba); -// color mapping related functions. -static WEBP_INLINE uint32_t VP8GetARGBIndex(uint32_t idx) { - return (idx >> 8) & 0xff; -} - -static WEBP_INLINE uint8_t VP8GetAlphaIndex(uint8_t idx) { - return idx; -} - -static WEBP_INLINE uint32_t VP8GetARGBValue(uint32_t val) { - return val; -} - -static WEBP_INLINE uint8_t VP8GetAlphaValue(uint32_t val) { - return (val >> 8) & 0xff; -} - typedef void (*VP8LMapARGBFunc)(const uint32_t* src, const uint32_t* const color_map, uint32_t* dst, int y_start, @@ -110,7 +102,8 @@ void VP8LColorIndexInverseTransformAlpha( // Expose some C-only fallback functions void VP8LTransformColorInverse_C(const VP8LMultipliers* const m, - uint32_t* data, int num_pixels); + const uint32_t* src, int num_pixels, + uint32_t* dst); void VP8LConvertBGRAToRGB_C(const uint32_t* src, int num_pixels, uint8_t* dst); void VP8LConvertBGRAToRGBA_C(const uint32_t* src, int num_pixels, uint8_t* dst); @@ -119,7 +112,8 @@ void VP8LConvertBGRAToRGBA4444_C(const uint32_t* src, void VP8LConvertBGRAToRGB565_C(const uint32_t* src, int num_pixels, uint8_t* dst); void VP8LConvertBGRAToBGR_C(const uint32_t* src, int num_pixels, uint8_t* dst); -void VP8LAddGreenToBlueAndRed_C(uint32_t* data, int num_pixels); +void VP8LAddGreenToBlueAndRed_C(const uint32_t* src, int num_pixels, + uint32_t* dst); // Must be called before calling any of the above methods. void VP8LDspInit(void); @@ -127,7 +121,10 @@ void VP8LDspInit(void); //------------------------------------------------------------------------------ // Encoding -extern VP8LProcessBlueAndRedFunc VP8LSubtractGreenFromBlueAndRed; +typedef void (*VP8LProcessEncBlueAndRedFunc)(uint32_t* dst, int num_pixels); +extern VP8LProcessEncBlueAndRedFunc VP8LSubtractGreenFromBlueAndRed; +typedef void (*VP8LTransformColorFunc)(const VP8LMultipliers* const m, + uint32_t* const dst, int num_pixels); extern VP8LTransformColorFunc VP8LTransformColor; typedef void (*VP8LCollectColorBlueTransformsFunc)( const uint32_t* argb, int stride, @@ -153,62 +150,8 @@ void VP8LCollectColorBlueTransforms_C(const uint32_t* argb, int stride, int green_to_blue, int red_to_blue, int histo[]); -//------------------------------------------------------------------------------ -// Image transforms. - -void VP8LResidualImage(int width, int height, int bits, int low_effort, - uint32_t* const argb, uint32_t* const argb_scratch, - uint32_t* const image, int near_lossless, int exact, - int used_subtract_green); - -void VP8LColorSpaceTransform(int width, int height, int bits, int quality, - uint32_t* const argb, uint32_t* image); - -//------------------------------------------------------------------------------ -// Misc methods. - -// Computes sampled size of 'size' when sampling using 'sampling bits'. -static WEBP_INLINE uint32_t VP8LSubSampleSize(uint32_t size, - uint32_t sampling_bits) { - return (size + (1 << sampling_bits) - 1) >> sampling_bits; -} - -// Converts near lossless quality into max number of bits shaved off. -static WEBP_INLINE int VP8LNearLosslessBits(int near_lossless_quality) { - // 100 -> 0 - // 80..99 -> 1 - // 60..79 -> 2 - // 40..59 -> 3 - // 20..39 -> 4 - // 0..19 -> 5 - return 5 - near_lossless_quality / 20; -} - -// ----------------------------------------------------------------------------- -// Faster logarithm for integers. Small values use a look-up table. - -// The threshold till approximate version of log_2 can be used. -// Practically, we can get rid of the call to log() as the two values match to -// very high degree (the ratio of these two is 0.99999x). -// Keeping a high threshold for now. -#define APPROX_LOG_WITH_CORRECTION_MAX 65536 -#define APPROX_LOG_MAX 4096 -#define LOG_2_RECIPROCAL 1.44269504088896338700465094007086 -#define LOG_LOOKUP_IDX_MAX 256 -extern const float kLog2Table[LOG_LOOKUP_IDX_MAX]; -extern const float kSLog2Table[LOG_LOOKUP_IDX_MAX]; -typedef float (*VP8LFastLog2SlowFunc)(uint32_t v); - -extern VP8LFastLog2SlowFunc VP8LFastLog2Slow; -extern VP8LFastLog2SlowFunc VP8LFastSLog2Slow; - -static WEBP_INLINE float VP8LFastLog2(uint32_t v) { - return (v < LOG_LOOKUP_IDX_MAX) ? kLog2Table[v] : VP8LFastLog2Slow(v); -} -// Fast calculation of v * log2(v) for integer input. -static WEBP_INLINE float VP8LFastSLog2(uint32_t v) { - return (v < LOG_LOOKUP_IDX_MAX) ? kSLog2Table[v] : VP8LFastSLog2Slow(v); -} +extern VP8LPredictorAddSubFunc VP8LPredictorsSub[16]; +extern VP8LPredictorAddSubFunc VP8LPredictorsSub_C[16]; // ----------------------------------------------------------------------------- // Huffman-cost related functions. @@ -228,11 +171,6 @@ typedef struct { // small struct to hold counters int streaks[2][2]; // [zero/non-zero][streak<3 / streak>=3] } VP8LStreaks; -typedef VP8LStreaks (*VP8LCostCombinedCountFunc)(const uint32_t* X, - const uint32_t* Y, int length); - -extern VP8LCostCombinedCountFunc VP8LHuffmanCostCombinedCount; - typedef struct { // small struct to hold bit entropy results double entropy; // entropy uint32_t sum; // sum of the population @@ -246,26 +184,20 @@ void VP8LBitEntropyInit(VP8LBitEntropy* const entropy); // Get the combined symbol bit entropy and Huffman cost stats for the // distributions 'X' and 'Y'. Those results can then be refined according to // codec specific heuristics. -void VP8LGetCombinedEntropyUnrefined(const uint32_t* const X, - const uint32_t* const Y, int length, - VP8LBitEntropy* const bit_entropy, - VP8LStreaks* const stats); +typedef void (*VP8LGetCombinedEntropyUnrefinedFunc)( + const uint32_t X[], const uint32_t Y[], int length, + VP8LBitEntropy* const bit_entropy, VP8LStreaks* const stats); +extern VP8LGetCombinedEntropyUnrefinedFunc VP8LGetCombinedEntropyUnrefined; + // Get the entropy for the distribution 'X'. -void VP8LGetEntropyUnrefined(const uint32_t* const X, int length, - VP8LBitEntropy* const bit_entropy, - VP8LStreaks* const stats); +typedef void (*VP8LGetEntropyUnrefinedFunc)(const uint32_t X[], int length, + VP8LBitEntropy* const bit_entropy, + VP8LStreaks* const stats); +extern VP8LGetEntropyUnrefinedFunc VP8LGetEntropyUnrefined; void VP8LBitsEntropyUnrefined(const uint32_t* const array, int n, VP8LBitEntropy* const entropy); -typedef void (*GetEntropyUnrefinedHelperFunc)(uint32_t val, int i, - uint32_t* const val_prev, - int* const i_prev, - VP8LBitEntropy* const bit_entropy, - VP8LStreaks* const stats); -// Internal function used by VP8LGet*EntropyUnrefined. -extern GetEntropyUnrefinedHelperFunc VP8LGetEntropyUnrefinedHelper; - typedef void (*VP8LHistogramAddFunc)(const VP8LHistogram* const a, const VP8LHistogram* const b, VP8LHistogram* const out); @@ -279,86 +211,11 @@ typedef int (*VP8LVectorMismatchFunc)(const uint32_t* const array1, // Returns the first index where array1 and array2 are different. extern VP8LVectorMismatchFunc VP8LVectorMismatch; -static WEBP_INLINE int VP8LBitsLog2Ceiling(uint32_t n) { - const int log_floor = BitsLog2Floor(n); - if (n == (n & ~(n - 1))) // zero or a power of two. - return log_floor; - else - return log_floor + 1; -} - -// Splitting of distance and length codes into prefixes and -// extra bits. The prefixes are encoded with an entropy code -// while the extra bits are stored just as normal bits. -static WEBP_INLINE void VP8LPrefixEncodeBitsNoLUT(int distance, int* const code, - int* const extra_bits) { - const int highest_bit = BitsLog2Floor(--distance); - const int second_highest_bit = (distance >> (highest_bit - 1)) & 1; - *extra_bits = highest_bit - 1; - *code = 2 * highest_bit + second_highest_bit; -} - -static WEBP_INLINE void VP8LPrefixEncodeNoLUT(int distance, int* const code, - int* const extra_bits, - int* const extra_bits_value) { - const int highest_bit = BitsLog2Floor(--distance); - const int second_highest_bit = (distance >> (highest_bit - 1)) & 1; - *extra_bits = highest_bit - 1; - *extra_bits_value = distance & ((1 << *extra_bits) - 1); - *code = 2 * highest_bit + second_highest_bit; -} - -#define PREFIX_LOOKUP_IDX_MAX 512 -typedef struct { - int8_t code_; - int8_t extra_bits_; -} VP8LPrefixCode; - -// These tables are derived using VP8LPrefixEncodeNoLUT. -extern const VP8LPrefixCode kPrefixEncodeCode[PREFIX_LOOKUP_IDX_MAX]; -extern const uint8_t kPrefixEncodeExtraBitsValue[PREFIX_LOOKUP_IDX_MAX]; -static WEBP_INLINE void VP8LPrefixEncodeBits(int distance, int* const code, - int* const extra_bits) { - if (distance < PREFIX_LOOKUP_IDX_MAX) { - const VP8LPrefixCode prefix_code = kPrefixEncodeCode[distance]; - *code = prefix_code.code_; - *extra_bits = prefix_code.extra_bits_; - } else { - VP8LPrefixEncodeBitsNoLUT(distance, code, extra_bits); - } -} - -static WEBP_INLINE void VP8LPrefixEncode(int distance, int* const code, - int* const extra_bits, - int* const extra_bits_value) { - if (distance < PREFIX_LOOKUP_IDX_MAX) { - const VP8LPrefixCode prefix_code = kPrefixEncodeCode[distance]; - *code = prefix_code.code_; - *extra_bits = prefix_code.extra_bits_; - *extra_bits_value = kPrefixEncodeExtraBitsValue[distance]; - } else { - VP8LPrefixEncodeNoLUT(distance, code, extra_bits, extra_bits_value); - } -} - -// Sum of each component, mod 256. -static WEBP_INLINE uint32_t VP8LAddPixels(uint32_t a, uint32_t b) { - const uint32_t alpha_and_green = (a & 0xff00ff00u) + (b & 0xff00ff00u); - const uint32_t red_and_blue = (a & 0x00ff00ffu) + (b & 0x00ff00ffu); - return (alpha_and_green & 0xff00ff00u) | (red_and_blue & 0x00ff00ffu); -} - -// Difference of each component, mod 256. -static WEBP_INLINE uint32_t VP8LSubPixels(uint32_t a, uint32_t b) { - const uint32_t alpha_and_green = - 0x00ff00ffu + (a & 0xff00ff00u) - (b & 0xff00ff00u); - const uint32_t red_and_blue = - 0xff00ff00u + (a & 0x00ff00ffu) - (b & 0x00ff00ffu); - return (alpha_and_green & 0xff00ff00u) | (red_and_blue & 0x00ff00ffu); -} - -void VP8LBundleColorMap(const uint8_t* const row, int width, - int xbits, uint32_t* const dst); +typedef void (*VP8LBundleColorMapFunc)(const uint8_t* const row, int width, + int xbits, uint32_t* dst); +extern VP8LBundleColorMapFunc VP8LBundleColorMap; +void VP8LBundleColorMap_C(const uint8_t* const row, int width, int xbits, + uint32_t* dst); // Must be called before calling any of the above methods. void VP8LEncDspInit(void); diff --git a/src/3rdparty/libwebp/src/dsp/lossless_common.h b/src/3rdparty/libwebp/src/dsp/lossless_common.h new file mode 100644 index 0000000..c40f711 --- /dev/null +++ b/src/3rdparty/libwebp/src/dsp/lossless_common.h @@ -0,0 +1,210 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Image transforms and color space conversion methods for lossless decoder. +// +// Authors: Vikas Arora (vikaas.arora@gmail.com) +// Jyrki Alakuijala (jyrki@google.com) +// Vincent Rabaud (vrabaud@google.com) + +#ifndef WEBP_DSP_LOSSLESS_COMMON_H_ +#define WEBP_DSP_LOSSLESS_COMMON_H_ + +#include "../webp/types.h" + +#include "../utils/utils.h" + +#ifdef __cplusplus +extern "C" { +#endif + +//------------------------------------------------------------------------------ +// Decoding + +// color mapping related functions. +static WEBP_INLINE uint32_t VP8GetARGBIndex(uint32_t idx) { + return (idx >> 8) & 0xff; +} + +static WEBP_INLINE uint8_t VP8GetAlphaIndex(uint8_t idx) { + return idx; +} + +static WEBP_INLINE uint32_t VP8GetARGBValue(uint32_t val) { + return val; +} + +static WEBP_INLINE uint8_t VP8GetAlphaValue(uint32_t val) { + return (val >> 8) & 0xff; +} + +//------------------------------------------------------------------------------ +// Misc methods. + +// Computes sampled size of 'size' when sampling using 'sampling bits'. +static WEBP_INLINE uint32_t VP8LSubSampleSize(uint32_t size, + uint32_t sampling_bits) { + return (size + (1 << sampling_bits) - 1) >> sampling_bits; +} + +// Converts near lossless quality into max number of bits shaved off. +static WEBP_INLINE int VP8LNearLosslessBits(int near_lossless_quality) { + // 100 -> 0 + // 80..99 -> 1 + // 60..79 -> 2 + // 40..59 -> 3 + // 20..39 -> 4 + // 0..19 -> 5 + return 5 - near_lossless_quality / 20; +} + +// ----------------------------------------------------------------------------- +// Faster logarithm for integers. Small values use a look-up table. + +// The threshold till approximate version of log_2 can be used. +// Practically, we can get rid of the call to log() as the two values match to +// very high degree (the ratio of these two is 0.99999x). +// Keeping a high threshold for now. +#define APPROX_LOG_WITH_CORRECTION_MAX 65536 +#define APPROX_LOG_MAX 4096 +#define LOG_2_RECIPROCAL 1.44269504088896338700465094007086 +#define LOG_LOOKUP_IDX_MAX 256 +extern const float kLog2Table[LOG_LOOKUP_IDX_MAX]; +extern const float kSLog2Table[LOG_LOOKUP_IDX_MAX]; +typedef float (*VP8LFastLog2SlowFunc)(uint32_t v); + +extern VP8LFastLog2SlowFunc VP8LFastLog2Slow; +extern VP8LFastLog2SlowFunc VP8LFastSLog2Slow; + +static WEBP_INLINE float VP8LFastLog2(uint32_t v) { + return (v < LOG_LOOKUP_IDX_MAX) ? kLog2Table[v] : VP8LFastLog2Slow(v); +} +// Fast calculation of v * log2(v) for integer input. +static WEBP_INLINE float VP8LFastSLog2(uint32_t v) { + return (v < LOG_LOOKUP_IDX_MAX) ? kSLog2Table[v] : VP8LFastSLog2Slow(v); +} + +// ----------------------------------------------------------------------------- +// PrefixEncode() + +static WEBP_INLINE int VP8LBitsLog2Ceiling(uint32_t n) { + const int log_floor = BitsLog2Floor(n); + if (n == (n & ~(n - 1))) { // zero or a power of two. + return log_floor; + } + return log_floor + 1; +} + +// Splitting of distance and length codes into prefixes and +// extra bits. The prefixes are encoded with an entropy code +// while the extra bits are stored just as normal bits. +static WEBP_INLINE void VP8LPrefixEncodeBitsNoLUT(int distance, int* const code, + int* const extra_bits) { + const int highest_bit = BitsLog2Floor(--distance); + const int second_highest_bit = (distance >> (highest_bit - 1)) & 1; + *extra_bits = highest_bit - 1; + *code = 2 * highest_bit + second_highest_bit; +} + +static WEBP_INLINE void VP8LPrefixEncodeNoLUT(int distance, int* const code, + int* const extra_bits, + int* const extra_bits_value) { + const int highest_bit = BitsLog2Floor(--distance); + const int second_highest_bit = (distance >> (highest_bit - 1)) & 1; + *extra_bits = highest_bit - 1; + *extra_bits_value = distance & ((1 << *extra_bits) - 1); + *code = 2 * highest_bit + second_highest_bit; +} + +#define PREFIX_LOOKUP_IDX_MAX 512 +typedef struct { + int8_t code_; + int8_t extra_bits_; +} VP8LPrefixCode; + +// These tables are derived using VP8LPrefixEncodeNoLUT. +extern const VP8LPrefixCode kPrefixEncodeCode[PREFIX_LOOKUP_IDX_MAX]; +extern const uint8_t kPrefixEncodeExtraBitsValue[PREFIX_LOOKUP_IDX_MAX]; +static WEBP_INLINE void VP8LPrefixEncodeBits(int distance, int* const code, + int* const extra_bits) { + if (distance < PREFIX_LOOKUP_IDX_MAX) { + const VP8LPrefixCode prefix_code = kPrefixEncodeCode[distance]; + *code = prefix_code.code_; + *extra_bits = prefix_code.extra_bits_; + } else { + VP8LPrefixEncodeBitsNoLUT(distance, code, extra_bits); + } +} + +static WEBP_INLINE void VP8LPrefixEncode(int distance, int* const code, + int* const extra_bits, + int* const extra_bits_value) { + if (distance < PREFIX_LOOKUP_IDX_MAX) { + const VP8LPrefixCode prefix_code = kPrefixEncodeCode[distance]; + *code = prefix_code.code_; + *extra_bits = prefix_code.extra_bits_; + *extra_bits_value = kPrefixEncodeExtraBitsValue[distance]; + } else { + VP8LPrefixEncodeNoLUT(distance, code, extra_bits, extra_bits_value); + } +} + +// Sum of each component, mod 256. +static WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW WEBP_INLINE +uint32_t VP8LAddPixels(uint32_t a, uint32_t b) { + const uint32_t alpha_and_green = (a & 0xff00ff00u) + (b & 0xff00ff00u); + const uint32_t red_and_blue = (a & 0x00ff00ffu) + (b & 0x00ff00ffu); + return (alpha_and_green & 0xff00ff00u) | (red_and_blue & 0x00ff00ffu); +} + +// Difference of each component, mod 256. +static WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW WEBP_INLINE +uint32_t VP8LSubPixels(uint32_t a, uint32_t b) { + const uint32_t alpha_and_green = + 0x00ff00ffu + (a & 0xff00ff00u) - (b & 0xff00ff00u); + const uint32_t red_and_blue = + 0xff00ff00u + (a & 0x00ff00ffu) - (b & 0x00ff00ffu); + return (alpha_and_green & 0xff00ff00u) | (red_and_blue & 0x00ff00ffu); +} + +//------------------------------------------------------------------------------ +// Transform-related functions use din both encoding and decoding. + +// Macros used to create a batch predictor that iteratively uses a +// one-pixel predictor. + +// The predictor is added to the output pixel (which +// is therefore considered as a residual) to get the final prediction. +#define GENERATE_PREDICTOR_ADD(PREDICTOR, PREDICTOR_ADD) \ +static void PREDICTOR_ADD(const uint32_t* in, const uint32_t* upper, \ + int num_pixels, uint32_t* out) { \ + int x; \ + for (x = 0; x < num_pixels; ++x) { \ + const uint32_t pred = (PREDICTOR)(out[x - 1], upper + x); \ + out[x] = VP8LAddPixels(in[x], pred); \ + } \ +} + +// It subtracts the prediction from the input pixel and stores the residual +// in the output pixel. +#define GENERATE_PREDICTOR_SUB(PREDICTOR, PREDICTOR_SUB) \ +static void PREDICTOR_SUB(const uint32_t* in, const uint32_t* upper, \ + int num_pixels, uint32_t* out) { \ + int x; \ + for (x = 0; x < num_pixels; ++x) { \ + const uint32_t pred = (PREDICTOR)(in[x - 1], upper + x); \ + out[x] = VP8LSubPixels(in[x], pred); \ + } \ +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // WEBP_DSP_LOSSLESS_COMMON_H_ diff --git a/src/3rdparty/libwebp/src/dsp/lossless_enc.c b/src/3rdparty/libwebp/src/dsp/lossless_enc.c index 256f6f5..4e46fba 100644 --- a/src/3rdparty/libwebp/src/dsp/lossless_enc.c +++ b/src/3rdparty/libwebp/src/dsp/lossless_enc.c @@ -17,16 +17,12 @@ #include <math.h> #include <stdlib.h> -#include "../dec/vp8li.h" -#include "../utils/endian_inl.h" +#include "../dec/vp8li_dec.h" +#include "../utils/endian_inl_utils.h" #include "./lossless.h" +#include "./lossless_common.h" #include "./yuv.h" -#define MAX_DIFF_COST (1e30f) - -static const int kPredLowEffort = 11; -static const uint32_t kMaskAlpha = 0xff000000; - // lookup table for small values of log2(int) const float kLog2Table[LOG_LOOKUP_IDX_MAX] = { 0.0000000000000000f, 0.0000000000000000f, @@ -380,26 +376,9 @@ static float FastLog2Slow(uint32_t v) { } } -// Mostly used to reduce code size + readability -static WEBP_INLINE int GetMin(int a, int b) { return (a > b) ? b : a; } -static WEBP_INLINE int GetMax(int a, int b) { return (a < b) ? b : a; } - //------------------------------------------------------------------------------ // Methods to calculate Entropy (Shannon). -static float PredictionCostSpatial(const int counts[256], int weight_0, - double exp_val) { - const int significant_symbols = 256 >> 4; - const double exp_decay_factor = 0.6; - double bits = weight_0 * counts[0]; - int i; - for (i = 1; i < significant_symbols; ++i) { - bits += exp_val * (counts[i] + counts[256 - i]); - exp_val *= exp_decay_factor; - } - return (float)(-0.1 * bits); -} - // Compute the combined Shanon's entropy for distribution {X} and {X+Y} static float CombinedShannonEntropy(const int X[256], const int Y[256]) { int i; @@ -422,18 +401,6 @@ static float CombinedShannonEntropy(const int X[256], const int Y[256]) { return (float)retval; } -static float PredictionCostSpatialHistogram(const int accumulated[4][256], - const int tile[4][256]) { - int i; - double retval = 0; - for (i = 0; i < 4; ++i) { - const double kExpValue = 0.94; - retval += PredictionCostSpatial(tile[i], 1, kExpValue); - retval += VP8LCombinedShannonEntropy(tile[i], accumulated[i]); - } - return (float)retval; -} - void VP8LBitEntropyInit(VP8LBitEntropy* const entropy) { entropy->entropy = 0.; entropy->sum = 0; @@ -486,9 +453,9 @@ static WEBP_INLINE void GetEntropyUnrefinedHelper( *i_prev = i; } -void VP8LGetEntropyUnrefined(const uint32_t* const X, int length, - VP8LBitEntropy* const bit_entropy, - VP8LStreaks* const stats) { +static void GetEntropyUnrefined(const uint32_t X[], int length, + VP8LBitEntropy* const bit_entropy, + VP8LStreaks* const stats) { int i; int i_prev = 0; uint32_t x_prev = X[0]; @@ -499,18 +466,18 @@ void VP8LGetEntropyUnrefined(const uint32_t* const X, int length, for (i = 1; i < length; ++i) { const uint32_t x = X[i]; if (x != x_prev) { - VP8LGetEntropyUnrefinedHelper(x, i, &x_prev, &i_prev, bit_entropy, stats); + GetEntropyUnrefinedHelper(x, i, &x_prev, &i_prev, bit_entropy, stats); } } - VP8LGetEntropyUnrefinedHelper(0, i, &x_prev, &i_prev, bit_entropy, stats); + GetEntropyUnrefinedHelper(0, i, &x_prev, &i_prev, bit_entropy, stats); bit_entropy->entropy += VP8LFastSLog2(bit_entropy->sum); } -void VP8LGetCombinedEntropyUnrefined(const uint32_t* const X, - const uint32_t* const Y, int length, - VP8LBitEntropy* const bit_entropy, - VP8LStreaks* const stats) { +static void GetCombinedEntropyUnrefined(const uint32_t X[], const uint32_t Y[], + int length, + VP8LBitEntropy* const bit_entropy, + VP8LStreaks* const stats) { int i = 1; int i_prev = 0; uint32_t xy_prev = X[0] + Y[0]; @@ -521,439 +488,29 @@ void VP8LGetCombinedEntropyUnrefined(const uint32_t* const X, for (i = 1; i < length; ++i) { const uint32_t xy = X[i] + Y[i]; if (xy != xy_prev) { - VP8LGetEntropyUnrefinedHelper(xy, i, &xy_prev, &i_prev, bit_entropy, - stats); + GetEntropyUnrefinedHelper(xy, i, &xy_prev, &i_prev, bit_entropy, stats); } } - VP8LGetEntropyUnrefinedHelper(0, i, &xy_prev, &i_prev, bit_entropy, stats); + GetEntropyUnrefinedHelper(0, i, &xy_prev, &i_prev, bit_entropy, stats); bit_entropy->entropy += VP8LFastSLog2(bit_entropy->sum); } -static WEBP_INLINE void UpdateHisto(int histo_argb[4][256], uint32_t argb) { - ++histo_argb[0][argb >> 24]; - ++histo_argb[1][(argb >> 16) & 0xff]; - ++histo_argb[2][(argb >> 8) & 0xff]; - ++histo_argb[3][argb & 0xff]; -} - //------------------------------------------------------------------------------ -static WEBP_INLINE uint32_t Predict(VP8LPredictorFunc pred_func, - int x, int y, - const uint32_t* current_row, - const uint32_t* upper_row) { - if (y == 0) { - return (x == 0) ? ARGB_BLACK : current_row[x - 1]; // Left. - } else if (x == 0) { - return upper_row[x]; // Top. - } else { - return pred_func(current_row[x - 1], upper_row + x); - } -} - -static int MaxDiffBetweenPixels(uint32_t p1, uint32_t p2) { - const int diff_a = abs((int)(p1 >> 24) - (int)(p2 >> 24)); - const int diff_r = abs((int)((p1 >> 16) & 0xff) - (int)((p2 >> 16) & 0xff)); - const int diff_g = abs((int)((p1 >> 8) & 0xff) - (int)((p2 >> 8) & 0xff)); - const int diff_b = abs((int)(p1 & 0xff) - (int)(p2 & 0xff)); - return GetMax(GetMax(diff_a, diff_r), GetMax(diff_g, diff_b)); -} - -static int MaxDiffAroundPixel(uint32_t current, uint32_t up, uint32_t down, - uint32_t left, uint32_t right) { - const int diff_up = MaxDiffBetweenPixels(current, up); - const int diff_down = MaxDiffBetweenPixels(current, down); - const int diff_left = MaxDiffBetweenPixels(current, left); - const int diff_right = MaxDiffBetweenPixels(current, right); - return GetMax(GetMax(diff_up, diff_down), GetMax(diff_left, diff_right)); -} - -static uint32_t AddGreenToBlueAndRed(uint32_t argb) { - const uint32_t green = (argb >> 8) & 0xff; - uint32_t red_blue = argb & 0x00ff00ffu; - red_blue += (green << 16) | green; - red_blue &= 0x00ff00ffu; - return (argb & 0xff00ff00u) | red_blue; -} - -static void MaxDiffsForRow(int width, int stride, const uint32_t* const argb, - uint8_t* const max_diffs, int used_subtract_green) { - uint32_t current, up, down, left, right; - int x; - if (width <= 2) return; - current = argb[0]; - right = argb[1]; - if (used_subtract_green) { - current = AddGreenToBlueAndRed(current); - right = AddGreenToBlueAndRed(right); - } - // max_diffs[0] and max_diffs[width - 1] are never used. - for (x = 1; x < width - 1; ++x) { - up = argb[-stride + x]; - down = argb[stride + x]; - left = current; - current = right; - right = argb[x + 1]; - if (used_subtract_green) { - up = AddGreenToBlueAndRed(up); - down = AddGreenToBlueAndRed(down); - right = AddGreenToBlueAndRed(right); - } - max_diffs[x] = MaxDiffAroundPixel(current, up, down, left, right); - } -} - -// Quantize the difference between the actual component value and its prediction -// to a multiple of quantization, working modulo 256, taking care not to cross -// a boundary (inclusive upper limit). -static uint8_t NearLosslessComponent(uint8_t value, uint8_t predict, - uint8_t boundary, int quantization) { - const int residual = (value - predict) & 0xff; - const int boundary_residual = (boundary - predict) & 0xff; - const int lower = residual & ~(quantization - 1); - const int upper = lower + quantization; - // Resolve ties towards a value closer to the prediction (i.e. towards lower - // if value comes after prediction and towards upper otherwise). - const int bias = ((boundary - value) & 0xff) < boundary_residual; - if (residual - lower < upper - residual + bias) { - // lower is closer to residual than upper. - if (residual > boundary_residual && lower <= boundary_residual) { - // Halve quantization step to avoid crossing boundary. This midpoint is - // on the same side of boundary as residual because midpoint >= residual - // (since lower is closer than upper) and residual is above the boundary. - return lower + (quantization >> 1); - } - return lower; - } else { - // upper is closer to residual than lower. - if (residual <= boundary_residual && upper > boundary_residual) { - // Halve quantization step to avoid crossing boundary. This midpoint is - // on the same side of boundary as residual because midpoint <= residual - // (since upper is closer than lower) and residual is below the boundary. - return lower + (quantization >> 1); - } - return upper & 0xff; - } -} - -// Quantize every component of the difference between the actual pixel value and -// its prediction to a multiple of a quantization (a power of 2, not larger than -// max_quantization which is a power of 2, smaller than max_diff). Take care if -// value and predict have undergone subtract green, which means that red and -// blue are represented as offsets from green. -static uint32_t NearLossless(uint32_t value, uint32_t predict, - int max_quantization, int max_diff, - int used_subtract_green) { - int quantization; - uint8_t new_green = 0; - uint8_t green_diff = 0; - uint8_t a, r, g, b; - if (max_diff <= 2) { - return VP8LSubPixels(value, predict); - } - quantization = max_quantization; - while (quantization >= max_diff) { - quantization >>= 1; - } - if ((value >> 24) == 0 || (value >> 24) == 0xff) { - // Preserve transparency of fully transparent or fully opaque pixels. - a = ((value >> 24) - (predict >> 24)) & 0xff; - } else { - a = NearLosslessComponent(value >> 24, predict >> 24, 0xff, quantization); - } - g = NearLosslessComponent((value >> 8) & 0xff, (predict >> 8) & 0xff, 0xff, - quantization); - if (used_subtract_green) { - // The green offset will be added to red and blue components during decoding - // to obtain the actual red and blue values. - new_green = ((predict >> 8) + g) & 0xff; - // The amount by which green has been adjusted during quantization. It is - // subtracted from red and blue for compensation, to avoid accumulating two - // quantization errors in them. - green_diff = (new_green - (value >> 8)) & 0xff; - } - r = NearLosslessComponent(((value >> 16) - green_diff) & 0xff, - (predict >> 16) & 0xff, 0xff - new_green, - quantization); - b = NearLosslessComponent((value - green_diff) & 0xff, predict & 0xff, - 0xff - new_green, quantization); - return ((uint32_t)a << 24) | ((uint32_t)r << 16) | ((uint32_t)g << 8) | b; -} - -// Returns the difference between the pixel and its prediction. In case of a -// lossy encoding, updates the source image to avoid propagating the deviation -// further to pixels which depend on the current pixel for their predictions. -static WEBP_INLINE uint32_t GetResidual(int width, int height, - uint32_t* const upper_row, - uint32_t* const current_row, - const uint8_t* const max_diffs, - int mode, VP8LPredictorFunc pred_func, - int x, int y, int max_quantization, - int exact, int used_subtract_green) { - const uint32_t predict = Predict(pred_func, x, y, current_row, upper_row); - uint32_t residual; - if (max_quantization == 1 || mode == 0 || y == 0 || y == height - 1 || - x == 0 || x == width - 1) { - residual = VP8LSubPixels(current_row[x], predict); - } else { - residual = NearLossless(current_row[x], predict, max_quantization, - max_diffs[x], used_subtract_green); - // Update the source image. - current_row[x] = VP8LAddPixels(predict, residual); - // x is never 0 here so we do not need to update upper_row like below. - } - if (!exact && (current_row[x] & kMaskAlpha) == 0) { - // If alpha is 0, cleanup RGB. We can choose the RGB values of the residual - // for best compression. The prediction of alpha itself can be non-zero and - // must be kept though. We choose RGB of the residual to be 0. - residual &= kMaskAlpha; - // Update the source image. - current_row[x] = predict & ~kMaskAlpha; - // The prediction for the rightmost pixel in a row uses the leftmost pixel - // in that row as its top-right context pixel. Hence if we change the - // leftmost pixel of current_row, the corresponding change must be applied - // to upper_row as well where top-right context is being read from. - if (x == 0 && y != 0) upper_row[width] = current_row[0]; - } - return residual; -} - -// Returns best predictor and updates the accumulated histogram. -// If max_quantization > 1, assumes that near lossless processing will be -// applied, quantizing residuals to multiples of quantization levels up to -// max_quantization (the actual quantization level depends on smoothness near -// the given pixel). -static int GetBestPredictorForTile(int width, int height, - int tile_x, int tile_y, int bits, - int accumulated[4][256], - uint32_t* const argb_scratch, - const uint32_t* const argb, - int max_quantization, - int exact, int used_subtract_green) { - const int kNumPredModes = 14; - const int start_x = tile_x << bits; - const int start_y = tile_y << bits; - const int tile_size = 1 << bits; - const int max_y = GetMin(tile_size, height - start_y); - const int max_x = GetMin(tile_size, width - start_x); - // Whether there exist columns just outside the tile. - const int have_left = (start_x > 0); - const int have_right = (max_x < width - start_x); - // Position and size of the strip covering the tile and adjacent columns if - // they exist. - const int context_start_x = start_x - have_left; - const int context_width = max_x + have_left + have_right; - // The width of upper_row and current_row is one pixel larger than image width - // to allow the top right pixel to point to the leftmost pixel of the next row - // when at the right edge. - uint32_t* upper_row = argb_scratch; - uint32_t* current_row = upper_row + width + 1; - uint8_t* const max_diffs = (uint8_t*)(current_row + width + 1); - float best_diff = MAX_DIFF_COST; - int best_mode = 0; - int mode; - int histo_stack_1[4][256]; - int histo_stack_2[4][256]; - // Need pointers to be able to swap arrays. - int (*histo_argb)[256] = histo_stack_1; - int (*best_histo)[256] = histo_stack_2; - int i, j; - - for (mode = 0; mode < kNumPredModes; ++mode) { - const VP8LPredictorFunc pred_func = VP8LPredictors[mode]; - float cur_diff; - int relative_y; - memset(histo_argb, 0, sizeof(histo_stack_1)); - if (start_y > 0) { - // Read the row above the tile which will become the first upper_row. - // Include a pixel to the left if it exists; include a pixel to the right - // in all cases (wrapping to the leftmost pixel of the next row if it does - // not exist). - memcpy(current_row + context_start_x, - argb + (start_y - 1) * width + context_start_x, - sizeof(*argb) * (max_x + have_left + 1)); - } - for (relative_y = 0; relative_y < max_y; ++relative_y) { - const int y = start_y + relative_y; - int relative_x; - uint32_t* tmp = upper_row; - upper_row = current_row; - current_row = tmp; - // Read current_row. Include a pixel to the left if it exists; include a - // pixel to the right in all cases except at the bottom right corner of - // the image (wrapping to the leftmost pixel of the next row if it does - // not exist in the current row). - memcpy(current_row + context_start_x, - argb + y * width + context_start_x, - sizeof(*argb) * (max_x + have_left + (y + 1 < height))); - if (max_quantization > 1 && y >= 1 && y + 1 < height) { - MaxDiffsForRow(context_width, width, argb + y * width + context_start_x, - max_diffs + context_start_x, used_subtract_green); - } - - for (relative_x = 0; relative_x < max_x; ++relative_x) { - const int x = start_x + relative_x; - UpdateHisto(histo_argb, - GetResidual(width, height, upper_row, current_row, - max_diffs, mode, pred_func, x, y, - max_quantization, exact, used_subtract_green)); - } - } - cur_diff = PredictionCostSpatialHistogram( - (const int (*)[256])accumulated, (const int (*)[256])histo_argb); - if (cur_diff < best_diff) { - int (*tmp)[256] = histo_argb; - histo_argb = best_histo; - best_histo = tmp; - best_diff = cur_diff; - best_mode = mode; - } - } - - for (i = 0; i < 4; i++) { - for (j = 0; j < 256; j++) { - accumulated[i][j] += best_histo[i][j]; - } - } - - return best_mode; -} - -// Converts pixels of the image to residuals with respect to predictions. -// If max_quantization > 1, applies near lossless processing, quantizing -// residuals to multiples of quantization levels up to max_quantization -// (the actual quantization level depends on smoothness near the given pixel). -static void CopyImageWithPrediction(int width, int height, - int bits, uint32_t* const modes, - uint32_t* const argb_scratch, - uint32_t* const argb, - int low_effort, int max_quantization, - int exact, int used_subtract_green) { - const int tiles_per_row = VP8LSubSampleSize(width, bits); - const int mask = (1 << bits) - 1; - // The width of upper_row and current_row is one pixel larger than image width - // to allow the top right pixel to point to the leftmost pixel of the next row - // when at the right edge. - uint32_t* upper_row = argb_scratch; - uint32_t* current_row = upper_row + width + 1; - uint8_t* current_max_diffs = (uint8_t*)(current_row + width + 1); - uint8_t* lower_max_diffs = current_max_diffs + width; - int y; - int mode = 0; - VP8LPredictorFunc pred_func = NULL; - - for (y = 0; y < height; ++y) { - int x; - uint32_t* const tmp32 = upper_row; - upper_row = current_row; - current_row = tmp32; - memcpy(current_row, argb + y * width, - sizeof(*argb) * (width + (y + 1 < height))); - - if (low_effort) { - for (x = 0; x < width; ++x) { - const uint32_t predict = Predict(VP8LPredictors[kPredLowEffort], x, y, - current_row, upper_row); - argb[y * width + x] = VP8LSubPixels(current_row[x], predict); - } - } else { - if (max_quantization > 1) { - // Compute max_diffs for the lower row now, because that needs the - // contents of argb for the current row, which we will overwrite with - // residuals before proceeding with the next row. - uint8_t* const tmp8 = current_max_diffs; - current_max_diffs = lower_max_diffs; - lower_max_diffs = tmp8; - if (y + 2 < height) { - MaxDiffsForRow(width, width, argb + (y + 1) * width, lower_max_diffs, - used_subtract_green); - } - } - for (x = 0; x < width; ++x) { - if ((x & mask) == 0) { - mode = (modes[(y >> bits) * tiles_per_row + (x >> bits)] >> 8) & 0xff; - pred_func = VP8LPredictors[mode]; - } - argb[y * width + x] = GetResidual( - width, height, upper_row, current_row, current_max_diffs, mode, - pred_func, x, y, max_quantization, exact, used_subtract_green); - } - } - } -} - -// Finds the best predictor for each tile, and converts the image to residuals -// with respect to predictions. If near_lossless_quality < 100, applies -// near lossless processing, shaving off more bits of residuals for lower -// qualities. -void VP8LResidualImage(int width, int height, int bits, int low_effort, - uint32_t* const argb, uint32_t* const argb_scratch, - uint32_t* const image, int near_lossless_quality, - int exact, int used_subtract_green) { - const int tiles_per_row = VP8LSubSampleSize(width, bits); - const int tiles_per_col = VP8LSubSampleSize(height, bits); - int tile_y; - int histo[4][256]; - const int max_quantization = 1 << VP8LNearLosslessBits(near_lossless_quality); - if (low_effort) { - int i; - for (i = 0; i < tiles_per_row * tiles_per_col; ++i) { - image[i] = ARGB_BLACK | (kPredLowEffort << 8); - } - } else { - memset(histo, 0, sizeof(histo)); - for (tile_y = 0; tile_y < tiles_per_col; ++tile_y) { - int tile_x; - for (tile_x = 0; tile_x < tiles_per_row; ++tile_x) { - const int pred = GetBestPredictorForTile(width, height, tile_x, tile_y, - bits, histo, argb_scratch, argb, max_quantization, exact, - used_subtract_green); - image[tile_y * tiles_per_row + tile_x] = ARGB_BLACK | (pred << 8); - } - } - } - - CopyImageWithPrediction(width, height, bits, image, argb_scratch, argb, - low_effort, max_quantization, exact, - used_subtract_green); -} - void VP8LSubtractGreenFromBlueAndRed_C(uint32_t* argb_data, int num_pixels) { int i; for (i = 0; i < num_pixels; ++i) { - const uint32_t argb = argb_data[i]; - const uint32_t green = (argb >> 8) & 0xff; + const int argb = argb_data[i]; + const int green = (argb >> 8) & 0xff; const uint32_t new_r = (((argb >> 16) & 0xff) - green) & 0xff; - const uint32_t new_b = ((argb & 0xff) - green) & 0xff; - argb_data[i] = (argb & 0xff00ff00) | (new_r << 16) | new_b; + const uint32_t new_b = (((argb >> 0) & 0xff) - green) & 0xff; + argb_data[i] = (argb & 0xff00ff00u) | (new_r << 16) | new_b; } } -static WEBP_INLINE void MultipliersClear(VP8LMultipliers* const m) { - m->green_to_red_ = 0; - m->green_to_blue_ = 0; - m->red_to_blue_ = 0; -} - -static WEBP_INLINE uint32_t ColorTransformDelta(int8_t color_pred, - int8_t color) { - return (uint32_t)((int)(color_pred) * color) >> 5; -} - -static WEBP_INLINE void ColorCodeToMultipliers(uint32_t color_code, - VP8LMultipliers* const m) { - m->green_to_red_ = (color_code >> 0) & 0xff; - m->green_to_blue_ = (color_code >> 8) & 0xff; - m->red_to_blue_ = (color_code >> 16) & 0xff; -} - -static WEBP_INLINE uint32_t MultipliersToColorCode( - const VP8LMultipliers* const m) { - return 0xff000000u | - ((uint32_t)(m->red_to_blue_) << 16) | - ((uint32_t)(m->green_to_blue_) << 8) | - m->green_to_red_; +static WEBP_INLINE int ColorTransformDelta(int8_t color_pred, int8_t color) { + return ((int)color_pred * color) >> 5; } void VP8LTransformColor_C(const VP8LMultipliers* const m, uint32_t* data, @@ -963,8 +520,8 @@ void VP8LTransformColor_C(const VP8LMultipliers* const m, uint32_t* data, const uint32_t argb = data[i]; const uint32_t green = argb >> 8; const uint32_t red = argb >> 16; - uint32_t new_red = red; - uint32_t new_blue = argb; + int new_red = red; + int new_blue = argb; new_red -= ColorTransformDelta(m->green_to_red_, green); new_red &= 0xff; new_blue -= ColorTransformDelta(m->green_to_blue_, green); @@ -977,7 +534,7 @@ void VP8LTransformColor_C(const VP8LMultipliers* const m, uint32_t* data, static WEBP_INLINE uint8_t TransformColorRed(uint8_t green_to_red, uint32_t argb) { const uint32_t green = argb >> 8; - uint32_t new_red = argb >> 16; + int new_red = argb >> 16; new_red -= ColorTransformDelta(green_to_red, green); return (new_red & 0xff); } @@ -993,15 +550,6 @@ static WEBP_INLINE uint8_t TransformColorBlue(uint8_t green_to_blue, return (new_blue & 0xff); } -static float PredictionCostCrossColor(const int accumulated[256], - const int counts[256]) { - // Favor low entropy, locally and globally. - // Favor small absolute values for PredictionCostSpatial - static const double kExpValue = 2.4; - return VP8LCombinedShannonEntropy(counts, accumulated) + - PredictionCostSpatial(counts, 3, kExpValue); -} - void VP8LCollectColorRedTransforms_C(const uint32_t* argb, int stride, int tile_width, int tile_height, int green_to_red, int histo[]) { @@ -1014,59 +562,6 @@ void VP8LCollectColorRedTransforms_C(const uint32_t* argb, int stride, } } -static float GetPredictionCostCrossColorRed( - const uint32_t* argb, int stride, int tile_width, int tile_height, - VP8LMultipliers prev_x, VP8LMultipliers prev_y, int green_to_red, - const int accumulated_red_histo[256]) { - int histo[256] = { 0 }; - float cur_diff; - - VP8LCollectColorRedTransforms(argb, stride, tile_width, tile_height, - green_to_red, histo); - - cur_diff = PredictionCostCrossColor(accumulated_red_histo, histo); - if ((uint8_t)green_to_red == prev_x.green_to_red_) { - cur_diff -= 3; // favor keeping the areas locally similar - } - if ((uint8_t)green_to_red == prev_y.green_to_red_) { - cur_diff -= 3; // favor keeping the areas locally similar - } - if (green_to_red == 0) { - cur_diff -= 3; - } - return cur_diff; -} - -static void GetBestGreenToRed( - const uint32_t* argb, int stride, int tile_width, int tile_height, - VP8LMultipliers prev_x, VP8LMultipliers prev_y, int quality, - const int accumulated_red_histo[256], VP8LMultipliers* const best_tx) { - const int kMaxIters = 4 + ((7 * quality) >> 8); // in range [4..6] - int green_to_red_best = 0; - int iter, offset; - float best_diff = GetPredictionCostCrossColorRed( - argb, stride, tile_width, tile_height, prev_x, prev_y, - green_to_red_best, accumulated_red_histo); - for (iter = 0; iter < kMaxIters; ++iter) { - // ColorTransformDelta is a 3.5 bit fixed point, so 32 is equal to - // one in color computation. Having initial delta here as 1 is sufficient - // to explore the range of (-2, 2). - const int delta = 32 >> iter; - // Try a negative and a positive delta from the best known value. - for (offset = -delta; offset <= delta; offset += 2 * delta) { - const int green_to_red_cur = offset + green_to_red_best; - const float cur_diff = GetPredictionCostCrossColorRed( - argb, stride, tile_width, tile_height, prev_x, prev_y, - green_to_red_cur, accumulated_red_histo); - if (cur_diff < best_diff) { - best_diff = cur_diff; - green_to_red_best = green_to_red_cur; - } - } - } - best_tx->green_to_red_ = green_to_red_best; -} - void VP8LCollectColorBlueTransforms_C(const uint32_t* argb, int stride, int tile_width, int tile_height, int green_to_blue, int red_to_blue, @@ -1080,187 +575,6 @@ void VP8LCollectColorBlueTransforms_C(const uint32_t* argb, int stride, } } -static float GetPredictionCostCrossColorBlue( - const uint32_t* argb, int stride, int tile_width, int tile_height, - VP8LMultipliers prev_x, VP8LMultipliers prev_y, - int green_to_blue, int red_to_blue, const int accumulated_blue_histo[256]) { - int histo[256] = { 0 }; - float cur_diff; - - VP8LCollectColorBlueTransforms(argb, stride, tile_width, tile_height, - green_to_blue, red_to_blue, histo); - - cur_diff = PredictionCostCrossColor(accumulated_blue_histo, histo); - if ((uint8_t)green_to_blue == prev_x.green_to_blue_) { - cur_diff -= 3; // favor keeping the areas locally similar - } - if ((uint8_t)green_to_blue == prev_y.green_to_blue_) { - cur_diff -= 3; // favor keeping the areas locally similar - } - if ((uint8_t)red_to_blue == prev_x.red_to_blue_) { - cur_diff -= 3; // favor keeping the areas locally similar - } - if ((uint8_t)red_to_blue == prev_y.red_to_blue_) { - cur_diff -= 3; // favor keeping the areas locally similar - } - if (green_to_blue == 0) { - cur_diff -= 3; - } - if (red_to_blue == 0) { - cur_diff -= 3; - } - return cur_diff; -} - -#define kGreenRedToBlueNumAxis 8 -#define kGreenRedToBlueMaxIters 7 -static void GetBestGreenRedToBlue( - const uint32_t* argb, int stride, int tile_width, int tile_height, - VP8LMultipliers prev_x, VP8LMultipliers prev_y, int quality, - const int accumulated_blue_histo[256], - VP8LMultipliers* const best_tx) { - const int8_t offset[kGreenRedToBlueNumAxis][2] = - {{0, -1}, {0, 1}, {-1, 0}, {1, 0}, {-1, -1}, {-1, 1}, {1, -1}, {1, 1}}; - const int8_t delta_lut[kGreenRedToBlueMaxIters] = { 16, 16, 8, 4, 2, 2, 2 }; - const int iters = - (quality < 25) ? 1 : (quality > 50) ? kGreenRedToBlueMaxIters : 4; - int green_to_blue_best = 0; - int red_to_blue_best = 0; - int iter; - // Initial value at origin: - float best_diff = GetPredictionCostCrossColorBlue( - argb, stride, tile_width, tile_height, prev_x, prev_y, - green_to_blue_best, red_to_blue_best, accumulated_blue_histo); - for (iter = 0; iter < iters; ++iter) { - const int delta = delta_lut[iter]; - int axis; - for (axis = 0; axis < kGreenRedToBlueNumAxis; ++axis) { - const int green_to_blue_cur = - offset[axis][0] * delta + green_to_blue_best; - const int red_to_blue_cur = offset[axis][1] * delta + red_to_blue_best; - const float cur_diff = GetPredictionCostCrossColorBlue( - argb, stride, tile_width, tile_height, prev_x, prev_y, - green_to_blue_cur, red_to_blue_cur, accumulated_blue_histo); - if (cur_diff < best_diff) { - best_diff = cur_diff; - green_to_blue_best = green_to_blue_cur; - red_to_blue_best = red_to_blue_cur; - } - if (quality < 25 && iter == 4) { - // Only axis aligned diffs for lower quality. - break; // next iter. - } - } - if (delta == 2 && green_to_blue_best == 0 && red_to_blue_best == 0) { - // Further iterations would not help. - break; // out of iter-loop. - } - } - best_tx->green_to_blue_ = green_to_blue_best; - best_tx->red_to_blue_ = red_to_blue_best; -} -#undef kGreenRedToBlueMaxIters -#undef kGreenRedToBlueNumAxis - -static VP8LMultipliers GetBestColorTransformForTile( - int tile_x, int tile_y, int bits, - VP8LMultipliers prev_x, - VP8LMultipliers prev_y, - int quality, int xsize, int ysize, - const int accumulated_red_histo[256], - const int accumulated_blue_histo[256], - const uint32_t* const argb) { - const int max_tile_size = 1 << bits; - const int tile_y_offset = tile_y * max_tile_size; - const int tile_x_offset = tile_x * max_tile_size; - const int all_x_max = GetMin(tile_x_offset + max_tile_size, xsize); - const int all_y_max = GetMin(tile_y_offset + max_tile_size, ysize); - const int tile_width = all_x_max - tile_x_offset; - const int tile_height = all_y_max - tile_y_offset; - const uint32_t* const tile_argb = argb + tile_y_offset * xsize - + tile_x_offset; - VP8LMultipliers best_tx; - MultipliersClear(&best_tx); - - GetBestGreenToRed(tile_argb, xsize, tile_width, tile_height, - prev_x, prev_y, quality, accumulated_red_histo, &best_tx); - GetBestGreenRedToBlue(tile_argb, xsize, tile_width, tile_height, - prev_x, prev_y, quality, accumulated_blue_histo, - &best_tx); - return best_tx; -} - -static void CopyTileWithColorTransform(int xsize, int ysize, - int tile_x, int tile_y, - int max_tile_size, - VP8LMultipliers color_transform, - uint32_t* argb) { - const int xscan = GetMin(max_tile_size, xsize - tile_x); - int yscan = GetMin(max_tile_size, ysize - tile_y); - argb += tile_y * xsize + tile_x; - while (yscan-- > 0) { - VP8LTransformColor(&color_transform, argb, xscan); - argb += xsize; - } -} - -void VP8LColorSpaceTransform(int width, int height, int bits, int quality, - uint32_t* const argb, uint32_t* image) { - const int max_tile_size = 1 << bits; - const int tile_xsize = VP8LSubSampleSize(width, bits); - const int tile_ysize = VP8LSubSampleSize(height, bits); - int accumulated_red_histo[256] = { 0 }; - int accumulated_blue_histo[256] = { 0 }; - int tile_x, tile_y; - VP8LMultipliers prev_x, prev_y; - MultipliersClear(&prev_y); - MultipliersClear(&prev_x); - for (tile_y = 0; tile_y < tile_ysize; ++tile_y) { - for (tile_x = 0; tile_x < tile_xsize; ++tile_x) { - int y; - const int tile_x_offset = tile_x * max_tile_size; - const int tile_y_offset = tile_y * max_tile_size; - const int all_x_max = GetMin(tile_x_offset + max_tile_size, width); - const int all_y_max = GetMin(tile_y_offset + max_tile_size, height); - const int offset = tile_y * tile_xsize + tile_x; - if (tile_y != 0) { - ColorCodeToMultipliers(image[offset - tile_xsize], &prev_y); - } - prev_x = GetBestColorTransformForTile(tile_x, tile_y, bits, - prev_x, prev_y, - quality, width, height, - accumulated_red_histo, - accumulated_blue_histo, - argb); - image[offset] = MultipliersToColorCode(&prev_x); - CopyTileWithColorTransform(width, height, tile_x_offset, tile_y_offset, - max_tile_size, prev_x, argb); - - // Gather accumulated histogram data. - for (y = tile_y_offset; y < all_y_max; ++y) { - int ix = y * width + tile_x_offset; - const int ix_end = ix + all_x_max - tile_x_offset; - for (; ix < ix_end; ++ix) { - const uint32_t pix = argb[ix]; - if (ix >= 2 && - pix == argb[ix - 2] && - pix == argb[ix - 1]) { - continue; // repeated pixels are handled by backward references - } - if (ix >= width + 2 && - argb[ix - 2] == argb[ix - width - 2] && - argb[ix - 1] == argb[ix - width - 1] && - pix == argb[ix - width]) { - continue; // repeated pixels are handled by backward references - } - ++accumulated_red_histo[(pix >> 16) & 0xff]; - ++accumulated_blue_histo[(pix >> 0) & 0xff]; - } - } - } - } -} - //------------------------------------------------------------------------------ static int VectorMismatch(const uint32_t* const array1, @@ -1274,8 +588,8 @@ static int VectorMismatch(const uint32_t* const array1, } // Bundles multiple (1, 2, 4 or 8) pixels into a single pixel. -void VP8LBundleColorMap(const uint8_t* const row, int width, - int xbits, uint32_t* const dst) { +void VP8LBundleColorMap_C(const uint8_t* const row, int width, int xbits, + uint32_t* dst) { int x; if (xbits > 0) { const int bit_depth = 1 << (3 - xbits); @@ -1350,8 +664,172 @@ static void HistogramAdd(const VP8LHistogram* const a, } //------------------------------------------------------------------------------ +// Image transforms. -VP8LProcessBlueAndRedFunc VP8LSubtractGreenFromBlueAndRed; +static WEBP_INLINE uint32_t Average2(uint32_t a0, uint32_t a1) { + return (((a0 ^ a1) & 0xfefefefeu) >> 1) + (a0 & a1); +} + +static WEBP_INLINE uint32_t Average3(uint32_t a0, uint32_t a1, uint32_t a2) { + return Average2(Average2(a0, a2), a1); +} + +static WEBP_INLINE uint32_t Average4(uint32_t a0, uint32_t a1, + uint32_t a2, uint32_t a3) { + return Average2(Average2(a0, a1), Average2(a2, a3)); +} + +static WEBP_INLINE uint32_t Clip255(uint32_t a) { + if (a < 256) { + return a; + } + // return 0, when a is a negative integer. + // return 255, when a is positive. + return ~a >> 24; +} + +static WEBP_INLINE int AddSubtractComponentFull(int a, int b, int c) { + return Clip255(a + b - c); +} + +static WEBP_INLINE uint32_t ClampedAddSubtractFull(uint32_t c0, uint32_t c1, + uint32_t c2) { + const int a = AddSubtractComponentFull(c0 >> 24, c1 >> 24, c2 >> 24); + const int r = AddSubtractComponentFull((c0 >> 16) & 0xff, + (c1 >> 16) & 0xff, + (c2 >> 16) & 0xff); + const int g = AddSubtractComponentFull((c0 >> 8) & 0xff, + (c1 >> 8) & 0xff, + (c2 >> 8) & 0xff); + const int b = AddSubtractComponentFull(c0 & 0xff, c1 & 0xff, c2 & 0xff); + return ((uint32_t)a << 24) | (r << 16) | (g << 8) | b; +} + +static WEBP_INLINE int AddSubtractComponentHalf(int a, int b) { + return Clip255(a + (a - b) / 2); +} + +static WEBP_INLINE uint32_t ClampedAddSubtractHalf(uint32_t c0, uint32_t c1, + uint32_t c2) { + const uint32_t ave = Average2(c0, c1); + const int a = AddSubtractComponentHalf(ave >> 24, c2 >> 24); + const int r = AddSubtractComponentHalf((ave >> 16) & 0xff, (c2 >> 16) & 0xff); + const int g = AddSubtractComponentHalf((ave >> 8) & 0xff, (c2 >> 8) & 0xff); + const int b = AddSubtractComponentHalf((ave >> 0) & 0xff, (c2 >> 0) & 0xff); + return ((uint32_t)a << 24) | (r << 16) | (g << 8) | b; +} + +// gcc-4.9 on ARM generates incorrect code in Select() when Sub3() is inlined. +#if defined(__arm__) && \ + (LOCAL_GCC_VERSION == 0x409 || LOCAL_GCC_VERSION == 0x408) +# define LOCAL_INLINE __attribute__ ((noinline)) +#else +# define LOCAL_INLINE WEBP_INLINE +#endif + +static LOCAL_INLINE int Sub3(int a, int b, int c) { + const int pb = b - c; + const int pa = a - c; + return abs(pb) - abs(pa); +} + +#undef LOCAL_INLINE + +static WEBP_INLINE uint32_t Select(uint32_t a, uint32_t b, uint32_t c) { + const int pa_minus_pb = + Sub3((a >> 24) , (b >> 24) , (c >> 24) ) + + Sub3((a >> 16) & 0xff, (b >> 16) & 0xff, (c >> 16) & 0xff) + + Sub3((a >> 8) & 0xff, (b >> 8) & 0xff, (c >> 8) & 0xff) + + Sub3((a ) & 0xff, (b ) & 0xff, (c ) & 0xff); + return (pa_minus_pb <= 0) ? a : b; +} + +//------------------------------------------------------------------------------ +// Predictors + +static uint32_t Predictor2(uint32_t left, const uint32_t* const top) { + (void)left; + return top[0]; +} +static uint32_t Predictor3(uint32_t left, const uint32_t* const top) { + (void)left; + return top[1]; +} +static uint32_t Predictor4(uint32_t left, const uint32_t* const top) { + (void)left; + return top[-1]; +} +static uint32_t Predictor5(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average3(left, top[0], top[1]); + return pred; +} +static uint32_t Predictor6(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average2(left, top[-1]); + return pred; +} +static uint32_t Predictor7(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average2(left, top[0]); + return pred; +} +static uint32_t Predictor8(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average2(top[-1], top[0]); + (void)left; + return pred; +} +static uint32_t Predictor9(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average2(top[0], top[1]); + (void)left; + return pred; +} +static uint32_t Predictor10(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Average4(left, top[-1], top[0], top[1]); + return pred; +} +static uint32_t Predictor11(uint32_t left, const uint32_t* const top) { + const uint32_t pred = Select(top[0], left, top[-1]); + return pred; +} +static uint32_t Predictor12(uint32_t left, const uint32_t* const top) { + const uint32_t pred = ClampedAddSubtractFull(left, top[0], top[-1]); + return pred; +} +static uint32_t Predictor13(uint32_t left, const uint32_t* const top) { + const uint32_t pred = ClampedAddSubtractHalf(left, top[0], top[-1]); + return pred; +} + +//------------------------------------------------------------------------------ + +static void PredictorSub0_C(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + for (i = 0; i < num_pixels; ++i) out[i] = VP8LSubPixels(in[i], ARGB_BLACK); + (void)upper; +} + +static void PredictorSub1_C(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + for (i = 0; i < num_pixels; ++i) out[i] = VP8LSubPixels(in[i], in[i - 1]); + (void)upper; +} + +GENERATE_PREDICTOR_SUB(Predictor2, PredictorSub2_C) +GENERATE_PREDICTOR_SUB(Predictor3, PredictorSub3_C) +GENERATE_PREDICTOR_SUB(Predictor4, PredictorSub4_C) +GENERATE_PREDICTOR_SUB(Predictor5, PredictorSub5_C) +GENERATE_PREDICTOR_SUB(Predictor6, PredictorSub6_C) +GENERATE_PREDICTOR_SUB(Predictor7, PredictorSub7_C) +GENERATE_PREDICTOR_SUB(Predictor8, PredictorSub8_C) +GENERATE_PREDICTOR_SUB(Predictor9, PredictorSub9_C) +GENERATE_PREDICTOR_SUB(Predictor10, PredictorSub10_C) +GENERATE_PREDICTOR_SUB(Predictor11, PredictorSub11_C) +GENERATE_PREDICTOR_SUB(Predictor12, PredictorSub12_C) +GENERATE_PREDICTOR_SUB(Predictor13, PredictorSub13_C) + +//------------------------------------------------------------------------------ + +VP8LProcessEncBlueAndRedFunc VP8LSubtractGreenFromBlueAndRed; VP8LTransformColorFunc VP8LTransformColor; @@ -1365,17 +843,23 @@ VP8LCostFunc VP8LExtraCost; VP8LCostCombinedFunc VP8LExtraCostCombined; VP8LCombinedShannonEntropyFunc VP8LCombinedShannonEntropy; -GetEntropyUnrefinedHelperFunc VP8LGetEntropyUnrefinedHelper; +VP8LGetEntropyUnrefinedFunc VP8LGetEntropyUnrefined; +VP8LGetCombinedEntropyUnrefinedFunc VP8LGetCombinedEntropyUnrefined; VP8LHistogramAddFunc VP8LHistogramAdd; VP8LVectorMismatchFunc VP8LVectorMismatch; +VP8LBundleColorMapFunc VP8LBundleColorMap; + +VP8LPredictorAddSubFunc VP8LPredictorsSub[16]; +VP8LPredictorAddSubFunc VP8LPredictorsSub_C[16]; extern void VP8LEncDspInitSSE2(void); extern void VP8LEncDspInitSSE41(void); extern void VP8LEncDspInitNEON(void); extern void VP8LEncDspInitMIPS32(void); extern void VP8LEncDspInitMIPSdspR2(void); +extern void VP8LEncDspInitMSA(void); static volatile VP8CPUInfo lossless_enc_last_cpuinfo_used = (VP8CPUInfo)&lossless_enc_last_cpuinfo_used; @@ -1399,11 +883,47 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInit(void) { VP8LExtraCostCombined = ExtraCostCombined; VP8LCombinedShannonEntropy = CombinedShannonEntropy; - VP8LGetEntropyUnrefinedHelper = GetEntropyUnrefinedHelper; + VP8LGetEntropyUnrefined = GetEntropyUnrefined; + VP8LGetCombinedEntropyUnrefined = GetCombinedEntropyUnrefined; VP8LHistogramAdd = HistogramAdd; VP8LVectorMismatch = VectorMismatch; + VP8LBundleColorMap = VP8LBundleColorMap_C; + + VP8LPredictorsSub[0] = PredictorSub0_C; + VP8LPredictorsSub[1] = PredictorSub1_C; + VP8LPredictorsSub[2] = PredictorSub2_C; + VP8LPredictorsSub[3] = PredictorSub3_C; + VP8LPredictorsSub[4] = PredictorSub4_C; + VP8LPredictorsSub[5] = PredictorSub5_C; + VP8LPredictorsSub[6] = PredictorSub6_C; + VP8LPredictorsSub[7] = PredictorSub7_C; + VP8LPredictorsSub[8] = PredictorSub8_C; + VP8LPredictorsSub[9] = PredictorSub9_C; + VP8LPredictorsSub[10] = PredictorSub10_C; + VP8LPredictorsSub[11] = PredictorSub11_C; + VP8LPredictorsSub[12] = PredictorSub12_C; + VP8LPredictorsSub[13] = PredictorSub13_C; + VP8LPredictorsSub[14] = PredictorSub0_C; // <- padding security sentinels + VP8LPredictorsSub[15] = PredictorSub0_C; + + VP8LPredictorsSub_C[0] = PredictorSub0_C; + VP8LPredictorsSub_C[1] = PredictorSub1_C; + VP8LPredictorsSub_C[2] = PredictorSub2_C; + VP8LPredictorsSub_C[3] = PredictorSub3_C; + VP8LPredictorsSub_C[4] = PredictorSub4_C; + VP8LPredictorsSub_C[5] = PredictorSub5_C; + VP8LPredictorsSub_C[6] = PredictorSub6_C; + VP8LPredictorsSub_C[7] = PredictorSub7_C; + VP8LPredictorsSub_C[8] = PredictorSub8_C; + VP8LPredictorsSub_C[9] = PredictorSub9_C; + VP8LPredictorsSub_C[10] = PredictorSub10_C; + VP8LPredictorsSub_C[11] = PredictorSub11_C; + VP8LPredictorsSub_C[12] = PredictorSub12_C; + VP8LPredictorsSub_C[13] = PredictorSub13_C; + VP8LPredictorsSub_C[14] = PredictorSub0_C; // <- padding security sentinels + VP8LPredictorsSub_C[15] = PredictorSub0_C; // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { @@ -1432,6 +952,11 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInit(void) { VP8LEncDspInitMIPSdspR2(); } #endif +#if defined(WEBP_USE_MSA) + if (VP8GetCPUInfo(kMSA)) { + VP8LEncDspInitMSA(); + } +#endif } lossless_enc_last_cpuinfo_used = VP8GetCPUInfo; } diff --git a/src/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c b/src/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c index 49c666d..4186b9f 100644 --- a/src/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c +++ b/src/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c @@ -14,6 +14,7 @@ #include "./dsp.h" #include "./lossless.h" +#include "./lossless_common.h" #if defined(WEBP_USE_MIPS32) @@ -240,6 +241,49 @@ static WEBP_INLINE void GetEntropyUnrefinedHelper( *i_prev = i; } +static void GetEntropyUnrefined(const uint32_t X[], int length, + VP8LBitEntropy* const bit_entropy, + VP8LStreaks* const stats) { + int i; + int i_prev = 0; + uint32_t x_prev = X[0]; + + memset(stats, 0, sizeof(*stats)); + VP8LBitEntropyInit(bit_entropy); + + for (i = 1; i < length; ++i) { + const uint32_t x = X[i]; + if (x != x_prev) { + GetEntropyUnrefinedHelper(x, i, &x_prev, &i_prev, bit_entropy, stats); + } + } + GetEntropyUnrefinedHelper(0, i, &x_prev, &i_prev, bit_entropy, stats); + + bit_entropy->entropy += VP8LFastSLog2(bit_entropy->sum); +} + +static void GetCombinedEntropyUnrefined(const uint32_t X[], const uint32_t Y[], + int length, + VP8LBitEntropy* const bit_entropy, + VP8LStreaks* const stats) { + int i = 1; + int i_prev = 0; + uint32_t xy_prev = X[0] + Y[0]; + + memset(stats, 0, sizeof(*stats)); + VP8LBitEntropyInit(bit_entropy); + + for (i = 1; i < length; ++i) { + const uint32_t xy = X[i] + Y[i]; + if (xy != xy_prev) { + GetEntropyUnrefinedHelper(xy, i, &xy_prev, &i_prev, bit_entropy, stats); + } + } + GetEntropyUnrefinedHelper(0, i, &xy_prev, &i_prev, bit_entropy, stats); + + bit_entropy->entropy += VP8LFastSLog2(bit_entropy->sum); +} + #define ASM_START \ __asm__ volatile( \ ".set push \n\t" \ @@ -375,7 +419,8 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitMIPS32(void) { VP8LFastLog2Slow = FastLog2Slow; VP8LExtraCost = ExtraCost; VP8LExtraCostCombined = ExtraCostCombined; - VP8LGetEntropyUnrefinedHelper = GetEntropyUnrefinedHelper; + VP8LGetEntropyUnrefined = GetEntropyUnrefined; + VP8LGetCombinedEntropyUnrefined = GetCombinedEntropyUnrefined; VP8LHistogramAdd = HistogramAdd; } diff --git a/src/3rdparty/libwebp/src/dsp/lossless_enc_msa.c b/src/3rdparty/libwebp/src/dsp/lossless_enc_msa.c new file mode 100644 index 0000000..2f69ba3 --- /dev/null +++ b/src/3rdparty/libwebp/src/dsp/lossless_enc_msa.c @@ -0,0 +1,147 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MSA variant of Image transform methods for lossless encoder. +// +// Authors: Prashant Patil (Prashant.Patil@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MSA) + +#include "./lossless.h" +#include "./msa_macro.h" + +#define TRANSFORM_COLOR_8(src0, src1, dst0, dst1, c0, c1, mask0, mask1) do { \ + v8i16 g0, g1, t0, t1, t2, t3; \ + v4i32 t4, t5; \ + VSHF_B2_SH(src0, src0, src1, src1, mask0, mask0, g0, g1); \ + DOTP_SB2_SH(g0, g1, c0, c0, t0, t1); \ + SRAI_H2_SH(t0, t1, 5); \ + t0 = __msa_subv_h((v8i16)src0, t0); \ + t1 = __msa_subv_h((v8i16)src1, t1); \ + t4 = __msa_srli_w((v4i32)src0, 16); \ + t5 = __msa_srli_w((v4i32)src1, 16); \ + DOTP_SB2_SH(t4, t5, c1, c1, t2, t3); \ + SRAI_H2_SH(t2, t3, 5); \ + SUB2(t0, t2, t1, t3, t0, t1); \ + VSHF_B2_UB(src0, t0, src1, t1, mask1, mask1, dst0, dst1); \ +} while (0) + +#define TRANSFORM_COLOR_4(src, dst, c0, c1, mask0, mask1) do { \ + const v16i8 g0 = VSHF_SB(src, src, mask0); \ + v8i16 t0 = __msa_dotp_s_h(c0, g0); \ + v8i16 t1; \ + v4i32 t2; \ + t0 = SRAI_H(t0, 5); \ + t0 = __msa_subv_h((v8i16)src, t0); \ + t2 = __msa_srli_w((v4i32)src, 16); \ + t1 = __msa_dotp_s_h(c1, (v16i8)t2); \ + t1 = SRAI_H(t1, 5); \ + t0 = t0 - t1; \ + dst = VSHF_UB(src, t0, mask1); \ +} while (0) + +static void TransformColor(const VP8LMultipliers* const m, uint32_t* data, + int num_pixels) { + v16u8 src0, dst0; + const v16i8 g2br = (v16i8)__msa_fill_w(m->green_to_blue_ | + (m->green_to_red_ << 16)); + const v16i8 r2b = (v16i8)__msa_fill_w(m->red_to_blue_); + const v16u8 mask0 = { 1, 255, 1, 255, 5, 255, 5, 255, 9, 255, 9, 255, + 13, 255, 13, 255 }; + const v16u8 mask1 = { 16, 1, 18, 3, 20, 5, 22, 7, 24, 9, 26, 11, + 28, 13, 30, 15 }; + + while (num_pixels >= 8) { + v16u8 src1, dst1; + LD_UB2(data, 4, src0, src1); + TRANSFORM_COLOR_8(src0, src1, dst0, dst1, g2br, r2b, mask0, mask1); + ST_UB2(dst0, dst1, data, 4); + data += 8; + num_pixels -= 8; + } + if (num_pixels > 0) { + if (num_pixels >= 4) { + src0 = LD_UB(data); + TRANSFORM_COLOR_4(src0, dst0, g2br, r2b, mask0, mask1); + ST_UB(dst0, data); + data += 4; + num_pixels -= 4; + } + if (num_pixels > 0) { + src0 = LD_UB(data); + TRANSFORM_COLOR_4(src0, dst0, g2br, r2b, mask0, mask1); + if (num_pixels == 3) { + const uint64_t pix_d = __msa_copy_s_d((v2i64)dst0, 0); + const uint32_t pix_w = __msa_copy_s_w((v4i32)dst0, 2); + SD(pix_d, data + 0); + SW(pix_w, data + 2); + } else if (num_pixels == 2) { + const uint64_t pix_d = __msa_copy_s_d((v2i64)dst0, 0); + SD(pix_d, data); + } else { + const uint32_t pix_w = __msa_copy_s_w((v4i32)dst0, 0); + SW(pix_w, data); + } + } + } +} + +static void SubtractGreenFromBlueAndRed(uint32_t* argb_data, int num_pixels) { + int i; + uint8_t* ptemp_data = (uint8_t*)argb_data; + v16u8 src0, dst0, tmp0; + const v16u8 mask = { 1, 255, 1, 255, 5, 255, 5, 255, 9, 255, 9, 255, + 13, 255, 13, 255 }; + + while (num_pixels >= 8) { + v16u8 src1, dst1, tmp1; + LD_UB2(ptemp_data, 16, src0, src1); + VSHF_B2_UB(src0, src1, src1, src0, mask, mask, tmp0, tmp1); + SUB2(src0, tmp0, src1, tmp1, dst0, dst1); + ST_UB2(dst0, dst1, ptemp_data, 16); + ptemp_data += 8 * 4; + num_pixels -= 8; + } + if (num_pixels > 0) { + if (num_pixels >= 4) { + src0 = LD_UB(ptemp_data); + tmp0 = VSHF_UB(src0, src0, mask); + dst0 = src0 - tmp0; + ST_UB(dst0, ptemp_data); + ptemp_data += 4 * 4; + num_pixels -= 4; + } + for (i = 0; i < num_pixels; i++) { + const uint8_t b = ptemp_data[0]; + const uint8_t g = ptemp_data[1]; + const uint8_t r = ptemp_data[2]; + ptemp_data[0] = (b - g) & 0xff; + ptemp_data[2] = (r - g) & 0xff; + ptemp_data += 4; + } + } +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8LEncDspInitMSA(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitMSA(void) { + VP8LSubtractGreenFromBlueAndRed = SubtractGreenFromBlueAndRed; + VP8LTransformColor = TransformColor; +} + +#else // !WEBP_USE_MSA + +WEBP_DSP_INIT_STUB(VP8LEncDspInitMSA) + +#endif // WEBP_USE_MSA diff --git a/src/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c b/src/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c index 7c894e7..8ad85d9 100644 --- a/src/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c +++ b/src/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c @@ -17,6 +17,8 @@ #include <assert.h> #include <emmintrin.h> #include "./lossless.h" +#include "./common_sse2.h" +#include "./lossless_common.h" // For sign-extended multiplying constants, pre-shifted by 5: #define CST_5b(X) (((int16_t)((uint16_t)X << 8)) >> 5) @@ -35,7 +37,9 @@ static void SubtractGreenFromBlueAndRed(uint32_t* argb_data, int num_pixels) { _mm_storeu_si128((__m128i*)&argb_data[i], out); } // fallthrough and finish off with plain-C - VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i); + if (i != num_pixels) { + VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i); + } } //------------------------------------------------------------------------------ @@ -69,7 +73,9 @@ static void TransformColor(const VP8LMultipliers* const m, _mm_storeu_si128((__m128i*)&argb_data[i], out); } // fallthrough and finish off with plain-C - VP8LTransformColor_C(m, argb_data + i, num_pixels - i); + if (i != num_pixels) { + VP8LTransformColor_C(m, argb_data + i, num_pixels - i); + } } //------------------------------------------------------------------------------ @@ -364,8 +370,9 @@ static int VectorMismatch(const uint32_t* const array1, if (length >= 8 && _mm_movemask_epi8(_mm_cmpeq_epi32( _mm_loadu_si128((const __m128i*)&array1[4]), - _mm_loadu_si128((const __m128i*)&array2[4]))) == 0xffff) + _mm_loadu_si128((const __m128i*)&array2[4]))) == 0xffff) { match_len = 8; + } } } @@ -375,6 +382,295 @@ static int VectorMismatch(const uint32_t* const array1, return match_len; } +// Bundles multiple (1, 2, 4 or 8) pixels into a single pixel. +static void BundleColorMap_SSE2(const uint8_t* const row, int width, int xbits, + uint32_t* dst) { + int x; + assert(xbits >= 0); + assert(xbits <= 3); + switch (xbits) { + case 0: { + const __m128i ff = _mm_set1_epi16(0xff00); + const __m128i zero = _mm_setzero_si128(); + // Store 0xff000000 | (row[x] << 8). + for (x = 0; x + 16 <= width; x += 16, dst += 16) { + const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]); + const __m128i in_lo = _mm_unpacklo_epi8(zero, in); + const __m128i dst0 = _mm_unpacklo_epi16(in_lo, ff); + const __m128i dst1 = _mm_unpackhi_epi16(in_lo, ff); + const __m128i in_hi = _mm_unpackhi_epi8(zero, in); + const __m128i dst2 = _mm_unpacklo_epi16(in_hi, ff); + const __m128i dst3 = _mm_unpackhi_epi16(in_hi, ff); + _mm_storeu_si128((__m128i*)&dst[0], dst0); + _mm_storeu_si128((__m128i*)&dst[4], dst1); + _mm_storeu_si128((__m128i*)&dst[8], dst2); + _mm_storeu_si128((__m128i*)&dst[12], dst3); + } + break; + } + case 1: { + const __m128i ff = _mm_set1_epi16(0xff00); + const __m128i mul = _mm_set1_epi16(0x110); + for (x = 0; x + 16 <= width; x += 16, dst += 8) { + // 0a0b | (where a/b are 4 bits). + const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]); + const __m128i tmp = _mm_mullo_epi16(in, mul); // aba0 + const __m128i pack = _mm_and_si128(tmp, ff); // ab00 + const __m128i dst0 = _mm_unpacklo_epi16(pack, ff); + const __m128i dst1 = _mm_unpackhi_epi16(pack, ff); + _mm_storeu_si128((__m128i*)&dst[0], dst0); + _mm_storeu_si128((__m128i*)&dst[4], dst1); + } + break; + } + case 2: { + const __m128i mask_or = _mm_set1_epi32(0xff000000); + const __m128i mul_cst = _mm_set1_epi16(0x0104); + const __m128i mask_mul = _mm_set1_epi16(0x0f00); + for (x = 0; x + 16 <= width; x += 16, dst += 4) { + // 000a000b000c000d | (where a/b/c/d are 2 bits). + const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]); + const __m128i mul = _mm_mullo_epi16(in, mul_cst); // 00ab00b000cd00d0 + const __m128i tmp = _mm_and_si128(mul, mask_mul); // 00ab000000cd0000 + const __m128i shift = _mm_srli_epi32(tmp, 12); // 00000000ab000000 + const __m128i pack = _mm_or_si128(shift, tmp); // 00000000abcd0000 + // Convert to 0xff00**00. + const __m128i res = _mm_or_si128(pack, mask_or); + _mm_storeu_si128((__m128i*)dst, res); + } + break; + } + default: { + assert(xbits == 3); + for (x = 0; x + 16 <= width; x += 16, dst += 2) { + // 0000000a00000000b... | (where a/b are 1 bit). + const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]); + const __m128i shift = _mm_slli_epi64(in, 7); + const uint32_t move = _mm_movemask_epi8(shift); + dst[0] = 0xff000000 | ((move & 0xff) << 8); + dst[1] = 0xff000000 | (move & 0xff00); + } + break; + } + } + if (x != width) { + VP8LBundleColorMap_C(row + x, width - x, xbits, dst); + } +} + +//------------------------------------------------------------------------------ +// Batch version of Predictor Transform subtraction + +static WEBP_INLINE void Average2_m128i(const __m128i* const a0, + const __m128i* const a1, + __m128i* const avg) { + // (a + b) >> 1 = ((a + b + 1) >> 1) - ((a ^ b) & 1) + const __m128i ones = _mm_set1_epi8(1); + const __m128i avg1 = _mm_avg_epu8(*a0, *a1); + const __m128i one = _mm_and_si128(_mm_xor_si128(*a0, *a1), ones); + *avg = _mm_sub_epi8(avg1, one); +} + +// Predictor0: ARGB_BLACK. +static void PredictorSub0_SSE2(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + const __m128i black = _mm_set1_epi32(ARGB_BLACK); + for (i = 0; i + 4 <= num_pixels; i += 4) { + const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); + const __m128i res = _mm_sub_epi8(src, black); + _mm_storeu_si128((__m128i*)&out[i], res); + } + if (i != num_pixels) { + VP8LPredictorsSub_C[0](in + i, upper + i, num_pixels - i, out + i); + } +} + +#define GENERATE_PREDICTOR_1(X, IN) \ +static void PredictorSub##X##_SSE2(const uint32_t* in, const uint32_t* upper, \ + int num_pixels, uint32_t* out) { \ + int i; \ + for (i = 0; i + 4 <= num_pixels; i += 4) { \ + const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \ + const __m128i pred = _mm_loadu_si128((const __m128i*)&(IN)); \ + const __m128i res = _mm_sub_epi8(src, pred); \ + _mm_storeu_si128((__m128i*)&out[i], res); \ + } \ + if (i != num_pixels) { \ + VP8LPredictorsSub_C[(X)](in + i, upper + i, num_pixels - i, out + i); \ + } \ +} + +GENERATE_PREDICTOR_1(1, in[i - 1]) // Predictor1: L +GENERATE_PREDICTOR_1(2, upper[i]) // Predictor2: T +GENERATE_PREDICTOR_1(3, upper[i + 1]) // Predictor3: TR +GENERATE_PREDICTOR_1(4, upper[i - 1]) // Predictor4: TL +#undef GENERATE_PREDICTOR_1 + +// Predictor5: avg2(avg2(L, TR), T) +static void PredictorSub5_SSE2(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + for (i = 0; i + 4 <= num_pixels; i += 4) { + const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]); + const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); + const __m128i TR = _mm_loadu_si128((const __m128i*)&upper[i + 1]); + const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); + __m128i avg, pred, res; + Average2_m128i(&L, &TR, &avg); + Average2_m128i(&avg, &T, &pred); + res = _mm_sub_epi8(src, pred); + _mm_storeu_si128((__m128i*)&out[i], res); + } + if (i != num_pixels) { + VP8LPredictorsSub_C[5](in + i, upper + i, num_pixels - i, out + i); + } +} + +#define GENERATE_PREDICTOR_2(X, A, B) \ +static void PredictorSub##X##_SSE2(const uint32_t* in, const uint32_t* upper, \ + int num_pixels, uint32_t* out) { \ + int i; \ + for (i = 0; i + 4 <= num_pixels; i += 4) { \ + const __m128i tA = _mm_loadu_si128((const __m128i*)&(A)); \ + const __m128i tB = _mm_loadu_si128((const __m128i*)&(B)); \ + const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \ + __m128i pred, res; \ + Average2_m128i(&tA, &tB, &pred); \ + res = _mm_sub_epi8(src, pred); \ + _mm_storeu_si128((__m128i*)&out[i], res); \ + } \ + if (i != num_pixels) { \ + VP8LPredictorsSub_C[(X)](in + i, upper + i, num_pixels - i, out + i); \ + } \ +} + +GENERATE_PREDICTOR_2(6, in[i - 1], upper[i - 1]) // Predictor6: avg(L, TL) +GENERATE_PREDICTOR_2(7, in[i - 1], upper[i]) // Predictor7: avg(L, T) +GENERATE_PREDICTOR_2(8, upper[i - 1], upper[i]) // Predictor8: avg(TL, T) +GENERATE_PREDICTOR_2(9, upper[i], upper[i + 1]) // Predictor9: average(T, TR) +#undef GENERATE_PREDICTOR_2 + +// Predictor10: avg(avg(L,TL), avg(T, TR)). +static void PredictorSub10_SSE2(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + for (i = 0; i + 4 <= num_pixels; i += 4) { + const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]); + const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); + const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); + const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); + const __m128i TR = _mm_loadu_si128((const __m128i*)&upper[i + 1]); + __m128i avgTTR, avgLTL, avg, res; + Average2_m128i(&T, &TR, &avgTTR); + Average2_m128i(&L, &TL, &avgLTL); + Average2_m128i(&avgTTR, &avgLTL, &avg); + res = _mm_sub_epi8(src, avg); + _mm_storeu_si128((__m128i*)&out[i], res); + } + if (i != num_pixels) { + VP8LPredictorsSub_C[10](in + i, upper + i, num_pixels - i, out + i); + } +} + +// Predictor11: select. +static void GetSumAbsDiff32(const __m128i* const A, const __m128i* const B, + __m128i* const out) { + // We can unpack with any value on the upper 32 bits, provided it's the same + // on both operands (to that their sum of abs diff is zero). Here we use *A. + const __m128i A_lo = _mm_unpacklo_epi32(*A, *A); + const __m128i B_lo = _mm_unpacklo_epi32(*B, *A); + const __m128i A_hi = _mm_unpackhi_epi32(*A, *A); + const __m128i B_hi = _mm_unpackhi_epi32(*B, *A); + const __m128i s_lo = _mm_sad_epu8(A_lo, B_lo); + const __m128i s_hi = _mm_sad_epu8(A_hi, B_hi); + *out = _mm_packs_epi32(s_lo, s_hi); +} + +static void PredictorSub11_SSE2(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + for (i = 0; i + 4 <= num_pixels; i += 4) { + const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]); + const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); + const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); + const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); + __m128i pa, pb; + GetSumAbsDiff32(&T, &TL, &pa); // pa = sum |T-TL| + GetSumAbsDiff32(&L, &TL, &pb); // pb = sum |L-TL| + { + const __m128i mask = _mm_cmpgt_epi32(pb, pa); + const __m128i A = _mm_and_si128(mask, L); + const __m128i B = _mm_andnot_si128(mask, T); + const __m128i pred = _mm_or_si128(A, B); // pred = (L > T)? L : T + const __m128i res = _mm_sub_epi8(src, pred); + _mm_storeu_si128((__m128i*)&out[i], res); + } + } + if (i != num_pixels) { + VP8LPredictorsSub_C[11](in + i, upper + i, num_pixels - i, out + i); + } +} + +// Predictor12: ClampedSubSubtractFull. +static void PredictorSub12_SSE2(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + const __m128i zero = _mm_setzero_si128(); + for (i = 0; i + 4 <= num_pixels; i += 4) { + const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); + const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]); + const __m128i L_lo = _mm_unpacklo_epi8(L, zero); + const __m128i L_hi = _mm_unpackhi_epi8(L, zero); + const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); + const __m128i T_lo = _mm_unpacklo_epi8(T, zero); + const __m128i T_hi = _mm_unpackhi_epi8(T, zero); + const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); + const __m128i TL_lo = _mm_unpacklo_epi8(TL, zero); + const __m128i TL_hi = _mm_unpackhi_epi8(TL, zero); + const __m128i diff_lo = _mm_sub_epi16(T_lo, TL_lo); + const __m128i diff_hi = _mm_sub_epi16(T_hi, TL_hi); + const __m128i pred_lo = _mm_add_epi16(L_lo, diff_lo); + const __m128i pred_hi = _mm_add_epi16(L_hi, diff_hi); + const __m128i pred = _mm_packus_epi16(pred_lo, pred_hi); + const __m128i res = _mm_sub_epi8(src, pred); + _mm_storeu_si128((__m128i*)&out[i], res); + } + if (i != num_pixels) { + VP8LPredictorsSub_C[12](in + i, upper + i, num_pixels - i, out + i); + } +} + +// Predictors13: ClampedAddSubtractHalf +static void PredictorSub13_SSE2(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + const __m128i zero = _mm_setzero_si128(); + for (i = 0; i + 2 <= num_pixels; i += 2) { + // we can only process two pixels at a time + const __m128i L = _mm_loadl_epi64((const __m128i*)&in[i - 1]); + const __m128i src = _mm_loadl_epi64((const __m128i*)&in[i]); + const __m128i T = _mm_loadl_epi64((const __m128i*)&upper[i]); + const __m128i TL = _mm_loadl_epi64((const __m128i*)&upper[i - 1]); + const __m128i L_lo = _mm_unpacklo_epi8(L, zero); + const __m128i T_lo = _mm_unpacklo_epi8(T, zero); + const __m128i TL_lo = _mm_unpacklo_epi8(TL, zero); + const __m128i sum = _mm_add_epi16(T_lo, L_lo); + const __m128i avg = _mm_srli_epi16(sum, 1); + const __m128i A1 = _mm_sub_epi16(avg, TL_lo); + const __m128i bit_fix = _mm_cmpgt_epi16(TL_lo, avg); + const __m128i A2 = _mm_sub_epi16(A1, bit_fix); + const __m128i A3 = _mm_srai_epi16(A2, 1); + const __m128i A4 = _mm_add_epi16(avg, A3); + const __m128i pred = _mm_packus_epi16(A4, A4); + const __m128i res = _mm_sub_epi8(src, pred); + _mm_storel_epi64((__m128i*)&out[i], res); + } + if (i != num_pixels) { + VP8LPredictorsSub_C[13](in + i, upper + i, num_pixels - i, out + i); + } +} + //------------------------------------------------------------------------------ // Entry point @@ -388,6 +684,24 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitSSE2(void) { VP8LHistogramAdd = HistogramAdd; VP8LCombinedShannonEntropy = CombinedShannonEntropy; VP8LVectorMismatch = VectorMismatch; + VP8LBundleColorMap = BundleColorMap_SSE2; + + VP8LPredictorsSub[0] = PredictorSub0_SSE2; + VP8LPredictorsSub[1] = PredictorSub1_SSE2; + VP8LPredictorsSub[2] = PredictorSub2_SSE2; + VP8LPredictorsSub[3] = PredictorSub3_SSE2; + VP8LPredictorsSub[4] = PredictorSub4_SSE2; + VP8LPredictorsSub[5] = PredictorSub5_SSE2; + VP8LPredictorsSub[6] = PredictorSub6_SSE2; + VP8LPredictorsSub[7] = PredictorSub7_SSE2; + VP8LPredictorsSub[8] = PredictorSub8_SSE2; + VP8LPredictorsSub[9] = PredictorSub9_SSE2; + VP8LPredictorsSub[10] = PredictorSub10_SSE2; + VP8LPredictorsSub[11] = PredictorSub11_SSE2; + VP8LPredictorsSub[12] = PredictorSub12_SSE2; + VP8LPredictorsSub[13] = PredictorSub13_SSE2; + VP8LPredictorsSub[14] = PredictorSub0_SSE2; // <- padding security sentinels + VP8LPredictorsSub[15] = PredictorSub0_SSE2; } #else // !WEBP_USE_SSE2 diff --git a/src/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c b/src/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c index 3e49319..821057c 100644 --- a/src/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c +++ b/src/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c @@ -32,7 +32,9 @@ static void SubtractGreenFromBlueAndRed(uint32_t* argb_data, int num_pixels) { _mm_storeu_si128((__m128i*)&argb_data[i], out); } // fallthrough and finish off with plain-C - VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i); + if (i != num_pixels) { + VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i); + } } //------------------------------------------------------------------------------ diff --git a/src/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c b/src/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c index 90aed7f..2984ce8 100644 --- a/src/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c +++ b/src/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c @@ -17,6 +17,7 @@ #if defined(WEBP_USE_MIPS_DSP_R2) #include "./lossless.h" +#include "./lossless_common.h" #define MAP_COLOR_FUNCS(FUNC_NAME, TYPE, GET_INDEX, GET_VALUE) \ static void FUNC_NAME(const TYPE* src, \ @@ -227,25 +228,27 @@ static uint32_t Predictor13(uint32_t left, const uint32_t* const top) { // Add green to blue and red channels (i.e. perform the inverse transform of // 'subtract green'). -static void AddGreenToBlueAndRed(uint32_t* data, int num_pixels) { +static void AddGreenToBlueAndRed(const uint32_t* src, int num_pixels, + uint32_t* dst) { uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; - uint32_t* const p_loop1_end = data + (num_pixels & ~3); - uint32_t* const p_loop2_end = data + num_pixels; + const uint32_t* const p_loop1_end = src + (num_pixels & ~3); + const uint32_t* const p_loop2_end = src + num_pixels; __asm__ volatile ( ".set push \n\t" ".set noreorder \n\t" - "beq %[data], %[p_loop1_end], 3f \n\t" + "beq %[src], %[p_loop1_end], 3f \n\t" " nop \n\t" "0: \n\t" - "lw %[temp0], 0(%[data]) \n\t" - "lw %[temp1], 4(%[data]) \n\t" - "lw %[temp2], 8(%[data]) \n\t" - "lw %[temp3], 12(%[data]) \n\t" + "lw %[temp0], 0(%[src]) \n\t" + "lw %[temp1], 4(%[src]) \n\t" + "lw %[temp2], 8(%[src]) \n\t" + "lw %[temp3], 12(%[src]) \n\t" "ext %[temp4], %[temp0], 8, 8 \n\t" "ext %[temp5], %[temp1], 8, 8 \n\t" "ext %[temp6], %[temp2], 8, 8 \n\t" "ext %[temp7], %[temp3], 8, 8 \n\t" - "addiu %[data], %[data], 16 \n\t" + "addiu %[src], %[src], 16 \n\t" + "addiu %[dst], %[dst], 16 \n\t" "replv.ph %[temp4], %[temp4] \n\t" "replv.ph %[temp5], %[temp5] \n\t" "replv.ph %[temp6], %[temp6] \n\t" @@ -254,44 +257,47 @@ static void AddGreenToBlueAndRed(uint32_t* data, int num_pixels) { "addu.qb %[temp1], %[temp1], %[temp5] \n\t" "addu.qb %[temp2], %[temp2], %[temp6] \n\t" "addu.qb %[temp3], %[temp3], %[temp7] \n\t" - "sw %[temp0], -16(%[data]) \n\t" - "sw %[temp1], -12(%[data]) \n\t" - "sw %[temp2], -8(%[data]) \n\t" - "bne %[data], %[p_loop1_end], 0b \n\t" - " sw %[temp3], -4(%[data]) \n\t" + "sw %[temp0], -16(%[dst]) \n\t" + "sw %[temp1], -12(%[dst]) \n\t" + "sw %[temp2], -8(%[dst]) \n\t" + "bne %[src], %[p_loop1_end], 0b \n\t" + " sw %[temp3], -4(%[dst]) \n\t" "3: \n\t" - "beq %[data], %[p_loop2_end], 2f \n\t" + "beq %[src], %[p_loop2_end], 2f \n\t" " nop \n\t" "1: \n\t" - "lw %[temp0], 0(%[data]) \n\t" - "addiu %[data], %[data], 4 \n\t" + "lw %[temp0], 0(%[src]) \n\t" + "addiu %[src], %[src], 4 \n\t" + "addiu %[dst], %[dst], 4 \n\t" "ext %[temp4], %[temp0], 8, 8 \n\t" "replv.ph %[temp4], %[temp4] \n\t" "addu.qb %[temp0], %[temp0], %[temp4] \n\t" - "bne %[data], %[p_loop2_end], 1b \n\t" - " sw %[temp0], -4(%[data]) \n\t" + "bne %[src], %[p_loop2_end], 1b \n\t" + " sw %[temp0], -4(%[dst]) \n\t" "2: \n\t" ".set pop \n\t" - : [data]"+&r"(data), [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), - [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), - [temp5]"=&r"(temp5), [temp6]"=&r"(temp6), [temp7]"=&r"(temp7) + : [dst]"+&r"(dst), [src]"+&r"(src), [temp0]"=&r"(temp0), + [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), + [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [temp6]"=&r"(temp6), + [temp7]"=&r"(temp7) : [p_loop1_end]"r"(p_loop1_end), [p_loop2_end]"r"(p_loop2_end) : "memory" ); } static void TransformColorInverse(const VP8LMultipliers* const m, - uint32_t* data, int num_pixels) { + const uint32_t* src, int num_pixels, + uint32_t* dst) { int temp0, temp1, temp2, temp3, temp4, temp5; uint32_t argb, argb1, new_red; const uint32_t G_to_R = m->green_to_red_; const uint32_t G_to_B = m->green_to_blue_; const uint32_t R_to_B = m->red_to_blue_; - uint32_t* const p_loop_end = data + (num_pixels & ~1); + const uint32_t* const p_loop_end = src + (num_pixels & ~1); __asm__ volatile ( ".set push \n\t" ".set noreorder \n\t" - "beq %[data], %[p_loop_end], 1f \n\t" + "beq %[src], %[p_loop_end], 1f \n\t" " nop \n\t" "replv.ph %[temp0], %[G_to_R] \n\t" "replv.ph %[temp1], %[G_to_B] \n\t" @@ -303,9 +309,12 @@ static void TransformColorInverse(const VP8LMultipliers* const m, "shra.ph %[temp1], %[temp1], 8 \n\t" "shra.ph %[temp2], %[temp2], 8 \n\t" "0: \n\t" - "lw %[argb], 0(%[data]) \n\t" - "lw %[argb1], 4(%[data]) \n\t" - "addiu %[data], %[data], 8 \n\t" + "lw %[argb], 0(%[src]) \n\t" + "lw %[argb1], 4(%[src]) \n\t" + "sw %[argb], 0(%[dst]) \n\t" + "sw %[argb1], 4(%[dst]) \n\t" + "addiu %[src], %[src], 8 \n\t" + "addiu %[dst], %[dst], 8 \n\t" "precrq.qb.ph %[temp3], %[argb], %[argb1] \n\t" "preceu.ph.qbra %[temp3], %[temp3] \n\t" "shll.ph %[temp3], %[temp3], 8 \n\t" @@ -322,29 +331,29 @@ static void TransformColorInverse(const VP8LMultipliers* const m, "shll.ph %[temp4], %[temp5], 8 \n\t" "shra.ph %[temp4], %[temp4], 8 \n\t" "mul.ph %[temp4], %[temp4], %[temp2] \n\t" - "sb %[temp5], -2(%[data]) \n\t" + "sb %[temp5], -2(%[dst]) \n\t" "sra %[temp5], %[temp5], 16 \n\t" "shra.ph %[temp4], %[temp4], 5 \n\t" "addu.ph %[argb1], %[argb1], %[temp4] \n\t" "preceu.ph.qbra %[temp3], %[argb1] \n\t" - "sb %[temp5], -6(%[data]) \n\t" - "sb %[temp3], -4(%[data]) \n\t" + "sb %[temp5], -6(%[dst]) \n\t" + "sb %[temp3], -4(%[dst]) \n\t" "sra %[temp3], %[temp3], 16 \n\t" - "bne %[data], %[p_loop_end], 0b \n\t" - " sb %[temp3], -8(%[data]) \n\t" + "bne %[src], %[p_loop_end], 0b \n\t" + " sb %[temp3], -8(%[dst]) \n\t" "1: \n\t" ".set pop \n\t" : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [new_red]"=&r"(new_red), [argb]"=&r"(argb), - [argb1]"=&r"(argb1), [data]"+&r"(data) + [argb1]"=&r"(argb1), [dst]"+&r"(dst), [src]"+&r"(src) : [G_to_R]"r"(G_to_R), [R_to_B]"r"(R_to_B), [G_to_B]"r"(G_to_B), [p_loop_end]"r"(p_loop_end) : "memory", "hi", "lo" ); // Fall-back to C-version for left-overs. - if (num_pixels & 1) VP8LTransformColorInverse_C(m, data, 1); + if (num_pixels & 1) VP8LTransformColorInverse_C(m, src, 1, dst); } static void ConvertBGRAToRGB(const uint32_t* src, diff --git a/src/3rdparty/libwebp/src/dsp/lossless_msa.c b/src/3rdparty/libwebp/src/dsp/lossless_msa.c new file mode 100644 index 0000000..f6dd564 --- /dev/null +++ b/src/3rdparty/libwebp/src/dsp/lossless_msa.c @@ -0,0 +1,355 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MSA variant of methods for lossless decoder +// +// Author: Prashant Patil (prashant.patil@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MSA) + +#include "./lossless.h" +#include "./msa_macro.h" + +//------------------------------------------------------------------------------ +// Colorspace conversion functions + +#define CONVERT16_BGRA_XXX(psrc, pdst, m0, m1, m2) do { \ + v16u8 src0, src1, src2, src3, dst0, dst1, dst2; \ + LD_UB4(psrc, 16, src0, src1, src2, src3); \ + VSHF_B2_UB(src0, src1, src1, src2, m0, m1, dst0, dst1); \ + dst2 = VSHF_UB(src2, src3, m2); \ + ST_UB2(dst0, dst1, pdst, 16); \ + ST_UB(dst2, pdst + 32); \ +} while (0) + +#define CONVERT12_BGRA_XXX(psrc, pdst, m0, m1, m2) do { \ + uint32_t pix_w; \ + v16u8 src0, src1, src2, dst0, dst1, dst2; \ + LD_UB3(psrc, 16, src0, src1, src2); \ + VSHF_B2_UB(src0, src1, src1, src2, m0, m1, dst0, dst1); \ + dst2 = VSHF_UB(src2, src2, m2); \ + ST_UB2(dst0, dst1, pdst, 16); \ + pix_w = __msa_copy_s_w((v4i32)dst2, 0); \ + SW(pix_w, pdst + 32); \ +} while (0) + +#define CONVERT8_BGRA_XXX(psrc, pdst, m0, m1) do { \ + uint64_t pix_d; \ + v16u8 src0, src1, src2, dst0, dst1; \ + LD_UB2(psrc, 16, src0, src1); \ + VSHF_B2_UB(src0, src1, src1, src2, m0, m1, dst0, dst1); \ + ST_UB(dst0, pdst); \ + pix_d = __msa_copy_s_d((v2i64)dst1, 0); \ + SD(pix_d, pdst + 16); \ +} while (0) + +#define CONVERT4_BGRA_XXX(psrc, pdst, m) do { \ + const v16u8 src0 = LD_UB(psrc); \ + const v16u8 dst0 = VSHF_UB(src0, src0, m); \ + uint64_t pix_d = __msa_copy_s_d((v2i64)dst0, 0); \ + uint32_t pix_w = __msa_copy_s_w((v4i32)dst0, 2); \ + SD(pix_d, pdst + 0); \ + SW(pix_w, pdst + 8); \ +} while (0) + +#define CONVERT1_BGRA_BGR(psrc, pdst) do { \ + const int32_t b = (psrc)[0]; \ + const int32_t g = (psrc)[1]; \ + const int32_t r = (psrc)[2]; \ + (pdst)[0] = b; \ + (pdst)[1] = g; \ + (pdst)[2] = r; \ +} while (0) + +#define CONVERT1_BGRA_RGB(psrc, pdst) do { \ + const int32_t b = (psrc)[0]; \ + const int32_t g = (psrc)[1]; \ + const int32_t r = (psrc)[2]; \ + (pdst)[0] = r; \ + (pdst)[1] = g; \ + (pdst)[2] = b; \ +} while (0) + +#define TRANSFORM_COLOR_INVERSE_8(src0, src1, dst0, dst1, \ + c0, c1, mask0, mask1) do { \ + v8i16 g0, g1, t0, t1, t2, t3; \ + v4i32 t4, t5; \ + VSHF_B2_SH(src0, src0, src1, src1, mask0, mask0, g0, g1); \ + DOTP_SB2_SH(g0, g1, c0, c0, t0, t1); \ + SRAI_H2_SH(t0, t1, 5); \ + t0 = __msa_addv_h(t0, (v8i16)src0); \ + t1 = __msa_addv_h(t1, (v8i16)src1); \ + t4 = __msa_srli_w((v4i32)t0, 16); \ + t5 = __msa_srli_w((v4i32)t1, 16); \ + DOTP_SB2_SH(t4, t5, c1, c1, t2, t3); \ + SRAI_H2_SH(t2, t3, 5); \ + ADD2(t0, t2, t1, t3, t0, t1); \ + VSHF_B2_UB(src0, t0, src1, t1, mask1, mask1, dst0, dst1); \ +} while (0) + +#define TRANSFORM_COLOR_INVERSE_4(src, dst, c0, c1, mask0, mask1) do { \ + const v16i8 g0 = VSHF_SB(src, src, mask0); \ + v8i16 t0 = __msa_dotp_s_h(c0, g0); \ + v8i16 t1; \ + v4i32 t2; \ + t0 = SRAI_H(t0, 5); \ + t0 = __msa_addv_h(t0, (v8i16)src); \ + t2 = __msa_srli_w((v4i32)t0, 16); \ + t1 = __msa_dotp_s_h(c1, (v16i8)t2); \ + t1 = SRAI_H(t1, 5); \ + t0 = t0 + t1; \ + dst = VSHF_UB(src, t0, mask1); \ +} while (0) + +static void ConvertBGRAToRGBA(const uint32_t* src, + int num_pixels, uint8_t* dst) { + int i; + const uint8_t* ptemp_src = (const uint8_t*)src; + uint8_t* ptemp_dst = (uint8_t*)dst; + v16u8 src0, dst0; + const v16u8 mask = { 2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15 }; + + while (num_pixels >= 8) { + v16u8 src1, dst1; + LD_UB2(ptemp_src, 16, src0, src1); + VSHF_B2_UB(src0, src0, src1, src1, mask, mask, dst0, dst1); + ST_UB2(dst0, dst1, ptemp_dst, 16); + ptemp_src += 32; + ptemp_dst += 32; + num_pixels -= 8; + } + if (num_pixels > 0) { + if (num_pixels >= 4) { + src0 = LD_UB(ptemp_src); + dst0 = VSHF_UB(src0, src0, mask); + ST_UB(dst0, ptemp_dst); + ptemp_src += 16; + ptemp_dst += 16; + num_pixels -= 4; + } + for (i = 0; i < num_pixels; i++) { + const uint8_t b = ptemp_src[2]; + const uint8_t g = ptemp_src[1]; + const uint8_t r = ptemp_src[0]; + const uint8_t a = ptemp_src[3]; + ptemp_dst[0] = b; + ptemp_dst[1] = g; + ptemp_dst[2] = r; + ptemp_dst[3] = a; + ptemp_src += 4; + ptemp_dst += 4; + } + } +} + +static void ConvertBGRAToBGR(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const uint8_t* ptemp_src = (const uint8_t*)src; + uint8_t* ptemp_dst = (uint8_t*)dst; + const v16u8 mask0 = { 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, + 16, 17, 18, 20 }; + const v16u8 mask1 = { 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, + 21, 22, 24, 25 }; + const v16u8 mask2 = { 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25, + 26, 28, 29, 30 }; + + while (num_pixels >= 16) { + CONVERT16_BGRA_XXX(ptemp_src, ptemp_dst, mask0, mask1, mask2); + ptemp_src += 64; + ptemp_dst += 48; + num_pixels -= 16; + } + if (num_pixels > 0) { + if (num_pixels >= 12) { + CONVERT12_BGRA_XXX(ptemp_src, ptemp_dst, mask0, mask1, mask2); + ptemp_src += 48; + ptemp_dst += 36; + num_pixels -= 12; + } else if (num_pixels >= 8) { + CONVERT8_BGRA_XXX(ptemp_src, ptemp_dst, mask0, mask1); + ptemp_src += 32; + ptemp_dst += 24; + num_pixels -= 8; + } else if (num_pixels >= 4) { + CONVERT4_BGRA_XXX(ptemp_src, ptemp_dst, mask0); + ptemp_src += 16; + ptemp_dst += 12; + num_pixels -= 4; + } + if (num_pixels == 3) { + CONVERT1_BGRA_BGR(ptemp_src + 0, ptemp_dst + 0); + CONVERT1_BGRA_BGR(ptemp_src + 4, ptemp_dst + 3); + CONVERT1_BGRA_BGR(ptemp_src + 8, ptemp_dst + 6); + } else if (num_pixels == 2) { + CONVERT1_BGRA_BGR(ptemp_src + 0, ptemp_dst + 0); + CONVERT1_BGRA_BGR(ptemp_src + 4, ptemp_dst + 3); + } else if (num_pixels == 1) { + CONVERT1_BGRA_BGR(ptemp_src, ptemp_dst); + } + } +} + +static void ConvertBGRAToRGB(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const uint8_t* ptemp_src = (const uint8_t*)src; + uint8_t* ptemp_dst = (uint8_t*)dst; + const v16u8 mask0 = { 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, + 18, 17, 16, 22 }; + const v16u8 mask1 = { 5, 4, 10, 9, 8, 14, 13, 12, 18, 17, 16, 22, + 21, 20, 26, 25 }; + const v16u8 mask2 = { 8, 14, 13, 12, 18, 17, 16, 22, 21, 20, 26, 25, + 24, 30, 29, 28 }; + + while (num_pixels >= 16) { + CONVERT16_BGRA_XXX(ptemp_src, ptemp_dst, mask0, mask1, mask2); + ptemp_src += 64; + ptemp_dst += 48; + num_pixels -= 16; + } + if (num_pixels) { + if (num_pixels >= 12) { + CONVERT12_BGRA_XXX(ptemp_src, ptemp_dst, mask0, mask1, mask2); + ptemp_src += 48; + ptemp_dst += 36; + num_pixels -= 12; + } else if (num_pixels >= 8) { + CONVERT8_BGRA_XXX(ptemp_src, ptemp_dst, mask0, mask1); + ptemp_src += 32; + ptemp_dst += 24; + num_pixels -= 8; + } else if (num_pixels >= 4) { + CONVERT4_BGRA_XXX(ptemp_src, ptemp_dst, mask0); + ptemp_src += 16; + ptemp_dst += 12; + num_pixels -= 4; + } + if (num_pixels == 3) { + CONVERT1_BGRA_RGB(ptemp_src + 0, ptemp_dst + 0); + CONVERT1_BGRA_RGB(ptemp_src + 4, ptemp_dst + 3); + CONVERT1_BGRA_RGB(ptemp_src + 8, ptemp_dst + 6); + } else if (num_pixels == 2) { + CONVERT1_BGRA_RGB(ptemp_src + 0, ptemp_dst + 0); + CONVERT1_BGRA_RGB(ptemp_src + 4, ptemp_dst + 3); + } else if (num_pixels == 1) { + CONVERT1_BGRA_RGB(ptemp_src, ptemp_dst); + } + } +} + +static void AddGreenToBlueAndRed(const uint32_t* const src, int num_pixels, + uint32_t* dst) { + int i; + const uint8_t* in = (const uint8_t*)src; + uint8_t* out = (uint8_t*)dst; + v16u8 src0, dst0, tmp0; + const v16u8 mask = { 1, 255, 1, 255, 5, 255, 5, 255, 9, 255, 9, 255, + 13, 255, 13, 255 }; + + while (num_pixels >= 8) { + v16u8 src1, dst1, tmp1; + LD_UB2(in, 16, src0, src1); + VSHF_B2_UB(src0, src1, src1, src0, mask, mask, tmp0, tmp1); + ADD2(src0, tmp0, src1, tmp1, dst0, dst1); + ST_UB2(dst0, dst1, out, 16); + in += 32; + out += 32; + num_pixels -= 8; + } + if (num_pixels > 0) { + if (num_pixels >= 4) { + src0 = LD_UB(in); + tmp0 = VSHF_UB(src0, src0, mask); + dst0 = src0 + tmp0; + ST_UB(dst0, out); + in += 16; + out += 16; + num_pixels -= 4; + } + for (i = 0; i < num_pixels; i++) { + const uint8_t b = in[0]; + const uint8_t g = in[1]; + const uint8_t r = in[2]; + out[0] = (b + g) & 0xff; + out[1] = g; + out[2] = (r + g) & 0xff; + out[4] = in[4]; + out += 4; + } + } +} + +static void TransformColorInverse(const VP8LMultipliers* const m, + const uint32_t* src, int num_pixels, + uint32_t* dst) { + v16u8 src0, dst0; + const v16i8 g2br = (v16i8)__msa_fill_w(m->green_to_blue_ | + (m->green_to_red_ << 16)); + const v16i8 r2b = (v16i8)__msa_fill_w(m->red_to_blue_); + const v16u8 mask0 = { 1, 255, 1, 255, 5, 255, 5, 255, 9, 255, 9, 255, + 13, 255, 13, 255 }; + const v16u8 mask1 = { 16, 1, 18, 3, 20, 5, 22, 7, 24, 9, 26, 11, + 28, 13, 30, 15 }; + + while (num_pixels >= 8) { + v16u8 src1, dst1; + LD_UB2(src, 4, src0, src1); + TRANSFORM_COLOR_INVERSE_8(src0, src1, dst0, dst1, g2br, r2b, mask0, mask1); + ST_UB2(dst0, dst1, dst, 4); + src += 8; + dst += 8; + num_pixels -= 8; + } + if (num_pixels > 0) { + if (num_pixels >= 4) { + src0 = LD_UB(src); + TRANSFORM_COLOR_INVERSE_4(src0, dst0, g2br, r2b, mask0, mask1); + ST_UB(dst0, dst); + src += 4; + dst += 4; + num_pixels -= 4; + } + if (num_pixels > 0) { + src0 = LD_UB(src); + TRANSFORM_COLOR_INVERSE_4(src0, dst0, g2br, r2b, mask0, mask1); + if (num_pixels == 3) { + const uint64_t pix_d = __msa_copy_s_d((v2i64)dst0, 0); + const uint32_t pix_w = __msa_copy_s_w((v4i32)dst0, 2); + SD(pix_d, dst + 0); + SW(pix_w, dst + 2); + } else if (num_pixels == 2) { + const uint64_t pix_d = __msa_copy_s_d((v2i64)dst0, 0); + SD(pix_d, dst); + } else { + const uint32_t pix_w = __msa_copy_s_w((v4i32)dst0, 0); + SW(pix_w, dst); + } + } + } +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8LDspInitMSA(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInitMSA(void) { + VP8LConvertBGRAToRGBA = ConvertBGRAToRGBA; + VP8LConvertBGRAToBGR = ConvertBGRAToBGR; + VP8LConvertBGRAToRGB = ConvertBGRAToRGB; + VP8LAddGreenToBlueAndRed = AddGreenToBlueAndRed; + VP8LTransformColorInverse = TransformColorInverse; +} + +#else // !WEBP_USE_MSA + +WEBP_DSP_INIT_STUB(VP8LDspInitMSA) + +#endif // WEBP_USE_MSA diff --git a/src/3rdparty/libwebp/src/dsp/lossless_neon.c b/src/3rdparty/libwebp/src/dsp/lossless_neon.c index 6faccb8..1145d5f 100644 --- a/src/3rdparty/libwebp/src/dsp/lossless_neon.c +++ b/src/3rdparty/libwebp/src/dsp/lossless_neon.c @@ -139,6 +139,357 @@ static void ConvertBGRAToRGB(const uint32_t* src, #endif // !WORK_AROUND_GCC + +//------------------------------------------------------------------------------ +// Predictor Transform + +#define LOAD_U32_AS_U8(IN) vreinterpret_u8_u32(vdup_n_u32((IN))) +#define LOAD_U32P_AS_U8(IN) vreinterpret_u8_u32(vld1_u32((IN))) +#define LOADQ_U32_AS_U8(IN) vreinterpretq_u8_u32(vdupq_n_u32((IN))) +#define LOADQ_U32P_AS_U8(IN) vreinterpretq_u8_u32(vld1q_u32((IN))) +#define GET_U8_AS_U32(IN) vget_lane_u32(vreinterpret_u32_u8((IN)), 0); +#define GETQ_U8_AS_U32(IN) vgetq_lane_u32(vreinterpretq_u32_u8((IN)), 0); +#define STOREQ_U8_AS_U32P(OUT, IN) vst1q_u32((OUT), vreinterpretq_u32_u8((IN))); +#define ROTATE32_LEFT(L) vextq_u8((L), (L), 12) // D|C|B|A -> C|B|A|D + +static WEBP_INLINE uint8x8_t Average2_u8_NEON(uint32_t a0, uint32_t a1) { + const uint8x8_t A0 = LOAD_U32_AS_U8(a0); + const uint8x8_t A1 = LOAD_U32_AS_U8(a1); + return vhadd_u8(A0, A1); +} + +static WEBP_INLINE uint32_t ClampedAddSubtractHalf_NEON(uint32_t c0, + uint32_t c1, + uint32_t c2) { + const uint8x8_t avg = Average2_u8_NEON(c0, c1); + // Remove one to c2 when bigger than avg. + const uint8x8_t C2 = LOAD_U32_AS_U8(c2); + const uint8x8_t cmp = vcgt_u8(C2, avg); + const uint8x8_t C2_1 = vadd_u8(C2, cmp); + // Compute half of the difference between avg and c2. + const int8x8_t diff_avg = vreinterpret_s8_u8(vhsub_u8(avg, C2_1)); + // Compute the sum with avg and saturate. + const int16x8_t avg_16 = vreinterpretq_s16_u16(vmovl_u8(avg)); + const uint8x8_t res = vqmovun_s16(vaddw_s8(avg_16, diff_avg)); + const uint32_t output = GET_U8_AS_U32(res); + return output; +} + +static WEBP_INLINE uint32_t Average2_NEON(uint32_t a0, uint32_t a1) { + const uint8x8_t avg_u8x8 = Average2_u8_NEON(a0, a1); + const uint32_t avg = GET_U8_AS_U32(avg_u8x8); + return avg; +} + +static WEBP_INLINE uint32_t Average3_NEON(uint32_t a0, uint32_t a1, + uint32_t a2) { + const uint8x8_t avg0 = Average2_u8_NEON(a0, a2); + const uint8x8_t A1 = LOAD_U32_AS_U8(a1); + const uint32_t avg = GET_U8_AS_U32(vhadd_u8(avg0, A1)); + return avg; +} + +static uint32_t Predictor5_NEON(uint32_t left, const uint32_t* const top) { + return Average3_NEON(left, top[0], top[1]); +} +static uint32_t Predictor6_NEON(uint32_t left, const uint32_t* const top) { + return Average2_NEON(left, top[-1]); +} +static uint32_t Predictor7_NEON(uint32_t left, const uint32_t* const top) { + return Average2_NEON(left, top[0]); +} +static uint32_t Predictor13_NEON(uint32_t left, const uint32_t* const top) { + return ClampedAddSubtractHalf_NEON(left, top[0], top[-1]); +} + +// Batch versions of those functions. + +// Predictor0: ARGB_BLACK. +static void PredictorAdd0_NEON(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + const uint8x16_t black = vreinterpretq_u8_u32(vdupq_n_u32(ARGB_BLACK)); + for (i = 0; i + 4 <= num_pixels; i += 4) { + const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]); + const uint8x16_t res = vaddq_u8(src, black); + STOREQ_U8_AS_U32P(&out[i], res); + } + VP8LPredictorsAdd_C[0](in + i, upper + i, num_pixels - i, out + i); +} + +// Predictor1: left. +static void PredictorAdd1_NEON(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + const uint8x16_t zero = LOADQ_U32_AS_U8(0); + for (i = 0; i + 4 <= num_pixels; i += 4) { + // a | b | c | d + const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]); + // 0 | a | b | c + const uint8x16_t shift0 = vextq_u8(zero, src, 12); + // a | a + b | b + c | c + d + const uint8x16_t sum0 = vaddq_u8(src, shift0); + // 0 | 0 | a | a + b + const uint8x16_t shift1 = vextq_u8(zero, sum0, 8); + // a | a + b | a + b + c | a + b + c + d + const uint8x16_t sum1 = vaddq_u8(sum0, shift1); + const uint8x16_t prev = LOADQ_U32_AS_U8(out[i - 1]); + const uint8x16_t res = vaddq_u8(sum1, prev); + STOREQ_U8_AS_U32P(&out[i], res); + } + VP8LPredictorsAdd_C[1](in + i, upper + i, num_pixels - i, out + i); +} + +// Macro that adds 32-bit integers from IN using mod 256 arithmetic +// per 8 bit channel. +#define GENERATE_PREDICTOR_1(X, IN) \ +static void PredictorAdd##X##_NEON(const uint32_t* in, \ + const uint32_t* upper, int num_pixels, \ + uint32_t* out) { \ + int i; \ + for (i = 0; i + 4 <= num_pixels; i += 4) { \ + const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]); \ + const uint8x16_t other = LOADQ_U32P_AS_U8(&(IN)); \ + const uint8x16_t res = vaddq_u8(src, other); \ + STOREQ_U8_AS_U32P(&out[i], res); \ + } \ + VP8LPredictorsAdd_C[(X)](in + i, upper + i, num_pixels - i, out + i); \ +} +// Predictor2: Top. +GENERATE_PREDICTOR_1(2, upper[i]) +// Predictor3: Top-right. +GENERATE_PREDICTOR_1(3, upper[i + 1]) +// Predictor4: Top-left. +GENERATE_PREDICTOR_1(4, upper[i - 1]) +#undef GENERATE_PREDICTOR_1 + +// Predictor5: average(average(left, TR), T) +#define DO_PRED5(LANE) do { \ + const uint8x16_t avgLTR = vhaddq_u8(L, TR); \ + const uint8x16_t avg = vhaddq_u8(avgLTR, T); \ + const uint8x16_t res = vaddq_u8(avg, src); \ + vst1q_lane_u32(&out[i + (LANE)], vreinterpretq_u32_u8(res), (LANE)); \ + L = ROTATE32_LEFT(res); \ +} while (0) + +static void PredictorAdd5_NEON(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + uint8x16_t L = LOADQ_U32_AS_U8(out[-1]); + for (i = 0; i + 4 <= num_pixels; i += 4) { + const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]); + const uint8x16_t T = LOADQ_U32P_AS_U8(&upper[i + 0]); + const uint8x16_t TR = LOADQ_U32P_AS_U8(&upper[i + 1]); + DO_PRED5(0); + DO_PRED5(1); + DO_PRED5(2); + DO_PRED5(3); + } + VP8LPredictorsAdd_C[5](in + i, upper + i, num_pixels - i, out + i); +} +#undef DO_PRED5 + +#define DO_PRED67(LANE) do { \ + const uint8x16_t avg = vhaddq_u8(L, top); \ + const uint8x16_t res = vaddq_u8(avg, src); \ + vst1q_lane_u32(&out[i + (LANE)], vreinterpretq_u32_u8(res), (LANE)); \ + L = ROTATE32_LEFT(res); \ +} while (0) + +// Predictor6: average(left, TL) +static void PredictorAdd6_NEON(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + uint8x16_t L = LOADQ_U32_AS_U8(out[-1]); + for (i = 0; i + 4 <= num_pixels; i += 4) { + const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]); + const uint8x16_t top = LOADQ_U32P_AS_U8(&upper[i - 1]); + DO_PRED67(0); + DO_PRED67(1); + DO_PRED67(2); + DO_PRED67(3); + } + VP8LPredictorsAdd_C[6](in + i, upper + i, num_pixels - i, out + i); +} + +// Predictor7: average(left, T) +static void PredictorAdd7_NEON(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + uint8x16_t L = LOADQ_U32_AS_U8(out[-1]); + for (i = 0; i + 4 <= num_pixels; i += 4) { + const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]); + const uint8x16_t top = LOADQ_U32P_AS_U8(&upper[i]); + DO_PRED67(0); + DO_PRED67(1); + DO_PRED67(2); + DO_PRED67(3); + } + VP8LPredictorsAdd_C[7](in + i, upper + i, num_pixels - i, out + i); +} +#undef DO_PRED67 + +#define GENERATE_PREDICTOR_2(X, IN) \ +static void PredictorAdd##X##_NEON(const uint32_t* in, \ + const uint32_t* upper, int num_pixels, \ + uint32_t* out) { \ + int i; \ + for (i = 0; i + 4 <= num_pixels; i += 4) { \ + const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]); \ + const uint8x16_t Tother = LOADQ_U32P_AS_U8(&(IN)); \ + const uint8x16_t T = LOADQ_U32P_AS_U8(&upper[i]); \ + const uint8x16_t avg = vhaddq_u8(T, Tother); \ + const uint8x16_t res = vaddq_u8(avg, src); \ + STOREQ_U8_AS_U32P(&out[i], res); \ + } \ + VP8LPredictorsAdd_C[(X)](in + i, upper + i, num_pixels - i, out + i); \ +} +// Predictor8: average TL T. +GENERATE_PREDICTOR_2(8, upper[i - 1]) +// Predictor9: average T TR. +GENERATE_PREDICTOR_2(9, upper[i + 1]) +#undef GENERATE_PREDICTOR_2 + +// Predictor10: average of (average of (L,TL), average of (T, TR)). +#define DO_PRED10(LANE) do { \ + const uint8x16_t avgLTL = vhaddq_u8(L, TL); \ + const uint8x16_t avg = vhaddq_u8(avgTTR, avgLTL); \ + const uint8x16_t res = vaddq_u8(avg, src); \ + vst1q_lane_u32(&out[i + (LANE)], vreinterpretq_u32_u8(res), (LANE)); \ + L = ROTATE32_LEFT(res); \ +} while (0) + +static void PredictorAdd10_NEON(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + uint8x16_t L = LOADQ_U32_AS_U8(out[-1]); + for (i = 0; i + 4 <= num_pixels; i += 4) { + const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]); + const uint8x16_t TL = LOADQ_U32P_AS_U8(&upper[i - 1]); + const uint8x16_t T = LOADQ_U32P_AS_U8(&upper[i]); + const uint8x16_t TR = LOADQ_U32P_AS_U8(&upper[i + 1]); + const uint8x16_t avgTTR = vhaddq_u8(T, TR); + DO_PRED10(0); + DO_PRED10(1); + DO_PRED10(2); + DO_PRED10(3); + } + VP8LPredictorsAdd_C[10](in + i, upper + i, num_pixels - i, out + i); +} +#undef DO_PRED10 + +// Predictor11: select. +#define DO_PRED11(LANE) do { \ + const uint8x16_t sumLin = vaddq_u8(L, src); /* in + L */ \ + const uint8x16_t pLTL = vabdq_u8(L, TL); /* |L - TL| */ \ + const uint16x8_t sum_LTL = vpaddlq_u8(pLTL); \ + const uint32x4_t pa = vpaddlq_u16(sum_LTL); \ + const uint32x4_t mask = vcleq_u32(pa, pb); \ + const uint8x16_t res = vbslq_u8(vreinterpretq_u8_u32(mask), sumTin, sumLin); \ + vst1q_lane_u32(&out[i + (LANE)], vreinterpretq_u32_u8(res), (LANE)); \ + L = ROTATE32_LEFT(res); \ +} while (0) + +static void PredictorAdd11_NEON(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + uint8x16_t L = LOADQ_U32_AS_U8(out[-1]); + for (i = 0; i + 4 <= num_pixels; i += 4) { + const uint8x16_t T = LOADQ_U32P_AS_U8(&upper[i]); + const uint8x16_t TL = LOADQ_U32P_AS_U8(&upper[i - 1]); + const uint8x16_t pTTL = vabdq_u8(T, TL); // |T - TL| + const uint16x8_t sum_TTL = vpaddlq_u8(pTTL); + const uint32x4_t pb = vpaddlq_u16(sum_TTL); + const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]); + const uint8x16_t sumTin = vaddq_u8(T, src); // in + T + DO_PRED11(0); + DO_PRED11(1); + DO_PRED11(2); + DO_PRED11(3); + } + VP8LPredictorsAdd_C[11](in + i, upper + i, num_pixels - i, out + i); +} +#undef DO_PRED11 + +// Predictor12: ClampedAddSubtractFull. +#define DO_PRED12(DIFF, LANE) do { \ + const uint8x8_t pred = \ + vqmovun_s16(vaddq_s16(vreinterpretq_s16_u16(L), (DIFF))); \ + const uint8x8_t res = \ + vadd_u8(pred, (LANE <= 1) ? vget_low_u8(src) : vget_high_u8(src)); \ + const uint16x8_t res16 = vmovl_u8(res); \ + vst1_lane_u32(&out[i + (LANE)], vreinterpret_u32_u8(res), (LANE) & 1); \ + /* rotate in the left predictor for next iteration */ \ + L = vextq_u16(res16, res16, 4); \ +} while (0) + +static void PredictorAdd12_NEON(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + uint16x8_t L = vmovl_u8(LOAD_U32_AS_U8(out[-1])); + for (i = 0; i + 4 <= num_pixels; i += 4) { + // load four pixels of source + const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]); + // precompute the difference T - TL once for all, stored as s16 + const uint8x16_t TL = LOADQ_U32P_AS_U8(&upper[i - 1]); + const uint8x16_t T = LOADQ_U32P_AS_U8(&upper[i]); + const int16x8_t diff_lo = + vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(T), vget_low_u8(TL))); + const int16x8_t diff_hi = + vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(T), vget_high_u8(TL))); + // loop over the four reconstructed pixels + DO_PRED12(diff_lo, 0); + DO_PRED12(diff_lo, 1); + DO_PRED12(diff_hi, 2); + DO_PRED12(diff_hi, 3); + } + VP8LPredictorsAdd_C[12](in + i, upper + i, num_pixels - i, out + i); +} +#undef DO_PRED12 + +// Predictor13: ClampedAddSubtractHalf +#define DO_PRED13(LANE, LOW_OR_HI) do { \ + const uint8x16_t avg = vhaddq_u8(L, T); \ + const uint8x16_t cmp = vcgtq_u8(TL, avg); \ + const uint8x16_t TL_1 = vaddq_u8(TL, cmp); \ + /* Compute half of the difference between avg and TL'. */ \ + const int8x8_t diff_avg = \ + vreinterpret_s8_u8(LOW_OR_HI(vhsubq_u8(avg, TL_1))); \ + /* Compute the sum with avg and saturate. */ \ + const int16x8_t avg_16 = vreinterpretq_s16_u16(vmovl_u8(LOW_OR_HI(avg))); \ + const uint8x8_t delta = vqmovun_s16(vaddw_s8(avg_16, diff_avg)); \ + const uint8x8_t res = vadd_u8(LOW_OR_HI(src), delta); \ + const uint8x16_t res2 = vcombine_u8(res, res); \ + vst1_lane_u32(&out[i + (LANE)], vreinterpret_u32_u8(res), (LANE) & 1); \ + L = ROTATE32_LEFT(res2); \ +} while (0) + +static void PredictorAdd13_NEON(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + uint8x16_t L = LOADQ_U32_AS_U8(out[-1]); + for (i = 0; i + 4 <= num_pixels; i += 4) { + const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]); + const uint8x16_t T = LOADQ_U32P_AS_U8(&upper[i]); + const uint8x16_t TL = LOADQ_U32P_AS_U8(&upper[i - 1]); + DO_PRED13(0, vget_low_u8); + DO_PRED13(1, vget_low_u8); + DO_PRED13(2, vget_high_u8); + DO_PRED13(3, vget_high_u8); + } + VP8LPredictorsAdd_C[13](in + i, upper + i, num_pixels - i, out + i); +} +#undef DO_PRED13 + +#undef LOAD_U32_AS_U8 +#undef LOAD_U32P_AS_U8 +#undef LOADQ_U32_AS_U8 +#undef LOADQ_U32P_AS_U8 +#undef GET_U8_AS_U32 +#undef GETQ_U8_AS_U32 +#undef STOREQ_U8_AS_U32P +#undef ROTATE32_LEFT + //------------------------------------------------------------------------------ // Subtract-Green Transform @@ -171,28 +522,30 @@ static WEBP_INLINE uint8x16_t DoGreenShuffle(const uint8x16_t argb, } #endif // USE_VTBLQ -static void AddGreenToBlueAndRed(uint32_t* argb_data, int num_pixels) { - const uint32_t* const end = argb_data + (num_pixels & ~3); +static void AddGreenToBlueAndRed(const uint32_t* src, int num_pixels, + uint32_t* dst) { + const uint32_t* const end = src + (num_pixels & ~3); #ifdef USE_VTBLQ const uint8x16_t shuffle = vld1q_u8(kGreenShuffle); #else const uint8x8_t shuffle = vld1_u8(kGreenShuffle); #endif - for (; argb_data < end; argb_data += 4) { - const uint8x16_t argb = vld1q_u8((uint8_t*)argb_data); + for (; src < end; src += 4, dst += 4) { + const uint8x16_t argb = vld1q_u8((const uint8_t*)src); const uint8x16_t greens = DoGreenShuffle(argb, shuffle); - vst1q_u8((uint8_t*)argb_data, vaddq_u8(argb, greens)); + vst1q_u8((uint8_t*)dst, vaddq_u8(argb, greens)); } // fallthrough and finish off with plain-C - VP8LAddGreenToBlueAndRed_C(argb_data, num_pixels & 3); + VP8LAddGreenToBlueAndRed_C(src, num_pixels & 3, dst); } //------------------------------------------------------------------------------ // Color Transform static void TransformColorInverse(const VP8LMultipliers* const m, - uint32_t* argb_data, int num_pixels) { - // sign-extended multiplying constants, pre-shifted by 6. + const uint32_t* const src, int num_pixels, + uint32_t* dst) { +// sign-extended multiplying constants, pre-shifted by 6. #define CST(X) (((int16_t)(m->X << 8)) >> 6) const int16_t rb[8] = { CST(green_to_blue_), CST(green_to_red_), @@ -219,7 +572,7 @@ static void TransformColorInverse(const VP8LMultipliers* const m, const uint32x4_t mask_ag = vdupq_n_u32(0xff00ff00u); int i; for (i = 0; i + 4 <= num_pixels; i += 4) { - const uint8x16_t in = vld1q_u8((uint8_t*)(argb_data + i)); + const uint8x16_t in = vld1q_u8((const uint8_t*)(src + i)); const uint32x4_t a0g0 = vandq_u32(vreinterpretq_u32_u8(in), mask_ag); // 0 g 0 g const uint8x16_t greens = DoGreenShuffle(in, shuffle); @@ -240,10 +593,10 @@ static void TransformColorInverse(const VP8LMultipliers* const m, // 0 r' 0 b'' const uint16x8_t G = vshrq_n_u16(vreinterpretq_u16_s8(F), 8); const uint32x4_t out = vorrq_u32(vreinterpretq_u32_u16(G), a0g0); - vst1q_u32(argb_data + i, out); + vst1q_u32(dst + i, out); } // Fall-back to C-version for left-overs. - VP8LTransformColorInverse_C(m, argb_data + i, num_pixels - i); + VP8LTransformColorInverse_C(m, src + i, num_pixels - i, dst + i); } #undef USE_VTBLQ @@ -254,6 +607,26 @@ static void TransformColorInverse(const VP8LMultipliers* const m, extern void VP8LDspInitNEON(void); WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInitNEON(void) { + VP8LPredictors[5] = Predictor5_NEON; + VP8LPredictors[6] = Predictor6_NEON; + VP8LPredictors[7] = Predictor7_NEON; + VP8LPredictors[13] = Predictor13_NEON; + + VP8LPredictorsAdd[0] = PredictorAdd0_NEON; + VP8LPredictorsAdd[1] = PredictorAdd1_NEON; + VP8LPredictorsAdd[2] = PredictorAdd2_NEON; + VP8LPredictorsAdd[3] = PredictorAdd3_NEON; + VP8LPredictorsAdd[4] = PredictorAdd4_NEON; + VP8LPredictorsAdd[5] = PredictorAdd5_NEON; + VP8LPredictorsAdd[6] = PredictorAdd6_NEON; + VP8LPredictorsAdd[7] = PredictorAdd7_NEON; + VP8LPredictorsAdd[8] = PredictorAdd8_NEON; + VP8LPredictorsAdd[9] = PredictorAdd9_NEON; + VP8LPredictorsAdd[10] = PredictorAdd10_NEON; + VP8LPredictorsAdd[11] = PredictorAdd11_NEON; + VP8LPredictorsAdd[12] = PredictorAdd12_NEON; + VP8LPredictorsAdd[13] = PredictorAdd13_NEON; + VP8LConvertBGRAToRGBA = ConvertBGRAToRGBA; VP8LConvertBGRAToBGR = ConvertBGRAToBGR; VP8LConvertBGRAToRGB = ConvertBGRAToRGB; diff --git a/src/3rdparty/libwebp/src/dsp/lossless_sse2.c b/src/3rdparty/libwebp/src/dsp/lossless_sse2.c index 2d016c2..15aae93 100644 --- a/src/3rdparty/libwebp/src/dsp/lossless_sse2.c +++ b/src/3rdparty/libwebp/src/dsp/lossless_sse2.c @@ -14,9 +14,12 @@ #include "./dsp.h" #if defined(WEBP_USE_SSE2) + +#include "./common_sse2.h" +#include "./lossless.h" +#include "./lossless_common.h" #include <assert.h> #include <emmintrin.h> -#include "./lossless.h" //------------------------------------------------------------------------------ // Predictor Transform @@ -75,25 +78,44 @@ static WEBP_INLINE uint32_t Select(uint32_t a, uint32_t b, uint32_t c) { return (pa_minus_pb <= 0) ? a : b; } -static WEBP_INLINE __m128i Average2_128i(uint32_t a0, uint32_t a1) { +static WEBP_INLINE void Average2_m128i(const __m128i* const a0, + const __m128i* const a1, + __m128i* const avg) { + // (a + b) >> 1 = ((a + b + 1) >> 1) - ((a ^ b) & 1) + const __m128i ones = _mm_set1_epi8(1); + const __m128i avg1 = _mm_avg_epu8(*a0, *a1); + const __m128i one = _mm_and_si128(_mm_xor_si128(*a0, *a1), ones); + *avg = _mm_sub_epi8(avg1, one); +} + +static WEBP_INLINE void Average2_uint32(const uint32_t a0, const uint32_t a1, + __m128i* const avg) { + // (a + b) >> 1 = ((a + b + 1) >> 1) - ((a ^ b) & 1) + const __m128i ones = _mm_set1_epi8(1); + const __m128i A0 = _mm_cvtsi32_si128(a0); + const __m128i A1 = _mm_cvtsi32_si128(a1); + const __m128i avg1 = _mm_avg_epu8(A0, A1); + const __m128i one = _mm_and_si128(_mm_xor_si128(A0, A1), ones); + *avg = _mm_sub_epi8(avg1, one); +} + +static WEBP_INLINE __m128i Average2_uint32_16(uint32_t a0, uint32_t a1) { const __m128i zero = _mm_setzero_si128(); const __m128i A0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(a0), zero); const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(a1), zero); const __m128i sum = _mm_add_epi16(A1, A0); - const __m128i avg = _mm_srli_epi16(sum, 1); - return avg; + return _mm_srli_epi16(sum, 1); } static WEBP_INLINE uint32_t Average2(uint32_t a0, uint32_t a1) { - const __m128i avg = Average2_128i(a0, a1); - const __m128i A2 = _mm_packus_epi16(avg, avg); - const uint32_t output = _mm_cvtsi128_si32(A2); - return output; + __m128i output; + Average2_uint32(a0, a1, &output); + return _mm_cvtsi128_si32(output); } static WEBP_INLINE uint32_t Average3(uint32_t a0, uint32_t a1, uint32_t a2) { const __m128i zero = _mm_setzero_si128(); - const __m128i avg1 = Average2_128i(a0, a2); + const __m128i avg1 = Average2_uint32_16(a0, a2); const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(a1), zero); const __m128i sum = _mm_add_epi16(avg1, A1); const __m128i avg2 = _mm_srli_epi16(sum, 1); @@ -104,8 +126,8 @@ static WEBP_INLINE uint32_t Average3(uint32_t a0, uint32_t a1, uint32_t a2) { static WEBP_INLINE uint32_t Average4(uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3) { - const __m128i avg1 = Average2_128i(a0, a1); - const __m128i avg2 = Average2_128i(a2, a3); + const __m128i avg1 = Average2_uint32_16(a0, a1); + const __m128i avg2 = Average2_uint32_16(a2, a3); const __m128i sum = _mm_add_epi16(avg2, avg1); const __m128i avg3 = _mm_srli_epi16(sum, 1); const __m128i A0 = _mm_packus_epi16(avg3, avg3); @@ -113,68 +135,289 @@ static WEBP_INLINE uint32_t Average4(uint32_t a0, uint32_t a1, return output; } -static uint32_t Predictor5(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor5_SSE2(uint32_t left, const uint32_t* const top) { const uint32_t pred = Average3(left, top[0], top[1]); return pred; } -static uint32_t Predictor6(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor6_SSE2(uint32_t left, const uint32_t* const top) { const uint32_t pred = Average2(left, top[-1]); return pred; } -static uint32_t Predictor7(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor7_SSE2(uint32_t left, const uint32_t* const top) { const uint32_t pred = Average2(left, top[0]); return pred; } -static uint32_t Predictor8(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor8_SSE2(uint32_t left, const uint32_t* const top) { const uint32_t pred = Average2(top[-1], top[0]); (void)left; return pred; } -static uint32_t Predictor9(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor9_SSE2(uint32_t left, const uint32_t* const top) { const uint32_t pred = Average2(top[0], top[1]); (void)left; return pred; } -static uint32_t Predictor10(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor10_SSE2(uint32_t left, const uint32_t* const top) { const uint32_t pred = Average4(left, top[-1], top[0], top[1]); return pred; } -static uint32_t Predictor11(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor11_SSE2(uint32_t left, const uint32_t* const top) { const uint32_t pred = Select(top[0], left, top[-1]); return pred; } -static uint32_t Predictor12(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor12_SSE2(uint32_t left, const uint32_t* const top) { const uint32_t pred = ClampedAddSubtractFull(left, top[0], top[-1]); return pred; } -static uint32_t Predictor13(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor13_SSE2(uint32_t left, const uint32_t* const top) { const uint32_t pred = ClampedAddSubtractHalf(left, top[0], top[-1]); return pred; } +// Batch versions of those functions. + +// Predictor0: ARGB_BLACK. +static void PredictorAdd0_SSE2(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + const __m128i black = _mm_set1_epi32(ARGB_BLACK); + for (i = 0; i + 4 <= num_pixels; i += 4) { + const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); + const __m128i res = _mm_add_epi8(src, black); + _mm_storeu_si128((__m128i*)&out[i], res); + } + if (i != num_pixels) { + VP8LPredictorsAdd_C[0](in + i, upper + i, num_pixels - i, out + i); + } +} + +// Predictor1: left. +static void PredictorAdd1_SSE2(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + __m128i prev = _mm_set1_epi32(out[-1]); + for (i = 0; i + 4 <= num_pixels; i += 4) { + // a | b | c | d + const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); + // 0 | a | b | c + const __m128i shift0 = _mm_slli_si128(src, 4); + // a | a + b | b + c | c + d + const __m128i sum0 = _mm_add_epi8(src, shift0); + // 0 | 0 | a | a + b + const __m128i shift1 = _mm_slli_si128(sum0, 8); + // a | a + b | a + b + c | a + b + c + d + const __m128i sum1 = _mm_add_epi8(sum0, shift1); + const __m128i res = _mm_add_epi8(sum1, prev); + _mm_storeu_si128((__m128i*)&out[i], res); + // replicate prev output on the four lanes + prev = _mm_shuffle_epi32(res, (3 << 0) | (3 << 2) | (3 << 4) | (3 << 6)); + } + if (i != num_pixels) { + VP8LPredictorsAdd_C[1](in + i, upper + i, num_pixels - i, out + i); + } +} + +// Macro that adds 32-bit integers from IN using mod 256 arithmetic +// per 8 bit channel. +#define GENERATE_PREDICTOR_1(X, IN) \ +static void PredictorAdd##X##_SSE2(const uint32_t* in, const uint32_t* upper, \ + int num_pixels, uint32_t* out) { \ + int i; \ + for (i = 0; i + 4 <= num_pixels; i += 4) { \ + const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \ + const __m128i other = _mm_loadu_si128((const __m128i*)&(IN)); \ + const __m128i res = _mm_add_epi8(src, other); \ + _mm_storeu_si128((__m128i*)&out[i], res); \ + } \ + if (i != num_pixels) { \ + VP8LPredictorsAdd_C[(X)](in + i, upper + i, num_pixels - i, out + i); \ + } \ +} + +// Predictor2: Top. +GENERATE_PREDICTOR_1(2, upper[i]) +// Predictor3: Top-right. +GENERATE_PREDICTOR_1(3, upper[i + 1]) +// Predictor4: Top-left. +GENERATE_PREDICTOR_1(4, upper[i - 1]) +#undef GENERATE_PREDICTOR_1 + +// Due to averages with integers, values cannot be accumulated in parallel for +// predictors 5 to 7. +GENERATE_PREDICTOR_ADD(Predictor5_SSE2, PredictorAdd5_SSE2) +GENERATE_PREDICTOR_ADD(Predictor6_SSE2, PredictorAdd6_SSE2) +GENERATE_PREDICTOR_ADD(Predictor7_SSE2, PredictorAdd7_SSE2) + +#define GENERATE_PREDICTOR_2(X, IN) \ +static void PredictorAdd##X##_SSE2(const uint32_t* in, const uint32_t* upper, \ + int num_pixels, uint32_t* out) { \ + int i; \ + for (i = 0; i + 4 <= num_pixels; i += 4) { \ + const __m128i Tother = _mm_loadu_si128((const __m128i*)&(IN)); \ + const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); \ + const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \ + __m128i avg, res; \ + Average2_m128i(&T, &Tother, &avg); \ + res = _mm_add_epi8(avg, src); \ + _mm_storeu_si128((__m128i*)&out[i], res); \ + } \ + if (i != num_pixels) { \ + VP8LPredictorsAdd_C[(X)](in + i, upper + i, num_pixels - i, out + i); \ + } \ +} +// Predictor8: average TL T. +GENERATE_PREDICTOR_2(8, upper[i - 1]) +// Predictor9: average T TR. +GENERATE_PREDICTOR_2(9, upper[i + 1]) +#undef GENERATE_PREDICTOR_2 + +// Predictor10: average of (average of (L,TL), average of (T, TR)). +static void PredictorAdd10_SSE2(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i, j; + __m128i L = _mm_cvtsi32_si128(out[-1]); + for (i = 0; i + 4 <= num_pixels; i += 4) { + __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); + __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); + const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); + const __m128i TR = _mm_loadu_si128((const __m128i*)&upper[i + 1]); + __m128i avgTTR; + Average2_m128i(&T, &TR, &avgTTR); + for (j = 0; j < 4; ++j) { + __m128i avgLTL, avg; + Average2_m128i(&L, &TL, &avgLTL); + Average2_m128i(&avgTTR, &avgLTL, &avg); + L = _mm_add_epi8(avg, src); + out[i + j] = _mm_cvtsi128_si32(L); + // Rotate the pre-computed values for the next iteration. + avgTTR = _mm_srli_si128(avgTTR, 4); + TL = _mm_srli_si128(TL, 4); + src = _mm_srli_si128(src, 4); + } + } + if (i != num_pixels) { + VP8LPredictorsAdd_C[10](in + i, upper + i, num_pixels - i, out + i); + } +} + +// Predictor11: select. +static void GetSumAbsDiff32(const __m128i* const A, const __m128i* const B, + __m128i* const out) { + // We can unpack with any value on the upper 32 bits, provided it's the same + // on both operands (to that their sum of abs diff is zero). Here we use *A. + const __m128i A_lo = _mm_unpacklo_epi32(*A, *A); + const __m128i B_lo = _mm_unpacklo_epi32(*B, *A); + const __m128i A_hi = _mm_unpackhi_epi32(*A, *A); + const __m128i B_hi = _mm_unpackhi_epi32(*B, *A); + const __m128i s_lo = _mm_sad_epu8(A_lo, B_lo); + const __m128i s_hi = _mm_sad_epu8(A_hi, B_hi); + *out = _mm_packs_epi32(s_lo, s_hi); +} + +static void PredictorAdd11_SSE2(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i, j; + __m128i L = _mm_cvtsi32_si128(out[-1]); + for (i = 0; i + 4 <= num_pixels; i += 4) { + __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); + __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); + __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); + __m128i pa; + GetSumAbsDiff32(&T, &TL, &pa); // pa = sum |T-TL| + for (j = 0; j < 4; ++j) { + const __m128i L_lo = _mm_unpacklo_epi32(L, L); + const __m128i TL_lo = _mm_unpacklo_epi32(TL, L); + const __m128i pb = _mm_sad_epu8(L_lo, TL_lo); // pb = sum |L-TL| + const __m128i mask = _mm_cmpgt_epi32(pb, pa); + const __m128i A = _mm_and_si128(mask, L); + const __m128i B = _mm_andnot_si128(mask, T); + const __m128i pred = _mm_or_si128(A, B); // pred = (L > T)? L : T + L = _mm_add_epi8(src, pred); + out[i + j] = _mm_cvtsi128_si32(L); + // Shift the pre-computed value for the next iteration. + T = _mm_srli_si128(T, 4); + TL = _mm_srli_si128(TL, 4); + src = _mm_srli_si128(src, 4); + pa = _mm_srli_si128(pa, 4); + } + } + if (i != num_pixels) { + VP8LPredictorsAdd_C[11](in + i, upper + i, num_pixels - i, out + i); + } +} + +// Predictor12: ClampedAddSubtractFull. +#define DO_PRED12(DIFF, LANE, OUT) \ +do { \ + const __m128i all = _mm_add_epi16(L, (DIFF)); \ + const __m128i alls = _mm_packus_epi16(all, all); \ + const __m128i res = _mm_add_epi8(src, alls); \ + out[i + (OUT)] = _mm_cvtsi128_si32(res); \ + L = _mm_unpacklo_epi8(res, zero); \ + /* Shift the pre-computed value for the next iteration.*/ \ + if (LANE == 0) (DIFF) = _mm_srli_si128((DIFF), 8); \ + src = _mm_srli_si128(src, 4); \ +} while (0) + +static void PredictorAdd12_SSE2(const uint32_t* in, const uint32_t* upper, + int num_pixels, uint32_t* out) { + int i; + const __m128i zero = _mm_setzero_si128(); + const __m128i L8 = _mm_cvtsi32_si128(out[-1]); + __m128i L = _mm_unpacklo_epi8(L8, zero); + for (i = 0; i + 4 <= num_pixels; i += 4) { + // Load 4 pixels at a time. + __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); + const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); + const __m128i T_lo = _mm_unpacklo_epi8(T, zero); + const __m128i T_hi = _mm_unpackhi_epi8(T, zero); + const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); + const __m128i TL_lo = _mm_unpacklo_epi8(TL, zero); + const __m128i TL_hi = _mm_unpackhi_epi8(TL, zero); + __m128i diff_lo = _mm_sub_epi16(T_lo, TL_lo); + __m128i diff_hi = _mm_sub_epi16(T_hi, TL_hi); + DO_PRED12(diff_lo, 0, 0); + DO_PRED12(diff_lo, 1, 1); + DO_PRED12(diff_hi, 0, 2); + DO_PRED12(diff_hi, 1, 3); + } + if (i != num_pixels) { + VP8LPredictorsAdd_C[12](in + i, upper + i, num_pixels - i, out + i); + } +} +#undef DO_PRED12 + +// Due to averages with integers, values cannot be accumulated in parallel for +// predictors 13. +GENERATE_PREDICTOR_ADD(Predictor13_SSE2, PredictorAdd13_SSE2) + //------------------------------------------------------------------------------ // Subtract-Green Transform -static void AddGreenToBlueAndRed(uint32_t* argb_data, int num_pixels) { +static void AddGreenToBlueAndRed(const uint32_t* const src, int num_pixels, + uint32_t* dst) { int i; for (i = 0; i + 4 <= num_pixels; i += 4) { - const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb + const __m128i in = _mm_loadu_si128((const __m128i*)&src[i]); // argb const __m128i A = _mm_srli_epi16(in, 8); // 0 a 0 g const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0)); const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // 0g0g const __m128i out = _mm_add_epi8(in, C); - _mm_storeu_si128((__m128i*)&argb_data[i], out); + _mm_storeu_si128((__m128i*)&dst[i], out); } // fallthrough and finish off with plain-C - VP8LAddGreenToBlueAndRed_C(argb_data + i, num_pixels - i); + if (i != num_pixels) { + VP8LAddGreenToBlueAndRed_C(src + i, num_pixels - i, dst + i); + } } //------------------------------------------------------------------------------ // Color Transform static void TransformColorInverse(const VP8LMultipliers* const m, - uint32_t* argb_data, int num_pixels) { - // sign-extended multiplying constants, pre-shifted by 5. + const uint32_t* const src, int num_pixels, + uint32_t* dst) { +// sign-extended multiplying constants, pre-shifted by 5. #define CST(X) (((int16_t)(m->X << 8)) >> 5) // sign-extend const __m128i mults_rb = _mm_set_epi16( CST(green_to_red_), CST(green_to_blue_), @@ -188,7 +431,7 @@ static void TransformColorInverse(const VP8LMultipliers* const m, const __m128i mask_ag = _mm_set1_epi32(0xff00ff00); // alpha-green masks int i; for (i = 0; i + 4 <= num_pixels; i += 4) { - const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb + const __m128i in = _mm_loadu_si128((const __m128i*)&src[i]); // argb const __m128i A = _mm_and_si128(in, mask_ag); // a 0 g 0 const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0)); const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // g0g0 @@ -200,15 +443,53 @@ static void TransformColorInverse(const VP8LMultipliers* const m, const __m128i I = _mm_add_epi8(H, F); // r' x b'' 0 const __m128i J = _mm_srli_epi16(I, 8); // 0 r' 0 b'' const __m128i out = _mm_or_si128(J, A); - _mm_storeu_si128((__m128i*)&argb_data[i], out); + _mm_storeu_si128((__m128i*)&dst[i], out); } // Fall-back to C-version for left-overs. - VP8LTransformColorInverse_C(m, argb_data + i, num_pixels - i); + if (i != num_pixels) { + VP8LTransformColorInverse_C(m, src + i, num_pixels - i, dst + i); + } } //------------------------------------------------------------------------------ // Color-space conversion functions +static void ConvertBGRAToRGB(const uint32_t* src, int num_pixels, + uint8_t* dst) { + const __m128i* in = (const __m128i*)src; + __m128i* out = (__m128i*)dst; + + while (num_pixels >= 32) { + // Load the BGRA buffers. + __m128i in0 = _mm_loadu_si128(in + 0); + __m128i in1 = _mm_loadu_si128(in + 1); + __m128i in2 = _mm_loadu_si128(in + 2); + __m128i in3 = _mm_loadu_si128(in + 3); + __m128i in4 = _mm_loadu_si128(in + 4); + __m128i in5 = _mm_loadu_si128(in + 5); + __m128i in6 = _mm_loadu_si128(in + 6); + __m128i in7 = _mm_loadu_si128(in + 7); + VP8L32bToPlanar(&in0, &in1, &in2, &in3); + VP8L32bToPlanar(&in4, &in5, &in6, &in7); + // At this points, in1/in5 contains red only, in2/in6 green only ... + // Pack the colors in 24b RGB. + VP8PlanarTo24b(&in1, &in5, &in2, &in6, &in3, &in7); + _mm_storeu_si128(out + 0, in1); + _mm_storeu_si128(out + 1, in5); + _mm_storeu_si128(out + 2, in2); + _mm_storeu_si128(out + 3, in6); + _mm_storeu_si128(out + 4, in3); + _mm_storeu_si128(out + 5, in7); + in += 8; + out += 6; + num_pixels -= 32; + } + // left-overs + if (num_pixels > 0) { + VP8LConvertBGRAToRGB_C((const uint32_t*)in, num_pixels, (uint8_t*)out); + } +} + static void ConvertBGRAToRGBA(const uint32_t* src, int num_pixels, uint8_t* dst) { const __m128i* in = (const __m128i*)src; @@ -233,7 +514,9 @@ static void ConvertBGRAToRGBA(const uint32_t* src, num_pixels -= 8; } // left-overs - VP8LConvertBGRAToRGBA_C((const uint32_t*)in, num_pixels, (uint8_t*)out); + if (num_pixels > 0) { + VP8LConvertBGRAToRGBA_C((const uint32_t*)in, num_pixels, (uint8_t*)out); + } } static void ConvertBGRAToRGBA4444(const uint32_t* src, @@ -267,7 +550,9 @@ static void ConvertBGRAToRGBA4444(const uint32_t* src, num_pixels -= 8; } // left-overs - VP8LConvertBGRAToRGBA4444_C((const uint32_t*)in, num_pixels, (uint8_t*)out); + if (num_pixels > 0) { + VP8LConvertBGRAToRGBA4444_C((const uint32_t*)in, num_pixels, (uint8_t*)out); + } } static void ConvertBGRAToRGB565(const uint32_t* src, @@ -306,7 +591,9 @@ static void ConvertBGRAToRGB565(const uint32_t* src, num_pixels -= 8; } // left-overs - VP8LConvertBGRAToRGB565_C((const uint32_t*)in, num_pixels, (uint8_t*)out); + if (num_pixels > 0) { + VP8LConvertBGRAToRGB565_C((const uint32_t*)in, num_pixels, (uint8_t*)out); + } } static void ConvertBGRAToBGR(const uint32_t* src, @@ -337,7 +624,9 @@ static void ConvertBGRAToBGR(const uint32_t* src, num_pixels -= 8; } // left-overs - VP8LConvertBGRAToBGR_C((const uint32_t*)in, num_pixels, dst); + if (num_pixels > 0) { + VP8LConvertBGRAToBGR_C((const uint32_t*)in, num_pixels, dst); + } } //------------------------------------------------------------------------------ @@ -346,19 +635,35 @@ static void ConvertBGRAToBGR(const uint32_t* src, extern void VP8LDspInitSSE2(void); WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInitSSE2(void) { - VP8LPredictors[5] = Predictor5; - VP8LPredictors[6] = Predictor6; - VP8LPredictors[7] = Predictor7; - VP8LPredictors[8] = Predictor8; - VP8LPredictors[9] = Predictor9; - VP8LPredictors[10] = Predictor10; - VP8LPredictors[11] = Predictor11; - VP8LPredictors[12] = Predictor12; - VP8LPredictors[13] = Predictor13; + VP8LPredictors[5] = Predictor5_SSE2; + VP8LPredictors[6] = Predictor6_SSE2; + VP8LPredictors[7] = Predictor7_SSE2; + VP8LPredictors[8] = Predictor8_SSE2; + VP8LPredictors[9] = Predictor9_SSE2; + VP8LPredictors[10] = Predictor10_SSE2; + VP8LPredictors[11] = Predictor11_SSE2; + VP8LPredictors[12] = Predictor12_SSE2; + VP8LPredictors[13] = Predictor13_SSE2; + + VP8LPredictorsAdd[0] = PredictorAdd0_SSE2; + VP8LPredictorsAdd[1] = PredictorAdd1_SSE2; + VP8LPredictorsAdd[2] = PredictorAdd2_SSE2; + VP8LPredictorsAdd[3] = PredictorAdd3_SSE2; + VP8LPredictorsAdd[4] = PredictorAdd4_SSE2; + VP8LPredictorsAdd[5] = PredictorAdd5_SSE2; + VP8LPredictorsAdd[6] = PredictorAdd6_SSE2; + VP8LPredictorsAdd[7] = PredictorAdd7_SSE2; + VP8LPredictorsAdd[8] = PredictorAdd8_SSE2; + VP8LPredictorsAdd[9] = PredictorAdd9_SSE2; + VP8LPredictorsAdd[10] = PredictorAdd10_SSE2; + VP8LPredictorsAdd[11] = PredictorAdd11_SSE2; + VP8LPredictorsAdd[12] = PredictorAdd12_SSE2; + VP8LPredictorsAdd[13] = PredictorAdd13_SSE2; VP8LAddGreenToBlueAndRed = AddGreenToBlueAndRed; VP8LTransformColorInverse = TransformColorInverse; + VP8LConvertBGRAToRGB = ConvertBGRAToRGB; VP8LConvertBGRAToRGBA = ConvertBGRAToRGBA; VP8LConvertBGRAToRGBA4444 = ConvertBGRAToRGBA4444; VP8LConvertBGRAToRGB565 = ConvertBGRAToRGB565; diff --git a/src/3rdparty/libwebp/src/dsp/msa_macro.h b/src/3rdparty/libwebp/src/dsp/msa_macro.h index 5c707f4..d0e5f45 100644 --- a/src/3rdparty/libwebp/src/dsp/msa_macro.h +++ b/src/3rdparty/libwebp/src/dsp/msa_macro.h @@ -23,12 +23,24 @@ #ifdef CLANG_BUILD #define ADDVI_H(a, b) __msa_addvi_h((v8i16)a, b) + #define ADDVI_W(a, b) __msa_addvi_w((v4i32)a, b) + #define SRAI_B(a, b) __msa_srai_b((v16i8)a, b) #define SRAI_H(a, b) __msa_srai_h((v8i16)a, b) #define SRAI_W(a, b) __msa_srai_w((v4i32)a, b) + #define SRLI_H(a, b) __msa_srli_h((v8i16)a, b) + #define SLLI_B(a, b) __msa_slli_b((v4i32)a, b) + #define ANDI_B(a, b) __msa_andi_b((v16u8)a, b) + #define ORI_B(a, b) __msa_ori_b((v16u8)a, b) #else #define ADDVI_H(a, b) (a + b) + #define ADDVI_W(a, b) (a + b) + #define SRAI_B(a, b) (a >> b) #define SRAI_H(a, b) (a >> b) #define SRAI_W(a, b) (a >> b) + #define SRLI_H(a, b) (a << b) + #define SLLI_B(a, b) (a << b) + #define ANDI_B(a, b) (a & b) + #define ORI_B(a, b) (a | b) #endif #define LD_B(RTYPE, psrc) *((RTYPE*)(psrc)) @@ -116,13 +128,13 @@ #define SH(val, pdst) MSA_STORE(val, pdst, msa_ush) MSA_STORE_FUNC(uint32_t, usw, msa_usw); #define SW(val, pdst) MSA_STORE(val, pdst, msa_usw) - #define SD(val, pdst) { \ + #define SD(val, pdst) do { \ uint8_t* const pdst_sd_m = (uint8_t*)(pdst); \ const uint32_t val0_m = (uint32_t)(val & 0x00000000FFFFFFFF); \ const uint32_t val1_m = (uint32_t)((val >> 32) & 0x00000000FFFFFFFF); \ SW(val0_m, pdst_sd_m); \ SW(val1_m, pdst_sd_m + 4); \ - } + } while (0) #endif // (__mips_isa_rev >= 6) /* Description : Load 4 words with stride @@ -133,34 +145,68 @@ * Load word in 'out2' from (psrc + 2 * stride) * Load word in 'out3' from (psrc + 3 * stride) */ -#define LW4(psrc, stride, out0, out1, out2, out3) { \ - const uint8_t* ptmp = (const uint8_t*)psrc; \ - out0 = LW(ptmp); \ - ptmp += stride; \ - out1 = LW(ptmp); \ - ptmp += stride; \ - out2 = LW(ptmp); \ - ptmp += stride; \ - out3 = LW(ptmp); \ -} +#define LW4(psrc, stride, out0, out1, out2, out3) do { \ + const uint8_t* ptmp = (const uint8_t*)psrc; \ + out0 = LW(ptmp); \ + ptmp += stride; \ + out1 = LW(ptmp); \ + ptmp += stride; \ + out2 = LW(ptmp); \ + ptmp += stride; \ + out3 = LW(ptmp); \ +} while (0) -/* Description : Store 4 words with stride +/* Description : Store words with stride * Arguments : Inputs - in0, in1, in2, in3, pdst, stride * Details : Store word from 'in0' to (pdst) * Store word from 'in1' to (pdst + stride) * Store word from 'in2' to (pdst + 2 * stride) * Store word from 'in3' to (pdst + 3 * stride) */ -#define SW4(in0, in1, in2, in3, pdst, stride) { \ - uint8_t* ptmp = (uint8_t*)pdst; \ - SW(in0, ptmp); \ - ptmp += stride; \ - SW(in1, ptmp); \ - ptmp += stride; \ - SW(in2, ptmp); \ - ptmp += stride; \ - SW(in3, ptmp); \ -} +#define SW4(in0, in1, in2, in3, pdst, stride) do { \ + uint8_t* ptmp = (uint8_t*)pdst; \ + SW(in0, ptmp); \ + ptmp += stride; \ + SW(in1, ptmp); \ + ptmp += stride; \ + SW(in2, ptmp); \ + ptmp += stride; \ + SW(in3, ptmp); \ +} while (0) + +#define SW3(in0, in1, in2, pdst, stride) do { \ + uint8_t* ptmp = (uint8_t*)pdst; \ + SW(in0, ptmp); \ + ptmp += stride; \ + SW(in1, ptmp); \ + ptmp += stride; \ + SW(in2, ptmp); \ +} while (0) + +#define SW2(in0, in1, pdst, stride) do { \ + uint8_t* ptmp = (uint8_t*)pdst; \ + SW(in0, ptmp); \ + ptmp += stride; \ + SW(in1, ptmp); \ +} while (0) + +/* Description : Store 4 double words with stride + * Arguments : Inputs - in0, in1, in2, in3, pdst, stride + * Details : Store double word from 'in0' to (pdst) + * Store double word from 'in1' to (pdst + stride) + * Store double word from 'in2' to (pdst + 2 * stride) + * Store double word from 'in3' to (pdst + 3 * stride) + */ +#define SD4(in0, in1, in2, in3, pdst, stride) do { \ + uint8_t* ptmp = (uint8_t*)pdst; \ + SD(in0, ptmp); \ + ptmp += stride; \ + SD(in1, ptmp); \ + ptmp += stride; \ + SD(in2, ptmp); \ + ptmp += stride; \ + SD(in3, ptmp); \ +} while (0) /* Description : Load vectors with 16 byte elements with stride * Arguments : Inputs - psrc, stride @@ -169,33 +215,169 @@ * Details : Load 16 byte elements in 'out0' from (psrc) * Load 16 byte elements in 'out1' from (psrc + stride) */ -#define LD_B2(RTYPE, psrc, stride, out0, out1) { \ - out0 = LD_B(RTYPE, psrc); \ - out1 = LD_B(RTYPE, psrc + stride); \ -} +#define LD_B2(RTYPE, psrc, stride, out0, out1) do { \ + out0 = LD_B(RTYPE, psrc); \ + out1 = LD_B(RTYPE, psrc + stride); \ +} while (0) #define LD_UB2(...) LD_B2(v16u8, __VA_ARGS__) #define LD_SB2(...) LD_B2(v16i8, __VA_ARGS__) -#define LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3) { \ - LD_B2(RTYPE, psrc, stride, out0, out1); \ - LD_B2(RTYPE, psrc + 2 * stride , stride, out2, out3); \ -} +#define LD_B3(RTYPE, psrc, stride, out0, out1, out2) do { \ + LD_B2(RTYPE, psrc, stride, out0, out1); \ + out2 = LD_B(RTYPE, psrc + 2 * stride); \ +} while (0) +#define LD_UB3(...) LD_B3(v16u8, __VA_ARGS__) +#define LD_SB3(...) LD_B3(v16i8, __VA_ARGS__) + +#define LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3) do { \ + LD_B2(RTYPE, psrc, stride, out0, out1); \ + LD_B2(RTYPE, psrc + 2 * stride , stride, out2, out3); \ +} while (0) #define LD_UB4(...) LD_B4(v16u8, __VA_ARGS__) #define LD_SB4(...) LD_B4(v16i8, __VA_ARGS__) +#define LD_B8(RTYPE, psrc, stride, \ + out0, out1, out2, out3, out4, out5, out6, out7) do { \ + LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3); \ + LD_B4(RTYPE, psrc + 4 * stride, stride, out4, out5, out6, out7); \ +} while (0) +#define LD_UB8(...) LD_B8(v16u8, __VA_ARGS__) +#define LD_SB8(...) LD_B8(v16i8, __VA_ARGS__) + /* Description : Load vectors with 8 halfword elements with stride * Arguments : Inputs - psrc, stride * Outputs - out0, out1 * Details : Load 8 halfword elements in 'out0' from (psrc) * Load 8 halfword elements in 'out1' from (psrc + stride) */ -#define LD_H2(RTYPE, psrc, stride, out0, out1) { \ - out0 = LD_H(RTYPE, psrc); \ - out1 = LD_H(RTYPE, psrc + stride); \ -} +#define LD_H2(RTYPE, psrc, stride, out0, out1) do { \ + out0 = LD_H(RTYPE, psrc); \ + out1 = LD_H(RTYPE, psrc + stride); \ +} while (0) #define LD_UH2(...) LD_H2(v8u16, __VA_ARGS__) #define LD_SH2(...) LD_H2(v8i16, __VA_ARGS__) +/* Description : Load vectors with 4 word elements with stride + * Arguments : Inputs - psrc, stride + * Outputs - out0, out1, out2, out3 + * Details : Load 4 word elements in 'out0' from (psrc + 0 * stride) + * Load 4 word elements in 'out1' from (psrc + 1 * stride) + * Load 4 word elements in 'out2' from (psrc + 2 * stride) + * Load 4 word elements in 'out3' from (psrc + 3 * stride) + */ +#define LD_W2(RTYPE, psrc, stride, out0, out1) do { \ + out0 = LD_W(RTYPE, psrc); \ + out1 = LD_W(RTYPE, psrc + stride); \ +} while (0) +#define LD_UW2(...) LD_W2(v4u32, __VA_ARGS__) +#define LD_SW2(...) LD_W2(v4i32, __VA_ARGS__) + +#define LD_W3(RTYPE, psrc, stride, out0, out1, out2) do { \ + LD_W2(RTYPE, psrc, stride, out0, out1); \ + out2 = LD_W(RTYPE, psrc + 2 * stride); \ +} while (0) +#define LD_UW3(...) LD_W3(v4u32, __VA_ARGS__) +#define LD_SW3(...) LD_W3(v4i32, __VA_ARGS__) + +#define LD_W4(RTYPE, psrc, stride, out0, out1, out2, out3) do { \ + LD_W2(RTYPE, psrc, stride, out0, out1); \ + LD_W2(RTYPE, psrc + 2 * stride, stride, out2, out3); \ +} while (0) +#define LD_UW4(...) LD_W4(v4u32, __VA_ARGS__) +#define LD_SW4(...) LD_W4(v4i32, __VA_ARGS__) + +/* Description : Store vectors of 16 byte elements with stride + * Arguments : Inputs - in0, in1, pdst, stride + * Details : Store 16 byte elements from 'in0' to (pdst) + * Store 16 byte elements from 'in1' to (pdst + stride) + */ +#define ST_B2(RTYPE, in0, in1, pdst, stride) do { \ + ST_B(RTYPE, in0, pdst); \ + ST_B(RTYPE, in1, pdst + stride); \ +} while (0) +#define ST_UB2(...) ST_B2(v16u8, __VA_ARGS__) +#define ST_SB2(...) ST_B2(v16i8, __VA_ARGS__) + +#define ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride) do { \ + ST_B2(RTYPE, in0, in1, pdst, stride); \ + ST_B2(RTYPE, in2, in3, pdst + 2 * stride, stride); \ +} while (0) +#define ST_UB4(...) ST_B4(v16u8, __VA_ARGS__) +#define ST_SB4(...) ST_B4(v16i8, __VA_ARGS__) + +#define ST_B8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ + pdst, stride) do { \ + ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride); \ + ST_B4(RTYPE, in4, in5, in6, in7, pdst + 4 * stride, stride); \ +} while (0) +#define ST_UB8(...) ST_B8(v16u8, __VA_ARGS__) + +/* Description : Store vectors of 4 word elements with stride + * Arguments : Inputs - in0, in1, in2, in3, pdst, stride + * Details : Store 4 word elements from 'in0' to (pdst + 0 * stride) + * Store 4 word elements from 'in1' to (pdst + 1 * stride) + * Store 4 word elements from 'in2' to (pdst + 2 * stride) + * Store 4 word elements from 'in3' to (pdst + 3 * stride) + */ +#define ST_W2(RTYPE, in0, in1, pdst, stride) do { \ + ST_W(RTYPE, in0, pdst); \ + ST_W(RTYPE, in1, pdst + stride); \ +} while (0) +#define ST_UW2(...) ST_W2(v4u32, __VA_ARGS__) +#define ST_SW2(...) ST_W2(v4i32, __VA_ARGS__) + +#define ST_W3(RTYPE, in0, in1, in2, pdst, stride) do { \ + ST_W2(RTYPE, in0, in1, pdst, stride); \ + ST_W(RTYPE, in2, pdst + 2 * stride); \ +} while (0) +#define ST_UW3(...) ST_W3(v4u32, __VA_ARGS__) +#define ST_SW3(...) ST_W3(v4i32, __VA_ARGS__) + +#define ST_W4(RTYPE, in0, in1, in2, in3, pdst, stride) do { \ + ST_W2(RTYPE, in0, in1, pdst, stride); \ + ST_W2(RTYPE, in2, in3, pdst + 2 * stride, stride); \ +} while (0) +#define ST_UW4(...) ST_W4(v4u32, __VA_ARGS__) +#define ST_SW4(...) ST_W4(v4i32, __VA_ARGS__) + +/* Description : Store vectors of 8 halfword elements with stride + * Arguments : Inputs - in0, in1, pdst, stride + * Details : Store 8 halfword elements from 'in0' to (pdst) + * Store 8 halfword elements from 'in1' to (pdst + stride) + */ +#define ST_H2(RTYPE, in0, in1, pdst, stride) do { \ + ST_H(RTYPE, in0, pdst); \ + ST_H(RTYPE, in1, pdst + stride); \ +} while (0) +#define ST_UH2(...) ST_H2(v8u16, __VA_ARGS__) +#define ST_SH2(...) ST_H2(v8i16, __VA_ARGS__) + +/* Description : Store 2x4 byte block to destination memory from input vector + * Arguments : Inputs - in, stidx, pdst, stride + * Details : Index 'stidx' halfword element from 'in' vector is copied to + * the GP register and stored to (pdst) + * Index 'stidx+1' halfword element from 'in' vector is copied to + * the GP register and stored to (pdst + stride) + * Index 'stidx+2' halfword element from 'in' vector is copied to + * the GP register and stored to (pdst + 2 * stride) + * Index 'stidx+3' halfword element from 'in' vector is copied to + * the GP register and stored to (pdst + 3 * stride) + */ +#define ST2x4_UB(in, stidx, pdst, stride) do { \ + uint8_t* pblk_2x4_m = (uint8_t*)pdst; \ + const uint16_t out0_m = __msa_copy_s_h((v8i16)in, stidx); \ + const uint16_t out1_m = __msa_copy_s_h((v8i16)in, stidx + 1); \ + const uint16_t out2_m = __msa_copy_s_h((v8i16)in, stidx + 2); \ + const uint16_t out3_m = __msa_copy_s_h((v8i16)in, stidx + 3); \ + SH(out0_m, pblk_2x4_m); \ + pblk_2x4_m += stride; \ + SH(out1_m, pblk_2x4_m); \ + pblk_2x4_m += stride; \ + SH(out2_m, pblk_2x4_m); \ + pblk_2x4_m += stride; \ + SH(out3_m, pblk_2x4_m); \ +} while (0) + /* Description : Store 4x4 byte block to destination memory from input vector * Arguments : Inputs - in0, in1, pdst, stride * Details : 'Idx0' word element from input vector 'in0' is copied to the @@ -207,14 +389,20 @@ * 'Idx3' word element from input vector 'in0' is copied to the * GP register and stored to (pdst + 3 * stride) */ -#define ST4x4_UB(in0, in1, idx0, idx1, idx2, idx3, pdst, stride) { \ - uint8_t* const pblk_4x4_m = (uint8_t*)pdst; \ - const uint32_t out0_m = __msa_copy_s_w((v4i32)in0, idx0); \ - const uint32_t out1_m = __msa_copy_s_w((v4i32)in0, idx1); \ - const uint32_t out2_m = __msa_copy_s_w((v4i32)in1, idx2); \ - const uint32_t out3_m = __msa_copy_s_w((v4i32)in1, idx3); \ - SW4(out0_m, out1_m, out2_m, out3_m, pblk_4x4_m, stride); \ -} +#define ST4x4_UB(in0, in1, idx0, idx1, idx2, idx3, pdst, stride) do { \ + uint8_t* const pblk_4x4_m = (uint8_t*)pdst; \ + const uint32_t out0_m = __msa_copy_s_w((v4i32)in0, idx0); \ + const uint32_t out1_m = __msa_copy_s_w((v4i32)in0, idx1); \ + const uint32_t out2_m = __msa_copy_s_w((v4i32)in1, idx2); \ + const uint32_t out3_m = __msa_copy_s_w((v4i32)in1, idx3); \ + SW4(out0_m, out1_m, out2_m, out3_m, pblk_4x4_m, stride); \ +} while (0) + +#define ST4x8_UB(in0, in1, pdst, stride) do { \ + uint8_t* const pblk_4x8 = (uint8_t*)pdst; \ + ST4x4_UB(in0, in0, 0, 1, 2, 3, pblk_4x8, stride); \ + ST4x4_UB(in1, in1, 0, 1, 2, 3, pblk_4x8 + 4 * stride, stride); \ +} while (0) /* Description : Immediate number of elements to slide * Arguments : Inputs - in0, in1, slide_val @@ -230,6 +418,30 @@ #define SLDI_SB(...) SLDI_B(v16i8, __VA_ARGS__) #define SLDI_SH(...) SLDI_B(v8i16, __VA_ARGS__) +/* Description : Shuffle byte vector elements as per mask vector + * Arguments : Inputs - in0, in1, in2, in3, mask0, mask1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Byte elements from 'in0' & 'in1' are copied selectively to + * 'out0' as per control vector 'mask0' + */ +#define VSHF_B(RTYPE, in0, in1, mask) \ + (RTYPE)__msa_vshf_b((v16i8)mask, (v16i8)in1, (v16i8)in0) + +#define VSHF_UB(...) VSHF_B(v16u8, __VA_ARGS__) +#define VSHF_SB(...) VSHF_B(v16i8, __VA_ARGS__) +#define VSHF_UH(...) VSHF_B(v8u16, __VA_ARGS__) +#define VSHF_SH(...) VSHF_B(v8i16, __VA_ARGS__) + +#define VSHF_B2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) do { \ + out0 = VSHF_B(RTYPE, in0, in1, mask0); \ + out1 = VSHF_B(RTYPE, in2, in3, mask1); \ +} while (0) +#define VSHF_B2_UB(...) VSHF_B2(v16u8, __VA_ARGS__) +#define VSHF_B2_SB(...) VSHF_B2(v16i8, __VA_ARGS__) +#define VSHF_B2_UH(...) VSHF_B2(v8u16, __VA_ARGS__) +#define VSHF_B2_SH(...) VSHF_B2(v8i16, __VA_ARGS__) + /* Description : Shuffle halfword vector elements as per mask vector * Arguments : Inputs - in0, in1, in2, in3, mask0, mask1 * Outputs - out0, out1 @@ -237,44 +449,219 @@ * Details : halfword elements from 'in0' & 'in1' are copied selectively to * 'out0' as per control vector 'mask0' */ -#define VSHF_H2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) { \ - out0 = (RTYPE)__msa_vshf_h((v8i16)mask0, (v8i16)in1, (v8i16)in0); \ - out1 = (RTYPE)__msa_vshf_h((v8i16)mask1, (v8i16)in3, (v8i16)in2); \ -} +#define VSHF_H2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) do { \ + out0 = (RTYPE)__msa_vshf_h((v8i16)mask0, (v8i16)in1, (v8i16)in0); \ + out1 = (RTYPE)__msa_vshf_h((v8i16)mask1, (v8i16)in3, (v8i16)in2); \ +} while (0) #define VSHF_H2_UH(...) VSHF_H2(v8u16, __VA_ARGS__) #define VSHF_H2_SH(...) VSHF_H2(v8i16, __VA_ARGS__) +/* Description : Dot product of byte vector elements + * Arguments : Inputs - mult0, mult1, cnst0, cnst1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Signed byte elements from 'mult0' are multiplied with + * signed byte elements from 'cnst0' producing a result + * twice the size of input i.e. signed halfword. + * The multiplication result of adjacent odd-even elements + * are added together and written to the 'out0' vector +*/ +#define DOTP_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \ + out0 = (RTYPE)__msa_dotp_s_h((v16i8)mult0, (v16i8)cnst0); \ + out1 = (RTYPE)__msa_dotp_s_h((v16i8)mult1, (v16i8)cnst1); \ +} while (0) +#define DOTP_SB2_SH(...) DOTP_SB2(v8i16, __VA_ARGS__) + +/* Description : Dot product of halfword vector elements + * Arguments : Inputs - mult0, mult1, cnst0, cnst1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Signed halfword elements from 'mult0' are multiplied with + * signed halfword elements from 'cnst0' producing a result + * twice the size of input i.e. signed word. + * The multiplication result of adjacent odd-even elements + * are added together and written to the 'out0' vector + */ +#define DOTP_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \ + out0 = (RTYPE)__msa_dotp_s_w((v8i16)mult0, (v8i16)cnst0); \ + out1 = (RTYPE)__msa_dotp_s_w((v8i16)mult1, (v8i16)cnst1); \ +} while (0) +#define DOTP_SH2_SW(...) DOTP_SH2(v4i32, __VA_ARGS__) + +/* Description : Dot product of unsigned word vector elements + * Arguments : Inputs - mult0, mult1, cnst0, cnst1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Unsigned word elements from 'mult0' are multiplied with + * unsigned word elements from 'cnst0' producing a result + * twice the size of input i.e. unsigned double word. + * The multiplication result of adjacent odd-even elements + * are added together and written to the 'out0' vector + */ +#define DOTP_UW2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \ + out0 = (RTYPE)__msa_dotp_u_d((v4u32)mult0, (v4u32)cnst0); \ + out1 = (RTYPE)__msa_dotp_u_d((v4u32)mult1, (v4u32)cnst1); \ +} while (0) +#define DOTP_UW2_UD(...) DOTP_UW2(v2u64, __VA_ARGS__) + +/* Description : Dot product & addition of halfword vector elements + * Arguments : Inputs - mult0, mult1, cnst0, cnst1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Signed halfword elements from 'mult0' are multiplied with + * signed halfword elements from 'cnst0' producing a result + * twice the size of input i.e. signed word. + * The multiplication result of adjacent odd-even elements + * are added to the 'out0' vector + */ +#define DPADD_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \ + out0 = (RTYPE)__msa_dpadd_s_w((v4i32)out0, (v8i16)mult0, (v8i16)cnst0); \ + out1 = (RTYPE)__msa_dpadd_s_w((v4i32)out1, (v8i16)mult1, (v8i16)cnst1); \ +} while (0) +#define DPADD_SH2_SW(...) DPADD_SH2(v4i32, __VA_ARGS__) + /* Description : Clips all signed halfword elements of input vector * between 0 & 255 * Arguments : Input/output - val * Return Type - signed halfword */ -#define CLIP_SH_0_255(val) { \ +#define CLIP_SH_0_255(val) do { \ const v8i16 max_m = __msa_ldi_h(255); \ val = __msa_maxi_s_h((v8i16)val, 0); \ val = __msa_min_s_h(max_m, (v8i16)val); \ -} -#define CLIP_SH2_0_255(in0, in1) { \ - CLIP_SH_0_255(in0); \ - CLIP_SH_0_255(in1); \ -} +} while (0) + +#define CLIP_SH2_0_255(in0, in1) do { \ + CLIP_SH_0_255(in0); \ + CLIP_SH_0_255(in1); \ +} while (0) + +#define CLIP_SH4_0_255(in0, in1, in2, in3) do { \ + CLIP_SH2_0_255(in0, in1); \ + CLIP_SH2_0_255(in2, in3); \ +} while (0) + +/* Description : Clips all unsigned halfword elements of input vector + * between 0 & 255 + * Arguments : Input - in + * Output - out_m + * Return Type - unsigned halfword + */ +#define CLIP_UH_0_255(in) do { \ + const v8u16 max_m = (v8u16)__msa_ldi_h(255); \ + in = __msa_maxi_u_h((v8u16) in, 0); \ + in = __msa_min_u_h((v8u16) max_m, (v8u16) in); \ +} while (0) + +#define CLIP_UH2_0_255(in0, in1) do { \ + CLIP_UH_0_255(in0); \ + CLIP_UH_0_255(in1); \ +} while (0) /* Description : Clips all signed word elements of input vector * between 0 & 255 * Arguments : Input/output - val * Return Type - signed word */ -#define CLIP_SW_0_255(val) { \ +#define CLIP_SW_0_255(val) do { \ const v4i32 max_m = __msa_ldi_w(255); \ val = __msa_maxi_s_w((v4i32)val, 0); \ val = __msa_min_s_w(max_m, (v4i32)val); \ +} while (0) + +#define CLIP_SW4_0_255(in0, in1, in2, in3) do { \ + CLIP_SW_0_255(in0); \ + CLIP_SW_0_255(in1); \ + CLIP_SW_0_255(in2); \ + CLIP_SW_0_255(in3); \ +} while (0) + +/* Description : Horizontal addition of 4 signed word elements of input vector + * Arguments : Input - in (signed word vector) + * Output - sum_m (i32 sum) + * Return Type - signed word (GP) + * Details : 4 signed word elements of 'in' vector are added together and + * the resulting integer sum is returned + */ +static WEBP_INLINE int32_t func_hadd_sw_s32(v4i32 in) { + const v2i64 res0_m = __msa_hadd_s_d((v4i32)in, (v4i32)in); + const v2i64 res1_m = __msa_splati_d(res0_m, 1); + const v2i64 out = res0_m + res1_m; + int32_t sum_m = __msa_copy_s_w((v4i32)out, 0); + return sum_m; } -#define CLIP_SW4_0_255(in0, in1, in2, in3) { \ - CLIP_SW_0_255(in0); \ - CLIP_SW_0_255(in1); \ - CLIP_SW_0_255(in2); \ - CLIP_SW_0_255(in3); \ +#define HADD_SW_S32(in) func_hadd_sw_s32(in) + +/* Description : Horizontal addition of 8 signed halfword elements + * Arguments : Input - in (signed halfword vector) + * Output - sum_m (s32 sum) + * Return Type - signed word + * Details : 8 signed halfword elements of input vector are added + * together and the resulting integer sum is returned + */ +static WEBP_INLINE int32_t func_hadd_sh_s32(v8i16 in) { + const v4i32 res = __msa_hadd_s_w(in, in); + const v2i64 res0 = __msa_hadd_s_d(res, res); + const v2i64 res1 = __msa_splati_d(res0, 1); + const v2i64 res2 = res0 + res1; + const int32_t sum_m = __msa_copy_s_w((v4i32)res2, 0); + return sum_m; +} +#define HADD_SH_S32(in) func_hadd_sh_s32(in) + +/* Description : Horizontal addition of 8 unsigned halfword elements + * Arguments : Input - in (unsigned halfword vector) + * Output - sum_m (u32 sum) + * Return Type - unsigned word + * Details : 8 unsigned halfword elements of input vector are added + * together and the resulting integer sum is returned + */ +static WEBP_INLINE uint32_t func_hadd_uh_u32(v8u16 in) { + uint32_t sum_m; + const v4u32 res_m = __msa_hadd_u_w(in, in); + v2u64 res0_m = __msa_hadd_u_d(res_m, res_m); + v2u64 res1_m = (v2u64)__msa_splati_d((v2i64)res0_m, 1); + res0_m = res0_m + res1_m; + sum_m = __msa_copy_s_w((v4i32)res0_m, 0); + return sum_m; } +#define HADD_UH_U32(in) func_hadd_uh_u32(in) + +/* Description : Horizontal addition of signed half word vector elements + Arguments : Inputs - in0, in1 + Outputs - out0, out1 + Return Type - as per RTYPE + Details : Each signed odd half word element from 'in0' is added to + even signed half word element from 'in0' (pairwise) and the + halfword result is written in 'out0' +*/ +#define HADD_SH2(RTYPE, in0, in1, out0, out1) do { \ + out0 = (RTYPE)__msa_hadd_s_w((v8i16)in0, (v8i16)in0); \ + out1 = (RTYPE)__msa_hadd_s_w((v8i16)in1, (v8i16)in1); \ +} while (0) +#define HADD_SH2_SW(...) HADD_SH2(v4i32, __VA_ARGS__) + +#define HADD_SH4(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) do { \ + HADD_SH2(RTYPE, in0, in1, out0, out1); \ + HADD_SH2(RTYPE, in2, in3, out2, out3); \ +} while (0) +#define HADD_SH4_SW(...) HADD_SH4(v4i32, __VA_ARGS__) + +/* Description : Horizontal subtraction of unsigned byte vector elements + * Arguments : Inputs - in0, in1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Each unsigned odd byte element from 'in0' is subtracted from + * even unsigned byte element from 'in0' (pairwise) and the + * halfword result is written to 'out0' + */ +#define HSUB_UB2(RTYPE, in0, in1, out0, out1) do { \ + out0 = (RTYPE)__msa_hsub_u_h((v16u8)in0, (v16u8)in0); \ + out1 = (RTYPE)__msa_hsub_u_h((v16u8)in1, (v16u8)in1); \ +} while (0) +#define HSUB_UB2_UH(...) HSUB_UB2(v8u16, __VA_ARGS__) +#define HSUB_UB2_SH(...) HSUB_UB2(v8i16, __VA_ARGS__) +#define HSUB_UB2_SW(...) HSUB_UB2(v4i32, __VA_ARGS__) /* Description : Set element n input vector to GPR value * Arguments : Inputs - in0, in1, in2, in3 @@ -282,23 +669,188 @@ * Return Type - as per RTYPE * Details : Set element 0 in vector 'out' to value specified in 'in0' */ -#define INSERT_W2(RTYPE, in0, in1, out) { \ +#define INSERT_W2(RTYPE, in0, in1, out) do { \ out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0); \ out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1); \ -} +} while (0) #define INSERT_W2_UB(...) INSERT_W2(v16u8, __VA_ARGS__) #define INSERT_W2_SB(...) INSERT_W2(v16i8, __VA_ARGS__) -#define INSERT_W4(RTYPE, in0, in1, in2, in3, out) { \ - out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0); \ - out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1); \ - out = (RTYPE)__msa_insert_w((v4i32)out, 2, in2); \ - out = (RTYPE)__msa_insert_w((v4i32)out, 3, in3); \ -} +#define INSERT_W4(RTYPE, in0, in1, in2, in3, out) do { \ + out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0); \ + out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1); \ + out = (RTYPE)__msa_insert_w((v4i32)out, 2, in2); \ + out = (RTYPE)__msa_insert_w((v4i32)out, 3, in3); \ +} while (0) #define INSERT_W4_UB(...) INSERT_W4(v16u8, __VA_ARGS__) #define INSERT_W4_SB(...) INSERT_W4(v16i8, __VA_ARGS__) #define INSERT_W4_SW(...) INSERT_W4(v4i32, __VA_ARGS__) +/* Description : Set element n of double word input vector to GPR value + * Arguments : Inputs - in0, in1 + * Output - out + * Return Type - as per RTYPE + * Details : Set element 0 in vector 'out' to GPR value specified in 'in0' + * Set element 1 in vector 'out' to GPR value specified in 'in1' + */ +#define INSERT_D2(RTYPE, in0, in1, out) do { \ + out = (RTYPE)__msa_insert_d((v2i64)out, 0, in0); \ + out = (RTYPE)__msa_insert_d((v2i64)out, 1, in1); \ +} while (0) +#define INSERT_D2_UB(...) INSERT_D2(v16u8, __VA_ARGS__) +#define INSERT_D2_SB(...) INSERT_D2(v16i8, __VA_ARGS__) + +/* Description : Interleave even byte elements from vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Even byte elements of 'in0' and 'in1' are interleaved + * and written to 'out0' + */ +#define ILVEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_ilvev_b((v16i8)in1, (v16i8)in0); \ + out1 = (RTYPE)__msa_ilvev_b((v16i8)in3, (v16i8)in2); \ +} while (0) +#define ILVEV_B2_UB(...) ILVEV_B2(v16u8, __VA_ARGS__) +#define ILVEV_B2_SB(...) ILVEV_B2(v16i8, __VA_ARGS__) +#define ILVEV_B2_UH(...) ILVEV_B2(v8u16, __VA_ARGS__) +#define ILVEV_B2_SH(...) ILVEV_B2(v8i16, __VA_ARGS__) +#define ILVEV_B2_SD(...) ILVEV_B2(v2i64, __VA_ARGS__) + +/* Description : Interleave odd byte elements from vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Odd byte elements of 'in0' and 'in1' are interleaved + * and written to 'out0' + */ +#define ILVOD_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_ilvod_b((v16i8)in1, (v16i8)in0); \ + out1 = (RTYPE)__msa_ilvod_b((v16i8)in3, (v16i8)in2); \ +} while (0) +#define ILVOD_B2_UB(...) ILVOD_B2(v16u8, __VA_ARGS__) +#define ILVOD_B2_SB(...) ILVOD_B2(v16i8, __VA_ARGS__) +#define ILVOD_B2_UH(...) ILVOD_B2(v8u16, __VA_ARGS__) +#define ILVOD_B2_SH(...) ILVOD_B2(v8i16, __VA_ARGS__) +#define ILVOD_B2_SD(...) ILVOD_B2(v2i64, __VA_ARGS__) + +/* Description : Interleave even halfword elements from vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Even halfword elements of 'in0' and 'in1' are interleaved + * and written to 'out0' + */ +#define ILVEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_ilvev_h((v8i16)in1, (v8i16)in0); \ + out1 = (RTYPE)__msa_ilvev_h((v8i16)in3, (v8i16)in2); \ +} while (0) +#define ILVEV_H2_UB(...) ILVEV_H2(v16u8, __VA_ARGS__) +#define ILVEV_H2_UH(...) ILVEV_H2(v8u16, __VA_ARGS__) +#define ILVEV_H2_SH(...) ILVEV_H2(v8i16, __VA_ARGS__) +#define ILVEV_H2_SW(...) ILVEV_H2(v4i32, __VA_ARGS__) + +/* Description : Interleave odd halfword elements from vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Odd halfword elements of 'in0' and 'in1' are interleaved + * and written to 'out0' + */ +#define ILVOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_ilvod_h((v8i16)in1, (v8i16)in0); \ + out1 = (RTYPE)__msa_ilvod_h((v8i16)in3, (v8i16)in2); \ +} while (0) +#define ILVOD_H2_UB(...) ILVOD_H2(v16u8, __VA_ARGS__) +#define ILVOD_H2_UH(...) ILVOD_H2(v8u16, __VA_ARGS__) +#define ILVOD_H2_SH(...) ILVOD_H2(v8i16, __VA_ARGS__) +#define ILVOD_H2_SW(...) ILVOD_H2(v4i32, __VA_ARGS__) + +/* Description : Interleave even word elements from vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Even word elements of 'in0' and 'in1' are interleaved + * and written to 'out0' + */ +#define ILVEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_ilvev_w((v4i32)in1, (v4i32)in0); \ + out1 = (RTYPE)__msa_ilvev_w((v4i32)in3, (v4i32)in2); \ +} while (0) +#define ILVEV_W2_UB(...) ILVEV_W2(v16u8, __VA_ARGS__) +#define ILVEV_W2_SB(...) ILVEV_W2(v16i8, __VA_ARGS__) +#define ILVEV_W2_UH(...) ILVEV_W2(v8u16, __VA_ARGS__) +#define ILVEV_W2_SD(...) ILVEV_W2(v2i64, __VA_ARGS__) + +/* Description : Interleave even-odd word elements from vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Even word elements of 'in0' and 'in1' are interleaved + * and written to 'out0' + * Odd word elements of 'in2' and 'in3' are interleaved + * and written to 'out1' + */ +#define ILVEVOD_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_ilvev_w((v4i32)in1, (v4i32)in0); \ + out1 = (RTYPE)__msa_ilvod_w((v4i32)in3, (v4i32)in2); \ +} while (0) +#define ILVEVOD_W2_UB(...) ILVEVOD_W2(v16u8, __VA_ARGS__) +#define ILVEVOD_W2_UH(...) ILVEVOD_W2(v8u16, __VA_ARGS__) +#define ILVEVOD_W2_SH(...) ILVEVOD_W2(v8i16, __VA_ARGS__) +#define ILVEVOD_W2_SW(...) ILVEVOD_W2(v4i32, __VA_ARGS__) + +/* Description : Interleave even-odd half-word elements from vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Even half-word elements of 'in0' and 'in1' are interleaved + * and written to 'out0' + * Odd half-word elements of 'in2' and 'in3' are interleaved + * and written to 'out1' + */ +#define ILVEVOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_ilvev_h((v8i16)in1, (v8i16)in0); \ + out1 = (RTYPE)__msa_ilvod_h((v8i16)in3, (v8i16)in2); \ +} while (0) +#define ILVEVOD_H2_UB(...) ILVEVOD_H2(v16u8, __VA_ARGS__) +#define ILVEVOD_H2_UH(...) ILVEVOD_H2(v8u16, __VA_ARGS__) +#define ILVEVOD_H2_SH(...) ILVEVOD_H2(v8i16, __VA_ARGS__) +#define ILVEVOD_H2_SW(...) ILVEVOD_H2(v4i32, __VA_ARGS__) + +/* Description : Interleave even double word elements from vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Even double word elements of 'in0' and 'in1' are interleaved + * and written to 'out0' + */ +#define ILVEV_D2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_ilvev_d((v2i64)in1, (v2i64)in0); \ + out1 = (RTYPE)__msa_ilvev_d((v2i64)in3, (v2i64)in2); \ +} while (0) +#define ILVEV_D2_UB(...) ILVEV_D2(v16u8, __VA_ARGS__) +#define ILVEV_D2_SB(...) ILVEV_D2(v16i8, __VA_ARGS__) +#define ILVEV_D2_SW(...) ILVEV_D2(v4i32, __VA_ARGS__) +#define ILVEV_D2_SD(...) ILVEV_D2(v2i64, __VA_ARGS__) + +/* Description : Interleave left half of byte elements from vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Left half of byte elements of 'in0' and 'in1' are interleaved + * and written to 'out0'. + */ +#define ILVL_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_ilvl_b((v16i8)in0, (v16i8)in1); \ + out1 = (RTYPE)__msa_ilvl_b((v16i8)in2, (v16i8)in3); \ +} while (0) +#define ILVL_B2_UB(...) ILVL_B2(v16u8, __VA_ARGS__) +#define ILVL_B2_SB(...) ILVL_B2(v16i8, __VA_ARGS__) +#define ILVL_B2_UH(...) ILVL_B2(v8u16, __VA_ARGS__) +#define ILVL_B2_SH(...) ILVL_B2(v8i16, __VA_ARGS__) +#define ILVL_B2_SW(...) ILVL_B2(v4i32, __VA_ARGS__) + /* Description : Interleave right half of byte elements from vectors * Arguments : Inputs - in0, in1, in2, in3 * Outputs - out0, out1 @@ -306,10 +858,10 @@ * Details : Right half of byte elements of 'in0' and 'in1' are interleaved * and written to out0. */ -#define ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1) { \ - out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1); \ - out1 = (RTYPE)__msa_ilvr_b((v16i8)in2, (v16i8)in3); \ -} +#define ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1); \ + out1 = (RTYPE)__msa_ilvr_b((v16i8)in2, (v16i8)in3); \ +} while (0) #define ILVR_B2_UB(...) ILVR_B2(v16u8, __VA_ARGS__) #define ILVR_B2_SB(...) ILVR_B2(v16i8, __VA_ARGS__) #define ILVR_B2_UH(...) ILVR_B2(v8u16, __VA_ARGS__) @@ -317,10 +869,10 @@ #define ILVR_B2_SW(...) ILVR_B2(v4i32, __VA_ARGS__) #define ILVR_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ - out0, out1, out2, out3) { \ + out0, out1, out2, out3) do { \ ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ ILVR_B2(RTYPE, in4, in5, in6, in7, out2, out3); \ -} +} while (0) #define ILVR_B4_UB(...) ILVR_B4(v16u8, __VA_ARGS__) #define ILVR_B4_SB(...) ILVR_B4(v16i8, __VA_ARGS__) #define ILVR_B4_UH(...) ILVR_B4(v8u16, __VA_ARGS__) @@ -334,19 +886,19 @@ * Details : Right half of halfword elements of 'in0' and 'in1' are * interleaved and written to 'out0'. */ -#define ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1) { \ - out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1); \ - out1 = (RTYPE)__msa_ilvr_h((v8i16)in2, (v8i16)in3); \ -} +#define ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1); \ + out1 = (RTYPE)__msa_ilvr_h((v8i16)in2, (v8i16)in3); \ +} while (0) #define ILVR_H2_UB(...) ILVR_H2(v16u8, __VA_ARGS__) #define ILVR_H2_SH(...) ILVR_H2(v8i16, __VA_ARGS__) #define ILVR_H2_SW(...) ILVR_H2(v4i32, __VA_ARGS__) #define ILVR_H4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ - out0, out1, out2, out3) { \ + out0, out1, out2, out3) do { \ ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1); \ ILVR_H2(RTYPE, in4, in5, in6, in7, out2, out3); \ -} +} while (0) #define ILVR_H4_UB(...) ILVR_H4(v16u8, __VA_ARGS__) #define ILVR_H4_SH(...) ILVR_H4(v8i16, __VA_ARGS__) #define ILVR_H4_SW(...) ILVR_H4(v4i32, __VA_ARGS__) @@ -358,31 +910,57 @@ * Details : Right half of double word elements of 'in0' and 'in1' are * interleaved and written to 'out0'. */ -#define ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1) { \ - out0 = (RTYPE)__msa_ilvr_d((v2i64)in0, (v2i64)in1); \ - out1 = (RTYPE)__msa_ilvr_d((v2i64)in2, (v2i64)in3); \ -} +#define ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_ilvr_d((v2i64)in0, (v2i64)in1); \ + out1 = (RTYPE)__msa_ilvr_d((v2i64)in2, (v2i64)in3); \ +} while (0) #define ILVR_D2_UB(...) ILVR_D2(v16u8, __VA_ARGS__) #define ILVR_D2_SB(...) ILVR_D2(v16i8, __VA_ARGS__) #define ILVR_D2_SH(...) ILVR_D2(v8i16, __VA_ARGS__) -#define ILVRL_H2(RTYPE, in0, in1, out0, out1) { \ +#define ILVR_D4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3) do { \ + ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1); \ + ILVR_D2(RTYPE, in4, in5, in6, in7, out2, out3); \ +} while (0) +#define ILVR_D4_SB(...) ILVR_D4(v16i8, __VA_ARGS__) +#define ILVR_D4_UB(...) ILVR_D4(v16u8, __VA_ARGS__) + +/* Description : Interleave both left and right half of input vectors + * Arguments : Inputs - in0, in1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Right half of byte elements from 'in0' and 'in1' are + * interleaved and written to 'out0' + */ +#define ILVRL_B2(RTYPE, in0, in1, out0, out1) do { \ + out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1); \ + out1 = (RTYPE)__msa_ilvl_b((v16i8)in0, (v16i8)in1); \ +} while (0) +#define ILVRL_B2_UB(...) ILVRL_B2(v16u8, __VA_ARGS__) +#define ILVRL_B2_SB(...) ILVRL_B2(v16i8, __VA_ARGS__) +#define ILVRL_B2_UH(...) ILVRL_B2(v8u16, __VA_ARGS__) +#define ILVRL_B2_SH(...) ILVRL_B2(v8i16, __VA_ARGS__) +#define ILVRL_B2_SW(...) ILVRL_B2(v4i32, __VA_ARGS__) + +#define ILVRL_H2(RTYPE, in0, in1, out0, out1) do { \ out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1); \ out1 = (RTYPE)__msa_ilvl_h((v8i16)in0, (v8i16)in1); \ -} +} while (0) #define ILVRL_H2_UB(...) ILVRL_H2(v16u8, __VA_ARGS__) #define ILVRL_H2_SB(...) ILVRL_H2(v16i8, __VA_ARGS__) #define ILVRL_H2_SH(...) ILVRL_H2(v8i16, __VA_ARGS__) #define ILVRL_H2_SW(...) ILVRL_H2(v4i32, __VA_ARGS__) #define ILVRL_H2_UW(...) ILVRL_H2(v4u32, __VA_ARGS__) -#define ILVRL_W2(RTYPE, in0, in1, out0, out1) { \ +#define ILVRL_W2(RTYPE, in0, in1, out0, out1) do { \ out0 = (RTYPE)__msa_ilvr_w((v4i32)in0, (v4i32)in1); \ out1 = (RTYPE)__msa_ilvl_w((v4i32)in0, (v4i32)in1); \ -} +} while (0) #define ILVRL_W2_UB(...) ILVRL_W2(v16u8, __VA_ARGS__) #define ILVRL_W2_SH(...) ILVRL_W2(v8i16, __VA_ARGS__) #define ILVRL_W2_SW(...) ILVRL_W2(v4i32, __VA_ARGS__) +#define ILVRL_W2_UW(...) ILVRL_W2(v4u32, __VA_ARGS__) /* Description : Pack even byte elements of vector pairs * Arguments : Inputs - in0, in1, in2, in3 @@ -392,15 +970,76 @@ * 'out0' & even byte elements of 'in1' are copied to the right * half of 'out0'. */ -#define PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) { \ - out0 = (RTYPE)__msa_pckev_b((v16i8)in0, (v16i8)in1); \ - out1 = (RTYPE)__msa_pckev_b((v16i8)in2, (v16i8)in3); \ -} +#define PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_pckev_b((v16i8)in0, (v16i8)in1); \ + out1 = (RTYPE)__msa_pckev_b((v16i8)in2, (v16i8)in3); \ +} while (0) #define PCKEV_B2_SB(...) PCKEV_B2(v16i8, __VA_ARGS__) #define PCKEV_B2_UB(...) PCKEV_B2(v16u8, __VA_ARGS__) #define PCKEV_B2_SH(...) PCKEV_B2(v8i16, __VA_ARGS__) #define PCKEV_B2_SW(...) PCKEV_B2(v4i32, __VA_ARGS__) +#define PCKEV_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3) do { \ + PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ + PCKEV_B2(RTYPE, in4, in5, in6, in7, out2, out3); \ +} while (0) +#define PCKEV_B4_SB(...) PCKEV_B4(v16i8, __VA_ARGS__) +#define PCKEV_B4_UB(...) PCKEV_B4(v16u8, __VA_ARGS__) +#define PCKEV_B4_SH(...) PCKEV_B4(v8i16, __VA_ARGS__) +#define PCKEV_B4_SW(...) PCKEV_B4(v4i32, __VA_ARGS__) + +/* Description : Pack even halfword elements of vector pairs + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Even halfword elements of 'in0' are copied to the left half of + * 'out0' & even halfword elements of 'in1' are copied to the + * right half of 'out0'. + */ +#define PCKEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_pckev_h((v8i16)in0, (v8i16)in1); \ + out1 = (RTYPE)__msa_pckev_h((v8i16)in2, (v8i16)in3); \ +} while (0) +#define PCKEV_H2_UH(...) PCKEV_H2(v8u16, __VA_ARGS__) +#define PCKEV_H2_SH(...) PCKEV_H2(v8i16, __VA_ARGS__) +#define PCKEV_H2_SW(...) PCKEV_H2(v4i32, __VA_ARGS__) +#define PCKEV_H2_UW(...) PCKEV_H2(v4u32, __VA_ARGS__) + +/* Description : Pack even word elements of vector pairs + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Even word elements of 'in0' are copied to the left half of + * 'out0' & even word elements of 'in1' are copied to the + * right half of 'out0'. + */ +#define PCKEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_pckev_w((v4i32)in0, (v4i32)in1); \ + out1 = (RTYPE)__msa_pckev_w((v4i32)in2, (v4i32)in3); \ +} while (0) +#define PCKEV_W2_UH(...) PCKEV_W2(v8u16, __VA_ARGS__) +#define PCKEV_W2_SH(...) PCKEV_W2(v8i16, __VA_ARGS__) +#define PCKEV_W2_SW(...) PCKEV_W2(v4i32, __VA_ARGS__) +#define PCKEV_W2_UW(...) PCKEV_W2(v4u32, __VA_ARGS__) + +/* Description : Pack odd halfword elements of vector pairs + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Odd halfword elements of 'in0' are copied to the left half of + * 'out0' & odd halfword elements of 'in1' are copied to the + * right half of 'out0'. + */ +#define PCKOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_pckod_h((v8i16)in0, (v8i16)in1); \ + out1 = (RTYPE)__msa_pckod_h((v8i16)in2, (v8i16)in3); \ +} while (0) +#define PCKOD_H2_UH(...) PCKOD_H2(v8u16, __VA_ARGS__) +#define PCKOD_H2_SH(...) PCKOD_H2(v8i16, __VA_ARGS__) +#define PCKOD_H2_SW(...) PCKOD_H2(v4i32, __VA_ARGS__) +#define PCKOD_H2_UW(...) PCKOD_H2(v4u32, __VA_ARGS__) + /* Description : Arithmetic immediate shift right all elements of word vector * Arguments : Inputs - in0, in1, shift * Outputs - in place operation @@ -408,17 +1047,17 @@ * Details : Each element of vector 'in0' is right shifted by 'shift' and * the result is written in-place. 'shift' is a GP variable. */ -#define SRAI_W2(RTYPE, in0, in1, shift_val) { \ - in0 = (RTYPE)SRAI_W(in0, shift_val); \ - in1 = (RTYPE)SRAI_W(in1, shift_val); \ -} +#define SRAI_W2(RTYPE, in0, in1, shift_val) do { \ + in0 = (RTYPE)SRAI_W(in0, shift_val); \ + in1 = (RTYPE)SRAI_W(in1, shift_val); \ +} while (0) #define SRAI_W2_SW(...) SRAI_W2(v4i32, __VA_ARGS__) #define SRAI_W2_UW(...) SRAI_W2(v4u32, __VA_ARGS__) -#define SRAI_W4(RTYPE, in0, in1, in2, in3, shift_val) { \ - SRAI_W2(RTYPE, in0, in1, shift_val); \ - SRAI_W2(RTYPE, in2, in3, shift_val); \ -} +#define SRAI_W4(RTYPE, in0, in1, in2, in3, shift_val) do { \ + SRAI_W2(RTYPE, in0, in1, shift_val); \ + SRAI_W2(RTYPE, in2, in3, shift_val); \ +} while (0) #define SRAI_W4_SW(...) SRAI_W4(v4i32, __VA_ARGS__) #define SRAI_W4_UW(...) SRAI_W4(v4u32, __VA_ARGS__) @@ -429,10 +1068,10 @@ * Details : Each element of vector 'in0' is right shifted by 'shift' and * the result is written in-place. 'shift' is a GP variable. */ -#define SRAI_H2(RTYPE, in0, in1, shift_val) { \ - in0 = (RTYPE)SRAI_H(in0, shift_val); \ - in1 = (RTYPE)SRAI_H(in1, shift_val); \ -} +#define SRAI_H2(RTYPE, in0, in1, shift_val) do { \ + in0 = (RTYPE)SRAI_H(in0, shift_val); \ + in1 = (RTYPE)SRAI_H(in1, shift_val); \ +} while (0) #define SRAI_H2_SH(...) SRAI_H2(v8i16, __VA_ARGS__) #define SRAI_H2_UH(...) SRAI_H2(v8u16, __VA_ARGS__) @@ -443,48 +1082,166 @@ * Details : Each element of vector 'in0' is right shifted by 'shift' and * the result is written in-place. 'shift' is a GP variable. */ -#define SRARI_W2(RTYPE, in0, in1, shift) { \ +#define SRARI_W2(RTYPE, in0, in1, shift) do { \ in0 = (RTYPE)__msa_srari_w((v4i32)in0, shift); \ in1 = (RTYPE)__msa_srari_w((v4i32)in1, shift); \ -} +} while (0) #define SRARI_W2_SW(...) SRARI_W2(v4i32, __VA_ARGS__) -#define SRARI_W4(RTYPE, in0, in1, in2, in3, shift) { \ - SRARI_W2(RTYPE, in0, in1, shift); \ - SRARI_W2(RTYPE, in2, in3, shift); \ -} +#define SRARI_W4(RTYPE, in0, in1, in2, in3, shift) do { \ + SRARI_W2(RTYPE, in0, in1, shift); \ + SRARI_W2(RTYPE, in2, in3, shift); \ +} while (0) #define SRARI_W4_SH(...) SRARI_W4(v8i16, __VA_ARGS__) #define SRARI_W4_UW(...) SRARI_W4(v4u32, __VA_ARGS__) #define SRARI_W4_SW(...) SRARI_W4(v4i32, __VA_ARGS__) +/* Description : Shift right arithmetic rounded double words + * Arguments : Inputs - in0, in1, shift + * Outputs - in place operation + * Return Type - as per RTYPE + * Details : Each element of vector 'in0' is shifted right arithmetically by + * the number of bits in the corresponding element in the vector + * 'shift'. The last discarded bit is added to shifted value for + * rounding and the result is written in-place. + * 'shift' is a vector. + */ +#define SRAR_D2(RTYPE, in0, in1, shift) do { \ + in0 = (RTYPE)__msa_srar_d((v2i64)in0, (v2i64)shift); \ + in1 = (RTYPE)__msa_srar_d((v2i64)in1, (v2i64)shift); \ +} while (0) +#define SRAR_D2_SW(...) SRAR_D2(v4i32, __VA_ARGS__) +#define SRAR_D2_SD(...) SRAR_D2(v2i64, __VA_ARGS__) +#define SRAR_D2_UD(...) SRAR_D2(v2u64, __VA_ARGS__) + +#define SRAR_D4(RTYPE, in0, in1, in2, in3, shift) do { \ + SRAR_D2(RTYPE, in0, in1, shift); \ + SRAR_D2(RTYPE, in2, in3, shift); \ +} while (0) +#define SRAR_D4_SD(...) SRAR_D4(v2i64, __VA_ARGS__) +#define SRAR_D4_UD(...) SRAR_D4(v2u64, __VA_ARGS__) + /* Description : Addition of 2 pairs of half-word vectors * Arguments : Inputs - in0, in1, in2, in3 * Outputs - out0, out1 * Details : Each element in 'in0' is added to 'in1' and result is written * to 'out0'. */ -#define ADDVI_H2(RTYPE, in0, in1, in2, in3, out0, out1) { \ - out0 = (RTYPE)ADDVI_H(in0, in1); \ - out1 = (RTYPE)ADDVI_H(in2, in3); \ -} +#define ADDVI_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)ADDVI_H(in0, in1); \ + out1 = (RTYPE)ADDVI_H(in2, in3); \ +} while (0) #define ADDVI_H2_SH(...) ADDVI_H2(v8i16, __VA_ARGS__) #define ADDVI_H2_UH(...) ADDVI_H2(v8u16, __VA_ARGS__) +/* Description : Addition of 2 pairs of word vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Details : Each element in 'in0' is added to 'in1' and result is written + * to 'out0'. + */ +#define ADDVI_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)ADDVI_W(in0, in1); \ + out1 = (RTYPE)ADDVI_W(in2, in3); \ +} while (0) +#define ADDVI_W2_SW(...) ADDVI_W2(v4i32, __VA_ARGS__) + +/* Description : Fill 2 pairs of word vectors with GP registers + * Arguments : Inputs - in0, in1 + * Outputs - out0, out1 + * Details : GP register in0 is replicated in each word element of out0 + * GP register in1 is replicated in each word element of out1 + */ +#define FILL_W2(RTYPE, in0, in1, out0, out1) do { \ + out0 = (RTYPE)__msa_fill_w(in0); \ + out1 = (RTYPE)__msa_fill_w(in1); \ +} while (0) +#define FILL_W2_SW(...) FILL_W2(v4i32, __VA_ARGS__) + /* Description : Addition of 2 pairs of vectors * Arguments : Inputs - in0, in1, in2, in3 * Outputs - out0, out1 * Details : Each element in 'in0' is added to 'in1' and result is written * to 'out0'. */ -#define ADD2(in0, in1, in2, in3, out0, out1) { \ - out0 = in0 + in1; \ - out1 = in2 + in3; \ -} +#define ADD2(in0, in1, in2, in3, out0, out1) do { \ + out0 = in0 + in1; \ + out1 = in2 + in3; \ +} while (0) + #define ADD4(in0, in1, in2, in3, in4, in5, in6, in7, \ - out0, out1, out2, out3) { \ + out0, out1, out2, out3) do { \ ADD2(in0, in1, in2, in3, out0, out1); \ ADD2(in4, in5, in6, in7, out2, out3); \ -} +} while (0) + +/* Description : Subtraction of 2 pairs of vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Details : Each element in 'in1' is subtracted from 'in0' and result is + * written to 'out0'. + */ +#define SUB2(in0, in1, in2, in3, out0, out1) do { \ + out0 = in0 - in1; \ + out1 = in2 - in3; \ +} while (0) + +#define SUB3(in0, in1, in2, in3, in4, in5, out0, out1, out2) do { \ + out0 = in0 - in1; \ + out1 = in2 - in3; \ + out2 = in4 - in5; \ +} while (0) + +#define SUB4(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3) do { \ + out0 = in0 - in1; \ + out1 = in2 - in3; \ + out2 = in4 - in5; \ + out3 = in6 - in7; \ +} while (0) + +/* Description : Addition - Subtraction of input vectors + * Arguments : Inputs - in0, in1 + * Outputs - out0, out1 + * Details : Each element in 'in1' is added to 'in0' and result is + * written to 'out0'. + * Each element in 'in1' is subtracted from 'in0' and result is + * written to 'out1'. + */ +#define ADDSUB2(in0, in1, out0, out1) do { \ + out0 = in0 + in1; \ + out1 = in0 - in1; \ +} while (0) + +/* Description : Multiplication of pairs of vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Details : Each element from 'in0' is multiplied with elements from 'in1' + * and the result is written to 'out0' + */ +#define MUL2(in0, in1, in2, in3, out0, out1) do { \ + out0 = in0 * in1; \ + out1 = in2 * in3; \ +} while (0) + +#define MUL4(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3) do { \ + MUL2(in0, in1, in2, in3, out0, out1); \ + MUL2(in4, in5, in6, in7, out2, out3); \ +} while (0) + +/* Description : Sign extend halfword elements from right half of the vector + * Arguments : Input - in (halfword vector) + * Output - out (sign extended word vector) + * Return Type - signed word + * Details : Sign bit of halfword elements from input vector 'in' is + * extracted and interleaved with same vector 'in0' to generate + * 4 word elements keeping sign intact + */ +#define UNPCK_R_SH_SW(in, out) do { \ + const v8i16 sign_m = __msa_clti_s_h((v8i16)in, 0); \ + out = (v4i32)__msa_ilvr_h(sign_m, (v8i16)in); \ +} while (0) /* Description : Sign extend halfword elements from input vector and return * the result in pair of vectors @@ -497,29 +1254,82 @@ * Then interleaved left with same vector 'in0' to * generate 4 signed word elements in 'out1' */ -#define UNPCK_SH_SW(in, out0, out1) { \ +#define UNPCK_SH_SW(in, out0, out1) do { \ const v8i16 tmp_m = __msa_clti_s_h((v8i16)in, 0); \ ILVRL_H2_SW(tmp_m, in, out0, out1); \ -} +} while (0) /* Description : Butterfly of 4 input vectors * Arguments : Inputs - in0, in1, in2, in3 * Outputs - out0, out1, out2, out3 * Details : Butterfly operation */ -#define BUTTERFLY_4(in0, in1, in2, in3, out0, out1, out2, out3) { \ - out0 = in0 + in3; \ - out1 = in1 + in2; \ - out2 = in1 - in2; \ - out3 = in0 - in3; \ -} +#define BUTTERFLY_4(in0, in1, in2, in3, out0, out1, out2, out3) do { \ + out0 = in0 + in3; \ + out1 = in1 + in2; \ + out2 = in1 - in2; \ + out3 = in0 - in3; \ +} while (0) + +/* Description : Transpose 16x4 block into 4x16 with byte elements in vectors + * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, + * in8, in9, in10, in11, in12, in13, in14, in15 + * Outputs - out0, out1, out2, out3 + * Return Type - unsigned byte + */ +#define TRANSPOSE16x4_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ + in8, in9, in10, in11, in12, in13, in14, in15, \ + out0, out1, out2, out3) do { \ + v2i64 tmp0_m, tmp1_m, tmp2_m, tmp3_m, tmp4_m, tmp5_m; \ + ILVEV_W2_SD(in0, in4, in8, in12, tmp2_m, tmp3_m); \ + ILVEV_W2_SD(in1, in5, in9, in13, tmp0_m, tmp1_m); \ + ILVEV_D2_UB(tmp2_m, tmp3_m, tmp0_m, tmp1_m, out1, out3); \ + ILVEV_W2_SD(in2, in6, in10, in14, tmp4_m, tmp5_m); \ + ILVEV_W2_SD(in3, in7, in11, in15, tmp0_m, tmp1_m); \ + ILVEV_D2_SD(tmp4_m, tmp5_m, tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ + ILVEV_B2_SD(out1, out3, tmp2_m, tmp3_m, tmp0_m, tmp1_m); \ + ILVEVOD_H2_UB(tmp0_m, tmp1_m, tmp0_m, tmp1_m, out0, out2); \ + ILVOD_B2_SD(out1, out3, tmp2_m, tmp3_m, tmp0_m, tmp1_m); \ + ILVEVOD_H2_UB(tmp0_m, tmp1_m, tmp0_m, tmp1_m, out1, out3); \ +} while (0) + +/* Description : Transpose 16x8 block into 8x16 with byte elements in vectors + * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, + * in8, in9, in10, in11, in12, in13, in14, in15 + * Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + * Return Type - unsigned byte + */ +#define TRANSPOSE16x8_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ + in8, in9, in10, in11, in12, in13, in14, in15, \ + out0, out1, out2, out3, out4, out5, \ + out6, out7) do { \ + v8i16 tmp0_m, tmp1_m, tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ + v4i32 tmp2_m, tmp3_m; \ + ILVEV_D2_UB(in0, in8, in1, in9, out7, out6); \ + ILVEV_D2_UB(in2, in10, in3, in11, out5, out4); \ + ILVEV_D2_UB(in4, in12, in5, in13, out3, out2); \ + ILVEV_D2_UB(in6, in14, in7, in15, out1, out0); \ + ILVEV_B2_SH(out7, out6, out5, out4, tmp0_m, tmp1_m); \ + ILVOD_B2_SH(out7, out6, out5, out4, tmp4_m, tmp5_m); \ + ILVEV_B2_UB(out3, out2, out1, out0, out5, out7); \ + ILVOD_B2_SH(out3, out2, out1, out0, tmp6_m, tmp7_m); \ + ILVEV_H2_SW(tmp0_m, tmp1_m, out5, out7, tmp2_m, tmp3_m); \ + ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out0, out4); \ + ILVOD_H2_SW(tmp0_m, tmp1_m, out5, out7, tmp2_m, tmp3_m); \ + ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out2, out6); \ + ILVEV_H2_SW(tmp4_m, tmp5_m, tmp6_m, tmp7_m, tmp2_m, tmp3_m); \ + ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out1, out5); \ + ILVOD_H2_SW(tmp4_m, tmp5_m, tmp6_m, tmp7_m, tmp2_m, tmp3_m); \ + ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out3, out7); \ +} while (0) /* Description : Transpose 4x4 block with word elements in vectors * Arguments : Inputs - in0, in1, in2, in3 * Outputs - out0, out1, out2, out3 * Return Type - as per RTYPE */ -#define TRANSPOSE4x4_W(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) { \ +#define TRANSPOSE4x4_W(RTYPE, in0, in1, in2, in3, \ + out0, out1, out2, out3) do { \ v4i32 s0_m, s1_m, s2_m, s3_m; \ ILVRL_W2_SW(in1, in0, s0_m, s1_m); \ ILVRL_W2_SW(in3, in2, s2_m, s3_m); \ @@ -527,7 +1337,7 @@ out1 = (RTYPE)__msa_ilvl_d((v2i64)s2_m, (v2i64)s0_m); \ out2 = (RTYPE)__msa_ilvr_d((v2i64)s3_m, (v2i64)s1_m); \ out3 = (RTYPE)__msa_ilvl_d((v2i64)s3_m, (v2i64)s1_m); \ -} +} while (0) #define TRANSPOSE4x4_SW_SW(...) TRANSPOSE4x4_W(v4i32, __VA_ARGS__) /* Description : Add block 4x4 @@ -535,7 +1345,7 @@ * Details : Least significant 4 bytes from each input vector are added to * the destination bytes, clipped between 0-255 and stored. */ -#define ADDBLK_ST4x4_UB(in0, in1, in2, in3, pdst, stride) { \ +#define ADDBLK_ST4x4_UB(in0, in1, in2, in3, pdst, stride) do { \ uint32_t src0_m, src1_m, src2_m, src3_m; \ v8i16 inp0_m, inp1_m, res0_m, res1_m; \ v16i8 dst0_m = { 0 }; \ @@ -550,6 +1360,31 @@ CLIP_SH2_0_255(res0_m, res1_m); \ PCKEV_B2_SB(res0_m, res0_m, res1_m, res1_m, dst0_m, dst1_m); \ ST4x4_UB(dst0_m, dst1_m, 0, 1, 0, 1, pdst, stride); \ -} +} while (0) + +/* Description : Pack even byte elements, extract 0 & 2 index words from pair + * of results and store 4 words in destination memory as per + * stride + * Arguments : Inputs - in0, in1, in2, in3, pdst, stride + */ +#define PCKEV_ST4x4_UB(in0, in1, in2, in3, pdst, stride) do { \ + v16i8 tmp0_m, tmp1_m; \ + PCKEV_B2_SB(in1, in0, in3, in2, tmp0_m, tmp1_m); \ + ST4x4_UB(tmp0_m, tmp1_m, 0, 2, 0, 2, pdst, stride); \ +} while (0) + +/* Description : average with rounding (in0 + in1 + 1) / 2. + * Arguments : Inputs - in0, in1, in2, in3, + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Each unsigned byte element from 'in0' vector is added with + * each unsigned byte element from 'in1' vector. Then the average + * with rounding is calculated and written to 'out0' + */ +#define AVER_UB2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ + out0 = (RTYPE)__msa_aver_u_b((v16u8)in0, (v16u8)in1); \ + out1 = (RTYPE)__msa_aver_u_b((v16u8)in2, (v16u8)in3); \ +} while (0) +#define AVER_UB2_UB(...) AVER_UB2(v16u8, __VA_ARGS__) #endif /* WEBP_DSP_MSA_MACRO_H_ */ diff --git a/src/3rdparty/libwebp/src/dsp/neon.h b/src/3rdparty/libwebp/src/dsp/neon.h index 0a06266..3b548a6 100644 --- a/src/3rdparty/libwebp/src/dsp/neon.h +++ b/src/3rdparty/libwebp/src/dsp/neon.h @@ -79,4 +79,22 @@ static WEBP_INLINE int32x4x4_t Transpose4x4(const int32x4x4_t rows) { } } +#if 0 // Useful debug macro. +#include <stdio.h> +#define PRINT_REG(REG, SIZE) do { \ + int i; \ + printf("%s \t[%d]: 0x", #REG, SIZE); \ + if (SIZE == 8) { \ + uint8_t _tmp[8]; \ + vst1_u8(_tmp, (REG)); \ + for (i = 0; i < 8; ++i) printf("%.2x ", _tmp[i]); \ + } else if (SIZE == 16) { \ + uint16_t _tmp[4]; \ + vst1_u16(_tmp, (REG)); \ + for (i = 0; i < 4; ++i) printf("%.4x ", _tmp[i]); \ + } \ + printf("\n"); \ +} while (0) +#endif + #endif // WEBP_DSP_NEON_H_ diff --git a/src/3rdparty/libwebp/src/dsp/rescaler.c b/src/3rdparty/libwebp/src/dsp/rescaler.c index bc743d5..0f54502 100644 --- a/src/3rdparty/libwebp/src/dsp/rescaler.c +++ b/src/3rdparty/libwebp/src/dsp/rescaler.c @@ -14,7 +14,7 @@ #include <assert.h> #include "./dsp.h" -#include "../utils/rescaler.h" +#include "../utils/rescaler_utils.h" //------------------------------------------------------------------------------ // Implementations of critical functions ImportRow / ExportRow @@ -173,10 +173,10 @@ void WebPRescalerExportRow(WebPRescaler* const wrk) { WebPRescalerExportRowExpand(wrk); } else if (wrk->fxy_scale) { WebPRescalerExportRowShrink(wrk); - } else { // very special case for src = dst = 1x1 + } else { // special case int i; + assert(wrk->src_height == wrk->dst_height && wrk->x_add == 1); assert(wrk->src_width == 1 && wrk->dst_width <= 2); - assert(wrk->src_height == 1 && wrk->dst_height == 1); for (i = 0; i < wrk->num_channels * wrk->dst_width; ++i) { wrk->dst[i] = wrk->irow[i]; wrk->irow[i] = 0; @@ -199,6 +199,7 @@ WebPRescalerExportRowFunc WebPRescalerExportRowShrink; extern void WebPRescalerDspInitSSE2(void); extern void WebPRescalerDspInitMIPS32(void); extern void WebPRescalerDspInitMIPSdspR2(void); +extern void WebPRescalerDspInitMSA(void); extern void WebPRescalerDspInitNEON(void); static volatile VP8CPUInfo rescaler_last_cpuinfo_used = @@ -233,6 +234,11 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInit(void) { WebPRescalerDspInitMIPSdspR2(); } #endif +#if defined(WEBP_USE_MSA) + if (VP8GetCPUInfo(kMSA)) { + WebPRescalerDspInitMSA(); + } +#endif } rescaler_last_cpuinfo_used = VP8GetCPUInfo; } diff --git a/src/3rdparty/libwebp/src/dsp/rescaler_mips32.c b/src/3rdparty/libwebp/src/dsp/rescaler_mips32.c index ddaa391..e09ad5d 100644 --- a/src/3rdparty/libwebp/src/dsp/rescaler_mips32.c +++ b/src/3rdparty/libwebp/src/dsp/rescaler_mips32.c @@ -16,7 +16,7 @@ #if defined(WEBP_USE_MIPS32) #include <assert.h> -#include "../utils/rescaler.h" +#include "../utils/rescaler_utils.h" //------------------------------------------------------------------------------ // Row import diff --git a/src/3rdparty/libwebp/src/dsp/rescaler_mips_dsp_r2.c b/src/3rdparty/libwebp/src/dsp/rescaler_mips_dsp_r2.c index b457d0a..2308d64 100644 --- a/src/3rdparty/libwebp/src/dsp/rescaler_mips_dsp_r2.c +++ b/src/3rdparty/libwebp/src/dsp/rescaler_mips_dsp_r2.c @@ -16,7 +16,7 @@ #if defined(WEBP_USE_MIPS_DSP_R2) #include <assert.h> -#include "../utils/rescaler.h" +#include "../utils/rescaler_utils.h" #define ROUNDER (WEBP_RESCALER_ONE >> 1) #define MULT_FIX(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX) diff --git a/src/3rdparty/libwebp/src/dsp/rescaler_msa.c b/src/3rdparty/libwebp/src/dsp/rescaler_msa.c new file mode 100644 index 0000000..2c10e55 --- /dev/null +++ b/src/3rdparty/libwebp/src/dsp/rescaler_msa.c @@ -0,0 +1,444 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MSA version of rescaling functions +// +// Author: Prashant Patil (prashant.patil@imgtec.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_MSA) + +#include <assert.h> + +#include "../utils/rescaler_utils.h" +#include "./msa_macro.h" + +#define ROUNDER (WEBP_RESCALER_ONE >> 1) +#define MULT_FIX(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX) + +#define CALC_MULT_FIX_16(in0, in1, in2, in3, scale, shift, dst) do { \ + v4u32 tmp0, tmp1, tmp2, tmp3; \ + v16u8 t0, t1, t2, t3, t4, t5; \ + v2u64 out0, out1, out2, out3; \ + ILVRL_W2_UW(zero, in0, tmp0, tmp1); \ + ILVRL_W2_UW(zero, in1, tmp2, tmp3); \ + DOTP_UW2_UD(tmp0, tmp1, scale, scale, out0, out1); \ + DOTP_UW2_UD(tmp2, tmp3, scale, scale, out2, out3); \ + SRAR_D4_UD(out0, out1, out2, out3, shift); \ + PCKEV_B2_UB(out1, out0, out3, out2, t0, t1); \ + ILVRL_W2_UW(zero, in2, tmp0, tmp1); \ + ILVRL_W2_UW(zero, in3, tmp2, tmp3); \ + DOTP_UW2_UD(tmp0, tmp1, scale, scale, out0, out1); \ + DOTP_UW2_UD(tmp2, tmp3, scale, scale, out2, out3); \ + SRAR_D4_UD(out0, out1, out2, out3, shift); \ + PCKEV_B2_UB(out1, out0, out3, out2, t2, t3); \ + PCKEV_B2_UB(t1, t0, t3, t2, t4, t5); \ + dst = (v16u8)__msa_pckev_b((v16i8)t5, (v16i8)t4); \ +} while (0) + +#define CALC_MULT_FIX_4(in0, scale, shift, dst) do { \ + v4u32 tmp0, tmp1; \ + v16i8 t0, t1; \ + v2u64 out0, out1; \ + ILVRL_W2_UW(zero, in0, tmp0, tmp1); \ + DOTP_UW2_UD(tmp0, tmp1, scale, scale, out0, out1); \ + SRAR_D2_UD(out0, out1, shift); \ + t0 = __msa_pckev_b((v16i8)out1, (v16i8)out0); \ + t1 = __msa_pckev_b(t0, t0); \ + t0 = __msa_pckev_b(t1, t1); \ + dst = __msa_copy_s_w((v4i32)t0, 0); \ +} while (0) + +#define CALC_MULT_FIX1_16(in0, in1, in2, in3, fyscale, shift, \ + dst0, dst1, dst2, dst3) do { \ + v4u32 tmp0, tmp1, tmp2, tmp3; \ + v2u64 out0, out1, out2, out3; \ + ILVRL_W2_UW(zero, in0, tmp0, tmp1); \ + ILVRL_W2_UW(zero, in1, tmp2, tmp3); \ + DOTP_UW2_UD(tmp0, tmp1, fyscale, fyscale, out0, out1); \ + DOTP_UW2_UD(tmp2, tmp3, fyscale, fyscale, out2, out3); \ + SRAR_D4_UD(out0, out1, out2, out3, shift); \ + PCKEV_W2_UW(out1, out0, out3, out2, dst0, dst1); \ + ILVRL_W2_UW(zero, in2, tmp0, tmp1); \ + ILVRL_W2_UW(zero, in3, tmp2, tmp3); \ + DOTP_UW2_UD(tmp0, tmp1, fyscale, fyscale, out0, out1); \ + DOTP_UW2_UD(tmp2, tmp3, fyscale, fyscale, out2, out3); \ + SRAR_D4_UD(out0, out1, out2, out3, shift); \ + PCKEV_W2_UW(out1, out0, out3, out2, dst2, dst3); \ +} while (0) + +#define CALC_MULT_FIX1_4(in0, scale, shift, dst) do { \ + v4u32 tmp0, tmp1; \ + v2u64 out0, out1; \ + ILVRL_W2_UW(zero, in0, tmp0, tmp1); \ + DOTP_UW2_UD(tmp0, tmp1, scale, scale, out0, out1); \ + SRAR_D2_UD(out0, out1, shift); \ + dst = (v4u32)__msa_pckev_w((v4i32)out1, (v4i32)out0); \ +} while (0) + +#define CALC_MULT_FIX2_16(in0, in1, in2, in3, mult, scale, shift, \ + dst0, dst1) do { \ + v4u32 tmp0, tmp1, tmp2, tmp3; \ + v2u64 out0, out1, out2, out3; \ + ILVRL_W2_UW(in0, in2, tmp0, tmp1); \ + ILVRL_W2_UW(in1, in3, tmp2, tmp3); \ + DOTP_UW2_UD(tmp0, tmp1, mult, mult, out0, out1); \ + DOTP_UW2_UD(tmp2, tmp3, mult, mult, out2, out3); \ + SRAR_D4_UD(out0, out1, out2, out3, shift); \ + DOTP_UW2_UD(out0, out1, scale, scale, out0, out1); \ + DOTP_UW2_UD(out2, out3, scale, scale, out2, out3); \ + SRAR_D4_UD(out0, out1, out2, out3, shift); \ + PCKEV_B2_UB(out1, out0, out3, out2, dst0, dst1); \ +} while (0) + +#define CALC_MULT_FIX2_4(in0, in1, mult, scale, shift, dst) do { \ + v4u32 tmp0, tmp1; \ + v2u64 out0, out1; \ + v16i8 t0, t1; \ + ILVRL_W2_UW(in0, in1, tmp0, tmp1); \ + DOTP_UW2_UD(tmp0, tmp1, mult, mult, out0, out1); \ + SRAR_D2_UD(out0, out1, shift); \ + DOTP_UW2_UD(out0, out1, scale, scale, out0, out1); \ + SRAR_D2_UD(out0, out1, shift); \ + t0 = __msa_pckev_b((v16i8)out1, (v16i8)out0); \ + t1 = __msa_pckev_b(t0, t0); \ + t0 = __msa_pckev_b(t1, t1); \ + dst = __msa_copy_s_w((v4i32)t0, 0); \ +} while (0) + +static WEBP_INLINE void ExportRowExpand_0(const uint32_t* frow, uint8_t* dst, + int length, + WebPRescaler* const wrk) { + const v4u32 scale = (v4u32)__msa_fill_w(wrk->fy_scale); + const v4u32 shift = (v4u32)__msa_fill_w(WEBP_RESCALER_RFIX); + const v4i32 zero = { 0 }; + + while (length >= 16) { + v4u32 src0, src1, src2, src3; + v16u8 out; + LD_UW4(frow, 4, src0, src1, src2, src3); + CALC_MULT_FIX_16(src0, src1, src2, src3, scale, shift, out); + ST_UB(out, dst); + length -= 16; + frow += 16; + dst += 16; + } + if (length > 0) { + int x_out; + if (length >= 12) { + uint32_t val0_m, val1_m, val2_m; + v4u32 src0, src1, src2; + LD_UW3(frow, 4, src0, src1, src2); + CALC_MULT_FIX_4(src0, scale, shift, val0_m); + CALC_MULT_FIX_4(src1, scale, shift, val1_m); + CALC_MULT_FIX_4(src2, scale, shift, val2_m); + SW3(val0_m, val1_m, val2_m, dst, 4); + length -= 12; + frow += 12; + dst += 12; + } else if (length >= 8) { + uint32_t val0_m, val1_m; + v4u32 src0, src1; + LD_UW2(frow, 4, src0, src1); + CALC_MULT_FIX_4(src0, scale, shift, val0_m); + CALC_MULT_FIX_4(src1, scale, shift, val1_m); + SW2(val0_m, val1_m, dst, 4); + length -= 8; + frow += 8; + dst += 8; + } else if (length >= 4) { + uint32_t val0_m; + const v4u32 src0 = LD_UW(frow); + CALC_MULT_FIX_4(src0, scale, shift, val0_m); + SW(val0_m, dst); + length -= 4; + frow += 4; + dst += 4; + } + for (x_out = 0; x_out < length; ++x_out) { + const uint32_t J = frow[x_out]; + const int v = (int)MULT_FIX(J, wrk->fy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + } + } +} + +static WEBP_INLINE void ExportRowExpand_1(const uint32_t* frow, uint32_t* irow, + uint8_t* dst, int length, + WebPRescaler* const wrk) { + const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); + const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B); + const v4i32 B1 = __msa_fill_w(B); + const v4i32 A1 = __msa_fill_w(A); + const v4i32 AB = __msa_ilvr_w(A1, B1); + const v4u32 scale = (v4u32)__msa_fill_w(wrk->fy_scale); + const v4u32 shift = (v4u32)__msa_fill_w(WEBP_RESCALER_RFIX); + + while (length >= 16) { + v4u32 frow0, frow1, frow2, frow3, irow0, irow1, irow2, irow3; + v16u8 t0, t1, t2, t3, t4, t5; + LD_UW4(frow, 4, frow0, frow1, frow2, frow3); + LD_UW4(irow, 4, irow0, irow1, irow2, irow3); + CALC_MULT_FIX2_16(frow0, frow1, irow0, irow1, AB, scale, shift, t0, t1); + CALC_MULT_FIX2_16(frow2, frow3, irow2, irow3, AB, scale, shift, t2, t3); + PCKEV_B2_UB(t1, t0, t3, t2, t4, t5); + t0 = (v16u8)__msa_pckev_b((v16i8)t5, (v16i8)t4); + ST_UB(t0, dst); + frow += 16; + irow += 16; + dst += 16; + length -= 16; + } + if (length > 0) { + int x_out; + if (length >= 12) { + uint32_t val0_m, val1_m, val2_m; + v4u32 frow0, frow1, frow2, irow0, irow1, irow2; + LD_UW3(frow, 4, frow0, frow1, frow2); + LD_UW3(irow, 4, irow0, irow1, irow2); + CALC_MULT_FIX2_4(frow0, irow0, AB, scale, shift, val0_m); + CALC_MULT_FIX2_4(frow1, irow1, AB, scale, shift, val1_m); + CALC_MULT_FIX2_4(frow2, irow2, AB, scale, shift, val2_m); + SW3(val0_m, val1_m, val2_m, dst, 4); + frow += 12; + irow += 12; + dst += 12; + length -= 12; + } else if (length >= 8) { + uint32_t val0_m, val1_m; + v4u32 frow0, frow1, irow0, irow1; + LD_UW2(frow, 4, frow0, frow1); + LD_UW2(irow, 4, irow0, irow1); + CALC_MULT_FIX2_4(frow0, irow0, AB, scale, shift, val0_m); + CALC_MULT_FIX2_4(frow1, irow1, AB, scale, shift, val1_m); + SW2(val0_m, val1_m, dst, 4); + frow += 4; + irow += 4; + dst += 4; + length -= 4; + } else if (length >= 4) { + uint32_t val0_m; + const v4u32 frow0 = LD_UW(frow + 0); + const v4u32 irow0 = LD_UW(irow + 0); + CALC_MULT_FIX2_4(frow0, irow0, AB, scale, shift, val0_m); + SW(val0_m, dst); + frow += 4; + irow += 4; + dst += 4; + length -= 4; + } + for (x_out = 0; x_out < length; ++x_out) { + const uint64_t I = (uint64_t)A * frow[x_out] + + (uint64_t)B * irow[x_out]; + const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); + const int v = (int)MULT_FIX(J, wrk->fy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + } + } +} + +static void RescalerExportRowExpand(WebPRescaler* const wrk) { + uint8_t* dst = wrk->dst; + rescaler_t* irow = wrk->irow; + const int x_out_max = wrk->dst_width * wrk->num_channels; + const rescaler_t* frow = wrk->frow; + assert(!WebPRescalerOutputDone(wrk)); + assert(wrk->y_accum <= 0); + assert(wrk->y_expand); + assert(wrk->y_sub != 0); + if (wrk->y_accum == 0) { + ExportRowExpand_0(frow, dst, x_out_max, wrk); + } else { + ExportRowExpand_1(frow, irow, dst, x_out_max, wrk); + } +} + +static WEBP_INLINE void ExportRowShrink_0(const uint32_t* frow, uint32_t* irow, + uint8_t* dst, int length, + const uint32_t yscale, + WebPRescaler* const wrk) { + const v4u32 y_scale = (v4u32)__msa_fill_w(yscale); + const v4u32 fxyscale = (v4u32)__msa_fill_w(wrk->fxy_scale); + const v4u32 shiftval = (v4u32)__msa_fill_w(WEBP_RESCALER_RFIX); + const v4i32 zero = { 0 }; + + while (length >= 16) { + v4u32 src0, src1, src2, src3, frac0, frac1, frac2, frac3; + v16u8 out; + LD_UW4(frow, 4, src0, src1, src2, src3); + CALC_MULT_FIX1_16(src0, src1, src2, src3, y_scale, shiftval, + frac0, frac1, frac2, frac3); + LD_UW4(irow, 4, src0, src1, src2, src3); + SUB4(src0, frac0, src1, frac1, src2, frac2, src3, frac3, + src0, src1, src2, src3); + CALC_MULT_FIX_16(src0, src1, src2, src3, fxyscale, shiftval, out); + ST_UB(out, dst); + ST_UW4(frac0, frac1, frac2, frac3, irow, 4); + frow += 16; + irow += 16; + dst += 16; + length -= 16; + } + if (length > 0) { + int x_out; + if (length >= 12) { + uint32_t val0_m, val1_m, val2_m; + v4u32 src0, src1, src2, frac0, frac1, frac2; + LD_UW3(frow, 4, src0, src1, src2); + CALC_MULT_FIX1_4(src0, y_scale, shiftval, frac0); + CALC_MULT_FIX1_4(src1, y_scale, shiftval, frac1); + CALC_MULT_FIX1_4(src2, y_scale, shiftval, frac2); + LD_UW3(irow, 4, src0, src1, src2); + SUB3(src0, frac0, src1, frac1, src2, frac2, src0, src1, src2); + CALC_MULT_FIX_4(src0, fxyscale, shiftval, val0_m); + CALC_MULT_FIX_4(src1, fxyscale, shiftval, val1_m); + CALC_MULT_FIX_4(src2, fxyscale, shiftval, val2_m); + SW3(val0_m, val1_m, val2_m, dst, 4); + ST_UW3(frac0, frac1, frac2, irow, 4); + frow += 12; + irow += 12; + dst += 12; + length -= 12; + } else if (length >= 8) { + uint32_t val0_m, val1_m; + v4u32 src0, src1, frac0, frac1; + LD_UW2(frow, 4, src0, src1); + CALC_MULT_FIX1_4(src0, y_scale, shiftval, frac0); + CALC_MULT_FIX1_4(src1, y_scale, shiftval, frac1); + LD_UW2(irow, 4, src0, src1); + SUB2(src0, frac0, src1, frac1, src0, src1); + CALC_MULT_FIX_4(src0, fxyscale, shiftval, val0_m); + CALC_MULT_FIX_4(src1, fxyscale, shiftval, val1_m); + SW2(val0_m, val1_m, dst, 4); + ST_UW2(frac0, frac1, irow, 4); + frow += 8; + irow += 8; + dst += 8; + length -= 8; + } else if (length >= 4) { + uint32_t val0_m; + v4u32 frac0; + v4u32 src0 = LD_UW(frow); + CALC_MULT_FIX1_4(src0, y_scale, shiftval, frac0); + src0 = LD_UW(irow); + src0 = src0 - frac0; + CALC_MULT_FIX_4(src0, fxyscale, shiftval, val0_m); + SW(val0_m, dst); + ST_UW(frac0, irow); + frow += 4; + irow += 4; + dst += 4; + length -= 4; + } + for (x_out = 0; x_out < length; ++x_out) { + const uint32_t frac = (uint32_t)MULT_FIX(frow[x_out], yscale); + const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + irow[x_out] = frac; + } + } +} + +static WEBP_INLINE void ExportRowShrink_1(uint32_t* irow, uint8_t* dst, + int length, + WebPRescaler* const wrk) { + const v4u32 scale = (v4u32)__msa_fill_w(wrk->fxy_scale); + const v4u32 shift = (v4u32)__msa_fill_w(WEBP_RESCALER_RFIX); + const v4i32 zero = { 0 }; + + while (length >= 16) { + v4u32 src0, src1, src2, src3; + v16u8 dst0; + LD_UW4(irow, 4, src0, src1, src2, src3); + CALC_MULT_FIX_16(src0, src1, src2, src3, scale, shift, dst0); + ST_UB(dst0, dst); + ST_SW4(zero, zero, zero, zero, irow, 4); + length -= 16; + irow += 16; + dst += 16; + } + if (length > 0) { + int x_out; + if (length >= 12) { + uint32_t val0_m, val1_m, val2_m; + v4u32 src0, src1, src2; + LD_UW3(irow, 4, src0, src1, src2); + CALC_MULT_FIX_4(src0, scale, shift, val0_m); + CALC_MULT_FIX_4(src1, scale, shift, val1_m); + CALC_MULT_FIX_4(src2, scale, shift, val2_m); + SW3(val0_m, val1_m, val2_m, dst, 4); + ST_SW3(zero, zero, zero, irow, 4); + length -= 12; + irow += 12; + dst += 12; + } else if (length >= 8) { + uint32_t val0_m, val1_m; + v4u32 src0, src1; + LD_UW2(irow, 4, src0, src1); + CALC_MULT_FIX_4(src0, scale, shift, val0_m); + CALC_MULT_FIX_4(src1, scale, shift, val1_m); + SW2(val0_m, val1_m, dst, 4); + ST_SW2(zero, zero, irow, 4); + length -= 8; + irow += 8; + dst += 8; + } else if (length >= 4) { + uint32_t val0_m; + const v4u32 src0 = LD_UW(irow + 0); + CALC_MULT_FIX_4(src0, scale, shift, val0_m); + SW(val0_m, dst); + ST_SW(zero, irow); + length -= 4; + irow += 4; + dst += 4; + } + for (x_out = 0; x_out < length; ++x_out) { + const int v = (int)MULT_FIX(irow[x_out], wrk->fxy_scale); + assert(v >= 0 && v <= 255); + dst[x_out] = v; + irow[x_out] = 0; + } + } +} + +static void RescalerExportRowShrink(WebPRescaler* const wrk) { + uint8_t* dst = wrk->dst; + rescaler_t* irow = wrk->irow; + const int x_out_max = wrk->dst_width * wrk->num_channels; + const rescaler_t* frow = wrk->frow; + const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum); + assert(!WebPRescalerOutputDone(wrk)); + assert(wrk->y_accum <= 0); + assert(!wrk->y_expand); + if (yscale) { + ExportRowShrink_0(frow, irow, dst, x_out_max, yscale, wrk); + } else { + ExportRowShrink_1(irow, dst, x_out_max, wrk); + } +} + +//------------------------------------------------------------------------------ +// Entry point + +extern void WebPRescalerDspInitMSA(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitMSA(void) { + WebPRescalerExportRowExpand = RescalerExportRowExpand; + WebPRescalerExportRowShrink = RescalerExportRowShrink; +} + +#else // !WEBP_USE_MSA + +WEBP_DSP_INIT_STUB(WebPRescalerDspInitMSA) + +#endif // WEBP_USE_MSA diff --git a/src/3rdparty/libwebp/src/dsp/rescaler_neon.c b/src/3rdparty/libwebp/src/dsp/rescaler_neon.c index 16fd450..b2dd8f3 100644 --- a/src/3rdparty/libwebp/src/dsp/rescaler_neon.c +++ b/src/3rdparty/libwebp/src/dsp/rescaler_neon.c @@ -18,7 +18,7 @@ #include <arm_neon.h> #include <assert.h> #include "./neon.h" -#include "../utils/rescaler.h" +#include "../utils/rescaler_utils.h" #define ROUNDER (WEBP_RESCALER_ONE >> 1) #define MULT_FIX_C(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX) diff --git a/src/3rdparty/libwebp/src/dsp/rescaler_sse2.c b/src/3rdparty/libwebp/src/dsp/rescaler_sse2.c index 5b97028..8271c22 100644 --- a/src/3rdparty/libwebp/src/dsp/rescaler_sse2.c +++ b/src/3rdparty/libwebp/src/dsp/rescaler_sse2.c @@ -17,7 +17,7 @@ #include <emmintrin.h> #include <assert.h> -#include "../utils/rescaler.h" +#include "../utils/rescaler_utils.h" #include "../utils/utils.h" //------------------------------------------------------------------------------ diff --git a/src/3rdparty/libwebp/src/dsp/upsampling.c b/src/3rdparty/libwebp/src/dsp/upsampling.c index 651274f..265e722 100644 --- a/src/3rdparty/libwebp/src/dsp/upsampling.c +++ b/src/3rdparty/libwebp/src/dsp/upsampling.c @@ -215,6 +215,7 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitYUV444Converters(void) { extern void WebPInitUpsamplersSSE2(void); extern void WebPInitUpsamplersNEON(void); extern void WebPInitUpsamplersMIPSdspR2(void); +extern void WebPInitUpsamplersMSA(void); static volatile VP8CPUInfo upsampling_last_cpuinfo_used2 = (VP8CPUInfo)&upsampling_last_cpuinfo_used2; @@ -252,6 +253,11 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitUpsamplers(void) { WebPInitUpsamplersMIPSdspR2(); } #endif +#if defined(WEBP_USE_MSA) + if (VP8GetCPUInfo(kMSA)) { + WebPInitUpsamplersMSA(); + } +#endif } #endif // FANCY_UPSAMPLING upsampling_last_cpuinfo_used2 = VP8GetCPUInfo; diff --git a/src/3rdparty/libwebp/src/dsp/upsampling_msa.c b/src/3rdparty/libwebp/src/dsp/upsampling_msa.c new file mode 100644 index 0000000..f24926f --- /dev/null +++ b/src/3rdparty/libwebp/src/dsp/upsampling_msa.c @@ -0,0 +1,678 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// MSA version of YUV to RGB upsampling functions. +// +// Author: Prashant Patil (prashant.patil@imgtec.com) + +#include <string.h> +#include "./dsp.h" + +#if defined(WEBP_USE_MSA) + +#include "./msa_macro.h" +#include "./yuv.h" + +#ifdef FANCY_UPSAMPLING + +#define ILVR_UW2(in, out0, out1) do { \ + const v8i16 t0 = (v8i16)__msa_ilvr_b((v16i8)zero, (v16i8)in); \ + out0 = (v4u32)__msa_ilvr_h((v8i16)zero, t0); \ + out1 = (v4u32)__msa_ilvl_h((v8i16)zero, t0); \ +} while (0) + +#define ILVRL_UW4(in, out0, out1, out2, out3) do { \ + v16u8 t0, t1; \ + ILVRL_B2_UB(zero, in, t0, t1); \ + ILVRL_H2_UW(zero, t0, out0, out1); \ + ILVRL_H2_UW(zero, t1, out2, out3); \ +} while (0) + +#define MULTHI_16(in0, in1, in2, in3, cnst, out0, out1) do { \ + const v4i32 const0 = (v4i32)__msa_fill_w(cnst * 256); \ + v4u32 temp0, temp1, temp2, temp3; \ + MUL4(in0, const0, in1, const0, in2, const0, in3, const0, \ + temp0, temp1, temp2, temp3); \ + PCKOD_H2_UH(temp1, temp0, temp3, temp2, out0, out1); \ +} while (0) + +#define MULTHI_8(in0, in1, cnst, out0) do { \ + const v4i32 const0 = (v4i32)__msa_fill_w(cnst * 256); \ + v4u32 temp0, temp1; \ + MUL2(in0, const0, in1, const0, temp0, temp1); \ + out0 = (v8u16)__msa_pckod_h((v8i16)temp1, (v8i16)temp0); \ +} while (0) + +#define CALC_R16(y0, y1, v0, v1, dst) do { \ + const v8i16 const_a = (v8i16)__msa_fill_h(14234); \ + const v8i16 a0 = __msa_adds_s_h((v8i16)y0, (v8i16)v0); \ + const v8i16 a1 = __msa_adds_s_h((v8i16)y1, (v8i16)v1); \ + v8i16 b0 = __msa_subs_s_h(a0, const_a); \ + v8i16 b1 = __msa_subs_s_h(a1, const_a); \ + SRAI_H2_SH(b0, b1, 6); \ + CLIP_SH2_0_255(b0, b1); \ + dst = (v16u8)__msa_pckev_b((v16i8)b1, (v16i8)b0); \ +} while (0) + +#define CALC_R8(y0, v0, dst) do { \ + const v8i16 const_a = (v8i16)__msa_fill_h(14234); \ + const v8i16 a0 = __msa_adds_s_h((v8i16)y0, (v8i16)v0); \ + v8i16 b0 = __msa_subs_s_h(a0, const_a); \ + b0 = SRAI_H(b0, 6); \ + CLIP_SH_0_255(b0); \ + dst = (v16u8)__msa_pckev_b((v16i8)b0, (v16i8)b0); \ +} while (0) + +#define CALC_G16(y0, y1, u0, u1, v0, v1, dst) do { \ + const v8i16 const_a = (v8i16)__msa_fill_h(8708); \ + v8i16 a0 = __msa_subs_s_h((v8i16)y0, (v8i16)u0); \ + v8i16 a1 = __msa_subs_s_h((v8i16)y1, (v8i16)u1); \ + const v8i16 b0 = __msa_subs_s_h(a0, (v8i16)v0); \ + const v8i16 b1 = __msa_subs_s_h(a1, (v8i16)v1); \ + a0 = __msa_adds_s_h(b0, const_a); \ + a1 = __msa_adds_s_h(b1, const_a); \ + SRAI_H2_SH(a0, a1, 6); \ + CLIP_SH2_0_255(a0, a1); \ + dst = (v16u8)__msa_pckev_b((v16i8)a1, (v16i8)a0); \ +} while (0) + +#define CALC_G8(y0, u0, v0, dst) do { \ + const v8i16 const_a = (v8i16)__msa_fill_h(8708); \ + v8i16 a0 = __msa_subs_s_h((v8i16)y0, (v8i16)u0); \ + const v8i16 b0 = __msa_subs_s_h(a0, (v8i16)v0); \ + a0 = __msa_adds_s_h(b0, const_a); \ + a0 = SRAI_H(a0, 6); \ + CLIP_SH_0_255(a0); \ + dst = (v16u8)__msa_pckev_b((v16i8)a0, (v16i8)a0); \ +} while (0) + +#define CALC_B16(y0, y1, u0, u1, dst) do { \ + const v8u16 const_a = (v8u16)__msa_fill_h(17685); \ + const v8u16 a0 = __msa_adds_u_h((v8u16)y0, u0); \ + const v8u16 a1 = __msa_adds_u_h((v8u16)y1, u1); \ + v8u16 b0 = __msa_subs_u_h(a0, const_a); \ + v8u16 b1 = __msa_subs_u_h(a1, const_a); \ + SRAI_H2_UH(b0, b1, 6); \ + CLIP_UH2_0_255(b0, b1); \ + dst = (v16u8)__msa_pckev_b((v16i8)b1, (v16i8)b0); \ +} while (0) + +#define CALC_B8(y0, u0, dst) do { \ + const v8u16 const_a = (v8u16)__msa_fill_h(17685); \ + const v8u16 a0 = __msa_adds_u_h((v8u16)y0, u0); \ + v8u16 b0 = __msa_subs_u_h(a0, const_a); \ + b0 = SRAI_H(b0, 6); \ + CLIP_UH_0_255(b0); \ + dst = (v16u8)__msa_pckev_b((v16i8)b0, (v16i8)b0); \ +} while (0) + +#define CALC_RGB16(y, u, v, R, G, B) do { \ + const v16u8 zero = { 0 }; \ + v8u16 y0, y1, u0, u1, v0, v1; \ + v4u32 p0, p1, p2, p3; \ + const v16u8 in_y = LD_UB(y); \ + const v16u8 in_u = LD_UB(u); \ + const v16u8 in_v = LD_UB(v); \ + ILVRL_UW4(in_y, p0, p1, p2, p3); \ + MULTHI_16(p0, p1, p2, p3, 19077, y0, y1); \ + ILVRL_UW4(in_v, p0, p1, p2, p3); \ + MULTHI_16(p0, p1, p2, p3, 26149, v0, v1); \ + CALC_R16(y0, y1, v0, v1, R); \ + MULTHI_16(p0, p1, p2, p3, 13320, v0, v1); \ + ILVRL_UW4(in_u, p0, p1, p2, p3); \ + MULTHI_16(p0, p1, p2, p3, 6419, u0, u1); \ + CALC_G16(y0, y1, u0, u1, v0, v1, G); \ + MULTHI_16(p0, p1, p2, p3, 33050, u0, u1); \ + CALC_B16(y0, y1, u0, u1, B); \ +} while (0) + +#define CALC_RGB8(y, u, v, R, G, B) do { \ + const v16u8 zero = { 0 }; \ + v8u16 y0, u0, v0; \ + v4u32 p0, p1; \ + const v16u8 in_y = LD_UB(y); \ + const v16u8 in_u = LD_UB(u); \ + const v16u8 in_v = LD_UB(v); \ + ILVR_UW2(in_y, p0, p1); \ + MULTHI_8(p0, p1, 19077, y0); \ + ILVR_UW2(in_v, p0, p1); \ + MULTHI_8(p0, p1, 26149, v0); \ + CALC_R8(y0, v0, R); \ + MULTHI_8(p0, p1, 13320, v0); \ + ILVR_UW2(in_u, p0, p1); \ + MULTHI_8(p0, p1, 6419, u0); \ + CALC_G8(y0, u0, v0, G); \ + MULTHI_8(p0, p1, 33050, u0); \ + CALC_B8(y0, u0, B); \ +} while (0) + +#define STORE16_3(a0, a1, a2, dst) do { \ + const v16u8 mask0 = { 0, 1, 16, 2, 3, 17, 4, 5, 18, 6, 7, 19, \ + 8, 9, 20, 10 }; \ + const v16u8 mask1 = { 0, 21, 1, 2, 22, 3, 4, 23, 5, 6, 24, 7, \ + 8, 25, 9, 10 }; \ + const v16u8 mask2 = { 26, 0, 1, 27, 2, 3, 28, 4, 5, 29, 6, 7, \ + 30, 8, 9, 31 }; \ + v16u8 out0, out1, out2, tmp0, tmp1, tmp2; \ + ILVRL_B2_UB(a1, a0, tmp0, tmp1); \ + out0 = VSHF_UB(tmp0, a2, mask0); \ + tmp2 = SLDI_UB(tmp1, tmp0, 11); \ + out1 = VSHF_UB(tmp2, a2, mask1); \ + tmp2 = SLDI_UB(tmp1, tmp1, 6); \ + out2 = VSHF_UB(tmp2, a2, mask2); \ + ST_UB(out0, dst + 0); \ + ST_UB(out1, dst + 16); \ + ST_UB(out2, dst + 32); \ +} while (0) + +#define STORE8_3(a0, a1, a2, dst) do { \ + int64_t out_m; \ + const v16u8 mask0 = { 0, 1, 16, 2, 3, 17, 4, 5, 18, 6, 7, 19, \ + 8, 9, 20, 10 }; \ + const v16u8 mask1 = { 11, 21, 12, 13, 22, 14, 15, 23, \ + 255, 255, 255, 255, 255, 255, 255, 255 }; \ + const v16u8 tmp0 = (v16u8)__msa_ilvr_b((v16i8)a1, (v16i8)a0); \ + v16u8 out0, out1; \ + VSHF_B2_UB(tmp0, a2, tmp0, a2, mask0, mask1, out0, out1); \ + ST_UB(out0, dst); \ + out_m = __msa_copy_s_d((v2i64)out1, 0); \ + SD(out_m, dst + 16); \ +} while (0) + +#define STORE16_4(a0, a1, a2, a3, dst) do { \ + v16u8 tmp0, tmp1, tmp2, tmp3; \ + v16u8 out0, out1, out2, out3; \ + ILVRL_B2_UB(a1, a0, tmp0, tmp1); \ + ILVRL_B2_UB(a3, a2, tmp2, tmp3); \ + ILVRL_H2_UB(tmp2, tmp0, out0, out1); \ + ILVRL_H2_UB(tmp3, tmp1, out2, out3); \ + ST_UB(out0, dst + 0); \ + ST_UB(out1, dst + 16); \ + ST_UB(out2, dst + 32); \ + ST_UB(out3, dst + 48); \ +} while (0) + +#define STORE8_4(a0, a1, a2, a3, dst) do { \ + v16u8 tmp0, tmp1, tmp2, tmp3; \ + ILVR_B2_UB(a1, a0, a3, a2, tmp0, tmp1); \ + ILVRL_H2_UB(tmp1, tmp0, tmp2, tmp3); \ + ST_UB(tmp2, dst + 0); \ + ST_UB(tmp3, dst + 16); \ +} while (0) + +#define STORE2_16(a0, a1, dst) do { \ + v16u8 out0, out1; \ + ILVRL_B2_UB(a1, a0, out0, out1); \ + ST_UB(out0, dst + 0); \ + ST_UB(out1, dst + 16); \ +} while (0) + +#define STORE2_8(a0, a1, dst) do { \ + const v16u8 out0 = (v16u8)__msa_ilvr_b((v16i8)a1, (v16i8)a0); \ + ST_UB(out0, dst); \ +} while (0) + +#define CALC_RGBA4444(y, u, v, out0, out1, N, dst) do { \ + CALC_RGB##N(y, u, v, R, G, B); \ + tmp0 = ANDI_B(R, 0xf0); \ + tmp1 = SRAI_B(G, 4); \ + RG = tmp0 | tmp1; \ + tmp0 = ANDI_B(B, 0xf0); \ + BA = ORI_B(tmp0, 0x0f); \ + STORE2_##N(out0, out1, dst); \ +} while (0) + +#define CALC_RGB565(y, u, v, out0, out1, N, dst) do { \ + CALC_RGB##N(y, u, v, R, G, B); \ + tmp0 = ANDI_B(R, 0xf8); \ + tmp1 = SRAI_B(G, 5); \ + RG = tmp0 | tmp1; \ + tmp0 = SLLI_B(G, 3); \ + tmp1 = ANDI_B(tmp0, 0xe0); \ + tmp0 = SRAI_B(B, 3); \ + GB = tmp0 | tmp1; \ + STORE2_##N(out0, out1, dst); \ +} while (0) + +static WEBP_INLINE int Clip8(int v) { + return v < 0 ? 0 : v > 255 ? 255 : v; +} + +static void YuvToRgb(int y, int u, int v, uint8_t* const rgb) { + const int y1 = MultHi(y, 19077); + const int r1 = y1 + MultHi(v, 26149) - 14234; + const int g1 = y1 - MultHi(u, 6419) - MultHi(v, 13320) + 8708; + const int b1 = y1 + MultHi(u, 33050) - 17685; + rgb[0] = Clip8(r1 >> 6); + rgb[1] = Clip8(g1 >> 6); + rgb[2] = Clip8(b1 >> 6); +} + +static void YuvToBgr(int y, int u, int v, uint8_t* const bgr) { + const int y1 = MultHi(y, 19077); + const int r1 = y1 + MultHi(v, 26149) - 14234; + const int g1 = y1 - MultHi(u, 6419) - MultHi(v, 13320) + 8708; + const int b1 = y1 + MultHi(u, 33050) - 17685; + bgr[0] = Clip8(b1 >> 6); + bgr[1] = Clip8(g1 >> 6); + bgr[2] = Clip8(r1 >> 6); +} + +static void YuvToRgb565(int y, int u, int v, uint8_t* const rgb) { + const int y1 = MultHi(y, 19077); + const int r1 = y1 + MultHi(v, 26149) - 14234; + const int g1 = y1 - MultHi(u, 6419) - MultHi(v, 13320) + 8708; + const int b1 = y1 + MultHi(u, 33050) - 17685; + const int r = Clip8(r1 >> 6); + const int g = Clip8(g1 >> 6); + const int b = Clip8(b1 >> 6); + const int rg = (r & 0xf8) | (g >> 5); + const int gb = ((g << 3) & 0xe0) | (b >> 3); +#ifdef WEBP_SWAP_16BIT_CSP + rgb[0] = gb; + rgb[1] = rg; +#else + rgb[0] = rg; + rgb[1] = gb; +#endif +} + +static void YuvToRgba4444(int y, int u, int v, uint8_t* const argb) { + const int y1 = MultHi(y, 19077); + const int r1 = y1 + MultHi(v, 26149) - 14234; + const int g1 = y1 - MultHi(u, 6419) - MultHi(v, 13320) + 8708; + const int b1 = y1 + MultHi(u, 33050) - 17685; + const int r = Clip8(r1 >> 6); + const int g = Clip8(g1 >> 6); + const int b = Clip8(b1 >> 6); + const int rg = (r & 0xf0) | (g >> 4); + const int ba = (b & 0xf0) | 0x0f; // overwrite the lower 4 bits +#ifdef WEBP_SWAP_16BIT_CSP + argb[0] = ba; + argb[1] = rg; +#else + argb[0] = rg; + argb[1] = ba; +#endif +} + +static void YuvToArgb(uint8_t y, uint8_t u, uint8_t v, uint8_t* const argb) { + argb[0] = 0xff; + YuvToRgb(y, u, v, argb + 1); +} + +static void YuvToBgra(uint8_t y, uint8_t u, uint8_t v, uint8_t* const bgra) { + YuvToBgr(y, u, v, bgra); + bgra[3] = 0xff; +} + +static void YuvToRgba(uint8_t y, uint8_t u, uint8_t v, uint8_t* const rgba) { + YuvToRgb(y, u, v, rgba); + rgba[3] = 0xff; +} + +static void YuvToRgbLine(const uint8_t* y, const uint8_t* u, + const uint8_t* v, uint8_t* dst, int length) { + v16u8 R, G, B; + while (length >= 16) { + CALC_RGB16(y, u, v, R, G, B); + STORE16_3(R, G, B, dst); + y += 16; + u += 16; + v += 16; + dst += 16 * 3; + length -= 16; + } + if (length > 8) { + uint8_t temp[3 * 16] = { 0 }; + memcpy(temp, y, length * sizeof(*temp)); + CALC_RGB16(temp, u, v, R, G, B); + STORE16_3(R, G, B, temp); + memcpy(dst, temp, length * 3 * sizeof(*dst)); + } else if (length > 0) { + uint8_t temp[3 * 8] = { 0 }; + memcpy(temp, y, length * sizeof(*temp)); + CALC_RGB8(temp, u, v, R, G, B); + STORE8_3(R, G, B, temp); + memcpy(dst, temp, length * 3 * sizeof(*dst)); + } +} + +static void YuvToBgrLine(const uint8_t* y, const uint8_t* u, + const uint8_t* v, uint8_t* dst, int length) { + v16u8 R, G, B; + while (length >= 16) { + CALC_RGB16(y, u, v, R, G, B); + STORE16_3(B, G, R, dst); + y += 16; + u += 16; + v += 16; + dst += 16 * 3; + length -= 16; + } + if (length > 8) { + uint8_t temp[3 * 16] = { 0 }; + memcpy(temp, y, length * sizeof(*temp)); + CALC_RGB16(temp, u, v, R, G, B); + STORE16_3(B, G, R, temp); + memcpy(dst, temp, length * 3 * sizeof(*dst)); + } else if (length > 0) { + uint8_t temp[3 * 8] = { 0 }; + memcpy(temp, y, length * sizeof(*temp)); + CALC_RGB8(temp, u, v, R, G, B); + STORE8_3(B, G, R, temp); + memcpy(dst, temp, length * 3 * sizeof(*dst)); + } +} + +static void YuvToRgbaLine(const uint8_t* y, const uint8_t* u, + const uint8_t* v, uint8_t* dst, int length) { + v16u8 R, G, B; + const v16u8 A = (v16u8)__msa_ldi_b(0xff); + while (length >= 16) { + CALC_RGB16(y, u, v, R, G, B); + STORE16_4(R, G, B, A, dst); + y += 16; + u += 16; + v += 16; + dst += 16 * 4; + length -= 16; + } + if (length > 8) { + uint8_t temp[4 * 16] = { 0 }; + memcpy(temp, y, length * sizeof(*temp)); + CALC_RGB16(&temp[0], u, v, R, G, B); + STORE16_4(R, G, B, A, temp); + memcpy(dst, temp, length * 4 * sizeof(*dst)); + } else if (length > 0) { + uint8_t temp[4 * 8] = { 0 }; + memcpy(temp, y, length * sizeof(*temp)); + CALC_RGB8(temp, u, v, R, G, B); + STORE8_4(R, G, B, A, temp); + memcpy(dst, temp, length * 4 * sizeof(*dst)); + } +} + +static void YuvToBgraLine(const uint8_t* y, const uint8_t* u, + const uint8_t* v, uint8_t* dst, int length) { + v16u8 R, G, B; + const v16u8 A = (v16u8)__msa_ldi_b(0xff); + while (length >= 16) { + CALC_RGB16(y, u, v, R, G, B); + STORE16_4(B, G, R, A, dst); + y += 16; + u += 16; + v += 16; + dst += 16 * 4; + length -= 16; + } + if (length > 8) { + uint8_t temp[4 * 16] = { 0 }; + memcpy(temp, y, length * sizeof(*temp)); + CALC_RGB16(temp, u, v, R, G, B); + STORE16_4(B, G, R, A, temp); + memcpy(dst, temp, length * 4 * sizeof(*dst)); + } else if (length > 0) { + uint8_t temp[4 * 8] = { 0 }; + memcpy(temp, y, length * sizeof(*temp)); + CALC_RGB8(temp, u, v, R, G, B); + STORE8_4(B, G, R, A, temp); + memcpy(dst, temp, length * 4 * sizeof(*dst)); + } +} + +static void YuvToArgbLine(const uint8_t* y, const uint8_t* u, + const uint8_t* v, uint8_t* dst, int length) { + v16u8 R, G, B; + const v16u8 A = (v16u8)__msa_ldi_b(0xff); + while (length >= 16) { + CALC_RGB16(y, u, v, R, G, B); + STORE16_4(A, R, G, B, dst); + y += 16; + u += 16; + v += 16; + dst += 16 * 4; + length -= 16; + } + if (length > 8) { + uint8_t temp[4 * 16] = { 0 }; + memcpy(temp, y, length * sizeof(*temp)); + CALC_RGB16(temp, u, v, R, G, B); + STORE16_4(A, R, G, B, temp); + memcpy(dst, temp, length * 4 * sizeof(*dst)); + } else if (length > 0) { + uint8_t temp[4 * 8] = { 0 }; + memcpy(temp, y, length * sizeof(*temp)); + CALC_RGB8(temp, u, v, R, G, B); + STORE8_4(A, R, G, B, temp); + memcpy(dst, temp, length * 4 * sizeof(*dst)); + } +} + +static void YuvToRgba4444Line(const uint8_t* y, const uint8_t* u, + const uint8_t* v, uint8_t* dst, int length) { + v16u8 R, G, B, RG, BA, tmp0, tmp1; + while (length >= 16) { + #ifdef WEBP_SWAP_16BIT_CSP + CALC_RGBA4444(y, u, v, BA, RG, 16, dst); + #else + CALC_RGBA4444(y, u, v, RG, BA, 16, dst); + #endif + y += 16; + u += 16; + v += 16; + dst += 16 * 2; + length -= 16; + } + if (length > 8) { + uint8_t temp[2 * 16] = { 0 }; + memcpy(temp, y, length * sizeof(*temp)); +#ifdef WEBP_SWAP_16BIT_CSP + CALC_RGBA4444(temp, u, v, BA, RG, 16, temp); +#else + CALC_RGBA4444(temp, u, v, RG, BA, 16, temp); +#endif + memcpy(dst, temp, length * 2 * sizeof(*dst)); + } else if (length > 0) { + uint8_t temp[2 * 8] = { 0 }; + memcpy(temp, y, length * sizeof(*temp)); +#ifdef WEBP_SWAP_16BIT_CSP + CALC_RGBA4444(temp, u, v, BA, RG, 8, temp); +#else + CALC_RGBA4444(temp, u, v, RG, BA, 8, temp); +#endif + memcpy(dst, temp, length * 2 * sizeof(*dst)); + } +} + +static void YuvToRgb565Line(const uint8_t* y, const uint8_t* u, + const uint8_t* v, uint8_t* dst, int length) { + v16u8 R, G, B, RG, GB, tmp0, tmp1; + while (length >= 16) { + #ifdef WEBP_SWAP_16BIT_CSP + CALC_RGB565(y, u, v, GB, RG, 16, dst); + #else + CALC_RGB565(y, u, v, RG, GB, 16, dst); + #endif + y += 16; + u += 16; + v += 16; + dst += 16 * 2; + length -= 16; + } + if (length > 8) { + uint8_t temp[2 * 16] = { 0 }; + memcpy(temp, y, length * sizeof(*temp)); +#ifdef WEBP_SWAP_16BIT_CSP + CALC_RGB565(temp, u, v, GB, RG, 16, temp); +#else + CALC_RGB565(temp, u, v, RG, GB, 16, temp); +#endif + memcpy(dst, temp, length * 2 * sizeof(*dst)); + } else if (length > 0) { + uint8_t temp[2 * 8] = { 0 }; + memcpy(temp, y, length * sizeof(*temp)); +#ifdef WEBP_SWAP_16BIT_CSP + CALC_RGB565(temp, u, v, GB, RG, 8, temp); +#else + CALC_RGB565(temp, u, v, RG, GB, 8, temp); +#endif + memcpy(dst, temp, length * 2 * sizeof(*dst)); + } +} + +#define UPSAMPLE_32PIXELS(a, b, c, d) do { \ + v16u8 s = __msa_aver_u_b(a, d); \ + v16u8 t = __msa_aver_u_b(b, c); \ + const v16u8 st = s ^ t; \ + v16u8 ad = a ^ d; \ + v16u8 bc = b ^ c; \ + v16u8 t0 = ad | bc; \ + v16u8 t1 = t0 | st; \ + v16u8 t2 = ANDI_B(t1, 1); \ + v16u8 t3 = __msa_aver_u_b(s, t); \ + const v16u8 k = t3 - t2; \ + v16u8 diag1, diag2; \ + AVER_UB2_UB(t, k, s, k, t0, t1); \ + bc = bc & st; \ + ad = ad & st; \ + t = t ^ k; \ + s = s ^ k; \ + t2 = bc | t; \ + t3 = ad | s; \ + t2 = ANDI_B(t2, 1); \ + t3 = ANDI_B(t3, 1); \ + SUB2(t0, t2, t1, t3, diag1, diag2); \ + AVER_UB2_UB(a, diag1, b, diag2, t0, t1); \ + ILVRL_B2_UB(t1, t0, a, b); \ + if (pbot_y != NULL) { \ + AVER_UB2_UB(c, diag2, d, diag1, t0, t1); \ + ILVRL_B2_UB(t1, t0, c, d); \ + } \ +} while (0) + +#define UPSAMPLE_FUNC(FUNC_NAME, FUNC, XSTEP) \ +static void FUNC_NAME(const uint8_t* top_y, const uint8_t* bot_y, \ + const uint8_t* top_u, const uint8_t* top_v, \ + const uint8_t* cur_u, const uint8_t* cur_v, \ + uint8_t* top_dst, uint8_t* bot_dst, int len) \ +{ \ + int size = (len - 1) >> 1; \ + uint8_t temp_u[64]; \ + uint8_t temp_v[64]; \ + const uint32_t tl_uv = ((top_u[0]) | ((top_v[0]) << 16)); \ + const uint32_t l_uv = ((cur_u[0]) | ((cur_v[0]) << 16)); \ + const uint32_t uv0 = (3 * tl_uv + l_uv + 0x00020002u) >> 2; \ + const uint8_t* ptop_y = &top_y[1]; \ + uint8_t *ptop_dst = top_dst + XSTEP; \ + const uint8_t* pbot_y = &bot_y[1]; \ + uint8_t *pbot_dst = bot_dst + XSTEP; \ + \ + FUNC(top_y[0], uv0 & 0xff, (uv0 >> 16), top_dst); \ + if (bot_y != NULL) { \ + const uint32_t uv1 = (3 * l_uv + tl_uv + 0x00020002u) >> 2; \ + FUNC(bot_y[0], uv1 & 0xff, (uv1 >> 16), bot_dst); \ + } \ + while (size >= 16) { \ + v16u8 tu0, tu1, tv0, tv1, cu0, cu1, cv0, cv1; \ + LD_UB2(top_u, 1, tu0, tu1); \ + LD_UB2(cur_u, 1, cu0, cu1); \ + LD_UB2(top_v, 1, tv0, tv1); \ + LD_UB2(cur_v, 1, cv0, cv1); \ + UPSAMPLE_32PIXELS(tu0, tu1, cu0, cu1); \ + UPSAMPLE_32PIXELS(tv0, tv1, cv0, cv1); \ + ST_UB4(tu0, tu1, cu0, cu1, &temp_u[0], 16); \ + ST_UB4(tv0, tv1, cv0, cv1, &temp_v[0], 16); \ + FUNC##Line(ptop_y, &temp_u[ 0], &temp_v[0], ptop_dst, 32); \ + if (bot_y != NULL) { \ + FUNC##Line(pbot_y, &temp_u[32], &temp_v[32], pbot_dst, 32); \ + } \ + ptop_y += 32; \ + pbot_y += 32; \ + ptop_dst += XSTEP * 32; \ + pbot_dst += XSTEP * 32; \ + top_u += 16; \ + top_v += 16; \ + cur_u += 16; \ + cur_v += 16; \ + size -= 16; \ + } \ + if (size > 0) { \ + v16u8 tu0, tu1, tv0, tv1, cu0, cu1, cv0, cv1; \ + memcpy(&temp_u[ 0], top_u, 17 * sizeof(uint8_t)); \ + memcpy(&temp_u[32], cur_u, 17 * sizeof(uint8_t)); \ + memcpy(&temp_v[ 0], top_v, 17 * sizeof(uint8_t)); \ + memcpy(&temp_v[32], cur_v, 17 * sizeof(uint8_t)); \ + LD_UB2(&temp_u[ 0], 1, tu0, tu1); \ + LD_UB2(&temp_u[32], 1, cu0, cu1); \ + LD_UB2(&temp_v[ 0], 1, tv0, tv1); \ + LD_UB2(&temp_v[32], 1, cv0, cv1); \ + UPSAMPLE_32PIXELS(tu0, tu1, cu0, cu1); \ + UPSAMPLE_32PIXELS(tv0, tv1, cv0, cv1); \ + ST_UB4(tu0, tu1, cu0, cu1, &temp_u[0], 16); \ + ST_UB4(tv0, tv1, cv0, cv1, &temp_v[0], 16); \ + FUNC##Line(ptop_y, &temp_u[ 0], &temp_v[0], ptop_dst, size * 2); \ + if (bot_y != NULL) { \ + FUNC##Line(pbot_y, &temp_u[32], &temp_v[32], pbot_dst, size * 2); \ + } \ + top_u += size; \ + top_v += size; \ + cur_u += size; \ + cur_v += size; \ + } \ + if (!(len & 1)) { \ + const uint32_t t0 = ((top_u[0]) | ((top_v[0]) << 16)); \ + const uint32_t c0 = ((cur_u[0]) | ((cur_v[0]) << 16)); \ + const uint32_t tmp0 = (3 * t0 + c0 + 0x00020002u) >> 2; \ + FUNC(top_y[len - 1], tmp0 & 0xff, (tmp0 >> 16), \ + top_dst + (len - 1) * XSTEP); \ + if (bot_y != NULL) { \ + const uint32_t tmp1 = (3 * c0 + t0 + 0x00020002u) >> 2; \ + FUNC(bot_y[len - 1], tmp1 & 0xff, (tmp1 >> 16), \ + bot_dst + (len - 1) * XSTEP); \ + } \ + } \ +} + +UPSAMPLE_FUNC(UpsampleRgbLinePair, YuvToRgb, 3) +UPSAMPLE_FUNC(UpsampleBgrLinePair, YuvToBgr, 3) +UPSAMPLE_FUNC(UpsampleRgbaLinePair, YuvToRgba, 4) +UPSAMPLE_FUNC(UpsampleBgraLinePair, YuvToBgra, 4) +UPSAMPLE_FUNC(UpsampleArgbLinePair, YuvToArgb, 4) +UPSAMPLE_FUNC(UpsampleRgba4444LinePair, YuvToRgba4444, 2) +UPSAMPLE_FUNC(UpsampleRgb565LinePair, YuvToRgb565, 2) + +//------------------------------------------------------------------------------ +// Entry point + +extern WebPUpsampleLinePairFunc WebPUpsamplers[/* MODE_LAST */]; + +extern void WebPInitUpsamplersMSA(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitUpsamplersMSA(void) { + WebPUpsamplers[MODE_RGB] = UpsampleRgbLinePair; + WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePair; + WebPUpsamplers[MODE_BGR] = UpsampleBgrLinePair; + WebPUpsamplers[MODE_BGRA] = UpsampleBgraLinePair; + WebPUpsamplers[MODE_ARGB] = UpsampleArgbLinePair; + WebPUpsamplers[MODE_rgbA] = UpsampleRgbaLinePair; + WebPUpsamplers[MODE_bgrA] = UpsampleBgraLinePair; + WebPUpsamplers[MODE_Argb] = UpsampleArgbLinePair; + WebPUpsamplers[MODE_RGB_565] = UpsampleRgb565LinePair; + WebPUpsamplers[MODE_RGBA_4444] = UpsampleRgba4444LinePair; + WebPUpsamplers[MODE_rgbA_4444] = UpsampleRgba4444LinePair; +} + +#endif // FANCY_UPSAMPLING + +#endif // WEBP_USE_MSA + +#if !(defined(FANCY_UPSAMPLING) && defined(WEBP_USE_MSA)) +WEBP_DSP_INIT_STUB(WebPInitUpsamplersMSA) +#endif diff --git a/src/3rdparty/libwebp/src/dsp/upsampling_neon.c b/src/3rdparty/libwebp/src/dsp/upsampling_neon.c index 2b0c99b..d371a83 100644 --- a/src/3rdparty/libwebp/src/dsp/upsampling_neon.c +++ b/src/3rdparty/libwebp/src/dsp/upsampling_neon.c @@ -28,47 +28,34 @@ // U/V upsampling // Loads 9 pixels each from rows r1 and r2 and generates 16 pixels. -#define UPSAMPLE_16PIXELS(r1, r2, out) { \ - uint8x8_t a = vld1_u8(r1); \ - uint8x8_t b = vld1_u8(r1 + 1); \ - uint8x8_t c = vld1_u8(r2); \ - uint8x8_t d = vld1_u8(r2 + 1); \ - \ - uint16x8_t al = vshll_n_u8(a, 1); \ - uint16x8_t bl = vshll_n_u8(b, 1); \ - uint16x8_t cl = vshll_n_u8(c, 1); \ - uint16x8_t dl = vshll_n_u8(d, 1); \ - \ - uint8x8_t diag1, diag2; \ - uint16x8_t sl; \ - \ +#define UPSAMPLE_16PIXELS(r1, r2, out) do { \ + const uint8x8_t a = vld1_u8(r1 + 0); \ + const uint8x8_t b = vld1_u8(r1 + 1); \ + const uint8x8_t c = vld1_u8(r2 + 0); \ + const uint8x8_t d = vld1_u8(r2 + 1); \ /* a + b + c + d */ \ - sl = vaddl_u8(a, b); \ - sl = vaddw_u8(sl, c); \ - sl = vaddw_u8(sl, d); \ - \ - al = vaddq_u16(sl, al); /* 3a + b + c + d */ \ - bl = vaddq_u16(sl, bl); /* a + 3b + c + d */ \ - \ - al = vaddq_u16(al, dl); /* 3a + b + c + 3d */ \ - bl = vaddq_u16(bl, cl); /* a + 3b + 3c + d */ \ + const uint16x8_t ad = vaddl_u8(a, d); \ + const uint16x8_t bc = vaddl_u8(b, c); \ + const uint16x8_t abcd = vaddq_u16(ad, bc); \ + /* 3a + b + c + 3d */ \ + const uint16x8_t al = vaddq_u16(abcd, vshlq_n_u16(ad, 1)); \ + /* a + 3b + 3c + d */ \ + const uint16x8_t bl = vaddq_u16(abcd, vshlq_n_u16(bc, 1)); \ \ - diag2 = vshrn_n_u16(al, 3); \ - diag1 = vshrn_n_u16(bl, 3); \ + const uint8x8_t diag2 = vshrn_n_u16(al, 3); \ + const uint8x8_t diag1 = vshrn_n_u16(bl, 3); \ \ - a = vrhadd_u8(a, diag1); \ - b = vrhadd_u8(b, diag2); \ - c = vrhadd_u8(c, diag2); \ - d = vrhadd_u8(d, diag1); \ + const uint8x8_t A = vrhadd_u8(a, diag1); \ + const uint8x8_t B = vrhadd_u8(b, diag2); \ + const uint8x8_t C = vrhadd_u8(c, diag2); \ + const uint8x8_t D = vrhadd_u8(d, diag1); \ \ - { \ - uint8x8x2_t a_b, c_d; \ - INIT_VECTOR2(a_b, a, b); \ - INIT_VECTOR2(c_d, c, d); \ - vst2_u8(out, a_b); \ - vst2_u8(out + 32, c_d); \ - } \ -} + uint8x8x2_t A_B, C_D; \ + INIT_VECTOR2(A_B, A, B); \ + INIT_VECTOR2(C_D, C, D); \ + vst2_u8(out + 0, A_B); \ + vst2_u8(out + 32, C_D); \ +} while (0) // Turn the macro into a function for reducing code-size when non-critical static void Upsample16Pixels(const uint8_t *r1, const uint8_t *r2, @@ -93,7 +80,6 @@ static void Upsample16Pixels(const uint8_t *r1, const uint8_t *r2, static const int16_t kCoeffs1[4] = { 19077, 26149, 6419, 13320 }; #define v255 vdup_n_u8(255) -#define v_0x0f vdup_n_u8(15) #define STORE_Rgb(out, r, g, b) do { \ uint8x8x3_t r_g_b; \ @@ -132,21 +118,16 @@ static const int16_t kCoeffs1[4] = { 19077, 26149, 6419, 13320 }; #endif #define STORE_Rgba4444(out, r, g, b) do { \ - const uint8x8_t r1 = vshl_n_u8(vshr_n_u8(r, 4), 4); /* 4bits */ \ - const uint8x8_t g1 = vshr_n_u8(g, 4); \ - const uint8x8_t ba = vorr_u8(b, v_0x0f); \ - const uint8x8_t rg = vorr_u8(r1, g1); \ + const uint8x8_t rg = vsri_n_u8(r, g, 4); /* shift g, insert r */ \ + const uint8x8_t ba = vsri_n_u8(b, v255, 4); /* shift a, insert b */ \ const uint8x8x2_t rgba4444 = ZIP_U8(rg, ba); \ vst1q_u8(out, vcombine_u8(rgba4444.val[0], rgba4444.val[1])); \ } while (0) #define STORE_Rgb565(out, r, g, b) do { \ - const uint8x8_t r1 = vshl_n_u8(vshr_n_u8(r, 3), 3); /* 5bits */ \ - const uint8x8_t g1 = vshr_n_u8(g, 5); /* upper 3bits */\ - const uint8x8_t g2 = vshl_n_u8(vshr_n_u8(g, 2), 5); /* lower 3bits */\ - const uint8x8_t b1 = vshr_n_u8(b, 3); /* 5bits */ \ - const uint8x8_t rg = vorr_u8(r1, g1); \ - const uint8x8_t gb = vorr_u8(g2, b1); \ + const uint8x8_t rg = vsri_n_u8(r, g, 5); /* shift g and insert r */ \ + const uint8x8_t g1 = vshl_n_u8(g, 3); /* pre-shift g: 3bits */ \ + const uint8x8_t gb = vsri_n_u8(g1, b, 3); /* shift b and insert g */ \ const uint8x8x2_t rgb565 = ZIP_U8(rg, gb); \ vst1q_u8(out, vcombine_u8(rgb565.val[0], rgb565.val[1])); \ } while (0) diff --git a/src/3rdparty/libwebp/src/dsp/yuv.c b/src/3rdparty/libwebp/src/dsp/yuv.c index f50a253..dd7d9de 100644 --- a/src/3rdparty/libwebp/src/dsp/yuv.c +++ b/src/3rdparty/libwebp/src/dsp/yuv.c @@ -13,6 +13,8 @@ #include "./yuv.h" +#include <stdlib.h> + #if defined(WEBP_YUV_USE_TABLE) static int done = 0; @@ -244,6 +246,48 @@ void WebPConvertRGBA32ToUV_C(const uint16_t* rgb, //----------------------------------------------------------------------------- +#define MAX_Y ((1 << 10) - 1) // 10b precision over 16b-arithmetic +static uint16_t clip_y(int v) { + return (v < 0) ? 0 : (v > MAX_Y) ? MAX_Y : (uint16_t)v; +} + +static uint64_t SharpYUVUpdateY_C(const uint16_t* ref, const uint16_t* src, + uint16_t* dst, int len) { + uint64_t diff = 0; + int i; + for (i = 0; i < len; ++i) { + const int diff_y = ref[i] - src[i]; + const int new_y = (int)dst[i] + diff_y; + dst[i] = clip_y(new_y); + diff += (uint64_t)abs(diff_y); + } + return diff; +} + +static void SharpYUVUpdateRGB_C(const int16_t* ref, const int16_t* src, + int16_t* dst, int len) { + int i; + for (i = 0; i < len; ++i) { + const int diff_uv = ref[i] - src[i]; + dst[i] += diff_uv; + } +} + +static void SharpYUVFilterRow_C(const int16_t* A, const int16_t* B, int len, + const uint16_t* best_y, uint16_t* out) { + int i; + for (i = 0; i < len; ++i, ++A, ++B) { + const int v0 = (A[0] * 9 + A[1] * 3 + B[0] * 3 + B[1] + 8) >> 4; + const int v1 = (A[1] * 9 + A[0] * 3 + B[1] * 3 + B[0] + 8) >> 4; + out[2 * i + 0] = clip_y(best_y[2 * i + 0] + v0); + out[2 * i + 1] = clip_y(best_y[2 * i + 1] + v1); + } +} + +#undef MAX_Y + +//----------------------------------------------------------------------------- + void (*WebPConvertRGB24ToY)(const uint8_t* rgb, uint8_t* y, int width); void (*WebPConvertBGR24ToY)(const uint8_t* bgr, uint8_t* y, int width); void (*WebPConvertRGBA32ToUV)(const uint16_t* rgb, @@ -253,10 +297,18 @@ void (*WebPConvertARGBToY)(const uint32_t* argb, uint8_t* y, int width); void (*WebPConvertARGBToUV)(const uint32_t* argb, uint8_t* u, uint8_t* v, int src_width, int do_store); +uint64_t (*WebPSharpYUVUpdateY)(const uint16_t* ref, const uint16_t* src, + uint16_t* dst, int len); +void (*WebPSharpYUVUpdateRGB)(const int16_t* ref, const int16_t* src, + int16_t* dst, int len); +void (*WebPSharpYUVFilterRow)(const int16_t* A, const int16_t* B, int len, + const uint16_t* best_y, uint16_t* out); + static volatile VP8CPUInfo rgba_to_yuv_last_cpuinfo_used = (VP8CPUInfo)&rgba_to_yuv_last_cpuinfo_used; extern void WebPInitConvertARGBToYUVSSE2(void); +extern void WebPInitSharpYUVSSE2(void); WEBP_TSAN_IGNORE_FUNCTION void WebPInitConvertARGBToYUV(void) { if (rgba_to_yuv_last_cpuinfo_used == VP8GetCPUInfo) return; @@ -269,10 +321,15 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitConvertARGBToYUV(void) { WebPConvertRGBA32ToUV = WebPConvertRGBA32ToUV_C; + WebPSharpYUVUpdateY = SharpYUVUpdateY_C; + WebPSharpYUVUpdateRGB = SharpYUVUpdateRGB_C; + WebPSharpYUVFilterRow = SharpYUVFilterRow_C; + if (VP8GetCPUInfo != NULL) { #if defined(WEBP_USE_SSE2) if (VP8GetCPUInfo(kSSE2)) { WebPInitConvertARGBToYUVSSE2(); + WebPInitSharpYUVSSE2(); } #endif // WEBP_USE_SSE2 } diff --git a/src/3rdparty/libwebp/src/dsp/yuv.h b/src/3rdparty/libwebp/src/dsp/yuv.h index 01c40fc..1d33b58 100644 --- a/src/3rdparty/libwebp/src/dsp/yuv.h +++ b/src/3rdparty/libwebp/src/dsp/yuv.h @@ -36,7 +36,7 @@ #define WEBP_DSP_YUV_H_ #include "./dsp.h" -#include "../dec/decode_vp8.h" +#include "../dec/vp8_dec.h" #if defined(WEBP_EXPERIMENTAL_FEATURES) // Do NOT activate this feature for real compression. This is only experimental! diff --git a/src/3rdparty/libwebp/src/dsp/yuv_sse2.c b/src/3rdparty/libwebp/src/dsp/yuv_sse2.c index e19bddf..e33c2bb 100644 --- a/src/3rdparty/libwebp/src/dsp/yuv_sse2.c +++ b/src/3rdparty/libwebp/src/dsp/yuv_sse2.c @@ -15,6 +15,8 @@ #if defined(WEBP_USE_SSE2) +#include "./common_sse2.h" +#include <stdlib.h> #include <emmintrin.h> //----------------------------------------------------------------------------- @@ -155,30 +157,13 @@ static WEBP_INLINE void PackAndStore565(const __m128i* const R, _mm_storeu_si128((__m128i*)dst, rgb565); } -// Function used several times in PlanarTo24b. -// It samples the in buffer as follows: one every two unsigned char is stored -// at the beginning of the buffer, while the other half is stored at the end. -static WEBP_INLINE void PlanarTo24bHelper(const __m128i* const in /*in[6]*/, - __m128i* const out /*out[6]*/) { - const __m128i v_mask = _mm_set1_epi16(0x00ff); - - // Take one every two upper 8b values. - out[0] = _mm_packus_epi16(_mm_and_si128(in[0], v_mask), - _mm_and_si128(in[1], v_mask)); - out[1] = _mm_packus_epi16(_mm_and_si128(in[2], v_mask), - _mm_and_si128(in[3], v_mask)); - out[2] = _mm_packus_epi16(_mm_and_si128(in[4], v_mask), - _mm_and_si128(in[5], v_mask)); - // Take one every two lower 8b values. - out[3] = _mm_packus_epi16(_mm_srli_epi16(in[0], 8), _mm_srli_epi16(in[1], 8)); - out[4] = _mm_packus_epi16(_mm_srli_epi16(in[2], 8), _mm_srli_epi16(in[3], 8)); - out[5] = _mm_packus_epi16(_mm_srli_epi16(in[4], 8), _mm_srli_epi16(in[5], 8)); -} - // Pack the planar buffers // rrrr... rrrr... gggg... gggg... bbbb... bbbb.... // triplet by triplet in the output buffer rgb as rgbrgbrgbrgb ... -static WEBP_INLINE void PlanarTo24b(__m128i* const in /*in[6]*/, uint8_t* rgb) { +static WEBP_INLINE void PlanarTo24b(__m128i* const in0, __m128i* const in1, + __m128i* const in2, __m128i* const in3, + __m128i* const in4, __m128i* const in5, + uint8_t* const rgb) { // The input is 6 registers of sixteen 8b but for the sake of explanation, // let's take 6 registers of four 8b values. // To pack, we will keep taking one every two 8b integer and move it @@ -191,22 +176,15 @@ static WEBP_INLINE void PlanarTo24b(__m128i* const in /*in[6]*/, uint8_t* rgb) { // Repeat the same permutations twice more: // r0r4g0g4 | b0b4r1r5 | g1g5b1b5 | r2r6g2g6 | b2b6r3r7 | g3g7b3b7 // r0g0b0r1 | g1b1r2g2 | b2r3g3b3 | r4g4b4r5 | g5b5r6g6 | b6r7g7b7 - __m128i tmp[6]; - PlanarTo24bHelper(in, tmp); - PlanarTo24bHelper(tmp, in); - PlanarTo24bHelper(in, tmp); - // We need to do it two more times than the example as we have sixteen bytes. - PlanarTo24bHelper(tmp, in); - PlanarTo24bHelper(in, tmp); - - _mm_storeu_si128((__m128i*)(rgb + 0), tmp[0]); - _mm_storeu_si128((__m128i*)(rgb + 16), tmp[1]); - _mm_storeu_si128((__m128i*)(rgb + 32), tmp[2]); - _mm_storeu_si128((__m128i*)(rgb + 48), tmp[3]); - _mm_storeu_si128((__m128i*)(rgb + 64), tmp[4]); - _mm_storeu_si128((__m128i*)(rgb + 80), tmp[5]); -} -#undef MK_UINT32 + VP8PlanarTo24b(in0, in1, in2, in3, in4, in5); + + _mm_storeu_si128((__m128i*)(rgb + 0), *in0); + _mm_storeu_si128((__m128i*)(rgb + 16), *in1); + _mm_storeu_si128((__m128i*)(rgb + 32), *in2); + _mm_storeu_si128((__m128i*)(rgb + 48), *in3); + _mm_storeu_si128((__m128i*)(rgb + 64), *in4); + _mm_storeu_si128((__m128i*)(rgb + 80), *in5); +} void VP8YuvToRgba32(const uint8_t* y, const uint8_t* u, const uint8_t* v, uint8_t* dst) { @@ -265,29 +243,29 @@ void VP8YuvToRgb56532(const uint8_t* y, const uint8_t* u, const uint8_t* v, void VP8YuvToRgb32(const uint8_t* y, const uint8_t* u, const uint8_t* v, uint8_t* dst) { __m128i R0, R1, R2, R3, G0, G1, G2, G3, B0, B1, B2, B3; - __m128i rgb[6]; + __m128i rgb0, rgb1, rgb2, rgb3, rgb4, rgb5; - YUV444ToRGB(y + 0, u + 0, v + 0, &R0, &G0, &B0); - YUV444ToRGB(y + 8, u + 8, v + 8, &R1, &G1, &B1); + YUV444ToRGB(y + 0, u + 0, v + 0, &R0, &G0, &B0); + YUV444ToRGB(y + 8, u + 8, v + 8, &R1, &G1, &B1); YUV444ToRGB(y + 16, u + 16, v + 16, &R2, &G2, &B2); YUV444ToRGB(y + 24, u + 24, v + 24, &R3, &G3, &B3); // Cast to 8b and store as RRRRGGGGBBBB. - rgb[0] = _mm_packus_epi16(R0, R1); - rgb[1] = _mm_packus_epi16(R2, R3); - rgb[2] = _mm_packus_epi16(G0, G1); - rgb[3] = _mm_packus_epi16(G2, G3); - rgb[4] = _mm_packus_epi16(B0, B1); - rgb[5] = _mm_packus_epi16(B2, B3); + rgb0 = _mm_packus_epi16(R0, R1); + rgb1 = _mm_packus_epi16(R2, R3); + rgb2 = _mm_packus_epi16(G0, G1); + rgb3 = _mm_packus_epi16(G2, G3); + rgb4 = _mm_packus_epi16(B0, B1); + rgb5 = _mm_packus_epi16(B2, B3); // Pack as RGBRGBRGBRGB. - PlanarTo24b(rgb, dst); + PlanarTo24b(&rgb0, &rgb1, &rgb2, &rgb3, &rgb4, &rgb5, dst); } void VP8YuvToBgr32(const uint8_t* y, const uint8_t* u, const uint8_t* v, uint8_t* dst) { __m128i R0, R1, R2, R3, G0, G1, G2, G3, B0, B1, B2, B3; - __m128i bgr[6]; + __m128i bgr0, bgr1, bgr2, bgr3, bgr4, bgr5; YUV444ToRGB(y + 0, u + 0, v + 0, &R0, &G0, &B0); YUV444ToRGB(y + 8, u + 8, v + 8, &R1, &G1, &B1); @@ -295,15 +273,15 @@ void VP8YuvToBgr32(const uint8_t* y, const uint8_t* u, const uint8_t* v, YUV444ToRGB(y + 24, u + 24, v + 24, &R3, &G3, &B3); // Cast to 8b and store as BBBBGGGGRRRR. - bgr[0] = _mm_packus_epi16(B0, B1); - bgr[1] = _mm_packus_epi16(B2, B3); - bgr[2] = _mm_packus_epi16(G0, G1); - bgr[3] = _mm_packus_epi16(G2, G3); - bgr[4] = _mm_packus_epi16(R0, R1); - bgr[5] = _mm_packus_epi16(R2, R3); + bgr0 = _mm_packus_epi16(B0, B1); + bgr1 = _mm_packus_epi16(B2, B3); + bgr2 = _mm_packus_epi16(G0, G1); + bgr3 = _mm_packus_epi16(G2, G3); + bgr4 = _mm_packus_epi16(R0, R1); + bgr5= _mm_packus_epi16(R2, R3); // Pack as BGRBGRBGRBGR. - PlanarTo24b(bgr, dst); + PlanarTo24b(&bgr0, &bgr1, &bgr2, &bgr3, &bgr4, &bgr5, dst); } //----------------------------------------------------------------------------- @@ -377,7 +355,7 @@ static void YuvToRgbRow(const uint8_t* y, const uint8_t* u, const uint8_t* v, int n; for (n = 0; n + 32 <= len; n += 32, dst += 32 * 3) { __m128i R0, R1, R2, R3, G0, G1, G2, G3, B0, B1, B2, B3; - __m128i rgb[6]; + __m128i rgb0, rgb1, rgb2, rgb3, rgb4, rgb5; YUV420ToRGB(y + 0, u + 0, v + 0, &R0, &G0, &B0); YUV420ToRGB(y + 8, u + 4, v + 4, &R1, &G1, &B1); @@ -385,15 +363,15 @@ static void YuvToRgbRow(const uint8_t* y, const uint8_t* u, const uint8_t* v, YUV420ToRGB(y + 24, u + 12, v + 12, &R3, &G3, &B3); // Cast to 8b and store as RRRRGGGGBBBB. - rgb[0] = _mm_packus_epi16(R0, R1); - rgb[1] = _mm_packus_epi16(R2, R3); - rgb[2] = _mm_packus_epi16(G0, G1); - rgb[3] = _mm_packus_epi16(G2, G3); - rgb[4] = _mm_packus_epi16(B0, B1); - rgb[5] = _mm_packus_epi16(B2, B3); + rgb0 = _mm_packus_epi16(R0, R1); + rgb1 = _mm_packus_epi16(R2, R3); + rgb2 = _mm_packus_epi16(G0, G1); + rgb3 = _mm_packus_epi16(G2, G3); + rgb4 = _mm_packus_epi16(B0, B1); + rgb5 = _mm_packus_epi16(B2, B3); // Pack as RGBRGBRGBRGB. - PlanarTo24b(rgb, dst); + PlanarTo24b(&rgb0, &rgb1, &rgb2, &rgb3, &rgb4, &rgb5, dst); y += 32; u += 16; @@ -413,7 +391,7 @@ static void YuvToBgrRow(const uint8_t* y, const uint8_t* u, const uint8_t* v, int n; for (n = 0; n + 32 <= len; n += 32, dst += 32 * 3) { __m128i R0, R1, R2, R3, G0, G1, G2, G3, B0, B1, B2, B3; - __m128i bgr[6]; + __m128i bgr0, bgr1, bgr2, bgr3, bgr4, bgr5; YUV420ToRGB(y + 0, u + 0, v + 0, &R0, &G0, &B0); YUV420ToRGB(y + 8, u + 4, v + 4, &R1, &G1, &B1); @@ -421,15 +399,15 @@ static void YuvToBgrRow(const uint8_t* y, const uint8_t* u, const uint8_t* v, YUV420ToRGB(y + 24, u + 12, v + 12, &R3, &G3, &B3); // Cast to 8b and store as BBBBGGGGRRRR. - bgr[0] = _mm_packus_epi16(B0, B1); - bgr[1] = _mm_packus_epi16(B2, B3); - bgr[2] = _mm_packus_epi16(G0, G1); - bgr[3] = _mm_packus_epi16(G2, G3); - bgr[4] = _mm_packus_epi16(R0, R1); - bgr[5] = _mm_packus_epi16(R2, R3); + bgr0 = _mm_packus_epi16(B0, B1); + bgr1 = _mm_packus_epi16(B2, B3); + bgr2 = _mm_packus_epi16(G0, G1); + bgr3 = _mm_packus_epi16(G2, G3); + bgr4 = _mm_packus_epi16(R0, R1); + bgr5 = _mm_packus_epi16(R2, R3); // Pack as BGRBGRBGRBGR. - PlanarTo24b(bgr, dst); + PlanarTo24b(&bgr0, &bgr1, &bgr2, &bgr3, &bgr4, &bgr5, dst); y += 32; u += 16; @@ -499,25 +477,19 @@ static WEBP_INLINE void RGB24PackedToPlanar(const uint8_t* const rgb, // Convert 8 packed ARGB to r[], g[], b[] static WEBP_INLINE void RGB32PackedToPlanar(const uint32_t* const argb, - __m128i* const r, - __m128i* const g, - __m128i* const b) { + __m128i* const rgb /*in[6]*/) { const __m128i zero = _mm_setzero_si128(); - const __m128i in0 = LOAD_16(argb + 0); // argb3 | argb2 | argb1 | argb0 - const __m128i in1 = LOAD_16(argb + 4); // argb7 | argb6 | argb5 | argb4 - // column-wise transpose - const __m128i A0 = _mm_unpacklo_epi8(in0, in1); - const __m128i A1 = _mm_unpackhi_epi8(in0, in1); - const __m128i B0 = _mm_unpacklo_epi8(A0, A1); - const __m128i B1 = _mm_unpackhi_epi8(A0, A1); - // C0 = g7 g6 ... g1 g0 | b7 b6 ... b1 b0 - // C1 = a7 a6 ... a1 a0 | r7 r6 ... r1 r0 - const __m128i C0 = _mm_unpacklo_epi8(B0, B1); - const __m128i C1 = _mm_unpackhi_epi8(B0, B1); - // store 16b - *r = _mm_unpacklo_epi8(C1, zero); - *g = _mm_unpackhi_epi8(C0, zero); - *b = _mm_unpacklo_epi8(C0, zero); + __m128i a0 = LOAD_16(argb + 0); + __m128i a1 = LOAD_16(argb + 4); + __m128i a2 = LOAD_16(argb + 8); + __m128i a3 = LOAD_16(argb + 12); + VP8L32bToPlanar(&a0, &a1, &a2, &a3); + rgb[0] = _mm_unpacklo_epi8(a1, zero); + rgb[1] = _mm_unpackhi_epi8(a1, zero); + rgb[2] = _mm_unpacklo_epi8(a2, zero); + rgb[3] = _mm_unpackhi_epi8(a2, zero); + rgb[4] = _mm_unpacklo_epi8(a3, zero); + rgb[5] = _mm_unpackhi_epi8(a3, zero); } // This macro computes (RG * MULT_RG + GB * MULT_GB + ROUNDER) >> DESCALE_FIX @@ -649,11 +621,10 @@ static void ConvertARGBToY(const uint32_t* argb, uint8_t* y, int width) { const int max_width = width & ~15; int i; for (i = 0; i < max_width; i += 16) { - __m128i r, g, b, Y0, Y1; - RGB32PackedToPlanar(&argb[i + 0], &r, &g, &b); - ConvertRGBToY(&r, &g, &b, &Y0); - RGB32PackedToPlanar(&argb[i + 8], &r, &g, &b); - ConvertRGBToY(&r, &g, &b, &Y1); + __m128i Y0, Y1, rgb[6]; + RGB32PackedToPlanar(&argb[i], rgb); + ConvertRGBToY(&rgb[0], &rgb[2], &rgb[4], &Y0); + ConvertRGBToY(&rgb[1], &rgb[3], &rgb[5], &Y1); STORE_16(_mm_packus_epi16(Y0, Y1), y + i); } for (; i < width; ++i) { // left-over @@ -678,20 +649,18 @@ static void ConvertARGBToUV(const uint32_t* argb, uint8_t* u, uint8_t* v, const int max_width = src_width & ~31; int i; for (i = 0; i < max_width; i += 32, u += 16, v += 16) { - __m128i r0, g0, b0, r1, g1, b1, U0, V0, U1, V1; - RGB32PackedToPlanar(&argb[i + 0], &r0, &g0, &b0); - RGB32PackedToPlanar(&argb[i + 8], &r1, &g1, &b1); - HorizontalAddPack(&r0, &r1, &r0); - HorizontalAddPack(&g0, &g1, &g0); - HorizontalAddPack(&b0, &b1, &b0); - ConvertRGBToUV(&r0, &g0, &b0, &U0, &V0); - - RGB32PackedToPlanar(&argb[i + 16], &r0, &g0, &b0); - RGB32PackedToPlanar(&argb[i + 24], &r1, &g1, &b1); - HorizontalAddPack(&r0, &r1, &r0); - HorizontalAddPack(&g0, &g1, &g0); - HorizontalAddPack(&b0, &b1, &b0); - ConvertRGBToUV(&r0, &g0, &b0, &U1, &V1); + __m128i rgb[6], U0, V0, U1, V1; + RGB32PackedToPlanar(&argb[i], rgb); + HorizontalAddPack(&rgb[0], &rgb[1], &rgb[0]); + HorizontalAddPack(&rgb[2], &rgb[3], &rgb[2]); + HorizontalAddPack(&rgb[4], &rgb[5], &rgb[4]); + ConvertRGBToUV(&rgb[0], &rgb[2], &rgb[4], &U0, &V0); + + RGB32PackedToPlanar(&argb[i + 16], rgb); + HorizontalAddPack(&rgb[0], &rgb[1], &rgb[0]); + HorizontalAddPack(&rgb[2], &rgb[3], &rgb[2]); + HorizontalAddPack(&rgb[4], &rgb[5], &rgb[4]); + ConvertRGBToUV(&rgb[0], &rgb[2], &rgb[4], &U1, &V1); U0 = _mm_packus_epi16(U0, U1); V0 = _mm_packus_epi16(V0, V1); @@ -767,9 +736,128 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitConvertARGBToYUVSSE2(void) { WebPConvertRGBA32ToUV = ConvertRGBA32ToUV; } +//------------------------------------------------------------------------------ + +#define MAX_Y ((1 << 10) - 1) // 10b precision over 16b-arithmetic +static uint16_t clip_y(int v) { + return (v < 0) ? 0 : (v > MAX_Y) ? MAX_Y : (uint16_t)v; +} + +static uint64_t SharpYUVUpdateY_SSE2(const uint16_t* ref, const uint16_t* src, + uint16_t* dst, int len) { + uint64_t diff = 0; + uint32_t tmp[4]; + int i; + const __m128i zero = _mm_setzero_si128(); + const __m128i max = _mm_set1_epi16(MAX_Y); + const __m128i one = _mm_set1_epi16(1); + __m128i sum = zero; + + for (i = 0; i + 8 <= len; i += 8) { + const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i)); + const __m128i B = _mm_loadu_si128((const __m128i*)(src + i)); + const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i)); + const __m128i D = _mm_sub_epi16(A, B); // diff_y + const __m128i E = _mm_cmpgt_epi16(zero, D); // sign (-1 or 0) + const __m128i F = _mm_add_epi16(C, D); // new_y + const __m128i G = _mm_or_si128(E, one); // -1 or 1 + const __m128i H = _mm_max_epi16(_mm_min_epi16(F, max), zero); + const __m128i I = _mm_madd_epi16(D, G); // sum(abs(...)) + _mm_storeu_si128((__m128i*)(dst + i), H); + sum = _mm_add_epi32(sum, I); + } + _mm_storeu_si128((__m128i*)tmp, sum); + diff = tmp[3] + tmp[2] + tmp[1] + tmp[0]; + for (; i < len; ++i) { + const int diff_y = ref[i] - src[i]; + const int new_y = (int)dst[i] + diff_y; + dst[i] = clip_y(new_y); + diff += (uint64_t)abs(diff_y); + } + return diff; +} + +static void SharpYUVUpdateRGB_SSE2(const int16_t* ref, const int16_t* src, + int16_t* dst, int len) { + int i = 0; + for (i = 0; i + 8 <= len; i += 8) { + const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i)); + const __m128i B = _mm_loadu_si128((const __m128i*)(src + i)); + const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i)); + const __m128i D = _mm_sub_epi16(A, B); // diff_uv + const __m128i E = _mm_add_epi16(C, D); // new_uv + _mm_storeu_si128((__m128i*)(dst + i), E); + } + for (; i < len; ++i) { + const int diff_uv = ref[i] - src[i]; + dst[i] += diff_uv; + } +} + +static void SharpYUVFilterRow_SSE2(const int16_t* A, const int16_t* B, int len, + const uint16_t* best_y, uint16_t* out) { + int i; + const __m128i kCst8 = _mm_set1_epi16(8); + const __m128i max = _mm_set1_epi16(MAX_Y); + const __m128i zero = _mm_setzero_si128(); + for (i = 0; i + 8 <= len; i += 8) { + const __m128i a0 = _mm_loadu_si128((const __m128i*)(A + i + 0)); + const __m128i a1 = _mm_loadu_si128((const __m128i*)(A + i + 1)); + const __m128i b0 = _mm_loadu_si128((const __m128i*)(B + i + 0)); + const __m128i b1 = _mm_loadu_si128((const __m128i*)(B + i + 1)); + const __m128i a0b1 = _mm_add_epi16(a0, b1); + const __m128i a1b0 = _mm_add_epi16(a1, b0); + const __m128i a0a1b0b1 = _mm_add_epi16(a0b1, a1b0); // A0+A1+B0+B1 + const __m128i a0a1b0b1_8 = _mm_add_epi16(a0a1b0b1, kCst8); + const __m128i a0b1_2 = _mm_add_epi16(a0b1, a0b1); // 2*(A0+B1) + const __m128i a1b0_2 = _mm_add_epi16(a1b0, a1b0); // 2*(A1+B0) + const __m128i c0 = _mm_srai_epi16(_mm_add_epi16(a0b1_2, a0a1b0b1_8), 3); + const __m128i c1 = _mm_srai_epi16(_mm_add_epi16(a1b0_2, a0a1b0b1_8), 3); + const __m128i d0 = _mm_add_epi16(c1, a0); + const __m128i d1 = _mm_add_epi16(c0, a1); + const __m128i e0 = _mm_srai_epi16(d0, 1); + const __m128i e1 = _mm_srai_epi16(d1, 1); + const __m128i f0 = _mm_unpacklo_epi16(e0, e1); + const __m128i f1 = _mm_unpackhi_epi16(e0, e1); + const __m128i g0 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 0)); + const __m128i g1 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 8)); + const __m128i h0 = _mm_add_epi16(g0, f0); + const __m128i h1 = _mm_add_epi16(g1, f1); + const __m128i i0 = _mm_max_epi16(_mm_min_epi16(h0, max), zero); + const __m128i i1 = _mm_max_epi16(_mm_min_epi16(h1, max), zero); + _mm_storeu_si128((__m128i*)(out + 2 * i + 0), i0); + _mm_storeu_si128((__m128i*)(out + 2 * i + 8), i1); + } + for (; i < len; ++i) { + // (9 * A0 + 3 * A1 + 3 * B0 + B1 + 8) >> 4 = + // = (8 * A0 + 2 * (A1 + B0) + (A0 + A1 + B0 + B1 + 8)) >> 4 + // We reuse the common sub-expressions. + const int a0b1 = A[i + 0] + B[i + 1]; + const int a1b0 = A[i + 1] + B[i + 0]; + const int a0a1b0b1 = a0b1 + a1b0 + 8; + const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4; + const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4; + out[2 * i + 0] = clip_y(best_y[2 * i + 0] + v0); + out[2 * i + 1] = clip_y(best_y[2 * i + 1] + v1); + } +} + +#undef MAX_Y + +//------------------------------------------------------------------------------ + +extern void WebPInitSharpYUVSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void WebPInitSharpYUVSSE2(void) { + WebPSharpYUVUpdateY = SharpYUVUpdateY_SSE2; + WebPSharpYUVUpdateRGB = SharpYUVUpdateRGB_SSE2; + WebPSharpYUVFilterRow = SharpYUVFilterRow_SSE2; +} + #else // !WEBP_USE_SSE2 WEBP_DSP_INIT_STUB(WebPInitSamplersSSE2) WEBP_DSP_INIT_STUB(WebPInitConvertARGBToYUVSSE2) +WEBP_DSP_INIT_STUB(WebPInitSharpYUVSSE2) #endif // WEBP_USE_SSE2 |