summaryrefslogtreecommitdiffstats
path: root/src/3rdparty/libwebp/src/dsp
diff options
context:
space:
mode:
authorLiang Qi <liang.qi@qt.io>2017-03-07 13:05:21 +0100
committerLiang Qi <liang.qi@qt.io>2017-03-13 10:47:45 +0000
commitb7ec9e78633d8f2c75a8b02e17e169497bb103e2 (patch)
treee4be04af4dbcf8cd635715efdf4e769281183746 /src/3rdparty/libwebp/src/dsp
parentf2dbc67c2b032a5f27d0224e020fb6dfcd3fd142 (diff)
Bundled libwebp updated to version 0.6.0
This commit imports libwebp 0.6.0, including AUTHORS, COPYING, ChangeLog, NEWS, PATENTS, README and src directories. In src, only includes header and source files. Upstream changes since 0.5.1 have been merged in. Also updated version in qt_attribution.json. Conflicts: src/3rdparty/libwebp.pri src/3rdparty/libwebp/qt_attribution.json src/3rdparty/libwebp/src/webp/config.h Change-Id: I001aa7a3fabf0130b54f9005c23aa822bc1d0ec1 Reviewed-by: Eirik Aavitsland <eirik.aavitsland@qt.io>
Diffstat (limited to 'src/3rdparty/libwebp/src/dsp')
-rw-r--r--src/3rdparty/libwebp/src/dsp/alpha_processing.c38
-rw-r--r--src/3rdparty/libwebp/src/dsp/alpha_processing_neon.c191
-rw-r--r--src/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c141
-rw-r--r--src/3rdparty/libwebp/src/dsp/common_sse2.h85
-rw-r--r--src/3rdparty/libwebp/src/dsp/cost.c2
-rw-r--r--src/3rdparty/libwebp/src/dsp/cost_mips32.c2
-rw-r--r--src/3rdparty/libwebp/src/dsp/cost_mips_dsp_r2.c2
-rw-r--r--src/3rdparty/libwebp/src/dsp/cost_sse2.c4
-rw-r--r--src/3rdparty/libwebp/src/dsp/cpu.c45
-rw-r--r--src/3rdparty/libwebp/src/dsp/dec.c4
-rw-r--r--src/3rdparty/libwebp/src/dsp/dec_clip_tables.c8
-rw-r--r--src/3rdparty/libwebp/src/dsp/dec_msa.c847
-rw-r--r--src/3rdparty/libwebp/src/dsp/dec_neon.c40
-rw-r--r--src/3rdparty/libwebp/src/dsp/dec_sse2.c4
-rw-r--r--src/3rdparty/libwebp/src/dsp/dec_sse41.c2
-rw-r--r--src/3rdparty/libwebp/src/dsp/dsp.h57
-rw-r--r--src/3rdparty/libwebp/src/dsp/enc.c180
-rw-r--r--src/3rdparty/libwebp/src/dsp/enc_mips32.c4
-rw-r--r--src/3rdparty/libwebp/src/dsp/enc_mips_dsp_r2.c4
-rw-r--r--src/3rdparty/libwebp/src/dsp/enc_msa.c892
-rw-r--r--src/3rdparty/libwebp/src/dsp/enc_neon.c42
-rw-r--r--src/3rdparty/libwebp/src/dsp/enc_sse2.c182
-rw-r--r--src/3rdparty/libwebp/src/dsp/enc_sse41.c2
-rw-r--r--src/3rdparty/libwebp/src/dsp/filters.c12
-rw-r--r--src/3rdparty/libwebp/src/dsp/filters_msa.c202
-rw-r--r--src/3rdparty/libwebp/src/dsp/filters_neon.c327
-rw-r--r--src/3rdparty/libwebp/src/dsp/lossless.c186
-rw-r--r--src/3rdparty/libwebp/src/dsp/lossless.h225
-rw-r--r--src/3rdparty/libwebp/src/dsp/lossless_common.h210
-rw-r--r--src/3rdparty/libwebp/src/dsp/lossless_enc.c953
-rw-r--r--src/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c47
-rw-r--r--src/3rdparty/libwebp/src/dsp/lossless_enc_msa.c147
-rw-r--r--src/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c320
-rw-r--r--src/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c4
-rw-r--r--src/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c79
-rw-r--r--src/3rdparty/libwebp/src/dsp/lossless_msa.c355
-rw-r--r--src/3rdparty/libwebp/src/dsp/lossless_neon.c395
-rw-r--r--src/3rdparty/libwebp/src/dsp/lossless_sse2.c389
-rw-r--r--src/3rdparty/libwebp/src/dsp/msa_macro.h1097
-rw-r--r--src/3rdparty/libwebp/src/dsp/neon.h18
-rw-r--r--src/3rdparty/libwebp/src/dsp/rescaler.c12
-rw-r--r--src/3rdparty/libwebp/src/dsp/rescaler_mips32.c2
-rw-r--r--src/3rdparty/libwebp/src/dsp/rescaler_mips_dsp_r2.c2
-rw-r--r--src/3rdparty/libwebp/src/dsp/rescaler_msa.c444
-rw-r--r--src/3rdparty/libwebp/src/dsp/rescaler_neon.c2
-rw-r--r--src/3rdparty/libwebp/src/dsp/rescaler_sse2.c2
-rw-r--r--src/3rdparty/libwebp/src/dsp/upsampling.c6
-rw-r--r--src/3rdparty/libwebp/src/dsp/upsampling_msa.c678
-rw-r--r--src/3rdparty/libwebp/src/dsp/upsampling_neon.c77
-rw-r--r--src/3rdparty/libwebp/src/dsp/yuv.c57
-rw-r--r--src/3rdparty/libwebp/src/dsp/yuv.h2
-rw-r--r--src/3rdparty/libwebp/src/dsp/yuv_sse2.c304
52 files changed, 7729 insertions, 1603 deletions
diff --git a/src/3rdparty/libwebp/src/dsp/alpha_processing.c b/src/3rdparty/libwebp/src/dsp/alpha_processing.c
index 1716cac..4b60e09 100644
--- a/src/3rdparty/libwebp/src/dsp/alpha_processing.c
+++ b/src/3rdparty/libwebp/src/dsp/alpha_processing.c
@@ -284,9 +284,9 @@ static void ApplyAlphaMultiply_16b(uint8_t* rgba4444,
#endif
}
-static int DispatchAlpha(const uint8_t* alpha, int alpha_stride,
- int width, int height,
- uint8_t* dst, int dst_stride) {
+static int DispatchAlpha_C(const uint8_t* alpha, int alpha_stride,
+ int width, int height,
+ uint8_t* dst, int dst_stride) {
uint32_t alpha_mask = 0xff;
int i, j;
@@ -303,9 +303,9 @@ static int DispatchAlpha(const uint8_t* alpha, int alpha_stride,
return (alpha_mask != 0xff);
}
-static void DispatchAlphaToGreen(const uint8_t* alpha, int alpha_stride,
- int width, int height,
- uint32_t* dst, int dst_stride) {
+static void DispatchAlphaToGreen_C(const uint8_t* alpha, int alpha_stride,
+ int width, int height,
+ uint32_t* dst, int dst_stride) {
int i, j;
for (j = 0; j < height; ++j) {
for (i = 0; i < width; ++i) {
@@ -316,9 +316,9 @@ static void DispatchAlphaToGreen(const uint8_t* alpha, int alpha_stride,
}
}
-static int ExtractAlpha(const uint8_t* argb, int argb_stride,
- int width, int height,
- uint8_t* alpha, int alpha_stride) {
+static int ExtractAlpha_C(const uint8_t* argb, int argb_stride,
+ int width, int height,
+ uint8_t* alpha, int alpha_stride) {
uint8_t alpha_mask = 0xff;
int i, j;
@@ -334,11 +334,17 @@ static int ExtractAlpha(const uint8_t* argb, int argb_stride,
return (alpha_mask == 0xff);
}
+static void ExtractGreen_C(const uint32_t* argb, uint8_t* alpha, int size) {
+ int i;
+ for (i = 0; i < size; ++i) alpha[i] = argb[i] >> 8;
+}
+
void (*WebPApplyAlphaMultiply)(uint8_t*, int, int, int, int);
void (*WebPApplyAlphaMultiply4444)(uint8_t*, int, int, int);
int (*WebPDispatchAlpha)(const uint8_t*, int, int, int, uint8_t*, int);
void (*WebPDispatchAlphaToGreen)(const uint8_t*, int, int, int, uint32_t*, int);
int (*WebPExtractAlpha)(const uint8_t*, int, int, int, uint8_t*, int);
+void (*WebPExtractGreen)(const uint32_t* argb, uint8_t* alpha, int size);
//------------------------------------------------------------------------------
// Init function
@@ -346,6 +352,7 @@ int (*WebPExtractAlpha)(const uint8_t*, int, int, int, uint8_t*, int);
extern void WebPInitAlphaProcessingMIPSdspR2(void);
extern void WebPInitAlphaProcessingSSE2(void);
extern void WebPInitAlphaProcessingSSE41(void);
+extern void WebPInitAlphaProcessingNEON(void);
static volatile VP8CPUInfo alpha_processing_last_cpuinfo_used =
(VP8CPUInfo)&alpha_processing_last_cpuinfo_used;
@@ -357,9 +364,11 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitAlphaProcessing(void) {
WebPMultRow = WebPMultRowC;
WebPApplyAlphaMultiply = ApplyAlphaMultiply;
WebPApplyAlphaMultiply4444 = ApplyAlphaMultiply_16b;
- WebPDispatchAlpha = DispatchAlpha;
- WebPDispatchAlphaToGreen = DispatchAlphaToGreen;
- WebPExtractAlpha = ExtractAlpha;
+
+ WebPDispatchAlpha = DispatchAlpha_C;
+ WebPDispatchAlphaToGreen = DispatchAlphaToGreen_C;
+ WebPExtractAlpha = ExtractAlpha_C;
+ WebPExtractGreen = ExtractGreen_C;
// If defined, use CPUInfo() to overwrite some pointers with faster versions.
if (VP8GetCPUInfo != NULL) {
@@ -373,6 +382,11 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitAlphaProcessing(void) {
#endif
}
#endif
+#if defined(WEBP_USE_NEON)
+ if (VP8GetCPUInfo(kNEON)) {
+ WebPInitAlphaProcessingNEON();
+ }
+#endif
#if defined(WEBP_USE_MIPS_DSP_R2)
if (VP8GetCPUInfo(kMIPSdspR2)) {
WebPInitAlphaProcessingMIPSdspR2();
diff --git a/src/3rdparty/libwebp/src/dsp/alpha_processing_neon.c b/src/3rdparty/libwebp/src/dsp/alpha_processing_neon.c
new file mode 100644
index 0000000..606a401
--- /dev/null
+++ b/src/3rdparty/libwebp/src/dsp/alpha_processing_neon.c
@@ -0,0 +1,191 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the COPYING file in the root of the source
+// tree. An additional intellectual property rights grant can be found
+// in the file PATENTS. All contributing project authors may
+// be found in the AUTHORS file in the root of the source tree.
+// -----------------------------------------------------------------------------
+//
+// Utilities for processing transparent channel, NEON version.
+//
+// Author: Skal (pascal.massimino@gmail.com)
+
+#include "./dsp.h"
+
+#if defined(WEBP_USE_NEON)
+
+#include "./neon.h"
+
+//------------------------------------------------------------------------------
+
+#define MULTIPLIER(a) ((a) * 0x8081)
+#define PREMULTIPLY(x, m) (((x) * (m)) >> 23)
+
+#define MULTIPLY_BY_ALPHA(V, ALPHA, OTHER) do { \
+ const uint8x8_t alpha = (V).val[(ALPHA)]; \
+ const uint16x8_t r1 = vmull_u8((V).val[1], alpha); \
+ const uint16x8_t g1 = vmull_u8((V).val[2], alpha); \
+ const uint16x8_t b1 = vmull_u8((V).val[(OTHER)], alpha); \
+ /* we use: v / 255 = (v + 1 + (v >> 8)) >> 8 */ \
+ const uint16x8_t r2 = vsraq_n_u16(r1, r1, 8); \
+ const uint16x8_t g2 = vsraq_n_u16(g1, g1, 8); \
+ const uint16x8_t b2 = vsraq_n_u16(b1, b1, 8); \
+ const uint16x8_t r3 = vaddq_u16(r2, kOne); \
+ const uint16x8_t g3 = vaddq_u16(g2, kOne); \
+ const uint16x8_t b3 = vaddq_u16(b2, kOne); \
+ (V).val[1] = vshrn_n_u16(r3, 8); \
+ (V).val[2] = vshrn_n_u16(g3, 8); \
+ (V).val[(OTHER)] = vshrn_n_u16(b3, 8); \
+} while (0)
+
+static void ApplyAlphaMultiply_NEON(uint8_t* rgba, int alpha_first,
+ int w, int h, int stride) {
+ const uint16x8_t kOne = vdupq_n_u16(1u);
+ while (h-- > 0) {
+ uint32_t* const rgbx = (uint32_t*)rgba;
+ int i = 0;
+ if (alpha_first) {
+ for (; i + 8 <= w; i += 8) {
+ // load aaaa...|rrrr...|gggg...|bbbb...
+ uint8x8x4_t RGBX = vld4_u8((const uint8_t*)(rgbx + i));
+ MULTIPLY_BY_ALPHA(RGBX, 0, 3);
+ vst4_u8((uint8_t*)(rgbx + i), RGBX);
+ }
+ } else {
+ for (; i + 8 <= w; i += 8) {
+ uint8x8x4_t RGBX = vld4_u8((const uint8_t*)(rgbx + i));
+ MULTIPLY_BY_ALPHA(RGBX, 3, 0);
+ vst4_u8((uint8_t*)(rgbx + i), RGBX);
+ }
+ }
+ // Finish with left-overs.
+ for (; i < w; ++i) {
+ uint8_t* const rgb = rgba + (alpha_first ? 1 : 0);
+ const uint8_t* const alpha = rgba + (alpha_first ? 0 : 3);
+ const uint32_t a = alpha[4 * i];
+ if (a != 0xff) {
+ const uint32_t mult = MULTIPLIER(a);
+ rgb[4 * i + 0] = PREMULTIPLY(rgb[4 * i + 0], mult);
+ rgb[4 * i + 1] = PREMULTIPLY(rgb[4 * i + 1], mult);
+ rgb[4 * i + 2] = PREMULTIPLY(rgb[4 * i + 2], mult);
+ }
+ }
+ rgba += stride;
+ }
+}
+#undef MULTIPLY_BY_ALPHA
+#undef MULTIPLIER
+#undef PREMULTIPLY
+
+//------------------------------------------------------------------------------
+
+static int DispatchAlpha_NEON(const uint8_t* alpha, int alpha_stride,
+ int width, int height,
+ uint8_t* dst, int dst_stride) {
+ uint32_t alpha_mask = 0xffffffffu;
+ uint8x8_t mask8 = vdup_n_u8(0xff);
+ uint32_t tmp[2];
+ int i, j;
+ for (j = 0; j < height; ++j) {
+ // We don't know if alpha is first or last in dst[] (depending on rgbA/Argb
+ // mode). So we must be sure dst[4*i + 8 - 1] is writable for the store.
+ // Hence the test with 'width - 1' instead of just 'width'.
+ for (i = 0; i + 8 <= width - 1; i += 8) {
+ uint8x8x4_t rgbX = vld4_u8((const uint8_t*)(dst + 4 * i));
+ const uint8x8_t alphas = vld1_u8(alpha + i);
+ rgbX.val[0] = alphas;
+ vst4_u8((uint8_t*)(dst + 4 * i), rgbX);
+ mask8 = vand_u8(mask8, alphas);
+ }
+ for (; i < width; ++i) {
+ const uint32_t alpha_value = alpha[i];
+ dst[4 * i] = alpha_value;
+ alpha_mask &= alpha_value;
+ }
+ alpha += alpha_stride;
+ dst += dst_stride;
+ }
+ vst1_u8((uint8_t*)tmp, mask8);
+ alpha_mask &= tmp[0];
+ alpha_mask &= tmp[1];
+ return (alpha_mask != 0xffffffffu);
+}
+
+static void DispatchAlphaToGreen_NEON(const uint8_t* alpha, int alpha_stride,
+ int width, int height,
+ uint32_t* dst, int dst_stride) {
+ int i, j;
+ uint8x8x4_t greens; // leave A/R/B channels zero'd.
+ greens.val[0] = vdup_n_u8(0);
+ greens.val[2] = vdup_n_u8(0);
+ greens.val[3] = vdup_n_u8(0);
+ for (j = 0; j < height; ++j) {
+ for (i = 0; i + 8 <= width; i += 8) {
+ greens.val[1] = vld1_u8(alpha + i);
+ vst4_u8((uint8_t*)(dst + i), greens);
+ }
+ for (; i < width; ++i) dst[i] = alpha[i] << 8;
+ alpha += alpha_stride;
+ dst += dst_stride;
+ }
+}
+
+static int ExtractAlpha_NEON(const uint8_t* argb, int argb_stride,
+ int width, int height,
+ uint8_t* alpha, int alpha_stride) {
+ uint32_t alpha_mask = 0xffffffffu;
+ uint8x8_t mask8 = vdup_n_u8(0xff);
+ uint32_t tmp[2];
+ int i, j;
+ for (j = 0; j < height; ++j) {
+ // We don't know if alpha is first or last in dst[] (depending on rgbA/Argb
+ // mode). So we must be sure dst[4*i + 8 - 1] is writable for the store.
+ // Hence the test with 'width - 1' instead of just 'width'.
+ for (i = 0; i + 8 <= width - 1; i += 8) {
+ const uint8x8x4_t rgbX = vld4_u8((const uint8_t*)(argb + 4 * i));
+ const uint8x8_t alphas = rgbX.val[0];
+ vst1_u8((uint8_t*)(alpha + i), alphas);
+ mask8 = vand_u8(mask8, alphas);
+ }
+ for (; i < width; ++i) {
+ alpha[i] = argb[4 * i];
+ alpha_mask &= alpha[i];
+ }
+ argb += argb_stride;
+ alpha += alpha_stride;
+ }
+ vst1_u8((uint8_t*)tmp, mask8);
+ alpha_mask &= tmp[0];
+ alpha_mask &= tmp[1];
+ return (alpha_mask == 0xffffffffu);
+}
+
+static void ExtractGreen_NEON(const uint32_t* argb,
+ uint8_t* alpha, int size) {
+ int i;
+ for (i = 0; i + 16 <= size; i += 16) {
+ const uint8x16x4_t rgbX = vld4q_u8((const uint8_t*)(argb + i));
+ const uint8x16_t greens = rgbX.val[1];
+ vst1q_u8(alpha + i, greens);
+ }
+ for (; i < size; ++i) alpha[i] = (argb[i] >> 8) & 0xff;
+}
+
+//------------------------------------------------------------------------------
+
+extern void WebPInitAlphaProcessingNEON(void);
+
+WEBP_TSAN_IGNORE_FUNCTION void WebPInitAlphaProcessingNEON(void) {
+ WebPApplyAlphaMultiply = ApplyAlphaMultiply_NEON;
+ WebPDispatchAlpha = DispatchAlpha_NEON;
+ WebPDispatchAlphaToGreen = DispatchAlphaToGreen_NEON;
+ WebPExtractAlpha = ExtractAlpha_NEON;
+ WebPExtractGreen = ExtractGreen_NEON;
+}
+
+#else // !WEBP_USE_NEON
+
+WEBP_DSP_INIT_STUB(WebPInitAlphaProcessingNEON)
+
+#endif // WEBP_USE_NEON
diff --git a/src/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c b/src/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c
index 5acb481..83dc559 100644
--- a/src/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c
+++ b/src/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c
@@ -150,46 +150,46 @@ static int ExtractAlpha(const uint8_t* argb, int argb_stride,
#define PREMULTIPLY(x, m) (((x) * (m)) >> 23)
// We can't use a 'const int' for the SHUFFLE value, because it has to be an
-// immediate in the _mm_shufflexx_epi16() instruction. We really a macro here.
-#define APPLY_ALPHA(RGBX, SHUFFLE, MASK, MULT) do { \
- const __m128i argb0 = _mm_loadl_epi64((__m128i*)&(RGBX)); \
- const __m128i argb1 = _mm_unpacklo_epi8(argb0, zero); \
- const __m128i alpha0 = _mm_and_si128(argb1, MASK); \
- const __m128i alpha1 = _mm_shufflelo_epi16(alpha0, SHUFFLE); \
- const __m128i alpha2 = _mm_shufflehi_epi16(alpha1, SHUFFLE); \
- /* alpha2 = [0 a0 a0 a0][0 a1 a1 a1] */ \
- const __m128i scale0 = _mm_mullo_epi16(alpha2, MULT); \
- const __m128i scale1 = _mm_mulhi_epu16(alpha2, MULT); \
- const __m128i argb2 = _mm_mulhi_epu16(argb1, scale0); \
- const __m128i argb3 = _mm_mullo_epi16(argb1, scale1); \
- const __m128i argb4 = _mm_adds_epu16(argb2, argb3); \
- const __m128i argb5 = _mm_srli_epi16(argb4, 7); \
- const __m128i argb6 = _mm_or_si128(argb5, alpha0); \
- const __m128i argb7 = _mm_packus_epi16(argb6, zero); \
- _mm_storel_epi64((__m128i*)&(RGBX), argb7); \
+// immediate in the _mm_shufflexx_epi16() instruction. We really need a macro.
+// We use: v / 255 = (v * 0x8081) >> 23, where v = alpha * {r,g,b} is a 16bit
+// value.
+#define APPLY_ALPHA(RGBX, SHUFFLE) do { \
+ const __m128i argb0 = _mm_loadu_si128((const __m128i*)&(RGBX)); \
+ const __m128i argb1_lo = _mm_unpacklo_epi8(argb0, zero); \
+ const __m128i argb1_hi = _mm_unpackhi_epi8(argb0, zero); \
+ const __m128i alpha0_lo = _mm_or_si128(argb1_lo, kMask); \
+ const __m128i alpha0_hi = _mm_or_si128(argb1_hi, kMask); \
+ const __m128i alpha1_lo = _mm_shufflelo_epi16(alpha0_lo, SHUFFLE); \
+ const __m128i alpha1_hi = _mm_shufflelo_epi16(alpha0_hi, SHUFFLE); \
+ const __m128i alpha2_lo = _mm_shufflehi_epi16(alpha1_lo, SHUFFLE); \
+ const __m128i alpha2_hi = _mm_shufflehi_epi16(alpha1_hi, SHUFFLE); \
+ /* alpha2 = [ff a0 a0 a0][ff a1 a1 a1] */ \
+ const __m128i A0_lo = _mm_mullo_epi16(alpha2_lo, argb1_lo); \
+ const __m128i A0_hi = _mm_mullo_epi16(alpha2_hi, argb1_hi); \
+ const __m128i A1_lo = _mm_mulhi_epu16(A0_lo, kMult); \
+ const __m128i A1_hi = _mm_mulhi_epu16(A0_hi, kMult); \
+ const __m128i A2_lo = _mm_srli_epi16(A1_lo, 7); \
+ const __m128i A2_hi = _mm_srli_epi16(A1_hi, 7); \
+ const __m128i A3 = _mm_packus_epi16(A2_lo, A2_hi); \
+ _mm_storeu_si128((__m128i*)&(RGBX), A3); \
} while (0)
-static void ApplyAlphaMultiply(uint8_t* rgba, int alpha_first,
- int w, int h, int stride) {
+static void ApplyAlphaMultiply_SSE2(uint8_t* rgba, int alpha_first,
+ int w, int h, int stride) {
const __m128i zero = _mm_setzero_si128();
- const int kSpan = 2;
- const int w2 = w & ~(kSpan - 1);
+ const __m128i kMult = _mm_set1_epi16(0x8081u);
+ const __m128i kMask = _mm_set_epi16(0, 0xff, 0xff, 0, 0, 0xff, 0xff, 0);
+ const int kSpan = 4;
while (h-- > 0) {
uint32_t* const rgbx = (uint32_t*)rgba;
int i;
if (!alpha_first) {
- const __m128i kMask = _mm_set_epi16(0xff, 0, 0, 0, 0xff, 0, 0, 0);
- const __m128i kMult =
- _mm_set_epi16(0, 0x8081, 0x8081, 0x8081, 0, 0x8081, 0x8081, 0x8081);
- for (i = 0; i < w2; i += kSpan) {
- APPLY_ALPHA(rgbx[i], _MM_SHUFFLE(0, 3, 3, 3), kMask, kMult);
+ for (i = 0; i + kSpan <= w; i += kSpan) {
+ APPLY_ALPHA(rgbx[i], _MM_SHUFFLE(2, 3, 3, 3));
}
} else {
- const __m128i kMask = _mm_set_epi16(0, 0, 0, 0xff, 0, 0, 0, 0xff);
- const __m128i kMult =
- _mm_set_epi16(0x8081, 0x8081, 0x8081, 0, 0x8081, 0x8081, 0x8081, 0);
- for (i = 0; i < w2; i += kSpan) {
- APPLY_ALPHA(rgbx[i], _MM_SHUFFLE(0, 0, 0, 3), kMask, kMult);
+ for (i = 0; i + kSpan <= w; i += kSpan) {
+ APPLY_ALPHA(rgbx[i], _MM_SHUFFLE(0, 0, 0, 1));
}
}
// Finish with left-overs.
@@ -213,64 +213,51 @@ static void ApplyAlphaMultiply(uint8_t* rgba, int alpha_first,
// -----------------------------------------------------------------------------
// Apply alpha value to rows
-// We use: kINV255 = (1 << 24) / 255 = 0x010101
-// So: a * kINV255 = (a << 16) | [(a << 8) | a]
-// -> _mm_mulhi_epu16() takes care of the (a<<16) part,
-// and _mm_mullo_epu16(a * 0x0101,...) takes care of the "(a << 8) | a" one.
-
-static void MultARGBRow(uint32_t* const ptr, int width, int inverse) {
+static void MultARGBRow_SSE2(uint32_t* const ptr, int width, int inverse) {
int x = 0;
if (!inverse) {
const int kSpan = 2;
const __m128i zero = _mm_setzero_si128();
- const __m128i kRound =
- _mm_set_epi16(0, 1 << 7, 1 << 7, 1 << 7, 0, 1 << 7, 1 << 7, 1 << 7);
- const __m128i kMult =
- _mm_set_epi16(0, 0x0101, 0x0101, 0x0101, 0, 0x0101, 0x0101, 0x0101);
- const __m128i kOne64 = _mm_set_epi16(1u << 8, 0, 0, 0, 1u << 8, 0, 0, 0);
- const int w2 = width & ~(kSpan - 1);
- for (x = 0; x < w2; x += kSpan) {
- const __m128i argb0 = _mm_loadl_epi64((__m128i*)&ptr[x]);
- const __m128i argb1 = _mm_unpacklo_epi8(argb0, zero);
- const __m128i tmp0 = _mm_shufflelo_epi16(argb1, _MM_SHUFFLE(3, 3, 3, 3));
- const __m128i tmp1 = _mm_shufflehi_epi16(tmp0, _MM_SHUFFLE(3, 3, 3, 3));
- const __m128i tmp2 = _mm_srli_epi64(tmp1, 16);
- const __m128i scale0 = _mm_mullo_epi16(tmp1, kMult);
- const __m128i scale1 = _mm_or_si128(tmp2, kOne64);
- const __m128i argb2 = _mm_mulhi_epu16(argb1, scale0);
- const __m128i argb3 = _mm_mullo_epi16(argb1, scale1);
- const __m128i argb4 = _mm_adds_epu16(argb2, argb3);
- const __m128i argb5 = _mm_adds_epu16(argb4, kRound);
- const __m128i argb6 = _mm_srli_epi16(argb5, 8);
- const __m128i argb7 = _mm_packus_epi16(argb6, zero);
- _mm_storel_epi64((__m128i*)&ptr[x], argb7);
+ const __m128i k128 = _mm_set1_epi16(128);
+ const __m128i kMult = _mm_set1_epi16(0x0101);
+ const __m128i kMask = _mm_set_epi16(0, 0xff, 0, 0, 0, 0xff, 0, 0);
+ for (x = 0; x + kSpan <= width; x += kSpan) {
+ // To compute 'result = (int)(a * x / 255. + .5)', we use:
+ // tmp = a * v + 128, result = (tmp * 0x0101u) >> 16
+ const __m128i A0 = _mm_loadl_epi64((const __m128i*)&ptr[x]);
+ const __m128i A1 = _mm_unpacklo_epi8(A0, zero);
+ const __m128i A2 = _mm_or_si128(A1, kMask);
+ const __m128i A3 = _mm_shufflelo_epi16(A2, _MM_SHUFFLE(2, 3, 3, 3));
+ const __m128i A4 = _mm_shufflehi_epi16(A3, _MM_SHUFFLE(2, 3, 3, 3));
+ // here, A4 = [ff a0 a0 a0][ff a1 a1 a1]
+ const __m128i A5 = _mm_mullo_epi16(A4, A1);
+ const __m128i A6 = _mm_add_epi16(A5, k128);
+ const __m128i A7 = _mm_mulhi_epu16(A6, kMult);
+ const __m128i A10 = _mm_packus_epi16(A7, zero);
+ _mm_storel_epi64((__m128i*)&ptr[x], A10);
}
}
width -= x;
if (width > 0) WebPMultARGBRowC(ptr + x, width, inverse);
}
-static void MultRow(uint8_t* const ptr, const uint8_t* const alpha,
- int width, int inverse) {
+static void MultRow_SSE2(uint8_t* const ptr, const uint8_t* const alpha,
+ int width, int inverse) {
int x = 0;
if (!inverse) {
- const int kSpan = 8;
const __m128i zero = _mm_setzero_si128();
- const __m128i kRound = _mm_set1_epi16(1 << 7);
- const int w2 = width & ~(kSpan - 1);
- for (x = 0; x < w2; x += kSpan) {
+ const __m128i k128 = _mm_set1_epi16(128);
+ const __m128i kMult = _mm_set1_epi16(0x0101);
+ for (x = 0; x + 8 <= width; x += 8) {
const __m128i v0 = _mm_loadl_epi64((__m128i*)&ptr[x]);
+ const __m128i a0 = _mm_loadl_epi64((const __m128i*)&alpha[x]);
const __m128i v1 = _mm_unpacklo_epi8(v0, zero);
- const __m128i alpha0 = _mm_loadl_epi64((const __m128i*)&alpha[x]);
- const __m128i alpha1 = _mm_unpacklo_epi8(alpha0, zero);
- const __m128i alpha2 = _mm_unpacklo_epi8(alpha0, alpha0);
- const __m128i v2 = _mm_mulhi_epu16(v1, alpha2);
- const __m128i v3 = _mm_mullo_epi16(v1, alpha1);
- const __m128i v4 = _mm_adds_epu16(v2, v3);
- const __m128i v5 = _mm_adds_epu16(v4, kRound);
- const __m128i v6 = _mm_srli_epi16(v5, 8);
- const __m128i v7 = _mm_packus_epi16(v6, zero);
- _mm_storel_epi64((__m128i*)&ptr[x], v7);
+ const __m128i a1 = _mm_unpacklo_epi8(a0, zero);
+ const __m128i v2 = _mm_mullo_epi16(v1, a1);
+ const __m128i v3 = _mm_add_epi16(v2, k128);
+ const __m128i v4 = _mm_mulhi_epu16(v3, kMult);
+ const __m128i v5 = _mm_packus_epi16(v4, zero);
+ _mm_storel_epi64((__m128i*)&ptr[x], v5);
}
}
width -= x;
@@ -283,9 +270,9 @@ static void MultRow(uint8_t* const ptr, const uint8_t* const alpha,
extern void WebPInitAlphaProcessingSSE2(void);
WEBP_TSAN_IGNORE_FUNCTION void WebPInitAlphaProcessingSSE2(void) {
- WebPMultARGBRow = MultARGBRow;
- WebPMultRow = MultRow;
- WebPApplyAlphaMultiply = ApplyAlphaMultiply;
+ WebPMultARGBRow = MultARGBRow_SSE2;
+ WebPMultRow = MultRow_SSE2;
+ WebPApplyAlphaMultiply = ApplyAlphaMultiply_SSE2;
WebPDispatchAlpha = DispatchAlpha;
WebPDispatchAlphaToGreen = DispatchAlphaToGreen;
WebPExtractAlpha = ExtractAlpha;
diff --git a/src/3rdparty/libwebp/src/dsp/common_sse2.h b/src/3rdparty/libwebp/src/dsp/common_sse2.h
index 7cea13f..995d7cf 100644
--- a/src/3rdparty/libwebp/src/dsp/common_sse2.h
+++ b/src/3rdparty/libwebp/src/dsp/common_sse2.h
@@ -100,6 +100,91 @@ static WEBP_INLINE void VP8Transpose_2_4x4_16b(
// a03 a13 a23 a33 b03 b13 b23 b33
}
+//------------------------------------------------------------------------------
+// Channel mixing.
+
+// Function used several times in VP8PlanarTo24b.
+// It samples the in buffer as follows: one every two unsigned char is stored
+// at the beginning of the buffer, while the other half is stored at the end.
+#define VP8PlanarTo24bHelper(IN, OUT) \
+ do { \
+ const __m128i v_mask = _mm_set1_epi16(0x00ff); \
+ /* Take one every two upper 8b values.*/ \
+ (OUT##0) = _mm_packus_epi16(_mm_and_si128((IN##0), v_mask), \
+ _mm_and_si128((IN##1), v_mask)); \
+ (OUT##1) = _mm_packus_epi16(_mm_and_si128((IN##2), v_mask), \
+ _mm_and_si128((IN##3), v_mask)); \
+ (OUT##2) = _mm_packus_epi16(_mm_and_si128((IN##4), v_mask), \
+ _mm_and_si128((IN##5), v_mask)); \
+ /* Take one every two lower 8b values.*/ \
+ (OUT##3) = _mm_packus_epi16(_mm_srli_epi16((IN##0), 8), \
+ _mm_srli_epi16((IN##1), 8)); \
+ (OUT##4) = _mm_packus_epi16(_mm_srli_epi16((IN##2), 8), \
+ _mm_srli_epi16((IN##3), 8)); \
+ (OUT##5) = _mm_packus_epi16(_mm_srli_epi16((IN##4), 8), \
+ _mm_srli_epi16((IN##5), 8)); \
+ } while (0)
+
+// Pack the planar buffers
+// rrrr... rrrr... gggg... gggg... bbbb... bbbb....
+// triplet by triplet in the output buffer rgb as rgbrgbrgbrgb ...
+static WEBP_INLINE void VP8PlanarTo24b(__m128i* const in0, __m128i* const in1,
+ __m128i* const in2, __m128i* const in3,
+ __m128i* const in4, __m128i* const in5) {
+ // The input is 6 registers of sixteen 8b but for the sake of explanation,
+ // let's take 6 registers of four 8b values.
+ // To pack, we will keep taking one every two 8b integer and move it
+ // around as follows:
+ // Input:
+ // r0r1r2r3 | r4r5r6r7 | g0g1g2g3 | g4g5g6g7 | b0b1b2b3 | b4b5b6b7
+ // Split the 6 registers in two sets of 3 registers: the first set as the even
+ // 8b bytes, the second the odd ones:
+ // r0r2r4r6 | g0g2g4g6 | b0b2b4b6 | r1r3r5r7 | g1g3g5g7 | b1b3b5b7
+ // Repeat the same permutations twice more:
+ // r0r4g0g4 | b0b4r1r5 | g1g5b1b5 | r2r6g2g6 | b2b6r3r7 | g3g7b3b7
+ // r0g0b0r1 | g1b1r2g2 | b2r3g3b3 | r4g4b4r5 | g5b5r6g6 | b6r7g7b7
+ __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
+ VP8PlanarTo24bHelper(*in, tmp);
+ VP8PlanarTo24bHelper(tmp, *in);
+ VP8PlanarTo24bHelper(*in, tmp);
+ // We need to do it two more times than the example as we have sixteen bytes.
+ {
+ __m128i out0, out1, out2, out3, out4, out5;
+ VP8PlanarTo24bHelper(tmp, out);
+ VP8PlanarTo24bHelper(out, *in);
+ }
+}
+
+#undef VP8PlanarTo24bHelper
+
+// Convert four packed four-channel buffers like argbargbargbargb... into the
+// split channels aaaaa ... rrrr ... gggg .... bbbbb ......
+static WEBP_INLINE void VP8L32bToPlanar(__m128i* const in0,
+ __m128i* const in1,
+ __m128i* const in2,
+ __m128i* const in3) {
+ // Column-wise transpose.
+ const __m128i A0 = _mm_unpacklo_epi8(*in0, *in1);
+ const __m128i A1 = _mm_unpackhi_epi8(*in0, *in1);
+ const __m128i A2 = _mm_unpacklo_epi8(*in2, *in3);
+ const __m128i A3 = _mm_unpackhi_epi8(*in2, *in3);
+ const __m128i B0 = _mm_unpacklo_epi8(A0, A1);
+ const __m128i B1 = _mm_unpackhi_epi8(A0, A1);
+ const __m128i B2 = _mm_unpacklo_epi8(A2, A3);
+ const __m128i B3 = _mm_unpackhi_epi8(A2, A3);
+ // C0 = g7 g6 ... g1 g0 | b7 b6 ... b1 b0
+ // C1 = a7 a6 ... a1 a0 | r7 r6 ... r1 r0
+ const __m128i C0 = _mm_unpacklo_epi8(B0, B1);
+ const __m128i C1 = _mm_unpackhi_epi8(B0, B1);
+ const __m128i C2 = _mm_unpacklo_epi8(B2, B3);
+ const __m128i C3 = _mm_unpackhi_epi8(B2, B3);
+ // Gather the channels.
+ *in0 = _mm_unpackhi_epi64(C1, C3);
+ *in1 = _mm_unpacklo_epi64(C1, C3);
+ *in2 = _mm_unpackhi_epi64(C0, C2);
+ *in3 = _mm_unpacklo_epi64(C0, C2);
+}
+
#endif // WEBP_USE_SSE2
#ifdef __cplusplus
diff --git a/src/3rdparty/libwebp/src/dsp/cost.c b/src/3rdparty/libwebp/src/dsp/cost.c
index fe72d26..58ddea7 100644
--- a/src/3rdparty/libwebp/src/dsp/cost.c
+++ b/src/3rdparty/libwebp/src/dsp/cost.c
@@ -10,7 +10,7 @@
// Author: Skal (pascal.massimino@gmail.com)
#include "./dsp.h"
-#include "../enc/cost.h"
+#include "../enc/cost_enc.h"
//------------------------------------------------------------------------------
// Boolean-cost cost table
diff --git a/src/3rdparty/libwebp/src/dsp/cost_mips32.c b/src/3rdparty/libwebp/src/dsp/cost_mips32.c
index d1e240e..3102da8 100644
--- a/src/3rdparty/libwebp/src/dsp/cost_mips32.c
+++ b/src/3rdparty/libwebp/src/dsp/cost_mips32.c
@@ -13,7 +13,7 @@
#if defined(WEBP_USE_MIPS32)
-#include "../enc/cost.h"
+#include "../enc/cost_enc.h"
static int GetResidualCost(int ctx0, const VP8Residual* const res) {
int temp0, temp1;
diff --git a/src/3rdparty/libwebp/src/dsp/cost_mips_dsp_r2.c b/src/3rdparty/libwebp/src/dsp/cost_mips_dsp_r2.c
index ce64067..6ec8aeb 100644
--- a/src/3rdparty/libwebp/src/dsp/cost_mips_dsp_r2.c
+++ b/src/3rdparty/libwebp/src/dsp/cost_mips_dsp_r2.c
@@ -13,7 +13,7 @@
#if defined(WEBP_USE_MIPS_DSP_R2)
-#include "../enc/cost.h"
+#include "../enc/cost_enc.h"
static int GetResidualCost(int ctx0, const VP8Residual* const res) {
int temp0, temp1;
diff --git a/src/3rdparty/libwebp/src/dsp/cost_sse2.c b/src/3rdparty/libwebp/src/dsp/cost_sse2.c
index 0cb1c1f..421d51f 100644
--- a/src/3rdparty/libwebp/src/dsp/cost_sse2.c
+++ b/src/3rdparty/libwebp/src/dsp/cost_sse2.c
@@ -16,8 +16,8 @@
#if defined(WEBP_USE_SSE2)
#include <emmintrin.h>
-#include "../enc/cost.h"
-#include "../enc/vp8enci.h"
+#include "../enc/cost_enc.h"
+#include "../enc/vp8i_enc.h"
#include "../utils/utils.h"
//------------------------------------------------------------------------------
diff --git a/src/3rdparty/libwebp/src/dsp/cpu.c b/src/3rdparty/libwebp/src/dsp/cpu.c
index cbb08db..b5583b6 100644
--- a/src/3rdparty/libwebp/src/dsp/cpu.c
+++ b/src/3rdparty/libwebp/src/dsp/cpu.c
@@ -95,26 +95,62 @@ static WEBP_INLINE uint64_t xgetbv(void) {
#endif
#if defined(__i386__) || defined(__x86_64__) || defined(WEBP_MSC_SSE2)
+
+// helper function for run-time detection of slow SSSE3 platforms
+static int CheckSlowModel(int info) {
+ // Table listing display models with longer latencies for the bsr instruction
+ // (ie 2 cycles vs 10/16 cycles) and some SSSE3 instructions like pshufb.
+ // Refer to Intel 64 and IA-32 Architectures Optimization Reference Manual.
+ static const uint8_t kSlowModels[] = {
+ 0x37, 0x4a, 0x4d, // Silvermont Microarchitecture
+ 0x1c, 0x26, 0x27 // Atom Microarchitecture
+ };
+ const uint32_t model = ((info & 0xf0000) >> 12) | ((info >> 4) & 0xf);
+ const uint32_t family = (info >> 8) & 0xf;
+ if (family == 0x06) {
+ size_t i;
+ for (i = 0; i < sizeof(kSlowModels) / sizeof(kSlowModels[0]); ++i) {
+ if (model == kSlowModels[i]) return 1;
+ }
+ }
+ return 0;
+}
+
static int x86CPUInfo(CPUFeature feature) {
int max_cpuid_value;
int cpu_info[4];
+ int is_intel = 0;
// get the highest feature value cpuid supports
GetCPUInfo(cpu_info, 0);
max_cpuid_value = cpu_info[0];
if (max_cpuid_value < 1) {
return 0;
+ } else {
+ const int VENDOR_ID_INTEL_EBX = 0x756e6547; // uneG
+ const int VENDOR_ID_INTEL_EDX = 0x49656e69; // Ieni
+ const int VENDOR_ID_INTEL_ECX = 0x6c65746e; // letn
+ is_intel = (cpu_info[1] == VENDOR_ID_INTEL_EBX &&
+ cpu_info[2] == VENDOR_ID_INTEL_ECX &&
+ cpu_info[3] == VENDOR_ID_INTEL_EDX); // genuine Intel?
}
GetCPUInfo(cpu_info, 1);
if (feature == kSSE2) {
- return 0 != (cpu_info[3] & 0x04000000);
+ return !!(cpu_info[3] & (1 << 26));
}
if (feature == kSSE3) {
- return 0 != (cpu_info[2] & 0x00000001);
+ return !!(cpu_info[2] & (1 << 0));
+ }
+ if (feature == kSlowSSSE3) {
+ if (is_intel && (cpu_info[2] & (1 << 0))) { // SSSE3?
+ return CheckSlowModel(cpu_info[0]);
+ }
+ return 0;
}
+
if (feature == kSSE4_1) {
- return 0 != (cpu_info[2] & 0x00080000);
+ return !!(cpu_info[2] & (1 << 19));
}
if (feature == kAVX) {
// bits 27 (OSXSAVE) & 28 (256-bit AVX)
@@ -126,7 +162,7 @@ static int x86CPUInfo(CPUFeature feature) {
if (feature == kAVX2) {
if (x86CPUInfo(kAVX) && max_cpuid_value >= 7) {
GetCPUInfo(cpu_info, 7);
- return ((cpu_info[1] & 0x00000020) == 0x00000020);
+ return !!(cpu_info[1] & (1 << 5));
}
}
return 0;
@@ -184,4 +220,3 @@ VP8CPUInfo VP8GetCPUInfo = mipsCPUInfo;
#else
VP8CPUInfo VP8GetCPUInfo = NULL;
#endif
-
diff --git a/src/3rdparty/libwebp/src/dsp/dec.c b/src/3rdparty/libwebp/src/dsp/dec.c
index e92d693..007e985 100644
--- a/src/3rdparty/libwebp/src/dsp/dec.c
+++ b/src/3rdparty/libwebp/src/dsp/dec.c
@@ -12,7 +12,7 @@
// Author: Skal (pascal.massimino@gmail.com)
#include "./dsp.h"
-#include "../dec/vp8i.h"
+#include "../dec/vp8i_dec.h"
#include "../utils/utils.h"
//------------------------------------------------------------------------------
@@ -239,7 +239,7 @@ VP8PredFunc VP8PredLuma16[NUM_B_DC_MODES];
//------------------------------------------------------------------------------
// 4x4
-#define AVG3(a, b, c) (((a) + 2 * (b) + (c) + 2) >> 2)
+#define AVG3(a, b, c) ((uint8_t)(((a) + 2 * (b) + (c) + 2) >> 2))
#define AVG2(a, b) (((a) + (b) + 1) >> 1)
static void VE4(uint8_t* dst) { // vertical
diff --git a/src/3rdparty/libwebp/src/dsp/dec_clip_tables.c b/src/3rdparty/libwebp/src/dsp/dec_clip_tables.c
index 3b6dde8..74ba34c 100644
--- a/src/3rdparty/libwebp/src/dsp/dec_clip_tables.c
+++ b/src/3rdparty/libwebp/src/dsp/dec_clip_tables.c
@@ -63,7 +63,7 @@ static const uint8_t abs0[255 + 255 + 1] = {
0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff
};
-static const int8_t sclip1[1020 + 1020 + 1] = {
+static const uint8_t sclip1[1020 + 1020 + 1] = {
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
@@ -236,7 +236,7 @@ static const int8_t sclip1[1020 + 1020 + 1] = {
0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f
};
-static const int8_t sclip2[112 + 112 + 1] = {
+static const uint8_t sclip2[112 + 112 + 1] = {
0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0,
0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0,
0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0,
@@ -339,8 +339,8 @@ static volatile int tables_ok = 0;
#endif
-const int8_t* const VP8ksclip1 = &sclip1[1020];
-const int8_t* const VP8ksclip2 = &sclip2[112];
+const int8_t* const VP8ksclip1 = (const int8_t*)&sclip1[1020];
+const int8_t* const VP8ksclip2 = (const int8_t*)&sclip2[112];
const uint8_t* const VP8kclip1 = &clip1[255];
const uint8_t* const VP8kabs0 = &abs0[255];
diff --git a/src/3rdparty/libwebp/src/dsp/dec_msa.c b/src/3rdparty/libwebp/src/dsp/dec_msa.c
index f76055c..8d9c98c 100644
--- a/src/3rdparty/libwebp/src/dsp/dec_msa.c
+++ b/src/3rdparty/libwebp/src/dsp/dec_msa.c
@@ -154,6 +154,820 @@ static void TransformAC3(const int16_t* in, uint8_t* dst) {
}
//------------------------------------------------------------------------------
+// Edge filtering functions
+
+#define FLIP_SIGN2(in0, in1, out0, out1) { \
+ out0 = (v16i8)__msa_xori_b(in0, 0x80); \
+ out1 = (v16i8)__msa_xori_b(in1, 0x80); \
+}
+
+#define FLIP_SIGN4(in0, in1, in2, in3, out0, out1, out2, out3) { \
+ FLIP_SIGN2(in0, in1, out0, out1); \
+ FLIP_SIGN2(in2, in3, out2, out3); \
+}
+
+#define FILT_VAL(q0_m, p0_m, mask, filt) do { \
+ v16i8 q0_sub_p0; \
+ q0_sub_p0 = __msa_subs_s_b(q0_m, p0_m); \
+ filt = __msa_adds_s_b(filt, q0_sub_p0); \
+ filt = __msa_adds_s_b(filt, q0_sub_p0); \
+ filt = __msa_adds_s_b(filt, q0_sub_p0); \
+ filt = filt & mask; \
+} while (0)
+
+#define FILT2(q_m, p_m, q, p) do { \
+ u_r = SRAI_H(temp1, 7); \
+ u_r = __msa_sat_s_h(u_r, 7); \
+ u_l = SRAI_H(temp3, 7); \
+ u_l = __msa_sat_s_h(u_l, 7); \
+ u = __msa_pckev_b((v16i8)u_l, (v16i8)u_r); \
+ q_m = __msa_subs_s_b(q_m, u); \
+ p_m = __msa_adds_s_b(p_m, u); \
+ q = __msa_xori_b((v16u8)q_m, 0x80); \
+ p = __msa_xori_b((v16u8)p_m, 0x80); \
+} while (0)
+
+#define LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev) do { \
+ v16i8 p1_m, p0_m, q0_m, q1_m; \
+ v16i8 filt, t1, t2; \
+ const v16i8 cnst4b = __msa_ldi_b(4); \
+ const v16i8 cnst3b = __msa_ldi_b(3); \
+ \
+ FLIP_SIGN4(p1, p0, q0, q1, p1_m, p0_m, q0_m, q1_m); \
+ filt = __msa_subs_s_b(p1_m, q1_m); \
+ filt = filt & hev; \
+ FILT_VAL(q0_m, p0_m, mask, filt); \
+ t1 = __msa_adds_s_b(filt, cnst4b); \
+ t1 = SRAI_B(t1, 3); \
+ t2 = __msa_adds_s_b(filt, cnst3b); \
+ t2 = SRAI_B(t2, 3); \
+ q0_m = __msa_subs_s_b(q0_m, t1); \
+ q0 = __msa_xori_b((v16u8)q0_m, 0x80); \
+ p0_m = __msa_adds_s_b(p0_m, t2); \
+ p0 = __msa_xori_b((v16u8)p0_m, 0x80); \
+ filt = __msa_srari_b(t1, 1); \
+ hev = __msa_xori_b(hev, 0xff); \
+ filt = filt & hev; \
+ q1_m = __msa_subs_s_b(q1_m, filt); \
+ q1 = __msa_xori_b((v16u8)q1_m, 0x80); \
+ p1_m = __msa_adds_s_b(p1_m, filt); \
+ p1 = __msa_xori_b((v16u8)p1_m, 0x80); \
+} while (0)
+
+#define LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev) do { \
+ v16i8 p2_m, p1_m, p0_m, q2_m, q1_m, q0_m; \
+ v16i8 u, filt, t1, t2, filt_sign; \
+ v8i16 filt_r, filt_l, u_r, u_l; \
+ v8i16 temp0, temp1, temp2, temp3; \
+ const v16i8 cnst4b = __msa_ldi_b(4); \
+ const v16i8 cnst3b = __msa_ldi_b(3); \
+ const v8i16 cnst9h = __msa_ldi_h(9); \
+ \
+ FLIP_SIGN4(p1, p0, q0, q1, p1_m, p0_m, q0_m, q1_m); \
+ filt = __msa_subs_s_b(p1_m, q1_m); \
+ FILT_VAL(q0_m, p0_m, mask, filt); \
+ FLIP_SIGN2(p2, q2, p2_m, q2_m); \
+ t2 = filt & hev; \
+ /* filt_val &= ~hev */ \
+ hev = __msa_xori_b(hev, 0xff); \
+ filt = filt & hev; \
+ t1 = __msa_adds_s_b(t2, cnst4b); \
+ t1 = SRAI_B(t1, 3); \
+ t2 = __msa_adds_s_b(t2, cnst3b); \
+ t2 = SRAI_B(t2, 3); \
+ q0_m = __msa_subs_s_b(q0_m, t1); \
+ p0_m = __msa_adds_s_b(p0_m, t2); \
+ filt_sign = __msa_clti_s_b(filt, 0); \
+ ILVRL_B2_SH(filt_sign, filt, filt_r, filt_l); \
+ /* update q2/p2 */ \
+ temp0 = filt_r * cnst9h; \
+ temp1 = ADDVI_H(temp0, 63); \
+ temp2 = filt_l * cnst9h; \
+ temp3 = ADDVI_H(temp2, 63); \
+ FILT2(q2_m, p2_m, q2, p2); \
+ /* update q1/p1 */ \
+ temp1 = temp1 + temp0; \
+ temp3 = temp3 + temp2; \
+ FILT2(q1_m, p1_m, q1, p1); \
+ /* update q0/p0 */ \
+ temp1 = temp1 + temp0; \
+ temp3 = temp3 + temp2; \
+ FILT2(q0_m, p0_m, q0, p0); \
+} while (0)
+
+#define LPF_MASK_HEV(p3_in, p2_in, p1_in, p0_in, \
+ q0_in, q1_in, q2_in, q3_in, \
+ limit_in, b_limit_in, thresh_in, \
+ hev_out, mask_out) do { \
+ v16u8 p3_asub_p2_m, p2_asub_p1_m, p1_asub_p0_m, q1_asub_q0_m; \
+ v16u8 p1_asub_q1_m, p0_asub_q0_m, q3_asub_q2_m, q2_asub_q1_m; \
+ v16u8 flat_out; \
+ \
+ /* absolute subtraction of pixel values */ \
+ p3_asub_p2_m = __msa_asub_u_b(p3_in, p2_in); \
+ p2_asub_p1_m = __msa_asub_u_b(p2_in, p1_in); \
+ p1_asub_p0_m = __msa_asub_u_b(p1_in, p0_in); \
+ q1_asub_q0_m = __msa_asub_u_b(q1_in, q0_in); \
+ q2_asub_q1_m = __msa_asub_u_b(q2_in, q1_in); \
+ q3_asub_q2_m = __msa_asub_u_b(q3_in, q2_in); \
+ p0_asub_q0_m = __msa_asub_u_b(p0_in, q0_in); \
+ p1_asub_q1_m = __msa_asub_u_b(p1_in, q1_in); \
+ /* calculation of hev */ \
+ flat_out = __msa_max_u_b(p1_asub_p0_m, q1_asub_q0_m); \
+ hev_out = (thresh_in < flat_out); \
+ /* calculation of mask */ \
+ p0_asub_q0_m = __msa_adds_u_b(p0_asub_q0_m, p0_asub_q0_m); \
+ p1_asub_q1_m = SRAI_B(p1_asub_q1_m, 1); \
+ p0_asub_q0_m = __msa_adds_u_b(p0_asub_q0_m, p1_asub_q1_m); \
+ mask_out = (b_limit_in < p0_asub_q0_m); \
+ mask_out = __msa_max_u_b(flat_out, mask_out); \
+ p3_asub_p2_m = __msa_max_u_b(p3_asub_p2_m, p2_asub_p1_m); \
+ mask_out = __msa_max_u_b(p3_asub_p2_m, mask_out); \
+ q2_asub_q1_m = __msa_max_u_b(q2_asub_q1_m, q3_asub_q2_m); \
+ mask_out = __msa_max_u_b(q2_asub_q1_m, mask_out); \
+ mask_out = (limit_in < mask_out); \
+ mask_out = __msa_xori_b(mask_out, 0xff); \
+} while (0)
+
+#define ST6x1_UB(in0, in0_idx, in1, in1_idx, pdst, stride) do { \
+ const uint16_t tmp0_h = __msa_copy_s_h((v8i16)in1, in1_idx); \
+ const uint32_t tmp0_w = __msa_copy_s_w((v4i32)in0, in0_idx); \
+ SW(tmp0_w, pdst); \
+ SH(tmp0_h, pdst + stride); \
+} while (0)
+
+#define ST6x4_UB(in0, start_in0_idx, in1, start_in1_idx, pdst, stride) do { \
+ uint8_t* ptmp1 = (uint8_t*)pdst; \
+ ST6x1_UB(in0, start_in0_idx, in1, start_in1_idx, ptmp1, 4); \
+ ptmp1 += stride; \
+ ST6x1_UB(in0, start_in0_idx + 1, in1, start_in1_idx + 1, ptmp1, 4); \
+ ptmp1 += stride; \
+ ST6x1_UB(in0, start_in0_idx + 2, in1, start_in1_idx + 2, ptmp1, 4); \
+ ptmp1 += stride; \
+ ST6x1_UB(in0, start_in0_idx + 3, in1, start_in1_idx + 3, ptmp1, 4); \
+} while (0)
+
+#define LPF_SIMPLE_FILT(p1_in, p0_in, q0_in, q1_in, mask) do { \
+ v16i8 p1_m, p0_m, q0_m, q1_m, filt, filt1, filt2; \
+ const v16i8 cnst4b = __msa_ldi_b(4); \
+ const v16i8 cnst3b = __msa_ldi_b(3); \
+ \
+ FLIP_SIGN4(p1_in, p0_in, q0_in, q1_in, p1_m, p0_m, q0_m, q1_m); \
+ filt = __msa_subs_s_b(p1_m, q1_m); \
+ FILT_VAL(q0_m, p0_m, mask, filt); \
+ filt1 = __msa_adds_s_b(filt, cnst4b); \
+ filt1 = SRAI_B(filt1, 3); \
+ filt2 = __msa_adds_s_b(filt, cnst3b); \
+ filt2 = SRAI_B(filt2, 3); \
+ q0_m = __msa_subs_s_b(q0_m, filt1); \
+ p0_m = __msa_adds_s_b(p0_m, filt2); \
+ q0_in = __msa_xori_b((v16u8)q0_m, 0x80); \
+ p0_in = __msa_xori_b((v16u8)p0_m, 0x80); \
+} while (0)
+
+#define LPF_SIMPLE_MASK(p1, p0, q0, q1, b_limit, mask) do { \
+ v16u8 p1_a_sub_q1, p0_a_sub_q0; \
+ \
+ p0_a_sub_q0 = __msa_asub_u_b(p0, q0); \
+ p1_a_sub_q1 = __msa_asub_u_b(p1, q1); \
+ p1_a_sub_q1 = (v16u8)__msa_srli_b((v16i8)p1_a_sub_q1, 1); \
+ p0_a_sub_q0 = __msa_adds_u_b(p0_a_sub_q0, p0_a_sub_q0); \
+ mask = __msa_adds_u_b(p0_a_sub_q0, p1_a_sub_q1); \
+ mask = (mask <= b_limit); \
+} while (0)
+
+static void VFilter16(uint8_t* src, int stride,
+ int b_limit_in, int limit_in, int thresh_in) {
+ uint8_t* ptemp = src - 4 * stride;
+ v16u8 p3, p2, p1, p0, q3, q2, q1, q0;
+ v16u8 mask, hev;
+ const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in);
+ const v16u8 limit = (v16u8)__msa_fill_b(limit_in);
+ const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
+
+ LD_UB8(ptemp, stride, p3, p2, p1, p0, q0, q1, q2, q3);
+ LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh,
+ hev, mask);
+ LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev);
+ ptemp = src - 3 * stride;
+ ST_UB4(p2, p1, p0, q0, ptemp, stride);
+ ptemp += (4 * stride);
+ ST_UB2(q1, q2, ptemp, stride);
+}
+
+static void HFilter16(uint8_t* src, int stride,
+ int b_limit_in, int limit_in, int thresh_in) {
+ uint8_t* ptmp = src - 4;
+ v16u8 p3, p2, p1, p0, q3, q2, q1, q0;
+ v16u8 mask, hev;
+ v16u8 row0, row1, row2, row3, row4, row5, row6, row7, row8;
+ v16u8 row9, row10, row11, row12, row13, row14, row15;
+ v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
+ const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
+ const v16u8 limit = (v16u8)__msa_fill_b(limit_in);
+ const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in);
+
+ LD_UB8(ptmp, stride, row0, row1, row2, row3, row4, row5, row6, row7);
+ ptmp += (8 * stride);
+ LD_UB8(ptmp, stride, row8, row9, row10, row11, row12, row13, row14, row15);
+ TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7,
+ row8, row9, row10, row11, row12, row13, row14, row15,
+ p3, p2, p1, p0, q0, q1, q2, q3);
+ LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh,
+ hev, mask);
+ LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev);
+ ILVR_B2_SH(p1, p2, q0, p0, tmp0, tmp1);
+ ILVRL_H2_SH(tmp1, tmp0, tmp3, tmp4);
+ ILVL_B2_SH(p1, p2, q0, p0, tmp0, tmp1);
+ ILVRL_H2_SH(tmp1, tmp0, tmp6, tmp7);
+ ILVRL_B2_SH(q2, q1, tmp2, tmp5);
+ ptmp = src - 3;
+ ST6x1_UB(tmp3, 0, tmp2, 0, ptmp, 4);
+ ptmp += stride;
+ ST6x1_UB(tmp3, 1, tmp2, 1, ptmp, 4);
+ ptmp += stride;
+ ST6x1_UB(tmp3, 2, tmp2, 2, ptmp, 4);
+ ptmp += stride;
+ ST6x1_UB(tmp3, 3, tmp2, 3, ptmp, 4);
+ ptmp += stride;
+ ST6x1_UB(tmp4, 0, tmp2, 4, ptmp, 4);
+ ptmp += stride;
+ ST6x1_UB(tmp4, 1, tmp2, 5, ptmp, 4);
+ ptmp += stride;
+ ST6x1_UB(tmp4, 2, tmp2, 6, ptmp, 4);
+ ptmp += stride;
+ ST6x1_UB(tmp4, 3, tmp2, 7, ptmp, 4);
+ ptmp += stride;
+ ST6x1_UB(tmp6, 0, tmp5, 0, ptmp, 4);
+ ptmp += stride;
+ ST6x1_UB(tmp6, 1, tmp5, 1, ptmp, 4);
+ ptmp += stride;
+ ST6x1_UB(tmp6, 2, tmp5, 2, ptmp, 4);
+ ptmp += stride;
+ ST6x1_UB(tmp6, 3, tmp5, 3, ptmp, 4);
+ ptmp += stride;
+ ST6x1_UB(tmp7, 0, tmp5, 4, ptmp, 4);
+ ptmp += stride;
+ ST6x1_UB(tmp7, 1, tmp5, 5, ptmp, 4);
+ ptmp += stride;
+ ST6x1_UB(tmp7, 2, tmp5, 6, ptmp, 4);
+ ptmp += stride;
+ ST6x1_UB(tmp7, 3, tmp5, 7, ptmp, 4);
+}
+
+// on three inner edges
+static void VFilterHorEdge16i(uint8_t* src, int stride,
+ int b_limit, int limit, int thresh) {
+ v16u8 mask, hev;
+ v16u8 p3, p2, p1, p0, q3, q2, q1, q0;
+ const v16u8 thresh0 = (v16u8)__msa_fill_b(thresh);
+ const v16u8 b_limit0 = (v16u8)__msa_fill_b(b_limit);
+ const v16u8 limit0 = (v16u8)__msa_fill_b(limit);
+
+ LD_UB8((src - 4 * stride), stride, p3, p2, p1, p0, q0, q1, q2, q3);
+ LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0,
+ hev, mask);
+ LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev);
+ ST_UB4(p1, p0, q0, q1, (src - 2 * stride), stride);
+}
+
+static void VFilter16i(uint8_t* src_y, int stride,
+ int b_limit, int limit, int thresh) {
+ VFilterHorEdge16i(src_y + 4 * stride, stride, b_limit, limit, thresh);
+ VFilterHorEdge16i(src_y + 8 * stride, stride, b_limit, limit, thresh);
+ VFilterHorEdge16i(src_y + 12 * stride, stride, b_limit, limit, thresh);
+}
+
+static void HFilterVertEdge16i(uint8_t* src, int stride,
+ int b_limit, int limit, int thresh) {
+ v16u8 mask, hev;
+ v16u8 p3, p2, p1, p0, q3, q2, q1, q0;
+ v16u8 row0, row1, row2, row3, row4, row5, row6, row7;
+ v16u8 row8, row9, row10, row11, row12, row13, row14, row15;
+ v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
+ const v16u8 thresh0 = (v16u8)__msa_fill_b(thresh);
+ const v16u8 b_limit0 = (v16u8)__msa_fill_b(b_limit);
+ const v16u8 limit0 = (v16u8)__msa_fill_b(limit);
+
+ LD_UB8(src - 4, stride, row0, row1, row2, row3, row4, row5, row6, row7);
+ LD_UB8(src - 4 + (8 * stride), stride,
+ row8, row9, row10, row11, row12, row13, row14, row15);
+ TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7,
+ row8, row9, row10, row11, row12, row13, row14, row15,
+ p3, p2, p1, p0, q0, q1, q2, q3);
+ LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0,
+ hev, mask);
+ LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev);
+ ILVR_B2_SH(p0, p1, q1, q0, tmp0, tmp1);
+ ILVRL_H2_SH(tmp1, tmp0, tmp2, tmp3);
+ ILVL_B2_SH(p0, p1, q1, q0, tmp0, tmp1);
+ ILVRL_H2_SH(tmp1, tmp0, tmp4, tmp5);
+ src -= 2;
+ ST4x8_UB(tmp2, tmp3, src, stride);
+ src += (8 * stride);
+ ST4x8_UB(tmp4, tmp5, src, stride);
+}
+
+static void HFilter16i(uint8_t* src_y, int stride,
+ int b_limit, int limit, int thresh) {
+ HFilterVertEdge16i(src_y + 4, stride, b_limit, limit, thresh);
+ HFilterVertEdge16i(src_y + 8, stride, b_limit, limit, thresh);
+ HFilterVertEdge16i(src_y + 12, stride, b_limit, limit, thresh);
+}
+
+// 8-pixels wide variants, for chroma filtering
+static void VFilter8(uint8_t* src_u, uint8_t* src_v, int stride,
+ int b_limit_in, int limit_in, int thresh_in) {
+ uint8_t* ptmp_src_u = src_u - 4 * stride;
+ uint8_t* ptmp_src_v = src_v - 4 * stride;
+ uint64_t p2_d, p1_d, p0_d, q0_d, q1_d, q2_d;
+ v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev;
+ v16u8 p3_u, p2_u, p1_u, p0_u, q3_u, q2_u, q1_u, q0_u;
+ v16u8 p3_v, p2_v, p1_v, p0_v, q3_v, q2_v, q1_v, q0_v;
+ const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
+ const v16u8 limit = (v16u8)__msa_fill_b(limit_in);
+ const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in);
+
+ LD_UB8(ptmp_src_u, stride, p3_u, p2_u, p1_u, p0_u, q0_u, q1_u, q2_u, q3_u);
+ LD_UB8(ptmp_src_v, stride, p3_v, p2_v, p1_v, p0_v, q0_v, q1_v, q2_v, q3_v);
+ ILVR_D4_UB(p3_v, p3_u, p2_v, p2_u, p1_v, p1_u, p0_v, p0_u, p3, p2, p1, p0);
+ ILVR_D4_UB(q0_v, q0_u, q1_v, q1_u, q2_v, q2_u, q3_v, q3_u, q0, q1, q2, q3);
+ LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh,
+ hev, mask);
+ LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev);
+ p2_d = __msa_copy_s_d((v2i64)p2, 0);
+ p1_d = __msa_copy_s_d((v2i64)p1, 0);
+ p0_d = __msa_copy_s_d((v2i64)p0, 0);
+ q0_d = __msa_copy_s_d((v2i64)q0, 0);
+ q1_d = __msa_copy_s_d((v2i64)q1, 0);
+ q2_d = __msa_copy_s_d((v2i64)q2, 0);
+ ptmp_src_u += stride;
+ SD4(p2_d, p1_d, p0_d, q0_d, ptmp_src_u, stride);
+ ptmp_src_u += (4 * stride);
+ SD(q1_d, ptmp_src_u);
+ ptmp_src_u += stride;
+ SD(q2_d, ptmp_src_u);
+ p2_d = __msa_copy_s_d((v2i64)p2, 1);
+ p1_d = __msa_copy_s_d((v2i64)p1, 1);
+ p0_d = __msa_copy_s_d((v2i64)p0, 1);
+ q0_d = __msa_copy_s_d((v2i64)q0, 1);
+ q1_d = __msa_copy_s_d((v2i64)q1, 1);
+ q2_d = __msa_copy_s_d((v2i64)q2, 1);
+ ptmp_src_v += stride;
+ SD4(p2_d, p1_d, p0_d, q0_d, ptmp_src_v, stride);
+ ptmp_src_v += (4 * stride);
+ SD(q1_d, ptmp_src_v);
+ ptmp_src_v += stride;
+ SD(q2_d, ptmp_src_v);
+}
+
+static void HFilter8(uint8_t* src_u, uint8_t* src_v, int stride,
+ int b_limit_in, int limit_in, int thresh_in) {
+ uint8_t* ptmp_src_u = src_u - 4;
+ uint8_t* ptmp_src_v = src_v - 4;
+ v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev;
+ v16u8 row0, row1, row2, row3, row4, row5, row6, row7, row8;
+ v16u8 row9, row10, row11, row12, row13, row14, row15;
+ v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
+ const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
+ const v16u8 limit = (v16u8)__msa_fill_b(limit_in);
+ const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in);
+
+ LD_UB8(ptmp_src_u, stride, row0, row1, row2, row3, row4, row5, row6, row7);
+ LD_UB8(ptmp_src_v, stride,
+ row8, row9, row10, row11, row12, row13, row14, row15);
+ TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7,
+ row8, row9, row10, row11, row12, row13, row14, row15,
+ p3, p2, p1, p0, q0, q1, q2, q3);
+ LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh,
+ hev, mask);
+ LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev);
+ ILVR_B2_SH(p1, p2, q0, p0, tmp0, tmp1);
+ ILVRL_H2_SH(tmp1, tmp0, tmp3, tmp4);
+ ILVL_B2_SH(p1, p2, q0, p0, tmp0, tmp1);
+ ILVRL_H2_SH(tmp1, tmp0, tmp6, tmp7);
+ ILVRL_B2_SH(q2, q1, tmp2, tmp5);
+ ptmp_src_u += 1;
+ ST6x4_UB(tmp3, 0, tmp2, 0, ptmp_src_u, stride);
+ ptmp_src_u += 4 * stride;
+ ST6x4_UB(tmp4, 0, tmp2, 4, ptmp_src_u, stride);
+ ptmp_src_v += 1;
+ ST6x4_UB(tmp6, 0, tmp5, 0, ptmp_src_v, stride);
+ ptmp_src_v += 4 * stride;
+ ST6x4_UB(tmp7, 0, tmp5, 4, ptmp_src_v, stride);
+}
+
+static void VFilter8i(uint8_t* src_u, uint8_t* src_v, int stride,
+ int b_limit_in, int limit_in, int thresh_in) {
+ uint64_t p1_d, p0_d, q0_d, q1_d;
+ v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev;
+ v16u8 p3_u, p2_u, p1_u, p0_u, q3_u, q2_u, q1_u, q0_u;
+ v16u8 p3_v, p2_v, p1_v, p0_v, q3_v, q2_v, q1_v, q0_v;
+ const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in);
+ const v16u8 limit = (v16u8)__msa_fill_b(limit_in);
+ const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
+
+ LD_UB8(src_u, stride, p3_u, p2_u, p1_u, p0_u, q0_u, q1_u, q2_u, q3_u);
+ src_u += (5 * stride);
+ LD_UB8(src_v, stride, p3_v, p2_v, p1_v, p0_v, q0_v, q1_v, q2_v, q3_v);
+ src_v += (5 * stride);
+ ILVR_D4_UB(p3_v, p3_u, p2_v, p2_u, p1_v, p1_u, p0_v, p0_u, p3, p2, p1, p0);
+ ILVR_D4_UB(q0_v, q0_u, q1_v, q1_u, q2_v, q2_u, q3_v, q3_u, q0, q1, q2, q3);
+ LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh,
+ hev, mask);
+ LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev);
+ p1_d = __msa_copy_s_d((v2i64)p1, 0);
+ p0_d = __msa_copy_s_d((v2i64)p0, 0);
+ q0_d = __msa_copy_s_d((v2i64)q0, 0);
+ q1_d = __msa_copy_s_d((v2i64)q1, 0);
+ SD4(q1_d, q0_d, p0_d, p1_d, src_u, -stride);
+ p1_d = __msa_copy_s_d((v2i64)p1, 1);
+ p0_d = __msa_copy_s_d((v2i64)p0, 1);
+ q0_d = __msa_copy_s_d((v2i64)q0, 1);
+ q1_d = __msa_copy_s_d((v2i64)q1, 1);
+ SD4(q1_d, q0_d, p0_d, p1_d, src_v, -stride);
+}
+
+static void HFilter8i(uint8_t* src_u, uint8_t* src_v, int stride,
+ int b_limit_in, int limit_in, int thresh_in) {
+ v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev;
+ v16u8 row0, row1, row2, row3, row4, row5, row6, row7, row8;
+ v16u8 row9, row10, row11, row12, row13, row14, row15;
+ v4i32 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
+ const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in);
+ const v16u8 limit = (v16u8)__msa_fill_b(limit_in);
+ const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
+
+ LD_UB8(src_u, stride, row0, row1, row2, row3, row4, row5, row6, row7);
+ LD_UB8(src_v, stride,
+ row8, row9, row10, row11, row12, row13, row14, row15);
+ TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7,
+ row8, row9, row10, row11, row12, row13, row14, row15,
+ p3, p2, p1, p0, q0, q1, q2, q3);
+ LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh,
+ hev, mask);
+ LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev);
+ ILVR_B2_SW(p0, p1, q1, q0, tmp0, tmp1);
+ ILVRL_H2_SW(tmp1, tmp0, tmp2, tmp3);
+ ILVL_B2_SW(p0, p1, q1, q0, tmp0, tmp1);
+ ILVRL_H2_SW(tmp1, tmp0, tmp4, tmp5);
+ src_u += 2;
+ ST4x4_UB(tmp2, tmp2, 0, 1, 2, 3, src_u, stride);
+ src_u += 4 * stride;
+ ST4x4_UB(tmp3, tmp3, 0, 1, 2, 3, src_u, stride);
+ src_v += 2;
+ ST4x4_UB(tmp4, tmp4, 0, 1, 2, 3, src_v, stride);
+ src_v += 4 * stride;
+ ST4x4_UB(tmp5, tmp5, 0, 1, 2, 3, src_v, stride);
+}
+
+static void SimpleVFilter16(uint8_t* src, int stride, int b_limit_in) {
+ v16u8 p1, p0, q1, q0, mask;
+ const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
+
+ LD_UB4(src - 2 * stride, stride, p1, p0, q0, q1);
+ LPF_SIMPLE_MASK(p1, p0, q0, q1, b_limit, mask);
+ LPF_SIMPLE_FILT(p1, p0, q0, q1, mask);
+ ST_UB2(p0, q0, src - stride, stride);
+}
+
+static void SimpleHFilter16(uint8_t* src, int stride, int b_limit_in) {
+ v16u8 p1, p0, q1, q0, mask, row0, row1, row2, row3, row4, row5, row6, row7;
+ v16u8 row8, row9, row10, row11, row12, row13, row14, row15;
+ v8i16 tmp0, tmp1;
+ const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
+ uint8_t* ptemp_src = src - 2;
+
+ LD_UB8(ptemp_src, stride, row0, row1, row2, row3, row4, row5, row6, row7);
+ LD_UB8(ptemp_src + 8 * stride, stride,
+ row8, row9, row10, row11, row12, row13, row14, row15);
+ TRANSPOSE16x4_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7,
+ row8, row9, row10, row11, row12, row13, row14, row15,
+ p1, p0, q0, q1);
+ LPF_SIMPLE_MASK(p1, p0, q0, q1, b_limit, mask);
+ LPF_SIMPLE_FILT(p1, p0, q0, q1, mask);
+ ILVRL_B2_SH(q0, p0, tmp1, tmp0);
+ ptemp_src += 1;
+ ST2x4_UB(tmp1, 0, ptemp_src, stride);
+ ptemp_src += 4 * stride;
+ ST2x4_UB(tmp1, 4, ptemp_src, stride);
+ ptemp_src += 4 * stride;
+ ST2x4_UB(tmp0, 0, ptemp_src, stride);
+ ptemp_src += 4 * stride;
+ ST2x4_UB(tmp0, 4, ptemp_src, stride);
+ ptemp_src += 4 * stride;
+}
+
+static void SimpleVFilter16i(uint8_t* src_y, int stride, int b_limit_in) {
+ SimpleVFilter16(src_y + 4 * stride, stride, b_limit_in);
+ SimpleVFilter16(src_y + 8 * stride, stride, b_limit_in);
+ SimpleVFilter16(src_y + 12 * stride, stride, b_limit_in);
+}
+
+static void SimpleHFilter16i(uint8_t* src_y, int stride, int b_limit_in) {
+ SimpleHFilter16(src_y + 4, stride, b_limit_in);
+ SimpleHFilter16(src_y + 8, stride, b_limit_in);
+ SimpleHFilter16(src_y + 12, stride, b_limit_in);
+}
+
+//------------------------------------------------------------------------------
+// Intra predictions
+//------------------------------------------------------------------------------
+
+// 4x4
+
+static void DC4(uint8_t* dst) { // DC
+ uint32_t dc = 4;
+ int i;
+ for (i = 0; i < 4; ++i) dc += dst[i - BPS] + dst[-1 + i * BPS];
+ dc >>= 3;
+ dc = dc | (dc << 8) | (dc << 16) | (dc << 24);
+ SW4(dc, dc, dc, dc, dst, BPS);
+}
+
+static void TM4(uint8_t* dst) {
+ const uint8_t* const ptemp = dst - BPS - 1;
+ v8i16 T, d, r0, r1, r2, r3;
+ const v16i8 zero = { 0 };
+ const v8i16 TL = (v8i16)__msa_fill_h(ptemp[0 * BPS]);
+ const v8i16 L0 = (v8i16)__msa_fill_h(ptemp[1 * BPS]);
+ const v8i16 L1 = (v8i16)__msa_fill_h(ptemp[2 * BPS]);
+ const v8i16 L2 = (v8i16)__msa_fill_h(ptemp[3 * BPS]);
+ const v8i16 L3 = (v8i16)__msa_fill_h(ptemp[4 * BPS]);
+ const v16u8 T1 = LD_UB(ptemp + 1);
+
+ T = (v8i16)__msa_ilvr_b(zero, (v16i8)T1);
+ d = T - TL;
+ ADD4(d, L0, d, L1, d, L2, d, L3, r0, r1, r2, r3);
+ CLIP_SH4_0_255(r0, r1, r2, r3);
+ PCKEV_ST4x4_UB(r0, r1, r2, r3, dst, BPS);
+}
+
+static void VE4(uint8_t* dst) { // vertical
+ const uint8_t* const ptop = dst - BPS - 1;
+ const uint32_t val0 = LW(ptop + 0);
+ const uint32_t val1 = LW(ptop + 4);
+ uint32_t out;
+ v16u8 A, B, C, AC, B2, R;
+
+ INSERT_W2_UB(val0, val1, A);
+ B = SLDI_UB(A, A, 1);
+ C = SLDI_UB(A, A, 2);
+ AC = __msa_ave_u_b(A, C);
+ B2 = __msa_ave_u_b(B, B);
+ R = __msa_aver_u_b(AC, B2);
+ out = __msa_copy_s_w((v4i32)R, 0);
+ SW4(out, out, out, out, dst, BPS);
+}
+
+static void RD4(uint8_t* dst) { // Down-right
+ const uint8_t* const ptop = dst - 1 - BPS;
+ uint32_t val0 = LW(ptop + 0);
+ uint32_t val1 = LW(ptop + 4);
+ uint32_t val2, val3;
+ v16u8 A, B, C, AC, B2, R, A1;
+
+ INSERT_W2_UB(val0, val1, A1);
+ A = SLDI_UB(A1, A1, 12);
+ A = (v16u8)__msa_insert_b((v16i8)A, 3, ptop[1 * BPS]);
+ A = (v16u8)__msa_insert_b((v16i8)A, 2, ptop[2 * BPS]);
+ A = (v16u8)__msa_insert_b((v16i8)A, 1, ptop[3 * BPS]);
+ A = (v16u8)__msa_insert_b((v16i8)A, 0, ptop[4 * BPS]);
+ B = SLDI_UB(A, A, 1);
+ C = SLDI_UB(A, A, 2);
+ AC = __msa_ave_u_b(A, C);
+ B2 = __msa_ave_u_b(B, B);
+ R = __msa_aver_u_b(AC, B2);
+ val3 = __msa_copy_s_w((v4i32)R, 0);
+ R = SLDI_UB(R, R, 1);
+ val2 = __msa_copy_s_w((v4i32)R, 0);
+ R = SLDI_UB(R, R, 1);
+ val1 = __msa_copy_s_w((v4i32)R, 0);
+ R = SLDI_UB(R, R, 1);
+ val0 = __msa_copy_s_w((v4i32)R, 0);
+ SW4(val0, val1, val2, val3, dst, BPS);
+}
+
+static void LD4(uint8_t* dst) { // Down-Left
+ const uint8_t* const ptop = dst - BPS;
+ uint32_t val0 = LW(ptop + 0);
+ uint32_t val1 = LW(ptop + 4);
+ uint32_t val2, val3;
+ v16u8 A, B, C, AC, B2, R;
+
+ INSERT_W2_UB(val0, val1, A);
+ B = SLDI_UB(A, A, 1);
+ C = SLDI_UB(A, A, 2);
+ C = (v16u8)__msa_insert_b((v16i8)C, 6, ptop[7]);
+ AC = __msa_ave_u_b(A, C);
+ B2 = __msa_ave_u_b(B, B);
+ R = __msa_aver_u_b(AC, B2);
+ val0 = __msa_copy_s_w((v4i32)R, 0);
+ R = SLDI_UB(R, R, 1);
+ val1 = __msa_copy_s_w((v4i32)R, 0);
+ R = SLDI_UB(R, R, 1);
+ val2 = __msa_copy_s_w((v4i32)R, 0);
+ R = SLDI_UB(R, R, 1);
+ val3 = __msa_copy_s_w((v4i32)R, 0);
+ SW4(val0, val1, val2, val3, dst, BPS);
+}
+
+// 16x16
+
+static void DC16(uint8_t* dst) { // DC
+ uint32_t dc = 16;
+ int i;
+ const v16u8 rtop = LD_UB(dst - BPS);
+ const v8u16 dctop = __msa_hadd_u_h(rtop, rtop);
+ v16u8 out;
+
+ for (i = 0; i < 16; ++i) {
+ dc += dst[-1 + i * BPS];
+ }
+ dc += HADD_UH_U32(dctop);
+ out = (v16u8)__msa_fill_b(dc >> 5);
+ ST_UB8(out, out, out, out, out, out, out, out, dst, BPS);
+ ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS);
+}
+
+static void TM16(uint8_t* dst) {
+ int j;
+ v8i16 d1, d2;
+ const v16i8 zero = { 0 };
+ const v8i16 TL = (v8i16)__msa_fill_h(dst[-1 - BPS]);
+ const v16i8 T = LD_SB(dst - BPS);
+
+ ILVRL_B2_SH(zero, T, d1, d2);
+ SUB2(d1, TL, d2, TL, d1, d2);
+ for (j = 0; j < 16; j += 4) {
+ v16i8 t0, t1, t2, t3;
+ v8i16 r0, r1, r2, r3, r4, r5, r6, r7;
+ const v8i16 L0 = (v8i16)__msa_fill_h(dst[-1 + 0 * BPS]);
+ const v8i16 L1 = (v8i16)__msa_fill_h(dst[-1 + 1 * BPS]);
+ const v8i16 L2 = (v8i16)__msa_fill_h(dst[-1 + 2 * BPS]);
+ const v8i16 L3 = (v8i16)__msa_fill_h(dst[-1 + 3 * BPS]);
+ ADD4(d1, L0, d1, L1, d1, L2, d1, L3, r0, r1, r2, r3);
+ ADD4(d2, L0, d2, L1, d2, L2, d2, L3, r4, r5, r6, r7);
+ CLIP_SH4_0_255(r0, r1, r2, r3);
+ CLIP_SH4_0_255(r4, r5, r6, r7);
+ PCKEV_B4_SB(r4, r0, r5, r1, r6, r2, r7, r3, t0, t1, t2, t3);
+ ST_SB4(t0, t1, t2, t3, dst, BPS);
+ dst += 4 * BPS;
+ }
+}
+
+static void VE16(uint8_t* dst) { // vertical
+ const v16u8 rtop = LD_UB(dst - BPS);
+ ST_UB8(rtop, rtop, rtop, rtop, rtop, rtop, rtop, rtop, dst, BPS);
+ ST_UB8(rtop, rtop, rtop, rtop, rtop, rtop, rtop, rtop, dst + 8 * BPS, BPS);
+}
+
+static void HE16(uint8_t* dst) { // horizontal
+ int j;
+ for (j = 16; j > 0; j -= 4) {
+ const v16u8 L0 = (v16u8)__msa_fill_b(dst[-1 + 0 * BPS]);
+ const v16u8 L1 = (v16u8)__msa_fill_b(dst[-1 + 1 * BPS]);
+ const v16u8 L2 = (v16u8)__msa_fill_b(dst[-1 + 2 * BPS]);
+ const v16u8 L3 = (v16u8)__msa_fill_b(dst[-1 + 3 * BPS]);
+ ST_UB4(L0, L1, L2, L3, dst, BPS);
+ dst += 4 * BPS;
+ }
+}
+
+static void DC16NoTop(uint8_t* dst) { // DC with top samples not available
+ int j;
+ uint32_t dc = 8;
+ v16u8 out;
+
+ for (j = 0; j < 16; ++j) {
+ dc += dst[-1 + j * BPS];
+ }
+ out = (v16u8)__msa_fill_b(dc >> 4);
+ ST_UB8(out, out, out, out, out, out, out, out, dst, BPS);
+ ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS);
+}
+
+static void DC16NoLeft(uint8_t* dst) { // DC with left samples not available
+ uint32_t dc = 8;
+ const v16u8 rtop = LD_UB(dst - BPS);
+ const v8u16 dctop = __msa_hadd_u_h(rtop, rtop);
+ v16u8 out;
+
+ dc += HADD_UH_U32(dctop);
+ out = (v16u8)__msa_fill_b(dc >> 4);
+ ST_UB8(out, out, out, out, out, out, out, out, dst, BPS);
+ ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS);
+}
+
+static void DC16NoTopLeft(uint8_t* dst) { // DC with nothing
+ const v16u8 out = (v16u8)__msa_fill_b(0x80);
+ ST_UB8(out, out, out, out, out, out, out, out, dst, BPS);
+ ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS);
+}
+
+// Chroma
+
+#define STORE8x8(out, dst) do { \
+ SD4(out, out, out, out, dst + 0 * BPS, BPS); \
+ SD4(out, out, out, out, dst + 4 * BPS, BPS); \
+} while (0)
+
+static void DC8uv(uint8_t* dst) { // DC
+ uint32_t dc = 8;
+ int i;
+ uint64_t out;
+ const v16u8 rtop = LD_UB(dst - BPS);
+ const v8u16 temp0 = __msa_hadd_u_h(rtop, rtop);
+ const v4u32 temp1 = __msa_hadd_u_w(temp0, temp0);
+ const v2u64 temp2 = __msa_hadd_u_d(temp1, temp1);
+ v16u8 dctemp;
+
+ for (i = 0; i < 8; ++i) {
+ dc += dst[-1 + i * BPS];
+ }
+ dc += __msa_copy_s_w((v4i32)temp2, 0);
+ dctemp = (v16u8)__msa_fill_b(dc >> 4);
+ out = __msa_copy_s_d((v2i64)dctemp, 0);
+ STORE8x8(out, dst);
+}
+
+static void TM8uv(uint8_t* dst) {
+ int j;
+ const v16i8 T1 = LD_SB(dst - BPS);
+ const v16i8 zero = { 0 };
+ const v8i16 T = (v8i16)__msa_ilvr_b(zero, T1);
+ const v8i16 TL = (v8i16)__msa_fill_h(dst[-1 - BPS]);
+ const v8i16 d = T - TL;
+
+ for (j = 0; j < 8; j += 4) {
+ v16i8 t0, t1;
+ v8i16 r0 = (v8i16)__msa_fill_h(dst[-1 + 0 * BPS]);
+ v8i16 r1 = (v8i16)__msa_fill_h(dst[-1 + 1 * BPS]);
+ v8i16 r2 = (v8i16)__msa_fill_h(dst[-1 + 2 * BPS]);
+ v8i16 r3 = (v8i16)__msa_fill_h(dst[-1 + 3 * BPS]);
+ ADD4(d, r0, d, r1, d, r2, d, r3, r0, r1, r2, r3);
+ CLIP_SH4_0_255(r0, r1, r2, r3);
+ PCKEV_B2_SB(r1, r0, r3, r2, t0, t1);
+ ST4x4_UB(t0, t1, 0, 2, 0, 2, dst, BPS);
+ ST4x4_UB(t0, t1, 1, 3, 1, 3, dst + 4, BPS);
+ dst += 4 * BPS;
+ }
+}
+
+static void VE8uv(uint8_t* dst) { // vertical
+ const v16u8 rtop = LD_UB(dst - BPS);
+ const uint64_t out = __msa_copy_s_d((v2i64)rtop, 0);
+ STORE8x8(out, dst);
+}
+
+static void HE8uv(uint8_t* dst) { // horizontal
+ int j;
+ for (j = 0; j < 8; j += 4) {
+ const v16u8 L0 = (v16u8)__msa_fill_b(dst[-1 + 0 * BPS]);
+ const v16u8 L1 = (v16u8)__msa_fill_b(dst[-1 + 1 * BPS]);
+ const v16u8 L2 = (v16u8)__msa_fill_b(dst[-1 + 2 * BPS]);
+ const v16u8 L3 = (v16u8)__msa_fill_b(dst[-1 + 3 * BPS]);
+ const uint64_t out0 = __msa_copy_s_d((v2i64)L0, 0);
+ const uint64_t out1 = __msa_copy_s_d((v2i64)L1, 0);
+ const uint64_t out2 = __msa_copy_s_d((v2i64)L2, 0);
+ const uint64_t out3 = __msa_copy_s_d((v2i64)L3, 0);
+ SD4(out0, out1, out2, out3, dst, BPS);
+ dst += 4 * BPS;
+ }
+}
+
+static void DC8uvNoLeft(uint8_t* dst) { // DC with no left samples
+ const uint32_t dc = 4;
+ const v16u8 rtop = LD_UB(dst - BPS);
+ const v8u16 temp0 = __msa_hadd_u_h(rtop, rtop);
+ const v4u32 temp1 = __msa_hadd_u_w(temp0, temp0);
+ const v2u64 temp2 = __msa_hadd_u_d(temp1, temp1);
+ const uint32_t sum_m = __msa_copy_s_w((v4i32)temp2, 0);
+ const v16u8 dcval = (v16u8)__msa_fill_b((dc + sum_m) >> 3);
+ const uint64_t out = __msa_copy_s_d((v2i64)dcval, 0);
+ STORE8x8(out, dst);
+}
+
+static void DC8uvNoTop(uint8_t* dst) { // DC with no top samples
+ uint32_t dc = 4;
+ int i;
+ uint64_t out;
+ v16u8 dctemp;
+
+ for (i = 0; i < 8; ++i) {
+ dc += dst[-1 + i * BPS];
+ }
+ dctemp = (v16u8)__msa_fill_b(dc >> 3);
+ out = __msa_copy_s_d((v2i64)dctemp, 0);
+ STORE8x8(out, dst);
+}
+
+static void DC8uvNoTopLeft(uint8_t* dst) { // DC with nothing
+ const uint64_t out = 0x8080808080808080ULL;
+ STORE8x8(out, dst);
+}
+
+//------------------------------------------------------------------------------
// Entry point
extern void VP8DspInitMSA(void);
@@ -163,6 +977,39 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8DspInitMSA(void) {
VP8Transform = TransformTwo;
VP8TransformDC = TransformDC;
VP8TransformAC3 = TransformAC3;
+
+ VP8VFilter16 = VFilter16;
+ VP8HFilter16 = HFilter16;
+ VP8VFilter16i = VFilter16i;
+ VP8HFilter16i = HFilter16i;
+ VP8VFilter8 = VFilter8;
+ VP8HFilter8 = HFilter8;
+ VP8VFilter8i = VFilter8i;
+ VP8HFilter8i = HFilter8i;
+ VP8SimpleVFilter16 = SimpleVFilter16;
+ VP8SimpleHFilter16 = SimpleHFilter16;
+ VP8SimpleVFilter16i = SimpleVFilter16i;
+ VP8SimpleHFilter16i = SimpleHFilter16i;
+
+ VP8PredLuma4[0] = DC4;
+ VP8PredLuma4[1] = TM4;
+ VP8PredLuma4[2] = VE4;
+ VP8PredLuma4[4] = RD4;
+ VP8PredLuma4[6] = LD4;
+ VP8PredLuma16[0] = DC16;
+ VP8PredLuma16[1] = TM16;
+ VP8PredLuma16[2] = VE16;
+ VP8PredLuma16[3] = HE16;
+ VP8PredLuma16[4] = DC16NoTop;
+ VP8PredLuma16[5] = DC16NoLeft;
+ VP8PredLuma16[6] = DC16NoTopLeft;
+ VP8PredChroma8[0] = DC8uv;
+ VP8PredChroma8[1] = TM8uv;
+ VP8PredChroma8[2] = VE8uv;
+ VP8PredChroma8[3] = HE8uv;
+ VP8PredChroma8[4] = DC8uvNoTop;
+ VP8PredChroma8[5] = DC8uvNoLeft;
+ VP8PredChroma8[6] = DC8uvNoTopLeft;
}
#else // !WEBP_USE_MSA
diff --git a/src/3rdparty/libwebp/src/dsp/dec_neon.c b/src/3rdparty/libwebp/src/dsp/dec_neon.c
index a63f43f..34796cf 100644
--- a/src/3rdparty/libwebp/src/dsp/dec_neon.c
+++ b/src/3rdparty/libwebp/src/dsp/dec_neon.c
@@ -17,7 +17,7 @@
#if defined(WEBP_USE_NEON)
#include "./neon.h"
-#include "../dec/vp8i.h"
+#include "../dec/vp8i_dec.h"
//------------------------------------------------------------------------------
// NxM Loading functions
@@ -666,9 +666,8 @@ static uint8x16_t NeedsHev(const uint8x16_t p1, const uint8x16_t p0,
const uint8x16_t hev_thresh_v = vdupq_n_u8((uint8_t)hev_thresh);
const uint8x16_t a_p1_p0 = vabdq_u8(p1, p0); // abs(p1 - p0)
const uint8x16_t a_q1_q0 = vabdq_u8(q1, q0); // abs(q1 - q0)
- const uint8x16_t mask1 = vcgtq_u8(a_p1_p0, hev_thresh_v);
- const uint8x16_t mask2 = vcgtq_u8(a_q1_q0, hev_thresh_v);
- const uint8x16_t mask = vorrq_u8(mask1, mask2);
+ const uint8x16_t a_max = vmaxq_u8(a_p1_p0, a_q1_q0);
+ const uint8x16_t mask = vcgtq_u8(a_max, hev_thresh_v);
return mask;
}
@@ -756,24 +755,25 @@ static void ApplyFilter6(
const int8x16_t delta,
uint8x16_t* const op2, uint8x16_t* const op1, uint8x16_t* const op0,
uint8x16_t* const oq0, uint8x16_t* const oq1, uint8x16_t* const oq2) {
- const int16x8_t kCst63 = vdupq_n_s16(63);
- const int8x8_t kCst27 = vdup_n_s8(27);
- const int8x8_t kCst18 = vdup_n_s8(18);
- const int8x8_t kCst9 = vdup_n_s8(9);
+ // We have to compute: X = (9*a+63) >> 7, Y = (18*a+63)>>7, Z = (27*a+63) >> 7
+ // Turns out, there's a common sub-expression S=9 * a - 1 that can be used
+ // with the special vqrshrn_n_s16 rounding-shift-and-narrow instruction:
+ // X = (S + 64) >> 7, Y = (S + 32) >> 6, Z = (18 * a + S + 64) >> 7
const int8x8_t delta_lo = vget_low_s8(delta);
const int8x8_t delta_hi = vget_high_s8(delta);
- const int16x8_t s1_lo = vmlal_s8(kCst63, kCst27, delta_lo); // 63 + 27 * a
- const int16x8_t s1_hi = vmlal_s8(kCst63, kCst27, delta_hi); // 63 + 27 * a
- const int16x8_t s2_lo = vmlal_s8(kCst63, kCst18, delta_lo); // 63 + 18 * a
- const int16x8_t s2_hi = vmlal_s8(kCst63, kCst18, delta_hi); // 63 + 18 * a
- const int16x8_t s3_lo = vmlal_s8(kCst63, kCst9, delta_lo); // 63 + 9 * a
- const int16x8_t s3_hi = vmlal_s8(kCst63, kCst9, delta_hi); // 63 + 9 * a
- const int8x8_t a1_lo = vqshrn_n_s16(s1_lo, 7);
- const int8x8_t a1_hi = vqshrn_n_s16(s1_hi, 7);
- const int8x8_t a2_lo = vqshrn_n_s16(s2_lo, 7);
- const int8x8_t a2_hi = vqshrn_n_s16(s2_hi, 7);
- const int8x8_t a3_lo = vqshrn_n_s16(s3_lo, 7);
- const int8x8_t a3_hi = vqshrn_n_s16(s3_hi, 7);
+ const int8x8_t kCst9 = vdup_n_s8(9);
+ const int16x8_t kCstm1 = vdupq_n_s16(-1);
+ const int8x8_t kCst18 = vdup_n_s8(18);
+ const int16x8_t S_lo = vmlal_s8(kCstm1, kCst9, delta_lo); // S = 9 * a - 1
+ const int16x8_t S_hi = vmlal_s8(kCstm1, kCst9, delta_hi);
+ const int16x8_t Z_lo = vmlal_s8(S_lo, kCst18, delta_lo); // S + 18 * a
+ const int16x8_t Z_hi = vmlal_s8(S_hi, kCst18, delta_hi);
+ const int8x8_t a3_lo = vqrshrn_n_s16(S_lo, 7); // (9 * a + 63) >> 7
+ const int8x8_t a3_hi = vqrshrn_n_s16(S_hi, 7);
+ const int8x8_t a2_lo = vqrshrn_n_s16(S_lo, 6); // (9 * a + 31) >> 6
+ const int8x8_t a2_hi = vqrshrn_n_s16(S_hi, 6);
+ const int8x8_t a1_lo = vqrshrn_n_s16(Z_lo, 7); // (27 * a + 63) >> 7
+ const int8x8_t a1_hi = vqrshrn_n_s16(Z_hi, 7);
const int8x16_t a1 = vcombine_s8(a1_lo, a1_hi);
const int8x16_t a2 = vcombine_s8(a2_lo, a2_hi);
const int8x16_t a3 = vcombine_s8(a3_lo, a3_hi);
diff --git a/src/3rdparty/libwebp/src/dsp/dec_sse2.c b/src/3rdparty/libwebp/src/dsp/dec_sse2.c
index f0a8ddc..411fb02 100644
--- a/src/3rdparty/libwebp/src/dsp/dec_sse2.c
+++ b/src/3rdparty/libwebp/src/dsp/dec_sse2.c
@@ -22,7 +22,7 @@
#include <emmintrin.h>
#include "./common_sse2.h"
-#include "../dec/vp8i.h"
+#include "../dec/vp8i_dec.h"
#include "../utils/utils.h"
//------------------------------------------------------------------------------
@@ -140,7 +140,7 @@ static void Transform(const int16_t* in, uint8_t* dst, int do_two) {
// Transpose the two 4x4.
VP8Transpose_2_4x4_16b(&shifted0, &shifted1, &shifted2, &shifted3, &T0, &T1,
- &T2, &T3);
+ &T2, &T3);
}
// Add inverse transform to 'dst' and store.
diff --git a/src/3rdparty/libwebp/src/dsp/dec_sse41.c b/src/3rdparty/libwebp/src/dsp/dec_sse41.c
index 8d6aed1..4e81ec4 100644
--- a/src/3rdparty/libwebp/src/dsp/dec_sse41.c
+++ b/src/3rdparty/libwebp/src/dsp/dec_sse41.c
@@ -16,7 +16,7 @@
#if defined(WEBP_USE_SSE41)
#include <smmintrin.h>
-#include "../dec/vp8i.h"
+#include "../dec/vp8i_dec.h"
#include "../utils/utils.h"
static void HE16(uint8_t* dst) { // horizontal
diff --git a/src/3rdparty/libwebp/src/dsp/dsp.h b/src/3rdparty/libwebp/src/dsp/dsp.h
index 1faac27..813fed4 100644
--- a/src/3rdparty/libwebp/src/dsp/dsp.h
+++ b/src/3rdparty/libwebp/src/dsp/dsp.h
@@ -111,8 +111,7 @@ extern "C" {
#define WEBP_UBSAN_IGNORE_UNDEF
#define WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW
-#if !defined(WEBP_FORCE_ALIGNED) && defined(__clang__) && \
- defined(__has_attribute)
+#if defined(__clang__) && defined(__has_attribute)
#if __has_attribute(no_sanitize)
// This macro prevents the undefined behavior sanitizer from reporting
// failures. This is only meant to silence unaligned loads on platforms that
@@ -133,6 +132,7 @@ extern "C" {
typedef enum {
kSSE2,
kSSE3,
+ kSlowSSSE3, // special feature for slow SSSE3 architectures
kSSE4_1,
kAVX,
kAVX2,
@@ -185,6 +185,11 @@ typedef int (*VP8WMetric)(const uint8_t* pix, const uint8_t* ref,
// 4 by 4 symmetric matrix.
extern VP8WMetric VP8TDisto4x4, VP8TDisto16x16;
+// Compute the average (DC) of four 4x4 blocks.
+// Each sub-4x4 block #i sum is stored in dc[i].
+typedef void (*VP8MeanMetric)(const uint8_t* ref, uint32_t dc[4]);
+extern VP8MeanMetric VP8Mean16x4;
+
typedef void (*VP8BlockCopy)(const uint8_t* src, uint8_t* dst);
extern VP8BlockCopy VP8Copy4x4;
extern VP8BlockCopy VP8Copy16x8;
@@ -246,30 +251,37 @@ extern VP8GetResidualCostFunc VP8GetResidualCost;
void VP8EncDspCostInit(void);
//------------------------------------------------------------------------------
-// SSIM utils
+// SSIM / PSNR utils
// struct for accumulating statistical moments
typedef struct {
- double w; // sum(w_i) : sum of weights
- double xm, ym; // sum(w_i * x_i), sum(w_i * y_i)
- double xxm, xym, yym; // sum(w_i * x_i * x_i), etc.
+ uint32_t w; // sum(w_i) : sum of weights
+ uint32_t xm, ym; // sum(w_i * x_i), sum(w_i * y_i)
+ uint32_t xxm, xym, yym; // sum(w_i * x_i * x_i), etc.
} VP8DistoStats;
+// Compute the final SSIM value
+// The non-clipped version assumes stats->w = (2 * VP8_SSIM_KERNEL + 1)^2.
+double VP8SSIMFromStats(const VP8DistoStats* const stats);
+double VP8SSIMFromStatsClipped(const VP8DistoStats* const stats);
+
#define VP8_SSIM_KERNEL 3 // total size of the kernel: 2 * VP8_SSIM_KERNEL + 1
-typedef void (*VP8SSIMAccumulateClippedFunc)(const uint8_t* src1, int stride1,
- const uint8_t* src2, int stride2,
- int xo, int yo, // center position
- int W, int H, // plane dimension
- VP8DistoStats* const stats);
+typedef double (*VP8SSIMGetClippedFunc)(const uint8_t* src1, int stride1,
+ const uint8_t* src2, int stride2,
+ int xo, int yo, // center position
+ int W, int H); // plane dimension
// This version is called with the guarantee that you can load 8 bytes and
// 8 rows at offset src1 and src2
-typedef void (*VP8SSIMAccumulateFunc)(const uint8_t* src1, int stride1,
- const uint8_t* src2, int stride2,
- VP8DistoStats* const stats);
+typedef double (*VP8SSIMGetFunc)(const uint8_t* src1, int stride1,
+ const uint8_t* src2, int stride2);
+
+extern VP8SSIMGetFunc VP8SSIMGet; // unclipped / unchecked
+extern VP8SSIMGetClippedFunc VP8SSIMGetClipped; // with clipping
-extern VP8SSIMAccumulateFunc VP8SSIMAccumulate; // unclipped / unchecked
-extern VP8SSIMAccumulateClippedFunc VP8SSIMAccumulateClipped; // with clipping
+typedef uint32_t (*VP8AccumulateSSEFunc)(const uint8_t* src1,
+ const uint8_t* src2, int len);
+extern VP8AccumulateSSEFunc VP8AccumulateSSE;
// must be called before using any of the above directly
void VP8SSIMDspInit(void);
@@ -416,6 +428,15 @@ extern void WebPConvertARGBToUV_C(const uint32_t* argb, uint8_t* u, uint8_t* v,
extern void WebPConvertRGBA32ToUV_C(const uint16_t* rgb,
uint8_t* u, uint8_t* v, int width);
+// utilities for accurate RGB->YUV conversion
+extern uint64_t (*WebPSharpYUVUpdateY)(const uint16_t* src, const uint16_t* ref,
+ uint16_t* dst, int len);
+extern void (*WebPSharpYUVUpdateRGB)(const int16_t* src, const int16_t* ref,
+ int16_t* dst, int len);
+extern void (*WebPSharpYUVFilterRow)(const int16_t* A, const int16_t* B,
+ int len,
+ const uint16_t* best_y, uint16_t* out);
+
// Must be called before using the above.
void WebPInitConvertARGBToYUV(void);
@@ -488,6 +509,10 @@ extern int (*WebPExtractAlpha)(const uint8_t* argb, int argb_stride,
int width, int height,
uint8_t* alpha, int alpha_stride);
+// Extract the green values from 32b values in argb[] and pack them into alpha[]
+// (this is the opposite of WebPDispatchAlphaToGreen).
+extern void (*WebPExtractGreen)(const uint32_t* argb, uint8_t* alpha, int size);
+
// Pre-Multiply operation transforms x into x * A / 255 (where x=Y,R,G or B).
// Un-Multiply operation transforms x into x * 255 / A.
diff --git a/src/3rdparty/libwebp/src/dsp/enc.c b/src/3rdparty/libwebp/src/dsp/enc.c
index f639f55..f31bc6d 100644
--- a/src/3rdparty/libwebp/src/dsp/enc.c
+++ b/src/3rdparty/libwebp/src/dsp/enc.c
@@ -15,7 +15,7 @@
#include <stdlib.h> // for abs()
#include "./dsp.h"
-#include "../enc/vp8enci.h"
+#include "../enc/vp8i_enc.h"
static WEBP_INLINE uint8_t clip_8b(int v) {
return (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255;
@@ -335,7 +335,7 @@ static void Intra16Preds(uint8_t* dst,
// luma 4x4 prediction
#define DST(x, y) dst[(x) + (y) * BPS]
-#define AVG3(a, b, c) (((a) + 2 * (b) + (c) + 2) >> 2)
+#define AVG3(a, b, c) ((uint8_t)(((a) + 2 * (b) + (c) + 2) >> 2))
#define AVG2(a, b) (((a) + (b) + 1) >> 1)
static void VE4(uint8_t* dst, const uint8_t* top) { // vertical
@@ -551,6 +551,20 @@ static int SSE4x4(const uint8_t* a, const uint8_t* b) {
return GetSSE(a, b, 4, 4);
}
+static void Mean16x4(const uint8_t* ref, uint32_t dc[4]) {
+ int k, x, y;
+ for (k = 0; k < 4; ++k) {
+ uint32_t avg = 0;
+ for (y = 0; y < 4; ++y) {
+ for (x = 0; x < 4; ++x) {
+ avg += ref[x + y * BPS];
+ }
+ }
+ dc[k] = avg;
+ ref += 4; // go to next 4x4 block.
+ }
+}
+
//------------------------------------------------------------------------------
// Texture distortion
//
@@ -656,32 +670,6 @@ static int Quantize2Blocks(int16_t in[32], int16_t out[32],
return nz;
}
-static int QuantizeBlockWHT(int16_t in[16], int16_t out[16],
- const VP8Matrix* const mtx) {
- int n, last = -1;
- for (n = 0; n < 16; ++n) {
- const int j = kZigzag[n];
- const int sign = (in[j] < 0);
- const uint32_t coeff = sign ? -in[j] : in[j];
- assert(mtx->sharpen_[j] == 0);
- if (coeff > mtx->zthresh_[j]) {
- const uint32_t Q = mtx->q_[j];
- const uint32_t iQ = mtx->iq_[j];
- const uint32_t B = mtx->bias_[j];
- int level = QUANTDIV(coeff, iQ, B);
- if (level > MAX_LEVEL) level = MAX_LEVEL;
- if (sign) level = -level;
- in[j] = level * (int)Q;
- out[n] = level;
- if (level) last = n;
- } else {
- out[n] = 0;
- in[j] = 0;
- }
- }
- return (last >= 0);
-}
-
//------------------------------------------------------------------------------
// Block copy
@@ -703,11 +691,51 @@ static void Copy16x8(const uint8_t* src, uint8_t* dst) {
}
//------------------------------------------------------------------------------
+// SSIM / PSNR
-static void SSIMAccumulateClipped(const uint8_t* src1, int stride1,
- const uint8_t* src2, int stride2,
- int xo, int yo, int W, int H,
- VP8DistoStats* const stats) {
+// hat-shaped filter. Sum of coefficients is equal to 16.
+static const uint32_t kWeight[2 * VP8_SSIM_KERNEL + 1] = {
+ 1, 2, 3, 4, 3, 2, 1
+};
+static const uint32_t kWeightSum = 16 * 16; // sum{kWeight}^2
+
+static WEBP_INLINE double SSIMCalculation(
+ const VP8DistoStats* const stats, uint32_t N /*num samples*/) {
+ const uint32_t w2 = N * N;
+ const uint32_t C1 = 20 * w2;
+ const uint32_t C2 = 60 * w2;
+ const uint32_t C3 = 8 * 8 * w2; // 'dark' limit ~= 6
+ const uint64_t xmxm = (uint64_t)stats->xm * stats->xm;
+ const uint64_t ymym = (uint64_t)stats->ym * stats->ym;
+ if (xmxm + ymym >= C3) {
+ const int64_t xmym = (int64_t)stats->xm * stats->ym;
+ const int64_t sxy = (int64_t)stats->xym * N - xmym; // can be negative
+ const uint64_t sxx = (uint64_t)stats->xxm * N - xmxm;
+ const uint64_t syy = (uint64_t)stats->yym * N - ymym;
+ // we descale by 8 to prevent overflow during the fnum/fden multiply.
+ const uint64_t num_S = (2 * (uint64_t)(sxy < 0 ? 0 : sxy) + C2) >> 8;
+ const uint64_t den_S = (sxx + syy + C2) >> 8;
+ const uint64_t fnum = (2 * xmym + C1) * num_S;
+ const uint64_t fden = (xmxm + ymym + C1) * den_S;
+ const double r = (double)fnum / fden;
+ assert(r >= 0. && r <= 1.0);
+ return r;
+ }
+ return 1.; // area is too dark to contribute meaningfully
+}
+
+double VP8SSIMFromStats(const VP8DistoStats* const stats) {
+ return SSIMCalculation(stats, kWeightSum);
+}
+
+double VP8SSIMFromStatsClipped(const VP8DistoStats* const stats) {
+ return SSIMCalculation(stats, stats->w);
+}
+
+static double SSIMGetClipped_C(const uint8_t* src1, int stride1,
+ const uint8_t* src2, int stride2,
+ int xo, int yo, int W, int H) {
+ VP8DistoStats stats = { 0, 0, 0, 0, 0, 0 };
const int ymin = (yo - VP8_SSIM_KERNEL < 0) ? 0 : yo - VP8_SSIM_KERNEL;
const int ymax = (yo + VP8_SSIM_KERNEL > H - 1) ? H - 1
: yo + VP8_SSIM_KERNEL;
@@ -719,38 +747,61 @@ static void SSIMAccumulateClipped(const uint8_t* src1, int stride1,
src2 += ymin * stride2;
for (y = ymin; y <= ymax; ++y, src1 += stride1, src2 += stride2) {
for (x = xmin; x <= xmax; ++x) {
- const int s1 = src1[x];
- const int s2 = src2[x];
- stats->w += 1;
- stats->xm += s1;
- stats->ym += s2;
- stats->xxm += s1 * s1;
- stats->xym += s1 * s2;
- stats->yym += s2 * s2;
+ const uint32_t w = kWeight[VP8_SSIM_KERNEL + x - xo]
+ * kWeight[VP8_SSIM_KERNEL + y - yo];
+ const uint32_t s1 = src1[x];
+ const uint32_t s2 = src2[x];
+ stats.w += w;
+ stats.xm += w * s1;
+ stats.ym += w * s2;
+ stats.xxm += w * s1 * s1;
+ stats.xym += w * s1 * s2;
+ stats.yym += w * s2 * s2;
}
}
+ return VP8SSIMFromStatsClipped(&stats);
}
-static void SSIMAccumulate(const uint8_t* src1, int stride1,
- const uint8_t* src2, int stride2,
- VP8DistoStats* const stats) {
+static double SSIMGet_C(const uint8_t* src1, int stride1,
+ const uint8_t* src2, int stride2) {
+ VP8DistoStats stats = { 0, 0, 0, 0, 0, 0 };
int x, y;
for (y = 0; y <= 2 * VP8_SSIM_KERNEL; ++y, src1 += stride1, src2 += stride2) {
for (x = 0; x <= 2 * VP8_SSIM_KERNEL; ++x) {
- const int s1 = src1[x];
- const int s2 = src2[x];
- stats->w += 1;
- stats->xm += s1;
- stats->ym += s2;
- stats->xxm += s1 * s1;
- stats->xym += s1 * s2;
- stats->yym += s2 * s2;
+ const uint32_t w = kWeight[x] * kWeight[y];
+ const uint32_t s1 = src1[x];
+ const uint32_t s2 = src2[x];
+ stats.xm += w * s1;
+ stats.ym += w * s2;
+ stats.xxm += w * s1 * s1;
+ stats.xym += w * s1 * s2;
+ stats.yym += w * s2 * s2;
}
}
+ return VP8SSIMFromStats(&stats);
+}
+
+//------------------------------------------------------------------------------
+
+static uint32_t AccumulateSSE(const uint8_t* src1,
+ const uint8_t* src2, int len) {
+ int i;
+ uint32_t sse2 = 0;
+ assert(len <= 65535); // to ensure that accumulation fits within uint32_t
+ for (i = 0; i < len; ++i) {
+ const int32_t diff = src1[i] - src2[i];
+ sse2 += diff * diff;
+ }
+ return sse2;
}
-VP8SSIMAccumulateFunc VP8SSIMAccumulate;
-VP8SSIMAccumulateClippedFunc VP8SSIMAccumulateClipped;
+//------------------------------------------------------------------------------
+
+VP8SSIMGetFunc VP8SSIMGet;
+VP8SSIMGetClippedFunc VP8SSIMGetClipped;
+VP8AccumulateSSEFunc VP8AccumulateSSE;
+
+extern void VP8SSIMDspInitSSE2(void);
static volatile VP8CPUInfo ssim_last_cpuinfo_used =
(VP8CPUInfo)&ssim_last_cpuinfo_used;
@@ -758,8 +809,17 @@ static volatile VP8CPUInfo ssim_last_cpuinfo_used =
WEBP_TSAN_IGNORE_FUNCTION void VP8SSIMDspInit(void) {
if (ssim_last_cpuinfo_used == VP8GetCPUInfo) return;
- VP8SSIMAccumulate = SSIMAccumulate;
- VP8SSIMAccumulateClipped = SSIMAccumulateClipped;
+ VP8SSIMGetClipped = SSIMGetClipped_C;
+ VP8SSIMGet = SSIMGet_C;
+
+ VP8AccumulateSSE = AccumulateSSE;
+ if (VP8GetCPUInfo != NULL) {
+#if defined(WEBP_USE_SSE2)
+ if (VP8GetCPUInfo(kSSE2)) {
+ VP8SSIMDspInitSSE2();
+ }
+#endif
+ }
ssim_last_cpuinfo_used = VP8GetCPUInfo;
}
@@ -783,6 +843,7 @@ VP8Metric VP8SSE16x8;
VP8Metric VP8SSE4x4;
VP8WMetric VP8TDisto4x4;
VP8WMetric VP8TDisto16x16;
+VP8MeanMetric VP8Mean16x4;
VP8QuantizeBlock VP8EncQuantizeBlock;
VP8Quantize2Blocks VP8EncQuantize2Blocks;
VP8QuantizeBlockWHT VP8EncQuantizeBlockWHT;
@@ -795,6 +856,7 @@ extern void VP8EncDspInitAVX2(void);
extern void VP8EncDspInitNEON(void);
extern void VP8EncDspInitMIPS32(void);
extern void VP8EncDspInitMIPSdspR2(void);
+extern void VP8EncDspInitMSA(void);
static volatile VP8CPUInfo enc_last_cpuinfo_used =
(VP8CPUInfo)&enc_last_cpuinfo_used;
@@ -820,9 +882,10 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInit(void) {
VP8SSE4x4 = SSE4x4;
VP8TDisto4x4 = Disto4x4;
VP8TDisto16x16 = Disto16x16;
+ VP8Mean16x4 = Mean16x4;
VP8EncQuantizeBlock = QuantizeBlock;
VP8EncQuantize2Blocks = Quantize2Blocks;
- VP8EncQuantizeBlockWHT = QuantizeBlockWHT;
+ VP8EncQuantizeBlockWHT = QuantizeBlock;
VP8Copy4x4 = Copy4x4;
VP8Copy16x8 = Copy16x8;
@@ -858,6 +921,11 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInit(void) {
VP8EncDspInitMIPSdspR2();
}
#endif
+#if defined(WEBP_USE_MSA)
+ if (VP8GetCPUInfo(kMSA)) {
+ VP8EncDspInitMSA();
+ }
+#endif
}
enc_last_cpuinfo_used = VP8GetCPUInfo;
}
diff --git a/src/3rdparty/libwebp/src/dsp/enc_mips32.c b/src/3rdparty/libwebp/src/dsp/enc_mips32.c
index fd10143..752b14d 100644
--- a/src/3rdparty/libwebp/src/dsp/enc_mips32.c
+++ b/src/3rdparty/libwebp/src/dsp/enc_mips32.c
@@ -18,8 +18,8 @@
#if defined(WEBP_USE_MIPS32)
#include "./mips_macro.h"
-#include "../enc/vp8enci.h"
-#include "../enc/cost.h"
+#include "../enc/vp8i_enc.h"
+#include "../enc/cost_enc.h"
static const int kC1 = 20091 + (1 << 16);
static const int kC2 = 35468;
diff --git a/src/3rdparty/libwebp/src/dsp/enc_mips_dsp_r2.c b/src/3rdparty/libwebp/src/dsp/enc_mips_dsp_r2.c
index 7ab96f6..6c8c1c6 100644
--- a/src/3rdparty/libwebp/src/dsp/enc_mips_dsp_r2.c
+++ b/src/3rdparty/libwebp/src/dsp/enc_mips_dsp_r2.c
@@ -17,8 +17,8 @@
#if defined(WEBP_USE_MIPS_DSP_R2)
#include "./mips_macro.h"
-#include "../enc/cost.h"
-#include "../enc/vp8enci.h"
+#include "../enc/cost_enc.h"
+#include "../enc/vp8i_enc.h"
static const int kC1 = 20091 + (1 << 16);
static const int kC2 = 35468;
diff --git a/src/3rdparty/libwebp/src/dsp/enc_msa.c b/src/3rdparty/libwebp/src/dsp/enc_msa.c
new file mode 100644
index 0000000..909b46d
--- /dev/null
+++ b/src/3rdparty/libwebp/src/dsp/enc_msa.c
@@ -0,0 +1,892 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the COPYING file in the root of the source
+// tree. An additional intellectual property rights grant can be found
+// in the file PATENTS. All contributing project authors may
+// be found in the AUTHORS file in the root of the source tree.
+// -----------------------------------------------------------------------------
+//
+// MSA version of encoder dsp functions.
+//
+// Author: Prashant Patil (prashant.patil@imgtec.com)
+
+#include "./dsp.h"
+
+#if defined(WEBP_USE_MSA)
+
+#include <stdlib.h>
+#include "./msa_macro.h"
+#include "../enc/vp8i_enc.h"
+
+//------------------------------------------------------------------------------
+// Transforms
+
+#define IDCT_1D_W(in0, in1, in2, in3, out0, out1, out2, out3) do { \
+ v4i32 a1_m, b1_m, c1_m, d1_m; \
+ const v4i32 cospi8sqrt2minus1 = __msa_fill_w(20091); \
+ const v4i32 sinpi8sqrt2 = __msa_fill_w(35468); \
+ v4i32 c_tmp1_m = in1 * sinpi8sqrt2; \
+ v4i32 c_tmp2_m = in3 * cospi8sqrt2minus1; \
+ v4i32 d_tmp1_m = in1 * cospi8sqrt2minus1; \
+ v4i32 d_tmp2_m = in3 * sinpi8sqrt2; \
+ \
+ ADDSUB2(in0, in2, a1_m, b1_m); \
+ SRAI_W2_SW(c_tmp1_m, c_tmp2_m, 16); \
+ c_tmp2_m = c_tmp2_m + in3; \
+ c1_m = c_tmp1_m - c_tmp2_m; \
+ SRAI_W2_SW(d_tmp1_m, d_tmp2_m, 16); \
+ d_tmp1_m = d_tmp1_m + in1; \
+ d1_m = d_tmp1_m + d_tmp2_m; \
+ BUTTERFLY_4(a1_m, b1_m, c1_m, d1_m, out0, out1, out2, out3); \
+} while (0)
+
+static WEBP_INLINE void ITransformOne(const uint8_t* ref, const int16_t* in,
+ uint8_t* dst) {
+ v8i16 input0, input1;
+ v4i32 in0, in1, in2, in3, hz0, hz1, hz2, hz3, vt0, vt1, vt2, vt3;
+ v4i32 res0, res1, res2, res3;
+ v16i8 dest0, dest1, dest2, dest3;
+ const v16i8 zero = { 0 };
+
+ LD_SH2(in, 8, input0, input1);
+ UNPCK_SH_SW(input0, in0, in1);
+ UNPCK_SH_SW(input1, in2, in3);
+ IDCT_1D_W(in0, in1, in2, in3, hz0, hz1, hz2, hz3);
+ TRANSPOSE4x4_SW_SW(hz0, hz1, hz2, hz3, hz0, hz1, hz2, hz3);
+ IDCT_1D_W(hz0, hz1, hz2, hz3, vt0, vt1, vt2, vt3);
+ SRARI_W4_SW(vt0, vt1, vt2, vt3, 3);
+ TRANSPOSE4x4_SW_SW(vt0, vt1, vt2, vt3, vt0, vt1, vt2, vt3);
+ LD_SB4(ref, BPS, dest0, dest1, dest2, dest3);
+ ILVR_B4_SW(zero, dest0, zero, dest1, zero, dest2, zero, dest3,
+ res0, res1, res2, res3);
+ ILVR_H4_SW(zero, res0, zero, res1, zero, res2, zero, res3,
+ res0, res1, res2, res3);
+ ADD4(res0, vt0, res1, vt1, res2, vt2, res3, vt3, res0, res1, res2, res3);
+ CLIP_SW4_0_255(res0, res1, res2, res3);
+ PCKEV_B2_SW(res0, res1, res2, res3, vt0, vt1);
+ res0 = (v4i32)__msa_pckev_b((v16i8)vt0, (v16i8)vt1);
+ ST4x4_UB(res0, res0, 3, 2, 1, 0, dst, BPS);
+}
+
+static void ITransform(const uint8_t* ref, const int16_t* in, uint8_t* dst,
+ int do_two) {
+ ITransformOne(ref, in, dst);
+ if (do_two) {
+ ITransformOne(ref + 4, in + 16, dst + 4);
+ }
+}
+
+static void FTransform(const uint8_t* src, const uint8_t* ref, int16_t* out) {
+ uint64_t out0, out1, out2, out3;
+ uint32_t in0, in1, in2, in3;
+ v4i32 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
+ v8i16 t0, t1, t2, t3;
+ v16u8 srcl0, srcl1, src0, src1;
+ const v8i16 mask0 = { 0, 4, 8, 12, 1, 5, 9, 13 };
+ const v8i16 mask1 = { 3, 7, 11, 15, 2, 6, 10, 14 };
+ const v8i16 mask2 = { 4, 0, 5, 1, 6, 2, 7, 3 };
+ const v8i16 mask3 = { 0, 4, 1, 5, 2, 6, 3, 7 };
+ const v8i16 cnst0 = { 2217, -5352, 2217, -5352, 2217, -5352, 2217, -5352 };
+ const v8i16 cnst1 = { 5352, 2217, 5352, 2217, 5352, 2217, 5352, 2217 };
+
+ LW4(src, BPS, in0, in1, in2, in3);
+ INSERT_W4_UB(in0, in1, in2, in3, src0);
+ LW4(ref, BPS, in0, in1, in2, in3);
+ INSERT_W4_UB(in0, in1, in2, in3, src1);
+ ILVRL_B2_UB(src0, src1, srcl0, srcl1);
+ HSUB_UB2_SH(srcl0, srcl1, t0, t1);
+ VSHF_H2_SH(t0, t1, t0, t1, mask0, mask1, t2, t3);
+ ADDSUB2(t2, t3, t0, t1);
+ t0 = SRLI_H(t0, 3);
+ VSHF_H2_SH(t0, t0, t1, t1, mask2, mask3, t3, t2);
+ tmp0 = __msa_hadd_s_w(t3, t3);
+ tmp2 = __msa_hsub_s_w(t3, t3);
+ FILL_W2_SW(1812, 937, tmp1, tmp3);
+ DPADD_SH2_SW(t2, t2, cnst0, cnst1, tmp3, tmp1);
+ SRAI_W2_SW(tmp1, tmp3, 9);
+ PCKEV_H2_SH(tmp1, tmp0, tmp3, tmp2, t0, t1);
+ VSHF_H2_SH(t0, t1, t0, t1, mask0, mask1, t2, t3);
+ ADDSUB2(t2, t3, t0, t1);
+ VSHF_H2_SH(t0, t0, t1, t1, mask2, mask3, t3, t2);
+ tmp0 = __msa_hadd_s_w(t3, t3);
+ tmp2 = __msa_hsub_s_w(t3, t3);
+ ADDVI_W2_SW(tmp0, 7, tmp2, 7, tmp0, tmp2);
+ SRAI_W2_SW(tmp0, tmp2, 4);
+ FILL_W2_SW(12000, 51000, tmp1, tmp3);
+ DPADD_SH2_SW(t2, t2, cnst0, cnst1, tmp3, tmp1);
+ SRAI_W2_SW(tmp1, tmp3, 16);
+ UNPCK_R_SH_SW(t1, tmp4);
+ tmp5 = __msa_ceqi_w(tmp4, 0);
+ tmp4 = (v4i32)__msa_nor_v((v16u8)tmp5, (v16u8)tmp5);
+ tmp5 = __msa_fill_w(1);
+ tmp5 = (v4i32)__msa_and_v((v16u8)tmp5, (v16u8)tmp4);
+ tmp1 += tmp5;
+ PCKEV_H2_SH(tmp1, tmp0, tmp3, tmp2, t0, t1);
+ out0 = __msa_copy_s_d((v2i64)t0, 0);
+ out1 = __msa_copy_s_d((v2i64)t0, 1);
+ out2 = __msa_copy_s_d((v2i64)t1, 0);
+ out3 = __msa_copy_s_d((v2i64)t1, 1);
+ SD4(out0, out1, out2, out3, out, 8);
+}
+
+static void FTransformWHT(const int16_t* in, int16_t* out) {
+ v8i16 in0 = { 0 };
+ v8i16 in1 = { 0 };
+ v8i16 tmp0, tmp1, tmp2, tmp3;
+ v8i16 out0, out1;
+ const v8i16 mask0 = { 0, 1, 2, 3, 8, 9, 10, 11 };
+ const v8i16 mask1 = { 4, 5, 6, 7, 12, 13, 14, 15 };
+ const v8i16 mask2 = { 0, 4, 8, 12, 1, 5, 9, 13 };
+ const v8i16 mask3 = { 3, 7, 11, 15, 2, 6, 10, 14 };
+
+ in0 = __msa_insert_h(in0, 0, in[ 0]);
+ in0 = __msa_insert_h(in0, 1, in[ 64]);
+ in0 = __msa_insert_h(in0, 2, in[128]);
+ in0 = __msa_insert_h(in0, 3, in[192]);
+ in0 = __msa_insert_h(in0, 4, in[ 16]);
+ in0 = __msa_insert_h(in0, 5, in[ 80]);
+ in0 = __msa_insert_h(in0, 6, in[144]);
+ in0 = __msa_insert_h(in0, 7, in[208]);
+ in1 = __msa_insert_h(in1, 0, in[ 48]);
+ in1 = __msa_insert_h(in1, 1, in[112]);
+ in1 = __msa_insert_h(in1, 2, in[176]);
+ in1 = __msa_insert_h(in1, 3, in[240]);
+ in1 = __msa_insert_h(in1, 4, in[ 32]);
+ in1 = __msa_insert_h(in1, 5, in[ 96]);
+ in1 = __msa_insert_h(in1, 6, in[160]);
+ in1 = __msa_insert_h(in1, 7, in[224]);
+ ADDSUB2(in0, in1, tmp0, tmp1);
+ VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask0, mask1, tmp2, tmp3);
+ ADDSUB2(tmp2, tmp3, tmp0, tmp1);
+ VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask2, mask3, in0, in1);
+ ADDSUB2(in0, in1, tmp0, tmp1);
+ VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask0, mask1, tmp2, tmp3);
+ ADDSUB2(tmp2, tmp3, out0, out1);
+ SRAI_H2_SH(out0, out1, 1);
+ ST_SH2(out0, out1, out, 8);
+}
+
+static int TTransform(const uint8_t* in, const uint16_t* w) {
+ int sum;
+ uint32_t in0_m, in1_m, in2_m, in3_m;
+ v16i8 src0;
+ v8i16 in0, in1, tmp0, tmp1, tmp2, tmp3;
+ v4i32 dst0, dst1;
+ const v16i8 zero = { 0 };
+ const v8i16 mask0 = { 0, 1, 2, 3, 8, 9, 10, 11 };
+ const v8i16 mask1 = { 4, 5, 6, 7, 12, 13, 14, 15 };
+ const v8i16 mask2 = { 0, 4, 8, 12, 1, 5, 9, 13 };
+ const v8i16 mask3 = { 3, 7, 11, 15, 2, 6, 10, 14 };
+
+ LW4(in, BPS, in0_m, in1_m, in2_m, in3_m);
+ INSERT_W4_SB(in0_m, in1_m, in2_m, in3_m, src0);
+ ILVRL_B2_SH(zero, src0, tmp0, tmp1);
+ VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask2, mask3, in0, in1);
+ ADDSUB2(in0, in1, tmp0, tmp1);
+ VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask0, mask1, tmp2, tmp3);
+ ADDSUB2(tmp2, tmp3, tmp0, tmp1);
+ VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask2, mask3, in0, in1);
+ ADDSUB2(in0, in1, tmp0, tmp1);
+ VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask0, mask1, tmp2, tmp3);
+ ADDSUB2(tmp2, tmp3, tmp0, tmp1);
+ tmp0 = __msa_add_a_h(tmp0, (v8i16)zero);
+ tmp1 = __msa_add_a_h(tmp1, (v8i16)zero);
+ LD_SH2(w, 8, tmp2, tmp3);
+ DOTP_SH2_SW(tmp0, tmp1, tmp2, tmp3, dst0, dst1);
+ dst0 = dst0 + dst1;
+ sum = HADD_SW_S32(dst0);
+ return sum;
+}
+
+static int Disto4x4(const uint8_t* const a, const uint8_t* const b,
+ const uint16_t* const w) {
+ const int sum1 = TTransform(a, w);
+ const int sum2 = TTransform(b, w);
+ return abs(sum2 - sum1) >> 5;
+}
+
+static int Disto16x16(const uint8_t* const a, const uint8_t* const b,
+ const uint16_t* const w) {
+ int D = 0;
+ int x, y;
+ for (y = 0; y < 16 * BPS; y += 4 * BPS) {
+ for (x = 0; x < 16; x += 4) {
+ D += Disto4x4(a + x + y, b + x + y, w);
+ }
+ }
+ return D;
+}
+
+//------------------------------------------------------------------------------
+// Histogram
+
+static void CollectHistogram(const uint8_t* ref, const uint8_t* pred,
+ int start_block, int end_block,
+ VP8Histogram* const histo) {
+ int j;
+ int distribution[MAX_COEFF_THRESH + 1] = { 0 };
+ for (j = start_block; j < end_block; ++j) {
+ int16_t out[16];
+ VP8FTransform(ref + VP8DspScan[j], pred + VP8DspScan[j], out);
+ {
+ int k;
+ v8i16 coeff0, coeff1;
+ const v8i16 zero = { 0 };
+ const v8i16 max_coeff_thr = __msa_ldi_h(MAX_COEFF_THRESH);
+ LD_SH2(&out[0], 8, coeff0, coeff1);
+ coeff0 = __msa_add_a_h(coeff0, zero);
+ coeff1 = __msa_add_a_h(coeff1, zero);
+ SRAI_H2_SH(coeff0, coeff1, 3);
+ coeff0 = __msa_min_s_h(coeff0, max_coeff_thr);
+ coeff1 = __msa_min_s_h(coeff1, max_coeff_thr);
+ ST_SH2(coeff0, coeff1, &out[0], 8);
+ for (k = 0; k < 16; ++k) {
+ ++distribution[out[k]];
+ }
+ }
+ }
+ VP8SetHistogramData(distribution, histo);
+}
+
+//------------------------------------------------------------------------------
+// Intra predictions
+
+// luma 4x4 prediction
+
+#define DST(x, y) dst[(x) + (y) * BPS]
+#define AVG3(a, b, c) (((a) + 2 * (b) + (c) + 2) >> 2)
+#define AVG2(a, b) (((a) + (b) + 1) >> 1)
+
+static WEBP_INLINE void VE4(uint8_t* dst, const uint8_t* top) { // vertical
+ const uint64_t val_m = LD(top - 1);
+ const v16u8 A = (v16u8)__msa_insert_d((v2i64)A, 0, val_m);
+ const v16u8 B = SLDI_UB(A, A, 1);
+ const v16u8 C = SLDI_UB(A, A, 2);
+ const v16u8 AC = __msa_ave_u_b(A, C);
+ const v16u8 B2 = __msa_ave_u_b(B, B);
+ const v16u8 R = __msa_aver_u_b(AC, B2);
+ const uint32_t out = __msa_copy_s_w((v4i32)R, 0);
+ SW4(out, out, out, out, dst, BPS);
+}
+
+static WEBP_INLINE void HE4(uint8_t* dst, const uint8_t* top) { // horizontal
+ const int X = top[-1];
+ const int I = top[-2];
+ const int J = top[-3];
+ const int K = top[-4];
+ const int L = top[-5];
+ WebPUint32ToMem(dst + 0 * BPS, 0x01010101U * AVG3(X, I, J));
+ WebPUint32ToMem(dst + 1 * BPS, 0x01010101U * AVG3(I, J, K));
+ WebPUint32ToMem(dst + 2 * BPS, 0x01010101U * AVG3(J, K, L));
+ WebPUint32ToMem(dst + 3 * BPS, 0x01010101U * AVG3(K, L, L));
+}
+
+static WEBP_INLINE void DC4(uint8_t* dst, const uint8_t* top) {
+ uint32_t dc = 4;
+ int i;
+ for (i = 0; i < 4; ++i) dc += top[i] + top[-5 + i];
+ dc >>= 3;
+ dc = dc | (dc << 8) | (dc << 16) | (dc << 24);
+ SW4(dc, dc, dc, dc, dst, BPS);
+}
+
+static WEBP_INLINE void RD4(uint8_t* dst, const uint8_t* top) {
+ const uint64_t val_m = LD(top - 5);
+ const v16u8 A1 = (v16u8)__msa_insert_d((v2i64)A1, 0, val_m);
+ const v16u8 A = (v16u8)__msa_insert_b((v16i8)A1, 8, top[3]);
+ const v16u8 B = SLDI_UB(A, A, 1);
+ const v16u8 C = SLDI_UB(A, A, 2);
+ const v16u8 AC = __msa_ave_u_b(A, C);
+ const v16u8 B2 = __msa_ave_u_b(B, B);
+ const v16u8 R0 = __msa_aver_u_b(AC, B2);
+ const v16u8 R1 = SLDI_UB(R0, R0, 1);
+ const v16u8 R2 = SLDI_UB(R1, R1, 1);
+ const v16u8 R3 = SLDI_UB(R2, R2, 1);
+ const uint32_t val0 = __msa_copy_s_w((v4i32)R0, 0);
+ const uint32_t val1 = __msa_copy_s_w((v4i32)R1, 0);
+ const uint32_t val2 = __msa_copy_s_w((v4i32)R2, 0);
+ const uint32_t val3 = __msa_copy_s_w((v4i32)R3, 0);
+ SW4(val3, val2, val1, val0, dst, BPS);
+}
+
+static WEBP_INLINE void LD4(uint8_t* dst, const uint8_t* top) {
+ const uint64_t val_m = LD(top);
+ const v16u8 A = (v16u8)__msa_insert_d((v2i64)A, 0, val_m);
+ const v16u8 B = SLDI_UB(A, A, 1);
+ const v16u8 C1 = SLDI_UB(A, A, 2);
+ const v16u8 C = (v16u8)__msa_insert_b((v16i8)C1, 6, top[7]);
+ const v16u8 AC = __msa_ave_u_b(A, C);
+ const v16u8 B2 = __msa_ave_u_b(B, B);
+ const v16u8 R0 = __msa_aver_u_b(AC, B2);
+ const v16u8 R1 = SLDI_UB(R0, R0, 1);
+ const v16u8 R2 = SLDI_UB(R1, R1, 1);
+ const v16u8 R3 = SLDI_UB(R2, R2, 1);
+ const uint32_t val0 = __msa_copy_s_w((v4i32)R0, 0);
+ const uint32_t val1 = __msa_copy_s_w((v4i32)R1, 0);
+ const uint32_t val2 = __msa_copy_s_w((v4i32)R2, 0);
+ const uint32_t val3 = __msa_copy_s_w((v4i32)R3, 0);
+ SW4(val0, val1, val2, val3, dst, BPS);
+}
+
+static WEBP_INLINE void VR4(uint8_t* dst, const uint8_t* top) {
+ const int X = top[-1];
+ const int I = top[-2];
+ const int J = top[-3];
+ const int K = top[-4];
+ const int A = top[0];
+ const int B = top[1];
+ const int C = top[2];
+ const int D = top[3];
+ DST(0, 0) = DST(1, 2) = AVG2(X, A);
+ DST(1, 0) = DST(2, 2) = AVG2(A, B);
+ DST(2, 0) = DST(3, 2) = AVG2(B, C);
+ DST(3, 0) = AVG2(C, D);
+ DST(0, 3) = AVG3(K, J, I);
+ DST(0, 2) = AVG3(J, I, X);
+ DST(0, 1) = DST(1, 3) = AVG3(I, X, A);
+ DST(1, 1) = DST(2, 3) = AVG3(X, A, B);
+ DST(2, 1) = DST(3, 3) = AVG3(A, B, C);
+ DST(3, 1) = AVG3(B, C, D);
+}
+
+static WEBP_INLINE void VL4(uint8_t* dst, const uint8_t* top) {
+ const int A = top[0];
+ const int B = top[1];
+ const int C = top[2];
+ const int D = top[3];
+ const int E = top[4];
+ const int F = top[5];
+ const int G = top[6];
+ const int H = top[7];
+ DST(0, 0) = AVG2(A, B);
+ DST(1, 0) = DST(0, 2) = AVG2(B, C);
+ DST(2, 0) = DST(1, 2) = AVG2(C, D);
+ DST(3, 0) = DST(2, 2) = AVG2(D, E);
+ DST(0, 1) = AVG3(A, B, C);
+ DST(1, 1) = DST(0, 3) = AVG3(B, C, D);
+ DST(2, 1) = DST(1, 3) = AVG3(C, D, E);
+ DST(3, 1) = DST(2, 3) = AVG3(D, E, F);
+ DST(3, 2) = AVG3(E, F, G);
+ DST(3, 3) = AVG3(F, G, H);
+}
+
+static WEBP_INLINE void HU4(uint8_t* dst, const uint8_t* top) {
+ const int I = top[-2];
+ const int J = top[-3];
+ const int K = top[-4];
+ const int L = top[-5];
+ DST(0, 0) = AVG2(I, J);
+ DST(2, 0) = DST(0, 1) = AVG2(J, K);
+ DST(2, 1) = DST(0, 2) = AVG2(K, L);
+ DST(1, 0) = AVG3(I, J, K);
+ DST(3, 0) = DST(1, 1) = AVG3(J, K, L);
+ DST(3, 1) = DST(1, 2) = AVG3(K, L, L);
+ DST(3, 2) = DST(2, 2) =
+ DST(0, 3) = DST(1, 3) = DST(2, 3) = DST(3, 3) = L;
+}
+
+static WEBP_INLINE void HD4(uint8_t* dst, const uint8_t* top) {
+ const int X = top[-1];
+ const int I = top[-2];
+ const int J = top[-3];
+ const int K = top[-4];
+ const int L = top[-5];
+ const int A = top[0];
+ const int B = top[1];
+ const int C = top[2];
+ DST(0, 0) = DST(2, 1) = AVG2(I, X);
+ DST(0, 1) = DST(2, 2) = AVG2(J, I);
+ DST(0, 2) = DST(2, 3) = AVG2(K, J);
+ DST(0, 3) = AVG2(L, K);
+ DST(3, 0) = AVG3(A, B, C);
+ DST(2, 0) = AVG3(X, A, B);
+ DST(1, 0) = DST(3, 1) = AVG3(I, X, A);
+ DST(1, 1) = DST(3, 2) = AVG3(J, I, X);
+ DST(1, 2) = DST(3, 3) = AVG3(K, J, I);
+ DST(1, 3) = AVG3(L, K, J);
+}
+
+static WEBP_INLINE void TM4(uint8_t* dst, const uint8_t* top) {
+ const v16i8 zero = { 0 };
+ const v8i16 TL = (v8i16)__msa_fill_h(top[-1]);
+ const v8i16 L0 = (v8i16)__msa_fill_h(top[-2]);
+ const v8i16 L1 = (v8i16)__msa_fill_h(top[-3]);
+ const v8i16 L2 = (v8i16)__msa_fill_h(top[-4]);
+ const v8i16 L3 = (v8i16)__msa_fill_h(top[-5]);
+ const v16u8 T1 = LD_UB(top);
+ const v8i16 T = (v8i16)__msa_ilvr_b(zero, (v16i8)T1);
+ const v8i16 d = T - TL;
+ v8i16 r0, r1, r2, r3;
+ ADD4(d, L0, d, L1, d, L2, d, L3, r0, r1, r2, r3);
+ CLIP_SH4_0_255(r0, r1, r2, r3);
+ PCKEV_ST4x4_UB(r0, r1, r2, r3, dst, BPS);
+}
+
+#undef DST
+#undef AVG3
+#undef AVG2
+
+static void Intra4Preds(uint8_t* dst, const uint8_t* top) {
+ DC4(I4DC4 + dst, top);
+ TM4(I4TM4 + dst, top);
+ VE4(I4VE4 + dst, top);
+ HE4(I4HE4 + dst, top);
+ RD4(I4RD4 + dst, top);
+ VR4(I4VR4 + dst, top);
+ LD4(I4LD4 + dst, top);
+ VL4(I4VL4 + dst, top);
+ HD4(I4HD4 + dst, top);
+ HU4(I4HU4 + dst, top);
+}
+
+// luma 16x16 prediction
+
+#define STORE16x16(out, dst) do { \
+ ST_UB8(out, out, out, out, out, out, out, out, dst + 0 * BPS, BPS); \
+ ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS); \
+} while (0)
+
+static WEBP_INLINE void VerticalPred16x16(uint8_t* dst, const uint8_t* top) {
+ if (top != NULL) {
+ const v16u8 out = LD_UB(top);
+ STORE16x16(out, dst);
+ } else {
+ const v16u8 out = (v16u8)__msa_fill_b(0x7f);
+ STORE16x16(out, dst);
+ }
+}
+
+static WEBP_INLINE void HorizontalPred16x16(uint8_t* dst,
+ const uint8_t* left) {
+ if (left != NULL) {
+ int j;
+ for (j = 0; j < 16; j += 4) {
+ const v16u8 L0 = (v16u8)__msa_fill_b(left[0]);
+ const v16u8 L1 = (v16u8)__msa_fill_b(left[1]);
+ const v16u8 L2 = (v16u8)__msa_fill_b(left[2]);
+ const v16u8 L3 = (v16u8)__msa_fill_b(left[3]);
+ ST_UB4(L0, L1, L2, L3, dst, BPS);
+ dst += 4 * BPS;
+ left += 4;
+ }
+ } else {
+ const v16u8 out = (v16u8)__msa_fill_b(0x81);
+ STORE16x16(out, dst);
+ }
+}
+
+static WEBP_INLINE void TrueMotion16x16(uint8_t* dst, const uint8_t* left,
+ const uint8_t* top) {
+ if (left != NULL) {
+ if (top != NULL) {
+ int j;
+ v8i16 d1, d2;
+ const v16i8 zero = { 0 };
+ const v8i16 TL = (v8i16)__msa_fill_h(left[-1]);
+ const v16u8 T = LD_UB(top);
+ ILVRL_B2_SH(zero, T, d1, d2);
+ SUB2(d1, TL, d2, TL, d1, d2);
+ for (j = 0; j < 16; j += 4) {
+ v16i8 t0, t1, t2, t3;
+ v8i16 r0, r1, r2, r3, r4, r5, r6, r7;
+ const v8i16 L0 = (v8i16)__msa_fill_h(left[j + 0]);
+ const v8i16 L1 = (v8i16)__msa_fill_h(left[j + 1]);
+ const v8i16 L2 = (v8i16)__msa_fill_h(left[j + 2]);
+ const v8i16 L3 = (v8i16)__msa_fill_h(left[j + 3]);
+ ADD4(d1, L0, d1, L1, d1, L2, d1, L3, r0, r1, r2, r3);
+ ADD4(d2, L0, d2, L1, d2, L2, d2, L3, r4, r5, r6, r7);
+ CLIP_SH4_0_255(r0, r1, r2, r3);
+ CLIP_SH4_0_255(r4, r5, r6, r7);
+ PCKEV_B4_SB(r4, r0, r5, r1, r6, r2, r7, r3, t0, t1, t2, t3);
+ ST_SB4(t0, t1, t2, t3, dst, BPS);
+ dst += 4 * BPS;
+ }
+ } else {
+ HorizontalPred16x16(dst, left);
+ }
+ } else {
+ if (top != NULL) {
+ VerticalPred16x16(dst, top);
+ } else {
+ const v16u8 out = (v16u8)__msa_fill_b(0x81);
+ STORE16x16(out, dst);
+ }
+ }
+}
+
+static WEBP_INLINE void DCMode16x16(uint8_t* dst, const uint8_t* left,
+ const uint8_t* top) {
+ int DC;
+ v16u8 out;
+ if (top != NULL && left != NULL) {
+ const v16u8 rtop = LD_UB(top);
+ const v8u16 dctop = __msa_hadd_u_h(rtop, rtop);
+ const v16u8 rleft = LD_UB(left);
+ const v8u16 dcleft = __msa_hadd_u_h(rleft, rleft);
+ const v8u16 dctemp = dctop + dcleft;
+ DC = HADD_UH_U32(dctemp);
+ DC = (DC + 16) >> 5;
+ } else if (left != NULL) { // left but no top
+ const v16u8 rleft = LD_UB(left);
+ const v8u16 dcleft = __msa_hadd_u_h(rleft, rleft);
+ DC = HADD_UH_U32(dcleft);
+ DC = (DC + DC + 16) >> 5;
+ } else if (top != NULL) { // top but no left
+ const v16u8 rtop = LD_UB(top);
+ const v8u16 dctop = __msa_hadd_u_h(rtop, rtop);
+ DC = HADD_UH_U32(dctop);
+ DC = (DC + DC + 16) >> 5;
+ } else { // no top, no left, nothing.
+ DC = 0x80;
+ }
+ out = (v16u8)__msa_fill_b(DC);
+ STORE16x16(out, dst);
+}
+
+static void Intra16Preds(uint8_t* dst,
+ const uint8_t* left, const uint8_t* top) {
+ DCMode16x16(I16DC16 + dst, left, top);
+ VerticalPred16x16(I16VE16 + dst, top);
+ HorizontalPred16x16(I16HE16 + dst, left);
+ TrueMotion16x16(I16TM16 + dst, left, top);
+}
+
+// Chroma 8x8 prediction
+
+#define CALC_DC8(in, out) do { \
+ const v8u16 temp0 = __msa_hadd_u_h(in, in); \
+ const v4u32 temp1 = __msa_hadd_u_w(temp0, temp0); \
+ const v2i64 temp2 = (v2i64)__msa_hadd_u_d(temp1, temp1); \
+ const v2i64 temp3 = __msa_splati_d(temp2, 1); \
+ const v2i64 temp4 = temp3 + temp2; \
+ const v16i8 temp5 = (v16i8)__msa_srari_d(temp4, 4); \
+ const v2i64 temp6 = (v2i64)__msa_splati_b(temp5, 0); \
+ out = __msa_copy_s_d(temp6, 0); \
+} while (0)
+
+#define STORE8x8(out, dst) do { \
+ SD4(out, out, out, out, dst + 0 * BPS, BPS); \
+ SD4(out, out, out, out, dst + 4 * BPS, BPS); \
+} while (0)
+
+static WEBP_INLINE void VerticalPred8x8(uint8_t* dst, const uint8_t* top) {
+ if (top != NULL) {
+ const uint64_t out = LD(top);
+ STORE8x8(out, dst);
+ } else {
+ const uint64_t out = 0x7f7f7f7f7f7f7f7fULL;
+ STORE8x8(out, dst);
+ }
+}
+
+static WEBP_INLINE void HorizontalPred8x8(uint8_t* dst, const uint8_t* left) {
+ if (left != NULL) {
+ int j;
+ for (j = 0; j < 8; j += 4) {
+ const v16u8 L0 = (v16u8)__msa_fill_b(left[0]);
+ const v16u8 L1 = (v16u8)__msa_fill_b(left[1]);
+ const v16u8 L2 = (v16u8)__msa_fill_b(left[2]);
+ const v16u8 L3 = (v16u8)__msa_fill_b(left[3]);
+ const uint64_t out0 = __msa_copy_s_d((v2i64)L0, 0);
+ const uint64_t out1 = __msa_copy_s_d((v2i64)L1, 0);
+ const uint64_t out2 = __msa_copy_s_d((v2i64)L2, 0);
+ const uint64_t out3 = __msa_copy_s_d((v2i64)L3, 0);
+ SD4(out0, out1, out2, out3, dst, BPS);
+ dst += 4 * BPS;
+ left += 4;
+ }
+ } else {
+ const uint64_t out = 0x8181818181818181ULL;
+ STORE8x8(out, dst);
+ }
+}
+
+static WEBP_INLINE void TrueMotion8x8(uint8_t* dst, const uint8_t* left,
+ const uint8_t* top) {
+ if (left != NULL) {
+ if (top != NULL) {
+ int j;
+ const v8i16 TL = (v8i16)__msa_fill_h(left[-1]);
+ const v16u8 T1 = LD_UB(top);
+ const v16i8 zero = { 0 };
+ const v8i16 T = (v8i16)__msa_ilvr_b(zero, (v16i8)T1);
+ const v8i16 d = T - TL;
+ for (j = 0; j < 8; j += 4) {
+ uint64_t out0, out1, out2, out3;
+ v16i8 t0, t1;
+ v8i16 r0 = (v8i16)__msa_fill_h(left[j + 0]);
+ v8i16 r1 = (v8i16)__msa_fill_h(left[j + 1]);
+ v8i16 r2 = (v8i16)__msa_fill_h(left[j + 2]);
+ v8i16 r3 = (v8i16)__msa_fill_h(left[j + 3]);
+ ADD4(d, r0, d, r1, d, r2, d, r3, r0, r1, r2, r3);
+ CLIP_SH4_0_255(r0, r1, r2, r3);
+ PCKEV_B2_SB(r1, r0, r3, r2, t0, t1);
+ out0 = __msa_copy_s_d((v2i64)t0, 0);
+ out1 = __msa_copy_s_d((v2i64)t0, 1);
+ out2 = __msa_copy_s_d((v2i64)t1, 0);
+ out3 = __msa_copy_s_d((v2i64)t1, 1);
+ SD4(out0, out1, out2, out3, dst, BPS);
+ dst += 4 * BPS;
+ }
+ } else {
+ HorizontalPred8x8(dst, left);
+ }
+ } else {
+ if (top != NULL) {
+ VerticalPred8x8(dst, top);
+ } else {
+ const uint64_t out = 0x8181818181818181ULL;
+ STORE8x8(out, dst);
+ }
+ }
+}
+
+static WEBP_INLINE void DCMode8x8(uint8_t* dst, const uint8_t* left,
+ const uint8_t* top) {
+ uint64_t out;
+ v16u8 src;
+ if (top != NULL && left != NULL) {
+ const uint64_t left_m = LD(left);
+ const uint64_t top_m = LD(top);
+ INSERT_D2_UB(left_m, top_m, src);
+ CALC_DC8(src, out);
+ } else if (left != NULL) { // left but no top
+ const uint64_t left_m = LD(left);
+ INSERT_D2_UB(left_m, left_m, src);
+ CALC_DC8(src, out);
+ } else if (top != NULL) { // top but no left
+ const uint64_t top_m = LD(top);
+ INSERT_D2_UB(top_m, top_m, src);
+ CALC_DC8(src, out);
+ } else { // no top, no left, nothing.
+ src = (v16u8)__msa_fill_b(0x80);
+ out = __msa_copy_s_d((v2i64)src, 0);
+ }
+ STORE8x8(out, dst);
+}
+
+static void IntraChromaPreds(uint8_t* dst, const uint8_t* left,
+ const uint8_t* top) {
+ // U block
+ DCMode8x8(C8DC8 + dst, left, top);
+ VerticalPred8x8(C8VE8 + dst, top);
+ HorizontalPred8x8(C8HE8 + dst, left);
+ TrueMotion8x8(C8TM8 + dst, left, top);
+ // V block
+ dst += 8;
+ if (top != NULL) top += 8;
+ if (left != NULL) left += 16;
+ DCMode8x8(C8DC8 + dst, left, top);
+ VerticalPred8x8(C8VE8 + dst, top);
+ HorizontalPred8x8(C8HE8 + dst, left);
+ TrueMotion8x8(C8TM8 + dst, left, top);
+}
+
+//------------------------------------------------------------------------------
+// Metric
+
+#define PACK_DOTP_UB4_SW(in0, in1, in2, in3, out0, out1, out2, out3) do { \
+ v16u8 tmp0, tmp1; \
+ v8i16 tmp2, tmp3; \
+ ILVRL_B2_UB(in0, in1, tmp0, tmp1); \
+ HSUB_UB2_SH(tmp0, tmp1, tmp2, tmp3); \
+ DOTP_SH2_SW(tmp2, tmp3, tmp2, tmp3, out0, out1); \
+ ILVRL_B2_UB(in2, in3, tmp0, tmp1); \
+ HSUB_UB2_SH(tmp0, tmp1, tmp2, tmp3); \
+ DOTP_SH2_SW(tmp2, tmp3, tmp2, tmp3, out2, out3); \
+} while (0)
+
+#define PACK_DPADD_UB4_SW(in0, in1, in2, in3, out0, out1, out2, out3) do { \
+ v16u8 tmp0, tmp1; \
+ v8i16 tmp2, tmp3; \
+ ILVRL_B2_UB(in0, in1, tmp0, tmp1); \
+ HSUB_UB2_SH(tmp0, tmp1, tmp2, tmp3); \
+ DPADD_SH2_SW(tmp2, tmp3, tmp2, tmp3, out0, out1); \
+ ILVRL_B2_UB(in2, in3, tmp0, tmp1); \
+ HSUB_UB2_SH(tmp0, tmp1, tmp2, tmp3); \
+ DPADD_SH2_SW(tmp2, tmp3, tmp2, tmp3, out2, out3); \
+} while (0)
+
+static int SSE16x16(const uint8_t* a, const uint8_t* b) {
+ uint32_t sum;
+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
+ v16u8 ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7;
+ v4i32 out0, out1, out2, out3;
+
+ LD_UB8(a, BPS, src0, src1, src2, src3, src4, src5, src6, src7);
+ LD_UB8(b, BPS, ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7);
+ PACK_DOTP_UB4_SW(src0, ref0, src1, ref1, out0, out1, out2, out3);
+ PACK_DPADD_UB4_SW(src2, ref2, src3, ref3, out0, out1, out2, out3);
+ PACK_DPADD_UB4_SW(src4, ref4, src5, ref5, out0, out1, out2, out3);
+ PACK_DPADD_UB4_SW(src6, ref6, src7, ref7, out0, out1, out2, out3);
+ a += 8 * BPS;
+ b += 8 * BPS;
+ LD_UB8(a, BPS, src0, src1, src2, src3, src4, src5, src6, src7);
+ LD_UB8(b, BPS, ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7);
+ PACK_DPADD_UB4_SW(src0, ref0, src1, ref1, out0, out1, out2, out3);
+ PACK_DPADD_UB4_SW(src2, ref2, src3, ref3, out0, out1, out2, out3);
+ PACK_DPADD_UB4_SW(src4, ref4, src5, ref5, out0, out1, out2, out3);
+ PACK_DPADD_UB4_SW(src6, ref6, src7, ref7, out0, out1, out2, out3);
+ out0 += out1;
+ out2 += out3;
+ out0 += out2;
+ sum = HADD_SW_S32(out0);
+ return sum;
+}
+
+static int SSE16x8(const uint8_t* a, const uint8_t* b) {
+ uint32_t sum;
+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
+ v16u8 ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7;
+ v4i32 out0, out1, out2, out3;
+
+ LD_UB8(a, BPS, src0, src1, src2, src3, src4, src5, src6, src7);
+ LD_UB8(b, BPS, ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7);
+ PACK_DOTP_UB4_SW(src0, ref0, src1, ref1, out0, out1, out2, out3);
+ PACK_DPADD_UB4_SW(src2, ref2, src3, ref3, out0, out1, out2, out3);
+ PACK_DPADD_UB4_SW(src4, ref4, src5, ref5, out0, out1, out2, out3);
+ PACK_DPADD_UB4_SW(src6, ref6, src7, ref7, out0, out1, out2, out3);
+ out0 += out1;
+ out2 += out3;
+ out0 += out2;
+ sum = HADD_SW_S32(out0);
+ return sum;
+}
+
+static int SSE8x8(const uint8_t* a, const uint8_t* b) {
+ uint32_t sum;
+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
+ v16u8 ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7;
+ v16u8 t0, t1, t2, t3;
+ v4i32 out0, out1, out2, out3;
+
+ LD_UB8(a, BPS, src0, src1, src2, src3, src4, src5, src6, src7);
+ LD_UB8(b, BPS, ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7);
+ ILVR_B4_UB(src0, src1, src2, src3, ref0, ref1, ref2, ref3, t0, t1, t2, t3);
+ PACK_DOTP_UB4_SW(t0, t2, t1, t3, out0, out1, out2, out3);
+ ILVR_B4_UB(src4, src5, src6, src7, ref4, ref5, ref6, ref7, t0, t1, t2, t3);
+ PACK_DPADD_UB4_SW(t0, t2, t1, t3, out0, out1, out2, out3);
+ out0 += out1;
+ out2 += out3;
+ out0 += out2;
+ sum = HADD_SW_S32(out0);
+ return sum;
+}
+
+static int SSE4x4(const uint8_t* a, const uint8_t* b) {
+ uint32_t sum = 0;
+ uint32_t src0, src1, src2, src3, ref0, ref1, ref2, ref3;
+ v16u8 src, ref, tmp0, tmp1;
+ v8i16 diff0, diff1;
+ v4i32 out0, out1;
+
+ LW4(a, BPS, src0, src1, src2, src3);
+ LW4(b, BPS, ref0, ref1, ref2, ref3);
+ INSERT_W4_UB(src0, src1, src2, src3, src);
+ INSERT_W4_UB(ref0, ref1, ref2, ref3, ref);
+ ILVRL_B2_UB(src, ref, tmp0, tmp1);
+ HSUB_UB2_SH(tmp0, tmp1, diff0, diff1);
+ DOTP_SH2_SW(diff0, diff1, diff0, diff1, out0, out1);
+ out0 += out1;
+ sum = HADD_SW_S32(out0);
+ return sum;
+}
+
+//------------------------------------------------------------------------------
+// Quantization
+
+static int QuantizeBlock(int16_t in[16], int16_t out[16],
+ const VP8Matrix* const mtx) {
+ int sum;
+ v8i16 in0, in1, sh0, sh1, out0, out1;
+ v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, sign0, sign1;
+ v4i32 s0, s1, s2, s3, b0, b1, b2, b3, t0, t1, t2, t3;
+ const v8i16 zero = { 0 };
+ const v8i16 zigzag0 = { 0, 1, 4, 8, 5, 2, 3, 6 };
+ const v8i16 zigzag1 = { 9, 12, 13, 10, 7, 11, 14, 15 };
+ const v8i16 maxlevel = __msa_fill_h(MAX_LEVEL);
+
+ LD_SH2(&in[0], 8, in0, in1);
+ LD_SH2(&mtx->sharpen_[0], 8, sh0, sh1);
+ tmp4 = __msa_add_a_h(in0, zero);
+ tmp5 = __msa_add_a_h(in1, zero);
+ ILVRL_H2_SH(sh0, tmp4, tmp0, tmp1);
+ ILVRL_H2_SH(sh1, tmp5, tmp2, tmp3);
+ HADD_SH4_SW(tmp0, tmp1, tmp2, tmp3, s0, s1, s2, s3);
+ sign0 = (in0 < zero);
+ sign1 = (in1 < zero); // sign
+ LD_SH2(&mtx->iq_[0], 8, tmp0, tmp1); // iq
+ ILVRL_H2_SW(zero, tmp0, t0, t1);
+ ILVRL_H2_SW(zero, tmp1, t2, t3);
+ LD_SW4(&mtx->bias_[0], 4, b0, b1, b2, b3); // bias
+ MUL4(t0, s0, t1, s1, t2, s2, t3, s3, t0, t1, t2, t3);
+ ADD4(b0, t0, b1, t1, b2, t2, b3, t3, b0, b1, b2, b3);
+ SRAI_W4_SW(b0, b1, b2, b3, 17);
+ PCKEV_H2_SH(b1, b0, b3, b2, tmp2, tmp3);
+ tmp0 = (tmp2 > maxlevel);
+ tmp1 = (tmp3 > maxlevel);
+ tmp2 = (v8i16)__msa_bmnz_v((v16u8)tmp2, (v16u8)maxlevel, (v16u8)tmp0);
+ tmp3 = (v8i16)__msa_bmnz_v((v16u8)tmp3, (v16u8)maxlevel, (v16u8)tmp1);
+ SUB2(0, tmp2, 0, tmp3, tmp0, tmp1);
+ tmp2 = (v8i16)__msa_bmnz_v((v16u8)tmp2, (v16u8)tmp0, (v16u8)sign0);
+ tmp3 = (v8i16)__msa_bmnz_v((v16u8)tmp3, (v16u8)tmp1, (v16u8)sign1);
+ LD_SW4(&mtx->zthresh_[0], 4, t0, t1, t2, t3); // zthresh
+ t0 = (s0 > t0);
+ t1 = (s1 > t1);
+ t2 = (s2 > t2);
+ t3 = (s3 > t3);
+ PCKEV_H2_SH(t1, t0, t3, t2, tmp0, tmp1);
+ tmp4 = (v8i16)__msa_bmnz_v((v16u8)zero, (v16u8)tmp2, (v16u8)tmp0);
+ tmp5 = (v8i16)__msa_bmnz_v((v16u8)zero, (v16u8)tmp3, (v16u8)tmp1);
+ LD_SH2(&mtx->q_[0], 8, tmp0, tmp1);
+ MUL2(tmp4, tmp0, tmp5, tmp1, in0, in1);
+ VSHF_H2_SH(tmp4, tmp5, tmp4, tmp5, zigzag0, zigzag1, out0, out1);
+ ST_SH2(in0, in1, &in[0], 8);
+ ST_SH2(out0, out1, &out[0], 8);
+ out0 = __msa_add_a_h(out0, out1);
+ sum = HADD_SH_S32(out0);
+ return (sum > 0);
+}
+
+static int Quantize2Blocks(int16_t in[32], int16_t out[32],
+ const VP8Matrix* const mtx) {
+ int nz;
+ nz = VP8EncQuantizeBlock(in + 0 * 16, out + 0 * 16, mtx) << 0;
+ nz |= VP8EncQuantizeBlock(in + 1 * 16, out + 1 * 16, mtx) << 1;
+ return nz;
+}
+
+//------------------------------------------------------------------------------
+// Entry point
+
+extern void VP8EncDspInitMSA(void);
+
+WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitMSA(void) {
+ VP8ITransform = ITransform;
+ VP8FTransform = FTransform;
+ VP8FTransformWHT = FTransformWHT;
+
+ VP8TDisto4x4 = Disto4x4;
+ VP8TDisto16x16 = Disto16x16;
+ VP8CollectHistogram = CollectHistogram;
+
+ VP8EncPredLuma4 = Intra4Preds;
+ VP8EncPredLuma16 = Intra16Preds;
+ VP8EncPredChroma8 = IntraChromaPreds;
+
+ VP8SSE16x16 = SSE16x16;
+ VP8SSE16x8 = SSE16x8;
+ VP8SSE8x8 = SSE8x8;
+ VP8SSE4x4 = SSE4x4;
+
+ VP8EncQuantizeBlock = QuantizeBlock;
+ VP8EncQuantize2Blocks = Quantize2Blocks;
+ VP8EncQuantizeBlockWHT = QuantizeBlock;
+}
+
+#else // !WEBP_USE_MSA
+
+WEBP_DSP_INIT_STUB(VP8EncDspInitMSA)
+
+#endif // WEBP_USE_MSA
diff --git a/src/3rdparty/libwebp/src/dsp/enc_neon.c b/src/3rdparty/libwebp/src/dsp/enc_neon.c
index 46f6bf9..6a078d6 100644
--- a/src/3rdparty/libwebp/src/dsp/enc_neon.c
+++ b/src/3rdparty/libwebp/src/dsp/enc_neon.c
@@ -18,7 +18,7 @@
#include <assert.h>
#include "./neon.h"
-#include "../enc/vp8enci.h"
+#include "../enc/vp8i_enc.h"
//------------------------------------------------------------------------------
// Transforms (Paragraph 14.4)
@@ -746,9 +746,14 @@ static WEBP_INLINE void AccumulateSSE16(const uint8_t* const a,
const uint8x16_t a0 = vld1q_u8(a);
const uint8x16_t b0 = vld1q_u8(b);
const uint8x16_t abs_diff = vabdq_u8(a0, b0);
- uint16x8_t prod = vmull_u8(vget_low_u8(abs_diff), vget_low_u8(abs_diff));
- prod = vmlal_u8(prod, vget_high_u8(abs_diff), vget_high_u8(abs_diff));
- *sum = vpadalq_u16(*sum, prod); // pair-wise add and accumulate
+ const uint16x8_t prod1 = vmull_u8(vget_low_u8(abs_diff),
+ vget_low_u8(abs_diff));
+ const uint16x8_t prod2 = vmull_u8(vget_high_u8(abs_diff),
+ vget_high_u8(abs_diff));
+ /* pair-wise adds and widen */
+ const uint32x4_t sum1 = vpaddlq_u16(prod1);
+ const uint32x4_t sum2 = vpaddlq_u16(prod2);
+ *sum = vaddq_u32(*sum, vaddq_u32(sum1, sum2));
}
// Horizontal sum of all four uint32_t values in 'sum'.
@@ -758,7 +763,7 @@ static int SumToInt(uint32x4_t sum) {
return (int)sum3;
}
-static int SSE16x16(const uint8_t* a, const uint8_t* b) {
+static int SSE16x16_NEON(const uint8_t* a, const uint8_t* b) {
uint32x4_t sum = vdupq_n_u32(0);
int y;
for (y = 0; y < 16; ++y) {
@@ -767,7 +772,7 @@ static int SSE16x16(const uint8_t* a, const uint8_t* b) {
return SumToInt(sum);
}
-static int SSE16x8(const uint8_t* a, const uint8_t* b) {
+static int SSE16x8_NEON(const uint8_t* a, const uint8_t* b) {
uint32x4_t sum = vdupq_n_u32(0);
int y;
for (y = 0; y < 8; ++y) {
@@ -776,7 +781,7 @@ static int SSE16x8(const uint8_t* a, const uint8_t* b) {
return SumToInt(sum);
}
-static int SSE8x8(const uint8_t* a, const uint8_t* b) {
+static int SSE8x8_NEON(const uint8_t* a, const uint8_t* b) {
uint32x4_t sum = vdupq_n_u32(0);
int y;
for (y = 0; y < 8; ++y) {
@@ -789,13 +794,18 @@ static int SSE8x8(const uint8_t* a, const uint8_t* b) {
return SumToInt(sum);
}
-static int SSE4x4(const uint8_t* a, const uint8_t* b) {
+static int SSE4x4_NEON(const uint8_t* a, const uint8_t* b) {
const uint8x16_t a0 = Load4x4(a);
const uint8x16_t b0 = Load4x4(b);
const uint8x16_t abs_diff = vabdq_u8(a0, b0);
- uint16x8_t prod = vmull_u8(vget_low_u8(abs_diff), vget_low_u8(abs_diff));
- prod = vmlal_u8(prod, vget_high_u8(abs_diff), vget_high_u8(abs_diff));
- return SumToInt(vpaddlq_u16(prod));
+ const uint16x8_t prod1 = vmull_u8(vget_low_u8(abs_diff),
+ vget_low_u8(abs_diff));
+ const uint16x8_t prod2 = vmull_u8(vget_high_u8(abs_diff),
+ vget_high_u8(abs_diff));
+ /* pair-wise adds and widen */
+ const uint32x4_t sum1 = vpaddlq_u16(prod1);
+ const uint32x4_t sum2 = vpaddlq_u16(prod2);
+ return SumToInt(vaddq_u32(sum1, sum2));
}
//------------------------------------------------------------------------------
@@ -903,10 +913,12 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitNEON(void) {
VP8TDisto4x4 = Disto4x4;
VP8TDisto16x16 = Disto16x16;
VP8CollectHistogram = CollectHistogram;
- VP8SSE16x16 = SSE16x16;
- VP8SSE16x8 = SSE16x8;
- VP8SSE8x8 = SSE8x8;
- VP8SSE4x4 = SSE4x4;
+
+ VP8SSE16x16 = SSE16x16_NEON;
+ VP8SSE16x8 = SSE16x8_NEON;
+ VP8SSE8x8 = SSE8x8_NEON;
+ VP8SSE4x4 = SSE4x4_NEON;
+
#if !defined(WORK_AROUND_GCC)
VP8EncQuantizeBlock = QuantizeBlock;
VP8EncQuantize2Blocks = Quantize2Blocks;
diff --git a/src/3rdparty/libwebp/src/dsp/enc_sse2.c b/src/3rdparty/libwebp/src/dsp/enc_sse2.c
index 4a2e3ce..2026a74 100644
--- a/src/3rdparty/libwebp/src/dsp/enc_sse2.c
+++ b/src/3rdparty/libwebp/src/dsp/enc_sse2.c
@@ -14,12 +14,13 @@
#include "./dsp.h"
#if defined(WEBP_USE_SSE2)
+#include <assert.h>
#include <stdlib.h> // for abs()
#include <emmintrin.h>
#include "./common_sse2.h"
-#include "../enc/cost.h"
-#include "../enc/vp8enci.h"
+#include "../enc/cost_enc.h"
+#include "../enc/vp8i_enc.h"
//------------------------------------------------------------------------------
// Transforms (Paragraph 14.4)
@@ -139,7 +140,7 @@ static void ITransform(const uint8_t* ref, const int16_t* in, uint8_t* dst,
// Transpose the two 4x4.
VP8Transpose_2_4x4_16b(&shifted0, &shifted1, &shifted2, &shifted3, &T0, &T1,
- &T2, &T3);
+ &T2, &T3);
}
// Add inverse transform to 'ref' and store.
@@ -250,25 +251,11 @@ static void FTransformPass2(const __m128i* const v01, const __m128i* const v32,
const __m128i k51000 = _mm_set1_epi32(51000);
// Same operations are done on the (0,3) and (1,2) pairs.
- // a0 = v0 + v3
- // a1 = v1 + v2
// a3 = v0 - v3
// a2 = v1 - v2
- const __m128i a01 = _mm_add_epi16(*v01, *v32);
const __m128i a32 = _mm_sub_epi16(*v01, *v32);
- const __m128i a11 = _mm_unpackhi_epi64(a01, a01);
const __m128i a22 = _mm_unpackhi_epi64(a32, a32);
- const __m128i a01_plus_7 = _mm_add_epi16(a01, seven);
- // d0 = (a0 + a1 + 7) >> 4;
- // d2 = (a0 - a1 + 7) >> 4;
- const __m128i c0 = _mm_add_epi16(a01_plus_7, a11);
- const __m128i c2 = _mm_sub_epi16(a01_plus_7, a11);
- const __m128i d0 = _mm_srai_epi16(c0, 4);
- const __m128i d2 = _mm_srai_epi16(c2, 4);
-
- // f1 = ((b3 * 5352 + b2 * 2217 + 12000) >> 16)
- // f3 = ((b3 * 2217 - b2 * 5352 + 51000) >> 16)
const __m128i b23 = _mm_unpacklo_epi16(a22, a32);
const __m128i c1 = _mm_madd_epi16(b23, k5352_2217);
const __m128i c3 = _mm_madd_epi16(b23, k2217_5352);
@@ -276,14 +263,28 @@ static void FTransformPass2(const __m128i* const v01, const __m128i* const v32,
const __m128i d3 = _mm_add_epi32(c3, k51000);
const __m128i e1 = _mm_srai_epi32(d1, 16);
const __m128i e3 = _mm_srai_epi32(d3, 16);
+ // f1 = ((b3 * 5352 + b2 * 2217 + 12000) >> 16)
+ // f3 = ((b3 * 2217 - b2 * 5352 + 51000) >> 16)
const __m128i f1 = _mm_packs_epi32(e1, e1);
const __m128i f3 = _mm_packs_epi32(e3, e3);
- // f1 = f1 + (a3 != 0);
+ // g1 = f1 + (a3 != 0);
// The compare will return (0xffff, 0) for (==0, !=0). To turn that into the
// desired (0, 1), we add one earlier through k12000_plus_one.
- // -> f1 = f1 + 1 - (a3 == 0)
+ // -> g1 = f1 + 1 - (a3 == 0)
const __m128i g1 = _mm_add_epi16(f1, _mm_cmpeq_epi16(a32, zero));
+ // a0 = v0 + v3
+ // a1 = v1 + v2
+ const __m128i a01 = _mm_add_epi16(*v01, *v32);
+ const __m128i a01_plus_7 = _mm_add_epi16(a01, seven);
+ const __m128i a11 = _mm_unpackhi_epi64(a01, a01);
+ const __m128i c0 = _mm_add_epi16(a01_plus_7, a11);
+ const __m128i c2 = _mm_sub_epi16(a01_plus_7, a11);
+ // d0 = (a0 + a1 + 7) >> 4;
+ // d2 = (a0 - a1 + 7) >> 4;
+ const __m128i d0 = _mm_srai_epi16(c0, 4);
+ const __m128i d2 = _mm_srai_epi16(c2, 4);
+
const __m128i d0_g1 = _mm_unpacklo_epi64(d0, g1);
const __m128i d2_f3 = _mm_unpacklo_epi64(d2, f3);
_mm_storeu_si128((__m128i*)&out[0], d0_g1);
@@ -1046,6 +1047,37 @@ static int SSE4x4(const uint8_t* a, const uint8_t* b) {
}
//------------------------------------------------------------------------------
+
+static void Mean16x4(const uint8_t* ref, uint32_t dc[4]) {
+ const __m128i mask = _mm_set1_epi16(0x00ff);
+ const __m128i a0 = _mm_loadu_si128((const __m128i*)&ref[BPS * 0]);
+ const __m128i a1 = _mm_loadu_si128((const __m128i*)&ref[BPS * 1]);
+ const __m128i a2 = _mm_loadu_si128((const __m128i*)&ref[BPS * 2]);
+ const __m128i a3 = _mm_loadu_si128((const __m128i*)&ref[BPS * 3]);
+ const __m128i b0 = _mm_srli_epi16(a0, 8); // hi byte
+ const __m128i b1 = _mm_srli_epi16(a1, 8);
+ const __m128i b2 = _mm_srli_epi16(a2, 8);
+ const __m128i b3 = _mm_srli_epi16(a3, 8);
+ const __m128i c0 = _mm_and_si128(a0, mask); // lo byte
+ const __m128i c1 = _mm_and_si128(a1, mask);
+ const __m128i c2 = _mm_and_si128(a2, mask);
+ const __m128i c3 = _mm_and_si128(a3, mask);
+ const __m128i d0 = _mm_add_epi32(b0, c0);
+ const __m128i d1 = _mm_add_epi32(b1, c1);
+ const __m128i d2 = _mm_add_epi32(b2, c2);
+ const __m128i d3 = _mm_add_epi32(b3, c3);
+ const __m128i e0 = _mm_add_epi32(d0, d1);
+ const __m128i e1 = _mm_add_epi32(d2, d3);
+ const __m128i f0 = _mm_add_epi32(e0, e1);
+ uint16_t tmp[8];
+ _mm_storeu_si128((__m128i*)tmp, f0);
+ dc[0] = tmp[0] + tmp[1];
+ dc[1] = tmp[2] + tmp[3];
+ dc[2] = tmp[4] + tmp[5];
+ dc[3] = tmp[6] + tmp[7];
+}
+
+//------------------------------------------------------------------------------
// Texture distortion
//
// We try to match the spectral content (weighted) between source and
@@ -1331,10 +1363,122 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitSSE2(void) {
VP8SSE4x4 = SSE4x4;
VP8TDisto4x4 = Disto4x4;
VP8TDisto16x16 = Disto16x16;
+ VP8Mean16x4 = Mean16x4;
+}
+
+//------------------------------------------------------------------------------
+// SSIM / PSNR entry point (TODO(skal): move to its own file later)
+
+static uint32_t AccumulateSSE_SSE2(const uint8_t* src1,
+ const uint8_t* src2, int len) {
+ int i = 0;
+ uint32_t sse2 = 0;
+ if (len >= 16) {
+ const int limit = len - 32;
+ int32_t tmp[4];
+ __m128i sum1;
+ __m128i sum = _mm_setzero_si128();
+ __m128i a0 = _mm_loadu_si128((const __m128i*)&src1[i]);
+ __m128i b0 = _mm_loadu_si128((const __m128i*)&src2[i]);
+ i += 16;
+ while (i <= limit) {
+ const __m128i a1 = _mm_loadu_si128((const __m128i*)&src1[i]);
+ const __m128i b1 = _mm_loadu_si128((const __m128i*)&src2[i]);
+ __m128i sum2;
+ i += 16;
+ SubtractAndAccumulate(a0, b0, &sum1);
+ sum = _mm_add_epi32(sum, sum1);
+ a0 = _mm_loadu_si128((const __m128i*)&src1[i]);
+ b0 = _mm_loadu_si128((const __m128i*)&src2[i]);
+ i += 16;
+ SubtractAndAccumulate(a1, b1, &sum2);
+ sum = _mm_add_epi32(sum, sum2);
+ }
+ SubtractAndAccumulate(a0, b0, &sum1);
+ sum = _mm_add_epi32(sum, sum1);
+ _mm_storeu_si128((__m128i*)tmp, sum);
+ sse2 += (tmp[3] + tmp[2] + tmp[1] + tmp[0]);
+ }
+
+ for (; i < len; ++i) {
+ const int32_t diff = src1[i] - src2[i];
+ sse2 += diff * diff;
+ }
+ return sse2;
+}
+
+static uint32_t HorizontalAdd16b(const __m128i* const m) {
+ uint16_t tmp[8];
+ const __m128i a = _mm_srli_si128(*m, 8);
+ const __m128i b = _mm_add_epi16(*m, a);
+ _mm_storeu_si128((__m128i*)tmp, b);
+ return (uint32_t)tmp[3] + tmp[2] + tmp[1] + tmp[0];
+}
+
+static uint32_t HorizontalAdd32b(const __m128i* const m) {
+ const __m128i a = _mm_srli_si128(*m, 8);
+ const __m128i b = _mm_add_epi32(*m, a);
+ const __m128i c = _mm_add_epi32(b, _mm_srli_si128(b, 4));
+ return (uint32_t)_mm_cvtsi128_si32(c);
+}
+
+static const uint16_t kWeight[] = { 1, 2, 3, 4, 3, 2, 1, 0 };
+
+#define ACCUMULATE_ROW(WEIGHT) do { \
+ /* compute row weight (Wx * Wy) */ \
+ const __m128i Wy = _mm_set1_epi16((WEIGHT)); \
+ const __m128i W = _mm_mullo_epi16(Wx, Wy); \
+ /* process 8 bytes at a time (7 bytes, actually) */ \
+ const __m128i a0 = _mm_loadl_epi64((const __m128i*)src1); \
+ const __m128i b0 = _mm_loadl_epi64((const __m128i*)src2); \
+ /* convert to 16b and multiply by weight */ \
+ const __m128i a1 = _mm_unpacklo_epi8(a0, zero); \
+ const __m128i b1 = _mm_unpacklo_epi8(b0, zero); \
+ const __m128i wa1 = _mm_mullo_epi16(a1, W); \
+ const __m128i wb1 = _mm_mullo_epi16(b1, W); \
+ /* accumulate */ \
+ xm = _mm_add_epi16(xm, wa1); \
+ ym = _mm_add_epi16(ym, wb1); \
+ xxm = _mm_add_epi32(xxm, _mm_madd_epi16(a1, wa1)); \
+ xym = _mm_add_epi32(xym, _mm_madd_epi16(a1, wb1)); \
+ yym = _mm_add_epi32(yym, _mm_madd_epi16(b1, wb1)); \
+ src1 += stride1; \
+ src2 += stride2; \
+} while (0)
+
+static double SSIMGet_SSE2(const uint8_t* src1, int stride1,
+ const uint8_t* src2, int stride2) {
+ VP8DistoStats stats;
+ const __m128i zero = _mm_setzero_si128();
+ __m128i xm = zero, ym = zero; // 16b accums
+ __m128i xxm = zero, yym = zero, xym = zero; // 32b accum
+ const __m128i Wx = _mm_loadu_si128((const __m128i*)kWeight);
+ assert(2 * VP8_SSIM_KERNEL + 1 == 7);
+ ACCUMULATE_ROW(1);
+ ACCUMULATE_ROW(2);
+ ACCUMULATE_ROW(3);
+ ACCUMULATE_ROW(4);
+ ACCUMULATE_ROW(3);
+ ACCUMULATE_ROW(2);
+ ACCUMULATE_ROW(1);
+ stats.xm = HorizontalAdd16b(&xm);
+ stats.ym = HorizontalAdd16b(&ym);
+ stats.xxm = HorizontalAdd32b(&xxm);
+ stats.xym = HorizontalAdd32b(&xym);
+ stats.yym = HorizontalAdd32b(&yym);
+ return VP8SSIMFromStats(&stats);
+}
+
+extern void VP8SSIMDspInitSSE2(void);
+
+WEBP_TSAN_IGNORE_FUNCTION void VP8SSIMDspInitSSE2(void) {
+ VP8AccumulateSSE = AccumulateSSE_SSE2;
+ VP8SSIMGet = SSIMGet_SSE2;
}
#else // !WEBP_USE_SSE2
WEBP_DSP_INIT_STUB(VP8EncDspInitSSE2)
+WEBP_DSP_INIT_STUB(VP8SSIMDspInitSSE2)
#endif // WEBP_USE_SSE2
diff --git a/src/3rdparty/libwebp/src/dsp/enc_sse41.c b/src/3rdparty/libwebp/src/dsp/enc_sse41.c
index a178390..e32086d 100644
--- a/src/3rdparty/libwebp/src/dsp/enc_sse41.c
+++ b/src/3rdparty/libwebp/src/dsp/enc_sse41.c
@@ -18,7 +18,7 @@
#include <stdlib.h> // for abs()
#include "./common_sse2.h"
-#include "../enc/vp8enci.h"
+#include "../enc/vp8i_enc.h"
//------------------------------------------------------------------------------
// Compute susceptibility based on DCT-coeff histograms.
diff --git a/src/3rdparty/libwebp/src/dsp/filters.c b/src/3rdparty/libwebp/src/dsp/filters.c
index 9f04faf..65f34aa 100644
--- a/src/3rdparty/libwebp/src/dsp/filters.c
+++ b/src/3rdparty/libwebp/src/dsp/filters.c
@@ -227,6 +227,8 @@ WebPFilterFunc WebPFilters[WEBP_FILTER_LAST];
WebPUnfilterFunc WebPUnfilters[WEBP_FILTER_LAST];
extern void VP8FiltersInitMIPSdspR2(void);
+extern void VP8FiltersInitMSA(void);
+extern void VP8FiltersInitNEON(void);
extern void VP8FiltersInitSSE2(void);
static volatile VP8CPUInfo filters_last_cpuinfo_used =
@@ -251,11 +253,21 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInit(void) {
VP8FiltersInitSSE2();
}
#endif
+#if defined(WEBP_USE_NEON)
+ if (VP8GetCPUInfo(kNEON)) {
+ VP8FiltersInitNEON();
+ }
+#endif
#if defined(WEBP_USE_MIPS_DSP_R2)
if (VP8GetCPUInfo(kMIPSdspR2)) {
VP8FiltersInitMIPSdspR2();
}
#endif
+#if defined(WEBP_USE_MSA)
+ if (VP8GetCPUInfo(kMSA)) {
+ VP8FiltersInitMSA();
+ }
+#endif
}
filters_last_cpuinfo_used = VP8GetCPUInfo;
}
diff --git a/src/3rdparty/libwebp/src/dsp/filters_msa.c b/src/3rdparty/libwebp/src/dsp/filters_msa.c
new file mode 100644
index 0000000..4b8922d
--- /dev/null
+++ b/src/3rdparty/libwebp/src/dsp/filters_msa.c
@@ -0,0 +1,202 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the COPYING file in the root of the source
+// tree. An additional intellectual property rights grant can be found
+// in the file PATENTS. All contributing project authors may
+// be found in the AUTHORS file in the root of the source tree.
+// -----------------------------------------------------------------------------
+//
+// MSA variant of alpha filters
+//
+// Author: Prashant Patil (prashant.patil@imgtec.com)
+
+#include "./dsp.h"
+
+#if defined(WEBP_USE_MSA)
+
+#include "./msa_macro.h"
+
+#include <assert.h>
+
+static WEBP_INLINE void PredictLineInverse0(const uint8_t* src,
+ const uint8_t* pred,
+ uint8_t* dst, int length) {
+ v16u8 src0, pred0, dst0;
+ assert(length >= 0);
+ while (length >= 32) {
+ v16u8 src1, pred1, dst1;
+ LD_UB2(src, 16, src0, src1);
+ LD_UB2(pred, 16, pred0, pred1);
+ SUB2(src0, pred0, src1, pred1, dst0, dst1);
+ ST_UB2(dst0, dst1, dst, 16);
+ src += 32;
+ pred += 32;
+ dst += 32;
+ length -= 32;
+ }
+ if (length > 0) {
+ int i;
+ if (length >= 16) {
+ src0 = LD_UB(src);
+ pred0 = LD_UB(pred);
+ dst0 = src0 - pred0;
+ ST_UB(dst0, dst);
+ src += 16;
+ pred += 16;
+ dst += 16;
+ length -= 16;
+ }
+ for (i = 0; i < length; i++) {
+ dst[i] = src[i] - pred[i];
+ }
+ }
+}
+
+//------------------------------------------------------------------------------
+// Helpful macro.
+
+#define SANITY_CHECK(in, out) \
+ assert(in != NULL); \
+ assert(out != NULL); \
+ assert(width > 0); \
+ assert(height > 0); \
+ assert(stride >= width);
+
+//------------------------------------------------------------------------------
+// Horrizontal filter
+
+static void HorizontalFilter(const uint8_t* data, int width, int height,
+ int stride, uint8_t* filtered_data) {
+ const uint8_t* preds = data;
+ const uint8_t* in = data;
+ uint8_t* out = filtered_data;
+ int row = 1;
+ SANITY_CHECK(in, out);
+
+ // Leftmost pixel is the same as input for topmost scanline.
+ out[0] = in[0];
+ PredictLineInverse0(in + 1, preds, out + 1, width - 1);
+ preds += stride;
+ in += stride;
+ out += stride;
+ // Filter line-by-line.
+ while (row < height) {
+ // Leftmost pixel is predicted from above.
+ PredictLineInverse0(in, preds - stride, out, 1);
+ PredictLineInverse0(in + 1, preds, out + 1, width - 1);
+ ++row;
+ preds += stride;
+ in += stride;
+ out += stride;
+ }
+}
+
+//------------------------------------------------------------------------------
+// Gradient filter
+
+static WEBP_INLINE void PredictLineGradient(const uint8_t* pinput,
+ const uint8_t* ppred,
+ uint8_t* poutput, int stride,
+ int size) {
+ int w;
+ const v16i8 zero = { 0 };
+ while (size >= 16) {
+ v16u8 pred0, dst0;
+ v8i16 a0, a1, b0, b1, c0, c1;
+ const v16u8 tmp0 = LD_UB(ppred - 1);
+ const v16u8 tmp1 = LD_UB(ppred - stride);
+ const v16u8 tmp2 = LD_UB(ppred - stride - 1);
+ const v16u8 src0 = LD_UB(pinput);
+ ILVRL_B2_SH(zero, tmp0, a0, a1);
+ ILVRL_B2_SH(zero, tmp1, b0, b1);
+ ILVRL_B2_SH(zero, tmp2, c0, c1);
+ ADD2(a0, b0, a1, b1, a0, a1);
+ SUB2(a0, c0, a1, c1, a0, a1);
+ CLIP_SH2_0_255(a0, a1);
+ pred0 = (v16u8)__msa_pckev_b((v16i8)a1, (v16i8)a0);
+ dst0 = src0 - pred0;
+ ST_UB(dst0, poutput);
+ ppred += 16;
+ pinput += 16;
+ poutput += 16;
+ size -= 16;
+ }
+ for (w = 0; w < size; ++w) {
+ const int pred = ppred[w - 1] + ppred[w - stride] - ppred[w - stride - 1];
+ poutput[w] = pinput[w] - (pred < 0 ? 0 : pred > 255 ? 255 : pred);
+ }
+}
+
+
+static void GradientFilter(const uint8_t* data, int width, int height,
+ int stride, uint8_t* filtered_data) {
+ const uint8_t* in = data;
+ const uint8_t* preds = data;
+ uint8_t* out = filtered_data;
+ int row = 1;
+ SANITY_CHECK(in, out);
+
+ // left prediction for top scan-line
+ out[0] = in[0];
+ PredictLineInverse0(in + 1, preds, out + 1, width - 1);
+ preds += stride;
+ in += stride;
+ out += stride;
+ // Filter line-by-line.
+ while (row < height) {
+ out[0] = in[0] - preds[- stride];
+ PredictLineGradient(preds + 1, in + 1, out + 1, stride, width - 1);
+ ++row;
+ preds += stride;
+ in += stride;
+ out += stride;
+ }
+}
+
+//------------------------------------------------------------------------------
+// Vertical filter
+
+static void VerticalFilter(const uint8_t* data, int width, int height,
+ int stride, uint8_t* filtered_data) {
+ const uint8_t* in = data;
+ const uint8_t* preds = data;
+ uint8_t* out = filtered_data;
+ int row = 1;
+ SANITY_CHECK(in, out);
+
+ // Very first top-left pixel is copied.
+ out[0] = in[0];
+ // Rest of top scan-line is left-predicted.
+ PredictLineInverse0(in + 1, preds, out + 1, width - 1);
+ in += stride;
+ out += stride;
+
+ // Filter line-by-line.
+ while (row < height) {
+ PredictLineInverse0(in, preds, out, width);
+ ++row;
+ preds += stride;
+ in += stride;
+ out += stride;
+ }
+}
+
+#undef SANITY_CHECK
+
+//------------------------------------------------------------------------------
+// Entry point
+
+extern void VP8FiltersInitMSA(void);
+
+WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInitMSA(void) {
+ WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter;
+ WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter;
+ WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter;
+}
+
+#else // !WEBP_USE_MSA
+
+WEBP_DSP_INIT_STUB(VP8FiltersInitMSA)
+
+#endif // WEBP_USE_MSA
diff --git a/src/3rdparty/libwebp/src/dsp/filters_neon.c b/src/3rdparty/libwebp/src/dsp/filters_neon.c
new file mode 100644
index 0000000..4d6e50c
--- /dev/null
+++ b/src/3rdparty/libwebp/src/dsp/filters_neon.c
@@ -0,0 +1,327 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the COPYING file in the root of the source
+// tree. An additional intellectual property rights grant can be found
+// in the file PATENTS. All contributing project authors may
+// be found in the AUTHORS file in the root of the source tree.
+// -----------------------------------------------------------------------------
+//
+// NEON variant of alpha filters
+//
+// Author: Skal (pascal.massimino@gmail.com)
+
+#include "./dsp.h"
+
+#if defined(WEBP_USE_NEON)
+
+#include <assert.h>
+#include "./neon.h"
+
+//------------------------------------------------------------------------------
+// Helpful macros.
+
+# define SANITY_CHECK(in, out) \
+ assert(in != NULL); \
+ assert(out != NULL); \
+ assert(width > 0); \
+ assert(height > 0); \
+ assert(stride >= width); \
+ assert(row >= 0 && num_rows > 0 && row + num_rows <= height); \
+ (void)height; // Silence unused warning.
+
+// load eight u8 and widen to s16
+#define U8_TO_S16(A) vreinterpretq_s16_u16(vmovl_u8(A))
+#define LOAD_U8_TO_S16(A) U8_TO_S16(vld1_u8(A))
+
+// shift left or right by N byte, inserting zeros
+#define SHIFT_RIGHT_N_Q(A, N) vextq_u8((A), zero, (N))
+#define SHIFT_LEFT_N_Q(A, N) vextq_u8(zero, (A), (16 - (N)) % 16)
+
+// rotate left by N bytes
+#define ROTATE_LEFT_N(A, N) vext_u8((A), (A), (N))
+// rotate right by N bytes
+#define ROTATE_RIGHT_N(A, N) vext_u8((A), (A), (8 - (N)) % 8)
+
+static void PredictLine_NEON(const uint8_t* src, const uint8_t* pred,
+ uint8_t* dst, int length) {
+ int i;
+ assert(length >= 0);
+ for (i = 0; i + 16 <= length; i += 16) {
+ const uint8x16_t A = vld1q_u8(&src[i]);
+ const uint8x16_t B = vld1q_u8(&pred[i]);
+ const uint8x16_t C = vsubq_u8(A, B);
+ vst1q_u8(&dst[i], C);
+ }
+ for (; i < length; ++i) dst[i] = src[i] - pred[i];
+}
+
+// Special case for left-based prediction (when preds==dst-1 or preds==src-1).
+static void PredictLineLeft_NEON(const uint8_t* src, uint8_t* dst, int length) {
+ PredictLine_NEON(src, src - 1, dst, length);
+}
+
+//------------------------------------------------------------------------------
+// Horizontal filter.
+
+static WEBP_INLINE void DoHorizontalFilter_NEON(const uint8_t* in,
+ int width, int height,
+ int stride,
+ int row, int num_rows,
+ uint8_t* out) {
+ const size_t start_offset = row * stride;
+ const int last_row = row + num_rows;
+ SANITY_CHECK(in, out);
+ in += start_offset;
+ out += start_offset;
+
+ if (row == 0) {
+ // Leftmost pixel is the same as input for topmost scanline.
+ out[0] = in[0];
+ PredictLineLeft_NEON(in + 1, out + 1, width - 1);
+ row = 1;
+ in += stride;
+ out += stride;
+ }
+
+ // Filter line-by-line.
+ while (row < last_row) {
+ // Leftmost pixel is predicted from above.
+ out[0] = in[0] - in[-stride];
+ PredictLineLeft_NEON(in + 1, out + 1, width - 1);
+ ++row;
+ in += stride;
+ out += stride;
+ }
+}
+
+static void HorizontalFilter_NEON(const uint8_t* data, int width, int height,
+ int stride, uint8_t* filtered_data) {
+ DoHorizontalFilter_NEON(data, width, height, stride, 0, height,
+ filtered_data);
+}
+
+//------------------------------------------------------------------------------
+// Vertical filter.
+
+static WEBP_INLINE void DoVerticalFilter_NEON(const uint8_t* in,
+ int width, int height, int stride,
+ int row, int num_rows,
+ uint8_t* out) {
+ const size_t start_offset = row * stride;
+ const int last_row = row + num_rows;
+ SANITY_CHECK(in, out);
+ in += start_offset;
+ out += start_offset;
+
+ if (row == 0) {
+ // Very first top-left pixel is copied.
+ out[0] = in[0];
+ // Rest of top scan-line is left-predicted.
+ PredictLineLeft_NEON(in + 1, out + 1, width - 1);
+ row = 1;
+ in += stride;
+ out += stride;
+ }
+
+ // Filter line-by-line.
+ while (row < last_row) {
+ PredictLine_NEON(in, in - stride, out, width);
+ ++row;
+ in += stride;
+ out += stride;
+ }
+}
+
+static void VerticalFilter_NEON(const uint8_t* data, int width, int height,
+ int stride, uint8_t* filtered_data) {
+ DoVerticalFilter_NEON(data, width, height, stride, 0, height,
+ filtered_data);
+}
+
+//------------------------------------------------------------------------------
+// Gradient filter.
+
+static WEBP_INLINE int GradientPredictor_C(uint8_t a, uint8_t b, uint8_t c) {
+ const int g = a + b - c;
+ return ((g & ~0xff) == 0) ? g : (g < 0) ? 0 : 255; // clip to 8bit
+}
+
+static void GradientPredictDirect_NEON(const uint8_t* const row,
+ const uint8_t* const top,
+ uint8_t* const out, int length) {
+ int i;
+ for (i = 0; i + 8 <= length; i += 8) {
+ const uint8x8_t A = vld1_u8(&row[i - 1]);
+ const uint8x8_t B = vld1_u8(&top[i + 0]);
+ const int16x8_t C = vreinterpretq_s16_u16(vaddl_u8(A, B));
+ const int16x8_t D = LOAD_U8_TO_S16(&top[i - 1]);
+ const uint8x8_t E = vqmovun_s16(vsubq_s16(C, D));
+ const uint8x8_t F = vld1_u8(&row[i + 0]);
+ vst1_u8(&out[i], vsub_u8(F, E));
+ }
+ for (; i < length; ++i) {
+ out[i] = row[i] - GradientPredictor_C(row[i - 1], top[i], top[i - 1]);
+ }
+}
+
+static WEBP_INLINE void DoGradientFilter_NEON(const uint8_t* in,
+ int width, int height,
+ int stride,
+ int row, int num_rows,
+ uint8_t* out) {
+ const size_t start_offset = row * stride;
+ const int last_row = row + num_rows;
+ SANITY_CHECK(in, out);
+ in += start_offset;
+ out += start_offset;
+
+ // left prediction for top scan-line
+ if (row == 0) {
+ out[0] = in[0];
+ PredictLineLeft_NEON(in + 1, out + 1, width - 1);
+ row = 1;
+ in += stride;
+ out += stride;
+ }
+
+ // Filter line-by-line.
+ while (row < last_row) {
+ out[0] = in[0] - in[-stride];
+ GradientPredictDirect_NEON(in + 1, in + 1 - stride, out + 1, width - 1);
+ ++row;
+ in += stride;
+ out += stride;
+ }
+}
+
+static void GradientFilter_NEON(const uint8_t* data, int width, int height,
+ int stride, uint8_t* filtered_data) {
+ DoGradientFilter_NEON(data, width, height, stride, 0, height,
+ filtered_data);
+}
+
+#undef SANITY_CHECK
+
+//------------------------------------------------------------------------------
+// Inverse transforms
+
+static void HorizontalUnfilter_NEON(const uint8_t* prev, const uint8_t* in,
+ uint8_t* out, int width) {
+ int i;
+ const uint8x16_t zero = vdupq_n_u8(0);
+ uint8x16_t last;
+ out[0] = in[0] + (prev == NULL ? 0 : prev[0]);
+ if (width <= 1) return;
+ last = vsetq_lane_u8(out[0], zero, 0);
+ for (i = 1; i + 16 <= width; i += 16) {
+ const uint8x16_t A0 = vld1q_u8(&in[i]);
+ const uint8x16_t A1 = vaddq_u8(A0, last);
+ const uint8x16_t A2 = SHIFT_LEFT_N_Q(A1, 1);
+ const uint8x16_t A3 = vaddq_u8(A1, A2);
+ const uint8x16_t A4 = SHIFT_LEFT_N_Q(A3, 2);
+ const uint8x16_t A5 = vaddq_u8(A3, A4);
+ const uint8x16_t A6 = SHIFT_LEFT_N_Q(A5, 4);
+ const uint8x16_t A7 = vaddq_u8(A5, A6);
+ const uint8x16_t A8 = SHIFT_LEFT_N_Q(A7, 8);
+ const uint8x16_t A9 = vaddq_u8(A7, A8);
+ vst1q_u8(&out[i], A9);
+ last = SHIFT_RIGHT_N_Q(A9, 15);
+ }
+ for (; i < width; ++i) out[i] = in[i] + out[i - 1];
+}
+
+static void VerticalUnfilter_NEON(const uint8_t* prev, const uint8_t* in,
+ uint8_t* out, int width) {
+ if (prev == NULL) {
+ HorizontalUnfilter_NEON(NULL, in, out, width);
+ } else {
+ int i;
+ assert(width >= 0);
+ for (i = 0; i + 16 <= width; i += 16) {
+ const uint8x16_t A = vld1q_u8(&in[i]);
+ const uint8x16_t B = vld1q_u8(&prev[i]);
+ const uint8x16_t C = vaddq_u8(A, B);
+ vst1q_u8(&out[i], C);
+ }
+ for (; i < width; ++i) out[i] = in[i] + prev[i];
+ }
+}
+
+// GradientUnfilter_NEON is correct but slower than the C-version,
+// at least on ARM64. For armv7, it's a wash.
+// So best is to disable it for now, but keep the idea around...
+// #define USE_GRADIENT_UNFILTER
+
+#if defined(USE_GRADIENT_UNFILTER)
+#define GRAD_PROCESS_LANE(L) do { \
+ const uint8x8_t tmp1 = ROTATE_RIGHT_N(pred, 1); /* rotate predictor in */ \
+ const int16x8_t tmp2 = vaddq_s16(BC, U8_TO_S16(tmp1)); \
+ const uint8x8_t delta = vqmovun_s16(tmp2); \
+ pred = vadd_u8(D, delta); \
+ out = vext_u8(out, ROTATE_LEFT_N(pred, (L)), 1); \
+} while (0)
+
+static void GradientPredictInverse_NEON(const uint8_t* const in,
+ const uint8_t* const top,
+ uint8_t* const row, int length) {
+ if (length > 0) {
+ int i;
+ uint8x8_t pred = vdup_n_u8(row[-1]); // left sample
+ uint8x8_t out = vdup_n_u8(0);
+ for (i = 0; i + 8 <= length; i += 8) {
+ const int16x8_t B = LOAD_U8_TO_S16(&top[i + 0]);
+ const int16x8_t C = LOAD_U8_TO_S16(&top[i - 1]);
+ const int16x8_t BC = vsubq_s16(B, C); // unclipped gradient basis B - C
+ const uint8x8_t D = vld1_u8(&in[i]); // base input
+ GRAD_PROCESS_LANE(0);
+ GRAD_PROCESS_LANE(1);
+ GRAD_PROCESS_LANE(2);
+ GRAD_PROCESS_LANE(3);
+ GRAD_PROCESS_LANE(4);
+ GRAD_PROCESS_LANE(5);
+ GRAD_PROCESS_LANE(6);
+ GRAD_PROCESS_LANE(7);
+ vst1_u8(&row[i], out);
+ }
+ for (; i < length; ++i) {
+ row[i] = in[i] + GradientPredictor_C(row[i - 1], top[i], top[i - 1]);
+ }
+ }
+}
+#undef GRAD_PROCESS_LANE
+
+static void GradientUnfilter_NEON(const uint8_t* prev, const uint8_t* in,
+ uint8_t* out, int width) {
+ if (prev == NULL) {
+ HorizontalUnfilter_NEON(NULL, in, out, width);
+ } else {
+ out[0] = in[0] + prev[0]; // predict from above
+ GradientPredictInverse_NEON(in + 1, prev + 1, out + 1, width - 1);
+ }
+}
+
+#endif // USE_GRADIENT_UNFILTER
+
+//------------------------------------------------------------------------------
+// Entry point
+
+extern void VP8FiltersInitNEON(void);
+
+WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInitNEON(void) {
+ WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter_NEON;
+ WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter_NEON;
+#if defined(USE_GRADIENT_UNFILTER)
+ WebPUnfilters[WEBP_FILTER_GRADIENT] = GradientUnfilter_NEON;
+#endif
+
+ WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter_NEON;
+ WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter_NEON;
+ WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter_NEON;
+}
+
+#else // !WEBP_USE_NEON
+
+WEBP_DSP_INIT_STUB(VP8FiltersInitNEON)
+
+#endif // WEBP_USE_NEON
diff --git a/src/3rdparty/libwebp/src/dsp/lossless.c b/src/3rdparty/libwebp/src/dsp/lossless.c
index af913ef..20d18f6 100644
--- a/src/3rdparty/libwebp/src/dsp/lossless.c
+++ b/src/3rdparty/libwebp/src/dsp/lossless.c
@@ -17,20 +17,16 @@
#include <math.h>
#include <stdlib.h>
-#include "../dec/vp8li.h"
-#include "../utils/endian_inl.h"
+#include "../dec/vp8li_dec.h"
+#include "../utils/endian_inl_utils.h"
#include "./lossless.h"
+#include "./lossless_common.h"
#define MAX_DIFF_COST (1e30f)
//------------------------------------------------------------------------------
// Image transforms.
-// In-place sum of each component with mod 256.
-static WEBP_INLINE void AddPixelsEq(uint32_t* a, uint32_t b) {
- *a = VP8LAddPixels(*a, b);
-}
-
static WEBP_INLINE uint32_t Average2(uint32_t a0, uint32_t a1) {
return (((a0 ^ a1) & 0xfefefefeu) >> 1) + (a0 & a1);
}
@@ -171,21 +167,41 @@ static uint32_t Predictor13(uint32_t left, const uint32_t* const top) {
return pred;
}
+GENERATE_PREDICTOR_ADD(Predictor0, PredictorAdd0)
+static void PredictorAdd1(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ uint32_t left = out[-1];
+ for (i = 0; i < num_pixels; ++i) {
+ out[i] = left = VP8LAddPixels(in[i], left);
+ }
+ (void)upper;
+}
+GENERATE_PREDICTOR_ADD(Predictor2, PredictorAdd2)
+GENERATE_PREDICTOR_ADD(Predictor3, PredictorAdd3)
+GENERATE_PREDICTOR_ADD(Predictor4, PredictorAdd4)
+GENERATE_PREDICTOR_ADD(Predictor5, PredictorAdd5)
+GENERATE_PREDICTOR_ADD(Predictor6, PredictorAdd6)
+GENERATE_PREDICTOR_ADD(Predictor7, PredictorAdd7)
+GENERATE_PREDICTOR_ADD(Predictor8, PredictorAdd8)
+GENERATE_PREDICTOR_ADD(Predictor9, PredictorAdd9)
+GENERATE_PREDICTOR_ADD(Predictor10, PredictorAdd10)
+GENERATE_PREDICTOR_ADD(Predictor11, PredictorAdd11)
+GENERATE_PREDICTOR_ADD(Predictor12, PredictorAdd12)
+GENERATE_PREDICTOR_ADD(Predictor13, PredictorAdd13)
+
//------------------------------------------------------------------------------
// Inverse prediction.
static void PredictorInverseTransform(const VP8LTransform* const transform,
- int y_start, int y_end, uint32_t* data) {
+ int y_start, int y_end,
+ const uint32_t* in, uint32_t* out) {
const int width = transform->xsize_;
if (y_start == 0) { // First Row follows the L (mode=1) mode.
- int x;
- const uint32_t pred0 = Predictor0(data[-1], NULL);
- AddPixelsEq(data, pred0);
- for (x = 1; x < width; ++x) {
- const uint32_t pred1 = Predictor1(data[x - 1], NULL);
- AddPixelsEq(data + x, pred1);
- }
- data += width;
+ PredictorAdd0(in, NULL, 1, out);
+ PredictorAdd1(in + 1, NULL, width - 1, out + 1);
+ in += width;
+ out += width;
++y_start;
}
@@ -193,36 +209,26 @@ static void PredictorInverseTransform(const VP8LTransform* const transform,
int y = y_start;
const int tile_width = 1 << transform->bits_;
const int mask = tile_width - 1;
- const int safe_width = width & ~mask;
const int tiles_per_row = VP8LSubSampleSize(width, transform->bits_);
const uint32_t* pred_mode_base =
transform->data_ + (y >> transform->bits_) * tiles_per_row;
while (y < y_end) {
- const uint32_t pred2 = Predictor2(data[-1], data - width);
const uint32_t* pred_mode_src = pred_mode_base;
- VP8LPredictorFunc pred_func;
int x = 1;
- int t = 1;
// First pixel follows the T (mode=2) mode.
- AddPixelsEq(data, pred2);
+ PredictorAdd2(in, out - width, 1, out);
// .. the rest:
- while (x < safe_width) {
- pred_func = VP8LPredictors[((*pred_mode_src++) >> 8) & 0xf];
- for (; t < tile_width; ++t, ++x) {
- const uint32_t pred = pred_func(data[x - 1], data + x - width);
- AddPixelsEq(data + x, pred);
- }
- t = 0;
- }
- if (x < width) {
- pred_func = VP8LPredictors[((*pred_mode_src++) >> 8) & 0xf];
- for (; x < width; ++x) {
- const uint32_t pred = pred_func(data[x - 1], data + x - width);
- AddPixelsEq(data + x, pred);
- }
+ while (x < width) {
+ const VP8LPredictorAddSubFunc pred_func =
+ VP8LPredictorsAdd[((*pred_mode_src++) >> 8) & 0xf];
+ int x_end = (x & ~mask) + tile_width;
+ if (x_end > width) x_end = width;
+ pred_func(in + x, out + x - width, x_end - x, out + x);
+ x = x_end;
}
- data += width;
+ in += width;
+ out += width;
++y;
if ((y & mask) == 0) { // Use the same mask, since tiles are squares.
pred_mode_base += tiles_per_row;
@@ -233,21 +239,22 @@ static void PredictorInverseTransform(const VP8LTransform* const transform,
// Add green to blue and red channels (i.e. perform the inverse transform of
// 'subtract green').
-void VP8LAddGreenToBlueAndRed_C(uint32_t* data, int num_pixels) {
+void VP8LAddGreenToBlueAndRed_C(const uint32_t* src, int num_pixels,
+ uint32_t* dst) {
int i;
for (i = 0; i < num_pixels; ++i) {
- const uint32_t argb = data[i];
+ const uint32_t argb = src[i];
const uint32_t green = ((argb >> 8) & 0xff);
uint32_t red_blue = (argb & 0x00ff00ffu);
red_blue += (green << 16) | green;
red_blue &= 0x00ff00ffu;
- data[i] = (argb & 0xff00ff00u) | red_blue;
+ dst[i] = (argb & 0xff00ff00u) | red_blue;
}
}
-static WEBP_INLINE uint32_t ColorTransformDelta(int8_t color_pred,
- int8_t color) {
- return (uint32_t)((int)(color_pred) * color) >> 5;
+static WEBP_INLINE int ColorTransformDelta(int8_t color_pred,
+ int8_t color) {
+ return ((int)color_pred * color) >> 5;
}
static WEBP_INLINE void ColorCodeToMultipliers(uint32_t color_code,
@@ -257,27 +264,29 @@ static WEBP_INLINE void ColorCodeToMultipliers(uint32_t color_code,
m->red_to_blue_ = (color_code >> 16) & 0xff;
}
-void VP8LTransformColorInverse_C(const VP8LMultipliers* const m, uint32_t* data,
- int num_pixels) {
+void VP8LTransformColorInverse_C(const VP8LMultipliers* const m,
+ const uint32_t* src, int num_pixels,
+ uint32_t* dst) {
int i;
for (i = 0; i < num_pixels; ++i) {
- const uint32_t argb = data[i];
+ const uint32_t argb = src[i];
const uint32_t green = argb >> 8;
const uint32_t red = argb >> 16;
- uint32_t new_red = red;
- uint32_t new_blue = argb;
+ int new_red = red;
+ int new_blue = argb;
new_red += ColorTransformDelta(m->green_to_red_, green);
new_red &= 0xff;
new_blue += ColorTransformDelta(m->green_to_blue_, green);
new_blue += ColorTransformDelta(m->red_to_blue_, new_red);
new_blue &= 0xff;
- data[i] = (argb & 0xff00ff00u) | (new_red << 16) | (new_blue);
+ dst[i] = (argb & 0xff00ff00u) | (new_red << 16) | (new_blue);
}
}
// Color space inverse transform.
static void ColorSpaceInverseTransform(const VP8LTransform* const transform,
- int y_start, int y_end, uint32_t* data) {
+ int y_start, int y_end,
+ const uint32_t* src, uint32_t* dst) {
const int width = transform->xsize_;
const int tile_width = 1 << transform->bits_;
const int mask = tile_width - 1;
@@ -291,17 +300,19 @@ static void ColorSpaceInverseTransform(const VP8LTransform* const transform,
while (y < y_end) {
const uint32_t* pred = pred_row;
VP8LMultipliers m = { 0, 0, 0 };
- const uint32_t* const data_safe_end = data + safe_width;
- const uint32_t* const data_end = data + width;
- while (data < data_safe_end) {
+ const uint32_t* const src_safe_end = src + safe_width;
+ const uint32_t* const src_end = src + width;
+ while (src < src_safe_end) {
ColorCodeToMultipliers(*pred++, &m);
- VP8LTransformColorInverse(&m, data, tile_width);
- data += tile_width;
+ VP8LTransformColorInverse(&m, src, tile_width, dst);
+ src += tile_width;
+ dst += tile_width;
}
- if (data < data_end) { // Left-overs using C-version.
+ if (src < src_end) { // Left-overs using C-version.
ColorCodeToMultipliers(*pred++, &m);
- VP8LTransformColorInverse(&m, data, remaining_width);
- data += remaining_width;
+ VP8LTransformColorInverse(&m, src, remaining_width, dst);
+ src += remaining_width;
+ dst += remaining_width;
}
++y;
if ((y & mask) == 0) pred_row += tiles_per_row;
@@ -366,10 +377,10 @@ void VP8LInverseTransform(const VP8LTransform* const transform,
assert(row_end <= transform->ysize_);
switch (transform->type_) {
case SUBTRACT_GREEN:
- VP8LAddGreenToBlueAndRed(out, (row_end - row_start) * width);
+ VP8LAddGreenToBlueAndRed(in, (row_end - row_start) * width, out);
break;
case PREDICTOR_TRANSFORM:
- PredictorInverseTransform(transform, row_start, row_end, out);
+ PredictorInverseTransform(transform, row_start, row_end, in, out);
if (row_end != transform->ysize_) {
// The last predicted row in this iteration will be the top-pred row
// for the first row in next iteration.
@@ -378,7 +389,7 @@ void VP8LInverseTransform(const VP8LTransform* const transform,
}
break;
case CROSS_COLOR_TRANSFORM:
- ColorSpaceInverseTransform(transform, row_start, row_end, out);
+ ColorSpaceInverseTransform(transform, row_start, row_end, in, out);
break;
case COLOR_INDEXING_TRANSFORM:
if (in == out && transform->bits_ > 0) {
@@ -555,10 +566,15 @@ void VP8LConvertFromBGRA(const uint32_t* const in_data, int num_pixels,
//------------------------------------------------------------------------------
-VP8LProcessBlueAndRedFunc VP8LAddGreenToBlueAndRed;
+VP8LProcessDecBlueAndRedFunc VP8LAddGreenToBlueAndRed;
+VP8LPredictorAddSubFunc VP8LPredictorsAdd[16];
VP8LPredictorFunc VP8LPredictors[16];
-VP8LTransformColorFunc VP8LTransformColorInverse;
+// exposed plain-C implementations
+VP8LPredictorAddSubFunc VP8LPredictorsAdd_C[16];
+VP8LPredictorFunc VP8LPredictors_C[16];
+
+VP8LTransformColorInverseFunc VP8LTransformColorInverse;
VP8LConvertFunc VP8LConvertBGRAToRGB;
VP8LConvertFunc VP8LConvertBGRAToRGBA;
@@ -572,29 +588,37 @@ VP8LMapAlphaFunc VP8LMapColor8b;
extern void VP8LDspInitSSE2(void);
extern void VP8LDspInitNEON(void);
extern void VP8LDspInitMIPSdspR2(void);
+extern void VP8LDspInitMSA(void);
static volatile VP8CPUInfo lossless_last_cpuinfo_used =
(VP8CPUInfo)&lossless_last_cpuinfo_used;
+#define COPY_PREDICTOR_ARRAY(IN, OUT) do { \
+ (OUT)[0] = IN##0; \
+ (OUT)[1] = IN##1; \
+ (OUT)[2] = IN##2; \
+ (OUT)[3] = IN##3; \
+ (OUT)[4] = IN##4; \
+ (OUT)[5] = IN##5; \
+ (OUT)[6] = IN##6; \
+ (OUT)[7] = IN##7; \
+ (OUT)[8] = IN##8; \
+ (OUT)[9] = IN##9; \
+ (OUT)[10] = IN##10; \
+ (OUT)[11] = IN##11; \
+ (OUT)[12] = IN##12; \
+ (OUT)[13] = IN##13; \
+ (OUT)[14] = IN##0; /* <- padding security sentinels*/ \
+ (OUT)[15] = IN##0; \
+} while (0);
+
WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInit(void) {
if (lossless_last_cpuinfo_used == VP8GetCPUInfo) return;
- VP8LPredictors[0] = Predictor0;
- VP8LPredictors[1] = Predictor1;
- VP8LPredictors[2] = Predictor2;
- VP8LPredictors[3] = Predictor3;
- VP8LPredictors[4] = Predictor4;
- VP8LPredictors[5] = Predictor5;
- VP8LPredictors[6] = Predictor6;
- VP8LPredictors[7] = Predictor7;
- VP8LPredictors[8] = Predictor8;
- VP8LPredictors[9] = Predictor9;
- VP8LPredictors[10] = Predictor10;
- VP8LPredictors[11] = Predictor11;
- VP8LPredictors[12] = Predictor12;
- VP8LPredictors[13] = Predictor13;
- VP8LPredictors[14] = Predictor0; // <- padding security sentinels
- VP8LPredictors[15] = Predictor0;
+ COPY_PREDICTOR_ARRAY(Predictor, VP8LPredictors)
+ COPY_PREDICTOR_ARRAY(Predictor, VP8LPredictors_C)
+ COPY_PREDICTOR_ARRAY(PredictorAdd, VP8LPredictorsAdd)
+ COPY_PREDICTOR_ARRAY(PredictorAdd, VP8LPredictorsAdd_C)
VP8LAddGreenToBlueAndRed = VP8LAddGreenToBlueAndRed_C;
@@ -626,8 +650,14 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInit(void) {
VP8LDspInitMIPSdspR2();
}
#endif
+#if defined(WEBP_USE_MSA)
+ if (VP8GetCPUInfo(kMSA)) {
+ VP8LDspInitMSA();
+ }
+#endif
}
lossless_last_cpuinfo_used = VP8GetCPUInfo;
}
+#undef COPY_PREDICTOR_ARRAY
//------------------------------------------------------------------------------
diff --git a/src/3rdparty/libwebp/src/dsp/lossless.h b/src/3rdparty/libwebp/src/dsp/lossless.h
index 9f0d7a2..352a54e 100644
--- a/src/3rdparty/libwebp/src/dsp/lossless.h
+++ b/src/3rdparty/libwebp/src/dsp/lossless.h
@@ -18,7 +18,7 @@
#include "../webp/types.h"
#include "../webp/decode.h"
-#include "../enc/histogram.h"
+#include "../enc/histogram_enc.h"
#include "../utils/utils.h"
#ifdef __cplusplus
@@ -26,7 +26,7 @@ extern "C" {
#endif
#ifdef WEBP_EXPERIMENTAL_FEATURES
-#include "../enc/delta_palettization.h"
+#include "../enc/delta_palettization_enc.h"
#endif // WEBP_EXPERIMENTAL_FEATURES
//------------------------------------------------------------------------------
@@ -34,9 +34,17 @@ extern "C" {
typedef uint32_t (*VP8LPredictorFunc)(uint32_t left, const uint32_t* const top);
extern VP8LPredictorFunc VP8LPredictors[16];
-
-typedef void (*VP8LProcessBlueAndRedFunc)(uint32_t* argb_data, int num_pixels);
-extern VP8LProcessBlueAndRedFunc VP8LAddGreenToBlueAndRed;
+extern VP8LPredictorFunc VP8LPredictors_C[16];
+// These Add/Sub function expects upper[-1] and out[-1] to be readable.
+typedef void (*VP8LPredictorAddSubFunc)(const uint32_t* in,
+ const uint32_t* upper, int num_pixels,
+ uint32_t* out);
+extern VP8LPredictorAddSubFunc VP8LPredictorsAdd[16];
+extern VP8LPredictorAddSubFunc VP8LPredictorsAdd_C[16];
+
+typedef void (*VP8LProcessDecBlueAndRedFunc)(const uint32_t* src,
+ int num_pixels, uint32_t* dst);
+extern VP8LProcessDecBlueAndRedFunc VP8LAddGreenToBlueAndRed;
typedef struct {
// Note: the members are uint8_t, so that any negative values are
@@ -45,9 +53,10 @@ typedef struct {
uint8_t green_to_blue_;
uint8_t red_to_blue_;
} VP8LMultipliers;
-typedef void (*VP8LTransformColorFunc)(const VP8LMultipliers* const m,
- uint32_t* argb_data, int num_pixels);
-extern VP8LTransformColorFunc VP8LTransformColorInverse;
+typedef void (*VP8LTransformColorInverseFunc)(const VP8LMultipliers* const m,
+ const uint32_t* src,
+ int num_pixels, uint32_t* dst);
+extern VP8LTransformColorInverseFunc VP8LTransformColorInverse;
struct VP8LTransform; // Defined in dec/vp8li.h.
@@ -72,23 +81,6 @@ extern VP8LConvertFunc VP8LConvertBGRAToBGR;
void VP8LConvertFromBGRA(const uint32_t* const in_data, int num_pixels,
WEBP_CSP_MODE out_colorspace, uint8_t* const rgba);
-// color mapping related functions.
-static WEBP_INLINE uint32_t VP8GetARGBIndex(uint32_t idx) {
- return (idx >> 8) & 0xff;
-}
-
-static WEBP_INLINE uint8_t VP8GetAlphaIndex(uint8_t idx) {
- return idx;
-}
-
-static WEBP_INLINE uint32_t VP8GetARGBValue(uint32_t val) {
- return val;
-}
-
-static WEBP_INLINE uint8_t VP8GetAlphaValue(uint32_t val) {
- return (val >> 8) & 0xff;
-}
-
typedef void (*VP8LMapARGBFunc)(const uint32_t* src,
const uint32_t* const color_map,
uint32_t* dst, int y_start,
@@ -110,7 +102,8 @@ void VP8LColorIndexInverseTransformAlpha(
// Expose some C-only fallback functions
void VP8LTransformColorInverse_C(const VP8LMultipliers* const m,
- uint32_t* data, int num_pixels);
+ const uint32_t* src, int num_pixels,
+ uint32_t* dst);
void VP8LConvertBGRAToRGB_C(const uint32_t* src, int num_pixels, uint8_t* dst);
void VP8LConvertBGRAToRGBA_C(const uint32_t* src, int num_pixels, uint8_t* dst);
@@ -119,7 +112,8 @@ void VP8LConvertBGRAToRGBA4444_C(const uint32_t* src,
void VP8LConvertBGRAToRGB565_C(const uint32_t* src,
int num_pixels, uint8_t* dst);
void VP8LConvertBGRAToBGR_C(const uint32_t* src, int num_pixels, uint8_t* dst);
-void VP8LAddGreenToBlueAndRed_C(uint32_t* data, int num_pixels);
+void VP8LAddGreenToBlueAndRed_C(const uint32_t* src, int num_pixels,
+ uint32_t* dst);
// Must be called before calling any of the above methods.
void VP8LDspInit(void);
@@ -127,7 +121,10 @@ void VP8LDspInit(void);
//------------------------------------------------------------------------------
// Encoding
-extern VP8LProcessBlueAndRedFunc VP8LSubtractGreenFromBlueAndRed;
+typedef void (*VP8LProcessEncBlueAndRedFunc)(uint32_t* dst, int num_pixels);
+extern VP8LProcessEncBlueAndRedFunc VP8LSubtractGreenFromBlueAndRed;
+typedef void (*VP8LTransformColorFunc)(const VP8LMultipliers* const m,
+ uint32_t* const dst, int num_pixels);
extern VP8LTransformColorFunc VP8LTransformColor;
typedef void (*VP8LCollectColorBlueTransformsFunc)(
const uint32_t* argb, int stride,
@@ -153,62 +150,8 @@ void VP8LCollectColorBlueTransforms_C(const uint32_t* argb, int stride,
int green_to_blue, int red_to_blue,
int histo[]);
-//------------------------------------------------------------------------------
-// Image transforms.
-
-void VP8LResidualImage(int width, int height, int bits, int low_effort,
- uint32_t* const argb, uint32_t* const argb_scratch,
- uint32_t* const image, int near_lossless, int exact,
- int used_subtract_green);
-
-void VP8LColorSpaceTransform(int width, int height, int bits, int quality,
- uint32_t* const argb, uint32_t* image);
-
-//------------------------------------------------------------------------------
-// Misc methods.
-
-// Computes sampled size of 'size' when sampling using 'sampling bits'.
-static WEBP_INLINE uint32_t VP8LSubSampleSize(uint32_t size,
- uint32_t sampling_bits) {
- return (size + (1 << sampling_bits) - 1) >> sampling_bits;
-}
-
-// Converts near lossless quality into max number of bits shaved off.
-static WEBP_INLINE int VP8LNearLosslessBits(int near_lossless_quality) {
- // 100 -> 0
- // 80..99 -> 1
- // 60..79 -> 2
- // 40..59 -> 3
- // 20..39 -> 4
- // 0..19 -> 5
- return 5 - near_lossless_quality / 20;
-}
-
-// -----------------------------------------------------------------------------
-// Faster logarithm for integers. Small values use a look-up table.
-
-// The threshold till approximate version of log_2 can be used.
-// Practically, we can get rid of the call to log() as the two values match to
-// very high degree (the ratio of these two is 0.99999x).
-// Keeping a high threshold for now.
-#define APPROX_LOG_WITH_CORRECTION_MAX 65536
-#define APPROX_LOG_MAX 4096
-#define LOG_2_RECIPROCAL 1.44269504088896338700465094007086
-#define LOG_LOOKUP_IDX_MAX 256
-extern const float kLog2Table[LOG_LOOKUP_IDX_MAX];
-extern const float kSLog2Table[LOG_LOOKUP_IDX_MAX];
-typedef float (*VP8LFastLog2SlowFunc)(uint32_t v);
-
-extern VP8LFastLog2SlowFunc VP8LFastLog2Slow;
-extern VP8LFastLog2SlowFunc VP8LFastSLog2Slow;
-
-static WEBP_INLINE float VP8LFastLog2(uint32_t v) {
- return (v < LOG_LOOKUP_IDX_MAX) ? kLog2Table[v] : VP8LFastLog2Slow(v);
-}
-// Fast calculation of v * log2(v) for integer input.
-static WEBP_INLINE float VP8LFastSLog2(uint32_t v) {
- return (v < LOG_LOOKUP_IDX_MAX) ? kSLog2Table[v] : VP8LFastSLog2Slow(v);
-}
+extern VP8LPredictorAddSubFunc VP8LPredictorsSub[16];
+extern VP8LPredictorAddSubFunc VP8LPredictorsSub_C[16];
// -----------------------------------------------------------------------------
// Huffman-cost related functions.
@@ -228,11 +171,6 @@ typedef struct { // small struct to hold counters
int streaks[2][2]; // [zero/non-zero][streak<3 / streak>=3]
} VP8LStreaks;
-typedef VP8LStreaks (*VP8LCostCombinedCountFunc)(const uint32_t* X,
- const uint32_t* Y, int length);
-
-extern VP8LCostCombinedCountFunc VP8LHuffmanCostCombinedCount;
-
typedef struct { // small struct to hold bit entropy results
double entropy; // entropy
uint32_t sum; // sum of the population
@@ -246,26 +184,20 @@ void VP8LBitEntropyInit(VP8LBitEntropy* const entropy);
// Get the combined symbol bit entropy and Huffman cost stats for the
// distributions 'X' and 'Y'. Those results can then be refined according to
// codec specific heuristics.
-void VP8LGetCombinedEntropyUnrefined(const uint32_t* const X,
- const uint32_t* const Y, int length,
- VP8LBitEntropy* const bit_entropy,
- VP8LStreaks* const stats);
+typedef void (*VP8LGetCombinedEntropyUnrefinedFunc)(
+ const uint32_t X[], const uint32_t Y[], int length,
+ VP8LBitEntropy* const bit_entropy, VP8LStreaks* const stats);
+extern VP8LGetCombinedEntropyUnrefinedFunc VP8LGetCombinedEntropyUnrefined;
+
// Get the entropy for the distribution 'X'.
-void VP8LGetEntropyUnrefined(const uint32_t* const X, int length,
- VP8LBitEntropy* const bit_entropy,
- VP8LStreaks* const stats);
+typedef void (*VP8LGetEntropyUnrefinedFunc)(const uint32_t X[], int length,
+ VP8LBitEntropy* const bit_entropy,
+ VP8LStreaks* const stats);
+extern VP8LGetEntropyUnrefinedFunc VP8LGetEntropyUnrefined;
void VP8LBitsEntropyUnrefined(const uint32_t* const array, int n,
VP8LBitEntropy* const entropy);
-typedef void (*GetEntropyUnrefinedHelperFunc)(uint32_t val, int i,
- uint32_t* const val_prev,
- int* const i_prev,
- VP8LBitEntropy* const bit_entropy,
- VP8LStreaks* const stats);
-// Internal function used by VP8LGet*EntropyUnrefined.
-extern GetEntropyUnrefinedHelperFunc VP8LGetEntropyUnrefinedHelper;
-
typedef void (*VP8LHistogramAddFunc)(const VP8LHistogram* const a,
const VP8LHistogram* const b,
VP8LHistogram* const out);
@@ -279,86 +211,11 @@ typedef int (*VP8LVectorMismatchFunc)(const uint32_t* const array1,
// Returns the first index where array1 and array2 are different.
extern VP8LVectorMismatchFunc VP8LVectorMismatch;
-static WEBP_INLINE int VP8LBitsLog2Ceiling(uint32_t n) {
- const int log_floor = BitsLog2Floor(n);
- if (n == (n & ~(n - 1))) // zero or a power of two.
- return log_floor;
- else
- return log_floor + 1;
-}
-
-// Splitting of distance and length codes into prefixes and
-// extra bits. The prefixes are encoded with an entropy code
-// while the extra bits are stored just as normal bits.
-static WEBP_INLINE void VP8LPrefixEncodeBitsNoLUT(int distance, int* const code,
- int* const extra_bits) {
- const int highest_bit = BitsLog2Floor(--distance);
- const int second_highest_bit = (distance >> (highest_bit - 1)) & 1;
- *extra_bits = highest_bit - 1;
- *code = 2 * highest_bit + second_highest_bit;
-}
-
-static WEBP_INLINE void VP8LPrefixEncodeNoLUT(int distance, int* const code,
- int* const extra_bits,
- int* const extra_bits_value) {
- const int highest_bit = BitsLog2Floor(--distance);
- const int second_highest_bit = (distance >> (highest_bit - 1)) & 1;
- *extra_bits = highest_bit - 1;
- *extra_bits_value = distance & ((1 << *extra_bits) - 1);
- *code = 2 * highest_bit + second_highest_bit;
-}
-
-#define PREFIX_LOOKUP_IDX_MAX 512
-typedef struct {
- int8_t code_;
- int8_t extra_bits_;
-} VP8LPrefixCode;
-
-// These tables are derived using VP8LPrefixEncodeNoLUT.
-extern const VP8LPrefixCode kPrefixEncodeCode[PREFIX_LOOKUP_IDX_MAX];
-extern const uint8_t kPrefixEncodeExtraBitsValue[PREFIX_LOOKUP_IDX_MAX];
-static WEBP_INLINE void VP8LPrefixEncodeBits(int distance, int* const code,
- int* const extra_bits) {
- if (distance < PREFIX_LOOKUP_IDX_MAX) {
- const VP8LPrefixCode prefix_code = kPrefixEncodeCode[distance];
- *code = prefix_code.code_;
- *extra_bits = prefix_code.extra_bits_;
- } else {
- VP8LPrefixEncodeBitsNoLUT(distance, code, extra_bits);
- }
-}
-
-static WEBP_INLINE void VP8LPrefixEncode(int distance, int* const code,
- int* const extra_bits,
- int* const extra_bits_value) {
- if (distance < PREFIX_LOOKUP_IDX_MAX) {
- const VP8LPrefixCode prefix_code = kPrefixEncodeCode[distance];
- *code = prefix_code.code_;
- *extra_bits = prefix_code.extra_bits_;
- *extra_bits_value = kPrefixEncodeExtraBitsValue[distance];
- } else {
- VP8LPrefixEncodeNoLUT(distance, code, extra_bits, extra_bits_value);
- }
-}
-
-// Sum of each component, mod 256.
-static WEBP_INLINE uint32_t VP8LAddPixels(uint32_t a, uint32_t b) {
- const uint32_t alpha_and_green = (a & 0xff00ff00u) + (b & 0xff00ff00u);
- const uint32_t red_and_blue = (a & 0x00ff00ffu) + (b & 0x00ff00ffu);
- return (alpha_and_green & 0xff00ff00u) | (red_and_blue & 0x00ff00ffu);
-}
-
-// Difference of each component, mod 256.
-static WEBP_INLINE uint32_t VP8LSubPixels(uint32_t a, uint32_t b) {
- const uint32_t alpha_and_green =
- 0x00ff00ffu + (a & 0xff00ff00u) - (b & 0xff00ff00u);
- const uint32_t red_and_blue =
- 0xff00ff00u + (a & 0x00ff00ffu) - (b & 0x00ff00ffu);
- return (alpha_and_green & 0xff00ff00u) | (red_and_blue & 0x00ff00ffu);
-}
-
-void VP8LBundleColorMap(const uint8_t* const row, int width,
- int xbits, uint32_t* const dst);
+typedef void (*VP8LBundleColorMapFunc)(const uint8_t* const row, int width,
+ int xbits, uint32_t* dst);
+extern VP8LBundleColorMapFunc VP8LBundleColorMap;
+void VP8LBundleColorMap_C(const uint8_t* const row, int width, int xbits,
+ uint32_t* dst);
// Must be called before calling any of the above methods.
void VP8LEncDspInit(void);
diff --git a/src/3rdparty/libwebp/src/dsp/lossless_common.h b/src/3rdparty/libwebp/src/dsp/lossless_common.h
new file mode 100644
index 0000000..c40f711
--- /dev/null
+++ b/src/3rdparty/libwebp/src/dsp/lossless_common.h
@@ -0,0 +1,210 @@
+// Copyright 2012 Google Inc. All Rights Reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the COPYING file in the root of the source
+// tree. An additional intellectual property rights grant can be found
+// in the file PATENTS. All contributing project authors may
+// be found in the AUTHORS file in the root of the source tree.
+// -----------------------------------------------------------------------------
+//
+// Image transforms and color space conversion methods for lossless decoder.
+//
+// Authors: Vikas Arora (vikaas.arora@gmail.com)
+// Jyrki Alakuijala (jyrki@google.com)
+// Vincent Rabaud (vrabaud@google.com)
+
+#ifndef WEBP_DSP_LOSSLESS_COMMON_H_
+#define WEBP_DSP_LOSSLESS_COMMON_H_
+
+#include "../webp/types.h"
+
+#include "../utils/utils.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//------------------------------------------------------------------------------
+// Decoding
+
+// color mapping related functions.
+static WEBP_INLINE uint32_t VP8GetARGBIndex(uint32_t idx) {
+ return (idx >> 8) & 0xff;
+}
+
+static WEBP_INLINE uint8_t VP8GetAlphaIndex(uint8_t idx) {
+ return idx;
+}
+
+static WEBP_INLINE uint32_t VP8GetARGBValue(uint32_t val) {
+ return val;
+}
+
+static WEBP_INLINE uint8_t VP8GetAlphaValue(uint32_t val) {
+ return (val >> 8) & 0xff;
+}
+
+//------------------------------------------------------------------------------
+// Misc methods.
+
+// Computes sampled size of 'size' when sampling using 'sampling bits'.
+static WEBP_INLINE uint32_t VP8LSubSampleSize(uint32_t size,
+ uint32_t sampling_bits) {
+ return (size + (1 << sampling_bits) - 1) >> sampling_bits;
+}
+
+// Converts near lossless quality into max number of bits shaved off.
+static WEBP_INLINE int VP8LNearLosslessBits(int near_lossless_quality) {
+ // 100 -> 0
+ // 80..99 -> 1
+ // 60..79 -> 2
+ // 40..59 -> 3
+ // 20..39 -> 4
+ // 0..19 -> 5
+ return 5 - near_lossless_quality / 20;
+}
+
+// -----------------------------------------------------------------------------
+// Faster logarithm for integers. Small values use a look-up table.
+
+// The threshold till approximate version of log_2 can be used.
+// Practically, we can get rid of the call to log() as the two values match to
+// very high degree (the ratio of these two is 0.99999x).
+// Keeping a high threshold for now.
+#define APPROX_LOG_WITH_CORRECTION_MAX 65536
+#define APPROX_LOG_MAX 4096
+#define LOG_2_RECIPROCAL 1.44269504088896338700465094007086
+#define LOG_LOOKUP_IDX_MAX 256
+extern const float kLog2Table[LOG_LOOKUP_IDX_MAX];
+extern const float kSLog2Table[LOG_LOOKUP_IDX_MAX];
+typedef float (*VP8LFastLog2SlowFunc)(uint32_t v);
+
+extern VP8LFastLog2SlowFunc VP8LFastLog2Slow;
+extern VP8LFastLog2SlowFunc VP8LFastSLog2Slow;
+
+static WEBP_INLINE float VP8LFastLog2(uint32_t v) {
+ return (v < LOG_LOOKUP_IDX_MAX) ? kLog2Table[v] : VP8LFastLog2Slow(v);
+}
+// Fast calculation of v * log2(v) for integer input.
+static WEBP_INLINE float VP8LFastSLog2(uint32_t v) {
+ return (v < LOG_LOOKUP_IDX_MAX) ? kSLog2Table[v] : VP8LFastSLog2Slow(v);
+}
+
+// -----------------------------------------------------------------------------
+// PrefixEncode()
+
+static WEBP_INLINE int VP8LBitsLog2Ceiling(uint32_t n) {
+ const int log_floor = BitsLog2Floor(n);
+ if (n == (n & ~(n - 1))) { // zero or a power of two.
+ return log_floor;
+ }
+ return log_floor + 1;
+}
+
+// Splitting of distance and length codes into prefixes and
+// extra bits. The prefixes are encoded with an entropy code
+// while the extra bits are stored just as normal bits.
+static WEBP_INLINE void VP8LPrefixEncodeBitsNoLUT(int distance, int* const code,
+ int* const extra_bits) {
+ const int highest_bit = BitsLog2Floor(--distance);
+ const int second_highest_bit = (distance >> (highest_bit - 1)) & 1;
+ *extra_bits = highest_bit - 1;
+ *code = 2 * highest_bit + second_highest_bit;
+}
+
+static WEBP_INLINE void VP8LPrefixEncodeNoLUT(int distance, int* const code,
+ int* const extra_bits,
+ int* const extra_bits_value) {
+ const int highest_bit = BitsLog2Floor(--distance);
+ const int second_highest_bit = (distance >> (highest_bit - 1)) & 1;
+ *extra_bits = highest_bit - 1;
+ *extra_bits_value = distance & ((1 << *extra_bits) - 1);
+ *code = 2 * highest_bit + second_highest_bit;
+}
+
+#define PREFIX_LOOKUP_IDX_MAX 512
+typedef struct {
+ int8_t code_;
+ int8_t extra_bits_;
+} VP8LPrefixCode;
+
+// These tables are derived using VP8LPrefixEncodeNoLUT.
+extern const VP8LPrefixCode kPrefixEncodeCode[PREFIX_LOOKUP_IDX_MAX];
+extern const uint8_t kPrefixEncodeExtraBitsValue[PREFIX_LOOKUP_IDX_MAX];
+static WEBP_INLINE void VP8LPrefixEncodeBits(int distance, int* const code,
+ int* const extra_bits) {
+ if (distance < PREFIX_LOOKUP_IDX_MAX) {
+ const VP8LPrefixCode prefix_code = kPrefixEncodeCode[distance];
+ *code = prefix_code.code_;
+ *extra_bits = prefix_code.extra_bits_;
+ } else {
+ VP8LPrefixEncodeBitsNoLUT(distance, code, extra_bits);
+ }
+}
+
+static WEBP_INLINE void VP8LPrefixEncode(int distance, int* const code,
+ int* const extra_bits,
+ int* const extra_bits_value) {
+ if (distance < PREFIX_LOOKUP_IDX_MAX) {
+ const VP8LPrefixCode prefix_code = kPrefixEncodeCode[distance];
+ *code = prefix_code.code_;
+ *extra_bits = prefix_code.extra_bits_;
+ *extra_bits_value = kPrefixEncodeExtraBitsValue[distance];
+ } else {
+ VP8LPrefixEncodeNoLUT(distance, code, extra_bits, extra_bits_value);
+ }
+}
+
+// Sum of each component, mod 256.
+static WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW WEBP_INLINE
+uint32_t VP8LAddPixels(uint32_t a, uint32_t b) {
+ const uint32_t alpha_and_green = (a & 0xff00ff00u) + (b & 0xff00ff00u);
+ const uint32_t red_and_blue = (a & 0x00ff00ffu) + (b & 0x00ff00ffu);
+ return (alpha_and_green & 0xff00ff00u) | (red_and_blue & 0x00ff00ffu);
+}
+
+// Difference of each component, mod 256.
+static WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW WEBP_INLINE
+uint32_t VP8LSubPixels(uint32_t a, uint32_t b) {
+ const uint32_t alpha_and_green =
+ 0x00ff00ffu + (a & 0xff00ff00u) - (b & 0xff00ff00u);
+ const uint32_t red_and_blue =
+ 0xff00ff00u + (a & 0x00ff00ffu) - (b & 0x00ff00ffu);
+ return (alpha_and_green & 0xff00ff00u) | (red_and_blue & 0x00ff00ffu);
+}
+
+//------------------------------------------------------------------------------
+// Transform-related functions use din both encoding and decoding.
+
+// Macros used to create a batch predictor that iteratively uses a
+// one-pixel predictor.
+
+// The predictor is added to the output pixel (which
+// is therefore considered as a residual) to get the final prediction.
+#define GENERATE_PREDICTOR_ADD(PREDICTOR, PREDICTOR_ADD) \
+static void PREDICTOR_ADD(const uint32_t* in, const uint32_t* upper, \
+ int num_pixels, uint32_t* out) { \
+ int x; \
+ for (x = 0; x < num_pixels; ++x) { \
+ const uint32_t pred = (PREDICTOR)(out[x - 1], upper + x); \
+ out[x] = VP8LAddPixels(in[x], pred); \
+ } \
+}
+
+// It subtracts the prediction from the input pixel and stores the residual
+// in the output pixel.
+#define GENERATE_PREDICTOR_SUB(PREDICTOR, PREDICTOR_SUB) \
+static void PREDICTOR_SUB(const uint32_t* in, const uint32_t* upper, \
+ int num_pixels, uint32_t* out) { \
+ int x; \
+ for (x = 0; x < num_pixels; ++x) { \
+ const uint32_t pred = (PREDICTOR)(in[x - 1], upper + x); \
+ out[x] = VP8LSubPixels(in[x], pred); \
+ } \
+}
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // WEBP_DSP_LOSSLESS_COMMON_H_
diff --git a/src/3rdparty/libwebp/src/dsp/lossless_enc.c b/src/3rdparty/libwebp/src/dsp/lossless_enc.c
index 256f6f5..4e46fba 100644
--- a/src/3rdparty/libwebp/src/dsp/lossless_enc.c
+++ b/src/3rdparty/libwebp/src/dsp/lossless_enc.c
@@ -17,16 +17,12 @@
#include <math.h>
#include <stdlib.h>
-#include "../dec/vp8li.h"
-#include "../utils/endian_inl.h"
+#include "../dec/vp8li_dec.h"
+#include "../utils/endian_inl_utils.h"
#include "./lossless.h"
+#include "./lossless_common.h"
#include "./yuv.h"
-#define MAX_DIFF_COST (1e30f)
-
-static const int kPredLowEffort = 11;
-static const uint32_t kMaskAlpha = 0xff000000;
-
// lookup table for small values of log2(int)
const float kLog2Table[LOG_LOOKUP_IDX_MAX] = {
0.0000000000000000f, 0.0000000000000000f,
@@ -380,26 +376,9 @@ static float FastLog2Slow(uint32_t v) {
}
}
-// Mostly used to reduce code size + readability
-static WEBP_INLINE int GetMin(int a, int b) { return (a > b) ? b : a; }
-static WEBP_INLINE int GetMax(int a, int b) { return (a < b) ? b : a; }
-
//------------------------------------------------------------------------------
// Methods to calculate Entropy (Shannon).
-static float PredictionCostSpatial(const int counts[256], int weight_0,
- double exp_val) {
- const int significant_symbols = 256 >> 4;
- const double exp_decay_factor = 0.6;
- double bits = weight_0 * counts[0];
- int i;
- for (i = 1; i < significant_symbols; ++i) {
- bits += exp_val * (counts[i] + counts[256 - i]);
- exp_val *= exp_decay_factor;
- }
- return (float)(-0.1 * bits);
-}
-
// Compute the combined Shanon's entropy for distribution {X} and {X+Y}
static float CombinedShannonEntropy(const int X[256], const int Y[256]) {
int i;
@@ -422,18 +401,6 @@ static float CombinedShannonEntropy(const int X[256], const int Y[256]) {
return (float)retval;
}
-static float PredictionCostSpatialHistogram(const int accumulated[4][256],
- const int tile[4][256]) {
- int i;
- double retval = 0;
- for (i = 0; i < 4; ++i) {
- const double kExpValue = 0.94;
- retval += PredictionCostSpatial(tile[i], 1, kExpValue);
- retval += VP8LCombinedShannonEntropy(tile[i], accumulated[i]);
- }
- return (float)retval;
-}
-
void VP8LBitEntropyInit(VP8LBitEntropy* const entropy) {
entropy->entropy = 0.;
entropy->sum = 0;
@@ -486,9 +453,9 @@ static WEBP_INLINE void GetEntropyUnrefinedHelper(
*i_prev = i;
}
-void VP8LGetEntropyUnrefined(const uint32_t* const X, int length,
- VP8LBitEntropy* const bit_entropy,
- VP8LStreaks* const stats) {
+static void GetEntropyUnrefined(const uint32_t X[], int length,
+ VP8LBitEntropy* const bit_entropy,
+ VP8LStreaks* const stats) {
int i;
int i_prev = 0;
uint32_t x_prev = X[0];
@@ -499,18 +466,18 @@ void VP8LGetEntropyUnrefined(const uint32_t* const X, int length,
for (i = 1; i < length; ++i) {
const uint32_t x = X[i];
if (x != x_prev) {
- VP8LGetEntropyUnrefinedHelper(x, i, &x_prev, &i_prev, bit_entropy, stats);
+ GetEntropyUnrefinedHelper(x, i, &x_prev, &i_prev, bit_entropy, stats);
}
}
- VP8LGetEntropyUnrefinedHelper(0, i, &x_prev, &i_prev, bit_entropy, stats);
+ GetEntropyUnrefinedHelper(0, i, &x_prev, &i_prev, bit_entropy, stats);
bit_entropy->entropy += VP8LFastSLog2(bit_entropy->sum);
}
-void VP8LGetCombinedEntropyUnrefined(const uint32_t* const X,
- const uint32_t* const Y, int length,
- VP8LBitEntropy* const bit_entropy,
- VP8LStreaks* const stats) {
+static void GetCombinedEntropyUnrefined(const uint32_t X[], const uint32_t Y[],
+ int length,
+ VP8LBitEntropy* const bit_entropy,
+ VP8LStreaks* const stats) {
int i = 1;
int i_prev = 0;
uint32_t xy_prev = X[0] + Y[0];
@@ -521,439 +488,29 @@ void VP8LGetCombinedEntropyUnrefined(const uint32_t* const X,
for (i = 1; i < length; ++i) {
const uint32_t xy = X[i] + Y[i];
if (xy != xy_prev) {
- VP8LGetEntropyUnrefinedHelper(xy, i, &xy_prev, &i_prev, bit_entropy,
- stats);
+ GetEntropyUnrefinedHelper(xy, i, &xy_prev, &i_prev, bit_entropy, stats);
}
}
- VP8LGetEntropyUnrefinedHelper(0, i, &xy_prev, &i_prev, bit_entropy, stats);
+ GetEntropyUnrefinedHelper(0, i, &xy_prev, &i_prev, bit_entropy, stats);
bit_entropy->entropy += VP8LFastSLog2(bit_entropy->sum);
}
-static WEBP_INLINE void UpdateHisto(int histo_argb[4][256], uint32_t argb) {
- ++histo_argb[0][argb >> 24];
- ++histo_argb[1][(argb >> 16) & 0xff];
- ++histo_argb[2][(argb >> 8) & 0xff];
- ++histo_argb[3][argb & 0xff];
-}
-
//------------------------------------------------------------------------------
-static WEBP_INLINE uint32_t Predict(VP8LPredictorFunc pred_func,
- int x, int y,
- const uint32_t* current_row,
- const uint32_t* upper_row) {
- if (y == 0) {
- return (x == 0) ? ARGB_BLACK : current_row[x - 1]; // Left.
- } else if (x == 0) {
- return upper_row[x]; // Top.
- } else {
- return pred_func(current_row[x - 1], upper_row + x);
- }
-}
-
-static int MaxDiffBetweenPixels(uint32_t p1, uint32_t p2) {
- const int diff_a = abs((int)(p1 >> 24) - (int)(p2 >> 24));
- const int diff_r = abs((int)((p1 >> 16) & 0xff) - (int)((p2 >> 16) & 0xff));
- const int diff_g = abs((int)((p1 >> 8) & 0xff) - (int)((p2 >> 8) & 0xff));
- const int diff_b = abs((int)(p1 & 0xff) - (int)(p2 & 0xff));
- return GetMax(GetMax(diff_a, diff_r), GetMax(diff_g, diff_b));
-}
-
-static int MaxDiffAroundPixel(uint32_t current, uint32_t up, uint32_t down,
- uint32_t left, uint32_t right) {
- const int diff_up = MaxDiffBetweenPixels(current, up);
- const int diff_down = MaxDiffBetweenPixels(current, down);
- const int diff_left = MaxDiffBetweenPixels(current, left);
- const int diff_right = MaxDiffBetweenPixels(current, right);
- return GetMax(GetMax(diff_up, diff_down), GetMax(diff_left, diff_right));
-}
-
-static uint32_t AddGreenToBlueAndRed(uint32_t argb) {
- const uint32_t green = (argb >> 8) & 0xff;
- uint32_t red_blue = argb & 0x00ff00ffu;
- red_blue += (green << 16) | green;
- red_blue &= 0x00ff00ffu;
- return (argb & 0xff00ff00u) | red_blue;
-}
-
-static void MaxDiffsForRow(int width, int stride, const uint32_t* const argb,
- uint8_t* const max_diffs, int used_subtract_green) {
- uint32_t current, up, down, left, right;
- int x;
- if (width <= 2) return;
- current = argb[0];
- right = argb[1];
- if (used_subtract_green) {
- current = AddGreenToBlueAndRed(current);
- right = AddGreenToBlueAndRed(right);
- }
- // max_diffs[0] and max_diffs[width - 1] are never used.
- for (x = 1; x < width - 1; ++x) {
- up = argb[-stride + x];
- down = argb[stride + x];
- left = current;
- current = right;
- right = argb[x + 1];
- if (used_subtract_green) {
- up = AddGreenToBlueAndRed(up);
- down = AddGreenToBlueAndRed(down);
- right = AddGreenToBlueAndRed(right);
- }
- max_diffs[x] = MaxDiffAroundPixel(current, up, down, left, right);
- }
-}
-
-// Quantize the difference between the actual component value and its prediction
-// to a multiple of quantization, working modulo 256, taking care not to cross
-// a boundary (inclusive upper limit).
-static uint8_t NearLosslessComponent(uint8_t value, uint8_t predict,
- uint8_t boundary, int quantization) {
- const int residual = (value - predict) & 0xff;
- const int boundary_residual = (boundary - predict) & 0xff;
- const int lower = residual & ~(quantization - 1);
- const int upper = lower + quantization;
- // Resolve ties towards a value closer to the prediction (i.e. towards lower
- // if value comes after prediction and towards upper otherwise).
- const int bias = ((boundary - value) & 0xff) < boundary_residual;
- if (residual - lower < upper - residual + bias) {
- // lower is closer to residual than upper.
- if (residual > boundary_residual && lower <= boundary_residual) {
- // Halve quantization step to avoid crossing boundary. This midpoint is
- // on the same side of boundary as residual because midpoint >= residual
- // (since lower is closer than upper) and residual is above the boundary.
- return lower + (quantization >> 1);
- }
- return lower;
- } else {
- // upper is closer to residual than lower.
- if (residual <= boundary_residual && upper > boundary_residual) {
- // Halve quantization step to avoid crossing boundary. This midpoint is
- // on the same side of boundary as residual because midpoint <= residual
- // (since upper is closer than lower) and residual is below the boundary.
- return lower + (quantization >> 1);
- }
- return upper & 0xff;
- }
-}
-
-// Quantize every component of the difference between the actual pixel value and
-// its prediction to a multiple of a quantization (a power of 2, not larger than
-// max_quantization which is a power of 2, smaller than max_diff). Take care if
-// value and predict have undergone subtract green, which means that red and
-// blue are represented as offsets from green.
-static uint32_t NearLossless(uint32_t value, uint32_t predict,
- int max_quantization, int max_diff,
- int used_subtract_green) {
- int quantization;
- uint8_t new_green = 0;
- uint8_t green_diff = 0;
- uint8_t a, r, g, b;
- if (max_diff <= 2) {
- return VP8LSubPixels(value, predict);
- }
- quantization = max_quantization;
- while (quantization >= max_diff) {
- quantization >>= 1;
- }
- if ((value >> 24) == 0 || (value >> 24) == 0xff) {
- // Preserve transparency of fully transparent or fully opaque pixels.
- a = ((value >> 24) - (predict >> 24)) & 0xff;
- } else {
- a = NearLosslessComponent(value >> 24, predict >> 24, 0xff, quantization);
- }
- g = NearLosslessComponent((value >> 8) & 0xff, (predict >> 8) & 0xff, 0xff,
- quantization);
- if (used_subtract_green) {
- // The green offset will be added to red and blue components during decoding
- // to obtain the actual red and blue values.
- new_green = ((predict >> 8) + g) & 0xff;
- // The amount by which green has been adjusted during quantization. It is
- // subtracted from red and blue for compensation, to avoid accumulating two
- // quantization errors in them.
- green_diff = (new_green - (value >> 8)) & 0xff;
- }
- r = NearLosslessComponent(((value >> 16) - green_diff) & 0xff,
- (predict >> 16) & 0xff, 0xff - new_green,
- quantization);
- b = NearLosslessComponent((value - green_diff) & 0xff, predict & 0xff,
- 0xff - new_green, quantization);
- return ((uint32_t)a << 24) | ((uint32_t)r << 16) | ((uint32_t)g << 8) | b;
-}
-
-// Returns the difference between the pixel and its prediction. In case of a
-// lossy encoding, updates the source image to avoid propagating the deviation
-// further to pixels which depend on the current pixel for their predictions.
-static WEBP_INLINE uint32_t GetResidual(int width, int height,
- uint32_t* const upper_row,
- uint32_t* const current_row,
- const uint8_t* const max_diffs,
- int mode, VP8LPredictorFunc pred_func,
- int x, int y, int max_quantization,
- int exact, int used_subtract_green) {
- const uint32_t predict = Predict(pred_func, x, y, current_row, upper_row);
- uint32_t residual;
- if (max_quantization == 1 || mode == 0 || y == 0 || y == height - 1 ||
- x == 0 || x == width - 1) {
- residual = VP8LSubPixels(current_row[x], predict);
- } else {
- residual = NearLossless(current_row[x], predict, max_quantization,
- max_diffs[x], used_subtract_green);
- // Update the source image.
- current_row[x] = VP8LAddPixels(predict, residual);
- // x is never 0 here so we do not need to update upper_row like below.
- }
- if (!exact && (current_row[x] & kMaskAlpha) == 0) {
- // If alpha is 0, cleanup RGB. We can choose the RGB values of the residual
- // for best compression. The prediction of alpha itself can be non-zero and
- // must be kept though. We choose RGB of the residual to be 0.
- residual &= kMaskAlpha;
- // Update the source image.
- current_row[x] = predict & ~kMaskAlpha;
- // The prediction for the rightmost pixel in a row uses the leftmost pixel
- // in that row as its top-right context pixel. Hence if we change the
- // leftmost pixel of current_row, the corresponding change must be applied
- // to upper_row as well where top-right context is being read from.
- if (x == 0 && y != 0) upper_row[width] = current_row[0];
- }
- return residual;
-}
-
-// Returns best predictor and updates the accumulated histogram.
-// If max_quantization > 1, assumes that near lossless processing will be
-// applied, quantizing residuals to multiples of quantization levels up to
-// max_quantization (the actual quantization level depends on smoothness near
-// the given pixel).
-static int GetBestPredictorForTile(int width, int height,
- int tile_x, int tile_y, int bits,
- int accumulated[4][256],
- uint32_t* const argb_scratch,
- const uint32_t* const argb,
- int max_quantization,
- int exact, int used_subtract_green) {
- const int kNumPredModes = 14;
- const int start_x = tile_x << bits;
- const int start_y = tile_y << bits;
- const int tile_size = 1 << bits;
- const int max_y = GetMin(tile_size, height - start_y);
- const int max_x = GetMin(tile_size, width - start_x);
- // Whether there exist columns just outside the tile.
- const int have_left = (start_x > 0);
- const int have_right = (max_x < width - start_x);
- // Position and size of the strip covering the tile and adjacent columns if
- // they exist.
- const int context_start_x = start_x - have_left;
- const int context_width = max_x + have_left + have_right;
- // The width of upper_row and current_row is one pixel larger than image width
- // to allow the top right pixel to point to the leftmost pixel of the next row
- // when at the right edge.
- uint32_t* upper_row = argb_scratch;
- uint32_t* current_row = upper_row + width + 1;
- uint8_t* const max_diffs = (uint8_t*)(current_row + width + 1);
- float best_diff = MAX_DIFF_COST;
- int best_mode = 0;
- int mode;
- int histo_stack_1[4][256];
- int histo_stack_2[4][256];
- // Need pointers to be able to swap arrays.
- int (*histo_argb)[256] = histo_stack_1;
- int (*best_histo)[256] = histo_stack_2;
- int i, j;
-
- for (mode = 0; mode < kNumPredModes; ++mode) {
- const VP8LPredictorFunc pred_func = VP8LPredictors[mode];
- float cur_diff;
- int relative_y;
- memset(histo_argb, 0, sizeof(histo_stack_1));
- if (start_y > 0) {
- // Read the row above the tile which will become the first upper_row.
- // Include a pixel to the left if it exists; include a pixel to the right
- // in all cases (wrapping to the leftmost pixel of the next row if it does
- // not exist).
- memcpy(current_row + context_start_x,
- argb + (start_y - 1) * width + context_start_x,
- sizeof(*argb) * (max_x + have_left + 1));
- }
- for (relative_y = 0; relative_y < max_y; ++relative_y) {
- const int y = start_y + relative_y;
- int relative_x;
- uint32_t* tmp = upper_row;
- upper_row = current_row;
- current_row = tmp;
- // Read current_row. Include a pixel to the left if it exists; include a
- // pixel to the right in all cases except at the bottom right corner of
- // the image (wrapping to the leftmost pixel of the next row if it does
- // not exist in the current row).
- memcpy(current_row + context_start_x,
- argb + y * width + context_start_x,
- sizeof(*argb) * (max_x + have_left + (y + 1 < height)));
- if (max_quantization > 1 && y >= 1 && y + 1 < height) {
- MaxDiffsForRow(context_width, width, argb + y * width + context_start_x,
- max_diffs + context_start_x, used_subtract_green);
- }
-
- for (relative_x = 0; relative_x < max_x; ++relative_x) {
- const int x = start_x + relative_x;
- UpdateHisto(histo_argb,
- GetResidual(width, height, upper_row, current_row,
- max_diffs, mode, pred_func, x, y,
- max_quantization, exact, used_subtract_green));
- }
- }
- cur_diff = PredictionCostSpatialHistogram(
- (const int (*)[256])accumulated, (const int (*)[256])histo_argb);
- if (cur_diff < best_diff) {
- int (*tmp)[256] = histo_argb;
- histo_argb = best_histo;
- best_histo = tmp;
- best_diff = cur_diff;
- best_mode = mode;
- }
- }
-
- for (i = 0; i < 4; i++) {
- for (j = 0; j < 256; j++) {
- accumulated[i][j] += best_histo[i][j];
- }
- }
-
- return best_mode;
-}
-
-// Converts pixels of the image to residuals with respect to predictions.
-// If max_quantization > 1, applies near lossless processing, quantizing
-// residuals to multiples of quantization levels up to max_quantization
-// (the actual quantization level depends on smoothness near the given pixel).
-static void CopyImageWithPrediction(int width, int height,
- int bits, uint32_t* const modes,
- uint32_t* const argb_scratch,
- uint32_t* const argb,
- int low_effort, int max_quantization,
- int exact, int used_subtract_green) {
- const int tiles_per_row = VP8LSubSampleSize(width, bits);
- const int mask = (1 << bits) - 1;
- // The width of upper_row and current_row is one pixel larger than image width
- // to allow the top right pixel to point to the leftmost pixel of the next row
- // when at the right edge.
- uint32_t* upper_row = argb_scratch;
- uint32_t* current_row = upper_row + width + 1;
- uint8_t* current_max_diffs = (uint8_t*)(current_row + width + 1);
- uint8_t* lower_max_diffs = current_max_diffs + width;
- int y;
- int mode = 0;
- VP8LPredictorFunc pred_func = NULL;
-
- for (y = 0; y < height; ++y) {
- int x;
- uint32_t* const tmp32 = upper_row;
- upper_row = current_row;
- current_row = tmp32;
- memcpy(current_row, argb + y * width,
- sizeof(*argb) * (width + (y + 1 < height)));
-
- if (low_effort) {
- for (x = 0; x < width; ++x) {
- const uint32_t predict = Predict(VP8LPredictors[kPredLowEffort], x, y,
- current_row, upper_row);
- argb[y * width + x] = VP8LSubPixels(current_row[x], predict);
- }
- } else {
- if (max_quantization > 1) {
- // Compute max_diffs for the lower row now, because that needs the
- // contents of argb for the current row, which we will overwrite with
- // residuals before proceeding with the next row.
- uint8_t* const tmp8 = current_max_diffs;
- current_max_diffs = lower_max_diffs;
- lower_max_diffs = tmp8;
- if (y + 2 < height) {
- MaxDiffsForRow(width, width, argb + (y + 1) * width, lower_max_diffs,
- used_subtract_green);
- }
- }
- for (x = 0; x < width; ++x) {
- if ((x & mask) == 0) {
- mode = (modes[(y >> bits) * tiles_per_row + (x >> bits)] >> 8) & 0xff;
- pred_func = VP8LPredictors[mode];
- }
- argb[y * width + x] = GetResidual(
- width, height, upper_row, current_row, current_max_diffs, mode,
- pred_func, x, y, max_quantization, exact, used_subtract_green);
- }
- }
- }
-}
-
-// Finds the best predictor for each tile, and converts the image to residuals
-// with respect to predictions. If near_lossless_quality < 100, applies
-// near lossless processing, shaving off more bits of residuals for lower
-// qualities.
-void VP8LResidualImage(int width, int height, int bits, int low_effort,
- uint32_t* const argb, uint32_t* const argb_scratch,
- uint32_t* const image, int near_lossless_quality,
- int exact, int used_subtract_green) {
- const int tiles_per_row = VP8LSubSampleSize(width, bits);
- const int tiles_per_col = VP8LSubSampleSize(height, bits);
- int tile_y;
- int histo[4][256];
- const int max_quantization = 1 << VP8LNearLosslessBits(near_lossless_quality);
- if (low_effort) {
- int i;
- for (i = 0; i < tiles_per_row * tiles_per_col; ++i) {
- image[i] = ARGB_BLACK | (kPredLowEffort << 8);
- }
- } else {
- memset(histo, 0, sizeof(histo));
- for (tile_y = 0; tile_y < tiles_per_col; ++tile_y) {
- int tile_x;
- for (tile_x = 0; tile_x < tiles_per_row; ++tile_x) {
- const int pred = GetBestPredictorForTile(width, height, tile_x, tile_y,
- bits, histo, argb_scratch, argb, max_quantization, exact,
- used_subtract_green);
- image[tile_y * tiles_per_row + tile_x] = ARGB_BLACK | (pred << 8);
- }
- }
- }
-
- CopyImageWithPrediction(width, height, bits, image, argb_scratch, argb,
- low_effort, max_quantization, exact,
- used_subtract_green);
-}
-
void VP8LSubtractGreenFromBlueAndRed_C(uint32_t* argb_data, int num_pixels) {
int i;
for (i = 0; i < num_pixels; ++i) {
- const uint32_t argb = argb_data[i];
- const uint32_t green = (argb >> 8) & 0xff;
+ const int argb = argb_data[i];
+ const int green = (argb >> 8) & 0xff;
const uint32_t new_r = (((argb >> 16) & 0xff) - green) & 0xff;
- const uint32_t new_b = ((argb & 0xff) - green) & 0xff;
- argb_data[i] = (argb & 0xff00ff00) | (new_r << 16) | new_b;
+ const uint32_t new_b = (((argb >> 0) & 0xff) - green) & 0xff;
+ argb_data[i] = (argb & 0xff00ff00u) | (new_r << 16) | new_b;
}
}
-static WEBP_INLINE void MultipliersClear(VP8LMultipliers* const m) {
- m->green_to_red_ = 0;
- m->green_to_blue_ = 0;
- m->red_to_blue_ = 0;
-}
-
-static WEBP_INLINE uint32_t ColorTransformDelta(int8_t color_pred,
- int8_t color) {
- return (uint32_t)((int)(color_pred) * color) >> 5;
-}
-
-static WEBP_INLINE void ColorCodeToMultipliers(uint32_t color_code,
- VP8LMultipliers* const m) {
- m->green_to_red_ = (color_code >> 0) & 0xff;
- m->green_to_blue_ = (color_code >> 8) & 0xff;
- m->red_to_blue_ = (color_code >> 16) & 0xff;
-}
-
-static WEBP_INLINE uint32_t MultipliersToColorCode(
- const VP8LMultipliers* const m) {
- return 0xff000000u |
- ((uint32_t)(m->red_to_blue_) << 16) |
- ((uint32_t)(m->green_to_blue_) << 8) |
- m->green_to_red_;
+static WEBP_INLINE int ColorTransformDelta(int8_t color_pred, int8_t color) {
+ return ((int)color_pred * color) >> 5;
}
void VP8LTransformColor_C(const VP8LMultipliers* const m, uint32_t* data,
@@ -963,8 +520,8 @@ void VP8LTransformColor_C(const VP8LMultipliers* const m, uint32_t* data,
const uint32_t argb = data[i];
const uint32_t green = argb >> 8;
const uint32_t red = argb >> 16;
- uint32_t new_red = red;
- uint32_t new_blue = argb;
+ int new_red = red;
+ int new_blue = argb;
new_red -= ColorTransformDelta(m->green_to_red_, green);
new_red &= 0xff;
new_blue -= ColorTransformDelta(m->green_to_blue_, green);
@@ -977,7 +534,7 @@ void VP8LTransformColor_C(const VP8LMultipliers* const m, uint32_t* data,
static WEBP_INLINE uint8_t TransformColorRed(uint8_t green_to_red,
uint32_t argb) {
const uint32_t green = argb >> 8;
- uint32_t new_red = argb >> 16;
+ int new_red = argb >> 16;
new_red -= ColorTransformDelta(green_to_red, green);
return (new_red & 0xff);
}
@@ -993,15 +550,6 @@ static WEBP_INLINE uint8_t TransformColorBlue(uint8_t green_to_blue,
return (new_blue & 0xff);
}
-static float PredictionCostCrossColor(const int accumulated[256],
- const int counts[256]) {
- // Favor low entropy, locally and globally.
- // Favor small absolute values for PredictionCostSpatial
- static const double kExpValue = 2.4;
- return VP8LCombinedShannonEntropy(counts, accumulated) +
- PredictionCostSpatial(counts, 3, kExpValue);
-}
-
void VP8LCollectColorRedTransforms_C(const uint32_t* argb, int stride,
int tile_width, int tile_height,
int green_to_red, int histo[]) {
@@ -1014,59 +562,6 @@ void VP8LCollectColorRedTransforms_C(const uint32_t* argb, int stride,
}
}
-static float GetPredictionCostCrossColorRed(
- const uint32_t* argb, int stride, int tile_width, int tile_height,
- VP8LMultipliers prev_x, VP8LMultipliers prev_y, int green_to_red,
- const int accumulated_red_histo[256]) {
- int histo[256] = { 0 };
- float cur_diff;
-
- VP8LCollectColorRedTransforms(argb, stride, tile_width, tile_height,
- green_to_red, histo);
-
- cur_diff = PredictionCostCrossColor(accumulated_red_histo, histo);
- if ((uint8_t)green_to_red == prev_x.green_to_red_) {
- cur_diff -= 3; // favor keeping the areas locally similar
- }
- if ((uint8_t)green_to_red == prev_y.green_to_red_) {
- cur_diff -= 3; // favor keeping the areas locally similar
- }
- if (green_to_red == 0) {
- cur_diff -= 3;
- }
- return cur_diff;
-}
-
-static void GetBestGreenToRed(
- const uint32_t* argb, int stride, int tile_width, int tile_height,
- VP8LMultipliers prev_x, VP8LMultipliers prev_y, int quality,
- const int accumulated_red_histo[256], VP8LMultipliers* const best_tx) {
- const int kMaxIters = 4 + ((7 * quality) >> 8); // in range [4..6]
- int green_to_red_best = 0;
- int iter, offset;
- float best_diff = GetPredictionCostCrossColorRed(
- argb, stride, tile_width, tile_height, prev_x, prev_y,
- green_to_red_best, accumulated_red_histo);
- for (iter = 0; iter < kMaxIters; ++iter) {
- // ColorTransformDelta is a 3.5 bit fixed point, so 32 is equal to
- // one in color computation. Having initial delta here as 1 is sufficient
- // to explore the range of (-2, 2).
- const int delta = 32 >> iter;
- // Try a negative and a positive delta from the best known value.
- for (offset = -delta; offset <= delta; offset += 2 * delta) {
- const int green_to_red_cur = offset + green_to_red_best;
- const float cur_diff = GetPredictionCostCrossColorRed(
- argb, stride, tile_width, tile_height, prev_x, prev_y,
- green_to_red_cur, accumulated_red_histo);
- if (cur_diff < best_diff) {
- best_diff = cur_diff;
- green_to_red_best = green_to_red_cur;
- }
- }
- }
- best_tx->green_to_red_ = green_to_red_best;
-}
-
void VP8LCollectColorBlueTransforms_C(const uint32_t* argb, int stride,
int tile_width, int tile_height,
int green_to_blue, int red_to_blue,
@@ -1080,187 +575,6 @@ void VP8LCollectColorBlueTransforms_C(const uint32_t* argb, int stride,
}
}
-static float GetPredictionCostCrossColorBlue(
- const uint32_t* argb, int stride, int tile_width, int tile_height,
- VP8LMultipliers prev_x, VP8LMultipliers prev_y,
- int green_to_blue, int red_to_blue, const int accumulated_blue_histo[256]) {
- int histo[256] = { 0 };
- float cur_diff;
-
- VP8LCollectColorBlueTransforms(argb, stride, tile_width, tile_height,
- green_to_blue, red_to_blue, histo);
-
- cur_diff = PredictionCostCrossColor(accumulated_blue_histo, histo);
- if ((uint8_t)green_to_blue == prev_x.green_to_blue_) {
- cur_diff -= 3; // favor keeping the areas locally similar
- }
- if ((uint8_t)green_to_blue == prev_y.green_to_blue_) {
- cur_diff -= 3; // favor keeping the areas locally similar
- }
- if ((uint8_t)red_to_blue == prev_x.red_to_blue_) {
- cur_diff -= 3; // favor keeping the areas locally similar
- }
- if ((uint8_t)red_to_blue == prev_y.red_to_blue_) {
- cur_diff -= 3; // favor keeping the areas locally similar
- }
- if (green_to_blue == 0) {
- cur_diff -= 3;
- }
- if (red_to_blue == 0) {
- cur_diff -= 3;
- }
- return cur_diff;
-}
-
-#define kGreenRedToBlueNumAxis 8
-#define kGreenRedToBlueMaxIters 7
-static void GetBestGreenRedToBlue(
- const uint32_t* argb, int stride, int tile_width, int tile_height,
- VP8LMultipliers prev_x, VP8LMultipliers prev_y, int quality,
- const int accumulated_blue_histo[256],
- VP8LMultipliers* const best_tx) {
- const int8_t offset[kGreenRedToBlueNumAxis][2] =
- {{0, -1}, {0, 1}, {-1, 0}, {1, 0}, {-1, -1}, {-1, 1}, {1, -1}, {1, 1}};
- const int8_t delta_lut[kGreenRedToBlueMaxIters] = { 16, 16, 8, 4, 2, 2, 2 };
- const int iters =
- (quality < 25) ? 1 : (quality > 50) ? kGreenRedToBlueMaxIters : 4;
- int green_to_blue_best = 0;
- int red_to_blue_best = 0;
- int iter;
- // Initial value at origin:
- float best_diff = GetPredictionCostCrossColorBlue(
- argb, stride, tile_width, tile_height, prev_x, prev_y,
- green_to_blue_best, red_to_blue_best, accumulated_blue_histo);
- for (iter = 0; iter < iters; ++iter) {
- const int delta = delta_lut[iter];
- int axis;
- for (axis = 0; axis < kGreenRedToBlueNumAxis; ++axis) {
- const int green_to_blue_cur =
- offset[axis][0] * delta + green_to_blue_best;
- const int red_to_blue_cur = offset[axis][1] * delta + red_to_blue_best;
- const float cur_diff = GetPredictionCostCrossColorBlue(
- argb, stride, tile_width, tile_height, prev_x, prev_y,
- green_to_blue_cur, red_to_blue_cur, accumulated_blue_histo);
- if (cur_diff < best_diff) {
- best_diff = cur_diff;
- green_to_blue_best = green_to_blue_cur;
- red_to_blue_best = red_to_blue_cur;
- }
- if (quality < 25 && iter == 4) {
- // Only axis aligned diffs for lower quality.
- break; // next iter.
- }
- }
- if (delta == 2 && green_to_blue_best == 0 && red_to_blue_best == 0) {
- // Further iterations would not help.
- break; // out of iter-loop.
- }
- }
- best_tx->green_to_blue_ = green_to_blue_best;
- best_tx->red_to_blue_ = red_to_blue_best;
-}
-#undef kGreenRedToBlueMaxIters
-#undef kGreenRedToBlueNumAxis
-
-static VP8LMultipliers GetBestColorTransformForTile(
- int tile_x, int tile_y, int bits,
- VP8LMultipliers prev_x,
- VP8LMultipliers prev_y,
- int quality, int xsize, int ysize,
- const int accumulated_red_histo[256],
- const int accumulated_blue_histo[256],
- const uint32_t* const argb) {
- const int max_tile_size = 1 << bits;
- const int tile_y_offset = tile_y * max_tile_size;
- const int tile_x_offset = tile_x * max_tile_size;
- const int all_x_max = GetMin(tile_x_offset + max_tile_size, xsize);
- const int all_y_max = GetMin(tile_y_offset + max_tile_size, ysize);
- const int tile_width = all_x_max - tile_x_offset;
- const int tile_height = all_y_max - tile_y_offset;
- const uint32_t* const tile_argb = argb + tile_y_offset * xsize
- + tile_x_offset;
- VP8LMultipliers best_tx;
- MultipliersClear(&best_tx);
-
- GetBestGreenToRed(tile_argb, xsize, tile_width, tile_height,
- prev_x, prev_y, quality, accumulated_red_histo, &best_tx);
- GetBestGreenRedToBlue(tile_argb, xsize, tile_width, tile_height,
- prev_x, prev_y, quality, accumulated_blue_histo,
- &best_tx);
- return best_tx;
-}
-
-static void CopyTileWithColorTransform(int xsize, int ysize,
- int tile_x, int tile_y,
- int max_tile_size,
- VP8LMultipliers color_transform,
- uint32_t* argb) {
- const int xscan = GetMin(max_tile_size, xsize - tile_x);
- int yscan = GetMin(max_tile_size, ysize - tile_y);
- argb += tile_y * xsize + tile_x;
- while (yscan-- > 0) {
- VP8LTransformColor(&color_transform, argb, xscan);
- argb += xsize;
- }
-}
-
-void VP8LColorSpaceTransform(int width, int height, int bits, int quality,
- uint32_t* const argb, uint32_t* image) {
- const int max_tile_size = 1 << bits;
- const int tile_xsize = VP8LSubSampleSize(width, bits);
- const int tile_ysize = VP8LSubSampleSize(height, bits);
- int accumulated_red_histo[256] = { 0 };
- int accumulated_blue_histo[256] = { 0 };
- int tile_x, tile_y;
- VP8LMultipliers prev_x, prev_y;
- MultipliersClear(&prev_y);
- MultipliersClear(&prev_x);
- for (tile_y = 0; tile_y < tile_ysize; ++tile_y) {
- for (tile_x = 0; tile_x < tile_xsize; ++tile_x) {
- int y;
- const int tile_x_offset = tile_x * max_tile_size;
- const int tile_y_offset = tile_y * max_tile_size;
- const int all_x_max = GetMin(tile_x_offset + max_tile_size, width);
- const int all_y_max = GetMin(tile_y_offset + max_tile_size, height);
- const int offset = tile_y * tile_xsize + tile_x;
- if (tile_y != 0) {
- ColorCodeToMultipliers(image[offset - tile_xsize], &prev_y);
- }
- prev_x = GetBestColorTransformForTile(tile_x, tile_y, bits,
- prev_x, prev_y,
- quality, width, height,
- accumulated_red_histo,
- accumulated_blue_histo,
- argb);
- image[offset] = MultipliersToColorCode(&prev_x);
- CopyTileWithColorTransform(width, height, tile_x_offset, tile_y_offset,
- max_tile_size, prev_x, argb);
-
- // Gather accumulated histogram data.
- for (y = tile_y_offset; y < all_y_max; ++y) {
- int ix = y * width + tile_x_offset;
- const int ix_end = ix + all_x_max - tile_x_offset;
- for (; ix < ix_end; ++ix) {
- const uint32_t pix = argb[ix];
- if (ix >= 2 &&
- pix == argb[ix - 2] &&
- pix == argb[ix - 1]) {
- continue; // repeated pixels are handled by backward references
- }
- if (ix >= width + 2 &&
- argb[ix - 2] == argb[ix - width - 2] &&
- argb[ix - 1] == argb[ix - width - 1] &&
- pix == argb[ix - width]) {
- continue; // repeated pixels are handled by backward references
- }
- ++accumulated_red_histo[(pix >> 16) & 0xff];
- ++accumulated_blue_histo[(pix >> 0) & 0xff];
- }
- }
- }
- }
-}
-
//------------------------------------------------------------------------------
static int VectorMismatch(const uint32_t* const array1,
@@ -1274,8 +588,8 @@ static int VectorMismatch(const uint32_t* const array1,
}
// Bundles multiple (1, 2, 4 or 8) pixels into a single pixel.
-void VP8LBundleColorMap(const uint8_t* const row, int width,
- int xbits, uint32_t* const dst) {
+void VP8LBundleColorMap_C(const uint8_t* const row, int width, int xbits,
+ uint32_t* dst) {
int x;
if (xbits > 0) {
const int bit_depth = 1 << (3 - xbits);
@@ -1350,8 +664,172 @@ static void HistogramAdd(const VP8LHistogram* const a,
}
//------------------------------------------------------------------------------
+// Image transforms.
-VP8LProcessBlueAndRedFunc VP8LSubtractGreenFromBlueAndRed;
+static WEBP_INLINE uint32_t Average2(uint32_t a0, uint32_t a1) {
+ return (((a0 ^ a1) & 0xfefefefeu) >> 1) + (a0 & a1);
+}
+
+static WEBP_INLINE uint32_t Average3(uint32_t a0, uint32_t a1, uint32_t a2) {
+ return Average2(Average2(a0, a2), a1);
+}
+
+static WEBP_INLINE uint32_t Average4(uint32_t a0, uint32_t a1,
+ uint32_t a2, uint32_t a3) {
+ return Average2(Average2(a0, a1), Average2(a2, a3));
+}
+
+static WEBP_INLINE uint32_t Clip255(uint32_t a) {
+ if (a < 256) {
+ return a;
+ }
+ // return 0, when a is a negative integer.
+ // return 255, when a is positive.
+ return ~a >> 24;
+}
+
+static WEBP_INLINE int AddSubtractComponentFull(int a, int b, int c) {
+ return Clip255(a + b - c);
+}
+
+static WEBP_INLINE uint32_t ClampedAddSubtractFull(uint32_t c0, uint32_t c1,
+ uint32_t c2) {
+ const int a = AddSubtractComponentFull(c0 >> 24, c1 >> 24, c2 >> 24);
+ const int r = AddSubtractComponentFull((c0 >> 16) & 0xff,
+ (c1 >> 16) & 0xff,
+ (c2 >> 16) & 0xff);
+ const int g = AddSubtractComponentFull((c0 >> 8) & 0xff,
+ (c1 >> 8) & 0xff,
+ (c2 >> 8) & 0xff);
+ const int b = AddSubtractComponentFull(c0 & 0xff, c1 & 0xff, c2 & 0xff);
+ return ((uint32_t)a << 24) | (r << 16) | (g << 8) | b;
+}
+
+static WEBP_INLINE int AddSubtractComponentHalf(int a, int b) {
+ return Clip255(a + (a - b) / 2);
+}
+
+static WEBP_INLINE uint32_t ClampedAddSubtractHalf(uint32_t c0, uint32_t c1,
+ uint32_t c2) {
+ const uint32_t ave = Average2(c0, c1);
+ const int a = AddSubtractComponentHalf(ave >> 24, c2 >> 24);
+ const int r = AddSubtractComponentHalf((ave >> 16) & 0xff, (c2 >> 16) & 0xff);
+ const int g = AddSubtractComponentHalf((ave >> 8) & 0xff, (c2 >> 8) & 0xff);
+ const int b = AddSubtractComponentHalf((ave >> 0) & 0xff, (c2 >> 0) & 0xff);
+ return ((uint32_t)a << 24) | (r << 16) | (g << 8) | b;
+}
+
+// gcc-4.9 on ARM generates incorrect code in Select() when Sub3() is inlined.
+#if defined(__arm__) && \
+ (LOCAL_GCC_VERSION == 0x409 || LOCAL_GCC_VERSION == 0x408)
+# define LOCAL_INLINE __attribute__ ((noinline))
+#else
+# define LOCAL_INLINE WEBP_INLINE
+#endif
+
+static LOCAL_INLINE int Sub3(int a, int b, int c) {
+ const int pb = b - c;
+ const int pa = a - c;
+ return abs(pb) - abs(pa);
+}
+
+#undef LOCAL_INLINE
+
+static WEBP_INLINE uint32_t Select(uint32_t a, uint32_t b, uint32_t c) {
+ const int pa_minus_pb =
+ Sub3((a >> 24) , (b >> 24) , (c >> 24) ) +
+ Sub3((a >> 16) & 0xff, (b >> 16) & 0xff, (c >> 16) & 0xff) +
+ Sub3((a >> 8) & 0xff, (b >> 8) & 0xff, (c >> 8) & 0xff) +
+ Sub3((a ) & 0xff, (b ) & 0xff, (c ) & 0xff);
+ return (pa_minus_pb <= 0) ? a : b;
+}
+
+//------------------------------------------------------------------------------
+// Predictors
+
+static uint32_t Predictor2(uint32_t left, const uint32_t* const top) {
+ (void)left;
+ return top[0];
+}
+static uint32_t Predictor3(uint32_t left, const uint32_t* const top) {
+ (void)left;
+ return top[1];
+}
+static uint32_t Predictor4(uint32_t left, const uint32_t* const top) {
+ (void)left;
+ return top[-1];
+}
+static uint32_t Predictor5(uint32_t left, const uint32_t* const top) {
+ const uint32_t pred = Average3(left, top[0], top[1]);
+ return pred;
+}
+static uint32_t Predictor6(uint32_t left, const uint32_t* const top) {
+ const uint32_t pred = Average2(left, top[-1]);
+ return pred;
+}
+static uint32_t Predictor7(uint32_t left, const uint32_t* const top) {
+ const uint32_t pred = Average2(left, top[0]);
+ return pred;
+}
+static uint32_t Predictor8(uint32_t left, const uint32_t* const top) {
+ const uint32_t pred = Average2(top[-1], top[0]);
+ (void)left;
+ return pred;
+}
+static uint32_t Predictor9(uint32_t left, const uint32_t* const top) {
+ const uint32_t pred = Average2(top[0], top[1]);
+ (void)left;
+ return pred;
+}
+static uint32_t Predictor10(uint32_t left, const uint32_t* const top) {
+ const uint32_t pred = Average4(left, top[-1], top[0], top[1]);
+ return pred;
+}
+static uint32_t Predictor11(uint32_t left, const uint32_t* const top) {
+ const uint32_t pred = Select(top[0], left, top[-1]);
+ return pred;
+}
+static uint32_t Predictor12(uint32_t left, const uint32_t* const top) {
+ const uint32_t pred = ClampedAddSubtractFull(left, top[0], top[-1]);
+ return pred;
+}
+static uint32_t Predictor13(uint32_t left, const uint32_t* const top) {
+ const uint32_t pred = ClampedAddSubtractHalf(left, top[0], top[-1]);
+ return pred;
+}
+
+//------------------------------------------------------------------------------
+
+static void PredictorSub0_C(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ for (i = 0; i < num_pixels; ++i) out[i] = VP8LSubPixels(in[i], ARGB_BLACK);
+ (void)upper;
+}
+
+static void PredictorSub1_C(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ for (i = 0; i < num_pixels; ++i) out[i] = VP8LSubPixels(in[i], in[i - 1]);
+ (void)upper;
+}
+
+GENERATE_PREDICTOR_SUB(Predictor2, PredictorSub2_C)
+GENERATE_PREDICTOR_SUB(Predictor3, PredictorSub3_C)
+GENERATE_PREDICTOR_SUB(Predictor4, PredictorSub4_C)
+GENERATE_PREDICTOR_SUB(Predictor5, PredictorSub5_C)
+GENERATE_PREDICTOR_SUB(Predictor6, PredictorSub6_C)
+GENERATE_PREDICTOR_SUB(Predictor7, PredictorSub7_C)
+GENERATE_PREDICTOR_SUB(Predictor8, PredictorSub8_C)
+GENERATE_PREDICTOR_SUB(Predictor9, PredictorSub9_C)
+GENERATE_PREDICTOR_SUB(Predictor10, PredictorSub10_C)
+GENERATE_PREDICTOR_SUB(Predictor11, PredictorSub11_C)
+GENERATE_PREDICTOR_SUB(Predictor12, PredictorSub12_C)
+GENERATE_PREDICTOR_SUB(Predictor13, PredictorSub13_C)
+
+//------------------------------------------------------------------------------
+
+VP8LProcessEncBlueAndRedFunc VP8LSubtractGreenFromBlueAndRed;
VP8LTransformColorFunc VP8LTransformColor;
@@ -1365,17 +843,23 @@ VP8LCostFunc VP8LExtraCost;
VP8LCostCombinedFunc VP8LExtraCostCombined;
VP8LCombinedShannonEntropyFunc VP8LCombinedShannonEntropy;
-GetEntropyUnrefinedHelperFunc VP8LGetEntropyUnrefinedHelper;
+VP8LGetEntropyUnrefinedFunc VP8LGetEntropyUnrefined;
+VP8LGetCombinedEntropyUnrefinedFunc VP8LGetCombinedEntropyUnrefined;
VP8LHistogramAddFunc VP8LHistogramAdd;
VP8LVectorMismatchFunc VP8LVectorMismatch;
+VP8LBundleColorMapFunc VP8LBundleColorMap;
+
+VP8LPredictorAddSubFunc VP8LPredictorsSub[16];
+VP8LPredictorAddSubFunc VP8LPredictorsSub_C[16];
extern void VP8LEncDspInitSSE2(void);
extern void VP8LEncDspInitSSE41(void);
extern void VP8LEncDspInitNEON(void);
extern void VP8LEncDspInitMIPS32(void);
extern void VP8LEncDspInitMIPSdspR2(void);
+extern void VP8LEncDspInitMSA(void);
static volatile VP8CPUInfo lossless_enc_last_cpuinfo_used =
(VP8CPUInfo)&lossless_enc_last_cpuinfo_used;
@@ -1399,11 +883,47 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInit(void) {
VP8LExtraCostCombined = ExtraCostCombined;
VP8LCombinedShannonEntropy = CombinedShannonEntropy;
- VP8LGetEntropyUnrefinedHelper = GetEntropyUnrefinedHelper;
+ VP8LGetEntropyUnrefined = GetEntropyUnrefined;
+ VP8LGetCombinedEntropyUnrefined = GetCombinedEntropyUnrefined;
VP8LHistogramAdd = HistogramAdd;
VP8LVectorMismatch = VectorMismatch;
+ VP8LBundleColorMap = VP8LBundleColorMap_C;
+
+ VP8LPredictorsSub[0] = PredictorSub0_C;
+ VP8LPredictorsSub[1] = PredictorSub1_C;
+ VP8LPredictorsSub[2] = PredictorSub2_C;
+ VP8LPredictorsSub[3] = PredictorSub3_C;
+ VP8LPredictorsSub[4] = PredictorSub4_C;
+ VP8LPredictorsSub[5] = PredictorSub5_C;
+ VP8LPredictorsSub[6] = PredictorSub6_C;
+ VP8LPredictorsSub[7] = PredictorSub7_C;
+ VP8LPredictorsSub[8] = PredictorSub8_C;
+ VP8LPredictorsSub[9] = PredictorSub9_C;
+ VP8LPredictorsSub[10] = PredictorSub10_C;
+ VP8LPredictorsSub[11] = PredictorSub11_C;
+ VP8LPredictorsSub[12] = PredictorSub12_C;
+ VP8LPredictorsSub[13] = PredictorSub13_C;
+ VP8LPredictorsSub[14] = PredictorSub0_C; // <- padding security sentinels
+ VP8LPredictorsSub[15] = PredictorSub0_C;
+
+ VP8LPredictorsSub_C[0] = PredictorSub0_C;
+ VP8LPredictorsSub_C[1] = PredictorSub1_C;
+ VP8LPredictorsSub_C[2] = PredictorSub2_C;
+ VP8LPredictorsSub_C[3] = PredictorSub3_C;
+ VP8LPredictorsSub_C[4] = PredictorSub4_C;
+ VP8LPredictorsSub_C[5] = PredictorSub5_C;
+ VP8LPredictorsSub_C[6] = PredictorSub6_C;
+ VP8LPredictorsSub_C[7] = PredictorSub7_C;
+ VP8LPredictorsSub_C[8] = PredictorSub8_C;
+ VP8LPredictorsSub_C[9] = PredictorSub9_C;
+ VP8LPredictorsSub_C[10] = PredictorSub10_C;
+ VP8LPredictorsSub_C[11] = PredictorSub11_C;
+ VP8LPredictorsSub_C[12] = PredictorSub12_C;
+ VP8LPredictorsSub_C[13] = PredictorSub13_C;
+ VP8LPredictorsSub_C[14] = PredictorSub0_C; // <- padding security sentinels
+ VP8LPredictorsSub_C[15] = PredictorSub0_C;
// If defined, use CPUInfo() to overwrite some pointers with faster versions.
if (VP8GetCPUInfo != NULL) {
@@ -1432,6 +952,11 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInit(void) {
VP8LEncDspInitMIPSdspR2();
}
#endif
+#if defined(WEBP_USE_MSA)
+ if (VP8GetCPUInfo(kMSA)) {
+ VP8LEncDspInitMSA();
+ }
+#endif
}
lossless_enc_last_cpuinfo_used = VP8GetCPUInfo;
}
diff --git a/src/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c b/src/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c
index 49c666d..4186b9f 100644
--- a/src/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c
+++ b/src/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c
@@ -14,6 +14,7 @@
#include "./dsp.h"
#include "./lossless.h"
+#include "./lossless_common.h"
#if defined(WEBP_USE_MIPS32)
@@ -240,6 +241,49 @@ static WEBP_INLINE void GetEntropyUnrefinedHelper(
*i_prev = i;
}
+static void GetEntropyUnrefined(const uint32_t X[], int length,
+ VP8LBitEntropy* const bit_entropy,
+ VP8LStreaks* const stats) {
+ int i;
+ int i_prev = 0;
+ uint32_t x_prev = X[0];
+
+ memset(stats, 0, sizeof(*stats));
+ VP8LBitEntropyInit(bit_entropy);
+
+ for (i = 1; i < length; ++i) {
+ const uint32_t x = X[i];
+ if (x != x_prev) {
+ GetEntropyUnrefinedHelper(x, i, &x_prev, &i_prev, bit_entropy, stats);
+ }
+ }
+ GetEntropyUnrefinedHelper(0, i, &x_prev, &i_prev, bit_entropy, stats);
+
+ bit_entropy->entropy += VP8LFastSLog2(bit_entropy->sum);
+}
+
+static void GetCombinedEntropyUnrefined(const uint32_t X[], const uint32_t Y[],
+ int length,
+ VP8LBitEntropy* const bit_entropy,
+ VP8LStreaks* const stats) {
+ int i = 1;
+ int i_prev = 0;
+ uint32_t xy_prev = X[0] + Y[0];
+
+ memset(stats, 0, sizeof(*stats));
+ VP8LBitEntropyInit(bit_entropy);
+
+ for (i = 1; i < length; ++i) {
+ const uint32_t xy = X[i] + Y[i];
+ if (xy != xy_prev) {
+ GetEntropyUnrefinedHelper(xy, i, &xy_prev, &i_prev, bit_entropy, stats);
+ }
+ }
+ GetEntropyUnrefinedHelper(0, i, &xy_prev, &i_prev, bit_entropy, stats);
+
+ bit_entropy->entropy += VP8LFastSLog2(bit_entropy->sum);
+}
+
#define ASM_START \
__asm__ volatile( \
".set push \n\t" \
@@ -375,7 +419,8 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitMIPS32(void) {
VP8LFastLog2Slow = FastLog2Slow;
VP8LExtraCost = ExtraCost;
VP8LExtraCostCombined = ExtraCostCombined;
- VP8LGetEntropyUnrefinedHelper = GetEntropyUnrefinedHelper;
+ VP8LGetEntropyUnrefined = GetEntropyUnrefined;
+ VP8LGetCombinedEntropyUnrefined = GetCombinedEntropyUnrefined;
VP8LHistogramAdd = HistogramAdd;
}
diff --git a/src/3rdparty/libwebp/src/dsp/lossless_enc_msa.c b/src/3rdparty/libwebp/src/dsp/lossless_enc_msa.c
new file mode 100644
index 0000000..2f69ba3
--- /dev/null
+++ b/src/3rdparty/libwebp/src/dsp/lossless_enc_msa.c
@@ -0,0 +1,147 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the COPYING file in the root of the source
+// tree. An additional intellectual property rights grant can be found
+// in the file PATENTS. All contributing project authors may
+// be found in the AUTHORS file in the root of the source tree.
+// -----------------------------------------------------------------------------
+//
+// MSA variant of Image transform methods for lossless encoder.
+//
+// Authors: Prashant Patil (Prashant.Patil@imgtec.com)
+
+#include "./dsp.h"
+
+#if defined(WEBP_USE_MSA)
+
+#include "./lossless.h"
+#include "./msa_macro.h"
+
+#define TRANSFORM_COLOR_8(src0, src1, dst0, dst1, c0, c1, mask0, mask1) do { \
+ v8i16 g0, g1, t0, t1, t2, t3; \
+ v4i32 t4, t5; \
+ VSHF_B2_SH(src0, src0, src1, src1, mask0, mask0, g0, g1); \
+ DOTP_SB2_SH(g0, g1, c0, c0, t0, t1); \
+ SRAI_H2_SH(t0, t1, 5); \
+ t0 = __msa_subv_h((v8i16)src0, t0); \
+ t1 = __msa_subv_h((v8i16)src1, t1); \
+ t4 = __msa_srli_w((v4i32)src0, 16); \
+ t5 = __msa_srli_w((v4i32)src1, 16); \
+ DOTP_SB2_SH(t4, t5, c1, c1, t2, t3); \
+ SRAI_H2_SH(t2, t3, 5); \
+ SUB2(t0, t2, t1, t3, t0, t1); \
+ VSHF_B2_UB(src0, t0, src1, t1, mask1, mask1, dst0, dst1); \
+} while (0)
+
+#define TRANSFORM_COLOR_4(src, dst, c0, c1, mask0, mask1) do { \
+ const v16i8 g0 = VSHF_SB(src, src, mask0); \
+ v8i16 t0 = __msa_dotp_s_h(c0, g0); \
+ v8i16 t1; \
+ v4i32 t2; \
+ t0 = SRAI_H(t0, 5); \
+ t0 = __msa_subv_h((v8i16)src, t0); \
+ t2 = __msa_srli_w((v4i32)src, 16); \
+ t1 = __msa_dotp_s_h(c1, (v16i8)t2); \
+ t1 = SRAI_H(t1, 5); \
+ t0 = t0 - t1; \
+ dst = VSHF_UB(src, t0, mask1); \
+} while (0)
+
+static void TransformColor(const VP8LMultipliers* const m, uint32_t* data,
+ int num_pixels) {
+ v16u8 src0, dst0;
+ const v16i8 g2br = (v16i8)__msa_fill_w(m->green_to_blue_ |
+ (m->green_to_red_ << 16));
+ const v16i8 r2b = (v16i8)__msa_fill_w(m->red_to_blue_);
+ const v16u8 mask0 = { 1, 255, 1, 255, 5, 255, 5, 255, 9, 255, 9, 255,
+ 13, 255, 13, 255 };
+ const v16u8 mask1 = { 16, 1, 18, 3, 20, 5, 22, 7, 24, 9, 26, 11,
+ 28, 13, 30, 15 };
+
+ while (num_pixels >= 8) {
+ v16u8 src1, dst1;
+ LD_UB2(data, 4, src0, src1);
+ TRANSFORM_COLOR_8(src0, src1, dst0, dst1, g2br, r2b, mask0, mask1);
+ ST_UB2(dst0, dst1, data, 4);
+ data += 8;
+ num_pixels -= 8;
+ }
+ if (num_pixels > 0) {
+ if (num_pixels >= 4) {
+ src0 = LD_UB(data);
+ TRANSFORM_COLOR_4(src0, dst0, g2br, r2b, mask0, mask1);
+ ST_UB(dst0, data);
+ data += 4;
+ num_pixels -= 4;
+ }
+ if (num_pixels > 0) {
+ src0 = LD_UB(data);
+ TRANSFORM_COLOR_4(src0, dst0, g2br, r2b, mask0, mask1);
+ if (num_pixels == 3) {
+ const uint64_t pix_d = __msa_copy_s_d((v2i64)dst0, 0);
+ const uint32_t pix_w = __msa_copy_s_w((v4i32)dst0, 2);
+ SD(pix_d, data + 0);
+ SW(pix_w, data + 2);
+ } else if (num_pixels == 2) {
+ const uint64_t pix_d = __msa_copy_s_d((v2i64)dst0, 0);
+ SD(pix_d, data);
+ } else {
+ const uint32_t pix_w = __msa_copy_s_w((v4i32)dst0, 0);
+ SW(pix_w, data);
+ }
+ }
+ }
+}
+
+static void SubtractGreenFromBlueAndRed(uint32_t* argb_data, int num_pixels) {
+ int i;
+ uint8_t* ptemp_data = (uint8_t*)argb_data;
+ v16u8 src0, dst0, tmp0;
+ const v16u8 mask = { 1, 255, 1, 255, 5, 255, 5, 255, 9, 255, 9, 255,
+ 13, 255, 13, 255 };
+
+ while (num_pixels >= 8) {
+ v16u8 src1, dst1, tmp1;
+ LD_UB2(ptemp_data, 16, src0, src1);
+ VSHF_B2_UB(src0, src1, src1, src0, mask, mask, tmp0, tmp1);
+ SUB2(src0, tmp0, src1, tmp1, dst0, dst1);
+ ST_UB2(dst0, dst1, ptemp_data, 16);
+ ptemp_data += 8 * 4;
+ num_pixels -= 8;
+ }
+ if (num_pixels > 0) {
+ if (num_pixels >= 4) {
+ src0 = LD_UB(ptemp_data);
+ tmp0 = VSHF_UB(src0, src0, mask);
+ dst0 = src0 - tmp0;
+ ST_UB(dst0, ptemp_data);
+ ptemp_data += 4 * 4;
+ num_pixels -= 4;
+ }
+ for (i = 0; i < num_pixels; i++) {
+ const uint8_t b = ptemp_data[0];
+ const uint8_t g = ptemp_data[1];
+ const uint8_t r = ptemp_data[2];
+ ptemp_data[0] = (b - g) & 0xff;
+ ptemp_data[2] = (r - g) & 0xff;
+ ptemp_data += 4;
+ }
+ }
+}
+
+//------------------------------------------------------------------------------
+// Entry point
+
+extern void VP8LEncDspInitMSA(void);
+
+WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitMSA(void) {
+ VP8LSubtractGreenFromBlueAndRed = SubtractGreenFromBlueAndRed;
+ VP8LTransformColor = TransformColor;
+}
+
+#else // !WEBP_USE_MSA
+
+WEBP_DSP_INIT_STUB(VP8LEncDspInitMSA)
+
+#endif // WEBP_USE_MSA
diff --git a/src/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c b/src/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c
index 7c894e7..8ad85d9 100644
--- a/src/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c
+++ b/src/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c
@@ -17,6 +17,8 @@
#include <assert.h>
#include <emmintrin.h>
#include "./lossless.h"
+#include "./common_sse2.h"
+#include "./lossless_common.h"
// For sign-extended multiplying constants, pre-shifted by 5:
#define CST_5b(X) (((int16_t)((uint16_t)X << 8)) >> 5)
@@ -35,7 +37,9 @@ static void SubtractGreenFromBlueAndRed(uint32_t* argb_data, int num_pixels) {
_mm_storeu_si128((__m128i*)&argb_data[i], out);
}
// fallthrough and finish off with plain-C
- VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i);
+ if (i != num_pixels) {
+ VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i);
+ }
}
//------------------------------------------------------------------------------
@@ -69,7 +73,9 @@ static void TransformColor(const VP8LMultipliers* const m,
_mm_storeu_si128((__m128i*)&argb_data[i], out);
}
// fallthrough and finish off with plain-C
- VP8LTransformColor_C(m, argb_data + i, num_pixels - i);
+ if (i != num_pixels) {
+ VP8LTransformColor_C(m, argb_data + i, num_pixels - i);
+ }
}
//------------------------------------------------------------------------------
@@ -364,8 +370,9 @@ static int VectorMismatch(const uint32_t* const array1,
if (length >= 8 &&
_mm_movemask_epi8(_mm_cmpeq_epi32(
_mm_loadu_si128((const __m128i*)&array1[4]),
- _mm_loadu_si128((const __m128i*)&array2[4]))) == 0xffff)
+ _mm_loadu_si128((const __m128i*)&array2[4]))) == 0xffff) {
match_len = 8;
+ }
}
}
@@ -375,6 +382,295 @@ static int VectorMismatch(const uint32_t* const array1,
return match_len;
}
+// Bundles multiple (1, 2, 4 or 8) pixels into a single pixel.
+static void BundleColorMap_SSE2(const uint8_t* const row, int width, int xbits,
+ uint32_t* dst) {
+ int x;
+ assert(xbits >= 0);
+ assert(xbits <= 3);
+ switch (xbits) {
+ case 0: {
+ const __m128i ff = _mm_set1_epi16(0xff00);
+ const __m128i zero = _mm_setzero_si128();
+ // Store 0xff000000 | (row[x] << 8).
+ for (x = 0; x + 16 <= width; x += 16, dst += 16) {
+ const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]);
+ const __m128i in_lo = _mm_unpacklo_epi8(zero, in);
+ const __m128i dst0 = _mm_unpacklo_epi16(in_lo, ff);
+ const __m128i dst1 = _mm_unpackhi_epi16(in_lo, ff);
+ const __m128i in_hi = _mm_unpackhi_epi8(zero, in);
+ const __m128i dst2 = _mm_unpacklo_epi16(in_hi, ff);
+ const __m128i dst3 = _mm_unpackhi_epi16(in_hi, ff);
+ _mm_storeu_si128((__m128i*)&dst[0], dst0);
+ _mm_storeu_si128((__m128i*)&dst[4], dst1);
+ _mm_storeu_si128((__m128i*)&dst[8], dst2);
+ _mm_storeu_si128((__m128i*)&dst[12], dst3);
+ }
+ break;
+ }
+ case 1: {
+ const __m128i ff = _mm_set1_epi16(0xff00);
+ const __m128i mul = _mm_set1_epi16(0x110);
+ for (x = 0; x + 16 <= width; x += 16, dst += 8) {
+ // 0a0b | (where a/b are 4 bits).
+ const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]);
+ const __m128i tmp = _mm_mullo_epi16(in, mul); // aba0
+ const __m128i pack = _mm_and_si128(tmp, ff); // ab00
+ const __m128i dst0 = _mm_unpacklo_epi16(pack, ff);
+ const __m128i dst1 = _mm_unpackhi_epi16(pack, ff);
+ _mm_storeu_si128((__m128i*)&dst[0], dst0);
+ _mm_storeu_si128((__m128i*)&dst[4], dst1);
+ }
+ break;
+ }
+ case 2: {
+ const __m128i mask_or = _mm_set1_epi32(0xff000000);
+ const __m128i mul_cst = _mm_set1_epi16(0x0104);
+ const __m128i mask_mul = _mm_set1_epi16(0x0f00);
+ for (x = 0; x + 16 <= width; x += 16, dst += 4) {
+ // 000a000b000c000d | (where a/b/c/d are 2 bits).
+ const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]);
+ const __m128i mul = _mm_mullo_epi16(in, mul_cst); // 00ab00b000cd00d0
+ const __m128i tmp = _mm_and_si128(mul, mask_mul); // 00ab000000cd0000
+ const __m128i shift = _mm_srli_epi32(tmp, 12); // 00000000ab000000
+ const __m128i pack = _mm_or_si128(shift, tmp); // 00000000abcd0000
+ // Convert to 0xff00**00.
+ const __m128i res = _mm_or_si128(pack, mask_or);
+ _mm_storeu_si128((__m128i*)dst, res);
+ }
+ break;
+ }
+ default: {
+ assert(xbits == 3);
+ for (x = 0; x + 16 <= width; x += 16, dst += 2) {
+ // 0000000a00000000b... | (where a/b are 1 bit).
+ const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]);
+ const __m128i shift = _mm_slli_epi64(in, 7);
+ const uint32_t move = _mm_movemask_epi8(shift);
+ dst[0] = 0xff000000 | ((move & 0xff) << 8);
+ dst[1] = 0xff000000 | (move & 0xff00);
+ }
+ break;
+ }
+ }
+ if (x != width) {
+ VP8LBundleColorMap_C(row + x, width - x, xbits, dst);
+ }
+}
+
+//------------------------------------------------------------------------------
+// Batch version of Predictor Transform subtraction
+
+static WEBP_INLINE void Average2_m128i(const __m128i* const a0,
+ const __m128i* const a1,
+ __m128i* const avg) {
+ // (a + b) >> 1 = ((a + b + 1) >> 1) - ((a ^ b) & 1)
+ const __m128i ones = _mm_set1_epi8(1);
+ const __m128i avg1 = _mm_avg_epu8(*a0, *a1);
+ const __m128i one = _mm_and_si128(_mm_xor_si128(*a0, *a1), ones);
+ *avg = _mm_sub_epi8(avg1, one);
+}
+
+// Predictor0: ARGB_BLACK.
+static void PredictorSub0_SSE2(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ const __m128i black = _mm_set1_epi32(ARGB_BLACK);
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
+ const __m128i res = _mm_sub_epi8(src, black);
+ _mm_storeu_si128((__m128i*)&out[i], res);
+ }
+ if (i != num_pixels) {
+ VP8LPredictorsSub_C[0](in + i, upper + i, num_pixels - i, out + i);
+ }
+}
+
+#define GENERATE_PREDICTOR_1(X, IN) \
+static void PredictorSub##X##_SSE2(const uint32_t* in, const uint32_t* upper, \
+ int num_pixels, uint32_t* out) { \
+ int i; \
+ for (i = 0; i + 4 <= num_pixels; i += 4) { \
+ const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \
+ const __m128i pred = _mm_loadu_si128((const __m128i*)&(IN)); \
+ const __m128i res = _mm_sub_epi8(src, pred); \
+ _mm_storeu_si128((__m128i*)&out[i], res); \
+ } \
+ if (i != num_pixels) { \
+ VP8LPredictorsSub_C[(X)](in + i, upper + i, num_pixels - i, out + i); \
+ } \
+}
+
+GENERATE_PREDICTOR_1(1, in[i - 1]) // Predictor1: L
+GENERATE_PREDICTOR_1(2, upper[i]) // Predictor2: T
+GENERATE_PREDICTOR_1(3, upper[i + 1]) // Predictor3: TR
+GENERATE_PREDICTOR_1(4, upper[i - 1]) // Predictor4: TL
+#undef GENERATE_PREDICTOR_1
+
+// Predictor5: avg2(avg2(L, TR), T)
+static void PredictorSub5_SSE2(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]);
+ const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]);
+ const __m128i TR = _mm_loadu_si128((const __m128i*)&upper[i + 1]);
+ const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
+ __m128i avg, pred, res;
+ Average2_m128i(&L, &TR, &avg);
+ Average2_m128i(&avg, &T, &pred);
+ res = _mm_sub_epi8(src, pred);
+ _mm_storeu_si128((__m128i*)&out[i], res);
+ }
+ if (i != num_pixels) {
+ VP8LPredictorsSub_C[5](in + i, upper + i, num_pixels - i, out + i);
+ }
+}
+
+#define GENERATE_PREDICTOR_2(X, A, B) \
+static void PredictorSub##X##_SSE2(const uint32_t* in, const uint32_t* upper, \
+ int num_pixels, uint32_t* out) { \
+ int i; \
+ for (i = 0; i + 4 <= num_pixels; i += 4) { \
+ const __m128i tA = _mm_loadu_si128((const __m128i*)&(A)); \
+ const __m128i tB = _mm_loadu_si128((const __m128i*)&(B)); \
+ const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \
+ __m128i pred, res; \
+ Average2_m128i(&tA, &tB, &pred); \
+ res = _mm_sub_epi8(src, pred); \
+ _mm_storeu_si128((__m128i*)&out[i], res); \
+ } \
+ if (i != num_pixels) { \
+ VP8LPredictorsSub_C[(X)](in + i, upper + i, num_pixels - i, out + i); \
+ } \
+}
+
+GENERATE_PREDICTOR_2(6, in[i - 1], upper[i - 1]) // Predictor6: avg(L, TL)
+GENERATE_PREDICTOR_2(7, in[i - 1], upper[i]) // Predictor7: avg(L, T)
+GENERATE_PREDICTOR_2(8, upper[i - 1], upper[i]) // Predictor8: avg(TL, T)
+GENERATE_PREDICTOR_2(9, upper[i], upper[i + 1]) // Predictor9: average(T, TR)
+#undef GENERATE_PREDICTOR_2
+
+// Predictor10: avg(avg(L,TL), avg(T, TR)).
+static void PredictorSub10_SSE2(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]);
+ const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
+ const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]);
+ const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]);
+ const __m128i TR = _mm_loadu_si128((const __m128i*)&upper[i + 1]);
+ __m128i avgTTR, avgLTL, avg, res;
+ Average2_m128i(&T, &TR, &avgTTR);
+ Average2_m128i(&L, &TL, &avgLTL);
+ Average2_m128i(&avgTTR, &avgLTL, &avg);
+ res = _mm_sub_epi8(src, avg);
+ _mm_storeu_si128((__m128i*)&out[i], res);
+ }
+ if (i != num_pixels) {
+ VP8LPredictorsSub_C[10](in + i, upper + i, num_pixels - i, out + i);
+ }
+}
+
+// Predictor11: select.
+static void GetSumAbsDiff32(const __m128i* const A, const __m128i* const B,
+ __m128i* const out) {
+ // We can unpack with any value on the upper 32 bits, provided it's the same
+ // on both operands (to that their sum of abs diff is zero). Here we use *A.
+ const __m128i A_lo = _mm_unpacklo_epi32(*A, *A);
+ const __m128i B_lo = _mm_unpacklo_epi32(*B, *A);
+ const __m128i A_hi = _mm_unpackhi_epi32(*A, *A);
+ const __m128i B_hi = _mm_unpackhi_epi32(*B, *A);
+ const __m128i s_lo = _mm_sad_epu8(A_lo, B_lo);
+ const __m128i s_hi = _mm_sad_epu8(A_hi, B_hi);
+ *out = _mm_packs_epi32(s_lo, s_hi);
+}
+
+static void PredictorSub11_SSE2(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]);
+ const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]);
+ const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]);
+ const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
+ __m128i pa, pb;
+ GetSumAbsDiff32(&T, &TL, &pa); // pa = sum |T-TL|
+ GetSumAbsDiff32(&L, &TL, &pb); // pb = sum |L-TL|
+ {
+ const __m128i mask = _mm_cmpgt_epi32(pb, pa);
+ const __m128i A = _mm_and_si128(mask, L);
+ const __m128i B = _mm_andnot_si128(mask, T);
+ const __m128i pred = _mm_or_si128(A, B); // pred = (L > T)? L : T
+ const __m128i res = _mm_sub_epi8(src, pred);
+ _mm_storeu_si128((__m128i*)&out[i], res);
+ }
+ }
+ if (i != num_pixels) {
+ VP8LPredictorsSub_C[11](in + i, upper + i, num_pixels - i, out + i);
+ }
+}
+
+// Predictor12: ClampedSubSubtractFull.
+static void PredictorSub12_SSE2(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ const __m128i zero = _mm_setzero_si128();
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
+ const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]);
+ const __m128i L_lo = _mm_unpacklo_epi8(L, zero);
+ const __m128i L_hi = _mm_unpackhi_epi8(L, zero);
+ const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]);
+ const __m128i T_lo = _mm_unpacklo_epi8(T, zero);
+ const __m128i T_hi = _mm_unpackhi_epi8(T, zero);
+ const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]);
+ const __m128i TL_lo = _mm_unpacklo_epi8(TL, zero);
+ const __m128i TL_hi = _mm_unpackhi_epi8(TL, zero);
+ const __m128i diff_lo = _mm_sub_epi16(T_lo, TL_lo);
+ const __m128i diff_hi = _mm_sub_epi16(T_hi, TL_hi);
+ const __m128i pred_lo = _mm_add_epi16(L_lo, diff_lo);
+ const __m128i pred_hi = _mm_add_epi16(L_hi, diff_hi);
+ const __m128i pred = _mm_packus_epi16(pred_lo, pred_hi);
+ const __m128i res = _mm_sub_epi8(src, pred);
+ _mm_storeu_si128((__m128i*)&out[i], res);
+ }
+ if (i != num_pixels) {
+ VP8LPredictorsSub_C[12](in + i, upper + i, num_pixels - i, out + i);
+ }
+}
+
+// Predictors13: ClampedAddSubtractHalf
+static void PredictorSub13_SSE2(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ const __m128i zero = _mm_setzero_si128();
+ for (i = 0; i + 2 <= num_pixels; i += 2) {
+ // we can only process two pixels at a time
+ const __m128i L = _mm_loadl_epi64((const __m128i*)&in[i - 1]);
+ const __m128i src = _mm_loadl_epi64((const __m128i*)&in[i]);
+ const __m128i T = _mm_loadl_epi64((const __m128i*)&upper[i]);
+ const __m128i TL = _mm_loadl_epi64((const __m128i*)&upper[i - 1]);
+ const __m128i L_lo = _mm_unpacklo_epi8(L, zero);
+ const __m128i T_lo = _mm_unpacklo_epi8(T, zero);
+ const __m128i TL_lo = _mm_unpacklo_epi8(TL, zero);
+ const __m128i sum = _mm_add_epi16(T_lo, L_lo);
+ const __m128i avg = _mm_srli_epi16(sum, 1);
+ const __m128i A1 = _mm_sub_epi16(avg, TL_lo);
+ const __m128i bit_fix = _mm_cmpgt_epi16(TL_lo, avg);
+ const __m128i A2 = _mm_sub_epi16(A1, bit_fix);
+ const __m128i A3 = _mm_srai_epi16(A2, 1);
+ const __m128i A4 = _mm_add_epi16(avg, A3);
+ const __m128i pred = _mm_packus_epi16(A4, A4);
+ const __m128i res = _mm_sub_epi8(src, pred);
+ _mm_storel_epi64((__m128i*)&out[i], res);
+ }
+ if (i != num_pixels) {
+ VP8LPredictorsSub_C[13](in + i, upper + i, num_pixels - i, out + i);
+ }
+}
+
//------------------------------------------------------------------------------
// Entry point
@@ -388,6 +684,24 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitSSE2(void) {
VP8LHistogramAdd = HistogramAdd;
VP8LCombinedShannonEntropy = CombinedShannonEntropy;
VP8LVectorMismatch = VectorMismatch;
+ VP8LBundleColorMap = BundleColorMap_SSE2;
+
+ VP8LPredictorsSub[0] = PredictorSub0_SSE2;
+ VP8LPredictorsSub[1] = PredictorSub1_SSE2;
+ VP8LPredictorsSub[2] = PredictorSub2_SSE2;
+ VP8LPredictorsSub[3] = PredictorSub3_SSE2;
+ VP8LPredictorsSub[4] = PredictorSub4_SSE2;
+ VP8LPredictorsSub[5] = PredictorSub5_SSE2;
+ VP8LPredictorsSub[6] = PredictorSub6_SSE2;
+ VP8LPredictorsSub[7] = PredictorSub7_SSE2;
+ VP8LPredictorsSub[8] = PredictorSub8_SSE2;
+ VP8LPredictorsSub[9] = PredictorSub9_SSE2;
+ VP8LPredictorsSub[10] = PredictorSub10_SSE2;
+ VP8LPredictorsSub[11] = PredictorSub11_SSE2;
+ VP8LPredictorsSub[12] = PredictorSub12_SSE2;
+ VP8LPredictorsSub[13] = PredictorSub13_SSE2;
+ VP8LPredictorsSub[14] = PredictorSub0_SSE2; // <- padding security sentinels
+ VP8LPredictorsSub[15] = PredictorSub0_SSE2;
}
#else // !WEBP_USE_SSE2
diff --git a/src/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c b/src/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c
index 3e49319..821057c 100644
--- a/src/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c
+++ b/src/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c
@@ -32,7 +32,9 @@ static void SubtractGreenFromBlueAndRed(uint32_t* argb_data, int num_pixels) {
_mm_storeu_si128((__m128i*)&argb_data[i], out);
}
// fallthrough and finish off with plain-C
- VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i);
+ if (i != num_pixels) {
+ VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i);
+ }
}
//------------------------------------------------------------------------------
diff --git a/src/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c b/src/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c
index 90aed7f..2984ce8 100644
--- a/src/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c
+++ b/src/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c
@@ -17,6 +17,7 @@
#if defined(WEBP_USE_MIPS_DSP_R2)
#include "./lossless.h"
+#include "./lossless_common.h"
#define MAP_COLOR_FUNCS(FUNC_NAME, TYPE, GET_INDEX, GET_VALUE) \
static void FUNC_NAME(const TYPE* src, \
@@ -227,25 +228,27 @@ static uint32_t Predictor13(uint32_t left, const uint32_t* const top) {
// Add green to blue and red channels (i.e. perform the inverse transform of
// 'subtract green').
-static void AddGreenToBlueAndRed(uint32_t* data, int num_pixels) {
+static void AddGreenToBlueAndRed(const uint32_t* src, int num_pixels,
+ uint32_t* dst) {
uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7;
- uint32_t* const p_loop1_end = data + (num_pixels & ~3);
- uint32_t* const p_loop2_end = data + num_pixels;
+ const uint32_t* const p_loop1_end = src + (num_pixels & ~3);
+ const uint32_t* const p_loop2_end = src + num_pixels;
__asm__ volatile (
".set push \n\t"
".set noreorder \n\t"
- "beq %[data], %[p_loop1_end], 3f \n\t"
+ "beq %[src], %[p_loop1_end], 3f \n\t"
" nop \n\t"
"0: \n\t"
- "lw %[temp0], 0(%[data]) \n\t"
- "lw %[temp1], 4(%[data]) \n\t"
- "lw %[temp2], 8(%[data]) \n\t"
- "lw %[temp3], 12(%[data]) \n\t"
+ "lw %[temp0], 0(%[src]) \n\t"
+ "lw %[temp1], 4(%[src]) \n\t"
+ "lw %[temp2], 8(%[src]) \n\t"
+ "lw %[temp3], 12(%[src]) \n\t"
"ext %[temp4], %[temp0], 8, 8 \n\t"
"ext %[temp5], %[temp1], 8, 8 \n\t"
"ext %[temp6], %[temp2], 8, 8 \n\t"
"ext %[temp7], %[temp3], 8, 8 \n\t"
- "addiu %[data], %[data], 16 \n\t"
+ "addiu %[src], %[src], 16 \n\t"
+ "addiu %[dst], %[dst], 16 \n\t"
"replv.ph %[temp4], %[temp4] \n\t"
"replv.ph %[temp5], %[temp5] \n\t"
"replv.ph %[temp6], %[temp6] \n\t"
@@ -254,44 +257,47 @@ static void AddGreenToBlueAndRed(uint32_t* data, int num_pixels) {
"addu.qb %[temp1], %[temp1], %[temp5] \n\t"
"addu.qb %[temp2], %[temp2], %[temp6] \n\t"
"addu.qb %[temp3], %[temp3], %[temp7] \n\t"
- "sw %[temp0], -16(%[data]) \n\t"
- "sw %[temp1], -12(%[data]) \n\t"
- "sw %[temp2], -8(%[data]) \n\t"
- "bne %[data], %[p_loop1_end], 0b \n\t"
- " sw %[temp3], -4(%[data]) \n\t"
+ "sw %[temp0], -16(%[dst]) \n\t"
+ "sw %[temp1], -12(%[dst]) \n\t"
+ "sw %[temp2], -8(%[dst]) \n\t"
+ "bne %[src], %[p_loop1_end], 0b \n\t"
+ " sw %[temp3], -4(%[dst]) \n\t"
"3: \n\t"
- "beq %[data], %[p_loop2_end], 2f \n\t"
+ "beq %[src], %[p_loop2_end], 2f \n\t"
" nop \n\t"
"1: \n\t"
- "lw %[temp0], 0(%[data]) \n\t"
- "addiu %[data], %[data], 4 \n\t"
+ "lw %[temp0], 0(%[src]) \n\t"
+ "addiu %[src], %[src], 4 \n\t"
+ "addiu %[dst], %[dst], 4 \n\t"
"ext %[temp4], %[temp0], 8, 8 \n\t"
"replv.ph %[temp4], %[temp4] \n\t"
"addu.qb %[temp0], %[temp0], %[temp4] \n\t"
- "bne %[data], %[p_loop2_end], 1b \n\t"
- " sw %[temp0], -4(%[data]) \n\t"
+ "bne %[src], %[p_loop2_end], 1b \n\t"
+ " sw %[temp0], -4(%[dst]) \n\t"
"2: \n\t"
".set pop \n\t"
- : [data]"+&r"(data), [temp0]"=&r"(temp0), [temp1]"=&r"(temp1),
- [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), [temp4]"=&r"(temp4),
- [temp5]"=&r"(temp5), [temp6]"=&r"(temp6), [temp7]"=&r"(temp7)
+ : [dst]"+&r"(dst), [src]"+&r"(src), [temp0]"=&r"(temp0),
+ [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), [temp3]"=&r"(temp3),
+ [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [temp6]"=&r"(temp6),
+ [temp7]"=&r"(temp7)
: [p_loop1_end]"r"(p_loop1_end), [p_loop2_end]"r"(p_loop2_end)
: "memory"
);
}
static void TransformColorInverse(const VP8LMultipliers* const m,
- uint32_t* data, int num_pixels) {
+ const uint32_t* src, int num_pixels,
+ uint32_t* dst) {
int temp0, temp1, temp2, temp3, temp4, temp5;
uint32_t argb, argb1, new_red;
const uint32_t G_to_R = m->green_to_red_;
const uint32_t G_to_B = m->green_to_blue_;
const uint32_t R_to_B = m->red_to_blue_;
- uint32_t* const p_loop_end = data + (num_pixels & ~1);
+ const uint32_t* const p_loop_end = src + (num_pixels & ~1);
__asm__ volatile (
".set push \n\t"
".set noreorder \n\t"
- "beq %[data], %[p_loop_end], 1f \n\t"
+ "beq %[src], %[p_loop_end], 1f \n\t"
" nop \n\t"
"replv.ph %[temp0], %[G_to_R] \n\t"
"replv.ph %[temp1], %[G_to_B] \n\t"
@@ -303,9 +309,12 @@ static void TransformColorInverse(const VP8LMultipliers* const m,
"shra.ph %[temp1], %[temp1], 8 \n\t"
"shra.ph %[temp2], %[temp2], 8 \n\t"
"0: \n\t"
- "lw %[argb], 0(%[data]) \n\t"
- "lw %[argb1], 4(%[data]) \n\t"
- "addiu %[data], %[data], 8 \n\t"
+ "lw %[argb], 0(%[src]) \n\t"
+ "lw %[argb1], 4(%[src]) \n\t"
+ "sw %[argb], 0(%[dst]) \n\t"
+ "sw %[argb1], 4(%[dst]) \n\t"
+ "addiu %[src], %[src], 8 \n\t"
+ "addiu %[dst], %[dst], 8 \n\t"
"precrq.qb.ph %[temp3], %[argb], %[argb1] \n\t"
"preceu.ph.qbra %[temp3], %[temp3] \n\t"
"shll.ph %[temp3], %[temp3], 8 \n\t"
@@ -322,29 +331,29 @@ static void TransformColorInverse(const VP8LMultipliers* const m,
"shll.ph %[temp4], %[temp5], 8 \n\t"
"shra.ph %[temp4], %[temp4], 8 \n\t"
"mul.ph %[temp4], %[temp4], %[temp2] \n\t"
- "sb %[temp5], -2(%[data]) \n\t"
+ "sb %[temp5], -2(%[dst]) \n\t"
"sra %[temp5], %[temp5], 16 \n\t"
"shra.ph %[temp4], %[temp4], 5 \n\t"
"addu.ph %[argb1], %[argb1], %[temp4] \n\t"
"preceu.ph.qbra %[temp3], %[argb1] \n\t"
- "sb %[temp5], -6(%[data]) \n\t"
- "sb %[temp3], -4(%[data]) \n\t"
+ "sb %[temp5], -6(%[dst]) \n\t"
+ "sb %[temp3], -4(%[dst]) \n\t"
"sra %[temp3], %[temp3], 16 \n\t"
- "bne %[data], %[p_loop_end], 0b \n\t"
- " sb %[temp3], -8(%[data]) \n\t"
+ "bne %[src], %[p_loop_end], 0b \n\t"
+ " sb %[temp3], -8(%[dst]) \n\t"
"1: \n\t"
".set pop \n\t"
: [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
[temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
[new_red]"=&r"(new_red), [argb]"=&r"(argb),
- [argb1]"=&r"(argb1), [data]"+&r"(data)
+ [argb1]"=&r"(argb1), [dst]"+&r"(dst), [src]"+&r"(src)
: [G_to_R]"r"(G_to_R), [R_to_B]"r"(R_to_B),
[G_to_B]"r"(G_to_B), [p_loop_end]"r"(p_loop_end)
: "memory", "hi", "lo"
);
// Fall-back to C-version for left-overs.
- if (num_pixels & 1) VP8LTransformColorInverse_C(m, data, 1);
+ if (num_pixels & 1) VP8LTransformColorInverse_C(m, src, 1, dst);
}
static void ConvertBGRAToRGB(const uint32_t* src,
diff --git a/src/3rdparty/libwebp/src/dsp/lossless_msa.c b/src/3rdparty/libwebp/src/dsp/lossless_msa.c
new file mode 100644
index 0000000..f6dd564
--- /dev/null
+++ b/src/3rdparty/libwebp/src/dsp/lossless_msa.c
@@ -0,0 +1,355 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the COPYING file in the root of the source
+// tree. An additional intellectual property rights grant can be found
+// in the file PATENTS. All contributing project authors may
+// be found in the AUTHORS file in the root of the source tree.
+// -----------------------------------------------------------------------------
+//
+// MSA variant of methods for lossless decoder
+//
+// Author: Prashant Patil (prashant.patil@imgtec.com)
+
+#include "./dsp.h"
+
+#if defined(WEBP_USE_MSA)
+
+#include "./lossless.h"
+#include "./msa_macro.h"
+
+//------------------------------------------------------------------------------
+// Colorspace conversion functions
+
+#define CONVERT16_BGRA_XXX(psrc, pdst, m0, m1, m2) do { \
+ v16u8 src0, src1, src2, src3, dst0, dst1, dst2; \
+ LD_UB4(psrc, 16, src0, src1, src2, src3); \
+ VSHF_B2_UB(src0, src1, src1, src2, m0, m1, dst0, dst1); \
+ dst2 = VSHF_UB(src2, src3, m2); \
+ ST_UB2(dst0, dst1, pdst, 16); \
+ ST_UB(dst2, pdst + 32); \
+} while (0)
+
+#define CONVERT12_BGRA_XXX(psrc, pdst, m0, m1, m2) do { \
+ uint32_t pix_w; \
+ v16u8 src0, src1, src2, dst0, dst1, dst2; \
+ LD_UB3(psrc, 16, src0, src1, src2); \
+ VSHF_B2_UB(src0, src1, src1, src2, m0, m1, dst0, dst1); \
+ dst2 = VSHF_UB(src2, src2, m2); \
+ ST_UB2(dst0, dst1, pdst, 16); \
+ pix_w = __msa_copy_s_w((v4i32)dst2, 0); \
+ SW(pix_w, pdst + 32); \
+} while (0)
+
+#define CONVERT8_BGRA_XXX(psrc, pdst, m0, m1) do { \
+ uint64_t pix_d; \
+ v16u8 src0, src1, src2, dst0, dst1; \
+ LD_UB2(psrc, 16, src0, src1); \
+ VSHF_B2_UB(src0, src1, src1, src2, m0, m1, dst0, dst1); \
+ ST_UB(dst0, pdst); \
+ pix_d = __msa_copy_s_d((v2i64)dst1, 0); \
+ SD(pix_d, pdst + 16); \
+} while (0)
+
+#define CONVERT4_BGRA_XXX(psrc, pdst, m) do { \
+ const v16u8 src0 = LD_UB(psrc); \
+ const v16u8 dst0 = VSHF_UB(src0, src0, m); \
+ uint64_t pix_d = __msa_copy_s_d((v2i64)dst0, 0); \
+ uint32_t pix_w = __msa_copy_s_w((v4i32)dst0, 2); \
+ SD(pix_d, pdst + 0); \
+ SW(pix_w, pdst + 8); \
+} while (0)
+
+#define CONVERT1_BGRA_BGR(psrc, pdst) do { \
+ const int32_t b = (psrc)[0]; \
+ const int32_t g = (psrc)[1]; \
+ const int32_t r = (psrc)[2]; \
+ (pdst)[0] = b; \
+ (pdst)[1] = g; \
+ (pdst)[2] = r; \
+} while (0)
+
+#define CONVERT1_BGRA_RGB(psrc, pdst) do { \
+ const int32_t b = (psrc)[0]; \
+ const int32_t g = (psrc)[1]; \
+ const int32_t r = (psrc)[2]; \
+ (pdst)[0] = r; \
+ (pdst)[1] = g; \
+ (pdst)[2] = b; \
+} while (0)
+
+#define TRANSFORM_COLOR_INVERSE_8(src0, src1, dst0, dst1, \
+ c0, c1, mask0, mask1) do { \
+ v8i16 g0, g1, t0, t1, t2, t3; \
+ v4i32 t4, t5; \
+ VSHF_B2_SH(src0, src0, src1, src1, mask0, mask0, g0, g1); \
+ DOTP_SB2_SH(g0, g1, c0, c0, t0, t1); \
+ SRAI_H2_SH(t0, t1, 5); \
+ t0 = __msa_addv_h(t0, (v8i16)src0); \
+ t1 = __msa_addv_h(t1, (v8i16)src1); \
+ t4 = __msa_srli_w((v4i32)t0, 16); \
+ t5 = __msa_srli_w((v4i32)t1, 16); \
+ DOTP_SB2_SH(t4, t5, c1, c1, t2, t3); \
+ SRAI_H2_SH(t2, t3, 5); \
+ ADD2(t0, t2, t1, t3, t0, t1); \
+ VSHF_B2_UB(src0, t0, src1, t1, mask1, mask1, dst0, dst1); \
+} while (0)
+
+#define TRANSFORM_COLOR_INVERSE_4(src, dst, c0, c1, mask0, mask1) do { \
+ const v16i8 g0 = VSHF_SB(src, src, mask0); \
+ v8i16 t0 = __msa_dotp_s_h(c0, g0); \
+ v8i16 t1; \
+ v4i32 t2; \
+ t0 = SRAI_H(t0, 5); \
+ t0 = __msa_addv_h(t0, (v8i16)src); \
+ t2 = __msa_srli_w((v4i32)t0, 16); \
+ t1 = __msa_dotp_s_h(c1, (v16i8)t2); \
+ t1 = SRAI_H(t1, 5); \
+ t0 = t0 + t1; \
+ dst = VSHF_UB(src, t0, mask1); \
+} while (0)
+
+static void ConvertBGRAToRGBA(const uint32_t* src,
+ int num_pixels, uint8_t* dst) {
+ int i;
+ const uint8_t* ptemp_src = (const uint8_t*)src;
+ uint8_t* ptemp_dst = (uint8_t*)dst;
+ v16u8 src0, dst0;
+ const v16u8 mask = { 2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15 };
+
+ while (num_pixels >= 8) {
+ v16u8 src1, dst1;
+ LD_UB2(ptemp_src, 16, src0, src1);
+ VSHF_B2_UB(src0, src0, src1, src1, mask, mask, dst0, dst1);
+ ST_UB2(dst0, dst1, ptemp_dst, 16);
+ ptemp_src += 32;
+ ptemp_dst += 32;
+ num_pixels -= 8;
+ }
+ if (num_pixels > 0) {
+ if (num_pixels >= 4) {
+ src0 = LD_UB(ptemp_src);
+ dst0 = VSHF_UB(src0, src0, mask);
+ ST_UB(dst0, ptemp_dst);
+ ptemp_src += 16;
+ ptemp_dst += 16;
+ num_pixels -= 4;
+ }
+ for (i = 0; i < num_pixels; i++) {
+ const uint8_t b = ptemp_src[2];
+ const uint8_t g = ptemp_src[1];
+ const uint8_t r = ptemp_src[0];
+ const uint8_t a = ptemp_src[3];
+ ptemp_dst[0] = b;
+ ptemp_dst[1] = g;
+ ptemp_dst[2] = r;
+ ptemp_dst[3] = a;
+ ptemp_src += 4;
+ ptemp_dst += 4;
+ }
+ }
+}
+
+static void ConvertBGRAToBGR(const uint32_t* src,
+ int num_pixels, uint8_t* dst) {
+ const uint8_t* ptemp_src = (const uint8_t*)src;
+ uint8_t* ptemp_dst = (uint8_t*)dst;
+ const v16u8 mask0 = { 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14,
+ 16, 17, 18, 20 };
+ const v16u8 mask1 = { 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20,
+ 21, 22, 24, 25 };
+ const v16u8 mask2 = { 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25,
+ 26, 28, 29, 30 };
+
+ while (num_pixels >= 16) {
+ CONVERT16_BGRA_XXX(ptemp_src, ptemp_dst, mask0, mask1, mask2);
+ ptemp_src += 64;
+ ptemp_dst += 48;
+ num_pixels -= 16;
+ }
+ if (num_pixels > 0) {
+ if (num_pixels >= 12) {
+ CONVERT12_BGRA_XXX(ptemp_src, ptemp_dst, mask0, mask1, mask2);
+ ptemp_src += 48;
+ ptemp_dst += 36;
+ num_pixels -= 12;
+ } else if (num_pixels >= 8) {
+ CONVERT8_BGRA_XXX(ptemp_src, ptemp_dst, mask0, mask1);
+ ptemp_src += 32;
+ ptemp_dst += 24;
+ num_pixels -= 8;
+ } else if (num_pixels >= 4) {
+ CONVERT4_BGRA_XXX(ptemp_src, ptemp_dst, mask0);
+ ptemp_src += 16;
+ ptemp_dst += 12;
+ num_pixels -= 4;
+ }
+ if (num_pixels == 3) {
+ CONVERT1_BGRA_BGR(ptemp_src + 0, ptemp_dst + 0);
+ CONVERT1_BGRA_BGR(ptemp_src + 4, ptemp_dst + 3);
+ CONVERT1_BGRA_BGR(ptemp_src + 8, ptemp_dst + 6);
+ } else if (num_pixels == 2) {
+ CONVERT1_BGRA_BGR(ptemp_src + 0, ptemp_dst + 0);
+ CONVERT1_BGRA_BGR(ptemp_src + 4, ptemp_dst + 3);
+ } else if (num_pixels == 1) {
+ CONVERT1_BGRA_BGR(ptemp_src, ptemp_dst);
+ }
+ }
+}
+
+static void ConvertBGRAToRGB(const uint32_t* src,
+ int num_pixels, uint8_t* dst) {
+ const uint8_t* ptemp_src = (const uint8_t*)src;
+ uint8_t* ptemp_dst = (uint8_t*)dst;
+ const v16u8 mask0 = { 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12,
+ 18, 17, 16, 22 };
+ const v16u8 mask1 = { 5, 4, 10, 9, 8, 14, 13, 12, 18, 17, 16, 22,
+ 21, 20, 26, 25 };
+ const v16u8 mask2 = { 8, 14, 13, 12, 18, 17, 16, 22, 21, 20, 26, 25,
+ 24, 30, 29, 28 };
+
+ while (num_pixels >= 16) {
+ CONVERT16_BGRA_XXX(ptemp_src, ptemp_dst, mask0, mask1, mask2);
+ ptemp_src += 64;
+ ptemp_dst += 48;
+ num_pixels -= 16;
+ }
+ if (num_pixels) {
+ if (num_pixels >= 12) {
+ CONVERT12_BGRA_XXX(ptemp_src, ptemp_dst, mask0, mask1, mask2);
+ ptemp_src += 48;
+ ptemp_dst += 36;
+ num_pixels -= 12;
+ } else if (num_pixels >= 8) {
+ CONVERT8_BGRA_XXX(ptemp_src, ptemp_dst, mask0, mask1);
+ ptemp_src += 32;
+ ptemp_dst += 24;
+ num_pixels -= 8;
+ } else if (num_pixels >= 4) {
+ CONVERT4_BGRA_XXX(ptemp_src, ptemp_dst, mask0);
+ ptemp_src += 16;
+ ptemp_dst += 12;
+ num_pixels -= 4;
+ }
+ if (num_pixels == 3) {
+ CONVERT1_BGRA_RGB(ptemp_src + 0, ptemp_dst + 0);
+ CONVERT1_BGRA_RGB(ptemp_src + 4, ptemp_dst + 3);
+ CONVERT1_BGRA_RGB(ptemp_src + 8, ptemp_dst + 6);
+ } else if (num_pixels == 2) {
+ CONVERT1_BGRA_RGB(ptemp_src + 0, ptemp_dst + 0);
+ CONVERT1_BGRA_RGB(ptemp_src + 4, ptemp_dst + 3);
+ } else if (num_pixels == 1) {
+ CONVERT1_BGRA_RGB(ptemp_src, ptemp_dst);
+ }
+ }
+}
+
+static void AddGreenToBlueAndRed(const uint32_t* const src, int num_pixels,
+ uint32_t* dst) {
+ int i;
+ const uint8_t* in = (const uint8_t*)src;
+ uint8_t* out = (uint8_t*)dst;
+ v16u8 src0, dst0, tmp0;
+ const v16u8 mask = { 1, 255, 1, 255, 5, 255, 5, 255, 9, 255, 9, 255,
+ 13, 255, 13, 255 };
+
+ while (num_pixels >= 8) {
+ v16u8 src1, dst1, tmp1;
+ LD_UB2(in, 16, src0, src1);
+ VSHF_B2_UB(src0, src1, src1, src0, mask, mask, tmp0, tmp1);
+ ADD2(src0, tmp0, src1, tmp1, dst0, dst1);
+ ST_UB2(dst0, dst1, out, 16);
+ in += 32;
+ out += 32;
+ num_pixels -= 8;
+ }
+ if (num_pixels > 0) {
+ if (num_pixels >= 4) {
+ src0 = LD_UB(in);
+ tmp0 = VSHF_UB(src0, src0, mask);
+ dst0 = src0 + tmp0;
+ ST_UB(dst0, out);
+ in += 16;
+ out += 16;
+ num_pixels -= 4;
+ }
+ for (i = 0; i < num_pixels; i++) {
+ const uint8_t b = in[0];
+ const uint8_t g = in[1];
+ const uint8_t r = in[2];
+ out[0] = (b + g) & 0xff;
+ out[1] = g;
+ out[2] = (r + g) & 0xff;
+ out[4] = in[4];
+ out += 4;
+ }
+ }
+}
+
+static void TransformColorInverse(const VP8LMultipliers* const m,
+ const uint32_t* src, int num_pixels,
+ uint32_t* dst) {
+ v16u8 src0, dst0;
+ const v16i8 g2br = (v16i8)__msa_fill_w(m->green_to_blue_ |
+ (m->green_to_red_ << 16));
+ const v16i8 r2b = (v16i8)__msa_fill_w(m->red_to_blue_);
+ const v16u8 mask0 = { 1, 255, 1, 255, 5, 255, 5, 255, 9, 255, 9, 255,
+ 13, 255, 13, 255 };
+ const v16u8 mask1 = { 16, 1, 18, 3, 20, 5, 22, 7, 24, 9, 26, 11,
+ 28, 13, 30, 15 };
+
+ while (num_pixels >= 8) {
+ v16u8 src1, dst1;
+ LD_UB2(src, 4, src0, src1);
+ TRANSFORM_COLOR_INVERSE_8(src0, src1, dst0, dst1, g2br, r2b, mask0, mask1);
+ ST_UB2(dst0, dst1, dst, 4);
+ src += 8;
+ dst += 8;
+ num_pixels -= 8;
+ }
+ if (num_pixels > 0) {
+ if (num_pixels >= 4) {
+ src0 = LD_UB(src);
+ TRANSFORM_COLOR_INVERSE_4(src0, dst0, g2br, r2b, mask0, mask1);
+ ST_UB(dst0, dst);
+ src += 4;
+ dst += 4;
+ num_pixels -= 4;
+ }
+ if (num_pixels > 0) {
+ src0 = LD_UB(src);
+ TRANSFORM_COLOR_INVERSE_4(src0, dst0, g2br, r2b, mask0, mask1);
+ if (num_pixels == 3) {
+ const uint64_t pix_d = __msa_copy_s_d((v2i64)dst0, 0);
+ const uint32_t pix_w = __msa_copy_s_w((v4i32)dst0, 2);
+ SD(pix_d, dst + 0);
+ SW(pix_w, dst + 2);
+ } else if (num_pixels == 2) {
+ const uint64_t pix_d = __msa_copy_s_d((v2i64)dst0, 0);
+ SD(pix_d, dst);
+ } else {
+ const uint32_t pix_w = __msa_copy_s_w((v4i32)dst0, 0);
+ SW(pix_w, dst);
+ }
+ }
+ }
+}
+
+//------------------------------------------------------------------------------
+// Entry point
+
+extern void VP8LDspInitMSA(void);
+
+WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInitMSA(void) {
+ VP8LConvertBGRAToRGBA = ConvertBGRAToRGBA;
+ VP8LConvertBGRAToBGR = ConvertBGRAToBGR;
+ VP8LConvertBGRAToRGB = ConvertBGRAToRGB;
+ VP8LAddGreenToBlueAndRed = AddGreenToBlueAndRed;
+ VP8LTransformColorInverse = TransformColorInverse;
+}
+
+#else // !WEBP_USE_MSA
+
+WEBP_DSP_INIT_STUB(VP8LDspInitMSA)
+
+#endif // WEBP_USE_MSA
diff --git a/src/3rdparty/libwebp/src/dsp/lossless_neon.c b/src/3rdparty/libwebp/src/dsp/lossless_neon.c
index 6faccb8..1145d5f 100644
--- a/src/3rdparty/libwebp/src/dsp/lossless_neon.c
+++ b/src/3rdparty/libwebp/src/dsp/lossless_neon.c
@@ -139,6 +139,357 @@ static void ConvertBGRAToRGB(const uint32_t* src,
#endif // !WORK_AROUND_GCC
+
+//------------------------------------------------------------------------------
+// Predictor Transform
+
+#define LOAD_U32_AS_U8(IN) vreinterpret_u8_u32(vdup_n_u32((IN)))
+#define LOAD_U32P_AS_U8(IN) vreinterpret_u8_u32(vld1_u32((IN)))
+#define LOADQ_U32_AS_U8(IN) vreinterpretq_u8_u32(vdupq_n_u32((IN)))
+#define LOADQ_U32P_AS_U8(IN) vreinterpretq_u8_u32(vld1q_u32((IN)))
+#define GET_U8_AS_U32(IN) vget_lane_u32(vreinterpret_u32_u8((IN)), 0);
+#define GETQ_U8_AS_U32(IN) vgetq_lane_u32(vreinterpretq_u32_u8((IN)), 0);
+#define STOREQ_U8_AS_U32P(OUT, IN) vst1q_u32((OUT), vreinterpretq_u32_u8((IN)));
+#define ROTATE32_LEFT(L) vextq_u8((L), (L), 12) // D|C|B|A -> C|B|A|D
+
+static WEBP_INLINE uint8x8_t Average2_u8_NEON(uint32_t a0, uint32_t a1) {
+ const uint8x8_t A0 = LOAD_U32_AS_U8(a0);
+ const uint8x8_t A1 = LOAD_U32_AS_U8(a1);
+ return vhadd_u8(A0, A1);
+}
+
+static WEBP_INLINE uint32_t ClampedAddSubtractHalf_NEON(uint32_t c0,
+ uint32_t c1,
+ uint32_t c2) {
+ const uint8x8_t avg = Average2_u8_NEON(c0, c1);
+ // Remove one to c2 when bigger than avg.
+ const uint8x8_t C2 = LOAD_U32_AS_U8(c2);
+ const uint8x8_t cmp = vcgt_u8(C2, avg);
+ const uint8x8_t C2_1 = vadd_u8(C2, cmp);
+ // Compute half of the difference between avg and c2.
+ const int8x8_t diff_avg = vreinterpret_s8_u8(vhsub_u8(avg, C2_1));
+ // Compute the sum with avg and saturate.
+ const int16x8_t avg_16 = vreinterpretq_s16_u16(vmovl_u8(avg));
+ const uint8x8_t res = vqmovun_s16(vaddw_s8(avg_16, diff_avg));
+ const uint32_t output = GET_U8_AS_U32(res);
+ return output;
+}
+
+static WEBP_INLINE uint32_t Average2_NEON(uint32_t a0, uint32_t a1) {
+ const uint8x8_t avg_u8x8 = Average2_u8_NEON(a0, a1);
+ const uint32_t avg = GET_U8_AS_U32(avg_u8x8);
+ return avg;
+}
+
+static WEBP_INLINE uint32_t Average3_NEON(uint32_t a0, uint32_t a1,
+ uint32_t a2) {
+ const uint8x8_t avg0 = Average2_u8_NEON(a0, a2);
+ const uint8x8_t A1 = LOAD_U32_AS_U8(a1);
+ const uint32_t avg = GET_U8_AS_U32(vhadd_u8(avg0, A1));
+ return avg;
+}
+
+static uint32_t Predictor5_NEON(uint32_t left, const uint32_t* const top) {
+ return Average3_NEON(left, top[0], top[1]);
+}
+static uint32_t Predictor6_NEON(uint32_t left, const uint32_t* const top) {
+ return Average2_NEON(left, top[-1]);
+}
+static uint32_t Predictor7_NEON(uint32_t left, const uint32_t* const top) {
+ return Average2_NEON(left, top[0]);
+}
+static uint32_t Predictor13_NEON(uint32_t left, const uint32_t* const top) {
+ return ClampedAddSubtractHalf_NEON(left, top[0], top[-1]);
+}
+
+// Batch versions of those functions.
+
+// Predictor0: ARGB_BLACK.
+static void PredictorAdd0_NEON(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ const uint8x16_t black = vreinterpretq_u8_u32(vdupq_n_u32(ARGB_BLACK));
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]);
+ const uint8x16_t res = vaddq_u8(src, black);
+ STOREQ_U8_AS_U32P(&out[i], res);
+ }
+ VP8LPredictorsAdd_C[0](in + i, upper + i, num_pixels - i, out + i);
+}
+
+// Predictor1: left.
+static void PredictorAdd1_NEON(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ const uint8x16_t zero = LOADQ_U32_AS_U8(0);
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ // a | b | c | d
+ const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]);
+ // 0 | a | b | c
+ const uint8x16_t shift0 = vextq_u8(zero, src, 12);
+ // a | a + b | b + c | c + d
+ const uint8x16_t sum0 = vaddq_u8(src, shift0);
+ // 0 | 0 | a | a + b
+ const uint8x16_t shift1 = vextq_u8(zero, sum0, 8);
+ // a | a + b | a + b + c | a + b + c + d
+ const uint8x16_t sum1 = vaddq_u8(sum0, shift1);
+ const uint8x16_t prev = LOADQ_U32_AS_U8(out[i - 1]);
+ const uint8x16_t res = vaddq_u8(sum1, prev);
+ STOREQ_U8_AS_U32P(&out[i], res);
+ }
+ VP8LPredictorsAdd_C[1](in + i, upper + i, num_pixels - i, out + i);
+}
+
+// Macro that adds 32-bit integers from IN using mod 256 arithmetic
+// per 8 bit channel.
+#define GENERATE_PREDICTOR_1(X, IN) \
+static void PredictorAdd##X##_NEON(const uint32_t* in, \
+ const uint32_t* upper, int num_pixels, \
+ uint32_t* out) { \
+ int i; \
+ for (i = 0; i + 4 <= num_pixels; i += 4) { \
+ const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]); \
+ const uint8x16_t other = LOADQ_U32P_AS_U8(&(IN)); \
+ const uint8x16_t res = vaddq_u8(src, other); \
+ STOREQ_U8_AS_U32P(&out[i], res); \
+ } \
+ VP8LPredictorsAdd_C[(X)](in + i, upper + i, num_pixels - i, out + i); \
+}
+// Predictor2: Top.
+GENERATE_PREDICTOR_1(2, upper[i])
+// Predictor3: Top-right.
+GENERATE_PREDICTOR_1(3, upper[i + 1])
+// Predictor4: Top-left.
+GENERATE_PREDICTOR_1(4, upper[i - 1])
+#undef GENERATE_PREDICTOR_1
+
+// Predictor5: average(average(left, TR), T)
+#define DO_PRED5(LANE) do { \
+ const uint8x16_t avgLTR = vhaddq_u8(L, TR); \
+ const uint8x16_t avg = vhaddq_u8(avgLTR, T); \
+ const uint8x16_t res = vaddq_u8(avg, src); \
+ vst1q_lane_u32(&out[i + (LANE)], vreinterpretq_u32_u8(res), (LANE)); \
+ L = ROTATE32_LEFT(res); \
+} while (0)
+
+static void PredictorAdd5_NEON(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ uint8x16_t L = LOADQ_U32_AS_U8(out[-1]);
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]);
+ const uint8x16_t T = LOADQ_U32P_AS_U8(&upper[i + 0]);
+ const uint8x16_t TR = LOADQ_U32P_AS_U8(&upper[i + 1]);
+ DO_PRED5(0);
+ DO_PRED5(1);
+ DO_PRED5(2);
+ DO_PRED5(3);
+ }
+ VP8LPredictorsAdd_C[5](in + i, upper + i, num_pixels - i, out + i);
+}
+#undef DO_PRED5
+
+#define DO_PRED67(LANE) do { \
+ const uint8x16_t avg = vhaddq_u8(L, top); \
+ const uint8x16_t res = vaddq_u8(avg, src); \
+ vst1q_lane_u32(&out[i + (LANE)], vreinterpretq_u32_u8(res), (LANE)); \
+ L = ROTATE32_LEFT(res); \
+} while (0)
+
+// Predictor6: average(left, TL)
+static void PredictorAdd6_NEON(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ uint8x16_t L = LOADQ_U32_AS_U8(out[-1]);
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]);
+ const uint8x16_t top = LOADQ_U32P_AS_U8(&upper[i - 1]);
+ DO_PRED67(0);
+ DO_PRED67(1);
+ DO_PRED67(2);
+ DO_PRED67(3);
+ }
+ VP8LPredictorsAdd_C[6](in + i, upper + i, num_pixels - i, out + i);
+}
+
+// Predictor7: average(left, T)
+static void PredictorAdd7_NEON(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ uint8x16_t L = LOADQ_U32_AS_U8(out[-1]);
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]);
+ const uint8x16_t top = LOADQ_U32P_AS_U8(&upper[i]);
+ DO_PRED67(0);
+ DO_PRED67(1);
+ DO_PRED67(2);
+ DO_PRED67(3);
+ }
+ VP8LPredictorsAdd_C[7](in + i, upper + i, num_pixels - i, out + i);
+}
+#undef DO_PRED67
+
+#define GENERATE_PREDICTOR_2(X, IN) \
+static void PredictorAdd##X##_NEON(const uint32_t* in, \
+ const uint32_t* upper, int num_pixels, \
+ uint32_t* out) { \
+ int i; \
+ for (i = 0; i + 4 <= num_pixels; i += 4) { \
+ const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]); \
+ const uint8x16_t Tother = LOADQ_U32P_AS_U8(&(IN)); \
+ const uint8x16_t T = LOADQ_U32P_AS_U8(&upper[i]); \
+ const uint8x16_t avg = vhaddq_u8(T, Tother); \
+ const uint8x16_t res = vaddq_u8(avg, src); \
+ STOREQ_U8_AS_U32P(&out[i], res); \
+ } \
+ VP8LPredictorsAdd_C[(X)](in + i, upper + i, num_pixels - i, out + i); \
+}
+// Predictor8: average TL T.
+GENERATE_PREDICTOR_2(8, upper[i - 1])
+// Predictor9: average T TR.
+GENERATE_PREDICTOR_2(9, upper[i + 1])
+#undef GENERATE_PREDICTOR_2
+
+// Predictor10: average of (average of (L,TL), average of (T, TR)).
+#define DO_PRED10(LANE) do { \
+ const uint8x16_t avgLTL = vhaddq_u8(L, TL); \
+ const uint8x16_t avg = vhaddq_u8(avgTTR, avgLTL); \
+ const uint8x16_t res = vaddq_u8(avg, src); \
+ vst1q_lane_u32(&out[i + (LANE)], vreinterpretq_u32_u8(res), (LANE)); \
+ L = ROTATE32_LEFT(res); \
+} while (0)
+
+static void PredictorAdd10_NEON(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ uint8x16_t L = LOADQ_U32_AS_U8(out[-1]);
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]);
+ const uint8x16_t TL = LOADQ_U32P_AS_U8(&upper[i - 1]);
+ const uint8x16_t T = LOADQ_U32P_AS_U8(&upper[i]);
+ const uint8x16_t TR = LOADQ_U32P_AS_U8(&upper[i + 1]);
+ const uint8x16_t avgTTR = vhaddq_u8(T, TR);
+ DO_PRED10(0);
+ DO_PRED10(1);
+ DO_PRED10(2);
+ DO_PRED10(3);
+ }
+ VP8LPredictorsAdd_C[10](in + i, upper + i, num_pixels - i, out + i);
+}
+#undef DO_PRED10
+
+// Predictor11: select.
+#define DO_PRED11(LANE) do { \
+ const uint8x16_t sumLin = vaddq_u8(L, src); /* in + L */ \
+ const uint8x16_t pLTL = vabdq_u8(L, TL); /* |L - TL| */ \
+ const uint16x8_t sum_LTL = vpaddlq_u8(pLTL); \
+ const uint32x4_t pa = vpaddlq_u16(sum_LTL); \
+ const uint32x4_t mask = vcleq_u32(pa, pb); \
+ const uint8x16_t res = vbslq_u8(vreinterpretq_u8_u32(mask), sumTin, sumLin); \
+ vst1q_lane_u32(&out[i + (LANE)], vreinterpretq_u32_u8(res), (LANE)); \
+ L = ROTATE32_LEFT(res); \
+} while (0)
+
+static void PredictorAdd11_NEON(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ uint8x16_t L = LOADQ_U32_AS_U8(out[-1]);
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ const uint8x16_t T = LOADQ_U32P_AS_U8(&upper[i]);
+ const uint8x16_t TL = LOADQ_U32P_AS_U8(&upper[i - 1]);
+ const uint8x16_t pTTL = vabdq_u8(T, TL); // |T - TL|
+ const uint16x8_t sum_TTL = vpaddlq_u8(pTTL);
+ const uint32x4_t pb = vpaddlq_u16(sum_TTL);
+ const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]);
+ const uint8x16_t sumTin = vaddq_u8(T, src); // in + T
+ DO_PRED11(0);
+ DO_PRED11(1);
+ DO_PRED11(2);
+ DO_PRED11(3);
+ }
+ VP8LPredictorsAdd_C[11](in + i, upper + i, num_pixels - i, out + i);
+}
+#undef DO_PRED11
+
+// Predictor12: ClampedAddSubtractFull.
+#define DO_PRED12(DIFF, LANE) do { \
+ const uint8x8_t pred = \
+ vqmovun_s16(vaddq_s16(vreinterpretq_s16_u16(L), (DIFF))); \
+ const uint8x8_t res = \
+ vadd_u8(pred, (LANE <= 1) ? vget_low_u8(src) : vget_high_u8(src)); \
+ const uint16x8_t res16 = vmovl_u8(res); \
+ vst1_lane_u32(&out[i + (LANE)], vreinterpret_u32_u8(res), (LANE) & 1); \
+ /* rotate in the left predictor for next iteration */ \
+ L = vextq_u16(res16, res16, 4); \
+} while (0)
+
+static void PredictorAdd12_NEON(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ uint16x8_t L = vmovl_u8(LOAD_U32_AS_U8(out[-1]));
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ // load four pixels of source
+ const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]);
+ // precompute the difference T - TL once for all, stored as s16
+ const uint8x16_t TL = LOADQ_U32P_AS_U8(&upper[i - 1]);
+ const uint8x16_t T = LOADQ_U32P_AS_U8(&upper[i]);
+ const int16x8_t diff_lo =
+ vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(T), vget_low_u8(TL)));
+ const int16x8_t diff_hi =
+ vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(T), vget_high_u8(TL)));
+ // loop over the four reconstructed pixels
+ DO_PRED12(diff_lo, 0);
+ DO_PRED12(diff_lo, 1);
+ DO_PRED12(diff_hi, 2);
+ DO_PRED12(diff_hi, 3);
+ }
+ VP8LPredictorsAdd_C[12](in + i, upper + i, num_pixels - i, out + i);
+}
+#undef DO_PRED12
+
+// Predictor13: ClampedAddSubtractHalf
+#define DO_PRED13(LANE, LOW_OR_HI) do { \
+ const uint8x16_t avg = vhaddq_u8(L, T); \
+ const uint8x16_t cmp = vcgtq_u8(TL, avg); \
+ const uint8x16_t TL_1 = vaddq_u8(TL, cmp); \
+ /* Compute half of the difference between avg and TL'. */ \
+ const int8x8_t diff_avg = \
+ vreinterpret_s8_u8(LOW_OR_HI(vhsubq_u8(avg, TL_1))); \
+ /* Compute the sum with avg and saturate. */ \
+ const int16x8_t avg_16 = vreinterpretq_s16_u16(vmovl_u8(LOW_OR_HI(avg))); \
+ const uint8x8_t delta = vqmovun_s16(vaddw_s8(avg_16, diff_avg)); \
+ const uint8x8_t res = vadd_u8(LOW_OR_HI(src), delta); \
+ const uint8x16_t res2 = vcombine_u8(res, res); \
+ vst1_lane_u32(&out[i + (LANE)], vreinterpret_u32_u8(res), (LANE) & 1); \
+ L = ROTATE32_LEFT(res2); \
+} while (0)
+
+static void PredictorAdd13_NEON(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ uint8x16_t L = LOADQ_U32_AS_U8(out[-1]);
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ const uint8x16_t src = LOADQ_U32P_AS_U8(&in[i]);
+ const uint8x16_t T = LOADQ_U32P_AS_U8(&upper[i]);
+ const uint8x16_t TL = LOADQ_U32P_AS_U8(&upper[i - 1]);
+ DO_PRED13(0, vget_low_u8);
+ DO_PRED13(1, vget_low_u8);
+ DO_PRED13(2, vget_high_u8);
+ DO_PRED13(3, vget_high_u8);
+ }
+ VP8LPredictorsAdd_C[13](in + i, upper + i, num_pixels - i, out + i);
+}
+#undef DO_PRED13
+
+#undef LOAD_U32_AS_U8
+#undef LOAD_U32P_AS_U8
+#undef LOADQ_U32_AS_U8
+#undef LOADQ_U32P_AS_U8
+#undef GET_U8_AS_U32
+#undef GETQ_U8_AS_U32
+#undef STOREQ_U8_AS_U32P
+#undef ROTATE32_LEFT
+
//------------------------------------------------------------------------------
// Subtract-Green Transform
@@ -171,28 +522,30 @@ static WEBP_INLINE uint8x16_t DoGreenShuffle(const uint8x16_t argb,
}
#endif // USE_VTBLQ
-static void AddGreenToBlueAndRed(uint32_t* argb_data, int num_pixels) {
- const uint32_t* const end = argb_data + (num_pixels & ~3);
+static void AddGreenToBlueAndRed(const uint32_t* src, int num_pixels,
+ uint32_t* dst) {
+ const uint32_t* const end = src + (num_pixels & ~3);
#ifdef USE_VTBLQ
const uint8x16_t shuffle = vld1q_u8(kGreenShuffle);
#else
const uint8x8_t shuffle = vld1_u8(kGreenShuffle);
#endif
- for (; argb_data < end; argb_data += 4) {
- const uint8x16_t argb = vld1q_u8((uint8_t*)argb_data);
+ for (; src < end; src += 4, dst += 4) {
+ const uint8x16_t argb = vld1q_u8((const uint8_t*)src);
const uint8x16_t greens = DoGreenShuffle(argb, shuffle);
- vst1q_u8((uint8_t*)argb_data, vaddq_u8(argb, greens));
+ vst1q_u8((uint8_t*)dst, vaddq_u8(argb, greens));
}
// fallthrough and finish off with plain-C
- VP8LAddGreenToBlueAndRed_C(argb_data, num_pixels & 3);
+ VP8LAddGreenToBlueAndRed_C(src, num_pixels & 3, dst);
}
//------------------------------------------------------------------------------
// Color Transform
static void TransformColorInverse(const VP8LMultipliers* const m,
- uint32_t* argb_data, int num_pixels) {
- // sign-extended multiplying constants, pre-shifted by 6.
+ const uint32_t* const src, int num_pixels,
+ uint32_t* dst) {
+// sign-extended multiplying constants, pre-shifted by 6.
#define CST(X) (((int16_t)(m->X << 8)) >> 6)
const int16_t rb[8] = {
CST(green_to_blue_), CST(green_to_red_),
@@ -219,7 +572,7 @@ static void TransformColorInverse(const VP8LMultipliers* const m,
const uint32x4_t mask_ag = vdupq_n_u32(0xff00ff00u);
int i;
for (i = 0; i + 4 <= num_pixels; i += 4) {
- const uint8x16_t in = vld1q_u8((uint8_t*)(argb_data + i));
+ const uint8x16_t in = vld1q_u8((const uint8_t*)(src + i));
const uint32x4_t a0g0 = vandq_u32(vreinterpretq_u32_u8(in), mask_ag);
// 0 g 0 g
const uint8x16_t greens = DoGreenShuffle(in, shuffle);
@@ -240,10 +593,10 @@ static void TransformColorInverse(const VP8LMultipliers* const m,
// 0 r' 0 b''
const uint16x8_t G = vshrq_n_u16(vreinterpretq_u16_s8(F), 8);
const uint32x4_t out = vorrq_u32(vreinterpretq_u32_u16(G), a0g0);
- vst1q_u32(argb_data + i, out);
+ vst1q_u32(dst + i, out);
}
// Fall-back to C-version for left-overs.
- VP8LTransformColorInverse_C(m, argb_data + i, num_pixels - i);
+ VP8LTransformColorInverse_C(m, src + i, num_pixels - i, dst + i);
}
#undef USE_VTBLQ
@@ -254,6 +607,26 @@ static void TransformColorInverse(const VP8LMultipliers* const m,
extern void VP8LDspInitNEON(void);
WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInitNEON(void) {
+ VP8LPredictors[5] = Predictor5_NEON;
+ VP8LPredictors[6] = Predictor6_NEON;
+ VP8LPredictors[7] = Predictor7_NEON;
+ VP8LPredictors[13] = Predictor13_NEON;
+
+ VP8LPredictorsAdd[0] = PredictorAdd0_NEON;
+ VP8LPredictorsAdd[1] = PredictorAdd1_NEON;
+ VP8LPredictorsAdd[2] = PredictorAdd2_NEON;
+ VP8LPredictorsAdd[3] = PredictorAdd3_NEON;
+ VP8LPredictorsAdd[4] = PredictorAdd4_NEON;
+ VP8LPredictorsAdd[5] = PredictorAdd5_NEON;
+ VP8LPredictorsAdd[6] = PredictorAdd6_NEON;
+ VP8LPredictorsAdd[7] = PredictorAdd7_NEON;
+ VP8LPredictorsAdd[8] = PredictorAdd8_NEON;
+ VP8LPredictorsAdd[9] = PredictorAdd9_NEON;
+ VP8LPredictorsAdd[10] = PredictorAdd10_NEON;
+ VP8LPredictorsAdd[11] = PredictorAdd11_NEON;
+ VP8LPredictorsAdd[12] = PredictorAdd12_NEON;
+ VP8LPredictorsAdd[13] = PredictorAdd13_NEON;
+
VP8LConvertBGRAToRGBA = ConvertBGRAToRGBA;
VP8LConvertBGRAToBGR = ConvertBGRAToBGR;
VP8LConvertBGRAToRGB = ConvertBGRAToRGB;
diff --git a/src/3rdparty/libwebp/src/dsp/lossless_sse2.c b/src/3rdparty/libwebp/src/dsp/lossless_sse2.c
index 2d016c2..15aae93 100644
--- a/src/3rdparty/libwebp/src/dsp/lossless_sse2.c
+++ b/src/3rdparty/libwebp/src/dsp/lossless_sse2.c
@@ -14,9 +14,12 @@
#include "./dsp.h"
#if defined(WEBP_USE_SSE2)
+
+#include "./common_sse2.h"
+#include "./lossless.h"
+#include "./lossless_common.h"
#include <assert.h>
#include <emmintrin.h>
-#include "./lossless.h"
//------------------------------------------------------------------------------
// Predictor Transform
@@ -75,25 +78,44 @@ static WEBP_INLINE uint32_t Select(uint32_t a, uint32_t b, uint32_t c) {
return (pa_minus_pb <= 0) ? a : b;
}
-static WEBP_INLINE __m128i Average2_128i(uint32_t a0, uint32_t a1) {
+static WEBP_INLINE void Average2_m128i(const __m128i* const a0,
+ const __m128i* const a1,
+ __m128i* const avg) {
+ // (a + b) >> 1 = ((a + b + 1) >> 1) - ((a ^ b) & 1)
+ const __m128i ones = _mm_set1_epi8(1);
+ const __m128i avg1 = _mm_avg_epu8(*a0, *a1);
+ const __m128i one = _mm_and_si128(_mm_xor_si128(*a0, *a1), ones);
+ *avg = _mm_sub_epi8(avg1, one);
+}
+
+static WEBP_INLINE void Average2_uint32(const uint32_t a0, const uint32_t a1,
+ __m128i* const avg) {
+ // (a + b) >> 1 = ((a + b + 1) >> 1) - ((a ^ b) & 1)
+ const __m128i ones = _mm_set1_epi8(1);
+ const __m128i A0 = _mm_cvtsi32_si128(a0);
+ const __m128i A1 = _mm_cvtsi32_si128(a1);
+ const __m128i avg1 = _mm_avg_epu8(A0, A1);
+ const __m128i one = _mm_and_si128(_mm_xor_si128(A0, A1), ones);
+ *avg = _mm_sub_epi8(avg1, one);
+}
+
+static WEBP_INLINE __m128i Average2_uint32_16(uint32_t a0, uint32_t a1) {
const __m128i zero = _mm_setzero_si128();
const __m128i A0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(a0), zero);
const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(a1), zero);
const __m128i sum = _mm_add_epi16(A1, A0);
- const __m128i avg = _mm_srli_epi16(sum, 1);
- return avg;
+ return _mm_srli_epi16(sum, 1);
}
static WEBP_INLINE uint32_t Average2(uint32_t a0, uint32_t a1) {
- const __m128i avg = Average2_128i(a0, a1);
- const __m128i A2 = _mm_packus_epi16(avg, avg);
- const uint32_t output = _mm_cvtsi128_si32(A2);
- return output;
+ __m128i output;
+ Average2_uint32(a0, a1, &output);
+ return _mm_cvtsi128_si32(output);
}
static WEBP_INLINE uint32_t Average3(uint32_t a0, uint32_t a1, uint32_t a2) {
const __m128i zero = _mm_setzero_si128();
- const __m128i avg1 = Average2_128i(a0, a2);
+ const __m128i avg1 = Average2_uint32_16(a0, a2);
const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(a1), zero);
const __m128i sum = _mm_add_epi16(avg1, A1);
const __m128i avg2 = _mm_srli_epi16(sum, 1);
@@ -104,8 +126,8 @@ static WEBP_INLINE uint32_t Average3(uint32_t a0, uint32_t a1, uint32_t a2) {
static WEBP_INLINE uint32_t Average4(uint32_t a0, uint32_t a1,
uint32_t a2, uint32_t a3) {
- const __m128i avg1 = Average2_128i(a0, a1);
- const __m128i avg2 = Average2_128i(a2, a3);
+ const __m128i avg1 = Average2_uint32_16(a0, a1);
+ const __m128i avg2 = Average2_uint32_16(a2, a3);
const __m128i sum = _mm_add_epi16(avg2, avg1);
const __m128i avg3 = _mm_srli_epi16(sum, 1);
const __m128i A0 = _mm_packus_epi16(avg3, avg3);
@@ -113,68 +135,289 @@ static WEBP_INLINE uint32_t Average4(uint32_t a0, uint32_t a1,
return output;
}
-static uint32_t Predictor5(uint32_t left, const uint32_t* const top) {
+static uint32_t Predictor5_SSE2(uint32_t left, const uint32_t* const top) {
const uint32_t pred = Average3(left, top[0], top[1]);
return pred;
}
-static uint32_t Predictor6(uint32_t left, const uint32_t* const top) {
+static uint32_t Predictor6_SSE2(uint32_t left, const uint32_t* const top) {
const uint32_t pred = Average2(left, top[-1]);
return pred;
}
-static uint32_t Predictor7(uint32_t left, const uint32_t* const top) {
+static uint32_t Predictor7_SSE2(uint32_t left, const uint32_t* const top) {
const uint32_t pred = Average2(left, top[0]);
return pred;
}
-static uint32_t Predictor8(uint32_t left, const uint32_t* const top) {
+static uint32_t Predictor8_SSE2(uint32_t left, const uint32_t* const top) {
const uint32_t pred = Average2(top[-1], top[0]);
(void)left;
return pred;
}
-static uint32_t Predictor9(uint32_t left, const uint32_t* const top) {
+static uint32_t Predictor9_SSE2(uint32_t left, const uint32_t* const top) {
const uint32_t pred = Average2(top[0], top[1]);
(void)left;
return pred;
}
-static uint32_t Predictor10(uint32_t left, const uint32_t* const top) {
+static uint32_t Predictor10_SSE2(uint32_t left, const uint32_t* const top) {
const uint32_t pred = Average4(left, top[-1], top[0], top[1]);
return pred;
}
-static uint32_t Predictor11(uint32_t left, const uint32_t* const top) {
+static uint32_t Predictor11_SSE2(uint32_t left, const uint32_t* const top) {
const uint32_t pred = Select(top[0], left, top[-1]);
return pred;
}
-static uint32_t Predictor12(uint32_t left, const uint32_t* const top) {
+static uint32_t Predictor12_SSE2(uint32_t left, const uint32_t* const top) {
const uint32_t pred = ClampedAddSubtractFull(left, top[0], top[-1]);
return pred;
}
-static uint32_t Predictor13(uint32_t left, const uint32_t* const top) {
+static uint32_t Predictor13_SSE2(uint32_t left, const uint32_t* const top) {
const uint32_t pred = ClampedAddSubtractHalf(left, top[0], top[-1]);
return pred;
}
+// Batch versions of those functions.
+
+// Predictor0: ARGB_BLACK.
+static void PredictorAdd0_SSE2(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ const __m128i black = _mm_set1_epi32(ARGB_BLACK);
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
+ const __m128i res = _mm_add_epi8(src, black);
+ _mm_storeu_si128((__m128i*)&out[i], res);
+ }
+ if (i != num_pixels) {
+ VP8LPredictorsAdd_C[0](in + i, upper + i, num_pixels - i, out + i);
+ }
+}
+
+// Predictor1: left.
+static void PredictorAdd1_SSE2(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ __m128i prev = _mm_set1_epi32(out[-1]);
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ // a | b | c | d
+ const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
+ // 0 | a | b | c
+ const __m128i shift0 = _mm_slli_si128(src, 4);
+ // a | a + b | b + c | c + d
+ const __m128i sum0 = _mm_add_epi8(src, shift0);
+ // 0 | 0 | a | a + b
+ const __m128i shift1 = _mm_slli_si128(sum0, 8);
+ // a | a + b | a + b + c | a + b + c + d
+ const __m128i sum1 = _mm_add_epi8(sum0, shift1);
+ const __m128i res = _mm_add_epi8(sum1, prev);
+ _mm_storeu_si128((__m128i*)&out[i], res);
+ // replicate prev output on the four lanes
+ prev = _mm_shuffle_epi32(res, (3 << 0) | (3 << 2) | (3 << 4) | (3 << 6));
+ }
+ if (i != num_pixels) {
+ VP8LPredictorsAdd_C[1](in + i, upper + i, num_pixels - i, out + i);
+ }
+}
+
+// Macro that adds 32-bit integers from IN using mod 256 arithmetic
+// per 8 bit channel.
+#define GENERATE_PREDICTOR_1(X, IN) \
+static void PredictorAdd##X##_SSE2(const uint32_t* in, const uint32_t* upper, \
+ int num_pixels, uint32_t* out) { \
+ int i; \
+ for (i = 0; i + 4 <= num_pixels; i += 4) { \
+ const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \
+ const __m128i other = _mm_loadu_si128((const __m128i*)&(IN)); \
+ const __m128i res = _mm_add_epi8(src, other); \
+ _mm_storeu_si128((__m128i*)&out[i], res); \
+ } \
+ if (i != num_pixels) { \
+ VP8LPredictorsAdd_C[(X)](in + i, upper + i, num_pixels - i, out + i); \
+ } \
+}
+
+// Predictor2: Top.
+GENERATE_PREDICTOR_1(2, upper[i])
+// Predictor3: Top-right.
+GENERATE_PREDICTOR_1(3, upper[i + 1])
+// Predictor4: Top-left.
+GENERATE_PREDICTOR_1(4, upper[i - 1])
+#undef GENERATE_PREDICTOR_1
+
+// Due to averages with integers, values cannot be accumulated in parallel for
+// predictors 5 to 7.
+GENERATE_PREDICTOR_ADD(Predictor5_SSE2, PredictorAdd5_SSE2)
+GENERATE_PREDICTOR_ADD(Predictor6_SSE2, PredictorAdd6_SSE2)
+GENERATE_PREDICTOR_ADD(Predictor7_SSE2, PredictorAdd7_SSE2)
+
+#define GENERATE_PREDICTOR_2(X, IN) \
+static void PredictorAdd##X##_SSE2(const uint32_t* in, const uint32_t* upper, \
+ int num_pixels, uint32_t* out) { \
+ int i; \
+ for (i = 0; i + 4 <= num_pixels; i += 4) { \
+ const __m128i Tother = _mm_loadu_si128((const __m128i*)&(IN)); \
+ const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); \
+ const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \
+ __m128i avg, res; \
+ Average2_m128i(&T, &Tother, &avg); \
+ res = _mm_add_epi8(avg, src); \
+ _mm_storeu_si128((__m128i*)&out[i], res); \
+ } \
+ if (i != num_pixels) { \
+ VP8LPredictorsAdd_C[(X)](in + i, upper + i, num_pixels - i, out + i); \
+ } \
+}
+// Predictor8: average TL T.
+GENERATE_PREDICTOR_2(8, upper[i - 1])
+// Predictor9: average T TR.
+GENERATE_PREDICTOR_2(9, upper[i + 1])
+#undef GENERATE_PREDICTOR_2
+
+// Predictor10: average of (average of (L,TL), average of (T, TR)).
+static void PredictorAdd10_SSE2(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i, j;
+ __m128i L = _mm_cvtsi32_si128(out[-1]);
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
+ __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]);
+ const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]);
+ const __m128i TR = _mm_loadu_si128((const __m128i*)&upper[i + 1]);
+ __m128i avgTTR;
+ Average2_m128i(&T, &TR, &avgTTR);
+ for (j = 0; j < 4; ++j) {
+ __m128i avgLTL, avg;
+ Average2_m128i(&L, &TL, &avgLTL);
+ Average2_m128i(&avgTTR, &avgLTL, &avg);
+ L = _mm_add_epi8(avg, src);
+ out[i + j] = _mm_cvtsi128_si32(L);
+ // Rotate the pre-computed values for the next iteration.
+ avgTTR = _mm_srli_si128(avgTTR, 4);
+ TL = _mm_srli_si128(TL, 4);
+ src = _mm_srli_si128(src, 4);
+ }
+ }
+ if (i != num_pixels) {
+ VP8LPredictorsAdd_C[10](in + i, upper + i, num_pixels - i, out + i);
+ }
+}
+
+// Predictor11: select.
+static void GetSumAbsDiff32(const __m128i* const A, const __m128i* const B,
+ __m128i* const out) {
+ // We can unpack with any value on the upper 32 bits, provided it's the same
+ // on both operands (to that their sum of abs diff is zero). Here we use *A.
+ const __m128i A_lo = _mm_unpacklo_epi32(*A, *A);
+ const __m128i B_lo = _mm_unpacklo_epi32(*B, *A);
+ const __m128i A_hi = _mm_unpackhi_epi32(*A, *A);
+ const __m128i B_hi = _mm_unpackhi_epi32(*B, *A);
+ const __m128i s_lo = _mm_sad_epu8(A_lo, B_lo);
+ const __m128i s_hi = _mm_sad_epu8(A_hi, B_hi);
+ *out = _mm_packs_epi32(s_lo, s_hi);
+}
+
+static void PredictorAdd11_SSE2(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i, j;
+ __m128i L = _mm_cvtsi32_si128(out[-1]);
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]);
+ __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]);
+ __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
+ __m128i pa;
+ GetSumAbsDiff32(&T, &TL, &pa); // pa = sum |T-TL|
+ for (j = 0; j < 4; ++j) {
+ const __m128i L_lo = _mm_unpacklo_epi32(L, L);
+ const __m128i TL_lo = _mm_unpacklo_epi32(TL, L);
+ const __m128i pb = _mm_sad_epu8(L_lo, TL_lo); // pb = sum |L-TL|
+ const __m128i mask = _mm_cmpgt_epi32(pb, pa);
+ const __m128i A = _mm_and_si128(mask, L);
+ const __m128i B = _mm_andnot_si128(mask, T);
+ const __m128i pred = _mm_or_si128(A, B); // pred = (L > T)? L : T
+ L = _mm_add_epi8(src, pred);
+ out[i + j] = _mm_cvtsi128_si32(L);
+ // Shift the pre-computed value for the next iteration.
+ T = _mm_srli_si128(T, 4);
+ TL = _mm_srli_si128(TL, 4);
+ src = _mm_srli_si128(src, 4);
+ pa = _mm_srli_si128(pa, 4);
+ }
+ }
+ if (i != num_pixels) {
+ VP8LPredictorsAdd_C[11](in + i, upper + i, num_pixels - i, out + i);
+ }
+}
+
+// Predictor12: ClampedAddSubtractFull.
+#define DO_PRED12(DIFF, LANE, OUT) \
+do { \
+ const __m128i all = _mm_add_epi16(L, (DIFF)); \
+ const __m128i alls = _mm_packus_epi16(all, all); \
+ const __m128i res = _mm_add_epi8(src, alls); \
+ out[i + (OUT)] = _mm_cvtsi128_si32(res); \
+ L = _mm_unpacklo_epi8(res, zero); \
+ /* Shift the pre-computed value for the next iteration.*/ \
+ if (LANE == 0) (DIFF) = _mm_srli_si128((DIFF), 8); \
+ src = _mm_srli_si128(src, 4); \
+} while (0)
+
+static void PredictorAdd12_SSE2(const uint32_t* in, const uint32_t* upper,
+ int num_pixels, uint32_t* out) {
+ int i;
+ const __m128i zero = _mm_setzero_si128();
+ const __m128i L8 = _mm_cvtsi32_si128(out[-1]);
+ __m128i L = _mm_unpacklo_epi8(L8, zero);
+ for (i = 0; i + 4 <= num_pixels; i += 4) {
+ // Load 4 pixels at a time.
+ __m128i src = _mm_loadu_si128((const __m128i*)&in[i]);
+ const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]);
+ const __m128i T_lo = _mm_unpacklo_epi8(T, zero);
+ const __m128i T_hi = _mm_unpackhi_epi8(T, zero);
+ const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]);
+ const __m128i TL_lo = _mm_unpacklo_epi8(TL, zero);
+ const __m128i TL_hi = _mm_unpackhi_epi8(TL, zero);
+ __m128i diff_lo = _mm_sub_epi16(T_lo, TL_lo);
+ __m128i diff_hi = _mm_sub_epi16(T_hi, TL_hi);
+ DO_PRED12(diff_lo, 0, 0);
+ DO_PRED12(diff_lo, 1, 1);
+ DO_PRED12(diff_hi, 0, 2);
+ DO_PRED12(diff_hi, 1, 3);
+ }
+ if (i != num_pixels) {
+ VP8LPredictorsAdd_C[12](in + i, upper + i, num_pixels - i, out + i);
+ }
+}
+#undef DO_PRED12
+
+// Due to averages with integers, values cannot be accumulated in parallel for
+// predictors 13.
+GENERATE_PREDICTOR_ADD(Predictor13_SSE2, PredictorAdd13_SSE2)
+
//------------------------------------------------------------------------------
// Subtract-Green Transform
-static void AddGreenToBlueAndRed(uint32_t* argb_data, int num_pixels) {
+static void AddGreenToBlueAndRed(const uint32_t* const src, int num_pixels,
+ uint32_t* dst) {
int i;
for (i = 0; i + 4 <= num_pixels; i += 4) {
- const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb
+ const __m128i in = _mm_loadu_si128((const __m128i*)&src[i]); // argb
const __m128i A = _mm_srli_epi16(in, 8); // 0 a 0 g
const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0));
const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // 0g0g
const __m128i out = _mm_add_epi8(in, C);
- _mm_storeu_si128((__m128i*)&argb_data[i], out);
+ _mm_storeu_si128((__m128i*)&dst[i], out);
}
// fallthrough and finish off with plain-C
- VP8LAddGreenToBlueAndRed_C(argb_data + i, num_pixels - i);
+ if (i != num_pixels) {
+ VP8LAddGreenToBlueAndRed_C(src + i, num_pixels - i, dst + i);
+ }
}
//------------------------------------------------------------------------------
// Color Transform
static void TransformColorInverse(const VP8LMultipliers* const m,
- uint32_t* argb_data, int num_pixels) {
- // sign-extended multiplying constants, pre-shifted by 5.
+ const uint32_t* const src, int num_pixels,
+ uint32_t* dst) {
+// sign-extended multiplying constants, pre-shifted by 5.
#define CST(X) (((int16_t)(m->X << 8)) >> 5) // sign-extend
const __m128i mults_rb = _mm_set_epi16(
CST(green_to_red_), CST(green_to_blue_),
@@ -188,7 +431,7 @@ static void TransformColorInverse(const VP8LMultipliers* const m,
const __m128i mask_ag = _mm_set1_epi32(0xff00ff00); // alpha-green masks
int i;
for (i = 0; i + 4 <= num_pixels; i += 4) {
- const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb
+ const __m128i in = _mm_loadu_si128((const __m128i*)&src[i]); // argb
const __m128i A = _mm_and_si128(in, mask_ag); // a 0 g 0
const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0));
const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // g0g0
@@ -200,15 +443,53 @@ static void TransformColorInverse(const VP8LMultipliers* const m,
const __m128i I = _mm_add_epi8(H, F); // r' x b'' 0
const __m128i J = _mm_srli_epi16(I, 8); // 0 r' 0 b''
const __m128i out = _mm_or_si128(J, A);
- _mm_storeu_si128((__m128i*)&argb_data[i], out);
+ _mm_storeu_si128((__m128i*)&dst[i], out);
}
// Fall-back to C-version for left-overs.
- VP8LTransformColorInverse_C(m, argb_data + i, num_pixels - i);
+ if (i != num_pixels) {
+ VP8LTransformColorInverse_C(m, src + i, num_pixels - i, dst + i);
+ }
}
//------------------------------------------------------------------------------
// Color-space conversion functions
+static void ConvertBGRAToRGB(const uint32_t* src, int num_pixels,
+ uint8_t* dst) {
+ const __m128i* in = (const __m128i*)src;
+ __m128i* out = (__m128i*)dst;
+
+ while (num_pixels >= 32) {
+ // Load the BGRA buffers.
+ __m128i in0 = _mm_loadu_si128(in + 0);
+ __m128i in1 = _mm_loadu_si128(in + 1);
+ __m128i in2 = _mm_loadu_si128(in + 2);
+ __m128i in3 = _mm_loadu_si128(in + 3);
+ __m128i in4 = _mm_loadu_si128(in + 4);
+ __m128i in5 = _mm_loadu_si128(in + 5);
+ __m128i in6 = _mm_loadu_si128(in + 6);
+ __m128i in7 = _mm_loadu_si128(in + 7);
+ VP8L32bToPlanar(&in0, &in1, &in2, &in3);
+ VP8L32bToPlanar(&in4, &in5, &in6, &in7);
+ // At this points, in1/in5 contains red only, in2/in6 green only ...
+ // Pack the colors in 24b RGB.
+ VP8PlanarTo24b(&in1, &in5, &in2, &in6, &in3, &in7);
+ _mm_storeu_si128(out + 0, in1);
+ _mm_storeu_si128(out + 1, in5);
+ _mm_storeu_si128(out + 2, in2);
+ _mm_storeu_si128(out + 3, in6);
+ _mm_storeu_si128(out + 4, in3);
+ _mm_storeu_si128(out + 5, in7);
+ in += 8;
+ out += 6;
+ num_pixels -= 32;
+ }
+ // left-overs
+ if (num_pixels > 0) {
+ VP8LConvertBGRAToRGB_C((const uint32_t*)in, num_pixels, (uint8_t*)out);
+ }
+}
+
static void ConvertBGRAToRGBA(const uint32_t* src,
int num_pixels, uint8_t* dst) {
const __m128i* in = (const __m128i*)src;
@@ -233,7 +514,9 @@ static void ConvertBGRAToRGBA(const uint32_t* src,
num_pixels -= 8;
}
// left-overs
- VP8LConvertBGRAToRGBA_C((const uint32_t*)in, num_pixels, (uint8_t*)out);
+ if (num_pixels > 0) {
+ VP8LConvertBGRAToRGBA_C((const uint32_t*)in, num_pixels, (uint8_t*)out);
+ }
}
static void ConvertBGRAToRGBA4444(const uint32_t* src,
@@ -267,7 +550,9 @@ static void ConvertBGRAToRGBA4444(const uint32_t* src,
num_pixels -= 8;
}
// left-overs
- VP8LConvertBGRAToRGBA4444_C((const uint32_t*)in, num_pixels, (uint8_t*)out);
+ if (num_pixels > 0) {
+ VP8LConvertBGRAToRGBA4444_C((const uint32_t*)in, num_pixels, (uint8_t*)out);
+ }
}
static void ConvertBGRAToRGB565(const uint32_t* src,
@@ -306,7 +591,9 @@ static void ConvertBGRAToRGB565(const uint32_t* src,
num_pixels -= 8;
}
// left-overs
- VP8LConvertBGRAToRGB565_C((const uint32_t*)in, num_pixels, (uint8_t*)out);
+ if (num_pixels > 0) {
+ VP8LConvertBGRAToRGB565_C((const uint32_t*)in, num_pixels, (uint8_t*)out);
+ }
}
static void ConvertBGRAToBGR(const uint32_t* src,
@@ -337,7 +624,9 @@ static void ConvertBGRAToBGR(const uint32_t* src,
num_pixels -= 8;
}
// left-overs
- VP8LConvertBGRAToBGR_C((const uint32_t*)in, num_pixels, dst);
+ if (num_pixels > 0) {
+ VP8LConvertBGRAToBGR_C((const uint32_t*)in, num_pixels, dst);
+ }
}
//------------------------------------------------------------------------------
@@ -346,19 +635,35 @@ static void ConvertBGRAToBGR(const uint32_t* src,
extern void VP8LDspInitSSE2(void);
WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInitSSE2(void) {
- VP8LPredictors[5] = Predictor5;
- VP8LPredictors[6] = Predictor6;
- VP8LPredictors[7] = Predictor7;
- VP8LPredictors[8] = Predictor8;
- VP8LPredictors[9] = Predictor9;
- VP8LPredictors[10] = Predictor10;
- VP8LPredictors[11] = Predictor11;
- VP8LPredictors[12] = Predictor12;
- VP8LPredictors[13] = Predictor13;
+ VP8LPredictors[5] = Predictor5_SSE2;
+ VP8LPredictors[6] = Predictor6_SSE2;
+ VP8LPredictors[7] = Predictor7_SSE2;
+ VP8LPredictors[8] = Predictor8_SSE2;
+ VP8LPredictors[9] = Predictor9_SSE2;
+ VP8LPredictors[10] = Predictor10_SSE2;
+ VP8LPredictors[11] = Predictor11_SSE2;
+ VP8LPredictors[12] = Predictor12_SSE2;
+ VP8LPredictors[13] = Predictor13_SSE2;
+
+ VP8LPredictorsAdd[0] = PredictorAdd0_SSE2;
+ VP8LPredictorsAdd[1] = PredictorAdd1_SSE2;
+ VP8LPredictorsAdd[2] = PredictorAdd2_SSE2;
+ VP8LPredictorsAdd[3] = PredictorAdd3_SSE2;
+ VP8LPredictorsAdd[4] = PredictorAdd4_SSE2;
+ VP8LPredictorsAdd[5] = PredictorAdd5_SSE2;
+ VP8LPredictorsAdd[6] = PredictorAdd6_SSE2;
+ VP8LPredictorsAdd[7] = PredictorAdd7_SSE2;
+ VP8LPredictorsAdd[8] = PredictorAdd8_SSE2;
+ VP8LPredictorsAdd[9] = PredictorAdd9_SSE2;
+ VP8LPredictorsAdd[10] = PredictorAdd10_SSE2;
+ VP8LPredictorsAdd[11] = PredictorAdd11_SSE2;
+ VP8LPredictorsAdd[12] = PredictorAdd12_SSE2;
+ VP8LPredictorsAdd[13] = PredictorAdd13_SSE2;
VP8LAddGreenToBlueAndRed = AddGreenToBlueAndRed;
VP8LTransformColorInverse = TransformColorInverse;
+ VP8LConvertBGRAToRGB = ConvertBGRAToRGB;
VP8LConvertBGRAToRGBA = ConvertBGRAToRGBA;
VP8LConvertBGRAToRGBA4444 = ConvertBGRAToRGBA4444;
VP8LConvertBGRAToRGB565 = ConvertBGRAToRGB565;
diff --git a/src/3rdparty/libwebp/src/dsp/msa_macro.h b/src/3rdparty/libwebp/src/dsp/msa_macro.h
index 5c707f4..d0e5f45 100644
--- a/src/3rdparty/libwebp/src/dsp/msa_macro.h
+++ b/src/3rdparty/libwebp/src/dsp/msa_macro.h
@@ -23,12 +23,24 @@
#ifdef CLANG_BUILD
#define ADDVI_H(a, b) __msa_addvi_h((v8i16)a, b)
+ #define ADDVI_W(a, b) __msa_addvi_w((v4i32)a, b)
+ #define SRAI_B(a, b) __msa_srai_b((v16i8)a, b)
#define SRAI_H(a, b) __msa_srai_h((v8i16)a, b)
#define SRAI_W(a, b) __msa_srai_w((v4i32)a, b)
+ #define SRLI_H(a, b) __msa_srli_h((v8i16)a, b)
+ #define SLLI_B(a, b) __msa_slli_b((v4i32)a, b)
+ #define ANDI_B(a, b) __msa_andi_b((v16u8)a, b)
+ #define ORI_B(a, b) __msa_ori_b((v16u8)a, b)
#else
#define ADDVI_H(a, b) (a + b)
+ #define ADDVI_W(a, b) (a + b)
+ #define SRAI_B(a, b) (a >> b)
#define SRAI_H(a, b) (a >> b)
#define SRAI_W(a, b) (a >> b)
+ #define SRLI_H(a, b) (a << b)
+ #define SLLI_B(a, b) (a << b)
+ #define ANDI_B(a, b) (a & b)
+ #define ORI_B(a, b) (a | b)
#endif
#define LD_B(RTYPE, psrc) *((RTYPE*)(psrc))
@@ -116,13 +128,13 @@
#define SH(val, pdst) MSA_STORE(val, pdst, msa_ush)
MSA_STORE_FUNC(uint32_t, usw, msa_usw);
#define SW(val, pdst) MSA_STORE(val, pdst, msa_usw)
- #define SD(val, pdst) { \
+ #define SD(val, pdst) do { \
uint8_t* const pdst_sd_m = (uint8_t*)(pdst); \
const uint32_t val0_m = (uint32_t)(val & 0x00000000FFFFFFFF); \
const uint32_t val1_m = (uint32_t)((val >> 32) & 0x00000000FFFFFFFF); \
SW(val0_m, pdst_sd_m); \
SW(val1_m, pdst_sd_m + 4); \
- }
+ } while (0)
#endif // (__mips_isa_rev >= 6)
/* Description : Load 4 words with stride
@@ -133,34 +145,68 @@
* Load word in 'out2' from (psrc + 2 * stride)
* Load word in 'out3' from (psrc + 3 * stride)
*/
-#define LW4(psrc, stride, out0, out1, out2, out3) { \
- const uint8_t* ptmp = (const uint8_t*)psrc; \
- out0 = LW(ptmp); \
- ptmp += stride; \
- out1 = LW(ptmp); \
- ptmp += stride; \
- out2 = LW(ptmp); \
- ptmp += stride; \
- out3 = LW(ptmp); \
-}
+#define LW4(psrc, stride, out0, out1, out2, out3) do { \
+ const uint8_t* ptmp = (const uint8_t*)psrc; \
+ out0 = LW(ptmp); \
+ ptmp += stride; \
+ out1 = LW(ptmp); \
+ ptmp += stride; \
+ out2 = LW(ptmp); \
+ ptmp += stride; \
+ out3 = LW(ptmp); \
+} while (0)
-/* Description : Store 4 words with stride
+/* Description : Store words with stride
* Arguments : Inputs - in0, in1, in2, in3, pdst, stride
* Details : Store word from 'in0' to (pdst)
* Store word from 'in1' to (pdst + stride)
* Store word from 'in2' to (pdst + 2 * stride)
* Store word from 'in3' to (pdst + 3 * stride)
*/
-#define SW4(in0, in1, in2, in3, pdst, stride) { \
- uint8_t* ptmp = (uint8_t*)pdst; \
- SW(in0, ptmp); \
- ptmp += stride; \
- SW(in1, ptmp); \
- ptmp += stride; \
- SW(in2, ptmp); \
- ptmp += stride; \
- SW(in3, ptmp); \
-}
+#define SW4(in0, in1, in2, in3, pdst, stride) do { \
+ uint8_t* ptmp = (uint8_t*)pdst; \
+ SW(in0, ptmp); \
+ ptmp += stride; \
+ SW(in1, ptmp); \
+ ptmp += stride; \
+ SW(in2, ptmp); \
+ ptmp += stride; \
+ SW(in3, ptmp); \
+} while (0)
+
+#define SW3(in0, in1, in2, pdst, stride) do { \
+ uint8_t* ptmp = (uint8_t*)pdst; \
+ SW(in0, ptmp); \
+ ptmp += stride; \
+ SW(in1, ptmp); \
+ ptmp += stride; \
+ SW(in2, ptmp); \
+} while (0)
+
+#define SW2(in0, in1, pdst, stride) do { \
+ uint8_t* ptmp = (uint8_t*)pdst; \
+ SW(in0, ptmp); \
+ ptmp += stride; \
+ SW(in1, ptmp); \
+} while (0)
+
+/* Description : Store 4 double words with stride
+ * Arguments : Inputs - in0, in1, in2, in3, pdst, stride
+ * Details : Store double word from 'in0' to (pdst)
+ * Store double word from 'in1' to (pdst + stride)
+ * Store double word from 'in2' to (pdst + 2 * stride)
+ * Store double word from 'in3' to (pdst + 3 * stride)
+ */
+#define SD4(in0, in1, in2, in3, pdst, stride) do { \
+ uint8_t* ptmp = (uint8_t*)pdst; \
+ SD(in0, ptmp); \
+ ptmp += stride; \
+ SD(in1, ptmp); \
+ ptmp += stride; \
+ SD(in2, ptmp); \
+ ptmp += stride; \
+ SD(in3, ptmp); \
+} while (0)
/* Description : Load vectors with 16 byte elements with stride
* Arguments : Inputs - psrc, stride
@@ -169,33 +215,169 @@
* Details : Load 16 byte elements in 'out0' from (psrc)
* Load 16 byte elements in 'out1' from (psrc + stride)
*/
-#define LD_B2(RTYPE, psrc, stride, out0, out1) { \
- out0 = LD_B(RTYPE, psrc); \
- out1 = LD_B(RTYPE, psrc + stride); \
-}
+#define LD_B2(RTYPE, psrc, stride, out0, out1) do { \
+ out0 = LD_B(RTYPE, psrc); \
+ out1 = LD_B(RTYPE, psrc + stride); \
+} while (0)
#define LD_UB2(...) LD_B2(v16u8, __VA_ARGS__)
#define LD_SB2(...) LD_B2(v16i8, __VA_ARGS__)
-#define LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3) { \
- LD_B2(RTYPE, psrc, stride, out0, out1); \
- LD_B2(RTYPE, psrc + 2 * stride , stride, out2, out3); \
-}
+#define LD_B3(RTYPE, psrc, stride, out0, out1, out2) do { \
+ LD_B2(RTYPE, psrc, stride, out0, out1); \
+ out2 = LD_B(RTYPE, psrc + 2 * stride); \
+} while (0)
+#define LD_UB3(...) LD_B3(v16u8, __VA_ARGS__)
+#define LD_SB3(...) LD_B3(v16i8, __VA_ARGS__)
+
+#define LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3) do { \
+ LD_B2(RTYPE, psrc, stride, out0, out1); \
+ LD_B2(RTYPE, psrc + 2 * stride , stride, out2, out3); \
+} while (0)
#define LD_UB4(...) LD_B4(v16u8, __VA_ARGS__)
#define LD_SB4(...) LD_B4(v16i8, __VA_ARGS__)
+#define LD_B8(RTYPE, psrc, stride, \
+ out0, out1, out2, out3, out4, out5, out6, out7) do { \
+ LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3); \
+ LD_B4(RTYPE, psrc + 4 * stride, stride, out4, out5, out6, out7); \
+} while (0)
+#define LD_UB8(...) LD_B8(v16u8, __VA_ARGS__)
+#define LD_SB8(...) LD_B8(v16i8, __VA_ARGS__)
+
/* Description : Load vectors with 8 halfword elements with stride
* Arguments : Inputs - psrc, stride
* Outputs - out0, out1
* Details : Load 8 halfword elements in 'out0' from (psrc)
* Load 8 halfword elements in 'out1' from (psrc + stride)
*/
-#define LD_H2(RTYPE, psrc, stride, out0, out1) { \
- out0 = LD_H(RTYPE, psrc); \
- out1 = LD_H(RTYPE, psrc + stride); \
-}
+#define LD_H2(RTYPE, psrc, stride, out0, out1) do { \
+ out0 = LD_H(RTYPE, psrc); \
+ out1 = LD_H(RTYPE, psrc + stride); \
+} while (0)
#define LD_UH2(...) LD_H2(v8u16, __VA_ARGS__)
#define LD_SH2(...) LD_H2(v8i16, __VA_ARGS__)
+/* Description : Load vectors with 4 word elements with stride
+ * Arguments : Inputs - psrc, stride
+ * Outputs - out0, out1, out2, out3
+ * Details : Load 4 word elements in 'out0' from (psrc + 0 * stride)
+ * Load 4 word elements in 'out1' from (psrc + 1 * stride)
+ * Load 4 word elements in 'out2' from (psrc + 2 * stride)
+ * Load 4 word elements in 'out3' from (psrc + 3 * stride)
+ */
+#define LD_W2(RTYPE, psrc, stride, out0, out1) do { \
+ out0 = LD_W(RTYPE, psrc); \
+ out1 = LD_W(RTYPE, psrc + stride); \
+} while (0)
+#define LD_UW2(...) LD_W2(v4u32, __VA_ARGS__)
+#define LD_SW2(...) LD_W2(v4i32, __VA_ARGS__)
+
+#define LD_W3(RTYPE, psrc, stride, out0, out1, out2) do { \
+ LD_W2(RTYPE, psrc, stride, out0, out1); \
+ out2 = LD_W(RTYPE, psrc + 2 * stride); \
+} while (0)
+#define LD_UW3(...) LD_W3(v4u32, __VA_ARGS__)
+#define LD_SW3(...) LD_W3(v4i32, __VA_ARGS__)
+
+#define LD_W4(RTYPE, psrc, stride, out0, out1, out2, out3) do { \
+ LD_W2(RTYPE, psrc, stride, out0, out1); \
+ LD_W2(RTYPE, psrc + 2 * stride, stride, out2, out3); \
+} while (0)
+#define LD_UW4(...) LD_W4(v4u32, __VA_ARGS__)
+#define LD_SW4(...) LD_W4(v4i32, __VA_ARGS__)
+
+/* Description : Store vectors of 16 byte elements with stride
+ * Arguments : Inputs - in0, in1, pdst, stride
+ * Details : Store 16 byte elements from 'in0' to (pdst)
+ * Store 16 byte elements from 'in1' to (pdst + stride)
+ */
+#define ST_B2(RTYPE, in0, in1, pdst, stride) do { \
+ ST_B(RTYPE, in0, pdst); \
+ ST_B(RTYPE, in1, pdst + stride); \
+} while (0)
+#define ST_UB2(...) ST_B2(v16u8, __VA_ARGS__)
+#define ST_SB2(...) ST_B2(v16i8, __VA_ARGS__)
+
+#define ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride) do { \
+ ST_B2(RTYPE, in0, in1, pdst, stride); \
+ ST_B2(RTYPE, in2, in3, pdst + 2 * stride, stride); \
+} while (0)
+#define ST_UB4(...) ST_B4(v16u8, __VA_ARGS__)
+#define ST_SB4(...) ST_B4(v16i8, __VA_ARGS__)
+
+#define ST_B8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \
+ pdst, stride) do { \
+ ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride); \
+ ST_B4(RTYPE, in4, in5, in6, in7, pdst + 4 * stride, stride); \
+} while (0)
+#define ST_UB8(...) ST_B8(v16u8, __VA_ARGS__)
+
+/* Description : Store vectors of 4 word elements with stride
+ * Arguments : Inputs - in0, in1, in2, in3, pdst, stride
+ * Details : Store 4 word elements from 'in0' to (pdst + 0 * stride)
+ * Store 4 word elements from 'in1' to (pdst + 1 * stride)
+ * Store 4 word elements from 'in2' to (pdst + 2 * stride)
+ * Store 4 word elements from 'in3' to (pdst + 3 * stride)
+ */
+#define ST_W2(RTYPE, in0, in1, pdst, stride) do { \
+ ST_W(RTYPE, in0, pdst); \
+ ST_W(RTYPE, in1, pdst + stride); \
+} while (0)
+#define ST_UW2(...) ST_W2(v4u32, __VA_ARGS__)
+#define ST_SW2(...) ST_W2(v4i32, __VA_ARGS__)
+
+#define ST_W3(RTYPE, in0, in1, in2, pdst, stride) do { \
+ ST_W2(RTYPE, in0, in1, pdst, stride); \
+ ST_W(RTYPE, in2, pdst + 2 * stride); \
+} while (0)
+#define ST_UW3(...) ST_W3(v4u32, __VA_ARGS__)
+#define ST_SW3(...) ST_W3(v4i32, __VA_ARGS__)
+
+#define ST_W4(RTYPE, in0, in1, in2, in3, pdst, stride) do { \
+ ST_W2(RTYPE, in0, in1, pdst, stride); \
+ ST_W2(RTYPE, in2, in3, pdst + 2 * stride, stride); \
+} while (0)
+#define ST_UW4(...) ST_W4(v4u32, __VA_ARGS__)
+#define ST_SW4(...) ST_W4(v4i32, __VA_ARGS__)
+
+/* Description : Store vectors of 8 halfword elements with stride
+ * Arguments : Inputs - in0, in1, pdst, stride
+ * Details : Store 8 halfword elements from 'in0' to (pdst)
+ * Store 8 halfword elements from 'in1' to (pdst + stride)
+ */
+#define ST_H2(RTYPE, in0, in1, pdst, stride) do { \
+ ST_H(RTYPE, in0, pdst); \
+ ST_H(RTYPE, in1, pdst + stride); \
+} while (0)
+#define ST_UH2(...) ST_H2(v8u16, __VA_ARGS__)
+#define ST_SH2(...) ST_H2(v8i16, __VA_ARGS__)
+
+/* Description : Store 2x4 byte block to destination memory from input vector
+ * Arguments : Inputs - in, stidx, pdst, stride
+ * Details : Index 'stidx' halfword element from 'in' vector is copied to
+ * the GP register and stored to (pdst)
+ * Index 'stidx+1' halfword element from 'in' vector is copied to
+ * the GP register and stored to (pdst + stride)
+ * Index 'stidx+2' halfword element from 'in' vector is copied to
+ * the GP register and stored to (pdst + 2 * stride)
+ * Index 'stidx+3' halfword element from 'in' vector is copied to
+ * the GP register and stored to (pdst + 3 * stride)
+ */
+#define ST2x4_UB(in, stidx, pdst, stride) do { \
+ uint8_t* pblk_2x4_m = (uint8_t*)pdst; \
+ const uint16_t out0_m = __msa_copy_s_h((v8i16)in, stidx); \
+ const uint16_t out1_m = __msa_copy_s_h((v8i16)in, stidx + 1); \
+ const uint16_t out2_m = __msa_copy_s_h((v8i16)in, stidx + 2); \
+ const uint16_t out3_m = __msa_copy_s_h((v8i16)in, stidx + 3); \
+ SH(out0_m, pblk_2x4_m); \
+ pblk_2x4_m += stride; \
+ SH(out1_m, pblk_2x4_m); \
+ pblk_2x4_m += stride; \
+ SH(out2_m, pblk_2x4_m); \
+ pblk_2x4_m += stride; \
+ SH(out3_m, pblk_2x4_m); \
+} while (0)
+
/* Description : Store 4x4 byte block to destination memory from input vector
* Arguments : Inputs - in0, in1, pdst, stride
* Details : 'Idx0' word element from input vector 'in0' is copied to the
@@ -207,14 +389,20 @@
* 'Idx3' word element from input vector 'in0' is copied to the
* GP register and stored to (pdst + 3 * stride)
*/
-#define ST4x4_UB(in0, in1, idx0, idx1, idx2, idx3, pdst, stride) { \
- uint8_t* const pblk_4x4_m = (uint8_t*)pdst; \
- const uint32_t out0_m = __msa_copy_s_w((v4i32)in0, idx0); \
- const uint32_t out1_m = __msa_copy_s_w((v4i32)in0, idx1); \
- const uint32_t out2_m = __msa_copy_s_w((v4i32)in1, idx2); \
- const uint32_t out3_m = __msa_copy_s_w((v4i32)in1, idx3); \
- SW4(out0_m, out1_m, out2_m, out3_m, pblk_4x4_m, stride); \
-}
+#define ST4x4_UB(in0, in1, idx0, idx1, idx2, idx3, pdst, stride) do { \
+ uint8_t* const pblk_4x4_m = (uint8_t*)pdst; \
+ const uint32_t out0_m = __msa_copy_s_w((v4i32)in0, idx0); \
+ const uint32_t out1_m = __msa_copy_s_w((v4i32)in0, idx1); \
+ const uint32_t out2_m = __msa_copy_s_w((v4i32)in1, idx2); \
+ const uint32_t out3_m = __msa_copy_s_w((v4i32)in1, idx3); \
+ SW4(out0_m, out1_m, out2_m, out3_m, pblk_4x4_m, stride); \
+} while (0)
+
+#define ST4x8_UB(in0, in1, pdst, stride) do { \
+ uint8_t* const pblk_4x8 = (uint8_t*)pdst; \
+ ST4x4_UB(in0, in0, 0, 1, 2, 3, pblk_4x8, stride); \
+ ST4x4_UB(in1, in1, 0, 1, 2, 3, pblk_4x8 + 4 * stride, stride); \
+} while (0)
/* Description : Immediate number of elements to slide
* Arguments : Inputs - in0, in1, slide_val
@@ -230,6 +418,30 @@
#define SLDI_SB(...) SLDI_B(v16i8, __VA_ARGS__)
#define SLDI_SH(...) SLDI_B(v8i16, __VA_ARGS__)
+/* Description : Shuffle byte vector elements as per mask vector
+ * Arguments : Inputs - in0, in1, in2, in3, mask0, mask1
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Byte elements from 'in0' & 'in1' are copied selectively to
+ * 'out0' as per control vector 'mask0'
+ */
+#define VSHF_B(RTYPE, in0, in1, mask) \
+ (RTYPE)__msa_vshf_b((v16i8)mask, (v16i8)in1, (v16i8)in0)
+
+#define VSHF_UB(...) VSHF_B(v16u8, __VA_ARGS__)
+#define VSHF_SB(...) VSHF_B(v16i8, __VA_ARGS__)
+#define VSHF_UH(...) VSHF_B(v8u16, __VA_ARGS__)
+#define VSHF_SH(...) VSHF_B(v8i16, __VA_ARGS__)
+
+#define VSHF_B2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) do { \
+ out0 = VSHF_B(RTYPE, in0, in1, mask0); \
+ out1 = VSHF_B(RTYPE, in2, in3, mask1); \
+} while (0)
+#define VSHF_B2_UB(...) VSHF_B2(v16u8, __VA_ARGS__)
+#define VSHF_B2_SB(...) VSHF_B2(v16i8, __VA_ARGS__)
+#define VSHF_B2_UH(...) VSHF_B2(v8u16, __VA_ARGS__)
+#define VSHF_B2_SH(...) VSHF_B2(v8i16, __VA_ARGS__)
+
/* Description : Shuffle halfword vector elements as per mask vector
* Arguments : Inputs - in0, in1, in2, in3, mask0, mask1
* Outputs - out0, out1
@@ -237,44 +449,219 @@
* Details : halfword elements from 'in0' & 'in1' are copied selectively to
* 'out0' as per control vector 'mask0'
*/
-#define VSHF_H2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) { \
- out0 = (RTYPE)__msa_vshf_h((v8i16)mask0, (v8i16)in1, (v8i16)in0); \
- out1 = (RTYPE)__msa_vshf_h((v8i16)mask1, (v8i16)in3, (v8i16)in2); \
-}
+#define VSHF_H2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) do { \
+ out0 = (RTYPE)__msa_vshf_h((v8i16)mask0, (v8i16)in1, (v8i16)in0); \
+ out1 = (RTYPE)__msa_vshf_h((v8i16)mask1, (v8i16)in3, (v8i16)in2); \
+} while (0)
#define VSHF_H2_UH(...) VSHF_H2(v8u16, __VA_ARGS__)
#define VSHF_H2_SH(...) VSHF_H2(v8i16, __VA_ARGS__)
+/* Description : Dot product of byte vector elements
+ * Arguments : Inputs - mult0, mult1, cnst0, cnst1
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Signed byte elements from 'mult0' are multiplied with
+ * signed byte elements from 'cnst0' producing a result
+ * twice the size of input i.e. signed halfword.
+ * The multiplication result of adjacent odd-even elements
+ * are added together and written to the 'out0' vector
+*/
+#define DOTP_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \
+ out0 = (RTYPE)__msa_dotp_s_h((v16i8)mult0, (v16i8)cnst0); \
+ out1 = (RTYPE)__msa_dotp_s_h((v16i8)mult1, (v16i8)cnst1); \
+} while (0)
+#define DOTP_SB2_SH(...) DOTP_SB2(v8i16, __VA_ARGS__)
+
+/* Description : Dot product of halfword vector elements
+ * Arguments : Inputs - mult0, mult1, cnst0, cnst1
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Signed halfword elements from 'mult0' are multiplied with
+ * signed halfword elements from 'cnst0' producing a result
+ * twice the size of input i.e. signed word.
+ * The multiplication result of adjacent odd-even elements
+ * are added together and written to the 'out0' vector
+ */
+#define DOTP_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \
+ out0 = (RTYPE)__msa_dotp_s_w((v8i16)mult0, (v8i16)cnst0); \
+ out1 = (RTYPE)__msa_dotp_s_w((v8i16)mult1, (v8i16)cnst1); \
+} while (0)
+#define DOTP_SH2_SW(...) DOTP_SH2(v4i32, __VA_ARGS__)
+
+/* Description : Dot product of unsigned word vector elements
+ * Arguments : Inputs - mult0, mult1, cnst0, cnst1
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Unsigned word elements from 'mult0' are multiplied with
+ * unsigned word elements from 'cnst0' producing a result
+ * twice the size of input i.e. unsigned double word.
+ * The multiplication result of adjacent odd-even elements
+ * are added together and written to the 'out0' vector
+ */
+#define DOTP_UW2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \
+ out0 = (RTYPE)__msa_dotp_u_d((v4u32)mult0, (v4u32)cnst0); \
+ out1 = (RTYPE)__msa_dotp_u_d((v4u32)mult1, (v4u32)cnst1); \
+} while (0)
+#define DOTP_UW2_UD(...) DOTP_UW2(v2u64, __VA_ARGS__)
+
+/* Description : Dot product & addition of halfword vector elements
+ * Arguments : Inputs - mult0, mult1, cnst0, cnst1
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Signed halfword elements from 'mult0' are multiplied with
+ * signed halfword elements from 'cnst0' producing a result
+ * twice the size of input i.e. signed word.
+ * The multiplication result of adjacent odd-even elements
+ * are added to the 'out0' vector
+ */
+#define DPADD_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \
+ out0 = (RTYPE)__msa_dpadd_s_w((v4i32)out0, (v8i16)mult0, (v8i16)cnst0); \
+ out1 = (RTYPE)__msa_dpadd_s_w((v4i32)out1, (v8i16)mult1, (v8i16)cnst1); \
+} while (0)
+#define DPADD_SH2_SW(...) DPADD_SH2(v4i32, __VA_ARGS__)
+
/* Description : Clips all signed halfword elements of input vector
* between 0 & 255
* Arguments : Input/output - val
* Return Type - signed halfword
*/
-#define CLIP_SH_0_255(val) { \
+#define CLIP_SH_0_255(val) do { \
const v8i16 max_m = __msa_ldi_h(255); \
val = __msa_maxi_s_h((v8i16)val, 0); \
val = __msa_min_s_h(max_m, (v8i16)val); \
-}
-#define CLIP_SH2_0_255(in0, in1) { \
- CLIP_SH_0_255(in0); \
- CLIP_SH_0_255(in1); \
-}
+} while (0)
+
+#define CLIP_SH2_0_255(in0, in1) do { \
+ CLIP_SH_0_255(in0); \
+ CLIP_SH_0_255(in1); \
+} while (0)
+
+#define CLIP_SH4_0_255(in0, in1, in2, in3) do { \
+ CLIP_SH2_0_255(in0, in1); \
+ CLIP_SH2_0_255(in2, in3); \
+} while (0)
+
+/* Description : Clips all unsigned halfword elements of input vector
+ * between 0 & 255
+ * Arguments : Input - in
+ * Output - out_m
+ * Return Type - unsigned halfword
+ */
+#define CLIP_UH_0_255(in) do { \
+ const v8u16 max_m = (v8u16)__msa_ldi_h(255); \
+ in = __msa_maxi_u_h((v8u16) in, 0); \
+ in = __msa_min_u_h((v8u16) max_m, (v8u16) in); \
+} while (0)
+
+#define CLIP_UH2_0_255(in0, in1) do { \
+ CLIP_UH_0_255(in0); \
+ CLIP_UH_0_255(in1); \
+} while (0)
/* Description : Clips all signed word elements of input vector
* between 0 & 255
* Arguments : Input/output - val
* Return Type - signed word
*/
-#define CLIP_SW_0_255(val) { \
+#define CLIP_SW_0_255(val) do { \
const v4i32 max_m = __msa_ldi_w(255); \
val = __msa_maxi_s_w((v4i32)val, 0); \
val = __msa_min_s_w(max_m, (v4i32)val); \
+} while (0)
+
+#define CLIP_SW4_0_255(in0, in1, in2, in3) do { \
+ CLIP_SW_0_255(in0); \
+ CLIP_SW_0_255(in1); \
+ CLIP_SW_0_255(in2); \
+ CLIP_SW_0_255(in3); \
+} while (0)
+
+/* Description : Horizontal addition of 4 signed word elements of input vector
+ * Arguments : Input - in (signed word vector)
+ * Output - sum_m (i32 sum)
+ * Return Type - signed word (GP)
+ * Details : 4 signed word elements of 'in' vector are added together and
+ * the resulting integer sum is returned
+ */
+static WEBP_INLINE int32_t func_hadd_sw_s32(v4i32 in) {
+ const v2i64 res0_m = __msa_hadd_s_d((v4i32)in, (v4i32)in);
+ const v2i64 res1_m = __msa_splati_d(res0_m, 1);
+ const v2i64 out = res0_m + res1_m;
+ int32_t sum_m = __msa_copy_s_w((v4i32)out, 0);
+ return sum_m;
}
-#define CLIP_SW4_0_255(in0, in1, in2, in3) { \
- CLIP_SW_0_255(in0); \
- CLIP_SW_0_255(in1); \
- CLIP_SW_0_255(in2); \
- CLIP_SW_0_255(in3); \
+#define HADD_SW_S32(in) func_hadd_sw_s32(in)
+
+/* Description : Horizontal addition of 8 signed halfword elements
+ * Arguments : Input - in (signed halfword vector)
+ * Output - sum_m (s32 sum)
+ * Return Type - signed word
+ * Details : 8 signed halfword elements of input vector are added
+ * together and the resulting integer sum is returned
+ */
+static WEBP_INLINE int32_t func_hadd_sh_s32(v8i16 in) {
+ const v4i32 res = __msa_hadd_s_w(in, in);
+ const v2i64 res0 = __msa_hadd_s_d(res, res);
+ const v2i64 res1 = __msa_splati_d(res0, 1);
+ const v2i64 res2 = res0 + res1;
+ const int32_t sum_m = __msa_copy_s_w((v4i32)res2, 0);
+ return sum_m;
+}
+#define HADD_SH_S32(in) func_hadd_sh_s32(in)
+
+/* Description : Horizontal addition of 8 unsigned halfword elements
+ * Arguments : Input - in (unsigned halfword vector)
+ * Output - sum_m (u32 sum)
+ * Return Type - unsigned word
+ * Details : 8 unsigned halfword elements of input vector are added
+ * together and the resulting integer sum is returned
+ */
+static WEBP_INLINE uint32_t func_hadd_uh_u32(v8u16 in) {
+ uint32_t sum_m;
+ const v4u32 res_m = __msa_hadd_u_w(in, in);
+ v2u64 res0_m = __msa_hadd_u_d(res_m, res_m);
+ v2u64 res1_m = (v2u64)__msa_splati_d((v2i64)res0_m, 1);
+ res0_m = res0_m + res1_m;
+ sum_m = __msa_copy_s_w((v4i32)res0_m, 0);
+ return sum_m;
}
+#define HADD_UH_U32(in) func_hadd_uh_u32(in)
+
+/* Description : Horizontal addition of signed half word vector elements
+ Arguments : Inputs - in0, in1
+ Outputs - out0, out1
+ Return Type - as per RTYPE
+ Details : Each signed odd half word element from 'in0' is added to
+ even signed half word element from 'in0' (pairwise) and the
+ halfword result is written in 'out0'
+*/
+#define HADD_SH2(RTYPE, in0, in1, out0, out1) do { \
+ out0 = (RTYPE)__msa_hadd_s_w((v8i16)in0, (v8i16)in0); \
+ out1 = (RTYPE)__msa_hadd_s_w((v8i16)in1, (v8i16)in1); \
+} while (0)
+#define HADD_SH2_SW(...) HADD_SH2(v4i32, __VA_ARGS__)
+
+#define HADD_SH4(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) do { \
+ HADD_SH2(RTYPE, in0, in1, out0, out1); \
+ HADD_SH2(RTYPE, in2, in3, out2, out3); \
+} while (0)
+#define HADD_SH4_SW(...) HADD_SH4(v4i32, __VA_ARGS__)
+
+/* Description : Horizontal subtraction of unsigned byte vector elements
+ * Arguments : Inputs - in0, in1
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Each unsigned odd byte element from 'in0' is subtracted from
+ * even unsigned byte element from 'in0' (pairwise) and the
+ * halfword result is written to 'out0'
+ */
+#define HSUB_UB2(RTYPE, in0, in1, out0, out1) do { \
+ out0 = (RTYPE)__msa_hsub_u_h((v16u8)in0, (v16u8)in0); \
+ out1 = (RTYPE)__msa_hsub_u_h((v16u8)in1, (v16u8)in1); \
+} while (0)
+#define HSUB_UB2_UH(...) HSUB_UB2(v8u16, __VA_ARGS__)
+#define HSUB_UB2_SH(...) HSUB_UB2(v8i16, __VA_ARGS__)
+#define HSUB_UB2_SW(...) HSUB_UB2(v4i32, __VA_ARGS__)
/* Description : Set element n input vector to GPR value
* Arguments : Inputs - in0, in1, in2, in3
@@ -282,23 +669,188 @@
* Return Type - as per RTYPE
* Details : Set element 0 in vector 'out' to value specified in 'in0'
*/
-#define INSERT_W2(RTYPE, in0, in1, out) { \
+#define INSERT_W2(RTYPE, in0, in1, out) do { \
out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0); \
out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1); \
-}
+} while (0)
#define INSERT_W2_UB(...) INSERT_W2(v16u8, __VA_ARGS__)
#define INSERT_W2_SB(...) INSERT_W2(v16i8, __VA_ARGS__)
-#define INSERT_W4(RTYPE, in0, in1, in2, in3, out) { \
- out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0); \
- out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1); \
- out = (RTYPE)__msa_insert_w((v4i32)out, 2, in2); \
- out = (RTYPE)__msa_insert_w((v4i32)out, 3, in3); \
-}
+#define INSERT_W4(RTYPE, in0, in1, in2, in3, out) do { \
+ out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0); \
+ out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1); \
+ out = (RTYPE)__msa_insert_w((v4i32)out, 2, in2); \
+ out = (RTYPE)__msa_insert_w((v4i32)out, 3, in3); \
+} while (0)
#define INSERT_W4_UB(...) INSERT_W4(v16u8, __VA_ARGS__)
#define INSERT_W4_SB(...) INSERT_W4(v16i8, __VA_ARGS__)
#define INSERT_W4_SW(...) INSERT_W4(v4i32, __VA_ARGS__)
+/* Description : Set element n of double word input vector to GPR value
+ * Arguments : Inputs - in0, in1
+ * Output - out
+ * Return Type - as per RTYPE
+ * Details : Set element 0 in vector 'out' to GPR value specified in 'in0'
+ * Set element 1 in vector 'out' to GPR value specified in 'in1'
+ */
+#define INSERT_D2(RTYPE, in0, in1, out) do { \
+ out = (RTYPE)__msa_insert_d((v2i64)out, 0, in0); \
+ out = (RTYPE)__msa_insert_d((v2i64)out, 1, in1); \
+} while (0)
+#define INSERT_D2_UB(...) INSERT_D2(v16u8, __VA_ARGS__)
+#define INSERT_D2_SB(...) INSERT_D2(v16i8, __VA_ARGS__)
+
+/* Description : Interleave even byte elements from vectors
+ * Arguments : Inputs - in0, in1, in2, in3
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Even byte elements of 'in0' and 'in1' are interleaved
+ * and written to 'out0'
+ */
+#define ILVEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_ilvev_b((v16i8)in1, (v16i8)in0); \
+ out1 = (RTYPE)__msa_ilvev_b((v16i8)in3, (v16i8)in2); \
+} while (0)
+#define ILVEV_B2_UB(...) ILVEV_B2(v16u8, __VA_ARGS__)
+#define ILVEV_B2_SB(...) ILVEV_B2(v16i8, __VA_ARGS__)
+#define ILVEV_B2_UH(...) ILVEV_B2(v8u16, __VA_ARGS__)
+#define ILVEV_B2_SH(...) ILVEV_B2(v8i16, __VA_ARGS__)
+#define ILVEV_B2_SD(...) ILVEV_B2(v2i64, __VA_ARGS__)
+
+/* Description : Interleave odd byte elements from vectors
+ * Arguments : Inputs - in0, in1, in2, in3
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Odd byte elements of 'in0' and 'in1' are interleaved
+ * and written to 'out0'
+ */
+#define ILVOD_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_ilvod_b((v16i8)in1, (v16i8)in0); \
+ out1 = (RTYPE)__msa_ilvod_b((v16i8)in3, (v16i8)in2); \
+} while (0)
+#define ILVOD_B2_UB(...) ILVOD_B2(v16u8, __VA_ARGS__)
+#define ILVOD_B2_SB(...) ILVOD_B2(v16i8, __VA_ARGS__)
+#define ILVOD_B2_UH(...) ILVOD_B2(v8u16, __VA_ARGS__)
+#define ILVOD_B2_SH(...) ILVOD_B2(v8i16, __VA_ARGS__)
+#define ILVOD_B2_SD(...) ILVOD_B2(v2i64, __VA_ARGS__)
+
+/* Description : Interleave even halfword elements from vectors
+ * Arguments : Inputs - in0, in1, in2, in3
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Even halfword elements of 'in0' and 'in1' are interleaved
+ * and written to 'out0'
+ */
+#define ILVEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_ilvev_h((v8i16)in1, (v8i16)in0); \
+ out1 = (RTYPE)__msa_ilvev_h((v8i16)in3, (v8i16)in2); \
+} while (0)
+#define ILVEV_H2_UB(...) ILVEV_H2(v16u8, __VA_ARGS__)
+#define ILVEV_H2_UH(...) ILVEV_H2(v8u16, __VA_ARGS__)
+#define ILVEV_H2_SH(...) ILVEV_H2(v8i16, __VA_ARGS__)
+#define ILVEV_H2_SW(...) ILVEV_H2(v4i32, __VA_ARGS__)
+
+/* Description : Interleave odd halfword elements from vectors
+ * Arguments : Inputs - in0, in1, in2, in3
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Odd halfword elements of 'in0' and 'in1' are interleaved
+ * and written to 'out0'
+ */
+#define ILVOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_ilvod_h((v8i16)in1, (v8i16)in0); \
+ out1 = (RTYPE)__msa_ilvod_h((v8i16)in3, (v8i16)in2); \
+} while (0)
+#define ILVOD_H2_UB(...) ILVOD_H2(v16u8, __VA_ARGS__)
+#define ILVOD_H2_UH(...) ILVOD_H2(v8u16, __VA_ARGS__)
+#define ILVOD_H2_SH(...) ILVOD_H2(v8i16, __VA_ARGS__)
+#define ILVOD_H2_SW(...) ILVOD_H2(v4i32, __VA_ARGS__)
+
+/* Description : Interleave even word elements from vectors
+ * Arguments : Inputs - in0, in1, in2, in3
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Even word elements of 'in0' and 'in1' are interleaved
+ * and written to 'out0'
+ */
+#define ILVEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_ilvev_w((v4i32)in1, (v4i32)in0); \
+ out1 = (RTYPE)__msa_ilvev_w((v4i32)in3, (v4i32)in2); \
+} while (0)
+#define ILVEV_W2_UB(...) ILVEV_W2(v16u8, __VA_ARGS__)
+#define ILVEV_W2_SB(...) ILVEV_W2(v16i8, __VA_ARGS__)
+#define ILVEV_W2_UH(...) ILVEV_W2(v8u16, __VA_ARGS__)
+#define ILVEV_W2_SD(...) ILVEV_W2(v2i64, __VA_ARGS__)
+
+/* Description : Interleave even-odd word elements from vectors
+ * Arguments : Inputs - in0, in1, in2, in3
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Even word elements of 'in0' and 'in1' are interleaved
+ * and written to 'out0'
+ * Odd word elements of 'in2' and 'in3' are interleaved
+ * and written to 'out1'
+ */
+#define ILVEVOD_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_ilvev_w((v4i32)in1, (v4i32)in0); \
+ out1 = (RTYPE)__msa_ilvod_w((v4i32)in3, (v4i32)in2); \
+} while (0)
+#define ILVEVOD_W2_UB(...) ILVEVOD_W2(v16u8, __VA_ARGS__)
+#define ILVEVOD_W2_UH(...) ILVEVOD_W2(v8u16, __VA_ARGS__)
+#define ILVEVOD_W2_SH(...) ILVEVOD_W2(v8i16, __VA_ARGS__)
+#define ILVEVOD_W2_SW(...) ILVEVOD_W2(v4i32, __VA_ARGS__)
+
+/* Description : Interleave even-odd half-word elements from vectors
+ * Arguments : Inputs - in0, in1, in2, in3
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Even half-word elements of 'in0' and 'in1' are interleaved
+ * and written to 'out0'
+ * Odd half-word elements of 'in2' and 'in3' are interleaved
+ * and written to 'out1'
+ */
+#define ILVEVOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_ilvev_h((v8i16)in1, (v8i16)in0); \
+ out1 = (RTYPE)__msa_ilvod_h((v8i16)in3, (v8i16)in2); \
+} while (0)
+#define ILVEVOD_H2_UB(...) ILVEVOD_H2(v16u8, __VA_ARGS__)
+#define ILVEVOD_H2_UH(...) ILVEVOD_H2(v8u16, __VA_ARGS__)
+#define ILVEVOD_H2_SH(...) ILVEVOD_H2(v8i16, __VA_ARGS__)
+#define ILVEVOD_H2_SW(...) ILVEVOD_H2(v4i32, __VA_ARGS__)
+
+/* Description : Interleave even double word elements from vectors
+ * Arguments : Inputs - in0, in1, in2, in3
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Even double word elements of 'in0' and 'in1' are interleaved
+ * and written to 'out0'
+ */
+#define ILVEV_D2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_ilvev_d((v2i64)in1, (v2i64)in0); \
+ out1 = (RTYPE)__msa_ilvev_d((v2i64)in3, (v2i64)in2); \
+} while (0)
+#define ILVEV_D2_UB(...) ILVEV_D2(v16u8, __VA_ARGS__)
+#define ILVEV_D2_SB(...) ILVEV_D2(v16i8, __VA_ARGS__)
+#define ILVEV_D2_SW(...) ILVEV_D2(v4i32, __VA_ARGS__)
+#define ILVEV_D2_SD(...) ILVEV_D2(v2i64, __VA_ARGS__)
+
+/* Description : Interleave left half of byte elements from vectors
+ * Arguments : Inputs - in0, in1, in2, in3
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Left half of byte elements of 'in0' and 'in1' are interleaved
+ * and written to 'out0'.
+ */
+#define ILVL_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_ilvl_b((v16i8)in0, (v16i8)in1); \
+ out1 = (RTYPE)__msa_ilvl_b((v16i8)in2, (v16i8)in3); \
+} while (0)
+#define ILVL_B2_UB(...) ILVL_B2(v16u8, __VA_ARGS__)
+#define ILVL_B2_SB(...) ILVL_B2(v16i8, __VA_ARGS__)
+#define ILVL_B2_UH(...) ILVL_B2(v8u16, __VA_ARGS__)
+#define ILVL_B2_SH(...) ILVL_B2(v8i16, __VA_ARGS__)
+#define ILVL_B2_SW(...) ILVL_B2(v4i32, __VA_ARGS__)
+
/* Description : Interleave right half of byte elements from vectors
* Arguments : Inputs - in0, in1, in2, in3
* Outputs - out0, out1
@@ -306,10 +858,10 @@
* Details : Right half of byte elements of 'in0' and 'in1' are interleaved
* and written to out0.
*/
-#define ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1) { \
- out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1); \
- out1 = (RTYPE)__msa_ilvr_b((v16i8)in2, (v16i8)in3); \
-}
+#define ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1); \
+ out1 = (RTYPE)__msa_ilvr_b((v16i8)in2, (v16i8)in3); \
+} while (0)
#define ILVR_B2_UB(...) ILVR_B2(v16u8, __VA_ARGS__)
#define ILVR_B2_SB(...) ILVR_B2(v16i8, __VA_ARGS__)
#define ILVR_B2_UH(...) ILVR_B2(v8u16, __VA_ARGS__)
@@ -317,10 +869,10 @@
#define ILVR_B2_SW(...) ILVR_B2(v4i32, __VA_ARGS__)
#define ILVR_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \
- out0, out1, out2, out3) { \
+ out0, out1, out2, out3) do { \
ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1); \
ILVR_B2(RTYPE, in4, in5, in6, in7, out2, out3); \
-}
+} while (0)
#define ILVR_B4_UB(...) ILVR_B4(v16u8, __VA_ARGS__)
#define ILVR_B4_SB(...) ILVR_B4(v16i8, __VA_ARGS__)
#define ILVR_B4_UH(...) ILVR_B4(v8u16, __VA_ARGS__)
@@ -334,19 +886,19 @@
* Details : Right half of halfword elements of 'in0' and 'in1' are
* interleaved and written to 'out0'.
*/
-#define ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1) { \
- out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1); \
- out1 = (RTYPE)__msa_ilvr_h((v8i16)in2, (v8i16)in3); \
-}
+#define ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1); \
+ out1 = (RTYPE)__msa_ilvr_h((v8i16)in2, (v8i16)in3); \
+} while (0)
#define ILVR_H2_UB(...) ILVR_H2(v16u8, __VA_ARGS__)
#define ILVR_H2_SH(...) ILVR_H2(v8i16, __VA_ARGS__)
#define ILVR_H2_SW(...) ILVR_H2(v4i32, __VA_ARGS__)
#define ILVR_H4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \
- out0, out1, out2, out3) { \
+ out0, out1, out2, out3) do { \
ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1); \
ILVR_H2(RTYPE, in4, in5, in6, in7, out2, out3); \
-}
+} while (0)
#define ILVR_H4_UB(...) ILVR_H4(v16u8, __VA_ARGS__)
#define ILVR_H4_SH(...) ILVR_H4(v8i16, __VA_ARGS__)
#define ILVR_H4_SW(...) ILVR_H4(v4i32, __VA_ARGS__)
@@ -358,31 +910,57 @@
* Details : Right half of double word elements of 'in0' and 'in1' are
* interleaved and written to 'out0'.
*/
-#define ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1) { \
- out0 = (RTYPE)__msa_ilvr_d((v2i64)in0, (v2i64)in1); \
- out1 = (RTYPE)__msa_ilvr_d((v2i64)in2, (v2i64)in3); \
-}
+#define ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_ilvr_d((v2i64)in0, (v2i64)in1); \
+ out1 = (RTYPE)__msa_ilvr_d((v2i64)in2, (v2i64)in3); \
+} while (0)
#define ILVR_D2_UB(...) ILVR_D2(v16u8, __VA_ARGS__)
#define ILVR_D2_SB(...) ILVR_D2(v16i8, __VA_ARGS__)
#define ILVR_D2_SH(...) ILVR_D2(v8i16, __VA_ARGS__)
-#define ILVRL_H2(RTYPE, in0, in1, out0, out1) { \
+#define ILVR_D4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \
+ out0, out1, out2, out3) do { \
+ ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1); \
+ ILVR_D2(RTYPE, in4, in5, in6, in7, out2, out3); \
+} while (0)
+#define ILVR_D4_SB(...) ILVR_D4(v16i8, __VA_ARGS__)
+#define ILVR_D4_UB(...) ILVR_D4(v16u8, __VA_ARGS__)
+
+/* Description : Interleave both left and right half of input vectors
+ * Arguments : Inputs - in0, in1
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Right half of byte elements from 'in0' and 'in1' are
+ * interleaved and written to 'out0'
+ */
+#define ILVRL_B2(RTYPE, in0, in1, out0, out1) do { \
+ out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1); \
+ out1 = (RTYPE)__msa_ilvl_b((v16i8)in0, (v16i8)in1); \
+} while (0)
+#define ILVRL_B2_UB(...) ILVRL_B2(v16u8, __VA_ARGS__)
+#define ILVRL_B2_SB(...) ILVRL_B2(v16i8, __VA_ARGS__)
+#define ILVRL_B2_UH(...) ILVRL_B2(v8u16, __VA_ARGS__)
+#define ILVRL_B2_SH(...) ILVRL_B2(v8i16, __VA_ARGS__)
+#define ILVRL_B2_SW(...) ILVRL_B2(v4i32, __VA_ARGS__)
+
+#define ILVRL_H2(RTYPE, in0, in1, out0, out1) do { \
out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1); \
out1 = (RTYPE)__msa_ilvl_h((v8i16)in0, (v8i16)in1); \
-}
+} while (0)
#define ILVRL_H2_UB(...) ILVRL_H2(v16u8, __VA_ARGS__)
#define ILVRL_H2_SB(...) ILVRL_H2(v16i8, __VA_ARGS__)
#define ILVRL_H2_SH(...) ILVRL_H2(v8i16, __VA_ARGS__)
#define ILVRL_H2_SW(...) ILVRL_H2(v4i32, __VA_ARGS__)
#define ILVRL_H2_UW(...) ILVRL_H2(v4u32, __VA_ARGS__)
-#define ILVRL_W2(RTYPE, in0, in1, out0, out1) { \
+#define ILVRL_W2(RTYPE, in0, in1, out0, out1) do { \
out0 = (RTYPE)__msa_ilvr_w((v4i32)in0, (v4i32)in1); \
out1 = (RTYPE)__msa_ilvl_w((v4i32)in0, (v4i32)in1); \
-}
+} while (0)
#define ILVRL_W2_UB(...) ILVRL_W2(v16u8, __VA_ARGS__)
#define ILVRL_W2_SH(...) ILVRL_W2(v8i16, __VA_ARGS__)
#define ILVRL_W2_SW(...) ILVRL_W2(v4i32, __VA_ARGS__)
+#define ILVRL_W2_UW(...) ILVRL_W2(v4u32, __VA_ARGS__)
/* Description : Pack even byte elements of vector pairs
* Arguments : Inputs - in0, in1, in2, in3
@@ -392,15 +970,76 @@
* 'out0' & even byte elements of 'in1' are copied to the right
* half of 'out0'.
*/
-#define PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) { \
- out0 = (RTYPE)__msa_pckev_b((v16i8)in0, (v16i8)in1); \
- out1 = (RTYPE)__msa_pckev_b((v16i8)in2, (v16i8)in3); \
-}
+#define PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_pckev_b((v16i8)in0, (v16i8)in1); \
+ out1 = (RTYPE)__msa_pckev_b((v16i8)in2, (v16i8)in3); \
+} while (0)
#define PCKEV_B2_SB(...) PCKEV_B2(v16i8, __VA_ARGS__)
#define PCKEV_B2_UB(...) PCKEV_B2(v16u8, __VA_ARGS__)
#define PCKEV_B2_SH(...) PCKEV_B2(v8i16, __VA_ARGS__)
#define PCKEV_B2_SW(...) PCKEV_B2(v4i32, __VA_ARGS__)
+#define PCKEV_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \
+ out0, out1, out2, out3) do { \
+ PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1); \
+ PCKEV_B2(RTYPE, in4, in5, in6, in7, out2, out3); \
+} while (0)
+#define PCKEV_B4_SB(...) PCKEV_B4(v16i8, __VA_ARGS__)
+#define PCKEV_B4_UB(...) PCKEV_B4(v16u8, __VA_ARGS__)
+#define PCKEV_B4_SH(...) PCKEV_B4(v8i16, __VA_ARGS__)
+#define PCKEV_B4_SW(...) PCKEV_B4(v4i32, __VA_ARGS__)
+
+/* Description : Pack even halfword elements of vector pairs
+ * Arguments : Inputs - in0, in1, in2, in3
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Even halfword elements of 'in0' are copied to the left half of
+ * 'out0' & even halfword elements of 'in1' are copied to the
+ * right half of 'out0'.
+ */
+#define PCKEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_pckev_h((v8i16)in0, (v8i16)in1); \
+ out1 = (RTYPE)__msa_pckev_h((v8i16)in2, (v8i16)in3); \
+} while (0)
+#define PCKEV_H2_UH(...) PCKEV_H2(v8u16, __VA_ARGS__)
+#define PCKEV_H2_SH(...) PCKEV_H2(v8i16, __VA_ARGS__)
+#define PCKEV_H2_SW(...) PCKEV_H2(v4i32, __VA_ARGS__)
+#define PCKEV_H2_UW(...) PCKEV_H2(v4u32, __VA_ARGS__)
+
+/* Description : Pack even word elements of vector pairs
+ * Arguments : Inputs - in0, in1, in2, in3
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Even word elements of 'in0' are copied to the left half of
+ * 'out0' & even word elements of 'in1' are copied to the
+ * right half of 'out0'.
+ */
+#define PCKEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_pckev_w((v4i32)in0, (v4i32)in1); \
+ out1 = (RTYPE)__msa_pckev_w((v4i32)in2, (v4i32)in3); \
+} while (0)
+#define PCKEV_W2_UH(...) PCKEV_W2(v8u16, __VA_ARGS__)
+#define PCKEV_W2_SH(...) PCKEV_W2(v8i16, __VA_ARGS__)
+#define PCKEV_W2_SW(...) PCKEV_W2(v4i32, __VA_ARGS__)
+#define PCKEV_W2_UW(...) PCKEV_W2(v4u32, __VA_ARGS__)
+
+/* Description : Pack odd halfword elements of vector pairs
+ * Arguments : Inputs - in0, in1, in2, in3
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Odd halfword elements of 'in0' are copied to the left half of
+ * 'out0' & odd halfword elements of 'in1' are copied to the
+ * right half of 'out0'.
+ */
+#define PCKOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_pckod_h((v8i16)in0, (v8i16)in1); \
+ out1 = (RTYPE)__msa_pckod_h((v8i16)in2, (v8i16)in3); \
+} while (0)
+#define PCKOD_H2_UH(...) PCKOD_H2(v8u16, __VA_ARGS__)
+#define PCKOD_H2_SH(...) PCKOD_H2(v8i16, __VA_ARGS__)
+#define PCKOD_H2_SW(...) PCKOD_H2(v4i32, __VA_ARGS__)
+#define PCKOD_H2_UW(...) PCKOD_H2(v4u32, __VA_ARGS__)
+
/* Description : Arithmetic immediate shift right all elements of word vector
* Arguments : Inputs - in0, in1, shift
* Outputs - in place operation
@@ -408,17 +1047,17 @@
* Details : Each element of vector 'in0' is right shifted by 'shift' and
* the result is written in-place. 'shift' is a GP variable.
*/
-#define SRAI_W2(RTYPE, in0, in1, shift_val) { \
- in0 = (RTYPE)SRAI_W(in0, shift_val); \
- in1 = (RTYPE)SRAI_W(in1, shift_val); \
-}
+#define SRAI_W2(RTYPE, in0, in1, shift_val) do { \
+ in0 = (RTYPE)SRAI_W(in0, shift_val); \
+ in1 = (RTYPE)SRAI_W(in1, shift_val); \
+} while (0)
#define SRAI_W2_SW(...) SRAI_W2(v4i32, __VA_ARGS__)
#define SRAI_W2_UW(...) SRAI_W2(v4u32, __VA_ARGS__)
-#define SRAI_W4(RTYPE, in0, in1, in2, in3, shift_val) { \
- SRAI_W2(RTYPE, in0, in1, shift_val); \
- SRAI_W2(RTYPE, in2, in3, shift_val); \
-}
+#define SRAI_W4(RTYPE, in0, in1, in2, in3, shift_val) do { \
+ SRAI_W2(RTYPE, in0, in1, shift_val); \
+ SRAI_W2(RTYPE, in2, in3, shift_val); \
+} while (0)
#define SRAI_W4_SW(...) SRAI_W4(v4i32, __VA_ARGS__)
#define SRAI_W4_UW(...) SRAI_W4(v4u32, __VA_ARGS__)
@@ -429,10 +1068,10 @@
* Details : Each element of vector 'in0' is right shifted by 'shift' and
* the result is written in-place. 'shift' is a GP variable.
*/
-#define SRAI_H2(RTYPE, in0, in1, shift_val) { \
- in0 = (RTYPE)SRAI_H(in0, shift_val); \
- in1 = (RTYPE)SRAI_H(in1, shift_val); \
-}
+#define SRAI_H2(RTYPE, in0, in1, shift_val) do { \
+ in0 = (RTYPE)SRAI_H(in0, shift_val); \
+ in1 = (RTYPE)SRAI_H(in1, shift_val); \
+} while (0)
#define SRAI_H2_SH(...) SRAI_H2(v8i16, __VA_ARGS__)
#define SRAI_H2_UH(...) SRAI_H2(v8u16, __VA_ARGS__)
@@ -443,48 +1082,166 @@
* Details : Each element of vector 'in0' is right shifted by 'shift' and
* the result is written in-place. 'shift' is a GP variable.
*/
-#define SRARI_W2(RTYPE, in0, in1, shift) { \
+#define SRARI_W2(RTYPE, in0, in1, shift) do { \
in0 = (RTYPE)__msa_srari_w((v4i32)in0, shift); \
in1 = (RTYPE)__msa_srari_w((v4i32)in1, shift); \
-}
+} while (0)
#define SRARI_W2_SW(...) SRARI_W2(v4i32, __VA_ARGS__)
-#define SRARI_W4(RTYPE, in0, in1, in2, in3, shift) { \
- SRARI_W2(RTYPE, in0, in1, shift); \
- SRARI_W2(RTYPE, in2, in3, shift); \
-}
+#define SRARI_W4(RTYPE, in0, in1, in2, in3, shift) do { \
+ SRARI_W2(RTYPE, in0, in1, shift); \
+ SRARI_W2(RTYPE, in2, in3, shift); \
+} while (0)
#define SRARI_W4_SH(...) SRARI_W4(v8i16, __VA_ARGS__)
#define SRARI_W4_UW(...) SRARI_W4(v4u32, __VA_ARGS__)
#define SRARI_W4_SW(...) SRARI_W4(v4i32, __VA_ARGS__)
+/* Description : Shift right arithmetic rounded double words
+ * Arguments : Inputs - in0, in1, shift
+ * Outputs - in place operation
+ * Return Type - as per RTYPE
+ * Details : Each element of vector 'in0' is shifted right arithmetically by
+ * the number of bits in the corresponding element in the vector
+ * 'shift'. The last discarded bit is added to shifted value for
+ * rounding and the result is written in-place.
+ * 'shift' is a vector.
+ */
+#define SRAR_D2(RTYPE, in0, in1, shift) do { \
+ in0 = (RTYPE)__msa_srar_d((v2i64)in0, (v2i64)shift); \
+ in1 = (RTYPE)__msa_srar_d((v2i64)in1, (v2i64)shift); \
+} while (0)
+#define SRAR_D2_SW(...) SRAR_D2(v4i32, __VA_ARGS__)
+#define SRAR_D2_SD(...) SRAR_D2(v2i64, __VA_ARGS__)
+#define SRAR_D2_UD(...) SRAR_D2(v2u64, __VA_ARGS__)
+
+#define SRAR_D4(RTYPE, in0, in1, in2, in3, shift) do { \
+ SRAR_D2(RTYPE, in0, in1, shift); \
+ SRAR_D2(RTYPE, in2, in3, shift); \
+} while (0)
+#define SRAR_D4_SD(...) SRAR_D4(v2i64, __VA_ARGS__)
+#define SRAR_D4_UD(...) SRAR_D4(v2u64, __VA_ARGS__)
+
/* Description : Addition of 2 pairs of half-word vectors
* Arguments : Inputs - in0, in1, in2, in3
* Outputs - out0, out1
* Details : Each element in 'in0' is added to 'in1' and result is written
* to 'out0'.
*/
-#define ADDVI_H2(RTYPE, in0, in1, in2, in3, out0, out1) { \
- out0 = (RTYPE)ADDVI_H(in0, in1); \
- out1 = (RTYPE)ADDVI_H(in2, in3); \
-}
+#define ADDVI_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)ADDVI_H(in0, in1); \
+ out1 = (RTYPE)ADDVI_H(in2, in3); \
+} while (0)
#define ADDVI_H2_SH(...) ADDVI_H2(v8i16, __VA_ARGS__)
#define ADDVI_H2_UH(...) ADDVI_H2(v8u16, __VA_ARGS__)
+/* Description : Addition of 2 pairs of word vectors
+ * Arguments : Inputs - in0, in1, in2, in3
+ * Outputs - out0, out1
+ * Details : Each element in 'in0' is added to 'in1' and result is written
+ * to 'out0'.
+ */
+#define ADDVI_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)ADDVI_W(in0, in1); \
+ out1 = (RTYPE)ADDVI_W(in2, in3); \
+} while (0)
+#define ADDVI_W2_SW(...) ADDVI_W2(v4i32, __VA_ARGS__)
+
+/* Description : Fill 2 pairs of word vectors with GP registers
+ * Arguments : Inputs - in0, in1
+ * Outputs - out0, out1
+ * Details : GP register in0 is replicated in each word element of out0
+ * GP register in1 is replicated in each word element of out1
+ */
+#define FILL_W2(RTYPE, in0, in1, out0, out1) do { \
+ out0 = (RTYPE)__msa_fill_w(in0); \
+ out1 = (RTYPE)__msa_fill_w(in1); \
+} while (0)
+#define FILL_W2_SW(...) FILL_W2(v4i32, __VA_ARGS__)
+
/* Description : Addition of 2 pairs of vectors
* Arguments : Inputs - in0, in1, in2, in3
* Outputs - out0, out1
* Details : Each element in 'in0' is added to 'in1' and result is written
* to 'out0'.
*/
-#define ADD2(in0, in1, in2, in3, out0, out1) { \
- out0 = in0 + in1; \
- out1 = in2 + in3; \
-}
+#define ADD2(in0, in1, in2, in3, out0, out1) do { \
+ out0 = in0 + in1; \
+ out1 = in2 + in3; \
+} while (0)
+
#define ADD4(in0, in1, in2, in3, in4, in5, in6, in7, \
- out0, out1, out2, out3) { \
+ out0, out1, out2, out3) do { \
ADD2(in0, in1, in2, in3, out0, out1); \
ADD2(in4, in5, in6, in7, out2, out3); \
-}
+} while (0)
+
+/* Description : Subtraction of 2 pairs of vectors
+ * Arguments : Inputs - in0, in1, in2, in3
+ * Outputs - out0, out1
+ * Details : Each element in 'in1' is subtracted from 'in0' and result is
+ * written to 'out0'.
+ */
+#define SUB2(in0, in1, in2, in3, out0, out1) do { \
+ out0 = in0 - in1; \
+ out1 = in2 - in3; \
+} while (0)
+
+#define SUB3(in0, in1, in2, in3, in4, in5, out0, out1, out2) do { \
+ out0 = in0 - in1; \
+ out1 = in2 - in3; \
+ out2 = in4 - in5; \
+} while (0)
+
+#define SUB4(in0, in1, in2, in3, in4, in5, in6, in7, \
+ out0, out1, out2, out3) do { \
+ out0 = in0 - in1; \
+ out1 = in2 - in3; \
+ out2 = in4 - in5; \
+ out3 = in6 - in7; \
+} while (0)
+
+/* Description : Addition - Subtraction of input vectors
+ * Arguments : Inputs - in0, in1
+ * Outputs - out0, out1
+ * Details : Each element in 'in1' is added to 'in0' and result is
+ * written to 'out0'.
+ * Each element in 'in1' is subtracted from 'in0' and result is
+ * written to 'out1'.
+ */
+#define ADDSUB2(in0, in1, out0, out1) do { \
+ out0 = in0 + in1; \
+ out1 = in0 - in1; \
+} while (0)
+
+/* Description : Multiplication of pairs of vectors
+ * Arguments : Inputs - in0, in1, in2, in3
+ * Outputs - out0, out1
+ * Details : Each element from 'in0' is multiplied with elements from 'in1'
+ * and the result is written to 'out0'
+ */
+#define MUL2(in0, in1, in2, in3, out0, out1) do { \
+ out0 = in0 * in1; \
+ out1 = in2 * in3; \
+} while (0)
+
+#define MUL4(in0, in1, in2, in3, in4, in5, in6, in7, \
+ out0, out1, out2, out3) do { \
+ MUL2(in0, in1, in2, in3, out0, out1); \
+ MUL2(in4, in5, in6, in7, out2, out3); \
+} while (0)
+
+/* Description : Sign extend halfword elements from right half of the vector
+ * Arguments : Input - in (halfword vector)
+ * Output - out (sign extended word vector)
+ * Return Type - signed word
+ * Details : Sign bit of halfword elements from input vector 'in' is
+ * extracted and interleaved with same vector 'in0' to generate
+ * 4 word elements keeping sign intact
+ */
+#define UNPCK_R_SH_SW(in, out) do { \
+ const v8i16 sign_m = __msa_clti_s_h((v8i16)in, 0); \
+ out = (v4i32)__msa_ilvr_h(sign_m, (v8i16)in); \
+} while (0)
/* Description : Sign extend halfword elements from input vector and return
* the result in pair of vectors
@@ -497,29 +1254,82 @@
* Then interleaved left with same vector 'in0' to
* generate 4 signed word elements in 'out1'
*/
-#define UNPCK_SH_SW(in, out0, out1) { \
+#define UNPCK_SH_SW(in, out0, out1) do { \
const v8i16 tmp_m = __msa_clti_s_h((v8i16)in, 0); \
ILVRL_H2_SW(tmp_m, in, out0, out1); \
-}
+} while (0)
/* Description : Butterfly of 4 input vectors
* Arguments : Inputs - in0, in1, in2, in3
* Outputs - out0, out1, out2, out3
* Details : Butterfly operation
*/
-#define BUTTERFLY_4(in0, in1, in2, in3, out0, out1, out2, out3) { \
- out0 = in0 + in3; \
- out1 = in1 + in2; \
- out2 = in1 - in2; \
- out3 = in0 - in3; \
-}
+#define BUTTERFLY_4(in0, in1, in2, in3, out0, out1, out2, out3) do { \
+ out0 = in0 + in3; \
+ out1 = in1 + in2; \
+ out2 = in1 - in2; \
+ out3 = in0 - in3; \
+} while (0)
+
+/* Description : Transpose 16x4 block into 4x16 with byte elements in vectors
+ * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7,
+ * in8, in9, in10, in11, in12, in13, in14, in15
+ * Outputs - out0, out1, out2, out3
+ * Return Type - unsigned byte
+ */
+#define TRANSPOSE16x4_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7, \
+ in8, in9, in10, in11, in12, in13, in14, in15, \
+ out0, out1, out2, out3) do { \
+ v2i64 tmp0_m, tmp1_m, tmp2_m, tmp3_m, tmp4_m, tmp5_m; \
+ ILVEV_W2_SD(in0, in4, in8, in12, tmp2_m, tmp3_m); \
+ ILVEV_W2_SD(in1, in5, in9, in13, tmp0_m, tmp1_m); \
+ ILVEV_D2_UB(tmp2_m, tmp3_m, tmp0_m, tmp1_m, out1, out3); \
+ ILVEV_W2_SD(in2, in6, in10, in14, tmp4_m, tmp5_m); \
+ ILVEV_W2_SD(in3, in7, in11, in15, tmp0_m, tmp1_m); \
+ ILVEV_D2_SD(tmp4_m, tmp5_m, tmp0_m, tmp1_m, tmp2_m, tmp3_m); \
+ ILVEV_B2_SD(out1, out3, tmp2_m, tmp3_m, tmp0_m, tmp1_m); \
+ ILVEVOD_H2_UB(tmp0_m, tmp1_m, tmp0_m, tmp1_m, out0, out2); \
+ ILVOD_B2_SD(out1, out3, tmp2_m, tmp3_m, tmp0_m, tmp1_m); \
+ ILVEVOD_H2_UB(tmp0_m, tmp1_m, tmp0_m, tmp1_m, out1, out3); \
+} while (0)
+
+/* Description : Transpose 16x8 block into 8x16 with byte elements in vectors
+ * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7,
+ * in8, in9, in10, in11, in12, in13, in14, in15
+ * Outputs - out0, out1, out2, out3, out4, out5, out6, out7
+ * Return Type - unsigned byte
+ */
+#define TRANSPOSE16x8_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7, \
+ in8, in9, in10, in11, in12, in13, in14, in15, \
+ out0, out1, out2, out3, out4, out5, \
+ out6, out7) do { \
+ v8i16 tmp0_m, tmp1_m, tmp4_m, tmp5_m, tmp6_m, tmp7_m; \
+ v4i32 tmp2_m, tmp3_m; \
+ ILVEV_D2_UB(in0, in8, in1, in9, out7, out6); \
+ ILVEV_D2_UB(in2, in10, in3, in11, out5, out4); \
+ ILVEV_D2_UB(in4, in12, in5, in13, out3, out2); \
+ ILVEV_D2_UB(in6, in14, in7, in15, out1, out0); \
+ ILVEV_B2_SH(out7, out6, out5, out4, tmp0_m, tmp1_m); \
+ ILVOD_B2_SH(out7, out6, out5, out4, tmp4_m, tmp5_m); \
+ ILVEV_B2_UB(out3, out2, out1, out0, out5, out7); \
+ ILVOD_B2_SH(out3, out2, out1, out0, tmp6_m, tmp7_m); \
+ ILVEV_H2_SW(tmp0_m, tmp1_m, out5, out7, tmp2_m, tmp3_m); \
+ ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out0, out4); \
+ ILVOD_H2_SW(tmp0_m, tmp1_m, out5, out7, tmp2_m, tmp3_m); \
+ ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out2, out6); \
+ ILVEV_H2_SW(tmp4_m, tmp5_m, tmp6_m, tmp7_m, tmp2_m, tmp3_m); \
+ ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out1, out5); \
+ ILVOD_H2_SW(tmp4_m, tmp5_m, tmp6_m, tmp7_m, tmp2_m, tmp3_m); \
+ ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out3, out7); \
+} while (0)
/* Description : Transpose 4x4 block with word elements in vectors
* Arguments : Inputs - in0, in1, in2, in3
* Outputs - out0, out1, out2, out3
* Return Type - as per RTYPE
*/
-#define TRANSPOSE4x4_W(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) { \
+#define TRANSPOSE4x4_W(RTYPE, in0, in1, in2, in3, \
+ out0, out1, out2, out3) do { \
v4i32 s0_m, s1_m, s2_m, s3_m; \
ILVRL_W2_SW(in1, in0, s0_m, s1_m); \
ILVRL_W2_SW(in3, in2, s2_m, s3_m); \
@@ -527,7 +1337,7 @@
out1 = (RTYPE)__msa_ilvl_d((v2i64)s2_m, (v2i64)s0_m); \
out2 = (RTYPE)__msa_ilvr_d((v2i64)s3_m, (v2i64)s1_m); \
out3 = (RTYPE)__msa_ilvl_d((v2i64)s3_m, (v2i64)s1_m); \
-}
+} while (0)
#define TRANSPOSE4x4_SW_SW(...) TRANSPOSE4x4_W(v4i32, __VA_ARGS__)
/* Description : Add block 4x4
@@ -535,7 +1345,7 @@
* Details : Least significant 4 bytes from each input vector are added to
* the destination bytes, clipped between 0-255 and stored.
*/
-#define ADDBLK_ST4x4_UB(in0, in1, in2, in3, pdst, stride) { \
+#define ADDBLK_ST4x4_UB(in0, in1, in2, in3, pdst, stride) do { \
uint32_t src0_m, src1_m, src2_m, src3_m; \
v8i16 inp0_m, inp1_m, res0_m, res1_m; \
v16i8 dst0_m = { 0 }; \
@@ -550,6 +1360,31 @@
CLIP_SH2_0_255(res0_m, res1_m); \
PCKEV_B2_SB(res0_m, res0_m, res1_m, res1_m, dst0_m, dst1_m); \
ST4x4_UB(dst0_m, dst1_m, 0, 1, 0, 1, pdst, stride); \
-}
+} while (0)
+
+/* Description : Pack even byte elements, extract 0 & 2 index words from pair
+ * of results and store 4 words in destination memory as per
+ * stride
+ * Arguments : Inputs - in0, in1, in2, in3, pdst, stride
+ */
+#define PCKEV_ST4x4_UB(in0, in1, in2, in3, pdst, stride) do { \
+ v16i8 tmp0_m, tmp1_m; \
+ PCKEV_B2_SB(in1, in0, in3, in2, tmp0_m, tmp1_m); \
+ ST4x4_UB(tmp0_m, tmp1_m, 0, 2, 0, 2, pdst, stride); \
+} while (0)
+
+/* Description : average with rounding (in0 + in1 + 1) / 2.
+ * Arguments : Inputs - in0, in1, in2, in3,
+ * Outputs - out0, out1
+ * Return Type - as per RTYPE
+ * Details : Each unsigned byte element from 'in0' vector is added with
+ * each unsigned byte element from 'in1' vector. Then the average
+ * with rounding is calculated and written to 'out0'
+ */
+#define AVER_UB2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
+ out0 = (RTYPE)__msa_aver_u_b((v16u8)in0, (v16u8)in1); \
+ out1 = (RTYPE)__msa_aver_u_b((v16u8)in2, (v16u8)in3); \
+} while (0)
+#define AVER_UB2_UB(...) AVER_UB2(v16u8, __VA_ARGS__)
#endif /* WEBP_DSP_MSA_MACRO_H_ */
diff --git a/src/3rdparty/libwebp/src/dsp/neon.h b/src/3rdparty/libwebp/src/dsp/neon.h
index 0a06266..3b548a6 100644
--- a/src/3rdparty/libwebp/src/dsp/neon.h
+++ b/src/3rdparty/libwebp/src/dsp/neon.h
@@ -79,4 +79,22 @@ static WEBP_INLINE int32x4x4_t Transpose4x4(const int32x4x4_t rows) {
}
}
+#if 0 // Useful debug macro.
+#include <stdio.h>
+#define PRINT_REG(REG, SIZE) do { \
+ int i; \
+ printf("%s \t[%d]: 0x", #REG, SIZE); \
+ if (SIZE == 8) { \
+ uint8_t _tmp[8]; \
+ vst1_u8(_tmp, (REG)); \
+ for (i = 0; i < 8; ++i) printf("%.2x ", _tmp[i]); \
+ } else if (SIZE == 16) { \
+ uint16_t _tmp[4]; \
+ vst1_u16(_tmp, (REG)); \
+ for (i = 0; i < 4; ++i) printf("%.4x ", _tmp[i]); \
+ } \
+ printf("\n"); \
+} while (0)
+#endif
+
#endif // WEBP_DSP_NEON_H_
diff --git a/src/3rdparty/libwebp/src/dsp/rescaler.c b/src/3rdparty/libwebp/src/dsp/rescaler.c
index bc743d5..0f54502 100644
--- a/src/3rdparty/libwebp/src/dsp/rescaler.c
+++ b/src/3rdparty/libwebp/src/dsp/rescaler.c
@@ -14,7 +14,7 @@
#include <assert.h>
#include "./dsp.h"
-#include "../utils/rescaler.h"
+#include "../utils/rescaler_utils.h"
//------------------------------------------------------------------------------
// Implementations of critical functions ImportRow / ExportRow
@@ -173,10 +173,10 @@ void WebPRescalerExportRow(WebPRescaler* const wrk) {
WebPRescalerExportRowExpand(wrk);
} else if (wrk->fxy_scale) {
WebPRescalerExportRowShrink(wrk);
- } else { // very special case for src = dst = 1x1
+ } else { // special case
int i;
+ assert(wrk->src_height == wrk->dst_height && wrk->x_add == 1);
assert(wrk->src_width == 1 && wrk->dst_width <= 2);
- assert(wrk->src_height == 1 && wrk->dst_height == 1);
for (i = 0; i < wrk->num_channels * wrk->dst_width; ++i) {
wrk->dst[i] = wrk->irow[i];
wrk->irow[i] = 0;
@@ -199,6 +199,7 @@ WebPRescalerExportRowFunc WebPRescalerExportRowShrink;
extern void WebPRescalerDspInitSSE2(void);
extern void WebPRescalerDspInitMIPS32(void);
extern void WebPRescalerDspInitMIPSdspR2(void);
+extern void WebPRescalerDspInitMSA(void);
extern void WebPRescalerDspInitNEON(void);
static volatile VP8CPUInfo rescaler_last_cpuinfo_used =
@@ -233,6 +234,11 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInit(void) {
WebPRescalerDspInitMIPSdspR2();
}
#endif
+#if defined(WEBP_USE_MSA)
+ if (VP8GetCPUInfo(kMSA)) {
+ WebPRescalerDspInitMSA();
+ }
+#endif
}
rescaler_last_cpuinfo_used = VP8GetCPUInfo;
}
diff --git a/src/3rdparty/libwebp/src/dsp/rescaler_mips32.c b/src/3rdparty/libwebp/src/dsp/rescaler_mips32.c
index ddaa391..e09ad5d 100644
--- a/src/3rdparty/libwebp/src/dsp/rescaler_mips32.c
+++ b/src/3rdparty/libwebp/src/dsp/rescaler_mips32.c
@@ -16,7 +16,7 @@
#if defined(WEBP_USE_MIPS32)
#include <assert.h>
-#include "../utils/rescaler.h"
+#include "../utils/rescaler_utils.h"
//------------------------------------------------------------------------------
// Row import
diff --git a/src/3rdparty/libwebp/src/dsp/rescaler_mips_dsp_r2.c b/src/3rdparty/libwebp/src/dsp/rescaler_mips_dsp_r2.c
index b457d0a..2308d64 100644
--- a/src/3rdparty/libwebp/src/dsp/rescaler_mips_dsp_r2.c
+++ b/src/3rdparty/libwebp/src/dsp/rescaler_mips_dsp_r2.c
@@ -16,7 +16,7 @@
#if defined(WEBP_USE_MIPS_DSP_R2)
#include <assert.h>
-#include "../utils/rescaler.h"
+#include "../utils/rescaler_utils.h"
#define ROUNDER (WEBP_RESCALER_ONE >> 1)
#define MULT_FIX(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX)
diff --git a/src/3rdparty/libwebp/src/dsp/rescaler_msa.c b/src/3rdparty/libwebp/src/dsp/rescaler_msa.c
new file mode 100644
index 0000000..2c10e55
--- /dev/null
+++ b/src/3rdparty/libwebp/src/dsp/rescaler_msa.c
@@ -0,0 +1,444 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the COPYING file in the root of the source
+// tree. An additional intellectual property rights grant can be found
+// in the file PATENTS. All contributing project authors may
+// be found in the AUTHORS file in the root of the source tree.
+// -----------------------------------------------------------------------------
+//
+// MSA version of rescaling functions
+//
+// Author: Prashant Patil (prashant.patil@imgtec.com)
+
+#include "./dsp.h"
+
+#if defined(WEBP_USE_MSA)
+
+#include <assert.h>
+
+#include "../utils/rescaler_utils.h"
+#include "./msa_macro.h"
+
+#define ROUNDER (WEBP_RESCALER_ONE >> 1)
+#define MULT_FIX(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX)
+
+#define CALC_MULT_FIX_16(in0, in1, in2, in3, scale, shift, dst) do { \
+ v4u32 tmp0, tmp1, tmp2, tmp3; \
+ v16u8 t0, t1, t2, t3, t4, t5; \
+ v2u64 out0, out1, out2, out3; \
+ ILVRL_W2_UW(zero, in0, tmp0, tmp1); \
+ ILVRL_W2_UW(zero, in1, tmp2, tmp3); \
+ DOTP_UW2_UD(tmp0, tmp1, scale, scale, out0, out1); \
+ DOTP_UW2_UD(tmp2, tmp3, scale, scale, out2, out3); \
+ SRAR_D4_UD(out0, out1, out2, out3, shift); \
+ PCKEV_B2_UB(out1, out0, out3, out2, t0, t1); \
+ ILVRL_W2_UW(zero, in2, tmp0, tmp1); \
+ ILVRL_W2_UW(zero, in3, tmp2, tmp3); \
+ DOTP_UW2_UD(tmp0, tmp1, scale, scale, out0, out1); \
+ DOTP_UW2_UD(tmp2, tmp3, scale, scale, out2, out3); \
+ SRAR_D4_UD(out0, out1, out2, out3, shift); \
+ PCKEV_B2_UB(out1, out0, out3, out2, t2, t3); \
+ PCKEV_B2_UB(t1, t0, t3, t2, t4, t5); \
+ dst = (v16u8)__msa_pckev_b((v16i8)t5, (v16i8)t4); \
+} while (0)
+
+#define CALC_MULT_FIX_4(in0, scale, shift, dst) do { \
+ v4u32 tmp0, tmp1; \
+ v16i8 t0, t1; \
+ v2u64 out0, out1; \
+ ILVRL_W2_UW(zero, in0, tmp0, tmp1); \
+ DOTP_UW2_UD(tmp0, tmp1, scale, scale, out0, out1); \
+ SRAR_D2_UD(out0, out1, shift); \
+ t0 = __msa_pckev_b((v16i8)out1, (v16i8)out0); \
+ t1 = __msa_pckev_b(t0, t0); \
+ t0 = __msa_pckev_b(t1, t1); \
+ dst = __msa_copy_s_w((v4i32)t0, 0); \
+} while (0)
+
+#define CALC_MULT_FIX1_16(in0, in1, in2, in3, fyscale, shift, \
+ dst0, dst1, dst2, dst3) do { \
+ v4u32 tmp0, tmp1, tmp2, tmp3; \
+ v2u64 out0, out1, out2, out3; \
+ ILVRL_W2_UW(zero, in0, tmp0, tmp1); \
+ ILVRL_W2_UW(zero, in1, tmp2, tmp3); \
+ DOTP_UW2_UD(tmp0, tmp1, fyscale, fyscale, out0, out1); \
+ DOTP_UW2_UD(tmp2, tmp3, fyscale, fyscale, out2, out3); \
+ SRAR_D4_UD(out0, out1, out2, out3, shift); \
+ PCKEV_W2_UW(out1, out0, out3, out2, dst0, dst1); \
+ ILVRL_W2_UW(zero, in2, tmp0, tmp1); \
+ ILVRL_W2_UW(zero, in3, tmp2, tmp3); \
+ DOTP_UW2_UD(tmp0, tmp1, fyscale, fyscale, out0, out1); \
+ DOTP_UW2_UD(tmp2, tmp3, fyscale, fyscale, out2, out3); \
+ SRAR_D4_UD(out0, out1, out2, out3, shift); \
+ PCKEV_W2_UW(out1, out0, out3, out2, dst2, dst3); \
+} while (0)
+
+#define CALC_MULT_FIX1_4(in0, scale, shift, dst) do { \
+ v4u32 tmp0, tmp1; \
+ v2u64 out0, out1; \
+ ILVRL_W2_UW(zero, in0, tmp0, tmp1); \
+ DOTP_UW2_UD(tmp0, tmp1, scale, scale, out0, out1); \
+ SRAR_D2_UD(out0, out1, shift); \
+ dst = (v4u32)__msa_pckev_w((v4i32)out1, (v4i32)out0); \
+} while (0)
+
+#define CALC_MULT_FIX2_16(in0, in1, in2, in3, mult, scale, shift, \
+ dst0, dst1) do { \
+ v4u32 tmp0, tmp1, tmp2, tmp3; \
+ v2u64 out0, out1, out2, out3; \
+ ILVRL_W2_UW(in0, in2, tmp0, tmp1); \
+ ILVRL_W2_UW(in1, in3, tmp2, tmp3); \
+ DOTP_UW2_UD(tmp0, tmp1, mult, mult, out0, out1); \
+ DOTP_UW2_UD(tmp2, tmp3, mult, mult, out2, out3); \
+ SRAR_D4_UD(out0, out1, out2, out3, shift); \
+ DOTP_UW2_UD(out0, out1, scale, scale, out0, out1); \
+ DOTP_UW2_UD(out2, out3, scale, scale, out2, out3); \
+ SRAR_D4_UD(out0, out1, out2, out3, shift); \
+ PCKEV_B2_UB(out1, out0, out3, out2, dst0, dst1); \
+} while (0)
+
+#define CALC_MULT_FIX2_4(in0, in1, mult, scale, shift, dst) do { \
+ v4u32 tmp0, tmp1; \
+ v2u64 out0, out1; \
+ v16i8 t0, t1; \
+ ILVRL_W2_UW(in0, in1, tmp0, tmp1); \
+ DOTP_UW2_UD(tmp0, tmp1, mult, mult, out0, out1); \
+ SRAR_D2_UD(out0, out1, shift); \
+ DOTP_UW2_UD(out0, out1, scale, scale, out0, out1); \
+ SRAR_D2_UD(out0, out1, shift); \
+ t0 = __msa_pckev_b((v16i8)out1, (v16i8)out0); \
+ t1 = __msa_pckev_b(t0, t0); \
+ t0 = __msa_pckev_b(t1, t1); \
+ dst = __msa_copy_s_w((v4i32)t0, 0); \
+} while (0)
+
+static WEBP_INLINE void ExportRowExpand_0(const uint32_t* frow, uint8_t* dst,
+ int length,
+ WebPRescaler* const wrk) {
+ const v4u32 scale = (v4u32)__msa_fill_w(wrk->fy_scale);
+ const v4u32 shift = (v4u32)__msa_fill_w(WEBP_RESCALER_RFIX);
+ const v4i32 zero = { 0 };
+
+ while (length >= 16) {
+ v4u32 src0, src1, src2, src3;
+ v16u8 out;
+ LD_UW4(frow, 4, src0, src1, src2, src3);
+ CALC_MULT_FIX_16(src0, src1, src2, src3, scale, shift, out);
+ ST_UB(out, dst);
+ length -= 16;
+ frow += 16;
+ dst += 16;
+ }
+ if (length > 0) {
+ int x_out;
+ if (length >= 12) {
+ uint32_t val0_m, val1_m, val2_m;
+ v4u32 src0, src1, src2;
+ LD_UW3(frow, 4, src0, src1, src2);
+ CALC_MULT_FIX_4(src0, scale, shift, val0_m);
+ CALC_MULT_FIX_4(src1, scale, shift, val1_m);
+ CALC_MULT_FIX_4(src2, scale, shift, val2_m);
+ SW3(val0_m, val1_m, val2_m, dst, 4);
+ length -= 12;
+ frow += 12;
+ dst += 12;
+ } else if (length >= 8) {
+ uint32_t val0_m, val1_m;
+ v4u32 src0, src1;
+ LD_UW2(frow, 4, src0, src1);
+ CALC_MULT_FIX_4(src0, scale, shift, val0_m);
+ CALC_MULT_FIX_4(src1, scale, shift, val1_m);
+ SW2(val0_m, val1_m, dst, 4);
+ length -= 8;
+ frow += 8;
+ dst += 8;
+ } else if (length >= 4) {
+ uint32_t val0_m;
+ const v4u32 src0 = LD_UW(frow);
+ CALC_MULT_FIX_4(src0, scale, shift, val0_m);
+ SW(val0_m, dst);
+ length -= 4;
+ frow += 4;
+ dst += 4;
+ }
+ for (x_out = 0; x_out < length; ++x_out) {
+ const uint32_t J = frow[x_out];
+ const int v = (int)MULT_FIX(J, wrk->fy_scale);
+ assert(v >= 0 && v <= 255);
+ dst[x_out] = v;
+ }
+ }
+}
+
+static WEBP_INLINE void ExportRowExpand_1(const uint32_t* frow, uint32_t* irow,
+ uint8_t* dst, int length,
+ WebPRescaler* const wrk) {
+ const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub);
+ const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B);
+ const v4i32 B1 = __msa_fill_w(B);
+ const v4i32 A1 = __msa_fill_w(A);
+ const v4i32 AB = __msa_ilvr_w(A1, B1);
+ const v4u32 scale = (v4u32)__msa_fill_w(wrk->fy_scale);
+ const v4u32 shift = (v4u32)__msa_fill_w(WEBP_RESCALER_RFIX);
+
+ while (length >= 16) {
+ v4u32 frow0, frow1, frow2, frow3, irow0, irow1, irow2, irow3;
+ v16u8 t0, t1, t2, t3, t4, t5;
+ LD_UW4(frow, 4, frow0, frow1, frow2, frow3);
+ LD_UW4(irow, 4, irow0, irow1, irow2, irow3);
+ CALC_MULT_FIX2_16(frow0, frow1, irow0, irow1, AB, scale, shift, t0, t1);
+ CALC_MULT_FIX2_16(frow2, frow3, irow2, irow3, AB, scale, shift, t2, t3);
+ PCKEV_B2_UB(t1, t0, t3, t2, t4, t5);
+ t0 = (v16u8)__msa_pckev_b((v16i8)t5, (v16i8)t4);
+ ST_UB(t0, dst);
+ frow += 16;
+ irow += 16;
+ dst += 16;
+ length -= 16;
+ }
+ if (length > 0) {
+ int x_out;
+ if (length >= 12) {
+ uint32_t val0_m, val1_m, val2_m;
+ v4u32 frow0, frow1, frow2, irow0, irow1, irow2;
+ LD_UW3(frow, 4, frow0, frow1, frow2);
+ LD_UW3(irow, 4, irow0, irow1, irow2);
+ CALC_MULT_FIX2_4(frow0, irow0, AB, scale, shift, val0_m);
+ CALC_MULT_FIX2_4(frow1, irow1, AB, scale, shift, val1_m);
+ CALC_MULT_FIX2_4(frow2, irow2, AB, scale, shift, val2_m);
+ SW3(val0_m, val1_m, val2_m, dst, 4);
+ frow += 12;
+ irow += 12;
+ dst += 12;
+ length -= 12;
+ } else if (length >= 8) {
+ uint32_t val0_m, val1_m;
+ v4u32 frow0, frow1, irow0, irow1;
+ LD_UW2(frow, 4, frow0, frow1);
+ LD_UW2(irow, 4, irow0, irow1);
+ CALC_MULT_FIX2_4(frow0, irow0, AB, scale, shift, val0_m);
+ CALC_MULT_FIX2_4(frow1, irow1, AB, scale, shift, val1_m);
+ SW2(val0_m, val1_m, dst, 4);
+ frow += 4;
+ irow += 4;
+ dst += 4;
+ length -= 4;
+ } else if (length >= 4) {
+ uint32_t val0_m;
+ const v4u32 frow0 = LD_UW(frow + 0);
+ const v4u32 irow0 = LD_UW(irow + 0);
+ CALC_MULT_FIX2_4(frow0, irow0, AB, scale, shift, val0_m);
+ SW(val0_m, dst);
+ frow += 4;
+ irow += 4;
+ dst += 4;
+ length -= 4;
+ }
+ for (x_out = 0; x_out < length; ++x_out) {
+ const uint64_t I = (uint64_t)A * frow[x_out]
+ + (uint64_t)B * irow[x_out];
+ const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX);
+ const int v = (int)MULT_FIX(J, wrk->fy_scale);
+ assert(v >= 0 && v <= 255);
+ dst[x_out] = v;
+ }
+ }
+}
+
+static void RescalerExportRowExpand(WebPRescaler* const wrk) {
+ uint8_t* dst = wrk->dst;
+ rescaler_t* irow = wrk->irow;
+ const int x_out_max = wrk->dst_width * wrk->num_channels;
+ const rescaler_t* frow = wrk->frow;
+ assert(!WebPRescalerOutputDone(wrk));
+ assert(wrk->y_accum <= 0);
+ assert(wrk->y_expand);
+ assert(wrk->y_sub != 0);
+ if (wrk->y_accum == 0) {
+ ExportRowExpand_0(frow, dst, x_out_max, wrk);
+ } else {
+ ExportRowExpand_1(frow, irow, dst, x_out_max, wrk);
+ }
+}
+
+static WEBP_INLINE void ExportRowShrink_0(const uint32_t* frow, uint32_t* irow,
+ uint8_t* dst, int length,
+ const uint32_t yscale,
+ WebPRescaler* const wrk) {
+ const v4u32 y_scale = (v4u32)__msa_fill_w(yscale);
+ const v4u32 fxyscale = (v4u32)__msa_fill_w(wrk->fxy_scale);
+ const v4u32 shiftval = (v4u32)__msa_fill_w(WEBP_RESCALER_RFIX);
+ const v4i32 zero = { 0 };
+
+ while (length >= 16) {
+ v4u32 src0, src1, src2, src3, frac0, frac1, frac2, frac3;
+ v16u8 out;
+ LD_UW4(frow, 4, src0, src1, src2, src3);
+ CALC_MULT_FIX1_16(src0, src1, src2, src3, y_scale, shiftval,
+ frac0, frac1, frac2, frac3);
+ LD_UW4(irow, 4, src0, src1, src2, src3);
+ SUB4(src0, frac0, src1, frac1, src2, frac2, src3, frac3,
+ src0, src1, src2, src3);
+ CALC_MULT_FIX_16(src0, src1, src2, src3, fxyscale, shiftval, out);
+ ST_UB(out, dst);
+ ST_UW4(frac0, frac1, frac2, frac3, irow, 4);
+ frow += 16;
+ irow += 16;
+ dst += 16;
+ length -= 16;
+ }
+ if (length > 0) {
+ int x_out;
+ if (length >= 12) {
+ uint32_t val0_m, val1_m, val2_m;
+ v4u32 src0, src1, src2, frac0, frac1, frac2;
+ LD_UW3(frow, 4, src0, src1, src2);
+ CALC_MULT_FIX1_4(src0, y_scale, shiftval, frac0);
+ CALC_MULT_FIX1_4(src1, y_scale, shiftval, frac1);
+ CALC_MULT_FIX1_4(src2, y_scale, shiftval, frac2);
+ LD_UW3(irow, 4, src0, src1, src2);
+ SUB3(src0, frac0, src1, frac1, src2, frac2, src0, src1, src2);
+ CALC_MULT_FIX_4(src0, fxyscale, shiftval, val0_m);
+ CALC_MULT_FIX_4(src1, fxyscale, shiftval, val1_m);
+ CALC_MULT_FIX_4(src2, fxyscale, shiftval, val2_m);
+ SW3(val0_m, val1_m, val2_m, dst, 4);
+ ST_UW3(frac0, frac1, frac2, irow, 4);
+ frow += 12;
+ irow += 12;
+ dst += 12;
+ length -= 12;
+ } else if (length >= 8) {
+ uint32_t val0_m, val1_m;
+ v4u32 src0, src1, frac0, frac1;
+ LD_UW2(frow, 4, src0, src1);
+ CALC_MULT_FIX1_4(src0, y_scale, shiftval, frac0);
+ CALC_MULT_FIX1_4(src1, y_scale, shiftval, frac1);
+ LD_UW2(irow, 4, src0, src1);
+ SUB2(src0, frac0, src1, frac1, src0, src1);
+ CALC_MULT_FIX_4(src0, fxyscale, shiftval, val0_m);
+ CALC_MULT_FIX_4(src1, fxyscale, shiftval, val1_m);
+ SW2(val0_m, val1_m, dst, 4);
+ ST_UW2(frac0, frac1, irow, 4);
+ frow += 8;
+ irow += 8;
+ dst += 8;
+ length -= 8;
+ } else if (length >= 4) {
+ uint32_t val0_m;
+ v4u32 frac0;
+ v4u32 src0 = LD_UW(frow);
+ CALC_MULT_FIX1_4(src0, y_scale, shiftval, frac0);
+ src0 = LD_UW(irow);
+ src0 = src0 - frac0;
+ CALC_MULT_FIX_4(src0, fxyscale, shiftval, val0_m);
+ SW(val0_m, dst);
+ ST_UW(frac0, irow);
+ frow += 4;
+ irow += 4;
+ dst += 4;
+ length -= 4;
+ }
+ for (x_out = 0; x_out < length; ++x_out) {
+ const uint32_t frac = (uint32_t)MULT_FIX(frow[x_out], yscale);
+ const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale);
+ assert(v >= 0 && v <= 255);
+ dst[x_out] = v;
+ irow[x_out] = frac;
+ }
+ }
+}
+
+static WEBP_INLINE void ExportRowShrink_1(uint32_t* irow, uint8_t* dst,
+ int length,
+ WebPRescaler* const wrk) {
+ const v4u32 scale = (v4u32)__msa_fill_w(wrk->fxy_scale);
+ const v4u32 shift = (v4u32)__msa_fill_w(WEBP_RESCALER_RFIX);
+ const v4i32 zero = { 0 };
+
+ while (length >= 16) {
+ v4u32 src0, src1, src2, src3;
+ v16u8 dst0;
+ LD_UW4(irow, 4, src0, src1, src2, src3);
+ CALC_MULT_FIX_16(src0, src1, src2, src3, scale, shift, dst0);
+ ST_UB(dst0, dst);
+ ST_SW4(zero, zero, zero, zero, irow, 4);
+ length -= 16;
+ irow += 16;
+ dst += 16;
+ }
+ if (length > 0) {
+ int x_out;
+ if (length >= 12) {
+ uint32_t val0_m, val1_m, val2_m;
+ v4u32 src0, src1, src2;
+ LD_UW3(irow, 4, src0, src1, src2);
+ CALC_MULT_FIX_4(src0, scale, shift, val0_m);
+ CALC_MULT_FIX_4(src1, scale, shift, val1_m);
+ CALC_MULT_FIX_4(src2, scale, shift, val2_m);
+ SW3(val0_m, val1_m, val2_m, dst, 4);
+ ST_SW3(zero, zero, zero, irow, 4);
+ length -= 12;
+ irow += 12;
+ dst += 12;
+ } else if (length >= 8) {
+ uint32_t val0_m, val1_m;
+ v4u32 src0, src1;
+ LD_UW2(irow, 4, src0, src1);
+ CALC_MULT_FIX_4(src0, scale, shift, val0_m);
+ CALC_MULT_FIX_4(src1, scale, shift, val1_m);
+ SW2(val0_m, val1_m, dst, 4);
+ ST_SW2(zero, zero, irow, 4);
+ length -= 8;
+ irow += 8;
+ dst += 8;
+ } else if (length >= 4) {
+ uint32_t val0_m;
+ const v4u32 src0 = LD_UW(irow + 0);
+ CALC_MULT_FIX_4(src0, scale, shift, val0_m);
+ SW(val0_m, dst);
+ ST_SW(zero, irow);
+ length -= 4;
+ irow += 4;
+ dst += 4;
+ }
+ for (x_out = 0; x_out < length; ++x_out) {
+ const int v = (int)MULT_FIX(irow[x_out], wrk->fxy_scale);
+ assert(v >= 0 && v <= 255);
+ dst[x_out] = v;
+ irow[x_out] = 0;
+ }
+ }
+}
+
+static void RescalerExportRowShrink(WebPRescaler* const wrk) {
+ uint8_t* dst = wrk->dst;
+ rescaler_t* irow = wrk->irow;
+ const int x_out_max = wrk->dst_width * wrk->num_channels;
+ const rescaler_t* frow = wrk->frow;
+ const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum);
+ assert(!WebPRescalerOutputDone(wrk));
+ assert(wrk->y_accum <= 0);
+ assert(!wrk->y_expand);
+ if (yscale) {
+ ExportRowShrink_0(frow, irow, dst, x_out_max, yscale, wrk);
+ } else {
+ ExportRowShrink_1(irow, dst, x_out_max, wrk);
+ }
+}
+
+//------------------------------------------------------------------------------
+// Entry point
+
+extern void WebPRescalerDspInitMSA(void);
+
+WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitMSA(void) {
+ WebPRescalerExportRowExpand = RescalerExportRowExpand;
+ WebPRescalerExportRowShrink = RescalerExportRowShrink;
+}
+
+#else // !WEBP_USE_MSA
+
+WEBP_DSP_INIT_STUB(WebPRescalerDspInitMSA)
+
+#endif // WEBP_USE_MSA
diff --git a/src/3rdparty/libwebp/src/dsp/rescaler_neon.c b/src/3rdparty/libwebp/src/dsp/rescaler_neon.c
index 16fd450..b2dd8f3 100644
--- a/src/3rdparty/libwebp/src/dsp/rescaler_neon.c
+++ b/src/3rdparty/libwebp/src/dsp/rescaler_neon.c
@@ -18,7 +18,7 @@
#include <arm_neon.h>
#include <assert.h>
#include "./neon.h"
-#include "../utils/rescaler.h"
+#include "../utils/rescaler_utils.h"
#define ROUNDER (WEBP_RESCALER_ONE >> 1)
#define MULT_FIX_C(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX)
diff --git a/src/3rdparty/libwebp/src/dsp/rescaler_sse2.c b/src/3rdparty/libwebp/src/dsp/rescaler_sse2.c
index 5b97028..8271c22 100644
--- a/src/3rdparty/libwebp/src/dsp/rescaler_sse2.c
+++ b/src/3rdparty/libwebp/src/dsp/rescaler_sse2.c
@@ -17,7 +17,7 @@
#include <emmintrin.h>
#include <assert.h>
-#include "../utils/rescaler.h"
+#include "../utils/rescaler_utils.h"
#include "../utils/utils.h"
//------------------------------------------------------------------------------
diff --git a/src/3rdparty/libwebp/src/dsp/upsampling.c b/src/3rdparty/libwebp/src/dsp/upsampling.c
index 651274f..265e722 100644
--- a/src/3rdparty/libwebp/src/dsp/upsampling.c
+++ b/src/3rdparty/libwebp/src/dsp/upsampling.c
@@ -215,6 +215,7 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitYUV444Converters(void) {
extern void WebPInitUpsamplersSSE2(void);
extern void WebPInitUpsamplersNEON(void);
extern void WebPInitUpsamplersMIPSdspR2(void);
+extern void WebPInitUpsamplersMSA(void);
static volatile VP8CPUInfo upsampling_last_cpuinfo_used2 =
(VP8CPUInfo)&upsampling_last_cpuinfo_used2;
@@ -252,6 +253,11 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitUpsamplers(void) {
WebPInitUpsamplersMIPSdspR2();
}
#endif
+#if defined(WEBP_USE_MSA)
+ if (VP8GetCPUInfo(kMSA)) {
+ WebPInitUpsamplersMSA();
+ }
+#endif
}
#endif // FANCY_UPSAMPLING
upsampling_last_cpuinfo_used2 = VP8GetCPUInfo;
diff --git a/src/3rdparty/libwebp/src/dsp/upsampling_msa.c b/src/3rdparty/libwebp/src/dsp/upsampling_msa.c
new file mode 100644
index 0000000..f24926f
--- /dev/null
+++ b/src/3rdparty/libwebp/src/dsp/upsampling_msa.c
@@ -0,0 +1,678 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the COPYING file in the root of the source
+// tree. An additional intellectual property rights grant can be found
+// in the file PATENTS. All contributing project authors may
+// be found in the AUTHORS file in the root of the source tree.
+// -----------------------------------------------------------------------------
+//
+// MSA version of YUV to RGB upsampling functions.
+//
+// Author: Prashant Patil (prashant.patil@imgtec.com)
+
+#include <string.h>
+#include "./dsp.h"
+
+#if defined(WEBP_USE_MSA)
+
+#include "./msa_macro.h"
+#include "./yuv.h"
+
+#ifdef FANCY_UPSAMPLING
+
+#define ILVR_UW2(in, out0, out1) do { \
+ const v8i16 t0 = (v8i16)__msa_ilvr_b((v16i8)zero, (v16i8)in); \
+ out0 = (v4u32)__msa_ilvr_h((v8i16)zero, t0); \
+ out1 = (v4u32)__msa_ilvl_h((v8i16)zero, t0); \
+} while (0)
+
+#define ILVRL_UW4(in, out0, out1, out2, out3) do { \
+ v16u8 t0, t1; \
+ ILVRL_B2_UB(zero, in, t0, t1); \
+ ILVRL_H2_UW(zero, t0, out0, out1); \
+ ILVRL_H2_UW(zero, t1, out2, out3); \
+} while (0)
+
+#define MULTHI_16(in0, in1, in2, in3, cnst, out0, out1) do { \
+ const v4i32 const0 = (v4i32)__msa_fill_w(cnst * 256); \
+ v4u32 temp0, temp1, temp2, temp3; \
+ MUL4(in0, const0, in1, const0, in2, const0, in3, const0, \
+ temp0, temp1, temp2, temp3); \
+ PCKOD_H2_UH(temp1, temp0, temp3, temp2, out0, out1); \
+} while (0)
+
+#define MULTHI_8(in0, in1, cnst, out0) do { \
+ const v4i32 const0 = (v4i32)__msa_fill_w(cnst * 256); \
+ v4u32 temp0, temp1; \
+ MUL2(in0, const0, in1, const0, temp0, temp1); \
+ out0 = (v8u16)__msa_pckod_h((v8i16)temp1, (v8i16)temp0); \
+} while (0)
+
+#define CALC_R16(y0, y1, v0, v1, dst) do { \
+ const v8i16 const_a = (v8i16)__msa_fill_h(14234); \
+ const v8i16 a0 = __msa_adds_s_h((v8i16)y0, (v8i16)v0); \
+ const v8i16 a1 = __msa_adds_s_h((v8i16)y1, (v8i16)v1); \
+ v8i16 b0 = __msa_subs_s_h(a0, const_a); \
+ v8i16 b1 = __msa_subs_s_h(a1, const_a); \
+ SRAI_H2_SH(b0, b1, 6); \
+ CLIP_SH2_0_255(b0, b1); \
+ dst = (v16u8)__msa_pckev_b((v16i8)b1, (v16i8)b0); \
+} while (0)
+
+#define CALC_R8(y0, v0, dst) do { \
+ const v8i16 const_a = (v8i16)__msa_fill_h(14234); \
+ const v8i16 a0 = __msa_adds_s_h((v8i16)y0, (v8i16)v0); \
+ v8i16 b0 = __msa_subs_s_h(a0, const_a); \
+ b0 = SRAI_H(b0, 6); \
+ CLIP_SH_0_255(b0); \
+ dst = (v16u8)__msa_pckev_b((v16i8)b0, (v16i8)b0); \
+} while (0)
+
+#define CALC_G16(y0, y1, u0, u1, v0, v1, dst) do { \
+ const v8i16 const_a = (v8i16)__msa_fill_h(8708); \
+ v8i16 a0 = __msa_subs_s_h((v8i16)y0, (v8i16)u0); \
+ v8i16 a1 = __msa_subs_s_h((v8i16)y1, (v8i16)u1); \
+ const v8i16 b0 = __msa_subs_s_h(a0, (v8i16)v0); \
+ const v8i16 b1 = __msa_subs_s_h(a1, (v8i16)v1); \
+ a0 = __msa_adds_s_h(b0, const_a); \
+ a1 = __msa_adds_s_h(b1, const_a); \
+ SRAI_H2_SH(a0, a1, 6); \
+ CLIP_SH2_0_255(a0, a1); \
+ dst = (v16u8)__msa_pckev_b((v16i8)a1, (v16i8)a0); \
+} while (0)
+
+#define CALC_G8(y0, u0, v0, dst) do { \
+ const v8i16 const_a = (v8i16)__msa_fill_h(8708); \
+ v8i16 a0 = __msa_subs_s_h((v8i16)y0, (v8i16)u0); \
+ const v8i16 b0 = __msa_subs_s_h(a0, (v8i16)v0); \
+ a0 = __msa_adds_s_h(b0, const_a); \
+ a0 = SRAI_H(a0, 6); \
+ CLIP_SH_0_255(a0); \
+ dst = (v16u8)__msa_pckev_b((v16i8)a0, (v16i8)a0); \
+} while (0)
+
+#define CALC_B16(y0, y1, u0, u1, dst) do { \
+ const v8u16 const_a = (v8u16)__msa_fill_h(17685); \
+ const v8u16 a0 = __msa_adds_u_h((v8u16)y0, u0); \
+ const v8u16 a1 = __msa_adds_u_h((v8u16)y1, u1); \
+ v8u16 b0 = __msa_subs_u_h(a0, const_a); \
+ v8u16 b1 = __msa_subs_u_h(a1, const_a); \
+ SRAI_H2_UH(b0, b1, 6); \
+ CLIP_UH2_0_255(b0, b1); \
+ dst = (v16u8)__msa_pckev_b((v16i8)b1, (v16i8)b0); \
+} while (0)
+
+#define CALC_B8(y0, u0, dst) do { \
+ const v8u16 const_a = (v8u16)__msa_fill_h(17685); \
+ const v8u16 a0 = __msa_adds_u_h((v8u16)y0, u0); \
+ v8u16 b0 = __msa_subs_u_h(a0, const_a); \
+ b0 = SRAI_H(b0, 6); \
+ CLIP_UH_0_255(b0); \
+ dst = (v16u8)__msa_pckev_b((v16i8)b0, (v16i8)b0); \
+} while (0)
+
+#define CALC_RGB16(y, u, v, R, G, B) do { \
+ const v16u8 zero = { 0 }; \
+ v8u16 y0, y1, u0, u1, v0, v1; \
+ v4u32 p0, p1, p2, p3; \
+ const v16u8 in_y = LD_UB(y); \
+ const v16u8 in_u = LD_UB(u); \
+ const v16u8 in_v = LD_UB(v); \
+ ILVRL_UW4(in_y, p0, p1, p2, p3); \
+ MULTHI_16(p0, p1, p2, p3, 19077, y0, y1); \
+ ILVRL_UW4(in_v, p0, p1, p2, p3); \
+ MULTHI_16(p0, p1, p2, p3, 26149, v0, v1); \
+ CALC_R16(y0, y1, v0, v1, R); \
+ MULTHI_16(p0, p1, p2, p3, 13320, v0, v1); \
+ ILVRL_UW4(in_u, p0, p1, p2, p3); \
+ MULTHI_16(p0, p1, p2, p3, 6419, u0, u1); \
+ CALC_G16(y0, y1, u0, u1, v0, v1, G); \
+ MULTHI_16(p0, p1, p2, p3, 33050, u0, u1); \
+ CALC_B16(y0, y1, u0, u1, B); \
+} while (0)
+
+#define CALC_RGB8(y, u, v, R, G, B) do { \
+ const v16u8 zero = { 0 }; \
+ v8u16 y0, u0, v0; \
+ v4u32 p0, p1; \
+ const v16u8 in_y = LD_UB(y); \
+ const v16u8 in_u = LD_UB(u); \
+ const v16u8 in_v = LD_UB(v); \
+ ILVR_UW2(in_y, p0, p1); \
+ MULTHI_8(p0, p1, 19077, y0); \
+ ILVR_UW2(in_v, p0, p1); \
+ MULTHI_8(p0, p1, 26149, v0); \
+ CALC_R8(y0, v0, R); \
+ MULTHI_8(p0, p1, 13320, v0); \
+ ILVR_UW2(in_u, p0, p1); \
+ MULTHI_8(p0, p1, 6419, u0); \
+ CALC_G8(y0, u0, v0, G); \
+ MULTHI_8(p0, p1, 33050, u0); \
+ CALC_B8(y0, u0, B); \
+} while (0)
+
+#define STORE16_3(a0, a1, a2, dst) do { \
+ const v16u8 mask0 = { 0, 1, 16, 2, 3, 17, 4, 5, 18, 6, 7, 19, \
+ 8, 9, 20, 10 }; \
+ const v16u8 mask1 = { 0, 21, 1, 2, 22, 3, 4, 23, 5, 6, 24, 7, \
+ 8, 25, 9, 10 }; \
+ const v16u8 mask2 = { 26, 0, 1, 27, 2, 3, 28, 4, 5, 29, 6, 7, \
+ 30, 8, 9, 31 }; \
+ v16u8 out0, out1, out2, tmp0, tmp1, tmp2; \
+ ILVRL_B2_UB(a1, a0, tmp0, tmp1); \
+ out0 = VSHF_UB(tmp0, a2, mask0); \
+ tmp2 = SLDI_UB(tmp1, tmp0, 11); \
+ out1 = VSHF_UB(tmp2, a2, mask1); \
+ tmp2 = SLDI_UB(tmp1, tmp1, 6); \
+ out2 = VSHF_UB(tmp2, a2, mask2); \
+ ST_UB(out0, dst + 0); \
+ ST_UB(out1, dst + 16); \
+ ST_UB(out2, dst + 32); \
+} while (0)
+
+#define STORE8_3(a0, a1, a2, dst) do { \
+ int64_t out_m; \
+ const v16u8 mask0 = { 0, 1, 16, 2, 3, 17, 4, 5, 18, 6, 7, 19, \
+ 8, 9, 20, 10 }; \
+ const v16u8 mask1 = { 11, 21, 12, 13, 22, 14, 15, 23, \
+ 255, 255, 255, 255, 255, 255, 255, 255 }; \
+ const v16u8 tmp0 = (v16u8)__msa_ilvr_b((v16i8)a1, (v16i8)a0); \
+ v16u8 out0, out1; \
+ VSHF_B2_UB(tmp0, a2, tmp0, a2, mask0, mask1, out0, out1); \
+ ST_UB(out0, dst); \
+ out_m = __msa_copy_s_d((v2i64)out1, 0); \
+ SD(out_m, dst + 16); \
+} while (0)
+
+#define STORE16_4(a0, a1, a2, a3, dst) do { \
+ v16u8 tmp0, tmp1, tmp2, tmp3; \
+ v16u8 out0, out1, out2, out3; \
+ ILVRL_B2_UB(a1, a0, tmp0, tmp1); \
+ ILVRL_B2_UB(a3, a2, tmp2, tmp3); \
+ ILVRL_H2_UB(tmp2, tmp0, out0, out1); \
+ ILVRL_H2_UB(tmp3, tmp1, out2, out3); \
+ ST_UB(out0, dst + 0); \
+ ST_UB(out1, dst + 16); \
+ ST_UB(out2, dst + 32); \
+ ST_UB(out3, dst + 48); \
+} while (0)
+
+#define STORE8_4(a0, a1, a2, a3, dst) do { \
+ v16u8 tmp0, tmp1, tmp2, tmp3; \
+ ILVR_B2_UB(a1, a0, a3, a2, tmp0, tmp1); \
+ ILVRL_H2_UB(tmp1, tmp0, tmp2, tmp3); \
+ ST_UB(tmp2, dst + 0); \
+ ST_UB(tmp3, dst + 16); \
+} while (0)
+
+#define STORE2_16(a0, a1, dst) do { \
+ v16u8 out0, out1; \
+ ILVRL_B2_UB(a1, a0, out0, out1); \
+ ST_UB(out0, dst + 0); \
+ ST_UB(out1, dst + 16); \
+} while (0)
+
+#define STORE2_8(a0, a1, dst) do { \
+ const v16u8 out0 = (v16u8)__msa_ilvr_b((v16i8)a1, (v16i8)a0); \
+ ST_UB(out0, dst); \
+} while (0)
+
+#define CALC_RGBA4444(y, u, v, out0, out1, N, dst) do { \
+ CALC_RGB##N(y, u, v, R, G, B); \
+ tmp0 = ANDI_B(R, 0xf0); \
+ tmp1 = SRAI_B(G, 4); \
+ RG = tmp0 | tmp1; \
+ tmp0 = ANDI_B(B, 0xf0); \
+ BA = ORI_B(tmp0, 0x0f); \
+ STORE2_##N(out0, out1, dst); \
+} while (0)
+
+#define CALC_RGB565(y, u, v, out0, out1, N, dst) do { \
+ CALC_RGB##N(y, u, v, R, G, B); \
+ tmp0 = ANDI_B(R, 0xf8); \
+ tmp1 = SRAI_B(G, 5); \
+ RG = tmp0 | tmp1; \
+ tmp0 = SLLI_B(G, 3); \
+ tmp1 = ANDI_B(tmp0, 0xe0); \
+ tmp0 = SRAI_B(B, 3); \
+ GB = tmp0 | tmp1; \
+ STORE2_##N(out0, out1, dst); \
+} while (0)
+
+static WEBP_INLINE int Clip8(int v) {
+ return v < 0 ? 0 : v > 255 ? 255 : v;
+}
+
+static void YuvToRgb(int y, int u, int v, uint8_t* const rgb) {
+ const int y1 = MultHi(y, 19077);
+ const int r1 = y1 + MultHi(v, 26149) - 14234;
+ const int g1 = y1 - MultHi(u, 6419) - MultHi(v, 13320) + 8708;
+ const int b1 = y1 + MultHi(u, 33050) - 17685;
+ rgb[0] = Clip8(r1 >> 6);
+ rgb[1] = Clip8(g1 >> 6);
+ rgb[2] = Clip8(b1 >> 6);
+}
+
+static void YuvToBgr(int y, int u, int v, uint8_t* const bgr) {
+ const int y1 = MultHi(y, 19077);
+ const int r1 = y1 + MultHi(v, 26149) - 14234;
+ const int g1 = y1 - MultHi(u, 6419) - MultHi(v, 13320) + 8708;
+ const int b1 = y1 + MultHi(u, 33050) - 17685;
+ bgr[0] = Clip8(b1 >> 6);
+ bgr[1] = Clip8(g1 >> 6);
+ bgr[2] = Clip8(r1 >> 6);
+}
+
+static void YuvToRgb565(int y, int u, int v, uint8_t* const rgb) {
+ const int y1 = MultHi(y, 19077);
+ const int r1 = y1 + MultHi(v, 26149) - 14234;
+ const int g1 = y1 - MultHi(u, 6419) - MultHi(v, 13320) + 8708;
+ const int b1 = y1 + MultHi(u, 33050) - 17685;
+ const int r = Clip8(r1 >> 6);
+ const int g = Clip8(g1 >> 6);
+ const int b = Clip8(b1 >> 6);
+ const int rg = (r & 0xf8) | (g >> 5);
+ const int gb = ((g << 3) & 0xe0) | (b >> 3);
+#ifdef WEBP_SWAP_16BIT_CSP
+ rgb[0] = gb;
+ rgb[1] = rg;
+#else
+ rgb[0] = rg;
+ rgb[1] = gb;
+#endif
+}
+
+static void YuvToRgba4444(int y, int u, int v, uint8_t* const argb) {
+ const int y1 = MultHi(y, 19077);
+ const int r1 = y1 + MultHi(v, 26149) - 14234;
+ const int g1 = y1 - MultHi(u, 6419) - MultHi(v, 13320) + 8708;
+ const int b1 = y1 + MultHi(u, 33050) - 17685;
+ const int r = Clip8(r1 >> 6);
+ const int g = Clip8(g1 >> 6);
+ const int b = Clip8(b1 >> 6);
+ const int rg = (r & 0xf0) | (g >> 4);
+ const int ba = (b & 0xf0) | 0x0f; // overwrite the lower 4 bits
+#ifdef WEBP_SWAP_16BIT_CSP
+ argb[0] = ba;
+ argb[1] = rg;
+#else
+ argb[0] = rg;
+ argb[1] = ba;
+#endif
+}
+
+static void YuvToArgb(uint8_t y, uint8_t u, uint8_t v, uint8_t* const argb) {
+ argb[0] = 0xff;
+ YuvToRgb(y, u, v, argb + 1);
+}
+
+static void YuvToBgra(uint8_t y, uint8_t u, uint8_t v, uint8_t* const bgra) {
+ YuvToBgr(y, u, v, bgra);
+ bgra[3] = 0xff;
+}
+
+static void YuvToRgba(uint8_t y, uint8_t u, uint8_t v, uint8_t* const rgba) {
+ YuvToRgb(y, u, v, rgba);
+ rgba[3] = 0xff;
+}
+
+static void YuvToRgbLine(const uint8_t* y, const uint8_t* u,
+ const uint8_t* v, uint8_t* dst, int length) {
+ v16u8 R, G, B;
+ while (length >= 16) {
+ CALC_RGB16(y, u, v, R, G, B);
+ STORE16_3(R, G, B, dst);
+ y += 16;
+ u += 16;
+ v += 16;
+ dst += 16 * 3;
+ length -= 16;
+ }
+ if (length > 8) {
+ uint8_t temp[3 * 16] = { 0 };
+ memcpy(temp, y, length * sizeof(*temp));
+ CALC_RGB16(temp, u, v, R, G, B);
+ STORE16_3(R, G, B, temp);
+ memcpy(dst, temp, length * 3 * sizeof(*dst));
+ } else if (length > 0) {
+ uint8_t temp[3 * 8] = { 0 };
+ memcpy(temp, y, length * sizeof(*temp));
+ CALC_RGB8(temp, u, v, R, G, B);
+ STORE8_3(R, G, B, temp);
+ memcpy(dst, temp, length * 3 * sizeof(*dst));
+ }
+}
+
+static void YuvToBgrLine(const uint8_t* y, const uint8_t* u,
+ const uint8_t* v, uint8_t* dst, int length) {
+ v16u8 R, G, B;
+ while (length >= 16) {
+ CALC_RGB16(y, u, v, R, G, B);
+ STORE16_3(B, G, R, dst);
+ y += 16;
+ u += 16;
+ v += 16;
+ dst += 16 * 3;
+ length -= 16;
+ }
+ if (length > 8) {
+ uint8_t temp[3 * 16] = { 0 };
+ memcpy(temp, y, length * sizeof(*temp));
+ CALC_RGB16(temp, u, v, R, G, B);
+ STORE16_3(B, G, R, temp);
+ memcpy(dst, temp, length * 3 * sizeof(*dst));
+ } else if (length > 0) {
+ uint8_t temp[3 * 8] = { 0 };
+ memcpy(temp, y, length * sizeof(*temp));
+ CALC_RGB8(temp, u, v, R, G, B);
+ STORE8_3(B, G, R, temp);
+ memcpy(dst, temp, length * 3 * sizeof(*dst));
+ }
+}
+
+static void YuvToRgbaLine(const uint8_t* y, const uint8_t* u,
+ const uint8_t* v, uint8_t* dst, int length) {
+ v16u8 R, G, B;
+ const v16u8 A = (v16u8)__msa_ldi_b(0xff);
+ while (length >= 16) {
+ CALC_RGB16(y, u, v, R, G, B);
+ STORE16_4(R, G, B, A, dst);
+ y += 16;
+ u += 16;
+ v += 16;
+ dst += 16 * 4;
+ length -= 16;
+ }
+ if (length > 8) {
+ uint8_t temp[4 * 16] = { 0 };
+ memcpy(temp, y, length * sizeof(*temp));
+ CALC_RGB16(&temp[0], u, v, R, G, B);
+ STORE16_4(R, G, B, A, temp);
+ memcpy(dst, temp, length * 4 * sizeof(*dst));
+ } else if (length > 0) {
+ uint8_t temp[4 * 8] = { 0 };
+ memcpy(temp, y, length * sizeof(*temp));
+ CALC_RGB8(temp, u, v, R, G, B);
+ STORE8_4(R, G, B, A, temp);
+ memcpy(dst, temp, length * 4 * sizeof(*dst));
+ }
+}
+
+static void YuvToBgraLine(const uint8_t* y, const uint8_t* u,
+ const uint8_t* v, uint8_t* dst, int length) {
+ v16u8 R, G, B;
+ const v16u8 A = (v16u8)__msa_ldi_b(0xff);
+ while (length >= 16) {
+ CALC_RGB16(y, u, v, R, G, B);
+ STORE16_4(B, G, R, A, dst);
+ y += 16;
+ u += 16;
+ v += 16;
+ dst += 16 * 4;
+ length -= 16;
+ }
+ if (length > 8) {
+ uint8_t temp[4 * 16] = { 0 };
+ memcpy(temp, y, length * sizeof(*temp));
+ CALC_RGB16(temp, u, v, R, G, B);
+ STORE16_4(B, G, R, A, temp);
+ memcpy(dst, temp, length * 4 * sizeof(*dst));
+ } else if (length > 0) {
+ uint8_t temp[4 * 8] = { 0 };
+ memcpy(temp, y, length * sizeof(*temp));
+ CALC_RGB8(temp, u, v, R, G, B);
+ STORE8_4(B, G, R, A, temp);
+ memcpy(dst, temp, length * 4 * sizeof(*dst));
+ }
+}
+
+static void YuvToArgbLine(const uint8_t* y, const uint8_t* u,
+ const uint8_t* v, uint8_t* dst, int length) {
+ v16u8 R, G, B;
+ const v16u8 A = (v16u8)__msa_ldi_b(0xff);
+ while (length >= 16) {
+ CALC_RGB16(y, u, v, R, G, B);
+ STORE16_4(A, R, G, B, dst);
+ y += 16;
+ u += 16;
+ v += 16;
+ dst += 16 * 4;
+ length -= 16;
+ }
+ if (length > 8) {
+ uint8_t temp[4 * 16] = { 0 };
+ memcpy(temp, y, length * sizeof(*temp));
+ CALC_RGB16(temp, u, v, R, G, B);
+ STORE16_4(A, R, G, B, temp);
+ memcpy(dst, temp, length * 4 * sizeof(*dst));
+ } else if (length > 0) {
+ uint8_t temp[4 * 8] = { 0 };
+ memcpy(temp, y, length * sizeof(*temp));
+ CALC_RGB8(temp, u, v, R, G, B);
+ STORE8_4(A, R, G, B, temp);
+ memcpy(dst, temp, length * 4 * sizeof(*dst));
+ }
+}
+
+static void YuvToRgba4444Line(const uint8_t* y, const uint8_t* u,
+ const uint8_t* v, uint8_t* dst, int length) {
+ v16u8 R, G, B, RG, BA, tmp0, tmp1;
+ while (length >= 16) {
+ #ifdef WEBP_SWAP_16BIT_CSP
+ CALC_RGBA4444(y, u, v, BA, RG, 16, dst);
+ #else
+ CALC_RGBA4444(y, u, v, RG, BA, 16, dst);
+ #endif
+ y += 16;
+ u += 16;
+ v += 16;
+ dst += 16 * 2;
+ length -= 16;
+ }
+ if (length > 8) {
+ uint8_t temp[2 * 16] = { 0 };
+ memcpy(temp, y, length * sizeof(*temp));
+#ifdef WEBP_SWAP_16BIT_CSP
+ CALC_RGBA4444(temp, u, v, BA, RG, 16, temp);
+#else
+ CALC_RGBA4444(temp, u, v, RG, BA, 16, temp);
+#endif
+ memcpy(dst, temp, length * 2 * sizeof(*dst));
+ } else if (length > 0) {
+ uint8_t temp[2 * 8] = { 0 };
+ memcpy(temp, y, length * sizeof(*temp));
+#ifdef WEBP_SWAP_16BIT_CSP
+ CALC_RGBA4444(temp, u, v, BA, RG, 8, temp);
+#else
+ CALC_RGBA4444(temp, u, v, RG, BA, 8, temp);
+#endif
+ memcpy(dst, temp, length * 2 * sizeof(*dst));
+ }
+}
+
+static void YuvToRgb565Line(const uint8_t* y, const uint8_t* u,
+ const uint8_t* v, uint8_t* dst, int length) {
+ v16u8 R, G, B, RG, GB, tmp0, tmp1;
+ while (length >= 16) {
+ #ifdef WEBP_SWAP_16BIT_CSP
+ CALC_RGB565(y, u, v, GB, RG, 16, dst);
+ #else
+ CALC_RGB565(y, u, v, RG, GB, 16, dst);
+ #endif
+ y += 16;
+ u += 16;
+ v += 16;
+ dst += 16 * 2;
+ length -= 16;
+ }
+ if (length > 8) {
+ uint8_t temp[2 * 16] = { 0 };
+ memcpy(temp, y, length * sizeof(*temp));
+#ifdef WEBP_SWAP_16BIT_CSP
+ CALC_RGB565(temp, u, v, GB, RG, 16, temp);
+#else
+ CALC_RGB565(temp, u, v, RG, GB, 16, temp);
+#endif
+ memcpy(dst, temp, length * 2 * sizeof(*dst));
+ } else if (length > 0) {
+ uint8_t temp[2 * 8] = { 0 };
+ memcpy(temp, y, length * sizeof(*temp));
+#ifdef WEBP_SWAP_16BIT_CSP
+ CALC_RGB565(temp, u, v, GB, RG, 8, temp);
+#else
+ CALC_RGB565(temp, u, v, RG, GB, 8, temp);
+#endif
+ memcpy(dst, temp, length * 2 * sizeof(*dst));
+ }
+}
+
+#define UPSAMPLE_32PIXELS(a, b, c, d) do { \
+ v16u8 s = __msa_aver_u_b(a, d); \
+ v16u8 t = __msa_aver_u_b(b, c); \
+ const v16u8 st = s ^ t; \
+ v16u8 ad = a ^ d; \
+ v16u8 bc = b ^ c; \
+ v16u8 t0 = ad | bc; \
+ v16u8 t1 = t0 | st; \
+ v16u8 t2 = ANDI_B(t1, 1); \
+ v16u8 t3 = __msa_aver_u_b(s, t); \
+ const v16u8 k = t3 - t2; \
+ v16u8 diag1, diag2; \
+ AVER_UB2_UB(t, k, s, k, t0, t1); \
+ bc = bc & st; \
+ ad = ad & st; \
+ t = t ^ k; \
+ s = s ^ k; \
+ t2 = bc | t; \
+ t3 = ad | s; \
+ t2 = ANDI_B(t2, 1); \
+ t3 = ANDI_B(t3, 1); \
+ SUB2(t0, t2, t1, t3, diag1, diag2); \
+ AVER_UB2_UB(a, diag1, b, diag2, t0, t1); \
+ ILVRL_B2_UB(t1, t0, a, b); \
+ if (pbot_y != NULL) { \
+ AVER_UB2_UB(c, diag2, d, diag1, t0, t1); \
+ ILVRL_B2_UB(t1, t0, c, d); \
+ } \
+} while (0)
+
+#define UPSAMPLE_FUNC(FUNC_NAME, FUNC, XSTEP) \
+static void FUNC_NAME(const uint8_t* top_y, const uint8_t* bot_y, \
+ const uint8_t* top_u, const uint8_t* top_v, \
+ const uint8_t* cur_u, const uint8_t* cur_v, \
+ uint8_t* top_dst, uint8_t* bot_dst, int len) \
+{ \
+ int size = (len - 1) >> 1; \
+ uint8_t temp_u[64]; \
+ uint8_t temp_v[64]; \
+ const uint32_t tl_uv = ((top_u[0]) | ((top_v[0]) << 16)); \
+ const uint32_t l_uv = ((cur_u[0]) | ((cur_v[0]) << 16)); \
+ const uint32_t uv0 = (3 * tl_uv + l_uv + 0x00020002u) >> 2; \
+ const uint8_t* ptop_y = &top_y[1]; \
+ uint8_t *ptop_dst = top_dst + XSTEP; \
+ const uint8_t* pbot_y = &bot_y[1]; \
+ uint8_t *pbot_dst = bot_dst + XSTEP; \
+ \
+ FUNC(top_y[0], uv0 & 0xff, (uv0 >> 16), top_dst); \
+ if (bot_y != NULL) { \
+ const uint32_t uv1 = (3 * l_uv + tl_uv + 0x00020002u) >> 2; \
+ FUNC(bot_y[0], uv1 & 0xff, (uv1 >> 16), bot_dst); \
+ } \
+ while (size >= 16) { \
+ v16u8 tu0, tu1, tv0, tv1, cu0, cu1, cv0, cv1; \
+ LD_UB2(top_u, 1, tu0, tu1); \
+ LD_UB2(cur_u, 1, cu0, cu1); \
+ LD_UB2(top_v, 1, tv0, tv1); \
+ LD_UB2(cur_v, 1, cv0, cv1); \
+ UPSAMPLE_32PIXELS(tu0, tu1, cu0, cu1); \
+ UPSAMPLE_32PIXELS(tv0, tv1, cv0, cv1); \
+ ST_UB4(tu0, tu1, cu0, cu1, &temp_u[0], 16); \
+ ST_UB4(tv0, tv1, cv0, cv1, &temp_v[0], 16); \
+ FUNC##Line(ptop_y, &temp_u[ 0], &temp_v[0], ptop_dst, 32); \
+ if (bot_y != NULL) { \
+ FUNC##Line(pbot_y, &temp_u[32], &temp_v[32], pbot_dst, 32); \
+ } \
+ ptop_y += 32; \
+ pbot_y += 32; \
+ ptop_dst += XSTEP * 32; \
+ pbot_dst += XSTEP * 32; \
+ top_u += 16; \
+ top_v += 16; \
+ cur_u += 16; \
+ cur_v += 16; \
+ size -= 16; \
+ } \
+ if (size > 0) { \
+ v16u8 tu0, tu1, tv0, tv1, cu0, cu1, cv0, cv1; \
+ memcpy(&temp_u[ 0], top_u, 17 * sizeof(uint8_t)); \
+ memcpy(&temp_u[32], cur_u, 17 * sizeof(uint8_t)); \
+ memcpy(&temp_v[ 0], top_v, 17 * sizeof(uint8_t)); \
+ memcpy(&temp_v[32], cur_v, 17 * sizeof(uint8_t)); \
+ LD_UB2(&temp_u[ 0], 1, tu0, tu1); \
+ LD_UB2(&temp_u[32], 1, cu0, cu1); \
+ LD_UB2(&temp_v[ 0], 1, tv0, tv1); \
+ LD_UB2(&temp_v[32], 1, cv0, cv1); \
+ UPSAMPLE_32PIXELS(tu0, tu1, cu0, cu1); \
+ UPSAMPLE_32PIXELS(tv0, tv1, cv0, cv1); \
+ ST_UB4(tu0, tu1, cu0, cu1, &temp_u[0], 16); \
+ ST_UB4(tv0, tv1, cv0, cv1, &temp_v[0], 16); \
+ FUNC##Line(ptop_y, &temp_u[ 0], &temp_v[0], ptop_dst, size * 2); \
+ if (bot_y != NULL) { \
+ FUNC##Line(pbot_y, &temp_u[32], &temp_v[32], pbot_dst, size * 2); \
+ } \
+ top_u += size; \
+ top_v += size; \
+ cur_u += size; \
+ cur_v += size; \
+ } \
+ if (!(len & 1)) { \
+ const uint32_t t0 = ((top_u[0]) | ((top_v[0]) << 16)); \
+ const uint32_t c0 = ((cur_u[0]) | ((cur_v[0]) << 16)); \
+ const uint32_t tmp0 = (3 * t0 + c0 + 0x00020002u) >> 2; \
+ FUNC(top_y[len - 1], tmp0 & 0xff, (tmp0 >> 16), \
+ top_dst + (len - 1) * XSTEP); \
+ if (bot_y != NULL) { \
+ const uint32_t tmp1 = (3 * c0 + t0 + 0x00020002u) >> 2; \
+ FUNC(bot_y[len - 1], tmp1 & 0xff, (tmp1 >> 16), \
+ bot_dst + (len - 1) * XSTEP); \
+ } \
+ } \
+}
+
+UPSAMPLE_FUNC(UpsampleRgbLinePair, YuvToRgb, 3)
+UPSAMPLE_FUNC(UpsampleBgrLinePair, YuvToBgr, 3)
+UPSAMPLE_FUNC(UpsampleRgbaLinePair, YuvToRgba, 4)
+UPSAMPLE_FUNC(UpsampleBgraLinePair, YuvToBgra, 4)
+UPSAMPLE_FUNC(UpsampleArgbLinePair, YuvToArgb, 4)
+UPSAMPLE_FUNC(UpsampleRgba4444LinePair, YuvToRgba4444, 2)
+UPSAMPLE_FUNC(UpsampleRgb565LinePair, YuvToRgb565, 2)
+
+//------------------------------------------------------------------------------
+// Entry point
+
+extern WebPUpsampleLinePairFunc WebPUpsamplers[/* MODE_LAST */];
+
+extern void WebPInitUpsamplersMSA(void);
+
+WEBP_TSAN_IGNORE_FUNCTION void WebPInitUpsamplersMSA(void) {
+ WebPUpsamplers[MODE_RGB] = UpsampleRgbLinePair;
+ WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePair;
+ WebPUpsamplers[MODE_BGR] = UpsampleBgrLinePair;
+ WebPUpsamplers[MODE_BGRA] = UpsampleBgraLinePair;
+ WebPUpsamplers[MODE_ARGB] = UpsampleArgbLinePair;
+ WebPUpsamplers[MODE_rgbA] = UpsampleRgbaLinePair;
+ WebPUpsamplers[MODE_bgrA] = UpsampleBgraLinePair;
+ WebPUpsamplers[MODE_Argb] = UpsampleArgbLinePair;
+ WebPUpsamplers[MODE_RGB_565] = UpsampleRgb565LinePair;
+ WebPUpsamplers[MODE_RGBA_4444] = UpsampleRgba4444LinePair;
+ WebPUpsamplers[MODE_rgbA_4444] = UpsampleRgba4444LinePair;
+}
+
+#endif // FANCY_UPSAMPLING
+
+#endif // WEBP_USE_MSA
+
+#if !(defined(FANCY_UPSAMPLING) && defined(WEBP_USE_MSA))
+WEBP_DSP_INIT_STUB(WebPInitUpsamplersMSA)
+#endif
diff --git a/src/3rdparty/libwebp/src/dsp/upsampling_neon.c b/src/3rdparty/libwebp/src/dsp/upsampling_neon.c
index 2b0c99b..d371a83 100644
--- a/src/3rdparty/libwebp/src/dsp/upsampling_neon.c
+++ b/src/3rdparty/libwebp/src/dsp/upsampling_neon.c
@@ -28,47 +28,34 @@
// U/V upsampling
// Loads 9 pixels each from rows r1 and r2 and generates 16 pixels.
-#define UPSAMPLE_16PIXELS(r1, r2, out) { \
- uint8x8_t a = vld1_u8(r1); \
- uint8x8_t b = vld1_u8(r1 + 1); \
- uint8x8_t c = vld1_u8(r2); \
- uint8x8_t d = vld1_u8(r2 + 1); \
- \
- uint16x8_t al = vshll_n_u8(a, 1); \
- uint16x8_t bl = vshll_n_u8(b, 1); \
- uint16x8_t cl = vshll_n_u8(c, 1); \
- uint16x8_t dl = vshll_n_u8(d, 1); \
- \
- uint8x8_t diag1, diag2; \
- uint16x8_t sl; \
- \
+#define UPSAMPLE_16PIXELS(r1, r2, out) do { \
+ const uint8x8_t a = vld1_u8(r1 + 0); \
+ const uint8x8_t b = vld1_u8(r1 + 1); \
+ const uint8x8_t c = vld1_u8(r2 + 0); \
+ const uint8x8_t d = vld1_u8(r2 + 1); \
/* a + b + c + d */ \
- sl = vaddl_u8(a, b); \
- sl = vaddw_u8(sl, c); \
- sl = vaddw_u8(sl, d); \
- \
- al = vaddq_u16(sl, al); /* 3a + b + c + d */ \
- bl = vaddq_u16(sl, bl); /* a + 3b + c + d */ \
- \
- al = vaddq_u16(al, dl); /* 3a + b + c + 3d */ \
- bl = vaddq_u16(bl, cl); /* a + 3b + 3c + d */ \
+ const uint16x8_t ad = vaddl_u8(a, d); \
+ const uint16x8_t bc = vaddl_u8(b, c); \
+ const uint16x8_t abcd = vaddq_u16(ad, bc); \
+ /* 3a + b + c + 3d */ \
+ const uint16x8_t al = vaddq_u16(abcd, vshlq_n_u16(ad, 1)); \
+ /* a + 3b + 3c + d */ \
+ const uint16x8_t bl = vaddq_u16(abcd, vshlq_n_u16(bc, 1)); \
\
- diag2 = vshrn_n_u16(al, 3); \
- diag1 = vshrn_n_u16(bl, 3); \
+ const uint8x8_t diag2 = vshrn_n_u16(al, 3); \
+ const uint8x8_t diag1 = vshrn_n_u16(bl, 3); \
\
- a = vrhadd_u8(a, diag1); \
- b = vrhadd_u8(b, diag2); \
- c = vrhadd_u8(c, diag2); \
- d = vrhadd_u8(d, diag1); \
+ const uint8x8_t A = vrhadd_u8(a, diag1); \
+ const uint8x8_t B = vrhadd_u8(b, diag2); \
+ const uint8x8_t C = vrhadd_u8(c, diag2); \
+ const uint8x8_t D = vrhadd_u8(d, diag1); \
\
- { \
- uint8x8x2_t a_b, c_d; \
- INIT_VECTOR2(a_b, a, b); \
- INIT_VECTOR2(c_d, c, d); \
- vst2_u8(out, a_b); \
- vst2_u8(out + 32, c_d); \
- } \
-}
+ uint8x8x2_t A_B, C_D; \
+ INIT_VECTOR2(A_B, A, B); \
+ INIT_VECTOR2(C_D, C, D); \
+ vst2_u8(out + 0, A_B); \
+ vst2_u8(out + 32, C_D); \
+} while (0)
// Turn the macro into a function for reducing code-size when non-critical
static void Upsample16Pixels(const uint8_t *r1, const uint8_t *r2,
@@ -93,7 +80,6 @@ static void Upsample16Pixels(const uint8_t *r1, const uint8_t *r2,
static const int16_t kCoeffs1[4] = { 19077, 26149, 6419, 13320 };
#define v255 vdup_n_u8(255)
-#define v_0x0f vdup_n_u8(15)
#define STORE_Rgb(out, r, g, b) do { \
uint8x8x3_t r_g_b; \
@@ -132,21 +118,16 @@ static const int16_t kCoeffs1[4] = { 19077, 26149, 6419, 13320 };
#endif
#define STORE_Rgba4444(out, r, g, b) do { \
- const uint8x8_t r1 = vshl_n_u8(vshr_n_u8(r, 4), 4); /* 4bits */ \
- const uint8x8_t g1 = vshr_n_u8(g, 4); \
- const uint8x8_t ba = vorr_u8(b, v_0x0f); \
- const uint8x8_t rg = vorr_u8(r1, g1); \
+ const uint8x8_t rg = vsri_n_u8(r, g, 4); /* shift g, insert r */ \
+ const uint8x8_t ba = vsri_n_u8(b, v255, 4); /* shift a, insert b */ \
const uint8x8x2_t rgba4444 = ZIP_U8(rg, ba); \
vst1q_u8(out, vcombine_u8(rgba4444.val[0], rgba4444.val[1])); \
} while (0)
#define STORE_Rgb565(out, r, g, b) do { \
- const uint8x8_t r1 = vshl_n_u8(vshr_n_u8(r, 3), 3); /* 5bits */ \
- const uint8x8_t g1 = vshr_n_u8(g, 5); /* upper 3bits */\
- const uint8x8_t g2 = vshl_n_u8(vshr_n_u8(g, 2), 5); /* lower 3bits */\
- const uint8x8_t b1 = vshr_n_u8(b, 3); /* 5bits */ \
- const uint8x8_t rg = vorr_u8(r1, g1); \
- const uint8x8_t gb = vorr_u8(g2, b1); \
+ const uint8x8_t rg = vsri_n_u8(r, g, 5); /* shift g and insert r */ \
+ const uint8x8_t g1 = vshl_n_u8(g, 3); /* pre-shift g: 3bits */ \
+ const uint8x8_t gb = vsri_n_u8(g1, b, 3); /* shift b and insert g */ \
const uint8x8x2_t rgb565 = ZIP_U8(rg, gb); \
vst1q_u8(out, vcombine_u8(rgb565.val[0], rgb565.val[1])); \
} while (0)
diff --git a/src/3rdparty/libwebp/src/dsp/yuv.c b/src/3rdparty/libwebp/src/dsp/yuv.c
index f50a253..dd7d9de 100644
--- a/src/3rdparty/libwebp/src/dsp/yuv.c
+++ b/src/3rdparty/libwebp/src/dsp/yuv.c
@@ -13,6 +13,8 @@
#include "./yuv.h"
+#include <stdlib.h>
+
#if defined(WEBP_YUV_USE_TABLE)
static int done = 0;
@@ -244,6 +246,48 @@ void WebPConvertRGBA32ToUV_C(const uint16_t* rgb,
//-----------------------------------------------------------------------------
+#define MAX_Y ((1 << 10) - 1) // 10b precision over 16b-arithmetic
+static uint16_t clip_y(int v) {
+ return (v < 0) ? 0 : (v > MAX_Y) ? MAX_Y : (uint16_t)v;
+}
+
+static uint64_t SharpYUVUpdateY_C(const uint16_t* ref, const uint16_t* src,
+ uint16_t* dst, int len) {
+ uint64_t diff = 0;
+ int i;
+ for (i = 0; i < len; ++i) {
+ const int diff_y = ref[i] - src[i];
+ const int new_y = (int)dst[i] + diff_y;
+ dst[i] = clip_y(new_y);
+ diff += (uint64_t)abs(diff_y);
+ }
+ return diff;
+}
+
+static void SharpYUVUpdateRGB_C(const int16_t* ref, const int16_t* src,
+ int16_t* dst, int len) {
+ int i;
+ for (i = 0; i < len; ++i) {
+ const int diff_uv = ref[i] - src[i];
+ dst[i] += diff_uv;
+ }
+}
+
+static void SharpYUVFilterRow_C(const int16_t* A, const int16_t* B, int len,
+ const uint16_t* best_y, uint16_t* out) {
+ int i;
+ for (i = 0; i < len; ++i, ++A, ++B) {
+ const int v0 = (A[0] * 9 + A[1] * 3 + B[0] * 3 + B[1] + 8) >> 4;
+ const int v1 = (A[1] * 9 + A[0] * 3 + B[1] * 3 + B[0] + 8) >> 4;
+ out[2 * i + 0] = clip_y(best_y[2 * i + 0] + v0);
+ out[2 * i + 1] = clip_y(best_y[2 * i + 1] + v1);
+ }
+}
+
+#undef MAX_Y
+
+//-----------------------------------------------------------------------------
+
void (*WebPConvertRGB24ToY)(const uint8_t* rgb, uint8_t* y, int width);
void (*WebPConvertBGR24ToY)(const uint8_t* bgr, uint8_t* y, int width);
void (*WebPConvertRGBA32ToUV)(const uint16_t* rgb,
@@ -253,10 +297,18 @@ void (*WebPConvertARGBToY)(const uint32_t* argb, uint8_t* y, int width);
void (*WebPConvertARGBToUV)(const uint32_t* argb, uint8_t* u, uint8_t* v,
int src_width, int do_store);
+uint64_t (*WebPSharpYUVUpdateY)(const uint16_t* ref, const uint16_t* src,
+ uint16_t* dst, int len);
+void (*WebPSharpYUVUpdateRGB)(const int16_t* ref, const int16_t* src,
+ int16_t* dst, int len);
+void (*WebPSharpYUVFilterRow)(const int16_t* A, const int16_t* B, int len,
+ const uint16_t* best_y, uint16_t* out);
+
static volatile VP8CPUInfo rgba_to_yuv_last_cpuinfo_used =
(VP8CPUInfo)&rgba_to_yuv_last_cpuinfo_used;
extern void WebPInitConvertARGBToYUVSSE2(void);
+extern void WebPInitSharpYUVSSE2(void);
WEBP_TSAN_IGNORE_FUNCTION void WebPInitConvertARGBToYUV(void) {
if (rgba_to_yuv_last_cpuinfo_used == VP8GetCPUInfo) return;
@@ -269,10 +321,15 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitConvertARGBToYUV(void) {
WebPConvertRGBA32ToUV = WebPConvertRGBA32ToUV_C;
+ WebPSharpYUVUpdateY = SharpYUVUpdateY_C;
+ WebPSharpYUVUpdateRGB = SharpYUVUpdateRGB_C;
+ WebPSharpYUVFilterRow = SharpYUVFilterRow_C;
+
if (VP8GetCPUInfo != NULL) {
#if defined(WEBP_USE_SSE2)
if (VP8GetCPUInfo(kSSE2)) {
WebPInitConvertARGBToYUVSSE2();
+ WebPInitSharpYUVSSE2();
}
#endif // WEBP_USE_SSE2
}
diff --git a/src/3rdparty/libwebp/src/dsp/yuv.h b/src/3rdparty/libwebp/src/dsp/yuv.h
index 01c40fc..1d33b58 100644
--- a/src/3rdparty/libwebp/src/dsp/yuv.h
+++ b/src/3rdparty/libwebp/src/dsp/yuv.h
@@ -36,7 +36,7 @@
#define WEBP_DSP_YUV_H_
#include "./dsp.h"
-#include "../dec/decode_vp8.h"
+#include "../dec/vp8_dec.h"
#if defined(WEBP_EXPERIMENTAL_FEATURES)
// Do NOT activate this feature for real compression. This is only experimental!
diff --git a/src/3rdparty/libwebp/src/dsp/yuv_sse2.c b/src/3rdparty/libwebp/src/dsp/yuv_sse2.c
index e19bddf..e33c2bb 100644
--- a/src/3rdparty/libwebp/src/dsp/yuv_sse2.c
+++ b/src/3rdparty/libwebp/src/dsp/yuv_sse2.c
@@ -15,6 +15,8 @@
#if defined(WEBP_USE_SSE2)
+#include "./common_sse2.h"
+#include <stdlib.h>
#include <emmintrin.h>
//-----------------------------------------------------------------------------
@@ -155,30 +157,13 @@ static WEBP_INLINE void PackAndStore565(const __m128i* const R,
_mm_storeu_si128((__m128i*)dst, rgb565);
}
-// Function used several times in PlanarTo24b.
-// It samples the in buffer as follows: one every two unsigned char is stored
-// at the beginning of the buffer, while the other half is stored at the end.
-static WEBP_INLINE void PlanarTo24bHelper(const __m128i* const in /*in[6]*/,
- __m128i* const out /*out[6]*/) {
- const __m128i v_mask = _mm_set1_epi16(0x00ff);
-
- // Take one every two upper 8b values.
- out[0] = _mm_packus_epi16(_mm_and_si128(in[0], v_mask),
- _mm_and_si128(in[1], v_mask));
- out[1] = _mm_packus_epi16(_mm_and_si128(in[2], v_mask),
- _mm_and_si128(in[3], v_mask));
- out[2] = _mm_packus_epi16(_mm_and_si128(in[4], v_mask),
- _mm_and_si128(in[5], v_mask));
- // Take one every two lower 8b values.
- out[3] = _mm_packus_epi16(_mm_srli_epi16(in[0], 8), _mm_srli_epi16(in[1], 8));
- out[4] = _mm_packus_epi16(_mm_srli_epi16(in[2], 8), _mm_srli_epi16(in[3], 8));
- out[5] = _mm_packus_epi16(_mm_srli_epi16(in[4], 8), _mm_srli_epi16(in[5], 8));
-}
-
// Pack the planar buffers
// rrrr... rrrr... gggg... gggg... bbbb... bbbb....
// triplet by triplet in the output buffer rgb as rgbrgbrgbrgb ...
-static WEBP_INLINE void PlanarTo24b(__m128i* const in /*in[6]*/, uint8_t* rgb) {
+static WEBP_INLINE void PlanarTo24b(__m128i* const in0, __m128i* const in1,
+ __m128i* const in2, __m128i* const in3,
+ __m128i* const in4, __m128i* const in5,
+ uint8_t* const rgb) {
// The input is 6 registers of sixteen 8b but for the sake of explanation,
// let's take 6 registers of four 8b values.
// To pack, we will keep taking one every two 8b integer and move it
@@ -191,22 +176,15 @@ static WEBP_INLINE void PlanarTo24b(__m128i* const in /*in[6]*/, uint8_t* rgb) {
// Repeat the same permutations twice more:
// r0r4g0g4 | b0b4r1r5 | g1g5b1b5 | r2r6g2g6 | b2b6r3r7 | g3g7b3b7
// r0g0b0r1 | g1b1r2g2 | b2r3g3b3 | r4g4b4r5 | g5b5r6g6 | b6r7g7b7
- __m128i tmp[6];
- PlanarTo24bHelper(in, tmp);
- PlanarTo24bHelper(tmp, in);
- PlanarTo24bHelper(in, tmp);
- // We need to do it two more times than the example as we have sixteen bytes.
- PlanarTo24bHelper(tmp, in);
- PlanarTo24bHelper(in, tmp);
-
- _mm_storeu_si128((__m128i*)(rgb + 0), tmp[0]);
- _mm_storeu_si128((__m128i*)(rgb + 16), tmp[1]);
- _mm_storeu_si128((__m128i*)(rgb + 32), tmp[2]);
- _mm_storeu_si128((__m128i*)(rgb + 48), tmp[3]);
- _mm_storeu_si128((__m128i*)(rgb + 64), tmp[4]);
- _mm_storeu_si128((__m128i*)(rgb + 80), tmp[5]);
-}
-#undef MK_UINT32
+ VP8PlanarTo24b(in0, in1, in2, in3, in4, in5);
+
+ _mm_storeu_si128((__m128i*)(rgb + 0), *in0);
+ _mm_storeu_si128((__m128i*)(rgb + 16), *in1);
+ _mm_storeu_si128((__m128i*)(rgb + 32), *in2);
+ _mm_storeu_si128((__m128i*)(rgb + 48), *in3);
+ _mm_storeu_si128((__m128i*)(rgb + 64), *in4);
+ _mm_storeu_si128((__m128i*)(rgb + 80), *in5);
+}
void VP8YuvToRgba32(const uint8_t* y, const uint8_t* u, const uint8_t* v,
uint8_t* dst) {
@@ -265,29 +243,29 @@ void VP8YuvToRgb56532(const uint8_t* y, const uint8_t* u, const uint8_t* v,
void VP8YuvToRgb32(const uint8_t* y, const uint8_t* u, const uint8_t* v,
uint8_t* dst) {
__m128i R0, R1, R2, R3, G0, G1, G2, G3, B0, B1, B2, B3;
- __m128i rgb[6];
+ __m128i rgb0, rgb1, rgb2, rgb3, rgb4, rgb5;
- YUV444ToRGB(y + 0, u + 0, v + 0, &R0, &G0, &B0);
- YUV444ToRGB(y + 8, u + 8, v + 8, &R1, &G1, &B1);
+ YUV444ToRGB(y + 0, u + 0, v + 0, &R0, &G0, &B0);
+ YUV444ToRGB(y + 8, u + 8, v + 8, &R1, &G1, &B1);
YUV444ToRGB(y + 16, u + 16, v + 16, &R2, &G2, &B2);
YUV444ToRGB(y + 24, u + 24, v + 24, &R3, &G3, &B3);
// Cast to 8b and store as RRRRGGGGBBBB.
- rgb[0] = _mm_packus_epi16(R0, R1);
- rgb[1] = _mm_packus_epi16(R2, R3);
- rgb[2] = _mm_packus_epi16(G0, G1);
- rgb[3] = _mm_packus_epi16(G2, G3);
- rgb[4] = _mm_packus_epi16(B0, B1);
- rgb[5] = _mm_packus_epi16(B2, B3);
+ rgb0 = _mm_packus_epi16(R0, R1);
+ rgb1 = _mm_packus_epi16(R2, R3);
+ rgb2 = _mm_packus_epi16(G0, G1);
+ rgb3 = _mm_packus_epi16(G2, G3);
+ rgb4 = _mm_packus_epi16(B0, B1);
+ rgb5 = _mm_packus_epi16(B2, B3);
// Pack as RGBRGBRGBRGB.
- PlanarTo24b(rgb, dst);
+ PlanarTo24b(&rgb0, &rgb1, &rgb2, &rgb3, &rgb4, &rgb5, dst);
}
void VP8YuvToBgr32(const uint8_t* y, const uint8_t* u, const uint8_t* v,
uint8_t* dst) {
__m128i R0, R1, R2, R3, G0, G1, G2, G3, B0, B1, B2, B3;
- __m128i bgr[6];
+ __m128i bgr0, bgr1, bgr2, bgr3, bgr4, bgr5;
YUV444ToRGB(y + 0, u + 0, v + 0, &R0, &G0, &B0);
YUV444ToRGB(y + 8, u + 8, v + 8, &R1, &G1, &B1);
@@ -295,15 +273,15 @@ void VP8YuvToBgr32(const uint8_t* y, const uint8_t* u, const uint8_t* v,
YUV444ToRGB(y + 24, u + 24, v + 24, &R3, &G3, &B3);
// Cast to 8b and store as BBBBGGGGRRRR.
- bgr[0] = _mm_packus_epi16(B0, B1);
- bgr[1] = _mm_packus_epi16(B2, B3);
- bgr[2] = _mm_packus_epi16(G0, G1);
- bgr[3] = _mm_packus_epi16(G2, G3);
- bgr[4] = _mm_packus_epi16(R0, R1);
- bgr[5] = _mm_packus_epi16(R2, R3);
+ bgr0 = _mm_packus_epi16(B0, B1);
+ bgr1 = _mm_packus_epi16(B2, B3);
+ bgr2 = _mm_packus_epi16(G0, G1);
+ bgr3 = _mm_packus_epi16(G2, G3);
+ bgr4 = _mm_packus_epi16(R0, R1);
+ bgr5= _mm_packus_epi16(R2, R3);
// Pack as BGRBGRBGRBGR.
- PlanarTo24b(bgr, dst);
+ PlanarTo24b(&bgr0, &bgr1, &bgr2, &bgr3, &bgr4, &bgr5, dst);
}
//-----------------------------------------------------------------------------
@@ -377,7 +355,7 @@ static void YuvToRgbRow(const uint8_t* y, const uint8_t* u, const uint8_t* v,
int n;
for (n = 0; n + 32 <= len; n += 32, dst += 32 * 3) {
__m128i R0, R1, R2, R3, G0, G1, G2, G3, B0, B1, B2, B3;
- __m128i rgb[6];
+ __m128i rgb0, rgb1, rgb2, rgb3, rgb4, rgb5;
YUV420ToRGB(y + 0, u + 0, v + 0, &R0, &G0, &B0);
YUV420ToRGB(y + 8, u + 4, v + 4, &R1, &G1, &B1);
@@ -385,15 +363,15 @@ static void YuvToRgbRow(const uint8_t* y, const uint8_t* u, const uint8_t* v,
YUV420ToRGB(y + 24, u + 12, v + 12, &R3, &G3, &B3);
// Cast to 8b and store as RRRRGGGGBBBB.
- rgb[0] = _mm_packus_epi16(R0, R1);
- rgb[1] = _mm_packus_epi16(R2, R3);
- rgb[2] = _mm_packus_epi16(G0, G1);
- rgb[3] = _mm_packus_epi16(G2, G3);
- rgb[4] = _mm_packus_epi16(B0, B1);
- rgb[5] = _mm_packus_epi16(B2, B3);
+ rgb0 = _mm_packus_epi16(R0, R1);
+ rgb1 = _mm_packus_epi16(R2, R3);
+ rgb2 = _mm_packus_epi16(G0, G1);
+ rgb3 = _mm_packus_epi16(G2, G3);
+ rgb4 = _mm_packus_epi16(B0, B1);
+ rgb5 = _mm_packus_epi16(B2, B3);
// Pack as RGBRGBRGBRGB.
- PlanarTo24b(rgb, dst);
+ PlanarTo24b(&rgb0, &rgb1, &rgb2, &rgb3, &rgb4, &rgb5, dst);
y += 32;
u += 16;
@@ -413,7 +391,7 @@ static void YuvToBgrRow(const uint8_t* y, const uint8_t* u, const uint8_t* v,
int n;
for (n = 0; n + 32 <= len; n += 32, dst += 32 * 3) {
__m128i R0, R1, R2, R3, G0, G1, G2, G3, B0, B1, B2, B3;
- __m128i bgr[6];
+ __m128i bgr0, bgr1, bgr2, bgr3, bgr4, bgr5;
YUV420ToRGB(y + 0, u + 0, v + 0, &R0, &G0, &B0);
YUV420ToRGB(y + 8, u + 4, v + 4, &R1, &G1, &B1);
@@ -421,15 +399,15 @@ static void YuvToBgrRow(const uint8_t* y, const uint8_t* u, const uint8_t* v,
YUV420ToRGB(y + 24, u + 12, v + 12, &R3, &G3, &B3);
// Cast to 8b and store as BBBBGGGGRRRR.
- bgr[0] = _mm_packus_epi16(B0, B1);
- bgr[1] = _mm_packus_epi16(B2, B3);
- bgr[2] = _mm_packus_epi16(G0, G1);
- bgr[3] = _mm_packus_epi16(G2, G3);
- bgr[4] = _mm_packus_epi16(R0, R1);
- bgr[5] = _mm_packus_epi16(R2, R3);
+ bgr0 = _mm_packus_epi16(B0, B1);
+ bgr1 = _mm_packus_epi16(B2, B3);
+ bgr2 = _mm_packus_epi16(G0, G1);
+ bgr3 = _mm_packus_epi16(G2, G3);
+ bgr4 = _mm_packus_epi16(R0, R1);
+ bgr5 = _mm_packus_epi16(R2, R3);
// Pack as BGRBGRBGRBGR.
- PlanarTo24b(bgr, dst);
+ PlanarTo24b(&bgr0, &bgr1, &bgr2, &bgr3, &bgr4, &bgr5, dst);
y += 32;
u += 16;
@@ -499,25 +477,19 @@ static WEBP_INLINE void RGB24PackedToPlanar(const uint8_t* const rgb,
// Convert 8 packed ARGB to r[], g[], b[]
static WEBP_INLINE void RGB32PackedToPlanar(const uint32_t* const argb,
- __m128i* const r,
- __m128i* const g,
- __m128i* const b) {
+ __m128i* const rgb /*in[6]*/) {
const __m128i zero = _mm_setzero_si128();
- const __m128i in0 = LOAD_16(argb + 0); // argb3 | argb2 | argb1 | argb0
- const __m128i in1 = LOAD_16(argb + 4); // argb7 | argb6 | argb5 | argb4
- // column-wise transpose
- const __m128i A0 = _mm_unpacklo_epi8(in0, in1);
- const __m128i A1 = _mm_unpackhi_epi8(in0, in1);
- const __m128i B0 = _mm_unpacklo_epi8(A0, A1);
- const __m128i B1 = _mm_unpackhi_epi8(A0, A1);
- // C0 = g7 g6 ... g1 g0 | b7 b6 ... b1 b0
- // C1 = a7 a6 ... a1 a0 | r7 r6 ... r1 r0
- const __m128i C0 = _mm_unpacklo_epi8(B0, B1);
- const __m128i C1 = _mm_unpackhi_epi8(B0, B1);
- // store 16b
- *r = _mm_unpacklo_epi8(C1, zero);
- *g = _mm_unpackhi_epi8(C0, zero);
- *b = _mm_unpacklo_epi8(C0, zero);
+ __m128i a0 = LOAD_16(argb + 0);
+ __m128i a1 = LOAD_16(argb + 4);
+ __m128i a2 = LOAD_16(argb + 8);
+ __m128i a3 = LOAD_16(argb + 12);
+ VP8L32bToPlanar(&a0, &a1, &a2, &a3);
+ rgb[0] = _mm_unpacklo_epi8(a1, zero);
+ rgb[1] = _mm_unpackhi_epi8(a1, zero);
+ rgb[2] = _mm_unpacklo_epi8(a2, zero);
+ rgb[3] = _mm_unpackhi_epi8(a2, zero);
+ rgb[4] = _mm_unpacklo_epi8(a3, zero);
+ rgb[5] = _mm_unpackhi_epi8(a3, zero);
}
// This macro computes (RG * MULT_RG + GB * MULT_GB + ROUNDER) >> DESCALE_FIX
@@ -649,11 +621,10 @@ static void ConvertARGBToY(const uint32_t* argb, uint8_t* y, int width) {
const int max_width = width & ~15;
int i;
for (i = 0; i < max_width; i += 16) {
- __m128i r, g, b, Y0, Y1;
- RGB32PackedToPlanar(&argb[i + 0], &r, &g, &b);
- ConvertRGBToY(&r, &g, &b, &Y0);
- RGB32PackedToPlanar(&argb[i + 8], &r, &g, &b);
- ConvertRGBToY(&r, &g, &b, &Y1);
+ __m128i Y0, Y1, rgb[6];
+ RGB32PackedToPlanar(&argb[i], rgb);
+ ConvertRGBToY(&rgb[0], &rgb[2], &rgb[4], &Y0);
+ ConvertRGBToY(&rgb[1], &rgb[3], &rgb[5], &Y1);
STORE_16(_mm_packus_epi16(Y0, Y1), y + i);
}
for (; i < width; ++i) { // left-over
@@ -678,20 +649,18 @@ static void ConvertARGBToUV(const uint32_t* argb, uint8_t* u, uint8_t* v,
const int max_width = src_width & ~31;
int i;
for (i = 0; i < max_width; i += 32, u += 16, v += 16) {
- __m128i r0, g0, b0, r1, g1, b1, U0, V0, U1, V1;
- RGB32PackedToPlanar(&argb[i + 0], &r0, &g0, &b0);
- RGB32PackedToPlanar(&argb[i + 8], &r1, &g1, &b1);
- HorizontalAddPack(&r0, &r1, &r0);
- HorizontalAddPack(&g0, &g1, &g0);
- HorizontalAddPack(&b0, &b1, &b0);
- ConvertRGBToUV(&r0, &g0, &b0, &U0, &V0);
-
- RGB32PackedToPlanar(&argb[i + 16], &r0, &g0, &b0);
- RGB32PackedToPlanar(&argb[i + 24], &r1, &g1, &b1);
- HorizontalAddPack(&r0, &r1, &r0);
- HorizontalAddPack(&g0, &g1, &g0);
- HorizontalAddPack(&b0, &b1, &b0);
- ConvertRGBToUV(&r0, &g0, &b0, &U1, &V1);
+ __m128i rgb[6], U0, V0, U1, V1;
+ RGB32PackedToPlanar(&argb[i], rgb);
+ HorizontalAddPack(&rgb[0], &rgb[1], &rgb[0]);
+ HorizontalAddPack(&rgb[2], &rgb[3], &rgb[2]);
+ HorizontalAddPack(&rgb[4], &rgb[5], &rgb[4]);
+ ConvertRGBToUV(&rgb[0], &rgb[2], &rgb[4], &U0, &V0);
+
+ RGB32PackedToPlanar(&argb[i + 16], rgb);
+ HorizontalAddPack(&rgb[0], &rgb[1], &rgb[0]);
+ HorizontalAddPack(&rgb[2], &rgb[3], &rgb[2]);
+ HorizontalAddPack(&rgb[4], &rgb[5], &rgb[4]);
+ ConvertRGBToUV(&rgb[0], &rgb[2], &rgb[4], &U1, &V1);
U0 = _mm_packus_epi16(U0, U1);
V0 = _mm_packus_epi16(V0, V1);
@@ -767,9 +736,128 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitConvertARGBToYUVSSE2(void) {
WebPConvertRGBA32ToUV = ConvertRGBA32ToUV;
}
+//------------------------------------------------------------------------------
+
+#define MAX_Y ((1 << 10) - 1) // 10b precision over 16b-arithmetic
+static uint16_t clip_y(int v) {
+ return (v < 0) ? 0 : (v > MAX_Y) ? MAX_Y : (uint16_t)v;
+}
+
+static uint64_t SharpYUVUpdateY_SSE2(const uint16_t* ref, const uint16_t* src,
+ uint16_t* dst, int len) {
+ uint64_t diff = 0;
+ uint32_t tmp[4];
+ int i;
+ const __m128i zero = _mm_setzero_si128();
+ const __m128i max = _mm_set1_epi16(MAX_Y);
+ const __m128i one = _mm_set1_epi16(1);
+ __m128i sum = zero;
+
+ for (i = 0; i + 8 <= len; i += 8) {
+ const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i));
+ const __m128i B = _mm_loadu_si128((const __m128i*)(src + i));
+ const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i));
+ const __m128i D = _mm_sub_epi16(A, B); // diff_y
+ const __m128i E = _mm_cmpgt_epi16(zero, D); // sign (-1 or 0)
+ const __m128i F = _mm_add_epi16(C, D); // new_y
+ const __m128i G = _mm_or_si128(E, one); // -1 or 1
+ const __m128i H = _mm_max_epi16(_mm_min_epi16(F, max), zero);
+ const __m128i I = _mm_madd_epi16(D, G); // sum(abs(...))
+ _mm_storeu_si128((__m128i*)(dst + i), H);
+ sum = _mm_add_epi32(sum, I);
+ }
+ _mm_storeu_si128((__m128i*)tmp, sum);
+ diff = tmp[3] + tmp[2] + tmp[1] + tmp[0];
+ for (; i < len; ++i) {
+ const int diff_y = ref[i] - src[i];
+ const int new_y = (int)dst[i] + diff_y;
+ dst[i] = clip_y(new_y);
+ diff += (uint64_t)abs(diff_y);
+ }
+ return diff;
+}
+
+static void SharpYUVUpdateRGB_SSE2(const int16_t* ref, const int16_t* src,
+ int16_t* dst, int len) {
+ int i = 0;
+ for (i = 0; i + 8 <= len; i += 8) {
+ const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i));
+ const __m128i B = _mm_loadu_si128((const __m128i*)(src + i));
+ const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i));
+ const __m128i D = _mm_sub_epi16(A, B); // diff_uv
+ const __m128i E = _mm_add_epi16(C, D); // new_uv
+ _mm_storeu_si128((__m128i*)(dst + i), E);
+ }
+ for (; i < len; ++i) {
+ const int diff_uv = ref[i] - src[i];
+ dst[i] += diff_uv;
+ }
+}
+
+static void SharpYUVFilterRow_SSE2(const int16_t* A, const int16_t* B, int len,
+ const uint16_t* best_y, uint16_t* out) {
+ int i;
+ const __m128i kCst8 = _mm_set1_epi16(8);
+ const __m128i max = _mm_set1_epi16(MAX_Y);
+ const __m128i zero = _mm_setzero_si128();
+ for (i = 0; i + 8 <= len; i += 8) {
+ const __m128i a0 = _mm_loadu_si128((const __m128i*)(A + i + 0));
+ const __m128i a1 = _mm_loadu_si128((const __m128i*)(A + i + 1));
+ const __m128i b0 = _mm_loadu_si128((const __m128i*)(B + i + 0));
+ const __m128i b1 = _mm_loadu_si128((const __m128i*)(B + i + 1));
+ const __m128i a0b1 = _mm_add_epi16(a0, b1);
+ const __m128i a1b0 = _mm_add_epi16(a1, b0);
+ const __m128i a0a1b0b1 = _mm_add_epi16(a0b1, a1b0); // A0+A1+B0+B1
+ const __m128i a0a1b0b1_8 = _mm_add_epi16(a0a1b0b1, kCst8);
+ const __m128i a0b1_2 = _mm_add_epi16(a0b1, a0b1); // 2*(A0+B1)
+ const __m128i a1b0_2 = _mm_add_epi16(a1b0, a1b0); // 2*(A1+B0)
+ const __m128i c0 = _mm_srai_epi16(_mm_add_epi16(a0b1_2, a0a1b0b1_8), 3);
+ const __m128i c1 = _mm_srai_epi16(_mm_add_epi16(a1b0_2, a0a1b0b1_8), 3);
+ const __m128i d0 = _mm_add_epi16(c1, a0);
+ const __m128i d1 = _mm_add_epi16(c0, a1);
+ const __m128i e0 = _mm_srai_epi16(d0, 1);
+ const __m128i e1 = _mm_srai_epi16(d1, 1);
+ const __m128i f0 = _mm_unpacklo_epi16(e0, e1);
+ const __m128i f1 = _mm_unpackhi_epi16(e0, e1);
+ const __m128i g0 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 0));
+ const __m128i g1 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 8));
+ const __m128i h0 = _mm_add_epi16(g0, f0);
+ const __m128i h1 = _mm_add_epi16(g1, f1);
+ const __m128i i0 = _mm_max_epi16(_mm_min_epi16(h0, max), zero);
+ const __m128i i1 = _mm_max_epi16(_mm_min_epi16(h1, max), zero);
+ _mm_storeu_si128((__m128i*)(out + 2 * i + 0), i0);
+ _mm_storeu_si128((__m128i*)(out + 2 * i + 8), i1);
+ }
+ for (; i < len; ++i) {
+ // (9 * A0 + 3 * A1 + 3 * B0 + B1 + 8) >> 4 =
+ // = (8 * A0 + 2 * (A1 + B0) + (A0 + A1 + B0 + B1 + 8)) >> 4
+ // We reuse the common sub-expressions.
+ const int a0b1 = A[i + 0] + B[i + 1];
+ const int a1b0 = A[i + 1] + B[i + 0];
+ const int a0a1b0b1 = a0b1 + a1b0 + 8;
+ const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4;
+ const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4;
+ out[2 * i + 0] = clip_y(best_y[2 * i + 0] + v0);
+ out[2 * i + 1] = clip_y(best_y[2 * i + 1] + v1);
+ }
+}
+
+#undef MAX_Y
+
+//------------------------------------------------------------------------------
+
+extern void WebPInitSharpYUVSSE2(void);
+
+WEBP_TSAN_IGNORE_FUNCTION void WebPInitSharpYUVSSE2(void) {
+ WebPSharpYUVUpdateY = SharpYUVUpdateY_SSE2;
+ WebPSharpYUVUpdateRGB = SharpYUVUpdateRGB_SSE2;
+ WebPSharpYUVFilterRow = SharpYUVFilterRow_SSE2;
+}
+
#else // !WEBP_USE_SSE2
WEBP_DSP_INIT_STUB(WebPInitSamplersSSE2)
WEBP_DSP_INIT_STUB(WebPInitConvertARGBToYUVSSE2)
+WEBP_DSP_INIT_STUB(WebPInitSharpYUVSSE2)
#endif // WEBP_USE_SSE2