NEON optimized bilinear sampling

Adds NEON version of interpolate_4_pixels used by smooth upscaling, and
bilinear sampling.

The SSE2 version is reordered to match the NEON version so they have
the same order of operations and a faster version that loads directly
into vector registers.

Testing is extended so we have a test of smoothness that can catch more
possible mistakes.

Change-Id: I0de4aecf5cb79468e7c8f19f421aa24b2955547c
Reviewed-by: Erik Verbruggen <erik.verbruggen@theqtcompany.com>

3 files changed, 78 insertions, 43 deletions

diff --git a/src/gui/painting/qdrawhelper.cpp b/src/gui/painting/qdrawhelper.cpp
index 8f5eb4d095..4e40d250d3 100644
--- a/src/gui/painting/qdrawhelper.cpp
+++ b/src/gui/painting/qdrawhelper.cpp
@@ -2342,8 +2342,8 @@ static const uint * QT_FASTCALL fetchTransformedBilinearARGB32PM(uint *buffer, c
                     uint bl = s2[x1];
                     uint br = s2[x2];
 
-#if defined(__SSE2__)
-                    // The SSE2 optimized interpolate_4_pixels is faster than interpolate_4_pixels_16.
+#if defined(__SSE2__) || defined(__ARM_NEON__)
+                    // The optimized interpolate_4_pixels are faster than interpolate_4_pixels_16.
                     int distx = (fx & 0x0000ffff) >> 8;
                     int disty = (fy & 0x0000ffff) >> 8;
                     *b = interpolate_4_pixels(tl, tr, bl, br, distx, disty);
@@ -2572,12 +2572,8 @@ static const uint *QT_FASTCALL fetchTransformedBilinear(uint *buffer, const Oper
                     if ((fdx < 0 && fdx > -(fixed_scale / 8)) || std::abs(data->m22) < (1./8.)) { // scale up more than 8x
                         int disty = (fy & 0x0000ffff) >> 8;
                         for (int i = 0; i < len; ++i) {
-                            uint tl = buf1[i * 2 + 0];
-                            uint tr = buf1[i * 2 + 1];
-                            uint bl = buf2[i * 2 + 0];
-                            uint br = buf2[i * 2 + 1];
                             int distx = (fracX & 0x0000ffff) >> 8;
-                            b[i] = interpolate_4_pixels(tl, tr, bl, br, distx, disty);
+                            b[i] = interpolate_4_pixels(buf1 + i * 2, buf2 + i * 2, distx, disty);
                             fracX += fdx;
                         }
                     } else { //scale down
@@ -2638,15 +2634,10 @@ static const uint *QT_FASTCALL fetchTransformedBilinear(uint *buffer, const Oper
                 if (std::abs(data->m11) > 8 || std::abs(data->m22) > 8) {
                     //if we are zooming more than 8 times, we use 8bit precision for the position.
                     for (int i = 0; i < len; ++i) {
-                        uint tl = buf1[i * 2 + 0];
-                        uint tr = buf1[i * 2 + 1];
-                        uint bl = buf2[i * 2 + 0];
-                        uint br = buf2[i * 2 + 1];
-
                         int distx = (fracX & 0x0000ffff) >> 8;
                         int disty = (fracY & 0x0000ffff) >> 8;
 
-                        b[i] = interpolate_4_pixels(tl, tr, bl, br, distx, disty);
+                        b[i] = interpolate_4_pixels(buf1 + i * 2, buf2 + i * 2, distx, disty);
                         fracX += fdx;
                         fracY += fdy;
                     }
@@ -2736,12 +2727,7 @@ static const uint *QT_FASTCALL fetchTransformedBilinear(uint *buffer, const Oper
                 int distx = distxs[i];
                 int disty = distys[i];
 
-                uint tl = buf1[i * 2 + 0];
-                uint tr = buf1[i * 2 + 1];
-                uint bl = buf2[i * 2 + 0];
-                uint br = buf2[i * 2 + 1];
-
-                b[i] = interpolate_4_pixels(tl, tr, bl, br, distx, disty);
+                b[i] = interpolate_4_pixels(buf1 + i * 2, buf2 + i * 2, distx, disty);
             }
             length -= len;
             b += len;
diff --git a/src/gui/painting/qdrawhelper_p.h b/src/gui/painting/qdrawhelper_p.h
index 1ff19f4e04..fc24e22cac 100644
--- a/src/gui/painting/qdrawhelper_p.h
+++ b/src/gui/painting/qdrawhelper_p.h
@@ -629,31 +629,75 @@ static Q_ALWAYS_INLINE uint BYTE_MUL(uint x, uint a) {
 }
 #endif
 
-#ifdef __SSE2__
+#if defined(__SSE2__)
+static Q_ALWAYS_INLINE uint interpolate_4_pixels_sse2(__m128i vt, __m128i vb, uint distx, uint disty)
+{
+    // First interpolate top and bottom pixels in parallel.
+    vt = _mm_unpacklo_epi8(vt, _mm_setzero_si128());
+    vb = _mm_unpacklo_epi8(vb, _mm_setzero_si128());
+    vt = _mm_mullo_epi16(vt, _mm_set1_epi16(256 - disty));
+    vb = _mm_mullo_epi16(vb, _mm_set1_epi16(disty));
+    __m128i vlr = _mm_add_epi16(vt, vb);
+    vlr = _mm_srli_epi16(vlr, 8);
+    // vlr now contains the result of the first two interpolate calls vlr = unpacked((xright << 64) | xleft)
+
+    // Now the last interpolate between left and right..
+    const __m128i vidistx = _mm_shufflelo_epi16(_mm_cvtsi32_si128(256 - distx), _MM_SHUFFLE(0, 0, 0, 0));
+    const __m128i vdistx = _mm_shufflelo_epi16(_mm_cvtsi32_si128(distx), _MM_SHUFFLE(0, 0, 0, 0));
+    const __m128i vmulx = _mm_unpacklo_epi16(vidistx, vdistx);
+    vlr = _mm_unpacklo_epi16(vlr, _mm_srli_si128(vlr, 8));
+    // vlr now contains the colors of left and right interleaved { la, ra, lr, rr, lg, rg, lb, rb }
+    vlr = _mm_madd_epi16(vlr, vmulx); // Multiply and horizontal add.
+    vlr = _mm_srli_epi32(vlr, 8);
+    vlr = _mm_packs_epi32(vlr, vlr);
+    vlr = _mm_packus_epi16(vlr, vlr);
+    return _mm_cvtsi128_si32(vlr);
+}
+
+static inline uint interpolate_4_pixels(uint tl, uint tr, uint bl, uint br, uint distx, uint disty)
+{
+    __m128i vt = _mm_unpacklo_epi32(_mm_cvtsi32_si128(tl), _mm_cvtsi32_si128(tr));
+    __m128i vb = _mm_unpacklo_epi32(_mm_cvtsi32_si128(bl), _mm_cvtsi32_si128(br));
+    return interpolate_4_pixels_sse2(vt, vb, distx, disty);
+}
+
+static inline uint interpolate_4_pixels(const uint t[], const uint b[], uint distx, uint disty)
+{
+    __m128i vt = _mm_loadl_epi64((const __m128i*)t);
+    __m128i vb = _mm_loadl_epi64((const __m128i*)b);
+    return interpolate_4_pixels_sse2(vt, vb, distx, disty);
+}
+#elif defined(__ARM_NEON__)
+static Q_ALWAYS_INLINE uint interpolate_4_pixels_neon(uint32x2_t vt32, uint32x2_t vb32, uint distx, uint disty)
+{
+    uint16x8_t vt16 = vmovl_u8(vreinterpret_u8_u32(vt32));
+    uint16x8_t vb16 = vmovl_u8(vreinterpret_u8_u32(vb32));
+    vt16 = vmulq_n_u16(vt16, 256 - disty);
+    vt16 = vmlaq_n_u16(vt16, vb16, disty);
+    vt16 = vshrq_n_u16(vt16, 8);
+    uint16x4_t vl16 = vget_low_u16(vt16);
+    uint16x4_t vr16 = vget_high_u16(vt16);
+    vl16 = vmul_n_u16(vl16, 256 - distx);
+    vl16 = vmla_n_u16(vl16, vr16, distx);
+    vl16 = vshr_n_u16(vl16, 8);
+    uint8x8_t vr = vmovn_u16(vcombine_u16(vl16, vl16));
+    return vget_lane_u32(vreinterpret_u32_u8(vr), 0);
+}
+
 static inline uint interpolate_4_pixels(uint tl, uint tr, uint bl, uint br, uint distx, uint disty)
 {
-    // First interpolate right and left pixels in parallel.
-    __m128i vl = _mm_unpacklo_epi32(_mm_cvtsi32_si128(tl), _mm_cvtsi32_si128(bl));
-    __m128i vr = _mm_unpacklo_epi32(_mm_cvtsi32_si128(tr), _mm_cvtsi32_si128(br));
-    vl = _mm_unpacklo_epi8(vl, _mm_setzero_si128());
-    vr = _mm_unpacklo_epi8(vr, _mm_setzero_si128());
-    vl = _mm_mullo_epi16(vl, _mm_set1_epi16(256 - distx));
-    vr = _mm_mullo_epi16(vr, _mm_set1_epi16(distx));
-    __m128i vtb = _mm_add_epi16(vl, vr);
-    vtb = _mm_srli_epi16(vtb, 8);
-    // vtb now contains the result of the first two interpolate calls vtb = unpacked((xbot << 64) | xtop)
-
-    // Now the last interpolate between top and bottom interpolations.
-    const __m128i vidisty = _mm_shufflelo_epi16(_mm_cvtsi32_si128(256 - disty), _MM_SHUFFLE(0, 0, 0, 0));
-    const __m128i vdisty = _mm_shufflelo_epi16(_mm_cvtsi32_si128(disty), _MM_SHUFFLE(0, 0, 0, 0));
-    const __m128i vmuly = _mm_unpacklo_epi16(vidisty, vdisty);
-    vtb = _mm_unpacklo_epi16(vtb, _mm_srli_si128(vtb, 8));
-    // vtb now contains the colors of top and bottom interleaved { ta, ba, tr, br, tg, bg, tb, bb }
-    vtb = _mm_madd_epi16(vtb, vmuly); // Multiply and horizontal add.
-    vtb = _mm_srli_epi32(vtb, 8);
-    vtb = _mm_packs_epi32(vtb, _mm_setzero_si128());
-    vtb = _mm_packus_epi16(vtb, _mm_setzero_si128());
-    return _mm_cvtsi128_si32(vtb);
+    uint32x2_t vt32 = vmov_n_u32(tl);
+    uint32x2_t vb32 = vmov_n_u32(bl);
+    vt32 = vset_lane_u32(tr, vt32, 1);
+    vb32 = vset_lane_u32(br, vb32, 1);
+    return interpolate_4_pixels_neon(vt32, vb32, distx, disty);
+}
+
+static inline uint interpolate_4_pixels(const uint t[], const uint b[], uint distx, uint disty)
+{
+    uint32x2_t vt32 = vld1_u32(t);
+    uint32x2_t vb32 = vld1_u32(b);
+    return interpolate_4_pixels_neon(vt32, vb32, distx, disty);
 }
 #else
 static inline uint interpolate_4_pixels(uint tl, uint tr, uint bl, uint br, uint distx, uint disty)
@@ -664,6 +708,11 @@ static inline uint interpolate_4_pixels(uint tl, uint tr, uint bl, uint br, uint
     uint xbot = INTERPOLATE_PIXEL_256(bl, idistx, br, distx);
     return INTERPOLATE_PIXEL_256(xtop, idisty, xbot, disty);
 }
+
+static inline uint interpolate_4_pixels(const uint t[], const uint b[], uint distx, uint disty)
+{
+    return interpolate_4_pixels(t[0], t[1], b[0], b[1], distx, disty);
+}
 #endif
 
 #if Q_BYTE_ORDER == Q_BIG_ENDIAN
diff --git a/src/gui/painting/qimagescale.cpp b/src/gui/painting/qimagescale.cpp
index d51d0bf4a8..9381afa8e7 100644
--- a/src/gui/painting/qimagescale.cpp
+++ b/src/gui/painting/qimagescale.cpp
@@ -308,7 +308,7 @@ static void qt_qimageScaleAARGBA_up_xy(QImageScaleInfo *isi, unsigned int *dest,
                 const unsigned int *pix = sptr + xpoints[x];
                 const int xap = xapoints[x];
                 if (xap > 0)
-                    *dptr = interpolate_4_pixels(pix[0], pix[1], pix[sow], pix[sow + 1], xap, yap);
+                    *dptr = interpolate_4_pixels(pix, pix + sow, xap, yap);
                 else
                     *dptr = INTERPOLATE_PIXEL_256(pix[0], 256 - yap, pix[sow], yap);
                 dptr++;