diff options
author | Simon Pilgrim <llvm-dev@redking.me.uk> | 2017-07-29 15:33:34 +0000 |
---|---|---|
committer | Simon Pilgrim <llvm-dev@redking.me.uk> | 2017-07-29 15:33:34 +0000 |
commit | 87f84e93649852012e6dfa3f8616789b2fd3a27d (patch) | |
tree | bce634c9a5feefccca255d337d8e6f865a0fdf0b /lib/Headers | |
parent | 18ebb1b364a028b96837d4145cdb485303e9bd57 (diff) |
[X86][AVX] Ensure vector non-temporal load/store intrinsics force pointer alignment (PR33830)
Clang specifies a max type alignment of 16 bytes on darwin targets (annoyingly in the driver not via cc1), meaning that the builtin nontemporal stores don't correctly align the loads/stores to 32 or 64 bytes when required, resulting in lowering to temporal unaligned loads/stores.
This patch casts the vectors to explicitly aligned types prior to the load/store to ensure that the require alignment is respected.
Differential Revision: https://reviews.llvm.org/D35996
git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@309488 91177308-0d34-0410-b5e6-96231b3b80d8
Diffstat (limited to 'lib/Headers')
-rw-r--r-- | lib/Headers/avx2intrin.h | 3 | ||||
-rw-r--r-- | lib/Headers/avx512fintrin.h | 12 | ||||
-rw-r--r-- | lib/Headers/avxintrin.h | 9 |
3 files changed, 16 insertions, 8 deletions
diff --git a/lib/Headers/avx2intrin.h b/lib/Headers/avx2intrin.h index 5d83a8db48..576f761b25 100644 --- a/lib/Headers/avx2intrin.h +++ b/lib/Headers/avx2intrin.h @@ -832,7 +832,8 @@ _mm256_xor_si256(__m256i __a, __m256i __b) static __inline__ __m256i __DEFAULT_FN_ATTRS _mm256_stream_load_si256(__m256i const *__V) { - return (__m256i)__builtin_nontemporal_load((const __v4di *)__V); + typedef __v4di __v4di_aligned __attribute__((aligned(32))); + return (__m256i)__builtin_nontemporal_load((const __v4di_aligned *)__V); } static __inline__ __m128 __DEFAULT_FN_ATTRS diff --git a/lib/Headers/avx512fintrin.h b/lib/Headers/avx512fintrin.h index 38737cdce5..4ce6945311 100644 --- a/lib/Headers/avx512fintrin.h +++ b/lib/Headers/avx512fintrin.h @@ -9035,25 +9035,29 @@ _mm512_kxor (__mmask16 __A, __mmask16 __B) static __inline__ void __DEFAULT_FN_ATTRS _mm512_stream_si512 (__m512i * __P, __m512i __A) { - __builtin_nontemporal_store((__v8di)__A, (__v8di*)__P); + typedef __v8di __v8di_aligned __attribute__((aligned(64))); + __builtin_nontemporal_store((__v8di_aligned)__A, (__v8di_aligned*)__P); } static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_stream_load_si512 (void *__P) { - return (__m512i) __builtin_nontemporal_load((const __v8di *)__P); + typedef __v8di __v8di_aligned __attribute__((aligned(64))); + return (__m512i) __builtin_nontemporal_load((const __v8di_aligned *)__P); } static __inline__ void __DEFAULT_FN_ATTRS _mm512_stream_pd (double *__P, __m512d __A) { - __builtin_nontemporal_store((__v8df)__A, (__v8df*)__P); + typedef __v8df __v8df_aligned __attribute__((aligned(64))); + __builtin_nontemporal_store((__v8df_aligned)__A, (__v8df_aligned*)__P); } static __inline__ void __DEFAULT_FN_ATTRS _mm512_stream_ps (float *__P, __m512 __A) { - __builtin_nontemporal_store((__v16sf)__A, (__v16sf*)__P); + typedef __v16sf __v16sf_aligned __attribute__((aligned(64))); + __builtin_nontemporal_store((__v16sf_aligned)__A, (__v16sf_aligned*)__P); } static __inline__ __m512d __DEFAULT_FN_ATTRS diff --git a/lib/Headers/avxintrin.h b/lib/Headers/avxintrin.h index 78bb70740b..dff5897b6b 100644 --- a/lib/Headers/avxintrin.h +++ b/lib/Headers/avxintrin.h @@ -3590,7 +3590,8 @@ _mm_maskstore_ps(float *__p, __m128i __m, __m128 __a) static __inline void __DEFAULT_FN_ATTRS _mm256_stream_si256(__m256i *__a, __m256i __b) { - __builtin_nontemporal_store((__v4di)__b, (__v4di*)__a); + typedef __v4di __v4di_aligned __attribute__((aligned(32))); + __builtin_nontemporal_store((__v4di_aligned)__b, (__v4di_aligned*)__a); } /// \brief Moves double-precision values from a 256-bit vector of [4 x double] @@ -3609,7 +3610,8 @@ _mm256_stream_si256(__m256i *__a, __m256i __b) static __inline void __DEFAULT_FN_ATTRS _mm256_stream_pd(double *__a, __m256d __b) { - __builtin_nontemporal_store((__v4df)__b, (__v4df*)__a); + typedef __v4df __v4df_aligned __attribute__((aligned(32))); + __builtin_nontemporal_store((__v4df_aligned)__b, (__v4df_aligned*)__a); } /// \brief Moves single-precision floating point values from a 256-bit vector @@ -3629,7 +3631,8 @@ _mm256_stream_pd(double *__a, __m256d __b) static __inline void __DEFAULT_FN_ATTRS _mm256_stream_ps(float *__p, __m256 __a) { - __builtin_nontemporal_store((__v8sf)__a, (__v8sf*)__p); + typedef __v8sf __v8sf_aligned __attribute__((aligned(32))); + __builtin_nontemporal_store((__v8sf_aligned)__a, (__v8sf_aligned*)__p); } /* Create vectors */ |