diff options
| author | Vitaly Buka <vitalybuka@google.com> | 2023-12-21 15:49:58 -0800 |
|---|---|---|
| committer | Vitaly Buka <vitalybuka@google.com> | 2023-12-21 15:49:58 -0800 |
| commit | 9d0e3a77eee290592620cf017c433bd7a751952d (patch) | |
| tree | ae4b405df5b87d749d54a7e75fa98edb47ea9478 | |
| parent | 25852ecd662256b4c0280d84ef6235e4f1834ea3 (diff) | |
| parent | 011024536963c7822c81f33434969e8eff08e180 (diff) | |
[𝘀𝗽𝗿] changes introduced through rebaseupstream/users/vitalybuka/spr/main.hwasan-respect-strip_path_prefix-printing-locals
Created using spr 1.3.4
[skip ci]
255 files changed, 15332 insertions, 3734 deletions
diff --git a/bolt/lib/Passes/SplitFunctions.cpp b/bolt/lib/Passes/SplitFunctions.cpp index d9f5da3e3bca..5de075973004 100644 --- a/bolt/lib/Passes/SplitFunctions.cpp +++ b/bolt/lib/Passes/SplitFunctions.cpp @@ -175,8 +175,12 @@ struct SplitCacheDirected final : public SplitStrategy { void fragment(const BlockIt Start, const BlockIt End) override { BasicBlockOrder BlockOrder(Start, End); BinaryFunction &BF = *BlockOrder.front()->getFunction(); + // No need to re-split small functions. + if (BlockOrder.size() <= 2) + return; size_t BestSplitIndex = findSplitIndex(BF, BlockOrder); + assert(BestSplitIndex < BlockOrder.size()); // Assign fragments based on the computed best split index. // All basic blocks with index up to the best split index become hot. @@ -200,10 +204,12 @@ private: }; struct SplitScore { - size_t SplitIndex; + size_t SplitIndex = size_t(-1); size_t HotSizeReduction = 0; double LocalScore = 0; double CoverCallScore = 0; + + double sum() const { return LocalScore + CoverCallScore; } }; // Auxiliary variables used by the algorithm. @@ -303,7 +309,7 @@ private: const size_t SplitIndex) { assert(SplitIndex < BlockOrder.size() && "Invalid split index"); - // Update function layout assuming hot-warm splitting at SplitIndex + // Update function layout assuming hot-warm splitting at SplitIndex. for (size_t Index = 0; Index < BlockOrder.size(); Index++) { BinaryBasicBlock *BB = BlockOrder[Index]; if (BB->getFragmentNum() == FragmentNum::cold()) @@ -319,8 +325,8 @@ private: // Populate BB.OutputAddressRange with estimated new start and end addresses // and compute the old end address of the hot section and the new end // address of the hot section. - size_t OldHotEndAddr; - size_t NewHotEndAddr; + size_t OldHotEndAddr{0}; + size_t NewHotEndAddr{0}; size_t CurrentAddr = BBOffsets[BlockOrder[0]]; for (BinaryBasicBlock *BB : BlockOrder) { // We only care about new addresses of blocks in hot/warm. @@ -492,20 +498,15 @@ private: } /// Compute the split score of splitting a function at a given index. - /// The split score consists of local score and cover score. Cover call score - /// is expensive to compute. As a result, we pass in a \p ReferenceScore and - /// compute cover score only when the local score exceeds that in the - /// ReferenceScore or that the size reduction of the hot fragment is larger - /// than that achieved by the split index of the ReferenceScore. This function - /// returns \p Score of SplitScore type. It contains the local score and cover - /// score (if computed) of the current splitting index. For easier book - /// keeping and comparison, it also stores the split index and the resulting - /// reduction in hot fragment size. + /// The split score consists of local score and cover score. This function + /// returns \p Score of SplitScore type. It contains the local score and + /// cover score of the current splitting index. For easier book keeping and + /// comparison, it also stores the split index and the resulting reduction + /// in hot fragment size. SplitScore computeSplitScore(const BinaryFunction &BF, const BasicBlockOrder &BlockOrder, const size_t SplitIndex, - const std::vector<CallInfo> &CoverCalls, - const SplitScore &ReferenceScore) { + const std::vector<CallInfo> &CoverCalls) { // Populate BinaryBasicBlock::OutputAddressRange with estimated // new start and end addresses after hot-warm splitting at SplitIndex. size_t OldHotEnd; @@ -533,47 +534,74 @@ private: // increamented in place. computeJumpScore(BlockOrder, SplitIndex, Score); - // There is no need to compute CoverCallScore if we have already found - // another split index with a bigger LocalScore and bigger HotSizeReduction. - if (Score.LocalScore <= ReferenceScore.LocalScore && - Score.HotSizeReduction <= ReferenceScore.HotSizeReduction) - return Score; - // Compute CoverCallScore and store in Score in place. computeCoverCallScore(BlockOrder, SplitIndex, CoverCalls, Score); return Score; } + /// Find the most likely successor of a basic block when it has one or two + /// successors. Return nullptr otherwise. + const BinaryBasicBlock *getMostLikelySuccessor(const BinaryBasicBlock *BB) { + if (BB->succ_size() == 1) + return BB->getSuccessor(); + if (BB->succ_size() == 2) { + uint64_t TakenCount = BB->getTakenBranchInfo().Count; + assert(TakenCount != BinaryBasicBlock::COUNT_NO_PROFILE); + uint64_t NonTakenCount = BB->getFallthroughBranchInfo().Count; + assert(NonTakenCount != BinaryBasicBlock::COUNT_NO_PROFILE); + if (TakenCount > NonTakenCount) + return BB->getConditionalSuccessor(true); + else if (TakenCount < NonTakenCount) + return BB->getConditionalSuccessor(false); + } + return nullptr; + } + /// Find the best index for splitting. The returned value is the index of the /// last hot basic block. Hence, "no splitting" is equivalent to returning the /// value which is one less than the size of the function. size_t findSplitIndex(const BinaryFunction &BF, const BasicBlockOrder &BlockOrder) { + assert(BlockOrder.size() > 2); // Find all function calls that can be shortened if we move blocks of the // current function to warm/cold const std::vector<CallInfo> CoverCalls = extractCoverCalls(BF); - // Try all possible split indices (blocks with Index <= SplitIndex are in - // hot) and find the one maximizing the splitting score. + // Find the existing hot-cold splitting index. + size_t HotColdIndex = 0; + while (HotColdIndex + 1 < BlockOrder.size()) { + if (BlockOrder[HotColdIndex + 1]->getFragmentNum() == FragmentNum::cold()) + break; + HotColdIndex++; + } + assert(HotColdIndex + 1 == BlockOrder.size() || + (BlockOrder[HotColdIndex]->getFragmentNum() == FragmentNum::main() && + BlockOrder[HotColdIndex + 1]->getFragmentNum() == + FragmentNum::cold())); + + // Try all possible split indices up to HotColdIndex (blocks that have + // Index <= SplitIndex are in hot) and find the one maximizing the + // splitting score. SplitScore BestScore; - double BestScoreSum = -1.0; - SplitScore ReferenceScore; - for (size_t Index = 0; Index < BlockOrder.size(); Index++) { + for (size_t Index = 0; Index <= HotColdIndex; Index++) { const BinaryBasicBlock *LastHotBB = BlockOrder[Index]; - // No need to keep cold blocks in the hot section. - if (LastHotBB->getFragmentNum() == FragmentNum::cold()) - break; + assert(LastHotBB->getFragmentNum() != FragmentNum::cold()); + + // Do not break jump to the most likely successor. + if (Index + 1 < BlockOrder.size() && + BlockOrder[Index + 1] == getMostLikelySuccessor(LastHotBB)) + continue; + const SplitScore Score = - computeSplitScore(BF, BlockOrder, Index, CoverCalls, ReferenceScore); - double ScoreSum = Score.LocalScore + Score.CoverCallScore; - if (ScoreSum > BestScoreSum) { - BestScoreSum = ScoreSum; + computeSplitScore(BF, BlockOrder, Index, CoverCalls); + if (Score.sum() > BestScore.sum()) BestScore = Score; - } - if (Score.LocalScore > ReferenceScore.LocalScore) - ReferenceScore = Score; } + // If we don't find a good splitting point, fallback to the original one. + if (BestScore.SplitIndex == size_t(-1)) + return HotColdIndex; + return BestScore.SplitIndex; } }; diff --git a/bolt/test/X86/cdsplit-call-scale.s b/bolt/test/X86/cdsplit-call-scale.s index 5b4f92832624..5701d9e6dfd6 100644 --- a/bolt/test/X86/cdsplit-call-scale.s +++ b/bolt/test/X86/cdsplit-call-scale.s @@ -2,8 +2,9 @@ # When -call-scale=0.0, the tested function is 2-way splitted. # When -call-scale=1.0, the tested function is 3-way splitted with 5 blocks # in warm because of the increased benefit of shortening the call edges. -# When -call-scale=1000.0, the tested function is 3-way splitted with 7 blocks -# in warm because of the strong benefit of shortening the call edges. +# When -call-scale=1000.0, the tested function is still 3-way splitted with +# 5 blocks in warm because cdsplit does not allow hot-warm splitting to break +# a fall through branch from a basic block to its most likely successor. # RUN: llvm-mc --filetype=obj --triple x86_64-unknown-unknown %s -o %t.o # RUN: link_fdata %s %t.o %t.fdata @@ -39,12 +40,10 @@ # MEDINCENTIVE: {{^\.Ltmp5}} # HIGHINCENTIVE: Binary Function "chain" after split-functions -# HIGHINCENTIVE: {{^\.LBB00}} +# HIGHINCENTIVE: {{^\.Ltmp1}} # HIGHINCENTIVE: ------- HOT-COLD SPLIT POINT ------- # HIGHINCENTIVE: {{^\.LFT1}} # HIGHINCENTIVE: ------- HOT-COLD SPLIT POINT ------- -# HIGHINCENTIVE: {{^\.LFT0}} -# HIGHINCENTIVE: {{^\.Ltmp1}} # HIGHINCENTIVE: {{^\.Ltmp0}} # HIGHINCENTIVE: {{^\.Ltmp2}} # HIGHINCENTIVE: {{^\.Ltmp3}} diff --git a/clang/docs/ControlFlowIntegrityDesign.rst b/clang/docs/ControlFlowIntegrityDesign.rst index f3a3c8294f7c..d66bd16155a9 100644 --- a/clang/docs/ControlFlowIntegrityDesign.rst +++ b/clang/docs/ControlFlowIntegrityDesign.rst @@ -349,7 +349,7 @@ address point. Note that libraries like libcxxabi do assume this property. (2) virtual function entry layout property -For each virtual function the distance between an virtual table entry for this function and the corresponding +For each virtual function the distance between a virtual table entry for this function and the corresponding address point is always the same. This property ensures that dynamic dispatch still works with the interleaving layout. Note that the interleaving scheme in the CFI implementation guarantees both properties above whereas the original scheme proposed diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst index 13fb7c345aa4..23a7f4f5d5b9 100644 --- a/clang/docs/LanguageExtensions.rst +++ b/clang/docs/LanguageExtensions.rst @@ -2019,7 +2019,7 @@ would be +1. ``ns_returns_autoreleased`` specifies that the returned object is autorelease pool. **Usage**: The ``ns_consumed`` and ``cf_consumed`` attributes can be placed on -an parameter declaration; they specify that the argument is expected to have a +a parameter declaration; they specify that the argument is expected to have a +1 retain count, which will be balanced in some way by the function or method. The ``ns_consumes_self`` attribute can only be placed on an Objective-C method; it specifies that the method expects its ``self`` parameter to have a @@ -3601,7 +3601,7 @@ scalar calls of ``__builtin_isfpclass`` applied to the input elementwise. The result of ``__builtin_isfpclass`` is a boolean value, if the first argument is a scalar, or an integer vector with the same element count as the first argument. The element type in this vector has the same bit length as the -element of the the first argument type. +element of the first argument type. This function never raises floating-point exceptions and does not canonicalize its input. The floating-point argument is not promoted, its data class is @@ -4959,7 +4959,7 @@ Clang supports the following match rules: - ``record(unless(is_union))``: Can be used to apply attributes only to ``struct`` and ``class`` declarations. -- ``enum``: Can be be used to apply attributes to enumeration declarations. +- ``enum``: Can be used to apply attributes to enumeration declarations. - ``enum_constant``: Can be used to apply attributes to enumerators. diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 39b9176865fc..ee211c16a48a 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -253,7 +253,7 @@ New Compiler Flags the preprocessed text to the output. This can greatly reduce the size of the preprocessed output, which can be helpful when trying to reduce a test case. * ``-fassume-nothrow-exception-dtor`` is added to assume that the destructor of - an thrown exception object will not throw. The generated code for catch + a thrown exception object will not throw. The generated code for catch handlers will be smaller. A throw expression of a type with a potentially-throwing destructor will lead to an error. diff --git a/clang/docs/SanitizerCoverage.rst b/clang/docs/SanitizerCoverage.rst index c7ced397c722..45ad03cb4377 100644 --- a/clang/docs/SanitizerCoverage.rst +++ b/clang/docs/SanitizerCoverage.rst @@ -496,7 +496,7 @@ offsets in the corresponding binary/DSO that were executed during the run. Sancov Tool ----------- -An simple ``sancov`` tool is provided to process coverage files. +A simple ``sancov`` tool is provided to process coverage files. The tool is part of LLVM project and is currently supported only on Linux. It can handle symbolization tasks autonomously without any extra support from the environment. You need to pass .sancov files (named diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h index 2a9f8dce74c0..e8c27d6c1203 100644 --- a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h +++ b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h @@ -727,20 +727,9 @@ RecordStorageLocation *getBaseObjectLocation(const MemberExpr &ME, std::vector<FieldDecl *> getFieldsForInitListExpr(const RecordDecl *RD); /// Associates a new `RecordValue` with `Loc` and returns the new value. -/// It is not defined whether the field values remain the same or not. -/// -/// This function is primarily intended for use by checks that set custom -/// properties on `RecordValue`s to model the state of these values. Such checks -/// should avoid modifying the properties of an existing `RecordValue` because -/// these changes would be visible to other `Environment`s that share the same -/// `RecordValue`. Instead, call `refreshRecordValue()`, then set the properties -/// on the new `RecordValue` that it returns. Typical usage: -/// -/// refreshRecordValue(Loc, Env).setProperty("my_prop", MyPropValue); RecordValue &refreshRecordValue(RecordStorageLocation &Loc, Environment &Env); /// Associates a new `RecordValue` with `Expr` and returns the new value. -/// See also documentation for the overload above. RecordValue &refreshRecordValue(const Expr &Expr, Environment &Env); } // namespace dataflow diff --git a/clang/include/clang/Analysis/FlowSensitive/RecordOps.h b/clang/include/clang/Analysis/FlowSensitive/RecordOps.h index 7b87840d626b..783e53e980aa 100644 --- a/clang/include/clang/Analysis/FlowSensitive/RecordOps.h +++ b/clang/include/clang/Analysis/FlowSensitive/RecordOps.h @@ -22,19 +22,13 @@ namespace dataflow { /// Copies a record (struct, class, or union) from `Src` to `Dst`. /// /// This performs a deep copy, i.e. it copies every field (including synthetic -/// fields) and recurses on fields of record type. It also copies properties -/// from the `RecordValue` associated with `Src` to the `RecordValue` associated -/// with `Dst` (if these `RecordValue`s exist). +/// fields) and recurses on fields of record type. /// /// If there is a `RecordValue` associated with `Dst` in the environment, this /// function creates a new `RecordValue` and associates it with `Dst`; clients /// need to be aware of this and must not assume that the `RecordValue` /// associated with `Dst` remains the same after the call. /// -/// We create a new `RecordValue` rather than modifying properties on the old -/// `RecordValue` because the old `RecordValue` may be shared with other -/// `Environment`s, and we don't want changes to properties to be visible there. -/// /// Requirements: /// /// `Src` and `Dst` must have the same canonical unqualified type. @@ -49,9 +43,7 @@ void copyRecord(RecordStorageLocation &Src, RecordStorageLocation &Dst, /// /// This performs a deep comparison, i.e. it compares every field (including /// synthetic fields) and recurses on fields of record type. Fields of reference -/// type compare equal if they refer to the same storage location. If -/// `RecordValue`s are associated with `Loc1` and Loc2`, it also compares the -/// properties on those `RecordValue`s. +/// type compare equal if they refer to the same storage location. /// /// Note on how to interpret the result: /// - If this returns true, the records are guaranteed to be equal at runtime. diff --git a/clang/include/clang/Analysis/FlowSensitive/Value.h b/clang/include/clang/Analysis/FlowSensitive/Value.h index e6c68e5b4e93..be1bf9324c87 100644 --- a/clang/include/clang/Analysis/FlowSensitive/Value.h +++ b/clang/include/clang/Analysis/FlowSensitive/Value.h @@ -63,7 +63,11 @@ public: /// Assigns `Val` as the value of the synthetic property with the given /// `Name`. + /// + /// Properties may not be set on `RecordValue`s; use synthetic fields instead + /// (for details, see documentation for `RecordStorageLocation`). void setProperty(llvm::StringRef Name, Value &Val) { + assert(getKind() != Kind::Record); Properties.insert_or_assign(Name, &Val); } @@ -184,33 +188,23 @@ private: /// In C++, prvalues of class type serve only a limited purpose: They can only /// be used to initialize a result object. It is not possible to access member /// variables or call member functions on a prvalue of class type. -/// Correspondingly, `RecordValue` also serves only two limited purposes: -/// - It conveys a prvalue of class type from the place where the object is -/// constructed to the result object that it initializes. +/// Correspondingly, `RecordValue` also serves only a limited purpose: It +/// conveys a prvalue of class type from the place where the object is +/// constructed to the result object that it initializes. /// -/// When creating a prvalue of class type, we already need a storage location -/// for `this`, even though prvalues are otherwise not associated with storage -/// locations. `RecordValue` is therefore essentially a wrapper for a storage -/// location, which is then used to set the storage location for the result -/// object when we process the AST node for that result object. +/// When creating a prvalue of class type, we already need a storage location +/// for `this`, even though prvalues are otherwise not associated with storage +/// locations. `RecordValue` is therefore essentially a wrapper for a storage +/// location, which is then used to set the storage location for the result +/// object when we process the AST node for that result object. /// -/// For example: -/// MyStruct S = MyStruct(3); +/// For example: +/// MyStruct S = MyStruct(3); /// -/// In this example, `MyStruct(3) is a prvalue, which is modeled as a -/// `RecordValue` that wraps a `RecordStorageLocation`. This -// `RecordStorageLocation` is then used as the storage location for `S`. +/// In this example, `MyStruct(3) is a prvalue, which is modeled as a +/// `RecordValue` that wraps a `RecordStorageLocation`. This +/// `RecordStorageLocation` is then used as the storage location for `S`. /// -/// - It allows properties to be associated with an object of class type. -/// Note that when doing so, you should avoid mutating the properties of an -/// existing `RecordValue` in place, as these changes would be visible to -/// other `Environment`s that share the same `RecordValue`. Instead, associate -/// a new `RecordValue` with the `RecordStorageLocation` and set the -/// properties on this new `RecordValue`. (See also `refreshRecordValue()` in -/// DataflowEnvironment.h, which makes this easy.) -/// Note also that this implies that it is common for the same -/// `RecordStorageLocation` to be associated with different `RecordValue`s -/// in different environments. /// Over time, we may eliminate `RecordValue` entirely. See also the discussion /// here: https://reviews.llvm.org/D155204#inline-1503204 class RecordValue final : public Value { diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def index 152d9f65f86d..21abc346cf17 100644 --- a/clang/include/clang/Basic/LangOptions.def +++ b/clang/include/clang/Basic/LangOptions.def @@ -456,6 +456,7 @@ ENUM_LANGOPT(SignReturnAddressScope, SignReturnAddressScopeKind, 2, SignReturnAd ENUM_LANGOPT(SignReturnAddressKey, SignReturnAddressKeyKind, 1, SignReturnAddressKeyKind::AKey, "Key used for return address signing") LANGOPT(BranchTargetEnforcement, 1, 0, "Branch-target enforcement enabled") +LANGOPT(BranchProtectionPAuthLR, 1, 0, "Use PC as a diversifier using PAuthLR NOP instructions.") LANGOPT(SpeculativeLoadHardening, 1, 0, "Speculative load hardening enabled") diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h index aa0f5023104a..ac3c324c6c29 100644 --- a/clang/include/clang/Basic/TargetInfo.h +++ b/clang/include/clang/Basic/TargetInfo.h @@ -1372,6 +1372,7 @@ public: LangOptions::SignReturnAddressKeyKind SignKey = LangOptions::SignReturnAddressKeyKind::AKey; bool BranchTargetEnforcement = false; + bool BranchProtectionPAuthLR = false; }; /// Determine if the Architecture in this TargetInfo supports branch diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td index ce99ca82c1d3..aac3bd486de9 100644 --- a/clang/include/clang/Basic/arm_sme.td +++ b/clang/include/clang/Basic/arm_sme.td @@ -351,6 +351,285 @@ let TargetGuard = "sme2" in { def SVBMOPA : Inst<"svbmopa_za32[_{d}]_m", "viPPdd", "iUi", MergeNone, "aarch64_sme_bmopa_za32", [IsSharedZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>; def SVBMOPS : Inst<"svbmops_za32[_{d}]_m", "viPPdd", "iUi", MergeNone, "aarch64_sme_bmops_za32", [IsSharedZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>; + + // VERTICAL DOT-PRODUCT + def SVVDOT_LANE_ZA32_VG1x2_S : Inst<"svvdot_lane_za32[_{d}]_vg1x2", "vm2di", "s", MergeNone, "aarch64_sme_svdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVVDOT_LANE_ZA32_VG1x4_S : Inst<"svvdot_lane_za32[_{d}]_vg1x4", "vm4di", "c", MergeNone, "aarch64_sme_svdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVVDOT_LANE_ZA32_VG1x2_U : Inst<"svvdot_lane_za32[_{d}]_vg1x2", "vm2di", "Us", MergeNone, "aarch64_sme_uvdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVVDOT_LANE_ZA32_VG1x4_U : Inst<"svvdot_lane_za32[_{d}]_vg1x4", "vm4di", "Uc", MergeNone, "aarch64_sme_uvdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVVDOT_LANE_ZA32_VG1x2_F : Inst<"svvdot_lane_za32[_{d}]_vg1x2", "vm2di", "hb", MergeNone, "aarch64_sme_fvdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVSUVDOT_LANE_ZA32_VG1x4 : Inst<"svsuvdot_lane_za32[_{d}]_vg1x4", "vm4di", "c", MergeNone, "aarch64_sme_suvdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVUSVDOT_LANE_ZA32_VG1x4 : Inst<"svusvdot_lane_za32[_{d}]_vg1x4", "vm4di", "Uc", MergeNone, "aarch64_sme_usvdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + + // Multi-vector signed & unsigned integer dot-product + def SVDOT_MULTI_ZA32_VG1x2_S : Inst<"svdot_za32[_{d}]_vg1x2", "vm22", "cs", MergeNone, "aarch64_sme_sdot_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_MULTI_ZA32_VG1x4_S : Inst<"svdot_za32[_{d}]_vg1x4", "vm44", "cs", MergeNone, "aarch64_sme_sdot_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_MULTI_ZA32_VG1x2_U : Inst<"svdot_za32[_{d}]_vg1x2", "vm22", "UcUs", MergeNone, "aarch64_sme_udot_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_MULTI_ZA32_VG1x4_U : Inst<"svdot_za32[_{d}]_vg1x4", "vm44", "UcUs", MergeNone, "aarch64_sme_udot_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA32_VG1x2_S : Inst<"svdot[_single]_za32[_{d}]_vg1x2", "vm2d", "cs", MergeNone, "aarch64_sme_sdot_single_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA32_VG1x4_S : Inst<"svdot[_single]_za32[_{d}]_vg1x4", "vm4d", "cs", MergeNone, "aarch64_sme_sdot_single_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA32_VG1x2_U : Inst<"svdot[_single]_za32[_{d}]_vg1x2", "vm2d", "UcUs", MergeNone, "aarch64_sme_udot_single_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA32_VG1x4_U : Inst<"svdot[_single]_za32[_{d}]_vg1x4", "vm4d", "UcUs", MergeNone, "aarch64_sme_udot_single_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_LANE_ZA32_VG1x2_S : Inst<"svdot_lane_za32[_{d}]_vg1x2", "vm2di", "cs", MergeNone, "aarch64_sme_sdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVDOT_LANE_ZA32_VG1x4_S : Inst<"svdot_lane_za32[_{d}]_vg1x4", "vm4di", "cs", MergeNone, "aarch64_sme_sdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVDOT_LANE_ZA32_VG1x2_U : Inst<"svdot_lane_za32[_{d}]_vg1x2", "vm2di", "UcUs", MergeNone, "aarch64_sme_udot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVDOT_LANE_ZA32_VG1x4_U : Inst<"svdot_lane_za32[_{d}]_vg1x4", "vm4di", "UcUs", MergeNone, "aarch64_sme_udot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + + def SVUSDOT_SINGLE_ZA32_VG1x2 : Inst<"svusdot[_single]_za32[_{d}]_vg1x2", "vm2.dx", "Uc", MergeNone, "aarch64_sme_usdot_single_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVUSDOT_SINGLE_ZA32_VG1x4 : Inst<"svusdot[_single]_za32[_{d}]_vg1x4", "vm4.dx", "Uc", MergeNone, "aarch64_sme_usdot_single_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVUSDOT_MULTI_ZA32_VG1x2 : Inst<"svusdot_za32[_{d}]_vg1x2", "vm2.d2.x", "Uc", MergeNone, "aarch64_sme_usdot_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVUSDOT_MULTI_ZA32_VG1x4 : Inst<"svusdot_za32[_{d}]_vg1x4", "vm4.d4.x", "Uc", MergeNone, "aarch64_sme_usdot_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVUSDOT_LANE_ZA32_VG1x2 : Inst<"svusdot_lane_za32[_{d}]_vg1x2", "vm2.dxi", "Uc", MergeNone, "aarch64_sme_usdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVUSDOT_LANE_ZA32_VG1x4 : Inst<"svusdot_lane_za32[_{d}]_vg1x4", "vm4.dxi", "Uc", MergeNone, "aarch64_sme_usdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + + def SVSUDOT_SINGLE_ZA32_VG1x2 : Inst<"svsudot[_single]_za32[_{d}]_vg1x2", "vm2.du", "c", MergeNone, "aarch64_sme_sudot_single_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVSUDOT_SINGLE_ZA32_VG1x4 : Inst<"svsudot[_single]_za32[_{d}]_vg1x4", "vm4.du", "c", MergeNone, "aarch64_sme_sudot_single_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + + // Multi-multi sudot builtins are mapped to usdot, with zn & zm operands swapped + def SVSUDOT_MULTI_ZA32_VG1x2 : Inst<"svsudot_za32[_{d}]_vg1x2", "vm2.d2.u", "c", MergeNone, "aarch64_sme_usdot_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVSUDOT_MULTI_ZA32_VG1x4 : Inst<"svsudot_za32[_{d}]_vg1x4", "vm4.d4.u", "c", MergeNone, "aarch64_sme_usdot_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + + def SVSUDOT_LANE_ZA32_VG1x2 : Inst<"svsudot_lane_za32[_{d}]_vg1x2", "vm2.dui", "c", MergeNone, "aarch64_sme_sudot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVSUDOT_LANE_ZA32_VG1x4 : Inst<"svsudot_lane_za32[_{d}]_vg1x4", "vm4.dui", "c", MergeNone, "aarch64_sme_sudot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + + // Multi-vector half-precision/BFloat16 floating-point dot-product + def SVDOT_MULTI_ZA32_VG1x2_F16 : Inst<"svdot_za32[_{d}]_vg1x2", "vm22", "bh", MergeNone, "aarch64_sme_fdot_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_MULTI_ZA32_VG1x4_F16 : Inst<"svdot_za32[_{d}]_vg1x4", "vm44", "bh", MergeNone, "aarch64_sme_fdot_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA32_VG1x2_F16 : Inst<"svdot[_single]_za32[_{d}]_vg1x2", "vm2d", "bh", MergeNone, "aarch64_sme_fdot_single_za32_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA32_VG1x4_F16 : Inst<"svdot[_single]_za32[_{d}]_vg1x4", "vm4d", "bh", MergeNone, "aarch64_sme_fdot_single_za32_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_LANE_ZA32_VG1x2_F16 : Inst<"svdot_lane_za32[_{d}]_vg1x2", "vm2di", "bh", MergeNone, "aarch64_sme_fdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVDOT_LANE_ZA32_VG1x4_F16 : Inst<"svdot_lane_za32[_{d}]_vg1x4", "vm4di", "bh", MergeNone, "aarch64_sme_fdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; +} + +let TargetGuard = "sme2,sme-i16i64" in { + def SVVDOT_LANE_ZA64_VG1x4_S : Inst<"svvdot_lane_za64[_{d}]_vg1x4", "vm4di", "s", MergeNone, "aarch64_sme_svdot_lane_za64_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + def SVVDOT_LANE_ZA64_VG1x4_U : Inst<"svvdot_lane_za64[_{d}]_vg1x4", "vm4di", "Us", MergeNone, "aarch64_sme_uvdot_lane_za64_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + + def SVDOT_MULTI_ZA64_VG1x2_S16 : Inst<"svdot_za64[_{d}]_vg1x2", "vm22", "s", MergeNone, "aarch64_sme_sdot_za64_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_MULTI_ZA64_VG1x4_S16 : Inst<"svdot_za64[_{d}]_vg1x4", "vm44", "s", MergeNone, "aarch64_sme_sdot_za64_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_MULTI_ZA64_VG1x2_U16 : Inst<"svdot_za64[_{d}]_vg1x2", "vm22", "Us", MergeNone, "aarch64_sme_udot_za64_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_MULTI_ZA64_VG1x4_U16 : Inst<"svdot_za64[_{d}]_vg1x4", "vm44", "Us", MergeNone, "aarch64_sme_udot_za64_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA64_VG1x2_S16 : Inst<"svdot[_single]_za64[_{d}]_vg1x2", "vm2d", "s", MergeNone, "aarch64_sme_sdot_single_za64_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA64_VG1x4_S16 : Inst<"svdot[_single]_za64[_{d}]_vg1x4", "vm4d", "s", MergeNone, "aarch64_sme_sdot_single_za64_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA64_VG1x2_U16 : Inst<"svdot[_single]_za64[_{d}]_vg1x2", "vm2d", "Us", MergeNone, "aarch64_sme_udot_single_za64_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVDOT_SINGLE_ZA64_VG1x4_U16 : Inst<"svdot[_single]_za64[_{d}]_vg1x4", "vm4d", "Us", MergeNone, "aarch64_sme_udot_single_za64_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVDOT_LANE_ZA64_VG1x2_S16 : Inst<"svdot_lane_za64[_{d}]_vg1x2", "vm2di", "s", MergeNone, "aarch64_sme_sdot_lane_za64_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + def SVDOT_LANE_ZA64_VG1x4_S16 : Inst<"svdot_lane_za64[_{d}]_vg1x4", "vm4di", "s", MergeNone, "aarch64_sme_sdot_lane_za64_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + def SVDOT_LANE_ZA64_VG1x2_U16 : Inst<"svdot_lane_za64[_{d}]_vg1x2", "vm2di", "Us", MergeNone, "aarch64_sme_udot_lane_za64_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + def SVDOT_LANE_ZA64_VG1x4_U16 : Inst<"svdot_lane_za64[_{d}]_vg1x4", "vm4di", "Us", MergeNone, "aarch64_sme_udot_lane_za64_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; +} + +// FMLA/FMLS +let TargetGuard = "sme2" in { + def SVMLA_MULTI_VG1x2_F32 : Inst<"svmla_za32[_{d}]_vg1x2", "vm22", "f", MergeNone, "aarch64_sme_fmla_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLA_MULTI_VG1x4_F32 : Inst<"svmla_za32[_{d}]_vg1x4", "vm44", "f", MergeNone, "aarch64_sme_fmla_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVMLS_MULTI_VG1x2_F32 : Inst<"svmls_za32[_{d}]_vg1x2", "vm22", "f", MergeNone, "aarch64_sme_fmls_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLS_MULTI_VG1x4_F32 : Inst<"svmls_za32[_{d}]_vg1x4", "vm44", "f", MergeNone, "aarch64_sme_fmls_vg1x4", [IsStreaming, IsSharedZA], []>; + + def SVMLA_SINGLE_VG1x2_F32 : Inst<"svmla[_single]_za32[_{d}]_vg1x2", "vm2d", "f", MergeNone, "aarch64_sme_fmla_single_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLA_SINGLE_VG1x4_F32 : Inst<"svmla[_single]_za32[_{d}]_vg1x4", "vm4d", "f", MergeNone, "aarch64_sme_fmla_single_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVMLS_SINGLE_VG1x2_F32 : Inst<"svmls[_single]_za32[_{d}]_vg1x2", "vm2d", "f", MergeNone, "aarch64_sme_fmls_single_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLS_SINGLE_VG1x4_F32 : Inst<"svmls[_single]_za32[_{d}]_vg1x4", "vm4d", "f", MergeNone, "aarch64_sme_fmls_single_vg1x4", [IsStreaming, IsSharedZA], []>; + + def SVMLA_LANE_VG1x2_F32 : Inst<"svmla_lane_za32[_{d}]_vg1x2", "vm2di", "f", MergeNone, "aarch64_sme_fmla_lane_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVMLA_LANE_VG1x4_F32 : Inst<"svmla_lane_za32[_{d}]_vg1x4", "vm4di", "f", MergeNone, "aarch64_sme_fmla_lane_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVMLS_LANE_VG1x2_F32 : Inst<"svmls_lane_za32[_{d}]_vg1x2", "vm2di", "f", MergeNone, "aarch64_sme_fmls_lane_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; + def SVMLS_LANE_VG1x4_F32 : Inst<"svmls_lane_za32[_{d}]_vg1x4", "vm4di", "f", MergeNone, "aarch64_sme_fmls_lane_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>; +} + +let TargetGuard = "sme2,sme-f64f64" in { + def SVMLA_MULTI_VG1x2_F64 : Inst<"svmla_za64[_{d}]_vg1x2", "vm22", "d", MergeNone, "aarch64_sme_fmla_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLA_MULTI_VG1x4_F64 : Inst<"svmla_za64[_{d}]_vg1x4", "vm44", "d", MergeNone, "aarch64_sme_fmla_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVMLS_MULTI_VG1x2_F64 : Inst<"svmls_za64[_{d}]_vg1x2", "vm22", "d", MergeNone, "aarch64_sme_fmls_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLS_MULTI_VG1x4_F64 : Inst<"svmls_za64[_{d}]_vg1x4", "vm44", "d", MergeNone, "aarch64_sme_fmls_vg1x4", [IsStreaming, IsSharedZA], []>; + + def SVMLA_SINGLE_VG1x2_F64 : Inst<"svmla[_single]_za64[_{d}]_vg1x2", "vm2d", "d", MergeNone, "aarch64_sme_fmla_single_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLA_SINGLE_VG1x4_F64 : Inst<"svmla[_single]_za64[_{d}]_vg1x4", "vm4d", "d", MergeNone, "aarch64_sme_fmla_single_vg1x4", [IsStreaming, IsSharedZA], []>; + def SVMLS_SINGLE_VG1x2_F64 : Inst<"svmls[_single]_za64[_{d}]_vg1x2", "vm2d", "d", MergeNone, "aarch64_sme_fmls_single_vg1x2", [IsStreaming, IsSharedZA], []>; + def SVMLS_SINGLE_VG1x4_F64 : Inst<"svmls[_single]_za64[_{d}]_vg1x4", "vm4d", "d", MergeNone, "aarch64_sme_fmls_single_vg1x4", [IsStreaming, IsSharedZA], []>; + + def SVMLA_LANE_VG1x2_F64 : Inst<"svmla_lane_za64[_{d}]_vg1x2", "vm2di", "d", MergeNone, "aarch64_sme_fmla_lane_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + def SVMLA_LANE_VG1x4_F64 : Inst<"svmla_lane_za64[_{d}]_vg1x4", "vm4di", "d", MergeNone, "aarch64_sme_fmla_lane_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + def SVMLS_LANE_VG1x2_F64 : Inst<"svmls_lane_za64[_{d}]_vg1x2", "vm2di", "d", MergeNone, "aarch64_sme_fmls_lane_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; + def SVMLS_LANE_VG1x4_F64 : Inst<"svmls_lane_za64[_{d}]_vg1x4", "vm4di", "d", MergeNone, "aarch64_sme_fmls_lane_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>; +} + +// FMLAL/FMLSL/UMLAL/SMLAL +// SMLALL/UMLALL/USMLALL/SUMLALL +let TargetGuard = "sme2" in { + // MULTI MLAL + def SVMLAL_MULTI_VG2x2_F16 : Inst<"svmla_za32[_{d}]_vg2x2", "vm22", "bh", MergeNone, "aarch64_sme_fmlal_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG2x4_F16 : Inst<"svmla_za32[_{d}]_vg2x4", "vm44", "bh", MergeNone, "aarch64_sme_fmlal_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG2x2_S16 : Inst<"svmla_za32[_{d}]_vg2x2", "vm22", "s", MergeNone, "aarch64_sme_smlal_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG2x4_S16 : Inst<"svmla_za32[_{d}]_vg2x4", "vm44", "s", MergeNone, "aarch64_sme_smlal_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG2x2_U16 : Inst<"svmla_za32[_{d}]_vg2x2", "vm22", "Us", MergeNone, "aarch64_sme_umlal_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG2x4_U16 : Inst<"svmla_za32[_{d}]_vg2x4", "vm44", "Us", MergeNone, "aarch64_sme_umlal_vg2x4", [IsStreaming, IsSharedZA], []>; + + def SVMLAL_MULTI_VG4x2_S8 : Inst<"svmla_za32[_{d}]_vg4x2", "vm22", "c", MergeNone, "aarch64_sme_smla_za32_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG4x2_U8 : Inst<"svmla_za32[_{d}]_vg4x2", "vm22", "Uc", MergeNone, "aarch64_sme_umla_za32_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG4x4_S8 : Inst<"svmla_za32[_{d}]_vg4x4", "vm44", "c", MergeNone, "aarch64_sme_smla_za32_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG4x4_U8 : Inst<"svmla_za32[_{d}]_vg4x4", "vm44", "Uc", MergeNone, "aarch64_sme_umla_za32_vg4x4", [IsStreaming, IsSharedZA], []>; + + // MULTI MLSL + def SVMLSL_MULTI_VG2x2_F16 : Inst<"svmls_za32[_{d}]_vg2x2", "vm22", "bh", MergeNone, "aarch64_sme_fmlsl_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG2x4_F16 : Inst<"svmls_za32[_{d}]_vg2x4", "vm44", "bh", MergeNone, "aarch64_sme_fmlsl_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG2x2_S16 : Inst<"svmls_za32[_{d}]_vg2x2", "vm22", "s", MergeNone, "aarch64_sme_smlsl_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG2x4_S16 : Inst<"svmls_za32[_{d}]_vg2x4", "vm44", "s", MergeNone, "aarch64_sme_smlsl_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG2x2_U16 : Inst<"svmls_za32[_{d}]_vg2x2", "vm22", "Us", MergeNone, "aarch64_sme_umlsl_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG2x4_U16 : Inst<"svmls_za32[_{d}]_vg2x4", "vm44", "Us", MergeNone, "aarch64_sme_umlsl_vg2x4", [IsStreaming, IsSharedZA], []>; + + def SVMLSL_MULTI_VG4x2_S8 : Inst<"svmls_za32[_{d}]_vg4x2", "vm22", "c", MergeNone, "aarch64_sme_smls_za32_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG4x2_U8 : Inst<"svmls_za32[_{d}]_vg4x2", "vm22", "Uc", MergeNone, "aarch64_sme_umls_za32_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG4x4_S8 : Inst<"svmls_za32[_{d}]_vg4x4", "vm44", "c", MergeNone, "aarch64_sme_smls_za32_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG4x4_U8 : Inst<"svmls_za32[_{d}]_vg4x4", "vm44", "Uc", MergeNone, "aarch64_sme_umls_za32_vg4x4", [IsStreaming, IsSharedZA], []>; + + // SINGLE MLAL + def SVMLAL_SINGLE_VG2x1_F16 : Inst<"svmla_za32[_{d}]_vg2x1", "vmdd", "bh", MergeNone, "aarch64_sme_fmlal_single_vg2x1", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x2_F16 : Inst<"svmla[_single]_za32[_{d}]_vg2x2", "vm2d", "bh", MergeNone, "aarch64_sme_fmlal_single_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x4_F16 : Inst<"svmla[_single]_za32[_{d}]_vg2x4", "vm4d", "bh", MergeNone, "aarch64_sme_fmlal_single_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x1_S16 : Inst<"svmla_za32[_{d}]_vg2x1", "vmdd", "s", MergeNone, "aarch64_sme_smlal_single_vg2x1", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x2_S16 : Inst<"svmla[_single]_za32[_{d}]_vg2x2", "vm2d", "s", MergeNone, "aarch64_sme_smlal_single_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x4_S16 : Inst<"svmla[_single]_za32[_{d}]_vg2x4", "vm4d", "s", MergeNone, "aarch64_sme_smlal_single_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x1_U16 : Inst<"svmla_za32[_{d}]_vg2x1", "vmdd", "Us", MergeNone, "aarch64_sme_umlal_single_vg2x1", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x2_U16 : Inst<"svmla[_single]_za32[_{d}]_vg2x2", "vm2d", "Us", MergeNone, "aarch64_sme_umlal_single_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG2x4_U16 : Inst<"svmla[_single]_za32[_{d}]_vg2x4", "vm4d", "Us", MergeNone, "aarch64_sme_umlal_single_vg2x4", [IsStreaming, IsSharedZA], []>; + + def SVMLAL_SINGLE_VG4x1_S8 : Inst<"svmla_za32[_{d}]_vg4x1", "vmdd", "c", MergeNone, "aarch64_sme_smla_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x1_U8 : Inst<"svmla_za32[_{d}]_vg4x1", "vmdd", "Uc", MergeNone, "aarch64_sme_umla_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x2_S8 : Inst<"svmla[_single]_za32[_{d}]_vg4x2", "vm2d", "c", MergeNone, "aarch64_sme_smla_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x2_U8 : Inst<"svmla[_single]_za32[_{d}]_vg4x2", "vm2d", "Uc", MergeNone, "aarch64_sme_umla_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x4_S8 : Inst<"svmla[_single]_za32[_{d}]_vg4x4", "vm4d", "c", MergeNone, "aarch64_sme_smla_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x4_U8 : Inst<"svmla[_single]_za32[_{d}]_vg4x4", "vm4d", "Uc", MergeNone, "aarch64_sme_umla_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>; + + // SINGLE MLSL + def SVMLSL_SINGLE_VG2x1_F16 : Inst<"svmls_za32[_{d}]_vg2x1", "vmdd", "bh", MergeNone, "aarch64_sme_fmlsl_single_vg2x1", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x2_F16 : Inst<"svmls[_single]_za32[_{d}]_vg2x2", "vm2d", "bh", MergeNone, "aarch64_sme_fmlsl_single_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x4_F16 : Inst<"svmls[_single]_za32[_{d}]_vg2x4", "vm4d", "bh", MergeNone, "aarch64_sme_fmlsl_single_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x1_S16 : Inst<"svmls_za32[_{d}]_vg2x1", "vmdd", "s", MergeNone, "aarch64_sme_smlsl_single_vg2x1", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x2_S16 : Inst<"svmls[_single]_za32[_{d}]_vg2x2", "vm2d", "s", MergeNone, "aarch64_sme_smlsl_single_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x4_S16 : Inst<"svmls[_single]_za32[_{d}]_vg2x4", "vm4d", "s", MergeNone, "aarch64_sme_smlsl_single_vg2x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x1_U16 : Inst<"svmls_za32[_{d}]_vg2x1", "vmdd", "Us", MergeNone, "aarch64_sme_umlsl_single_vg2x1", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x2_U16 : Inst<"svmls[_single]_za32[_{d}]_vg2x2", "vm2d", "Us", MergeNone, "aarch64_sme_umlsl_single_vg2x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG2x4_U16 : Inst<"svmls[_single]_za32[_{d}]_vg2x4", "vm4d", "Us", MergeNone, "aarch64_sme_umlsl_single_vg2x4", [IsStreaming, IsSharedZA], []>; + + def SVMLSL_SINGLE_VG4x1_S8 : Inst<"svmls_za32[_{d}]_vg4x1", "vmdd", "c", MergeNone, "aarch64_sme_smls_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x1_U8 : Inst<"svmls_za32[_{d}]_vg4x1", "vmdd", "Uc", MergeNone, "aarch64_sme_umls_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x2_S8 : Inst<"svmls[_single]_za32[_{d}]_vg4x2", "vm2d", "c", MergeNone, "aarch64_sme_smls_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x2_U8 : Inst<"svmls[_single]_za32[_{d}]_vg4x2", "vm2d", "Uc", MergeNone, "aarch64_sme_umls_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x4_S8 : Inst<"svmls[_single]_za32[_{d}]_vg4x4", "vm4d", "c", MergeNone, "aarch64_sme_smls_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x4_U8 : Inst<"svmls[_single]_za32[_{d}]_vg4x4", "vm4d", "Uc", MergeNone, "aarch64_sme_umls_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>; + + // INDEXED MLAL + def SVMLAL_LANE_VG2x1_F16 : Inst<"svmla_lane_za32[_{d}]_vg2x1", "vmddi", "bh", MergeNone, "aarch64_sme_fmlal_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x2_F16 : Inst<"svmla_lane_za32[_{d}]_vg2x2", "vm2di", "bh", MergeNone, "aarch64_sme_fmlal_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x4_F16 : Inst<"svmla_lane_za32[_{d}]_vg2x4", "vm4di", "bh", MergeNone, "aarch64_sme_fmlal_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x1_S16 : Inst<"svmla_lane_za32[_{d}]_vg2x1", "vmddi", "s", MergeNone, "aarch64_sme_smlal_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x2_S16 : Inst<"svmla_lane_za32[_{d}]_vg2x2", "vm2di", "s", MergeNone, "aarch64_sme_smlal_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x4_S16 : Inst<"svmla_lane_za32[_{d}]_vg2x4", "vm4di", "s", MergeNone, "aarch64_sme_smlal_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x1_U16 : Inst<"svmla_lane_za32[_{d}]_vg2x1", "vmddi", "Us", MergeNone, "aarch64_sme_umlal_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x2_U16 : Inst<"svmla_lane_za32[_{d}]_vg2x2", "vm2di", "Us", MergeNone, "aarch64_sme_umlal_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG2x4_U16 : Inst<"svmla_lane_za32[_{d}]_vg2x4", "vm4di", "Us", MergeNone, "aarch64_sme_umlal_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + + def SVMLAL_LANE_VG4x1_S8 : Inst<"svmla_lane_za32[_{d}]_vg4x1", "vmddi", "c", MergeNone, "aarch64_sme_smla_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLAL_LANE_VG4x1_U8 : Inst<"svmla_lane_za32[_{d}]_vg4x1", "vmddi", "Uc", MergeNone, "aarch64_sme_umla_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLAL_LANE_VG4x2_S8 : Inst<"svmla_lane_za32[_{d}]_vg4x2", "vm2di", "c", MergeNone, "aarch64_sme_smla_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLAL_LANE_VG4x2_U8 : Inst<"svmla_lane_za32[_{d}]_vg4x2", "vm2di", "Uc", MergeNone, "aarch64_sme_umla_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLAL_LANE_VG4x4_S8 : Inst<"svmla_lane_za32[_{d}]_vg4x4", "vm4di", "c", MergeNone, "aarch64_sme_smla_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLAL_LANE_VG4x4_U8 : Inst<"svmla_lane_za32[_{d}]_vg4x4", "vm4di", "Uc", MergeNone, "aarch64_sme_umla_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + + // INDEXED MLSL + def SVMLSL_LANE_VG2x1_F16 : Inst<"svmls_lane_za32[_{d}]_vg2x1", "vmddi", "bh", MergeNone, "aarch64_sme_fmlsl_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x2_F16 : Inst<"svmls_lane_za32[_{d}]_vg2x2", "vm2di", "bh", MergeNone, "aarch64_sme_fmlsl_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x4_F16 : Inst<"svmls_lane_za32[_{d}]_vg2x4", "vm4di", "bh", MergeNone, "aarch64_sme_fmlsl_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x1_S16 : Inst<"svmls_lane_za32[_{d}]_vg2x1", "vmddi", "s", MergeNone, "aarch64_sme_smlsl_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x2_S16 : Inst<"svmls_lane_za32[_{d}]_vg2x2", "vm2di", "s", MergeNone, "aarch64_sme_smlsl_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x4_S16 : Inst<"svmls_lane_za32[_{d}]_vg2x4", "vm4di", "s", MergeNone, "aarch64_sme_smlsl_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x1_U16 : Inst<"svmls_lane_za32[_{d}]_vg2x1", "vmddi", "Us", MergeNone, "aarch64_sme_umlsl_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x2_U16 : Inst<"svmls_lane_za32[_{d}]_vg2x2", "vm2di", "Us", MergeNone, "aarch64_sme_umlsl_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG2x4_U16 : Inst<"svmls_lane_za32[_{d}]_vg2x4", "vm4di", "Us", MergeNone, "aarch64_sme_umlsl_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + + def SVMLSL_LANE_VG4x1_S8 : Inst<"svmls_lane_za32[_{d}]_vg4x1", "vmddi", "c", MergeNone, "aarch64_sme_smls_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLSL_LANE_VG4x1_U8 : Inst<"svmls_lane_za32[_{d}]_vg4x1", "vmddi", "Uc", MergeNone, "aarch64_sme_umls_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLSL_LANE_VG4x2_S8 : Inst<"svmls_lane_za32[_{d}]_vg4x2", "vm2di", "c", MergeNone, "aarch64_sme_smls_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLSL_LANE_VG4x2_U8 : Inst<"svmls_lane_za32[_{d}]_vg4x2", "vm2di", "Uc", MergeNone, "aarch64_sme_umls_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLSL_LANE_VG4x4_S8 : Inst<"svmls_lane_za32[_{d}]_vg4x4", "vm4di", "c", MergeNone, "aarch64_sme_smls_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVMLSL_LANE_VG4x4_U8 : Inst<"svmls_lane_za32[_{d}]_vg4x4", "vm4di", "Uc", MergeNone, "aarch64_sme_umls_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + + // SINGLE SUMLALL + // Single sumla maps to usmla, with zn & zm operands swapped + def SVSUMLALL_SINGLE_VG4x1 : Inst<"svsumla_za32[_{d}]_vg4x1", "vmdu", "c", MergeNone, "aarch64_sme_usmla_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>; + + def SVSUMLALL_SINGLE_VG4x2 : Inst<"svsumla[_single]_za32[_{d}]_vg4x2", "vm2.du", "c", MergeNone, "aarch64_sme_sumla_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVSUMLALL_SINGLE_VG4x4 : Inst<"svsumla[_single]_za32[_{d}]_vg4x4", "vm4.du", "c", MergeNone, "aarch64_sme_sumla_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>; + + // Multi-multi sumla builtins are mapped to usmla, with zn & zm operands swapped + def SVSUMLALL_MULTI_VG4x2 : Inst<"svsumla_za32[_{d}]_vg4x2", "vm2.d2.u", "c", MergeNone, "aarch64_sme_usmla_za32_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVSUMLALL_MULTI_VG4x4 : Inst<"svsumla_za32[_{d}]_vg4x4", "vm4.d4.u", "c", MergeNone, "aarch64_sme_usmla_za32_vg4x4", [IsStreaming, IsSharedZA], []>; + + // INDEXED SUMLALL + def SVSUMLALL_LANE_VG4x1 : Inst<"svsumla_lane_za32[_{d}]_vg4x1", "vmdui", "c", MergeNone, "aarch64_sme_sumla_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVSUMLALL_LANE_VG4x2 : Inst<"svsumla_lane_za32[_{d}]_vg4x2", "vm2ui", "c", MergeNone, "aarch64_sme_sumla_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVSUMLALL_LANE_VG4x4 : Inst<"svsumla_lane_za32[_{d}]_vg4x4", "vm4ui", "c", MergeNone, "aarch64_sme_sumla_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + + // SINGLE USMLALL + def SVUSMLALL_SINGLE_VG4x1 : Inst<"svusmla_za32[_{d}]_vg4x1", "vmdx", "Uc", MergeNone, "aarch64_sme_usmla_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVUSMLALL_SINGLE_VG4x2 : Inst<"svusmla[_single]_za32[_{d}]_vg4x2", "vm2.dx", "Uc", MergeNone, "aarch64_sme_usmla_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVUSMLALL_SINGLE_VG4x4 : Inst<"svusmla[_single]_za32[_{d}]_vg4x4", "vm4.dx", "Uc", MergeNone, "aarch64_sme_usmla_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>; + + // MULTI USMLALL + def SVUSMLALL_MULTI_VG4x2 : Inst<"svusmla_za32[_{d}]_vg4x2", "vm2.d2.x", "Uc", MergeNone, "aarch64_sme_usmla_za32_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVUSMLALL_MULTI_VG4x4 : Inst<"svusmla_za32[_{d}]_vg4x4", "vm4.d4.x", "Uc", MergeNone, "aarch64_sme_usmla_za32_vg4x4", [IsStreaming, IsSharedZA], []>; + + // INDEXED USMLALL + def SVUSMLALL_LANE_VG4x1 : Inst<"svusmla_lane_za32[_{d}]_vg4x1", "vmdxi", "Uc", MergeNone, "aarch64_sme_usmla_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVUSMLALL_LANE_VG4x2 : Inst<"svusmla_lane_za32[_{d}]_vg4x2", "vm2xi", "Uc", MergeNone, "aarch64_sme_usmla_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; + def SVUSMLALL_LANE_VG4x4 : Inst<"svusmla_lane_za32[_{d}]_vg4x4", "vm4xi", "Uc", MergeNone, "aarch64_sme_usmla_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>; +} + +let TargetGuard = "sme2,sme-i16i64" in { + // MULTI MLAL + def SVMLAL_MULTI_VG4x2_S16 : Inst<"svmla_za64[_{d}]_vg4x2", "vm22", "s", MergeNone, "aarch64_sme_smla_za64_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG4x2_U16 : Inst<"svmla_za64[_{d}]_vg4x2", "vm22", "Us", MergeNone, "aarch64_sme_umla_za64_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG4x4_S16 : Inst<"svmla_za64[_{d}]_vg4x4", "vm44", "s", MergeNone, "aarch64_sme_smla_za64_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_MULTI_VG4x4_U16 : Inst<"svmla_za64[_{d}]_vg4x4", "vm44", "Us", MergeNone, "aarch64_sme_umla_za64_vg4x4", [IsStreaming, IsSharedZA], []>; + + // MULTI MLSL + def SVMLSL_MULTI_VG4x2_S16 : Inst<"svmls_za64[_{d}]_vg4x2", "vm22", "s", MergeNone, "aarch64_sme_smls_za64_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG4x2_U16 : Inst<"svmls_za64[_{d}]_vg4x2", "vm22", "Us", MergeNone, "aarch64_sme_umls_za64_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG4x4_S16 : Inst<"svmls_za64[_{d}]_vg4x4", "vm44", "s", MergeNone, "aarch64_sme_smls_za64_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_MULTI_VG4x4_U16 : Inst<"svmls_za64[_{d}]_vg4x4", "vm44", "Us", MergeNone, "aarch64_sme_umls_za64_vg4x4", [IsStreaming, IsSharedZA], []>; + + // SINGLE MLAL + def SVMLAL_SINGLE_VG4x1_S16 : Inst<"svmla_za64[_{d}]_vg4x1", "vmdd", "s", MergeNone, "aarch64_sme_smla_za64_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x1_U16 : Inst<"svmla_za64[_{d}]_vg4x1", "vmdd", "Us", MergeNone, "aarch64_sme_umla_za64_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x2_S16 : Inst<"svmla[_single]_za64[_{d}]_vg4x2", "vm2d", "s", MergeNone, "aarch64_sme_smla_za64_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x2_U16 : Inst<"svmla[_single]_za64[_{d}]_vg4x2", "vm2d", "Us", MergeNone, "aarch64_sme_umla_za64_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x4_S16 : Inst<"svmla[_single]_za64[_{d}]_vg4x4", "vm4d", "s", MergeNone, "aarch64_sme_smla_za64_single_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLAL_SINGLE_VG4x4_U16 : Inst<"svmla[_single]_za64[_{d}]_vg4x4", "vm4d", "Us", MergeNone, "aarch64_sme_umla_za64_single_vg4x4", [IsStreaming, IsSharedZA], []>; + + // SINGLE MLSL + def SVMLSL_SINGLE_VG4x1_S16 : Inst<"svmls_za64[_{d}]_vg4x1", "vmdd", "s", MergeNone, "aarch64_sme_smls_za64_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x1_U16 : Inst<"svmls_za64[_{d}]_vg4x1", "vmdd", "Us", MergeNone, "aarch64_sme_umls_za64_single_vg4x1", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x2_S16 : Inst<"svmls[_single]_za64[_{d}]_vg4x2", "vm2d", "s", MergeNone, "aarch64_sme_smls_za64_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x2_U16 : Inst<"svmls[_single]_za64[_{d}]_vg4x2", "vm2d", "Us", MergeNone, "aarch64_sme_umls_za64_single_vg4x2", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x4_S16 : Inst<"svmls[_single]_za64[_{d}]_vg4x4", "vm4d", "s", MergeNone, "aarch64_sme_smls_za64_single_vg4x4", [IsStreaming, IsSharedZA], []>; + def SVMLSL_SINGLE_VG4x4_U16 : Inst<"svmls[_single]_za64[_{d}]_vg4x4", "vm4d", "Us", MergeNone, "aarch64_sme_umls_za64_single_vg4x4", [IsStreaming, IsSharedZA], []>; + + // INDEXED MLAL + def SVMLAL_LANE_VG4x1_S16 : Inst<"svmla_lane_za64[_{d}]_vg4x1", "vmddi", "s", MergeNone, "aarch64_sme_smla_za64_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG4x1_U16 : Inst<"svmla_lane_za64[_{d}]_vg4x1", "vmddi", "Us", MergeNone, "aarch64_sme_umla_za64_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG4x2_S16 : Inst<"svmla_lane_za64[_{d}]_vg4x2", "vm2di", "s", MergeNone, "aarch64_sme_smla_za64_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG4x2_U16 : Inst<"svmla_lane_za64[_{d}]_vg4x2", "vm2di", "Us", MergeNone, "aarch64_sme_umla_za64_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG4x4_S16 : Inst<"svmla_lane_za64[_{d}]_vg4x4", "vm4di", "s", MergeNone, "aarch64_sme_smla_za64_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLAL_LANE_VG4x4_U16 : Inst<"svmla_lane_za64[_{d}]_vg4x4", "vm4di", "Us", MergeNone, "aarch64_sme_umla_za64_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + + // INDEXED MLSL + def SVMLSL_LANE_VG4x1_S16 : Inst<"svmls_lane_za64[_{d}]_vg4x1", "vmddi", "s", MergeNone, "aarch64_sme_smls_za64_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG4x1_U16 : Inst<"svmls_lane_za64[_{d}]_vg4x1", "vmddi", "Us", MergeNone, "aarch64_sme_umls_za64_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG4x2_S16 : Inst<"svmls_lane_za64[_{d}]_vg4x2", "vm2di", "s", MergeNone, "aarch64_sme_smls_za64_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG4x2_U16 : Inst<"svmls_lane_za64[_{d}]_vg4x2", "vm2di", "Us", MergeNone, "aarch64_sme_umls_za64_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG4x4_S16 : Inst<"svmls_lane_za64[_{d}]_vg4x4", "vm4di", "s", MergeNone, "aarch64_sme_smls_za64_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; + def SVMLSL_LANE_VG4x4_U16 : Inst<"svmls_lane_za64[_{d}]_vg4x4", "vm4di", "Us", MergeNone, "aarch64_sme_umls_za64_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>; } // diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td index 04bf7acdeba7..91f62c4c7633 100644 --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -1988,79 +1988,61 @@ def SVWHILELO_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNo def SVWHILELS_COUNT : SInst<"svwhilele_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; def SVWHILEHI_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; def SVWHILEHS_COUNT : SInst<"svwhilege_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +} + +multiclass MultiVecLoad<string i> { + // FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available (SME2 requires __arm_streaming) + def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + + def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; +} -def SVLD1B_X2 : MInst<"svld1[_{2}]_x2", "2}c", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1H_X2 : MInst<"svld1[_{2}]_x2", "2}c", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1W_X2 : MInst<"svld1[_{2}]_x2", "2}c", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1D_X2 : MInst<"svld1[_{2}]_x2", "2}c", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1B_X4 : MInst<"svld1[_{2}]_x4", "4}c", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; -def SVLD1H_X4 : MInst<"svld1[_{2}]_x4", "4}c", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; -def SVLD1W_X4 : MInst<"svld1[_{2}]_x4", "4}c", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; -def SVLD1D_X4 : MInst<"svld1[_{2}]_x4", "4}c", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; - -def SVLDNT1B_X2 : MInst<"svldnt1[_{2}]_x2", "2}c", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1H_X2 : MInst<"svldnt1[_{2}]_x2", "2}c", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1W_X2 : MInst<"svldnt1[_{2}]_x2", "2}c", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1D_X2 : MInst<"svldnt1[_{2}]_x2", "2}c", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1B_X4 : MInst<"svldnt1[_{2}]_x4", "4}c", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; -def SVLDNT1H_X4 : MInst<"svldnt1[_{2}]_x4", "4}c", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; -def SVLDNT1W_X4 : MInst<"svldnt1[_{2}]_x4", "4}c", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; -def SVLDNT1D_X4 : MInst<"svldnt1[_{2}]_x4", "4}c", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; - -def SVLD1B_VNUM_X2 : MInst<"svld1_vnum[_{2}]_x2", "2}cl", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1H_VNUM_X2 : MInst<"svld1_vnum[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1W_VNUM_X2 : MInst<"svld1_vnum[_{2}]_x2", "2}cl", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1D_VNUM_X2 : MInst<"svld1_vnum[_{2}]_x2", "2}cl", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">; -def SVLD1B_VNUM_X4 : MInst<"svld1_vnum[_{2}]_x4", "4}cl", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; -def SVLD1H_VNUM_X4 : MInst<"svld1_vnum[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; -def SVLD1W_VNUM_X4 : MInst<"svld1_vnum[_{2}]_x4", "4}cl", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; -def SVLD1D_VNUM_X4 : MInst<"svld1_vnum[_{2}]_x4", "4}cl", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">; - -def SVLDNT1B_VNUM_X2 : MInst<"svldnt1_vnum[_{2}]_x2", "2}cl", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1H_VNUM_X2 : MInst<"svldnt1_vnum[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1W_VNUM_X2 : MInst<"svldnt1_vnum[_{2}]_x2", "2}cl", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1D_VNUM_X2 : MInst<"svldnt1_vnum[_{2}]_x2", "2}cl", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">; -def SVLDNT1B_VNUM_X4 : MInst<"svldnt1_vnum[_{2}]_x4", "4}cl", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; -def SVLDNT1H_VNUM_X4 : MInst<"svldnt1_vnum[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; -def SVLDNT1W_VNUM_X4 : MInst<"svldnt1_vnum[_{2}]_x4", "4}cl", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; -def SVLDNT1D_VNUM_X4 : MInst<"svldnt1_vnum[_{2}]_x4", "4}cl", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">; - -def SVST1B_X2 : MInst<"svst1[_{2}_x2]", "v}p2", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1H_X2 : MInst<"svst1[_{2}_x2]", "v}p2", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1W_X2 : MInst<"svst1[_{2}_x2]", "v}p2", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1D_X2 : MInst<"svst1[_{2}_x2]", "v}p2", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1B_X4 : MInst<"svst1[_{2}_x4]", "v}p4", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; -def SVST1H_X4 : MInst<"svst1[_{2}_x4]", "v}p4", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; -def SVST1W_X4 : MInst<"svst1[_{2}_x4]", "v}p4", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; -def SVST1D_X4 : MInst<"svst1[_{2}_x4]", "v}p4", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; - -def SVST1B_VNUM_X2 : MInst<"svst1_vnum[_{2}_x2]", "v}pl2", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1H_VNUM_X2 : MInst<"svst1_vnum[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1W_VNUM_X2 : MInst<"svst1_vnum[_{2}_x2]", "v}pl2", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1D_VNUM_X2 : MInst<"svst1_vnum[_{2}_x2]", "v}pl2", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">; -def SVST1B_VNUM_X4 : MInst<"svst1_vnum[_{2}_x4]", "v}pl4", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; -def SVST1H_VNUM_X4 : MInst<"svst1_vnum[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; -def SVST1W_VNUM_X4 : MInst<"svst1_vnum[_{2}_x4]", "v}pl4", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; -def SVST1D_VNUM_X4 : MInst<"svst1_vnum[_{2}_x4]", "v}pl4", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">; - -def SVSTNT1B_X2 : MInst<"svstnt1[_{2}_x2]", "v}p2", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1H_X2 : MInst<"svstnt1[_{2}_x2]", "v}p2", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1W_X2 : MInst<"svstnt1[_{2}_x2]", "v}p2", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1D_X2 : MInst<"svstnt1[_{2}_x2]", "v}p2", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1B_X4 : MInst<"svstnt1[_{2}_x4]", "v}p4", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; -def SVSTNT1H_X4 : MInst<"svstnt1[_{2}_x4]", "v}p4", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; -def SVSTNT1W_X4 : MInst<"svstnt1[_{2}_x4]", "v}p4", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; -def SVSTNT1D_X4 : MInst<"svstnt1[_{2}_x4]", "v}p4", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; - -def SVSTNT1B_VNUM_X2 : MInst<"svstnt1_vnum[_{2}_x2]", "v}pl2", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1H_VNUM_X2 : MInst<"svstnt1_vnum[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1W_VNUM_X2 : MInst<"svstnt1_vnum[_{2}_x2]", "v}pl2", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1D_VNUM_X2 : MInst<"svstnt1_vnum[_{2}_x2]", "v}pl2", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">; -def SVSTNT1B_VNUM_X4 : MInst<"svstnt1_vnum[_{2}_x4]", "v}pl4", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; -def SVSTNT1H_VNUM_X4 : MInst<"svstnt1_vnum[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; -def SVSTNT1W_VNUM_X4 : MInst<"svstnt1_vnum[_{2}_x4]", "v}pl4", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; -def SVSTNT1D_VNUM_X4 : MInst<"svstnt1_vnum[_{2}_x4]", "v}pl4", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">; +let TargetGuard = "sve2p1|sme2" in { + defm LD1 : MultiVecLoad<"ld1">; + defm LDNT1 : MultiVecLoad<"ldnt1">; +} + +multiclass MultiVecStore<string i> { + // FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available (SME2 requires __arm_streaming) + def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + + def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; +} + +let TargetGuard = "sve2p1|sme2" in { + defm ST1 : MultiVecStore<"st1">; + defm STNT1 : MultiVecStore<"stnt1">; +} +let TargetGuard = "sve2p1" in { def SVDOT_X2_S : SInst<"svdot[_{d}_{2}_{3}]", "ddhh", "i", MergeNone, "aarch64_sve_sdot_x2", [], []>; def SVDOT_X2_U : SInst<"svdot[_{d}_{2}_{3}]", "ddhh", "Ui", MergeNone, "aarch64_sve_udot_x2", [], []>; def SVDOT_X2_F : SInst<"svdot[_{d}_{2}_{3}]", "ddhh", "f", MergeNone, "aarch64_sve_fdot_x2", [], []>; @@ -2142,6 +2124,21 @@ let TargetGuard = "sme2" in { } let TargetGuard = "sme2" in { + // FRINTA / FRINTM / FRINTN / FRINTP + def SVRINTA_X2 : SInst<"svrinta[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frinta_x2", [IsStreaming], []>; + def SVRINTA_X4 : SInst<"svrinta[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frinta_x4", [IsStreaming], []>; + + def SVRINTM_X2 : SInst<"svrintm[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frintm_x2", [IsStreaming], []>; + def SVRINTM_X4 : SInst<"svrintm[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frintm_x4", [IsStreaming], []>; + + def SVRINTN_X2 : SInst<"svrintn[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frintn_x2", [IsStreaming], []>; + def SVRINTN_X4 : SInst<"svrintn[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frintn_x4", [IsStreaming], []>; + + def SVRINTP_X2 : SInst<"svrintp[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frintp_x2", [IsStreaming], []>; + def SVRINTP_X4 : SInst<"svrintp[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frintp_x4", [IsStreaming], []>; +} + +let TargetGuard = "sme2" in { def SVSCLAMP_X2 : SInst<"svclamp[_single_{d}_x2]", "22dd", "csil", MergeNone, "aarch64_sve_sclamp_single_x2", [IsStreaming], []>; def SVUCLAMP_X2 : SInst<"svclamp[_single_{d}_x2]", "22dd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp_single_x2", [IsStreaming], []>; def SVFCLAMP_X2 : SInst<"svclamp[_single_{d}_x2]", "22dd", "hfd", MergeNone, "aarch64_sve_fclamp_single_x2", [IsStreaming], []>; diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 1b02087425b7..2b93ddf03349 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -5308,7 +5308,8 @@ def rewrite_objc : Flag<["-"], "rewrite-objc">, Flags<[NoXarchOption]>, def rewrite_legacy_objc : Flag<["-"], "rewrite-legacy-objc">, Flags<[NoXarchOption]>, HelpText<"Rewrite Legacy Objective-C source to C++">; -def rdynamic : Flag<["-"], "rdynamic">, Group<Link_Group>; +def rdynamic : Flag<["-"], "rdynamic">, Group<Link_Group>, + Visibility<[ClangOption, FlangOption]>; def resource_dir : Separate<["-"], "resource-dir">, Flags<[NoXarchOption, HelpHidden]>, Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>, @@ -6999,6 +7000,8 @@ def msign_return_address_key_EQ : Joined<["-"], "msign-return-address-key=">, Values<"a_key,b_key">; def mbranch_target_enforce : Flag<["-"], "mbranch-target-enforce">, MarshallingInfoFlag<LangOpts<"BranchTargetEnforcement">>; +def mbranch_protection_pauth_lr : Flag<["-"], "mbranch-protection-pauth-lr">, + MarshallingInfoFlag<LangOpts<"BranchProtectionPAuthLR">>; def fno_dllexport_inlines : Flag<["-"], "fno-dllexport-inlines">, MarshallingInfoNegativeFlag<LangOpts<"DllExportInlines">>; def cfguard_no_checks : Flag<["-"], "cfguard-no-checks">, diff --git a/clang/lib/Analysis/FlowSensitive/RecordOps.cpp b/clang/lib/Analysis/FlowSensitive/RecordOps.cpp index a326826db239..da4dd6dc0785 100644 --- a/clang/lib/Analysis/FlowSensitive/RecordOps.cpp +++ b/clang/lib/Analysis/FlowSensitive/RecordOps.cpp @@ -66,19 +66,8 @@ void clang::dataflow::copyRecord(RecordStorageLocation &Src, } } - RecordValue *SrcVal = Env.get<RecordValue>(Src); - RecordValue *DstVal = Env.get<RecordValue>(Dst); - - DstVal = &Env.create<RecordValue>(Dst); + RecordValue *DstVal = &Env.create<RecordValue>(Dst); Env.setValue(Dst, *DstVal); - - if (SrcVal == nullptr) - return; - - for (const auto &[Name, Value] : SrcVal->properties()) { - if (Value != nullptr) - DstVal->setProperty(Name, *Value); - } } bool clang::dataflow::recordsEqual(const RecordStorageLocation &Loc1, @@ -125,25 +114,5 @@ bool clang::dataflow::recordsEqual(const RecordStorageLocation &Loc1, } } - llvm::StringMap<Value *> Props1, Props2; - - if (RecordValue *Val1 = Env1.get<RecordValue>(Loc1)) - for (const auto &[Name, Value] : Val1->properties()) - Props1[Name] = Value; - if (RecordValue *Val2 = Env2.get<RecordValue>(Loc2)) - for (const auto &[Name, Value] : Val2->properties()) - Props2[Name] = Value; - - if (Props1.size() != Props2.size()) - return false; - - for (const auto &[Name, Value] : Props1) { - auto It = Props2.find(Name); - if (It == Props2.end()) - return false; - if (Value != It->second) - return false; - } - return true; } diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp index def16c032c86..3ee39133fcee 100644 --- a/clang/lib/Basic/Targets/AArch64.cpp +++ b/clang/lib/Basic/Targets/AArch64.cpp @@ -225,6 +225,7 @@ bool AArch64TargetInfo::validateBranchProtection(StringRef Spec, StringRef, BPI.SignKey = LangOptions::SignReturnAddressKeyKind::BKey; BPI.BranchTargetEnforcement = PBP.BranchTargetEnforcement; + BPI.BranchProtectionPAuthLR = PBP.BranchProtectionPAuthLR; return true; } diff --git a/clang/lib/Basic/Targets/ARM.cpp b/clang/lib/Basic/Targets/ARM.cpp index ce7e4d4639ce..6e1842fc64e5 100644 --- a/clang/lib/Basic/Targets/ARM.cpp +++ b/clang/lib/Basic/Targets/ARM.cpp @@ -419,6 +419,7 @@ bool ARMTargetInfo::validateBranchProtection(StringRef Spec, StringRef Arch, BPI.SignKey = LangOptions::SignReturnAddressKeyKind::AKey; BPI.BranchTargetEnforcement = PBP.BranchTargetEnforcement; + BPI.BranchProtectionPAuthLR = PBP.BranchProtectionPAuthLR; return true; } diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index a29304c81928..5081062da286 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -10318,6 +10318,30 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, return nullptr; } +static void swapCommutativeSMEOperands(unsigned BuiltinID, + SmallVectorImpl<Value *> &Ops) { + unsigned MultiVec; + switch (BuiltinID) { + default: + return; + case SME::BI__builtin_sme_svsumla_za32_s8_vg4x1: + MultiVec = 1; + break; + case SME::BI__builtin_sme_svsumla_za32_s8_vg4x2: + case SME::BI__builtin_sme_svsudot_za32_s8_vg1x2: + MultiVec = 2; + break; + case SME::BI__builtin_sme_svsudot_za32_s8_vg1x4: + case SME::BI__builtin_sme_svsumla_za32_s8_vg4x4: + MultiVec = 4; + break; + } + + if (MultiVec > 0) + for (unsigned I = 0; I < MultiVec; ++I) + std::swap(Ops[I + 1], Ops[I + 1 + MultiVec]); +} + Value *CodeGenFunction::EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E) { auto *Builtin = findARMVectorIntrinsicInMap(AArch64SMEIntrinsicMap, BuiltinID, @@ -10340,6 +10364,9 @@ Value *CodeGenFunction::EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, BuiltinID == SME::BI__builtin_sme_svstr_za) return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic); + // Handle builtins which require their multi-vector operands to be swapped + swapCommutativeSMEOperands(BuiltinID, Ops); + // Should not happen! if (Builtin->LLVMIntrinsic == 0) return nullptr; diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index b2e173d0d694..d78f2594a237 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -1106,6 +1106,9 @@ void CodeGenModule::Release() { if (LangOpts.BranchTargetEnforcement) getModule().addModuleFlag(llvm::Module::Min, "branch-target-enforcement", 1); + if (LangOpts.BranchProtectionPAuthLR) + getModule().addModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr", + 1); if (LangOpts.hasSignReturnAddress()) getModule().addModuleFlag(llvm::Module::Min, "sign-return-address", 1); if (LangOpts.isSignReturnAddressScopeAll()) diff --git a/clang/lib/CodeGen/Targets/AArch64.cpp b/clang/lib/CodeGen/Targets/AArch64.cpp index be5145daa00b..7102d190fe00 100644 --- a/clang/lib/CodeGen/Targets/AArch64.cpp +++ b/clang/lib/CodeGen/Targets/AArch64.cpp @@ -136,6 +136,8 @@ public: Fn->addFnAttr("branch-target-enforcement", BPI.BranchTargetEnforcement ? "true" : "false"); + Fn->addFnAttr("branch-protection-pauth-lr", + BPI.BranchProtectionPAuthLR ? "true" : "false"); } bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF, diff --git a/clang/lib/Driver/ToolChains/AIX.cpp b/clang/lib/Driver/ToolChains/AIX.cpp index f9670ea6f251..e6126ff62db3 100644 --- a/clang/lib/Driver/ToolChains/AIX.cpp +++ b/clang/lib/Driver/ToolChains/AIX.cpp @@ -328,6 +328,12 @@ void aix::Linker::ConstructJob(Compilation &C, const JobAction &JA, } } + if (D.IsFlangMode()) { + addFortranRuntimeLibraryPath(ToolChain, Args, CmdArgs); + addFortranRuntimeLibs(ToolChain, Args, CmdArgs); + CmdArgs.push_back("-lm"); + CmdArgs.push_back("-lpthread"); + } const char *Exec = Args.MakeArgString(ToolChain.GetLinkerPath()); C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(), Exec, CmdArgs, Inputs, Output)); diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index de9fd5eaa1e0..4783affd3220 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -1497,7 +1497,7 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args, << Triple.getArchName(); StringRef Scope, Key; - bool IndirectBranches; + bool IndirectBranches, BranchProtectionPAuthLR; if (A->getOption().matches(options::OPT_msign_return_address_EQ)) { Scope = A->getValue(); @@ -1506,6 +1506,7 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args, << A->getSpelling() << Scope; Key = "a_key"; IndirectBranches = false; + BranchProtectionPAuthLR = false; } else { StringRef DiagMsg; llvm::ARM::ParsedBranchProtection PBP; @@ -1517,6 +1518,7 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args, << "b-key" << A->getAsString(Args); Scope = PBP.Scope; Key = PBP.Key; + BranchProtectionPAuthLR = PBP.BranchProtectionPAuthLR; IndirectBranches = PBP.BranchTargetEnforcement; } @@ -1525,6 +1527,9 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args, if (!Scope.equals("none")) CmdArgs.push_back( Args.MakeArgString(Twine("-msign-return-address-key=") + Key)); + if (BranchProtectionPAuthLR) + CmdArgs.push_back( + Args.MakeArgString(Twine("-mbranch-protection-pauth-lr"))); if (IndirectBranches) CmdArgs.push_back("-mbranch-target-enforce"); } diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 4f4bdac793be..6eb0ed8f3fed 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1174,8 +1174,9 @@ static void addFortranMain(const ToolChain &TC, const ArgList &Args, // The --whole-archive option needs to be part of the link line to make // sure that the main() function from Fortran_main.a is pulled in by the // linker. However, it shouldn't be used if it's already active. - // TODO: Find an equivalent of `--whole-archive` for Darwin. - if (!isWholeArchivePresent(Args) && !TC.getTriple().isMacOSX()) { + // TODO: Find an equivalent of `--whole-archive` for Darwin and AIX. + if (!isWholeArchivePresent(Args) && !TC.getTriple().isMacOSX() && + !TC.getTriple().isOSAIX()) { CmdArgs.push_back("--whole-archive"); CmdArgs.push_back("-lFortran_main"); CmdArgs.push_back("--no-whole-archive"); diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp index 209fb0434208..d989707d5575 100644 --- a/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/clang/lib/Serialization/ASTReaderDecl.cpp @@ -584,7 +584,18 @@ void ASTDeclReader::Visit(Decl *D) { void ASTDeclReader::VisitDecl(Decl *D) { BitsUnpacker DeclBits(Record.readInt()); + auto ModuleOwnership = + (Decl::ModuleOwnershipKind)DeclBits.getNextBits(/*Width=*/3); + D->setReferenced(DeclBits.getNextBit()); + D->Used = DeclBits.getNextBit(); + IsDeclMarkedUsed |= D->Used; + D->setAccess((AccessSpecifier)DeclBits.getNextBits(/*Width=*/2)); + D->setImplicit(DeclBits.getNextBit()); bool HasStandaloneLexicalDC = DeclBits.getNextBit(); + bool HasAttrs = DeclBits.getNextBit(); + D->setTopLevelDeclInObjCContainer(DeclBits.getNextBit()); + D->InvalidDecl = DeclBits.getNextBit(); + D->FromASTFile = true; if (D->isTemplateParameter() || D->isTemplateParameterPack() || isa<ParmVarDecl, ObjCTypeParamDecl>(D)) { @@ -623,20 +634,6 @@ void ASTDeclReader::VisitDecl(Decl *D) { } D->setLocation(ThisDeclLoc); - D->InvalidDecl = DeclBits.getNextBit(); - bool HasAttrs = DeclBits.getNextBit(); - D->setImplicit(DeclBits.getNextBit()); - D->Used = DeclBits.getNextBit(); - IsDeclMarkedUsed |= D->Used; - D->setReferenced(DeclBits.getNextBit()); - D->setTopLevelDeclInObjCContainer(DeclBits.getNextBit()); - D->setAccess((AccessSpecifier)DeclBits.getNextBits(/*Width=*/2)); - D->FromASTFile = true; - auto ModuleOwnership = - (Decl::ModuleOwnershipKind)DeclBits.getNextBits(/*Width=*/3); - bool ModulePrivate = - (ModuleOwnership == Decl::ModuleOwnershipKind::ModulePrivate); - if (HasAttrs) { AttrVec Attrs; Record.readAttributes(Attrs); @@ -647,8 +644,9 @@ void ASTDeclReader::VisitDecl(Decl *D) { // Determine whether this declaration is part of a (sub)module. If so, it // may not yet be visible. + bool ModulePrivate = + (ModuleOwnership == Decl::ModuleOwnershipKind::ModulePrivate); if (unsigned SubmoduleID = readSubmoduleID()) { - switch (ModuleOwnership) { case Decl::ModuleOwnershipKind::Visible: ModuleOwnership = Decl::ModuleOwnershipKind::VisibleWhenImported; @@ -1065,9 +1063,11 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) { // after everything else is read. BitsUnpacker FunctionDeclBits(Record.readInt()); + FD->setCachedLinkage((Linkage)FunctionDeclBits.getNextBits(/*Width=*/3)); FD->setStorageClass((StorageClass)FunctionDeclBits.getNextBits(/*Width=*/3)); FD->setInlineSpecified(FunctionDeclBits.getNextBit()); FD->setImplicitlyInline(FunctionDeclBits.getNextBit()); + FD->setHasSkippedBody(FunctionDeclBits.getNextBit()); FD->setVirtualAsWritten(FunctionDeclBits.getNextBit()); // We defer calling `FunctionDecl::setPure()` here as for methods of // `CXXTemplateSpecializationDecl`s, we may not have connected up the @@ -1081,16 +1081,14 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) { FD->setDefaulted(FunctionDeclBits.getNextBit()); FD->setExplicitlyDefaulted(FunctionDeclBits.getNextBit()); FD->setIneligibleOrNotSelected(FunctionDeclBits.getNextBit()); - FD->setHasImplicitReturnZero(FunctionDeclBits.getNextBit()); FD->setConstexprKind( (ConstexprSpecKind)FunctionDeclBits.getNextBits(/*Width=*/2)); - FD->setUsesSEHTry(FunctionDeclBits.getNextBit()); - FD->setHasSkippedBody(FunctionDeclBits.getNextBit()); + FD->setHasImplicitReturnZero(FunctionDeclBits.getNextBit()); FD->setIsMultiVersion(FunctionDeclBits.getNextBit()); FD->setLateTemplateParsed(FunctionDeclBits.getNextBit()); FD->setFriendConstraintRefersToEnclosingTemplate( FunctionDeclBits.getNextBit()); - FD->setCachedLinkage((Linkage)FunctionDeclBits.getNextBits(/*Width=*/3)); + FD->setUsesSEHTry(FunctionDeclBits.getNextBit()); FD->EndRangeLoc = readSourceLocation(); if (FD->isExplicitlyDefaulted()) @@ -1597,6 +1595,8 @@ ASTDeclReader::RedeclarableResult ASTDeclReader::VisitVarDeclImpl(VarDecl *VD) { VisitDeclaratorDecl(VD); BitsUnpacker VarDeclBits(Record.readInt()); + auto VarLinkage = Linkage(VarDeclBits.getNextBits(/*Width=*/3)); + bool DefGeneratedInModule = VarDeclBits.getNextBit(); VD->VarDeclBits.SClass = (StorageClass)VarDeclBits.getNextBits(/*Width=*/3); VD->VarDeclBits.TSCSpec = VarDeclBits.getNextBits(/*Width=*/2); VD->VarDeclBits.InitStyle = VarDeclBits.getNextBits(/*Width=*/2); @@ -1608,17 +1608,20 @@ ASTDeclReader::RedeclarableResult ASTDeclReader::VisitVarDeclImpl(VarDecl *VD) { VD->NonParmVarDeclBits.ExceptionVar = VarDeclBits.getNextBit(); VD->NonParmVarDeclBits.NRVOVariable = VarDeclBits.getNextBit(); VD->NonParmVarDeclBits.CXXForRangeDecl = VarDeclBits.getNextBit(); - VD->NonParmVarDeclBits.ObjCForDecl = VarDeclBits.getNextBit(); + VD->NonParmVarDeclBits.IsInline = VarDeclBits.getNextBit(); VD->NonParmVarDeclBits.IsInlineSpecified = VarDeclBits.getNextBit(); VD->NonParmVarDeclBits.IsConstexpr = VarDeclBits.getNextBit(); VD->NonParmVarDeclBits.IsInitCapture = VarDeclBits.getNextBit(); VD->NonParmVarDeclBits.PreviousDeclInSameBlockScope = VarDeclBits.getNextBit(); - VD->NonParmVarDeclBits.ImplicitParamKind = - VarDeclBits.getNextBits(/*Width*/ 3); + VD->NonParmVarDeclBits.EscapingByref = VarDeclBits.getNextBit(); HasDeducedType = VarDeclBits.getNextBit(); + VD->NonParmVarDeclBits.ImplicitParamKind = + VarDeclBits.getNextBits(/*Width*/ 3); + + VD->NonParmVarDeclBits.ObjCForDecl = VarDeclBits.getNextBit(); } // If this variable has a deduced type, defer reading that type until we are @@ -1630,7 +1633,6 @@ ASTDeclReader::RedeclarableResult ASTDeclReader::VisitVarDeclImpl(VarDecl *VD) { VD->setType(Reader.GetType(DeferredTypeID)); DeferredTypeID = 0; - auto VarLinkage = Linkage(VarDeclBits.getNextBits(/*Width=*/3)); VD->setCachedLinkage(VarLinkage); // Reconstruct the one piece of the IdentifierNamespace that we need. @@ -1638,7 +1640,7 @@ ASTDeclReader::RedeclarableResult ASTDeclReader::VisitVarDeclImpl(VarDecl *VD) { VD->getLexicalDeclContext()->isFunctionOrMethod()) VD->setLocalExternDecl(); - if (VarDeclBits.getNextBit()) { + if (DefGeneratedInModule) { Reader.DefinitionSource[VD] = Loc.F->Kind == ModuleKind::MK_MainFile || Reader.getContext().getLangOpts().BuildingPCHWithObjectFile; diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp index d2424bffc228..cf37ffe4c38b 100644 --- a/clang/lib/Serialization/ASTReaderStmt.cpp +++ b/clang/lib/Serialization/ASTReaderStmt.cpp @@ -152,10 +152,9 @@ void ASTStmtReader::VisitNullStmt(NullStmt *S) { void ASTStmtReader::VisitCompoundStmt(CompoundStmt *S) { VisitStmt(S); - CurrentUnpackingBits.emplace(Record.readInt()); SmallVector<Stmt *, 16> Stmts; - unsigned NumStmts = CurrentUnpackingBits->getNextBits(/*Width=*/20); - unsigned HasFPFeatures = CurrentUnpackingBits->getNextBit(); + unsigned NumStmts = Record.readInt(); + unsigned HasFPFeatures = Record.readInt(); assert(S->hasStoredFPFeatures() == HasFPFeatures); while (NumStmts--) Stmts.push_back(Record.readSubStmt()); @@ -600,20 +599,21 @@ void ASTStmtReader::VisitPredefinedExpr(PredefinedExpr *E) { void ASTStmtReader::VisitDeclRefExpr(DeclRefExpr *E) { VisitExpr(E); - E->DeclRefExprBits.HasQualifier = CurrentUnpackingBits->getNextBit(); - E->DeclRefExprBits.HasFoundDecl = CurrentUnpackingBits->getNextBit(); - E->DeclRefExprBits.HasTemplateKWAndArgsInfo = - CurrentUnpackingBits->getNextBit(); + CurrentUnpackingBits.emplace(Record.readInt()); E->DeclRefExprBits.HadMultipleCandidates = CurrentUnpackingBits->getNextBit(); E->DeclRefExprBits.RefersToEnclosingVariableOrCapture = CurrentUnpackingBits->getNextBit(); E->DeclRefExprBits.NonOdrUseReason = CurrentUnpackingBits->getNextBits(/*Width=*/2); E->DeclRefExprBits.IsImmediateEscalating = CurrentUnpackingBits->getNextBit(); + E->DeclRefExprBits.HasFoundDecl = CurrentUnpackingBits->getNextBit(); + E->DeclRefExprBits.HasQualifier = CurrentUnpackingBits->getNextBit(); + E->DeclRefExprBits.HasTemplateKWAndArgsInfo = + CurrentUnpackingBits->getNextBit(); E->DeclRefExprBits.CapturedByCopyInLambdaWithExplicitObjectParameter = false; unsigned NumTemplateArgs = 0; if (E->hasTemplateKWAndArgsInfo()) - NumTemplateArgs = CurrentUnpackingBits->getNextBits(/*Width=*/12); + NumTemplateArgs = Record.readInt(); if (E->hasQualifier()) new (E->getTrailingObjects<NestedNameSpecifierLoc>()) @@ -1013,10 +1013,11 @@ void ASTStmtReader::VisitOMPIteratorExpr(OMPIteratorExpr *E) { void ASTStmtReader::VisitCallExpr(CallExpr *E) { VisitExpr(E); - unsigned NumArgs = CurrentUnpackingBits->getNextBits(/*Width=*/13); - bool HasFPFeatures = CurrentUnpackingBits->getNextBit(); + unsigned NumArgs = Record.readInt(); + CurrentUnpackingBits.emplace(Record.readInt()); E->setADLCallKind( static_cast<CallExpr::ADLCallKind>(CurrentUnpackingBits->getNextBit())); + bool HasFPFeatures = CurrentUnpackingBits->getNextBit(); assert((NumArgs == E->getNumArgs()) && "Wrong NumArgs!"); E->setRParenLoc(readSourceLocation()); E->setCallee(Record.readSubExpr()); @@ -1035,10 +1036,11 @@ void ASTStmtReader::VisitCXXMemberCallExpr(CXXMemberCallExpr *E) { void ASTStmtReader::VisitMemberExpr(MemberExpr *E) { VisitExpr(E); + CurrentUnpackingBits.emplace(Record.readInt()); bool HasQualifier = CurrentUnpackingBits->getNextBit(); bool HasFoundDecl = CurrentUnpackingBits->getNextBit(); bool HasTemplateInfo = CurrentUnpackingBits->getNextBit(); - unsigned NumTemplateArgs = CurrentUnpackingBits->getNextBits(/*Width=*/12); + unsigned NumTemplateArgs = Record.readInt(); E->Base = Record.readSubExpr(); E->MemberDecl = Record.readDeclAs<ValueDecl>(); @@ -1103,10 +1105,14 @@ void ASTStmtReader::VisitCastExpr(CastExpr *E) { VisitExpr(E); unsigned NumBaseSpecs = Record.readInt(); assert(NumBaseSpecs == E->path_size()); + + CurrentUnpackingBits.emplace(Record.readInt()); + E->setCastKind((CastKind)CurrentUnpackingBits->getNextBits(/*Width=*/7)); unsigned HasFPFeatures = CurrentUnpackingBits->getNextBit(); assert(E->hasStoredFPFeatures() == HasFPFeatures); + E->setSubExpr(Record.readSubExpr()); - E->setCastKind((CastKind)CurrentUnpackingBits->getNextBits(/*Width=*/7)); + CastExpr::path_iterator BaseI = E->path_begin(); while (NumBaseSpecs--) { auto *BaseSpec = new (Record.getContext()) CXXBaseSpecifier; @@ -1119,12 +1125,12 @@ void ASTStmtReader::VisitCastExpr(CastExpr *E) { } void ASTStmtReader::VisitBinaryOperator(BinaryOperator *E) { - VisitExpr(E); - bool hasFP_Features = CurrentUnpackingBits->getNextBit(); - E->setHasStoredFPFeatures(hasFP_Features); + CurrentUnpackingBits.emplace(Record.readInt()); E->setOpcode( (BinaryOperator::Opcode)CurrentUnpackingBits->getNextBits(/*Width=*/6)); + bool hasFP_Features = CurrentUnpackingBits->getNextBit(); + E->setHasStoredFPFeatures(hasFP_Features); E->setLHS(Record.readSubExpr()); E->setRHS(Record.readSubExpr()); E->setOperatorLoc(readSourceLocation()); @@ -1700,8 +1706,7 @@ void ASTStmtReader::VisitMSDependentExistsStmt(MSDependentExistsStmt *S) { void ASTStmtReader::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) { VisitCallExpr(E); - E->CXXOperatorCallExprBits.OperatorKind = - CurrentUnpackingBits->getNextBits(/*Width=*/6); + E->CXXOperatorCallExprBits.OperatorKind = Record.readInt(); E->Range = Record.readSourceRange(); } @@ -1976,8 +1981,9 @@ void ASTStmtReader::VisitCXXDependentScopeMemberExpr( CXXDependentScopeMemberExpr *E) { VisitExpr(E); + unsigned NumTemplateArgs = Record.readInt(); + CurrentUnpackingBits.emplace(Record.readInt()); bool HasTemplateKWAndArgsInfo = CurrentUnpackingBits->getNextBit(); - unsigned NumTemplateArgs = CurrentUnpackingBits->getNextBits(/*Width=*/16); bool HasFirstQualifierFoundInScope = CurrentUnpackingBits->getNextBit(); assert((HasTemplateKWAndArgsInfo == E->hasTemplateKWAndArgsInfo()) && @@ -2044,15 +2050,15 @@ ASTStmtReader::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) { void ASTStmtReader::VisitOverloadExpr(OverloadExpr *E) { VisitExpr(E); + unsigned NumResults = Record.readInt(); CurrentUnpackingBits.emplace(Record.readInt()); - unsigned NumResults = CurrentUnpackingBits->getNextBits(/*Width=*/12); bool HasTemplateKWAndArgsInfo = CurrentUnpackingBits->getNextBit(); assert((E->getNumDecls() == NumResults) && "Wrong NumResults!"); assert((E->hasTemplateKWAndArgsInfo() == HasTemplateKWAndArgsInfo) && "Wrong HasTemplateKWAndArgsInfo!"); if (HasTemplateKWAndArgsInfo) { - unsigned NumTemplateArgs = CurrentUnpackingBits->getNextBits(/*Width=*/12); + unsigned NumTemplateArgs = Record.readInt(); ReadTemplateKWAndArgsInfo(*E->getTrailingASTTemplateKWAndArgsInfo(), E->getTrailingTemplateArgumentLoc(), NumTemplateArgs); @@ -2869,9 +2875,8 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; case STMT_COMPOUND: { - BitsUnpacker StmtCompoundBits(Record[ASTStmtReader::NumStmtFields]); - unsigned NumStmts = StmtCompoundBits.getNextBits(/*Width=*/20); - bool HasFPFeatures = StmtCompoundBits.getNextBit(); + unsigned NumStmts = Record[ASTStmtReader::NumStmtFields]; + bool HasFPFeatures = Record[ASTStmtReader::NumStmtFields + 1]; S = CompoundStmt::CreateEmpty(Context, NumStmts, HasFPFeatures); break; } @@ -2981,14 +2986,13 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; case EXPR_DECL_REF: { - BitsUnpacker DeclRefExprBits(Record[ASTStmtReader::NumStmtFields]); - DeclRefExprBits.advance(ASTStmtReader::NumExprBits); - bool HasQualifier = DeclRefExprBits.getNextBit(); + BitsUnpacker DeclRefExprBits(Record[ASTStmtReader::NumExprFields]); + DeclRefExprBits.advance(5); bool HasFoundDecl = DeclRefExprBits.getNextBit(); + bool HasQualifier = DeclRefExprBits.getNextBit(); bool HasTemplateKWAndArgsInfo = DeclRefExprBits.getNextBit(); - DeclRefExprBits.advance(5); unsigned NumTemplateArgs = HasTemplateKWAndArgsInfo - ? DeclRefExprBits.getNextBits(/*Width=*/12) + ? Record[ASTStmtReader::NumExprFields + 1] : 0; S = DeclRefExpr::CreateEmpty(Context, HasQualifier, HasFoundDecl, HasTemplateKWAndArgsInfo, NumTemplateArgs); @@ -3074,9 +3078,9 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; case EXPR_CALL: { - BitsUnpacker CallExprBits(Record[ASTStmtReader::NumStmtFields]); - CallExprBits.advance(ASTStmtReader::NumExprBits); - auto NumArgs = CallExprBits.getNextBits(/*Width=*/13); + auto NumArgs = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CallExprBits.advance(1); auto HasFPFeatures = CallExprBits.getNextBit(); S = CallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); break; @@ -3088,28 +3092,27 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; case EXPR_MEMBER: { - BitsUnpacker ExprMemberBits(Record[ASTStmtReader::NumStmtFields]); - ExprMemberBits.advance(ASTStmtReader::NumExprBits); + BitsUnpacker ExprMemberBits(Record[ASTStmtReader::NumExprFields]); bool HasQualifier = ExprMemberBits.getNextBit(); bool HasFoundDecl = ExprMemberBits.getNextBit(); bool HasTemplateInfo = ExprMemberBits.getNextBit(); - unsigned NumTemplateArgs = ExprMemberBits.getNextBits(/*Width=*/12); + unsigned NumTemplateArgs = Record[ASTStmtReader::NumExprFields + 1]; S = MemberExpr::CreateEmpty(Context, HasQualifier, HasFoundDecl, HasTemplateInfo, NumTemplateArgs); break; } case EXPR_BINARY_OPERATOR: { - BitsUnpacker BinaryOperatorBits(Record[ASTStmtReader::NumStmtFields]); - BinaryOperatorBits.advance(ASTStmtReader::NumExprBits); + BitsUnpacker BinaryOperatorBits(Record[ASTStmtReader::NumExprFields]); + BinaryOperatorBits.advance(/*Size of opcode*/ 6); bool HasFPFeatures = BinaryOperatorBits.getNextBit(); S = BinaryOperator::CreateEmpty(Context, HasFPFeatures); break; } case EXPR_COMPOUND_ASSIGN_OPERATOR: { - BitsUnpacker BinaryOperatorBits(Record[ASTStmtReader::NumStmtFields]); - BinaryOperatorBits.advance(ASTStmtReader::NumExprBits); + BitsUnpacker BinaryOperatorBits(Record[ASTStmtReader::NumExprFields]); + BinaryOperatorBits.advance(/*Size of opcode*/ 6); bool HasFPFeatures = BinaryOperatorBits.getNextBit(); S = CompoundAssignOperator::CreateEmpty(Context, HasFPFeatures); break; @@ -3124,18 +3127,18 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; case EXPR_IMPLICIT_CAST: { - BitsUnpacker CastExprBits(Record[ASTStmtReader::NumStmtFields]); - CastExprBits.advance(ASTStmtReader::NumExprBits); unsigned PathSize = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CastExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CastExprBits.advance(7); bool HasFPFeatures = CastExprBits.getNextBit(); S = ImplicitCastExpr::CreateEmpty(Context, PathSize, HasFPFeatures); break; } case EXPR_CSTYLE_CAST: { - BitsUnpacker CastExprBits(Record[ASTStmtReader::NumStmtFields]); - CastExprBits.advance(ASTStmtReader::NumExprBits); unsigned PathSize = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CastExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CastExprBits.advance(7); bool HasFPFeatures = CastExprBits.getNextBit(); S = CStyleCastExpr::CreateEmpty(Context, PathSize, HasFPFeatures); break; @@ -3834,9 +3837,9 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { } case EXPR_CXX_OPERATOR_CALL: { - BitsUnpacker CallExprBits(Record[ASTStmtReader::NumStmtFields]); - CallExprBits.advance(ASTStmtReader::NumExprBits); - auto NumArgs = CallExprBits.getNextBits(/*Width=*/13); + auto NumArgs = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CallExprBits.advance(1); auto HasFPFeatures = CallExprBits.getNextBit(); S = CXXOperatorCallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); @@ -3844,9 +3847,9 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { } case EXPR_CXX_MEMBER_CALL: { - BitsUnpacker CallExprBits(Record[ASTStmtReader::NumStmtFields]); - CallExprBits.advance(ASTStmtReader::NumExprBits); - auto NumArgs = CallExprBits.getNextBits(/*Width=*/13); + auto NumArgs = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CallExprBits.advance(1); auto HasFPFeatures = CallExprBits.getNextBit(); S = CXXMemberCallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); @@ -3874,9 +3877,9 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; case EXPR_CXX_STATIC_CAST: { - BitsUnpacker CastExprBits(Record[ASTStmtReader::NumStmtFields]); - CastExprBits.advance(ASTStmtReader::NumExprBits); unsigned PathSize = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CastExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CastExprBits.advance(7); bool HasFPFeatures = CastExprBits.getNextBit(); S = CXXStaticCastExpr::CreateEmpty(Context, PathSize, HasFPFeatures); break; @@ -3903,9 +3906,9 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; case EXPR_CXX_FUNCTIONAL_CAST: { - BitsUnpacker CastExprBits(Record[ASTStmtReader::NumStmtFields]); - CastExprBits.advance(ASTStmtReader::NumExprBits); unsigned PathSize = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CastExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CastExprBits.advance(7); bool HasFPFeatures = CastExprBits.getNextBit(); S = CXXFunctionalCastExpr::CreateEmpty(Context, PathSize, HasFPFeatures); break; @@ -3921,9 +3924,9 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { } case EXPR_USER_DEFINED_LITERAL: { - BitsUnpacker CallExprBits(Record[ASTStmtReader::NumStmtFields]); - CallExprBits.advance(ASTStmtReader::NumExprBits); - auto NumArgs = CallExprBits.getNextBits(/*Width=*/13); + auto NumArgs = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CallExprBits.advance(1); auto HasFPFeatures = CallExprBits.getNextBit(); S = UserDefinedLiteral::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); @@ -4015,12 +4018,11 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; case EXPR_CXX_DEPENDENT_SCOPE_MEMBER: { + unsigned NumTemplateArgs = Record[ASTStmtReader::NumExprFields]; BitsUnpacker DependentScopeMemberBits( - Record[ASTStmtReader::NumStmtFields]); - DependentScopeMemberBits.advance(ASTStmtReader::NumExprBits); + Record[ASTStmtReader::NumExprFields + 1]); bool HasTemplateKWAndArgsInfo = DependentScopeMemberBits.getNextBit(); - unsigned NumTemplateArgs = - DependentScopeMemberBits.getNextBits(/*Width=*/16); + bool HasFirstQualifierFoundInScope = DependentScopeMemberBits.getNextBit(); S = CXXDependentScopeMemberExpr::CreateEmpty( @@ -4049,11 +4051,11 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; case EXPR_CXX_UNRESOLVED_MEMBER: { - BitsUnpacker OverloadExprBits(Record[ASTStmtReader::NumExprFields]); - auto NumResults = OverloadExprBits.getNextBits(/*Width=*/12); + auto NumResults = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker OverloadExprBits(Record[ASTStmtReader::NumExprFields + 1]); auto HasTemplateKWAndArgsInfo = OverloadExprBits.getNextBit(); auto NumTemplateArgs = HasTemplateKWAndArgsInfo - ? OverloadExprBits.getNextBits(/*Width=*/12) + ? Record[ASTStmtReader::NumExprFields + 2] : 0; S = UnresolvedMemberExpr::CreateEmpty( Context, NumResults, HasTemplateKWAndArgsInfo, NumTemplateArgs); @@ -4061,11 +4063,11 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { } case EXPR_CXX_UNRESOLVED_LOOKUP: { - BitsUnpacker OverloadExprBits(Record[ASTStmtReader::NumExprFields]); - auto NumResults = OverloadExprBits.getNextBits(/*Width=*/12); + auto NumResults = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker OverloadExprBits(Record[ASTStmtReader::NumExprFields + 1]); auto HasTemplateKWAndArgsInfo = OverloadExprBits.getNextBit(); auto NumTemplateArgs = HasTemplateKWAndArgsInfo - ? OverloadExprBits.getNextBits(/*Width=*/12) + ? Record[ASTStmtReader::NumExprFields + 2] : 0; S = UnresolvedLookupExpr::CreateEmpty( Context, NumResults, HasTemplateKWAndArgsInfo, NumTemplateArgs); @@ -4130,9 +4132,9 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; case EXPR_CUDA_KERNEL_CALL: { - BitsUnpacker CallExprBits(Record[ASTStmtReader::NumStmtFields]); - CallExprBits.advance(ASTStmtReader::NumExprBits); - auto NumArgs = CallExprBits.getNextBits(/*Width=*/13); + auto NumArgs = Record[ASTStmtReader::NumExprFields]; + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields + 1]); + CallExprBits.advance(1); auto HasFPFeatures = CallExprBits.getNextBit(); S = CUDAKernelCallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp index 53128133588f..2554abc682a1 100644 --- a/clang/lib/Serialization/ASTWriterDecl.cpp +++ b/clang/lib/Serialization/ASTWriterDecl.cpp @@ -321,15 +321,25 @@ void ASTDeclWriter::Visit(Decl *D) { void ASTDeclWriter::VisitDecl(Decl *D) { BitsPacker DeclBits; + + // The order matters here. It will be better to put the bit with higher + // probability to be 0 in the end of the bits. + // + // Since we're using VBR6 format to store it. + // It will be pretty effient if all the higher bits are 0. + // For example, if we need to pack 8 bits into a value and the stored value + // is 0xf0, the actual stored value will be 0b000111'110000, which takes 12 + // bits actually. However, if we changed the order to be 0x0f, then we can + // store it as 0b001111, which takes 6 bits only now. + DeclBits.addBits((uint64_t)D->getModuleOwnershipKind(), /*BitWidth=*/3); + DeclBits.addBit(D->isReferenced()); + DeclBits.addBit(D->isUsed(false)); + DeclBits.addBits(D->getAccess(), /*BitWidth=*/2); + DeclBits.addBit(D->isImplicit()); DeclBits.addBit(D->getDeclContext() != D->getLexicalDeclContext()); - DeclBits.addBit(D->isInvalidDecl()); DeclBits.addBit(D->hasAttrs()); - DeclBits.addBit(D->isImplicit()); - DeclBits.addBit(D->isUsed(false)); - DeclBits.addBit(D->isReferenced()); DeclBits.addBit(D->isTopLevelDeclInObjCContainer()); - DeclBits.addBits(D->getAccess(), /*BitWidth=*/2); - DeclBits.addBits((uint64_t)D->getModuleOwnershipKind(), /*BitWidth=*/3); + DeclBits.addBit(D->isInvalidDecl()); Record.push_back(DeclBits); Record.AddDeclRef(cast_or_null<Decl>(D->getDeclContext())); @@ -493,16 +503,13 @@ void ASTDeclWriter::VisitEnumDecl(EnumDecl *D) { Record.AddDeclRef(nullptr); } - if (D->getDeclContext() == D->getLexicalDeclContext() && - !D->hasAttrs() && - !D->isImplicit() && - !D->hasExtInfo() && + if (D->getDeclContext() == D->getLexicalDeclContext() && !D->hasAttrs() && + !D->isInvalidDecl() && !D->isImplicit() && !D->hasExtInfo() && !D->getTypedefNameForAnonDecl() && D->getFirstDecl() == D->getMostRecentDecl() && !D->isTopLevelDeclInObjCContainer() && !CXXRecordDecl::classofKind(D->getKind()) && - !D->getIntegerTypeSourceInfo() && - !D->getMemberSpecializationInfo() && + !D->getIntegerTypeSourceInfo() && !D->getMemberSpecializationInfo() && !needsAnonymousDeclarationNumber(D) && D->getDeclName().getNameKind() == DeclarationName::Identifier) AbbrevToUse = Writer.getDeclEnumAbbrev(); @@ -537,9 +544,8 @@ void ASTDeclWriter::VisitRecordDecl(RecordDecl *D) { if (!isa<CXXRecordDecl>(D)) Record.push_back(D->getODRHash()); - if (D->getDeclContext() == D->getLexicalDeclContext() && - !D->hasAttrs() && - !D->hasExtInfo() && + if (D->getDeclContext() == D->getLexicalDeclContext() && !D->hasAttrs() && + !D->isImplicit() && !D->isInvalidDecl() && !D->hasExtInfo() && !D->getTypedefNameForAnonDecl() && D->getFirstDecl() == D->getMostRecentDecl() && !D->isTopLevelDeclInObjCContainer() && @@ -663,11 +669,16 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) { Record.AddDeclarationNameLoc(D->DNLoc, D->getDeclName()); Record.push_back(D->getIdentifierNamespace()); + // The order matters here. It will be better to put the bit with higher + // probability to be 0 in the end of the bits. See the comments in VisitDecl + // for details. BitsPacker FunctionDeclBits; // FIXME: stable encoding + FunctionDeclBits.addBits(llvm::to_underlying(D->getLinkageInternal()), 3); FunctionDeclBits.addBits((uint32_t)D->getStorageClass(), /*BitWidth=*/3); FunctionDeclBits.addBit(D->isInlineSpecified()); FunctionDeclBits.addBit(D->isInlined()); + FunctionDeclBits.addBit(D->hasSkippedBody()); FunctionDeclBits.addBit(D->isVirtualAsWritten()); FunctionDeclBits.addBit(D->isPure()); FunctionDeclBits.addBit(D->hasInheritedPrototype()); @@ -678,14 +689,12 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) { FunctionDeclBits.addBit(D->isDefaulted()); FunctionDeclBits.addBit(D->isExplicitlyDefaulted()); FunctionDeclBits.addBit(D->isIneligibleOrNotSelected()); - FunctionDeclBits.addBit(D->hasImplicitReturnZero()); FunctionDeclBits.addBits((uint64_t)(D->getConstexprKind()), /*BitWidth=*/2); - FunctionDeclBits.addBit(D->usesSEHTry()); - FunctionDeclBits.addBit(D->hasSkippedBody()); + FunctionDeclBits.addBit(D->hasImplicitReturnZero()); FunctionDeclBits.addBit(D->isMultiVersion()); FunctionDeclBits.addBit(D->isLateTemplateParsed()); FunctionDeclBits.addBit(D->FriendConstraintRefersToEnclosingTemplate()); - FunctionDeclBits.addBits(llvm::to_underlying(D->getLinkageInternal()), 3); + FunctionDeclBits.addBit(D->usesSEHTry()); Record.push_back(FunctionDeclBits); Record.AddSourceLocation(D->getEndLoc()); @@ -1049,7 +1058,28 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) { VisitRedeclarable(D); VisitDeclaratorDecl(D); + // The order matters here. It will be better to put the bit with higher + // probability to be 0 in the end of the bits. See the comments in VisitDecl + // for details. BitsPacker VarDeclBits; + VarDeclBits.addBits(llvm::to_underlying(D->getLinkageInternal()), + /*BitWidth=*/3); + + bool ModulesCodegen = false; + if (Writer.WritingModule && D->getStorageDuration() == SD_Static && + !D->getDescribedVarTemplate()) { + // When building a C++20 module interface unit or a partition unit, a + // strong definition in the module interface is provided by the + // compilation of that unit, not by its users. (Inline variables are still + // emitted in module users.) + ModulesCodegen = + (Writer.WritingModule->isInterfaceOrPartition() || + (D->hasAttr<DLLExportAttr>() && + Writer.Context->getLangOpts().BuildingPCHWithObjectFile)) && + Writer.Context->GetGVALinkageForVariable(D) >= GVA_StrongExternal; + } + VarDeclBits.addBit(ModulesCodegen); + VarDeclBits.addBits(D->getStorageClass(), /*BitWidth=*/3); VarDeclBits.addBits(D->getTSCSpec(), /*BitWidth=*/2); VarDeclBits.addBits(D->getInitStyle(), /*BitWidth=*/2); @@ -1061,41 +1091,26 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) { VarDeclBits.addBit(D->isExceptionVariable()); VarDeclBits.addBit(D->isNRVOVariable()); VarDeclBits.addBit(D->isCXXForRangeDecl()); - VarDeclBits.addBit(D->isObjCForDecl()); + VarDeclBits.addBit(D->isInline()); VarDeclBits.addBit(D->isInlineSpecified()); VarDeclBits.addBit(D->isConstexpr()); VarDeclBits.addBit(D->isInitCapture()); VarDeclBits.addBit(D->isPreviousDeclInSameBlockScope()); + VarDeclBits.addBit(D->isEscapingByref()); + HasDeducedType = D->getType()->getContainedDeducedType(); + VarDeclBits.addBit(HasDeducedType); + if (const auto *IPD = dyn_cast<ImplicitParamDecl>(D)) VarDeclBits.addBits(llvm::to_underlying(IPD->getParameterKind()), /*Width=*/3); else VarDeclBits.addBits(0, /*Width=*/3); - VarDeclBits.addBit(D->isEscapingByref()); - HasDeducedType = D->getType()->getContainedDeducedType(); - VarDeclBits.addBit(HasDeducedType); - } - - VarDeclBits.addBits(llvm::to_underlying(D->getLinkageInternal()), /*BitWidth=*/3); - - bool ModulesCodegen = false; - if (Writer.WritingModule && D->getStorageDuration() == SD_Static && - !D->getDescribedVarTemplate()) { - // When building a C++20 module interface unit or a partition unit, a - // strong definition in the module interface is provided by the - // compilation of that unit, not by its users. (Inline variables are still - // emitted in module users.) - ModulesCodegen = - (Writer.WritingModule->isInterfaceOrPartition() || - (D->hasAttr<DLLExportAttr>() && - Writer.Context->getLangOpts().BuildingPCHWithObjectFile)) && - Writer.Context->GetGVALinkageForVariable(D) >= GVA_StrongExternal; + VarDeclBits.addBit(D->isObjCForDecl()); } - VarDeclBits.addBit(ModulesCodegen); Record.push_back(VarDeclBits); if (ModulesCodegen) @@ -1124,23 +1139,17 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) { Record.push_back(VarNotTemplate); } - if (D->getDeclContext() == D->getLexicalDeclContext() && - !D->hasAttrs() && + if (D->getDeclContext() == D->getLexicalDeclContext() && !D->hasAttrs() && !D->isTopLevelDeclInObjCContainer() && !needsAnonymousDeclarationNumber(D) && D->getDeclName().getNameKind() == DeclarationName::Identifier && - !D->hasExtInfo() && - D->getFirstDecl() == D->getMostRecentDecl() && - D->getKind() == Decl::Var && - !D->isInline() && - !D->isConstexpr() && - !D->isInitCapture() && - !D->isPreviousDeclInSameBlockScope() && - !D->isEscapingByref() && - !HasDeducedType && - D->getStorageDuration() != SD_Static && - !D->getDescribedVarTemplate() && - !D->getMemberSpecializationInfo()) + !D->hasExtInfo() && D->getFirstDecl() == D->getMostRecentDecl() && + D->getKind() == Decl::Var && !D->isInline() && !D->isConstexpr() && + !D->isInitCapture() && !D->isPreviousDeclInSameBlockScope() && + !D->isEscapingByref() && !HasDeducedType && + D->getStorageDuration() != SD_Static && !D->getDescribedVarTemplate() && + !D->getMemberSpecializationInfo() && !D->isObjCForDecl() && + !isa<ImplicitParamDecl>(D) && !D->isEscapingByref()) AbbrevToUse = Writer.getDeclVarAbbrev(); Code = serialization::DECL_VAR; @@ -1176,7 +1185,8 @@ void ASTDeclWriter::VisitParmVarDecl(ParmVarDecl *D) { // we dynamically check for the properties that we optimize for, but don't // know are true of all PARM_VAR_DECLs. if (D->getDeclContext() == D->getLexicalDeclContext() && !D->hasAttrs() && - !D->hasExtInfo() && D->getStorageClass() == 0 && + !D->hasExtInfo() && D->getStorageClass() == 0 && !D->isInvalidDecl() && + !D->isTopLevelDeclInObjCContainer() && D->getInitStyle() == VarDecl::CInit && // Can params have anything else? D->getInit() == nullptr) // No default expr. AbbrevToUse = Writer.getDeclParmVarAbbrev(); @@ -1810,7 +1820,7 @@ void ASTDeclWriter::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) { if (!TC && !OwnsDefaultArg && D->getDeclContext() == D->getLexicalDeclContext() && !D->isInvalidDecl() && !D->hasAttrs() && - !D->isTopLevelDeclInObjCContainer() && + !D->isTopLevelDeclInObjCContainer() && !D->isImplicit() && D->getDeclName().getNameKind() == DeclarationName::Identifier) AbbrevToUse = Writer.getDeclTemplateTypeParmAbbrev(); @@ -2094,10 +2104,14 @@ getFunctionDeclAbbrev(serialization::DeclCode Code) { } // Decl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 12)); // Packed DeclBits: HasStandaloneLexicalDC, - // isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, - // AccessSpecifier, ModuleOwnershipKind + 8)); // Packed DeclBits: ModuleOwnershipKind, + // isUsed, isReferenced, AccessSpecifier, + // isImplicit + // + // The following bits should be 0: + // HasStandaloneLexicalDC, HasAttrs, + // TopLevelDeclInObjCContainer, + // isInvalidDecl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl @@ -2131,15 +2145,13 @@ getFunctionDeclAbbrev(serialization::DeclCode Code) { // // Add an AbbrevOp for 'size then elements' and use it here. Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); return Abv; } template <FunctionDecl::TemplatedKind Kind> std::shared_ptr<llvm::BitCodeAbbrev> getCXXMethodAbbrev() { - using namespace llvm; - auto Abv = getFunctionDeclAbbrev<Kind>(serialization::DECL_CXX_METHOD); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - return Abv; + return getFunctionDeclAbbrev<Kind>(serialization::DECL_CXX_METHOD); } } // namespace @@ -2153,10 +2165,13 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(serialization::DECL_FIELD)); // Decl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 12)); // Packed DeclBits: HasStandaloneLexicalDC, - // isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, - // AccessSpecifier, ModuleOwnershipKind + 7)); // Packed DeclBits: ModuleOwnershipKind, + // isUsed, isReferenced, AccessSpecifier, + // + // The following bits should be 0: + // isImplicit, HasStandaloneLexicalDC, HasAttrs, + // TopLevelDeclInObjCContainer, + // isInvalidDecl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl @@ -2216,10 +2231,13 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 12)); // Packed DeclBits: HasStandaloneLexicalDC, - // isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, - // AccessSpecifier, ModuleOwnershipKind + 7)); // Packed DeclBits: ModuleOwnershipKind, + // isUsed, isReferenced, AccessSpecifier, + // + // The following bits should be 0: + // isImplicit, HasStandaloneLexicalDC, HasAttrs, + // TopLevelDeclInObjCContainer, + // isInvalidDecl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl @@ -2257,10 +2275,13 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 12)); // Packed DeclBits: HasStandaloneLexicalDC, - // isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, - // AccessSpecifier, ModuleOwnershipKind + 7)); // Packed DeclBits: ModuleOwnershipKind, + // isUsed, isReferenced, AccessSpecifier, + // + // The following bits should be 0: + // isImplicit, HasStandaloneLexicalDC, HasAttrs, + // TopLevelDeclInObjCContainer, + // isInvalidDecl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl @@ -2305,10 +2326,11 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 12)); // Packed DeclBits: HasStandaloneLexicalDC, - // isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, - // AccessSpecifier, ModuleOwnershipKind + 8)); // Packed DeclBits: ModuleOwnershipKind, isUsed, + // isReferenced, AccessSpecifier, + // HasStandaloneLexicalDC, HasAttrs, isImplicit, + // TopLevelDeclInObjCContainer, + // isInvalidDecl, Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl @@ -2345,10 +2367,11 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 12)); // Packed DeclBits: HasStandaloneLexicalDC, - // isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, - // AccessSpecifier, ModuleOwnershipKind + 7)); // Packed DeclBits: ModuleOwnershipKind, + // isReferenced, isUsed, AccessSpecifier. Other + // higher bits should be 0: isImplicit, + // HasStandaloneLexicalDC, HasAttrs, + // TopLevelDeclInObjCContainer, isInvalidDecl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl @@ -2389,12 +2412,13 @@ void ASTWriter::WriteDeclAbbrevs() { // VarDecl Abv->Add(BitCodeAbbrevOp( BitCodeAbbrevOp::Fixed, - 27)); // Packed Var Decl bits: SClass, TSCSpec, InitStyle, + 21)); // Packed Var Decl bits: Linkage, ModulesCodegen, + // SClass, TSCSpec, InitStyle, // isARCPseudoStrong, IsThisDeclarationADemotedDefinition, // isExceptionVariable, isNRVOVariable, isCXXForRangeDecl, - // isObjCForDecl, isInline, isInlineSpecified, isConstexpr, - // isInitCapture, isPrevDeclInSameScope, ImplicitParamKind, - // EscapingByref, HasDeducedType, Linkage, ModulesCodegen + // isInline, isInlineSpecified, isConstexpr, + // isInitCapture, isPrevDeclInSameScope, + // EscapingByref, HasDeducedType, ImplicitParamKind, isObjCForDecl Abv->Add(BitCodeAbbrevOp(0)); // VarKind (local enum) // Type Source Info Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); @@ -2422,10 +2446,11 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(0)); // hasTypeConstraint // Decl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, - 12)); // Packed DeclBits: HasStandaloneLexicalDC, - // isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, - // AccessSpecifier, ModuleOwnershipKind + 7)); // Packed DeclBits: ModuleOwnershipKind, + // isReferenced, isUsed, AccessSpecifier. Other + // higher bits should be 0: isImplicit, + // HasStandaloneLexicalDC, HasAttrs, + // TopLevelDeclInObjCContainer, isInvalidDecl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl @@ -2471,12 +2496,14 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(serialization::EXPR_DECL_REF)); // Stmt // Expr - // PackingBits: DependenceKind, ValueKind, ObjectKind, HasQualifier, - // GetDeclFound, ExplicitTemplateArgs, HadMultipleCandidates, - // NonOdrUseReason, RefersToEnclosingVariableOrCapture, IsImmediateEscalating - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 18)); + // PackingBits: DependenceKind, ValueKind. ObjectKind should be 0. + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type // DeclRefExpr + // Packing Bits: , HadMultipleCandidates, RefersToEnclosingVariableOrCapture, + // IsImmediateEscalating, NonOdrUseReason. + // GetDeclFound, HasQualifier and ExplicitTemplateArgs should be 0. + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclRef Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Location DeclRefExprAbbrev = Stream.EmitAbbrev(std::move(Abv)); @@ -2515,11 +2542,12 @@ void ASTWriter::WriteDeclAbbrevs() { // Stmt // Expr // Packing Bits: DependenceKind, ValueKind, ObjectKind, - // HasFPFeatures, CastKind, PartOfExplicitCast - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 19)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type // CastExpr Abv->Add(BitCodeAbbrevOp(0)); // PathSize + // Packing Bits: CastKind, StoredFPFeatures, isPartOfExplicitCast + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 9)); // ImplicitCastExpr ExprImplicitCastAbbrev = Stream.EmitAbbrev(std::move(Abv)); @@ -2528,11 +2556,13 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(serialization::EXPR_BINARY_OPERATOR)); // Stmt // Expr - // Packing Bits: DependenceKind, ValueKind, ObjectKind, - // HasFPFeatures, OpKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 17)); + // Packing Bits: DependenceKind. ValueKind and ObjectKind should + // be 0 in this case. + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type // BinaryOperator + Abv->Add( + BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // OpCode and HasFPFeatures Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location BinaryOperatorAbbrev = Stream.EmitAbbrev(std::move(Abv)); @@ -2541,11 +2571,14 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(serialization::EXPR_COMPOUND_ASSIGN_OPERATOR)); // Stmt // Expr - // Packing Bits: DependenceKind, ValueKind, ObjectKind, - // HasFPFeatures, OpKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 17)); + // Packing Bits: DependenceKind. ValueKind and ObjectKind should + // be 0 in this case. + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type // BinaryOperator + // Packing Bits: OpCode. The HasFPFeatures bit should be 0 + Abv->Add( + BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // OpCode and HasFPFeatures Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location // CompoundAssignOperator Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHSType @@ -2558,10 +2591,11 @@ void ASTWriter::WriteDeclAbbrevs() { // Stmt // Expr // Packing Bits: DependenceKind, ValueKind, ObjectKind, - // NumArgs, hasStoredFPFeatures, ADLCallKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 25)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type // CallExpr + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // NumArgs + Abv->Add(BitCodeAbbrevOp(0)); // ADLCallKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location CallExprAbbrev = Stream.EmitAbbrev(std::move(Abv)); @@ -2571,12 +2605,14 @@ void ASTWriter::WriteDeclAbbrevs() { // Stmt // Expr // Packing Bits: DependenceKind, ValueKind, ObjectKind, - // NumArgs, hasStoredFPFeatures, ADLCallKind, OperatorKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 31)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type // CallExpr + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // NumArgs + Abv->Add(BitCodeAbbrevOp(0)); // ADLCallKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location // CXXOperatorCallExpr + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Operator Kind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location CXXOperatorCallExprAbbrev = Stream.EmitAbbrev(std::move(Abv)); @@ -2587,10 +2623,11 @@ void ASTWriter::WriteDeclAbbrevs() { // Stmt // Expr // Packing Bits: DependenceKind, ValueKind, ObjectKind, - // NumArgs, hasStoredFPFeatures, ADLCallKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 25)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type // CallExpr + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // NumArgs + Abv->Add(BitCodeAbbrevOp(0)); // ADLCallKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location // CXXMemberCallExpr CXXMemberCallExprAbbrev = Stream.EmitAbbrev(std::move(Abv)); @@ -2600,8 +2637,8 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(serialization::STMT_COMPOUND)); // Stmt // CompoundStmt - // Packing Bits: Num Stmts, hasStoredFPFeatures - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 21)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Num Stmts + Abv->Add(BitCodeAbbrevOp(0)); // hasStoredFPFeatures Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location CompoundStmtAbbrev = Stream.EmitAbbrev(std::move(Abv)); diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp index 02cc7798abdb..7f888e44dde1 100644 --- a/clang/lib/Serialization/ASTWriterStmt.cpp +++ b/clang/lib/Serialization/ASTWriterStmt.cpp @@ -138,10 +138,8 @@ void ASTStmtWriter::VisitNullStmt(NullStmt *S) { void ASTStmtWriter::VisitCompoundStmt(CompoundStmt *S) { VisitStmt(S); - CurrentPackingBits.updateBits(); - // 20 bits should be enough to store the size of stmts. - CurrentPackingBits.addBits(S->size(), /*Width=*/20); - CurrentPackingBits.addBit(S->hasStoredFPFeatures()); + Record.push_back(S->size()); + Record.push_back(S->hasStoredFPFeatures()); for (auto *CS : S->body()) Record.AddStmt(CS); @@ -675,25 +673,26 @@ void ASTStmtWriter::VisitPredefinedExpr(PredefinedExpr *E) { void ASTStmtWriter::VisitDeclRefExpr(DeclRefExpr *E) { VisitExpr(E); - CurrentPackingBits.addBit(E->hasQualifier()); - CurrentPackingBits.addBit(E->getDecl() != E->getFoundDecl()); - CurrentPackingBits.addBit(E->hasTemplateKWAndArgsInfo()); + CurrentPackingBits.updateBits(); + CurrentPackingBits.addBit(E->hadMultipleCandidates()); CurrentPackingBits.addBit(E->refersToEnclosingVariableOrCapture()); CurrentPackingBits.addBits(E->isNonOdrUse(), /*Width=*/2); CurrentPackingBits.addBit(E->isImmediateEscalating()); + CurrentPackingBits.addBit(E->getDecl() != E->getFoundDecl()); + CurrentPackingBits.addBit(E->hasQualifier()); + CurrentPackingBits.addBit(E->hasTemplateKWAndArgsInfo()); if (E->hasTemplateKWAndArgsInfo()) { unsigned NumTemplateArgs = E->getNumTemplateArgs(); - // 12 bits should be sufficient to store the number of template args. - CurrentPackingBits.addBits(NumTemplateArgs, /*Width=*/12); + Record.push_back(NumTemplateArgs); } DeclarationName::NameKind nk = (E->getDecl()->getDeclName().getNameKind()); if ((!E->hasTemplateKWAndArgsInfo()) && (!E->hasQualifier()) && (E->getDecl() == E->getFoundDecl()) && - nk == DeclarationName::Identifier) { + nk == DeclarationName::Identifier && E->getObjectKind() == OK_Ordinary) { AbbrevToUse = Writer.getDeclRefExprAbbrev(); } @@ -936,10 +935,10 @@ void ASTStmtWriter::VisitOMPIteratorExpr(OMPIteratorExpr *E) { void ASTStmtWriter::VisitCallExpr(CallExpr *E) { VisitExpr(E); - // 13 bits should be sufficient to store the number args; - CurrentPackingBits.addBits(E->getNumArgs(), /*BitsWidth=*/13); - CurrentPackingBits.addBit(E->hasStoredFPFeatures()); + Record.push_back(E->getNumArgs()); + CurrentPackingBits.updateBits(); CurrentPackingBits.addBit(static_cast<bool>(E->getADLCallKind())); + CurrentPackingBits.addBit(E->hasStoredFPFeatures()); Record.AddSourceLocation(E->getRParenLoc()); Record.AddStmt(E->getCallee()); @@ -950,7 +949,8 @@ void ASTStmtWriter::VisitCallExpr(CallExpr *E) { if (E->hasStoredFPFeatures()) Record.push_back(E->getFPFeatures().getAsOpaqueInt()); - if (!E->hasStoredFPFeatures() && E->getStmtClass() == Stmt::CallExprClass) + if (!E->hasStoredFPFeatures() && !static_cast<bool>(E->getADLCallKind()) && + E->getStmtClass() == Stmt::CallExprClass) AbbrevToUse = Writer.getCallExprAbbrev(); Code = serialization::EXPR_CALL; @@ -979,12 +979,11 @@ void ASTStmtWriter::VisitMemberExpr(MemberExpr *E) { // Write these first for easy access when deserializing, as they affect the // size of the MemberExpr. - + CurrentPackingBits.updateBits(); CurrentPackingBits.addBit(HasQualifier); CurrentPackingBits.addBit(HasFoundDecl); CurrentPackingBits.addBit(HasTemplateInfo); - // 12 bits should be enough to store the number of args - CurrentPackingBits.addBits(NumTemplateArgs, /*Width=*/12); + Record.push_back(NumTemplateArgs); Record.AddStmt(E->getBase()); Record.AddDeclRef(E->getMemberDecl()); @@ -1041,9 +1040,10 @@ void ASTStmtWriter::VisitCastExpr(CastExpr *E) { VisitExpr(E); Record.push_back(E->path_size()); - CurrentPackingBits.addBit(E->hasStoredFPFeatures()); + CurrentPackingBits.updateBits(); // 7 bits should be enough to store the casting kinds. CurrentPackingBits.addBits(E->getCastKind(), /*Width=*/7); + CurrentPackingBits.addBit(E->hasStoredFPFeatures()); Record.AddStmt(E->getSubExpr()); for (CastExpr::path_iterator @@ -1056,18 +1056,21 @@ void ASTStmtWriter::VisitCastExpr(CastExpr *E) { void ASTStmtWriter::VisitBinaryOperator(BinaryOperator *E) { VisitExpr(E); - bool HasFPFeatures = E->hasStoredFPFeatures(); + // Write this first for easy access when deserializing, as they affect the // size of the UnaryOperator. - CurrentPackingBits.addBit(HasFPFeatures); + CurrentPackingBits.updateBits(); CurrentPackingBits.addBits(E->getOpcode(), /*Width=*/6); + bool HasFPFeatures = E->hasStoredFPFeatures(); + CurrentPackingBits.addBit(HasFPFeatures); Record.AddStmt(E->getLHS()); Record.AddStmt(E->getRHS()); Record.AddSourceLocation(E->getOperatorLoc()); if (HasFPFeatures) Record.push_back(E->getStoredFPFeatures().getAsOpaqueInt()); - if (!HasFPFeatures) + if (!HasFPFeatures && E->getValueKind() == VK_PRValue && + E->getObjectKind() == OK_Ordinary) AbbrevToUse = Writer.getBinaryOperatorAbbrev(); Code = serialization::EXPR_BINARY_OPERATOR; @@ -1078,7 +1081,8 @@ void ASTStmtWriter::VisitCompoundAssignOperator(CompoundAssignOperator *E) { Record.AddTypeRef(E->getComputationLHSType()); Record.AddTypeRef(E->getComputationResultType()); - if (!E->hasStoredFPFeatures()) + if (!E->hasStoredFPFeatures() && E->getValueKind() == VK_PRValue && + E->getObjectKind() == OK_Ordinary) AbbrevToUse = Writer.getCompoundAssignOperatorAbbrev(); Code = serialization::EXPR_COMPOUND_ASSIGN_OPERATOR; @@ -1664,10 +1668,10 @@ void ASTStmtWriter::VisitMSDependentExistsStmt(MSDependentExistsStmt *S) { void ASTStmtWriter::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) { VisitCallExpr(E); - CurrentPackingBits.addBits(E->getOperator(), /*Width=*/6); + Record.push_back(E->getOperator()); Record.AddSourceRange(E->Range); - if (!E->hasStoredFPFeatures()) + if (!E->hasStoredFPFeatures() && !static_cast<bool>(E->getADLCallKind())) AbbrevToUse = Writer.getCXXOperatorCallExprAbbrev(); Code = serialization::EXPR_CXX_OPERATOR_CALL; @@ -1676,7 +1680,7 @@ void ASTStmtWriter::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) { void ASTStmtWriter::VisitCXXMemberCallExpr(CXXMemberCallExpr *E) { VisitCallExpr(E); - if (!E->hasStoredFPFeatures()) + if (!E->hasStoredFPFeatures() && !static_cast<bool>(E->getADLCallKind())) AbbrevToUse = Writer.getCXXMemberCallExprAbbrev(); Code = serialization::EXPR_CXX_MEMBER_CALL; @@ -1838,6 +1842,7 @@ void ASTStmtWriter::VisitCXXThisExpr(CXXThisExpr *E) { VisitExpr(E); Record.AddSourceLocation(E->getLocation()); Record.push_back(E->isImplicit()); + Code = serialization::EXPR_CXX_THIS; } @@ -1971,10 +1976,9 @@ void ASTStmtWriter::VisitCXXDependentScopeMemberExpr( // Don't emit anything here (or if you do you will have to update // the corresponding deserialization function). - + Record.push_back(E->getNumTemplateArgs()); + CurrentPackingBits.updateBits(); CurrentPackingBits.addBit(E->hasTemplateKWAndArgsInfo()); - // 16 bits should be enough to store the number of template args. - CurrentPackingBits.addBits(E->getNumTemplateArgs(), /*Width=*/16); CurrentPackingBits.addBit(E->hasFirstQualifierFoundInScope()); if (E->hasTemplateKWAndArgsInfo()) { @@ -2041,15 +2045,14 @@ ASTStmtWriter::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) { void ASTStmtWriter::VisitOverloadExpr(OverloadExpr *E) { VisitExpr(E); + Record.push_back(E->getNumDecls()); + CurrentPackingBits.updateBits(); - // 12 Bits should enough to store the number of decls. - CurrentPackingBits.addBits(E->getNumDecls(), /*BitWidth=*/12); CurrentPackingBits.addBit(E->hasTemplateKWAndArgsInfo()); if (E->hasTemplateKWAndArgsInfo()) { const ASTTemplateKWAndArgsInfo &ArgInfo = *E->getTrailingASTTemplateKWAndArgsInfo(); - // 12 Bits should enough to store the number of template args. - CurrentPackingBits.addBits(ArgInfo.NumTemplateArgs, /*BitWidth=*/12); + Record.push_back(ArgInfo.NumTemplateArgs); AddTemplateKWAndArgsInfo(ArgInfo, E->getTrailingTemplateArgumentLoc()); } diff --git a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp index 925fc90e3554..254b36ed0396 100644 --- a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp @@ -266,6 +266,8 @@ private: {&StreamChecker::preFseek, &StreamChecker::evalFseek, 0}}, {{{"ftell"}, 1}, {&StreamChecker::preDefault, &StreamChecker::evalFtell, 0}}, + {{{"fflush"}, 1}, + {&StreamChecker::preFflush, &StreamChecker::evalFflush, 0}}, {{{"rewind"}, 1}, {&StreamChecker::preDefault, &StreamChecker::evalRewind, 0}}, {{{"fgetpos"}, 2}, @@ -360,6 +362,12 @@ private: CheckerContext &C, const StreamErrorState &ErrorKind) const; + void preFflush(const FnDescription *Desc, const CallEvent &Call, + CheckerContext &C) const; + + void evalFflush(const FnDescription *Desc, const CallEvent &Call, + CheckerContext &C) const; + /// Check that the stream (in StreamVal) is not NULL. /// If it can only be NULL a fatal error is emitted and nullptr returned. /// Otherwise the return value is a new state where the stream is constrained @@ -1191,6 +1199,84 @@ void StreamChecker::evalSetFeofFerror(const FnDescription *Desc, C.addTransition(State); } +void StreamChecker::preFflush(const FnDescription *Desc, const CallEvent &Call, + CheckerContext &C) const { + ProgramStateRef State = C.getState(); + SVal StreamVal = getStreamArg(Desc, Call); + std::optional<DefinedSVal> Stream = StreamVal.getAs<DefinedSVal>(); + if (!Stream) + return; + + ProgramStateRef StateNotNull, StateNull; + std::tie(StateNotNull, StateNull) = + C.getConstraintManager().assumeDual(State, *Stream); + if (StateNotNull && !StateNull) + ensureStreamOpened(StreamVal, C, StateNotNull); +} + +void StreamChecker::evalFflush(const FnDescription *Desc, const CallEvent &Call, + CheckerContext &C) const { + ProgramStateRef State = C.getState(); + SVal StreamVal = getStreamArg(Desc, Call); + std::optional<DefinedSVal> Stream = StreamVal.getAs<DefinedSVal>(); + if (!Stream) + return; + + // Skip if the stream can be both NULL and non-NULL. + ProgramStateRef StateNotNull, StateNull; + std::tie(StateNotNull, StateNull) = + C.getConstraintManager().assumeDual(State, *Stream); + if (StateNotNull && StateNull) + return; + if (StateNotNull && !StateNull) + State = StateNotNull; + else + State = StateNull; + + const CallExpr *CE = dyn_cast_or_null<CallExpr>(Call.getOriginExpr()); + if (!CE) + return; + + // `fflush` returns EOF on failure, otherwise returns 0. + ProgramStateRef StateFailed = bindInt(*EofVal, State, C, CE); + ProgramStateRef StateNotFailed = bindInt(0, State, C, CE); + + // Clear error states if `fflush` returns 0, but retain their EOF flags. + auto ClearErrorInNotFailed = [&StateNotFailed, Desc](SymbolRef Sym, + const StreamState *SS) { + if (SS->ErrorState & ErrorFError) { + StreamErrorState NewES = + (SS->ErrorState & ErrorFEof) ? ErrorFEof : ErrorNone; + StreamState NewSS = StreamState::getOpened(Desc, NewES, false); + StateNotFailed = StateNotFailed->set<StreamMap>(Sym, NewSS); + } + }; + + if (StateNotNull && !StateNull) { + // Skip if the input stream's state is unknown, open-failed or closed. + if (SymbolRef StreamSym = StreamVal.getAsSymbol()) { + const StreamState *SS = State->get<StreamMap>(StreamSym); + if (SS) { + assert(SS->isOpened() && "Stream is expected to be opened"); + ClearErrorInNotFailed(StreamSym, SS); + } else + return; + } + } else { + // Clear error states for all streams. + const StreamMapTy &Map = StateNotFailed->get<StreamMap>(); + for (const auto &I : Map) { + SymbolRef Sym = I.first; + const StreamState &SS = I.second; + if (SS.isOpened()) + ClearErrorInNotFailed(Sym, &SS); + } + } + + C.addTransition(StateNotFailed); + C.addTransition(StateFailed); +} + ProgramStateRef StreamChecker::ensureStreamNonNull(SVal StreamVal, const Expr *StreamE, CheckerContext &C, diff --git a/clang/test/Analysis/Inputs/system-header-simulator.h b/clang/test/Analysis/Inputs/system-header-simulator.h index 7089bd8bfc9d..409a969a0d4c 100644 --- a/clang/test/Analysis/Inputs/system-header-simulator.h +++ b/clang/test/Analysis/Inputs/system-header-simulator.h @@ -61,6 +61,7 @@ void clearerr(FILE *stream); int feof(FILE *stream); int ferror(FILE *stream); int fileno(FILE *stream); +int fflush(FILE *stream); size_t strlen(const char *); diff --git a/clang/test/Analysis/stream-error.c b/clang/test/Analysis/stream-error.c index c8332bcbfa8c..37e1e54dfc89 100644 --- a/clang/test/Analysis/stream-error.c +++ b/clang/test/Analysis/stream-error.c @@ -299,6 +299,73 @@ void error_fseek_0(void) { fclose(F); } +void error_fflush_after_fclose(void) { + FILE *F = tmpfile(); + int Ret; + fflush(NULL); // no-warning + if (!F) + return; + if ((Ret = fflush(F)) != 0) + clang_analyzer_eval(Ret == EOF); // expected-warning {{TRUE}} + fclose(F); + fflush(F); // expected-warning {{Stream might be already closed}} +} + +void error_fflush_on_open_failed_stream(void) { + FILE *F = tmpfile(); + if (!F) { + fflush(F); // no-warning + return; + } + fclose(F); +} + +void error_fflush_on_unknown_stream(FILE *F) { + fflush(F); // no-warning + fclose(F); // no-warning +} + +void error_fflush_on_non_null_stream_clear_error_states(void) { + FILE *F0 = tmpfile(), *F1 = tmpfile(); + // `fflush` clears a non-EOF stream's error state. + if (F0) { + StreamTesterChecker_make_ferror_stream(F0); + if (fflush(F0) == 0) { // no-warning + clang_analyzer_eval(ferror(F0)); // expected-warning {{FALSE}} + clang_analyzer_eval(feof(F0)); // expected-warning {{FALSE}} + } + fclose(F0); + } + // `fflush` clears an EOF stream's error state. + if (F1) { + StreamTesterChecker_make_feof_stream(F1); + if (fflush(F1) == 0) { // no-warning + clang_analyzer_eval(ferror(F1)); // expected-warning {{FALSE}} + clang_analyzer_eval(feof(F1)); // expected-warning {{TRUE}} + } + fclose(F1); + } +} + +void error_fflush_on_null_stream_clear_error_states(void) { + FILE *F0 = tmpfile(), *F1 = tmpfile(); + // `fflush` clears all stream's error states, while retains their EOF states. + if (F0 && F1) { + StreamTesterChecker_make_ferror_stream(F0); + StreamTesterChecker_make_feof_stream(F1); + if (fflush(NULL) == 0) { // no-warning + clang_analyzer_eval(ferror(F0)); // expected-warning {{FALSE}} + clang_analyzer_eval(feof(F0)); // expected-warning {{FALSE}} + clang_analyzer_eval(ferror(F1)); // expected-warning {{FALSE}} + clang_analyzer_eval(feof(F1)); // expected-warning {{TRUE}} + } + } + if (F0) + fclose(F0); + if (F1) + fclose(F1); +} + void error_indeterminate(void) { FILE *F = fopen("file", "r+"); if (!F) diff --git a/clang/test/CodeGen/aarch64-branch-protection-attr.c b/clang/test/CodeGen/aarch64-branch-protection-attr.c index 3c2714e2feda..8ab3e17ade42 100644 --- a/clang/test/CodeGen/aarch64-branch-protection-attr.c +++ b/clang/test/CodeGen/aarch64-branch-protection-attr.c @@ -46,6 +46,24 @@ __attribute__ ((target("branch-protection=pac-ret+leaf+bti"))) void btileaf() {} // CHECK: define{{.*}} void @btileaf() #[[#BTIPACLEAF:]] + +__attribute__ ((target("branch-protection=pac-ret+pc"))) +void pauthlr() {} +// CHECK: define{{.*}} void @pauthlr() #[[#PAUTHLR:]] + +__attribute__ ((target("branch-protection=pac-ret+pc+b-key"))) +void pauthlr_bkey() {} +// CHECK: define{{.*}} void @pauthlr_bkey() #[[#PAUTHLR_BKEY:]] + +__attribute__ ((target("branch-protection=pac-ret+pc+leaf"))) +void pauthlr_leaf() {} +// CHECK: define{{.*}} void @pauthlr_leaf() #[[#PAUTHLR_LEAF:]] + +__attribute__ ((target("branch-protection=pac-ret+pc+bti"))) +void pauthlr_bti() {} +// CHECK: define{{.*}} void @pauthlr_bti() #[[#PAUTHLR_BTI:]] + + // CHECK-DAG: attributes #[[#NONE]] = { {{.*}} "branch-target-enforcement"="false" {{.*}} "sign-return-address"="none" // CHECK-DAG: attributes #[[#STD]] = { {{.*}} "branch-target-enforcement"="true" {{.*}} "sign-return-address"="non-leaf" "sign-return-address-key"="a_key" @@ -61,3 +79,13 @@ void btileaf() {} // CHECK-DAG: attributes #[[#PACBKEYLEAF]] = { {{.*}} "branch-target-enforcement"="false" {{.*}}"sign-return-address"="all" "sign-return-address-key"="b_key" // CHECK-DAG: attributes #[[#BTIPACLEAF]] = { {{.*}}"branch-target-enforcement"="true" {{.*}} "sign-return-address"="all" "sign-return-address-key"="a_key" + + +// CHECK-DAG: attributes #[[#PAUTHLR]] = { {{.*}}"branch-protection-pauth-lr"="true" {{.*}}"branch-target-enforcement"="false" {{.*}}"sign-return-address"="non-leaf" "sign-return-address-key"="a_key" + +// CHECK-DAG: attributes #[[#PAUTHLR_BKEY]] = { {{.*}}"branch-protection-pauth-lr"="true" {{.*}}"branch-target-enforcement"="false" {{.*}}"sign-return-address"="non-leaf" "sign-return-address-key"="b_key" + +// CHECK-DAG: attributes #[[#PAUTHLR_LEAF]] = { {{.*}}"branch-protection-pauth-lr"="true" {{.*}}"branch-target-enforcement"="false" {{.*}}"sign-return-address"="all" "sign-return-address-key"="a_key" + +// CHECK-DAG: attributes #[[#PAUTHLR_BTI]] = { {{.*}}"branch-protection-pauth-lr"="true" {{.*}}"branch-target-enforcement"="true" {{.*}}"sign-return-address"="non-leaf" "sign-return-address-key"="a_key" + diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_fp_dots.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_fp_dots.c new file mode 100644 index 000000000000..ff4176530710 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_fp_dots.c @@ -0,0 +1,297 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: aarch64-registered-target + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +#include <arm_sme_draft_spec_subject_to_change.h> + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED,A5) A1##A3##A5 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4,A5) A1##A2##A3##A4##A5 +#endif + +// +// Multi, multi (half) +// CHECK-LABEL: @test_svdot_multi_za32_vg1x2_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.za32.vg1x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_multi_za32_vg1x2_f16j13svfloat16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.za32.vg1x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za32_vg1x2_f16(uint32_t slice_base, svfloat16x2_t zn, svfloat16x2_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za32,,,_f16,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_multi_za32_vg1x4_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.za32.vg1x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], <vscale x 8 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_multi_za32_vg1x4_f16j13svfloat16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.za32.vg1x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], <vscale x 8 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za32_vg1x4_f16(uint32_t slice_base, svfloat16x4_t zn, svfloat16x4_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za32,,,_f16,_vg1x4)(slice_base, zn, zm); +} + + +// +// Multi, single (half) +// CHECK-LABEL: @test_svdot_single_za32_vg1x2_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.single.za32.vg1x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svdot_single_za32_vg1x2_f16j13svfloat16x2_tu13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.single.za32.vg1x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za32_vg1x2_f16(uint32_t slice_base, svfloat16x2_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za32,,_f16,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_single_za32_vg1x4_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.single.za32.vg1x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svdot_single_za32_vg1x4_f16j13svfloat16x4_tu13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.single.za32.vg1x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za32_vg1x4_f16(uint32_t slice_base, svfloat16x4_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za32,,_f16,,_vg1x4)(slice_base, zn, zm); +} + + +// +// Multi, indexed (half) +// CHECK-LABEL: @test_svdot_lane_za32_vg1x2_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svdot_lane_za32_vg1x2_f16j13svfloat16x2_tu13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za32_vg1x2_f16(uint32_t slice_base, svfloat16x2_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za32,,,_f16,_vg1x2)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svdot_lane_za32_vg1x4_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svdot_lane_za32_vg1x4_f16j13svfloat16x4_tu13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za32_vg1x4_f16(uint32_t slice_base, svfloat16x4_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za32,,,_f16,_vg1x4)(slice_base, zn, zm, 3); +} + + +// +// Multi, multi (bfloat) +// CHECK-LABEL: @test_svdot_multi_za32_vg1x2_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.za32.vg1x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svdot_multi_za32_vg1x2_bf16j14svbfloat16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.za32.vg1x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za32_vg1x2_bf16(uint32_t slice_base, svbfloat16x2_t zn, svbfloat16x2_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za32,,,_bf16,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_multi_za32_vg1x4_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.za32.vg1x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svdot_multi_za32_vg1x4_bf16j14svbfloat16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.za32.vg1x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za32_vg1x4_bf16(uint32_t slice_base, svbfloat16x4_t zn, svbfloat16x4_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za32,,,_bf16,_vg1x4)(slice_base, zn, zm); +} + + +// +// Multi, single (bfloat) +// CHECK-LABEL: @test_svdot_single_za32_vg1x2_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.single.za32.vg1x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z33test_svdot_single_za32_vg1x2_bf16j14svbfloat16x2_tu14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.single.za32.vg1x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za32_vg1x2_bf16(uint32_t slice_base, svbfloat16x2_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za32,,_bf16,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_single_za32_vg1x4_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.single.za32.vg1x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z33test_svdot_single_za32_vg1x4_bf16j14svbfloat16x4_tu14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.single.za32.vg1x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za32_vg1x4_bf16(uint32_t slice_base, svbfloat16x4_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za32,,_bf16,,_vg1x4)(slice_base, zn, zm); +} + + +// +// Multi, indexed (bfloat) +// CHECK-LABEL: @test_svdot_lane_za32_vg1x2_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_lane_za32_vg1x2_bf16j14svbfloat16x2_tu14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za32_vg1x2_bf16(uint32_t slice_base, svbfloat16x2_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za32,,_bf16,,_vg1x2)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svdot_lane_za32_vg1x4_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_lane_za32_vg1x4_bf16j14svbfloat16x4_tu14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za32_vg1x4_bf16(uint32_t slice_base, svbfloat16x4_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za32,,_bf16,,_vg1x4)(slice_base, zn, zm, 3); +} diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_frint.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_frint.c new file mode 100644 index 000000000000..2a34b0e2878e --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_frint.c @@ -0,0 +1,282 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py + +// REQUIRES: aarch64-registered-target + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + +#include <arm_sme_draft_spec_subject_to_change.h> + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED) A1 +#else +#define SVE_ACLE_FUNC(A1,A2) A1##A2 +#endif + +// FRINTA + +// CHECK-LABEL: @test_svfrinta_f32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frinta.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], i64 4) +// CHECK-NEXT: ret <vscale x 8 x float> [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z20test_svfrinta_f32_x213svfloat32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frinta.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret <vscale x 8 x float> [[TMP6]] +// +svfloat32x2_t test_svfrinta_f32_x2(svfloat32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svrinta,_f32_x2)(zn); +} + +// CHECK-LABEL: @test_svfrinta_f32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frinta.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP8]], <vscale x 4 x float> [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP10]], <vscale x 4 x float> [[TMP11]], i64 12) +// CHECK-NEXT: ret <vscale x 16 x float> [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z20test_svfrinta_f32_x413svfloat32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frinta.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP8]], <vscale x 4 x float> [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP10]], <vscale x 4 x float> [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret <vscale x 16 x float> [[TMP12]] +// +svfloat32x4_t test_svfrinta_f32_x4(svfloat32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svrinta,_f32_x4)(zn); +} + +// FRINTM + +// CHECK-LABEL: @test_svfrintam_f32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frintm.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], i64 4) +// CHECK-NEXT: ret <vscale x 8 x float> [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z21test_svfrintam_f32_x213svfloat32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frintm.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret <vscale x 8 x float> [[TMP6]] +// +svfloat32x2_t test_svfrintam_f32_x2(svfloat32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svrintm,_f32_x2)(zn); +} + +// CHECK-LABEL: @test_svfrintm_f32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frintm.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP8]], <vscale x 4 x float> [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP10]], <vscale x 4 x float> [[TMP11]], i64 12) +// CHECK-NEXT: ret <vscale x 16 x float> [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z20test_svfrintm_f32_x413svfloat32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frintm.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP8]], <vscale x 4 x float> [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP10]], <vscale x 4 x float> [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret <vscale x 16 x float> [[TMP12]] +// +svfloat32x4_t test_svfrintm_f32_x4(svfloat32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svrintm,_f32_x4)(zn); +} + +// FRINTN + +// CHECK-LABEL: @test_svfrintn_f32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frintn.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], i64 4) +// CHECK-NEXT: ret <vscale x 8 x float> [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z20test_svfrintn_f32_x213svfloat32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frintn.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret <vscale x 8 x float> [[TMP6]] +// +svfloat32x2_t test_svfrintn_f32_x2(svfloat32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svrintn,_f32_x2)(zn); +} + +// CHECK-LABEL: @test_svfrintn_f32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frintn.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP8]], <vscale x 4 x float> [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP10]], <vscale x 4 x float> [[TMP11]], i64 12) +// CHECK-NEXT: ret <vscale x 16 x float> [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z20test_svfrintn_f32_x413svfloat32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frintn.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP8]], <vscale x 4 x float> [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP10]], <vscale x 4 x float> [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret <vscale x 16 x float> [[TMP12]] +// +svfloat32x4_t test_svfrintn_f32_x4(svfloat32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svrintn,_f32_x4)(zn); +} + +// FRINTP + +// CHECK-LABEL: @test_svfrintp_f32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frintp.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 0 +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP3]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 1 +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], i64 4) +// CHECK-NEXT: ret <vscale x 8 x float> [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z20test_svfrintp_f32_x213svfloat32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frintp.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 0 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP3]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP2]], 1 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], i64 4) +// CPP-CHECK-NEXT: ret <vscale x 8 x float> [[TMP6]] +// +svfloat32x2_t test_svfrintp_f32_x2(svfloat32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svrintp,_f32_x2)(zn); +} + +// CHECK-LABEL: @test_svfrintp_f32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frintp.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]]) +// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 0 +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP5]], i64 0) +// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]], i64 4) +// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP8]], <vscale x 4 x float> [[TMP9]], i64 8) +// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP10]], <vscale x 4 x float> [[TMP11]], i64 12) +// CHECK-NEXT: ret <vscale x 16 x float> [[TMP12]] +// +// CPP-CHECK-LABEL: @_Z20test_svfrintp_f32_x413svfloat32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.frintp.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]]) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 0 +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP5]], i64 0) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 1 +// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]], i64 4) +// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 2 +// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP8]], <vscale x 4 x float> [[TMP9]], i64 8) +// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 3 +// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP10]], <vscale x 4 x float> [[TMP11]], i64 12) +// CPP-CHECK-NEXT: ret <vscale x 16 x float> [[TMP12]] +// +svfloat32x4_t test_svfrintp_f32_x4(svfloat32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svrintp,_f32_x4)(zn); +} diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_int_dots.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_int_dots.c new file mode 100644 index 000000000000..0d85071b7fc3 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_int_dots.c @@ -0,0 +1,1103 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: aarch64-registered-target + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-i16i64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-i16i64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-i16i64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-i16i64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-i16i64 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +#include <arm_sme_draft_spec_subject_to_change.h> + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED,A5) A1##A3##A5 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4,A5) A1##A2##A3##A4##A5 +#endif + +// +// Multi, multi (unsigned) +// CHECK-LABEL: @test_svdot_multi_za32_vg1x2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_multi_za32_vg1x2_u16j12svuint16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za32_vg1x2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16x2_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za32,,,_u16,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_multi_za32_vg1x4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.za32.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_multi_za32_vg1x4_u16j12svuint16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.za32.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za32_vg1x4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16x4_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za32,,,_u16,_vg1x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_multi_za32_vg1x2_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svdot_multi_za32_vg1x2_u8j11svuint8x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za32_vg1x2_u8(uint32_t slice_base, svuint8x2_t zn, svuint8x2_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za32,,,_u8,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_multi_za32_vg1x4_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svdot_multi_za32_vg1x4_u8j11svuint8x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za32_vg1x4_u8(uint32_t slice_base, svuint8x4_t zn, svuint8x4_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za32,,,_u8,_vg1x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_multi_za64_vg1x2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.za64.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_multi_za64_vg1x2_u16j12svuint16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.za64.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za64_vg1x2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16x2_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za64,,,_u16,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_multi_za64_vg1x4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_multi_za64_vg1x4_u16j12svuint16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za64_vg1x4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16x4_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za64,,,_u16,_vg1x4)(slice_base, zn, zm); +} + + +// +// Multi, multi (signed) +// CHECK-LABEL: @test_svdot_multi_za32_vg1x2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_multi_za32_vg1x2_s16j11svint16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za32_vg1x2_s16(uint32_t slice_base, svint16x2_t zn, svint16x2_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za32,,,_s16,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_multi_za32_vg1x4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.za32.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_multi_za32_vg1x4_s16j11svint16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.za32.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za32_vg1x4_s16(uint32_t slice_base, svint16x4_t zn, svint16x4_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za32,,,_s16,_vg1x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_multi_za32_vg1x2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svdot_multi_za32_vg1x2_s8j10svint8x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za32_vg1x2_s8(uint32_t slice_base, svint8x2_t zn, svint8x2_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za32,,,_s8,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_multi_za32_vg1x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svdot_multi_za32_vg1x4_s8j10svint8x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za32_vg1x4_s8(uint32_t slice_base, svint8x4_t zn, svint8x4_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za32,,,_s8,_vg1x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_multi_za64_vg1x2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.za64.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_multi_za64_vg1x2_s16j11svint16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.za64.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za64_vg1x2_s16(uint32_t slice_base, svint16x2_t zn, svint16x2_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za64,,,_s16,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_multi_za64_vg1x4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_multi_za64_vg1x4_s16j11svint16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_multi_za64_vg1x4_s16(uint32_t slice_base, svint16x4_t zn, svint16x4_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_za64,,,_s16,_vg1x4)(slice_base, zn, zm); +} + + +// +// Multi, single (unsigned) +// CHECK-LABEL: @test_svdot_single_za32_vg1x2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svdot_single_za32_vg1x2_u16j12svuint16x2_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za32_vg1x2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za32,,_u16,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_single_za32_vg1x4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svdot_single_za32_vg1x4_u16j12svuint16x4_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za32_vg1x4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za32,,_u16,,_vg1x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_single_za32_vg1x2_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_single_za32_vg1x2_u8j11svuint8x2_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za32_vg1x2_u8(uint32_t slice_base, svuint8x2_t zn, svuint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za32,,_u8,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_single_za32_vg1x4_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_single_za32_vg1x4_u8j11svuint8x4_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za32_vg1x4_u8(uint32_t slice_base, svuint8x4_t zn, svuint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za32,,_u8,,_vg1x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_single_za64_vg1x2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.single.za64.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svdot_single_za64_vg1x2_u16j12svuint16x2_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.single.za64.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za64_vg1x2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za64,,_u16,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_single_za64_vg1x4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.single.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svdot_single_za64_vg1x4_u16j12svuint16x4_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.single.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za64_vg1x4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za64,,_u16,,_vg1x4)(slice_base, zn, zm); +} + + +// +// Multi, single (signed) +// CHECK-LABEL: @test_svdot_single_za32_vg1x2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.single.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svdot_single_za32_vg1x2_s16j11svint16x2_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.single.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za32_vg1x2_s16(uint32_t slice_base, svint16x2_t zn, svint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za32,,_s16,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_single_za32_vg1x4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.single.za32.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svdot_single_za32_vg1x4_s16j11svint16x4_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.single.za32.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za32_vg1x4_s16(uint32_t slice_base, svint16x4_t zn, svint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za32,,_s16,,_vg1x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_single_za32_vg1x2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.single.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_single_za32_vg1x2_s8j10svint8x2_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.single.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za32_vg1x2_s8(uint32_t slice_base, svint8x2_t zn, svint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za32,,_s8,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_single_za32_vg1x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.single.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svdot_single_za32_vg1x4_s8j10svint8x4_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.single.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za32_vg1x4_s8(uint32_t slice_base, svint8x4_t zn, svint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za32,,_s8,,_vg1x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_single_za64_vg1x2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.single.za64.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svdot_single_za64_vg1x2_s16j11svint16x2_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.single.za64.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za64_vg1x2_s16(uint32_t slice_base, svint16x2_t zn, svint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za64,,_s16,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svdot_single_za64_vg1x4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.single.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svdot_single_za64_vg1x4_s16j11svint16x4_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.single.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_single_za64_vg1x4_s16(uint32_t slice_base, svint16x4_t zn, svint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_single_za64,,_s16,,_vg1x4)(slice_base, zn, zm); +} + +// +// Multi, indexed (unsigned) +// CHECK-LABEL: @test_svdot_lane_za32_vg1x2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svdot_lane_za32_vg1x2_u16j12svuint16x2_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za32_vg1x2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za32,,_u16,,_vg1x2)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svdot_lane_za32_vg1x4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svdot_lane_za32_vg1x4_u16j12svuint16x4_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za32_vg1x4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za32,,_u16,,_vg1x4)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svdot_lane_za32_vg1x2_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z29test_svdot_lane_za32_vg1x2_u8j11svuint8x2_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za32_vg1x2_u8(uint32_t slice_base, svuint8x2_t zn, svuint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za32,,_u8,,_vg1x2)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svdot_lane_za32_vg1x4_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z29test_svdot_lane_za32_vg1x4_u8j11svuint8x4_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za32_vg1x4_u8(uint32_t slice_base, svuint8x4_t zn, svuint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za32,,_u8,,_vg1x4)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svdot_lane_za64_vg1x2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.lane.za64.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 1) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svdot_lane_za64_vg1x2_u16j12svuint16x2_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.lane.za64.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 1) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za64_vg1x2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za64,,,_u16,_vg1x2)(slice_base, zn, zm, 1); +} + +// CHECK-LABEL: @test_svdot_lane_za64_vg1x4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.lane.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 1) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svdot_lane_za64_vg1x4_u16j12svuint16x4_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.udot.lane.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 1) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za64_vg1x4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za64,,,_u16,_vg1x4)(slice_base, zn, zm, 1); +} + + +// +// Multi, indexed (signed) +// CHECK-LABEL: @test_svdot_lane_za32_vg1x2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svdot_lane_za32_vg1x2_s16j11svint16x2_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za32_vg1x2_s16(uint32_t slice_base, svint16x2_t zn, svint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za32,,,_s16,_vg1x2)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svdot_lane_za32_vg1x4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svdot_lane_za32_vg1x4_s16j11svint16x4_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za32_vg1x4_s16(uint32_t slice_base, svint16x4_t zn, svint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za32,,,_s16,_vg1x4)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svdot_lane_za32_vg1x2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z29test_svdot_lane_za32_vg1x2_s8j10svint8x2_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za32_vg1x2_s8(uint32_t slice_base, svint8x2_t zn, svint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za32,,,_s8,_vg1x2)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svdot_lane_za32_vg1x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z29test_svdot_lane_za32_vg1x4_s8j10svint8x4_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za32_vg1x4_s8(uint32_t slice_base, svint8x4_t zn, svint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za32,,,_s8,_vg1x4)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svdot_lane_za64_vg1x2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.lane.za64.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 1) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svdot_lane_za64_vg1x2_s16j11svint16x2_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.lane.za64.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 1) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za64_vg1x2_s16(uint32_t slice_base, svint16x2_t zn, svint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za64,,,_s16,_vg1x2)(slice_base, zn, zm, 1); +} + +// CHECK-LABEL: @test_svdot_lane_za64_vg1x4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.lane.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 1) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svdot_lane_za64_vg1x4_s16j11svint16x4_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sdot.lane.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 1) +// CPP-CHECK-NEXT: ret void +// +void test_svdot_lane_za64_vg1x4_s16(uint32_t slice_base, svint16x4_t zn, svint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svdot_lane_za64,,,_s16,_vg1x4)(slice_base, zn, zm, 1); +} + + +// +// Multi, multi (unsigned by signed) +// CHECK-LABEL: @test_svusdot_multi_za32_vg1x2_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svusdot_multi_za32_vg1x2_u8j11svuint8x2_t10svint8x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svusdot_multi_za32_vg1x2_u8(uint32_t slice_base, svuint8x2_t zn, svint8x2_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svusdot_za32,,,_u8,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svusdot_multi_za32_vg1x4_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svusdot_multi_za32_vg1x4_u8j11svuint8x4_t10svint8x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svusdot_multi_za32_vg1x4_u8(uint32_t slice_base, svuint8x4_t zn, svint8x4_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svusdot_za32,,,_u8,_vg1x4)(slice_base, zn, zm); +} + + +// +// Multi, single (unsigned by signed) +// CHECK-LABEL: @test_svusdot_single_za32_vg1x2_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.single.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z33test_svusdot_single_za32_vg1x2_u8j11svuint8x2_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.single.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svusdot_single_za32_vg1x2_u8(uint32_t slice_base, svuint8x2_t zn, svint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svusdot_single_za32,,_u8,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svusdot_single_za32_vg1x4_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.single.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z33test_svusdot_single_za32_vg1x4_u8j11svuint8x4_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.single.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svusdot_single_za32_vg1x4_u8(uint32_t slice_base, svuint8x4_t zn, svint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svusdot_single_za32,,_u8,,_vg1x4)(slice_base, zn, zm); +} + +// +// Multi, indexed (unsigned by signed) +// CHECK-LABEL: @test_svusdot_lane_za32_vg1x2_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svusdot_lane_za32_vg1x2_u8j11svuint8x2_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svusdot_lane_za32_vg1x2_u8(uint32_t slice_base, svuint8x2_t zn, svint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svusdot_lane_za32,,_u8,,_vg1x2)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svusdot_lane_za32_vg1x4_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svusdot_lane_za32_vg1x4_u8j11svuint8x4_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svusdot_lane_za32_vg1x4_u8(uint32_t slice_base, svuint8x4_t zn, svint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svusdot_lane_za32,,_u8,,_vg1x4)(slice_base, zn, zm, 3); +} + + +// +// Multi, single (signed by unsigned) +// CHECK-LABEL: @test_svsudot_single_za32_vg1x2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sudot.single.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z33test_svsudot_single_za32_vg1x2_s8j10svint8x2_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sudot.single.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svsudot_single_za32_vg1x2_s8(uint32_t slice_base, svint8x2_t zn, svuint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svsudot_single_za32,,_s8,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svsudot_single_za32_vg1x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sudot.single.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z33test_svsudot_single_za32_vg1x4_s8j10svint8x4_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sudot.single.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svsudot_single_za32_vg1x4_s8(uint32_t slice_base, svint8x4_t zn, svuint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svsudot_single_za32,,_s8,,_vg1x4)(slice_base, zn, zm); +} + +// +// Multi, multi (signed by unsigned) +// CHECK-LABEL: @test_svsudot_multi_za32_vg1x2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svsudot_multi_za32_vg1x2_s8j10svint8x2_t11svuint8x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]]) +// CPP-CHECK-NEXT: ret void +// +void test_svsudot_multi_za32_vg1x2_s8(uint32_t slice_base, svint8x2_t zn, svuint8x2_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svsudot_za32,,_s8,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svsudot_multi_za32_vg1x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svsudot_multi_za32_vg1x4_s8j10svint8x4_t11svuint8x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usdot.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svsudot_multi_za32_vg1x4_s8(uint32_t slice_base, svint8x4_t zn, svuint8x4_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svsudot_za32,,_s8,,_vg1x4)(slice_base, zn, zm); +} + +// +// Multi, indexed (signed by unsigned) +// CHECK-LABEL: @test_svsudot_lane_za32_vg1x2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svsudot_lane_za32_vg1x2_s8j10svint8x2_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svsudot_lane_za32_vg1x2_s8(uint32_t slice_base, svint8x2_t zn, svuint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svsudot_lane_za32,,_s8,,_vg1x2)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svsudot_lane_za32_vg1x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svsudot_lane_za32_vg1x4_s8j10svint8x4_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svsudot_lane_za32_vg1x4_s8(uint32_t slice_base, svint8x4_t zn, svuint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svsudot_lane_za32,,_s8,,_vg1x4)(slice_base, zn, zm, 3); +} diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mla.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mla.c new file mode 100644 index 000000000000..f52edd9888da --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mla.c @@ -0,0 +1,292 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: aarch64-registered-target + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +#include <arm_sme_draft_spec_subject_to_change.h> + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED,A5) A1##A3##A5 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4,A5) A1##A2##A3##A4##A5 +#endif + +// +// Multi, multi +// CHECK-LABEL: @test_svmla2_f32( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZM]], i64 4) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.vg1x2.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmla2_f32j13svfloat32x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZM]], i64 4) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.vg1x2.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla2_f32(uint32_t slice_base, svfloat32x2_t zn, svfloat32x2_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmla_za32,_f32,_vg1x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla4_f32( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM]], i64 4) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM]], i64 12) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.vg1x4.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], <vscale x 4 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmla4_f32j13svfloat32x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM]], i64 4) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM]], i64 12) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.vg1x4.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], <vscale x 4 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla4_f32(uint32_t slice_base, svfloat32x4_t zn, svfloat32x4_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmla_za32,_f32,_vg1x4,,)(slice_base, zn, zm); +} + +// +// Multi, single +// CHECK-LABEL: @test_svmla_single2_f32( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.single.vg1x2.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmla_single2_f32j13svfloat32x2_tu13__SVFloat32_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.single.vg1x2.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single2_f32(uint32_t slice_base, svfloat32x2_t zn, svfloat32_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmla_single_za32,,_f32,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single4_f32( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.single.vg1x4.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmla_single4_f32j13svfloat32x4_tu13__SVFloat32_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.single.vg1x4.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single4_f32(uint32_t slice_base, svfloat32x4_t zn, svfloat32_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmla_single_za32,,_f32,,_vg1x4)(slice_base, zn, zm); +} + +// +// Multi, indexed +// CHECK-LABEL: @test_svmla_lane2_f32( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.lane.vg1x2.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmla_lane2_f32j13svfloat32x2_tu13__SVFloat32_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.lane.vg1x2.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane2_f32(uint32_t slice_base, svfloat32x2_t zn, svfloat32_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmla_lane_za32,_f32,_vg1x2,,)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svmla_lane4_f32( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.lane.vg1x4.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmla_lane4_f32j13svfloat32x4_tu13__SVFloat32_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.lane.vg1x4.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane4_f32(uint32_t slice_base, svfloat32x4_t zn, svfloat32_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmla_lane_za32,_f32,_vg1x4,,)(slice_base, zn, zm, 3); +} + +// +// Multi, multi +// CHECK-LABEL: @test_svmla2_f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZM]], i64 2) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.vg1x2.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmla2_f64j13svfloat64x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZM]], i64 2) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.vg1x2.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla2_f64(uint32_t slice_base, svfloat64x2_t zn, svfloat64x2_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmla_za64,_f64,_vg1x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla4_f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM]], i64 2) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM]], i64 4) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM]], i64 6) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.vg1x4.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], <vscale x 2 x double> [[TMP5]], <vscale x 2 x double> [[TMP6]], <vscale x 2 x double> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmla4_f64j13svfloat64x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM]], i64 2) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM]], i64 4) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM]], i64 6) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.vg1x4.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], <vscale x 2 x double> [[TMP5]], <vscale x 2 x double> [[TMP6]], <vscale x 2 x double> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla4_f64(uint32_t slice_base, svfloat64x4_t zn, svfloat64x4_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmla_za64,_f64,_vg1x4,,)(slice_base, zn, zm); +} + +// +// Multi, single +// CHECK-LABEL: @test_svmla_single2_f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN]], i64 2) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.single.vg1x2.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmla_single2_f64j13svfloat64x2_tu13__SVFloat64_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN]], i64 2) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.single.vg1x2.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single2_f64(uint32_t slice_base, svfloat64x2_t zn, svfloat64_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmla_single_za64,,_f64,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single4_f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 6) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.single.vg1x4.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmla_single4_f64j13svfloat64x4_tu13__SVFloat64_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 6) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.single.vg1x4.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single4_f64(uint32_t slice_base, svfloat64x4_t zn, svfloat64_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmla_single_za64,,_f64,,_vg1x4)(slice_base, zn, zm); +} + +// +// Multi, indexed +// CHECK-LABEL: @test_svmla_lane2_f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN]], i64 2) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.lane.vg1x2.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[ZM:%.*]], i32 1) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmla_lane2_f64j13svfloat64x2_tu13__SVFloat64_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN]], i64 2) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.lane.vg1x2.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[ZM:%.*]], i32 1) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane2_f64(uint32_t slice_base, svfloat64x2_t zn, svfloat64_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmla_lane_za64,_f64,_vg1x2,,)(slice_base, zn, zm, 1); +} + +// CHECK-LABEL: @test_svmla_lane4_f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 6) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.lane.vg1x4.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[ZM:%.*]], i32 1) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmla_lane4_f64j13svfloat64x4_tu13__SVFloat64_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 6) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmla.lane.vg1x4.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[ZM:%.*]], i32 1) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane4_f64(uint32_t slice_base, svfloat64x4_t zn, svfloat64_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmla_lane_za64,_f64,_vg1x4,,)(slice_base, zn, zm, 1); +} diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mlal.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mlal.c new file mode 100644 index 000000000000..834ade753507 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mlal.c @@ -0,0 +1,696 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: aarch64-registered-target + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +#include <arm_sme_draft_spec_subject_to_change.h> + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED,A5) A1##A3##A5 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4,A5) A1##A2##A3##A4##A5 +#endif + +// +// Multi, multi +// CHECK-LABEL: @test_svmla2_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.vg2x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmla2_f16j13svfloat16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.vg2x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla2_f16(uint32_t slice_base, svfloat16x2_t zn, svfloat16x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_f16,_vg2x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla2_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.vg2x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z16test_svmla2_bf16j14svbfloat16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.vg2x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla2_bf16(uint32_t slice_base, svbfloat16x2_t zn, svbfloat16x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_bf16,_vg2x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmla2_u16j12svuint16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_u16,_vg2x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmla2_s16j11svint16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla2_s16(uint32_t slice_base, svint16x2_t zn, svint16x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_s16,_vg2x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla4_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.vg2x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], <vscale x 8 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmla4_f16j13svfloat16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.vg2x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], <vscale x 8 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla4_f16(uint32_t slice_base, svfloat16x4_t zn, svfloat16x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_f16,_vg2x4,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla4_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.vg2x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z16test_svmla4_bf16j14svbfloat16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.vg2x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla4_bf16(uint32_t slice_base, svbfloat16x4_t zn, svbfloat16x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_bf16,_vg2x4,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmla4_u16j12svuint16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_u16,_vg2x4,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmla4_s16j11svint16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla4_s16(uint32_t slice_base, svint16x4_t zn, svint16x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_s16,_vg2x4,,)(slice_base, zn, zm); +} + +// +// Multi, single +// CHECK-LABEL: @test_svmla_single1_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.single.vg2x1.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmla_single1_f16ju13__SVFloat16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.single.vg2x1.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single1_f16(uint32_t slice_base, svfloat16_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_f16,_vg2x1,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single1_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.single.vg2x1.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmla_single1_bf16ju14__SVBfloat16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.single.vg2x1.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single1_bf16(uint32_t slice_base, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_bf16,_vg2x1,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single1_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.single.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmla_single1_u16ju12__SVUint16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.single.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single1_u16(uint32_t slice_base, svuint16_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_u16,_vg2x1,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single1_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.single.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmla_single1_s16ju11__SVInt16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.single.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single1_s16(uint32_t slice_base, svint16_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_s16,_vg2x1,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single2_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.single.vg2x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmla_single2_f16j13svfloat16x2_tu13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.single.vg2x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single2_f16(uint32_t slice_base, svfloat16x2_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za32,,_f16,,_vg2x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single2_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.single.vg2x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmla_single2_bf16j14svbfloat16x2_tu14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.single.vg2x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single2_bf16(uint32_t slice_base, svbfloat16x2_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za32,,_bf16,,_vg2x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.single.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmla_single2_u16j12svuint16x2_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.single.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za32,,_u16,,_vg2x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.single.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmla_single2_s16j11svint16x2_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.single.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single2_s16(uint32_t slice_base, svint16x2_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za32,,_s16,,_vg2x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single4_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.single.vg2x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmla_single4_f16j13svfloat16x4_tu13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.single.vg2x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single4_f16(uint32_t slice_base, svfloat16x4_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za32,,_f16,,_vg2x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single4_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.single.vg2x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmla_single4_bf16j14svbfloat16x4_tu14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.single.vg2x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single4_bf16(uint32_t slice_base, svbfloat16x4_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za32,,_bf16,,_vg2x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.single.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmla_single4_u16j12svuint16x4_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.single.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za32,,_u16,,_vg2x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.single.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmla_single4_s16j11svint16x4_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.single.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single4_s16(uint32_t slice_base, svint16x4_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za32,,_s16,,_vg2x4)(slice_base, zn, zm); +} + +// +// Multi, indexed +// + +// CHECK-LABEL: @test_svmla_lane1_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.lane.vg2x1.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmla_lane1_f16ju13__SVFloat16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.lane.vg2x1.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane1_f16(uint32_t slice_base, svfloat16_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,,,_f16,_vg2x1)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmla_lane1_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.lane.vg2x1.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_svmla_lane1_bf16ju14__SVBfloat16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.lane.vg2x1.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane1_bf16(uint32_t slice_base, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,,,_bf16,_vg2x1)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmla_lane1_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.lane.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmla_lane1_u16ju12__SVUint16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.lane.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane1_u16(uint32_t slice_base, svuint16_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,,,_u16,_vg2x1)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmla_lane1_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.lane.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmla_lane1_s16ju11__SVInt16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.lane.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane1_s16(uint32_t slice_base, svint16_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,,,_s16,_vg2x1)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmla_lane2_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.lane.vg2x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmla_lane2_f16j13svfloat16x2_tu13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.lane.vg2x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane2_f16(uint32_t slice_base, svfloat16x2_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,,,_f16,_vg2x2)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmla_lane2_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.lane.vg2x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_svmla_lane2_bf16j14svbfloat16x2_tu14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.lane.vg2x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane2_bf16(uint32_t slice_base, svbfloat16x2_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,,,_bf16,_vg2x2)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmla_lane2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.lane.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmla_lane2_u16j12svuint16x2_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.lane.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,,,_u16,_vg2x2)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmla_lane2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.lane.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmla_lane2_s16j11svint16x2_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.lane.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane2_s16(uint32_t slice_base, svint16x2_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,,,_s16,_vg2x2)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmla_lane4_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.lane.vg2x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmla_lane4_f16j13svfloat16x4_tu13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.lane.vg2x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane4_f16(uint32_t slice_base, svfloat16x4_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,,,_f16,_vg2x4)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmla_lane4_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.lane.vg2x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_svmla_lane4_bf16j14svbfloat16x4_tu14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlal.lane.vg2x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane4_bf16(uint32_t slice_base, svbfloat16x4_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,,,_bf16,_vg2x4)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmla_lane4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.lane.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmla_lane4_u16j12svuint16x4_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlal.lane.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,,,_u16,_vg2x4)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmla_lane4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.lane.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmla_lane4_s16j11svint16x4_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlal.lane.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_lane4_s16(uint32_t slice_base, svint16x4_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,,,_s16,_vg2x4)(slice_base, zn, zm, 7); +} diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mlall.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mlall.c new file mode 100644 index 000000000000..fceb16a48260 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mlall.c @@ -0,0 +1,1790 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: aarch64-registered-target + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-i16i64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-i16i64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-i16i64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -S -target-feature +sme-i16i64 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-i16i64 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +#include <arm_sme_draft_spec_subject_to_change.h> + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED,A5) A1##A3##A5 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4,A5) A1##A2##A3##A4##A5 +#endif + +// +// Single x 1 +// + +// MLAL + +// CHECK-LABEL: @test_svmla_single_x1_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.single.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmla_single_x1_s8ju10__SVInt8_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.single.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single_x1_s8(uint32_t slice_base, svint8_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_s8,_vg4x1,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single_x1_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.single.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z24test_svmla_single_x1_s16ju11__SVInt16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.single.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single_x1_s16(uint32_t slice_base, svint16_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za64,_s16,_vg4x1,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_uvmlal_single_x1_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.single.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z24test_uvmlal_single_x1_u8ju11__SVUint8_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.single.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_uvmlal_single_x1_u8(uint32_t slice_base, svuint8_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_u8,_vg4x1,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_uvmlal_single_x1_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.single.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z25test_uvmlal_single_x1_u16ju12__SVUint16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.single.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_uvmlal_single_x1_u16(uint32_t slice_base, svuint16_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za64,_u16,_vg4x1,,)(slice_base, zn, zm); +} + +// MLSL + +// CHECK-LABEL: @test_svmls_single_x1_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.single.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmls_single_x1_s8ju10__SVInt8_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.single.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single_x1_s8(uint32_t slice_base, svint8_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_s8,_vg4x1,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single_x1_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.single.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z24test_svmls_single_x1_s16ju11__SVInt16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.single.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single_x1_s16(uint32_t slice_base, svint16_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za64,_s16,_vg4x1,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_uvmlsl_single_x1_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.single.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z24test_uvmlsl_single_x1_u8ju11__SVUint8_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.single.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_uvmlsl_single_x1_u8(uint32_t slice_base, svuint8_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_u8,_vg4x1,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_uvmlsl_single_x1_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.single.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z25test_uvmlsl_single_x1_u16ju12__SVUint16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.single.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_uvmlsl_single_x1_u16(uint32_t slice_base, svuint16_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za64,_u16,_vg4x1,,)(slice_base, zn, zm); +} + +// SUMLALL + +// CHECK-LABEL: @test_sumlall_single_x1_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.single.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z25test_sumlall_single_x1_s8ju10__SVInt8_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.single.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_sumlall_single_x1_s8(uint32_t slice_base, svint8_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svsumla_za32,_s8,_vg4x1,,)(slice_base, zn, zm); +} + +// USMLALL + +// CHECK-LABEL: @test_usmlall_single_x1_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.single.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z25test_usmlall_single_x1_u8ju11__SVUint8_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.single.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_usmlall_single_x1_u8(uint32_t slice_base, svuint8_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svusmla_za32,_u8,_vg4x1,,)(slice_base, zn, zm); +} + +// +// Single x 2 +// + +// MLAL + +// CHECK-LABEL: @test_svmla_single_x2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.single.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmla_single_x2_s8j10svint8x2_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.single.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single_x2_s8(uint32_t slice_base, svint8x2_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za32,,_s8,,_vg4x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single_x2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.single.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z24test_svmla_single_x2_s16j11svint16x2_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.single.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single_x2_s16(uint32_t slice_base, svint16x2_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za64,,_s16,,_vg4x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single_x2_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.single.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmla_single_x2_u8j11svuint8x2_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.single.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single_x2_u8(uint32_t slice_base, svuint8x2_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za32,,_u8,,_vg4x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single_x2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.single.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z24test_svmla_single_x2_u16j12svuint16x2_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.single.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single_x2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za64,,_u16,,_vg4x2)(slice_base, zn, zm); +} + +// MLSL + +// CHECK-LABEL: @test_svmls_single_x2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.single.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmls_single_x2_s8j10svint8x2_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.single.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single_x2_s8(uint32_t slice_base, svint8x2_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za32,,_s8,,_vg4x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single_x2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.single.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z24test_svmls_single_x2_s16j11svint16x2_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.single.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single_x2_s16(uint32_t slice_base, svint16x2_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za64,,_s16,,_vg4x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single_x2_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.single.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmls_single_x2_u8j11svuint8x2_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.single.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single_x2_u8(uint32_t slice_base, svuint8x2_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za32,,_u8,,_vg4x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single_x2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.single.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z24test_svmls_single_x2_u16j12svuint16x2_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.single.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single_x2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za64,,_u16,,_vg4x2)(slice_base, zn, zm); +} + +// SUMLALL + +// CHECK-LABEL: @test_svsumla_single_x2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sumla.za32.single.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z25test_svsumla_single_x2_s8j10svint8x2_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sumla.za32.single.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svsumla_single_x2_s8(uint32_t slice_base, svint8x2_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svsumla_single_za32,,_s8,,_vg4x2)(slice_base, zn, zm); +} + +// USMLALL + +// CHECK-LABEL: @test_usmlall_single_x2_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.single.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z25test_usmlall_single_x2_u8j11svuint8x2_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.single.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_usmlall_single_x2_u8(uint32_t slice_base, svuint8x2_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svusmla_single_za32,,_u8,,_vg4x2)(slice_base, zn, zm); +} + +// +// Single x 4 +// + +// MLAL + +// CHECK-LABEL: @test_svmla_single_x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.single.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmla_single_x4_s8j10svint8x4_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.single.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single_x4_s8(uint32_t slice_base, svint8x4_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za32,,_s8,,_vg4x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single_x4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.single.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z24test_svmla_single_x4_s16j11svint16x4_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.single.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single_x4_s16(uint32_t slice_base, svint16x4_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za64,,_s16,,_vg4x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single_x4_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.single.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmla_single_x4_u8j11svuint8x4_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.single.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single_x4_u8(uint32_t slice_base, svuint8x4_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za32,,_u8,,_vg4x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmla_single_x4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.single.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z24test_svmla_single_x4_u16j12svuint16x4_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.single.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmla_single_x4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_single_za64,,_u16,,_vg4x4)(slice_base, zn, zm); +} + +// MLSL + +// CHECK-LABEL: @test_svmls_single_x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.single.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmls_single_x4_s8j10svint8x4_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.single.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single_x4_s8(uint32_t slice_base, svint8x4_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za32,,_s8,,_vg4x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single_x4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.single.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z24test_svmls_single_x4_s16j11svint16x4_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.single.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single_x4_s16(uint32_t slice_base, svint16x4_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za64,,_s16,,_vg4x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single_x4_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.single.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmls_single_x4_u8j11svuint8x4_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.single.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single_x4_u8(uint32_t slice_base, svuint8x4_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za32,,_u8,,_vg4x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single_x4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.single.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z24test_svmls_single_x4_u16j12svuint16x4_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.single.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single_x4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za64,,_u16,,_vg4x4)(slice_base, zn, zm); +} + +// SUMLALL + +// CHECK-LABEL: @test_svsumla_single_x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sumla.za32.single.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z25test_svsumla_single_x4_s8j10svint8x4_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sumla.za32.single.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svsumla_single_x4_s8(uint32_t slice_base, svint8x4_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svsumla_single_za32,,_s8,,_vg4x4)(slice_base, zn, zm); +} + +// USMLALL + +// CHECK-LABEL: @test_usmlall_single_x4_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.single.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z25test_usmlall_single_x4_u8j11svuint8x4_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.single.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_usmlall_single_x4_u8(uint32_t slice_base, svuint8x4_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svusmla_single_za32,,_u8,,_vg4x4)(slice_base, zn, zm); +} + +// +// Multi x 2 +// + +// MLAL + +// CHECK-LABEL: @test_mlal_multi_x2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_mlal_multi_x2_s8j10svint8x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlal_multi_x2_s8(uint32_t slice_base, svint8x2_t zn, svint8x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_s8,_vg4x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_mlal_multi_x2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_mlal_multi_x2_s16j11svint16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlal_multi_x2_s16(uint32_t slice_base, svint16x2_t zn, svint16x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za64,_s16,_vg4x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_mlal_multi_x2_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_mlal_multi_x2_u8j11svuint8x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlal_multi_x2_u8(uint32_t slice_base, svuint8x2_t zn, svuint8x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_u8,_vg4x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_mlal_multi_x2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_mlal_multi_x2_u16j12svuint16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlal_multi_x2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za64,_u16,_vg4x2,,)(slice_base, zn, zm); +} + +// MLSL + +// CHECK-LABEL: @test_mlsl_multi_x2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_mlsl_multi_x2_s8j10svint8x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlsl_multi_x2_s8(uint32_t slice_base, svint8x2_t zn, svint8x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_s8,_vg4x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_mlsl_multi_x2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_mlsl_multi_x2_s16j11svint16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlsl_multi_x2_s16(uint32_t slice_base, svint16x2_t zn, svint16x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za64,_s16,_vg4x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_mlsl_multi_x2_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_mlsl_multi_x2_u8j11svuint8x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlsl_multi_x2_u8(uint32_t slice_base, svuint8x2_t zn, svuint8x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_u8,_vg4x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_mlsl_multi_x2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_mlsl_multi_x2_u16j12svuint16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlsl_multi_x2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za64,_u16,_vg4x2,,)(slice_base, zn, zm); +} + +// SUMLALL + +// CHECK-LABEL: @test_sumlal_multi_x2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_sumlal_multi_x2_s8j10svint8x2_t11svuint8x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]]) +// CPP-CHECK-NEXT: ret void +// +void test_sumlal_multi_x2_s8(uint32_t slice_base, svint8x2_t zn, svuint8x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svsumla_za32,_s8,_vg4x2,,)(slice_base, zn, zm); +} + +// USMLALL + +// CHECK-LABEL: @test_usmlal_multi_x2_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_usmlal_multi_x2_u8j11svuint8x2_t10svint8x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_usmlal_multi_x2_u8(uint32_t slice_base, svuint8x2_t zn, svint8x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svusmla_za32,_u8,_vg4x2,,)(slice_base, zn, zm); +} + +// +// Multi x 4 +// + +// MLAL + +// CHECK-LABEL: @test_mlal_multi_x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_mlal_multi_x4_s8j10svint8x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlal_multi_x4_s8(uint32_t slice_base, svint8x4_t zn, svint8x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_s8,_vg4x4,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_mlal_multi_x4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_mlal_multi_x4_s16j11svint16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlal_multi_x4_s16(uint32_t slice_base, svint16x4_t zn, svint16x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za64,_s16,_vg4x4,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_mlal_multi_x4_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_mlal_multi_x4_u8j11svuint8x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlal_multi_x4_u8(uint32_t slice_base, svuint8x4_t zn, svuint8x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za32,_u8,_vg4x4,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_mlal_multi_x4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_mlal_multi_x4_u16j12svuint16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlal_multi_x4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_za64,_u16,_vg4x4,,)(slice_base, zn, zm); +} + +// MLSL + +// CHECK-LABEL: @test_mlsl_multi_x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_mlsl_multi_x4_s8j10svint8x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlsl_multi_x4_s8(uint32_t slice_base, svint8x4_t zn, svint8x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_s8,_vg4x4,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_mlsl_multi_x4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_mlsl_multi_x4_s16j11svint16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlsl_multi_x4_s16(uint32_t slice_base, svint16x4_t zn, svint16x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za64,_s16,_vg4x4,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_mlsl_multi_x4_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_mlsl_multi_x4_u8j11svuint8x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlsl_multi_x4_u8(uint32_t slice_base, svuint8x4_t zn, svuint8x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_u8,_vg4x4,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_mlsl_multi_x4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_mlsl_multi_x4_u16j12svuint16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_mlsl_multi_x4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za64,_u16,_vg4x4,,)(slice_base, zn, zm); +} + +// SUMLALL + +// CHECK-LABEL: @test_sumlal_multi_x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_sumlal_multi_x4_s8j10svint8x4_t11svuint8x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_sumlal_multi_x4_s8(uint32_t slice_base, svint8x4_t zn, svuint8x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svsumla_za32,_s8,_vg4x4,,)(slice_base, zn, zm); +} + +// USMLALL + +// CHECK-LABEL: @test_usmlal_multi_x4_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_usmlal_multi_x4_u8j11svuint8x4_t10svint8x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_usmlal_multi_x4_u8(uint32_t slice_base, svuint8x4_t zn, svint8x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svusmla_za32,_u8,_vg4x4,,)(slice_base, zn, zm); +} + +// +// Indexed x 1 +// + +// SMLAL + +// CHECK-LABEL: @test_smlal_lane_x1_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.lane.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_smlal_lane_x1_s8ju10__SVInt8_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.lane.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_smlal_lane_x1_s8(uint32_t slice_base, svint8_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,_s8,_vg4x1,,)(slice_base, zn, zm, 15); +} + +// CHECK-LABEL: @test_smlal_lane_x1_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.lane.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_smlal_lane_x1_s16ju11__SVInt16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.lane.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_smlal_lane_x1_s16(uint32_t slice_base, svint16_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za64,_s16,_vg4x1,,)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_smlal_lane_x1_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.lane.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_smlal_lane_x1_u8ju11__SVUint8_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.lane.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_smlal_lane_x1_u8(uint32_t slice_base, svuint8_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,_u8,_vg4x1,,)(slice_base, zn, zm, 15); +} + +// CHECK-LABEL: @test_smlal_lane_x1_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.lane.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_smlal_lane_x1_u16ju12__SVUint16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.lane.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_smlal_lane_x1_u16(uint32_t slice_base, svuint16_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za64,_u16,_vg4x1,,)(slice_base, zn, zm, 7); +} + +// SMLSL + +// CHECK-LABEL: @test_smlsl_lane_x1_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.lane.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_smlsl_lane_x1_s8ju10__SVInt8_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.lane.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_smlsl_lane_x1_s8(uint32_t slice_base, svint8_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,_s8,_vg4x1,,)(slice_base, zn, zm, 15); +} + +// CHECK-LABEL: @test_smlsl_lane_x1_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.lane.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_smlsl_lane_x1_s16ju11__SVInt16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.lane.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_smlsl_lane_x1_s16(uint32_t slice_base, svint16_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za64,_s16,_vg4x1,,)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_smlsl_lane_x1_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.lane.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_smlsl_lane_x1_u8ju11__SVUint8_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.lane.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_smlsl_lane_x1_u8(uint32_t slice_base, svuint8_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,_u8,_vg4x1,,)(slice_base, zn, zm, 15); +} + +// CHECK-LABEL: @test_smlsl_lane_x1_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.lane.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_smlsl_lane_x1_u16ju12__SVUint16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.lane.vg4x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_smlsl_lane_x1_u16(uint32_t slice_base, svuint16_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za64,_u16,_vg4x1,,)(slice_base, zn, zm, 7); +} + +// SUMLALL + +// CHECK-LABEL: @test_sumlall_lane_x1_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sumla.za32.lane.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_sumlall_lane_x1_s8ju10__SVInt8_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sumla.za32.lane.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_sumlall_lane_x1_s8(uint32_t slice_base, svint8_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svsumla_lane_za32,_s8,_vg4x1,,)(slice_base, zn, zm, 15); +} + +// USMLALL + +// CHECK-LABEL: @test_usmlall_lane_x1_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.lane.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_usmlall_lane_x1_u8ju11__SVUint8_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.lane.vg4x1.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_usmlall_lane_x1_u8(uint32_t slice_base, svuint8_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svusmla_lane_za32,_u8,_vg4x1,,)(slice_base, zn, zm, 15); +} + +// +// Indexed x 2 +// + +// SMLAL + +// CHECK-LABEL: @test_smlal_lane_x2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.lane.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_smlal_lane_x2_s8j10svint8x2_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.lane.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_smlal_lane_x2_s8(uint32_t slice_base, svint8x2_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,_s8,_vg4x2,,)(slice_base, zn, zm, 15); +} + +// CHECK-LABEL: @test_smlal_lane_x2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.lane.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_smlal_lane_x2_s16j11svint16x2_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.lane.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_smlal_lane_x2_s16(uint32_t slice_base, svint16x2_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za64,_s16,_vg4x2,,)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_smlal_lane_x2_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.lane.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_smlal_lane_x2_u8j11svuint8x2_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.lane.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_smlal_lane_x2_u8(uint32_t slice_base, svuint8x2_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,_u8,_vg4x2,,)(slice_base, zn, zm, 15); +} + +// CHECK-LABEL: @test_smlal_lane_x2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.lane.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_smlal_lane_x2_u16j12svuint16x2_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.lane.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_smlal_lane_x2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za64,_u16,_vg4x2,,)(slice_base, zn, zm, 7); +} + +// SMLSL + +// CHECK-LABEL: @test_smlsl_lane_x2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.lane.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_smlsl_lane_x2_s8j10svint8x2_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.lane.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_smlsl_lane_x2_s8(uint32_t slice_base, svint8x2_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,_s8,_vg4x2,,)(slice_base, zn, zm, 15); +} + +// CHECK-LABEL: @test_smlsl_lane_x2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.lane.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_smlsl_lane_x2_s16j11svint16x2_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.lane.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_smlsl_lane_x2_s16(uint32_t slice_base, svint16x2_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za64,_s16,_vg4x2,,)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_smlsl_lane_x2_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.lane.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_smlsl_lane_x2_u8j11svuint8x2_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.lane.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_smlsl_lane_x2_u8(uint32_t slice_base, svuint8x2_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,_u8,_vg4x2,,)(slice_base, zn, zm, 15); +} + +// CHECK-LABEL: @test_smlsl_lane_x2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.lane.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_smlsl_lane_x2_u16j12svuint16x2_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.lane.vg4x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_smlsl_lane_x2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za64,_u16,_vg4x2,,)(slice_base, zn, zm, 7); +} + +// SUMLALL + +// CHECK-LABEL: @test_sumlall_lane_x2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sumla.za32.lane.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_sumlall_lane_x2_s8j10svint8x2_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sumla.za32.lane.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_sumlall_lane_x2_s8(uint32_t slice_base, svint8x2_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svsumla_lane_za32,_s8,_vg4x2,,)(slice_base, zn, zm, 15); +} + +// CHECK-LABEL: @test_usmlall_lane_x2_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.lane.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_usmlall_lane_x2_u8j11svuint8x2_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.lane.vg4x2.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_usmlall_lane_x2_u8(uint32_t slice_base, svuint8x2_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svusmla_lane_za32,_u8,_vg4x2,,)(slice_base, zn, zm, 15); +} + +// +// Indexed x 4 +// + +// MLAL + +// CHECK-LABEL: @test_smlal_lane_x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.lane.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_smlal_lane_x4_s8j10svint8x4_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za32.lane.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_smlal_lane_x4_s8(uint32_t slice_base, svint8x4_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,_s8,_vg4x4,,)(slice_base, zn, zm, 15); +} + +// CHECK-LABEL: @test_smlal_lane_x4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.lane.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_smlal_lane_x4_s16j11svint16x4_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smla.za64.lane.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_smlal_lane_x4_s16(uint32_t slice_base, svint16x4_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za64,_s16,_vg4x4,,)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_smlal_lane_x4_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.lane.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_smlal_lane_x4_u8j11svuint8x4_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za32.lane.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_smlal_lane_x4_u8(uint32_t slice_base, svuint8x4_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za32,_u8,_vg4x4,,)(slice_base, zn, zm, 15); +} + +// CHECK-LABEL: @test_smlal_lane_x4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.lane.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_smlal_lane_x4_u16j12svuint16x4_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umla.za64.lane.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_smlal_lane_x4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmla_lane_za64,_u16,_vg4x4,,)(slice_base, zn, zm, 7); +} + +// MLSL + +// CHECK-LABEL: @test_smlsl_lane_x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.lane.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_smlsl_lane_x4_s8j10svint8x4_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za32.lane.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_smlsl_lane_x4_s8(uint32_t slice_base, svint8x4_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,_s8,_vg4x4,,)(slice_base, zn, zm, 15); +} + +// CHECK-LABEL: @test_smlsl_lane_x4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.lane.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_smlsl_lane_x4_s16j11svint16x4_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smls.za64.lane.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_smlsl_lane_x4_s16(uint32_t slice_base, svint16x4_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za64,_s16,_vg4x4,,)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_smlsl_lane_x4_u8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.lane.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_smlsl_lane_x4_u8j11svuint8x4_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za32.lane.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_smlsl_lane_x4_u8(uint32_t slice_base, svuint8x4_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,_u8,_vg4x4,,)(slice_base, zn, zm, 15); +} + +// CHECK-LABEL: @test_smlsl_lane_x4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.lane.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_smlsl_lane_x4_u16j12svuint16x4_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umls.za64.lane.vg4x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_smlsl_lane_x4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za64,_u16,_vg4x4,,)(slice_base, zn, zm, 7); +} + +// SUMLALL + +// CHECK-LABEL: @test_sumlall_lane_x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.sumla.za32.lane.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_sumlall_lane_x4_s8j10svint8x4_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sumla.za32.lane.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_sumlall_lane_x4_s8(uint32_t slice_base, svint8x4_t zn, svuint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svsumla_lane_za32,_s8,_vg4x4,,)(slice_base, zn, zm, 15); +} + +// USMLALL + +// CHECK-LABEL: @test_usmlall_lane_x4_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.lane.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_usmlall_lane_x4_s8j11svuint8x4_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usmla.za32.lane.vg4x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 15) +// CPP-CHECK-NEXT: ret void +// +void test_usmlall_lane_x4_s8(uint32_t slice_base, svuint8x4_t zn, svint8_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svusmla_lane_za32,_u8,_vg4x4,,)(slice_base, zn, zm, 15); +} diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mls.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mls.c new file mode 100644 index 000000000000..6830a399e91d --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mls.c @@ -0,0 +1,292 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: aarch64-registered-target + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +#include <arm_sme_draft_spec_subject_to_change.h> + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED,A5) A1##A3##A5 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4,A5) A1##A2##A3##A4##A5 +#endif + +// +// Multi, multi +// CHECK-LABEL: @test_svmls2_f32( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZM]], i64 4) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.vg1x2.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmls2_f32j13svfloat32x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZM]], i64 4) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.vg1x2.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls2_f32(uint32_t slice_base, svfloat32x2_t zn, svfloat32x2_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmls_za32,,_f32,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls4_f32( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM]], i64 4) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM]], i64 12) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.vg1x4.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], <vscale x 4 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmls4_f32j13svfloat32x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM]], i64 4) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZM]], i64 12) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.vg1x4.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], <vscale x 4 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls4_f32(uint32_t slice_base, svfloat32x4_t zn, svfloat32x4_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmls_za32,,_f32,,_vg1x4)(slice_base, zn, zm); +} + +// +// Multi, single +// CHECK-LABEL: @test_svmls_single2_f32( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.single.vg1x2.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmls_single2_f32j13svfloat32x2_tu13__SVFloat32_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.single.vg1x2.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single2_f32(uint32_t slice_base, svfloat32x2_t zn, svfloat32_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmls_single_za32,,_f32,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single4_f32( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.single.vg1x4.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmls_single4_f32j13svfloat32x4_tu13__SVFloat32_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.single.vg1x4.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single4_f32(uint32_t slice_base, svfloat32x4_t zn, svfloat32_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmls_single_za32,,_f32,,_vg1x4)(slice_base, zn, zm); +} + +// +// Multi, indexed +// CHECK-LABEL: @test_svmls_lane2_f32( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.lane.vg1x2.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmls_lane2_f32j13svfloat32x2_tu13__SVFloat32_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.lane.vg1x2.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane2_f32(uint32_t slice_base, svfloat32x2_t zn, svfloat32_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmls_lane_za32,,_f32,,_vg1x2)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svmls_lane4_f32( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.lane.vg1x4.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmls_lane4_f32j13svfloat32x4_tu13__SVFloat32_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[ZN]], i64 12) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.lane.vg1x4.nxv4f32(i32 [[SLICE_BASE:%.*]], <vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane4_f32(uint32_t slice_base, svfloat32x4_t zn, svfloat32_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmls_lane_za32,,_f32,,_vg1x4)(slice_base, zn, zm, 3); +} + +// +// Multi, multi +// CHECK-LABEL: @test_svmls2_f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZM]], i64 2) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.vg1x2.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmls2_f64j13svfloat64x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZM]], i64 2) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.vg1x2.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls2_f64(uint32_t slice_base, svfloat64x2_t zn, svfloat64x2_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmls_za64,,_f64,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls4_f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM]], i64 2) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM]], i64 4) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM]], i64 6) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.vg1x4.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], <vscale x 2 x double> [[TMP5]], <vscale x 2 x double> [[TMP6]], <vscale x 2 x double> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmls4_f64j13svfloat64x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM]], i64 2) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM]], i64 4) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZM]], i64 6) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.vg1x4.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], <vscale x 2 x double> [[TMP5]], <vscale x 2 x double> [[TMP6]], <vscale x 2 x double> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls4_f64(uint32_t slice_base, svfloat64x4_t zn, svfloat64x4_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmls_za64,,_f64,,_vg1x4)(slice_base, zn, zm); +} + +// +// Multi, single +// CHECK-LABEL: @test_svmls_single2_f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN]], i64 2) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.single.vg1x2.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmls_single2_f64j13svfloat64x2_tu13__SVFloat64_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN]], i64 2) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.single.vg1x2.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single2_f64(uint32_t slice_base, svfloat64x2_t zn, svfloat64_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmls_single_za64,,_f64,,_vg1x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single4_f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 6) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.single.vg1x4.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmls_single4_f64j13svfloat64x4_tu13__SVFloat64_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 6) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.single.vg1x4.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single4_f64(uint32_t slice_base, svfloat64x4_t zn, svfloat64_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmls_single_za64,,_f64,,_vg1x4)(slice_base, zn, zm); +} + +// +// Multi, indexed +// CHECK-LABEL: @test_svmls_lane2_f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN]], i64 2) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.lane.vg1x2.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[ZM:%.*]], i32 1) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmls_lane2_f64j13svfloat64x2_tu13__SVFloat64_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[ZN]], i64 2) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.lane.vg1x2.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[ZM:%.*]], i32 1) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane2_f64(uint32_t slice_base, svfloat64x2_t zn, svfloat64_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmls_lane_za64,,_f64,,_vg1x2)(slice_base, zn, zm, 1); +} + +// CHECK-LABEL: @test_svmls_lane4_f64( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 6) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.lane.vg1x4.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[ZM:%.*]], i32 1) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmls_lane4_f64j13svfloat64x4_tu13__SVFloat64_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[ZN]], i64 6) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmls.lane.vg1x4.nxv2f64(i32 [[SLICE_BASE:%.*]], <vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[ZM:%.*]], i32 1) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane4_f64(uint32_t slice_base, svfloat64x4_t zn, svfloat64_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svmls_lane_za64,,_f64,,_vg1x4)(slice_base, zn, zm, 1); +} diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mlsl.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mlsl.c new file mode 100644 index 000000000000..0a87d97f649c --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mlsl.c @@ -0,0 +1,696 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: aarch64-registered-target + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +#include <arm_sme_draft_spec_subject_to_change.h> + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED,A5) A1##A3##A5 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4,A5) A1##A2##A3##A4##A5 +#endif + +// +// Multi, multi +// CHECK-LABEL: @test_svmls2_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.vg2x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmls2_f16j13svfloat16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.vg2x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls2_f16(uint32_t slice_base, svfloat16x2_t zn, svfloat16x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_f16,_vg2x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls2_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.vg2x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z16test_svmls2_bf16j14svbfloat16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.vg2x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls2_bf16(uint32_t slice_base, svbfloat16x2_t zn, svbfloat16x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_bf16,_vg2x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmls2_u16j12svuint16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_u16,_vg2x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmls2_s16j11svint16x2_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls2_s16(uint32_t slice_base, svint16x2_t zn, svint16x2_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_s16,_vg2x2,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls4_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.vg2x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], <vscale x 8 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmls4_f16j13svfloat16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.vg2x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], <vscale x 8 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls4_f16(uint32_t slice_base, svfloat16x4_t zn, svfloat16x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_f16,_vg2x4,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls4_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.vg2x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z16test_svmls4_bf16j14svbfloat16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.vg2x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls4_bf16(uint32_t slice_base, svbfloat16x4_t zn, svbfloat16x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_bf16,_vg2x4,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmls4_u16j12svuint16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_u16,_vg2x4,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z15test_svmls4_s16j11svint16x4_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16) +// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls4_s16(uint32_t slice_base, svint16x4_t zn, svint16x4_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_s16,_vg2x4,,)(slice_base, zn, zm); +} + +// +// Multi, single +// CHECK-LABEL: @test_svmls_single1_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.single.vg2x1.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmls_single1_f16ju13__SVFloat16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.single.vg2x1.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single1_f16(uint32_t slice_base, svfloat16_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_f16,_vg2x1,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single1_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.single.vg2x1.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmls_single1_bf16ju14__SVBfloat16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.single.vg2x1.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single1_bf16(uint32_t slice_base, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_bf16,_vg2x1,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single1_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.single.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmls_single1_u16ju12__SVUint16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.single.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single1_u16(uint32_t slice_base, svuint16_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_u16,_vg2x1,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single1_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.single.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmls_single1_s16ju11__SVInt16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.single.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single1_s16(uint32_t slice_base, svint16_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_za32,_s16,_vg2x1,,)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single2_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.single.vg2x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmls_single2_f16j13svfloat16x2_tu13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.single.vg2x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single2_f16(uint32_t slice_base, svfloat16x2_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za32,,_f16,,_vg2x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single2_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.single.vg2x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmls_single2_bf16j14svbfloat16x2_tu14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.single.vg2x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single2_bf16(uint32_t slice_base, svbfloat16x2_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za32,,_bf16,,_vg2x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.single.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmls_single2_u16j12svuint16x2_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.single.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za32,,_u16,,_vg2x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.single.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmls_single2_s16j11svint16x2_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.single.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single2_s16(uint32_t slice_base, svint16x2_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za32,,_s16,,_vg2x2)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single4_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.single.vg2x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmls_single4_f16j13svfloat16x4_tu13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.single.vg2x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single4_f16(uint32_t slice_base, svfloat16x4_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za32,,_f16,,_vg2x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single4_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.single.vg2x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z23test_svmls_single4_bf16j14svbfloat16x4_tu14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.single.vg2x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single4_bf16(uint32_t slice_base, svbfloat16x4_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za32,,_bf16,,_vg2x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.single.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmls_single4_u16j12svuint16x4_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.single.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za32,,_u16,,_vg2x4)(slice_base, zn, zm); +} + +// CHECK-LABEL: @test_svmls_single4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.single.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z22test_svmls_single4_s16j11svint16x4_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.single.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]]) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_single4_s16(uint32_t slice_base, svint16x4_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_single_za32,,_s16,,_vg2x4)(slice_base, zn, zm); +} + +// +// Multi, indexed +// + +// CHECK-LABEL: @test_svmls_lane1_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.lane.vg2x1.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmls_lane1_f16ju13__SVFloat16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.lane.vg2x1.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane1_f16(uint32_t slice_base, svfloat16_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,,_f16,,_vg2x1)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmls_lane1_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.lane.vg2x1.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_svmls_lane1_bf16ju14__SVBfloat16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.lane.vg2x1.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane1_bf16(uint32_t slice_base, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,,_bf16,,_vg2x1)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmls_lane1_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.lane.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmls_lane1_u16ju12__SVUint16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.lane.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane1_u16(uint32_t slice_base, svuint16_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,,_u16,,_vg2x1)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmls_lane1_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.lane.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmls_lane1_s16ju11__SVInt16_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.lane.vg2x1.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane1_s16(uint32_t slice_base, svint16_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,,_s16,,_vg2x1)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmls_lane2_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.lane.vg2x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmls_lane2_f16j13svfloat16x2_tu13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.lane.vg2x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane2_f16(uint32_t slice_base, svfloat16x2_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,,_f16,,_vg2x2)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmls_lane2_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.lane.vg2x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_svmls_lane2_bf16j14svbfloat16x2_tu14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.lane.vg2x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane2_bf16(uint32_t slice_base, svbfloat16x2_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,,_bf16,,_vg2x2)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmls_lane2_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.lane.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmls_lane2_u16j12svuint16x2_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.lane.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane2_u16(uint32_t slice_base, svuint16x2_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,,_u16,,_vg2x2)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmls_lane2_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.lane.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmls_lane2_s16j11svint16x2_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.lane.vg2x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane2_s16(uint32_t slice_base, svint16x2_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,,_s16,,_vg2x2)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmls_lane4_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.lane.vg2x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmls_lane4_f16j13svfloat16x4_tu13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.lane.vg2x4.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane4_f16(uint32_t slice_base, svfloat16x4_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,,_f16,,_vg2x4)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmls_lane4_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.lane.vg2x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z21test_svmls_lane4_bf16j14svbfloat16x4_tu14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fmlsl.lane.vg2x4.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane4_bf16(uint32_t slice_base, svbfloat16x4_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,,_bf16,,_vg2x4)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmls_lane4_u16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.lane.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmls_lane4_u16j12svuint16x4_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.umlsl.lane.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane4_u16(uint32_t slice_base, svuint16x4_t zn, svuint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,,_u16,,_vg2x4)(slice_base, zn, zm, 7); +} + +// CHECK-LABEL: @test_svmls_lane4_s16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.lane.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z20test_svmls_lane4_s16j11svint16x4_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.smlsl.lane.vg2x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 7) +// CPP-CHECK-NEXT: ret void +// +void test_svmls_lane4_s16(uint32_t slice_base, svint16x4_t zn, svint16_t zm) __arm_streaming __arm_shared_za +{ + SVE_ACLE_FUNC(svmls_lane_za32,,_s16,,_vg2x4)(slice_base, zn, zm, 7); +} diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vdot.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vdot.c new file mode 100644 index 000000000000..fb313d4cebd7 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vdot.c @@ -0,0 +1,222 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: aarch64-registered-target +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-i16i64 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-i16i64 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-i16i64 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-i16i64 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-i16i64 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +#include <arm_sme_draft_spec_subject_to_change.h> + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3) A1##A2##A3 +#endif + +// CHECK-LABEL: @test_svvdot_lane_za32_bf16_vg1x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fvdot.lane.za32.vg1x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svvdot_lane_za32_bf16_vg1x2j14svbfloat16x2_tu14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fvdot.lane.za32.vg1x2.nxv8bf16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svvdot_lane_za32_bf16_vg1x2(uint32_t slice_base, svbfloat16x2_t zn, svbfloat16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svvdot_lane_za32,_bf16,_vg1x2)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svvdot_lane_za32_f16_vg1x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.fvdot.lane.za32.vg1x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svvdot_lane_za32_f16_vg1x2j13svfloat16x2_tu13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fvdot.lane.za32.vg1x2.nxv8f16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svvdot_lane_za32_f16_vg1x2(uint32_t slice_base, svfloat16x2_t zn, svfloat16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svvdot_lane_za32,_f16,_vg1x2)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svvdot_lane_za32_s16_vg1x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.svdot.lane.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svvdot_lane_za32_s16_vg1x2j11svint16x2_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.svdot.lane.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svvdot_lane_za32_s16_vg1x2(uint32_t slice_base, svint16x2_t zn, svint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svvdot_lane_za32,_s16,_vg1x2)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svvdot_lane_za32_u16_vg1x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.uvdot.lane.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svvdot_lane_za32_u16_vg1x2j12svuint16x2_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.uvdot.lane.za32.vg1x2.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svvdot_lane_za32_u16_vg1x2(uint32_t slice_base, svuint16x2_t zn, svuint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svvdot_lane_za32,_u16,_vg1x2)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svvdot_lane_za32_s8_vg1x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.svdot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svvdot_lane_za32_s8_vg1x4j10svint8x4_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.svdot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svvdot_lane_za32_s8_vg1x4(uint32_t slice_base, svint8x4_t zn, svint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svvdot_lane_za32,_s8,_vg1x4)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svvdot_lane_za32_u8_vg1x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.uvdot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z30test_svvdot_lane_za32_u8_vg1x4j11svuint8x4_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.uvdot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svvdot_lane_za32_u8_vg1x4(uint32_t slice_base, svuint8x4_t zn, svuint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svvdot_lane_za32,_u8,_vg1x4)(slice_base, zn, zm, 3); +} + +// CHECK-LABEL: @test_svvdot_lane_za64_s16_vg1x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.svdot.lane.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 1) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svvdot_lane_za64_s16_vg1x4j11svint16x4_tu11__SVInt16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.svdot.lane.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 1) +// CPP-CHECK-NEXT: ret void +// +void test_svvdot_lane_za64_s16_vg1x4(uint32_t slice_base, svint16x4_t zn, svint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svvdot_lane_za64,_s16,_vg1x4)(slice_base, zn, zm, 1); +} + +// CHECK-LABEL: @test_svvdot_lane_za64_u16_vg1x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.uvdot.lane.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 1) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z31test_svvdot_lane_za64_u16_vg1x4j12svuint16x4_tu12__SVUint16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZN]], i64 24) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.uvdot.lane.za64.vg1x4.nxv8i16(i32 [[SLICE_BASE:%.*]], <vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]], i32 1) +// CPP-CHECK-NEXT: ret void +// +void test_svvdot_lane_za64_u16_vg1x4(uint32_t slice_base, svuint16x4_t zn, svuint16_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svvdot_lane_za64,_u16,_vg1x4)(slice_base, zn, zm, 1); +} + + +// CHECK-LABEL: @test_svsuvdot_lane_za32_s8_vg1x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.suvdot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svsuvdot_lane_za32_s8_vg1x4j10svint8x4_tu10__SVInt8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.suvdot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svsuvdot_lane_za32_s8_vg1x4(uint32_t slice_base, svint8x4_t zn, svint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svsuvdot_lane_za32,_s8,_vg1x4)(slice_base, zn, zm, 3); +} + + +// CHECK-LABEL: @test_svusvdot_lane_za32_u8_vg1x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CHECK-NEXT: tail call void @llvm.aarch64.sme.usvdot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CHECK-NEXT: ret void +// +// CPP-CHECK-LABEL: @_Z32test_svusvdot_lane_za32_u8_vg1x4j11svuint8x4_tu11__SVUint8_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZN]], i64 48) +// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.usvdot.lane.za32.vg1x4.nxv16i8(i32 [[SLICE_BASE:%.*]], <vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]], i32 3) +// CPP-CHECK-NEXT: ret void +// +void test_svusvdot_lane_za32_u8_vg1x4(uint32_t slice_base, svuint8x4_t zn, svuint8_t zm) __arm_streaming __arm_shared_za { + SVE_ACLE_FUNC(svusvdot_lane_za32,_u8,_vg1x4)(slice_base, zn, zm, 3); +} + diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c index 7a25d31de013..6f1231e776aa 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c @@ -1,9 +1,12 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -DTEST_SME2 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -DTEST_SME2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + #include <arm_sve.h> #ifdef SVE_OVERLOADED_FORMS @@ -13,6 +16,12 @@ #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 #endif +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif + // CHECK-LABEL: @test_svld1_u8_x2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) @@ -31,7 +40,7 @@ // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16) // CPP-CHECK-NEXT: ret <vscale x 32 x i8> [[TMP4]] // -svuint8x2_t test_svld1_u8_x2(svcount_t pn, const uint8_t *base) +svuint8x2_t test_svld1_u8_x2(svcount_t pn, const uint8_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_u8,_x2,)(pn, base); } @@ -54,7 +63,7 @@ svuint8x2_t test_svld1_u8_x2(svcount_t pn, const uint8_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8) // CPP-CHECK-NEXT: ret <vscale x 16 x i16> [[TMP4]] // -svuint16x2_t test_svld1_u16_x2(svcount_t pn, const uint16_t *base) +svuint16x2_t test_svld1_u16_x2(svcount_t pn, const uint16_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_u16,_x2,)(pn, base); } @@ -77,7 +86,7 @@ svuint16x2_t test_svld1_u16_x2(svcount_t pn, const uint16_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4) // CPP-CHECK-NEXT: ret <vscale x 8 x i32> [[TMP4]] // -svuint32x2_t test_svld1_u32_x2(svcount_t pn, const uint32_t *base) +svuint32x2_t test_svld1_u32_x2(svcount_t pn, const uint32_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_u32,_x2,)(pn, base); } @@ -100,7 +109,7 @@ svuint32x2_t test_svld1_u32_x2(svcount_t pn, const uint32_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2) // CPP-CHECK-NEXT: ret <vscale x 4 x i64> [[TMP4]] // -svuint64x2_t test_svld1_u64_x2(svcount_t pn, const uint64_t *base) +svuint64x2_t test_svld1_u64_x2(svcount_t pn, const uint64_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_u64,_x2,)(pn, base); } @@ -131,7 +140,7 @@ svuint64x2_t test_svld1_u64_x2(svcount_t pn, const uint64_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 48) // CPP-CHECK-NEXT: ret <vscale x 64 x i8> [[TMP8]] // -svuint8x4_t test_svld1_u8_x4(svcount_t pn, const uint8_t *base) +svuint8x4_t test_svld1_u8_x4(svcount_t pn, const uint8_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_u8,_x4,)(pn, base); } @@ -162,7 +171,7 @@ svuint8x4_t test_svld1_u8_x4(svcount_t pn, const uint8_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 24) // CPP-CHECK-NEXT: ret <vscale x 32 x i16> [[TMP8]] // -svuint16x4_t test_svld1_u16_x4(svcount_t pn, const uint16_t *base) +svuint16x4_t test_svld1_u16_x4(svcount_t pn, const uint16_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_u16,_x4,)(pn, base); } @@ -193,7 +202,7 @@ svuint16x4_t test_svld1_u16_x4(svcount_t pn, const uint16_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 12) // CPP-CHECK-NEXT: ret <vscale x 16 x i32> [[TMP8]] // -svuint32x4_t test_svld1_u32_x4(svcount_t pn, const uint32_t *base) +svuint32x4_t test_svld1_u32_x4(svcount_t pn, const uint32_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_u32,_x4,)(pn, base); } @@ -224,7 +233,7 @@ svuint32x4_t test_svld1_u32_x4(svcount_t pn, const uint32_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 6) // CPP-CHECK-NEXT: ret <vscale x 8 x i64> [[TMP8]] // -svuint64x4_t test_svld1_u64_x4(svcount_t pn, const uint64_t *base) +svuint64x4_t test_svld1_u64_x4(svcount_t pn, const uint64_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_u64,_x4,)(pn, base); } @@ -247,7 +256,7 @@ svuint64x4_t test_svld1_u64_x4(svcount_t pn, const uint64_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16) // CPP-CHECK-NEXT: ret <vscale x 32 x i8> [[TMP4]] // -svint8x2_t test_svld1_s8_x2(svcount_t pn, const int8_t *base) +svint8x2_t test_svld1_s8_x2(svcount_t pn, const int8_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_s8,_x2,)(pn, base); } @@ -270,7 +279,7 @@ svint8x2_t test_svld1_s8_x2(svcount_t pn, const int8_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8) // CPP-CHECK-NEXT: ret <vscale x 16 x i16> [[TMP4]] // -svint16x2_t test_svld1_s16_x2(svcount_t pn, const int16_t *base) +svint16x2_t test_svld1_s16_x2(svcount_t pn, const int16_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_s16,_x2,)(pn, base); } @@ -293,7 +302,7 @@ svint16x2_t test_svld1_s16_x2(svcount_t pn, const int16_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4) // CPP-CHECK-NEXT: ret <vscale x 8 x i32> [[TMP4]] // -svint32x2_t test_svld1_s32_x2(svcount_t pn, const int32_t *base) +svint32x2_t test_svld1_s32_x2(svcount_t pn, const int32_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_s32,_x2,)(pn, base); } @@ -316,7 +325,7 @@ svint32x2_t test_svld1_s32_x2(svcount_t pn, const int32_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2) // CPP-CHECK-NEXT: ret <vscale x 4 x i64> [[TMP4]] // -svint64x2_t test_svld1_s64_x2(svcount_t pn, const int64_t *base) +svint64x2_t test_svld1_s64_x2(svcount_t pn, const int64_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_s64,_x2,)(pn, base); } @@ -347,7 +356,7 @@ svint64x2_t test_svld1_s64_x2(svcount_t pn, const int64_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 48) // CPP-CHECK-NEXT: ret <vscale x 64 x i8> [[TMP8]] // -svint8x4_t test_svld1_s8_x4(svcount_t pn, const int8_t *base) +svint8x4_t test_svld1_s8_x4(svcount_t pn, const int8_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_s8,_x4,)(pn, base); } @@ -378,7 +387,7 @@ svint8x4_t test_svld1_s8_x4(svcount_t pn, const int8_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 24) // CPP-CHECK-NEXT: ret <vscale x 32 x i16> [[TMP8]] // -svint16x4_t test_svld1_s16_x4(svcount_t pn, const int16_t *base) +svint16x4_t test_svld1_s16_x4(svcount_t pn, const int16_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_s16,_x4,)(pn, base); } @@ -409,7 +418,7 @@ svint16x4_t test_svld1_s16_x4(svcount_t pn, const int16_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 12) // CPP-CHECK-NEXT: ret <vscale x 16 x i32> [[TMP8]] // -svint32x4_t test_svld1_s32_x4(svcount_t pn, const int32_t *base) +svint32x4_t test_svld1_s32_x4(svcount_t pn, const int32_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_s32,_x4,)(pn, base); } @@ -440,7 +449,7 @@ svint32x4_t test_svld1_s32_x4(svcount_t pn, const int32_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 6) // CPP-CHECK-NEXT: ret <vscale x 8 x i64> [[TMP8]] // -svint64x4_t test_svld1_s64_x4(svcount_t pn, const int64_t *base) +svint64x4_t test_svld1_s64_x4(svcount_t pn, const int64_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_s64,_x4,)(pn, base); } @@ -463,7 +472,7 @@ svint64x4_t test_svld1_s64_x4(svcount_t pn, const int64_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], i64 8) // CPP-CHECK-NEXT: ret <vscale x 16 x half> [[TMP4]] // -svfloat16x2_t test_svld1_f16_x2(svcount_t pn, const float16_t *base) +svfloat16x2_t test_svld1_f16_x2(svcount_t pn, const float16_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_f16,_x2,)(pn, base); } @@ -486,7 +495,7 @@ svfloat16x2_t test_svld1_f16_x2(svcount_t pn, const float16_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], i64 4) // CPP-CHECK-NEXT: ret <vscale x 8 x float> [[TMP4]] // -svfloat32x2_t test_svld1_f32_x2(svcount_t pn, const float32_t *base) +svfloat32x2_t test_svld1_f32_x2(svcount_t pn, const float32_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_f32,_x2,)(pn, base); } @@ -509,7 +518,7 @@ svfloat32x2_t test_svld1_f32_x2(svcount_t pn, const float32_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], i64 2) // CPP-CHECK-NEXT: ret <vscale x 4 x double> [[TMP4]] // -svfloat64x2_t test_svld1_f64_x2(svcount_t pn, const float64_t *base) +svfloat64x2_t test_svld1_f64_x2(svcount_t pn, const float64_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_f64,_x2,)(pn, base); } @@ -540,7 +549,7 @@ svfloat64x2_t test_svld1_f64_x2(svcount_t pn, const float64_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]], i64 24) // CPP-CHECK-NEXT: ret <vscale x 32 x half> [[TMP8]] // -svfloat16x4_t test_svld1_f16_x4(svcount_t pn, const float16_t *base) +svfloat16x4_t test_svld1_f16_x4(svcount_t pn, const float16_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_f16,_x4,)(pn, base); } @@ -571,7 +580,7 @@ svfloat16x4_t test_svld1_f16_x4(svcount_t pn, const float16_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]], i64 12) // CPP-CHECK-NEXT: ret <vscale x 16 x float> [[TMP8]] // -svfloat32x4_t test_svld1_f32_x4(svcount_t pn, const float32_t *base) +svfloat32x4_t test_svld1_f32_x4(svcount_t pn, const float32_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_f32,_x4,)(pn, base); } @@ -602,7 +611,7 @@ svfloat32x4_t test_svld1_f32_x4(svcount_t pn, const float32_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP6]], <vscale x 2 x double> [[TMP7]], i64 6) // CPP-CHECK-NEXT: ret <vscale x 8 x double> [[TMP8]] // -svfloat64x4_t test_svld1_f64_x4(svcount_t pn, const float64_t *base) +svfloat64x4_t test_svld1_f64_x4(svcount_t pn, const float64_t *base) ATTR { return SVE_ACLE_FUNC(svld1,_f64,_x4,)(pn, base); } @@ -631,7 +640,7 @@ svfloat64x4_t test_svld1_f64_x4(svcount_t pn, const float64_t *base) // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16) // CPP-CHECK-NEXT: ret <vscale x 32 x i8> [[TMP5]] // -svuint8x2_t test_svld1_vnum_u8_x2(svcount_t pn, const uint8_t *base, int64_t vnum) +svuint8x2_t test_svld1_vnum_u8_x2(svcount_t pn, const uint8_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_u8,_x2,)(pn, base, vnum); } @@ -656,7 +665,7 @@ svuint8x2_t test_svld1_vnum_u8_x2(svcount_t pn, const uint8_t *base, int64_t vnu // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8) // CPP-CHECK-NEXT: ret <vscale x 16 x i16> [[TMP5]] // -svuint16x2_t test_svld1_vnum_u16_x2(svcount_t pn, const uint16_t *base, int64_t vnum) +svuint16x2_t test_svld1_vnum_u16_x2(svcount_t pn, const uint16_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_u16,_x2,)(pn, base, vnum); } @@ -681,7 +690,7 @@ svuint16x2_t test_svld1_vnum_u16_x2(svcount_t pn, const uint16_t *base, int64_t // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4) // CPP-CHECK-NEXT: ret <vscale x 8 x i32> [[TMP5]] // -svuint32x2_t test_svld1_vnum_u32_x2(svcount_t pn, const uint32_t *base, int64_t vnum) +svuint32x2_t test_svld1_vnum_u32_x2(svcount_t pn, const uint32_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_u32,_x2,)(pn, base, vnum); } @@ -706,7 +715,7 @@ svuint32x2_t test_svld1_vnum_u32_x2(svcount_t pn, const uint32_t *base, int64_t // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2) // CPP-CHECK-NEXT: ret <vscale x 4 x i64> [[TMP5]] // -svuint64x2_t test_svld1_vnum_u64_x2(svcount_t pn, const uint64_t *base, int64_t vnum) +svuint64x2_t test_svld1_vnum_u64_x2(svcount_t pn, const uint64_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_u64,_x2,)(pn, base, vnum); } @@ -739,7 +748,7 @@ svuint64x2_t test_svld1_vnum_u64_x2(svcount_t pn, const uint64_t *base, int64_t // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP8]], i64 48) // CPP-CHECK-NEXT: ret <vscale x 64 x i8> [[TMP9]] // -svuint8x4_t test_svld1_vnum_u8_x4(svcount_t pn, const uint8_t *base, int64_t vnum) +svuint8x4_t test_svld1_vnum_u8_x4(svcount_t pn, const uint8_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_u8,_x4,)(pn, base, vnum); } @@ -772,7 +781,7 @@ svuint8x4_t test_svld1_vnum_u8_x4(svcount_t pn, const uint8_t *base, int64_t vnu // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP7]], <vscale x 8 x i16> [[TMP8]], i64 24) // CPP-CHECK-NEXT: ret <vscale x 32 x i16> [[TMP9]] // -svuint16x4_t test_svld1_vnum_u16_x4(svcount_t pn, const uint16_t *base, int64_t vnum) +svuint16x4_t test_svld1_vnum_u16_x4(svcount_t pn, const uint16_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_u16,_x4,)(pn, base, vnum); } @@ -805,7 +814,7 @@ svuint16x4_t test_svld1_vnum_u16_x4(svcount_t pn, const uint16_t *base, int64_t // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], i64 12) // CPP-CHECK-NEXT: ret <vscale x 16 x i32> [[TMP9]] // -svuint32x4_t test_svld1_vnum_u32_x4(svcount_t pn, const uint32_t *base, int64_t vnum) +svuint32x4_t test_svld1_vnum_u32_x4(svcount_t pn, const uint32_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_u32,_x4,)(pn, base, vnum); } @@ -838,7 +847,7 @@ svuint32x4_t test_svld1_vnum_u32_x4(svcount_t pn, const uint32_t *base, int64_t // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP8]], i64 6) // CPP-CHECK-NEXT: ret <vscale x 8 x i64> [[TMP9]] // -svuint64x4_t test_svld1_vnum_u64_x4(svcount_t pn, const uint64_t *base, int64_t vnum) +svuint64x4_t test_svld1_vnum_u64_x4(svcount_t pn, const uint64_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_u64,_x4,)(pn, base, vnum); } @@ -863,7 +872,7 @@ svuint64x4_t test_svld1_vnum_u64_x4(svcount_t pn, const uint64_t *base, int64_t // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16) // CPP-CHECK-NEXT: ret <vscale x 32 x i8> [[TMP5]] // -svint8x2_t test_svld1_vnum_s8_x2(svcount_t pn, const int8_t *base, int64_t vnum) +svint8x2_t test_svld1_vnum_s8_x2(svcount_t pn, const int8_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_s8,_x2,)(pn, base, vnum); } @@ -888,7 +897,7 @@ svint8x2_t test_svld1_vnum_s8_x2(svcount_t pn, const int8_t *base, int64_t vnum) // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8) // CPP-CHECK-NEXT: ret <vscale x 16 x i16> [[TMP5]] // -svint16x2_t test_svld1_vnum_s16_x2(svcount_t pn, const int16_t *base, int64_t vnum) +svint16x2_t test_svld1_vnum_s16_x2(svcount_t pn, const int16_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_s16,_x2,)(pn, base, vnum); } @@ -913,7 +922,7 @@ svint16x2_t test_svld1_vnum_s16_x2(svcount_t pn, const int16_t *base, int64_t vn // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4) // CPP-CHECK-NEXT: ret <vscale x 8 x i32> [[TMP5]] // -svint32x2_t test_svld1_vnum_s32_x2(svcount_t pn, const int32_t *base, int64_t vnum) +svint32x2_t test_svld1_vnum_s32_x2(svcount_t pn, const int32_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_s32,_x2,)(pn, base, vnum); } @@ -938,7 +947,7 @@ svint32x2_t test_svld1_vnum_s32_x2(svcount_t pn, const int32_t *base, int64_t vn // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2) // CPP-CHECK-NEXT: ret <vscale x 4 x i64> [[TMP5]] // -svint64x2_t test_svld1_vnum_s64_x2(svcount_t pn, const int64_t *base, int64_t vnum) +svint64x2_t test_svld1_vnum_s64_x2(svcount_t pn, const int64_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_s64,_x2,)(pn, base, vnum); } @@ -971,7 +980,7 @@ svint64x2_t test_svld1_vnum_s64_x2(svcount_t pn, const int64_t *base, int64_t vn // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP8]], i64 48) // CPP-CHECK-NEXT: ret <vscale x 64 x i8> [[TMP9]] // -svint8x4_t test_svld1_vnum_s8_x4(svcount_t pn, const int8_t *base, int64_t vnum) +svint8x4_t test_svld1_vnum_s8_x4(svcount_t pn, const int8_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_s8,_x4,)(pn, base, vnum); } @@ -1004,7 +1013,7 @@ svint8x4_t test_svld1_vnum_s8_x4(svcount_t pn, const int8_t *base, int64_t vnum) // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP7]], <vscale x 8 x i16> [[TMP8]], i64 24) // CPP-CHECK-NEXT: ret <vscale x 32 x i16> [[TMP9]] // -svint16x4_t test_svld1_vnum_s16_x4(svcount_t pn, const int16_t *base, int64_t vnum) +svint16x4_t test_svld1_vnum_s16_x4(svcount_t pn, const int16_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_s16,_x4,)(pn, base, vnum); } @@ -1037,7 +1046,7 @@ svint16x4_t test_svld1_vnum_s16_x4(svcount_t pn, const int16_t *base, int64_t vn // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], i64 12) // CPP-CHECK-NEXT: ret <vscale x 16 x i32> [[TMP9]] // -svint32x4_t test_svld1_vnum_s32_x4(svcount_t pn, const int32_t *base, int64_t vnum) +svint32x4_t test_svld1_vnum_s32_x4(svcount_t pn, const int32_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_s32,_x4,)(pn, base, vnum); } @@ -1070,7 +1079,7 @@ svint32x4_t test_svld1_vnum_s32_x4(svcount_t pn, const int32_t *base, int64_t vn // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP8]], i64 6) // CPP-CHECK-NEXT: ret <vscale x 8 x i64> [[TMP9]] // -svint64x4_t test_svld1_vnum_s64_x4(svcount_t pn, const int64_t *base, int64_t vnum) +svint64x4_t test_svld1_vnum_s64_x4(svcount_t pn, const int64_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_s64,_x4,)(pn, base, vnum); } @@ -1095,7 +1104,7 @@ svint64x4_t test_svld1_vnum_s64_x4(svcount_t pn, const int64_t *base, int64_t vn // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], i64 8) // CPP-CHECK-NEXT: ret <vscale x 16 x half> [[TMP5]] // -svfloat16x2_t test_svld1_vnum_f16_x2(svcount_t pn, const float16_t *base, int64_t vnum) +svfloat16x2_t test_svld1_vnum_f16_x2(svcount_t pn, const float16_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_f16,_x2,)(pn, base, vnum); } @@ -1120,7 +1129,7 @@ svfloat16x2_t test_svld1_vnum_f16_x2(svcount_t pn, const float16_t *base, int64_ // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], i64 4) // CPP-CHECK-NEXT: ret <vscale x 8 x float> [[TMP5]] // -svfloat32x2_t test_svld1_vnum_f32_x2(svcount_t pn, const float32_t *base, int64_t vnum) +svfloat32x2_t test_svld1_vnum_f32_x2(svcount_t pn, const float32_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_f32,_x2,)(pn, base, vnum); } @@ -1145,7 +1154,7 @@ svfloat32x2_t test_svld1_vnum_f32_x2(svcount_t pn, const float32_t *base, int64_ // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], i64 2) // CPP-CHECK-NEXT: ret <vscale x 4 x double> [[TMP5]] // -svfloat64x2_t test_svld1_vnum_f64_x2(svcount_t pn, const float64_t *base, int64_t vnum) +svfloat64x2_t test_svld1_vnum_f64_x2(svcount_t pn, const float64_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_f64,_x2,)(pn, base, vnum); } @@ -1178,7 +1187,7 @@ svfloat64x2_t test_svld1_vnum_f64_x2(svcount_t pn, const float64_t *base, int64_ // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP7]], <vscale x 8 x half> [[TMP8]], i64 24) // CPP-CHECK-NEXT: ret <vscale x 32 x half> [[TMP9]] // -svfloat16x4_t test_svld1_vnum_f16_x4(svcount_t pn, const float16_t *base, int64_t vnum) +svfloat16x4_t test_svld1_vnum_f16_x4(svcount_t pn, const float16_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_f16,_x4,)(pn, base, vnum); } @@ -1211,7 +1220,7 @@ svfloat16x4_t test_svld1_vnum_f16_x4(svcount_t pn, const float16_t *base, int64_ // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP7]], <vscale x 4 x float> [[TMP8]], i64 12) // CPP-CHECK-NEXT: ret <vscale x 16 x float> [[TMP9]] // -svfloat32x4_t test_svld1_vnum_f32_x4(svcount_t pn, const float32_t *base, int64_t vnum) +svfloat32x4_t test_svld1_vnum_f32_x4(svcount_t pn, const float32_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_f32,_x4,)(pn, base, vnum); } @@ -1244,7 +1253,7 @@ svfloat32x4_t test_svld1_vnum_f32_x4(svcount_t pn, const float32_t *base, int64_ // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP7]], <vscale x 2 x double> [[TMP8]], i64 6) // CPP-CHECK-NEXT: ret <vscale x 8 x double> [[TMP9]] // -svfloat64x4_t test_svld1_vnum_f64_x4(svcount_t pn, const float64_t *base, int64_t vnum) +svfloat64x4_t test_svld1_vnum_f64_x4(svcount_t pn, const float64_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svld1_vnum,_f64,_x4,)(pn, base, vnum); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ldnt1.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ldnt1.c index 7a0fcde819dc..3f61cc3de139 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ldnt1.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ldnt1.c @@ -1,9 +1,12 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -DTEST_SME2 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -DTEST_SME2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + #include <arm_sve.h> #ifdef SVE_OVERLOADED_FORMS @@ -13,6 +16,12 @@ #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 #endif +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif + // CHECK-LABEL: @test_svldnt1_u8_x2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) @@ -31,7 +40,7 @@ // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16) // CPP-CHECK-NEXT: ret <vscale x 32 x i8> [[TMP4]] // -svuint8x2_t test_svldnt1_u8_x2(svcount_t pn, const uint8_t *base) +svuint8x2_t test_svldnt1_u8_x2(svcount_t pn, const uint8_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_u8,_x2,)(pn, base); } @@ -54,7 +63,7 @@ svuint8x2_t test_svldnt1_u8_x2(svcount_t pn, const uint8_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8) // CPP-CHECK-NEXT: ret <vscale x 16 x i16> [[TMP4]] // -svuint16x2_t test_svldnt1_u16_x2(svcount_t pn, const uint16_t *base) +svuint16x2_t test_svldnt1_u16_x2(svcount_t pn, const uint16_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_u16,_x2,)(pn, base); } @@ -77,7 +86,7 @@ svuint16x2_t test_svldnt1_u16_x2(svcount_t pn, const uint16_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4) // CPP-CHECK-NEXT: ret <vscale x 8 x i32> [[TMP4]] // -svuint32x2_t test_svldnt1_u32_x2(svcount_t pn, const uint32_t *base) +svuint32x2_t test_svldnt1_u32_x2(svcount_t pn, const uint32_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_u32,_x2,)(pn, base); } @@ -100,7 +109,7 @@ svuint32x2_t test_svldnt1_u32_x2(svcount_t pn, const uint32_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2) // CPP-CHECK-NEXT: ret <vscale x 4 x i64> [[TMP4]] // -svuint64x2_t test_svldnt1_u64_x2(svcount_t pn, const uint64_t *base) +svuint64x2_t test_svldnt1_u64_x2(svcount_t pn, const uint64_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_u64,_x2,)(pn, base); } @@ -131,7 +140,7 @@ svuint64x2_t test_svldnt1_u64_x2(svcount_t pn, const uint64_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 48) // CPP-CHECK-NEXT: ret <vscale x 64 x i8> [[TMP8]] // -svuint8x4_t test_svldnt1_u8_x4(svcount_t pn, const uint8_t *base) +svuint8x4_t test_svldnt1_u8_x4(svcount_t pn, const uint8_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_u8,_x4,)(pn, base); } @@ -162,7 +171,7 @@ svuint8x4_t test_svldnt1_u8_x4(svcount_t pn, const uint8_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 24) // CPP-CHECK-NEXT: ret <vscale x 32 x i16> [[TMP8]] // -svuint16x4_t test_svldnt1_u16_x4(svcount_t pn, const uint16_t *base) +svuint16x4_t test_svldnt1_u16_x4(svcount_t pn, const uint16_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_u16,_x4,)(pn, base); } @@ -193,7 +202,7 @@ svuint16x4_t test_svldnt1_u16_x4(svcount_t pn, const uint16_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 12) // CPP-CHECK-NEXT: ret <vscale x 16 x i32> [[TMP8]] // -svuint32x4_t test_svldnt1_u32_x4(svcount_t pn, const uint32_t *base) +svuint32x4_t test_svldnt1_u32_x4(svcount_t pn, const uint32_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_u32,_x4,)(pn, base); } @@ -224,7 +233,7 @@ svuint32x4_t test_svldnt1_u32_x4(svcount_t pn, const uint32_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 6) // CPP-CHECK-NEXT: ret <vscale x 8 x i64> [[TMP8]] // -svuint64x4_t test_svldnt1_u64_x4(svcount_t pn, const uint64_t *base) +svuint64x4_t test_svldnt1_u64_x4(svcount_t pn, const uint64_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_u64,_x4,)(pn, base); } @@ -247,7 +256,7 @@ svuint64x4_t test_svldnt1_u64_x4(svcount_t pn, const uint64_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16) // CPP-CHECK-NEXT: ret <vscale x 32 x i8> [[TMP4]] // -svint8x2_t test_svldnt1_s8_x2(svcount_t pn, const int8_t *base) +svint8x2_t test_svldnt1_s8_x2(svcount_t pn, const int8_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_s8,_x2,)(pn, base); } @@ -270,7 +279,7 @@ svint8x2_t test_svldnt1_s8_x2(svcount_t pn, const int8_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8) // CPP-CHECK-NEXT: ret <vscale x 16 x i16> [[TMP4]] // -svint16x2_t test_svldnt1_s16_x2(svcount_t pn, const int16_t *base) +svint16x2_t test_svldnt1_s16_x2(svcount_t pn, const int16_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_s16,_x2,)(pn, base); } @@ -293,7 +302,7 @@ svint16x2_t test_svldnt1_s16_x2(svcount_t pn, const int16_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4) // CPP-CHECK-NEXT: ret <vscale x 8 x i32> [[TMP4]] // -svint32x2_t test_svldnt1_s32_x2(svcount_t pn, const int32_t *base) +svint32x2_t test_svldnt1_s32_x2(svcount_t pn, const int32_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_s32,_x2,)(pn, base); } @@ -316,7 +325,7 @@ svint32x2_t test_svldnt1_s32_x2(svcount_t pn, const int32_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2) // CPP-CHECK-NEXT: ret <vscale x 4 x i64> [[TMP4]] // -svint64x2_t test_svldnt1_s64_x2(svcount_t pn, const int64_t *base) +svint64x2_t test_svldnt1_s64_x2(svcount_t pn, const int64_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_s64,_x2,)(pn, base); } @@ -347,7 +356,7 @@ svint64x2_t test_svldnt1_s64_x2(svcount_t pn, const int64_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 48) // CPP-CHECK-NEXT: ret <vscale x 64 x i8> [[TMP8]] // -svint8x4_t test_svldnt1_s8_x4(svcount_t pn, const int8_t *base) +svint8x4_t test_svldnt1_s8_x4(svcount_t pn, const int8_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_s8,_x4,)(pn, base); } @@ -378,7 +387,7 @@ svint8x4_t test_svldnt1_s8_x4(svcount_t pn, const int8_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 24) // CPP-CHECK-NEXT: ret <vscale x 32 x i16> [[TMP8]] // -svint16x4_t test_svldnt1_s16_x4(svcount_t pn, const int16_t *base) +svint16x4_t test_svldnt1_s16_x4(svcount_t pn, const int16_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_s16,_x4,)(pn, base); } @@ -409,7 +418,7 @@ svint16x4_t test_svldnt1_s16_x4(svcount_t pn, const int16_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 12) // CPP-CHECK-NEXT: ret <vscale x 16 x i32> [[TMP8]] // -svint32x4_t test_svldnt1_s32_x4(svcount_t pn, const int32_t *base) +svint32x4_t test_svldnt1_s32_x4(svcount_t pn, const int32_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_s32,_x4,)(pn, base); } @@ -440,7 +449,7 @@ svint32x4_t test_svldnt1_s32_x4(svcount_t pn, const int32_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 6) // CPP-CHECK-NEXT: ret <vscale x 8 x i64> [[TMP8]] // -svint64x4_t test_svldnt1_s64_x4(svcount_t pn, const int64_t *base) +svint64x4_t test_svldnt1_s64_x4(svcount_t pn, const int64_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_s64,_x4,)(pn, base); } @@ -463,7 +472,7 @@ svint64x4_t test_svldnt1_s64_x4(svcount_t pn, const int64_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], i64 8) // CPP-CHECK-NEXT: ret <vscale x 16 x half> [[TMP4]] // -svfloat16x2_t test_svldnt1_f16_x2(svcount_t pn, const float16_t *base) +svfloat16x2_t test_svldnt1_f16_x2(svcount_t pn, const float16_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_f16,_x2,)(pn, base); } @@ -486,7 +495,7 @@ svfloat16x2_t test_svldnt1_f16_x2(svcount_t pn, const float16_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], i64 4) // CPP-CHECK-NEXT: ret <vscale x 8 x float> [[TMP4]] // -svfloat32x2_t test_svldnt1_f32_x2(svcount_t pn, const float32_t *base) +svfloat32x2_t test_svldnt1_f32_x2(svcount_t pn, const float32_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_f32,_x2,)(pn, base); } @@ -509,7 +518,7 @@ svfloat32x2_t test_svldnt1_f32_x2(svcount_t pn, const float32_t *base) // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], i64 2) // CPP-CHECK-NEXT: ret <vscale x 4 x double> [[TMP4]] // -svfloat64x2_t test_svldnt1_f64_x2(svcount_t pn, const float64_t *base) +svfloat64x2_t test_svldnt1_f64_x2(svcount_t pn, const float64_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_f64,_x2,)(pn, base); } @@ -540,7 +549,7 @@ svfloat64x2_t test_svldnt1_f64_x2(svcount_t pn, const float64_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]], i64 24) // CPP-CHECK-NEXT: ret <vscale x 32 x half> [[TMP8]] // -svfloat16x4_t test_svldnt1_f16_x4(svcount_t pn, const float16_t *base) +svfloat16x4_t test_svldnt1_f16_x4(svcount_t pn, const float16_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_f16,_x4,)(pn, base); } @@ -571,7 +580,7 @@ svfloat16x4_t test_svldnt1_f16_x4(svcount_t pn, const float16_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]], i64 12) // CPP-CHECK-NEXT: ret <vscale x 16 x float> [[TMP8]] // -svfloat32x4_t test_svldnt1_f32_x4(svcount_t pn, const float32_t *base) +svfloat32x4_t test_svldnt1_f32_x4(svcount_t pn, const float32_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_f32,_x4,)(pn, base); } @@ -602,7 +611,7 @@ svfloat32x4_t test_svldnt1_f32_x4(svcount_t pn, const float32_t *base) // CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP6]], <vscale x 2 x double> [[TMP7]], i64 6) // CPP-CHECK-NEXT: ret <vscale x 8 x double> [[TMP8]] // -svfloat64x4_t test_svldnt1_f64_x4(svcount_t pn, const float64_t *base) +svfloat64x4_t test_svldnt1_f64_x4(svcount_t pn, const float64_t *base) ATTR { return SVE_ACLE_FUNC(svldnt1,_f64,_x4,)(pn, base); } @@ -631,7 +640,7 @@ svfloat64x4_t test_svldnt1_f64_x4(svcount_t pn, const float64_t *base) // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16) // CPP-CHECK-NEXT: ret <vscale x 32 x i8> [[TMP5]] // -svuint8x2_t test_svldnt1_vnum_u8_x2(svcount_t pn, const uint8_t *base, int64_t vnum) +svuint8x2_t test_svldnt1_vnum_u8_x2(svcount_t pn, const uint8_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_u8,_x2,)(pn, base, vnum); } @@ -656,7 +665,7 @@ svuint8x2_t test_svldnt1_vnum_u8_x2(svcount_t pn, const uint8_t *base, int64_t v // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8) // CPP-CHECK-NEXT: ret <vscale x 16 x i16> [[TMP5]] // -svuint16x2_t test_svldnt1_vnum_u16_x2(svcount_t pn, const uint16_t *base, int64_t vnum) +svuint16x2_t test_svldnt1_vnum_u16_x2(svcount_t pn, const uint16_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_u16,_x2,)(pn, base, vnum); } @@ -681,7 +690,7 @@ svuint16x2_t test_svldnt1_vnum_u16_x2(svcount_t pn, const uint16_t *base, int64_ // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4) // CPP-CHECK-NEXT: ret <vscale x 8 x i32> [[TMP5]] // -svuint32x2_t test_svldnt1_vnum_u32_x2(svcount_t pn, const uint32_t *base, int64_t vnum) +svuint32x2_t test_svldnt1_vnum_u32_x2(svcount_t pn, const uint32_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_u32,_x2,)(pn, base, vnum); } @@ -706,7 +715,7 @@ svuint32x2_t test_svldnt1_vnum_u32_x2(svcount_t pn, const uint32_t *base, int64_ // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2) // CPP-CHECK-NEXT: ret <vscale x 4 x i64> [[TMP5]] // -svuint64x2_t test_svldnt1_vnum_u64_x2(svcount_t pn, const uint64_t *base, int64_t vnum) +svuint64x2_t test_svldnt1_vnum_u64_x2(svcount_t pn, const uint64_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_u64,_x2,)(pn, base, vnum); } @@ -739,7 +748,7 @@ svuint64x2_t test_svldnt1_vnum_u64_x2(svcount_t pn, const uint64_t *base, int64_ // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP8]], i64 48) // CPP-CHECK-NEXT: ret <vscale x 64 x i8> [[TMP9]] // -svuint8x4_t test_svldnt1_vnum_u8_x4(svcount_t pn, const uint8_t *base, int64_t vnum) +svuint8x4_t test_svldnt1_vnum_u8_x4(svcount_t pn, const uint8_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_u8,_x4,)(pn, base, vnum); } @@ -772,7 +781,7 @@ svuint8x4_t test_svldnt1_vnum_u8_x4(svcount_t pn, const uint8_t *base, int64_t v // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP7]], <vscale x 8 x i16> [[TMP8]], i64 24) // CPP-CHECK-NEXT: ret <vscale x 32 x i16> [[TMP9]] // -svuint16x4_t test_svldnt1_vnum_u16_x4(svcount_t pn, const uint16_t *base, int64_t vnum) +svuint16x4_t test_svldnt1_vnum_u16_x4(svcount_t pn, const uint16_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_u16,_x4,)(pn, base, vnum); } @@ -805,7 +814,7 @@ svuint16x4_t test_svldnt1_vnum_u16_x4(svcount_t pn, const uint16_t *base, int64_ // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], i64 12) // CPP-CHECK-NEXT: ret <vscale x 16 x i32> [[TMP9]] // -svuint32x4_t test_svldnt1_vnum_u32_x4(svcount_t pn, const uint32_t *base, int64_t vnum) +svuint32x4_t test_svldnt1_vnum_u32_x4(svcount_t pn, const uint32_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_u32,_x4,)(pn, base, vnum); } @@ -838,7 +847,7 @@ svuint32x4_t test_svldnt1_vnum_u32_x4(svcount_t pn, const uint32_t *base, int64_ // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP8]], i64 6) // CPP-CHECK-NEXT: ret <vscale x 8 x i64> [[TMP9]] // -svuint64x4_t test_svldnt1_vnum_u64_x4(svcount_t pn, const uint64_t *base, int64_t vnum) +svuint64x4_t test_svldnt1_vnum_u64_x4(svcount_t pn, const uint64_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_u64,_x4,)(pn, base, vnum); } @@ -863,7 +872,7 @@ svuint64x4_t test_svldnt1_vnum_u64_x4(svcount_t pn, const uint64_t *base, int64_ // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16) // CPP-CHECK-NEXT: ret <vscale x 32 x i8> [[TMP5]] // -svint8x2_t test_svldnt1_vnum_s8_x2(svcount_t pn, const int8_t *base, int64_t vnum) +svint8x2_t test_svldnt1_vnum_s8_x2(svcount_t pn, const int8_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_s8,_x2,)(pn, base, vnum); } @@ -888,7 +897,7 @@ svint8x2_t test_svldnt1_vnum_s8_x2(svcount_t pn, const int8_t *base, int64_t vnu // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8) // CPP-CHECK-NEXT: ret <vscale x 16 x i16> [[TMP5]] // -svint16x2_t test_svldnt1_vnum_s16_x2(svcount_t pn, const int16_t *base, int64_t vnum) +svint16x2_t test_svldnt1_vnum_s16_x2(svcount_t pn, const int16_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_s16,_x2,)(pn, base, vnum); } @@ -913,7 +922,7 @@ svint16x2_t test_svldnt1_vnum_s16_x2(svcount_t pn, const int16_t *base, int64_t // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4) // CPP-CHECK-NEXT: ret <vscale x 8 x i32> [[TMP5]] // -svint32x2_t test_svldnt1_vnum_s32_x2(svcount_t pn, const int32_t *base, int64_t vnum) +svint32x2_t test_svldnt1_vnum_s32_x2(svcount_t pn, const int32_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_s32,_x2,)(pn, base, vnum); } @@ -938,7 +947,7 @@ svint32x2_t test_svldnt1_vnum_s32_x2(svcount_t pn, const int32_t *base, int64_t // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2) // CPP-CHECK-NEXT: ret <vscale x 4 x i64> [[TMP5]] // -svint64x2_t test_svldnt1_vnum_s64_x2(svcount_t pn, const int64_t *base, int64_t vnum) +svint64x2_t test_svldnt1_vnum_s64_x2(svcount_t pn, const int64_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_s64,_x2,)(pn, base, vnum); } @@ -971,7 +980,7 @@ svint64x2_t test_svldnt1_vnum_s64_x2(svcount_t pn, const int64_t *base, int64_t // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP8]], i64 48) // CPP-CHECK-NEXT: ret <vscale x 64 x i8> [[TMP9]] // -svint8x4_t test_svldnt1_vnum_s8_x4(svcount_t pn, const int8_t *base, int64_t vnum) +svint8x4_t test_svldnt1_vnum_s8_x4(svcount_t pn, const int8_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_s8,_x4,)(pn, base, vnum); } @@ -1004,7 +1013,7 @@ svint8x4_t test_svldnt1_vnum_s8_x4(svcount_t pn, const int8_t *base, int64_t vnu // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP7]], <vscale x 8 x i16> [[TMP8]], i64 24) // CPP-CHECK-NEXT: ret <vscale x 32 x i16> [[TMP9]] // -svint16x4_t test_svldnt1_vnum_s16_x4(svcount_t pn, const int16_t *base, int64_t vnum) +svint16x4_t test_svldnt1_vnum_s16_x4(svcount_t pn, const int16_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_s16,_x4,)(pn, base, vnum); } @@ -1037,7 +1046,7 @@ svint16x4_t test_svldnt1_vnum_s16_x4(svcount_t pn, const int16_t *base, int64_t // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], i64 12) // CPP-CHECK-NEXT: ret <vscale x 16 x i32> [[TMP9]] // -svint32x4_t test_svldnt1_vnum_s32_x4(svcount_t pn, const int32_t *base, int64_t vnum) +svint32x4_t test_svldnt1_vnum_s32_x4(svcount_t pn, const int32_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_s32,_x4,)(pn, base, vnum); } @@ -1070,7 +1079,7 @@ svint32x4_t test_svldnt1_vnum_s32_x4(svcount_t pn, const int32_t *base, int64_t // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP8]], i64 6) // CPP-CHECK-NEXT: ret <vscale x 8 x i64> [[TMP9]] // -svint64x4_t test_svldnt1_vnum_s64_x4(svcount_t pn, const int64_t *base, int64_t vnum) +svint64x4_t test_svldnt1_vnum_s64_x4(svcount_t pn, const int64_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_s64,_x4,)(pn, base, vnum); } @@ -1095,7 +1104,7 @@ svint64x4_t test_svldnt1_vnum_s64_x4(svcount_t pn, const int64_t *base, int64_t // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], i64 8) // CPP-CHECK-NEXT: ret <vscale x 16 x half> [[TMP5]] // -svfloat16x2_t test_svldnt1_vnum_f16_x2(svcount_t pn, const float16_t *base, int64_t vnum) +svfloat16x2_t test_svldnt1_vnum_f16_x2(svcount_t pn, const float16_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_f16,_x2,)(pn, base, vnum); } @@ -1120,7 +1129,7 @@ svfloat16x2_t test_svldnt1_vnum_f16_x2(svcount_t pn, const float16_t *base, int6 // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], i64 4) // CPP-CHECK-NEXT: ret <vscale x 8 x float> [[TMP5]] // -svfloat32x2_t test_svldnt1_vnum_f32_x2(svcount_t pn, const float32_t *base, int64_t vnum) +svfloat32x2_t test_svldnt1_vnum_f32_x2(svcount_t pn, const float32_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_f32,_x2,)(pn, base, vnum); } @@ -1145,7 +1154,7 @@ svfloat32x2_t test_svldnt1_vnum_f32_x2(svcount_t pn, const float32_t *base, int6 // CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], i64 2) // CPP-CHECK-NEXT: ret <vscale x 4 x double> [[TMP5]] // -svfloat64x2_t test_svldnt1_vnum_f64_x2(svcount_t pn, const float64_t *base, int64_t vnum) +svfloat64x2_t test_svldnt1_vnum_f64_x2(svcount_t pn, const float64_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_f64,_x2,)(pn, base, vnum); } @@ -1178,7 +1187,7 @@ svfloat64x2_t test_svldnt1_vnum_f64_x2(svcount_t pn, const float64_t *base, int6 // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP7]], <vscale x 8 x half> [[TMP8]], i64 24) // CPP-CHECK-NEXT: ret <vscale x 32 x half> [[TMP9]] // -svfloat16x4_t test_svldnt1_vnum_f16_x4(svcount_t pn, const float16_t *base, int64_t vnum) +svfloat16x4_t test_svldnt1_vnum_f16_x4(svcount_t pn, const float16_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_f16,_x4,)(pn, base, vnum); } @@ -1211,7 +1220,7 @@ svfloat16x4_t test_svldnt1_vnum_f16_x4(svcount_t pn, const float16_t *base, int6 // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP7]], <vscale x 4 x float> [[TMP8]], i64 12) // CPP-CHECK-NEXT: ret <vscale x 16 x float> [[TMP9]] // -svfloat32x4_t test_svldnt1_vnum_f32_x4(svcount_t pn, const float32_t *base, int64_t vnum) +svfloat32x4_t test_svldnt1_vnum_f32_x4(svcount_t pn, const float32_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_f32,_x4,)(pn, base, vnum); } @@ -1244,7 +1253,7 @@ svfloat32x4_t test_svldnt1_vnum_f32_x4(svcount_t pn, const float32_t *base, int6 // CPP-CHECK-NEXT: [[TMP9:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP7]], <vscale x 2 x double> [[TMP8]], i64 6) // CPP-CHECK-NEXT: ret <vscale x 8 x double> [[TMP9]] // -svfloat64x4_t test_svldnt1_vnum_f64_x4(svcount_t pn, const float64_t *base, int64_t vnum) +svfloat64x4_t test_svldnt1_vnum_f64_x4(svcount_t pn, const float64_t *base, int64_t vnum) ATTR { return SVE_ACLE_FUNC(svldnt1_vnum,_f64,_x4,)(pn, base, vnum); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_st1.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_st1.c index 9efc37a1dd58..7aa994345a8c 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_st1.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_st1.c @@ -1,9 +1,11 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -DTEST_SME2 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -DTEST_SME2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s #include <arm_sve.h> #ifdef SVE_OVERLOADED_FORMS @@ -13,6 +15,12 @@ #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 #endif +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif + // CHECK-LABEL: @test_svst1_u8_x2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0) @@ -27,7 +35,7 @@ // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_u8_x2(svcount_t pn, uint8_t *base, svuint8x2_t v) +void test_svst1_u8_x2(svcount_t pn, uint8_t *base, svuint8x2_t v) ATTR { return SVE_ACLE_FUNC(svst1,_u8_x2,,)(pn, base, v); } @@ -46,7 +54,7 @@ void test_svst1_u8_x2(svcount_t pn, uint8_t *base, svuint8x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_u16_x2(svcount_t pn, uint16_t *base, svuint16x2_t v) +void test_svst1_u16_x2(svcount_t pn, uint16_t *base, svuint16x2_t v) ATTR { return SVE_ACLE_FUNC(svst1,_u16_x2,,)(pn, base, v); } @@ -65,7 +73,7 @@ void test_svst1_u16_x2(svcount_t pn, uint16_t *base, svuint16x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_u32_x2(svcount_t pn, uint32_t *base, svuint32x2_t v) +void test_svst1_u32_x2(svcount_t pn, uint32_t *base, svuint32x2_t v) ATTR { return SVE_ACLE_FUNC(svst1,_u32_x2,,)(pn, base, v); } @@ -84,7 +92,7 @@ void test_svst1_u32_x2(svcount_t pn, uint32_t *base, svuint32x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_u64_x2(svcount_t pn, uint64_t *base, svuint64x2_t v) +void test_svst1_u64_x2(svcount_t pn, uint64_t *base, svuint64x2_t v) ATTR { return SVE_ACLE_FUNC(svst1,_u64_x2,,)(pn, base, v); } @@ -107,7 +115,7 @@ void test_svst1_u64_x2(svcount_t pn, uint64_t *base, svuint64x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_u8_x4(svcount_t pn, uint8_t *base, svuint8x4_t v) +void test_svst1_u8_x4(svcount_t pn, uint8_t *base, svuint8x4_t v) ATTR { return SVE_ACLE_FUNC(svst1,_u8_x4,,)(pn, base, v); } @@ -130,7 +138,7 @@ void test_svst1_u8_x4(svcount_t pn, uint8_t *base, svuint8x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_u16_x4(svcount_t pn, uint16_t *base, svuint16x4_t v) +void test_svst1_u16_x4(svcount_t pn, uint16_t *base, svuint16x4_t v) ATTR { return SVE_ACLE_FUNC(svst1,_u16_x4,,)(pn, base, v); } @@ -153,7 +161,7 @@ void test_svst1_u16_x4(svcount_t pn, uint16_t *base, svuint16x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_u32_x4(svcount_t pn, uint32_t *base, svuint32x4_t v) +void test_svst1_u32_x4(svcount_t pn, uint32_t *base, svuint32x4_t v) ATTR { return SVE_ACLE_FUNC(svst1,_u32_x4,,)(pn, base, v); } @@ -176,7 +184,7 @@ void test_svst1_u32_x4(svcount_t pn, uint32_t *base, svuint32x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_u64_x4(svcount_t pn, uint64_t *base, svuint64x4_t v) +void test_svst1_u64_x4(svcount_t pn, uint64_t *base, svuint64x4_t v) ATTR { return SVE_ACLE_FUNC(svst1,_u64_x4,,)(pn, base, v); } @@ -195,7 +203,7 @@ void test_svst1_u64_x4(svcount_t pn, uint64_t *base, svuint64x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_s8_x2(svcount_t pn, int8_t *base, svint8x2_t v) +void test_svst1_s8_x2(svcount_t pn, int8_t *base, svint8x2_t v) ATTR { return SVE_ACLE_FUNC(svst1,_s8_x2,,)(pn, base, v); } @@ -214,7 +222,7 @@ void test_svst1_s8_x2(svcount_t pn, int8_t *base, svint8x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_s16_x2(svcount_t pn, int16_t *base, svint16x2_t v) +void test_svst1_s16_x2(svcount_t pn, int16_t *base, svint16x2_t v) ATTR { return SVE_ACLE_FUNC(svst1,_s16_x2,,)(pn, base, v); } @@ -233,7 +241,7 @@ void test_svst1_s16_x2(svcount_t pn, int16_t *base, svint16x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_s32_x2(svcount_t pn, int32_t *base, svint32x2_t v) +void test_svst1_s32_x2(svcount_t pn, int32_t *base, svint32x2_t v) ATTR { return SVE_ACLE_FUNC(svst1,_s32_x2,,)(pn, base, v); } @@ -252,7 +260,7 @@ void test_svst1_s32_x2(svcount_t pn, int32_t *base, svint32x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_s64_x2(svcount_t pn, int64_t *base, svint64x2_t v) +void test_svst1_s64_x2(svcount_t pn, int64_t *base, svint64x2_t v) ATTR { return SVE_ACLE_FUNC(svst1,_s64_x2,,)(pn, base, v); } @@ -275,7 +283,7 @@ void test_svst1_s64_x2(svcount_t pn, int64_t *base, svint64x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_s8_x4(svcount_t pn, int8_t *base, svint8x4_t v) +void test_svst1_s8_x4(svcount_t pn, int8_t *base, svint8x4_t v) ATTR { return SVE_ACLE_FUNC(svst1,_s8_x4,,)(pn, base, v); } @@ -298,7 +306,7 @@ void test_svst1_s8_x4(svcount_t pn, int8_t *base, svint8x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_s16_x4(svcount_t pn, int16_t *base, svint16x4_t v) +void test_svst1_s16_x4(svcount_t pn, int16_t *base, svint16x4_t v) ATTR { return SVE_ACLE_FUNC(svst1,_s16_x4,,)(pn, base, v); } @@ -321,7 +329,7 @@ void test_svst1_s16_x4(svcount_t pn, int16_t *base, svint16x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_s32_x4(svcount_t pn, int32_t *base, svint32x4_t v) +void test_svst1_s32_x4(svcount_t pn, int32_t *base, svint32x4_t v) ATTR { return SVE_ACLE_FUNC(svst1,_s32_x4,,)(pn, base, v); } @@ -344,7 +352,7 @@ void test_svst1_s32_x4(svcount_t pn, int32_t *base, svint32x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_s64_x4(svcount_t pn, int64_t *base, svint64x4_t v) +void test_svst1_s64_x4(svcount_t pn, int64_t *base, svint64x4_t v) ATTR { return SVE_ACLE_FUNC(svst1,_s64_x4,,)(pn, base, v); } @@ -363,7 +371,7 @@ void test_svst1_s64_x4(svcount_t pn, int64_t *base, svint64x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_f16_x2(svcount_t pn, float16_t *base, svfloat16x2_t v) +void test_svst1_f16_x2(svcount_t pn, float16_t *base, svfloat16x2_t v) ATTR { return SVE_ACLE_FUNC(svst1,_f16_x2,,)(pn, base, v); } @@ -382,7 +390,7 @@ void test_svst1_f16_x2(svcount_t pn, float16_t *base, svfloat16x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_f32_x2(svcount_t pn, float32_t *base, svfloat32x2_t v) +void test_svst1_f32_x2(svcount_t pn, float32_t *base, svfloat32x2_t v) ATTR { return SVE_ACLE_FUNC(svst1,_f32_x2,,)(pn, base, v); } @@ -401,7 +409,7 @@ void test_svst1_f32_x2(svcount_t pn, float32_t *base, svfloat32x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_f64_x2(svcount_t pn, float64_t *base, svfloat64x2_t v) +void test_svst1_f64_x2(svcount_t pn, float64_t *base, svfloat64x2_t v) ATTR { return SVE_ACLE_FUNC(svst1,_f64_x2,,)(pn, base, v); } @@ -424,7 +432,7 @@ void test_svst1_f64_x2(svcount_t pn, float64_t *base, svfloat64x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_f16_x4(svcount_t pn, float16_t *base, svfloat16x4_t v) +void test_svst1_f16_x4(svcount_t pn, float16_t *base, svfloat16x4_t v) ATTR { return SVE_ACLE_FUNC(svst1,_f16_x4,,)(pn, base, v); } @@ -447,7 +455,7 @@ void test_svst1_f16_x4(svcount_t pn, float16_t *base, svfloat16x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_f32_x4(svcount_t pn, float32_t *base, svfloat32x4_t v) +void test_svst1_f32_x4(svcount_t pn, float32_t *base, svfloat32x4_t v) ATTR { return SVE_ACLE_FUNC(svst1,_f32_x4,,)(pn, base, v); } @@ -470,7 +478,7 @@ void test_svst1_f32_x4(svcount_t pn, float32_t *base, svfloat32x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_f64_x4(svcount_t pn, float64_t *base, svfloat64x4_t v) +void test_svst1_f64_x4(svcount_t pn, float64_t *base, svfloat64x4_t v) ATTR { return SVE_ACLE_FUNC(svst1,_f64_x4,,)(pn, base, v); } @@ -495,7 +503,7 @@ void test_svst1_f64_x4(svcount_t pn, float64_t *base, svfloat64x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_u8_x2(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x2_t v) +void test_svst1_vnum_u8_x2(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x2_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_u8_x2,,)(pn, base, vnum, v); } @@ -516,7 +524,7 @@ void test_svst1_vnum_u8_x2(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x2_ // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_u16_x2(svcount_t pn, uint16_t *base, int64_t vnum, svuint16x2_t v) +void test_svst1_vnum_u16_x2(svcount_t pn, uint16_t *base, int64_t vnum, svuint16x2_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_u16_x2,,)(pn, base, vnum, v); } @@ -537,7 +545,7 @@ void test_svst1_vnum_u16_x2(svcount_t pn, uint16_t *base, int64_t vnum, svuint16 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_u32_x2(svcount_t pn, uint32_t *base, int64_t vnum, svuint32x2_t v) +void test_svst1_vnum_u32_x2(svcount_t pn, uint32_t *base, int64_t vnum, svuint32x2_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_u32_x2,,)(pn, base, vnum, v); } @@ -558,7 +566,7 @@ void test_svst1_vnum_u32_x2(svcount_t pn, uint32_t *base, int64_t vnum, svuint32 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_u64_x2(svcount_t pn, uint64_t *base, int64_t vnum, svuint64x2_t v) +void test_svst1_vnum_u64_x2(svcount_t pn, uint64_t *base, int64_t vnum, svuint64x2_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_u64_x2,,)(pn, base, vnum, v); } @@ -583,7 +591,7 @@ void test_svst1_vnum_u64_x2(svcount_t pn, uint64_t *base, int64_t vnum, svuint64 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_u8_x4(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x4_t v) +void test_svst1_vnum_u8_x4(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x4_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_u8_x4,,)(pn, base, vnum, v); } @@ -608,7 +616,7 @@ void test_svst1_vnum_u8_x4(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x4_ // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_u16_x4(svcount_t pn, uint16_t *base, int64_t vnum, svuint16x4_t v) +void test_svst1_vnum_u16_x4(svcount_t pn, uint16_t *base, int64_t vnum, svuint16x4_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_u16_x4,,)(pn, base, vnum, v); } @@ -633,7 +641,7 @@ void test_svst1_vnum_u16_x4(svcount_t pn, uint16_t *base, int64_t vnum, svuint16 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_u32_x4(svcount_t pn, uint32_t *base, int64_t vnum, svuint32x4_t v) +void test_svst1_vnum_u32_x4(svcount_t pn, uint32_t *base, int64_t vnum, svuint32x4_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_u32_x4,,)(pn, base, vnum, v); } @@ -658,7 +666,7 @@ void test_svst1_vnum_u32_x4(svcount_t pn, uint32_t *base, int64_t vnum, svuint32 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_u64_x4(svcount_t pn, uint64_t *base, int64_t vnum, svuint64x4_t v) +void test_svst1_vnum_u64_x4(svcount_t pn, uint64_t *base, int64_t vnum, svuint64x4_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_u64_x4,,)(pn, base, vnum, v); } @@ -679,7 +687,7 @@ void test_svst1_vnum_u64_x4(svcount_t pn, uint64_t *base, int64_t vnum, svuint64 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_s8_x2(svcount_t pn, int8_t *base, int64_t vnum, svint8x2_t v) +void test_svst1_vnum_s8_x2(svcount_t pn, int8_t *base, int64_t vnum, svint8x2_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_s8_x2,,)(pn, base, vnum, v); } @@ -700,7 +708,7 @@ void test_svst1_vnum_s8_x2(svcount_t pn, int8_t *base, int64_t vnum, svint8x2_t // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_s16_x2(svcount_t pn, int16_t *base, int64_t vnum, svint16x2_t v) +void test_svst1_vnum_s16_x2(svcount_t pn, int16_t *base, int64_t vnum, svint16x2_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_s16_x2,,)(pn, base, vnum, v); } @@ -721,7 +729,7 @@ void test_svst1_vnum_s16_x2(svcount_t pn, int16_t *base, int64_t vnum, svint16x2 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_s32_x2(svcount_t pn, int32_t *base, int64_t vnum, svint32x2_t v) +void test_svst1_vnum_s32_x2(svcount_t pn, int32_t *base, int64_t vnum, svint32x2_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_s32_x2,,)(pn, base, vnum, v); } @@ -742,7 +750,7 @@ void test_svst1_vnum_s32_x2(svcount_t pn, int32_t *base, int64_t vnum, svint32x2 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_s64_x2(svcount_t pn, int64_t *base, int64_t vnum, svint64x2_t v) +void test_svst1_vnum_s64_x2(svcount_t pn, int64_t *base, int64_t vnum, svint64x2_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_s64_x2,,)(pn, base, vnum, v); } @@ -767,7 +775,7 @@ void test_svst1_vnum_s64_x2(svcount_t pn, int64_t *base, int64_t vnum, svint64x2 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_s8_x4(svcount_t pn, int8_t *base, int64_t vnum, svint8x4_t v) +void test_svst1_vnum_s8_x4(svcount_t pn, int8_t *base, int64_t vnum, svint8x4_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_s8_x4,,)(pn, base, vnum, v); } @@ -792,7 +800,7 @@ void test_svst1_vnum_s8_x4(svcount_t pn, int8_t *base, int64_t vnum, svint8x4_t // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_s16_x4(svcount_t pn, int16_t *base, int64_t vnum, svint16x4_t v) +void test_svst1_vnum_s16_x4(svcount_t pn, int16_t *base, int64_t vnum, svint16x4_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_s16_x4,,)(pn, base, vnum, v); } @@ -817,7 +825,7 @@ void test_svst1_vnum_s16_x4(svcount_t pn, int16_t *base, int64_t vnum, svint16x4 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_s32_x4(svcount_t pn, int32_t *base, int64_t vnum, svint32x4_t v) +void test_svst1_vnum_s32_x4(svcount_t pn, int32_t *base, int64_t vnum, svint32x4_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_s32_x4,,)(pn, base, vnum, v); } @@ -842,7 +850,7 @@ void test_svst1_vnum_s32_x4(svcount_t pn, int32_t *base, int64_t vnum, svint32x4 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_s64_x4(svcount_t pn, int64_t *base, int64_t vnum, svint64x4_t v) +void test_svst1_vnum_s64_x4(svcount_t pn, int64_t *base, int64_t vnum, svint64x4_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_s64_x4,,)(pn, base, vnum, v); } @@ -865,7 +873,7 @@ void test_svst1_vnum_s64_x4(svcount_t pn, int64_t *base, int64_t vnum, svint64x4 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_f16_x2(svcount_t pn, float16_t *base, float64_t vnum, svfloat16x2_t v) +void test_svst1_vnum_f16_x2(svcount_t pn, float16_t *base, float64_t vnum, svfloat16x2_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_f16_x2,,)(pn, base, vnum, v); } @@ -888,7 +896,7 @@ void test_svst1_vnum_f16_x2(svcount_t pn, float16_t *base, float64_t vnum, svflo // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_f32_x2(svcount_t pn, float32_t *base, float64_t vnum, svfloat32x2_t v) +void test_svst1_vnum_f32_x2(svcount_t pn, float32_t *base, float64_t vnum, svfloat32x2_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_f32_x2,,)(pn, base, vnum, v); } @@ -911,7 +919,7 @@ void test_svst1_vnum_f32_x2(svcount_t pn, float32_t *base, float64_t vnum, svflo // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_f64_x2(svcount_t pn, float64_t *base, float64_t vnum, svfloat64x2_t v) +void test_svst1_vnum_f64_x2(svcount_t pn, float64_t *base, float64_t vnum, svfloat64x2_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_f64_x2,,)(pn, base, vnum, v); } @@ -938,7 +946,7 @@ void test_svst1_vnum_f64_x2(svcount_t pn, float64_t *base, float64_t vnum, svflo // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_f16_x4(svcount_t pn, float16_t *base, float64_t vnum, svfloat16x4_t v) +void test_svst1_vnum_f16_x4(svcount_t pn, float16_t *base, float64_t vnum, svfloat16x4_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_f16_x4,,)(pn, base, vnum, v); } @@ -965,7 +973,7 @@ void test_svst1_vnum_f16_x4(svcount_t pn, float16_t *base, float64_t vnum, svflo // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_f32_x4(svcount_t pn, float32_t *base, float64_t vnum, svfloat32x4_t v) +void test_svst1_vnum_f32_x4(svcount_t pn, float32_t *base, float64_t vnum, svfloat32x4_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_f32_x4,,)(pn, base, vnum, v); } @@ -992,7 +1000,7 @@ void test_svst1_vnum_f32_x4(svcount_t pn, float32_t *base, float64_t vnum, svflo // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svst1_vnum_f64_x4(svcount_t pn, float64_t *base, float64_t vnum, svfloat64x4_t v) +void test_svst1_vnum_f64_x4(svcount_t pn, float64_t *base, float64_t vnum, svfloat64x4_t v) ATTR { return SVE_ACLE_FUNC(svst1_vnum,_f64_x4,,)(pn, base, vnum, v); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c index 9b860fe7180e..0d8696a7634a 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c @@ -1,9 +1,12 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -DTEST_SME2 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -DTEST_SME2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + #include <arm_sve.h> #ifdef SVE_OVERLOADED_FORMS @@ -13,6 +16,11 @@ #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 #endif +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif // CHECK-LABEL: @test_svstnt1_u8_x2( // CHECK-NEXT: entry: @@ -28,7 +36,7 @@ // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_u8_x2(svcount_t pn, uint8_t *base, svuint8x2_t v) +void test_svstnt1_u8_x2(svcount_t pn, uint8_t *base, svuint8x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_u8_x2,,)(pn, base, v); } @@ -48,7 +56,7 @@ void test_svstnt1_u8_x2(svcount_t pn, uint8_t *base, svuint8x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_u16_x2(svcount_t pn, uint16_t *base, svuint16x2_t v) +void test_svstnt1_u16_x2(svcount_t pn, uint16_t *base, svuint16x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_u16_x2,,)(pn, base, v); } @@ -68,7 +76,7 @@ void test_svstnt1_u16_x2(svcount_t pn, uint16_t *base, svuint16x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_u32_x2(svcount_t pn, uint32_t *base, svuint32x2_t v) +void test_svstnt1_u32_x2(svcount_t pn, uint32_t *base, svuint32x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_u32_x2,,)(pn, base, v); } @@ -88,7 +96,7 @@ void test_svstnt1_u32_x2(svcount_t pn, uint32_t *base, svuint32x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_u64_x2(svcount_t pn, uint64_t *base, svuint64x2_t v) +void test_svstnt1_u64_x2(svcount_t pn, uint64_t *base, svuint64x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_u64_x2,,)(pn, base, v); } @@ -112,7 +120,7 @@ void test_svstnt1_u64_x2(svcount_t pn, uint64_t *base, svuint64x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_u8_x4(svcount_t pn, uint8_t *base, svuint8x4_t v) +void test_svstnt1_u8_x4(svcount_t pn, uint8_t *base, svuint8x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_u8_x4,,)(pn, base, v); } @@ -136,7 +144,7 @@ void test_svstnt1_u8_x4(svcount_t pn, uint8_t *base, svuint8x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_u16_x4(svcount_t pn, uint16_t *base, svuint16x4_t v) +void test_svstnt1_u16_x4(svcount_t pn, uint16_t *base, svuint16x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_u16_x4,,)(pn, base, v); } @@ -160,7 +168,7 @@ void test_svstnt1_u16_x4(svcount_t pn, uint16_t *base, svuint16x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_u32_x4(svcount_t pn, uint32_t *base, svuint32x4_t v) +void test_svstnt1_u32_x4(svcount_t pn, uint32_t *base, svuint32x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_u32_x4,,)(pn, base, v); } @@ -184,7 +192,7 @@ void test_svstnt1_u32_x4(svcount_t pn, uint32_t *base, svuint32x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_u64_x4(svcount_t pn, uint64_t *base, svuint64x4_t v) +void test_svstnt1_u64_x4(svcount_t pn, uint64_t *base, svuint64x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_u64_x4,,)(pn, base, v); } @@ -204,7 +212,7 @@ void test_svstnt1_u64_x4(svcount_t pn, uint64_t *base, svuint64x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_s8_x2(svcount_t pn, int8_t *base, svint8x2_t v) +void test_svstnt1_s8_x2(svcount_t pn, int8_t *base, svint8x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_s8_x2,,)(pn, base, v); } @@ -224,7 +232,7 @@ void test_svstnt1_s8_x2(svcount_t pn, int8_t *base, svint8x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_s16_x2(svcount_t pn, int16_t *base, svint16x2_t v) +void test_svstnt1_s16_x2(svcount_t pn, int16_t *base, svint16x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_s16_x2,,)(pn, base, v); } @@ -244,7 +252,7 @@ void test_svstnt1_s16_x2(svcount_t pn, int16_t *base, svint16x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_s32_x2(svcount_t pn, int32_t *base, svint32x2_t v) +void test_svstnt1_s32_x2(svcount_t pn, int32_t *base, svint32x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_s32_x2,,)(pn, base, v); } @@ -264,7 +272,7 @@ void test_svstnt1_s32_x2(svcount_t pn, int32_t *base, svint32x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_s64_x2(svcount_t pn, int64_t *base, svint64x2_t v) +void test_svstnt1_s64_x2(svcount_t pn, int64_t *base, svint64x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_s64_x2,,)(pn, base, v); } @@ -288,7 +296,7 @@ void test_svstnt1_s64_x2(svcount_t pn, int64_t *base, svint64x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_s8_x4(svcount_t pn, int8_t *base, svint8x4_t v) +void test_svstnt1_s8_x4(svcount_t pn, int8_t *base, svint8x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_s8_x4,,)(pn, base, v); } @@ -312,7 +320,7 @@ void test_svstnt1_s8_x4(svcount_t pn, int8_t *base, svint8x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_s16_x4(svcount_t pn, int16_t *base, svint16x4_t v) +void test_svstnt1_s16_x4(svcount_t pn, int16_t *base, svint16x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_s16_x4,,)(pn, base, v); } @@ -336,7 +344,7 @@ void test_svstnt1_s16_x4(svcount_t pn, int16_t *base, svint16x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_s32_x4(svcount_t pn, int32_t *base, svint32x4_t v) +void test_svstnt1_s32_x4(svcount_t pn, int32_t *base, svint32x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_s32_x4,,)(pn, base, v); } @@ -360,7 +368,7 @@ void test_svstnt1_s32_x4(svcount_t pn, int32_t *base, svint32x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_s64_x4(svcount_t pn, int64_t *base, svint64x4_t v) +void test_svstnt1_s64_x4(svcount_t pn, int64_t *base, svint64x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_s64_x4,,)(pn, base, v); } @@ -380,7 +388,7 @@ void test_svstnt1_s64_x4(svcount_t pn, int64_t *base, svint64x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_f16_x2(svcount_t pn, float16_t *base, svfloat16x2_t v) +void test_svstnt1_f16_x2(svcount_t pn, float16_t *base, svfloat16x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_f16_x2,,)(pn, base, v); } @@ -400,7 +408,7 @@ void test_svstnt1_f16_x2(svcount_t pn, float16_t *base, svfloat16x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_f32_x2(svcount_t pn, float32_t *base, svfloat32x2_t v) +void test_svstnt1_f32_x2(svcount_t pn, float32_t *base, svfloat32x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_f32_x2,,)(pn, base, v); } @@ -420,7 +428,7 @@ void test_svstnt1_f32_x2(svcount_t pn, float32_t *base, svfloat32x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_f64_x2(svcount_t pn, float64_t *base, svfloat64x2_t v) +void test_svstnt1_f64_x2(svcount_t pn, float64_t *base, svfloat64x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_f64_x2,,)(pn, base, v); } @@ -444,7 +452,7 @@ void test_svstnt1_f64_x2(svcount_t pn, float64_t *base, svfloat64x2_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_f16_x4(svcount_t pn, float16_t *base, svfloat16x4_t v) +void test_svstnt1_f16_x4(svcount_t pn, float16_t *base, svfloat16x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_f16_x4,,)(pn, base, v); } @@ -468,7 +476,7 @@ void test_svstnt1_f16_x4(svcount_t pn, float16_t *base, svfloat16x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_f32_x4(svcount_t pn, float32_t *base, svfloat32x4_t v) +void test_svstnt1_f32_x4(svcount_t pn, float32_t *base, svfloat32x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_f32_x4,,)(pn, base, v); } @@ -492,7 +500,7 @@ void test_svstnt1_f32_x4(svcount_t pn, float32_t *base, svfloat32x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_f64_x4(svcount_t pn, float64_t *base, svfloat64x4_t v) +void test_svstnt1_f64_x4(svcount_t pn, float64_t *base, svfloat64x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1,_f64_x4,,)(pn, base, v); } @@ -518,7 +526,7 @@ void test_svstnt1_f64_x4(svcount_t pn, float64_t *base, svfloat64x4_t v) // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_u8_x2(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x2_t v) +void test_svstnt1_vnum_u8_x2(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_u8_x2,,)(pn, base, vnum, v); } @@ -540,7 +548,7 @@ void test_svstnt1_vnum_u8_x2(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_u16_x2(svcount_t pn, uint16_t *base, int64_t vnum, svuint16x2_t v) +void test_svstnt1_vnum_u16_x2(svcount_t pn, uint16_t *base, int64_t vnum, svuint16x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_u16_x2,,)(pn, base, vnum, v); } @@ -562,7 +570,7 @@ void test_svstnt1_vnum_u16_x2(svcount_t pn, uint16_t *base, int64_t vnum, svuint // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_u32_x2(svcount_t pn, uint32_t *base, int64_t vnum, svuint32x2_t v) +void test_svstnt1_vnum_u32_x2(svcount_t pn, uint32_t *base, int64_t vnum, svuint32x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_u32_x2,,)(pn, base, vnum, v); } @@ -584,7 +592,7 @@ void test_svstnt1_vnum_u32_x2(svcount_t pn, uint32_t *base, int64_t vnum, svuint // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_u64_x2(svcount_t pn, uint64_t *base, int64_t vnum, svuint64x2_t v) +void test_svstnt1_vnum_u64_x2(svcount_t pn, uint64_t *base, int64_t vnum, svuint64x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_u64_x2,,)(pn, base, vnum, v); } @@ -610,7 +618,7 @@ void test_svstnt1_vnum_u64_x2(svcount_t pn, uint64_t *base, int64_t vnum, svuint // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_u8_x4(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x4_t v) +void test_svstnt1_vnum_u8_x4(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_u8_x4,,)(pn, base, vnum, v); } @@ -636,7 +644,7 @@ void test_svstnt1_vnum_u8_x4(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_u16_x4(svcount_t pn, uint16_t *base, int64_t vnum, svuint16x4_t v) +void test_svstnt1_vnum_u16_x4(svcount_t pn, uint16_t *base, int64_t vnum, svuint16x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_u16_x4,,)(pn, base, vnum, v); } @@ -662,7 +670,7 @@ void test_svstnt1_vnum_u16_x4(svcount_t pn, uint16_t *base, int64_t vnum, svuint // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_u32_x4(svcount_t pn, uint32_t *base, int64_t vnum, svuint32x4_t v) +void test_svstnt1_vnum_u32_x4(svcount_t pn, uint32_t *base, int64_t vnum, svuint32x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_u32_x4,,)(pn, base, vnum, v); } @@ -688,7 +696,7 @@ void test_svstnt1_vnum_u32_x4(svcount_t pn, uint32_t *base, int64_t vnum, svuint // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_u64_x4(svcount_t pn, uint64_t *base, int64_t vnum, svuint64x4_t v) +void test_svstnt1_vnum_u64_x4(svcount_t pn, uint64_t *base, int64_t vnum, svuint64x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_u64_x4,,)(pn, base, vnum, v); } @@ -710,7 +718,7 @@ void test_svstnt1_vnum_u64_x4(svcount_t pn, uint64_t *base, int64_t vnum, svuint // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_s8_x2(svcount_t pn, int8_t *base, int64_t vnum, svint8x2_t v) +void test_svstnt1_vnum_s8_x2(svcount_t pn, int8_t *base, int64_t vnum, svint8x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_s8_x2,,)(pn, base, vnum, v); } @@ -732,7 +740,7 @@ void test_svstnt1_vnum_s8_x2(svcount_t pn, int8_t *base, int64_t vnum, svint8x2_ // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_s16_x2(svcount_t pn, int16_t *base, int64_t vnum, svint16x2_t v) +void test_svstnt1_vnum_s16_x2(svcount_t pn, int16_t *base, int64_t vnum, svint16x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_s16_x2,,)(pn, base, vnum, v); } @@ -754,7 +762,7 @@ void test_svstnt1_vnum_s16_x2(svcount_t pn, int16_t *base, int64_t vnum, svint16 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_s32_x2(svcount_t pn, int32_t *base, int64_t vnum, svint32x2_t v) +void test_svstnt1_vnum_s32_x2(svcount_t pn, int32_t *base, int64_t vnum, svint32x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_s32_x2,,)(pn, base, vnum, v); } @@ -776,7 +784,7 @@ void test_svstnt1_vnum_s32_x2(svcount_t pn, int32_t *base, int64_t vnum, svint32 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_s64_x2(svcount_t pn, int64_t *base, int64_t vnum, svint64x2_t v) +void test_svstnt1_vnum_s64_x2(svcount_t pn, int64_t *base, int64_t vnum, svint64x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_s64_x2,,)(pn, base, vnum, v); } @@ -802,7 +810,7 @@ void test_svstnt1_vnum_s64_x2(svcount_t pn, int64_t *base, int64_t vnum, svint64 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_s8_x4(svcount_t pn, int8_t *base, int64_t vnum, svint8x4_t v) +void test_svstnt1_vnum_s8_x4(svcount_t pn, int8_t *base, int64_t vnum, svint8x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_s8_x4,,)(pn, base, vnum, v); } @@ -828,7 +836,7 @@ void test_svstnt1_vnum_s8_x4(svcount_t pn, int8_t *base, int64_t vnum, svint8x4_ // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_s16_x4(svcount_t pn, int16_t *base, int64_t vnum, svint16x4_t v) +void test_svstnt1_vnum_s16_x4(svcount_t pn, int16_t *base, int64_t vnum, svint16x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_s16_x4,,)(pn, base, vnum, v); } @@ -854,7 +862,7 @@ void test_svstnt1_vnum_s16_x4(svcount_t pn, int16_t *base, int64_t vnum, svint16 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_s32_x4(svcount_t pn, int32_t *base, int64_t vnum, svint32x4_t v) +void test_svstnt1_vnum_s32_x4(svcount_t pn, int32_t *base, int64_t vnum, svint32x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_s32_x4,,)(pn, base, vnum, v); } @@ -880,7 +888,7 @@ void test_svstnt1_vnum_s32_x4(svcount_t pn, int32_t *base, int64_t vnum, svint32 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_s64_x4(svcount_t pn, int64_t *base, int64_t vnum, svint64x4_t v) +void test_svstnt1_vnum_s64_x4(svcount_t pn, int64_t *base, int64_t vnum, svint64x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_s64_x4,,)(pn, base, vnum, v); } @@ -904,7 +912,7 @@ void test_svstnt1_vnum_s64_x4(svcount_t pn, int64_t *base, int64_t vnum, svint64 // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_f16_x2(svcount_t pn, float16_t *base, float64_t vnum, svfloat16x2_t v) +void test_svstnt1_vnum_f16_x2(svcount_t pn, float16_t *base, float64_t vnum, svfloat16x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_f16_x2,,)(pn, base, vnum, v); } @@ -928,7 +936,7 @@ void test_svstnt1_vnum_f16_x2(svcount_t pn, float16_t *base, float64_t vnum, svf // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_f32_x2(svcount_t pn, float32_t *base, float64_t vnum, svfloat32x2_t v) +void test_svstnt1_vnum_f32_x2(svcount_t pn, float32_t *base, float64_t vnum, svfloat32x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_f32_x2,,)(pn, base, vnum, v); } @@ -952,7 +960,7 @@ void test_svstnt1_vnum_f32_x2(svcount_t pn, float32_t *base, float64_t vnum, svf // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP2]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_f64_x2(svcount_t pn, float64_t *base, float64_t vnum, svfloat64x2_t v) +void test_svstnt1_vnum_f64_x2(svcount_t pn, float64_t *base, float64_t vnum, svfloat64x2_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_f64_x2,,)(pn, base, vnum, v); } @@ -980,7 +988,7 @@ void test_svstnt1_vnum_f64_x2(svcount_t pn, float64_t *base, float64_t vnum, svf // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_f16_x4(svcount_t pn, float16_t *base, float64_t vnum, svfloat16x4_t v) +void test_svstnt1_vnum_f16_x4(svcount_t pn, float16_t *base, float64_t vnum, svfloat16x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_f16_x4,,)(pn, base, vnum, v); } @@ -1008,7 +1016,7 @@ void test_svstnt1_vnum_f16_x4(svcount_t pn, float16_t *base, float64_t vnum, svf // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_f32_x4(svcount_t pn, float32_t *base, float64_t vnum, svfloat32x4_t v) +void test_svstnt1_vnum_f32_x4(svcount_t pn, float32_t *base, float64_t vnum, svfloat32x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_f32_x4,,)(pn, base, vnum, v); } @@ -1036,7 +1044,7 @@ void test_svstnt1_vnum_f32_x4(svcount_t pn, float32_t *base, float64_t vnum, svf // CPP-CHECK-NEXT: tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP4]]) // CPP-CHECK-NEXT: ret void // -void test_svstnt1_vnum_f64_x4(svcount_t pn, float64_t *base, float64_t vnum, svfloat64x4_t v) +void test_svstnt1_vnum_f64_x4(svcount_t pn, float64_t *base, float64_t vnum, svfloat64x4_t v) ATTR { return SVE_ACLE_FUNC(svstnt1_vnum,_f64_x4,,)(pn, base, vnum, v); } diff --git a/clang/test/Driver/aarch64-pauth-lr.c b/clang/test/Driver/aarch64-pauth-lr.c new file mode 100644 index 000000000000..2e1b530fc989 --- /dev/null +++ b/clang/test/Driver/aarch64-pauth-lr.c @@ -0,0 +1,23 @@ +// Check the -cc1 flags for the various forms of -mbranch-protection=pac-ret+pc. + +// RUN: %clang -target aarch64-arm-none-eabi -c %s -### -mbranch-protection=pac-ret+pc 2>&1 | FileCheck %s --check-prefixes=PAUTH-LR +// RUN: %clang -target aarch64-arm-none-eabi -c %s -### -mbranch-protection=pac-ret+pc+b-key 2>&1 | FileCheck %s --check-prefixes=PAUTH-LR-B-KEY +// RUN: %clang -target aarch64-arm-none-eabi -c %s -### -mbranch-protection=pac-ret+pc+leaf 2>&1 | FileCheck %s --check-prefixes=PAUTH-LR-LEAF +// RUN: %clang -target aarch64-arm-none-eabi -c %s -### -mbranch-protection=pac-ret+pc+bti 2>&1 | FileCheck %s --check-prefixes=PAUTH-LR-BTI +// RUN: %clang -target aarch64-arm-none-eabi -c %s -### -mbranch-protection=pac-ret+pc+leaf+b-key+bti 2>&1 | FileCheck %s --check-prefixes=PAUTH-LR-LEAF-B-KEY-BTI +// RUN: %clang -target aarch64-arm-none-eabi -c %s -### -mbranch-protection=pac-ret+pc -march=armv9.5-a 2>&1 | FileCheck %s --check-prefixes=PAUTH-LR +// RUN: %clang -target aarch64-arm-none-eabi -c %s -### -mbranch-protection=pac-ret+pc+b-key -march=armv9.5-a 2>&1 | FileCheck %s --check-prefixes=PAUTH-LR-B-KEY +// RUN: %clang -target aarch64-arm-none-eabi -c %s -### -mbranch-protection=pac-ret+pc+leaf -march=armv9.5-a 2>&1 | FileCheck %s --check-prefixes=PAUTH-LR-LEAF +// RUN: %clang -target aarch64-arm-none-eabi -c %s -### -mbranch-protection=pac-ret+pc+bti -march=armv9.5-a 2>&1 | FileCheck %s --check-prefixes=PAUTH-LR-BTI +// RUN: %clang -target aarch64-arm-none-eabi -c %s -### -mbranch-protection=pac-ret+pc+leaf+b-key+bti -march=armv9.5-a 2>&1 | FileCheck %s --check-prefixes=PAUTH-LR-LEAF-B-KEY-BTI + +// PAUTH-LR: "-msign-return-address=non-leaf" "-msign-return-address-key=a_key" "-mbranch-protection-pauth-lr" +// PAUTH-LR-B-KEY: "-msign-return-address=non-leaf" "-msign-return-address-key=b_key" "-mbranch-protection-pauth-lr" +// PAUTH-LR-LEAF: "-msign-return-address=all" "-msign-return-address-key=a_key" "-mbranch-protection-pauth-lr" +// PAUTH-LR-BTI: "-msign-return-address=non-leaf" "-msign-return-address-key=a_key" "-mbranch-protection-pauth-lr" +// PAUTH-LR-LEAF-B-KEY-BTI: "-msign-return-address=all" "-msign-return-address-key=b_key" "-mbranch-protection-pauth-lr" "-mbranch-target-enforce" + +// NOT-PAUTH-LR: "-mbranch-target-enforce" +// NOT-PAUTH-LR-B-KEY: "-mbranch-target-enforce" +// NOT-PAUTH-LR-LEAF: "-mbranch-target-enforce" +// NOT-PAUTH-LR-BTI: "-mbranch-target-enforce" diff --git a/clang/test/Driver/aarch64-v95a.c b/clang/test/Driver/aarch64-v95a.c index 366cade86a9f..6fac62e8b389 100644 --- a/clang/test/Driver/aarch64-v95a.c +++ b/clang/test/Driver/aarch64-v95a.c @@ -1,3 +1,5 @@ +// ===== Base v9.5a architecture ===== + // RUN: %clang -target aarch64 -march=armv9.5a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV95A %s // RUN: %clang -target aarch64 -march=armv9.5-a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV95A %s // RUN: %clang -target aarch64 -mlittle-endian -march=armv9.5a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV95A %s @@ -5,6 +7,7 @@ // RUN: %clang -target aarch64_be -mlittle-endian -march=armv9.5a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV95A %s // RUN: %clang -target aarch64_be -mlittle-endian -march=armv9.5-a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV95A %s // GENERICV95A: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "generic" "-target-feature" "+neon" "-target-feature" "+v9.5a" + // RUN: %clang -target aarch64_be -march=armv9.5a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV95A-BE %s // RUN: %clang -target aarch64_be -march=armv9.5-a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV95A-BE %s // RUN: %clang -target aarch64 -mbig-endian -march=armv9.5a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV95A-BE %s @@ -18,3 +21,7 @@ // RUN: %clang -target aarch64 -march=armv9.5a+cpa -### -c %s 2>&1 | FileCheck -check-prefix=V95A-CPA %s // RUN: %clang -target aarch64 -march=armv9.5-a+cpa -### -c %s 2>&1 | FileCheck -check-prefix=V95A-CPA %s // V95A-CPA: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "generic" "-target-feature" "+neon" "-target-feature" "+v9.5a" "-target-feature" "+cpa" + +// RUN: %clang -target aarch64 -march=armv9.5a+pauth-lr -### -c %s 2>&1 | FileCheck -check-prefix=V95A-PAUTHLR %s +// RUN: %clang -target aarch64 -march=armv9.5-a+pauth-lr -### -c %s 2>&1 | FileCheck -check-prefix=V95A-PAUTHLR %s +// V95A-PAUTHLR: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "generic" "-target-feature" "+neon" "-target-feature" "+v9.5a" "-target-feature" "+pauth-lr" diff --git a/clang/test/Driver/modules.m b/clang/test/Driver/modules.m index 9eb356980556..d1a65f5cb071 100644 --- a/clang/test/Driver/modules.m +++ b/clang/test/Driver/modules.m @@ -57,7 +57,21 @@ // CHECK-MODULE-MAP-FILES: "-fmodule-map-file=foo.map" // CHECK-MODULE-MAP-FILES: "-fmodule-map-file=bar.map" -// RUN: %clang -fmodules -fbuiltin-module-map -### %s 2>&1 | FileCheck -check-prefix=CHECK-BUILTIN-MODULE-MAP %s +// Verify that the driver propagates -fmodule-name and -fmodule-map-file flags when +// -fmodules-decluse or -fmodules-strict-decluse, as used for layering check. +// RUN: %clang -fmodules-decluse -fmodule-name=foo -c -### %s 2>&1 | FileCheck -check-prefix=CHECK-DECLUSE-PROPAGATE-MODULE-NAME %s +// CHECK-DECLUSE-PROPAGATE-MODULE-NAME: -fmodule-name=foo + +// RUN: %clang -fmodules-decluse -fmodule-map-file=foo.map -c -### %s 2>&1 | FileCheck -check-prefix=CHECK-DECLUSE-PROPAGATE-MODULE-MAPS %s +// CHECK-DECLUSE-PROPAGATE-MODULE-MAPS: -fmodule-map-file=foo.map + +// RUN: %clang -fmodules-strict-decluse -fmodule-name=foo -c -### %s 2>&1 | FileCheck -check-prefix=CHECK-STRICT-DECLUSE-PROPAGATE-MODULE-NAME %s +// CHECK-STRICT-DECLUSE-PROPAGATE-MODULE-NAME: -fmodule-name=foo + +// RUN: %clang -fmodules-strict-decluse -fmodule-map-file=foo.map -c -### %s 2>&1 | FileCheck -check-prefix=CHECK-STRICT-DECLUSE-PROPAGATE-MODULE-MAPS %s +// CHECK-STRICT-DECLUSE-PROPAGATE-MODULE-MAPS: -fmodule-map-file=foo.map + + // RUN: %clang -fmodules -fbuiltin-module-map -### %s 2>&1 | FileCheck -check-prefix=CHECK-BUILTIN-MODULE-MAP %s // CHECK-BUILTIN-MODULE-MAP: "-fmodules" // CHECK-BUILTIN-MODULE-MAP: "-fmodule-map-file={{.*}}include{{/|\\\\}}module.modulemap" diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c index db89aa7b608a..b3da54162da0 100644 --- a/clang/test/Preprocessor/aarch64-target-features.c +++ b/clang/test/Preprocessor/aarch64-target-features.c @@ -600,6 +600,7 @@ // RUN: %clang -target aarch64-none-elf -march=armv9.1-a -x c -E -dM %s -o - | FileCheck --check-prefixes=CHECK-V81-OR-LATER,CHECK-V83-OR-LATER,CHECK-V85-OR-LATER %s // RUN: %clang -target aarch64-none-elf -march=armv9.2-a -x c -E -dM %s -o - | FileCheck --check-prefixes=CHECK-V81-OR-LATER,CHECK-V83-OR-LATER,CHECK-V85-OR-LATER %s // RUN: %clang -target aarch64-none-elf -march=armv9.3-a -x c -E -dM %s -o - | FileCheck --check-prefixes=CHECK-V81-OR-LATER,CHECK-V83-OR-LATER,CHECK-V85-OR-LATER %s +// RUN: %clang -target aarch64-none-elf -march=armv9.4-a -x c -E -dM %s -o - | FileCheck --check-prefixes=CHECK-V81-OR-LATER,CHECK-V83-OR-LATER,CHECK-V85-OR-LATER %s // RUN: %clang -target aarch64-none-elf -march=armv9.5-a -x c -E -dM %s -o - | FileCheck --check-prefixes=CHECK-V81-OR-LATER,CHECK-V83-OR-LATER,CHECK-V85-OR-LATER %s // CHECK-V81-OR-LATER: __ARM_FEATURE_ATOMICS 1 // CHECK-V85-OR-LATER: __ARM_FEATURE_BTI 1 diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp index 6ab6dabd92b1..6a6370bf99b1 100644 --- a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp +++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp @@ -241,3 +241,112 @@ void test_bfmlslb_bad_lane(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) __ svbfmlslb_lane_f32(zda, zn, zm, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} svbfmlslt_lane_f32(zda, zn, zm, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} } + +void test_multiply_add_sub_long(uint32_t base, svint8_t s8, svuint8_t u8, + svint16_t s16, svuint16_t u16, svint8x2_t s8x2, + svuint8x2_t u8x2, svint16x2_t s16x2, svuint16x2_t u16x2, + svint8x4_t s8x4, svuint8x4_t u8x4, svint16x4_t s16x4, svuint16x4_t u16x4) __arm_streaming __arm_shared_za { + + svmla_lane_za32_s8_vg4x1(base, s8, s8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svmla_lane_za32_u8_vg4x1(base, u8, u8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svmla_lane_za64_s16_vg4x1(base, s16, s16, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + svmla_lane_za64_u16_vg4x1(base, u16, u16, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + + svmla_lane_za32_s8_vg4x2(base, s8x2, s8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svmla_lane_za32_u8_vg4x2(base, u8x2, u8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svmla_lane_za64_s16_vg4x2(base, s16x2, s16, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + svmla_lane_za64_u16_vg4x2(base, u16x2, u16, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + + svmla_lane_za32_s8_vg4x4(base, s8x4, s8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svmla_lane_za32_u8_vg4x4(base, u8x4, u8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svmla_lane_za64_s16_vg4x4(base, s16x4, s16, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + svmla_lane_za64_u16_vg4x4(base, u16x4, u16, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + + svmls_lane_za32_s8_vg4x1(base, s8, s8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svmls_lane_za32_u8_vg4x1(base, u8, u8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svmls_lane_za64_s16_vg4x1(base, s16, s16, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + svmls_lane_za64_u16_vg4x1(base, u16, u16, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + + svmls_lane_za32_s8_vg4x2(base, s8x2, s8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svmls_lane_za32_u8_vg4x2(base, u8x2, u8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svmls_lane_za64_s16_vg4x2(base, s16x2, s16, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + svmls_lane_za64_u16_vg4x2(base, u16x2, u16, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + + svmls_lane_za32_s8_vg4x4(base, s8x4, s8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svmls_lane_za32_u8_vg4x4(base, u8x4, u8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svmls_lane_za64_s16_vg4x4(base, s16x4, s16, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + svmls_lane_za64_u16_vg4x4(base, u16x4, u16, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + + svsumla_lane_za32_s8_vg4x1(base, s8, u8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svsumla_lane_za32_s8_vg4x2(base, s8x2, u8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svsumla_lane_za32_s8_vg4x4(base, s8x4, u8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + + svusmla_lane_za32_u8_vg4x1(base, u8, s8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svusmla_lane_za32_u8_vg4x2(base, u8x2, s8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + svusmla_lane_za32_u8_vg4x4(base, u8x4, s8, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} +} + +void test_vertical_dot_product(uint32_t base, svint16x2_t s16x2, svuint16x2_t u16x2, + svint8x4_t s8x4, svuint8x4_t u8x4, + svint16x4_t s16x4, svuint16x4_t u16x4, + svfloat16x2_t f16x2, svbfloat16x2_t bf16x2, + svint16_t s16, svuint16_t u16, + svint8_t s8, svuint8_t u8, + svfloat16_t f16, svbfloat16_t b16) __arm_streaming __arm_shared_za { + // Test lane indices. + svvdot_lane_za32_s16_vg1x2(base, s16x2, s16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svvdot_lane_za32_u16_vg1x2(base, u16x2, u16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svvdot_lane_za32_s8_vg1x4(base, s8x4, s8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svvdot_lane_za32_u8_vg1x4(base, u8x4, u8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svvdot_lane_za64_s16_vg1x4(base, s16x4, s16, 2); // expected-error {{argument value 2 is outside the valid range [0, 1]}} + svvdot_lane_za64_u16_vg1x4(base, u16x4, u16, 2); // expected-error {{argument value 2 is outside the valid range [0, 1]}} + svvdot_lane_za32_f16_vg1x2(base, f16x2, f16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svvdot_lane_za32_bf16_vg1x2(base, bf16x2, b16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svsuvdot_lane_za32_s8_vg1x4(base, s8x4, s8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svusvdot_lane_za32_u8_vg1x4(base, u8x4, u8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} +} + +void test_fdot_za32_bad_lane(uint32_t slice_base, svfloat16_t z_f16, + svfloat16x2_t z_f16x2, svfloat16x4_t z_f16x4, + svbfloat16_t z_bf16, svbfloat16x2_t z_bf16x2, + svbfloat16x4_t z_bf16x4) __arm_streaming __arm_shared_za { + // 16-bit float + svdot_lane_za32_f16_vg1x2(slice_base, z_f16x2, z_f16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svdot_lane_za32_f16_vg1x4(slice_base, z_f16x4, z_f16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + + // 16-bit binary float + svdot_lane_za32_bf16_vg1x2(slice_base, z_bf16x2, z_bf16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svdot_lane_za32_bf16_vg1x4(slice_base, z_bf16x4, z_bf16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} +} + +void test_svdot_multi_za32_bad_lane(uint32_t slice_base, svuint16_t z_u16, + svuint16x2_t z_u16x2, svuint16x4_t z_u16x4, + svint16_t z_s16, svint16x2_t z_s16x2, + svint16x4_t z_s16x4, svuint8_t z_u8, + svuint8x2_t z_u8x2, svuint8x4_t z_u8x4, + svint8_t z_s8, svint8x2_t z_s8x2, + svint8x4_t z_s8x4) __arm_streaming __arm_shared_za { + // Multi, indexed (unsigned) + svdot_lane_za32_u16_vg1x2(slice_base, z_u16x2, z_u16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svdot_lane_za32_u16_vg1x4(slice_base, z_u16x4, z_u16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svdot_lane_za32_u8_vg1x2(slice_base, z_u8x2, z_u8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svdot_lane_za32_u8_vg1x4(slice_base, z_u8x4, z_u8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svdot_lane_za64_u16_vg1x2(slice_base, z_u16x2, z_u16, 2); // expected-error {{argument value 2 is outside the valid range [0, 1]}} + svdot_lane_za64_u16_vg1x4(slice_base, z_u16x4, z_u16, 2); // expected-error {{argument value 2 is outside the valid range [0, 1]}} + + // Multi, indexed (signed) + svdot_lane_za32_s16_vg1x2(slice_base, z_s16x2, z_s16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svdot_lane_za32_s16_vg1x4(slice_base, z_s16x4, z_s16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svdot_lane_za32_s8_vg1x2(slice_base, z_s8x2, z_s8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svdot_lane_za32_s8_vg1x4(slice_base, z_s8x4, z_s8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svdot_lane_za64_s16_vg1x2(slice_base, z_s16x2, z_s16, 2); // expected-error {{argument value 2 is outside the valid range [0, 1]}} + svdot_lane_za64_s16_vg1x4(slice_base, z_s16x4, z_s16, 2); // expected-error {{argument value 2 is outside the valid range [0, 1]}} + + // Multi, indexed (unsigned by signed) + svusdot_lane_za32_u8_vg1x2(slice_base, z_u8x2, z_s8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svusdot_lane_za32_u8_vg1x4(slice_base, z_u8x4, z_s8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + + // Multi, indexed (unsigned by signed) + svsudot_lane_za32_s8_vg1x2(slice_base, z_s8x2, z_u8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} + svsudot_lane_za32_s8_vg1x4(slice_base, z_s8x4, z_u8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} +} diff --git a/clang/unittests/Analysis/FlowSensitive/RecordOpsTest.cpp b/clang/unittests/Analysis/FlowSensitive/RecordOpsTest.cpp index 84fe675c32c2..cd6a37d370e8 100644 --- a/clang/unittests/Analysis/FlowSensitive/RecordOpsTest.cpp +++ b/clang/unittests/Analysis/FlowSensitive/RecordOpsTest.cpp @@ -89,8 +89,6 @@ TEST(RecordOpsTest, CopyRecord) { auto *S2Val = cast<RecordValue>(Env.getValue(S2)); EXPECT_NE(S1Val, S2Val); - S1Val->setProperty("prop", Env.getBoolLiteralValue(true)); - copyRecord(S1, S2, Env); EXPECT_EQ(getFieldValue(&S1, *OuterIntDecl, Env), @@ -104,8 +102,6 @@ TEST(RecordOpsTest, CopyRecord) { S1Val = cast<RecordValue>(Env.getValue(S1)); S2Val = cast<RecordValue>(Env.getValue(S2)); EXPECT_NE(S1Val, S2Val); - - EXPECT_EQ(S2Val->getProperty("prop"), &Env.getBoolLiteralValue(true)); }); } @@ -150,9 +146,6 @@ TEST(RecordOpsTest, RecordsEqual) { Env.setValue(S1.getSyntheticField("synth_int"), Env.create<IntegerValue>()); - cast<RecordValue>(Env.getValue(S1)) - ->setProperty("prop", Env.getBoolLiteralValue(true)); - // Strategy: Create two equal records, then verify each of the various // ways in which records can differ causes recordsEqual to return false. // changes we can make to the record. @@ -202,36 +195,6 @@ TEST(RecordOpsTest, RecordsEqual) { EXPECT_FALSE(recordsEqual(S1, S2, Env)); copyRecord(S1, S2, Env); EXPECT_TRUE(recordsEqual(S1, S2, Env)); - - // S1 and S2 have the same property with different values. - cast<RecordValue>(Env.getValue(S2)) - ->setProperty("prop", Env.getBoolLiteralValue(false)); - EXPECT_FALSE(recordsEqual(S1, S2, Env)); - copyRecord(S1, S2, Env); - EXPECT_TRUE(recordsEqual(S1, S2, Env)); - - // S1 has a property that S2 doesn't have. - cast<RecordValue>(Env.getValue(S1)) - ->setProperty("other_prop", Env.getBoolLiteralValue(false)); - EXPECT_FALSE(recordsEqual(S1, S2, Env)); - // We modified S1 this time, so need to copy back the other way. - copyRecord(S2, S1, Env); - EXPECT_TRUE(recordsEqual(S1, S2, Env)); - - // S2 has a property that S1 doesn't have. - cast<RecordValue>(Env.getValue(S2)) - ->setProperty("other_prop", Env.getBoolLiteralValue(false)); - EXPECT_FALSE(recordsEqual(S1, S2, Env)); - copyRecord(S1, S2, Env); - EXPECT_TRUE(recordsEqual(S1, S2, Env)); - - // S1 and S2 have the same number of properties, but with different - // names. - cast<RecordValue>(Env.getValue(S1)) - ->setProperty("prop1", Env.getBoolLiteralValue(false)); - cast<RecordValue>(Env.getValue(S2)) - ->setProperty("prop2", Env.getBoolLiteralValue(false)); - EXPECT_FALSE(recordsEqual(S1, S2, Env)); }); } diff --git a/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp b/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp index 4c3cb322eacf..8d481788af20 100644 --- a/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp +++ b/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp @@ -623,11 +623,11 @@ TEST_F(JoinFlowConditionsTest, JoinDistinctButProvablyEquivalentValues) { }); } -class OptionalIntAnalysis final - : public DataflowAnalysis<OptionalIntAnalysis, NoopLattice> { +class NullPointerAnalysis final + : public DataflowAnalysis<NullPointerAnalysis, NoopLattice> { public: - explicit OptionalIntAnalysis(ASTContext &Context) - : DataflowAnalysis<OptionalIntAnalysis, NoopLattice>(Context) {} + explicit NullPointerAnalysis(ASTContext &Context) + : DataflowAnalysis<NullPointerAnalysis, NoopLattice>(Context) {} static NoopLattice initialElement() { return {}; } @@ -636,40 +636,37 @@ public: if (!CS) return; const Stmt *S = CS->getStmt(); - auto OptionalIntRecordDecl = recordDecl(hasName("OptionalInt")); - auto HasOptionalIntType = hasType(OptionalIntRecordDecl); - - SmallVector<BoundNodes, 1> Matches = match( - stmt(anyOf(cxxConstructExpr(HasOptionalIntType).bind("construct"), - cxxOperatorCallExpr( - callee(cxxMethodDecl(ofClass(OptionalIntRecordDecl)))) - .bind("operator"))), - *S, getASTContext()); - if (const auto *E = selectFirst<CXXConstructExpr>( - "construct", Matches)) { - cast<RecordValue>(Env.getValue(*E)) - ->setProperty("has_value", Env.getBoolLiteralValue(false)); - } else if (const auto *E = - selectFirst<CXXOperatorCallExpr>("operator", Matches)) { - assert(E->getNumArgs() > 0); - auto *Object = E->getArg(0); - assert(Object != nullptr); - - refreshRecordValue(*Object, Env) - .setProperty("has_value", Env.getBoolLiteralValue(true)); + const Expr *E = dyn_cast<Expr>(S); + if (!E) + return; + + if (!E->getType()->isPointerType()) + return; + + // Make sure we have a `PointerValue` for `E`. + auto *PtrVal = cast_or_null<PointerValue>(Env.getValue(*E)); + if (PtrVal == nullptr) { + PtrVal = cast<PointerValue>(Env.createValue(E->getType())); + Env.setValue(*E, *PtrVal); } + + if (auto *Cast = dyn_cast<ImplicitCastExpr>(E); + Cast && Cast->getCastKind() == CK_NullToPointer) + PtrVal->setProperty("is_null", Env.getBoolLiteralValue(true)); + else if (auto *Op = dyn_cast<UnaryOperator>(E); + Op && Op->getOpcode() == UO_AddrOf) + PtrVal->setProperty("is_null", Env.getBoolLiteralValue(false)); } ComparisonResult compare(QualType Type, const Value &Val1, const Environment &Env1, const Value &Val2, const Environment &Env2) override { - // Nothing to say about a value that does not model an `OptionalInt`. - if (!Type->isRecordType() || - Type->getAsCXXRecordDecl()->getQualifiedNameAsString() != "OptionalInt") + // Nothing to say about a value that is not a pointer. + if (!Type->isPointerType()) return ComparisonResult::Unknown; - auto *Prop1 = Val1.getProperty("has_value"); - auto *Prop2 = Val2.getProperty("has_value"); + auto *Prop1 = Val1.getProperty("is_null"); + auto *Prop2 = Val2.getProperty("is_null"); assert(Prop1 != nullptr && Prop2 != nullptr); return areEquivalentValues(*Prop1, *Prop2) ? ComparisonResult::Same : ComparisonResult::Different; @@ -678,23 +675,22 @@ public: bool merge(QualType Type, const Value &Val1, const Environment &Env1, const Value &Val2, const Environment &Env2, Value &MergedVal, Environment &MergedEnv) override { - // Nothing to say about a value that does not model an `OptionalInt`. - if (!Type->isRecordType() || - Type->getAsCXXRecordDecl()->getQualifiedNameAsString() != "OptionalInt") + // Nothing to say about a value that is not a pointer. + if (!Type->isPointerType()) return false; - auto *HasValue1 = cast_or_null<BoolValue>(Val1.getProperty("has_value")); - if (HasValue1 == nullptr) + auto *IsNull1 = cast_or_null<BoolValue>(Val1.getProperty("is_null")); + if (IsNull1 == nullptr) return false; - auto *HasValue2 = cast_or_null<BoolValue>(Val2.getProperty("has_value")); - if (HasValue2 == nullptr) + auto *IsNull2 = cast_or_null<BoolValue>(Val2.getProperty("is_null")); + if (IsNull2 == nullptr) return false; - if (HasValue1 == HasValue2) - MergedVal.setProperty("has_value", *HasValue1); + if (IsNull1 == IsNull2) + MergedVal.setProperty("is_null", *IsNull1); else - MergedVal.setProperty("has_value", MergedEnv.makeTopBoolValue()); + MergedVal.setProperty("is_null", MergedEnv.makeTopBoolValue()); return true; } }; @@ -703,23 +699,14 @@ class WideningTest : public Test { protected: template <typename Matcher> void runDataflow(llvm::StringRef Code, Matcher Match) { - tooling::FileContentMappings FilesContents; - FilesContents.push_back( - std::make_pair<std::string, std::string>("widening_test_defs.h", R"( - struct OptionalInt { - OptionalInt() = default; - OptionalInt& operator=(int); - }; - )")); ASSERT_THAT_ERROR( - checkDataflow<OptionalIntAnalysis>( - AnalysisInputs<OptionalIntAnalysis>( + checkDataflow<NullPointerAnalysis>( + AnalysisInputs<NullPointerAnalysis>( Code, ast_matchers::hasName("target"), [](ASTContext &Context, Environment &Env) { - return OptionalIntAnalysis(Context); + return NullPointerAnalysis(Context); }) - .withASTBuildArgs({"-fsyntax-only", "-std=c++17"}) - .withASTBuildVirtualMappedFiles(std::move(FilesContents)), + .withASTBuildArgs({"-fsyntax-only", "-std=c++17"}), /*VerifyResults=*/[&Match](const llvm::StringMap< DataflowAnalysisState<NoopLattice>> &Results, @@ -731,13 +718,12 @@ protected: TEST_F(WideningTest, JoinDistinctValuesWithDistinctProperties) { std::string Code = R"( - #include "widening_test_defs.h" - void target(bool Cond) { - OptionalInt Foo; + int *Foo = nullptr; + int i = 0; /*[[p1]]*/ if (Cond) { - Foo = 1; + Foo = &i; /*[[p2]]*/ } (void)0; @@ -760,27 +746,27 @@ TEST_F(WideningTest, JoinDistinctValuesWithDistinctProperties) { return Env.getValue(*FooDecl); }; - EXPECT_EQ(GetFooValue(Env1)->getProperty("has_value"), - &Env1.getBoolLiteralValue(false)); - EXPECT_EQ(GetFooValue(Env2)->getProperty("has_value"), - &Env2.getBoolLiteralValue(true)); + EXPECT_EQ(GetFooValue(Env1)->getProperty("is_null"), + &Env1.getBoolLiteralValue(true)); + EXPECT_EQ(GetFooValue(Env2)->getProperty("is_null"), + &Env2.getBoolLiteralValue(false)); EXPECT_TRUE( - isa<TopBoolValue>(GetFooValue(Env3)->getProperty("has_value"))); + isa<TopBoolValue>(GetFooValue(Env3)->getProperty("is_null"))); }); } TEST_F(WideningTest, JoinDistinctValuesWithSameProperties) { std::string Code = R"( - #include "widening_test_defs.h" - void target(bool Cond) { - OptionalInt Foo; + int *Foo = nullptr; + int i1 = 0; + int i2 = 0; /*[[p1]]*/ if (Cond) { - Foo = 1; + Foo = &i1; /*[[p2]]*/ } else { - Foo = 2; + Foo = &i2; /*[[p3]]*/ } (void)0; @@ -805,14 +791,14 @@ TEST_F(WideningTest, JoinDistinctValuesWithSameProperties) { return Env.getValue(*FooDecl); }; - EXPECT_EQ(GetFooValue(Env1)->getProperty("has_value"), - &Env1.getBoolLiteralValue(false)); - EXPECT_EQ(GetFooValue(Env2)->getProperty("has_value"), - &Env2.getBoolLiteralValue(true)); - EXPECT_EQ(GetFooValue(Env3)->getProperty("has_value"), - &Env3.getBoolLiteralValue(true)); - EXPECT_EQ(GetFooValue(Env4)->getProperty("has_value"), - &Env4.getBoolLiteralValue(true)); + EXPECT_EQ(GetFooValue(Env1)->getProperty("is_null"), + &Env1.getBoolLiteralValue(true)); + EXPECT_EQ(GetFooValue(Env2)->getProperty("is_null"), + &Env2.getBoolLiteralValue(false)); + EXPECT_EQ(GetFooValue(Env3)->getProperty("is_null"), + &Env3.getBoolLiteralValue(false)); + EXPECT_EQ(GetFooValue(Env4)->getProperty("is_null"), + &Env4.getBoolLiteralValue(false)); }); } @@ -849,13 +835,13 @@ TEST_F(WideningTest, DistinctPointersToTheSameLocationAreEquivalent) { TEST_F(WideningTest, DistinctValuesWithSamePropertiesAreEquivalent) { std::string Code = R"( - #include "widening_test_defs.h" - void target(bool Cond) { - OptionalInt Foo; - Foo = 1; + int *Foo; + int i1 = 0; + int i2 = 0; + Foo = &i1; while (Cond) { - Foo = 2; + Foo = &i2; } (void)0; /*[[p]]*/ @@ -872,8 +858,8 @@ TEST_F(WideningTest, DistinctValuesWithSamePropertiesAreEquivalent) { ASSERT_THAT(FooDecl, NotNull()); const auto *FooVal = Env.getValue(*FooDecl); - EXPECT_EQ(FooVal->getProperty("has_value"), - &Env.getBoolLiteralValue(true)); + EXPECT_EQ(FooVal->getProperty("is_null"), + &Env.getBoolLiteralValue(false)); }); } diff --git a/compiler-rt/lib/hwasan/hwasan_report.cpp b/compiler-rt/lib/hwasan/hwasan_report.cpp index 71155c9814c1..bbe89112e4db 100644 --- a/compiler-rt/lib/hwasan/hwasan_report.cpp +++ b/compiler-rt/lib/hwasan/hwasan_report.cpp @@ -648,23 +648,19 @@ void BaseReport::PrintHeapOrGlobalCandidate() const { if (candidate.heap.is_allocated) { uptr offset; const char *whence; - const char *cause; if (candidate.heap.begin <= untagged_addr && untagged_addr < candidate.heap.end) { offset = untagged_addr - candidate.heap.begin; whence = "inside"; - cause = "heap-use-after-free"; } else if (candidate.after) { offset = untagged_addr - candidate.heap.end; whence = "after"; - cause = "heap-buffer-overflow"; } else { offset = candidate.heap.begin - untagged_addr; whence = "before"; - cause = "heap-buffer-underflow"; } Printf("%s", d.Error()); - Printf("\nCause: %s\n", cause); + Printf("\nCause: heap-buffer-overflow\n"); Printf("%s", d.Default()); Printf("%s", d.Location()); Printf("%p is located %zd bytes %s a %zd-byte region [%p,%p)\n", diff --git a/compiler-rt/test/hwasan/TestCases/Linux/syscalls.cpp b/compiler-rt/test/hwasan/TestCases/Linux/syscalls.cpp index eee43f458fac..154b69898993 100644 --- a/compiler-rt/test/hwasan/TestCases/Linux/syscalls.cpp +++ b/compiler-rt/test/hwasan/TestCases/Linux/syscalls.cpp @@ -26,7 +26,7 @@ int main(int argc, char *argv[]) { __sanitizer_syscall_pre_recvmsg(0, buf - 1, 0); // CHECK: HWAddressSanitizer: tag-mismatch on address [[PTR:0x[a-f0-9]+]] - // CHECK: Cause: heap-buffer-underflow + // CHECK: Cause: heap-buffer-overflow // CHECK: [[PTR]] is located 1 bytes before a 1000-byte region free(buf); diff --git a/compiler-rt/test/hwasan/TestCases/heap-buffer-overflow.c b/compiler-rt/test/hwasan/TestCases/heap-buffer-overflow.c index 44643eef57e2..d390017dd755 100644 --- a/compiler-rt/test/hwasan/TestCases/heap-buffer-overflow.c +++ b/compiler-rt/test/hwasan/TestCases/heap-buffer-overflow.c @@ -29,8 +29,7 @@ int main(int argc, char **argv) { if (size == 1000000) { fprintf(stderr, "is a large allocated heap chunk; size: 1003520 offset: %d\n", offset); - fprintf(stderr, "Cause: heap-buffer-%s\n", - offset == -30 ? "underflow" : "overflow"); + fprintf(stderr, "Cause: heap-buffer-overflow\n"); fprintf(stderr, "is located %s a 1000000-byte region\n", offset == -30 ? "30 bytes before" : "0 bytes after"); return -1; @@ -45,11 +44,11 @@ int main(int argc, char **argv) { // CHECK80: Cause: heap-buffer-overflow // CHECK80: is located 50 bytes after a 30-byte region // - // CHECKm30: Cause: heap-buffer-underflow + // CHECKm30: Cause: heap-buffer-overflow // CHECKm30: is located 30 bytes before a 30-byte region // // CHECKMm30: is a large allocated heap chunk; size: 1003520 offset: -30 - // CHECKMm30: Cause: heap-buffer-underflow + // CHECKMm30: Cause: heap-buffer-overflow // CHECKMm30: is located 30 bytes before a 1000000-byte region // // CHECKM: is a large allocated heap chunk; size: 1003520 offset: 1000000 diff --git a/compiler-rt/test/hwasan/TestCases/stack-uar.c b/compiler-rt/test/hwasan/TestCases/stack-uar.c index 48440a47d5f5..29941e617ad6 100644 --- a/compiler-rt/test/hwasan/TestCases/stack-uar.c +++ b/compiler-rt/test/hwasan/TestCases/stack-uar.c @@ -51,14 +51,14 @@ int main() { // CHECK: is located in stack of thread // CHECK: Potentially referenced stack objects: // CHECK-NEXT: {{zzz|yyy}} in buggy {{.*}}stack-uar.c: - // CHECK-NEXT: Memory tags around the buggy address + // CHECK: Memory tags around the buggy address // NOSYM: Previously allocated frames: // NOSYM-NEXT: record_addr:0x{{.*}} record:0x{{.*}} ({{.*}}/stack-uar.c.tmp+0x{{.*}}){{$}} // NOSYM-NEXT: record_addr:0x{{.*}} record:0x{{.*}} ({{.*}}/stack-uar.c.tmp+0x{{.*}}){{$}} // NOSYM-NEXT: record_addr:0x{{.*}} record:0x{{.*}} ({{.*}}/stack-uar.c.tmp+0x{{.*}}){{$}} // NOSYM-NEXT: record_addr:0x{{.*}} record:0x{{.*}} ({{.*}}/stack-uar.c.tmp+0x{{.*}}){{$}} - // NOSYM-NEXT: Memory tags around the buggy address + // NOSYM: Memory tags around the buggy address // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in main } diff --git a/compiler-rt/test/hwasan/TestCases/stack-uas.c b/compiler-rt/test/hwasan/TestCases/stack-uas.c index 4455e5910074..d38eedb87fc2 100644 --- a/compiler-rt/test/hwasan/TestCases/stack-uas.c +++ b/compiler-rt/test/hwasan/TestCases/stack-uas.c @@ -70,14 +70,14 @@ int main() { // CHECK: is located in stack of thread // CHECK: Potentially referenced stack objects: // CHECK-NEXT: {{zzz|yyy}} in buggy {{.*}}stack-uas.c: - // CHECK-NEXT: Memory tags around the buggy address + // CHECK: Memory tags around the buggy address // NOSYM: Previously allocated frames: // NOSYM-NEXT: record_addr:0x{{.*}} record:0x{{.*}} ({{.*}}/stack-uas.c.tmp+0x{{.*}}){{$}} // NOSYM-NEXT: record_addr:0x{{.*}} record:0x{{.*}} ({{.*}}/stack-uas.c.tmp+0x{{.*}}){{$}} // NOSYM-NEXT: record_addr:0x{{.*}} record:0x{{.*}} ({{.*}}/stack-uas.c.tmp+0x{{.*}}){{$}} // NOSYM-NEXT: record_addr:0x{{.*}} record:0x{{.*}} ({{.*}}/stack-uas.c.tmp+0x{{.*}}){{$}} - // NOSYM-NEXT: Memory tags around the buggy address + // NOSYM: Memory tags around the buggy address // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in buggy } diff --git a/flang/docs/Intrinsics.md b/flang/docs/Intrinsics.md index fef2b4ea4dd8..189920a0881b 100644 --- a/flang/docs/Intrinsics.md +++ b/flang/docs/Intrinsics.md @@ -695,6 +695,11 @@ CACHESIZE, EOF, FP_CLASS, INT_PTR_KIND, ISNAN, LOC MALLOC ``` +### Library subroutine +``` +CALL GETLOG(USRNAME) +``` + ## Intrinsic Procedure Name Resolution When the name of a procedure in a program is the same as the one of an intrinsic @@ -754,6 +759,7 @@ This phase currently supports all the intrinsic procedures listed above but the | Intrinsic subroutines |MVBITS (elemental), CPU_TIME, DATE_AND_TIME, EVENT_QUERY, EXECUTE_COMMAND_LINE, GET_COMMAND, GET_COMMAND_ARGUMENT, GET_ENVIRONMENT_VARIABLE, MOVE_ALLOC, RANDOM_INIT, RANDOM_NUMBER, RANDOM_SEED, SYSTEM_CLOCK | | Atomic intrinsic subroutines | ATOMIC_ADD | | Collective intrinsic subroutines | CO_REDUCE | +| Library subroutines | GETLOG| ### Intrinsic Function Folding diff --git a/flang/include/flang/Runtime/extensions.h b/flang/include/flang/Runtime/extensions.h index ad592814e5ac..175113c57ccb 100644 --- a/flang/include/flang/Runtime/extensions.h +++ b/flang/include/flang/Runtime/extensions.h @@ -14,6 +14,7 @@ #define FORTRAN_PROCEDURE_NAME(name) name##_ +#include <cstddef> #include <cstdint> extern "C" { @@ -28,5 +29,8 @@ std::int32_t FORTRAN_PROCEDURE_NAME(iargc)(); void FORTRAN_PROCEDURE_NAME(getarg)( std::int32_t &n, std::int8_t *arg, std::int64_t length); +// GNU extension subroutine GETLOG(C). +void FORTRAN_PROCEDURE_NAME(getlog)(std::byte *name, std::int64_t length); + } // extern "C" #endif // FORTRAN_RUNTIME_EXTENSIONS_H_ diff --git a/flang/lib/Lower/OpenMP.cpp b/flang/lib/Lower/OpenMP.cpp index 1acc49abb1da..8ed31766725e 100644 --- a/flang/lib/Lower/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP.cpp @@ -2197,10 +2197,12 @@ static void genBodyOfTargetDataOp( if (fir::isa_builtin_cptr_type(refType.getElementType())) { converter.bindSymbol(*argSymbol, arg); } else { + // Avoid capture of a reference to a structured binding. + const Fortran::semantics::Symbol *sym = argSymbol; extVal.match( [&](const fir::MutableBoxValue &mbv) { converter.bindSymbol( - *argSymbol, + *sym, fir::MutableBoxValue( arg, fir::factory::getNonDeferredLenParams(extVal), {})); }, @@ -2489,7 +2491,7 @@ static void genBodyOfTargetOp( // Bind the symbols to their corresponding block arguments. for (auto [argIndex, argSymbol] : llvm::enumerate(mapSymbols)) { const mlir::BlockArgument &arg = region.getArgument(argIndex); - // Avoid capture of reference to a structured binding. + // Avoid capture of a reference to a structured binding. const Fortran::semantics::Symbol *sym = argSymbol; fir::ExtendedValue extVal = converter.getSymbolExtendedValue(*sym); extVal.match( diff --git a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp index 12f354a47c2b..c89ee6d5e203 100644 --- a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp +++ b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp @@ -1162,14 +1162,11 @@ void SimplifyIntrinsicsPass::simplifyMinMaxlocReduction( mlir::Operation::operand_range args = call.getArgs(); - mlir::SymbolRefAttr callee = call.getCalleeAttr(); - mlir::StringRef funcNameBase = callee.getLeafReference().getValue(); - bool isDim = funcNameBase.ends_with("Dim"); - mlir::Value back = args[isDim ? 7 : 6]; + mlir::Value back = args[6]; if (isTrueOrNotConstant(back)) return; - mlir::Value mask = args[isDim ? 6 : 5]; + mlir::Value mask = args[5]; mlir::Value maskDef = findMaskDef(mask); // maskDef is set to NULL when the defining op is not one we accept. @@ -1178,8 +1175,10 @@ void SimplifyIntrinsicsPass::simplifyMinMaxlocReduction( if (maskDef == NULL) return; + mlir::SymbolRefAttr callee = call.getCalleeAttr(); + mlir::StringRef funcNameBase = callee.getLeafReference().getValue(); unsigned rank = getDimCount(args[1]); - if ((isDim && rank != 1) || !(rank > 0)) + if (funcNameBase.ends_with("Dim") || !(rank > 0)) return; fir::FirOpBuilder builder{getSimplificationBuilder(call, kindMap)}; @@ -1235,7 +1234,7 @@ void SimplifyIntrinsicsPass::simplifyMinMaxlocReduction( mlir::func::FuncOp newFunc = getOrCreateFunction(builder, funcName, typeGenerator, bodyGenerator); builder.create<fir::CallOp>(loc, newFunc, - mlir::ValueRange{args[0], args[1], mask}); + mlir::ValueRange{args[0], args[1], args[5]}); call->dropAllReferences(); call->erase(); } diff --git a/flang/lib/Optimizer/Transforms/StackArrays.cpp b/flang/lib/Optimizer/Transforms/StackArrays.cpp index b51e2aae1a9d..1c213abefe6f 100644 --- a/flang/lib/Optimizer/Transforms/StackArrays.cpp +++ b/flang/lib/Optimizer/Transforms/StackArrays.cpp @@ -449,7 +449,7 @@ StackArraysAnalysisWrapper::analyseFunction(mlir::Operation *func) { const LatticePoint *lattice = solver.lookupState<LatticePoint>(op); // there will be no lattice for an unreachable block if (lattice) - point.join(*lattice); + (void)point.join(*lattice); }; func->walk([&](mlir::func::ReturnOp child) { joinOperationLattice(child); }); func->walk([&](fir::UnreachableOp child) { joinOperationLattice(child); }); diff --git a/flang/runtime/character.cpp b/flang/runtime/character.cpp index 2afde7cd5e83..084aa0c9c8b6 100644 --- a/flang/runtime/character.cpp +++ b/flang/runtime/character.cpp @@ -11,6 +11,7 @@ #include "tools.h" #include "flang/Common/bit-population-count.h" #include "flang/Common/uint128.h" +#include "flang/Runtime/character.h" #include "flang/Runtime/cpp-type.h" #include "flang/Runtime/descriptor.h" #include <algorithm> @@ -464,27 +465,6 @@ static void GeneralCharFuncKind(Descriptor &result, const Descriptor &string, } } -template <typename TO, typename FROM> -static void CopyAndPad( - TO *to, const FROM *from, std::size_t toChars, std::size_t fromChars) { - if constexpr (sizeof(TO) != sizeof(FROM)) { - std::size_t copyChars{std::min(toChars, fromChars)}; - for (std::size_t j{0}; j < copyChars; ++j) { - to[j] = from[j]; - } - for (std::size_t j{copyChars}; j < toChars; ++j) { - to[j] = static_cast<TO>(' '); - } - } else if (toChars <= fromChars) { - std::memcpy(to, from, toChars * sizeof(TO)); - } else { - std::memcpy(to, from, fromChars * sizeof(TO)); - for (std::size_t j{fromChars}; j < toChars; ++j) { - to[j] = static_cast<TO>(' '); - } - } -} - template <typename CHAR, bool ISMIN> static void MaxMinHelper(Descriptor &accumulator, const Descriptor &x, const Terminator &terminator) { diff --git a/flang/runtime/extensions.cpp b/flang/runtime/extensions.cpp index b8e9b6eae132..1c025d40b395 100644 --- a/flang/runtime/extensions.cpp +++ b/flang/runtime/extensions.cpp @@ -10,13 +10,29 @@ // extensions that will eventually be implemented in Fortran. #include "flang/Runtime/extensions.h" +#include "tools.h" #include "flang/Runtime/command.h" #include "flang/Runtime/descriptor.h" #include "flang/Runtime/io-api.h" +#if _REENTRANT || _POSIX_C_SOURCE >= 199506L +// System is posix-compliant and has getlogin_r +#include <unistd.h> +#endif + extern "C" { namespace Fortran::runtime { + +void GetUsernameEnvVar( + const char *envName, std::byte *arg, std::int64_t length) { + Descriptor name{*Descriptor::Create( + 1, std::strlen(envName) + 1, const_cast<char *>(envName), 0)}; + Descriptor value{*Descriptor::Create(1, length, arg, 0)}; + + RTNAME(GetEnvVariable) + (name, &value, nullptr, false, nullptr, __FILE__, __LINE__); +} namespace io { // SUBROUTINE FLUSH(N) // FLUSH N @@ -37,5 +53,28 @@ void FORTRAN_PROCEDURE_NAME(getarg)( (void)RTNAME(GetCommandArgument)( n, &value, nullptr, nullptr, __FILE__, __LINE__); } + +// CALL GETLOG(USRNAME) +void FORTRAN_PROCEDURE_NAME(getlog)(std::byte *arg, std::int64_t length) { +#if _REENTRANT || _POSIX_C_SOURCE >= 199506L + const int nameMaxLen{LOGIN_NAME_MAX + 1}; + char str[nameMaxLen]; + + int error{getlogin_r(str, nameMaxLen)}; + if (error == 0) { + // no error: find first \0 in string then pad from there + CopyAndPad(reinterpret_cast<char *>(arg), str, length, std::strlen(str)); + } else { + // error occur: get username from environment variable + GetUsernameEnvVar("LOGNAME", arg, length); + } +#elif _WIN32 + // Get username from environment to avoid link to Advapi32.lib + GetUsernameEnvVar("USERNAME", arg, length); +#else + GetUsernameEnvVar("LOGNAME", arg, length); +#endif +} + } // namespace Fortran::runtime } // extern "C" diff --git a/flang/runtime/tools.h b/flang/runtime/tools.h index ea659190e143..9811bce25acd 100644 --- a/flang/runtime/tools.h +++ b/flang/runtime/tools.h @@ -411,5 +411,27 @@ RT_API_ATTRS void ShallowCopy(const Descriptor &to, const Descriptor &from, bool toIsContiguous, bool fromIsContiguous); RT_API_ATTRS void ShallowCopy(const Descriptor &to, const Descriptor &from); +// Defines a utility function for copying and padding characters +template <typename TO, typename FROM> +RT_API_ATTRS void CopyAndPad( + TO *to, const FROM *from, std::size_t toChars, std::size_t fromChars) { + if constexpr (sizeof(TO) != sizeof(FROM)) { + std::size_t copyChars{std::min(toChars, fromChars)}; + for (std::size_t j{0}; j < copyChars; ++j) { + to[j] = from[j]; + } + for (std::size_t j{copyChars}; j < toChars; ++j) { + to[j] = static_cast<TO>(' '); + } + } else if (toChars <= fromChars) { + std::memcpy(to, from, toChars * sizeof(TO)); + } else { + std::memcpy(to, from, std::min(toChars, fromChars) * sizeof(TO)); + for (std::size_t j{fromChars}; j < toChars; ++j) { + to[j] = static_cast<TO>(' '); + } + } +} + } // namespace Fortran::runtime #endif // FORTRAN_RUNTIME_TOOLS_H_ diff --git a/flang/test/Driver/dynamic-linker.f90 b/flang/test/Driver/dynamic-linker.f90 index df119c22a2ea..1cbd407d21ce 100644 --- a/flang/test/Driver/dynamic-linker.f90 +++ b/flang/test/Driver/dynamic-linker.f90 @@ -7,6 +7,7 @@ ! RUN: %flang -### --target=x86_64-windows-msvc -rpath /path/to/dir -shared \ ! RUN: -static %s 2>&1 | FileCheck \ ! RUN: --check-prefixes=MSVC-LINKER-OPTIONS %s +! RUN: %flang -### --target=aarch64-linux-none -rdynamic %s 2>&1 | FileCheck --check-prefixes=RDYNAMIC-LINKER-OPTION %s ! TODO: Could the linker have an extension or a suffix? ! GNU-LINKER-OPTIONS: "{{.*}}ld{{(.exe)?}}" @@ -14,6 +15,9 @@ ! GNU-LINKER-OPTIONS-SAME: "-static" ! GNU-LINKER-OPTIONS-SAME: "-rpath" "/path/to/dir" +! RDYNAMIC-LINKER-OPTION: "{{.*}}ld" +! RDYNAMIC-LINKER-OPTION-SAME: "-export-dynamic" + ! For MSVC, adding -static does not add any additional linker options. ! MSVC-LINKER-OPTIONS: "{{.*}}link{{(.exe)?}}" ! MSVC-LINKER-OPTIONS-SAME: "-dll" diff --git a/flang/test/Driver/no-duplicate-main.f90 b/flang/test/Driver/no-duplicate-main.f90 index b884e7ecd7f1..88f4430828e0 100644 --- a/flang/test/Driver/no-duplicate-main.f90 +++ b/flang/test/Driver/no-duplicate-main.f90 @@ -1,4 +1,4 @@ -! UNSUPPORTED: system-windows, system-darwin +! UNSUPPORTED: system-windows, system-darwin, system-aix ! RUN: %flang -x ir -o %t.c-object -c %S/Inputs/no_duplicate_main.ll ! RUN: %flang -o %t -c %s diff --git a/flang/test/Runtime/no-cpp-dep.c b/flang/test/Runtime/no-cpp-dep.c index f8fe97b5bf78..654bebed345b 100644 --- a/flang/test/Runtime/no-cpp-dep.c +++ b/flang/test/Runtime/no-cpp-dep.c @@ -5,7 +5,10 @@ a C compiler. REQUIRES: c-compiler -RUN: %cc -std=c99 %s -I%include %libruntime %libdecimal -lm -o /dev/null +RUN: %if system-aix %{ export OBJECT_MODE=64 %} +RUN: %cc -std=c99 %s -I%include %libruntime %libdecimal -lm \ +RUN: %if system-aix %{-lpthread %} +RUN: rm a.out */ #include "flang/Runtime/entry-names.h" diff --git a/flang/test/Transforms/simplifyintrinsics.fir b/flang/test/Transforms/simplifyintrinsics.fir index d42924a17a80..0bd6ac7c436f 100644 --- a/flang/test/Transforms/simplifyintrinsics.fir +++ b/flang/test/Transforms/simplifyintrinsics.fir @@ -2115,13 +2115,13 @@ func.func @_QPtestminloc_doesntwork1d_back(%arg0: !fir.ref<!fir.array<10xi32>> { // CHECK-NOT: fir.call @_FortranAMinlocInteger4x1_i32_contract_simplified({{.*}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>) -> () // ----- -// Check Minloc is simplified when DIM arg is set so long as the result is scalar +// Check Minloc is not simplified when DIM arg is set -func.func @_QPtestminloc_1d_dim(%arg0: !fir.ref<!fir.array<10xi32>> {fir.bindc_name = "a"}) -> !fir.array<1xi32> { +func.func @_QPtestminloc_doesntwork1d_dim(%arg0: !fir.ref<!fir.array<10xi32>> {fir.bindc_name = "a"}) -> !fir.array<1xi32> { %0 = fir.alloca !fir.box<!fir.heap<i32>> %c10 = arith.constant 10 : index %c1 = arith.constant 1 : index - %1 = fir.alloca !fir.array<1xi32> {bindc_name = "testminloc_1d_dim", uniq_name = "_QFtestminloc_1d_dimEtestminloc_1d_dim"} + %1 = fir.alloca !fir.array<1xi32> {bindc_name = "testminloc_doesntwork1d_dim", uniq_name = "_QFtestminloc_doesntwork1d_dimEtestminloc_doesntwork1d_dim"} %2 = fir.shape %c1 : (index) -> !fir.shape<1> %3 = fir.array_load %1(%2) : (!fir.ref<!fir.array<1xi32>>, !fir.shape<1>) -> !fir.array<1xi32> %4 = fir.shape %c10 : (index) -> !fir.shape<1> @@ -2156,65 +2156,11 @@ func.func @_QPtestminloc_1d_dim(%arg0: !fir.ref<!fir.array<10xi32>> {fir.bindc_n %21 = fir.load %1 : !fir.ref<!fir.array<1xi32>> return %21 : !fir.array<1xi32> } -// CHECK-LABEL: func.func @_QPtestminloc_1d_dim( +// CHECK-LABEL: func.func @_QPtestminloc_doesntwork1d_dim( // CHECK-SAME: %[[ARR:.*]]: !fir.ref<!fir.array<10xi32>> {fir.bindc_name = "a"}) -> !fir.array<1xi32> { -// CHECK: fir.call @_FortranAMinlocDimx1_i32_contract_simplified({{.*}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>) -> () - -// CHECK-LABEL: func.func private @_FortranAMinlocDimx1_i32_contract_simplified(%arg0: !fir.ref<!fir.box<none>>, %arg1: !fir.box<none>, %arg2: !fir.box<none>) attributes {llvm.linkage = #llvm.linkage<linkonce_odr>} { -// CHECK-NEXT: %[[V0:.*]] = fir.alloca i32 -// CHECK-NEXT: %c0_i32 = arith.constant 0 : i32 -// CHECK-NEXT: %c1 = arith.constant 1 : index -// CHECK-NEXT: %[[V1:.*]] = fir.allocmem !fir.array<1xi32> -// CHECK-NEXT: %[[V2:.*]] = fir.shape %c1 : (index) -> !fir.shape<1> -// CHECK-NEXT: %[[V3:.*]] = fir.embox %[[V1]](%[[V2]]) : (!fir.heap<!fir.array<1xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<1xi32>>> -// CHECK-NEXT: %c0 = arith.constant 0 : index -// CHECK-NEXT: %[[V4:.*]] = fir.coordinate_of %[[V3]], %c0 : (!fir.box<!fir.heap<!fir.array<1xi32>>>, index) -> !fir.ref<i32> -// CHECK-NEXT: fir.store %c0_i32 to %[[V4]] : !fir.ref<i32> -// CHECK-NEXT: %c0_0 = arith.constant 0 : index -// CHECK-NEXT: %[[V5:.*]] = fir.convert %arg1 : (!fir.box<none>) -> !fir.box<!fir.array<?xi32>> -// CHECK-NEXT: %c1_i32 = arith.constant 1 : i32 -// CHECK-NEXT: %c0_i32_1 = arith.constant 0 : i32 -// CHECK-NEXT: fir.store %c0_i32_1 to %[[V0]] : !fir.ref<i32> -// CHECK-NEXT: %c2147483647_i32 = arith.constant 2147483647 : i32 -// CHECK-NEXT: %c1_2 = arith.constant 1 : index -// CHECK-NEXT: %c0_3 = arith.constant 0 : index -// CHECK-NEXT: %[[V6:.*]]:3 = fir.box_dims %[[V5]], %c0_3 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index) -// CHECK-NEXT: %[[V7:.*]] = arith.subi %[[V6]]#1, %c1_2 : index -// CHECK-NEXT: %[[V8:.*]] = fir.do_loop %arg3 = %c0_0 to %[[V7]] step %c1_2 iter_args(%arg4 = %c2147483647_i32) -> (i32) { -// CHECK-NEXT: fir.store %c1_i32 to %[[V0]] : !fir.ref<i32> -// CHECK-NEXT: %[[V12:.*]] = fir.coordinate_of %[[V5]], %arg3 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32> -// CHECK-NEXT: %[[V13:.*]] = fir.load %[[V12]] : !fir.ref<i32> -// CHECK-NEXT: %[[V14:.*]] = arith.cmpi slt, %[[V13]], %arg4 : i32 -// CHECK-NEXT: %[[V15:.*]] = fir.if %[[V14]] -> (i32) { -// CHECK-NEXT: %c1_i32_4 = arith.constant 1 : i32 -// CHECK-NEXT: %c0_5 = arith.constant 0 : index -// CHECK-NEXT: %[[V16:.*]] = fir.coordinate_of %[[V3]], %c0_5 : (!fir.box<!fir.heap<!fir.array<1xi32>>>, index) -> !fir.ref<i32> -// CHECK-NEXT: %[[V17:.*]] = fir.convert %arg3 : (index) -> i32 -// CHECK-NEXT: %[[V18:.*]] = arith.addi %[[V17]], %c1_i32_4 : i32 -// CHECK-NEXT: fir.store %[[V18]] to %[[V16]] : !fir.ref<i32> -// CHECK-NEXT: fir.result %[[V13]] : i32 -// CHECK-NEXT: } else { -// CHECK-NEXT: fir.result %arg4 : i32 -// CHECK-NEXT: } -// CHECK-NEXT: fir.result %[[V15]] : i32 -// CHECK-NEXT: } -// CHECK-NEXT: %[[V9:.*]] = fir.load %[[V0]] : !fir.ref<i32> -// CHECK-NEXT: %[[V10:.*]] = arith.cmpi eq, %[[V9]], %c1_i32 : i32 -// CHECK-NEXT: fir.if %[[V10]] { -// CHECK-NEXT: %c2147483647_i32_4 = arith.constant 2147483647 : i32 -// CHECK-NEXT: %[[V12]] = arith.cmpi eq, %c2147483647_i32_4, %[[V8]] : i32 -// CHECK-NEXT: fir.if %[[V12]] { -// CHECK-NEXT: %c0_5 = arith.constant 0 : index -// CHECK-NEXT: %[[V13]] = fir.coordinate_of %[[V3]], %c0_5 : (!fir.box<!fir.heap<!fir.array<1xi32>>>, index) -> !fir.ref<i32> -// CHECK-NEXT: fir.store %c1_i32 to %[[V13]] : !fir.ref<i32> -// CHECK-NEXT: } -// CHECK-NEXT: } -// CHECK-NEXT: %[[V11:.*]] = fir.convert %arg0 : (!fir.ref<!fir.box<none>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<1xi32>>>> -// CHECK-NEXT: fir.store %[[V3]] to %[[V11]] : !fir.ref<!fir.box<!fir.heap<!fir.array<1xi32>>>> -// CHECK-NEXT: return -// CHECK-NEXT: } - - +// CHECK-NOT: fir.call @_FortranAMinlocDimx1_i32_contract_simplified({{.*}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>) -> () +// CHECK: fir.call @_FortranAMinlocDim({{.*}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32, i32, !fir.ref<i8>, i32, !fir.box<none>, i1) -> none +// CHECK-NOT: fir.call @_FortranAMinlocDimx1_i32_contract_simplified({{.*}}) fastmath<contract> : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>) -> () // ----- // Check Minloc is not simplified when dimension of inputArr is unknown diff --git a/flang/unittests/Runtime/CommandTest.cpp b/flang/unittests/Runtime/CommandTest.cpp index 2b648b31666a..dfc3ad68b3ab 100644 --- a/flang/unittests/Runtime/CommandTest.cpp +++ b/flang/unittests/Runtime/CommandTest.cpp @@ -10,9 +10,15 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "flang/Runtime/descriptor.h" +#include "flang/Runtime/extensions.h" #include "flang/Runtime/main.h" +#include <cstddef> #include <cstdlib> +#if _REENTRANT || _POSIX_C_SOURCE >= 199506L +#include <limits.h> // LOGIN_NAME_MAX used in getlog test +#endif + using namespace Fortran::runtime; template <std::size_t n = 64> @@ -59,6 +65,13 @@ protected: return res; } + void CheckCharEqStr(const char *value, const std::string &expected) const { + ASSERT_NE(value, nullptr); + EXPECT_EQ(std::strncmp(value, expected.c_str(), expected.size()), 0) + << "expected: " << expected << "\n" + << "value: " << value; + } + void CheckDescriptorEqStr( const Descriptor *value, const std::string &expected) const { ASSERT_NE(value, nullptr); @@ -397,6 +410,11 @@ class EnvironmentVariables : public CommandFixture { protected: EnvironmentVariables() : CommandFixture(0, nullptr) { SetEnv("NAME", "VALUE"); +#ifdef _WIN32 + SetEnv("USERNAME", "loginName"); +#else + SetEnv("LOGNAME", "loginName"); +#endif SetEnv("EMPTY", ""); } @@ -494,3 +512,68 @@ TEST_F(EnvironmentVariables, ErrMsgTooShort) { 1); CheckDescriptorEqStr(errMsg.get(), "Mis"); } + +// username first char must not be null +TEST_F(EnvironmentVariables, GetlogGetName) { + const int charLen{3}; + char input[charLen]{"\0\0"}; + + FORTRAN_PROCEDURE_NAME(getlog) + (reinterpret_cast<std::byte *>(input), charLen); + + EXPECT_NE(input[0], '\0'); +} + +#if _REENTRANT || _POSIX_C_SOURCE >= 199506L +TEST_F(EnvironmentVariables, GetlogPadSpace) { + // guarantee 1 char longer than max, last char should be pad space + const int charLen{LOGIN_NAME_MAX + 2}; + char input[charLen]; + + FORTRAN_PROCEDURE_NAME(getlog) + (reinterpret_cast<std::byte *>(input), charLen); + + EXPECT_EQ(input[charLen - 1], ' '); +} +#endif + +#ifdef _WIN32 // Test ability to get name from environment variable +TEST_F(EnvironmentVariables, GetlogEnvGetName) { + if (EnableFineGrainedTests()) { + ASSERT_NE(std::getenv("USERNAME"), nullptr) + << "Environment variable USERNAME does not exist"; + + char input[]{"XXXXXXXXX"}; + FORTRAN_PROCEDURE_NAME(getlog) + (reinterpret_cast<std::byte *>(input), sizeof(input)); + + CheckCharEqStr(input, "loginName"); + } +} + +TEST_F(EnvironmentVariables, GetlogEnvBufferShort) { + if (EnableFineGrainedTests()) { + ASSERT_NE(std::getenv("USERNAME"), nullptr) + << "Environment variable USERNAME does not exist"; + + char input[]{"XXXXXX"}; + FORTRAN_PROCEDURE_NAME(getlog) + (reinterpret_cast<std::byte *>(input), sizeof(input)); + + CheckCharEqStr(input, "loginN"); + } +} + +TEST_F(EnvironmentVariables, GetlogEnvPadSpace) { + if (EnableFineGrainedTests()) { + ASSERT_NE(std::getenv("USERNAME"), nullptr) + << "Environment variable USERNAME does not exist"; + + char input[]{"XXXXXXXXXX"}; + FORTRAN_PROCEDURE_NAME(getlog) + (reinterpret_cast<std::byte *>(input), sizeof(input)); + + CheckCharEqStr(input, "loginName "); + } +} +#endif diff --git a/libc/src/__support/CPP/bit.h b/libc/src/__support/CPP/bit.h index 4de142b56165..122f6b8c3328 100644 --- a/libc/src/__support/CPP/bit.h +++ b/libc/src/__support/CPP/bit.h @@ -29,10 +29,10 @@ namespace LIBC_NAMESPACE::cpp { // UB in the implementation. template < typename To, typename From, - typename = cpp::enable_if_t<sizeof(To) == sizeof(From)>, - typename = cpp::enable_if_t<cpp::is_trivially_constructible<To>::value>, - typename = cpp::enable_if_t<cpp::is_trivially_copyable<To>::value>, - typename = cpp::enable_if_t<cpp::is_trivially_copyable<From>::value>> + typename = cpp::enable_if_t<sizeof(To) == sizeof(From) && + cpp::is_trivially_constructible<To>::value && + cpp::is_trivially_copyable<To>::value && + cpp::is_trivially_copyable<From>::value>> LIBC_INLINE constexpr To bit_cast(const From &from) { MSAN_UNPOISON(&from, sizeof(From)); #if LIBC_HAS_BUILTIN(__builtin_bit_cast) diff --git a/libc/src/__support/FPUtil/FPBits.h b/libc/src/__support/FPUtil/FPBits.h index 37e2820eab85..790449cc0080 100644 --- a/libc/src/__support/FPUtil/FPBits.h +++ b/libc/src/__support/FPUtil/FPBits.h @@ -11,6 +11,7 @@ #include "src/__support/CPP/bit.h" #include "src/__support/CPP/type_traits.h" +#include "src/__support/UInt128.h" #include "src/__support/common.h" #include "src/__support/macros/attributes.h" // LIBC_INLINE @@ -20,41 +21,37 @@ namespace LIBC_NAMESPACE { namespace fputil { -// A generic class to represent single precision, double precision, and quad -// precision IEEE 754 floating point formats. -// On most platforms, the 'float' type corresponds to single precision floating -// point numbers, the 'double' type corresponds to double precision floating -// point numers, and the 'long double' type corresponds to the quad precision -// floating numbers. On x86 platforms however, the 'long double' type maps to -// an x87 floating point format. This format is an IEEE 754 extension format. -// It is handled as an explicit specialization of this class. -template <typename T> struct FPBits : private FloatProperties<T> { - static_assert(cpp::is_floating_point_v<T>, - "FPBits instantiated with invalid type."); - using typename FloatProperties<T>::StorageType; - using FloatProperties<T>::TOTAL_LEN; +namespace internal { -private: - using FloatProperties<T>::EXP_SIG_MASK; - -public: - using FloatProperties<T>::EXP_MASK; - using FloatProperties<T>::EXP_BIAS; - using FloatProperties<T>::EXP_LEN; - using FloatProperties<T>::FRACTION_MASK; - using FloatProperties<T>::FRACTION_LEN; +// This is a temporary class to unify common methods and properties between +// FPBits and FPBits<long double>. +template <FPType fp_type> struct FPBitsCommon : private FPProperties<fp_type> { + using UP = FPProperties<fp_type>; + using typename UP::StorageType; + using UP::TOTAL_LEN; -private: - using FloatProperties<T>::QUIET_NAN_MASK; +protected: + using UP::EXP_SIG_MASK; + using UP::QUIET_NAN_MASK; public: - using FloatProperties<T>::SIGN_MASK; + using UP::EXP_BIAS; + using UP::EXP_LEN; + using UP::EXP_MASK; + using UP::EXP_MASK_SHIFT; + using UP::FP_MASK; + using UP::FRACTION_LEN; + using UP::FRACTION_MASK; + using UP::SIGN_MASK; // Reinterpreting bits as an integer value and interpreting the bits of an // integer value as a floating point value is used in tests. So, a convenient // type is provided for such reinterpretations. StorageType bits; + LIBC_INLINE constexpr FPBitsCommon() : bits(0) {} + LIBC_INLINE explicit constexpr FPBitsCommon(StorageType bits) : bits(bits) {} + LIBC_INLINE constexpr void set_mantissa(StorageType mantVal) { mantVal &= FRACTION_MASK; bits &= ~FRACTION_MASK; @@ -65,16 +62,89 @@ public: return bits & FRACTION_MASK; } - LIBC_INLINE constexpr void set_biased_exponent(StorageType expVal) { - expVal = (expVal << FRACTION_LEN) & EXP_MASK; + LIBC_INLINE constexpr void set_sign(bool signVal) { + if (get_sign() != signVal) + bits ^= SIGN_MASK; + } + + LIBC_INLINE constexpr bool get_sign() const { + return (bits & SIGN_MASK) != 0; + } + + LIBC_INLINE constexpr void set_biased_exponent(StorageType biased) { + // clear exponent bits bits &= ~EXP_MASK; - bits |= expVal; + // set exponent bits + bits |= (biased << EXP_MASK_SHIFT) & EXP_MASK; } LIBC_INLINE constexpr uint16_t get_biased_exponent() const { - return uint16_t((bits & EXP_MASK) >> FRACTION_LEN); + return uint16_t((bits & EXP_MASK) >> EXP_MASK_SHIFT); + } + + LIBC_INLINE constexpr int get_exponent() const { + return int(get_biased_exponent()) - EXP_BIAS; } + // If the number is subnormal, the exponent is treated as if it were the + // minimum exponent for a normal number. This is to keep continuity between + // the normal and subnormal ranges, but it causes problems for functions where + // values are calculated from the exponent, since just subtracting the bias + // will give a slightly incorrect result. Additionally, zero has an exponent + // of zero, and that should actually be treated as zero. + LIBC_INLINE constexpr int get_explicit_exponent() const { + const int biased_exp = int(get_biased_exponent()); + if (is_zero()) { + return 0; + } else if (biased_exp == 0) { + return 1 - EXP_BIAS; + } else { + return biased_exp - EXP_BIAS; + } + } + + LIBC_INLINE constexpr StorageType uintval() const { return bits & FP_MASK; } + + LIBC_INLINE constexpr bool is_zero() const { + return (bits & EXP_SIG_MASK) == 0; + } +}; + +} // namespace internal + +// A generic class to represent single precision, double precision, and quad +// precision IEEE 754 floating point formats. +// On most platforms, the 'float' type corresponds to single precision floating +// point numbers, the 'double' type corresponds to double precision floating +// point numers, and the 'long double' type corresponds to the quad precision +// floating numbers. On x86 platforms however, the 'long double' type maps to +// an x87 floating point format. This format is an IEEE 754 extension format. +// It is handled as an explicit specialization of this class. +template <typename T> +struct FPBits : public internal::FPBitsCommon<get_fp_type<T>()> { + static_assert(cpp::is_floating_point_v<T>, + "FPBits instantiated with invalid type."); + using UP = internal::FPBitsCommon<get_fp_type<T>()>; + using StorageType = typename UP::StorageType; + using UP::bits; + +private: + using UP::EXP_SIG_MASK; + using UP::QUIET_NAN_MASK; + +public: + using UP::EXP_BIAS; + using UP::EXP_LEN; + using UP::EXP_MASK; + using UP::EXP_MASK_SHIFT; + using UP::FRACTION_LEN; + using UP::FRACTION_MASK; + using UP::SIGN_MASK; + using UP::TOTAL_LEN; + + using UP::get_biased_exponent; + using UP::is_zero; + // The function return mantissa with the implicit bit set iff the current // value is a valid normal number. LIBC_INLINE constexpr StorageType get_explicit_mantissa() { @@ -84,19 +154,6 @@ public: (FRACTION_MASK & bits); } - LIBC_INLINE constexpr void set_sign(bool signVal) { - bits |= SIGN_MASK; - if (!signVal) - bits -= SIGN_MASK; - } - - LIBC_INLINE constexpr bool get_sign() const { - return (bits & SIGN_MASK) != 0; - } - - static_assert(sizeof(T) == sizeof(StorageType), - "Data type and integral representation have different sizes."); - static constexpr int MAX_BIASED_EXPONENT = (1 << EXP_LEN) - 1; static constexpr StorageType MIN_SUBNORMAL = StorageType(1); static constexpr StorageType MAX_SUBNORMAL = FRACTION_MASK; @@ -108,49 +165,21 @@ public: // type match. template <typename XType, cpp::enable_if_t<cpp::is_same_v<T, XType>, int> = 0> LIBC_INLINE constexpr explicit FPBits(XType x) - : bits(cpp::bit_cast<StorageType>(x)) {} + : UP(cpp::bit_cast<StorageType>(x)) {} template <typename XType, cpp::enable_if_t<cpp::is_same_v<XType, StorageType>, int> = 0> - LIBC_INLINE constexpr explicit FPBits(XType x) : bits(x) {} - - LIBC_INLINE constexpr FPBits() : bits(0) {} + LIBC_INLINE constexpr explicit FPBits(XType x) : UP(x) {} - LIBC_INLINE constexpr T get_val() const { return cpp::bit_cast<T>(bits); } + LIBC_INLINE constexpr FPBits() : UP() {} LIBC_INLINE constexpr void set_val(T value) { bits = cpp::bit_cast<StorageType>(value); } - LIBC_INLINE constexpr explicit operator T() const { return get_val(); } - - LIBC_INLINE constexpr StorageType uintval() const { return bits; } - - LIBC_INLINE constexpr int get_exponent() const { - return int(get_biased_exponent()) - EXP_BIAS; - } - - // If the number is subnormal, the exponent is treated as if it were the - // minimum exponent for a normal number. This is to keep continuity between - // the normal and subnormal ranges, but it causes problems for functions where - // values are calculated from the exponent, since just subtracting the bias - // will give a slightly incorrect result. Additionally, zero has an exponent - // of zero, and that should actually be treated as zero. - LIBC_INLINE constexpr int get_explicit_exponent() const { - const int biased_exp = int(get_biased_exponent()); - if (is_zero()) { - return 0; - } else if (biased_exp == 0) { - return 1 - EXP_BIAS; - } else { - return biased_exp - EXP_BIAS; - } - } + LIBC_INLINE constexpr T get_val() const { return cpp::bit_cast<T>(bits); } - LIBC_INLINE constexpr bool is_zero() const { - // Remove sign bit by shift - return (bits << 1) == 0; - } + LIBC_INLINE constexpr explicit operator T() const { return get_val(); } LIBC_INLINE constexpr bool is_inf() const { return (bits & EXP_SIG_MASK) == EXP_MASK; diff --git a/libc/src/__support/FPUtil/FloatProperties.h b/libc/src/__support/FPUtil/FloatProperties.h index bcf1f7cfabd3..6bf75b7167d3 100644 --- a/libc/src/__support/FPUtil/FloatProperties.h +++ b/libc/src/__support/FPUtil/FloatProperties.h @@ -111,7 +111,7 @@ public: (1U << (EXP_LEN - 1U)) - 1U; static_assert(EXP_BIAS > 0); -private: +protected: // The shift amount to get the *significand* part to the least significant // bit. Always `0` but kept for consistency. LIBC_INLINE_VAR static constexpr int SIG_MASK_SHIFT = 0; diff --git a/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h b/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h index b2b016adb661..1011e61f03fd 100644 --- a/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h +++ b/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h @@ -26,20 +26,27 @@ namespace LIBC_NAMESPACE { namespace fputil { -template <> struct FPBits<long double> : private FloatProperties<long double> { - using typename FloatProperties<long double>::StorageType; - using FloatProperties<long double>::TOTAL_LEN; - using FloatProperties<long double>::EXP_MASK; - using FloatProperties<long double>::EXP_BIAS; - using FloatProperties<long double>::EXP_LEN; - using FloatProperties<long double>::FRACTION_MASK; - using FloatProperties<long double>::FRACTION_LEN; +template <> +struct FPBits<long double> + : public internal::FPBitsCommon<FPType::X86_Binary80> { + using UP = internal::FPBitsCommon<FPType::X86_Binary80>; + using StorageType = typename UP::StorageType; + using UP::bits; private: - using FloatProperties<long double>::QUIET_NAN_MASK; + using UP::EXP_SIG_MASK; + using UP::QUIET_NAN_MASK; public: - using FloatProperties<long double>::SIGN_MASK; + using UP::EXP_BIAS; + using UP::EXP_LEN; + using UP::EXP_MASK; + using UP::EXP_MASK_SHIFT; + using UP::FP_MASK; + using UP::FRACTION_LEN; + using UP::FRACTION_MASK; + using UP::SIGN_MASK; + using UP::TOTAL_LEN; static constexpr int MAX_BIASED_EXPONENT = 0x7FFF; static constexpr StorageType MIN_SUBNORMAL = StorageType(1); @@ -51,18 +58,6 @@ public: (StorageType(MAX_BIASED_EXPONENT - 1) << (FRACTION_LEN + 1)) | (StorageType(1) << FRACTION_LEN) | MAX_SUBNORMAL; - StorageType bits; - - LIBC_INLINE constexpr void set_mantissa(StorageType mantVal) { - mantVal &= FRACTION_MASK; - bits &= ~FRACTION_MASK; - bits |= mantVal; - } - - LIBC_INLINE constexpr StorageType get_mantissa() const { - return bits & FRACTION_MASK; - } - LIBC_INLINE constexpr StorageType get_explicit_mantissa() const { // The x86 80 bit float represents the leading digit of the mantissa // explicitly. This is the mask for that bit. @@ -70,16 +65,6 @@ public: return bits & (FRACTION_MASK | EXPLICIT_BIT_MASK); } - LIBC_INLINE constexpr void set_biased_exponent(StorageType expVal) { - expVal = (expVal << (TOTAL_LEN - 1 - EXP_LEN)) & EXP_MASK; - bits &= ~EXP_MASK; - bits |= expVal; - } - - LIBC_INLINE constexpr uint16_t get_biased_exponent() const { - return uint16_t((bits & EXP_MASK) >> (TOTAL_LEN - 1 - EXP_LEN)); - } - LIBC_INLINE constexpr void set_implicit_bit(bool implicitVal) { bits &= ~(StorageType(1) << FRACTION_LEN); bits |= (StorageType(implicitVal) << FRACTION_LEN); @@ -89,22 +74,12 @@ public: return bool((bits & (StorageType(1) << FRACTION_LEN)) >> FRACTION_LEN); } - LIBC_INLINE constexpr void set_sign(bool signVal) { - bits &= ~SIGN_MASK; - StorageType sign1 = StorageType(signVal) << (TOTAL_LEN - 1); - bits |= sign1; - } - - LIBC_INLINE constexpr bool get_sign() const { - return bool((bits & SIGN_MASK) >> (TOTAL_LEN - 1)); - } - - LIBC_INLINE constexpr FPBits() : bits(0) {} + LIBC_INLINE constexpr FPBits() : UP() {} template <typename XType, cpp::enable_if_t<cpp::is_same_v<long double, XType>, int> = 0> LIBC_INLINE constexpr explicit FPBits(XType x) - : bits(cpp::bit_cast<StorageType>(x)) { + : UP(cpp::bit_cast<StorageType>(x)) { // bits starts uninitialized, and setting it to a long double only // overwrites the first 80 bits. This clears those upper bits. bits = bits & ((StorageType(1) << 80) - 1); @@ -112,47 +87,16 @@ public: template <typename XType, cpp::enable_if_t<cpp::is_same_v<XType, StorageType>, int> = 0> - LIBC_INLINE constexpr explicit FPBits(XType x) : bits(x) {} + LIBC_INLINE constexpr explicit FPBits(XType x) : UP(x) {} LIBC_INLINE constexpr operator long double() { return cpp::bit_cast<long double>(bits); } - LIBC_INLINE constexpr StorageType uintval() { - // We zero the padding bits as they can contain garbage. - return bits & FP_MASK; - } - LIBC_INLINE constexpr long double get_val() const { return cpp::bit_cast<long double>(bits); } - LIBC_INLINE constexpr int get_exponent() const { - return int(get_biased_exponent()) - EXP_BIAS; - } - - // If the number is subnormal, the exponent is treated as if it were the - // minimum exponent for a normal number. This is to keep continuity between - // the normal and subnormal ranges, but it causes problems for functions where - // values are calculated from the exponent, since just subtracting the bias - // will give a slightly incorrect result. Additionally, zero has an exponent - // of zero, and that should actually be treated as zero. - LIBC_INLINE constexpr int get_explicit_exponent() const { - const int biased_exp = int(get_biased_exponent()); - if (is_zero()) { - return 0; - } else if (biased_exp == 0) { - return 1 - EXP_BIAS; - } else { - return biased_exp - EXP_BIAS; - } - } - - LIBC_INLINE constexpr bool is_zero() const { - return get_biased_exponent() == 0 && get_mantissa() == 0 && - get_implicit_bit() == 0; - } - LIBC_INLINE constexpr bool is_inf() const { return get_biased_exponent() == MAX_BIASED_EXPONENT && get_mantissa() == 0 && get_implicit_bit() == 1; diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h index cfd495c58618..79e05940f027 100644 --- a/libc/src/__support/UInt.h +++ b/libc/src/__support/UInt.h @@ -30,7 +30,7 @@ template <size_t Bits, bool Signed> struct BigInt { static_assert(Bits > 0 && Bits % 64 == 0, "Number of bits in BigInt should be a multiple of 64."); LIBC_INLINE_VAR static constexpr size_t WORDCOUNT = Bits / 64; - uint64_t val[WORDCOUNT]{}; + cpp::array<uint64_t, WORDCOUNT> val{}; LIBC_INLINE_VAR static constexpr uint64_t MASK32 = 0xFFFFFFFFu; @@ -954,6 +954,48 @@ struct make_signed<UInt<Bits>> : type_identity<Int<Bits>> { "Number of bits in Int should be a multiple of 64."); }; +namespace internal { +template <typename T> struct is_custom_uint : cpp::false_type {}; +template <size_t Bits> struct is_custom_uint<UInt<Bits>> : cpp::true_type {}; +} // namespace internal + +// bit_cast to UInt +// Note: The standard scheme for SFINAE selection is to have exactly one +// function instanciation valid at a time. This is usually done by having a +// predicate in one function and the negated predicate in the other one. +// e.g. +// template<typename = cpp::enable_if_t< is_custom_uint<To>::value == true> ... +// template<typename = cpp::enable_if_t< is_custom_uint<To>::value == false> ... +// +// Unfortunately this would make the default 'cpp::bit_cast' aware of +// 'is_custom_uint' (or any other customization). To prevent exposing all +// customizations in the original function, we create a different function with +// four 'typename's instead of three - otherwise it would be considered as a +// redeclaration of the same function leading to "error: template parameter +// redefines default argument". +template <typename To, typename From, + typename = cpp::enable_if_t<sizeof(To) == sizeof(From) && + cpp::is_trivially_copyable<To>::value && + cpp::is_trivially_copyable<From>::value>, + typename = cpp::enable_if_t<internal::is_custom_uint<To>::value>> +LIBC_INLINE constexpr To bit_cast(const From &from) { + To out; + using Storage = decltype(out.val); + out.val = cpp::bit_cast<Storage>(from); + return out; +} + +// bit_cast from UInt +template < + typename To, size_t Bits, + typename = cpp::enable_if_t<sizeof(To) == sizeof(UInt<Bits>) && + cpp::is_trivially_constructible<To>::value && + cpp::is_trivially_copyable<To>::value && + cpp::is_trivially_copyable<UInt<Bits>>::value>> +LIBC_INLINE constexpr To bit_cast(const UInt<Bits> &from) { + return cpp::bit_cast<To>(from.val); +} + } // namespace LIBC_NAMESPACE::cpp #endif // LLVM_LIBC_SRC___SUPPORT_UINT_H diff --git a/libc/test/src/__support/uint_test.cpp b/libc/test/src/__support/uint_test.cpp index 971bac55bd9d..0ad72c35645c 100644 --- a/libc/test/src/__support/uint_test.cpp +++ b/libc/test/src/__support/uint_test.cpp @@ -10,19 +10,62 @@ #include "src/__support/UInt.h" #include "test/UnitTest/Test.h" +#include <math.h> // HUGE_VALF, HUGE_VALF -// We want to test LIBC_NAMESPACE::cpp::UInt<128> explicitly. So, for +namespace LIBC_NAMESPACE { + +using LL_UInt64 = cpp::UInt<64>; +// We want to test cpp::UInt<128> explicitly. So, for // convenience, we use a sugar which does not conflict with the UInt128 type // which can resolve to __uint128_t if the platform has it. -using LL_UInt128 = LIBC_NAMESPACE::cpp::UInt<128>; -using LL_UInt192 = LIBC_NAMESPACE::cpp::UInt<192>; -using LL_UInt256 = LIBC_NAMESPACE::cpp::UInt<256>; -using LL_UInt320 = LIBC_NAMESPACE::cpp::UInt<320>; -using LL_UInt512 = LIBC_NAMESPACE::cpp::UInt<512>; -using LL_UInt1024 = LIBC_NAMESPACE::cpp::UInt<1024>; +using LL_UInt128 = cpp::UInt<128>; +using LL_UInt192 = cpp::UInt<192>; +using LL_UInt256 = cpp::UInt<256>; +using LL_UInt320 = cpp::UInt<320>; +using LL_UInt512 = cpp::UInt<512>; +using LL_UInt1024 = cpp::UInt<1024>; + +using LL_Int128 = cpp::Int<128>; +using LL_Int192 = cpp::Int<192>; + +TEST(LlvmLibcUIntClassTest, BitCastToFromDouble) { + static_assert(cpp::is_trivially_copyable<LL_UInt64>::value); + static_assert(sizeof(LL_UInt64) == sizeof(double)); + const double inf = HUGE_VAL; + const double max = DBL_MAX; + const double array[] = {0.0, 0.1, 1.0, max, inf}; + for (double value : array) { + LL_UInt64 back = cpp::bit_cast<LL_UInt64>(value); + double forth = cpp::bit_cast<double>(back); + EXPECT_TRUE(value == forth); + } +} -using LL_Int128 = LIBC_NAMESPACE::cpp::Int<128>; -using LL_Int192 = LIBC_NAMESPACE::cpp::Int<192>; +#ifdef __SIZEOF_INT128__ +TEST(LlvmLibcUIntClassTest, BitCastToFromNativeUint128) { + static_assert(cpp::is_trivially_copyable<LL_UInt128>::value); + static_assert(sizeof(LL_UInt128) == sizeof(__uint128_t)); + const __uint128_t array[] = {0, 1, ~__uint128_t(0)}; + for (__uint128_t value : array) { + LL_UInt128 back = cpp::bit_cast<LL_UInt128>(value); + __uint128_t forth = cpp::bit_cast<__uint128_t>(back); + EXPECT_TRUE(value == forth); + } +} +#endif + +#ifdef LIBC_COMPILER_HAS_FLOAT128 +TEST(LlvmLibcUIntClassTest, BitCastToFromNativeFloat128) { + static_assert(cpp::is_trivially_copyable<LL_UInt128>::value); + static_assert(sizeof(LL_UInt128) == sizeof(float128)); + const float128 array[] = {0, 0.1, 1}; + for (float128 value : array) { + LL_UInt128 back = cpp::bit_cast<LL_UInt128>(value); + float128 forth = cpp::bit_cast<float128>(back); + EXPECT_TRUE(value == forth); + } +} +#endif TEST(LlvmLibcUIntClassTest, BasicInit) { LL_UInt128 half_val(12345); @@ -634,3 +677,5 @@ TEST(LlvmLibcUIntClassTest, ConstructorFromUInt128Tests) { } #endif // __SIZEOF_INT128__ + +} // namespace LIBC_NAMESPACE diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp index e1275ce711fc..00651df48b62 100644 --- a/lldb/source/Interpreter/CommandInterpreter.cpp +++ b/lldb/source/Interpreter/CommandInterpreter.cpp @@ -1160,7 +1160,11 @@ Status CommandInterpreter::AddUserCommand(llvm::StringRef name, if (UserCommandExists(name)) { if (!can_replace) { - result.SetErrorString("user command exists and force replace not set"); + result.SetErrorStringWithFormatv( + "user command \"{0}\" already exists and force replace was not set " + "by --overwrite or 'settings set interpreter.require-overwrite " + "false'", + name); return result; } if (cmd_sp->IsMultiwordObject()) { diff --git a/lldb/source/Plugins/Process/Windows/Common/TargetThreadWindows.cpp b/lldb/source/Plugins/Process/Windows/Common/TargetThreadWindows.cpp index 37dc8f6d6d14..ad67e764fe10 100644 --- a/lldb/source/Plugins/Process/Windows/Common/TargetThreadWindows.cpp +++ b/lldb/source/Plugins/Process/Windows/Common/TargetThreadWindows.cpp @@ -7,18 +7,14 @@ //===----------------------------------------------------------------------===// #include "lldb/Host/HostInfo.h" -#include "lldb/Host/HostNativeThreadBase.h" -#include "lldb/Host/windows/HostThreadWindows.h" -#include "lldb/Host/windows/windows.h" -#include "lldb/Target/RegisterContext.h" #include "lldb/Target/Unwind.h" #include "lldb/Utility/LLDBLog.h" #include "lldb/Utility/Log.h" -#include "lldb/Utility/State.h" #include "ProcessWindows.h" -#include "ProcessWindowsLog.h" #include "TargetThreadWindows.h" +#include "lldb/Host/windows/HostThreadWindows.h" +#include <llvm/Support/ConvertUTF.h> #if defined(__x86_64__) || defined(_M_AMD64) #include "x64/RegisterContextWindows_x64.h" @@ -33,6 +29,9 @@ using namespace lldb; using namespace lldb_private; +using GetThreadDescriptionFunctionPtr = HRESULT +WINAPI (*)(HANDLE hThread, PWSTR *ppszThreadDescription); + TargetThreadWindows::TargetThreadWindows(ProcessWindows &process, const HostThread &thread) : Thread(process, thread.GetNativeThread().GetThreadId()), @@ -175,3 +174,29 @@ Status TargetThreadWindows::DoResume() { return Status(); } + +const char *TargetThreadWindows::GetName() { + Log *log = GetLog(LLDBLog::Thread); + static GetThreadDescriptionFunctionPtr GetThreadDescription = []() { + HMODULE hModule = ::LoadLibraryW(L"Kernel32.dll"); + return hModule ? reinterpret_cast<GetThreadDescriptionFunctionPtr>( + ::GetProcAddress(hModule, "GetThreadDescription")) + : nullptr; + }(); + LLDB_LOGF(log, "GetProcAddress: %p", + reinterpret_cast<void *>(GetThreadDescription)); + if (!GetThreadDescription) + return m_name.c_str(); + PWSTR pszThreadName; + if (SUCCEEDED(GetThreadDescription( + m_host_thread.GetNativeThread().GetSystemHandle(), &pszThreadName))) { + LLDB_LOGF(log, "GetThreadDescription: %ls", pszThreadName); + llvm::convertUTF16ToUTF8String( + llvm::ArrayRef(reinterpret_cast<char *>(pszThreadName), + wcslen(pszThreadName) * sizeof(wchar_t)), + m_name); + ::LocalFree(pszThreadName); + } + + return m_name.c_str(); +} diff --git a/lldb/source/Plugins/Process/Windows/Common/TargetThreadWindows.h b/lldb/source/Plugins/Process/Windows/Common/TargetThreadWindows.h index 2845847738f6..07e1db464ad5 100644 --- a/lldb/source/Plugins/Process/Windows/Common/TargetThreadWindows.h +++ b/lldb/source/Plugins/Process/Windows/Common/TargetThreadWindows.h @@ -34,6 +34,7 @@ public: lldb::RegisterContextSP CreateRegisterContextForFrame(StackFrame *frame) override; bool CalculateStopInfo() override; + const char *GetName() override; Status DoResume(); @@ -42,6 +43,7 @@ public: private: lldb::RegisterContextSP m_thread_reg_ctx_sp; HostThread m_host_thread; + std::string m_name; }; } // namespace lldb_private diff --git a/lldb/test/API/commands/command/script/TestCommandScript.py b/lldb/test/API/commands/command/script/TestCommandScript.py index cac11834fa73..850552032902 100644 --- a/lldb/test/API/commands/command/script/TestCommandScript.py +++ b/lldb/test/API/commands/command/script/TestCommandScript.py @@ -161,6 +161,19 @@ class CmdPythonTestCase(TestBase): ) self.expect("my_command", substrs=["a.out"]) + # Test that without --overwrite we are not allowed to redefine the command. + self.expect( + "command script add my_command --class welcome.TargetnameCommand", + substrs=[ + ( + 'user command "my_command" already exists and force replace was' + " not set by --overwrite or 'settings set" + " interpreter.require-overwrite false'" + ), + ], + error=True, + ) + self.runCmd("command script clear") self.expect( diff --git a/lldb/test/API/python_api/global_module_cache/TestGlobalModuleCache.py b/lldb/test/API/python_api/global_module_cache/TestGlobalModuleCache.py index 6bb22c46efb4..5b6e9e8a588a 100644 --- a/lldb/test/API/python_api/global_module_cache/TestGlobalModuleCache.py +++ b/lldb/test/API/python_api/global_module_cache/TestGlobalModuleCache.py @@ -26,6 +26,13 @@ class GlobalModuleCacheTestCase(TestBase): # a previous build, so sleep a bit here to ensure that the touch is later. time.sleep(2) try: + # Make sure dst is writeable before trying to write to it. + subprocess.run( + ["chmod", "777", dst], + stdin=None, + capture_output=False, + encoding="utf-8", + ) shutil.copy(src, dst) except: self.fail(f"Could not copy {src} to {dst}") diff --git a/lldb/unittests/Thread/CMakeLists.txt b/lldb/unittests/Thread/CMakeLists.txt index d6e365adac5d..8fc44da6f5b3 100644 --- a/lldb/unittests/Thread/CMakeLists.txt +++ b/lldb/unittests/Thread/CMakeLists.txt @@ -1,3 +1,8 @@ +if (CMAKE_SYSTEM_NAME MATCHES "Windows") + list(APPEND LLDB_WINDOWS_LIBS lldbPluginPlatformWindows) + list(APPEND LLDB_WINDOWS_LIBS lldbPluginProcessWindowsCommon) +endif() + add_lldb_unittest(ThreadTests ThreadTest.cpp @@ -11,5 +16,6 @@ add_lldb_unittest(ThreadTests lldbInterpreter lldbBreakpoint lldbPluginPlatformLinux + ${LLDB_WINDOWS_LIBS} ) diff --git a/lldb/unittests/Thread/ThreadTest.cpp b/lldb/unittests/Thread/ThreadTest.cpp index bd8cdce99f17..4c660e9815c3 100644 --- a/lldb/unittests/Thread/ThreadTest.cpp +++ b/lldb/unittests/Thread/ThreadTest.cpp @@ -8,9 +8,20 @@ #include "lldb/Target/Thread.h" #include "Plugins/Platform/Linux/PlatformLinux.h" +#include <thread> +#ifdef _WIN32 +#include "lldb/Host/windows/HostThreadWindows.h" +#include "lldb/Host/windows/windows.h" + +#include "Plugins/Platform/Windows/PlatformWindows.h" +#include "Plugins/Process/Windows/Common/LocalDebugDelegate.h" +#include "Plugins/Process/Windows/Common/ProcessWindows.h" +#include "Plugins/Process/Windows/Common/TargetThreadWindows.h" +#endif #include "lldb/Core/Debugger.h" #include "lldb/Host/FileSystem.h" #include "lldb/Host/HostInfo.h" +#include "lldb/Host/HostThread.h" #include "lldb/Target/Process.h" #include "lldb/Target/StopInfo.h" #include "lldb/Utility/ArchSpec.h" @@ -21,14 +32,33 @@ using namespace lldb_private::repro; using namespace lldb; namespace { + +#ifdef _WIN32 +using SetThreadDescriptionFunctionPtr = HRESULT +WINAPI (*)(HANDLE hThread, PCWSTR lpThreadDescription); + +static SetThreadDescriptionFunctionPtr SetThreadName; +#endif + class ThreadTest : public ::testing::Test { public: void SetUp() override { FileSystem::Initialize(); HostInfo::Initialize(); +#ifdef _WIN32 + HMODULE hModule = ::LoadLibraryW(L"Kernel32.dll"); + if (hModule) { + SetThreadName = reinterpret_cast<SetThreadDescriptionFunctionPtr>( + ::GetProcAddress(hModule, "SetThreadDescription")); + } + PlatformWindows::Initialize(); +#endif platform_linux::PlatformLinux::Initialize(); } void TearDown() override { +#ifdef _WIN32 + PlatformWindows::Terminate(); +#endif platform_linux::PlatformLinux::Terminate(); HostInfo::Terminate(); FileSystem::Terminate(); @@ -88,6 +118,47 @@ TargetSP CreateTarget(DebuggerSP &debugger_sp, ArchSpec &arch) { return target_sp; } +#ifdef _WIN32 +std::shared_ptr<TargetThreadWindows> +CreateWindowsThread(const ProcessWindowsSP &process_sp, std::thread &t) { + HostThread host_thread((lldb::thread_t)t.native_handle()); + ThreadSP thread_sp = + std::make_shared<TargetThreadWindows>(*process_sp.get(), host_thread); + return std::static_pointer_cast<TargetThreadWindows>(thread_sp); +} + +TEST_F(ThreadTest, GetThreadDescription) { + if (!SetThreadName) + return; + + ArchSpec arch(HostInfo::GetArchitecture()); + Platform::SetHostPlatform(PlatformWindows::CreateInstance(true, &arch)); + + DebuggerSP debugger_sp = Debugger::CreateInstance(); + ASSERT_TRUE(debugger_sp); + + TargetSP target_sp = CreateTarget(debugger_sp, arch); + ASSERT_TRUE(target_sp); + + ListenerSP listener_sp(Listener::MakeListener("dummy")); + auto process_sp = std::static_pointer_cast<ProcessWindows>( + ProcessWindows::CreateInstance(target_sp, listener_sp, nullptr, false)); + ASSERT_TRUE(process_sp); + + std::thread t([]() {}); + auto thread_sp = CreateWindowsThread(process_sp, t); + DWORD tid = thread_sp->GetHostThread().GetNativeThread().GetThreadId(); + HANDLE hThread = ::OpenThread(THREAD_SET_LIMITED_INFORMATION, FALSE, tid); + ASSERT_TRUE(hThread); + + SetThreadName(hThread, L"thread name"); + ::CloseHandle(hThread); + ASSERT_STREQ(thread_sp->GetName(), "thread name"); + + t.join(); +} +#endif + TEST_F(ThreadTest, SetStopInfo) { ArchSpec arch("powerpc64-pc-linux"); diff --git a/lldb/unittests/Utility/ChecksumTest.cpp b/lldb/unittests/Utility/ChecksumTest.cpp index 7537d30b5ff5..a81aba2ee98c 100644 --- a/lldb/unittests/Utility/ChecksumTest.cpp +++ b/lldb/unittests/Utility/ChecksumTest.cpp @@ -12,14 +12,14 @@ using namespace lldb_private; -static llvm::MD5::MD5Result hash1 = {0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15}; +static llvm::MD5::MD5Result hash1 = { + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}}; -static llvm::MD5::MD5Result hash2 = {0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15}; +static llvm::MD5::MD5Result hash2 = { + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}}; -static llvm::MD5::MD5Result hash3 = {8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7}; +static llvm::MD5::MD5Result hash3 = { + {8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7}}; TEST(ChecksumTest, TestConstructor) { Checksum checksum1; diff --git a/lldb/unittests/Utility/FileSpecTest.cpp b/lldb/unittests/Utility/FileSpecTest.cpp index 565395a495be..9faad10e4730 100644 --- a/lldb/unittests/Utility/FileSpecTest.cpp +++ b/lldb/unittests/Utility/FileSpecTest.cpp @@ -536,7 +536,8 @@ TEST(FileSpecTest, TestGetComponents) { } TEST(FileSpecTest, TestChecksum) { - Checksum checksum({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}); + Checksum checksum(llvm::MD5::MD5Result{ + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}}); FileSpec file_spec("/foo/bar", FileSpec::Style::posix, checksum); EXPECT_TRUE(static_cast<bool>(file_spec.GetChecksum())); EXPECT_EQ(file_spec.GetChecksum(), checksum); diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h index 2ffd4d4b7143..daf1d8e2079f 100644 --- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h +++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h @@ -156,6 +156,10 @@ public: /// FDecl is assumed to have a parent Module when using this function. bool getLibFunc(const Function &FDecl, LibFunc &F) const; + /// Searches for a function name using an Instruction \p Opcode. + /// Currently, only the frem instruction is supported. + bool getLibFunc(unsigned int Opcode, Type *Ty, LibFunc &F) const; + /// Forces a function to be marked as unavailable. void setUnavailable(LibFunc F) { setState(F, Unavailable); @@ -360,6 +364,12 @@ public: getLibFunc(*(CB.getCalledFunction()), F); } + /// Searches for a function name using an Instruction \p Opcode. + /// Currently, only the frem instruction is supported. + bool getLibFunc(unsigned int Opcode, Type *Ty, LibFunc &F) const { + return Impl->getLibFunc(Opcode, Ty, F); + } + /// Disables all builtins. /// /// This can be used for options like -fno-builtin. diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index f5114fa40c70..735be3680aea 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -560,6 +560,10 @@ public: // (set to UINT_MAX to disable). This does not apply in cases where the // loop is being fully unrolled. unsigned MaxCount; + /// Set the maximum upper bound of trip count. Allowing the MaxUpperBound + /// to be overrided by a target gives more flexiblity on certain cases. + /// By default, MaxUpperBound uses UnrollMaxUpperBound which value is 8. + unsigned MaxUpperBound; /// Set the maximum unrolling factor for full unrolling. Like MaxCount, but /// applies even if full unrolling is selected. This allows a target to fall /// back to Partial unrolling if full unrolling is above FullUnrollMaxCount. diff --git a/llvm/include/llvm/CodeGen/AccelTable.h b/llvm/include/llvm/CodeGen/AccelTable.h index af874aa5e91a..6eb09f32f9f9 100644 --- a/llvm/include/llvm/CodeGen/AccelTable.h +++ b/llvm/include/llvm/CodeGen/AccelTable.h @@ -270,16 +270,14 @@ public: #endif uint64_t getDieOffset() const { - assert(std::holds_alternative<uint64_t>(OffsetVal) && - "Accessing DIE Offset before normalizing."); + assert(isNormalized() && "Accessing DIE Offset before normalizing."); return std::get<uint64_t>(OffsetVal); } unsigned getDieTag() const { return DieTag; } unsigned getUnitID() const { return UnitID; } bool isTU() const { return IsTU; } void normalizeDIEToOffset() { - assert(std::holds_alternative<const DIE *>(OffsetVal) && - "Accessing offset after normalizing."); + assert(!isNormalized() && "Accessing offset after normalizing."); OffsetVal = std::get<const DIE *>(OffsetVal)->getOffset(); } bool isNormalized() const { @@ -309,7 +307,7 @@ class DWARF5AccelTable : public AccelTable<DWARF5AccelTableData> { public: struct UnitIndexAndEncoding { unsigned Index; - DWARF5AccelTableData::AttributeEncoding Endoding; + DWARF5AccelTableData::AttributeEncoding Encoding; }; /// Returns type units that were constructed. const TUVectorTy &getTypeUnitsSymbols() { return TUSymbolsOrHashes; } diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index 51bd9b63c127..cb48f54b13a6 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -2353,14 +2353,14 @@ def int_amdgcn_s_get_waveid_in_workgroup : Intrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrHasSideEffects, IntrWillReturn, IntrNoCallback, IntrNoFree]>; -class AMDGPUGlobalAtomicRtn<LLVMType vt> : Intrinsic < +class AMDGPUAtomicRtn<LLVMType vt> : Intrinsic < [vt], [llvm_anyptr_ty, // vaddr vt], // vdata(VGPR) [IntrArgMemOnly, IntrWillReturn, NoCapture<ArgIndex<0>>, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>; -def int_amdgcn_global_atomic_csub : AMDGPUGlobalAtomicRtn<llvm_i32_ty>; +def int_amdgcn_global_atomic_csub : AMDGPUAtomicRtn<llvm_i32_ty>; // uint4 llvm.amdgcn.image.bvh.intersect.ray <node_ptr>, <ray_extent>, <ray_origin>, // <ray_dir>, <ray_inv_dir>, <texture_descr> @@ -2486,10 +2486,10 @@ def int_amdgcn_permlanex16_var : ClangBuiltin<"__builtin_amdgcn_permlanex16_var" [IntrNoMem, IntrConvergent, IntrWillReturn, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, IntrNoCallback, IntrNoFree]>; -def int_amdgcn_flat_atomic_fmin_num : AMDGPUGlobalAtomicRtn<llvm_anyfloat_ty>; -def int_amdgcn_flat_atomic_fmax_num : AMDGPUGlobalAtomicRtn<llvm_anyfloat_ty>; -def int_amdgcn_global_atomic_fmin_num : AMDGPUGlobalAtomicRtn<llvm_anyfloat_ty>; -def int_amdgcn_global_atomic_fmax_num : AMDGPUGlobalAtomicRtn<llvm_anyfloat_ty>; +def int_amdgcn_flat_atomic_fmin_num : AMDGPUAtomicRtn<llvm_anyfloat_ty>; +def int_amdgcn_flat_atomic_fmax_num : AMDGPUAtomicRtn<llvm_anyfloat_ty>; +def int_amdgcn_global_atomic_fmin_num : AMDGPUAtomicRtn<llvm_anyfloat_ty>; +def int_amdgcn_global_atomic_fmax_num : AMDGPUAtomicRtn<llvm_anyfloat_ty>; //===----------------------------------------------------------------------===// // Deep learning intrinsics. @@ -2692,7 +2692,7 @@ def int_amdgcn_udot8 : // gfx908 intrinsics // ===----------------------------------------------------------------------===// -def int_amdgcn_global_atomic_fadd : AMDGPUGlobalAtomicRtn<llvm_anyfloat_ty>; +def int_amdgcn_global_atomic_fadd : AMDGPUAtomicRtn<llvm_anyfloat_ty>; // llvm.amdgcn.mfma.*.* vdst, srcA, srcB, srcC, cbsz, abid, blgp class AMDGPUMfmaIntrinsic<LLVMType DestTy, LLVMType SrcABTy> : @@ -2728,11 +2728,11 @@ def int_amdgcn_mfma_f32_16x16x8bf16 : AMDGPUMfmaIntrinsic<llvm_v4f32_ty, llvm_v // gfx90a intrinsics // ===----------------------------------------------------------------------===// -def int_amdgcn_global_atomic_fmin : AMDGPUGlobalAtomicRtn<llvm_anyfloat_ty>; -def int_amdgcn_global_atomic_fmax : AMDGPUGlobalAtomicRtn<llvm_anyfloat_ty>; -def int_amdgcn_flat_atomic_fadd : AMDGPUGlobalAtomicRtn<llvm_anyfloat_ty>; -def int_amdgcn_flat_atomic_fmin : AMDGPUGlobalAtomicRtn<llvm_anyfloat_ty>; -def int_amdgcn_flat_atomic_fmax : AMDGPUGlobalAtomicRtn<llvm_anyfloat_ty>; +def int_amdgcn_global_atomic_fmin : AMDGPUAtomicRtn<llvm_anyfloat_ty>; +def int_amdgcn_global_atomic_fmax : AMDGPUAtomicRtn<llvm_anyfloat_ty>; +def int_amdgcn_flat_atomic_fadd : AMDGPUAtomicRtn<llvm_anyfloat_ty>; +def int_amdgcn_flat_atomic_fmin : AMDGPUAtomicRtn<llvm_anyfloat_ty>; +def int_amdgcn_flat_atomic_fmax : AMDGPUAtomicRtn<llvm_anyfloat_ty>; def int_amdgcn_mfma_f32_32x32x4bf16_1k : AMDGPUMfmaIntrinsic<llvm_v32f32_ty, llvm_v4i16_ty>; def int_amdgcn_mfma_f32_16x16x4bf16_1k : AMDGPUMfmaIntrinsic<llvm_v16f32_ty, llvm_v4i16_ty>; @@ -2751,8 +2751,8 @@ def int_amdgcn_mfma_f64_4x4x4f64 : AMDGPUMfmaIntrinsic<llvm_double_ty, ll // ===----------------------------------------------------------------------===// // bf16 atomics use v2i16 argument since there is no bf16 data type in the llvm. -def int_amdgcn_global_atomic_fadd_v2bf16 : AMDGPUGlobalAtomicRtn<llvm_v2i16_ty>; -def int_amdgcn_flat_atomic_fadd_v2bf16 : AMDGPUGlobalAtomicRtn<llvm_v2i16_ty>; +def int_amdgcn_global_atomic_fadd_v2bf16 : AMDGPUAtomicRtn<llvm_v2i16_ty>; +def int_amdgcn_flat_atomic_fadd_v2bf16 : AMDGPUAtomicRtn<llvm_v2i16_ty>; def int_amdgcn_ds_fadd_v2bf16 : DefaultAttrsIntrinsic< [llvm_v2i16_ty], [LLVMQualPointerType<3>, llvm_v2i16_ty], diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index 096d1688af3f..48afdb867ba6 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -1656,6 +1656,19 @@ template <typename Op_t> struct PtrToIntSameSize_match { } }; +template <typename Op_t> struct NNegZExt_match { + Op_t Op; + + NNegZExt_match(const Op_t &OpMatch) : Op(OpMatch) {} + + template <typename OpTy> bool match(OpTy *V) { + if (auto *I = dyn_cast<Instruction>(V)) + return I->getOpcode() == Instruction::ZExt && I->hasNonNeg() && + Op.match(I->getOperand(0)); + return false; + } +}; + /// Matches BitCast. template <typename OpTy> inline CastOperator_match<OpTy, Instruction::BitCast> @@ -1708,6 +1721,11 @@ inline CastInst_match<OpTy, Instruction::ZExt> m_ZExt(const OpTy &Op) { } template <typename OpTy> +inline NNegZExt_match<OpTy> m_NNegZExt(const OpTy &Op) { + return NNegZExt_match<OpTy>(Op); +} + +template <typename OpTy> inline match_combine_or<CastInst_match<OpTy, Instruction::ZExt>, OpTy> m_ZExtOrSelf(const OpTy &Op) { return m_CombineOr(m_ZExt(Op), Op); @@ -1719,6 +1737,14 @@ m_SExtOrSelf(const OpTy &Op) { return m_CombineOr(m_SExt(Op), Op); } +/// Match either "sext" or "zext nneg". +template <typename OpTy> +inline match_combine_or<CastInst_match<OpTy, Instruction::SExt>, + NNegZExt_match<OpTy>> +m_SExtLike(const OpTy &Op) { + return m_CombineOr(m_SExt(Op), m_NNegZExt(Op)); +} + template <typename OpTy> inline match_combine_or<CastInst_match<OpTy, Instruction::ZExt>, CastInst_match<OpTy, Instruction::SExt>> diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h index 493689f6a61e..2757b8cd54a6 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -447,7 +447,7 @@ public: bool isConditionIndependencePairCovered(unsigned Condition) const { auto It = PosToID.find(Condition); if (It != PosToID.end()) - return (IndependencePairs.find(It->second) != IndependencePairs.end()); + return IndependencePairs.contains(It->second); llvm_unreachable("Condition ID without an Ordinal mapping"); } diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h index f0b35790133f..6c7410a8b8f7 100644 --- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h +++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h @@ -174,6 +174,7 @@ enum ArchExtKind : unsigned { AEK_SMEF8F32 = 70, // FEAT_SME_F8F32 AEK_SMEFA64 = 71, // FEAT_SME_FA64 AEK_CPA = 72, // FEAT_CPA + AEK_PAUTHLR = 73, // FEAT_PAuth_LR AEK_NUM_EXTENSIONS }; using ExtensionBitset = Bitset<AEK_NUM_EXTENSIONS>; @@ -297,6 +298,7 @@ inline constexpr ExtensionInfo Extensions[] = { {"sme-f8f32", AArch64::AEK_SMEF8F32, "+sme-f8f32", "-sme-f8f32", FEAT_INIT, "+sme2,+fp8", 0}, {"sme-fa64", AArch64::AEK_SMEFA64, "+sme-fa64", "-sme-fa64", FEAT_INIT, "", 0}, {"cpa", AArch64::AEK_CPA, "+cpa", "-cpa", FEAT_INIT, "", 0}, + {"pauth-lr", AArch64::AEK_PAUTHLR, "+pauth-lr", "-pauth-lr", FEAT_INIT, "", 0}, // Special cases {"none", AArch64::AEK_NONE, {}, {}, FEAT_INIT, "", ExtensionInfo::MaxFMVPriority}, }; diff --git a/llvm/include/llvm/TargetParser/ARMTargetParserCommon.h b/llvm/include/llvm/TargetParser/ARMTargetParserCommon.h index e3d9ffc1d4db..1e4187c6fb11 100644 --- a/llvm/include/llvm/TargetParser/ARMTargetParserCommon.h +++ b/llvm/include/llvm/TargetParser/ARMTargetParserCommon.h @@ -41,6 +41,7 @@ struct ParsedBranchProtection { StringRef Scope; StringRef Key; bool BranchTargetEnforcement; + bool BranchProtectionPAuthLR; }; bool parseBranchProtection(StringRef Spec, ParsedBranchProtection &PBP, diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp index 20959cf6948f..bbb7c86d2185 100644 --- a/llvm/lib/Analysis/TargetLibraryInfo.cpp +++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp @@ -1149,6 +1149,16 @@ bool TargetLibraryInfoImpl::getLibFunc(const Function &FDecl, return isValidProtoForLibFunc(*FDecl.getFunctionType(), F, *M); } +bool TargetLibraryInfoImpl::getLibFunc(unsigned int Opcode, Type *Ty, + LibFunc &F) const { + // Must be a frem instruction with float or double arguments. + if (Opcode != Instruction::FRem || (!Ty->isDoubleTy() && !Ty->isFloatTy())) + return false; + + F = Ty->isDoubleTy() ? LibFunc_fmod : LibFunc_fmodf; + return true; +} + void TargetLibraryInfoImpl::disableAllFunctions() { memset(AvailableArray, 0, sizeof(AvailableArray)); } diff --git a/llvm/lib/Analysis/VFABIDemangling.cpp b/llvm/lib/Analysis/VFABIDemangling.cpp index 22fc52070015..426f98c0c628 100644 --- a/llvm/lib/Analysis/VFABIDemangling.cpp +++ b/llvm/lib/Analysis/VFABIDemangling.cpp @@ -126,7 +126,7 @@ static ParseRet tryParseLinearTokenWithRuntimeStep(StringRef &ParseString, return ParseRet::None; } -/// The function looks for the following stringt at the beginning of +/// The function looks for the following string at the beginning of /// the input string `ParseString`: /// /// <token> <number> diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 9b8376833d2e..769d921eb1e8 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -1826,6 +1826,8 @@ void computeKnownBits(const Value *V, const APInt &DemandedElts, Known.Zero &= ~Elt; Known.One &= Elt; } + if (Known.hasConflict()) + Known.resetAll(); return; } @@ -1849,6 +1851,8 @@ void computeKnownBits(const Value *V, const APInt &DemandedElts, Known.Zero &= ~Elt; Known.One &= Elt; } + if (Known.hasConflict()) + Known.resetAll(); return; } @@ -6556,10 +6560,25 @@ static bool shiftAmountKnownInRange(const Value *ShiftAmount) { return Safe; } -static bool canCreateUndefOrPoison(const Operator *Op, bool PoisonOnly, +enum class UndefPoisonKind { + PoisonOnly = (1 << 0), + UndefOnly = (1 << 1), + UndefOrPoison = PoisonOnly | UndefOnly, +}; + +static bool includesPoison(UndefPoisonKind Kind) { + return (unsigned(Kind) & unsigned(UndefPoisonKind::PoisonOnly)) != 0; +} + +static bool includesUndef(UndefPoisonKind Kind) { + return (unsigned(Kind) & unsigned(UndefPoisonKind::UndefOnly)) != 0; +} + +static bool canCreateUndefOrPoison(const Operator *Op, UndefPoisonKind Kind, bool ConsiderFlagsAndMetadata) { - if (ConsiderFlagsAndMetadata && Op->hasPoisonGeneratingFlagsOrMetadata()) + if (ConsiderFlagsAndMetadata && includesPoison(Kind) && + Op->hasPoisonGeneratingFlagsOrMetadata()) return true; unsigned Opcode = Op->getOpcode(); @@ -6569,7 +6588,7 @@ static bool canCreateUndefOrPoison(const Operator *Op, bool PoisonOnly, case Instruction::Shl: case Instruction::AShr: case Instruction::LShr: - return !shiftAmountKnownInRange(Op->getOperand(1)); + return includesPoison(Kind) && !shiftAmountKnownInRange(Op->getOperand(1)); case Instruction::FPToSI: case Instruction::FPToUI: // fptosi/ui yields poison if the resulting value does not fit in the @@ -6610,7 +6629,8 @@ static bool canCreateUndefOrPoison(const Operator *Op, bool PoisonOnly, return false; case Intrinsic::sshl_sat: case Intrinsic::ushl_sat: - return !shiftAmountKnownInRange(II->getArgOperand(1)); + return includesPoison(Kind) && + !shiftAmountKnownInRange(II->getArgOperand(1)); case Intrinsic::fma: case Intrinsic::fmuladd: case Intrinsic::sqrt: @@ -6665,18 +6685,16 @@ static bool canCreateUndefOrPoison(const Operator *Op, bool PoisonOnly, auto *VTy = cast<VectorType>(Op->getOperand(0)->getType()); unsigned IdxOp = Op->getOpcode() == Instruction::InsertElement ? 2 : 1; auto *Idx = dyn_cast<ConstantInt>(Op->getOperand(IdxOp)); - if (!Idx || Idx->getValue().uge(VTy->getElementCount().getKnownMinValue())) - return true; + if (includesPoison(Kind)) + return !Idx || + Idx->getValue().uge(VTy->getElementCount().getKnownMinValue()); return false; } case Instruction::ShuffleVector: { - // shufflevector may return undef. - if (PoisonOnly) - return false; ArrayRef<int> Mask = isa<ConstantExpr>(Op) ? cast<ConstantExpr>(Op)->getShuffleMask() : cast<ShuffleVectorInst>(Op)->getShuffleMask(); - return is_contained(Mask, PoisonMaskElem); + return includesPoison(Kind) && is_contained(Mask, PoisonMaskElem); } case Instruction::FNeg: case Instruction::PHI: @@ -6712,17 +6730,17 @@ static bool canCreateUndefOrPoison(const Operator *Op, bool PoisonOnly, bool llvm::canCreateUndefOrPoison(const Operator *Op, bool ConsiderFlagsAndMetadata) { - return ::canCreateUndefOrPoison(Op, /*PoisonOnly=*/false, + return ::canCreateUndefOrPoison(Op, UndefPoisonKind::UndefOrPoison, ConsiderFlagsAndMetadata); } bool llvm::canCreatePoison(const Operator *Op, bool ConsiderFlagsAndMetadata) { - return ::canCreateUndefOrPoison(Op, /*PoisonOnly=*/true, + return ::canCreateUndefOrPoison(Op, UndefPoisonKind::PoisonOnly, ConsiderFlagsAndMetadata); } -static bool directlyImpliesPoison(const Value *ValAssumedPoison, - const Value *V, unsigned Depth) { +static bool directlyImpliesPoison(const Value *ValAssumedPoison, const Value *V, + unsigned Depth) { if (ValAssumedPoison == V) return true; @@ -6774,14 +6792,11 @@ bool llvm::impliesPoison(const Value *ValAssumedPoison, const Value *V) { return ::impliesPoison(ValAssumedPoison, V, /* Depth */ 0); } -static bool programUndefinedIfUndefOrPoison(const Value *V, - bool PoisonOnly); +static bool programUndefinedIfUndefOrPoison(const Value *V, bool PoisonOnly); -static bool isGuaranteedNotToBeUndefOrPoison(const Value *V, - AssumptionCache *AC, - const Instruction *CtxI, - const DominatorTree *DT, - unsigned Depth, bool PoisonOnly) { +static bool isGuaranteedNotToBeUndefOrPoison( + const Value *V, AssumptionCache *AC, const Instruction *CtxI, + const DominatorTree *DT, unsigned Depth, UndefPoisonKind Kind) { if (Depth >= MaxAnalysisRecursionDepth) return false; @@ -6796,16 +6811,19 @@ static bool isGuaranteedNotToBeUndefOrPoison(const Value *V, } if (auto *C = dyn_cast<Constant>(V)) { + if (isa<PoisonValue>(C)) + return !includesPoison(Kind); + if (isa<UndefValue>(C)) - return PoisonOnly && !isa<PoisonValue>(C); + return !includesUndef(Kind); if (isa<ConstantInt>(C) || isa<GlobalVariable>(C) || isa<ConstantFP>(V) || isa<ConstantPointerNull>(C) || isa<Function>(C)) return true; if (C->getType()->isVectorTy() && !isa<ConstantExpr>(C)) - return (PoisonOnly ? !C->containsPoisonElement() - : !C->containsUndefOrPoisonElement()) && + return (!includesUndef(Kind) ? !C->containsPoisonElement() + : !C->containsUndefOrPoisonElement()) && !C->containsConstantExpression(); } @@ -6823,8 +6841,7 @@ static bool isGuaranteedNotToBeUndefOrPoison(const Value *V, return true; auto OpCheck = [&](const Value *V) { - return isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth + 1, - PoisonOnly); + return isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth + 1, Kind); }; if (auto *Opr = dyn_cast<Operator>(V)) { @@ -6846,14 +6863,16 @@ static bool isGuaranteedNotToBeUndefOrPoison(const Value *V, for (unsigned i = 0; i < Num; ++i) { auto *TI = PN->getIncomingBlock(i)->getTerminator(); if (!isGuaranteedNotToBeUndefOrPoison(PN->getIncomingValue(i), AC, TI, - DT, Depth + 1, PoisonOnly)) { + DT, Depth + 1, Kind)) { IsWellDefined = false; break; } } if (IsWellDefined) return true; - } else if (!canCreateUndefOrPoison(Opr) && all_of(Opr->operands(), OpCheck)) + } else if (!::canCreateUndefOrPoison(Opr, Kind, + /*ConsiderFlagsAndMetadata*/ true) && + all_of(Opr->operands(), OpCheck)) return true; } @@ -6863,7 +6882,7 @@ static bool isGuaranteedNotToBeUndefOrPoison(const Value *V, I->hasMetadata(LLVMContext::MD_dereferenceable_or_null)) return true; - if (programUndefinedIfUndefOrPoison(V, PoisonOnly)) + if (programUndefinedIfUndefOrPoison(V, !includesUndef(Kind))) return true; // CxtI may be null or a cloned instruction. @@ -6895,7 +6914,7 @@ static bool isGuaranteedNotToBeUndefOrPoison(const Value *V, if (Cond) { if (Cond == V) return true; - else if (PoisonOnly && isa<Operator>(Cond)) { + else if (!includesUndef(Kind) && isa<Operator>(Cond)) { // For poison, we can analyze further auto *Opr = cast<Operator>(Cond); if (any_of(Opr->operands(), @@ -6917,20 +6936,22 @@ bool llvm::isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC, const Instruction *CtxI, const DominatorTree *DT, unsigned Depth) { - return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth, false); + return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth, + UndefPoisonKind::UndefOrPoison); } bool llvm::isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC, const Instruction *CtxI, const DominatorTree *DT, unsigned Depth) { - return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth, true); + return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth, + UndefPoisonKind::PoisonOnly); } bool llvm::isGuaranteedNotToBeUndef(const Value *V, AssumptionCache *AC, const Instruction *CtxI, const DominatorTree *DT, unsigned Depth) { - // TODO: This is currently equivalent to isGuaranteedNotToBeUndefOrPoison(). - return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth, false); + return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth, + UndefPoisonKind::UndefOnly); } /// Return true if undefined behavior would provably be executed on the path to diff --git a/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp b/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp index d6f487c18b03..30ea7eef3a12 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp @@ -412,7 +412,7 @@ static uint32_t constructAbbreviationTag( const std::optional<DWARF5AccelTable::UnitIndexAndEncoding> &EntryRet) { uint32_t AbbrvTag = 0; if (EntryRet) - AbbrvTag |= 1 << EntryRet->Endoding.Index; + AbbrvTag |= 1 << EntryRet->Encoding.Index; AbbrvTag |= 1 << dwarf::DW_IDX_die_offset; AbbrvTag |= Tag << LowerBitSize; return AbbrvTag; @@ -429,7 +429,7 @@ void Dwarf5AccelTableWriter<DataT>::populateAbbrevsMap() { if (Abbreviations.count(AbbrvTag) == 0) { SmallVector<DWARF5AccelTableData::AttributeEncoding, 2> UA; if (EntryRet) - UA.push_back(EntryRet->Endoding); + UA.push_back(EntryRet->Encoding); UA.push_back({dwarf::DW_IDX_die_offset, dwarf::DW_FORM_ref4}); Abbreviations.try_emplace(AbbrvTag, UA); } diff --git a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp index 36c91b7fa97e..893aa4a91828 100644 --- a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp +++ b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp @@ -15,14 +15,17 @@ #include "llvm/CodeGen/ReplaceWithVeclib.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Analysis/DemandedBits.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/CodeGen/Passes.h" +#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstIterator.h" +#include "llvm/Support/TypeSize.h" #include "llvm/Transforms/Utils/ModuleUtils.h" using namespace llvm; @@ -38,138 +41,137 @@ STATISTIC(NumTLIFuncDeclAdded, STATISTIC(NumFuncUsedAdded, "Number of functions added to `llvm.compiler.used`"); -static bool replaceWithTLIFunction(CallInst &CI, const StringRef TLIName) { - Module *M = CI.getModule(); - - Function *OldFunc = CI.getCalledFunction(); - - // Check if the vector library function is already declared in this module, - // otherwise insert it. +/// Returns a vector Function that it adds to the Module \p M. When an \p +/// ScalarFunc is not null, it copies its attributes to the newly created +/// Function. +Function *getTLIFunction(Module *M, FunctionType *VectorFTy, + const StringRef TLIName, + Function *ScalarFunc = nullptr) { Function *TLIFunc = M->getFunction(TLIName); if (!TLIFunc) { - TLIFunc = Function::Create(OldFunc->getFunctionType(), - Function::ExternalLinkage, TLIName, *M); - TLIFunc->copyAttributesFrom(OldFunc); + TLIFunc = + Function::Create(VectorFTy, Function::ExternalLinkage, TLIName, *M); + if (ScalarFunc) + TLIFunc->copyAttributesFrom(ScalarFunc); LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added vector library function `" << TLIName << "` of type `" << *(TLIFunc->getType()) << "` to module.\n"); ++NumTLIFuncDeclAdded; - - // Add the freshly created function to llvm.compiler.used, - // similar to as it is done in InjectTLIMappings + // Add the freshly created function to llvm.compiler.used, similar to as it + // is done in InjectTLIMappings. appendToCompilerUsed(*M, {TLIFunc}); - LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << TLIName << "` to `@llvm.compiler.used`.\n"); ++NumFuncUsedAdded; } + return TLIFunc; +} - // Replace the call to the vector intrinsic with a call - // to the corresponding function from the vector library. - IRBuilder<> IRBuilder(&CI); - SmallVector<Value *> Args(CI.args()); - // Preserve the operand bundles. - SmallVector<OperandBundleDef, 1> OpBundles; - CI.getOperandBundlesAsDefs(OpBundles); - CallInst *Replacement = IRBuilder.CreateCall(TLIFunc, Args, OpBundles); - assert(OldFunc->getFunctionType() == TLIFunc->getFunctionType() && - "Expecting function types to be identical"); - CI.replaceAllUsesWith(Replacement); - if (isa<FPMathOperator>(Replacement)) { - // Preserve fast math flags for FP math. - Replacement->copyFastMathFlags(&CI); +/// Replace the call to the vector intrinsic ( \p CalltoReplace ) with a call to +/// the corresponding function from the vector library ( \p TLIVecFunc ). +static void replaceWithTLIFunction(CallInst &CalltoReplace, VFInfo &Info, + Function *TLIVecFunc) { + IRBuilder<> IRBuilder(&CalltoReplace); + SmallVector<Value *> Args(CalltoReplace.args()); + if (auto OptMaskpos = Info.getParamIndexForOptionalMask()) { + auto *MaskTy = VectorType::get(Type::getInt1Ty(CalltoReplace.getContext()), + Info.Shape.VF); + Args.insert(Args.begin() + OptMaskpos.value(), + Constant::getAllOnesValue(MaskTy)); } - LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Replaced call to `" - << OldFunc->getName() << "` with call to `" << TLIName - << "`.\n"); - ++NumCallsReplaced; - return true; + // Preserve the operand bundles. + SmallVector<OperandBundleDef, 1> OpBundles; + CalltoReplace.getOperandBundlesAsDefs(OpBundles); + CallInst *Replacement = IRBuilder.CreateCall(TLIVecFunc, Args, OpBundles); + CalltoReplace.replaceAllUsesWith(Replacement); + // Preserve fast math flags for FP math. + if (isa<FPMathOperator>(Replacement)) + Replacement->copyFastMathFlags(&CalltoReplace); } +/// Returns true when successfully replaced \p CallToReplace with a suitable +/// function taking vector arguments, based on available mappings in the \p TLI. +/// Currently only works when \p CallToReplace is a call to vectorized +/// intrinsic. static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, - CallInst &CI) { - if (!CI.getCalledFunction()) { + CallInst &CallToReplace) { + if (!CallToReplace.getCalledFunction()) return false; - } - auto IntrinsicID = CI.getCalledFunction()->getIntrinsicID(); - if (IntrinsicID == Intrinsic::not_intrinsic) { - // Replacement is only performed for intrinsic functions + auto IntrinsicID = CallToReplace.getCalledFunction()->getIntrinsicID(); + // Replacement is only performed for intrinsic functions. + if (IntrinsicID == Intrinsic::not_intrinsic) return false; - } - // Convert vector arguments to scalar type and check that - // all vector operands have identical vector width. + // Compute arguments types of the corresponding scalar call. Additionally + // checks if in the vector call, all vector operands have the same EC. ElementCount VF = ElementCount::getFixed(0); - SmallVector<Type *> ScalarTypes; - for (auto Arg : enumerate(CI.args())) { - auto *ArgType = Arg.value()->getType(); - // Vector calls to intrinsics can still have - // scalar operands for specific arguments. + SmallVector<Type *> ScalarArgTypes; + for (auto Arg : enumerate(CallToReplace.args())) { + auto *ArgTy = Arg.value()->getType(); if (isVectorIntrinsicWithScalarOpAtArg(IntrinsicID, Arg.index())) { - ScalarTypes.push_back(ArgType); - } else { - // The argument in this place should be a vector if - // this is a call to a vector intrinsic. - auto *VectorArgTy = dyn_cast<VectorType>(ArgType); - if (!VectorArgTy) { - // The argument is not a vector, do not perform - // the replacement. - return false; - } - ElementCount NumElements = VectorArgTy->getElementCount(); - if (NumElements.isScalable()) { - // The current implementation does not support - // scalable vectors. + ScalarArgTypes.push_back(ArgTy); + } else if (auto *VectorArgTy = dyn_cast<VectorType>(ArgTy)) { + ScalarArgTypes.push_back(ArgTy->getScalarType()); + // Disallow vector arguments with different VFs. When processing the first + // vector argument, store it's VF, and for the rest ensure that they match + // it. + if (VF.isZero()) + VF = VectorArgTy->getElementCount(); + else if (VF != VectorArgTy->getElementCount()) return false; - } - if (VF.isNonZero() && VF != NumElements) { - // The different arguments differ in vector size. - return false; - } else { - VF = NumElements; - } - ScalarTypes.push_back(VectorArgTy->getElementType()); - } + } else + // Exit when it is supposed to be a vector argument but it isn't. + return false; } - // Try to reconstruct the name for the scalar version of this - // intrinsic using the intrinsic ID and the argument types - // converted to scalar above. - std::string ScalarName; - if (Intrinsic::isOverloaded(IntrinsicID)) { - ScalarName = Intrinsic::getName(IntrinsicID, ScalarTypes, CI.getModule()); - } else { - ScalarName = Intrinsic::getName(IntrinsicID).str(); - } + // Try to reconstruct the name for the scalar version of this intrinsic using + // the intrinsic ID and the argument types converted to scalar above. + std::string ScalarName = + (Intrinsic::isOverloaded(IntrinsicID) + ? Intrinsic::getName(IntrinsicID, ScalarArgTypes, + CallToReplace.getModule()) + : Intrinsic::getName(IntrinsicID).str()); + + // Try to find the mapping for the scalar version of this intrinsic and the + // exact vector width of the call operands in the TargetLibraryInfo. First, + // check with a non-masked variant, and if that fails try with a masked one. + const VecDesc *VD = + TLI.getVectorMappingInfo(ScalarName, VF, /*Masked*/ false); + if (!VD && !(VD = TLI.getVectorMappingInfo(ScalarName, VF, /*Masked*/ true))) + return false; - if (!TLI.isFunctionVectorizable(ScalarName)) { - // The TargetLibraryInfo does not contain a vectorized version of - // the scalar function. + LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Found TLI mapping from: `" << ScalarName + << "` and vector width " << VF << " to: `" + << VD->getVectorFnName() << "`.\n"); + + // Replace the call to the intrinsic with a call to the vector library + // function. + Type *ScalarRetTy = CallToReplace.getType()->getScalarType(); + FunctionType *ScalarFTy = + FunctionType::get(ScalarRetTy, ScalarArgTypes, /*isVarArg*/ false); + const std::string MangledName = VD->getVectorFunctionABIVariantString(); + auto OptInfo = VFABI::tryDemangleForVFABI(MangledName, ScalarFTy); + if (!OptInfo) return false; - } - // Try to find the mapping for the scalar version of this intrinsic - // and the exact vector width of the call operands in the - // TargetLibraryInfo. - StringRef TLIName = TLI.getVectorizedFunction(ScalarName, VF); - - LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Looking up TLI mapping for `" - << ScalarName << "` and vector width " << VF << ".\n"); - - if (!TLIName.empty()) { - // Found the correct mapping in the TargetLibraryInfo, - // replace the call to the intrinsic with a call to - // the vector library function. - LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Found TLI function `" << TLIName - << "`.\n"); - return replaceWithTLIFunction(CI, TLIName); - } + FunctionType *VectorFTy = VFABI::createFunctionType(*OptInfo, ScalarFTy); + if (!VectorFTy) + return false; + + Function *FuncToReplace = CallToReplace.getCalledFunction(); + Function *TLIFunc = getTLIFunction(CallToReplace.getModule(), VectorFTy, + VD->getVectorFnName(), FuncToReplace); + replaceWithTLIFunction(CallToReplace, *OptInfo, TLIFunc); - return false; + LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Replaced call to `" + << FuncToReplace->getName() << "` with call to `" + << TLIFunc->getName() << "`.\n"); + ++NumCallsReplaced; + return true; } static bool runImpl(const TargetLibraryInfo &TLI, Function &F) { @@ -185,9 +187,8 @@ static bool runImpl(const TargetLibraryInfo &TLI, Function &F) { } // Erase the calls to the intrinsics that have been replaced // with calls to the vector library. - for (auto *CI : ReplacedCalls) { + for (auto *CI : ReplacedCalls) CI->eraseFromParent(); - } return Changed; } @@ -207,10 +208,10 @@ PreservedAnalyses ReplaceWithVeclib::run(Function &F, PA.preserve<DemandedBitsAnalysis>(); PA.preserve<OptimizationRemarkEmitterAnalysis>(); return PA; - } else { - // The pass did not replace any calls, hence it preserves all analyses. - return PreservedAnalyses::all(); } + + // The pass did not replace any calls, hence it preserves all analyses. + return PreservedAnalyses::all(); } //////////////////////////////////////////////////////////////////////////////// diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index 362fa92dd44b..3d21bd22e6ef 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -1871,6 +1871,9 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { case ISD::EXPERIMENTAL_VP_STRIDED_STORE: Res = PromoteIntOp_VP_STRIDED(N, OpNo); break; + case ISD::EXPERIMENTAL_VP_SPLICE: + Res = PromoteIntOp_VP_SPLICE(N, OpNo); + break; } // If the result is null, the sub-method took care of registering results etc. @@ -2549,6 +2552,20 @@ SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); } +SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo) { + SmallVector<SDValue, 6> NewOps(N->op_begin(), N->op_end()); + + if (OpNo == 2) { // Offset operand + NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); + return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); + } + + assert((OpNo == 4 || OpNo == 5) && "Unexpected operand for promotion"); + + NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); + return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); +} + //===----------------------------------------------------------------------===// // Integer Result Expansion //===----------------------------------------------------------------------===// diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index 9d5931b44ac6..4a249e7a2dc9 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -410,6 +410,7 @@ private: SDValue PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo); SDValue PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo); SDValue PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo); + SDValue PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo); void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 5be1892a44f6..81facf92e55a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -6858,8 +6858,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, // expanding copies of large vectors from registers. This only works for // fixed length vectors, since we need to know the exact number of // elements. - if (N2C && N1.getOperand(0).getValueType().isFixedLengthVector() && - N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0) { + if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS && + N1.getOperand(0).getValueType().isFixedLengthVector()) { unsigned Factor = N1.getOperand(0).getValueType().getVectorNumElements(); return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, @@ -6976,7 +6976,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of // the concat have the same type as the extract. - if (N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0 && + if (N1.getOpcode() == ISD::CONCAT_VECTORS && VT == N1.getOperand(0).getValueType()) { unsigned Factor = VT.getVectorMinNumElements(); return N1.getOperand(N2C->getZExtValue() / Factor); diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td index db92a94e40e4..97e92a57a7ff 100644 --- a/llvm/lib/Target/AArch64/AArch64.td +++ b/llvm/lib/Target/AArch64/AArch64.td @@ -622,8 +622,13 @@ def FeatureLdpAlignedOnly : SubtargetFeature<"ldp-aligned-only", "HasLdpAlignedO def FeatureStpAlignedOnly : SubtargetFeature<"stp-aligned-only", "HasStpAlignedOnly", "true", "In order to emit stp, first check if the store will be aligned to 2 * element_size">; +// AArch64 2023 Architecture Extensions (v9.5-A) + def FeatureCPA : SubtargetFeature<"cpa", "HasCPA", "true", - "Enable ARMv9.5-A Checked Pointer Arithmetic (FEAT_CPA)">; + "Enable Armv9.5-A Checked Pointer Arithmetic (FEAT_CPA)">; + +def FeaturePAuthLR : SubtargetFeature<"pauth-lr", "HasPAuthLR", + "true", "Enable Armv9.5-A PAC enhancements (FEAT_PAuth_LR)">; //===----------------------------------------------------------------------===// // Architectures. @@ -810,7 +815,7 @@ def SMEUnsupported : AArch64Unsupported { SME2Unsupported.F); } -let F = [HasPAuth] in +let F = [HasPAuth, HasPAuthLR] in def PAUnsupported : AArch64Unsupported; include "AArch64SchedA53.td" diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td index d6c00be80bd9..99f256b88782 100644 --- a/llvm/lib/Target/AArch64/AArch64Combine.td +++ b/llvm/lib/Target/AArch64/AArch64Combine.td @@ -134,6 +134,14 @@ def shuffle_vector_lowering : GICombineGroup<[dup, rev, ext, zip, uzp, trn, form_duplane, shuf_to_ins]>; +// Turn G_UNMERGE_VALUES -> G_EXTRACT_VECTOR_ELT's +def vector_unmerge_lowering : GICombineRule < + (defs root:$root), + (match (wip_match_opcode G_UNMERGE_VALUES):$root, + [{ return matchScalarizeVectorUnmerge(*${root}, MRI); }]), + (apply [{ applyScalarizeVectorUnmerge(*${root}, MRI, B); }]) +>; + def adjust_icmp_imm_matchdata : GIDefMatchData<"std::pair<uint64_t, CmpInst::Predicate>">; def adjust_icmp_imm : GICombineRule < @@ -251,7 +259,8 @@ def AArch64PostLegalizerLowering icmp_lowering, build_vector_lowering, lower_vector_fcmp, form_truncstore, vector_sext_inreg_to_shift, - unmerge_ext_to_unmerge, lower_mull]> { + unmerge_ext_to_unmerge, lower_mull, + vector_unmerge_lowering]> { } // Post-legalization combines which are primarily optimizations. diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td index 690ac0dcda62..cb63d8726744 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td +++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td @@ -2368,6 +2368,80 @@ class ClearAuth<bits<1> data, string asm> let Inst{4-0} = Rd; } +// v9.5-A FEAT_PAuth_LR + +class SignAuthFixedRegs<bits<5> opcode2, bits<6> opcode, string asm> + : I<(outs), (ins), asm, "", "", []>, + Sched<[WriteI, ReadI]> { + let Inst{31} = 0b1; // sf + let Inst{30} = 0b1; + let Inst{29} = 0b0; // S + let Inst{28-21} = 0b11010110; + let Inst{20-16} = opcode2; + let Inst{15-10} = opcode; + let Inst{9-5} = 0b11111; // Rn + let Inst{4-0} = 0b11110; // Rd +} + +def PAuthPCRelLabel16Operand : PCRelLabel<16> { + let Name = "PAuthPCRelLabel16"; + let PredicateMethod = "isPAuthPCRelLabel16Operand"; +} +def am_pauth_pcrel : Operand<OtherVT> { + let EncoderMethod = "getPAuthPCRelOpValue"; + let DecoderMethod = "DecodePCRelLabel16"; + let PrintMethod = "printAlignedLabel"; + let ParserMatchClass = PAuthPCRelLabel16Operand; + let OperandType = "OPERAND_PCREL"; +} + +class SignAuthPCRel<bits<2> opc, string asm> + : I<(outs), (ins am_pauth_pcrel:$label), asm, "\t$label", "", []>, + Sched<[]> { + bits<16> label; + let Inst{31} = 0b1; // sf + let Inst{30-23} = 0b11100111; + let Inst{22-21} = opc; + let Inst{20-5} = label; // imm + let Inst{4-0} = 0b11111; // Rd +} + +class SignAuthOneReg<bits<5> opcode2, bits<6> opcode, string asm> + : I<(outs), (ins GPR64:$Rn), asm, "\t$Rn", "", []>, + Sched<[]> { + bits<5> Rn; + let Inst{31} = 0b1; // sf + let Inst{30} = 0b1; + let Inst{29} = 0b0; // S + let Inst{28-21} = 0b11010110; + let Inst{20-16} = opcode2; + let Inst{15-10} = opcode; + let Inst{9-5} = Rn; + let Inst{4-0} = 0b11110; // Rd +} + +class SignAuthReturnPCRel<bits<3> opc, bits<5> op2, string asm> + : I<(outs), (ins am_pauth_pcrel:$label), asm, "\t$label", "", []>, + Sched<[WriteAtomic]> { + bits<16> label; + let Inst{31-24} = 0b01010101; + let Inst{23-21} = opc; + let Inst{20-5} = label; // imm16 + let Inst{4-0} = op2; +} + +class SignAuthReturnReg<bits<6> op3, string asm> + : I<(outs), (ins GPR64common:$Rm), asm, "\t$Rm", "", []>, + Sched<[WriteAtomic]> { + bits<5> Rm; + let Inst{31-25} = 0b1101011; + let Inst{24-21} = 0b0010; // opc + let Inst{20-16} = 0b11111; // op2 + let Inst{15-10} = op3; + let Inst{9-5} = 0b11111; // Rn + let Inst{4-0} = Rm; // op4 (Rm) +} + // Base class for the Armv8.4-A 8 and 16-bit flag manipulation instructions class BaseFlagManipulation<bit sf, bit sz, dag iops, string asm, string ops> : I<(outs), iops, asm, ops, "", []>, diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 7d71c316bcb0..6d85e1fb5fbf 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -4089,6 +4089,29 @@ AArch64InstrInfo::getLdStOffsetOp(const MachineInstr &MI) { return MI.getOperand(Idx); } +const MachineOperand & +AArch64InstrInfo::getLdStAmountOp(const MachineInstr &MI) { + switch (MI.getOpcode()) { + default: + llvm_unreachable("Unexpected opcode"); + case AArch64::LDRBroX: + case AArch64::LDRBBroX: + case AArch64::LDRSBXroX: + case AArch64::LDRSBWroX: + case AArch64::LDRHroX: + case AArch64::LDRHHroX: + case AArch64::LDRSHXroX: + case AArch64::LDRSHWroX: + case AArch64::LDRWroX: + case AArch64::LDRSroX: + case AArch64::LDRSWroX: + case AArch64::LDRDroX: + case AArch64::LDRXroX: + case AArch64::LDRQroX: + return MI.getOperand(4); + } +} + static const TargetRegisterClass *getRegClass(const MachineInstr &MI, Register Reg) { if (MI.getParent() == nullptr) @@ -8779,12 +8802,23 @@ AArch64InstrInfo::getOutliningTypeImpl(MachineBasicBlock::iterator &MIT, // Don't outline anything used for return address signing. The outlined // function will get signed later if needed switch (MI.getOpcode()) { + case AArch64::PACM: case AArch64::PACIASP: case AArch64::PACIBSP: + case AArch64::PACIASPPC: + case AArch64::PACIBSPPC: case AArch64::AUTIASP: case AArch64::AUTIBSP: + case AArch64::AUTIASPPCi: + case AArch64::AUTIASPPCr: + case AArch64::AUTIBSPPCi: + case AArch64::AUTIBSPPCr: case AArch64::RETAA: case AArch64::RETAB: + case AArch64::RETAASPPCi: + case AArch64::RETAASPPCr: + case AArch64::RETABSPPCi: + case AArch64::RETABSPPCr: case AArch64::EMITBKEY: case AArch64::PAUTH_PROLOGUE: case AArch64::PAUTH_EPILOGUE: diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h index 6526f6740747..db24a19fe5f8 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h @@ -111,6 +111,9 @@ public: /// Returns the immediate offset operator of a load/store. static const MachineOperand &getLdStOffsetOp(const MachineInstr &MI); + /// Returns the shift amount operator of a load/store. + static const MachineOperand &getLdStAmountOp(const MachineInstr &MI); + /// Returns whether the instruction is FP or NEON. static bool isFpOrNEON(const MachineInstr &MI); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index bdb38f0c3789..977729bb082b 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -61,6 +61,9 @@ def HasLOR : Predicate<"Subtarget->hasLOR()">, def HasPAuth : Predicate<"Subtarget->hasPAuth()">, AssemblerPredicateWithAll<(all_of FeaturePAuth), "pauth">; +def HasPAuthLR : Predicate<"Subtarget->hasPAuthLR()">, + AssemblerPredicateWithAll<(all_of FeaturePAuthLR), "pauth-lr">; + def HasJS : Predicate<"Subtarget->hasJS()">, AssemblerPredicateWithAll<(all_of FeatureJS), "jsconv">; @@ -1646,6 +1649,42 @@ let Predicates = [HasPAuth] in { } +// v9.5-A pointer authentication extensions + +// Always accept "pacm" as an alias for "hint #39", but don't emit it when +// disassembling if we don't have the pauth-lr feature. +let CRm = 0b0100 in { + def PACM : SystemNoOperands<0b111, "hint\t#39">; +} +def : InstAlias<"pacm", (PACM), 0>; + +let Predicates = [HasPAuthLR] in { + let Defs = [LR], Uses = [LR, SP] in { + // opcode2, opcode, asm + def PACIASPPC : SignAuthFixedRegs<0b00001, 0b101000, "paciasppc">; + def PACIBSPPC : SignAuthFixedRegs<0b00001, 0b101001, "pacibsppc">; + def PACNBIASPPC : SignAuthFixedRegs<0b00001, 0b100000, "pacnbiasppc">; + def PACNBIBSPPC : SignAuthFixedRegs<0b00001, 0b100001, "pacnbibsppc">; + // opc, asm + def AUTIASPPCi : SignAuthPCRel<0b00, "autiasppc">; + def AUTIBSPPCi : SignAuthPCRel<0b01, "autibsppc">; + // opcode2, opcode, asm + def AUTIASPPCr : SignAuthOneReg<0b00001, 0b100100, "autiasppc">; + def AUTIBSPPCr : SignAuthOneReg<0b00001, 0b100101, "autibsppc">; + } + + let Uses = [LR, SP], isReturn = 1, isTerminator = 1, isBarrier = 1 in { + // opc, op2, asm + def RETAASPPCi : SignAuthReturnPCRel<0b000, 0b11111, "retaasppc">; + def RETABSPPCi : SignAuthReturnPCRel<0b001, 0b11111, "retabsppc">; + // op3, asm + def RETAASPPCr : SignAuthReturnReg<0b000010, "retaasppc">; + def RETABSPPCr : SignAuthReturnReg<0b000011, "retabsppc">; + } + def : InstAlias<"pacm", (PACM), 1>; +} + + // v8.3a floating point conversion for javascript let Predicates = [HasJS, HasFPARMv8], Defs = [NZCV] in def FJCVTZS : BaseFPToIntegerUnscaled<0b01, 0b11, 0b110, FPR64, GPR32, @@ -6482,23 +6521,23 @@ def : Pat<(v2i64 (vector_insert v2i64:$src, (i64 (bitconvert (f64 FPR64:$Sn))), // f32 bitcast(vector_extract(v4i32 src, lane)) -> EXTRACT_SUBREG(INSvi32lane(-, 0, src, lane)) def : Pat<(f32 (bitconvert (i32 (vector_extract v4i32:$src, imm:$Immd)))), (EXTRACT_SUBREG (INSvi32lane (IMPLICIT_DEF), 0, V128:$src, imm:$Immd), ssub)>; -def : Pat<(f32 (bitconvert (i32 (vector_extract v4i32:$src, 0)))), +def : Pat<(f32 (bitconvert (i32 (vector_extract v4i32:$src, (i64 0))))), (EXTRACT_SUBREG V128:$src, ssub)>; def : Pat<(f64 (bitconvert (i64 (vector_extract v2i64:$src, imm:$Immd)))), (EXTRACT_SUBREG (INSvi64lane (IMPLICIT_DEF), 0, V128:$src, imm:$Immd), dsub)>; -def : Pat<(f64 (bitconvert (i64 (vector_extract v2i64:$src, 0)))), +def : Pat<(f64 (bitconvert (i64 (vector_extract v2i64:$src, (i64 0))))), (EXTRACT_SUBREG V128:$src, dsub)>; // Floating point vector extractions are codegen'd as either a sequence of // subregister extractions, or a MOV (aka DUP here) if // the lane number is anything other than zero. -def : Pat<(vector_extract (v2f64 V128:$Rn), 0), +def : Pat<(f64 (vector_extract (v2f64 V128:$Rn), (i64 0))), (f64 (EXTRACT_SUBREG V128:$Rn, dsub))>; -def : Pat<(vector_extract (v4f32 V128:$Rn), 0), +def : Pat<(f32 (vector_extract (v4f32 V128:$Rn), (i64 0))), (f32 (EXTRACT_SUBREG V128:$Rn, ssub))>; -def : Pat<(vector_extract (v8f16 V128:$Rn), 0), +def : Pat<(f16 (vector_extract (v8f16 V128:$Rn), (i64 0))), (f16 (EXTRACT_SUBREG V128:$Rn, hsub))>; -def : Pat<(vector_extract (v8bf16 V128:$Rn), 0), +def : Pat<(bf16 (vector_extract (v8bf16 V128:$Rn), (i64 0))), (bf16 (EXTRACT_SUBREG V128:$Rn, hsub))>; diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp index dc6d5b8950c3..aa7a4bc23536 100644 --- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp +++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp @@ -62,6 +62,8 @@ STATISTIC(NumUnscaledPairCreated, "Number of load/store from unscaled generated"); STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted"); STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted"); +STATISTIC(NumConstOffsetFolded, + "Number of const offset of index address folded"); DEBUG_COUNTER(RegRenamingCounter, DEBUG_TYPE "-reg-renaming", "Controls which pairs are considered for renaming"); @@ -75,6 +77,11 @@ static cl::opt<unsigned> LdStLimit("aarch64-load-store-scan-limit", static cl::opt<unsigned> UpdateLimit("aarch64-update-scan-limit", cl::init(100), cl::Hidden); +// The LdStConstLimit limits how far we search for const offset instructions +// when we form index address load/store instructions. +static cl::opt<unsigned> LdStConstLimit("aarch64-load-store-const-scan-limit", + cl::init(10), cl::Hidden); + // Enable register renaming to find additional store pairing opportunities. static cl::opt<bool> EnableRenaming("aarch64-load-store-renaming", cl::init(true), cl::Hidden); @@ -171,6 +178,13 @@ struct AArch64LoadStoreOpt : public MachineFunctionPass { findMatchingUpdateInsnForward(MachineBasicBlock::iterator I, int UnscaledOffset, unsigned Limit); + // Scan the instruction list to find a register assigned with a const + // value that can be combined with the current instruction (a load or store) + // using base addressing with writeback. Scan backwards. + MachineBasicBlock::iterator + findMatchingConstOffsetBackward(MachineBasicBlock::iterator I, unsigned Limit, + unsigned &Offset); + // Scan the instruction list to find a base register update that can // be combined with the current instruction (a load or store) using // pre or post indexed addressing with writeback. Scan backwards. @@ -182,11 +196,19 @@ struct AArch64LoadStoreOpt : public MachineFunctionPass { bool isMatchingUpdateInsn(MachineInstr &MemMI, MachineInstr &MI, unsigned BaseReg, int Offset); + bool isMatchingMovConstInsn(MachineInstr &MemMI, MachineInstr &MI, + unsigned IndexReg, unsigned &Offset); + // Merge a pre- or post-index base register update into a ld/st instruction. MachineBasicBlock::iterator mergeUpdateInsn(MachineBasicBlock::iterator I, MachineBasicBlock::iterator Update, bool IsPreIdx); + MachineBasicBlock::iterator + mergeConstOffsetInsn(MachineBasicBlock::iterator I, + MachineBasicBlock::iterator Update, unsigned Offset, + int Scale); + // Find and merge zero store instructions. bool tryToMergeZeroStInst(MachineBasicBlock::iterator &MBBI); @@ -199,6 +221,9 @@ struct AArch64LoadStoreOpt : public MachineFunctionPass { // Find and merge a base register updates before or after a ld/st instruction. bool tryToMergeLdStUpdate(MachineBasicBlock::iterator &MBBI); + // Find and merge a index ldr/st instruction into a base ld/st instruction. + bool tryToMergeIndexLdSt(MachineBasicBlock::iterator &MBBI, int Scale); + bool optimizeBlock(MachineBasicBlock &MBB, bool EnableNarrowZeroStOpt); bool runOnMachineFunction(MachineFunction &Fn) override; @@ -481,6 +506,42 @@ static unsigned getPreIndexedOpcode(unsigned Opc) { } } +static unsigned getBaseAddressOpcode(unsigned Opc) { + // TODO: Add more index address loads/stores. + switch (Opc) { + default: + llvm_unreachable("Opcode has no base address equivalent!"); + case AArch64::LDRBroX: + return AArch64::LDRBui; + case AArch64::LDRBBroX: + return AArch64::LDRBBui; + case AArch64::LDRSBXroX: + return AArch64::LDRSBXui; + case AArch64::LDRSBWroX: + return AArch64::LDRSBWui; + case AArch64::LDRHroX: + return AArch64::LDRHui; + case AArch64::LDRHHroX: + return AArch64::LDRHHui; + case AArch64::LDRSHXroX: + return AArch64::LDRSHXui; + case AArch64::LDRSHWroX: + return AArch64::LDRSHWui; + case AArch64::LDRWroX: + return AArch64::LDRWui; + case AArch64::LDRSroX: + return AArch64::LDRSui; + case AArch64::LDRSWroX: + return AArch64::LDRSWui; + case AArch64::LDRDroX: + return AArch64::LDRDui; + case AArch64::LDRXroX: + return AArch64::LDRXui; + case AArch64::LDRQroX: + return AArch64::LDRQui; + } +} + static unsigned getPostIndexedOpcode(unsigned Opc) { switch (Opc) { default: @@ -722,6 +783,41 @@ static bool isMergeableLdStUpdate(MachineInstr &MI) { } } +// Make sure this is a reg+reg Ld/St +static bool isMergeableIndexLdSt(MachineInstr &MI, int &Scale) { + unsigned Opc = MI.getOpcode(); + switch (Opc) { + default: + return false; + // Scaled instructions. + // TODO: Add more index address stores. + case AArch64::LDRBroX: + case AArch64::LDRBBroX: + case AArch64::LDRSBXroX: + case AArch64::LDRSBWroX: + Scale = 1; + return true; + case AArch64::LDRHroX: + case AArch64::LDRHHroX: + case AArch64::LDRSHXroX: + case AArch64::LDRSHWroX: + Scale = 2; + return true; + case AArch64::LDRWroX: + case AArch64::LDRSroX: + case AArch64::LDRSWroX: + Scale = 4; + return true; + case AArch64::LDRDroX: + case AArch64::LDRXroX: + Scale = 8; + return true; + case AArch64::LDRQroX: + Scale = 16; + return true; + } +} + static bool isRewritableImplicitDef(unsigned Opc) { switch (Opc) { default: @@ -2018,6 +2114,63 @@ AArch64LoadStoreOpt::mergeUpdateInsn(MachineBasicBlock::iterator I, return NextI; } +MachineBasicBlock::iterator +AArch64LoadStoreOpt::mergeConstOffsetInsn(MachineBasicBlock::iterator I, + MachineBasicBlock::iterator Update, + unsigned Offset, int Scale) { + assert((Update->getOpcode() == AArch64::MOVKWi) && + "Unexpected const mov instruction to merge!"); + MachineBasicBlock::iterator E = I->getParent()->end(); + MachineBasicBlock::iterator NextI = next_nodbg(I, E); + MachineBasicBlock::iterator PrevI = prev_nodbg(Update, E); + MachineInstr &MemMI = *I; + unsigned Mask = (1 << 12) * Scale - 1; + unsigned Low = Offset & Mask; + unsigned High = Offset - Low; + Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MemMI).getReg(); + Register IndexReg = AArch64InstrInfo::getLdStOffsetOp(MemMI).getReg(); + MachineInstrBuilder AddMIB, MemMIB; + + // Add IndexReg, BaseReg, High (the BaseReg may be SP) + AddMIB = + BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(AArch64::ADDXri)) + .addDef(IndexReg) + .addUse(BaseReg) + .addImm(High >> 12) // shifted value + .addImm(12); // shift 12 + (void)AddMIB; + // Ld/St DestReg, IndexReg, Imm12 + unsigned NewOpc = getBaseAddressOpcode(I->getOpcode()); + MemMIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(NewOpc)) + .add(getLdStRegOp(MemMI)) + .add(AArch64InstrInfo::getLdStOffsetOp(MemMI)) + .addImm(Low / Scale) + .setMemRefs(I->memoperands()) + .setMIFlags(I->mergeFlagsWith(*Update)); + (void)MemMIB; + + ++NumConstOffsetFolded; + LLVM_DEBUG(dbgs() << "Creating base address load/store.\n"); + LLVM_DEBUG(dbgs() << " Replacing instructions:\n "); + LLVM_DEBUG(PrevI->print(dbgs())); + LLVM_DEBUG(dbgs() << " "); + LLVM_DEBUG(Update->print(dbgs())); + LLVM_DEBUG(dbgs() << " "); + LLVM_DEBUG(I->print(dbgs())); + LLVM_DEBUG(dbgs() << " with instruction:\n "); + LLVM_DEBUG(((MachineInstr *)AddMIB)->print(dbgs())); + LLVM_DEBUG(dbgs() << " "); + LLVM_DEBUG(((MachineInstr *)MemMIB)->print(dbgs())); + LLVM_DEBUG(dbgs() << "\n"); + + // Erase the old instructions for the block. + I->eraseFromParent(); + PrevI->eraseFromParent(); + Update->eraseFromParent(); + + return NextI; +} + bool AArch64LoadStoreOpt::isMatchingUpdateInsn(MachineInstr &MemMI, MachineInstr &MI, unsigned BaseReg, int Offset) { @@ -2065,6 +2218,31 @@ bool AArch64LoadStoreOpt::isMatchingUpdateInsn(MachineInstr &MemMI, return false; } +bool AArch64LoadStoreOpt::isMatchingMovConstInsn(MachineInstr &MemMI, + MachineInstr &MI, + unsigned IndexReg, + unsigned &Offset) { + // The update instruction source and destination register must be the + // same as the load/store index register. + if (MI.getOpcode() == AArch64::MOVKWi && + TRI->isSuperOrSubRegisterEq(IndexReg, MI.getOperand(1).getReg())) { + + // movz + movk hold a large offset of a Ld/St instruction. + MachineBasicBlock::iterator B = MI.getParent()->begin(); + MachineBasicBlock::iterator MBBI = &MI; + MBBI = prev_nodbg(MBBI, B); + MachineInstr &MovzMI = *MBBI; + if (MovzMI.getOpcode() == AArch64::MOVZWi) { + unsigned Low = MovzMI.getOperand(1).getImm(); + unsigned High = MI.getOperand(2).getImm() << MI.getOperand(3).getImm(); + Offset = High + Low; + // 12-bit optionally shifted immediates are legal for adds. + return Offset >> 24 == 0; + } + } + return false; +} + MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnForward( MachineBasicBlock::iterator I, int UnscaledOffset, unsigned Limit) { MachineBasicBlock::iterator E = I->getParent()->end(); @@ -2220,6 +2398,60 @@ MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward( return E; } +MachineBasicBlock::iterator +AArch64LoadStoreOpt::findMatchingConstOffsetBackward( + MachineBasicBlock::iterator I, unsigned Limit, unsigned &Offset) { + MachineBasicBlock::iterator B = I->getParent()->begin(); + MachineBasicBlock::iterator E = I->getParent()->end(); + MachineInstr &MemMI = *I; + MachineBasicBlock::iterator MBBI = I; + + // If the load is the first instruction in the block, there's obviously + // not any matching load or store. + if (MBBI == B) + return E; + + // Make sure the IndexReg is killed and the shift amount is zero. + // TODO: Relex this restriction to extend, simplify processing now. + if (!AArch64InstrInfo::getLdStOffsetOp(MemMI).isKill() || + !AArch64InstrInfo::getLdStAmountOp(MemMI).isImm() || + (AArch64InstrInfo::getLdStAmountOp(MemMI).getImm() != 0)) + return E; + + Register IndexReg = AArch64InstrInfo::getLdStOffsetOp(MemMI).getReg(); + + // Track which register units have been modified and used between the first + // insn (inclusive) and the second insn. + ModifiedRegUnits.clear(); + UsedRegUnits.clear(); + unsigned Count = 0; + do { + MBBI = prev_nodbg(MBBI, B); + MachineInstr &MI = *MBBI; + + // Don't count transient instructions towards the search limit since there + // may be different numbers of them if e.g. debug information is present. + if (!MI.isTransient()) + ++Count; + + // If we found a match, return it. + if (isMatchingMovConstInsn(*I, MI, IndexReg, Offset)) { + return MBBI; + } + + // Update the status of what the instruction clobbered and used. + LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI); + + // Otherwise, if the index register is used or modified, we have no match, + // so return early. + if (!ModifiedRegUnits.available(IndexReg) || + !UsedRegUnits.available(IndexReg)) + return E; + + } while (MBBI != B && Count < Limit); + return E; +} + bool AArch64LoadStoreOpt::tryToPromoteLoadFromStore( MachineBasicBlock::iterator &MBBI) { MachineInstr &MI = *MBBI; @@ -2404,6 +2636,34 @@ bool AArch64LoadStoreOpt::tryToMergeLdStUpdate return false; } +bool AArch64LoadStoreOpt::tryToMergeIndexLdSt(MachineBasicBlock::iterator &MBBI, + int Scale) { + MachineInstr &MI = *MBBI; + MachineBasicBlock::iterator E = MI.getParent()->end(); + MachineBasicBlock::iterator Update; + + // Don't know how to handle unscaled pre/post-index versions below, so bail. + if (TII->hasUnscaledLdStOffset(MI.getOpcode())) + return false; + + // Look back to try to find a const offset for index LdSt instruction. For + // example, + // mov x8, #LargeImm ; = a * (1<<12) + imm12 + // ldr x1, [x0, x8] + // merged into: + // add x8, x0, a * (1<<12) + // ldr x1, [x8, imm12] + unsigned Offset; + Update = findMatchingConstOffsetBackward(MBBI, LdStConstLimit, Offset); + if (Update != E && (Offset & (Scale - 1)) == 0) { + // Merge the imm12 into the ld/st. + MBBI = mergeConstOffsetInsn(MBBI, Update, Offset, Scale); + return true; + } + + return false; +} + bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB, bool EnableNarrowZeroStOpt) { @@ -2482,6 +2742,22 @@ bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB, ++MBBI; } + // 5) Find a register assigned with a const value that can be combined with + // into the load or store. e.g., + // mov x8, #LargeImm ; = a * (1<<12) + imm12 + // ldr x1, [x0, x8] + // ; becomes + // add x8, x0, a * (1<<12) + // ldr x1, [x8, imm12] + for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); + MBBI != E;) { + int Scale; + if (isMergeableIndexLdSt(*MBBI, Scale) && tryToMergeIndexLdSt(MBBI, Scale)) + Modified = true; + else + ++MBBI; + } + return Modified; } diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp index 9da59ef2a806..1a8c71888a85 100644 --- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp @@ -93,16 +93,24 @@ AArch64FunctionInfo::AArch64FunctionInfo(const Function &F, // TODO: skip functions that have no instrumented allocas for optimization IsMTETagged = F.hasFnAttribute(Attribute::SanitizeMemTag); - if (!F.hasFnAttribute("branch-target-enforcement")) { - if (const auto *BTE = mdconst::extract_or_null<ConstantInt>( - F.getParent()->getModuleFlag("branch-target-enforcement"))) - BranchTargetEnforcement = BTE->getZExtValue(); - } else { - const StringRef BTIEnable = - F.getFnAttribute("branch-target-enforcement").getValueAsString(); - assert(BTIEnable == "true" || BTIEnable == "false"); - BranchTargetEnforcement = BTIEnable == "true"; - } + // BTI/PAuthLR may be set either on the function or the module. Set Bool from + // either the function attribute or module attribute, depending on what is + // set. + // Note: the module attributed is numeric (0 or 1) but the function attribute + // is stringy ("true" or "false"). + auto TryFnThenModule = [&](StringRef AttrName, bool &Bool) { + if (F.hasFnAttribute(AttrName)) { + const StringRef V = F.getFnAttribute(AttrName).getValueAsString(); + assert(V.equals_insensitive("true") || V.equals_insensitive("false")); + Bool = V.equals_insensitive("true"); + } else if (const auto *ModVal = mdconst::extract_or_null<ConstantInt>( + F.getParent()->getModuleFlag(AttrName))) { + Bool = ModVal->getZExtValue(); + } + }; + + TryFnThenModule("branch-target-enforcement", BranchTargetEnforcement); + TryFnThenModule("branch-protection-pauth-lr", BranchProtectionPAuthLR); // The default stack probe size is 4096 if the function has no // stack-probe-size attribute. This is a safe default because it is the diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h index 219f83cfd32e..cd4a18bfbc23 100644 --- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h +++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h @@ -22,6 +22,7 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/IR/Function.h" #include "llvm/MC/MCLinkerOptimizationHint.h" +#include "llvm/MC/MCSymbol.h" #include <cassert> #include <optional> @@ -164,10 +165,21 @@ class AArch64FunctionInfo final : public MachineFunctionInfo { /// SignWithBKey modifies the default PAC-RET mode to signing with the B key. bool SignWithBKey = false; + /// SigningInstrOffset captures the offset of the PAC-RET signing instruction + /// within the prologue, so it can be re-used for authentication in the + /// epilogue when using PC as a second salt (FEAT_PAuth_LR) + MCSymbol *SignInstrLabel = nullptr; + /// BranchTargetEnforcement enables placing BTI instructions at potential /// indirect branch destinations. bool BranchTargetEnforcement = false; + /// Indicates that SP signing should be diversified with PC as-per PAuthLR. + /// This is set by -mbranch-protection and will emit NOP instructions unless + /// the subtarget feature +pauthlr is also used (in which case non-NOP + /// instructions are emitted). + bool BranchProtectionPAuthLR = false; + /// Whether this function has an extended frame record [Ctx, FP, LR]. If so, /// bit 60 of the in-memory FP will be 1 to enable other tools to detect the /// extended record. @@ -436,10 +448,16 @@ public: bool needsShadowCallStackPrologueEpilogue(MachineFunction &MF) const; bool shouldSignWithBKey() const { return SignWithBKey; } + + MCSymbol *getSigningInstrLabel() const { return SignInstrLabel; } + void setSigningInstrLabel(MCSymbol *Label) { SignInstrLabel = Label; } + bool isMTETagged() const { return IsMTETagged; } bool branchTargetEnforcement() const { return BranchTargetEnforcement; } + bool branchProtectionPAuthLR() const { return BranchProtectionPAuthLR; } + void setHasSwiftAsyncContext(bool HasContext) { HasSwiftAsyncContext = HasContext; } diff --git a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp index 7576d2a899d1..7509afaeb5fe 100644 --- a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp +++ b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp @@ -60,11 +60,35 @@ FunctionPass *llvm::createAArch64PointerAuthPass() { char AArch64PointerAuth::ID = 0; +// Where PAuthLR support is not known at compile time, it is supported using +// PACM. PACM is in the hint space so has no effect when PAuthLR is not +// supported by the hardware, but will alter the behaviour of PACI*SP, AUTI*SP +// and RETAA/RETAB if the hardware supports PAuthLR. +static void BuildPACM(const AArch64Subtarget &Subtarget, MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, DebugLoc DL, + MachineInstr::MIFlag Flags, MCSymbol *PACSym = nullptr) { + const TargetInstrInfo *TII = Subtarget.getInstrInfo(); + auto &MFnI = *MBB.getParent()->getInfo<AArch64FunctionInfo>(); + + // ADR X16,<address_of_PACIASP> + if (PACSym) { + assert(Flags == MachineInstr::FrameDestroy); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADR)) + .addReg(AArch64::X16, RegState::Define) + .addSym(PACSym); + } + + // Only emit PACM if -mbranch-protection has +pc and the target does not + // have feature +pauth-lr. + if (MFnI.branchProtectionPAuthLR() && !Subtarget.hasPAuthLR()) + BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACM)).setMIFlag(Flags); +} + void AArch64PointerAuth::signLR(MachineFunction &MF, MachineBasicBlock::iterator MBBI) const { - const AArch64FunctionInfo *MFnI = MF.getInfo<AArch64FunctionInfo>(); - bool UseBKey = MFnI->shouldSignWithBKey(); - bool EmitCFI = MFnI->needsDwarfUnwindInfo(MF); + auto &MFnI = *MF.getInfo<AArch64FunctionInfo>(); + bool UseBKey = MFnI.shouldSignWithBKey(); + bool EmitCFI = MFnI.needsDwarfUnwindInfo(MF); bool NeedsWinCFI = MF.hasWinCFI(); MachineBasicBlock &MBB = *MBBI->getParent(); @@ -77,11 +101,29 @@ void AArch64PointerAuth::signLR(MachineFunction &MF, .setMIFlag(MachineInstr::FrameSetup); } + // PAuthLR authentication instructions need to know the value of PC at the + // point of signing (PACI*). + if (MFnI.branchProtectionPAuthLR()) { + MCSymbol *PACSym = MF.getMMI().getContext().createTempSymbol(); + MFnI.setSigningInstrLabel(PACSym); + } + // No SEH opcode for this one; it doesn't materialize into an // instruction on Windows. - BuildMI(MBB, MBBI, DL, - TII->get(UseBKey ? AArch64::PACIBSP : AArch64::PACIASP)) - .setMIFlag(MachineInstr::FrameSetup); + if (MFnI.branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) { + BuildMI(MBB, MBBI, DL, + TII->get(MFnI.shouldSignWithBKey() ? AArch64::PACIBSPPC + : AArch64::PACIASPPC)) + .setMIFlag(MachineInstr::FrameSetup) + ->setPreInstrSymbol(MF, MFnI.getSigningInstrLabel()); + } else { + BuildPACM(*Subtarget, MBB, MBBI, DL, MachineInstr::FrameSetup); + BuildMI(MBB, MBBI, DL, + TII->get(MFnI.shouldSignWithBKey() ? AArch64::PACIBSP + : AArch64::PACIASP)) + .setMIFlag(MachineInstr::FrameSetup) + ->setPreInstrSymbol(MF, MFnI.getSigningInstrLabel()); + } if (EmitCFI) { unsigned CFIIndex = @@ -118,15 +160,37 @@ void AArch64PointerAuth::authenticateLR( // DW_CFA_AARCH64_negate_ra_state can't be emitted. bool TerminatorIsCombinable = TI != MBB.end() && TI->getOpcode() == AArch64::RET; + MCSymbol *PACSym = MFnI->getSigningInstrLabel(); + if (Subtarget->hasPAuth() && TerminatorIsCombinable && !NeedsWinCFI && !MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) { - unsigned CombinedRetOpcode = UseBKey ? AArch64::RETAB : AArch64::RETAA; - BuildMI(MBB, TI, DL, TII->get(CombinedRetOpcode)).copyImplicitOps(*TI); + if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) { + assert(PACSym && "No PAC instruction to refer to"); + BuildMI(MBB, TI, DL, + TII->get(UseBKey ? AArch64::RETABSPPCi : AArch64::RETAASPPCi)) + .addSym(PACSym) + .copyImplicitOps(*MBBI) + .setMIFlag(MachineInstr::FrameDestroy); + } else { + BuildPACM(*Subtarget, MBB, TI, DL, MachineInstr::FrameDestroy, PACSym); + BuildMI(MBB, TI, DL, TII->get(UseBKey ? AArch64::RETAB : AArch64::RETAA)) + .copyImplicitOps(*MBBI) + .setMIFlag(MachineInstr::FrameDestroy); + } MBB.erase(TI); } else { - unsigned AutOpcode = UseBKey ? AArch64::AUTIBSP : AArch64::AUTIASP; - BuildMI(MBB, MBBI, DL, TII->get(AutOpcode)) - .setMIFlag(MachineInstr::FrameDestroy); + if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) { + assert(PACSym && "No PAC instruction to refer to"); + BuildMI(MBB, MBBI, DL, + TII->get(UseBKey ? AArch64::AUTIBSPPCi : AArch64::AUTIASPPCi)) + .addSym(PACSym) + .setMIFlag(MachineInstr::FrameDestroy); + } else { + BuildPACM(*Subtarget, MBB, MBBI, DL, MachineInstr::FrameDestroy, PACSym); + BuildMI(MBB, MBBI, DL, + TII->get(UseBKey ? AArch64::AUTIBSP : AArch64::AUTIASP)) + .setMIFlag(MachineInstr::FrameDestroy); + } if (EmitAsyncCFI) { unsigned CFIIndex = diff --git a/llvm/lib/Target/AArch64/AArch64SchedA64FX.td b/llvm/lib/Target/AArch64/AArch64SchedA64FX.td index 813b4a3affcf..7edce4b61605 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedA64FX.td +++ b/llvm/lib/Target/AArch64/AArch64SchedA64FX.td @@ -22,7 +22,7 @@ def A64FXModel : SchedMachineModel { list<Predicate> UnsupportedFeatures = !listconcat(SMEUnsupported.F, SVEUnsupported.F, [HasMTE, HasMatMulInt8, HasBF16, - HasPAuth, HasCPA]); + HasPAuth, HasPAuthLR, HasCPA]); let FullInstRWOverlapCheck = 0; } diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td index 53cf725f0e23..a6fab5e6245f 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td +++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td @@ -19,7 +19,7 @@ def NeoverseN2Model : SchedMachineModel { let CompleteModel = 1; list<Predicate> UnsupportedFeatures = !listconcat(SMEUnsupported.F, - [HasSVE2p1, HasCPA]); + [HasSVE2p1, HasPAuthLR, HasCPA]); } //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp index 74afa4183e67..38a92cb09602 100644 --- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp +++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp @@ -1696,6 +1696,21 @@ public: return DiagnosticPredicateTy::Match; } + bool isPAuthPCRelLabel16Operand() const { + // PAuth PCRel16 operands are similar to regular branch targets, but only + // negative values are allowed for concrete immediates as signing instr + // should be in a lower address. + if (!isImm()) + return false; + const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm()); + if (!MCE) + return true; + int64_t Val = MCE->getValue(); + if (Val & 0b11) + return false; + return (Val <= 0) && (Val > -(1 << 18)); + } + void addExpr(MCInst &Inst, const MCExpr *Expr) const { // Add as immediates when possible. Null MCExpr = 0. if (!Expr) @@ -1997,6 +2012,19 @@ public: Inst.addOperand(MCOperand::createImm(MCE->getValue() >> 2)); } + void addPAuthPCRelLabel16Operands(MCInst &Inst, unsigned N) const { + // PC-relative operands don't encode the low bits, so shift them off + // here. If it's a label, however, just put it on directly as there's + // not enough information now to do anything. + assert(N == 1 && "Invalid number of operands!"); + const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm()); + if (!MCE) { + addExpr(Inst, getImm()); + return; + } + Inst.addOperand(MCOperand::createImm(MCE->getValue() >> 2)); + } + void addPCRelLabel19Operands(MCInst &Inst, unsigned N) const { // Branch operands don't encode the low bits, so shift them off // here. If it's a label, however, just put it on directly as there's diff --git a/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp b/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp index cf2d3879292d..e3220d103ae0 100644 --- a/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp +++ b/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp @@ -165,6 +165,9 @@ static DecodeStatus DecodeFixedPointScaleImm32(MCInst &Inst, unsigned Imm, static DecodeStatus DecodeFixedPointScaleImm64(MCInst &Inst, unsigned Imm, uint64_t Address, const MCDisassembler *Decoder); +static DecodeStatus DecodePCRelLabel16(MCInst &Inst, unsigned Imm, + uint64_t Address, + const MCDisassembler *Decoder); static DecodeStatus DecodePCRelLabel19(MCInst &Inst, unsigned Imm, uint64_t Address, const MCDisassembler *Decoder); @@ -887,6 +890,21 @@ static DecodeStatus DecodeFixedPointScaleImm64(MCInst &Inst, unsigned Imm, return Success; } +static DecodeStatus DecodePCRelLabel16(MCInst &Inst, unsigned Imm, + uint64_t Addr, + const MCDisassembler *Decoder) { + // Immediate is encoded as the top 16-bits of an unsigned 18-bit negative + // PC-relative offset. + uint64_t ImmVal = Imm; + if (ImmVal < 0 || ImmVal > (1 << 16)) + return Fail; + ImmVal = -ImmVal; + if (!Decoder->tryAddingSymbolicOperand(Inst, (ImmVal << 2), Addr, + /*IsBranch=*/false, 0, 0, 4)) + Inst.addOperand(MCOperand::createImm(ImmVal)); + return Success; +} + static DecodeStatus DecodePCRelLabel19(MCInst &Inst, unsigned Imm, uint64_t Addr, const MCDisassembler *Decoder) { diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp index 687063873a16..830203b61c58 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp @@ -769,6 +769,27 @@ void applyDupLane(MachineInstr &MI, MachineRegisterInfo &MRI, MI.eraseFromParent(); } +bool matchScalarizeVectorUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI) { + auto &Unmerge = cast<GUnmerge>(MI); + Register Src1Reg = Unmerge.getReg(Unmerge.getNumOperands() - 1); + const LLT SrcTy = MRI.getType(Src1Reg); + return SrcTy.isVector() && !SrcTy.isScalable() && + Unmerge.getNumOperands() == (unsigned)SrcTy.getNumElements() + 1; +} + +void applyScalarizeVectorUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &B) { + auto &Unmerge = cast<GUnmerge>(MI); + Register Src1Reg = Unmerge.getReg(Unmerge.getNumOperands() - 1); + const LLT SrcTy = MRI.getType(Src1Reg); + assert((SrcTy.isVector() && !SrcTy.isScalable()) && + "Expected a fixed length vector"); + + for (int I = 0; I < SrcTy.getNumElements(); ++I) + B.buildExtractVectorElementConstant(Unmerge.getReg(I), Src1Reg, I); + MI.eraseFromParent(); +} + bool matchBuildVectorToDup(MachineInstr &MI, MachineRegisterInfo &MRI) { assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR); auto Splat = getAArch64VectorSplat(MI, MRI); diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp index a6900b8963bb..30ef3680ae79 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp @@ -67,6 +67,7 @@ public: {"fixup_aarch64_ldr_pcrel_imm19", 5, 19, PCRelFlagVal}, {"fixup_aarch64_movw", 5, 16, 0}, {"fixup_aarch64_pcrel_branch14", 5, 14, PCRelFlagVal}, + {"fixup_aarch64_pcrel_branch16", 5, 16, PCRelFlagVal}, {"fixup_aarch64_pcrel_branch19", 5, 19, PCRelFlagVal}, {"fixup_aarch64_pcrel_branch26", 0, 26, PCRelFlagVal}, {"fixup_aarch64_pcrel_call26", 0, 26, PCRelFlagVal}}; @@ -121,6 +122,7 @@ static unsigned getFixupKindNumBytes(unsigned Kind) { case AArch64::fixup_aarch64_movw: case AArch64::fixup_aarch64_pcrel_branch14: + case AArch64::fixup_aarch64_pcrel_branch16: case AArch64::fixup_aarch64_add_imm12: case AArch64::fixup_aarch64_ldst_imm12_scale1: case AArch64::fixup_aarch64_ldst_imm12_scale2: @@ -314,6 +316,17 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target, if (Value & 0x3) Ctx.reportError(Fixup.getLoc(), "fixup not sufficiently aligned"); return (Value >> 2) & 0x3fff; + case AArch64::fixup_aarch64_pcrel_branch16: + // Unsigned PC-relative offset, so invert the negative immediate. + SignedValue = -SignedValue; + Value = static_cast<uint64_t>(SignedValue); + // Check valid 18-bit unsigned range. + if (SignedValue < 0 || SignedValue > ((1 << 18) - 1)) + Ctx.reportError(Fixup.getLoc(), "fixup value out of range"); + // Low two bits are not encoded (4-byte alignment assumed). + if (Value & 0b11) + Ctx.reportError(Fixup.getLoc(), "fixup not sufficiently aligned"); + return (Value >> 2) & 0xffff; case AArch64::fixup_aarch64_pcrel_branch26: case AArch64::fixup_aarch64_pcrel_call26: if (TheTriple.isOSBinFormatCOFF() && !IsResolved && SignedValue != 0) { @@ -380,6 +393,7 @@ unsigned AArch64AsmBackend::getFixupKindContainereSizeInBytes(unsigned Kind) con case AArch64::fixup_aarch64_movw: case AArch64::fixup_aarch64_pcrel_branch14: + case AArch64::fixup_aarch64_pcrel_branch16: case AArch64::fixup_aarch64_add_imm12: case AArch64::fixup_aarch64_ldst_imm12_scale1: case AArch64::fixup_aarch64_ldst_imm12_scale2: diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp index 9de40661298c..496ab18e9b19 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp @@ -186,6 +186,10 @@ unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx, return R_CLS(LD_PREL_LO19); case AArch64::fixup_aarch64_pcrel_branch14: return R_CLS(TSTBR14); + case AArch64::fixup_aarch64_pcrel_branch16: + Ctx.reportError(Fixup.getLoc(), + "relocation of PAC/AUT instructions is not supported"); + return ELF::R_AARCH64_NONE; case AArch64::fixup_aarch64_pcrel_branch19: return R_CLS(CONDBR19); default: diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64FixupKinds.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64FixupKinds.h index 767dd8805520..fdee2d5ad2bf 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64FixupKinds.h +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64FixupKinds.h @@ -43,6 +43,11 @@ enum Fixups { // The high 14 bits of a 21-bit pc-relative immediate. fixup_aarch64_pcrel_branch14, + // The high 16 bits of a 18-bit unsigned PC-relative immediate. Used by + // pointer authentication, only within a function, so no relocation can be + // generated. + fixup_aarch64_pcrel_branch16, + // The high 19 bits of a 21-bit pc-relative immediate. Same encoding as // fixup_aarch64_pcrel_adrhi, except this is use by b.cc and generates // relocations directly when necessary. diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp index dbc4323a860f..c3e12b6d8024 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp @@ -88,6 +88,12 @@ public: SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const; + /// getPAuthPCRelOpValue - Return the encoded value for a pointer + /// authentication pc-relative operand. + uint32_t getPAuthPCRelOpValue(const MCInst &MI, unsigned OpIdx, + SmallVectorImpl<MCFixup> &Fixups, + const MCSubtargetInfo &STI) const; + /// getLoadLiteralOpValue - Return the encoded value for a load-literal /// pc-relative address. uint32_t getLoadLiteralOpValue(const MCInst &MI, unsigned OpIdx, @@ -327,6 +333,29 @@ uint32_t AArch64MCCodeEmitter::getCondBranchTargetOpValue( return 0; } +/// getPAuthPCRelOpValue - Return the encoded value for a pointer +/// authentication pc-relative operand. +uint32_t +AArch64MCCodeEmitter::getPAuthPCRelOpValue(const MCInst &MI, unsigned OpIdx, + SmallVectorImpl<MCFixup> &Fixups, + const MCSubtargetInfo &STI) const { + const MCOperand &MO = MI.getOperand(OpIdx); + + // If the destination is an immediate, invert sign as it's a negative value + // that should be encoded as unsigned + if (MO.isImm()) + return -(MO.getImm()); + assert(MO.isExpr() && "Unexpected target type!"); + + MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_pcrel_branch16); + Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc())); + + ++MCNumFixups; + + // All of the information is in the fixup. + return 0; +} + /// getLoadLiteralOpValue - Return the encoded value for a load-literal /// pc-relative address. uint32_t diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td index 060fb66d38f7..d2a325d5ad89 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.td +++ b/llvm/lib/Target/AMDGPU/AMDGPU.td @@ -1100,8 +1100,8 @@ def FeatureGFX12 : GCNSubtargetFeatureGeneration<"GFX12", FeatureVOP3Literal, FeatureDPP8, FeatureNoDataDepHazard, FeaturePkFmacF16Inst, FeatureA16, FeatureFastDenormalF32, FeatureG16, - FeatureUnalignedBufferAccess, FeatureUnalignedDSAccess, FeatureGDS, - FeatureGWS, FeatureTrue16BitInsts + FeatureUnalignedBufferAccess, FeatureUnalignedDSAccess, + FeatureTrue16BitInsts ] >; diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td index 3a895923fa4b..bc9049b4ef33 100644 --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -1147,7 +1147,8 @@ def : GCNPat < >; } // End SubtargetPredicate = HasAtomicDsPkAdd16Insts -def : Pat < +let OtherPredicates = [HasGDS] in +def : GCNPat < (SIds_ordered_count i32:$value, i16:$offset), (DS_ORDERED_COUNT $value, (as_i16imm $offset)) >; @@ -1189,7 +1190,8 @@ def : GCNPat < //===----------------------------------------------------------------------===// class Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<bits<8> op, DS_Pseudo ps, int ef, - string opName = ps.Mnemonic> + string opName = ps.Mnemonic, + bit hasGFX12Enc = 0> : DS_Real<ps, opName>, SIMCInstr <ps.Mnemonic, ef> { let Inst{7-0} = !if(ps.has_offset0, offset0, 0); @@ -1201,6 +1203,8 @@ class Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<bits<8> op, DS_Pseudo ps, int ef, let Inst{47-40} = !if(ps.has_data0, data0{7-0}, 0); let Inst{55-48} = !if(ps.has_data1, data1{7-0}, 0); let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, 0); + + let gds = !if(hasGFX12Enc, 0, ?); } //===----------------------------------------------------------------------===// @@ -1212,7 +1216,7 @@ let AssemblerPredicate = isGFX12Plus, DecoderNamespace = "GFX12" in { defvar ps = !cast<DS_Pseudo>(NAME); def _gfx12 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op, ps, SIEncodingFamily.GFX12, - ps.Mnemonic>; + ps.Mnemonic, 1>; } multiclass DS_Real_Renamed_gfx12<bits<8> op, DS_Pseudo backing_pseudo, @@ -1220,7 +1224,7 @@ let AssemblerPredicate = isGFX12Plus, DecoderNamespace = "GFX12" in { def _gfx12 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op, backing_pseudo, SIEncodingFamily.GFX12, - real_name>, + real_name, 1>, MnemonicAlias<backing_pseudo.Mnemonic, real_name>, Requires<[isGFX12Plus]>; } diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp index ed2e7e4f189e..7939d0036568 100644 --- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp +++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp @@ -702,6 +702,11 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size, AMDGPU::OpName::src2_modifiers); } + if (Res && (MCII->get(MI.getOpcode()).TSFlags & SIInstrFlags::DS) && + !AMDGPU::hasGDS(STI)) { + insertNamedMCOperand(MI, MCOperand::createImm(0), AMDGPU::OpName::gds); + } + if (Res && (MCII->get(MI.getOpcode()).TSFlags & (SIInstrFlags::MUBUF | SIInstrFlags::FLAT | SIInstrFlags::SMRD))) { int CPolPos = AMDGPU::getNamedOperandIdx(MI.getOpcode(), diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index e599f23101c8..29ac08b6895e 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -4983,6 +4983,14 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI, } } + if (isDS(MI) && !ST.hasGDS()) { + const MachineOperand *GDSOp = getNamedOperand(MI, AMDGPU::OpName::gds); + if (GDSOp && GDSOp->getImm() != 0) { + ErrInfo = "GDS is not supported on this subtarget"; + return false; + } + } + if (isImage(MI)) { const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim); if (DimOp) { diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index de15bea72e46..d6dedd669ffd 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -675,7 +675,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND, ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN, ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX, - ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE}; + ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE}; static const unsigned FloatingPointVPOps[] = { ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL, @@ -688,7 +688,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS, - ISD::EXPERIMENTAL_VP_REVERSE}; + ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE}; static const unsigned IntegerVecReduceOps[] = { ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, @@ -773,6 +773,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setOperationAction(ISD::VECTOR_REVERSE, VT, Custom); + setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom); setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom); setOperationPromotedToType( @@ -1147,6 +1148,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ISD::VP_SETCC, ISD::VP_TRUNCATE}, VT, Custom); + setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom); setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom); continue; } @@ -6637,6 +6639,8 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, !Subtarget.hasVInstructionsF16())) return SplitVPOp(Op, DAG); return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget); + case ISD::EXPERIMENTAL_VP_SPLICE: + return lowerVPSpliceExperimental(Op, DAG); case ISD::EXPERIMENTAL_VP_REVERSE: return lowerVPReverseExperimental(Op, DAG); } @@ -10583,6 +10587,87 @@ SDValue RISCVTargetLowering::lowerVPFPIntConvOp(SDValue Op, } SDValue +RISCVTargetLowering::lowerVPSpliceExperimental(SDValue Op, + SelectionDAG &DAG) const { + SDLoc DL(Op); + + SDValue Op1 = Op.getOperand(0); + SDValue Op2 = Op.getOperand(1); + SDValue Offset = Op.getOperand(2); + SDValue Mask = Op.getOperand(3); + SDValue EVL1 = Op.getOperand(4); + SDValue EVL2 = Op.getOperand(5); + + const MVT XLenVT = Subtarget.getXLenVT(); + MVT VT = Op.getSimpleValueType(); + MVT ContainerVT = VT; + if (VT.isFixedLengthVector()) { + ContainerVT = getContainerForFixedLengthVector(VT); + Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget); + Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget); + MVT MaskVT = getMaskTypeFor(ContainerVT); + Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); + } + + bool IsMaskVector = VT.getVectorElementType() == MVT::i1; + if (IsMaskVector) { + ContainerVT = ContainerVT.changeVectorElementType(MVT::i8); + + // Expand input operands + SDValue SplatOneOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, + DAG.getUNDEF(ContainerVT), + DAG.getConstant(1, DL, XLenVT), EVL1); + SDValue SplatZeroOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, + DAG.getUNDEF(ContainerVT), + DAG.getConstant(0, DL, XLenVT), EVL1); + Op1 = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, Op1, SplatOneOp1, + SplatZeroOp1, EVL1); + + SDValue SplatOneOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, + DAG.getUNDEF(ContainerVT), + DAG.getConstant(1, DL, XLenVT), EVL2); + SDValue SplatZeroOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, + DAG.getUNDEF(ContainerVT), + DAG.getConstant(0, DL, XLenVT), EVL2); + Op2 = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, Op2, SplatOneOp2, + SplatZeroOp2, EVL2); + } + + int64_t ImmValue = cast<ConstantSDNode>(Offset)->getSExtValue(); + SDValue DownOffset, UpOffset; + if (ImmValue >= 0) { + // The operand is a TargetConstant, we need to rebuild it as a regular + // constant. + DownOffset = DAG.getConstant(ImmValue, DL, XLenVT); + UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, DownOffset); + } else { + // The operand is a TargetConstant, we need to rebuild it as a regular + // constant rather than negating the original operand. + UpOffset = DAG.getConstant(-ImmValue, DL, XLenVT); + DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, UpOffset); + } + + SDValue SlideDown = + getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT), + Op1, DownOffset, Mask, UpOffset); + SDValue Result = getVSlideup(DAG, Subtarget, DL, ContainerVT, SlideDown, Op2, + UpOffset, Mask, EVL2, RISCVII::TAIL_AGNOSTIC); + + if (IsMaskVector) { + // Truncate Result back to a mask vector (Result has same EVL as Op2) + Result = DAG.getNode( + RISCVISD::SETCC_VL, DL, ContainerVT.changeVectorElementType(MVT::i1), + {Result, DAG.getConstant(0, DL, ContainerVT), + DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)), + Mask, EVL2}); + } + + if (!VT.isFixedLengthVector()) + return Result; + return convertFromScalableVector(VT, Result, DAG, Subtarget); +} + +SDValue RISCVTargetLowering::lowerVPReverseExperimental(SDValue Op, SelectionDAG &DAG) const { SDLoc DL(Op); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h index 41a2dc5771c8..2d9f716cdf9a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.h +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h @@ -910,6 +910,7 @@ private: SDValue lowerLogicVPOp(SDValue Op, SelectionDAG &DAG) const; SDValue lowerVPExtMaskOp(SDValue Op, SelectionDAG &DAG) const; SDValue lowerVPSetCCMaskOp(SDValue Op, SelectionDAG &DAG) const; + SDValue lowerVPSpliceExperimental(SDValue Op, SelectionDAG &DAG) const; SDValue lowerVPReverseExperimental(SDValue Op, SelectionDAG &DAG) const; SDValue lowerVPFPIntConvOp(SDValue Op, SelectionDAG &DAG) const; SDValue lowerVPStridedLoad(SDValue Op, SelectionDAG &DAG) const; diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index b8a6784ff3c6..3a34a0bfae46 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -645,7 +645,7 @@ SPIRVType *SPIRVGlobalRegistry::findSPIRVType( Register Reg = DT.find(Ty, &MIRBuilder.getMF()); if (Reg.isValid()) return getSPIRVTypeForVReg(Reg); - if (ForwardPointerTypes.find(Ty) != ForwardPointerTypes.end()) + if (ForwardPointerTypes.contains(Ty)) return ForwardPointerTypes[Ty]; return restOfCreateSPIRVType(Ty, MIRBuilder, AccQual, EmitIR); } @@ -712,14 +712,14 @@ SPIRVType *SPIRVGlobalRegistry::createSPIRVType( // Null pointer means we have a loop in type definitions, make and // return corresponding OpTypeForwardPointer. if (SpvElementType == nullptr) { - if (ForwardPointerTypes.find(Ty) == ForwardPointerTypes.end()) + if (!ForwardPointerTypes.contains(Ty)) ForwardPointerTypes[PType] = getOpTypeForwardPointer(SC, MIRBuilder); return ForwardPointerTypes[PType]; } Register Reg(0); // If we have forward pointer associated with this type, use its register // operand to create OpTypePointer. - if (ForwardPointerTypes.find(PType) != ForwardPointerTypes.end()) + if (ForwardPointerTypes.contains(PType)) Reg = getSPIRVTypeID(ForwardPointerTypes[PType]); return getOpTypePointer(SC, SpvElementType, MIRBuilder, Reg); diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp index 779036016560..2a830535a2aa 100644 --- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp @@ -391,7 +391,7 @@ void SPIRVModuleAnalysis::numberRegistersGlobally(const Module &M) { if (MI.getOpcode() != SPIRV::OpExtInst) continue; auto Set = MI.getOperand(2).getImm(); - if (MAI.ExtInstSetMap.find(Set) == MAI.ExtInstSetMap.end()) + if (!MAI.ExtInstSetMap.contains(Set)) MAI.ExtInstSetMap[Set] = Register::index2VirtReg(MAI.getNextID()); } } diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h index 5124181b49e2..d0b8027edd42 100644 --- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h +++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h @@ -189,7 +189,7 @@ struct ModuleAnalysisInfo { } unsigned getNextID() { return MaxID++; } bool hasMBBRegister(const MachineBasicBlock &MBB) { - return BBNumToRegMap.find(MBB.getNumber()) != BBNumToRegMap.end(); + return BBNumToRegMap.contains(MBB.getNumber()); } // Convert MBB's number to corresponding ID register. Register getOrCreateMBBRegister(const MachineBasicBlock &MBB) { diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index f4076be2a7b7..1bfce70fedc0 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -83,7 +83,7 @@ static void addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR) { } for (MachineInstr *MI : ToErase) { Register Reg = MI->getOperand(2).getReg(); - if (RegsAlreadyAddedToDT.find(MI) != RegsAlreadyAddedToDT.end()) + if (RegsAlreadyAddedToDT.contains(MI)) Reg = RegsAlreadyAddedToDT[MI]; auto *RC = MRI.getRegClassOrNull(MI->getOperand(0).getReg()); if (!MRI.getRegClassOrNull(Reg) && RC) diff --git a/llvm/lib/Target/TargetMachine.cpp b/llvm/lib/Target/TargetMachine.cpp index 3f96bd37755e..2a4383314e46 100644 --- a/llvm/lib/Target/TargetMachine.cpp +++ b/llvm/lib/Target/TargetMachine.cpp @@ -61,8 +61,6 @@ bool TargetMachine::isLargeGlobalValue(const GlobalValue *GVal) const { // We should properly mark well-known section name prefixes as small/large, // because otherwise the output section may have the wrong section flags and // the linker will lay it out in an unexpected way. - // TODO: bring back lbss/ldata/lrodata checks after fixing accesses to large - // globals in the small code model. StringRef Name = GV->getSection(); if (!Name.empty()) { auto IsPrefix = [&](StringRef Prefix) { @@ -71,6 +69,8 @@ bool TargetMachine::isLargeGlobalValue(const GlobalValue *GVal) const { }; if (IsPrefix(".bss") || IsPrefix(".data") || IsPrefix(".rodata")) return false; + if (IsPrefix(".lbss") || IsPrefix(".ldata") || IsPrefix(".lrodata")) + return true; } // For x86-64, we treat an explicit GlobalVariable small code model to mean diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp index 7ec59c74f5f5..77a997588c4f 100644 --- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp +++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp @@ -1828,9 +1828,7 @@ bool X86DAGToDAGISel::matchWrapper(SDValue N, X86ISelAddressMode &AM) { // That signifies access to globals that are known to be "near", // such as the GOT itself. CodeModel::Model M = TM.getCodeModel(); - if (Subtarget->is64Bit() && - ((M == CodeModel::Large && !IsRIPRelTLS) || - (M == CodeModel::Medium && !IsRIPRel))) + if (Subtarget->is64Bit() && M == CodeModel::Large && !IsRIPRelTLS) return true; // Base and index reg must be 0 in order to use %rip as base. @@ -1866,6 +1864,13 @@ bool X86DAGToDAGISel::matchWrapper(SDValue N, X86ISelAddressMode &AM) { } else llvm_unreachable("Unhandled symbol reference node."); + // Can't use an addressing mode with large globals. + if (Subtarget->is64Bit() && !IsRIPRel && AM.GV && + TM.isLargeGlobalValue(AM.GV)) { + AM = Backup; + return true; + } + if (foldOffsetIntoAddress(Offset, AM)) { AM = Backup; return true; @@ -1910,20 +1915,12 @@ bool X86DAGToDAGISel::matchAddress(SDValue N, X86ISelAddressMode &AM) { // Post-processing: Convert foo to foo(%rip), even in non-PIC mode, // because it has a smaller encoding. - // TODO: Which other code models can use this? - switch (TM.getCodeModel()) { - default: break; - case CodeModel::Small: - case CodeModel::Kernel: - if (Subtarget->is64Bit() && - AM.Scale == 1 && - AM.BaseType == X86ISelAddressMode::RegBase && - AM.Base_Reg.getNode() == nullptr && - AM.IndexReg.getNode() == nullptr && - AM.SymbolFlags == X86II::MO_NO_FLAG && - AM.hasSymbolicDisplacement()) - AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64); - break; + if (TM.getCodeModel() != CodeModel::Large && + (!AM.GV || !TM.isLargeGlobalValue(AM.GV)) && Subtarget->is64Bit() && + AM.Scale == 1 && AM.BaseType == X86ISelAddressMode::RegBase && + AM.Base_Reg.getNode() == nullptr && AM.IndexReg.getNode() == nullptr && + AM.SymbolFlags == X86II::MO_NO_FLAG && AM.hasSymbolicDisplacement()) { + AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64); } return false; diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index db5e4fe84f41..49112862a314 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -2674,34 +2674,33 @@ SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const { return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy(DAG.getDataLayout())); } -bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, - bool hasSymbolicDisplacement) { +bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model CM, + bool HasSymbolicDisplacement) { // Offset should fit into 32 bit immediate field. if (!isInt<32>(Offset)) return false; // If we don't have a symbolic displacement - we don't have any extra // restrictions. - if (!hasSymbolicDisplacement) + if (!HasSymbolicDisplacement) return true; - // FIXME: Some tweaks might be needed for medium code model. - if (M != CodeModel::Small && M != CodeModel::Kernel) - return false; - - // For small code model we assume that latest object is 16MB before end of 31 - // bits boundary. We may also accept pretty large negative constants knowing - // that all objects are in the positive half of address space. - if (M == CodeModel::Small && Offset < 16*1024*1024) + // We can fold large offsets in the large code model because we always use + // 64-bit offsets. + if (CM == CodeModel::Large) return true; // For kernel code model we know that all object resist in the negative half // of 32bits address space. We may not accept negative offsets, since they may // be just off and we may accept pretty large positive ones. - if (M == CodeModel::Kernel && Offset >= 0) - return true; - - return false; + if (CM == CodeModel::Kernel) + return Offset >= 0; + + // For other non-large code models we assume that latest small object is 16MB + // before end of 31 bits boundary. We may also accept pretty large negative + // constants knowing that all objects are in the positive half of address + // space. + return Offset < 16 * 1024 * 1024; } /// Return true if the condition is an signed comparison operation. diff --git a/llvm/lib/Target/X86/X86InstrArithmetic.td b/llvm/lib/Target/X86/X86InstrArithmetic.td index 46b430a842ef..87feb7dc3b4e 100644 --- a/llvm/lib/Target/X86/X86InstrArithmetic.td +++ b/llvm/lib/Target/X86/X86InstrArithmetic.td @@ -44,107 +44,69 @@ def PLEA32r : PseudoI<(outs GR32:$dst), (ins anymem:$src), []>; def PLEA64r : PseudoI<(outs GR64:$dst), (ins anymem:$src), []>; } -//===----------------------------------------------------------------------===// -// Fixed-Register Multiplication and Division Instructions. -// - -// BinOpRR - Binary instructions with inputs "reg, reg". -class BinOpRR<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - dag outlist, X86FoldableSchedWrite sched, list<dag> pattern> - : ITy<opcode, MRMDestReg, typeinfo, outlist, - (ins typeinfo.RegClass:$src1, typeinfo.RegClass:$src2), - mnemonic, "{$src2, $src1|$src1, $src2}", pattern>, - Sched<[sched]>; - -// BinOpRR_F - Binary instructions with inputs "reg, reg", where the pattern -// has just a EFLAGS as a result. -class BinOpRR_F<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - SDPatternOperator opnode> - : BinOpRR<opcode, mnemonic, typeinfo, (outs), WriteALU, - [(set EFLAGS, - (opnode typeinfo.RegClass:$src1, typeinfo.RegClass:$src2))]>; - -// BinOpRR_RF - Binary instructions with inputs "reg, reg", where the pattern -// has both a regclass and EFLAGS as a result. -class BinOpRR_RF<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - SDNode opnode> - : BinOpRR<opcode, mnemonic, typeinfo, (outs typeinfo.RegClass:$dst), WriteALU, - [(set typeinfo.RegClass:$dst, EFLAGS, - (opnode typeinfo.RegClass:$src1, typeinfo.RegClass:$src2))]>; - -// BinOpRR_RFF - Binary instructions with inputs "reg, reg", where the pattern -// has both a regclass and EFLAGS as a result, and has EFLAGS as input. -class BinOpRR_RFF<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - SDNode opnode> - : BinOpRR<opcode, mnemonic, typeinfo, (outs typeinfo.RegClass:$dst), WriteADC, - [(set typeinfo.RegClass:$dst, EFLAGS, - (opnode typeinfo.RegClass:$src1, typeinfo.RegClass:$src2, - EFLAGS))]>; - -// BinOpRR_Rev - Binary instructions with inputs "reg, reg"(reversed encoding). -class BinOpRR_Rev<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - X86FoldableSchedWrite sched = WriteALU> - : ITy<opcode, MRMSrcReg, typeinfo, - (outs typeinfo.RegClass:$dst), - (ins typeinfo.RegClass:$src1, typeinfo.RegClass:$src2), - mnemonic, "{$src2, $dst|$dst, $src2}", []>, - Sched<[sched]> { - // The disassembler should know about this, but not the asmparser. - let isCodeGenOnly = 1; - let ForceDisassemble = 1; - let hasSideEffects = 0; +// BinOpRR - Instructions that read "reg, reg". +class BinOpRR<bits<8> o, string m, X86TypeInfo t, dag out, list<dag> p> + : ITy<o, MRMDestReg, t, out, (ins t.RegClass:$src1, t.RegClass:$src2), m, + "{$src2, $src1|$src1, $src2}", p>, Sched<[WriteALU]>; +// BinOpRR_F - Instructions that read "reg, reg" and write EFLAGS only. +class BinOpRR_F<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> + : BinOpRR<o, m, t, (outs), + [(set EFLAGS, (node t.RegClass:$src1, t.RegClass:$src2))]>, + DefEFLAGS; +// BinOpRR_F_Rev - Reversed encoding of BinOpRR_F +class BinOpRR_F_Rev<bits<8> o, string m, X86TypeInfo t> + : BinOpRR_F<o, m, t, null_frag>, DisassembleOnly { + let Form = MRMSrcReg; } - -// BinOpRR_RFF_Rev - Binary instructions with inputs "reg, reg"(reversed -// encoding), with sched = WriteADC. -class BinOpRR_RFF_Rev<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo> - : BinOpRR_Rev<opcode, mnemonic, typeinfo, WriteADC>; - -// BinOpRR_F_Rev - Binary instructions with inputs "reg, reg"(reversed -// encoding), without outlist dag. -class BinOpRR_F_Rev<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo> - : ITy<opcode, MRMSrcReg, typeinfo, (outs), - (ins typeinfo.RegClass:$src1, typeinfo.RegClass:$src2), - mnemonic, "{$src2, $src1|$src1, $src2}", []>, - Sched<[WriteALU]> { - // The disassembler should know about this, but not the asmparser. - let isCodeGenOnly = 1; - let ForceDisassemble = 1; - let hasSideEffects = 0; +// BinOpRR_RF - Instructions that read "reg, reg", and write "reg", EFLAGS. +class BinOpRR_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> + : BinOpRR<o, m, t, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, + (node t.RegClass:$src1, t.RegClass:$src2))]>, DefEFLAGS; +// BinOpRR_RF_Rev - Reversed encoding of BinOpRR_RF. +class BinOpRR_RF_Rev<bits<8> o, string m, X86TypeInfo t> + : BinOpRR_RF<o, m, t, null_frag>, DisassembleOnly { + let Form = MRMSrcReg; +} +// BinOpRRF_RF - Instructions that read "reg, reg", write "reg" and read/write +// EFLAGS. +class BinOpRRF_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> + : BinOpRR<o, m, t, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, + (node t.RegClass:$src1, t.RegClass:$src2, + EFLAGS))]>, DefEFLAGS, UseEFLAGS { + let SchedRW = [WriteADC]; +} +// BinOpRRF_RF_Rev - Reversed encoding of BinOpRRF_RF +class BinOpRRF_RF_Rev<bits<8> o, string m, X86TypeInfo t> + : BinOpRRF_RF<o, m, t, null_frag>, DisassembleOnly { + let Form = MRMSrcReg; } -// BinOpRM - Binary instructions with inputs "reg, [mem]". -class BinOpRM<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - dag outlist, X86FoldableSchedWrite sched, list<dag> pattern> - : ITy<opcode, MRMSrcMem, typeinfo, outlist, - (ins typeinfo.RegClass:$src1, typeinfo.MemOperand:$src2), - mnemonic, "{$src2, $src1|$src1, $src2}", pattern>, - Sched<[sched.Folded, sched.ReadAfterFold]>; - -// BinOpRM_F - Binary instructions with inputs "reg, [mem]", where the pattern -// has just a EFLAGS as a result. -class BinOpRM_F<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - SDNode opnode> - : BinOpRM<opcode, mnemonic, typeinfo, (outs), WriteALU, - [(set EFLAGS, - (opnode typeinfo.RegClass:$src1, (typeinfo.LoadNode addr:$src2)))]>; - -// BinOpRM_RF - Binary instructions with inputs "reg, [mem]", where the pattern -// has both a regclass and EFLAGS as a result. -class BinOpRM_RF<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - SDNode opnode> - : BinOpRM<opcode, mnemonic, typeinfo, (outs typeinfo.RegClass:$dst), WriteALU, - [(set typeinfo.RegClass:$dst, EFLAGS, - (opnode typeinfo.RegClass:$src1, (typeinfo.LoadNode addr:$src2)))]>; - -// BinOpRM_RFF - Binary instructions with inputs "reg, [mem]", where the pattern -// has both a regclass and EFLAGS as a result, and has EFLAGS as input. -class BinOpRM_RFF<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - SDNode opnode> - : BinOpRM<opcode, mnemonic, typeinfo, (outs typeinfo.RegClass:$dst), WriteADC, - [(set typeinfo.RegClass:$dst, EFLAGS, - (opnode typeinfo.RegClass:$src1, (typeinfo.LoadNode addr:$src2), - EFLAGS))]> { +// BinOpRM - Instructions that read "reg, [mem]". +class BinOpRM<bits<8> o, string m, X86TypeInfo t, dag out, list<dag> p> + : ITy<o, MRMSrcMem, t, out, (ins t.RegClass:$src1, t.MemOperand:$src2), m, + "{$src2, $src1|$src1, $src2}", p>, + Sched<[WriteALU.Folded, WriteALU.ReadAfterFold]> { + let mayLoad = 1; +} +// BinOpRM_F - Instructions that read "reg, [mem]" and write EFLAGS only. +class BinOpRM_F<bits<8> o, string m, X86TypeInfo t, SDNode node> + : BinOpRM<o, m, t, (outs), + [(set EFLAGS, (node t.RegClass:$src1, + (t.LoadNode addr:$src2)))]>, DefEFLAGS; +// BinOpRM_RF - Instructions that read "reg, reg", and write "reg", EFLAGS. +class BinOpRM_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> + : BinOpRM<o, m, t, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, (node t.RegClass:$src1, + (t.LoadNode addr:$src2)))]>, DefEFLAGS; +// BinOpRMF_RF - Instructions that read "reg, [mem]", write "reg" and read/write +// EFLAGS. +class BinOpRMF_RF<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> + : BinOpRM<o, m, t, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, + (node t.RegClass:$src1, (t.LoadNode addr:$src2), EFLAGS))]>, + DefEFLAGS, UseEFLAGS { let SchedRW = [WriteADC.Folded, WriteADC.ReadAfterFold, // base, scale, index, offset, segment. ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault, @@ -152,395 +114,326 @@ class BinOpRM_RFF<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, WriteADC.ReadAfterFold]; } -// BinOpRI - Binary instructions with inputs "reg, imm". -class BinOpRI<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - Format f, dag outlist, X86FoldableSchedWrite sched, list<dag> pattern> - : ITy<opcode, f, typeinfo, outlist, - (ins typeinfo.RegClass:$src1, typeinfo.ImmOperand:$src2), - mnemonic, "{$src2, $src1|$src1, $src2}", pattern>, - Sched<[sched]> { - let ImmT = typeinfo.ImmEncoding; +// BinOpRI - Instructions that read "reg, imm". +class BinOpRI<bits<8> o, string m, X86TypeInfo t, Format f, dag out, list<dag> p> + : ITy<o, f, t, out, (ins t.RegClass:$src1, t.ImmOperand:$src2), m, + "{$src2, $src1|$src1, $src2}", p>, Sched<[WriteALU]> { + let ImmT = t.ImmEncoding; } - -// BinOpRI_F - Binary instructions with inputs "reg, imm", where the pattern -// has EFLAGS as a result. -class BinOpRI_F<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - SDPatternOperator opnode, Format f> - : BinOpRI<opcode, mnemonic, typeinfo, f, (outs), WriteALU, - [(set EFLAGS, - (opnode typeinfo.RegClass:$src1, typeinfo.ImmOperator:$src2))]>; - -// BinOpRI_RF - Binary instructions with inputs "reg, imm", where the pattern -// has both a regclass and EFLAGS as a result. -class BinOpRI_RF<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - SDNode opnode, Format f> - : BinOpRI<opcode, mnemonic, typeinfo, f, (outs typeinfo.RegClass:$dst), WriteALU, - [(set typeinfo.RegClass:$dst, EFLAGS, - (opnode typeinfo.RegClass:$src1, typeinfo.ImmOperator:$src2))]>; - -// BinOpRI_RFF - Binary instructions with inputs "reg, imm", where the pattern -// has both a regclass and EFLAGS as a result, and has EFLAGS as input. -class BinOpRI_RFF<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - SDNode opnode, Format f> - : BinOpRI<opcode, mnemonic, typeinfo, f, (outs typeinfo.RegClass:$dst), WriteADC, - [(set typeinfo.RegClass:$dst, EFLAGS, - (opnode typeinfo.RegClass:$src1, typeinfo.ImmOperator:$src2, - EFLAGS))]>; - -// BinOpRI8 - Binary instructions with inputs "reg, imm8". -class BinOpRI8<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - Format f, dag outlist, X86FoldableSchedWrite sched, list<dag> pattern> - : ITy<opcode, f, typeinfo, outlist, - (ins typeinfo.RegClass:$src1, typeinfo.Imm8Operand:$src2), - mnemonic, "{$src2, $src1|$src1, $src2}", pattern>, - Sched<[sched]> { - let ImmT = Imm8; // Always 8-bit immediate. +// BinOpRI_F - Instructions that read "reg, imm" and write EFLAGS only. +class BinOpRI_F<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node, + Format f> + : BinOpRI<o, m, t, f, (outs), + [(set EFLAGS, (node t.RegClass:$src1, + t.ImmOperator:$src2))]>, DefEFLAGS; +// BinOpRI_RF - Instructions that read "reg, imm" and write "reg", EFLAGS. +class BinOpRI_RF<bits<8> o, string m, X86TypeInfo t, SDNode node, Format f> + : BinOpRI<o, m, t, f, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, + (node t.RegClass:$src1, t.ImmOperator:$src2))]>, DefEFLAGS; +// BinOpRIF_RF - Instructions that read "reg, imm", write "reg" and read/write +// EFLAGS. +class BinOpRIF_RF<bits<8> o, string m, X86TypeInfo t, SDNode node, Format f> + : BinOpRI<o, m, t, f, (outs t.RegClass:$dst), + [(set t.RegClass:$dst, EFLAGS, + (node t.RegClass:$src1, t.ImmOperator:$src2, + EFLAGS))]>, DefEFLAGS, UseEFLAGS { + let SchedRW = [WriteADC]; +} +// BinOpRI8 - Instructions that read "reg, imm8". +class BinOpRI8<bits<8> o, string m, X86TypeInfo t, Format f, dag out> + : ITy<o, f, t, out, (ins t.RegClass:$src1, t.Imm8Operand:$src2), m, + "{$src2, $src1|$src1, $src2}", []>, Sched<[WriteALU]> { + let ImmT = Imm8; +} +// BinOpRI8_F - Instructions that read "reg, imm8" and write EFLAGS only. +class BinOpRI8_F<bits<8> o, string m, X86TypeInfo t, Format f> + : BinOpRI8<o, m, t, f, (outs)>, DefEFLAGS; +// BinOpRI8_RF - Instructions that read "reg, imm8" and write "reg", EFLAGS. +class BinOpRI8_RF<bits<8> o, string m, X86TypeInfo t, Format f> + : BinOpRI8<o, m, t, f, (outs t.RegClass:$dst)>, DefEFLAGS; +// BinOpRI8F_RF - Instructions that read "reg, imm", write "reg" and read/write +// EFLAGS. +class BinOpRI8F_RF<bits<8> o, string m, X86TypeInfo t, Format f> + : BinOpRI8<o, m, t, f, (outs t.RegClass:$dst)>, DefEFLAGS, UseEFLAGS { + let SchedRW = [WriteADC]; } -// BinOpRI8_F - Binary instructions with inputs "reg, imm8". -class BinOpRI8_F<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, Format f> - : BinOpRI8<opcode, mnemonic, typeinfo, f, (outs), WriteALU, []>; - -// BinOpRI8_RF - Binary instructions with inputs "reg, imm8". -class BinOpRI8_RF<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, Format f> - : BinOpRI8<opcode, mnemonic, typeinfo, f, (outs typeinfo.RegClass:$dst), WriteALU, []>; - -// BinOpRI8_RFF - Binary instructions with inputs "reg, imm8". -class BinOpRI8_RFF<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, Format f> - : BinOpRI8<opcode, mnemonic, typeinfo, f, (outs typeinfo.RegClass:$dst), WriteADC, []>; - -// BinOpMR - Binary instructions with inputs "[mem], reg". -class BinOpMR<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - list<dag> pattern> - : ITy<opcode, MRMDestMem, typeinfo, - (outs), (ins typeinfo.MemOperand:$dst, typeinfo.RegClass:$src), - mnemonic, "{$src, $dst|$dst, $src}", pattern>; - -// BinOpMR_RMW - Binary instructions with inputs "[mem], reg", where the pattern -// implicitly use EFLAGS. -class BinOpMR_RMW<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - SDNode opnode> - : BinOpMR<opcode, mnemonic, typeinfo, - [(store (opnode (load addr:$dst), typeinfo.RegClass:$src), addr:$dst), +// BinOpMR - Instructions that read "[mem], reg". +class BinOpMR<bits<8> o, string m, X86TypeInfo t, list<dag> p> + : ITy<o, MRMDestMem, t, (outs), (ins t.MemOperand:$dst, t.RegClass:$src), m, + "{$src, $dst|$dst, $src}", p> { + let mayLoad = 1; +} +// BinOpMR_F - Instructions that read "[mem], imm8" and write EFLAGS only. +class BinOpMR_F<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node> + : BinOpMR<o, m, t, + [(set EFLAGS, (node (t.LoadNode addr:$dst), t.RegClass:$src))]>, + Sched<[WriteALU.Folded, ReadDefault, ReadDefault, ReadDefault, + ReadDefault, ReadDefault, WriteALU.ReadAfterFold]>, DefEFLAGS; +// BinOpMR_MF - Instructions that read "[mem], reg" and write "[mem]", EFLAGS. +class BinOpMR_MF<bits<8> o, string m, X86TypeInfo t, SDNode node> + : BinOpMR<o, m, t, + [(store (node (load addr:$dst), t.RegClass:$src), addr:$dst), (implicit EFLAGS)]>, Sched<[WriteALURMW, // base, scale, index, offset, segment - ReadDefault, ReadDefault, ReadDefault, - ReadDefault, ReadDefault, - WriteALU.ReadAfterFold]>; // reg - -// BinOpMR_RMW_FF - Binary instructions with inputs "[mem], reg", where the -// pattern sets EFLAGS and implicitly uses EFLAGS. -class BinOpMR_RMW_FF<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - SDNode opnode> - : BinOpMR<opcode, mnemonic, typeinfo, - [(store (opnode (load addr:$dst), typeinfo.RegClass:$src, EFLAGS), - addr:$dst), - (implicit EFLAGS)]>, + ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault, + WriteALU.ReadAfterFold]>, // reg + DefEFLAGS { + let mayStore = 1; +} +// BinOpMRF_MF - Instructions that read "[mem], reg", write "[mem]" and +// read/write EFLAGS. +class BinOpMRF_MF<bits<8> o, string m, X86TypeInfo t, SDNode node> + : BinOpMR<o, m, t, + [(store (node (load addr:$dst), t.RegClass:$src, EFLAGS), + addr:$dst), (implicit EFLAGS)]>, Sched<[WriteADCRMW, // base, scale, index, offset, segment ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault, WriteALU.ReadAfterFold, // reg - WriteALU.ReadAfterFold]>; // EFLAGS - -// BinOpMR_F - Binary instructions with inputs "[mem], reg", where the pattern -// has EFLAGS as a result. -class BinOpMR_F<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - SDPatternOperator opnode> - : BinOpMR<opcode, mnemonic, typeinfo, - [(set EFLAGS, (opnode (typeinfo.LoadNode addr:$dst), - typeinfo.RegClass:$src))]>, - Sched<[WriteALU.Folded, ReadDefault, ReadDefault, ReadDefault, - ReadDefault, ReadDefault, WriteALU.ReadAfterFold]>; - -// BinOpMI - Binary instructions with inputs "[mem], imm". -class BinOpMI<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - Format f, list<dag> pattern> - : ITy<opcode, f, typeinfo, - (outs), (ins typeinfo.MemOperand:$dst, typeinfo.ImmOperand:$src), - mnemonic, "{$src, $dst|$dst, $src}", pattern> { - let ImmT = typeinfo.ImmEncoding; + WriteALU.ReadAfterFold]>, // EFLAGS + DefEFLAGS, UseEFLAGS { + let mayStore = 1; } -// BinOpMI_RMW - Binary instructions with inputs "[mem], imm", where the -// pattern implicitly use EFLAGS. -class BinOpMI_RMW<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - SDNode opnode, Format f> - : BinOpMI<opcode, mnemonic, typeinfo, f, - [(store (opnode (typeinfo.VT (load addr:$dst)), - typeinfo.ImmOperator:$src), addr:$dst), - (implicit EFLAGS)]>, - Sched<[WriteALURMW]>; - -// BinOpMI_RMW_FF - Binary instructions with inputs "[mem], imm", where the -// pattern sets EFLAGS and implicitly uses EFLAGS. -class BinOpMI_RMW_FF<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - SDNode opnode, Format f> - : BinOpMI<opcode, mnemonic, typeinfo, f, - [(store (opnode (typeinfo.VT (load addr:$dst)), - typeinfo.ImmOperator:$src, EFLAGS), addr:$dst), - (implicit EFLAGS)]>, - Sched<[WriteADCRMW]>; - -// BinOpMI_F - Binary instructions with inputs "[mem], imm", where the pattern -// has EFLAGS as a result. -class BinOpMI_F<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - SDPatternOperator opnode, Format f> - : BinOpMI<opcode, mnemonic, typeinfo, f, - [(set EFLAGS, (opnode (typeinfo.LoadNode addr:$dst), - typeinfo.ImmOperator:$src))]>, - Sched<[WriteALU.Folded]>; - -// BinOpMI8 - Binary instructions with inputs "[mem], imm8". -class BinOpMI8<string mnemonic, X86TypeInfo typeinfo, - Format f, list<dag> pattern> - : ITy<0x82, f, typeinfo, - (outs), (ins typeinfo.MemOperand:$dst, typeinfo.Imm8Operand:$src), - mnemonic, "{$src, $dst|$dst, $src}", pattern> { - let ImmT = Imm8; // Always 8-bit immediate. +// BinOpMI - Instructions that read "[mem], imm". +class BinOpMI<bits<8> o, string m, X86TypeInfo t, Format f, list<dag> p> + : ITy<o, f, t, (outs), (ins t.MemOperand:$dst, t.ImmOperand:$src), m, + "{$src, $dst|$dst, $src}", p> { + let ImmT = t.ImmEncoding; + let mayLoad = 1; +} +// BinOpMI_F - Instructions that read "[mem], imm" and write EFLAGS only. +class BinOpMI_F<bits<8> o, string m, X86TypeInfo t, SDPatternOperator node, + Format f> + : BinOpMI<o, m, t, f, + [(set EFLAGS, (node (t.LoadNode addr:$dst), t.ImmOperator:$src))]>, + Sched<[WriteALU.Folded]>, DefEFLAGS; +// BinOpMI_MF - Instructions that read "[mem], imm" and write "[mem]", EFLAGS. +class BinOpMI_MF<bits<8> o, string m, X86TypeInfo t, SDNode node, Format f> + : BinOpMI<o, m, t, f, + [(store (node (t.VT (load addr:$dst)), + t.ImmOperator:$src), addr:$dst), (implicit EFLAGS)]>, + Sched<[WriteALURMW]>, DefEFLAGS { + let mayStore = 1; +} +// BinOpMIF_MF - Instructions that read "[mem], imm", write "[mem]" and +// read/write EFLAGS. +class BinOpMIF_MF<bits<8> o, string m, X86TypeInfo t, SDNode node, Format f> + : BinOpMI<o, m, t, f, + [(store (node (t.VT (load addr:$dst)), + t.ImmOperator:$src, EFLAGS), addr:$dst), (implicit EFLAGS)]>, + Sched<[WriteADCRMW]>, DefEFLAGS, UseEFLAGS { + let mayStore = 1; } -// BinOpMI8_RMW - Binary instructions with inputs "[mem], imm8". -class BinOpMI8_RMW<string mnemonic, X86TypeInfo typeinfo, Format f> - : BinOpMI8<mnemonic, typeinfo, f, []>, Sched<[WriteALURMW]>; - -// BinOpMI8_RMW_FF - Binary instructions with inputs "[mem], imm8". -class BinOpMI8_RMW_FF<string mnemonic, X86TypeInfo typeinfo, Format f> - : BinOpMI8<mnemonic, typeinfo, f, []>, Sched<[WriteADCRMW]>; - -// BinOpMI8_F - Binary instructions with inputs "[mem], imm8" -class BinOpMI8_F<string mnemonic, X86TypeInfo typeinfo, Format f> - : BinOpMI8<mnemonic, typeinfo, f, []>, Sched<[WriteALU.Folded]>; - -// BinOpAI - Binary instructions with input imm, that implicitly use A reg and -// implicitly define Areg and EFLAGS. -class BinOpAI<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - Register areg, string operands, X86FoldableSchedWrite sched = WriteALU> - : ITy<opcode, RawFrm, typeinfo, - (outs), (ins typeinfo.ImmOperand:$src), - mnemonic, operands, []>, - Sched<[sched]> { - let ImmT = typeinfo.ImmEncoding; +// BinOpMI8 - Instructions that read "[mem], imm8". +class BinOpMI8<string m, X86TypeInfo t, Format f> + : ITy<0x83, f, t, (outs), (ins t.MemOperand:$dst, t.Imm8Operand:$src), m, + "{$src, $dst|$dst, $src}", []> { + let ImmT = Imm8; + let mayLoad = 1; +} +// BinOpMI8_F - Instructions that read "[mem], imm8" and write EFLAGS only. +class BinOpMI8_F<string m, X86TypeInfo t, Format f> + : BinOpMI8<m, t, f>, Sched<[WriteALU.Folded]>, DefEFLAGS; +// BinOpMI8_MF - Instructions that read "[mem], imm8" and write "[mem]", EFLAGS. +class BinOpMI8_MF<string m, X86TypeInfo t, Format f> + : BinOpMI8<m, t, f>, Sched<[WriteALURMW]>, DefEFLAGS { + let mayStore = 1; +} +// BinOpMI8F_MF - Instructions that read "[mem], imm8", write "[mem]" and +// read/write EFLAGS. +class BinOpMI8F_MF<string m, X86TypeInfo t, Format f> + : BinOpMI8<m, t, f>, Sched<[WriteADCRMW]>, DefEFLAGS, UseEFLAGS { + let mayStore = 1; +} + +// BinOpAI - Instructions that read "a-reg imm" (Accumulator register). +class BinOpAI<bits<8> o, string m, X86TypeInfo t, Register areg, string args> + : ITy<o, RawFrm, t, (outs), (ins t.ImmOperand:$src), m, args, []>, + Sched<[WriteALU]> { + let ImmT = t.ImmEncoding; let Uses = [areg]; - let Defs = [areg, EFLAGS]; - let hasSideEffects = 0; } +// BinOpAI_F - Instructions that read "a-reg imm" and write EFLAGS only. +class BinOpAI_F<bits<8> o, string m, X86TypeInfo t, Register areg, string args> + : BinOpAI<o, m, t, areg, args>, DefEFLAGS; -// BinOpAI_RFF - Binary instructions with input imm, that implicitly use and -// define Areg and EFLAGS. -class BinOpAI_RFF<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - Register areg, string operands> - : BinOpAI<opcode, mnemonic, typeinfo, areg, operands, WriteADC> { +// BinOpAI_AF - Instructions that read "a-reg imm" and write a-reg/EFLAGS. +class BinOpAI_AF<bits<8> o, string m, X86TypeInfo t, Register areg, + string args> : BinOpAI<o, m, t, areg, args> { + let Defs = [areg, EFLAGS]; +} +// BinOpAIF_AF - Instructions that read "a-reg imm", write a-reg and read/write +// EFLAGS. +class BinOpAIF_AF<bits<8> o, string m, X86TypeInfo t, Register areg, + string args> : BinOpAI<o, m, t, areg, args> { let Uses = [areg, EFLAGS]; + let Defs = [areg, EFLAGS]; + let SchedRW = [WriteADC]; } -// BinOpAI_F - Binary instructions with input imm, that implicitly use A reg and -// implicitly define EFLAGS. -class BinOpAI_F<bits<8> opcode, string mnemonic, X86TypeInfo typeinfo, - Register areg, string operands> - : BinOpAI<opcode, mnemonic, typeinfo, areg, operands> { - let Defs = [EFLAGS]; +// UnaryOpR - Instructions that read "reg" and write "reg". +class UnaryOpR<bits<8> o, Format f, string m, X86TypeInfo t, list<dag> p> + : ITy<o, f, t, (outs t.RegClass:$dst), + (ins t.RegClass:$src1), m, "$dst", p>, Sched<[WriteALU]>; + +// UnaryOpM - Instructions that read "[mem]" and writes "[mem]". +class UnaryOpM<bits<8> o, Format f, string m, X86TypeInfo t, list<dag> p> + : ITy<o, f, t, (outs), (ins t.MemOperand:$dst), m, "$dst", p>, + Sched<[WriteALURMW]> { + let mayLoad = 1; + let mayStore = 1; } -// UnaryOpM - Unary instructions with a memory operand. -class UnaryOpM<bits<8> opcode, Format f, string mnemonic, X86TypeInfo info, - list<dag> pattern> - : ITy<opcode, f, info, (outs), (ins info.MemOperand:$dst), mnemonic, - "$dst", pattern>; - -// UnaryOpR - Unary instructions with a register. -class UnaryOpR<bits<8> opcode, Format f, string mnemonic, X86TypeInfo info, - list<dag> pattern> - : ITy<opcode, f, info, (outs info.RegClass:$dst), - (ins info.RegClass:$src1), mnemonic, "$dst", pattern>; - -// INCDECR - Instructions like "inc reg". -class INCDECR<Format f, string mnemonic, X86TypeInfo info, - SDPatternOperator node> - : UnaryOpR<0xFE, f, mnemonic, info, - [(set info.RegClass:$dst, EFLAGS, - (node info.RegClass:$src1, 1))]>; - -// INCDECM - Instructions like "inc [mem]". -class INCDECM<Format f, string mnemonic, X86TypeInfo info, int num> - : UnaryOpM<0xFE, f, mnemonic, info, - [(store (add (info.LoadNode addr:$dst), num), addr:$dst), - (implicit EFLAGS)]>; - -// INCDECR_ALT - Instructions like "inc reg" short forms. -class INCDECR_ALT<bits<8> opcode, string mnemonic, X86TypeInfo info> - : UnaryOpR<opcode, AddRegFrm, mnemonic, info, []>{ +// INCDECR - Instructions like "inc reg". +class INCDECR<Format f, string m, X86TypeInfo t, SDPatternOperator node> + : UnaryOpR<0xFF, f, m, t, + [(set t.RegClass:$dst, EFLAGS, (node t.RegClass:$src1, 1))]>, + DefEFLAGS { + let isConvertibleToThreeAddress = 1; // Can xform into LEA. +} + +// INCDECM - Instructions like "inc [mem]". +class INCDECM<Format f, string m, X86TypeInfo t, int num> + : UnaryOpM<0xFF, f, m, t, + [(store (add (t.LoadNode addr:$dst), num), addr:$dst), + (implicit EFLAGS)]>, DefEFLAGS; + +// INCDECR_ALT - Instructions like "inc reg" short forms. +class INCDECR_ALT<bits<8> o, string m, X86TypeInfo t> + : UnaryOpR<o, AddRegFrm, m, t, []>, DefEFLAGS { + // Short forms only valid in 32-bit mode. Selected during MCInst lowering. let Predicates = [Not64BitMode]; - let Opcode = opcode; } -// MulOpR - Instructions like "mul reg". -class MulOpR<bits<8> opcode, Format f, string mnemonic, X86TypeInfo info, - X86FoldableSchedWrite sched, list<dag> pattern> - : ITy<opcode, f, info, (outs), (ins info.RegClass:$src), mnemonic, - "$src", pattern>, - Sched<[sched]>; - -// MulOpM - Instructions like "mul [mem]". -class MulOpM<bits<8> opcode, Format f, string mnemonic, X86TypeInfo info, - X86FoldableSchedWrite sched, list<dag> pattern> - : ITy<opcode, f, info, (outs), (ins info.MemOperand:$src), mnemonic, - "$src", pattern>, SchedLoadReg<sched>; - -// NegOpR - Instructions like "neg reg", with implicit EFLAGS. -class NegOpR<bits<8> opcode, string mnemonic, X86TypeInfo info> - : UnaryOpR<opcode, MRM3r, mnemonic, info, - [(set info.RegClass:$dst, (ineg info.RegClass:$src1)), - (implicit EFLAGS)]>; - -// NotOpR - Instructions like "not reg". -class NotOpR<bits<8> opcode, string mnemonic, X86TypeInfo info> - : UnaryOpR<opcode, MRM2r, mnemonic, info, - [(set info.RegClass:$dst, - (not info.RegClass:$src1))]>; - -// NegOpM - Instructions like "neg [mem]", with implicit EFLAGS. -class NegOpM<bits<8> opcode, string mnemonic, X86TypeInfo info> - : UnaryOpM<opcode, MRM3m, mnemonic, info, - [(store (ineg (info.LoadNode addr:$dst)), addr:$dst), - (implicit EFLAGS)]>; - -// NotOpM - Instructions like "neg [mem]". -class NotOpM<bits<8> opcode, string mnemonic, X86TypeInfo info> - : UnaryOpM<opcode, MRM2m, mnemonic, info, - [(store (not (info.LoadNode addr:$dst)), addr:$dst)]>; - -// BinOpRR_C - Binary instructions with inputs "reg, reg", which used mainly -// with Constraints = "$src1 = $dst". -class BinOpRR_C<bits<8> opcode, Format f, string mnemonic, X86TypeInfo info, - list<dag> pattern> - : ITy<opcode, f, info, (outs info.RegClass:$dst), - (ins info.RegClass:$src1, info.RegClass:$src2), - mnemonic, "{$src2, $dst|$dst, $src2}", pattern>; - -// BinOpRM_C - Binary instructions with inputs "reg, [mem]", which used mainly -// with Constraints = "$src1 = $dst". -class BinOpRM_C<bits<8> opcode, Format f, string mnemonic, X86TypeInfo info, - list<dag> pattern> - : ITy<opcode, f, info, (outs info.RegClass:$dst), - (ins info.RegClass:$src1, info.MemOperand:$src2), - mnemonic, "{$src2, $dst|$dst, $src2}", pattern>; +// MulOpR - Instructions like "mul reg". +class MulOpR<bits<8> o, Format f, string m, X86TypeInfo t, + X86FoldableSchedWrite sched, list<dag> p> + : ITy<o, f, t, (outs), (ins t.RegClass:$src), m, "$src", p>, Sched<[sched]>; + +// MulOpM - Instructions like "mul [mem]". +class MulOpM<bits<8> o, Format f, string m, X86TypeInfo t, + X86FoldableSchedWrite sched, list<dag> p> + : ITy<o, f, t, (outs), (ins t.MemOperand:$src), m, + "$src", p>, SchedLoadReg<sched> { + let mayLoad = 1; +} + +// NegOpR - Instructions like "neg reg". +class NegOpR<bits<8> o, string m, X86TypeInfo t> + : UnaryOpR<o, MRM3r, m, t, + [(set t.RegClass:$dst, (ineg t.RegClass:$src1)), + (implicit EFLAGS)]>, DefEFLAGS; + +// NegOpM - Instructions like "neg [mem]". +class NegOpM<bits<8> o, string m, X86TypeInfo t> + : UnaryOpM<o, MRM3m, m, t, + [(store (ineg (t.LoadNode addr:$dst)), addr:$dst), + (implicit EFLAGS)]>, DefEFLAGS; + +// NOTE: NOT does not set EFLAGS! +// NotOpR - Instructions like "not reg". +class NotOpR<bits<8> o, string m, X86TypeInfo t> + : UnaryOpR<o, MRM2r, m, t, [(set t.RegClass:$dst, (not t.RegClass:$src1))]>; + +// NotOpM - Instructions like "neg [mem]". +class NotOpM<bits<8> o, string m, X86TypeInfo t> + : UnaryOpM<o, MRM2m, m, t, + [(store (not (t.LoadNode addr:$dst)), addr:$dst)]>; // IMulOpRR - Instructions like "imul reg, reg, i8". -class IMulOpRR<bits<8> opcode, string mnemonic, X86TypeInfo info, - X86FoldableSchedWrite sched> - : BinOpRR_C<opcode, MRMSrcReg, mnemonic, info, - [(set info.RegClass:$dst, EFLAGS, - (X86smul_flag info.RegClass:$src1, - info.RegClass:$src2))]>, - Sched<[sched]>, TB; +class IMulOpRR<bits<8> o, string m, X86TypeInfo t, X86FoldableSchedWrite sched> + : BinOpRR_RF<o, m, t, X86smul_flag>, TB { + let Form = MRMSrcReg; + let SchedRW = [sched]; + // X = IMUL Y, Z --> X = IMUL Z, Y + let isCommutable = 1; +} // IMulOpRM - Instructions like "imul reg, reg, [mem]". -class IMulOpRM<bits<8> opcode, string mnemonic, X86TypeInfo info, - X86FoldableSchedWrite sched> - : BinOpRM_C<opcode, MRMSrcMem, mnemonic, info, - [(set info.RegClass:$dst, EFLAGS, - (X86smul_flag info.RegClass:$src1, (info.LoadNode addr:$src2)))]>, - Sched<[sched.Folded, sched.ReadAfterFold]>, TB; +class IMulOpRM<bits<8> o, string m, X86TypeInfo t, X86FoldableSchedWrite sched> + : BinOpRM_RF<o, m, t, X86smul_flag>, TB { +let Form = MRMSrcMem; +let SchedRW = [sched.Folded, sched.ReadAfterFold]; +} // IMulOpRRI8 - Instructions like "imul reg, reg, i8". -class IMulOpRRI8<bits<8> opcode, string mnemonic, X86TypeInfo info, +class IMulOpRRI8<bits<8> o, string m, X86TypeInfo t, X86FoldableSchedWrite sched> - : ITy<opcode, MRMSrcReg, info, (outs info.RegClass:$dst), - (ins info.RegClass:$src1, info.Imm8Operand:$src2), mnemonic, - "{$src2, $src1, $dst|$dst, $src1, $src2}", []>, Sched<[sched]> { + : ITy<o, MRMSrcReg, t, (outs t.RegClass:$dst), + (ins t.RegClass:$src1, t.Imm8Operand:$src2), m, + "{$src2, $src1, $dst|$dst, $src1, $src2}", []>, Sched<[sched]>, DefEFLAGS { let ImmT = Imm8; } // IMulOpRRI - Instructions like "imul reg, reg, i16/i32/i64". -class IMulOpRRI<bits<8> opcode, string mnemonic, X86TypeInfo info, +class IMulOpRRI<bits<8> o, string m, X86TypeInfo t, X86FoldableSchedWrite sched> - : ITy<opcode, MRMSrcReg, info, (outs info.RegClass:$dst), - (ins info.RegClass:$src1, info.ImmOperand:$src2), mnemonic, + : ITy<o, MRMSrcReg, t, (outs t.RegClass:$dst), + (ins t.RegClass:$src1, t.ImmOperand:$src2), m, "{$src2, $src1, $dst|$dst, $src1, $src2}", - [(set info.RegClass:$dst, EFLAGS, - (X86smul_flag info.RegClass:$src1, - info.ImmNoSuOperator:$src2))]>, - Sched<[sched]>{ - let ImmT = info.ImmEncoding; + [(set t.RegClass:$dst, EFLAGS, (X86smul_flag t.RegClass:$src1, + t.ImmNoSuOperator:$src2))]>, + Sched<[sched]>, DefEFLAGS { + let ImmT = t.ImmEncoding; } // IMulOpRMI8 - Instructions like "imul reg, [mem], i8". -class IMulOpRMI8<bits<8> opcode, string mnemonic, X86TypeInfo info, +class IMulOpRMI8<bits<8> o, string m, X86TypeInfo t, X86FoldableSchedWrite sched> - : ITy<opcode, MRMSrcMem, info, (outs info.RegClass:$dst), - (ins info.MemOperand:$src1, info.Imm8Operand:$src2), mnemonic, - "{$src2, $src1, $dst|$dst, $src1, $src2}", []>, Sched<[sched.Folded]> { + : ITy<o, MRMSrcMem, t, (outs t.RegClass:$dst), + (ins t.MemOperand:$src1, t.Imm8Operand:$src2), m, + "{$src2, $src1, $dst|$dst, $src1, $src2}", []>, Sched<[sched.Folded]>, + DefEFLAGS { let ImmT = Imm8; + let mayLoad = 1; } // IMulOpRMI - Instructions like "imul reg, [mem], i16/i32/i64". -class IMulOpRMI<bits<8> opcode, string mnemonic, X86TypeInfo info, +class IMulOpRMI<bits<8> o, string m, X86TypeInfo t, X86FoldableSchedWrite sched> - : ITy<opcode, MRMSrcMem, info, (outs info.RegClass:$dst), - (ins info.MemOperand:$src1, info.ImmOperand:$src2), mnemonic, + : ITy<o, MRMSrcMem, t, (outs t.RegClass:$dst), + (ins t.MemOperand:$src1, t.ImmOperand:$src2), m, "{$src2, $src1, $dst|$dst, $src1, $src2}", - [(set info.RegClass:$dst, EFLAGS, - (X86smul_flag (info.LoadNode addr:$src1), - info.ImmNoSuOperator:$src2))]>, - Sched<[sched.Folded]>{ - let ImmT = info.ImmEncoding; + [(set t.RegClass:$dst, EFLAGS, + (X86smul_flag (t.LoadNode addr:$src1), t.ImmNoSuOperator:$src2))]>, + Sched<[sched.Folded]>, DefEFLAGS { + let ImmT = t.ImmEncoding; } -let Defs = [EFLAGS] in { -let Constraints = "$src1 = $dst", SchedRW = [WriteALU] in { -// Short forms only valid in 32-bit mode. Selected during MCInst lowering. -let hasSideEffects = 0 in { +let Constraints = "$src1 = $dst" in { def INC16r_alt : INCDECR_ALT<0x40, "inc", Xi16>; def INC32r_alt : INCDECR_ALT<0x40, "inc", Xi32>; -} // hasSideEffects = 0 - -let isConvertibleToThreeAddress = 1 in { // Can xform into LEA. def INC8r : INCDECR<MRM0r, "inc", Xi8, X86add_flag_nocf>; def INC16r : INCDECR<MRM0r, "inc", Xi16, X86add_flag_nocf>; def INC32r : INCDECR<MRM0r, "inc", Xi32, X86add_flag_nocf>; def INC64r : INCDECR<MRM0r, "inc", Xi64, X86add_flag_nocf>; -} // isConvertibleToThreeAddress = 1 -} // Constraints = "$src1 = $dst", SchedRW -let SchedRW = [WriteALURMW] in { -let Predicates = [UseIncDec] in { - def INC8m : INCDECM<MRM0m, "inc", Xi8, 1>; - def INC16m : INCDECM<MRM0m, "inc", Xi16, 1>; - def INC32m : INCDECM<MRM0m, "inc", Xi32, 1>; -} // Predicates -let Predicates = [UseIncDec, In64BitMode] in { - def INC64m : INCDECM<MRM0m, "inc", Xi64, 1>; -} // Predicates -} // SchedRW - -let Constraints = "$src1 = $dst", SchedRW = [WriteALU] in { -// Short forms only valid in 32-bit mode. Selected during MCInst lowering. -let hasSideEffects = 0 in { def DEC16r_alt : INCDECR_ALT<0x48, "dec", Xi16>; def DEC32r_alt : INCDECR_ALT<0x48, "dec", Xi32>; -} // hasSideEffects = 0 - -let isConvertibleToThreeAddress = 1 in { // Can xform into LEA. def DEC8r : INCDECR<MRM1r, "dec", Xi8, X86sub_flag_nocf>; def DEC16r : INCDECR<MRM1r, "dec", Xi16, X86sub_flag_nocf>; def DEC32r : INCDECR<MRM1r, "dec", Xi32, X86sub_flag_nocf>; def DEC64r : INCDECR<MRM1r, "dec", Xi64, X86sub_flag_nocf>; -} // isConvertibleToThreeAddress = 1 -} // Constraints = "$src1 = $dst", SchedRW +} -let SchedRW = [WriteALURMW] in { let Predicates = [UseIncDec] in { - def DEC8m : INCDECM<MRM1m, "dec", Xi8, -1>; - def DEC16m : INCDECM<MRM1m, "dec", Xi16, -1>; - def DEC32m : INCDECM<MRM1m, "dec", Xi32, -1>; -} // Predicates +def INC8m : INCDECM<MRM0m, "inc", Xi8, 1>; +def INC16m : INCDECM<MRM0m, "inc", Xi16, 1>; +def INC32m : INCDECM<MRM0m, "inc", Xi32, 1>; +def DEC8m : INCDECM<MRM1m, "dec", Xi8, -1>; +def DEC16m : INCDECM<MRM1m, "dec", Xi16, -1>; +def DEC32m : INCDECM<MRM1m, "dec", Xi32, -1>; +} let Predicates = [UseIncDec, In64BitMode] in { - def DEC64m : INCDECM<MRM1m, "dec", Xi64, -1>; -} // Predicates -} // SchedRW -} // Defs = [EFLAGS] +def INC64m : INCDECM<MRM0m, "inc", Xi64, 1>; +def DEC64m : INCDECM<MRM1m, "dec", Xi64, -1>; +} // Extra precision multiplication @@ -554,14 +447,14 @@ def MUL8r : MulOpR<0xF6, MRM4r, "mul", Xi8, WriteIMul8, // syntax can be accepted. [(set AL, (mul AL, GR8:$src)), (implicit EFLAGS)]>; // AX,DX = AX*GR16 -let Defs = [AX,DX,EFLAGS], Uses = [AX], hasSideEffects = 0 in +let Defs = [AX,DX,EFLAGS], Uses = [AX] in def MUL16r : MulOpR<0xF7, MRM4r, "mul", Xi16, WriteIMul16, []>; // EAX,EDX = EAX*GR32 -let Defs = [EAX,EDX,EFLAGS], Uses = [EAX], hasSideEffects = 0 in +let Defs = [EAX,EDX,EFLAGS], Uses = [EAX] in def MUL32r : MulOpR<0xF7, MRM4r, "mul", Xi32, WriteIMul32, [/*(set EAX, EDX, EFLAGS, (X86umul_flag EAX, GR32:$src))*/]>; // RAX,RDX = RAX*GR64 -let Defs = [RAX,RDX,EFLAGS], Uses = [RAX], hasSideEffects = 0 in +let Defs = [RAX,RDX,EFLAGS], Uses = [RAX] in def MUL64r : MulOpR<0xF7, MRM4r, "mul", Xi64, WriteIMul64, [/*(set RAX, RDX, EFLAGS, (X86umul_flag RAX, GR64:$src))*/]>; // AL,AH = AL*[mem8] @@ -573,7 +466,6 @@ def MUL8m : MulOpM<0xF6, MRM4m, "mul", Xi8, WriteIMul8, [(set AL, (mul AL, (loadi8 addr:$src))), (implicit EFLAGS)]>; // AX,DX = AX*[mem16] -let mayLoad = 1, hasSideEffects = 0 in { let Defs = [AX,DX,EFLAGS], Uses = [AX] in def MUL16m : MulOpM<0xF7, MRM4m, "mul", Xi16, WriteIMul16, []>; // EAX,EDX = EAX*[mem32] @@ -582,10 +474,8 @@ def MUL32m : MulOpM<0xF7, MRM4m, "mul", Xi32, WriteIMul32, []>; // RAX,RDX = RAX*[mem64] let Defs = [RAX,RDX,EFLAGS], Uses = [RAX] in def MUL64m : MulOpM<0xF7, MRM4m, "mul", Xi64, WriteIMul64, []>, - Requires<[In64BitMode]>; -} + Requires<[In64BitMode]>; -let hasSideEffects = 0 in { // AL,AH = AL*GR8 let Defs = [AL,EFLAGS,AX], Uses = [AL] in def IMUL8r : MulOpR<0xF6, MRM5r, "imul", Xi8, WriteIMul8, []>; @@ -599,7 +489,6 @@ def IMUL32r : MulOpR<0xF7, MRM5r, "imul", Xi32, WriteIMul32, []>; let Defs = [RAX,RDX,EFLAGS], Uses = [RAX] in def IMUL64r : MulOpR<0xF7, MRM5r, "imul", Xi64, WriteIMul64, []>; -let mayLoad = 1 in { // AL,AH = AL*[mem8] let Defs = [AL,EFLAGS,AX], Uses = [AL] in def IMUL8m : MulOpM<0xF6, MRM5m, "imul", Xi8, WriteIMul8, []>; @@ -612,28 +501,21 @@ def IMUL32m : MulOpM<0xF7, MRM5m, "imul", Xi32, WriteIMul32, []>; // RAX,RDX = RAX*[mem64] let Defs = [RAX,RDX,EFLAGS], Uses = [RAX] in def IMUL64m : MulOpM<0xF7, MRM5m, "imul", Xi64, WriteIMul64, []>, - Requires<[In64BitMode]>; -} + Requires<[In64BitMode]>; -let Defs = [EFLAGS] in { let Constraints = "$src1 = $dst" in { -let isCommutable = 1 in { -// X = IMUL Y, Z --> X = IMUL Z, Y // Register-Register Signed Integer Multiply def IMUL16rr : IMulOpRR<0xAF, "imul", Xi16, WriteIMul16Reg>; def IMUL32rr : IMulOpRR<0xAF, "imul", Xi32, WriteIMul32Reg>; def IMUL64rr : IMulOpRR<0xAF, "imul", Xi64, WriteIMul64Reg>; -} // isCommutable // Register-Memory Signed Integer Multiply def IMUL16rm : IMulOpRM<0xAF, "imul", Xi16, WriteIMul16Reg>; def IMUL32rm : IMulOpRM<0xAF, "imul", Xi32, WriteIMul32Reg>; def IMUL64rm : IMulOpRM<0xAF, "imul", Xi64, WriteIMul64Reg>; -} // Constraints = "$src1 = $dst" -} // Defs = [EFLAGS] +} // Surprisingly enough, these are not two address instructions! -let Defs = [EFLAGS] in { // NOTE: These are order specific, we want the ri8 forms to be listed // first so that they are slightly preferred to the ri forms. @@ -653,7 +535,6 @@ def IMUL64rri32 : IMulOpRRI<0x69, "imul", Xi64, WriteIMul64Imm>; // Memory-Integer Signed Integer Multiply // GR16 = [mem16]*I8 -let mayLoad = 1 in { def IMUL16rmi8 : IMulOpRMI8<0x6B, "imul", Xi16, WriteIMul16Imm>; // GR16 = [mem16]*I16 def IMUL16rmi : IMulOpRMI<0x69, "imul", Xi16, WriteIMul16Imm>; @@ -665,9 +546,6 @@ def IMUL32rmi : IMulOpRMI<0x69, "imul", Xi32, WriteIMul32Imm>; def IMUL64rmi8 : IMulOpRMI8<0x6B, "imul", Xi64, WriteIMul64Imm>; // GR64 = [mem64]*I32 def IMUL64rmi32 : IMulOpRMI<0x69, "imul", Xi64, WriteIMul64Imm>; -} // mayLoad -} // Defs = [EFLAGS] -} // hasSideEffects // unsigned division/remainder let hasSideEffects = 1 in { // so that we don't speculatively execute @@ -713,7 +591,6 @@ def IDIV32r: MulOpR<0xF7, MRM7r, "idiv", Xi32, WriteIDiv32, []>; let Defs = [RAX,RDX,EFLAGS], Uses = [RAX,RDX] in def IDIV64r: MulOpR<0xF7, MRM7r, "idiv", Xi64, WriteIDiv64, []>; -let mayLoad = 1 in { let Defs = [AL,AH,EFLAGS], Uses = [AX] in // AX/[mem8] = AL,AH def IDIV8m : MulOpM<0xF6, MRM7m, "idiv", Xi8, WriteIDiv8, []>; @@ -727,47 +604,31 @@ let Defs = [RAX,RDX,EFLAGS], Uses = [RAX,RDX] in // RDX:RAX/[mem64] = RAX,RDX // RDX:RAX/[mem64] = RAX,RDX def IDIV64m: MulOpM<0xF7, MRM7m, "idiv", Xi64, WriteIDiv64, []>, Requires<[In64BitMode]>; -} } // hasSideEffects = 1 -//===----------------------------------------------------------------------===// -// Two address Instructions. -// - -// unary instructions -let Defs = [EFLAGS] in { -let Constraints = "$src1 = $dst", SchedRW = [WriteALU] in { +let Constraints = "$src1 = $dst" in { def NEG8r : NegOpR<0xF6, "neg", Xi8>; def NEG16r : NegOpR<0xF7, "neg", Xi16>; def NEG32r : NegOpR<0xF7, "neg", Xi32>; def NEG64r : NegOpR<0xF7, "neg", Xi64>; -} // Constraints = "$src1 = $dst", SchedRW +} -// Read-modify-write negate. -let SchedRW = [WriteALURMW] in { def NEG8m : NegOpM<0xF6, "neg", Xi8>; def NEG16m : NegOpM<0xF7, "neg", Xi16>; def NEG32m : NegOpM<0xF7, "neg", Xi32>; def NEG64m : NegOpM<0xF7, "neg", Xi64>, Requires<[In64BitMode]>; -} // SchedRW -} // Defs = [EFLAGS] - -// Note: NOT does not set EFLAGS! - -let Constraints = "$src1 = $dst", SchedRW = [WriteALU] in { +let Constraints = "$src1 = $dst" in { def NOT8r : NotOpR<0xF6, "not", Xi8>; def NOT16r : NotOpR<0xF7, "not", Xi16>; def NOT32r : NotOpR<0xF7, "not", Xi32>; def NOT64r : NotOpR<0xF7, "not", Xi64>; -} // Constraints = "$src1 = $dst", SchedRW +} -let SchedRW = [WriteALURMW] in { def NOT8m : NotOpM<0xF6, "not", Xi8>; def NOT16m : NotOpM<0xF7, "not", Xi16>; def NOT32m : NotOpM<0xF7, "not", Xi32>; def NOT64m : NotOpM<0xF7, "not", Xi64>, Requires<[In64BitMode]>; -} // SchedRW /// ArithBinOp_RF - This is an arithmetic binary operator where the pattern is /// defined with "(set GPR:$dst, EFLAGS, (...". @@ -779,81 +640,73 @@ multiclass ArithBinOp_RF<bits<8> BaseOpc, bits<8> BaseOpc2, bits<8> BaseOpc4, SDNode opnodeflag, SDNode opnode, bit CommutableRR, bit ConvertibleToThreeAddress, bit ConvertibleToThreeAddressRR> { - let Defs = [EFLAGS] in { - let Constraints = "$src1 = $dst" in { - let isCommutable = CommutableRR in { - let isConvertibleToThreeAddress = ConvertibleToThreeAddressRR in { - def NAME#8rr : BinOpRR_RF<BaseOpc, mnemonic, Xi8 , opnodeflag>; - def NAME#16rr : BinOpRR_RF<BaseOpc, mnemonic, Xi16, opnodeflag>; - def NAME#32rr : BinOpRR_RF<BaseOpc, mnemonic, Xi32, opnodeflag>; - def NAME#64rr : BinOpRR_RF<BaseOpc, mnemonic, Xi64, opnodeflag>; - } // isConvertibleToThreeAddress - } // isCommutable - - def NAME#8rr_REV : BinOpRR_Rev<BaseOpc2, mnemonic, Xi8>; - def NAME#16rr_REV : BinOpRR_Rev<BaseOpc2, mnemonic, Xi16>; - def NAME#32rr_REV : BinOpRR_Rev<BaseOpc2, mnemonic, Xi32>; - def NAME#64rr_REV : BinOpRR_Rev<BaseOpc2, mnemonic, Xi64>; - - def NAME#8rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi8 , opnodeflag>; - def NAME#16rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi16, opnodeflag>; - def NAME#32rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi32, opnodeflag>; - def NAME#64rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi64, opnodeflag>; - - let isConvertibleToThreeAddress = ConvertibleToThreeAddress, hasSideEffects= 0 in { - def NAME#8ri : BinOpRI_RF<0x80, mnemonic, Xi8 , opnodeflag, RegMRM>; - - // NOTE: These are order specific, we want the ri8 forms to be listed - // first so that they are slightly preferred to the ri forms. - def NAME#16ri8 : BinOpRI8_RF<0x82, mnemonic, Xi16, RegMRM>; - def NAME#32ri8 : BinOpRI8_RF<0x82, mnemonic, Xi32, RegMRM>; - def NAME#64ri8 : BinOpRI8_RF<0x82, mnemonic, Xi64, RegMRM>; - - def NAME#16ri : BinOpRI_RF<0x80, mnemonic, Xi16, opnodeflag, RegMRM>; - def NAME#32ri : BinOpRI_RF<0x80, mnemonic, Xi32, opnodeflag, RegMRM>; - def NAME#64ri32: BinOpRI_RF<0x80, mnemonic, Xi64, opnodeflag, RegMRM>; - } - } // Constraints = "$src1 = $dst" - - let mayLoad = 1, mayStore = 1, hasSideEffects = 0 in { - def NAME#8mr : BinOpMR_RMW<BaseOpc, mnemonic, Xi8 , opnode>; - def NAME#16mr : BinOpMR_RMW<BaseOpc, mnemonic, Xi16, opnode>; - def NAME#32mr : BinOpMR_RMW<BaseOpc, mnemonic, Xi32, opnode>; - def NAME#64mr : BinOpMR_RMW<BaseOpc, mnemonic, Xi64, opnode>; - - // NOTE: These are order specific, we want the mi8 forms to be listed - // first so that they are slightly preferred to the mi forms. - def NAME#16mi8 : BinOpMI8_RMW<mnemonic, Xi16, MemMRM>; - def NAME#32mi8 : BinOpMI8_RMW<mnemonic, Xi32, MemMRM>; - let Predicates = [In64BitMode] in - def NAME#64mi8 : BinOpMI8_RMW<mnemonic, Xi64, MemMRM>; - - def NAME#8mi : BinOpMI_RMW<0x80, mnemonic, Xi8 , opnode, MemMRM>; - def NAME#16mi : BinOpMI_RMW<0x80, mnemonic, Xi16, opnode, MemMRM>; - def NAME#32mi : BinOpMI_RMW<0x80, mnemonic, Xi32, opnode, MemMRM>; - let Predicates = [In64BitMode] in - def NAME#64mi32 : BinOpMI_RMW<0x80, mnemonic, Xi64, opnode, MemMRM>; + let Constraints = "$src1 = $dst" in { + let isCommutable = CommutableRR, + isConvertibleToThreeAddress = ConvertibleToThreeAddressRR in { + def NAME#8rr : BinOpRR_RF<BaseOpc, mnemonic, Xi8 , opnodeflag>; + def NAME#16rr : BinOpRR_RF<BaseOpc, mnemonic, Xi16, opnodeflag>; + def NAME#32rr : BinOpRR_RF<BaseOpc, mnemonic, Xi32, opnodeflag>; + def NAME#64rr : BinOpRR_RF<BaseOpc, mnemonic, Xi64, opnodeflag>; } - // These are for the disassembler since 0x82 opcode behaves like 0x80, but - // not in 64-bit mode. - let Predicates = [Not64BitMode], isCodeGenOnly = 1, ForceDisassemble = 1, - hasSideEffects = 0 in { - let Constraints = "$src1 = $dst" in - def NAME#8ri8 : BinOpRI8_RF<0x82, mnemonic, Xi8, RegMRM>; - let mayLoad = 1, mayStore = 1 in - def NAME#8mi8 : BinOpMI8_RMW<mnemonic, Xi8, MemMRM>; + def NAME#8rr_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi8>; + def NAME#16rr_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi16>; + def NAME#32rr_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi32>; + def NAME#64rr_REV : BinOpRR_RF_Rev<BaseOpc2, mnemonic, Xi64>; + + def NAME#8rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi8 , opnodeflag>; + def NAME#16rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi16, opnodeflag>; + def NAME#32rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi32, opnodeflag>; + def NAME#64rm : BinOpRM_RF<BaseOpc2, mnemonic, Xi64, opnodeflag>; + + let isConvertibleToThreeAddress = ConvertibleToThreeAddress in { + def NAME#8ri : BinOpRI_RF<0x80, mnemonic, Xi8 , opnodeflag, RegMRM>; + // NOTE: These are order specific, we want the ri8 forms to be listed + // first so that they are slightly preferred to the ri forms. + def NAME#16ri8 : BinOpRI8_RF<0x83, mnemonic, Xi16, RegMRM>; + def NAME#32ri8 : BinOpRI8_RF<0x83, mnemonic, Xi32, RegMRM>; + def NAME#64ri8 : BinOpRI8_RF<0x83, mnemonic, Xi64, RegMRM>; + + def NAME#16ri : BinOpRI_RF<0x81, mnemonic, Xi16, opnodeflag, RegMRM>; + def NAME#32ri : BinOpRI_RF<0x81, mnemonic, Xi32, opnodeflag, RegMRM>; + def NAME#64ri32: BinOpRI_RF<0x81, mnemonic, Xi64, opnodeflag, RegMRM>; } - } // Defs = [EFLAGS] - - def NAME#8i8 : BinOpAI<BaseOpc4, mnemonic, Xi8 , AL, - "{$src, %al|al, $src}">; - def NAME#16i16 : BinOpAI<BaseOpc4, mnemonic, Xi16, AX, - "{$src, %ax|ax, $src}">; - def NAME#32i32 : BinOpAI<BaseOpc4, mnemonic, Xi32, EAX, - "{$src, %eax|eax, $src}">; - def NAME#64i32 : BinOpAI<BaseOpc4, mnemonic, Xi64, RAX, - "{$src, %rax|rax, $src}">; + } // Constraints = "$src1 = $dst" + + def NAME#8mr : BinOpMR_MF<BaseOpc, mnemonic, Xi8 , opnode>; + def NAME#16mr : BinOpMR_MF<BaseOpc, mnemonic, Xi16, opnode>; + def NAME#32mr : BinOpMR_MF<BaseOpc, mnemonic, Xi32, opnode>; + def NAME#64mr : BinOpMR_MF<BaseOpc, mnemonic, Xi64, opnode>; + + // NOTE: These are order specific, we want the mi8 forms to be listed + // first so that they are slightly preferred to the mi forms. + def NAME#16mi8 : BinOpMI8_MF<mnemonic, Xi16, MemMRM>; + def NAME#32mi8 : BinOpMI8_MF<mnemonic, Xi32, MemMRM>; + let Predicates = [In64BitMode] in + def NAME#64mi8 : BinOpMI8_MF<mnemonic, Xi64, MemMRM>; + + def NAME#8mi : BinOpMI_MF<0x80, mnemonic, Xi8 , opnode, MemMRM>; + def NAME#16mi : BinOpMI_MF<0x81, mnemonic, Xi16, opnode, MemMRM>; + def NAME#32mi : BinOpMI_MF<0x81, mnemonic, Xi32, opnode, MemMRM>; + let Predicates = [In64BitMode] in + def NAME#64mi32 : BinOpMI_MF<0x81, mnemonic, Xi64, opnode, MemMRM>; + + // These are for the disassembler since 0x82 opcode behaves like 0x80, but + // not in 64-bit mode. + let Predicates = [Not64BitMode] in { + let Constraints = "$src1 = $dst" in + def NAME#8ri8 : BinOpRI8_RF<0x82, mnemonic, Xi8, RegMRM>, DisassembleOnly; + def NAME#8mi8 : BinOpMI8_MF<mnemonic, Xi8, MemMRM>, DisassembleOnly; + } + + def NAME#8i8 : BinOpAI_AF<BaseOpc4, mnemonic, Xi8 , AL, + "{$src, %al|al, $src}">; + def NAME#16i16 : BinOpAI_AF<BaseOpc4, mnemonic, Xi16, AX, + "{$src, %ax|ax, $src}">; + def NAME#32i32 : BinOpAI_AF<BaseOpc4, mnemonic, Xi32, EAX, + "{$src, %eax|eax, $src}">; + def NAME#64i32 : BinOpAI_AF<BaseOpc4, mnemonic, Xi64, RAX, + "{$src, %rax|rax, $src}">; } /// ArithBinOp_RFF - This is an arithmetic binary operator where the pattern is @@ -866,80 +719,73 @@ multiclass ArithBinOp_RFF<bits<8> BaseOpc, bits<8> BaseOpc2, bits<8> BaseOpc4, string mnemonic, Format RegMRM, Format MemMRM, SDNode opnode, bit CommutableRR, bit ConvertibleToThreeAddress> { - let Uses = [EFLAGS], Defs = [EFLAGS] in { - let Constraints = "$src1 = $dst" in { - let isCommutable = CommutableRR in { - def NAME#8rr : BinOpRR_RFF<BaseOpc, mnemonic, Xi8 , opnode>; - let isConvertibleToThreeAddress = ConvertibleToThreeAddress in { - def NAME#16rr : BinOpRR_RFF<BaseOpc, mnemonic, Xi16, opnode>; - def NAME#32rr : BinOpRR_RFF<BaseOpc, mnemonic, Xi32, opnode>; - def NAME#64rr : BinOpRR_RFF<BaseOpc, mnemonic, Xi64, opnode>; - } // isConvertibleToThreeAddress - } // isCommutable - - def NAME#8rr_REV : BinOpRR_RFF_Rev<BaseOpc2, mnemonic, Xi8>; - def NAME#16rr_REV : BinOpRR_RFF_Rev<BaseOpc2, mnemonic, Xi16>; - def NAME#32rr_REV : BinOpRR_RFF_Rev<BaseOpc2, mnemonic, Xi32>; - def NAME#64rr_REV : BinOpRR_RFF_Rev<BaseOpc2, mnemonic, Xi64>; - - def NAME#8rm : BinOpRM_RFF<BaseOpc2, mnemonic, Xi8 , opnode>; - def NAME#16rm : BinOpRM_RFF<BaseOpc2, mnemonic, Xi16, opnode>; - def NAME#32rm : BinOpRM_RFF<BaseOpc2, mnemonic, Xi32, opnode>; - def NAME#64rm : BinOpRM_RFF<BaseOpc2, mnemonic, Xi64, opnode>; - - def NAME#8ri : BinOpRI_RFF<0x80, mnemonic, Xi8 , opnode, RegMRM>; - - let isConvertibleToThreeAddress = ConvertibleToThreeAddress, hasSideEffects = 0 in { - // NOTE: These are order specific, we want the ri8 forms to be listed - // first so that they are slightly preferred to the ri forms. - def NAME#16ri8 : BinOpRI8_RFF<0x82, mnemonic, Xi16, RegMRM>; - def NAME#32ri8 : BinOpRI8_RFF<0x82, mnemonic, Xi32, RegMRM>; - def NAME#64ri8 : BinOpRI8_RFF<0x82, mnemonic, Xi64, RegMRM>; - - def NAME#16ri : BinOpRI_RFF<0x80, mnemonic, Xi16, opnode, RegMRM>; - def NAME#32ri : BinOpRI_RFF<0x80, mnemonic, Xi32, opnode, RegMRM>; - def NAME#64ri32: BinOpRI_RFF<0x80, mnemonic, Xi64, opnode, RegMRM>; - } - } // Constraints = "$src1 = $dst" - - def NAME#8mr : BinOpMR_RMW_FF<BaseOpc, mnemonic, Xi8 , opnode>; - def NAME#16mr : BinOpMR_RMW_FF<BaseOpc, mnemonic, Xi16, opnode>; - def NAME#32mr : BinOpMR_RMW_FF<BaseOpc, mnemonic, Xi32, opnode>; - def NAME#64mr : BinOpMR_RMW_FF<BaseOpc, mnemonic, Xi64, opnode>; - - // NOTE: These are order specific, we want the mi8 forms to be listed - // first so that they are slightly preferred to the mi forms. - let mayLoad = 1, mayStore = 1, hasSideEffects = 0 in { - def NAME#16mi8 : BinOpMI8_RMW_FF<mnemonic, Xi16, MemMRM>; - def NAME#32mi8 : BinOpMI8_RMW_FF<mnemonic, Xi32, MemMRM>; - let Predicates = [In64BitMode] in - def NAME#64mi8 : BinOpMI8_RMW_FF<mnemonic, Xi64, MemMRM>; - - def NAME#8mi : BinOpMI_RMW_FF<0x80, mnemonic, Xi8 , opnode, MemMRM>; - def NAME#16mi : BinOpMI_RMW_FF<0x80, mnemonic, Xi16, opnode, MemMRM>; - def NAME#32mi : BinOpMI_RMW_FF<0x80, mnemonic, Xi32, opnode, MemMRM>; - let Predicates = [In64BitMode] in - def NAME#64mi32 : BinOpMI_RMW_FF<0x80, mnemonic, Xi64, opnode, MemMRM>; - } - - // These are for the disassembler since 0x82 opcode behaves like 0x80, but - // not in 64-bit mode. - let Predicates = [Not64BitMode], isCodeGenOnly = 1, ForceDisassemble = 1, - hasSideEffects = 0 in { - let Constraints = "$src1 = $dst" in - def NAME#8ri8 : BinOpRI8_RFF<0x82, mnemonic, Xi8, RegMRM>; - let mayLoad = 1, mayStore = 1 in - def NAME#8mi8 : BinOpMI8_RMW_FF<mnemonic, Xi8, MemMRM>; + let Constraints = "$src1 = $dst" in { + let isCommutable = CommutableRR in { + def NAME#8rr : BinOpRRF_RF<BaseOpc, mnemonic, Xi8 , opnode>; + let isConvertibleToThreeAddress = ConvertibleToThreeAddress in { + def NAME#16rr : BinOpRRF_RF<BaseOpc, mnemonic, Xi16, opnode>; + def NAME#32rr : BinOpRRF_RF<BaseOpc, mnemonic, Xi32, opnode>; + def NAME#64rr : BinOpRRF_RF<BaseOpc, mnemonic, Xi64, opnode>; + } // isConvertibleToThreeAddress + } // isCommutable + + def NAME#8rr_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi8>; + def NAME#16rr_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi16>; + def NAME#32rr_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi32>; + def NAME#64rr_REV : BinOpRRF_RF_Rev<BaseOpc2, mnemonic, Xi64>; + + def NAME#8rm : BinOpRMF_RF<BaseOpc2, mnemonic, Xi8 , opnode>; + def NAME#16rm : BinOpRMF_RF<BaseOpc2, mnemonic, Xi16, opnode>; + def NAME#32rm : BinOpRMF_RF<BaseOpc2, mnemonic, Xi32, opnode>; + def NAME#64rm : BinOpRMF_RF<BaseOpc2, mnemonic, Xi64, opnode>; + + def NAME#8ri : BinOpRIF_RF<0x80, mnemonic, Xi8 , opnode, RegMRM>; + let isConvertibleToThreeAddress = ConvertibleToThreeAddress in { + // NOTE: These are order specific, we want the ri8 forms to be listed + // first so that they are slightly preferred to the ri forms. + def NAME#16ri8 : BinOpRI8F_RF<0x83, mnemonic, Xi16, RegMRM>; + def NAME#32ri8 : BinOpRI8F_RF<0x83, mnemonic, Xi32, RegMRM>; + def NAME#64ri8 : BinOpRI8F_RF<0x83, mnemonic, Xi64, RegMRM>; + + def NAME#16ri : BinOpRIF_RF<0x81, mnemonic, Xi16, opnode, RegMRM>; + def NAME#32ri : BinOpRIF_RF<0x81, mnemonic, Xi32, opnode, RegMRM>; + def NAME#64ri32: BinOpRIF_RF<0x81, mnemonic, Xi64, opnode, RegMRM>; } - } // Uses = [EFLAGS], Defs = [EFLAGS] - - def NAME#8i8 : BinOpAI_RFF<BaseOpc4, mnemonic, Xi8 , AL, - "{$src, %al|al, $src}">; - def NAME#16i16 : BinOpAI_RFF<BaseOpc4, mnemonic, Xi16, AX, + } // Constraints = "$src1 = $dst" + + def NAME#8mr : BinOpMRF_MF<BaseOpc, mnemonic, Xi8 , opnode>; + def NAME#16mr : BinOpMRF_MF<BaseOpc, mnemonic, Xi16, opnode>; + def NAME#32mr : BinOpMRF_MF<BaseOpc, mnemonic, Xi32, opnode>; + def NAME#64mr : BinOpMRF_MF<BaseOpc, mnemonic, Xi64, opnode>; + + // NOTE: These are order specific, we want the mi8 forms to be listed + // first so that they are slightly preferred to the mi forms. + def NAME#16mi8 : BinOpMI8F_MF<mnemonic, Xi16, MemMRM>; + def NAME#32mi8 : BinOpMI8F_MF<mnemonic, Xi32, MemMRM>; + let Predicates = [In64BitMode] in + def NAME#64mi8 : BinOpMI8F_MF<mnemonic, Xi64, MemMRM>; + + def NAME#8mi : BinOpMIF_MF<0x80, mnemonic, Xi8 , opnode, MemMRM>; + def NAME#16mi : BinOpMIF_MF<0x81, mnemonic, Xi16, opnode, MemMRM>; + def NAME#32mi : BinOpMIF_MF<0x81, mnemonic, Xi32, opnode, MemMRM>; + let Predicates = [In64BitMode] in + def NAME#64mi32 : BinOpMIF_MF<0x81, mnemonic, Xi64, opnode, MemMRM>; + + // These are for the disassembler since 0x82 opcode behaves like 0x80, but + // not in 64-bit mode. + let Predicates = [Not64BitMode] in { + let Constraints = "$src1 = $dst" in + def NAME#8ri8 : BinOpRI8F_RF<0x82, mnemonic, Xi8, RegMRM>, DisassembleOnly; + def NAME#8mi8 : BinOpMI8F_MF<mnemonic, Xi8, MemMRM>, DisassembleOnly; + } + + def NAME#8i8 : BinOpAIF_AF<BaseOpc4, mnemonic, Xi8 , AL, + "{$src, %al|al, $src}">; + def NAME#16i16 : BinOpAIF_AF<BaseOpc4, mnemonic, Xi16, AX, "{$src, %ax|ax, $src}">; - def NAME#32i32 : BinOpAI_RFF<BaseOpc4, mnemonic, Xi32, EAX, + def NAME#32i32 : BinOpAIF_AF<BaseOpc4, mnemonic, Xi32, EAX, "{$src, %eax|eax, $src}">; - def NAME#64i32 : BinOpAI_RFF<BaseOpc4, mnemonic, Xi64, RAX, + def NAME#64i32 : BinOpAIF_AF<BaseOpc4, mnemonic, Xi64, RAX, "{$src, %rax|rax, $src}">; } @@ -949,93 +795,88 @@ multiclass ArithBinOp_RFF<bits<8> BaseOpc, bits<8> BaseOpc2, bits<8> BaseOpc4, /// multiclass ArithBinOp_F<bits<8> BaseOpc, bits<8> BaseOpc2, bits<8> BaseOpc4, string mnemonic, Format RegMRM, Format MemMRM, - SDNode opnode, - bit CommutableRR, bit ConvertibleToThreeAddress> { - let Defs = [EFLAGS] in { - let isCommutable = CommutableRR in { - def NAME#8rr : BinOpRR_F<BaseOpc, mnemonic, Xi8 , opnode>; - let isConvertibleToThreeAddress = ConvertibleToThreeAddress in { - def NAME#16rr : BinOpRR_F<BaseOpc, mnemonic, Xi16, opnode>; - def NAME#32rr : BinOpRR_F<BaseOpc, mnemonic, Xi32, opnode>; - def NAME#64rr : BinOpRR_F<BaseOpc, mnemonic, Xi64, opnode>; - } - } // isCommutable - - def NAME#8rr_REV : BinOpRR_F_Rev<BaseOpc2, mnemonic, Xi8>; - def NAME#16rr_REV : BinOpRR_F_Rev<BaseOpc2, mnemonic, Xi16>; - def NAME#32rr_REV : BinOpRR_F_Rev<BaseOpc2, mnemonic, Xi32>; - def NAME#64rr_REV : BinOpRR_F_Rev<BaseOpc2, mnemonic, Xi64>; - - def NAME#8rm : BinOpRM_F<BaseOpc2, mnemonic, Xi8 , opnode>; - def NAME#16rm : BinOpRM_F<BaseOpc2, mnemonic, Xi16, opnode>; - def NAME#32rm : BinOpRM_F<BaseOpc2, mnemonic, Xi32, opnode>; - def NAME#64rm : BinOpRM_F<BaseOpc2, mnemonic, Xi64, opnode>; - - def NAME#8ri : BinOpRI_F<0x80, mnemonic, Xi8 , opnode, RegMRM>; - - let isConvertibleToThreeAddress = ConvertibleToThreeAddress, hasSideEffects = 0 in { - // NOTE: These are order specific, we want the ri8 forms to be listed - // first so that they are slightly preferred to the ri forms. - def NAME#16ri8 : BinOpRI8_F<0x82, mnemonic, Xi16, RegMRM>; - def NAME#32ri8 : BinOpRI8_F<0x82, mnemonic, Xi32, RegMRM>; - def NAME#64ri8 : BinOpRI8_F<0x82, mnemonic, Xi64, RegMRM>; - - def NAME#16ri : BinOpRI_F<0x80, mnemonic, Xi16, opnode, RegMRM>; - def NAME#32ri : BinOpRI_F<0x80, mnemonic, Xi32, opnode, RegMRM>; - def NAME#64ri32: BinOpRI_F<0x80, mnemonic, Xi64, opnode, RegMRM>; - } - - def NAME#8mr : BinOpMR_F<BaseOpc, mnemonic, Xi8 , opnode>; - def NAME#16mr : BinOpMR_F<BaseOpc, mnemonic, Xi16, opnode>; - def NAME#32mr : BinOpMR_F<BaseOpc, mnemonic, Xi32, opnode>; - def NAME#64mr : BinOpMR_F<BaseOpc, mnemonic, Xi64, opnode>; - - // NOTE: These are order specific, we want the mi8 forms to be listed - // first so that they are slightly preferred to the mi forms. - let mayLoad = 1, hasSideEffects = 0 in { - def NAME#16mi8 : BinOpMI8_F<mnemonic, Xi16, MemMRM>; - def NAME#32mi8 : BinOpMI8_F<mnemonic, Xi32, MemMRM>; - let Predicates = [In64BitMode] in - def NAME#64mi8 : BinOpMI8_F<mnemonic, Xi64, MemMRM>; - - def NAME#8mi : BinOpMI_F<0x80, mnemonic, Xi8 , opnode, MemMRM>; - def NAME#16mi : BinOpMI_F<0x80, mnemonic, Xi16, opnode, MemMRM>; - def NAME#32mi : BinOpMI_F<0x80, mnemonic, Xi32, opnode, MemMRM>; - let Predicates = [In64BitMode] in - def NAME#64mi32 : BinOpMI_F<0x80, mnemonic, Xi64, opnode, MemMRM>; - } - - // These are for the disassembler since 0x82 opcode behaves like 0x80, but - // not in 64-bit mode. - let Predicates = [Not64BitMode], isCodeGenOnly = 1, ForceDisassemble = 1, - hasSideEffects = 0 in { - def NAME#8ri8 : BinOpRI8_F<0x82, mnemonic, Xi8, RegMRM>; - let mayLoad = 1 in - def NAME#8mi8 : BinOpMI8_F<mnemonic, Xi8, MemMRM>; - } - } // Defs = [EFLAGS] - - def NAME#8i8 : BinOpAI_F<BaseOpc4, mnemonic, Xi8 , AL, - "{$src, %al|al, $src}">; + SDNode opnode, bit CommutableRR, + bit ConvertibleToThreeAddress> { + let isCommutable = CommutableRR in { + def NAME#8rr : BinOpRR_F<BaseOpc, mnemonic, Xi8 , opnode>; + let isConvertibleToThreeAddress = ConvertibleToThreeAddress in { + def NAME#16rr : BinOpRR_F<BaseOpc, mnemonic, Xi16, opnode>; + def NAME#32rr : BinOpRR_F<BaseOpc, mnemonic, Xi32, opnode>; + def NAME#64rr : BinOpRR_F<BaseOpc, mnemonic, Xi64, opnode>; + } // isConvertibleToThreeAddress + } // isCommutable + + def NAME#8rr_REV : BinOpRR_F_Rev<BaseOpc2, mnemonic, Xi8>; + def NAME#16rr_REV : BinOpRR_F_Rev<BaseOpc2, mnemonic, Xi16>; + def NAME#32rr_REV : BinOpRR_F_Rev<BaseOpc2, mnemonic, Xi32>; + def NAME#64rr_REV : BinOpRR_F_Rev<BaseOpc2, mnemonic, Xi64>; + + def NAME#8rm : BinOpRM_F<BaseOpc2, mnemonic, Xi8 , opnode>; + def NAME#16rm : BinOpRM_F<BaseOpc2, mnemonic, Xi16, opnode>; + def NAME#32rm : BinOpRM_F<BaseOpc2, mnemonic, Xi32, opnode>; + def NAME#64rm : BinOpRM_F<BaseOpc2, mnemonic, Xi64, opnode>; + + def NAME#8ri : BinOpRI_F<0x80, mnemonic, Xi8 , opnode, RegMRM>; + + let isConvertibleToThreeAddress = ConvertibleToThreeAddress in { + // NOTE: These are order specific, we want the ri8 forms to be listed + // first so that they are slightly preferred to the ri forms. + def NAME#16ri8 : BinOpRI8_F<0x83, mnemonic, Xi16, RegMRM>; + def NAME#32ri8 : BinOpRI8_F<0x83, mnemonic, Xi32, RegMRM>; + def NAME#64ri8 : BinOpRI8_F<0x83, mnemonic, Xi64, RegMRM>; + + def NAME#16ri : BinOpRI_F<0x81, mnemonic, Xi16, opnode, RegMRM>; + def NAME#32ri : BinOpRI_F<0x81, mnemonic, Xi32, opnode, RegMRM>; + def NAME#64ri32: BinOpRI_F<0x81, mnemonic, Xi64, opnode, RegMRM>; + } + + def NAME#8mr : BinOpMR_F<BaseOpc, mnemonic, Xi8 , opnode>; + def NAME#16mr : BinOpMR_F<BaseOpc, mnemonic, Xi16, opnode>; + def NAME#32mr : BinOpMR_F<BaseOpc, mnemonic, Xi32, opnode>; + def NAME#64mr : BinOpMR_F<BaseOpc, mnemonic, Xi64, opnode>; + + // NOTE: These are order specific, we want the mi8 forms to be listed + // first so that they are slightly preferred to the mi forms. + def NAME#16mi8 : BinOpMI8_F<mnemonic, Xi16, MemMRM>; + def NAME#32mi8 : BinOpMI8_F<mnemonic, Xi32, MemMRM>; + let Predicates = [In64BitMode] in + def NAME#64mi8 : BinOpMI8_F<mnemonic, Xi64, MemMRM>; + + def NAME#8mi : BinOpMI_F<0x80, mnemonic, Xi8 , opnode, MemMRM>; + def NAME#16mi : BinOpMI_F<0x81, mnemonic, Xi16, opnode, MemMRM>; + def NAME#32mi : BinOpMI_F<0x81, mnemonic, Xi32, opnode, MemMRM>; + let Predicates = [In64BitMode] in + def NAME#64mi32 : BinOpMI_F<0x81, mnemonic, Xi64, opnode, MemMRM>; + + // These are for the disassembler since 0x82 opcode behaves like 0x80, but + // not in 64-bit mode. + let Predicates = [Not64BitMode] in { + def NAME#8ri8 : BinOpRI8_F<0x82, mnemonic, Xi8, RegMRM>, DisassembleOnly; + let mayLoad = 1 in + def NAME#8mi8 : BinOpMI8_F<mnemonic, Xi8, MemMRM>; + } + + def NAME#8i8 : BinOpAI_F<BaseOpc4, mnemonic, Xi8 , AL, + "{$src, %al|al, $src}">; def NAME#16i16 : BinOpAI_F<BaseOpc4, mnemonic, Xi16, AX, - "{$src, %ax|ax, $src}">; + "{$src, %ax|ax, $src}">; def NAME#32i32 : BinOpAI_F<BaseOpc4, mnemonic, Xi32, EAX, - "{$src, %eax|eax, $src}">; + "{$src, %eax|eax, $src}">; def NAME#64i32 : BinOpAI_F<BaseOpc4, mnemonic, Xi64, RAX, - "{$src, %rax|rax, $src}">; + "{$src, %rax|rax, $src}">; } -defm AND : ArithBinOp_RF<0x20, 0x22, 0x24, "and", MRM4r, MRM4m, +defm AND : ArithBinOp_RF<0x21, 0x23, 0x25, "and", MRM4r, MRM4m, X86and_flag, and, 1, 0, 0>; -defm OR : ArithBinOp_RF<0x08, 0x0A, 0x0C, "or", MRM1r, MRM1m, +defm OR : ArithBinOp_RF<0x09, 0x0B, 0x0D, "or", MRM1r, MRM1m, X86or_flag, or, 1, 0, 0>; -defm XOR : ArithBinOp_RF<0x30, 0x32, 0x34, "xor", MRM6r, MRM6m, +defm XOR : ArithBinOp_RF<0x31, 0x33, 0x35, "xor", MRM6r, MRM6m, X86xor_flag, xor, 1, 0, 0>; -defm ADD : ArithBinOp_RF<0x00, 0x02, 0x04, "add", MRM0r, MRM0m, +defm ADD : ArithBinOp_RF<0x01, 0x03, 0x05, "add", MRM0r, MRM0m, X86add_flag, add, 1, 1, 1>; let isCompare = 1 in { -defm SUB : ArithBinOp_RF<0x28, 0x2A, 0x2C, "sub", MRM5r, MRM5m, +defm SUB : ArithBinOp_RF<0x29, 0x2B, 0x2D, "sub", MRM5r, MRM5m, X86sub_flag, sub, 0, 1, 0>; } @@ -1049,13 +890,13 @@ def XOR8rr_NOREX : I<0x30, MRMDestReg, (outs GR8_NOREX:$dst), Sched<[WriteALU]>; // Arithmetic. -defm ADC : ArithBinOp_RFF<0x10, 0x12, 0x14, "adc", MRM2r, MRM2m, X86adc_flag, +defm ADC : ArithBinOp_RFF<0x11, 0x13, 0x15, "adc", MRM2r, MRM2m, X86adc_flag, 1, 0>; -defm SBB : ArithBinOp_RFF<0x18, 0x1A, 0x1C, "sbb", MRM3r, MRM3m, X86sbb_flag, +defm SBB : ArithBinOp_RFF<0x19, 0x1B, 0x1D, "sbb", MRM3r, MRM3m, X86sbb_flag, 0, 0>; let isCompare = 1 in { -defm CMP : ArithBinOp_F<0x38, 0x3A, 0x3C, "cmp", MRM7r, MRM7m, X86cmp, 0, 0>; +defm CMP : ArithBinOp_F<0x39, 0x3B, 0x3D, "cmp", MRM7r, MRM7m, X86cmp, 0, 0>; } // Patterns to recognize loads on the LHS of an ADC. We can't make X86adc_flag @@ -1193,44 +1034,37 @@ def : Pat<(store (X86adc_flag i64relocImmSExt32_su:$src, (load addr:$dst), EFLAG // they don't have all the usual imm8 and REV forms, and are encoded into a // different space. let isCompare = 1 in { - let Defs = [EFLAGS] in { - let isCommutable = 1 in { - // Avoid selecting these and instead use a test+and. Post processing will - // combine them. This gives bunch of other patterns that start with - // and a chance to match. - def TEST8rr : BinOpRR_F<0x84, "test", Xi8 , null_frag>; - def TEST16rr : BinOpRR_F<0x84, "test", Xi16, null_frag>; - def TEST32rr : BinOpRR_F<0x84, "test", Xi32, null_frag>; - def TEST64rr : BinOpRR_F<0x84, "test", Xi64, null_frag>; - } // isCommutable - - let hasSideEffects = 0, mayLoad = 1 in { - def TEST8mr : BinOpMR_F<0x84, "test", Xi8 , null_frag>; - def TEST16mr : BinOpMR_F<0x84, "test", Xi16, null_frag>; - def TEST32mr : BinOpMR_F<0x84, "test", Xi32, null_frag>; - def TEST64mr : BinOpMR_F<0x84, "test", Xi64, null_frag>; - } - - def TEST8ri : BinOpRI_F<0xF6, "test", Xi8 , X86testpat, MRM0r>; - def TEST16ri : BinOpRI_F<0xF6, "test", Xi16, X86testpat, MRM0r>; - def TEST32ri : BinOpRI_F<0xF6, "test", Xi32, X86testpat, MRM0r>; - def TEST64ri32 : BinOpRI_F<0xF6, "test", Xi64, X86testpat, MRM0r>; - - def TEST8mi : BinOpMI_F<0xF6, "test", Xi8 , X86testpat, MRM0m>; - def TEST16mi : BinOpMI_F<0xF6, "test", Xi16, X86testpat, MRM0m>; - def TEST32mi : BinOpMI_F<0xF6, "test", Xi32, X86testpat, MRM0m>; - let Predicates = [In64BitMode] in - def TEST64mi32 : BinOpMI_F<0xF6, "test", Xi64, X86testpat, MRM0m>; - } // Defs = [EFLAGS] - - def TEST8i8 : BinOpAI_F<0xA8, "test", Xi8 , AL, - "{$src, %al|al, $src}">; - def TEST16i16 : BinOpAI_F<0xA8, "test", Xi16, AX, - "{$src, %ax|ax, $src}">; - def TEST32i32 : BinOpAI_F<0xA8, "test", Xi32, EAX, - "{$src, %eax|eax, $src}">; - def TEST64i32 : BinOpAI_F<0xA8, "test", Xi64, RAX, - "{$src, %rax|rax, $src}">; + let isCommutable = 1 in { + // Avoid selecting these and instead use a test+and. Post processing will + // combine them. This gives bunch of other patterns that start with + // and a chance to match. + def TEST8rr : BinOpRR_F<0x84, "test", Xi8 , null_frag>; + def TEST16rr : BinOpRR_F<0x85, "test", Xi16, null_frag>; + def TEST32rr : BinOpRR_F<0x85, "test", Xi32, null_frag>; + def TEST64rr : BinOpRR_F<0x85, "test", Xi64, null_frag>; + } // isCommutable + +def TEST8mr : BinOpMR_F<0x84, "test", Xi8 , null_frag>; +def TEST16mr : BinOpMR_F<0x85, "test", Xi16, null_frag>; +def TEST32mr : BinOpMR_F<0x85, "test", Xi32, null_frag>; +def TEST64mr : BinOpMR_F<0x85, "test", Xi64, null_frag>; + +def TEST8ri : BinOpRI_F<0xF6, "test", Xi8 , X86testpat, MRM0r>; +def TEST16ri : BinOpRI_F<0xF7, "test", Xi16, X86testpat, MRM0r>; +def TEST32ri : BinOpRI_F<0xF7, "test", Xi32, X86testpat, MRM0r>; +def TEST64ri32 : BinOpRI_F<0xF7, "test", Xi64, X86testpat, MRM0r>; + +def TEST8mi : BinOpMI_F<0xF6, "test", Xi8 , X86testpat, MRM0m>; +def TEST16mi : BinOpMI_F<0xF7, "test", Xi16, X86testpat, MRM0m>; +def TEST32mi : BinOpMI_F<0xF7, "test", Xi32, X86testpat, MRM0m>; + + let Predicates = [In64BitMode] in + def TEST64mi32 : BinOpMI_F<0xF7, "test", Xi64, X86testpat, MRM0m>; + +def TEST8i8 : BinOpAI_F<0xA8, "test", Xi8 , AL, "{$src, %al|al, $src}">; +def TEST16i16 : BinOpAI_F<0xA9, "test", Xi16, AX, "{$src, %ax|ax, $src}">; +def TEST32i32 : BinOpAI_F<0xA9, "test", Xi32, EAX, "{$src, %eax|eax, $src}">; +def TEST64i32 : BinOpAI_F<0xA9, "test", Xi64, RAX, "{$src, %rax|rax, $src}">; } // isCompare // Patterns to match a relocImm into the immediate field. @@ -1354,38 +1188,25 @@ let Uses = [RDX] in // // We don't have patterns for these as there is no advantage over ADC for // most code. -class ADCOXOpRR <bits<8> opcode, string mnemonic, X86TypeInfo info> - : BinOpRR_C<opcode, MRMSrcReg, mnemonic, info, []>{ - let Opcode = opcode; - let OpSize = OpSizeFixed; +class ADCOXOpRR <string m, X86TypeInfo t> + : BinOpRRF_RF<0xF6, m, t, null_frag> { + let Form = MRMSrcReg; + let isCommutable = 1; } -class ADCOXOpRM <bits<8> opcode, string mnemonic, X86TypeInfo info> - : BinOpRM_C<opcode, MRMSrcMem, mnemonic, info, []>{ - let Opcode = opcode; - let OpSize = OpSizeFixed; +class ADCOXOpRM <string m, X86TypeInfo t> + : BinOpRMF_RF<0xF6, m, t, null_frag> { + let Form = MRMSrcMem; } -let Predicates = [HasADX], Defs = [EFLAGS], Uses = [EFLAGS], - Constraints = "$src1 = $dst", hasSideEffects = 0 in { - let SchedRW = [WriteADC], isCommutable = 1 in { - def ADCX32rr : ADCOXOpRR<0xF6, "adcx", Xi32>, T8PD; - def ADCX64rr : ADCOXOpRR<0xF6, "adcx", Xi64>, T8PD; - - def ADOX32rr : ADCOXOpRR<0xF6, "adox", Xi32>, T8XS; - def ADOX64rr : ADCOXOpRR<0xF6, "adox", Xi64>, T8XS; - } // SchedRW - - let mayLoad = 1, - SchedRW = [WriteADC.Folded, WriteADC.ReadAfterFold, - // Memory operand. - ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault, - // Implicit read of EFLAGS - WriteADC.ReadAfterFold] in { - def ADCX32rm : ADCOXOpRM<0xF6, "adcx", Xi32>, T8PD; - def ADCX64rm : ADCOXOpRM<0xF6, "adcx", Xi64>, T8PD; - - def ADOX32rm : ADCOXOpRM<0xF6, "adox", Xi32>, T8XS; - def ADOX64rm : ADCOXOpRM<0xF6, "adox", Xi64>, T8XS; - } // mayLoad, SchedRW +let OpSize = OpSizeFixed, Constraints = "$src1 = $dst", + Predicates = [HasADX] in { +def ADCX32rr : ADCOXOpRR<"adcx", Xi32>, T8PD; +def ADCX64rr : ADCOXOpRR<"adcx", Xi64>, T8PD; +def ADOX32rr : ADCOXOpRR<"adox", Xi32>, T8XS; +def ADOX64rr : ADCOXOpRR<"adox", Xi64>, T8XS; +def ADCX32rm : ADCOXOpRM<"adcx", Xi32>, T8PD; +def ADCX64rm : ADCOXOpRM<"adcx", Xi64>, T8PD; +def ADOX32rm : ADCOXOpRM<"adox", Xi32>, T8XS; +def ADOX64rm : ADCOXOpRM<"adox", Xi64>, T8XS; } diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td index 457833f8cc33..c77c77ee4a3e 100644 --- a/llvm/lib/Target/X86/X86InstrCompiler.td +++ b/llvm/lib/Target/X86/X86InstrCompiler.td @@ -1515,6 +1515,23 @@ def : Pat<(X86add_flag_nocf GR32:$src1, 128), def : Pat<(X86add_flag_nocf GR64:$src1, 128), (SUB64ri32 GR64:$src1, -128)>; +// Depositing value to 8/16 bit subreg: +def : Pat<(or (and GR64:$dst, -256), + (i64 (zextloadi8 addr:$src))), + (INSERT_SUBREG (i64 (COPY $dst)), (MOV8rm i8mem:$src), sub_8bit)>; + +def : Pat<(or (and GR32:$dst, -256), + (i32 (zextloadi8 addr:$src))), + (INSERT_SUBREG (i32 (COPY $dst)), (MOV8rm i8mem:$src), sub_8bit)>; + +def : Pat<(or (and GR64:$dst, -65536), + (i64 (zextloadi16 addr:$src))), + (INSERT_SUBREG (i64 (COPY $dst)), (MOV16rm i16mem:$src), sub_16bit)>; + +def : Pat<(or (and GR32:$dst, -65536), + (i32 (zextloadi16 addr:$src))), + (INSERT_SUBREG (i32 (COPY $dst)), (MOV16rm i16mem:$src), sub_16bit)>; + // The same trick applies for 32-bit immediate fields in 64-bit // instructions. def : Pat<(add GR64:$src1, 0x0000000080000000), diff --git a/llvm/lib/Target/X86/X86InstrUtils.td b/llvm/lib/Target/X86/X86InstrUtils.td index 3d8f4e642aeb..a94efd2b1a05 100644 --- a/llvm/lib/Target/X86/X86InstrUtils.td +++ b/llvm/lib/Target/X86/X86InstrUtils.td @@ -116,6 +116,14 @@ class NotEVEX2VEXConvertible { bit notEVEX2VEXConvertible = 1; } class ExplicitREX2Prefix { ExplicitOpPrefix explicitOpPrefix = ExplicitREX2; } class ExplicitVEXPrefix { ExplicitOpPrefix explicitOpPrefix = ExplicitVEX; } class ExplicitEVEXPrefix { ExplicitOpPrefix explicitOpPrefix = ExplicitEVEX; } +class DefEFLAGS { list<Register> Defs = [EFLAGS]; } +class UseEFLAGS { list<Register> Uses = [EFLAGS]; } +class DisassembleOnly { + // The disassembler should know about this, but not the asmparser. + bit isCodeGenOnly = 1; + bit ForceDisassemble = 1; +} + // SchedModel info for instruction that loads one value and gets the second // (and possibly third) value from a register. @@ -139,7 +147,7 @@ class X86TypeInfo<ValueType vt, string instrsuffix, RegisterClass regclass, Operand immoperand, SDPatternOperator immoperator, SDPatternOperator immnosuoperator, Operand imm8operand, SDPatternOperator imm8operator, SDPatternOperator imm8nosuoperator, - bit hasOddOpcode, OperandSize opSize, + bit hasEvenOpcode, OperandSize opSize, bit hasREX_W> { /// VT - This is the value type itself. ValueType VT = vt; @@ -189,10 +197,10 @@ class X86TypeInfo<ValueType vt, string instrsuffix, RegisterClass regclass, SDPatternOperator Imm8NoSuOperator = imm8nosuoperator; - /// HasOddOpcode - This bit is true if the instruction should have an odd (as - /// opposed to even) opcode. Operations on i8 are usually even, operations on - /// other datatypes are odd. - bit HasOddOpcode = hasOddOpcode; + /// HasEvenOpcode - This bit is true if the instruction should have an even (as + /// opposed to odd) opcode. Operations on i8 are even, operations on + /// other datatypes are usually odd. + bit HasEvenOpcode = hasEvenOpcode; /// OpSize - Selects whether the instruction needs a 0x66 prefix based on /// 16-bit vs 32-bit mode. i8/i64 set this to OpSizeFixed. i16 sets this @@ -208,16 +216,16 @@ def invalid_node : SDNode<"<<invalid_node>>", SDTIntLeaf,[],"<<invalid_node>>">; def Xi8 : X86TypeInfo<i8, "b", GR8, loadi8, i8mem, Imm8, i8imm, imm_su, imm, i8imm, invalid_node, invalid_node, - 0, OpSizeFixed, 0>; + 1, OpSizeFixed, 0>; def Xi16 : X86TypeInfo<i16, "w", GR16, loadi16, i16mem, Imm16, i16imm, imm_su, imm, i16i8imm, i16immSExt8_su, i16immSExt8, - 1, OpSize16, 0>; + 0, OpSize16, 0>; def Xi32 : X86TypeInfo<i32, "l", GR32, loadi32, i32mem, Imm32, i32imm, imm_su, imm, i32i8imm, i32immSExt8_su, i32immSExt8, - 1, OpSize32, 0>; + 0, OpSize32, 0>; def Xi64 : X86TypeInfo<i64, "q", GR64, loadi64, i64mem, Imm32S, i64i32imm, i64immSExt32_su, i64immSExt32, i64i8imm, i64immSExt8_su, - i64immSExt8, 1, OpSizeFixed, 1>; + i64immSExt8, 0, OpSizeFixed, 1>; // Group template arguments that can be derived from the vector type (EltNum x // EltVT). These are things like the register class for the writemask, etc. @@ -984,10 +992,11 @@ class MMXIi8<bits<8> o, Format F, dag outs, dag ins, string asm, class ITy<bits<8> opcode, Format f, X86TypeInfo typeinfo, dag outs, dag ins, string mnemonic, string args, list<dag> pattern> : I<{opcode{7}, opcode{6}, opcode{5}, opcode{4}, - opcode{3}, opcode{2}, opcode{1}, typeinfo.HasOddOpcode }, - f, outs, ins, + opcode{3}, opcode{2}, opcode{1}, + !if(!eq(typeinfo.HasEvenOpcode, 1), 0, opcode{0})}, f, outs, ins, !strconcat(mnemonic, "{", typeinfo.InstrSuffix, "}\t", args), pattern> { + let hasSideEffects = 0; // Infer instruction prefixes from type info. let OpSize = typeinfo.OpSize; let hasREX_W = typeinfo.HasREX_W; diff --git a/llvm/lib/TargetParser/ARMTargetParserCommon.cpp b/llvm/lib/TargetParser/ARMTargetParserCommon.cpp index 10b80cad4347..6d3a59d532fd 100644 --- a/llvm/lib/TargetParser/ARMTargetParserCommon.cpp +++ b/llvm/lib/TargetParser/ARMTargetParserCommon.cpp @@ -134,13 +134,13 @@ ARM::EndianKind ARM::parseArchEndian(StringRef Arch) { } // Parse a branch protection specification, which has the form -// standard | none | [bti,pac-ret[+b-key,+leaf]*] +// standard | none | [bti,pac-ret[+b-key,+leaf,+pc]*] // Returns true on success, with individual elements of the specification // returned in `PBP`. Returns false in error, with `Err` containing // an erroneous part of the spec. bool ARM::parseBranchProtection(StringRef Spec, ParsedBranchProtection &PBP, StringRef &Err) { - PBP = {"none", "a_key", false}; + PBP = {"none", "a_key", false, false}; if (Spec == "none") return true; // defaults are ok @@ -166,6 +166,8 @@ bool ARM::parseBranchProtection(StringRef Spec, ParsedBranchProtection &PBP, PBP.Scope = "all"; else if (PACOpt == "b-key") PBP.Key = "b_key"; + else if (PACOpt == "pc") + PBP.BranchProtectionPAuthLR = true; else break; } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index c496f9c7419b..a272357fa04a 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -1539,6 +1539,9 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { if (Instruction *I = foldCommutativeIntrinsicOverSelects(*II)) return I; + if (Instruction *I = foldCommutativeIntrinsicOverPhis(*II)) + return I; + if (CallInst *NewCall = canonicalizeConstantArg0ToArg1(CI)) return NewCall; } @@ -4237,3 +4240,22 @@ InstCombinerImpl::foldCommutativeIntrinsicOverSelects(IntrinsicInst &II) { return nullptr; } + +Instruction * +InstCombinerImpl::foldCommutativeIntrinsicOverPhis(IntrinsicInst &II) { + assert(II.isCommutative() && "Instruction should be commutative"); + + PHINode *LHS = dyn_cast<PHINode>(II.getOperand(0)); + PHINode *RHS = dyn_cast<PHINode>(II.getOperand(1)); + + if (!LHS || !RHS) + return nullptr; + + if (auto P = matchSymmetricPhiNodesPair(LHS, RHS)) { + replaceOperand(II, 0, P->first); + replaceOperand(II, 1, P->second); + return &II; + } + + return nullptr; +} diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp index 0ad87eeb4c91..289976718e52 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -4624,35 +4624,27 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I, } bool NoOp0WrapProblem = false, NoOp1WrapProblem = false; - bool Op0HasNUW = false, Op1HasNUW = false; - bool Op0HasNSW = false, Op1HasNSW = false; + if (BO0 && isa<OverflowingBinaryOperator>(BO0)) + NoOp0WrapProblem = + ICmpInst::isEquality(Pred) || + (CmpInst::isUnsigned(Pred) && BO0->hasNoUnsignedWrap()) || + (CmpInst::isSigned(Pred) && BO0->hasNoSignedWrap()); + if (BO1 && isa<OverflowingBinaryOperator>(BO1)) + NoOp1WrapProblem = + ICmpInst::isEquality(Pred) || + (CmpInst::isUnsigned(Pred) && BO1->hasNoUnsignedWrap()) || + (CmpInst::isSigned(Pred) && BO1->hasNoSignedWrap()); + // Analyze the case when either Op0 or Op1 is an add instruction. // Op0 = A + B (or A and B are null); Op1 = C + D (or C and D are null). - auto hasNoWrapProblem = [](const BinaryOperator &BO, CmpInst::Predicate Pred, - bool &HasNSW, bool &HasNUW) -> bool { - if (isa<OverflowingBinaryOperator>(BO)) { - HasNUW = BO.hasNoUnsignedWrap(); - HasNSW = BO.hasNoSignedWrap(); - return ICmpInst::isEquality(Pred) || - (CmpInst::isUnsigned(Pred) && HasNUW) || - (CmpInst::isSigned(Pred) && HasNSW); - } else if (BO.getOpcode() == Instruction::Or) { - HasNUW = true; - HasNSW = true; - return true; - } else { - return false; - } - }; Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr; - - if (BO0) { - match(BO0, m_AddLike(m_Value(A), m_Value(B))); - NoOp0WrapProblem = hasNoWrapProblem(*BO0, Pred, Op0HasNSW, Op0HasNUW); + if (BO0 && BO0->getOpcode() == Instruction::Add) { + A = BO0->getOperand(0); + B = BO0->getOperand(1); } - if (BO1) { - match(BO1, m_AddLike(m_Value(C), m_Value(D))); - NoOp1WrapProblem = hasNoWrapProblem(*BO1, Pred, Op1HasNSW, Op1HasNUW); + if (BO1 && BO1->getOpcode() == Instruction::Add) { + C = BO1->getOperand(0); + D = BO1->getOperand(1); } // icmp (A+B), A -> icmp B, 0 for equalities or if there is no overflow. @@ -4772,15 +4764,17 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I, APInt AP2Abs = AP2->abs(); if (AP1Abs.uge(AP2Abs)) { APInt Diff = *AP1 - *AP2; + bool HasNUW = BO0->hasNoUnsignedWrap() && Diff.ule(*AP1); + bool HasNSW = BO0->hasNoSignedWrap(); Constant *C3 = Constant::getIntegerValue(BO0->getType(), Diff); - Value *NewAdd = Builder.CreateAdd( - A, C3, "", Op0HasNUW && Diff.ule(*AP1), Op0HasNSW); + Value *NewAdd = Builder.CreateAdd(A, C3, "", HasNUW, HasNSW); return new ICmpInst(Pred, NewAdd, C); } else { APInt Diff = *AP2 - *AP1; + bool HasNUW = BO1->hasNoUnsignedWrap() && Diff.ule(*AP2); + bool HasNSW = BO1->hasNoSignedWrap(); Constant *C3 = Constant::getIntegerValue(BO0->getType(), Diff); - Value *NewAdd = Builder.CreateAdd( - C, C3, "", Op1HasNUW && Diff.ule(*AP1), Op1HasNSW); + Value *NewAdd = Builder.CreateAdd(C, C3, "", HasNUW, HasNSW); return new ICmpInst(Pred, A, NewAdd); } } @@ -4874,14 +4868,16 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I, isKnownNonZero(Z, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT); // if Z != 0 and nsw(X * Z) and nsw(Y * Z) // X * Z eq/ne Y * Z -> X eq/ne Y - if (NonZero && BO0 && BO1 && Op0HasNSW && Op1HasNSW) + if (NonZero && BO0 && BO1 && BO0->hasNoSignedWrap() && + BO1->hasNoSignedWrap()) return new ICmpInst(Pred, X, Y); } else NonZero = isKnownNonZero(Z, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT); // If Z != 0 and nuw(X * Z) and nuw(Y * Z) // X * Z u{lt/le/gt/ge}/eq/ne Y * Z -> X u{lt/le/gt/ge}/eq/ne Y - if (NonZero && BO0 && BO1 && Op0HasNUW && Op1HasNUW) + if (NonZero && BO0 && BO1 && BO0->hasNoUnsignedWrap() && + BO1->hasNoUnsignedWrap()) return new ICmpInst(Pred, X, Y); } } @@ -4980,8 +4976,8 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I, return new ICmpInst(Pred, BO0->getOperand(0), BO1->getOperand(0)); case Instruction::Shl: { - bool NUW = Op0HasNUW && Op1HasNUW; - bool NSW = Op0HasNSW && Op1HasNSW; + bool NUW = BO0->hasNoUnsignedWrap() && BO1->hasNoUnsignedWrap(); + bool NSW = BO0->hasNoSignedWrap() && BO1->hasNoSignedWrap(); if (!NUW && !NSW) break; if (!NSW && I.isSigned()) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h index f86db698ef8f..9e76a0cf17b1 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h +++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h @@ -278,6 +278,16 @@ private: IntrinsicInst &Tramp); Instruction *foldCommutativeIntrinsicOverSelects(IntrinsicInst &II); + // Match a pair of Phi Nodes like + // phi [a, BB0], [b, BB1] & phi [b, BB0], [a, BB1] + // Return the matched two operands. + std::optional<std::pair<Value *, Value *>> + matchSymmetricPhiNodesPair(PHINode *LHS, PHINode *RHS); + + // Tries to fold (op phi(a, b) phi(b, a)) -> (op a, b) + // while op is a commutative intrinsic call. + Instruction *foldCommutativeIntrinsicOverPhis(IntrinsicInst &II); + Value *simplifyMaskedLoad(IntrinsicInst &II); Instruction *simplifyMaskedStore(IntrinsicInst &II); Instruction *simplifyMaskedGather(IntrinsicInst &II); @@ -492,6 +502,11 @@ public: /// X % (C0 * C1) Value *SimplifyAddWithRemainder(BinaryOperator &I); + // Tries to fold (Binop phi(a, b) phi(b, a)) -> (Binop a, b) + // while Binop is commutative. + Value *SimplifyPhiCommutativeBinaryOp(BinaryOperator &I, Value *LHS, + Value *RHS); + // Binary Op helper for select operations where the expression can be // efficiently reorganized. Value *SimplifySelectsFeedingBinaryOp(BinaryOperator &I, Value *LHS, diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp index e5566578869d..f0ea3d9fcad5 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp @@ -350,6 +350,13 @@ Instruction *InstCombinerImpl::visitMul(BinaryOperator &I) { if (match(&I, m_c_Mul(m_OneUse(m_Neg(m_Value(X))), m_Value(Y)))) return BinaryOperator::CreateNeg(Builder.CreateMul(X, Y)); + // (-X * Y) * -X --> (X * Y) * X + // (-X << Y) * -X --> (X << Y) * X + if (match(Op1, m_Neg(m_Value(X)))) { + if (Value *NegOp0 = Negator::Negate(false, /*IsNSW*/ false, Op0, *this)) + return BinaryOperator::CreateMul(NegOp0, X); + } + // (X / Y) * Y = X - (X % Y) // (X / Y) * -Y = (X % Y) - X { diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index 4188b5b46e87..7f5a7b666903 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -1096,6 +1096,54 @@ Value *InstCombinerImpl::foldUsingDistributiveLaws(BinaryOperator &I) { return SimplifySelectsFeedingBinaryOp(I, LHS, RHS); } +std::optional<std::pair<Value *, Value *>> +InstCombinerImpl::matchSymmetricPhiNodesPair(PHINode *LHS, PHINode *RHS) { + if (LHS->getParent() != RHS->getParent()) + return std::nullopt; + + if (LHS->getNumIncomingValues() < 2) + return std::nullopt; + + if (!equal(LHS->blocks(), RHS->blocks())) + return std::nullopt; + + Value *L0 = LHS->getIncomingValue(0); + Value *R0 = RHS->getIncomingValue(0); + + for (unsigned I = 1, E = LHS->getNumIncomingValues(); I != E; ++I) { + Value *L1 = LHS->getIncomingValue(I); + Value *R1 = RHS->getIncomingValue(I); + + if ((L0 == L1 && R0 == R1) || (L0 == R1 && R0 == L1)) + continue; + + return std::nullopt; + } + + return std::optional(std::pair(L0, R0)); +} + +Value *InstCombinerImpl::SimplifyPhiCommutativeBinaryOp(BinaryOperator &I, + Value *Op0, + Value *Op1) { + assert(I.isCommutative() && "Instruction should be commutative"); + + PHINode *LHS = dyn_cast<PHINode>(Op0); + PHINode *RHS = dyn_cast<PHINode>(Op1); + + if (!LHS || !RHS) + return nullptr; + + if (auto P = matchSymmetricPhiNodesPair(LHS, RHS)) { + Value *BI = Builder.CreateBinOp(I.getOpcode(), P->first, P->second); + if (auto *BO = dyn_cast<BinaryOperator>(BI)) + BO->copyIRFlags(&I); + return BI; + } + + return nullptr; +} + Value *InstCombinerImpl::SimplifySelectsFeedingBinaryOp(BinaryOperator &I, Value *LHS, Value *RHS) { @@ -1529,6 +1577,11 @@ Instruction *InstCombinerImpl::foldBinopWithPhiOperands(BinaryOperator &BO) { BO.getParent() != Phi1->getParent()) return nullptr; + if (BO.isCommutative()) { + if (Value *V = SimplifyPhiCommutativeBinaryOp(BO, Phi0, Phi1)) + return replaceInstUsesWith(BO, V); + } + // Fold if there is at least one specific constant value in phi0 or phi1's // incoming values that comes from the same block and this specific constant // value can be used to do optimization for specific binary operator. @@ -2463,7 +2516,7 @@ Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) { Idx2); } ConstantInt *C; - if (match(GEP.getOperand(1), m_OneUse(m_SExt(m_OneUse(m_NSWAdd( + if (match(GEP.getOperand(1), m_OneUse(m_SExtLike(m_OneUse(m_NSWAdd( m_Value(Idx1), m_ConstantInt(C))))))) { // %add = add nsw i32 %idx1, idx2 // %sidx = sext i32 %add to i64 diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp index f14541a1a037..7cfeb019af97 100644 --- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp +++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp @@ -200,6 +200,7 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences( UP.Count = 0; UP.DefaultUnrollRuntimeCount = 8; UP.MaxCount = std::numeric_limits<unsigned>::max(); + UP.MaxUpperBound = UnrollMaxUpperBound; UP.FullUnrollMaxCount = std::numeric_limits<unsigned>::max(); UP.BEInsns = 2; UP.Partial = false; @@ -237,6 +238,8 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences( UP.MaxPercentThresholdBoost = UnrollMaxPercentThresholdBoost; if (UnrollMaxCount.getNumOccurrences() > 0) UP.MaxCount = UnrollMaxCount; + if (UnrollMaxUpperBound.getNumOccurrences() > 0) + UP.MaxUpperBound = UnrollMaxUpperBound; if (UnrollFullMaxCount.getNumOccurrences() > 0) UP.FullUnrollMaxCount = UnrollFullMaxCount; if (UnrollAllowPartial.getNumOccurrences() > 0) @@ -777,7 +780,7 @@ shouldPragmaUnroll(Loop *L, const PragmaInfo &PInfo, return TripCount; if (PInfo.PragmaEnableUnroll && !TripCount && MaxTripCount && - MaxTripCount <= UnrollMaxUpperBound) + MaxTripCount <= UP.MaxUpperBound) return MaxTripCount; // if didn't return until here, should continue to other priorties @@ -952,7 +955,7 @@ bool llvm::computeUnrollCount( // cost of exact full unrolling. As such, if we have an exact count and // found it unprofitable, we'll never chose to bounded unroll. if (!TripCount && MaxTripCount && (UP.UpperBound || MaxOrZero) && - MaxTripCount <= UnrollMaxUpperBound) { + MaxTripCount <= UP.MaxUpperBound) { UP.Count = MaxTripCount; if (auto UnrollFactor = shouldFullUnroll(L, TTI, DT, SE, EphValues, MaxTripCount, UCE, UP)) { @@ -1026,7 +1029,7 @@ bool llvm::computeUnrollCount( } // Don't unroll a small upper bound loop unless user or TTI asked to do so. - if (MaxTripCount && !UP.Force && MaxTripCount < UnrollMaxUpperBound) { + if (MaxTripCount && !UP.Force && MaxTripCount < UP.MaxUpperBound) { UP.Count = 0; return false; } diff --git a/llvm/test/Analysis/ValueTracking/pr75505.ll b/llvm/test/Analysis/ValueTracking/pr75505.ll new file mode 100644 index 000000000000..79368ff18e06 --- /dev/null +++ b/llvm/test/Analysis/ValueTracking/pr75505.ll @@ -0,0 +1,17 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -passes=instsimplify < %s | FileCheck %s + +; Just make sure that we don't assert. +define i32 @test(<2 x i16> %a, i32 %b) { +; CHECK-LABEL: define i32 @test( +; CHECK-SAME: <2 x i16> [[A:%.*]], i32 [[B:%.*]]) { +; CHECK-NEXT: [[MUL:%.*]] = mul <2 x i16> [[A]], <i16 -1, i16 poison> +; CHECK-NEXT: [[BC:%.*]] = bitcast <2 x i16> [[MUL]] to i32 +; CHECK-NEXT: [[LSHR:%.*]] = lshr i32 [[B]], [[BC]] +; CHECK-NEXT: ret i32 [[LSHR]] +; + %mul = mul <2 x i16> %a, <i16 -1, i16 poison> + %bc = bitcast <2 x i16> %mul to i32 + %lshr = lshr i32 %b, %bc + ret i32 %lshr +} diff --git a/llvm/test/CodeGen/AArch64/aarch64-minmaxv.ll b/llvm/test/CodeGen/AArch64/aarch64-minmaxv.ll index 5e477e8947d1..194fe5be40c2 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-minmaxv.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-minmaxv.ll @@ -516,20 +516,17 @@ define i8 @sminv_v4i8(<4 x i8> %a) { ; CHECK-GI-LABEL: sminv_v4i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: mov h3, v0.h[3] +; CHECK-GI-NEXT: umov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[1] +; CHECK-GI-NEXT: umov w10, v0.h[2] +; CHECK-GI-NEXT: umov w12, v0.h[3] +; CHECK-GI-NEXT: sxtb w11, w8 +; CHECK-GI-NEXT: cmp w11, w9, sxtb +; CHECK-GI-NEXT: sxtb w11, w10 +; CHECK-GI-NEXT: csel w8, w8, w9, lt +; CHECK-GI-NEXT: cmp w11, w12, sxtb ; CHECK-GI-NEXT: sxtb w9, w8 -; CHECK-GI-NEXT: fmov w10, s1 -; CHECK-GI-NEXT: fmov w11, s2 -; CHECK-GI-NEXT: cmp w9, w10, sxtb -; CHECK-GI-NEXT: sxtb w9, w11 -; CHECK-GI-NEXT: csel w8, w8, w10, lt -; CHECK-GI-NEXT: fmov w10, s3 -; CHECK-GI-NEXT: cmp w9, w10, sxtb -; CHECK-GI-NEXT: sxtb w9, w8 -; CHECK-GI-NEXT: csel w10, w11, w10, lt +; CHECK-GI-NEXT: csel w10, w10, w12, lt ; CHECK-GI-NEXT: cmp w9, w10, sxtb ; CHECK-GI-NEXT: csel w0, w8, w10, lt ; CHECK-GI-NEXT: ret @@ -611,19 +608,16 @@ define i16 @sminv_v3i16(<3 x i16> %a) { ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w9, s0 -; CHECK-GI-NEXT: sxth w8, w8 -; CHECK-GI-NEXT: fmov w10, s1 -; CHECK-GI-NEXT: fmov w11, s2 +; CHECK-GI-NEXT: smov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[0] +; CHECK-GI-NEXT: umov w10, v0.h[1] +; CHECK-GI-NEXT: smov w11, v0.h[2] +; CHECK-GI-NEXT: umov w13, v0.h[2] ; CHECK-GI-NEXT: fmov w12, s1 -; CHECK-GI-NEXT: cmp w8, w10, sxth -; CHECK-GI-NEXT: sxth w8, w11 -; CHECK-GI-NEXT: fmov w10, s2 -; CHECK-GI-NEXT: csel w9, w9, w12, lt -; CHECK-GI-NEXT: cmp w8, w9, sxth -; CHECK-GI-NEXT: csel w0, w9, w10, gt +; CHECK-GI-NEXT: cmp w8, w12, sxth +; CHECK-GI-NEXT: csel w8, w9, w10, lt +; CHECK-GI-NEXT: cmp w11, w8, sxth +; CHECK-GI-NEXT: csel w0, w8, w13, gt ; CHECK-GI-NEXT: ret entry: %arg1 = call i16 @llvm.vector.reduce.smin.v3i16(<3 x i16> %a) @@ -887,20 +881,17 @@ define i8 @smaxv_v4i8(<4 x i8> %a) { ; CHECK-GI-LABEL: smaxv_v4i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: mov h3, v0.h[3] -; CHECK-GI-NEXT: sxtb w9, w8 -; CHECK-GI-NEXT: fmov w10, s1 -; CHECK-GI-NEXT: fmov w11, s2 -; CHECK-GI-NEXT: cmp w9, w10, sxtb -; CHECK-GI-NEXT: sxtb w9, w11 -; CHECK-GI-NEXT: csel w8, w8, w10, gt -; CHECK-GI-NEXT: fmov w10, s3 -; CHECK-GI-NEXT: cmp w9, w10, sxtb +; CHECK-GI-NEXT: umov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[1] +; CHECK-GI-NEXT: umov w10, v0.h[2] +; CHECK-GI-NEXT: umov w12, v0.h[3] +; CHECK-GI-NEXT: sxtb w11, w8 +; CHECK-GI-NEXT: cmp w11, w9, sxtb +; CHECK-GI-NEXT: sxtb w11, w10 +; CHECK-GI-NEXT: csel w8, w8, w9, gt +; CHECK-GI-NEXT: cmp w11, w12, sxtb ; CHECK-GI-NEXT: sxtb w9, w8 -; CHECK-GI-NEXT: csel w10, w11, w10, gt +; CHECK-GI-NEXT: csel w10, w10, w12, gt ; CHECK-GI-NEXT: cmp w9, w10, sxtb ; CHECK-GI-NEXT: csel w0, w8, w10, gt ; CHECK-GI-NEXT: ret @@ -982,19 +973,16 @@ define i16 @smaxv_v3i16(<3 x i16> %a) { ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w9, s0 -; CHECK-GI-NEXT: sxth w8, w8 -; CHECK-GI-NEXT: fmov w10, s1 -; CHECK-GI-NEXT: fmov w11, s2 +; CHECK-GI-NEXT: smov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[0] +; CHECK-GI-NEXT: umov w10, v0.h[1] +; CHECK-GI-NEXT: smov w11, v0.h[2] +; CHECK-GI-NEXT: umov w13, v0.h[2] ; CHECK-GI-NEXT: fmov w12, s1 -; CHECK-GI-NEXT: cmp w8, w10, sxth -; CHECK-GI-NEXT: sxth w8, w11 -; CHECK-GI-NEXT: fmov w10, s2 -; CHECK-GI-NEXT: csel w9, w9, w12, gt -; CHECK-GI-NEXT: cmp w8, w9, sxth -; CHECK-GI-NEXT: csel w0, w9, w10, lt +; CHECK-GI-NEXT: cmp w8, w12, sxth +; CHECK-GI-NEXT: csel w8, w9, w10, gt +; CHECK-GI-NEXT: cmp w11, w8, sxth +; CHECK-GI-NEXT: csel w0, w8, w13, lt ; CHECK-GI-NEXT: ret entry: %arg1 = call i16 @llvm.vector.reduce.smax.v3i16(<3 x i16> %a) @@ -1256,19 +1244,16 @@ define i8 @uminv_v4i8(<4 x i8> %a) { ; CHECK-GI-LABEL: uminv_v4i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: mov h3, v0.h[3] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w10, s1 -; CHECK-GI-NEXT: fmov w11, s2 -; CHECK-GI-NEXT: fmov w12, s3 -; CHECK-GI-NEXT: and w9, w8, #0xff -; CHECK-GI-NEXT: cmp w9, w10, uxtb -; CHECK-GI-NEXT: and w9, w11, #0xff -; CHECK-GI-NEXT: csel w8, w8, w10, lo -; CHECK-GI-NEXT: cmp w9, w12, uxtb -; CHECK-GI-NEXT: csel w9, w11, w12, lo +; CHECK-GI-NEXT: umov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[1] +; CHECK-GI-NEXT: umov w10, v0.h[2] +; CHECK-GI-NEXT: umov w11, v0.h[3] +; CHECK-GI-NEXT: and w12, w8, #0xff +; CHECK-GI-NEXT: cmp w12, w9, uxtb +; CHECK-GI-NEXT: and w12, w10, #0xff +; CHECK-GI-NEXT: csel w8, w8, w9, lo +; CHECK-GI-NEXT: cmp w12, w11, uxtb +; CHECK-GI-NEXT: csel w9, w10, w11, lo ; CHECK-GI-NEXT: and w10, w8, #0xff ; CHECK-GI-NEXT: cmp w10, w9, uxtb ; CHECK-GI-NEXT: csel w0, w8, w9, lo @@ -1351,19 +1336,16 @@ define i16 @uminv_v3i16(<3 x i16> %a) { ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w9, s0 -; CHECK-GI-NEXT: uxth w8, w8 -; CHECK-GI-NEXT: fmov w10, s1 -; CHECK-GI-NEXT: fmov w11, s2 +; CHECK-GI-NEXT: umov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[0] +; CHECK-GI-NEXT: umov w10, v0.h[1] +; CHECK-GI-NEXT: umov w11, v0.h[2] +; CHECK-GI-NEXT: umov w13, v0.h[2] ; CHECK-GI-NEXT: fmov w12, s1 -; CHECK-GI-NEXT: cmp w8, w10, uxth -; CHECK-GI-NEXT: uxth w8, w11 -; CHECK-GI-NEXT: fmov w10, s2 -; CHECK-GI-NEXT: csel w9, w9, w12, lo -; CHECK-GI-NEXT: cmp w8, w9, uxth -; CHECK-GI-NEXT: csel w0, w9, w10, hi +; CHECK-GI-NEXT: cmp w8, w12, uxth +; CHECK-GI-NEXT: csel w8, w9, w10, lo +; CHECK-GI-NEXT: cmp w11, w8, uxth +; CHECK-GI-NEXT: csel w0, w8, w13, hi ; CHECK-GI-NEXT: ret entry: %arg1 = call i16 @llvm.vector.reduce.umin.v3i16(<3 x i16> %a) @@ -1625,19 +1607,16 @@ define i8 @umaxv_v4i8(<4 x i8> %a) { ; CHECK-GI-LABEL: umaxv_v4i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: mov h3, v0.h[3] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w10, s1 -; CHECK-GI-NEXT: fmov w11, s2 -; CHECK-GI-NEXT: fmov w12, s3 -; CHECK-GI-NEXT: and w9, w8, #0xff -; CHECK-GI-NEXT: cmp w9, w10, uxtb -; CHECK-GI-NEXT: and w9, w11, #0xff -; CHECK-GI-NEXT: csel w8, w8, w10, hi -; CHECK-GI-NEXT: cmp w9, w12, uxtb -; CHECK-GI-NEXT: csel w9, w11, w12, hi +; CHECK-GI-NEXT: umov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[1] +; CHECK-GI-NEXT: umov w10, v0.h[2] +; CHECK-GI-NEXT: umov w11, v0.h[3] +; CHECK-GI-NEXT: and w12, w8, #0xff +; CHECK-GI-NEXT: cmp w12, w9, uxtb +; CHECK-GI-NEXT: and w12, w10, #0xff +; CHECK-GI-NEXT: csel w8, w8, w9, hi +; CHECK-GI-NEXT: cmp w12, w11, uxtb +; CHECK-GI-NEXT: csel w9, w10, w11, hi ; CHECK-GI-NEXT: and w10, w8, #0xff ; CHECK-GI-NEXT: cmp w10, w9, uxtb ; CHECK-GI-NEXT: csel w0, w8, w9, hi @@ -1719,19 +1698,16 @@ define i16 @umaxv_v3i16(<3 x i16> %a) { ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w9, s0 -; CHECK-GI-NEXT: uxth w8, w8 -; CHECK-GI-NEXT: fmov w10, s1 -; CHECK-GI-NEXT: fmov w11, s2 +; CHECK-GI-NEXT: umov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[0] +; CHECK-GI-NEXT: umov w10, v0.h[1] +; CHECK-GI-NEXT: umov w11, v0.h[2] +; CHECK-GI-NEXT: umov w13, v0.h[2] ; CHECK-GI-NEXT: fmov w12, s1 -; CHECK-GI-NEXT: cmp w8, w10, uxth -; CHECK-GI-NEXT: uxth w8, w11 -; CHECK-GI-NEXT: fmov w10, s2 -; CHECK-GI-NEXT: csel w9, w9, w12, hi -; CHECK-GI-NEXT: cmp w8, w9, uxth -; CHECK-GI-NEXT: csel w0, w9, w10, lo +; CHECK-GI-NEXT: cmp w8, w12, uxth +; CHECK-GI-NEXT: csel w8, w9, w10, hi +; CHECK-GI-NEXT: cmp w11, w8, uxth +; CHECK-GI-NEXT: csel w0, w8, w13, lo ; CHECK-GI-NEXT: ret entry: %arg1 = call i16 @llvm.vector.reduce.umax.v3i16(<3 x i16> %a) diff --git a/llvm/test/CodeGen/AArch64/aarch64-mulv.ll b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll index 90f09379e68f..7b7ca9d8ffc2 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-mulv.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll @@ -73,13 +73,10 @@ define i8 @mulv_v4i8(<4 x i8> %a) { ; CHECK-GI-LABEL: mulv_v4i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: mov h3, v0.h[3] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: fmov w10, s2 -; CHECK-GI-NEXT: fmov w11, s3 +; CHECK-GI-NEXT: umov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[1] +; CHECK-GI-NEXT: umov w10, v0.h[2] +; CHECK-GI-NEXT: umov w11, v0.h[3] ; CHECK-GI-NEXT: mul w8, w8, w9 ; CHECK-GI-NEXT: mul w9, w10, w11 ; CHECK-GI-NEXT: mul w0, w8, w9 @@ -113,27 +110,20 @@ define i8 @mulv_v8i8(<8 x i8> %a) { ; CHECK-GI-LABEL: mulv_v8i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov b1, v0.b[1] -; CHECK-GI-NEXT: mov b2, v0.b[2] -; CHECK-GI-NEXT: mov b3, v0.b[3] -; CHECK-GI-NEXT: mov b4, v0.b[4] -; CHECK-GI-NEXT: mov b5, v0.b[5] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: mov b6, v0.b[6] -; CHECK-GI-NEXT: mov b7, v0.b[7] -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: fmov w10, s2 -; CHECK-GI-NEXT: fmov w11, s3 -; CHECK-GI-NEXT: fmov w12, s5 +; CHECK-GI-NEXT: umov w8, v0.b[0] +; CHECK-GI-NEXT: umov w9, v0.b[1] +; CHECK-GI-NEXT: umov w10, v0.b[2] +; CHECK-GI-NEXT: umov w11, v0.b[3] +; CHECK-GI-NEXT: umov w12, v0.b[4] +; CHECK-GI-NEXT: umov w13, v0.b[5] +; CHECK-GI-NEXT: umov w14, v0.b[6] +; CHECK-GI-NEXT: umov w15, v0.b[7] ; CHECK-GI-NEXT: mul w8, w8, w9 -; CHECK-GI-NEXT: fmov w9, s4 -; CHECK-GI-NEXT: mul w10, w10, w11 -; CHECK-GI-NEXT: fmov w11, s6 -; CHECK-GI-NEXT: mul w9, w9, w12 -; CHECK-GI-NEXT: fmov w12, s7 -; CHECK-GI-NEXT: mul w8, w8, w10 -; CHECK-GI-NEXT: mul w11, w11, w12 -; CHECK-GI-NEXT: mul w9, w9, w11 +; CHECK-GI-NEXT: mul w9, w10, w11 +; CHECK-GI-NEXT: mul w10, w12, w13 +; CHECK-GI-NEXT: mul w11, w14, w15 +; CHECK-GI-NEXT: mul w8, w8, w9 +; CHECK-GI-NEXT: mul w9, w10, w11 ; CHECK-GI-NEXT: mul w0, w8, w9 ; CHECK-GI-NEXT: ret entry: @@ -167,27 +157,20 @@ define i8 @mulv_v16i8(<16 x i8> %a) { ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov d1, v0.d[1] ; CHECK-GI-NEXT: mul v0.8b, v0.8b, v1.8b -; CHECK-GI-NEXT: mov b1, v0.b[1] -; CHECK-GI-NEXT: mov b2, v0.b[2] -; CHECK-GI-NEXT: mov b3, v0.b[3] -; CHECK-GI-NEXT: mov b4, v0.b[4] -; CHECK-GI-NEXT: mov b5, v0.b[5] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: mov b6, v0.b[6] -; CHECK-GI-NEXT: mov b7, v0.b[7] -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: fmov w10, s2 -; CHECK-GI-NEXT: fmov w11, s3 -; CHECK-GI-NEXT: fmov w12, s5 +; CHECK-GI-NEXT: umov w8, v0.b[0] +; CHECK-GI-NEXT: umov w9, v0.b[1] +; CHECK-GI-NEXT: umov w10, v0.b[2] +; CHECK-GI-NEXT: umov w11, v0.b[3] +; CHECK-GI-NEXT: umov w12, v0.b[4] +; CHECK-GI-NEXT: umov w13, v0.b[5] +; CHECK-GI-NEXT: umov w14, v0.b[6] +; CHECK-GI-NEXT: umov w15, v0.b[7] +; CHECK-GI-NEXT: mul w8, w8, w9 +; CHECK-GI-NEXT: mul w9, w10, w11 +; CHECK-GI-NEXT: mul w10, w12, w13 +; CHECK-GI-NEXT: mul w11, w14, w15 ; CHECK-GI-NEXT: mul w8, w8, w9 -; CHECK-GI-NEXT: fmov w9, s4 -; CHECK-GI-NEXT: mul w10, w10, w11 -; CHECK-GI-NEXT: fmov w11, s6 -; CHECK-GI-NEXT: mul w9, w9, w12 -; CHECK-GI-NEXT: fmov w12, s7 -; CHECK-GI-NEXT: mul w8, w8, w10 -; CHECK-GI-NEXT: mul w11, w11, w12 -; CHECK-GI-NEXT: mul w9, w9, w11 +; CHECK-GI-NEXT: mul w9, w10, w11 ; CHECK-GI-NEXT: mul w0, w8, w9 ; CHECK-GI-NEXT: ret entry: @@ -225,27 +208,20 @@ define i8 @mulv_v32i8(<32 x i8> %a) { ; CHECK-GI-NEXT: mul v0.8b, v0.8b, v2.8b ; CHECK-GI-NEXT: mul v1.8b, v1.8b, v3.8b ; CHECK-GI-NEXT: mul v0.8b, v0.8b, v1.8b -; CHECK-GI-NEXT: mov b1, v0.b[1] -; CHECK-GI-NEXT: mov b2, v0.b[2] -; CHECK-GI-NEXT: mov b3, v0.b[3] -; CHECK-GI-NEXT: mov b4, v0.b[4] -; CHECK-GI-NEXT: mov b5, v0.b[5] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: mov b6, v0.b[6] -; CHECK-GI-NEXT: mov b7, v0.b[7] -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: fmov w10, s2 -; CHECK-GI-NEXT: fmov w11, s3 -; CHECK-GI-NEXT: fmov w12, s5 +; CHECK-GI-NEXT: umov w8, v0.b[0] +; CHECK-GI-NEXT: umov w9, v0.b[1] +; CHECK-GI-NEXT: umov w10, v0.b[2] +; CHECK-GI-NEXT: umov w11, v0.b[3] +; CHECK-GI-NEXT: umov w12, v0.b[4] +; CHECK-GI-NEXT: umov w13, v0.b[5] +; CHECK-GI-NEXT: umov w14, v0.b[6] +; CHECK-GI-NEXT: umov w15, v0.b[7] ; CHECK-GI-NEXT: mul w8, w8, w9 -; CHECK-GI-NEXT: fmov w9, s4 -; CHECK-GI-NEXT: mul w10, w10, w11 -; CHECK-GI-NEXT: fmov w11, s6 -; CHECK-GI-NEXT: mul w9, w9, w12 -; CHECK-GI-NEXT: fmov w12, s7 -; CHECK-GI-NEXT: mul w8, w8, w10 -; CHECK-GI-NEXT: mul w11, w11, w12 -; CHECK-GI-NEXT: mul w9, w9, w11 +; CHECK-GI-NEXT: mul w9, w10, w11 +; CHECK-GI-NEXT: mul w10, w12, w13 +; CHECK-GI-NEXT: mul w11, w14, w15 +; CHECK-GI-NEXT: mul w8, w8, w9 +; CHECK-GI-NEXT: mul w9, w10, w11 ; CHECK-GI-NEXT: mul w0, w8, w9 ; CHECK-GI-NEXT: ret entry: @@ -289,13 +265,11 @@ define i16 @mulv_v3i16(<3 x i16> %a) { ; CHECK-GI-LABEL: mulv_v3i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: umov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[1] +; CHECK-GI-NEXT: umov w10, v0.h[2] ; CHECK-GI-NEXT: mul w8, w8, w9 -; CHECK-GI-NEXT: fmov w9, s2 -; CHECK-GI-NEXT: mul w0, w8, w9 +; CHECK-GI-NEXT: mul w0, w8, w10 ; CHECK-GI-NEXT: ret entry: %arg1 = call i16 @llvm.vector.reduce.mul.v3i16(<3 x i16> %a) @@ -318,13 +292,10 @@ define i16 @mulv_v4i16(<4 x i16> %a) { ; CHECK-GI-LABEL: mulv_v4i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: mov h3, v0.h[3] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: fmov w10, s2 -; CHECK-GI-NEXT: fmov w11, s3 +; CHECK-GI-NEXT: umov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[1] +; CHECK-GI-NEXT: umov w10, v0.h[2] +; CHECK-GI-NEXT: umov w11, v0.h[3] ; CHECK-GI-NEXT: mul w8, w8, w9 ; CHECK-GI-NEXT: mul w9, w10, w11 ; CHECK-GI-NEXT: mul w0, w8, w9 @@ -352,13 +323,10 @@ define i16 @mulv_v8i16(<8 x i16> %a) { ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov d1, v0.d[1] ; CHECK-GI-NEXT: mul v0.4h, v0.4h, v1.4h -; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: mov h3, v0.h[3] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: fmov w10, s2 -; CHECK-GI-NEXT: fmov w11, s3 +; CHECK-GI-NEXT: umov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[1] +; CHECK-GI-NEXT: umov w10, v0.h[2] +; CHECK-GI-NEXT: umov w11, v0.h[3] ; CHECK-GI-NEXT: mul w8, w8, w9 ; CHECK-GI-NEXT: mul w9, w10, w11 ; CHECK-GI-NEXT: mul w0, w8, w9 @@ -390,15 +358,12 @@ define i16 @mulv_v16i16(<16 x i16> %a) { ; CHECK-GI-NEXT: mul v0.4h, v0.4h, v2.4h ; CHECK-GI-NEXT: mul v1.4h, v1.4h, v3.4h ; CHECK-GI-NEXT: mul v0.4h, v0.4h, v1.4h -; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: mov h3, v0.h[3] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: fmov w10, s2 +; CHECK-GI-NEXT: umov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[1] +; CHECK-GI-NEXT: umov w10, v0.h[2] +; CHECK-GI-NEXT: umov w11, v0.h[3] ; CHECK-GI-NEXT: mul w8, w8, w9 -; CHECK-GI-NEXT: fmov w9, s3 -; CHECK-GI-NEXT: mul w9, w10, w9 +; CHECK-GI-NEXT: mul w9, w10, w11 ; CHECK-GI-NEXT: mul w0, w8, w9 ; CHECK-GI-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/aarch64-smull.ll b/llvm/test/CodeGen/AArch64/aarch64-smull.ll index 99aa28d859e1..dbc5417e2313 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-smull.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-smull.ll @@ -3,6 +3,19 @@ ; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+sve < %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SVE ; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI +; CHECK-GI: warning: Instruction selection used fallback path for smull_zext_v4i16_v4i32 +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for pmlsl2_v8i16_uzp1 +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for smlsl2_v8i16_uzp1 +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for umlsl2_v8i16_uzp1 +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for smlsl2_v4i32_uzp1 +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for umlsl2_v4i32_uzp1 +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for pmlsl_pmlsl2_v8i16_uzp1 +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for smlsl_smlsl2_v8i16_uzp1 +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for umlsl_umlsl2_v8i16_uzp1 +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for smlsl_smlsl2_v4i32_uzp1 +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for umlsl_umlsl2_v4i32_uzp1 +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for do_stuff + define <8 x i16> @smull_v8i8_v8i16(ptr %A, ptr %B) nounwind { ; CHECK-LABEL: smull_v8i8_v8i16: ; CHECK: // %bb.0: @@ -226,11 +239,10 @@ define <2 x i64> @smull_zext_v2i32_v2i64(ptr %A, ptr %B) nounwind { ; CHECK-GI-NEXT: movi d0, #0x00ffff0000ffff ; CHECK-GI-NEXT: mov v1.s[1], v2.s[0] ; CHECK-GI-NEXT: and v0.8b, v1.8b, v0.8b -; CHECK-GI-NEXT: mov s1, v0.s[1] -; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: mov w8, v0.s[0] +; CHECK-GI-NEXT: mov w9, v0.s[1] ; CHECK-GI-NEXT: ldr d0, [x1] ; CHECK-GI-NEXT: sshll v0.2d, v0.2s, #0 -; CHECK-GI-NEXT: fmov w9, s1 ; CHECK-GI-NEXT: fmov d1, x8 ; CHECK-GI-NEXT: mov d3, v0.d[1] ; CHECK-GI-NEXT: mov v1.d[1], x9 diff --git a/llvm/test/CodeGen/AArch64/arm64-addrmode.ll b/llvm/test/CodeGen/AArch64/arm64-addrmode.ll index 3d4749a7b8e7..bfef61abd8c1 100644 --- a/llvm/test/CodeGen/AArch64/arm64-addrmode.ll +++ b/llvm/test/CodeGen/AArch64/arm64-addrmode.ll @@ -210,12 +210,24 @@ define void @t17(i64 %a) { ret void } -define i32 @LdOffset_i8(ptr %a) { +; LDRBBroX +define i8 @LdOffset_i8(ptr %a) { ; CHECK-LABEL: LdOffset_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #56952 // =0xde78 -; CHECK-NEXT: movk w8, #15, lsl #16 -; CHECK-NEXT: ldrb w0, [x0, x8] +; CHECK-NEXT: add x8, x0, #253, lsl #12 // =1036288 +; CHECK-NEXT: ldrb w0, [x8, #3704] +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992 + %val = load i8, ptr %arrayidx, align 1 + ret i8 %val +} + +; LDRBBroX +define i32 @LdOffset_i8_zext32(ptr %a) { +; CHECK-LABEL: LdOffset_i8_zext32: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #253, lsl #12 // =1036288 +; CHECK-NEXT: ldrb w0, [x8, #3704] ; CHECK-NEXT: ret %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992 %val = load i8, ptr %arrayidx, align 1 @@ -223,12 +235,76 @@ define i32 @LdOffset_i8(ptr %a) { ret i32 %conv } -define i32 @LdOffset_i16(ptr %a) { +; LDRSBWroX +define i32 @LdOffset_i8_sext32(ptr %a) { +; CHECK-LABEL: LdOffset_i8_sext32: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #253, lsl #12 // =1036288 +; CHECK-NEXT: ldrsb w0, [x8, #3704] +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992 + %val = load i8, ptr %arrayidx, align 1 + %conv = sext i8 %val to i32 + ret i32 %conv +} + +; LDRBBroX +define i64 @LdOffset_i8_zext64(ptr %a) { +; CHECK-LABEL: LdOffset_i8_zext64: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #253, lsl #12 // =1036288 +; CHECK-NEXT: ldrb w0, [x8, #3704] +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992 + %val = load i8, ptr %arrayidx, align 1 + %conv = zext i8 %val to i64 + ret i64 %conv +} + +; LDRSBXroX +define i64 @LdOffset_i8_sext64(ptr %a) { +; CHECK-LABEL: LdOffset_i8_sext64: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #253, lsl #12 // =1036288 +; CHECK-NEXT: ldrsb x0, [x8, #3704] +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992 + %val = load i8, ptr %arrayidx, align 1 + %conv = sext i8 %val to i64 + ret i64 %conv +} + +; LDRHHroX +define i16 @LdOffset_i16(ptr %a) { ; CHECK-LABEL: LdOffset_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #48368 // =0xbcf0 -; CHECK-NEXT: movk w8, #31, lsl #16 -; CHECK-NEXT: ldrsh w0, [x0, x8] +; CHECK-NEXT: add x8, x0, #506, lsl #12 // =2072576 +; CHECK-NEXT: ldrh w0, [x8, #7408] +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992 + %val = load i16, ptr %arrayidx, align 2 + ret i16 %val +} + +; LDRHHroX +define i32 @LdOffset_i16_zext32(ptr %a) { +; CHECK-LABEL: LdOffset_i16_zext32: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #506, lsl #12 // =2072576 +; CHECK-NEXT: ldrh w0, [x8, #7408] +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992 + %val = load i16, ptr %arrayidx, align 2 + %conv = zext i16 %val to i32 + ret i32 %conv +} + +; LDRSHWroX +define i32 @LdOffset_i16_sext32(ptr %a) { +; CHECK-LABEL: LdOffset_i16_sext32: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #506, lsl #12 // =2072576 +; CHECK-NEXT: ldrsh w0, [x8, #7408] ; CHECK-NEXT: ret %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992 %val = load i16, ptr %arrayidx, align 2 @@ -236,18 +312,162 @@ define i32 @LdOffset_i16(ptr %a) { ret i32 %conv } +; LDRHHroX +define i64 @LdOffset_i16_zext64(ptr %a) { +; CHECK-LABEL: LdOffset_i16_zext64: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #506, lsl #12 // =2072576 +; CHECK-NEXT: ldrh w0, [x8, #7408] +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992 + %val = load i16, ptr %arrayidx, align 2 + %conv = zext i16 %val to i64 + ret i64 %conv +} + +; LDRSHXroX +define i64 @LdOffset_i16_sext64(ptr %a) { +; CHECK-LABEL: LdOffset_i16_sext64: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #506, lsl #12 // =2072576 +; CHECK-NEXT: ldrsh x0, [x8, #7408] +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992 + %val = load i16, ptr %arrayidx, align 2 + %conv = sext i16 %val to i64 + ret i64 %conv +} + +; LDRWroX define i32 @LdOffset_i32(ptr %a) { ; CHECK-LABEL: LdOffset_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #31200 // =0x79e0 -; CHECK-NEXT: movk w8, #63, lsl #16 -; CHECK-NEXT: ldr w0, [x0, x8] +; CHECK-NEXT: add x8, x0, #1012, lsl #12 // =4145152 +; CHECK-NEXT: ldr w0, [x8, #14816] ; CHECK-NEXT: ret %arrayidx = getelementptr inbounds i32, ptr %a, i64 1039992 %val = load i32, ptr %arrayidx, align 4 ret i32 %val } +; LDRWroX +define i64 @LdOffset_i32_zext64(ptr %a) { +; CHECK-LABEL: LdOffset_i32_zext64: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #1012, lsl #12 // =4145152 +; CHECK-NEXT: ldr w0, [x8, #14816] +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i32, ptr %a, i64 1039992 + %val = load i32, ptr %arrayidx, align 2 + %conv = zext i32 %val to i64 + ret i64 %conv +} + +; LDRSWroX +define i64 @LdOffset_i32_sext64(ptr %a) { +; CHECK-LABEL: LdOffset_i32_sext64: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #1012, lsl #12 // =4145152 +; CHECK-NEXT: ldrsw x0, [x8, #14816] +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i32, ptr %a, i64 1039992 + %val = load i32, ptr %arrayidx, align 2 + %conv = sext i32 %val to i64 + ret i64 %conv +} + +; LDRXroX +define i64 @LdOffset_i64(ptr %a) { +; CHECK-LABEL: LdOffset_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #2024, lsl #12 // =8290304 +; CHECK-NEXT: ldr x0, [x8, #29632] +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i64, ptr %a, i64 1039992 + %val = load i64, ptr %arrayidx, align 4 + ret i64 %val +} + +; LDRDroX +define <2 x i32> @LdOffset_v2i32(ptr %a) { +; CHECK-LABEL: LdOffset_v2i32: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #2024, lsl #12 // =8290304 +; CHECK-NEXT: ldr d0, [x8, #29632] +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds <2 x i32>, ptr %a, i64 1039992 + %val = load <2 x i32>, ptr %arrayidx, align 4 + ret <2 x i32> %val +} + +; LDRQroX +define <2 x i64> @LdOffset_v2i64(ptr %a) { +; CHECK-LABEL: LdOffset_v2i64: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #4048, lsl #12 // =16580608 +; CHECK-NEXT: ldr q0, [x8, #59264] +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds <2 x i64>, ptr %a, i64 1039992 + %val = load <2 x i64>, ptr %arrayidx, align 4 + ret <2 x i64> %val +} + +; LDRSBWroX +define double @LdOffset_i8_f64(ptr %a) { +; CHECK-LABEL: LdOffset_i8_f64: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #253, lsl #12 // =1036288 +; CHECK-NEXT: ldrsb w8, [x8, #3704] +; CHECK-NEXT: scvtf d0, w8 +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992 + %val = load i8, ptr %arrayidx, align 1 + %conv = sitofp i8 %val to double + ret double %conv +} + +; LDRSHWroX +define double @LdOffset_i16_f64(ptr %a) { +; CHECK-LABEL: LdOffset_i16_f64: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #506, lsl #12 // =2072576 +; CHECK-NEXT: ldrsh w8, [x8, #7408] +; CHECK-NEXT: scvtf d0, w8 +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992 + %val = load i16, ptr %arrayidx, align 2 + %conv = sitofp i16 %val to double + ret double %conv +} + +; LDRSroX +define double @LdOffset_i32_f64(ptr %a) { +; CHECK-LABEL: LdOffset_i32_f64: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #1012, lsl #12 // =4145152 +; CHECK-NEXT: ldr s0, [x8, #14816] +; CHECK-NEXT: ucvtf d0, d0 +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i32, ptr %a, i64 1039992 + %val = load i32, ptr %arrayidx, align 4 + %conv = uitofp i32 %val to double + ret double %conv +} + +; LDRDroX +define double @LdOffset_i64_f64(ptr %a) { +; CHECK-LABEL: LdOffset_i64_f64: +; CHECK: // %bb.0: +; CHECK-NEXT: add x8, x0, #2024, lsl #12 // =8290304 +; CHECK-NEXT: ldr d0, [x8, #29632] +; CHECK-NEXT: scvtf d0, d0 +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i64, ptr %a, i64 1039992 + %val = load i64, ptr %arrayidx, align 8 + %conv = sitofp i64 %val to double + ret double %conv +} + define i64 @LdOffset_i64_multi_offset(ptr %a) { ; CHECK-LABEL: LdOffset_i64_multi_offset: ; CHECK: // %bb.0: @@ -295,3 +515,27 @@ define i32 @LdOffset_i16_odd_offset(ptr nocapture noundef readonly %a) { ret i32 %conv } +; Already encoded with a single mov MOVNWi +define i8 @LdOffset_i8_movnwi(ptr %a) { +; CHECK-LABEL: LdOffset_i8_movnwi: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, #16777215 // =0xffffff +; CHECK-NEXT: ldrb w0, [x0, x8] +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i8, ptr %a, i64 16777215 + %val = load i8, ptr %arrayidx, align 1 + ret i8 %val +} + +; Negative test: the offset is too large to encoded with a add +define i8 @LdOffset_i8_too_large(ptr %a) { +; CHECK-LABEL: LdOffset_i8_too_large: +; CHECK: // %bb.0: +; CHECK-NEXT: mov w8, #1 // =0x1 +; CHECK-NEXT: movk w8, #256, lsl #16 +; CHECK-NEXT: ldrb w0, [x0, x8] +; CHECK-NEXT: ret + %arrayidx = getelementptr inbounds i8, ptr %a, i64 16777217 + %val = load i8, ptr %arrayidx, align 1 + ret i8 %val +} diff --git a/llvm/test/CodeGen/AArch64/fptoi.ll b/llvm/test/CodeGen/AArch64/fptoi.ll index f30dad966492..23ba85d54c7a 100644 --- a/llvm/test/CodeGen/AArch64/fptoi.ll +++ b/llvm/test/CodeGen/AArch64/fptoi.ll @@ -5846,11 +5846,9 @@ define <3 x i8> @fptos_v3f16_v3i8(<3 x half> %a) { ; CHECK-GI-FP16-LABEL: fptos_v3f16_v3i8: ; CHECK-GI-FP16: // %bb.0: // %entry ; CHECK-GI-FP16-NEXT: fcvtzs v0.4h, v0.4h -; CHECK-GI-FP16-NEXT: mov h1, v0.h[1] -; CHECK-GI-FP16-NEXT: mov h2, v0.h[2] -; CHECK-GI-FP16-NEXT: fmov w0, s0 -; CHECK-GI-FP16-NEXT: fmov w1, s1 -; CHECK-GI-FP16-NEXT: fmov w2, s2 +; CHECK-GI-FP16-NEXT: umov w0, v0.h[0] +; CHECK-GI-FP16-NEXT: umov w1, v0.h[1] +; CHECK-GI-FP16-NEXT: umov w2, v0.h[2] ; CHECK-GI-FP16-NEXT: ret entry: %c = fptosi <3 x half> %a to <3 x i8> @@ -5890,11 +5888,9 @@ define <3 x i8> @fptou_v3f16_v3i8(<3 x half> %a) { ; CHECK-GI-FP16-LABEL: fptou_v3f16_v3i8: ; CHECK-GI-FP16: // %bb.0: // %entry ; CHECK-GI-FP16-NEXT: fcvtzu v0.4h, v0.4h -; CHECK-GI-FP16-NEXT: mov h1, v0.h[1] -; CHECK-GI-FP16-NEXT: mov h2, v0.h[2] -; CHECK-GI-FP16-NEXT: fmov w0, s0 -; CHECK-GI-FP16-NEXT: fmov w1, s1 -; CHECK-GI-FP16-NEXT: fmov w2, s2 +; CHECK-GI-FP16-NEXT: umov w0, v0.h[0] +; CHECK-GI-FP16-NEXT: umov w1, v0.h[1] +; CHECK-GI-FP16-NEXT: umov w2, v0.h[2] ; CHECK-GI-FP16-NEXT: ret entry: %c = fptoui <3 x half> %a to <3 x i8> diff --git a/llvm/test/CodeGen/AArch64/large-offset-ldr-merge.mir b/llvm/test/CodeGen/AArch64/large-offset-ldr-merge.mir new file mode 100755 index 000000000000..15b6700398ea --- /dev/null +++ b/llvm/test/CodeGen/AArch64/large-offset-ldr-merge.mir @@ -0,0 +1,47 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=aarch64 -run-pass aarch64-ldst-opt %s -o - | FileCheck %s + + +--- +name: LdOffset +tracksRegLiveness: true +liveins: + - { reg: '$x0', virtual-reg: '' } +body: | + bb.0.entry: + liveins: $x0 + + ; CHECK-LABEL: name: LdOffset + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: $x8 = ADDXri $x0, 253, 12 + ; CHECK-NEXT: renamable $w0 = LDRBBui killed renamable $x8, 3704 + ; CHECK-NEXT: RET undef $lr, implicit $w0 + renamable $w8 = MOVZWi 56952, 0 + renamable $w8 = MOVKWi $w8, 15, 16, implicit-def $x8 + renamable $w0 = LDRBBroX killed renamable $x0, killed renamable $x8, 0, 0 + RET undef $lr, implicit $w0 +... + +# Negative test: the IndexReg missing killed flags +--- +name: LdOffset_missing_killed +tracksRegLiveness: true +liveins: + - { reg: '$x0', virtual-reg: '' } +body: | + bb.0.entry: + liveins: $x0 + + ; CHECK-LABEL: name: LdOffset_missing_killed + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $w8 = MOVZWi 56952, 0 + ; CHECK-NEXT: renamable $w8 = MOVKWi $w8, 15, 16, implicit-def $x8 + ; CHECK-NEXT: renamable $w0 = LDRBBroX killed renamable $x0, renamable $x8, 0, 0 + ; CHECK-NEXT: RET undef $lr, implicit $w0 + renamable $w8 = MOVZWi 56952, 0 + renamable $w8 = MOVKWi $w8, 15, 16, implicit-def $x8 + renamable $w0 = LDRBBroX killed renamable $x0, renamable $x8, 0, 0 + RET undef $lr, implicit $w0 +... diff --git a/llvm/test/CodeGen/AArch64/reduce-and.ll b/llvm/test/CodeGen/AArch64/reduce-and.ll index a20a76c00418..8b7438a42b71 100644 --- a/llvm/test/CodeGen/AArch64/reduce-and.ll +++ b/llvm/test/CodeGen/AArch64/reduce-and.ll @@ -53,13 +53,10 @@ define i1 @test_redand_v4i1(<4 x i1> %a) { ; GISEL-LABEL: test_redand_v4i1: ; GISEL: // %bb.0: ; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0 -; GISEL-NEXT: mov h1, v0.h[1] -; GISEL-NEXT: mov h2, v0.h[2] -; GISEL-NEXT: mov h3, v0.h[3] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 +; GISEL-NEXT: umov w8, v0.h[0] +; GISEL-NEXT: umov w9, v0.h[1] +; GISEL-NEXT: umov w10, v0.h[2] +; GISEL-NEXT: umov w11, v0.h[3] ; GISEL-NEXT: and w8, w8, w9 ; GISEL-NEXT: and w9, w10, w11 ; GISEL-NEXT: and w8, w8, w9 @@ -82,27 +79,20 @@ define i1 @test_redand_v8i1(<8 x i1> %a) { ; GISEL-LABEL: test_redand_v8i1: ; GISEL: // %bb.0: ; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0 -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s4 -; GISEL-NEXT: fmov w13, s5 -; GISEL-NEXT: fmov w14, s6 +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] ; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: fmov w9, s7 -; GISEL-NEXT: and w10, w10, w11 -; GISEL-NEXT: and w11, w12, w13 -; GISEL-NEXT: and w8, w8, w10 -; GISEL-NEXT: and w9, w14, w9 -; GISEL-NEXT: and w9, w11, w9 +; GISEL-NEXT: and w9, w10, w11 +; GISEL-NEXT: and w10, w12, w13 +; GISEL-NEXT: and w11, w14, w15 +; GISEL-NEXT: and w8, w8, w9 +; GISEL-NEXT: and w9, w10, w11 ; GISEL-NEXT: and w8, w8, w9 ; GISEL-NEXT: and w0, w8, #0x1 ; GISEL-NEXT: ret @@ -122,49 +112,34 @@ define i1 @test_redand_v16i1(<16 x i1> %a) { ; ; GISEL-LABEL: test_redand_v16i1: ; GISEL: // %bb.0: -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: mov b16, v0.b[8] -; GISEL-NEXT: mov b17, v0.b[9] -; GISEL-NEXT: mov b18, v0.b[10] -; GISEL-NEXT: mov b19, v0.b[11] -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s6 -; GISEL-NEXT: mov b20, v0.b[12] -; GISEL-NEXT: mov b21, v0.b[13] -; GISEL-NEXT: fmov w13, s7 -; GISEL-NEXT: mov b22, v0.b[14] -; GISEL-NEXT: mov b23, v0.b[15] +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] +; GISEL-NEXT: umov w16, v0.b[8] +; GISEL-NEXT: umov w17, v0.b[9] +; GISEL-NEXT: umov w18, v0.b[10] +; GISEL-NEXT: umov w0, v0.b[11] ; GISEL-NEXT: and w8, w8, w9 +; GISEL-NEXT: umov w1, v0.b[12] +; GISEL-NEXT: umov w2, v0.b[13] ; GISEL-NEXT: and w9, w10, w11 -; GISEL-NEXT: fmov w10, s4 +; GISEL-NEXT: and w10, w12, w13 +; GISEL-NEXT: umov w3, v0.b[14] +; GISEL-NEXT: and w11, w14, w15 ; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: fmov w11, s5 -; GISEL-NEXT: fmov w14, s18 -; GISEL-NEXT: fmov w15, s19 -; GISEL-NEXT: fmov w16, s22 -; GISEL-NEXT: fmov w17, s23 -; GISEL-NEXT: and w10, w10, w11 -; GISEL-NEXT: and w11, w12, w13 -; GISEL-NEXT: fmov w12, s16 +; GISEL-NEXT: umov w4, v0.b[15] +; GISEL-NEXT: and w12, w16, w17 +; GISEL-NEXT: and w13, w18, w0 ; GISEL-NEXT: and w9, w10, w11 -; GISEL-NEXT: fmov w13, s17 -; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: and w12, w12, w13 -; GISEL-NEXT: and w13, w14, w15 -; GISEL-NEXT: fmov w14, s20 -; GISEL-NEXT: fmov w15, s21 +; GISEL-NEXT: and w14, w1, w2 ; GISEL-NEXT: and w10, w12, w13 -; GISEL-NEXT: and w14, w14, w15 -; GISEL-NEXT: and w15, w16, w17 +; GISEL-NEXT: and w8, w8, w9 +; GISEL-NEXT: and w15, w3, w4 ; GISEL-NEXT: and w11, w14, w15 ; GISEL-NEXT: and w9, w10, w11 ; GISEL-NEXT: and w8, w8, w9 @@ -184,49 +159,34 @@ define <16 x i1> @test_redand_ins_v16i1(<16 x i1> %a) { ; ; GISEL-LABEL: test_redand_ins_v16i1: ; GISEL: // %bb.0: -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: mov b16, v0.b[8] -; GISEL-NEXT: mov b17, v0.b[9] -; GISEL-NEXT: mov b18, v0.b[10] -; GISEL-NEXT: mov b19, v0.b[11] -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s6 -; GISEL-NEXT: mov b20, v0.b[12] -; GISEL-NEXT: mov b21, v0.b[13] -; GISEL-NEXT: fmov w13, s7 -; GISEL-NEXT: mov b22, v0.b[14] -; GISEL-NEXT: mov b23, v0.b[15] +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] +; GISEL-NEXT: umov w16, v0.b[8] +; GISEL-NEXT: umov w17, v0.b[9] +; GISEL-NEXT: umov w18, v0.b[10] +; GISEL-NEXT: umov w0, v0.b[11] ; GISEL-NEXT: and w8, w8, w9 +; GISEL-NEXT: umov w1, v0.b[12] +; GISEL-NEXT: umov w2, v0.b[13] ; GISEL-NEXT: and w9, w10, w11 -; GISEL-NEXT: fmov w10, s4 +; GISEL-NEXT: and w10, w12, w13 +; GISEL-NEXT: umov w3, v0.b[14] +; GISEL-NEXT: and w11, w14, w15 ; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: fmov w11, s5 -; GISEL-NEXT: fmov w14, s18 -; GISEL-NEXT: fmov w15, s19 -; GISEL-NEXT: fmov w16, s22 -; GISEL-NEXT: fmov w17, s23 -; GISEL-NEXT: and w10, w10, w11 -; GISEL-NEXT: and w11, w12, w13 -; GISEL-NEXT: fmov w12, s16 +; GISEL-NEXT: umov w4, v0.b[15] +; GISEL-NEXT: and w12, w16, w17 +; GISEL-NEXT: and w13, w18, w0 ; GISEL-NEXT: and w9, w10, w11 -; GISEL-NEXT: fmov w13, s17 -; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: and w12, w12, w13 -; GISEL-NEXT: and w13, w14, w15 -; GISEL-NEXT: fmov w14, s20 -; GISEL-NEXT: fmov w15, s21 +; GISEL-NEXT: and w14, w1, w2 ; GISEL-NEXT: and w10, w12, w13 -; GISEL-NEXT: and w14, w14, w15 -; GISEL-NEXT: and w15, w16, w17 +; GISEL-NEXT: and w8, w8, w9 +; GISEL-NEXT: and w15, w3, w4 ; GISEL-NEXT: and w11, w14, w15 ; GISEL-NEXT: and w9, w10, w11 ; GISEL-NEXT: and w8, w8, w9 @@ -287,13 +247,10 @@ define i8 @test_redand_v4i8(<4 x i8> %a) { ; GISEL-LABEL: test_redand_v4i8: ; GISEL: // %bb.0: ; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0 -; GISEL-NEXT: mov h1, v0.h[1] -; GISEL-NEXT: mov h2, v0.h[2] -; GISEL-NEXT: mov h3, v0.h[3] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 +; GISEL-NEXT: umov w8, v0.h[0] +; GISEL-NEXT: umov w9, v0.h[1] +; GISEL-NEXT: umov w10, v0.h[2] +; GISEL-NEXT: umov w11, v0.h[3] ; GISEL-NEXT: and w8, w8, w9 ; GISEL-NEXT: and w9, w10, w11 ; GISEL-NEXT: and w0, w8, w9 @@ -315,27 +272,20 @@ define i8 @test_redand_v8i8(<8 x i8> %a) { ; GISEL-LABEL: test_redand_v8i8: ; GISEL: // %bb.0: ; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0 -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s4 -; GISEL-NEXT: fmov w13, s5 -; GISEL-NEXT: fmov w14, s6 +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] ; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: fmov w9, s7 -; GISEL-NEXT: and w10, w10, w11 -; GISEL-NEXT: and w11, w12, w13 -; GISEL-NEXT: and w8, w8, w10 -; GISEL-NEXT: and w9, w14, w9 -; GISEL-NEXT: and w9, w11, w9 +; GISEL-NEXT: and w9, w10, w11 +; GISEL-NEXT: and w10, w12, w13 +; GISEL-NEXT: and w11, w14, w15 +; GISEL-NEXT: and w8, w8, w9 +; GISEL-NEXT: and w9, w10, w11 ; GISEL-NEXT: and w0, w8, w9 ; GISEL-NEXT: ret %and_result = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> %a) @@ -358,27 +308,20 @@ define i8 @test_redand_v16i8(<16 x i8> %a) { ; GISEL: // %bb.0: ; GISEL-NEXT: mov d1, v0.d[1] ; GISEL-NEXT: and v0.8b, v0.8b, v1.8b -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s4 -; GISEL-NEXT: fmov w13, s5 -; GISEL-NEXT: fmov w14, s6 +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] ; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: fmov w9, s7 -; GISEL-NEXT: and w10, w10, w11 -; GISEL-NEXT: and w11, w12, w13 -; GISEL-NEXT: and w8, w8, w10 -; GISEL-NEXT: and w9, w14, w9 -; GISEL-NEXT: and w9, w11, w9 +; GISEL-NEXT: and w9, w10, w11 +; GISEL-NEXT: and w10, w12, w13 +; GISEL-NEXT: and w11, w14, w15 +; GISEL-NEXT: and w8, w8, w9 +; GISEL-NEXT: and w9, w10, w11 ; GISEL-NEXT: and w0, w8, w9 ; GISEL-NEXT: ret %and_result = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> %a) @@ -403,27 +346,20 @@ define i8 @test_redand_v32i8(<32 x i8> %a) { ; GISEL-NEXT: and v0.16b, v0.16b, v1.16b ; GISEL-NEXT: mov d1, v0.d[1] ; GISEL-NEXT: and v0.8b, v0.8b, v1.8b -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s4 -; GISEL-NEXT: fmov w13, s5 -; GISEL-NEXT: fmov w14, s6 +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] +; GISEL-NEXT: and w8, w8, w9 +; GISEL-NEXT: and w9, w10, w11 +; GISEL-NEXT: and w10, w12, w13 +; GISEL-NEXT: and w11, w14, w15 ; GISEL-NEXT: and w8, w8, w9 -; GISEL-NEXT: fmov w9, s7 -; GISEL-NEXT: and w10, w10, w11 -; GISEL-NEXT: and w11, w12, w13 -; GISEL-NEXT: and w8, w8, w10 -; GISEL-NEXT: and w9, w14, w9 -; GISEL-NEXT: and w9, w11, w9 +; GISEL-NEXT: and w9, w10, w11 ; GISEL-NEXT: and w0, w8, w9 ; GISEL-NEXT: ret %and_result = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> %a) @@ -442,13 +378,10 @@ define i16 @test_redand_v4i16(<4 x i16> %a) { ; GISEL-LABEL: test_redand_v4i16: ; GISEL: // %bb.0: ; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0 -; GISEL-NEXT: mov h1, v0.h[1] -; GISEL-NEXT: mov h2, v0.h[2] -; GISEL-NEXT: mov h3, v0.h[3] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 +; GISEL-NEXT: umov w8, v0.h[0] +; GISEL-NEXT: umov w9, v0.h[1] +; GISEL-NEXT: umov w10, v0.h[2] +; GISEL-NEXT: umov w11, v0.h[3] ; GISEL-NEXT: and w8, w8, w9 ; GISEL-NEXT: and w9, w10, w11 ; GISEL-NEXT: and w0, w8, w9 @@ -472,13 +405,10 @@ define i16 @test_redand_v8i16(<8 x i16> %a) { ; GISEL: // %bb.0: ; GISEL-NEXT: mov d1, v0.d[1] ; GISEL-NEXT: and v0.8b, v0.8b, v1.8b -; GISEL-NEXT: mov h1, v0.h[1] -; GISEL-NEXT: mov h2, v0.h[2] -; GISEL-NEXT: mov h3, v0.h[3] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 +; GISEL-NEXT: umov w8, v0.h[0] +; GISEL-NEXT: umov w9, v0.h[1] +; GISEL-NEXT: umov w10, v0.h[2] +; GISEL-NEXT: umov w11, v0.h[3] ; GISEL-NEXT: and w8, w8, w9 ; GISEL-NEXT: and w9, w10, w11 ; GISEL-NEXT: and w0, w8, w9 @@ -504,13 +434,10 @@ define i16 @test_redand_v16i16(<16 x i16> %a) { ; GISEL-NEXT: and v0.16b, v0.16b, v1.16b ; GISEL-NEXT: mov d1, v0.d[1] ; GISEL-NEXT: and v0.8b, v0.8b, v1.8b -; GISEL-NEXT: mov h1, v0.h[1] -; GISEL-NEXT: mov h2, v0.h[2] -; GISEL-NEXT: mov h3, v0.h[3] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 +; GISEL-NEXT: umov w8, v0.h[0] +; GISEL-NEXT: umov w9, v0.h[1] +; GISEL-NEXT: umov w10, v0.h[2] +; GISEL-NEXT: umov w11, v0.h[3] ; GISEL-NEXT: and w8, w8, w9 ; GISEL-NEXT: and w9, w10, w11 ; GISEL-NEXT: and w0, w8, w9 diff --git a/llvm/test/CodeGen/AArch64/reduce-or.ll b/llvm/test/CodeGen/AArch64/reduce-or.ll index 4c30a3293496..c4ac01f32e36 100644 --- a/llvm/test/CodeGen/AArch64/reduce-or.ll +++ b/llvm/test/CodeGen/AArch64/reduce-or.ll @@ -53,13 +53,10 @@ define i1 @test_redor_v4i1(<4 x i1> %a) { ; GISEL-LABEL: test_redor_v4i1: ; GISEL: // %bb.0: ; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0 -; GISEL-NEXT: mov h1, v0.h[1] -; GISEL-NEXT: mov h2, v0.h[2] -; GISEL-NEXT: mov h3, v0.h[3] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 +; GISEL-NEXT: umov w8, v0.h[0] +; GISEL-NEXT: umov w9, v0.h[1] +; GISEL-NEXT: umov w10, v0.h[2] +; GISEL-NEXT: umov w11, v0.h[3] ; GISEL-NEXT: orr w8, w8, w9 ; GISEL-NEXT: orr w9, w10, w11 ; GISEL-NEXT: orr w8, w8, w9 @@ -82,27 +79,20 @@ define i1 @test_redor_v8i1(<8 x i1> %a) { ; GISEL-LABEL: test_redor_v8i1: ; GISEL: // %bb.0: ; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0 -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s4 -; GISEL-NEXT: fmov w13, s5 -; GISEL-NEXT: fmov w14, s6 +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] ; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: fmov w9, s7 -; GISEL-NEXT: orr w10, w10, w11 -; GISEL-NEXT: orr w11, w12, w13 -; GISEL-NEXT: orr w8, w8, w10 -; GISEL-NEXT: orr w9, w14, w9 -; GISEL-NEXT: orr w9, w11, w9 +; GISEL-NEXT: orr w9, w10, w11 +; GISEL-NEXT: orr w10, w12, w13 +; GISEL-NEXT: orr w11, w14, w15 +; GISEL-NEXT: orr w8, w8, w9 +; GISEL-NEXT: orr w9, w10, w11 ; GISEL-NEXT: orr w8, w8, w9 ; GISEL-NEXT: and w0, w8, #0x1 ; GISEL-NEXT: ret @@ -122,49 +112,34 @@ define i1 @test_redor_v16i1(<16 x i1> %a) { ; ; GISEL-LABEL: test_redor_v16i1: ; GISEL: // %bb.0: -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: mov b16, v0.b[8] -; GISEL-NEXT: mov b17, v0.b[9] -; GISEL-NEXT: mov b18, v0.b[10] -; GISEL-NEXT: mov b19, v0.b[11] -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s6 -; GISEL-NEXT: mov b20, v0.b[12] -; GISEL-NEXT: mov b21, v0.b[13] -; GISEL-NEXT: fmov w13, s7 -; GISEL-NEXT: mov b22, v0.b[14] -; GISEL-NEXT: mov b23, v0.b[15] +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] +; GISEL-NEXT: umov w16, v0.b[8] +; GISEL-NEXT: umov w17, v0.b[9] +; GISEL-NEXT: umov w18, v0.b[10] +; GISEL-NEXT: umov w0, v0.b[11] ; GISEL-NEXT: orr w8, w8, w9 +; GISEL-NEXT: umov w1, v0.b[12] +; GISEL-NEXT: umov w2, v0.b[13] ; GISEL-NEXT: orr w9, w10, w11 -; GISEL-NEXT: fmov w10, s4 +; GISEL-NEXT: orr w10, w12, w13 +; GISEL-NEXT: umov w3, v0.b[14] +; GISEL-NEXT: orr w11, w14, w15 ; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: fmov w11, s5 -; GISEL-NEXT: fmov w14, s18 -; GISEL-NEXT: fmov w15, s19 -; GISEL-NEXT: fmov w16, s22 -; GISEL-NEXT: fmov w17, s23 -; GISEL-NEXT: orr w10, w10, w11 -; GISEL-NEXT: orr w11, w12, w13 -; GISEL-NEXT: fmov w12, s16 +; GISEL-NEXT: umov w4, v0.b[15] +; GISEL-NEXT: orr w12, w16, w17 +; GISEL-NEXT: orr w13, w18, w0 ; GISEL-NEXT: orr w9, w10, w11 -; GISEL-NEXT: fmov w13, s17 -; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: orr w12, w12, w13 -; GISEL-NEXT: orr w13, w14, w15 -; GISEL-NEXT: fmov w14, s20 -; GISEL-NEXT: fmov w15, s21 +; GISEL-NEXT: orr w14, w1, w2 ; GISEL-NEXT: orr w10, w12, w13 -; GISEL-NEXT: orr w14, w14, w15 -; GISEL-NEXT: orr w15, w16, w17 +; GISEL-NEXT: orr w8, w8, w9 +; GISEL-NEXT: orr w15, w3, w4 ; GISEL-NEXT: orr w11, w14, w15 ; GISEL-NEXT: orr w9, w10, w11 ; GISEL-NEXT: orr w8, w8, w9 @@ -184,49 +159,34 @@ define <16 x i1> @test_redor_ins_v16i1(<16 x i1> %a) { ; ; GISEL-LABEL: test_redor_ins_v16i1: ; GISEL: // %bb.0: -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: mov b16, v0.b[8] -; GISEL-NEXT: mov b17, v0.b[9] -; GISEL-NEXT: mov b18, v0.b[10] -; GISEL-NEXT: mov b19, v0.b[11] -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s6 -; GISEL-NEXT: mov b20, v0.b[12] -; GISEL-NEXT: mov b21, v0.b[13] -; GISEL-NEXT: fmov w13, s7 -; GISEL-NEXT: mov b22, v0.b[14] -; GISEL-NEXT: mov b23, v0.b[15] +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] +; GISEL-NEXT: umov w16, v0.b[8] +; GISEL-NEXT: umov w17, v0.b[9] +; GISEL-NEXT: umov w18, v0.b[10] +; GISEL-NEXT: umov w0, v0.b[11] ; GISEL-NEXT: orr w8, w8, w9 +; GISEL-NEXT: umov w1, v0.b[12] +; GISEL-NEXT: umov w2, v0.b[13] ; GISEL-NEXT: orr w9, w10, w11 -; GISEL-NEXT: fmov w10, s4 +; GISEL-NEXT: orr w10, w12, w13 +; GISEL-NEXT: umov w3, v0.b[14] +; GISEL-NEXT: orr w11, w14, w15 ; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: fmov w11, s5 -; GISEL-NEXT: fmov w14, s18 -; GISEL-NEXT: fmov w15, s19 -; GISEL-NEXT: fmov w16, s22 -; GISEL-NEXT: fmov w17, s23 -; GISEL-NEXT: orr w10, w10, w11 -; GISEL-NEXT: orr w11, w12, w13 -; GISEL-NEXT: fmov w12, s16 +; GISEL-NEXT: umov w4, v0.b[15] +; GISEL-NEXT: orr w12, w16, w17 +; GISEL-NEXT: orr w13, w18, w0 ; GISEL-NEXT: orr w9, w10, w11 -; GISEL-NEXT: fmov w13, s17 -; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: orr w12, w12, w13 -; GISEL-NEXT: orr w13, w14, w15 -; GISEL-NEXT: fmov w14, s20 -; GISEL-NEXT: fmov w15, s21 +; GISEL-NEXT: orr w14, w1, w2 ; GISEL-NEXT: orr w10, w12, w13 -; GISEL-NEXT: orr w14, w14, w15 -; GISEL-NEXT: orr w15, w16, w17 +; GISEL-NEXT: orr w8, w8, w9 +; GISEL-NEXT: orr w15, w3, w4 ; GISEL-NEXT: orr w11, w14, w15 ; GISEL-NEXT: orr w9, w10, w11 ; GISEL-NEXT: orr w8, w8, w9 @@ -289,13 +249,10 @@ define i8 @test_redor_v4i8(<4 x i8> %a) { ; GISEL-LABEL: test_redor_v4i8: ; GISEL: // %bb.0: ; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0 -; GISEL-NEXT: mov h1, v0.h[1] -; GISEL-NEXT: mov h2, v0.h[2] -; GISEL-NEXT: mov h3, v0.h[3] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 +; GISEL-NEXT: umov w8, v0.h[0] +; GISEL-NEXT: umov w9, v0.h[1] +; GISEL-NEXT: umov w10, v0.h[2] +; GISEL-NEXT: umov w11, v0.h[3] ; GISEL-NEXT: orr w8, w8, w9 ; GISEL-NEXT: orr w9, w10, w11 ; GISEL-NEXT: orr w0, w8, w9 @@ -317,27 +274,20 @@ define i8 @test_redor_v8i8(<8 x i8> %a) { ; GISEL-LABEL: test_redor_v8i8: ; GISEL: // %bb.0: ; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0 -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s4 -; GISEL-NEXT: fmov w13, s5 -; GISEL-NEXT: fmov w14, s6 +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] ; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: fmov w9, s7 -; GISEL-NEXT: orr w10, w10, w11 -; GISEL-NEXT: orr w11, w12, w13 -; GISEL-NEXT: orr w8, w8, w10 -; GISEL-NEXT: orr w9, w14, w9 -; GISEL-NEXT: orr w9, w11, w9 +; GISEL-NEXT: orr w9, w10, w11 +; GISEL-NEXT: orr w10, w12, w13 +; GISEL-NEXT: orr w11, w14, w15 +; GISEL-NEXT: orr w8, w8, w9 +; GISEL-NEXT: orr w9, w10, w11 ; GISEL-NEXT: orr w0, w8, w9 ; GISEL-NEXT: ret %or_result = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> %a) @@ -360,27 +310,20 @@ define i8 @test_redor_v16i8(<16 x i8> %a) { ; GISEL: // %bb.0: ; GISEL-NEXT: mov d1, v0.d[1] ; GISEL-NEXT: orr v0.8b, v0.8b, v1.8b -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s4 -; GISEL-NEXT: fmov w13, s5 -; GISEL-NEXT: fmov w14, s6 +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] ; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: fmov w9, s7 -; GISEL-NEXT: orr w10, w10, w11 -; GISEL-NEXT: orr w11, w12, w13 -; GISEL-NEXT: orr w8, w8, w10 -; GISEL-NEXT: orr w9, w14, w9 -; GISEL-NEXT: orr w9, w11, w9 +; GISEL-NEXT: orr w9, w10, w11 +; GISEL-NEXT: orr w10, w12, w13 +; GISEL-NEXT: orr w11, w14, w15 +; GISEL-NEXT: orr w8, w8, w9 +; GISEL-NEXT: orr w9, w10, w11 ; GISEL-NEXT: orr w0, w8, w9 ; GISEL-NEXT: ret %or_result = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> %a) @@ -405,27 +348,20 @@ define i8 @test_redor_v32i8(<32 x i8> %a) { ; GISEL-NEXT: orr v0.16b, v0.16b, v1.16b ; GISEL-NEXT: mov d1, v0.d[1] ; GISEL-NEXT: orr v0.8b, v0.8b, v1.8b -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s4 -; GISEL-NEXT: fmov w13, s5 -; GISEL-NEXT: fmov w14, s6 +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] +; GISEL-NEXT: orr w8, w8, w9 +; GISEL-NEXT: orr w9, w10, w11 +; GISEL-NEXT: orr w10, w12, w13 +; GISEL-NEXT: orr w11, w14, w15 ; GISEL-NEXT: orr w8, w8, w9 -; GISEL-NEXT: fmov w9, s7 -; GISEL-NEXT: orr w10, w10, w11 -; GISEL-NEXT: orr w11, w12, w13 -; GISEL-NEXT: orr w8, w8, w10 -; GISEL-NEXT: orr w9, w14, w9 -; GISEL-NEXT: orr w9, w11, w9 +; GISEL-NEXT: orr w9, w10, w11 ; GISEL-NEXT: orr w0, w8, w9 ; GISEL-NEXT: ret %or_result = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> %a) @@ -444,13 +380,10 @@ define i16 @test_redor_v4i16(<4 x i16> %a) { ; GISEL-LABEL: test_redor_v4i16: ; GISEL: // %bb.0: ; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0 -; GISEL-NEXT: mov h1, v0.h[1] -; GISEL-NEXT: mov h2, v0.h[2] -; GISEL-NEXT: mov h3, v0.h[3] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 +; GISEL-NEXT: umov w8, v0.h[0] +; GISEL-NEXT: umov w9, v0.h[1] +; GISEL-NEXT: umov w10, v0.h[2] +; GISEL-NEXT: umov w11, v0.h[3] ; GISEL-NEXT: orr w8, w8, w9 ; GISEL-NEXT: orr w9, w10, w11 ; GISEL-NEXT: orr w0, w8, w9 @@ -474,13 +407,10 @@ define i16 @test_redor_v8i16(<8 x i16> %a) { ; GISEL: // %bb.0: ; GISEL-NEXT: mov d1, v0.d[1] ; GISEL-NEXT: orr v0.8b, v0.8b, v1.8b -; GISEL-NEXT: mov h1, v0.h[1] -; GISEL-NEXT: mov h2, v0.h[2] -; GISEL-NEXT: mov h3, v0.h[3] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 +; GISEL-NEXT: umov w8, v0.h[0] +; GISEL-NEXT: umov w9, v0.h[1] +; GISEL-NEXT: umov w10, v0.h[2] +; GISEL-NEXT: umov w11, v0.h[3] ; GISEL-NEXT: orr w8, w8, w9 ; GISEL-NEXT: orr w9, w10, w11 ; GISEL-NEXT: orr w0, w8, w9 @@ -506,13 +436,10 @@ define i16 @test_redor_v16i16(<16 x i16> %a) { ; GISEL-NEXT: orr v0.16b, v0.16b, v1.16b ; GISEL-NEXT: mov d1, v0.d[1] ; GISEL-NEXT: orr v0.8b, v0.8b, v1.8b -; GISEL-NEXT: mov h1, v0.h[1] -; GISEL-NEXT: mov h2, v0.h[2] -; GISEL-NEXT: mov h3, v0.h[3] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 +; GISEL-NEXT: umov w8, v0.h[0] +; GISEL-NEXT: umov w9, v0.h[1] +; GISEL-NEXT: umov w10, v0.h[2] +; GISEL-NEXT: umov w11, v0.h[3] ; GISEL-NEXT: orr w8, w8, w9 ; GISEL-NEXT: orr w9, w10, w11 ; GISEL-NEXT: orr w0, w8, w9 diff --git a/llvm/test/CodeGen/AArch64/reduce-xor.ll b/llvm/test/CodeGen/AArch64/reduce-xor.ll index c74b3734a1b7..5c2a808ef2e8 100644 --- a/llvm/test/CodeGen/AArch64/reduce-xor.ll +++ b/llvm/test/CodeGen/AArch64/reduce-xor.ll @@ -48,13 +48,10 @@ define i1 @test_redxor_v4i1(<4 x i1> %a) { ; GISEL-LABEL: test_redxor_v4i1: ; GISEL: // %bb.0: ; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0 -; GISEL-NEXT: mov h1, v0.h[1] -; GISEL-NEXT: mov h2, v0.h[2] -; GISEL-NEXT: mov h3, v0.h[3] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 +; GISEL-NEXT: umov w8, v0.h[0] +; GISEL-NEXT: umov w9, v0.h[1] +; GISEL-NEXT: umov w10, v0.h[2] +; GISEL-NEXT: umov w11, v0.h[3] ; GISEL-NEXT: eor w8, w8, w9 ; GISEL-NEXT: eor w9, w10, w11 ; GISEL-NEXT: eor w8, w8, w9 @@ -75,27 +72,20 @@ define i1 @test_redxor_v8i1(<8 x i1> %a) { ; GISEL-LABEL: test_redxor_v8i1: ; GISEL: // %bb.0: ; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0 -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s4 -; GISEL-NEXT: fmov w13, s5 -; GISEL-NEXT: fmov w14, s6 +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] ; GISEL-NEXT: eor w8, w8, w9 -; GISEL-NEXT: fmov w9, s7 -; GISEL-NEXT: eor w10, w10, w11 -; GISEL-NEXT: eor w11, w12, w13 -; GISEL-NEXT: eor w8, w8, w10 -; GISEL-NEXT: eor w9, w14, w9 -; GISEL-NEXT: eor w9, w11, w9 +; GISEL-NEXT: eor w9, w10, w11 +; GISEL-NEXT: eor w10, w12, w13 +; GISEL-NEXT: eor w11, w14, w15 +; GISEL-NEXT: eor w8, w8, w9 +; GISEL-NEXT: eor w9, w10, w11 ; GISEL-NEXT: eor w8, w8, w9 ; GISEL-NEXT: and w0, w8, #0x1 ; GISEL-NEXT: ret @@ -113,49 +103,34 @@ define i1 @test_redxor_v16i1(<16 x i1> %a) { ; ; GISEL-LABEL: test_redxor_v16i1: ; GISEL: // %bb.0: -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: mov b16, v0.b[8] -; GISEL-NEXT: mov b17, v0.b[9] -; GISEL-NEXT: mov b18, v0.b[10] -; GISEL-NEXT: mov b19, v0.b[11] -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s6 -; GISEL-NEXT: mov b20, v0.b[12] -; GISEL-NEXT: mov b21, v0.b[13] -; GISEL-NEXT: fmov w13, s7 -; GISEL-NEXT: mov b22, v0.b[14] -; GISEL-NEXT: mov b23, v0.b[15] +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] +; GISEL-NEXT: umov w16, v0.b[8] +; GISEL-NEXT: umov w17, v0.b[9] +; GISEL-NEXT: umov w18, v0.b[10] +; GISEL-NEXT: umov w0, v0.b[11] ; GISEL-NEXT: eor w8, w8, w9 +; GISEL-NEXT: umov w1, v0.b[12] +; GISEL-NEXT: umov w2, v0.b[13] ; GISEL-NEXT: eor w9, w10, w11 -; GISEL-NEXT: fmov w10, s4 +; GISEL-NEXT: eor w10, w12, w13 +; GISEL-NEXT: umov w3, v0.b[14] +; GISEL-NEXT: eor w11, w14, w15 ; GISEL-NEXT: eor w8, w8, w9 -; GISEL-NEXT: fmov w11, s5 -; GISEL-NEXT: fmov w14, s18 -; GISEL-NEXT: fmov w15, s19 -; GISEL-NEXT: fmov w16, s22 -; GISEL-NEXT: fmov w17, s23 -; GISEL-NEXT: eor w10, w10, w11 -; GISEL-NEXT: eor w11, w12, w13 -; GISEL-NEXT: fmov w12, s16 +; GISEL-NEXT: umov w4, v0.b[15] +; GISEL-NEXT: eor w12, w16, w17 +; GISEL-NEXT: eor w13, w18, w0 ; GISEL-NEXT: eor w9, w10, w11 -; GISEL-NEXT: fmov w13, s17 -; GISEL-NEXT: eor w8, w8, w9 -; GISEL-NEXT: eor w12, w12, w13 -; GISEL-NEXT: eor w13, w14, w15 -; GISEL-NEXT: fmov w14, s20 -; GISEL-NEXT: fmov w15, s21 +; GISEL-NEXT: eor w14, w1, w2 ; GISEL-NEXT: eor w10, w12, w13 -; GISEL-NEXT: eor w14, w14, w15 -; GISEL-NEXT: eor w15, w16, w17 +; GISEL-NEXT: eor w8, w8, w9 +; GISEL-NEXT: eor w15, w3, w4 ; GISEL-NEXT: eor w11, w14, w15 ; GISEL-NEXT: eor w9, w10, w11 ; GISEL-NEXT: eor w8, w8, w9 @@ -173,49 +148,34 @@ define <16 x i1> @test_redxor_ins_v16i1(<16 x i1> %a) { ; ; GISEL-LABEL: test_redxor_ins_v16i1: ; GISEL: // %bb.0: -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: mov b16, v0.b[8] -; GISEL-NEXT: mov b17, v0.b[9] -; GISEL-NEXT: mov b18, v0.b[10] -; GISEL-NEXT: mov b19, v0.b[11] -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s6 -; GISEL-NEXT: mov b20, v0.b[12] -; GISEL-NEXT: mov b21, v0.b[13] -; GISEL-NEXT: fmov w13, s7 -; GISEL-NEXT: mov b22, v0.b[14] -; GISEL-NEXT: mov b23, v0.b[15] +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] +; GISEL-NEXT: umov w16, v0.b[8] +; GISEL-NEXT: umov w17, v0.b[9] +; GISEL-NEXT: umov w18, v0.b[10] +; GISEL-NEXT: umov w0, v0.b[11] ; GISEL-NEXT: eor w8, w8, w9 +; GISEL-NEXT: umov w1, v0.b[12] +; GISEL-NEXT: umov w2, v0.b[13] ; GISEL-NEXT: eor w9, w10, w11 -; GISEL-NEXT: fmov w10, s4 +; GISEL-NEXT: eor w10, w12, w13 +; GISEL-NEXT: umov w3, v0.b[14] +; GISEL-NEXT: eor w11, w14, w15 ; GISEL-NEXT: eor w8, w8, w9 -; GISEL-NEXT: fmov w11, s5 -; GISEL-NEXT: fmov w14, s18 -; GISEL-NEXT: fmov w15, s19 -; GISEL-NEXT: fmov w16, s22 -; GISEL-NEXT: fmov w17, s23 -; GISEL-NEXT: eor w10, w10, w11 -; GISEL-NEXT: eor w11, w12, w13 -; GISEL-NEXT: fmov w12, s16 +; GISEL-NEXT: umov w4, v0.b[15] +; GISEL-NEXT: eor w12, w16, w17 +; GISEL-NEXT: eor w13, w18, w0 ; GISEL-NEXT: eor w9, w10, w11 -; GISEL-NEXT: fmov w13, s17 -; GISEL-NEXT: eor w8, w8, w9 -; GISEL-NEXT: eor w12, w12, w13 -; GISEL-NEXT: eor w13, w14, w15 -; GISEL-NEXT: fmov w14, s20 -; GISEL-NEXT: fmov w15, s21 +; GISEL-NEXT: eor w14, w1, w2 ; GISEL-NEXT: eor w10, w12, w13 -; GISEL-NEXT: eor w14, w14, w15 -; GISEL-NEXT: eor w15, w16, w17 +; GISEL-NEXT: eor w8, w8, w9 +; GISEL-NEXT: eor w15, w3, w4 ; GISEL-NEXT: eor w11, w14, w15 ; GISEL-NEXT: eor w9, w10, w11 ; GISEL-NEXT: eor w8, w8, w9 @@ -278,13 +238,10 @@ define i8 @test_redxor_v4i8(<4 x i8> %a) { ; GISEL-LABEL: test_redxor_v4i8: ; GISEL: // %bb.0: ; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0 -; GISEL-NEXT: mov h1, v0.h[1] -; GISEL-NEXT: mov h2, v0.h[2] -; GISEL-NEXT: mov h3, v0.h[3] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 +; GISEL-NEXT: umov w8, v0.h[0] +; GISEL-NEXT: umov w9, v0.h[1] +; GISEL-NEXT: umov w10, v0.h[2] +; GISEL-NEXT: umov w11, v0.h[3] ; GISEL-NEXT: eor w8, w8, w9 ; GISEL-NEXT: eor w9, w10, w11 ; GISEL-NEXT: eor w0, w8, w9 @@ -306,27 +263,20 @@ define i8 @test_redxor_v8i8(<8 x i8> %a) { ; GISEL-LABEL: test_redxor_v8i8: ; GISEL: // %bb.0: ; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0 -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s4 -; GISEL-NEXT: fmov w13, s5 -; GISEL-NEXT: fmov w14, s6 +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] ; GISEL-NEXT: eor w8, w8, w9 -; GISEL-NEXT: fmov w9, s7 -; GISEL-NEXT: eor w10, w10, w11 -; GISEL-NEXT: eor w11, w12, w13 -; GISEL-NEXT: eor w8, w8, w10 -; GISEL-NEXT: eor w9, w14, w9 -; GISEL-NEXT: eor w9, w11, w9 +; GISEL-NEXT: eor w9, w10, w11 +; GISEL-NEXT: eor w10, w12, w13 +; GISEL-NEXT: eor w11, w14, w15 +; GISEL-NEXT: eor w8, w8, w9 +; GISEL-NEXT: eor w9, w10, w11 ; GISEL-NEXT: eor w0, w8, w9 ; GISEL-NEXT: ret %xor_result = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> %a) @@ -349,27 +299,20 @@ define i8 @test_redxor_v16i8(<16 x i8> %a) { ; GISEL: // %bb.0: ; GISEL-NEXT: mov d1, v0.d[1] ; GISEL-NEXT: eor v0.8b, v0.8b, v1.8b -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s4 -; GISEL-NEXT: fmov w13, s5 -; GISEL-NEXT: fmov w14, s6 +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] ; GISEL-NEXT: eor w8, w8, w9 -; GISEL-NEXT: fmov w9, s7 -; GISEL-NEXT: eor w10, w10, w11 -; GISEL-NEXT: eor w11, w12, w13 -; GISEL-NEXT: eor w8, w8, w10 -; GISEL-NEXT: eor w9, w14, w9 -; GISEL-NEXT: eor w9, w11, w9 +; GISEL-NEXT: eor w9, w10, w11 +; GISEL-NEXT: eor w10, w12, w13 +; GISEL-NEXT: eor w11, w14, w15 +; GISEL-NEXT: eor w8, w8, w9 +; GISEL-NEXT: eor w9, w10, w11 ; GISEL-NEXT: eor w0, w8, w9 ; GISEL-NEXT: ret %xor_result = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> %a) @@ -394,27 +337,20 @@ define i8 @test_redxor_v32i8(<32 x i8> %a) { ; GISEL-NEXT: eor v0.16b, v0.16b, v1.16b ; GISEL-NEXT: mov d1, v0.d[1] ; GISEL-NEXT: eor v0.8b, v0.8b, v1.8b -; GISEL-NEXT: mov b1, v0.b[1] -; GISEL-NEXT: mov b2, v0.b[2] -; GISEL-NEXT: mov b3, v0.b[3] -; GISEL-NEXT: mov b4, v0.b[4] -; GISEL-NEXT: mov b5, v0.b[5] -; GISEL-NEXT: mov b6, v0.b[6] -; GISEL-NEXT: mov b7, v0.b[7] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 -; GISEL-NEXT: fmov w12, s4 -; GISEL-NEXT: fmov w13, s5 -; GISEL-NEXT: fmov w14, s6 +; GISEL-NEXT: umov w8, v0.b[0] +; GISEL-NEXT: umov w9, v0.b[1] +; GISEL-NEXT: umov w10, v0.b[2] +; GISEL-NEXT: umov w11, v0.b[3] +; GISEL-NEXT: umov w12, v0.b[4] +; GISEL-NEXT: umov w13, v0.b[5] +; GISEL-NEXT: umov w14, v0.b[6] +; GISEL-NEXT: umov w15, v0.b[7] +; GISEL-NEXT: eor w8, w8, w9 +; GISEL-NEXT: eor w9, w10, w11 +; GISEL-NEXT: eor w10, w12, w13 +; GISEL-NEXT: eor w11, w14, w15 ; GISEL-NEXT: eor w8, w8, w9 -; GISEL-NEXT: fmov w9, s7 -; GISEL-NEXT: eor w10, w10, w11 -; GISEL-NEXT: eor w11, w12, w13 -; GISEL-NEXT: eor w8, w8, w10 -; GISEL-NEXT: eor w9, w14, w9 -; GISEL-NEXT: eor w9, w11, w9 +; GISEL-NEXT: eor w9, w10, w11 ; GISEL-NEXT: eor w0, w8, w9 ; GISEL-NEXT: ret %xor_result = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> %a) @@ -433,13 +369,10 @@ define i16 @test_redxor_v4i16(<4 x i16> %a) { ; GISEL-LABEL: test_redxor_v4i16: ; GISEL: // %bb.0: ; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0 -; GISEL-NEXT: mov h1, v0.h[1] -; GISEL-NEXT: mov h2, v0.h[2] -; GISEL-NEXT: mov h3, v0.h[3] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 +; GISEL-NEXT: umov w8, v0.h[0] +; GISEL-NEXT: umov w9, v0.h[1] +; GISEL-NEXT: umov w10, v0.h[2] +; GISEL-NEXT: umov w11, v0.h[3] ; GISEL-NEXT: eor w8, w8, w9 ; GISEL-NEXT: eor w9, w10, w11 ; GISEL-NEXT: eor w0, w8, w9 @@ -463,13 +396,10 @@ define i16 @test_redxor_v8i16(<8 x i16> %a) { ; GISEL: // %bb.0: ; GISEL-NEXT: mov d1, v0.d[1] ; GISEL-NEXT: eor v0.8b, v0.8b, v1.8b -; GISEL-NEXT: mov h1, v0.h[1] -; GISEL-NEXT: mov h2, v0.h[2] -; GISEL-NEXT: mov h3, v0.h[3] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 +; GISEL-NEXT: umov w8, v0.h[0] +; GISEL-NEXT: umov w9, v0.h[1] +; GISEL-NEXT: umov w10, v0.h[2] +; GISEL-NEXT: umov w11, v0.h[3] ; GISEL-NEXT: eor w8, w8, w9 ; GISEL-NEXT: eor w9, w10, w11 ; GISEL-NEXT: eor w0, w8, w9 @@ -495,13 +425,10 @@ define i16 @test_redxor_v16i16(<16 x i16> %a) { ; GISEL-NEXT: eor v0.16b, v0.16b, v1.16b ; GISEL-NEXT: mov d1, v0.d[1] ; GISEL-NEXT: eor v0.8b, v0.8b, v1.8b -; GISEL-NEXT: mov h1, v0.h[1] -; GISEL-NEXT: mov h2, v0.h[2] -; GISEL-NEXT: mov h3, v0.h[3] -; GISEL-NEXT: fmov w8, s0 -; GISEL-NEXT: fmov w9, s1 -; GISEL-NEXT: fmov w10, s2 -; GISEL-NEXT: fmov w11, s3 +; GISEL-NEXT: umov w8, v0.h[0] +; GISEL-NEXT: umov w9, v0.h[1] +; GISEL-NEXT: umov w10, v0.h[2] +; GISEL-NEXT: umov w11, v0.h[3] ; GISEL-NEXT: eor w8, w8, w9 ; GISEL-NEXT: eor w9, w10, w11 ; GISEL-NEXT: eor w0, w8, w9 diff --git a/llvm/test/CodeGen/AArch64/replace-intrinsics-with-veclib-armpl.ll b/llvm/test/CodeGen/AArch64/replace-intrinsics-with-veclib-armpl.ll index 18431ae021f9..d41870ec6e79 100644 --- a/llvm/test/CodeGen/AArch64/replace-intrinsics-with-veclib-armpl.ll +++ b/llvm/test/CodeGen/AArch64/replace-intrinsics-with-veclib-armpl.ll @@ -15,7 +15,7 @@ declare <vscale x 2 x double> @llvm.cos.nxv2f64(<vscale x 2 x double>) declare <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float>) ;. -; CHECK: @llvm.compiler.used = appending global [16 x ptr] [ptr @armpl_vcosq_f64, ptr @armpl_vcosq_f32, ptr @armpl_vsinq_f64, ptr @armpl_vsinq_f32, ptr @armpl_vexpq_f64, ptr @armpl_vexpq_f32, ptr @armpl_vexp2q_f64, ptr @armpl_vexp2q_f32, ptr @armpl_vexp10q_f64, ptr @armpl_vexp10q_f32, ptr @armpl_vlogq_f64, ptr @armpl_vlogq_f32, ptr @armpl_vlog2q_f64, ptr @armpl_vlog2q_f32, ptr @armpl_vlog10q_f64, ptr @armpl_vlog10q_f32], section "llvm.metadata" +; CHECK: @llvm.compiler.used = appending global [32 x ptr] [ptr @armpl_vcosq_f64, ptr @armpl_vcosq_f32, ptr @armpl_svcos_f64_x, ptr @armpl_svcos_f32_x, ptr @armpl_vsinq_f64, ptr @armpl_vsinq_f32, ptr @armpl_svsin_f64_x, ptr @armpl_svsin_f32_x, ptr @armpl_vexpq_f64, ptr @armpl_vexpq_f32, ptr @armpl_svexp_f64_x, ptr @armpl_svexp_f32_x, ptr @armpl_vexp2q_f64, ptr @armpl_vexp2q_f32, ptr @armpl_svexp2_f64_x, ptr @armpl_svexp2_f32_x, ptr @armpl_vexp10q_f64, ptr @armpl_vexp10q_f32, ptr @armpl_svexp10_f64_x, ptr @armpl_svexp10_f32_x, ptr @armpl_vlogq_f64, ptr @armpl_vlogq_f32, ptr @armpl_svlog_f64_x, ptr @armpl_svlog_f32_x, ptr @armpl_vlog2q_f64, ptr @armpl_vlog2q_f32, ptr @armpl_svlog2_f64_x, ptr @armpl_svlog2_f32_x, ptr @armpl_vlog10q_f64, ptr @armpl_vlog10q_f32, ptr @armpl_svlog10_f64_x, ptr @armpl_svlog10_f32_x], section "llvm.metadata" ;. define <2 x double> @llvm_cos_f64(<2 x double> %in) { ; CHECK-LABEL: define <2 x double> @llvm_cos_f64 @@ -40,7 +40,7 @@ define <4 x float> @llvm_cos_f32(<4 x float> %in) { define <vscale x 2 x double> @llvm_cos_vscale_f64(<vscale x 2 x double> %in) #0 { ; CHECK-LABEL: define <vscale x 2 x double> @llvm_cos_vscale_f64 ; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]]) #[[ATTR1:[0-9]+]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.cos.nxv2f64(<vscale x 2 x double> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svcos_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.cos.nxv2f64(<vscale x 2 x double> %in) @@ -50,7 +50,7 @@ define <vscale x 2 x double> @llvm_cos_vscale_f64(<vscale x 2 x double> %in) #0 define <vscale x 4 x float> @llvm_cos_vscale_f32(<vscale x 4 x float> %in) #0 { ; CHECK-LABEL: define <vscale x 4 x float> @llvm_cos_vscale_f32 ; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svcos_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float> %in) @@ -85,7 +85,7 @@ define <4 x float> @llvm_sin_f32(<4 x float> %in) { define <vscale x 2 x double> @llvm_sin_vscale_f64(<vscale x 2 x double> %in) #0 { ; CHECK-LABEL: define <vscale x 2 x double> @llvm_sin_vscale_f64 ; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.sin.nxv2f64(<vscale x 2 x double> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svsin_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.sin.nxv2f64(<vscale x 2 x double> %in) @@ -95,7 +95,7 @@ define <vscale x 2 x double> @llvm_sin_vscale_f64(<vscale x 2 x double> %in) #0 define <vscale x 4 x float> @llvm_sin_vscale_f32(<vscale x 4 x float> %in) #0 { ; CHECK-LABEL: define <vscale x 4 x float> @llvm_sin_vscale_f32 ; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svsin_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float> %in) @@ -130,7 +130,7 @@ define <4 x float> @llvm_exp_f32(<4 x float> %in) { define <vscale x 2 x double> @llvm_exp_vscale_f64(<vscale x 2 x double> %in) #0 { ; CHECK-LABEL: define <vscale x 2 x double> @llvm_exp_vscale_f64 ; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.exp.nxv2f64(<vscale x 2 x double> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svexp_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.exp.nxv2f64(<vscale x 2 x double> %in) @@ -140,7 +140,7 @@ define <vscale x 2 x double> @llvm_exp_vscale_f64(<vscale x 2 x double> %in) #0 define <vscale x 4 x float> @llvm_exp_vscale_f32(<vscale x 4 x float> %in) #0 { ; CHECK-LABEL: define <vscale x 4 x float> @llvm_exp_vscale_f32 ; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svexp_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %in) @@ -175,7 +175,7 @@ define <4 x float> @llvm_exp2_f32(<4 x float> %in) { define <vscale x 2 x double> @llvm_exp2_vscale_f64(<vscale x 2 x double> %in) #0 { ; CHECK-LABEL: define <vscale x 2 x double> @llvm_exp2_vscale_f64 ; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.exp2.nxv2f64(<vscale x 2 x double> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svexp2_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.exp2.nxv2f64(<vscale x 2 x double> %in) @@ -185,7 +185,7 @@ define <vscale x 2 x double> @llvm_exp2_vscale_f64(<vscale x 2 x double> %in) #0 define <vscale x 4 x float> @llvm_exp2_vscale_f32(<vscale x 4 x float> %in) #0 { ; CHECK-LABEL: define <vscale x 4 x float> @llvm_exp2_vscale_f32 ; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svexp2_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> %in) @@ -220,7 +220,7 @@ define <4 x float> @llvm_exp10_f32(<4 x float> %in) { define <vscale x 2 x double> @llvm_exp10_vscale_f64(<vscale x 2 x double> %in) #0 { ; CHECK-LABEL: define <vscale x 2 x double> @llvm_exp10_vscale_f64 ; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.exp10.nxv2f64(<vscale x 2 x double> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svexp10_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.exp10.nxv2f64(<vscale x 2 x double> %in) @@ -230,7 +230,7 @@ define <vscale x 2 x double> @llvm_exp10_vscale_f64(<vscale x 2 x double> %in) # define <vscale x 4 x float> @llvm_exp10_vscale_f32(<vscale x 4 x float> %in) #0 { ; CHECK-LABEL: define <vscale x 4 x float> @llvm_exp10_vscale_f32 ; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.exp10.nxv4f32(<vscale x 4 x float> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svexp10_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.exp10.nxv4f32(<vscale x 4 x float> %in) @@ -265,7 +265,7 @@ define <4 x float> @llvm_log_f32(<4 x float> %in) { define <vscale x 2 x double> @llvm_log_vscale_f64(<vscale x 2 x double> %in) #0 { ; CHECK-LABEL: define <vscale x 2 x double> @llvm_log_vscale_f64 ; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.log.nxv2f64(<vscale x 2 x double> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svlog_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.log.nxv2f64(<vscale x 2 x double> %in) @@ -275,7 +275,7 @@ define <vscale x 2 x double> @llvm_log_vscale_f64(<vscale x 2 x double> %in) #0 define <vscale x 4 x float> @llvm_log_vscale_f32(<vscale x 4 x float> %in) #0 { ; CHECK-LABEL: define <vscale x 4 x float> @llvm_log_vscale_f32 ; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.log.nxv4f32(<vscale x 4 x float> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svlog_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.log.nxv4f32(<vscale x 4 x float> %in) @@ -310,7 +310,7 @@ define <4 x float> @llvm_log2_f32(<4 x float> %in) { define <vscale x 2 x double> @llvm_log2_vscale_f64(<vscale x 2 x double> %in) #0 { ; CHECK-LABEL: define <vscale x 2 x double> @llvm_log2_vscale_f64 ; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.log2.nxv2f64(<vscale x 2 x double> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svlog2_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.log2.nxv2f64(<vscale x 2 x double> %in) @@ -320,7 +320,7 @@ define <vscale x 2 x double> @llvm_log2_vscale_f64(<vscale x 2 x double> %in) #0 define <vscale x 4 x float> @llvm_log2_vscale_f32(<vscale x 4 x float> %in) #0 { ; CHECK-LABEL: define <vscale x 4 x float> @llvm_log2_vscale_f32 ; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.log2.nxv4f32(<vscale x 4 x float> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svlog2_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.log2.nxv4f32(<vscale x 4 x float> %in) @@ -355,7 +355,7 @@ define <4 x float> @llvm_log10_f32(<4 x float> %in) { define <vscale x 2 x double> @llvm_log10_vscale_f64(<vscale x 2 x double> %in) #0 { ; CHECK-LABEL: define <vscale x 2 x double> @llvm_log10_vscale_f64 ; CHECK-SAME: (<vscale x 2 x double> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.log10.nxv2f64(<vscale x 2 x double> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @armpl_svlog10_f64_x(<vscale x 2 x double> [[IN]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.log10.nxv2f64(<vscale x 2 x double> %in) @@ -365,7 +365,7 @@ define <vscale x 2 x double> @llvm_log10_vscale_f64(<vscale x 2 x double> %in) # define <vscale x 4 x float> @llvm_log10_vscale_f32(<vscale x 4 x float> %in) #0 { ; CHECK-LABEL: define <vscale x 4 x float> @llvm_log10_vscale_f32 ; CHECK-SAME: (<vscale x 4 x float> [[IN:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.log10.nxv4f32(<vscale x 4 x float> [[IN]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @armpl_svlog10_f32_x(<vscale x 4 x float> [[IN]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.log10.nxv4f32(<vscale x 4 x float> %in) @@ -380,7 +380,7 @@ declare <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float>, <vscale x 4 ; ; There is a bug in the replace-with-veclib pass, and for intrinsics which take ; more than one arguments, but has just one overloaded type, it incorrectly -; reconstructs the scalar name, for pow specificlly it is searching for: +; reconstructs the scalar name, for pow specifically it is searching for: ; llvm.pow.f64.f64 and llvm.pow.f32.f32 ; diff --git a/llvm/test/CodeGen/AArch64/replace-intrinsics-with-veclib-sleef-scalable.ll b/llvm/test/CodeGen/AArch64/replace-intrinsics-with-veclib-sleef-scalable.ll index 8b06c41bcb1a..c2ff6014bc69 100644 --- a/llvm/test/CodeGen/AArch64/replace-intrinsics-with-veclib-sleef-scalable.ll +++ b/llvm/test/CodeGen/AArch64/replace-intrinsics-with-veclib-sleef-scalable.ll @@ -3,8 +3,9 @@ target triple = "aarch64-unknown-linux-gnu" -; NOTE: The existing TLI mappings are not used since the -replace-with-veclib pass is broken for scalable vectors. - +;. +; CHECK: @llvm.compiler.used = appending global [16 x ptr] [ptr @_ZGVsMxv_cos, ptr @_ZGVsMxv_cosf, ptr @_ZGVsMxv_exp, ptr @_ZGVsMxv_expf, ptr @_ZGVsMxv_exp2, ptr @_ZGVsMxv_exp2f, ptr @_ZGVsMxv_exp10, ptr @_ZGVsMxv_exp10f, ptr @_ZGVsMxv_log, ptr @_ZGVsMxv_logf, ptr @_ZGVsMxv_log10, ptr @_ZGVsMxv_log10f, ptr @_ZGVsMxv_log2, ptr @_ZGVsMxv_log2f, ptr @_ZGVsMxv_sin, ptr @_ZGVsMxv_sinf], section "llvm.metadata" +;. define <vscale x 2 x double> @llvm_ceil_vscale_f64(<vscale x 2 x double> %in) { ; CHECK-LABEL: @llvm_ceil_vscale_f64( ; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> [[IN:%.*]]) @@ -43,7 +44,7 @@ define <vscale x 4 x float> @llvm_copysign_vscale_f32(<vscale x 4 x float> %mag, define <vscale x 2 x double> @llvm_cos_vscale_f64(<vscale x 2 x double> %in) { ; CHECK-LABEL: @llvm_cos_vscale_f64( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.cos.nxv2f64(<vscale x 2 x double> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_cos(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.cos.nxv2f64(<vscale x 2 x double> %in) @@ -52,7 +53,7 @@ define <vscale x 2 x double> @llvm_cos_vscale_f64(<vscale x 2 x double> %in) { define <vscale x 4 x float> @llvm_cos_vscale_f32(<vscale x 4 x float> %in) { ; CHECK-LABEL: @llvm_cos_vscale_f32( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxv_cosf(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float> %in) @@ -61,7 +62,7 @@ define <vscale x 4 x float> @llvm_cos_vscale_f32(<vscale x 4 x float> %in) { define <vscale x 2 x double> @llvm_exp_vscale_f64(<vscale x 2 x double> %in) { ; CHECK-LABEL: @llvm_exp_vscale_f64( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.exp.nxv2f64(<vscale x 2 x double> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_exp(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.exp.nxv2f64(<vscale x 2 x double> %in) @@ -70,7 +71,7 @@ define <vscale x 2 x double> @llvm_exp_vscale_f64(<vscale x 2 x double> %in) { define <vscale x 4 x float> @llvm_exp_vscale_f32(<vscale x 4 x float> %in) { ; CHECK-LABEL: @llvm_exp_vscale_f32( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxv_expf(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %in) @@ -79,7 +80,7 @@ define <vscale x 4 x float> @llvm_exp_vscale_f32(<vscale x 4 x float> %in) { define <vscale x 2 x double> @llvm_exp2_vscale_f64(<vscale x 2 x double> %in) { ; CHECK-LABEL: @llvm_exp2_vscale_f64( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.exp2.nxv2f64(<vscale x 2 x double> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_exp2(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.exp2.nxv2f64(<vscale x 2 x double> %in) @@ -88,7 +89,7 @@ define <vscale x 2 x double> @llvm_exp2_vscale_f64(<vscale x 2 x double> %in) { define <vscale x 4 x float> @llvm_exp2_vscale_f32(<vscale x 4 x float> %in) { ; CHECK-LABEL: @llvm_exp2_vscale_f32( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxv_exp2f(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> %in) @@ -97,7 +98,7 @@ define <vscale x 4 x float> @llvm_exp2_vscale_f32(<vscale x 4 x float> %in) { define <vscale x 2 x double> @llvm_exp10_vscale_f64(<vscale x 2 x double> %in) { ; CHECK-LABEL: @llvm_exp10_vscale_f64( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.exp10.nxv2f64(<vscale x 2 x double> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_exp10(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.exp10.nxv2f64(<vscale x 2 x double> %in) @@ -106,7 +107,7 @@ define <vscale x 2 x double> @llvm_exp10_vscale_f64(<vscale x 2 x double> %in) { define <vscale x 4 x float> @llvm_exp10_vscale_f32(<vscale x 4 x float> %in) { ; CHECK-LABEL: @llvm_exp10_vscale_f32( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.exp10.nxv4f32(<vscale x 4 x float> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxv_exp10f(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.exp10.nxv4f32(<vscale x 4 x float> %in) @@ -169,7 +170,7 @@ define <vscale x 4 x float> @llvm_fma_vscale_f32(<vscale x 4 x float> %a, <vscal define <vscale x 2 x double> @llvm_log_vscale_f64(<vscale x 2 x double> %in) { ; CHECK-LABEL: @llvm_log_vscale_f64( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.log.nxv2f64(<vscale x 2 x double> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_log(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.log.nxv2f64(<vscale x 2 x double> %in) @@ -178,7 +179,7 @@ define <vscale x 2 x double> @llvm_log_vscale_f64(<vscale x 2 x double> %in) { define <vscale x 4 x float> @llvm_log_vscale_f32(<vscale x 4 x float> %in) { ; CHECK-LABEL: @llvm_log_vscale_f32( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.log.nxv4f32(<vscale x 4 x float> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxv_logf(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.log.nxv4f32(<vscale x 4 x float> %in) @@ -187,7 +188,7 @@ define <vscale x 4 x float> @llvm_log_vscale_f32(<vscale x 4 x float> %in) { define <vscale x 2 x double> @llvm_log10_vscale_f64(<vscale x 2 x double> %in) { ; CHECK-LABEL: @llvm_log10_vscale_f64( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.log10.nxv2f64(<vscale x 2 x double> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_log10(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.log10.nxv2f64(<vscale x 2 x double> %in) @@ -196,7 +197,7 @@ define <vscale x 2 x double> @llvm_log10_vscale_f64(<vscale x 2 x double> %in) { define <vscale x 4 x float> @llvm_log10_vscale_f32(<vscale x 4 x float> %in) { ; CHECK-LABEL: @llvm_log10_vscale_f32( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.log10.nxv4f32(<vscale x 4 x float> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxv_log10f(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.log10.nxv4f32(<vscale x 4 x float> %in) @@ -205,7 +206,7 @@ define <vscale x 4 x float> @llvm_log10_vscale_f32(<vscale x 4 x float> %in) { define <vscale x 2 x double> @llvm_log2_vscale_f64(<vscale x 2 x double> %in) { ; CHECK-LABEL: @llvm_log2_vscale_f64( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.log2.nxv2f64(<vscale x 2 x double> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_log2(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.log2.nxv2f64(<vscale x 2 x double> %in) @@ -214,7 +215,7 @@ define <vscale x 2 x double> @llvm_log2_vscale_f64(<vscale x 2 x double> %in) { define <vscale x 4 x float> @llvm_log2_vscale_f32(<vscale x 4 x float> %in) { ; CHECK-LABEL: @llvm_log2_vscale_f32( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.log2.nxv4f32(<vscale x 4 x float> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxv_log2f(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.log2.nxv4f32(<vscale x 4 x float> %in) @@ -331,7 +332,7 @@ define <vscale x 4 x float> @llvm_round_vscale_f32(<vscale x 4 x float> %in) { define <vscale x 2 x double> @llvm_sin_vscale_f64(<vscale x 2 x double> %in) { ; CHECK-LABEL: @llvm_sin_vscale_f64( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @llvm.sin.nxv2f64(<vscale x 2 x double> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 2 x double> @_ZGVsMxv_sin(<vscale x 2 x double> [[IN:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 2 x double> [[TMP1]] ; %1 = call fast <vscale x 2 x double> @llvm.sin.nxv2f64(<vscale x 2 x double> %in) @@ -340,7 +341,7 @@ define <vscale x 2 x double> @llvm_sin_vscale_f64(<vscale x 2 x double> %in) { define <vscale x 4 x float> @llvm_sin_vscale_f32(<vscale x 4 x float> %in) { ; CHECK-LABEL: @llvm_sin_vscale_f32( -; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float> [[IN:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = call fast <vscale x 4 x float> @_ZGVsMxv_sinf(<vscale x 4 x float> [[IN:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)) ; CHECK-NEXT: ret <vscale x 4 x float> [[TMP1]] ; %1 = call fast <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float> %in) diff --git a/llvm/test/CodeGen/AArch64/sext.ll b/llvm/test/CodeGen/AArch64/sext.ll index 4d26228caf62..014e4071a4bf 100644 --- a/llvm/test/CodeGen/AArch64/sext.ll +++ b/llvm/test/CodeGen/AArch64/sext.ll @@ -289,18 +289,14 @@ define <3 x i32> @sext_v3i16_v3i32(<3 x i16> %a) { ; CHECK-GI-LABEL: sext_v3i16_v3i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: sxth w8, w8 -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: fmov s0, w8 -; CHECK-GI-NEXT: fmov w8, s2 -; CHECK-GI-NEXT: sxth w9, w9 -; CHECK-GI-NEXT: sxth w8, w8 -; CHECK-GI-NEXT: mov v0.s[1], w9 -; CHECK-GI-NEXT: mov v0.s[2], w8 -; CHECK-GI-NEXT: mov v0.s[3], w8 +; CHECK-GI-NEXT: smov w8, v0.h[0] +; CHECK-GI-NEXT: smov w9, v0.h[1] +; CHECK-GI-NEXT: fmov s1, w8 +; CHECK-GI-NEXT: smov w8, v0.h[2] +; CHECK-GI-NEXT: mov v1.s[1], w9 +; CHECK-GI-NEXT: mov v1.s[2], w8 +; CHECK-GI-NEXT: mov v1.s[3], w8 +; CHECK-GI-NEXT: mov v0.16b, v1.16b ; CHECK-GI-NEXT: ret entry: %c = sext <3 x i16> %a to <3 x i32> @@ -322,15 +318,10 @@ define <3 x i64> @sext_v3i16_v3i64(<3 x i16> %a) { ; CHECK-GI-LABEL: sext_v3i16_v3i64: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: sxth x8, w8 -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: fmov w10, s2 +; CHECK-GI-NEXT: smov x8, v0.h[0] +; CHECK-GI-NEXT: smov x9, v0.h[1] +; CHECK-GI-NEXT: smov x10, v0.h[2] ; CHECK-GI-NEXT: fmov d0, x8 -; CHECK-GI-NEXT: sxth x9, w9 -; CHECK-GI-NEXT: sxth x10, w10 ; CHECK-GI-NEXT: fmov d1, x9 ; CHECK-GI-NEXT: fmov d2, x10 ; CHECK-GI-NEXT: ret @@ -352,15 +343,10 @@ define <3 x i64> @sext_v3i32_v3i64(<3 x i32> %a) { ; ; CHECK-GI-LABEL: sext_v3i32_v3i64: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: mov s1, v0.s[1] -; CHECK-GI-NEXT: mov s2, v0.s[2] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: sxtw x8, w8 -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: fmov w10, s2 +; CHECK-GI-NEXT: smov x8, v0.s[0] +; CHECK-GI-NEXT: smov x9, v0.s[1] +; CHECK-GI-NEXT: smov x10, v0.s[2] ; CHECK-GI-NEXT: fmov d0, x8 -; CHECK-GI-NEXT: sxtw x9, w9 -; CHECK-GI-NEXT: sxtw x10, w10 ; CHECK-GI-NEXT: fmov d1, x9 ; CHECK-GI-NEXT: fmov d2, x10 ; CHECK-GI-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sign-return-address-pauth-lr.ll b/llvm/test/CodeGen/AArch64/sign-return-address-pauth-lr.ll new file mode 100644 index 000000000000..a78fa853d99d --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sign-return-address-pauth-lr.ll @@ -0,0 +1,542 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 + +; PauthLR is controlled via a combination of -mbranch-protection and +pauth-lr. +; -mbranch-protection=+pc enables branch protection. If the feature +pauth-lr +; is available (v9.5a onwards) then non-NOP instructions are used; otherwise +; NOP instructions are used. + +; There are 6 cases to cover: + +; feature \ -mbranch-protection= | none | pac-ret | pac-ret+pc +; ------------------------------------------------------------------------ +; without +pauth-lr | no codegen | old pac | NOP pauth-lr +; with +pauth-lr | no codegen | old pac | non-NOP pauth-lr + +; sign-return-address.ll tests combinations of -mbranch-protection=none/pac-ret +; and whether +pauth-lr is present or not. + +; sign-return-address-pauth-lr.ll is identical, with the addition of this module +; attribute, which enables -mbranch-protection=pac-ret+pc, and therefore tests +; the remaining parameter combinations in the table: +!llvm.module.flags = !{!1} +!1 = !{i32 1, !"branch-protection-pauth-lr", i32 1} + +; RUN: llc -mtriple=aarch64 < %s | FileCheck --check-prefixes=CHECK,COMPAT %s +; RUN: llc -mtriple=aarch64 -mattr=v8.3a < %s | FileCheck --check-prefixes=CHECK,V83A %s +; RUN: llc -mtriple=aarch64 -mattr=v9a -mattr=pauth-lr < %s | FileCheck --check-prefixes=PAUTHLR %s + +define i32 @leaf(i32 %x) { +; CHECK-LABEL: leaf: +; CHECK: // %bb.0: +; CHECK-NEXT: ret +; +; PAUTHLR-LABEL: leaf: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: ret + ret i32 %x +} + +define i32 @leaf_sign_none(i32 %x) "sign-return-address"="none" { +; CHECK-LABEL: leaf_sign_none: +; CHECK: // %bb.0: +; CHECK-NEXT: ret +; +; PAUTHLR-LABEL: leaf_sign_none: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: ret + ret i32 %x +} + +define i32 @leaf_sign_non_leaf(i32 %x) "sign-return-address"="non-leaf" { +; CHECK-LABEL: leaf_sign_non_leaf: +; CHECK: // %bb.0: +; CHECK-NEXT: ret +; +; PAUTHLR-LABEL: leaf_sign_non_leaf: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: ret + ret i32 %x +} + +define i32 @leaf_sign_all(i32 %x) "sign-return-address"="all" { +; COMPAT-LABEL: leaf_sign_all: +; COMPAT: // %bb.0: +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: .Ltmp0: +; COMPAT-NEXT: hint #25 +; COMPAT-NEXT: .cfi_negate_ra_state +; COMPAT-NEXT: adr x16, .Ltmp0 +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: hint #29 +; COMPAT-NEXT: ret +; +; V83A-LABEL: leaf_sign_all: +; V83A: // %bb.0: +; V83A-NEXT: hint #39 +; V83A-NEXT: .Ltmp0: +; V83A-NEXT: paciasp +; V83A-NEXT: .cfi_negate_ra_state +; V83A-NEXT: adr x16, .Ltmp0 +; V83A-NEXT: hint #39 +; V83A-NEXT: retaa +; +; PAUTHLR-LABEL: leaf_sign_all: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: .Ltmp0: +; PAUTHLR-NEXT: paciasppc +; PAUTHLR-NEXT: .cfi_negate_ra_state +; PAUTHLR-NEXT: retaasppc .Ltmp0 + ret i32 %x +} + +define i64 @leaf_clobbers_lr(i64 %x) "sign-return-address"="non-leaf" { +; COMPAT-LABEL: leaf_clobbers_lr: +; COMPAT: // %bb.0: +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: .Ltmp1: +; COMPAT-NEXT: hint #25 +; COMPAT-NEXT: .cfi_negate_ra_state +; COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; COMPAT-NEXT: .cfi_def_cfa_offset 16 +; COMPAT-NEXT: .cfi_offset w30, -16 +; COMPAT-NEXT: //APP +; COMPAT-NEXT: mov x30, x0 +; COMPAT-NEXT: //NO_APP +; COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; COMPAT-NEXT: adr x16, .Ltmp1 +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: hint #29 +; COMPAT-NEXT: ret +; +; V83A-LABEL: leaf_clobbers_lr: +; V83A: // %bb.0: +; V83A-NEXT: hint #39 +; V83A-NEXT: .Ltmp1: +; V83A-NEXT: paciasp +; V83A-NEXT: .cfi_negate_ra_state +; V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; V83A-NEXT: .cfi_def_cfa_offset 16 +; V83A-NEXT: .cfi_offset w30, -16 +; V83A-NEXT: //APP +; V83A-NEXT: mov x30, x0 +; V83A-NEXT: //NO_APP +; V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; V83A-NEXT: adr x16, .Ltmp1 +; V83A-NEXT: hint #39 +; V83A-NEXT: retaa +; +; PAUTHLR-LABEL: leaf_clobbers_lr: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: .Ltmp1: +; PAUTHLR-NEXT: paciasppc +; PAUTHLR-NEXT: .cfi_negate_ra_state +; PAUTHLR-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; PAUTHLR-NEXT: .cfi_def_cfa_offset 16 +; PAUTHLR-NEXT: .cfi_offset w30, -16 +; PAUTHLR-NEXT: //APP +; PAUTHLR-NEXT: mov x30, x0 +; PAUTHLR-NEXT: //NO_APP +; PAUTHLR-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; PAUTHLR-NEXT: retaasppc .Ltmp1 + call void asm sideeffect "mov x30, $0", "r,~{lr}"(i64 %x) #1 + ret i64 %x +} + +declare i32 @foo(i32) + +define i32 @non_leaf_sign_all(i32 %x) "sign-return-address"="all" { +; COMPAT-LABEL: non_leaf_sign_all: +; COMPAT: // %bb.0: +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: .Ltmp2: +; COMPAT-NEXT: hint #25 +; COMPAT-NEXT: .cfi_negate_ra_state +; COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; COMPAT-NEXT: .cfi_def_cfa_offset 16 +; COMPAT-NEXT: .cfi_offset w30, -16 +; COMPAT-NEXT: bl foo +; COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; COMPAT-NEXT: adr x16, .Ltmp2 +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: hint #29 +; COMPAT-NEXT: ret +; +; V83A-LABEL: non_leaf_sign_all: +; V83A: // %bb.0: +; V83A-NEXT: hint #39 +; V83A-NEXT: .Ltmp2: +; V83A-NEXT: paciasp +; V83A-NEXT: .cfi_negate_ra_state +; V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; V83A-NEXT: .cfi_def_cfa_offset 16 +; V83A-NEXT: .cfi_offset w30, -16 +; V83A-NEXT: bl foo +; V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; V83A-NEXT: adr x16, .Ltmp2 +; V83A-NEXT: hint #39 +; V83A-NEXT: retaa +; +; PAUTHLR-LABEL: non_leaf_sign_all: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: .Ltmp2: +; PAUTHLR-NEXT: paciasppc +; PAUTHLR-NEXT: .cfi_negate_ra_state +; PAUTHLR-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; PAUTHLR-NEXT: .cfi_def_cfa_offset 16 +; PAUTHLR-NEXT: .cfi_offset w30, -16 +; PAUTHLR-NEXT: bl foo +; PAUTHLR-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; PAUTHLR-NEXT: retaasppc .Ltmp2 + %call = call i32 @foo(i32 %x) + ret i32 %call +} + +define i32 @non_leaf_sign_non_leaf(i32 %x) "sign-return-address"="non-leaf" { +; COMPAT-LABEL: non_leaf_sign_non_leaf: +; COMPAT: // %bb.0: +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: .Ltmp3: +; COMPAT-NEXT: hint #25 +; COMPAT-NEXT: .cfi_negate_ra_state +; COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; COMPAT-NEXT: .cfi_def_cfa_offset 16 +; COMPAT-NEXT: .cfi_offset w30, -16 +; COMPAT-NEXT: bl foo +; COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; COMPAT-NEXT: adr x16, .Ltmp3 +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: hint #29 +; COMPAT-NEXT: ret +; +; V83A-LABEL: non_leaf_sign_non_leaf: +; V83A: // %bb.0: +; V83A-NEXT: hint #39 +; V83A-NEXT: .Ltmp3: +; V83A-NEXT: paciasp +; V83A-NEXT: .cfi_negate_ra_state +; V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; V83A-NEXT: .cfi_def_cfa_offset 16 +; V83A-NEXT: .cfi_offset w30, -16 +; V83A-NEXT: bl foo +; V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; V83A-NEXT: adr x16, .Ltmp3 +; V83A-NEXT: hint #39 +; V83A-NEXT: retaa +; +; PAUTHLR-LABEL: non_leaf_sign_non_leaf: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: .Ltmp3: +; PAUTHLR-NEXT: paciasppc +; PAUTHLR-NEXT: .cfi_negate_ra_state +; PAUTHLR-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; PAUTHLR-NEXT: .cfi_def_cfa_offset 16 +; PAUTHLR-NEXT: .cfi_offset w30, -16 +; PAUTHLR-NEXT: bl foo +; PAUTHLR-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; PAUTHLR-NEXT: retaasppc .Ltmp3 + %call = call i32 @foo(i32 %x) + ret i32 %call +} + +; Should not use the RETAA instruction. +define i32 @non_leaf_scs(i32 %x) "sign-return-address"="non-leaf" shadowcallstack "target-features"="+v8.3a,+reserve-x18" { +; CHECK-LABEL: non_leaf_scs: +; CHECK: // %bb.0: +; CHECK-NEXT: str x30, [x18], #8 +; CHECK-NEXT: .cfi_escape 0x16, 0x12, 0x02, 0x82, 0x78 // +; CHECK-NEXT: hint #39 +; CHECK-NEXT: .Ltmp4: +; CHECK-NEXT: paciasp +; CHECK-NEXT: .cfi_negate_ra_state +; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset w30, -16 +; CHECK-NEXT: bl foo +; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: adr x16, .Ltmp4 +; CHECK-NEXT: hint #39 +; CHECK-NEXT: autiasp +; CHECK-NEXT: ldr x30, [x18, #-8]! +; CHECK-NEXT: ret +; +; PAUTHLR-LABEL: non_leaf_scs: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: str x30, [x18], #8 +; PAUTHLR-NEXT: .cfi_escape 0x16, 0x12, 0x02, 0x82, 0x78 // +; PAUTHLR-NEXT: .Ltmp4: +; PAUTHLR-NEXT: paciasppc +; PAUTHLR-NEXT: .cfi_negate_ra_state +; PAUTHLR-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; PAUTHLR-NEXT: .cfi_def_cfa_offset 16 +; PAUTHLR-NEXT: .cfi_offset w30, -16 +; PAUTHLR-NEXT: bl foo +; PAUTHLR-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; PAUTHLR-NEXT: autiasppc .Ltmp4 +; PAUTHLR-NEXT: ldr x30, [x18, #-8]! +; PAUTHLR-NEXT: ret + %call = call i32 @foo(i32 %x) + ret i32 %call +} + +define i32 @leaf_sign_all_v83(i32 %x) "sign-return-address"="all" "target-features"="+v8.3a" { +; CHECK-LABEL: leaf_sign_all_v83: +; CHECK: // %bb.0: +; CHECK-NEXT: hint #39 +; CHECK-NEXT: .Ltmp5: +; CHECK-NEXT: paciasp +; CHECK-NEXT: .cfi_negate_ra_state +; CHECK-NEXT: adr x16, .Ltmp5 +; CHECK-NEXT: hint #39 +; CHECK-NEXT: retaa +; +; PAUTHLR-LABEL: leaf_sign_all_v83: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: .Ltmp5: +; PAUTHLR-NEXT: paciasppc +; PAUTHLR-NEXT: .cfi_negate_ra_state +; PAUTHLR-NEXT: retaasppc .Ltmp5 + ret i32 %x +} + +declare fastcc i64 @bar(i64) + +define fastcc void @spill_lr_and_tail_call(i64 %x) "sign-return-address"="all" { +; COMPAT-LABEL: spill_lr_and_tail_call: +; COMPAT: // %bb.0: +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: .Ltmp6: +; COMPAT-NEXT: hint #25 +; COMPAT-NEXT: .cfi_negate_ra_state +; COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; COMPAT-NEXT: .cfi_def_cfa_offset 16 +; COMPAT-NEXT: .cfi_offset w30, -16 +; COMPAT-NEXT: //APP +; COMPAT-NEXT: mov x30, x0 +; COMPAT-NEXT: //NO_APP +; COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; COMPAT-NEXT: adr x16, .Ltmp6 +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: hint #29 +; COMPAT-NEXT: b bar +; +; V83A-LABEL: spill_lr_and_tail_call: +; V83A: // %bb.0: +; V83A-NEXT: hint #39 +; V83A-NEXT: .Ltmp6: +; V83A-NEXT: paciasp +; V83A-NEXT: .cfi_negate_ra_state +; V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; V83A-NEXT: .cfi_def_cfa_offset 16 +; V83A-NEXT: .cfi_offset w30, -16 +; V83A-NEXT: //APP +; V83A-NEXT: mov x30, x0 +; V83A-NEXT: //NO_APP +; V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; V83A-NEXT: adr x16, .Ltmp6 +; V83A-NEXT: hint #39 +; V83A-NEXT: autiasp +; V83A-NEXT: b bar +; +; PAUTHLR-LABEL: spill_lr_and_tail_call: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: .Ltmp6: +; PAUTHLR-NEXT: paciasppc +; PAUTHLR-NEXT: .cfi_negate_ra_state +; PAUTHLR-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; PAUTHLR-NEXT: .cfi_def_cfa_offset 16 +; PAUTHLR-NEXT: .cfi_offset w30, -16 +; PAUTHLR-NEXT: //APP +; PAUTHLR-NEXT: mov x30, x0 +; PAUTHLR-NEXT: //NO_APP +; PAUTHLR-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; PAUTHLR-NEXT: autiasppc .Ltmp6 +; PAUTHLR-NEXT: b bar + call void asm sideeffect "mov x30, $0", "r,~{lr}"(i64 %x) #1 + tail call fastcc i64 @bar(i64 %x) + ret void +} + +define i32 @leaf_sign_all_a_key(i32 %x) "sign-return-address"="all" "sign-return-address-key"="a_key" { +; COMPAT-LABEL: leaf_sign_all_a_key: +; COMPAT: // %bb.0: +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: .Ltmp7: +; COMPAT-NEXT: hint #25 +; COMPAT-NEXT: .cfi_negate_ra_state +; COMPAT-NEXT: adr x16, .Ltmp7 +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: hint #29 +; COMPAT-NEXT: ret +; +; V83A-LABEL: leaf_sign_all_a_key: +; V83A: // %bb.0: +; V83A-NEXT: hint #39 +; V83A-NEXT: .Ltmp7: +; V83A-NEXT: paciasp +; V83A-NEXT: .cfi_negate_ra_state +; V83A-NEXT: adr x16, .Ltmp7 +; V83A-NEXT: hint #39 +; V83A-NEXT: retaa +; +; PAUTHLR-LABEL: leaf_sign_all_a_key: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: .Ltmp7: +; PAUTHLR-NEXT: paciasppc +; PAUTHLR-NEXT: .cfi_negate_ra_state +; PAUTHLR-NEXT: retaasppc .Ltmp7 + ret i32 %x +} + +define i32 @leaf_sign_all_b_key(i32 %x) "sign-return-address"="all" "sign-return-address-key"="b_key" { +; COMPAT-LABEL: leaf_sign_all_b_key: +; COMPAT: // %bb.0: +; COMPAT-NEXT: .cfi_b_key_frame +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: .Ltmp8: +; COMPAT-NEXT: hint #27 +; COMPAT-NEXT: .cfi_negate_ra_state +; COMPAT-NEXT: adr x16, .Ltmp8 +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: hint #31 +; COMPAT-NEXT: ret +; +; V83A-LABEL: leaf_sign_all_b_key: +; V83A: // %bb.0: +; V83A-NEXT: .cfi_b_key_frame +; V83A-NEXT: hint #39 +; V83A-NEXT: .Ltmp8: +; V83A-NEXT: pacibsp +; V83A-NEXT: .cfi_negate_ra_state +; V83A-NEXT: adr x16, .Ltmp8 +; V83A-NEXT: hint #39 +; V83A-NEXT: retab +; +; PAUTHLR-LABEL: leaf_sign_all_b_key: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: .cfi_b_key_frame +; PAUTHLR-NEXT: .Ltmp8: +; PAUTHLR-NEXT: pacibsppc +; PAUTHLR-NEXT: .cfi_negate_ra_state +; PAUTHLR-NEXT: retabsppc .Ltmp8 + ret i32 %x +} + +define i32 @leaf_sign_all_v83_b_key(i32 %x) "sign-return-address"="all" "target-features"="+v8.3a" "sign-return-address-key"="b_key" { +; CHECK-LABEL: leaf_sign_all_v83_b_key: +; CHECK: // %bb.0: +; CHECK-NEXT: .cfi_b_key_frame +; CHECK-NEXT: hint #39 +; CHECK-NEXT: .Ltmp9: +; CHECK-NEXT: pacibsp +; CHECK-NEXT: .cfi_negate_ra_state +; CHECK-NEXT: adr x16, .Ltmp9 +; CHECK-NEXT: hint #39 +; CHECK-NEXT: retab +; +; PAUTHLR-LABEL: leaf_sign_all_v83_b_key: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: .cfi_b_key_frame +; PAUTHLR-NEXT: .Ltmp9: +; PAUTHLR-NEXT: pacibsppc +; PAUTHLR-NEXT: .cfi_negate_ra_state +; PAUTHLR-NEXT: retabsppc .Ltmp9 + ret i32 %x +} + +; Note that BTI instruction is not needed before PACIASP. +define i32 @leaf_sign_all_a_key_bti(i32 %x) "sign-return-address"="all" "sign-return-address-key"="a_key" "branch-target-enforcement"="true"{ +; COMPAT-LABEL: leaf_sign_all_a_key_bti: +; COMPAT: // %bb.0: +; COMPAT-NEXT: hint #34 +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: .Ltmp10: +; COMPAT-NEXT: hint #25 +; COMPAT-NEXT: .cfi_negate_ra_state +; COMPAT-NEXT: adr x16, .Ltmp10 +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: hint #29 +; COMPAT-NEXT: ret +; +; V83A-LABEL: leaf_sign_all_a_key_bti: +; V83A: // %bb.0: +; V83A-NEXT: hint #34 +; V83A-NEXT: hint #39 +; V83A-NEXT: .Ltmp10: +; V83A-NEXT: paciasp +; V83A-NEXT: .cfi_negate_ra_state +; V83A-NEXT: adr x16, .Ltmp10 +; V83A-NEXT: hint #39 +; V83A-NEXT: retaa +; +; PAUTHLR-LABEL: leaf_sign_all_a_key_bti: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: bti c +; PAUTHLR-NEXT: .Ltmp10: +; PAUTHLR-NEXT: paciasppc +; PAUTHLR-NEXT: .cfi_negate_ra_state +; PAUTHLR-NEXT: retaasppc .Ltmp10 + ret i32 %x +} + +; Note that BTI instruction is not needed before PACIBSP. +define i32 @leaf_sign_all_b_key_bti(i32 %x) "sign-return-address"="all" "sign-return-address-key"="b_key" "branch-target-enforcement"="true"{ +; COMPAT-LABEL: leaf_sign_all_b_key_bti: +; COMPAT: // %bb.0: +; COMPAT-NEXT: hint #34 +; COMPAT-NEXT: .cfi_b_key_frame +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: .Ltmp11: +; COMPAT-NEXT: hint #27 +; COMPAT-NEXT: .cfi_negate_ra_state +; COMPAT-NEXT: adr x16, .Ltmp11 +; COMPAT-NEXT: hint #39 +; COMPAT-NEXT: hint #31 +; COMPAT-NEXT: ret +; +; V83A-LABEL: leaf_sign_all_b_key_bti: +; V83A: // %bb.0: +; V83A-NEXT: hint #34 +; V83A-NEXT: .cfi_b_key_frame +; V83A-NEXT: hint #39 +; V83A-NEXT: .Ltmp11: +; V83A-NEXT: pacibsp +; V83A-NEXT: .cfi_negate_ra_state +; V83A-NEXT: adr x16, .Ltmp11 +; V83A-NEXT: hint #39 +; V83A-NEXT: retab +; +; PAUTHLR-LABEL: leaf_sign_all_b_key_bti: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: bti c +; PAUTHLR-NEXT: .cfi_b_key_frame +; PAUTHLR-NEXT: .Ltmp11: +; PAUTHLR-NEXT: pacibsppc +; PAUTHLR-NEXT: .cfi_negate_ra_state +; PAUTHLR-NEXT: retabsppc .Ltmp11 + ret i32 %x +} + +; Note that BTI instruction is not needed before PACIBSP. +define i32 @leaf_sign_all_v83_b_key_bti(i32 %x) "sign-return-address"="all" "target-features"="+v8.3a" "sign-return-address-key"="b_key" "branch-target-enforcement"="true" { +; CHECK-LABEL: leaf_sign_all_v83_b_key_bti: +; CHECK: // %bb.0: +; CHECK-NEXT: hint #34 +; CHECK-NEXT: .cfi_b_key_frame +; CHECK-NEXT: hint #39 +; CHECK-NEXT: .Ltmp12: +; CHECK-NEXT: pacibsp +; CHECK-NEXT: .cfi_negate_ra_state +; CHECK-NEXT: adr x16, .Ltmp12 +; CHECK-NEXT: hint #39 +; CHECK-NEXT: retab +; +; PAUTHLR-LABEL: leaf_sign_all_v83_b_key_bti: +; PAUTHLR: // %bb.0: +; PAUTHLR-NEXT: bti c +; PAUTHLR-NEXT: .cfi_b_key_frame +; PAUTHLR-NEXT: .Ltmp12: +; PAUTHLR-NEXT: pacibsppc +; PAUTHLR-NEXT: .cfi_negate_ra_state +; PAUTHLR-NEXT: retabsppc .Ltmp12 + ret i32 %x +} diff --git a/llvm/test/CodeGen/AArch64/sign-return-address.ll b/llvm/test/CodeGen/AArch64/sign-return-address.ll index 5680915c7f41..1481d4beb50d 100644 --- a/llvm/test/CodeGen/AArch64/sign-return-address.ll +++ b/llvm/test/CodeGen/AArch64/sign-return-address.ll @@ -2,6 +2,9 @@ ; RUN: llc -mtriple=aarch64 < %s | FileCheck --check-prefixes=CHECK,COMPAT %s ; RUN: llc -mtriple=aarch64 -mattr=v8.3a < %s | FileCheck --check-prefixes=CHECK,V83A %s +; v9.5-A is not expected to change codegen without -mbranch-protection=+pc, so reuse V83A. +; RUN: llc -mtriple=aarch64 -mattr=v9.5a < %s | FileCheck --check-prefixes=CHECK,V83A %s + define i32 @leaf(i32 %x) { ; CHECK-LABEL: leaf: ; CHECK: // %bb.0: diff --git a/llvm/test/CodeGen/AArch64/vecreduce-umax-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-umax-legalization.ll index 53aefaf3d336..7f804fe48fd8 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-umax-legalization.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-umax-legalization.ll @@ -168,53 +168,32 @@ define i8 @test_v9i8(<9 x i8> %a) nounwind { ; CHECK-GI-LABEL: test_v9i8: ; CHECK-GI: // %bb.0: ; CHECK-GI-NEXT: mov b1, v0.b[1] -; CHECK-GI-NEXT: mov b2, v0.b[2] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: mov b3, v0.b[3] -; CHECK-GI-NEXT: mov b4, v0.b[4] -; CHECK-GI-NEXT: fmov w9, s0 -; CHECK-GI-NEXT: uxtb w8, w8 -; CHECK-GI-NEXT: fmov w10, s1 -; CHECK-GI-NEXT: fmov w11, s2 -; CHECK-GI-NEXT: fmov w12, s1 -; CHECK-GI-NEXT: mov b1, v0.b[5] -; CHECK-GI-NEXT: mov b2, v0.b[6] -; CHECK-GI-NEXT: cmp w8, w10, uxtb -; CHECK-GI-NEXT: fmov w10, s3 -; CHECK-GI-NEXT: uxtb w8, w11 -; CHECK-GI-NEXT: csel w9, w9, w12, hi -; CHECK-GI-NEXT: cmp w8, w9, uxtb -; CHECK-GI-NEXT: uxtb w8, w10 -; CHECK-GI-NEXT: fmov w10, s4 -; CHECK-GI-NEXT: csel w9, w9, w11, lo -; CHECK-GI-NEXT: fmov w11, s3 -; CHECK-GI-NEXT: mov b3, v0.b[7] -; CHECK-GI-NEXT: mov b0, v0.b[8] -; CHECK-GI-NEXT: cmp w8, w9, uxtb -; CHECK-GI-NEXT: uxtb w8, w10 -; CHECK-GI-NEXT: fmov w10, s1 -; CHECK-GI-NEXT: csel w9, w9, w11, lo -; CHECK-GI-NEXT: fmov w11, s4 -; CHECK-GI-NEXT: cmp w8, w9, uxtb -; CHECK-GI-NEXT: uxtb w8, w10 -; CHECK-GI-NEXT: fmov w10, s2 -; CHECK-GI-NEXT: csel w9, w9, w11, lo +; CHECK-GI-NEXT: umov w8, v0.b[0] +; CHECK-GI-NEXT: umov w9, v0.b[1] +; CHECK-GI-NEXT: umov w10, v0.b[2] ; CHECK-GI-NEXT: fmov w11, s1 -; CHECK-GI-NEXT: cmp w8, w9, uxtb -; CHECK-GI-NEXT: uxtb w8, w10 -; CHECK-GI-NEXT: fmov w10, s3 -; CHECK-GI-NEXT: csel w9, w9, w11, lo -; CHECK-GI-NEXT: fmov w11, s2 -; CHECK-GI-NEXT: cmp w8, w9, uxtb -; CHECK-GI-NEXT: uxtb w8, w10 -; CHECK-GI-NEXT: fmov w10, s0 -; CHECK-GI-NEXT: csel w9, w9, w11, lo -; CHECK-GI-NEXT: fmov w11, s3 -; CHECK-GI-NEXT: cmp w8, w9, uxtb -; CHECK-GI-NEXT: uxtb w8, w10 -; CHECK-GI-NEXT: csel w9, w9, w11, lo -; CHECK-GI-NEXT: cmp w8, w9, uxtb -; CHECK-GI-NEXT: csel w0, w9, w10, lo +; CHECK-GI-NEXT: cmp w8, w11, uxtb +; CHECK-GI-NEXT: umov w11, v0.b[3] +; CHECK-GI-NEXT: csel w8, w8, w9, hi +; CHECK-GI-NEXT: umov w9, v0.b[4] +; CHECK-GI-NEXT: cmp w10, w8, uxtb +; CHECK-GI-NEXT: csel w8, w8, w10, lo +; CHECK-GI-NEXT: umov w10, v0.b[5] +; CHECK-GI-NEXT: cmp w11, w8, uxtb +; CHECK-GI-NEXT: csel w8, w8, w11, lo +; CHECK-GI-NEXT: umov w11, v0.b[6] +; CHECK-GI-NEXT: cmp w9, w8, uxtb +; CHECK-GI-NEXT: csel w8, w8, w9, lo +; CHECK-GI-NEXT: umov w9, v0.b[7] +; CHECK-GI-NEXT: cmp w10, w8, uxtb +; CHECK-GI-NEXT: csel w8, w8, w10, lo +; CHECK-GI-NEXT: umov w10, v0.b[8] +; CHECK-GI-NEXT: cmp w11, w8, uxtb +; CHECK-GI-NEXT: csel w8, w8, w11, lo +; CHECK-GI-NEXT: cmp w9, w8, uxtb +; CHECK-GI-NEXT: csel w8, w8, w9, lo +; CHECK-GI-NEXT: cmp w10, w8, uxtb +; CHECK-GI-NEXT: csel w0, w8, w10, lo ; CHECK-GI-NEXT: ret %b = call i8 @llvm.vector.reduce.umax.v9i8(<9 x i8> %a) ret i8 %b @@ -259,21 +238,18 @@ define i1 @test_v4i1(<4 x i1> %a) nounwind { ; CHECK-GI-LABEL: test_v4i1: ; CHECK-GI: // %bb.0: ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: mov h3, v0.h[3] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w10, s1 -; CHECK-GI-NEXT: fmov w12, s2 -; CHECK-GI-NEXT: fmov w13, s3 -; CHECK-GI-NEXT: and w9, w8, #0x1 -; CHECK-GI-NEXT: and w11, w10, #0x1 -; CHECK-GI-NEXT: cmp w9, w11 -; CHECK-GI-NEXT: and w9, w12, #0x1 -; CHECK-GI-NEXT: and w11, w13, #0x1 -; CHECK-GI-NEXT: csel w8, w8, w10, hi -; CHECK-GI-NEXT: cmp w9, w11 -; CHECK-GI-NEXT: csel w9, w12, w13, hi +; CHECK-GI-NEXT: umov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[1] +; CHECK-GI-NEXT: umov w10, v0.h[2] +; CHECK-GI-NEXT: umov w11, v0.h[3] +; CHECK-GI-NEXT: and w12, w8, #0x1 +; CHECK-GI-NEXT: and w13, w9, #0x1 +; CHECK-GI-NEXT: cmp w12, w13 +; CHECK-GI-NEXT: and w12, w10, #0x1 +; CHECK-GI-NEXT: and w13, w11, #0x1 +; CHECK-GI-NEXT: csel w8, w8, w9, hi +; CHECK-GI-NEXT: cmp w12, w13 +; CHECK-GI-NEXT: csel w9, w10, w11, hi ; CHECK-GI-NEXT: and w10, w8, #0x1 ; CHECK-GI-NEXT: and w11, w9, #0x1 ; CHECK-GI-NEXT: cmp w10, w11 diff --git a/llvm/test/CodeGen/AArch64/xtn.ll b/llvm/test/CodeGen/AArch64/xtn.ll index 0dd4e3644b78..21982fadbe80 100644 --- a/llvm/test/CodeGen/AArch64/xtn.ll +++ b/llvm/test/CodeGen/AArch64/xtn.ll @@ -224,23 +224,13 @@ entry: } define <3 x i8> @xtn_v3i16_v3i8(<3 x i16> %a) { -; CHECK-SD-LABEL: xtn_v3i16_v3i8: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-SD-NEXT: umov w0, v0.h[0] -; CHECK-SD-NEXT: umov w1, v0.h[1] -; CHECK-SD-NEXT: umov w2, v0.h[2] -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: xtn_v3i16_v3i8: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: fmov w0, s0 -; CHECK-GI-NEXT: fmov w1, s1 -; CHECK-GI-NEXT: fmov w2, s2 -; CHECK-GI-NEXT: ret +; CHECK-LABEL: xtn_v3i16_v3i8: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: umov w0, v0.h[0] +; CHECK-NEXT: umov w1, v0.h[1] +; CHECK-NEXT: umov w2, v0.h[2] +; CHECK-NEXT: ret entry: %arg1 = trunc <3 x i16> %a to <3 x i8> ret <3 x i8> %arg1 diff --git a/llvm/test/CodeGen/AArch64/zext.ll b/llvm/test/CodeGen/AArch64/zext.ll index 42c0bf79e778..e513340f5b18 100644 --- a/llvm/test/CodeGen/AArch64/zext.ll +++ b/llvm/test/CodeGen/AArch64/zext.ll @@ -2,6 +2,8 @@ ; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-SD ; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI +; CHECK-GI: warning: Instruction selection used fallback path for zext_v16i10_v16i16 + define i16 @zext_i8_to_i16(i8 %a) { ; CHECK-LABEL: zext_i8_to_i16: ; CHECK: // %bb.0: // %entry @@ -333,18 +335,14 @@ define <3 x i32> @zext_v3i16_v3i32(<3 x i16> %a) { ; CHECK-GI-LABEL: zext_v3i16_v3i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: uxth w8, w8 -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: fmov s0, w8 -; CHECK-GI-NEXT: fmov w8, s2 -; CHECK-GI-NEXT: uxth w9, w9 -; CHECK-GI-NEXT: uxth w8, w8 -; CHECK-GI-NEXT: mov v0.s[1], w9 -; CHECK-GI-NEXT: mov v0.s[2], w8 -; CHECK-GI-NEXT: mov v0.s[3], w8 +; CHECK-GI-NEXT: umov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[1] +; CHECK-GI-NEXT: fmov s1, w8 +; CHECK-GI-NEXT: umov w8, v0.h[2] +; CHECK-GI-NEXT: mov v1.s[1], w9 +; CHECK-GI-NEXT: mov v1.s[2], w8 +; CHECK-GI-NEXT: mov v1.s[3], w8 +; CHECK-GI-NEXT: mov v0.16b, v1.16b ; CHECK-GI-NEXT: ret entry: %c = zext <3 x i16> %a to <3 x i32> @@ -366,15 +364,10 @@ define <3 x i64> @zext_v3i16_v3i64(<3 x i16> %a) { ; CHECK-GI-LABEL: zext_v3i16_v3i64: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov h1, v0.h[1] -; CHECK-GI-NEXT: mov h2, v0.h[2] -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: ubfx x8, x8, #0, #16 -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: fmov w10, s2 +; CHECK-GI-NEXT: umov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[1] +; CHECK-GI-NEXT: umov w10, v0.h[2] ; CHECK-GI-NEXT: fmov d0, x8 -; CHECK-GI-NEXT: ubfx x9, x9, #0, #16 -; CHECK-GI-NEXT: ubfx x10, x10, #0, #16 ; CHECK-GI-NEXT: fmov d1, x9 ; CHECK-GI-NEXT: fmov d2, x10 ; CHECK-GI-NEXT: ret @@ -396,12 +389,10 @@ define <3 x i64> @zext_v3i32_v3i64(<3 x i32> %a) { ; ; CHECK-GI-LABEL: zext_v3i32_v3i64: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: mov s1, v0.s[1] -; CHECK-GI-NEXT: mov s2, v0.s[2] -; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: mov w8, v0.s[0] +; CHECK-GI-NEXT: mov w9, v0.s[1] +; CHECK-GI-NEXT: mov w10, v0.s[2] ; CHECK-GI-NEXT: fmov d0, x8 -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: fmov w10, s2 ; CHECK-GI-NEXT: fmov d1, x9 ; CHECK-GI-NEXT: fmov d2, x10 ; CHECK-GI-NEXT: ret diff --git a/llvm/test/CodeGen/AMDGPU/gds-unsupported.ll b/llvm/test/CodeGen/AMDGPU/gds-unsupported.ll index 174de0d58295..d35b9f79e23a 100644 --- a/llvm/test/CodeGen/AMDGPU/gds-unsupported.ll +++ b/llvm/test/CodeGen/AMDGPU/gds-unsupported.ll @@ -1,4 +1,5 @@ ; RUN: not --crash llc -march=amdgcn -mcpu=gfx90a < %s 2>&1 | FileCheck %s +; RUN: not --crash llc -march=amdgcn -mcpu=gfx1200 < %s 2>&1 | FileCheck %s ; GDS is not supported on GFX90A+ ; CHECK: LLVM ERROR: Cannot select: {{.*}} AtomicLoadAdd diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.add.gfx11.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.add.gfx11.ll index a9c2c2790389..3e977c054ec2 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.add.gfx11.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.add.gfx11.ll @@ -1,5 +1,8 @@ ; RUN: llc -march=amdgcn -mcpu=gfx1100 -amdgpu-enable-vopd=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,FUNC %s ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1100 -amdgpu-enable-vopd=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,FUNC %s +; RUN: not --crash llc -march=amdgcn -mcpu=gfx1200 -amdgpu-enable-vopd=0 -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=GFX12-ERR %s + +; GFX12-ERR: LLVM ERROR: Cannot select: {{.*}} = DS_ORDERED_COUNT ; FUNC-LABEL: {{^}}ds_ordered_add: ; GCN-DAG: v_mov_b32_e32 v[[INCR:[0-9]+]], 31 diff --git a/llvm/test/CodeGen/AMDGPU/verify-gfx12-gds.mir b/llvm/test/CodeGen/AMDGPU/verify-gfx12-gds.mir new file mode 100644 index 000000000000..a2182aa8d6ef --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/verify-gfx12-gds.mir @@ -0,0 +1,10 @@ +# RUN: not --crash llc -march=amdgcn -mcpu=gfx1200 -run-pass=none -o /dev/null %s 2>&1 | FileCheck -check-prefix=GFX12 %s + +--- +name: gds +body: | + bb.0: + ; GFX12: *** Bad machine code: GDS is not supported on this subtarget *** + ; GFX12: - instruction: DS_ADD_U32 %0:vgpr_32, %1:vgpr_32, 0, 1, implicit $m0, implicit $exec :: (load store acq_rel (s32), addrspace 2) + DS_ADD_U32 %0:vgpr_32, %2:vgpr_32, 0, 1, implicit $m0, implicit $exec :: (load store acq_rel (s32), addrspace 2) +... diff --git a/llvm/test/CodeGen/PowerPC/aix-return55.ll b/llvm/test/CodeGen/PowerPC/aix-return55.ll index c7d481ced140..a36deda9c146 100644 --- a/llvm/test/CodeGen/PowerPC/aix-return55.ll +++ b/llvm/test/CodeGen/PowerPC/aix-return55.ll @@ -39,7 +39,7 @@ entry: ;CHECKOBJ-NEXT: 1c: 67 8a bc de oris 10, 28, 48350{{[[:space:]] *}} ;CHECKOBJ32-NEXT: 00000020 <d>: ;CHECKOBJ64-NEXT: 0000000000000020 <d>: -;CHECKOBJ-NEXT: 20: 40 14 00 00 bdnzf 20, 0x20 +;CHECKOBJ-NEXT: 20: 40 14 00 00 bdnzf 20, 0x20 <d> ;CHECKOBJ-NEXT: 24: 00 00 00 00 <unknown>{{[[:space:]] *}} ;CHECKOBJ32-NEXT: 00000028 <foo>: ;CHECKOBJ32-NEXT: 28: 00 00 00 00 <unknown> diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect.ll index a5056d407b76..a557b6f4f171 100644 --- a/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect.ll +++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect.ll @@ -189,7 +189,7 @@ entry: ; DIS32-NEXT: 40: 7c 08 02 a6 mflr 0 ; DIS32-NEXT: 44: 94 21 ff c0 stwu 1, -64(1) ; DIS32-NEXT: 48: 90 01 00 48 stw 0, 72(1) -; DIS32-NEXT: 4c: 4b ff ff b5 bl 0x0 <.alias_foo> +; DIS32-NEXT: 4c: 4b ff ff b5 bl 0x0 <.foo> ; DIS32-NEXT: 0000004c: R_RBR (idx: 7) .foo[PR] ; DIS32-NEXT: 50: 60 00 00 00 nop ; DIS32-NEXT: 54: 48 00 00 6d bl 0xc0 <.static_overalign_foo> @@ -198,7 +198,7 @@ entry: ; DIS32-NEXT: 5c: 4b ff ff a5 bl 0x0 <.alias_foo> ; DIS32-NEXT: 0000005c: R_RBR (idx: 9) .alias_foo ; DIS32-NEXT: 60: 60 00 00 00 nop -; DIS32-NEXT: 64: 4b ff ff 9d bl 0x0 <.alias_foo> +; DIS32-NEXT: 64: 4b ff ff 9d bl 0x0 <.extern_foo> ; DIS32-NEXT: 00000064: R_RBR (idx: 1) .extern_foo[PR] ; DIS32-NEXT: 68: 60 00 00 00 nop ; DIS32-NEXT: 6c: 4b ff ff b5 bl 0x20 <.hidden_foo> @@ -212,7 +212,7 @@ entry: ; DIS64-NEXT: 40: 7c 08 02 a6 mflr 0 ; DIS64-NEXT: 44: f8 21 ff 91 stdu 1, -112(1) ; DIS64-NEXT: 48: f8 01 00 80 std 0, 128(1) -; DIS64-NEXT: 4c: 4b ff ff b5 bl 0x0 <.alias_foo> +; DIS64-NEXT: 4c: 4b ff ff b5 bl 0x0 <.foo> ; DIS64-NEXT: 000000000000004c: R_RBR (idx: 7) .foo[PR] ; DIS64-NEXT: 50: 60 00 00 00 nop ; DIS64-NEXT: 54: 48 00 00 6d bl 0xc0 <.static_overalign_foo> @@ -221,7 +221,7 @@ entry: ; DIS64-NEXT: 5c: 4b ff ff a5 bl 0x0 <.alias_foo> ; DIS64-NEXT: 000000000000005c: R_RBR (idx: 9) .alias_foo ; DIS64-NEXT: 60: 60 00 00 00 nop -; DIS64-NEXT: 64: 4b ff ff 9d bl 0x0 <.alias_foo> +; DIS64-NEXT: 64: 4b ff ff 9d bl 0x0 <.extern_foo> ; DIS64-NEXT: 0000000000000064: R_RBR (idx: 1) .extern_foo[PR] ; DIS64-NEXT: 68: 60 00 00 00 nop ; DIS64-NEXT: 6c: 4b ff ff b5 bl 0x20 <.hidden_foo> diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll index 97a5fbcf78f5..5ac6a7af0db2 100644 --- a/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll +++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll @@ -456,7 +456,7 @@ declare i32 @bar(i32) ; SYM-NEXT: ] -; DIS: {{.*}}aix-xcoff-reloc.ll.tmp.o: file format aixcoff-rs6000 +; DIS: : file format aixcoff-rs6000 ; DIS: Disassembly of section .text: ; DIS: 00000000 <.foo>: ; DIS-NEXT: 0: 7c 08 02 a6 mflr 0 @@ -495,7 +495,7 @@ declare i32 @bar(i32) ; DIS: 00000084 <globalB>: ; DIS-NEXT: 84: 00 00 00 44 <unknown> -; DIS_REL: {{.*}}aix-xcoff-reloc.ll.tmp.o: file format aixcoff-rs6000 +; DIS_REL: : file format aixcoff-rs6000 ; DIS_REL: RELOCATION RECORDS FOR [.text]: ; DIS_REL-NEXT: OFFSET TYPE VALUE ; DIS_REL-NEXT: 00000010 R_RBR .bar @@ -515,7 +515,7 @@ declare i32 @bar(i32) ; DIS64-NEXT: 4: f8 21 ff 91 stdu 1, -112(1) ; DIS64-NEXT: 8: 38 60 00 01 li 3, 1 ; DIS64-NEXT: c: f8 01 00 80 std 0, 128(1) -; DIS64-NEXT: 10: 4b ff ff f1 bl 0x0 <.foo> +; DIS64-NEXT: 10: 4b ff ff f1 bl 0x0 <.bar> ; DIS64-NEXT: 14: 60 00 00 00 nop ; DIS64-NEXT: 18: e8 82 00 00 ld 4, 0(2) ; DIS64-NEXT: 1c: e8 a2 00 08 ld 5, 8(2) diff --git a/llvm/test/CodeGen/RISCV/vp-splice-fixed-vectors.ll b/llvm/test/CodeGen/RISCV/vp-splice-fixed-vectors.ll new file mode 100644 index 000000000000..f7c8c251e197 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/vp-splice-fixed-vectors.ll @@ -0,0 +1,281 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple riscv64 -mattr=+f,+d,+v -verify-machineinstrs -riscv-v-vector-bits-min=128 \ +; RUN: < %s | FileCheck %s + +declare <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64>, <2 x i64>, i32, <2 x i1>, i32, i32) +declare <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32>, <4 x i32>, i32, <4 x i1>, i32, i32) +declare <8 x i16> @llvm.experimental.vp.splice.v8i16(<8 x i16>, <8 x i16>, i32, <8 x i1>, i32, i32) +declare <16 x i8> @llvm.experimental.vp.splice.v16i8(<16 x i8>, <16 x i8>, i32, <16 x i1>, i32, i32) + +declare <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double>, <2 x double>, i32, <2 x i1>, i32, i32) +declare <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float>, <4 x float>, i32, <4 x i1>, i32, i32) + +define <2 x i64> @test_vp_splice_v2i64(<2 x i64> %va, <2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v2i64: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e64, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; CHECK-NEXT: vslideup.vx v8, v9, a0 +; CHECK-NEXT: ret + %head = insertelement <2 x i1> undef, i1 1, i32 0 + %allones = shufflevector <2 x i1> %head, <2 x i1> undef, <2 x i32> zeroinitializer + + %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 5, <2 x i1> %allones, i32 %evla, i32 %evlb) + ret <2 x i64> %v +} + +define <2 x i64> @test_vp_splice_v2i64_negative_offset(<2 x i64> %va, <2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v2i64_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e64, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 5 +; CHECK-NEXT: ret + %head = insertelement <2 x i1> undef, i1 1, i32 0 + %allones = shufflevector <2 x i1> %head, <2 x i1> undef, <2 x i32> zeroinitializer + + %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 -5, <2 x i1> %allones, i32 %evla, i32 %evlb) + ret <2 x i64> %v +} + +define <2 x i64> @test_vp_splice_v2i64_masked(<2 x i64> %va, <2 x i64> %vb, <2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v2i64_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e64, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, mu +; CHECK-NEXT: vslideup.vx v8, v9, a0, v0.t +; CHECK-NEXT: ret + %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 5, <2 x i1> %mask, i32 %evla, i32 %evlb) + ret <2 x i64> %v +} + +define <4 x i32> @test_vp_splice_v4i32(<4 x i32> %va, <4 x i32> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v4i32: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vx v8, v9, a0 +; CHECK-NEXT: ret + %head = insertelement <4 x i1> undef, i1 1, i32 0 + %allones = shufflevector <4 x i1> %head, <4 x i1> undef, <4 x i32> zeroinitializer + + %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 5, <4 x i1> %allones, i32 %evla, i32 %evlb) + ret <4 x i32> %v +} + +define <4 x i32> @test_vp_splice_v4i32_negative_offset(<4 x i32> %va, <4 x i32> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v4i32_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e32, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 5 +; CHECK-NEXT: ret + %head = insertelement <4 x i1> undef, i1 1, i32 0 + %allones = shufflevector <4 x i1> %head, <4 x i1> undef, <4 x i32> zeroinitializer + + %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 -5, <4 x i1> %allones, i32 %evla, i32 %evlb) + ret <4 x i32> %v +} + +define <4 x i32> @test_vp_splice_v4i32_masked(<4 x i32> %va, <4 x i32> %vb, <4 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v4i32_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, mu +; CHECK-NEXT: vslideup.vx v8, v9, a0, v0.t +; CHECK-NEXT: ret + %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 5, <4 x i1> %mask, i32 %evla, i32 %evlb) + ret <4 x i32> %v +} + +define <8 x i16> @test_vp_splice_v8i16(<8 x i16> %va, <8 x i16> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e16, m1, ta, ma +; CHECK-NEXT: vslideup.vx v8, v9, a0 +; CHECK-NEXT: ret + %head = insertelement <8 x i1> undef, i1 1, i32 0 + %allones = shufflevector <8 x i1> %head, <8 x i1> undef, <8 x i32> zeroinitializer + + %v = call <8 x i16> @llvm.experimental.vp.splice.v8i16(<8 x i16> %va, <8 x i16> %vb, i32 5, <8 x i1> %allones, i32 %evla, i32 %evlb) + ret <8 x i16> %v +} + +define <8 x i16> @test_vp_splice_v8i16_negative_offset(<8 x i16> %va, <8 x i16> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v8i16_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e16, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e16, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 5 +; CHECK-NEXT: ret + %head = insertelement <8 x i1> undef, i1 1, i32 0 + %allones = shufflevector <8 x i1> %head, <8 x i1> undef, <8 x i32> zeroinitializer + + %v = call <8 x i16> @llvm.experimental.vp.splice.v8i16(<8 x i16> %va, <8 x i16> %vb, i32 -5, <8 x i1> %allones, i32 %evla, i32 %evlb) + ret <8 x i16> %v +} + +define <8 x i16> @test_vp_splice_v8i16_masked(<8 x i16> %va, <8 x i16> %vb, <8 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v8i16_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e16, m1, ta, mu +; CHECK-NEXT: vslideup.vx v8, v9, a0, v0.t +; CHECK-NEXT: ret + %v = call <8 x i16> @llvm.experimental.vp.splice.v8i16(<8 x i16> %va, <8 x i16> %vb, i32 5, <8 x i1> %mask, i32 %evla, i32 %evlb) + ret <8 x i16> %v +} + +define <16 x i8> @test_vp_splice_v16i8(<16 x i8> %va, <16 x i8> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v16i8: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vslideup.vx v8, v9, a0 +; CHECK-NEXT: ret + %head = insertelement <16 x i1> undef, i1 1, i32 0 + %allones = shufflevector <16 x i1> %head, <16 x i1> undef, <16 x i32> zeroinitializer + + %v = call <16 x i8> @llvm.experimental.vp.splice.v16i8(<16 x i8> %va, <16 x i8> %vb, i32 5, <16 x i1> %allones, i32 %evla, i32 %evlb) + ret <16 x i8> %v +} + +define <16 x i8> @test_vp_splice_v16i8_negative_offset(<16 x i8> %va, <16 x i8> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v16i8_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e8, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 5 +; CHECK-NEXT: ret + %head = insertelement <16 x i1> undef, i1 1, i32 0 + %allones = shufflevector <16 x i1> %head, <16 x i1> undef, <16 x i32> zeroinitializer + + %v = call <16 x i8> @llvm.experimental.vp.splice.v16i8(<16 x i8> %va, <16 x i8> %vb, i32 -5, <16 x i1> %allones, i32 %evla, i32 %evlb) + ret <16 x i8> %v +} + +define <16 x i8> @test_vp_splice_v16i8_masked(<16 x i8> %va, <16 x i8> %vb, <16 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v16i8_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, mu +; CHECK-NEXT: vslideup.vx v8, v9, a0, v0.t +; CHECK-NEXT: ret + %v = call <16 x i8> @llvm.experimental.vp.splice.v16i8(<16 x i8> %va, <16 x i8> %vb, i32 5, <16 x i1> %mask, i32 %evla, i32 %evlb) + ret <16 x i8> %v +} + +define <2 x double> @test_vp_splice_v2f64(<2 x double> %va, <2 x double> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v2f64: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e64, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; CHECK-NEXT: vslideup.vx v8, v9, a0 +; CHECK-NEXT: ret + %head = insertelement <2 x i1> undef, i1 1, i32 0 + %allones = shufflevector <2 x i1> %head, <2 x i1> undef, <2 x i32> zeroinitializer + + %v = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %va, <2 x double> %vb, i32 5, <2 x i1> %allones, i32 %evla, i32 %evlb) + ret <2 x double> %v +} + +define <2 x double> @test_vp_splice_v2f64_negative_offset(<2 x double> %va, <2 x double> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v2f64_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e64, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 5 +; CHECK-NEXT: ret + %head = insertelement <2 x i1> undef, i1 1, i32 0 + %allones = shufflevector <2 x i1> %head, <2 x i1> undef, <2 x i32> zeroinitializer + + %v = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %va, <2 x double> %vb, i32 -5, <2 x i1> %allones, i32 %evla, i32 %evlb) + ret <2 x double> %v +} + +define <2 x double> @test_vp_splice_v2f64_masked(<2 x double> %va, <2 x double> %vb, <2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v2f64_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e64, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, mu +; CHECK-NEXT: vslideup.vx v8, v9, a0, v0.t +; CHECK-NEXT: ret + %v = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %va, <2 x double> %vb, i32 5, <2 x i1> %mask, i32 %evla, i32 %evlb) + ret <2 x double> %v +} + +define <4 x float> @test_vp_splice_v4f32(<4 x float> %va, <4 x float> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v4f32: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vx v8, v9, a0 +; CHECK-NEXT: ret + %head = insertelement <4 x i1> undef, i1 1, i32 0 + %allones = shufflevector <4 x i1> %head, <4 x i1> undef, <4 x i32> zeroinitializer + + %v = call <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float> %va, <4 x float> %vb, i32 5, <4 x i1> %allones, i32 %evla, i32 %evlb) + ret <4 x float> %v +} + +define <4 x float> @test_vp_splice_v4f32_negative_offset(<4 x float> %va, <4 x float> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v4f32_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e32, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 5 +; CHECK-NEXT: ret + %head = insertelement <4 x i1> undef, i1 1, i32 0 + %allones = shufflevector <4 x i1> %head, <4 x i1> undef, <4 x i32> zeroinitializer + + %v = call <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float> %va, <4 x float> %vb, i32 -5, <4 x i1> %allones, i32 %evla, i32 %evlb) + ret <4 x float> %v +} + +define <4 x float> @test_vp_splice_v4f32_masked(<4 x float> %va, <4 x float> %vb, <4 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v4f32_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, mu +; CHECK-NEXT: vslideup.vx v8, v9, a0, v0.t +; CHECK-NEXT: ret + %v = call <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float> %va, <4 x float> %vb, i32 5, <4 x i1> %mask, i32 %evla, i32 %evlb) + ret <4 x float> %v +} diff --git a/llvm/test/CodeGen/RISCV/vp-splice-mask-fixed-vectors.ll b/llvm/test/CodeGen/RISCV/vp-splice-mask-fixed-vectors.ll new file mode 100644 index 000000000000..9579973aee0d --- /dev/null +++ b/llvm/test/CodeGen/RISCV/vp-splice-mask-fixed-vectors.ll @@ -0,0 +1,316 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs -riscv-v-vector-bits-min=128 \ +; RUN: < %s | FileCheck %s + +declare <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1>, <2 x i1>, i32, <2 x i1>, i32, i32) +declare <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1>, <4 x i1>, i32, <4 x i1>, i32, i32) +declare <8 x i1> @llvm.experimental.vp.splice.v8i1(<8 x i1>, <8 x i1>, i32, <8 x i1>, i32, i32) +declare <16 x i1> @llvm.experimental.vp.splice.v16i1(<16 x i1>, <16 x i1>, i32, <16 x i1>, i32, i32) + +define <2 x i1> @test_vp_splice_v2i1(<2 x i1> %va, <2 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v2i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, mf8, ta, ma +; CHECK-NEXT: vslidedown.vi v9, v9, 5 +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vslideup.vx v9, v8, a0 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <2 x i1> undef, i1 1, i32 0 + %allones = shufflevector <2 x i1> %head, <2 x i1> undef, <2 x i32> zeroinitializer + + %v = call <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1> %va, <2 x i1> %vb, i32 5, <2 x i1> %allones, i32 %evla, i32 %evlb) + ret <2 x i1> %v +} + +define <2 x i1> @test_vp_splice_v2i1_negative_offset(<2 x i1> %va, <2 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v2i1_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e8, mf8, ta, ma +; CHECK-NEXT: vslidedown.vx v9, v9, a0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vslideup.vi v9, v8, 5 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <2 x i1> undef, i1 1, i32 0 + %allones = shufflevector <2 x i1> %head, <2 x i1> undef, <2 x i32> zeroinitializer + + %v = call <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1> %va, <2 x i1> %vb, i32 -5, <2 x i1> %allones, i32 %evla, i32 %evlb) + ret <2 x i1> %v +} + +define <2 x i1> @test_vp_splice_v2i1_masked(<2 x i1> %va, <2 x i1> %vb, <2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v2i1_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v10, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v11, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v10 +; CHECK-NEXT: vmerge.vim v10, v11, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, mf8, ta, ma +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vslidedown.vi v10, v10, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, mu +; CHECK-NEXT: vslideup.vx v10, v8, a0, v0.t +; CHECK-NEXT: vsetvli zero, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmsne.vi v0, v10, 0, v0.t +; CHECK-NEXT: ret + %v = call <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1> %va, <2 x i1> %vb, i32 5, <2 x i1> %mask, i32 %evla, i32 %evlb) + ret <2 x i1> %v +} + +define <4 x i1> @test_vp_splice_v4i1(<4 x i1> %va, <4 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v4i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, mf4, ta, ma +; CHECK-NEXT: vslidedown.vi v9, v9, 5 +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; CHECK-NEXT: vslideup.vx v9, v8, a0 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <4 x i1> undef, i1 1, i32 0 + %allones = shufflevector <4 x i1> %head, <4 x i1> undef, <4 x i32> zeroinitializer + + %v = call <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1> %va, <4 x i1> %vb, i32 5, <4 x i1> %allones, i32 %evla, i32 %evlb) + ret <4 x i1> %v +} + +define <4 x i1> @test_vp_splice_v4i1_negative_offset(<4 x i1> %va, <4 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v4i1_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e8, mf4, ta, ma +; CHECK-NEXT: vslidedown.vx v9, v9, a0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; CHECK-NEXT: vslideup.vi v9, v8, 5 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <4 x i1> undef, i1 1, i32 0 + %allones = shufflevector <4 x i1> %head, <4 x i1> undef, <4 x i32> zeroinitializer + + %v = call <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1> %va, <4 x i1> %vb, i32 -5, <4 x i1> %allones, i32 %evla, i32 %evlb) + ret <4 x i1> %v +} + +define <4 x i1> @test_vp_splice_v4i1_masked(<4 x i1> %va, <4 x i1> %vb, <4 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v4i1_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v10, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v11, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v10 +; CHECK-NEXT: vmerge.vim v10, v11, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, mf4, ta, ma +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vslidedown.vi v10, v10, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, mu +; CHECK-NEXT: vslideup.vx v10, v8, a0, v0.t +; CHECK-NEXT: vsetvli zero, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmsne.vi v0, v10, 0, v0.t +; CHECK-NEXT: ret + %v = call <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1> %va, <4 x i1> %vb, i32 5, <4 x i1> %mask, i32 %evla, i32 %evlb) + ret <4 x i1> %v +} + +define <8 x i1> @test_vp_splice_v8i1(<8 x i1> %va, <8 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v8i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, mf2, ta, ma +; CHECK-NEXT: vslidedown.vi v9, v9, 5 +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; CHECK-NEXT: vslideup.vx v9, v8, a0 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <8 x i1> undef, i1 1, i32 0 + %allones = shufflevector <8 x i1> %head, <8 x i1> undef, <8 x i32> zeroinitializer + + %v = call <8 x i1> @llvm.experimental.vp.splice.v8i1(<8 x i1> %va, <8 x i1> %vb, i32 5, <8 x i1> %allones, i32 %evla, i32 %evlb) + ret <8 x i1> %v +} + +define <8 x i1> @test_vp_splice_v8i1_negative_offset(<8 x i1> %va, <8 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v8i1_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e8, mf2, ta, ma +; CHECK-NEXT: vslidedown.vx v9, v9, a0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v9, v8, 5 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <8 x i1> undef, i1 1, i32 0 + %allones = shufflevector <8 x i1> %head, <8 x i1> undef, <8 x i32> zeroinitializer + + %v = call <8 x i1> @llvm.experimental.vp.splice.v8i1(<8 x i1> %va, <8 x i1> %vb, i32 -5, <8 x i1> %allones, i32 %evla, i32 %evlb) + ret <8 x i1> %v +} + +define <8 x i1> @test_vp_splice_v8i1_masked(<8 x i1> %va, <8 x i1> %vb, <8 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v8i1_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v10, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v11, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v10 +; CHECK-NEXT: vmerge.vim v10, v11, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, mf2, ta, ma +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vslidedown.vi v10, v10, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, mu +; CHECK-NEXT: vslideup.vx v10, v8, a0, v0.t +; CHECK-NEXT: vsetvli zero, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmsne.vi v0, v10, 0, v0.t +; CHECK-NEXT: ret + %v = call <8 x i1> @llvm.experimental.vp.splice.v8i1(<8 x i1> %va, <8 x i1> %vb, i32 5, <8 x i1> %mask, i32 %evla, i32 %evlb) + ret <8 x i1> %v +} + +define <16 x i1> @test_vp_splice_v16i1(<16 x i1> %va, <16 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v16i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v9, v9, 5 +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vslideup.vx v9, v8, a0 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <16 x i1> undef, i1 1, i32 0 + %allones = shufflevector <16 x i1> %head, <16 x i1> undef, <16 x i32> zeroinitializer + + %v = call <16 x i1> @llvm.experimental.vp.splice.v16i1(<16 x i1> %va, <16 x i1> %vb, i32 5, <16 x i1> %allones, i32 %evla, i32 %evlb) + ret <16 x i1> %v +} + +define <16 x i1> @test_vp_splice_v16i1_negative_offset(<16 x i1> %va, <16 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v16i1_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e8, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v9, v9, a0 +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vslideup.vi v9, v8, 5 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <16 x i1> undef, i1 1, i32 0 + %allones = shufflevector <16 x i1> %head, <16 x i1> undef, <16 x i32> zeroinitializer + + %v = call <16 x i1> @llvm.experimental.vp.splice.v16i1(<16 x i1> %va, <16 x i1> %vb, i32 -5, <16 x i1> %allones, i32 %evla, i32 %evlb) + ret <16 x i1> %v +} + +define <16 x i1> @test_vp_splice_v16i1_masked(<16 x i1> %va, <16 x i1> %vb, <16 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_v16i1_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v10, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v11, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v10 +; CHECK-NEXT: vmerge.vim v10, v11, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, ma +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vslidedown.vi v10, v10, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, mu +; CHECK-NEXT: vslideup.vx v10, v8, a0, v0.t +; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma +; CHECK-NEXT: vmsne.vi v0, v10, 0, v0.t +; CHECK-NEXT: ret + %v = call <16 x i1> @llvm.experimental.vp.splice.v16i1(<16 x i1> %va, <16 x i1> %vb, i32 5, <16 x i1> %mask, i32 %evla, i32 %evlb) + ret <16 x i1> %v +} diff --git a/llvm/test/CodeGen/RISCV/vp-splice-mask-vectors.ll b/llvm/test/CodeGen/RISCV/vp-splice-mask-vectors.ll new file mode 100644 index 000000000000..4eaadb3c24fb --- /dev/null +++ b/llvm/test/CodeGen/RISCV/vp-splice-mask-vectors.ll @@ -0,0 +1,553 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs \ +; RUN: < %s | FileCheck %s + +declare <vscale x 1 x i1> @llvm.experimental.vp.splice.nxv1i1(<vscale x 1 x i1>, <vscale x 1 x i1>, i32, <vscale x 1 x i1>, i32, i32) +declare <vscale x 2 x i1> @llvm.experimental.vp.splice.nxv2i1(<vscale x 2 x i1>, <vscale x 2 x i1>, i32, <vscale x 2 x i1>, i32, i32) +declare <vscale x 4 x i1> @llvm.experimental.vp.splice.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>, i32, <vscale x 4 x i1>, i32, i32) +declare <vscale x 8 x i1> @llvm.experimental.vp.splice.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>, i32, <vscale x 8 x i1>, i32, i32) +declare <vscale x 16 x i1> @llvm.experimental.vp.splice.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>, i32, <vscale x 16 x i1>, i32, i32) +declare <vscale x 32 x i1> @llvm.experimental.vp.splice.nxv32i1(<vscale x 32 x i1>, <vscale x 32 x i1>, i32, <vscale x 32 x i1>, i32, i32) +declare <vscale x 64 x i1> @llvm.experimental.vp.splice.nxv64i1(<vscale x 64 x i1>, <vscale x 64 x i1>, i32, <vscale x 64 x i1>, i32, i32) + +define <vscale x 1 x i1> @test_vp_splice_nxv1i1(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv1i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, mf8, ta, ma +; CHECK-NEXT: vslidedown.vi v9, v9, 5 +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vslideup.vx v9, v8, a0 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 1 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 1 x i1> %head, <vscale x 1 x i1> undef, <vscale x 1 x i32> zeroinitializer + + %v = call <vscale x 1 x i1> @llvm.experimental.vp.splice.nxv1i1(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 5, <vscale x 1 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 1 x i1> %v +} + +define <vscale x 1 x i1> @test_vp_splice_nxv1i1_negative_offset(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv1i1_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e8, mf8, ta, ma +; CHECK-NEXT: vslidedown.vx v9, v9, a0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vslideup.vi v9, v8, 5 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 1 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 1 x i1> %head, <vscale x 1 x i1> undef, <vscale x 1 x i32> zeroinitializer + + %v = call <vscale x 1 x i1> @llvm.experimental.vp.splice.nxv1i1(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 -5, <vscale x 1 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 1 x i1> %v +} + +define <vscale x 1 x i1> @test_vp_splice_nxv1i1_masked(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, <vscale x 1 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv1i1_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v10, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v11, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v10 +; CHECK-NEXT: vmerge.vim v10, v11, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, mf8, ta, ma +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vslidedown.vi v10, v10, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, mu +; CHECK-NEXT: vslideup.vx v10, v8, a0, v0.t +; CHECK-NEXT: vsetvli zero, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmsne.vi v0, v10, 0, v0.t +; CHECK-NEXT: ret + %v = call <vscale x 1 x i1> @llvm.experimental.vp.splice.nxv1i1(<vscale x 1 x i1> %va, <vscale x 1 x i1> %vb, i32 5, <vscale x 1 x i1> %mask, i32 %evla, i32 %evlb) + ret <vscale x 1 x i1> %v +} + +define <vscale x 2 x i1> @test_vp_splice_nxv2i1(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv2i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, mf4, ta, ma +; CHECK-NEXT: vslidedown.vi v9, v9, 5 +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; CHECK-NEXT: vslideup.vx v9, v8, a0 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 2 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 2 x i1> %head, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer + + %v = call <vscale x 2 x i1> @llvm.experimental.vp.splice.nxv2i1(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 5, <vscale x 2 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 2 x i1> %v +} + +define <vscale x 2 x i1> @test_vp_splice_nxv2i1_negative_offset(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv2i1_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e8, mf4, ta, ma +; CHECK-NEXT: vslidedown.vx v9, v9, a0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; CHECK-NEXT: vslideup.vi v9, v8, 5 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 2 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 2 x i1> %head, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer + + %v = call <vscale x 2 x i1> @llvm.experimental.vp.splice.nxv2i1(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 -5, <vscale x 2 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 2 x i1> %v +} + +define <vscale x 2 x i1> @test_vp_splice_nxv2i1_masked(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv2i1_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v10, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v11, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v10 +; CHECK-NEXT: vmerge.vim v10, v11, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, mf4, ta, ma +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vslidedown.vi v10, v10, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, mu +; CHECK-NEXT: vslideup.vx v10, v8, a0, v0.t +; CHECK-NEXT: vsetvli zero, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmsne.vi v0, v10, 0, v0.t +; CHECK-NEXT: ret + %v = call <vscale x 2 x i1> @llvm.experimental.vp.splice.nxv2i1(<vscale x 2 x i1> %va, <vscale x 2 x i1> %vb, i32 5, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb) + ret <vscale x 2 x i1> %v +} + +define <vscale x 4 x i1> @test_vp_splice_nxv4i1(<vscale x 4 x i1> %va, <vscale x 4 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv4i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, mf2, ta, ma +; CHECK-NEXT: vslidedown.vi v9, v9, 5 +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; CHECK-NEXT: vslideup.vx v9, v8, a0 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 4 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 4 x i1> %head, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer + + %v = call <vscale x 4 x i1> @llvm.experimental.vp.splice.nxv4i1(<vscale x 4 x i1> %va, <vscale x 4 x i1> %vb, i32 5, <vscale x 4 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 4 x i1> %v +} + +define <vscale x 4 x i1> @test_vp_splice_nxv4i1_negative_offset(<vscale x 4 x i1> %va, <vscale x 4 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv4i1_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e8, mf2, ta, ma +; CHECK-NEXT: vslidedown.vx v9, v9, a0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v9, v8, 5 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 4 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 4 x i1> %head, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer + + %v = call <vscale x 4 x i1> @llvm.experimental.vp.splice.nxv4i1(<vscale x 4 x i1> %va, <vscale x 4 x i1> %vb, i32 -5, <vscale x 4 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 4 x i1> %v +} + +define <vscale x 4 x i1> @test_vp_splice_nxv4i1_masked(<vscale x 4 x i1> %va, <vscale x 4 x i1> %vb, <vscale x 4 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv4i1_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v10, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v11, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v10 +; CHECK-NEXT: vmerge.vim v10, v11, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, mf2, ta, ma +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vslidedown.vi v10, v10, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, mu +; CHECK-NEXT: vslideup.vx v10, v8, a0, v0.t +; CHECK-NEXT: vsetvli zero, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmsne.vi v0, v10, 0, v0.t +; CHECK-NEXT: ret + %v = call <vscale x 4 x i1> @llvm.experimental.vp.splice.nxv4i1(<vscale x 4 x i1> %va, <vscale x 4 x i1> %vb, i32 5, <vscale x 4 x i1> %mask, i32 %evla, i32 %evlb) + ret <vscale x 4 x i1> %v +} + +define <vscale x 8 x i1> @test_vp_splice_nxv8i1(<vscale x 8 x i1> %va, <vscale x 8 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv8i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v9, v9, 5 +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vslideup.vx v9, v8, a0 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 8 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 8 x i1> %head, <vscale x 8 x i1> undef, <vscale x 8 x i32> zeroinitializer + + %v = call <vscale x 8 x i1> @llvm.experimental.vp.splice.nxv8i1(<vscale x 8 x i1> %va, <vscale x 8 x i1> %vb, i32 5, <vscale x 8 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 8 x i1> %v +} + +define <vscale x 8 x i1> @test_vp_splice_nxv8i1_negative_offset(<vscale x 8 x i1> %va, <vscale x 8 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv8i1_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v9, v10, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e8, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v9, v9, a0 +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vslideup.vi v9, v8, 5 +; CHECK-NEXT: vmsne.vi v0, v9, 0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 8 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 8 x i1> %head, <vscale x 8 x i1> undef, <vscale x 8 x i32> zeroinitializer + + %v = call <vscale x 8 x i1> @llvm.experimental.vp.splice.nxv8i1(<vscale x 8 x i1> %va, <vscale x 8 x i1> %vb, i32 -5, <vscale x 8 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 8 x i1> %v +} + +define <vscale x 8 x i1> @test_vp_splice_nxv8i1_masked(<vscale x 8 x i1> %va, <vscale x 8 x i1> %vb, <vscale x 8 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv8i1_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v10, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v8, v11, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.i v11, 0 +; CHECK-NEXT: vmv1r.v v0, v10 +; CHECK-NEXT: vmerge.vim v10, v11, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, ma +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vslidedown.vi v10, v10, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, mu +; CHECK-NEXT: vslideup.vx v10, v8, a0, v0.t +; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma +; CHECK-NEXT: vmsne.vi v0, v10, 0, v0.t +; CHECK-NEXT: ret + %v = call <vscale x 8 x i1> @llvm.experimental.vp.splice.nxv8i1(<vscale x 8 x i1> %va, <vscale x 8 x i1> %vb, i32 5, <vscale x 8 x i1> %mask, i32 %evla, i32 %evlb) + ret <vscale x 8 x i1> %v +} + +define <vscale x 16 x i1> @test_vp_splice_nxv16i1(<vscale x 16 x i1> %va, <vscale x 16 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv16i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v10, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, m2, ta, ma +; CHECK-NEXT: vmv.v.i v12, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v8, v12, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, m2, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; CHECK-NEXT: vslideup.vx v8, v10, a0 +; CHECK-NEXT: vmsne.vi v0, v8, 0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 16 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 16 x i1> %head, <vscale x 16 x i1> undef, <vscale x 16 x i32> zeroinitializer + + %v = call <vscale x 16 x i1> @llvm.experimental.vp.splice.nxv16i1(<vscale x 16 x i1> %va, <vscale x 16 x i1> %vb, i32 5, <vscale x 16 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 16 x i1> %v +} + +define <vscale x 16 x i1> @test_vp_splice_nxv16i1_negative_offset(<vscale x 16 x i1> %va, <vscale x 16 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv16i1_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v10, v10, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, m2, ta, ma +; CHECK-NEXT: vmv.v.i v12, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v8, v12, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e8, m2, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v10, 5 +; CHECK-NEXT: vmsne.vi v0, v8, 0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 16 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 16 x i1> %head, <vscale x 16 x i1> undef, <vscale x 16 x i32> zeroinitializer + + %v = call <vscale x 16 x i1> @llvm.experimental.vp.splice.nxv16i1(<vscale x 16 x i1> %va, <vscale x 16 x i1> %vb, i32 -5, <vscale x 16 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 16 x i1> %v +} + +define <vscale x 16 x i1> @test_vp_splice_nxv16i1_masked(<vscale x 16 x i1> %va, <vscale x 16 x i1> %vb, <vscale x 16 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv16i1_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v10, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; CHECK-NEXT: vmv.v.i v12, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v12, v12, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, m2, ta, ma +; CHECK-NEXT: vmv.v.i v14, 0 +; CHECK-NEXT: vmv1r.v v0, v10 +; CHECK-NEXT: vmerge.vim v10, v14, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, m2, ta, ma +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vslidedown.vi v10, v10, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, mu +; CHECK-NEXT: vslideup.vx v10, v12, a0, v0.t +; CHECK-NEXT: vsetvli zero, zero, e8, m2, ta, ma +; CHECK-NEXT: vmsne.vi v8, v10, 0, v0.t +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: ret + %v = call <vscale x 16 x i1> @llvm.experimental.vp.splice.nxv16i1(<vscale x 16 x i1> %va, <vscale x 16 x i1> %vb, i32 5, <vscale x 16 x i1> %mask, i32 %evla, i32 %evlb) + ret <vscale x 16 x i1> %v +} + +define <vscale x 32 x i1> @test_vp_splice_nxv32i1(<vscale x 32 x i1> %va, <vscale x 32 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv32i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, m4, ta, ma +; CHECK-NEXT: vmv.v.i v12, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v12, v12, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, m4, ta, ma +; CHECK-NEXT: vmv.v.i v16, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v8, v16, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, m4, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e8, m4, ta, ma +; CHECK-NEXT: vslideup.vx v8, v12, a0 +; CHECK-NEXT: vmsne.vi v0, v8, 0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 32 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 32 x i1> %head, <vscale x 32 x i1> undef, <vscale x 32 x i32> zeroinitializer + + %v = call <vscale x 32 x i1> @llvm.experimental.vp.splice.nxv32i1(<vscale x 32 x i1> %va, <vscale x 32 x i1> %vb, i32 5, <vscale x 32 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 32 x i1> %v +} + +define <vscale x 32 x i1> @test_vp_splice_nxv32i1_negative_offset(<vscale x 32 x i1> %va, <vscale x 32 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv32i1_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, m4, ta, ma +; CHECK-NEXT: vmv.v.i v12, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v12, v12, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, m4, ta, ma +; CHECK-NEXT: vmv.v.i v16, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v8, v16, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e8, m4, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e8, m4, ta, ma +; CHECK-NEXT: vslideup.vi v8, v12, 5 +; CHECK-NEXT: vmsne.vi v0, v8, 0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 32 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 32 x i1> %head, <vscale x 32 x i1> undef, <vscale x 32 x i32> zeroinitializer + + %v = call <vscale x 32 x i1> @llvm.experimental.vp.splice.nxv32i1(<vscale x 32 x i1> %va, <vscale x 32 x i1> %vb, i32 -5, <vscale x 32 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 32 x i1> %v +} + +define <vscale x 32 x i1> @test_vp_splice_nxv32i1_masked(<vscale x 32 x i1> %va, <vscale x 32 x i1> %vb, <vscale x 32 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv32i1_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v10, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, m4, ta, ma +; CHECK-NEXT: vmv.v.i v12, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v12, v12, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, m4, ta, ma +; CHECK-NEXT: vmv.v.i v16, 0 +; CHECK-NEXT: vmv1r.v v0, v10 +; CHECK-NEXT: vmerge.vim v16, v16, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, m4, ta, ma +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vslidedown.vi v16, v16, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e8, m4, ta, mu +; CHECK-NEXT: vslideup.vx v16, v12, a0, v0.t +; CHECK-NEXT: vsetvli zero, zero, e8, m4, ta, ma +; CHECK-NEXT: vmsne.vi v8, v16, 0, v0.t +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: ret + %v = call <vscale x 32 x i1> @llvm.experimental.vp.splice.nxv32i1(<vscale x 32 x i1> %va, <vscale x 32 x i1> %vb, i32 5, <vscale x 32 x i1> %mask, i32 %evla, i32 %evlb) + ret <vscale x 32 x i1> %v +} + +define <vscale x 64 x i1> @test_vp_splice_nxv64i1(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv64i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma +; CHECK-NEXT: vmv.v.i v16, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v16, v16, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, m8, ta, ma +; CHECK-NEXT: vmv.v.i v24, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v8, v24, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, m8, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma +; CHECK-NEXT: vslideup.vx v8, v16, a0 +; CHECK-NEXT: vmsne.vi v0, v8, 0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 64 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 64 x i1> %head, <vscale x 64 x i1> undef, <vscale x 64 x i32> zeroinitializer + + %v = call <vscale x 64 x i1> @llvm.experimental.vp.splice.nxv64i1(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, i32 5, <vscale x 64 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 64 x i1> %v +} + +define <vscale x 64 x i1> @test_vp_splice_nxv64i1_negative_offset(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv64i1_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v9, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma +; CHECK-NEXT: vmv.v.i v16, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v16, v16, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, m8, ta, ma +; CHECK-NEXT: vmv.v.i v24, 0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vmerge.vim v8, v24, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e8, m8, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma +; CHECK-NEXT: vslideup.vi v8, v16, 5 +; CHECK-NEXT: vmsne.vi v0, v8, 0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 64 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 64 x i1> %head, <vscale x 64 x i1> undef, <vscale x 64 x i32> zeroinitializer + + %v = call <vscale x 64 x i1> @llvm.experimental.vp.splice.nxv64i1(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, i32 -5, <vscale x 64 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 64 x i1> %v +} + +define <vscale x 64 x i1> @test_vp_splice_nxv64i1_masked(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, <vscale x 64 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv64i1_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: vmv1r.v v10, v0 +; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma +; CHECK-NEXT: vmv.v.i v16, 0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vmerge.vim v16, v16, 1, v0 +; CHECK-NEXT: vsetvli zero, a0, e8, m8, ta, ma +; CHECK-NEXT: vmv.v.i v24, 0 +; CHECK-NEXT: vmv1r.v v0, v10 +; CHECK-NEXT: vmerge.vim v24, v24, 1, v0 +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, m8, ta, ma +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vslidedown.vi v24, v24, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, mu +; CHECK-NEXT: vslideup.vx v24, v16, a0, v0.t +; CHECK-NEXT: vsetvli zero, zero, e8, m8, ta, ma +; CHECK-NEXT: vmsne.vi v8, v24, 0, v0.t +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: ret + %v = call <vscale x 64 x i1> @llvm.experimental.vp.splice.nxv64i1(<vscale x 64 x i1> %va, <vscale x 64 x i1> %vb, i32 5, <vscale x 64 x i1> %mask, i32 %evla, i32 %evlb) + ret <vscale x 64 x i1> %v +} diff --git a/llvm/test/CodeGen/RISCV/vp-splice.ll b/llvm/test/CodeGen/RISCV/vp-splice.ll new file mode 100644 index 000000000000..7d85370e390b --- /dev/null +++ b/llvm/test/CodeGen/RISCV/vp-splice.ll @@ -0,0 +1,330 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple riscv64 -mattr=+f,+d,+v -verify-machineinstrs \ +; RUN: < %s | FileCheck %s + +declare <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, i32, <vscale x 2 x i1>, i32, i32) + +declare <vscale x 1 x i64> @llvm.experimental.vp.splice.nxv1i64(<vscale x 1 x i64>, <vscale x 1 x i64>, i32, <vscale x 1 x i1>, i32, i32) +declare <vscale x 2 x i32> @llvm.experimental.vp.splice.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>, i32, <vscale x 2 x i1>, i32, i32) +declare <vscale x 4 x i16> @llvm.experimental.vp.splice.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>, i32, <vscale x 4 x i1>, i32, i32) +declare <vscale x 8 x i8> @llvm.experimental.vp.splice.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>, i32, <vscale x 8 x i1>, i32, i32) + +declare <vscale x 1 x double> @llvm.experimental.vp.splice.nxv1f64(<vscale x 1 x double>, <vscale x 1 x double>, i32, <vscale x 1 x i1>, i32, i32) +declare <vscale x 2 x float> @llvm.experimental.vp.splice.nxv2f32(<vscale x 2 x float>, <vscale x 2 x float>, i32, <vscale x 2 x i1>, i32, i32) + +declare <vscale x 16 x i64> @llvm.experimental.vp.splice.nxv16i64(<vscale x 16 x i64>, <vscale x 16 x i64>, i32, <vscale x 16 x i1>, i32, i32) + +define <vscale x 2 x i64> @test_vp_splice_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv2i64: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e64, m2, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e64, m2, ta, ma +; CHECK-NEXT: vslideup.vx v8, v10, a0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 2 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 2 x i1> %head, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer + + %v = call <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 5, <vscale x 2 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 2 x i64> %v +} + +define <vscale x 2 x i64> @test_vp_splice_nxv2i64_negative_offset(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv2i64_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e64, m2, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e64, m2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v10, 5 +; CHECK-NEXT: ret + %head = insertelement <vscale x 2 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 2 x i1> %head, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer + + %v = call <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 -5, <vscale x 2 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 2 x i64> %v +} + +define <vscale x 2 x i64> @test_vp_splice_nxv2i64_masked(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv2i64_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e64, m2, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e64, m2, ta, mu +; CHECK-NEXT: vslideup.vx v8, v10, a0, v0.t +; CHECK-NEXT: ret + %v = call <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 5, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb) + ret <vscale x 2 x i64> %v +} + +define <vscale x 1 x i64> @test_vp_splice_nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv1i64: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e64, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; CHECK-NEXT: vslideup.vx v8, v9, a0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 1 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 1 x i1> %head, <vscale x 1 x i1> undef, <vscale x 1 x i32> zeroinitializer + + %v = call <vscale x 1 x i64> @llvm.experimental.vp.splice.nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, i32 5, <vscale x 1 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 1 x i64> %v +} + +define <vscale x 1 x i64> @test_vp_splice_nxv1i64_negative_offset(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv1i64_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e64, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 5 +; CHECK-NEXT: ret + %head = insertelement <vscale x 1 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 1 x i1> %head, <vscale x 1 x i1> undef, <vscale x 1 x i32> zeroinitializer + + %v = call <vscale x 1 x i64> @llvm.experimental.vp.splice.nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, i32 -5, <vscale x 1 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 1 x i64> %v +} + +define <vscale x 1 x i64> @test_vp_splice_nxv1i64_masked(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, <vscale x 1 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv1i64_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e64, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, mu +; CHECK-NEXT: vslideup.vx v8, v9, a0, v0.t +; CHECK-NEXT: ret + %v = call <vscale x 1 x i64> @llvm.experimental.vp.splice.nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x i64> %vb, i32 5, <vscale x 1 x i1> %mask, i32 %evla, i32 %evlb) + ret <vscale x 1 x i64> %v +} + +define <vscale x 2 x i32> @test_vp_splice_nxv2i32(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv2i32: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vx v8, v9, a0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 2 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 2 x i1> %head, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer + + %v = call <vscale x 2 x i32> @llvm.experimental.vp.splice.nxv2i32(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 5, <vscale x 2 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 2 x i32> %v +} + +define <vscale x 2 x i32> @test_vp_splice_nxv2i32_negative_offset(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv2i32_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e32, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 5 +; CHECK-NEXT: ret + %head = insertelement <vscale x 2 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 2 x i1> %head, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer + + %v = call <vscale x 2 x i32> @llvm.experimental.vp.splice.nxv2i32(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 -5, <vscale x 2 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 2 x i32> %v +} + +define <vscale x 2 x i32> @test_vp_splice_nxv2i32_masked(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv2i32_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, mu +; CHECK-NEXT: vslideup.vx v8, v9, a0, v0.t +; CHECK-NEXT: ret + %v = call <vscale x 2 x i32> @llvm.experimental.vp.splice.nxv2i32(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 5, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb) + ret <vscale x 2 x i32> %v +} + +define <vscale x 4 x i16> @test_vp_splice_nxv4i16(<vscale x 4 x i16> %va, <vscale x 4 x i16> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e16, m1, ta, ma +; CHECK-NEXT: vslideup.vx v8, v9, a0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 4 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 4 x i1> %head, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer + + %v = call <vscale x 4 x i16> @llvm.experimental.vp.splice.nxv4i16(<vscale x 4 x i16> %va, <vscale x 4 x i16> %vb, i32 5, <vscale x 4 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 4 x i16> %v +} + +define <vscale x 4 x i16> @test_vp_splice_nxv4i16_negative_offset(<vscale x 4 x i16> %va, <vscale x 4 x i16> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv4i16_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e16, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e16, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 5 +; CHECK-NEXT: ret + %head = insertelement <vscale x 4 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 4 x i1> %head, <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer + + %v = call <vscale x 4 x i16> @llvm.experimental.vp.splice.nxv4i16(<vscale x 4 x i16> %va, <vscale x 4 x i16> %vb, i32 -5, <vscale x 4 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 4 x i16> %v +} + +define <vscale x 4 x i16> @test_vp_splice_nxv4i16_masked(<vscale x 4 x i16> %va, <vscale x 4 x i16> %vb, <vscale x 4 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv4i16_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e16, m1, ta, mu +; CHECK-NEXT: vslideup.vx v8, v9, a0, v0.t +; CHECK-NEXT: ret + %v = call <vscale x 4 x i16> @llvm.experimental.vp.splice.nxv4i16(<vscale x 4 x i16> %va, <vscale x 4 x i16> %vb, i32 5, <vscale x 4 x i1> %mask, i32 %evla, i32 %evlb) + ret <vscale x 4 x i16> %v +} + +define <vscale x 8 x i8> @test_vp_splice_nxv8i8(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vslideup.vx v8, v9, a0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 8 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 8 x i1> %head, <vscale x 8 x i1> undef, <vscale x 8 x i32> zeroinitializer + + %v = call <vscale x 8 x i8> @llvm.experimental.vp.splice.nxv8i8(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb, i32 5, <vscale x 8 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 8 x i8> %v +} + +define <vscale x 8 x i8> @test_vp_splice_nxv8i8_negative_offset(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv8i8_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e8, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 5 +; CHECK-NEXT: ret + %head = insertelement <vscale x 8 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 8 x i1> %head, <vscale x 8 x i1> undef, <vscale x 8 x i32> zeroinitializer + + %v = call <vscale x 8 x i8> @llvm.experimental.vp.splice.nxv8i8(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb, i32 -5, <vscale x 8 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 8 x i8> %v +} + +define <vscale x 8 x i8> @test_vp_splice_nxv8i8_masked(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb, <vscale x 8 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv8i8_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e8, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, mu +; CHECK-NEXT: vslideup.vx v8, v9, a0, v0.t +; CHECK-NEXT: ret + %v = call <vscale x 8 x i8> @llvm.experimental.vp.splice.nxv8i8(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb, i32 5, <vscale x 8 x i1> %mask, i32 %evla, i32 %evlb) + ret <vscale x 8 x i8> %v +} + +define <vscale x 1 x double> @test_vp_splice_nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv1f64: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e64, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; CHECK-NEXT: vslideup.vx v8, v9, a0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 1 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 1 x i1> %head, <vscale x 1 x i1> undef, <vscale x 1 x i32> zeroinitializer + + %v = call <vscale x 1 x double> @llvm.experimental.vp.splice.nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, i32 5, <vscale x 1 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 1 x double> %v +} + +define <vscale x 1 x double> @test_vp_splice_nxv1f64_negative_offset(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv1f64_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e64, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 5 +; CHECK-NEXT: ret + %head = insertelement <vscale x 1 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 1 x i1> %head, <vscale x 1 x i1> undef, <vscale x 1 x i32> zeroinitializer + + %v = call <vscale x 1 x double> @llvm.experimental.vp.splice.nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, i32 -5, <vscale x 1 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 1 x double> %v +} + +define <vscale x 1 x double> @test_vp_splice_nxv1f64_masked(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, <vscale x 1 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv1f64_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e64, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, mu +; CHECK-NEXT: vslideup.vx v8, v9, a0, v0.t +; CHECK-NEXT: ret + %v = call <vscale x 1 x double> @llvm.experimental.vp.splice.nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, i32 5, <vscale x 1 x i1> %mask, i32 %evla, i32 %evlb) + ret <vscale x 1 x double> %v +} + +define <vscale x 2 x float> @test_vp_splice_nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv2f32: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vx v8, v9, a0 +; CHECK-NEXT: ret + %head = insertelement <vscale x 2 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 2 x i1> %head, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer + + %v = call <vscale x 2 x float> @llvm.experimental.vp.splice.nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 5, <vscale x 2 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 2 x float> %v +} + +define <vscale x 2 x float> @test_vp_splice_nxv2f32_negative_offset(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv2f32_negative_offset: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetivli zero, 5, e32, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 5 +; CHECK-NEXT: ret + %head = insertelement <vscale x 2 x i1> undef, i1 1, i32 0 + %allones = shufflevector <vscale x 2 x i1> %head, <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer + + %v = call <vscale x 2 x float> @llvm.experimental.vp.splice.nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 -5, <vscale x 2 x i1> %allones, i32 %evla, i32 %evlb) + ret <vscale x 2 x float> %v +} + +define <vscale x 2 x float> @test_vp_splice_nxv2f32_masked(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) { +; CHECK-LABEL: test_vp_splice_nxv2f32_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: addi a0, a0, -5 +; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma +; CHECK-NEXT: vslidedown.vi v8, v8, 5, v0.t +; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, mu +; CHECK-NEXT: vslideup.vx v8, v9, a0, v0.t +; CHECK-NEXT: ret + %v = call <vscale x 2 x float> @llvm.experimental.vp.splice.nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 5, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb) + ret <vscale x 2 x float> %v +} diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-05.ll b/llvm/test/CodeGen/SystemZ/atomic-load-05.ll index 979f1e684e89..f406bc6d2ce6 100644 --- a/llvm/test/CodeGen/SystemZ/atomic-load-05.ll +++ b/llvm/test/CodeGen/SystemZ/atomic-load-05.ll @@ -1,14 +1,22 @@ -; Test 128-bit atomic loads. +; Test 128-bit integer atomic loads. ; ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s define i128 @f1(ptr %src) { ; CHECK-LABEL: f1: -; CHECK: lpq %r0, 0(%r3) -; CHECK-DAG: stg %r1, 8(%r2) -; CHECK-DAG: stg %r0, 0(%r2) -; CHECK: br %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: lpq %r0, 0(%r3) +; CHECK-NEXT: stg %r1, 8(%r2) +; CHECK-NEXT: stg %r0, 0(%r2) +; CHECK-NEXT: br %r14 %val = load atomic i128, ptr %src seq_cst, align 16 ret i128 %val } + +define i128 @f2(ptr %src) { +; CHECK-LABEL: f2: +; CHECK: brasl %r14, __atomic_load@PLT + %val = load atomic i128, ptr %src seq_cst, align 8 + ret i128 %val +} diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-08.ll b/llvm/test/CodeGen/SystemZ/atomic-load-08.ll index 069d2168e19a..4d914e3ea0e1 100644 --- a/llvm/test/CodeGen/SystemZ/atomic-load-08.ll +++ b/llvm/test/CodeGen/SystemZ/atomic-load-08.ll @@ -1,19 +1,25 @@ -; Test long double atomic loads. Expect a libcall. +; Test long double atomic loads. These are emitted by the Clang FE as i128 +; loads with a bitcast, and this test case gets converted into that form as +; well by the AtomicExpand pass. ; ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s +; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s define void @f1(ptr %ret, ptr %src) { ; CHECK-LABEL: f1: -; CHECK: lgr [[RET:%r[0-9]+]], %r2 -; CHECK: la %r4, 160(%r15) -; CHECK: lghi %r2, 16 -; CHECK: lhi %r5, 5 +; CHECK: # %bb.0: +; CHECK-NEXT: lpq %r0, 0(%r3) +; CHECK-NEXT: stg %r1, 8(%r2) +; CHECK-NEXT: stg %r0, 0(%r2) +; CHECK-NEXT: br %r14 + %val = load atomic fp128, ptr %src seq_cst, align 16 + store fp128 %val, ptr %ret, align 8 + ret void +} + +define void @f2(ptr %ret, ptr %src) { +; CHECK-LABEL: f2: ; CHECK: brasl %r14, __atomic_load@PLT -; CHECK: ld [[FL:%f[0-9]+]], 160(%r15) -; CHECK: ld [[FH:%f[0-9]+]], 168(%r15) -; CHECK: std [[FL]], 0([[RET]]) -; CHECK: std [[FH]], 8([[RET]]) -; CHECK: br %r14 %val = load atomic fp128, ptr %src seq_cst, align 8 store fp128 %val, ptr %ret, align 8 ret void diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-05.ll b/llvm/test/CodeGen/SystemZ/atomic-store-05.ll index dad7d9527b84..e4af7ad57e38 100644 --- a/llvm/test/CodeGen/SystemZ/atomic-store-05.ll +++ b/llvm/test/CodeGen/SystemZ/atomic-store-05.ll @@ -1,4 +1,4 @@ -; Test 128-bit atomic stores. +; Test 128-bit integer atomic stores. ; ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s @@ -24,3 +24,10 @@ define void @f2(i128 %val, ptr %src) { store atomic i128 %val, ptr %src monotonic, align 16 ret void } + +define void @f3(i128 %val, ptr %src) { +; CHECK-LABEL: f3: +; CHECK: brasl %r14, __atomic_store@PLT + store atomic i128 %val, ptr %src seq_cst, align 8 + ret void +} diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-06.ll b/llvm/test/CodeGen/SystemZ/atomic-store-06.ll index fd39793faefc..b748bfc767a4 100644 --- a/llvm/test/CodeGen/SystemZ/atomic-store-06.ll +++ b/llvm/test/CodeGen/SystemZ/atomic-store-06.ll @@ -1,13 +1,17 @@ -; Test float atomic loads. +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; Test float atomic stores. ; ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s define void @f1(ptr %src, float %val) { ; CHECK-LABEL: f1: -; CHECK: lgdr [[R:%r[0-9]+]], %f0 -; CHECK: srlg [[R]], [[R]], 32 -; CHECK: st [[R]], 0(%r2) -; CHECK: br %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: # kill: def $f0s killed $f0s def $f0d +; CHECK-NEXT: lgdr %r0, %f0 +; CHECK-NEXT: srlg %r0, %r0, 32 +; CHECK-NEXT: st %r0, 0(%r2) +; CHECK-NEXT: bcr 15, %r0 +; CHECK-NEXT: br %r14 store atomic float %val, ptr %src seq_cst, align 4 ret void } diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-07.ll b/llvm/test/CodeGen/SystemZ/atomic-store-07.ll index c904b738f2c5..11f81ae1e07d 100644 --- a/llvm/test/CodeGen/SystemZ/atomic-store-07.ll +++ b/llvm/test/CodeGen/SystemZ/atomic-store-07.ll @@ -1,11 +1,14 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; Test double atomic stores. ; ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s define void @f1(ptr %dst, double %val) { ; CHECK-LABEL: f1: -; CHECK: std %f0, 0(%r2) -; CHECK: br %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: std %f0, 0(%r2) +; CHECK-NEXT: bcr 15, %r0 +; CHECK-NEXT: br %r14 store atomic double %val, ptr %dst seq_cst, align 8 ret void } diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-08.ll b/llvm/test/CodeGen/SystemZ/atomic-store-08.ll index b33b283e8dbd..f7f4f4d967db 100644 --- a/llvm/test/CodeGen/SystemZ/atomic-store-08.ll +++ b/llvm/test/CodeGen/SystemZ/atomic-store-08.ll @@ -1,19 +1,25 @@ -; Test long double atomic stores. Expect a libcall. +; Test long double atomic stores. The atomic store is converted to i128 by +; the AtomicExpand pass. ; ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s +; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s define void @f1(ptr %dst, ptr %src) { ; CHECK-LABEL: f1: -; CHECK: ld [[FL:%f[0-9]+]], 0(%r3) -; CHECK: ld [[FH:%f[0-9]+]], 8(%r3) -; CHECK: lgr %r3, %r2 -; CHECK: std [[FL]], 160(%r15) -; CHECK: std [[FH]], 168(%r15) -; CHECK: la %r4, 160(%r15) -; CHECK: lghi %r2, 16 -; CHECK: lhi %r5, 5 +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r1, 8(%r3) +; CHECK-NEXT: lg %r0, 0(%r3) +; CHECK-NEXT: stpq %r0, 0(%r2) +; CHECK-NEXT: bcr 1{{[45]}}, %r0 +; CHECK-NEXT: br %r14 + %val = load fp128, ptr %src, align 8 + store atomic fp128 %val, ptr %dst seq_cst, align 16 + ret void +} + +define void @f2(ptr %dst, ptr %src) { +; CHECK-LABEL: f2: ; CHECK: brasl %r14, __atomic_store@PLT -; CHECK: br %r14 %val = load fp128, ptr %src, align 8 store atomic fp128 %val, ptr %dst seq_cst, align 8 ret void diff --git a/llvm/test/CodeGen/X86/code-model-elf-sections.ll b/llvm/test/CodeGen/X86/code-model-elf-sections.ll index cb19f0d34f59..749d5b6bf904 100644 --- a/llvm/test/CodeGen/X86/code-model-elf-sections.ll +++ b/llvm/test/CodeGen/X86/code-model-elf-sections.ll @@ -21,16 +21,16 @@ ; SMALL: .data {{.*}} WA {{.*}} ; SMALL: .data.x {{.*}} WA {{.*}} ; SMALL: .data0 {{.*}} WA {{.*}} -; SMALL: .ldata {{.*}} WA {{.*}} -; SMALL: .ldata.x {{.*}} WA {{.*}} +; SMALL: .ldata {{.*}} WAl {{.*}} +; SMALL: .ldata.x {{.*}} WAl {{.*}} ; SMALL: .ldata0 {{.*}} WA {{.*}} ; SMALL: force_small {{.*}} WA {{.*}} ; SMALL: force_large {{.*}} WAl {{.*}} ; SMALL: foo {{.*}} WA {{.*}} ; SMALL: .bss {{.*}} WA {{.*}} -; SMALL: .lbss {{.*}} WA {{.*}} +; SMALL: .lbss {{.*}} WAl {{.*}} ; SMALL: .rodata {{.*}} A {{.*}} -; SMALL: .lrodata {{.*}} A {{.*}} +; SMALL: .lrodata {{.*}} Al {{.*}} ; SMALL: .data.rel.ro {{.*}} WA {{.*}} ; SMALL: .tbss {{.*}} WAT {{.*}} ; SMALL: .tdata {{.*}} WAT {{.*}} @@ -38,16 +38,16 @@ ; SMALL-DS: .data {{.*}} WA {{.*}} ; SMALL-DS: .data.x {{.*}} WA {{.*}} ; SMALL-DS: .data0 {{.*}} WA {{.*}} -; SMALL-DS: .ldata {{.*}} WA {{.*}} -; SMALL-DS: .ldata.x {{.*}} WA {{.*}} +; SMALL-DS: .ldata {{.*}} WAl {{.*}} +; SMALL-DS: .ldata.x {{.*}} WAl {{.*}} ; SMALL-DS: .ldata0 {{.*}} WA {{.*}} ; SMALL-DS: .data.data {{.*}} WA {{.*}} ; SMALL-DS: force_small {{.*}} WA {{.*}} ; SMALL-DS: force_large {{.*}} WAl {{.*}} ; SMALL-DS: foo {{.*}} WA {{.*}} -; SMALL-DS: .lbss {{.*}} WA {{.*}} +; SMALL-DS: .lbss {{.*}} WAl {{.*}} ; SMALL-DS: .bss.bss {{.*}} WA {{.*}} -; SMALL-DS: .lrodata {{.*}} A {{.*}} +; SMALL-DS: .lrodata {{.*}} Al {{.*}} ; SMALL-DS: .rodata.rodata {{.*}} A {{.*}} ; SMALL-DS: .data.rel.ro.relro {{.*}} WA {{.*}} ; SMALL-DS: .tbss.tbss {{.*}} WAT {{.*}} diff --git a/llvm/test/CodeGen/X86/code-model-elf.ll b/llvm/test/CodeGen/X86/code-model-elf.ll index 6112f2a57b82..afcffb3a7ade 100644 --- a/llvm/test/CodeGen/X86/code-model-elf.ll +++ b/llvm/test/CodeGen/X86/code-model-elf.ll @@ -11,6 +11,16 @@ ; RUN: llc -verify-machineinstrs < %s -relocation-model=pic -code-model=large | FileCheck %s --check-prefix=CHECK --check-prefix=LARGE-PIC ; RUN: llc -verify-machineinstrs < %s -relocation-model=pic -code-model=large -large-data-threshold=1000 | FileCheck %s --check-prefix=CHECK --check-prefix=LARGE-SMALL-DATA-PIC +; Check that the relocations we emit are valid. +; RUN: llc -verify-machineinstrs < %s -relocation-model=static -code-model=small -filetype=obj -o /dev/null +; RUN: llc -verify-machineinstrs < %s -relocation-model=static -code-model=medium -filetype=obj -o /dev/null +; RUN: llc -verify-machineinstrs < %s -relocation-model=static -code-model=large -filetype=obj -o /dev/null +; RUN: llc -verify-machineinstrs < %s -relocation-model=pic -code-model=small -filetype=obj -o /dev/null +; RUN: llc -verify-machineinstrs < %s -relocation-model=pic -code-model=medium -large-data-threshold=1000 -filetype=obj -o /dev/null +; RUN: llc -verify-machineinstrs < %s -relocation-model=pic -code-model=medium -filetype=obj -o /dev/null +; RUN: llc -verify-machineinstrs < %s -relocation-model=pic -code-model=large -filetype=obj -o /dev/null +; RUN: llc -verify-machineinstrs < %s -relocation-model=pic -code-model=large -large-data-threshold=1000 -filetype=obj -o /dev/null + ; Generated from this C source: ; ; static int static_data[10]; @@ -376,7 +386,6 @@ define dso_local ptr @lea_forced_small_data() #0 { ret ptr @forced_small_data } -; TODO: make small and medium instruction sequence the same define dso_local i32 @load_forced_small_data() #0 { ; SMALL-STATIC-LABEL: load_forced_small_data: ; SMALL-STATIC: # %bb.0: @@ -385,14 +394,13 @@ define dso_local i32 @load_forced_small_data() #0 { ; ; MEDIUM-STATIC-LABEL: load_forced_small_data: ; MEDIUM-STATIC: # %bb.0: -; MEDIUM-STATIC-NEXT: movl $forced_small_data, %eax -; MEDIUM-STATIC-NEXT: movl 8(%rax), %eax +; MEDIUM-STATIC-NEXT: movl forced_small_data+8(%rip), %eax ; MEDIUM-STATIC-NEXT: retq ; ; LARGE-STATIC-LABEL: load_forced_small_data: ; LARGE-STATIC: # %bb.0: -; LARGE-STATIC-NEXT: movl $forced_small_data, %eax -; LARGE-STATIC-NEXT: movl 8(%rax), %eax +; LARGE-STATIC-NEXT: movl $forced_small_data+8, %eax +; LARGE-STATIC-NEXT: movl (%rax), %eax ; LARGE-STATIC-NEXT: retq ; ; SMALL-PIC-LABEL: load_forced_small_data: @@ -402,14 +410,12 @@ define dso_local i32 @load_forced_small_data() #0 { ; ; MEDIUM-SMALL-DATA-PIC-LABEL: load_forced_small_data: ; MEDIUM-SMALL-DATA-PIC: # %bb.0: -; MEDIUM-SMALL-DATA-PIC-NEXT: leaq forced_small_data(%rip), %rax -; MEDIUM-SMALL-DATA-PIC-NEXT: movl 8(%rax), %eax +; MEDIUM-SMALL-DATA-PIC-NEXT: movl forced_small_data+8(%rip), %eax ; MEDIUM-SMALL-DATA-PIC-NEXT: retq ; ; MEDIUM-PIC-LABEL: load_forced_small_data: ; MEDIUM-PIC: # %bb.0: -; MEDIUM-PIC-NEXT: leaq forced_small_data(%rip), %rax -; MEDIUM-PIC-NEXT: movl 8(%rax), %eax +; MEDIUM-PIC-NEXT: movl forced_small_data+8(%rip), %eax ; MEDIUM-PIC-NEXT: retq ; ; LARGE-PIC-LABEL: load_forced_small_data: @@ -435,7 +441,6 @@ define dso_local i32 @load_forced_small_data() #0 { ret i32 %rv } -; TODO: fix small code model instruction sequences to use 64-bit constants define dso_local ptr @lea_forced_large_data() #0 { ; SMALL-STATIC-LABEL: lea_forced_large_data: ; SMALL-STATIC: # %bb.0: @@ -454,8 +459,9 @@ define dso_local ptr @lea_forced_large_data() #0 { ; ; SMALL-PIC-LABEL: lea_forced_large_data: ; SMALL-PIC: # %bb.0: -; SMALL-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax -; SMALL-PIC-NEXT: leaq forced_large_data@GOTOFF(%rax), %rax +; SMALL-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rcx +; SMALL-PIC-NEXT: movabsq $forced_large_data@GOTOFF, %rax +; SMALL-PIC-NEXT: addq %rcx, %rax ; SMALL-PIC-NEXT: retq ; ; MEDIUM-SMALL-DATA-PIC-LABEL: lea_forced_large_data: @@ -497,25 +503,27 @@ define dso_local ptr @lea_forced_large_data() #0 { define dso_local i32 @load_forced_large_data() #0 { ; SMALL-STATIC-LABEL: load_forced_large_data: ; SMALL-STATIC: # %bb.0: -; SMALL-STATIC-NEXT: movl forced_large_data+8(%rip), %eax +; SMALL-STATIC-NEXT: movabsq $forced_large_data+8, %rax +; SMALL-STATIC-NEXT: movl (%rax), %eax ; SMALL-STATIC-NEXT: retq ; ; MEDIUM-STATIC-LABEL: load_forced_large_data: ; MEDIUM-STATIC: # %bb.0: -; MEDIUM-STATIC-NEXT: movabsq $forced_large_data, %rax -; MEDIUM-STATIC-NEXT: movl 8(%rax), %eax +; MEDIUM-STATIC-NEXT: movabsq $forced_large_data+8, %rax +; MEDIUM-STATIC-NEXT: movl (%rax), %eax ; MEDIUM-STATIC-NEXT: retq ; ; LARGE-STATIC-LABEL: load_forced_large_data: ; LARGE-STATIC: # %bb.0: -; LARGE-STATIC-NEXT: movabsq $forced_large_data, %rax -; LARGE-STATIC-NEXT: movl 8(%rax), %eax +; LARGE-STATIC-NEXT: movabsq $forced_large_data+8, %rax +; LARGE-STATIC-NEXT: movl (%rax), %eax ; LARGE-STATIC-NEXT: retq ; ; SMALL-PIC-LABEL: load_forced_large_data: ; SMALL-PIC: # %bb.0: ; SMALL-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax -; SMALL-PIC-NEXT: movl forced_large_data@GOTOFF+8(%rax), %eax +; SMALL-PIC-NEXT: movabsq $forced_large_data@GOTOFF, %rcx +; SMALL-PIC-NEXT: movl 8(%rax,%rcx), %eax ; SMALL-PIC-NEXT: retq ; ; MEDIUM-SMALL-DATA-PIC-LABEL: load_forced_large_data: @@ -563,14 +571,14 @@ define dso_local i32 @load_global_data() #0 { ; ; MEDIUM-STATIC-LABEL: load_global_data: ; MEDIUM-STATIC: # %bb.0: -; MEDIUM-STATIC-NEXT: movabsq $global_data, %rax -; MEDIUM-STATIC-NEXT: movl 8(%rax), %eax +; MEDIUM-STATIC-NEXT: movabsq $global_data+8, %rax +; MEDIUM-STATIC-NEXT: movl (%rax), %eax ; MEDIUM-STATIC-NEXT: retq ; ; LARGE-STATIC-LABEL: load_global_data: ; LARGE-STATIC: # %bb.0: -; LARGE-STATIC-NEXT: movabsq $global_data, %rax -; LARGE-STATIC-NEXT: movl 8(%rax), %eax +; LARGE-STATIC-NEXT: movabsq $global_data+8, %rax +; LARGE-STATIC-NEXT: movl (%rax), %eax ; LARGE-STATIC-NEXT: retq ; ; SMALL-PIC-LABEL: load_global_data: @@ -580,8 +588,7 @@ define dso_local i32 @load_global_data() #0 { ; ; MEDIUM-SMALL-DATA-PIC-LABEL: load_global_data: ; MEDIUM-SMALL-DATA-PIC: # %bb.0: -; MEDIUM-SMALL-DATA-PIC-NEXT: leaq global_data(%rip), %rax -; MEDIUM-SMALL-DATA-PIC-NEXT: movl 8(%rax), %eax +; MEDIUM-SMALL-DATA-PIC-NEXT: movl global_data+8(%rip), %eax ; MEDIUM-SMALL-DATA-PIC-NEXT: retq ; ; MEDIUM-PIC-LABEL: load_global_data: @@ -684,14 +691,14 @@ define dso_local i32 @load_unknown_size_data() #0 { ; ; MEDIUM-STATIC-LABEL: load_unknown_size_data: ; MEDIUM-STATIC: # %bb.0: -; MEDIUM-STATIC-NEXT: movabsq $unknown_size_data, %rax -; MEDIUM-STATIC-NEXT: movl 8(%rax), %eax +; MEDIUM-STATIC-NEXT: movabsq $unknown_size_data+8, %rax +; MEDIUM-STATIC-NEXT: movl (%rax), %eax ; MEDIUM-STATIC-NEXT: retq ; ; LARGE-STATIC-LABEL: load_unknown_size_data: ; LARGE-STATIC: # %bb.0: -; LARGE-STATIC-NEXT: movabsq $unknown_size_data, %rax -; LARGE-STATIC-NEXT: movl 8(%rax), %eax +; LARGE-STATIC-NEXT: movabsq $unknown_size_data+8, %rax +; LARGE-STATIC-NEXT: movl (%rax), %eax ; LARGE-STATIC-NEXT: retq ; ; SMALL-PIC-LABEL: load_unknown_size_data: @@ -1127,8 +1134,7 @@ define dso_local float @load_constant_pool(float %x) #0 { ; ; MEDIUM-STATIC-LABEL: load_constant_pool: ; MEDIUM-STATIC: # %bb.0: -; MEDIUM-STATIC-NEXT: movl ${{\.?LCPI[0-9]+_[0-9]+}}, %eax -; MEDIUM-STATIC-NEXT: addss (%rax), %xmm0 +; MEDIUM-STATIC-NEXT: addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; MEDIUM-STATIC-NEXT: retq ; ; LARGE-STATIC-LABEL: load_constant_pool: diff --git a/llvm/test/CodeGen/X86/fast-isel-large-object.ll b/llvm/test/CodeGen/X86/fast-isel-large-object.ll index 6ca2c4240723..9acdfdeaf7cc 100644 --- a/llvm/test/CodeGen/X86/fast-isel-large-object.ll +++ b/llvm/test/CodeGen/X86/fast-isel-large-object.ll @@ -6,8 +6,9 @@ define ptr @f() { ; CHECK-LABEL: f: ; CHECK: # %bb.0: -; CHECK-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax -; CHECK-NEXT: leaq g@GOTOFF(%rax), %rax +; CHECK-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rcx +; CHECK-NEXT: movabsq $g@GOTOFF, %rax +; CHECK-NEXT: addq %rcx, %rax ; CHECK-NEXT: retq ret ptr @g } diff --git a/llvm/test/CodeGen/X86/fold-add.ll b/llvm/test/CodeGen/X86/fold-add.ll index 597e51d877eb..8c28d66597fb 100644 --- a/llvm/test/CodeGen/X86/fold-add.ll +++ b/llvm/test/CodeGen/X86/fold-add.ll @@ -45,8 +45,7 @@ define dso_local i64 @one() #0 { ; ; MSTATIC-LABEL: one: ; MSTATIC: # %bb.0: # %entry -; MSTATIC-NEXT: movabsq $foo, %rax -; MSTATIC-NEXT: incq %rax +; MSTATIC-NEXT: movabsq $foo+1, %rax ; MSTATIC-NEXT: retq ; ; MPIC-LABEL: one: diff --git a/llvm/test/CodeGen/X86/insert.ll b/llvm/test/CodeGen/X86/insert.ll new file mode 100644 index 000000000000..381de2ecaa16 --- /dev/null +++ b/llvm/test/CodeGen/X86/insert.ll @@ -0,0 +1,93 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=i386-unknown-unknown | FileCheck %s --check-prefixes=X86 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=X64 + +define i64 @sub8(i64 noundef %res, ptr %byte) { +; X86-LABEL: sub8: +; X86: # %bb.0: # %entry +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movb (%ecx), %al +; X86-NEXT: retl +; +; X64-LABEL: sub8: +; X64: # %bb.0: # %entry +; X64-NEXT: movq %rdi, %rax +; X64-NEXT: movb (%rsi), %al +; X64-NEXT: retq +entry: + %and = and i64 %res, -256 + %d = load i8, ptr %byte, align 1 + %conv2 = zext i8 %d to i64 + %or = or i64 %and, %conv2 + ret i64 %or +} + +define i64 @sub16(i64 noundef %res, ptr %byte) { +; X86-LABEL: sub16: +; X86: # %bb.0: # %entry +; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: shll $16, %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzwl (%eax), %eax +; X86-NEXT: orl %ecx, %eax +; X86-NEXT: retl +; +; X64-LABEL: sub16: +; X64: # %bb.0: # %entry +; X64-NEXT: movq %rdi, %rax +; X64-NEXT: movw (%rsi), %ax +; X64-NEXT: retq +entry: + %and = and i64 %res, -65536 + %d = load i16, ptr %byte, align 1 + %conv2 = zext i16 %d to i64 + %or = or i64 %and, %conv2 + ret i64 %or +} + +define i32 @sub8_32(i32 noundef %res, ptr %byte) { +; X86-LABEL: sub8_32: +; X86: # %bb.0: # %entry +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movb (%ecx), %al +; X86-NEXT: retl +; +; X64-LABEL: sub8_32: +; X64: # %bb.0: # %entry +; X64-NEXT: movl %edi, %eax +; X64-NEXT: movb (%rsi), %al +; X64-NEXT: retq +entry: + %and = and i32 %res, -256 + %d = load i8, ptr %byte, align 1 + %conv2 = zext i8 %d to i32 + %or = or i32 %and, %conv2 + ret i32 %or +} + +define i32 @sub16_32(i32 noundef %res, ptr %byte) { +; X86-LABEL: sub16_32: +; X86: # %bb.0: # %entry +; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: shll $16, %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzwl (%eax), %eax +; X86-NEXT: orl %ecx, %eax +; X86-NEXT: retl +; +; X64-LABEL: sub16_32: +; X64: # %bb.0: # %entry +; X64-NEXT: movl %edi, %eax +; X64-NEXT: movw (%rsi), %ax +; X64-NEXT: retq +entry: + %and = and i32 %res, -65536 + %d = load i16, ptr %byte, align 1 + %conv2 = zext i16 %d to i32 + %or = or i32 %and, %conv2 + ret i32 %or +} diff --git a/llvm/test/MC/AArch64/armv9.5a-pauthlr-diagnostics.s b/llvm/test/MC/AArch64/armv9.5a-pauthlr-diagnostics.s new file mode 100644 index 000000000000..d06183be9da3 --- /dev/null +++ b/llvm/test/MC/AArch64/armv9.5a-pauthlr-diagnostics.s @@ -0,0 +1,57 @@ +// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+pauth-lr 2>&1 < %s | FileCheck %s + + autiasppc #2 +// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: expected label or encodable integer pc offset +// CHECK-NEXT: autiasppc #2 +// CHECK-NOT: [[@LINE-3]]:{{[0-9]+}}: + + autiasppc #1<<17 +// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: expected label or encodable integer pc offset +// CHECK-NEXT: autiasppc #1<<17 +// CHECK-NOT: [[@LINE-3]]:{{[0-9]+}}: + + autiasppc #-2 +// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: expected label or encodable integer pc offset +// CHECK-NEXT: autiasppc #-2 +// CHECK-NOT: [[@LINE-3]]:{{[0-9]+}}: + + autiasppc w0 +// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: expected label or encodable integer pc offset +// CHECK-NEXT: autiasppc w0 +// CHECK-NOT: [[@LINE-3]]:{{[0-9]+}}: + + autiasppc sp +// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: expected label or encodable integer pc offset +// CHECK-NEXT: autiasppc sp +// CHECK-NOT: [[@LINE-3]]:{{[0-9]+}}: + + retabsppc #2 +// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: expected label or encodable integer pc offset +// CHECK-NEXT: retabsppc #2 +// CHECK-NOT: [[@LINE-3]]:{{[0-9]+}}: + + retabsppc #(1<<17) +// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: expected label or encodable integer pc offset +// CHECK-NEXT: retabsppc #(1<<17) +// CHECK-NOT: [[@LINE-3]]:{{[0-9]+}}: + + retabsppc #-2 +// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: expected label or encodable integer pc offset +// CHECK-NEXT: retabsppc #-2 +// CHECK-NOT: [[@LINE-3]]:{{[0-9]+}}: + + retaasppc w0 +// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: expected label or encodable integer pc offset +// CHECK-NEXT: retaasppc w0 +// CHECK-NOT: [[@LINE-3]]:{{[0-9]+}}: + + retaasppc sp +// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: expected label or encodable integer pc offset +// CHECK-NEXT: retaasppc sp +// CHECK-NOT: [[@LINE-3]]:{{[0-9]+}}: + + retaasppc xzr +// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: expected label or encodable integer pc offset +// CHECK-NEXT: retaasppc xzr +// CHECK-NOT: [[@LINE-3]]:{{[0-9]+}}: + diff --git a/llvm/test/MC/AArch64/armv9.5a-pauthlr-reloc.s b/llvm/test/MC/AArch64/armv9.5a-pauthlr-reloc.s new file mode 100644 index 000000000000..c10142a19976 --- /dev/null +++ b/llvm/test/MC/AArch64/armv9.5a-pauthlr-reloc.s @@ -0,0 +1,12 @@ +// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+pauth-lr -filetype=obj -o /dev/null 2>&1 < %s | FileCheck %s + + autiasppc undef_label +// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: relocation of PAC/AUT instructions is not supported +// CHECK-NEXT: autiasppc undef_label +// CHECK-NOT: [[@LINE-3]]:{{[0-9]+}}: + + autibsppc undef_label +// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: relocation of PAC/AUT instructions is not supported +// CHECK-NEXT: autibsppc undef_label +// CHECK-NOT: [[@LINE-3]]:{{[0-9]+}}: + diff --git a/llvm/test/MC/AArch64/armv9.5a-pauthlr.s b/llvm/test/MC/AArch64/armv9.5a-pauthlr.s new file mode 100644 index 000000000000..24e9c4498468 --- /dev/null +++ b/llvm/test/MC/AArch64/armv9.5a-pauthlr.s @@ -0,0 +1,151 @@ +// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+pauth-lr < %s \ +// RUN: | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST +// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-ERROR +// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+pauth-lr < %s \ +// RUN: | llvm-objdump -d --mattr=+pauth-lr - | FileCheck %s --check-prefix=CHECK-DISASS +// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+pauth-lr < %s \ +// RUN: | llvm-objdump -d --mattr=-pauth-lr - | FileCheck %s --check-prefix=CHECK-UNKNOWN + +// Label at address 4, so we can test that the address shows up in the +// disassembly. + nop +label1: + + paciasppc +// CHECK-INST: paciasppc +// CHECK-DISASS: paciasppc +// CHECK-ENCODING: [0xfe,0xa3,0xc1,0xda] +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: dac1a3fe <unknown> + + pacibsppc +// CHECK-INST: pacibsppc +// CHECK-DISASS: pacibsppc +// CHECK-ENCODING: [0xfe,0xa7,0xc1,0xda] +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: dac1a7fe <unknown> + + pacnbiasppc +// CHECK-INST: pacnbiasppc +// CHECK-DISASS: pacnbiasppc +// CHECK-ENCODING: [0xfe,0x83,0xc1,0xda] +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: dac183fe <unknown> + + pacnbibsppc +// CHECK-INST: pacnbibsppc +// CHECK-DISASS: pacnbibsppc +// CHECK-ENCODING: [0xfe,0x87,0xc1,0xda] +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: dac187fe <unknown> + + autiasppc label1 +// CHECK-INST: autiasppc label1 +// CHECK-DISASS: autiasppc 0x4 <label1> +// CHECK-ENCODING: [0bAAA11111,A,0b100AAAAA,0xf3] +// CHECK-ENCODING: fixup A - offset: 0, value: label1, kind: fixup_aarch64_pcrel_branch16 +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: f380009f <unknown> + + autibsppc label1 +// CHECK-INST: autibsppc label1 +// CHECK-DISASS: autibsppc 0x4 <label1> +// CHECK-ENCODING: [0bAAA11111,A,0b101AAAAA,0xf3] +// CHECK-ENCODING: fixup A - offset: 0, value: label1, kind: fixup_aarch64_pcrel_branch16 +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: f3a000bf <unknown> + + autibsppc #0 +// CHECK-INST: autibsppc #0 +// CHECK-DISASS: autibsppc 0x1c <label1+0x18> +// CHECK-ENCODING: [0x1f,0x00,0xa0,0xf3] +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: f3a0001f <unknown> + + autibsppc #-(1<<18)+4 +// CHECK-INST: autibsppc #-262140 +// CHECK-DISASS: autibsppc 0xfffffffffffc0024 <label1+0xfffffffffffc0020> +// CHECK-ENCODING: [0xff,0xff,0xbf,0xf3] +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: f3bfffff <unknown> + + autiasppc x0 +// CHECK-INST: autiasppc x0 +// CHECK-DISASS: autiasppc x0 +// CHECK-ENCODING: [0x1e,0x90,0xc1,0xda] +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: dac1901e <unknown> + + autibsppc x1 +// CHECK-INST: autibsppc x1 +// CHECK-DISASS: autibsppc x1 +// CHECK-ENCODING: [0x3e,0x94,0xc1,0xda] +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: dac1943e <unknown> + + autiasppc xzr +// CHECK-INST: autiasppc xzr +// CHECK-DISASS: autiasppc xzr +// CHECK-ENCODING: [0xfe,0x93,0xc1,0xda] +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: dac193fe <unknown> + + autibsppc xzr +// CHECK-INST: autibsppc xzr +// CHECK-DISASS: autibsppc xzr +// CHECK-ENCODING: [0xfe,0x97,0xc1,0xda] +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: dac197fe <unknown> + + + retaasppc label1 +// CHECK-INST: retaasppc label1 +// CHECK-DISASS: retaasppc 0x4 <label1> +// CHECK-ENCODING: [0bAAA11111,A,0b000AAAAA,0x55] +// CHECK-ENCODING: // fixup A - offset: 0, value: label1, kind: fixup_aarch64_pcrel_branch16 +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: 5500019f <unknown> + + retabsppc label1 +// CHECK-INST: retabsppc label1 +// CHECK-DISASS: retabsppc 0x4 <label1> +// CHECK-ENCODING: [0bAAA11111,A,0b001AAAAA,0x55] +// CHECK-ENCODING: // fixup A - offset: 0, value: label1, kind: fixup_aarch64_pcrel_branch16 +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: 552001bf <unknown> + + retaasppc #0 +// CHECK-INST: retaasppc #0 +// CHECK-DISASS: retaasppc 0x3c <label1+0x38> +// CHECK-ENCODING: [0x1f,0x00,0x00,0x55] +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: 5500001f <unknown> + + retaasppc #-(1<<18)+4 +// CHECK-INST: retaasppc #-262140 +// CHECK-DISASS: retaasppc 0xfffffffffffc0044 <label1+0xfffffffffffc0040> +// CHECK-ENCODING: [0xff,0xff,0x1f,0x55] +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: 551fffff <unknown> + + retaasppc x2 +// CHECK-INST: retaasppc x2 +// CHECK-DISASS: retaasppc x2 +// CHECK-ENCODING: [0xe2,0x0b,0x5f,0xd6] +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: d65f0be2 <unknown> + + retabsppc x3 +// CHECK-INST: retabsppc x3 +// CHECK-DISASS: retabsppc x3 +// CHECK-ENCODING: [0xe3,0x0f,0x5f,0xd6] +// CHECK-ERROR: instruction requires: pauth-lr +// CHECK-UNKNOWN: d65f0fe3 <unknown> + + pacm +// CHECK-INST: pacm +// CHECK-DISASS: pacm +// CHECK-ENCODING: [0xff,0x24,0x03,0xd5] +// CHECK-ERROR-NOT: instruction requires: +// CHECK-UNKNOWN: d50324ff hint #39 diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_features.s b/llvm/test/MC/AMDGPU/gfx12_asm_features.s new file mode 100644 index 000000000000..7e58bdb3b444 --- /dev/null +++ b/llvm/test/MC/AMDGPU/gfx12_asm_features.s @@ -0,0 +1,29 @@ +// RUN: llvm-mc -arch=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s + +// +// Elements of CPol operand can be given in any order +// + +image_load v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_LOAD_HT scope:SCOPE_SE +// GFX12: encoding: [0x00,0x00,0x40,0xd0,0x00,0x00,0x24,0x00,0x00,0x00,0x00,0x00] + +image_load v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D scope:SCOPE_SE th:TH_LOAD_HT +// GFX12: encoding: [0x00,0x00,0x40,0xd0,0x00,0x00,0x24,0x00,0x00,0x00,0x00,0x00] + +image_sample v[29:30], [v31, v32, v33], s[32:39], s[68:71] dmask:0x3 dim:SQ_RSRC_IMG_3D th:TH_LOAD_NT scope:SCOPE_SYS +// GFX12: encoding: [0x02,0xc0,0xc6,0xe4,0x1d,0x40,0x1c,0x22,0x1f,0x20,0x21,0x00] + +image_sample v[29:30], [v31, v32, v33], s[32:39], s[68:71] dmask:0x3 dim:SQ_RSRC_IMG_3D scope:SCOPE_SYS th:TH_LOAD_NT +// GFX12: encoding: [0x02,0xc0,0xc6,0xe4,0x1d,0x40,0x1c,0x22,0x1f,0x20,0x21,0x00] + +buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV +// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f] + +buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 scope:SCOPE_DEV th:TH_LOAD_NT_HT +// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f] + +tbuffer_load_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_UINT] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS +// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0xe0,0xbc,0x02,0x00,0xff,0xff,0x7f] + +tbuffer_load_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_UINT] offset:8388607 scope:SCOPE_SYS th:TH_LOAD_BYPASS +// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0xe0,0xbc,0x02,0x00,0xff,0xff,0x7f] diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.5a-pauthlr.txt b/llvm/test/MC/Disassembler/AArch64/armv9.5a-pauthlr.txt new file mode 100644 index 000000000000..caf1fde2c2b7 --- /dev/null +++ b/llvm/test/MC/Disassembler/AArch64/armv9.5a-pauthlr.txt @@ -0,0 +1,78 @@ +# RUN: llvm-mc -triple aarch64 -disassemble -mattr=+pauth-lr < %s | FileCheck %s +# RUN: not llvm-mc -triple aarch64 -disassemble < %s 2>&1 | FileCheck %s --check-prefix=NO-PAUTHLR + +[0xfe,0xa3,0xc1,0xda] +# CHECK: paciasppc +# NO-PAUTHLR: invalid instruction encoding + +[0xfe,0xa7,0xc1,0xda] +# CHECK: pacibsppc +# NO-PAUTHLR: invalid instruction encoding + +[0xfe,0x83,0xc1,0xda] +# CHECK: pacnbiasppc +# NO-PAUTHLR: invalid instruction encoding + +[0xfe,0x87,0xc1,0xda] +# CHECK: pacnbibsppc +# NO-PAUTHLR: invalid instruction encoding + +[0x9f,0x00,0x80,0xf3] +# CHECK: autiasppc #-16 +# NO-PAUTHLR: invalid instruction encoding + +[0xbf,0x00,0xa0,0xf3] +# CHECK: autibsppc #-20 +# NO-PAUTHLR: invalid instruction encoding + +[0x1f,0x00,0xa0,0xf3] +# CHECK: autibsppc #0 +# NO-PAUTHLR: invalid instruction encoding + +[0xff,0xff,0xbf,0xf3] +# CHECK: autibsppc #-262140 +# NO-PAUTHLR: invalid instruction encoding + +[0x1e,0x90,0xc1,0xda] +# CHECK: autiasppc x0 +# NO-PAUTHLR: invalid instruction encoding + +[0x3e,0x94,0xc1,0xda] +# CHECK: autibsppc x1 +# NO-PAUTHLR: invalid instruction encoding + +[0xfe,0x93,0xc1,0xda] +# CHECK: autiasppc xzr +# NO-PAUTHLR: invalid instruction encoding + +[0xfe,0x97,0xc1,0xda] +# CHECK: autibsppc xzr +# NO-PAUTHLR: invalid instruction encoding + +[0xbf,0x01,0x00,0x55] +# CHECK: retaasppc #-52 +# NO-PAUTHLR: invalid instruction encoding + +[0xdf,0x01,0x20,0x55] +# CHECK: retabsppc #-56 +# NO-PAUTHLR: invalid instruction encoding + +[0x1f,0x00,0x00,0x55] +# CHECK: retaasppc #0 +# NO-PAUTHLR: invalid instruction encoding + +[0xff,0xff,0x1f,0x55] +# CHECK: retaasppc #-262140 +# NO-PAUTHLR: invalid instruction encoding + +[0xe2,0x0b,0x5f,0xd6] +# CHECK: retaasppc x2 +# NO-PAUTHLR: invalid instruction encoding + +[0xe3,0x0f,0x5f,0xd6] +# CHECK: retabsppc x3 +# NO-PAUTHLR: invalid instruction encoding + +[0xff,0x24,0x03,0xd5] +# CHECK: pacm +# NO-PAUTHLR: hint #39 diff --git a/llvm/test/MC/Disassembler/AMDGPU/decode-err.txt b/llvm/test/MC/Disassembler/AMDGPU/decode-err.txt index 24716b1226e4..e1bb7ad51171 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/decode-err.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/decode-err.txt @@ -34,6 +34,14 @@ # W64: v_wmma_f32_16x16x16_f16 v[16:19], v[0:7], v[8:15], s[0:3]/*Invalid register, operand has 'VReg_128' register class*/ ; encoding: [0x10,0x40,0x40,0xcc,0x00,0x11,0x02,0x18] 0x10,0x40,0x40,0xcc,0x00,0x11,0x02,0x18 # src2 sgpr0 +# this is ds_add_f32 with gds bit which is not valid on gfx12+ +# GFX12: [[@LINE+1]]:1: warning: invalid instruction encoding +0x00,0x00,0x56,0xd8,0x00,0x01,0x00,0x00 + # this is image_msaa_load where samp field for gfx12 VSAMPLE is not all zeros # GFX12: [[@LINE+1]]:1: warning: invalid instruction encoding 0x06,0x00,0x46,0xe4,0x01,0x10,0x80,0x00,0x05,0x06,0x07,0x00 + +# This is ds_read_b32 with gds bit which is not valid on gfx90a. +# GFX90A: [[@LINE+1]]:1: warning: invalid instruction encoding +0x00,0x00,0x6d,0xd8,0x01,0x00,0x00,0x00 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx90a_features.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx90a_features.txt index 8746ee79c8f5..b348e8b5ef01 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx90a_features.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx90a_features.txt @@ -773,8 +773,3 @@ # GFX90A: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] glc ; encoding: [0x00,0x00,0x41,0xdd,0x00,0x02,0x00,0x00] 0x00,0x00,0x41,0xdd,0x00,0x02,0x00,0x00 - -# Disassembler still decodes the gds modifier even though the assembler does -# not accept it. -# GFX90A: ds_read_b32 v0, v1 gds ; encoding: [0x00,0x00,0x6d,0xd8,0x01,0x00,0x00,0x00] -0x00,0x00,0x6d,0xd8,0x01,0x00,0x00,0x00 diff --git a/llvm/test/Transforms/ConstraintElimination/and-implied-by-operands.ll b/llvm/test/Transforms/ConstraintElimination/and-implied-by-operands.ll index 2de7592d5ccc..5c49ca0e96f3 100644 --- a/llvm/test/Transforms/ConstraintElimination/and-implied-by-operands.ll +++ b/llvm/test/Transforms/ConstraintElimination/and-implied-by-operands.ll @@ -478,3 +478,24 @@ entry: %and = select i1 %c.1, i1 %c.2, i1 false ret i1 %and } + +define i1 @and_select_second_implies_first_guaranteed_not_poison(ptr noundef %A, ptr noundef %B) { +; CHECK-LABEL: @and_select_second_implies_first_guaranteed_not_poison( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[C_1:%.*]] = icmp ne ptr [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds ptr, ptr [[B]], i64 -1 +; CHECK-NEXT: [[C_2:%.*]] = icmp ugt ptr [[GEP]], [[A]] +; CHECK-NEXT: call void @no_noundef(i1 [[C_2]]) +; CHECK-NEXT: [[AND:%.*]] = select i1 [[C_1]], i1 [[C_2]], i1 false +; CHECK-NEXT: ret i1 [[AND]] +; +entry: + %c.1 = icmp ne ptr %A, %B + %gep = getelementptr inbounds ptr, ptr %B, i64 -1 + %c.2 = icmp ugt ptr %gep, %A + call void @no_noundef(i1 %c.2) + %and = select i1 %c.1, i1 %c.2, i1 false + ret i1 %and +} + +declare void @no_noundef(i1 noundef) diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll b/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll index 7ec1028d65e0..0b95139f3dcb 100644 --- a/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll +++ b/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll @@ -567,3 +567,32 @@ define i16 @cond_value_may_not_well_defined(i16 %x) { %sel = select i1 %cmp, i16 %and, i16 24 ret i16 %sel } + +define i16 @and_elide_poison_flags(i16 noundef %a) { +; CHECK-LABEL: @and_elide_poison_flags( +; CHECK-NEXT: [[X:%.*]] = add nuw i16 [[A:%.*]], 1 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i16 [[X]], 8 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], i16 [[X]], i16 24 +; CHECK-NEXT: ret i16 [[SEL]] +; + %x = add nuw i16 %a, 1 + %and = and i16 %x, 7 + %cmp = icmp ult i16 %x, 8 + %sel = select i1 %cmp, i16 %and, i16 24 + ret i16 %sel +} + +define i16 @and_elide_poison_flags_missing_noundef(i16 %a) { +; CHECK-LABEL: @and_elide_poison_flags_missing_noundef( +; CHECK-NEXT: [[X:%.*]] = add nuw i16 [[A:%.*]], 1 +; CHECK-NEXT: [[AND:%.*]] = and i16 [[X]], 7 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i16 [[X]], 8 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], i16 [[AND]], i16 24 +; CHECK-NEXT: ret i16 [[SEL]] +; + %x = add nuw i16 %a, 1 + %and = and i16 %x, 7 + %cmp = icmp ult i16 %x, 8 + %sel = select i1 %cmp, i16 %and, i16 24 + ret i16 %sel +} diff --git a/llvm/test/Transforms/InstCombine/array.ll b/llvm/test/Transforms/InstCombine/array.ll index 8bab3104fd8c..396a7aa340f6 100644 --- a/llvm/test/Transforms/InstCombine/array.ll +++ b/llvm/test/Transforms/InstCombine/array.ll @@ -72,3 +72,39 @@ entry: store i32 %b, ptr %gep ret void } + +define void @test_zext_nneg(ptr %ptr, i32 %a, i32 %b) { +; CHECK-LABEL: define void @test_zext_nneg( +; CHECK-SAME: ptr [[PTR:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[A]] to i64 +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[TMP0]] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[TMP1]], i64 10 +; CHECK-NEXT: store i32 [[B]], ptr [[GEP]], align 4 +; CHECK-NEXT: ret void +; +entry: + %add = add nsw i32 %a, 10 + %idx = zext nneg i32 %add to i64 + %gep = getelementptr inbounds i32, ptr %ptr, i64 %idx + store i32 %b, ptr %gep + ret void +} + +define void @test_zext_missing_nneg(ptr %ptr, i32 %a, i32 %b) { +; CHECK-LABEL: define void @test_zext_missing_nneg( +; CHECK-SAME: ptr [[PTR:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[A]], 10 +; CHECK-NEXT: [[IDX:%.*]] = zext i32 [[ADD]] to i64 +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[IDX]] +; CHECK-NEXT: store i32 [[B]], ptr [[GEP]], align 4 +; CHECK-NEXT: ret void +; +entry: + %add = add nsw i32 %a, 10 + %idx = zext i32 %add to i64 + %gep = getelementptr inbounds i32, ptr %ptr, i64 %idx + store i32 %b, ptr %gep + ret void +} diff --git a/llvm/test/Transforms/InstCombine/commutative-operation-over-phis.ll b/llvm/test/Transforms/InstCombine/commutative-operation-over-phis.ll new file mode 100644 index 000000000000..e8b0fb198bd1 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/commutative-operation-over-phis.ll @@ -0,0 +1,645 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes=instcombine -S < %s | FileCheck %s + +declare void @dummy() + +declare i32 @llvm.smax.i32(i32 %a, i32 %b) +declare i32 @llvm.smin.i32(i32 %a, i32 %b) +declare i32 @llvm.umax.i32(i32 %a, i32 %b) +declare i32 @llvm.umin.i32(i32 %a, i32 %b) +declare float @llvm.maxnum.f32(float %a, float %b) +declare float @llvm.minnum.f32(float %a, float %b) +declare float @llvm.maximum.f32(float %a, float %b) +declare float @llvm.minimum.f32(float %a, float %b) +declare float @llvm.pow.f32(float %a, float %b) + +define i8 @fold_phi_mul(i1 %c, i8 %a, i8 %b) { +; CHECK-LABEL: define i8 @fold_phi_mul( +; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = mul i8 [[A]], [[B]] +; CHECK-NEXT: ret i8 [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi i8 [%a, %entry], [%b, %then] + %phi2 = phi i8 [%b, %entry], [%a, %then] + %ret = mul i8 %phi1, %phi2 + ret i8 %ret +} + +define i8 @fold_phi_mul_three(i1 %c, i1 %d, i8 %a, i8 %b) { +; CHECK-LABEL: define i8 @fold_phi_mul_three( +; CHECK-SAME: i1 [[C:%.*]], i1 [[D:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN1:%.*]], label [[END:%.*]] +; CHECK: then1: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br i1 [[D]], label [[THEN2:%.*]], label [[END]] +; CHECK: then2: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = mul i8 [[A]], [[B]] +; CHECK-NEXT: ret i8 [[RET]] +; +entry: + br i1 %c, label %then1, label %end +then1: + call void @dummy() + br i1 %d, label %then2, label %end +then2: + call void @dummy() + br label %end +end: + %phi1 = phi i8 [%a, %entry], [%b, %then1], [%a, %then2] + %phi2 = phi i8 [%b, %entry], [%a, %then1], [%b, %then2] + %ret = mul i8 %phi1, %phi2 + ret i8 %ret +} + +define i8 @fold_phi_mul_three_notopt(i1 %c, i1 %d, i8 %a, i8 %b) { +; CHECK-LABEL: define i8 @fold_phi_mul_three_notopt( +; CHECK-SAME: i1 [[C:%.*]], i1 [[D:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN1:%.*]], label [[END:%.*]] +; CHECK: then1: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br i1 [[D]], label [[THEN2:%.*]], label [[END]] +; CHECK: then2: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[PHI1:%.*]] = phi i8 [ [[A]], [[ENTRY:%.*]] ], [ [[B]], [[THEN1]] ], [ [[A]], [[THEN2]] ] +; CHECK-NEXT: [[PHI2:%.*]] = phi i8 [ [[B]], [[ENTRY]] ], [ [[A]], [[THEN1]] ], [ [[A]], [[THEN2]] ] +; CHECK-NEXT: [[RET:%.*]] = mul i8 [[PHI1]], [[PHI2]] +; CHECK-NEXT: ret i8 [[RET]] +; +entry: + br i1 %c, label %then1, label %end +then1: + call void @dummy() + br i1 %d, label %then2, label %end +then2: + call void @dummy() + br label %end +end: + %phi1 = phi i8 [%a, %entry], [%b, %then1], [%a, %then2] + %phi2 = phi i8 [%b, %entry], [%a, %then1], [%a, %then2] + %ret = mul i8 %phi1, %phi2 + ret i8 %ret +} + +define i8 @fold_phi_mul_nsw_nuw(i1 %c, i8 %a, i8 %b) { +; CHECK-LABEL: define i8 @fold_phi_mul_nsw_nuw( +; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = mul nuw nsw i8 [[A]], [[B]] +; CHECK-NEXT: ret i8 [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi i8 [%a, %entry], [%b, %then] + %phi2 = phi i8 [%b, %entry], [%a, %then] + %ret = mul nsw nuw i8 %phi1, %phi2 + ret i8 %ret +} + +define <2 x i8> @fold_phi_mul_fix_vec(i1 %c, <2 x i8> %a, <2 x i8> %b) { +; CHECK-LABEL: define <2 x i8> @fold_phi_mul_fix_vec( +; CHECK-SAME: i1 [[C:%.*]], <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = mul <2 x i8> [[A]], [[B]] +; CHECK-NEXT: ret <2 x i8> [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi <2 x i8> [%a, %entry], [%b, %then] + %phi2 = phi <2 x i8> [%b, %entry], [%a, %then] + %ret = mul <2 x i8> %phi1, %phi2 + ret <2 x i8> %ret +} + +define <vscale x 2 x i8> @fold_phi_mul_scale_vec(i1 %c, <vscale x 2 x i8> %a, <vscale x 2 x i8> %b) { +; CHECK-LABEL: define <vscale x 2 x i8> @fold_phi_mul_scale_vec( +; CHECK-SAME: i1 [[C:%.*]], <vscale x 2 x i8> [[A:%.*]], <vscale x 2 x i8> [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = mul <vscale x 2 x i8> [[A]], [[B]] +; CHECK-NEXT: ret <vscale x 2 x i8> [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi <vscale x 2 x i8> [%a, %entry], [%b, %then] + %phi2 = phi <vscale x 2 x i8> [%b, %entry], [%a, %then] + %ret = mul <vscale x 2 x i8> %phi1, %phi2 + ret <vscale x 2 x i8> %ret +} + +define i8 @fold_phi_mul_commute(i1 %c, i8 %a, i8 %b) { +; CHECK-LABEL: define i8 @fold_phi_mul_commute( +; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = mul i8 [[A]], [[B]] +; CHECK-NEXT: ret i8 [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi i8 [%a, %entry], [%b, %then] + %phi2 = phi i8 [%a, %then], [%b, %entry] + %ret = mul i8 %phi1, %phi2 + ret i8 %ret +} + + +define i8 @fold_phi_mul_notopt(i1 %c, i8 %a, i8 %b, i8 %d) { +; CHECK-LABEL: define i8 @fold_phi_mul_notopt( +; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]], i8 [[D:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[PHI1:%.*]] = phi i8 [ [[A]], [[ENTRY:%.*]] ], [ [[B]], [[THEN]] ] +; CHECK-NEXT: [[PHI2:%.*]] = phi i8 [ [[B]], [[ENTRY]] ], [ [[D]], [[THEN]] ] +; CHECK-NEXT: [[RET:%.*]] = mul i8 [[PHI1]], [[PHI2]] +; CHECK-NEXT: ret i8 [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi i8 [%a, %entry], [%b, %then] + %phi2 = phi i8 [%b, %entry], [%d, %then] + %ret = mul i8 %phi1, %phi2 + ret i8 %ret +} + + +define i8 @fold_phi_sub(i1 %c, i8 %a, i8 %b) { +; CHECK-LABEL: define i8 @fold_phi_sub( +; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[PHI1:%.*]] = phi i8 [ [[A]], [[ENTRY:%.*]] ], [ [[B]], [[THEN]] ] +; CHECK-NEXT: [[PHI2:%.*]] = phi i8 [ [[B]], [[ENTRY]] ], [ [[A]], [[THEN]] ] +; CHECK-NEXT: [[RET:%.*]] = sub i8 [[PHI1]], [[PHI2]] +; CHECK-NEXT: ret i8 [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi i8 [%a, %entry], [%b, %then] + %phi2 = phi i8 [%b, %entry], [%a, %then] + %ret = sub i8 %phi1, %phi2 + ret i8 %ret +} + + +define i8 @fold_phi_add(i1 %c, i8 %a, i8 %b) { +; CHECK-LABEL: define i8 @fold_phi_add( +; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = add i8 [[A]], [[B]] +; CHECK-NEXT: ret i8 [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi i8 [%a, %entry], [%b, %then] + %phi2 = phi i8 [%b, %entry], [%a, %then] + %ret = add i8 %phi1, %phi2 + ret i8 %ret +} + +define i8 @fold_phi_and(i1 %c, i8 %a, i8 %b) { +; CHECK-LABEL: define i8 @fold_phi_and( +; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = and i8 [[A]], [[B]] +; CHECK-NEXT: ret i8 [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi i8 [%a, %entry], [%b, %then] + %phi2 = phi i8 [%b, %entry], [%a, %then] + %ret = and i8 %phi1, %phi2 + ret i8 %ret +} + +define i8 @fold_phi_or(i1 %c, i8 %a, i8 %b) { +; CHECK-LABEL: define i8 @fold_phi_or( +; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = or i8 [[A]], [[B]] +; CHECK-NEXT: ret i8 [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi i8 [%a, %entry], [%b, %then] + %phi2 = phi i8 [%b, %entry], [%a, %then] + %ret = or i8 %phi1, %phi2 + ret i8 %ret +} + + +define i8 @fold_phi_xor(i1 %c, i8 %a, i8 %b) { +; CHECK-LABEL: define i8 @fold_phi_xor( +; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = xor i8 [[A]], [[B]] +; CHECK-NEXT: ret i8 [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi i8 [%a, %entry], [%b, %then] + %phi2 = phi i8 [%b, %entry], [%a, %then] + %ret = xor i8 %phi1, %phi2 + ret i8 %ret +} + + +define float @fold_phi_fadd(i1 %c, float %a, float %b) { +; CHECK-LABEL: define float @fold_phi_fadd( +; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = fadd float [[A]], [[B]] +; CHECK-NEXT: ret float [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi float [%a, %entry], [%b, %then] + %phi2 = phi float [%b, %entry], [%a, %then] + %ret = fadd float %phi1, %phi2 + ret float %ret +} + +define float @fold_phi_fadd_nnan(i1 %c, float %a, float %b) { +; CHECK-LABEL: define float @fold_phi_fadd_nnan( +; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = fadd nnan float [[A]], [[B]] +; CHECK-NEXT: ret float [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi float [%a, %entry], [%b, %then] + %phi2 = phi float [%b, %entry], [%a, %then] + %ret = fadd nnan float %phi1, %phi2 + ret float %ret +} + + +define float @fold_phi_fmul(i1 %c, float %a, float %b) { +; CHECK-LABEL: define float @fold_phi_fmul( +; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = fmul float [[A]], [[B]] +; CHECK-NEXT: ret float [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi float [%a, %entry], [%b, %then] + %phi2 = phi float [%b, %entry], [%a, %then] + %ret = fmul float %phi1, %phi2 + ret float %ret +} + + +define i32 @fold_phi_smax(i1 %c, i32 %a, i32 %b) { +; CHECK-LABEL: define i32 @fold_phi_smax( +; CHECK-SAME: i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.smax.i32(i32 [[A]], i32 [[B]]) +; CHECK-NEXT: ret i32 [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi i32 [%a, %entry], [%b, %then] + %phi2 = phi i32 [%b, %entry], [%a, %then] + %ret = call i32 @llvm.smax.i32(i32 %phi1, i32 %phi2) + ret i32 %ret +} + + +define i32 @fold_phi_smin(i1 %c, i32 %a, i32 %b) { +; CHECK-LABEL: define i32 @fold_phi_smin( +; CHECK-SAME: i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.smin.i32(i32 [[A]], i32 [[B]]) +; CHECK-NEXT: ret i32 [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi i32 [%a, %entry], [%b, %then] + %phi2 = phi i32 [%b, %entry], [%a, %then] + %ret = call i32 @llvm.smin.i32(i32 %phi1, i32 %phi2) + ret i32 %ret +} + + +define i32 @fold_phi_umax(i1 %c, i32 %a, i32 %b) { +; CHECK-LABEL: define i32 @fold_phi_umax( +; CHECK-SAME: i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.umax.i32(i32 [[A]], i32 [[B]]) +; CHECK-NEXT: ret i32 [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi i32 [%a, %entry], [%b, %then] + %phi2 = phi i32 [%b, %entry], [%a, %then] + %ret = call i32 @llvm.umax.i32(i32 %phi1, i32 %phi2) + ret i32 %ret +} + +define i32 @fold_phi_umin(i1 %c, i32 %a, i32 %b) { +; CHECK-LABEL: define i32 @fold_phi_umin( +; CHECK-SAME: i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.umin.i32(i32 [[A]], i32 [[B]]) +; CHECK-NEXT: ret i32 [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi i32 [%a, %entry], [%b, %then] + %phi2 = phi i32 [%b, %entry], [%a, %then] + %ret = call i32 @llvm.umin.i32(i32 %phi1, i32 %phi2) + ret i32 %ret +} + + +define float @fold_phi_maxnum(i1 %c, float %a, float %b) { +; CHECK-LABEL: define float @fold_phi_maxnum( +; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = call float @llvm.maxnum.f32(float [[A]], float [[B]]) +; CHECK-NEXT: ret float [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi float [%a, %entry], [%b, %then] + %phi2 = phi float [%b, %entry], [%a, %then] + %ret = call float @llvm.maxnum.f32(float %phi1, float %phi2) + ret float %ret +} + +define float @fold_phi_pow(i1 %c, float %a, float %b) { +; CHECK-LABEL: define float @fold_phi_pow( +; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[PHI1:%.*]] = phi float [ [[A]], [[ENTRY:%.*]] ], [ [[B]], [[THEN]] ] +; CHECK-NEXT: [[PHI2:%.*]] = phi float [ [[B]], [[ENTRY]] ], [ [[A]], [[THEN]] ] +; CHECK-NEXT: [[RET:%.*]] = call float @llvm.pow.f32(float [[PHI1]], float [[PHI2]]) +; CHECK-NEXT: ret float [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi float [%a, %entry], [%b, %then] + %phi2 = phi float [%b, %entry], [%a, %then] + %ret = call float @llvm.pow.f32(float %phi1, float %phi2) + ret float %ret +} + +define float @fold_phi_minnum(i1 %c, float %a, float %b) { +; CHECK-LABEL: define float @fold_phi_minnum( +; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = call float @llvm.minnum.f32(float [[A]], float [[B]]) +; CHECK-NEXT: ret float [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi float [%a, %entry], [%b, %then] + %phi2 = phi float [%b, %entry], [%a, %then] + %ret = call float @llvm.minnum.f32(float %phi1, float %phi2) + ret float %ret +} + +define float @fold_phi_maximum(i1 %c, float %a, float %b) { +; CHECK-LABEL: define float @fold_phi_maximum( +; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = call float @llvm.maximum.f32(float [[A]], float [[B]]) +; CHECK-NEXT: ret float [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi float [%a, %entry], [%b, %then] + %phi2 = phi float [%b, %entry], [%a, %then] + %ret = call float @llvm.maximum.f32(float %phi1, float %phi2) + ret float %ret +} + +define float @fold_phi_minimum(i1 %c, float %a, float %b) { +; CHECK-LABEL: define float @fold_phi_minimum( +; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[C]], label [[THEN:%.*]], label [[END:%.*]] +; CHECK: then: +; CHECK-NEXT: call void @dummy() +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET:%.*]] = call float @llvm.minimum.f32(float [[A]], float [[B]]) +; CHECK-NEXT: ret float [[RET]] +; +entry: + br i1 %c, label %then, label %end +then: + call void @dummy() + br label %end +end: + %phi1 = phi float [%a, %entry], [%b, %then] + %phi2 = phi float [%b, %entry], [%a, %then] + %ret = call float @llvm.minimum.f32(float %phi1, float %phi2) + ret float %ret +} + diff --git a/llvm/test/Transforms/InstCombine/icmp.ll b/llvm/test/Transforms/InstCombine/icmp.ll index fe831934f4f3..1c7bb36f0d34 100644 --- a/llvm/test/Transforms/InstCombine/icmp.ll +++ b/llvm/test/Transforms/InstCombine/icmp.ll @@ -3862,9 +3862,10 @@ define <8 x i1> @bitreverse_vec_ne(<8 x i16> %x, <8 x i16> %y) { define i1 @knownbits1(i8 %a, i8 %b) { ; CHECK-LABEL: @knownbits1( ; CHECK-NEXT: [[A1:%.*]] = and i8 [[A:%.*]], 1 +; CHECK-NEXT: [[A2:%.*]] = or disjoint i8 [[A1]], 4 ; CHECK-NEXT: [[B1:%.*]] = and i8 [[B:%.*]], 2 -; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i8 [[B1]], 1 -; CHECK-NEXT: [[C:%.*]] = icmp eq i8 [[A1]], [[TMP1]] +; CHECK-NEXT: [[B2:%.*]] = or disjoint i8 [[B1]], 5 +; CHECK-NEXT: [[C:%.*]] = icmp eq i8 [[A2]], [[B2]] ; CHECK-NEXT: ret i1 [[C]] ; %a1 = and i8 %a, 5 @@ -3878,9 +3879,10 @@ define i1 @knownbits1(i8 %a, i8 %b) { define i1 @knownbits2(i8 %a, i8 %b) { ; CHECK-LABEL: @knownbits2( ; CHECK-NEXT: [[A1:%.*]] = and i8 [[A:%.*]], 1 +; CHECK-NEXT: [[A2:%.*]] = or disjoint i8 [[A1]], 4 ; CHECK-NEXT: [[B1:%.*]] = and i8 [[B:%.*]], 2 -; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i8 [[B1]], 1 -; CHECK-NEXT: [[C:%.*]] = icmp ne i8 [[A1]], [[TMP1]] +; CHECK-NEXT: [[B2:%.*]] = or disjoint i8 [[B1]], 5 +; CHECK-NEXT: [[C:%.*]] = icmp ne i8 [[A2]], [[B2]] ; CHECK-NEXT: ret i1 [[C]] ; %a1 = and i8 %a, 5 @@ -3894,9 +3896,10 @@ define i1 @knownbits2(i8 %a, i8 %b) { define i1 @knownbits3(i8 %a, i8 %b) { ; CHECK-LABEL: @knownbits3( ; CHECK-NEXT: [[A1:%.*]] = and i8 [[A:%.*]], 1 +; CHECK-NEXT: [[A2:%.*]] = or disjoint i8 [[A1]], 4 ; CHECK-NEXT: [[B1:%.*]] = and i8 [[B:%.*]], 2 -; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i8 [[B1]], 1 -; CHECK-NEXT: [[C:%.*]] = icmp eq i8 [[TMP1]], [[A1]] +; CHECK-NEXT: [[B2:%.*]] = or disjoint i8 [[B1]], 5 +; CHECK-NEXT: [[C:%.*]] = icmp eq i8 [[B2]], [[A2]] ; CHECK-NEXT: ret i1 [[C]] ; %a1 = and i8 %a, 5 @@ -3910,9 +3913,10 @@ define i1 @knownbits3(i8 %a, i8 %b) { define <2 x i1> @knownbits4(<2 x i8> %a, <2 x i8> %b) { ; CHECK-LABEL: @knownbits4( ; CHECK-NEXT: [[A1:%.*]] = and <2 x i8> [[A:%.*]], <i8 1, i8 1> +; CHECK-NEXT: [[A2:%.*]] = or disjoint <2 x i8> [[A1]], <i8 4, i8 4> ; CHECK-NEXT: [[B1:%.*]] = and <2 x i8> [[B:%.*]], <i8 2, i8 2> -; CHECK-NEXT: [[TMP1:%.*]] = or disjoint <2 x i8> [[B1]], <i8 1, i8 1> -; CHECK-NEXT: [[C:%.*]] = icmp ne <2 x i8> [[TMP1]], [[A1]] +; CHECK-NEXT: [[B2:%.*]] = or disjoint <2 x i8> [[B1]], <i8 5, i8 5> +; CHECK-NEXT: [[C:%.*]] = icmp ne <2 x i8> [[B2]], [[A2]] ; CHECK-NEXT: ret <2 x i1> [[C]] ; %a1 = and <2 x i8> %a, <i8 5, i8 5> @@ -3928,9 +3932,10 @@ define <2 x i1> @knownbits4(<2 x i8> %a, <2 x i8> %b) { define i1 @knownbits5(i8 %a, i8 %b) { ; CHECK-LABEL: @knownbits5( ; CHECK-NEXT: [[A1:%.*]] = and i8 [[A:%.*]], -127 +; CHECK-NEXT: [[A2:%.*]] = or disjoint i8 [[A1]], 4 ; CHECK-NEXT: [[B1:%.*]] = and i8 [[B:%.*]], 2 -; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i8 [[B1]], 1 -; CHECK-NEXT: [[C:%.*]] = icmp eq i8 [[A1]], [[TMP1]] +; CHECK-NEXT: [[B2:%.*]] = or disjoint i8 [[B1]], 5 +; CHECK-NEXT: [[C:%.*]] = icmp eq i8 [[A2]], [[B2]] ; CHECK-NEXT: ret i1 [[C]] ; %a1 = and i8 %a, 133 @@ -3944,9 +3949,10 @@ define i1 @knownbits5(i8 %a, i8 %b) { define i1 @knownbits6(i8 %a, i8 %b) { ; CHECK-LABEL: @knownbits6( ; CHECK-NEXT: [[A1:%.*]] = and i8 [[A:%.*]], -127 +; CHECK-NEXT: [[A2:%.*]] = or disjoint i8 [[A1]], 4 ; CHECK-NEXT: [[B1:%.*]] = and i8 [[B:%.*]], 2 -; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i8 [[B1]], 1 -; CHECK-NEXT: [[C:%.*]] = icmp ne i8 [[A1]], [[TMP1]] +; CHECK-NEXT: [[B2:%.*]] = or disjoint i8 [[B1]], 5 +; CHECK-NEXT: [[C:%.*]] = icmp ne i8 [[A2]], [[B2]] ; CHECK-NEXT: ret i1 [[C]] ; %a1 = and i8 %a, 133 @@ -3960,9 +3966,10 @@ define i1 @knownbits6(i8 %a, i8 %b) { define <2 x i1> @knownbits7(<2 x i8> %a, <2 x i8> %b) { ; CHECK-LABEL: @knownbits7( ; CHECK-NEXT: [[A1:%.*]] = and <2 x i8> [[A:%.*]], <i8 -127, i8 -127> +; CHECK-NEXT: [[A2:%.*]] = or disjoint <2 x i8> [[A1]], <i8 4, i8 4> ; CHECK-NEXT: [[B1:%.*]] = and <2 x i8> [[B:%.*]], <i8 2, i8 2> -; CHECK-NEXT: [[TMP1:%.*]] = or disjoint <2 x i8> [[B1]], <i8 1, i8 1> -; CHECK-NEXT: [[C:%.*]] = icmp eq <2 x i8> [[TMP1]], [[A1]] +; CHECK-NEXT: [[B2:%.*]] = or disjoint <2 x i8> [[B1]], <i8 5, i8 5> +; CHECK-NEXT: [[C:%.*]] = icmp eq <2 x i8> [[B2]], [[A2]] ; CHECK-NEXT: ret <2 x i1> [[C]] ; %a1 = and <2 x i8> %a, <i8 133, i8 133> @@ -3976,9 +3983,10 @@ define <2 x i1> @knownbits7(<2 x i8> %a, <2 x i8> %b) { define i1 @knownbits8(i8 %a, i8 %b) { ; CHECK-LABEL: @knownbits8( ; CHECK-NEXT: [[A1:%.*]] = and i8 [[A:%.*]], -127 +; CHECK-NEXT: [[A2:%.*]] = or disjoint i8 [[A1]], 4 ; CHECK-NEXT: [[B1:%.*]] = and i8 [[B:%.*]], 2 -; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i8 [[B1]], 1 -; CHECK-NEXT: [[C:%.*]] = icmp ne i8 [[TMP1]], [[A1]] +; CHECK-NEXT: [[B2:%.*]] = or disjoint i8 [[B1]], 5 +; CHECK-NEXT: [[C:%.*]] = icmp ne i8 [[B2]], [[A2]] ; CHECK-NEXT: ret i1 [[C]] ; %a1 = and i8 %a, 133 @@ -4904,80 +4912,3 @@ define i1 @or_positive_sgt_zero_multi_use(i8 %a) { %cmp = icmp sgt i8 %b, 0 ret i1 %cmp } - - -define i1 @disjoint_or_sgt_1(i8 %a, i8 %b) { -; CHECK-LABEL: @disjoint_or_sgt_1( -; CHECK-NEXT: [[B1:%.*]] = add nsw i8 [[B:%.*]], 2 -; CHECK-NEXT: [[ICMP_:%.*]] = icmp sle i8 [[B1]], [[A:%.*]] -; CHECK-NEXT: ret i1 [[ICMP_]] -; - %a1 = or disjoint i8 %a, 1 - %b1 = add nsw i8 %b, 2 - %icmp_ = icmp sgt i8 %a1, %b1 - ret i1 %icmp_ -} - -define i1 @disjoint_or_sgt_2(i8 %a, i8 %b) { -; CHECK-LABEL: @disjoint_or_sgt_2( -; CHECK-NEXT: [[A1:%.*]] = or disjoint i8 [[A:%.*]], 2 -; CHECK-NEXT: [[B1:%.*]] = add i8 [[B:%.*]], 1 -; CHECK-NEXT: [[ICMP_:%.*]] = icmp sgt i8 [[A1]], [[B1]] -; CHECK-NEXT: ret i1 [[ICMP_]] -; - %a1 = or disjoint i8 %a, 2 - %b1 = add i8 %b, 1 - %icmp_ = icmp sgt i8 %a1, %b1 - ret i1 %icmp_ -} - -define i1 @disjoint_or_sgt_3(i8 %a, i8 %b) { -; CHECK-LABEL: @disjoint_or_sgt_3( -; CHECK-NEXT: [[A1:%.*]] = or disjoint i8 [[A:%.*]], 2 -; CHECK-NEXT: [[B1:%.*]] = add nuw i8 [[B:%.*]], 1 -; CHECK-NEXT: [[ICMP_:%.*]] = icmp sgt i8 [[A1]], [[B1]] -; CHECK-NEXT: ret i1 [[ICMP_]] -; - %a1 = or disjoint i8 %a, 2 - %b1 = add nuw i8 %b, 1 - %icmp_ = icmp sgt i8 %a1, %b1 - ret i1 %icmp_ -} - -define i1 @disjoint_or_ugt_1(i8 %a, i8 %b) { -; CHECK-LABEL: @disjoint_or_ugt_1( -; CHECK-NEXT: [[B1:%.*]] = add nsw i8 [[B:%.*]], 2 -; CHECK-NEXT: [[ICMP_:%.*]] = icmp ule i8 [[B1]], [[A:%.*]] -; CHECK-NEXT: ret i1 [[ICMP_]] -; - %a1 = or disjoint i8 %a, 1 - %b1 = add nsw i8 %b, 2 - %icmp_ = icmp ugt i8 %a1, %b1 - ret i1 %icmp_ -} - -define i1 @disjoint_or_ugt_2(i8 %a, i8 %b) { -; CHECK-LABEL: @disjoint_or_ugt_2( -; CHECK-NEXT: [[A1:%.*]] = or disjoint i8 [[A:%.*]], 2 -; CHECK-NEXT: [[B1:%.*]] = add i8 [[B:%.*]], 1 -; CHECK-NEXT: [[ICMP_:%.*]] = icmp ugt i8 [[A1]], [[B1]] -; CHECK-NEXT: ret i1 [[ICMP_]] -; - %a1 = or disjoint i8 %a, 2 - %b1 = add i8 %b, 1 - %icmp_ = icmp ugt i8 %a1, %b1 - ret i1 %icmp_ -} - -define i1 @disjoint_or_ugt_3(i8 %a, i8 %b) { -; CHECK-LABEL: @disjoint_or_ugt_3( -; CHECK-NEXT: [[A1:%.*]] = or disjoint i8 [[A:%.*]], 2 -; CHECK-NEXT: [[B1:%.*]] = add nuw i8 [[B:%.*]], 1 -; CHECK-NEXT: [[ICMP_:%.*]] = icmp ugt i8 [[A1]], [[B1]] -; CHECK-NEXT: ret i1 [[ICMP_]] -; - %a1 = or disjoint i8 %a, 2 - %b1 = add nuw i8 %b, 1 - %icmp_ = icmp ugt i8 %a1, %b1 - ret i1 %icmp_ -} diff --git a/llvm/test/Transforms/InstCombine/mul-inseltpoison.ll b/llvm/test/Transforms/InstCombine/mul-inseltpoison.ll index 448558d755a5..8fe4261bbf00 100644 --- a/llvm/test/Transforms/InstCombine/mul-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/mul-inseltpoison.ll @@ -672,9 +672,8 @@ define <2 x i32> @test_mul_canonicalize_vec(<2 x i32> %x, <2 x i32> %y) { define i32 @test_mul_canonicalize_multiple_uses(i32 %x, i32 %y) { ; CHECK-LABEL: @test_mul_canonicalize_multiple_uses( -; CHECK-NEXT: [[NEG:%.*]] = sub i32 0, [[X:%.*]] -; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[NEG]], [[Y:%.*]] -; CHECK-NEXT: [[MUL2:%.*]] = mul i32 [[MUL]], [[NEG]] +; CHECK-NEXT: [[MUL_NEG:%.*]] = mul i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[MUL2:%.*]] = mul i32 [[MUL_NEG]], [[X]] ; CHECK-NEXT: ret i32 [[MUL2]] ; %neg = sub i32 0, %x diff --git a/llvm/test/Transforms/InstCombine/mul.ll b/llvm/test/Transforms/InstCombine/mul.ll index 9fe8462c5d31..b404fcffbf42 100644 --- a/llvm/test/Transforms/InstCombine/mul.ll +++ b/llvm/test/Transforms/InstCombine/mul.ll @@ -1255,17 +1255,190 @@ define <2 x i32> @test_mul_canonicalize_vec(<2 x i32> %x, <2 x i32> %y) { define i32 @test_mul_canonicalize_multiple_uses(i32 %x, i32 %y) { ; CHECK-LABEL: @test_mul_canonicalize_multiple_uses( +; CHECK-NEXT: [[MUL_NEG:%.*]] = mul i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[MUL2:%.*]] = mul i32 [[MUL_NEG]], [[X]] +; CHECK-NEXT: ret i32 [[MUL2]] +; + %neg = sub i32 0, %x + %mul = mul i32 %neg, %y + %mul2 = mul i32 %mul, %neg + ret i32 %mul2 +} + +define i32 @mul_nsw_mul_nsw_neg(i32 %x, i32 %y) { +; CHECK-LABEL: @mul_nsw_mul_nsw_neg( +; CHECK-NEXT: [[MUL_NEG:%.*]] = mul i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[MUL2:%.*]] = mul i32 [[MUL_NEG]], [[X]] +; CHECK-NEXT: ret i32 [[MUL2]] +; + %neg = sub i32 0, %x + %mul = mul nsw i32 %neg, %y + %mul2 = mul nsw i32 %mul, %neg + ret i32 %mul2 +} + +define i32 @mul_mul_nsw_neg(i32 %x,i32 %y) { +; CHECK-LABEL: @mul_mul_nsw_neg( +; CHECK-NEXT: [[MUL_NEG:%.*]] = mul i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[MUL2:%.*]] = mul i32 [[MUL_NEG]], [[X]] +; CHECK-NEXT: ret i32 [[MUL2]] +; + %neg = sub i32 0, %x + %mul = mul nsw i32 %neg, %y + %mul2 = mul i32 %mul, %neg + ret i32 %mul2 +} + +define i32 @mul_nsw_mul_neg(i32 %x,i32 %y) { +; CHECK-LABEL: @mul_nsw_mul_neg( +; CHECK-NEXT: [[MUL_NEG:%.*]] = mul i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[MUL2:%.*]] = mul i32 [[MUL_NEG]], [[X]] +; CHECK-NEXT: ret i32 [[MUL2]] +; + %neg = sub i32 0, %x + %mul = mul i32 %neg, %y + %mul2 = mul nsw i32 %mul, %neg + ret i32 %mul2 +} + +define i32 @mul_nsw_mul_neg_onearg(i32 %x) { +; CHECK-LABEL: @mul_nsw_mul_neg_onearg( +; CHECK-NEXT: [[MUL_NEG:%.*]] = mul i32 [[X:%.*]], [[X]] +; CHECK-NEXT: [[MUL2:%.*]] = mul i32 [[MUL_NEG]], [[X]] +; CHECK-NEXT: ret i32 [[MUL2]] +; + %neg = sub i32 0, %x + %mul = mul i32 %neg, %x + %mul2 = mul nsw i32 %mul, %neg + ret i32 %mul2 +} + +define i8 @mul_mul_nsw_neg_onearg(i8 %x) { +; CHECK-LABEL: @mul_mul_nsw_neg_onearg( +; CHECK-NEXT: [[MUL_NEG:%.*]] = mul i8 [[X:%.*]], [[X]] +; CHECK-NEXT: [[MUL2:%.*]] = mul i8 [[MUL_NEG]], [[X]] +; CHECK-NEXT: ret i8 [[MUL2]] +; + %neg = sub i8 0, %x + %mul = mul nsw i8 %neg, %x + %mul2 = mul i8 %mul, %neg + ret i8 %mul2 +} + +define i32 @mul_nsw_mul_nsw_neg_onearg(i32 %x) { +; CHECK-LABEL: @mul_nsw_mul_nsw_neg_onearg( +; CHECK-NEXT: [[MUL_NEG:%.*]] = mul i32 [[X:%.*]], [[X]] +; CHECK-NEXT: [[MUL2:%.*]] = mul i32 [[MUL_NEG]], [[X]] +; CHECK-NEXT: ret i32 [[MUL2]] +; + %neg = sub i32 0, %x + %mul = mul nsw i32 %neg, %x + %mul2 = mul nsw i32 %mul, %neg + ret i32 %mul2 +} + +define i32 @mul_nsw_shl_nsw_neg(i32 %x, i32 %y) { +; CHECK-LABEL: @mul_nsw_shl_nsw_neg( +; CHECK-NEXT: [[SHL_NEG:%.*]] = shl i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[SHL_NEG]], [[X]] +; CHECK-NEXT: ret i32 [[MUL]] +; + %neg = sub i32 0, %x + %shl = shl nsw i32 %neg, %y + %mul = mul nsw i32 %shl, %neg + ret i32 %mul +} + +define i32 @mul_shl_nsw_neg(i32 %x,i32 %y) { +; CHECK-LABEL: @mul_shl_nsw_neg( +; CHECK-NEXT: [[SHL_NEG:%.*]] = shl i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[SHL_NEG]], [[X]] +; CHECK-NEXT: ret i32 [[MUL]] +; + %neg = sub i32 0, %x + %shl = shl nsw i32 %neg, %y + %mul = mul i32 %shl, %neg + ret i32 %mul +} + +define i32 @mul_nsw_shl_neg(i32 %x,i32 %y) { +; CHECK-LABEL: @mul_nsw_shl_neg( +; CHECK-NEXT: [[SHL_NEG:%.*]] = shl i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[SHL_NEG]], [[X]] +; CHECK-NEXT: ret i32 [[MUL]] +; + %neg = sub i32 0, %x + %shl = shl i32 %neg, %y + %mul = mul nsw i32 %shl, %neg + ret i32 %mul +} + +define i32 @mul_nsw_shl_neg_onearg(i32 %x) { +; CHECK-LABEL: @mul_nsw_shl_neg_onearg( +; CHECK-NEXT: [[SHL_NEG:%.*]] = shl i32 [[X:%.*]], [[X]] +; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[SHL_NEG]], [[X]] +; CHECK-NEXT: ret i32 [[MUL]] +; + %neg = sub i32 0, %x + %shl = shl i32 %neg, %x + %mul = mul nsw i32 %shl, %neg + ret i32 %mul +} + +define i8 @mul_shl_nsw_neg_onearg(i8 %x) { +; CHECK-LABEL: @mul_shl_nsw_neg_onearg( +; CHECK-NEXT: [[SHL_NEG:%.*]] = shl i8 [[X:%.*]], [[X]] +; CHECK-NEXT: [[MUL:%.*]] = mul i8 [[SHL_NEG]], [[X]] +; CHECK-NEXT: ret i8 [[MUL]] +; + %neg = sub i8 0, %x + %shl = shl nsw i8 %neg, %x + %mul = mul i8 %shl, %neg + ret i8 %mul +} + +define i32 @mul_nsw_shl_nsw_neg_onearg(i32 %x) { +; CHECK-LABEL: @mul_nsw_shl_nsw_neg_onearg( +; CHECK-NEXT: [[SHL_NEG:%.*]] = mul i32 [[X:%.*]], [[X]] +; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[SHL_NEG]], [[X]] +; CHECK-NEXT: ret i32 [[MUL]] +; + %neg = sub i32 0, %x + %shl = mul nsw i32 %neg, %x + %mul = mul nsw i32 %shl, %neg + ret i32 %mul +} + +define i32 @mul_use_mul_neg(i32 %x,i32 %y) { +; CHECK-LABEL: @mul_use_mul_neg( ; CHECK-NEXT: [[NEG:%.*]] = sub i32 0, [[X:%.*]] ; CHECK-NEXT: [[MUL:%.*]] = mul i32 [[NEG]], [[Y:%.*]] +; CHECK-NEXT: call void @use32(i32 [[MUL]]) ; CHECK-NEXT: [[MUL2:%.*]] = mul i32 [[MUL]], [[NEG]] ; CHECK-NEXT: ret i32 [[MUL2]] ; %neg = sub i32 0, %x %mul = mul i32 %neg, %y + call void @use32(i32 %mul) %mul2 = mul i32 %mul, %neg ret i32 %mul2 } +define i32 @mul_shl_use_mul_neg(i32 %x,i32 %y) { +; CHECK-LABEL: @mul_shl_use_mul_neg( +; CHECK-NEXT: [[NEG:%.*]] = sub i32 0, [[X:%.*]] +; CHECK-NEXT: [[SHL:%.*]] = shl i32 [[NEG]], [[Y:%.*]] +; CHECK-NEXT: call void @use32(i32 [[SHL]]) +; CHECK-NEXT: [[MUL2:%.*]] = mul i32 [[SHL]], [[NEG]] +; CHECK-NEXT: ret i32 [[MUL2]] +; + %neg = sub i32 0, %x + %shl = shl i32 %neg, %y + call void @use32(i32 %shl) + %mul2 = mul i32 %shl, %neg + ret i32 %mul2 +} + @X = global i32 5 define i64 @test_mul_canonicalize_neg_is_not_undone(i64 %L1) { diff --git a/llvm/test/tools/llvm-objdump/XCOFF/disassemble-symbolize-operands.ll b/llvm/test/tools/llvm-objdump/XCOFF/disassemble-symbolize-operands.ll index adedb6b7a5ab..2b4d6806292c 100644 --- a/llvm/test/tools/llvm-objdump/XCOFF/disassemble-symbolize-operands.ll +++ b/llvm/test/tools/llvm-objdump/XCOFF/disassemble-symbolize-operands.ll @@ -21,7 +21,7 @@ ; CHECK-NEXT: 68: cmplwi 3, 11 ; CHECK-NEXT: 6c: bt 0, 0x60 <L2> ; CHECK-NEXT: 70: mr 31, 3 -; CHECK-NEXT: 74: bl 0x0 <.internal> +; CHECK-NEXT: 74: bl 0x0 <.extern> ; CHECK-NEXT: 78: nop ; CHECK-NEXT: 7c: mr 3, 31 ; CHECK-NEXT: 80: b 0x60 <L2> diff --git a/llvm/test/tools/llvm-objdump/XCOFF/disassemble-symbolize-operands2.ll b/llvm/test/tools/llvm-objdump/XCOFF/disassemble-symbolize-operands2.ll new file mode 100644 index 000000000000..a9cee924845f --- /dev/null +++ b/llvm/test/tools/llvm-objdump/XCOFF/disassemble-symbolize-operands2.ll @@ -0,0 +1,65 @@ +; RUN: llc -mtriple=powerpc-ibm-aix-xcoff %s -filetype=obj -o %t +; RUN: llvm-objdump %t -r -d --symbolize-operands --no-show-raw-insn \ +; RUN: | FileCheck %s + +; CHECK-LABEL: <.a>: +;; No <L0> should appear +; CHECK-NEXT: 0: mflr 0 +; CHECK-NEXT: 4: stwu 1, -64(1) +; CHECK-NEXT: 8: lwz 3, 0(2) +; CHECK-NEXT:0000000a: R_TOC var +; CHECK-NEXT: c: stw 0, 72(1) +; CHECK-NEXT: 10: lwz 3, 0(3) +; CHECK-NEXT: 14: bl 0x4c <.b> +; CHECK-NEXT: 18: nop +; CHECK-NEXT: 1c: li 3, 1 +; CHECK-NEXT: 20: bl 0x0 <.c> +; CHECK-NEXT:00000020: R_RBR .c + +; CHECK-LABEL: <.b>: +; CHECK-NEXT: 4c: mflr 0 +; CHECK-NEXT: 50: stwu 1, -64(1) +; CHECK-NEXT: 54: cmplwi 3, 1 +; CHECK-NEXT: 58: stw 0, 72(1) +; CHECK-NEXT: 5c: stw 3, 60(1) +; CHECK-NEXT: 60: bf 2, 0x6c <L0> +; CHECK-NEXT: 64: bl 0x0 <.a> +; CHECK-NEXT: 68: nop +; CHECK-NEXT:<L0>: +; CHECK-NEXT: 6c: li 3, 2 +; CHECK-NEXT: 70: bl 0x0 <.c> +; CHECK-NEXT:00000070: R_RBR .c + +target triple = "powerpc-ibm-aix7.2.0.0" + +@var = external global i32, align 4 + +; Function Attrs: noinline nounwind optnone +define i32 @a() { +entry: + %0 = load i32, ptr @var, align 4 + %call = call i32 @b(i32 noundef %0) + %call1 = call i32 @c(i32 noundef 1) + ret i32 %call1 +} + +; Function Attrs: noinline nounwind optnone +define i32 @b(i32 noundef %x) { +entry: + %x.addr = alloca i32, align 4 + store i32 %x, ptr %x.addr, align 4 + %0 = load i32, ptr %x.addr, align 4 + %cmp = icmp eq i32 %0, 1 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + %call = call i32 @a() + br label %if.end + +if.end: ; preds = %if.then, %entry + %call1 = call i32 @c(i32 noundef 2) + ret i32 %call1 +} + +declare i32 @c(i32 noundef) + diff --git a/llvm/test/tools/llvm-readtapi/stubify-invalid.test b/llvm/test/tools/llvm-readtapi/stubify-invalid.test index 3ecc9ff1aeee..ae97d47c5199 100644 --- a/llvm/test/tools/llvm-readtapi/stubify-invalid.test +++ b/llvm/test/tools/llvm-readtapi/stubify-invalid.test @@ -1,6 +1,7 @@ ; RUN: rm -rf %t -; RUN: not llvm-readtapi -stubify %t/objc.dylib %t/flat_namespace.dylib %t/thread_local.dylib %t/fat.dylib --o tmp.tbd 2>&1 | FileCheck %s --allow-empty --check-prefix OUT -; RUN: not llvm-readtapi -stubify --o tmp.tbd 2>&1 | FileCheck %s --allow-empty --check-prefix IN +; RUN: mkdir -p %t +; RUN: not llvm-readtapi -stubify %t/objc.dylib %t/flat_namespace.dylib %t/thread_local.dylib %t/fat.dylib --o %t/tmp.tbd 2>&1 | FileCheck %s --allow-empty --check-prefix OUT +; RUN: not llvm-readtapi -stubify --o %t/tmp.tbd 2>&1 | FileCheck %s --allow-empty --check-prefix IN ; OUT: error: cannot write multiple inputs into single output file ; IN: error: stubify requires at least one input file diff --git a/llvm/tools/llvm-objdump/XCOFFDump.cpp b/llvm/tools/llvm-objdump/XCOFFDump.cpp index 0f6147924f8a..d9c00c096209 100644 --- a/llvm/tools/llvm-objdump/XCOFFDump.cpp +++ b/llvm/tools/llvm-objdump/XCOFFDump.cpp @@ -43,6 +43,7 @@ objdump::createXCOFFDumper(const object::XCOFFObjectFile &Obj) { Error objdump::getXCOFFRelocationValueString(const XCOFFObjectFile &Obj, const RelocationRef &Rel, + bool SymbolDescription, SmallVectorImpl<char> &Result) { symbol_iterator SymI = Rel.getSymbol(); if (SymI == Obj.symbol_end()) diff --git a/llvm/tools/llvm-objdump/XCOFFDump.h b/llvm/tools/llvm-objdump/XCOFFDump.h index cf5b19f910ea..0ba6ba4cdaaa 100644 --- a/llvm/tools/llvm-objdump/XCOFFDump.h +++ b/llvm/tools/llvm-objdump/XCOFFDump.h @@ -33,6 +33,7 @@ std::string getXCOFFSymbolDescription(const SymbolInfoTy &SymbolInfo, Error getXCOFFRelocationValueString(const object::XCOFFObjectFile &Obj, const object::RelocationRef &RelRef, + bool SymbolDescription, llvm::SmallVectorImpl<char> &Result); void dumpTracebackTable(ArrayRef<uint8_t> Bytes, uint64_t Address, diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp index 463d73e73ef8..7467a6062b5a 100644 --- a/llvm/tools/llvm-objdump/llvm-objdump.cpp +++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp @@ -424,6 +424,7 @@ bool objdump::isRelocAddressLess(RelocationRef A, RelocationRef B) { } static Error getRelocationValueString(const RelocationRef &Rel, + bool SymbolDescription, SmallVectorImpl<char> &Result) { const ObjectFile *Obj = Rel.getObject(); if (auto *ELF = dyn_cast<ELFObjectFileBase>(Obj)) @@ -435,7 +436,8 @@ static Error getRelocationValueString(const RelocationRef &Rel, if (auto *MachO = dyn_cast<MachOObjectFile>(Obj)) return getMachORelocationValueString(MachO, Rel, Result); if (auto *XCOFF = dyn_cast<XCOFFObjectFile>(Obj)) - return getXCOFFRelocationValueString(*XCOFF, Rel, Result); + return getXCOFFRelocationValueString(*XCOFF, Rel, SymbolDescription, + Result); llvm_unreachable("unknown object file format"); } @@ -527,7 +529,7 @@ static void printRelocation(formatted_raw_ostream &OS, StringRef FileName, SmallString<16> Name; SmallString<32> Val; Rel.getTypeName(Name); - if (Error E = getRelocationValueString(Rel, Val)) + if (Error E = getRelocationValueString(Rel, SymbolDescription, Val)) reportError(std::move(E), FileName); OS << (Is64Bits || !LeadingAddr ? "\t\t" : "\t\t\t"); if (LeadingAddr) @@ -1289,7 +1291,8 @@ collectLocalBranchTargets(ArrayRef<uint8_t> Bytes, MCInstrAnalysis *MIA, uint64_t Start, uint64_t End, std::unordered_map<uint64_t, std::string> &Labels) { // So far only supports PowerPC and X86. - if (!STI->getTargetTriple().isPPC() && !STI->getTargetTriple().isX86()) + const bool isPPC = STI->getTargetTriple().isPPC(); + if (!isPPC && !STI->getTargetTriple().isX86()) return; if (MIA) @@ -1299,8 +1302,8 @@ collectLocalBranchTargets(ArrayRef<uint8_t> Bytes, MCInstrAnalysis *MIA, unsigned LabelCount = 0; Start += SectionAddr; End += SectionAddr; - uint64_t Index = Start; - while (Index < End) { + const bool isXCOFF = STI->getTargetTriple().isOSBinFormatXCOFF(); + for (uint64_t Index = Start; Index < End;) { // Disassemble a real instruction and record function-local branch labels. MCInst Inst; uint64_t Size; @@ -1311,18 +1314,22 @@ collectLocalBranchTargets(ArrayRef<uint8_t> Bytes, MCInstrAnalysis *MIA, Size = std::min<uint64_t>(ThisBytes.size(), DisAsm->suggestBytesToSkip(ThisBytes, Index)); - if (Disassembled && MIA) { - uint64_t Target; - bool TargetKnown = MIA->evaluateBranch(Inst, Index, Size, Target); - // On PowerPC, if the address of a branch is the same as the target, it - // means that it's a function call. Do not mark the label for this case. - if (TargetKnown && (Target >= Start && Target < End) && - !Labels.count(Target) && - !(STI->getTargetTriple().isPPC() && Target == Index)) - Labels[Target] = ("L" + Twine(LabelCount++)).str(); - MIA->updateState(Inst, Index); - } else if (!Disassembled && MIA) { - MIA->resetState(); + if (MIA) { + if (Disassembled) { + uint64_t Target; + bool TargetKnown = MIA->evaluateBranch(Inst, Index, Size, Target); + if (TargetKnown && (Target >= Start && Target < End) && + !Labels.count(Target)) { + // On PowerPC and AIX, a function call is encoded as a branch to 0. + // On other PowerPC platforms (ELF), a function call is encoded as + // a branch to self. Do not add a label for these cases. + if (!(isPPC && + ((Target == 0 && isXCOFF) || (Target == Index && !isXCOFF)))) + Labels[Target] = ("L" + Twine(LabelCount++)).str(); + } + MIA->updateState(Inst, Index); + } else + MIA->resetState(); } Index += Size; } @@ -1486,7 +1493,7 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj, } std::map<SectionRef, std::vector<RelocationRef>> RelocMap; - if (InlineRelocs) + if (InlineRelocs || Obj.isXCOFF()) RelocMap = getRelocsMap(Obj); bool Is64Bits = Obj.getBytesInAddress() > 4; @@ -1985,6 +1992,8 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj, DT->InstrAnalysis->resetState(); while (Index < End) { + uint64_t RelOffset; + // ARM and AArch64 ELF binaries can interleave data and text in the // same section. We rely on the markers introduced to understand what // we need to dump. If the data marker is within a function, it is @@ -2019,6 +2028,26 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj, } } + auto findRel = [&]() { + while (RelCur != RelEnd) { + RelOffset = RelCur->getOffset() - RelAdjustment; + // If this relocation is hidden, skip it. + if (getHidden(*RelCur) || SectionAddr + RelOffset < StartAddress) { + ++RelCur; + continue; + } + + // Stop when RelCur's offset is past the disassembled + // instruction/data. + if (RelOffset >= Index + Size) + return false; + if (RelOffset >= Index) + return true; + ++RelCur; + } + return false; + }; + if (DumpARMELFData) { Size = dumpARMELFData(SectionAddr, Index, End, Obj, Bytes, MappingSymbols, *DT->SubtargetInfo, FOS); @@ -2029,7 +2058,7 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj, uint64_t MaxOffset = End - Index; // For --reloc: print zero blocks patched by relocations, so that // relocations can be shown in the dump. - if (RelCur != RelEnd) + if (InlineRelocs && RelCur != RelEnd) MaxOffset = std::min(RelCur->getOffset() - RelAdjustment - Index, MaxOffset); @@ -2087,17 +2116,19 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj, DT->InstPrinter->setCommentStream(llvm::nulls()); - // If disassembly has failed, avoid analysing invalid/incomplete - // instruction information. Otherwise, try to resolve the target - // address (jump target or memory operand address) and print it on the + // If disassembly succeeds, we try to resolve the target address + // (jump target or memory operand address) and print it to the // right of the instruction. + // + // Otherwise, we don't print anything else so that we avoid + // analyzing invalid or incomplete instruction information. if (Disassembled && DT->InstrAnalysis) { - // Branch targets are printed just after the instructions. llvm::raw_ostream *TargetOS = &FOS; uint64_t Target; bool PrintTarget = DT->InstrAnalysis->evaluateBranch( Inst, SectionAddr + Index, Size, Target); - if (!PrintTarget) + + if (!PrintTarget) { if (std::optional<uint64_t> MaybeTarget = DT->InstrAnalysis->evaluateMemoryOperandAddress( Inst, DT->SubtargetInfo.get(), SectionAddr + Index, @@ -2111,6 +2142,8 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj, *TargetOS << "0x" << Twine::utohexstr(Target); } } + } + if (PrintTarget) { // In a relocatable object, the target's section must reside in // the same section as the call instruction or it is accessed @@ -2120,7 +2153,8 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj, // In that case, locate the section(s) containing the target // address and find the symbol in one of those, if possible. // - // N.B. We don't walk the relocations in the relocatable case yet. + // N.B. Except for XCOFF, we don't walk the relocations in the + // relocatable case yet. std::vector<const SectionSymbolsTy *> TargetSectionSymbols; if (!Obj.isRelocatableObject()) { auto It = llvm::partition_point( @@ -2166,19 +2200,65 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj, break; } + // Branch targets are printed just after the instructions. // Print the labels corresponding to the target if there's any. bool BBAddrMapLabelAvailable = BBAddrMapLabels.count(Target); bool LabelAvailable = AllLabels.count(Target); + if (TargetSym != nullptr) { uint64_t TargetAddress = TargetSym->Addr; uint64_t Disp = Target - TargetAddress; std::string TargetName = Demangle ? demangle(TargetSym->Name) : TargetSym->Name.str(); + bool RelFixedUp = false; + SmallString<32> Val; *TargetOS << " <"; - if (!Disp) { - // Always Print the binary symbol precisely corresponding to - // the target address. + // On XCOFF, we use relocations, even without -r, so we + // can print the correct name for an extern function call. + if (Obj.isXCOFF() && findRel()) { + // Check for possible branch relocations and + // branches to fixup code. + bool BranchRelocationType = true; + XCOFF::RelocationType RelocType; + if (Obj.is64Bit()) { + const XCOFFRelocation64 *Reloc = + reinterpret_cast<XCOFFRelocation64 *>( + RelCur->getRawDataRefImpl().p); + RelFixedUp = Reloc->isFixupIndicated(); + RelocType = Reloc->Type; + } else { + const XCOFFRelocation32 *Reloc = + reinterpret_cast<XCOFFRelocation32 *>( + RelCur->getRawDataRefImpl().p); + RelFixedUp = Reloc->isFixupIndicated(); + RelocType = Reloc->Type; + } + BranchRelocationType = + RelocType == XCOFF::R_BA || RelocType == XCOFF::R_BR || + RelocType == XCOFF::R_RBA || RelocType == XCOFF::R_RBR; + + // If we have a valid relocation, try to print its + // corresponding symbol name. Multiple relocations on the + // same instruction are not handled. + // Branches to fixup code will have the RelFixedUp flag set in + // the RLD. For these instructions, we print the correct + // branch target, but print the referenced symbol as a + // comment. + if (Error E = getRelocationValueString(*RelCur, false, Val)) { + // If -r was used, this error will be printed later. + // Otherwise, we ignore the error and print what + // would have been printed without using relocations. + consumeError(std::move(E)); + *TargetOS << TargetName; + RelFixedUp = false; // Suppress comment for RLD sym name + } else if (BranchRelocationType && !RelFixedUp) + *TargetOS << Val; + else + *TargetOS << TargetName; + if (Disp) + *TargetOS << "+0x" << Twine::utohexstr(Disp); + } else if (!Disp) { *TargetOS << TargetName; } else if (BBAddrMapLabelAvailable) { *TargetOS << BBAddrMapLabels[Target].front(); @@ -2190,6 +2270,12 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj, *TargetOS << TargetName << "+0x" << Twine::utohexstr(Disp); } *TargetOS << ">"; + if (RelFixedUp && !InlineRelocs) { + // We have fixup code for a relocation. We print the + // referenced symbol as a comment. + *TargetOS << "\t# " << Val; + } + } else if (BBAddrMapLabelAvailable) { *TargetOS << " <" << BBAddrMapLabels[Target].front() << ">"; } else if (LabelAvailable) { @@ -2215,36 +2301,20 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj, if (BTF) printBTFRelocation(FOS, *BTF, {Index, Section.getIndex()}, LVP); - // Hexagon does this in pretty printer - if (Obj.getArch() != Triple::hexagon) { - // Print relocation for instruction and data. - while (RelCur != RelEnd) { - uint64_t Offset = RelCur->getOffset() - RelAdjustment; - // If this relocation is hidden, skip it. - if (getHidden(*RelCur)) { - ++RelCur; - continue; - } - - // Stop when RelCur's offset is past the disassembled - // instruction/data. Note that it's possible the disassembled data - // is not the complete data: we might see the relocation printed in - // the middle of the data, but this matches the binutils objdump - // output. - if (Offset >= Index + Size) - break; - + // Hexagon handles relocs in pretty printer + if (InlineRelocs && Obj.getArch() != Triple::hexagon) { + while (findRel()) { // When --adjust-vma is used, update the address printed. if (RelCur->getSymbol() != Obj.symbol_end()) { Expected<section_iterator> SymSI = RelCur->getSymbol()->getSection(); if (SymSI && *SymSI != Obj.section_end() && shouldAdjustVA(**SymSI)) - Offset += AdjustVMA; + RelOffset += AdjustVMA; } printRelocation(FOS, Obj.getFileName(), *RelCur, - SectionAddr + Offset, Is64Bits); + SectionAddr + RelOffset, Is64Bits); LVP.printAfterOtherLine(FOS, true); ++RelCur; } @@ -2428,7 +2498,8 @@ void Dumper::printRelocations() { if (Address < StartAddress || Address > StopAddress || getHidden(Reloc)) continue; Reloc.getTypeName(RelocName); - if (Error E = getRelocationValueString(Reloc, ValueStr)) + if (Error E = + getRelocationValueString(Reloc, SymbolDescription, ValueStr)) reportUniqueWarning(std::move(E)); outs() << format(Fmt.data(), Address) << " " diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 322b7da2678f..12b81d411cfa 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -1651,10 +1651,10 @@ struct SampleOverlapStats { namespace { struct FuncSampleStats { - uint64_t SampleSum; - uint64_t MaxSample; - uint64_t HotBlockCount; - FuncSampleStats() : SampleSum(0), MaxSample(0), HotBlockCount(0) {} + uint64_t SampleSum = 0; + uint64_t MaxSample = 0; + uint64_t HotBlockCount = 0; + FuncSampleStats() = default; FuncSampleStats(uint64_t SampleSum, uint64_t MaxSample, uint64_t HotBlockCount) : SampleSum(SampleSum), MaxSample(MaxSample), @@ -2563,12 +2563,10 @@ static int overlap_main(int argc, const char *argv[]) { namespace { struct ValueSitesStats { - ValueSitesStats() - : TotalNumValueSites(0), TotalNumValueSitesWithValueProfile(0), - TotalNumValues(0) {} - uint64_t TotalNumValueSites; - uint64_t TotalNumValueSitesWithValueProfile; - uint64_t TotalNumValues; + ValueSitesStats() = default; + uint64_t TotalNumValueSites = 0; + uint64_t TotalNumValueSitesWithValueProfile = 0; + uint64_t TotalNumValues = 0; std::vector<unsigned> ValueSitesHistogram; }; } // namespace @@ -2867,13 +2865,12 @@ static void showSectionInfo(sampleprof::SampleProfileReader *Reader, namespace { struct HotFuncInfo { std::string FuncName; - uint64_t TotalCount; - double TotalCountPercent; - uint64_t MaxCount; - uint64_t EntryCount; + uint64_t TotalCount = 0; + double TotalCountPercent = 0.0f; + uint64_t MaxCount = 0; + uint64_t EntryCount = 0; - HotFuncInfo() - : TotalCount(0), TotalCountPercent(0.0f), MaxCount(0), EntryCount(0) {} + HotFuncInfo() = default; HotFuncInfo(StringRef FN, uint64_t TS, double TSP, uint64_t MS, uint64_t ES) : FuncName(FN.begin(), FN.end()), TotalCount(TS), TotalCountPercent(TSP), diff --git a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp index 292b5cade950..34b06fe480f3 100644 --- a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp +++ b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp @@ -621,3 +621,38 @@ TEST_F(TargetLibraryInfoTest, ValidProto) { EXPECT_TRUE(isLibFunc(F, LF)); } } + +namespace { + +/// Creates TLI for AArch64 and uses it to get the LibFunc names for the given +/// Instruction opcode and Type. +class TLITestAarch64 : public ::testing::Test { +private: + const Triple TargetTriple; + +protected: + LLVMContext Ctx; + std::unique_ptr<TargetLibraryInfoImpl> TLII; + std::unique_ptr<TargetLibraryInfo> TLI; + + /// Create TLI for AArch64 + TLITestAarch64() : TargetTriple(Triple("aarch64-unknown-linux-gnu")) { + TLII = std::make_unique<TargetLibraryInfoImpl>( + TargetLibraryInfoImpl(TargetTriple)); + TLI = std::make_unique<TargetLibraryInfo>(TargetLibraryInfo(*TLII)); + } + + /// Returns the TLI function name for the given \p Opcode and type \p Ty. + StringRef getScalarName(unsigned int Opcode, Type *Ty) { + LibFunc Func; + if (!TLI->getLibFunc(Opcode, Ty, Func)) + return ""; + return TLI->getName(Func); + } +}; +} // end anonymous namespace + +TEST_F(TLITestAarch64, TestFrem) { + EXPECT_EQ(getScalarName(Instruction::FRem, Type::getDoubleTy(Ctx)), "fmod"); + EXPECT_EQ(getScalarName(Instruction::FRem, Type::getFloatTy(Ctx)), "fmodf"); +}
\ No newline at end of file diff --git a/llvm/unittests/Analysis/ValueTrackingTest.cpp b/llvm/unittests/Analysis/ValueTrackingTest.cpp index 0d3a594da0c0..27f631884072 100644 --- a/llvm/unittests/Analysis/ValueTrackingTest.cpp +++ b/llvm/unittests/Analysis/ValueTrackingTest.cpp @@ -1177,12 +1177,12 @@ TEST(ValueTracking, canCreatePoisonOrUndef) { {{false, false}, "shufflevector <4 x i32> %vx, <4 x i32> %vx2, " "<4 x i32> <i32 0, i32 1, i32 2, i32 3>"}, - {{false, true}, + {{true, false}, "shufflevector <4 x i32> %vx, <4 x i32> %vx2, " - "<4 x i32> <i32 0, i32 1, i32 2, i32 undef>"}, - {{false, true}, + "<4 x i32> <i32 0, i32 1, i32 2, i32 poison>"}, + {{true, false}, "shufflevector <vscale x 4 x i32> %svx, " - "<vscale x 4 x i32> %svx, <vscale x 4 x i32> undef"}, + "<vscale x 4 x i32> %svx, <vscale x 4 x i32> poison"}, {{true, false}, "call i32 @g(i32 %x)"}, {{false, false}, "call noundef i32 @g(i32 %x)"}, {{true, false}, "fcmp nnan oeq float %fx, %fy"}, diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp index 30e60ad92b68..866176ab0983 100644 --- a/llvm/unittests/TargetParser/TargetParserTest.cpp +++ b/llvm/unittests/TargetParser/TargetParserTest.cpp @@ -1812,7 +1812,8 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) { AArch64::AEK_SSVE_FP8DOT4, AArch64::AEK_LUT, AArch64::AEK_SME_LUTv2, AArch64::AEK_SMEF8F16, AArch64::AEK_SMEF8F32, AArch64::AEK_SMEFA64, - AArch64::AEK_CPA}; + AArch64::AEK_CPA, AArch64::AEK_PAUTHLR, + }; std::vector<StringRef> Features; @@ -1899,6 +1900,7 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) { EXPECT_TRUE(llvm::is_contained(Features, "+sme-f8f32")); EXPECT_TRUE(llvm::is_contained(Features, "+sme-fa64")); EXPECT_TRUE(llvm::is_contained(Features, "+cpa")); + EXPECT_TRUE(llvm::is_contained(Features, "+pauth-lr")); // Assuming we listed every extension above, this should produce the same // result. (note that AEK_NONE doesn't have a name so it won't be in the diff --git a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn index 3debc48a4bb9..6059074dfa27 100644 --- a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn @@ -120,6 +120,7 @@ copy("Headers") { "__stddef_wint_t.h", "__wmmintrin_aes.h", "__wmmintrin_pclmul.h", + "adcintrin.h", "adxintrin.h", "altivec.h", "ammintrin.h", diff --git a/llvm/utils/gn/secondary/llvm/lib/TextAPI/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/TextAPI/BUILD.gn index f1c604aa420b..ed312b248656 100644 --- a/llvm/utils/gn/secondary/llvm/lib/TextAPI/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/TextAPI/BUILD.gn @@ -20,5 +20,6 @@ static_library("TextAPI") { "TextStub.cpp", "TextStubCommon.cpp", "TextStubV5.cpp", + "Utils.cpp", ] } diff --git a/llvm/utils/gn/secondary/llvm/lib/TextAPI/BinaryReader/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/TextAPI/BinaryReader/BUILD.gn new file mode 100644 index 000000000000..3eeb32aae4e3 --- /dev/null +++ b/llvm/utils/gn/secondary/llvm/lib/TextAPI/BinaryReader/BUILD.gn @@ -0,0 +1,10 @@ +static_library("BinaryReader") { + output_name = "LLVMTextAPIBinaryReader" + deps = [ + "//llvm/lib/Object", + "//llvm/lib/Support", + "//llvm/lib/TargetParser", + "//llvm/lib/TextAPI", + ] + sources = [ "DylibReader.cpp" ] +} diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-readtapi/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-readtapi/BUILD.gn index a562d04c4e90..1e67b1dcf036 100644 --- a/llvm/utils/gn/secondary/llvm/tools/llvm-readtapi/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/tools/llvm-readtapi/BUILD.gn @@ -26,10 +26,12 @@ group("symlinks") { executable("llvm-readtapi") { deps = [ ":TapiOpts", + "//llvm/lib/BinaryFormat", "//llvm/lib/Object", "//llvm/lib/Option", "//llvm/lib/Support", "//llvm/lib/TextAPI", + "//llvm/lib/TextAPI/BinaryReader", ] sources = [ "DiffEngine.cpp", diff --git a/mlir/include/mlir/Analysis/DataFlowFramework.h b/mlir/include/mlir/Analysis/DataFlowFramework.h index 541cdb1e237c..c76cfac07fc7 100644 --- a/mlir/include/mlir/Analysis/DataFlowFramework.h +++ b/mlir/include/mlir/Analysis/DataFlowFramework.h @@ -30,8 +30,8 @@ namespace mlir { //===----------------------------------------------------------------------===// /// A result type used to indicate if a change happened. Boolean operations on -/// ChangeResult behave as though `Change` is truthy. -enum class ChangeResult { +/// ChangeResult behave as though `Change` is truth. +enum class [[nodiscard]] ChangeResult { NoChange, Change, }; diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td index c72fde2ab351..b536b6c97cef 100644 --- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td +++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td @@ -19,10 +19,11 @@ include "mlir/Dialect/GPU/IR/CompilationAttrInterfaces.td" include "mlir/Dialect/GPU/IR/CompilationAttrs.td" include "mlir/Dialect/GPU/IR/ParallelLoopMapperAttr.td" include "mlir/Dialect/GPU/TransformOps/GPUDeviceMappingAttr.td" +include "mlir/IR/CommonTypeConstraints.td" include "mlir/IR/EnumAttr.td" -include "mlir/Interfaces/FunctionInterfaces.td" include "mlir/IR/SymbolInterfaces.td" include "mlir/Interfaces/DataLayoutInterfaces.td" +include "mlir/Interfaces/FunctionInterfaces.td" include "mlir/Interfaces/InferIntRangeInterface.td" include "mlir/Interfaces/InferTypeOpInterface.td" include "mlir/Interfaces/SideEffectInterfaces.td" @@ -1023,16 +1024,23 @@ def GPU_AllReduceOp : GPU_Op<"all_reduce", let hasRegionVerifier = 1; } +def AnyIntegerOrFloatOr1DVector : + AnyTypeOf<[AnyIntegerOrFloat, VectorOfRankAndType<[1], [AnyIntegerOrFloat]>]>; + def GPU_SubgroupReduceOp : GPU_Op<"subgroup_reduce", [SameOperandsAndResultType]> { let summary = "Reduce values among subgroup."; let description = [{ The `subgroup_reduce` op reduces the value of every work item across a subgroup. The result is equal for all work items of a subgroup. + When the reduced value is of a vector type, each vector element is reduced + independently. Only 1-d vector types are allowed. + Example: ```mlir - %1 = gpu.subgroup_reduce add %0 : (f32) -> (f32) + %1 = gpu.subgroup_reduce add %a : (f32) -> (f32) + %2 = gpu.subgroup_reduce add %b : (vector<4xf16>) -> (vector<4xf16>) ``` If `uniform` flag is set either none or all work items of a subgroup @@ -1045,11 +1053,11 @@ def GPU_SubgroupReduceOp : GPU_Op<"subgroup_reduce", [SameOperandsAndResultType] }]; let arguments = (ins - AnyIntegerOrFloat:$value, + AnyIntegerOrFloatOr1DVector:$value, GPU_AllReduceOperationAttr:$op, UnitAttr:$uniform ); - let results = (outs AnyIntegerOrFloat:$result); + let results = (outs AnyIntegerOrFloatOr1DVector:$result); let assemblyFormat = [{ custom<AllReduceOperation>($op) $value (`uniform` $uniform^)? attr-dict diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVArithmeticOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVArithmeticOps.td index 51124e141c6d..22d5afcd7738 100644 --- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVArithmeticOps.td +++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVArithmeticOps.td @@ -582,6 +582,8 @@ def SPIRV_SNegateOp : SPIRV_ArithmeticUnaryOp<"SNegate", %3 = spirv.SNegate %2 : vector<4xi32> ``` }]; + + let hasFolder = 1; } // ----- diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBitOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBitOps.td index b460c8e68aa0..38639a175ab4 100644 --- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBitOps.td +++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBitOps.td @@ -462,6 +462,8 @@ def SPIRV_NotOp : SPIRV_BitUnaryOp<"Not", [UsableInSpecConstantOp]> { %3 = spirv.Not %1 : vector<4xi32> ``` }]; + + let hasFolder = 1; } #endif // MLIR_DIALECT_SPIRV_IR_BIT_OPS diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td index 2e26c44de281..e48a56f0625d 100644 --- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td +++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td @@ -534,6 +534,7 @@ def SPIRV_LogicalNotOp : SPIRV_LogicalUnaryOp<"LogicalNot", }]; let hasCanonicalizer = 1; + let hasFolder = 1; } // ----- diff --git a/mlir/include/mlir/Transforms/FoldUtils.h b/mlir/include/mlir/Transforms/FoldUtils.h index 2600da361496..2e7a6fe3e362 100644 --- a/mlir/include/mlir/Transforms/FoldUtils.h +++ b/mlir/include/mlir/Transforms/FoldUtils.h @@ -33,7 +33,8 @@ class Value; class OperationFolder { public: OperationFolder(MLIRContext *ctx, OpBuilder::Listener *listener = nullptr) - : interfaces(ctx), rewriter(ctx, listener) {} + : erasedFoldedLocation(UnknownLoc::get(ctx)), interfaces(ctx), + rewriter(ctx, listener) {} /// Tries to perform folding on the given `op`, including unifying /// deduplicated constants. If successful, replaces `op`'s uses with @@ -65,7 +66,7 @@ public: /// be created in a parent block. On success this returns the constant /// operation, nullptr otherwise. Value getOrCreateConstant(Block *block, Dialect *dialect, Attribute value, - Type type, Location loc); + Type type); private: /// This map keeps track of uniqued constants by dialect, attribute, and type. @@ -95,6 +96,9 @@ private: Dialect *dialect, Attribute value, Type type, Location loc); + /// The location to overwrite with for folder-owned constants. + UnknownLoc erasedFoldedLocation; + /// A mapping between an insertion region and the constants that have been /// created within it. DenseMap<Region *, ConstantMap> foldScopes; diff --git a/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp b/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp index 2820d27b65f7..7875fa9d43d9 100644 --- a/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp +++ b/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp @@ -191,7 +191,7 @@ void LivenessAnalysis::visitCallOperand(OpOperand &operand) { void LivenessAnalysis::setToExitState(Liveness *lattice) { // This marks values of type (2) liveness as "live". - lattice->markLive(); + (void)lattice->markLive(); } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Bindings/Python/IRCore.cpp b/mlir/lib/Bindings/Python/IRCore.cpp index 39757dfad5be..5412c3dec4b1 100644 --- a/mlir/lib/Bindings/Python/IRCore.cpp +++ b/mlir/lib/Bindings/Python/IRCore.cpp @@ -602,7 +602,7 @@ py::object PyMlirContext::createFromCapsule(py::object capsule) { MlirContext rawContext = mlirPythonCapsuleToContext(capsule.ptr()); if (mlirContextIsNull(rawContext)) throw py::error_already_set(); - return stealExternalContext(rawContext).releaseObject(); + return forContext(rawContext).releaseObject(); } PyMlirContext *PyMlirContext::createNewContextForInit() { @@ -615,35 +615,18 @@ PyMlirContextRef PyMlirContext::forContext(MlirContext context) { auto &liveContexts = getLiveContexts(); auto it = liveContexts.find(context.ptr); if (it == liveContexts.end()) { - throw std::runtime_error( - "Cannot use a context that is not owned by the Python bindings."); + // Create. + PyMlirContext *unownedContextWrapper = new PyMlirContext(context); + py::object pyRef = py::cast(unownedContextWrapper); + assert(pyRef && "cast to py::object failed"); + liveContexts[context.ptr] = unownedContextWrapper; + return PyMlirContextRef(unownedContextWrapper, std::move(pyRef)); } - // Use existing. py::object pyRef = py::cast(it->second); return PyMlirContextRef(it->second, std::move(pyRef)); } -PyMlirContextRef PyMlirContext::stealExternalContext(MlirContext context) { - py::gil_scoped_acquire acquire; - auto &liveContexts = getLiveContexts(); - auto it = liveContexts.find(context.ptr); - if (it != liveContexts.end()) { - throw std::runtime_error( - "Cannot transfer ownership of the context to Python " - "as it is already owned by Python."); - } - - PyMlirContext *unownedContextWrapper = new PyMlirContext(context); - // Note that the default return value policy on cast is automatic_reference, - // which does not take ownership (delete will not be called). - // Just be explicit. - py::object pyRef = - py::cast(unownedContextWrapper, py::return_value_policy::take_ownership); - assert(pyRef && "cast to py::object failed"); - return PyMlirContextRef(unownedContextWrapper, std::move(pyRef)); -} - PyMlirContext::LiveContextMap &PyMlirContext::getLiveContexts() { static LiveContextMap liveContexts; return liveContexts; @@ -1162,18 +1145,6 @@ PyOperationRef PyOperation::forOperation(PyMlirContextRef contextRef, return PyOperationRef(existing, std::move(pyRef)); } -PyOperationRef PyOperation::stealExternalOperation(PyMlirContextRef contextRef, - MlirOperation operation) { - auto &liveOperations = contextRef->liveOperations; - auto it = liveOperations.find(operation.ptr); - if (it != liveOperations.end()) { - throw std::runtime_error( - "Cannot transfer ownership of the operation to Python " - "as it is already owned by Python."); - } - return createInstance(std::move(contextRef), operation, py::none()); -} - PyOperationRef PyOperation::createDetached(PyMlirContextRef contextRef, MlirOperation operation, py::object parentKeepAlive) { @@ -1345,8 +1316,7 @@ py::object PyOperation::createFromCapsule(py::object capsule) { if (mlirOperationIsNull(rawOperation)) throw py::error_already_set(); MlirContext rawCtxt = mlirOperationGetContext(rawOperation); - return stealExternalOperation(PyMlirContext::forContext(rawCtxt), - rawOperation) + return forOperation(PyMlirContext::forContext(rawCtxt), rawOperation) .releaseObject(); } @@ -2578,16 +2548,6 @@ void mlir::python::populateIRCore(py::module &m) { .def("_get_live_operation_count", &PyMlirContext::getLiveOperationCount) .def("_clear_live_operations", &PyMlirContext::clearLiveOperations) .def("_get_live_module_count", &PyMlirContext::getLiveModuleCount) - .def_static("_testing_create_raw_context_capsule", - []() { - // Creates an MlirContext not known to the Python bindings - // and puts it in a capsule. Used to test interop. Using - // this without passing it back to the capsule creation - // API will leak. - return py::reinterpret_steal<py::object>( - mlirPythonContextToCapsule( - mlirContextCreateWithThreading(false))); - }) .def_property_readonly(MLIR_PYTHON_CAPI_PTR_ATTR, &PyMlirContext::getCapsule) .def(MLIR_PYTHON_CAPI_FACTORY_ATTR, &PyMlirContext::createFromCapsule) @@ -3013,7 +2973,8 @@ void mlir::python::populateIRCore(py::module &m) { py::arg("binary") = false, kOperationPrintStateDocstring) .def("print", py::overload_cast<std::optional<int64_t>, bool, bool, bool, bool, - bool, py::object, bool>(&PyOperationBase::print), + bool, py::object, bool>( + &PyOperationBase::print), // Careful: Lots of arguments must match up with print method. py::arg("large_elements_limit") = py::none(), py::arg("enable_debug_info") = false, @@ -3085,25 +3046,6 @@ void mlir::python::populateIRCore(py::module &m) { .def_property_readonly(MLIR_PYTHON_CAPI_PTR_ATTR, &PyOperation::getCapsule) .def(MLIR_PYTHON_CAPI_FACTORY_ATTR, &PyOperation::createFromCapsule) - .def_static( - "_testing_create_raw_capsule", - [](std::string sourceStr) { - // Creates a raw context and an operation via parsing the given - // source and returns them in a capsule. Error handling is - // minimal as this is purely intended for testing interop with - // operation creation from capsule functions. - MlirContext context = mlirContextCreateWithThreading(false); - MlirOperation op = mlirOperationCreateParse( - context, toMlirStringRef(sourceStr), toMlirStringRef("temp")); - if (mlirOperationIsNull(op)) { - mlirContextDestroy(context); - throw std::invalid_argument("Failed to parse"); - } - return py::make_tuple(py::reinterpret_steal<py::object>( - mlirPythonContextToCapsule(context)), - py::reinterpret_steal<py::object>( - mlirPythonOperationToCapsule(op))); - }) .def_property_readonly("operation", [](py::object self) { return self; }) .def_property_readonly("opview", &PyOperation::createOpView) .def_property_readonly( diff --git a/mlir/lib/Bindings/Python/IRModule.h b/mlir/lib/Bindings/Python/IRModule.h index 04164b78b3e2..79b7e0c96188 100644 --- a/mlir/lib/Bindings/Python/IRModule.h +++ b/mlir/lib/Bindings/Python/IRModule.h @@ -176,19 +176,8 @@ public: static PyMlirContext *createNewContextForInit(); /// Returns a context reference for the singleton PyMlirContext wrapper for - /// the given context. It is only valid to call this on an MlirContext that - /// is already owned by the Python bindings. Typically this will be because - /// it came in some fashion from createNewContextForInit(). However, it - /// is also possible to explicitly transfer ownership of an existing - /// MlirContext to the Python bindings via stealExternalContext(). + /// the given context. static PyMlirContextRef forContext(MlirContext context); - - /// Explicitly takes ownership of an MlirContext that must not already be - /// known to the Python bindings. Once done, the life-cycle of the context - /// will be controlled by the Python bindings, and it will be destroyed - /// when the reference count goes to zero. - static PyMlirContextRef stealExternalContext(MlirContext context); - ~PyMlirContext(); /// Accesses the underlying MlirContext. @@ -617,12 +606,6 @@ public: forOperation(PyMlirContextRef contextRef, MlirOperation operation, pybind11::object parentKeepAlive = pybind11::object()); - /// Explicitly takes ownership of an operation that must not already be known - /// to the Python bindings. Once done, the life-cycle of the operation - /// will be controlled by the Python bindings. - static PyOperationRef stealExternalOperation(PyMlirContextRef contextRef, - MlirOperation operation); - /// Creates a detached operation. The operation must not be associated with /// any existing live operation. static PyOperationRef diff --git a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp index f9d6f04a811f..0c6e2e80b88a 100644 --- a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp +++ b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp @@ -32,6 +32,95 @@ using namespace mlir; namespace { +/// Helper to create an arm_sme.intr.ld1*.(horiz|vert)' intrinsic. +static Operation *createLoadTileSliceIntrinsic( + RewriterBase &rewriter, Location loc, arm_sme::ArmSMETileType type, + arm_sme::TileSliceLayout layout, Value maskOp, Value ptr, + IntegerAttr tileId, Value tileSliceI32) { + if (layout == arm_sme::TileSliceLayout::Horizontal) { + switch (type) { + case arm_sme::ArmSMETileType::ZAB: + return rewriter.create<arm_sme::aarch64_sme_ld1b_horiz>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAH: + return rewriter.create<arm_sme::aarch64_sme_ld1h_horiz>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAS: + return rewriter.create<arm_sme::aarch64_sme_ld1w_horiz>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAD: + return rewriter.create<arm_sme::aarch64_sme_ld1d_horiz>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAQ: + return rewriter.create<arm_sme::aarch64_sme_ld1q_horiz>( + loc, maskOp, ptr, tileId, tileSliceI32); + } + } else { + switch (type) { + case arm_sme::ArmSMETileType::ZAB: + return rewriter.create<arm_sme::aarch64_sme_ld1b_vert>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAH: + return rewriter.create<arm_sme::aarch64_sme_ld1h_vert>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAS: + return rewriter.create<arm_sme::aarch64_sme_ld1w_vert>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAD: + return rewriter.create<arm_sme::aarch64_sme_ld1d_vert>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAQ: + return rewriter.create<arm_sme::aarch64_sme_ld1q_vert>( + loc, maskOp, ptr, tileId, tileSliceI32); + break; + } + } +} + +/// Helper to create an arm_sme.intr.st1*.(horiz|vert)' intrinsic. +static Operation *createStoreTileSliceIntrinsic( + RewriterBase &rewriter, Location loc, arm_sme::ArmSMETileType type, + arm_sme::TileSliceLayout layout, Value maskOp, Value ptr, + IntegerAttr tileId, Value tileSliceI32) { + if (layout == arm_sme::TileSliceLayout::Horizontal) { + switch (type) { + case arm_sme::ArmSMETileType::ZAB: + return rewriter.create<arm_sme::aarch64_sme_st1b_horiz>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAH: + return rewriter.create<arm_sme::aarch64_sme_st1h_horiz>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAS: + return rewriter.create<arm_sme::aarch64_sme_st1w_horiz>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAD: + return rewriter.create<arm_sme::aarch64_sme_st1d_horiz>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAQ: + return rewriter.create<arm_sme::aarch64_sme_st1q_horiz>( + loc, maskOp, ptr, tileId, tileSliceI32); + } + } else { + switch (type) { + case arm_sme::ArmSMETileType::ZAB: + return rewriter.create<arm_sme::aarch64_sme_st1b_vert>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAH: + return rewriter.create<arm_sme::aarch64_sme_st1h_vert>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAS: + return rewriter.create<arm_sme::aarch64_sme_st1w_vert>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAD: + return rewriter.create<arm_sme::aarch64_sme_st1d_vert>( + loc, maskOp, ptr, tileId, tileSliceI32); + case arm_sme::ArmSMETileType::ZAQ: + return rewriter.create<arm_sme::aarch64_sme_st1q_vert>( + loc, maskOp, ptr, tileId, tileSliceI32); + } + } +} + IntegerAttr getTileIdOrError(arm_sme::ArmSMETileOpInterface op) { auto tileId = op.getTileId(); if (!tileId) @@ -75,9 +164,6 @@ struct ZeroOpConversion : public ConvertOpToLLVMPattern<arm_sme::ZeroOp> { ConversionPatternRewriter &rewriter) const override { auto loc = zero.getLoc(); - unsigned tileElementWidth = - zero.getVectorType().getElementType().getIntOrFloatBitWidth(); - auto tileId = getTileIdOrError(zero); if (!tileId) return failure(); @@ -86,23 +172,24 @@ struct ZeroOpConversion : public ConvertOpToLLVMPattern<arm_sme::ZeroOp> { // The base mask is just the mask to zero the first tile (of a size). // These masks are derived from: // https://developer.arm.com/documentation/ddi0602/2022-06/SME-Instructions/ZERO--Zero-a-list-of-64-bit-element-ZA-tiles- + arm_sme::ArmSMETileType tileType = *zero.getAllocatedTileType(); auto baseMaskForSize = [&] { - switch (tileElementWidth) { - case 8: + switch (tileType) { + case arm_sme::ArmSMETileType::ZAB: // Zeroing the 8-bit ZA0.B tile is equivalent to zeroing all eight // 64-bit element tiles named ZA0.D to ZA7.D. return 0b1111'1111; - case 16: - // Zeroing the 16-bit ZA0.H tile is equivalent to zeroing 64-bit element - // tiles named ZA0.D, ZA2.D, ZA4.D, and ZA6.D. - // Shift this left once for ZA1.H. + case arm_sme::ArmSMETileType::ZAH: + // Zeroing the 16-bit ZA0.H tile is equivalent to zeroing 64-bit + // element tiles named ZA0.D, ZA2.D, ZA4.D, and ZA6.D. Shift this left + // once for ZA1.H. return 0b0101'0101; - case 32: + case arm_sme::ArmSMETileType::ZAS: // Zeroing the 32-bit ZA0.S tile is equivalent to zeroing 64-bit // element tiles named ZA0.D and ZA4.D. // Shift left by 1, 2, or 3 respectively for ZA1.S, ZA2.S, ZA3.S. return 0b0001'0001; - case 64: + case arm_sme::ArmSMETileType::ZAD: // Zeroing one of the a 64-bit tiles ZA0.D to ZA7.D just requires // setting the bit for that tile. return 0b0000'0001; @@ -172,63 +259,13 @@ struct LoadTileSliceConversion // Create all active predicate mask. auto maskOp = loadTileSliceOp.getMask(); - auto tileType = loadTileSliceOp.getVectorType(); - auto tileElementType = tileType.getElementType(); - unsigned tileElementWidth = tileElementType.getIntOrFloatBitWidth(); + auto tileVectorType = loadTileSliceOp.getVectorType(); + arm_sme::ArmSMETileType tileType = *arm_sme::getSMETileType(tileVectorType); arm_sme::TileSliceLayout layout = loadTileSliceOp.getLayout(); // Create 'arm_sme.intr.ld1*.(horiz|vert)' intrinsic to load ZA tile slice. - if (layout == arm_sme::TileSliceLayout::Horizontal) { - switch (tileElementWidth) { - default: - llvm_unreachable("unexpected element type!"); - case 8: - rewriter.create<arm_sme::aarch64_sme_ld1b_horiz>(loc, maskOp, ptr, - tileId, tileSliceI32); - break; - case 16: - rewriter.create<arm_sme::aarch64_sme_ld1h_horiz>(loc, maskOp, ptr, - tileId, tileSliceI32); - break; - case 32: - rewriter.create<arm_sme::aarch64_sme_ld1w_horiz>(loc, maskOp, ptr, - tileId, tileSliceI32); - break; - case 64: - rewriter.create<arm_sme::aarch64_sme_ld1d_horiz>(loc, maskOp, ptr, - tileId, tileSliceI32); - break; - case 128: - rewriter.create<arm_sme::aarch64_sme_ld1q_horiz>(loc, maskOp, ptr, - tileId, tileSliceI32); - break; - } - } else { - switch (tileElementWidth) { - default: - llvm_unreachable("unexpected element type!"); - case 8: - rewriter.create<arm_sme::aarch64_sme_ld1b_vert>(loc, maskOp, ptr, - tileId, tileSliceI32); - break; - case 16: - rewriter.create<arm_sme::aarch64_sme_ld1h_vert>(loc, maskOp, ptr, - tileId, tileSliceI32); - break; - case 32: - rewriter.create<arm_sme::aarch64_sme_ld1w_vert>(loc, maskOp, ptr, - tileId, tileSliceI32); - break; - case 64: - rewriter.create<arm_sme::aarch64_sme_ld1d_vert>(loc, maskOp, ptr, - tileId, tileSliceI32); - break; - case 128: - rewriter.create<arm_sme::aarch64_sme_ld1q_vert>(loc, maskOp, ptr, - tileId, tileSliceI32); - break; - } - } + createLoadTileSliceIntrinsic(rewriter, loc, tileType, layout, maskOp, ptr, + tileId, tileSliceI32); // The load intrinsics have no result, replace 'arm_sme.tile_load' with // the input tile to preserve dataflow. @@ -249,9 +286,7 @@ struct StoreTileSliceConversion arm_sme::StoreTileSliceOp::Adaptor adaptor, ConversionPatternRewriter &rewriter) const override { auto loc = storeTileSliceOp.getLoc(); - auto tileType = storeTileSliceOp.getVectorType(); - auto tileElementType = tileType.getElementType(); - unsigned tileElementWidth = tileElementType.getIntOrFloatBitWidth(); + auto tileVectorType = storeTileSliceOp.getVectorType(); auto tileId = getTileIdOrError(storeTileSliceOp); if (!tileId) @@ -271,58 +306,12 @@ struct StoreTileSliceConversion auto maskOp = storeTileSliceOp.getMask(); arm_sme::TileSliceLayout layout = storeTileSliceOp.getLayout(); + arm_sme::ArmSMETileType tileType = *arm_sme::getSMETileType(tileVectorType); - if (layout == arm_sme::TileSliceLayout::Horizontal) { - switch (tileElementWidth) { - default: - llvm_unreachable("unexpected element type!"); - case 8: - rewriter.replaceOpWithNewOp<arm_sme::aarch64_sme_st1b_horiz>( - storeTileSliceOp, maskOp, ptr, tileId, tileSliceI32); - break; - case 16: - rewriter.replaceOpWithNewOp<arm_sme::aarch64_sme_st1h_horiz>( - storeTileSliceOp, maskOp, ptr, tileId, tileSliceI32); - break; - case 32: - rewriter.replaceOpWithNewOp<arm_sme::aarch64_sme_st1w_horiz>( - storeTileSliceOp, maskOp, ptr, tileId, tileSliceI32); - break; - case 64: - rewriter.replaceOpWithNewOp<arm_sme::aarch64_sme_st1d_horiz>( - storeTileSliceOp, maskOp, ptr, tileId, tileSliceI32); - break; - case 128: - rewriter.replaceOpWithNewOp<arm_sme::aarch64_sme_st1q_horiz>( - storeTileSliceOp, maskOp, ptr, tileId, tileSliceI32); - break; - } - } else { - switch (tileElementWidth) { - default: - llvm_unreachable("unexpected element type!"); - case 8: - rewriter.replaceOpWithNewOp<arm_sme::aarch64_sme_st1b_vert>( - storeTileSliceOp, maskOp, ptr, tileId, tileSliceI32); - break; - case 16: - rewriter.replaceOpWithNewOp<arm_sme::aarch64_sme_st1h_vert>( - storeTileSliceOp, maskOp, ptr, tileId, tileSliceI32); - break; - case 32: - rewriter.replaceOpWithNewOp<arm_sme::aarch64_sme_st1w_vert>( - storeTileSliceOp, maskOp, ptr, tileId, tileSliceI32); - break; - case 64: - rewriter.replaceOpWithNewOp<arm_sme::aarch64_sme_st1d_vert>( - storeTileSliceOp, maskOp, ptr, tileId, tileSliceI32); - break; - case 128: - rewriter.replaceOpWithNewOp<arm_sme::aarch64_sme_st1q_vert>( - storeTileSliceOp, maskOp, ptr, tileId, tileSliceI32); - break; - } - } + rewriter.replaceOp(storeTileSliceOp, + createStoreTileSliceIntrinsic(rewriter, loc, tileType, + layout, maskOp, ptr, + tileId, tileSliceI32)); return success(); } diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp index d383c16949f0..d7885e035959 100644 --- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp +++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp @@ -16,10 +16,12 @@ #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h" #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h" #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h" +#include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h" #include "mlir/Dialect/SPIRV/IR/TargetAndABI.h" #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h" #include "mlir/IR/BuiltinOps.h" #include "mlir/IR/Matchers.h" +#include "mlir/Support/LogicalResult.h" #include "mlir/Transforms/DialectConversion.h" #include <optional> @@ -591,10 +593,12 @@ public: LogicalResult matchAndRewrite(gpu::SubgroupReduceOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { - auto opType = op.getOp(); - auto result = - createGroupReduceOp(rewriter, op.getLoc(), adaptor.getValue(), opType, - /*isGroup*/ false, op.getUniform()); + if (!isa<spirv::ScalarType>(adaptor.getValue().getType())) + return rewriter.notifyMatchFailure(op, "reduction type is not a scalar"); + + auto result = createGroupReduceOp(rewriter, op.getLoc(), adaptor.getValue(), + adaptor.getOp(), + /*isGroup=*/false, adaptor.getUniform()); if (!result) return failure(); diff --git a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp index 597846e31e21..8aa51f352f82 100644 --- a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp +++ b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp @@ -223,8 +223,10 @@ struct AssignTileIDsPattern if (failed(tileId)) return tileOp.emitError("ran out of SME virtual tiles!"); - func->setDiscardableAttr(kTilesInUseAttr, - rewriter.getI32IntegerAttr((unsigned)tilesInUse)); + rewriter.updateRootInPlace(func, [&]() { + func->setDiscardableAttr( + kTilesInUseAttr, rewriter.getI32IntegerAttr((unsigned)tilesInUse)); + }); // Find all the ops that (transitively) depend on this tile. SetVector<Operation *> dependantOps; @@ -245,14 +247,15 @@ struct AssignTileIDsPattern // scf.if, and moving the contents of %tileA or %tileB to result tile (based // on the %some_cond). auto tileIDAttr = rewriter.getI32IntegerAttr(*tileId); - tileOp.setTileId(tileIDAttr); + rewriter.updateRootInPlace(tileOp, [&]() { tileOp.setTileId(tileIDAttr); }); for (auto *op : dependantOps) { if (auto tileOp = llvm::dyn_cast<ArmSMETileOpInterface>(op)) { auto currentTileId = tileOp.getTileId(); if (currentTileId && unsigned(currentTileId.getInt()) != tileId) return tileOp.emitOpError( "already assigned different SME virtual tile!"); - tileOp.setTileId(tileIDAttr); + rewriter.updateRootInPlace(tileOp, + [&]() { tileOp.setTileId(tileIDAttr); }); } } diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp index 7c3330f4c238..dd482f305fcb 100644 --- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp +++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp @@ -19,6 +19,7 @@ #include "mlir/IR/BuiltinAttributes.h" #include "mlir/IR/BuiltinOps.h" #include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Diagnostics.h" #include "mlir/IR/DialectImplementation.h" #include "mlir/IR/Matchers.h" #include "mlir/IR/OpImplementation.h" @@ -588,8 +589,16 @@ static void printAllReduceOperation(AsmPrinter &printer, Operation *op, //===----------------------------------------------------------------------===// LogicalResult gpu::SubgroupReduceOp::verify() { + Type elemType = getType(); + if (auto vecTy = dyn_cast<VectorType>(elemType)) { + if (vecTy.isScalable()) + return emitOpError() << "is not compatible with scalable vector types"; + + elemType = vecTy.getElementType(); + } + gpu::AllReduceOperation opName = getOp(); - if (failed(verifyReduceOpAndType(opName, getType()))) { + if (failed(verifyReduceOpAndType(opName, elemType))) { return emitError() << '`' << gpu::stringifyAllReduceOperation(opName) << "` reduction operation is not compatible with type " << getType(); diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp index 08ddc7c25aa9..4c62289a1e94 100644 --- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp +++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp @@ -644,6 +644,45 @@ OpFoldResult spirv::UModOp::fold(FoldAdaptor adaptor) { } //===----------------------------------------------------------------------===// +// spirv.SNegate +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::SNegateOp::fold(FoldAdaptor adaptor) { + // -(-x) = 0 - (0 - x) = x + auto op = getOperand(); + if (auto negateOp = op.getDefiningOp<spirv::SNegateOp>()) + return negateOp->getOperand(0); + + // According to the SPIR-V spec: + // + // Signed-integer subtract of Operand from zero. + return constFoldUnaryOp<IntegerAttr>( + adaptor.getOperands(), [](const APInt &a) { + APInt zero = APInt::getZero(a.getBitWidth()); + return zero - a; + }); +} + +//===----------------------------------------------------------------------===// +// spirv.NotOp +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::NotOp::fold(spirv::NotOp::FoldAdaptor adaptor) { + // !(!x) = x + auto op = getOperand(); + if (auto notOp = op.getDefiningOp<spirv::NotOp>()) + return notOp->getOperand(0); + + // According to the SPIR-V spec: + // + // Complement the bits of Operand. + return constFoldUnaryOp<IntegerAttr>(adaptor.getOperands(), [&](APInt a) { + a.flipAllBits(); + return a; + }); +} + +//===----------------------------------------------------------------------===// // spirv.LogicalAnd //===----------------------------------------------------------------------===// @@ -714,6 +753,22 @@ OpFoldResult spirv::LogicalNotEqualOp::fold(FoldAdaptor adaptor) { // spirv.LogicalNot //===----------------------------------------------------------------------===// +OpFoldResult spirv::LogicalNotOp::fold(FoldAdaptor adaptor) { + // !(!x) = x + auto op = getOperand(); + if (auto notOp = op.getDefiningOp<spirv::LogicalNotOp>()) + return notOp->getOperand(0); + + // According to the SPIR-V spec: + // + // Complement the bits of Operand. + return constFoldUnaryOp<IntegerAttr>(adaptor.getOperands(), + [](const APInt &a) { + APInt zero = APInt::getZero(1); + return a == 1 ? zero : (zero + 1); + }); +} + void spirv::LogicalNotOp::getCanonicalizationPatterns( RewritePatternSet &results, MLIRContext *context) { results diff --git a/mlir/lib/Transforms/SCCP.cpp b/mlir/lib/Transforms/SCCP.cpp index 14435b37acc9..b2d3929b0459 100644 --- a/mlir/lib/Transforms/SCCP.cpp +++ b/mlir/lib/Transforms/SCCP.cpp @@ -53,7 +53,7 @@ static LogicalResult replaceWithConstant(DataFlowSolver &solver, Dialect *dialect = latticeValue.getConstantDialect(); Value constant = folder.getOrCreateConstant( builder.getInsertionBlock(), dialect, latticeValue.getConstantValue(), - value.getType(), value.getLoc()); + value.getType()); if (!constant) return failure(); diff --git a/mlir/lib/Transforms/Utils/FoldUtils.cpp b/mlir/lib/Transforms/Utils/FoldUtils.cpp index eb4dcb251a22..e5f78abf7fca 100644 --- a/mlir/lib/Transforms/Utils/FoldUtils.cpp +++ b/mlir/lib/Transforms/Utils/FoldUtils.cpp @@ -77,8 +77,10 @@ LogicalResult OperationFolder::tryToFold(Operation *op, bool *inPlaceUpdate) { // Check to see if we should rehoist, i.e. if a non-constant operation was // inserted before this one. Block *opBlock = op->getBlock(); - if (&opBlock->front() != op && !isFolderOwnedConstant(op->getPrevNode())) + if (&opBlock->front() != op && !isFolderOwnedConstant(op->getPrevNode())) { op->moveBefore(&opBlock->front()); + op->setLoc(erasedFoldedLocation); + } return failure(); } @@ -112,8 +114,10 @@ bool OperationFolder::insertKnownConstant(Operation *op, Attribute constValue) { // If this is a constant we unique'd, we don't need to insert, but we can // check to see if we should rehoist it. if (isFolderOwnedConstant(op)) { - if (&opBlock->front() != op && !isFolderOwnedConstant(op->getPrevNode())) + if (&opBlock->front() != op && !isFolderOwnedConstant(op->getPrevNode())) { op->moveBefore(&opBlock->front()); + op->setLoc(erasedFoldedLocation); + } return true; } @@ -142,6 +146,7 @@ bool OperationFolder::insertKnownConstant(Operation *op, Attribute constValue) { if (folderConstOp) { notifyRemoval(op); rewriter.replaceOp(op, folderConstOp->getResults()); + folderConstOp->setLoc(erasedFoldedLocation); return false; } @@ -151,8 +156,10 @@ bool OperationFolder::insertKnownConstant(Operation *op, Attribute constValue) { // anything. Otherwise, we move the constant to the insertion block. Block *insertBlock = &insertRegion->front(); if (opBlock != insertBlock || (&insertBlock->front() != op && - !isFolderOwnedConstant(op->getPrevNode()))) + !isFolderOwnedConstant(op->getPrevNode()))) { op->moveBefore(&insertBlock->front()); + op->setLoc(erasedFoldedLocation); + } folderConstOp = op; referencedDialects[op].push_back(op->getDialect()); @@ -193,17 +200,17 @@ void OperationFolder::clear() { /// Get or create a constant using the given builder. On success this returns /// the constant operation, nullptr otherwise. Value OperationFolder::getOrCreateConstant(Block *block, Dialect *dialect, - Attribute value, Type type, - Location loc) { + Attribute value, Type type) { // Find an insertion point for the constant. auto *insertRegion = getInsertionRegion(interfaces, block); auto &entry = insertRegion->front(); rewriter.setInsertionPoint(&entry, entry.begin()); // Get the constant map for the insertion region of this operation. + // Use erased location since the op is being built at the front of block. auto &uniquedConstants = foldScopes[insertRegion]; - Operation *constOp = - tryGetOrCreateConstant(uniquedConstants, dialect, value, type, loc); + Operation *constOp = tryGetOrCreateConstant(uniquedConstants, dialect, value, + type, erasedFoldedLocation); return constOp ? constOp->getResult(0) : Value(); } @@ -254,8 +261,9 @@ OperationFolder::processFoldResults(Operation *op, // Check to see if there is a canonicalized version of this constant. auto res = op->getResult(i); Attribute attrRepl = foldResults[i].get<Attribute>(); - if (auto *constOp = tryGetOrCreateConstant( - uniquedConstants, dialect, attrRepl, res.getType(), op->getLoc())) { + if (auto *constOp = + tryGetOrCreateConstant(uniquedConstants, dialect, attrRepl, + res.getType(), erasedFoldedLocation)) { // Ensure that this constant dominates the operation we are replacing it // with. This may not automatically happen if the operation being folded // was inserted before the constant within the insertion block. @@ -290,8 +298,11 @@ OperationFolder::tryGetOrCreateConstant(ConstantMap &uniquedConstants, // Check if an existing mapping already exists. auto constKey = std::make_tuple(dialect, value, type); Operation *&constOp = uniquedConstants[constKey]; - if (constOp) + if (constOp) { + if (loc != constOp->getLoc()) + constOp->setLoc(erasedFoldedLocation); return constOp; + } // If one doesn't exist, try to materialize one. if (!(constOp = materializeConstant(dialect, rewriter, value, type, loc))) @@ -314,6 +325,8 @@ OperationFolder::tryGetOrCreateConstant(ConstantMap &uniquedConstants, notifyRemoval(constOp); rewriter.eraseOp(constOp); referencedDialects[existingOp].push_back(dialect); + if (loc != existingOp->getLoc()) + existingOp->setLoc(erasedFoldedLocation); return constOp = existingOp; } diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt index 55c5973e40e5..3c9cf304d88a 100644 --- a/mlir/python/CMakeLists.txt +++ b/mlir/python/CMakeLists.txt @@ -21,7 +21,6 @@ declare_mlir_python_sources(MLIRPythonSources.Core.Python _mlir_libs/__init__.py ir.py passmanager.py - extras/types.py dialects/_ods_common.py # The main _mlir module has submodules: include stubs from each. @@ -30,6 +29,14 @@ declare_mlir_python_sources(MLIRPythonSources.Core.Python _mlir_libs/_mlir/passmanager.pyi ) +declare_mlir_python_sources(MLIRPythonSources.Core.Python.Extras + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + ADD_TO_PARENT MLIRPythonSources.Core.Python + SOURCES + extras/types.py + extras/meta.py +) + declare_mlir_python_sources(MLIRPythonSources.ExecutionEngine ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" ADD_TO_PARENT MLIRPythonSources diff --git a/mlir/python/mlir/dialects/arith.py b/mlir/python/mlir/dialects/arith.py index 83aca0d58bf2..663a53660a64 100644 --- a/mlir/python/mlir/dialects/arith.py +++ b/mlir/python/mlir/dialects/arith.py @@ -11,6 +11,8 @@ try: from ._ods_common import ( get_default_loc_context as _get_default_loc_context, _cext as _ods_cext, + get_op_result_or_op_results as _get_op_result_or_op_results, + SubClassValueT as _SubClassValueT, ) from typing import Any, List, Union @@ -75,3 +77,9 @@ class ConstantOp(ConstantOp): return FloatAttr(self.value).value else: raise ValueError("only integer and float constants have literal values") + + +def constant( + result: Type, value: Union[int, float, Attribute], *, loc=None, ip=None +) -> _SubClassValueT: + return _get_op_result_or_op_results(ConstantOp(result, value, loc=loc, ip=ip)) diff --git a/mlir/python/mlir/dialects/builtin.py b/mlir/python/mlir/dialects/builtin.py index b71cc2466d46..1c69d6d7c3a0 100644 --- a/mlir/python/mlir/dialects/builtin.py +++ b/mlir/python/mlir/dialects/builtin.py @@ -2,8 +2,11 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +from typing import Dict, Optional + from ._builtin_ops_gen import * from ._builtin_ops_gen import _Dialect +from ..extras.meta import region_op try: from ..ir import * @@ -23,3 +26,23 @@ class ModuleOp(ModuleOp): @property def body(self): return self.regions[0].blocks[0] + + +@region_op +def module( + *, + sym_name=None, + sym_visibility=None, + attrs: Optional[Dict[str, Attribute]] = None, + loc=None, + ip=None, +): + mod = ModuleOp.__base__( + sym_name=sym_name, sym_visibility=sym_visibility, loc=loc, ip=ip + ) + if attrs is None: + attrs = {} + for attr_name, attr in attrs.items(): + mod.operation.attributes[attr_name] = attr + + return mod diff --git a/mlir/python/mlir/dialects/func.py b/mlir/python/mlir/dialects/func.py index 6599f67b7078..24fdcbcd85b2 100644 --- a/mlir/python/mlir/dialects/func.py +++ b/mlir/python/mlir/dialects/func.py @@ -243,6 +243,9 @@ class FuncOp(FuncOp): return decorator +func = FuncOp.from_py_func + + @_ods_cext.register_operation(_Dialect, replace=True) class CallOp(CallOp): """Specialization for the call op class.""" diff --git a/mlir/python/mlir/dialects/pdl.py b/mlir/python/mlir/dialects/pdl.py index 90d7d706238e..db07dc50aabd 100644 --- a/mlir/python/mlir/dialects/pdl.py +++ b/mlir/python/mlir/dialects/pdl.py @@ -5,6 +5,7 @@ from ._pdl_ops_gen import * from ._pdl_ops_gen import _Dialect from .._mlir_libs._mlirDialectsPDL import * +from .._mlir_libs._mlirDialectsPDL import OperationType try: @@ -13,7 +14,7 @@ try: except ImportError as e: raise RuntimeError("Error loading imports from extension module") from e -from typing import Union, Optional, Sequence, Mapping +from typing import Union, Optional, Sequence, Mapping, NewType from ._ods_common import ( get_op_result_or_value as _get_value, get_op_results_or_values as _get_values, @@ -220,3 +221,10 @@ class TypesOp(TypesOp): constantTypes = [] result = pdl.RangeType.get(pdl.TypeType.get()) super().__init__(result, constantTypes=constantTypes, loc=loc, ip=ip) + + +OperationTypeT = NewType("OperationType", OperationType) + + +def op_t() -> OperationTypeT: + return OperationTypeT(OperationType.get()) diff --git a/mlir/python/mlir/dialects/scf.py b/mlir/python/mlir/dialects/scf.py index 20bbed9bc93d..dad7377987e5 100644 --- a/mlir/python/mlir/dialects/scf.py +++ b/mlir/python/mlir/dialects/scf.py @@ -120,7 +120,7 @@ def for_( params = [start, stop, step] for i, p in enumerate(params): if isinstance(p, int): - p = constant(IntegerAttr.get(IndexType.get(), p)) + p = constant(IndexType.get(), p) elif isinstance(p, float): raise ValueError(f"{p=} must be int.") params[i] = p diff --git a/mlir/python/mlir/dialects/tensor.py b/mlir/python/mlir/dialects/tensor.py index 67248748eaf3..79dd9476ad0f 100644 --- a/mlir/python/mlir/dialects/tensor.py +++ b/mlir/python/mlir/dialects/tensor.py @@ -4,6 +4,7 @@ from ._tensor_ops_gen import * from ._tensor_ops_gen import _Dialect +from ..extras.meta import region_op try: from ..ir import * @@ -40,3 +41,9 @@ class EmptyOp(EmptyOp): dynamic_sizes.append(s) result_type = RankedTensorType.get(static_sizes, element_type) super().__init__(result_type, dynamic_sizes, loc=loc, ip=ip) + + +generate = region_op( + lambda result, dynamic_extents: GenerateOp(result, dynamic_extents), + terminator=lambda args: YieldOp(args[0]), +) diff --git a/mlir/python/mlir/dialects/transform/__init__.py b/mlir/python/mlir/dialects/transform/__init__.py index 175634c7d458..5b158ec6b65f 100644 --- a/mlir/python/mlir/dialects/transform/__init__.py +++ b/mlir/python/mlir/dialects/transform/__init__.py @@ -18,7 +18,7 @@ try: except ImportError as e: raise RuntimeError("Error loading imports from extension module") from e -from typing import Optional, Sequence, Union +from typing import Optional, Sequence, Union, NewType @_ods_cext.register_operation(_Dialect, replace=True) @@ -175,7 +175,7 @@ class NamedSequenceOp(NamedSequenceOp): result_types: Sequence[Type], sym_visibility=None, arg_attrs=None, - res_attrs=None + res_attrs=None, ): function_type = FunctionType.get(input_types, result_types) super().__init__( @@ -183,7 +183,7 @@ class NamedSequenceOp(NamedSequenceOp): function_type=TypeAttr.get(function_type), sym_visibility=sym_visibility, arg_attrs=arg_attrs, - res_attrs=res_attrs + res_attrs=res_attrs, ) self.regions[0].blocks.append(*input_types) @@ -212,3 +212,10 @@ class YieldOp(YieldOp): if operands is None: operands = [] super().__init__(_get_op_results_or_values(operands), loc=loc, ip=ip) + + +AnyOpTypeT = NewType("AnyOpType", AnyOpType) + + +def any_op_t() -> AnyOpTypeT: + return AnyOpTypeT(AnyOpType.get()) diff --git a/mlir/python/mlir/dialects/transform/extras/__init__.py b/mlir/python/mlir/dialects/transform/extras/__init__.py index c715dac1ef7e..e4d47e9064f2 100644 --- a/mlir/python/mlir/dialects/transform/extras/__init__.py +++ b/mlir/python/mlir/dialects/transform/extras/__init__.py @@ -4,8 +4,16 @@ from typing import Callable, Optional, Sequence, Union +from ....extras.meta import region_op from .... import ir -from .. import AnyOpType, OperationType, NamedSequenceOp, YieldOp +from .. import ( + AnyOpType, + OperationType, + NamedSequenceOp, + YieldOp, + SequenceOp, + ApplyPatternsOp, +) from .. import structured @@ -147,3 +155,8 @@ def insert_transform_script( if dump_script: print(named_sequence_op) + + +sequence = region_op(SequenceOp.__base__, terminator=YieldOp) +named_sequence = region_op(NamedSequenceOp, terminator=YieldOp) +apply_patterns = region_op(ApplyPatternsOp) diff --git a/mlir/python/mlir/extras/meta.py b/mlir/python/mlir/extras/meta.py new file mode 100644 index 000000000000..3f2defadf794 --- /dev/null +++ b/mlir/python/mlir/extras/meta.py @@ -0,0 +1,83 @@ +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import inspect +from functools import wraps + +from ..dialects._ods_common import get_op_result_or_op_results +from ..ir import Type, InsertionPoint + + +def op_region_builder(op, op_region, terminator=None): + def builder_wrapper(body_builder): + # Add a block with block args having types determined by type hints on the wrapped function. + if len(op_region.blocks) == 0: + sig = inspect.signature(body_builder) + types = [p.annotation for p in sig.parameters.values()] + if not ( + len(types) == len(sig.parameters) + and all(isinstance(t, Type) for t in types) + ): + raise ValueError( + f"for {body_builder=} either missing a type annotation or type annotation isn't a mlir type: {sig}" + ) + + op_region.blocks.append(*types) + + with InsertionPoint(op_region.blocks[0]): + results = body_builder(*list(op_region.blocks[0].arguments)) + + with InsertionPoint(list(op_region.blocks)[-1]): + if terminator is not None: + res = [] + if isinstance(results, (tuple, list)): + res.extend(results) + elif results is not None: + res.append(results) + terminator(res) + + return get_op_result_or_op_results(op) + + return builder_wrapper + + +def region_op(op_constructor, terminator=None): + """Decorator to define an MLIR Op specified as a python function. + + Requires that an `mlir.ir.InsertionPoint` and `mlir.ir.Location` are + active for the current thread (i.e. established in a `with` block). + + Supports "naked" usage i.e., no parens if no args need to be passed to the Op constructor. + + When applied as a decorator to a Python function, an entry block will + be constructed for the Op with types as specified **as type hints on the args of the function**. + The block arguments will be passed positionally to the Python function. + + If a terminator is specified then the return from the decorated function will be passed + to the terminator as the last statement in the entry block. Note, the API for the terminator + is a (possibly empty) list; terminator accepting single values should be wrapped in a + `lambda args: term(args[0])` + + The identifier (name) of the function will become: + 1. A single value result if the Op returns a single value; + 2. An OpResultList (as a list) if the Op returns multiple values; + 3. The Operation if the Op returns no results. + + See examples in tensor.py and transform.extras. + """ + + def op_decorator(*args, **kwargs): + op = op_constructor(*args, **kwargs) + op_region = op.regions[0] + + return op_region_builder(op, op_region, terminator) + + @wraps(op_decorator) + def maybe_no_args(*args, **kwargs): + if len(args) == 1 and len(kwargs) == 0 and callable(args[0]): + return op_decorator()(args[0]) + else: + return op_decorator(*args, **kwargs) + + return maybe_no_args diff --git a/mlir/test/Conversion/GPUToSPIRV/reductions.mlir b/mlir/test/Conversion/GPUToSPIRV/reductions.mlir index af58f4173136..44f85f68587f 100644 --- a/mlir/test/Conversion/GPUToSPIRV/reductions.mlir +++ b/mlir/test/Conversion/GPUToSPIRV/reductions.mlir @@ -655,6 +655,26 @@ gpu.module @kernels { // ----- +module attributes { + gpu.container_module, + spirv.target_env = #spirv.target_env<#spirv.vce<v1.3, [Kernel, Addresses, Groups, GroupNonUniformArithmetic, GroupUniformArithmeticKHR], []>, #spirv.resource_limits<>> +} { + +gpu.module @kernels { + // CHECK-LABEL: spirv.func @test + // CHECK-SAME: (%[[ARG:.*]]: i32) + gpu.func @test(%arg : vector<1xi32>) kernel + attributes {spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [16, 1, 1]>} { + // CHECK: %{{.*}} = spirv.GroupNonUniformSMax "Subgroup" "Reduce" %[[ARG]] : i32 + %r0 = gpu.subgroup_reduce maxsi %arg : (vector<1xi32>) -> (vector<1xi32>) + gpu.return + } +} + +} + +// ----- + // TODO: Handle boolean reductions. module attributes { @@ -751,3 +771,21 @@ gpu.module @kernels { } } } + +// ----- + +// Vector reductions need to be lowered to scalar reductions first. + +module attributes { + gpu.container_module, + spirv.target_env = #spirv.target_env<#spirv.vce<v1.3, [Kernel, Addresses, Groups, GroupNonUniformArithmetic, GroupUniformArithmeticKHR], []>, #spirv.resource_limits<>> +} { +gpu.module @kernels { + gpu.func @maxui(%arg : vector<2xi32>) kernel + attributes {spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [16, 1, 1]>} { + // expected-error @+1 {{failed to legalize operation 'gpu.subgroup_reduce'}} + %r0 = gpu.subgroup_reduce maxui %arg : (vector<2xi32>) -> (vector<2xi32>) + gpu.return + } +} +} diff --git a/mlir/test/Dialect/GPU/invalid.mlir b/mlir/test/Dialect/GPU/invalid.mlir index d8a40f89f80a..8a34d6432607 100644 --- a/mlir/test/Dialect/GPU/invalid.mlir +++ b/mlir/test/Dialect/GPU/invalid.mlir @@ -333,9 +333,17 @@ func.func @reduce_invalid_op_type_maximumf(%arg0 : i32) { // ----- -func.func @subgroup_reduce_bad_type(%arg0 : vector<2xf32>) { - // expected-error@+1 {{'gpu.subgroup_reduce' op operand #0 must be Integer or Float}} - %res = gpu.subgroup_reduce add %arg0 : (vector<2xf32>) -> vector<2xf32> +func.func @subgroup_reduce_bad_type(%arg0 : vector<2x2xf32>) { + // expected-error@+1 {{'gpu.subgroup_reduce' op operand #0 must be Integer or Float or vector of}} + %res = gpu.subgroup_reduce add %arg0 : (vector<2x2xf32>) -> vector<2x2xf32> + return +} + +// ----- + +func.func @subgroup_reduce_bad_type_scalable(%arg0 : vector<[2]xf32>) { + // expected-error@+1 {{is not compatible with scalable vector types}} + %res = gpu.subgroup_reduce add %arg0 : (vector<[2]xf32>) -> vector<[2]xf32> return } diff --git a/mlir/test/Dialect/GPU/ops.mlir b/mlir/test/Dialect/GPU/ops.mlir index 481934364156..605124243830 100644 --- a/mlir/test/Dialect/GPU/ops.mlir +++ b/mlir/test/Dialect/GPU/ops.mlir @@ -84,6 +84,8 @@ module attributes {gpu.container_module} { %one = arith.constant 1.0 : f32 + %vec = vector.broadcast %arg0 : f32 to vector<4xf32> + // CHECK: %{{.*}} = gpu.all_reduce add %{{.*}} { // CHECK-NEXT: } : (f32) -> f32 %sum = gpu.all_reduce add %one {} : (f32) -> (f32) @@ -98,6 +100,9 @@ module attributes {gpu.container_module} { // CHECK: %{{.*}} = gpu.subgroup_reduce add %{{.*}} uniform : (f32) -> f32 %sum_subgroup1 = gpu.subgroup_reduce add %one uniform : (f32) -> f32 + // CHECK: %{{.*}} = gpu.subgroup_reduce add %{{.*}} : (vector<4xf32>) -> vector<4xf32> + %sum_subgroup2 = gpu.subgroup_reduce add %vec : (vector<4xf32>) -> vector<4xf32> + %width = arith.constant 7 : i32 %offset = arith.constant 3 : i32 // CHECK: gpu.shuffle xor %{{.*}}, %{{.*}}, %{{.*}} : f32 diff --git a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir index 871ecd4f28b1..1cb69891a70e 100644 --- a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir +++ b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir @@ -1007,6 +1007,102 @@ func.func @umod_fail_fold(%arg0: i32) -> (i32, i32) { // ----- //===----------------------------------------------------------------------===// +// spirv.SNegate +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @snegate_twice +// CHECK-SAME: (%[[ARG:.*]]: i32) +func.func @snegate_twice(%arg0 : i32) -> i32 { + %0 = spirv.SNegate %arg0 : i32 + %1 = spirv.SNegate %0 : i32 + + // CHECK: return %[[ARG]] : i32 + return %1 : i32 +} + +// CHECK-LABEL: @snegate_min +func.func @snegate_min() -> (i8, i8) { + // CHECK: %[[MIN:.*]] = spirv.Constant -128 : i8 + %cmin = spirv.Constant -128 : i8 + + %0 = spirv.SNegate %cmin : i8 + %1 = spirv.SNegate %0 : i8 + + // CHECK: return %[[MIN]], %[[MIN]] + return %0, %1 : i8, i8 +} + +// CHECK-LABEL: @const_fold_scalar_snegate +func.func @const_fold_scalar_snegate() -> (i32, i32, i32) { + %c0 = spirv.Constant 0 : i32 + %c3 = spirv.Constant 3 : i32 + %cn3 = spirv.Constant -3 : i32 + + // CHECK-DAG: %[[THREE:.*]] = spirv.Constant 3 : i32 + // CHECK-DAG: %[[NTHREE:.*]] = spirv.Constant -3 : i32 + // CHECK-DAG: %[[ZERO:.*]] = spirv.Constant 0 : i32 + %0 = spirv.SNegate %c0 : i32 + %1 = spirv.SNegate %c3 : i32 + %2 = spirv.SNegate %cn3 : i32 + + // CHECK: return %[[ZERO]], %[[NTHREE]], %[[THREE]] + return %0, %1, %2 : i32, i32, i32 +} + +// CHECK-LABEL: @const_fold_vector_snegate +func.func @const_fold_vector_snegate() -> vector<3xi32> { + // CHECK: spirv.Constant dense<[0, 3, -3]> + %cv = spirv.Constant dense<[0, -3, 3]> : vector<3xi32> + %0 = spirv.SNegate %cv : vector<3xi32> + return %0 : vector<3xi32> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.Not +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @not_twice +// CHECK-SAME: (%[[ARG:.*]]: i32) +func.func @not_twice(%arg0 : i32) -> i32 { + %0 = spirv.Not %arg0 : i32 + %1 = spirv.Not %0 : i32 + + // CHECK: return %[[ARG]] : i32 + return %1 : i32 +} + +// CHECK-LABEL: @const_fold_scalar_not +func.func @const_fold_scalar_not() -> (i32, i32, i32) { + %c0 = spirv.Constant 0 : i32 + %c3 = spirv.Constant 3 : i32 + %cn3 = spirv.Constant -3 : i32 + + // CHECK-DAG: %[[TWO:.*]] = spirv.Constant 2 : i32 + // CHECK-DAG: %[[NFOUR:.*]] = spirv.Constant -4 : i32 + // CHECK-DAG: %[[NONE:.*]] = spirv.Constant -1 : i32 + %0 = spirv.Not %c0 : i32 + %1 = spirv.Not %c3 : i32 + %2 = spirv.Not %cn3 : i32 + + // CHECK: return %[[NONE]], %[[NFOUR]], %[[TWO]] + return %0, %1, %2 : i32, i32, i32 +} + +// CHECK-LABEL: @const_fold_vector_not +func.func @const_fold_vector_not() -> vector<3xi32> { + %cv = spirv.Constant dense<[-1, -4, 2]> : vector<3xi32> + + // CHECK: spirv.Constant dense<[0, 3, -3]> + %0 = spirv.Not %cv : vector<3xi32> + + return %0 : vector<3xi32> +} + +// ----- + +//===----------------------------------------------------------------------===// // spirv.LogicalAnd //===----------------------------------------------------------------------===// @@ -1040,6 +1136,38 @@ func.func @convert_logical_and_true_false_vector(%arg: vector<3xi1>) -> (vector< // spirv.LogicalNot //===----------------------------------------------------------------------===// +// CHECK-LABEL: @logical_not_twice +// CHECK-SAME: (%[[ARG:.*]]: i1) +func.func @logical_not_twice(%arg0 : i1) -> i1 { + %0 = spirv.LogicalNot %arg0 : i1 + %1 = spirv.LogicalNot %0 : i1 + + // CHECK: return %[[ARG]] : i1 + return %1 : i1 +} + +// CHECK-LABEL: @const_fold_scalar_logical_not +func.func @const_fold_scalar_logical_not() -> i1 { + %true = spirv.Constant true + + // CHECK: spirv.Constant false + %0 = spirv.LogicalNot %true : i1 + + return %0 : i1 +} + +// CHECK-LABEL: @const_fold_vector_logical_not +func.func @const_fold_vector_logical_not() -> vector<2xi1> { + %cv = spirv.Constant dense<[true, false]> : vector<2xi1> + + // CHECK: spirv.Constant dense<[false, true]> + %0 = spirv.LogicalNot %cv : vector<2xi1> + + return %0 : vector<2xi1> +} + +// ----- + func.func @convert_logical_not_to_not_equal(%arg0: vector<3xi64>, %arg1: vector<3xi64>) -> vector<3xi1> { // CHECK: %[[RESULT:.*]] = spirv.INotEqual {{%.*}}, {{%.*}} : vector<3xi64> // CHECK-NEXT: spirv.ReturnValue %[[RESULT]] : vector<3xi1> diff --git a/mlir/test/Dialect/Transform/test-pattern-application.mlir b/mlir/test/Dialect/Transform/test-pattern-application.mlir index 2fd47c6bae39..ff9a535c8384 100644 --- a/mlir/test/Dialect/Transform/test-pattern-application.mlir +++ b/mlir/test/Dialect/Transform/test-pattern-application.mlir @@ -179,7 +179,6 @@ module { // CHECK: return %[[c5]] func.func @canonicalization(%t: tensor<5xf32>) -> index { %c0 = arith.constant 0 : index - // expected-remark @below {{op was replaced}} %dim = tensor.dim %t, %c0 : tensor<5xf32> return %dim : index } @@ -191,7 +190,6 @@ transform.sequence failures(propagate) { transform.apply_patterns to %1 { transform.apply_patterns.canonicalization } : !transform.any_op - transform.test_print_remark_at_operand %0, "op was replaced" : !transform.any_op } // ----- diff --git a/mlir/test/Dialect/Vector/fold-arith-extf-into-vector-contract.mlir b/mlir/test/Dialect/Vector/fold-arith-extf-into-vector-contract.mlir index 79429afd8ff2..31ae126906f2 100644 --- a/mlir/test/Dialect/Vector/fold-arith-extf-into-vector-contract.mlir +++ b/mlir/test/Dialect/Vector/fold-arith-extf-into-vector-contract.mlir @@ -10,9 +10,41 @@ // CHECK-SAME: iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>} // CHECK-SAME: %[[ARG0]], %[[ARG1]], %[[ARG2]] : vector<64x64xf16>, vector<64x64xf16> into vector<64x64xf32> // CHECK-NEXT: return %[[R]] : vector<64x64xf32> -func.func @fold_arith_extf_into_contract(%arg0: vector<64x64xf16>, %arg1: vector<64x64xf16>, %arg2: vector<64x64xf32>) -> vector<64x64xf32> { +func.func @fold_arith_extf_into_contract( + %arg0: vector<64x64xf16>, + %arg1: vector<64x64xf16>, + %arg2: vector<64x64xf32>) -> vector<64x64xf32> { %lhs_f32 = arith.extf %arg0 : vector<64x64xf16> to vector<64x64xf32> %rhs_f32 = arith.extf %arg1 : vector<64x64xf16> to vector<64x64xf32> - %result = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d2, d1)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>} %lhs_f32, %rhs_f32, %arg2 : vector<64x64xf32>, vector<64x64xf32> into vector<64x64xf32> + %result = vector.contract { + indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d2, d1)>, affine_map<(d0, d1, d2) -> (d0, d1)>], + iterator_types = ["parallel", "parallel", "reduction"], + kind = #vector.kind<add>} + %lhs_f32, %rhs_f32, %arg2 : vector<64x64xf32>, vector<64x64xf32> into vector<64x64xf32> return %result : vector<64x64xf32> -}
\ No newline at end of file +} + +// ----- + +// CHECK-DAG: #[[$map0:.*]] = affine_map<(d0, d1, d2) -> (d0, d2)> +// CHECK-DAG: #[[$map1:.*]] = affine_map<(d0, d1, d2) -> (d2, d1)> +// CHECK-DAG: #[[$map2:.*]] = affine_map<(d0, d1, d2) -> (d0, d1)> +// CHECK-LABEL: func.func @fold_arith_extf_into_contract_scalable +// CHECK-SAME: (%[[ARG0:.*]]: vector<[64]x64xf16>, %[[ARG1:.*]]: vector<64x64xf16>, %[[ARG2:.*]]: vector<[64]x64xf32>) +// CHECK-NEXT: %[[R:.+]] = vector.contract {indexing_maps = [#[[$map0]], #[[$map1]], #[[$map2]]], +// CHECK-SAME: iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>} +// CHECK-SAME: %[[ARG0]], %[[ARG1]], %[[ARG2]] : vector<[64]x64xf16>, vector<64x64xf16> into vector<[64]x64xf32> +// CHECK-NEXT: return %[[R]] : vector<[64]x64xf32> +func.func @fold_arith_extf_into_contract_scalable( + %arg0: vector<[64]x64xf16>, + %arg1: vector<64x64xf16>, + %arg2: vector<[64]x64xf32>) -> vector<[64]x64xf32> { + %lhs_f32 = arith.extf %arg0 : vector<[64]x64xf16> to vector<[64]x64xf32> + %rhs_f32 = arith.extf %arg1 : vector<64x64xf16> to vector<64x64xf32> + %result = vector.contract { + indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d2, d1)>, affine_map<(d0, d1, d2) -> (d0, d1)>], + iterator_types = ["parallel", "parallel", "reduction"], + kind = #vector.kind<add>} + %lhs_f32, %rhs_f32, %arg2 : vector<[64]x64xf32>, vector<64x64xf32> into vector<[64]x64xf32> + return %result : vector<[64]x64xf32> +} diff --git a/mlir/test/Dialect/Vector/vector-contract-to-outerproduct-transforms.mlir b/mlir/test/Dialect/Vector/vector-contract-to-outerproduct-matmul-transforms.mlir index 7588b738ff9a..7a60ff8ea858 100644 --- a/mlir/test/Dialect/Vector/vector-contract-to-outerproduct-transforms.mlir +++ b/mlir/test/Dialect/Vector/vector-contract-to-outerproduct-matmul-transforms.mlir @@ -1,20 +1,22 @@ // RUN: mlir-opt %s --transform-interpreter --split-input-file | FileCheck %s -// NOTE - tests in this file are duplicated so that there's a version for -// * _fixed width_ and for _scalable_ vectors. -// In order for the "vector.contract -> vector.outerproduct" patterns to work, -// only the non-reduction dimension can be scalable (*). For Matmul operations -// that is set to be the N dimension (i.e. rows of the output matrix), which -// matches how matrix multiplication are normally implemented for e.g. -// Arm SVE. However, making the M dimension scalable (i.e. columns of the -// output matrix) should work as well. -// -// (*) The conversion tested in this file unrolls along the reduction -// dimension, which is not supported for scalable vectors. +/// Tests for `vector.contract` -> `vector.outerproduct` transformations for +/// matmul operations: +/// C += A * B. +/// (A, B and C are 2-d matrices). ATM three different variants / are tested: +/// * plain (no mask, fixed-wdith vectors), +/// * masked (fixed-width vectors, +/// * scalable (mask + scalable vectors). +/// In order for the "vector.contract -> vector.outerproduct" patterns to work, +/// only the non-reduction dimension can be scalable (*). For matmul operations +/// that is set to be the N dimension (i.e. rows of the output matrix), which +/// matches how matrix multiplication are normally implemented for e.g. +/// Arm SVE. However, making the M dimension scalable (i.e. columns of the +/// output matrix) should work as well. +/// +/// (*) The conversion tested in this file unrolls along the reduction +/// dimension, which is not supported for scalable vectors. -// ============================================================================ -// Matmul 0 (plain + masked + mixed types) -// ============================================================================ #matmat_accesses_0 = [ affine_map<(m, n, k) -> (m, k)>, affine_map<(m, n, k) -> (k, n)>, @@ -25,6 +27,49 @@ iterator_types = ["parallel", "parallel", "reduction"] } +#matmat_accesses_1 = [ + affine_map<(m, n, k) -> (m, k)>, + affine_map<(m, n, k) -> (n, k)>, + affine_map<(m, n, k) -> (m, n)> +] +#matmat_trait_1 = { + indexing_maps = #matmat_accesses_1, + iterator_types = ["parallel", "parallel", "reduction"] +} + +#matmat_accesses_2 = [ + affine_map<(m, n, k) -> (k, m)>, + affine_map<(m, n, k) -> (k, n)>, + affine_map<(m, n, k) -> (m, n)> +] +#matmat_trait_2 = { + indexing_maps = #matmat_accesses_2, + iterator_types = ["parallel", "parallel", "reduction"] +} + +#matmat_accesses_3 = [ + affine_map<(m, n, k) -> (k, m)>, + affine_map<(m, n, k) -> (n, k)>, + affine_map<(m, n, k) -> (m, n)> +] +#matmat_trait_3 = { + indexing_maps = #matmat_accesses_3, + iterator_types = ["parallel", "parallel", "reduction"] +} + +#matmat_accesses_4 = [ + affine_map<(m, n, k) -> (m, k)>, + affine_map<(m, n, k) -> (k, n)>, + affine_map<(m, n, k) -> (n, m)> +] +#matmat_trait_4 = { + indexing_maps = #matmat_accesses_4, + iterator_types = ["parallel", "parallel", "reduction"] +} + +// ============================================================================ +// Matmul 0 (plain + masked + mixed types) +// ============================================================================ // CHECK-LABEL: func @matmul // CHECK-SAME: %[[A:[a-zA-Z0-9]*]]: vector<2x4xf32>, // CHECK-SAME: %[[B:[a-zA-Z0-9]*]]: vector<4x3xf32>, @@ -53,10 +98,10 @@ // CHECK-SAME: : vector<2xf32>, vector<3xf32> // // CHECK: return %[[c3]] : vector<2x3xf32> -func.func @matmul(%arg0: vector<2x4xf32>, - %arg1: vector<4x3xf32>, - %arg2: vector<2x3xf32>) -> vector<2x3xf32> { - %0 = vector.contract #matmat_trait_0 %arg0, %arg1, %arg2 +func.func @matmul(%A: vector<2x4xf32>, + %B: vector<4x3xf32>, + %C: vector<2x3xf32>) -> vector<2x3xf32> { + %0 = vector.contract #matmat_trait_0 %A, %B, %C : vector<2x4xf32>, vector<4x3xf32> into vector<2x3xf32> return %0 : vector<2x3xf32> } @@ -89,10 +134,10 @@ func.func @matmul(%arg0: vector<2x4xf32>, // CHECK-SAME: : vector<2xf32>, vector<[3]xf32> // // CHECK: return %[[c3]] : vector<2x[3]xf32> -func.func @matmul_scalable(%arg0: vector<2x4xf32>, - %arg1: vector<4x[3]xf32>, - %arg2: vector<2x[3]xf32>) -> vector<2x[3]xf32> { - %0 = vector.contract #matmat_trait_0 %arg0, %arg1, %arg2 +func.func @matmul_scalable(%A: vector<2x4xf32>, + %B: vector<4x[3]xf32>, + %C: vector<2x[3]xf32>) -> vector<2x[3]xf32> { + %0 = vector.contract #matmat_trait_0 %A, %B, %C : vector<2x4xf32>, vector<4x[3]xf32> into vector<2x[3]xf32> return %0 : vector<2x[3]xf32> } @@ -114,11 +159,11 @@ func.func @matmul_scalable(%arg0: vector<2x4xf32>, // CHECK: %[[T_MASK_R4:.*]] = vector.extract %[[T_MASK]][4] : vector<3x7xi1> from vector<5x3x7xi1> // CHECK: %{{.*}} = vector.mask %[[T_MASK_R4]] { vector.outerproduct %{{.*}} {kind = #vector.kind<add>} : vector<3xf32>, vector<7xf32> } : vector<3x7xi1> -> vector<3x7xf32> -func.func @masked_matmul(%arg0: vector<3x5xf32>, - %arg1: vector<5x7xf32>, - %arg2: vector<3x7xf32>, +func.func @masked_matmul(%A: vector<3x5xf32>, + %B: vector<5x7xf32>, + %C: vector<3x7xf32>, %m : vector<3x7x5xi1>) -> vector<3x7xf32> { - %0 = vector.mask %m { vector.contract #matmat_trait_0 %arg0, %arg1, %arg2 + %0 = vector.mask %m { vector.contract #matmat_trait_0 %A, %B, %C : vector<3x5xf32>, vector<5x7xf32> into vector<3x7xf32> } : vector<3x7x5xi1> -> vector<3x7xf32> return %0 : vector<3x7xf32> } @@ -140,11 +185,11 @@ func.func @masked_matmul(%arg0: vector<3x5xf32>, // CHECK: %[[T_MASK_R4:.*]] = vector.extract %[[T_MASK]][4] : vector<3x[7]xi1> from vector<5x3x[7]xi1> // CHECK: %{{.*}} = vector.mask %[[T_MASK_R4]] { vector.outerproduct %{{.*}} {kind = #vector.kind<add>} : vector<3xf32>, vector<[7]xf32> } : vector<3x[7]xi1> -> vector<3x[7]xf32> -func.func @masked_matmul_scalable(%arg0: vector<3x5xf32>, - %arg1: vector<5x[7]xf32>, - %arg2: vector<3x[7]xf32>, +func.func @masked_matmul_scalable(%A: vector<3x5xf32>, + %B: vector<5x[7]xf32>, + %C: vector<3x[7]xf32>, %m : vector<3x[7]x5xi1>) -> vector<3x[7]xf32> { - %0 = vector.mask %m { vector.contract #matmat_trait_0 %arg0, %arg1, %arg2 + %0 = vector.mask %m { vector.contract #matmat_trait_0 %A, %B, %C : vector<3x5xf32>, vector<5x[7]xf32> into vector<3x[7]xf32> } : vector<3x[7]x5xi1> -> vector<3x[7]xf32> return %0 : vector<3x[7]xf32> } @@ -160,11 +205,11 @@ func.func @masked_matmul_scalable(%arg0: vector<3x5xf32>, // CHECK: %[[b1:.*]] = arith.extf %[[b0]] : vector<3xf16> to vector<3xf32> // CHECK: %[[c0:.*]] = vector.outerproduct %[[a1]], %[[b1]], %[[C]] // CHECK: return %[[c0]] : vector<2x3xf32> -func.func @matmul_mixed(%arg0: vector<2x1xf16>, - %arg1: vector<1x3xf16>, - %arg2: vector<2x3xf32>) -> vector<2x3xf32> +func.func @matmul_mixed(%A: vector<2x1xf16>, + %B: vector<1x3xf16>, + %C: vector<2x3xf32>) -> vector<2x3xf32> { - %0 = vector.contract #matmat_trait_0 %arg0, %arg1, %arg2 + %0 = vector.contract #matmat_trait_0 %A, %B, %C : vector<2x1xf16>, vector<1x3xf16> into vector<2x3xf32> return %0 : vector<2x3xf32> } @@ -180,28 +225,18 @@ func.func @matmul_mixed(%arg0: vector<2x1xf16>, // CHECK: %[[b1:.*]] = arith.extf %[[b0]] : vector<[3]xf16> to vector<[3]xf32> // CHECK: %[[c0:.*]] = vector.outerproduct %[[a1]], %[[b1]], %[[C]] // CHECK: return %[[c0]] : vector<2x[3]xf32> -func.func @matmul_mixed_scalable(%arg0: vector<2x1xf16>, - %arg1: vector<1x[3]xf16>, - %arg2: vector<2x[3]xf32>) -> vector<2x[3]xf32> +func.func @matmul_mixed_scalable(%A: vector<2x1xf16>, + %B: vector<1x[3]xf16>, + %C: vector<2x[3]xf32>) -> vector<2x[3]xf32> { - %0 = vector.contract #matmat_trait_0 %arg0, %arg1, %arg2 + %0 = vector.contract #matmat_trait_0 %A, %B, %C : vector<2x1xf16>, vector<1x[3]xf16> into vector<2x[3]xf32> return %0 : vector<2x[3]xf32> } // ============================================================================ -// Matmul 1 (plain) +// Matmul 1 (plain + scalable) // ============================================================================ -#matmat_accesses_1 = [ - affine_map<(m, n, k) -> (m, k)>, - affine_map<(m, n, k) -> (n, k)>, - affine_map<(m, n, k) -> (m, n)> -] -#matmat_trait_1 = { - indexing_maps = #matmat_accesses_1, - iterator_types = ["parallel", "parallel", "reduction"] -} - // CHECK-LABEL: func @matmul_1 // CHECK-SAME: %[[A:[a-zA-Z0-9]*]]: vector<2x1xf32>, // CHECK-SAME: %[[B:[a-zA-Z0-9]*]]: vector<3x1xf32>, @@ -212,11 +247,11 @@ func.func @matmul_mixed_scalable(%arg0: vector<2x1xf16>, // CHECK: %[[b0:.*]] = vector.extract %[[Bt]][0] : vector<3xf32> from vector<1x3xf32> // CHECK: %[[c0:.*]] = vector.outerproduct %[[a0]], %[[b0]], %[[C]] // CHECK: return %[[c0]] : vector<2x3xf32> -func.func @matmul_1(%arg0: vector<2x1xf32>, - %arg1: vector<3x1xf32>, - %arg2: vector<2x3xf32>) -> vector<2x3xf32> +func.func @matmul_1(%A: vector<2x1xf32>, + %B: vector<3x1xf32>, + %C: vector<2x3xf32>) -> vector<2x3xf32> { - %0 = vector.contract #matmat_trait_1 %arg0, %arg1, %arg2 + %0 = vector.contract #matmat_trait_1 %A, %B, %C : vector<2x1xf32>, vector<3x1xf32> into vector<2x3xf32> return %0 : vector<2x3xf32> } @@ -231,28 +266,18 @@ func.func @matmul_1(%arg0: vector<2x1xf32>, // CHECK: %[[b0:.*]] = vector.extract %[[Bt]][0] : vector<[3]xf32> from vector<1x[3]xf32> // CHECK: %[[c0:.*]] = vector.outerproduct %[[a0]], %[[b0]], %[[C]] // CHECK: return %[[c0]] : vector<2x[3]xf32> -func.func @matmul_1_scalable(%arg0: vector<2x1xf32>, - %arg1: vector<[3]x1xf32>, - %arg2: vector<2x[3]xf32>) -> vector<2x[3]xf32> +func.func @matmul_1_scalable(%A: vector<2x1xf32>, + %B: vector<[3]x1xf32>, + %C: vector<2x[3]xf32>) -> vector<2x[3]xf32> { - %0 = vector.contract #matmat_trait_1 %arg0, %arg1, %arg2 + %0 = vector.contract #matmat_trait_1 %A, %B, %C : vector<2x1xf32>, vector<[3]x1xf32> into vector<2x[3]xf32> return %0 : vector<2x[3]xf32> } // ============================================================================ -// Matmul 2 (plain) +// Matmul 2 (plain + scalable) // ============================================================================ -#matmat_accesses_2 = [ - affine_map<(m, n, k) -> (k, m)>, - affine_map<(m, n, k) -> (k, n)>, - affine_map<(m, n, k) -> (m, n)> -] -#matmat_trait_2 = { - indexing_maps = #matmat_accesses_2, - iterator_types = ["parallel", "parallel", "reduction"] -} - // CHECK-LABEL: func @matmul_2 // CHECK-SAME: %[[A:[a-zA-Z0-9]*]]: vector<1x2xf32>, // CHECK-SAME: %[[B:[a-zA-Z0-9]*]]: vector<1x3xf32>, @@ -261,11 +286,11 @@ func.func @matmul_1_scalable(%arg0: vector<2x1xf32>, // CHECK: %[[b0:.*]] = vector.extract %[[B]][0] : vector<3xf32> from vector<1x3xf32> // CHECK: %[[c0:.*]] = vector.outerproduct %[[a0]], %[[b0]], %[[C]] // CHECK: return %[[c0]] : vector<2x3xf32> -func.func @matmul_2(%arg0: vector<1x2xf32>, - %arg1: vector<1x3xf32>, - %arg2: vector<2x3xf32>) -> vector<2x3xf32> +func.func @matmul_2(%A: vector<1x2xf32>, + %B: vector<1x3xf32>, + %C: vector<2x3xf32>) -> vector<2x3xf32> { - %0 = vector.contract #matmat_trait_2 %arg0, %arg1, %arg2 + %0 = vector.contract #matmat_trait_2 %A, %B, %C : vector<1x2xf32>, vector<1x3xf32> into vector<2x3xf32> return %0 : vector<2x3xf32> } @@ -278,28 +303,18 @@ func.func @matmul_2(%arg0: vector<1x2xf32>, // CHECK: %[[b0:.*]] = vector.extract %[[B]][0] : vector<[3]xf32> from vector<1x[3]xf32> // CHECK: %[[c0:.*]] = vector.outerproduct %[[a0]], %[[b0]], %[[C]] // CHECK: return %[[c0]] : vector<2x[3]xf32> -func.func @matmul_2_scalable(%arg0: vector<1x2xf32>, - %arg1: vector<1x[3]xf32>, - %arg2: vector<2x[3]xf32>) -> vector<2x[3]xf32> +func.func @matmul_2_scalable(%A: vector<1x2xf32>, + %B: vector<1x[3]xf32>, + %C: vector<2x[3]xf32>) -> vector<2x[3]xf32> { - %0 = vector.contract #matmat_trait_2 %arg0, %arg1, %arg2 + %0 = vector.contract #matmat_trait_2 %A, %B, %C : vector<1x2xf32>, vector<1x[3]xf32> into vector<2x[3]xf32> return %0 : vector<2x[3]xf32> } // ============================================================================ -// Matmul 3 (plain) +// Matmul 3 (plain + scalable) // ============================================================================ -#matmat_accesses_3 = [ - affine_map<(m, n, k) -> (k, m)>, - affine_map<(m, n, k) -> (n, k)>, - affine_map<(m, n, k) -> (m, n)> -] -#matmat_trait_3 = { - indexing_maps = #matmat_accesses_3, - iterator_types = ["parallel", "parallel", "reduction"] -} - // CHECK-LABEL: func @matmul_3 // CHECK-SAME: %[[A:[a-zA-Z0-9]*]]: vector<1x2xf32>, // CHECK-SAME: %[[B:[a-zA-Z0-9]*]]: vector<3x1xf32>, @@ -309,11 +324,11 @@ func.func @matmul_2_scalable(%arg0: vector<1x2xf32>, // CHECK: %[[b0:.*]] = vector.extract %[[Bt]][0] : vector<3xf32> from vector<1x3xf32> // CHECK: %[[c0:.*]] = vector.outerproduct %[[a0]], %[[b0]], %[[C]] // CHECK: return %[[c0]] : vector<2x3xf32> -func.func @matmul_3(%arg0: vector<1x2xf32>, - %arg1: vector<3x1xf32>, - %arg2: vector<2x3xf32>) -> vector<2x3xf32> +func.func @matmul_3(%A: vector<1x2xf32>, + %B: vector<3x1xf32>, + %C: vector<2x3xf32>) -> vector<2x3xf32> { - %0 = vector.contract #matmat_trait_3 %arg0, %arg1, %arg2 + %0 = vector.contract #matmat_trait_3 %A, %B, %C : vector<1x2xf32>, vector<3x1xf32> into vector<2x3xf32> return %0 : vector<2x3xf32> } @@ -327,28 +342,18 @@ func.func @matmul_3(%arg0: vector<1x2xf32>, // CHECK: %[[b0:.*]] = vector.extract %[[Bt]][0] : vector<[3]xf32> from vector<1x[3]xf32> // CHECK: %[[c0:.*]] = vector.outerproduct %[[a0]], %[[b0]], %[[C]] // CHECK: return %[[c0]] : vector<2x[3]xf32> -func.func @matmul_3_scalable(%arg0: vector<1x2xf32>, - %arg1: vector<[3]x1xf32>, - %arg2: vector<2x[3]xf32>) -> vector<2x[3]xf32> +func.func @matmul_3_scalable(%A: vector<1x2xf32>, + %B: vector<[3]x1xf32>, + %C: vector<2x[3]xf32>) -> vector<2x[3]xf32> { - %0 = vector.contract #matmat_trait_3 %arg0, %arg1, %arg2 + %0 = vector.contract #matmat_trait_3 %A, %B, %C : vector<1x2xf32>, vector<[3]x1xf32> into vector<2x[3]xf32> return %0 : vector<2x[3]xf32> } // ============================================================================ -// Matmul 4 (plain) +// Matmul 4 (plain + scalable) // ============================================================================ -#matmat_accesses_4 = [ - affine_map<(m, n, k) -> (m, k)>, - affine_map<(m, n, k) -> (k, n)>, - affine_map<(m, n, k) -> (n, m)> -] -#matmat_trait_4 = { - indexing_maps = #matmat_accesses_4, - iterator_types = ["parallel", "parallel", "reduction"] -} - // CHECK-LABEL: func @matmul_4 // CHECK-SAME: %[[A:[a-zA-Z0-9]*]]: vector<2x1xf32>, // CHECK-SAME: %[[B:[a-zA-Z0-9]*]]: vector<1x3xf32>, @@ -358,11 +363,11 @@ func.func @matmul_3_scalable(%arg0: vector<1x2xf32>, // CHECK: %[[a0:.*]] = vector.extract %[[At]][0] : vector<2xf32> from vector<1x2xf32> // CHECK: %[[c0:.*]] = vector.outerproduct %[[b0]], %[[a0]], %[[C]] // CHECK: return %[[c0]] : vector<3x2xf32> -func.func @matmul_4(%arg0: vector<2x1xf32>, - %arg1: vector<1x3xf32>, - %arg2: vector<3x2xf32>) -> vector<3x2xf32> +func.func @matmul_4(%A: vector<2x1xf32>, + %B: vector<1x3xf32>, + %C: vector<3x2xf32>) -> vector<3x2xf32> { - %0 = vector.contract #matmat_trait_4 %arg0, %arg1, %arg2 + %0 = vector.contract #matmat_trait_4 %A, %B, %C : vector<2x1xf32>, vector<1x3xf32> into vector<3x2xf32> return %0 : vector<3x2xf32> } @@ -376,11 +381,11 @@ func.func @matmul_4(%arg0: vector<2x1xf32>, // CHECK: %[[a0:.*]] = vector.extract %[[At]][0] : vector<[2]xf32> from vector<1x[2]xf32> // CHECK: %[[c0:.*]] = vector.outerproduct %[[b0]], %[[a0]], %[[C]] // CHECK: return %[[c0]] : vector<3x[2]xf32> -func.func @matmul_4_scalable(%arg0: vector<[2]x1xf32>, - %arg1: vector<1x3xf32>, - %arg2: vector<3x[2]xf32>) -> vector<3x[2]xf32> +func.func @matmul_4_scalable(%A: vector<[2]x1xf32>, + %B: vector<1x3xf32>, + %C: vector<3x[2]xf32>) -> vector<3x[2]xf32> { - %0 = vector.contract #matmat_trait_4 %arg0, %arg1, %arg2 + %0 = vector.contract #matmat_trait_4 %A, %B, %C : vector<[2]x1xf32>, vector<1x3xf32> into vector<3x[2]xf32> return %0 : vector<3x[2]xf32> } diff --git a/mlir/test/Dialect/Vector/vector-contract-to-outerproduct-matvec-transforms.mlir b/mlir/test/Dialect/Vector/vector-contract-to-outerproduct-matvec-transforms.mlir index c09a4d569638..d86c6158bcdf 100644 --- a/mlir/test/Dialect/Vector/vector-contract-to-outerproduct-matvec-transforms.mlir +++ b/mlir/test/Dialect/Vector/vector-contract-to-outerproduct-matvec-transforms.mlir @@ -235,6 +235,23 @@ func.func @masked_matvec_mk_k_m_max_scalable_parallel_dim(%A: vector<[2]x3xf32>, // ============================================================================ // Matvec 2 (plain + masked + scalable) // ============================================================================ +// CHECK-LABEL: func @matvec_km_k_m +// CHECK-SAME: %[[A:.*0]]: vector<2x2xf32> +// CHECK-SAME: %[[X:.*1]]: vector<2xf32> +// CHECK-SAME: %[[B:.*2]]: vector<2xf32> +// CHECK: %[[T3:.*]] = vector.extract %[[A]][0] : vector<2xf32> from vector<2x2xf32> +// CHECK: %[[T4:.*]] = vector.extract %[[X]][0] : f32 from vector<2xf32> +// CHECK: %[[T5:.*]] = vector.outerproduct %[[T3]], %[[T4]], %[[B]] {kind = #vector.kind<add>} : vector<2xf32>, f32 +// CHECK: %[[T6:.*]] = vector.extract %[[A]][1] : vector<2xf32> from vector<2x2xf32> +// CHECK: %[[T7:.*]] = vector.extract %[[X]][1] : f32 from vector<2xf32> +// CHECK: %[[T8:.*]] = vector.outerproduct %[[T6]], %[[T7]], %[[T5]] {kind = #vector.kind<add>} : vector<2xf32>, f32 +func.func @matvec_km_k_m(%A: vector<2x2xf32>, + %x: vector<2xf32>, + %b: vector<2xf32>) -> vector<2xf32> { + %0 = vector.contract #matvec_trait_2 %A, %x, %b : vector<2x2xf32>, vector<2xf32> into vector<2xf32> + return %0 : vector<2xf32> +} + // CHECK-LABEL: @masked_matvec_km_k_m // CHECK-SAME: %[[A:.+]]: vector<2x4xf32> // CHECK-SAME: %[[X:.+]]: vector<2xf32> @@ -273,26 +290,27 @@ func.func @masked_matvec_km_k_m_scalable_parallel_dim(%A: vector<2x[4]xf32>, return %res : vector<[4]xf32> } -// CHECK-LABEL: func @matvec_km_k_m +// ============================================================================ +// Matvec 3 (plain + masked + scalable) +// ============================================================================ +// CHECK-LABEL: func @matvec_k_mk_m // CHECK-SAME: %[[A:.*0]]: vector<2x2xf32> // CHECK-SAME: %[[X:.*1]]: vector<2xf32> // CHECK-SAME: %[[B:.*2]]: vector<2xf32> -// CHECK: %[[T3:.*]] = vector.extract %[[A]][0] : vector<2xf32> from vector<2x2xf32> -// CHECK: %[[T4:.*]] = vector.extract %[[X]][0] : f32 from vector<2xf32> -// CHECK: %[[T5:.*]] = vector.outerproduct %[[T3]], %[[T4]], %[[B]] {kind = #vector.kind<add>} : vector<2xf32>, f32 -// CHECK: %[[T6:.*]] = vector.extract %[[A]][1] : vector<2xf32> from vector<2x2xf32> -// CHECK: %[[T7:.*]] = vector.extract %[[X]][1] : f32 from vector<2xf32> -// CHECK: %[[T8:.*]] = vector.outerproduct %[[T6]], %[[T7]], %[[T5]] {kind = #vector.kind<add>} : vector<2xf32>, f32 -func.func @matvec_km_k_m(%A: vector<2x2xf32>, +// CHECK: %[[T3:.*]] = vector.transpose %[[A]], [1, 0] : vector<2x2xf32> to vector<2x2xf32> +// CHECK: %[[T4:.*]] = vector.extract %[[T3]][0] : vector<2xf32> from vector<2x2xf32> +// CHECK: %[[T5:.*]] = vector.extract %[[X]][0] : f32 from vector<2xf32> +// CHECK: %[[T6:.*]] = vector.outerproduct %[[T4]], %[[T5]], %[[B]] {kind = #vector.kind<add>} : vector<2xf32>, f32 +// CHECK: %[[T7:.*]] = vector.extract %[[T3]][1] : vector<2xf32> from vector<2x2xf32> +// CHECK: %[[T8:.*]] = vector.extract %[[X]][1] : f32 from vector<2xf32> +// CHECK: %[[T9:.*]] = vector.outerproduct %[[T7]], %[[T8]], %[[T6]] {kind = #vector.kind<add>} : vector<2xf32>, f32 +func.func @matvec_k_mk_m(%A: vector<2x2xf32>, %x: vector<2xf32>, %b: vector<2xf32>) -> vector<2xf32> { - %0 = vector.contract #matvec_trait_2 %A, %x, %b : vector<2x2xf32>, vector<2xf32> into vector<2xf32> + %0 = vector.contract #matvec_trait_3 %x, %A, %b : vector<2xf32>, vector<2x2xf32> into vector<2xf32> return %0 : vector<2xf32> } -// ============================================================================ -// Matvec 3 (plain + masked + scalable) -// ============================================================================ // CHECK-LABEL: @masked_matvec_k_mk_m // CHECK-SAME: %[[A:.+]]: vector<4x2xf32> // CHECK-SAME: %[[X:.+]]: vector<2xf32> @@ -331,24 +349,6 @@ func.func @masked_matvec_k_mk_m_scalable_parallel_dim(%A: vector<[4]x2xf32>, return %res : vector<[4]xf32> } -// CHECK-LABEL: func @matvec_k_mk_m -// CHECK-SAME: %[[A:.*0]]: vector<2x2xf32> -// CHECK-SAME: %[[X:.*1]]: vector<2xf32> -// CHECK-SAME: %[[B:.*2]]: vector<2xf32> -// CHECK: %[[T3:.*]] = vector.transpose %[[A]], [1, 0] : vector<2x2xf32> to vector<2x2xf32> -// CHECK: %[[T4:.*]] = vector.extract %[[T3]][0] : vector<2xf32> from vector<2x2xf32> -// CHECK: %[[T5:.*]] = vector.extract %[[X]][0] : f32 from vector<2xf32> -// CHECK: %[[T6:.*]] = vector.outerproduct %[[T4]], %[[T5]], %[[B]] {kind = #vector.kind<add>} : vector<2xf32>, f32 -// CHECK: %[[T7:.*]] = vector.extract %[[T3]][1] : vector<2xf32> from vector<2x2xf32> -// CHECK: %[[T8:.*]] = vector.extract %[[X]][1] : f32 from vector<2xf32> -// CHECK: %[[T9:.*]] = vector.outerproduct %[[T7]], %[[T8]], %[[T6]] {kind = #vector.kind<add>} : vector<2xf32>, f32 -func.func @matvec_k_mk_m(%A: vector<2x2xf32>, - %x: vector<2xf32>, - %b: vector<2xf32>) -> vector<2xf32> { - %0 = vector.contract #matvec_trait_3 %x, %A, %b : vector<2xf32>, vector<2x2xf32> into vector<2xf32> - return %0 : vector<2xf32> -} - // ============================================================================ // Matvec 4 (plain + masked + scalable) // ============================================================================ diff --git a/mlir/test/Transforms/canonicalize-debuginfo.mlir b/mlir/test/Transforms/canonicalize-debuginfo.mlir new file mode 100644 index 000000000000..30c8022daa76 --- /dev/null +++ b/mlir/test/Transforms/canonicalize-debuginfo.mlir @@ -0,0 +1,46 @@ +// RUN: mlir-opt %s -pass-pipeline='builtin.module(func.func(canonicalize{test-convergence}))' -split-input-file -mlir-print-debuginfo | FileCheck %s + +// CHECK-LABEL: func @merge_constants +func.func @merge_constants() -> (index, index, index, index) { + // CHECK-NEXT: arith.constant 42 : index loc(#[[UnknownLoc:.*]]) + %0 = arith.constant 42 : index loc("merge_constants":0:0) + %1 = arith.constant 42 : index loc("merge_constants":1:0) + %2 = arith.constant 42 : index loc("merge_constants":2:0) + %3 = arith.constant 42 : index loc("merge_constants":2:0) + return %0, %1, %2, %3 : index, index, index, index +} +// CHECK: #[[UnknownLoc]] = loc(unknown) + +// ----- + +// CHECK-LABEL: func @simple_hoist +func.func @simple_hoist(%arg0: memref<8xi32>) -> i32 { + // CHECK: arith.constant 88 : i32 loc(#[[UnknownLoc:.*]]) + // CHECK: arith.constant 42 : i32 loc(#[[ConstLoc0:.*]]) + // CHECK: arith.constant 0 : index loc(#[[ConstLoc1:.*]]) + %0 = arith.constant 42 : i32 loc("simple_hoist":0:0) + %1 = arith.constant 0 : index loc("simple_hoist":1:0) + memref.store %0, %arg0[%1] : memref<8xi32> + + %2 = arith.constant 88 : i32 loc("simple_hoist":2:0) + + return %2 : i32 +} +// CHECK-DAG: #[[UnknownLoc]] = loc(unknown) +// CHECK-DAG: #[[ConstLoc0]] = loc("simple_hoist":0:0) +// CHECK-DAG: #[[ConstLoc1]] = loc("simple_hoist":1:0) + +// ----- + +// CHECK-LABEL: func @hoist_and_merge +func.func @hoist_and_merge(%arg0: memref<8xi32>) { + // CHECK-NEXT: arith.constant 42 : i32 loc(#[[UnknownLoc:.*]]) + affine.for %arg1 = 0 to 8 { + %0 = arith.constant 42 : i32 loc("hoist_and_merge":0:0) + %1 = arith.constant 42 : i32 loc("hoist_and_merge":1:0) + memref.store %0, %arg0[%arg1] : memref<8xi32> + memref.store %1, %arg0[%arg1] : memref<8xi32> + } + return +} loc("hoist_and_merge":2:0) +// CHECK: #[[UnknownLoc]] = loc(unknown) diff --git a/mlir/test/Transforms/constant-fold-debuginfo.mlir b/mlir/test/Transforms/constant-fold-debuginfo.mlir new file mode 100644 index 000000000000..c308bc477bee --- /dev/null +++ b/mlir/test/Transforms/constant-fold-debuginfo.mlir @@ -0,0 +1,42 @@ +// RUN: mlir-opt %s -split-input-file -test-constant-fold -mlir-print-debuginfo | FileCheck %s + +// CHECK-LABEL: func @fold_and_merge +func.func @fold_and_merge() -> (i32, i32) { + // CHECK-NEXT: [[C:%.+]] = arith.constant 6 : i32 loc(#[[UnknownLoc:.*]]) + %0 = arith.constant 1 : i32 loc("fold_and_merge":0:0) + %1 = arith.constant 5 : i32 loc("fold_and_merge":1:0) + %2 = arith.addi %0, %1 : i32 loc("fold_and_merge":2:0) + + %3 = arith.constant 6 : i32 loc("fold_and_merge":3:0) + + return %2, %3: i32, i32 +} +// CHECK: #[[UnknownLoc]] = loc(unknown) + +// ----- + +// CHECK-LABEL: func @materialize_different_dialect +func.func @materialize_different_dialect() -> (f32, f32) { + // CHECK: arith.constant 1.{{0*}}e+00 : f32 loc(#[[UnknownLoc:.*]]) + %0 = arith.constant -1.0 : f32 loc("materialize_different_dialect":0:0) + %1 = math.absf %0 : f32 loc("materialize_different_dialect":1:0) + %2 = arith.constant 1.0 : f32 loc("materialize_different_dialect":2:0) + + return %1, %2: f32, f32 +} +// CHECK: #[[UnknownLoc]] = loc(unknown) + +// ----- + +// CHECK-LABEL: func @materialize_in_front +func.func @materialize_in_front(%arg0: memref<8xi32>) { + // CHECK-NEXT: arith.constant 6 : i32 loc(#[[UnknownLoc:.*]]) + affine.for %arg1 = 0 to 8 { + %1 = arith.constant 1 : i32 loc("materialize_in_front":0:0) + %2 = arith.constant 5 : i32 loc("materialize_in_front":1:0) + %3 = arith.addi %1, %2 : i32 loc("materialize_in_front":2:0) + memref.store %3, %arg0[%arg1] : memref<8xi32> + } + return +} loc("materialize_in_front":3:0) +// CHECK: #[[UnknownLoc]] = loc(unknown) diff --git a/mlir/test/lib/Analysis/TestDataFlowFramework.cpp b/mlir/test/lib/Analysis/TestDataFlowFramework.cpp index ed361b5a0e27..b6b33182440c 100644 --- a/mlir/test/lib/Analysis/TestDataFlowFramework.cpp +++ b/mlir/test/lib/Analysis/TestDataFlowFramework.cpp @@ -100,7 +100,7 @@ LogicalResult FooAnalysis::initialize(Operation *top) { return top->emitError("expected at least one block in the region"); // Initialize the top-level state. - getOrCreate<FooState>(&top->getRegion(0).front())->join(0); + (void)getOrCreate<FooState>(&top->getRegion(0).front())->join(0); // Visit all nested blocks and operations. for (Block &block : top->getRegion(0)) { diff --git a/mlir/test/lib/Transforms/TestIntRangeInference.cpp b/mlir/test/lib/Transforms/TestIntRangeInference.cpp index 2f6dd5b8095d..5758f6acf2f0 100644 --- a/mlir/test/lib/Transforms/TestIntRangeInference.cpp +++ b/mlir/test/lib/Transforms/TestIntRangeInference.cpp @@ -40,9 +40,8 @@ static LogicalResult replaceWithConstant(DataFlowSolver &solver, OpBuilder &b, maybeDefiningOp ? maybeDefiningOp->getDialect() : value.getParentRegion()->getParentOp()->getDialect(); Attribute constAttr = b.getIntegerAttr(value.getType(), *maybeConstValue); - Value constant = - folder.getOrCreateConstant(b.getInsertionBlock(), valueDialect, constAttr, - value.getType(), value.getLoc()); + Value constant = folder.getOrCreateConstant( + b.getInsertionBlock(), valueDialect, constAttr, value.getType()); if (!constant) return failure(); diff --git a/mlir/test/python/dialects/arith_dialect.py b/mlir/test/python/dialects/arith_dialect.py index f80f2c084a0f..8bb80eed2b81 100644 --- a/mlir/test/python/dialects/arith_dialect.py +++ b/mlir/test/python/dialects/arith_dialect.py @@ -75,7 +75,7 @@ def testArithValue(): f64_t = F64Type.get() with InsertionPoint(module.body): - a = arith.constant(value=FloatAttr.get(f16_t, 42.42)) + a = arith.constant(f16_t, 42.42) # CHECK: ArithValue(%cst = arith.constant 4.240 print(a) @@ -83,12 +83,12 @@ def testArithValue(): # CHECK: ArithValue(%0 = arith.addf %cst, %cst : f16) print(b) - a = arith.constant(value=FloatAttr.get(f32_t, 42.42)) + a = arith.constant(f32_t, 42.42) b = a - a # CHECK: ArithValue(%1 = arith.subf %cst_0, %cst_0 : f32) print(b) - a = arith.constant(value=FloatAttr.get(f64_t, 42.42)) + a = arith.constant(f64_t, 42.42) b = a * a # CHECK: ArithValue(%2 = arith.mulf %cst_1, %cst_1 : f64) print(b) diff --git a/mlir/test/python/dialects/tensor.py b/mlir/test/python/dialects/tensor.py index b690c934dc46..ca9066b23911 100644 --- a/mlir/test/python/dialects/tensor.py +++ b/mlir/test/python/dialects/tensor.py @@ -4,6 +4,7 @@ from mlir.ir import * import mlir.dialects.arith as arith import mlir.dialects.func as func import mlir.dialects.tensor as tensor +from mlir.extras import types as T def run(f): @@ -139,3 +140,37 @@ def testFromElementsOp(): t = tensor.FromElementsOp(RankedTensorType.get((1, 2), f32), [c0, c1]) # CHECK: %{{.*}} = "tensor.from_elements"(%[[C0]], %[[C1]]) : (f32, f32) -> tensor<1x2xf32> print(t) + + +# CHECK-LABEL: TEST: testGenerateRegionOp +@run +def testGenerateRegionOp(): + S = ShapedType.get_dynamic_size() + with Context(), Location.unknown(): + module = Module.create() + with InsertionPoint(module.body): + # CHECK: %[[VAL_0:.*]] = arith.constant 1 : index + # CHECK: %[[VAL_1:.*]] = arith.constant 2 : index + one = arith.constant(T.index(), 1) + two = arith.constant(T.index(), 2) + + @tensor.generate(T.tensor(S, 3, S, T.index()), dynamic_extents=[one, two]) + def generate_one(i: T.index(), j: T.index(), k: T.index()): + ij = arith.addi(i, j) + ijk = arith.addi(ij, k) + return ijk + + assert ( + isinstance(generate_one, Value) + and generate_one.owner.name == "tensor.generate" + ) + + # CHECK: %[[GENERATED:.*]] = tensor.generate + # CHECK-SAME: %[[VAL_0]], + # CHECK-SAME: %[[VAL_1]] { + # CHECK: ^bb0(%[[VAL_1:.*]]: index, %[[VAL_2:.*]]: index, %[[VAL_3:.*]]: index): + # CHECK: %[[VAL_4:.*]] = arith.addi %[[VAL_1]], %[[VAL_2]] : index + # CHECK: %[[VAL_5:.*]] = arith.addi %[[VAL_4]], %[[VAL_3]] : index + # CHECK: tensor.yield %[[VAL_5]] : index + # CHECK: } : tensor<?x3x?xindex> + print(module) diff --git a/mlir/test/python/dialects/transform_extras.py b/mlir/test/python/dialects/transform_extras.py index e7b43ea63c31..358f8c32f75c 100644 --- a/mlir/test/python/dialects/transform_extras.py +++ b/mlir/test/python/dialects/transform_extras.py @@ -2,9 +2,34 @@ from typing import Callable from mlir import ir -from mlir.dialects import scf -from mlir.dialects.transform import structured -from mlir.dialects.transform.extras import OpHandle, insert_transform_script +from mlir.dialects import scf, pdl +from mlir.dialects.transform import ( + structured, + get_parent_op, + apply_patterns_canonicalization, + apply_cse, + any_op_t, +) +from mlir.dialects.transform import FailurePropagationMode +from mlir.dialects.transform.structured import structured_match +from mlir.dialects.transform.loop import loop_unroll +from mlir.dialects.transform.extras import ( + OpHandle, + insert_transform_script, + sequence, + apply_patterns, +) +from mlir.extras import types as T + + +def construct_and_print_in_module(f): + print("\nTEST:", f.__name__) + with ir.Context(), ir.Location.unknown(): + module = ir.Module.create() + with ir.InsertionPoint(module.body): + f() + print(module) + return f def build_transform_script(script: Callable[[OpHandle], None]): @@ -93,3 +118,45 @@ def test_match_ops_mixed(op: OpHandle): # CHECK-NEXT: %[[VAL_1:.*]] = transform.structured.match # CHECK-SAME: ops{["scf.for", "linalg.matmul", "scf.forall"]} in %[[VAL_0]] # CHECK-SAME: -> !transform.any_op + + +# CHECK-LABEL: TEST: test_sequence_region +@construct_and_print_in_module +def test_sequence_region(): + # CHECK: transform.sequence failures(propagate) { + # CHECK: ^{{.*}}(%[[VAL_0:.*]]: !transform.any_op): + # CHECK: %[[VAL_1:.*]] = transform.structured.match ops{["arith.addi"]} in %[[VAL_0]] : (!transform.any_op) -> !transform.any_op + # CHECK: %[[VAL_2:.*]] = get_parent_op %[[VAL_1]] {op_name = "scf.for"} : (!transform.any_op) -> !pdl.operation + # CHECK: transform.loop.unroll %[[VAL_2]] {factor = 4 : i64} : !pdl.operation + # CHECK: } + @sequence([], FailurePropagationMode.Propagate, []) + def basic(target: any_op_t()): + m = structured_match(any_op_t(), target, ops=["arith.addi"]) + loop = get_parent_op(pdl.op_t(), m, op_name="scf.for") + loop_unroll(loop, 4) + + +# CHECK-LABEL: TEST: test_apply_patterns +@construct_and_print_in_module +def test_apply_patterns(): + # CHECK: transform.sequence failures(propagate) { + # CHECK: ^{{.*}}(%[[VAL_0:.*]]: !transform.any_op): + # CHECK: %[[VAL_1:.*]] = transform.structured.match ops{["linalg.matmul"]} in %[[VAL_0]] : (!transform.any_op) -> !transform.any_op + # CHECK: %[[VAL_2:.*]] = get_parent_op %[[VAL_1]] {op_name = "func.func"} : (!transform.any_op) -> !pdl.operation + # CHECK: apply_patterns to %[[VAL_2]] { + # CHECK: transform.apply_patterns.canonicalization + # CHECK: } : !pdl.operation + # CHECK: %[[VAL_3:.*]] = transform.structured.match ops{["func.func"]} in %[[VAL_0]] : (!transform.any_op) -> !transform.any_op + # CHECK: apply_cse to %[[VAL_3]] : !transform.any_op + # CHECK: } + @sequence([], FailurePropagationMode.Propagate, []) + def basic(variant_op: any_op_t()): + matmul = structured_match(any_op_t(), variant_op, ops=["linalg.matmul"]) + top_func = get_parent_op(pdl.op_t(), matmul, op_name="func.func") + + @apply_patterns(top_func) + def pats(): + apply_patterns_canonicalization() + + top_func = structured_match(any_op_t(), variant_op, ops=["func.func"]) + apply_cse(top_func) diff --git a/mlir/test/python/integration/dialects/transform.py b/mlir/test/python/integration/dialects/transform.py new file mode 100644 index 000000000000..bc88a61314d0 --- /dev/null +++ b/mlir/test/python/integration/dialects/transform.py @@ -0,0 +1,155 @@ +# RUN: %PYTHON %s 2>&1 | FileCheck %s + +from mlir.passmanager import PassManager +from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from mlir.dialects import scf, pdl, func, arith, linalg +from mlir.dialects.transform import ( + get_parent_op, + apply_patterns_canonicalization, + apply_cse, + any_op_t, +) +from mlir.dialects.transform.structured import structured_match +from mlir.dialects.transform.loop import loop_unroll +from mlir.dialects.transform.extras import named_sequence, apply_patterns +from mlir.extras import types as T +from mlir.dialects.builtin import module, ModuleOp + + +def construct_and_print_in_module(f): + print("\nTEST:", f.__name__) + with Context(), Location.unknown(): + module = Module.create() + with InsertionPoint(module.body): + module = f(module) + if module is not None: + print(module) + return f + + +# CHECK-LABEL: TEST: test_named_sequence +@construct_and_print_in_module +def test_named_sequence(module_): + # CHECK-LABEL: func.func @loop_unroll_op() { + # CHECK: %[[VAL_0:.*]] = arith.constant 0 : index + # CHECK: %[[VAL_1:.*]] = arith.constant 42 : index + # CHECK: %[[VAL_2:.*]] = arith.constant 5 : index + # CHECK: scf.for %[[VAL_3:.*]] = %[[VAL_0]] to %[[VAL_1]] step %[[VAL_2]] { + # CHECK: %[[VAL_4:.*]] = arith.addi %[[VAL_3]], %[[VAL_3]] : index + # CHECK: } + # CHECK: return + # CHECK: } + @func.func() + def loop_unroll_op(): + for i in scf.for_(0, 42, 5): + v = arith.addi(i, i) + scf.yield_([]) + + # CHECK-LABEL: module attributes {transform.with_named_sequence} { + # CHECK: transform.named_sequence @__transform_main(%[[VAL_0:.*]]: !transform.any_op) { + # CHECK: %[[VAL_1:.*]] = transform.structured.match ops{["arith.addi"]} in %[[VAL_0]] : (!transform.any_op) -> !transform.any_op + # CHECK: %[[VAL_2:.*]] = transform.get_parent_op %[[VAL_1]] {op_name = "scf.for"} : (!transform.any_op) -> !pdl.operation + # CHECK: transform.loop.unroll %[[VAL_2]] {factor = 4 : i64} : !pdl.operation + # CHECK: transform.yield + # CHECK: } + # CHECK: } + @module(attrs={"transform.with_named_sequence": UnitAttr.get()}) + def mod(): + @named_sequence("__transform_main", [any_op_t()], []) + def basic(target: any_op_t()): + m = structured_match(any_op_t(), target, ops=["arith.addi"]) + loop = get_parent_op(pdl.op_t(), m, op_name="scf.for") + loop_unroll(loop, 4) + + # The identifier (name) of the function becomes the Operation + assert isinstance(mod.opview, ModuleOp) + + print(module_) + + pm = PassManager.parse("builtin.module(transform-interpreter)") + pm.run(module_.operation) + + # CHECK-LABEL: func.func @loop_unroll_op() { + # CHECK: %[[VAL_0:.*]] = arith.constant 0 : index + # CHECK: %[[VAL_1:.*]] = arith.constant 42 : index + # CHECK: %[[VAL_2:.*]] = arith.constant 5 : index + # CHECK: %[[VAL_6:.*]] = arith.constant 40 : index + # CHECK: %[[VAL_7:.*]] = arith.constant 20 : index + # CHECK: scf.for %[[VAL_3:.*]] = %[[VAL_0]] to %[[VAL_6]] step %[[VAL_7]] { + # CHECK: %[[VAL_5:.*]] = arith.addi %[[VAL_3]], %[[VAL_3]] : index + # CHECK: %[[VAL_8:.*]] = arith.constant 1 : index + # CHECK: %[[VAL_9:.*]] = arith.muli %[[VAL_2]], %[[VAL_8]] : index + # CHECK: %[[VAL_10:.*]] = arith.addi %[[VAL_3]], %[[VAL_9]] : index + # CHECK: %[[VAL_11:.*]] = arith.addi %[[VAL_10]], %[[VAL_10]] : index + # CHECK: %[[VAL_12:.*]] = arith.constant 2 : index + # CHECK: %[[VAL_13:.*]] = arith.muli %[[VAL_2]], %[[VAL_12]] : index + # CHECK: %[[VAL_14:.*]] = arith.addi %[[VAL_3]], %[[VAL_13]] : index + # CHECK: %[[VAL_15:.*]] = arith.addi %[[VAL_14]], %[[VAL_14]] : index + # CHECK: %[[VAL_16:.*]] = arith.constant 3 : index + # CHECK: %[[VAL_17:.*]] = arith.muli %[[VAL_2]], %[[VAL_16]] : index + # CHECK: %[[VAL_18:.*]] = arith.addi %[[VAL_3]], %[[VAL_17]] : index + # CHECK: %[[VAL_19:.*]] = arith.addi %[[VAL_18]], %[[VAL_18]] : index + # CHECK: } + # CHECK: %[[VAL_4:.*]] = arith.addi %[[VAL_6]], %[[VAL_6]] : index + # CHECK: return + # CHECK: } + print(module_) + + +# CHECK-LABEL: TEST: test_apply_patterns +@construct_and_print_in_module +def test_apply_patterns(module_): + M, N, K = 3, 5, 3 + + # CHECK-LABEL: func.func @matmul( + # CHECK-SAME: %[[VAL_0:.*]]: tensor<3x5xf32>, %[[VAL_1:.*]]: tensor<5x3xf32>, %[[VAL_2:.*]]: tensor<3x3xf32>) -> tensor<3x3xf32> { + # CHECK: %[[VAL_3:.*]] = arith.constant 1 : i32 + # CHECK: %[[VAL_4:.*]] = arith.addi %[[VAL_3]], %[[VAL_3]] : i32 + # CHECK: %[[VAL_5:.*]] = linalg.matmul {cast = #linalg.type_fn<cast_signed>} ins(%[[VAL_0]], %[[VAL_1]] : tensor<3x5xf32>, tensor<5x3xf32>) outs(%[[VAL_2]] : tensor<3x3xf32>) -> tensor<3x3xf32> + # CHECK: return %[[VAL_5]] : tensor<3x3xf32> + # CHECK: } + @func.func( + T.tensor(M, N, T.f32()), T.tensor(N, K, T.f32()), T.tensor(M, K, T.f32()) + ) + def matmul(A, B, C): + i = arith.constant(T.i32(), 1) + v = arith.addi(i, i) + return linalg.matmul(A, B, outs=[C]) + + # CHECK-LABEL: module attributes {transform.with_named_sequence} { + # CHECK: transform.named_sequence @__transform_main(%[[VAL_0:.*]]: !transform.any_op) { + # CHECK: %[[VAL_1:.*]] = transform.structured.match ops{["linalg.matmul"]} in %[[VAL_0]] : (!transform.any_op) -> !transform.any_op + # CHECK: %[[VAL_2:.*]] = transform.get_parent_op %[[VAL_1]] {op_name = "func.func"} : (!transform.any_op) -> !pdl.operation + # CHECK: transform.apply_patterns to %[[VAL_2]] { + # CHECK: transform.apply_patterns.canonicalization + # CHECK: } : !pdl.operation + # CHECK: %[[VAL_3:.*]] = transform.structured.match ops{["func.func"]} in %[[VAL_0]] : (!transform.any_op) -> !transform.any_op + # CHECK: transform.apply_cse to %[[VAL_3]] : !transform.any_op + # CHECK: transform.yield + # CHECK: } + # CHECK: } + @module(attrs={"transform.with_named_sequence": UnitAttr.get()}) + def mod(): + @named_sequence("__transform_main", [any_op_t()], []) + def basic(variant_op: any_op_t()): + matmul = structured_match(any_op_t(), variant_op, ops=["linalg.matmul"]) + top_func = get_parent_op(pdl.op_t(), matmul, op_name="func.func") + + @apply_patterns(top_func) + def pats(): + apply_patterns_canonicalization() + + top_func = structured_match(any_op_t(), variant_op, ops=["func.func"]) + apply_cse(top_func) + + print(module_) + + pm = PassManager.parse("builtin.module(transform-interpreter)") + pm.run(module_.operation) + + # CHECK-LABEL: func.func @matmul( + # CHECK-SAME: %[[VAL_0:.*]]: tensor<3x5xf32>, %[[VAL_1:.*]]: tensor<5x3xf32>, %[[VAL_2:.*]]: tensor<3x3xf32>) -> tensor<3x3xf32> { + # CHECK: %[[VAL_3:.*]] = linalg.matmul {cast = #linalg.type_fn<cast_signed>} ins(%[[VAL_0]], %[[VAL_1]] : tensor<3x5xf32>, tensor<5x3xf32>) outs(%[[VAL_2]] : tensor<3x3xf32>) -> tensor<3x3xf32> + # CHECK: return %[[VAL_3]] : tensor<3x3xf32> + # CHECK: } + print(module_) diff --git a/mlir/test/python/ir/context_lifecycle.py b/mlir/test/python/ir/context_lifecycle.py index fbd1851ba70a..c20270999425 100644 --- a/mlir/test/python/ir/context_lifecycle.py +++ b/mlir/test/python/ir/context_lifecycle.py @@ -45,46 +45,5 @@ assert mlir.ir.Context._get_live_count() == 0 c4 = mlir.ir.Context() c4_capsule = c4._CAPIPtr assert '"mlir.ir.Context._CAPIPtr"' in repr(c4_capsule) -# Because the context is already owned by Python, it cannot be created -# a second time. -try: - c5 = mlir.ir.Context._CAPICreate(c4_capsule) -except RuntimeError: - pass -else: - raise AssertionError( - "Should have gotten a RuntimeError when attempting to " - "re-create an already owned context" - ) -c4 = None -c4_capsule = None -gc.collect() -assert mlir.ir.Context._get_live_count() == 0 - -# Use a private testing method to create an unowned context capsule and -# import it. -c6_capsule = mlir.ir.Context._testing_create_raw_context_capsule() -c6 = mlir.ir.Context._CAPICreate(c6_capsule) -assert mlir.ir.Context._get_live_count() == 1 -c6_capsule = None -c6 = None -gc.collect() -assert mlir.ir.Context._get_live_count() == 0 - -# Also test operation import/export as it is tightly coupled to the context. -( - raw_context_capsule, - raw_operation_capsule, -) = mlir.ir.Operation._testing_create_raw_capsule("builtin.module {}") -assert '"mlir.ir.Operation._CAPIPtr"' in repr(raw_operation_capsule) -# Attempting to import an operation for an unknown context should fail. -try: - mlir.ir.Operation._CAPICreate(raw_operation_capsule) -except RuntimeError: - pass -else: - raise AssertionError("Expected exception for unknown context") - -# Try again having imported the context. -c7 = mlir.ir.Context._CAPICreate(raw_context_capsule) -op7 = mlir.ir.Operation._CAPICreate(raw_operation_capsule) +c5 = mlir.ir.Context._CAPICreate(c4_capsule) +assert c4 is c5 diff --git a/mlir/test/python/ir/operation.py b/mlir/test/python/ir/operation.py index f59b1a26ba48..04f8a9936e31 100644 --- a/mlir/test/python/ir/operation.py +++ b/mlir/test/python/ir/operation.py @@ -844,6 +844,19 @@ def testOperationName(): print(op.operation.name) +# CHECK-LABEL: TEST: testCapsuleConversions +@run +def testCapsuleConversions(): + ctx = Context() + ctx.allow_unregistered_dialects = True + with Location.unknown(ctx): + m = Operation.create("custom.op1").operation + m_capsule = m._CAPIPtr + assert '"mlir.ir.Operation._CAPIPtr"' in repr(m_capsule) + m2 = Operation._CAPICreate(m_capsule) + assert m2 is m + + # CHECK-LABEL: TEST: testOperationErase @run def testOperationErase(): diff --git a/openmp/libomptarget/CMakeLists.txt b/openmp/libomptarget/CMakeLists.txt index 7060e20af093..31e475d86960 100644 --- a/openmp/libomptarget/CMakeLists.txt +++ b/openmp/libomptarget/CMakeLists.txt @@ -153,6 +153,8 @@ add_subdirectory(test) # Add unit tests if GMock/GTest is present if (EXISTS ${LLVM_THIRD_PARTY_DIR}/unittest) - add_subdirectory(${LLVM_THIRD_PARTY_DIR}/unittest ${CMAKE_CURRENT_BINARY_DIR}/third-party/unittest) + if (NOT TARGET llvm_gtest) + add_subdirectory(${LLVM_THIRD_PARTY_DIR}/unittest ${CMAKE_CURRENT_BINARY_DIR}/third-party/unittest) + endif() add_subdirectory(unittests) endif() diff --git a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel index 049098b158f2..5882a311c9b6 100644 --- a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel @@ -1451,7 +1451,7 @@ filegroup( filegroup( name = "TransformExtrasPackagePyFiles", - srcs = glob(["mlir/extras/dialects/transform/*.py"]), + srcs = glob(["mlir/dialects/transform/extras/*.py"]), ) ##---------------------------------------------------------------------------## |