diff options
Diffstat (limited to 'lib/CodeGen/CGLoopInfo.cpp')
| -rw-r--r-- | lib/CodeGen/CGLoopInfo.cpp | 551 |
1 files changed, 454 insertions, 97 deletions
diff --git a/lib/CodeGen/CGLoopInfo.cpp b/lib/CodeGen/CGLoopInfo.cpp index fd0a9c773a..b2bc42bfa0 100644 --- a/lib/CodeGen/CGLoopInfo.cpp +++ b/lib/CodeGen/CGLoopInfo.cpp @@ -1,9 +1,8 @@ //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -19,138 +18,396 @@ using namespace clang::CodeGen; using namespace llvm; -static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs, - const llvm::DebugLoc &StartLoc, - const llvm::DebugLoc &EndLoc, MDNode *&AccGroup) { +MDNode * +LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) { + LLVMContext &Ctx = Header->getContext(); + SmallVector<Metadata *, 4> NewLoopProperties; + TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); + NewLoopProperties.push_back(TempNode.get()); + NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); - if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && - Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && - Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled && - Attrs.PipelineInitiationInterval == 0 && - Attrs.VectorizeEnable == LoopAttributes::Unspecified && - Attrs.UnrollEnable == LoopAttributes::Unspecified && - Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified && - Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc && - !EndLoc) - return nullptr; + MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties); + LoopID->replaceOperandWith(0, LoopID); + return LoopID; +} + +MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs, + ArrayRef<Metadata *> LoopProperties, + bool &HasUserTransforms) { + LLVMContext &Ctx = Header->getContext(); + + Optional<bool> Enabled; + if (Attrs.PipelineDisabled) + Enabled = false; + else if (Attrs.PipelineInitiationInterval != 0) + Enabled = true; + + if (Enabled != true) { + SmallVector<Metadata *, 4> NewLoopProperties; + if (Enabled == false) { + NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + NewLoopProperties.push_back( + MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"), + ConstantAsMetadata::get(ConstantInt::get( + llvm::Type::getInt1Ty(Ctx), 1))})); + LoopProperties = NewLoopProperties; + } + return createLoopPropertiesMetadata(LoopProperties); + } SmallVector<Metadata *, 4> Args; - // Reserve operand 0 for loop id self reference. - auto TempNode = MDNode::getTemporary(Ctx, None); + TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); Args.push_back(TempNode.get()); + Args.append(LoopProperties.begin(), LoopProperties.end()); - // If we have a valid start debug location for the loop, add it. - if (StartLoc) { - Args.push_back(StartLoc.getAsMDNode()); - - // If we also have a valid end debug location for the loop, add it. - if (EndLoc) - Args.push_back(EndLoc.getAsMDNode()); - } - - // Setting vectorize.width - if (Attrs.VectorizeWidth > 0) { - Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"), - ConstantAsMetadata::get(ConstantInt::get( - Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))}; + if (Attrs.PipelineInitiationInterval > 0) { + Metadata *Vals[] = { + MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"), + ConstantAsMetadata::get(ConstantInt::get( + llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))}; Args.push_back(MDNode::get(Ctx, Vals)); } - // Setting interleave.count - if (Attrs.InterleaveCount > 0) { - Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"), - ConstantAsMetadata::get(ConstantInt::get( - Type::getInt32Ty(Ctx), Attrs.InterleaveCount))}; - Args.push_back(MDNode::get(Ctx, Vals)); + // No follow-up: This is the last transformation. + + MDNode *LoopID = MDNode::getDistinct(Ctx, Args); + LoopID->replaceOperandWith(0, LoopID); + HasUserTransforms = true; + return LoopID; +} + +MDNode * +LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs, + ArrayRef<Metadata *> LoopProperties, + bool &HasUserTransforms) { + LLVMContext &Ctx = Header->getContext(); + + Optional<bool> Enabled; + if (Attrs.UnrollEnable == LoopAttributes::Disable) + Enabled = false; + else if (Attrs.UnrollEnable == LoopAttributes::Full) + Enabled = None; + else if (Attrs.UnrollEnable != LoopAttributes::Unspecified || + Attrs.UnrollCount != 0) + Enabled = true; + + if (Enabled != true) { + // createFullUnrollMetadata will already have added llvm.loop.unroll.disable + // if unrolling is disabled. + return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms); } + SmallVector<Metadata *, 4> FollowupLoopProperties; + + // Apply all loop properties to the unrolled loop. + FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + + // Don't unroll an already unrolled loop. + FollowupLoopProperties.push_back( + MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable"))); + + bool FollowupHasTransforms = false; + MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties, + FollowupHasTransforms); + + SmallVector<Metadata *, 4> Args; + TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); + Args.push_back(TempNode.get()); + Args.append(LoopProperties.begin(), LoopProperties.end()); + // Setting unroll.count if (Attrs.UnrollCount > 0) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"), ConstantAsMetadata::get(ConstantInt::get( - Type::getInt32Ty(Ctx), Attrs.UnrollCount))}; + llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))}; Args.push_back(MDNode::get(Ctx, Vals)); } - // Setting unroll_and_jam.count - if (Attrs.UnrollAndJamCount > 0) { - Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"), - ConstantAsMetadata::get(ConstantInt::get( - Type::getInt32Ty(Ctx), Attrs.UnrollAndJamCount))}; + // Setting unroll.full or unroll.disable + if (Attrs.UnrollEnable == LoopAttributes::Enable) { + Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")}; Args.push_back(MDNode::get(Ctx, Vals)); } - // Setting vectorize.enable - if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) { - Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"), - ConstantAsMetadata::get(ConstantInt::get( - Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable == - LoopAttributes::Enable)))}; - Args.push_back(MDNode::get(Ctx, Vals)); - } + if (FollowupHasTransforms) + Args.push_back(MDNode::get( + Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup})); - // Setting unroll.full or unroll.disable - if (Attrs.UnrollEnable != LoopAttributes::Unspecified) { - std::string Name; - if (Attrs.UnrollEnable == LoopAttributes::Enable) - Name = "llvm.loop.unroll.enable"; - else if (Attrs.UnrollEnable == LoopAttributes::Full) - Name = "llvm.loop.unroll.full"; - else - Name = "llvm.loop.unroll.disable"; - Metadata *Vals[] = {MDString::get(Ctx, Name)}; - Args.push_back(MDNode::get(Ctx, Vals)); + MDNode *LoopID = MDNode::getDistinct(Ctx, Args); + LoopID->replaceOperandWith(0, LoopID); + HasUserTransforms = true; + return LoopID; +} + +MDNode * +LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs, + ArrayRef<Metadata *> LoopProperties, + bool &HasUserTransforms) { + LLVMContext &Ctx = Header->getContext(); + + Optional<bool> Enabled; + if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable) + Enabled = false; + else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable || + Attrs.UnrollAndJamCount != 0) + Enabled = true; + + if (Enabled != true) { + SmallVector<Metadata *, 4> NewLoopProperties; + if (Enabled == false) { + NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + NewLoopProperties.push_back(MDNode::get( + Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable"))); + LoopProperties = NewLoopProperties; + } + return createPartialUnrollMetadata(Attrs, LoopProperties, + HasUserTransforms); } - // Setting unroll_and_jam.full or unroll_and_jam.disable - if (Attrs.UnrollAndJamEnable != LoopAttributes::Unspecified) { - std::string Name; - if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) - Name = "llvm.loop.unroll_and_jam.enable"; - else if (Attrs.UnrollAndJamEnable == LoopAttributes::Full) - Name = "llvm.loop.unroll_and_jam.full"; - else - Name = "llvm.loop.unroll_and_jam.disable"; - Metadata *Vals[] = {MDString::get(Ctx, Name)}; + SmallVector<Metadata *, 4> FollowupLoopProperties; + FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + FollowupLoopProperties.push_back( + MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable"))); + + bool FollowupHasTransforms = false; + MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties, + FollowupHasTransforms); + + SmallVector<Metadata *, 4> Args; + TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); + Args.push_back(TempNode.get()); + Args.append(LoopProperties.begin(), LoopProperties.end()); + + // Setting unroll_and_jam.count + if (Attrs.UnrollAndJamCount > 0) { + Metadata *Vals[] = { + MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"), + ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), + Attrs.UnrollAndJamCount))}; Args.push_back(MDNode::get(Ctx, Vals)); } - if (Attrs.DistributeEnable != LoopAttributes::Unspecified) { - Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"), - ConstantAsMetadata::get(ConstantInt::get( - Type::getInt1Ty(Ctx), (Attrs.DistributeEnable == - LoopAttributes::Enable)))}; + if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) { + Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")}; Args.push_back(MDNode::get(Ctx, Vals)); } - if (Attrs.IsParallel) { - AccGroup = MDNode::getDistinct(Ctx, {}); + if (FollowupHasTransforms) Args.push_back(MDNode::get( - Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup})); + Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"), + Followup})); + + if (UnrollAndJamInnerFollowup) + Args.push_back(MDNode::get( + Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"), + UnrollAndJamInnerFollowup})); + + MDNode *LoopID = MDNode::getDistinct(Ctx, Args); + LoopID->replaceOperandWith(0, LoopID); + HasUserTransforms = true; + return LoopID; +} + +MDNode * +LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs, + ArrayRef<Metadata *> LoopProperties, + bool &HasUserTransforms) { + LLVMContext &Ctx = Header->getContext(); + + Optional<bool> Enabled; + if (Attrs.VectorizeEnable == LoopAttributes::Disable) + Enabled = false; + else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified || + Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0) + Enabled = true; + + if (Enabled != true) { + SmallVector<Metadata *, 4> NewLoopProperties; + if (Enabled == false) { + NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + NewLoopProperties.push_back( + MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"), + ConstantAsMetadata::get(ConstantInt::get( + llvm::Type::getInt1Ty(Ctx), 0))})); + LoopProperties = NewLoopProperties; + } + return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms); + } + + // Apply all loop properties to the vectorized loop. + SmallVector<Metadata *, 4> FollowupLoopProperties; + FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + + // Don't vectorize an already vectorized loop. + FollowupLoopProperties.push_back( + MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized"))); + + bool FollowupHasTransforms = false; + MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties, + FollowupHasTransforms); + + SmallVector<Metadata *, 4> Args; + TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); + Args.push_back(TempNode.get()); + Args.append(LoopProperties.begin(), LoopProperties.end()); + + // Setting vectorize.width + if (Attrs.VectorizeWidth > 0) { + Metadata *Vals[] = { + MDString::get(Ctx, "llvm.loop.vectorize.width"), + ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), + Attrs.VectorizeWidth))}; + Args.push_back(MDNode::get(Ctx, Vals)); } - if (Attrs.PipelineDisabled) { + // Setting interleave.count + if (Attrs.InterleaveCount > 0) { Metadata *Vals[] = { - MDString::get(Ctx, "llvm.loop.pipeline.disable"), - ConstantAsMetadata::get(ConstantInt::get( - Type::getInt1Ty(Ctx), (Attrs.PipelineDisabled == true)))}; + MDString::get(Ctx, "llvm.loop.interleave.count"), + ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), + Attrs.InterleaveCount))}; Args.push_back(MDNode::get(Ctx, Vals)); } - if (Attrs.PipelineInitiationInterval > 0) { + // Setting vectorize.enable + if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) { Metadata *Vals[] = { - MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"), + MDString::get(Ctx, "llvm.loop.vectorize.enable"), ConstantAsMetadata::get(ConstantInt::get( - Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))}; + llvm::Type::getInt1Ty(Ctx), + (Attrs.VectorizeEnable == LoopAttributes::Enable)))}; Args.push_back(MDNode::get(Ctx, Vals)); } - // Set the first operand to itself. + if (FollowupHasTransforms) + Args.push_back(MDNode::get( + Ctx, + {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup})); + MDNode *LoopID = MDNode::get(Ctx, Args); LoopID->replaceOperandWith(0, LoopID); + HasUserTransforms = true; return LoopID; } +MDNode * +LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs, + ArrayRef<Metadata *> LoopProperties, + bool &HasUserTransforms) { + LLVMContext &Ctx = Header->getContext(); + + Optional<bool> Enabled; + if (Attrs.DistributeEnable == LoopAttributes::Disable) + Enabled = false; + if (Attrs.DistributeEnable == LoopAttributes::Enable) + Enabled = true; + + if (Enabled != true) { + SmallVector<Metadata *, 4> NewLoopProperties; + if (Enabled == false) { + NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + NewLoopProperties.push_back( + MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"), + ConstantAsMetadata::get(ConstantInt::get( + llvm::Type::getInt1Ty(Ctx), 0))})); + LoopProperties = NewLoopProperties; + } + return createLoopVectorizeMetadata(Attrs, LoopProperties, + HasUserTransforms); + } + + bool FollowupHasTransforms = false; + MDNode *Followup = + createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms); + + SmallVector<Metadata *, 4> Args; + TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); + Args.push_back(TempNode.get()); + Args.append(LoopProperties.begin(), LoopProperties.end()); + + Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"), + ConstantAsMetadata::get(ConstantInt::get( + llvm::Type::getInt1Ty(Ctx), + (Attrs.DistributeEnable == LoopAttributes::Enable)))}; + Args.push_back(MDNode::get(Ctx, Vals)); + + if (FollowupHasTransforms) + Args.push_back(MDNode::get( + Ctx, + {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup})); + + MDNode *LoopID = MDNode::get(Ctx, Args); + LoopID->replaceOperandWith(0, LoopID); + HasUserTransforms = true; + return LoopID; +} + +MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs, + ArrayRef<Metadata *> LoopProperties, + bool &HasUserTransforms) { + LLVMContext &Ctx = Header->getContext(); + + Optional<bool> Enabled; + if (Attrs.UnrollEnable == LoopAttributes::Disable) + Enabled = false; + else if (Attrs.UnrollEnable == LoopAttributes::Full) + Enabled = true; + + if (Enabled != true) { + SmallVector<Metadata *, 4> NewLoopProperties; + if (Enabled == false) { + NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); + NewLoopProperties.push_back( + MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable"))); + LoopProperties = NewLoopProperties; + } + return createLoopDistributeMetadata(Attrs, LoopProperties, + HasUserTransforms); + } + + SmallVector<Metadata *, 4> Args; + TempMDTuple TempNode = MDNode::getTemporary(Ctx, None); + Args.push_back(TempNode.get()); + Args.append(LoopProperties.begin(), LoopProperties.end()); + Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full"))); + + // No follow-up: there is no loop after full unrolling. + // TODO: Warn if there are transformations after full unrolling. + + MDNode *LoopID = MDNode::getDistinct(Ctx, Args); + LoopID->replaceOperandWith(0, LoopID); + HasUserTransforms = true; + return LoopID; +} + +MDNode *LoopInfo::createMetadata( + const LoopAttributes &Attrs, + llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties, + bool &HasUserTransforms) { + SmallVector<Metadata *, 3> LoopProperties; + + // If we have a valid start debug location for the loop, add it. + if (StartLoc) { + LoopProperties.push_back(StartLoc.getAsMDNode()); + + // If we also have a valid end debug location for the loop, add it. + if (EndLoc) + LoopProperties.push_back(EndLoc.getAsMDNode()); + } + + assert(!!AccGroup == Attrs.IsParallel && + "There must be an access group iff the loop is parallel"); + if (Attrs.IsParallel) { + LLVMContext &Ctx = Header->getContext(); + LoopProperties.push_back(MDNode::get( + Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup})); + } + + LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(), + AdditionalLoopProperties.end()); + return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms); +} + LoopAttributes::LoopAttributes(bool IsParallel) : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified), UnrollEnable(LoopAttributes::Unspecified), @@ -174,15 +431,114 @@ void LoopAttributes::clear() { } LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs, - const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc) - : LoopID(nullptr), Header(Header), Attrs(Attrs) { - LoopID = - createMetadata(Header->getContext(), Attrs, StartLoc, EndLoc, AccGroup); + const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, + LoopInfo *Parent) + : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc), + Parent(Parent) { + + if (Attrs.IsParallel) { + // Create an access group for this loop. + LLVMContext &Ctx = Header->getContext(); + AccGroup = MDNode::getDistinct(Ctx, {}); + } + + if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && + Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && + Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled && + Attrs.PipelineInitiationInterval == 0 && + Attrs.VectorizeEnable == LoopAttributes::Unspecified && + Attrs.UnrollEnable == LoopAttributes::Unspecified && + Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified && + Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc && + !EndLoc) + return; + + TempLoopID = MDNode::getTemporary(Header->getContext(), None); +} + +void LoopInfo::finish() { + // We did not annotate the loop body instructions because there are no + // attributes for this loop. + if (!TempLoopID) + return; + + MDNode *LoopID; + LoopAttributes CurLoopAttr = Attrs; + LLVMContext &Ctx = Header->getContext(); + + if (Parent && (Parent->Attrs.UnrollAndJamEnable || + Parent->Attrs.UnrollAndJamCount != 0)) { + // Parent unroll-and-jams this loop. + // Split the transformations in those that happens before the unroll-and-jam + // and those after. + + LoopAttributes BeforeJam, AfterJam; + + BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel; + + BeforeJam.VectorizeWidth = Attrs.VectorizeWidth; + BeforeJam.InterleaveCount = Attrs.InterleaveCount; + BeforeJam.VectorizeEnable = Attrs.VectorizeEnable; + BeforeJam.DistributeEnable = Attrs.DistributeEnable; + + switch (Attrs.UnrollEnable) { + case LoopAttributes::Unspecified: + case LoopAttributes::Disable: + BeforeJam.UnrollEnable = Attrs.UnrollEnable; + AfterJam.UnrollEnable = Attrs.UnrollEnable; + break; + case LoopAttributes::Full: + BeforeJam.UnrollEnable = LoopAttributes::Full; + break; + case LoopAttributes::Enable: + AfterJam.UnrollEnable = LoopAttributes::Enable; + break; + } + + AfterJam.UnrollCount = Attrs.UnrollCount; + AfterJam.PipelineDisabled = Attrs.PipelineDisabled; + AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval; + + // If this loop is subject of an unroll-and-jam by the parent loop, and has + // an unroll-and-jam annotation itself, we have to decide whether to first + // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The + // UnrollAndJam pass processes loops from inner to outer, so we apply the + // inner first. + BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount; + BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable; + + // Set the inner followup metadata to process by the outer loop. Only + // consider the first inner loop. + if (!Parent->UnrollAndJamInnerFollowup) { + // Splitting the attributes into a BeforeJam and an AfterJam part will + // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam) + // to be forwarded to the AfterJam part. We detect the situation here and + // add it manually. + SmallVector<Metadata *, 1> BeforeLoopProperties; + if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified || + BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0) + BeforeLoopProperties.push_back( + MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized"))); + + bool InnerFollowupHasTransform = false; + MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties, + InnerFollowupHasTransform); + if (InnerFollowupHasTransform) + Parent->UnrollAndJamInnerFollowup = InnerFollowup; + } + + CurLoopAttr = BeforeJam; + } + + bool HasUserTransforms = false; + LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms); + TempLoopID->replaceAllUsesWith(LoopID); } void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc) { - Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc)); + Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc, + Active.empty() ? nullptr : &Active.back())); // Clear the attributes so nested loops do not inherit them. StagedAttrs.clear(); } @@ -209,13 +565,13 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, // Translate opencl_unroll_hint attribute argument to // equivalent LoopHintAttr enums. // OpenCL v2.0 s6.11.5: - // 0 - full unroll (no argument). + // 0 - enable unroll (no argument). // 1 - disable unroll. // other positive integer n - unroll by n. if (OpenCLHint) { ValueInt = OpenCLHint->getUnrollHint(); if (ValueInt == 0) { - State = LoopHintAttr::Full; + State = LoopHintAttr::Enable; } else if (ValueInt != 1) { Option = LoopHintAttr::UnrollCount; State = LoopHintAttr::Numeric; @@ -365,6 +721,7 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, void LoopInfoStack::pop() { assert(!Active.empty() && "No active loops to pop"); + Active.back().finish(); Active.pop_back(); } |