diff options
author | Vitaly Buka <vitalybuka@google.com> | 2024-03-27 13:41:43 -0700 |
---|---|---|
committer | Vitaly Buka <vitalybuka@google.com> | 2024-03-27 13:41:43 -0700 |
commit | b164a11cdb4440334f0ca0e7be84e74db49c8e83 (patch) | |
tree | b594732f4b6b11986d128a8f9952aeb9aa50d7f4 | |
parent | 97b5af6b077728e78b62d16ac06616e046396fe4 (diff) | |
parent | 453a63caebc35378bcdb31da9d0f358a3998e51b (diff) |
[𝘀𝗽𝗿] changes introduced through rebaseupstream/users/vitalybuka/spr/main.hwasan-dont-instrument-loads-from-global-if-globals-are-not-tagged
Created using spr 1.3.4
[skip ci]
114 files changed, 3104 insertions, 2296 deletions
diff --git a/clang/lib/CodeGen/ABIInfoImpl.cpp b/clang/lib/CodeGen/ABIInfoImpl.cpp index dd59101ecc81..3e34d82cb399 100644 --- a/clang/lib/CodeGen/ABIInfoImpl.cpp +++ b/clang/lib/CodeGen/ABIInfoImpl.cpp @@ -187,7 +187,7 @@ CodeGen::emitVoidPtrDirectVAArg(CodeGenFunction &CGF, Address VAListAddr, CharUnits FullDirectSize = DirectSize.alignTo(SlotSize); Address NextPtr = CGF.Builder.CreateConstInBoundsByteGEP(Addr, FullDirectSize, "argp.next"); - CGF.Builder.CreateStore(NextPtr.getPointer(), VAListAddr); + CGF.Builder.CreateStore(NextPtr.emitRawPointer(CGF), VAListAddr); // If the argument is smaller than a slot, and this is a big-endian // target, the argument will be right-adjusted in its slot. @@ -239,8 +239,8 @@ Address CodeGen::emitMergePHI(CodeGenFunction &CGF, Address Addr1, const llvm::Twine &Name) { assert(Addr1.getType() == Addr2.getType()); llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name); - PHI->addIncoming(Addr1.getPointer(), Block1); - PHI->addIncoming(Addr2.getPointer(), Block2); + PHI->addIncoming(Addr1.emitRawPointer(CGF), Block1); + PHI->addIncoming(Addr2.emitRawPointer(CGF), Block2); CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment()); return Address(PHI, Addr1.getElementType(), Align); } @@ -400,7 +400,7 @@ Address CodeGen::EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, llvm::Type *ElementTy = CGF.ConvertTypeForMem(Ty); llvm::Type *BaseTy = llvm::PointerType::getUnqual(ElementTy); llvm::Value *Addr = - CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy); + CGF.Builder.CreateVAArg(VAListAddr.emitRawPointer(CGF), BaseTy); return Address(Addr, ElementTy, TyAlignForABI); } else { assert((AI.isDirect() || AI.isExtend()) && @@ -416,7 +416,7 @@ Address CodeGen::EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, "Unexpected CoerceToType seen in arginfo in generic VAArg emitter!"); Address Temp = CGF.CreateMemTemp(Ty, "varet"); - Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(), + Val = CGF.Builder.CreateVAArg(VAListAddr.emitRawPointer(CGF), CGF.ConvertTypeForMem(Ty)); CGF.Builder.CreateStore(Val, Temp); return Temp; diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h index cf48df8f5e73..35ec370a139c 100644 --- a/clang/lib/CodeGen/Address.h +++ b/clang/lib/CodeGen/Address.h @@ -15,6 +15,7 @@ #define LLVM_CLANG_LIB_CODEGEN_ADDRESS_H #include "clang/AST/CharUnits.h" +#include "clang/AST/Type.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/IR/Constants.h" #include "llvm/Support/MathExtras.h" @@ -22,28 +23,41 @@ namespace clang { namespace CodeGen { +class Address; +class CGBuilderTy; +class CodeGenFunction; +class CodeGenModule; + // Indicates whether a pointer is known not to be null. enum KnownNonNull_t { NotKnownNonNull, KnownNonNull }; -/// An aligned address. -class Address { +/// An abstract representation of an aligned address. This is designed to be an +/// IR-level abstraction, carrying just the information necessary to perform IR +/// operations on an address like loads and stores. In particular, it doesn't +/// carry C type information or allow the representation of things like +/// bit-fields; clients working at that level should generally be using +/// `LValue`. +/// The pointer contained in this class is known to be unsigned. +class RawAddress { llvm::PointerIntPair<llvm::Value *, 1, bool> PointerAndKnownNonNull; llvm::Type *ElementType; CharUnits Alignment; protected: - Address(std::nullptr_t) : ElementType(nullptr) {} + RawAddress(std::nullptr_t) : ElementType(nullptr) {} public: - Address(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment, - KnownNonNull_t IsKnownNonNull = NotKnownNonNull) + RawAddress(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull) : PointerAndKnownNonNull(Pointer, IsKnownNonNull), ElementType(ElementType), Alignment(Alignment) { assert(Pointer != nullptr && "Pointer cannot be null"); assert(ElementType != nullptr && "Element type cannot be null"); } - static Address invalid() { return Address(nullptr); } + inline RawAddress(Address Addr); + + static RawAddress invalid() { return RawAddress(nullptr); } bool isValid() const { return PointerAndKnownNonNull.getPointer() != nullptr; } @@ -80,6 +94,133 @@ public: return Alignment; } + /// Return address with different element type, but same pointer and + /// alignment. + RawAddress withElementType(llvm::Type *ElemTy) const { + return RawAddress(getPointer(), ElemTy, getAlignment(), isKnownNonNull()); + } + + KnownNonNull_t isKnownNonNull() const { + assert(isValid()); + return (KnownNonNull_t)PointerAndKnownNonNull.getInt(); + } +}; + +/// Like RawAddress, an abstract representation of an aligned address, but the +/// pointer contained in this class is possibly signed. +class Address { + friend class CGBuilderTy; + + // The boolean flag indicates whether the pointer is known to be non-null. + llvm::PointerIntPair<llvm::Value *, 1, bool> Pointer; + + /// The expected IR type of the pointer. Carrying accurate element type + /// information in Address makes it more convenient to work with Address + /// values and allows frontend assertions to catch simple mistakes. + llvm::Type *ElementType = nullptr; + + CharUnits Alignment; + + /// Offset from the base pointer. + llvm::Value *Offset = nullptr; + + llvm::Value *emitRawPointerSlow(CodeGenFunction &CGF) const; + +protected: + Address(std::nullptr_t) : ElementType(nullptr) {} + +public: + Address(llvm::Value *pointer, llvm::Type *elementType, CharUnits alignment, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull) + : Pointer(pointer, IsKnownNonNull), ElementType(elementType), + Alignment(alignment) { + assert(pointer != nullptr && "Pointer cannot be null"); + assert(elementType != nullptr && "Element type cannot be null"); + assert(!alignment.isZero() && "Alignment cannot be zero"); + } + + Address(llvm::Value *BasePtr, llvm::Type *ElementType, CharUnits Alignment, + llvm::Value *Offset, KnownNonNull_t IsKnownNonNull = NotKnownNonNull) + : Pointer(BasePtr, IsKnownNonNull), ElementType(ElementType), + Alignment(Alignment), Offset(Offset) {} + + Address(RawAddress RawAddr) + : Pointer(RawAddr.isValid() ? RawAddr.getPointer() : nullptr), + ElementType(RawAddr.isValid() ? RawAddr.getElementType() : nullptr), + Alignment(RawAddr.isValid() ? RawAddr.getAlignment() + : CharUnits::Zero()) {} + + static Address invalid() { return Address(nullptr); } + bool isValid() const { return Pointer.getPointer() != nullptr; } + + /// This function is used in situations where the caller is doing some sort of + /// opaque "laundering" of the pointer. + void replaceBasePointer(llvm::Value *P) { + assert(isValid() && "pointer isn't valid"); + assert(P->getType() == Pointer.getPointer()->getType() && + "Pointer's type changed"); + Pointer.setPointer(P); + assert(isValid() && "pointer is invalid after replacement"); + } + + CharUnits getAlignment() const { return Alignment; } + + void setAlignment(CharUnits Value) { Alignment = Value; } + + llvm::Value *getBasePointer() const { + assert(isValid() && "pointer isn't valid"); + return Pointer.getPointer(); + } + + /// Return the type of the pointer value. + llvm::PointerType *getType() const { + return llvm::PointerType::get( + ElementType, + llvm::cast<llvm::PointerType>(Pointer.getPointer()->getType()) + ->getAddressSpace()); + } + + /// Return the type of the values stored in this address. + llvm::Type *getElementType() const { + assert(isValid()); + return ElementType; + } + + /// Return the address space that this address resides in. + unsigned getAddressSpace() const { return getType()->getAddressSpace(); } + + /// Return the IR name of the pointer value. + llvm::StringRef getName() const { return Pointer.getPointer()->getName(); } + + // This function is called only in CGBuilderBaseTy::CreateElementBitCast. + void setElementType(llvm::Type *Ty) { + assert(hasOffset() && + "this funcion shouldn't be called when there is no offset"); + ElementType = Ty; + } + + /// Whether the pointer is known not to be null. + KnownNonNull_t isKnownNonNull() const { + assert(isValid()); + return (KnownNonNull_t)Pointer.getInt(); + } + + Address setKnownNonNull() { + assert(isValid()); + Pointer.setInt(KnownNonNull); + return *this; + } + + bool hasOffset() const { return Offset; } + + llvm::Value *getOffset() const { return Offset; } + + /// Return the pointer contained in this class after authenticating it and + /// adding offset to it if necessary. + llvm::Value *emitRawPointer(CodeGenFunction &CGF) const { + return getBasePointer(); + } + /// Return address with different pointer, but same element type and /// alignment. Address withPointer(llvm::Value *NewPointer, @@ -91,61 +232,59 @@ public: /// Return address with different alignment, but same pointer and element /// type. Address withAlignment(CharUnits NewAlignment) const { - return Address(getPointer(), getElementType(), NewAlignment, + return Address(Pointer.getPointer(), getElementType(), NewAlignment, isKnownNonNull()); } /// Return address with different element type, but same pointer and /// alignment. Address withElementType(llvm::Type *ElemTy) const { - return Address(getPointer(), ElemTy, getAlignment(), isKnownNonNull()); - } - - /// Whether the pointer is known not to be null. - KnownNonNull_t isKnownNonNull() const { - assert(isValid()); - return (KnownNonNull_t)PointerAndKnownNonNull.getInt(); - } - - /// Set the non-null bit. - Address setKnownNonNull() { - assert(isValid()); - PointerAndKnownNonNull.setInt(true); - return *this; + if (!hasOffset()) + return Address(getBasePointer(), ElemTy, getAlignment(), nullptr, + isKnownNonNull()); + Address A(*this); + A.ElementType = ElemTy; + return A; } }; +inline RawAddress::RawAddress(Address Addr) + : PointerAndKnownNonNull(Addr.isValid() ? Addr.getBasePointer() : nullptr, + Addr.isValid() ? Addr.isKnownNonNull() + : NotKnownNonNull), + ElementType(Addr.isValid() ? Addr.getElementType() : nullptr), + Alignment(Addr.isValid() ? Addr.getAlignment() : CharUnits::Zero()) {} + /// A specialization of Address that requires the address to be an /// LLVM Constant. -class ConstantAddress : public Address { - ConstantAddress(std::nullptr_t) : Address(nullptr) {} +class ConstantAddress : public RawAddress { + ConstantAddress(std::nullptr_t) : RawAddress(nullptr) {} public: ConstantAddress(llvm::Constant *pointer, llvm::Type *elementType, CharUnits alignment) - : Address(pointer, elementType, alignment) {} + : RawAddress(pointer, elementType, alignment) {} static ConstantAddress invalid() { return ConstantAddress(nullptr); } llvm::Constant *getPointer() const { - return llvm::cast<llvm::Constant>(Address::getPointer()); + return llvm::cast<llvm::Constant>(RawAddress::getPointer()); } ConstantAddress withElementType(llvm::Type *ElemTy) const { return ConstantAddress(getPointer(), ElemTy, getAlignment()); } - static bool isaImpl(Address addr) { + static bool isaImpl(RawAddress addr) { return llvm::isa<llvm::Constant>(addr.getPointer()); } - static ConstantAddress castImpl(Address addr) { + static ConstantAddress castImpl(RawAddress addr) { return ConstantAddress(llvm::cast<llvm::Constant>(addr.getPointer()), addr.getElementType(), addr.getAlignment()); } }; - } // Present a minimal LLVM-like casting interface. diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp index fb03d013e8af..56198385de9d 100644 --- a/clang/lib/CodeGen/CGAtomic.cpp +++ b/clang/lib/CodeGen/CGAtomic.cpp @@ -80,7 +80,7 @@ namespace { AtomicSizeInBits = C.toBits( C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) .alignTo(lvalue.getAlignment())); - llvm::Value *BitFieldPtr = lvalue.getBitFieldPointer(); + llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF); auto OffsetInChars = (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * lvalue.getAlignment(); @@ -139,13 +139,13 @@ namespace { const LValue &getAtomicLValue() const { return LVal; } llvm::Value *getAtomicPointer() const { if (LVal.isSimple()) - return LVal.getPointer(CGF); + return LVal.emitRawPointer(CGF); else if (LVal.isBitField()) - return LVal.getBitFieldPointer(); + return LVal.getRawBitFieldPointer(CGF); else if (LVal.isVectorElt()) - return LVal.getVectorPointer(); + return LVal.getRawVectorPointer(CGF); assert(LVal.isExtVectorElt()); - return LVal.getExtVectorPointer(); + return LVal.getRawExtVectorPointer(CGF); } Address getAtomicAddress() const { llvm::Type *ElTy; @@ -368,7 +368,7 @@ bool AtomicInfo::emitMemSetZeroIfNecessary() const { return false; CGF.Builder.CreateMemSet( - addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0), + addr.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.Int8Ty, 0), CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), LVal.getAlignment().getAsAlign()); return true; @@ -1055,7 +1055,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { return getTargetHooks().performAddrSpaceCast( *this, V, AS, LangAS::opencl_generic, DestType, false); }; - Args.add(RValue::get(CastToGenericAddrSpace(Ptr.getPointer(), + + Args.add(RValue::get(CastToGenericAddrSpace(Ptr.emitRawPointer(*this), E->getPtr()->getType())), getContext().VoidPtrTy); @@ -1086,10 +1087,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { LibCallName = "__atomic_compare_exchange"; RetTy = getContext().BoolTy; HaveRetTy = true; - Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(), + Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), E->getVal1()->getType())), getContext().VoidPtrTy); - Args.add(RValue::get(CastToGenericAddrSpace(Val2.getPointer(), + Args.add(RValue::get(CastToGenericAddrSpace(Val2.emitRawPointer(*this), E->getVal2()->getType())), getContext().VoidPtrTy); Args.add(RValue::get(Order), getContext().IntTy); @@ -1105,7 +1106,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__scoped_atomic_exchange: case AtomicExpr::AO__scoped_atomic_exchange_n: LibCallName = "__atomic_exchange"; - Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(), + Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), E->getVal1()->getType())), getContext().VoidPtrTy); break; @@ -1120,7 +1121,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { LibCallName = "__atomic_store"; RetTy = getContext().VoidTy; HaveRetTy = true; - Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(), + Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), E->getVal1()->getType())), getContext().VoidPtrTy); break; @@ -1199,7 +1200,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { if (!HaveRetTy) { // Value is returned through parameter before the order. RetTy = getContext().VoidTy; - Args.add(RValue::get(CastToGenericAddrSpace(Dest.getPointer(), RetTy)), + Args.add(RValue::get( + CastToGenericAddrSpace(Dest.emitRawPointer(*this), RetTy)), getContext().VoidPtrTy); } // Order is always the last parameter. @@ -1513,7 +1515,7 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, } else TempAddr = CreateTempAlloca(); - EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); + EmitAtomicLoadLibcall(TempAddr.emitRawPointer(CGF), AO, IsVolatile); // Okay, turn that back into the original value or whole atomic (for // non-simple lvalues) type. @@ -1673,9 +1675,9 @@ std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( if (shouldUseLibcall()) { // Produce a source address. Address ExpectedAddr = materializeRValue(Expected); - Address DesiredAddr = materializeRValue(Desired); - auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), - DesiredAddr.getPointer(), + llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); + llvm::Value *DesiredPtr = materializeRValue(Desired).emitRawPointer(CGF); + auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, Success, Failure); return std::make_pair( convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), @@ -1757,7 +1759,7 @@ void AtomicInfo::EmitAtomicUpdateLibcall( Address ExpectedAddr = CreateTempAlloca(); - EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); + EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile); auto *ContBB = CGF.createBasicBlock("atomic_cont"); auto *ExitBB = CGF.createBasicBlock("atomic_exit"); CGF.EmitBlock(ContBB); @@ -1771,10 +1773,10 @@ void AtomicInfo::EmitAtomicUpdateLibcall( AggValueSlot::ignored(), SourceLocation(), /*AsValue=*/false); EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); + llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); + llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF); auto *Res = - EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), - DesiredAddr.getPointer(), - AO, Failure); + EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure); CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); CGF.EmitBlock(ExitBB, /*IsFinished=*/true); } @@ -1843,7 +1845,7 @@ void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, Address ExpectedAddr = CreateTempAlloca(); - EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); + EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile); auto *ContBB = CGF.createBasicBlock("atomic_cont"); auto *ExitBB = CGF.createBasicBlock("atomic_exit"); CGF.EmitBlock(ContBB); @@ -1854,10 +1856,10 @@ void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, CGF.Builder.CreateStore(OldVal, DesiredAddr); } EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); + llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); + llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF); auto *Res = - EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), - DesiredAddr.getPointer(), - AO, Failure); + EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure); CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); CGF.EmitBlock(ExitBB, /*IsFinished=*/true); } @@ -1957,7 +1959,8 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, args.add(RValue::get(atomics.getAtomicSizeValue()), getContext().getSizeType()); args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy); - args.add(RValue::get(srcAddr.getPointer()), getContext().VoidPtrTy); + args.add(RValue::get(srcAddr.emitRawPointer(*this)), + getContext().VoidPtrTy); args.add( RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), getContext().IntTy); diff --git a/clang/lib/CodeGen/CGBlocks.cpp b/clang/lib/CodeGen/CGBlocks.cpp index ad0b50d79961..a01f2c7c9798 100644 --- a/clang/lib/CodeGen/CGBlocks.cpp +++ b/clang/lib/CodeGen/CGBlocks.cpp @@ -36,7 +36,8 @@ CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name) : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false), NoEscape(false), HasCXXObject(false), UsesStret(false), HasCapturedVariableLayout(false), CapturesNonExternalType(false), - LocalAddress(Address::invalid()), StructureType(nullptr), Block(block) { + LocalAddress(RawAddress::invalid()), StructureType(nullptr), + Block(block) { // Skip asm prefix, if any. 'name' is usually taken directly from // the mangled name of the enclosing function. @@ -794,7 +795,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // Otherwise, we have to emit this as a local block. - Address blockAddr = blockInfo.LocalAddress; + RawAddress blockAddr = blockInfo.LocalAddress; assert(blockAddr.isValid() && "block has no address!"); llvm::Constant *isa; @@ -939,7 +940,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { if (CI.isNested()) byrefPointer = Builder.CreateLoad(src, "byref.capture"); else - byrefPointer = src.getPointer(); + byrefPointer = src.emitRawPointer(*this); // Write that void* into the capture field. Builder.CreateStore(byrefPointer, blockField); @@ -961,10 +962,10 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { } // If it's a reference variable, copy the reference into the block field. - } else if (type->isReferenceType()) { - Builder.CreateStore(src.getPointer(), blockField); + } else if (auto refType = type->getAs<ReferenceType>()) { + Builder.CreateStore(src.emitRawPointer(*this), blockField); - // If type is const-qualified, copy the value into the block field. + // If type is const-qualified, copy the value into the block field. } else if (type.isConstQualified() && type.getObjCLifetime() == Qualifiers::OCL_Strong && CGM.getCodeGenOpts().OptimizationLevel != 0) { @@ -1377,7 +1378,7 @@ void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D, // Allocate a stack slot like for any local variable to guarantee optimal // debug info at -O0. The mem2reg pass will eliminate it when optimizing. - Address alloc = CreateMemTemp(D->getType(), D->getName() + ".addr"); + RawAddress alloc = CreateMemTemp(D->getType(), D->getName() + ".addr"); Builder.CreateStore(arg, alloc); if (CGDebugInfo *DI = getDebugInfo()) { if (CGM.getCodeGenOpts().hasReducedDebugInfo()) { @@ -1497,7 +1498,7 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction( // frame setup instruction by llvm::DwarfDebug::beginFunction(). auto NL = ApplyDebugLocation::CreateEmpty(*this); Builder.CreateStore(BlockPointer, Alloca); - BlockPointerDbgLoc = Alloca.getPointer(); + BlockPointerDbgLoc = Alloca.emitRawPointer(*this); } // If we have a C++ 'this' reference, go ahead and force it into @@ -1557,8 +1558,8 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction( const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable); if (capture.isConstant()) { auto addr = LocalDeclMap.find(variable)->second; - (void)DI->EmitDeclareOfAutoVariable(variable, addr.getPointer(), - Builder); + (void)DI->EmitDeclareOfAutoVariable( + variable, addr.emitRawPointer(*this), Builder); continue; } @@ -1662,7 +1663,7 @@ struct CallBlockRelease final : EHScopeStack::Cleanup { if (LoadBlockVarAddr) { BlockVarAddr = CGF.Builder.CreateLoad(Addr); } else { - BlockVarAddr = Addr.getPointer(); + BlockVarAddr = Addr.emitRawPointer(CGF); } CGF.BuildBlockRelease(BlockVarAddr, FieldFlags, CanThrow); @@ -1962,13 +1963,15 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) { // it. It's not quite worth the annoyance to avoid creating it in the // first place. if (!needsEHCleanup(captureType.isDestructedType())) - cast<llvm::Instruction>(dstField.getPointer())->eraseFromParent(); + if (auto *I = + cast_or_null<llvm::Instruction>(dstField.getBasePointer())) + I->eraseFromParent(); } break; } case BlockCaptureEntityKind::BlockObject: { llvm::Value *srcValue = Builder.CreateLoad(srcField, "blockcopy.src"); - llvm::Value *dstAddr = dstField.getPointer(); + llvm::Value *dstAddr = dstField.emitRawPointer(*this); llvm::Value *args[] = { dstAddr, srcValue, llvm::ConstantInt::get(Int32Ty, flags.getBitMask()) }; @@ -2139,7 +2142,7 @@ public: llvm::Value *flagsVal = llvm::ConstantInt::get(CGF.Int32Ty, flags); llvm::FunctionCallee fn = CGF.CGM.getBlockObjectAssign(); - llvm::Value *args[] = { destField.getPointer(), srcValue, flagsVal }; + llvm::Value *args[] = {destField.emitRawPointer(CGF), srcValue, flagsVal}; CGF.EmitNounwindRuntimeCall(fn, args); } @@ -2696,7 +2699,8 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { storeHeaderField(V, getPointerSize(), "byref.isa"); // Store the address of the variable into its own forwarding pointer. - storeHeaderField(addr.getPointer(), getPointerSize(), "byref.forwarding"); + storeHeaderField(addr.emitRawPointer(*this), getPointerSize(), + "byref.forwarding"); // Blocks ABI: // c) the flags field is set to either 0 if no helper functions are diff --git a/clang/lib/CodeGen/CGBlocks.h b/clang/lib/CodeGen/CGBlocks.h index 4ef1ae9f3365..8d10c4f69b20 100644 --- a/clang/lib/CodeGen/CGBlocks.h +++ b/clang/lib/CodeGen/CGBlocks.h @@ -271,7 +271,8 @@ public: /// The block's captures. Non-constant captures are sorted by their offsets. llvm::SmallVector<Capture, 4> SortedCaptures; - Address LocalAddress; + // Currently we assume that block-pointer types are never signed. + RawAddress LocalAddress; llvm::StructType *StructureType; const BlockDecl *Block; const BlockExpr *BlockExpression; diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h index bf5ab171d720..6dd9da7c4cad 100644 --- a/clang/lib/CodeGen/CGBuilder.h +++ b/clang/lib/CodeGen/CGBuilder.h @@ -10,7 +10,9 @@ #define LLVM_CLANG_LIB_CODEGEN_CGBUILDER_H #include "Address.h" +#include "CGValue.h" #include "CodeGenTypeCache.h" +#include "llvm/Analysis/Utils/Local.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Type.h" @@ -18,12 +20,15 @@ namespace clang { namespace CodeGen { +class CGBuilderTy; class CodeGenFunction; /// This is an IRBuilder insertion helper that forwards to /// CodeGenFunction::InsertHelper, which adds necessary metadata to /// instructions. class CGBuilderInserter final : public llvm::IRBuilderDefaultInserter { + friend CGBuilderTy; + public: CGBuilderInserter() = default; explicit CGBuilderInserter(CodeGenFunction *CGF) : CGF(CGF) {} @@ -43,10 +48,42 @@ typedef llvm::IRBuilder<llvm::ConstantFolder, CGBuilderInserterTy> CGBuilderBaseTy; class CGBuilderTy : public CGBuilderBaseTy { + friend class Address; + /// Storing a reference to the type cache here makes it a lot easier /// to build natural-feeling, target-specific IR. const CodeGenTypeCache &TypeCache; + CodeGenFunction *getCGF() const { return getInserter().CGF; } + + llvm::Value *emitRawPointerFromAddress(Address Addr) const { + return Addr.getBasePointer(); + } + + template <bool IsInBounds> + Address createConstGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1, + const llvm::Twine &Name) { + const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); + llvm::GetElementPtrInst *GEP; + if (IsInBounds) + GEP = cast<llvm::GetElementPtrInst>(CreateConstInBoundsGEP2_32( + Addr.getElementType(), emitRawPointerFromAddress(Addr), Idx0, Idx1, + Name)); + else + GEP = cast<llvm::GetElementPtrInst>(CreateConstGEP2_32( + Addr.getElementType(), emitRawPointerFromAddress(Addr), Idx0, Idx1, + Name)); + llvm::APInt Offset( + DL.getIndexSizeInBits(Addr.getType()->getPointerAddressSpace()), 0, + /*isSigned=*/true); + if (!GEP->accumulateConstantOffset(DL, Offset)) + llvm_unreachable("offset of GEP with constants is always computable"); + return Address(GEP, GEP->getResultElementType(), + Addr.getAlignment().alignmentAtOffset( + CharUnits::fromQuantity(Offset.getSExtValue())), + IsInBounds ? Addr.isKnownNonNull() : NotKnownNonNull); + } + public: CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::LLVMContext &C) : CGBuilderBaseTy(C), TypeCache(TypeCache) {} @@ -69,20 +106,22 @@ public: // Note that we intentionally hide the CreateLoad APIs that don't // take an alignment. llvm::LoadInst *CreateLoad(Address Addr, const llvm::Twine &Name = "") { - return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(), + return CreateAlignedLoad(Addr.getElementType(), + emitRawPointerFromAddress(Addr), Addr.getAlignment().getAsAlign(), Name); } llvm::LoadInst *CreateLoad(Address Addr, const char *Name) { // This overload is required to prevent string literals from // ending up in the IsVolatile overload. - return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(), + return CreateAlignedLoad(Addr.getElementType(), + emitRawPointerFromAddress(Addr), Addr.getAlignment().getAsAlign(), Name); } llvm::LoadInst *CreateLoad(Address Addr, bool IsVolatile, const llvm::Twine &Name = "") { - return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(), - Addr.getAlignment().getAsAlign(), IsVolatile, - Name); + return CreateAlignedLoad( + Addr.getElementType(), emitRawPointerFromAddress(Addr), + Addr.getAlignment().getAsAlign(), IsVolatile, Name); } using CGBuilderBaseTy::CreateAlignedLoad; @@ -96,7 +135,7 @@ public: // take an alignment. llvm::StoreInst *CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile = false) { - return CreateAlignedStore(Val, Addr.getPointer(), + return CreateAlignedStore(Val, emitRawPointerFromAddress(Addr), Addr.getAlignment().getAsAlign(), IsVolatile); } @@ -132,33 +171,41 @@ public: llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID = llvm::SyncScope::System) { return CGBuilderBaseTy::CreateAtomicCmpXchg( - Addr.getPointer(), Cmp, New, Addr.getAlignment().getAsAlign(), - SuccessOrdering, FailureOrdering, SSID); + Addr.emitRawPointer(*getCGF()), Cmp, New, + Addr.getAlignment().getAsAlign(), SuccessOrdering, FailureOrdering, + SSID); } llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID = llvm::SyncScope::System) { - return CGBuilderBaseTy::CreateAtomicRMW(Op, Addr.getPointer(), Val, - Addr.getAlignment().getAsAlign(), - Ordering, SSID); + return CGBuilderBaseTy::CreateAtomicRMW( + Op, Addr.emitRawPointer(*getCGF()), Val, + Addr.getAlignment().getAsAlign(), Ordering, SSID); } using CGBuilderBaseTy::CreateAddrSpaceCast; Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, + llvm::Type *ElementTy, const llvm::Twine &Name = "") { - return Addr.withPointer(CreateAddrSpaceCast(Addr.getPointer(), Ty, Name), - Addr.isKnownNonNull()); + if (!Addr.hasOffset()) + return Address(CreateAddrSpaceCast(Addr.getBasePointer(), Ty, Name), + ElementTy, Addr.getAlignment(), nullptr, + Addr.isKnownNonNull()); + // Eagerly force a raw address if these is an offset. + return RawAddress( + CreateAddrSpaceCast(Addr.emitRawPointer(*getCGF()), Ty, Name), + ElementTy, Addr.getAlignment(), Addr.isKnownNonNull()); } using CGBuilderBaseTy::CreatePointerBitCastOrAddrSpaceCast; Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name = "") { - llvm::Value *Ptr = - CreatePointerBitCastOrAddrSpaceCast(Addr.getPointer(), Ty, Name); - return Address(Ptr, ElementTy, Addr.getAlignment(), Addr.isKnownNonNull()); + if (Addr.getType()->getAddressSpace() == Ty->getPointerAddressSpace()) + return Addr.withElementType(ElementTy); + return CreateAddrSpaceCast(Addr, Ty, ElementTy, Name); } /// Given @@ -176,10 +223,11 @@ public: const llvm::StructLayout *Layout = DL.getStructLayout(ElTy); auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index)); - return Address( - CreateStructGEP(Addr.getElementType(), Addr.getPointer(), Index, Name), - ElTy->getElementType(Index), - Addr.getAlignment().alignmentAtOffset(Offset), Addr.isKnownNonNull()); + return Address(CreateStructGEP(Addr.getElementType(), Addr.getBasePointer(), + Index, Name), + ElTy->getElementType(Index), + Addr.getAlignment().alignmentAtOffset(Offset), + Addr.isKnownNonNull()); } /// Given @@ -198,7 +246,7 @@ public: CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy->getElementType())); return Address( - CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), + CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(), {getSize(CharUnits::Zero()), getSize(Index)}, Name), ElTy->getElementType(), Addr.getAlignment().alignmentAtOffset(Index * EltSize), @@ -216,10 +264,10 @@ public: const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy)); - return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), - getSize(Index), Name), - ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize), - Addr.isKnownNonNull()); + return Address( + CreateInBoundsGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name), + ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize), + Addr.isKnownNonNull()); } /// Given @@ -229,110 +277,133 @@ public: /// where i64 is actually the target word size. Address CreateConstGEP(Address Addr, uint64_t Index, const llvm::Twine &Name = "") { + llvm::Type *ElTy = Addr.getElementType(); const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); - CharUnits EltSize = - CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType())); + CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy)); - return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), - getSize(Index), Name), + return Address(CreateGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name), Addr.getElementType(), - Addr.getAlignment().alignmentAtOffset(Index * EltSize), - NotKnownNonNull); + Addr.getAlignment().alignmentAtOffset(Index * EltSize)); } /// Create GEP with single dynamic index. The address alignment is reduced /// according to the element size. using CGBuilderBaseTy::CreateGEP; - Address CreateGEP(Address Addr, llvm::Value *Index, + Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index, const llvm::Twine &Name = "") { const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType())); return Address( - CreateGEP(Addr.getElementType(), Addr.getPointer(), Index, Name), + CreateGEP(Addr.getElementType(), Addr.emitRawPointer(CGF), Index, Name), Addr.getElementType(), - Addr.getAlignment().alignmentOfArrayElement(EltSize), NotKnownNonNull); + Addr.getAlignment().alignmentOfArrayElement(EltSize)); } /// Given a pointer to i8, adjust it by a given constant offset. Address CreateConstInBoundsByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name = "") { assert(Addr.getElementType() == TypeCache.Int8Ty); - return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(), - getSize(Offset), Name), - Addr.getElementType(), - Addr.getAlignment().alignmentAtOffset(Offset), - Addr.isKnownNonNull()); + return Address( + CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(), + getSize(Offset), Name), + Addr.getElementType(), Addr.getAlignment().alignmentAtOffset(Offset), + Addr.isKnownNonNull()); } + Address CreateConstByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name = "") { assert(Addr.getElementType() == TypeCache.Int8Ty); - return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), + return Address(CreateGEP(Addr.getElementType(), Addr.getBasePointer(), getSize(Offset), Name), Addr.getElementType(), - Addr.getAlignment().alignmentAtOffset(Offset), - NotKnownNonNull); + Addr.getAlignment().alignmentAtOffset(Offset)); } using CGBuilderBaseTy::CreateConstInBoundsGEP2_32; Address CreateConstInBoundsGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1, const llvm::Twine &Name = "") { - const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); + return createConstGEP2_32<true>(Addr, Idx0, Idx1, Name); + } - auto *GEP = cast<llvm::GetElementPtrInst>(CreateConstInBoundsGEP2_32( - Addr.getElementType(), Addr.getPointer(), Idx0, Idx1, Name)); - llvm::APInt Offset( - DL.getIndexSizeInBits(Addr.getType()->getPointerAddressSpace()), 0, - /*isSigned=*/true); - if (!GEP->accumulateConstantOffset(DL, Offset)) - llvm_unreachable("offset of GEP with constants is always computable"); - return Address(GEP, GEP->getResultElementType(), - Addr.getAlignment().alignmentAtOffset( - CharUnits::fromQuantity(Offset.getSExtValue())), - Addr.isKnownNonNull()); + using CGBuilderBaseTy::CreateConstGEP2_32; + Address CreateConstGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1, + const llvm::Twine &Name = "") { + return createConstGEP2_32<false>(Addr, Idx0, Idx1, Name); + } + + Address CreateGEP(Address Addr, ArrayRef<llvm::Value *> IdxList, + llvm::Type *ElementType, CharUnits Align, + const Twine &Name = "") { + llvm::Value *Ptr = emitRawPointerFromAddress(Addr); + return RawAddress(CreateGEP(Addr.getElementType(), Ptr, IdxList, Name), + ElementType, Align); + } + + using CGBuilderBaseTy::CreateInBoundsGEP; + Address CreateInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList, + llvm::Type *ElementType, CharUnits Align, + const Twine &Name = "") { + return RawAddress(CreateInBoundsGEP(Addr.getElementType(), + emitRawPointerFromAddress(Addr), + IdxList, Name), + ElementType, Align, Addr.isKnownNonNull()); + } + + using CGBuilderBaseTy::CreateIsNull; + llvm::Value *CreateIsNull(Address Addr, const Twine &Name = "") { + if (!Addr.hasOffset()) + return CreateIsNull(Addr.getBasePointer(), Name); + // The pointer isn't null if Addr has an offset since offsets can always + // be applied inbound. + return llvm::ConstantInt::getFalse(Context); } using CGBuilderBaseTy::CreateMemCpy; llvm::CallInst *CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile = false) { - return CreateMemCpy(Dest.getPointer(), Dest.getAlignment().getAsAlign(), - Src.getPointer(), Src.getAlignment().getAsAlign(), Size, - IsVolatile); + llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); + llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); + return CreateMemCpy(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, + Src.getAlignment().getAsAlign(), Size, IsVolatile); } llvm::CallInst *CreateMemCpy(Address Dest, Address Src, uint64_t Size, bool IsVolatile = false) { - return CreateMemCpy(Dest.getPointer(), Dest.getAlignment().getAsAlign(), - Src.getPointer(), Src.getAlignment().getAsAlign(), Size, - IsVolatile); + llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); + llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); + return CreateMemCpy(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, + Src.getAlignment().getAsAlign(), Size, IsVolatile); } using CGBuilderBaseTy::CreateMemCpyInline; llvm::CallInst *CreateMemCpyInline(Address Dest, Address Src, uint64_t Size) { - return CreateMemCpyInline( - Dest.getPointer(), Dest.getAlignment().getAsAlign(), Src.getPointer(), - Src.getAlignment().getAsAlign(), getInt64(Size)); + llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); + llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); + return CreateMemCpyInline(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, + Src.getAlignment().getAsAlign(), getInt64(Size)); } using CGBuilderBaseTy::CreateMemMove; llvm::CallInst *CreateMemMove(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile = false) { - return CreateMemMove(Dest.getPointer(), Dest.getAlignment().getAsAlign(), - Src.getPointer(), Src.getAlignment().getAsAlign(), - Size, IsVolatile); + llvm::Value *DestPtr = emitRawPointerFromAddress(Dest); + llvm::Value *SrcPtr = emitRawPointerFromAddress(Src); + return CreateMemMove(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr, + Src.getAlignment().getAsAlign(), Size, IsVolatile); } using CGBuilderBaseTy::CreateMemSet; llvm::CallInst *CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile = false) { - return CreateMemSet(Dest.getPointer(), Value, Size, + return CreateMemSet(emitRawPointerFromAddress(Dest), Value, Size, Dest.getAlignment().getAsAlign(), IsVolatile); } using CGBuilderBaseTy::CreateMemSetInline; llvm::CallInst *CreateMemSetInline(Address Dest, llvm::Value *Value, uint64_t Size) { - return CreateMemSetInline(Dest.getPointer(), + return CreateMemSetInline(emitRawPointerFromAddress(Dest), Dest.getAlignment().getAsAlign(), Value, getInt64(Size)); } @@ -346,16 +417,31 @@ public: const llvm::StructLayout *Layout = DL.getStructLayout(ElTy); auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index)); - return Address(CreatePreserveStructAccessIndex(ElTy, Addr.getPointer(), - Index, FieldIndex, DbgInfo), - ElTy->getElementType(Index), - Addr.getAlignment().alignmentAtOffset(Offset)); + return Address( + CreatePreserveStructAccessIndex(ElTy, emitRawPointerFromAddress(Addr), + Index, FieldIndex, DbgInfo), + ElTy->getElementType(Index), + Addr.getAlignment().alignmentAtOffset(Offset)); + } + + using CGBuilderBaseTy::CreatePreserveUnionAccessIndex; + Address CreatePreserveUnionAccessIndex(Address Addr, unsigned FieldIndex, + llvm::MDNode *DbgInfo) { + Addr.replaceBasePointer(CreatePreserveUnionAccessIndex( + Addr.getBasePointer(), FieldIndex, DbgInfo)); + return Addr; } using CGBuilderBaseTy::CreateLaunderInvariantGroup; Address CreateLaunderInvariantGroup(Address Addr) { - return Addr.withPointer(CreateLaunderInvariantGroup(Addr.getPointer()), - Addr.isKnownNonNull()); + Addr.replaceBasePointer(CreateLaunderInvariantGroup(Addr.getBasePointer())); + return Addr; + } + + using CGBuilderBaseTy::CreateStripInvariantGroup; + Address CreateStripInvariantGroup(Address Addr) { + Addr.replaceBasePointer(CreateStripInvariantGroup(Addr.getBasePointer())); + return Addr; } }; diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index fdb517eb254d..5ab5917c0c8d 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -2117,9 +2117,9 @@ llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction( auto AL = ApplyDebugLocation::CreateArtificial(*this); CharUnits Offset; - Address BufAddr = - Address(Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Int8Ty, - BufferAlignment); + Address BufAddr = makeNaturalAddressForPointer( + Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Ctx.VoidTy, + BufferAlignment); Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()), Builder.CreateConstByteGEP(BufAddr, Offset++, "summary")); Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()), @@ -2162,7 +2162,7 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) { // Ignore argument 1, the format string. It is not currently used. CallArgList Args; - Args.add(RValue::get(BufAddr.getPointer()), Ctx.VoidPtrTy); + Args.add(RValue::get(BufAddr.emitRawPointer(*this)), Ctx.VoidPtrTy); for (const auto &Item : Layout.Items) { int Size = Item.getSizeByte(); @@ -2202,8 +2202,8 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) { if (!isa<Constant>(ArgVal)) { CleanupKind Cleanup = getARCCleanupKind(); QualType Ty = TheExpr->getType(); - Address Alloca = Address::invalid(); - Address Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca); + RawAddress Alloca = RawAddress::invalid(); + RawAddress Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca); ArgVal = EmitARCRetain(Ty, ArgVal); Builder.CreateStore(ArgVal, Addr); pushLifetimeExtendedDestroy(Cleanup, Alloca, Ty, @@ -2236,7 +2236,7 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) { llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction( Layout, BufAddr.getAlignment()); EmitCall(FI, CGCallee::forDirect(F), ReturnValueSlot(), Args); - return RValue::get(BufAddr.getPointer()); + return RValue::get(BufAddr, *this); } static bool isSpecialUnsignedMultiplySignedResult( @@ -2984,7 +2984,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, // Check NonnullAttribute/NullabilityArg and Alignment. auto EmitArgCheck = [&](TypeCheckKind Kind, Address A, const Expr *Arg, unsigned ParmNum) { - Value *Val = A.getPointer(); + Value *Val = A.emitRawPointer(*this); EmitNonNullArgCheck(RValue::get(Val), Arg->getType(), Arg->getExprLoc(), FD, ParmNum); @@ -3013,12 +3013,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_va_end: EmitVAStartEnd(BuiltinID == Builtin::BI__va_start ? EmitScalarExpr(E->getArg(0)) - : EmitVAListRef(E->getArg(0)).getPointer(), + : EmitVAListRef(E->getArg(0)).emitRawPointer(*this), BuiltinID != Builtin::BI__builtin_va_end); return RValue::get(nullptr); case Builtin::BI__builtin_va_copy: { - Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer(); - Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer(); + Value *DstPtr = EmitVAListRef(E->getArg(0)).emitRawPointer(*this); + Value *SrcPtr = EmitVAListRef(E->getArg(1)).emitRawPointer(*this); Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy, {DstPtr->getType()}), {DstPtr, SrcPtr}); return RValue::get(nullptr); @@ -3849,13 +3849,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified(); Address Src = EmitPointerWithAlignment(E->getArg(0)); - EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(), - E->getArg(0)->getExprLoc(), FD, 0); + EmitNonNullArgCheck(RValue::get(Src.emitRawPointer(*this)), + E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, + 0); Value *Result = MB.CreateColumnMajorLoad( - Src.getElementType(), Src.getPointer(), + Src.getElementType(), Src.emitRawPointer(*this), Align(Src.getAlignment().getQuantity()), Stride, IsVolatile, - ResultTy->getNumRows(), ResultTy->getNumColumns(), - "matrix"); + ResultTy->getNumRows(), ResultTy->getNumColumns(), "matrix"); return RValue::get(Result); } @@ -3870,11 +3870,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, assert(PtrTy && "arg1 must be of pointer type"); bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified(); - EmitNonNullArgCheck(RValue::get(Dst.getPointer()), E->getArg(1)->getType(), - E->getArg(1)->getExprLoc(), FD, 0); + EmitNonNullArgCheck(RValue::get(Dst.emitRawPointer(*this)), + E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD, + 0); Value *Result = MB.CreateColumnMajorStore( - Matrix, Dst.getPointer(), Align(Dst.getAlignment().getQuantity()), - Stride, IsVolatile, MatrixTy->getNumRows(), MatrixTy->getNumColumns()); + Matrix, Dst.emitRawPointer(*this), + Align(Dst.getAlignment().getQuantity()), Stride, IsVolatile, + MatrixTy->getNumRows(), MatrixTy->getNumColumns()); return RValue::get(Result); } @@ -4033,7 +4035,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_bzero: { Address Dest = EmitPointerWithAlignment(E->getArg(0)); Value *SizeVal = EmitScalarExpr(E->getArg(1)); - EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), + EmitNonNullArgCheck(Dest, E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, 0); Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false); return RValue::get(nullptr); @@ -4044,10 +4046,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Address Src = EmitPointerWithAlignment(E->getArg(0)); Address Dest = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = EmitScalarExpr(E->getArg(2)); - EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(), - E->getArg(0)->getExprLoc(), FD, 0); - EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(1)->getType(), - E->getArg(1)->getExprLoc(), FD, 0); + EmitNonNullArgCheck(RValue::get(Src.emitRawPointer(*this)), + E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, + 0); + EmitNonNullArgCheck(RValue::get(Dest.emitRawPointer(*this)), + E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD, + 0); Builder.CreateMemMove(Dest, Src, SizeVal, false); return RValue::get(nullptr); } @@ -4064,10 +4068,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Builder.CreateMemCpy(Dest, Src, SizeVal, false); if (BuiltinID == Builtin::BImempcpy || BuiltinID == Builtin::BI__builtin_mempcpy) - return RValue::get(Builder.CreateInBoundsGEP(Dest.getElementType(), - Dest.getPointer(), SizeVal)); + return RValue::get(Builder.CreateInBoundsGEP( + Dest.getElementType(), Dest.emitRawPointer(*this), SizeVal)); else - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BI__builtin_memcpy_inline: { @@ -4099,7 +4103,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Address Src = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); Builder.CreateMemCpy(Dest, Src, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BI__builtin_objc_memmove_collectable: { @@ -4108,7 +4112,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Value *SizeVal = EmitScalarExpr(E->getArg(2)); CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestAddr, SrcAddr, SizeVal); - return RValue::get(DestAddr.getPointer()); + return RValue::get(DestAddr, *this); } case Builtin::BI__builtin___memmove_chk: { @@ -4125,7 +4129,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Address Src = EmitPointerWithAlignment(E->getArg(1)); Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); Builder.CreateMemMove(Dest, Src, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BImemmove: @@ -4136,7 +4140,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0); EmitArgCheck(TCK_Load, Src, E->getArg(1), 1); Builder.CreateMemMove(Dest, Src, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BImemset: case Builtin::BI__builtin_memset: { @@ -4144,10 +4148,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty()); Value *SizeVal = EmitScalarExpr(E->getArg(2)); - EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), + EmitNonNullArgCheck(Dest, E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, 0); Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BI__builtin_memset_inline: { Address Dest = EmitPointerWithAlignment(E->getArg(0)); @@ -4155,8 +4159,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty()); uint64_t Size = E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue(); - EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(), - E->getArg(0)->getExprLoc(), FD, 0); + EmitNonNullArgCheck(RValue::get(Dest.emitRawPointer(*this)), + E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD, + 0); Builder.CreateMemSetInline(Dest, ByteVal, Size); return RValue::get(nullptr); } @@ -4175,7 +4180,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, Builder.getInt8Ty()); Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); Builder.CreateMemSet(Dest, ByteVal, SizeVal, false); - return RValue::get(Dest.getPointer()); + return RValue::get(Dest, *this); } case Builtin::BI__builtin_wmemchr: { // The MSVC runtime library does not provide a definition of wmemchr, so we @@ -4397,14 +4402,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, // Store the stack pointer to the setjmp buffer. Value *StackAddr = Builder.CreateStackSave(); - assert(Buf.getPointer()->getType() == StackAddr->getType()); + assert(Buf.emitRawPointer(*this)->getType() == StackAddr->getType()); Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Buf, 2); Builder.CreateStore(StackAddr, StackSaveSlot); // Call LLVM's EH setjmp, which is lightweight. Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp); - return RValue::get(Builder.CreateCall(F, Buf.getPointer())); + return RValue::get(Builder.CreateCall(F, Buf.emitRawPointer(*this))); } case Builtin::BI__builtin_longjmp: { Value *Buf = EmitScalarExpr(E->getArg(0)); @@ -5577,7 +5582,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, llvm::Value *Queue = EmitScalarExpr(E->getArg(0)); llvm::Value *Flags = EmitScalarExpr(E->getArg(1)); LValue NDRangeL = EmitAggExprToLValue(E->getArg(2)); - llvm::Value *Range = NDRangeL.getAddress(*this).getPointer(); + llvm::Value *Range = NDRangeL.getAddress(*this).emitRawPointer(*this); llvm::Type *RangeTy = NDRangeL.getAddress(*this).getType(); if (NumArgs == 4) { @@ -5686,9 +5691,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, getContext(), Expr::NPC_ValueDependentIsNotNull)) { EventWaitList = llvm::ConstantPointerNull::get(PtrTy); } else { - EventWaitList = E->getArg(4)->getType()->isArrayType() - ? EmitArrayToPointerDecay(E->getArg(4)).getPointer() - : EmitScalarExpr(E->getArg(4)); + EventWaitList = + E->getArg(4)->getType()->isArrayType() + ? EmitArrayToPointerDecay(E->getArg(4)).emitRawPointer(*this) + : EmitScalarExpr(E->getArg(4)); // Convert to generic address space. EventWaitList = Builder.CreatePointerCast(EventWaitList, PtrTy); } @@ -5784,7 +5790,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, llvm::Type *GenericVoidPtrTy = Builder.getPtrTy( getContext().getTargetAddressSpace(LangAS::opencl_generic)); LValue NDRangeL = EmitAggExprToLValue(E->getArg(0)); - llvm::Value *NDRange = NDRangeL.getAddress(*this).getPointer(); + llvm::Value *NDRange = NDRangeL.getAddress(*this).emitRawPointer(*this); auto Info = CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(1)); Value *Kernel = @@ -5869,7 +5875,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, auto PTy0 = FTy->getParamType(0); if (PTy0 != Arg0Val->getType()) { if (Arg0Ty->isArrayType()) - Arg0Val = EmitArrayToPointerDecay(Arg0).getPointer(); + Arg0Val = EmitArrayToPointerDecay(Arg0).emitRawPointer(*this); else Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0); } @@ -5907,7 +5913,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, auto PTy1 = FTy->getParamType(1); if (PTy1 != Arg1Val->getType()) { if (Arg1Ty->isArrayType()) - Arg1Val = EmitArrayToPointerDecay(Arg1).getPointer(); + Arg1Val = EmitArrayToPointerDecay(Arg1).emitRawPointer(*this); else Arg1Val = Builder.CreatePointerCast(Arg1Val, PTy1); } @@ -5921,7 +5927,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_ms_va_start: case Builtin::BI__builtin_ms_va_end: return RValue::get( - EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(), + EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).emitRawPointer(*this), BuiltinID == Builtin::BI__builtin_ms_va_start)); case Builtin::BI__builtin_ms_va_copy: { @@ -5963,8 +5969,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, // If this is a predefined lib function (e.g. malloc), emit the call // using exactly the normal call path. if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID)) - return emitLibraryCall(*this, FD, E, - cast<llvm::Constant>(EmitScalarExpr(E->getCallee()))); + return emitLibraryCall( + *this, FD, E, cast<llvm::Constant>(EmitScalarExpr(E->getCallee()))); // Check that a call to a target specific builtin has the correct target // features. @@ -6081,7 +6087,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, return RValue::get(nullptr); return RValue::get(V); case TEK_Aggregate: - return RValue::getAggregate(ReturnValue.getValue(), + return RValue::getAggregate(ReturnValue.getAddress(), ReturnValue.isVolatile()); case TEK_Complex: llvm_unreachable("No current target builtin returns complex"); @@ -8851,7 +8857,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, // Get the alignment for the argument in addition to the value; // we'll use it later. PtrOp0 = EmitPointerWithAlignment(E->getArg(0)); - Ops.push_back(PtrOp0.getPointer()); + Ops.push_back(PtrOp0.emitRawPointer(*this)); continue; } } @@ -8878,7 +8884,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, // Get the alignment for the argument in addition to the value; // we'll use it later. PtrOp1 = EmitPointerWithAlignment(E->getArg(1)); - Ops.push_back(PtrOp1.getPointer()); + Ops.push_back(PtrOp1.emitRawPointer(*this)); continue; } } @@ -9299,7 +9305,7 @@ Value *CodeGenFunction::EmitARMMVEBuiltinExpr(unsigned BuiltinID, if (ReturnValue.isNull()) return MvecOut; else - return Builder.CreateStore(MvecOut, ReturnValue.getValue()); + return Builder.CreateStore(MvecOut, ReturnValue.getAddress()); } case CustomCodeGen::VST24: { @@ -11479,7 +11485,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, // Get the alignment for the argument in addition to the value; // we'll use it later. PtrOp0 = EmitPointerWithAlignment(E->getArg(0)); - Ops.push_back(PtrOp0.getPointer()); + Ops.push_back(PtrOp0.emitRawPointer(*this)); continue; } } @@ -13345,15 +13351,15 @@ Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID, if (!getDebugInfo()) { CGM.Error(E->getExprLoc(), "using __builtin_preserve_field_info() without -g"); - return IsBitField ? EmitLValue(Arg).getBitFieldPointer() - : EmitLValue(Arg).getPointer(*this); + return IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this) + : EmitLValue(Arg).emitRawPointer(*this); } // Enable underlying preserve_*_access_index() generation. bool OldIsInPreservedAIRegion = IsInPreservedAIRegion; IsInPreservedAIRegion = true; - Value *FieldAddr = IsBitField ? EmitLValue(Arg).getBitFieldPointer() - : EmitLValue(Arg).getPointer(*this); + Value *FieldAddr = IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this) + : EmitLValue(Arg).emitRawPointer(*this); IsInPreservedAIRegion = OldIsInPreservedAIRegion; ConstantInt *C = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); @@ -14345,14 +14351,14 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, } case X86::BI_mm_setcsr: case X86::BI__builtin_ia32_ldmxcsr: { - Address Tmp = CreateMemTemp(E->getArg(0)->getType()); + RawAddress Tmp = CreateMemTemp(E->getArg(0)->getType()); Builder.CreateStore(Ops[0], Tmp); return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr), Tmp.getPointer()); } case X86::BI_mm_getcsr: case X86::BI__builtin_ia32_stmxcsr: { - Address Tmp = CreateMemTemp(E->getType()); + RawAddress Tmp = CreateMemTemp(E->getType()); Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr), Tmp.getPointer()); return Builder.CreateLoad(Tmp, "stmxcsr"); @@ -17627,7 +17633,8 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, SmallVector<Value *, 4> Ops; for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) if (E->getArg(i)->getType()->isArrayType()) - Ops.push_back(EmitArrayToPointerDecay(E->getArg(i)).getPointer()); + Ops.push_back( + EmitArrayToPointerDecay(E->getArg(i)).emitRawPointer(*this)); else Ops.push_back(EmitScalarExpr(E->getArg(i))); // The first argument of these two builtins is a pointer used to store their @@ -20089,14 +20096,14 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, // Save returned values. assert(II.NumResults); if (II.NumResults == 1) { - Builder.CreateAlignedStore(Result, Dst.getPointer(), + Builder.CreateAlignedStore(Result, Dst.emitRawPointer(*this), CharUnits::fromQuantity(4)); } else { for (unsigned i = 0; i < II.NumResults; ++i) { Builder.CreateAlignedStore( Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), Dst.getElementType()), - Builder.CreateGEP(Dst.getElementType(), Dst.getPointer(), + Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); } @@ -20136,7 +20143,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < II.NumResults; ++i) { Value *V = Builder.CreateAlignedLoad( Src.getElementType(), - Builder.CreateGEP(Src.getElementType(), Src.getPointer(), + Builder.CreateGEP(Src.getElementType(), Src.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, ParamType)); @@ -20208,7 +20215,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsA; ++i) { Value *V = Builder.CreateAlignedLoad( SrcA.getElementType(), - Builder.CreateGEP(SrcA.getElementType(), SrcA.getPointer(), + Builder.CreateGEP(SrcA.getElementType(), SrcA.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, AType)); @@ -20218,7 +20225,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsB; ++i) { Value *V = Builder.CreateAlignedLoad( SrcB.getElementType(), - Builder.CreateGEP(SrcB.getElementType(), SrcB.getPointer(), + Builder.CreateGEP(SrcB.getElementType(), SrcB.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, BType)); @@ -20229,7 +20236,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsC; ++i) { Value *V = Builder.CreateAlignedLoad( SrcC.getElementType(), - Builder.CreateGEP(SrcC.getElementType(), SrcC.getPointer(), + Builder.CreateGEP(SrcC.getElementType(), SrcC.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); Values.push_back(Builder.CreateBitCast(V, CType)); @@ -20239,7 +20246,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, for (unsigned i = 0; i < MI.NumEltsD; ++i) Builder.CreateAlignedStore( Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), DType), - Builder.CreateGEP(Dst.getElementType(), Dst.getPointer(), + Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this), llvm::ConstantInt::get(IntTy, i)), CharUnits::fromQuantity(4)); return Result; @@ -20497,7 +20504,7 @@ struct BuiltinAlignArgs { BuiltinAlignArgs(const CallExpr *E, CodeGenFunction &CGF) { QualType AstType = E->getArg(0)->getType(); if (AstType->isArrayType()) - Src = CGF.EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Src = CGF.EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(CGF); else Src = CGF.EmitScalarExpr(E->getArg(0)); SrcType = Src->getType(); @@ -21115,7 +21122,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_get: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); Value *Index = EmitScalarExpr(E->getArg(1)); Function *Callee; if (E->getType().isWebAssemblyExternrefType()) @@ -21129,7 +21136,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_set: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); Value *Index = EmitScalarExpr(E->getArg(1)); Value *Val = EmitScalarExpr(E->getArg(2)); Function *Callee; @@ -21144,13 +21151,13 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_size: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Value = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *Value = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_table_size); return Builder.CreateCall(Callee, Value); } case WebAssembly::BI__builtin_wasm_table_grow: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); Value *Val = EmitScalarExpr(E->getArg(1)); Value *NElems = EmitScalarExpr(E->getArg(2)); @@ -21167,7 +21174,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_fill: { assert(E->getArg(0)->getType()->isArrayType()); - Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); + Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); Value *Index = EmitScalarExpr(E->getArg(1)); Value *Val = EmitScalarExpr(E->getArg(2)); Value *NElems = EmitScalarExpr(E->getArg(3)); @@ -21185,8 +21192,8 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, } case WebAssembly::BI__builtin_wasm_table_copy: { assert(E->getArg(0)->getType()->isArrayType()); - Value *TableX = EmitArrayToPointerDecay(E->getArg(0)).getPointer(); - Value *TableY = EmitArrayToPointerDecay(E->getArg(1)).getPointer(); + Value *TableX = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this); + Value *TableY = EmitArrayToPointerDecay(E->getArg(1)).emitRawPointer(*this); Value *DstIdx = EmitScalarExpr(E->getArg(2)); Value *SrcIdx = EmitScalarExpr(E->getArg(3)); Value *NElems = EmitScalarExpr(E->getArg(4)); @@ -21265,7 +21272,7 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, auto MakeCircOp = [this, E](unsigned IntID, bool IsLoad) { // The base pointer is passed by address, so it needs to be loaded. Address A = EmitPointerWithAlignment(E->getArg(0)); - Address BP = Address(A.getPointer(), Int8PtrTy, A.getAlignment()); + Address BP = Address(A.emitRawPointer(*this), Int8PtrTy, A.getAlignment()); llvm::Value *Base = Builder.CreateLoad(BP); // The treatment of both loads and stores is the same: the arguments for // the builtin are the same as the arguments for the intrinsic. @@ -21306,8 +21313,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, // EmitPointerWithAlignment and EmitScalarExpr evaluates the expression // per call. Address DestAddr = EmitPointerWithAlignment(E->getArg(1)); - DestAddr = Address(DestAddr.getPointer(), Int8Ty, DestAddr.getAlignment()); - llvm::Value *DestAddress = DestAddr.getPointer(); + DestAddr = DestAddr.withElementType(Int8Ty); + llvm::Value *DestAddress = DestAddr.emitRawPointer(*this); // Operands are Base, Dest, Modifier. // The intrinsic format in LLVM IR is defined as @@ -21358,8 +21365,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn}); llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1); - Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.getPointer(), - PredAddr.getAlignment()); + Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.emitRawPointer(*this), + PredAddr.getAlignment()); return Builder.CreateExtractValue(Result, 0); } // These are identical to the builtins above, except they don't consume @@ -21377,8 +21384,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID, {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))}); llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1); - Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.getPointer(), - PredAddr.getAlignment()); + Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.emitRawPointer(*this), + PredAddr.getAlignment()); return Builder.CreateExtractValue(Result, 0); } diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp index b756318c46a9..0cb5b06a519c 100644 --- a/clang/lib/CodeGen/CGCUDANV.cpp +++ b/clang/lib/CodeGen/CGCUDANV.cpp @@ -331,11 +331,11 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, llvm::ConstantInt::get(SizeTy, std::max<size_t>(1, Args.size()))); // Store pointers to the arguments in a locally allocated launch_args. for (unsigned i = 0; i < Args.size(); ++i) { - llvm::Value* VarPtr = CGF.GetAddrOfLocalVar(Args[i]).getPointer(); + llvm::Value *VarPtr = CGF.GetAddrOfLocalVar(Args[i]).emitRawPointer(CGF); llvm::Value *VoidVarPtr = CGF.Builder.CreatePointerCast(VarPtr, PtrTy); CGF.Builder.CreateDefaultAlignedStore( - VoidVarPtr, - CGF.Builder.CreateConstGEP1_32(PtrTy, KernelArgs.getPointer(), i)); + VoidVarPtr, CGF.Builder.CreateConstGEP1_32( + PtrTy, KernelArgs.emitRawPointer(CGF), i)); } llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end"); @@ -393,9 +393,10 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, /*isVarArg=*/false), addUnderscoredPrefixToName("PopCallConfiguration")); - CGF.EmitRuntimeCallOrInvoke(cudaPopConfigFn, - {GridDim.getPointer(), BlockDim.getPointer(), - ShmemSize.getPointer(), Stream.getPointer()}); + CGF.EmitRuntimeCallOrInvoke(cudaPopConfigFn, {GridDim.emitRawPointer(CGF), + BlockDim.emitRawPointer(CGF), + ShmemSize.emitRawPointer(CGF), + Stream.emitRawPointer(CGF)}); // Emit the call to cudaLaunch llvm::Value *Kernel = @@ -405,7 +406,7 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF, cudaLaunchKernelFD->getParamDecl(0)->getType()); LaunchKernelArgs.add(RValue::getAggregate(GridDim), Dim3Ty); LaunchKernelArgs.add(RValue::getAggregate(BlockDim), Dim3Ty); - LaunchKernelArgs.add(RValue::get(KernelArgs.getPointer()), + LaunchKernelArgs.add(RValue::get(KernelArgs, CGF), cudaLaunchKernelFD->getParamDecl(3)->getType()); LaunchKernelArgs.add(RValue::get(CGF.Builder.CreateLoad(ShmemSize)), cudaLaunchKernelFD->getParamDecl(4)->getType()); @@ -438,8 +439,8 @@ void CGNVCUDARuntime::emitDeviceStubBodyLegacy(CodeGenFunction &CGF, auto TInfo = CGM.getContext().getTypeInfoInChars(A->getType()); Offset = Offset.alignTo(TInfo.Align); llvm::Value *Args[] = { - CGF.Builder.CreatePointerCast(CGF.GetAddrOfLocalVar(A).getPointer(), - PtrTy), + CGF.Builder.CreatePointerCast( + CGF.GetAddrOfLocalVar(A).emitRawPointer(CGF), PtrTy), llvm::ConstantInt::get(SizeTy, TInfo.Width.getQuantity()), llvm::ConstantInt::get(SizeTy, Offset.getQuantity()), }; diff --git a/clang/lib/CodeGen/CGCXXABI.cpp b/clang/lib/CodeGen/CGCXXABI.cpp index a8bf57a277e9..7c6dfc3e59d8 100644 --- a/clang/lib/CodeGen/CGCXXABI.cpp +++ b/clang/lib/CodeGen/CGCXXABI.cpp @@ -20,6 +20,12 @@ using namespace CodeGen; CGCXXABI::~CGCXXABI() { } +Address CGCXXABI::getThisAddress(CodeGenFunction &CGF) { + return CGF.makeNaturalAddressForPointer( + CGF.CXXABIThisValue, CGF.CXXABIThisDecl->getType()->getPointeeType(), + CGF.CXXABIThisAlignment); +} + void CGCXXABI::ErrorUnsupportedABI(CodeGenFunction &CGF, StringRef S) { DiagnosticsEngine &Diags = CGF.CGM.getDiags(); unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error, @@ -44,8 +50,12 @@ CGCallee CGCXXABI::EmitLoadOfMemberFunctionPointer( llvm::Value *MemPtr, const MemberPointerType *MPT) { ErrorUnsupportedABI(CGF, "calls through member pointers"); - ThisPtrForCall = This.getPointer(); - const auto *FPT = MPT->getPointeeType()->castAs<FunctionProtoType>(); + const auto *RD = + cast<CXXRecordDecl>(MPT->getClass()->castAs<RecordType>()->getDecl()); + ThisPtrForCall = + CGF.getAsNaturalPointerTo(This, CGF.getContext().getRecordType(RD)); + const FunctionProtoType *FPT = + MPT->getPointeeType()->getAs<FunctionProtoType>(); llvm::Constant *FnPtr = llvm::Constant::getNullValue( llvm::PointerType::getUnqual(CGM.getLLVMContext())); return CGCallee::forDirect(FnPtr, FPT); @@ -251,16 +261,15 @@ void CGCXXABI::ReadArrayCookie(CodeGenFunction &CGF, Address ptr, // If we don't need an array cookie, bail out early. if (!requiresArrayCookie(expr, eltTy)) { - allocPtr = ptr.getPointer(); + allocPtr = ptr.emitRawPointer(CGF); numElements = nullptr; cookieSize = CharUnits::Zero(); return; } cookieSize = getArrayCookieSizeImpl(eltTy); - Address allocAddr = - CGF.Builder.CreateConstInBoundsByteGEP(ptr, -cookieSize); - allocPtr = allocAddr.getPointer(); + Address allocAddr = CGF.Builder.CreateConstInBoundsByteGEP(ptr, -cookieSize); + allocPtr = allocAddr.emitRawPointer(CGF); numElements = readArrayCookieImpl(CGF, allocAddr, cookieSize); } diff --git a/clang/lib/CodeGen/CGCXXABI.h b/clang/lib/CodeGen/CGCXXABI.h index ad1ad08d0856..c7eccbd0095a 100644 --- a/clang/lib/CodeGen/CGCXXABI.h +++ b/clang/lib/CodeGen/CGCXXABI.h @@ -57,12 +57,8 @@ protected: llvm::Value *getThisValue(CodeGenFunction &CGF) { return CGF.CXXABIThisValue; } - Address getThisAddress(CodeGenFunction &CGF) { - return Address( - CGF.CXXABIThisValue, - CGF.ConvertTypeForMem(CGF.CXXABIThisDecl->getType()->getPointeeType()), - CGF.CXXABIThisAlignment); - } + + Address getThisAddress(CodeGenFunction &CGF); /// Issue a diagnostic about unsupported features in the ABI. void ErrorUnsupportedABI(CodeGenFunction &CGF, StringRef S); @@ -475,12 +471,6 @@ public: BaseSubobject Base, const CXXRecordDecl *NearestVBase) = 0; - /// Get the address point of the vtable for the given base subobject while - /// building a constexpr. - virtual llvm::Constant * - getVTableAddressPointForConstExpr(BaseSubobject Base, - const CXXRecordDecl *VTableClass) = 0; - /// Get the address of the vtable for the given record decl which should be /// used for the vptr at the given offset in RD. virtual llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD, diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index b8adf5c26b3a..fb0078214b07 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -1037,15 +1037,9 @@ static void forConstantArrayExpansion(CodeGenFunction &CGF, ConstantArrayExpansion *CAE, Address BaseAddr, llvm::function_ref<void(Address)> Fn) { - CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy); - CharUnits EltAlign = - BaseAddr.getAlignment().alignmentOfArrayElement(EltSize); - llvm::Type *EltTy = CGF.ConvertTypeForMem(CAE->EltTy); - for (int i = 0, n = CAE->NumElts; i < n; i++) { - llvm::Value *EltAddr = CGF.Builder.CreateConstGEP2_32( - BaseAddr.getElementType(), BaseAddr.getPointer(), 0, i); - Fn(Address(EltAddr, EltTy, EltAlign)); + Address EltAddr = CGF.Builder.CreateConstGEP2_32(BaseAddr, 0, i); + Fn(EltAddr); } } @@ -1160,9 +1154,10 @@ void CodeGenFunction::ExpandTypeToArgs( } /// Create a temporary allocation for the purposes of coercion. -static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty, - CharUnits MinAlign, - const Twine &Name = "tmp") { +static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF, + llvm::Type *Ty, + CharUnits MinAlign, + const Twine &Name = "tmp") { // Don't use an alignment that's worse than what LLVM would prefer. auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty); CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign)); @@ -1332,11 +1327,11 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, } // Otherwise do coercion through memory. This is stupid, but simple. - Address Tmp = + RawAddress Tmp = CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName()); CGF.Builder.CreateMemCpy( - Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(), - Src.getAlignment().getAsAlign(), + Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), + Src.emitRawPointer(CGF), Src.getAlignment().getAsAlign(), llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinValue())); return CGF.Builder.CreateLoad(Tmp); } @@ -1420,11 +1415,12 @@ static void CreateCoercedStore(llvm::Value *Src, // // FIXME: Assert that we aren't truncating non-padding bits when have access // to that information. - Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment()); + RawAddress Tmp = + CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment()); CGF.Builder.CreateStore(Src, Tmp); CGF.Builder.CreateMemCpy( - Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(), - Tmp.getAlignment().getAsAlign(), + Dst.emitRawPointer(CGF), Dst.getAlignment().getAsAlign(), + Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedValue())); } } @@ -3024,17 +3020,17 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, case ABIArgInfo::Indirect: case ABIArgInfo::IndirectAliased: { assert(NumIRArgs == 1); - Address ParamAddr = Address(Fn->getArg(FirstIRArg), ConvertTypeForMem(Ty), - ArgI.getIndirectAlign(), KnownNonNull); + Address ParamAddr = makeNaturalAddressForPointer( + Fn->getArg(FirstIRArg), Ty, ArgI.getIndirectAlign(), false, nullptr, + nullptr, KnownNonNull); if (!hasScalarEvaluationKind(Ty)) { // Aggregates and complex variables are accessed by reference. All we // need to do is realign the value, if requested. Also, if the address // may be aliased, copy it to ensure that the parameter variable is // mutable and has a unique adress, as C requires. - Address V = ParamAddr; if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) { - Address AlignedTemp = CreateMemTemp(Ty, "coerce"); + RawAddress AlignedTemp = CreateMemTemp(Ty, "coerce"); // Copy from the incoming argument pointer to the temporary with the // appropriate alignment. @@ -3044,11 +3040,12 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, CharUnits Size = getContext().getTypeSizeInChars(Ty); Builder.CreateMemCpy( AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(), - ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(), + ParamAddr.emitRawPointer(*this), + ParamAddr.getAlignment().getAsAlign(), llvm::ConstantInt::get(IntPtrTy, Size.getQuantity())); - V = AlignedTemp; + ParamAddr = AlignedTemp; } - ArgVals.push_back(ParamValue::forIndirect(V)); + ArgVals.push_back(ParamValue::forIndirect(ParamAddr)); } else { // Load scalar value from indirect argument. llvm::Value *V = @@ -3162,10 +3159,10 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, == ParameterABI::SwiftErrorResult) { QualType pointeeTy = Ty->getPointeeType(); assert(pointeeTy->isPointerType()); - Address temp = - CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); - Address arg(V, ConvertTypeForMem(pointeeTy), - getContext().getTypeAlignInChars(pointeeTy)); + RawAddress temp = + CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); + Address arg = makeNaturalAddressForPointer( + V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy)); llvm::Value *incomingErrorValue = Builder.CreateLoad(arg); Builder.CreateStore(incomingErrorValue, temp); V = temp.getPointer(); @@ -3502,7 +3499,7 @@ static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF, llvm::LoadInst *load = dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts()); if (!load || load->isAtomic() || load->isVolatile() || - load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer()) + load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getBasePointer()) return nullptr; // Okay! Burn it all down. This relies for correctness on the @@ -3539,12 +3536,15 @@ static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF, /// Heuristically search for a dominating store to the return-value slot. static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { + llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer(); + // Check if a User is a store which pointerOperand is the ReturnValue. // We are looking for stores to the ReturnValue, not for stores of the // ReturnValue to some other location. - auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * { + auto GetStoreIfValid = [&CGF, + ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * { auto *SI = dyn_cast<llvm::StoreInst>(U); - if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer() || + if (!SI || SI->getPointerOperand() != ReturnValuePtr || SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType()) return nullptr; // These aren't actually possible for non-coerced returns, and we @@ -3558,7 +3558,7 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { // for something immediately preceding the IP. Sometimes this can // happen with how we generate implicit-returns; it can also happen // with noreturn cleanups. - if (!CGF.ReturnValue.getPointer()->hasOneUse()) { + if (!ReturnValuePtr->hasOneUse()) { llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock(); if (IP->empty()) return nullptr; @@ -3576,8 +3576,7 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { return nullptr; } - llvm::StoreInst *store = - GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back()); + llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back()); if (!store) return nullptr; // Now do a first-and-dirty dominance check: just walk up the @@ -4121,7 +4120,11 @@ void CodeGenFunction::EmitDelegateCallArg(CallArgList &args, } static bool isProvablyNull(llvm::Value *addr) { - return isa<llvm::ConstantPointerNull>(addr); + return llvm::isa_and_nonnull<llvm::ConstantPointerNull>(addr); +} + +static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) { + return llvm::isKnownNonZero(Addr.getBasePointer(), CGF.CGM.getDataLayout()); } /// Emit the actual writing-back of a writeback. @@ -4129,21 +4132,20 @@ static void emitWriteback(CodeGenFunction &CGF, const CallArgList::Writeback &writeback) { const LValue &srcLV = writeback.Source; Address srcAddr = srcLV.getAddress(CGF); - assert(!isProvablyNull(srcAddr.getPointer()) && + assert(!isProvablyNull(srcAddr.getBasePointer()) && "shouldn't have writeback for provably null argument"); llvm::BasicBlock *contBB = nullptr; // If the argument wasn't provably non-null, we need to null check // before doing the store. - bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(), - CGF.CGM.getDataLayout()); + bool provablyNonNull = isProvablyNonNull(srcAddr, CGF); + if (!provablyNonNull) { llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback"); contBB = CGF.createBasicBlock("icr.done"); - llvm::Value *isNull = - CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull"); + llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull"); CGF.Builder.CreateCondBr(isNull, contBB, writebackBB); CGF.EmitBlock(writebackBB); } @@ -4247,7 +4249,7 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, CGF.ConvertTypeForMem(CRE->getType()->getPointeeType()); // If the address is a constant null, just pass the appropriate null. - if (isProvablyNull(srcAddr.getPointer())) { + if (isProvablyNull(srcAddr.getBasePointer())) { args.add(RValue::get(llvm::ConstantPointerNull::get(destType)), CRE->getType()); return; @@ -4276,17 +4278,16 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, // If the address is *not* known to be non-null, we need to switch. llvm::Value *finalArgument; - bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(), - CGF.CGM.getDataLayout()); + bool provablyNonNull = isProvablyNonNull(srcAddr, CGF); + if (provablyNonNull) { - finalArgument = temp.getPointer(); + finalArgument = temp.emitRawPointer(CGF); } else { - llvm::Value *isNull = - CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull"); + llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull"); - finalArgument = CGF.Builder.CreateSelect(isNull, - llvm::ConstantPointerNull::get(destType), - temp.getPointer(), "icr.argument"); + finalArgument = CGF.Builder.CreateSelect( + isNull, llvm::ConstantPointerNull::get(destType), + temp.emitRawPointer(CGF), "icr.argument"); // If we need to copy, then the load has to be conditional, which // means we need control flow. @@ -4410,6 +4411,16 @@ void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType, EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, std::nullopt); } +void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType, + SourceLocation ArgLoc, + AbstractCallee AC, unsigned ParmNum) { + if (!AC.getDecl() || !(SanOpts.has(SanitizerKind::NonnullAttribute) || + SanOpts.has(SanitizerKind::NullabilityArg))) + return; + + EmitNonNullArgCheck(RValue::get(Addr, *this), ArgType, ArgLoc, AC, ParmNum); +} + // Check if the call is going to use the inalloca convention. This needs to // agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged // later, so we can't check it directly. @@ -4750,12 +4761,22 @@ CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) { llvm::CallInst * CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const llvm::Twine &name) { - return EmitNounwindRuntimeCall(callee, std::nullopt, name); + return EmitNounwindRuntimeCall(callee, ArrayRef<llvm::Value *>(), name); } /// Emits a call to the given nounwind runtime function. llvm::CallInst * CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, + ArrayRef<Address> args, + const llvm::Twine &name) { + SmallVector<llvm::Value *, 3> values; + for (auto arg : args) + values.push_back(arg.emitRawPointer(*this)); + return EmitNounwindRuntimeCall(callee, values, name); +} + +llvm::CallInst * +CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args, const llvm::Twine &name) { llvm::CallInst *call = EmitRuntimeCall(callee, args, name); @@ -5032,7 +5053,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If we're using inalloca, insert the allocation after the stack save. // FIXME: Do this earlier rather than hacking it in here! - Address ArgMemory = Address::invalid(); + RawAddress ArgMemory = RawAddress::invalid(); if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) { const llvm::DataLayout &DL = CGM.getDataLayout(); llvm::Instruction *IP = CallArgs.getStackBase(); @@ -5048,7 +5069,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, AI->setAlignment(Align.getAsAlign()); AI->setUsedWithInAlloca(true); assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca()); - ArgMemory = Address(AI, ArgStruct, Align); + ArgMemory = RawAddress(AI, ArgStruct, Align); } ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo); @@ -5057,11 +5078,11 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If the call returns a temporary with struct return, create a temporary // alloca to hold the result, unless one is given to us. Address SRetPtr = Address::invalid(); - Address SRetAlloca = Address::invalid(); + RawAddress SRetAlloca = RawAddress::invalid(); llvm::Value *UnusedReturnSizePtr = nullptr; if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) { if (!ReturnValue.isNull()) { - SRetPtr = ReturnValue.getValue(); + SRetPtr = ReturnValue.getAddress(); } else { SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca); if (HaveInsertPoint() && ReturnValue.isUnused()) { @@ -5071,15 +5092,16 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, } } if (IRFunctionArgs.hasSRetArg()) { - IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer(); + IRCallArgs[IRFunctionArgs.getSRetArgNo()] = + getAsNaturalPointerTo(SRetPtr, RetTy); } else if (RetAI.isInAlloca()) { Address Addr = Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex()); - Builder.CreateStore(SRetPtr.getPointer(), Addr); + Builder.CreateStore(getAsNaturalPointerTo(SRetPtr, RetTy), Addr); } } - Address swiftErrorTemp = Address::invalid(); + RawAddress swiftErrorTemp = RawAddress::invalid(); Address swiftErrorArg = Address::invalid(); // When passing arguments using temporary allocas, we need to add the @@ -5112,9 +5134,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, assert(NumIRArgs == 0); assert(getTarget().getTriple().getArch() == llvm::Triple::x86); if (I->isAggregate()) { - Address Addr = I->hasLValue() - ? I->getKnownLValue().getAddress(*this) - : I->getKnownRValue().getAggregateAddress(); + RawAddress Addr = I->hasLValue() + ? I->getKnownLValue().getAddress(*this) + : I->getKnownRValue().getAggregateAddress(); llvm::Instruction *Placeholder = cast<llvm::Instruction>(Addr.getPointer()); @@ -5138,7 +5160,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, } else if (ArgInfo.getInAllocaIndirect()) { // Make a temporary alloca and store the address of it into the argument // struct. - Address Addr = CreateMemTempWithoutCast( + RawAddress Addr = CreateMemTempWithoutCast( I->Ty, getContext().getTypeAlignInChars(I->Ty), "indirect-arg-temp"); I->copyInto(*this, Addr); @@ -5160,12 +5182,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, assert(NumIRArgs == 1); if (!I->isAggregate()) { // Make a temporary alloca to pass the argument. - Address Addr = CreateMemTempWithoutCast( + RawAddress Addr = CreateMemTempWithoutCast( I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp"); - llvm::Value *Val = Addr.getPointer(); + llvm::Value *Val = getAsNaturalPointerTo(Addr, I->Ty); if (ArgHasMaybeUndefAttr) - Val = Builder.CreateFreeze(Addr.getPointer()); + Val = Builder.CreateFreeze(Val); IRCallArgs[FirstIRArg] = Val; I->copyInto(*this, Addr); @@ -5181,7 +5203,6 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, Address Addr = I->hasLValue() ? I->getKnownLValue().getAddress(*this) : I->getKnownRValue().getAggregateAddress(); - llvm::Value *V = Addr.getPointer(); CharUnits Align = ArgInfo.getIndirectAlign(); const llvm::DataLayout *TD = &CGM.getDataLayout(); @@ -5192,8 +5213,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, bool NeedCopy = false; if (Addr.getAlignment() < Align && - llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) < - Align.getAsAlign()) { + llvm::getOrEnforceKnownAlignment(Addr.emitRawPointer(*this), + Align.getAsAlign(), + *TD) < Align.getAsAlign()) { NeedCopy = true; } else if (I->hasLValue()) { auto LV = I->getKnownLValue(); @@ -5224,11 +5246,11 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, if (NeedCopy) { // Create an aligned temporary, and copy to it. - Address AI = CreateMemTempWithoutCast( + RawAddress AI = CreateMemTempWithoutCast( I->Ty, ArgInfo.getIndirectAlign(), "byval-temp"); - llvm::Value *Val = AI.getPointer(); + llvm::Value *Val = getAsNaturalPointerTo(AI, I->Ty); if (ArgHasMaybeUndefAttr) - Val = Builder.CreateFreeze(AI.getPointer()); + Val = Builder.CreateFreeze(Val); IRCallArgs[FirstIRArg] = Val; // Emit lifetime markers for the temporary alloca. @@ -5245,6 +5267,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, I->copyInto(*this, AI); } else { // Skip the extra memcpy call. + llvm::Value *V = getAsNaturalPointerTo(Addr, I->Ty); auto *T = llvm::PointerType::get( CGM.getLLVMContext(), CGM.getDataLayout().getAllocaAddrSpace()); @@ -5284,8 +5307,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, assert(!swiftErrorTemp.isValid() && "multiple swifterror args"); QualType pointeeTy = I->Ty->getPointeeType(); - swiftErrorArg = Address(V, ConvertTypeForMem(pointeeTy), - getContext().getTypeAlignInChars(pointeeTy)); + swiftErrorArg = makeNaturalAddressForPointer( + V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy)); swiftErrorTemp = CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); @@ -5422,7 +5445,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, llvm::Value *tempSize = nullptr; Address addr = Address::invalid(); - Address AllocaAddr = Address::invalid(); + RawAddress AllocaAddr = RawAddress::invalid(); if (I->isAggregate()) { addr = I->hasLValue() ? I->getKnownLValue().getAddress(*this) : I->getKnownRValue().getAggregateAddress(); @@ -5856,7 +5879,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, return RValue::getComplex(std::make_pair(Real, Imag)); } case TEK_Aggregate: { - Address DestPtr = ReturnValue.getValue(); + Address DestPtr = ReturnValue.getAddress(); bool DestIsVolatile = ReturnValue.isVolatile(); if (!DestPtr.isValid()) { diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h index 1bd48a072593..6b676ac196db 100644 --- a/clang/lib/CodeGen/CGCall.h +++ b/clang/lib/CodeGen/CGCall.h @@ -377,6 +377,7 @@ public: Address getValue() const { return Addr; } bool isUnused() const { return IsUnused; } bool isExternallyDestructed() const { return IsExternallyDestructed; } + Address getAddress() const { return Addr; } }; /// Adds attributes to \p F according to our \p CodeGenOpts and \p LangOpts, as diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp index 34319381901a..8c1c8ee455d2 100644 --- a/clang/lib/CodeGen/CGClass.cpp +++ b/clang/lib/CodeGen/CGClass.cpp @@ -139,8 +139,9 @@ Address CodeGenFunction::LoadCXXThisAddress() { CXXThisAlignment = CGM.getClassPointerAlignment(MD->getParent()); } - llvm::Type *Ty = ConvertType(MD->getFunctionObjectParameterType()); - return Address(LoadCXXThis(), Ty, CXXThisAlignment, KnownNonNull); + return makeNaturalAddressForPointer( + LoadCXXThis(), MD->getFunctionObjectParameterType(), CXXThisAlignment, + false, nullptr, nullptr, KnownNonNull); } /// Emit the address of a field using a member data pointer. @@ -270,7 +271,7 @@ ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr, } // Apply the base offset. - llvm::Value *ptr = addr.getPointer(); + llvm::Value *ptr = addr.emitRawPointer(CGF); ptr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ptr, baseOffset, "add.ptr"); // If we have a virtual component, the alignment of the result will @@ -338,8 +339,8 @@ Address CodeGenFunction::GetAddressOfBaseClass( if (sanitizePerformTypeCheck()) { SanitizerSet SkippedChecks; SkippedChecks.set(SanitizerKind::Null, !NullCheckValue); - EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(), - DerivedTy, DerivedAlign, SkippedChecks); + EmitTypeCheck(TCK_Upcast, Loc, Value.emitRawPointer(*this), DerivedTy, + DerivedAlign, SkippedChecks); } return Value.withElementType(BaseValueTy); } @@ -354,7 +355,7 @@ Address CodeGenFunction::GetAddressOfBaseClass( llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull"); endBB = createBasicBlock("cast.end"); - llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer()); + llvm::Value *isNull = Builder.CreateIsNull(Value); Builder.CreateCondBr(isNull, endBB, notNullBB); EmitBlock(notNullBB); } @@ -363,14 +364,15 @@ Address CodeGenFunction::GetAddressOfBaseClass( SanitizerSet SkippedChecks; SkippedChecks.set(SanitizerKind::Null, true); EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc, - Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks); + Value.emitRawPointer(*this), DerivedTy, DerivedAlign, + SkippedChecks); } // Compute the virtual offset. llvm::Value *VirtualOffset = nullptr; if (VBase) { VirtualOffset = - CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase); + CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase); } // Apply both offsets. @@ -387,7 +389,7 @@ Address CodeGenFunction::GetAddressOfBaseClass( EmitBlock(endBB); llvm::PHINode *PHI = Builder.CreatePHI(PtrTy, 2, "cast.result"); - PHI->addIncoming(Value.getPointer(), notNullBB); + PHI->addIncoming(Value.emitRawPointer(*this), notNullBB); PHI->addIncoming(llvm::Constant::getNullValue(PtrTy), origBB); Value = Value.withPointer(PHI, NotKnownNonNull); } @@ -424,15 +426,19 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr, CastNotNull = createBasicBlock("cast.notnull"); CastEnd = createBasicBlock("cast.end"); - llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer()); + llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr); Builder.CreateCondBr(IsNull, CastNull, CastNotNull); EmitBlock(CastNotNull); } // Apply the offset. - llvm::Value *Value = BaseAddr.getPointer(); - Value = Builder.CreateInBoundsGEP( - Int8Ty, Value, Builder.CreateNeg(NonVirtualOffset), "sub.ptr"); + Address Addr = BaseAddr.withElementType(Int8Ty); + Addr = Builder.CreateInBoundsGEP( + Addr, Builder.CreateNeg(NonVirtualOffset), Int8Ty, + CGM.getClassPointerAlignment(Derived), "sub.ptr"); + + // Just cast. + Addr = Addr.withElementType(DerivedValueTy); // Produce a PHI if we had a null-check. if (NullCheckValue) { @@ -441,13 +447,15 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr, Builder.CreateBr(CastEnd); EmitBlock(CastEnd); + llvm::Value *Value = Addr.emitRawPointer(*this); llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2); PHI->addIncoming(Value, CastNotNull); PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull); - Value = PHI; + return Address(PHI, Addr.getElementType(), + CGM.getClassPointerAlignment(Derived)); } - return Address(Value, DerivedValueTy, CGM.getClassPointerAlignment(Derived)); + return Addr; } llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD, @@ -1719,7 +1727,7 @@ namespace { // Use the base class declaration location as inline DebugLocation. All // fields of the class are destroyed. DeclAsInlineDebugLocation InlineHere(CGF, *BaseClass); - EmitSanitizerDtorFieldsCallback(CGF, Addr.getPointer(), + EmitSanitizerDtorFieldsCallback(CGF, Addr.emitRawPointer(CGF), BaseSize.getQuantity()); // Prevent the current stack frame from disappearing from the stack trace. @@ -2022,7 +2030,7 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, // Find the end of the array. llvm::Type *elementType = arrayBase.getElementType(); - llvm::Value *arrayBegin = arrayBase.getPointer(); + llvm::Value *arrayBegin = arrayBase.emitRawPointer(*this); llvm::Value *arrayEnd = Builder.CreateInBoundsGEP( elementType, arrayBegin, numElements, "arrayctor.end"); @@ -2118,14 +2126,15 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, Address This = ThisAVS.getAddress(); LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace(); LangAS ThisAS = D->getFunctionObjectParameterType().getAddressSpace(); - llvm::Value *ThisPtr = This.getPointer(); + llvm::Value *ThisPtr = + getAsNaturalPointerTo(This, D->getThisType()->getPointeeType()); if (SlotAS != ThisAS) { unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS); llvm::Type *NewType = llvm::PointerType::get(getLLVMContext(), TargetThisAS); - ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(), - ThisAS, SlotAS, NewType); + ThisPtr = getTargetHooks().performAddrSpaceCast(*this, ThisPtr, ThisAS, + SlotAS, NewType); } // Push the this ptr. @@ -2194,7 +2203,7 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, const CXXRecordDecl *ClassDecl = D->getParent(); if (!NewPointerIsChecked) - EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This.getPointer(), + EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This, getContext().getRecordType(ClassDecl), CharUnits::Zero()); if (D->isTrivial() && D->isDefaultConstructor()) { @@ -2207,10 +2216,9 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, // model that copy. if (isMemcpyEquivalentSpecialMember(D)) { assert(Args.size() == 2 && "unexpected argcount for trivial ctor"); - QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType(); - Address Src = Address(Args[1].getRValue(*this).getScalarVal(), ConvertTypeForMem(SrcTy), - CGM.getNaturalTypeAlignment(SrcTy)); + Address Src = makeNaturalAddressForPointer( + Args[1].getRValue(*this).getScalarVal(), SrcTy); LValue SrcLVal = MakeAddrLValue(Src, SrcTy); QualType DestTy = getContext().getTypeDeclType(ClassDecl); LValue DestLVal = MakeAddrLValue(This, DestTy); @@ -2263,7 +2271,9 @@ void CodeGenFunction::EmitInheritedCXXConstructorCall( const CXXConstructorDecl *D, bool ForVirtualBase, Address This, bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) { CallArgList Args; - CallArg ThisArg(RValue::get(This.getPointer()), D->getThisType()); + CallArg ThisArg(RValue::get(getAsNaturalPointerTo( + This, D->getThisType()->getPointeeType())), + D->getThisType()); // Forward the parameters. if (InheritedFromVBase && @@ -2388,12 +2398,14 @@ CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, CallArgList Args; // Push the this ptr. - Args.add(RValue::get(This.getPointer()), D->getThisType()); + Args.add(RValue::get(getAsNaturalPointerTo(This, D->getThisType())), + D->getThisType()); // Push the src ptr. QualType QT = *(FPT->param_type_begin()); llvm::Type *t = CGM.getTypes().ConvertType(QT); - llvm::Value *SrcVal = Builder.CreateBitCast(Src.getPointer(), t); + llvm::Value *Val = getAsNaturalPointerTo(Src, D->getThisType()); + llvm::Value *SrcVal = Builder.CreateBitCast(Val, t); Args.add(RValue::get(SrcVal), QT); // Skip over first argument (Src). @@ -2418,7 +2430,9 @@ CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor, // this Address This = LoadCXXThisAddress(); - DelegateArgs.add(RValue::get(This.getPointer()), (*I)->getType()); + DelegateArgs.add(RValue::get(getAsNaturalPointerTo( + This, (*I)->getType()->getPointeeType())), + (*I)->getType()); ++I; // FIXME: The location of the VTT parameter in the parameter list is @@ -2775,7 +2789,7 @@ void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived, if (MayBeNull) { llvm::Value *DerivedNotNull = - Builder.CreateIsNotNull(Derived.getPointer(), "cast.nonnull"); + Builder.CreateIsNotNull(Derived.emitRawPointer(*this), "cast.nonnull"); llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check"); ContBlock = createBasicBlock("cast.cont"); @@ -2976,7 +2990,7 @@ void CodeGenFunction::EmitLambdaBlockInvokeBody() { QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda)); Address ThisPtr = GetAddrOfBlockDecl(variable); - CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType); + CallArgs.add(RValue::get(getAsNaturalPointerTo(ThisPtr, ThisType)), ThisType); // Add the rest of the parameters. for (auto *param : BD->parameters()) @@ -3004,7 +3018,7 @@ void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) { QualType LambdaType = getContext().getRecordType(Lambda); QualType ThisType = getContext().getPointerType(LambdaType); Address ThisPtr = CreateMemTemp(LambdaType, "unused.capture"); - CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType); + CallArgs.add(RValue::get(ThisPtr.emitRawPointer(*this)), ThisType); EmitLambdaDelegatingInvokeBody(MD, CallArgs); } diff --git a/clang/lib/CodeGen/CGCleanup.cpp b/clang/lib/CodeGen/CGCleanup.cpp index f87caf050eea..e6f8e6873004 100644 --- a/clang/lib/CodeGen/CGCleanup.cpp +++ b/clang/lib/CodeGen/CGCleanup.cpp @@ -27,7 +27,7 @@ bool DominatingValue<RValue>::saved_type::needsSaving(RValue rv) { if (rv.isScalar()) return DominatingLLVMValue::needsSaving(rv.getScalarVal()); if (rv.isAggregate()) - return DominatingLLVMValue::needsSaving(rv.getAggregatePointer()); + return DominatingValue<Address>::needsSaving(rv.getAggregateAddress()); return true; } @@ -35,69 +35,40 @@ DominatingValue<RValue>::saved_type DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) { if (rv.isScalar()) { llvm::Value *V = rv.getScalarVal(); - - // These automatically dominate and don't need to be saved. - if (!DominatingLLVMValue::needsSaving(V)) - return saved_type(V, nullptr, ScalarLiteral); - - // Everything else needs an alloca. - Address addr = - CGF.CreateDefaultAlignTempAlloca(V->getType(), "saved-rvalue"); - CGF.Builder.CreateStore(V, addr); - return saved_type(addr.getPointer(), nullptr, ScalarAddress); + return saved_type(DominatingLLVMValue::save(CGF, V), + DominatingLLVMValue::needsSaving(V) ? ScalarAddress + : ScalarLiteral); } if (rv.isComplex()) { CodeGenFunction::ComplexPairTy V = rv.getComplexVal(); - llvm::Type *ComplexTy = - llvm::StructType::get(V.first->getType(), V.second->getType()); - Address addr = CGF.CreateDefaultAlignTempAlloca(ComplexTy, "saved-complex"); - CGF.Builder.CreateStore(V.first, CGF.Builder.CreateStructGEP(addr, 0)); - CGF.Builder.CreateStore(V.second, CGF.Builder.CreateStructGEP(addr, 1)); - return saved_type(addr.getPointer(), nullptr, ComplexAddress); + return saved_type(DominatingLLVMValue::save(CGF, V.first), + DominatingLLVMValue::save(CGF, V.second)); } assert(rv.isAggregate()); - Address V = rv.getAggregateAddress(); // TODO: volatile? - if (!DominatingLLVMValue::needsSaving(V.getPointer())) - return saved_type(V.getPointer(), V.getElementType(), AggregateLiteral, - V.getAlignment().getQuantity()); - - Address addr = - CGF.CreateTempAlloca(V.getType(), CGF.getPointerAlign(), "saved-rvalue"); - CGF.Builder.CreateStore(V.getPointer(), addr); - return saved_type(addr.getPointer(), V.getElementType(), AggregateAddress, - V.getAlignment().getQuantity()); + Address V = rv.getAggregateAddress(); + return saved_type( + DominatingValue<Address>::save(CGF, V), rv.isVolatileQualified(), + DominatingValue<Address>::needsSaving(V) ? AggregateAddress + : AggregateLiteral); } /// Given a saved r-value produced by SaveRValue, perform the code /// necessary to restore it to usability at the current insertion /// point. RValue DominatingValue<RValue>::saved_type::restore(CodeGenFunction &CGF) { - auto getSavingAddress = [&](llvm::Value *value) { - auto *AI = cast<llvm::AllocaInst>(value); - return Address(value, AI->getAllocatedType(), - CharUnits::fromQuantity(AI->getAlign().value())); - }; switch (K) { case ScalarLiteral: - return RValue::get(Value); case ScalarAddress: - return RValue::get(CGF.Builder.CreateLoad(getSavingAddress(Value))); + return RValue::get(DominatingLLVMValue::restore(CGF, Vals.first)); case AggregateLiteral: + case AggregateAddress: return RValue::getAggregate( - Address(Value, ElementType, CharUnits::fromQuantity(Align))); - case AggregateAddress: { - auto addr = CGF.Builder.CreateLoad(getSavingAddress(Value)); - return RValue::getAggregate( - Address(addr, ElementType, CharUnits::fromQuantity(Align))); - } + DominatingValue<Address>::restore(CGF, AggregateAddr), IsVolatile); case ComplexAddress: { - Address address = getSavingAddress(Value); - llvm::Value *real = - CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(address, 0)); - llvm::Value *imag = - CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(address, 1)); + llvm::Value *real = DominatingLLVMValue::restore(CGF, Vals.first); + llvm::Value *imag = DominatingLLVMValue::restore(CGF, Vals.second); return RValue::getComplex(real, imag); } } @@ -294,14 +265,14 @@ void EHScopeStack::popNullFixups() { BranchFixups.pop_back(); } -Address CodeGenFunction::createCleanupActiveFlag() { +RawAddress CodeGenFunction::createCleanupActiveFlag() { // Create a variable to decide whether the cleanup needs to be run. - Address active = CreateTempAllocaWithoutCast( + RawAddress active = CreateTempAllocaWithoutCast( Builder.getInt1Ty(), CharUnits::One(), "cleanup.cond"); // Initialize it to false at a site that's guaranteed to be run // before each evaluation. - setBeforeOutermostConditional(Builder.getFalse(), active); + setBeforeOutermostConditional(Builder.getFalse(), active, *this); // Initialize it to true at the current location. Builder.CreateStore(Builder.getTrue(), active); @@ -309,7 +280,7 @@ Address CodeGenFunction::createCleanupActiveFlag() { return active; } -void CodeGenFunction::initFullExprCleanupWithFlag(Address ActiveFlag) { +void CodeGenFunction::initFullExprCleanupWithFlag(RawAddress ActiveFlag) { // Set that as the active flag in the cleanup. EHCleanupScope &cleanup = cast<EHCleanupScope>(*EHStack.begin()); assert(!cleanup.hasActiveFlag() && "cleanup already has active flag?"); @@ -322,15 +293,17 @@ void CodeGenFunction::initFullExprCleanupWithFlag(Address ActiveFlag) { void EHScopeStack::Cleanup::anchor() {} static void createStoreInstBefore(llvm::Value *value, Address addr, - llvm::Instruction *beforeInst) { - auto store = new llvm::StoreInst(value, addr.getPointer(), beforeInst); + llvm::Instruction *beforeInst, + CodeGenFunction &CGF) { + auto store = new llvm::StoreInst(value, addr.emitRawPointer(CGF), beforeInst); store->setAlignment(addr.getAlignment().getAsAlign()); } static llvm::LoadInst *createLoadInstBefore(Address addr, const Twine &name, - llvm::Instruction *beforeInst) { - return new llvm::LoadInst(addr.getElementType(), addr.getPointer(), name, - false, addr.getAlignment().getAsAlign(), + llvm::Instruction *beforeInst, + CodeGenFunction &CGF) { + return new llvm::LoadInst(addr.getElementType(), addr.emitRawPointer(CGF), + name, false, addr.getAlignment().getAsAlign(), beforeInst); } @@ -357,8 +330,8 @@ static void ResolveAllBranchFixups(CodeGenFunction &CGF, // entry which we're currently popping. if (Fixup.OptimisticBranchBlock == nullptr) { createStoreInstBefore(CGF.Builder.getInt32(Fixup.DestinationIndex), - CGF.getNormalCleanupDestSlot(), - Fixup.InitialBranch); + CGF.getNormalCleanupDestSlot(), Fixup.InitialBranch, + CGF); Fixup.InitialBranch->setSuccessor(0, CleanupEntry); } @@ -385,7 +358,7 @@ static llvm::SwitchInst *TransitionToCleanupSwitch(CodeGenFunction &CGF, if (llvm::BranchInst *Br = dyn_cast<llvm::BranchInst>(Term)) { assert(Br->isUnconditional()); auto Load = createLoadInstBefore(CGF.getNormalCleanupDestSlot(), - "cleanup.dest", Term); + "cleanup.dest", Term, CGF); llvm::SwitchInst *Switch = llvm::SwitchInst::Create(Load, Br->getSuccessor(0), 4, Block); Br->eraseFromParent(); @@ -513,8 +486,8 @@ void CodeGenFunction::PopCleanupBlocks( I += Header.getSize(); if (Header.isConditional()) { - Address ActiveFlag = - reinterpret_cast<Address &>(LifetimeExtendedCleanupStack[I]); + RawAddress ActiveFlag = + reinterpret_cast<RawAddress &>(LifetimeExtendedCleanupStack[I]); initFullExprCleanupWithFlag(ActiveFlag); I += sizeof(ActiveFlag); } @@ -888,7 +861,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { if (NormalCleanupDestSlot->hasOneUse()) { NormalCleanupDestSlot->user_back()->eraseFromParent(); NormalCleanupDestSlot->eraseFromParent(); - NormalCleanupDest = Address::invalid(); + NormalCleanupDest = RawAddress::invalid(); } llvm::BasicBlock *BranchAfter = Scope.getBranchAfterBlock(0); @@ -912,9 +885,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { // pass the abnormal exit flag to Fn (SEH cleanup) cleanupFlags.setHasExitSwitch(); - llvm::LoadInst *Load = - createLoadInstBefore(getNormalCleanupDestSlot(), "cleanup.dest", - nullptr); + llvm::LoadInst *Load = createLoadInstBefore( + getNormalCleanupDestSlot(), "cleanup.dest", nullptr, *this); llvm::SwitchInst *Switch = llvm::SwitchInst::Create(Load, Default, SwitchCapacity); @@ -961,8 +933,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { if (!Fixup.Destination) continue; if (!Fixup.OptimisticBranchBlock) { createStoreInstBefore(Builder.getInt32(Fixup.DestinationIndex), - getNormalCleanupDestSlot(), - Fixup.InitialBranch); + getNormalCleanupDestSlot(), Fixup.InitialBranch, + *this); Fixup.InitialBranch->setSuccessor(0, NormalEntry); } Fixup.OptimisticBranchBlock = NormalExit; @@ -1135,7 +1107,7 @@ void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) { // Store the index at the start. llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex()); - createStoreInstBefore(Index, getNormalCleanupDestSlot(), BI); + createStoreInstBefore(Index, getNormalCleanupDestSlot(), BI, *this); // Adjust BI to point to the first cleanup block. { @@ -1269,9 +1241,9 @@ static void SetupCleanupBlockActivation(CodeGenFunction &CGF, // If we're in a conditional block, ignore the dominating IP and // use the outermost conditional branch. if (CGF.isInConditionalBranch()) { - CGF.setBeforeOutermostConditional(value, var); + CGF.setBeforeOutermostConditional(value, var, CGF); } else { - createStoreInstBefore(value, var, dominatingIP); + createStoreInstBefore(value, var, dominatingIP, CGF); } } @@ -1321,7 +1293,7 @@ void CodeGenFunction::DeactivateCleanupBlock(EHScopeStack::stable_iterator C, Scope.setActive(false); } -Address CodeGenFunction::getNormalCleanupDestSlot() { +RawAddress CodeGenFunction::getNormalCleanupDestSlot() { if (!NormalCleanupDest.isValid()) NormalCleanupDest = CreateDefaultAlignTempAlloca(Builder.getInt32Ty(), "cleanup.dest.slot"); diff --git a/clang/lib/CodeGen/CGCleanup.h b/clang/lib/CodeGen/CGCleanup.h index 7a7344c07160..03e4a29d7b3d 100644 --- a/clang/lib/CodeGen/CGCleanup.h +++ b/clang/lib/CodeGen/CGCleanup.h @@ -333,7 +333,7 @@ public: Address getActiveFlag() const { return ActiveFlag; } - void setActiveFlag(Address Var) { + void setActiveFlag(RawAddress Var) { assert(Var.getAlignment().isOne()); ActiveFlag = Var; } diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp index b7142ec08af9..93ca711f716f 100644 --- a/clang/lib/CodeGen/CGCoroutine.cpp +++ b/clang/lib/CodeGen/CGCoroutine.cpp @@ -867,8 +867,8 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) { EmitStmt(S.getPromiseDeclStmt()); Address PromiseAddr = GetAddrOfLocalVar(S.getPromiseDecl()); - auto *PromiseAddrVoidPtr = - new llvm::BitCastInst(PromiseAddr.getPointer(), VoidPtrTy, "", CoroId); + auto *PromiseAddrVoidPtr = new llvm::BitCastInst( + PromiseAddr.emitRawPointer(*this), VoidPtrTy, "", CoroId); // Update CoroId to refer to the promise. We could not do it earlier because // promise local variable was not emitted yet. CoroId->setArgOperand(1, PromiseAddrVoidPtr); diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp index 2ef5ed04af30..267f2e40a7bb 100644 --- a/clang/lib/CodeGen/CGDecl.cpp +++ b/clang/lib/CodeGen/CGDecl.cpp @@ -1461,7 +1461,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { bool EmitDebugInfo = DI && CGM.getCodeGenOpts().hasReducedDebugInfo(); Address address = Address::invalid(); - Address AllocaAddr = Address::invalid(); + RawAddress AllocaAddr = RawAddress::invalid(); Address OpenMPLocalAddr = Address::invalid(); if (CGM.getLangOpts().OpenMPIRBuilder) OpenMPLocalAddr = OMPBuilderCBHelpers::getAddressOfLocalVariable(*this, &D); @@ -1524,7 +1524,10 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { // return slot, so that we can elide the copy when returning this // variable (C++0x [class.copy]p34). address = ReturnValue; - AllocaAddr = ReturnValue; + AllocaAddr = + RawAddress(ReturnValue.emitRawPointer(*this), + ReturnValue.getElementType(), ReturnValue.getAlignment()); + ; if (const RecordType *RecordTy = Ty->getAs<RecordType>()) { const auto *RD = RecordTy->getDecl(); @@ -1535,7 +1538,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { // to this variable. Set it to zero to indicate that NRVO was not // applied. llvm::Value *Zero = Builder.getFalse(); - Address NRVOFlag = + RawAddress NRVOFlag = CreateTempAlloca(Zero->getType(), CharUnits::One(), "nrvo"); EnsureInsertPoint(); Builder.CreateStore(Zero, NRVOFlag); @@ -1678,7 +1681,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { } if (D.hasAttr<AnnotateAttr>() && HaveInsertPoint()) - EmitVarAnnotations(&D, address.getPointer()); + EmitVarAnnotations(&D, address.emitRawPointer(*this)); // Make sure we call @llvm.lifetime.end. if (emission.useLifetimeMarkers()) @@ -1851,12 +1854,13 @@ void CodeGenFunction::emitZeroOrPatternForAutoVarInit(QualType type, llvm::Value *BaseSizeInChars = llvm::ConstantInt::get(IntPtrTy, EltSize.getQuantity()); Address Begin = Loc.withElementType(Int8Ty); - llvm::Value *End = Builder.CreateInBoundsGEP( - Begin.getElementType(), Begin.getPointer(), SizeVal, "vla.end"); + llvm::Value *End = Builder.CreateInBoundsGEP(Begin.getElementType(), + Begin.emitRawPointer(*this), + SizeVal, "vla.end"); llvm::BasicBlock *OriginBB = Builder.GetInsertBlock(); EmitBlock(LoopBB); llvm::PHINode *Cur = Builder.CreatePHI(Begin.getType(), 2, "vla.cur"); - Cur->addIncoming(Begin.getPointer(), OriginBB); + Cur->addIncoming(Begin.emitRawPointer(*this), OriginBB); CharUnits CurAlign = Loc.getAlignment().alignmentOfArrayElement(EltSize); auto *I = Builder.CreateMemCpy(Address(Cur, Int8Ty, CurAlign), @@ -2283,7 +2287,7 @@ void CodeGenFunction::emitDestroy(Address addr, QualType type, checkZeroLength = false; } - llvm::Value *begin = addr.getPointer(); + llvm::Value *begin = addr.emitRawPointer(*this); llvm::Value *end = Builder.CreateInBoundsGEP(addr.getElementType(), begin, length); emitArrayDestroy(begin, end, type, elementAlign, destroyer, @@ -2543,7 +2547,7 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg, } Address DeclPtr = Address::invalid(); - Address AllocaPtr = Address::invalid(); + RawAddress AllocaPtr = Address::invalid(); bool DoStore = false; bool IsScalar = hasScalarEvaluationKind(Ty); bool UseIndirectDebugAddress = false; @@ -2555,8 +2559,8 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg, // Indirect argument is in alloca address space, which may be different // from the default address space. auto AllocaAS = CGM.getASTAllocaAddressSpace(); - auto *V = DeclPtr.getPointer(); - AllocaPtr = DeclPtr; + auto *V = DeclPtr.emitRawPointer(*this); + AllocaPtr = RawAddress(V, DeclPtr.getElementType(), DeclPtr.getAlignment()); // For truly ABI indirect arguments -- those that are not `byval` -- store // the address of the argument on the stack to preserve debug information. @@ -2695,7 +2699,7 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg, } if (D.hasAttr<AnnotateAttr>()) - EmitVarAnnotations(&D, DeclPtr.getPointer()); + EmitVarAnnotations(&D, DeclPtr.emitRawPointer(*this)); // We can only check return value nullability if all arguments to the // function satisfy their nullability preconditions. This makes it necessary diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp index 5a9d06da12de..34f289334a7d 100644 --- a/clang/lib/CodeGen/CGException.cpp +++ b/clang/lib/CodeGen/CGException.cpp @@ -397,7 +397,7 @@ namespace { void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) { // Make sure the exception object is cleaned up if there's an // exception during initialization. - pushFullExprCleanup<FreeException>(EHCleanup, addr.getPointer()); + pushFullExprCleanup<FreeException>(EHCleanup, addr.emitRawPointer(*this)); EHScopeStack::stable_iterator cleanup = EHStack.stable_begin(); // __cxa_allocate_exception returns a void*; we need to cast this @@ -416,8 +416,8 @@ void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) { /*IsInit*/ true); // Deactivate the cleanup block. - DeactivateCleanupBlock(cleanup, - cast<llvm::Instruction>(typedAddr.getPointer())); + DeactivateCleanupBlock( + cleanup, cast<llvm::Instruction>(typedAddr.emitRawPointer(*this))); } Address CodeGenFunction::getExceptionSlot() { @@ -1834,7 +1834,8 @@ Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF, llvm::Value *ParentFP) { llvm::CallInst *RecoverCall = nullptr; CGBuilderTy Builder(*this, AllocaInsertPt); - if (auto *ParentAlloca = dyn_cast<llvm::AllocaInst>(ParentVar.getPointer())) { + if (auto *ParentAlloca = + dyn_cast_or_null<llvm::AllocaInst>(ParentVar.getBasePointer())) { // Mark the variable escaped if nobody else referenced it and compute the // localescape index. auto InsertPair = ParentCGF.EscapedLocals.insert( @@ -1851,8 +1852,8 @@ Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF, // If the parent didn't have an alloca, we're doing some nested outlining. // Just clone the existing localrecover call, but tweak the FP argument to // use our FP value. All other arguments are constants. - auto *ParentRecover = - cast<llvm::IntrinsicInst>(ParentVar.getPointer()->stripPointerCasts()); + auto *ParentRecover = cast<llvm::IntrinsicInst>( + ParentVar.emitRawPointer(*this)->stripPointerCasts()); assert(ParentRecover->getIntrinsicID() == llvm::Intrinsic::localrecover && "expected alloca or localrecover in parent LocalDeclMap"); RecoverCall = cast<llvm::CallInst>(ParentRecover->clone()); @@ -1925,7 +1926,8 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF, if (isa<ImplicitParamDecl>(D) && D->getType() == getContext().VoidPtrTy) { assert(D->getName().starts_with("frame_pointer")); - FramePtrAddrAlloca = cast<llvm::AllocaInst>(I.second.getPointer()); + FramePtrAddrAlloca = + cast<llvm::AllocaInst>(I.second.getBasePointer()); break; } } @@ -1986,7 +1988,8 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF, LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField); if (!LambdaThisCaptureField->getType()->isPointerType()) { - CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer(); + CXXThisValue = + ThisFieldLValue.getAddress(*this).emitRawPointer(*this); } else { CXXThisValue = EmitLoadOfLValue(ThisFieldLValue, SourceLocation()) .getScalarVal(); diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp index 6491835cf8ef..36872c0fedb7 100644 --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -65,21 +65,21 @@ static llvm::cl::opt<bool> ClSanitizeDebugDeoptimization( /// CreateTempAlloca - This creates a alloca and inserts it into the entry /// block. -Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, - CharUnits Align, - const Twine &Name, - llvm::Value *ArraySize) { +RawAddress +CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits Align, + const Twine &Name, + llvm::Value *ArraySize) { auto Alloca = CreateTempAlloca(Ty, Name, ArraySize); Alloca->setAlignment(Align.getAsAlign()); - return Address(Alloca, Ty, Align, KnownNonNull); + return RawAddress(Alloca, Ty, Align, KnownNonNull); } /// CreateTempAlloca - This creates a alloca and inserts it into the entry /// block. The alloca is casted to default address space if necessary. -Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, - const Twine &Name, - llvm::Value *ArraySize, - Address *AllocaAddr) { +RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, + const Twine &Name, + llvm::Value *ArraySize, + RawAddress *AllocaAddr) { auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize); if (AllocaAddr) *AllocaAddr = Alloca; @@ -101,7 +101,7 @@ Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, Ty->getPointerTo(DestAddrSpace), /*non-null*/ true); } - return Address(V, Ty, Align, KnownNonNull); + return RawAddress(V, Ty, Align, KnownNonNull); } /// CreateTempAlloca - This creates an alloca and inserts it into the entry @@ -120,28 +120,29 @@ llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, /// default alignment of the corresponding LLVM type, which is *not* /// guaranteed to be related in any way to the expected alignment of /// an AST type that might have been lowered to Ty. -Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty, - const Twine &Name) { +RawAddress CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty, + const Twine &Name) { CharUnits Align = CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlign(Ty)); return CreateTempAlloca(Ty, Align, Name); } -Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) { +RawAddress CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) { CharUnits Align = getContext().getTypeAlignInChars(Ty); return CreateTempAlloca(ConvertType(Ty), Align, Name); } -Address CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name, - Address *Alloca) { +RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name, + RawAddress *Alloca) { // FIXME: Should we prefer the preferred type alignment here? return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name, Alloca); } -Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, - const Twine &Name, Address *Alloca) { - Address Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name, - /*ArraySize=*/nullptr, Alloca); +RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, + const Twine &Name, + RawAddress *Alloca) { + RawAddress Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name, + /*ArraySize=*/nullptr, Alloca); if (Ty->isConstantMatrixType()) { auto *ArrayTy = cast<llvm::ArrayType>(Result.getElementType()); @@ -154,13 +155,14 @@ Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, return Result; } -Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, CharUnits Align, - const Twine &Name) { +RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, + CharUnits Align, + const Twine &Name) { return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name); } -Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, - const Twine &Name) { +RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, + const Twine &Name) { return CreateMemTempWithoutCast(Ty, getContext().getTypeAlignInChars(Ty), Name); } @@ -359,7 +361,7 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, } else { CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor( GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete)); - CleanupArg = cast<llvm::Constant>(ReferenceTemporary.getPointer()); + CleanupArg = cast<llvm::Constant>(ReferenceTemporary.emitRawPointer(CGF)); } CGF.CGM.getCXXABI().registerGlobalDtor( CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg); @@ -384,10 +386,10 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, } } -static Address createReferenceTemporary(CodeGenFunction &CGF, - const MaterializeTemporaryExpr *M, - const Expr *Inner, - Address *Alloca = nullptr) { +static RawAddress createReferenceTemporary(CodeGenFunction &CGF, + const MaterializeTemporaryExpr *M, + const Expr *Inner, + RawAddress *Alloca = nullptr) { auto &TCG = CGF.getTargetHooks(); switch (M->getStorageDuration()) { case SD_FullExpression: @@ -416,7 +418,7 @@ static Address createReferenceTemporary(CodeGenFunction &CGF, GV->getValueType()->getPointerTo( CGF.getContext().getTargetAddressSpace(LangAS::Default))); // FIXME: Should we put the new global into a COMDAT? - return Address(C, GV->getValueType(), alignment); + return RawAddress(C, GV->getValueType(), alignment); } return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca); } @@ -448,7 +450,7 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { auto ownership = M->getType().getObjCLifetime(); if (ownership != Qualifiers::OCL_None && ownership != Qualifiers::OCL_ExplicitNone) { - Address Object = createReferenceTemporary(*this, M, E); + RawAddress Object = createReferenceTemporary(*this, M, E); if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) { llvm::Type *Ty = ConvertTypeForMem(E->getType()); Object = Object.withElementType(Ty); @@ -502,8 +504,8 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { } // Create and initialize the reference temporary. - Address Alloca = Address::invalid(); - Address Object = createReferenceTemporary(*this, M, E, &Alloca); + RawAddress Alloca = Address::invalid(); + RawAddress Object = createReferenceTemporary(*this, M, E, &Alloca); if (auto *Var = dyn_cast<llvm::GlobalVariable>( Object.getPointer()->stripPointerCasts())) { llvm::Type *TemporaryType = ConvertTypeForMem(E->getType()); @@ -1111,12 +1113,12 @@ llvm::Value *CodeGenFunction::EmitCountedByFieldExpr( } else if (const MemberExpr *ME = dyn_cast<MemberExpr>(StructBase)) { LValue LV = EmitMemberExpr(ME); Address Addr = LV.getAddress(*this); - Res = Addr.getPointer(); + Res = Addr.emitRawPointer(*this); } else if (StructBase->getType()->isPointerType()) { LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; Address Addr = EmitPointerWithAlignment(StructBase, &BaseInfo, &TBAAInfo); - Res = Addr.getPointer(); + Res = Addr.emitRawPointer(*this); } else { return nullptr; } @@ -1282,8 +1284,7 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) { if (BaseInfo) BaseInfo->mergeForCast(TargetTypeBaseInfo); - Addr = Address(Addr.getPointer(), Addr.getElementType(), Align, - IsKnownNonNull); + Addr.setAlignment(Align); } } @@ -1300,8 +1301,8 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, CGF.ConvertTypeForMem(E->getType()->getPointeeType()); Addr = Addr.withElementType(ElemTy); if (CE->getCastKind() == CK_AddressSpaceConversion) - Addr = CGF.Builder.CreateAddrSpaceCast(Addr, - CGF.ConvertType(E->getType())); + Addr = CGF.Builder.CreateAddrSpaceCast( + Addr, CGF.ConvertType(E->getType()), ElemTy); return Addr; } break; @@ -1364,10 +1365,9 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, // TODO: conditional operators, comma. // Otherwise, use the alignment of the type. - CharUnits Align = - CGF.CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo); - llvm::Type *ElemTy = CGF.ConvertTypeForMem(E->getType()->getPointeeType()); - return Address(CGF.EmitScalarExpr(E), ElemTy, Align, IsKnownNonNull); + return CGF.makeNaturalAddressForPointer( + CGF.EmitScalarExpr(E), E->getType()->getPointeeType(), CharUnits(), + /*ForPointeeType=*/true, BaseInfo, TBAAInfo, IsKnownNonNull); } /// EmitPointerWithAlignment - Given an expression of pointer type, try to @@ -1468,8 +1468,7 @@ LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) { if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase())) SkippedChecks.set(SanitizerKind::Null, true); } - EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(*this), E->getType(), - LV.getAlignment(), SkippedChecks); + EmitTypeCheck(TCK, E->getExprLoc(), LV, E->getType(), SkippedChecks); } return LV; } @@ -1581,11 +1580,11 @@ LValue CodeGenFunction::EmitLValueHelper(const Expr *E, // Defend against branches out of gnu statement expressions surrounded by // cleanups. Address Addr = LV.getAddress(*this); - llvm::Value *V = Addr.getPointer(); + llvm::Value *V = Addr.getBasePointer(); Scope.ForceCleanup({&V}); - return LValue::MakeAddr(Addr.withPointer(V, Addr.isKnownNonNull()), - LV.getType(), getContext(), LV.getBaseInfo(), - LV.getTBAAInfo()); + Addr.replaceBasePointer(V); + return LValue::MakeAddr(Addr, LV.getType(), getContext(), + LV.getBaseInfo(), LV.getTBAAInfo()); } // FIXME: Is it possible to create an ExprWithCleanups that produces a // bitfield lvalue or some other non-simple lvalue? @@ -1929,7 +1928,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo, bool isNontemporal) { - if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer())) + if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getBasePointer())) if (GV->isThreadLocal()) Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV), NotKnownNonNull); @@ -2039,8 +2038,9 @@ llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) { // Convert the pointer of \p Addr to a pointer to a vector (the value type of // MatrixType), if it points to a array (the memory type of MatrixType). -static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF, - bool IsVector = true) { +static RawAddress MaybeConvertMatrixAddress(RawAddress Addr, + CodeGenFunction &CGF, + bool IsVector = true) { auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.getElementType()); if (ArrayTy && IsVector) { auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(), @@ -2077,7 +2077,7 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo, bool isInit, bool isNontemporal) { - if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer())) + if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getBasePointer())) if (GV->isThreadLocal()) Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV), NotKnownNonNull); @@ -2432,14 +2432,12 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL"); llvm::Type *ResultType = IntPtrTy; Address dst = EmitPointerWithAlignment(Dst.getBaseIvarExp()); - llvm::Value *RHS = dst.getPointer(); + llvm::Value *RHS = dst.emitRawPointer(*this); RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast"); - llvm::Value *LHS = - Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType, - "sub.ptr.lhs.cast"); + llvm::Value *LHS = Builder.CreatePtrToInt(LvalueDst.emitRawPointer(*this), + ResultType, "sub.ptr.lhs.cast"); llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset"); - CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst, - BytesBetween); + CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst, BytesBetween); } else if (Dst.isGlobalObjCRef()) { CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst, Dst.isThreadLocalRef()); @@ -2770,12 +2768,9 @@ CodeGenFunction::EmitLoadOfReference(LValue RefLVal, llvm::LoadInst *Load = Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile()); CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo()); - - QualType PointeeType = RefLVal.getType()->getPointeeType(); - CharUnits Align = CGM.getNaturalTypeAlignment( - PointeeType, PointeeBaseInfo, PointeeTBAAInfo, - /* forPointeeType= */ true); - return Address(Load, ConvertTypeForMem(PointeeType), Align); + return makeNaturalAddressForPointer(Load, RefLVal.getType()->getPointeeType(), + CharUnits(), /*ForPointeeType=*/true, + PointeeBaseInfo, PointeeTBAAInfo); } LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) { @@ -2792,10 +2787,9 @@ Address CodeGenFunction::EmitLoadOfPointer(Address Ptr, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo) { llvm::Value *Addr = Builder.CreateLoad(Ptr); - return Address(Addr, ConvertTypeForMem(PtrTy->getPointeeType()), - CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), BaseInfo, - TBAAInfo, - /*forPointeeType=*/true)); + return makeNaturalAddressForPointer(Addr, PtrTy->getPointeeType(), + CharUnits(), /*ForPointeeType=*/true, + BaseInfo, TBAAInfo); } LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr, @@ -2991,7 +2985,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { /* BaseInfo= */ nullptr, /* TBAAInfo= */ nullptr, /* forPointeeType= */ true); - Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment); + Addr = makeNaturalAddressForPointer(Val, T, Alignment); } return MakeAddrLValue(Addr, T, AlignmentSource::Decl); } @@ -3023,11 +3017,12 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD), CapturedStmtInfo->getContextValue()); Address LValueAddress = CapLVal.getAddress(*this); - CapLVal = MakeAddrLValue( - Address(LValueAddress.getPointer(), LValueAddress.getElementType(), - getContext().getDeclAlign(VD)), - CapLVal.getType(), LValueBaseInfo(AlignmentSource::Decl), - CapLVal.getTBAAInfo()); + CapLVal = MakeAddrLValue(Address(LValueAddress.emitRawPointer(*this), + LValueAddress.getElementType(), + getContext().getDeclAlign(VD)), + CapLVal.getType(), + LValueBaseInfo(AlignmentSource::Decl), + CapLVal.getTBAAInfo()); // Mark lvalue as nontemporal if the variable is marked as nontemporal // in simd context. if (getLangOpts().OpenMP && @@ -3083,7 +3078,8 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { // Handle threadlocal function locals. if (VD->getTLSKind() != VarDecl::TLS_None) addr = addr.withPointer( - Builder.CreateThreadLocalAddress(addr.getPointer()), NotKnownNonNull); + Builder.CreateThreadLocalAddress(addr.getBasePointer()), + NotKnownNonNull); // Check for OpenMP threadprivate variables. if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd && @@ -3351,7 +3347,7 @@ llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) { // Pointers are passed directly, everything else is passed by address. if (!V->getType()->isPointerTy()) { - Address Ptr = CreateDefaultAlignTempAlloca(V->getType()); + RawAddress Ptr = CreateDefaultAlignTempAlloca(V->getType()); Builder.CreateStore(V, Ptr); V = Ptr.getPointer(); } @@ -3924,6 +3920,21 @@ static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF, } } +static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, + ArrayRef<llvm::Value *> indices, + llvm::Type *elementType, bool inbounds, + bool signedIndices, SourceLocation loc, + CharUnits align, + const llvm::Twine &name = "arrayidx") { + if (inbounds) { + return CGF.EmitCheckedInBoundsGEP(addr, indices, elementType, signedIndices, + CodeGenFunction::NotSubtraction, loc, + align, name); + } else { + return CGF.Builder.CreateGEP(addr, indices, elementType, align, name); + } +} + static CharUnits getArrayElementAlign(CharUnits arrayAlign, llvm::Value *idx, CharUnits eltSize) { @@ -3971,7 +3982,7 @@ static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF, llvm::Function *Fn = CGF.CGM.getIntrinsic(llvm::Intrinsic::preserve_static_offset); - llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.getPointer()}); + llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.emitRawPointer(CGF)}); return Address(Call, Addr.getElementType(), Addr.getAlignment()); } @@ -4034,7 +4045,7 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, // We can use that to compute the best alignment of the element. CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType); CharUnits eltAlign = - getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize); + getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize); if (hasBPFPreserveStaticOffset(Base)) addr = wrapWithBPFPreserveStaticOffset(CGF, addr); @@ -4043,19 +4054,19 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back()); if (!LastIndex || (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, Base))) { - eltPtr = emitArraySubscriptGEP( - CGF, addr.getElementType(), addr.getPointer(), indices, inbounds, - signedIndices, loc, name); + addr = emitArraySubscriptGEP(CGF, addr, indices, + CGF.ConvertTypeForMem(eltType), inbounds, + signedIndices, loc, eltAlign, name); + return addr; } else { // Remember the original array subscript for bpf target unsigned idx = LastIndex->getZExtValue(); llvm::DIType *DbgInfo = nullptr; if (arrayType) DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(*arrayType, loc); - eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getElementType(), - addr.getPointer(), - indices.size() - 1, - idx, DbgInfo); + eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex( + addr.getElementType(), addr.emitRawPointer(CGF), indices.size() - 1, + idx, DbgInfo); } return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign); @@ -4224,8 +4235,8 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, CharUnits EltAlign = getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize); llvm::Value *EltPtr = - emitArraySubscriptGEP(*this, Int8Ty, Addr.getPointer(), ScaledIdx, - false, SignedIndices, E->getExprLoc()); + emitArraySubscriptGEP(*this, Int8Ty, Addr.emitRawPointer(*this), + ScaledIdx, false, SignedIndices, E->getExprLoc()); Addr = Address(EltPtr, OrigBaseElemTy, EltAlign); } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) { // If this is A[i] where A is an array, the frontend will have decayed the @@ -4271,7 +4282,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, llvm::Type *CountTy = ConvertType(CountFD->getType()); llvm::Value *Res = Builder.CreateInBoundsGEP( - Int8Ty, Addr.getPointer(), + Int8Ty, Addr.emitRawPointer(*this), Builder.getInt32(OffsetDiff.getQuantity()), ".counted_by.gep"); Res = Builder.CreateAlignedLoad(CountTy, Res, getIntAlign(), ".counted_by.load"); @@ -4517,9 +4528,9 @@ LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, BaseInfo = ArrayLV.getBaseInfo(); TBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, ResultExprTy); } else { - Address Base = emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, - TBAAInfo, BaseTy, ResultExprTy, - IsLowerBound); + Address Base = + emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo, BaseTy, + ResultExprTy, IsLowerBound); EltPtr = emitArraySubscriptGEP(*this, Base, Idx, ResultExprTy, !getLangOpts().isSignedOverflowDefined(), /*signedIndices=*/false, E->getExprLoc()); @@ -4606,7 +4617,7 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) { SkippedChecks.set(SanitizerKind::Alignment, true); if (IsBaseCXXThis || isa<DeclRefExpr>(BaseExpr)) SkippedChecks.set(SanitizerKind::Null, true); - EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr.getPointer(), PtrTy, + EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr, PtrTy, /*Alignment=*/CharUnits::Zero(), SkippedChecks); BaseLV = MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo); } else @@ -4655,8 +4666,8 @@ LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field, LambdaLV = EmitLoadOfReferenceLValue(AddrOfExplicitObject, D->getType(), AlignmentSource::Decl); else - LambdaLV = MakeNaturalAlignAddrLValue(AddrOfExplicitObject.getPointer(), - D->getType().getNonReferenceType()); + LambdaLV = MakeAddrLValue(AddrOfExplicitObject, + D->getType().getNonReferenceType()); } else { QualType LambdaTagType = getContext().getTagDeclType(Field->getParent()); LambdaLV = MakeNaturalAlignAddrLValue(ThisValue, LambdaTagType); @@ -4846,7 +4857,8 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, // information provided by invariant.group. This is because accessing // fields may leak the real address of dynamic object, which could result // in miscompilation when leaked pointer would be compared. - auto *stripped = Builder.CreateStripInvariantGroup(addr.getPointer()); + auto *stripped = + Builder.CreateStripInvariantGroup(addr.emitRawPointer(*this)); addr = Address(stripped, addr.getElementType(), addr.getAlignment()); } } @@ -4865,10 +4877,11 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, // Remember the original union field index llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(base.getType(), rec->getLocation()); - addr = Address( - Builder.CreatePreserveUnionAccessIndex( - addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo), - addr.getElementType(), addr.getAlignment()); + addr = + Address(Builder.CreatePreserveUnionAccessIndex( + addr.emitRawPointer(*this), + getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo), + addr.getElementType(), addr.getAlignment()); } if (FieldType->isReferenceType()) @@ -5105,11 +5118,9 @@ LValue CodeGenFunction::EmitConditionalOperatorLValue( if (Info.LHS && Info.RHS) { Address lhsAddr = Info.LHS->getAddress(*this); Address rhsAddr = Info.RHS->getAddress(*this); - llvm::PHINode *phi = Builder.CreatePHI(lhsAddr.getType(), 2, "cond-lvalue"); - phi->addIncoming(lhsAddr.getPointer(), Info.lhsBlock); - phi->addIncoming(rhsAddr.getPointer(), Info.rhsBlock); - Address result(phi, lhsAddr.getElementType(), - std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment())); + Address result = mergeAddressesInConditionalExpr( + lhsAddr, rhsAddr, Info.lhsBlock, Info.rhsBlock, + Builder.GetInsertBlock(), expr->getType()); AlignmentSource alignSource = std::max(Info.LHS->getBaseInfo().getAlignmentSource(), Info.RHS->getBaseInfo().getAlignmentSource()); @@ -5196,7 +5207,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { LValue LV = EmitLValue(E->getSubExpr()); Address V = LV.getAddress(*this); const auto *DCE = cast<CXXDynamicCastExpr>(E); - return MakeNaturalAlignAddrLValue(EmitDynamicCast(V, DCE), E->getType()); + return MakeNaturalAlignRawAddrLValue(EmitDynamicCast(V, DCE), E->getType()); } case CK_ConstructorConversion: @@ -5261,8 +5272,8 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is // performed and the object is not of the derived type. if (sanitizePerformTypeCheck()) - EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(), - Derived.getPointer(), E->getType()); + EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(), Derived, + E->getType()); if (SanOpts.has(SanitizerKind::CFIDerivedCast)) EmitVTablePtrCheckForCast(E->getType(), Derived, @@ -5618,7 +5629,7 @@ LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) { LValue CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) { - return MakeNaturalAlignAddrLValue(EmitCXXTypeidExpr(E), E->getType()); + return MakeNaturalAlignRawAddrLValue(EmitCXXTypeidExpr(E), E->getType()); } Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) { diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp index 5190b22bcc16..143855aa84ca 100644 --- a/clang/lib/CodeGen/CGExprAgg.cpp +++ b/clang/lib/CodeGen/CGExprAgg.cpp @@ -294,10 +294,10 @@ void AggExprEmitter::withReturnValueSlot( // Otherwise, EmitCall will emit its own, notice that it's "unused", and end // its lifetime before we have the chance to emit a proper destructor call. bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() || - (RequiresDestruction && !Dest.getAddress().isValid()); + (RequiresDestruction && Dest.isIgnored()); Address RetAddr = Address::invalid(); - Address RetAllocaAddr = Address::invalid(); + RawAddress RetAllocaAddr = RawAddress::invalid(); EHScopeStack::stable_iterator LifetimeEndBlock; llvm::Value *LifetimeSizePtr = nullptr; @@ -329,7 +329,8 @@ void AggExprEmitter::withReturnValueSlot( if (!UseTemp) return; - assert(Dest.isIgnored() || Dest.getPointer() != Src.getAggregatePointer()); + assert(Dest.isIgnored() || Dest.emitRawPointer(CGF) != + Src.getAggregatePointer(E->getType(), CGF)); EmitFinalDestCopy(E->getType(), Src); if (!RequiresDestruction && LifetimeStartInst) { @@ -448,7 +449,8 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) { llvm::Value *Zero = llvm::ConstantInt::get(CGF.PtrDiffTy, 0); llvm::Value *IdxStart[] = { Zero, Zero }; llvm::Value *ArrayStart = Builder.CreateInBoundsGEP( - ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxStart, "arraystart"); + ArrayPtr.getElementType(), ArrayPtr.emitRawPointer(CGF), IdxStart, + "arraystart"); CGF.EmitStoreThroughLValue(RValue::get(ArrayStart), Start); ++Field; @@ -465,7 +467,8 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) { // End pointer. llvm::Value *IdxEnd[] = { Zero, Size }; llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP( - ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxEnd, "arrayend"); + ArrayPtr.getElementType(), ArrayPtr.emitRawPointer(CGF), IdxEnd, + "arrayend"); CGF.EmitStoreThroughLValue(RValue::get(ArrayEnd), EndOrLength); } else if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) { // Length. @@ -516,9 +519,9 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, // down a level. llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0); llvm::Value *indices[] = { zero, zero }; - llvm::Value *begin = Builder.CreateInBoundsGEP( - DestPtr.getElementType(), DestPtr.getPointer(), indices, - "arrayinit.begin"); + llvm::Value *begin = Builder.CreateInBoundsGEP(DestPtr.getElementType(), + DestPtr.emitRawPointer(CGF), + indices, "arrayinit.begin"); CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType); CharUnits elementAlign = @@ -1059,7 +1062,7 @@ void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) { if (RV.isScalar()) return {RV.getScalarVal(), nullptr}; if (RV.isAggregate()) - return {RV.getAggregatePointer(), nullptr}; + return {RV.getAggregatePointer(E->getType(), CGF), nullptr}; assert(RV.isComplex()); return RV.getComplexVal(); }; @@ -1818,7 +1821,7 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr( // else, clean it up for -O0 builds and general tidiness. if (!pushedCleanup && LV.isSimple()) if (llvm::GetElementPtrInst *GEP = - dyn_cast<llvm::GetElementPtrInst>(LV.getPointer(CGF))) + dyn_cast<llvm::GetElementPtrInst>(LV.emitRawPointer(CGF))) if (GEP->use_empty()) GEP->eraseFromParent(); } @@ -1849,9 +1852,9 @@ void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E, // destPtr is an array*. Construct an elementType* by drilling down a level. llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0); llvm::Value *indices[] = {zero, zero}; - llvm::Value *begin = Builder.CreateInBoundsGEP( - destPtr.getElementType(), destPtr.getPointer(), indices, - "arrayinit.begin"); + llvm::Value *begin = Builder.CreateInBoundsGEP(destPtr.getElementType(), + destPtr.emitRawPointer(CGF), + indices, "arrayinit.begin"); // Prepare to special-case multidimensional array initialization: we avoid // emitting multiple destructor loops in that case. diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp index 35da0f1a89bc..5c16a48d3ee6 100644 --- a/clang/lib/CodeGen/CGExprCXX.cpp +++ b/clang/lib/CodeGen/CGExprCXX.cpp @@ -280,7 +280,8 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; Address ThisValue = EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo); - This = MakeAddrLValue(ThisValue, Base->getType(), BaseInfo, TBAAInfo); + This = MakeAddrLValue(ThisValue, Base->getType()->getPointeeType(), + BaseInfo, TBAAInfo); } else { This = EmitLValue(Base); } @@ -353,10 +354,8 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( if (IsImplicitObjectCXXThis || isa<DeclRefExpr>(IOA)) SkippedChecks.set(SanitizerKind::Null, true); } - EmitTypeCheck(CodeGenFunction::TCK_MemberCall, CallLoc, - This.getPointer(*this), - C.getRecordType(CalleeDecl->getParent()), - /*Alignment=*/CharUnits::Zero(), SkippedChecks); + EmitTypeCheck(CodeGenFunction::TCK_MemberCall, CallLoc, This, + C.getRecordType(CalleeDecl->getParent()), SkippedChecks); // C++ [class.virtual]p12: // Explicit qualification with the scope operator (5.1) suppresses the @@ -455,7 +454,7 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E, else This = EmitLValue(BaseExpr, KnownNonNull).getAddress(*this); - EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.getPointer(), + EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.emitRawPointer(*this), QualType(MPT->getClass(), 0)); // Get the member function pointer. @@ -1109,9 +1108,10 @@ void CodeGenFunction::EmitNewArrayInitializer( // alloca. EndOfInit = CreateTempAlloca(BeginPtr.getType(), getPointerAlign(), "array.init.end"); - CleanupDominator = Builder.CreateStore(BeginPtr.getPointer(), EndOfInit); - pushIrregularPartialArrayCleanup(BeginPtr.getPointer(), EndOfInit, - ElementType, ElementAlign, + CleanupDominator = + Builder.CreateStore(BeginPtr.emitRawPointer(*this), EndOfInit); + pushIrregularPartialArrayCleanup(BeginPtr.emitRawPointer(*this), + EndOfInit, ElementType, ElementAlign, getDestroyer(DtorKind)); Cleanup = EHStack.stable_begin(); } @@ -1123,16 +1123,17 @@ void CodeGenFunction::EmitNewArrayInitializer( // element. TODO: some of these stores can be trivially // observed to be unnecessary. if (EndOfInit.isValid()) { - Builder.CreateStore(CurPtr.getPointer(), EndOfInit); + Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit); } // FIXME: If the last initializer is an incomplete initializer list for // an array, and we have an array filler, we can fold together the two // initialization loops. StoreAnyExprIntoOneUnit(*this, IE, IE->getType(), CurPtr, AggValueSlot::DoesNotOverlap); - CurPtr = Address(Builder.CreateInBoundsGEP( - CurPtr.getElementType(), CurPtr.getPointer(), - Builder.getSize(1), "array.exp.next"), + CurPtr = Address(Builder.CreateInBoundsGEP(CurPtr.getElementType(), + CurPtr.emitRawPointer(*this), + Builder.getSize(1), + "array.exp.next"), CurPtr.getElementType(), StartAlign.alignmentAtOffset((++i) * ElementSize)); } @@ -1186,7 +1187,7 @@ void CodeGenFunction::EmitNewArrayInitializer( // FIXME: Share this cleanup with the constructor call emission rather than // having it create a cleanup of its own. if (EndOfInit.isValid()) - Builder.CreateStore(CurPtr.getPointer(), EndOfInit); + Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit); // Emit a constructor call loop to initialize the remaining elements. if (InitListElements) @@ -1249,15 +1250,15 @@ void CodeGenFunction::EmitNewArrayInitializer( llvm::BasicBlock *ContBB = createBasicBlock("new.loop.end"); // Find the end of the array, hoisted out of the loop. - llvm::Value *EndPtr = - Builder.CreateInBoundsGEP(BeginPtr.getElementType(), BeginPtr.getPointer(), - NumElements, "array.end"); + llvm::Value *EndPtr = Builder.CreateInBoundsGEP( + BeginPtr.getElementType(), BeginPtr.emitRawPointer(*this), NumElements, + "array.end"); // If the number of elements isn't constant, we have to now check if there is // anything left to initialize. if (!ConstNum) { - llvm::Value *IsEmpty = - Builder.CreateICmpEQ(CurPtr.getPointer(), EndPtr, "array.isempty"); + llvm::Value *IsEmpty = Builder.CreateICmpEQ(CurPtr.emitRawPointer(*this), + EndPtr, "array.isempty"); Builder.CreateCondBr(IsEmpty, ContBB, LoopBB); } @@ -1267,19 +1268,20 @@ void CodeGenFunction::EmitNewArrayInitializer( // Set up the current-element phi. llvm::PHINode *CurPtrPhi = Builder.CreatePHI(CurPtr.getType(), 2, "array.cur"); - CurPtrPhi->addIncoming(CurPtr.getPointer(), EntryBB); + CurPtrPhi->addIncoming(CurPtr.emitRawPointer(*this), EntryBB); CurPtr = Address(CurPtrPhi, CurPtr.getElementType(), ElementAlign); // Store the new Cleanup position for irregular Cleanups. if (EndOfInit.isValid()) - Builder.CreateStore(CurPtr.getPointer(), EndOfInit); + Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit); // Enter a partial-destruction Cleanup if necessary. if (!CleanupDominator && needsEHCleanup(DtorKind)) { - pushRegularPartialArrayCleanup(BeginPtr.getPointer(), CurPtr.getPointer(), - ElementType, ElementAlign, - getDestroyer(DtorKind)); + llvm::Value *BeginPtrRaw = BeginPtr.emitRawPointer(*this); + llvm::Value *CurPtrRaw = CurPtr.emitRawPointer(*this); + pushRegularPartialArrayCleanup(BeginPtrRaw, CurPtrRaw, ElementType, + ElementAlign, getDestroyer(DtorKind)); Cleanup = EHStack.stable_begin(); CleanupDominator = Builder.CreateUnreachable(); } @@ -1295,9 +1297,8 @@ void CodeGenFunction::EmitNewArrayInitializer( } // Advance to the next element by adjusting the pointer type as necessary. - llvm::Value *NextPtr = - Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr.getPointer(), 1, - "array.next"); + llvm::Value *NextPtr = Builder.CreateConstInBoundsGEP1_32( + ElementTy, CurPtr.emitRawPointer(*this), 1, "array.next"); // Check whether we've gotten to the end of the array and, if so, // exit the loop. @@ -1523,14 +1524,9 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF, typedef CallDeleteDuringNew<DirectCleanupTraits> DirectCleanup; - DirectCleanup *Cleanup = CGF.EHStack - .pushCleanupWithExtra<DirectCleanup>(EHCleanup, - E->getNumPlacementArgs(), - E->getOperatorDelete(), - NewPtr.getPointer(), - AllocSize, - E->passAlignment(), - AllocAlign); + DirectCleanup *Cleanup = CGF.EHStack.pushCleanupWithExtra<DirectCleanup>( + EHCleanup, E->getNumPlacementArgs(), E->getOperatorDelete(), + NewPtr.emitRawPointer(CGF), AllocSize, E->passAlignment(), AllocAlign); for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) { auto &Arg = NewArgs[I + NumNonPlacementArgs]; Cleanup->setPlacementArg(I, Arg.getRValue(CGF), Arg.Ty); @@ -1541,7 +1537,7 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF, // Otherwise, we need to save all this stuff. DominatingValue<RValue>::saved_type SavedNewPtr = - DominatingValue<RValue>::save(CGF, RValue::get(NewPtr.getPointer())); + DominatingValue<RValue>::save(CGF, RValue::get(NewPtr, CGF)); DominatingValue<RValue>::saved_type SavedAllocSize = DominatingValue<RValue>::save(CGF, RValue::get(AllocSize)); @@ -1618,14 +1614,14 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { // In these cases, discard the computed alignment and use the // formal alignment of the allocated type. if (BaseInfo.getAlignmentSource() != AlignmentSource::Decl) - allocation = allocation.withAlignment(allocAlign); + allocation.setAlignment(allocAlign); // Set up allocatorArgs for the call to operator delete if it's not // the reserved global operator. if (E->getOperatorDelete() && !E->getOperatorDelete()->isReservedGlobalPlacementOperator()) { allocatorArgs.add(RValue::get(allocSize), getContext().getSizeType()); - allocatorArgs.add(RValue::get(allocation.getPointer()), arg->getType()); + allocatorArgs.add(RValue::get(allocation, *this), arg->getType()); } } else { @@ -1713,8 +1709,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { llvm::BasicBlock *notNullBB = createBasicBlock("new.notnull"); contBB = createBasicBlock("new.cont"); - llvm::Value *isNull = - Builder.CreateIsNull(allocation.getPointer(), "new.isnull"); + llvm::Value *isNull = Builder.CreateIsNull(allocation, "new.isnull"); Builder.CreateCondBr(isNull, contBB, notNullBB); EmitBlock(notNullBB); } @@ -1760,12 +1755,12 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { SkippedChecks.set(SanitizerKind::Null, nullCheck); EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, E->getAllocatedTypeSourceInfo()->getTypeLoc().getBeginLoc(), - result.getPointer(), allocType, result.getAlignment(), - SkippedChecks, numElements); + result, allocType, result.getAlignment(), SkippedChecks, + numElements); EmitNewInitializer(*this, E, allocType, elementTy, result, numElements, allocSizeWithoutCookie); - llvm::Value *resultPtr = result.getPointer(); + llvm::Value *resultPtr = result.emitRawPointer(*this); if (E->isArray()) { // NewPtr is a pointer to the base element type. If we're // allocating an array of arrays, we'll need to cast back to the @@ -1909,7 +1904,8 @@ static void EmitDestroyingObjectDelete(CodeGenFunction &CGF, CGF.CGM.getCXXABI().emitVirtualObjectDelete(CGF, DE, Ptr, ElementType, Dtor); else - CGF.EmitDeleteCall(DE->getOperatorDelete(), Ptr.getPointer(), ElementType); + CGF.EmitDeleteCall(DE->getOperatorDelete(), Ptr.emitRawPointer(CGF), + ElementType); } /// Emit the code for deleting a single object. @@ -1925,8 +1921,7 @@ static bool EmitObjectDelete(CodeGenFunction &CGF, // dynamic type, the static type shall be a base class of the dynamic type // of the object to be deleted and the static type shall have a virtual // destructor or the behavior is undefined. - CGF.EmitTypeCheck(CodeGenFunction::TCK_MemberCall, - DE->getExprLoc(), Ptr.getPointer(), + CGF.EmitTypeCheck(CodeGenFunction::TCK_MemberCall, DE->getExprLoc(), Ptr, ElementType); const FunctionDecl *OperatorDelete = DE->getOperatorDelete(); @@ -1975,9 +1970,8 @@ static bool EmitObjectDelete(CodeGenFunction &CGF, // Make sure that we call delete even if the dtor throws. // This doesn't have to a conditional cleanup because we're going // to pop it off in a second. - CGF.EHStack.pushCleanup<CallObjectDelete>(NormalAndEHCleanup, - Ptr.getPointer(), - OperatorDelete, ElementType); + CGF.EHStack.pushCleanup<CallObjectDelete>( + NormalAndEHCleanup, Ptr.emitRawPointer(CGF), OperatorDelete, ElementType); if (Dtor) CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, @@ -2064,7 +2058,7 @@ static void EmitArrayDelete(CodeGenFunction &CGF, CharUnits elementAlign = deletedPtr.getAlignment().alignmentOfArrayElement(elementSize); - llvm::Value *arrayBegin = deletedPtr.getPointer(); + llvm::Value *arrayBegin = deletedPtr.emitRawPointer(CGF); llvm::Value *arrayEnd = CGF.Builder.CreateInBoundsGEP( deletedPtr.getElementType(), arrayBegin, numElements, "delete.end"); @@ -2095,7 +2089,7 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) { llvm::BasicBlock *DeleteNotNull = createBasicBlock("delete.notnull"); llvm::BasicBlock *DeleteEnd = createBasicBlock("delete.end"); - llvm::Value *IsNull = Builder.CreateIsNull(Ptr.getPointer(), "isnull"); + llvm::Value *IsNull = Builder.CreateIsNull(Ptr, "isnull"); Builder.CreateCondBr(IsNull, DeleteEnd, DeleteNotNull); EmitBlock(DeleteNotNull); @@ -2130,10 +2124,8 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) { GEP.push_back(Zero); } - Ptr = Address(Builder.CreateInBoundsGEP(Ptr.getElementType(), - Ptr.getPointer(), GEP, "del.first"), - ConvertTypeForMem(DeleteTy), Ptr.getAlignment(), - Ptr.isKnownNonNull()); + Ptr = Builder.CreateInBoundsGEP(Ptr, GEP, ConvertTypeForMem(DeleteTy), + Ptr.getAlignment(), "del.first"); } assert(ConvertTypeForMem(DeleteTy) == Ptr.getElementType()); @@ -2191,7 +2183,7 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E, // destruction and the static type of the operand is neither the constructor // or destructor’s class nor one of its bases, the behavior is undefined. CGF.EmitTypeCheck(CodeGenFunction::TCK_DynamicOperation, E->getExprLoc(), - ThisPtr.getPointer(), SrcRecordTy); + ThisPtr, SrcRecordTy); // C++ [expr.typeid]p2: // If the glvalue expression is obtained by applying the unary * operator to @@ -2207,7 +2199,7 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E, CGF.createBasicBlock("typeid.bad_typeid"); llvm::BasicBlock *EndBlock = CGF.createBasicBlock("typeid.end"); - llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr.getPointer()); + llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr); CGF.Builder.CreateCondBr(IsNull, BadTypeidBlock, EndBlock); CGF.EmitBlock(BadTypeidBlock); @@ -2293,8 +2285,7 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr, // construction or destruction and the static type of the operand is not a // pointer to or object of the constructor or destructor’s own class or one // of its bases, the dynamic_cast results in undefined behavior. - EmitTypeCheck(TCK_DynamicOperation, DCE->getExprLoc(), ThisAddr.getPointer(), - SrcRecordTy); + EmitTypeCheck(TCK_DynamicOperation, DCE->getExprLoc(), ThisAddr, SrcRecordTy); if (DCE->isAlwaysNull()) { if (llvm::Value *T = EmitDynamicCastToNull(*this, DestTy)) { @@ -2329,7 +2320,7 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr, CastNull = createBasicBlock("dynamic_cast.null"); CastNotNull = createBasicBlock("dynamic_cast.notnull"); - llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr.getPointer()); + llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr); Builder.CreateCondBr(IsNull, CastNull, CastNotNull); EmitBlock(CastNotNull); } diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp index 67a3cdc77042..36d7493d9a6b 100644 --- a/clang/lib/CodeGen/CGExprConstant.cpp +++ b/clang/lib/CodeGen/CGExprConstant.cpp @@ -800,8 +800,8 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD, // Add a vtable pointer, if we need one and it hasn't already been added. if (Layout.hasOwnVFPtr()) { llvm::Constant *VTableAddressPoint = - CGM.getCXXABI().getVTableAddressPointForConstExpr( - BaseSubobject(CD, Offset), VTableClass); + CGM.getCXXABI().getVTableAddressPoint(BaseSubobject(CD, Offset), + VTableClass); if (!AppendBytes(Offset, VTableAddressPoint)) return false; } diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp index 8536570087ad..83247aa48f86 100644 --- a/clang/lib/CodeGen/CGExprScalar.cpp +++ b/clang/lib/CodeGen/CGExprScalar.cpp @@ -2250,7 +2250,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { // performed and the object is not of the derived type. if (CGF.sanitizePerformTypeCheck()) CGF.EmitTypeCheck(CodeGenFunction::TCK_DowncastPointer, CE->getExprLoc(), - Derived.getPointer(), DestTy->getPointeeType()); + Derived, DestTy->getPointeeType()); if (CGF.SanOpts.has(SanitizerKind::CFIDerivedCast)) CGF.EmitVTablePtrCheckForCast(DestTy->getPointeeType(), Derived, @@ -2258,13 +2258,14 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { CodeGenFunction::CFITCK_DerivedCast, CE->getBeginLoc()); - return Derived.getPointer(); + return CGF.getAsNaturalPointerTo(Derived, CE->getType()->getPointeeType()); } case CK_UncheckedDerivedToBase: case CK_DerivedToBase: { // The EmitPointerWithAlignment path does this fine; just discard // the alignment. - return CGF.EmitPointerWithAlignment(CE).getPointer(); + return CGF.getAsNaturalPointerTo(CGF.EmitPointerWithAlignment(CE), + CE->getType()->getPointeeType()); } case CK_Dynamic: { @@ -2274,7 +2275,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { } case CK_ArrayToPointerDecay: - return CGF.EmitArrayToPointerDecay(E).getPointer(); + return CGF.getAsNaturalPointerTo(CGF.EmitArrayToPointerDecay(E), + CE->getType()->getPointeeType()); case CK_FunctionToPointerDecay: return EmitLValue(E).getPointer(CGF); @@ -5588,3 +5590,16 @@ CodeGenFunction::EmitCheckedInBoundsGEP(llvm::Type *ElemTy, Value *Ptr, return GEPVal; } + +Address CodeGenFunction::EmitCheckedInBoundsGEP( + Address Addr, ArrayRef<Value *> IdxList, llvm::Type *elementType, + bool SignedIndices, bool IsSubtraction, SourceLocation Loc, CharUnits Align, + const Twine &Name) { + if (!SanOpts.has(SanitizerKind::PointerOverflow)) + return Builder.CreateInBoundsGEP(Addr, IdxList, elementType, Align, Name); + + return RawAddress( + EmitCheckedInBoundsGEP(Addr.getElementType(), Addr.emitRawPointer(*this), + IdxList, SignedIndices, IsSubtraction, Loc, Name), + elementType, Align); +} diff --git a/clang/lib/CodeGen/CGNonTrivialStruct.cpp b/clang/lib/CodeGen/CGNonTrivialStruct.cpp index 75c1d7fbea84..8fade0fac21e 100644 --- a/clang/lib/CodeGen/CGNonTrivialStruct.cpp +++ b/clang/lib/CodeGen/CGNonTrivialStruct.cpp @@ -366,7 +366,7 @@ template <class Derived> struct GenFuncBase { llvm::Value *SizeInBytes = CGF.Builder.CreateNUWMul(BaseEltSizeVal, NumElts); llvm::Value *DstArrayEnd = CGF.Builder.CreateInBoundsGEP( - CGF.Int8Ty, DstAddr.getPointer(), SizeInBytes); + CGF.Int8Ty, DstAddr.emitRawPointer(CGF), SizeInBytes); llvm::BasicBlock *PreheaderBB = CGF.Builder.GetInsertBlock(); // Create the header block and insert the phi instructions. @@ -376,7 +376,7 @@ template <class Derived> struct GenFuncBase { for (unsigned I = 0; I < N; ++I) { PHIs[I] = CGF.Builder.CreatePHI(CGF.CGM.Int8PtrPtrTy, 2, "addr.cur"); - PHIs[I]->addIncoming(StartAddrs[I].getPointer(), PreheaderBB); + PHIs[I]->addIncoming(StartAddrs[I].emitRawPointer(CGF), PreheaderBB); } // Create the exit and loop body blocks. @@ -410,7 +410,7 @@ template <class Derived> struct GenFuncBase { // Instrs to update the destination and source addresses. // Update phi instructions. NewAddrs[I] = getAddrWithOffset(NewAddrs[I], EltSize); - PHIs[I]->addIncoming(NewAddrs[I].getPointer(), LoopBB); + PHIs[I]->addIncoming(NewAddrs[I].emitRawPointer(CGF), LoopBB); } // Insert an unconditional branch to the header block. @@ -488,7 +488,7 @@ template <class Derived> struct GenFuncBase { for (unsigned I = 0; I < N; ++I) { Alignments[I] = Addrs[I].getAlignment(); - Ptrs[I] = Addrs[I].getPointer(); + Ptrs[I] = Addrs[I].emitRawPointer(CallerCGF); } if (llvm::Function *F = diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp index f3a948cf13f9..c7f497a7c845 100644 --- a/clang/lib/CodeGen/CGObjC.cpp +++ b/clang/lib/CodeGen/CGObjC.cpp @@ -94,8 +94,8 @@ CodeGenFunction::EmitObjCBoxedExpr(const ObjCBoxedExpr *E) { // and cast value to correct type Address Temporary = CreateMemTemp(SubExpr->getType()); EmitAnyExprToMem(SubExpr, Temporary, Qualifiers(), /*isInit*/ true); - llvm::Value *BitCast = - Builder.CreateBitCast(Temporary.getPointer(), ConvertType(ArgQT)); + llvm::Value *BitCast = Builder.CreateBitCast( + Temporary.emitRawPointer(*this), ConvertType(ArgQT)); Args.add(RValue::get(BitCast), ArgQT); // Create char array to store type encoding @@ -204,11 +204,11 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E, ObjCMethodDecl::param_const_iterator PI = MethodWithObjects->param_begin(); const ParmVarDecl *argDecl = *PI++; QualType ArgQT = argDecl->getType().getUnqualifiedType(); - Args.add(RValue::get(Objects.getPointer()), ArgQT); + Args.add(RValue::get(Objects, *this), ArgQT); if (DLE) { argDecl = *PI++; ArgQT = argDecl->getType().getUnqualifiedType(); - Args.add(RValue::get(Keys.getPointer()), ArgQT); + Args.add(RValue::get(Keys, *this), ArgQT); } argDecl = *PI; ArgQT = argDecl->getType().getUnqualifiedType(); @@ -827,7 +827,7 @@ static void emitStructGetterCall(CodeGenFunction &CGF, ObjCIvarDecl *ivar, // sizeof (Type of Ivar), isAtomic, false); CallArgList args; - llvm::Value *dest = CGF.ReturnValue.getPointer(); + llvm::Value *dest = CGF.ReturnValue.emitRawPointer(CGF); args.add(RValue::get(dest), Context.VoidPtrTy); args.add(RValue::get(src), Context.VoidPtrTy); @@ -1147,8 +1147,8 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, callCStructCopyConstructor(Dst, Src); } else { ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl(); - emitCPPObjectAtomicGetterCall(*this, ReturnValue.getPointer(), ivar, - AtomicHelperFn); + emitCPPObjectAtomicGetterCall(*this, ReturnValue.emitRawPointer(*this), + ivar, AtomicHelperFn); } return; } @@ -1163,7 +1163,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, } else { ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl(); - emitCPPObjectAtomicGetterCall(*this, ReturnValue.getPointer(), + emitCPPObjectAtomicGetterCall(*this, ReturnValue.emitRawPointer(*this), ivar, AtomicHelperFn); } return; @@ -1287,7 +1287,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl, case TEK_Scalar: { llvm::Value *value; if (propType->isReferenceType()) { - value = LV.getAddress(*this).getPointer(); + value = LV.getAddress(*this).emitRawPointer(*this); } else { // We want to load and autoreleaseReturnValue ARC __weak ivars. if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) { @@ -1821,16 +1821,14 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ CallArgList Args; // The first argument is a temporary of the enumeration-state type. - Args.add(RValue::get(StatePtr.getPointer()), - getContext().getPointerType(StateTy)); + Args.add(RValue::get(StatePtr, *this), getContext().getPointerType(StateTy)); // The second argument is a temporary array with space for NumItems // pointers. We'll actually be loading elements from the array // pointer written into the control state; this buffer is so that // collections that *aren't* backed by arrays can still queue up // batches of elements. - Args.add(RValue::get(ItemsPtr.getPointer()), - getContext().getPointerType(ItemsTy)); + Args.add(RValue::get(ItemsPtr, *this), getContext().getPointerType(ItemsTy)); // The third argument is the capacity of that temporary array. llvm::Type *NSUIntegerTy = ConvertType(getContext().getNSUIntegerType()); @@ -2198,7 +2196,7 @@ static llvm::Value *emitARCLoadOperation(CodeGenFunction &CGF, Address addr, if (!fn) fn = getARCIntrinsic(IntID, CGF.CGM); - return CGF.EmitNounwindRuntimeCall(fn, addr.getPointer()); + return CGF.EmitNounwindRuntimeCall(fn, addr.emitRawPointer(CGF)); } /// Perform an operation having the following signature: @@ -2216,9 +2214,8 @@ static llvm::Value *emitARCStoreOperation(CodeGenFunction &CGF, Address addr, llvm::Type *origType = value->getType(); llvm::Value *args[] = { - CGF.Builder.CreateBitCast(addr.getPointer(), CGF.Int8PtrPtrTy), - CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy) - }; + CGF.Builder.CreateBitCast(addr.emitRawPointer(CGF), CGF.Int8PtrPtrTy), + CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy)}; llvm::CallInst *result = CGF.EmitNounwindRuntimeCall(fn, args); if (ignored) return nullptr; @@ -2237,9 +2234,8 @@ static void emitARCCopyOperation(CodeGenFunction &CGF, Address dst, Address src, fn = getARCIntrinsic(IntID, CGF.CGM); llvm::Value *args[] = { - CGF.Builder.CreateBitCast(dst.getPointer(), CGF.Int8PtrPtrTy), - CGF.Builder.CreateBitCast(src.getPointer(), CGF.Int8PtrPtrTy) - }; + CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), CGF.Int8PtrPtrTy), + CGF.Builder.CreateBitCast(src.emitRawPointer(CGF), CGF.Int8PtrPtrTy)}; CGF.EmitNounwindRuntimeCall(fn, args); } @@ -2490,9 +2486,8 @@ llvm::Value *CodeGenFunction::EmitARCStoreStrongCall(Address addr, fn = getARCIntrinsic(llvm::Intrinsic::objc_storeStrong, CGM); llvm::Value *args[] = { - Builder.CreateBitCast(addr.getPointer(), Int8PtrPtrTy), - Builder.CreateBitCast(value, Int8PtrTy) - }; + Builder.CreateBitCast(addr.emitRawPointer(*this), Int8PtrPtrTy), + Builder.CreateBitCast(value, Int8PtrTy)}; EmitNounwindRuntimeCall(fn, args); if (ignored) return nullptr; @@ -2643,7 +2638,7 @@ void CodeGenFunction::EmitARCDestroyWeak(Address addr) { if (!fn) fn = getARCIntrinsic(llvm::Intrinsic::objc_destroyWeak, CGM); - EmitNounwindRuntimeCall(fn, addr.getPointer()); + EmitNounwindRuntimeCall(fn, addr.emitRawPointer(*this)); } /// void \@objc_moveWeak(i8** %dest, i8** %src) diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp index a36b0cdddaf0..4e7f777ba1d9 100644 --- a/clang/lib/CodeGen/CGObjCGNU.cpp +++ b/clang/lib/CodeGen/CGObjCGNU.cpp @@ -706,7 +706,8 @@ protected: llvm::Value *cmd, MessageSendInfo &MSI) override { CGBuilderTy &Builder = CGF.Builder; llvm::Value *lookupArgs[] = { - EnforceType(Builder, ObjCSuper.getPointer(), PtrToObjCSuperTy), cmd}; + EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), PtrToObjCSuperTy), + cmd}; return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs); } @@ -761,8 +762,8 @@ class CGObjCGNUstep : public CGObjCGNU { llvm::FunctionCallee LookupFn = SlotLookupFn; // Store the receiver on the stack so that we can reload it later - Address ReceiverPtr = - CGF.CreateTempAlloca(Receiver->getType(), CGF.getPointerAlign()); + RawAddress ReceiverPtr = + CGF.CreateTempAlloca(Receiver->getType(), CGF.getPointerAlign()); Builder.CreateStore(Receiver, ReceiverPtr); llvm::Value *self; @@ -778,9 +779,9 @@ class CGObjCGNUstep : public CGObjCGNU { LookupFn2->addParamAttr(0, llvm::Attribute::NoCapture); llvm::Value *args[] = { - EnforceType(Builder, ReceiverPtr.getPointer(), PtrToIdTy), - EnforceType(Builder, cmd, SelectorTy), - EnforceType(Builder, self, IdTy) }; + EnforceType(Builder, ReceiverPtr.getPointer(), PtrToIdTy), + EnforceType(Builder, cmd, SelectorTy), + EnforceType(Builder, self, IdTy)}; llvm::CallBase *slot = CGF.EmitRuntimeCallOrInvoke(LookupFn, args); slot->setOnlyReadsMemory(); slot->setMetadata(msgSendMDKind, node); @@ -800,7 +801,7 @@ class CGObjCGNUstep : public CGObjCGNU { llvm::Value *cmd, MessageSendInfo &MSI) override { CGBuilderTy &Builder = CGF.Builder; - llvm::Value *lookupArgs[] = {ObjCSuper.getPointer(), cmd}; + llvm::Value *lookupArgs[] = {ObjCSuper.emitRawPointer(CGF), cmd}; llvm::CallInst *slot = CGF.EmitNounwindRuntimeCall(SlotLookupSuperFn, lookupArgs); @@ -1221,10 +1222,10 @@ class CGObjCGNUstep2 : public CGObjCGNUstep { llvm::Value *cmd, MessageSendInfo &MSI) override { // Don't access the slot unless we're trying to cache the result. CGBuilderTy &Builder = CGF.Builder; - llvm::Value *lookupArgs[] = {CGObjCGNU::EnforceType(Builder, - ObjCSuper.getPointer(), - PtrToObjCSuperTy), - cmd}; + llvm::Value *lookupArgs[] = { + CGObjCGNU::EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), + PtrToObjCSuperTy), + cmd}; return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs); } @@ -2186,7 +2187,8 @@ protected: llvm::Value *cmd, MessageSendInfo &MSI) override { CGBuilderTy &Builder = CGF.Builder; llvm::Value *lookupArgs[] = { - EnforceType(Builder, ObjCSuper.getPointer(), PtrToObjCSuperTy), cmd, + EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), PtrToObjCSuperTy), + cmd, }; if (CGM.ReturnTypeUsesSRet(MSI.CallInfo)) @@ -4201,15 +4203,15 @@ void CGObjCGNU::EmitThrowStmt(CodeGenFunction &CGF, llvm::Value * CGObjCGNU::EmitObjCWeakRead(CodeGenFunction &CGF, Address AddrWeakObj) { CGBuilderTy &B = CGF.Builder; - return B.CreateCall(WeakReadFn, - EnforceType(B, AddrWeakObj.getPointer(), PtrToIdTy)); + return B.CreateCall( + WeakReadFn, EnforceType(B, AddrWeakObj.emitRawPointer(CGF), PtrToIdTy)); } void CGObjCGNU::EmitObjCWeakAssign(CodeGenFunction &CGF, llvm::Value *src, Address dst) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy); + llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy); B.CreateCall(WeakAssignFn, {src, dstVal}); } @@ -4218,7 +4220,7 @@ void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF, bool threadlocal) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy); + llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy); // FIXME. Add threadloca assign API assert(!threadlocal && "EmitObjCGlobalAssign - Threal Local API NYI"); B.CreateCall(GlobalAssignFn, {src, dstVal}); @@ -4229,7 +4231,7 @@ void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF, llvm::Value *ivarOffset) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.getPointer(), IdTy); + llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), IdTy); B.CreateCall(IvarAssignFn, {src, dstVal, ivarOffset}); } @@ -4237,7 +4239,7 @@ void CGObjCGNU::EmitObjCStrongCastAssign(CodeGenFunction &CGF, llvm::Value *src, Address dst) { CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); - llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy); + llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy); B.CreateCall(StrongCastAssignFn, {src, dstVal}); } @@ -4246,8 +4248,8 @@ void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF, Address SrcPtr, llvm::Value *Size) { CGBuilderTy &B = CGF.Builder; - llvm::Value *DestPtrVal = EnforceType(B, DestPtr.getPointer(), PtrTy); - llvm::Value *SrcPtrVal = EnforceType(B, SrcPtr.getPointer(), PtrTy); + llvm::Value *DestPtrVal = EnforceType(B, DestPtr.emitRawPointer(CGF), PtrTy); + llvm::Value *SrcPtrVal = EnforceType(B, SrcPtr.emitRawPointer(CGF), PtrTy); B.CreateCall(MemMoveFn, {DestPtrVal, SrcPtrVal, Size}); } diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp index ed8d7b9a065d..8a599c10e1ca 100644 --- a/clang/lib/CodeGen/CGObjCMac.cpp +++ b/clang/lib/CodeGen/CGObjCMac.cpp @@ -1310,7 +1310,7 @@ private: /// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy, /// for the given selector. llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel); - Address EmitSelectorAddr(Selector Sel); + ConstantAddress EmitSelectorAddr(Selector Sel); public: CGObjCMac(CodeGen::CodeGenModule &cgm); @@ -1538,7 +1538,7 @@ private: /// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy, /// for the given selector. llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel); - Address EmitSelectorAddr(Selector Sel); + ConstantAddress EmitSelectorAddr(Selector Sel); /// GetInterfaceEHType - Get the cached ehtype for the given Objective-C /// interface. The return value has type EHTypePtrTy. @@ -2064,9 +2064,8 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, const ObjCMethodDecl *Method) { // Create and init a super structure; this is a (receiver, class) // pair we will pass to objc_msgSendSuper. - Address ObjCSuper = - CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(), - "objc_super"); + RawAddress ObjCSuper = CGF.CreateTempAlloca( + ObjCTypes.SuperTy, CGF.getPointerAlign(), "objc_super"); llvm::Value *ReceiverAsObject = CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy); CGF.Builder.CreateStore(ReceiverAsObject, @@ -4259,7 +4258,7 @@ namespace { CGF.EmitBlock(FinallyCallExit); CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryExitFn(), - ExceptionData.getPointer()); + ExceptionData.emitRawPointer(CGF)); CGF.EmitBlock(FinallyNoCallExit); @@ -4425,7 +4424,9 @@ void FragileHazards::emitHazardsInNewBlocks() { } static void addIfPresent(llvm::DenseSet<llvm::Value*> &S, Address V) { - if (V.isValid()) S.insert(V.getPointer()); + if (V.isValid()) + if (llvm::Value *Ptr = V.getBasePointer()) + S.insert(Ptr); } void FragileHazards::collectLocals() { @@ -4628,13 +4629,13 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // - Call objc_exception_try_enter to push ExceptionData on top of // the EH stack. CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(), - ExceptionData.getPointer()); + ExceptionData.emitRawPointer(CGF)); // - Call setjmp on the exception data buffer. llvm::Constant *Zero = llvm::ConstantInt::get(CGF.Builder.getInt32Ty(), 0); llvm::Value *GEPIndexes[] = { Zero, Zero, Zero }; llvm::Value *SetJmpBuffer = CGF.Builder.CreateGEP( - ObjCTypes.ExceptionDataTy, ExceptionData.getPointer(), GEPIndexes, + ObjCTypes.ExceptionDataTy, ExceptionData.emitRawPointer(CGF), GEPIndexes, "setjmp_buffer"); llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall( ObjCTypes.getSetJmpFn(), SetJmpBuffer, "setjmp_result"); @@ -4673,9 +4674,9 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, } else { // Retrieve the exception object. We may emit multiple blocks but // nothing can cross this so the value is already in SSA form. - llvm::CallInst *Caught = - CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(), - ExceptionData.getPointer(), "caught"); + llvm::CallInst *Caught = CGF.EmitNounwindRuntimeCall( + ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF), + "caught"); // Push the exception to rethrow onto the EH value stack for the // benefit of any @throws in the handlers. @@ -4698,7 +4699,7 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Enter a new exception try block (in case a @catch block // throws an exception). CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(), - ExceptionData.getPointer()); + ExceptionData.emitRawPointer(CGF)); llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall(ObjCTypes.getSetJmpFn(), @@ -4829,9 +4830,9 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Extract the new exception and save it to the // propagating-exception slot. assert(PropagatingExnVar.isValid()); - llvm::CallInst *NewCaught = - CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(), - ExceptionData.getPointer(), "caught"); + llvm::CallInst *NewCaught = CGF.EmitNounwindRuntimeCall( + ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF), + "caught"); CGF.Builder.CreateStore(NewCaught, PropagatingExnVar); // Don't pop the catch handler; the throw already did. @@ -4861,9 +4862,8 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Otherwise, just look in the buffer for the exception to throw. } else { - llvm::CallInst *Caught = - CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(), - ExceptionData.getPointer()); + llvm::CallInst *Caught = CGF.EmitNounwindRuntimeCall( + ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF)); PropagatingExn = Caught; } @@ -4906,7 +4906,7 @@ llvm::Value * CGObjCMac::EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF, Address AddrWeakObj) { llvm::Type* DestTy = AddrWeakObj.getElementType(); llvm::Value *AddrWeakObjVal = CGF.Builder.CreateBitCast( - AddrWeakObj.getPointer(), ObjCTypes.PtrObjectPtrTy); + AddrWeakObj.emitRawPointer(CGF), ObjCTypes.PtrObjectPtrTy); llvm::Value *read_weak = CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(), AddrWeakObjVal, "weakread"); @@ -4928,8 +4928,8 @@ void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = { src, dstVal }; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(), args, "weakassign"); @@ -4950,8 +4950,8 @@ void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; if (!threadlocal) CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(), @@ -4977,8 +4977,8 @@ void CGObjCMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal, ivarOffset}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args); } @@ -4997,8 +4997,8 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(), args, "strongassign"); @@ -5007,7 +5007,8 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF, void CGObjCMac::EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *size) { - llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), size }; + llvm::Value *args[] = {DestPtr.emitRawPointer(CGF), + SrcPtr.emitRawPointer(CGF), size}; CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args); } @@ -5243,7 +5244,7 @@ llvm::Value *CGObjCMac::EmitSelector(CodeGenFunction &CGF, Selector Sel) { return CGF.Builder.CreateLoad(EmitSelectorAddr(Sel)); } -Address CGObjCMac::EmitSelectorAddr(Selector Sel) { +ConstantAddress CGObjCMac::EmitSelectorAddr(Selector Sel) { CharUnits Align = CGM.getPointerAlign(); llvm::GlobalVariable *&Entry = SelectorReferences[Sel]; @@ -5254,7 +5255,7 @@ Address CGObjCMac::EmitSelectorAddr(Selector Sel) { Entry->setExternallyInitialized(true); } - return Address(Entry, ObjCTypes.SelectorPtrTy, Align); + return ConstantAddress(Entry, ObjCTypes.SelectorPtrTy, Align); } llvm::Constant *CGObjCCommonMac::GetClassName(StringRef RuntimeName) { @@ -7323,7 +7324,7 @@ CGObjCNonFragileABIMac::EmitVTableMessageSend(CodeGenFunction &CGF, ObjCTypes.MessageRefTy, CGF.getPointerAlign()); // Update the message ref argument. - args[1].setRValue(RValue::get(mref.getPointer())); + args[1].setRValue(RValue::get(mref, CGF)); // Load the function to call from the message ref table. Address calleeAddr = CGF.Builder.CreateStructGEP(mref, 0); @@ -7552,9 +7553,8 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, // ... // Create and init a super structure; this is a (receiver, class) // pair we will pass to objc_msgSendSuper. - Address ObjCSuper = - CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(), - "objc_super"); + RawAddress ObjCSuper = CGF.CreateTempAlloca( + ObjCTypes.SuperTy, CGF.getPointerAlign(), "objc_super"); llvm::Value *ReceiverAsObject = CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy); @@ -7594,7 +7594,7 @@ llvm::Value *CGObjCNonFragileABIMac::EmitSelector(CodeGenFunction &CGF, return LI; } -Address CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) { +ConstantAddress CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) { llvm::GlobalVariable *&Entry = SelectorReferences[Sel]; CharUnits Align = CGM.getPointerAlign(); if (!Entry) { @@ -7610,7 +7610,7 @@ Address CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) { CGM.addCompilerUsedGlobal(Entry); } - return Address(Entry, ObjCTypes.SelectorPtrTy, Align); + return ConstantAddress(Entry, ObjCTypes.SelectorPtrTy, Align); } /// EmitObjCIvarAssign - Code gen for assigning to a __strong object. @@ -7629,8 +7629,8 @@ void CGObjCNonFragileABIMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal, ivarOffset}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args); } @@ -7650,8 +7650,8 @@ void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign( src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(), args, "weakassign"); @@ -7660,7 +7660,8 @@ void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign( void CGObjCNonFragileABIMac::EmitGCMemmoveCollectable( CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *Size) { - llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), Size }; + llvm::Value *args[] = {DestPtr.emitRawPointer(CGF), + SrcPtr.emitRawPointer(CGF), Size}; CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args); } @@ -7672,7 +7673,7 @@ llvm::Value * CGObjCNonFragileABIMac::EmitObjCWeakRead( Address AddrWeakObj) { llvm::Type *DestTy = AddrWeakObj.getElementType(); llvm::Value *AddrWeakObjVal = CGF.Builder.CreateBitCast( - AddrWeakObj.getPointer(), ObjCTypes.PtrObjectPtrTy); + AddrWeakObj.emitRawPointer(CGF), ObjCTypes.PtrObjectPtrTy); llvm::Value *read_weak = CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(), AddrWeakObjVal, "weakread"); @@ -7694,8 +7695,8 @@ void CGObjCNonFragileABIMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(), args, "weakassign"); @@ -7716,8 +7717,8 @@ void CGObjCNonFragileABIMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF, src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy); } src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy); - llvm::Value *dstVal = - CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy); + llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), + ObjCTypes.PtrObjectPtrTy); llvm::Value *args[] = {src, dstVal}; if (!threadlocal) CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(), diff --git a/clang/lib/CodeGen/CGObjCRuntime.cpp b/clang/lib/CodeGen/CGObjCRuntime.cpp index 424564f97599..01d0f35da196 100644 --- a/clang/lib/CodeGen/CGObjCRuntime.cpp +++ b/clang/lib/CodeGen/CGObjCRuntime.cpp @@ -67,7 +67,7 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF, V = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, V, Offset, "add.ptr"); if (!Ivar->isBitField()) { - LValue LV = CGF.MakeNaturalAlignAddrLValue(V, IvarTy); + LValue LV = CGF.MakeNaturalAlignRawAddrLValue(V, IvarTy); return LV; } @@ -233,7 +233,7 @@ void CGObjCRuntime::EmitTryCatchStmt(CodeGenFunction &CGF, llvm::Instruction *CPICandidate = Handler.Block->getFirstNonPHI(); if (auto *CPI = dyn_cast_or_null<llvm::CatchPadInst>(CPICandidate)) { CGF.CurrentFuncletPad = CPI; - CPI->setOperand(2, CGF.getExceptionSlot().getPointer()); + CPI->setOperand(2, CGF.getExceptionSlot().emitRawPointer(CGF)); CGF.EHStack.pushCleanup<CatchRetScope>(NormalCleanup, CPI); } } @@ -405,7 +405,7 @@ bool CGObjCRuntime::canMessageReceiverBeNull(CodeGenFunction &CGF, auto self = curMethod->getSelfDecl(); if (self->getType().isConstQualified()) { if (auto LI = dyn_cast<llvm::LoadInst>(receiver->stripPointerCasts())) { - llvm::Value *selfAddr = CGF.GetAddrOfLocalVar(self).getPointer(); + llvm::Value *selfAddr = CGF.GetAddrOfLocalVar(self).emitRawPointer(CGF); if (selfAddr == LI->getPointerOperand()) { return false; } diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 00e395c2a207..bc363313dec6 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -622,7 +622,7 @@ static void emitInitWithReductionInitializer(CodeGenFunction &CGF, auto *GV = new llvm::GlobalVariable( CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, Init, Name); - LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty); + LValue LV = CGF.MakeNaturalAlignRawAddrLValue(GV, Ty); RValue InitRVal; switch (CGF.getEvaluationKind(Ty)) { case TEK_Scalar: @@ -668,8 +668,8 @@ static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, llvm::Value *SrcBegin = nullptr; if (DRD) - SrcBegin = SrcAddr.getPointer(); - llvm::Value *DestBegin = DestAddr.getPointer(); + SrcBegin = SrcAddr.emitRawPointer(CGF); + llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF); // Cast from pointer to array type to pointer to single element. llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements); @@ -912,7 +912,7 @@ static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, Address OriginalBaseAddress, llvm::Value *Addr) { - Address Tmp = Address::invalid(); + RawAddress Tmp = RawAddress::invalid(); Address TopTmp = Address::invalid(); Address MostTopTmp = Address::invalid(); BaseTy = BaseTy.getNonReferenceType(); @@ -971,10 +971,10 @@ Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address SharedAddr = SharedAddresses[N].first.getAddress(CGF); llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff( SharedAddr.getElementType(), BaseLValue.getPointer(CGF), - SharedAddr.getPointer()); + SharedAddr.emitRawPointer(CGF)); llvm::Value *PrivatePointer = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - PrivateAddr.getPointer(), SharedAddr.getType()); + PrivateAddr.emitRawPointer(CGF), SharedAddr.getType()); llvm::Value *Ptr = CGF.Builder.CreateGEP( SharedAddr.getElementType(), PrivatePointer, Adjustment); return castToBase(CGF, OrigVD->getType(), @@ -1557,7 +1557,7 @@ static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc( return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack, ParentName); } -Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) { +ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) { auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(VD); }; auto LinkageForVariable = [&VD, this]() { @@ -1579,8 +1579,8 @@ Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) { LinkageForVariable); if (!addr) - return Address::invalid(); - return Address(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD)); + return ConstantAddress::invalid(); + return ConstantAddress(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD)); } llvm::Constant * @@ -1604,7 +1604,7 @@ Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, llvm::Type *VarTy = VDAddr.getElementType(); llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), - CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy), + CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy), CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)), getOrCreateThreadPrivateCache(VD)}; return Address( @@ -1627,7 +1627,8 @@ void CGOpenMPRuntime::emitThreadPrivateVarInit( // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor) // to register constructor/destructor for variable. llvm::Value *Args[] = { - OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy), + OMPLoc, + CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.VoidPtrTy), Ctor, CopyCtor, Dtor}; CGF.EmitRuntimeCall( OMPBuilder.getOrCreateRuntimeFunction( @@ -1900,13 +1901,13 @@ void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, // OutlinedFn(>id, &zero_bound, CapturedStruct); Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc); - Address ZeroAddrBound = + RawAddress ZeroAddrBound = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, /*Name=*/".bound.zero.addr"); CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddrBound); llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs; // ThreadId for serialized parallels is 0. - OutlinedFnArgs.push_back(ThreadIDAddr.getPointer()); + OutlinedFnArgs.push_back(ThreadIDAddr.emitRawPointer(CGF)); OutlinedFnArgs.push_back(ZeroAddrBound.getPointer()); OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end()); @@ -2272,7 +2273,7 @@ void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, emitUpdateLocation(CGF, Loc), // ident_t *<loc> getThreadID(CGF, Loc), // i32 <gtid> BufSize, // size_t <buf_size> - CL.getPointer(), // void *<copyprivate list> + CL.emitRawPointer(CGF), // void *<copyprivate list> CpyFn, // void (*) (void *, void *) <copy_func> DidItVal // i32 did_it }; @@ -2591,10 +2592,10 @@ static void emitForStaticInitCall( ThreadId, CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1, M2)), // Schedule type - Values.IL.getPointer(), // &isLastIter - Values.LB.getPointer(), // &LB - Values.UB.getPointer(), // &UB - Values.ST.getPointer(), // &Stride + Values.IL.emitRawPointer(CGF), // &isLastIter + Values.LB.emitRawPointer(CGF), // &LB + Values.UB.emitRawPointer(CGF), // &UB + Values.ST.emitRawPointer(CGF), // &Stride CGF.Builder.getIntN(Values.IVSize, 1), // Incr Chunk // Chunk }; @@ -2697,12 +2698,11 @@ llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper, // kmp_int[32|64] *p_stride); llvm::Value *Args[] = { - emitUpdateLocation(CGF, Loc), - getThreadID(CGF, Loc), - IL.getPointer(), // &isLastIter - LB.getPointer(), // &Lower - UB.getPointer(), // &Upper - ST.getPointer() // &Stride + emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), + IL.emitRawPointer(CGF), // &isLastIter + LB.emitRawPointer(CGF), // &Lower + UB.emitRawPointer(CGF), // &Upper + ST.emitRawPointer(CGF) // &Stride }; llvm::Value *Call = CGF.EmitRuntimeCall( OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args); @@ -3047,7 +3047,7 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, CGF.Builder .CreatePointerBitCastOrAddrSpaceCast(TDBase.getAddress(CGF), CGF.VoidPtrTy, CGF.Int8Ty) - .getPointer()}; + .emitRawPointer(CGF)}; SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs), std::end(CommonArgs)); if (isOpenMPTaskLoopDirective(Kind)) { @@ -3574,7 +3574,8 @@ getPointerAndSize(CodeGenFunction &CGF, const Expr *E) { CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false); Address UpAddrAddress = UpAddrLVal.getAddress(CGF); llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32( - UpAddrAddress.getElementType(), UpAddrAddress.getPointer(), /*Idx0=*/1); + UpAddrAddress.getElementType(), UpAddrAddress.emitRawPointer(CGF), + /*Idx0=*/1); llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy); llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy); SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr); @@ -3888,8 +3889,9 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *Size; std::tie(Addr, Size) = getPointerAndSize(CGF, E); llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); - LValue Base = CGF.MakeAddrLValue( - CGF.Builder.CreateGEP(AffinitiesArray, Idx), KmpTaskAffinityInfoTy); + LValue Base = + CGF.MakeAddrLValue(CGF.Builder.CreateGEP(CGF, AffinitiesArray, Idx), + KmpTaskAffinityInfoTy); // affs[i].base_addr = &<Affinities[i].second>; LValue BaseAddrLVal = CGF.EmitLValueForField( Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr)); @@ -3910,7 +3912,7 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *LocRef = emitUpdateLocation(CGF, Loc); llvm::Value *GTid = getThreadID(CGF, Loc); llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - AffinitiesArray.getPointer(), CGM.VoidPtrTy); + AffinitiesArray.emitRawPointer(CGF), CGM.VoidPtrTy); // FIXME: Emit the function and ignore its result for now unless the // runtime function is properly implemented. (void)CGF.EmitRuntimeCall( @@ -3921,8 +3923,8 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *NewTaskNewTaskTTy = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( NewTask, KmpTaskTWithPrivatesPtrTy); - LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy, - KmpTaskTWithPrivatesQTy); + LValue Base = CGF.MakeNaturalAlignRawAddrLValue(NewTaskNewTaskTTy, + KmpTaskTWithPrivatesQTy); LValue TDBase = CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); // Fill the data in the resulting kmp_task_t record. @@ -4047,7 +4049,7 @@ CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, CGF.ConvertTypeForMem(KmpDependInfoPtrTy)), KmpDependInfoPtrTy->castAs<PointerType>()); Address DepObjAddr = CGF.Builder.CreateGEP( - Base.getAddress(CGF), + CGF, Base.getAddress(CGF), llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); LValue NumDepsBase = CGF.MakeAddrLValue( DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo()); @@ -4097,7 +4099,7 @@ static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, LValue &PosLVal = *Pos.get<LValue *>(); llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); Base = CGF.MakeAddrLValue( - CGF.Builder.CreateGEP(DependenciesArray, Idx), KmpDependInfoTy); + CGF.Builder.CreateGEP(CGF, DependenciesArray, Idx), KmpDependInfoTy); } // deps[i].base_addr = &<Dependencies[i].second>; LValue BaseAddrLVal = CGF.EmitLValueForField( @@ -4195,7 +4197,7 @@ void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF, ElSize, CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false)); llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); - Address DepAddr = CGF.Builder.CreateGEP(DependenciesArray, Pos); + Address DepAddr = CGF.Builder.CreateGEP(CGF, DependenciesArray, Pos); CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size); // Increase pos. @@ -4430,7 +4432,7 @@ void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy), CGF.ConvertTypeForMem(KmpDependInfoTy)); llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( - Addr.getElementType(), Addr.getPointer(), + Addr.getElementType(), Addr.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr, CGF.VoidPtrTy); @@ -4460,8 +4462,8 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, Address Begin = Base.getAddress(CGF); // Cast from pointer to array type to pointer to single element. - llvm::Value *End = CGF.Builder.CreateGEP( - Begin.getElementType(), Begin.getPointer(), NumDeps); + llvm::Value *End = CGF.Builder.CreateGEP(Begin.getElementType(), + Begin.emitRawPointer(CGF), NumDeps); // The basic structure here is a while-do loop. llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body"); llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done"); @@ -4469,7 +4471,7 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, CGF.EmitBlock(BodyBB); llvm::PHINode *ElementPHI = CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast"); - ElementPHI->addIncoming(Begin.getPointer(), EntryBB); + ElementPHI->addIncoming(Begin.emitRawPointer(CGF), EntryBB); Begin = Begin.withPointer(ElementPHI, KnownNonNull); Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo()); @@ -4483,12 +4485,12 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, FlagsLVal); // Shift the address forward by one element. - Address ElementNext = - CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext"); - ElementPHI->addIncoming(ElementNext.getPointer(), - CGF.Builder.GetInsertBlock()); + llvm::Value *ElementNext = + CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext") + .emitRawPointer(CGF); + ElementPHI->addIncoming(ElementNext, CGF.Builder.GetInsertBlock()); llvm::Value *IsEmpty = - CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty"); + CGF.Builder.CreateICmpEQ(ElementNext, End, "omp.isempty"); CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); // Done. CGF.EmitBlock(DoneBB, /*IsFinished=*/true); @@ -4531,7 +4533,7 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, DepTaskArgs[1] = ThreadID; DepTaskArgs[2] = NewTask; DepTaskArgs[3] = NumOfElements; - DepTaskArgs[4] = DependenciesArray.getPointer(); + DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF); DepTaskArgs[5] = CGF.Builder.getInt32(0); DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); } @@ -4563,7 +4565,7 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, DepWaitTaskArgs[0] = UpLoc; DepWaitTaskArgs[1] = ThreadID; DepWaitTaskArgs[2] = NumOfElements; - DepWaitTaskArgs[3] = DependenciesArray.getPointer(); + DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF); DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); DepWaitTaskArgs[6] = @@ -4725,8 +4727,8 @@ static void EmitOMPAggregateReduction( const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe(); llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr); - llvm::Value *RHSBegin = RHSAddr.getPointer(); - llvm::Value *LHSBegin = LHSAddr.getPointer(); + llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF); + llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF); // Cast from pointer to array type to pointer to single element. llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSAddr.getElementType(), LHSBegin, NumElements); @@ -4990,7 +4992,7 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, QualType ReductionArrayTy = C.getConstantArrayType( C.VoidPtrTy, ArraySize, nullptr, ArraySizeModifier::Normal, /*IndexTypeQuals=*/0); - Address ReductionList = + RawAddress ReductionList = CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); const auto *IPriv = Privates.begin(); unsigned Idx = 0; @@ -5462,7 +5464,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit( C.getConstantArrayType(RDType, ArraySize, nullptr, ArraySizeModifier::Normal, /*IndexTypeQuals=*/0); // kmp_task_red_input_t .rd_input.[Size]; - Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); + RawAddress TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionOrigs, Data.ReductionCopies, Data.ReductionOps); for (unsigned Cnt = 0; Cnt < Size; ++Cnt) { @@ -5473,7 +5475,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit( TaskRedInput.getElementType(), TaskRedInput.getPointer(), Idxs, /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc, ".rd_input.gep."); - LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType); + LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(GEP, RDType); // ElemLVal.reduce_shar = &Shareds[Cnt]; LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD); RCG.emitSharedOrigLValue(CGF, Cnt); @@ -5629,7 +5631,7 @@ void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, DepWaitTaskArgs[0] = UpLoc; DepWaitTaskArgs[1] = ThreadID; DepWaitTaskArgs[2] = NumOfElements; - DepWaitTaskArgs[3] = DependenciesArray.getPointer(); + DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF); DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); DepWaitTaskArgs[6] = @@ -5852,7 +5854,7 @@ void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF, AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy, AllocatorTraitsLVal.getBaseInfo(), AllocatorTraitsLVal.getTBAAInfo()); - llvm::Value *Traits = Addr.getPointer(); + llvm::Value *Traits = Addr.emitRawPointer(CGF); llvm::Value *AllocatorVal = CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( @@ -7312,17 +7314,19 @@ private: CGF.EmitOMPSharedLValue(MC.getAssociatedExpression()) .getAddress(CGF); } - Size = CGF.Builder.CreatePtrDiff( - CGF.Int8Ty, ComponentLB.getPointer(), LB.getPointer()); + llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF); + llvm::Value *LBPtr = LB.emitRawPointer(CGF); + Size = CGF.Builder.CreatePtrDiff(CGF.Int8Ty, ComponentLBPtr, + LBPtr); break; } } assert(Size && "Failed to determine structure size"); CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - CombinedInfo.BasePointers.push_back(BP.getPointer()); + CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF)); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - CombinedInfo.Pointers.push_back(LB.getPointer()); + CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( Size, CGF.Int64Ty, /*isSigned=*/true)); CombinedInfo.Types.push_back(Flags); @@ -7332,13 +7336,14 @@ private: LB = CGF.Builder.CreateConstGEP(ComponentLB, 1); } CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - CombinedInfo.BasePointers.push_back(BP.getPointer()); + CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF)); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - CombinedInfo.Pointers.push_back(LB.getPointer()); + CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); + llvm::Value *LBPtr = LB.emitRawPointer(CGF); Size = CGF.Builder.CreatePtrDiff( - CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).getPointer(), - LB.getPointer()); + CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).emitRawPointer(CGF), + LBPtr); CombinedInfo.Sizes.push_back( CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); CombinedInfo.Types.push_back(Flags); @@ -7356,20 +7361,21 @@ private: (Next == CE && MapType != OMPC_MAP_unknown)) { if (!IsMappingWholeStruct) { CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - CombinedInfo.BasePointers.push_back(BP.getPointer()); + CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF)); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - CombinedInfo.Pointers.push_back(LB.getPointer()); + CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( Size, CGF.Int64Ty, /*isSigned=*/true)); CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize : 1); } else { StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); - StructBaseCombinedInfo.BasePointers.push_back(BP.getPointer()); + StructBaseCombinedInfo.BasePointers.push_back( + BP.emitRawPointer(CGF)); StructBaseCombinedInfo.DevicePtrDecls.push_back(nullptr); StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); - StructBaseCombinedInfo.Pointers.push_back(LB.getPointer()); + StructBaseCombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); StructBaseCombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( Size, CGF.Int64Ty, /*isSigned=*/true)); StructBaseCombinedInfo.NonContigInfo.Dims.push_back( @@ -8211,11 +8217,11 @@ public: } CombinedInfo.Exprs.push_back(VD); // Base is the base of the struct - CombinedInfo.BasePointers.push_back(PartialStruct.Base.getPointer()); + CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF)); CombinedInfo.DevicePtrDecls.push_back(nullptr); CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); // Pointer is the address of the lowest element - llvm::Value *LB = LBAddr.getPointer(); + llvm::Value *LB = LBAddr.emitRawPointer(CGF); const CXXMethodDecl *MD = CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(CGF.CurFuncDecl) : nullptr; const CXXRecordDecl *RD = MD ? MD->getParent() : nullptr; @@ -8229,7 +8235,7 @@ public: // if the this[:1] expression had appeared in a map clause with a map-type // of tofrom. // Emit this[:1] - CombinedInfo.Pointers.push_back(PartialStruct.Base.getPointer()); + CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF)); QualType Ty = MD->getFunctionObjectParameterType(); llvm::Value *Size = CGF.Builder.CreateIntCast(CGF.getTypeSize(Ty), CGF.Int64Ty, @@ -8238,7 +8244,7 @@ public: } else { CombinedInfo.Pointers.push_back(LB); // Size is (addr of {highest+1} element) - (addr of lowest element) - llvm::Value *HB = HBAddr.getPointer(); + llvm::Value *HB = HBAddr.emitRawPointer(CGF); llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32( HBAddr.getElementType(), HB, /*Idx0=*/1); llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy); @@ -8747,7 +8753,7 @@ public: Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue( CV, ElementType, CGF.getContext().getDeclAlign(VD), AlignmentSource::Decl)); - CombinedInfo.Pointers.push_back(PtrAddr.getPointer()); + CombinedInfo.Pointers.push_back(PtrAddr.emitRawPointer(CGF)); } else { CombinedInfo.Pointers.push_back(CV); } @@ -9558,10 +9564,11 @@ static void emitTargetCallKernelLaunch( bool HasNoWait = D.hasClausesOfKind<OMPNowaitClause>(); unsigned NumTargetItems = InputInfo.NumberOfTargetItems; - llvm::Value *BasePointersArray = InputInfo.BasePointersArray.getPointer(); - llvm::Value *PointersArray = InputInfo.PointersArray.getPointer(); - llvm::Value *SizesArray = InputInfo.SizesArray.getPointer(); - llvm::Value *MappersArray = InputInfo.MappersArray.getPointer(); + llvm::Value *BasePointersArray = + InputInfo.BasePointersArray.emitRawPointer(CGF); + llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF); + llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF); + llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF); auto &&EmitTargetCallFallbackCB = [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS, @@ -10309,15 +10316,16 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall( // Source location for the ident struct llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); - llvm::Value *OffloadingArgs[] = {RTLoc, - DeviceID, - PointerNum, - InputInfo.BasePointersArray.getPointer(), - InputInfo.PointersArray.getPointer(), - InputInfo.SizesArray.getPointer(), - MapTypesArray, - MapNamesArray, - InputInfo.MappersArray.getPointer()}; + llvm::Value *OffloadingArgs[] = { + RTLoc, + DeviceID, + PointerNum, + InputInfo.BasePointersArray.emitRawPointer(CGF), + InputInfo.PointersArray.emitRawPointer(CGF), + InputInfo.SizesArray.emitRawPointer(CGF), + MapTypesArray, + MapNamesArray, + InputInfo.MappersArray.emitRawPointer(CGF)}; // Select the right runtime function call for each standalone // directive. @@ -11128,7 +11136,7 @@ void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF, getThreadID(CGF, D.getBeginLoc()), llvm::ConstantInt::getSigned(CGM.Int32Ty, NumIterations.size()), CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(), + CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).emitRawPointer(CGF), CGM.VoidPtrTy)}; llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction( @@ -11162,7 +11170,8 @@ static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, /*Volatile=*/false, Int64Ty); } llvm::Value *Args[] = { - ULoc, ThreadID, CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()}; + ULoc, ThreadID, + CGF.Builder.CreateConstArrayGEP(CntAddr, 0).emitRawPointer(CGF)}; llvm::FunctionCallee RTLFn; llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder(); OMPDoacrossKind<T> ODK; @@ -11332,7 +11341,7 @@ Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF, Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID( CGF, SourceLocation::getFromRawEncoding(LocEncoding)); Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - Addr.getPointer(), CGF.VoidPtrTy); + Addr.emitRawPointer(CGF), CGF.VoidPtrTy); llvm::Value *AllocVal = getAllocatorVal(CGF, AllocExpr); Args[2] = AllocVal; CGF.EmitRuntimeCall(RTLFn, Args); @@ -11690,15 +11699,17 @@ void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LLIVTy, getName({UniqueDeclName, "iv"})); cast<llvm::GlobalVariable>(LastIV)->setAlignment( IVLVal.getAlignment().getAsAlign()); - LValue LastIVLVal = CGF.MakeNaturalAlignAddrLValue(LastIV, IVLVal.getType()); + LValue LastIVLVal = + CGF.MakeNaturalAlignRawAddrLValue(LastIV, IVLVal.getType()); // Last value of the lastprivate conditional. // decltype(priv_a) last_a; llvm::GlobalVariable *Last = OMPBuilder.getOrCreateInternalVariable( CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName); - Last->setAlignment(LVal.getAlignment().getAsAlign()); - LValue LastLVal = CGF.MakeAddrLValue( - Address(Last, Last->getValueType(), LVal.getAlignment()), LVal.getType()); + cast<llvm::GlobalVariable>(Last)->setAlignment( + LVal.getAlignment().getAsAlign()); + LValue LastLVal = + CGF.MakeRawAddrLValue(Last, LVal.getType(), LVal.getAlignment()); // Global loop counter. Required to handle inner parallel-for regions. // iv @@ -11871,9 +11882,8 @@ void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate( // The variable was not updated in the region - exit. if (!GV) return; - LValue LPLVal = CGF.MakeAddrLValue( - Address(GV, GV->getValueType(), PrivLVal.getAlignment()), - PrivLVal.getType().getNonReferenceType()); + LValue LPLVal = CGF.MakeRawAddrLValue( + GV, PrivLVal.getType().getNonReferenceType(), PrivLVal.getAlignment()); llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc); CGF.EmitStoreOfScalar(Res, PrivLVal); } diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h index c3206427b143..522ae3d35d22 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.h +++ b/clang/lib/CodeGen/CGOpenMPRuntime.h @@ -1068,13 +1068,12 @@ public: /// \param Loc Location of the reference to threadprivate var. /// \return Address of the threadprivate variable for the current thread. virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, - const VarDecl *VD, - Address VDAddr, + const VarDecl *VD, Address VDAddr, SourceLocation Loc); /// Returns the address of the variable marked as declare target with link /// clause OR as declare target with to clause and unified memory. - virtual Address getAddrOfDeclareTargetVar(const VarDecl *VD); + virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD); /// Emit a code for initialization of threadprivate variable. It emits /// a call to runtime library which adds initial value to the newly created diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp index 299ee1460b3d..5baac8f0e3e2 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp @@ -1096,7 +1096,8 @@ void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF, llvm::PointerType *VarPtrTy = CGF.ConvertTypeForMem(VarTy)->getPointerTo(); llvm::Value *CastedVoidPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( VoidPtr, VarPtrTy, VD->getName() + "_on_stack"); - LValue VarAddr = CGF.MakeNaturalAlignAddrLValue(CastedVoidPtr, VarTy); + LValue VarAddr = + CGF.MakeNaturalAlignPointeeRawAddrLValue(CastedVoidPtr, VarTy); Rec.second.PrivateAddr = VarAddr.getAddress(CGF); Rec.second.GlobalizedVal = VoidPtr; @@ -1206,8 +1207,8 @@ void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF, bool IsBareKernel = D.getSingleClause<OMPXBareClause>(); - Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, - /*Name=*/".zero.addr"); + RawAddress ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, + /*Name=*/".zero.addr"); CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddr); llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs; // We don't emit any thread id function call in bare kernel, but because the @@ -1215,7 +1216,7 @@ void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF, if (IsBareKernel) OutlinedFnArgs.push_back(llvm::ConstantPointerNull::get(CGM.VoidPtrTy)); else - OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).getPointer()); + OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).emitRawPointer(CGF)); OutlinedFnArgs.push_back(ZeroAddr.getPointer()); OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end()); emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); @@ -1289,7 +1290,7 @@ void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF, llvm::ConstantInt::get(CGF.Int32Ty, -1), FnPtr, ID, - Bld.CreateBitOrPointerCast(CapturedVarsAddrs.getPointer(), + Bld.CreateBitOrPointerCast(CapturedVarsAddrs.emitRawPointer(CGF), CGF.VoidPtrPtrTy), llvm::ConstantInt::get(CGM.SizeTy, CapturedVars.size())}; CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( @@ -1503,17 +1504,18 @@ static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr, CGF.EmitBlock(PreCondBB); llvm::PHINode *PhiSrc = Bld.CreatePHI(Ptr.getType(), /*NumReservedValues=*/2); - PhiSrc->addIncoming(Ptr.getPointer(), CurrentBB); + PhiSrc->addIncoming(Ptr.emitRawPointer(CGF), CurrentBB); llvm::PHINode *PhiDest = Bld.CreatePHI(ElemPtr.getType(), /*NumReservedValues=*/2); - PhiDest->addIncoming(ElemPtr.getPointer(), CurrentBB); + PhiDest->addIncoming(ElemPtr.emitRawPointer(CGF), CurrentBB); Ptr = Address(PhiSrc, Ptr.getElementType(), Ptr.getAlignment()); ElemPtr = Address(PhiDest, ElemPtr.getElementType(), ElemPtr.getAlignment()); + llvm::Value *PtrEndRaw = PtrEnd.emitRawPointer(CGF); + llvm::Value *PtrRaw = Ptr.emitRawPointer(CGF); llvm::Value *PtrDiff = Bld.CreatePtrDiff( - CGF.Int8Ty, PtrEnd.getPointer(), - Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr.getPointer(), - CGF.VoidPtrTy)); + CGF.Int8Ty, PtrEndRaw, + Bld.CreatePointerBitCastOrAddrSpaceCast(PtrRaw, CGF.VoidPtrTy)); Bld.CreateCondBr(Bld.CreateICmpSGT(PtrDiff, Bld.getInt64(IntSize - 1)), ThenBB, ExitBB); CGF.EmitBlock(ThenBB); @@ -1528,8 +1530,8 @@ static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr, TBAAAccessInfo()); Address LocalPtr = Bld.CreateConstGEP(Ptr, 1); Address LocalElemPtr = Bld.CreateConstGEP(ElemPtr, 1); - PhiSrc->addIncoming(LocalPtr.getPointer(), ThenBB); - PhiDest->addIncoming(LocalElemPtr.getPointer(), ThenBB); + PhiSrc->addIncoming(LocalPtr.emitRawPointer(CGF), ThenBB); + PhiDest->addIncoming(LocalElemPtr.emitRawPointer(CGF), ThenBB); CGF.EmitBranch(PreCondBB); CGF.EmitBlock(ExitBB); } else { @@ -1676,10 +1678,10 @@ static void emitReductionListCopy( // scope and that of functions it invokes (i.e., reduce_function). // RemoteReduceData[i] = (void*)&RemoteElem if (UpdateDestListPtr) { - CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast( - DestElementAddr.getPointer(), CGF.VoidPtrTy), - DestElementPtrAddr, /*Volatile=*/false, - C.VoidPtrTy); + CGF.EmitStoreOfScalar( + Bld.CreatePointerBitCastOrAddrSpaceCast( + DestElementAddr.emitRawPointer(CGF), CGF.VoidPtrTy), + DestElementPtrAddr, /*Volatile=*/false, C.VoidPtrTy); } ++Idx; @@ -1830,7 +1832,7 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM, // elemptr = ((CopyType*)(elemptrptr)) + I Address ElemPtr(ElemPtrPtr, CopyType, Align); if (NumIters > 1) - ElemPtr = Bld.CreateGEP(ElemPtr, Cnt); + ElemPtr = Bld.CreateGEP(CGF, ElemPtr, Cnt); // Get pointer to location in transfer medium. // MediumPtr = &medium[warp_id] @@ -1894,7 +1896,7 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM, TargetElemPtrPtr, /*Volatile=*/false, C.VoidPtrTy, Loc); Address TargetElemPtr(TargetElemPtrVal, CopyType, Align); if (NumIters > 1) - TargetElemPtr = Bld.CreateGEP(TargetElemPtr, Cnt); + TargetElemPtr = Bld.CreateGEP(CGF, TargetElemPtr, Cnt); // *TargetElemPtr = SrcMediumVal; llvm::Value *SrcMediumValue = @@ -2105,9 +2107,9 @@ static llvm::Function *emitShuffleAndReduceFunction( CGF.EmitBlock(ThenBB); // reduce_function(LocalReduceList, RemoteReduceList) llvm::Value *LocalReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( - LocalReduceList.getPointer(), CGF.VoidPtrTy); + LocalReduceList.emitRawPointer(CGF), CGF.VoidPtrTy); llvm::Value *RemoteReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast( - RemoteReduceList.getPointer(), CGF.VoidPtrTy); + RemoteReduceList.emitRawPointer(CGF), CGF.VoidPtrTy); CGM.getOpenMPRuntime().emitOutlinedFunctionCall( CGF, Loc, ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr}); Bld.CreateBr(MergeBB); @@ -2218,9 +2220,9 @@ static llvm::Value *emitListToGlobalCopyFunction( llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); + CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD); Address GlobAddr = GlobLVal.getAddress(CGF); - GlobLVal.setAddress(Address(GlobAddr.getPointer(), + GlobLVal.setAddress(Address(GlobAddr.emitRawPointer(CGF), CGF.ConvertTypeForMem(Private->getType()), GlobAddr.getAlignment())); switch (CGF.getEvaluationKind(Private->getType())) { @@ -2304,7 +2306,7 @@ static llvm::Value *emitListToGlobalReduceFunction( // 1. Build a list of reduction variables. // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]}; - Address ReductionList = + RawAddress ReductionList = CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); auto IPriv = Privates.begin(); llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg), @@ -2319,10 +2321,10 @@ static llvm::Value *emitListToGlobalReduceFunction( llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); + CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD); Address GlobAddr = GlobLVal.getAddress(CGF); - CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false, - C.VoidPtrTy); + CGF.EmitStoreOfScalar(GlobAddr.emitRawPointer(CGF), Elem, + /*Volatile=*/false, C.VoidPtrTy); if ((*IPriv)->getType()->isVariablyModifiedType()) { // Store array size. ++Idx; @@ -2425,9 +2427,9 @@ static llvm::Value *emitGlobalToListCopyFunction( llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); + CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD); Address GlobAddr = GlobLVal.getAddress(CGF); - GlobLVal.setAddress(Address(GlobAddr.getPointer(), + GlobLVal.setAddress(Address(GlobAddr.emitRawPointer(CGF), CGF.ConvertTypeForMem(Private->getType()), GlobAddr.getAlignment())); switch (CGF.getEvaluationKind(Private->getType())) { @@ -2526,10 +2528,10 @@ static llvm::Value *emitGlobalToListReduceFunction( llvm::Value *BufferPtr = Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs); LValue GlobLVal = CGF.EmitLValueForField( - CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD); + CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD); Address GlobAddr = GlobLVal.getAddress(CGF); - CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false, - C.VoidPtrTy); + CGF.EmitStoreOfScalar(GlobAddr.emitRawPointer(CGF), Elem, + /*Volatile=*/false, C.VoidPtrTy); if ((*IPriv)->getType()->isVariablyModifiedType()) { // Store array size. ++Idx; @@ -2545,7 +2547,7 @@ static llvm::Value *emitGlobalToListReduceFunction( } // Call reduce_function(ReduceList, GlobalReduceList) - llvm::Value *GlobalReduceList = ReductionList.getPointer(); + llvm::Value *GlobalReduceList = ReductionList.emitRawPointer(CGF); Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg); llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar( AddrReduceListArg, /*Volatile=*/false, C.VoidPtrTy, Loc); @@ -2876,7 +2878,7 @@ void CGOpenMPRuntimeGPU::emitReduction( } llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( - ReductionList.getPointer(), CGF.VoidPtrTy); + ReductionList.emitRawPointer(CGF), CGF.VoidPtrTy); llvm::Function *ReductionFn = emitReductionFunction( CGF.CurFn->getName(), Loc, CGF.ConvertTypeForMem(ReductionArrayTy), Privates, LHSExprs, RHSExprs, ReductionOps); @@ -3106,15 +3108,15 @@ llvm::Function *CGOpenMPRuntimeGPU::createParallelDataSharingWrapper( // Get the array of arguments. SmallVector<llvm::Value *, 8> Args; - Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).getPointer()); - Args.emplace_back(ZeroAddr.getPointer()); + Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).emitRawPointer(CGF)); + Args.emplace_back(ZeroAddr.emitRawPointer(CGF)); CGBuilderTy &Bld = CGF.Builder; auto CI = CS.capture_begin(); // Use global memory for data sharing. // Handle passing of global args to workers. - Address GlobalArgs = + RawAddress GlobalArgs = CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "global_args"); llvm::Value *GlobalArgsPtr = GlobalArgs.getPointer(); llvm::Value *DataSharingArgs[] = {GlobalArgsPtr}; @@ -3400,7 +3402,7 @@ void CGOpenMPRuntimeGPU::adjustTargetSpecificDataForLambdas( VDAddr = CGF.EmitLoadOfReferenceLValue(VDAddr, VD->getType().getCanonicalType()) .getAddress(CGF); - CGF.EmitStoreOfScalar(VDAddr.getPointer(), VarLVal); + CGF.EmitStoreOfScalar(VDAddr.emitRawPointer(CGF), VarLVal); } } } diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp index cb5a004e4f4a..576fe2f7a2d4 100644 --- a/clang/lib/CodeGen/CGStmt.cpp +++ b/clang/lib/CodeGen/CGStmt.cpp @@ -2294,7 +2294,7 @@ std::pair<llvm::Value*, llvm::Type *> CodeGenFunction::EmitAsmInputLValue( Address Addr = InputValue.getAddress(*this); ConstraintStr += '*'; - return {Addr.getPointer(), Addr.getElementType()}; + return {InputValue.getPointer(*this), Addr.getElementType()}; } std::pair<llvm::Value *, llvm::Type *> @@ -2701,7 +2701,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { ArgTypes.push_back(DestAddr.getType()); ArgElemTypes.push_back(DestAddr.getElementType()); - Args.push_back(DestAddr.getPointer()); + Args.push_back(DestAddr.emitRawPointer(*this)); Constraints += "=*"; Constraints += OutputConstraint; ReadOnly = ReadNone = false; @@ -3076,8 +3076,8 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) { CapturedStmtInfo->setContextValue(Builder.CreateLoad(DeclPtr)); // Initialize variable-length arrays. - LValue Base = MakeNaturalAlignAddrLValue(CapturedStmtInfo->getContextValue(), - Ctx.getTagDeclType(RD)); + LValue Base = MakeNaturalAlignRawAddrLValue( + CapturedStmtInfo->getContextValue(), Ctx.getTagDeclType(RD)); for (auto *FD : RD->fields()) { if (FD->hasCapturedVLAType()) { auto *ExprArg = diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp index f37ac549d10a..e6d504bcdeca 100644 --- a/clang/lib/CodeGen/CGStmtOpenMP.cpp +++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp @@ -350,7 +350,8 @@ void CodeGenFunction::GenerateOpenMPCapturedVars( LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType()); llvm::Value *SrcAddrVal = EmitScalarConversion( - DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()), + DstAddr.emitRawPointer(*this), + Ctx.getPointerType(Ctx.getUIntPtrType()), Ctx.getPointerType(CurField->getType()), CurCap->getLocation()); LValue SrcLV = MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType()); @@ -364,7 +365,8 @@ void CodeGenFunction::GenerateOpenMPCapturedVars( CapturedVars.push_back(CV); } else { assert(CurCap->capturesVariable() && "Expected capture by reference."); - CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer()); + CapturedVars.push_back( + EmitLValue(*I).getAddress(*this).emitRawPointer(*this)); } } } @@ -375,8 +377,9 @@ static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, ASTContext &Ctx = CGF.getContext(); llvm::Value *CastedPtr = CGF.EmitScalarConversion( - AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(), + AddrLV.getAddress(CGF).emitRawPointer(CGF), Ctx.getUIntPtrType(), Ctx.getPointerType(DstType), Loc); + // FIXME: should the pointee type (DstType) be passed? Address TmpAddr = CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(CGF); return TmpAddr; @@ -702,8 +705,8 @@ void CodeGenFunction::EmitOMPAggregateAssign( llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr); SrcAddr = SrcAddr.withElementType(DestAddr.getElementType()); - llvm::Value *SrcBegin = SrcAddr.getPointer(); - llvm::Value *DestBegin = DestAddr.getPointer(); + llvm::Value *SrcBegin = SrcAddr.emitRawPointer(*this); + llvm::Value *DestBegin = DestAddr.emitRawPointer(*this); // Cast from pointer to array type to pointer to single element. llvm::Value *DestEnd = Builder.CreateInBoundsGEP(DestAddr.getElementType(), DestBegin, NumElements); @@ -1007,10 +1010,10 @@ bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { CopyBegin = createBasicBlock("copyin.not.master"); CopyEnd = createBasicBlock("copyin.not.master.end"); // TODO: Avoid ptrtoint conversion. - auto *MasterAddrInt = - Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy); - auto *PrivateAddrInt = - Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy); + auto *MasterAddrInt = Builder.CreatePtrToInt( + MasterAddr.emitRawPointer(*this), CGM.IntPtrTy); + auto *PrivateAddrInt = Builder.CreatePtrToInt( + PrivateAddr.emitRawPointer(*this), CGM.IntPtrTy); Builder.CreateCondBr( Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin, CopyEnd); @@ -1666,7 +1669,7 @@ Address CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate( llvm::Type *VarTy = VDAddr.getElementType(); llvm::Value *Data = - CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy); + CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy); llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)); std::string Suffix = getNameWithSeparators({"cache", ""}); llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix); @@ -2045,7 +2048,7 @@ void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) { ->getParam(0) ->getType() .getNonReferenceType(); - Address CountAddr = CreateMemTemp(LogicalTy, ".count.addr"); + RawAddress CountAddr = CreateMemTemp(LogicalTy, ".count.addr"); emitCapturedStmtCall(*this, DistanceClosure, {CountAddr.getPointer()}); llvm::Value *DistVal = Builder.CreateLoad(CountAddr, ".count"); @@ -2061,7 +2064,7 @@ void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) { LValue LCVal = EmitLValue(LoopVarRef); Address LoopVarAddress = LCVal.getAddress(*this); emitCapturedStmtCall(*this, LoopVarClosure, - {LoopVarAddress.getPointer(), IndVar}); + {LoopVarAddress.emitRawPointer(*this), IndVar}); RunCleanupsScope BodyScope(*this); EmitStmt(BodyStmt); @@ -4795,7 +4798,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( ParamTypes.push_back(PrivatesPtr->getType()); for (const Expr *E : Data.PrivateVars) { const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); - Address PrivatePtr = CGF.CreateMemTemp( + RawAddress PrivatePtr = CGF.CreateMemTemp( CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); CallArgs.push_back(PrivatePtr.getPointer()); @@ -4803,7 +4806,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( } for (const Expr *E : Data.FirstprivateVars) { const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); - Address PrivatePtr = + RawAddress PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".firstpriv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); @@ -4813,7 +4816,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( } for (const Expr *E : Data.LastprivateVars) { const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); - Address PrivatePtr = + RawAddress PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".lastpriv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); @@ -4826,7 +4829,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( Ty = CGF.getContext().getPointerType(Ty); if (isAllocatableDecl(VD)) Ty = CGF.getContext().getPointerType(Ty); - Address PrivatePtr = CGF.CreateMemTemp( + RawAddress PrivatePtr = CGF.CreateMemTemp( CGF.getContext().getPointerType(Ty), ".local.ptr.addr"); auto Result = UntiedLocalVars.insert( std::make_pair(VD, std::make_pair(PrivatePtr, Address::invalid()))); @@ -4859,7 +4862,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( if (auto *DI = CGF.getDebugInfo()) if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo()) (void)DI->EmitDeclareOfAutoVariable( - Pair.first, Pair.second.getPointer(), CGF.Builder, + Pair.first, Pair.second.getBasePointer(), CGF.Builder, /*UsePointerValue*/ true); } // Adjust mapping for internal locals by mapping actual memory instead of @@ -4912,14 +4915,14 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( RedCG, Cnt); Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); - Replacement = - Address(CGF.EmitScalarConversion( - Replacement.getPointer(), CGF.getContext().VoidPtrTy, - CGF.getContext().getPointerType( - Data.ReductionCopies[Cnt]->getType()), - Data.ReductionCopies[Cnt]->getExprLoc()), - CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), - Replacement.getAlignment()); + Replacement = Address( + CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF), + CGF.getContext().VoidPtrTy, + CGF.getContext().getPointerType( + Data.ReductionCopies[Cnt]->getType()), + Data.ReductionCopies[Cnt]->getExprLoc()), + CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), + Replacement.getAlignment()); Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement); } @@ -4970,7 +4973,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective( CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); Replacement = Address( CGF.EmitScalarConversion( - Replacement.getPointer(), CGF.getContext().VoidPtrTy, + Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy, CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), InRedPrivs[Cnt]->getExprLoc()), CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()), @@ -5089,7 +5092,7 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective( // If there is no user-defined mapper, the mapper array will be nullptr. In // this case, we don't need to privatize it. if (!isa_and_nonnull<llvm::ConstantPointerNull>( - InputInfo.MappersArray.getPointer())) { + InputInfo.MappersArray.emitRawPointer(*this))) { MVD = createImplicitFirstprivateForType( getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc()); TargetScope.addPrivate(MVD, InputInfo.MappersArray); @@ -5115,7 +5118,7 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective( ParamTypes.push_back(PrivatesPtr->getType()); for (const Expr *E : Data.FirstprivateVars) { const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); - Address PrivatePtr = + RawAddress PrivatePtr = CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), ".firstpriv.ptr.addr"); PrivatePtrs.emplace_back(VD, PrivatePtr); @@ -5194,14 +5197,14 @@ void CodeGenFunction::processInReduction(const OMPExecutableDirective &S, RedCG, Cnt); Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); - Replacement = - Address(CGF.EmitScalarConversion( - Replacement.getPointer(), CGF.getContext().VoidPtrTy, - CGF.getContext().getPointerType( - Data.ReductionCopies[Cnt]->getType()), - Data.ReductionCopies[Cnt]->getExprLoc()), - CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), - Replacement.getAlignment()); + Replacement = Address( + CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF), + CGF.getContext().VoidPtrTy, + CGF.getContext().getPointerType( + Data.ReductionCopies[Cnt]->getType()), + Data.ReductionCopies[Cnt]->getExprLoc()), + CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), + Replacement.getAlignment()); Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement); } @@ -5247,7 +5250,7 @@ void CodeGenFunction::processInReduction(const OMPExecutableDirective &S, CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); Replacement = Address( CGF.EmitScalarConversion( - Replacement.getPointer(), CGF.getContext().VoidPtrTy, + Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy, CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), InRedPrivs[Cnt]->getExprLoc()), CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()), @@ -5394,7 +5397,7 @@ void CodeGenFunction::EmitOMPDepobjDirective(const OMPDepobjDirective &S) { Dependencies.DepExprs.append(DC->varlist_begin(), DC->varlist_end()); Address DepAddr = CGM.getOpenMPRuntime().emitDepobjDependClause( *this, Dependencies, DC->getBeginLoc()); - EmitStoreOfScalar(DepAddr.getPointer(), DOLVal); + EmitStoreOfScalar(DepAddr.emitRawPointer(*this), DOLVal); return; } if (const auto *DC = S.getSingleClause<OMPDestroyClause>()) { @@ -6471,21 +6474,21 @@ static void emitOMPAtomicCompareExpr( D->getType()->hasSignedIntegerRepresentation()); llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{ - XAddr.getPointer(), XAddr.getElementType(), + XAddr.emitRawPointer(CGF), XAddr.getElementType(), X->getType()->hasSignedIntegerRepresentation(), X->getType().isVolatileQualified()}; llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal; if (V) { LValue LV = CGF.EmitLValue(V); Address Addr = LV.getAddress(CGF); - VOpVal = {Addr.getPointer(), Addr.getElementType(), + VOpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(), V->getType()->hasSignedIntegerRepresentation(), V->getType().isVolatileQualified()}; } if (R) { LValue LV = CGF.EmitLValue(R); Address Addr = LV.getAddress(CGF); - ROpVal = {Addr.getPointer(), Addr.getElementType(), + ROpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(), R->getType()->hasSignedIntegerRepresentation(), R->getType().isVolatileQualified()}; } @@ -7029,7 +7032,7 @@ void CodeGenFunction::EmitOMPInteropDirective(const OMPInteropDirective &S) { std::tie(NumDependences, DependenciesArray) = CGM.getOpenMPRuntime().emitDependClause(*this, Data.Dependences, S.getBeginLoc()); - DependenceList = DependenciesArray.getPointer(); + DependenceList = DependenciesArray.emitRawPointer(*this); } Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>(); diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp index 8dee3f74b44b..862369ae009f 100644 --- a/clang/lib/CodeGen/CGVTables.cpp +++ b/clang/lib/CodeGen/CGVTables.cpp @@ -201,14 +201,13 @@ CodeGenFunction::GenerateVarArgsThunk(llvm::Function *Fn, // Find the first store of "this", which will be to the alloca associated // with "this". - Address ThisPtr = - Address(&*AI, ConvertTypeForMem(MD->getFunctionObjectParameterType()), - CGM.getClassPointerAlignment(MD->getParent())); + Address ThisPtr = makeNaturalAddressForPointer( + &*AI, MD->getFunctionObjectParameterType(), + CGM.getClassPointerAlignment(MD->getParent())); llvm::BasicBlock *EntryBB = &Fn->front(); llvm::BasicBlock::iterator ThisStore = llvm::find_if(*EntryBB, [&](llvm::Instruction &I) { - return isa<llvm::StoreInst>(I) && - I.getOperand(0) == ThisPtr.getPointer(); + return isa<llvm::StoreInst>(I) && I.getOperand(0) == &*AI; }); assert(ThisStore != EntryBB->end() && "Store of this should be in entry block?"); diff --git a/clang/lib/CodeGen/CGValue.h b/clang/lib/CodeGen/CGValue.h index 1e6f67250583..cc9ad10ae596 100644 --- a/clang/lib/CodeGen/CGValue.h +++ b/clang/lib/CodeGen/CGValue.h @@ -14,12 +14,13 @@ #ifndef LLVM_CLANG_LIB_CODEGEN_CGVALUE_H #define LLVM_CLANG_LIB_CODEGEN_CGVALUE_H +#include "Address.h" +#include "CodeGenTBAA.h" +#include "EHScopeStack.h" #include "clang/AST/ASTContext.h" #include "clang/AST/Type.h" -#include "llvm/IR/Value.h" #include "llvm/IR/Type.h" -#include "Address.h" -#include "CodeGenTBAA.h" +#include "llvm/IR/Value.h" namespace llvm { class Constant; @@ -28,57 +29,64 @@ namespace llvm { namespace clang { namespace CodeGen { - class AggValueSlot; - class CodeGenFunction; - struct CGBitFieldInfo; +class AggValueSlot; +class CGBuilderTy; +class CodeGenFunction; +struct CGBitFieldInfo; /// RValue - This trivial value class is used to represent the result of an /// expression that is evaluated. It can be one of three things: either a /// simple LLVM SSA value, a pair of SSA values for complex numbers, or the /// address of an aggregate value in memory. class RValue { - enum Flavor { Scalar, Complex, Aggregate }; + friend struct DominatingValue<RValue>; - // The shift to make to an aggregate's alignment to make it look - // like a pointer. - enum { AggAlignShift = 4 }; + enum FlavorEnum { Scalar, Complex, Aggregate }; - // Stores first value and flavor. - llvm::PointerIntPair<llvm::Value *, 2, Flavor> V1; - // Stores second value and volatility. - llvm::PointerIntPair<llvm::Value *, 1, bool> V2; - // Stores element type for aggregate values. - llvm::Type *ElementType; + union { + // Stores first and second value. + struct { + llvm::Value *first; + llvm::Value *second; + } Vals; + + // Stores aggregate address. + Address AggregateAddr; + }; + + unsigned IsVolatile : 1; + unsigned Flavor : 2; public: - bool isScalar() const { return V1.getInt() == Scalar; } - bool isComplex() const { return V1.getInt() == Complex; } - bool isAggregate() const { return V1.getInt() == Aggregate; } + RValue() : Vals{nullptr, nullptr}, Flavor(Scalar) {} + + bool isScalar() const { return Flavor == Scalar; } + bool isComplex() const { return Flavor == Complex; } + bool isAggregate() const { return Flavor == Aggregate; } - bool isVolatileQualified() const { return V2.getInt(); } + bool isVolatileQualified() const { return IsVolatile; } /// getScalarVal() - Return the Value* of this scalar value. llvm::Value *getScalarVal() const { assert(isScalar() && "Not a scalar!"); - return V1.getPointer(); + return Vals.first; } /// getComplexVal - Return the real/imag components of this complex value. /// std::pair<llvm::Value *, llvm::Value *> getComplexVal() const { - return std::make_pair(V1.getPointer(), V2.getPointer()); + return std::make_pair(Vals.first, Vals.second); } /// getAggregateAddr() - Return the Value* of the address of the aggregate. Address getAggregateAddress() const { assert(isAggregate() && "Not an aggregate!"); - auto align = reinterpret_cast<uintptr_t>(V2.getPointer()) >> AggAlignShift; - return Address( - V1.getPointer(), ElementType, CharUnits::fromQuantity(align)); + return AggregateAddr; } - llvm::Value *getAggregatePointer() const { - assert(isAggregate() && "Not an aggregate!"); - return V1.getPointer(); + + llvm::Value *getAggregatePointer(QualType PointeeType, + CodeGenFunction &CGF) const { + return getAggregateAddress().getBasePointer(); } static RValue getIgnored() { @@ -88,17 +96,19 @@ public: static RValue get(llvm::Value *V) { RValue ER; - ER.V1.setPointer(V); - ER.V1.setInt(Scalar); - ER.V2.setInt(false); + ER.Vals.first = V; + ER.Flavor = Scalar; + ER.IsVolatile = false; return ER; } + static RValue get(Address Addr, CodeGenFunction &CGF) { + return RValue::get(Addr.emitRawPointer(CGF)); + } static RValue getComplex(llvm::Value *V1, llvm::Value *V2) { RValue ER; - ER.V1.setPointer(V1); - ER.V2.setPointer(V2); - ER.V1.setInt(Complex); - ER.V2.setInt(false); + ER.Vals = {V1, V2}; + ER.Flavor = Complex; + ER.IsVolatile = false; return ER; } static RValue getComplex(const std::pair<llvm::Value *, llvm::Value *> &C) { @@ -107,15 +117,15 @@ public: // FIXME: Aggregate rvalues need to retain information about whether they are // volatile or not. Remove default to find all places that probably get this // wrong. + + /// Convert an Address to an RValue. If the Address is not + /// signed, create an RValue using the unsigned address. Otherwise, resign the + /// address using the provided type. static RValue getAggregate(Address addr, bool isVolatile = false) { RValue ER; - ER.V1.setPointer(addr.getPointer()); - ER.V1.setInt(Aggregate); - ER.ElementType = addr.getElementType(); - - auto align = static_cast<uintptr_t>(addr.getAlignment().getQuantity()); - ER.V2.setPointer(reinterpret_cast<llvm::Value*>(align << AggAlignShift)); - ER.V2.setInt(isVolatile); + ER.AggregateAddr = addr; + ER.Flavor = Aggregate; + ER.IsVolatile = isVolatile; return ER; } }; @@ -178,8 +188,10 @@ class LValue { MatrixElt // This is a matrix element, use getVector* } LVType; - llvm::Value *V; - llvm::Type *ElementType; + union { + Address Addr = Address::invalid(); + llvm::Value *V; + }; union { // Index into a vector subscript: V[i] @@ -197,10 +209,6 @@ class LValue { // 'const' is unused here Qualifiers Quals; - // The alignment to use when accessing this lvalue. (For vector elements, - // this is the alignment of the whole vector.) - unsigned Alignment; - // objective-c's ivar bool Ivar:1; @@ -234,23 +242,19 @@ class LValue { Expr *BaseIvarExp; private: - void Initialize(QualType Type, Qualifiers Quals, CharUnits Alignment, + void Initialize(QualType Type, Qualifiers Quals, Address Addr, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { - assert((!Alignment.isZero() || Type->isIncompleteType()) && - "initializing l-value with zero alignment!"); - if (isGlobalReg()) - assert(ElementType == nullptr && "Global reg does not store elem type"); - else - assert(ElementType != nullptr && "Must have elem type"); - this->Type = Type; this->Quals = Quals; const unsigned MaxAlign = 1U << 31; - this->Alignment = Alignment.getQuantity() <= MaxAlign - ? Alignment.getQuantity() - : MaxAlign; - assert(this->Alignment == Alignment.getQuantity() && - "Alignment exceeds allowed max!"); + CharUnits Alignment = Addr.getAlignment(); + assert((isGlobalReg() || !Alignment.isZero() || Type->isIncompleteType()) && + "initializing l-value with zero alignment!"); + if (Alignment.getQuantity() > MaxAlign) { + assert(false && "Alignment exceeds allowed max!"); + Alignment = CharUnits::fromQuantity(MaxAlign); + } + this->Addr = Addr; this->BaseInfo = BaseInfo; this->TBAAInfo = TBAAInfo; @@ -259,9 +263,20 @@ private: this->ImpreciseLifetime = false; this->Nontemporal = false; this->ThreadLocalRef = false; + this->IsKnownNonNull = false; this->BaseIvarExp = nullptr; } + void initializeSimpleLValue(Address Addr, QualType Type, + LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo, + ASTContext &Context) { + Qualifiers QS = Type.getQualifiers(); + QS.setObjCGCAttr(Context.getObjCGCAttrKind(Type)); + LVType = Simple; + Initialize(Type, QS, Addr, BaseInfo, TBAAInfo); + assert(Addr.getBasePointer()->getType()->isPointerTy()); + } + public: bool isSimple() const { return LVType == Simple; } bool isVectorElt() const { return LVType == VectorElt; } @@ -328,8 +343,8 @@ public: LangAS getAddressSpace() const { return Quals.getAddressSpace(); } - CharUnits getAlignment() const { return CharUnits::fromQuantity(Alignment); } - void setAlignment(CharUnits A) { Alignment = A.getQuantity(); } + CharUnits getAlignment() const { return Addr.getAlignment(); } + void setAlignment(CharUnits A) { Addr.setAlignment(A); } LValueBaseInfo getBaseInfo() const { return BaseInfo; } void setBaseInfo(LValueBaseInfo Info) { BaseInfo = Info; } @@ -345,28 +360,32 @@ public: // simple lvalue llvm::Value *getPointer(CodeGenFunction &CGF) const { assert(isSimple()); - return V; + return Addr.getBasePointer(); } - Address getAddress(CodeGenFunction &CGF) const { - return Address(getPointer(CGF), ElementType, getAlignment(), - isKnownNonNull()); - } - void setAddress(Address address) { + llvm::Value *emitRawPointer(CodeGenFunction &CGF) const { assert(isSimple()); - V = address.getPointer(); - ElementType = address.getElementType(); - Alignment = address.getAlignment().getQuantity(); - IsKnownNonNull = address.isKnownNonNull(); + return Addr.isValid() ? Addr.emitRawPointer(CGF) : nullptr; } + Address getAddress(CodeGenFunction &CGF) const { + // FIXME: remove parameter. + return Addr; + } + + void setAddress(Address address) { Addr = address; } + // vector elt lvalue Address getVectorAddress() const { - return Address(getVectorPointer(), ElementType, getAlignment(), - (KnownNonNull_t)isKnownNonNull()); + assert(isVectorElt()); + return Addr; + } + llvm::Value *getRawVectorPointer(CodeGenFunction &CGF) const { + assert(isVectorElt()); + return Addr.emitRawPointer(CGF); } llvm::Value *getVectorPointer() const { assert(isVectorElt()); - return V; + return Addr.getBasePointer(); } llvm::Value *getVectorIdx() const { assert(isVectorElt()); @@ -374,12 +393,12 @@ public: } Address getMatrixAddress() const { - return Address(getMatrixPointer(), ElementType, getAlignment(), - (KnownNonNull_t)isKnownNonNull()); + assert(isMatrixElt()); + return Addr; } llvm::Value *getMatrixPointer() const { assert(isMatrixElt()); - return V; + return Addr.getBasePointer(); } llvm::Value *getMatrixIdx() const { assert(isMatrixElt()); @@ -388,12 +407,12 @@ public: // extended vector elements. Address getExtVectorAddress() const { - return Address(getExtVectorPointer(), ElementType, getAlignment(), - (KnownNonNull_t)isKnownNonNull()); + assert(isExtVectorElt()); + return Addr; } - llvm::Value *getExtVectorPointer() const { + llvm::Value *getRawExtVectorPointer(CodeGenFunction &CGF) const { assert(isExtVectorElt()); - return V; + return Addr.emitRawPointer(CGF); } llvm::Constant *getExtVectorElts() const { assert(isExtVectorElt()); @@ -402,10 +421,14 @@ public: // bitfield lvalue Address getBitFieldAddress() const { - return Address(getBitFieldPointer(), ElementType, getAlignment(), - (KnownNonNull_t)isKnownNonNull()); + assert(isBitField()); + return Addr; + } + llvm::Value *getRawBitFieldPointer(CodeGenFunction &CGF) const { + assert(isBitField()); + return Addr.emitRawPointer(CGF); } - llvm::Value *getBitFieldPointer() const { assert(isBitField()); return V; } + const CGBitFieldInfo &getBitFieldInfo() const { assert(isBitField()); return *BitFieldInfo; @@ -414,18 +437,13 @@ public: // global register lvalue llvm::Value *getGlobalReg() const { assert(isGlobalReg()); return V; } - static LValue MakeAddr(Address address, QualType type, ASTContext &Context, + static LValue MakeAddr(Address Addr, QualType type, ASTContext &Context, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { - Qualifiers qs = type.getQualifiers(); - qs.setObjCGCAttr(Context.getObjCGCAttrKind(type)); - LValue R; R.LVType = Simple; - assert(address.getPointer()->getType()->isPointerTy()); - R.V = address.getPointer(); - R.ElementType = address.getElementType(); - R.IsKnownNonNull = address.isKnownNonNull(); - R.Initialize(type, qs, address.getAlignment(), BaseInfo, TBAAInfo); + R.initializeSimpleLValue(Addr, type, BaseInfo, TBAAInfo, Context); + R.Addr = Addr; + assert(Addr.getType()->isPointerTy()); return R; } @@ -434,26 +452,18 @@ public: TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = VectorElt; - R.V = vecAddress.getPointer(); - R.ElementType = vecAddress.getElementType(); R.VectorIdx = Idx; - R.IsKnownNonNull = vecAddress.isKnownNonNull(); - R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(), - BaseInfo, TBAAInfo); + R.Initialize(type, type.getQualifiers(), vecAddress, BaseInfo, TBAAInfo); return R; } - static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts, + static LValue MakeExtVectorElt(Address Addr, llvm::Constant *Elts, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = ExtVectorElt; - R.V = vecAddress.getPointer(); - R.ElementType = vecAddress.getElementType(); R.VectorElts = Elts; - R.IsKnownNonNull = vecAddress.isKnownNonNull(); - R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(), - BaseInfo, TBAAInfo); + R.Initialize(type, type.getQualifiers(), Addr, BaseInfo, TBAAInfo); return R; } @@ -468,12 +478,8 @@ public: TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = BitField; - R.V = Addr.getPointer(); - R.ElementType = Addr.getElementType(); R.BitFieldInfo = &Info; - R.IsKnownNonNull = Addr.isKnownNonNull(); - R.Initialize(type, type.getQualifiers(), Addr.getAlignment(), BaseInfo, - TBAAInfo); + R.Initialize(type, type.getQualifiers(), Addr, BaseInfo, TBAAInfo); return R; } @@ -481,11 +487,9 @@ public: QualType type) { LValue R; R.LVType = GlobalReg; - R.V = V; - R.ElementType = nullptr; - R.IsKnownNonNull = true; - R.Initialize(type, type.getQualifiers(), alignment, + R.Initialize(type, type.getQualifiers(), Address::invalid(), LValueBaseInfo(AlignmentSource::Decl), TBAAAccessInfo()); + R.V = V; return R; } @@ -494,12 +498,8 @@ public: TBAAAccessInfo TBAAInfo) { LValue R; R.LVType = MatrixElt; - R.V = matAddress.getPointer(); - R.ElementType = matAddress.getElementType(); R.VectorIdx = Idx; - R.IsKnownNonNull = matAddress.isKnownNonNull(); - R.Initialize(type, type.getQualifiers(), matAddress.getAlignment(), - BaseInfo, TBAAInfo); + R.Initialize(type, type.getQualifiers(), matAddress, BaseInfo, TBAAInfo); return R; } @@ -643,17 +643,17 @@ public: return NeedsGCBarriers_t(ObjCGCFlag); } - llvm::Value *getPointer() const { - return Addr.getPointer(); + llvm::Value *getPointer(QualType PointeeTy, CodeGenFunction &CGF) const; + + llvm::Value *emitRawPointer(CodeGenFunction &CGF) const { + return Addr.isValid() ? Addr.emitRawPointer(CGF) : nullptr; } Address getAddress() const { return Addr; } - bool isIgnored() const { - return !Addr.isValid(); - } + bool isIgnored() const { return !Addr.isValid(); } CharUnits getAlignment() const { return Addr.getAlignment(); diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp index f2ebaf767452..44103884940f 100644 --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -193,26 +193,35 @@ CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() { CGF.Builder.setDefaultConstrainedRounding(OldRounding); } -LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) { +static LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, + bool ForPointeeType, + CodeGenFunction &CGF) { LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; - CharUnits Alignment = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo); - Address Addr(V, ConvertTypeForMem(T), Alignment); - return LValue::MakeAddr(Addr, T, getContext(), BaseInfo, TBAAInfo); + CharUnits Alignment = + CGF.CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo, ForPointeeType); + Address Addr = Address(V, CGF.ConvertTypeForMem(T), Alignment); + return CGF.MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo); +} + +LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) { + return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false, *this); } -/// Given a value of type T* that may not be to a complete object, -/// construct an l-value with the natural pointee alignment of T. LValue CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) { - LValueBaseInfo BaseInfo; - TBAAAccessInfo TBAAInfo; - CharUnits Align = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo, - /* forPointeeType= */ true); - Address Addr(V, ConvertTypeForMem(T), Align); - return MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo); + return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true, *this); +} + +LValue CodeGenFunction::MakeNaturalAlignRawAddrLValue(llvm::Value *V, + QualType T) { + return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false, *this); } +LValue CodeGenFunction::MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, + QualType T) { + return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true, *this); +} llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) { return CGM.getTypes().ConvertTypeForMem(T); @@ -525,7 +534,8 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) { ReturnBlock.getBlock()->eraseFromParent(); } if (ReturnValue.isValid()) { - auto *RetAlloca = dyn_cast<llvm::AllocaInst>(ReturnValue.getPointer()); + auto *RetAlloca = + dyn_cast<llvm::AllocaInst>(ReturnValue.emitRawPointer(*this)); if (RetAlloca && RetAlloca->use_empty()) { RetAlloca->eraseFromParent(); ReturnValue = Address::invalid(); @@ -1122,13 +1132,14 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, auto AI = CurFn->arg_begin(); if (CurFnInfo->getReturnInfo().isSRetAfterThis()) ++AI; - ReturnValue = - Address(&*AI, ConvertType(RetTy), - CurFnInfo->getReturnInfo().getIndirectAlign(), KnownNonNull); + ReturnValue = makeNaturalAddressForPointer( + &*AI, RetTy, CurFnInfo->getReturnInfo().getIndirectAlign(), false, + nullptr, nullptr, KnownNonNull); if (!CurFnInfo->getReturnInfo().getIndirectByVal()) { - ReturnValuePointer = CreateDefaultAlignTempAlloca( - ReturnValue.getPointer()->getType(), "result.ptr"); - Builder.CreateStore(ReturnValue.getPointer(), ReturnValuePointer); + ReturnValuePointer = + CreateDefaultAlignTempAlloca(ReturnValue.getType(), "result.ptr"); + Builder.CreateStore(ReturnValue.emitRawPointer(*this), + ReturnValuePointer); } } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca && !hasScalarEvaluationKind(CurFnInfo->getReturnType())) { @@ -1189,8 +1200,9 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, // or contains the address of the enclosing object). LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField); if (!LambdaThisCaptureField->getType()->isPointerType()) { - // If the enclosing object was captured by value, just use its address. - CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer(); + // If the enclosing object was captured by value, just use its + // address. Sign this pointer. + CXXThisValue = ThisFieldLValue.getPointer(*this); } else { // Load the lvalue pointed to by the field, since '*this' was captured // by reference. @@ -2012,8 +2024,9 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity()); Address begin = dest.withElementType(CGF.Int8Ty); - llvm::Value *end = Builder.CreateInBoundsGEP( - begin.getElementType(), begin.getPointer(), sizeInChars, "vla.end"); + llvm::Value *end = Builder.CreateInBoundsGEP(begin.getElementType(), + begin.emitRawPointer(CGF), + sizeInChars, "vla.end"); llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock(); llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop"); @@ -2024,7 +2037,7 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, CGF.EmitBlock(loopBB); llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur"); - cur->addIncoming(begin.getPointer(), originBB); + cur->addIncoming(begin.emitRawPointer(CGF), originBB); CharUnits curAlign = dest.getAlignment().alignmentOfArrayElement(baseSize); @@ -2217,10 +2230,10 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType, addr = addr.withElementType(baseType); } else { // Create the actual GEP. - addr = Address(Builder.CreateInBoundsGEP( - addr.getElementType(), addr.getPointer(), gepIndices, "array.begin"), - ConvertTypeForMem(eltType), - addr.getAlignment()); + addr = Address(Builder.CreateInBoundsGEP(addr.getElementType(), + addr.emitRawPointer(*this), + gepIndices, "array.begin"), + ConvertTypeForMem(eltType), addr.getAlignment()); } baseType = eltType; @@ -2561,7 +2574,7 @@ void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) { Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D, Address Addr) { assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute"); - llvm::Value *V = Addr.getPointer(); + llvm::Value *V = Addr.emitRawPointer(*this); llvm::Type *VTy = V->getType(); auto *PTy = dyn_cast<llvm::PointerType>(VTy); unsigned AS = PTy ? PTy->getAddressSpace() : 0; diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index e8f8aa601ed0..8dd6da5f85f1 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -151,6 +151,9 @@ struct DominatingLLVMValue { /// Answer whether the given value needs extra work to be saved. static bool needsSaving(llvm::Value *value) { + if (!value) + return false; + // If it's not an instruction, we don't need to save. if (!isa<llvm::Instruction>(value)) return false; @@ -177,21 +180,28 @@ template <> struct DominatingValue<Address> { typedef Address type; struct saved_type { - DominatingLLVMValue::saved_type SavedValue; + DominatingLLVMValue::saved_type BasePtr; llvm::Type *ElementType; CharUnits Alignment; + DominatingLLVMValue::saved_type Offset; + llvm::PointerType *EffectiveType; }; static bool needsSaving(type value) { - return DominatingLLVMValue::needsSaving(value.getPointer()); + if (DominatingLLVMValue::needsSaving(value.getBasePointer()) || + DominatingLLVMValue::needsSaving(value.getOffset())) + return true; + return false; } static saved_type save(CodeGenFunction &CGF, type value) { - return { DominatingLLVMValue::save(CGF, value.getPointer()), - value.getElementType(), value.getAlignment() }; + return {DominatingLLVMValue::save(CGF, value.getBasePointer()), + value.getElementType(), value.getAlignment(), + DominatingLLVMValue::save(CGF, value.getOffset()), value.getType()}; } static type restore(CodeGenFunction &CGF, saved_type value) { - return Address(DominatingLLVMValue::restore(CGF, value.SavedValue), - value.ElementType, value.Alignment); + return Address(DominatingLLVMValue::restore(CGF, value.BasePtr), + value.ElementType, value.Alignment, + DominatingLLVMValue::restore(CGF, value.Offset)); } }; @@ -201,14 +211,26 @@ template <> struct DominatingValue<RValue> { class saved_type { enum Kind { ScalarLiteral, ScalarAddress, AggregateLiteral, AggregateAddress, ComplexAddress }; - - llvm::Value *Value; - llvm::Type *ElementType; + union { + struct { + DominatingLLVMValue::saved_type first, second; + } Vals; + DominatingValue<Address>::saved_type AggregateAddr; + }; LLVM_PREFERRED_TYPE(Kind) unsigned K : 3; - unsigned Align : 29; - saved_type(llvm::Value *v, llvm::Type *e, Kind k, unsigned a = 0) - : Value(v), ElementType(e), K(k), Align(a) {} + unsigned IsVolatile : 1; + + saved_type(DominatingLLVMValue::saved_type Val1, unsigned K) + : Vals{Val1, DominatingLLVMValue::saved_type()}, K(K) {} + + saved_type(DominatingLLVMValue::saved_type Val1, + DominatingLLVMValue::saved_type Val2) + : Vals{Val1, Val2}, K(ComplexAddress) {} + + saved_type(DominatingValue<Address>::saved_type AggregateAddr, + bool IsVolatile, unsigned K) + : AggregateAddr(AggregateAddr), K(K) {} public: static bool needsSaving(RValue value); @@ -659,7 +681,7 @@ public: llvm::Value *Size; public: - CallLifetimeEnd(Address addr, llvm::Value *size) + CallLifetimeEnd(RawAddress addr, llvm::Value *size) : Addr(addr.getPointer()), Size(size) {} void Emit(CodeGenFunction &CGF, Flags flags) override { @@ -684,7 +706,7 @@ public: }; /// i32s containing the indexes of the cleanup destinations. - Address NormalCleanupDest = Address::invalid(); + RawAddress NormalCleanupDest = RawAddress::invalid(); unsigned NextCleanupDestIndex = 1; @@ -819,10 +841,10 @@ public: template <class T, class... As> void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) { if (!isInConditionalBranch()) - return pushCleanupAfterFullExprWithActiveFlag<T>(Kind, Address::invalid(), - A...); + return pushCleanupAfterFullExprWithActiveFlag<T>( + Kind, RawAddress::invalid(), A...); - Address ActiveFlag = createCleanupActiveFlag(); + RawAddress ActiveFlag = createCleanupActiveFlag(); assert(!DominatingValue<Address>::needsSaving(ActiveFlag) && "cleanup active flag should never need saving"); @@ -835,7 +857,7 @@ public: template <class T, class... As> void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind, - Address ActiveFlag, As... A) { + RawAddress ActiveFlag, As... A) { LifetimeExtendedCleanupHeader Header = {sizeof(T), Kind, ActiveFlag.isValid()}; @@ -850,7 +872,7 @@ public: new (Buffer) LifetimeExtendedCleanupHeader(Header); new (Buffer + sizeof(Header)) T(A...); if (Header.IsConditional) - new (Buffer + sizeof(Header) + sizeof(T)) Address(ActiveFlag); + new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag); } /// Set up the last cleanup that was pushed as a conditional @@ -859,8 +881,8 @@ public: initFullExprCleanupWithFlag(createCleanupActiveFlag()); } - void initFullExprCleanupWithFlag(Address ActiveFlag); - Address createCleanupActiveFlag(); + void initFullExprCleanupWithFlag(RawAddress ActiveFlag); + RawAddress createCleanupActiveFlag(); /// PushDestructorCleanup - Push a cleanup to call the /// complete-object destructor of an object of the given type at the @@ -1048,7 +1070,7 @@ public: QualType VarTy = LocalVD->getType(); if (VarTy->isReferenceType()) { Address Temp = CGF.CreateMemTemp(VarTy); - CGF.Builder.CreateStore(TempAddr.getPointer(), Temp); + CGF.Builder.CreateStore(TempAddr.emitRawPointer(CGF), Temp); TempAddr = Temp; } SavedTempAddresses.try_emplace(LocalVD, TempAddr); @@ -1243,10 +1265,12 @@ public: /// one branch or the other of a conditional expression. bool isInConditionalBranch() const { return OutermostConditional != nullptr; } - void setBeforeOutermostConditional(llvm::Value *value, Address addr) { + void setBeforeOutermostConditional(llvm::Value *value, Address addr, + CodeGenFunction &CGF) { assert(isInConditionalBranch()); llvm::BasicBlock *block = OutermostConditional->getStartingBlock(); - auto store = new llvm::StoreInst(value, addr.getPointer(), &block->back()); + auto store = + new llvm::StoreInst(value, addr.emitRawPointer(CGF), &block->back()); store->setAlignment(addr.getAlignment().getAsAlign()); } @@ -1601,7 +1625,7 @@ public: /// If \p StepV is null, the default increment is 1. void maybeUpdateMCDCTestVectorBitmap(const Expr *E) { if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) { - PGO.emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr); + PGO.emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr, *this); PGO.setCurrentStmt(E); } } @@ -1609,7 +1633,7 @@ public: /// Update the MCDC temp value with the condition's evaluated result. void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val) { if (isMCDCCoverageEnabled()) { - PGO.emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val); + PGO.emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val, *this); PGO.setCurrentStmt(E); } } @@ -1704,7 +1728,7 @@ public: : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue), OldCXXThisAlignment(CGF.CXXThisAlignment), SourceLocScope(E, CGF.CurSourceLocExprScope) { - CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getPointer(); + CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer(); CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment(); } ~CXXDefaultInitExprScope() { @@ -2090,7 +2114,7 @@ public: llvm::Value *getExceptionFromSlot(); llvm::Value *getSelectorFromSlot(); - Address getNormalCleanupDestSlot(); + RawAddress getNormalCleanupDestSlot(); llvm::BasicBlock *getUnreachableBlock() { if (!UnreachableBlock) { @@ -2579,10 +2603,40 @@ public: // Helpers //===--------------------------------------------------------------------===// + Address mergeAddressesInConditionalExpr(Address LHS, Address RHS, + llvm::BasicBlock *LHSBlock, + llvm::BasicBlock *RHSBlock, + llvm::BasicBlock *MergeBlock, + QualType MergedType) { + Builder.SetInsertPoint(MergeBlock); + llvm::PHINode *PtrPhi = Builder.CreatePHI(LHS.getType(), 2, "cond"); + PtrPhi->addIncoming(LHS.getBasePointer(), LHSBlock); + PtrPhi->addIncoming(RHS.getBasePointer(), RHSBlock); + LHS.replaceBasePointer(PtrPhi); + LHS.setAlignment(std::min(LHS.getAlignment(), RHS.getAlignment())); + return LHS; + } + + /// Construct an address with the natural alignment of T. If a pointer to T + /// is expected to be signed, the pointer passed to this function must have + /// been signed, and the returned Address will have the pointer authentication + /// information needed to authenticate the signed pointer. + Address makeNaturalAddressForPointer( + llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(), + bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr, + TBAAAccessInfo *TBAAInfo = nullptr, + KnownNonNull_t IsKnownNonNull = NotKnownNonNull) { + if (Alignment.isZero()) + Alignment = + CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, ForPointeeType); + return Address(Ptr, ConvertTypeForMem(T), Alignment, nullptr, + IsKnownNonNull); + } + LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source = AlignmentSource::Type) { - return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source), - CGM.getTBAAAccessInfo(T)); + return MakeAddrLValue(Addr, T, LValueBaseInfo(Source), + CGM.getTBAAAccessInfo(T)); } LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo, @@ -2592,6 +2646,14 @@ public: LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, AlignmentSource Source = AlignmentSource::Type) { + return MakeAddrLValue(makeNaturalAddressForPointer(V, T, Alignment), T, + LValueBaseInfo(Source), CGM.getTBAAAccessInfo(T)); + } + + /// Same as MakeAddrLValue above except that the pointer is known to be + /// unsigned. + LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment, + AlignmentSource Source = AlignmentSource::Type) { Address Addr(V, ConvertTypeForMem(T), Alignment); return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source), CGM.getTBAAAccessInfo(T)); @@ -2604,9 +2666,18 @@ public: TBAAAccessInfo()); } + /// Given a value of type T* that may not be to a complete object, construct + /// an l-value with the natural pointee alignment of T. LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T); + LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T); + /// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known + /// to be unsigned. + LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T); + + LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T); + Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo = nullptr, TBAAAccessInfo *PointeeTBAAInfo = nullptr); @@ -2655,13 +2726,13 @@ public: /// more efficient if the caller knows that the address will not be exposed. llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp", llvm::Value *ArraySize = nullptr); - Address CreateTempAlloca(llvm::Type *Ty, CharUnits align, - const Twine &Name = "tmp", - llvm::Value *ArraySize = nullptr, - Address *Alloca = nullptr); - Address CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align, - const Twine &Name = "tmp", - llvm::Value *ArraySize = nullptr); + RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align, + const Twine &Name = "tmp", + llvm::Value *ArraySize = nullptr, + RawAddress *Alloca = nullptr); + RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align, + const Twine &Name = "tmp", + llvm::Value *ArraySize = nullptr); /// CreateDefaultAlignedTempAlloca - This creates an alloca with the /// default ABI alignment of the given LLVM type. @@ -2673,8 +2744,8 @@ public: /// not hand this address off to arbitrary IRGen routines, and especially /// do not pass it as an argument to a function that might expect a /// properly ABI-aligned value. - Address CreateDefaultAlignTempAlloca(llvm::Type *Ty, - const Twine &Name = "tmp"); + RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty, + const Twine &Name = "tmp"); /// CreateIRTemp - Create a temporary IR object of the given type, with /// appropriate alignment. This routine should only be used when an temporary @@ -2684,32 +2755,31 @@ public: /// /// That is, this is exactly equivalent to CreateMemTemp, but calling /// ConvertType instead of ConvertTypeForMem. - Address CreateIRTemp(QualType T, const Twine &Name = "tmp"); + RawAddress CreateIRTemp(QualType T, const Twine &Name = "tmp"); /// CreateMemTemp - Create a temporary memory object of the given type, with /// appropriate alignmen and cast it to the default address space. Returns /// the original alloca instruction by \p Alloca if it is not nullptr. - Address CreateMemTemp(QualType T, const Twine &Name = "tmp", - Address *Alloca = nullptr); - Address CreateMemTemp(QualType T, CharUnits Align, const Twine &Name = "tmp", - Address *Alloca = nullptr); + RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp", + RawAddress *Alloca = nullptr); + RawAddress CreateMemTemp(QualType T, CharUnits Align, + const Twine &Name = "tmp", + RawAddress *Alloca = nullptr); /// CreateMemTemp - Create a temporary memory object of the given type, with /// appropriate alignmen without casting it to the default address space. - Address CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp"); - Address CreateMemTempWithoutCast(QualType T, CharUnits Align, - const Twine &Name = "tmp"); + RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp"); + RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align, + const Twine &Name = "tmp"); /// CreateAggTemp - Create a temporary memory object for the given /// aggregate type. AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp", - Address *Alloca = nullptr) { - return AggValueSlot::forAddr(CreateMemTemp(T, Name, Alloca), - T.getQualifiers(), - AggValueSlot::IsNotDestructed, - AggValueSlot::DoesNotNeedGCBarriers, - AggValueSlot::IsNotAliased, - AggValueSlot::DoesNotOverlap); + RawAddress *Alloca = nullptr) { + return AggValueSlot::forAddr( + CreateMemTemp(T, Name, Alloca), T.getQualifiers(), + AggValueSlot::IsNotDestructed, AggValueSlot::DoesNotNeedGCBarriers, + AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap); } /// EvaluateExprAsBool - Perform the usual unary conversions on the specified @@ -3083,6 +3153,25 @@ public: /// calls to EmitTypeCheck can be skipped. bool sanitizePerformTypeCheck() const; + void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV, + QualType Type, SanitizerSet SkippedChecks = SanitizerSet(), + llvm::Value *ArraySize = nullptr) { + if (!sanitizePerformTypeCheck()) + return; + EmitTypeCheck(TCK, Loc, LV.emitRawPointer(*this), Type, LV.getAlignment(), + SkippedChecks, ArraySize); + } + + void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr, + QualType Type, CharUnits Alignment = CharUnits::Zero(), + SanitizerSet SkippedChecks = SanitizerSet(), + llvm::Value *ArraySize = nullptr) { + if (!sanitizePerformTypeCheck()) + return; + EmitTypeCheck(TCK, Loc, Addr.emitRawPointer(*this), Type, Alignment, + SkippedChecks, ArraySize); + } + /// Emit a check that \p V is the address of storage of the /// appropriate size and alignment for an object of type \p Type /// (or if ArraySize is provided, for an array of that bound). @@ -3183,17 +3272,17 @@ public: /// Address with original alloca instruction. Invalid if the variable was /// emitted as a global constant. - Address AllocaAddr; + RawAddress AllocaAddr; struct Invalid {}; AutoVarEmission(Invalid) : Variable(nullptr), Addr(Address::invalid()), - AllocaAddr(Address::invalid()) {} + AllocaAddr(RawAddress::invalid()) {} AutoVarEmission(const VarDecl &variable) : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr), IsEscapingByRef(false), IsConstantAggregate(false), - SizeForLifetimeMarkers(nullptr), AllocaAddr(Address::invalid()) {} + SizeForLifetimeMarkers(nullptr), AllocaAddr(RawAddress::invalid()) {} bool wasEmittedAsGlobal() const { return !Addr.isValid(); } @@ -3216,7 +3305,7 @@ public: } /// Returns the address for the original alloca instruction. - Address getOriginalAllocatedAddress() const { return AllocaAddr; } + RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; } /// Returns the address of the object within this declaration. /// Note that this does not chase the forwarding pointer for @@ -3246,23 +3335,32 @@ public: llvm::GlobalValue::LinkageTypes Linkage); class ParamValue { - llvm::Value *Value; - llvm::Type *ElementType; - unsigned Alignment; - ParamValue(llvm::Value *V, llvm::Type *T, unsigned A) - : Value(V), ElementType(T), Alignment(A) {} + union { + Address Addr; + llvm::Value *Value; + }; + + bool IsIndirect; + + ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {} + ParamValue(Address A) : Addr(A), IsIndirect(true) {} + public: static ParamValue forDirect(llvm::Value *value) { - return ParamValue(value, nullptr, 0); + return ParamValue(value); } static ParamValue forIndirect(Address addr) { assert(!addr.getAlignment().isZero()); - return ParamValue(addr.getPointer(), addr.getElementType(), - addr.getAlignment().getQuantity()); + return ParamValue(addr); } - bool isIndirect() const { return Alignment != 0; } - llvm::Value *getAnyValue() const { return Value; } + bool isIndirect() const { return IsIndirect; } + llvm::Value *getAnyValue() const { + if (!isIndirect()) + return Value; + assert(!Addr.hasOffset() && "unexpected offset"); + return Addr.getBasePointer(); + } llvm::Value *getDirectValue() const { assert(!isIndirect()); @@ -3271,8 +3369,7 @@ public: Address getIndirectAddress() const { assert(isIndirect()); - return Address(Value, ElementType, CharUnits::fromQuantity(Alignment), - KnownNonNull); + return Addr; } }; @@ -4183,6 +4280,9 @@ public: llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name = ""); llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee, + ArrayRef<Address> args, + const Twine &name = ""); + llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args, const Twine &name = ""); @@ -4208,6 +4308,12 @@ public: CXXDtorType Type, const CXXRecordDecl *RD); + llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) { + return Addr.getBasePointer(); + } + + bool isPointerKnownNonNull(const Expr *E); + // Return the copy constructor name with the prefix "__copy_constructor_" // removed. static std::string getNonTrivialCopyConstructorStr(QualType QT, @@ -4780,6 +4886,11 @@ public: SourceLocation Loc, const Twine &Name = ""); + Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList, + llvm::Type *elementType, bool SignedIndices, + bool IsSubtraction, SourceLocation Loc, + CharUnits Align, const Twine &Name = ""); + /// Specifies which type of sanitizer check to apply when handling a /// particular builtin. enum BuiltinCheckKind { @@ -4842,6 +4953,10 @@ public: void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc, AbstractCallee AC, unsigned ParmNum); + void EmitNonNullArgCheck(Address Addr, QualType ArgType, + SourceLocation ArgLoc, AbstractCallee AC, + unsigned ParmNum); + /// EmitCallArg - Emit a single call argument. void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType); @@ -5050,7 +5165,7 @@ DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) { CGF.CreateTempAlloca(value->getType(), align, "cond-cleanup.save"); CGF.Builder.CreateStore(value, alloca); - return saved_type(alloca.getPointer(), true); + return saved_type(alloca.emitRawPointer(CGF), true); } inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF, diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index e3ed5e90f2d3..00b3bfcaa0bc 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -7267,7 +7267,7 @@ void CodeGenFunction::EmitDeclMetadata() { for (auto &I : LocalDeclMap) { const Decl *D = I.first; - llvm::Value *Addr = I.second.getPointer(); + llvm::Value *Addr = I.second.emitRawPointer(*this); if (auto *Alloca = dyn_cast<llvm::AllocaInst>(Addr)) { llvm::Value *DAddr = GetPointerConstant(getLLVMContext(), D); Alloca->setMetadata( diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp index 2619edfeb7dc..76704c4d7be4 100644 --- a/clang/lib/CodeGen/CodeGenPGO.cpp +++ b/clang/lib/CodeGen/CodeGenPGO.cpp @@ -1239,7 +1239,8 @@ void CodeGenPGO::emitMCDCParameters(CGBuilderTy &Builder) { void CodeGenPGO::emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, const Expr *S, - Address MCDCCondBitmapAddr) { + Address MCDCCondBitmapAddr, + CodeGenFunction &CGF) { if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState) return; @@ -1262,7 +1263,7 @@ void CodeGenPGO::emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, Builder.getInt64(FunctionHash), Builder.getInt32(RegionMCDCState->BitmapBytes), Builder.getInt32(MCDCTestVectorBitmapOffset), - MCDCCondBitmapAddr.getPointer()}; + MCDCCondBitmapAddr.emitRawPointer(CGF)}; Builder.CreateCall( CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_tvbitmap_update), Args); } @@ -1283,7 +1284,8 @@ void CodeGenPGO::emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S, void CodeGenPGO::emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, Address MCDCCondBitmapAddr, - llvm::Value *Val) { + llvm::Value *Val, + CodeGenFunction &CGF) { if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState) return; @@ -1312,7 +1314,7 @@ void CodeGenPGO::emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, llvm::Value *Args[5] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), Builder.getInt64(FunctionHash), Builder.getInt32(Branch.ID), - MCDCCondBitmapAddr.getPointer(), Val}; + MCDCCondBitmapAddr.emitRawPointer(CGF), Val}; Builder.CreateCall( CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_condbitmap_update), Args); diff --git a/clang/lib/CodeGen/CodeGenPGO.h b/clang/lib/CodeGen/CodeGenPGO.h index 036fbf6815a4..9d66ffad6f43 100644 --- a/clang/lib/CodeGen/CodeGenPGO.h +++ b/clang/lib/CodeGen/CodeGenPGO.h @@ -113,12 +113,14 @@ public: void emitCounterSetOrIncrement(CGBuilderTy &Builder, const Stmt *S, llvm::Value *StepV); void emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, const Expr *S, - Address MCDCCondBitmapAddr); + Address MCDCCondBitmapAddr, + CodeGenFunction &CGF); void emitMCDCParameters(CGBuilderTy &Builder); void emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S, Address MCDCCondBitmapAddr); void emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, - Address MCDCCondBitmapAddr, llvm::Value *Val); + Address MCDCCondBitmapAddr, llvm::Value *Val, + CodeGenFunction &CGF); /// Return the region count for the counter at the given index. uint64_t getRegionCount(const Stmt *S) { diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp index bdd53a192f82..fd71317572f0 100644 --- a/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -307,10 +307,6 @@ public: CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base, const CXXRecordDecl *NearestVBase); - llvm::Constant * - getVTableAddressPointForConstExpr(BaseSubobject Base, - const CXXRecordDecl *VTableClass) override; - llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) override; @@ -646,7 +642,7 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer( // Apply the adjustment and cast back to the original struct type // for consistency. - llvm::Value *This = ThisAddr.getPointer(); + llvm::Value *This = ThisAddr.emitRawPointer(CGF); This = Builder.CreateInBoundsGEP(Builder.getInt8Ty(), This, Adj); ThisPtrForCall = This; @@ -850,7 +846,7 @@ llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress( CGBuilderTy &Builder = CGF.Builder; // Apply the offset, which we assume is non-null. - return Builder.CreateInBoundsGEP(CGF.Int8Ty, Base.getPointer(), MemPtr, + return Builder.CreateInBoundsGEP(CGF.Int8Ty, Base.emitRawPointer(CGF), MemPtr, "memptr.offset"); } @@ -1245,7 +1241,7 @@ void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF, CGF.getPointerAlign()); // Apply the offset. - llvm::Value *CompletePtr = Ptr.getPointer(); + llvm::Value *CompletePtr = Ptr.emitRawPointer(CGF); CompletePtr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, CompletePtr, Offset); @@ -1482,7 +1478,8 @@ llvm::Value *ItaniumCXXABI::emitDynamicCastCall( computeOffsetHint(CGF.getContext(), SrcDecl, DestDecl).getQuantity()); // Emit the call to __dynamic_cast. - llvm::Value *Args[] = {ThisAddr.getPointer(), SrcRTTI, DestRTTI, OffsetHint}; + llvm::Value *Args[] = {ThisAddr.emitRawPointer(CGF), SrcRTTI, DestRTTI, + OffsetHint}; llvm::Value *Value = CGF.EmitNounwindRuntimeCall(getItaniumDynamicCastFn(CGF), Args); @@ -1571,7 +1568,7 @@ llvm::Value *ItaniumCXXABI::emitExactDynamicCast( VPtr, CGM.getTBAAVTablePtrAccessInfo(CGF.VoidPtrPtrTy)); llvm::Value *Success = CGF.Builder.CreateICmpEQ( VPtr, getVTableAddressPoint(BaseSubobject(SrcDecl, *Offset), DestDecl)); - llvm::Value *Result = ThisAddr.getPointer(); + llvm::Value *Result = ThisAddr.emitRawPointer(CGF); if (!Offset->isZero()) Result = CGF.Builder.CreateInBoundsGEP( CGF.CharTy, Result, @@ -1611,7 +1608,7 @@ llvm::Value *ItaniumCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF, PtrDiffLTy, OffsetToTop, CGF.getPointerAlign(), "offset.to.top"); } // Finally, add the offset to the pointer. - return CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ThisAddr.getPointer(), + return CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ThisAddr.emitRawPointer(CGF), OffsetToTop); } @@ -1792,8 +1789,8 @@ void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF, else Callee = CGCallee::forDirect(CGM.getAddrOfCXXStructor(GD), GD); - CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, VTT, VTTTy, - nullptr); + CGF.EmitCXXDestructorCall(GD, Callee, CGF.getAsNaturalPointerTo(This, ThisTy), + ThisTy, VTT, VTTTy, nullptr); } void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, @@ -1952,11 +1949,6 @@ llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructorWithVTT( CGF.getPointerAlign()); } -llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr( - BaseSubobject Base, const CXXRecordDecl *VTableClass) { - return getVTableAddressPoint(Base, VTableClass); -} - llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) { assert(VPtrOffset.isZero() && "Itanium ABI only supports zero vptr offsets"); @@ -2088,8 +2080,8 @@ llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall( ThisTy = D->getDestroyedType(); } - CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, nullptr, - QualType(), nullptr); + CGF.EmitCXXDestructorCall(GD, Callee, This.emitRawPointer(CGF), ThisTy, + nullptr, QualType(), nullptr); return nullptr; } @@ -2162,7 +2154,7 @@ static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, int64_t VirtualAdjustment, bool IsReturnAdjustment) { if (!NonVirtualAdjustment && !VirtualAdjustment) - return InitialPtr.getPointer(); + return InitialPtr.emitRawPointer(CGF); Address V = InitialPtr.withElementType(CGF.Int8Ty); @@ -2195,10 +2187,10 @@ static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, CGF.getPointerAlign()); } // Adjust our pointer. - ResultPtr = CGF.Builder.CreateInBoundsGEP( - V.getElementType(), V.getPointer(), Offset); + ResultPtr = CGF.Builder.CreateInBoundsGEP(V.getElementType(), + V.emitRawPointer(CGF), Offset); } else { - ResultPtr = V.getPointer(); + ResultPtr = V.emitRawPointer(CGF); } // In a derived-to-base conversion, the non-virtual adjustment is @@ -2284,7 +2276,7 @@ Address ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF, llvm::FunctionType::get(CGM.VoidTy, NumElementsPtr.getType(), false); llvm::FunctionCallee F = CGM.CreateRuntimeFunction(FTy, "__asan_poison_cxx_array_cookie"); - CGF.Builder.CreateCall(F, NumElementsPtr.getPointer()); + CGF.Builder.CreateCall(F, NumElementsPtr.emitRawPointer(CGF)); } // Finally, compute a pointer to the actual data buffer by skipping @@ -2315,7 +2307,7 @@ llvm::Value *ItaniumCXXABI::readArrayCookieImpl(CodeGenFunction &CGF, llvm::FunctionType::get(CGF.SizeTy, CGF.UnqualPtrTy, false); llvm::FunctionCallee F = CGM.CreateRuntimeFunction(FTy, "__asan_load_cxx_array_cookie"); - return CGF.Builder.CreateCall(F, numElementsPtr.getPointer()); + return CGF.Builder.CreateCall(F, numElementsPtr.emitRawPointer(CGF)); } CharUnits ARMCXXABI::getArrayCookieSizeImpl(QualType elementType) { @@ -2627,7 +2619,7 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF, // Call __cxa_guard_release. This cannot throw. CGF.EmitNounwindRuntimeCall(getGuardReleaseFn(CGM, guardPtrTy), - guardAddr.getPointer()); + guardAddr.emitRawPointer(CGF)); } else if (D.isLocalVarDecl()) { // For local variables, store 1 into the first byte of the guard variable // after the object initialization completes so that initialization is @@ -3120,10 +3112,10 @@ LValue ItaniumCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, LValue LV; if (VD->getType()->isReferenceType()) - LV = CGF.MakeNaturalAlignAddrLValue(CallVal, LValType); + LV = CGF.MakeNaturalAlignRawAddrLValue(CallVal, LValType); else - LV = CGF.MakeAddrLValue(CallVal, LValType, - CGF.getContext().getDeclAlign(VD)); + LV = CGF.MakeRawAddrLValue(CallVal, LValType, + CGF.getContext().getDeclAlign(VD)); // FIXME: need setObjCGCLValueClass? return LV; } @@ -4604,7 +4596,7 @@ static void InitCatchParam(CodeGenFunction &CGF, CGF.Builder.CreateStore(Casted, ExnPtrTmp); // Bind the reference to the temporary. - AdjustedExn = ExnPtrTmp.getPointer(); + AdjustedExn = ExnPtrTmp.emitRawPointer(CGF); } } diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp index 172c4c937b97..d38a26940a3c 100644 --- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -327,10 +327,6 @@ public: CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base, const CXXRecordDecl *NearestVBase) override; - llvm::Constant * - getVTableAddressPointForConstExpr(BaseSubobject Base, - const CXXRecordDecl *VTableClass) override; - llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) override; @@ -937,7 +933,7 @@ void MicrosoftCXXABI::emitBeginCatch(CodeGenFunction &CGF, } CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(*CatchParam); - CPI->setArgOperand(2, var.getObjectAddress(CGF).getPointer()); + CPI->setArgOperand(2, var.getObjectAddress(CGF).emitRawPointer(CGF)); CGF.EHStack.pushCleanup<CatchRetScope>(NormalCleanup, CPI); CGF.EmitAutoVarCleanups(var); } @@ -974,7 +970,7 @@ MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, Address Value, llvm::Value *Offset = GetVirtualBaseClassOffset(CGF, Value, SrcDecl, PolymorphicBase); llvm::Value *Ptr = CGF.Builder.CreateInBoundsGEP( - Value.getElementType(), Value.getPointer(), Offset); + Value.getElementType(), Value.emitRawPointer(CGF), Offset); CharUnits VBaseAlign = CGF.CGM.getVBaseAlignment(Value.getAlignment(), SrcDecl, PolymorphicBase); return std::make_tuple(Address(Ptr, CGF.Int8Ty, VBaseAlign), Offset, @@ -1011,7 +1007,7 @@ llvm::Value *MicrosoftCXXABI::EmitTypeid(CodeGenFunction &CGF, llvm::Type *StdTypeInfoPtrTy) { std::tie(ThisPtr, std::ignore, std::ignore) = performBaseAdjustment(CGF, ThisPtr, SrcRecordTy); - llvm::CallBase *Typeid = emitRTtypeidCall(CGF, ThisPtr.getPointer()); + llvm::CallBase *Typeid = emitRTtypeidCall(CGF, ThisPtr.emitRawPointer(CGF)); return CGF.Builder.CreateBitCast(Typeid, StdTypeInfoPtrTy); } @@ -1033,7 +1029,7 @@ llvm::Value *MicrosoftCXXABI::emitDynamicCastCall( llvm::Value *Offset; std::tie(This, Offset, std::ignore) = performBaseAdjustment(CGF, This, SrcRecordTy); - llvm::Value *ThisPtr = This.getPointer(); + llvm::Value *ThisPtr = This.emitRawPointer(CGF); Offset = CGF.Builder.CreateTrunc(Offset, CGF.Int32Ty); // PVOID __RTDynamicCast( @@ -1065,7 +1061,7 @@ llvm::Value *MicrosoftCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF, llvm::FunctionCallee Function = CGF.CGM.CreateRuntimeFunction( llvm::FunctionType::get(CGF.Int8PtrTy, ArgTypes, false), "__RTCastToVoid"); - llvm::Value *Args[] = {Value.getPointer()}; + llvm::Value *Args[] = {Value.emitRawPointer(CGF)}; return CGF.EmitRuntimeCall(Function, Args); } @@ -1493,7 +1489,7 @@ Address MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall( llvm::Value *VBaseOffset = GetVirtualBaseClassOffset(CGF, Result, Derived, VBase); llvm::Value *VBasePtr = CGF.Builder.CreateInBoundsGEP( - Result.getElementType(), Result.getPointer(), VBaseOffset); + Result.getElementType(), Result.emitRawPointer(CGF), VBaseOffset); CharUnits VBaseAlign = CGF.CGM.getVBaseAlignment(Result.getAlignment(), Derived, VBase); Result = Address(VBasePtr, CGF.Int8Ty, VBaseAlign); @@ -1660,7 +1656,8 @@ void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF, llvm::Value *Implicit = getCXXDestructorImplicitParam(CGF, DD, Type, ForVirtualBase, Delegating); // = nullptr - CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, + CGF.EmitCXXDestructorCall(GD, Callee, CGF.getAsNaturalPointerTo(This, ThisTy), + ThisTy, /*ImplicitParam=*/Implicit, /*ImplicitParamTy=*/QualType(), nullptr); if (BaseDtorEndBB) { @@ -1791,13 +1788,6 @@ MicrosoftCXXABI::getVTableAddressPoint(BaseSubobject Base, return VFTablesMap[ID]; } -llvm::Constant *MicrosoftCXXABI::getVTableAddressPointForConstExpr( - BaseSubobject Base, const CXXRecordDecl *VTableClass) { - llvm::Constant *VFTable = getVTableAddressPoint(Base, VTableClass); - assert(VFTable && "Couldn't find a vftable for the given base?"); - return VFTable; -} - llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, CharUnits VPtrOffset) { // getAddrOfVTable may return 0 if asked to get an address of a vtable which @@ -2013,8 +2003,9 @@ llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall( } This = adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true); - RValue RV = CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, - ImplicitParam, Context.IntTy, CE); + RValue RV = + CGF.EmitCXXDestructorCall(GD, Callee, This.emitRawPointer(CGF), ThisTy, + ImplicitParam, Context.IntTy, CE); return RV.getScalarVal(); } @@ -2212,13 +2203,13 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF, Address This, const ThisAdjustment &TA) { if (TA.isEmpty()) - return This.getPointer(); + return This.emitRawPointer(CGF); This = This.withElementType(CGF.Int8Ty); llvm::Value *V; if (TA.Virtual.isEmpty()) { - V = This.getPointer(); + V = This.emitRawPointer(CGF); } else { assert(TA.Virtual.Microsoft.VtordispOffset < 0); // Adjust the this argument based on the vtordisp value. @@ -2227,7 +2218,7 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF, CharUnits::fromQuantity(TA.Virtual.Microsoft.VtordispOffset)); VtorDispPtr = VtorDispPtr.withElementType(CGF.Int32Ty); llvm::Value *VtorDisp = CGF.Builder.CreateLoad(VtorDispPtr, "vtordisp"); - V = CGF.Builder.CreateGEP(This.getElementType(), This.getPointer(), + V = CGF.Builder.CreateGEP(This.getElementType(), This.emitRawPointer(CGF), CGF.Builder.CreateNeg(VtorDisp)); // Unfortunately, having applied the vtordisp means that we no @@ -2264,11 +2255,11 @@ llvm::Value * MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret, const ReturnAdjustment &RA) { if (RA.isEmpty()) - return Ret.getPointer(); + return Ret.emitRawPointer(CGF); Ret = Ret.withElementType(CGF.Int8Ty); - llvm::Value *V = Ret.getPointer(); + llvm::Value *V = Ret.emitRawPointer(CGF); if (RA.Virtual.Microsoft.VBIndex) { assert(RA.Virtual.Microsoft.VBIndex > 0); int32_t IntSize = CGF.getIntSize().getQuantity(); @@ -2583,7 +2574,7 @@ struct ResetGuardBit final : EHScopeStack::Cleanup { struct CallInitThreadAbort final : EHScopeStack::Cleanup { llvm::Value *Guard; - CallInitThreadAbort(Address Guard) : Guard(Guard.getPointer()) {} + CallInitThreadAbort(RawAddress Guard) : Guard(Guard.getPointer()) {} void Emit(CodeGenFunction &CGF, Flags flags) override { // Calling _Init_thread_abort will reset the guard's state. @@ -3123,8 +3114,8 @@ MicrosoftCXXABI::GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF, llvm::Value **VBPtrOut) { CGBuilderTy &Builder = CGF.Builder; // Load the vbtable pointer from the vbptr in the instance. - llvm::Value *VBPtr = Builder.CreateInBoundsGEP(CGM.Int8Ty, This.getPointer(), - VBPtrOffset, "vbptr"); + llvm::Value *VBPtr = Builder.CreateInBoundsGEP( + CGM.Int8Ty, This.emitRawPointer(CGF), VBPtrOffset, "vbptr"); if (VBPtrOut) *VBPtrOut = VBPtr; @@ -3203,7 +3194,7 @@ llvm::Value *MicrosoftCXXABI::AdjustVirtualBase( Builder.CreateBr(SkipAdjustBB); CGF.EmitBlock(SkipAdjustBB); llvm::PHINode *Phi = Builder.CreatePHI(CGM.Int8PtrTy, 2, "memptr.base"); - Phi->addIncoming(Base.getPointer(), OriginalBB); + Phi->addIncoming(Base.emitRawPointer(CGF), OriginalBB); Phi->addIncoming(AdjustedBase, VBaseAdjustBB); return Phi; } @@ -3238,7 +3229,7 @@ llvm::Value *MicrosoftCXXABI::EmitMemberDataPointerAddress( Addr = AdjustVirtualBase(CGF, E, RD, Base, VirtualBaseAdjustmentOffset, VBPtrOffset); } else { - Addr = Base.getPointer(); + Addr = Base.emitRawPointer(CGF); } // Apply the offset, which we assume is non-null. @@ -3526,7 +3517,7 @@ CGCallee MicrosoftCXXABI::EmitLoadOfMemberFunctionPointer( ThisPtrForCall = AdjustVirtualBase(CGF, E, RD, This, VirtualBaseAdjustmentOffset, VBPtrOffset); } else { - ThisPtrForCall = This.getPointer(); + ThisPtrForCall = This.emitRawPointer(CGF); } if (NonVirtualBaseAdjustment) @@ -4445,10 +4436,7 @@ void MicrosoftCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) { llvm::GlobalVariable *TI = getThrowInfo(ThrowType); // Call into the runtime to throw the exception. - llvm::Value *Args[] = { - AI.getPointer(), - TI - }; + llvm::Value *Args[] = {AI.emitRawPointer(CGF), TI}; CGF.EmitNoreturnRuntimeCallOrInvoke(getThrowFn(), Args); } diff --git a/clang/lib/CodeGen/TargetInfo.h b/clang/lib/CodeGen/TargetInfo.h index 6893b50a3cfe..b1dfe5bf8f27 100644 --- a/clang/lib/CodeGen/TargetInfo.h +++ b/clang/lib/CodeGen/TargetInfo.h @@ -295,6 +295,11 @@ public: /// Get the AST address space for alloca. virtual LangAS getASTAllocaAddressSpace() const { return LangAS::Default; } + Address performAddrSpaceCast(CodeGen::CodeGenFunction &CGF, Address Addr, + LangAS SrcAddr, LangAS DestAddr, + llvm::Type *DestTy, + bool IsNonNull = false) const; + /// Perform address space cast of an expression of pointer type. /// \param V is the LLVM value to be casted to another address space. /// \param SrcAddr is the language address space of \p V. diff --git a/clang/lib/CodeGen/Targets/NVPTX.cpp b/clang/lib/CodeGen/Targets/NVPTX.cpp index 8718f1ecf3a7..7dce5042c3dc 100644 --- a/clang/lib/CodeGen/Targets/NVPTX.cpp +++ b/clang/lib/CodeGen/Targets/NVPTX.cpp @@ -85,7 +85,7 @@ private: LValue Src) { llvm::Value *Handle = nullptr; llvm::Constant *C = - llvm::dyn_cast<llvm::Constant>(Src.getAddress(CGF).getPointer()); + llvm::dyn_cast<llvm::Constant>(Src.getAddress(CGF).emitRawPointer(CGF)); // Lookup `addrspacecast` through the constant pointer if any. if (auto *ASC = llvm::dyn_cast_or_null<llvm::AddrSpaceCastOperator>(C)) C = llvm::cast<llvm::Constant>(ASC->getPointerOperand()); diff --git a/clang/lib/CodeGen/Targets/PPC.cpp b/clang/lib/CodeGen/Targets/PPC.cpp index 00b04723f17d..362add30b435 100644 --- a/clang/lib/CodeGen/Targets/PPC.cpp +++ b/clang/lib/CodeGen/Targets/PPC.cpp @@ -513,9 +513,10 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 8); llvm::Value *RegOffset = Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity())); - RegAddr = Address( - Builder.CreateInBoundsGEP(CGF.Int8Ty, RegAddr.getPointer(), RegOffset), - DirectTy, RegAddr.getAlignment().alignmentOfArrayElement(RegSize)); + RegAddr = Address(Builder.CreateInBoundsGEP( + CGF.Int8Ty, RegAddr.emitRawPointer(CGF), RegOffset), + DirectTy, + RegAddr.getAlignment().alignmentOfArrayElement(RegSize)); // Increase the used-register count. NumRegs = @@ -551,7 +552,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, // Round up address of argument to alignment CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty); if (Align > OverflowAreaAlign) { - llvm::Value *Ptr = OverflowArea.getPointer(); + llvm::Value *Ptr = OverflowArea.emitRawPointer(CGF); OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align), OverflowArea.getElementType(), Align); } @@ -560,7 +561,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList, // Increase the overflow area. OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size); - Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr); + Builder.CreateStore(OverflowArea.emitRawPointer(CGF), OverflowAreaAddr); CGF.EmitBranch(Cont); } diff --git a/clang/lib/CodeGen/Targets/Sparc.cpp b/clang/lib/CodeGen/Targets/Sparc.cpp index a337a52a94ec..9025a633f328 100644 --- a/clang/lib/CodeGen/Targets/Sparc.cpp +++ b/clang/lib/CodeGen/Targets/Sparc.cpp @@ -326,7 +326,7 @@ Address SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, // Update VAList. Address NextPtr = Builder.CreateConstInBoundsByteGEP(Addr, Stride, "ap.next"); - Builder.CreateStore(NextPtr.getPointer(), VAListAddr); + Builder.CreateStore(NextPtr.emitRawPointer(CGF), VAListAddr); return ArgAddr.withElementType(ArgTy); } diff --git a/clang/lib/CodeGen/Targets/SystemZ.cpp b/clang/lib/CodeGen/Targets/SystemZ.cpp index 6eb0c6ef2f7d..deaafc85a315 100644 --- a/clang/lib/CodeGen/Targets/SystemZ.cpp +++ b/clang/lib/CodeGen/Targets/SystemZ.cpp @@ -306,7 +306,7 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, // Update overflow_arg_area_ptr pointer llvm::Value *NewOverflowArgArea = CGF.Builder.CreateGEP( - OverflowArgArea.getElementType(), OverflowArgArea.getPointer(), + OverflowArgArea.getElementType(), OverflowArgArea.emitRawPointer(CGF), PaddedSizeV, "overflow_arg_area"); CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr); @@ -382,10 +382,9 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, Address MemAddr = RawMemAddr.withElementType(DirectTy); // Update overflow_arg_area_ptr pointer - llvm::Value *NewOverflowArgArea = - CGF.Builder.CreateGEP(OverflowArgArea.getElementType(), - OverflowArgArea.getPointer(), PaddedSizeV, - "overflow_arg_area"); + llvm::Value *NewOverflowArgArea = CGF.Builder.CreateGEP( + OverflowArgArea.getElementType(), OverflowArgArea.emitRawPointer(CGF), + PaddedSizeV, "overflow_arg_area"); CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr); CGF.EmitBranch(ContBlock); diff --git a/clang/lib/CodeGen/Targets/XCore.cpp b/clang/lib/CodeGen/Targets/XCore.cpp index aeb48f851e16..88edb781a947 100644 --- a/clang/lib/CodeGen/Targets/XCore.cpp +++ b/clang/lib/CodeGen/Targets/XCore.cpp @@ -180,7 +180,7 @@ Address XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, // Increment the VAList. if (!ArgSize.isZero()) { Address APN = Builder.CreateConstInBoundsByteGEP(AP, ArgSize); - Builder.CreateStore(APN.getPointer(), VAListAddr); + Builder.CreateStore(APN.emitRawPointer(CGF), VAListAddr); } return Val; diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp index 710bf8d8a8ec..d06c42d5f4c5 100644 --- a/clang/lib/Format/WhitespaceManager.cpp +++ b/clang/lib/Format/WhitespaceManager.cpp @@ -464,10 +464,11 @@ AlignTokenSequence(const FormatStyle &Style, unsigned Start, unsigned End, if (i + 1 != Changes.size()) Changes[i + 1].PreviousEndOfTokenColumn += Shift; - // If PointerAlignment is PAS_Right, keep *s or &s next to the token + // If PointerAlignment is PAS_Right, keep *s or &s next to the token, + // except if the token is equal, then a space is needed. if ((Style.PointerAlignment == FormatStyle::PAS_Right || Style.ReferenceAlignment == FormatStyle::RAS_Right) && - CurrentChange.Spaces != 0) { + CurrentChange.Spaces != 0 && CurrentChange.Tok->isNot(tok::equal)) { const bool ReferenceNotRightAligned = Style.ReferenceAlignment != FormatStyle::RAS_Right && Style.ReferenceAlignment != FormatStyle::RAS_Pointer; diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h index 18472f728eef..d47eab453f87 100644 --- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h +++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h @@ -1149,19 +1149,6 @@ float4 pow(float4, float4); #ifdef __HLSL_ENABLE_16_BIT _HLSL_AVAILABILITY(shadermodel, 6.2) _HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) -int16_t reversebits(int16_t); -_HLSL_AVAILABILITY(shadermodel, 6.2) -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) -int16_t2 reversebits(int16_t2); -_HLSL_AVAILABILITY(shadermodel, 6.2) -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) -int16_t3 reversebits(int16_t3); -_HLSL_AVAILABILITY(shadermodel, 6.2) -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) -int16_t4 reversebits(int16_t4); - -_HLSL_AVAILABILITY(shadermodel, 6.2) -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) uint16_t reversebits(uint16_t); _HLSL_AVAILABILITY(shadermodel, 6.2) _HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) @@ -1175,15 +1162,6 @@ uint16_t4 reversebits(uint16_t4); #endif _HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) -int reversebits(int); -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) -int2 reversebits(int2); -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) -int3 reversebits(int3); -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) -int4 reversebits(int4); - -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) uint reversebits(uint); _HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) uint2 reversebits(uint2); @@ -1193,15 +1171,6 @@ _HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) uint4 reversebits(uint4); _HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) -int64_t reversebits(int64_t); -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) -int64_t2 reversebits(int64_t2); -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) -int64_t3 reversebits(int64_t3); -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) -int64_t4 reversebits(int64_t4); - -_HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) uint64_t reversebits(uint64_t); _HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) uint64_t2 reversebits(uint64_t2); diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 447e73686b4f..f5c0c761da75 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -5541,6 +5541,14 @@ bool CheckNoDoubleVectors(Sema *S, CallExpr *TheCall) { checkDoubleVector); } +bool CheckUnsignedIntRepresentation(Sema *S, CallExpr *TheCall) { + auto checkAllUnsignedTypes = [](clang::QualType PassedType) -> bool { + return !PassedType->hasUnsignedIntegerRepresentation(); + }; + return CheckArgsTypesAreCorrect(S, TheCall, S->Context.UnsignedIntTy, + checkAllUnsignedTypes); +} + void SetElementTypeAsReturnType(Sema *S, CallExpr *TheCall, QualType ReturnType) { auto *VecTyA = TheCall->getArg(0)->getType()->getAs<VectorType>(); @@ -5628,6 +5636,11 @@ bool Sema::CheckHLSLBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { } // Note these are llvm builtins that we want to catch invalid intrinsic // generation. Normal handling of these builitns will occur elsewhere. + case Builtin::BI__builtin_elementwise_bitreverse: { + if (CheckUnsignedIntRepresentation(this, TheCall)) + return true; + break; + } case Builtin::BI__builtin_elementwise_cos: case Builtin::BI__builtin_elementwise_sin: case Builtin::BI__builtin_elementwise_log: diff --git a/clang/test/C/C99/digraphs.c b/clang/test/C/C99/digraphs.c new file mode 100644 index 000000000000..870a44111816 --- /dev/null +++ b/clang/test/C/C99/digraphs.c @@ -0,0 +1,90 @@ +// RUN: %clang_cc1 -verify -ffreestanding %s + +/* WG14 ???: yes + * restricted character set support via digraphs and <iso646.h> + * + * NB: I cannot find a definitive document number associated with the feature, + * which was pulled from the editor's report in the C99 front matter. However, + * based on discussion in the C99 rationale document, I believe this is + * referring to features added by AMD1 to support ISO 646 and digraphs. + */ + +// Validate that we provide iso646.h in freestanding mode. +#include <iso646.h> + +// Validate that we define all the expected macros and their expected +// expansions (when suitable for a constant expression) as well. +#ifndef and +#error "missing and" +#else +_Static_assert((1 and 1) == (1 && 1), ""); +#endif + +#ifndef and_eq +#error "missing and_eq" +#endif + +#ifndef bitand +#error "missing bitand" +#else +_Static_assert((1 bitand 3) == (1 & 3), ""); +#endif + +#ifndef bitor +#error "missing bitor" +#else +_Static_assert((1 bitor 2) == (1 | 2), ""); +#endif + +#ifndef compl +#error "missing compl" +#else +_Static_assert((compl 0) == (~0), ""); +#endif + +#ifndef not +#error "missing not" +#else +_Static_assert((not 12) == (!12), ""); +#endif + +#ifndef not_eq +#error "missing not_eq" +#else +_Static_assert((0 not_eq 12) == (0 != 12), ""); +#endif + +#ifndef or +#error "missing or" +#else +// This intentionally diagnoses use of '||' only, because the user likely did +// not confuse the operator when using 'or' instead. +_Static_assert((0 or 12) == (0 || 12), ""); // expected-warning {{use of logical '||' with constant operand}} \ + expected-note {{use '|' for a bitwise operation}} +#endif + +#ifndef or_eq +#error "missing or_eq" +#endif + +#ifndef xor +#error "missing xor" +#else +_Static_assert((1 xor 3) == (1 ^ 3), ""); +#endif + +#ifndef xor_eq +#error "missing xor_eq" +#endif + +// Validate that digraphs behave the same as their expected counterparts. The +// definition should match the declaration in every way except spelling. +#define DI_NAME(f, b) f %:%: b +#define STD_NAME(f, b) f ## b +void DI_NAME(foo, bar)(int (*array)<: 0 :>); +void STD_NAME(foo, bar)(int (*array)[0]) {} + +#define DI_STR(f) %:f +#define STD_STR(f) #f +_Static_assert(__builtin_strcmp(DI_STR(testing), STD_STR(testing)) == 0, ""); + diff --git a/clang/test/CodeGenHLSL/builtins/bitreverse.hlsl b/clang/test/CodeGenHLSL/builtins/bitreverse.hlsl deleted file mode 100644 index e7609a2b61e2..000000000000 --- a/clang/test/CodeGenHLSL/builtins/bitreverse.hlsl +++ /dev/null @@ -1,155 +0,0 @@ -// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -x hlsl -triple \
-// RUN: dxil-pc-shadermodel6.3-library %s -D__HLSL_ENABLE_16_BIT \
-// RUN: -emit-llvm -disable-llvm-passes -O3 -o - | FileCheck %s
-
-#ifdef __HLSL_ENABLE_16_BIT
-// CHECK: define noundef i16 @
-// CHECK: call i16 @llvm.bitreverse.i16(
-int16_t test_bitreverse_short(int16_t p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <2 x i16> @
-// CHECK: call <2 x i16> @llvm.bitreverse.v2i16(
-int16_t2 test_bitreverse_short2(int16_t2 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <3 x i16> @
-// CHECK: call <3 x i16> @llvm.bitreverse.v3i16
-int16_t3 test_bitreverse_short3(int16_t3 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <4 x i16> @
-// CHECK: call <4 x i16> @llvm.bitreverse.v4i16
-int16_t4 test_bitreverse_short4(int16_t4 p0)
-{
- return reversebits(p0);
-}
-
-// CHECK: define noundef i16 @
-// CHECK: call i16 @llvm.bitreverse.i16(
-uint16_t test_bitreverse_ushort(uint16_t p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <2 x i16> @
-// CHECK: call <2 x i16> @llvm.bitreverse.v2i16
-uint16_t2 test_bitreverse_ushort2(uint16_t2 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <3 x i16> @
-// CHECK: call <3 x i16> @llvm.bitreverse.v3i16
-uint16_t3 test_bitreverse_ushort3(uint16_t3 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <4 x i16> @
-// CHECK: call <4 x i16> @llvm.bitreverse.v4i16
-uint16_t4 test_bitreverse_ushort4(uint16_t4 p0)
-{
- return reversebits(p0);
-}
-#endif
-
-// CHECK: define noundef i32 @
-// CHECK: call i32 @llvm.bitreverse.i32(
-int test_bitreverse_int(int p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <2 x i32> @
-// CHECK: call <2 x i32> @llvm.bitreverse.v2i32
-int2 test_bitreverse_int2(int2 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <3 x i32> @
-// CHECK: call <3 x i32> @llvm.bitreverse.v3i32
-int3 test_bitreverse_int3(int3 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <4 x i32> @
-// CHECK: call <4 x i32> @llvm.bitreverse.v4i32
-int4 test_bitreverse_int4(int4 p0)
-{
- return reversebits(p0);
-}
-
-// CHECK: define noundef i32 @
-// CHECK: call i32 @llvm.bitreverse.i32(
-int test_bitreverse_uint(uint p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <2 x i32> @
-// CHECK: call <2 x i32> @llvm.bitreverse.v2i32
-uint2 test_bitreverse_uint2(uint2 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <3 x i32> @
-// CHECK: call <3 x i32> @llvm.bitreverse.v3i32
-uint3 test_bitreverse_uint3(uint3 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <4 x i32> @
-// CHECK: call <4 x i32> @llvm.bitreverse.v4i32
-uint4 test_bitreverse_uint4(uint4 p0)
-{
- return reversebits(p0);
-}
-
-// CHECK: define noundef i64 @
-// CHECK: call i64 @llvm.bitreverse.i64(
-int64_t test_bitreverse_long(int64_t p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <2 x i64> @
-// CHECK: call <2 x i64> @llvm.bitreverse.v2i64
-int64_t2 test_bitreverse_long2(int64_t2 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <3 x i64> @
-// CHECK: call <3 x i64> @llvm.bitreverse.v3i64
-int64_t3 test_bitreverse_long3(int64_t3 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <4 x i64> @
-// CHECK: call <4 x i64> @llvm.bitreverse.v4i64
-int64_t4 test_bitreverse_long4(int64_t4 p0)
-{
- return reversebits(p0);
-}
-
-// CHECK: define noundef i64 @
-// CHECK: call i64 @llvm.bitreverse.i64(
-uint64_t test_bitreverse_long(uint64_t p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <2 x i64> @
-// CHECK: call <2 x i64> @llvm.bitreverse.v2i64
-uint64_t2 test_bitreverse_long2(uint64_t2 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <3 x i64> @
-// CHECK: call <3 x i64> @llvm.bitreverse.v3i64
-uint64_t3 test_bitreverse_long3(uint64_t3 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <4 x i64> @
-// CHECK: call <4 x i64> @llvm.bitreverse.v4i64
-uint64_t4 test_bitreverse_long4(uint64_t4 p0)
-{
- return reversebits(p0);
-}
diff --git a/clang/test/CodeGenHLSL/builtins/reversebits.hlsl b/clang/test/CodeGenHLSL/builtins/reversebits.hlsl index 6da7d289f82e..a319417e97a4 100644 --- a/clang/test/CodeGenHLSL/builtins/reversebits.hlsl +++ b/clang/test/CodeGenHLSL/builtins/reversebits.hlsl @@ -5,31 +5,6 @@ #ifdef __HLSL_ENABLE_16_BIT
// CHECK: define noundef i16 @
// CHECK: call i16 @llvm.bitreverse.i16(
-int16_t test_bitreverse_short(int16_t p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <2 x i16> @
-// CHECK: call <2 x i16> @llvm.bitreverse.v2i16(
-int16_t2 test_bitreverse_short2(int16_t2 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <3 x i16> @
-// CHECK: call <3 x i16> @llvm.bitreverse.v3i16
-int16_t3 test_bitreverse_short3(int16_t3 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <4 x i16> @
-// CHECK: call <4 x i16> @llvm.bitreverse.v4i16
-int16_t4 test_bitreverse_short4(int16_t4 p0)
-{
- return reversebits(p0);
-}
-
-// CHECK: define noundef i16 @
-// CHECK: call i16 @llvm.bitreverse.i16(
uint16_t test_bitreverse_ushort(uint16_t p0)
{
return reversebits(p0);
@@ -56,31 +31,6 @@ uint16_t4 test_bitreverse_ushort4(uint16_t4 p0) // CHECK: define noundef i32 @
// CHECK: call i32 @llvm.bitreverse.i32(
-int test_bitreverse_int(int p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <2 x i32> @
-// CHECK: call <2 x i32> @llvm.bitreverse.v2i32
-int2 test_bitreverse_int2(int2 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <3 x i32> @
-// CHECK: call <3 x i32> @llvm.bitreverse.v3i32
-int3 test_bitreverse_int3(int3 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <4 x i32> @
-// CHECK: call <4 x i32> @llvm.bitreverse.v4i32
-int4 test_bitreverse_int4(int4 p0)
-{
- return reversebits(p0);
-}
-
-// CHECK: define noundef i32 @
-// CHECK: call i32 @llvm.bitreverse.i32(
int test_bitreverse_uint(uint p0)
{
return reversebits(p0);
@@ -106,31 +56,6 @@ uint4 test_bitreverse_uint4(uint4 p0) // CHECK: define noundef i64 @
// CHECK: call i64 @llvm.bitreverse.i64(
-int64_t test_bitreverse_long(int64_t p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <2 x i64> @
-// CHECK: call <2 x i64> @llvm.bitreverse.v2i64
-int64_t2 test_bitreverse_long2(int64_t2 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <3 x i64> @
-// CHECK: call <3 x i64> @llvm.bitreverse.v3i64
-int64_t3 test_bitreverse_long3(int64_t3 p0)
-{
- return reversebits(p0);
-}
-// CHECK: define noundef <4 x i64> @
-// CHECK: call <4 x i64> @llvm.bitreverse.v4i64
-int64_t4 test_bitreverse_long4(int64_t4 p0)
-{
- return reversebits(p0);
-}
-
-// CHECK: define noundef i64 @
-// CHECK: call i64 @llvm.bitreverse.i64(
uint64_t test_bitreverse_long(uint64_t p0)
{
return reversebits(p0);
diff --git a/clang/test/Driver/linker-wrapper-image.c b/clang/test/Driver/linker-wrapper-image.c index 754752641352..d01445e3aed0 100644 --- a/clang/test/Driver/linker-wrapper-image.c +++ b/clang/test/Driver/linker-wrapper-image.c @@ -26,11 +26,11 @@ // OPENMP: @.omp_offloading.device_image = internal unnamed_addr constant [[[SIZE:[0-9]+]] x i8] c"\10\FF\10\AD{{.*}}", section ".llvm.offloading", align 8 // OPENMP-NEXT: @.omp_offloading.device_images = internal unnamed_addr constant [1 x %__tgt_device_image] [%__tgt_device_image { ptr getelementptr inbounds ([[[BEGIN:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 1, i64 0), ptr getelementptr inbounds ([[[END:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 1, i64 0), ptr @__start_omp_offloading_entries, ptr @__stop_omp_offloading_entries }] // OPENMP-NEXT: @.omp_offloading.descriptor = internal constant %__tgt_bin_desc { i32 1, ptr @.omp_offloading.device_images, ptr @__start_omp_offloading_entries, ptr @__stop_omp_offloading_entries } -// OPENMP-NEXT: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @.omp_offloading.descriptor_reg, ptr null }] -// OPENMP-NEXT: @llvm.global_dtors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @.omp_offloading.descriptor_unreg, ptr null }] +// OPENMP-NEXT: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 101, ptr @.omp_offloading.descriptor_reg, ptr null }] // OPENMP: define internal void @.omp_offloading.descriptor_reg() section ".text.startup" { // OPENMP-NEXT: entry: +// OPENMP-NEXT: %0 = call i32 @atexit(ptr @.omp_offloading.descriptor_unreg) // OPENMP-NEXT: call void @__tgt_register_lib(ptr @.omp_offloading.descriptor) // OPENMP-NEXT: ret void // OPENMP-NEXT: } @@ -62,7 +62,7 @@ // CUDA-NEXT: @.fatbin_wrapper = internal constant %fatbin_wrapper { i32 1180844977, i32 1, ptr @.fatbin_image, ptr null }, section ".nvFatBinSegment", align 8 // CUDA-NEXT: @.cuda.binary_handle = internal global ptr null -// CUDA: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @.cuda.fatbin_reg, ptr null }] +// CUDA: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 101, ptr @.cuda.fatbin_reg, ptr null }] // CUDA: define internal void @.cuda.fatbin_reg() section ".text.startup" { // CUDA-NEXT: entry: @@ -162,7 +162,7 @@ // HIP-NEXT: @.fatbin_wrapper = internal constant %fatbin_wrapper { i32 1212764230, i32 1, ptr @.fatbin_image, ptr null }, section ".hipFatBinSegment", align 8 // HIP-NEXT: @.hip.binary_handle = internal global ptr null -// HIP: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @.hip.fatbin_reg, ptr null }] +// HIP: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 101, ptr @.hip.fatbin_reg, ptr null }] // HIP: define internal void @.hip.fatbin_reg() section ".text.startup" { // HIP-NEXT: entry: diff --git a/clang/test/SemaHLSL/BuiltIns/reversebits-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/reversebits-errors.hlsl new file mode 100644 index 000000000000..6e66db6d1cca --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/reversebits-errors.hlsl @@ -0,0 +1,12 @@ +// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify + + +double2 test_int_builtin(double2 p0) { + return __builtin_elementwise_bitreverse(p0); + // expected-error@-1 {{1st argument must be a vector of integers (was 'double2' (aka 'vector<double, 2>'))}} +} + +int2 test_int_builtin(int2 p0) { + return __builtin_elementwise_bitreverse(p0); + // expected-error@-1 {{passing 'int2' (aka 'vector<int, 2>') to parameter of incompatible type '__attribute__((__vector_size__(2 * sizeof(unsigned int)))) unsigned int' (vector of 2 'unsigned int' values)}} +} diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index 03005384a6f6..d1e977dfa66a 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -19066,6 +19066,11 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) { verifyFormat("int a(int x);\n" "double b();", Alignment); + verifyFormat("int a(const Test & = Test());\n" + "int a1(int &foo, const Test & = Test());\n" + "int a2(int &foo, const Test &name = Test());\n" + "double b();", + Alignment); verifyFormat("struct Test {\n" " Test(const Test &) = default;\n" " ~Test() = default;\n" @@ -19102,6 +19107,13 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) { " int x,\n" " bool y);", Alignment); + // Set ColumnLimit low so that we break the argument list in multiple lines. + Alignment.ColumnLimit = 35; + verifyFormat("int a3(SomeTypeName1 &x,\n" + " SomeTypeName2 &y,\n" + " const Test & = Test());\n" + "double b();", + Alignment); Alignment.ColumnLimit = OldColumnLimit; // Ensure function pointers don't screw up recursive alignment verifyFormat("int a(int x, void (*fp)(int y));\n" @@ -19287,6 +19299,10 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) { "int foobar;", AlignmentLeft); + verifyFormat("int a(SomeType& foo, const Test& = Test());\n" + "double b();", + AlignmentLeft); + // PAS_Middle FormatStyle AlignmentMiddle = Alignment; AlignmentMiddle.PointerAlignment = FormatStyle::PAS_Middle; @@ -19347,6 +19363,10 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) { "int foobar;", AlignmentMiddle); + verifyFormat("int a(SomeType & foo, const Test & = Test());\n" + "double b();", + AlignmentMiddle); + Alignment.AlignConsecutiveAssignments.Enabled = false; Alignment.AlignEscapedNewlines = FormatStyle::ENAS_DontAlign; verifyFormat("#define A \\\n" diff --git a/clang/utils/TableGen/MveEmitter.cpp b/clang/utils/TableGen/MveEmitter.cpp index 3a90eee5f1c9..aa20c758d84a 100644 --- a/clang/utils/TableGen/MveEmitter.cpp +++ b/clang/utils/TableGen/MveEmitter.cpp @@ -575,7 +575,7 @@ public: // Emit code to generate this result as a Value *. std::string asValue() override { if (AddressType) - return "(" + varname() + ".getPointer())"; + return "(" + varname() + ".emitRawPointer(*this))"; return Result::asValue(); } bool hasIntegerValue() const override { return Immediate; } diff --git a/clang/www/c_status.html b/clang/www/c_status.html index 60f48aba2788..55aca94cd2ec 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -105,7 +105,7 @@ conformance.</p> <tr> <td>restricted character set support via digraphs and <iso646.h></td> <td>Unknown</td> - <td class="unknown" align="center">Unknown</td> + <td class="full" align="center">Yes</td> </tr> <tr> <td>more precise aliasing rules via effective type</td> diff --git a/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp b/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp index 3e41f67ba922..17a596d712d0 100644 --- a/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp @@ -136,7 +136,7 @@ TEST(ScudoStringsTest, CapacityIncreaseFails) { rlimit Limit = {}; EXPECT_EQ(0, getrlimit(RLIMIT_AS, &Limit)); - rlimit EmptyLimit = {.rlim_max = Limit.rlim_max}; + rlimit EmptyLimit = {.rlim_cur = 0, .rlim_max = Limit.rlim_max}; EXPECT_EQ(0, setrlimit(RLIMIT_AS, &EmptyLimit)); // Test requires that the default length is at least 6 characters. diff --git a/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp b/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp index b7678678d8a2..add62c5a42a3 100644 --- a/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp @@ -55,7 +55,7 @@ TEST(ScudoVectorTest, ReallocateFails) { rlimit Limit = {}; EXPECT_EQ(0, getrlimit(RLIMIT_AS, &Limit)); - rlimit EmptyLimit = {.rlim_max = Limit.rlim_max}; + rlimit EmptyLimit = {.rlim_cur = 0, .rlim_max = Limit.rlim_max}; EXPECT_EQ(0, setrlimit(RLIMIT_AS, &EmptyLimit)); V.resize(capacity); diff --git a/compiler-rt/lib/scudo/standalone/tsd.h b/compiler-rt/lib/scudo/standalone/tsd.h index b2108a01900b..72773f2f72b1 100644 --- a/compiler-rt/lib/scudo/standalone/tsd.h +++ b/compiler-rt/lib/scudo/standalone/tsd.h @@ -41,9 +41,9 @@ template <class Allocator> struct alignas(SCUDO_CACHE_LINE_SIZE) TSD { return true; } if (atomic_load_relaxed(&Precedence) == 0) - atomic_store_relaxed(&Precedence, - static_cast<uptr>(getMonotonicTimeFast() >> - FIRST_32_SECOND_64(16, 0))); + atomic_store_relaxed( + &Precedence, + static_cast<uptr>(getMonotonicTime() >> FIRST_32_SECOND_64(16, 0))); return false; } inline void lock() NO_THREAD_SAFETY_ANALYSIS { diff --git a/libcxx/include/__chrono/tzdb_list.h b/libcxx/include/__chrono/tzdb_list.h index 112e04ff2ee6..e8aaf31e3631 100644 --- a/libcxx/include/__chrono/tzdb_list.h +++ b/libcxx/include/__chrono/tzdb_list.h @@ -52,19 +52,29 @@ public: using const_iterator = forward_list<tzdb>::const_iterator; - _LIBCPP_NODISCARD_EXT _LIBCPP_EXPORTED_FROM_ABI const tzdb& front() const noexcept; + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI const tzdb& front() const noexcept { return __front(); } - _LIBCPP_EXPORTED_FROM_ABI const_iterator erase_after(const_iterator __p); + _LIBCPP_HIDE_FROM_ABI const_iterator erase_after(const_iterator __p) { return __erase_after(__p); } - _LIBCPP_NODISCARD_EXT _LIBCPP_EXPORTED_FROM_ABI const_iterator begin() const noexcept; - _LIBCPP_NODISCARD_EXT _LIBCPP_EXPORTED_FROM_ABI const_iterator end() const noexcept; + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI const_iterator begin() const noexcept { return __begin(); } + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI const_iterator end() const noexcept { return __end(); } - _LIBCPP_NODISCARD_EXT _LIBCPP_EXPORTED_FROM_ABI const_iterator cbegin() const noexcept; - _LIBCPP_NODISCARD_EXT _LIBCPP_EXPORTED_FROM_ABI const_iterator cend() const noexcept; + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const noexcept { return __cbegin(); } + _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI const_iterator cend() const noexcept { return __cend(); } [[nodiscard]] _LIBCPP_HIDE_FROM_ABI __impl& __implementation() { return *__impl_; } private: + [[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI const tzdb& __front() const noexcept; + + _LIBCPP_EXPORTED_FROM_ABI const_iterator __erase_after(const_iterator __p); + + [[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI const_iterator __begin() const noexcept; + [[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI const_iterator __end() const noexcept; + + [[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI const_iterator __cbegin() const noexcept; + [[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI const_iterator __cend() const noexcept; + __impl* __impl_; }; diff --git a/libcxx/src/include/tzdb/tzdb_list_private.h b/libcxx/src/include/tzdb/tzdb_list_private.h index f43d7d8ea772..969b2b9f8a9f 100644 --- a/libcxx/src/include/tzdb/tzdb_list_private.h +++ b/libcxx/src/include/tzdb/tzdb_list_private.h @@ -54,14 +54,14 @@ public: using const_iterator = tzdb_list::const_iterator; - const tzdb& front() const noexcept { + const tzdb& __front() const noexcept { #ifndef _LIBCPP_HAS_NO_THREADS shared_lock __lock{__mutex_}; #endif return __tzdb_.front(); } - const_iterator erase_after(const_iterator __p) { + const_iterator __erase_after(const_iterator __p) { #ifndef _LIBCPP_HAS_NO_THREADS unique_lock __lock{__mutex_}; #endif @@ -70,20 +70,17 @@ public: return __tzdb_.erase_after(__p); } - const_iterator begin() const noexcept { + const_iterator __begin() const noexcept { #ifndef _LIBCPP_HAS_NO_THREADS shared_lock __lock{__mutex_}; #endif return __tzdb_.begin(); } - const_iterator end() const noexcept { + const_iterator __end() const noexcept { // forward_list<T>::end does not access the list, so no need to take a lock. return __tzdb_.end(); } - const_iterator cbegin() const noexcept { return begin(); } - const_iterator cend() const noexcept { return end(); } - private: // Loads the tzdbs // pre: The caller ensures the locking, if needed, is done. diff --git a/libcxx/src/tzdb_list.cpp b/libcxx/src/tzdb_list.cpp index d3ee8b58f98b..b99c30a9b9e6 100644 --- a/libcxx/src/tzdb_list.cpp +++ b/libcxx/src/tzdb_list.cpp @@ -18,26 +18,24 @@ namespace chrono { _LIBCPP_EXPORTED_FROM_ABI tzdb_list::~tzdb_list() { delete __impl_; } -_LIBCPP_NODISCARD_EXT _LIBCPP_EXPORTED_FROM_ABI const tzdb& tzdb_list::front() const noexcept { - return __impl_->front(); -} +[[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI const tzdb& tzdb_list::__front() const noexcept { return __impl_->__front(); } -_LIBCPP_EXPORTED_FROM_ABI tzdb_list::const_iterator tzdb_list::erase_after(const_iterator __p) { - return __impl_->erase_after(__p); +_LIBCPP_EXPORTED_FROM_ABI tzdb_list::const_iterator tzdb_list::__erase_after(const_iterator __p) { + return __impl_->__erase_after(__p); } -_LIBCPP_NODISCARD_EXT _LIBCPP_EXPORTED_FROM_ABI tzdb_list::const_iterator tzdb_list::begin() const noexcept { - return __impl_->begin(); +[[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI tzdb_list::const_iterator tzdb_list::__begin() const noexcept { + return __impl_->__begin(); } -_LIBCPP_NODISCARD_EXT _LIBCPP_EXPORTED_FROM_ABI tzdb_list::const_iterator tzdb_list::end() const noexcept { - return __impl_->end(); +[[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI tzdb_list::const_iterator tzdb_list::__end() const noexcept { + return __impl_->__end(); } -_LIBCPP_NODISCARD_EXT _LIBCPP_EXPORTED_FROM_ABI tzdb_list::const_iterator tzdb_list::cbegin() const noexcept { - return __impl_->cbegin(); +[[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI tzdb_list::const_iterator tzdb_list::__cbegin() const noexcept { + return __impl_->__begin(); } -_LIBCPP_NODISCARD_EXT _LIBCPP_EXPORTED_FROM_ABI tzdb_list::const_iterator tzdb_list::cend() const noexcept { - return __impl_->cend(); +[[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI tzdb_list::const_iterator tzdb_list::__cend() const noexcept { + return __impl_->__end(); } } // namespace chrono diff --git a/libcxx/utils/libcxx/test/params.py b/libcxx/utils/libcxx/test/params.py index 695e01115aa4..5e42562ed5db 100644 --- a/libcxx/utils/libcxx/test/params.py +++ b/libcxx/utils/libcxx/test/params.py @@ -407,6 +407,6 @@ DEFAULT_PARAMETERS = [ AddFeature('has-clang-tidy'), AddSubstitution('%{clang-tidy}', exe), ] - ), + ), ] # fmt: on diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h index 2a0c1e9e8c44..2e2ec9a1c830 100644 --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -2708,6 +2708,7 @@ public: IRBuilder(const IRBuilder &) = delete; InserterTy &getInserter() { return Inserter; } + const InserterTy &getInserter() const { return Inserter; } }; template <typename FolderTy, typename InserterTy> diff --git a/llvm/include/llvm/IR/ProfDataUtils.h b/llvm/include/llvm/IR/ProfDataUtils.h index 255fa2ff1c79..c0897408986f 100644 --- a/llvm/include/llvm/IR/ProfDataUtils.h +++ b/llvm/include/llvm/IR/ProfDataUtils.h @@ -108,5 +108,8 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalWeights); /// a `prof` metadata reference to instruction `I`. void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights); +/// Scaling the profile data attached to 'I' using the ratio of S/T. +void scaleProfData(Instruction &I, uint64_t S, uint64_t T); + } // namespace llvm #endif diff --git a/llvm/include/llvm/Object/GOFF.h b/llvm/include/llvm/Object/GOFF.h index 9fb8876e893d..91762457ae05 100644 --- a/llvm/include/llvm/Object/GOFF.h +++ b/llvm/include/llvm/Object/GOFF.h @@ -73,26 +73,6 @@ protected: } }; -class TXTRecord : public Record { -public: - /// \brief Maximum length of data; any more must go in continuation. - static const uint8_t TXTMaxDataLength = 56; - - static Error getData(const uint8_t *Record, SmallString<256> &CompleteData); - - static void getElementEsdId(const uint8_t *Record, uint32_t &EsdId) { - get<uint32_t>(Record, 4, EsdId); - } - - static void getOffset(const uint8_t *Record, uint32_t &Offset) { - get<uint32_t>(Record, 12, Offset); - } - - static void getDataLength(const uint8_t *Record, uint16_t &Length) { - get<uint16_t>(Record, 22, Length); - } -}; - class HDRRecord : public Record { public: static Error getData(const uint8_t *Record, SmallString<256> &CompleteData); diff --git a/llvm/include/llvm/Object/GOFFObjectFile.h b/llvm/include/llvm/Object/GOFFObjectFile.h index 6871641e97ec..7e1ceb95f667 100644 --- a/llvm/include/llvm/Object/GOFFObjectFile.h +++ b/llvm/include/llvm/Object/GOFFObjectFile.h @@ -29,10 +29,7 @@ namespace llvm { namespace object { class GOFFObjectFile : public ObjectFile { - friend class GOFFSymbolRef; - IndexedMap<const uint8_t *> EsdPtrs; // Indexed by EsdId. - SmallVector<const uint8_t *, 256> TextPtrs; mutable DenseMap<uint32_t, std::pair<size_t, std::unique_ptr<char[]>>> EsdNamesCache; @@ -41,7 +38,7 @@ class GOFFObjectFile : public ObjectFile { // (EDID, 0) code, r/o data section // (EDID,PRID) r/w data section SmallVector<SectionEntryImpl, 256> SectionList; - mutable DenseMap<uint32_t, SmallVector<uint8_t>> SectionDataCache; + mutable DenseMap<uint32_t, std::string> SectionDataCache; public: Expected<StringRef> getSymbolName(SymbolRef Symbol) const; @@ -69,10 +66,6 @@ public: return true; } - bool isSectionNoLoad(DataRefImpl Sec) const; - bool isSectionReadOnlyData(DataRefImpl Sec) const; - bool isSectionZeroInit(DataRefImpl Sec) const; - private: // SymbolRef. Expected<StringRef> getSymbolName(DataRefImpl Symb) const override; @@ -82,24 +75,27 @@ private: Expected<uint32_t> getSymbolFlags(DataRefImpl Symb) const override; Expected<SymbolRef::Type> getSymbolType(DataRefImpl Symb) const override; Expected<section_iterator> getSymbolSection(DataRefImpl Symb) const override; - uint64_t getSymbolSize(DataRefImpl Symb) const; const uint8_t *getSymbolEsdRecord(DataRefImpl Symb) const; bool isSymbolUnresolved(DataRefImpl Symb) const; bool isSymbolIndirect(DataRefImpl Symb) const; // SectionRef. - void moveSectionNext(DataRefImpl &Sec) const override; - virtual Expected<StringRef> getSectionName(DataRefImpl Sec) const override; - uint64_t getSectionAddress(DataRefImpl Sec) const override; - uint64_t getSectionSize(DataRefImpl Sec) const override; + void moveSectionNext(DataRefImpl &Sec) const override {} + virtual Expected<StringRef> getSectionName(DataRefImpl Sec) const override { + return StringRef(); + } + uint64_t getSectionAddress(DataRefImpl Sec) const override { return 0; } + uint64_t getSectionSize(DataRefImpl Sec) const override { return 0; } virtual Expected<ArrayRef<uint8_t>> - getSectionContents(DataRefImpl Sec) const override; - uint64_t getSectionIndex(DataRefImpl Sec) const override { return Sec.d.a; } - uint64_t getSectionAlignment(DataRefImpl Sec) const override; + getSectionContents(DataRefImpl Sec) const override { + return ArrayRef<uint8_t>(); + } + uint64_t getSectionIndex(DataRefImpl Sec) const override { return 0; } + uint64_t getSectionAlignment(DataRefImpl Sec) const override { return 0; } bool isSectionCompressed(DataRefImpl Sec) const override { return false; } - bool isSectionText(DataRefImpl Sec) const override; - bool isSectionData(DataRefImpl Sec) const override; + bool isSectionText(DataRefImpl Sec) const override { return false; } + bool isSectionData(DataRefImpl Sec) const override { return false; } bool isSectionBSS(DataRefImpl Sec) const override { return false; } bool isSectionVirtual(DataRefImpl Sec) const override { return false; } relocation_iterator section_rel_begin(DataRefImpl Sec) const override { @@ -113,7 +109,6 @@ private: const uint8_t *getSectionPrEsdRecord(DataRefImpl &Sec) const; const uint8_t *getSectionEdEsdRecord(uint32_t SectionIndex) const; const uint8_t *getSectionPrEsdRecord(uint32_t SectionIndex) const; - uint32_t getSectionDefEsdId(DataRefImpl &Sec) const; // RelocationRef. void moveRelocationNext(DataRefImpl &Rel) const override {} @@ -127,29 +122,6 @@ private: SmallVectorImpl<char> &Result) const override {} }; -class GOFFSymbolRef : public SymbolRef { -public: - GOFFSymbolRef(const SymbolRef &B) : SymbolRef(B) { - assert(isa<GOFFObjectFile>(SymbolRef::getObject())); - } - - const GOFFObjectFile *getObject() const { - return cast<GOFFObjectFile>(BasicSymbolRef::getObject()); - } - - Expected<uint32_t> getSymbolGOFFFlags() const { - return getObject()->getSymbolFlags(getRawDataRefImpl()); - } - - Expected<SymbolRef::Type> getSymbolGOFFType() const { - return getObject()->getSymbolType(getRawDataRefImpl()); - } - - uint64_t getSize() const { - return getObject()->getSymbolSize(getRawDataRefImpl()); - } -}; - } // namespace object } // namespace llvm diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td index cb1c0ed2513d..1f7dc6922f13 100644 --- a/llvm/include/llvm/Target/Target.td +++ b/llvm/include/llvm/Target/Target.td @@ -83,10 +83,27 @@ class RegInfoByHwMode<list<HwMode> Ms = [], list<RegInfo> Ts = []> list<RegInfo> Objects = Ts; } +class SubRegRange<int size, int offset = 0> { + int Size = size; // Sub register size in bits. + int Offset = offset; // Offset of the first bit of the sub-reg index. +} + +class SubRegRangeByHwMode<list<HwMode> Ms = [], list<SubRegRange> Ts = []> + : HwModeSelect<Ms> { + // The length of this list must be the same as the length of Ms. + list<SubRegRange> Objects = Ts; +} + // SubRegIndex - Use instances of SubRegIndex to identify subregisters. class SubRegIndex<int size, int offset = 0> { string Namespace = ""; + // The size/offset information, parameterized by a HW mode. + // If the HwModes provided for SubRegRanges does not include the DefaultMode, + // the/ Size and Offset fields below will be used for the default. Otherwise, + // the Size and Offset fields are ignored. + SubRegRangeByHwMode SubRegRanges; + // Size - Size (in bits) of the sub-registers represented by this index. int Size = size; diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index e10b8bc8c5e2..24f69ea1b742 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -1455,6 +1455,9 @@ SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { // First store the whole vector. SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo); + // Freeze the index so we don't poison the clamping code we're about to emit. + Idx = DAG.getFreeze(Idx); + // Then store the inserted part. if (PartVT.isVector()) { SDValue SubStackPtr = diff --git a/llvm/lib/CodeGen/TargetRegisterInfo.cpp b/llvm/lib/CodeGen/TargetRegisterInfo.cpp index 4120c74c23b1..4e06393f4cc1 100644 --- a/llvm/lib/CodeGen/TargetRegisterInfo.cpp +++ b/llvm/lib/CodeGen/TargetRegisterInfo.cpp @@ -595,13 +595,13 @@ bool TargetRegisterInfo::getCoveringSubRegIndexes( unsigned TargetRegisterInfo::getSubRegIdxSize(unsigned Idx) const { assert(Idx && Idx < getNumSubRegIndices() && "This is not a subregister index"); - return SubRegIdxRanges[Idx].Size; + return SubRegIdxRanges[HwMode * getNumSubRegIndices() + Idx].Size; } unsigned TargetRegisterInfo::getSubRegIdxOffset(unsigned Idx) const { assert(Idx && Idx < getNumSubRegIndices() && "This is not a subregister index"); - return SubRegIdxRanges[Idx].Offset; + return SubRegIdxRanges[HwMode * getNumSubRegIndices() + Idx].Offset; } Register diff --git a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp index fec1bdbe9d8c..7241d15ed1c6 100644 --- a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp +++ b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp @@ -186,57 +186,62 @@ GlobalVariable *createBinDesc(Module &M, ArrayRef<ArrayRef<char>> Bufs, ".omp_offloading.descriptor" + Suffix); } -void createRegisterFunction(Module &M, GlobalVariable *BinDesc, - StringRef Suffix) { +Function *createUnregisterFunction(Module &M, GlobalVariable *BinDesc, + StringRef Suffix) { LLVMContext &C = M.getContext(); auto *FuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false); - auto *Func = Function::Create(FuncTy, GlobalValue::InternalLinkage, - ".omp_offloading.descriptor_reg" + Suffix, &M); + auto *Func = + Function::Create(FuncTy, GlobalValue::InternalLinkage, + ".omp_offloading.descriptor_unreg" + Suffix, &M); Func->setSection(".text.startup"); - // Get __tgt_register_lib function declaration. - auto *RegFuncTy = FunctionType::get(Type::getVoidTy(C), getBinDescPtrTy(M), - /*isVarArg*/ false); - FunctionCallee RegFuncC = - M.getOrInsertFunction("__tgt_register_lib", RegFuncTy); + // Get __tgt_unregister_lib function declaration. + auto *UnRegFuncTy = FunctionType::get(Type::getVoidTy(C), getBinDescPtrTy(M), + /*isVarArg*/ false); + FunctionCallee UnRegFuncC = + M.getOrInsertFunction("__tgt_unregister_lib", UnRegFuncTy); // Construct function body IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func)); - Builder.CreateCall(RegFuncC, BinDesc); + Builder.CreateCall(UnRegFuncC, BinDesc); Builder.CreateRetVoid(); - // Add this function to constructors. - // Set priority to 1 so that __tgt_register_lib is executed AFTER - // __tgt_register_requires (we want to know what requirements have been - // asked for before we load a libomptarget plugin so that by the time the - // plugin is loaded it can report how many devices there are which can - // satisfy these requirements). - appendToGlobalCtors(M, Func, /*Priority*/ 1); + return Func; } -void createUnregisterFunction(Module &M, GlobalVariable *BinDesc, - StringRef Suffix) { +void createRegisterFunction(Module &M, GlobalVariable *BinDesc, + StringRef Suffix) { LLVMContext &C = M.getContext(); auto *FuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false); - auto *Func = - Function::Create(FuncTy, GlobalValue::InternalLinkage, - ".omp_offloading.descriptor_unreg" + Suffix, &M); + auto *Func = Function::Create(FuncTy, GlobalValue::InternalLinkage, + ".omp_offloading.descriptor_reg" + Suffix, &M); Func->setSection(".text.startup"); - // Get __tgt_unregister_lib function declaration. - auto *UnRegFuncTy = FunctionType::get(Type::getVoidTy(C), getBinDescPtrTy(M), - /*isVarArg*/ false); - FunctionCallee UnRegFuncC = - M.getOrInsertFunction("__tgt_unregister_lib", UnRegFuncTy); + // Get __tgt_register_lib function declaration. + auto *RegFuncTy = FunctionType::get(Type::getVoidTy(C), getBinDescPtrTy(M), + /*isVarArg*/ false); + FunctionCallee RegFuncC = + M.getOrInsertFunction("__tgt_register_lib", RegFuncTy); + + auto *AtExitTy = FunctionType::get( + Type::getInt32Ty(C), PointerType::getUnqual(C), /*isVarArg=*/false); + FunctionCallee AtExit = M.getOrInsertFunction("atexit", AtExitTy); + + Function *UnregFunc = createUnregisterFunction(M, BinDesc, Suffix); // Construct function body IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func)); - Builder.CreateCall(UnRegFuncC, BinDesc); + + // Register the destructors with 'atexit'. This is expected by the CUDA + // runtime and ensures that we clean up before dynamic objects are destroyed. + // This needs to be done before the runtime is called and registers its own. + Builder.CreateCall(AtExit, UnregFunc); + + Builder.CreateCall(RegFuncC, BinDesc); Builder.CreateRetVoid(); - // Add this function to global destructors. - // Match priority of __tgt_register_lib - appendToGlobalDtors(M, Func, /*Priority*/ 1); + // Add this function to constructors. + appendToGlobalCtors(M, Func, /*Priority=*/101); } // struct fatbin_wrapper { @@ -578,7 +583,7 @@ void createRegisterFatbinFunction(Module &M, GlobalVariable *FatbinDesc, DtorBuilder.CreateRetVoid(); // Add this function to constructors. - appendToGlobalCtors(M, CtorFunc, /*Priority*/ 1); + appendToGlobalCtors(M, CtorFunc, /*Priority=*/101); } } // namespace @@ -591,7 +596,6 @@ Error offloading::wrapOpenMPBinaries(Module &M, ArrayRef<ArrayRef<char>> Images, return createStringError(inconvertibleErrorCode(), "No binary descriptors created."); createRegisterFunction(M, Desc, Suffix); - createUnregisterFunction(M, Desc, Suffix); return Error::success(); } diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index c2abe875e019..cec02e2ce633 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -833,15 +833,6 @@ CallInst *CallInst::Create(CallInst *CI, ArrayRef<OperandBundleDef> OpB, // of S/T. The meaning of "branch_weights" meta data for call instruction is // transfered to represent call count. void CallInst::updateProfWeight(uint64_t S, uint64_t T) { - auto *ProfileData = getMetadata(LLVMContext::MD_prof); - if (ProfileData == nullptr) - return; - - auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0)); - if (!ProfDataName || (!ProfDataName->getString().equals("branch_weights") && - !ProfDataName->getString().equals("VP"))) - return; - if (T == 0) { LLVM_DEBUG(dbgs() << "Attempting to update profile weights will result in " "div by 0. Ignoring. Likely the function " @@ -850,42 +841,7 @@ void CallInst::updateProfWeight(uint64_t S, uint64_t T) { "with non-zero prof info."); return; } - - MDBuilder MDB(getContext()); - SmallVector<Metadata *, 3> Vals; - Vals.push_back(ProfileData->getOperand(0)); - APInt APS(128, S), APT(128, T); - if (ProfDataName->getString().equals("branch_weights") && - ProfileData->getNumOperands() > 0) { - // Using APInt::div may be expensive, but most cases should fit 64 bits. - APInt Val(128, mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(1)) - ->getValue() - .getZExtValue()); - Val *= APS; - Vals.push_back(MDB.createConstant( - ConstantInt::get(Type::getInt32Ty(getContext()), - Val.udiv(APT).getLimitedValue(UINT32_MAX)))); - } else if (ProfDataName->getString().equals("VP")) - for (unsigned i = 1; i < ProfileData->getNumOperands(); i += 2) { - // The first value is the key of the value profile, which will not change. - Vals.push_back(ProfileData->getOperand(i)); - uint64_t Count = - mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(i + 1)) - ->getValue() - .getZExtValue(); - // Don't scale the magic number. - if (Count == NOMORE_ICP_MAGICNUM) { - Vals.push_back(ProfileData->getOperand(i + 1)); - continue; - } - // Using APInt::div may be expensive, but most cases should fit 64 bits. - APInt Val(128, Count); - Val *= APS; - Vals.push_back(MDB.createConstant( - ConstantInt::get(Type::getInt64Ty(getContext()), - Val.udiv(APT).getLimitedValue()))); - } - setMetadata(LLVMContext::MD_prof, MDNode::get(getContext(), Vals)); + scaleProfData(*this, S, T); } //===----------------------------------------------------------------------===// diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp index b1a10d0ce5a5..dc86f4204b1a 100644 --- a/llvm/lib/IR/ProfDataUtils.cpp +++ b/llvm/lib/IR/ProfDataUtils.cpp @@ -190,4 +190,52 @@ void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights) { I.setMetadata(LLVMContext::MD_prof, BranchWeights); } +void scaleProfData(Instruction &I, uint64_t S, uint64_t T) { + assert(T != 0 && "Caller should guarantee"); + auto *ProfileData = I.getMetadata(LLVMContext::MD_prof); + if (ProfileData == nullptr) + return; + + auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0)); + if (!ProfDataName || (!ProfDataName->getString().equals("branch_weights") && + !ProfDataName->getString().equals("VP"))) + return; + + LLVMContext &C = I.getContext(); + + MDBuilder MDB(C); + SmallVector<Metadata *, 3> Vals; + Vals.push_back(ProfileData->getOperand(0)); + APInt APS(128, S), APT(128, T); + if (ProfDataName->getString().equals("branch_weights") && + ProfileData->getNumOperands() > 0) { + // Using APInt::div may be expensive, but most cases should fit 64 bits. + APInt Val(128, mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(1)) + ->getValue() + .getZExtValue()); + Val *= APS; + Vals.push_back(MDB.createConstant(ConstantInt::get( + Type::getInt32Ty(C), Val.udiv(APT).getLimitedValue(UINT32_MAX)))); + } else if (ProfDataName->getString().equals("VP")) + for (unsigned i = 1; i < ProfileData->getNumOperands(); i += 2) { + // The first value is the key of the value profile, which will not change. + Vals.push_back(ProfileData->getOperand(i)); + uint64_t Count = + mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(i + 1)) + ->getValue() + .getZExtValue(); + // Don't scale the magic number. + if (Count == NOMORE_ICP_MAGICNUM) { + Vals.push_back(ProfileData->getOperand(i + 1)); + continue; + } + // Using APInt::div may be expensive, but most cases should fit 64 bits. + APInt Val(128, Count); + Val *= APS; + Vals.push_back(MDB.createConstant(ConstantInt::get( + Type::getInt64Ty(C), Val.udiv(APT).getLimitedValue()))); + } + I.setMetadata(LLVMContext::MD_prof, MDNode::get(C, Vals)); +} + } // namespace llvm diff --git a/llvm/lib/Object/GOFFObjectFile.cpp b/llvm/lib/Object/GOFFObjectFile.cpp index 2845d9362544..76a13559ebfe 100644 --- a/llvm/lib/Object/GOFFObjectFile.cpp +++ b/llvm/lib/Object/GOFFObjectFile.cpp @@ -168,11 +168,6 @@ GOFFObjectFile::GOFFObjectFile(MemoryBufferRef Object, Error &Err) LLVM_DEBUG(dbgs() << " -- ESD " << EsdId << "\n"); break; } - case GOFF::RT_TXT: - // Save TXT records. - TextPtrs.emplace_back(I); - LLVM_DEBUG(dbgs() << " -- TXT\n"); - break; case GOFF::RT_END: LLVM_DEBUG(dbgs() << " -- END (GOFF record type) unhandled\n"); break; @@ -369,13 +364,6 @@ GOFFObjectFile::getSymbolSection(DataRefImpl Symb) const { std::to_string(SymEdId)); } -uint64_t GOFFObjectFile::getSymbolSize(DataRefImpl Symb) const { - const uint8_t *Record = getSymbolEsdRecord(Symb); - uint32_t Length; - ESDRecord::getLength(Record, Length); - return Length; -} - const uint8_t *GOFFObjectFile::getSectionEdEsdRecord(DataRefImpl &Sec) const { SectionEntryImpl EsdIds = SectionList[Sec.d.a]; const uint8_t *EsdRecord = EsdPtrs[EsdIds.d.a]; @@ -406,154 +394,6 @@ GOFFObjectFile::getSectionPrEsdRecord(uint32_t SectionIndex) const { return EsdRecord; } -uint32_t GOFFObjectFile::getSectionDefEsdId(DataRefImpl &Sec) const { - const uint8_t *EsdRecord = getSectionEdEsdRecord(Sec); - uint32_t Length; - ESDRecord::getLength(EsdRecord, Length); - if (Length == 0) { - const uint8_t *PrEsdRecord = getSectionPrEsdRecord(Sec); - if (PrEsdRecord) - EsdRecord = PrEsdRecord; - } - - uint32_t DefEsdId; - ESDRecord::getEsdId(EsdRecord, DefEsdId); - LLVM_DEBUG(dbgs() << "Got def EsdId: " << DefEsdId << '\n'); - return DefEsdId; -} - -void GOFFObjectFile::moveSectionNext(DataRefImpl &Sec) const { - Sec.d.a++; - if ((Sec.d.a) >= SectionList.size()) - Sec.d.a = 0; -} - -Expected<StringRef> GOFFObjectFile::getSectionName(DataRefImpl Sec) const { - DataRefImpl EdSym; - SectionEntryImpl EsdIds = SectionList[Sec.d.a]; - EdSym.d.a = EsdIds.d.a; - Expected<StringRef> Name = getSymbolName(EdSym); - if (Name) { - StringRef Res = *Name; - LLVM_DEBUG(dbgs() << "Got section: " << Res << '\n'); - LLVM_DEBUG(dbgs() << "Final section name: " << Res << '\n'); - Name = Res; - } - return Name; -} - -uint64_t GOFFObjectFile::getSectionAddress(DataRefImpl Sec) const { - uint32_t Offset; - const uint8_t *EsdRecord = getSectionEdEsdRecord(Sec); - ESDRecord::getOffset(EsdRecord, Offset); - return Offset; -} - -uint64_t GOFFObjectFile::getSectionSize(DataRefImpl Sec) const { - uint32_t Length; - uint32_t DefEsdId = getSectionDefEsdId(Sec); - const uint8_t *EsdRecord = EsdPtrs[DefEsdId]; - ESDRecord::getLength(EsdRecord, Length); - LLVM_DEBUG(dbgs() << "Got section size: " << Length << '\n'); - return static_cast<uint64_t>(Length); -} - -// Unravel TXT records and expand fill characters to produce -// a contiguous sequence of bytes. -Expected<ArrayRef<uint8_t>> -GOFFObjectFile::getSectionContents(DataRefImpl Sec) const { - if (SectionDataCache.count(Sec.d.a)) { - auto &Buf = SectionDataCache[Sec.d.a]; - return ArrayRef<uint8_t>(Buf); - } - uint64_t SectionSize = getSectionSize(Sec); - uint32_t DefEsdId = getSectionDefEsdId(Sec); - - const uint8_t *EdEsdRecord = getSectionEdEsdRecord(Sec); - bool FillBytePresent; - ESDRecord::getFillBytePresent(EdEsdRecord, FillBytePresent); - uint8_t FillByte = '\0'; - if (FillBytePresent) - ESDRecord::getFillByteValue(EdEsdRecord, FillByte); - - // Initialize section with fill byte. - SmallVector<uint8_t> Data(SectionSize, FillByte); - - // Replace section with content from text records. - for (const uint8_t *TxtRecordInt : TextPtrs) { - const uint8_t *TxtRecordPtr = TxtRecordInt; - uint32_t TxtEsdId; - TXTRecord::getElementEsdId(TxtRecordPtr, TxtEsdId); - LLVM_DEBUG(dbgs() << "Got txt EsdId: " << TxtEsdId << '\n'); - - if (TxtEsdId != DefEsdId) - continue; - - uint32_t TxtDataOffset; - TXTRecord::getOffset(TxtRecordPtr, TxtDataOffset); - - uint16_t TxtDataSize; - TXTRecord::getDataLength(TxtRecordPtr, TxtDataSize); - - LLVM_DEBUG(dbgs() << "Record offset " << TxtDataOffset << ", data size " - << TxtDataSize << "\n"); - - SmallString<256> CompleteData; - CompleteData.reserve(TxtDataSize); - if (Error Err = TXTRecord::getData(TxtRecordPtr, CompleteData)) - return std::move(Err); - assert(CompleteData.size() == TxtDataSize && "Wrong length of data"); - std::copy(CompleteData.data(), CompleteData.data() + TxtDataSize, - Data.begin() + TxtDataOffset); - } - SectionDataCache[Sec.d.a] = Data; - return ArrayRef<uint8_t>(Data); -} - -uint64_t GOFFObjectFile::getSectionAlignment(DataRefImpl Sec) const { - const uint8_t *EsdRecord = getSectionEdEsdRecord(Sec); - GOFF::ESDAlignment Pow2Alignment; - ESDRecord::getAlignment(EsdRecord, Pow2Alignment); - return 1ULL << static_cast<uint64_t>(Pow2Alignment); -} - -bool GOFFObjectFile::isSectionText(DataRefImpl Sec) const { - const uint8_t *EsdRecord = getSectionEdEsdRecord(Sec); - GOFF::ESDExecutable Executable; - ESDRecord::getExecutable(EsdRecord, Executable); - return Executable == GOFF::ESD_EXE_CODE; -} - -bool GOFFObjectFile::isSectionData(DataRefImpl Sec) const { - const uint8_t *EsdRecord = getSectionEdEsdRecord(Sec); - GOFF::ESDExecutable Executable; - ESDRecord::getExecutable(EsdRecord, Executable); - return Executable == GOFF::ESD_EXE_DATA; -} - -bool GOFFObjectFile::isSectionNoLoad(DataRefImpl Sec) const { - const uint8_t *EsdRecord = getSectionEdEsdRecord(Sec); - GOFF::ESDLoadingBehavior LoadingBehavior; - ESDRecord::getLoadingBehavior(EsdRecord, LoadingBehavior); - return LoadingBehavior == GOFF::ESD_LB_NoLoad; -} - -bool GOFFObjectFile::isSectionReadOnlyData(DataRefImpl Sec) const { - if (!isSectionData(Sec)) - return false; - - const uint8_t *EsdRecord = getSectionEdEsdRecord(Sec); - GOFF::ESDLoadingBehavior LoadingBehavior; - ESDRecord::getLoadingBehavior(EsdRecord, LoadingBehavior); - return LoadingBehavior == GOFF::ESD_LB_Initial; -} - -bool GOFFObjectFile::isSectionZeroInit(DataRefImpl Sec) const { - // GOFF uses fill characters and fill characters are applied - // on getSectionContents() - so we say false to zero init. - return false; -} - section_iterator GOFFObjectFile::section_begin() const { DataRefImpl Sec; moveSectionNext(Sec); @@ -636,13 +476,6 @@ Error ESDRecord::getData(const uint8_t *Record, return getContinuousData(Record, DataSize, 72, CompleteData); } -Error TXTRecord::getData(const uint8_t *Record, - SmallString<256> &CompleteData) { - uint16_t Length; - getDataLength(Record, Length); - return getContinuousData(Record, Length, 24, CompleteData); -} - Error ENDRecord::getData(const uint8_t *Record, SmallString<256> &CompleteData) { uint16_t Length = getNameLength(Record); diff --git a/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp b/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp index 6865850cf04f..22da7ddef98a 100644 --- a/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp +++ b/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp @@ -680,9 +680,11 @@ bool AArch64MIPeepholeOpt::visitFMOVDr(MachineInstr &MI) { // Let's remove MIs for high 64-bits. Register OldDef = MI.getOperand(0).getReg(); Register NewDef = MI.getOperand(1).getReg(); + LLVM_DEBUG(dbgs() << "Removing: " << MI << "\n"); + MRI->clearKillFlags(OldDef); + MRI->clearKillFlags(NewDef); MRI->constrainRegClass(NewDef, MRI->getRegClass(OldDef)); MRI->replaceRegWith(OldDef, NewDef); - LLVM_DEBUG(dbgs() << "Removed: " << MI << "\n"); MI.eraseFromParent(); return true; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td index 875f2e382d54..e68fb42ece9f 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td @@ -531,14 +531,6 @@ class VALUVs2<bits<6> funct6, bits<5> vs1, RISCVVFormat opv, string opcodestr> : RVInstV<funct6, vs1, opv, (outs VR:$vd), (ins VR:$vs2, VMaskOp:$vm), opcodestr, "$vd, $vs2$vm">; - -// op vd, vs2 (use vs1 as instruction encoding) -class VALUVs2NoVm<bits<6> funct6, bits<5> vs1, RISCVVFormat opv, string opcodestr> - : RVInstV<funct6, vs1, opv, (outs VR:$vd), - (ins VR:$vs2), opcodestr, - "$vd, $vs2"> { - let vm = 1; -} } // hasSideEffects = 0, mayLoad = 0, mayStore = 0 //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td index b388bb00b0f2..60db03d68e47 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td @@ -67,6 +67,16 @@ class PALUVVNoVm<bits<6> funct6, RISCVVFormat opv, string opcodestr> let Inst{6-0} = OPC_OP_P.Value; } +// op vd, vs2, vs1 +class PALUVVNoVmTernary<bits<6> funct6, RISCVVFormat opv, string opcodestr> + : RVInstVV<funct6, opv, (outs VR:$vd_wb), + (ins VR:$vd, VR:$vs2, VR:$vs1), + opcodestr, "$vd, $vs2, $vs1"> { + let Constraints = "$vd = $vd_wb"; + let vm = 1; + let Inst{6-0} = OPC_OP_P.Value; +} + // op vd, vs2, imm class PALUVINoVm<bits<6> funct6, string opcodestr, Operand optype> : VALUVINoVm<funct6, opcodestr, optype> { @@ -74,18 +84,34 @@ class PALUVINoVm<bits<6> funct6, string opcodestr, Operand optype> let Inst{14-12} = OPMVV.Value; } -// op vd, vs2 (use vs1 as instruction encoding) -class PALUVs2NoVm<bits<6> funct6, bits<5> vs1, RISCVVFormat opv, string opcodestr> - : VALUVs2NoVm<funct6, vs1, opv, opcodestr> { +// op vd, vs2, imm where vd is also a source regardless of tail policy +class PALUVINoVmBinary<bits<6> funct6, string opcodestr, Operand optype> + : RVInstIVI<funct6, (outs VR:$vd_wb), + (ins VR:$vd, VR:$vs2, optype:$imm), + opcodestr, "$vd, $vs2, $imm"> { + let Constraints = "$vd = $vd_wb"; + let vm = 1; + let Inst{6-0} = OPC_OP_P.Value; + let Inst{14-12} = OPMVV.Value; +} + +// op vd, vs2 (use vs1 as instruction encoding) where vd is also a source +// regardless of tail policy +class PALUVs2NoVmBinary<bits<6> funct6, bits<5> vs1, RISCVVFormat opv, + string opcodestr> + : RVInstV<funct6, vs1, opv, (outs VR:$vd_wb), (ins VR:$vd, VR:$vs2), + opcodestr, "$vd, $vs2"> { + let Constraints = "$vd = $vd_wb"; + let vm = 1; let Inst{6-0} = OPC_OP_P.Value; } multiclass VAES_MV_V_S<bits<6> funct6_vv, bits<6> funct6_vs, bits<5> vs1, RISCVVFormat opv, string opcodestr> { let RVVConstraint = NoConstraint in - def NAME # _VV : PALUVs2NoVm<funct6_vv, vs1, opv, opcodestr # ".vv">; + def NAME # _VV : PALUVs2NoVmBinary<funct6_vv, vs1, opv, opcodestr # ".vv">; let RVVConstraint = VS2Constraint in - def NAME # _VS : PALUVs2NoVm<funct6_vs, vs1, opv, opcodestr # ".vs">; + def NAME # _VS : PALUVs2NoVmBinary<funct6_vs, vs1, opv, opcodestr # ".vs">; } } // hasSideEffects = 0, mayLoad = 0, mayStore = 0 @@ -116,14 +142,14 @@ let Predicates = [HasStdExtZvkb] in { } // Predicates = [HasStdExtZvkb] let Predicates = [HasStdExtZvkg], RVVConstraint = NoConstraint in { - def VGHSH_VV : PALUVVNoVm<0b101100, OPMVV, "vghsh.vv">; - def VGMUL_VV : PALUVs2NoVm<0b101000, 0b10001, OPMVV, "vgmul.vv">; + def VGHSH_VV : PALUVVNoVmTernary<0b101100, OPMVV, "vghsh.vv">; + def VGMUL_VV : PALUVs2NoVmBinary<0b101000, 0b10001, OPMVV, "vgmul.vv">; } // Predicates = [HasStdExtZvkg] let Predicates = [HasStdExtZvknhaOrZvknhb], RVVConstraint = Sha2Constraint in { - def VSHA2CH_VV : PALUVVNoVm<0b101110, OPMVV, "vsha2ch.vv">; - def VSHA2CL_VV : PALUVVNoVm<0b101111, OPMVV, "vsha2cl.vv">; - def VSHA2MS_VV : PALUVVNoVm<0b101101, OPMVV, "vsha2ms.vv">; + def VSHA2CH_VV : PALUVVNoVmTernary<0b101110, OPMVV, "vsha2ch.vv">; + def VSHA2CL_VV : PALUVVNoVmTernary<0b101111, OPMVV, "vsha2cl.vv">; + def VSHA2MS_VV : PALUVVNoVmTernary<0b101101, OPMVV, "vsha2ms.vv">; } // Predicates = [HasStdExtZvknhaOrZvknhb] let Predicates = [HasStdExtZvkned]in { @@ -132,9 +158,9 @@ let Predicates = [HasStdExtZvkned]in { defm VAESEF : VAES_MV_V_S<0b101000, 0b101001, 0b00011, OPMVV, "vaesef">; defm VAESEM : VAES_MV_V_S<0b101000, 0b101001, 0b00010, OPMVV, "vaesem">; def VAESKF1_VI : PALUVINoVm<0b100010, "vaeskf1.vi", uimm5>; - def VAESKF2_VI : PALUVINoVm<0b101010, "vaeskf2.vi", uimm5>; + def VAESKF2_VI : PALUVINoVmBinary<0b101010, "vaeskf2.vi", uimm5>; let RVVConstraint = VS2Constraint in - def VAESZ_VS : PALUVs2NoVm<0b101001, 0b00111, OPMVV, "vaesz.vs">; + def VAESZ_VS : PALUVs2NoVmBinary<0b101001, 0b00111, OPMVV, "vaesz.vs">; } // Predicates = [HasStdExtZvkned] let Predicates = [HasStdExtZvksed] in { @@ -144,7 +170,7 @@ let Predicates = [HasStdExtZvksed] in { } // Predicates = [HasStdExtZvksed] let Predicates = [HasStdExtZvksh], RVVConstraint = VS2Constraint in { - def VSM3C_VI : PALUVINoVm<0b101011, "vsm3c.vi", uimm5>; + def VSM3C_VI : PALUVINoVmBinary<0b101011, "vsm3c.vi", uimm5>; def VSM3ME_VV : PALUVVNoVm<0b100000, OPMVV, "vsm3me.vv">; } // Predicates = [HasStdExtZvksh] diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td index 9da1f73681c6..90c4a7193ee3 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td @@ -64,9 +64,14 @@ def sub_vrm1_6 : ComposedSubRegIndex<sub_vrm2_3, sub_vrm1_0>; def sub_vrm1_7 : ComposedSubRegIndex<sub_vrm2_3, sub_vrm1_1>; // GPR sizes change with HwMode. -// FIXME: Support HwMode in SubRegIndex? -def sub_gpr_even : SubRegIndex<-1>; -def sub_gpr_odd : SubRegIndex<-1, -1>; +def sub_gpr_even : SubRegIndex<32> { + let SubRegRanges = SubRegRangeByHwMode<[RV32, RV64], + [SubRegRange<32>, SubRegRange<64>]>; +} +def sub_gpr_odd : SubRegIndex<32, 32> { + let SubRegRanges = SubRegRangeByHwMode<[RV32, RV64], + [SubRegRange<32, 32>, SubRegRange<64, 64>]>; +} } // Namespace = "RISCV" // Integer registers diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp index 027ee1086bf4..0788d0c3a721 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp @@ -153,7 +153,7 @@ /// %__THREW__.val = __THREW__; /// __THREW__ = 0; /// %__threwValue.val = __threwValue; -/// if (%__THREW__.val != 0 & %__threwValue.val != 0) { +/// if (%__THREW__.val != 0) { /// %label = __wasm_setjmp_test(%__THREW__.val, functionInvocationId); /// if (%label == 0) /// emscripten_longjmp(%__THREW__.val, %__threwValue.val); @@ -712,12 +712,10 @@ void WebAssemblyLowerEmscriptenEHSjLj::wrapTestSetjmp( BasicBlock *ThenBB1 = BasicBlock::Create(C, "if.then1", F); BasicBlock *ElseBB1 = BasicBlock::Create(C, "if.else1", F); BasicBlock *EndBB1 = BasicBlock::Create(C, "if.end", F); - Value *ThrewCmp = IRB.CreateICmpNE(Threw, getAddrSizeInt(M, 0)); Value *ThrewValue = IRB.CreateLoad(IRB.getInt32Ty(), ThrewValueGV, ThrewValueGV->getName() + ".val"); - Value *ThrewValueCmp = IRB.CreateICmpNE(ThrewValue, IRB.getInt32(0)); - Value *Cmp1 = IRB.CreateAnd(ThrewCmp, ThrewValueCmp, "cmp1"); - IRB.CreateCondBr(Cmp1, ThenBB1, ElseBB1); + Value *ThrewCmp = IRB.CreateICmpNE(Threw, getAddrSizeInt(M, 0)); + IRB.CreateCondBr(ThrewCmp, ThenBB1, ElseBB1); // Generate call.em.longjmp BB once and share it within the function if (!CallEmLongjmpBB) { diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp index 9a8040bc4b06..2cbef8a7ae61 100644 --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -234,6 +234,21 @@ static cl::opt<unsigned> ProfileICPRelativeHotnessSkip( cl::desc( "Skip relative hotness check for ICP up to given number of targets.")); +static cl::opt<unsigned> HotFuncCutoffForStalenessError( + "hot-func-cutoff-for-staleness-error", cl::Hidden, cl::init(800000), + cl::desc("A function is considered hot for staleness error check if its " + "total sample count is above the specified percentile")); + +static cl::opt<unsigned> MinfuncsForStalenessError( + "min-functions-for-staleness-error", cl::Hidden, cl::init(50), + cl::desc("Skip the check if the number of hot functions is smaller than " + "the specified number.")); + +static cl::opt<unsigned> PrecentMismatchForStalenessError( + "precent-mismatch-for-staleness-error", cl::Hidden, cl::init(80), + cl::desc("Reject the profile if the mismatch percent is higher than the " + "given number.")); + static cl::opt<bool> CallsitePrioritizedInline( "sample-profile-prioritized-inline", cl::Hidden, @@ -630,6 +645,8 @@ protected: std::vector<Function *> buildFunctionOrder(Module &M, LazyCallGraph &CG); std::unique_ptr<ProfiledCallGraph> buildProfiledCallGraph(Module &M); void generateMDProfMetadata(Function &F); + bool rejectHighStalenessProfile(Module &M, ProfileSummaryInfo *PSI, + const SampleProfileMap &Profiles); /// Map from function name to Function *. Used to find the function from /// the function name. If the function name contains suffix, additional @@ -2187,6 +2204,55 @@ bool SampleProfileLoader::doInitialization(Module &M, return true; } +// Note that this is a module-level check. Even if one module is errored out, +// the entire build will be errored out. However, the user could make big +// changes to functions in single module but those changes might not be +// performance significant to the whole binary. Therefore, to avoid those false +// positives, we select a reasonable big set of hot functions that are supposed +// to be globally performance significant, only compute and check the mismatch +// within those functions. The function selection is based on two criteria: +// 1) The function is hot enough, which is tuned by a hotness-based +// flag(HotFuncCutoffForStalenessError). 2) The num of function is large enough +// which is tuned by the MinfuncsForStalenessError flag. +bool SampleProfileLoader::rejectHighStalenessProfile( + Module &M, ProfileSummaryInfo *PSI, const SampleProfileMap &Profiles) { + assert(FunctionSamples::ProfileIsProbeBased && + "Only support for probe-based profile"); + uint64_t TotalHotFunc = 0; + uint64_t NumMismatchedFunc = 0; + for (const auto &I : Profiles) { + const auto &FS = I.second; + const auto *FuncDesc = ProbeManager->getDesc(FS.getGUID()); + if (!FuncDesc) + continue; + + // Use a hotness-based threshold to control the function selection. + if (!PSI->isHotCountNthPercentile(HotFuncCutoffForStalenessError, + FS.getTotalSamples())) + continue; + + TotalHotFunc++; + if (ProbeManager->profileIsHashMismatched(*FuncDesc, FS)) + NumMismatchedFunc++; + } + // Make sure that the num of selected function is not too small to distinguish + // from the user's benign changes. + if (TotalHotFunc < MinfuncsForStalenessError) + return false; + + // Finally check the mismatch percentage against the threshold. + if (NumMismatchedFunc * 100 >= + TotalHotFunc * PrecentMismatchForStalenessError) { + auto &Ctx = M.getContext(); + const char *Msg = + "The input profile significantly mismatches current source code. " + "Please recollect profile to avoid performance regression."; + Ctx.diagnose(DiagnosticInfoSampleProfile(M.getModuleIdentifier(), Msg)); + return true; + } + return false; +} + void SampleProfileMatcher::findIRAnchors( const Function &F, std::map<LineLocation, StringRef> &IRAnchors) { // For inlined code, recover the original callsite and callee by finding the @@ -2718,6 +2784,11 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM, ProfileSummary::PSK_Sample); PSI->refresh(); } + + if (FunctionSamples::ProfileIsProbeBased && + rejectHighStalenessProfile(M, PSI, Reader->getProfiles())) + return false; + // Compute the total number of samples collected in this profile. for (const auto &I : Reader->getProfiles()) TotalCollectedSamples += I.second.getTotalSamples(); diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index e86705e89889..5d03b66b0ce3 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -68,6 +68,9 @@ class VPBuilder { public: VPBuilder() = default; VPBuilder(VPBasicBlock *InsertBB) { setInsertPoint(InsertBB); } + VPBuilder(VPRecipeBase *InsertPt) { + setInsertPoint(InsertPt->getParent(), InsertPt->getIterator()); + } /// Clear the insertion point: created instructions will not be inserted into /// a block. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index f70ccf8270b2..2f56799a7b51 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1127,6 +1127,12 @@ public: return WrapFlags.HasNSW; } + bool isDisjoint() const { + assert(OpType == OperationType::DisjointOp && + "recipe cannot have a disjoing flag"); + return DisjointFlags.IsDisjoint; + } + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void printFlags(raw_ostream &O) const; #endif diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index aa2535906945..a03a408686ef 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -261,6 +261,11 @@ m_Mul(const Op0_t &Op0, const Op1_t &Op1) { return m_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1); } +template <typename Op0_t, typename Op1_t> +inline AllBinaryRecipe_match<Op0_t, Op1_t, Instruction::Or> +m_Or(const Op0_t &Op0, const Op1_t &Op1) { + return m_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1); +} } // namespace VPlanPatternMatch } // namespace llvm diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index e6ab27a918bc..006f4349d6fb 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1255,7 +1255,24 @@ void VPlanTransforms::dropPoisonGeneratingRecipes( // load/store. If the underlying instruction has poison-generating flags, // drop them directly. if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(CurRec)) { - RecWithFlags->dropPoisonGeneratingFlags(); + VPValue *A, *B; + using namespace llvm::VPlanPatternMatch; + // Dropping disjoint from an OR may yield incorrect results, as some + // analysis may have converted it to an Add implicitly (e.g. SCEV used + // for dependence analysis). Instead, replace it with an equivalent Add. + // This is possible as all users of the disjoint OR only access lanes + // where the operands are disjoint or poison otherwise. + if (match(RecWithFlags, m_Or(m_VPValue(A), m_VPValue(B))) && + RecWithFlags->isDisjoint()) { + VPBuilder Builder(RecWithFlags); + VPInstruction *New = Builder.createOverflowingOp( + Instruction::Add, {A, B}, {false, false}, + RecWithFlags->getDebugLoc()); + RecWithFlags->replaceAllUsesWith(New); + RecWithFlags->eraseFromParent(); + CurRec = New; + } else + RecWithFlags->dropPoisonGeneratingFlags(); } else { Instruction *Instr = dyn_cast_or_null<Instruction>( CurRec->getVPSingleValue()->getUnderlyingValue()); diff --git a/llvm/test/CodeGen/AArch64/peephole-movd.mir b/llvm/test/CodeGen/AArch64/peephole-movd.mir new file mode 100644 index 000000000000..bd7f0ab3f044 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/peephole-movd.mir @@ -0,0 +1,60 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -run-pass=aarch64-mi-peephole-opt -o - -mtriple=aarch64-unknown-linux -verify-machineinstrs %s | FileCheck %s + +--- +name: remove_kill_flags +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $w0 + ; CHECK-LABEL: name: remove_kill_flags + ; CHECK: liveins: $w0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[MOVIv2d_ns:%[0-9]+]]:fpr128 = MOVIv2d_ns 0 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr64 = COPY [[MOVIv2d_ns]].dsub + ; CHECK-NEXT: [[UQSHLv8i8_shift:%[0-9]+]]:fpr64 = UQSHLv8i8_shift killed [[COPY]], 1 + ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, [[UQSHLv8i8_shift]], %subreg.dsub + ; CHECK-NEXT: [[TBLv8i8One:%[0-9]+]]:fpr64 = TBLv8i8One killed [[SUBREG_TO_REG]], [[UQSHLv8i8_shift]] + ; CHECK-NEXT: [[DEF:%[0-9]+]]:fpr128 = IMPLICIT_DEF + ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF]], [[UQSHLv8i8_shift]], %subreg.dsub + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %0:fpr128 = MOVIv2d_ns 0 + %1:fpr64 = COPY %0.dsub:fpr128 + %2:fpr64 = UQSHLv8i8_shift killed %1:fpr64, 1 + %3:fpr64 = FMOVDr %2:fpr64 + %4:fpr128 = SUBREG_TO_REG 0, killed %3:fpr64, %subreg.dsub + %5:fpr64 = TBLv8i8One killed %4:fpr128, %2:fpr64 + %7:fpr128 = IMPLICIT_DEF + %6:fpr128 = INSERT_SUBREG %7:fpr128, killed %2:fpr64, %subreg.dsub + RET_ReallyLR implicit $w0 +... +--- +name: remove_kill_flags2 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $w0 + ; CHECK-LABEL: name: remove_kill_flags2 + ; CHECK: liveins: $w0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[MOVIv2d_ns:%[0-9]+]]:fpr128 = MOVIv2d_ns 0 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr64 = COPY [[MOVIv2d_ns]].dsub + ; CHECK-NEXT: [[UQSHLv8i8_shift:%[0-9]+]]:fpr64 = UQSHLv8i8_shift killed [[COPY]], 1 + ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, [[UQSHLv8i8_shift]], %subreg.dsub + ; CHECK-NEXT: [[DEF:%[0-9]+]]:fpr128 = IMPLICIT_DEF + ; CHECK-NEXT: [[INSERT_SUBREG:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF]], [[UQSHLv8i8_shift]], %subreg.dsub + ; CHECK-NEXT: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF + ; CHECK-NEXT: [[INSERT_SUBREG1:%[0-9]+]]:fpr128 = INSERT_SUBREG [[DEF1]], [[UQSHLv8i8_shift]], %subreg.dsub + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %0:fpr128 = MOVIv2d_ns 0 + %1:fpr64 = COPY %0.dsub:fpr128 + %2:fpr64 = UQSHLv8i8_shift killed %1:fpr64, 1 + %3:fpr64 = FMOVDr %2:fpr64 + %4:fpr128 = SUBREG_TO_REG 0, %3:fpr64, %subreg.dsub + %7:fpr128 = IMPLICIT_DEF + %6:fpr128 = INSERT_SUBREG %7:fpr128, killed %2:fpr64, %subreg.dsub + %9:fpr128 = IMPLICIT_DEF + %8:fpr128 = INSERT_SUBREG %9:fpr128, killed %3:fpr64, %subreg.dsub + RET_ReallyLR implicit $w0 +... + diff --git a/llvm/test/CodeGen/AArch64/pr86717.ll b/llvm/test/CodeGen/AArch64/pr86717.ll new file mode 100644 index 000000000000..aa8be954be72 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/pr86717.ll @@ -0,0 +1,22 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=aarch64 | FileCheck %s + +define <16 x i8> @f(i32 %0) { +; CHECK-LABEL: f: +; CHECK: // %bb.0: +; CHECK-NEXT: sub sp, sp, #16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: mov w8, #1 // =0x1 +; CHECK-NEXT: mov x9, sp +; CHECK-NEXT: sub w8, w8, w0 +; CHECK-NEXT: bfxil x9, x8, #0, #4 +; CHECK-NEXT: mov w8, #3 // =0x3 +; CHECK-NEXT: str q0, [sp] +; CHECK-NEXT: strb w8, [x9] +; CHECK-NEXT: ldr q0, [sp], #16 +; CHECK-NEXT: ret + %2 = sub nuw i32 1, %0 + %3 = insertelement <16 x i8> zeroinitializer, i8 3, i32 %2 + ret <16 x i8> %3 +} diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll index 25106b456d2f..6629d3440549 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll @@ -123,9 +123,10 @@ define void @insert_32xi8_idx(ptr %src, ptr %dst, i8 %in, i32 %idx) nounwind { ; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 ; CHECK-NEXT: xvld $xr0, $a0, 0 ; CHECK-NEXT: xvst $xr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 4, 0 -; CHECK-NEXT: st.b $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 4, 0 +; CHECK-NEXT: st.b $a2, $a3, 0 ; CHECK-NEXT: xvld $xr0, $sp, 0 ; CHECK-NEXT: xvst $xr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $fp, -64 @@ -149,9 +150,10 @@ define void @insert_16xi16_idx(ptr %src, ptr %dst, i16 %in, i32 %idx) nounwind { ; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 ; CHECK-NEXT: xvld $xr0, $a0, 0 ; CHECK-NEXT: xvst $xr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 4, 1 -; CHECK-NEXT: st.h $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 4, 1 +; CHECK-NEXT: st.h $a2, $a3, 0 ; CHECK-NEXT: xvld $xr0, $sp, 0 ; CHECK-NEXT: xvst $xr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $fp, -64 @@ -175,9 +177,10 @@ define void @insert_8xi32_idx(ptr %src, ptr %dst, i32 %in, i32 %idx) nounwind { ; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 ; CHECK-NEXT: xvld $xr0, $a0, 0 ; CHECK-NEXT: xvst $xr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 4, 2 -; CHECK-NEXT: st.w $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 4, 2 +; CHECK-NEXT: st.w $a2, $a3, 0 ; CHECK-NEXT: xvld $xr0, $sp, 0 ; CHECK-NEXT: xvst $xr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $fp, -64 @@ -201,9 +204,10 @@ define void @insert_4xi64_idx(ptr %src, ptr %dst, i64 %in, i32 %idx) nounwind { ; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 ; CHECK-NEXT: xvld $xr0, $a0, 0 ; CHECK-NEXT: xvst $xr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 4, 3 -; CHECK-NEXT: st.d $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 4, 3 +; CHECK-NEXT: st.d $a2, $a3, 0 ; CHECK-NEXT: xvld $xr0, $sp, 0 ; CHECK-NEXT: xvst $xr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $fp, -64 @@ -227,9 +231,10 @@ define void @insert_8xfloat_idx(ptr %src, ptr %dst, float %in, i32 %idx) nounwin ; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 ; CHECK-NEXT: xvld $xr1, $a0, 0 ; CHECK-NEXT: xvst $xr1, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a2, 4, 2 -; CHECK-NEXT: fst.s $fa0, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a2, 31, 0 +; CHECK-NEXT: addi.d $a2, $sp, 0 +; CHECK-NEXT: bstrins.d $a2, $a0, 4, 2 +; CHECK-NEXT: fst.s $fa0, $a2, 0 ; CHECK-NEXT: xvld $xr0, $sp, 0 ; CHECK-NEXT: xvst $xr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $fp, -64 @@ -253,9 +258,10 @@ define void @insert_4xdouble_idx(ptr %src, ptr %dst, double %in, i32 %idx) nounw ; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0 ; CHECK-NEXT: xvld $xr1, $a0, 0 ; CHECK-NEXT: xvst $xr1, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a2, 4, 3 -; CHECK-NEXT: fst.d $fa0, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a2, 31, 0 +; CHECK-NEXT: addi.d $a2, $sp, 0 +; CHECK-NEXT: bstrins.d $a2, $a0, 4, 3 +; CHECK-NEXT: fst.d $fa0, $a2, 0 ; CHECK-NEXT: xvld $xr0, $sp, 0 ; CHECK-NEXT: xvst $xr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $fp, -64 diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insertelement.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insertelement.ll index 7f232073ae12..19171b7d8ed7 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insertelement.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insertelement.ll @@ -87,9 +87,10 @@ define void @insert_16xi8_idx(ptr %src, ptr %dst, i8 %ins, i32 %idx) nounwind { ; CHECK-NEXT: addi.d $sp, $sp, -16 ; CHECK-NEXT: vld $vr0, $a0, 0 ; CHECK-NEXT: vst $vr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 3, 0 -; CHECK-NEXT: st.b $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 3, 0 +; CHECK-NEXT: st.b $a2, $a3, 0 ; CHECK-NEXT: vld $vr0, $sp, 0 ; CHECK-NEXT: vst $vr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $sp, 16 @@ -106,9 +107,10 @@ define void @insert_8xi16_idx(ptr %src, ptr %dst, i16 %ins, i32 %idx) nounwind { ; CHECK-NEXT: addi.d $sp, $sp, -16 ; CHECK-NEXT: vld $vr0, $a0, 0 ; CHECK-NEXT: vst $vr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 3, 1 -; CHECK-NEXT: st.h $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 3, 1 +; CHECK-NEXT: st.h $a2, $a3, 0 ; CHECK-NEXT: vld $vr0, $sp, 0 ; CHECK-NEXT: vst $vr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $sp, 16 @@ -125,9 +127,10 @@ define void @insert_4xi32_idx(ptr %src, ptr %dst, i32 %ins, i32 %idx) nounwind { ; CHECK-NEXT: addi.d $sp, $sp, -16 ; CHECK-NEXT: vld $vr0, $a0, 0 ; CHECK-NEXT: vst $vr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 3, 2 -; CHECK-NEXT: st.w $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 3, 2 +; CHECK-NEXT: st.w $a2, $a3, 0 ; CHECK-NEXT: vld $vr0, $sp, 0 ; CHECK-NEXT: vst $vr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $sp, 16 @@ -144,9 +147,10 @@ define void @insert_2xi64_idx(ptr %src, ptr %dst, i64 %ins, i32 %idx) nounwind { ; CHECK-NEXT: addi.d $sp, $sp, -16 ; CHECK-NEXT: vld $vr0, $a0, 0 ; CHECK-NEXT: vst $vr0, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a3, 3, 3 -; CHECK-NEXT: st.d $a2, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a3, 31, 0 +; CHECK-NEXT: addi.d $a3, $sp, 0 +; CHECK-NEXT: bstrins.d $a3, $a0, 3, 3 +; CHECK-NEXT: st.d $a2, $a3, 0 ; CHECK-NEXT: vld $vr0, $sp, 0 ; CHECK-NEXT: vst $vr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $sp, 16 @@ -163,9 +167,10 @@ define void @insert_4xfloat_idx(ptr %src, ptr %dst, float %ins, i32 %idx) nounwi ; CHECK-NEXT: addi.d $sp, $sp, -16 ; CHECK-NEXT: vld $vr1, $a0, 0 ; CHECK-NEXT: vst $vr1, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a2, 3, 2 -; CHECK-NEXT: fst.s $fa0, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a2, 31, 0 +; CHECK-NEXT: addi.d $a2, $sp, 0 +; CHECK-NEXT: bstrins.d $a2, $a0, 3, 2 +; CHECK-NEXT: fst.s $fa0, $a2, 0 ; CHECK-NEXT: vld $vr0, $sp, 0 ; CHECK-NEXT: vst $vr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $sp, 16 @@ -182,9 +187,10 @@ define void @insert_2xdouble_idx(ptr %src, ptr %dst, double %ins, i32 %idx) noun ; CHECK-NEXT: addi.d $sp, $sp, -16 ; CHECK-NEXT: vld $vr1, $a0, 0 ; CHECK-NEXT: vst $vr1, $sp, 0 -; CHECK-NEXT: addi.d $a0, $sp, 0 -; CHECK-NEXT: bstrins.d $a0, $a2, 3, 3 -; CHECK-NEXT: fst.d $fa0, $a0, 0 +; CHECK-NEXT: bstrpick.d $a0, $a2, 31, 0 +; CHECK-NEXT: addi.d $a2, $sp, 0 +; CHECK-NEXT: bstrins.d $a2, $a0, 3, 3 +; CHECK-NEXT: fst.d $fa0, $a2, 0 ; CHECK-NEXT: vld $vr0, $sp, 0 ; CHECK-NEXT: vst $vr0, $a1, 0 ; CHECK-NEXT: addi.d $sp, $sp, 16 diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj.ll b/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj.ll index 32942cd92e68..d88f42a4dc58 100644 --- a/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj.ll +++ b/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj.ll @@ -22,10 +22,8 @@ entry: to label %try.cont unwind label %lpad ; CHECK: entry.split.split: -; CHECK: %[[CMP0:.*]] = icmp ne i32 %__THREW__.val, 0 -; CHECK-NEXT: %__threwValue.val = load i32, ptr @__threwValue -; CHECK-NEXT: %[[CMP1:.*]] = icmp ne i32 %__threwValue.val, 0 -; CHECK-NEXT: %[[CMP:.*]] = and i1 %[[CMP0]], %[[CMP1]] +; CHECK: %__threwValue.val = load i32, ptr @__threwValue +; CHECK-NEXT: %[[CMP:.*]] = icmp ne i32 %__THREW__.val, 0 ; CHECK-NEXT: br i1 %[[CMP]], label %if.then1, label %if.else1 ; This is exception checking part. %if.else1 leads here @@ -121,6 +119,7 @@ if.end: ; preds = %entry ; CHECK-NEXT: unreachable ; CHECK: normal: +; CHECK-NEXT: %__threwValue.val = load i32, ptr @__threwValue, align 4 ; CHECK-NEXT: icmp ne i32 %__THREW__.val, 0 return: ; preds = %if.end, %entry diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll index 27ec95a2c462..dca4c59d7c87 100644 --- a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll +++ b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll @@ -37,10 +37,8 @@ entry: ; CHECK-NEXT: call cc{{.*}} void @__invoke_void_[[PTR]]_i32(ptr @emscripten_longjmp, [[PTR]] %[[JMPBUF]], i32 1) ; CHECK-NEXT: %[[__THREW__VAL:.*]] = load [[PTR]], ptr @__THREW__ ; CHECK-NEXT: store [[PTR]] 0, ptr @__THREW__ -; CHECK-NEXT: %[[CMP0:.*]] = icmp ne [[PTR]] %__THREW__.val, 0 ; CHECK-NEXT: %[[THREWVALUE_VAL:.*]] = load i32, ptr @__threwValue -; CHECK-NEXT: %[[CMP1:.*]] = icmp ne i32 %[[THREWVALUE_VAL]], 0 -; CHECK-NEXT: %[[CMP:.*]] = and i1 %[[CMP0]], %[[CMP1]] +; CHECK-NEXT: %[[CMP:.*]] = icmp ne [[PTR]] %__THREW__.val, 0 ; CHECK-NEXT: br i1 %[[CMP]], label %if.then1, label %if.else1 ; CHECK: entry.split.split.split: diff --git a/llvm/test/CodeGen/X86/2009-06-05-VariableIndexInsert.ll b/llvm/test/CodeGen/X86/2009-06-05-VariableIndexInsert.ll index 535450a52ff6..695a2d0cd806 100644 --- a/llvm/test/CodeGen/X86/2009-06-05-VariableIndexInsert.ll +++ b/llvm/test/CodeGen/X86/2009-06-05-VariableIndexInsert.ll @@ -9,11 +9,11 @@ define <2 x i64> @_mm_insert_epi16(<2 x i64> %a, i32 %b, i32 %imm) nounwind read ; X86-NEXT: movl %esp, %ebp ; X86-NEXT: andl $-16, %esp ; X86-NEXT: subl $32, %esp -; X86-NEXT: movzwl 8(%ebp), %eax -; X86-NEXT: movl 12(%ebp), %ecx -; X86-NEXT: andl $7, %ecx +; X86-NEXT: movl 12(%ebp), %eax +; X86-NEXT: movzwl 8(%ebp), %ecx +; X86-NEXT: andl $7, %eax ; X86-NEXT: movaps %xmm0, (%esp) -; X86-NEXT: movw %ax, (%esp,%ecx,2) +; X86-NEXT: movw %cx, (%esp,%eax,2) ; X86-NEXT: movaps (%esp), %xmm0 ; X86-NEXT: movl %ebp, %esp ; X86-NEXT: popl %ebp diff --git a/llvm/test/CodeGen/X86/insertelement-var-index.ll b/llvm/test/CodeGen/X86/insertelement-var-index.ll index 8ed8495d7a46..5420e6b5ce86 100644 --- a/llvm/test/CodeGen/X86/insertelement-var-index.ll +++ b/llvm/test/CodeGen/X86/insertelement-var-index.ll @@ -1009,18 +1009,19 @@ define <2 x i64> @arg_i64_v2i64(<2 x i64> %v, i64 %x, i32 %y) nounwind { ; X86AVX2-NEXT: pushl %esi ; X86AVX2-NEXT: andl $-16, %esp ; X86AVX2-NEXT: subl $48, %esp -; X86AVX2-NEXT: movl 8(%ebp), %eax -; X86AVX2-NEXT: movl 12(%ebp), %ecx -; X86AVX2-NEXT: movl 16(%ebp), %edx +; X86AVX2-NEXT: movl 8(%ebp), %edx +; X86AVX2-NEXT: movl 12(%ebp), %eax +; X86AVX2-NEXT: movl 16(%ebp), %ecx ; X86AVX2-NEXT: vmovaps %xmm0, (%esp) -; X86AVX2-NEXT: leal (%edx,%edx), %esi +; X86AVX2-NEXT: addl %ecx, %ecx +; X86AVX2-NEXT: movl %ecx, %esi ; X86AVX2-NEXT: andl $3, %esi -; X86AVX2-NEXT: movl %eax, (%esp,%esi,4) +; X86AVX2-NEXT: movl %edx, (%esp,%esi,4) ; X86AVX2-NEXT: vmovaps (%esp), %xmm0 ; X86AVX2-NEXT: vmovaps %xmm0, {{[0-9]+}}(%esp) -; X86AVX2-NEXT: leal 1(%edx,%edx), %eax -; X86AVX2-NEXT: andl $3, %eax -; X86AVX2-NEXT: movl %ecx, 16(%esp,%eax,4) +; X86AVX2-NEXT: incl %ecx +; X86AVX2-NEXT: andl $3, %ecx +; X86AVX2-NEXT: movl %eax, 16(%esp,%ecx,4) ; X86AVX2-NEXT: vmovaps {{[0-9]+}}(%esp), %xmm0 ; X86AVX2-NEXT: leal -4(%ebp), %esp ; X86AVX2-NEXT: popl %esi @@ -1362,12 +1363,13 @@ define <2 x i64> @load_i64_v2i64(<2 x i64> %v, ptr %p, i32 %y) nounwind { ; X86AVX2-NEXT: movl (%ecx), %edx ; X86AVX2-NEXT: movl 4(%ecx), %ecx ; X86AVX2-NEXT: vmovaps %xmm0, (%esp) -; X86AVX2-NEXT: leal (%eax,%eax), %esi +; X86AVX2-NEXT: addl %eax, %eax +; X86AVX2-NEXT: movl %eax, %esi ; X86AVX2-NEXT: andl $3, %esi ; X86AVX2-NEXT: movl %edx, (%esp,%esi,4) ; X86AVX2-NEXT: vmovaps (%esp), %xmm0 ; X86AVX2-NEXT: vmovaps %xmm0, {{[0-9]+}}(%esp) -; X86AVX2-NEXT: leal 1(%eax,%eax), %eax +; X86AVX2-NEXT: incl %eax ; X86AVX2-NEXT: andl $3, %eax ; X86AVX2-NEXT: movl %ecx, 16(%esp,%eax,4) ; X86AVX2-NEXT: vmovaps {{[0-9]+}}(%esp), %xmm0 @@ -1742,18 +1744,19 @@ define <4 x i64> @arg_i64_v4i64(<4 x i64> %v, i64 %x, i32 %y) nounwind { ; X86AVX2-NEXT: pushl %esi ; X86AVX2-NEXT: andl $-32, %esp ; X86AVX2-NEXT: subl $96, %esp -; X86AVX2-NEXT: movl 8(%ebp), %eax -; X86AVX2-NEXT: movl 12(%ebp), %ecx -; X86AVX2-NEXT: movl 16(%ebp), %edx +; X86AVX2-NEXT: movl 8(%ebp), %edx +; X86AVX2-NEXT: movl 12(%ebp), %eax +; X86AVX2-NEXT: movl 16(%ebp), %ecx ; X86AVX2-NEXT: vmovaps %ymm0, (%esp) -; X86AVX2-NEXT: leal (%edx,%edx), %esi +; X86AVX2-NEXT: addl %ecx, %ecx +; X86AVX2-NEXT: movl %ecx, %esi ; X86AVX2-NEXT: andl $7, %esi -; X86AVX2-NEXT: movl %eax, (%esp,%esi,4) +; X86AVX2-NEXT: movl %edx, (%esp,%esi,4) ; X86AVX2-NEXT: vmovaps (%esp), %ymm0 ; X86AVX2-NEXT: vmovaps %ymm0, {{[0-9]+}}(%esp) -; X86AVX2-NEXT: leal 1(%edx,%edx), %eax -; X86AVX2-NEXT: andl $7, %eax -; X86AVX2-NEXT: movl %ecx, 32(%esp,%eax,4) +; X86AVX2-NEXT: incl %ecx +; X86AVX2-NEXT: andl $7, %ecx +; X86AVX2-NEXT: movl %eax, 32(%esp,%ecx,4) ; X86AVX2-NEXT: vmovaps {{[0-9]+}}(%esp), %ymm0 ; X86AVX2-NEXT: leal -4(%ebp), %esp ; X86AVX2-NEXT: popl %esi @@ -2128,12 +2131,13 @@ define <4 x i64> @load_i64_v4i64(<4 x i64> %v, ptr %p, i32 %y) nounwind { ; X86AVX2-NEXT: movl (%ecx), %edx ; X86AVX2-NEXT: movl 4(%ecx), %ecx ; X86AVX2-NEXT: vmovaps %ymm0, (%esp) -; X86AVX2-NEXT: leal (%eax,%eax), %esi +; X86AVX2-NEXT: addl %eax, %eax +; X86AVX2-NEXT: movl %eax, %esi ; X86AVX2-NEXT: andl $7, %esi ; X86AVX2-NEXT: movl %edx, (%esp,%esi,4) ; X86AVX2-NEXT: vmovaps (%esp), %ymm0 ; X86AVX2-NEXT: vmovaps %ymm0, {{[0-9]+}}(%esp) -; X86AVX2-NEXT: leal 1(%eax,%eax), %eax +; X86AVX2-NEXT: incl %eax ; X86AVX2-NEXT: andl $7, %eax ; X86AVX2-NEXT: movl %ecx, 32(%esp,%eax,4) ; X86AVX2-NEXT: vmovaps {{[0-9]+}}(%esp), %ymm0 diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll b/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll index c83911f60149..f8d13fc4237e 100644 --- a/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll +++ b/llvm/test/Instrumentation/HWAddressSanitizer/globals-access.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --global-value-regex "x" --version 4 -; RUN: opt < %s -S -passes=hwasan -mtriple=aarch64 -hwasan-globals=0 | FileCheck %s --check-prefixes=NOGLOB -; RUN: opt < %s -S -passes=hwasan -mtriple=aarch64 -hwasan-globals=1 | FileCheck %s +; RUN: opt < %s -S -passes=hwasan -mtriple=aarch64-linux-gnu -hwasan-globals=0 | FileCheck %s --check-prefixes=NOGLOB +; RUN: opt < %s -S -passes=hwasan -mtriple=aarch64-linux-gnu -hwasan-globals=1 | FileCheck %s + +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" @x = dso_local global i32 0, align 4 diff --git a/llvm/test/TableGen/ConcatenatedSubregs.td b/llvm/test/TableGen/ConcatenatedSubregs.td index 5b354c94dca5..ea4e7f01a2e2 100644 --- a/llvm/test/TableGen/ConcatenatedSubregs.td +++ b/llvm/test/TableGen/ConcatenatedSubregs.td @@ -90,16 +90,19 @@ def TestTarget : Target; // CHECK-LABEL: RegisterClass DRegs: // CHECK-LABEL: SubRegIndex ssub1: -// CHECK: Offset, Size: 16, 16 +// CHECK: Offset: { Default:16 } +// CHECK: Size: { Default:16 } // CHECK-LABEL: SubRegIndex sub0: // CHECK-LABEL: SubRegIndex sub1: // CHECK-LABEL: SubRegIndex sub2: // Check inferred indexes: // CHECK-LABEL: SubRegIndex ssub1_ssub2: -// CHECK: Offset, Size: 16, 65535 +// CHECK: Offset: { Default:16 } +// CHECK: Size: { Default:65535 } // CHECK-LABEL: SubRegIndex ssub3_ssub4: // CHECK-LABEL: SubRegIndex ssub0_ssub1_ssub2_ssub3: -// CHECK: Offset, Size: 65535, 65535 +// CHECK: Offset: { Default:65535 } +// CHECK: Size: { Default:65535 } // CHECK-LABEL: SubRegIndex ssub1_ssub2_ssub3_ssub4: // Check that all subregs are generated on some examples diff --git a/llvm/test/TableGen/HwModeSubRegs.td b/llvm/test/TableGen/HwModeSubRegs.td new file mode 100644 index 000000000000..2bf7a917979d --- /dev/null +++ b/llvm/test/TableGen/HwModeSubRegs.td @@ -0,0 +1,75 @@ +// RUN: llvm-tblgen -gen-register-info -register-info-debug -I %p/../../include %s -o /dev/null 2>&1 | FileCheck %s +include "llvm/Target/Target.td" + +def HasFeat : Predicate<"Subtarget->hasFeat()">; + +def TestMode : HwMode<"+feat1", [HasFeat]>; + +class MyReg<string n> + : Register<n> { + let Namespace = "Test"; +} +class MyClass<int size, list<ValueType> types, dag registers> + : RegisterClass<"Test", types, size, registers> { + let Size = size; +} + +def X0 : MyReg<"x0">; +def X1 : MyReg<"x1">; +def X2 : MyReg<"x2">; +def X3 : MyReg<"x3">; +def X4 : MyReg<"x4">; +def X5 : MyReg<"x5">; +def X6 : MyReg<"x6">; +def X7 : MyReg<"x7">; +def X8 : MyReg<"x8">; +def X9 : MyReg<"x9">; +def X10 : MyReg<"x10">; +def X11 : MyReg<"x11">; +def X12 : MyReg<"x12">; +def X13 : MyReg<"x13">; +def X14 : MyReg<"x14">; +def X15 : MyReg<"x15">; + +def ModeVT : ValueTypeByHwMode<[DefaultMode, TestMode], + [i32, i64]>; +let RegInfos = RegInfoByHwMode<[DefaultMode, TestMode], + [RegInfo<32,32,32>, RegInfo<64,64,64>]> in +def XRegs : MyClass<32, [ModeVT], (sequence "X%u", 0, 15)>; + +def sub_even : SubRegIndex<32> { + let SubRegRanges = SubRegRangeByHwMode<[DefaultMode, TestMode], + [SubRegRange<32>, SubRegRange<64>]>; +} +def sub_odd : SubRegIndex<32, 32> { + let SubRegRanges = SubRegRangeByHwMode<[DefaultMode, TestMode], + [SubRegRange<32, 32>, SubRegRange<64, 64>]>; +} + +def XPairs : RegisterTuples<[sub_even, sub_odd], + [(decimate (rotl XRegs, 0), 2), + (decimate (rotl XRegs, 1), 2)]>; + +let RegInfos = RegInfoByHwMode<[DefaultMode, TestMode], + [RegInfo<64,64,32>, RegInfo<128,128,64>]> in +def XPairsClass : MyClass<64, [untyped], (add XPairs)>; + +def TestTarget : Target; + +// CHECK-LABEL: RegisterClass XRegs: +// CHECK: SpillSize: { Default:32 TestMode:64 } +// CHECK: SpillAlignment: { Default:32 TestMode:64 } +// CHECK: Regs: X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + +// CHECK-LABEL: RegisterClass XPairsClass: +// CHECK: SpillSize: { Default:64 TestMode:128 } +// CHECK: SpillAlignment: { Default:32 TestMode:64 } +// CHECK: CoveredBySubRegs: 1 +// CHECK: Regs: X0_X1 X2_X3 X4_X5 X6_X7 X8_X9 X10_X11 X12_X13 X14_X15 + +// CHECK-LABEL: SubRegIndex sub_even: +// CHECK: Offset: { Default:0 TestMode:0 } +// CHECK: Size: { Default:32 TestMode:64 } +// CHECK-LABEL: SubRegIndex sub_odd: +// CHECK: Offset: { Default:32 TestMode:64 } +// CHECK: Size: { Default:32 TestMode:64 } diff --git a/llvm/test/Transforms/Inline/update_invoke_prof.ll b/llvm/test/Transforms/Inline/update_invoke_prof.ll new file mode 100644 index 000000000000..5f09c7cf8fe0 --- /dev/null +++ b/llvm/test/Transforms/Inline/update_invoke_prof.ll @@ -0,0 +1,64 @@ +; A pre-commit test to show that branch weights and value profiles associated with invoke are not updated. +; RUN: opt < %s -passes='require<profile-summary>,cgscc(inline)' -S | FileCheck %s + +declare i32 @__gxx_personality_v0(...) + +define void @caller(ptr %func) personality ptr @__gxx_personality_v0 !prof !15 { + call void @callee(ptr %func), !prof !16 + ret void +} + +declare void @inner_callee(ptr %func) + +define void @callee(ptr %func) personality ptr @__gxx_personality_v0 !prof !17 { + invoke void %func() + to label %next unwind label %lpad, !prof !18 + +next: + invoke void @inner_callee(ptr %func) + to label %ret unwind label %lpad, !prof !19 + +lpad: + %exn = landingpad {ptr, i32} + cleanup + unreachable + +ret: + ret void +} + +!llvm.module.flags = !{!1} +!1 = !{i32 1, !"ProfileSummary", !2} +!2 = !{!3, !4, !5, !6, !7, !8, !9, !10} +!3 = !{!"ProfileFormat", !"SampleProfile"} +!4 = !{!"TotalCount", i64 10000} +!5 = !{!"MaxCount", i64 10} +!6 = !{!"MaxInternalCount", i64 1} +!7 = !{!"MaxFunctionCount", i64 2000} +!8 = !{!"NumCounts", i64 2} +!9 = !{!"NumFunctions", i64 2} +!10 = !{!"DetailedSummary", !11} +!11 = !{!12, !13, !14} +!12 = !{i32 10000, i64 100, i32 1} +!13 = !{i32 999000, i64 100, i32 1} +!14 = !{i32 999999, i64 1, i32 2} +!15 = !{!"function_entry_count", i64 1000} +!16 = !{!"branch_weights", i64 1000} +!17 = !{!"function_entry_count", i32 1500} +!18 = !{!"VP", i32 0, i64 1500, i64 123, i64 900, i64 456, i64 600} +!19 = !{!"branch_weights", i32 1500} + +; CHECK-LABEL: @caller( +; CHECK: invoke void %func( +; CHECK-NEXT: {{.*}} !prof ![[PROF1:[0-9]+]] +; CHECK: invoke void @inner_callee( +; CHECK-NEXT: {{.*}} !prof ![[PROF2:[0-9]+]] + +; CHECK-LABL: @callee( +; CHECK: invoke void %func( +; CHECK-NEXT: {{.*}} !prof ![[PROF1]] +; CHECK: invoke void @inner_callee( +; CHECK-NEXT: {{.*}} !prof ![[PROF2]] + +; CHECK: ![[PROF1]] = !{!"VP", i32 0, i64 1500, i64 123, i64 900, i64 456, i64 600} +; CHECK: ![[PROF2]] = !{!"branch_weights", i32 1500} diff --git a/llvm/test/Transforms/Inline/update_value_profile.ll b/llvm/test/Transforms/Inline/update_value_profile.ll new file mode 100644 index 000000000000..daa95e93b68e --- /dev/null +++ b/llvm/test/Transforms/Inline/update_value_profile.ll @@ -0,0 +1,81 @@ +; RUN: opt < %s -passes='require<profile-summary>,cgscc(inline)' -inline-threshold=100 -S | FileCheck %s +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; When 'callee' is inlined into caller1 and caller2, the indirect call value +; profiles of the inlined copy should be scaled based on callers' profiles, +; and the indirect call value profiles in 'callee' should be updated. +define i32 @callee(ptr %0, i32 %1) !prof !20 { +; CHECK-LABEL: define i32 @callee( +; CHECK-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) !prof [[PROF0:![0-9]+]] { +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP0]], align 8 +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 8 +; CHECK-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP4]], align 8 +; CHECK-NEXT: [[TMP6:%.*]] = tail call i32 [[TMP5]](ptr [[TMP0]], i32 [[TMP1]]), !prof [[PROF1:![0-9]+]] +; CHECK-NEXT: ret i32 [[TMP6]] +; + %3 = load ptr, ptr %0 + %5 = getelementptr inbounds i8, ptr %3, i64 8 + %6 = load ptr, ptr %5 + %7 = tail call i32 %6(ptr %0, i32 %1), !prof !17 + ret i32 %7 +} + +define i32 @caller1(i32 %0) !prof !18 { +; CHECK-LABEL: define i32 @caller1( +; CHECK-SAME: i32 [[TMP0:%.*]]) !prof [[PROF2:![0-9]+]] { +; CHECK-NEXT: [[TMP2:%.*]] = tail call ptr @_Z10createTypei(i32 [[TMP0]]) +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8 +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 8 +; CHECK-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP4]], align 8 +; CHECK-NEXT: [[TMP6:%.*]] = tail call i32 [[TMP5]](ptr [[TMP2]], i32 [[TMP0]]), !prof [[PROF3:![0-9]+]] +; CHECK-NEXT: ret i32 [[TMP6]] +; + %2 = tail call ptr @_Z10createTypei(i32 %0) + %3 = tail call i32 @callee(ptr %2, i32 %0) + ret i32 %3 +} + +define i32 @caller2(i32 %0) !prof !19 { +; CHECK-LABEL: define i32 @caller2( +; CHECK-SAME: i32 [[TMP0:%.*]]) !prof [[PROF4:![0-9]+]] { +; CHECK-NEXT: [[TMP2:%.*]] = tail call ptr @_Z10createTypei(i32 [[TMP0]]) +; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8 +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 8 +; CHECK-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP4]], align 8 +; CHECK-NEXT: [[TMP6:%.*]] = tail call i32 [[TMP5]](ptr [[TMP2]], i32 [[TMP0]]), !prof [[PROF5:![0-9]+]] +; CHECK-NEXT: ret i32 [[TMP6]] +; + %2 = tail call ptr @_Z10createTypei(i32 %0) + %3 = tail call i32 @callee(ptr %2, i32 %0) + ret i32 %3 +} + +declare ptr @_Z10createTypei(i32) + +!1 = !{i32 1, !"ProfileSummary", !2} +!2 = !{!3, !4, !5, !6, !7, !8, !9, !10} +!3 = !{!"ProfileFormat", !"InstrProf"} +!4 = !{!"TotalCount", i64 10000} +!5 = !{!"MaxCount", i64 10} +!6 = !{!"MaxInternalCount", i64 1} +!7 = !{!"MaxFunctionCount", i64 1000} +!8 = !{!"NumCounts", i64 3} +!9 = !{!"NumFunctions", i64 3} +!10 = !{!"DetailedSummary", !11} +!11 = !{!12, !13, !14} +!12 = !{i32 10000, i64 100, i32 1} +!13 = !{i32 999000, i64 100, i32 1} +!14 = !{i32 999999, i64 1, i32 2} +!17 = !{!"VP", i32 0, i64 1600, i64 123, i64 1000, i64 456, i64 600} +!18 = !{!"function_entry_count", i64 1000} +!19 = !{!"function_entry_count", i64 600} +!20 = !{!"function_entry_count", i64 1700} +;. +; CHECK: [[PROF0]] = !{!"function_entry_count", i64 100} +; CHECK: [[PROF1]] = !{!"VP", i32 0, i64 94, i64 123, i64 58, i64 456, i64 35} +; CHECK: [[PROF2]] = !{!"function_entry_count", i64 1000} +; CHECK: [[PROF3]] = !{!"VP", i32 0, i64 941, i64 123, i64 588, i64 456, i64 352} +; CHECK: [[PROF4]] = !{!"function_entry_count", i64 600} +; CHECK: [[PROF5]] = !{!"VP", i32 0, i64 564, i64 123, i64 352, i64 456, i64 211} +;. diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll b/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll index 14acb6f57aa0..3f38abc75a58 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll @@ -29,7 +29,7 @@ define void @test(ptr noundef align 8 dereferenceable_or_null(16) %arr) #0 { ; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i64> [[VEC_IND]], <i64 1, i64 1, i64 1, i64 1> ; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i64> [[TMP2]], zeroinitializer ; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer -; CHECK-NEXT: [[TMP5:%.*]] = or i64 [[TMP0]], 1 +; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP0]], 1 ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[ARR]], i64 [[TMP5]] ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[TMP6]], i32 0 ; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[TMP7]], i32 -3 diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch-error.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch-error.ll new file mode 100644 index 000000000000..2bb8f677f40c --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch-error.ll @@ -0,0 +1,7 @@ +; REQUIRES: x86_64-linux +; RUN: not opt < %S/pseudo-probe-profile-mismatch.ll -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch.prof -min-functions-for-staleness-error=1 -precent-mismatch-for-staleness-error=1 -S 2>&1 | FileCheck %s +; RUN: opt < %S/pseudo-probe-profile-mismatch.ll -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch.prof -min-functions-for-staleness-error=3 -precent-mismatch-for-staleness-error=70 -S 2>&1 +; RUN: opt < %S/pseudo-probe-profile-mismatch.ll -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch.prof -min-functions-for-staleness-error=4 -precent-mismatch-for-staleness-error=1 -S 2>&1 + + +; CHECK: error: {{.*}}: The input profile significantly mismatches current source code. Please recollect profile to avoid performance regression. diff --git a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp index 1fd81bd407be..0a947f6e206f 100644 --- a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp +++ b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp @@ -63,6 +63,10 @@ Error SubprocessMemory::addMemoryDefinition( SharedMemoryNames.push_back(SharedMemoryName); int SharedMemoryFD = shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + if (SharedMemoryFD == -1) + return make_error<Failure>( + "Failed to create shared memory object for memory definition: " + + Twine(strerror(errno))); if (ftruncate(SharedMemoryFD, MemVal.SizeBytes) != 0) { return make_error<Failure>("Truncating a memory definiton failed: " + Twine(strerror(errno))); @@ -100,7 +104,8 @@ Expected<int> SubprocessMemory::setupAuxiliaryMemoryInSubprocess( shm_open(AuxiliaryMemoryName.c_str(), O_RDWR, S_IRUSR | S_IWUSR); if (AuxiliaryMemoryFileDescriptor == -1) return make_error<Failure>( - "Getting file descriptor for auxiliary memory failed"); + "Getting file descriptor for auxiliary memory failed: " + + Twine(strerror(errno))); // set up memory value file descriptors int *AuxiliaryMemoryMapping = (int *)mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, diff --git a/llvm/unittests/Object/GOFFObjectFileTest.cpp b/llvm/unittests/Object/GOFFObjectFileTest.cpp index 69f60d016a80..734dac6b8507 100644 --- a/llvm/unittests/Object/GOFFObjectFileTest.cpp +++ b/llvm/unittests/Object/GOFFObjectFileTest.cpp @@ -502,100 +502,3 @@ TEST(GOFFObjectFileTest, InvalidERSymbolType) { FailedWithMessage("ESD record 1 has unknown Executable type 0x03")); } } - -TEST(GOFFObjectFileTest, TXTConstruct) { - char GOFFData[GOFF::RecordLength * 6] = {}; - - // HDR record. - GOFFData[0] = 0x03; - GOFFData[1] = 0xF0; - GOFFData[50] = 0x01; - - // ESD record. - GOFFData[GOFF::RecordLength] = 0x03; - GOFFData[GOFF::RecordLength + 7] = 0x01; // ESDID. - GOFFData[GOFF::RecordLength + 71] = 0x05; // Size of symbol name. - GOFFData[GOFF::RecordLength + 72] = 0xa5; // Symbol name is v. - GOFFData[GOFF::RecordLength + 73] = 0x81; // Symbol name is a. - GOFFData[GOFF::RecordLength + 74] = 0x99; // Symbol name is r. - GOFFData[GOFF::RecordLength + 75] = 0x7b; // Symbol name is #. - GOFFData[GOFF::RecordLength + 76] = 0x83; // Symbol name is c. - - // ESD record. - GOFFData[GOFF::RecordLength * 2] = 0x03; - GOFFData[GOFF::RecordLength * 2 + 3] = 0x01; - GOFFData[GOFF::RecordLength * 2 + 7] = 0x02; // ESDID. - GOFFData[GOFF::RecordLength * 2 + 11] = 0x01; // Parent ESDID. - GOFFData[GOFF::RecordLength * 2 + 27] = 0x08; // Length. - GOFFData[GOFF::RecordLength * 2 + 40] = 0x01; // Name Space ID. - GOFFData[GOFF::RecordLength * 2 + 41] = 0x80; - GOFFData[GOFF::RecordLength * 2 + 60] = 0x04; // Size of symbol name. - GOFFData[GOFF::RecordLength * 2 + 61] = 0x04; // Size of symbol name. - GOFFData[GOFF::RecordLength * 2 + 63] = 0x0a; // Size of symbol name. - GOFFData[GOFF::RecordLength * 2 + 66] = 0x03; // Size of symbol name. - GOFFData[GOFF::RecordLength * 2 + 71] = 0x08; // Size of symbol name. - GOFFData[GOFF::RecordLength * 2 + 72] = 0xc3; // Symbol name is c. - GOFFData[GOFF::RecordLength * 2 + 73] = 0x6d; // Symbol name is _. - GOFFData[GOFF::RecordLength * 2 + 74] = 0xc3; // Symbol name is c. - GOFFData[GOFF::RecordLength * 2 + 75] = 0xd6; // Symbol name is o. - GOFFData[GOFF::RecordLength * 2 + 76] = 0xc4; // Symbol name is D. - GOFFData[GOFF::RecordLength * 2 + 77] = 0xc5; // Symbol name is E. - GOFFData[GOFF::RecordLength * 2 + 78] = 0xf6; // Symbol name is 6. - GOFFData[GOFF::RecordLength * 2 + 79] = 0xf4; // Symbol name is 4. - - // ESD record. - GOFFData[GOFF::RecordLength * 3] = 0x03; - GOFFData[GOFF::RecordLength * 3 + 3] = 0x02; - GOFFData[GOFF::RecordLength * 3 + 7] = 0x03; // ESDID. - GOFFData[GOFF::RecordLength * 3 + 11] = 0x02; // Parent ESDID. - GOFFData[GOFF::RecordLength * 3 + 71] = 0x05; // Size of symbol name. - GOFFData[GOFF::RecordLength * 3 + 72] = 0xa5; // Symbol name is v. - GOFFData[GOFF::RecordLength * 3 + 73] = 0x81; // Symbol name is a. - GOFFData[GOFF::RecordLength * 3 + 74] = 0x99; // Symbol name is r. - GOFFData[GOFF::RecordLength * 3 + 75] = 0x7b; // Symbol name is #. - GOFFData[GOFF::RecordLength * 3 + 76] = 0x83; // Symbol name is c. - - // TXT record. - GOFFData[GOFF::RecordLength * 4] = 0x03; - GOFFData[GOFF::RecordLength * 4 + 1] = 0x10; - GOFFData[GOFF::RecordLength * 4 + 7] = 0x02; - GOFFData[GOFF::RecordLength * 4 + 23] = 0x08; // Data Length. - GOFFData[GOFF::RecordLength * 4 + 24] = 0x12; - GOFFData[GOFF::RecordLength * 4 + 25] = 0x34; - GOFFData[GOFF::RecordLength * 4 + 26] = 0x56; - GOFFData[GOFF::RecordLength * 4 + 27] = 0x78; - GOFFData[GOFF::RecordLength * 4 + 28] = 0x9a; - GOFFData[GOFF::RecordLength * 4 + 29] = 0xbc; - GOFFData[GOFF::RecordLength * 4 + 30] = 0xde; - GOFFData[GOFF::RecordLength * 4 + 31] = 0xf0; - - // END record. - GOFFData[GOFF::RecordLength * 5] = 0x03; - GOFFData[GOFF::RecordLength * 5 + 1] = 0x40; - GOFFData[GOFF::RecordLength * 5 + 11] = 0x06; - - StringRef Data(GOFFData, GOFF::RecordLength * 6); - - Expected<std::unique_ptr<ObjectFile>> GOFFObjOrErr = - object::ObjectFile::createGOFFObjectFile( - MemoryBufferRef(Data, "dummyGOFF")); - - ASSERT_THAT_EXPECTED(GOFFObjOrErr, Succeeded()); - - GOFFObjectFile *GOFFObj = dyn_cast<GOFFObjectFile>((*GOFFObjOrErr).get()); - auto Symbols = GOFFObj->symbols(); - ASSERT_EQ(std::distance(Symbols.begin(), Symbols.end()), 1); - SymbolRef Symbol = *Symbols.begin(); - Expected<StringRef> SymbolNameOrErr = GOFFObj->getSymbolName(Symbol); - ASSERT_THAT_EXPECTED(SymbolNameOrErr, Succeeded()); - StringRef SymbolName = SymbolNameOrErr.get(); - EXPECT_EQ(SymbolName, "var#c"); - - auto Sections = GOFFObj->sections(); - ASSERT_EQ(std::distance(Sections.begin(), Sections.end()), 1); - SectionRef Section = *Sections.begin(); - Expected<StringRef> SectionContent = Section.getContents(); - ASSERT_THAT_EXPECTED(SectionContent, Succeeded()); - StringRef Contents = SectionContent.get(); - EXPECT_EQ(Contents, "\x12\x34\x56\x78\x9a\xbc\xde\xf0"); -} diff --git a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp index e851d4c16bf7..624e8d5d54ba 100644 --- a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp +++ b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp @@ -47,19 +47,24 @@ using namespace llvm; // CodeGenSubRegIndex //===----------------------------------------------------------------------===// -CodeGenSubRegIndex::CodeGenSubRegIndex(Record *R, unsigned Enum) +CodeGenSubRegIndex::CodeGenSubRegIndex(Record *R, unsigned Enum, + const CodeGenHwModes &CGH) : TheDef(R), EnumValue(Enum), AllSuperRegsCovered(true), Artificial(true) { Name = std::string(R->getName()); if (R->getValue("Namespace")) Namespace = std::string(R->getValueAsString("Namespace")); - Size = R->getValueAsInt("Size"); - Offset = R->getValueAsInt("Offset"); + + if (const RecordVal *RV = R->getValue("SubRegRanges")) + if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) + Range = SubRegRangeByHwMode(DI->getDef(), CGH); + if (!Range.hasDefault()) + Range.insertSubRegRangeForMode(DefaultMode, SubRegRange(R)); } CodeGenSubRegIndex::CodeGenSubRegIndex(StringRef N, StringRef Nspace, unsigned Enum) : TheDef(nullptr), Name(std::string(N)), Namespace(std::string(Nspace)), - Size(-1), Offset(-1), EnumValue(Enum), AllSuperRegsCovered(true), + Range(SubRegRange(-1, -1)), EnumValue(Enum), AllSuperRegsCovered(true), Artificial(true) {} std::string CodeGenSubRegIndex::getQualifiedName() const { @@ -81,7 +86,7 @@ void CodeGenSubRegIndex::updateComponents(CodeGenRegBank &RegBank) { "ComposedOf must have exactly two entries"); CodeGenSubRegIndex *A = RegBank.getSubRegIdx(Comps[0]); CodeGenSubRegIndex *B = RegBank.getSubRegIdx(Comps[1]); - CodeGenSubRegIndex *X = A->addComposite(B, this); + CodeGenSubRegIndex *X = A->addComposite(B, this, RegBank.getHwModes()); if (X) PrintFatalError(TheDef->getLoc(), "Ambiguous ComposedOf entries"); } @@ -518,7 +523,8 @@ void CodeGenRegister::computeSecondarySubRegs(CodeGenRegBank &RegBank) { // Each part of Cand is a sub-register of this. Make the full Cand also // a sub-register with a concatenated sub-register index. - CodeGenSubRegIndex *Concat = RegBank.getConcatSubRegIndex(Parts); + CodeGenSubRegIndex *Concat = + RegBank.getConcatSubRegIndex(Parts, RegBank.getHwModes()); std::pair<CodeGenSubRegIndex *, CodeGenRegister *> NewSubReg = std::pair(Concat, Cand); @@ -542,7 +548,7 @@ void CodeGenRegister::computeSecondarySubRegs(CodeGenRegBank &RegBank) { PrintFatalError(TheDef->getLoc(), "No SubRegIndex for " + SubReg.second->getName() + " in " + getName()); - NewIdx->addComposite(SubReg.first, SubIdx); + NewIdx->addComposite(SubReg.first, SubIdx, RegBank.getHwModes()); } } } @@ -1315,7 +1321,7 @@ CodeGenSubRegIndex *CodeGenRegBank::getSubRegIdx(Record *Def) { CodeGenSubRegIndex *&Idx = Def2SubRegIdx[Def]; if (Idx) return Idx; - SubRegIndices.emplace_back(Def, SubRegIndices.size() + 1); + SubRegIndices.emplace_back(Def, SubRegIndices.size() + 1, getHwModes()); Idx = &SubRegIndices.back(); return Idx; } @@ -1379,12 +1385,13 @@ CodeGenRegBank::getCompositeSubRegIndex(CodeGenSubRegIndex *A, // None exists, synthesize one. std::string Name = A->getName() + "_then_" + B->getName(); Comp = createSubRegIndex(Name, A->getNamespace()); - A->addComposite(B, Comp); + A->addComposite(B, Comp, getHwModes()); return Comp; } CodeGenSubRegIndex *CodeGenRegBank::getConcatSubRegIndex( - const SmallVector<CodeGenSubRegIndex *, 8> &Parts) { + const SmallVector<CodeGenSubRegIndex *, 8> &Parts, + const CodeGenHwModes &CGH) { assert(Parts.size() > 1 && "Need two parts to concatenate"); #ifndef NDEBUG for (CodeGenSubRegIndex *Idx : Parts) { @@ -1399,28 +1406,47 @@ CodeGenSubRegIndex *CodeGenRegBank::getConcatSubRegIndex( // None exists, synthesize one. std::string Name = Parts.front()->getName(); - // Determine whether all parts are contiguous. - bool IsContinuous = true; - unsigned Size = Parts.front()->Size; - unsigned LastOffset = Parts.front()->Offset; - unsigned LastSize = Parts.front()->Size; const unsigned UnknownSize = (uint16_t)-1; + for (unsigned i = 1, e = Parts.size(); i != e; ++i) { Name += '_'; Name += Parts[i]->getName(); - if (Size == UnknownSize || Parts[i]->Size == UnknownSize) - Size = UnknownSize; - else - Size += Parts[i]->Size; - if (LastSize == UnknownSize || Parts[i]->Offset != (LastOffset + LastSize)) - IsContinuous = false; - LastOffset = Parts[i]->Offset; - LastSize = Parts[i]->Size; } + Idx = createSubRegIndex(Name, Parts.front()->getNamespace()); - Idx->Size = Size; - Idx->Offset = IsContinuous ? Parts.front()->Offset : -1; Idx->ConcatenationOf.assign(Parts.begin(), Parts.end()); + + unsigned NumModes = CGH.getNumModeIds(); + for (unsigned M = 0; M < NumModes; ++M) { + const CodeGenSubRegIndex *Part = Parts.front(); + + // Determine whether all parts are contiguous. + bool IsContinuous = true; + const SubRegRange &FirstPartRange = Part->Range.get(M); + unsigned Size = FirstPartRange.Size; + unsigned LastOffset = FirstPartRange.Offset; + unsigned LastSize = FirstPartRange.Size; + + for (unsigned i = 1, e = Parts.size(); i != e; ++i) { + Part = Parts[i]; + Name += '_'; + Name += Part->getName(); + + const SubRegRange &PartRange = Part->Range.get(M); + if (Size == UnknownSize || PartRange.Size == UnknownSize) + Size = UnknownSize; + else + Size += PartRange.Size; + if (LastSize == UnknownSize || + PartRange.Offset != (LastOffset + LastSize)) + IsContinuous = false; + LastOffset = PartRange.Offset; + LastSize = PartRange.Size; + } + unsigned Offset = IsContinuous ? FirstPartRange.Offset : -1; + Idx->Range.get(M) = SubRegRange(Size, Offset); + } + return Idx; } @@ -1504,7 +1530,8 @@ void CodeGenRegBank::computeComposites() { assert(Idx3 && "Sub-register doesn't have an index"); // Conflicting composition? Emit a warning but allow it. - if (CodeGenSubRegIndex *Prev = Idx1->addComposite(Idx2, Idx3)) { + if (CodeGenSubRegIndex *Prev = + Idx1->addComposite(Idx2, Idx3, getHwModes())) { // If the composition was not user-defined, always emit a warning. if (!UserDefined.count({Idx1, Idx2}) || agree(compose(Idx1, Idx2), SubRegAction.at(Idx3))) diff --git a/llvm/utils/TableGen/Common/CodeGenRegisters.h b/llvm/utils/TableGen/Common/CodeGenRegisters.h index c34f376ea99d..9058baea2b23 100644 --- a/llvm/utils/TableGen/Common/CodeGenRegisters.h +++ b/llvm/utils/TableGen/Common/CodeGenRegisters.h @@ -68,8 +68,7 @@ class CodeGenSubRegIndex { std::string Namespace; public: - uint16_t Size; - uint16_t Offset; + SubRegRangeByHwMode Range; const unsigned EnumValue; mutable LaneBitmask LaneMask; mutable SmallVector<MaskRolPair, 1> CompositionLaneMaskTransform; @@ -86,7 +85,7 @@ public: // indexes are not used to create new register classes. bool Artificial; - CodeGenSubRegIndex(Record *R, unsigned Enum); + CodeGenSubRegIndex(Record *R, unsigned Enum, const CodeGenHwModes &CGH); CodeGenSubRegIndex(StringRef N, StringRef Nspace, unsigned Enum); CodeGenSubRegIndex(CodeGenSubRegIndex &) = delete; @@ -108,19 +107,42 @@ public: // Add a composite subreg index: this+A = B. // Return a conflicting composite, or NULL - CodeGenSubRegIndex *addComposite(CodeGenSubRegIndex *A, - CodeGenSubRegIndex *B) { + CodeGenSubRegIndex *addComposite(CodeGenSubRegIndex *A, CodeGenSubRegIndex *B, + const CodeGenHwModes &CGH) { assert(A && B); std::pair<CompMap::iterator, bool> Ins = Composed.insert(std::pair(A, B)); + // Synthetic subreg indices that aren't contiguous (for instance ARM // register tuples) don't have a bit range, so it's OK to let // B->Offset == -1. For the other cases, accumulate the offset and set // the size here. Only do so if there is no offset yet though. - if ((Offset != (uint16_t)-1 && A->Offset != (uint16_t)-1) && - (B->Offset == (uint16_t)-1)) { - B->Offset = Offset + A->Offset; - B->Size = A->Size; + unsigned NumModes = CGH.getNumModeIds(); + // Skip default mode. + for (unsigned M = 0; M < NumModes; ++M) { + // Handle DefaultMode last. + if (M == DefaultMode) + continue; + SubRegRange &Range = this->Range.get(M); + SubRegRange &ARange = A->Range.get(M); + SubRegRange &BRange = B->Range.get(M); + + if (Range.Offset != (uint16_t)-1 && ARange.Offset != (uint16_t)-1 && + BRange.Offset == (uint16_t)-1) { + BRange.Offset = Range.Offset + ARange.Offset; + BRange.Size = ARange.Size; + } + } + + // Now handle default. + SubRegRange &Range = this->Range.get(DefaultMode); + SubRegRange &ARange = A->Range.get(DefaultMode); + SubRegRange &BRange = B->Range.get(DefaultMode); + if (Range.Offset != (uint16_t)-1 && ARange.Offset != (uint16_t)-1 && + BRange.Offset == (uint16_t)-1) { + BRange.Offset = Range.Offset + ARange.Offset; + BRange.Size = ARange.Size; } + return (Ins.second || Ins.first->second == B) ? nullptr : Ins.first->second; } @@ -681,7 +703,8 @@ public: // Find or create a sub-register index representing the concatenation of // non-overlapping sibling indices. CodeGenSubRegIndex * - getConcatSubRegIndex(const SmallVector<CodeGenSubRegIndex *, 8> &); + getConcatSubRegIndex(const SmallVector<CodeGenSubRegIndex *, 8> &, + const CodeGenHwModes &CGH); const std::deque<CodeGenRegister> &getRegisters() const { return Registers; } diff --git a/llvm/utils/TableGen/Common/InfoByHwMode.cpp b/llvm/utils/TableGen/Common/InfoByHwMode.cpp index 5496408bb0d3..cacf4ece6671 100644 --- a/llvm/utils/TableGen/Common/InfoByHwMode.cpp +++ b/llvm/utils/TableGen/Common/InfoByHwMode.cpp @@ -183,6 +183,20 @@ void RegSizeInfoByHwMode::writeToStream(raw_ostream &OS) const { OS << '}'; } +SubRegRange::SubRegRange(Record *R) { + Size = R->getValueAsInt("Size"); + Offset = R->getValueAsInt("Offset"); +} + +SubRegRangeByHwMode::SubRegRangeByHwMode(Record *R, const CodeGenHwModes &CGH) { + const HwModeSelect &MS = CGH.getHwModeSelect(R); + for (const HwModeSelect::PairType &P : MS.Items) { + auto I = Map.insert({P.first, SubRegRange(P.second)}); + assert(I.second && "Duplicate entry?"); + (void)I; + } +} + EncodingInfoByHwMode::EncodingInfoByHwMode(Record *R, const CodeGenHwModes &CGH) { const HwModeSelect &MS = CGH.getHwModeSelect(R); diff --git a/llvm/utils/TableGen/Common/InfoByHwMode.h b/llvm/utils/TableGen/Common/InfoByHwMode.h index 1909913c50c6..dd0b9830d757 100644 --- a/llvm/utils/TableGen/Common/InfoByHwMode.h +++ b/llvm/utils/TableGen/Common/InfoByHwMode.h @@ -176,6 +176,8 @@ struct ValueTypeByHwMode : public InfoByHwMode<MVT> { ValueTypeByHwMode getValueTypeByHwMode(Record *Rec, const CodeGenHwModes &CGH); +raw_ostream &operator<<(raw_ostream &OS, const ValueTypeByHwMode &T); + struct RegSizeInfo { unsigned RegSize; unsigned SpillSize; @@ -213,10 +215,27 @@ struct RegSizeInfoByHwMode : public InfoByHwMode<RegSizeInfo> { } }; -raw_ostream &operator<<(raw_ostream &OS, const ValueTypeByHwMode &T); raw_ostream &operator<<(raw_ostream &OS, const RegSizeInfo &T); raw_ostream &operator<<(raw_ostream &OS, const RegSizeInfoByHwMode &T); +struct SubRegRange { + uint16_t Size; + uint16_t Offset; + + SubRegRange(Record *R); + SubRegRange(uint16_t Size, uint16_t Offset) : Size(Size), Offset(Offset) {} +}; + +struct SubRegRangeByHwMode : public InfoByHwMode<SubRegRange> { + SubRegRangeByHwMode(Record *R, const CodeGenHwModes &CGH); + SubRegRangeByHwMode(SubRegRange Range) { Map.insert({DefaultMode, Range}); } + SubRegRangeByHwMode() = default; + + void insertSubRegRangeForMode(unsigned Mode, SubRegRange Info) { + Map.insert(std::pair(Mode, Info)); + } +}; + struct EncodingInfoByHwMode : public InfoByHwMode<Record *> { EncodingInfoByHwMode(Record *R, const CodeGenHwModes &CGH); EncodingInfoByHwMode() = default; diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp index a1259bff6ba8..ee8830edeedb 100644 --- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp +++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp @@ -1245,10 +1245,13 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, // Emit the table of sub-register index sizes. OS << "static const TargetRegisterInfo::SubRegCoveredBits " "SubRegIdxRangeTable[] = {\n"; - OS << " { " << (uint16_t)-1 << ", " << (uint16_t)-1 << " },\n"; - for (const auto &Idx : SubRegIndices) { - OS << " { " << Idx.Offset << ", " << Idx.Size << " },\t// " - << Idx.getName() << "\n"; + for (unsigned M = 0; M < NumModes; ++M) { + OS << " { " << (uint16_t)-1 << ", " << (uint16_t)-1 << " },\n"; + for (const auto &Idx : SubRegIndices) { + const SubRegRange &Range = Idx.Range.get(M); + OS << " { " << Range.Offset << ", " << Range.Size << " },\t// " + << Idx.getName() << "\n"; + } } OS << "};\n\n"; @@ -1864,7 +1867,13 @@ void RegisterInfoEmitter::debugDump(raw_ostream &OS) { OS << "SubRegIndex " << SRI.getName() << ":\n"; OS << "\tLaneMask: " << PrintLaneMask(SRI.LaneMask) << '\n'; OS << "\tAllSuperRegsCovered: " << SRI.AllSuperRegsCovered << '\n'; - OS << "\tOffset, Size: " << SRI.Offset << ", " << SRI.Size << '\n'; + OS << "\tOffset: {"; + for (unsigned M = 0; M != NumModes; ++M) + OS << ' ' << getModeName(M) << ':' << SRI.Range.get(M).Offset; + OS << " }\n\tSize: {"; + for (unsigned M = 0; M != NumModes; ++M) + OS << ' ' << getModeName(M) << ':' << SRI.Range.get(M).Size; + OS << " }\n"; } for (const CodeGenRegister &R : RegBank.getRegisters()) { diff --git a/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h b/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h index b16b081e5eff..79e8464bfda5 100644 --- a/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h +++ b/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h @@ -1065,6 +1065,132 @@ protected: return (DeviceId >= 0 && DeviceId < getNumDevices()); } +public: + // TODO: This plugin interface needs to be cleaned up. + + /// Returns non-zero if the provided \p Image can be executed by the runtime. + int32_t is_valid_binary(__tgt_device_image *Image); + + /// Initialize the device inside of the plugin. + int32_t init_device(int32_t DeviceId); + + /// Return the number of devices this plugin can support. + int32_t number_of_devices(); + + /// Initializes the OpenMP register requires information. + int64_t init_requires(int64_t RequiresFlags); + + /// Returns non-zero if the data can be exchanged between the two devices. + int32_t is_data_exchangable(int32_t SrcDeviceId, int32_t DstDeviceId); + + /// Initializes the record and replay mechanism inside the plugin. + int32_t initialize_record_replay(int32_t DeviceId, int64_t MemorySize, + void *VAddr, bool isRecord, bool SaveOutput, + uint64_t &ReqPtrArgOffset); + + /// Loads the associated binary into the plugin and returns a handle to it. + int32_t load_binary(int32_t DeviceId, __tgt_device_image *TgtImage, + __tgt_device_binary *Binary); + + /// Allocates memory that is accessively to the given device. + void *data_alloc(int32_t DeviceId, int64_t Size, void *HostPtr, int32_t Kind); + + /// Deallocates memory on the given device. + int32_t data_delete(int32_t DeviceId, void *TgtPtr, int32_t Kind); + + /// Locks / pins host memory using the plugin runtime. + int32_t data_lock(int32_t DeviceId, void *Ptr, int64_t Size, + void **LockedPtr); + + /// Unlocks / unpins host memory using the plugin runtime. + int32_t data_unlock(int32_t DeviceId, void *Ptr); + + /// Notify the runtime about a new mapping that has been created outside. + int32_t data_notify_mapped(int32_t DeviceId, void *HstPtr, int64_t Size); + + /// Notify t he runtime about a mapping that has been deleted. + int32_t data_notify_unmapped(int32_t DeviceId, void *HstPtr); + + /// Copy data to the given device. + int32_t data_submit(int32_t DeviceId, void *TgtPtr, void *HstPtr, + int64_t Size); + + /// Copy data to the given device asynchronously. + int32_t data_submit_async(int32_t DeviceId, void *TgtPtr, void *HstPtr, + int64_t Size, __tgt_async_info *AsyncInfoPtr); + + /// Copy data from the given device. + int32_t data_retrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr, + int64_t Size); + + /// Copy data from the given device asynchornously. + int32_t data_retrieve_async(int32_t DeviceId, void *HstPtr, void *TgtPtr, + int64_t Size, __tgt_async_info *AsyncInfoPtr); + + /// Exchange memory addresses between two devices. + int32_t data_exchange(int32_t SrcDeviceId, void *SrcPtr, int32_t DstDeviceId, + void *DstPtr, int64_t Size); + + /// Exchange memory addresses between two devices asynchronously. + int32_t data_exchange_async(int32_t SrcDeviceId, void *SrcPtr, + int DstDeviceId, void *DstPtr, int64_t Size, + __tgt_async_info *AsyncInfo); + + /// Begin executing a kernel on the given device. + int32_t launch_kernel(int32_t DeviceId, void *TgtEntryPtr, void **TgtArgs, + ptrdiff_t *TgtOffsets, KernelArgsTy *KernelArgs, + __tgt_async_info *AsyncInfoPtr); + + /// Synchronize an asyncrhonous queue with the plugin runtime. + int32_t synchronize(int32_t DeviceId, __tgt_async_info *AsyncInfoPtr); + + /// Query the current state of an asynchronous queue. + int32_t query_async(int32_t DeviceId, __tgt_async_info *AsyncInfoPtr); + + /// Prints information about the given devices supported by the plugin. + void print_device_info(int32_t DeviceId); + + /// Creates an event in the given plugin if supported. + int32_t create_event(int32_t DeviceId, void **EventPtr); + + /// Records an event that has occurred. + int32_t record_event(int32_t DeviceId, void *EventPtr, + __tgt_async_info *AsyncInfoPtr); + + /// Wait until an event has occurred. + int32_t wait_event(int32_t DeviceId, void *EventPtr, + __tgt_async_info *AsyncInfoPtr); + + /// Syncrhonize execution until an event is done. + int32_t sync_event(int32_t DeviceId, void *EventPtr); + + /// Remove the event from the plugin. + int32_t destroy_event(int32_t DeviceId, void *EventPtr); + + /// Remove the event from the plugin. + void set_info_flag(uint32_t NewInfoLevel); + + /// Creates an asynchronous queue for the given plugin. + int32_t init_async_info(int32_t DeviceId, __tgt_async_info **AsyncInfoPtr); + + /// Creates device information to be used for diagnostics. + int32_t init_device_info(int32_t DeviceId, __tgt_device_info *DeviceInfo, + const char **ErrStr); + + /// Sets the offset into the devices for use by OMPT. + int32_t set_device_offset(int32_t DeviceIdOffset); + + /// Returns if the plugin can support auotmatic copy. + int32_t use_auto_zero_copy(int32_t DeviceId); + + /// Look up a global symbol in the given binary. + int32_t get_global(__tgt_device_binary Binary, uint64_t Size, + const char *Name, void **DevicePtr); + + /// Look up a kernel function in the given binary. + int32_t get_function(__tgt_device_binary Binary, const char *Name, + void **KernelPtr); + private: /// Number of devices available for the plugin. int32_t NumDevices = 0; diff --git a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp index c32bd089d8a9..a4e6c9319215 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp @@ -1566,36 +1566,7 @@ Expected<bool> GenericPluginTy::checkELFImage(StringRef Image) const { return isELFCompatible(Image); } -bool llvm::omp::target::plugin::libomptargetSupportsRPC() { -#ifdef LIBOMPTARGET_RPC_SUPPORT - return true; -#else - return false; -#endif -} - -/// Exposed library API function, basically wrappers around the GenericDeviceTy -/// functionality with the same name. All non-async functions are redirected -/// to the async versions right away with a NULL AsyncInfoPtr. -#ifdef __cplusplus -extern "C" { -#endif - -int32_t __tgt_rtl_init_plugin() { - auto Err = PluginTy::initIfNeeded(); - if (Err) { - [[maybe_unused]] std::string ErrStr = toString(std::move(Err)); - DP("Failed to init plugin: %s", ErrStr.c_str()); - return OFFLOAD_FAIL; - } - - return OFFLOAD_SUCCESS; -} - -int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image) { - if (!PluginTy::isActive()) - return false; - +int32_t GenericPluginTy::is_valid_binary(__tgt_device_image *Image) { StringRef Buffer(reinterpret_cast<const char *>(Image->ImageStart), target::getPtrDiff(Image->ImageEnd, Image->ImageStart)); @@ -1610,13 +1581,13 @@ int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image) { case file_magic::elf_executable: case file_magic::elf_shared_object: case file_magic::elf_core: { - auto MatchOrErr = PluginTy::get().checkELFImage(Buffer); + auto MatchOrErr = checkELFImage(Buffer); if (Error Err = MatchOrErr.takeError()) return HandleError(std::move(Err)); return *MatchOrErr; } case file_magic::bitcode: { - auto MatchOrErr = PluginTy::get().getJIT().checkBitcodeImage(Buffer); + auto MatchOrErr = getJIT().checkBitcodeImage(Buffer); if (Error Err = MatchOrErr.takeError()) return HandleError(std::move(Err)); return *MatchOrErr; @@ -1626,8 +1597,8 @@ int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image) { } } -int32_t __tgt_rtl_init_device(int32_t DeviceId) { - auto Err = PluginTy::get().initDevice(DeviceId); +int32_t GenericPluginTy::init_device(int32_t DeviceId) { + auto Err = initDevice(DeviceId); if (Err) { REPORT("Failure to initialize device %d: %s\n", DeviceId, toString(std::move(Err)).data()); @@ -1637,26 +1608,24 @@ int32_t __tgt_rtl_init_device(int32_t DeviceId) { return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_number_of_devices() { - return PluginTy::get().getNumDevices(); -} +int32_t GenericPluginTy::number_of_devices() { return getNumDevices(); } -int64_t __tgt_rtl_init_requires(int64_t RequiresFlags) { - PluginTy::get().setRequiresFlag(RequiresFlags); +int64_t GenericPluginTy::init_requires(int64_t RequiresFlags) { + setRequiresFlag(RequiresFlags); return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_is_data_exchangable(int32_t SrcDeviceId, - int32_t DstDeviceId) { - return PluginTy::get().isDataExchangable(SrcDeviceId, DstDeviceId); +int32_t GenericPluginTy::is_data_exchangable(int32_t SrcDeviceId, + int32_t DstDeviceId) { + return isDataExchangable(SrcDeviceId, DstDeviceId); } -int32_t __tgt_rtl_initialize_record_replay(int32_t DeviceId, int64_t MemorySize, - void *VAddr, bool isRecord, - bool SaveOutput, - uint64_t &ReqPtrArgOffset) { - GenericPluginTy &Plugin = PluginTy::get(); - GenericDeviceTy &Device = Plugin.getDevice(DeviceId); +int32_t GenericPluginTy::initialize_record_replay(int32_t DeviceId, + int64_t MemorySize, + void *VAddr, bool isRecord, + bool SaveOutput, + uint64_t &ReqPtrArgOffset) { + GenericDeviceTy &Device = getDevice(DeviceId); RecordReplayTy::RRStatusTy Status = isRecord ? RecordReplayTy::RRStatusTy::RRRecording : RecordReplayTy::RRStatusTy::RRReplaying; @@ -1675,12 +1644,12 @@ int32_t __tgt_rtl_initialize_record_replay(int32_t DeviceId, int64_t MemorySize, return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_load_binary(int32_t DeviceId, __tgt_device_image *TgtImage, - __tgt_device_binary *Binary) { - GenericPluginTy &Plugin = PluginTy::get(); - GenericDeviceTy &Device = Plugin.getDevice(DeviceId); +int32_t GenericPluginTy::load_binary(int32_t DeviceId, + __tgt_device_image *TgtImage, + __tgt_device_binary *Binary) { + GenericDeviceTy &Device = getDevice(DeviceId); - auto ImageOrErr = Device.loadBinary(Plugin, TgtImage); + auto ImageOrErr = Device.loadBinary(*this, TgtImage); if (!ImageOrErr) { auto Err = ImageOrErr.takeError(); REPORT("Failure to load binary image %p on device %d: %s\n", TgtImage, @@ -1696,10 +1665,10 @@ int32_t __tgt_rtl_load_binary(int32_t DeviceId, __tgt_device_image *TgtImage, return OFFLOAD_SUCCESS; } -void *__tgt_rtl_data_alloc(int32_t DeviceId, int64_t Size, void *HostPtr, - int32_t Kind) { - auto AllocOrErr = PluginTy::get().getDevice(DeviceId).dataAlloc( - Size, HostPtr, (TargetAllocTy)Kind); +void *GenericPluginTy::data_alloc(int32_t DeviceId, int64_t Size, void *HostPtr, + int32_t Kind) { + auto AllocOrErr = + getDevice(DeviceId).dataAlloc(Size, HostPtr, (TargetAllocTy)Kind); if (!AllocOrErr) { auto Err = AllocOrErr.takeError(); REPORT("Failure to allocate device memory: %s\n", @@ -1711,9 +1680,10 @@ void *__tgt_rtl_data_alloc(int32_t DeviceId, int64_t Size, void *HostPtr, return *AllocOrErr; } -int32_t __tgt_rtl_data_delete(int32_t DeviceId, void *TgtPtr, int32_t Kind) { - auto Err = PluginTy::get().getDevice(DeviceId).dataDelete( - TgtPtr, (TargetAllocTy)Kind); +int32_t GenericPluginTy::data_delete(int32_t DeviceId, void *TgtPtr, + int32_t Kind) { + auto Err = + getDevice(DeviceId).dataDelete(TgtPtr, static_cast<TargetAllocTy>(Kind)); if (Err) { REPORT("Failure to deallocate device pointer %p: %s\n", TgtPtr, toString(std::move(Err)).data()); @@ -1723,9 +1693,9 @@ int32_t __tgt_rtl_data_delete(int32_t DeviceId, void *TgtPtr, int32_t Kind) { return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_data_lock(int32_t DeviceId, void *Ptr, int64_t Size, - void **LockedPtr) { - auto LockedPtrOrErr = PluginTy::get().getDevice(DeviceId).dataLock(Ptr, Size); +int32_t GenericPluginTy::data_lock(int32_t DeviceId, void *Ptr, int64_t Size, + void **LockedPtr) { + auto LockedPtrOrErr = getDevice(DeviceId).dataLock(Ptr, Size); if (!LockedPtrOrErr) { auto Err = LockedPtrOrErr.takeError(); REPORT("Failure to lock memory %p: %s\n", Ptr, @@ -1742,8 +1712,8 @@ int32_t __tgt_rtl_data_lock(int32_t DeviceId, void *Ptr, int64_t Size, return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_data_unlock(int32_t DeviceId, void *Ptr) { - auto Err = PluginTy::get().getDevice(DeviceId).dataUnlock(Ptr); +int32_t GenericPluginTy::data_unlock(int32_t DeviceId, void *Ptr) { + auto Err = getDevice(DeviceId).dataUnlock(Ptr); if (Err) { REPORT("Failure to unlock memory %p: %s\n", Ptr, toString(std::move(Err)).data()); @@ -1753,9 +1723,9 @@ int32_t __tgt_rtl_data_unlock(int32_t DeviceId, void *Ptr) { return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_data_notify_mapped(int32_t DeviceId, void *HstPtr, - int64_t Size) { - auto Err = PluginTy::get().getDevice(DeviceId).notifyDataMapped(HstPtr, Size); +int32_t GenericPluginTy::data_notify_mapped(int32_t DeviceId, void *HstPtr, + int64_t Size) { + auto Err = getDevice(DeviceId).notifyDataMapped(HstPtr, Size); if (Err) { REPORT("Failure to notify data mapped %p: %s\n", HstPtr, toString(std::move(Err)).data()); @@ -1765,8 +1735,8 @@ int32_t __tgt_rtl_data_notify_mapped(int32_t DeviceId, void *HstPtr, return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_data_notify_unmapped(int32_t DeviceId, void *HstPtr) { - auto Err = PluginTy::get().getDevice(DeviceId).notifyDataUnmapped(HstPtr); +int32_t GenericPluginTy::data_notify_unmapped(int32_t DeviceId, void *HstPtr) { + auto Err = getDevice(DeviceId).notifyDataUnmapped(HstPtr); if (Err) { REPORT("Failure to notify data unmapped %p: %s\n", HstPtr, toString(std::move(Err)).data()); @@ -1776,17 +1746,16 @@ int32_t __tgt_rtl_data_notify_unmapped(int32_t DeviceId, void *HstPtr) { return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_data_submit(int32_t DeviceId, void *TgtPtr, void *HstPtr, - int64_t Size) { - return __tgt_rtl_data_submit_async(DeviceId, TgtPtr, HstPtr, Size, - /*AsyncInfoPtr=*/nullptr); +int32_t GenericPluginTy::data_submit(int32_t DeviceId, void *TgtPtr, + void *HstPtr, int64_t Size) { + return data_submit_async(DeviceId, TgtPtr, HstPtr, Size, + /*AsyncInfoPtr=*/nullptr); } -int32_t __tgt_rtl_data_submit_async(int32_t DeviceId, void *TgtPtr, - void *HstPtr, int64_t Size, - __tgt_async_info *AsyncInfoPtr) { - auto Err = PluginTy::get().getDevice(DeviceId).dataSubmit(TgtPtr, HstPtr, - Size, AsyncInfoPtr); +int32_t GenericPluginTy::data_submit_async(int32_t DeviceId, void *TgtPtr, + void *HstPtr, int64_t Size, + __tgt_async_info *AsyncInfoPtr) { + auto Err = getDevice(DeviceId).dataSubmit(TgtPtr, HstPtr, Size, AsyncInfoPtr); if (Err) { REPORT("Failure to copy data from host to device. Pointers: host " "= " DPxMOD ", device = " DPxMOD ", size = %" PRId64 ": %s\n", @@ -1798,17 +1767,17 @@ int32_t __tgt_rtl_data_submit_async(int32_t DeviceId, void *TgtPtr, return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_data_retrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr, - int64_t Size) { - return __tgt_rtl_data_retrieve_async(DeviceId, HstPtr, TgtPtr, Size, - /*AsyncInfoPtr=*/nullptr); +int32_t GenericPluginTy::data_retrieve(int32_t DeviceId, void *HstPtr, + void *TgtPtr, int64_t Size) { + return data_retrieve_async(DeviceId, HstPtr, TgtPtr, Size, + /*AsyncInfoPtr=*/nullptr); } -int32_t __tgt_rtl_data_retrieve_async(int32_t DeviceId, void *HstPtr, - void *TgtPtr, int64_t Size, - __tgt_async_info *AsyncInfoPtr) { - auto Err = PluginTy::get().getDevice(DeviceId).dataRetrieve( - HstPtr, TgtPtr, Size, AsyncInfoPtr); +int32_t GenericPluginTy::data_retrieve_async(int32_t DeviceId, void *HstPtr, + void *TgtPtr, int64_t Size, + __tgt_async_info *AsyncInfoPtr) { + auto Err = + getDevice(DeviceId).dataRetrieve(HstPtr, TgtPtr, Size, AsyncInfoPtr); if (Err) { REPORT("Faliure to copy data from device to host. Pointers: host " "= " DPxMOD ", device = " DPxMOD ", size = %" PRId64 ": %s\n", @@ -1820,20 +1789,19 @@ int32_t __tgt_rtl_data_retrieve_async(int32_t DeviceId, void *HstPtr, return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_data_exchange(int32_t SrcDeviceId, void *SrcPtr, - int32_t DstDeviceId, void *DstPtr, - int64_t Size) { - return __tgt_rtl_data_exchange_async(SrcDeviceId, SrcPtr, DstDeviceId, DstPtr, - Size, - /*AsyncInfoPtr=*/nullptr); +int32_t GenericPluginTy::data_exchange(int32_t SrcDeviceId, void *SrcPtr, + int32_t DstDeviceId, void *DstPtr, + int64_t Size) { + return data_exchange_async(SrcDeviceId, SrcPtr, DstDeviceId, DstPtr, Size, + /*AsyncInfoPtr=*/nullptr); } -int32_t __tgt_rtl_data_exchange_async(int32_t SrcDeviceId, void *SrcPtr, - int DstDeviceId, void *DstPtr, - int64_t Size, - __tgt_async_info *AsyncInfo) { - GenericDeviceTy &SrcDevice = PluginTy::get().getDevice(SrcDeviceId); - GenericDeviceTy &DstDevice = PluginTy::get().getDevice(DstDeviceId); +int32_t GenericPluginTy::data_exchange_async(int32_t SrcDeviceId, void *SrcPtr, + int DstDeviceId, void *DstPtr, + int64_t Size, + __tgt_async_info *AsyncInfo) { + GenericDeviceTy &SrcDevice = getDevice(SrcDeviceId); + GenericDeviceTy &DstDevice = getDevice(DstDeviceId); auto Err = SrcDevice.dataExchange(SrcPtr, DstDevice, DstPtr, Size, AsyncInfo); if (Err) { REPORT("Failure to copy data from device (%d) to device (%d). Pointers: " @@ -1846,12 +1814,12 @@ int32_t __tgt_rtl_data_exchange_async(int32_t SrcDeviceId, void *SrcPtr, return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_launch_kernel(int32_t DeviceId, void *TgtEntryPtr, - void **TgtArgs, ptrdiff_t *TgtOffsets, - KernelArgsTy *KernelArgs, - __tgt_async_info *AsyncInfoPtr) { - auto Err = PluginTy::get().getDevice(DeviceId).launchKernel( - TgtEntryPtr, TgtArgs, TgtOffsets, *KernelArgs, AsyncInfoPtr); +int32_t GenericPluginTy::launch_kernel(int32_t DeviceId, void *TgtEntryPtr, + void **TgtArgs, ptrdiff_t *TgtOffsets, + KernelArgsTy *KernelArgs, + __tgt_async_info *AsyncInfoPtr) { + auto Err = getDevice(DeviceId).launchKernel(TgtEntryPtr, TgtArgs, TgtOffsets, + *KernelArgs, AsyncInfoPtr); if (Err) { REPORT("Failure to run target region " DPxMOD " in device %d: %s\n", DPxPTR(TgtEntryPtr), DeviceId, toString(std::move(Err)).data()); @@ -1861,9 +1829,9 @@ int32_t __tgt_rtl_launch_kernel(int32_t DeviceId, void *TgtEntryPtr, return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_synchronize(int32_t DeviceId, - __tgt_async_info *AsyncInfoPtr) { - auto Err = PluginTy::get().getDevice(DeviceId).synchronize(AsyncInfoPtr); +int32_t GenericPluginTy::synchronize(int32_t DeviceId, + __tgt_async_info *AsyncInfoPtr) { + auto Err = getDevice(DeviceId).synchronize(AsyncInfoPtr); if (Err) { REPORT("Failure to synchronize stream %p: %s\n", AsyncInfoPtr->Queue, toString(std::move(Err)).data()); @@ -1873,9 +1841,9 @@ int32_t __tgt_rtl_synchronize(int32_t DeviceId, return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_query_async(int32_t DeviceId, - __tgt_async_info *AsyncInfoPtr) { - auto Err = PluginTy::get().getDevice(DeviceId).queryAsync(AsyncInfoPtr); +int32_t GenericPluginTy::query_async(int32_t DeviceId, + __tgt_async_info *AsyncInfoPtr) { + auto Err = getDevice(DeviceId).queryAsync(AsyncInfoPtr); if (Err) { REPORT("Failure to query stream %p: %s\n", AsyncInfoPtr->Queue, toString(std::move(Err)).data()); @@ -1885,14 +1853,14 @@ int32_t __tgt_rtl_query_async(int32_t DeviceId, return OFFLOAD_SUCCESS; } -void __tgt_rtl_print_device_info(int32_t DeviceId) { - if (auto Err = PluginTy::get().getDevice(DeviceId).printInfo()) +void GenericPluginTy::print_device_info(int32_t DeviceId) { + if (auto Err = getDevice(DeviceId).printInfo()) REPORT("Failure to print device %d info: %s\n", DeviceId, toString(std::move(Err)).data()); } -int32_t __tgt_rtl_create_event(int32_t DeviceId, void **EventPtr) { - auto Err = PluginTy::get().getDevice(DeviceId).createEvent(EventPtr); +int32_t GenericPluginTy::create_event(int32_t DeviceId, void **EventPtr) { + auto Err = getDevice(DeviceId).createEvent(EventPtr); if (Err) { REPORT("Failure to create event: %s\n", toString(std::move(Err)).data()); return OFFLOAD_FAIL; @@ -1901,10 +1869,9 @@ int32_t __tgt_rtl_create_event(int32_t DeviceId, void **EventPtr) { return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_record_event(int32_t DeviceId, void *EventPtr, - __tgt_async_info *AsyncInfoPtr) { - auto Err = - PluginTy::get().getDevice(DeviceId).recordEvent(EventPtr, AsyncInfoPtr); +int32_t GenericPluginTy::record_event(int32_t DeviceId, void *EventPtr, + __tgt_async_info *AsyncInfoPtr) { + auto Err = getDevice(DeviceId).recordEvent(EventPtr, AsyncInfoPtr); if (Err) { REPORT("Failure to record event %p: %s\n", EventPtr, toString(std::move(Err)).data()); @@ -1914,10 +1881,9 @@ int32_t __tgt_rtl_record_event(int32_t DeviceId, void *EventPtr, return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_wait_event(int32_t DeviceId, void *EventPtr, - __tgt_async_info *AsyncInfoPtr) { - auto Err = - PluginTy::get().getDevice(DeviceId).waitEvent(EventPtr, AsyncInfoPtr); +int32_t GenericPluginTy::wait_event(int32_t DeviceId, void *EventPtr, + __tgt_async_info *AsyncInfoPtr) { + auto Err = getDevice(DeviceId).waitEvent(EventPtr, AsyncInfoPtr); if (Err) { REPORT("Failure to wait event %p: %s\n", EventPtr, toString(std::move(Err)).data()); @@ -1927,8 +1893,8 @@ int32_t __tgt_rtl_wait_event(int32_t DeviceId, void *EventPtr, return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_sync_event(int32_t DeviceId, void *EventPtr) { - auto Err = PluginTy::get().getDevice(DeviceId).syncEvent(EventPtr); +int32_t GenericPluginTy::sync_event(int32_t DeviceId, void *EventPtr) { + auto Err = getDevice(DeviceId).syncEvent(EventPtr); if (Err) { REPORT("Failure to synchronize event %p: %s\n", EventPtr, toString(std::move(Err)).data()); @@ -1938,8 +1904,8 @@ int32_t __tgt_rtl_sync_event(int32_t DeviceId, void *EventPtr) { return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_destroy_event(int32_t DeviceId, void *EventPtr) { - auto Err = PluginTy::get().getDevice(DeviceId).destroyEvent(EventPtr); +int32_t GenericPluginTy::destroy_event(int32_t DeviceId, void *EventPtr) { + auto Err = getDevice(DeviceId).destroyEvent(EventPtr); if (Err) { REPORT("Failure to destroy event %p: %s\n", EventPtr, toString(std::move(Err)).data()); @@ -1949,16 +1915,16 @@ int32_t __tgt_rtl_destroy_event(int32_t DeviceId, void *EventPtr) { return OFFLOAD_SUCCESS; } -void __tgt_rtl_set_info_flag(uint32_t NewInfoLevel) { +void GenericPluginTy::set_info_flag(uint32_t NewInfoLevel) { std::atomic<uint32_t> &InfoLevel = getInfoLevelInternal(); InfoLevel.store(NewInfoLevel); } -int32_t __tgt_rtl_init_async_info(int32_t DeviceId, - __tgt_async_info **AsyncInfoPtr) { +int32_t GenericPluginTy::init_async_info(int32_t DeviceId, + __tgt_async_info **AsyncInfoPtr) { assert(AsyncInfoPtr && "Invalid async info"); - auto Err = PluginTy::get().getDevice(DeviceId).initAsyncInfo(AsyncInfoPtr); + auto Err = getDevice(DeviceId).initAsyncInfo(AsyncInfoPtr); if (Err) { REPORT("Failure to initialize async info at " DPxMOD " on device %d: %s\n", DPxPTR(*AsyncInfoPtr), DeviceId, toString(std::move(Err)).data()); @@ -1968,12 +1934,12 @@ int32_t __tgt_rtl_init_async_info(int32_t DeviceId, return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_init_device_info(int32_t DeviceId, - __tgt_device_info *DeviceInfo, - const char **ErrStr) { +int32_t GenericPluginTy::init_device_info(int32_t DeviceId, + __tgt_device_info *DeviceInfo, + const char **ErrStr) { *ErrStr = ""; - auto Err = PluginTy::get().getDevice(DeviceId).initDeviceInfo(DeviceInfo); + auto Err = getDevice(DeviceId).initDeviceInfo(DeviceInfo); if (Err) { REPORT("Failure to initialize device info at " DPxMOD " on device %d: %s\n", DPxPTR(DeviceInfo), DeviceId, toString(std::move(Err)).data()); @@ -1983,31 +1949,30 @@ int32_t __tgt_rtl_init_device_info(int32_t DeviceId, return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_set_device_offset(int32_t DeviceIdOffset) { - PluginTy::get().setDeviceIdStartIndex(DeviceIdOffset); +int32_t GenericPluginTy::set_device_offset(int32_t DeviceIdOffset) { + setDeviceIdStartIndex(DeviceIdOffset); return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_use_auto_zero_copy(int32_t DeviceId) { +int32_t GenericPluginTy::use_auto_zero_copy(int32_t DeviceId) { // Automatic zero-copy only applies to programs that did // not request unified_shared_memory and are deployed on an // APU with XNACK enabled. - if (PluginTy::get().getRequiresFlags() & OMP_REQ_UNIFIED_SHARED_MEMORY) + if (getRequiresFlags() & OMP_REQ_UNIFIED_SHARED_MEMORY) return false; - return PluginTy::get().getDevice(DeviceId).useAutoZeroCopy(); + return getDevice(DeviceId).useAutoZeroCopy(); } -int32_t __tgt_rtl_get_global(__tgt_device_binary Binary, uint64_t Size, - const char *Name, void **DevicePtr) { +int32_t GenericPluginTy::get_global(__tgt_device_binary Binary, uint64_t Size, + const char *Name, void **DevicePtr) { assert(Binary.handle && "Invalid device binary handle"); DeviceImageTy &Image = *reinterpret_cast<DeviceImageTy *>(Binary.handle); - GenericPluginTy &Plugin = PluginTy::get(); GenericDeviceTy &Device = Image.getDevice(); GlobalTy DeviceGlobal(Name, Size); - GenericGlobalHandlerTy &GHandler = Plugin.getGlobalHandler(); + GenericGlobalHandlerTy &GHandler = getGlobalHandler(); if (auto Err = GHandler.getGlobalMetadataFromDevice(Device, Image, DeviceGlobal)) { REPORT("Failure to look up global address: %s\n", @@ -2025,8 +1990,8 @@ int32_t __tgt_rtl_get_global(__tgt_device_binary Binary, uint64_t Size, return OFFLOAD_SUCCESS; } -int32_t __tgt_rtl_get_function(__tgt_device_binary Binary, const char *Name, - void **KernelPtr) { +int32_t GenericPluginTy::get_function(__tgt_device_binary Binary, + const char *Name, void **KernelPtr) { assert(Binary.handle && "Invalid device binary handle"); DeviceImageTy &Image = *reinterpret_cast<DeviceImageTy *>(Binary.handle); @@ -2049,6 +2014,212 @@ int32_t __tgt_rtl_get_function(__tgt_device_binary Binary, const char *Name, return OFFLOAD_SUCCESS; } +bool llvm::omp::target::plugin::libomptargetSupportsRPC() { +#ifdef LIBOMPTARGET_RPC_SUPPORT + return true; +#else + return false; +#endif +} + +/// Exposed library API function, basically wrappers around the GenericDeviceTy +/// functionality with the same name. All non-async functions are redirected +/// to the async versions right away with a NULL AsyncInfoPtr. +#ifdef __cplusplus +extern "C" { +#endif + +int32_t __tgt_rtl_init_plugin() { + auto Err = PluginTy::initIfNeeded(); + if (Err) { + [[maybe_unused]] std::string ErrStr = toString(std::move(Err)); + DP("Failed to init plugin: %s", ErrStr.c_str()); + return OFFLOAD_FAIL; + } + + return OFFLOAD_SUCCESS; +} + +int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image) { + if (!PluginTy::isActive()) + return false; + + return PluginTy::get().is_valid_binary(Image); +} + +int32_t __tgt_rtl_init_device(int32_t DeviceId) { + return PluginTy::get().init_device(DeviceId); +} + +int32_t __tgt_rtl_number_of_devices() { + return PluginTy::get().number_of_devices(); +} + +int64_t __tgt_rtl_init_requires(int64_t RequiresFlags) { + return PluginTy::get().init_requires(RequiresFlags); +} + +int32_t __tgt_rtl_is_data_exchangable(int32_t SrcDeviceId, + int32_t DstDeviceId) { + return PluginTy::get().is_data_exchangable(SrcDeviceId, DstDeviceId); +} + +int32_t __tgt_rtl_initialize_record_replay(int32_t DeviceId, int64_t MemorySize, + void *VAddr, bool isRecord, + bool SaveOutput, + uint64_t &ReqPtrArgOffset) { + return PluginTy::get().initialize_record_replay( + DeviceId, MemorySize, VAddr, isRecord, SaveOutput, ReqPtrArgOffset); +} + +int32_t __tgt_rtl_load_binary(int32_t DeviceId, __tgt_device_image *TgtImage, + __tgt_device_binary *Binary) { + return PluginTy::get().load_binary(DeviceId, TgtImage, Binary); +} + +void *__tgt_rtl_data_alloc(int32_t DeviceId, int64_t Size, void *HostPtr, + int32_t Kind) { + return PluginTy::get().data_alloc(DeviceId, Size, HostPtr, Kind); +} + +int32_t __tgt_rtl_data_delete(int32_t DeviceId, void *TgtPtr, int32_t Kind) { + return PluginTy::get().data_delete(DeviceId, TgtPtr, Kind); +} + +int32_t __tgt_rtl_data_lock(int32_t DeviceId, void *Ptr, int64_t Size, + void **LockedPtr) { + return PluginTy::get().data_lock(DeviceId, Ptr, Size, LockedPtr); +} + +int32_t __tgt_rtl_data_unlock(int32_t DeviceId, void *Ptr) { + return PluginTy::get().data_unlock(DeviceId, Ptr); +} + +int32_t __tgt_rtl_data_notify_mapped(int32_t DeviceId, void *HstPtr, + int64_t Size) { + return PluginTy::get().data_notify_mapped(DeviceId, HstPtr, Size); +} + +int32_t __tgt_rtl_data_notify_unmapped(int32_t DeviceId, void *HstPtr) { + return PluginTy::get().data_notify_unmapped(DeviceId, HstPtr); +} + +int32_t __tgt_rtl_data_submit(int32_t DeviceId, void *TgtPtr, void *HstPtr, + int64_t Size) { + return PluginTy::get().data_submit(DeviceId, TgtPtr, HstPtr, Size); +} + +int32_t __tgt_rtl_data_submit_async(int32_t DeviceId, void *TgtPtr, + void *HstPtr, int64_t Size, + __tgt_async_info *AsyncInfoPtr) { + return PluginTy::get().data_submit_async(DeviceId, TgtPtr, HstPtr, Size, + AsyncInfoPtr); +} + +int32_t __tgt_rtl_data_retrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr, + int64_t Size) { + return PluginTy::get().data_retrieve(DeviceId, HstPtr, TgtPtr, Size); +} + +int32_t __tgt_rtl_data_retrieve_async(int32_t DeviceId, void *HstPtr, + void *TgtPtr, int64_t Size, + __tgt_async_info *AsyncInfoPtr) { + return PluginTy::get().data_retrieve_async(DeviceId, HstPtr, TgtPtr, Size, + AsyncInfoPtr); +} + +int32_t __tgt_rtl_data_exchange(int32_t SrcDeviceId, void *SrcPtr, + int32_t DstDeviceId, void *DstPtr, + int64_t Size) { + return PluginTy::get().data_exchange(SrcDeviceId, SrcPtr, DstDeviceId, DstPtr, + Size); +} + +int32_t __tgt_rtl_data_exchange_async(int32_t SrcDeviceId, void *SrcPtr, + int DstDeviceId, void *DstPtr, + int64_t Size, + __tgt_async_info *AsyncInfo) { + return PluginTy::get().data_exchange_async(SrcDeviceId, SrcPtr, DstDeviceId, + DstPtr, Size, AsyncInfo); +} + +int32_t __tgt_rtl_launch_kernel(int32_t DeviceId, void *TgtEntryPtr, + void **TgtArgs, ptrdiff_t *TgtOffsets, + KernelArgsTy *KernelArgs, + __tgt_async_info *AsyncInfoPtr) { + return PluginTy::get().launch_kernel(DeviceId, TgtEntryPtr, TgtArgs, + TgtOffsets, KernelArgs, AsyncInfoPtr); +} + +int32_t __tgt_rtl_synchronize(int32_t DeviceId, + __tgt_async_info *AsyncInfoPtr) { + return PluginTy::get().synchronize(DeviceId, AsyncInfoPtr); +} + +int32_t __tgt_rtl_query_async(int32_t DeviceId, + __tgt_async_info *AsyncInfoPtr) { + return PluginTy::get().query_async(DeviceId, AsyncInfoPtr); +} + +void __tgt_rtl_print_device_info(int32_t DeviceId) { + PluginTy::get().print_device_info(DeviceId); +} + +int32_t __tgt_rtl_create_event(int32_t DeviceId, void **EventPtr) { + return PluginTy::get().create_event(DeviceId, EventPtr); +} + +int32_t __tgt_rtl_record_event(int32_t DeviceId, void *EventPtr, + __tgt_async_info *AsyncInfoPtr) { + return PluginTy::get().record_event(DeviceId, EventPtr, AsyncInfoPtr); +} + +int32_t __tgt_rtl_wait_event(int32_t DeviceId, void *EventPtr, + __tgt_async_info *AsyncInfoPtr) { + return PluginTy::get().wait_event(DeviceId, EventPtr, AsyncInfoPtr); +} + +int32_t __tgt_rtl_sync_event(int32_t DeviceId, void *EventPtr) { + return PluginTy::get().sync_event(DeviceId, EventPtr); +} + +int32_t __tgt_rtl_destroy_event(int32_t DeviceId, void *EventPtr) { + return PluginTy::get().destroy_event(DeviceId, EventPtr); +} + +void __tgt_rtl_set_info_flag(uint32_t NewInfoLevel) { + return PluginTy::get().set_info_flag(NewInfoLevel); +} + +int32_t __tgt_rtl_init_async_info(int32_t DeviceId, + __tgt_async_info **AsyncInfoPtr) { + return PluginTy::get().init_async_info(DeviceId, AsyncInfoPtr); +} + +int32_t __tgt_rtl_init_device_info(int32_t DeviceId, + __tgt_device_info *DeviceInfo, + const char **ErrStr) { + return PluginTy::get().init_device_info(DeviceId, DeviceInfo, ErrStr); +} + +int32_t __tgt_rtl_set_device_offset(int32_t DeviceIdOffset) { + return PluginTy::get().set_device_offset(DeviceIdOffset); +} + +int32_t __tgt_rtl_use_auto_zero_copy(int32_t DeviceId) { + return PluginTy::get().use_auto_zero_copy(DeviceId); +} + +int32_t __tgt_rtl_get_global(__tgt_device_binary Binary, uint64_t Size, + const char *Name, void **DevicePtr) { + return PluginTy::get().get_global(Binary, Size, Name, DevicePtr); +} + +int32_t __tgt_rtl_get_function(__tgt_device_binary Binary, const char *Name, + void **KernelPtr) { + return PluginTy::get().get_function(Binary, Name, KernelPtr); +} + #ifdef __cplusplus } #endif diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp index 3345277d91d3..44a2facc8d3d 100644 --- a/openmp/libomptarget/src/device.cpp +++ b/openmp/libomptarget/src/device.cpp @@ -79,8 +79,7 @@ DeviceTy::~DeviceTy() { llvm::Error DeviceTy::init() { // Make call to init_requires if it exists for this plugin. int32_t Ret = 0; - if (RTL->init_requires) - Ret = RTL->init_requires(PM->getRequirements()); + Ret = RTL->init_requires(PM->getRequirements()); if (Ret != OFFLOAD_SUCCESS) return llvm::createStringError( llvm::inconvertibleErrorCode(), @@ -154,8 +153,6 @@ int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size, omp_get_initial_device(), HstPtrBegin, DeviceID, TgtPtrBegin, Size, /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);) - if (!AsyncInfo || !RTL->data_submit_async || !RTL->synchronize) - return RTL->data_submit(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size); return RTL->data_submit_async(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size, AsyncInfo); } @@ -176,8 +173,6 @@ int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin, DeviceID, TgtPtrBegin, omp_get_initial_device(), HstPtrBegin, Size, /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);) - if (!RTL->data_retrieve_async || !RTL->synchronize) - return RTL->data_retrieve(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size); return RTL->data_retrieve_async(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size, AsyncInfo); } @@ -196,7 +191,7 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr, RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(), RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size, /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);) - if (!AsyncInfo || !RTL->data_exchange_async || !RTL->synchronize) { + if (!AsyncInfo) { assert(RTL->data_exchange && "RTL->data_exchange is nullptr"); return RTL->data_exchange(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size); @@ -206,9 +201,6 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr, } int32_t DeviceTy::notifyDataMapped(void *HstPtr, int64_t Size) { - if (!RTL->data_notify_mapped) - return OFFLOAD_SUCCESS; - DP("Notifying about new mapping: HstPtr=" DPxMOD ", Size=%" PRId64 "\n", DPxPTR(HstPtr), Size); @@ -220,9 +212,6 @@ int32_t DeviceTy::notifyDataMapped(void *HstPtr, int64_t Size) { } int32_t DeviceTy::notifyDataUnmapped(void *HstPtr) { - if (!RTL->data_notify_unmapped) - return OFFLOAD_SUCCESS; - DP("Notifying about an unmapping: HstPtr=" DPxMOD "\n", DPxPTR(HstPtr)); if (RTL->data_notify_unmapped(RTLDeviceID, HstPtr)) { @@ -242,70 +231,46 @@ int32_t DeviceTy::launchKernel(void *TgtEntryPtr, void **TgtVarsPtr, // Run region on device bool DeviceTy::printDeviceInfo() { - if (!RTL->print_device_info) - return false; RTL->print_device_info(RTLDeviceID); return true; } // Whether data can be copied to DstDevice directly bool DeviceTy::isDataExchangable(const DeviceTy &DstDevice) { - if (RTL != DstDevice.RTL || !RTL->is_data_exchangable) + if (RTL != DstDevice.RTL) return false; if (RTL->is_data_exchangable(RTLDeviceID, DstDevice.RTLDeviceID)) - return (RTL->data_exchange != nullptr) || - (RTL->data_exchange_async != nullptr); - + return true; return false; } int32_t DeviceTy::synchronize(AsyncInfoTy &AsyncInfo) { - if (RTL->synchronize) - return RTL->synchronize(RTLDeviceID, AsyncInfo); - return OFFLOAD_SUCCESS; + return RTL->synchronize(RTLDeviceID, AsyncInfo); } int32_t DeviceTy::queryAsync(AsyncInfoTy &AsyncInfo) { - if (RTL->query_async) - return RTL->query_async(RTLDeviceID, AsyncInfo); - - return synchronize(AsyncInfo); + return RTL->query_async(RTLDeviceID, AsyncInfo); } int32_t DeviceTy::createEvent(void **Event) { - if (RTL->create_event) - return RTL->create_event(RTLDeviceID, Event); - - return OFFLOAD_SUCCESS; + return RTL->create_event(RTLDeviceID, Event); } int32_t DeviceTy::recordEvent(void *Event, AsyncInfoTy &AsyncInfo) { - if (RTL->record_event) - return RTL->record_event(RTLDeviceID, Event, AsyncInfo); - - return OFFLOAD_SUCCESS; + return RTL->record_event(RTLDeviceID, Event, AsyncInfo); } int32_t DeviceTy::waitEvent(void *Event, AsyncInfoTy &AsyncInfo) { - if (RTL->wait_event) - return RTL->wait_event(RTLDeviceID, Event, AsyncInfo); - - return OFFLOAD_SUCCESS; + return RTL->wait_event(RTLDeviceID, Event, AsyncInfo); } int32_t DeviceTy::syncEvent(void *Event) { - if (RTL->sync_event) - return RTL->sync_event(RTLDeviceID, Event); - - return OFFLOAD_SUCCESS; + return RTL->sync_event(RTLDeviceID, Event); } int32_t DeviceTy::destroyEvent(void *Event) { - if (RTL->create_event) - return RTL->destroy_event(RTLDeviceID, Event); - - return OFFLOAD_SUCCESS; + return RTL->destroy_event(RTLDeviceID, Event); } void DeviceTy::dumpOffloadEntries() { @@ -321,7 +286,5 @@ void DeviceTy::dumpOffloadEntries() { } bool DeviceTy::useAutoZeroCopy() { - if (RTL->use_auto_zero_copy) - return RTL->use_auto_zero_copy(RTLDeviceID); - return false; + return RTL->use_auto_zero_copy(RTLDeviceID); } diff --git a/openmp/libomptarget/src/interface.cpp b/openmp/libomptarget/src/interface.cpp index b7f547f1ec3d..b562ba8818c3 100644 --- a/openmp/libomptarget/src/interface.cpp +++ b/openmp/libomptarget/src/interface.cpp @@ -456,10 +456,8 @@ EXTERN void __tgt_set_info_flag(uint32_t NewInfoLevel) { assert(PM && "Runtime not initialized"); std::atomic<uint32_t> &InfoLevel = getInfoLevelInternal(); InfoLevel.store(NewInfoLevel); - for (auto &R : PM->pluginAdaptors()) { - if (R.set_info_flag) - R.set_info_flag(NewInfoLevel); - } + for (auto &R : PM->pluginAdaptors()) + R.set_info_flag(NewInfoLevel); } EXTERN int __tgt_print_device_info(int64_t DeviceId) { diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp index 5bbf3a455c72..803e941fe838 100644 --- a/openmp/libomptarget/src/omptarget.cpp +++ b/openmp/libomptarget/src/omptarget.cpp @@ -481,12 +481,10 @@ void *targetLockExplicit(void *HostPtr, size_t Size, int DeviceNum, FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str()); int32_t Err = 0; - if (!DeviceOrErr->RTL->data_lock) { - Err = DeviceOrErr->RTL->data_lock(DeviceNum, HostPtr, Size, &RC); - if (Err) { - DP("Could not lock ptr %p\n", HostPtr); - return nullptr; - } + Err = DeviceOrErr->RTL->data_lock(DeviceNum, HostPtr, Size, &RC); + if (Err) { + DP("Could not lock ptr %p\n", HostPtr); + return nullptr; } DP("%s returns device ptr " DPxMOD "\n", Name, DPxPTR(RC)); return RC; @@ -499,9 +497,7 @@ void targetUnlockExplicit(void *HostPtr, int DeviceNum, const char *Name) { if (!DeviceOrErr) FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str()); - if (!DeviceOrErr->RTL->data_unlock) - DeviceOrErr->RTL->data_unlock(DeviceNum, HostPtr); - + DeviceOrErr->RTL->data_unlock(DeviceNum, HostPtr); DP("%s returns\n", Name); } |