diff --git a/lib/Transforms/Vectorize/VPlan.cpp b/lib/Transforms/Vectorize/VPlan.cpp index 9ee2744df0c..5c7c01e2f0f 100644 --- a/lib/Transforms/Vectorize/VPlan.cpp +++ b/lib/Transforms/Vectorize/VPlan.cpp @@ -503,10 +503,75 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent, } #endif +bool VPRecipeBase::mayWriteToMemory() const { + switch (getVPDefID()) { + case VPWidenMemoryInstructionSC: { + return cast(this)->isStore(); + } + case VPReplicateSC: + case VPWidenCallSC: + return cast(getVPValue()->getUnderlyingValue()) + ->mayWriteToMemory(); + case VPBranchOnMaskSC: + return false; + case VPWidenIntOrFpInductionSC: + case VPWidenCanonicalIVSC: + case VPWidenPHISC: + case VPBlendSC: + case VPWidenSC: + case VPWidenGEPSC: + case VPReductionSC: + case VPWidenSelectSC: { + const Instruction *I = + dyn_cast_or_null(getVPValue()->getUnderlyingValue()); + (void)I; + assert((!I || !I->mayWriteToMemory()) && + "underlying instruction may write to memory"); + return false; + } + default: + return true; + } +} + +bool VPRecipeBase::mayReadFromMemory() const { + switch (getVPDefID()) { + case VPWidenMemoryInstructionSC: { + return !cast(this)->isStore(); + } + case VPReplicateSC: + case VPWidenCallSC: + return cast(getVPValue()->getUnderlyingValue()) + ->mayReadFromMemory(); + case VPBranchOnMaskSC: + return false; + case VPWidenIntOrFpInductionSC: + case VPWidenCanonicalIVSC: + case VPWidenPHISC: + case VPBlendSC: + case VPWidenSC: + case VPWidenGEPSC: + case VPReductionSC: + case VPWidenSelectSC: { + const Instruction *I = + dyn_cast_or_null(getVPValue()->getUnderlyingValue()); + (void)I; + assert((!I || !I->mayReadFromMemory()) && + "underlying instruction may read from memory"); + return false; + } + default: + return true; + } +} + bool VPRecipeBase::mayHaveSideEffects() const { switch (getVPDefID()) { case VPBranchOnMaskSC: return false; + case VPWidenIntOrFpInductionSC: + case VPWidenCanonicalIVSC: + case VPWidenPHISC: case VPBlendSC: case VPWidenSC: case VPWidenGEPSC: diff --git a/lib/Transforms/Vectorize/VPlan.h b/lib/Transforms/Vectorize/VPlan.h index bfb8adb7ae6..0a308772f3d 100644 --- a/lib/Transforms/Vectorize/VPlan.h +++ b/lib/Transforms/Vectorize/VPlan.h @@ -729,6 +729,17 @@ public: return getVPDefID() == VPWidenIntOrFpInductionSC || getVPDefID() == VPWidenPHISC || getVPDefID() == VPPredInstPHISC || getVPDefID() == VPWidenCanonicalIVSC; } + + /// Returns true if the recipe may read from memory. + bool mayReadFromMemory() const; + + /// Returns true if the recipe may write to memory. + bool mayWriteToMemory() const; + + /// Returns true if the recipe may read from or write to memory. + bool mayReadOrWriteMemory() const { + return mayReadFromMemory() || mayWriteToMemory(); + } }; inline bool VPUser::classof(const VPDef *Def) { diff --git a/unittests/Transforms/Vectorize/VPlanTest.cpp b/unittests/Transforms/Vectorize/VPlanTest.cpp index c881591ec9e..22ca09e6949 100644 --- a/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -949,7 +949,7 @@ TEST(VPRecipeTest, CastVPWidenMemoryInstructionRecipeToVPUserAndVPDef) { delete Load; } -TEST(VPRecipeTest, MayHaveSideEffects) { +TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { LLVMContext C; IntegerType *Int1 = IntegerType::get(C, 1); IntegerType *Int32 = IntegerType::get(C, 32); @@ -965,7 +965,9 @@ TEST(VPRecipeTest, MayHaveSideEffects) { Args.push_back(&Op1); VPWidenRecipe Recipe(*AI, make_range(Args.begin(), Args.end())); EXPECT_FALSE(Recipe.mayHaveSideEffects()); - + EXPECT_FALSE(Recipe.mayReadFromMemory()); + EXPECT_FALSE(Recipe.mayWriteToMemory()); + EXPECT_FALSE(Recipe.mayReadOrWriteMemory()); delete AI; } @@ -982,6 +984,9 @@ TEST(VPRecipeTest, MayHaveSideEffects) { VPWidenSelectRecipe Recipe(*SelectI, make_range(Args.begin(), Args.end()), false); EXPECT_FALSE(Recipe.mayHaveSideEffects()); + EXPECT_FALSE(Recipe.mayReadFromMemory()); + EXPECT_FALSE(Recipe.mayWriteToMemory()); + EXPECT_FALSE(Recipe.mayReadOrWriteMemory()); delete SelectI; } @@ -995,6 +1000,9 @@ TEST(VPRecipeTest, MayHaveSideEffects) { Args.push_back(&Op2); VPWidenGEPRecipe Recipe(GEP, make_range(Args.begin(), Args.end())); EXPECT_FALSE(Recipe.mayHaveSideEffects()); + EXPECT_FALSE(Recipe.mayReadFromMemory()); + EXPECT_FALSE(Recipe.mayWriteToMemory()); + EXPECT_FALSE(Recipe.mayReadOrWriteMemory()); delete GEP; } @@ -1002,6 +1010,9 @@ TEST(VPRecipeTest, MayHaveSideEffects) { VPValue Mask; VPBranchOnMaskRecipe Recipe(&Mask); EXPECT_FALSE(Recipe.mayHaveSideEffects()); + EXPECT_FALSE(Recipe.mayReadFromMemory()); + EXPECT_FALSE(Recipe.mayWriteToMemory()); + EXPECT_FALSE(Recipe.mayReadOrWriteMemory()); } { @@ -1011,6 +1022,9 @@ TEST(VPRecipeTest, MayHaveSideEffects) { VPReductionRecipe Recipe(nullptr, nullptr, &ChainOp, &CondOp, &VecOp, nullptr); EXPECT_FALSE(Recipe.mayHaveSideEffects()); + EXPECT_FALSE(Recipe.mayReadFromMemory()); + EXPECT_FALSE(Recipe.mayWriteToMemory()); + EXPECT_FALSE(Recipe.mayReadOrWriteMemory()); } { @@ -1020,10 +1034,26 @@ TEST(VPRecipeTest, MayHaveSideEffects) { VPValue Mask; VPWidenMemoryInstructionRecipe Recipe(*Load, &Addr, &Mask); EXPECT_TRUE(Recipe.mayHaveSideEffects()); - + EXPECT_TRUE(Recipe.mayReadFromMemory()); + EXPECT_FALSE(Recipe.mayWriteToMemory()); + EXPECT_TRUE(Recipe.mayReadOrWriteMemory()); delete Load; } + { + auto *Store = new StoreInst(UndefValue::get(Int32), + UndefValue::get(Int32Ptr), false, Align(1)); + VPValue Addr; + VPValue Mask; + VPValue StoredV; + VPWidenMemoryInstructionRecipe Recipe(*Store, &Addr, &StoredV, &Mask); + EXPECT_TRUE(Recipe.mayHaveSideEffects()); + EXPECT_FALSE(Recipe.mayReadFromMemory()); + EXPECT_TRUE(Recipe.mayWriteToMemory()); + EXPECT_TRUE(Recipe.mayReadOrWriteMemory()); + delete Store; + } + { FunctionType *FTy = FunctionType::get(Int32, false); auto *Call = CallInst::Create(FTy, UndefValue::get(FTy)); @@ -1034,6 +1064,9 @@ TEST(VPRecipeTest, MayHaveSideEffects) { Args.push_back(&Op2); VPWidenCallRecipe Recipe(*Call, make_range(Args.begin(), Args.end())); EXPECT_TRUE(Recipe.mayHaveSideEffects()); + EXPECT_TRUE(Recipe.mayReadFromMemory()); + EXPECT_TRUE(Recipe.mayWriteToMemory()); + EXPECT_TRUE(Recipe.mayReadOrWriteMemory()); delete Call; } @@ -1041,8 +1074,12 @@ TEST(VPRecipeTest, MayHaveSideEffects) { { VPValue Op1; VPValue Op2; - VPInstruction Recipe(Instruction::Add, {&Op1, &Op2}); + VPInstruction VPInst(Instruction::Add, {&Op1, &Op2}); + VPRecipeBase &Recipe = VPInst; EXPECT_TRUE(Recipe.mayHaveSideEffects()); + EXPECT_TRUE(Recipe.mayReadFromMemory()); + EXPECT_TRUE(Recipe.mayWriteToMemory()); + EXPECT_TRUE(Recipe.mayReadOrWriteMemory()); } }