[dfsan] Add full fast8 support

Complete support for fast8:
- amend shadow size and mapping in runtime
- remove fast16 mode and -dfsan-fast-16-labels flag
- remove legacy mode and make fast8 mode the default
- remove dfsan-fast-8-labels flag
- remove functions in dfsan interface only applicable to legacy
- remove legacy-related instrumentation code and tests
- update documentation.

Reviewed By: stephan.yichao.zhao, browneee

Differential Revision: https://reviews.llvm.org/D103745

lib/Transforms/Instrumentation/DataFlowSanitizer.cpp

@@ -18,11 +18,9 @@
 /// The analysis is based on automatic propagation of data flow labels (also
 /// known as taint labels) through a program as it performs computation.
 ///
-/// There are two possible memory layouts. In the first one, each byte of
+/// Each byte of application memory is backed by a shadow memory byte. The
-/// application memory is backed by a shadow memory byte. The shadow byte can
+/// shadow byte can represent up to 8 labels. On Linux/x86_64, memory is then
-/// represent up to 8 labels. To enable this you must specify the
+/// laid out as follows:
-/// -dfsan-fast-8-labels flag. On Linux/x86_64, memory is then laid out as
-/// follows:
 ///
 /// +--------------------+ 0x800000000000 (top of memory)
 /// | application memory |
@@ -30,11 +28,11 @@
 /// |                    |
 /// |       unused       |
 /// |                    |
-/// +--------------------+ 0x300200000000 (kUnusedAddr)
+/// +--------------------+ 0x300000000000 (kUnusedAddr)
-/// |    union table     |
-/// +--------------------+ 0x300000000000 (kUnionTableAddr)
 /// |       origin       |
 /// +--------------------+ 0x200000008000 (kOriginAddr)
+/// |       unused       |
+/// +--------------------+ 0x200000000000
 /// |   shadow memory    |
 /// +--------------------+ 0x100000008000 (kShadowAddr)
 /// |       unused       |
@@ -43,34 +41,9 @@
 /// +--------------------+ 0x000000000000
 ///
 ///
-/// In the second memory layout, each byte of application memory is backed by
+/// To derive a shadow memory address from an application memory address, bits
-/// two bytes of shadow memory which hold the label. That means we can represent
+/// 45-46 are cleared to bring the address into the range
-/// either 16 labels (with -dfsan-fast-16-labels flag) or 2^16 labels (on the
+/// [0x100000008000,0x200000000000). See the function
-/// default legacy mode) per byte. On Linux/x86_64, memory is then laid out as
-/// follows:
-///
-/// +--------------------+ 0x800000000000 (top of memory)
-/// | application memory |
-/// +--------------------+ 0x700000008000 (kAppAddr)
-/// |                    |
-/// |       unused       |
-/// |                    |
-/// +--------------------+ 0x300200000000 (kUnusedAddr)
-/// |    union table     |
-/// +--------------------+ 0x300000000000 (kUnionTableAddr)
-/// |       origin       |
-/// +--------------------+ 0x200000008000 (kOriginAddr)
-/// |   shadow memory    |
-/// +--------------------+ 0x000000010000 (kShadowAddr)
-/// | reserved by kernel |
-/// +--------------------+ 0x000000000000
-///
-///
-/// To derive a shadow memory address from an application memory address,
-/// bits 44-46 are cleared to bring the address into the range
-/// [0x000000008000,0x100000000000).  Then the address is shifted left by 1 to
-/// account for the double byte representation of shadow labels and move the
-/// address into the shadow memory range.  See the function
 /// DataFlowSanitizer::getShadowAddress below.
 ///
 /// For more information, please refer to the design document:
@@ -230,22 +203,6 @@ static cl::opt<bool> ClEventCallbacks(
     cl::desc("Insert calls to __dfsan_*_callback functions on data events."),
     cl::Hidden, cl::init(false));
 
-// Use a distinct bit for each base label, enabling faster unions with less
-// instrumentation.  Limits the max number of base labels to 16.
-static cl::opt<bool> ClFast16Labels(
-    "dfsan-fast-16-labels",
-    cl::desc("Use more efficient instrumentation, limiting the number of "
-             "labels to 16."),
-    cl::Hidden, cl::init(false));
-
-// Use a distinct bit for each base label, enabling faster unions with less
-// instrumentation.  Limits the max number of base labels to 8.
-static cl::opt<bool> ClFast8Labels(
-    "dfsan-fast-8-labels",
-    cl::desc("Use more efficient instrumentation, limiting the number of "
-             "labels to 8."),
-    cl::Hidden, cl::init(false));
-
 // Controls whether the pass tracks the control flow of select instructions.
 static cl::opt<bool> ClTrackSelectControlFlow(
     "dfsan-track-select-control-flow",
@@ -387,6 +344,8 @@ class DataFlowSanitizer {
   friend struct DFSanFunction;
   friend class DFSanVisitor;
 
+  enum { ShadowWidthBits = 8, ShadowWidthBytes = ShadowWidthBits / 8 };
+
   enum {
     OriginWidthBits = 32,
     OriginWidthBytes = OriginWidthBits / 8
@@ -428,15 +387,11 @@ class DataFlowSanitizer {
     WK_Custom
   };
 
-  unsigned ShadowWidthBits;
-  unsigned ShadowWidthBytes;
-
   Module *Mod;
   LLVMContext *Ctx;
   Type *Int8Ptr;
   IntegerType *OriginTy;
   PointerType *OriginPtrTy;
-  ConstantInt *OriginBase;
   ConstantInt *ZeroOrigin;
   /// The shadow type for all primitive types and vector types.
   IntegerType *PrimitiveShadowTy;
@@ -444,14 +399,14 @@ class DataFlowSanitizer {
   IntegerType *IntptrTy;
   ConstantInt *ZeroPrimitiveShadow;
   ConstantInt *ShadowPtrMask;
-  ConstantInt *ShadowPtrMul;
+  ConstantInt *ShadowBase;
+  ConstantInt *OriginBase;
   Constant *ArgTLS;
   ArrayType *ArgOriginTLSTy;
   Constant *ArgOriginTLS;
   Constant *RetvalTLS;
   Constant *RetvalOriginTLS;
   Constant *ExternalShadowMask;
-  FunctionType *DFSanUnionFnTy;
   FunctionType *DFSanUnionLoadFnTy;
   FunctionType *DFSanLoadLabelAndOriginFnTy;
   FunctionType *DFSanUnimplementedFnTy;
@@ -465,10 +420,7 @@ class DataFlowSanitizer {
   FunctionType *DFSanChainOriginIfTaintedFnTy;
   FunctionType *DFSanMemOriginTransferFnTy;
   FunctionType *DFSanMaybeStoreOriginFnTy;
-  FunctionCallee DFSanUnionFn;
-  FunctionCallee DFSanCheckedUnionFn;
   FunctionCallee DFSanUnionLoadFn;
-  FunctionCallee DFSanUnionLoadFastLabelsFn;
   FunctionCallee DFSanLoadLabelAndOriginFn;
   FunctionCallee DFSanUnimplementedFn;
   FunctionCallee DFSanSetLabelFn;
@@ -510,7 +462,6 @@ class DataFlowSanitizer {
   void initializeCallbackFunctions(Module &M);
   void initializeRuntimeFunctions(Module &M);
   void injectMetadataGlobals(Module &M);
-
   bool init(Module &M);
 
   /// Advances \p OriginAddr to point to the next 32-bit origin and then loads
@@ -518,19 +469,15 @@ class DataFlowSanitizer {
   Value *loadNextOrigin(Instruction *Pos, Align OriginAlign,
                         Value **OriginAddr);
 
-  /// Returns whether fast8 or fast16 mode has been specified.
-  bool hasFastLabelsEnabled();
-
   /// Returns whether the given load byte size is amenable to inlined
   /// optimization patterns.
   bool hasLoadSizeForFastPath(uint64_t Size);
 
-  /// Returns whether the pass tracks origins. Support only fast16 mode in TLS
+  /// Returns whether the pass tracks origins. Supports only TLS ABI mode.
-  /// ABI mode.
   bool shouldTrackOrigins();
 
   /// Returns whether the pass tracks labels for struct fields and array
-  /// indices. Support only fast16 mode in TLS ABI mode.
+  /// indices. Supports only TLS ABI mode.
   bool shouldTrackFieldsAndIndices();
 
   /// Returns a zero constant with the shadow type of OrigTy.
@@ -590,7 +537,6 @@ struct DFSanFunction {
 
   DenseSet<Instruction *> SkipInsts;
   std::vector<Value *> NonZeroChecks;
-  bool AvoidNewBlocks;
 
   struct CachedShadow {
     BasicBlock *Block; // The block where Shadow is defined.
@@ -608,9 +554,6 @@ struct DFSanFunction {
   DFSanFunction(DataFlowSanitizer &DFS, Function *F, bool IsNativeABI)
       : DFS(DFS), F(F), IA(DFS.getInstrumentedABI()), IsNativeABI(IsNativeABI) {
     DT.recalculate(*F);
-    // FIXME: Need to track down the register allocator issue which causes poor
-    // performance in pathological cases with large numbers of basic blocks.
-    AvoidNewBlocks = F->size() > 1000;
   }
 
   /// Computes the shadow address for a given function argument.
@@ -702,15 +645,11 @@ private:
   /// Returns the shadow value of an argument A.
   Value *getShadowForTLSArgument(Argument *A);
 
-  /// The fast path of loading shadow in legacy mode.
+  /// The fast path of loading shadows.
-  Value *loadLegacyShadowFast(Value *ShadowAddr, uint64_t Size,
-                              Align ShadowAlign, Instruction *Pos);
-
-  /// The fast path of loading shadow in fast-16-label mode.
   std::pair<Value *, Value *>
-  loadFast16ShadowFast(Value *ShadowAddr, Value *OriginAddr, uint64_t Size,
+  loadShadowFast(Value *ShadowAddr, Value *OriginAddr, uint64_t Size,
-                       Align ShadowAlign, Align OriginAlign, Value *FirstOrigin,
+                 Align ShadowAlign, Align OriginAlign, Value *FirstOrigin,
-                       Instruction *Pos);
+                 Instruction *Pos);
 
   Align getOriginAlign(Align InstAlignment);
 
@@ -820,14 +759,6 @@ private:
 
 DataFlowSanitizer::DataFlowSanitizer(
     const std::vector<std::string> &ABIListFiles) {
-  if (ClFast8Labels && ClFast16Labels) {
-    report_fatal_error(
-        "cannot set both -dfsan-fast-8-labels and -dfsan-fast-16-labels");
-  }
-
-  ShadowWidthBits = ClFast8Labels ? 8 : 16;
-  ShadowWidthBytes = ShadowWidthBits / 8;
-
   std::vector<std::string> AllABIListFiles(std::move(ABIListFiles));
   llvm::append_range(AllABIListFiles, ClABIListFiles);
   // FIXME: should we propagate vfs::FileSystem to this constructor?
@@ -922,11 +853,6 @@ bool DataFlowSanitizer::isZeroShadow(Value *V) {
   return isa<ConstantAggregateZero>(V);
 }
 
-bool DataFlowSanitizer::hasFastLabelsEnabled() {
-  static const bool HasFastLabelsEnabled = ClFast8Labels || ClFast16Labels;
-  return HasFastLabelsEnabled;
-}
-
 bool DataFlowSanitizer::hasLoadSizeForFastPath(uint64_t Size) {
   uint64_t ShadowSize = Size * ShadowWidthBytes;
   return ShadowSize % 8 == 0 || ShadowSize == 4;
@@ -934,14 +860,12 @@ bool DataFlowSanitizer::hasLoadSizeForFastPath(uint64_t Size) {
 
 bool DataFlowSanitizer::shouldTrackOrigins() {
   static const bool ShouldTrackOrigins =
-      ClTrackOrigins && getInstrumentedABI() == DataFlowSanitizer::IA_TLS &&
+      ClTrackOrigins && getInstrumentedABI() == DataFlowSanitizer::IA_TLS;
-      hasFastLabelsEnabled();
   return ShouldTrackOrigins;
 }
 
 bool DataFlowSanitizer::shouldTrackFieldsAndIndices() {
-  return getInstrumentedABI() == DataFlowSanitizer::IA_TLS &&
+  return getInstrumentedABI() == DataFlowSanitizer::IA_TLS;
-         hasFastLabelsEnabled();
 }
 
 Constant *DataFlowSanitizer::getZeroShadow(Type *OrigTy) {
@@ -1100,21 +1024,19 @@ bool DataFlowSanitizer::init(Module &M) {
   PrimitiveShadowPtrTy = PointerType::getUnqual(PrimitiveShadowTy);
   IntptrTy = DL.getIntPtrType(*Ctx);
   ZeroPrimitiveShadow = ConstantInt::getSigned(PrimitiveShadowTy, 0);
-  ShadowPtrMul = ConstantInt::getSigned(IntptrTy, ShadowWidthBytes);
-  OriginBase = ConstantInt::get(IntptrTy, 0x200000000000LL);
   ZeroOrigin = ConstantInt::getSigned(OriginTy, 0);
 
+  // TODO: these should be platform-specific and set in the switch-stmt below.
+  ShadowBase = ConstantInt::get(IntptrTy, 0x100000008000LL);
+  OriginBase = ConstantInt::get(IntptrTy, 0x200000008000LL);
+
   switch (TargetTriple.getArch()) {
   case Triple::x86_64:
-    ShadowPtrMask = ClFast8Labels
+    ShadowPtrMask = ConstantInt::getSigned(IntptrTy, ~0x600000000000LL);
-                        ? ConstantInt::getSigned(IntptrTy, ~0x600000000000LL)
-                        : ConstantInt::getSigned(IntptrTy, ~0x700000000000LL);
     break;
   case Triple::mips64:
   case Triple::mips64el:
-    ShadowPtrMask = ClFast8Labels
+    ShadowPtrMask = ConstantInt::getSigned(IntptrTy, ~0xE000000000LL);
-                        ? ConstantInt::getSigned(IntptrTy, ~0xE000000000LL)
-                        : ConstantInt::getSigned(IntptrTy, ~0xF000000000LL);
     break;
   case Triple::aarch64:
   case Triple::aarch64_be:
@@ -1125,9 +1047,6 @@ bool DataFlowSanitizer::init(Module &M) {
     report_fatal_error("unsupported triple");
   }
 
-  Type *DFSanUnionArgs[2] = {PrimitiveShadowTy, PrimitiveShadowTy};
-  DFSanUnionFnTy =
-      FunctionType::get(PrimitiveShadowTy, DFSanUnionArgs, /*isVarArg=*/false);
   Type *DFSanUnionLoadArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
   DFSanUnionLoadFnTy = FunctionType::get(PrimitiveShadowTy, DFSanUnionLoadArgs,
                                          /*isVarArg=*/false);
@@ -1306,32 +1225,6 @@ Constant *DataFlowSanitizer::getOrBuildTrampolineFunction(FunctionType *FT,
 
 // Initialize DataFlowSanitizer runtime functions and declare them in the module
 void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) {
-  {
-    AttributeList AL;
-    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
-                         Attribute::NoUnwind);
-    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
-                         Attribute::ReadNone);
-    AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex,
-                         Attribute::ZExt);
-    AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
-    AL = AL.addParamAttribute(M.getContext(), 1, Attribute::ZExt);
-    DFSanUnionFn =
-        Mod->getOrInsertFunction("__dfsan_union", DFSanUnionFnTy, AL);
-  }
-  {
-    AttributeList AL;
-    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
-                         Attribute::NoUnwind);
-    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
-                         Attribute::ReadNone);
-    AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex,
-                         Attribute::ZExt);
-    AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
-    AL = AL.addParamAttribute(M.getContext(), 1, Attribute::ZExt);
-    DFSanCheckedUnionFn =
-        Mod->getOrInsertFunction("dfsan_union", DFSanUnionFnTy, AL);
-  }
   {
     AttributeList AL;
     AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
@@ -1343,17 +1236,6 @@ void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) {
     DFSanUnionLoadFn =
         Mod->getOrInsertFunction("__dfsan_union_load", DFSanUnionLoadFnTy, AL);
   }
-  {
-    AttributeList AL;
-    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
-                         Attribute::NoUnwind);
-    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
-                         Attribute::ReadOnly);
-    AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex,
-                         Attribute::ZExt);
-    DFSanUnionLoadFastLabelsFn = Mod->getOrInsertFunction(
-        "__dfsan_union_load_fast16labels", DFSanUnionLoadFnTy, AL);
-  }
   {
     AttributeList AL;
     AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
@@ -1406,13 +1288,8 @@ void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) {
         "__dfsan_maybe_store_origin", DFSanMaybeStoreOriginFnTy, AL);
   }
 
-  DFSanRuntimeFunctions.insert(DFSanUnionFn.getCallee()->stripPointerCasts());
-  DFSanRuntimeFunctions.insert(
-      DFSanCheckedUnionFn.getCallee()->stripPointerCasts());
   DFSanRuntimeFunctions.insert(
       DFSanUnionLoadFn.getCallee()->stripPointerCasts());
-  DFSanRuntimeFunctions.insert(
-      DFSanUnionLoadFastLabelsFn.getCallee()->stripPointerCasts());
   DFSanRuntimeFunctions.insert(
       DFSanLoadLabelAndOriginFn.getCallee()->stripPointerCasts());
   DFSanRuntimeFunctions.insert(
@@ -1878,13 +1755,16 @@ Value *DataFlowSanitizer::getShadowOffset(Value *Addr, IRBuilder<> &IRB) {
 std::pair<Value *, Value *>
 DataFlowSanitizer::getShadowOriginAddress(Value *Addr, Align InstAlignment,
                                           Instruction *Pos) {
-  // Returns ((Addr & shadow_mask) + origin_base) & ~4UL
+  // Returns ((Addr & shadow_mask) + origin_base - shadow_base) & ~4UL
   IRBuilder<> IRB(Pos);
   Value *ShadowOffset = getShadowOffset(Addr, IRB);
   Value *ShadowPtr = getShadowAddress(Addr, Pos, ShadowOffset);
   Value *OriginPtr = nullptr;
   if (shouldTrackOrigins()) {
-    Value *OriginLong = IRB.CreateAdd(ShadowOffset, OriginBase);
+    static Value *OriginByShadowOffset = ConstantInt::get(
+        IntptrTy, OriginBase->getZExtValue() - ShadowBase->getZExtValue());
+
+    Value *OriginLong = IRB.CreateAdd(ShadowOffset, OriginByShadowOffset);
     const Align Alignment = llvm::assumeAligned(InstAlignment.value());
     // When alignment is >= 4, Addr must be aligned to 4, otherwise it is UB.
     // So Mask is unnecessary.
@@ -1900,15 +1780,11 @@ DataFlowSanitizer::getShadowOriginAddress(Value *Addr, Align InstAlignment,
 Value *DataFlowSanitizer::getShadowAddress(Value *Addr, Instruction *Pos,
                                            Value *ShadowOffset) {
   IRBuilder<> IRB(Pos);
-
-  if (!ShadowPtrMul->isOne())
-    ShadowOffset = IRB.CreateMul(ShadowOffset, ShadowPtrMul);
-
   return IRB.CreateIntToPtr(ShadowOffset, PrimitiveShadowPtrTy);
 }
 
 Value *DataFlowSanitizer::getShadowAddress(Value *Addr, Instruction *Pos) {
-  // Returns (Addr & shadow_mask) x 2
+  // Returns (Addr & shadow_mask)
   IRBuilder<> IRB(Pos);
   Value *ShadowOffset = getShadowOffset(Addr, IRB);
   return getShadowAddress(Addr, Pos, ShadowOffset);
@@ -1961,37 +1837,8 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
   Value *PV2 = collapseToPrimitiveShadow(V2, Pos);
 
   IRBuilder<> IRB(Pos);
-  if (DFS.hasFastLabelsEnabled()) {
+  CCS.Block = Pos->getParent();
-    CCS.Block = Pos->getParent();
+  CCS.Shadow = IRB.CreateOr(PV1, PV2);
-    CCS.Shadow = IRB.CreateOr(PV1, PV2);
-  } else if (AvoidNewBlocks) {
-    CallInst *Call = IRB.CreateCall(DFS.DFSanCheckedUnionFn, {PV1, PV2});
-    Call->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
-    Call->addParamAttr(0, Attribute::ZExt);
-    Call->addParamAttr(1, Attribute::ZExt);
-
-    CCS.Block = Pos->getParent();
-    CCS.Shadow = Call;
-  } else {
-    BasicBlock *Head = Pos->getParent();
-    Value *Ne = IRB.CreateICmpNE(PV1, PV2);
-    BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen(
-        Ne, Pos, /*Unreachable=*/false, DFS.ColdCallWeights, &DT));
-    IRBuilder<> ThenIRB(BI);
-    CallInst *Call = ThenIRB.CreateCall(DFS.DFSanUnionFn, {PV1, PV2});
-    Call->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
-    Call->addParamAttr(0, Attribute::ZExt);
-    Call->addParamAttr(1, Attribute::ZExt);
-
-    BasicBlock *Tail = BI->getSuccessor(0);
-    PHINode *Phi =
-        PHINode::Create(DFS.PrimitiveShadowTy, 2, "", &Tail->front());
-    Phi->addIncoming(Call, Call->getParent());
-    Phi->addIncoming(PV1, Head);
-
-    CCS.Block = Tail;
-    CCS.Shadow = Phi;
-  }
 
   std::set<Value *> UnionElems;
   if (V1Elems != ShadowElements.end()) {
@@ -2116,7 +1963,7 @@ Value *DataFlowSanitizer::loadNextOrigin(Instruction *Pos, Align OriginAlign,
   return IRB.CreateAlignedLoad(OriginTy, *OriginAddr, OriginAlign);
 }
 
-std::pair<Value *, Value *> DFSanFunction::loadFast16ShadowFast(
+std::pair<Value *, Value *> DFSanFunction::loadShadowFast(
     Value *ShadowAddr, Value *OriginAddr, uint64_t Size, Align ShadowAlign,
     Align OriginAlign, Value *FirstOrigin, Instruction *Pos) {
   const bool ShouldTrackOrigins = DFS.shouldTrackOrigins();
@@ -2135,7 +1982,7 @@ std::pair<Value *, Value *> DFSanFunction::loadFast16ShadowFast(
   // Specifically, when the shadow size in bytes (i.e., loaded bytes x shadow
   // per byte) is either:
   // - a multiple of 8  (common)
-  // - equal to 4       (only for load32 in fast-8 mode)
+  // - equal to 4       (only for load32)
   //
   // For the second case, we can fit the wide shadow in a 32-bit integer. In all
   // other cases, we use a 64-bit integer to hold the wide shadow.
@@ -2204,84 +2051,6 @@ std::pair<Value *, Value *> DFSanFunction::loadFast16ShadowFast(
               : DFS.ZeroOrigin};
 }
 
-Value *DFSanFunction::loadLegacyShadowFast(Value *ShadowAddr, uint64_t Size,
-                                           Align ShadowAlign,
-                                           Instruction *Pos) {
-  // Fast path for the common case where each byte has identical shadow: load
-  // shadow 64 (or 32) bits at a time, fall out to a __dfsan_union_load call if
-  // any shadow is non-equal.
-  BasicBlock *FallbackBB = BasicBlock::Create(*DFS.Ctx, "", F);
-  IRBuilder<> FallbackIRB(FallbackBB);
-  CallInst *FallbackCall = FallbackIRB.CreateCall(
-      DFS.DFSanUnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
-  FallbackCall->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
-
-  const uint64_t ShadowSize = Size * DFS.ShadowWidthBytes;
-  assert(Size >= 4 && "Not large enough load size for fast path!");
-
-  // Same as in loadFast16AShadowsFast. In the case of load32, we can fit the
-  // wide shadow in a 32-bit integer instead.
-  Type *WideShadowTy =
-      ShadowSize == 4 ? Type::getInt32Ty(*DFS.Ctx) : Type::getInt64Ty(*DFS.Ctx);
-
-  // Compare each of the shadows stored in the loaded 64 bits to each other,
-  // by computing (WideShadow rotl ShadowWidthBits) == WideShadow.
-  IRBuilder<> IRB(Pos);
-  unsigned WideShadowBitWidth = WideShadowTy->getIntegerBitWidth();
-  Value *WideAddr = IRB.CreateBitCast(ShadowAddr, WideShadowTy->getPointerTo());
-  Value *WideShadow =
-      IRB.CreateAlignedLoad(WideShadowTy, WideAddr, ShadowAlign);
-  Value *TruncShadow = IRB.CreateTrunc(WideShadow, DFS.PrimitiveShadowTy);
-  Value *ShlShadow = IRB.CreateShl(WideShadow, DFS.ShadowWidthBits);
-  Value *ShrShadow =
-      IRB.CreateLShr(WideShadow, WideShadowBitWidth - DFS.ShadowWidthBits);
-  Value *RotShadow = IRB.CreateOr(ShlShadow, ShrShadow);
-  Value *ShadowsEq = IRB.CreateICmpEQ(WideShadow, RotShadow);
-
-  BasicBlock *Head = Pos->getParent();
-  BasicBlock *Tail = Head->splitBasicBlock(Pos->getIterator());
-
-  if (DomTreeNode *OldNode = DT.getNode(Head)) {
-    std::vector<DomTreeNode *> Children(OldNode->begin(), OldNode->end());
-
-    DomTreeNode *NewNode = DT.addNewBlock(Tail, Head);
-    for (auto *Child : Children)
-      DT.changeImmediateDominator(Child, NewNode);
-  }
-
-  // In the following code LastBr will refer to the previous basic block's
-  // conditional branch instruction, whose true successor is fixed up to point
-  // to the next block during the loop below or to the tail after the final
-  // iteration.
-  BranchInst *LastBr = BranchInst::Create(FallbackBB, FallbackBB, ShadowsEq);
-  ReplaceInstWithInst(Head->getTerminator(), LastBr);
-  DT.addNewBlock(FallbackBB, Head);
-
-  const uint64_t BytesPerWideShadow = WideShadowBitWidth / DFS.ShadowWidthBits;
-
-  for (uint64_t ByteOfs = BytesPerWideShadow; ByteOfs < Size;
-       ByteOfs += BytesPerWideShadow) {
-    BasicBlock *NextBB = BasicBlock::Create(*DFS.Ctx, "", F);
-    DT.addNewBlock(NextBB, LastBr->getParent());
-    IRBuilder<> NextIRB(NextBB);
-    WideAddr = NextIRB.CreateGEP(WideShadowTy, WideAddr,
-                                 ConstantInt::get(DFS.IntptrTy, 1));
-    Value *NextWideShadow =
-        NextIRB.CreateAlignedLoad(WideShadowTy, WideAddr, ShadowAlign);
-    ShadowsEq = NextIRB.CreateICmpEQ(WideShadow, NextWideShadow);
-    LastBr->setSuccessor(0, NextBB);
-    LastBr = NextIRB.CreateCondBr(ShadowsEq, FallbackBB, FallbackBB);
-  }
-
-  LastBr->setSuccessor(0, Tail);
-  FallbackIRB.CreateBr(Tail);
-  PHINode *Shadow =
-      PHINode::Create(DFS.PrimitiveShadowTy, 2, "", &Tail->front());
-  Shadow->addIncoming(FallbackCall, FallbackBB);
-  Shadow->addIncoming(TruncShadow, LastBr->getParent());
-  return Shadow;
-}
-
 std::pair<Value *, Value *> DFSanFunction::loadShadowOriginSansLoadTracking(
     Value *Addr, uint64_t Size, Align InstAlignment, Instruction *Pos) {
   const bool ShouldTrackOrigins = DFS.shouldTrackOrigins();
@@ -2369,21 +2138,14 @@ std::pair<Value *, Value *> DFSanFunction::loadShadowOriginSansLoadTracking(
   }
   }
   bool HasSizeForFastPath = DFS.hasLoadSizeForFastPath(Size);
-  bool HasFastLabelsEnabled = DFS.hasFastLabelsEnabled();
 
-  if (HasFastLabelsEnabled && HasSizeForFastPath)
+  if (HasSizeForFastPath)
-    return loadFast16ShadowFast(ShadowAddr, OriginAddr, Size, ShadowAlign,
+    return loadShadowFast(ShadowAddr, OriginAddr, Size, ShadowAlign,
-                                OriginAlign, Origin, Pos);
+                          OriginAlign, Origin, Pos);
-
-  if (!AvoidNewBlocks && HasSizeForFastPath)
-    return {loadLegacyShadowFast(ShadowAddr, Size, ShadowAlign, Pos), Origin};
 
   IRBuilder<> IRB(Pos);
-  FunctionCallee &UnionLoadFn = HasFastLabelsEnabled
-                                    ? DFS.DFSanUnionLoadFastLabelsFn
-                                    : DFS.DFSanUnionLoadFn;
   CallInst *FallbackCall = IRB.CreateCall(
-      UnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
+      DFS.DFSanUnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
   FallbackCall->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
   return {FallbackCall, Origin};
 }

test/Instrumentation/DataFlowSanitizer/abilist.ll

@@ -1,6 +1,4 @@
 ; RUN: opt < %s -dfsan -dfsan-args-abi -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-args-abi -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-args-abi -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 

test/Instrumentation/DataFlowSanitizer/abilist_aggregate.ll

@@ -1,6 +1,4 @@
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s --check-prefixes=CHECK,TLS_ABI
+; RUN: opt < %s -dfsan -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s --check-prefixes=CHECK,TLS_ABI
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s --check-prefixes=CHECK,TLS_ABI
-; RUN: opt < %s -dfsan -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s --check-prefixes=CHECK,LEGACY
 ; RUN: opt < %s -dfsan -dfsan-args-abi -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s --check-prefixes=CHECK,ARGS_ABI
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -33,19 +31,12 @@ define {i1, i7} @call_functional({i32, i1} %a, [2 x i7] %b) {
   ; TLS_ABI-NEXT: %[[#REG+10]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } %[[#REG+9]], i[[#SBITS]] %[[#REG+8]], 1
   ; TLS_ABI: store { i[[#SBITS]], i[[#SBITS]] } %[[#REG+10]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
 
-  ; LEGACY: @"dfs$call_functional"
+  ; ARGS_ABI: @"dfs$call_functional"({ i32, i1 } %0, [2 x i7] %1, i[[#SBITS]] %2, i[[#SBITS]] %3)
-  ; LEGACY: [[B:%.*]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN:2]]
+  ; ARGS_ABI: %[[#U:]]  = or i[[#SBITS]] %2, %3
-  ; LEGACY: [[A:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; ARGS_ABI: %r = call { i1, i7 } @functional({ i32, i1 } %0, [2 x i7] %1)
-  ; LEGACY: [[U:%.*]] = call zeroext i[[#SBITS]] @__dfsan_union(i[[#SBITS]] zeroext [[A]], i[[#SBITS]] zeroext [[B]])
+  ; ARGS_ABI: %[[#R:]] = insertvalue { { i1, i7 }, i[[#SBITS]] } undef, { i1, i7 } %r, 0
-  ; LEGACY: [[PH:%.*]] = phi i[[#SBITS]] [ [[U]], {{.*}} ], [ [[A]], {{.*}} ]
+  ; ARGS_ABI: %[[#R+1]] = insertvalue { { i1, i7 }, i[[#SBITS]] } %[[#R]], i[[#SBITS]] %[[#U]], 1
-  ; LEGACY: store i[[#SBITS]] [[PH]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; ARGS_ABI: ret { { i1, i7 }, i[[#SBITS]] } %[[#R+1]]
-
-  ; ARGS_ABI: @"dfs$call_functional"
-  ; ARGS_ABI: [[U:%.*]]  = call zeroext i[[#SBITS]] @__dfsan_union(i[[#SBITS]] zeroext %2, i[[#SBITS]] zeroext %3)
-  ; ARGS_ABI: [[PH:%.*]] = phi i[[#SBITS]] [ %7, {{.*}} ], [ %2, {{.*}} ]
-  ; ARGS_ABI: [[R0:%.*]] = insertvalue { { i1, i7 }, i[[#SBITS]] } undef, { i1, i7 } %r, 0
-  ; ARGS_ABI: [[R1:%.*]] = insertvalue { { i1, i7 }, i[[#SBITS]] } [[R0]], i[[#SBITS]] [[PH]], 1
-  ; ARGS_ABI: ret { { i1, i7 }, i[[#SBITS]] } [[R1]]
 
   %r = call {i1, i7} @functional({i32, i1} %a, [2 x i7] %b)
   ret {i1, i7} %r

test/Instrumentation/DataFlowSanitizer/arith.ll

@@ -7,13 +7,12 @@ target triple = "x86_64-unknown-linux-gnu"
 
 define i8 @add(i8 %a, i8 %b) {
   ; CHECK: @"dfs$add"
-  ; CHECK-DAG: %[[ALABEL:.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[ARGTLSTYPE:\[100 x i64\]]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
+  ; CHECK-DAG: %[[#ALABEL:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[ARGTLSTYPE:\[100 x i64\]]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
-  ; CHECK-DAG: %[[BLABEL:.*]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[ARGTLSTYPE]]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN]]
+  ; CHECK-DAG: %[[#BLABEL:]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[ARGTLSTYPE]]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN]]
-  ; CHECK: %[[UNION:.*]] = call zeroext i[[#SBITS]] @__dfsan_union(i[[#SBITS]] zeroext %[[ALABEL]], i[[#SBITS]] zeroext %[[BLABEL]])
+  ; CHECK: %[[#UNION:]] = or i[[#SBITS]] %[[#ALABEL]], %[[#BLABEL]]
-  ; CHECK: %[[ADDLABEL:.*]] = phi i[[#SBITS]] [ %[[UNION]], {{.*}} ], [ %[[ALABEL]], {{.*}} ]
+  ; CHECK: %c = add i8 %a, %b
-  ; CHECK: add i8
+  ; CHECK: store i[[#SBITS]] %[[#UNION]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; CHECK: store i[[#SBITS]] %[[ADDLABEL]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; CHECK: ret i8 %c
-  ; CHECK: ret i8
   %c = add i8 %a, %b
   ret i8 %c
 }
@@ -22,10 +21,10 @@ define i8 @sub(i8 %a, i8 %b) {
   ; CHECK: @"dfs$sub"
   ; CHECK: load{{.*}}__dfsan_arg_tls
   ; CHECK: load{{.*}}__dfsan_arg_tls
-  ; CHECK: call{{.*}}__dfsan_union
+  ; CHECK: or i[[#SBITS]]
-  ; CHECK: sub i8
+  ; CHECK: %c = sub i8 %a, %b
   ; CHECK: store{{.*}}__dfsan_retval_tls
-  ; CHECK: ret i8
+  ; CHECK: ret i8 %c
   %c = sub i8 %a, %b
   ret i8 %c
 }
@@ -34,10 +33,10 @@ define i8 @mul(i8 %a, i8 %b) {
   ; CHECK: @"dfs$mul"
   ; CHECK: load{{.*}}__dfsan_arg_tls
   ; CHECK: load{{.*}}__dfsan_arg_tls
-  ; CHECK: call{{.*}}__dfsan_union
+  ; CHECK: or i[[#SBITS]]
-  ; CHECK: mul i8
+  ; CHECK: %c = mul i8 %a, %b
   ; CHECK: store{{.*}}__dfsan_retval_tls
-  ; CHECK: ret i8
+  ; CHECK: ret i8 %c
   %c = mul i8 %a, %b
   ret i8 %c
 }
@@ -46,10 +45,10 @@ define i8 @sdiv(i8 %a, i8 %b) {
   ; CHECK: @"dfs$sdiv"
   ; CHECK: load{{.*}}__dfsan_arg_tls
   ; CHECK: load{{.*}}__dfsan_arg_tls
-  ; CHECK: call{{.*}}__dfsan_union
+  ; CHECK: or i[[#SBITS]]
-  ; CHECK: sdiv i8
+  ; CHECK: %c = sdiv i8 %a, %b
   ; CHECK: store{{.*}}__dfsan_retval_tls
-  ; CHECK: ret i8
+  ; CHECK: ret i8 %c
   %c = sdiv i8 %a, %b
   ret i8 %c
 }
@@ -58,10 +57,10 @@ define i8 @udiv(i8 %a, i8 %b) {
   ; CHECK: @"dfs$udiv"
   ; CHECK: load{{.*}}__dfsan_arg_tls
   ; CHECK: load{{.*}}__dfsan_arg_tls
-  ; CHECK: call{{.*}}__dfsan_union
+  ; CHECK: or i[[#SBITS]]
-  ; CHECK: udiv i8
+  ; CHECK: %c = udiv i8 %a, %b
   ; CHECK: store{{.*}}__dfsan_retval_tls
-  ; CHECK: ret i8
+  ; CHECK: ret i8 %c
   %c = udiv i8 %a, %b
   ret i8 %c
 }
@@ -69,9 +68,9 @@ define i8 @udiv(i8 %a, i8 %b) {
 define double @fneg(double %a) {
   ; CHECK: @"dfs$fneg"
   ; CHECK: load{{.*}}__dfsan_arg_tls
-  ; CHECK: fneg double
+  ; CHECK: %c = fneg double %a
   ; CHECK: store{{.*}}__dfsan_retval_tls
-  ; CHECK: ret double
+  ; CHECK: ret double %c
   %c = fneg double %a
   ret double %c
 }

test/Instrumentation/DataFlowSanitizer/array.ll

@@ -1,15 +1,9 @@
-; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,LEGACY
+; RUN: opt < %s -dfsan -dfsan-event-callbacks=true -S | FileCheck %s --check-prefixes=CHECK,EVENT_CALLBACKS
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-event-callbacks=true -S | FileCheck %s --check-prefixes=CHECK,EVENT_CALLBACKS
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-event-callbacks=true -S | FileCheck %s --check-prefixes=CHECK,EVENT_CALLBACKS
 ; RUN: opt < %s -dfsan -dfsan-args-abi -S | FileCheck %s --check-prefixes=CHECK,ARGS_ABI
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,FAST
+; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,FAST
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s --check-prefixes=CHECK,NO_COMBINE_LOAD_PTR
+; RUN: opt < %s -dfsan -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s --check-prefixes=CHECK,NO_COMBINE_LOAD_PTR
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-combine-pointer-labels-on-store=true -S | FileCheck %s --check-prefixes=CHECK,COMBINE_STORE_PTR
+; RUN: opt < %s -dfsan -dfsan-combine-pointer-labels-on-store=true -S | FileCheck %s --check-prefixes=CHECK,COMBINE_STORE_PTR
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-debug-nonzero-labels -S | FileCheck %s --check-prefixes=CHECK,DEBUG_NONZERO_LABELS
+; RUN: opt < %s -dfsan -dfsan-debug-nonzero-labels -S | FileCheck %s --check-prefixes=CHECK,DEBUG_NONZERO_LABELS
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -S | FileCheck %s --check-prefixes=CHECK,FAST
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s --check-prefixes=CHECK,NO_COMBINE_LOAD_PTR
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-combine-pointer-labels-on-store=true -S | FileCheck %s --check-prefixes=CHECK,COMBINE_STORE_PTR
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-debug-nonzero-labels -S | FileCheck %s --check-prefixes=CHECK,DEBUG_NONZERO_LABELS
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -102,13 +96,6 @@ define [1 x i1] @load_array1([1 x i1]* %p) {
   ; FAST: [[S1:%.*]] = insertvalue [1 x i[[#SBITS]]] undef, i[[#SBITS]] [[U]], 0
   ; FAST: store [1 x i[[#SBITS]]] [[S1]], [1 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [1 x i[[#SBITS]]]*), align [[ALIGN]]
 
-  ; LEGACY: @"dfs$load_array1"
-  ; LEGACY: [[P:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
-  ; LEGACY: [[L:%.*]] = load i[[#SBITS]], i[[#SBITS]]* {{.*}}, align [[#SBYTES]]
-  ; LEGACY: [[U:%.*]] = call zeroext i[[#SBITS]] @__dfsan_union(i[[#SBITS]] zeroext [[L]], i[[#SBITS]] zeroext [[P]])
-  ; LEGACY: [[PH:%.*]] = phi i[[#SBITS]] [ [[U]], {{.*}} ], [ [[L]], {{.*}} ]
-  ; LEGACY: store i[[#SBITS]] [[PH]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
-
   %a = load [1 x i1], [1 x i1]* %p
   ret [1 x i1] %a
 }
@@ -164,13 +151,6 @@ define [4 x i1] @load_array4([4 x i1]* %p) {
   ; FAST: [[S4:%.*]] = insertvalue [4 x i[[#SBITS]]] [[S3]], i[[#SBITS]] [[O]], 3
   ; FAST: store [4 x i[[#SBITS]]] [[S4]], [4 x i[[#SBITS]]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to [4 x i[[#SBITS]]]*), align 2
 
-  ; LEGACY: @"dfs$load_array4"
-  ; LEGACY: [[P:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
-  ; LEGACY: [[PH1:%.*]] = phi i[[#SBITS]]
-  ; LEGACY: [[U:%.*]] = call zeroext i[[#SBITS]] @__dfsan_union(i[[#SBITS]] zeroext [[PH1]], i[[#SBITS]] zeroext [[P]])
-  ; LEGACY: [[PH:%.*]] = phi i[[#SBITS]] [ [[U]], {{.*}} ], [ [[PH1]], {{.*}} ]
-  ; LEGACY: store i[[#SBITS]] [[PH]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
-
   %a = load [4 x i1], [4 x i1]* %p
   ret [4 x i1] %a
 }
@@ -220,13 +200,6 @@ define void @store_zero_array([4 x i1]* %p) {
 }
 
 define void @store_array2([2 x i1] %a, [2 x i1]* %p) {
-  ; LEGACY: @"dfs$store_array2"
-  ; LEGACY: [[S:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
-  ; LEGACY: [[SP0:%.*]] = getelementptr i[[#SBITS]], i[[#SBITS]]* [[SP:%.*]], i32 0
-  ; LEGACY: store i[[#SBITS]] [[S]], i[[#SBITS]]* [[SP0]], align [[#SBYTES]]
-  ; LEGACY: [[SP1:%.*]] = getelementptr i[[#SBITS]], i[[#SBITS]]* [[SP]], i32 1
-  ; LEGACY: store i[[#SBITS]] [[S]], i[[#SBITS]]* [[SP1]], align [[#SBYTES]]
-
   ; EVENT_CALLBACKS: @"dfs$store_array2"
   ; EVENT_CALLBACKS: [[E12:%.*]] = or i[[#SBITS]]
   ; EVENT_CALLBACKS: [[P:%.*]] = bitcast [2 x i1]* %p to i8*

test/Instrumentation/DataFlowSanitizer/atomics.ll

@@ -1,9 +1,6 @@
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -S | FileCheck %s
+; RUN: opt < %s -dfsan -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,CHECK16
+; RUN: opt < %s -dfsan -dfsan-track-origins=1 -S | FileCheck %s --check-prefixes=CHECK,CHECK_ORIGIN
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-8-labels=true -S | FileCheck %s --check-prefixes=CHECK,CHECK_ORIGIN
+; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-instrument-with-call-threshold=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK_ORIGIN
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-8-labels=true -dfsan-instrument-with-call-threshold=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK_ORIGIN
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,CHECK16,CHECK_ORIGIN
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels=true -dfsan-instrument-with-call-threshold=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK16,CHECK_ORIGIN
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -21,7 +18,6 @@ entry:
   ; CHECK-NOT:         @__dfsan_arg_tls
   ; CHECK:             %[[#INTP:]] = ptrtoint i32* %p to i64
   ; CHECK-NEXT:        %[[#SHADOW_ADDR:INTP+1]] = and i64 %[[#INTP]], [[#%.10d,MASK:]]
-  ; CHECK16-NEXT:      %[[#SHADOW_ADDR:INTP+2]] = mul i64 %[[#INTP+1]], 2
   ; CHECK-NEXT:        %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
   ; CHECK-NEXT:        %[[#SHADOW_PTR64:]] = bitcast i[[#SBITS]]* %[[#SHADOW_PTR]] to i[[#NUM_BITS:mul(SBITS,4)]]*
   ; CHECK-NEXT:        store i[[#NUM_BITS]] 0, i[[#NUM_BITS]]* %[[#SHADOW_PTR64]], align [[#SBYTES]]
@@ -42,7 +38,6 @@ define i32 @AtomicRmwMax(i32* %p, i32 %x) {
   ; CHECK-NOT:         @__dfsan_arg_tls
   ; CHECK:             %[[#INTP:]] = ptrtoint i32* %p to i64
   ; CHECK-NEXT:        %[[#SHADOW_ADDR:INTP+1]] = and i64 %[[#INTP]], [[#%.10d,MASK:]]
-  ; CHECK16-NEXT:      %[[#SHADOW_ADDR:INTP+2]] = mul i64 %[[#INTP+1]], 2
   ; CHECK-NEXT:        %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
   ; CHECK-NEXT:        %[[#SHADOW_PTR64:]] = bitcast i[[#SBITS]]* %[[#SHADOW_PTR]] to i[[#NUM_BITS:mul(SBITS,4)]]*
   ; CHECK-NEXT:        store i[[#NUM_BITS]] 0, i[[#NUM_BITS]]* %[[#SHADOW_PTR64]], align [[#SBYTES]]
@@ -65,7 +60,6 @@ define i32 @Cmpxchg(i32* %p, i32 %a, i32 %b) {
   ; CHECK-NOT:         @__dfsan_arg_tls
   ; CHECK:             %[[#INTP:]] = ptrtoint i32* %p to i64
   ; CHECK-NEXT:        %[[#SHADOW_ADDR:INTP+1]] = and i64 %[[#INTP]], [[#%.10d,MASK:]]
-  ; CHECK16-NEXT:      %[[#SHADOW_ADDR:INTP+2]] = mul i64 %[[#INTP+1]], 2
   ; CHECK-NEXT:        %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
   ; CHECK-NEXT:        %[[#SHADOW_PTR64:]] = bitcast i[[#SBITS]]* %[[#SHADOW_PTR]] to i[[#NUM_BITS:mul(SBITS,4)]]*
   ; CHECK-NEXT:        store i[[#NUM_BITS]] 0, i[[#NUM_BITS]]* %[[#SHADOW_PTR64]], align [[#SBYTES]]
@@ -89,7 +83,6 @@ define i32 @CmpxchgMonotonic(i32* %p, i32 %a, i32 %b) {
   ; CHECK-NOT:         @__dfsan_arg_tls
   ; CHECK:             %[[#INTP:]] = ptrtoint i32* %p to i64
   ; CHECK-NEXT:        %[[#SHADOW_ADDR:INTP+1]] = and i64 %[[#INTP]], [[#%.10d,MASK:]]
-  ; CHECK16-NEXT:      %[[#SHADOW_ADDR:INTP+2]] = mul i64 %[[#INTP+1]], 2
   ; CHECK-NEXT:        %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
   ; CHECK-NEXT:        %[[#SHADOW_PTR64:]] = bitcast i[[#SBITS]]* %[[#SHADOW_PTR]] to i[[#NUM_BITS:mul(SBITS,4)]]*
   ; CHECK-NEXT:        store i[[#NUM_BITS]] 0, i[[#NUM_BITS]]* %[[#SHADOW_PTR64]], align [[#SBYTES]]
@@ -213,7 +206,6 @@ define void @AtomicStore(i32* %p, i32 %x) {
   ; CHECK_ORIGIN-NOT:  35184372088832
   ; CHECK:             %[[#INTP:]] = ptrtoint i32* %p to i64
   ; CHECK-NEXT:        %[[#SHADOW_ADDR:INTP+1]] = and i64 %[[#INTP]], [[#%.10d,MASK:]]
-  ; CHECK16-NEXT:      %[[#SHADOW_ADDR:INTP+2]] = mul i64 %[[#INTP+1]], 2
   ; CHECK-NEXT:        %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
   ; CHECK-NEXT:        %[[#SHADOW_PTR64:]] = bitcast i[[#SBITS]]* %[[#SHADOW_PTR]] to i[[#NUM_BITS:mul(SBITS,4)]]*
   ; CHECK-NEXT:        store i[[#NUM_BITS]] 0, i[[#NUM_BITS]]* %[[#SHADOW_PTR64]], align [[#SBYTES]]
@@ -234,7 +226,6 @@ define void @AtomicStoreRelease(i32* %p, i32 %x) {
   ; CHECK_ORIGIN-NOT:  35184372088832
   ; CHECK:             %[[#INTP:]] = ptrtoint i32* %p to i64
   ; CHECK-NEXT:        %[[#SHADOW_ADDR:INTP+1]] = and i64 %[[#INTP]], [[#%.10d,MASK:]]
-  ; CHECK16-NEXT:      %[[#SHADOW_ADDR:INTP+2]] = mul i64 %[[#INTP+1]], 2
   ; CHECK-NEXT:        %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
   ; CHECK-NEXT:        %[[#SHADOW_PTR64:]] = bitcast i[[#SBITS]]* %[[#SHADOW_PTR]] to i[[#NUM_BITS:mul(SBITS,4)]]*
   ; CHECK-NEXT:        store i[[#NUM_BITS]] 0, i[[#NUM_BITS]]* %[[#SHADOW_PTR64]], align [[#SBYTES]]
@@ -255,7 +246,6 @@ define void @AtomicStoreMonotonic(i32* %p, i32 %x) {
   ; CHECK_ORIGIN-NOT:  35184372088832
   ; CHECK:             %[[#INTP:]] = ptrtoint i32* %p to i64
   ; CHECK-NEXT:        %[[#SHADOW_ADDR:INTP+1]] = and i64 %[[#INTP]], [[#%.10d,MASK:]]
-  ; CHECK16-NEXT:      %[[#SHADOW_ADDR:INTP+2]] = mul i64 %[[#INTP+1]], 2
   ; CHECK-NEXT:        %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
   ; CHECK-NEXT:        %[[#SHADOW_PTR64:]] = bitcast i[[#SBITS]]* %[[#SHADOW_PTR]] to i[[#NUM_BITS:mul(SBITS,4)]]*
   ; CHECK-NEXT:        store i[[#NUM_BITS]] 0, i[[#NUM_BITS]]* %[[#SHADOW_PTR64]], align [[#SBYTES]]
@@ -276,7 +266,6 @@ define void @AtomicStoreUnordered(i32* %p, i32 %x) {
   ; CHECK_ORIGIN-NOT:  35184372088832
   ; CHECK:             %[[#INTP:]] = ptrtoint i32* %p to i64
   ; CHECK-NEXT:        %[[#SHADOW_ADDR:INTP+1]] = and i64 %[[#INTP]], [[#%.10d,MASK:]]
-  ; CHECK16-NEXT:      %[[#SHADOW_ADDR:INTP+2]] = mul i64 %[[#INTP+1]], 2
   ; CHECK-NEXT:        %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
   ; CHECK-NEXT:        %[[#SHADOW_PTR64:]] = bitcast i[[#SBITS]]* %[[#SHADOW_PTR]] to i[[#NUM_BITS:mul(SBITS,4)]]*
   ; CHECK-NEXT:        store i[[#NUM_BITS]] 0, i[[#NUM_BITS]]* %[[#SHADOW_PTR64]], align [[#SBYTES]]

test/Instrumentation/DataFlowSanitizer/basic.ll

@@ -1,6 +1,5 @@
-; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,CHECK_NO_ORIGIN -DSHADOW_MASK=-123145302310913
+; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,CHECK_NO_ORIGIN -DSHADOW_MASK=-105553116266497   --dump-input-context=100
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,CHECK_ORIGIN -DSHADOW_MASK=-123145302310913
+; RUN: opt < %s -dfsan -dfsan-track-origins=1  -S | FileCheck %s --check-prefixes=CHECK,CHECK_ORIGIN -DSHADOW_MASK=-105553116266497  --dump-input-context=100
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-8-labels=true  -S | FileCheck %s --check-prefixes=CHECK,CHECK_ORIGIN -DSHADOW_MASK=-105553116266497
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -35,20 +34,11 @@ define void @store(i8* %p) {
 ; CHECK: declare void @__dfsan_mem_transfer_callback(i[[#SBITS]]*, i64)
 ; CHECK: declare void @__dfsan_cmp_callback(i[[#SBITS]])
 
-; CHECK: ; Function Attrs: nounwind readnone
+; CHECK: ; Function Attrs: nounwind readonly
-; CHECK-NEXT: declare zeroext i[[#SBITS]] @__dfsan_union(i[[#SBITS]] zeroext, i[[#SBITS]] zeroext) #0
+; CHECK-NEXT: declare zeroext i[[#SBITS]] @__dfsan_union_load(i[[#SBITS]]*, i64)
-
-; CHECK: ; Function Attrs: nounwind readnone
-; CHECK-NEXT: declare zeroext i[[#SBITS]] @dfsan_union(i[[#SBITS]] zeroext, i[[#SBITS]] zeroext) #0
 
 ; CHECK: ; Function Attrs: nounwind readonly
-; CHECK-NEXT: declare zeroext i[[#SBITS]] @__dfsan_union_load(i[[#SBITS]]*, i64) #1
+; CHECK-NEXT: declare zeroext i64 @__dfsan_load_label_and_origin(i8*, i64)
-
-; CHECK: ; Function Attrs: nounwind readonly
-; CHECK-NEXT: declare zeroext i[[#SBITS]] @__dfsan_union_load_fast16labels(i[[#SBITS]]*, i64) #1
-
-; CHECK: ; Function Attrs: nounwind readonly
-; CHECK-NEXT: declare zeroext i64 @__dfsan_load_label_and_origin(i8*, i64) #1
 
 ; CHECK: declare void @__dfsan_unimplemented(i8*)
 ; CHECK: declare void @__dfsan_set_label(i[[#SBITS]] zeroext, i32 zeroext, i8*, i64)

test/Instrumentation/DataFlowSanitizer/call.ll

@@ -1,6 +1,4 @@
 ; RUN: opt < %s -dfsan -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels -S | FileCheck %s
 ; RUN: opt < %s -passes=dfsan -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"

test/Instrumentation/DataFlowSanitizer/dont_combine_offset_labels_on_gep.ll

@@ -1,5 +1,5 @@
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-combine-offset-labels-on-gep=false -dfsan-fast-8-labels=true -S | FileCheck %s --check-prefixes=CHECK,CHECK_ORIGIN
+; RUN: opt < %s -dfsan -dfsan-combine-offset-labels-on-gep=false -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-combine-offset-labels-on-gep=false -dfsan-fast-8-labels=true -S | FileCheck %s
+; RUN: opt < %s -dfsan -dfsan-combine-offset-labels-on-gep=false -dfsan-track-origins=1 -S | FileCheck %s --check-prefixes=CHECK,CHECK_ORIGIN
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 

test/Instrumentation/DataFlowSanitizer/external_mask.ll

@@ -1,15 +1,12 @@
-; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,CHECK16
+; RUN: opt < %s -dfsan -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,CHECK16
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -S | FileCheck %s --check-prefixes=CHECK
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-gnu"
 
 define i32 @test(i32 %a, i32* nocapture readonly %b) #0 {
-; CHECK: @"dfs$test"
+  ; CHECK: @"dfs$test"
-; CHECK: %[[RV:.*]] load{{.*}}__dfsan_shadow_ptr_mask
+  ; CHECK: %[[RV:.*]] load{{.*}}__dfsan_shadow_ptr_mask
-; CHECK: ptrtoint i32* {{.*}} to i64
+  ; CHECK: ptrtoint i32* {{.*}} to i64
-; CHECK: and {{.*}}%[[RV:.*]]
+  ; CHECK: and {{.*}}%[[RV:.*]]
-; CHECK16: mul i64
   %1 = load i32, i32* %b, align 4
   %2 = add nsw i32 %1, %a
   ret i32 %2

test/Instrumentation/DataFlowSanitizer/fast16labels.ll

@@ -1,172 +0,0 @@
-; Test that -dfsan-fast-16-labels mode uses inline ORs rather than calling
-; __dfsan_union or __dfsan_union_load.
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels -S | FileCheck %s --implicit-check-not="call{{.*}}__dfsan_union" --check-prefixes=CHECK,CHECK16
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels -S | FileCheck %s --implicit-check-not="call{{.*}}__dfsan_union" --check-prefixes=CHECK,CHECK8
-target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-; CHECK: @__dfsan_arg_tls = external thread_local(initialexec) global [[TLS_ARR:\[100 x i64\]]]
-; CHECK: @__dfsan_retval_tls = external thread_local(initialexec) global [[TLS_ARR]]
-; CHECK: @__dfsan_shadow_width_bits = weak_odr constant i32 [[#SBITS:]]
-; CHECK: @__dfsan_shadow_width_bytes = weak_odr constant i32 [[#SBYTES:]]
-
-define i8 @add(i8 %a, i8 %b) {
-  ; CHECK-LABEL: define i8 @"dfs$add"
-  ; CHECK-DAG: %[[ALABEL:.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
-  ; CHECK-DAG: %[[BLABEL:.*]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN]]
-  ; CHECK: %[[ADDLABEL:.*]] = or i[[#SBITS]] %[[ALABEL]], %[[BLABEL]]
-  ; CHECK: %c = add i8 %a, %b
-  ; CHECK: store i[[#SBITS]] %[[ADDLABEL]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; CHECK: ret i8 %c
-  %c = add i8 %a, %b
-  ret i8 %c
-}
-
-define i8 @load8(i8* %p) {
-  ; CHECK-LABEL:  define i8 @"dfs$load8"
-  ; CHECK-SAME:   (i8* %[[PADDR:.*]])
-  ; CHECK-NEXT:   %[[#ARG:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; CHECK-NEXT:   %[[#R:]] = ptrtoint i8* %[[PADDR]] to i64
-  ; CHECK-NEXT:   %[[#PS:R+1]] = and i64 %[[#R]], [[#%.10d,MASK:]]
-  ; CHECK16-NEXT: %[[#PS:R+2]] = mul i64 %[[#R+1]], 2
-  ; CHECK-NEXT:   %[[#SADDR:]] = inttoptr i64 %[[#PS]] to i[[#SBITS]]*
-  ; CHECK-NEXT:   %[[#S:]] = load i[[#SBITS]], i[[#SBITS]]* %[[#SADDR]]
-  ; CHECK-NEXT:   %[[#S_OUT:S+1]] = or i[[#SBITS]] %[[#S]], %[[#ARG]]
-  ; CHECK-NEXT:   %a = load i8, i8* %p
-  ; CHECK-NEXT:   store i[[#SBITS]] %[[#S_OUT]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; CHECK-NEXT:   ret i8 %a
-
-  %a = load i8, i8* %p
-  ret i8 %a
-}
-
-define i16 @load16(i16* %p) {
-  ; CHECK-LABEL:  define i16 @"dfs$load16"
-  ; CHECK-SAME:   (i16* %[[PADDR:.*]])
-  ; CHECK-NEXT:   %[[#ARG:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; CHECK-NEXT:   %[[#R:]] = ptrtoint i16* %[[PADDR]] to i64
-  ; CHECK-NEXT:   %[[#PS:R+1]] = and i64 %[[#R]], [[#%.10d,MASK:]]
-  ; CHECK16-NEXT: %[[#PS:R+2]] = mul i64 %[[#R+1]], 2
-  ; CHECK-NEXT:   %[[#SADDR:]]  = inttoptr i64 %[[#PS]] to i[[#SBITS]]*
-  ; CHECK-NEXT:   %[[#SADDR+1]] = getelementptr i[[#SBITS]], i[[#SBITS]]* %[[#SADDR]], i64 1
-  ; CHECK-NEXT:   %[[#S:]]  = load i[[#SBITS]], i[[#SBITS]]* %[[#SADDR]]
-  ; CHECK-NEXT:   %[[#S+1]] = load i[[#SBITS]], i[[#SBITS]]* %[[#SADDR+1]]
-  ; CHECK-NEXT:   %[[#S+2]] = or i[[#SBITS]] %[[#S]], %[[#S+1]]
-  ; CHECK-NEXT:   %[[#S_OUT:S+3]] = or i[[#SBITS]] %[[#S+2]], %[[#ARG]]
-  ; CHECK-NEXT:   %a = load i16, i16* %p
-  ; CHECK-NEXT:   store i[[#SBITS]] %[[#S_OUT]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; CHECK-NEXT:   ret i16 %a
-
-  %a = load i16, i16* %p
-  ret i16 %a
-}
-
-define i32 @load32(i32* %p) {
-  ; CHECK-LABEL: define i32 @"dfs$load32"
-  ; CHECK-SAME:   (i32* %[[PADDR:.*]])
-  ; CHECK-NEXT:   %[[#ARG:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; CHECK-NEXT:   %[[#R:]] = ptrtoint i32* %[[PADDR]] to i64
-  ; CHECK-NEXT:   %[[#PS:R+1]] = and i64 %[[#R]], [[#%.10d,MASK:]]
-  ; CHECK16-NEXT: %[[#PS:R+2]] = mul i64 %[[#R+1]], 2
-  ; CHECK-NEXT:   %[[#SADDR:]] = inttoptr i64 %[[#PS]] to i[[#SBITS]]*
-  ; CHECK-NEXT:   %[[#SADDR+1]] = bitcast i[[#SBITS]]* %[[#SADDR]] to i[[#WBITS:mul(SBITS,4)]]*
-  ; CHECK-NEXT:   %[[#WS:]]  = load i[[#WBITS]], i[[#WBITS]]* %[[#SADDR+1]]
-  ; CHECK-NEXT:   %[[#WS+1]] = lshr i[[#WBITS]] %[[#WS]], [[#mul(SBITS,2)]]
-  ; CHECK-NEXT:   %[[#WS+2]] = or i[[#WBITS]] %[[#WS]], %[[#WS+1]]
-  ; CHECK-NEXT:   %[[#WS+3]] = lshr i[[#WBITS]] %[[#WS+2]], [[#SBITS]]
-  ; CHECK-NEXT:   %[[#WS+4]] = or i[[#WBITS]] %[[#WS+2]], %[[#WS+3]]
-  ; CHECK-NEXT:   %[[#WS+5]] = trunc i[[#WBITS]] %[[#WS+4]] to i[[#SBITS]]
-  ; CHECK-NEXT:   %[[#S_OUT:WS+6]] = or i[[#SBITS]] %[[#WS+5]], %[[#ARG]]
-  ; CHECK-NEXT:   %a = load i32, i32* %p
-  ; CHECK-NEXT:   store i[[#SBITS]] %[[#S_OUT]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; CHECK-NEXT:   ret i32 %a
-
-  %a = load i32, i32* %p
-  ret i32 %a
-}
-
-define i64 @load64(i64* %p) {
-  ; CHECK-LABEL:  define i64 @"dfs$load64"
-  ; CHECK-SAME:   (i64* %[[PADDR:.*]])
-  ; CHECK-NEXT:   %[[#ARG:]]        = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; CHECK-NEXT:   %[[#R:]]          = ptrtoint i64* %[[PADDR]] to i64
-  ; CHECK-NEXT:   %[[#PS:R+1]]      = and i64 %[[#R]], [[#%.10d,MASK:]]
-  ; CHECK16-NEXT: %[[#PS:R+2]]      = mul i64 %[[#R+1]], 2
-  ; CHECK-NEXT:   %[[#SADDR:]]      = inttoptr i64 %[[#PS]] to i[[#SBITS]]*
-  ; CHECK-NEXT:   %[[#SADDR+1]]     = bitcast i[[#SBITS]]* %[[#SADDR]] to i64*
-  ; CHECK-NEXT:   %[[#WS:]]         = load i64, i64* %[[#SADDR+1]]
-
-  ; COMM: On fast16, the 2x64 shadow bits need to be ORed first.
-  ; CHECK16-NEXT: %[[#SADDR_NEXT:]] = getelementptr i64, i64* %[[#SADDR+1]], i64 1
-  ; CHECK16-NEXT: %[[#WS_NEXT:]]    = load i64, i64* %[[#SADDR_NEXT]]
-  ; CHECK16-NEXT: %[[#WS:]]         = or i64 %[[#WS]], %[[#WS_NEXT]]
-  ; CHECK16-NEXT: %[[#WS+1]]        = lshr i64 %[[#WS]], 32
-  ; CHECK16-NEXT: %[[#WS+2]]        = or i64 %[[#WS]], %[[#WS+1]]
-  ; CHECK16-NEXT: %[[#WS+3]]        = lshr i64 %[[#WS+2]], 16
-  ; CHECK16-NEXT: %[[#WS+4]]        = or i64 %[[#WS+2]], %[[#WS+3]]
-  ; CHECK16-NEXT: %[[#WS+5]]        = trunc i64 %[[#WS+4]] to i[[#SBITS]]
-  ; CHECK16-NEXT: %[[#S_OUT:]]      = or i[[#SBITS]] %[[#WS+5]], %[[#ARG]]
-
-  ; COMM: On fast8, no need to OR the wide shadow but one more shift is needed.
-  ; CHECK8-NEXT: %[[#WS+1]]         = lshr i64 %[[#WS]], 32
-  ; CHECK8-NEXT: %[[#WS+2]]         = or i64 %[[#WS]], %[[#WS+1]]
-  ; CHECK8-NEXT: %[[#WS+3]]         = lshr i64 %[[#WS+2]], 16
-  ; CHECK8-NEXT: %[[#WS+4]]         = or i64 %[[#WS+2]], %[[#WS+3]]
-  ; CHECK8-NEXT: %[[#WS+5]]         = lshr i64 %[[#WS+4]], 8
-  ; CHECK8-NEXT: %[[#WS+6]]         = or i64 %[[#WS+4]], %[[#WS+5]]
-  ; CHECK8-NEXT: %[[#WS+7]]         = trunc i64 %[[#WS+6]] to i[[#SBITS]]
-  ; CHECK8-NEXT: %[[#S_OUT:]]       = or i[[#SBITS]] %[[#WS+7]], %[[#ARG]]
-
-  ; CHECK-NEXT:   %a = load i64, i64* %p
-  ; CHECK-NEXT:   store i[[#SBITS]] %[[#S_OUT]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; CHECK-NEXT:   ret i64 %a
-
-  %a = load i64, i64* %p
-  ret i64 %a
-}
-
-define i128 @load128(i128* %p) {
-  ; CHECK-LABEL:  define i128 @"dfs$load128"
-  ; CHECK-SAME:   (i128* %[[PADDR:.*]])
-  ; CHECK-NEXT:   %[[#ARG:]]    = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; CHECK-NEXT:   %[[#R:]]      = ptrtoint i128* %[[PADDR]] to i64
-  ; CHECK-NEXT:   %[[#PS:R+1]]  = and i64 %[[#R]], [[#%.10d,MASK:]]
-  ; CHECK16-NEXT: %[[#PS:R+2]]  = mul i64 %[[#R+1]], 2
-  ; CHECK-NEXT:   %[[#SADDR:]]  = inttoptr i64 %[[#PS]] to i[[#SBITS]]*
-  ; CHECK-NEXT:   %[[#SADDR+1]] = bitcast i[[#SBITS]]* %[[#SADDR]] to i64*
-  ; CHECK-NEXT:   %[[#S:]]      = load i64, i64* %[[#SADDR+1]]
-  ; CHECK-NEXT:   %[[#S+1]]     = getelementptr i64, i64* %[[#SADDR+1]], i64 1
-  ; CHECK-NEXT:   %[[#S+2]]     = load i64, i64* %[[#S+1]]
-  ; CHECK-NEXT:   %[[#WS:S+3]]  = or i64 %[[#S]], %[[#S+2]]
-
-  ; COMM: On fast16, we need to OR 4x64bits for the wide shadow, before ORing its bytes.
-  ; CHECK16-NEXT: %[[#S+4]]     = getelementptr i64, i64* %[[#S+1]], i64 1
-  ; CHECK16-NEXT: %[[#S+5]]     = load i64, i64* %[[#S+4]]
-  ; CHECK16-NEXT: %[[#S+6]]     = or i64 %[[#S+3]], %[[#S+5]]
-  ; CHECK16-NEXT: %[[#S+7]]     = getelementptr i64, i64* %[[#S+4]], i64 1
-  ; CHECK16-NEXT: %[[#S+8]]     = load i64, i64* %[[#S+7]]
-  ; CHECK16-NEXT: %[[#WS:S+9]]  = or i64 %[[#S+6]], %[[#S+8]]
-  ; CHECK16-NEXT: %[[#WS+1]]    = lshr i64 %[[#WS]], 32
-  ; CHECK16-NEXT: %[[#WS+2]]    = or i64 %[[#WS]], %[[#WS+1]]
-  ; CHECK16-NEXT: %[[#WS+3]]    = lshr i64 %[[#WS+2]], 16
-  ; CHECK16-NEXT: %[[#WS+4]]    = or i64 %[[#WS+2]], %[[#WS+3]]
-  ; CHECK16-NEXT: %[[#WS+5]]    = trunc i64 %[[#WS+4]] to i[[#SBITS]]
-  ; CHECK16-NEXT: %[[#S_OUT:]]  = or i[[#SBITS]] %[[#WS+5]], %[[#ARG]]
-
-  ; COMM: On fast8, we need to OR 2x64bits for the wide shadow, before ORing its bytes (one more shift).
-  ; CHECK8-NEXT: %[[#WS+1]]     = lshr i64 %[[#WS]], 32
-  ; CHECK8-NEXT: %[[#WS+2]]     = or i64 %[[#WS]], %[[#WS+1]]
-  ; CHECK8-NEXT: %[[#WS+3]]     = lshr i64 %[[#WS+2]], 16
-  ; CHECK8-NEXT: %[[#WS+4]]     = or i64 %[[#WS+2]], %[[#WS+3]]
-  ; CHECK8-NEXT: %[[#WS+5]]     = lshr i64 %[[#WS+4]], 8
-  ; CHECK8-NEXT: %[[#WS+6]]     = or i64 %[[#WS+4]], %[[#WS+5]]
-  ; CHECK8-NEXT: %[[#WS+7]]     = trunc i64 %[[#WS+6]] to i[[#SBITS]]
-  ; CHECK8-NEXT: %[[#S_OUT:]]   = or i[[#SBITS]] %[[#WS+7]], %[[#ARG]]
-
-  ; CHECK-NEXT: %a = load i128, i128* %p
-  ; CHECK-NEXT: store i[[#SBITS]] %[[#S_OUT]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; CHECK-NEXT: ret i128 %a
-
-  %a = load i128, i128* %p
-  ret i128 %a
-}

test/Instrumentation/DataFlowSanitizer/load.ll

@@ -1,176 +1,156 @@
-; RUN: opt < %s -dfsan -dfsan-combine-pointer-labels-on-load=1 -S | FileCheck %s --check-prefix=COMBINE_PTR_LABEL
+; RUN: opt < %s -dfsan -dfsan-combine-pointer-labels-on-load=true -S | FileCheck %s --check-prefixes=CHECK,COMBINE_LOAD_PTR
-; RUN: opt < %s -dfsan -dfsan-combine-pointer-labels-on-load=0 -S | FileCheck %s --check-prefix=NO_COMBINE_PTR_LABEL
+; RUN: opt < %s -dfsan -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-define {} @load0({}* %p) {
+; CHECK: @__dfsan_arg_tls = external thread_local(initialexec) global [[TLS_ARR:\[100 x i64\]]]
-  ; COMBINE_PTR_LABEL: @"dfs$load0"
+; CHECK: @__dfsan_retval_tls = external thread_local(initialexec) global [[TLS_ARR]]
-  ; COMBINE_PTR_LABEL: load
+; CHECK: @__dfsan_shadow_width_bits = weak_odr constant i32 [[#SBITS:]]
-  ; COMBINE_PTR_LABEL-NOT: load
+; CHECK: @__dfsan_shadow_width_bytes = weak_odr constant i32 [[#SBYTES:]]
+
+
+define {} @load0({}* %p) {
+  ; CHECK-LABEL:           @"dfs$load0"
+  ; CHECK-NEXT:            %a = load {}, {}* %p, align 1
+  ; CHECK-NEXT:            store {} zeroinitializer, {}* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to {}*), align [[ALIGN:2]]
+  ; CHECK-NEXT:            ret {} %a
 
-  ; NO_COMBINE_PTR_LABEL: @"dfs$load0"
-  ; NO_COMBINE_PTR_LABEL: load
-  ; NO_COMBINE_PTR_LABEL-NOT: load
   %a = load {}, {}* %p
   ret {} %a
 }
 
 define i8 @load8(i8* %p) {
-  ; COMBINE_PTR_LABEL: @"dfs$load8"
+  ; CHECK-LABEL:           @"dfs$load8"
-  ; COMBINE_PTR_LABEL: load i16, i16*
+  ; COMBINE_LOAD_PTR-NEXT: %[[#PS:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; COMBINE_PTR_LABEL: ptrtoint i8* {{.*}} to i64
+  ; CHECK-NEXT:            %[[#INTP:]] = ptrtoint i8* %p to i64
-  ; COMBINE_PTR_LABEL: and i64
+  ; CHECK-NEXT:            %[[#SHADOW_ADDR:]] = and i64 %[[#INTP]], [[#%.10d,MASK:]]
-  ; COMBINE_PTR_LABEL: mul i64
+  ; CHECK-NEXT:            %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
-  ; COMBINE_PTR_LABEL: inttoptr i64
+  ; CHECK-NEXT:            %[[#SHADOW:]] = load i[[#SBITS]], i[[#SBITS]]* %[[#SHADOW_PTR]]
-  ; COMBINE_PTR_LABEL: load i16, i16*
+  ; COMBINE_LOAD_PTR-NEXT: %[[#SHADOW:]] = or i[[#SBITS]] %[[#SHADOW]], %[[#PS]]
-  ; COMBINE_PTR_LABEL: icmp ne i16
+  ; CHECK-NEXT:            %a = load i8, i8* %p
-  ; COMBINE_PTR_LABEL: call zeroext i16 @__dfsan_union
+  ; CHECK-NEXT:            store i[[#SBITS]] %[[#SHADOW]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; COMBINE_PTR_LABEL: load i8, i8*
+  ; CHECK-NEXT:            ret i8 %a
-  ; COMBINE_PTR_LABEL: store i16 {{.*}} @__dfsan_retval_tls
-  ; COMBINE_PTR_LABEL: ret i8
-
-  ; NO_COMBINE_PTR_LABEL: @"dfs$load8"
-  ; NO_COMBINE_PTR_LABEL: ptrtoint i8*
-  ; NO_COMBINE_PTR_LABEL: and i64
-  ; NO_COMBINE_PTR_LABEL: mul i64
-  ; NO_COMBINE_PTR_LABEL: inttoptr i64 {{.*}} to i16*
-  ; NO_COMBINE_PTR_LABEL: load i16, i16*
-  ; NO_COMBINE_PTR_LABEL: load i8, i8*
-  ; NO_COMBINE_PTR_LABEL: store i16 {{.*}} @__dfsan_retval_tls
-  ; NO_COMBINE_PTR_LABEL: ret i8
 
   %a = load i8, i8* %p
   ret i8 %a
 }
 
 define i16 @load16(i16* %p) {
-  ; COMBINE_PTR_LABEL: @"dfs$load16"
+  ; CHECK-LABEL:           @"dfs$load16"
-  ; COMBINE_PTR_LABEL: ptrtoint i16*
+  ; COMBINE_LOAD_PTR-NEXT: %[[#PS:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; COMBINE_PTR_LABEL: and i64
+  ; CHECK-NEXT:            %[[#INTP:]] = ptrtoint i16* %p to i64
-  ; COMBINE_PTR_LABEL: mul i64
+  ; CHECK-NEXT:            %[[#SHADOW_ADDR:]] = and i64 %[[#INTP]], [[#MASK]]
-  ; COMBINE_PTR_LABEL: inttoptr i64 {{.*}} i16*
+  ; CHECK-NEXT:            %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
-  ; COMBINE_PTR_LABEL: getelementptr i16
+  ; CHECK-NEXT:            %[[#SHADOW_PTR+1]] = getelementptr i[[#SBITS]], i[[#SBITS]]* %[[#SHADOW_PTR]], i64 1
-  ; COMBINE_PTR_LABEL: load i16, i16*
+  ; CHECK-NEXT:            %[[#SHADOW:]]  = load i[[#SBITS]], i[[#SBITS]]* %[[#SHADOW_PTR]]
-  ; COMBINE_PTR_LABEL: load i16, i16*
+  ; CHECK-NEXT:            %[[#SHADOW+1]] = load i[[#SBITS]], i[[#SBITS]]* %[[#SHADOW_PTR+1]]
-  ; COMBINE_PTR_LABEL: icmp ne
-  ; COMBINE_PTR_LABEL: call {{.*}} @__dfsan_union
-  ; COMBINE_PTR_LABEL: icmp ne i16
-  ; COMBINE_PTR_LABEL: call {{.*}} @__dfsan_union
-  ; COMBINE_PTR_LABEL: load i16, i16*
-  ; COMBINE_PTR_LABEL: store {{.*}} @__dfsan_retval_tls
-  ; COMBINE_PTR_LABEL: ret i16
 
-  ; NO_COMBINE_PTR_LABEL: @"dfs$load16"
+  ; CHECK-NEXT:            %[[#SHADOW:]] = or i[[#SBITS]] %[[#SHADOW]], %[[#SHADOW+1]]
-  ; NO_COMBINE_PTR_LABEL: ptrtoint i16*
+  ; COMBINE_LOAD_PTR-NEXT: %[[#SHADOW:]] = or i[[#SBITS]] %[[#SHADOW]], %[[#PS]]
-  ; NO_COMBINE_PTR_LABEL: and i64
+  ; CHECK-NEXT:            %a = load i16, i16* %p
-  ; NO_COMBINE_PTR_LABEL: mul i64
+  ; CHECK-NEXT:            store i[[#SBITS]] %[[#SHADOW]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; NO_COMBINE_PTR_LABEL: inttoptr i64 {{.*}} i16*
+  ; CHECK-NEXT:            ret i16 %a
-  ; NO_COMBINE_PTR_LABEL: getelementptr i16, i16*
-  ; NO_COMBINE_PTR_LABEL: load i16, i16*
-  ; NO_COMBINE_PTR_LABEL: load i16, i16*
-  ; NO_COMBINE_PTR_LABEL: icmp ne i16
-  ; NO_COMBINE_PTR_LABEL: call {{.*}} @__dfsan_union
-  ; NO_COMBINE_PTR_LABEL: load i16, i16*
-  ; NO_COMBINE_PTR_LABEL: store i16 {{.*}} @__dfsan_retval_tls
-  ; NO_COMBINE_PTR_LABEL: ret i16
 
   %a = load i16, i16* %p
   ret i16 %a
 }
 
 define i32 @load32(i32* %p) {
-  ; COMBINE_PTR_LABEL: @"dfs$load32"
+  ; CHECK-LABEL:           @"dfs$load32"
-  ; COMBINE_PTR_LABEL: ptrtoint i32*
+  ; COMBINE_LOAD_PTR-NEXT: %[[#PS:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; COMBINE_PTR_LABEL: and i64
+  ; CHECK-NEXT:            %[[#INTP:]] = ptrtoint i32* %p to i64
-  ; COMBINE_PTR_LABEL: mul i64
+  ; CHECK-NEXT:            %[[#SHADOW_ADDR:]] = and i64 %[[#INTP]], [[#MASK]]
-  ; COMBINE_PTR_LABEL: inttoptr i64 {{.*}} i16*
+  ; CHECK-NEXT:            %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
-  ; COMBINE_PTR_LABEL: bitcast i16* {{.*}} i64*
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_PTR:]] = bitcast i[[#SBITS]]* %[[#SHADOW_PTR]] to i[[#WSBITS:mul(SBITS,4)]]*
-  ; COMBINE_PTR_LABEL: load i64, i64*
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = load i[[#WSBITS]], i[[#WSBITS]]* %[[#WIDE_SHADOW_PTR]], align [[#SBYTES]]
-  ; COMBINE_PTR_LABEL: trunc i64 {{.*}} i16
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_SHIFTED:]] = lshr i[[#WSBITS]] %[[#WIDE_SHADOW]], [[#mul(SBITS,2)]]
-  ; COMBINE_PTR_LABEL: shl i64
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i[[#WSBITS]] %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; COMBINE_PTR_LABEL: lshr i64
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_SHIFTED:]] = lshr i[[#WSBITS]] %[[#WIDE_SHADOW]], [[#SBITS]]
-  ; COMBINE_PTR_LABEL: or i64
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i[[#WSBITS]] %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; COMBINE_PTR_LABEL: icmp eq i64
+  ; CHECK-NEXT:            %[[#SHADOW:]] = trunc i[[#WSBITS]] %[[#WIDE_SHADOW]] to i[[#SBITS]]
-  ; COMBINE_PTR_LABEL: icmp ne i16
+  ; COMBINE_LOAD_PTR-NEXT: %[[#SHADOW:]] = or i[[#SBITS]] %[[#SHADOW]], %[[#PS]]
-  ; COMBINE_PTR_LABEL: call {{.*}} @__dfsan_union
+  ; CHECK-NEXT:            %a = load i32, i32* %p, align 4
-  ; COMBINE_PTR_LABEL: load i32, i32*
+  ; CHECK-NEXT:            store i[[#SBITS]] %[[#SHADOW]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; COMBINE_PTR_LABEL: store i16 {{.*}} @__dfsan_retval_tls
+  ; CHECK-NEXT:            ret i32 %a
-  ; COMBINE_PTR_LABEL: ret i32
-  ; COMBINE_PTR_LABEL: call {{.*}} @__dfsan_union_load
-
-  ; NO_COMBINE_PTR_LABEL: @"dfs$load32"
-  ; NO_COMBINE_PTR_LABEL: ptrtoint i32*
-  ; NO_COMBINE_PTR_LABEL: and i64
-  ; NO_COMBINE_PTR_LABEL: mul i64
-  ; NO_COMBINE_PTR_LABEL: inttoptr i64 {{.*}} i16*
-  ; NO_COMBINE_PTR_LABEL: bitcast i16* {{.*}} i64*
-  ; NO_COMBINE_PTR_LABEL: load i64, i64*
-  ; NO_COMBINE_PTR_LABEL: trunc i64 {{.*}} i16
-  ; NO_COMBINE_PTR_LABEL: shl i64
-  ; NO_COMBINE_PTR_LABEL: lshr i64
-  ; NO_COMBINE_PTR_LABEL: or i64
-  ; NO_COMBINE_PTR_LABEL: icmp eq i64
-  ; NO_COMBINE_PTR_LABEL: load i32, i32*
-  ; NO_COMBINE_PTR_LABEL: store i16 {{.*}} @__dfsan_retval_tls
-  ; NO_COMBINE_PTR_LABEL: ret i32
-  ; NO_COMBINE_PTR_LABEL: call {{.*}} @__dfsan_union_load
-  
 
   %a = load i32, i32* %p
   ret i32 %a
 }
 
 define i64 @load64(i64* %p) {
-  ; COMBINE_PTR_LABEL: @"dfs$load64"
+  ; CHECK-LABEL:           @"dfs$load64"
-  ; COMBINE_PTR_LABEL: ptrtoint i64*
+  ; COMBINE_LOAD_PTR-NEXT: %[[#PS:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; COMBINE_PTR_LABEL: and i64
+  ; CHECK-NEXT:            %[[#INTP:]] = ptrtoint i64* %p to i64
-  ; COMBINE_PTR_LABEL: mul i64
+  ; CHECK-NEXT:            %[[#SHADOW_ADDR:]] = and i64 %[[#INTP]], [[#MASK]]
-  ; COMBINE_PTR_LABEL: inttoptr i64 {{.*}} i16*
+  ; CHECK-NEXT:            %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
-  ; COMBINE_PTR_LABEL: bitcast i16* {{.*}} i64*
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_PTR:]] = bitcast i[[#SBITS]]* %[[#SHADOW_PTR]] to i64*
-  ; COMBINE_PTR_LABEL: load i64, i64*
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = load i64, i64* %[[#WIDE_SHADOW_PTR]], align [[#SBYTES]]
-  ; COMBINE_PTR_LABEL: trunc i64 {{.*}} i16
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 32
-  ; COMBINE_PTR_LABEL: shl i64
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; COMBINE_PTR_LABEL: lshr i64
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 16
-  ; COMBINE_PTR_LABEL: or i64
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; COMBINE_PTR_LABEL: icmp eq i64
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 8
-  ; COMBINE_PTR_LABEL: icmp ne i16
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; COMBINE_PTR_LABEL: call {{.*}} @__dfsan_union
+  ; CHECK-NEXT:            %[[#SHADOW:]] = trunc i64 %[[#WIDE_SHADOW]] to i[[#SBITS]]
-  ; COMBINE_PTR_LABEL: load i64, i64*
+  ; COMBINE_LOAD_PTR-NEXT: %[[#SHADOW:]] = or i[[#SBITS]] %[[#SHADOW]], %[[#PS]]
-  ; COMBINE_PTR_LABEL: store i16 {{.*}} @__dfsan_retval_tls
+  ; CHECK-NEXT:            %a = load i64, i64* %p, align 8
-  ; COMBINE_PTR_LABEL: ret i64
+  ; CHECK-NEXT:            store i[[#SBITS]] %[[#SHADOW]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; COMBINE_PTR_LABEL: call {{.*}} @__dfsan_union_load
+  ; CHECK-NEXT:            ret i64 %a
-  ; COMBINE_PTR_LABEL: getelementptr i64, i64* {{.*}} i64
-  ; COMBINE_PTR_LABEL: load i64, i64*
-  ; COMBINE_PTR_LABEL: icmp eq i64
-
-  ; NO_COMBINE_PTR_LABEL: @"dfs$load64"
-  ; NO_COMBINE_PTR_LABEL: ptrtoint i64*
-  ; NO_COMBINE_PTR_LABEL: and i64
-  ; NO_COMBINE_PTR_LABEL: mul i64
-  ; NO_COMBINE_PTR_LABEL: inttoptr i64 {{.*}} i16*
-  ; NO_COMBINE_PTR_LABEL: bitcast i16* {{.*}} i64*
-  ; NO_COMBINE_PTR_LABEL: load i64, i64*
-  ; NO_COMBINE_PTR_LABEL: trunc i64 {{.*}} i16
-  ; NO_COMBINE_PTR_LABEL: shl i64
-  ; NO_COMBINE_PTR_LABEL: lshr i64
-  ; NO_COMBINE_PTR_LABEL: or i64
-  ; NO_COMBINE_PTR_LABEL: icmp eq i64
-  ; NO_COMBINE_PTR_LABEL: load i64, i64*
-  ; NO_COMBINE_PTR_LABEL: store i16 {{.*}} @__dfsan_retval_tls
-  ; NO_COMBINE_PTR_LABEL: ret i64
-  ; NO_COMBINE_PTR_LABEL: call {{.*}} @__dfsan_union_load
-  ; NO_COMBINE_PTR_LABEL: getelementptr i64, i64* {{.*}} i64
-  ; NO_COMBINE_PTR_LABEL: load i64, i64*
-  ; NO_COMBINE_PTR_LABEL: icmp eq i64
 
   %a = load i64, i64* %p
   ret i64 %a
 }
 
+define i128 @load128(i128* %p) {
+  ; CHECK-LABEL:           @"dfs$load128"
+  ; COMBINE_LOAD_PTR-NEXT: %[[#PS:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; CHECK-NEXT:            %[[#INTP:]] = ptrtoint i128* %p to i64
+  ; CHECK-NEXT:            %[[#SHADOW_ADDR:]] = and i64 %[[#INTP]], [[#MASK]]
+  ; CHECK-NEXT:            %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_PTR:]] = bitcast i[[#SBITS]]* %[[#SHADOW_PTR]] to i64*
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = load i64, i64* %[[#WIDE_SHADOW_PTR]], align [[#SBYTES]]
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_PTR2:]] = getelementptr i64, i64* %[[#WIDE_SHADOW_PTR]], i64 1
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW2:]] = load i64, i64* %[[#WIDE_SHADOW_PTR2]], align [[#SBYTES]]
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW2]]
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 32
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 16
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 8
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
+  ; CHECK-NEXT:            %[[#SHADOW:]] = trunc i64 %[[#WIDE_SHADOW]] to i[[#SBITS]]
+  ; COMBINE_LOAD_PTR-NEXT: %[[#SHADOW:]] = or i[[#SBITS]] %[[#SHADOW]], %[[#PS]]
+  ; CHECK-NEXT:            %a = load i128, i128* %p, align 8
+  ; CHECK-NEXT:            store i[[#SBITS]] %[[#SHADOW]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; CHECK-NEXT:            ret i128 %a
+
+  %a = load i128, i128* %p
+  ret i128 %a
+}
+
+
+define i17 @load17(i17* %p) {
+  ; CHECK-LABEL:           @"dfs$load17"
+  ; COMBINE_LOAD_PTR-NEXT: %[[#PS:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; CHECK-NEXT:            %[[#INTP:]] = ptrtoint i17* %p to i64
+  ; CHECK-NEXT:            %[[#SHADOW_ADDR:]] = and i64 %[[#INTP]], [[#MASK]]
+  ; CHECK-NEXT:            %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
+  ; CHECK-NEXT:            %[[#SHADOW:]] = call zeroext i8 @__dfsan_union_load(i[[#SBITS]]* %[[#SHADOW_PTR]], i64 3)
+  ; COMBINE_LOAD_PTR-NEXT: %[[#SHADOW:]] = or i[[#SBITS]] %[[#SHADOW]], %[[#PS]]
+  ; CHECK-NEXT:            %a = load i17, i17* %p
+  ; CHECK-NEXT:            store i[[#SBITS]] %[[#SHADOW]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; CHECK-NEXT:            ret i17 %a
+
+  %a = load i17, i17* %p
+  ret i17 %a
+}
+
 @X = constant i1 1
 define i1 @load_global() {
-  ; NO_COMBINE_PTR_LABEL: @"dfs$load_global"
+  ; CHECK-LABEL:           @"dfs$load_global"
-  ; NO_COMBINE_PTR_LABEL: store i16 0, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align 2
+  ; CHECK-NEXT:            %a = load i1, i1* @X
+  ; CHECK-NEXT:            store i[[#SBITS]] 0, i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; CHECK-NEXT:            ret i1 %a
 
   %a = load i1, i1* @X
   ret i1 %a

test/Instrumentation/DataFlowSanitizer/musttailcall.ll

@@ -1,7 +1,5 @@
 ; RUN: opt < %s -dfsan -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels -S | FileCheck %s
+; RUN: opt < %s -dfsan -dfsan-track-origins=1 -S | FileCheck %s --check-prefixes=CHECK,CHECK_ORIGIN
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels -dfsan-track-origins=1 -S | FileCheck %s --check-prefixes=CHECK,CHECK_ORIGIN
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 

test/Instrumentation/DataFlowSanitizer/origin_abilist.ll

@@ -1,5 +1,4 @@
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-8-labels=true  -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s
+; RUN: opt < %s -dfsan -dfsan-track-origins=1  -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels=true -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 

test/Instrumentation/DataFlowSanitizer/origin_cached_shadows.ll

@@ -1,5 +1,4 @@
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-8-labels=true  -S | FileCheck %s
+; RUN: opt < %s -dfsan -dfsan-track-origins=1  -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels=true -S | FileCheck %s
 ;
 ; %15 and %17 have the same key in shadow cache. They should not reuse the same
 ; shadow because their blocks do not dominate each other. Origin tracking

test/Instrumentation/DataFlowSanitizer/origin_call.ll

@@ -1,5 +1,4 @@
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-8-labels=true  -S | FileCheck %s
+; RUN: opt < %s -dfsan -dfsan-track-origins=1  -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels=true -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 

test/Instrumentation/DataFlowSanitizer/origin_load.ll

@@ -1,7 +1,5 @@
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-8-labels -S | FileCheck %s --check-prefixes=CHECK,CHECK8,COMBINE_LOAD_PTR
+; RUN: opt < %s -dfsan -dfsan-track-origins=1 -S | FileCheck %s --check-prefixes=CHECK,COMBINE_LOAD_PTR
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-8-labels -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s --check-prefixes=CHECK,CHECK8
+; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels -S | FileCheck %s --check-prefixes=CHECK,CHECK16,COMBINE_LOAD_PTR
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s --check-prefixes=CHECK,CHECK16
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -38,7 +36,6 @@ define i16* @load_escaped_alloca() {
   ; CHECK-LABEL:  @"dfs$load_escaped_alloca"
   ; CHECK:        %[[#INTP:]] = ptrtoint i16* %p to i64
   ; CHECK-NEXT:   %[[#SHADOW_ADDR:]] = and i64 %[[#INTP]], [[#%.10d,MASK:]]
-  ; CHECK16-NEXT: %[[#SHADOW_ADDR:]] = mul i64 %[[#SHADOW_ADDR]], 2
   ; CHECK-NEXT:   %[[#SHADOW_PTR0:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
   ; CHECK-NEXT:   %[[#ORIGIN_OFFSET:]] = add i64 %[[#INTP+1]], [[#%.10d,ORIGIN_MASK:]]
   ; CHECK-NEXT:   %[[#ORIGIN_ADDR:]] = and i64 %[[#ORIGIN_OFFSET]], -4
@@ -76,7 +73,6 @@ define i1 @load1(i1* %p) {
 
   ; CHECK-NEXT:            %[[#INTP:]] = ptrtoint i1* %p to i64
   ; CHECK-NEXT:            %[[#SHADOW_ADDR:]] = and i64 %[[#INTP]], [[#MASK]]
-  ; CHECK16-NEXT:          %[[#SHADOW_ADDR:]] = mul i64 %[[#SHADOW_ADDR]], 2
   ; CHECK-NEXT:            %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
   ; CHECK-NEXT:            %[[#ORIGIN_OFFSET:]] = add i64 %[[#INTP+1]], [[#ORIGIN_MASK]]
   ; CHECK-NEXT:            %[[#ORIGIN_ADDR:]] = and i64 %[[#ORIGIN_OFFSET]], -4
@@ -104,7 +100,6 @@ define i16 @load16(i1 %i, i16* %p) {
 
   ; CHECK-NEXT:            %[[#INTP:]] = ptrtoint i16* %p to i64
   ; CHECK-NEXT:            %[[#SHADOW_ADDR:]] = and i64 %[[#INTP]], [[#MASK]]
-  ; CHECK16-NEXT:          %[[#SHADOW_ADDR:]] = mul i64 %[[#SHADOW_ADDR]], 2
   ; CHECK-NEXT:            %[[#SHADOW_PTR0:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
   ; CHECK-NEXT:            %[[#ORIGIN_OFFSET:]] = add i64 %[[#INTP+1]], [[#ORIGIN_MASK]]
   ; CHECK-NEXT:            %[[#ORIGIN_ADDR:]] = and i64 %[[#ORIGIN_OFFSET]], -4
@@ -133,9 +128,8 @@ define i32 @load32(i32* %p) {
   ; COMBINE_LOAD_PTR-NEXT: %[[#PO:]] = load i32, i32* getelementptr inbounds ([200 x i32], [200 x i32]* @__dfsan_arg_origin_tls, i64 0, i64 0), align 4
   ; COMBINE_LOAD_PTR-NEXT: %[[#PS:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
 
-  ; CHECK:                 %[[#INTP:]] = ptrtoint i32* %p to i64
+  ; CHECK-NEXT:            %[[#INTP:]] = ptrtoint i32* %p to i64
   ; CHECK-NEXT:            %[[#SHADOW_ADDR:INTP+1]] = and i64 %[[#INTP]], [[#MASK]]
-  ; CHECK16-NEXT:          %[[#SHADOW_ADDR:]] = mul i64 %[[#SHADOW_ADDR]], 2
   ; CHECK-NEXT:            %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
   ; CHECK-NEXT:            %[[#ORIGIN_ADDR:]] = add i64 %[[#INTP+1]], [[#ORIGIN_MASK]]
   ; CHECK-NEXT:            %[[#ORIGIN_PTR:]] = inttoptr i64 %[[#ORIGIN_ADDR]] to i32*
@@ -166,41 +160,26 @@ define i64 @load64(i64* %p) {
   ; COMBINE_LOAD_PTR-NEXT: %[[#PO:]] = load i32, i32* getelementptr inbounds ([200 x i32], [200 x i32]* @__dfsan_arg_origin_tls, i64 0, i64 0), align 4
   ; COMBINE_LOAD_PTR-NEXT: %[[#PS:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
 
-  ; CHECK:                 %[[#INTP:]] = ptrtoint i64* %p to i64
+  ; CHECK-NEXT:            %[[#INTP:]] = ptrtoint i64* %p to i64
   ; CHECK-NEXT:            %[[#SHADOW_ADDR:INTP+1]] = and i64 %[[#INTP]], [[#MASK]]
-  ; CHECK16-NEXT:          %[[#SHADOW_ADDR:]] = mul i64 %[[#SHADOW_ADDR]], 2
   ; CHECK-NEXT:            %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
   ; CHECK-NEXT:            %[[#ORIGIN_ADDR:]] = add i64 %[[#INTP+1]], [[#ORIGIN_MASK]]
   ; CHECK-NEXT:            %[[#ORIGIN_PTR:]] = inttoptr i64 %[[#ORIGIN_ADDR]] to i32*
   ; CHECK-NEXT:            %[[#ORIGIN:]] = load i32, i32* %[[#ORIGIN_PTR]], align 8
   ; CHECK-NEXT:            %[[#WIDE_SHADOW_PTR:]] = bitcast i[[#SBITS]]* %[[#SHADOW_PTR]] to i64*
   ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = load i64, i64* %[[#WIDE_SHADOW_PTR]], align [[#SBYTES]]
-
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_LO:]] = shl i64 %[[#WIDE_SHADOW]], 32
-  ; COMM: On fast16, the 2x64 shadow bits need to be ORed first.
+  ; CHECK-NEXT:            %[[#ORIGIN2_PTR:]] = getelementptr i32, i32* %[[#ORIGIN_PTR]], i64 1
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW_PTR2:]] = getelementptr i64, i64* %[[#WIDE_SHADOW_PTR]], i64 1
+  ; CHECK-NEXT:            %[[#ORIGIN2:]] = load i32, i32* %[[#ORIGIN2_PTR]], align 8
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW2:]] = load i64, i64* %[[#WIDE_SHADOW_PTR2]], align [[#SBYTES]]
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 32
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW2]]
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; CHECK16-NEXT:          %[[#ORIGIN_PTR2:]] = getelementptr i32, i32* %[[#ORIGIN_PTR]], i64 1
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 16
-  ; CHECK16-NEXT:          %[[#ORIGIN2:]] = load i32, i32* %[[#ORIGIN_PTR2]], align 8
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 32
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 8
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 16
+  ; CHECK-NEXT:            %[[#SHADOW:]] = trunc i64 %[[#WIDE_SHADOW]] to i[[#SBITS]]
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
+  ; CHECK-NEXT:            %[[#SHADOW_NZ:]] = icmp ne i64 %[[#WIDE_SHADOW_LO]], 0
-  ; CHECK16-NEXT:          %[[#SHADOW:]] = trunc i64 %[[#WIDE_SHADOW]] to i[[#SBITS]]
+  ; CHECK-NEXT:            %[[#ORIGIN:]] = select i1 %[[#SHADOW_NZ]], i32 %[[#ORIGIN]], i32 %[[#ORIGIN2]]
-  ; CHECK16-NEXT:          %[[#SHADOW_NZ:]] = icmp ne i64 %[[#WIDE_SHADOW2]], 0
-  ; CHECK16-NEXT:          %[[#ORIGIN:]] = select i1 %[[#SHADOW_NZ]], i32 %[[#ORIGIN2]], i32 %[[#ORIGIN]]
-
-  ; COMM: On fast8, no need to OR the wide shadow but one more shift is needed.
-  ; CHECK8-NEXT:           %[[#WIDE_SHADOW_LO:]] = shl i64 %[[#WIDE_SHADOW]], 32
-  ; CHECK8-NEXT:           %[[#ORIGIN_PTR2:]] = getelementptr i32, i32* %[[#ORIGIN_PTR]], i64 1
-  ; CHECK8-NEXT:           %[[#ORIGIN2:]] = load i32, i32* %[[#ORIGIN_PTR2]], align 8
-  ; CHECK8-NEXT:           %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 32
-  ; CHECK8-NEXT:           %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; CHECK8-NEXT:           %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 16
-  ; CHECK8-NEXT:           %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; CHECK8-NEXT:           %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 8
-  ; CHECK8-NEXT:           %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; CHECK8-NEXT:           %[[#SHADOW:]] = trunc i64 %[[#WIDE_SHADOW]] to i[[#SBITS]]
   ; CHECK8-NEXT:           %[[#SHADOW_NZ:]] = icmp ne i64 %[[#WIDE_SHADOW_LO]], 0
   ; CHECK8-NEXT:           %[[#ORIGIN:]] = select i1 %[[#SHADOW_NZ]], i32 %[[#ORIGIN]], i32 %[[#ORIGIN2]]
 
@@ -246,66 +225,38 @@ define i128 @load128(i128* %p) {
   ; COMBINE_LOAD_PTR-NEXT: %[[#PO:]] = load i32, i32* getelementptr inbounds ([200 x i32], [200 x i32]* @__dfsan_arg_origin_tls, i64 0, i64 0), align 4
   ; COMBINE_LOAD_PTR-NEXT: %[[#PS:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
 
-  ; CHECK:                 %[[#INTP:]] = ptrtoint i128* %p to i64
+  ; CHECK-NEXT:            %[[#INTP:]] = ptrtoint i128* %p to i64
   ; CHECK-NEXT:            %[[#SHADOW_ADDR:INTP+1]] = and i64 %[[#INTP]], [[#MASK]]
-  ; CHECK16-NEXT:          %[[#SHADOW_ADDR:]] = mul i64 %[[#SHADOW_ADDR]], 2
   ; CHECK-NEXT:            %[[#SHADOW_PTR:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
-  ; CHECK-NEXT:            %[[#ORIGIN_ADDR:]] = add i64 %[[#INTP+1]], [[#ORIGIN_MASK]]
+  ; CHECK-NEXT:            %[[#ORIGIN_ADDR:]] = add i64 %[[#SHADOW_ADDR]], [[#ORIGIN_MASK]]
-  ; CHECK-NEXT:            %[[#ORIGIN_PTR:]] = inttoptr i64 %[[#ORIGIN_ADDR]] to i32*
+  ; CHECK-NEXT:            %[[#ORIGIN1_PTR:]] = inttoptr i64 %[[#ORIGIN_ADDR]] to i32*
-  ; CHECK-NEXT:            %[[#ORIGIN:]] = load i32, i32* %[[#ORIGIN_PTR]], align 8
+  ; CHECK-NEXT:            %[[#ORIGIN1:]] = load i32, i32* %[[#ORIGIN1_PTR]], align 8
-  ; CHECK-NEXT:            %[[#WIDE_SHADOW_PTR:]] = bitcast i[[#SBITS]]* %[[#SHADOW_PTR]] to i64*
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW1_PTR:]] = bitcast i[[#SBITS]]* %[[#SHADOW_PTR]] to i64*
-  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = load i64, i64* %[[#WIDE_SHADOW_PTR]], align [[#SBYTES]]
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW1:]] = load i64, i64* %[[#WIDE_SHADOW1_PTR]], align [[#SBYTES]]
-  ; CHECK8-NEXT:           %[[#WIDE_SHADOW_LO:]] = shl i64 %[[#WIDE_SHADOW]], 32
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW1_LO:]] = shl i64 %[[#WIDE_SHADOW1]], 32
-  ; CHECK8-NEXT:           %[[#ORIGIN_PTR2:]] = getelementptr i32, i32* %[[#ORIGIN_PTR]], i64 1
+  ; CHECK-NEXT:            %[[#ORIGIN2_PTR:]] = getelementptr i32, i32* %[[#ORIGIN1_PTR]], i64 1
-  ; CHECK8-NEXT:           %[[#ORIGIN2:]] = load i32, i32* %[[#ORIGIN_PTR2]], align 8
+  ; CHECK-NEXT:            %[[#ORIGIN2:]] = load i32, i32* %[[#ORIGIN2_PTR]], align 8
-  ; CHECK-NEXT:            %[[#WIDE_SHADOW_PTR2:]] = getelementptr i64, i64* %[[#WIDE_SHADOW_PTR]], i64 1
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW2_PTR:]] = getelementptr i64, i64* %[[#WIDE_SHADOW1_PTR]], i64 1
-  ; CHECK-NEXT:            %[[#WIDE_SHADOW2:]] = load i64, i64* %[[#WIDE_SHADOW_PTR2]], align [[#SBYTES]]
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW2:]] = load i64, i64* %[[#WIDE_SHADOW2_PTR]], align [[#SBYTES]]
-  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW2]]
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW1]], %[[#WIDE_SHADOW2]]
-
+  ; CHECK-NEXT:            %[[#ORIGIN3_PTR:]] = getelementptr i32, i32* %[[#ORIGIN2_PTR]], i64 1
-  ; COMM: On fast16, we need to OR 4x64bits for the wide shadow, before ORing its bytes.
+  ; CHECK-NEXT:            %[[#ORIGIN3:]] = load i32, i32* %[[#ORIGIN3_PTR]], align 8
-  ; CHECK16-NEXT:          %[[#ORIGIN_PTR2:]] = getelementptr i32, i32* %[[#ORIGIN_PTR]], i64 1
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW2_LO:]] = shl i64 %[[#WIDE_SHADOW2]], 32
-  ; CHECK16-NEXT:          %[[#ORIGIN2:]] = load i32, i32* %[[#ORIGIN_PTR2]], align 8
+  ; CHECK-NEXT:            %[[#ORIGIN4_PTR:]] = getelementptr i32, i32* %[[#ORIGIN3_PTR]], i64 1
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW_PTR3:]] = getelementptr i64, i64* %[[#WIDE_SHADOW_PTR2]], i64 1
+  ; CHECK-NEXT:            %[[#ORIGIN4:]] = load i32, i32* %[[#ORIGIN4_PTR]], align 8
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW3:]] = load i64, i64* %[[#WIDE_SHADOW_PTR3]], align [[#SBYTES]]
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 32
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW3]]
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; CHECK16-NEXT:          %[[#ORIGIN_PTR3:]] = getelementptr i32, i32* %[[#ORIGIN_PTR2]], i64 1
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 16
-  ; CHECK16-NEXT:          %[[#ORIGIN3:]] = load i32, i32* %[[#ORIGIN_PTR3]], align 8
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW_PTR4:]] = getelementptr i64, i64* %[[#WIDE_SHADOW_PTR3]], i64 1
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 8
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW4:]] = load i64, i64* %[[#WIDE_SHADOW_PTR4]], align [[#SBYTES]]
+  ; CHECK-NEXT:            %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW4]]
+  ; CHECK-NEXT:            %[[#SHADOW:]] = trunc i64 %[[#WIDE_SHADOW]] to i[[#SBITS]]
-  ; CHECK16-NEXT:          %[[#ORIGIN_PTR4:]] = getelementptr i32, i32* %[[#ORIGIN_PTR3]], i64 1
+  ; CHECK-NEXT:            %[[#SHADOW1_LO_NZ:]] = icmp ne i64 %[[#WIDE_SHADOW1_LO]], 0
-  ; CHECK16-NEXT:          %[[#ORIGIN4:]] = load i32, i32* %[[#ORIGIN_PTR4]], align 8
+  ; CHECK-NEXT:            %[[#ORIGIN12:]] = select i1 %[[#SHADOW1_LO_NZ]], i32 %[[#ORIGIN1]], i32 %[[#ORIGIN2]]
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 32
+  ; CHECK-NEXT:            %[[#SHADOW2_NZ:]] = icmp ne i64 %[[#WIDE_SHADOW2]], 0
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
+  ; CHECK-NEXT:            %[[#ORIGIN124:]] = select i1 %[[#SHADOW2_NZ]], i32 %[[#ORIGIN4]], i32 %[[#ORIGIN12]]
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 16
+  ; CHECK-NEXT:            %[[#SHADOW2_LO_NZ:]] = icmp ne i64 %[[#WIDE_SHADOW2_LO]], 0
-  ; CHECK16-NEXT:          %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
+  ; CHECK-NEXT:            %[[#ORIGIN:]] = select i1 %[[#SHADOW2_LO_NZ]], i32 %[[#ORIGIN3]], i32 %[[#ORIGIN124]]
-  ; CHECK16-NEXT:          %[[#SHADOW:]] = trunc i64 %[[#WIDE_SHADOW]] to i[[#SBITS]]
-  ; CHECK16-NEXT:          %[[#SHADOW2_NZ:]] = icmp ne i64 %[[#WIDE_SHADOW2]], 0
-  ; CHECK16-NEXT:          %[[#ORIGIN:]] = select i1 %[[#SHADOW2_NZ]], i32 %[[#ORIGIN2]], i32 %[[#ORIGIN]]
-  ; CHECK16-NEXT:          %[[#SHADOW3_NZ:]] = icmp ne i64 %[[#WIDE_SHADOW3]], 0
-  ; CHECK16-NEXT:          %[[#ORIGIN:]] = select i1 %[[#SHADOW3_NZ]], i32 %[[#ORIGIN3]], i32 %[[#ORIGIN]]
-  ; CHECK16-NEXT:          %[[#SHADOW4_NZ:]] = icmp ne i64 %[[#WIDE_SHADOW4]], 0
-  ; CHECK16-NEXT:          %[[#ORIGIN:]] = select i1 %[[#SHADOW4_NZ]], i32 %[[#ORIGIN4]], i32 %[[#ORIGIN]]
-  
-  ; COMM: On fast8, we need to OR 2x64bits for the wide shadow, before ORing its bytes (one more shift).
-  ; CHECK8-NEXT:           %[[#ORIGIN_PTR3:]] = getelementptr i32, i32* %[[#ORIGIN_PTR2]], i64 1
-  ; CHECK8-NEXT:           %[[#ORIGIN3:]] = load i32, i32* %[[#ORIGIN_PTR3]], align 8
-  ; CHECK8-NEXT:           %[[#WIDE_SHADOW2_LO:]] = shl i64 %[[#WIDE_SHADOW2]], 32
-  ; CHECK8-NEXT:           %[[#ORIGIN_PTR4:]] = getelementptr i32, i32* %[[#ORIGIN_PTR3]], i64 1
-  ; CHECK8-NEXT:           %[[#ORIGIN4:]] = load i32, i32* %[[#ORIGIN_PTR4]], align 8
-  ; CHECK8-NEXT:           %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 32
-  ; CHECK8-NEXT:           %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; CHECK8-NEXT:           %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 16
-  ; CHECK8-NEXT:           %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; CHECK8-NEXT:           %[[#WIDE_SHADOW_SHIFTED:]] = lshr i64 %[[#WIDE_SHADOW]], 8
-  ; CHECK8-NEXT:           %[[#WIDE_SHADOW:]] = or i64 %[[#WIDE_SHADOW]], %[[#WIDE_SHADOW_SHIFTED]]
-  ; CHECK8-NEXT:           %[[#SHADOW:]] = trunc i64 %[[#WIDE_SHADOW]] to i[[#SBITS]]
-  ; CHECK8-NEXT:           %[[#SHADOW_LO_NZ:]] = icmp ne i64 %[[#WIDE_SHADOW_LO]], 0
-  ; CHECK8-NEXT:           %[[#ORIGIN:]] = select i1 %[[#SHADOW_LO_NZ]], i32 %[[#ORIGIN]], i32 %[[#ORIGIN2]]
-  ; CHECK8-NEXT:           %[[#SHADOW2_NZ:]] = icmp ne i64 %[[#WIDE_SHADOW2]], 0
-  ; CHECK8-NEXT:           %[[#ORIGIN:]] = select i1 %[[#SHADOW2_NZ]], i32 %[[#ORIGIN4]], i32 %[[#ORIGIN]]
-  ; CHECK8-NEXT:           %[[#SHADOW2_LO_NZ:]] = icmp ne i64 %[[#WIDE_SHADOW2_LO]], 0
-  ; CHECK8-NEXT:           %[[#ORIGIN:]] = select i1 %[[#SHADOW2_LO_NZ]], i32 %[[#ORIGIN3]], i32 %[[#ORIGIN]]
 
   ; COMBINE_LOAD_PTR-NEXT: %[[#SHADOW:]] = or i[[#SBITS]] %[[#SHADOW]], %[[#PS]]
   ; COMBINE_LOAD_PTR-NEXT: %[[#NZ:]] = icmp ne i[[#SBITS]] %[[#PS]], 0

test/Instrumentation/DataFlowSanitizer/origin_mem_intrinsic.ll

@@ -1,5 +1,4 @@
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-8-labels=true  -S | FileCheck %s
+; RUN: opt < %s -dfsan -dfsan-track-origins=1  -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels=true -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 

test/Instrumentation/DataFlowSanitizer/origin_other_ops.ll

@@ -1,5 +1,4 @@
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-8-labels=true  -S | FileCheck %s
+; RUN: opt < %s -dfsan -dfsan-track-origins=1  -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels=true -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 

test/Instrumentation/DataFlowSanitizer/origin_phi.ll

@@ -1,5 +1,4 @@
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-8-labels=true  -S | FileCheck %s
+; RUN: opt < %s -dfsan -dfsan-track-origins=1  -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels=true -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 

test/Instrumentation/DataFlowSanitizer/origin_select.ll

@@ -1,7 +1,5 @@
-; RUN: opt < %s -dfsan -dfsan-track-select-control-flow=1 -dfsan-track-origins=1 -dfsan-fast-8-labels=true  -S | FileCheck %s --check-prefixes=CHECK,TRACK_CONTROL_FLOW
+; RUN: opt < %s -dfsan -dfsan-track-select-control-flow=1 -dfsan-track-origins=1  -S | FileCheck %s --check-prefixes=CHECK,TRACK_CONTROL_FLOW
-; RUN: opt < %s -dfsan -dfsan-track-select-control-flow=0 -dfsan-track-origins=1 -dfsan-fast-8-labels=true  -S | FileCheck %s --check-prefixes=CHECK,NO_TRACK_CONTROL_FLOW
+; RUN: opt < %s -dfsan -dfsan-track-select-control-flow=0 -dfsan-track-origins=1  -S | FileCheck %s --check-prefixes=CHECK,NO_TRACK_CONTROL_FLOW
-; RUN: opt < %s -dfsan -dfsan-track-select-control-flow=1 -dfsan-track-origins=1 -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,TRACK_CONTROL_FLOW
-; RUN: opt < %s -dfsan -dfsan-track-select-control-flow=0 -dfsan-track-origins=1 -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,NO_TRACK_CONTROL_FLOW
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 

test/Instrumentation/DataFlowSanitizer/origin_store.ll

@@ -1,7 +1,5 @@
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-8-labels -S | FileCheck %s
+; RUN: opt < %s -dfsan -dfsan-track-origins=1 -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-8-labels -dfsan-combine-pointer-labels-on-store -S | FileCheck %s --check-prefixes=CHECK,COMBINE_STORE_PTR
+; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-combine-pointer-labels-on-store -S | FileCheck %s --check-prefixes=CHECK,COMBINE_STORE_PTR
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels -S | FileCheck %s --check-prefixes=CHECK,CHECK16
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels -dfsan-combine-pointer-labels-on-store -S | FileCheck %s --check-prefixes=CHECK,CHECK16,COMBINE_STORE_PTR
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -55,7 +53,6 @@ define void @store_nonzero_to_escaped_alloca(i16 %a) {
   ; CHECK-NEXT:   %[[#AS:]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
   ; CHECK:        %[[#INTP:]] = ptrtoint i16* %p to i64
   ; CHECK-NEXT:   %[[#SHADOW_ADDR:]] = and i64 %[[#INTP]], [[#%.10d,MASK:]]
-  ; CHECK16-NEXT: %[[#SHADOW_ADDR:]] = mul i64 %[[#SHADOW_ADDR]], 2
   ; CHECK-NEXT:   %[[#SHADOW_PTR0:]] = inttoptr i64 %[[#SHADOW_ADDR]] to i[[#SBITS]]*
   ; CHECK-NEXT:   %[[#ORIGIN_OFFSET:]] = add i64 %[[#INTP+1]], [[#%.10d,ORIGIN_MASK:]]
   ; CHECK-NEXT:   %[[#ORIGIN_ADDR:]] = and i64 %[[#ORIGIN_OFFSET]], -4

test/Instrumentation/DataFlowSanitizer/origin_store_threshold.ll

@@ -1,5 +1,4 @@
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-8-labels=true  -dfsan-instrument-with-call-threshold=0 -S | FileCheck %s
+; RUN: opt < %s -dfsan -dfsan-track-origins=1  -dfsan-instrument-with-call-threshold=0 -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-track-origins=1 -dfsan-fast-16-labels=true -dfsan-instrument-with-call-threshold=0 -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 

test/Instrumentation/DataFlowSanitizer/origin_track_load.ll

@@ -1,5 +1,4 @@
-; RUN: opt < %s -dfsan -dfsan-track-origins=2 -dfsan-fast-8-labels -S | FileCheck %s
+; RUN: opt < %s -dfsan -dfsan-track-origins=2 -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-track-origins=2 -dfsan-fast-16-labels -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 

test/Instrumentation/DataFlowSanitizer/phi.ll

@@ -1,6 +1,4 @@
-; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,LEGACY
+; RUN: opt < %s -dfsan -S | FileCheck %s
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,FAST
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -S | FileCheck %s --check-prefixes=CHECK,FAST
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -8,15 +6,11 @@ target triple = "x86_64-unknown-linux-gnu"
 ; CHECK: @__dfsan_shadow_width_bytes = weak_odr constant i32 [[#SBYTES:]]
 
 define {i32, i32} @test({i32, i32} %a, i1 %c) {
-  ; LEGACY: [[AL:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
+  ; CHECK: %[[#AL:]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN:2]]
-  ; LEGACY: [[PL:%.*]] = phi i[[#SBITS]] [ [[AL]], %T ], [ [[AL]], %F ]
+  ; CHECK: %[[#AL0:]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } %[[#AL]], i[[#SBITS]] 0, 0
-  ; LEGACY: store i[[#SBITS]] [[PL]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
+  ; CHECK: %[[#AL1:]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } %[[#AL]], i[[#SBITS]] 0, 1
-
+  ; CHECK: %[[#PL:]] = phi { i[[#SBITS]], i[[#SBITS]] } [ %[[#AL0]], %T ], [ %[[#AL1]], %F ]
-  ; FAST: [[AL:%.*]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN:2]]
+  ; CHECK: store { i[[#SBITS]], i[[#SBITS]] } %[[#PL]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
-  ; FAST: [[AL0:%.*]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } [[AL]], i[[#SBITS]] 0, 0
-  ; FAST: [[AL1:%.*]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } [[AL]], i[[#SBITS]] 0, 1
-  ; FAST: [[PL:%.*]] = phi { i[[#SBITS]], i[[#SBITS]] } [ [[AL0]], %T ], [ [[AL1]], %F ]
-  ; FAST: store { i[[#SBITS]], i[[#SBITS]] } [[PL]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
 
 entry:
   br i1 %c, label %T, label %F

test/Instrumentation/DataFlowSanitizer/select.ll

@@ -1,9 +1,5 @@
-; RUN: opt < %s -dfsan -dfsan-track-select-control-flow=1 -S | FileCheck %s --check-prefixes=CHECK,TRACK_CF,TRACK_CF_LEGACY
+; RUN: opt < %s -dfsan -dfsan-track-select-control-flow=true -S | FileCheck %s --check-prefixes=CHECK,TRACK_CF
-; RUN: opt < %s -dfsan -dfsan-track-select-control-flow=0 -S | FileCheck %s --check-prefixes=CHECK,NO_TRACK_CF,NO_TRACK_CF_LEGACY
+; RUN: opt < %s -dfsan -dfsan-track-select-control-flow=false -S | FileCheck %s --check-prefixes=CHECK,NO_TRACK_CF
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels -dfsan-track-select-control-flow=1 -S | FileCheck %s --check-prefixes=CHECK,TRACK_CF,TRACK_CF_FAST
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels -dfsan-track-select-control-flow=0 -S | FileCheck %s --check-prefixes=CHECK,NO_TRACK_CF,NO_TRACK_CF_FAST
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels -dfsan-track-select-control-flow=1 -S | FileCheck %s --check-prefixes=CHECK,TRACK_CF,TRACK_CF_FAST
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels -dfsan-track-select-control-flow=0 -S | FileCheck %s --check-prefixes=CHECK,NO_TRACK_CF,NO_TRACK_CF_FAST
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -18,11 +14,7 @@ define i8 @select8(i1 %c, i8 %t, i8 %f) {
   ; TRACK_CF: %[[#R+1]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN]]
   ; TRACK_CF: %[[#R+2]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
   ; TRACK_CF: %[[#R+3]] = select i1 %c, i[[#SBITS]] %[[#R+1]], i[[#SBITS]] %[[#R]]
-  ; TRACK_CF_LEGACY: %[[#R+4]] = icmp ne i[[#SBITS]] %[[#R+2]], %[[#R+3]]
+  ; TRACK_CF: %[[#RO:]] = or i[[#SBITS]] %[[#R+2]], %[[#R+3]]
-  ; TRACK_CF_LEGACY: %[[#R+6]] = call {{.*}} i[[#SBITS]] @__dfsan_union(i[[#SBITS]] {{.*}} %[[#R+2]], i[[#SBITS]] {{.*}} %[[#R+3]])
-  ; TRACK_CF_LEGACY: %[[#RO:]] = phi i[[#SBITS]] [ %[[#R+6]], {{.*}} ], [ %[[#R+2]], {{.*}} ]
-  ; COMM: The union is simply an OR when fast labels are used.
-  ; TRACK_CF_FAST: %[[#RO:]] = or i[[#SBITS]] %[[#R+2]], %[[#R+3]]
   ; TRACK_CF: %a = select i1 %c, i8 %t, i8 %f
   ; TRACK_CF: store i[[#SBITS]] %[[#RO]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
   ; TRACK_CF: ret i8 %a
@@ -44,11 +36,7 @@ define i8 @select8e(i1 %c, i8 %tf) {
   ; TRACK_CF: @"dfs$select8e"
   ; TRACK_CF: %[[#R:]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN]]
   ; TRACK_CF: %[[#R+1]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; TRACK_CF_LEGACY: %[[#R+2]] = icmp ne i[[#SBITS]] %[[#R+1]], %[[#R]]
+  ; TRACK_CF: %[[#RO:]] = or i[[#SBITS]] %[[#R+1]], %[[#R]]
-  ; TRACK_CF_LEGACY: %[[#R+4]] = call {{.*}} i[[#SBITS]] @__dfsan_union(i[[#SBITS]] {{.*}} %[[#R+1]], i[[#SBITS]] {{.*}} %[[#R]])
-  ; TRACK_CF_LEGACY: %[[#RO:]] = phi i[[#SBITS]] [ %[[#R+4]], {{.*}} ], [ %[[#R+1]], {{.*}} ]
-  ; COMM: The union is simply an OR when fast labels are used.
-  ; TRACK_CF_FAST: %[[#RO:]] = or i[[#SBITS]] %[[#R+1]], %[[#R]]
   ; TRACK_CF: %a = select i1 %c, i8 %tf, i8 %tf
   ; TRACK_CF: store i[[#SBITS]] %[[#RO]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
   ; TRACK_CF: ret i8 %a
@@ -69,15 +57,8 @@ define <4 x i8> @select8v(<4 x i1> %c, <4 x i8> %t, <4 x i8> %f) {
   ; TRACK_CF: %[[#R:]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 4) to i[[#SBITS]]*), align [[ALIGN:2]]
   ; TRACK_CF: %[[#R+1]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN]]
   ; TRACK_CF: %[[#R+2]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; TRACK_CF_LEGACY: %[[#R+3]] = icmp ne i[[#SBITS]] %[[#R+1]], %[[#R]]
+  ; TRACK_CF: %[[#R+3]] = or i[[#SBITS]] %[[#R+1]], %[[#R]]
-  ; TRACK_CF_LEGACY: %[[#R+5]] = call {{.*}} i[[#SBITS]] @__dfsan_union(i[[#SBITS]] {{.*}} %[[#R+1]], i[[#SBITS]] zeroext %[[#R]])
+  ; TRACK_CF: %[[#RO:]] = or i[[#SBITS]] %[[#R+2]], %[[#R+3]]
-  ; TRACK_CF_LEGACY: %[[#R+7]] = phi i[[#SBITS]] [ %[[#R+5]], {{.*}} ], [ %[[#R+1]], {{.*}} ]
-  ; TRACK_CF_LEGACY: %[[#R+8]] = icmp ne i[[#SBITS]] %[[#R+2]], %[[#R+7]]
-  ; TRACK_CF_LEGACY: %[[#R+10]] = call {{.*}} i[[#SBITS]] @__dfsan_union(i[[#SBITS]] {{.*}} %[[#R+2]], i[[#SBITS]] zeroext %[[#R+7]])
-  ; TRACK_CF_LEGACY: %[[#RO:]] = phi i[[#SBITS]] [ %[[#R+10]], {{.*}} ], [ %[[#R+2]], {{.*}} ]
-  ; COMM: The union is simply an OR when fast labels are used.
-  ; TRACK_CF_FAST: %[[#R+3]] = or i[[#SBITS]] %[[#R+1]], %[[#R]]
-  ; TRACK_CF_FAST: %[[#RO:]] = or i[[#SBITS]] %[[#R+2]], %[[#R+3]]
   ; TRACK_CF: %a = select <4 x i1> %c, <4 x i8> %t, <4 x i8> %f
   ; TRACK_CF: store i[[#SBITS]] %[[#RO]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
   ; TRACK_CF: ret <4 x i8> %a
@@ -86,11 +67,7 @@ define <4 x i8> @select8v(<4 x i1> %c, <4 x i8> %t, <4 x i8> %f) {
   ; NO_TRACK_CF: %[[#R:]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 4) to i[[#SBITS]]*), align [[ALIGN:2]]
   ; NO_TRACK_CF: %[[#R+1]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN]]
   ; NO_TRACK_CF: %[[#R+2]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; NO_TRACK_CF_LEGACY: %[[#R+3]] = icmp ne i[[#SBITS]] %[[#R+1]], %[[#R]]
+  ; NO_TRACK_CF: %[[#RO:]] = or i[[#SBITS]] %[[#R+1]], %[[#R]]
-  ; NO_TRACK_CF_LEGACY: %[[#R+5]] = call {{.*}} i[[#SBITS]] @__dfsan_union(i[[#SBITS]] {{.*}} %[[#R+1]], i[[#SBITS]] {{.*}} %[[#R]])
-  ; NO_TRACK_CF_LEGACY: %[[#RO:]] = phi i[[#SBITS]] [ %6, {{.*}} ], [ %2, {{.*}} ]
-  ; COMM: The union is simply an OR when fast labels are used.
-  ; NO_TRACK_CF_FAST: %[[#RO:]] = or i[[#SBITS]] %[[#R+1]], %[[#R]]
   ; NO_TRACK_CF: %a = select <4 x i1> %c, <4 x i8> %t, <4 x i8> %f
   ; NO_TRACK_CF: store i[[#SBITS]] %[[#RO]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
   ; NO_TRACK_CF: ret <4 x i8> %a

test/Instrumentation/DataFlowSanitizer/shadow-args-zext.ll

@@ -1,6 +1,4 @@
 ; RUN: opt -mtriple=x86_64-unknown-linux-gnu < %s -dfsan -S --dfsan-abilist=%S/Inputs/shadow-args-abilist.txt | FileCheck %s
-; RUN: opt -mtriple=x86_64-unknown-linux-gnu < %s -dfsan -S --dfsan-abilist=%S/Inputs/shadow-args-abilist.txt -dfsan-fast-16-labels | FileCheck %s
-; RUN: opt -mtriple=x86_64-unknown-linux-gnu < %s -dfsan -S --dfsan-abilist=%S/Inputs/shadow-args-abilist.txt -dfsan-fast-8-labels | FileCheck %s
 
 ; REQUIRES: x86-registered-target
 

test/Instrumentation/DataFlowSanitizer/store.ll

@@ -1,9 +1,5 @@
-; RUN: opt < %s -dfsan -dfsan-combine-pointer-labels-on-store=1 -S | FileCheck %s --check-prefixes=CHECK,CHECK16,COMBINE_PTR_LABEL
+; RUN: opt < %s -dfsan -dfsan-combine-pointer-labels-on-store=1 -S | FileCheck %s --check-prefixes=CHECK,COMBINE_PTR_LABEL
-; RUN: opt < %s -dfsan -dfsan-combine-pointer-labels-on-store=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK16,NO_COMBINE_PTR_LABEL
+; RUN: opt < %s -dfsan -dfsan-combine-pointer-labels-on-store=0 -S | FileCheck %s --check-prefixes=CHECK,NO_COMBINE_PTR_LABEL
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels -dfsan-combine-pointer-labels-on-store=1 -S | FileCheck %s --check-prefixes=CHECK,CHECK16,COMBINE_PTR_LABEL_FAST
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels -dfsan-combine-pointer-labels-on-store=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK16,NO_COMBINE_PTR_LABEL
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels -dfsan-combine-pointer-labels-on-store=1 -S | FileCheck %s --check-prefixes=CHECK,COMBINE_PTR_LABEL_FAST
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels -dfsan-combine-pointer-labels-on-store=0 -S | FileCheck %s --check-prefixes=CHECK,NO_COMBINE_PTR_LABEL
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -23,19 +19,10 @@ define void @store0({} %v, {}* %p) {
 define void @store8(i8 %v, i8* %p) {
   ; CHECK-LABEL:       @"dfs$store8"
   ; CHECK:             load i[[#SBITS]], i[[#SBITS]]* {{.*}} @__dfsan_arg_tls
-
   ; COMBINE_PTR_LABEL: load i[[#SBITS]], i[[#SBITS]]* {{.*}} @__dfsan_arg_tls
-  ; COMBINE_PTR_LABEL: icmp ne i[[#SBITS]]
+  ; COMBINE_PTR_LABEL: or i[[#SBITS]]
-  ; COMBINE_PTR_LABEL: call {{.*}} @__dfsan_union
-
-  ; COMM: When not in legacy mode, the three instructions above will
-  ;        be replaced with the following:
-  ; COMBINE_PTR_LABEL_FAST: load i[[#SBITS]], i[[#SBITS]]* {{.*}} @__dfsan_arg_tls
-  ; COMBINE_PTR_LABEL_FAST: or i[[#SBITS]]
-
   ; CHECK:             ptrtoint i8* {{.*}} i64
   ; CHECK-NEXT:        and i64
-  ; CHECK16-NEXT:      mul i64
   ; CHECK-NEXT:        inttoptr i64 {{.*}} i[[#SBITS]]*
   ; CHECK-NEXT:        getelementptr i[[#SBITS]], i[[#SBITS]]*
   ; CHECK-NEXT:        store i[[#SBITS]]
@@ -49,19 +36,10 @@ define void @store8(i8 %v, i8* %p) {
 define void @store16(i16 %v, i16* %p) {
   ; CHECK-LABEL:       @"dfs$store16"
   ; CHECK:             load i[[#SBITS]], i[[#SBITS]]* {{.*}} @__dfsan_arg_tls
-
   ; COMBINE_PTR_LABEL: load i[[#SBITS]], i[[#SBITS]]* {{.*}} @__dfsan_arg_tls
-  ; COMBINE_PTR_LABEL: icmp ne i[[#SBITS]]
+  ; COMBINE_PTR_LABEL: or i[[#SBITS]]
-  ; COMBINE_PTR_LABEL: call {{.*}} @__dfsan_union
-
-  ; COMM: When not in legacy mode, the three instructions above will
-  ;        be replaced with the following:
-  ; COMBINE_PTR_LABEL_FAST: load i[[#SBITS]], i[[#SBITS]]* {{.*}} @__dfsan_arg_tls
-  ; COMBINE_PTR_LABEL_FAST: or i[[#SBITS]]
-
   ; CHECK:             ptrtoint i16* {{.*}} i64
   ; CHECK-NEXT:        and i64
-  ; CHECK16-NEXT:      mul i64
   ; CHECK-NEXT:        inttoptr i64 {{.*}} i[[#SBITS]]*
   ; CHECK-NEXT:        getelementptr i[[#SBITS]], i[[#SBITS]]*
   ; CHECK-NEXT:        store i[[#SBITS]]
@@ -77,19 +55,10 @@ define void @store16(i16 %v, i16* %p) {
 define void @store32(i32 %v, i32* %p) {
   ; CHECK-LABEL:       @"dfs$store32"
   ; CHECK:             load i[[#SBITS]], i[[#SBITS]]* {{.*}} @__dfsan_arg_tls
-
   ; COMBINE_PTR_LABEL: load i[[#SBITS]], i[[#SBITS]]* {{.*}} @__dfsan_arg_tls
-  ; COMBINE_PTR_LABEL: icmp ne i[[#SBITS]]
+  ; COMBINE_PTR_LABEL: or i[[#SBITS]]
-  ; COMBINE_PTR_LABEL: call {{.*}} @__dfsan_union
-
-  ; COMM: When not in legacy mode, the three instructions above will
-  ;        be replaced with the following:
-  ; COMBINE_PTR_LABEL_FAST: load i[[#SBITS]], i[[#SBITS]]* {{.*}} @__dfsan_arg_tls
-  ; COMBINE_PTR_LABEL_FAST: or i[[#SBITS]]
-
   ; CHECK:             ptrtoint i32* {{.*}} i64
   ; CHECK-NEXT:        and i64
-  ; CHECK16-NEXT:      mul i64
   ; CHECK-NEXT:        inttoptr i64 {{.*}} i[[#SBITS]]*
   ; CHECK-NEXT:        getelementptr i[[#SBITS]], i[[#SBITS]]*
   ; CHECK-NEXT:        store i[[#SBITS]]
@@ -109,19 +78,10 @@ define void @store32(i32 %v, i32* %p) {
 define void @store64(i64 %v, i64* %p) {
   ; CHECK-LABEL:       @"dfs$store64"
   ; CHECK:             load i[[#SBITS]], i[[#SBITS]]* {{.*}} @__dfsan_arg_tls
-
   ; COMBINE_PTR_LABEL: load i[[#SBITS]], i[[#SBITS]]* {{.*}} @__dfsan_arg_tls
-  ; COMBINE_PTR_LABEL: icmp ne i[[#SBITS]]
+  ; COMBINE_PTR_LABEL: or i[[#SBITS]]
-  ; COMBINE_PTR_LABEL: call {{.*}} @__dfsan_union
-
-  ; COMM: When not in legacy mode, the three instructions above will
-  ;       be replaced with the following:
-  ; COMBINE_PTR_LABEL_FAST: load i[[#SBITS]], i[[#SBITS]]* {{.*}} @__dfsan_arg_tls
-  ; COMBINE_PTR_LABEL_FAST: or i[[#SBITS]]
-
   ; CHECK:             ptrtoint i64* {{.*}} i64
   ; CHECK-NEXT:        and i64
-  ; CHECK16-NEXT:      mul i64
   ; CHECK-NEXT:        inttoptr i64 {{.*}} i[[#SBITS]]*
   ; CHECK-COUNT-8:     insertelement {{.*}} i[[#SBITS]]
   ; CHECK-NEXT:        bitcast i[[#SBITS]]* {{.*}} <8 x i[[#SBITS]]>*

test/Instrumentation/DataFlowSanitizer/struct.ll

@@ -1,16 +1,10 @@
-; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,LEGACY
+; RUN: opt < %s -dfsan -dfsan-event-callbacks=true -S | FileCheck %s --check-prefixes=CHECK,EVENT_CALLBACKS
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-event-callbacks=true -S | FileCheck %s --check-prefixes=CHECK,EVENT_CALLBACKS
 ; RUN: opt < %s -dfsan -dfsan-args-abi -S | FileCheck %s --check-prefixes=CHECK,ARGS_ABI
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,FAST
+; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,FAST
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s --check-prefixes=CHECK,NO_COMBINE_LOAD_PTR
+; RUN: opt < %s -dfsan -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s --check-prefixes=CHECK,NO_COMBINE_LOAD_PTR
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-combine-pointer-labels-on-store=true -S | FileCheck %s --check-prefixes=CHECK,COMBINE_STORE_PTR
+; RUN: opt < %s -dfsan -dfsan-combine-pointer-labels-on-store=true -S | FileCheck %s --check-prefixes=CHECK,COMBINE_STORE_PTR
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-track-select-control-flow=false -S | FileCheck %s --check-prefixes=CHECK,NO_SELECT_CONTROL
+; RUN: opt < %s -dfsan -dfsan-track-select-control-flow=false -S | FileCheck %s --check-prefixes=CHECK,NO_SELECT_CONTROL
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-debug-nonzero-labels -S | FileCheck %s --check-prefixes=CHECK,DEBUG_NONZERO_LABELS
+; RUN: opt < %s -dfsan -dfsan-debug-nonzero-labels -S | FileCheck %s --check-prefixes=CHECK,DEBUG_NONZERO_LABELS
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -S | FileCheck %s --check-prefixes=CHECK,FAST
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s --check-prefixes=CHECK,NO_COMBINE_LOAD_PTR
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-combine-pointer-labels-on-store=true -S | FileCheck %s --check-prefixes=CHECK,COMBINE_STORE_PTR
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-track-select-control-flow=false -S | FileCheck %s --check-prefixes=CHECK,NO_SELECT_CONTROL
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -dfsan-debug-nonzero-labels -S | FileCheck %s --check-prefixes=CHECK,DEBUG_NONZERO_LABELS
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -91,11 +85,6 @@ define {i1, i32} @select_struct(i1 %c, {i1, i32} %a, {i1, i32} %b) {
   ; FAST: %[[#R+9]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } %[[#R+8]], i[[#SBITS]] %[[#R+7]], 1
   ; FAST: store { i[[#SBITS]], i[[#SBITS]] } %[[#R+9]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
 
-  ; LEGACY: @"dfs$select_struct"
-  ; LEGACY: [[U:%.*]] = call zeroext i[[#SBITS]] @__dfsan_union
-  ; LEGACY: [[P:%.*]] = phi i[[#SBITS]] [ [[U]],
-  ; LEGACY: store i[[#SBITS]] [[P]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align 2
-
   %s = select i1 %c, {i1, i32} %a, {i1, i32} %b
   ret {i1, i32} %s
 }
@@ -109,13 +98,6 @@ define { i32, i32 } @asm_struct(i32 %0, i32 %1) {
   ; FAST: [[S1:%.*]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } [[S0]], i[[#SBITS]] [[E01]], 1
   ; FAST: store { i[[#SBITS]], i[[#SBITS]] } [[S1]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
 
-  ; LEGACY: @"dfs$asm_struct"
-  ; LEGACY: [[E1:%.*]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN:2]]
-  ; LEGACY: [[E0:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; LEGACY: [[E01:%.*]] = call zeroext i[[#SBITS]] @__dfsan_union(i[[#SBITS]] zeroext [[E0]], i[[#SBITS]] zeroext [[E1]])
-  ; LEGACY: [[P:%.*]] = phi i[[#SBITS]] [ [[E01]], {{.*}} ], [ [[E0]], {{.*}} ]
-  ; LEGACY: store i[[#SBITS]] [[P]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
-
 entry:
   %a = call { i32, i32 } asm "", "=r,=r,r,r,~{dirflag},~{fpsr},~{flags}"(i32 %0, i32 %1)
   ret { i32, i32 } %a
@@ -124,9 +106,6 @@ entry:
 define {i32, i32} @const_struct() {
   ; FAST: @"dfs$const_struct"
   ; FAST: store { i[[#SBITS]], i[[#SBITS]] } zeroinitializer, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align 2
-
-  ; LEGACY: @"dfs$const_struct"
-  ; LEGACY: store i[[#SBITS]] 0, i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align 2
   ret {i32, i32} { i32 42, i32 11 }
 }
 
@@ -136,9 +115,6 @@ define i1 @extract_struct({i1, i5} %s) {
   ; FAST: [[EM:%.*]] = extractvalue { i[[#SBITS]], i[[#SBITS]] } [[SM]], 0
   ; FAST: store i[[#SBITS]] [[EM]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
 
-  ; LEGACY: @"dfs$extract_struct"
-  ; LEGACY: [[SM:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN:2]]
-  ; LEGACY: store i[[#SBITS]] [[SM]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
   %e2 = extractvalue {i1, i5} %s, 0
   ret i1 %e2
 }
@@ -149,13 +125,6 @@ define {i1, i5} @insert_struct({i1, i5} %s, i5 %e1) {
   ; FAST: [[SM:%.*]] = load { i[[#SBITS]], i[[#SBITS]] }, { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
   ; FAST: [[SM1:%.*]] = insertvalue { i[[#SBITS]], i[[#SBITS]] } [[SM]], i[[#SBITS]] [[EM]], 1
   ; FAST: store { i[[#SBITS]], i[[#SBITS]] } [[SM1]], { i[[#SBITS]], i[[#SBITS]] }* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to { i[[#SBITS]], i[[#SBITS]] }*), align [[ALIGN]]
-
-  ; LEGACY: @"dfs$insert_struct"
-  ; LEGACY: [[EM:%.*]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([[TLS_ARR]]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN:2]]
-  ; LEGACY: [[SM:%.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-  ; LEGACY: [[U:%.*]] = call zeroext i[[#SBITS]] @__dfsan_union(i[[#SBITS]] zeroext [[SM]], i[[#SBITS]] zeroext [[EM]])
-  ; LEGACY: [[P:%.*]] = phi i[[#SBITS]] [ [[U]], {{.*}} ], [ [[SM]], {{.*}} ]
-  ; LEGACY: store i[[#SBITS]] [[P]], i[[#SBITS]]* bitcast ([[TLS_ARR]]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]
   %s1 = insertvalue {i1, i5} %s, i5 %e1, 1
   ret {i1, i5} %s1
 }

test/Instrumentation/DataFlowSanitizer/union.ll

@@ -2,6 +2,9 @@
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
+; CHECK: @__dfsan_shadow_width_bits = weak_odr constant i32 [[#SBITS:]]
+; CHECK: @__dfsan_shadow_width_bytes = weak_odr constant i32 [[#SBYTES:]]
+
 @a = common global i32 0
 @b = common global i32 0
 
@@ -9,10 +12,10 @@ target triple = "x86_64-unknown-linux-gnu"
 
 ; CHECK-LABEL: @"dfs$f"
 define void @f(i32 %x, i32 %y) {
-  ; CHECK: call{{.*}}__dfsan_union
+  ; CHECK: or i[[#SBITS]]
   %xay = add i32 %x, %y
   store i32 %xay, i32* @a
-  ; CHECK-NOT: call{{.*}}__dfsan_union
+  ; CHECK-NOT: or i[[#SBITS]]
   %xmy = mul i32 %x, %y
   store i32 %xmy, i32* @b
   ret void
@@ -26,13 +29,13 @@ define void @g(i1 %p, i32 %x, i32 %y) {
   br i1 %p, label %l1, label %l2
 
 l1:
-  ; CHECK: call{{.*}}__dfsan_union
+  ; CHECK: or i[[#SBITS]]
   %xay = add i32 %x, %y
   store i32 %xay, i32* @a
   br label %l3
 
 l2:
-  ; CHECK: call{{.*}}__dfsan_union
+  ; CHECK: or i[[#SBITS]]
   %xmy = mul i32 %x, %y
   store i32 %xmy, i32* @b
   br label %l3
@@ -45,9 +48,9 @@ l3:
 
 ; CHECK-LABEL: @"dfs$h"
 define i32 @h(i32 %x, i32 %y) {
-  ; CHECK: call{{.*}}__dfsan_union
+  ; CHECK: or i[[#SBITS]]
   %xay = add i32 %x, %y
-  ; CHECK-NOT: call{{.*}}__dfsan_union
+  ; CHECK-NOT: or i[[#SBITS]]
   %xayax = add i32 %xay, %x
   ret i32 %xayax
 }

test/Instrumentation/DataFlowSanitizer/vector.ll

@@ -1,7 +1,5 @@
-; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,TLS_ABI,TLS_ABI_LEGACY
 ; RUN: opt < %s -dfsan -dfsan-args-abi -S | FileCheck %s --check-prefixes=CHECK,ARGS_ABI
-; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefixes=CHECK,TLS_ABI,TLS_ABI_FAST
+; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefixes=CHECK,TLS_ABI
-; RUN: opt < %s -dfsan -dfsan-fast-8-labels=true -S | FileCheck %s --check-prefixes=CHECK,TLS_ABI,TLS_ABI_FAST
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -37,12 +35,7 @@ define <4 x i1> @icmp_vector(<4 x i8> %a, <4 x i8> %b) {
   ; TLS_ABI-LABEL: @"dfs$icmp_vector"
   ; TLS_ABI-NEXT: %[[B:.*]] = load i[[#SBITS]], i[[#SBITS]]* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 2) to i[[#SBITS]]*), align [[ALIGN:2]]
   ; TLS_ABI-NEXT: %[[A:.*]] = load i[[#SBITS]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_arg_tls to i[[#SBITS]]*), align [[ALIGN]]
-
+  ; TLS_ABI:       %[[L:.*]] = or i[[#SBITS]] %[[A]], %[[B]]
-  ; TLS_ABI_LEGACY: %[[U:.*]] = call zeroext i[[#SBITS]] @__dfsan_union(i[[#SBITS]] zeroext %[[A]], i[[#SBITS]] zeroext %[[B]])
-  ; TLS_ABI_LEGACY: %[[L:.*]] = phi i[[#SBITS]] [ %[[U]], {{.*}} ], [ %[[A]], {{.*}} ]
-
-  ; COM: With fast labels enabled, union is just an OR.
-  ; TLS_ABI_FAST: %[[L:.*]] = or i[[#SBITS]] %[[A]], %[[B]]
 
   ; TLS_ABI: %r = icmp eq <4 x i8> %a, %b
   ; TLS_ABI: store i[[#SBITS]] %[[L]], i[[#SBITS]]* bitcast ([100 x i64]* @__dfsan_retval_tls to i[[#SBITS]]*), align [[ALIGN]]