[gicombiner] Add a CodeExpander to handle C++ fragments with variable expansion

Summary:
This will handle expansion of C++ fragments in the declarative combiner
including custom predicates, and escapes into C++ to aid the migration
effort.

Fixed the -DLLVM_LINK_LLVM_DYLIB=ON using DISABLE_LLVM_LINK_LLVM_DYLIB when
creating the library. Apparently it automatically links to libLLVM.dylib
and we don't want that from tablegen.

Reviewers: bogner, volkan

Subscribers: mgorny, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D68288

> llvm-svn: 373551

llvm-svn: 373651

unittests/CMakeLists.txt

@@ -31,8 +31,9 @@ add_subdirectory(Remarks)
 add_subdirectory(Passes)
 add_subdirectory(ProfileData)
 add_subdirectory(Support)
-add_subdirectory(TextAPI)
+add_subdirectory(TableGen)
 add_subdirectory(Target)
+add_subdirectory(TextAPI)
 add_subdirectory(Transforms)
 add_subdirectory(XRay)
 add_subdirectory(tools)

unittests/TableGen/CMakeLists.txt

@@ -0,0 +1,10 @@
+set(LLVM_LINK_COMPONENTS
+  TableGen
+  Support
+  )
+
+add_llvm_unittest(TableGenTests
+  CodeExpanderTest.cpp
+  )
+include_directories(${CMAKE_SOURCE_DIR}/utils/TableGen)
+target_link_libraries(TableGenTests PRIVATE LLVMTableGenGlobalISel LLVMTableGen)

unittests/TableGen/CodeExpanderTest.cpp

@@ -0,0 +1,203 @@
+//===- llvm/unittest/TableGen/CodeExpanderTest.cpp - Tests ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "GlobalISel/CodeExpander.h"
+#include "GlobalISel/CodeExpansions.h"
+
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/TableGen/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+static StringRef bufferize(StringRef Str) {
+  std::unique_ptr<MemoryBuffer> Buffer =
+      MemoryBuffer::getMemBufferCopy(Str, "TestBuffer");
+  StringRef StrBufferRef = Buffer->getBuffer();
+  SrcMgr.AddNewSourceBuffer(std::move(Buffer), SMLoc());
+  return StrBufferRef;
+}
+
+class RAIIDiagnosticChecker {
+  std::string EmittedDiags;
+  raw_string_ostream OS;
+  std::vector<SMDiagnostic> Expected;
+  std::vector<SMDiagnostic> Received;
+
+public:
+  RAIIDiagnosticChecker() : OS(EmittedDiags) {
+    SrcMgr.setDiagHandler(handler, this);
+  }
+  ~RAIIDiagnosticChecker() {
+    SrcMgr.setDiagHandler(nullptr);
+    EXPECT_EQ(Received.size(), Expected.size());
+    for (unsigned i = 0; i < Received.size() && i < Expected.size(); ++i) {
+      EXPECT_EQ(Received[i].getLoc(), Expected[i].getLoc());
+      EXPECT_EQ(Received[i].getFilename(), Expected[i].getFilename());
+      EXPECT_EQ(Received[i].getKind(), Expected[i].getKind());
+      EXPECT_EQ(Received[i].getLineNo(), Expected[i].getLineNo());
+      EXPECT_EQ(Received[i].getColumnNo(), Expected[i].getColumnNo());
+      EXPECT_EQ(Received[i].getMessage(), Expected[i].getMessage());
+      EXPECT_EQ(Received[i].getLineContents(), Expected[i].getLineContents());
+      EXPECT_EQ(Received[i].getRanges(), Expected[i].getRanges());
+    }
+
+    if (testing::Test::HasFailure())
+      errs() << "Emitted diagnostic:\n" << OS.str();
+  }
+
+  void expect(SMDiagnostic D) { Expected.push_back(D); }
+
+  void diag(const SMDiagnostic &D) {
+    Received.push_back(D);
+  }
+
+  static void handler(const SMDiagnostic &D, void *Context) {
+    RAIIDiagnosticChecker *Self = static_cast<RAIIDiagnosticChecker *>(Context);
+    Self->diag(D);
+    SrcMgr.setDiagHandler(nullptr);
+    SrcMgr.PrintMessage(Self->OS, D);
+    SrcMgr.setDiagHandler(handler, Context);
+  };
+};
+
+TEST(CodeExpander, NoExpansions) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("No expansions", Expansions, SMLoc(), false).emit(OS);
+  EXPECT_EQ(OS.str(), "No expansions");
+}
+
+// Indentation is applied to all lines except the first
+TEST(CodeExpander, Indentation) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("No expansions\nsecond line\nthird line", Expansions, SMLoc(),
+               false, "  ")
+      .emit(OS);
+  EXPECT_EQ(OS.str(), "No expansions\n  second line\n  third line");
+}
+
+// \ is an escape character that removes special meanings from the next
+// character.
+TEST(CodeExpander, Escape) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("\\\\\\a\\$", Expansions, SMLoc(), false).emit(OS);
+  EXPECT_EQ(OS.str(), "\\a$");
+}
+
+// $foo is not an expansion. It should warn though.
+TEST(CodeExpander, NotAnExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  StringRef In = bufferize(" $foo");
+  CodeExpander(" $foo", Expansions, SMLoc::getFromPointer(In.data()), false)
+      .emit(OS);
+  EXPECT_EQ(OS.str(), " $foo");
+  DiagChecker.expect(SMDiagnostic(
+      SrcMgr, SMLoc::getFromPointer(In.data() + 1), "TestBuffer", 1, 1,
+      SourceMgr::DK_Warning, "Assuming missing escape character", " $foo", {}));
+}
+
+// \$foo is not an expansion but shouldn't warn as it's using the escape.
+TEST(CodeExpander, EscapedNotAnExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("\\$foo", Expansions, SMLoc(), false).emit(OS);
+  EXPECT_EQ(OS.str(), "$foo");
+}
+
+// \${foo is not an expansion but shouldn't warn as it's using the escape.
+TEST(CodeExpander, EscapedUnterminatedExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("\\${foo", Expansions, SMLoc(), false).emit(OS);
+  EXPECT_EQ(OS.str(), "${foo");
+}
+
+// \${foo is not an expansion but shouldn't warn as it's using the escape.
+TEST(CodeExpander, EscapedExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("\\${foo}", Expansions, SMLoc(), false).emit(OS);
+  EXPECT_EQ(OS.str(), "${foo}");
+}
+
+// ${foo} is an undefined expansion and should error.
+TEST(CodeExpander, UndefinedExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+  Expansions.declare("bar", "expansion");
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("${foo}${bar}", Expansions, SMLoc(), false).emit(OS);
+  EXPECT_EQ(OS.str(), "expansion");
+  DiagChecker.expect(
+      SMDiagnostic(SrcMgr, SMLoc(), "<unknown>", 0, -1, SourceMgr::DK_Error,
+                   "Attempting to expand an undeclared variable foo", "", {}));
+}
+
+// ${foo} is an undefined expansion and should error. When given a valid
+// location for the start of the buffer it should correctly point at the
+// expansion being performed.
+TEST(CodeExpander, UndefinedExpansionWithLoc) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+  Expansions.declare("bar", "expansion");
+
+  RAIIDiagnosticChecker DiagChecker;
+  StringRef In = bufferize("Padding ${foo}${bar}");
+  CodeExpander(In, Expansions, SMLoc::getFromPointer(In.data()), false)
+      .emit(OS);
+  EXPECT_EQ(OS.str(), "Padding expansion");
+  DiagChecker.expect(SMDiagnostic(
+      SrcMgr, SMLoc::getFromPointer(In.data() + 8), "TestBuffer", 1, 8,
+      SourceMgr::DK_Error, "Attempting to expand an undeclared variable foo",
+      "Padding ${foo}${bar}", {}));
+}
+
+// ${bar is an unterminated expansion. Warn and implicitly terminate it.
+TEST(CodeExpander, UnterminatedExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+  Expansions.declare("bar", "expansion");
+
+  RAIIDiagnosticChecker DiagChecker;
+  StringRef In = bufferize(" ${bar");
+  CodeExpander(In, Expansions, SMLoc::getFromPointer(In.data()), false)
+      .emit(OS);
+  EXPECT_EQ(OS.str(), " expansion");
+  DiagChecker.expect(SMDiagnostic(SrcMgr, SMLoc::getFromPointer(In.data() + 1),
+                                  "TestBuffer", 1, 1, SourceMgr::DK_Warning,
+                                  "Unterminated expansion", " ${bar", {}));
+}

utils/TableGen/CMakeLists.txt

@@ -1,3 +1,5 @@
+add_subdirectory(GlobalISel)
+
 set(LLVM_LINK_COMPONENTS Support)
 
 add_tablegen(llvm-tblgen LLVM
@@ -50,4 +52,5 @@ add_tablegen(llvm-tblgen LLVM
   WebAssemblyDisassemblerEmitter.cpp
   CTagsEmitter.cpp
   )
+target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenGlobalISel)
 set_target_properties(llvm-tblgen PROPERTIES FOLDER "Tablegenning")

utils/TableGen/GICombinerEmitter.cpp

@@ -26,6 +26,11 @@ cl::OptionCategory
 static cl::list<std::string>
     SelectedCombiners("combiners", cl::desc("Emit the specified combiners"),
                       cl::cat(GICombinerEmitterCat), cl::CommaSeparated);
+static cl::opt<bool> ShowExpansions(
+    "gicombiner-show-expansions",
+    cl::desc("Use C++ comments to indicate occurence of code expansion"),
+    cl::cat(GICombinerEmitterCat));
+
 namespace {
 class GICombinerEmitter {
   StringRef Name;

utils/TableGen/GlobalISel/CMakeLists.txt

@@ -0,0 +1,7 @@
+set(LLVM_LINK_COMPONENTS
+  Support
+  )
+
+llvm_add_library(LLVMTableGenGlobalISel STATIC DISABLE_LLVM_LINK_LLVM_DYLIB
+                 CodeExpander.cpp
+                )

utils/TableGen/GlobalISel/CodeExpander.cpp

@@ -0,0 +1,93 @@
+//===- CodeExpander.cpp - Expand variables in a string --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Expand the variables in a string.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CodeExpander.h"
+#include "CodeExpansions.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/TableGen/Error.h"
+
+using namespace llvm;
+
+void CodeExpander::emit(raw_ostream &OS) const {
+  StringRef Current = Code;
+
+  while (!Current.empty()) {
+    size_t Pos = Current.find_first_of("$\n\\");
+    if (Pos == StringRef::npos) {
+      OS << Current;
+      Current = "";
+      continue;
+    }
+
+    OS << Current.substr(0, Pos);
+    Current = Current.substr(Pos);
+
+    if (Current.startswith("\n")) {
+      OS << "\n" << Indent;
+      Current = Current.drop_front(1);
+      continue;
+    }
+
+    if (Current.startswith("\\$") || Current.startswith("\\\\")) {
+      OS << Current[1];
+      Current = Current.drop_front(2);
+      continue;
+    }
+
+    if (Current.startswith("\\")) {
+      Current = Current.drop_front(1);
+      continue;
+    }
+
+    if (Current.startswith("${")) {
+      StringRef StartVar = Current;
+      Current = Current.drop_front(2);
+      StringRef Var;
+      std::tie(Var, Current) = Current.split("}");
+
+      // Warn if we split because no terminator was found.
+      StringRef EndVar = StartVar.drop_front(2 /* ${ */ + Var.size());
+      if (EndVar.empty()) {
+        size_t LocOffset = StartVar.data() - Code.data();
+        PrintWarning(
+            Loc.size() > 0 && Loc[0].isValid()
+                ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset)
+                : SMLoc(),
+            "Unterminated expansion");
+      }
+
+      auto ValueI = Expansions.find(Var);
+      if (ValueI == Expansions.end()) {
+        size_t LocOffset = StartVar.data() - Code.data();
+        PrintError(Loc.size() > 0 && Loc[0].isValid()
+                       ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset)
+                       : SMLoc(),
+                   "Attempting to expand an undeclared variable " + Var);
+      }
+      if (ShowExpansions)
+        OS << "/*$" << Var << "{*/";
+      OS << Expansions.lookup(Var);
+      if (ShowExpansions)
+        OS << "/*}*/";
+      continue;
+    }
+
+    size_t LocOffset = Current.data() - Code.data();
+    PrintWarning(Loc.size() > 0 && Loc[0].isValid()
+                     ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset)
+                     : SMLoc(),
+                 "Assuming missing escape character");
+    OS << "$";
+    Current = Current.drop_front(1);
+  }
+}

utils/TableGen/GlobalISel/CodeExpander.h

@@ -0,0 +1,55 @@
+//===- CodeExpander.h - Expand variables in a string ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Expand the variables in a string.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_UTILS_TABLEGEN_CODEEXPANDER_H
+#define LLVM_UTILS_TABLEGEN_CODEEXPANDER_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/SMLoc.h"
+
+namespace llvm {
+class CodeExpansions;
+class raw_ostream;
+
+/// Emit the given code with all '${foo}' placeholders expanded to their
+/// replacements.
+///
+/// It's an error to use an undefined expansion and expansion-like output that
+/// needs to be emitted verbatim can be escaped as '\${foo}'
+///
+/// The emitted code can be given a custom indent to enable both indentation by
+/// an arbitrary amount of whitespace and emission of the code as a comment.
+class CodeExpander {
+  StringRef Code;
+  const CodeExpansions &Expansions;
+  const ArrayRef<SMLoc> &Loc;
+  bool ShowExpansions;
+  StringRef Indent;
+
+public:
+  CodeExpander(StringRef Code, const CodeExpansions &Expansions,
+               const ArrayRef<SMLoc> &Loc, bool ShowExpansions,
+               StringRef Indent = "    ")
+      : Code(Code), Expansions(Expansions), Loc(Loc),
+        ShowExpansions(ShowExpansions), Indent(Indent) {}
+
+  void emit(raw_ostream &OS) const;
+};
+
+inline raw_ostream &operator<<(raw_ostream &OS, const CodeExpander &Expander) {
+  Expander.emit(OS);
+  return OS;
+}
+} // end namespace llvm
+
+#endif // ifndef LLVM_UTILS_TABLEGEN_CODEEXPANDER_H

utils/TableGen/GlobalISel/CodeExpansions.h

@@ -0,0 +1,43 @@
+//===- CodeExpansions.h - Record expansions for CodeExpander --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Record the expansions to use in a CodeExpander.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringMap.h"
+
+#ifndef LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H
+#define LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H
+namespace llvm {
+class CodeExpansions {
+public:
+  using const_iterator = StringMap<std::string>::const_iterator;
+
+protected:
+  StringMap<std::string> Expansions;
+
+public:
+  void declare(StringRef Name, StringRef Expansion) {
+    bool Inserted = Expansions.try_emplace(Name, Expansion).second;
+    assert(Inserted && "Declared variable twice");
+    (void)Inserted;
+  }
+
+  std::string lookup(StringRef Variable) const {
+    return Expansions.lookup(Variable);
+  }
+
+  const_iterator begin() const { return Expansions.begin(); }
+  const_iterator end() const { return Expansions.end(); }
+  const_iterator find(StringRef Variable) const {
+    return Expansions.find(Variable);
+  }
+};
+} // end namespace llvm
+#endif // ifndef LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H