[llvm-objcopy] Dwarf .debug section compression support (zlib, zlib-gnu).

Third Attempt:
    - Alignment issues resolved.
    - zlib::isAvailable() detected.
    - ArrayRef misuse fixed.

  Usage:

  llvm-objcopy --compress-debug-sections=zlib foo.o
  llvm-objcopy --compress-debug-sections=zlib-gnu foo.o

  In both cases the debug section contents is compressed with zlib. In the GNU
  style case the header is the "ZLIB" magic string followed by the uint64 big-
  endian decompressed size. In the non-GNU mode the header is the
  Elf(32|64)_Chdr.

  Decompression support is coming soon.

  Differential Revision: https://reviews.llvm.org/D49678

llvm-svn: 341635

test/tools/llvm-objcopy/Inputs/compress-debug-sections.yaml

@@ -0,0 +1,21 @@
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .debug_foo
+    Type:            SHT_PROGBITS
+    Content:         0000000000000000
+  - Name:            .notdebug_foo
+    Type:            SHT_PROGBITS
+    Content:         0000000000000000
+  - Name:            .rela.debug_foo
+    Type:            SHT_RELA
+    Info:            .debug_foo
+    Relocations:
+      - Offset:          0x1
+        Symbol:          .debug_foo
+        Type:            R_X86_64_32
+...

test/tools/llvm-objcopy/compress-debug-sections-default-gnu.test

@@ -0,0 +1,9 @@
+# REQUIRES: zlib
+
+# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o
+# RUN: llvm-objcopy --compress-debug-sections --compress-debug-sections=zlib-gnu %t.o %t-compressed.o
+# RUN: llvm-objdump -s %t-compressed.o | FileCheck %s
+
+# CHECK: .zdebug_foo:
+# CHECK: ZLIB
+

test/tools/llvm-objcopy/compress-debug-sections-default.test

@@ -0,0 +1,13 @@
+# REQUIRES: zlib
+
+# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o
+# RUN: llvm-objcopy --compress-debug-sections %t.o %t-compressed.o
+# RUN: llvm-readobj -s %t-compressed.o | FileCheck %s
+
+# CHECK: Name: .debug_foo
+# CHECK-NEXT: Type: SHT_PROGBITS
+# CHECK-NEXT: Flags [
+# CHECK-NEXT: SHF_COMPRESSED
+# CHECK-NEXT: ]
+# CHECK-NOT: Name: .debug_foo
+

test/tools/llvm-objcopy/compress-debug-sections-invalid-format.test

@@ -0,0 +1,5 @@
+# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o
+# RUN: not llvm-objcopy --compress-debug-sections=zlib-fake %t.o 2>&1 | FileCheck %s
+
+# CHECK: Invalid or unsupported --compress-debug-sections format: zlib-fake.
+

test/tools/llvm-objcopy/compress-debug-sections-zlib-gnu.test

@@ -0,0 +1,49 @@
+# REQUIRES: zlib
+
+# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o
+# RUN: llvm-objcopy --compress-debug-sections=zlib-gnu %t.o %t-compressed.o
+
+# RUN: llvm-objdump -s %t.o -section=.debug_foo | FileCheck %s
+# RUN: llvm-objdump -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-COMPRESSED
+# RUN: llvm-readobj -relocations -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-FLAGS
+
+# CHECK: .debug_foo:
+
+# CHECK-COMPRESSED: .zdebug_foo:
+# CHECK-COMPRESSED: ZLIB
+# CHECK-COMPRESSED: .notdebug_foo:
+
+# CHECK-FLAGS-NOT: Name: .debug_foo
+# CHECK-FLAGS: Index: 1
+# CHECK-FLAGS-NEXT: Name: .zdebug_foo
+# CHECK-FLAGS-NEXT: Type: SHT_PROGBITS
+# CHECK-FLAGS-NEXT: Flags [
+# CHECK-FLAGS-NEXT: ]
+# CHECK-FLAGS-NEXT: Address:
+# CHECK-FLAGS-NEXT: Offset:
+# CHECK-FLAGS-NEXT: Size: 23
+
+# CHECK-FLAGS: Name: .notdebug_foo
+# CHECK-FLAGS-NEXT: Type: SHT_PROGBITS
+# CHECK-FLAGS-NEXT: Flags [
+# CHECK-FLAGS-NEXT: ]
+# CHECK-FLAGS-NEXT: Address:
+# CHECK-FLAGS-NEXT: Offset:
+# CHECK-FLAGS-NEXT: Size: 8
+
+# CHECK-FLAGS: Name: .rela.debug_foo
+# CHECK-FLAGS-NEXT: Type: SHT_RELA
+# CHECK-FLAGS-NEXT: Flags [
+# CHECK-FLAGS-NEXT: ]
+# CHECK-FLAGS-NEXT: Address:
+# CHECK-FLAGS-NEXT: Offset:
+# CHECK-FLAGS-NEXT: Size:
+# CHECK-FLAGS-NEXT: Link:
+# CHECK-FLAGS-NEXT: Info: 1
+
+# CHECK-FLAGS: Relocations [
+# CHECK-FLAGS-NEXT:   .rela.debug_foo {
+# CHECK-FLAGS-NEXT:     0x1 R_X86_64_32 - 0x0
+# CHECK-FLAGS-NEXT:   }
+# CHECK-FLAGS-NEXT: ]
+

test/tools/llvm-objcopy/compress-debug-sections-zlib.test

@@ -0,0 +1,49 @@
+# REQUIRES: zlib
+
+# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o
+# RUN: llvm-objcopy --compress-debug-sections=zlib %t.o %t-compressed.o
+
+# RUN: llvm-objdump -s %t.o -section=.debug_foo | FileCheck %s
+# RUN: llvm-objdump -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-COMPRESSED
+# RUN: llvm-readobj -relocations -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-FLAGS
+
+# CHECK: .debug_foo:
+
+# CHECK-COMPRESSED: .debug_foo:
+# CHECK-COMPRESSED: .notdebug_foo:
+
+# CHECK-FLAGS: Index: 1
+# CHECK-FLAGS-NEXT: Name: .debug_foo
+# CHECK-FLAGS-NEXT: Type: SHT_PROGBITS
+# CHECK-FLAGS-NEXT: Flags [
+# CHECK-FLAGS-NEXT: SHF_COMPRESSED
+# CHECK-FLAGS-NEXT: ]
+# CHECK-FLAGS-NEXT: Address:
+# CHECK-FLAGS-NEXT: Offset:
+# CHECK-FLAGS-NEXT: Size: 35
+# CHECK-FLAGS-NOT: Name: .debug_foo
+
+# CHECK-FLAGS: Name: .notdebug_foo
+# CHECK-FLAGS-NEXT: Type: SHT_PROGBITS
+# CHECK-FLAGS-NEXT: Flags [
+# CHECK-FLAGS-NEXT: ]
+# CHECK-FLAGS-NEXT: Address:
+# CHECK-FLAGS-NEXT: Offset:
+# CHECK-FLAGS-NEXT: Size: 8
+
+# CHECK-FLAGS: Name: .rela.debug_foo
+# CHECK-FLAGS-NEXT: Type: SHT_RELA
+# CHECK-FLAGS-NEXT: Flags [
+# CHECK-FLAGS-NEXT: ]
+# CHECK-FLAGS-NEXT: Address:
+# CHECK-FLAGS-NEXT: Offset:
+# CHECK-FLAGS-NEXT: Size:
+# CHECK-FLAGS-NEXT: Link:
+# CHECK-FLAGS-NEXT: Info: 1
+
+# CHECK-FLAGS: Relocations [
+# CHECK-FLAGS-NEXT:   .rela.debug_foo {
+# CHECK-FLAGS-NEXT:     0x1 R_X86_64_32 - 0x0
+# CHECK-FLAGS-NEXT:   }
+# CHECK-FLAGS-NEXT: ]
+

tools/llvm-objcopy/ObjcopyOpts.td

@@ -17,6 +17,12 @@ def I : JoinedOrSeparate<[ "-" ], "I">, Alias<input_target>;
 defm output_target : Eq<"output-target">,
                      HelpText<"Format of the output file">,
                      Values<"binary">;
+def compress_debug_sections : Flag<["--", "-"], "compress-debug-sections">;
+def compress_debug_sections_eq : Joined<["--", "-"], "compress-debug-sections=">,
+                                 MetaVarName<"[ zlib | zlib-gnu ]">,
+                                 HelpText<"Compress DWARF debug sections using "
+                                          "specified style. Supported styles: "
+                                          "'zlib-gnu' and 'zlib'">;
 def O : JoinedOrSeparate<["-"], "O">,
         Alias<output_target>;
 defm split_dwo : Eq<"split-dwo">,

tools/llvm-objcopy/Object.cpp

@@ -15,7 +15,9 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Support/Compression.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileOutputBuffer.h"
 #include "llvm/Support/Path.h"
@@ -138,6 +140,76 @@ void OwnedDataSection::accept(SectionVisitor &Visitor) const {
   Visitor.visit(*this);
 }
 
+void BinarySectionWriter::visit(const CompressedSection &Sec) {
+  error("Cannot write compressed section '" + Sec.Name + "' ");
+}
+
+template <class ELFT>
+void ELFSectionWriter<ELFT>::visit(const CompressedSection &Sec) {
+  uint8_t *Buf = Out.getBufferStart();
+  Buf += Sec.Offset;
+
+  if (Sec.CompressionType == DebugCompressionType::None) {
+    std::copy(Sec.OriginalData.begin(), Sec.OriginalData.end(), Buf);
+    return;
+  }
+
+  if (Sec.CompressionType == DebugCompressionType::GNU) {
+    const char *Magic = "ZLIB";
+    memcpy(Buf, Magic, strlen(Magic));
+    Buf += strlen(Magic);
+    const uint64_t DecompressedSize =
+        support::endian::read64be(&Sec.DecompressedSize);
+    memcpy(Buf, &DecompressedSize, sizeof(DecompressedSize));
+    Buf += sizeof(DecompressedSize);
+  } else {
+    Elf_Chdr_Impl<ELFT> Chdr;
+    Chdr.ch_type = ELF::ELFCOMPRESS_ZLIB;
+    Chdr.ch_size = Sec.DecompressedSize;
+    Chdr.ch_addralign = Sec.DecompressedAlign;
+    memcpy(Buf, &Chdr, sizeof(Chdr));
+    Buf += sizeof(Chdr);
+  }
+
+  std::copy(Sec.CompressedData.begin(), Sec.CompressedData.end(), Buf);
+}
+
+CompressedSection::CompressedSection(const SectionBase &Sec,
+                                     DebugCompressionType CompressionType)
+    : SectionBase(Sec), CompressionType(CompressionType),
+      DecompressedSize(Sec.OriginalData.size()), DecompressedAlign(Sec.Align) {
+
+  if (!zlib::isAvailable()) {
+    CompressionType = DebugCompressionType::None;
+    return;
+  }
+
+  if (Error E = zlib::compress(
+          StringRef(reinterpret_cast<const char *>(OriginalData.data()),
+                    OriginalData.size()),
+          CompressedData))
+    reportError(Name, std::move(E));
+
+  size_t ChdrSize;
+  if (CompressionType == DebugCompressionType::GNU) {
+    Name = ".z" + Sec.Name.substr(1);
+    ChdrSize = sizeof("ZLIB") - 1 + sizeof(uint64_t);
+  } else {
+    Flags |= ELF::SHF_COMPRESSED;
+    ChdrSize =
+        std::max(std::max(sizeof(object::Elf_Chdr_Impl<object::ELF64LE>),
+                          sizeof(object::Elf_Chdr_Impl<object::ELF64BE>)),
+                 std::max(sizeof(object::Elf_Chdr_Impl<object::ELF32LE>),
+                          sizeof(object::Elf_Chdr_Impl<object::ELF32BE>)));
+  }
+  Size = ChdrSize + CompressedData.size();
+  Align = 8;
+}
+
+void CompressedSection::accept(SectionVisitor &Visitor) const {
+  Visitor.visit(*this);
+}
+
 void StringTableSection::addString(StringRef Name) {
   StrTabBuilder.add(Name);
   Size = StrTabBuilder.getSize();

tools/llvm-objcopy/Object.h

@@ -26,6 +26,7 @@
 #include <vector>
 
 namespace llvm {
+enum class DebugCompressionType;
 namespace objcopy {
 
 class Buffer;
@@ -39,6 +40,7 @@ class DynamicRelocationSection;
 class GnuDebugLinkSection;
 class GroupSection;
 class SectionIndexSection;
+class CompressedSection;
 class Segment;
 class Object;
 struct Symbol;
@@ -86,6 +88,7 @@ public:
   virtual void visit(const GnuDebugLinkSection &Sec) = 0;
   virtual void visit(const GroupSection &Sec) = 0;
   virtual void visit(const SectionIndexSection &Sec) = 0;
+  virtual void visit(const CompressedSection &Sec) = 0;
 };
 
 class SectionWriter : public SectionVisitor {
@@ -104,6 +107,7 @@ public:
   virtual void visit(const GnuDebugLinkSection &Sec) override = 0;
   virtual void visit(const GroupSection &Sec) override = 0;
   virtual void visit(const SectionIndexSection &Sec) override = 0;
+  virtual void visit(const CompressedSection &Sec) override = 0;
 
   explicit SectionWriter(Buffer &Buf) : Out(Buf) {}
 };
@@ -122,6 +126,7 @@ public:
   void visit(const GnuDebugLinkSection &Sec) override;
   void visit(const GroupSection &Sec) override;
   void visit(const SectionIndexSection &Sec) override;
+  void visit(const CompressedSection &Sec) override;
 
   explicit ELFSectionWriter(Buffer &Buf) : SectionWriter(Buf) {}
 };
@@ -139,6 +144,7 @@ public:
   void visit(const GnuDebugLinkSection &Sec) override;
   void visit(const GroupSection &Sec) override;
   void visit(const SectionIndexSection &Sec) override;
+  void visit(const CompressedSection &Sec) override;
 
   explicit BinarySectionWriter(Buffer &Buf) : SectionWriter(Buf) {}
 };
@@ -246,7 +252,7 @@ public:
 
 class SectionBase {
 public:
-  StringRef Name;
+  std::string Name;
   Segment *ParentSegment = nullptr;
   uint64_t HeaderOffset;
   uint64_t OriginalOffset = std::numeric_limits<uint64_t>::max();
@@ -265,6 +271,9 @@ public:
   uint64_t Type = ELF::SHT_NULL;
   ArrayRef<uint8_t> OriginalData;
 
+  SectionBase() = default;
+  SectionBase(const SectionBase &) = default;
+
   virtual ~SectionBase() = default;
 
   virtual void initialize(SectionTableRef SecTable);
@@ -341,7 +350,7 @@ class OwnedDataSection : public SectionBase {
 public:
   OwnedDataSection(StringRef SecName, ArrayRef<uint8_t> Data)
       : Data(std::begin(Data), std::end(Data)) {
-    Name = SecName;
+    Name = SecName.str();
     Type = ELF::SHT_PROGBITS;
     Size = Data.size();
     OriginalOffset = std::numeric_limits<uint64_t>::max();
@@ -350,6 +359,20 @@ public:
   void accept(SectionVisitor &Sec) const override;
 };
 
+class CompressedSection : public SectionBase {
+  MAKE_SEC_WRITER_FRIEND
+
+  DebugCompressionType CompressionType;
+  uint64_t DecompressedSize;
+  uint64_t DecompressedAlign;
+  SmallVector<char, 128> CompressedData;
+
+public:
+  CompressedSection(const SectionBase &Sec,
+                    DebugCompressionType CompressionType);
+  void accept(SectionVisitor &Visitor) const override;
+};
+
 // There are two types of string tables that can exist, dynamic and not dynamic.
 // In the dynamic case the string table is allocated. Changing a dynamic string
 // table would mean altering virtual addresses and thus the memory image. So

tools/llvm-objcopy/llvm-objcopy.cpp

@@ -17,6 +17,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/ArchiveWriter.h"
 #include "llvm/Object/Binary.h"
@@ -29,6 +30,7 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/Compression.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ErrorOr.h"
@@ -176,6 +178,7 @@ struct CopyConfig {
   bool StripSections = false;
   bool StripUnneeded = false;
   bool Weaken = false;
+  DebugCompressionType CompressionType = DebugCompressionType::None;
 };
 
 // Configuration for the overall invocation of this tool. When invoked as
@@ -298,12 +301,12 @@ static SectionRename parseRenameSectionValue(StringRef FlagValue) {
 }
 
 static bool isDebugSection(const SectionBase &Sec) {
-  return Sec.Name.startswith(".debug") || Sec.Name.startswith(".zdebug") ||
-         Sec.Name == ".gdb_index";
+  return StringRef(Sec.Name).startswith(".debug") ||
+         StringRef(Sec.Name).startswith(".zdebug") || Sec.Name == ".gdb_index";
 }
 
 static bool isDWOSection(const SectionBase &Sec) {
-  return Sec.Name.endswith(".dwo");
+  return StringRef(Sec.Name).endswith(".dwo");
 }
 
 static bool onlyKeepDWOPred(const Object &Obj, const SectionBase &Sec) {
@@ -413,6 +416,50 @@ static Error dumpSectionToFile(StringRef SecName, StringRef Filename,
                                  object_error::parse_failed);
 }
 
+static bool isCompressed(const SectionBase &Section) {
+  const char *Magic = "ZLIB";
+  return StringRef(Section.Name).startswith(".zdebug") ||
+         (Section.OriginalData.size() > strlen(Magic) &&
+          !strncmp(reinterpret_cast<const char *>(Section.OriginalData.data()),
+                   Magic, strlen(Magic))) ||
+         (Section.Flags & ELF::SHF_COMPRESSED);
+}
+
+static bool isCompressable(const SectionBase &Section) {
+  return !isCompressed(Section) && isDebugSection(Section) &&
+         Section.Name != ".gdb_index";
+}
+
+static void compressSections(const CopyConfig &Config, Object &Obj,
+                             SectionPred &RemovePred) {
+  SmallVector<SectionBase *, 13> ToCompress;
+  SmallVector<RelocationSection *, 13> RelocationSections;
+  for (auto &Sec : Obj.sections()) {
+    if (RelocationSection *R = dyn_cast<RelocationSection>(&Sec)) {
+      if (isCompressable(*R->getSection()))
+        RelocationSections.push_back(R);
+      continue;
+    }
+
+    if (isCompressable(Sec))
+      ToCompress.push_back(&Sec);
+  }
+
+  for (SectionBase *S : ToCompress) {
+    CompressedSection &CS =
+        Obj.addSection<CompressedSection>(*S, Config.CompressionType);
+
+    for (RelocationSection *RS : RelocationSections) {
+      if (RS->getSection() == S)
+        RS->setSection(&CS);
+    }
+  }
+
+  RemovePred = [RemovePred](const SectionBase &Sec) {
+    return isCompressable(Sec) || RemovePred(Sec);
+  };
+}
+
 // This function handles the high level operations of GNU objcopy including
 // handling command line options. It's important to outline certain properties
 // we expect to hold of the command line operations. Any operation that "keeps"
@@ -572,7 +619,7 @@ static void handleArgs(const CopyConfig &Config, Object &Obj,
         return true;
       if (&Sec == Obj.SectionNames)
         return false;
-      if (Sec.Name.startswith(".gnu.warning"))
+      if (StringRef(Sec.Name).startswith(".gnu.warning"))
         return false;
       return (Sec.Flags & SHF_ALLOC) == 0;
     };
@@ -624,6 +671,9 @@ static void handleArgs(const CopyConfig &Config, Object &Obj,
     };
   }
 
+  if (Config.CompressionType != DebugCompressionType::None)
+    compressSections(Config, Obj, RemovePred);
+
   Obj.removeSections(RemovePred);
 
   if (!Config.SectionsToRename.empty()) {
@@ -868,6 +918,25 @@ static DriverConfig parseObjcopyOptions(ArrayRef<const char *> ArgsArr) {
     Config.BinaryArch = getMachineInfo(BinaryArch);
   }
 
+  if (auto Arg = InputArgs.getLastArg(OBJCOPY_compress_debug_sections,
+                                      OBJCOPY_compress_debug_sections_eq)) {
+    Config.CompressionType = DebugCompressionType::Z;
+
+    if (Arg->getOption().getID() == OBJCOPY_compress_debug_sections_eq) {
+      Config.CompressionType =
+          StringSwitch<DebugCompressionType>(
+              InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections_eq))
+              .Case("zlib-gnu", DebugCompressionType::GNU)
+              .Case("zlib", DebugCompressionType::Z)
+              .Default(DebugCompressionType::None);
+      if (Config.CompressionType == DebugCompressionType::None)
+        error("Invalid or unsupported --compress-debug-sections format: " +
+              InputArgs.getLastArgValue(OBJCOPY_compress_debug_sections_eq));
+      if (!zlib::isAvailable())
+        error("LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress.");
+    }
+  }
+
   Config.SplitDWO = InputArgs.getLastArgValue(OBJCOPY_split_dwo);
   Config.AddGnuDebugLink = InputArgs.getLastArgValue(OBJCOPY_add_gnu_debuglink);
   Config.SymbolsPrefix = InputArgs.getLastArgValue(OBJCOPY_prefix_symbols);