1
0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2024-11-26 12:43:36 +01:00

Reapply [llvm-lipo] Implement -create (with hardcoded alignments)

This reapplies r366142 with a fix for the failing Windows test.

Original commit message:

Creates universal binary output file from input files. Currently uses
hard coded value for alignment.  Want to get the create functionality
approved before implementing the alignment function.

Patch by Anusha Basana <anusha.basana@gmail.com>

Differential Revision: https://reviews.llvm.org/D64102

llvm-svn: 366512
This commit is contained in:
Shoaib Meenai 2019-07-18 22:48:38 +00:00
parent 3ea2024537
commit 1528fde232
8 changed files with 528 additions and 18 deletions

View File

@ -0,0 +1,101 @@
--- !mach-o
FileHeader:
magic: 0xFEEDFACF
cputype: 0x0100000C
cpusubtype: 0x00000000
filetype: 0x00000001
ncmds: 4
sizeofcmds: 352
flags: 0x00002000
reserved: 0x00000000
LoadCommands:
- cmd: LC_SEGMENT_64
cmdsize: 232
segname: ''
vmaddr: 0
vmsize: 56
fileoff: 384
filesize: 56
maxprot: 7
initprot: 7
nsects: 2
flags: 0
Sections:
- sectname: __text
segname: __TEXT
addr: 0x0000000000000000
size: 20
offset: 0x00000180
align: 2
reloff: 0x00000000
nreloc: 0
flags: 0x80000400
reserved1: 0x00000000
reserved2: 0x00000000
reserved3: 0x00000000
- sectname: __compact_unwind
segname: __LD
addr: 0x0000000000000018
size: 32
offset: 0x00000198
align: 3
reloff: 0x000001B8
nreloc: 1
flags: 0x02000000
reserved1: 0x00000000
reserved2: 0x00000000
reserved3: 0x00000000
- cmd: LC_VERSION_MIN_IPHONEOS
cmdsize: 16
version: 327680
sdk: 0
- cmd: LC_SYMTAB
cmdsize: 24
symoff: 448
nsyms: 3
stroff: 496
strsize: 20
- cmd: LC_DYSYMTAB
cmdsize: 80
ilocalsym: 0
nlocalsym: 2
iextdefsym: 2
nextdefsym: 1
iundefsym: 3
nundefsym: 0
tocoff: 0
ntoc: 0
modtaboff: 0
nmodtab: 0
extrefsymoff: 0
nextrefsyms: 0
indirectsymoff: 0
nindirectsyms: 0
extreloff: 0
nextrel: 0
locreloff: 0
nlocrel: 0
LinkEditData:
NameList:
- n_strx: 13
n_type: 0x0E
n_sect: 1
n_desc: 0
n_value: 0
- n_strx: 7
n_type: 0x0E
n_sect: 2
n_desc: 0
n_value: 24
- n_strx: 1
n_type: 0x0F
n_sect: 1
n_desc: 0
n_value: 0
StringTable:
- ''
- _main
- ltmp1
- ltmp0
- ''
...

View File

@ -0,0 +1,76 @@
--- !mach-o
FileHeader:
magic: 0xFEEDFACE
cputype: 0x0000000C
cpusubtype: 0x00000009
filetype: 0x00000001
ncmds: 4
sizeofcmds: 244
flags: 0x00002000
LoadCommands:
- cmd: LC_SEGMENT
cmdsize: 124
segname: ''
vmaddr: 0
vmsize: 10
fileoff: 272
filesize: 10
maxprot: 7
initprot: 7
nsects: 1
flags: 0
Sections:
- sectname: __text
segname: __TEXT
addr: 0x0000000000000000
size: 10
offset: 0x00000110
align: 1
reloff: 0x00000000
nreloc: 0
flags: 0x80000400
reserved1: 0x00000000
reserved2: 0x00000000
reserved3: 0x00000000
- cmd: LC_VERSION_MIN_IPHONEOS
cmdsize: 16
version: 327680
sdk: 0
- cmd: LC_SYMTAB
cmdsize: 24
symoff: 284
nsyms: 1
stroff: 296
strsize: 8
- cmd: LC_DYSYMTAB
cmdsize: 80
ilocalsym: 0
nlocalsym: 0
iextdefsym: 0
nextdefsym: 1
iundefsym: 1
nundefsym: 0
tocoff: 0
ntoc: 0
modtaboff: 0
nmodtab: 0
extrefsymoff: 0
nextrefsyms: 0
indirectsymoff: 0
nindirectsyms: 0
extreloff: 0
nextrel: 0
locreloff: 0
nlocrel: 0
LinkEditData:
NameList:
- n_strx: 1
n_type: 0x0F
n_sect: 1
n_desc: 8
n_value: 0
StringTable:
- ''
- _main
- ''
...

View File

@ -0,0 +1,89 @@
--- !mach-o
FileHeader:
magic: 0xFEEDFACF
cputype: 0x01000007
cpusubtype: 0x00000003
filetype: 0x00000001
ncmds: 4
sizeofcmds: 352
flags: 0x00002000
reserved: 0x00000000
LoadCommands:
- cmd: LC_SEGMENT_64
cmdsize: 232
segname: ''
vmaddr: 0
vmsize: 80
fileoff: 384
filesize: 80
maxprot: 7
initprot: 7
nsects: 2
flags: 0
Sections:
- sectname: __text
segname: __TEXT
addr: 0x0000000000000000
size: 15
offset: 0x00000180
align: 4
reloff: 0x00000000
nreloc: 0
flags: 0x80000400
reserved1: 0x00000000
reserved2: 0x00000000
reserved3: 0x00000000
- sectname: __eh_frame
segname: __TEXT
addr: 0x0000000000000010
size: 64
offset: 0x00000190
align: 3
reloff: 0x00000000
nreloc: 0
flags: 0x6800000B
reserved1: 0x00000000
reserved2: 0x00000000
reserved3: 0x00000000
- cmd: LC_VERSION_MIN_MACOSX
cmdsize: 16
version: 656384
sdk: 0
- cmd: LC_SYMTAB
cmdsize: 24
symoff: 464
nsyms: 1
stroff: 480
strsize: 8
- cmd: LC_DYSYMTAB
cmdsize: 80
ilocalsym: 0
nlocalsym: 0
iextdefsym: 0
nextdefsym: 1
iundefsym: 1
nundefsym: 0
tocoff: 0
ntoc: 0
modtaboff: 0
nmodtab: 0
extrefsymoff: 0
nextrefsyms: 0
indirectsymoff: 0
nindirectsyms: 0
extreloff: 0
nextrel: 0
locreloff: 0
nlocrel: 0
LinkEditData:
NameList:
- n_strx: 1
n_type: 0x0F
n_sect: 1
n_desc: 0
n_value: 0
StringTable:
- ''
- _main
- ''
...

View File

@ -0,0 +1,13 @@
# Executable testing is not supported on Windows, since all files are considered executable
# UNSUPPORTED: system-windows
# RUN: yaml2obj %p/Inputs/i386-slice.yaml > %t-i386.o
# RUN: yaml2obj %p/Inputs/x86_64-slice.yaml > %t-x86_64.o
# RUN: chmod -x %t-i386.o
# RUN: chmod -x %t-x86_64.o
# RUN: llvm-lipo %t-i386.o %t-x86_64.o -create -output %t-universal.o
# RUN: test ! -x %t-universal.o
# RUN: chmod +x %t-i386.o
# RUN: llvm-lipo %t-i386.o %t-x86_64.o -create -output %t-universal.o
# RUN: test -x %t-universal.o

View File

@ -0,0 +1,8 @@
# RUN: yaml2obj %p/Inputs/i386-slice.yaml > %t-32.o
# RUN: yaml2obj %p/Inputs/i386-x86_64-universal.yaml > %t-universal.o
# RUN: not llvm-lipo %t-32.o -create 2>&1 | FileCheck --check-prefix=NO_OUTPUT %s
# NO_OUTPUT: error: create expects a single output file to be specified
# RUN: not llvm-lipo %t-universal.o %t-32.o -create -output %t.o 2>&1 | FileCheck --check-prefix=DUPLICATE_ARCHS %s
# DUPLICATE_ARCHS: have the same architecture i386 and therefore cannot be in the same universal binary

View File

@ -0,0 +1,32 @@
# RUN: yaml2obj %p/Inputs/i386-slice.yaml > %t-i386.o
# RUN: yaml2obj %p/Inputs/x86_64-slice.yaml > %t-x86_64.o
# RUN: llvm-lipo %t-i386.o %t-x86_64.o -create -output %t-universal-llvm.o
# RUN: yaml2obj %p/Inputs/i386-x86_64-universal.yaml > %t-universal.o
# RUN: cmp %t-universal-llvm.o %t-universal.o
# RUN: yaml2obj %p/Inputs/armv7-slice.yaml > %t-armv7.o
# RUN: yaml2obj %p/Inputs/arm64-slice.yaml > %t-arm64.o
# RUN: llvm-lipo %t-arm64.o %t-armv7.o %t-universal.o -create -output %t-universal-2.o
# RUN: llvm-lipo %t-universal-2.o -thin x86_64 -output %t-x86_64_extracted.o
# RUN: cmp %t-x86_64_extracted.o %t-x86_64.o
# RUN: llvm-lipo %t-universal-2.o -thin armv7 -output %t-armv7-extracted.o
# RUN: cmp %t-armv7-extracted.o %t-armv7.o
# RUN: llvm-objdump %t-universal-2.o -m --universal-headers | FileCheck %s
# CHECK: fat_magic FAT_MAGIC
# CHECK: nfat_arch 4
# CHECK: architecture i386
# CHECK: offset 4096
# CHECK: align 2^12 (4096)
# CHECK: architecture x86_64
# CHECK: offset 8192
# CHECK: align 2^12 (4096)
# CHECK: architecture armv7
# CHECK: offset 16384
# CHECK: align 2^14 (16384)
# CHECK: architecture arm64
# CHECK: offset 32768
# CHECK: align 2^14 (16384)

View File

@ -23,6 +23,11 @@ def thin : Option<["-", "--"], "thin", KIND_SEPARATE>,
HelpText<"Create a thin output file of specified arch_type from the " HelpText<"Create a thin output file of specified arch_type from the "
"fat input file. Requires -output option">; "fat input file. Requires -output option">;
def create : Option<["-", "--"], "create", KIND_FLAG>,
Group<action_group>,
HelpText<"Create a universal binary output file from the input "
"files. Requires -output option">;
def output : Option<["-", "--"], "output", KIND_SEPARATE>, def output : Option<["-", "--"], "output", KIND_SEPARATE>,
HelpText<"Create output file with specified name">; HelpText<"Create output file with specified name">;
def o : JoinedOrSeparate<["-"], "o">, Alias<output>; def o : JoinedOrSeparate<["-"], "o">, Alias<output>;

View File

@ -80,6 +80,7 @@ enum class LipoAction {
PrintArchs, PrintArchs,
VerifyArch, VerifyArch,
ThinArch, ThinArch,
CreateUniversal,
}; };
struct Config { struct Config {
@ -90,6 +91,14 @@ struct Config {
LipoAction ActionToPerform; LipoAction ActionToPerform;
}; };
struct Slice {
const MachOObjectFile *ObjectFile;
// Requires Alignment field to store slice alignment values from universal
// binaries. Also needed to order the slices using compareSlices, so the total
// file size can be calculated before creating the output buffer.
uint32_t Alignment;
};
} // end namespace } // end namespace
static void validateArchitectureName(StringRef ArchitectureName) { static void validateArchitectureName(StringRef ArchitectureName) {
@ -108,7 +117,7 @@ static Config parseLipoOptions(ArrayRef<const char *> ArgsArr) {
Config C; Config C;
LipoOptTable T; LipoOptTable T;
unsigned MissingArgumentIndex, MissingArgumentCount; unsigned MissingArgumentIndex, MissingArgumentCount;
llvm::opt::InputArgList InputArgs = opt::InputArgList InputArgs =
T.ParseArgs(ArgsArr, MissingArgumentIndex, MissingArgumentCount); T.ParseArgs(ArgsArr, MissingArgumentIndex, MissingArgumentCount);
if (MissingArgumentCount) if (MissingArgumentCount)
@ -186,6 +195,12 @@ static Config parseLipoOptions(ArrayRef<const char *> ArgsArr) {
C.ActionToPerform = LipoAction::ThinArch; C.ActionToPerform = LipoAction::ThinArch;
return C; return C;
case LIPO_create:
if (C.OutputFile.empty())
reportError("create expects a single output file to be specified");
C.ActionToPerform = LipoAction::CreateUniversal;
return C;
default: default:
reportError("llvm-lipo action unspecified"); reportError("llvm-lipo action unspecified");
} }
@ -195,8 +210,7 @@ static SmallVector<OwningBinary<Binary>, 1>
readInputBinaries(ArrayRef<std::string> InputFiles) { readInputBinaries(ArrayRef<std::string> InputFiles) {
SmallVector<OwningBinary<Binary>, 1> InputBinaries; SmallVector<OwningBinary<Binary>, 1> InputBinaries;
for (StringRef InputFile : InputFiles) { for (StringRef InputFile : InputFiles) {
Expected<OwningBinary<llvm::object::Binary>> BinaryOrErr = Expected<OwningBinary<Binary>> BinaryOrErr = createBinary(InputFile);
createBinary(InputFile);
if (!BinaryOrErr) if (!BinaryOrErr)
reportError(InputFile, BinaryOrErr.takeError()); reportError(InputFile, BinaryOrErr.takeError());
// TODO: Add compatibility for archive files // TODO: Add compatibility for archive files
@ -241,33 +255,35 @@ static void verifyArch(ArrayRef<OwningBinary<Binary>> InputBinaries,
exit(EXIT_SUCCESS); exit(EXIT_SUCCESS);
} }
static void printArchOrUnknown(const MachOObjectFile *ObjectFile) { // Returns a string of the given Object file's architecture type
// Prints trailing space and unknown in this format for compatibility with // Unknown architectures formatted unknown(CPUType,CPUSubType) for compatibility
// cctools lipo. // with cctools lipo
const std::string ObjectArch = ObjectFile->getArchTriple().getArchName(); static std::string getArchString(const MachOObjectFile &ObjectFile) {
if (ObjectArch.empty()) const Triple T = ObjectFile.getArchTriple();
outs() << "unknown(" << ObjectFile->getHeader().cputype << "," const StringRef ObjectArch = T.getArchName();
<< ObjectFile->getHeader().cpusubtype << ") "; if (!ObjectArch.empty())
else return ObjectArch;
outs() << ObjectArch + " "; return ("unknown(" + Twine(ObjectFile.getHeader().cputype) + "," +
Twine(ObjectFile.getHeader().cpusubtype & ~MachO::CPU_SUBTYPE_MASK) +
")")
.str();
} }
LLVM_ATTRIBUTE_NORETURN LLVM_ATTRIBUTE_NORETURN
static void printArchs(ArrayRef<OwningBinary<Binary>> InputBinaries) { static void printArchs(ArrayRef<OwningBinary<Binary>> InputBinaries) {
// Prints trailing space for compatibility with cctools lipo.
assert(InputBinaries.size() == 1 && "Incorrect number of input binaries"); assert(InputBinaries.size() == 1 && "Incorrect number of input binaries");
const Binary *InputBinary = InputBinaries.front().getBinary(); const Binary *InputBinary = InputBinaries.front().getBinary();
if (auto UO = dyn_cast<MachOUniversalBinary>(InputBinary)) { if (auto UO = dyn_cast<MachOUniversalBinary>(InputBinary)) {
for (MachOUniversalBinary::object_iterator I = UO->begin_objects(), for (const auto &O : UO->objects()) {
E = UO->end_objects();
I != E; ++I) {
Expected<std::unique_ptr<MachOObjectFile>> BinaryOrError = Expected<std::unique_ptr<MachOObjectFile>> BinaryOrError =
I->getAsObjectFile(); O.getAsObjectFile();
if (!BinaryOrError) if (!BinaryOrError)
reportError(InputBinary->getFileName(), BinaryOrError.takeError()); reportError(InputBinary->getFileName(), BinaryOrError.takeError());
printArchOrUnknown(BinaryOrError.get().get()); outs() << getArchString(*BinaryOrError.get().get()) << " ";
} }
} else if (auto O = dyn_cast<MachOObjectFile>(InputBinary)) { } else if (auto O = dyn_cast<MachOObjectFile>(InputBinary)) {
printArchOrUnknown(O); outs() << getArchString(*O) << " ";
} else { } else {
llvm_unreachable("Unexpected binary format"); llvm_unreachable("Unexpected binary format");
} }
@ -314,6 +330,173 @@ static void extractSlice(ArrayRef<OwningBinary<Binary>> InputBinaries,
exit(EXIT_SUCCESS); exit(EXIT_SUCCESS);
} }
static void checkArchDuplicates(const ArrayRef<Slice> &Slices) {
DenseMap<uint64_t, const MachOObjectFile *> CPUIds;
auto CPUIDForSlice = [](const Slice &S) {
return static_cast<uint64_t>(S.ObjectFile->getHeader().cputype) << 32 |
S.ObjectFile->getHeader().cpusubtype;
};
for (const auto &S : Slices) {
auto Entry = CPUIds.try_emplace(CPUIDForSlice(S), S.ObjectFile);
if (!Entry.second)
reportError(Entry.first->second->getFileName() + " and " +
S.ObjectFile->getFileName() + " have the same architecture " +
getArchString(*S.ObjectFile) +
" and therefore cannot be in the same universal binary");
}
}
static uint32_t calculateAlignment(const MachOObjectFile *ObjectFile) {
// TODO: Implement getAlign() and remove hard coding
// Will be implemented in a follow-up.
switch (ObjectFile->getHeader().cputype) {
case MachO::CPU_TYPE_I386:
case MachO::CPU_TYPE_X86_64:
case MachO::CPU_TYPE_POWERPC:
case MachO::CPU_TYPE_POWERPC64:
return 12; // log2 value of page size(4k) for x86 and PPC
case MachO::CPU_TYPE_ARM:
case MachO::CPU_TYPE_ARM64:
case MachO::CPU_TYPE_ARM64_32:
return 14; // log2 value of page size(16k) for Darwin ARM
default:
return 12;
}
}
// This function replicates ordering from cctools lipo for consistency
static bool compareSlices(const Slice &Lhs, const Slice &Rhs) {
if (Lhs.ObjectFile->getHeader().cputype ==
Rhs.ObjectFile->getHeader().cputype)
return Lhs.ObjectFile->getHeader().cpusubtype <
Rhs.ObjectFile->getHeader().cpusubtype;
// force arm64-family to follow after all other slices for compatibility
// with cctools lipo
if (Lhs.ObjectFile->getHeader().cputype == MachO::CPU_TYPE_ARM64)
return false;
if (Rhs.ObjectFile->getHeader().cputype == MachO::CPU_TYPE_ARM64)
return true;
// Sort by alignment to minimize file size
return Lhs.Alignment < Rhs.Alignment;
}
// Updates vector ExtractedObjects with the MachOObjectFiles extracted from
// Universal Binary files to transfer ownership.
static SmallVector<Slice, 2> buildSlices(
ArrayRef<OwningBinary<Binary>> InputBinaries,
SmallVectorImpl<std::unique_ptr<MachOObjectFile>> &ExtractedObjects) {
SmallVector<Slice, 2> Slices;
for (auto &IB : InputBinaries) {
const Binary *InputBinary = IB.getBinary();
if (auto UO = dyn_cast<MachOUniversalBinary>(InputBinary)) {
for (const auto &O : UO->objects()) {
Expected<std::unique_ptr<MachOObjectFile>> BinaryOrError =
O.getAsObjectFile();
if (!BinaryOrError)
reportError(InputBinary->getFileName(), BinaryOrError.takeError());
ExtractedObjects.push_back(std::move(BinaryOrError.get()));
Slices.push_back(Slice{ExtractedObjects.back().get(), O.getAlign()});
}
} else if (auto O = dyn_cast<MachOObjectFile>(InputBinary)) {
Slices.push_back(Slice{O, calculateAlignment(O)});
} else {
llvm_unreachable("Unexpected binary format");
}
}
return Slices;
}
static SmallVector<MachO::fat_arch, 2>
buildFatArchList(ArrayRef<Slice> Slices) {
SmallVector<MachO::fat_arch, 2> FatArchList;
uint64_t Offset =
sizeof(MachO::fat_header) + Slices.size() * sizeof(MachO::fat_arch);
for (size_t Index = 0, Size = Slices.size(); Index < Size; ++Index) {
Offset = alignTo(Offset, 1 << Slices[Index].Alignment);
const MachOObjectFile *ObjectFile = Slices[Index].ObjectFile;
if (Offset > UINT32_MAX)
reportError("fat file too large to be created because the offset "
"field in struct fat_arch is only 32-bits and the offset " +
Twine(Offset) + " for " + ObjectFile->getFileName() +
" for architecture " + getArchString(*ObjectFile) +
"exceeds that.");
MachO::fat_arch FatArch;
FatArch.cputype = ObjectFile->getHeader().cputype;
FatArch.cpusubtype = ObjectFile->getHeader().cpusubtype;
FatArch.offset = Offset;
FatArch.size = ObjectFile->getMemoryBufferRef().getBufferSize();
FatArch.align = Slices[Index].Alignment;
Offset += FatArch.size;
FatArchList.push_back(FatArch);
}
return FatArchList;
}
static void createUniversalBinary(SmallVectorImpl<Slice> &Slices,
StringRef OutputFileName) {
MachO::fat_header FatHeader;
FatHeader.magic = MachO::FAT_MAGIC;
FatHeader.nfat_arch = Slices.size();
stable_sort(Slices, compareSlices);
SmallVector<MachO::fat_arch, 2> FatArchList = buildFatArchList(Slices);
const bool IsExecutable = any_of(Slices, [](Slice S) {
return sys::fs::can_execute(S.ObjectFile->getFileName());
});
const uint64_t OutputFileSize =
FatArchList.back().offset + FatArchList.back().size;
Expected<std::unique_ptr<FileOutputBuffer>> OutFileOrError =
FileOutputBuffer::create(OutputFileName, OutputFileSize,
IsExecutable ? FileOutputBuffer::F_executable
: 0);
if (!OutFileOrError)
reportError(OutputFileName, OutFileOrError.takeError());
std::unique_ptr<FileOutputBuffer> OutFile = std::move(OutFileOrError.get());
std::memset(OutFile->getBufferStart(), 0, OutputFileSize);
if (sys::IsLittleEndianHost)
MachO::swapStruct(FatHeader);
std::memcpy(OutFile->getBufferStart(), &FatHeader, sizeof(MachO::fat_header));
for (size_t Index = 0, Size = Slices.size(); Index < Size; ++Index) {
MemoryBufferRef BufferRef = Slices[Index].ObjectFile->getMemoryBufferRef();
std::copy(BufferRef.getBufferStart(), BufferRef.getBufferEnd(),
OutFile->getBufferStart() + FatArchList[Index].offset);
}
// FatArchs written after Slices in order reduce the number of swaps for the
// LittleEndian case
if (sys::IsLittleEndianHost)
for (MachO::fat_arch &FA : FatArchList)
MachO::swapStruct(FA);
std::memcpy(OutFile->getBufferStart() + sizeof(MachO::fat_header),
FatArchList.begin(),
sizeof(MachO::fat_arch) * FatArchList.size());
if (Error E = OutFile->commit())
reportError(OutputFileName, std::move(E));
}
LLVM_ATTRIBUTE_NORETURN
static void createUniversalBinary(ArrayRef<OwningBinary<Binary>> InputBinaries,
StringRef OutputFileName) {
assert(InputBinaries.size() >= 1 && "Incorrect number of input binaries");
assert(!OutputFileName.empty() && "Create expects a single output file");
SmallVector<std::unique_ptr<MachOObjectFile>, 1> ExtractedObjects;
SmallVector<Slice, 1> Slices = buildSlices(InputBinaries, ExtractedObjects);
checkArchDuplicates(Slices);
createUniversalBinary(Slices, OutputFileName);
exit(EXIT_SUCCESS);
}
int main(int argc, char **argv) { int main(int argc, char **argv) {
InitLLVM X(argc, argv); InitLLVM X(argc, argv);
Config C = parseLipoOptions(makeArrayRef(argv + 1, argc)); Config C = parseLipoOptions(makeArrayRef(argv + 1, argc));
@ -330,6 +513,9 @@ int main(int argc, char **argv) {
case LipoAction::ThinArch: case LipoAction::ThinArch:
extractSlice(InputBinaries, C.ThinArchType, C.OutputFile); extractSlice(InputBinaries, C.ThinArchType, C.OutputFile);
break; break;
case LipoAction::CreateUniversal:
createUniversalBinary(InputBinaries, C.OutputFile);
break;
} }
return EXIT_SUCCESS; return EXIT_SUCCESS;
} }