RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2025-01-31 12:41:49 +01:00
Code Issues Projects Releases Wiki Activity

[clang][AArch64] Correctly align HFA arguments when passed on the stack

Browse Source 
When we pass a AArch64 Homogeneous Floating-Point
Aggregate (HFA) argument with increased alignment
requirements, for example

    struct S {
      __attribute__ ((__aligned__(16))) double v[4];
    };

Clang uses `[4 x double]` for the parameter, which is passed
on the stack at alignment 8, whereas it should be at
alignment 16, following Rule C.4 in
AAPCS (https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst#642parameter-passing-rules)

Currently we don't have a way to express in LLVM IR the
alignment requirements of the function arguments. The align
attribute is applicable to pointers only, and only for some
special ways of passing arguments (e..g byval). When
implementing AAPCS32/AAPCS64, clang resorts to dubious hacks
of coercing to types, which naturally have the needed
alignment. We don't have enough types to cover all the
cases, though.

This patch introduces a new use of the stackalign attribute
to control stack slot alignment, when and if an argument is
passed in memory.

The attribute align is left as an optimizer hint - it still
applies to pointer types only and pertains to the content of
the pointer, whereas the alignment of the pointer itself is
determined by the stackalign attribute.

For byval arguments, the stackalign attribute assumes the
role, previously perfomed by align, falling back to align if
stackalign` is absent.

On the clang side, when passing arguments using the "direct"
style (cf. `ABIArgInfo::Kind`), now we can optionally
specify an alignment, which is emitted as the new
`stackalign` attribute.

Patch by Momchil Velikov and Lucas Prates.

Differential Revision: https://reviews.llvm.org/D98794
This commit is contained in:
Momchil Velikov 2021-04-15 19:58:54 +01:00
parent 6b675e5ab5
commit d98e321d12
17 changed files with 148 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
docs/LangRef.rst
View File

@ -1300,6 +1300,15 @@ Currently, only the following parameter attributes are defined:
undefined. Note that this does not refer to padding introduced by the
type's storage representation.
``alignstack(<n>)``
This indicates the alignment that should be considered by the backend when
assigning this parameter to a stack slot during calling convention
lowering. The enforcement of the specified alignment is target-dependent,
as target-specific calling convention rules may override this value. This
attribute serves the purpose of carrying language specific alignment
information that is not mapped to base types in the backend (for example,
over-alignment specification through language attributes).
.. _gc:
Garbage Collector Strategy Names

35
include/llvm/CodeGen/TargetCallingConv.h
View File

@ -44,7 +44,8 @@ namespace ISD {
unsigned IsHva : 1; ///< HVA field for
unsigned IsHvaStart : 1; ///< HVA structure start
unsigned IsSecArgPass : 1; ///< Second argument
unsigned ByValOrByRefAlign : 4; ///< Log 2 of byval/byref alignment
unsigned MemAlign : 4; ///< Log 2 of alignment when arg is passed in memory
///< (including byval/byref)
unsigned OrigAlign : 5; ///< Log 2 of original alignment
unsigned IsInConsecutiveRegsLast : 1;
unsigned IsInConsecutiveRegs : 1;
@ -55,18 +56,12 @@ namespace ISD {
unsigned PointerAddrSpace; ///< Address space of pointer argument
/// Set the alignment used by byref or byval parameters.
void setAlignImpl(Align A) {
ByValOrByRefAlign = encode(A);
assert(getNonZeroByValAlign() == A && "bitfield overflow");
}
public:
ArgFlagsTy()
: IsZExt(0), IsSExt(0), IsInReg(0), IsSRet(0), IsByVal(0), IsByRef(0),
IsNest(0), IsReturned(0), IsSplit(0), IsInAlloca(0), IsPreallocated(0),
IsSplitEnd(0), IsSwiftSelf(0), IsSwiftError(0), IsCFGuardTarget(0),
IsHva(0), IsHvaStart(0), IsSecArgPass(0), ByValOrByRefAlign(0),
IsHva(0), IsHvaStart(0), IsSecArgPass(0), MemAlign(0),
OrigAlign(0), IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0),
IsCopyElisionCandidate(0), IsPointer(0), ByValOrByRefSize(0),
PointerAddrSpace(0) {
@ -141,24 +136,26 @@ namespace ISD {
bool isPointer() const { return IsPointer; }
void setPointer() { IsPointer = 1; }
Align getNonZeroByValAlign() const {
MaybeAlign A = decodeMaybeAlign(ByValOrByRefAlign);
assert(A && "ByValAlign must be defined");
return *A;
}
void setByValAlign(Align A) {
assert(isByVal() && !isByRef());
setAlignImpl(A);
Align getNonZeroMemAlign() const {
return decodeMaybeAlign(MemAlign).valueOrOne();
}
void setByRefAlign(Align A) {
assert(!isByVal() && isByRef());
setAlignImpl(A);
void setMemAlign(Align A) {
MemAlign = encode(A);
assert(getNonZeroMemAlign() == A && "bitfield overflow");
}
Align getNonZeroByValAlign() const {
assert(isByVal());
MaybeAlign A = decodeMaybeAlign(MemAlign);
assert(A && "ByValAlign must be defined");
return *A;
}
Align getNonZeroOrigAlign() const {
return decodeMaybeAlign(OrigAlign).valueOrOne();
}
void setOrigAlign(Align A) {
OrigAlign = encode(A);
assert(getNonZeroOrigAlign() == A && "bitfield overflow");

2
include/llvm/IR/Argument.h
View File

@ -102,6 +102,8 @@ public:
/// If this is a byval or inalloca argument, return its alignment.
MaybeAlign getParamAlign() const;
MaybeAlign getParamStackAlign() const;
/// If this is a byval argument, return its type.
Type *getParamByValType() const;

3
include/llvm/IR/Attributes.h
View File

@ -674,6 +674,9 @@ public:
/// Return the alignment for the specified function parameter.
MaybeAlign getParamAlignment(unsigned ArgNo) const;
/// Return the stack alignment for the specified function parameter.
MaybeAlign getParamStackAlignment(unsigned ArgNo) const;
/// Return the byval type for the specified function parameter.
Type *getParamByValType(unsigned ArgNo) const;

4
include/llvm/IR/Function.h
View File

@ -483,6 +483,10 @@ public:
return AttributeSets.getParamAlignment(ArgNo);
}
MaybeAlign getParamStackAlign(unsigned ArgNo) const {
return AttributeSets.getParamStackAlignment(ArgNo);
}
/// Extract the byval type for a parameter.
Type *getParamByValType(unsigned ArgNo) const {
return AttributeSets.getParamByValType(ArgNo);

4
include/llvm/IR/InstrTypes.h
View File

@ -1731,6 +1731,10 @@ public:
return Attrs.getParamAlignment(ArgNo);
}
MaybeAlign getParamStackAlign(unsigned ArgNo) const {
return Attrs.getParamStackAlignment(ArgNo);
}
/// Extract the byval type for a call or parameter.
Type *getParamByValType(unsigned ArgNo) const {
Type *Ty = Attrs.getParamByValType(ArgNo);

8
lib/AsmParser/LLParser.cpp
View File

@ -1715,6 +1715,13 @@ bool LLParser::parseOptionalParamAttrs(AttrBuilder &B) {
B.addAlignmentAttr(Alignment);
continue;
}
case lltok::kw_alignstack: {
unsigned Alignment;
if (parseOptionalStackAlignment(Alignment))
return true;
B.addStackAlignmentAttr(Alignment);
continue;
}
case lltok::kw_byval: {
Type *Ty;
if (parseRequiredTypeAttr(Ty, lltok::kw_byval))
@ -1783,7 +1790,6 @@ bool LLParser::parseOptionalParamAttrs(AttrBuilder &B) {
case lltok::kw_zeroext: B.addAttribute(Attribute::ZExt); break;
case lltok::kw_immarg: B.addAttribute(Attribute::ImmArg); break;
case lltok::kw_alignstack:
case lltok::kw_alwaysinline:
case lltok::kw_argmemonly:
case lltok::kw_builtin:

16
lib/CodeGen/GlobalISel/CallLowering.cpp
View File

@ -154,6 +154,7 @@ void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
const AttributeList &Attrs = FuncInfo.getAttributes();
addArgFlagsFromAttributes(Flags, Attrs, OpIdx);
Align MemAlign;
if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated()) {
Type *ElementTy = cast<PointerType>(Arg.Ty)->getElementType();
@ -162,13 +163,18 @@ void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
// For ByVal, alignment should be passed from FE. BE will guess if
// this info is not there but there are cases it cannot get right.
Align FrameAlign;
if (auto ParamAlign = FuncInfo.getParamAlign(OpIdx - 1))
FrameAlign = *ParamAlign;
if (auto ParamAlign = FuncInfo.getParamStackAlign(OpIdx - 1))
MemAlign = *ParamAlign;
else if ((ParamAlign = FuncInfo.getParamAlign(OpIdx - 1)))
MemAlign = *ParamAlign;
else
FrameAlign = Align(getTLI()->getByValTypeAlignment(ElementTy, DL));
Flags.setByValAlign(FrameAlign);
MemAlign = Align(getTLI()->getByValTypeAlignment(ElementTy, DL));
} else if (auto ParamAlign = FuncInfo.getParamStackAlign(OpIdx - 1)) {
MemAlign = *ParamAlign;
} else {
MemAlign = Align(DL.getABITypeAlign(Arg.Ty));
}
Flags.setMemAlign(MemAlign);
Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty));
// Don't try to use the returned attribute if the argument is marked as

11
lib/CodeGen/SelectionDAG/FastISel.cpp
View File

@ -1072,6 +1072,7 @@ bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
// preallocated handling in the various CC lowering callbacks.
Flags.setByVal();
}
MaybeAlign MemAlign = Arg.Alignment;
if (Arg.IsByVal || Arg.IsInAlloca || Arg.IsPreallocated) {
PointerType *Ty = cast<PointerType>(Arg.Ty);
Type *ElementTy = Ty->getElementType();
@ -1080,18 +1081,18 @@ bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
// For ByVal, alignment should come from FE. BE will guess if this info
// is not there, but there are cases it cannot get right.
MaybeAlign FrameAlign = Arg.Alignment;
if (!FrameAlign)
FrameAlign = Align(TLI.getByValTypeAlignment(ElementTy, DL));
if (!MemAlign)
MemAlign = Align(TLI.getByValTypeAlignment(ElementTy, DL));
Flags.setByValSize(FrameSize);
Flags.setByValAlign(*FrameAlign);
} else if (!MemAlign) {
MemAlign = DL.getABITypeAlign(Arg.Ty);
}
Flags.setMemAlign(*MemAlign);
if (Arg.IsNest)
Flags.setNest();
if (NeedsRegBlock)
Flags.setInConsecutiveRegs();
Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty));
CLI.OutVals.push_back(Arg.Val);
CLI.OutFlags.push_back(Flags);
}

48
lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
View File

@ -9425,6 +9425,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
// for a type depending on the context. Give the target a chance to
// specify the alignment it wants.
const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
Flags.setOrigAlign(OriginalAlignment);
if (Args[i].Ty->isPointerTy()) {
Flags.setPointer();
@ -9478,6 +9479,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
// in the various CC lowering callbacks.
Flags.setByVal();
}
Align MemAlign;
if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
PointerType *Ty = cast<PointerType>(Args[i].Ty);
Type *ElementTy = Ty->getElementType();
@ -9487,18 +9489,20 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
Flags.setByValSize(FrameSize);
// info is not there but there are cases it cannot get right.
Align FrameAlign;
if (auto MA = Args[i].Alignment)
FrameAlign = *MA;
MemAlign = *MA;
else
FrameAlign = Align(getByValTypeAlignment(ElementTy, DL));
Flags.setByValAlign(FrameAlign);
MemAlign = Align(getByValTypeAlignment(ElementTy, DL));
} else if (auto MA = Args[i].Alignment) {
MemAlign = *MA;
} else {
MemAlign = OriginalAlignment;
}
Flags.setMemAlign(MemAlign);
if (Args[i].IsNest)
Flags.setNest();
if (NeedsRegBlock)
Flags.setInConsecutiveRegs();
Flags.setOrigAlign(OriginalAlignment);
MVT PartVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
CLI.CallConv, VT);
@ -9960,11 +9964,6 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
ISD::ArgFlagsTy Flags;
// Certain targets (such as MIPS), may have a different ABI alignment
// for a type depending on the context. Give the target a chance to
// specify the alignment it wants.
const Align OriginalAlignment(
TLI->getABIAlignmentForCallingConv(ArgTy, DL));
if (Arg.getType()->isPointerTy()) {
Flags.setPointer();
@ -10017,6 +10016,14 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
Flags.setByVal();
}
// Certain targets (such as MIPS), may have a different ABI alignment
// for a type depending on the context. Give the target a chance to
// specify the alignment it wants.
const Align OriginalAlignment(
TLI->getABIAlignmentForCallingConv(ArgTy, DL));
Flags.setOrigAlign(OriginalAlignment);
Align MemAlign;
Type *ArgMemTy = nullptr;
if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
Flags.isByRef()) {
@ -10028,24 +10035,27 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
// For in-memory arguments, size and alignment should be passed from FE.
// BE will guess if this info is not there but there are cases it cannot
// get right.
MaybeAlign MemAlign = Arg.getParamAlign();
if (!MemAlign)
if (auto ParamAlign = Arg.getParamStackAlign())
MemAlign = *ParamAlign;
else if ((ParamAlign = Arg.getParamAlign()))
MemAlign = *ParamAlign;
else
MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL));
if (Flags.isByRef()) {
if (Flags.isByRef())
Flags.setByRefSize(MemSize);
Flags.setByRefAlign(*MemAlign);
} else {
else
Flags.setByValSize(MemSize);
Flags.setByValAlign(*MemAlign);
}
} else if (auto ParamAlign = Arg.getParamStackAlign()) {
MemAlign = *ParamAlign;
} else {
MemAlign = OriginalAlignment;
}
Flags.setMemAlign(MemAlign);
if (Arg.hasAttribute(Attribute::Nest))
Flags.setNest();
if (NeedsRegBlock)
Flags.setInConsecutiveRegs();
Flags.setOrigAlign(OriginalAlignment);
if (ArgCopyElisionCandidates.count(&Arg))
Flags.setCopyElisionCandidate();
if (Arg.hasAttribute(Attribute::Returned))

7
lib/CodeGen/SelectionDAG/TargetLowering.cpp
View File

@ -115,10 +115,13 @@ void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
Alignment = Call->getParamAlign(ArgIdx);
Alignment = Call->getParamStackAlign(ArgIdx);
ByValType = nullptr;
if (IsByVal)
if (IsByVal) {
ByValType = Call->getParamByValType(ArgIdx);
if (!Alignment)
Alignment = Call->getParamAlign(ArgIdx);
}
PreallocatedType = nullptr;
if (IsPreallocated)
PreallocatedType = Call->getParamPreallocatedType(ArgIdx);

4
lib/IR/Attributes.cpp
View File

@ -1583,6 +1583,10 @@ MaybeAlign AttributeList::getParamAlignment(unsigned ArgNo) const {
return getAttributes(ArgNo + FirstArgIndex).getAlignment();
}
MaybeAlign AttributeList::getParamStackAlignment(unsigned ArgNo) const {
return getAttributes(ArgNo + FirstArgIndex).getStackAlignment();
}
Type *AttributeList::getParamByValType(unsigned Index) const {
return getAttributes(Index+FirstArgIndex).getByValType();
}

4
lib/IR/Function.cpp
View File

@ -200,6 +200,10 @@ MaybeAlign Argument::getParamAlign() const {
return getParent()->getParamAlign(getArgNo());
}
MaybeAlign Argument::getParamStackAlign() const {
return getParent()->getParamStackAlign(getArgNo());
}
Type *Argument::getParamByValType() const {
assert(getType()->isPointerTy() && "Only pointers have byval types");
return getParent()->getParamByValType(getArgNo());

5
lib/IR/Verifier.cpp
View File

@ -1648,7 +1648,6 @@ static bool isFuncOnlyAttr(Attribute::AttrKind Kind) {
case Attribute::NoImplicitFloat:
case Attribute::Naked:
case Attribute::InlineHint:
case Attribute::StackAlignment:
case Attribute::UWTable:
case Attribute::VScaleRange:
case Attribute::NonLazyBind:
@ -1691,7 +1690,7 @@ static bool isFuncOnlyAttr(Attribute::AttrKind Kind) {
static bool isFuncOrArgAttr(Attribute::AttrKind Kind) {
return Kind == Attribute::ReadOnly || Kind == Attribute::WriteOnly ||
Kind == Attribute::ReadNone || Kind == Attribute::NoFree ||
Kind == Attribute::Preallocated;
Kind == Attribute::Preallocated || Kind == Attribute::StackAlignment;
}
void Verifier::verifyAttributeTypes(AttributeSet Attrs, bool IsFunction,
@ -3313,7 +3312,7 @@ static AttrBuilder getParameterABIAttributes(int I, AttributeList Attrs) {
static const Attribute::AttrKind ABIAttrs[] = {
Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca,
Attribute::InReg, Attribute::SwiftSelf, Attribute::SwiftError,
Attribute::Preallocated, Attribute::ByRef};
Attribute::Preallocated, Attribute::ByRef, Attribute::StackAlignment};
AttrBuilder Copy;
for (auto AK : ABIAttrs) {
if (Attrs.hasParamAttribute(I, AK))

14
lib/Target/AArch64/AArch64CallingConvention.cpp
View File

@ -88,13 +88,8 @@ static bool finishStackBlock(SmallVectorImpl<CCValAssign> &PendingMembers,
}
unsigned Size = LocVT.getSizeInBits() / 8;
const Align StackAlign =
State.getMachineFunction().getDataLayout().getStackAlignment();
const Align OrigAlign = ArgFlags.getNonZeroOrigAlign();
const Align Alignment = std::min(OrigAlign, StackAlign);
for (auto &It : PendingMembers) {
It.convertToMem(State.AllocateStack(Size, std::max(Alignment, SlotAlign)));
It.convertToMem(State.AllocateStack(Size, SlotAlign));
State.addLoc(It);
SlotAlign = Align(1);
}
@ -197,7 +192,12 @@ static bool CC_AArch64_Custom_Block(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
State.AllocateReg(Reg);
}
const Align SlotAlign = Subtarget.isTargetDarwin() ? Align(1) : Align(8);
const Align StackAlign =
State.getMachineFunction().getDataLayout().getStackAlignment();
const Align MemAlign = ArgFlags.getNonZeroMemAlign();
Align SlotAlign = std::min(MemAlign, StackAlign);
if (!Subtarget.isTargetDarwin())
SlotAlign = std::max(SlotAlign, Align(8));
return finishStackBlock(PendingMembers, LocVT, ArgFlags, State, SlotAlign);
}

2
test/Bitcode/compatibility.ll
View File

@ -550,6 +550,8 @@ declare void @f.param.dereferenceable(i8* dereferenceable(4))
; CHECK: declare void @f.param.dereferenceable(i8* dereferenceable(4))
declare void @f.param.dereferenceable_or_null(i8* dereferenceable_or_null(4))
; CHECK: declare void @f.param.dereferenceable_or_null(i8* dereferenceable_or_null(4))
declare void @f.param.stack_align([2 x double] alignstack(16))
; CHECK: declare void @f.param.stack_align([2 x double] alignstack(16))
; Functions -- unnamed_addr and local_unnamed_addr
declare void @f.unnamed_addr() unnamed_addr

33
test/CodeGen/AArch64/arm64-abi-hfa-args.ll Normal file
View File

@ -0,0 +1,33 @@
; RUN: llc < %s -mtriple=arm64-none-eabi | FileCheck %s
; Over-aligned HFA argument placed on register - one element per register
define double @test_hfa_align_arg_reg([2 x double] alignstack(16) %h.coerce) local_unnamed_addr #0 {
entry:
; CHECK-LABEL: test_hfa_align_arg_reg:
; CHECK-NOT: mov
; CHECK-NOT: ld
; CHECK: ret
%h.coerce.fca.0.extract = extractvalue [2 x double] %h.coerce, 0
ret double %h.coerce.fca.0.extract
}
; Call with over-aligned HFA argument placed on register - one element per register
define double @test_hfa_align_call_reg() local_unnamed_addr #0 {
entry:
; CHECK-LABEL: test_hfa_align_call_reg:
; CHECK-DAG: fmov d0, #1.00000000
; CHECK-DAG: fmov d1, #2.00000000
; CHECK: bl test_hfa_align_arg_reg
%call = call double @test_hfa_align_arg_reg([2 x double] alignstack(16) [double 1.000000e+00, double 2.000000e+00])
ret double %call
}
; Over-aligned HFA argument placed on stack - stack round up to alignment
define double @test_hfa_align_arg_stack(double %d0, double %d1, double %d2, double %d3, double %d4, double %d5, double %d6, double %d7, float %f, [2 x double] alignstack(16) %h.coerce) local_unnamed_addr #0 {
entry:
; CHECK-LABEL: test_hfa_align_arg_stack:
; CHECK: ldr d0, [sp, #16]
; CHECK-NEXT: ret
%h.coerce.fca.0.extract = extractvalue [2 x double] %h.coerce, 0
ret double %h.coerce.fca.0.extract
}