From 61b877a57de9fc37b8a94a71840ec66b4abfee2c Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Tue, 21 Jul 2026 19:56:37 +0200 Subject: [PATCH 01/50] cmake: Filter dangling symlinks from add_td_sources glob (#211051) Editor lock files (e.g. Emacs' `.#foo.td`) are dangling symlinks that `file(GLOB *.td)` picks up and passes to add_library(), which then fails with "Cannot find source file". Filter them out with an EXISTS check, as already done for header files in add_header_files_for_glob. Co-authored-by: Claude (Claude Opus 4.8) --- llvm/cmake/modules/LLVMProcessSources.cmake | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/llvm/cmake/modules/LLVMProcessSources.cmake b/llvm/cmake/modules/LLVMProcessSources.cmake index 0670d60bf2afd..1bd05a9674649 100644 --- a/llvm/cmake/modules/LLVMProcessSources.cmake +++ b/llvm/cmake/modules/LLVMProcessSources.cmake @@ -18,10 +18,19 @@ endfunction(llvm_replace_compiler_option) macro(add_td_sources srcs) file(GLOB tds *.td) - if( tds ) - source_group("TableGen descriptions" FILES ${tds}) - set_source_files_properties(${tds} PROPERTIES HEADER_FILE_ONLY ON) - list(APPEND ${srcs} ${tds}) + # Explicit existence check is necessary to filter dangling symlinks out + # (e.g. editor lock files such as Emacs' `.#foo.td`), as also done in + # add_header_files_for_glob below. + set(existing_tds) + foreach(td ${tds}) + if(EXISTS ${td}) + list(APPEND existing_tds ${td}) + endif() + endforeach() + if( existing_tds ) + source_group("TableGen descriptions" FILES ${existing_tds}) + set_source_files_properties(${existing_tds} PROPERTIES HEADER_FILE_ONLY ON) + list(APPEND ${srcs} ${existing_tds}) endif() endmacro(add_td_sources) From f63615b2fdbf153ceaf5e2444e89fdb1aa59ab6c Mon Sep 17 00:00:00 2001 From: Arseniy Obolenskiy Date: Tue, 21 Jul 2026 20:02:01 +0200 Subject: [PATCH 02/50] [AMDGPU] Fix wrong min/max for amdgcn.fmed3 NaN 3rd operand (#203822) --- .../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 6 ++-- .../Transforms/InstCombine/AMDGPU/fmed3.ll | 36 ++++++++----------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp index aefb8d5787e22..64020bfe5fd80 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp @@ -1605,9 +1605,9 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const { : IC.Builder.CreateMinNum(Src0, Src1); break; case KnownIEEEMode::Off: - V = (ConstSrc2 && ConstSrc2->isNegInfinity()) - ? IC.Builder.CreateMinimumNum(Src0, Src1) - : IC.Builder.CreateMaximumNum(Src0, Src1); + V = (ConstSrc2 && ConstSrc2->isPosInfinity()) + ? IC.Builder.CreateMaximumNum(Src0, Src1) + : IC.Builder.CreateMinimumNum(Src0, Src1); break; case KnownIEEEMode::Unknown: break; diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/fmed3.ll b/llvm/test/Transforms/InstCombine/AMDGPU/fmed3.ll index 90080c5bd5738..ac350f8abeea3 100644 --- a/llvm/test/Transforms/InstCombine/AMDGPU/fmed3.ll +++ b/llvm/test/Transforms/InstCombine/AMDGPU/fmed3.ll @@ -137,7 +137,7 @@ define float @fmed3_x_y_undef_f32(float %x, float %y) #1 { ; ; IEEE0-LABEL: define float @fmed3_x_y_undef_f32( ; IEEE0-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] { -; IEEE0-NEXT: [[MED3:%.*]] = call float @llvm.maximumnum.f32(float [[X]], float [[Y]]) +; IEEE0-NEXT: [[MED3:%.*]] = call float @llvm.minimumnum.f32(float [[X]], float [[Y]]) ; IEEE0-NEXT: ret float [[MED3]] ; %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float %y, float undef) @@ -182,7 +182,7 @@ define float @fmed3_x_y_qnan0_f32(float %x, float %y) #1 { ; ; IEEE0-LABEL: define float @fmed3_x_y_qnan0_f32( ; IEEE0-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] { -; IEEE0-NEXT: [[MED3:%.*]] = call float @llvm.maximumnum.f32(float [[X]], float [[Y]]) +; IEEE0-NEXT: [[MED3:%.*]] = call float @llvm.minimumnum.f32(float [[X]], float [[Y]]) ; IEEE0-NEXT: ret float [[MED3]] ; %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float %y, float 0x7FF8000000000000) @@ -374,13 +374,9 @@ define float @fmed3_0_nan_1_f32() #1 { } define float @fmed3_0_1_nan_f32() #1 { -; IEEE1-LABEL: define float @fmed3_0_1_nan_f32( -; IEEE1-SAME: ) #[[ATTR1]] { -; IEEE1-NEXT: ret float 0.000000e+00 -; -; IEEE0-LABEL: define float @fmed3_0_1_nan_f32( -; IEEE0-SAME: ) #[[ATTR1]] { -; IEEE0-NEXT: ret float 1.000000e+00 +; CHECK-LABEL: define float @fmed3_0_1_nan_f32( +; CHECK-SAME: ) #[[ATTR1]] { +; CHECK-NEXT: ret float 0.000000e+00 ; %med = call float @llvm.amdgcn.fmed3.f32(float 0.0, float 1.0, float 0x7FF8001000000000) ret float %med @@ -405,13 +401,9 @@ define float @fmed3_0_undef_1_f32() #1 { } define float @fmed3_0_1_undef_f32() #1 { -; IEEE1-LABEL: define float @fmed3_0_1_undef_f32( -; IEEE1-SAME: ) #[[ATTR1]] { -; IEEE1-NEXT: ret float 0.000000e+00 -; -; IEEE0-LABEL: define float @fmed3_0_1_undef_f32( -; IEEE0-SAME: ) #[[ATTR1]] { -; IEEE0-NEXT: ret float 1.000000e+00 +; CHECK-LABEL: define float @fmed3_0_1_undef_f32( +; CHECK-SAME: ) #[[ATTR1]] { +; CHECK-NEXT: ret float 0.000000e+00 ; %med = call float @llvm.amdgcn.fmed3.f32(float 0.0, float 1.0, float undef) ret float %med @@ -479,7 +471,7 @@ define float @fmed3_x_y_snan1_f32(float %x, float %y) #1 { ; ; IEEE0-LABEL: define float @fmed3_x_y_snan1_f32( ; IEEE0-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] { -; IEEE0-NEXT: [[MED3:%.*]] = call float @llvm.maximumnum.f32(float [[X]], float [[Y]]) +; IEEE0-NEXT: [[MED3:%.*]] = call float @llvm.minimumnum.f32(float [[X]], float [[Y]]) ; IEEE0-NEXT: ret float [[MED3]] ; %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float %y, float 0x7FF4000000000000) @@ -558,7 +550,7 @@ define float @fmed3_neg2_3_snan1_f32(float %x, float %y) #1 { ; ; IEEE0-LABEL: define float @fmed3_neg2_3_snan1_f32( ; IEEE0-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] { -; IEEE0-NEXT: ret float 3.000000e+00 +; IEEE0-NEXT: ret float -2.000000e+00 ; %med3 = call float @llvm.amdgcn.fmed3.f32(float -2.0, float 3.0, float 0x7FF4000000000000) ret float %med3 @@ -699,12 +691,12 @@ define amdgpu_ps float @amdgpu_ps_default_fmed3_x_snan1_y_f32(float %x, float %y define amdgpu_ps float @amdgpu_ps_default_fmed3_x_y_snan1_f32(float %x, float %y) { ; HAS-IEEE-BIT1-LABEL: define amdgpu_ps float @amdgpu_ps_default_fmed3_x_y_snan1_f32( ; HAS-IEEE-BIT1-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; HAS-IEEE-BIT1-NEXT: [[MED3:%.*]] = call float @llvm.maximumnum.f32(float [[X]], float [[Y]]) +; HAS-IEEE-BIT1-NEXT: [[MED3:%.*]] = call float @llvm.minimumnum.f32(float [[X]], float [[Y]]) ; HAS-IEEE-BIT1-NEXT: ret float [[MED3]] ; ; IEEE0-LABEL: define amdgpu_ps float @amdgpu_ps_default_fmed3_x_y_snan1_f32( ; IEEE0-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; IEEE0-NEXT: [[MED3:%.*]] = call float @llvm.maximumnum.f32(float [[X]], float [[Y]]) +; IEEE0-NEXT: [[MED3:%.*]] = call float @llvm.minimumnum.f32(float [[X]], float [[Y]]) ; IEEE0-NEXT: ret float [[MED3]] ; ; NO-IEEE-BIT-LABEL: define amdgpu_ps float @amdgpu_ps_default_fmed3_x_y_snan1_f32( @@ -753,7 +745,7 @@ define amdgpu_ps float @amdgpu_ps_attr_fmed3_x_y_snan1_f32(float %x, float %y) # ; ; IEEE0-LABEL: define amdgpu_ps float @amdgpu_ps_attr_fmed3_x_y_snan1_f32( ; IEEE0-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] { -; IEEE0-NEXT: [[MED3:%.*]] = call float @llvm.maximumnum.f32(float [[X]], float [[Y]]) +; IEEE0-NEXT: [[MED3:%.*]] = call float @llvm.minimumnum.f32(float [[X]], float [[Y]]) ; IEEE0-NEXT: ret float [[MED3]] ; %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float %y, float 0x7FF4000000000000) @@ -893,7 +885,7 @@ define float @fmed3_x_y_qnan0_f32_flags(float %x, float %y) #1 { ; ; IEEE0-LABEL: define float @fmed3_x_y_qnan0_f32_flags( ; IEEE0-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] { -; IEEE0-NEXT: [[MED3:%.*]] = call ninf nsz float @llvm.maximumnum.f32(float [[X]], float [[Y]]) +; IEEE0-NEXT: [[MED3:%.*]] = call ninf nsz float @llvm.minimumnum.f32(float [[X]], float [[Y]]) ; IEEE0-NEXT: ret float [[MED3]] ; %med3 = call nsz ninf float @llvm.amdgcn.fmed3.f32(float %x, float %y, float 0x7FF8000000000000) From 879db2f9478f454f8dcf961b660a9078b73bf1b5 Mon Sep 17 00:00:00 2001 From: Arseniy Obolenskiy Date: Tue, 21 Jul 2026 20:02:36 +0200 Subject: [PATCH 03/50] [AMDGPU] Fix ISel crash on llvm.amdgcn.class with bf16 operand (#210971) --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 17 ++++++++++++++--- .../CodeGen/AMDGPU/llvm.amdgcn.class.bf16.ll | 19 +++++++++++++++++++ .../AMDGPU/llvm.amdgcn.class.f16.err.ll | 18 ++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.bf16.ll create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.err.ll diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index f92b1731c3f7d..705efd3aa8ef0 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -11185,9 +11185,20 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, case Intrinsic::amdgcn_fract: return DAG.getNode(AMDGPUISD::FRACT, DL, VT, Op.getOperand(1)); - case Intrinsic::amdgcn_class: - return DAG.getNode(AMDGPUISD::FP_CLASS, DL, VT, Op.getOperand(1), - Op.getOperand(2)); + case Intrinsic::amdgcn_class: { + SDValue Src = Op.getOperand(1); + EVT SrcVT = Src.getValueType(); + bool IsLegal = SrcVT == MVT::f32 || SrcVT == MVT::f64 || + (SrcVT == MVT::f16 && Subtarget->has16BitInsts()); + if (!IsLegal) { + DAG.getContext()->diagnose(DiagnosticInfoUnsupported( + DAG.getMachineFunction().getFunction(), + "llvm.amdgcn.class only supports f16, f32, and f64", + DL.getDebugLoc())); + return DAG.getPOISON(VT); + } + return DAG.getNode(AMDGPUISD::FP_CLASS, DL, VT, Src, Op.getOperand(2)); + } case Intrinsic::amdgcn_div_fmas: return DAG.getNode(AMDGPUISD::DIV_FMAS, DL, VT, Op.getOperand(1), Op.getOperand(2), Op.getOperand(3), Op.getOperand(4)); diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.bf16.ll new file mode 100644 index 0000000000000..46531efe31725 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.bf16.ll @@ -0,0 +1,19 @@ +; RUN: not llc -mtriple=amdgpu9.50 < %s 2>&1 | FileCheck -check-prefix=ERROR %s +; RUN: not llc -mtriple=amdgpu6.00 < %s 2>&1 | FileCheck -check-prefix=ERROR %s + +; ERROR: error: :0:0: in function class_bf16 void (ptr addrspace(1), ptr addrspace(1), ptr addrspace(1)): llvm.amdgcn.class only supports f16, f32, and f64 + +declare i1 @llvm.amdgcn.class.bf16(bfloat %a, i32 %b) + +define amdgpu_kernel void @class_bf16( + ptr addrspace(1) %r, + ptr addrspace(1) %a, + ptr addrspace(1) %b) { +entry: + %a.val = load bfloat, ptr addrspace(1) %a + %b.val = load i32, ptr addrspace(1) %b + %r.val = call i1 @llvm.amdgcn.class.bf16(bfloat %a.val, i32 %b.val) + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, ptr addrspace(1) %r + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.err.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.err.ll new file mode 100644 index 0000000000000..4d60b9d57a0b4 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.err.ll @@ -0,0 +1,18 @@ +; RUN: not llc -mtriple=amdgpu7.00 < %s 2>&1 | FileCheck -check-prefix=ERROR %s + +; ERROR: error: :0:0: in function class_f16 void (ptr addrspace(1), ptr addrspace(1), ptr addrspace(1)): llvm.amdgcn.class only supports f16, f32, and f64 + +declare i1 @llvm.amdgcn.class.f16(half %a, i32 %b) + +define amdgpu_kernel void @class_f16( + ptr addrspace(1) %r, + ptr addrspace(1) %a, + ptr addrspace(1) %b) { +entry: + %a.val = load half, ptr addrspace(1) %a + %b.val = load i32, ptr addrspace(1) %b + %r.val = call i1 @llvm.amdgcn.class.f16(half %a.val, i32 %b.val) + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, ptr addrspace(1) %r + ret void +} From 35d69e3fe344e063174f54a9785a08249ba18cd9 Mon Sep 17 00:00:00 2001 From: Alexis Engelke Date: Tue, 21 Jul 2026 20:05:57 +0200 Subject: [PATCH 04/50] [Analysis] Use CycleInfo for BranchProbabilityInfo (#210301) Instead of computing a cycle info (SccInfo) internally in BranchProbabilityInfo, use CycleInfo and remove the use of LoopInfo. After recent improvements to CycleInfo, the extra analysis is cheaper than computing the same information internally. Replacing LoopInfo with CycleInfo has some impliciation that the loop/cycle forest can differ if the header of an irreducible loop (which is ignored by loop info) also happens to be the header of a different natural loop. (Simple example: S -> A, B; A -> A, B; B -> A, B -- LoopInfo will find {{A}, {B}}; CycleInfo will find {A, {B}}].) In these cases, the default weights will differ. This change also makes the handling of entering/exiting edges consistent between reducible and irreducible loops, therefore, there are also minor changes for reducible CFGs. --- .../llvm/Analysis/BranchProbabilityInfo.h | 9 +- .../llvm/Analysis/LazyBranchProbabilityInfo.h | 14 +- llvm/lib/Analysis/BranchProbabilityInfo.cpp | 329 +++--------------- .../Analysis/LazyBranchProbabilityInfo.cpp | 14 +- llvm/lib/Analysis/ModuleSummaryAnalysis.cpp | 5 +- .../Analysis/OptimizationRemarkEmitter.cpp | 5 +- llvm/lib/CodeGen/CodeGenPrepare.cpp | 5 +- llvm/lib/CodeGen/MachineFunctionPass.cpp | 2 + llvm/lib/Transforms/IPO/PartialInlining.cpp | 17 +- .../Instrumentation/InstrProfiling.cpp | 5 +- .../Instrumentation/PGOInstrumentation.cpp | 13 +- .../Analysis/BranchProbabilityInfo/loop.ll | 77 +++- .../GlobalISel/gisel-commandline-option.ll | 1 + llvm/test/CodeGen/AArch64/O3-pipeline.ll | 9 +- llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 29 +- llvm/test/CodeGen/ARM/O3-pipeline.ll | 14 +- .../CodeGen/LoongArch/annotate-tablejump.ll | 36 +- llvm/test/CodeGen/LoongArch/opt-pipeline.ll | 7 +- llvm/test/CodeGen/M68k/pipeline.ll | 7 +- llvm/test/CodeGen/PowerPC/O3-pipeline.ll | 9 +- .../GlobalISel/gisel-commandline-option.ll | 1 + llvm/test/CodeGen/RISCV/O3-pipeline.ll | 17 +- llvm/test/CodeGen/SPIRV/llc-pipeline.ll | 10 +- .../GlobalISel/gisel-commandline-option.ll | 3 +- llvm/test/CodeGen/X86/opt-pipeline.ll | 7 +- llvm/test/CodeGen/X86/pr38795.ll | 99 +++--- .../new-pm-thinlto-postlink-pgo-defaults.ll | 4 +- ...-pm-thinlto-postlink-samplepgo-defaults.ll | 4 +- .../new-pm-thinlto-prelink-pgo-defaults.ll | 7 +- ...w-pm-thinlto-prelink-samplepgo-defaults.ll | 4 +- .../Inline/cgscc-incremental-invalidate.ll | 9 +- .../PGOProfile/Inputs/irreducible.proftext | 8 +- .../Inputs/irreducible_entry.proftext | 6 +- .../Analysis/BlockFrequencyInfoTest.cpp | 6 +- .../Analysis/BranchProbabilityInfoTest.cpp | 9 +- .../Analysis/ProfileSummaryInfoTest.cpp | 6 +- .../IPO/FunctionSpecializationTest.cpp | 4 +- .../Transforms/Utils/BasicBlockUtilsTest.cpp | 14 +- .../Transforms/Utils/SizeOptsTest.cpp | 8 +- 39 files changed, 400 insertions(+), 433 deletions(-) diff --git a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h index dfc0079724d52..970fd532a1485 100644 --- a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h +++ b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h @@ -27,8 +27,7 @@ namespace llvm { class Function; -class Loop; -class LoopInfo; +class CycleInfo; class raw_ostream; class DominatorTree; class PostDominatorTree; @@ -110,11 +109,11 @@ class BranchProbabilityInfo { public: BranchProbabilityInfo() = default; - BranchProbabilityInfo(const Function &F, const LoopInfo &LI, + BranchProbabilityInfo(const Function &F, const CycleInfo &CI, const TargetLibraryInfo *TLI = nullptr, DominatorTree *DT = nullptr, PostDominatorTree *PDT = nullptr) { - calculate(F, LI, TLI, DT, PDT); + calculate(F, CI, TLI, DT, PDT); } LLVM_ABI bool invalidate(Function &, const PreservedAnalyses &PA, @@ -174,7 +173,7 @@ class BranchProbabilityInfo { return IsLikely ? LikelyProb : LikelyProb.getCompl(); } - LLVM_ABI void calculate(const Function &F, const LoopInfo &LI, + LLVM_ABI void calculate(const Function &F, const CycleInfo &CI, const TargetLibraryInfo *TLI, DominatorTree *DT, PostDominatorTree *PDT); diff --git a/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h b/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h index 0f8932f784881..6cd4528d05a64 100644 --- a/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h +++ b/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h @@ -20,8 +20,8 @@ #include "llvm/Pass.h" namespace llvm { +class CycleInfo; class Function; -class LoopInfo; class TargetLibraryInfo; /// This is an alternative analysis pass to @@ -41,7 +41,7 @@ class TargetLibraryInfo; /// LazyBranchProbabilityInfoPass::getLazyBPIAnalysisUsage(AU) /// /// 3. The computed BPI should be requested with -/// getAnalysis().getBPI() before LoopInfo +/// getAnalysis().getBPI() before CycleInfo /// could be invalidated for example by changing the CFG. /// /// Note that it is expected that we wouldn't need this functionality for the @@ -54,15 +54,15 @@ class LLVM_ABI LazyBranchProbabilityInfoPass : public FunctionPass { /// analysis without paying for the overhead if BPI doesn't end up being used. class LazyBranchProbabilityInfo { public: - LazyBranchProbabilityInfo(const Function *F, const LoopInfo *LI, + LazyBranchProbabilityInfo(const Function *F, const CycleInfo *CI, const TargetLibraryInfo *TLI) - : F(F), LI(LI), TLI(TLI) {} + : F(F), CI(CI), TLI(TLI) {} /// Retrieve the BPI with the branch probabilities computed. BranchProbabilityInfo &getCalculated() { if (!Calculated) { - assert(F && LI && "call setAnalysis"); - BPI.calculate(*F, *LI, TLI, nullptr, nullptr); + assert(F && CI && "call setAnalysis"); + BPI.calculate(*F, *CI, TLI, nullptr, nullptr); Calculated = true; } return BPI; @@ -76,7 +76,7 @@ class LLVM_ABI LazyBranchProbabilityInfoPass : public FunctionPass { BranchProbabilityInfo BPI; bool Calculated = false; const Function *F; - const LoopInfo *LI; + const CycleInfo *CI; const TargetLibraryInfo *TLI; }; diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp index 1b0a8b8e74f4a..c409c68909e71 100644 --- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp +++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp @@ -16,7 +16,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/ConstantFolding.h" -#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/CycleAnalysis.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/Attributes.h" @@ -61,7 +61,7 @@ static cl::opt PrintBranchProbFuncName( INITIALIZE_PASS_BEGIN(BranchProbabilityInfoWrapperPass, "branch-prob", "Branch Probability Analysis", false, true) -INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(CycleInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) INITIALIZE_PASS_DEPENDENCY(PostDominatorTreeWrapperPass) @@ -165,98 +165,24 @@ namespace { class BPIConstruction { public: BPIConstruction(BranchProbabilityInfo &BPI) : BPI(BPI) {} - void calculate(const Function &F, const LoopInfo &LI, + void calculate(const Function &F, const CycleInfo &CI, const TargetLibraryInfo *TLI, DominatorTree *DT, PostDominatorTree *PDT); private: - // Data structure to track SCCs for handling irreducible loops. - class SccInfo { - // Enum of types to classify basic blocks in SCC. Basic block belonging to - // SCC is 'Inner' until it is either 'Header' or 'Exiting'. Note that a - // basic block can be 'Header' and 'Exiting' at the same time. - enum SccBlockType { - Inner = 0x0, - Header = 0x1, - Exiting = 0x2, - }; - // Map of basic blocks to SCC IDs they belong to. If basic block doesn't - // belong to any SCC it is not in the map. - using SccMap = DenseMap; - // Each basic block in SCC is attributed with one or several types from - // SccBlockType. Map value has uint32_t type (instead of SccBlockType) - // since basic block may be for example "Header" and "Exiting" at the same - // time and we need to be able to keep more than one value from - // SccBlockType. - using SccBlockTypeMap = DenseMap; - // Vector containing classification of basic blocks for all SCCs where i'th - // vector element corresponds to SCC with ID equal to i. - using SccBlockTypeMaps = std::vector; - - SccMap SccNums; - SccBlockTypeMaps SccBlocks; - - public: - explicit SccInfo(const Function &F); - - /// If \p BB belongs to some SCC then ID of that SCC is returned, otherwise - /// -1 is returned. If \p BB belongs to more than one SCC at the same time - /// result is undefined. - int getSCCNum(const BasicBlock *BB) const; - /// Returns true if \p BB is a 'header' block in SCC with \p SccNum ID, - /// false otherwise. - bool isSCCHeader(const BasicBlock *BB, int SccNum) const { - return getSccBlockType(BB, SccNum) & Header; - } - /// Returns true if \p BB is an 'exiting' block in SCC with \p SccNum ID, - /// false otherwise. - bool isSCCExitingBlock(const BasicBlock *BB, int SccNum) const { - return getSccBlockType(BB, SccNum) & Exiting; - } - /// Fills in \p Enters vector with all such blocks that don't belong to - /// SCC with \p SccNum ID but there is an edge to a block belonging to the - /// SCC. - void getSccEnterBlocks(int SccNum, - SmallVectorImpl &Enters) const; - /// Fills in \p Exits vector with all such blocks that don't belong to - /// SCC with \p SccNum ID but there is an edge from a block belonging to the - /// SCC. - void getSccExitBlocks(int SccNum, - SmallVectorImpl &Exits) const; - - private: - /// Returns \p BB's type according to classification given by SccBlockType - /// enum. Please note that \p BB must belong to SSC with \p SccNum ID. - uint32_t getSccBlockType(const BasicBlock *BB, int SccNum) const; - /// Calculates \p BB's type and stores it in internal data structures for - /// future use. Please note that \p BB must belong to SSC with \p SccNum ID. - void calculateSccBlockType(const BasicBlock *BB, int SccNum); - }; - - /// Pair of Loop and SCC ID number. Used to unify handling of normal and - /// SCC based loop representations. - using LoopData = std::pair; /// Helper class to keep basic block along with its loop data information. class LoopBlock { public: - explicit LoopBlock(const BasicBlock *BB, const LoopInfo &LI, - const SccInfo &SccI); + explicit LoopBlock(const BasicBlock *BB, const CycleInfo &CI) + : BB(BB), C(CI.getCycle(BB)) {} const BasicBlock *getBlock() const { return BB; } BasicBlock *getBlock() { return const_cast(BB); } - LoopData getLoopData() const { return LD; } - Loop *getLoop() const { return LD.first; } - int getSccNum() const { return LD.second; } - - bool belongsToLoop() const { return getLoop() || getSccNum() != -1; } - bool belongsToSameLoop(const LoopBlock &LB) const { - return (LB.getLoop() && getLoop() == LB.getLoop()) || - (LB.getSccNum() != -1 && getSccNum() == LB.getSccNum()); - } + CycleRef getCycle() const { return C; } private: - const BasicBlock *const BB = nullptr; - LoopData LD = {nullptr, -1}; + const BasicBlock *const BB; + CycleRef C; }; // Pair of LoopBlocks representing an edge from first to second block. @@ -264,7 +190,7 @@ class BPIConstruction { /// Helper to construct LoopBlock for \p BB. LoopBlock getLoopBlock(const BasicBlock *BB) const { - return LoopBlock(BB, *LI, *SccI); + return LoopBlock(BB, *CI); } /// Returns true if destination block belongs to some loop and source block is @@ -278,24 +204,18 @@ class BPIConstruction { /// Returns true if \p Edge is either enters to or exits from some loop, false /// in all other cases. bool isLoopEnteringExitingEdge(const LoopEdge &Edge) const; - /// Returns true if source and destination blocks belongs to the same loop and - /// destination block is loop header. - bool isLoopBackEdge(const LoopEdge &Edge) const; // Fills in \p Enters vector with all "enter" blocks to a loop \LB belongs to. void getLoopEnterBlocks(const LoopBlock &LB, SmallVectorImpl &Enters) const; - // Fills in \p Exits vector with all "exit" blocks from a loop \LB belongs to. - void getLoopExitBlocks(const LoopBlock &LB, - SmallVectorImpl &Exits) const; /// Returns estimated weight for \p BB. std::nullopt if \p BB has no estimated /// weight. std::optional getEstimatedBlockWeight(const BasicBlock *BB) const; /// Returns estimated weight to enter \p L. In other words it is weight of - /// loop's header block not scaled by trip count. Returns std::nullopt if \p L + /// loop's header block not scaled by trip count. Returns std::nullopt if \p C /// has no no estimated weight. - std::optional getEstimatedLoopWeight(const LoopData &L) const; + std::optional getEstimatedLoopWeight(CycleRef C) const; /// Return estimated weight for \p Edge. Returns std::nullopt if estimated /// weight is unknown. @@ -341,133 +261,23 @@ class BPIConstruction { BranchProbabilityInfo &BPI; - const LoopInfo *LI = nullptr; - - /// Keeps information about all SCCs in a function. - std::unique_ptr SccI; + const CycleInfo *CI = nullptr; /// Keeps mapping of a basic block to its estimated weight. SmallDenseMap EstimatedBlockWeight; /// Keeps mapping of a loop to estimated weight to enter the loop. - SmallDenseMap EstimatedLoopWeight; + SmallDenseMap EstimatedLoopWeight; }; -BPIConstruction::SccInfo::SccInfo(const Function &F) { - // Record SCC numbers of blocks in the CFG to identify irreducible loops. - // FIXME: We could only calculate this if the CFG is known to be irreducible - // (perhaps cache this info in LoopInfo if we can easily calculate it there?). - int SccNum = 0; - for (scc_iterator It = scc_begin(&F); !It.isAtEnd(); - ++It, ++SccNum) { - // Ignore single-block SCCs since they either aren't loops or LoopInfo will - // catch them. - const std::vector &Scc = *It; - if (Scc.size() == 1) - continue; - - LLVM_DEBUG(dbgs() << "BPI: SCC " << SccNum << ":"); - for (const auto *BB : Scc) { - LLVM_DEBUG(dbgs() << " " << BB->getName()); - SccNums[BB] = SccNum; - calculateSccBlockType(BB, SccNum); - } - LLVM_DEBUG(dbgs() << "\n"); - } -} - -int BPIConstruction::SccInfo::getSCCNum(const BasicBlock *BB) const { - auto SccIt = SccNums.find(BB); - if (SccIt == SccNums.end()) - return -1; - return SccIt->second; -} - -void BPIConstruction::SccInfo::getSccEnterBlocks( - int SccNum, SmallVectorImpl &Enters) const { - - for (auto MapIt : SccBlocks[SccNum]) { - const auto *BB = MapIt.first; - if (isSCCHeader(BB, SccNum)) - for (const auto *Pred : predecessors(BB)) - if (getSCCNum(Pred) != SccNum) - Enters.push_back(const_cast(BB)); - } -} - -void BPIConstruction::SccInfo::getSccExitBlocks( - int SccNum, SmallVectorImpl &Exits) const { - for (auto MapIt : SccBlocks[SccNum]) { - const auto *BB = MapIt.first; - if (isSCCExitingBlock(BB, SccNum)) - for (const auto *Succ : successors(BB)) - if (getSCCNum(Succ) != SccNum) - Exits.push_back(const_cast(Succ)); - } -} - -uint32_t BPIConstruction::SccInfo::getSccBlockType(const BasicBlock *BB, - int SccNum) const { - assert(getSCCNum(BB) == SccNum); - - assert(SccBlocks.size() > static_cast(SccNum) && "Unknown SCC"); - const auto &SccBlockTypes = SccBlocks[SccNum]; - - auto It = SccBlockTypes.find(BB); - if (It != SccBlockTypes.end()) { - return It->second; - } - return Inner; -} - -void BPIConstruction::SccInfo::calculateSccBlockType(const BasicBlock *BB, - int SccNum) { - assert(getSCCNum(BB) == SccNum); - uint32_t BlockType = Inner; - - if (llvm::any_of(predecessors(BB), [&](const BasicBlock *Pred) { - // Consider any block that is an entry point to the SCC as - // a header. - return getSCCNum(Pred) != SccNum; - })) - BlockType |= Header; - - if (llvm::any_of(successors(BB), [&](const BasicBlock *Succ) { - return getSCCNum(Succ) != SccNum; - })) - BlockType |= Exiting; - - // Lazily compute the set of headers for a given SCC and cache the results - // in the SccHeaderMap. - if (SccBlocks.size() <= static_cast(SccNum)) - SccBlocks.resize(SccNum + 1); - auto &SccBlockTypes = SccBlocks[SccNum]; - - if (BlockType != Inner) { - bool IsInserted; - std::tie(std::ignore, IsInserted) = - SccBlockTypes.insert(std::make_pair(BB, BlockType)); - assert(IsInserted && "Duplicated block in SCC"); - } -} - -BPIConstruction::LoopBlock::LoopBlock(const BasicBlock *BB, const LoopInfo &LI, - const SccInfo &SccI) - : BB(BB) { - LD.first = LI.getLoopFor(BB); - if (!LD.first) { - LD.second = SccI.getSCCNum(BB); - } -} - bool BPIConstruction::isLoopEnteringEdge(const LoopEdge &Edge) const { const auto &SrcBlock = Edge.first; const auto &DstBlock = Edge.second; - return (DstBlock.getLoop() && - !DstBlock.getLoop()->contains(SrcBlock.getLoop())) || - // Assume that SCCs can't be nested. - (DstBlock.getSccNum() != -1 && - SrcBlock.getSccNum() != DstBlock.getSccNum()); + if (!DstBlock.getCycle()) // Edge into no-cycle is not entering. + return false; + if (!SrcBlock.getCycle()) // Edge from no-cycle into cycle is entering. + return true; + return !CI->contains(DstBlock.getCycle(), SrcBlock.getCycle()); } bool BPIConstruction::isLoopExitingEdge(const LoopEdge &Edge) const { @@ -478,35 +288,13 @@ bool BPIConstruction::isLoopEnteringExitingEdge(const LoopEdge &Edge) const { return isLoopEnteringEdge(Edge) || isLoopExitingEdge(Edge); } -bool BPIConstruction::isLoopBackEdge(const LoopEdge &Edge) const { - const auto &SrcBlock = Edge.first; - const auto &DstBlock = Edge.second; - return SrcBlock.belongsToSameLoop(DstBlock) && - ((DstBlock.getLoop() && - DstBlock.getLoop()->getHeader() == DstBlock.getBlock()) || - (DstBlock.getSccNum() != -1 && - SccI->isSCCHeader(DstBlock.getBlock(), DstBlock.getSccNum()))); -} - void BPIConstruction::getLoopEnterBlocks( const LoopBlock &LB, SmallVectorImpl &Enters) const { - if (LB.getLoop()) { - auto *Header = LB.getLoop()->getHeader(); - Enters.append(pred_begin(Header), pred_end(Header)); - } else { - assert(LB.getSccNum() != -1 && "LB doesn't belong to any loop?"); - SccI->getSccEnterBlocks(LB.getSccNum(), Enters); - } -} - -void BPIConstruction::getLoopExitBlocks( - const LoopBlock &LB, SmallVectorImpl &Exits) const { - if (LB.getLoop()) { - LB.getLoop()->getExitBlocks(Exits); - } else { - assert(LB.getSccNum() != -1 && "LB doesn't belong to any loop?"); - SccI->getSccExitBlocks(LB.getSccNum(), Exits); - } + CycleRef C = LB.getCycle(); + for (BasicBlock *Entry : CI->getEntries(C)) + for (const auto *Pred : predecessors(Entry)) + if (!CI->contains(C, Pred)) + Enters.push_back(const_cast(Pred)); } // Propagate existing explicit probabilities from either profile data or @@ -679,12 +467,12 @@ bool BPIConstruction::calcPointerHeuristics(const BasicBlock *BB) { } } -// Compute the unlikely successors to the block BB in the loop L, specifically +// Compute the unlikely successors to the block BB in the cycle C, specifically // those that are unlikely because this is a loop, and add them to the // UnlikelyBlocks set. static void -computeUnlikelySuccessors(const BasicBlock *BB, Loop *L, - SmallPtrSetImpl &UnlikelyBlocks) { +computeUnlikelySuccessors(const BasicBlock *BB, const CycleInfo &CI, CycleRef C, + SmallPtrSetImpl &UnlikelyBlocks) { // Sometimes in a loop we have a branch whose condition is made false by // taking it. This is typically something like // int n = 0; @@ -711,30 +499,30 @@ computeUnlikelySuccessors(const BasicBlock *BB, Loop *L, return; // Check if the branch is based on an instruction compared with a constant - CmpInst *CI = dyn_cast(BI->getCondition()); - if (!CI || !isa(CI->getOperand(0)) || - !isa(CI->getOperand(1))) + CmpInst *Cmp = dyn_cast(BI->getCondition()); + if (!Cmp || !isa(Cmp->getOperand(0)) || + !isa(Cmp->getOperand(1))) return; // Either the instruction must be a PHI, or a chain of operations involving // constants that ends in a PHI which we can then collapse into a single value // if the PHI value is known. - Instruction *CmpLHS = dyn_cast(CI->getOperand(0)); + Instruction *CmpLHS = dyn_cast(Cmp->getOperand(0)); PHINode *CmpPHI = dyn_cast(CmpLHS); - Constant *CmpConst = dyn_cast(CI->getOperand(1)); + Constant *CmpConst = dyn_cast(Cmp->getOperand(1)); // Collect the instructions until we hit a PHI SmallVector InstChain; while (!CmpPHI && CmpLHS && isa(CmpLHS) && isa(CmpLHS->getOperand(1))) { // Stop if the chain extends outside of the loop - if (!L->contains(CmpLHS)) + if (!CI.contains(C, CmpLHS->getParent())) return; InstChain.push_back(cast(CmpLHS)); CmpLHS = dyn_cast(CmpLHS->getOperand(0)); if (CmpLHS) CmpPHI = dyn_cast(CmpLHS); } - if (!CmpPHI || !L->contains(CmpPHI)) + if (!CmpPHI || !CI.contains(C, CmpPHI->getParent())) return; // Trace the phi node to find all values that come from successors of BB @@ -746,7 +534,7 @@ computeUnlikelySuccessors(const BasicBlock *BB, Loop *L, PHINode *P = WorkList.pop_back_val(); for (BasicBlock *B : P->blocks()) { // Skip blocks that aren't part of the loop - if (!L->contains(B)) + if (!CI.contains(C, B)) continue; Value *V = P->getIncomingValueForBlock(B); // If the source is a PHI add it to the work list if we haven't @@ -774,7 +562,7 @@ computeUnlikelySuccessors(const BasicBlock *BB, Loop *L, continue; // Now constant-evaluate the compare Constant *Result = ConstantFoldCompareInstOperands( - CI->getPredicate(), CmpLHSConst, CmpConst, DL); + Cmp->getPredicate(), CmpLHSConst, CmpConst, DL); // If the result means we don't branch to the block then that block is // unlikely. if (Result && ((Result->isNullValue() && B == BI->getSuccessor(0)) || @@ -793,8 +581,8 @@ BPIConstruction::getEstimatedBlockWeight(const BasicBlock *BB) const { } std::optional -BPIConstruction::getEstimatedLoopWeight(const LoopData &L) const { - auto WeightIt = EstimatedLoopWeight.find(L); +BPIConstruction::getEstimatedLoopWeight(CycleRef C) const { + auto WeightIt = EstimatedLoopWeight.find(C); if (WeightIt == EstimatedLoopWeight.end()) return std::nullopt; return WeightIt->second; @@ -805,7 +593,7 @@ BPIConstruction::getEstimatedEdgeWeight(const LoopEdge &Edge) const { // For edges entering a loop take weight of a loop rather than an individual // block in the loop. return isLoopEnteringEdge(Edge) - ? getEstimatedLoopWeight(Edge.second.getLoopData()) + ? getEstimatedLoopWeight(Edge.second.getCycle()) : getEstimatedBlockWeight(Edge.second.getBlock()); } @@ -850,7 +638,7 @@ bool BPIConstruction::updateEstimatedBlockWeight( LoopBlock PredLoop = getLoopBlock(PredBlock); // Add affected block/loop to a working list. if (isLoopExitingEdge({PredLoop, LoopBB})) { - if (!EstimatedLoopWeight.count(PredLoop.getLoopData())) + if (!EstimatedLoopWeight.count(PredLoop.getCycle())) LoopWorkList.push_back(PredLoop); } else if (!EstimatedBlockWeight.count(PredBlock)) BlockWorkList.push_back(PredBlock); @@ -948,7 +736,7 @@ void BPIConstruction::estimateBlockWeights(const Function &F, DominatorTree *DT, PostDominatorTree *PDT) { SmallVector BlockWorkList; SmallVector LoopWorkList; - SmallDenseMap> LoopExitBlocks; + SmallDenseMap> LoopExitBlocks; // By doing RPO we make sure that all predecessors already have weights // calculated before visiting theirs successors. @@ -967,14 +755,14 @@ void BPIConstruction::estimateBlockWeights(const Function &F, DominatorTree *DT, do { while (!LoopWorkList.empty()) { const LoopBlock LoopBB = LoopWorkList.pop_back_val(); - const LoopData LD = LoopBB.getLoopData(); - if (EstimatedLoopWeight.count(LD)) + CycleRef C = LoopBB.getCycle(); + if (EstimatedLoopWeight.count(C)) continue; - auto Res = LoopExitBlocks.try_emplace(LD); + auto Res = LoopExitBlocks.try_emplace(C); SmallVectorImpl &Exits = Res.first->second; if (Res.second) - getLoopExitBlocks(LoopBB, Exits); + CI->getExitBlocks(C, Exits); auto LoopWeight = getMaxEstimatedEdgeWeight( LoopBB, make_range(Exits.begin(), Exits.end())); @@ -983,7 +771,7 @@ void BPIConstruction::estimateBlockWeights(const Function &F, DominatorTree *DT, if (LoopWeight <= static_cast(BlockExecWeight::UNREACHABLE)) LoopWeight = static_cast(BlockExecWeight::LOWEST_NON_ZERO); - EstimatedLoopWeight.insert({LD, *LoopWeight}); + EstimatedLoopWeight.insert({C, *LoopWeight}); // Add all blocks entering the loop into working list. getLoopEnterBlocks(LoopBB, BlockWorkList); } @@ -1022,8 +810,8 @@ bool BPIConstruction::calcEstimatedHeuristics(const BasicBlock *BB) { SmallPtrSet UnlikelyBlocks; uint32_t TC = LBH_TAKEN_WEIGHT / LBH_NONTAKEN_WEIGHT; - if (LoopBB.getLoop()) - computeUnlikelySuccessors(BB, LoopBB.getLoop(), UnlikelyBlocks); + if (LoopBB.getCycle()) + computeUnlikelySuccessors(BB, *CI, LoopBB.getCycle(), UnlikelyBlocks); // Changed to 'true' if at least one successor has estimated weight. bool FoundEstimatedWeight = false; @@ -1046,7 +834,7 @@ bool BPIConstruction::calcEstimatedHeuristics(const BasicBlock *BB) { Weight.value_or(static_cast(BlockExecWeight::DEFAULT)) / TC); } - bool IsUnlikelyEdge = LoopBB.getLoop() && UnlikelyBlocks.contains(SuccBB); + bool IsUnlikelyEdge = LoopBB.getCycle() && UnlikelyBlocks.contains(SuccBB); if (IsUnlikelyEdge && // Avoid adjustment of ZERO weight since it should remain unchanged. Weight != static_cast(BlockExecWeight::ZERO)) { @@ -1223,12 +1011,10 @@ bool BPIConstruction::calcFloatingPointHeuristics(const BasicBlock *BB) { } return true; } -void BPIConstruction::calculate(const Function &F, const LoopInfo &LoopI, +void BPIConstruction::calculate(const Function &F, const CycleInfo &CycleI, const TargetLibraryInfo *TLI, DominatorTree *DT, PostDominatorTree *PDT) { - LI = &LoopI; - - SccI = std::make_unique(F); + CI = &CycleI; std::unique_ptr DTPtr; std::unique_ptr PDTPtr; @@ -1435,7 +1221,8 @@ void BranchProbabilityInfo::eraseBlock(const BasicBlock *BB) { EdgeStarts[BB->getNumber()] = 0; } -void BranchProbabilityInfo::calculate(const Function &F, const LoopInfo &LoopI, +void BranchProbabilityInfo::calculate(const Function &F, + const CycleInfo &CycleI, const TargetLibraryInfo *TLI, DominatorTree *DT, PostDominatorTree *PDT) { @@ -1445,7 +1232,7 @@ void BranchProbabilityInfo::calculate(const Function &F, const LoopInfo &LoopI, BlockNumberEpoch = F.getBlockNumberEpoch(); Probs.clear(); EdgeStarts.clear(); - BPIConstruction(*this).calculate(F, LoopI, TLI, DT, PDT); + BPIConstruction(*this).calculate(F, CycleI, TLI, DT, PDT); if (PrintBranchProb && (PrintBranchProbFuncName.empty() || F.getName() == PrintBranchProbFuncName)) { @@ -1459,7 +1246,7 @@ void BranchProbabilityInfoWrapperPass::getAnalysisUsage( // asserts that DT is also present so if we don't make sure that we have DT // here, that assert will trigger. AU.addRequired(); - AU.addRequired(); + AU.addRequired(); AU.addRequired(); AU.addRequired(); AU.addRequired(); @@ -1467,13 +1254,13 @@ void BranchProbabilityInfoWrapperPass::getAnalysisUsage( } bool BranchProbabilityInfoWrapperPass::runOnFunction(Function &F) { - const LoopInfo &LI = getAnalysis().getLoopInfo(); + const CycleInfo &CI = getAnalysis().getResult(); const TargetLibraryInfo &TLI = getAnalysis().getTLI(F); DominatorTree &DT = getAnalysis().getDomTree(); PostDominatorTree &PDT = getAnalysis().getPostDomTree(); - BPI.calculate(F, LI, &TLI, &DT, &PDT); + BPI.calculate(F, CI, &TLI, &DT, &PDT); return false; } @@ -1485,12 +1272,12 @@ void BranchProbabilityInfoWrapperPass::print(raw_ostream &OS, AnalysisKey BranchProbabilityAnalysis::Key; BranchProbabilityInfo BranchProbabilityAnalysis::run(Function &F, FunctionAnalysisManager &AM) { - auto &LI = AM.getResult(F); + auto &CI = AM.getResult(F); auto &TLI = AM.getResult(F); auto &DT = AM.getResult(F); auto &PDT = AM.getResult(F); BranchProbabilityInfo BPI; - BPI.calculate(F, LI, &TLI, &DT, &PDT); + BPI.calculate(F, CI, &TLI, &DT, &PDT); return BPI; } diff --git a/llvm/lib/Analysis/LazyBranchProbabilityInfo.cpp b/llvm/lib/Analysis/LazyBranchProbabilityInfo.cpp index 7297905a85f60..740747e7c086b 100644 --- a/llvm/lib/Analysis/LazyBranchProbabilityInfo.cpp +++ b/llvm/lib/Analysis/LazyBranchProbabilityInfo.cpp @@ -14,7 +14,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/LazyBranchProbabilityInfo.h" -#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/CycleAnalysis.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/InitializePasses.h" @@ -25,7 +25,7 @@ using namespace llvm; INITIALIZE_PASS_BEGIN(LazyBranchProbabilityInfoPass, DEBUG_TYPE, "Lazy Branch Probability Analysis", true, true) -INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(CycleInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) INITIALIZE_PASS_END(LazyBranchProbabilityInfoPass, DEBUG_TYPE, "Lazy Branch Probability Analysis", true, true) @@ -45,7 +45,7 @@ void LazyBranchProbabilityInfoPass::getAnalysisUsage(AnalysisUsage &AU) const { // asserts that DT is also present so if we don't make sure that we have DT // here, that assert will trigger. AU.addRequiredTransitive(); - AU.addRequiredTransitive(); + AU.addRequiredTransitive(); AU.addRequiredTransitive(); AU.setPreservesAll(); } @@ -53,21 +53,21 @@ void LazyBranchProbabilityInfoPass::getAnalysisUsage(AnalysisUsage &AU) const { void LazyBranchProbabilityInfoPass::releaseMemory() { LBPI.reset(); } bool LazyBranchProbabilityInfoPass::runOnFunction(Function &F) { - LoopInfo &LI = getAnalysis().getLoopInfo(); + CycleInfo &CI = getAnalysis().getResult(); TargetLibraryInfo &TLI = getAnalysis().getTLI(F); - LBPI = std::make_unique(&F, &LI, &TLI); + LBPI = std::make_unique(&F, &CI, &TLI); return false; } void LazyBranchProbabilityInfoPass::getLazyBPIAnalysisUsage(AnalysisUsage &AU) { AU.addRequiredTransitive(); - AU.addRequiredTransitive(); + AU.addRequiredTransitive(); AU.addRequiredTransitive(); } void llvm::initializeLazyBPIPassPass(PassRegistry &Registry) { INITIALIZE_PASS_DEPENDENCY(LazyBranchProbabilityInfoPass); - INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass); + INITIALIZE_PASS_DEPENDENCY(CycleInfoWrapperPass); INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass); } diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp index 33caa268b1c6e..ef16310d1f249 100644 --- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -33,6 +33,7 @@ #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/CycleInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalAlias.h" @@ -1084,7 +1085,9 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex( BFI = GetBFICallback(F); else if (F.hasProfileData()) { LoopInfo LI{DT}; - BranchProbabilityInfo BPI{F, LI}; + CycleInfo CI; + CI.compute(const_cast(F)); + BranchProbabilityInfo BPI{F, CI}; BFIPtr = std::make_unique(F, BPI, LI); BFI = BFIPtr.get(); } diff --git a/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp b/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp index fbb5971e10750..520029349f487 100644 --- a/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp +++ b/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp @@ -16,6 +16,7 @@ #include "llvm/Analysis/LazyBlockFrequencyInfo.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/ProfileSummaryInfo.h" +#include "llvm/IR/CycleInfo.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/LLVMContext.h" @@ -34,11 +35,13 @@ OptimizationRemarkEmitter::OptimizationRemarkEmitter(const Function *F) DT.recalculate(*const_cast(F)); // Generate LoopInfo from it. + CycleInfo CI; + CI.compute(*const_cast(F)); LoopInfo LI; LI.analyze(DT); // Then compute BranchProbabilityInfo. - BranchProbabilityInfo BPI(*F, LI, nullptr, &DT, nullptr); + BranchProbabilityInfo BPI(*F, CI, nullptr, &DT, nullptr); // Finally compute BFI. OwnedBFI = std::make_unique(*F, BPI, LI); diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index 9d3f18c8fce63..4cc93d21da5c0 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -50,6 +50,7 @@ #include "llvm/IR/CFG.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/CycleInfo.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DerivedTypes.h" @@ -853,8 +854,10 @@ void CodeGenPrepare::removeAllAssertingVHReferences(Value *V) { // Verify BFI has been updated correctly by recomputing BFI and comparing them. [[maybe_unused]] void CodeGenPrepare::verifyBFIUpdates(Function &F) { DominatorTree NewDT(F); + CycleInfo NewCI; + NewCI.compute(F); LoopInfo NewLI(NewDT); - BranchProbabilityInfo NewBPI(F, NewLI, TLInfo); + BranchProbabilityInfo NewBPI(F, NewCI, TLInfo); BlockFrequencyInfo NewBFI(F, NewBPI, NewLI); NewBFI.verifyMatch(*BFI); } diff --git a/llvm/lib/CodeGen/MachineFunctionPass.cpp b/llvm/lib/CodeGen/MachineFunctionPass.cpp index f69d1c0997ecc..67ae3613263dc 100644 --- a/llvm/lib/CodeGen/MachineFunctionPass.cpp +++ b/llvm/lib/CodeGen/MachineFunctionPass.cpp @@ -13,6 +13,7 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/Analysis/BasicAliasAnalysis.h" #include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/Analysis/CycleAnalysis.h" #include "llvm/Analysis/DominanceFrontier.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/IVUsers.h" @@ -152,6 +153,7 @@ void MachineFunctionPass::getAnalysisUsage(AnalysisUsage &AU) const { AU.addPreserved(); AU.addPreserved(); AU.addPreserved(); + AU.addPreserved(); AU.addPreserved(); AU.addPreserved(); AU.addPreserved(); diff --git a/llvm/lib/Transforms/IPO/PartialInlining.cpp b/llvm/lib/Transforms/IPO/PartialInlining.cpp index b95ab41993d19..e01bb9f78fca3 100644 --- a/llvm/lib/Transforms/IPO/PartialInlining.cpp +++ b/llvm/lib/Transforms/IPO/PartialInlining.cpp @@ -29,6 +29,7 @@ #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" +#include "llvm/IR/CycleInfo.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Dominators.h" @@ -348,8 +349,10 @@ PartialInlinerImpl::computeOutliningColdRegionsInfo( BasicBlock *EntryBlock = &F.front(); DominatorTree DT(F); + CycleInfo CI; + CI.compute(F); LoopInfo LI(DT); - BranchProbabilityInfo BPI(F, LI); + BranchProbabilityInfo BPI(F, CI); std::unique_ptr ScopedBFI; BlockFrequencyInfo *BFI; if (!GetBFI) { @@ -903,8 +906,10 @@ void PartialInlinerImpl::computeCallsiteToProfCountMap( // For the old pass manager: if (!GetBFI) { DominatorTree DT(*Caller); + CycleInfo CI; + CI.compute(*Caller); LoopInfo LI(DT); - BranchProbabilityInfo BPI(*Caller, LI); + BranchProbabilityInfo BPI(*Caller, CI); TempBFI.reset(new BlockFrequencyInfo(*Caller, BPI, LI)); CurrentCallerBFI = TempBFI.get(); } else { @@ -1088,8 +1093,10 @@ bool PartialInlinerImpl::FunctionCloner::doMultiRegionFunctionOutlining() { DT.recalculate(*ClonedFunc); // Manually calculate a BlockFrequencyInfo and BranchProbabilityInfo. + CycleInfo CI; + CI.compute(*ClonedFunc); LoopInfo LI(DT); - BranchProbabilityInfo BPI(*ClonedFunc, LI); + BranchProbabilityInfo BPI(*ClonedFunc, CI); ClonedFuncBFI.reset(new BlockFrequencyInfo(*ClonedFunc, BPI, LI)); // Cache and recycle the CodeExtractor analysis to avoid O(n^2) compile-time. @@ -1163,8 +1170,10 @@ PartialInlinerImpl::FunctionCloner::doSingleRegionFunctionOutlining() { DT.recalculate(*ClonedFunc); // Manually calculate a BlockFrequencyInfo and BranchProbabilityInfo. + CycleInfo CI; + CI.compute(*ClonedFunc); LoopInfo LI(DT); - BranchProbabilityInfo BPI(*ClonedFunc, LI); + BranchProbabilityInfo BPI(*ClonedFunc, CI); ClonedFuncBFI.reset(new BlockFrequencyInfo(*ClonedFunc, BPI, LI)); // Gather up the blocks that we're going to extract. diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index 5e58b5c95f776..ef97dac7a5f0a 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -29,6 +29,7 @@ #include "llvm/IR/CFG.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/CycleInfo.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/DiagnosticInfo.h" @@ -973,13 +974,15 @@ void InstrLowerer::promoteCounterLoadStores(Function *F) { return; DominatorTree DT(*F); + CycleInfo CI; + CI.compute(*F); LoopInfo LI(DT); DenseMap> LoopPromotionCandidates; std::unique_ptr BFI; if (Options.UseBFIInPromotion) { std::unique_ptr BPI; - BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F))); + BPI.reset(new BranchProbabilityInfo(*F, CI, &GetTLI(*F))); BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI)); } diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index f6fcb41636000..47f98d98a50e9 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -72,6 +72,7 @@ #include "llvm/IR/Comdat.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/CycleInfo.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/EHPersonalities.h" @@ -1582,8 +1583,10 @@ void PGOUseFunc::populateCoverage() { unsigned NumCorruptCoverage = 0; DominatorTree DT(F); + CycleInfo CI; + CI.compute(F); LoopInfo LI(DT); - BranchProbabilityInfo BPI(F, LI); + BranchProbabilityInfo BPI(F, CI); BlockFrequencyInfo BFI(F, BPI, LI); auto IsBlockDead = [&](const BasicBlock &BB) -> std::optional { if (auto C = BFI.getBlockProfileCount(&BB)) @@ -2340,8 +2343,10 @@ static bool annotateAllFunctions( (ViewBlockFreqFuncName.empty() || F.getName() == ViewBlockFreqFuncName)) { LoopInfo LI{DominatorTree(F)}; + CycleInfo CI; + CI.compute(F); std::unique_ptr NewBPI = - std::make_unique(F, LI); + std::make_unique(F, CI); std::unique_ptr NewBFI = std::make_unique(F, *NewBPI, LI); if (PGOViewCounts == PGOVCT_Graph) @@ -2366,8 +2371,10 @@ static bool annotateAllFunctions( } if (PGOVerifyBFI || PGOVerifyHotBFI || PGOFixEntryCount) { + CycleInfo CI; + CI.compute(F); LoopInfo LI{DominatorTree(F)}; - BranchProbabilityInfo NBPI(F, LI); + BranchProbabilityInfo NBPI(F, CI); // Fix func entry count. if (PGOFixEntryCount) diff --git a/llvm/test/Analysis/BranchProbabilityInfo/loop.ll b/llvm/test/Analysis/BranchProbabilityInfo/loop.ll index ffac1cd466641..278f57a509c4f 100644 --- a/llvm/test/Analysis/BranchProbabilityInfo/loop.ll +++ b/llvm/test/Analysis/BranchProbabilityInfo/loop.ll @@ -6,6 +6,8 @@ declare void @g2() declare void @g3() declare void @g4() declare i32 @g5() +declare void @cold() cold +declare i1 @opaque() define void @test1(i32 %a, i32 %b) { entry: @@ -519,7 +521,6 @@ exit: } declare i32 @InvokeCall() -declare void @cold() cold ; If loop has single exit and it leads to 'cold' block then edge leading to loop enter ; should be considered 'cold' as well. @@ -721,5 +722,79 @@ exit: ret void } +define void @test19() { +; CHECK-LABEL: Printing analysis {{.*}} for function 'test19' +entry: + call void @cold() + %c0 = call i1 @opaque() + br i1 %c0, label %outer.header, label %exit +; CHECK: edge %entry -> %outer.header probability is 0x40000000 / 0x80000000 = 50.00% +; CHECK: edge %entry -> %exit probability is 0x40000000 / 0x80000000 = 50.00% + +outer.header: + call void @cold() + br label %outer.body +; CHECK: edge %outer.header -> %outer.body probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] + +outer.body: + %c2 = call i1 @opaque() + br i1 %c2, label %inner.header, label %bypass +; CHECK: edge %outer.body -> %inner.header probability is 0x40000000 / 0x80000000 = 50.00% +; CHECK: edge %outer.body -> %bypass probability is 0x40000000 / 0x80000000 = 50.00% + +inner.header: + call void @cold() + %c3 = call i1 @opaque() + br i1 %c3, label %outer.latch, label %inner.latch +; CHECK: edge %inner.header -> %outer.latch probability is 0x0041edfd / 0x80000000 = 0.20% +; CHECK: edge %inner.header -> %inner.latch probability is 0x7fbe1203 / 0x80000000 = 99.80% [HOT edge] + +outer.latch: + br label %outer.header +; CHECK: edge %outer.latch -> %outer.header probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] + +bypass: + br label %outer.latch +; CHECK: edge %bypass -> %outer.latch probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] + +exit: + ret void + +inner.latch: + %c7 = call i1 @opaque() + br i1 %c7, label %inner.header, label %outer.latch +; CHECK: edge %inner.latch -> %inner.header probability is 0x7c0003e0 / 0x80000000 = 96.88% [HOT edge] +; CHECK: edge %inner.latch -> %outer.latch probability is 0x03fffc20 / 0x80000000 = 3.12% +} + +; Check that a cold irreducible loop propagates its estimated weight through +; the blocks entering the SCC. +define void @test20(i1 %arg, i1 %arg2, i1 %arg3, i1 %arg4) { +; CHECK-LABEL: Printing analysis {{.*}} for function 'test20' +entry: + br i1 %arg, label %dispatch, label %exit +; CHECK: edge %entry -> %dispatch probability is 0x078780e3 / 0x80000000 = 5.88% +; CHECK: edge %entry -> %exit probability is 0x78787f1d / 0x80000000 = 94.12% [HOT edge] + +dispatch: + br i1 %arg2, label %entry1, label %entry2 + +entry1: + br label %loop1 + +entry2: + br label %loop2 + +loop1: + br i1 %arg3, label %loop2, label %cold +loop2: + br i1 %arg4, label %loop1, label %cold +cold: + call void @cold() + br label %exit + +exit: + ret void +} diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll b/llvm/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll index 19cd2b5a453f2..33e8b854d86b1 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll @@ -73,6 +73,7 @@ ; VERIFY-NEXT: Verify generated machine code ; ENABLED-NEXT: Analysis for ComputingKnownBits ; ENABLED-O1-NEXT: Lazy Branch Probability Analysis +; ENABLED-O1-NEXT: Natural Loop Information ; ENABLED-O1-NEXT: Lazy Block Frequency Analysis ; ENABLED-NEXT: InstructionSelect ; ENABLED-O1-NEXT: AArch64 Post Select Optimizer diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll index 213559d8e24e8..6ffdb0f1305b9 100644 --- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll +++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll @@ -36,6 +36,7 @@ ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Canonicalize natural loops +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Optimization Remark Emitter @@ -52,9 +53,10 @@ ; CHECK-NEXT: Lower Garbage Collection Instructions ; CHECK-NEXT: Shadow Stack GC Lowering ; CHECK-NEXT: Remove unreachable blocks from the CFG -; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis +; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Block Frequency Analysis ; CHECK-NEXT: Constant Hoisting ; CHECK-NEXT: Replace intrinsics with calls to vector library @@ -66,6 +68,7 @@ ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics ; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis ; CHECK-NEXT: Block Frequency Analysis @@ -95,6 +98,7 @@ ; CHECK-NEXT: Interleaved Access Pass ; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Type Promotion +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis ; CHECK-NEXT: Block Frequency Analysis @@ -112,7 +116,7 @@ ; CHECK-NEXT: Module Verifier ; CHECK-NEXT: Analysis containing CSE Info ; CHECK-NEXT: Dominator Tree Construction -; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis ; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) @@ -127,6 +131,7 @@ ; CHECK-NEXT: RegBankSelect ; CHECK-NEXT: Analysis for ComputingKnownBits ; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: InstructionSelect ; CHECK-NEXT: ResetMachineFunction diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll index d10df2ed1d11f..f8101eba8106b 100644 --- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll +++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll @@ -231,9 +231,10 @@ ; GCN-O1-NEXT: Induction Variable Users ; GCN-O1-NEXT: Loop Strength Reduction ; GCN-O1-NEXT: Remove unreachable blocks from the CFG -; GCN-O1-NEXT: Natural Loop Information +; GCN-O1-NEXT: Cycle Info Analysis ; GCN-O1-NEXT: Post-Dominator Tree Construction ; GCN-O1-NEXT: Branch Probability Analysis +; GCN-O1-NEXT: Natural Loop Information ; GCN-O1-NEXT: Block Frequency Analysis ; GCN-O1-NEXT: Constant Hoisting ; GCN-O1-NEXT: Replace intrinsics with calls to vector library @@ -249,6 +250,7 @@ ; GCN-O1-NEXT: Dominator Tree Construction ; GCN-O1-NEXT: AMDGPU Lower Kernel Arguments ; GCN-O1-NEXT: Natural Loop Information +; GCN-O1-NEXT: Cycle Info Analysis ; GCN-O1-NEXT: Post-Dominator Tree Construction ; GCN-O1-NEXT: Branch Probability Analysis ; GCN-O1-NEXT: Block Frequency Analysis @@ -314,11 +316,11 @@ ; GCN-O1-NEXT: Uniformity Analysis ; GCN-O1-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O1-NEXT: Function Alias Analysis Results -; GCN-O1-NEXT: Natural Loop Information ; GCN-O1-NEXT: Post-Dominator Tree Construction ; GCN-O1-NEXT: Branch Probability Analysis ; GCN-O1-NEXT: Assignment Tracking Analysis ; GCN-O1-NEXT: Lazy Branch Probability Analysis +; GCN-O1-NEXT: Natural Loop Information ; GCN-O1-NEXT: Lazy Block Frequency Analysis ; GCN-O1-NEXT: AMDGPU DAG->DAG Pattern Instruction Selection ; GCN-O1-NEXT: MachineDominator Tree Construction @@ -516,6 +518,7 @@ ; GCN-O1-OPTS-NEXT: Natural Loop Information ; GCN-O1-OPTS-NEXT: AMDGPU Promote Alloca ; GCN-O1-OPTS-NEXT: Canonicalize natural loops +; GCN-O1-OPTS-NEXT: Cycle Info Analysis ; GCN-O1-OPTS-NEXT: Lazy Branch Probability Analysis ; GCN-O1-OPTS-NEXT: Lazy Block Frequency Analysis ; GCN-O1-OPTS-NEXT: Optimization Remark Emitter @@ -539,9 +542,10 @@ ; GCN-O1-OPTS-NEXT: Induction Variable Users ; GCN-O1-OPTS-NEXT: Loop Strength Reduction ; GCN-O1-OPTS-NEXT: Remove unreachable blocks from the CFG -; GCN-O1-OPTS-NEXT: Natural Loop Information +; GCN-O1-OPTS-NEXT: Cycle Info Analysis ; GCN-O1-OPTS-NEXT: Post-Dominator Tree Construction ; GCN-O1-OPTS-NEXT: Branch Probability Analysis +; GCN-O1-OPTS-NEXT: Natural Loop Information ; GCN-O1-OPTS-NEXT: Block Frequency Analysis ; GCN-O1-OPTS-NEXT: Constant Hoisting ; GCN-O1-OPTS-NEXT: Replace intrinsics with calls to vector library @@ -558,6 +562,7 @@ ; GCN-O1-OPTS-NEXT: Dominator Tree Construction ; GCN-O1-OPTS-NEXT: AMDGPU Lower Kernel Arguments ; GCN-O1-OPTS-NEXT: Natural Loop Information +; GCN-O1-OPTS-NEXT: Cycle Info Analysis ; GCN-O1-OPTS-NEXT: Post-Dominator Tree Construction ; GCN-O1-OPTS-NEXT: Branch Probability Analysis ; GCN-O1-OPTS-NEXT: Block Frequency Analysis @@ -629,11 +634,11 @@ ; GCN-O1-OPTS-NEXT: Uniformity Analysis ; GCN-O1-OPTS-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O1-OPTS-NEXT: Function Alias Analysis Results -; GCN-O1-OPTS-NEXT: Natural Loop Information ; GCN-O1-OPTS-NEXT: Post-Dominator Tree Construction ; GCN-O1-OPTS-NEXT: Branch Probability Analysis ; GCN-O1-OPTS-NEXT: Assignment Tracking Analysis ; GCN-O1-OPTS-NEXT: Lazy Branch Probability Analysis +; GCN-O1-OPTS-NEXT: Natural Loop Information ; GCN-O1-OPTS-NEXT: Lazy Block Frequency Analysis ; GCN-O1-OPTS-NEXT: AMDGPU DAG->DAG Pattern Instruction Selection ; GCN-O1-OPTS-NEXT: MachineDominator Tree Construction @@ -855,6 +860,7 @@ ; GCN-O2-NEXT: LCSSA Verifier ; GCN-O2-NEXT: Loop-Closed SSA Form Pass ; GCN-O2-NEXT: Scalar Evolution Analysis +; GCN-O2-NEXT: Cycle Info Analysis ; GCN-O2-NEXT: Lazy Branch Probability Analysis ; GCN-O2-NEXT: Lazy Block Frequency Analysis ; GCN-O2-NEXT: Loop Pass Manager @@ -864,9 +870,10 @@ ; GCN-O2-NEXT: Induction Variable Users ; GCN-O2-NEXT: Loop Strength Reduction ; GCN-O2-NEXT: Remove unreachable blocks from the CFG -; GCN-O2-NEXT: Natural Loop Information +; GCN-O2-NEXT: Cycle Info Analysis ; GCN-O2-NEXT: Post-Dominator Tree Construction ; GCN-O2-NEXT: Branch Probability Analysis +; GCN-O2-NEXT: Natural Loop Information ; GCN-O2-NEXT: Block Frequency Analysis ; GCN-O2-NEXT: Constant Hoisting ; GCN-O2-NEXT: Replace intrinsics with calls to vector library @@ -883,6 +890,7 @@ ; GCN-O2-NEXT: Dominator Tree Construction ; GCN-O2-NEXT: AMDGPU Lower Kernel Arguments ; GCN-O2-NEXT: Natural Loop Information +; GCN-O2-NEXT: Cycle Info Analysis ; GCN-O2-NEXT: Post-Dominator Tree Construction ; GCN-O2-NEXT: Branch Probability Analysis ; GCN-O2-NEXT: Block Frequency Analysis @@ -955,11 +963,11 @@ ; GCN-O2-NEXT: Uniformity Analysis ; GCN-O2-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O2-NEXT: Function Alias Analysis Results -; GCN-O2-NEXT: Natural Loop Information ; GCN-O2-NEXT: Post-Dominator Tree Construction ; GCN-O2-NEXT: Branch Probability Analysis ; GCN-O2-NEXT: Assignment Tracking Analysis ; GCN-O2-NEXT: Lazy Branch Probability Analysis +; GCN-O2-NEXT: Natural Loop Information ; GCN-O2-NEXT: Lazy Block Frequency Analysis ; GCN-O2-NEXT: AMDGPU DAG->DAG Pattern Instruction Selection ; GCN-O2-NEXT: MachineDominator Tree Construction @@ -1171,6 +1179,7 @@ ; GCN-O3-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O3-NEXT: Function Alias Analysis Results ; GCN-O3-NEXT: Memory Dependence Analysis +; GCN-O3-NEXT: Cycle Info Analysis ; GCN-O3-NEXT: Lazy Branch Probability Analysis ; GCN-O3-NEXT: Lazy Block Frequency Analysis ; GCN-O3-NEXT: Optimization Remark Emitter @@ -1188,6 +1197,7 @@ ; GCN-O3-NEXT: LCSSA Verifier ; GCN-O3-NEXT: Loop-Closed SSA Form Pass ; GCN-O3-NEXT: Scalar Evolution Analysis +; GCN-O3-NEXT: Cycle Info Analysis ; GCN-O3-NEXT: Lazy Branch Probability Analysis ; GCN-O3-NEXT: Lazy Block Frequency Analysis ; GCN-O3-NEXT: Loop Pass Manager @@ -1197,9 +1207,10 @@ ; GCN-O3-NEXT: Induction Variable Users ; GCN-O3-NEXT: Loop Strength Reduction ; GCN-O3-NEXT: Remove unreachable blocks from the CFG -; GCN-O3-NEXT: Natural Loop Information +; GCN-O3-NEXT: Cycle Info Analysis ; GCN-O3-NEXT: Post-Dominator Tree Construction ; GCN-O3-NEXT: Branch Probability Analysis +; GCN-O3-NEXT: Natural Loop Information ; GCN-O3-NEXT: Block Frequency Analysis ; GCN-O3-NEXT: Constant Hoisting ; GCN-O3-NEXT: Replace intrinsics with calls to vector library @@ -1214,6 +1225,7 @@ ; GCN-O3-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O3-NEXT: Function Alias Analysis Results ; GCN-O3-NEXT: Memory Dependence Analysis +; GCN-O3-NEXT: Cycle Info Analysis ; GCN-O3-NEXT: Lazy Branch Probability Analysis ; GCN-O3-NEXT: Lazy Block Frequency Analysis ; GCN-O3-NEXT: Optimization Remark Emitter @@ -1223,6 +1235,7 @@ ; GCN-O3-NEXT: Dominator Tree Construction ; GCN-O3-NEXT: AMDGPU Lower Kernel Arguments ; GCN-O3-NEXT: Natural Loop Information +; GCN-O3-NEXT: Cycle Info Analysis ; GCN-O3-NEXT: Post-Dominator Tree Construction ; GCN-O3-NEXT: Branch Probability Analysis ; GCN-O3-NEXT: Block Frequency Analysis @@ -1295,11 +1308,11 @@ ; GCN-O3-NEXT: Uniformity Analysis ; GCN-O3-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O3-NEXT: Function Alias Analysis Results -; GCN-O3-NEXT: Natural Loop Information ; GCN-O3-NEXT: Post-Dominator Tree Construction ; GCN-O3-NEXT: Branch Probability Analysis ; GCN-O3-NEXT: Assignment Tracking Analysis ; GCN-O3-NEXT: Lazy Branch Probability Analysis +; GCN-O3-NEXT: Natural Loop Information ; GCN-O3-NEXT: Lazy Block Frequency Analysis ; GCN-O3-NEXT: AMDGPU DAG->DAG Pattern Instruction Selection ; GCN-O3-NEXT: MachineDominator Tree Construction diff --git a/llvm/test/CodeGen/ARM/O3-pipeline.ll b/llvm/test/CodeGen/ARM/O3-pipeline.ll index 9f4d70531a3f7..d57e8785a94c8 100644 --- a/llvm/test/CodeGen/ARM/O3-pipeline.ll +++ b/llvm/test/CodeGen/ARM/O3-pipeline.ll @@ -28,9 +28,10 @@ ; CHECK-NEXT: Lower Garbage Collection Instructions ; CHECK-NEXT: Shadow Stack GC Lowering ; CHECK-NEXT: Remove unreachable blocks from the CFG -; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis +; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Block Frequency Analysis ; CHECK-NEXT: Constant Hoisting ; CHECK-NEXT: Replace intrinsics with calls to vector library @@ -49,6 +50,7 @@ ; CHECK-NEXT: Complex Deinterleaving Pass ; CHECK-NEXT: Interleaved Access Pass ; CHECK-NEXT: Type Promotion +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis ; CHECK-NEXT: Block Frequency Analysis @@ -58,14 +60,15 @@ ; CHECK-NEXT: Merge internal globals ; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Optimization Remark Emitter ; CHECK-NEXT: Hardware Loop Insertion ; CHECK-NEXT: Loop Pass Manager ; CHECK-NEXT: Transform predicated vector loops to use MVE tail predication -; CHECK-NEXT: A No-Op Barrier Pass -; CHECK-NEXT: FunctionPass Manager +; CHECK-NEXT: A No-Op Barrier Pass +; CHECK-NEXT: FunctionPass Manager ; CHECK-NEXT: Prepare inline asm insts ; CHECK-NEXT: Safe Stack instrumentation pass ; CHECK-NEXT: Insert stack protectors @@ -73,11 +76,12 @@ ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) ; CHECK-NEXT: Function Alias Analysis Results -; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis ; CHECK-NEXT: Assignment Tracking Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: ARM Instruction Selection ; CHECK-NEXT: Finalize ISel and expand pseudo-instructions @@ -108,7 +112,7 @@ ; CHECK-NEXT: Modulo Software Pipelining ; CHECK-NEXT: MachineDominator Tree Construction ; CHECK-NEXT: Machine Natural Loop Construction -; CHECK-NEXT: MVE TailPred and VPT Optimisation Pass +; CHECK-NEXT: ARM MVE TailPred and VPT Optimisation Pass ; CHECK-NEXT: ARM MLA / MLS expansion pass ; CHECK-NEXT: MachineDominator Tree Construction ; CHECK-NEXT: ARM pre- register allocation load / store optimization pass diff --git a/llvm/test/CodeGen/LoongArch/annotate-tablejump.ll b/llvm/test/CodeGen/LoongArch/annotate-tablejump.ll index 2af80789086e6..f47b8fde2efdf 100644 --- a/llvm/test/CodeGen/LoongArch/annotate-tablejump.ll +++ b/llvm/test/CodeGen/LoongArch/annotate-tablejump.ll @@ -13,7 +13,7 @@ define void @switch_4_arms(i32 %in, ptr %out) nounwind { ; LA32-JT: # %bb.0: # %entry ; LA32-JT-NEXT: addi.w $a3, $a0, -1 ; LA32-JT-NEXT: ori $a2, $zero, 3 -; LA32-JT-NEXT: bltu $a2, $a3, .LBB0_7 +; LA32-JT-NEXT: bltu $a2, $a3, .LBB0_6 ; LA32-JT-NEXT: # %bb.1: # %entry ; LA32-JT-NEXT: pcalau12i $a4, %pc_hi20(.LJTI0_0) ; LA32-JT-NEXT: addi.w $a4, $a4, %pc_lo12(.LJTI0_0) @@ -23,27 +23,27 @@ define void @switch_4_arms(i32 %in, ptr %out) nounwind { ; LA32-JT-NEXT: jr $a3 ; LA32-JT-NEXT: .LBB0_2: # %bb1 ; LA32-JT-NEXT: ori $a3, $zero, 4 -; LA32-JT-NEXT: b .LBB0_6 +; LA32-JT-NEXT: b .LBB0_5 ; LA32-JT-NEXT: .LBB0_3: # %bb2 ; LA32-JT-NEXT: ori $a3, $zero, 3 -; LA32-JT-NEXT: b .LBB0_6 +; LA32-JT-NEXT: b .LBB0_5 ; LA32-JT-NEXT: .LBB0_4: # %bb3 ; LA32-JT-NEXT: ori $a3, $zero, 2 -; LA32-JT-NEXT: b .LBB0_6 -; LA32-JT-NEXT: .LBB0_5: # %bb4 -; LA32-JT-NEXT: ori $a3, $zero, 1 -; LA32-JT-NEXT: .LBB0_6: # %exit +; LA32-JT-NEXT: .LBB0_5: # %exit ; LA32-JT-NEXT: st.w $a3, $a1, 0 -; LA32-JT-NEXT: .LBB0_7: # %exit +; LA32-JT-NEXT: .LBB0_6: # %exit ; LA32-JT-NEXT: addi.w $a3, $a0, -5 ; LA32-JT-NEXT: bltu $a2, $a3, .LBB0_9 -; LA32-JT-NEXT: # %bb.8: # %exit +; LA32-JT-NEXT: # %bb.7: # %exit ; LA32-JT-NEXT: pcalau12i $a4, %pc_hi20(.LJTI0_1) ; LA32-JT-NEXT: addi.w $a4, $a4, %pc_lo12(.LJTI0_1) ; LA32-JT-NEXT: alsl.w $a3, $a3, $a4, 2 ; LA32-JT-NEXT: ld.w $a3, $a3, 0 ; LA32-JT-NEXT: .Ljrtb_1: ; LA32-JT-NEXT: jr $a3 +; LA32-JT-NEXT: .LBB0_8: # %bb4 +; LA32-JT-NEXT: ori $a3, $zero, 1 +; LA32-JT-NEXT: b .LBB0_5 ; LA32-JT-NEXT: .LBB0_9: # %exit2 ; LA32-JT-NEXT: ret ; @@ -52,7 +52,7 @@ define void @switch_4_arms(i32 %in, ptr %out) nounwind { ; LA64-JT-NEXT: addi.w $a0, $a0, 0 ; LA64-JT-NEXT: addi.d $a3, $a0, -1 ; LA64-JT-NEXT: ori $a2, $zero, 3 -; LA64-JT-NEXT: bltu $a2, $a3, .LBB0_7 +; LA64-JT-NEXT: bltu $a2, $a3, .LBB0_6 ; LA64-JT-NEXT: # %bb.1: # %entry ; LA64-JT-NEXT: slli.d $a3, $a3, 3 ; LA64-JT-NEXT: pcalau12i $a4, %pc_hi20(.LJTI0_0) @@ -62,27 +62,27 @@ define void @switch_4_arms(i32 %in, ptr %out) nounwind { ; LA64-JT-NEXT: jr $a3 ; LA64-JT-NEXT: .LBB0_2: # %bb1 ; LA64-JT-NEXT: ori $a3, $zero, 4 -; LA64-JT-NEXT: b .LBB0_6 +; LA64-JT-NEXT: b .LBB0_5 ; LA64-JT-NEXT: .LBB0_3: # %bb2 ; LA64-JT-NEXT: ori $a3, $zero, 3 -; LA64-JT-NEXT: b .LBB0_6 +; LA64-JT-NEXT: b .LBB0_5 ; LA64-JT-NEXT: .LBB0_4: # %bb3 ; LA64-JT-NEXT: ori $a3, $zero, 2 -; LA64-JT-NEXT: b .LBB0_6 -; LA64-JT-NEXT: .LBB0_5: # %bb4 -; LA64-JT-NEXT: ori $a3, $zero, 1 -; LA64-JT-NEXT: .LBB0_6: # %exit +; LA64-JT-NEXT: .LBB0_5: # %exit ; LA64-JT-NEXT: st.w $a3, $a1, 0 -; LA64-JT-NEXT: .LBB0_7: # %exit +; LA64-JT-NEXT: .LBB0_6: # %exit ; LA64-JT-NEXT: addi.d $a3, $a0, -5 ; LA64-JT-NEXT: bltu $a2, $a3, .LBB0_9 -; LA64-JT-NEXT: # %bb.8: # %exit +; LA64-JT-NEXT: # %bb.7: # %exit ; LA64-JT-NEXT: slli.d $a3, $a3, 3 ; LA64-JT-NEXT: pcalau12i $a4, %pc_hi20(.LJTI0_1) ; LA64-JT-NEXT: addi.d $a4, $a4, %pc_lo12(.LJTI0_1) ; LA64-JT-NEXT: ldx.d $a3, $a4, $a3 ; LA64-JT-NEXT: .Ljrtb_1: ; LA64-JT-NEXT: jr $a3 +; LA64-JT-NEXT: .LBB0_8: # %bb4 +; LA64-JT-NEXT: ori $a3, $zero, 1 +; LA64-JT-NEXT: b .LBB0_5 ; LA64-JT-NEXT: .LBB0_9: # %exit2 ; LA64-JT-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll index 2657a575aa8af..bda71c7faebf4 100644 --- a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll +++ b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll @@ -53,9 +53,10 @@ ; LAXX-NEXT: Lower Garbage Collection Instructions ; LAXX-NEXT: Shadow Stack GC Lowering ; LAXX-NEXT: Remove unreachable blocks from the CFG -; LAXX-NEXT: Natural Loop Information +; LAXX-NEXT: Cycle Info Analysis ; LAXX-NEXT: Post-Dominator Tree Construction ; LAXX-NEXT: Branch Probability Analysis +; LAXX-NEXT: Natural Loop Information ; LAXX-NEXT: Block Frequency Analysis ; LAXX-NEXT: Constant Hoisting ; LAXX-NEXT: Replace intrinsics with calls to vector library @@ -68,6 +69,7 @@ ; LAXX-NEXT: Expand reduction intrinsics ; LAXX-NEXT: Natural Loop Information ; LAXX-NEXT: Type Promotion +; LAXX-NEXT: Cycle Info Analysis ; LAXX-NEXT: Post-Dominator Tree Construction ; LAXX-NEXT: Branch Probability Analysis ; LAXX-NEXT: Block Frequency Analysis @@ -80,11 +82,12 @@ ; LAXX-NEXT: Module Verifier ; LAXX-NEXT: Basic Alias Analysis (stateless AA impl) ; LAXX-NEXT: Function Alias Analysis Results -; LAXX-NEXT: Natural Loop Information +; LAXX-NEXT: Cycle Info Analysis ; LAXX-NEXT: Post-Dominator Tree Construction ; LAXX-NEXT: Branch Probability Analysis ; LAXX-NEXT: Assignment Tracking Analysis ; LAXX-NEXT: Lazy Branch Probability Analysis +; LAXX-NEXT: Natural Loop Information ; LAXX-NEXT: Lazy Block Frequency Analysis ; LAXX-NEXT: LoongArch DAG->DAG Pattern Instruction Selection ; LAXX-NEXT: Finalize ISel and expand pseudo-instructions diff --git a/llvm/test/CodeGen/M68k/pipeline.ll b/llvm/test/CodeGen/M68k/pipeline.ll index 24329b2749fca..66eb993ade52d 100644 --- a/llvm/test/CodeGen/M68k/pipeline.ll +++ b/llvm/test/CodeGen/M68k/pipeline.ll @@ -22,9 +22,10 @@ ; CHECK-NEXT: Lower Garbage Collection Instructions ; CHECK-NEXT: Shadow Stack GC Lowering ; CHECK-NEXT: Remove unreachable blocks from the CFG -; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis +; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Block Frequency Analysis ; CHECK-NEXT: Constant Hoisting ; CHECK-NEXT: Replace intrinsics with calls to vector library @@ -36,6 +37,7 @@ ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics ; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis ; CHECK-NEXT: Block Frequency Analysis @@ -48,11 +50,12 @@ ; CHECK-NEXT: Module Verifier ; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) ; CHECK-NEXT: Function Alias Analysis Results -; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis ; CHECK-NEXT: Assignment Tracking Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: M68k DAG->DAG Pattern Instruction Selection ; CHECK-NEXT: M68k PIC Global Base Reg Initialization diff --git a/llvm/test/CodeGen/PowerPC/O3-pipeline.ll b/llvm/test/CodeGen/PowerPC/O3-pipeline.ll index 3901d122f4494..127c7872fd01d 100644 --- a/llvm/test/CodeGen/PowerPC/O3-pipeline.ll +++ b/llvm/test/CodeGen/PowerPC/O3-pipeline.ll @@ -42,6 +42,7 @@ ; CHECK-NEXT: LCSSA Verifier ; CHECK-NEXT: Loop-Closed SSA Form Pass ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Loop Pass Manager @@ -54,9 +55,10 @@ ; CHECK-NEXT: Lower Garbage Collection Instructions ; CHECK-NEXT: Shadow Stack GC Lowering ; CHECK-NEXT: Remove unreachable blocks from the CFG -; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis +; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Block Frequency Analysis ; CHECK-NEXT: Constant Hoisting ; CHECK-NEXT: Replace intrinsics with calls to vector library @@ -68,6 +70,7 @@ ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics ; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis ; CHECK-NEXT: Block Frequency Analysis @@ -79,6 +82,7 @@ ; CHECK-NEXT: Scalar Evolution Analysis ; CHECK-NEXT: Prepare loop for ppc preferred instruction forms ; CHECK-NEXT: Scalar Evolution Analysis +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Optimization Remark Emitter @@ -89,11 +93,12 @@ ; CHECK-NEXT: Module Verifier ; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) ; CHECK-NEXT: Function Alias Analysis Results -; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis ; CHECK-NEXT: Assignment Tracking Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: PowerPC DAG->DAG Pattern Instruction Selection ; CHECK-NEXT: MachineDominator Tree Construction diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/gisel-commandline-option.ll b/llvm/test/CodeGen/RISCV/GlobalISel/gisel-commandline-option.ll index 82c66c23b26ba..e5f993b7552d2 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/gisel-commandline-option.ll +++ b/llvm/test/CodeGen/RISCV/GlobalISel/gisel-commandline-option.ll @@ -26,6 +26,7 @@ ; ENABLED-NEXT: RegBankSelect ; ENABLED-NEXT: Analysis for ComputingKnownBits ; ENABLED-O1-NEXT: Lazy Branch Probability Analysis +; ENABLED-O1-NEXT: Natural Loop Information ; ENABLED-O1-NEXT: Lazy Block Frequency Analysis ; ENABLED-NEXT: InstructionSelect ; ENABLED-NEXT: ResetMachineFunction diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll index 12b0895e0f158..d5bb2797c974a 100644 --- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll +++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll @@ -32,10 +32,11 @@ ; CHECK-NEXT: FunctionPass Manager ; CHECK-NEXT: Expand IR instructions ; CHECK-NEXT: Expand Atomic instructions -; CHECK-NEXT: RISC-V Zacas ABI fix +; CHECK-NEXT: RISC-V Zacas ABI fix ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Canonicalize natural loops +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Optimization Remark Emitter @@ -56,9 +57,10 @@ ; CHECK-NEXT: Lower Garbage Collection Instructions ; CHECK-NEXT: Shadow Stack GC Lowering ; CHECK-NEXT: Remove unreachable blocks from the CFG -; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis +; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Block Frequency Analysis ; CHECK-NEXT: Constant Hoisting ; CHECK-NEXT: Replace intrinsics with calls to vector library @@ -70,6 +72,7 @@ ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics ; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis ; CHECK-NEXT: Block Frequency Analysis @@ -80,6 +83,7 @@ ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Type Promotion +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis ; CHECK-NEXT: Block Frequency Analysis @@ -97,11 +101,12 @@ ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) ; CHECK-NEXT: Function Alias Analysis Results -; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis ; CHECK-NEXT: Assignment Tracking Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: RISC-V DAG->DAG Pattern Instruction Selection ; CHECK-NEXT: Finalize ISel and expand pseudo-instructions @@ -118,9 +123,9 @@ ; CHECK-NEXT: Remove dead machine instructions ; CHECK-NEXT: MachineDominator Tree Construction ; CHECK-NEXT: Machine Natural Loop Construction -; CHECK-NEXT: Machine Trace Metrics -; CHECK-NEXT: Lazy Machine Block Frequency Analysis -; CHECK-NEXT: Machine InstCombiner +; CHECK-NEXT: Machine Trace Metrics +; CHECK-NEXT: Lazy Machine Block Frequency Analysis +; CHECK-NEXT: Machine InstCombiner ; CHECK-NEXT: Machine Block Frequency Analysis ; CHECK-NEXT: Early Machine Loop Invariant Code Motion ; CHECK-NEXT: MachineDominator Tree Construction diff --git a/llvm/test/CodeGen/SPIRV/llc-pipeline.ll b/llvm/test/CodeGen/SPIRV/llc-pipeline.ll index c9234576df30d..3318fe41d0760 100644 --- a/llvm/test/CodeGen/SPIRV/llc-pipeline.ll +++ b/llvm/test/CodeGen/SPIRV/llc-pipeline.ll @@ -64,8 +64,9 @@ ; SPIRV-O0-NEXT: SPIRV post legalizer ; SPIRV-O0-NEXT: Analysis for ComputingKnownBits ; SPIRV-O0-NEXT: Dominator Tree Construction -; SPIRV-O0-NEXT: Natural Loop Information +; SPIRV-O0-NEXT: Cycle Info Analysis ; SPIRV-O0-NEXT: Lazy Branch Probability Analysis +; SPIRV-O0-NEXT: Natural Loop Information ; SPIRV-O0-NEXT: Lazy Block Frequency Analysis ; SPIRV-O0-NEXT: InstructionSelect ; SPIRV-O0-NEXT: ResetMachineFunction @@ -125,9 +126,10 @@ ; SPIRV-Opt-NEXT: Lower Garbage Collection Instructions ; SPIRV-Opt-NEXT: Shadow Stack GC Lowering ; SPIRV-Opt-NEXT: Remove unreachable blocks from the CFG -; SPIRV-Opt-NEXT: Natural Loop Information +; SPIRV-Opt-NEXT: Cycle Info Analysis ; SPIRV-Opt-NEXT: Post-Dominator Tree Construction ; SPIRV-Opt-NEXT: Branch Probability Analysis +; SPIRV-Opt-NEXT: Natural Loop Information ; SPIRV-Opt-NEXT: Block Frequency Analysis ; SPIRV-Opt-NEXT: Constant Hoisting ; SPIRV-Opt-NEXT: Replace intrinsics with calls to vector library @@ -147,6 +149,7 @@ ; SPIRV-Opt-NEXT: FunctionPass Manager ; SPIRV-Opt-NEXT: Dominator Tree Construction ; SPIRV-Opt-NEXT: Natural Loop Information +; SPIRV-Opt-NEXT: Cycle Info Analysis ; SPIRV-Opt-NEXT: Post-Dominator Tree Construction ; SPIRV-Opt-NEXT: Branch Probability Analysis ; SPIRV-Opt-NEXT: Block Frequency Analysis @@ -169,7 +172,7 @@ ; SPIRV-Opt-NEXT: Insert stack protectors ; SPIRV-Opt-NEXT: Analysis containing CSE Info ; SPIRV-Opt-NEXT: Dominator Tree Construction -; SPIRV-Opt-NEXT: Natural Loop Information +; SPIRV-Opt-NEXT: Cycle Info Analysis ; SPIRV-Opt-NEXT: Post-Dominator Tree Construction ; SPIRV-Opt-NEXT: Branch Probability Analysis ; SPIRV-Opt-NEXT: Basic Alias Analysis (stateless AA impl) @@ -184,6 +187,7 @@ ; SPIRV-Opt-NEXT: SPIRV post legalizer ; SPIRV-Opt-NEXT: Analysis for ComputingKnownBits ; SPIRV-Opt-NEXT: Lazy Branch Probability Analysis +; SPIRV-Opt-NEXT: Natural Loop Information ; SPIRV-Opt-NEXT: Lazy Block Frequency Analysis ; SPIRV-Opt-NEXT: InstructionSelect ; SPIRV-Opt-NEXT: ResetMachineFunction diff --git a/llvm/test/CodeGen/WebAssembly/GlobalISel/gisel-commandline-option.ll b/llvm/test/CodeGen/WebAssembly/GlobalISel/gisel-commandline-option.ll index bff1f6912d48b..b1ed82a7fea8e 100644 --- a/llvm/test/CodeGen/WebAssembly/GlobalISel/gisel-commandline-option.ll +++ b/llvm/test/CodeGen/WebAssembly/GlobalISel/gisel-commandline-option.ll @@ -26,8 +26,9 @@ ; ENABLED-NEXT: RegBankSelect ; ENABLED-NEXT: Analysis for ComputingKnownBits ; ENABLED-O1-NEXT: Dominator Tree Construction -; ENABLED-O1-NEXT: Natural Loop Information +; ENABLED-O1-NEXT: Cycle Info Analysis ; ENABLED-O1-NEXT: Lazy Branch Probability Analysis +; ENABLED-O1-NEXT: Natural Loop Information ; ENABLED-O1-NEXT: Lazy Block Frequency Analysis ; ENABLED-NEXT: InstructionSelect ; NOFALLBACK-NEXT: WebAssembly Argument Move diff --git a/llvm/test/CodeGen/X86/opt-pipeline.ll b/llvm/test/CodeGen/X86/opt-pipeline.ll index 24390f2d852d3..ad4d0bf7fe0e4 100644 --- a/llvm/test/CodeGen/X86/opt-pipeline.ll +++ b/llvm/test/CodeGen/X86/opt-pipeline.ll @@ -51,9 +51,10 @@ ; CHECK-NEXT: Lower Garbage Collection Instructions ; CHECK-NEXT: Shadow Stack GC Lowering ; CHECK-NEXT: Remove unreachable blocks from the CFG -; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis +; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Block Frequency Analysis ; CHECK-NEXT: Constant Hoisting ; CHECK-NEXT: Replace intrinsics with calls to vector library @@ -68,6 +69,7 @@ ; CHECK-NEXT: X86 Partial Reduction ; CHECK-NEXT: Expand indirectbr instructions ; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis ; CHECK-NEXT: Block Frequency Analysis @@ -80,11 +82,12 @@ ; CHECK-NEXT: Module Verifier ; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) ; CHECK-NEXT: Function Alias Analysis Results -; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Cycle Info Analysis ; CHECK-NEXT: Post-Dominator Tree Construction ; CHECK-NEXT: Branch Probability Analysis ; CHECK-NEXT: Assignment Tracking Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis +; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: X86 DAG->DAG Instruction Selection ; CHECK-NEXT: MachineDominator Tree Construction diff --git a/llvm/test/CodeGen/X86/pr38795.ll b/llvm/test/CodeGen/X86/pr38795.ll index 6a0c13526ac18..3c23fdafa97e3 100644 --- a/llvm/test/CodeGen/X86/pr38795.ll +++ b/llvm/test/CodeGen/X86/pr38795.ll @@ -25,7 +25,7 @@ define dso_local void @fn() { ; CHECK-NEXT: xorl %ebx, %ebx ; CHECK-NEXT: # implicit-def: $ecx ; CHECK-NEXT: # implicit-def: $edi -; CHECK-NEXT: # implicit-def: $dh +; CHECK-NEXT: # implicit-def: $dl ; CHECK-NEXT: # implicit-def: $al ; CHECK-NEXT: # kill: killed $al ; CHECK-NEXT: # implicit-def: $ebp @@ -33,7 +33,8 @@ define dso_local void @fn() { ; CHECK-NEXT: .p2align 4 ; CHECK-NEXT: .LBB0_15: # %for.inc ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 -; CHECK-NEXT: movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill +; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload +; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill ; CHECK-NEXT: .LBB0_1: # %for.cond ; CHECK-NEXT: # =>This Loop Header: Depth=1 ; CHECK-NEXT: # Child Loop BB0_19 Depth 2 @@ -41,46 +42,73 @@ define dso_local void @fn() { ; CHECK-NEXT: jne .LBB0_3 ; CHECK-NEXT: # %bb.2: # %if.then ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 -; CHECK-NEXT: movb %dh, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill +; CHECK-NEXT: movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill ; CHECK-NEXT: movl $.str, (%esp) ; CHECK-NEXT: calll printf ; CHECK-NEXT: # implicit-def: $eax -; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: jne .LBB0_10 -; CHECK-NEXT: jmp .LBB0_6 +; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload +; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill +; CHECK-NEXT: jmp .LBB0_5 ; CHECK-NEXT: .p2align 4 ; CHECK-NEXT: .LBB0_3: # %if.end ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: movl %ecx, %eax ; CHECK-NEXT: cltd ; CHECK-NEXT: idivl a -; CHECK-NEXT: movl %ecx, %edx +; CHECK-NEXT: movl %eax, %esi +; CHECK-NEXT: movl %ecx, %eax ; CHECK-NEXT: movl $0, h -; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %dh # 1-byte Reload -; CHECK-NEXT: cmpb $8, %dh +; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload +; CHECK-NEXT: cmpb $8, %dl ; CHECK-NEXT: jg .LBB0_7 ; CHECK-NEXT: # %bb.4: # %if.then13 ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 -; CHECK-NEXT: movl %eax, %esi +; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill ; CHECK-NEXT: movl $.str, (%esp) -; CHECK-NEXT: movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill ; CHECK-NEXT: calll printf -; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %dh # 1-byte Reload -; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %dl # 1-byte Reload +; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload ; CHECK-NEXT: testb %bl, %bl ; CHECK-NEXT: movl %esi, %ecx ; CHECK-NEXT: # implicit-def: $eax -; CHECK-NEXT: movb %dh, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill +; CHECK-NEXT: movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill ; CHECK-NEXT: jne .LBB0_15 +; CHECK-NEXT: jmp .LBB0_5 +; CHECK-NEXT: .p2align 4 +; CHECK-NEXT: .LBB0_7: # %if.end21 +; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 +; CHECK-NEXT: # implicit-def: $ebp +; CHECK-NEXT: jmp .LBB0_8 ; CHECK-NEXT: .p2align 4 -; CHECK-NEXT: # %bb.5: # %for.cond35 +; CHECK-NEXT: .LBB0_5: # %for.cond35 ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: testl %edi, %edi ; CHECK-NEXT: je .LBB0_6 -; CHECK-NEXT: .LBB0_10: # %af +; CHECK-NEXT: # %bb.10: # %af ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: testb %bl, %bl +; CHECK-NEXT: je .LBB0_16 +; CHECK-NEXT: .LBB0_11: # in Loop: Header=BB0_1 Depth=1 +; CHECK-NEXT: # implicit-def: $edi +; CHECK-NEXT: # implicit-def: $cl +; CHECK-NEXT: # kill: killed $cl +; CHECK-NEXT: # implicit-def: $cl +; CHECK-NEXT: # kill: killed $cl +; CHECK-NEXT: # implicit-def: $ebp +; CHECK-NEXT: jmp .LBB0_5 +; CHECK-NEXT: .p2align 4 +; CHECK-NEXT: .LBB0_6: # in Loop: Header=BB0_1 Depth=1 +; CHECK-NEXT: xorl %edi, %edi +; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload +; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload +; CHECK-NEXT: jmp .LBB0_19 +; CHECK-NEXT: .p2align 4 +; CHECK-NEXT: .LBB0_8: # %ae +; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 +; CHECK-NEXT: testb %bl, %bl +; CHECK-NEXT: je .LBB0_12 +; CHECK-NEXT: # %bb.9: # in Loop: Header=BB0_1 Depth=1 +; CHECK-NEXT: # implicit-def: $eax +; CHECK-NEXT: testb %bl, %bl ; CHECK-NEXT: jne .LBB0_11 ; CHECK-NEXT: .LBB0_16: # %if.end39 ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 @@ -94,33 +122,22 @@ define dso_local void @fn() { ; CHECK-NEXT: calll printf ; CHECK-NEXT: .LBB0_18: # %for.end46 ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 -; CHECK-NEXT: # implicit-def: $dh ; CHECK-NEXT: # implicit-def: $dl +; CHECK-NEXT: # implicit-def: $al ; CHECK-NEXT: # implicit-def: $ebp -; CHECK-NEXT: jmp .LBB0_19 -; CHECK-NEXT: .p2align 4 -; CHECK-NEXT: .LBB0_7: # %if.end21 -; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 -; CHECK-NEXT: # implicit-def: $ebp -; CHECK-NEXT: jmp .LBB0_8 -; CHECK-NEXT: .p2align 4 -; CHECK-NEXT: .LBB0_6: # in Loop: Header=BB0_1 Depth=1 -; CHECK-NEXT: xorl %edi, %edi -; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %dh # 1-byte Reload ; CHECK-NEXT: .p2align 4 ; CHECK-NEXT: .LBB0_19: # %for.cond47 ; CHECK-NEXT: # Parent Loop BB0_1 Depth=1 ; CHECK-NEXT: # => This Inner Loop Header: Depth=2 ; CHECK-NEXT: testb %bl, %bl ; CHECK-NEXT: jne .LBB0_19 -; CHECK-NEXT: .LBB0_8: # %ae -; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 -; CHECK-NEXT: testb %bl, %bl -; CHECK-NEXT: jne .LBB0_9 -; CHECK-NEXT: # %bb.12: # %if.end26 +; CHECK-NEXT: jmp .LBB0_8 +; CHECK-NEXT: .p2align 4 +; CHECK-NEXT: .LBB0_12: # %if.end26 ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 +; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill ; CHECK-NEXT: xorl %ecx, %ecx -; CHECK-NEXT: testb %dh, %dh +; CHECK-NEXT: testb %dl, %dl ; CHECK-NEXT: je .LBB0_15 ; CHECK-NEXT: # %bb.13: # %if.end26 ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 @@ -131,20 +148,6 @@ define dso_local void @fn() { ; CHECK-NEXT: xorl %ecx, %ecx ; CHECK-NEXT: xorl %ebp, %ebp ; CHECK-NEXT: jmp .LBB0_15 -; CHECK-NEXT: .p2align 4 -; CHECK-NEXT: .LBB0_9: # in Loop: Header=BB0_1 Depth=1 -; CHECK-NEXT: # implicit-def: $eax -; CHECK-NEXT: testb %bl, %bl -; CHECK-NEXT: je .LBB0_16 -; CHECK-NEXT: .LBB0_11: # in Loop: Header=BB0_1 Depth=1 -; CHECK-NEXT: # implicit-def: $edi -; CHECK-NEXT: # implicit-def: $cl -; CHECK-NEXT: # kill: killed $cl -; CHECK-NEXT: # implicit-def: $dl -; CHECK-NEXT: # implicit-def: $ebp -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: jne .LBB0_10 -; CHECK-NEXT: jmp .LBB0_6 entry: br label %for.cond diff --git a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll index f098951e1403d..aa0b2864eaab8 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll @@ -42,8 +42,9 @@ ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: BlockFrequencyAnalysis on foo ; CHECK-O-NEXT: Running analysis: BranchProbabilityAnalysis on foo -; CHECK-O-NEXT: Running analysis: LoopAnalysis on foo +; CHECK-O-NEXT: Running analysis: CycleAnalysis on foo ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo +; CHECK-O-NEXT: Running analysis: LoopAnalysis on foo ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running pass: AlwaysInlinerPass ; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass @@ -120,7 +121,6 @@ ; CHECK-O-NEXT: Running pass: ADCEPass ; CHECK-O23-NEXT: Running pass: MemCpyOptPass ; CHECK-O23-NEXT: Running pass: DSEPass -; CHECK-O23-NEXT: Running analysis: CycleAnalysis ; CHECK-O23-NEXT: Running pass: MoveAutoInitPass on foo ; CHECK-O23-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23-NEXT: Running pass: LCSSAPass diff --git a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll index 702920e5a6dbf..e9d64c4a43163 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll @@ -48,8 +48,9 @@ ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: BlockFrequencyAnalysis on foo ; CHECK-O-NEXT: Running analysis: BranchProbabilityAnalysis on foo -; CHECK-O-NEXT: Running analysis: LoopAnalysis on foo +; CHECK-O-NEXT: Running analysis: CycleAnalysis on foo ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo +; CHECK-O-NEXT: Running analysis: LoopAnalysis on foo ; CHECK-O-NEXT: Running pass: SimplifyCFGPass on foo ; CHECK-O-NEXT: Running pass: PGOForceFunctionAttrsPass @@ -127,7 +128,6 @@ ; CHECK-O-NEXT: Running pass: ADCEPass ; CHECK-O23-NEXT: Running pass: MemCpyOptPass ; CHECK-O23-NEXT: Running pass: DSEPass -; CHECK-O23-NEXT: Running analysis: CycleAnalysis ; CHECK-O23-NEXT: Running pass: MoveAutoInitPass ; CHECK-O23-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23-NEXT: Running pass: LCSSAPass diff --git a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll index 2c2046b0d0934..b60a8af7be33e 100644 --- a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll @@ -67,9 +67,10 @@ ; CHECK-O-NEXT: Running pass: PGOInstrumentationUse ; CHECK-O-NEXT: Running analysis: ProfileSummaryAnalysis ; CHECK-O-NEXT: Running analysis: BranchProbabilityAnalysis on foo -; CHECK-O-NEXT: Running analysis: LoopAnalysis on foo +; CHECK-O-NEXT: Running analysis: CycleAnalysis on foo ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo ; CHECK-O-NEXT: Running analysis: BlockFrequencyAnalysis on foo +; CHECK-O-NEXT: Running analysis: LoopAnalysis on foo ; CHECK-O-NEXT: Invalidating analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Invalidating analysis: LazyCallGraphAnalysis on ; CHECK-O-NEXT: Invalidating analysis: InnerAnalysisManagerProxy @@ -119,8 +120,9 @@ ; CHECK-O-NEXT: Running analysis: LastRunTrackingAnalysis ; CHECK-O-NEXT: Running analysis: BlockFrequencyAnalysis on foo ; CHECK-O-NEXT: Running analysis: BranchProbabilityAnalysis on foo -; CHECK-O-NEXT: Running analysis: LoopAnalysis on foo +; CHECK-O-NEXT: Running analysis: CycleAnalysis on foo ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo +; CHECK-O-NEXT: Running analysis: LoopAnalysis on foo ; CHECK-O23-NEXT: Running pass: AggressiveInstCombinePass ; CHECK-O-NEXT: Running pass: LibCallsShrinkWrapPass ; CHECK-O23-NEXT: Running pass: PGOMemOPSizeOpt @@ -166,7 +168,6 @@ ; CHECK-O-NEXT: Running pass: ADCEPass ; CHECK-O23-NEXT: Running pass: MemCpyOptPass ; CHECK-O23-NEXT: Running pass: DSEPass -; CHECK-O23-NEXT: Running analysis: CycleAnalysis ; CHECK-O23-NEXT: Running pass: MoveAutoInitPass on foo ; CHECK-O23-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23-NEXT: Running pass: LCSSAPass diff --git a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll index ef01b14df972a..34ff5ae50f30b 100644 --- a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll @@ -54,8 +54,9 @@ ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: BlockFrequencyAnalysis on foo ; CHECK-O-NEXT: Running analysis: BranchProbabilityAnalysis on foo -; CHECK-O-NEXT: Running analysis: LoopAnalysis on foo +; CHECK-O-NEXT: Running analysis: CycleAnalysis on foo ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo +; CHECK-O-NEXT: Running analysis: LoopAnalysis on foo ; CHECK-O-NEXT: Running pass: SimplifyCFGPass on foo ; CHECK-O-NEXT: Running pass: PGOForceFunctionAttrsPass ; CHECK-O-NEXT: Running pass: AlwaysInlinerPass @@ -131,7 +132,6 @@ ; CHECK-O-NEXT: Running pass: ADCEPass ; CHECK-O23-NEXT: Running pass: MemCpyOptPass ; CHECK-O23-NEXT: Running pass: DSEPass -; CHECK-O23-NEXT: Running analysis: CycleAnalysis ; CHECK-O23-NEXT: Running pass: MoveAutoInitPass on foo ; CHECK-O23-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23-NEXT: Running pass: LCSSAPass diff --git a/llvm/test/Transforms/Inline/cgscc-incremental-invalidate.ll b/llvm/test/Transforms/Inline/cgscc-incremental-invalidate.ll index bc31bca614a40..373bc01c9f473 100644 --- a/llvm/test/Transforms/Inline/cgscc-incremental-invalidate.ll +++ b/llvm/test/Transforms/Inline/cgscc-incremental-invalidate.ll @@ -8,20 +8,23 @@ ; ; CHECK: Running pass: InlinerPass on (test1_f, test1_g, test1_h) ; CHECK: Running analysis: DominatorTreeAnalysis on test1_f +; CHECK: Invalidating analysis: CycleAnalysis on test1_f ; CHECK: Invalidating analysis: DominatorTreeAnalysis on test1_f -; CHECK: Invalidating analysis: LoopAnalysis on test1_f ; CHECK: Invalidating analysis: BranchProbabilityAnalysis on test1_f +; CHECK: Invalidating analysis: LoopAnalysis on test1_f ; CHECK: Invalidating analysis: BlockFrequencyAnalysis on test1_f ; CHECK: Invalidating analysis: EphemeralValuesAnalysis on test1_f ; CHECK: Running analysis: DominatorTreeAnalysis on test1_g +; CHECK: Invalidating analysis: CycleAnalysis on test1_g ; CHECK: Invalidating analysis: DominatorTreeAnalysis on test1_g -; CHECK: Invalidating analysis: LoopAnalysis on test1_g ; CHECK: Invalidating analysis: BranchProbabilityAnalysis on test1_g +; CHECK: Invalidating analysis: LoopAnalysis on test1_g ; CHECK: Invalidating analysis: BlockFrequencyAnalysis on test1_g ; CHECK: Invalidating analysis: EphemeralValuesAnalysis on test1_g +; CHECK: Invalidating analysis: CycleAnalysis on test1_h ; CHECK: Invalidating analysis: DominatorTreeAnalysis on test1_h -; CHECK: Invalidating analysis: LoopAnalysis on test1_h ; CHECK: Invalidating analysis: BranchProbabilityAnalysis on test1_h +; CHECK: Invalidating analysis: LoopAnalysis on test1_h ; CHECK: Invalidating analysis: BlockFrequencyAnalysis on test1_h ; CHECK: Invalidating analysis: EphemeralValuesAnalysis on test1_h ; CHECK-NOT: Invalidating analysis: diff --git a/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext b/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext index b8828cc1238d7..d0fdac4193553 100644 --- a/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext +++ b/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext @@ -7,8 +7,8 @@ _Z11irreducibleii # Counter Values: 1000 950 -100 373 +100 1 0 @@ -19,11 +19,11 @@ _Z11irreduciblePh 9 # Counter Values: 300 -99 -300 -201 1 +201 1 +99 0 0 +99 0 diff --git a/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext index da84cd0ff34a0..361a63c8e3e7f 100644 --- a/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext +++ b/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext @@ -9,8 +9,8 @@ _Z11irreducibleii 1 1000 950 -100 373 +100 0 _Z11irreduciblePh @@ -21,10 +21,10 @@ _Z11irreduciblePh # Counter Values: 1 300 -99 -300 201 1 +99 0 0 +99 0 diff --git a/llvm/unittests/Analysis/BlockFrequencyInfoTest.cpp b/llvm/unittests/Analysis/BlockFrequencyInfoTest.cpp index c1e94cd5898fb..fd6daf4ae1230 100644 --- a/llvm/unittests/Analysis/BlockFrequencyInfoTest.cpp +++ b/llvm/unittests/Analysis/BlockFrequencyInfoTest.cpp @@ -12,6 +12,7 @@ #include "llvm/Analysis/LoopInfo.h" #include "llvm/AsmParser/Parser.h" #include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CycleInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/LLVMContext.h" @@ -28,13 +29,16 @@ class BlockFrequencyInfoTest : public testing::Test { protected: std::unique_ptr BPI; std::unique_ptr DT; + std::unique_ptr CI; std::unique_ptr LI; LLVMContext C; BlockFrequencyInfo buildBFI(Function &F) { DT.reset(new DominatorTree(F)); LI.reset(new LoopInfo(*DT)); - BPI.reset(new BranchProbabilityInfo(F, *LI)); + CI.reset(new CycleInfo()); + CI->compute(F); + BPI.reset(new BranchProbabilityInfo(F, *CI)); return BlockFrequencyInfo(F, *BPI, *LI); } std::unique_ptr makeLLVMModule() { diff --git a/llvm/unittests/Analysis/BranchProbabilityInfoTest.cpp b/llvm/unittests/Analysis/BranchProbabilityInfoTest.cpp index 38d9c0dd46e6f..ce4e763b553f7 100644 --- a/llvm/unittests/Analysis/BranchProbabilityInfoTest.cpp +++ b/llvm/unittests/Analysis/BranchProbabilityInfoTest.cpp @@ -7,10 +7,10 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/BranchProbabilityInfo.h" -#include "llvm/Analysis/LoopInfo.h" #include "llvm/AsmParser/Parser.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/CycleInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/Instructions.h" @@ -27,13 +27,14 @@ namespace { struct BranchProbabilityInfoTest : public testing::Test { std::unique_ptr BPI; std::unique_ptr DT; - std::unique_ptr LI; + std::unique_ptr CI; LLVMContext C; BranchProbabilityInfo &buildBPI(Function &F) { DT.reset(new DominatorTree(F)); - LI.reset(new LoopInfo(*DT)); - BPI.reset(new BranchProbabilityInfo(F, *LI)); + CI.reset(new CycleInfo()); + CI->compute(F); + BPI.reset(new BranchProbabilityInfo(F, *CI)); return *BPI; } diff --git a/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp b/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp index c8752c78d1c35..629d7b0ffedcb 100644 --- a/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp +++ b/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp @@ -12,6 +12,7 @@ #include "llvm/Analysis/LoopInfo.h" #include "llvm/AsmParser/Parser.h" #include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CycleInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/LLVMContext.h" @@ -37,6 +38,7 @@ class ProfileSummaryInfoTest : public testing::Test { std::unique_ptr BPI; std::unique_ptr DT; std::unique_ptr LI; + std::unique_ptr CI; ProfileSummaryInfo buildPSI(Module *M) { return ProfileSummaryInfo(*M); @@ -44,7 +46,9 @@ class ProfileSummaryInfoTest : public testing::Test { BlockFrequencyInfo buildBFI(Function &F) { DT.reset(new DominatorTree(F)); LI.reset(new LoopInfo(*DT)); - BPI.reset(new BranchProbabilityInfo(F, *LI)); + CI.reset(new CycleInfo()); + CI->compute(F); + BPI.reset(new BranchProbabilityInfo(F, *CI)); return BlockFrequencyInfo(F, *BPI, *LI); } std::unique_ptr makeLLVMModule(const char *ProfKind = nullptr, diff --git a/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp b/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp index 400f23a932540..4941b153cc6f2 100644 --- a/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp +++ b/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp @@ -6,9 +6,11 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Transforms/IPO/FunctionSpecialization.h" #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/Analysis/CycleAnalysis.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/TargetLibraryInfo.h" @@ -17,7 +19,6 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/PassInstrumentation.h" #include "llvm/Support/SourceMgr.h" -#include "llvm/Transforms/IPO/FunctionSpecialization.h" #include "llvm/Transforms/Utils/SCCPSolver.h" #include "gtest/gtest.h" #include @@ -49,6 +50,7 @@ class FunctionSpecializationTest : public testing::Test { FAM.registerPass([&] { return TargetIRAnalysis(); }); FAM.registerPass([&] { return BlockFrequencyAnalysis(); }); FAM.registerPass([&] { return BranchProbabilityAnalysis(); }); + FAM.registerPass([&] { return CycleAnalysis(); }); FAM.registerPass([&] { return LoopAnalysis(); }); FAM.registerPass([&] { return AssumptionAnalysis(); }); FAM.registerPass([&] { return DominatorTreeAnalysis(); }); diff --git a/llvm/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp b/llvm/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp index a91d67ff6c550..5785037687d28 100644 --- a/llvm/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp +++ b/llvm/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp @@ -20,6 +20,7 @@ #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/AsmParser/Parser.h" #include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CycleInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" @@ -651,7 +652,9 @@ define void @crit_edge(ptr %tgt, i1 %cond0, i1 %cond1) { Function *F = M->getFunction("crit_edge"); DominatorTree DT(*F); LoopInfo LI(DT); - BranchProbabilityInfo BPI(*F, LI); + CycleInfo CI; + CI.compute(*F); + BranchProbabilityInfo BPI(*F, CI); BlockFrequencyInfo BFI(*F, BPI, LI); ASSERT_TRUE(SplitIndirectBrCriticalEdges(*F, /*IgnoreBlocksWithoutPHI=*/true, @@ -693,7 +696,9 @@ define void @crit_edge(ptr %tgt, i1 %cond0, i1 %cond1) { Function *F = M->getFunction("crit_edge"); DominatorTree DT(*F); LoopInfo LI(DT); - BranchProbabilityInfo BPI(*F, LI); + CycleInfo CI; + CI.compute(*F); + BranchProbabilityInfo BPI(*F, CI); BlockFrequencyInfo BFI(*F, BPI, LI); ASSERT_TRUE(SplitIndirectBrCriticalEdges(*F, /*IgnoreBlocksWithoutPHI=*/false, @@ -794,8 +799,9 @@ switch i32 %0, label %LD [ )IR"); Function *F = M->getFunction("edge_probability"); DominatorTree DT(*F); - LoopInfo LI(DT); - BranchProbabilityInfo BPI(*F, LI); + CycleInfo CI; + CI.compute(*F); + BranchProbabilityInfo BPI(*F, CI); // Check that the unreachable block has the minimal probability. const BasicBlock *EntryBB = getBasicBlockByName(*F, "entry"); diff --git a/llvm/unittests/Transforms/Utils/SizeOptsTest.cpp b/llvm/unittests/Transforms/Utils/SizeOptsTest.cpp index 9c3d1657aa03a..f1057fb964b80 100644 --- a/llvm/unittests/Transforms/Utils/SizeOptsTest.cpp +++ b/llvm/unittests/Transforms/Utils/SizeOptsTest.cpp @@ -7,12 +7,13 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Utils/SizeOpts.h" -#include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/AsmParser/Parser.h" #include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CycleInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/LLVMContext.h" @@ -33,12 +34,15 @@ class SizeOptsTest : public testing::Test { struct BFIData { std::unique_ptr DT; std::unique_ptr LI; + std::unique_ptr CI; std::unique_ptr BPI; std::unique_ptr BFI; BFIData(Function &F) { DT.reset(new DominatorTree(F)); LI.reset(new LoopInfo(*DT)); - BPI.reset(new BranchProbabilityInfo(F, *LI)); + CI.reset(new CycleInfo()); + CI->compute(F); + BPI.reset(new BranchProbabilityInfo(F, *CI)); BFI.reset(new BlockFrequencyInfo(F, *BPI, *LI)); } BlockFrequencyInfo *get() { return BFI.get(); } From c2fac16a870fd77772b05ca11f7aa1f4e5a4d595 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Tue, 21 Jul 2026 11:12:17 -0700 Subject: [PATCH 05/50] Revert "[IR] Make semantics of strictfp consistent" (#211057) Reverts llvm/llvm-project#209465 Causes verifier issues: https://github.com/llvm/llvm-project/pull/209465#issuecomment-5025559698 --- clang/lib/CodeGen/TargetInfo.cpp | 3 - .../cl20-device-side-enqueue-attributes.cl | 4 +- llvm/docs/LangRef.md | 6 +- llvm/include/llvm/IR/Function.h | 3 - llvm/lib/IR/Verifier.cpp | 5 -- llvm/test/CodeGen/AArch64/is_fpclass.ll | 4 +- .../sms-loop-carried-fp-exceptions1.mir | 2 +- .../sve-streaming-mode-cvt-fp-to-int.ll | 4 +- .../sve-streaming-mode-cvt-int-to-fp.ll | 4 +- llvm/test/CodeGen/PowerPC/milicode32.ll | 2 +- .../ppcf128-constrained-fp-intrinsics.ll | 16 ++++- .../CodeGen/RISCV/double-intrinsics-strict.ll | 2 +- .../CodeGen/RISCV/float-intrinsics-strict.ll | 2 +- .../RISCV/zfh-half-intrinsics-strict.ll | 2 +- .../RISCV/zfhmin-half-intrinsics-strict.ll | 2 +- .../llvm-intrinsics/constrained-fmuladd.ll | 12 ++-- .../EarlyCSE/replace-calls-def-attrs.ll | 67 +++++++++---------- llvm/test/Transforms/InstCombine/erf.ll | 40 +++++------ llvm/test/Transforms/InstCombine/fdim.ll | 7 +- .../InstSimplify/disable_folding.ll | 14 ++-- .../merge-compatible-invokes-of-landingpad.ll | 30 ++++----- llvm/unittests/IR/VerifierTest.cpp | 24 ------- .../test/Target/LLVMIR/llvmir-intrinsics.mlir | 50 +++++++------- 23 files changed, 136 insertions(+), 169 deletions(-) diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp index 2f89154c2d2d9..641caf6d5cd4b 100644 --- a/clang/lib/CodeGen/TargetInfo.cpp +++ b/clang/lib/CodeGen/TargetInfo.cpp @@ -195,9 +195,6 @@ llvm::Value *TargetCodeGenInfo::createEnqueuedBlockKernel( CGF.CGM.addDefaultFunctionDefinitionAttributes(KernelAttrs); F->addFnAttrs(KernelAttrs); - if (CGF.Builder.getIsFPConstrained()) - F->addFnAttr(llvm::Attribute::StrictFP); - auto IP = CGF.Builder.saveIP(); auto *BB = llvm::BasicBlock::Create(C, "entry", F); auto &Builder = CGF.Builder; diff --git a/clang/test/CodeGenOpenCL/cl20-device-side-enqueue-attributes.cl b/clang/test/CodeGenOpenCL/cl20-device-side-enqueue-attributes.cl index c93f623016660..962ac4afa4991 100644 --- a/clang/test/CodeGenOpenCL/cl20-device-side-enqueue-attributes.cl +++ b/clang/test/CodeGenOpenCL/cl20-device-side-enqueue-attributes.cl @@ -188,7 +188,7 @@ kernel void device_side_enqueue(global float *a, global float *b, int i) { // STRICTFP-NEXT: ret void // // -// STRICTFP: Function Attrs: convergent nounwind strictfp +// STRICTFP: Function Attrs: convergent nounwind // STRICTFP-LABEL: define spir_kernel void @__device_side_enqueue_block_invoke_kernel( // STRICTFP-SAME: ptr addrspace(4) [[TMP0:%.*]]) #[[ATTR4:[0-9]+]] { // STRICTFP-NEXT: [[ENTRY:.*:]] @@ -207,7 +207,7 @@ kernel void device_side_enqueue(global float *a, global float *b, int i) { // STRICTFP: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } // STRICTFP: attributes #[[ATTR2]] = { convergent noinline nounwind optnone strictfp "stack-protector-buffer-size"="8" } // STRICTFP: attributes #[[ATTR3:[0-9]+]] = { nocallback nofree nosync nounwind strictfp willreturn memory(inaccessiblemem: readwrite) } -// STRICTFP: attributes #[[ATTR4]] = { convergent nounwind strictfp "stack-protector-buffer-size"="8" } +// STRICTFP: attributes #[[ATTR4]] = { convergent nounwind "stack-protector-buffer-size"="8" } // STRICTFP: attributes #[[ATTR5]] = { convergent nounwind strictfp } // STRICTFP: attributes #[[ATTR6]] = { strictfp } //. diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md index 461177d24bfa5..415d015d39a93 100644 --- a/llvm/docs/LangRef.md +++ b/llvm/docs/LangRef.md @@ -2794,8 +2794,7 @@ fn -> other_fn -> other_fn ; fn is norecurse optimizations that require assumptions about the floating-point rounding mode or that might alter the state of floating-point status flags that might otherwise be set or cleared by calling this function. LLVM will - not introduce any new floating-point instructions that may trap. All - function definitions that contain strictfp calls must be marked strictfp. + not introduce any new floating-point instructions that may trap. (denormal_fpenv)= @@ -26624,6 +26623,9 @@ point intrinsics must have the `strictfp` attribute either on the calling instruction or on the declaration or definition of the function being called. +All function *definitions* that use constrained floating point intrinsics +must have the `strictfp` attribute. + #### '`llvm.experimental.constrained.fadd`' Intrinsic ##### Syntax: diff --git a/llvm/include/llvm/IR/Function.h b/llvm/include/llvm/IR/Function.h index 0238c9b352f5f..bd28f0d9902da 100644 --- a/llvm/include/llvm/IR/Function.h +++ b/llvm/include/llvm/IR/Function.h @@ -632,9 +632,6 @@ class LLVM_ABI Function : public GlobalObject, public ilist_node { addFnAttr(Attribute::NoRecurse); } - /// Determine if the function has strict floating point sematics. - bool isStrictFP() const { return hasFnAttribute(Attribute::StrictFP); } - /// Determine if the function is required to make forward progress. bool mustProgress() const { return hasFnAttribute(Attribute::MustProgress) || diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 41c45be8de882..f6b7d30cf42c3 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -3897,11 +3897,6 @@ void Verifier::visitCallBase(CallBase &Call) { Check(!Attrs.hasFnAttr(Attribute::DenormalFPEnv), "denormal_fpenv attribute may not apply to call sites", Call); - Check(!Attrs.hasFnAttr(Attribute::StrictFP) || - Call.getFunction()->isStrictFP(), - "call site marked strictfp without caller function marked strictfp", - Call); - // Verify call attributes. verifyFunctionAttrs(FTy, Attrs, &Call, IsIntrinsic, Call.isInlineAsm()); diff --git a/llvm/test/CodeGen/AArch64/is_fpclass.ll b/llvm/test/CodeGen/AArch64/is_fpclass.ll index 19ce95641fe98..71a8f42a7b4db 100644 --- a/llvm/test/CodeGen/AArch64/is_fpclass.ll +++ b/llvm/test/CodeGen/AArch64/is_fpclass.ll @@ -249,7 +249,7 @@ entry: ret i1 %0 } -define i1 @isfinite_d_strictfp(double %x) strictfp { +define i1 @isfinite_d_strictfp(double %x) { ; CHECK-SD-LABEL: isfinite_d_strictfp: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fmov x8, d0 @@ -278,7 +278,7 @@ entry: ret i1 %0 } -define i1 @not_isfinite_d_strictfp(double %x) strictfp { +define i1 @not_isfinite_d_strictfp(double %x) { ; CHECK-SD-LABEL: not_isfinite_d_strictfp: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fmov x8, d0 diff --git a/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir b/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir index 4d3552a056df5..878f7b782a9a1 100644 --- a/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir +++ b/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir @@ -23,7 +23,7 @@ --- | @x = dso_local global i32 0, align 4 - define dso_local void @f(ptr nocapture noundef writeonly %a, float noundef %y, i32 noundef %n) #2 { + define dso_local void @f(ptr nocapture noundef writeonly %a, float noundef %y, i32 noundef %n) { entry: %cmp6 = icmp sgt i32 %n, 0 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-cvt-fp-to-int.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-cvt-fp-to-int.ll index 9320908403679..cfdc1baf8c282 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-cvt-fp-to-int.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-cvt-fp-to-int.ll @@ -209,7 +209,7 @@ define i64 @f64_to_u64(double %x) { ret i64 %cvt } -define i32 @strict_convert_signed(double %x) #0 { +define i32 @strict_convert_signed(double %x) { ; CHECK-LABEL: strict_convert_signed: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: fcvtzs w0, d0 @@ -224,7 +224,7 @@ define i32 @strict_convert_signed(double %x) #0 { ret i32 %cvt } -define i32 @strict_convert_unsigned(float %x) #0 { +define i32 @strict_convert_unsigned(float %x) { ; CHECK-LABEL: strict_convert_unsigned: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: fcvtzu w0, s0 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-cvt-int-to-fp.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-cvt-int-to-fp.ll index 9e96355fd598d..83403fc4ad86f 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-cvt-int-to-fp.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-cvt-int-to-fp.ll @@ -209,7 +209,7 @@ entry: ret double %cvt } -define float @strict_convert_signed(i32 %x) #0 { +define float @strict_convert_signed(i32 %x) { ; CHECK-LABEL: strict_convert_signed: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: scvtf s0, w0 @@ -224,7 +224,7 @@ entry: ret float %cvt } -define float @strict_convert_unsigned(i64 %x) #0 { +define float @strict_convert_unsigned(i64 %x) { ; CHECK-LABEL: strict_convert_unsigned: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ucvtf s0, x0 diff --git a/llvm/test/CodeGen/PowerPC/milicode32.ll b/llvm/test/CodeGen/PowerPC/milicode32.ll index f2caec7e0045f..2c7e5c9867137 100644 --- a/llvm/test/CodeGen/PowerPC/milicode32.ll +++ b/llvm/test/CodeGen/PowerPC/milicode32.ll @@ -68,7 +68,7 @@ entry: ret i32 %call } -define i32 @strlen_test_fp_strict(ptr noundef %str) strictfp nounwind { +define i32 @strlen_test_fp_strict(ptr noundef %str) nounwind { ; CHECK-AIX-32-P9-LABEL: strlen_test_fp_strict: ; CHECK-AIX-32-P9: # %bb.0: # %entry ; CHECK-AIX-32-P9-NEXT: mflr r0 diff --git a/llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll b/llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll index 756cc6dc9408c..c1ee436a40c55 100644 --- a/llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll +++ b/llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll @@ -1972,12 +1972,14 @@ entry: ret ppc_fp128 %conv } -define i1 @ppcq_to_s1(ppc_fp128 %a) #0 { +define i1 @ppcq_to_s1(ppc_fp128 %a) { ; PC64LE-LABEL: ppcq_to_s1: ; PC64LE: # %bb.0: # %entry ; PC64LE-NEXT: mflr 0 ; PC64LE-NEXT: stdu 1, -32(1) ; PC64LE-NEXT: std 0, 48(1) +; PC64LE-NEXT: .cfi_def_cfa_offset 32 +; PC64LE-NEXT: .cfi_offset lr, 16 ; PC64LE-NEXT: bl __gcc_qtou ; PC64LE-NEXT: nop ; PC64LE-NEXT: addi 1, 1, 32 @@ -1990,6 +1992,8 @@ define i1 @ppcq_to_s1(ppc_fp128 %a) #0 { ; PC64LE9-NEXT: mflr 0 ; PC64LE9-NEXT: stdu 1, -32(1) ; PC64LE9-NEXT: std 0, 48(1) +; PC64LE9-NEXT: .cfi_def_cfa_offset 32 +; PC64LE9-NEXT: .cfi_offset lr, 16 ; PC64LE9-NEXT: bl __gcc_qtou ; PC64LE9-NEXT: nop ; PC64LE9-NEXT: addi 1, 1, 32 @@ -2002,6 +2006,8 @@ define i1 @ppcq_to_s1(ppc_fp128 %a) #0 { ; PC64-NEXT: mflr 0 ; PC64-NEXT: stdu 1, -112(1) ; PC64-NEXT: std 0, 128(1) +; PC64-NEXT: .cfi_def_cfa_offset 112 +; PC64-NEXT: .cfi_offset lr, 16 ; PC64-NEXT: bl __gcc_qtou ; PC64-NEXT: nop ; PC64-NEXT: addi 1, 1, 112 @@ -2013,12 +2019,14 @@ entry: ret i1 %conv } -define i1 @ppcq_to_u1(ppc_fp128 %a) #0 { +define i1 @ppcq_to_u1(ppc_fp128 %a) { ; PC64LE-LABEL: ppcq_to_u1: ; PC64LE: # %bb.0: # %entry ; PC64LE-NEXT: mflr 0 ; PC64LE-NEXT: stdu 1, -32(1) ; PC64LE-NEXT: std 0, 48(1) +; PC64LE-NEXT: .cfi_def_cfa_offset 32 +; PC64LE-NEXT: .cfi_offset lr, 16 ; PC64LE-NEXT: bl __fixunstfsi ; PC64LE-NEXT: nop ; PC64LE-NEXT: addi 1, 1, 32 @@ -2031,6 +2039,8 @@ define i1 @ppcq_to_u1(ppc_fp128 %a) #0 { ; PC64LE9-NEXT: mflr 0 ; PC64LE9-NEXT: stdu 1, -32(1) ; PC64LE9-NEXT: std 0, 48(1) +; PC64LE9-NEXT: .cfi_def_cfa_offset 32 +; PC64LE9-NEXT: .cfi_offset lr, 16 ; PC64LE9-NEXT: bl __fixunstfsi ; PC64LE9-NEXT: nop ; PC64LE9-NEXT: addi 1, 1, 32 @@ -2043,6 +2053,8 @@ define i1 @ppcq_to_u1(ppc_fp128 %a) #0 { ; PC64-NEXT: mflr 0 ; PC64-NEXT: stdu 1, -112(1) ; PC64-NEXT: std 0, 128(1) +; PC64-NEXT: .cfi_def_cfa_offset 112 +; PC64-NEXT: .cfi_offset lr, 16 ; PC64-NEXT: bl __fixunstfsi ; PC64-NEXT: nop ; PC64-NEXT: addi 1, 1, 112 diff --git a/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll b/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll index 654d99948bd40..53fcfa19725db 100644 --- a/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll +++ b/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll @@ -1989,7 +1989,7 @@ define i64 @llround_f64(double %a) nounwind strictfp { ret i64 %1 } -define double @ldexp_f64(double %x, i32 signext %y) nounwind strictfp { +define double @ldexp_f64(double %x, i32 signext %y) nounwind { ; RV32IFD-LABEL: ldexp_f64: ; RV32IFD: # %bb.0: ; RV32IFD-NEXT: addi sp, sp, -16 diff --git a/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll b/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll index b22661976e727..3a4acfd8a41ee 100644 --- a/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll +++ b/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll @@ -1955,7 +1955,7 @@ define i64 @llround_f32(float %a) nounwind strictfp { ret i64 %1 } -define float @ldexp_f32(float %x, i32 signext %y) nounwind strictfp { +define float @ldexp_f32(float %x, i32 signext %y) nounwind { ; RV32IF-LABEL: ldexp_f32: ; RV32IF: # %bb.0: ; RV32IF-NEXT: addi sp, sp, -16 diff --git a/llvm/test/CodeGen/RISCV/zfh-half-intrinsics-strict.ll b/llvm/test/CodeGen/RISCV/zfh-half-intrinsics-strict.ll index c876b76b8f0d9..eb1848965a9ba 100644 --- a/llvm/test/CodeGen/RISCV/zfh-half-intrinsics-strict.ll +++ b/llvm/test/CodeGen/RISCV/zfh-half-intrinsics-strict.ll @@ -714,7 +714,7 @@ define i64 @llround_f16(half %a) nounwind strictfp { ret i64 %1 } -define half @ldexp_f16(half %x, i32 signext %y) nounwind strictfp { +define half @ldexp_f16(half %x, i32 signext %y) nounwind { ; RV32IZFH-LABEL: ldexp_f16: ; RV32IZFH: # %bb.0: ; RV32IZFH-NEXT: addi sp, sp, -16 diff --git a/llvm/test/CodeGen/RISCV/zfhmin-half-intrinsics-strict.ll b/llvm/test/CodeGen/RISCV/zfhmin-half-intrinsics-strict.ll index 891ffa0a0c96b..0529819a4f4e2 100644 --- a/llvm/test/CodeGen/RISCV/zfhmin-half-intrinsics-strict.ll +++ b/llvm/test/CodeGen/RISCV/zfhmin-half-intrinsics-strict.ll @@ -744,7 +744,7 @@ define i64 @llround_f16(half %a) nounwind strictfp { ret i64 %1 } -define half @ldexp_f16(half %x, i32 signext %y) nounwind strictfp { +define half @ldexp_f16(half %x, i32 signext %y) nounwind { ; RV32IZFHMIN-LABEL: ldexp_f16: ; RV32IZFHMIN: # %bb.0: ; RV32IZFHMIN-NEXT: addi sp, sp, -16 diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-fmuladd.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-fmuladd.ll index 67eed241354d2..b2d4f570afbd9 100644 --- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-fmuladd.ll +++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/constrained-fmuladd.ll @@ -10,7 +10,7 @@ ; CHECK: OpFMul %[[#]] %[[#]] %[[#]] ; CHECK: OpFAdd %[[#]] %[[#]] %[[#]] -define spir_kernel void @test_f32(float %a) strictfp { +define spir_kernel void @test_f32(float %a) { entry: %r = tail call float @llvm.experimental.constrained.fmuladd.f32( float %a, float %a, float %a, @@ -18,9 +18,9 @@ entry: ret void } -; CHECK: OpFMul %[[#]] %[[#]] %[[#]] +; CHECK: OpFMul %[[#]] %[[#]] %[[#]] ; CHECK: OpFAdd %[[#]] %[[#]] %[[#]] -define spir_kernel void @test_f64(double %a) strictfp { +define spir_kernel void @test_f64(double %a) { entry: %r = tail call double @llvm.experimental.constrained.fmuladd.f64( double %a, double %a, double %a, @@ -30,7 +30,7 @@ entry: ; CHECK: OpFMul %[[#]] %[[#]] %[[#]] ; CHECK: OpFAdd %[[#]] %[[#]] %[[#]] -define spir_kernel void @test_v2f32(<2 x float> %a) strictfp { +define spir_kernel void @test_v2f32(<2 x float> %a) { entry: %r = tail call <2 x float> @llvm.experimental.constrained.fmuladd.v2f32( <2 x float> %a, <2 x float> %a, <2 x float> %a, @@ -40,7 +40,7 @@ entry: ; CHECK: OpFMul %[[#]] %[[#]] %[[#]] ; CHECK: OpFAdd %[[#]] %[[#]] %[[#]] -define spir_kernel void @test_v4f32(<4 x float> %a) strictfp { +define spir_kernel void @test_v4f32(<4 x float> %a) { entry: %r = tail call <4 x float> @llvm.experimental.constrained.fmuladd.v4f32( <4 x float> %a, <4 x float> %a, <4 x float> %a, @@ -50,7 +50,7 @@ entry: ; CHECK: OpFMul %[[#]] %[[#]] %[[#]] ; CHECK: OpFAdd %[[#]] %[[#]] %[[#]] -define spir_kernel void @test_v2f64(<2 x double> %a) strictfp { +define spir_kernel void @test_v2f64(<2 x double> %a) { entry: %r = tail call <2 x double> @llvm.experimental.constrained.fmuladd.v2f64( <2 x double> %a, <2 x double> %a, <2 x double> %a, diff --git a/llvm/test/Transforms/EarlyCSE/replace-calls-def-attrs.ll b/llvm/test/Transforms/EarlyCSE/replace-calls-def-attrs.ll index b8854e3061dbf..1dbffd962a638 100644 --- a/llvm/test/Transforms/EarlyCSE/replace-calls-def-attrs.ll +++ b/llvm/test/Transforms/EarlyCSE/replace-calls-def-attrs.ll @@ -13,7 +13,7 @@ declare i8 @buz.fp(float, float) define i8 @same_parent_combine_diff_attrs(i8 %x, i8 %y) { ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs( ; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) { -; CHECK-NEXT: [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR2:[0-9]+]] +; CHECK-NEXT: [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1:[0-9]+]] ; CHECK-NEXT: [[R:%.*]] = call i8 @buz(i8 [[C1]], i8 [[C1]]) ; CHECK-NEXT: ret i8 [[R]] ; @@ -27,7 +27,7 @@ define i8 @same_parent_combine_diff_attrs(i8 %x, i8 %y) { define i8 @same_parent_combine_diff_attrs_needs_intersect(i8 %x, i8 %y) { ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_needs_intersect( ; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) { -; CHECK-NEXT: [[C1:%.*]] = call ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR2]] +; CHECK-NEXT: [[C1:%.*]] = call ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]] ; CHECK-NEXT: [[R:%.*]] = call i8 @buz.ptr(ptr [[C1]], ptr [[C1]]) ; CHECK-NEXT: ret i8 [[R]] ; @@ -41,7 +41,7 @@ define i8 @same_parent_combine_diff_attrs_needs_intersect(i8 %x, i8 %y) { define i8 @same_parent_combine_diff_attrs_fmf(float %x, float %y) { ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_fmf( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[C1:%.*]] = call nnan float @baz.fp(float [[X]], float noundef [[Y]]) #[[ATTR3:[0-9]+]] +; CHECK-NEXT: [[C1:%.*]] = call nnan float @baz.fp(float [[X]], float noundef [[Y]]) #[[ATTR2:[0-9]+]] ; CHECK-NEXT: [[R:%.*]] = call i8 @buz.fp(float [[C1]], float [[C1]]) ; CHECK-NEXT: ret i8 [[R]] ; @@ -55,7 +55,7 @@ define i8 @same_parent_combine_diff_attrs_fmf(float %x, float %y) { define i8 @same_parent_combine_diff_attrs_fmf2(float %x, float %y) { ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_fmf2( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[C1:%.*]] = call nnan float @baz.fp(float [[X]], float noundef [[Y]]) #[[ATTR2]] +; CHECK-NEXT: [[C1:%.*]] = call nnan float @baz.fp(float [[X]], float noundef [[Y]]) #[[ATTR1]] ; CHECK-NEXT: [[R:%.*]] = call i8 @buz.fp(float [[C1]], float [[C1]]) ; CHECK-NEXT: ret i8 [[R]] ; @@ -69,7 +69,7 @@ define i8 @same_parent_combine_diff_attrs_fmf2(float %x, float %y) { define i8 @same_parent_combine_diff_attrs_needs_intersect2(i8 %x, i8 %y) { ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_needs_intersect2( ; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) { -; CHECK-NEXT: [[C1:%.*]] = call ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR3]] +; CHECK-NEXT: [[C1:%.*]] = call ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR2]] ; CHECK-NEXT: [[R:%.*]] = call i8 @buz.ptr(ptr [[C1]], ptr [[C1]]) ; CHECK-NEXT: ret i8 [[R]] ; @@ -83,7 +83,7 @@ define i8 @same_parent_combine_diff_attrs_needs_intersect2(i8 %x, i8 %y) { define i8 @same_parent_combine_diff_attrs_really_needs_intersect(i8 %x, i8 %y) { ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_really_needs_intersect( ; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) { -; CHECK-NEXT: [[C1:%.*]] = call ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR3]] +; CHECK-NEXT: [[C1:%.*]] = call ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR2]] ; CHECK-NEXT: [[R:%.*]] = call i8 @buz.ptr(ptr [[C1]], ptr noundef [[C1]]) ; CHECK-NEXT: ret i8 [[R]] ; @@ -112,7 +112,7 @@ define i8 @same_parent_combine_diff_attrs_fail_side_effects(i8 %x, i8 %y) { define i8 @same_parent_combine_diff_attrs_quasi_side_effects2(i8 %x, i8 %y) { ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_quasi_side_effects2( ; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) { -; CHECK-NEXT: [[C1:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR2]] +; CHECK-NEXT: [[C1:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR1]] ; CHECK-NEXT: [[C0:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) ; CHECK-NEXT: [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]]) ; CHECK-NEXT: ret i8 [[R]] @@ -127,10 +127,10 @@ define i8 @same_parent_combine_diff_attrs_quasi_side_effects2(i8 %x, i8 %y) { define i8 @diff_parent_combine_diff_attrs(i1 %c, i8 %x, i8 %y) { ; CHECK-LABEL: define i8 @diff_parent_combine_diff_attrs( ; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) { -; CHECK-NEXT: [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR2]] +; CHECK-NEXT: [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]] ; CHECK-NEXT: br i1 [[C]], label %[[T:.*]], label %[[F:.*]] ; CHECK: [[T]]: -; CHECK-NEXT: [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR3]] +; CHECK-NEXT: [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR2]] ; CHECK-NEXT: [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]]) ; CHECK-NEXT: ret i8 [[R]] ; CHECK: [[F]]: @@ -151,7 +151,7 @@ F: define i8 @diff_parent_combine_diff_attrs_preserves_return_attrs(i1 %c, i8 %x, i8 %y) { ; CHECK-LABEL: define i8 @diff_parent_combine_diff_attrs_preserves_return_attrs( ; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) { -; CHECK-NEXT: [[C1:%.*]] = call nonnull ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR3]] +; CHECK-NEXT: [[C1:%.*]] = call nonnull ptr @baz.ptr(i8 [[X]], i8 noundef [[Y]]) #[[ATTR2]] ; CHECK-NEXT: br i1 [[C]], label %[[T:.*]], label %[[F:.*]] ; CHECK: [[T]]: ; CHECK-NEXT: [[R:%.*]] = call i8 @buz.ptr(ptr [[C1]], ptr noundef [[C1]]) @@ -172,8 +172,8 @@ F: define i8 @same_parent_combine_diff_attrs_todo(i8 %x, i8 %y) { ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_todo( ; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) { -; CHECK-NEXT: [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR2]] -; CHECK-NEXT: [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR4:[0-9]+]] +; CHECK-NEXT: [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]] +; CHECK-NEXT: [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR3:[0-9]+]] ; CHECK-NEXT: [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]]) ; CHECK-NEXT: ret i8 [[R]] ; @@ -184,12 +184,11 @@ define i8 @same_parent_combine_diff_attrs_todo(i8 %x, i8 %y) { } -define i8 @same_parent_combine_diff_attrs_fail(i8 %x, i8 %y) strictfp { -; CHECK: Function Attrs: strictfp +define i8 @same_parent_combine_diff_attrs_fail(i8 %x, i8 %y) { ; CHECK-LABEL: define i8 @same_parent_combine_diff_attrs_fail( -; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) #[[ATTR0:[0-9]+]] { -; CHECK-NEXT: [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR2]] -; CHECK-NEXT: [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR5:[0-9]+]] +; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) { +; CHECK-NEXT: [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]] +; CHECK-NEXT: [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR4:[0-9]+]] ; CHECK-NEXT: [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]]) ; CHECK-NEXT: ret i8 [[R]] ; @@ -203,10 +202,10 @@ define i8 @same_parent_combine_diff_attrs_fail(i8 %x, i8 %y) strictfp { define i8 @diff_parent_combine_diff_attrs_todo(i1 %c, i8 %x, i8 %y) { ; CHECK-LABEL: define i8 @diff_parent_combine_diff_attrs_todo( ; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) { -; CHECK-NEXT: [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR2]] +; CHECK-NEXT: [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]] ; CHECK-NEXT: br i1 [[C]], label %[[T:.*]], label %[[F:.*]] ; CHECK: [[T]]: -; CHECK-NEXT: [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR6:[0-9]+]] +; CHECK-NEXT: [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR5:[0-9]+]] ; CHECK-NEXT: [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]]) ; CHECK-NEXT: ret i8 [[R]] ; CHECK: [[F]]: @@ -224,14 +223,13 @@ F: ret i8 %r2 } -define i8 @diff_parent_combine_diff_attrs_fail(i1 %c, i8 %x, i8 %y) strictfp { -; CHECK: Function Attrs: strictfp +define i8 @diff_parent_combine_diff_attrs_fail(i1 %c, i8 %x, i8 %y) { ; CHECK-LABEL: define i8 @diff_parent_combine_diff_attrs_fail( -; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR2]] +; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) { +; CHECK-NEXT: [[C1:%.*]] = call i8 @baz(i8 [[X]], i8 noundef [[Y]]) #[[ATTR1]] ; CHECK-NEXT: br i1 [[C]], label %[[T:.*]], label %[[F:.*]] ; CHECK: [[T]]: -; CHECK-NEXT: [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR5]] +; CHECK-NEXT: [[C0:%.*]] = call i8 @baz(i8 noundef [[X]], i8 noundef [[Y]]) #[[ATTR4]] ; CHECK-NEXT: [[R:%.*]] = call i8 @buz(i8 [[C0]], i8 [[C1]]) ; CHECK-NEXT: ret i8 [[R]] ; CHECK: [[F]]: @@ -249,11 +247,10 @@ F: ret i8 %r2 } -define i32 @commutative_intrinsic_intersection_failure(i32 %arg, i32 %arg1) strictfp { -; CHECK: Function Attrs: strictfp +define i32 @commutative_intrinsic_intersection_failure(i32 %arg, i32 %arg1) { ; CHECK-LABEL: define i32 @commutative_intrinsic_intersection_failure( -; CHECK-SAME: i32 [[ARG:%.*]], i32 [[ARG1:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[CALL:%.*]] = call i32 @llvm.smin.i32(i32 [[ARG]], i32 [[ARG1]]) #[[ATTR0]] +; CHECK-SAME: i32 [[ARG:%.*]], i32 [[ARG1:%.*]]) { +; CHECK-NEXT: [[CALL:%.*]] = call i32 @llvm.smin.i32(i32 [[ARG]], i32 [[ARG1]]) #[[ATTR6:[0-9]+]] ; CHECK-NEXT: [[CALL2:%.*]] = call i32 @llvm.smin.i32(i32 [[ARG1]], i32 [[ARG]]) ; CHECK-NEXT: [[OR:%.*]] = or i32 [[CALL2]], [[CALL]] ; CHECK-NEXT: ret i32 [[OR]] @@ -265,11 +262,11 @@ define i32 @commutative_intrinsic_intersection_failure(i32 %arg, i32 %arg1) stri } ;. -; CHECK: attributes #[[ATTR0]] = { strictfp } -; CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) } -; CHECK: attributes #[[ATTR2]] = { memory(none) } -; CHECK: attributes #[[ATTR3]] = { memory(read) } -; CHECK: attributes #[[ATTR4]] = { alwaysinline memory(none) } -; CHECK: attributes #[[ATTR5]] = { strictfp memory(none) } -; CHECK: attributes #[[ATTR6]] = { noinline optnone memory(none) } +; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) } +; CHECK: attributes #[[ATTR1]] = { memory(none) } +; CHECK: attributes #[[ATTR2]] = { memory(read) } +; CHECK: attributes #[[ATTR3]] = { alwaysinline memory(none) } +; CHECK: attributes #[[ATTR4]] = { strictfp memory(none) } +; CHECK: attributes #[[ATTR5]] = { noinline optnone memory(none) } +; CHECK: attributes #[[ATTR6]] = { strictfp } ;. diff --git a/llvm/test/Transforms/InstCombine/erf.ll b/llvm/test/Transforms/InstCombine/erf.ll index 57015fb490a7e..9f83acabb6fb7 100644 --- a/llvm/test/Transforms/InstCombine/erf.ll +++ b/llvm/test/Transforms/InstCombine/erf.ll @@ -75,7 +75,7 @@ define double @erf_inf() { define float @erff_inf_memory_none() { ; CHECK-LABEL: define float @erff_inf_memory_none() { -; CHECK-NEXT: [[R:%.*]] = call float @erff(float +inf) #[[ATTR2:[0-9]+]] +; CHECK-NEXT: [[R:%.*]] = call float @erff(float +inf) #[[ATTR1:[0-9]+]] ; CHECK-NEXT: ret float [[R]] ; %r = call float @erff(float 0x7FF0000000000000) readnone @@ -84,7 +84,7 @@ define float @erff_inf_memory_none() { define double @erf_inf_memory_none() { ; CHECK-LABEL: define double @erf_inf_memory_none() { -; CHECK-NEXT: [[R:%.*]] = call double @erf(double +inf) #[[ATTR2]] +; CHECK-NEXT: [[R:%.*]] = call double @erf(double +inf) #[[ATTR1]] ; CHECK-NEXT: ret double [[R]] ; %r = call double @erf(double 0x7FF0000000000000) readnone @@ -111,7 +111,7 @@ define double @erf_neg_inf() { define float @erff_neg_inf_memory_none() { ; CHECK-LABEL: define float @erff_neg_inf_memory_none() { -; CHECK-NEXT: [[R:%.*]] = call float @erff(float -inf) #[[ATTR2]] +; CHECK-NEXT: [[R:%.*]] = call float @erff(float -inf) #[[ATTR1]] ; CHECK-NEXT: ret float [[R]] ; %r = call float @erff(float 0xFFF0000000000000) readnone @@ -120,7 +120,7 @@ define float @erff_neg_inf_memory_none() { define double @erf_neg_inf_memory_none() { ; CHECK-LABEL: define double @erf_neg_inf_memory_none() { -; CHECK-NEXT: [[R:%.*]] = call double @erf(double -inf) #[[ATTR2]] +; CHECK-NEXT: [[R:%.*]] = call double @erf(double -inf) #[[ATTR1]] ; CHECK-NEXT: ret double [[R]] ; %r = call double @erf(double 0xFFF0000000000000) readnone @@ -147,7 +147,7 @@ define double @erf_nan() { define float @erff_nan_memory_none() { ; CHECK-LABEL: define float @erff_nan_memory_none() { -; CHECK-NEXT: [[R:%.*]] = call float @erff(float +qnan) #[[ATTR2]] +; CHECK-NEXT: [[R:%.*]] = call float @erff(float +qnan) #[[ATTR1]] ; CHECK-NEXT: ret float [[R]] ; %r = call float @erff(float 0x7FF8000000000000) readnone @@ -156,7 +156,7 @@ define float @erff_nan_memory_none() { define double @erf_nan_memory_none() { ; CHECK-LABEL: define double @erf_nan_memory_none() { -; CHECK-NEXT: [[R:%.*]] = call double @erf(double +qnan) #[[ATTR2]] +; CHECK-NEXT: [[R:%.*]] = call double @erf(double +qnan) #[[ATTR1]] ; CHECK-NEXT: ret double [[R]] ; %r = call double @erf(double 0x7FF8000000000000) readnone @@ -181,40 +181,36 @@ define double @erf_poison() { ret double %r } -define float @erff_const_strictfp() strictfp { -; CHECK-LABEL: define float @erff_const_strictfp( -; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { -; CHECK-NEXT: [[R:%.*]] = call float @erff(float 5.000000e-01) #[[ATTR0]] +define float @erff_const_strictfp() { +; CHECK-LABEL: define float @erff_const_strictfp() { +; CHECK-NEXT: [[R:%.*]] = call float @erff(float 5.000000e-01) #[[ATTR2:[0-9]+]] ; CHECK-NEXT: ret float [[R]] ; %r = call float @erff(float 5.000000e-01) strictfp ret float %r } -define double @erf_const_strictfp() strictfp { -; CHECK-LABEL: define double @erf_const_strictfp( -; CHECK-SAME: ) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call double @erf(double -5.000000e-01) #[[ATTR0]] +define double @erf_const_strictfp() { +; CHECK-LABEL: define double @erf_const_strictfp() { +; CHECK-NEXT: [[R:%.*]] = call double @erf(double -5.000000e-01) #[[ATTR2]] ; CHECK-NEXT: ret double [[R]] ; %r = call double @erf(double -5.000000e-01) strictfp ret double %r } -define float @erff_nan_strictfp() strictfp { -; CHECK-LABEL: define float @erff_nan_strictfp( -; CHECK-SAME: ) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call float @erff(float +qnan) #[[ATTR0]] +define float @erff_nan_strictfp() { +; CHECK-LABEL: define float @erff_nan_strictfp() { +; CHECK-NEXT: [[R:%.*]] = call float @erff(float +qnan) #[[ATTR2]] ; CHECK-NEXT: ret float [[R]] ; %r = call float @erff(float 0x7FF8000000000000) strictfp ret float %r } -define double @erf_nan_strictfp() strictfp { -; CHECK-LABEL: define double @erf_nan_strictfp( -; CHECK-SAME: ) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call double @erf(double +qnan) #[[ATTR0]] +define double @erf_nan_strictfp() { +; CHECK-LABEL: define double @erf_nan_strictfp() { +; CHECK-NEXT: [[R:%.*]] = call double @erf(double +qnan) #[[ATTR2]] ; CHECK-NEXT: ret double [[R]] ; %r = call double @erf(double 0x7FF8000000000000) strictfp diff --git a/llvm/test/Transforms/InstCombine/fdim.ll b/llvm/test/Transforms/InstCombine/fdim.ll index f167a842642db..edbdaedc975c0 100644 --- a/llvm/test/Transforms/InstCombine/fdim.ll +++ b/llvm/test/Transforms/InstCombine/fdim.ll @@ -99,10 +99,9 @@ define double @fdim_nzero() { ret double %dim } -define double @fdim_strictfp() strictfp { -; CHECK-LABEL: define double @fdim_strictfp( -; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { -; CHECK-NEXT: [[DIM:%.*]] = call double @fdim(double 1.000000e+01, double 8.000000e+00) #[[ATTR0]] +define double @fdim_strictfp() { +; CHECK-LABEL: define double @fdim_strictfp() { +; CHECK-NEXT: [[DIM:%.*]] = call double @fdim(double 1.000000e+01, double 8.000000e+00) #[[ATTR1:[0-9]+]] ; CHECK-NEXT: ret double [[DIM]] ; %dim = call double @fdim(double 10.0, double 8.0) strictfp diff --git a/llvm/test/Transforms/InstSimplify/disable_folding.ll b/llvm/test/Transforms/InstSimplify/disable_folding.ll index 02ec4877e51d9..69892012629bc 100644 --- a/llvm/test/Transforms/InstSimplify/disable_folding.ll +++ b/llvm/test/Transforms/InstSimplify/disable_folding.ll @@ -31,10 +31,9 @@ define float @test_llvm_sin() { } ; Should not be folded, even when -disable-fp-call-folding is not set, as it is marked as strictfp. -define float @test_fmax_ftz_nan_f_strictfp() #1 { -; CHECK-LABEL: define float @test_fmax_ftz_nan_f_strictfp( -; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { -; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fmax.ftz.nan.f(float 1.250000e+00, float -2.000000e+00) #[[ATTR0]] +define float @test_fmax_ftz_nan_f_strictfp() { +; CHECK-LABEL: define float @test_fmax_ftz_nan_f_strictfp() { +; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fmax.ftz.nan.f(float 1.250000e+00, float -2.000000e+00) #[[ATTR1:[0-9]+]] ; CHECK-NEXT: ret float [[RES]] ; %res = call float @llvm.nvvm.fmax.ftz.nan.f(float 1.25, float -2.0) #1 @@ -43,10 +42,9 @@ define float @test_fmax_ftz_nan_f_strictfp() #1 { ; Check that strictfp disables folding for LLVM math intrinsics like sin.f32 ; even when -disable-fp-call-folding is not set. -define float @test_llvm_sin_strictfp() #1 { -; CHECK-LABEL: define float @test_llvm_sin_strictfp( -; CHECK-SAME: ) #[[ATTR0]] { -; CHECK-NEXT: [[RES:%.*]] = call float @llvm.sin.f32(float 5.000000e-01) #[[ATTR0]] +define float @test_llvm_sin_strictfp() { +; CHECK-LABEL: define float @test_llvm_sin_strictfp() { +; CHECK-NEXT: [[RES:%.*]] = call float @llvm.sin.f32(float 5.000000e-01) #[[ATTR1]] ; CHECK-NEXT: ret float [[RES]] ; %res = call float @llvm.sin.f32(float 0.5) #1 diff --git a/llvm/test/Transforms/SimplifyCFG/X86/merge-compatible-invokes-of-landingpad.ll b/llvm/test/Transforms/SimplifyCFG/X86/merge-compatible-invokes-of-landingpad.ll index 0479fdb183f4c..95114abb4ef5f 100644 --- a/llvm/test/Transforms/SimplifyCFG/X86/merge-compatible-invokes-of-landingpad.ll +++ b/llvm/test/Transforms/SimplifyCFG/X86/merge-compatible-invokes-of-landingpad.ll @@ -411,7 +411,7 @@ define void @t7_nomerge0() personality ptr @__gxx_personality_v0 { ; CHECK-NEXT: [[C1:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C1]], label %[[IF_THEN1:.*]], label %[[IF_END:.*]] ; CHECK: [[IF_THEN1]]: -; CHECK-NEXT: invoke void @simple_throw() #[[ATTR2:[0-9]+]] +; CHECK-NEXT: invoke void @simple_throw() #[[ATTR1:[0-9]+]] ; CHECK-NEXT: to label %[[INVOKE_CONT2:.*]] unwind label %[[LPAD]] ; CHECK: [[INVOKE_CONT2]]: ; CHECK-NEXT: unreachable @@ -454,7 +454,7 @@ define void @t8_nomerge1() personality ptr @__gxx_personality_v0 { ; CHECK-NEXT: [[C0:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C0]], label %[[IF_THEN0:.*]], label %[[IF_ELSE:.*]] ; CHECK: [[IF_THEN0]]: -; CHECK-NEXT: invoke void @simple_throw() #[[ATTR2]] +; CHECK-NEXT: invoke void @simple_throw() #[[ATTR1]] ; CHECK-NEXT: to label %[[INVOKE_CONT0:.*]] unwind label %[[LPAD:.*]] ; CHECK: [[INVOKE_CONT0]]: ; CHECK-NEXT: unreachable @@ -510,7 +510,7 @@ define void @t9_nomerge2() personality ptr @__gxx_personality_v0 { ; CHECK-NEXT: [[C0:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C0]], label %[[IF_THEN0:.*]], label %[[IF_ELSE:.*]] ; CHECK: [[IF_THEN0]]: -; CHECK-NEXT: invoke void @simple_throw() #[[ATTR2]] +; CHECK-NEXT: invoke void @simple_throw() #[[ATTR1]] ; CHECK-NEXT: to label %[[INVOKE_CONT0:.*]] unwind label %[[LPAD:.*]] ; CHECK: [[INVOKE_CONT0]]: ; CHECK-NEXT: unreachable @@ -523,7 +523,7 @@ define void @t9_nomerge2() personality ptr @__gxx_personality_v0 { ; CHECK-NEXT: [[C1:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C1]], label %[[IF_THEN1:.*]], label %[[IF_END:.*]] ; CHECK: [[IF_THEN1]]: -; CHECK-NEXT: invoke void @simple_throw() #[[ATTR2]] +; CHECK-NEXT: invoke void @simple_throw() #[[ATTR1]] ; CHECK-NEXT: to label %[[INVOKE_CONT2:.*]] unwind label %[[LPAD]] ; CHECK: [[INVOKE_CONT2]]: ; CHECK-NEXT: unreachable @@ -1075,7 +1075,7 @@ define void @t17_mismatched_attrs_okay_merge_intersect() personality ptr @__gxx_ ; CHECK-NEXT: [[C1:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C1]], label %[[IF_THEN1_INVOKE]], label %[[IF_END:.*]] ; CHECK: [[IF_THEN1_INVOKE]]: -; CHECK-NEXT: invoke void @simple_throw() #[[ATTR3:[0-9]+]] +; CHECK-NEXT: invoke void @simple_throw() #[[ATTR2:[0-9]+]] ; CHECK-NEXT: to label %[[IF_THEN1_CONT:.*]] unwind label %[[LPAD]] ; CHECK: [[IF_THEN1_CONT]]: ; CHECK-NEXT: unreachable @@ -1127,7 +1127,7 @@ define void @t17_mismatched_attrs_okay_merge_intersect2() personality ptr @__gxx ; CHECK-NEXT: [[C1:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C1]], label %[[IF_THEN1_INVOKE]], label %[[IF_END:.*]] ; CHECK: [[IF_THEN1_INVOKE]]: -; CHECK-NEXT: invoke void @simple_throw() #[[ATTR3]] +; CHECK-NEXT: invoke void @simple_throw() #[[ATTR2]] ; CHECK-NEXT: to label %[[IF_THEN1_CONT:.*]] unwind label %[[LPAD]] ; CHECK: [[IF_THEN1_CONT]]: ; CHECK-NEXT: unreachable @@ -1166,15 +1166,13 @@ if.end: } -define void @t17_mismatched_attrs_prevent_merge() strictfp personality ptr @__gxx_personality_v0 { -; CHECK: Function Attrs: strictfp -; CHECK-LABEL: define void @t17_mismatched_attrs_prevent_merge( -; CHECK-SAME: ) #[[ATTR0:[0-9]+]] personality ptr @__gxx_personality_v0 { +define void @t17_mismatched_attrs_prevent_merge() personality ptr @__gxx_personality_v0 { +; CHECK-LABEL: define void @t17_mismatched_attrs_prevent_merge() personality ptr @__gxx_personality_v0 { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: [[C0:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C0]], label %[[IF_THEN0:.*]], label %[[IF_ELSE:.*]] ; CHECK: [[IF_THEN0]]: -; CHECK-NEXT: invoke void @simple_throw() #[[ATTR0]] +; CHECK-NEXT: invoke void @simple_throw() #[[ATTR3:[0-9]+]] ; CHECK-NEXT: to label %[[INVOKE_CONT0:.*]] unwind label %[[LPAD:.*]] ; CHECK: [[INVOKE_CONT0]]: ; CHECK-NEXT: unreachable @@ -1241,7 +1239,7 @@ define void @t18_attributes_are_preserved() personality ptr @__gxx_personality_v ; CHECK-NEXT: [[C1:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C1]], label %[[IF_THEN1_INVOKE]], label %[[IF_END:.*]] ; CHECK: [[IF_THEN1_INVOKE]]: -; CHECK-NEXT: invoke void @simple_throw() #[[ATTR3]] +; CHECK-NEXT: invoke void @simple_throw() #[[ATTR2]] ; CHECK-NEXT: to label %[[IF_THEN1_CONT:.*]] unwind label %[[LPAD]] ; CHECK: [[IF_THEN1_CONT]]: ; CHECK-NEXT: unreachable @@ -2677,8 +2675,8 @@ declare void @consume(i32) declare dso_local i32 @__gxx_personality_v0(...) ;. -; CHECK: attributes #[[ATTR0]] = { strictfp } -; CHECK: attributes #[[ATTR1:[0-9]+]] = { noreturn } -; CHECK: attributes #[[ATTR2]] = { nomerge } -; CHECK: attributes #[[ATTR3]] = { memory(none) } +; CHECK: attributes #[[ATTR0:[0-9]+]] = { noreturn } +; CHECK: attributes #[[ATTR1]] = { nomerge } +; CHECK: attributes #[[ATTR2]] = { memory(none) } +; CHECK: attributes #[[ATTR3]] = { strictfp } ;. diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp index f6a89cde0901b..48d461cab330a 100644 --- a/llvm/unittests/IR/VerifierTest.cpp +++ b/llvm/unittests/IR/VerifierTest.cpp @@ -699,28 +699,4 @@ TEST(VerifierTest, IntrinsicRetInvalidStruct) { } } -TEST(VerifierTest, InvalidStrictFPAttribute) { - LLVMContext Ctx; - Module M("M", Ctx); - FunctionType *FuncTy = - FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false); - Function *F = - Function::Create(FuncTy, Function::ExternalLinkage, "strictfp_test", M); - BasicBlock *Entry = BasicBlock::Create(Ctx, "entry", F); - Type *FloatTy = Type::getFloatTy(Ctx); - - Function *CosF = - Intrinsic::getOrInsertDeclaration(&M, Intrinsic::cos, FloatTy); - CallInst *CI = CallInst::Create(CosF, ConstantFP::getNullValue(FloatTy), - "strictfp_call", Entry); - CI->addFnAttr(Attribute::StrictFP); - ReturnInst::Create(Ctx, Entry); - - std::string Error; - raw_string_ostream ErrorOS(Error); - EXPECT_TRUE(verifyModule(M, &ErrorOS)); - EXPECT_TRUE(StringRef(Error).starts_with( - "call site marked strictfp without caller function marked strictfp")) - << Error; -} } // end anonymous namespace diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir index 7a362ebe2421e..6910a06992b55 100644 --- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir @@ -596,9 +596,9 @@ llvm.func @masked_gather_scatter_intrinsics(%M: vector<7 x !llvm.ptr>, %mask: ve } // CHECK-LABEL: @masked_expand_compress_intrinsics -llvm.func @masked_expand_compress_intrinsics(%ptr: !llvm.ptr, %mask: vector<7xi1>, %passthrough: vector<7xf32>) { +llvm.func @masked_expand_compress_intrinsics(%ptr: !llvm.ptr, %mask: vector<7xi1>, %passthru: vector<7xf32>) { // CHECK: call <7 x float> @llvm.masked.expandload.v7f32.p0(ptr %{{.*}}, <7 x i1> %{{.*}}, <7 x float> %{{.*}}) - %0 = "llvm.intr.masked.expandload"(%ptr, %mask, %passthrough) + %0 = "llvm.intr.masked.expandload"(%ptr, %mask, %passthru) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> (vector<7xf32>) // CHECK: call void @llvm.masked.compressstore.v7f32.p0(<7 x float> %{{.*}}, ptr %{{.*}}, <7 x i1> %{{.*}}) "llvm.intr.masked.compressstore"(%0, %ptr, %mask) @@ -607,9 +607,9 @@ llvm.func @masked_expand_compress_intrinsics(%ptr: !llvm.ptr, %mask: vector<7xi1 } // CHECK-LABEL: @masked_expand_compress_intrinsics_with_alignment -llvm.func @masked_expand_compress_intrinsics_with_alignment(%ptr: !llvm.ptr, %mask: vector<7xi1>, %passthrough: vector<7xf32>) { +llvm.func @masked_expand_compress_intrinsics_with_alignment(%ptr: !llvm.ptr, %mask: vector<7xi1>, %passthru: vector<7xf32>) { // CHECK: call <7 x float> @llvm.masked.expandload.v7f32.p0(ptr align 8 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> %{{.*}}) - %0 = "llvm.intr.masked.expandload"(%ptr, %mask, %passthrough) {arg_attrs = [{llvm.align = 8 : i32}, {}, {}]} + %0 = "llvm.intr.masked.expandload"(%ptr, %mask, %passthru) {arg_attrs = [{llvm.align = 8 : i32}, {}, {}]} : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> (vector<7xf32>) // CHECK: call void @llvm.masked.compressstore.v7f32.p0(<7 x float> %{{.*}}, ptr align 8 %{{.*}}, <7 x i1> %{{.*}}) "llvm.intr.masked.compressstore"(%0, %ptr, %mask) {arg_attrs = [{}, {llvm.align = 8 : i32}, {}]} @@ -1202,7 +1202,7 @@ llvm.func @vector_ptrmask(%p: vector<8 x !llvm.ptr>, %mask: vector<8 x i64>) -> } // CHECK-LABEL: @experimental_constrained_fadd -llvm.func @experimental_constrained_fadd(%s: f32, %v: vector<4 x f32>) attributes { passthrough = ["strictfp"] } { +llvm.func @experimental_constrained_fadd(%s: f32, %v: vector<4 x f32>) { // CHECK: call float @llvm.experimental.constrained.fadd.f32( // CHECK: metadata !"round.towardzero" // CHECK: metadata !"fpexcept.ignore" @@ -1215,7 +1215,7 @@ llvm.func @experimental_constrained_fadd(%s: f32, %v: vector<4 x f32>) attribute } // CHECK-LABEL: @experimental_constrained_fsub -llvm.func @experimental_constrained_fsub(%s: f32, %v: vector<4 x f32>) attributes { passthrough = ["strictfp"] } { +llvm.func @experimental_constrained_fsub(%s: f32, %v: vector<4 x f32>) { // CHECK: call float @llvm.experimental.constrained.fsub.f32( // CHECK: metadata !"round.towardzero" // CHECK: metadata !"fpexcept.ignore" @@ -1228,7 +1228,7 @@ llvm.func @experimental_constrained_fsub(%s: f32, %v: vector<4 x f32>) attribute } // CHECK-LABEL: @experimental_constrained_fmul -llvm.func @experimental_constrained_fmul(%s: f32, %v: vector<4 x f32>) attributes { passthrough = ["strictfp"] } { +llvm.func @experimental_constrained_fmul(%s: f32, %v: vector<4 x f32>) { // CHECK: call float @llvm.experimental.constrained.fmul.f32( // CHECK: metadata !"round.towardzero" // CHECK: metadata !"fpexcept.ignore" @@ -1241,7 +1241,7 @@ llvm.func @experimental_constrained_fmul(%s: f32, %v: vector<4 x f32>) attribute } // CHECK-LABEL: @experimental_constrained_fdiv -llvm.func @experimental_constrained_fdiv(%s: f32, %v: vector<4 x f32>) attributes { passthrough = ["strictfp"] } { +llvm.func @experimental_constrained_fdiv(%s: f32, %v: vector<4 x f32>) { // CHECK: call float @llvm.experimental.constrained.fdiv.f32( // CHECK: metadata !"round.towardzero" // CHECK: metadata !"fpexcept.ignore" @@ -1254,7 +1254,7 @@ llvm.func @experimental_constrained_fdiv(%s: f32, %v: vector<4 x f32>) attribute } // CHECK-LABEL: @experimental_constrained_frem -llvm.func @experimental_constrained_frem(%s: f32, %v: vector<4 x f32>) attributes { passthrough = ["strictfp"] } { +llvm.func @experimental_constrained_frem(%s: f32, %v: vector<4 x f32>) { // CHECK: call float @llvm.experimental.constrained.frem.f32( // CHECK: metadata !"round.towardzero" // CHECK: metadata !"fpexcept.ignore" @@ -1267,7 +1267,7 @@ llvm.func @experimental_constrained_frem(%s: f32, %v: vector<4 x f32>) attribute } // CHECK-LABEL: @experimental_constrained_fma -llvm.func @experimental_constrained_fma(%s: f32, %v: vector<4 x f32>) attributes { passthrough = ["strictfp"] } { +llvm.func @experimental_constrained_fma(%s: f32, %v: vector<4 x f32>) { // CHECK: call float @llvm.experimental.constrained.fma.f32( // CHECK: metadata !"round.towardzero" // CHECK: metadata !"fpexcept.ignore" @@ -1280,7 +1280,7 @@ llvm.func @experimental_constrained_fma(%s: f32, %v: vector<4 x f32>) attributes } // CHECK-LABEL: @experimental_constrained_fmuladd -llvm.func @experimental_constrained_fmuladd(%s: f32, %v: vector<4 x f32>) attributes { passthrough = ["strictfp"] } { +llvm.func @experimental_constrained_fmuladd(%s: f32, %v: vector<4 x f32>) { // CHECK: call float @llvm.experimental.constrained.fmuladd.f32( // CHECK: metadata !"round.towardzero" // CHECK: metadata !"fpexcept.ignore" @@ -1293,7 +1293,7 @@ llvm.func @experimental_constrained_fmuladd(%s: f32, %v: vector<4 x f32>) attrib } // CHECK-LABEL: @experimental_constrained_uitofp -llvm.func @experimental_constrained_uitofp(%s: i32, %v: vector<4 x i32>) attributes { passthrough = ["strictfp"] } { +llvm.func @experimental_constrained_uitofp(%s: i32, %v: vector<4 x i32>) { // CHECK: call float @llvm.experimental.constrained.uitofp.f32.i32( // CHECK: metadata !"round.towardzero" // CHECK: metadata !"fpexcept.ignore" @@ -1322,7 +1322,7 @@ llvm.func @experimental_constrained_uitofp(%s: i32, %v: vector<4 x i32>) attribu } // CHECK-LABEL: @experimental_constrained_sitofp -llvm.func @experimental_constrained_sitofp(%s: i32, %v: vector<4 x i32>) attributes { passthrough = ["strictfp"] } { +llvm.func @experimental_constrained_sitofp(%s: i32, %v: vector<4 x i32>) { // CHECK: call float @llvm.experimental.constrained.sitofp.f32.i32( // CHECK: metadata !"round.towardzero" // CHECK: metadata !"fpexcept.ignore" @@ -1351,7 +1351,7 @@ llvm.func @experimental_constrained_sitofp(%s: i32, %v: vector<4 x i32>) attribu } // CHECK-LABEL: @experimental_constrained_fptrunc -llvm.func @experimental_constrained_fptrunc(%s: f64, %v: vector<4xf32>) attributes { passthrough = ["strictfp"] } { +llvm.func @experimental_constrained_fptrunc(%s: f64, %v: vector<4xf32>) { // CHECK: call float @llvm.experimental.constrained.fptrunc.f32.f64( // CHECK: metadata !"round.towardzero" // CHECK: metadata !"fpexcept.ignore" @@ -1380,7 +1380,7 @@ llvm.func @experimental_constrained_fptrunc(%s: f64, %v: vector<4xf32>) attribut } // CHECK-LABEL: @experimental_constrained_fpext -llvm.func @experimental_constrained_fpext(%s: f32, %v: vector<4xf32>) attributes { passthrough = ["strictfp"] } { +llvm.func @experimental_constrained_fpext(%s: f32, %v: vector<4xf32>) { // CHECK: call double @llvm.experimental.constrained.fpext.f64.f32( // CHECK: metadata !"fpexcept.ignore" %0 = llvm.intr.experimental.constrained.fpext %s ignore : f32 to f64 @@ -1426,13 +1426,13 @@ llvm.func @vector_scmp(%a: vector<4 x i32>, %b: vector<4 x i32>) -> vector<4 x i // Check that intrinsics are declared with appropriate types. // CHECK-DAG: declare float @llvm.fma.f32(float, float, float) -// CHECK-DAG: declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>) +// CHECK-DAG: declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>) #0 // CHECK-DAG: declare float @llvm.fmuladd.f32(float, float, float) -// CHECK-DAG: declare <8 x float> @llvm.fmuladd.v8f32(<8 x float>, <8 x float>, <8 x float>) +// CHECK-DAG: declare <8 x float> @llvm.fmuladd.v8f32(<8 x float>, <8 x float>, <8 x float>) #0 // CHECK-DAG: declare void @llvm.prefetch.p0(ptr readonly captures(none), i32 immarg range(i32 0, 2), i32 immarg range(i32 0, 4), i32 immarg range(i32 0, 2)) // CHECK-DAG: declare i1 @llvm.is.fpclass.f32(float, i32 immarg) // CHECK-DAG: declare float @llvm.exp.f32(float) -// CHECK-DAG: declare <8 x float> @llvm.exp.v8f32(<8 x float>) +// CHECK-DAG: declare <8 x float> @llvm.exp.v8f32(<8 x float>) #0 // CHECK-DAG: declare float @llvm.exp2.f32(float) // CHECK-DAG: declare <8 x float> @llvm.exp2.v8f32(<8 x float>) // CHECK-DAG: declare float @llvm.exp10.f32(float) @@ -1442,19 +1442,19 @@ llvm.func @vector_scmp(%a: vector<4 x i32>, %b: vector<4 x i32>) -> vector<4 x i // CHECK-DAG: declare { float, i32 } @llvm.frexp.f32.i32(float) // CHECK-DAG: declare { <8 x float>, i32 } @llvm.frexp.v8f32.i32(<8 x float>) // CHECK-DAG: declare float @llvm.log.f32(float) -// CHECK-DAG: declare <8 x float> @llvm.log.v8f32(<8 x float>) +// CHECK-DAG: declare <8 x float> @llvm.log.v8f32(<8 x float>) #0 // CHECK-DAG: declare float @llvm.log10.f32(float) -// CHECK-DAG: declare <8 x float> @llvm.log10.v8f32(<8 x float>) +// CHECK-DAG: declare <8 x float> @llvm.log10.v8f32(<8 x float>) #0 // CHECK-DAG: declare float @llvm.log2.f32(float) -// CHECK-DAG: declare <8 x float> @llvm.log2.v8f32(<8 x float>) +// CHECK-DAG: declare <8 x float> @llvm.log2.v8f32(<8 x float>) #0 // CHECK-DAG: declare float @llvm.fabs.f32(float) -// CHECK-DAG: declare <8 x float> @llvm.fabs.v8f32(<8 x float>) +// CHECK-DAG: declare <8 x float> @llvm.fabs.v8f32(<8 x float>) #0 // CHECK-DAG: declare float @llvm.sqrt.f32(float) -// CHECK-DAG: declare <8 x float> @llvm.sqrt.v8f32(<8 x float>) +// CHECK-DAG: declare <8 x float> @llvm.sqrt.v8f32(<8 x float>) #0 // CHECK-DAG: declare float @llvm.ceil.f32(float) -// CHECK-DAG: declare <8 x float> @llvm.ceil.v8f32(<8 x float>) +// CHECK-DAG: declare <8 x float> @llvm.ceil.v8f32(<8 x float>) #0 // CHECK-DAG: declare float @llvm.cos.f32(float) -// CHECK-DAG: declare <8 x float> @llvm.cos.v8f32(<8 x float>) +// CHECK-DAG: declare <8 x float> @llvm.cos.v8f32(<8 x float>) #0 // CHECK-DAG: declare { float, float } @llvm.sincos.f32(float) // CHECK-DAG: declare { <8 x float>, <8 x float> } @llvm.sincos.v8f32(<8 x float>) // CHECK-DAG: declare float @llvm.copysign.f32(float, float) From 48883c5aa6428ea91cf22417a519e6302e88a529 Mon Sep 17 00:00:00 2001 From: Derek Schuff Date: Tue, 21 Jul 2026 11:12:39 -0700 Subject: [PATCH 06/50] [WebAssembly] Swap encoding bits for HAS_MEMORY_ORDER for relaxed atomics (#210854) Following the decision in https://github.com/WebAssembly/relaxed-atomics/issues/3 --- llvm/include/llvm/BinaryFormat/Wasm.h | 2 +- llvm/test/MC/WebAssembly/atomics-orderings.s | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/llvm/include/llvm/BinaryFormat/Wasm.h b/llvm/include/llvm/BinaryFormat/Wasm.h index 5c7571b233ef2..a73272dc17048 100644 --- a/llvm/include/llvm/BinaryFormat/Wasm.h +++ b/llvm/include/llvm/BinaryFormat/Wasm.h @@ -88,7 +88,7 @@ enum : unsigned { // RMW/CMPXCHG operations have 2 orderings but they must currently match. WASM_MEM_ORDER_RMW_ACQ_REL = 0x11, }; -const unsigned WASM_MEMARG_HAS_MEM_ORDER = 0x20; +const unsigned WASM_MEMARG_HAS_MEM_ORDER = 0x10; // Kinds of externals (for imports and exports). enum : unsigned { diff --git a/llvm/test/MC/WebAssembly/atomics-orderings.s b/llvm/test/MC/WebAssembly/atomics-orderings.s index f493b108d6b88..fe1c2d6006e44 100644 --- a/llvm/test/MC/WebAssembly/atomics-orderings.s +++ b/llvm/test/MC/WebAssembly/atomics-orderings.s @@ -34,27 +34,27 @@ main: # DISASM: i32.atomic.load seqcst 0 i32.atomic.load seqcst 0 - # CHECK: i32.atomic.load acqrel 0 # encoding: [0xfe,0x10,0x22,0x01,0x00] + # CHECK: i32.atomic.load acqrel 0 # encoding: [0xfe,0x10,0x12,0x01,0x00] # DISASM: i32.atomic.load acqrel 0 i32.atomic.load acqrel 0 - # CHECK: i64.atomic.load acqrel 0 # encoding: [0xfe,0x11,0x23,0x01,0x00] + # CHECK: i64.atomic.load acqrel 0 # encoding: [0xfe,0x11,0x13,0x01,0x00] # DISASM: i64.atomic.load acqrel 0 i64.atomic.load acqrel 0 - # CHECK: i32.atomic.store acqrel 0 # encoding: [0xfe,0x17,0x22,0x01,0x00] + # CHECK: i32.atomic.store acqrel 0 # encoding: [0xfe,0x17,0x12,0x01,0x00] # DISASM: i32.atomic.store acqrel 0 i32.atomic.store acqrel 0 - # CHECK: i64.atomic.store acqrel 8 # encoding: [0xfe,0x18,0x23,0x01,0x08] + # CHECK: i64.atomic.store acqrel 8 # encoding: [0xfe,0x18,0x13,0x01,0x08] # DISASM: i64.atomic.store acqrel 8 i64.atomic.store acqrel 8 - # CHECK: i32.atomic.rmw.add acqrel 0 # encoding: [0xfe,0x1e,0x22,0x11,0x00] + # CHECK: i32.atomic.rmw.add acqrel 0 # encoding: [0xfe,0x1e,0x12,0x11,0x00] # DISASM: i32.atomic.rmw.add acqrel 0 i32.atomic.rmw.add acqrel 0 - # CHECK: i64.atomic.rmw.cmpxchg acqrel 0 # encoding: [0xfe,0x49,0x23,0x11,0x00] + # CHECK: i64.atomic.rmw.cmpxchg acqrel 0 # encoding: [0xfe,0x49,0x13,0x11,0x00] # DISASM: i64.atomic.rmw.cmpxchg acqrel 0 i64.atomic.rmw.cmpxchg acqrel 0 @@ -62,7 +62,7 @@ main: # DISASM: i32.atomic.load8_u seqcst 0 i32.atomic.load8_u seqcst 0:p2align=0 - # CHECK: i64.atomic.rmw32.xchg_u acqrel 0 # encoding: [0xfe,0x47,0x22,0x11,0x00] + # CHECK: i64.atomic.rmw32.xchg_u acqrel 0 # encoding: [0xfe,0x47,0x12,0x11,0x00] # DISASM: i64.atomic.rmw32.xchg_u acqrel 0 i64.atomic.rmw32.xchg_u acqrel 0 From 15de5a2c4c14070d0c42ab0ca436ce4e192a9baf Mon Sep 17 00:00:00 2001 From: Brad Smith Date: Tue, 21 Jul 2026 14:17:33 -0400 Subject: [PATCH 07/50] [Driver] Remove SUSE SLES special casing for hash-style (#210223) --- clang/lib/Driver/ToolChains/Linux.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp index 6dca14d8bf0a8..a01442f3b23e2 100644 --- a/clang/lib/Driver/ToolChains/Linux.cpp +++ b/clang/lib/Driver/ToolChains/Linux.cpp @@ -317,13 +317,8 @@ Linux::Linux(const Driver &D, const llvm::Triple &Triple, const ArgList &Args) // .gnu.hash needs symbols to be grouped by hash code whereas the MIPS // ABI requires a mapping between the GOT and the symbol table. // Hexagon linker/loader does not support .gnu.hash. - // SUSE SLES 11 will stop being supported Mar 2028. - if (!IsMips && !IsHexagon) { - if (Distro.IsOpenSUSE()) - ExtraOpts.push_back("--hash-style=both"); - else - ExtraOpts.push_back("--hash-style=gnu"); - } + if (!IsMips && !IsHexagon) + ExtraOpts.push_back("--hash-style=gnu"); #ifdef ENABLE_LINKER_BUILD_ID ExtraOpts.push_back("--build-id"); From d81e22df458858e0ab1fdd56710d1d12251674b0 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Tue, 21 Jul 2026 14:18:14 -0400 Subject: [PATCH 08/50] [SLP][NFC]Add extra tests for blended loads vectorization, NFC Reviewers: Pull Request: https://github.com/llvm/llvm-project/pull/211080 --- .../SLPVectorizer/X86/masked-blended-loads.ll | 264 ++++++++++++++++++ 1 file changed, 264 insertions(+) diff --git a/llvm/test/Transforms/SLPVectorizer/X86/masked-blended-loads.ll b/llvm/test/Transforms/SLPVectorizer/X86/masked-blended-loads.ll index dd8b0e584bcfc..68b2b14a03d37 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/masked-blended-loads.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/masked-blended-loads.ll @@ -185,3 +185,267 @@ entry: %vecins.15 = insertelement <16 x i32> %vecins.14, i32 %l15, i64 15 ret <16 x i32> %vecins.15 } + +define <8 x i32> @test_cmp_two_vectors(ptr %a, ptr %b, <8 x i32> %x, <8 x i32> %y) { +; CHECK-LABEL: define <8 x i32> @test_cmp_two_vectors( +; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], <8 x i32> [[X:%.*]], <8 x i32> [[Y:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt <8 x i32> [[X]], [[Y]] +; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i1> [[TMP0]], i64 0 +; CHECK-NEXT: [[P0:%.*]] = select i1 [[TMP1]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[L0:%.*]] = load i32, ptr [[P0]], align 4 +; CHECK-NEXT: [[V0:%.*]] = insertelement <8 x i32> poison, i32 [[L0]], i64 0 +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i1> [[TMP0]], i64 1 +; CHECK-NEXT: [[S1:%.*]] = select i1 [[TMP2]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[S1]], i64 4 +; CHECK-NEXT: [[L1:%.*]] = load i32, ptr [[P1]], align 4 +; CHECK-NEXT: [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[L1]], i64 1 +; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i1> [[TMP0]], i64 2 +; CHECK-NEXT: [[S2:%.*]] = select i1 [[TMP3]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[S2]], i64 8 +; CHECK-NEXT: [[L2:%.*]] = load i32, ptr [[P2]], align 4 +; CHECK-NEXT: [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[L2]], i64 2 +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i1> [[TMP0]], i64 3 +; CHECK-NEXT: [[S3:%.*]] = select i1 [[TMP9]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[S3]], i64 12 +; CHECK-NEXT: [[L3:%.*]] = load i32, ptr [[P3]], align 4 +; CHECK-NEXT: [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[L3]], i64 3 +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[TMP0]], i64 4 +; CHECK-NEXT: [[S4:%.*]] = select i1 [[TMP5]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[S4]], i64 16 +; CHECK-NEXT: [[L4:%.*]] = load i32, ptr [[P4]], align 4 +; CHECK-NEXT: [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[L4]], i64 4 +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP0]], i64 5 +; CHECK-NEXT: [[S5:%.*]] = select i1 [[TMP6]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[S5]], i64 20 +; CHECK-NEXT: [[L5:%.*]] = load i32, ptr [[P5]], align 4 +; CHECK-NEXT: [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[L5]], i64 5 +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i1> [[TMP0]], i64 6 +; CHECK-NEXT: [[S6:%.*]] = select i1 [[TMP7]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds i8, ptr [[S6]], i64 24 +; CHECK-NEXT: [[L6:%.*]] = load i32, ptr [[P6]], align 4 +; CHECK-NEXT: [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[L6]], i64 6 +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[TMP0]], i64 7 +; CHECK-NEXT: [[S7:%.*]] = select i1 [[TMP8]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i8, ptr [[S7]], i64 28 +; CHECK-NEXT: [[L7:%.*]] = load i32, ptr [[P7]], align 4 +; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> [[V6]], i32 [[L7]], i64 7 +; CHECK-NEXT: ret <8 x i32> [[TMP4]] +; +entry: + %x0 = extractelement <8 x i32> %x, i64 0 + %y0 = extractelement <8 x i32> %y, i64 0 + %c0 = icmp sgt i32 %x0, %y0 + %p0 = select i1 %c0, ptr %a, ptr %b + %l0 = load i32, ptr %p0, align 4 + %v0 = insertelement <8 x i32> poison, i32 %l0, i64 0 + %x1 = extractelement <8 x i32> %x, i64 1 + %y1 = extractelement <8 x i32> %y, i64 1 + %c1 = icmp sgt i32 %x1, %y1 + %s1 = select i1 %c1, ptr %a, ptr %b + %p1 = getelementptr inbounds i8, ptr %s1, i64 4 + %l1 = load i32, ptr %p1, align 4 + %v1 = insertelement <8 x i32> %v0, i32 %l1, i64 1 + %x2 = extractelement <8 x i32> %x, i64 2 + %y2 = extractelement <8 x i32> %y, i64 2 + %c2 = icmp sgt i32 %x2, %y2 + %s2 = select i1 %c2, ptr %a, ptr %b + %p2 = getelementptr inbounds i8, ptr %s2, i64 8 + %l2 = load i32, ptr %p2, align 4 + %v2 = insertelement <8 x i32> %v1, i32 %l2, i64 2 + %x3 = extractelement <8 x i32> %x, i64 3 + %y3 = extractelement <8 x i32> %y, i64 3 + %c3 = icmp sgt i32 %x3, %y3 + %s3 = select i1 %c3, ptr %a, ptr %b + %p3 = getelementptr inbounds i8, ptr %s3, i64 12 + %l3 = load i32, ptr %p3, align 4 + %v3 = insertelement <8 x i32> %v2, i32 %l3, i64 3 + %x4 = extractelement <8 x i32> %x, i64 4 + %y4 = extractelement <8 x i32> %y, i64 4 + %c4 = icmp sgt i32 %x4, %y4 + %s4 = select i1 %c4, ptr %a, ptr %b + %p4 = getelementptr inbounds i8, ptr %s4, i64 16 + %l4 = load i32, ptr %p4, align 4 + %v4 = insertelement <8 x i32> %v3, i32 %l4, i64 4 + %x5 = extractelement <8 x i32> %x, i64 5 + %y5 = extractelement <8 x i32> %y, i64 5 + %c5 = icmp sgt i32 %x5, %y5 + %s5 = select i1 %c5, ptr %a, ptr %b + %p5 = getelementptr inbounds i8, ptr %s5, i64 20 + %l5 = load i32, ptr %p5, align 4 + %v5 = insertelement <8 x i32> %v4, i32 %l5, i64 5 + %x6 = extractelement <8 x i32> %x, i64 6 + %y6 = extractelement <8 x i32> %y, i64 6 + %c6 = icmp sgt i32 %x6, %y6 + %s6 = select i1 %c6, ptr %a, ptr %b + %p6 = getelementptr inbounds i8, ptr %s6, i64 24 + %l6 = load i32, ptr %p6, align 4 + %v6 = insertelement <8 x i32> %v5, i32 %l6, i64 6 + %x7 = extractelement <8 x i32> %x, i64 7 + %y7 = extractelement <8 x i32> %y, i64 7 + %c7 = icmp sgt i32 %x7, %y7 + %s7 = select i1 %c7, ptr %a, ptr %b + %p7 = getelementptr inbounds i8, ptr %s7, i64 28 + %l7 = load i32, ptr %p7, align 4 + %v7 = insertelement <8 x i32> %v6, i32 %l7, i64 7 + ret <8 x i32> %v7 +} + +define <8 x i32> @test_and_of_compares(ptr %a, ptr %b, <8 x i32> %x, <8 x i32> %y, <8 x i32> %z) { +; CHECK-LABEL: define <8 x i32> @test_and_of_compares( +; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], <8 x i32> [[X:%.*]], <8 x i32> [[Y:%.*]], <8 x i32> [[Z:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> [[Y]], <8 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[Z]], <8 x i32> [[X]], <8 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = icmp slt <8 x i32> [[TMP0]], [[TMP1]] +; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i1> [[TMP2]], i64 6 +; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP2]], i64 7 +; CHECK-NEXT: [[C0:%.*]] = and i1 [[TMP4]], [[TMP3]] +; CHECK-NEXT: [[P0:%.*]] = select i1 [[C0]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[L0:%.*]] = load i32, ptr [[P0]], align 4 +; CHECK-NEXT: [[V0:%.*]] = insertelement <8 x i32> poison, i32 [[L0]], i64 0 +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[TMP2]], i64 4 +; CHECK-NEXT: [[TMP31:%.*]] = extractelement <8 x i1> [[TMP2]], i64 5 +; CHECK-NEXT: [[C1:%.*]] = and i1 [[TMP31]], [[TMP5]] +; CHECK-NEXT: [[S1:%.*]] = select i1 [[C1]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[S1]], i64 4 +; CHECK-NEXT: [[L1:%.*]] = load i32, ptr [[P1]], align 4 +; CHECK-NEXT: [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[L1]], i64 1 +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i1> [[TMP2]], i64 2 +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[TMP2]], i64 3 +; CHECK-NEXT: [[C2:%.*]] = and i1 [[TMP8]], [[TMP7]] +; CHECK-NEXT: [[S2:%.*]] = select i1 [[C2]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[S2]], i64 8 +; CHECK-NEXT: [[L2:%.*]] = load i32, ptr [[P2]], align 4 +; CHECK-NEXT: [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[L2]], i64 2 +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i1> [[TMP2]], i64 0 +; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i1> [[TMP2]], i64 1 +; CHECK-NEXT: [[C3:%.*]] = and i1 [[TMP10]], [[TMP9]] +; CHECK-NEXT: [[S3:%.*]] = select i1 [[C3]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[S3]], i64 12 +; CHECK-NEXT: [[L3:%.*]] = load i32, ptr [[P3]], align 4 +; CHECK-NEXT: [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[L3]], i64 3 +; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> [[Z]], <2 x i32> +; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i32> [[Y]], <8 x i32> [[X]], <2 x i32> +; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <2 x i32> [[TMP11]], [[TMP12]] +; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP13]], i64 0 +; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x i1> [[TMP13]], i64 1 +; CHECK-NEXT: [[C4:%.*]] = and i1 [[TMP14]], [[TMP15]] +; CHECK-NEXT: [[S4:%.*]] = select i1 [[C4]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[S4]], i64 16 +; CHECK-NEXT: [[L4:%.*]] = load i32, ptr [[P4]], align 4 +; CHECK-NEXT: [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[L4]], i64 4 +; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> [[Z]], <2 x i32> +; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <8 x i32> [[Y]], <8 x i32> [[X]], <2 x i32> +; CHECK-NEXT: [[TMP18:%.*]] = icmp sgt <2 x i32> [[TMP16]], [[TMP17]] +; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x i1> [[TMP18]], i64 0 +; CHECK-NEXT: [[TMP20:%.*]] = extractelement <2 x i1> [[TMP18]], i64 1 +; CHECK-NEXT: [[C5:%.*]] = and i1 [[TMP19]], [[TMP20]] +; CHECK-NEXT: [[S5:%.*]] = select i1 [[C5]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[S5]], i64 20 +; CHECK-NEXT: [[L5:%.*]] = load i32, ptr [[P5]], align 4 +; CHECK-NEXT: [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[L5]], i64 5 +; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> [[Z]], <2 x i32> +; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <8 x i32> [[Y]], <8 x i32> [[X]], <2 x i32> +; CHECK-NEXT: [[TMP23:%.*]] = icmp sgt <2 x i32> [[TMP21]], [[TMP22]] +; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x i1> [[TMP23]], i64 0 +; CHECK-NEXT: [[TMP25:%.*]] = extractelement <2 x i1> [[TMP23]], i64 1 +; CHECK-NEXT: [[C6:%.*]] = and i1 [[TMP24]], [[TMP25]] +; CHECK-NEXT: [[S6:%.*]] = select i1 [[C6]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds i8, ptr [[S6]], i64 24 +; CHECK-NEXT: [[L6:%.*]] = load i32, ptr [[P6]], align 4 +; CHECK-NEXT: [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[L6]], i64 6 +; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> [[Z]], <2 x i32> +; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <8 x i32> [[Y]], <8 x i32> [[X]], <2 x i32> +; CHECK-NEXT: [[TMP28:%.*]] = icmp sgt <2 x i32> [[TMP26]], [[TMP27]] +; CHECK-NEXT: [[TMP29:%.*]] = extractelement <2 x i1> [[TMP28]], i64 0 +; CHECK-NEXT: [[TMP30:%.*]] = extractelement <2 x i1> [[TMP28]], i64 1 +; CHECK-NEXT: [[C7:%.*]] = and i1 [[TMP29]], [[TMP30]] +; CHECK-NEXT: [[S7:%.*]] = select i1 [[C7]], ptr [[A]], ptr [[B]] +; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i8, ptr [[S7]], i64 28 +; CHECK-NEXT: [[L7:%.*]] = load i32, ptr [[P7]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x i32> [[V6]], i32 [[L7]], i64 7 +; CHECK-NEXT: ret <8 x i32> [[TMP6]] +; +entry: + %x0 = extractelement <8 x i32> %x, i64 0 + %y0 = extractelement <8 x i32> %y, i64 0 + %z0 = extractelement <8 x i32> %z, i64 0 + %m0 = icmp sgt i32 %x0, %y0 + %n0 = icmp slt i32 %x0, %z0 + %c0 = and i1 %m0, %n0 + %p0 = select i1 %c0, ptr %a, ptr %b + %l0 = load i32, ptr %p0, align 4 + %v0 = insertelement <8 x i32> poison, i32 %l0, i64 0 + %x1 = extractelement <8 x i32> %x, i64 1 + %y1 = extractelement <8 x i32> %y, i64 1 + %z1 = extractelement <8 x i32> %z, i64 1 + %m1 = icmp sgt i32 %x1, %y1 + %n1 = icmp slt i32 %x1, %z1 + %c1 = and i1 %m1, %n1 + %s1 = select i1 %c1, ptr %a, ptr %b + %p1 = getelementptr inbounds i8, ptr %s1, i64 4 + %l1 = load i32, ptr %p1, align 4 + %v1 = insertelement <8 x i32> %v0, i32 %l1, i64 1 + %x2 = extractelement <8 x i32> %x, i64 2 + %y2 = extractelement <8 x i32> %y, i64 2 + %z2 = extractelement <8 x i32> %z, i64 2 + %m2 = icmp sgt i32 %x2, %y2 + %n2 = icmp slt i32 %x2, %z2 + %c2 = and i1 %m2, %n2 + %s2 = select i1 %c2, ptr %a, ptr %b + %p2 = getelementptr inbounds i8, ptr %s2, i64 8 + %l2 = load i32, ptr %p2, align 4 + %v2 = insertelement <8 x i32> %v1, i32 %l2, i64 2 + %x3 = extractelement <8 x i32> %x, i64 3 + %y3 = extractelement <8 x i32> %y, i64 3 + %z3 = extractelement <8 x i32> %z, i64 3 + %m3 = icmp sgt i32 %x3, %y3 + %n3 = icmp slt i32 %x3, %z3 + %c3 = and i1 %m3, %n3 + %s3 = select i1 %c3, ptr %a, ptr %b + %p3 = getelementptr inbounds i8, ptr %s3, i64 12 + %l3 = load i32, ptr %p3, align 4 + %v3 = insertelement <8 x i32> %v2, i32 %l3, i64 3 + %x4 = extractelement <8 x i32> %x, i64 4 + %y4 = extractelement <8 x i32> %y, i64 4 + %z4 = extractelement <8 x i32> %z, i64 4 + %m4 = icmp sgt i32 %x4, %y4 + %n4 = icmp slt i32 %x4, %z4 + %c4 = and i1 %m4, %n4 + %s4 = select i1 %c4, ptr %a, ptr %b + %p4 = getelementptr inbounds i8, ptr %s4, i64 16 + %l4 = load i32, ptr %p4, align 4 + %v4 = insertelement <8 x i32> %v3, i32 %l4, i64 4 + %x5 = extractelement <8 x i32> %x, i64 5 + %y5 = extractelement <8 x i32> %y, i64 5 + %z5 = extractelement <8 x i32> %z, i64 5 + %m5 = icmp sgt i32 %x5, %y5 + %n5 = icmp slt i32 %x5, %z5 + %c5 = and i1 %m5, %n5 + %s5 = select i1 %c5, ptr %a, ptr %b + %p5 = getelementptr inbounds i8, ptr %s5, i64 20 + %l5 = load i32, ptr %p5, align 4 + %v5 = insertelement <8 x i32> %v4, i32 %l5, i64 5 + %x6 = extractelement <8 x i32> %x, i64 6 + %y6 = extractelement <8 x i32> %y, i64 6 + %z6 = extractelement <8 x i32> %z, i64 6 + %m6 = icmp sgt i32 %x6, %y6 + %n6 = icmp slt i32 %x6, %z6 + %c6 = and i1 %m6, %n6 + %s6 = select i1 %c6, ptr %a, ptr %b + %p6 = getelementptr inbounds i8, ptr %s6, i64 24 + %l6 = load i32, ptr %p6, align 4 + %v6 = insertelement <8 x i32> %v5, i32 %l6, i64 6 + %x7 = extractelement <8 x i32> %x, i64 7 + %y7 = extractelement <8 x i32> %y, i64 7 + %z7 = extractelement <8 x i32> %z, i64 7 + %m7 = icmp sgt i32 %x7, %y7 + %n7 = icmp slt i32 %x7, %z7 + %c7 = and i1 %m7, %n7 + %s7 = select i1 %c7, ptr %a, ptr %b + %p7 = getelementptr inbounds i8, ptr %s7, i64 28 + %l7 = load i32, ptr %p7, align 4 + %v7 = insertelement <8 x i32> %v6, i32 %l7, i64 7 + ret <8 x i32> %v7 +} From 0aa8e45b04e548e34d7e6b1262cf7fec53678061 Mon Sep 17 00:00:00 2001 From: adams381 Date: Tue, 21 Jul 2026 13:20:00 -0500 Subject: [PATCH 09/50] [CIR] Lower _BitInt to its in-memory storage integer (#206606) ClangIR keeps a `_BitInt(N)` at its literal width, but in memory it has to live in a wider padded integer, the same way `bool` is `i1` as a value and `i8` in memory. Because CIRGen emitted the constant at its exact width, the storage padding was left undefined: a `signed _BitInt(6) = -1` wrote `0x3f` where classic CodeGen and GCC write `0xff`. @andykaylor pointed this out on [#205605](https://github.com/llvm/llvm-project/pull/205605). The fix widens `_BitInt` to its storage integer `iM` and sign/zero-extends by signedness during lowering to the LLVM dialect, the same place `bool` becomes `i8` (`convertTypeForMemory` and the to/from-memory helpers). CIRGen keeps the literal `!cir.int` unchanged. The byte-array storage form for wide "split" widths, where the storage integer's alloc size exceeds its store size (e.g. `_BitInt(129)` to `i192`, 32 vs 24 bytes), is not implemented. The global, alloca, load, and store lowerings report an NYI error and the aggregate type converters fail the conversion, rather than emit a wrong-sized integer. Atomic read-modify-write on a padded `_BitInt` and vector-of-`_BitInt` are also unimplemented. The storage width comes from `IntType::getABIAlignment`, which assumes a fixed 64-bit maximum alignment (x86-64). It is verified byte-for-byte on x86-64; on targets with a larger maximum (e.g. AArch64) any width that would be mis-sized instead hits the split-storage NYI. On a target whose `_BitInt` maximum alignment is below 64 bits (e.g. a 32-bit target) the storage can be mis-sized. Making the computation target-aware needs the target's `getBitIntAlign`, which is not in the data layout, and is left as a follow-up. The approach follows the RFC [Widen bool / _BitInt / ext-vector-bool to their in-memory types in lowering](https://discourse.llvm.org/t/rfc-clangir-widen-bool-bitint-ext-vector-bool-to-their-in-memory-types-in-lowering/91160). It supersedes [#205605](https://github.com/llvm/llvm-project/pull/205605), which emitted the constant at its literal width. --- clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp | 7 +- .../CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp | 177 ++++++++++++++++-- clang/lib/CIR/Lowering/LoweringHelpers.cpp | 15 +- clang/test/CIR/CodeGen/array.cpp | 7 +- clang/test/CIR/CodeGen/bitint-memory.c | 61 ++++++ .../CIR/CodeGen/bitint-split-storage-nyi.c | 27 +++ clang/test/CIR/CodeGen/bitint.c | 12 +- clang/test/CIR/Lowering/global-var-simple.cpp | 4 +- 8 files changed, 272 insertions(+), 38 deletions(-) create mode 100644 clang/test/CIR/CodeGen/bitint-memory.c create mode 100644 clang/test/CIR/CodeGen/bitint-split-storage-nyi.c diff --git a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp index 3fcf577776663..f1016493b39cd 100644 --- a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp @@ -1309,9 +1309,10 @@ mlir::Attribute ConstantEmitter::emitForMemory(CIRGenModule &cgm, cgm.errorNYI("emitForMemory: zero-extend HLSL bool vectors"); } - if (destType->isBitIntType()) { - cgm.errorNYI("emitForMemory: _BitInt type"); - } + // CIR represents source types as literally as possible. Some types, such as + // bool and _BitInt(N), are kept at their literal width here and expanded to + // their wider "in memory" types during lowering to the LLVM dialect, so the + // constant is already in the right form and needs no adjustment. return c; } diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp index f52f5683e50c0..a368df24e0144 100644 --- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp +++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp @@ -76,6 +76,32 @@ mlir::Type elementTypeIfVector(mlir::Type type) { } } // namespace +/// In-memory storage width in bits for a _BitInt(N): N rounded up to the type's +/// ABI alignment. This equals sizeof(_BitInt(N)) * 8 on the default target +/// (e.g. _BitInt(6) -> 8, _BitInt(17) -> 32, _BitInt(128) -> 128). +static unsigned getBitIntMemoryStorageBits(cir::IntType ty, + const mlir::DataLayout &dataLayout) { + uint64_t alignBits = ty.getABIAlignment(dataLayout, {}) * 8; + return llvm::alignTo(ty.getWidth(), alignBits); +} + +/// A _BitInt(N) whose padded storage integer iM has a larger alloc size than +/// its M/8 store size is laid out by clang as a byte array, not a plain integer +/// (e.g. _BitInt(129) -> i192 with alloc size 32 != store size 24). That +/// "split" storage form is not yet implemented; lowerings must detect it and +/// report errorNYI rather than emit the wrong-sized integer. +static bool isSplitStorageBitInt(cir::IntType ty, + const mlir::DataLayout &dataLayout) { + if (!ty.isBitInt()) + return false; + unsigned storageBits = getBitIntMemoryStorageBits(ty, dataLayout); + auto storageTy = mlir::IntegerType::get(ty.getContext(), storageBits); + uint64_t storeSize = storageBits / 8; + uint64_t allocSize = + llvm::alignTo(storeSize, dataLayout.getTypeABIAlignment(storageTy)); + return allocSize != storeSize; +} + /// Given a type convertor and a data layout, convert the given type to a type /// that is suitable for memory operations. For example, this can be used to /// lower cir.bool accesses to i8. @@ -89,9 +115,33 @@ static mlir::Type convertTypeForMemory(const mlir::TypeConverter &converter, dataLayout.getTypeSizeInBits(type)); } + // _BitInt(N) keeps its literal width as a value but is stored in a padded + // integer iM in memory, the same way bool is i1 as a value and i8 in memory. + // The byte-array storage form for wide split widths is not implemented; a + // null return signals that, and op lowerings turn it into errorNYI. + if (auto intTy = mlir::dyn_cast(type); + intTy && intTy.isBitInt()) { + if (isSplitStorageBitInt(intTy, dataLayout)) + return {}; + return mlir::IntegerType::get( + type.getContext(), getBitIntMemoryStorageBits(intTy, dataLayout)); + } + return converter.convertType(type); } +/// Alignment to use for a memory access whose op carries no explicit alignment. +/// For _BitInt the storage integer iM's ABI alignment (e.g. i128's 16) +/// over-aligns the value, so use the CIR _BitInt ABI alignment (e.g. 8). +static uint64_t getMemoryFallbackAlignment(mlir::Type cirType, + mlir::Type llvmMemType, + const mlir::DataLayout &dataLayout) { + if (auto intTy = mlir::dyn_cast(cirType); + intTy && intTy.isBitInt()) + return intTy.getABIAlignment(dataLayout, {}); + return dataLayout.getTypeABIAlignment(llvmMemType); +} + static mlir::Value createIntCast(mlir::OpBuilder &bld, mlir::Value src, mlir::IntegerType dstTy, bool isSigned = false) { @@ -111,6 +161,23 @@ static mlir::Value createIntCast(mlir::OpBuilder &bld, mlir::Value src, return mlir::LLVM::BitcastOp::create(bld, loc, dstTy, src); } +/// Cast a _BitInt(N) value between its literal width iN and its padded +/// in-memory storage iM (sign/zero-extend to memory per signedness so the +/// padding bits are well-defined, truncate back on load; matches classic +/// CodeGen). Callers must first rule out the unsupported byte-array storage +/// form (isSplitStorageBitInt / a null convertTypeForMemory result). +static mlir::Value +castBitIntMemoryStorage(mlir::ConversionPatternRewriter &rewriter, + const mlir::DataLayout &dataLayout, cir::IntType intTy, + mlir::Value value, bool toMemory) { + unsigned storageBits = getBitIntMemoryStorageBits(intTy, dataLayout); + if (storageBits == intTy.getWidth()) + return value; + unsigned dstBits = toMemory ? storageBits : intTy.getWidth(); + return createIntCast(rewriter, value, rewriter.getIntegerType(dstBits), + /*isSigned=*/toMemory && intTy.isSigned()); +} + static mlir::LLVM::Visibility lowerCIRVisibilityToLLVMVisibility(cir::VisibilityKind visibilityKind) { switch (visibilityKind) { @@ -137,6 +204,12 @@ static mlir::Value emitFromMemory(mlir::ConversionPatternRewriter &rewriter, return createIntCast(rewriter, value, rewriter.getI1Type()); } + // Truncate the padded storage integer back to the _BitInt's literal width. + if (auto intTy = mlir::dyn_cast(op.getType()); + intTy && intTy.isBitInt()) + return castBitIntMemoryStorage(rewriter, dataLayout, intTy, value, + /*toMemory=*/false); + return value; } @@ -155,6 +228,12 @@ static mlir::Value emitToMemory(mlir::ConversionPatternRewriter &rewriter, return createIntCast(rewriter, value, memType); } + // Sign/zero-extend the _BitInt value to its padded storage integer. + if (auto intTy = mlir::dyn_cast(origType); + intTy && intTy.isBitInt()) + return castBitIntMemoryStorage(rewriter, dataLayout, intTy, value, + /*toMemory=*/true); + return value; } @@ -443,9 +522,14 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::BoolAttr boolAttr) { /// IntAttr visitor. mlir::Value CIRAttrToValue::visitCirAttr(cir::IntAttr intAttr) { mlir::Location loc = parentOp->getLoc(); - return mlir::LLVM::ConstantOp::create( + mlir::DataLayout layout(parentOp->getParentOfType()); + // Materialize the value at its literal width, then widen to the in-memory + // storage type (a no-op except for _BitInt) so aggregate members built here + // match the iM struct/array fields produced by convertTypeForMemory. + mlir::Value val = mlir::LLVM::ConstantOp::create( rewriter, loc, converter->convertType(intAttr.getType()), intAttr.getValue()); + return emitToMemory(rewriter, layout, intAttr.getType(), val); } /// FPAttr visitor. @@ -731,15 +815,19 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::TypeInfoAttr typeInfoAttr) { /// UndefAttr visitor. mlir::Value CIRAttrToValue::visitCirAttr(cir::UndefAttr undefAttr) { mlir::Location loc = parentOp->getLoc(); + mlir::DataLayout layout(parentOp->getParentOfType()); return mlir::LLVM::UndefOp::create( - rewriter, loc, converter->convertType(undefAttr.getType())); + rewriter, loc, + convertTypeForMemory(*converter, layout, undefAttr.getType())); } /// PoisonAttr visitor. mlir::Value CIRAttrToValue::visitCirAttr(cir::PoisonAttr poisonAttr) { mlir::Location loc = parentOp->getLoc(); + mlir::DataLayout layout(parentOp->getParentOfType()); return mlir::LLVM::PoisonOp::create( - rewriter, loc, converter->convertType(poisonAttr.getType())); + rewriter, loc, + convertTypeForMemory(*converter, layout, poisonAttr.getType())); } // VTableAttr visitor. @@ -760,8 +848,9 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::VTableAttr vtableArr) { /// ZeroAttr visitor. mlir::Value CIRAttrToValue::visitCirAttr(cir::ZeroAttr attr) { mlir::Location loc = parentOp->getLoc(); - return mlir::LLVM::ZeroOp::create(rewriter, loc, - converter->convertType(attr.getType())); + mlir::DataLayout layout(parentOp->getParentOfType()); + return mlir::LLVM::ZeroOp::create( + rewriter, loc, convertTypeForMemory(*converter, layout, attr.getType())); } // This class handles rewriting initializer attributes for types that do not @@ -780,7 +869,17 @@ class GlobalInitAttrRewriter { } mlir::Attribute visitCirAttr(cir::IntAttr attr) { - return rewriter.getIntegerAttr(llvmType, attr.getValue()); + // A _BitInt(N) global stores its value in a padded integer iM; sign/zero- + // extend the APInt to that width (a no-op for plain integers, whose value + // width already matches llvmType) so the IntegerAttr is well-typed. + llvm::APInt val = attr.getValue(); + auto destTy = mlir::cast(llvmType); + if (val.getBitWidth() != destTy.getWidth()) { + auto cirIntTy = mlir::cast(attr.getType()); + val = cirIntTy.isSigned() ? val.sext(destTy.getWidth()) + : val.zext(destTy.getWidth()); + } + return rewriter.getIntegerAttr(llvmType, val); } mlir::Attribute visitCirAttr(cir::FPAttr attr) { @@ -1732,6 +1831,9 @@ mlir::LogicalResult CIRToLLVMAllocaOpLowering::matchAndRewrite( typeConverter->convertType(rewriter.getIndexType()), 1); mlir::Type elementTy = convertTypeForMemory(*getTypeConverter(), dataLayout, op.getAllocaType()); + if (!elementTy) + return op.emitError() + << "NYI: lowering alloca of a type with no memory representation"; mlir::Type resultTy = convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType()); @@ -1938,10 +2040,13 @@ mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite( mlir::ConversionPatternRewriter &rewriter) const { const mlir::Type llvmTy = convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType()); + if (!llvmTy) + return op.emitError() + << "NYI: lowering load of a type with no memory representation"; mlir::LLVM::AtomicOrdering ordering = getLLVMMemOrder(op.getMemOrder()); std::optional opAlign = op.getAlignment(); - unsigned alignment = - (unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmTy)); + unsigned alignment = (unsigned)opAlign.value_or( + getMemoryFallbackAlignment(op.getType(), llvmTy, dataLayout)); assert(!cir::MissingFeatures::lowerModeOptLevel()); @@ -1970,6 +2075,10 @@ cir::direct::CIRToLLVMVecMaskedLoadOpLowering::matchAndRewrite( mlir::ConversionPatternRewriter &rewriter) const { const mlir::Type llvmResTy = convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType()); + if (!llvmResTy) + return op.emitError() + << "NYI: lowering masked load of a type with no memory " + "representation"; std::optional opAlign = op.getAlignment(); unsigned alignment = @@ -1989,11 +2098,15 @@ mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite( cir::StoreOp op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) const { mlir::LLVM::AtomicOrdering memorder = getLLVMMemOrder(op.getMemOrder()); + mlir::Type valueType = op.getValue().getType(); const mlir::Type llvmTy = - getTypeConverter()->convertType(op.getValue().getType()); + convertTypeForMemory(*getTypeConverter(), dataLayout, valueType); + if (!llvmTy) + return op.emitError() + << "NYI: lowering store of a type with no memory representation"; std::optional opAlign = op.getAlignment(); - unsigned alignment = - (unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmTy)); + unsigned alignment = (unsigned)opAlign.value_or( + getMemoryFallbackAlignment(valueType, llvmTy, dataLayout)); assert(!cir::MissingFeatures::lowerModeOptLevel()); @@ -2537,6 +2650,9 @@ mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite( // This is the LLVM dialect type. mlir::Type llvmType = convertTypeForMemory(*getTypeConverter(), dataLayout, cirSymType); + if (!llvmType) + return op.emitError() + << "NYI: lowering global of a type with no memory representation"; // A flexible array member initializer makes the constant larger than the // record's declared type, so the global must use an oversized anonymous @@ -3327,9 +3443,19 @@ static void prepareTypeConverter(mlir::LLVMTypeConverter &converter, converter.addConversion([&](cir::ArrayType type) -> mlir::Type { mlir::Type ty = convertTypeForMemory(converter, dataLayout, type.getElementType()); + // A null element type means an unsupported member (e.g. a _BitInt with + // byte-array storage); propagate the conversion failure. + if (!ty) + return {}; return mlir::LLVM::LLVMArrayType::get(ty, type.getSize()); }); converter.addConversion([&](cir::VectorType type) -> mlir::Type { + // Vector-of-_BitInt memory layout is not modeled here (the element would + // stay at its literal width); report a conversion failure rather than emit + // an unvalidated vector. + if (auto intTy = mlir::dyn_cast(type.getElementType()); + intTy && intTy.isBitInt()) + return {}; const mlir::Type ty = converter.convertType(type.getElementType()); return mlir::VectorType::get(type.getSize(), ty, {type.getIsScalable()}); }); @@ -3381,8 +3507,15 @@ static void prepareTypeConverter(mlir::LLVMTypeConverter &converter, }); converter.addConversion([&](cir::StructType type) -> mlir::Type { llvm::SmallVector llvmMembers; - for (mlir::Type ty : type.getMembers()) - llvmMembers.push_back(convertTypeForMemory(converter, dataLayout, ty)); + for (mlir::Type ty : type.getMembers()) { + mlir::Type memberTy = convertTypeForMemory(converter, dataLayout, ty); + // A null member means an unsupported type (e.g. a _BitInt with byte-array + // storage); propagate the conversion failure instead of building an + // invalid struct body. + if (!memberTy) + return {}; + llvmMembers.push_back(memberTy); + } mlir::LLVM::LLVMStructType llvmStruct; if (type.getName()) { @@ -3400,11 +3533,19 @@ static void prepareTypeConverter(mlir::LLVMTypeConverter &converter, converter.addConversion([&](cir::UnionType type) -> mlir::Type { llvm::SmallVector llvmMembers; if (!type.getMembers().empty()) - if (auto storage = type.getUnionStorageType(dataLayout)) - llvmMembers.push_back( - convertTypeForMemory(converter, dataLayout, storage)); - if (mlir::Type pad = type.getPadding()) - llvmMembers.push_back(convertTypeForMemory(converter, dataLayout, pad)); + if (auto storage = type.getUnionStorageType(dataLayout)) { + mlir::Type storageTy = + convertTypeForMemory(converter, dataLayout, storage); + if (!storageTy) + return {}; + llvmMembers.push_back(storageTy); + } + if (mlir::Type pad = type.getPadding()) { + mlir::Type padTy = convertTypeForMemory(converter, dataLayout, pad); + if (!padTy) + return {}; + llvmMembers.push_back(padTy); + } mlir::LLVM::LLVMStructType llvmStruct; if (type.getName()) { diff --git a/clang/lib/CIR/Lowering/LoweringHelpers.cpp b/clang/lib/CIR/Lowering/LoweringHelpers.cpp index 1b3448dbaabe9..5cbe436224f6a 100644 --- a/clang/lib/CIR/Lowering/LoweringHelpers.cpp +++ b/clang/lib/CIR/Lowering/LoweringHelpers.cpp @@ -238,9 +238,15 @@ lowerConstArrayAttr(cir::ConstArrayAttr constArr, if (mlir::isa(constArr.getElts())) return convertStringAttrToDenseElementsAttr(constArr, converter->convertType(type)); - if (mlir::isa(type)) + if (mlir::isa(type)) { + // A _BitInt array element is stored in a padded integer in memory; the + // dense-attribute path here cannot express that widening, so fall back to + // the insertvalue region, which widens each element via emitToMemory. + if (mlir::cast(type).isBitInt()) + return std::nullopt; return convertToDenseElementsAttr( constArr, dims, type, converter->convertType(type)); + } if (mlir::isa(type)) return convertToDenseElementsAttr( @@ -299,9 +305,14 @@ lowerConstRecordMemberAttr(mlir::Attribute attr, if (mlir::isa(attr)) return mlir::LLVM::UndefAttr::get(ctx); - if (auto intAttr = mlir::dyn_cast(attr)) + if (auto intAttr = mlir::dyn_cast(attr)) { + // A _BitInt member is stored in a padded integer in memory; defer to the + // insertvalue region (CIRAttrToValue), which performs the extension. + if (mlir::cast(intAttr.getType()).isBitInt()) + return std::nullopt; return mlir::IntegerAttr::get(converter->convertType(intAttr.getType()), intAttr.getValue()); + } if (auto boolAttr = mlir::dyn_cast(attr)) return mlir::IntegerAttr::get(converter->convertType(boolAttr.getType()), diff --git a/clang/test/CIR/CodeGen/array.cpp b/clang/test/CIR/CodeGen/array.cpp index 1ff733ee17ea6..1f5cf676da979 100644 --- a/clang/test/CIR/CodeGen/array.cpp +++ b/clang/test/CIR/CodeGen/array.cpp @@ -534,11 +534,12 @@ void bitint_index_access(_BitInt(7) i) { // CIR: cir.store{{.*}} %[[VAL]], %[[N_ADDR]] : !s32i, !cir.ptr // LLVM: define{{.*}} @_Z19bitint_index_accessDB7_(i7 {{[^)]*}}) -// LLVM: %[[I_ADDR:.*]] = alloca i7 +// LLVM: %[[I_ADDR:.*]] = alloca i8 // LLVM: %[[ARR_ADDR:.*]] = alloca [10 x i32] // LLVM: %[[N_ADDR:.*]] = alloca i32 -// LLVM: %[[I:.*]] = load i7, ptr %[[I_ADDR]] -// LLVM: %[[IDX:.*]] = sext i7 %[[I]] to i64 +// LLVM: %[[I:.*]] = load i8, ptr %[[I_ADDR]] +// LLVM: %[[I_CAST:.*]] = trunc i8 %[[I]] to i7 +// LLVM: %[[IDX:.*]] = sext i7 %[[I_CAST]] to i64 // LLVM: %[[ELE:.*]] = getelementptr [10 x i32], ptr %[[ARR_ADDR]], i32 0, i64 %[[IDX]] // LLVM: %[[VAL:.*]] = load i32, ptr %[[ELE]] // LLVM: store i32 %[[VAL]], ptr %[[N_ADDR]] diff --git a/clang/test/CIR/CodeGen/bitint-memory.c b/clang/test/CIR/CodeGen/bitint-memory.c new file mode 100644 index 0000000000000..ba71546e61160 --- /dev/null +++ b/clang/test/CIR/CodeGen/bitint-memory.c @@ -0,0 +1,61 @@ +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir +// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll +// RUN: FileCheck --check-prefix=LLVM --input-file=%t-cir.ll %s +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll +// RUN: FileCheck --check-prefix=LLVM --input-file=%t.ll %s + +signed _BitInt(6) s6_neg1 = -1; +unsigned _BitInt(6) u6_63 = 63; +signed _BitInt(6) s6_neg2 = -2; +signed _BitInt(2) s2_neg1 = -1; +unsigned _BitInt(4) u4_15 = 15; +signed _BitInt(17) s17_neg1 = -1; +unsigned _BitInt(17) u17_100 = 100; +signed _BitInt(128) s128 = 1234; + +// CIR-DAG: cir.global external @s6_neg1 = #cir.int<-1> : !cir.int {alignment = 1 : i64} +// CIR-DAG: cir.global external @u6_63 = #cir.int<63> : !cir.int {alignment = 1 : i64} +// CIR-DAG: cir.global external @s6_neg2 = #cir.int<-2> : !cir.int {alignment = 1 : i64} +// CIR-DAG: cir.global external @s2_neg1 = #cir.int<-1> : !cir.int {alignment = 1 : i64} +// CIR-DAG: cir.global external @u4_15 = #cir.int<15> : !cir.int {alignment = 1 : i64} +// CIR-DAG: cir.global external @s17_neg1 = #cir.int<-1> : !cir.int {alignment = 4 : i64} +// CIR-DAG: cir.global external @u17_100 = #cir.int<100> : !cir.int {alignment = 4 : i64} +// CIR-DAG: cir.global external @s128 = #cir.int<1234> : !s128i_bitint {alignment = 8 : i64} + +// A signed _BitInt is sign-extended across its padded storage integer; an +// unsigned one is zero-extended. Storage width is the ABI size (i6 -> i8, +// i17 -> i32, i128 stays i128 with align 8). +// LLVM-DAG: @s6_neg1 = global i8 -1, align 1 +// LLVM-DAG: @u6_63 = global i8 63, align 1 +// LLVM-DAG: @s6_neg2 = global i8 -2, align 1 +// LLVM-DAG: @s2_neg1 = global i8 -1, align 1 +// LLVM-DAG: @u4_15 = global i8 15, align 1 +// LLVM-DAG: @s17_neg1 = global i32 -1, align 4 +// LLVM-DAG: @u17_100 = global i32 100, align 4 +// LLVM-DAG: @s128 = global i128 1234, align 8 + +struct S { _BitInt(17) m; }; +struct S gs = {-1}; + +// CIR-DAG: cir.global external @gs = #cir.const_record<{#cir.int<-1> : !cir.int}> : !rec_S {alignment = 4 : i64} +// LLVM-DAG: @gs = global %struct.S { i32 -1 }, align 4 + +signed _BitInt(17) garr[3] = {-1, 2, -3}; + +// CIR-DAG: cir.global external @garr = #cir.const_array<[#cir.int<-1> : !cir.int, #cir.int<2> : !cir.int, #cir.int<-3> : !cir.int]> : !cir.array x 3> {alignment = 4 : i64} +// LLVM-DAG: @garr = global [3 x i32] [i32 -1, i32 2, i32 -3], align 4 + +void store_load(signed _BitInt(17) *p, signed _BitInt(17) v) { *p = v; } + +// CIR-LABEL: cir.func {{.*}} @store_load +// CIR: %[[V:.+]] = cir.load {{.*}} : !cir.ptr>, !cir.int +// CIR: cir.store {{.*}} %[[V]], {{.*}} : !cir.int, !cir.ptr> + +// The value is sign-extended on store into i32 memory and truncated back to +// i17 on load. +// LLVM-LABEL: @store_load +// LLVM: %[[EXT:.+]] = sext i17 %{{.+}} to i32 +// LLVM: store i32 %[[EXT]], ptr %{{.+}}, align 4 +// LLVM: %[[LD:.+]] = load i32, ptr %{{.+}}, align 4 +// LLVM: %[[TR:.+]] = trunc i32 %[[LD]] to i17 diff --git a/clang/test/CIR/CodeGen/bitint-split-storage-nyi.c b/clang/test/CIR/CodeGen/bitint-split-storage-nyi.c new file mode 100644 index 0000000000000..ee9fc168124e6 --- /dev/null +++ b/clang/test/CIR/CodeGen/bitint-split-storage-nyi.c @@ -0,0 +1,27 @@ +// RUN: not %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -DGLOBAL %s -o - 2>&1 | FileCheck %s --check-prefix=GLOBAL +// RUN: not %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -DALLOCA %s -o - 2>&1 | FileCheck %s --check-prefix=ALLOCA +// RUN: not %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -DSTORE %s -o - 2>&1 | FileCheck %s --check-prefix=STORE +// RUN: not %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -DLOAD %s -o - 2>&1 | FileCheck %s --check-prefix=LOAD + +#ifdef GLOBAL +signed _BitInt(129) g129 = 1; +// GLOBAL: NYI: lowering global of a type with no memory representation +#endif + +#ifdef ALLOCA +signed _BitInt(129) use_local(signed _BitInt(129) a) { + signed _BitInt(129) x = a; + return x; +} +// ALLOCA: NYI: lowering alloca of a type with no memory representation +#endif + +#ifdef STORE +void store_lit(signed _BitInt(129) *p) { *p = (signed _BitInt(129))1; } +// STORE: NYI: lowering store of a type with no memory representation +#endif + +#ifdef LOAD +int load_cmp(signed _BitInt(129) *p) { return *p != 0; } +// LOAD: NYI: lowering load of a type with no memory representation +#endif diff --git a/clang/test/CIR/CodeGen/bitint.c b/clang/test/CIR/CodeGen/bitint.c index e84b610c171b6..ef67366571138 100644 --- a/clang/test/CIR/CodeGen/bitint.c +++ b/clang/test/CIR/CodeGen/bitint.c @@ -20,25 +20,22 @@ signed _BitInt(128) bitint128_var; __int128 int128_var; signed _BitInt(256) bitint256_var; signed _BitInt(254) bitint254_var; -signed _BitInt(257) bitint257_var; // CIR: cir.global external @bitint128_var = #cir.int<0> : !s128i_bitint {alignment = 8 : i64} // CIR: cir.global external @int128_var = #cir.int<0> : !s128i {alignment = 16 : i64} // CIR: cir.global external @bitint256_var = #cir.int<0> : !s256i_bitint // CIR: cir.global external @bitint254_var = #cir.int<0> : !cir.int -// CIR: cir.global external @bitint257_var = #cir.int<0> : !cir.int +// _BitInt is stored in its padded storage integer in memory (e.g. i254 -> i256). // LLVM: @bitint128_var = global i128 0, align 8 // LLVM: @int128_var = global i128 0, align 16 // LLVM: @bitint256_var = global i256 0 -// LLVM: @bitint254_var = global i254 0, align 8 -// LLVM: @bitint257_var = global i257 0, align 8 +// LLVM: @bitint254_var = global i256 0, align 8 // OGCG: @bitint128_var = global i128 0, align 8 // OGCG: @int128_var = global i128 0, align 16 // OGCG: @bitint256_var = global i256 0 // OGCG: @bitint254_var = global i256 0, align 8 -// OGCG: @bitint257_var = global [40 x i8] zeroinitializer, align 8 void take_bitint_32(_BitInt(32) x) {} // CIR: cir.func {{.*}} @take_bitint_32(%arg0: !s32i_bitint @@ -60,11 +57,6 @@ void take_bitint_254(signed _BitInt(254) x) {} // LLVM: define {{.*}} void @take_bitint_254(i254 {{.*}}) // OGCG: define {{.*}} void @take_bitint_254(ptr noundef byval(i256) align 8 {{.*}}) -void take_bitint_257(signed _BitInt(257) x) {} -// CIR: cir.func {{.*}} @take_bitint_257(%arg0: !cir.int -// LLVM: define {{.*}} void @take_bitint_257(i257 {{.*}}) -// OGCG: define {{.*}} void @take_bitint_257(ptr noundef byval([40 x i8]) align 8 {{.*}}) - // Regular __int128 should NOT have the bitint flag. void take_int128(__int128 x) {} // CIR: cir.func {{.*}} @take_int128(%arg0: !s128i diff --git a/clang/test/CIR/Lowering/global-var-simple.cpp b/clang/test/CIR/Lowering/global-var-simple.cpp index c470e9b5f06f0..96f8353b2f1a2 100644 --- a/clang/test/CIR/Lowering/global-var-simple.cpp +++ b/clang/test/CIR/Lowering/global-var-simple.cpp @@ -76,11 +76,11 @@ char32_t c32; // OGCG: @c32 = global i32 0, align 4 _BitInt(20) sb20; -// CHECK: @sb20 = global i20 0, align 4 +// CHECK: @sb20 = global i32 0, align 4 // OGCG: @sb20 = global i32 0, align 4 unsigned _BitInt(48) ub48; -// CHECK: @ub48 = global i48 0, align 8 +// CHECK: @ub48 = global i64 0, align 8 // OGCG: @ub48 = global i64 0, align 8 bool boolfalse = false; From 49d568a8d60d5912a5a7d8b2c3ac1caf58dab3e6 Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Tue, 21 Jul 2026 19:27:15 +0100 Subject: [PATCH 10/50] [DTLTO] Forward loop interchange to DTLTO remote compilations (#208591) Recent upstream LLVM changes enabled `LoopInterchangePass` by default through `PipelineTuningOptions`. TLTO observes that state directly from the LTO config, but DTLTO was not forwarding the equivalent Clang option to the remote compiler. As a result, TLTO could run loop interchange pass while DTLTO did not, producing codegen differences for configurations that exposed profitable/legally interchangeable loops. Note that this was a pre-existing issue, the change in default has simply exposed it. Forward `C.PTO.LoopInterchange` as `-floop-interchange` when constructing the common DTLTO remote compiler options. Extend the testing to check that the SN-DBS remote command line includes the forwarded option. The code for forwarding such options is only intended as a temporary measure. A more comprehensive solution is in discussion here: https://discourse.llvm.org/t/synchronizing-lto-code-generation-configuration-between-clang-and-lld-dtlto --- cross-project-tests/dtlto/remote-options.test | 9 +++++++++ llvm/lib/DTLTO/DTLTO.cpp | 2 ++ llvm/test/ThinLTO/X86/dtlto/json.ll | 1 + 3 files changed, 12 insertions(+) diff --git a/cross-project-tests/dtlto/remote-options.test b/cross-project-tests/dtlto/remote-options.test index e808adf9cf9e1..ddc9883199c49 100644 --- a/cross-project-tests/dtlto/remote-options.test +++ b/cross-project-tests/dtlto/remote-options.test @@ -47,6 +47,15 @@ NO-SECTIONS-NOT: "-faddrsig" NO-SECTIONS-NOT: "-ffunction-sections" NO-SECTIONS-NOT: "-fdata-sections" +## Loop interchange. +RUN: %{lto2-elf} -enable-loopinterchange=true 2>&1 | \ +RUN: FileCheck %s --check-prefixes=ARGS,LOOP-INTERCHANGE,ERR +LOOP-INTERCHANGE: "-floop-interchange" + +RUN: %{lto2-elf} -enable-loopinterchange=false 2>&1 | \ +RUN: FileCheck %s --check-prefixes=ARGS,NO-LOOP-INTERCHANGE,ERR +NO-LOOP-INTERCHANGE-NOT: "-floop-interchange" + ## PIC is forwarded as -fpic for ELF targets but not for COFF targets because ## Clang does not accept -fpic for every target. RUN: %{lto2-elf} -relocation-model=pic 2>&1 | \ diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp index 1d42d09d0820a..6946abe470773 100644 --- a/llvm/lib/DTLTO/DTLTO.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -227,6 +227,8 @@ void lto::DTLTO::buildCommonRemoteCompilerOptions() { Ops.push_back("-ffunction-sections"); if (C.Options.DataSections) Ops.push_back("-fdata-sections"); + if (C.PTO.LoopInterchange) + Ops.push_back("-floop-interchange"); if (C.RelocModel == Reloc::PIC_) // Clang doesn't have -fpic for all triples. diff --git a/llvm/test/ThinLTO/X86/dtlto/json.ll b/llvm/test/ThinLTO/X86/dtlto/json.ll index ee1c428c31c7c..e705e97bd8858 100644 --- a/llvm/test/ThinLTO/X86/dtlto/json.ll +++ b/llvm/test/ThinLTO/X86/dtlto/json.ll @@ -37,6 +37,7 @@ LLVM-NEXT: "clang" CHECK-NEXT: "-c" CHECK-NEXT: "--target=x86_64-unknown-linux-gnu" CHECK-NEXT: "-O2" +CHECK-NEXT: "-floop-interchange" CHECK-NEXT: "-fpic" CHECK-NEXT: "-Wno-unused-command-line-argument" CHECK-NEXT: "--rota1=10" From b35c1aab487dd89537114612c5a1ed3dd0a3ce11 Mon Sep 17 00:00:00 2001 From: "Deric C." Date: Tue, 21 Jul 2026 11:29:35 -0700 Subject: [PATCH 11/50] Revert "[clang] Update diagnostics to include matrices as accepted types" (#211070) Reverts llvm/llvm-project#201237. The PR did not update the test assertions for `Sema/builtins-elementwise-math.c` https://github.com/llvm/llvm-project/pull/201237#issuecomment-5037239584 --- .../clang/Basic/DiagnosticSemaKinds.td | 3 +- clang/lib/Sema/SemaChecking.cpp | 11 +- clang/lib/Sema/SemaHLSL.cpp | 10 +- clang/lib/Sema/SemaSPIRV.cpp | 12 +- clang/test/Sema/aarch64-sve-vector-exp-ops.c | 4 +- clang/test/Sema/aarch64-sve-vector-log-ops.c | 6 +- clang/test/Sema/aarch64-sve-vector-pow-ops.c | 2 +- clang/test/Sema/aarch64-sve-vector-trig-ops.c | 20 +- clang/test/Sema/builtins-elementwise-math.c | 320 +++++++++--------- clang/test/Sema/riscv-rvv-vector-exp-ops.c | 4 +- clang/test/Sema/riscv-rvv-vector-log-ops.c | 6 +- clang/test/Sema/riscv-rvv-vector-trig-ops.c | 20 +- clang/test/Sema/riscv-sve-vector-pow-ops.c | 2 +- .../SemaHLSL/BuiltIns/atan2-errors_mat.hlsl | 2 +- .../SemaHLSL/BuiltIns/ddx-coarse-errors.hlsl | 4 +- .../SemaHLSL/BuiltIns/ddx-fine-errors.hlsl | 4 +- .../SemaHLSL/BuiltIns/ddy-coarse-errors.hlsl | 4 +- .../SemaHLSL/BuiltIns/ddy-fine-errors.hlsl | 4 +- .../SemaHLSL/BuiltIns/degrees-errors.hlsl | 6 +- clang/test/SemaHLSL/BuiltIns/exp-errors.hlsl | 6 +- .../SemaHLSL/BuiltIns/f16tof32-errors.hlsl | 16 +- clang/test/SemaHLSL/BuiltIns/frac-errors.hlsl | 6 +- .../BuiltIns/half-float-only-errors.hlsl | 4 +- .../BuiltIns/half-float-only-errors2.hlsl | 4 +- .../test/SemaHLSL/BuiltIns/isinf-errors.hlsl | 10 +- .../test/SemaHLSL/BuiltIns/isnan-errors.hlsl | 10 +- clang/test/SemaHLSL/BuiltIns/lerp-errors.hlsl | 20 +- .../SemaHLSL/BuiltIns/normalize-errors.hlsl | 6 +- .../SemaHLSL/BuiltIns/radians-errors.hlsl | 6 +- clang/test/SemaHLSL/BuiltIns/rcp-errors.hlsl | 6 +- .../SemaHLSL/BuiltIns/reversebits-errors.hlsl | 4 +- .../test/SemaHLSL/BuiltIns/round-errors.hlsl | 6 +- .../test/SemaHLSL/BuiltIns/rsqrt-errors.hlsl | 8 +- clang/test/SemaHLSL/BuiltIns/step-errors.hlsl | 6 +- clang/test/SemaSPIRV/BuiltIns/ddx-errors.c | 2 +- clang/test/SemaSPIRV/BuiltIns/ddy-errors.c | 2 +- .../SemaSPIRV/BuiltIns/faceforward-errors.c | 2 +- clang/test/SemaSPIRV/BuiltIns/fwidth-errors.c | 2 +- .../SemaSPIRV/BuiltIns/smoothstep-errors.c | 2 +- 39 files changed, 286 insertions(+), 286 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 471aa1e9c06a2..89e2f956971b3 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -13393,8 +13393,7 @@ def err_builtin_is_within_lifetime_invalid_arg : Error< def err_builtin_invalid_arg_type: Error< "%ordinal0 argument must be a " // First component: scalar or container types - "%select{|scalar|vector|matrix|vector of|scalar or vector of|" - "scalar, vector, or matrix of}1" + "%select{|scalar|vector|matrix|vector of|scalar or vector of}1" // A comma after generic vector/matrix types if there are non-empty second // and third components, to initiate a list. "%plural{[2,3]:%plural{0:|:%plural{0:|:,}2}3|:}1" diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 174c96f2e45f4..ef07e8c6133ed 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -2360,22 +2360,23 @@ checkMathBuiltinElementType(Sema &S, SourceLocation Loc, QualType ArgTy, break; case Sema::EltwiseBuiltinArgTyRestriction::FloatTy: if (!EltTy->isRealFloatingType()) { + // FIXME: make diagnostic's wording correct for matrices return S.Diag(Loc, diag::err_builtin_invalid_arg_type) - << ArgOrdinal << /* scalar, vector, or matrix */ 6 - << /* no int */ 0 << /* floating-point */ 1 << ArgTy; + << ArgOrdinal << /* scalar or vector */ 5 << /* no int */ 0 + << /* floating-point */ 1 << ArgTy; } break; case Sema::EltwiseBuiltinArgTyRestriction::IntegerTy: if (!EltTy->isIntegerType()) { return S.Diag(Loc, diag::err_builtin_invalid_arg_type) - << ArgOrdinal << /* scalar, vector, or matrix */ 6 - << /* integer */ 1 << /* no fp */ 0 << ArgTy; + << ArgOrdinal << /* scalar or vector */ 5 << /* integer */ 1 + << /* no fp */ 0 << ArgTy; } break; case Sema::EltwiseBuiltinArgTyRestriction::SignedIntOrFloatTy: if (!EltTy->isSignedIntegerType() && !EltTy->isRealFloatingType()) { return S.Diag(Loc, diag::err_builtin_invalid_arg_type) - << 1 << /* scalar, vector, or matrix */ 6 << /* signed int */ 2 + << 1 << /* scalar or vector */ 5 << /* signed int */ 2 << /* or fp */ 1 << ArgTy; } break; diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp index 3465418745068..c333f0dd4c872 100644 --- a/clang/lib/Sema/SemaHLSL.cpp +++ b/clang/lib/Sema/SemaHLSL.cpp @@ -3402,7 +3402,7 @@ static bool CheckFloatOrHalfRepresentation(Sema *S, SourceLocation Loc, if (!BaseType->isHalfType() && !BaseType->isFloat32Type()) return S->Diag(Loc, diag::err_builtin_invalid_arg_type) - << ArgOrdinal << /* scalar, vector, or matrix */ 6 << /* no int */ 0 + << ArgOrdinal << /* scalar or vector of */ 5 << /* no int */ 0 << /* half or float */ 2 << PassedType; return false; } @@ -3493,8 +3493,8 @@ static bool CheckUnsignedIntRepresentation(Sema *S, SourceLocation Loc, clang::QualType PassedType) { if (!PassedType->hasUnsignedIntegerRepresentation()) return S->Diag(Loc, diag::err_builtin_invalid_arg_type) - << ArgOrdinal << /* scalar, vector, or matrix */ 6 - << /* unsigned int */ 3 << /* no fp */ 0 << PassedType; + << ArgOrdinal << /* scalar or vector of */ 5 << /* unsigned int */ 3 + << /* no fp */ 0 << PassedType; return false; } @@ -4325,8 +4325,8 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { ->hasFloatingRepresentation()) // half or float or double return SemaRef.Diag(TheCall->getArg(0)->getBeginLoc(), diag::err_builtin_invalid_arg_type) - << /* ordinal */ 1 << /* scalar, vector, or matrix */ 6 - << /* no int */ 0 << /* fp */ 1 << TheCall->getArg(0)->getType(); + << /* ordinal */ 1 << /* scalar or vector */ 5 << /* no int */ 0 + << /* fp */ 1 << TheCall->getArg(0)->getType(); if (SemaRef.PrepareBuiltinElementwiseMathOneArgCall(TheCall)) return true; break; diff --git a/clang/lib/Sema/SemaSPIRV.cpp b/clang/lib/Sema/SemaSPIRV.cpp index 99842fa4de5ef..8c2af5053bde2 100644 --- a/clang/lib/Sema/SemaSPIRV.cpp +++ b/clang/lib/Sema/SemaSPIRV.cpp @@ -324,8 +324,8 @@ bool SemaSPIRV::CheckSPIRVBuiltinFunctionCall(const TargetInfo &TI, QualType ArgTyA = A.get()->getType(); if (!ArgTyA->hasFloatingRepresentation()) { SemaRef.Diag(A.get()->getBeginLoc(), diag::err_builtin_invalid_arg_type) - << /* ordinal */ 1 << /* scalar, vector, or matrix */ 6 - << /* no int */ 0 << /* fp */ 1 << ArgTyA; + << /* ordinal */ 1 << /* scalar or vector */ 5 << /* no int */ 0 + << /* fp */ 1 << ArgTyA; return true; } @@ -345,8 +345,8 @@ bool SemaSPIRV::CheckSPIRVBuiltinFunctionCall(const TargetInfo &TI, QualType ArgTyA = A.get()->getType(); if (!ArgTyA->hasFloatingRepresentation()) { SemaRef.Diag(A.get()->getBeginLoc(), diag::err_builtin_invalid_arg_type) - << /* ordinal */ 1 << /* scalar, vector, or matrix */ 6 - << /* no int */ 0 << /* fp */ 1 << ArgTyA; + << /* ordinal */ 1 << /* scalar or vector */ 5 << /* no int */ 0 + << /* fp */ 1 << ArgTyA; return true; } @@ -371,8 +371,8 @@ bool SemaSPIRV::CheckSPIRVBuiltinFunctionCall(const TargetInfo &TI, QualType ArgTyA = A.get()->getType(); if (!ArgTyA->hasFloatingRepresentation()) { SemaRef.Diag(A.get()->getBeginLoc(), diag::err_builtin_invalid_arg_type) - << /* ordinal */ 1 << /* scalar, vector, or matrix */ 6 - << /* no int */ 0 << /* fp */ 1 << ArgTyA; + << /* ordinal */ 1 << /* scalar or vector */ 5 << /* no int */ 0 + << /* fp */ 1 << ArgTyA; return true; } diff --git a/clang/test/Sema/aarch64-sve-vector-exp-ops.c b/clang/test/Sema/aarch64-sve-vector-exp-ops.c index 9172c1abbff43..c0bcfe1318679 100644 --- a/clang/test/Sema/aarch64-sve-vector-exp-ops.c +++ b/clang/test/Sema/aarch64-sve-vector-exp-ops.c @@ -7,11 +7,11 @@ svfloat32_t test_exp_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_exp(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } svfloat32_t test_exp2_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_exp2(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } diff --git a/clang/test/Sema/aarch64-sve-vector-log-ops.c b/clang/test/Sema/aarch64-sve-vector-log-ops.c index 17fbd8fb5d812..cd0b5607d1e2a 100644 --- a/clang/test/Sema/aarch64-sve-vector-log-ops.c +++ b/clang/test/Sema/aarch64-sve-vector-log-ops.c @@ -7,17 +7,17 @@ svfloat32_t test_log_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_log(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } svfloat32_t test_log10_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_log10(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } svfloat32_t test_log2_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_log2(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } diff --git a/clang/test/Sema/aarch64-sve-vector-pow-ops.c b/clang/test/Sema/aarch64-sve-vector-pow-ops.c index 94287c2ce3b47..e4977615e98d2 100644 --- a/clang/test/Sema/aarch64-sve-vector-pow-ops.c +++ b/clang/test/Sema/aarch64-sve-vector-pow-ops.c @@ -7,5 +7,5 @@ svfloat32_t test_pow_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_pow(v, v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } diff --git a/clang/test/Sema/aarch64-sve-vector-trig-ops.c b/clang/test/Sema/aarch64-sve-vector-trig-ops.c index 06fee06f5f706..04e496031fff4 100644 --- a/clang/test/Sema/aarch64-sve-vector-trig-ops.c +++ b/clang/test/Sema/aarch64-sve-vector-trig-ops.c @@ -7,59 +7,59 @@ svfloat32_t test_asin_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_asin(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } svfloat32_t test_acos_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_acos(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } svfloat32_t test_atan_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_atan(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } svfloat32_t test_atan2_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_atan2(v, v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } svfloat32_t test_sin_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_sin(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } svfloat32_t test_cos_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_cos(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } svfloat32_t test_tan_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_tan(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } svfloat32_t test_sinh_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_sinh(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } svfloat32_t test_cosh_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_cosh(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } svfloat32_t test_tanh_vv_i8mf8(svfloat32_t v) { return __builtin_elementwise_tanh(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } diff --git a/clang/test/Sema/builtins-elementwise-math.c b/clang/test/Sema/builtins-elementwise-math.c index 0b1fbb130a72e..511d1d8b43329 100644 --- a/clang/test/Sema/builtins-elementwise-math.c +++ b/clang/test/Sema/builtins-elementwise-math.c @@ -39,18 +39,18 @@ void test_builtin_elementwise_abs(int i, double d, float4 v, int3 iv, unsigned u // expected-error@-1 {{assigning to 'int' from incompatible type 'float4' (vector of 4 'float' values)}} u = __builtin_elementwise_abs(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of signed integer or floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of signed integer or floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_abs(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of signed integer or floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of signed integer or floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} i = __builtin_elementwise_abs(&i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of signed integer or floating-point types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of signed integer or floating-point types (was 'int *')}} } void test_builtin_elementwise_add_sat(int i, short s, double d, float4 v, int3 iv, unsigned3 uv, int *p) { i = __builtin_elementwise_add_sat(p, d); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'int *')}} struct Foo foo = __builtin_elementwise_add_sat(i, i); // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'int'}} @@ -65,7 +65,7 @@ void test_builtin_elementwise_add_sat(int i, short s, double d, float4 v, int3 i // expected-error@-1 {{too many arguments to function call, expected 2, have 3}} i = __builtin_elementwise_add_sat(v, iv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'float4' (vector of 4 'float' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'float4' (vector of 4 'float' values))}} i = __builtin_elementwise_add_sat(iv, v); // expected-error@-1 {{arguments are of different types ('int3' (vector of 3 'int' values) vs 'float4' (vector of 4 'float' values))}} @@ -74,7 +74,7 @@ void test_builtin_elementwise_add_sat(int i, short s, double d, float4 v, int3 i // expected-error@-1 {{arguments are of different types ('unsigned3' (vector of 3 'unsigned int' values) vs 'int3' (vector of 3 'int' values))}} v = __builtin_elementwise_add_sat(v, v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'float4' (vector of 4 'float' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'float4' (vector of 4 'float' values))}} s = __builtin_elementwise_add_sat(i, s); // expected-error@-1 {{arguments are of different types ('int' vs 'short')}} @@ -103,7 +103,7 @@ void test_builtin_elementwise_add_sat(int i, short s, double d, float4 v, int3 i int A[10]; A = __builtin_elementwise_add_sat(A, A); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'int *')}} int(ii); int j; @@ -111,12 +111,12 @@ void test_builtin_elementwise_add_sat(int i, short s, double d, float4 v, int3 i _Complex float c1, c2; c1 = __builtin_elementwise_add_sat(c1, c2); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was '_Complex float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was '_Complex float')}} } void test_builtin_elementwise_sub_sat(int i, short s, double d, float4 v, int3 iv, unsigned3 uv, int *p) { i = __builtin_elementwise_sub_sat(p, d); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'int *')}} struct Foo foo = __builtin_elementwise_sub_sat(i, i); // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'int'}} @@ -131,7 +131,7 @@ void test_builtin_elementwise_sub_sat(int i, short s, double d, float4 v, int3 i // expected-error@-1 {{too many arguments to function call, expected 2, have 3}} i = __builtin_elementwise_sub_sat(v, iv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'float4' (vector of 4 'float' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'float4' (vector of 4 'float' values))}} i = __builtin_elementwise_sub_sat(iv, v); // expected-error@-1 {{arguments are of different types ('int3' (vector of 3 'int' values) vs 'float4' (vector of 4 'float' values))}} @@ -140,7 +140,7 @@ void test_builtin_elementwise_sub_sat(int i, short s, double d, float4 v, int3 i // expected-error@-1 {{arguments are of different types ('unsigned3' (vector of 3 'unsigned int' values) vs 'int3' (vector of 3 'int' values))}} v = __builtin_elementwise_sub_sat(v, v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'float4' (vector of 4 'float' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'float4' (vector of 4 'float' values))}} s = __builtin_elementwise_sub_sat(i, s); // expected-error@-1 {{arguments are of different types ('int' vs 'short')}} @@ -169,7 +169,7 @@ void test_builtin_elementwise_sub_sat(int i, short s, double d, float4 v, int3 i int A[10]; A = __builtin_elementwise_sub_sat(A, A); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'int *')}} int(ii); int j; @@ -177,14 +177,14 @@ void test_builtin_elementwise_sub_sat(int i, short s, double d, float4 v, int3 i _Complex float c1, c2; c1 = __builtin_elementwise_sub_sat(c1, c2); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was '_Complex float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was '_Complex float')}} } void test_builtin_elementwise_clmul(int i, short s, double d, float4 v, int3 iv, unsigned3 uv, unsigned u, unsigned4 vu, int *p) { i = __builtin_elementwise_clmul(p, d); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'int *')}} struct Foo foo = __builtin_elementwise_clmul(i, i); // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'int'}} @@ -199,7 +199,7 @@ void test_builtin_elementwise_clmul(int i, short s, double d, float4 v, // expected-error@-1 {{too many arguments to function call, expected 2, have 3}} i = __builtin_elementwise_clmul(v, v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'float4' (vector of 4 'float' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'float4' (vector of 4 'float' values))}} i = __builtin_elementwise_clmul(i, s); // expected-error@-1 {{arguments are of different types ('int' vs 'short')}} @@ -420,7 +420,7 @@ void test_builtin_elementwise_min(int i, short s, double d, float4 v, int3 iv, u void test_builtin_elementwise_maximum(int i, short s, float f, double d, float4 fv, double4 dv, int3 iv, unsigned3 uv, int *p) { i = __builtin_elementwise_maximum(p, d); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int *')}} struct Foo foo = __builtin_elementwise_maximum(d, d); // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'double'}} @@ -438,7 +438,7 @@ void test_builtin_elementwise_maximum(int i, short s, float f, double d, float4 // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'int3' (vector of 3 'int' values))}} i = __builtin_elementwise_maximum(uv, iv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned3' (vector of 3 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned3' (vector of 3 'unsigned int' values))}} dv = __builtin_elementwise_maximum(fv, dv); // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'double4' (vector of 4 'double' values))}} @@ -449,23 +449,23 @@ void test_builtin_elementwise_maximum(int i, short s, float f, double d, float4 fv = __builtin_elementwise_maximum(fv, fv); i = __builtin_elementwise_maximum(iv, iv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int3' (vector of 3 'int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int3' (vector of 3 'int' values))}} i = __builtin_elementwise_maximum(i, i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} int A[10]; A = __builtin_elementwise_maximum(A, A); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int *')}} _Complex float c1, c2; c1 = __builtin_elementwise_maximum(c1, c2); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was '_Complex float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was '_Complex float')}} } void test_builtin_elementwise_minimum(int i, short s, float f, double d, float4 fv, double4 dv, int3 iv, unsigned3 uv, int *p) { i = __builtin_elementwise_minimum(p, d); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int *')}} struct Foo foo = __builtin_elementwise_minimum(d, d); // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'double'}} @@ -483,7 +483,7 @@ void test_builtin_elementwise_minimum(int i, short s, float f, double d, float4 // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'int3' (vector of 3 'int' values))}} i = __builtin_elementwise_minimum(uv, iv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned3' (vector of 3 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned3' (vector of 3 'unsigned int' values))}} dv = __builtin_elementwise_minimum(fv, dv); // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'double4' (vector of 4 'double' values))}} @@ -494,23 +494,23 @@ void test_builtin_elementwise_minimum(int i, short s, float f, double d, float4 fv = __builtin_elementwise_minimum(fv, fv); i = __builtin_elementwise_minimum(iv, iv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int3' (vector of 3 'int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int3' (vector of 3 'int' values))}} i = __builtin_elementwise_minimum(i, i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} int A[10]; A = __builtin_elementwise_minimum(A, A); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int *')}} _Complex float c1, c2; c1 = __builtin_elementwise_minimum(c1, c2); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was '_Complex float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was '_Complex float')}} } void test_builtin_elementwise_maximumnum(int i, short s, float f, double d, float4 fv, double4 dv, int3 iv, unsigned3 uv, int *p) { i = __builtin_elementwise_maximumnum(p, d); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int *')}} struct Foo foo = __builtin_elementwise_maximumnum(d, d); // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'double'}} @@ -528,7 +528,7 @@ void test_builtin_elementwise_maximumnum(int i, short s, float f, double d, floa // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'int3' (vector of 3 'int' values))}} i = __builtin_elementwise_maximumnum(uv, iv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned3' (vector of 3 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned3' (vector of 3 'unsigned int' values))}} dv = __builtin_elementwise_maximumnum(fv, dv); // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'double4' (vector of 4 'double' values))}} @@ -539,23 +539,23 @@ void test_builtin_elementwise_maximumnum(int i, short s, float f, double d, floa fv = __builtin_elementwise_maximumnum(fv, fv); i = __builtin_elementwise_maximumnum(iv, iv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int3' (vector of 3 'int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int3' (vector of 3 'int' values))}} i = __builtin_elementwise_maximumnum(i, i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} int A[10]; A = __builtin_elementwise_maximumnum(A, A); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int *')}} _Complex float c1, c2; c1 = __builtin_elementwise_maximumnum(c1, c2); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was '_Complex float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was '_Complex float')}} } void test_builtin_elementwise_minimumnum(int i, short s, float f, double d, float4 fv, double4 dv, int3 iv, unsigned3 uv, int *p) { i = __builtin_elementwise_minimumnum(p, d); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int *')}} struct Foo foo = __builtin_elementwise_minimumnum(d, d); // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'double'}} @@ -573,7 +573,7 @@ void test_builtin_elementwise_minimumnum(int i, short s, float f, double d, floa // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'int3' (vector of 3 'int' values))}} i = __builtin_elementwise_minimumnum(uv, iv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned3' (vector of 3 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned3' (vector of 3 'unsigned int' values))}} dv = __builtin_elementwise_minimumnum(fv, dv); // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'double4' (vector of 4 'double' values))}} @@ -584,18 +584,18 @@ void test_builtin_elementwise_minimumnum(int i, short s, float f, double d, floa fv = __builtin_elementwise_minimumnum(fv, fv); i = __builtin_elementwise_minimumnum(iv, iv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int3' (vector of 3 'int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int3' (vector of 3 'int' values))}} i = __builtin_elementwise_minimumnum(i, i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} int A[10]; A = __builtin_elementwise_minimumnum(A, A); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int *')}} _Complex float c1, c2; c1 = __builtin_elementwise_minimumnum(c1, c2); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was '_Complex float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was '_Complex float')}} } void test_builtin_elementwise_bitreverse(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -607,16 +607,16 @@ void test_builtin_elementwise_bitreverse(int i, float f, double d, float4 v, int // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_bitreverse(f); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'float')}} i = __builtin_elementwise_bitreverse(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_bitreverse(d); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'double')}} v = __builtin_elementwise_bitreverse(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'float4' (vector of 4 'float' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'float4' (vector of 4 'float' values))}} } void test_builtin_elementwise_ceil(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -628,16 +628,16 @@ void test_builtin_elementwise_ceil(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_ceil(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_ceil(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_ceil(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_ceil(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_acos(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -649,16 +649,16 @@ void test_builtin_elementwise_acos(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_acos(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_acos(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_acos(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_acos(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_cos(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -670,16 +670,16 @@ void test_builtin_elementwise_cos(int i, float f, double d, float4 v, int3 iv, u // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_cos(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_cos(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_cos(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_cos(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_cosh(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -691,16 +691,16 @@ void test_builtin_elementwise_cosh(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_cosh(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_cosh(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_cosh(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_cosh(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_exp(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -712,16 +712,16 @@ void test_builtin_elementwise_exp(int i, float f, double d, float4 v, int3 iv, u // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_exp(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_exp(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_exp(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_exp(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_exp2(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -733,16 +733,16 @@ void test_builtin_elementwise_exp2(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_exp2(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_exp2(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_exp2(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_exp2(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_exp10(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -754,16 +754,16 @@ void test_builtin_elementwise_exp10(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_exp10(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_exp10(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_exp10(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_exp10(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_ldexp(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -781,10 +781,10 @@ void test_builtin_elementwise_ldexp(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too many arguments to function call, expected 2, have 3}} f = __builtin_elementwise_ldexp(i, i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} f = __builtin_elementwise_ldexp(f, f); - // expected-error@-1 {{2nd argument must be a scalar, vector, or matrix of integer types (was 'float')}} + // expected-error@-1 {{2nd argument must be a scalar or vector of integer types (was 'float')}} f = __builtin_elementwise_ldexp(v, iv); // expected-error@-1 {{vector operands do not have the same number of elements ('float4' (vector of 4 'float' values) and 'int3' (vector of 3 'int' values))}} @@ -796,10 +796,10 @@ void test_builtin_elementwise_ldexp(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{vector operands do not have the same number of elements ('float' and 'int3' (vector of 3 'int' values))}} f = __builtin_elementwise_ldexp(u, i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} f = __builtin_elementwise_ldexp(uv, i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_floor(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -811,16 +811,16 @@ void test_builtin_elementwise_floor(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_floor(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_floor(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_floor(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_floor(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_log(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -832,16 +832,16 @@ void test_builtin_elementwise_log(int i, float f, double d, float4 v, int3 iv, u // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_log(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_log(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_log(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_log(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_log10(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -853,16 +853,16 @@ void test_builtin_elementwise_log10(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_log10(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_log10(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_log10(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_log10(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_log2(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -874,16 +874,16 @@ void test_builtin_elementwise_log2(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_log2(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_log2(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_log2(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_log2(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_popcount(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -895,16 +895,16 @@ void test_builtin_elementwise_popcount(int i, float f, double d, float4 v, int3 // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_popcount(f); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'float')}} i = __builtin_elementwise_popcount(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_popcount(d); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'double')}} v = __builtin_elementwise_popcount(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'float4' (vector of 4 'float' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'float4' (vector of 4 'float' values))}} int2 i2 = __builtin_elementwise_popcount(iv); // expected-error@-1 {{initializing 'int2' (vector of 2 'int' values) with an expression of incompatible type 'int3' (vector of 3 'int' values)}} @@ -921,10 +921,10 @@ void test_builtin_elementwise_popcount(int i, float f, double d, float4 v, int3 void test_builtin_elementwise_fmod(int i, short s, double d, float4 v, int3 iv, unsigned3 uv, int *p) { i = __builtin_elementwise_fmod(p, d); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int *')}} struct Foo foo = __builtin_elementwise_fmod(i, i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_fmod(i); // expected-error@-1 {{too few arguments to function call, expected 2, have 1}} @@ -939,7 +939,7 @@ void test_builtin_elementwise_fmod(int i, short s, double d, float4 v, int3 iv, // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'int3' (vector of 3 'int' values))}} i = __builtin_elementwise_fmod(uv, iv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned3' (vector of 3 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned3' (vector of 3 'unsigned int' values))}} i = __builtin_elementwise_fmod(d, v); // expected-error@-1 {{arguments are of different types ('double' vs 'float4' (vector of 4 'float' values))}} @@ -947,10 +947,10 @@ void test_builtin_elementwise_fmod(int i, short s, double d, float4 v, int3 iv, void test_builtin_elementwise_pow(int i, short s, double d, float4 v, int3 iv, unsigned3 uv, int *p) { i = __builtin_elementwise_pow(p, d); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int *')}} struct Foo foo = __builtin_elementwise_pow(i, i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_pow(i); // expected-error@-1 {{too few arguments to function call, expected 2, have 1}} @@ -965,7 +965,7 @@ void test_builtin_elementwise_pow(int i, short s, double d, float4 v, int3 iv, u // expected-error@-1 {{arguments are of different types ('float4' (vector of 4 'float' values) vs 'int3' (vector of 3 'int' values))}} i = __builtin_elementwise_pow(uv, iv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned3' (vector of 3 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned3' (vector of 3 'unsigned int' values))}} } @@ -978,16 +978,16 @@ void test_builtin_elementwise_roundeven(int i, float f, double d, float4 v, int3 // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_roundeven(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_roundeven(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_roundeven(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_roundeven(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_round(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -998,20 +998,20 @@ void test_builtin_elementwise_round(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_round(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_round(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_round(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_round(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} _Complex float c1, c2; c1 = __builtin_elementwise_round(c1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was '_Complex float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was '_Complex float')}} } void test_builtin_elementwise_rint(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -1022,20 +1022,20 @@ void test_builtin_elementwise_rint(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_rint(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_rint(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_rint(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_rint(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} _Complex float c1, c2; c1 = __builtin_elementwise_rint(c1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was '_Complex float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was '_Complex float')}} } void test_builtin_elementwise_nearbyint(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -1046,20 +1046,20 @@ void test_builtin_elementwise_nearbyint(int i, float f, double d, float4 v, int3 // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_nearbyint(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_nearbyint(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_nearbyint(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_nearbyint(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} _Complex float c1, c2; c1 = __builtin_elementwise_nearbyint(c1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was '_Complex float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was '_Complex float')}} } void test_builtin_elementwise_asin(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -1071,16 +1071,16 @@ void test_builtin_elementwise_asin(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_asin(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_asin(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_asin(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_asin(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_sin(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -1092,16 +1092,16 @@ void test_builtin_elementwise_sin(int i, float f, double d, float4 v, int3 iv, u // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_sin(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_sin(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_sin(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_sin(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_sinh(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -1113,16 +1113,16 @@ void test_builtin_elementwise_sinh(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_sinh(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_sinh(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_sinh(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_sinh(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_sqrt(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -1134,16 +1134,16 @@ void test_builtin_elementwise_sqrt(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_sqrt(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_sqrt(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_sqrt(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_sqrt(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_atan(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -1155,16 +1155,16 @@ void test_builtin_elementwise_atan(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_atan(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_atan(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_atan(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_atan(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_atan2(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -1179,16 +1179,16 @@ void test_builtin_elementwise_atan2(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 2, have 1}} i = __builtin_elementwise_atan2(i, i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_atan2(f, f, f); // expected-error@-1 {{too many arguments to function call, expected 2, have 3}} u = __builtin_elementwise_atan2(u, u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_atan2(uv, uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_tan(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -1200,16 +1200,16 @@ void test_builtin_elementwise_tan(int i, float f, double d, float4 v, int3 iv, u // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_tan(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_tan(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_tan(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_tan(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_tanh(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -1221,16 +1221,16 @@ void test_builtin_elementwise_tanh(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_tanh(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_tanh(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_tanh(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_tanh(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_trunc(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -1242,16 +1242,16 @@ void test_builtin_elementwise_trunc(int i, float f, double d, float4 v, int3 iv, // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_trunc(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_trunc(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_trunc(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_trunc(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_canonicalize(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) { @@ -1263,24 +1263,24 @@ void test_builtin_elementwise_canonicalize(int i, float f, double d, float4 v, i // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} i = __builtin_elementwise_canonicalize(i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_canonicalize(f, f); // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} u = __builtin_elementwise_canonicalize(u); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned int')}} uv = __builtin_elementwise_canonicalize(uv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned4' (vector of 4 'unsigned int' values))}} } void test_builtin_elementwise_copysign(int i, short s, double d, float f, float4 v, int3 iv, unsigned3 uv, int *p) { i = __builtin_elementwise_copysign(p, d); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int *')}} i = __builtin_elementwise_copysign(i, i); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} i = __builtin_elementwise_copysign(i); // expected-error@-1 {{too few arguments to function call, expected 2, have 1}} @@ -1292,32 +1292,32 @@ void test_builtin_elementwise_copysign(int i, short s, double d, float f, float4 // expected-error@-1 {{too many arguments to function call, expected 2, have 3}} i = __builtin_elementwise_copysign(v, iv); - // expected-error@-1 {{2nd argument must be a scalar, vector, or matrix of floating-point types (was 'int3' (vector of 3 'int' values))}} + // expected-error@-1 {{2nd argument must be a scalar or vector of floating-point types (was 'int3' (vector of 3 'int' values))}} i = __builtin_elementwise_copysign(uv, iv); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'unsigned3' (vector of 3 'unsigned int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'unsigned3' (vector of 3 'unsigned int' values))}} s = __builtin_elementwise_copysign(i, s); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} f = __builtin_elementwise_copysign(f, i); - // expected-error@-1 {{2nd argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{2nd argument must be a scalar or vector of floating-point types (was 'int')}} f = __builtin_elementwise_copysign(i, f); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} enum e { one, two }; i = __builtin_elementwise_copysign(one, two); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} enum f { three }; enum f x = __builtin_elementwise_copysign(one, three); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} _BitInt(32) ext; // expected-warning {{'_BitInt' in C17 and earlier is a Clang extension}} ext = __builtin_elementwise_copysign(ext, ext); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was '_BitInt(32)')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was '_BitInt(32)')}} const float cf32 = 0.0f; f = __builtin_elementwise_copysign(cf32, f); @@ -1329,7 +1329,7 @@ void test_builtin_elementwise_copysign(int i, short s, double d, float f, float4 float A[10]; A = __builtin_elementwise_copysign(A, A); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'float *')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'float *')}} float(ii); float j; @@ -1337,7 +1337,7 @@ void test_builtin_elementwise_copysign(int i, short s, double d, float f, float4 _Complex float c1, c2; c1 = __builtin_elementwise_copysign(c1, c2); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was '_Complex float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was '_Complex float')}} double f64 = 0.0; double tmp0 = __builtin_elementwise_copysign(f64, f); @@ -1426,30 +1426,30 @@ void test_builtin_elementwise_fma(int i32, int2 v2i32, short i16, // expected-error@-1 {{arguments are of different types ('double' vs 'double2' (vector of 2 'double' values)}} i32 = __builtin_elementwise_fma(i32, i32, i32); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} v2i32 = __builtin_elementwise_fma(v2i32, v2i32, v2i32); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int2' (vector of 2 'int' values))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int2' (vector of 2 'int' values))}} f32 = __builtin_elementwise_fma(f32, f32, i32); - // expected-error@-1 {{3rd argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{3rd argument must be a scalar or vector of floating-point types (was 'int')}} f32 = __builtin_elementwise_fma(f32, i32, f32); - // expected-error@-1 {{2nd argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{2nd argument must be a scalar or vector of floating-point types (was 'int')}} f32 = __builtin_elementwise_fma(f32, f32, i32); - // expected-error@-1 {{3rd argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{3rd argument must be a scalar or vector of floating-point types (was 'int')}} _Complex float c1, c2, c3; c1 = __builtin_elementwise_fma(c1, f32, f32); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was '_Complex float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was '_Complex float')}} c2 = __builtin_elementwise_fma(f32, c2, f32); - // expected-error@-1 {{2nd argument must be a scalar, vector, or matrix of floating-point types (was '_Complex float')}} + // expected-error@-1 {{2nd argument must be a scalar or vector of floating-point types (was '_Complex float')}} c3 = __builtin_elementwise_fma(f32, f32, c3); - // expected-error@-1 {{3rd argument must be a scalar, vector, or matrix of floating-point types (was '_Complex float')}} + // expected-error@-1 {{3rd argument must be a scalar or vector of floating-point types (was '_Complex float')}} } void test_builtin_elementwise_fsh(int i32, int2 v2i32, short i16, int3 v3i32, @@ -1473,13 +1473,13 @@ void test_builtin_elementwise_fsh(int i32, int2 v2i32, short i16, int3 v3i32, // expected-error@-1 {{arguments are of different types ('short' vs 'int')}} f32 = __builtin_elementwise_fshl(f32, f32, f32); - // expected-error@-1 {{argument must be a scalar, vector, or matrix of integer types (was 'float')}} + // expected-error@-1 {{argument must be a scalar or vector of integer types (was 'float')}} f64 = __builtin_elementwise_fshr(f64, f64, f64); - // expected-error@-1 {{argument must be a scalar, vector, or matrix of integer types (was 'double')}} + // expected-error@-1 {{argument must be a scalar or vector of integer types (was 'double')}} v2i32 = __builtin_elementwise_fshl(v2i32, v2i32, v2f32); - // expected-error@-1 {{argument must be a scalar, vector, or matrix of integer types (was 'float2' (vector of 2 'float' values))}} + // expected-error@-1 {{argument must be a scalar or vector of integer types (was 'float2' (vector of 2 'float' values))}} v2i32 = __builtin_elementwise_fshr(v2i32, v2i32, v3i32); // expected-error@-1 {{arguments are of different types ('int2' (vector of 2 'int' values) vs 'int3' (vector of 3 'int' values))}} @@ -1518,11 +1518,11 @@ cfloat4 quux(cfloat4 x, float4 y) { void test_builtin_elementwise_clzg(int i32, int2 v2i32, short i16, double f64, double2 v2f64) { f64 = __builtin_elementwise_clzg(f64); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'double')}} _Complex float c1; c1 = __builtin_elementwise_clzg(c1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was '_Complex float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was '_Complex float')}} v2i32 = __builtin_elementwise_clzg(v2i32, i32); // expected-error@-1 {{arguments are of different types ('int2' (vector of 2 'int' values) vs 'int')}} @@ -1540,11 +1540,11 @@ void test_builtin_elementwise_clzg(int i32, int2 v2i32, short i16, void test_builtin_elementwise_ctzg(int i32, int2 v2i32, short i16, double f64, double2 v2f64) { f64 = __builtin_elementwise_ctzg(f64); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'double')}} _Complex float c1; c1 = __builtin_elementwise_ctzg(c1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was '_Complex float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was '_Complex float')}} v2i32 = __builtin_elementwise_ctzg(v2i32, i32); // expected-error@-1 {{arguments are of different types ('int2' (vector of 2 'int' values) vs 'int')}} diff --git a/clang/test/Sema/riscv-rvv-vector-exp-ops.c b/clang/test/Sema/riscv-rvv-vector-exp-ops.c index fd2197b3718a7..f6b544123fa1c 100644 --- a/clang/test/Sema/riscv-rvv-vector-exp-ops.c +++ b/clang/test/Sema/riscv-rvv-vector-exp-ops.c @@ -9,11 +9,11 @@ vfloat32mf2_t test_exp_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_exp(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } vfloat32mf2_t test_exp2_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_exp2(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } diff --git a/clang/test/Sema/riscv-rvv-vector-log-ops.c b/clang/test/Sema/riscv-rvv-vector-log-ops.c index e58ce1345cd43..2674bed95a108 100644 --- a/clang/test/Sema/riscv-rvv-vector-log-ops.c +++ b/clang/test/Sema/riscv-rvv-vector-log-ops.c @@ -9,17 +9,17 @@ vfloat32mf2_t test_log_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_log(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } vfloat32mf2_t test_log10_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_log10(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } vfloat32mf2_t test_log2_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_log2(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } diff --git a/clang/test/Sema/riscv-rvv-vector-trig-ops.c b/clang/test/Sema/riscv-rvv-vector-trig-ops.c index 8d4bdee19587f..7825816d820b6 100644 --- a/clang/test/Sema/riscv-rvv-vector-trig-ops.c +++ b/clang/test/Sema/riscv-rvv-vector-trig-ops.c @@ -8,60 +8,60 @@ vfloat32mf2_t test_asin_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_asin(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } vfloat32mf2_t test_acos_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_acos(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } vfloat32mf2_t test_atan_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_atan(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } vfloat32mf2_t test_atan2_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_atan2(v, v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } vfloat32mf2_t test_sin_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_sin(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } vfloat32mf2_t test_cos_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_cos(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } vfloat32mf2_t test_tan_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_tan(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } vfloat32mf2_t test_sinh_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_sinh(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } vfloat32mf2_t test_cosh_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_cosh(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } vfloat32mf2_t test_tanh_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_tanh(v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types}} } diff --git a/clang/test/Sema/riscv-sve-vector-pow-ops.c b/clang/test/Sema/riscv-sve-vector-pow-ops.c index b1000eb6b747a..7edaf4099c314 100644 --- a/clang/test/Sema/riscv-sve-vector-pow-ops.c +++ b/clang/test/Sema/riscv-sve-vector-pow-ops.c @@ -9,5 +9,5 @@ vfloat32mf2_t test_pow_vv_i8mf8(vfloat32mf2_t v) { return __builtin_elementwise_pow(v, v); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point type}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point type}} } diff --git a/clang/test/SemaHLSL/BuiltIns/atan2-errors_mat.hlsl b/clang/test/SemaHLSL/BuiltIns/atan2-errors_mat.hlsl index c0b13ca7ba535..19467d99d2292 100644 --- a/clang/test/SemaHLSL/BuiltIns/atan2-errors_mat.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/atan2-errors_mat.hlsl @@ -3,5 +3,5 @@ double2x2 test_vec_double_builtin(double2x2 p0, double2x2 p1) { return __builtin_elementwise_atan2(p0, p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double2x2' (aka 'matrix'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double2x2' (aka 'matrix'))}} } diff --git a/clang/test/SemaHLSL/BuiltIns/ddx-coarse-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ddx-coarse-errors.hlsl index 1479793770672..ebad1cc6826d8 100644 --- a/clang/test/SemaHLSL/BuiltIns/ddx-coarse-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/ddx-coarse-errors.hlsl @@ -13,10 +13,10 @@ float too_many_args(float val) { float test_integer_scalar_input(int val) { return __builtin_hlsl_elementwise_ddx_coarse(val); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int')}} } double test_double_scalar_input(double val) { return __builtin_hlsl_elementwise_ddx_coarse(val); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double')}} } diff --git a/clang/test/SemaHLSL/BuiltIns/ddx-fine-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ddx-fine-errors.hlsl index 8f96caa25a2b2..71196943c322a 100644 --- a/clang/test/SemaHLSL/BuiltIns/ddx-fine-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/ddx-fine-errors.hlsl @@ -13,10 +13,10 @@ float too_many_args(float val) { float test_integer_scalar_input(int val) { return __builtin_hlsl_elementwise_ddx_fine(val); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int')}} } double test_double_scalar_input(double val) { return __builtin_hlsl_elementwise_ddx_fine(val); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double')}} } diff --git a/clang/test/SemaHLSL/BuiltIns/ddy-coarse-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ddy-coarse-errors.hlsl index 1303b84d4a92b..9cc23665882c8 100644 --- a/clang/test/SemaHLSL/BuiltIns/ddy-coarse-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/ddy-coarse-errors.hlsl @@ -13,10 +13,10 @@ float too_many_args(float val) { float test_integer_scalar_input(int val) { return __builtin_hlsl_elementwise_ddy_coarse(val); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int')}} } double test_double_scalar_input(double val) { return __builtin_hlsl_elementwise_ddy_coarse(val); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double')}} } diff --git a/clang/test/SemaHLSL/BuiltIns/ddy-fine-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/ddy-fine-errors.hlsl index 637dd2f012a25..e17d5f36832b2 100644 --- a/clang/test/SemaHLSL/BuiltIns/ddy-fine-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/ddy-fine-errors.hlsl @@ -13,10 +13,10 @@ float too_many_args(float val) { float test_integer_scalar_input(int val) { return __builtin_hlsl_elementwise_ddy_fine(val); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int')}} } double test_double_scalar_input(double val) { return __builtin_hlsl_elementwise_ddy_fine(val); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double')}} } diff --git a/clang/test/SemaHLSL/BuiltIns/degrees-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/degrees-errors.hlsl index 1255874239c8c..637a6eecfb35a 100644 --- a/clang/test/SemaHLSL/BuiltIns/degrees-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/degrees-errors.hlsl @@ -12,15 +12,15 @@ float2 test_too_many_arg(float2 p0) { float builtin_bool_to_float_type_promotion(bool p1) { return __builtin_hlsl_elementwise_degrees(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'bool')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'bool')}} } float builtin_degrees_int_to_float_promotion(int p1) { return __builtin_hlsl_elementwise_degrees(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int')}} } float2 builtin_degrees_int2_to_float2_promotion(int2 p1) { return __builtin_hlsl_elementwise_degrees(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} } diff --git a/clang/test/SemaHLSL/BuiltIns/exp-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/exp-errors.hlsl index 087a7edeb13cb..f0076ac4e5881 100644 --- a/clang/test/SemaHLSL/BuiltIns/exp-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/exp-errors.hlsl @@ -14,15 +14,15 @@ float2 test_too_many_arg(float2 p0) { float builtin_bool_to_float_type_promotion(bool p1) { return TEST_FUNC(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'bool')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'bool')}} } float builtin_exp_int_to_float_promotion(int p1) { return TEST_FUNC(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} } float2 builtin_exp_int2_to_float2_promotion(int2 p1) { return TEST_FUNC(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int2' (aka 'vector'))}} } diff --git a/clang/test/SemaHLSL/BuiltIns/f16tof32-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/f16tof32-errors.hlsl index 545bc27bd366d..a39a9970a176c 100644 --- a/clang/test/SemaHLSL/BuiltIns/f16tof32-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/f16tof32-errors.hlsl @@ -24,7 +24,7 @@ float builtin_f16tof32_bool4(bool4 p0) { float builtin_f16tof32_short(short p0) { return __builtin_hlsl_elementwise_f16tof32(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of unsigned integer types (was 'short')}} + // expected-error@-1 {{1st argument must be a scalar or vector of unsigned integer types (was 'short')}} } float builtin_f16tof32_unsigned_short(unsigned short p0) { @@ -34,37 +34,37 @@ float builtin_f16tof32_unsigned_short(unsigned short p0) { float builtin_f16tof32_int(int p0) { return __builtin_hlsl_elementwise_f16tof32(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of unsigned integer types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of unsigned integer types (was 'int')}} } float builtin_f16tof32_int64_t(long p0) { return __builtin_hlsl_elementwise_f16tof32(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of unsigned integer types (was 'long')}} + // expected-error@-1 {{1st argument must be a scalar or vector of unsigned integer types (was 'long')}} } float2 builtin_f16tof32_int2_to_float2_promotion(int2 p0) { return __builtin_hlsl_elementwise_f16tof32(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of unsigned integer types (was 'int2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of unsigned integer types (was 'int2' (aka 'vector'))}} } float builtin_f16tof32_half(half p0) { return __builtin_hlsl_elementwise_f16tof32(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of unsigned integer types (was 'half')}} + // expected-error@-1 {{1st argument must be a scalar or vector of unsigned integer types (was 'half')}} } float builtin_f16tof32_half4(half4 p0) { return __builtin_hlsl_elementwise_f16tof32(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of unsigned integer types (was 'half4' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of unsigned integer types (was 'half4' (aka 'vector'))}} } float builtin_f16tof32_float(float p0) { return __builtin_hlsl_elementwise_f16tof32(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of unsigned integer types (was 'float')}} + // expected-error@-1 {{1st argument must be a scalar or vector of unsigned integer types (was 'float')}} } float builtin_f16tof32_double(double p0) { return __builtin_hlsl_elementwise_f16tof32(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of unsigned integer types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of unsigned integer types (was 'double')}} } float f16tof32_too_few_arg() { diff --git a/clang/test/SemaHLSL/BuiltIns/frac-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/frac-errors.hlsl index a95ea1b65af60..cdf2b61c45207 100644 --- a/clang/test/SemaHLSL/BuiltIns/frac-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/frac-errors.hlsl @@ -13,16 +13,16 @@ float2 test_too_many_arg(float2 p0) { float builtin_bool_to_float_type_promotion(bool p1) { return __builtin_hlsl_elementwise_frac(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'bool')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'bool')}} } float builtin_frac_int_to_float_promotion(int p1) { return __builtin_hlsl_elementwise_frac(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int')}} } float2 builtin_frac_int2_to_float2_promotion(int2 p1) { return __builtin_hlsl_elementwise_frac(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} } diff --git a/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl index 84364afec1229..e9cc0ed338e3e 100644 --- a/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl @@ -23,10 +23,10 @@ double test_double_builtin(double p0) { return TEST_FUNC(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double')}} } double2 test_vec_double_builtin(double2 p0) { return TEST_FUNC(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double2' (aka 'vector'))}} } diff --git a/clang/test/SemaHLSL/BuiltIns/half-float-only-errors2.hlsl b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors2.hlsl index ab7aec0ae080e..9e10e1afa9385 100644 --- a/clang/test/SemaHLSL/BuiltIns/half-float-only-errors2.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors2.hlsl @@ -4,10 +4,10 @@ double test_double_builtin(double p0, double p1) { return TEST_FUNC(p0, p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double')}} } double2 test_vec_double_builtin(double2 p0, double2 p1) { return TEST_FUNC(p0, p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double2' (aka 'vector'))}} } diff --git a/clang/test/SemaHLSL/BuiltIns/isinf-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/isinf-errors.hlsl index 4084399a8dc79..a32bc9628a295 100644 --- a/clang/test/SemaHLSL/BuiltIns/isinf-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/isinf-errors.hlsl @@ -13,26 +13,26 @@ bool2 test_too_many_arg(float2 p0) { bool builtin_bool_to_float_type_promotion(bool p1) { return __builtin_hlsl_elementwise_isinf(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'bool')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'bool')}} } bool builtin_isinf_int_to_float_promotion(int p1) { return __builtin_hlsl_elementwise_isinf(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int')}} } bool2 builtin_isinf_int2_to_float2_promotion(int2 p1) { return __builtin_hlsl_elementwise_isinf(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} } // builtins are variadic functions and so are subject to DefaultVariadicArgumentPromotion half builtin_isinf_half_scalar (half p0) { return __builtin_hlsl_elementwise_isinf (p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double')}} } float builtin_isinf_float_scalar ( float p0) { return __builtin_hlsl_elementwise_isinf (p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double')}} } diff --git a/clang/test/SemaHLSL/BuiltIns/isnan-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/isnan-errors.hlsl index 0f5722fa52aa9..625c415f91de2 100644 --- a/clang/test/SemaHLSL/BuiltIns/isnan-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/isnan-errors.hlsl @@ -13,26 +13,26 @@ bool2 test_too_many_arg(float2 p0) { bool builtin_bool_to_float_type_promotion(bool p1) { return __builtin_hlsl_elementwise_isnan(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'bool')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'bool')}} } bool builtin_isnan_int_to_float_promotion(int p1) { return __builtin_hlsl_elementwise_isnan(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int')}} } bool2 builtin_isnan_int2_to_float2_promotion(int2 p1) { return __builtin_hlsl_elementwise_isnan(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} } // builtins are variadic functions and so are subject to DefaultVariadicArgumentPromotion half builtin_isnan_half_scalar (half p0) { return __builtin_hlsl_elementwise_isnan (p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double')}} } float builtin_isnan_float_scalar ( float p0) { return __builtin_hlsl_elementwise_isnan (p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double')}} } diff --git a/clang/test/SemaHLSL/BuiltIns/lerp-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/lerp-errors.hlsl index d2a551cab5e20..bc82991eb3e6b 100644 --- a/clang/test/SemaHLSL/BuiltIns/lerp-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/lerp-errors.hlsl @@ -71,12 +71,12 @@ float2 test_builtin_lerp_float2_splat(float p0, float2 p1) { float2 test_builtin_lerp_float2_splat2(double p0, double2 p1) { return __builtin_hlsl_lerp(p1, p0, p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double2' (aka 'vector'))}} } float2 test_builtin_lerp_float2_splat3(double p0, double2 p1) { return __builtin_hlsl_lerp(p1, p1, p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double2' (aka 'vector'))}} } float3 test_builtin_lerp_float3_splat(float p0, float3 p1) { @@ -91,40 +91,40 @@ float4 test_builtin_lerp_float4_splat(float p0, float4 p1) { float2 test_lerp_float2_int_splat(float2 p0, int p1) { return __builtin_hlsl_lerp(p0, p1, p1); - // expected-error@-1 {{2nd argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int')}} + // expected-error@-1 {{2nd argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int')}} } float3 test_lerp_float3_int_splat(float3 p0, int p1) { return __builtin_hlsl_lerp(p0, p1, p1); - // expected-error@-1 {{2nd argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int')}} + // expected-error@-1 {{2nd argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int')}} } float2 test_builtin_lerp_int_vect_to_float_vec_promotion(int2 p0, float p1) { return __builtin_hlsl_lerp(p0, p1, p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} } float test_builtin_lerp_bool_type_promotion(bool p0) { return __builtin_hlsl_lerp(p0, p0, p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'bool')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'bool')}} } float builtin_bool_to_float_type_promotion(float p0, bool p1) { return __builtin_hlsl_lerp(p0, p0, p1); - // expected-error@-1 {{3rd argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'bool')}} + // expected-error@-1 {{3rd argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'bool')}} } float builtin_bool_to_float_type_promotion2(bool p0, float p1) { return __builtin_hlsl_lerp(p1, p0, p1); - // expected-error@-1 {{2nd argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'bool')}} + // expected-error@-1 {{2nd argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'bool')}} } float builtin_lerp_int_to_float_promotion(float p0, int p1) { return __builtin_hlsl_lerp(p0, p0, p1); - // expected-error@-1 {{3rd argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int')}} + // expected-error@-1 {{3rd argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int')}} } float4 test_lerp_int4(int4 p0, int4 p1, int4 p2) { return __builtin_hlsl_lerp(p0, p1, p2); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int4' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int4' (aka 'vector'))}} } diff --git a/clang/test/SemaHLSL/BuiltIns/normalize-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/normalize-errors.hlsl index f8e3367c99fde..377c2d5e41a73 100644 --- a/clang/test/SemaHLSL/BuiltIns/normalize-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/normalize-errors.hlsl @@ -15,17 +15,17 @@ void test_too_many_arg(float2 p0) bool builtin_bool_to_float_type_promotion(bool p1) { return __builtin_hlsl_normalize(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'bool')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'bool')}} } bool builtin_normalize_int_to_float_promotion(int p1) { return __builtin_hlsl_normalize(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int')}} } bool2 builtin_normalize_int2_to_float2_promotion(int2 p1) { return __builtin_hlsl_normalize(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} } diff --git a/clang/test/SemaHLSL/BuiltIns/radians-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/radians-errors.hlsl index 7b24ee780e8ea..70e5b671bb3c9 100644 --- a/clang/test/SemaHLSL/BuiltIns/radians-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/radians-errors.hlsl @@ -12,16 +12,16 @@ float2 test_too_many_arg(float2 p0) { float builtin_bool_to_float_type_promotion(bool p1) { return __builtin_hlsl_elementwise_radians(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'bool')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'bool')}} } float builtin_radians_int_to_float_promotion(int p1) { return __builtin_hlsl_elementwise_radians(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int')}} } float2 builtin_radians_int2_to_float2_promotion(int2 p1) { return __builtin_hlsl_elementwise_radians(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} } diff --git a/clang/test/SemaHLSL/BuiltIns/rcp-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/rcp-errors.hlsl index 3d10f9901f53b..79076b4815a6e 100644 --- a/clang/test/SemaHLSL/BuiltIns/rcp-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/rcp-errors.hlsl @@ -12,15 +12,15 @@ float2 test_too_many_arg(float2 p0) { float builtin_bool_to_float_type_promotion(bool p1) { return __builtin_hlsl_elementwise_rcp(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'bool')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'bool')}} } float builtin_rcp_int_to_float_promotion(int p1) { return __builtin_hlsl_elementwise_rcp(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} } float2 builtin_rcp_int2_to_float2_promotion(int2 p1) { return __builtin_hlsl_elementwise_rcp(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int2' (aka 'vector'))}} } diff --git a/clang/test/SemaHLSL/BuiltIns/reversebits-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/reversebits-errors.hlsl index 7b6ed7b334343..5b33b89cb8eb8 100644 --- a/clang/test/SemaHLSL/BuiltIns/reversebits-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/reversebits-errors.hlsl @@ -3,10 +3,10 @@ double2 test_int_builtin(double2 p0) { return __builtin_elementwise_bitreverse(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of integer types (was 'double2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'double2' (aka 'vector'))}} } int2 test_int_builtin(int2 p0) { return __builtin_elementwise_bitreverse(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of unsigned integer types (was 'int2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of unsigned integer types (was 'int2' (aka 'vector'))}} } diff --git a/clang/test/SemaHLSL/BuiltIns/round-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/round-errors.hlsl index 9ea0840b94a67..54feed35379d7 100644 --- a/clang/test/SemaHLSL/BuiltIns/round-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/round-errors.hlsl @@ -13,15 +13,15 @@ float2 test_too_many_arg(float2 p0) { float builtin_bool_to_float_type_promotion(bool p1) { return __builtin_elementwise_round(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'bool')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'bool')}} } float builtin_round_int_to_float_promotion(int p1) { return __builtin_elementwise_round(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} } float2 builtin_round_int2_to_float2_promotion(int2 p1) { return __builtin_elementwise_round(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int2' (aka 'vector'))}} } diff --git a/clang/test/SemaHLSL/BuiltIns/rsqrt-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/rsqrt-errors.hlsl index bade79cb50afc..cedfcca35225e 100644 --- a/clang/test/SemaHLSL/BuiltIns/rsqrt-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/rsqrt-errors.hlsl @@ -12,20 +12,20 @@ float2 test_too_many_arg(float2 p0) { float builtin_bool_to_float_type_promotion(bool p1) { return __builtin_hlsl_elementwise_rsqrt(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'bool')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'bool')}} } float builtin_rsqrt_int_to_float_promotion(int p1) { return __builtin_hlsl_elementwise_rsqrt(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int')}} } float2 builtin_rsqrt_int2_to_float2_promotion(int2 p1) { return __builtin_hlsl_elementwise_rsqrt(p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} } double builtin_rsqrt_double(double p0) { return __builtin_hlsl_elementwise_rsqrt(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'double')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'double')}} } diff --git a/clang/test/SemaHLSL/BuiltIns/step-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/step-errors.hlsl index 224ebd7f5cc19..993450a17ebfb 100644 --- a/clang/test/SemaHLSL/BuiltIns/step-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/step-errors.hlsl @@ -15,17 +15,17 @@ void test_too_many_arg(float2 p0) bool builtin_bool_to_float_type_promotion(bool p1) { return __builtin_hlsl_step(p1, p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'bool')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'bool')}} } bool builtin_step_int_to_float_promotion(int p1) { return __builtin_hlsl_step(p1, p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int')}} } bool2 builtin_step_int2_to_float2_promotion(int2 p1) { return __builtin_hlsl_step(p1, p1); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} + // expected-error@-1 {{1st argument must be a scalar or vector of 16 or 32 bit floating-point types (was 'int2' (aka 'vector'))}} } diff --git a/clang/test/SemaSPIRV/BuiltIns/ddx-errors.c b/clang/test/SemaSPIRV/BuiltIns/ddx-errors.c index 58ca5d767f08c..a29af5dc43e30 100644 --- a/clang/test/SemaSPIRV/BuiltIns/ddx-errors.c +++ b/clang/test/SemaSPIRV/BuiltIns/ddx-errors.c @@ -15,7 +15,7 @@ float test_too_many_arg(float p0) { float test_int_scalar_inputs(int p0) { return __builtin_spirv_ddx(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} } float test_mismatched_return(float2 p0) { diff --git a/clang/test/SemaSPIRV/BuiltIns/ddy-errors.c b/clang/test/SemaSPIRV/BuiltIns/ddy-errors.c index 7164f728a73cf..65c37af1369c5 100644 --- a/clang/test/SemaSPIRV/BuiltIns/ddy-errors.c +++ b/clang/test/SemaSPIRV/BuiltIns/ddy-errors.c @@ -15,7 +15,7 @@ float test_too_many_arg(float p0) { float test_int_scalar_inputs(int p0) { return __builtin_spirv_ddy(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} } float test_mismatched_return(float2 p0) { diff --git a/clang/test/SemaSPIRV/BuiltIns/faceforward-errors.c b/clang/test/SemaSPIRV/BuiltIns/faceforward-errors.c index eef1e1fdb103d..d708df2843fab 100644 --- a/clang/test/SemaSPIRV/BuiltIns/faceforward-errors.c +++ b/clang/test/SemaSPIRV/BuiltIns/faceforward-errors.c @@ -20,7 +20,7 @@ float2 test_too_many_arg(float2 p0) { int test_int_scalar_inputs(int p0) { return __builtin_spirv_faceforward(p0, p0, p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} } float test_int_scalar_inputs2(float p0, int p1) { diff --git a/clang/test/SemaSPIRV/BuiltIns/fwidth-errors.c b/clang/test/SemaSPIRV/BuiltIns/fwidth-errors.c index 57e18f7e58942..44cdd819e4332 100644 --- a/clang/test/SemaSPIRV/BuiltIns/fwidth-errors.c +++ b/clang/test/SemaSPIRV/BuiltIns/fwidth-errors.c @@ -15,7 +15,7 @@ float test_too_many_arg(float p0) { float test_int_scalar_inputs(int p0) { return __builtin_spirv_fwidth(p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} } float test_mismatched_return(float2 p0) { diff --git a/clang/test/SemaSPIRV/BuiltIns/smoothstep-errors.c b/clang/test/SemaSPIRV/BuiltIns/smoothstep-errors.c index b39c853f41c54..1f21bf3cb8210 100644 --- a/clang/test/SemaSPIRV/BuiltIns/smoothstep-errors.c +++ b/clang/test/SemaSPIRV/BuiltIns/smoothstep-errors.c @@ -20,7 +20,7 @@ float2 test_too_many_arg(float2 p0) { int test_int_scalar_inputs(int p0) { return __builtin_spirv_smoothstep(p0, p0, p0); - // expected-error@-1 {{1st argument must be a scalar, vector, or matrix of floating-point types (was 'int')}} + // expected-error@-1 {{1st argument must be a scalar or vector of floating-point types (was 'int')}} } float test_int_scalar_inputs2(float p0, int p1) { From 18220e793fbda204b66c08989cde11ad4583f26c Mon Sep 17 00:00:00 2001 From: Demetrius Kanios Date: Tue, 21 Jul 2026 11:30:12 -0700 Subject: [PATCH 12/50] [SelectionDAG][GlobalISel] Move `to_tframeindex` & `renderFrameIndex` from targets into common code. (#210896) Consolidates the multiple copies across the targets of the `to_tframeindex` `SDNodeXForm` (frame index => target frame index) and its GISel counterpart/equiv `renderFrameIndex`, moving them into appropriate common areas. As suggested/requested in https://github.com/llvm/llvm-project/pull/206885#discussion_r3504014131 --- .../llvm/CodeGen/GlobalISel/InstructionSelector.h | 6 ++++++ .../llvm/Target/GlobalISel/SelectionDAGCompat.td | 3 +++ llvm/include/llvm/Target/TargetSelectionDAG.td | 4 ++++ llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp | 9 +++++++++ llvm/lib/Target/AMDGPU/AMDGPUGISel.td | 3 --- llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp | 6 ------ llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h | 3 --- llvm/lib/Target/AMDGPU/SIInstrInfo.td | 5 ----- llvm/lib/Target/AMDGPU/SIInstructions.td | 4 ++-- llvm/lib/Target/CSKY/CSKYInstrInfo.td | 5 ----- llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 4 ---- .../Target/RISCV/GISel/RISCVInstructionSelector.cpp | 10 ---------- llvm/lib/Target/RISCV/RISCVInstrInfo.td | 8 -------- llvm/lib/Target/Sparc/SparcInstrInfo.td | 3 --- .../TableGen/GlobalISelEmitter/GlobalISelEmitter.td | 2 ++ llvm/test/TableGen/GlobalISelEmitter/frameindex.td | 5 ----- 16 files changed, 26 insertions(+), 54 deletions(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h index 483afb426fa10..09cade01bdadb 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -14,6 +14,8 @@ #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTOR_H #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/Support/Compiler.h" namespace llvm { @@ -41,6 +43,10 @@ class LLVM_ABI InstructionSelector : public GIMatchTableExecutor { /// changingInstr() and changedInstr() will never be called on these /// observers. GISelObserverWrapper *AllObservers = nullptr; + +protected: + void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI, + int OpIdx) const; }; } // namespace llvm diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index c483d2310b8d2..e06c96fa837df 100644 --- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -312,3 +312,6 @@ class GIComplexPatternEquiv { class GISDNodeXFormEquiv { SDNodeXForm SelDAGEquivalent = seldag; } + +def : GICustomOperandRenderer<"renderFrameIndex">, + GISDNodeXFormEquiv; diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index 69be608107359..8bebbbcce17be 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -1024,6 +1024,10 @@ class SDNodeXForm { // The default transform does not change the matched node. def NOOP_SDNodeXForm : SDNodeXForm; +def to_tframeindex : SDNodeXFormgetTargetFrameIndex(N->getIndex(), N->getValueType(0)); +}]>; + //===----------------------------------------------------------------------===// // Selection DAG Pattern Fragments. // diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp index c48591cc2f02f..bac9ce5a7a0fe 100644 --- a/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp +++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp @@ -7,10 +7,19 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" +#include "llvm/CodeGen/TargetOpcodes.h" namespace llvm { // vtable anchor InstructionSelector::~InstructionSelector() = default; +void InstructionSelector::renderFrameIndex(MachineInstrBuilder &MIB, + const MachineInstr &MI, + int OpIdx) const { + assert(MI.getOpcode() == TargetOpcode::G_FRAME_INDEX && OpIdx == -1 && + "Expected G_FRAME_INDEX"); + MIB.add(MI.getOperand(1)); +} + } // namespace llvm diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td index a88322df79036..5df9834f4ef80 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td @@ -474,9 +474,6 @@ def gi_extract_swz : GICustomOperandRenderer<"renderExtractSWZ">, def gi_extract_cpol_set_glc : GICustomOperandRenderer<"renderExtractCpolSetGLC">, GISDNodeXFormEquiv; -def gi_frameindex_to_targetframeindex : GICustomOperandRenderer<"renderFrameIndex">, - GISDNodeXFormEquiv; - def gi_fp_pow2_to_exponent : GICustomOperandRenderer<"renderFPPow2ToExponent">, GISDNodeXFormEquiv; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp index 32b7507842842..22b8b10554928 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp @@ -7610,12 +7610,6 @@ void AMDGPUInstructionSelector::renderExtractCpolSetGLC( MIB.addImm(Cpol | AMDGPU::CPol::GLC); } -void AMDGPUInstructionSelector::renderFrameIndex(MachineInstrBuilder &MIB, - const MachineInstr &MI, - int OpIdx) const { - MIB.addFrameIndex(MI.getOperand(1).getIndex()); -} - void AMDGPUInstructionSelector::renderFPPow2ToExponent(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h index bd8ec0a769398..1f9531ce2fa13 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h @@ -417,9 +417,6 @@ class AMDGPUInstructionSelector final : public InstructionSelector { void renderExtractCpolSetGLC(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const; - void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI, - int OpIdx) const; - void renderFPPow2ToExponent(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const; diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td index 714627bee2a2c..8208a6ea2d5b5 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -881,11 +881,6 @@ return CurDAG->getTargetConstant( N->getValueAPF().bitcastToAPInt().getZExtValue(), SDLoc(N), MVT::i32); }]>; -def frameindex_to_targetframeindex : SDNodeXForm(N); - return CurDAG->getTargetFrameIndex(FI->getIndex(), MVT::i32); -}]>; - // Copied from the AArch64 backend: def bitcast_fpimm_to_i64 : SDNodeXFormgetTargetConstant( diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index d978735a92db8..f04373d3163a9 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -2580,12 +2580,12 @@ foreach vt = Reg32Types.types in { // unnecessary copy from SGPR to VGPR. def : GCNPat < (VGPRImm<(p5 frameindex)>:$fi), - (V_MOV_B32_e32 (p5 (frameindex_to_targetframeindex $fi))) + (V_MOV_B32_e32 (p5 (to_tframeindex $fi))) >; def : GCNPat < (p5 frameindex:$fi), - (S_MOV_B32 (p5 (frameindex_to_targetframeindex $fi))) + (S_MOV_B32 (p5 (to_tframeindex $fi))) >; def : GCNPat < diff --git a/llvm/lib/Target/CSKY/CSKYInstrInfo.td b/llvm/lib/Target/CSKY/CSKYInstrInfo.td index 82e271e5b5565..c5c15462eb3e1 100644 --- a/llvm/lib/Target/CSKY/CSKYInstrInfo.td +++ b/llvm/lib/Target/CSKY/CSKYInstrInfo.td @@ -72,11 +72,6 @@ class OImmAsmOperand : ImmAsmOperand<"O", width, suffix> { } -def to_tframeindex : SDNodeXForm(N); - return CurDAG->getTargetFrameIndex(FI->getIndex(), TLI->getPointerTy(CurDAG->getDataLayout())); -}]>; - def to_tconstpool : SDNodeXForm(N); return CurDAG->getTargetConstantPool(CP->getConstVal(), TLI->getPointerTy(CurDAG->getDataLayout()), diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td index dcf89f9a73956..45e2935020d71 100644 --- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td +++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td @@ -1773,10 +1773,6 @@ def to_texternsym : SDNodeXFormgetTargetFlags()); }]>; -def to_tframeindex : SDNodeXFormgetTargetFrameIndex(N->getIndex(), N->getValueType(0)); -}]>; - def : Pat<(i32 globaladdr:$dst), (MOV_B32_sym (to_tglobaladdr $dst))>; def : Pat<(i64 globaladdr:$dst), (MOV_B64_sym (to_tglobaladdr $dst))>; diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp index bf91c53270067..16af89c715861 100644 --- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp +++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp @@ -144,8 +144,6 @@ class RISCVInstructionSelector : public InstructionSelector { int OpIdx) const; void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const; - void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI, - int OpIdx) const; void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const; @@ -1554,14 +1552,6 @@ void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB, MIB.addImm(CstVal + 1); } -void RISCVInstructionSelector::renderFrameIndex(MachineInstrBuilder &MIB, - const MachineInstr &MI, - int OpIdx) const { - assert(MI.getOpcode() == TargetOpcode::G_FRAME_INDEX && OpIdx == -1 && - "Expected G_FRAME_INDEX"); - MIB.add(MI.getOperand(1)); -} - void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const { diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td index e10ee7389db0e..9b4f805880f29 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td @@ -1552,14 +1552,6 @@ def PseudoAddTPRel : Pseudo<(outs GPR:$rd), /// FrameIndex calculations -// Transforms frameindex -> tframeindex. -def to_tframeindex : SDNodeXFormgetTargetFrameIndex(N->getIndex(), N->getValueType(0)); -}]>; - -def : GICustomOperandRenderer<"renderFrameIndex">, - GISDNodeXFormEquiv; - def : Pat<(frameindex:$fi), (ADDI (iPTR (to_tframeindex $fi)), 0)>; def : Pat<(riscv_add_like frameindex:$fi, simm12_lo:$offset), diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.td b/llvm/lib/Target/Sparc/SparcInstrInfo.td index 40af4dae8783b..99cb89ea342e8 100644 --- a/llvm/lib/Target/Sparc/SparcInstrInfo.td +++ b/llvm/lib/Target/Sparc/SparcInstrInfo.td @@ -1945,9 +1945,6 @@ def : Pat<(i32 imm:$val), (ORri (SETHIi (HI22 imm:$val)), (LO10 imm:$val))>; // Frame index. -def to_tframeindex : SDNodeXFormgetTargetFrameIndex(N->getIndex(), N->getValueType(0)); -}]>; def : Pat<(i32 (frameindex:$ptr)), (ADDri (i32 (to_tframeindex $ptr)), (i32 0))>; def : Pat<(i64 (frameindex:$ptr)), (ADDri (i64 (to_tframeindex $ptr)), (i64 0))>; diff --git a/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td index d51bb87bb5896..fa02ca90b0a16 100644 --- a/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td +++ b/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td @@ -249,11 +249,13 @@ def HasC : Predicate<"Subtarget->hasC()"> { let RecomputePerFunction = 1; } // CHECK-LABEL: // Custom renderers. // CHECK-NEXT: enum { // CHECK-NEXT: GICR_Invalid, +// CHECK-NEXT: GICR_renderFrameIndex, // CHECK-NEXT: GICR_renderImm, // CHECK-NEXT: }; // CHECK-NEXT: MyTargetInstructionSelector::CustomRendererFn // CHECK-NEXT: MyTargetInstructionSelector::CustomRenderers[] = { // CHECK-NEXT: nullptr, // GICR_Invalid +// CHECK-NEXT: &MyTargetInstructionSelector::renderFrameIndex, // CHECK-NEXT: &MyTargetInstructionSelector::renderImm, // CHECK-NEXT: }; diff --git a/llvm/test/TableGen/GlobalISelEmitter/frameindex.td b/llvm/test/TableGen/GlobalISelEmitter/frameindex.td index 27784526a65ba..7e5bf0a6cdbdb 100644 --- a/llvm/test/TableGen/GlobalISelEmitter/frameindex.td +++ b/llvm/test/TableGen/GlobalISelEmitter/frameindex.td @@ -5,11 +5,6 @@ include "GlobalISelEmitterCommon.td" def ADDI : I<(outs GPR32:$dst), (ins GPR32:$src1, i32imm:$src2), []>; -def to_tframeindex : SDNodeXForm; - -def : GICustomOperandRenderer<"renderFrameIndex">, - GISDNodeXFormEquiv; - def : Pat<(frameindex:$fi), (ADDI (to_tframeindex $fi), 0)>; def : Pat<(ptradd frameindex:$fi, (i32 imm:$offset)), From 0ee0b9701bc1486d084014185f674321e714c079 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Tue, 21 Jul 2026 11:37:16 -0700 Subject: [PATCH 13/50] [mlir][openacc] Support already existing global in GPU module (#210806) --- .../ACCDeclareGPUModuleInsertion.cpp | 15 +++++++++++---- .../acc-declare-gpu-module-insertion.mlir | 19 ++++++++++++++++++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/mlir/lib/Dialect/OpenACC/Transforms/ACCDeclareGPUModuleInsertion.cpp b/mlir/lib/Dialect/OpenACC/Transforms/ACCDeclareGPUModuleInsertion.cpp index f815245882630..705b4f9df7770 100644 --- a/mlir/lib/Dialect/OpenACC/Transforms/ACCDeclareGPUModuleInsertion.cpp +++ b/mlir/lib/Dialect/OpenACC/Transforms/ACCDeclareGPUModuleInsertion.cpp @@ -99,20 +99,27 @@ class ACCDeclareGPUModuleInsertion StringAttr name = symOp.getNameAttr(); if (Operation *existing = gpuSymTable.lookup(name.getValue())) { - // Reuse only when the existing GPU symbol is structurally equivalent to - // the global we would insert. Otherwise treat as a conflict (different - // op type or different definition). + // Reuse when structurally equivalent ignoring locations and discardable + // attrs such as `acc.declare` attributes. Only a different op type or a + // true definition mismatch is a conflict. if (existing->getName() != globalOp.getName() || !OperationEquivalence::isEquivalentTo( existing, &globalOp, OperationEquivalence::ignoreValueEquivalence, /*markEquivalent=*/nullptr, - OperationEquivalence::IgnoreLocations)) { + OperationEquivalence::IgnoreLocations | + OperationEquivalence::IgnoreDiscardableAttrs)) { accSupport.emitNYI(globalOp.getLoc(), llvm::Twine("duplicate global symbol '") + name.getValue() + "' in gpu module"); return failure(); } + // Propagate acc.declare onto the GPU copy if it was cloned before the + // host global was marked. + if (!existing->getAttr(acc::getDeclareAttrName())) + if (Attribute declareAttr = + globalOp.getAttr(acc::getDeclareAttrName())) + existing->setAttr(acc::getDeclareAttrName(), declareAttr); continue; } diff --git a/mlir/test/Dialect/OpenACC/acc-declare-gpu-module-insertion.mlir b/mlir/test/Dialect/OpenACC/acc-declare-gpu-module-insertion.mlir index 8d206673ac02d..2e5910d155641 100644 --- a/mlir/test/Dialect/OpenACC/acc-declare-gpu-module-insertion.mlir +++ b/mlir/test/Dialect/OpenACC/acc-declare-gpu-module-insertion.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -acc-declare-gpu-module-insertion | FileCheck %s +// RUN: mlir-opt %s -split-input-file -acc-declare-gpu-module-insertion | FileCheck %s // Test that globals with acc.declare are copied into the GPU module. // The host global stays in the module; a copy is inserted into the GPU module. @@ -12,3 +12,20 @@ module { memref.global @arr : memref<7xf32> = dense<0.0> {acc.declare = #acc.declare} } + +// ----- + +// If the GPU module already has the global (e.g. from CUDA Fortran pass before +// ACCImplicitDeclare marked the host), reuse it and propagate acc.declare. +// CHECK-LABEL: module attributes {gpu.container_module} +// CHECK: memref.global @precloned {{.*}} {acc.declare = #acc.declare} +// CHECK: gpu.module @acc_gpu_module { +// CHECK-NEXT: memref.global @precloned {{.*}} {acc.declare = #acc.declare} +// CHECK-NEXT: } + +module attributes {gpu.container_module} { + memref.global @precloned : memref<4xf32> = dense<0.0> {acc.declare = #acc.declare} + gpu.module @acc_gpu_module { + memref.global @precloned : memref<4xf32> = dense<0.0> + } +} From 3f928aec54e28dd470a3f0fbbe2ebafb73691dcc Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Tue, 21 Jul 2026 14:49:43 -0400 Subject: [PATCH 14/50] [SLP]Make the instruction-count check loop-aware Raw getNum{Scalar,Vector}Insts() counted one-time, LICM-hoisted broadcasts/buildvectors against the loop body, rejecting profitable loop trees (508.namd_r). Weight each entry by its loop-nest trip count and drop nest-invariant ones; flat code is unchanged (scale 1). Fixes #207572 Reviewers: bababuck, RKSimon, hiraditya Pull Request: https://github.com/llvm/llvm-project/pull/210074 --- .../Transforms/Vectorize/SLPVectorizer.cpp | 100 +++++++++----- .../X86/deleted-instructions-clear.ll | 10 +- .../X86/loop-invariant-gather-inst-count.ll | 128 +++++++++--------- .../phi-removed-on-operand-vectorization.ll | 15 +- .../Transforms/SLPVectorizer/X86/rgb_phi.ll | 18 +-- 5 files changed, 150 insertions(+), 121 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 7adac62541fe0..8121e695698c8 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -3434,12 +3434,15 @@ class slpvectorizer::BoUpSLP { Instruction *I, const SmallDenseSet *VectorizedVals = nullptr) const; - /// Estimates the number of scalar instructions in the tree. - unsigned getNumScalarInsts() const; + /// Estimates the number of scalar instructions in the tree, each weighted by + /// its loop-nest trip count (nest-invariant entries are dropped when + /// \p TreeLoop is non-null). + uint64_t getNumScalarInsts(bool HasTreeLoop); /// Estimates the number of vector instructions (including buildvectors, - /// shuffles, and extracts) that the tree will produce. - unsigned getNumVectorInsts() const; + /// shuffles, and extracts) the tree produces, weighted like + /// getNumScalarInsts(). + uint64_t getNumVectorInsts(bool HasTreeLoop); /// Return information about the vector formed for the specified index /// of a vector of (the same) instruction. @@ -3489,6 +3492,10 @@ class slpvectorizer::BoUpSLP { uint64_t getGatherNodeEffectiveScale(const TreeEntry &TE, Instruction *U = nullptr); + /// \returns the loop-nest execution scale of \p TE. + uint64_t getEntryEffectiveScale(const TreeEntry &TE, + Instruction *U = nullptr); + /// Get the loop nest for the given loop \p L. ArrayRef getLoopNest(const Loop *L); @@ -13505,12 +13512,16 @@ static InstructionCost canConvertToFMA(ArrayRef VL, TargetTransformInfo &TTI, const TargetLibraryInfo &TLI); -unsigned BoUpSLP::getNumScalarInsts() const { - unsigned Count = 0; +uint64_t BoUpSLP::getNumScalarInsts(bool HasTreeLoop) { + uint64_t Total = 0; for (const std::unique_ptr &Ptr : VectorizableTree) { const TreeEntry &TE = *Ptr; if (DeletedNodes.contains(&TE)) continue; + uint64_t Scale = getEntryEffectiveScale(TE); + if (HasTreeLoop && Scale <= 1) + continue; + unsigned Count = 0; if (TE.isGather() || TransformedToGatherNodes.contains(&TE)) { // Count extractelement scalars in gathers - they exist in the scalar // code regardless of vectorization. ExtractElement instructions @@ -13518,12 +13529,13 @@ unsigned BoUpSLP::getNumScalarInsts() const { for (Value *V : TE.Scalars) if (isa(V)) ++Count; + Total = SaturatingMultiplyAdd(Count, Scale, Total); continue; } // CombinedVectorize entries (e.g. the fmul child of an FMulAdd, or the // cmp child of a MinMax select) are absorbed into the parent on both - // scalar and vector sides. The backend fuses fadd+fmul → fma and - // select+cmp → smin/smax even for scalar code, so skip to avoid + // scalar and vector sides. The backend fuses fadd+fmul -> fma and + // select+cmp -> smin/smax even for scalar code, so skip to avoid // double-counting. if (TE.State == TreeEntry::CombinedVectorize) continue; @@ -13548,7 +13560,7 @@ unsigned BoUpSLP::getNumScalarInsts() const { } // Even when the whole node is not combined, individual scalar // instructions may be fused by the backend. Each fused pair (e.g. - // fadd+fmul → fma, select+cmp → smin/smax) becomes a single scalar + // fadd+fmul -> fma, select+cmp -> smin/smax) becomes a single scalar // instruction, absorbing the operand instruction. Subtract 1 for each // such match to avoid over-counting the scalar side. if (TE.CombinedOp == TreeEntry::NotCombinedOp && TE.hasState()) { @@ -13582,22 +13594,27 @@ unsigned BoUpSLP::getNumScalarInsts() const { } } } + Total = SaturatingMultiplyAdd(Count, Scale, Total); } - return Count; + return Total; } -unsigned BoUpSLP::getNumVectorInsts() const { - unsigned Count = 0; - SmallPtrSet GatherExtractSourceVecs; +uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop) { + uint64_t Total = 0; + // Source vector -> max scale among the gather entries sharing it, so the + // combined shufflevector is still weighted like an in-loop entry below. + SmallDenseMap GatherExtractSourceVecs; for (const std::unique_ptr &Ptr : VectorizableTree) { const TreeEntry &TE = *Ptr; if (DeletedNodes.contains(&TE)) continue; if (TE.State == TreeEntry::CombinedVectorize) continue; - bool IsGatherOrTransformed = - TE.isGather() || TransformedToGatherNodes.contains(&TE); - if (IsGatherOrTransformed) { + uint64_t Scale = getEntryEffectiveScale(TE); + if (HasTreeLoop && Scale <= 1) + continue; + unsigned Count = 0; + if (TE.isGather() || TransformedToGatherNodes.contains(&TE)) { if (TE.hasState()) { if (const TreeEntry *E = getSameValuesTreeEntry(TE.getMainOp(), TE.Scalars); @@ -13607,7 +13624,7 @@ unsigned BoUpSLP::getNumVectorInsts() const { if (const TreeEntry *E = getSameValuesTreeEntry(TE.getMainOp(), RevScalars); E && E->getVectorFactor() == TE.getVectorFactor()) { - ++Count; + Total = SaturatingAdd(Total, Scale); continue; } } @@ -13617,14 +13634,19 @@ unsigned BoUpSLP::getNumVectorInsts() const { if (all_of(TE.Scalars, IsaPred)) { for (Value *V : TE.Scalars) - if (auto *EE = dyn_cast(V)) - GatherExtractSourceVecs.insert(EE->getVectorOperand()); + if (auto *EE = dyn_cast(V)) { + uint64_t &VecScale = + GatherExtractSourceVecs.try_emplace(EE->getVectorOperand(), 0) + .first->second; + VecScale = std::max(VecScale, Scale); + } } else { for (Value *V : TE.Scalars) { if (!isConstant(V)) ++Count; } } + Total = SaturatingMultiplyAdd(Count, Scale, Total); continue; } // InsertElement/ExtractElement vectorize entries don't produce real @@ -13640,6 +13662,7 @@ unsigned BoUpSLP::getNumVectorInsts() const { Count += 2; if (!TE.ReorderIndices.empty() || !TE.ReuseShuffleIndices.empty()) ++Count; + Total = SaturatingMultiplyAdd(Count, Scale, Total); continue; } if (TE.State == TreeEntry::SplitVectorize) @@ -13648,8 +13671,10 @@ unsigned BoUpSLP::getNumVectorInsts() const { ++Count; if (!TE.ReorderIndices.empty() || !TE.ReuseShuffleIndices.empty()) ++Count; + Total = SaturatingMultiplyAdd(Count, Scale, Total); } - Count += GatherExtractSourceVecs.size(); + for (const auto &VecAndScale : GatherExtractSourceVecs) + Total = SaturatingAdd(Total, VecAndScale.second); // Count extract instructions from ExternalUses, skipping insertelements // (those get folded into shuffles, not real extracts). SmallPtrSet CountedExtracts; @@ -13662,9 +13687,9 @@ unsigned BoUpSLP::getNumVectorInsts() const { continue; if (!CountedExtracts.insert(EU.Scalar).second) continue; - ++Count; + ++Total; } - return Count; + return Total; } void BoUpSLP::TreeEntry::buildAltOpShuffleMask( @@ -16441,6 +16466,12 @@ uint64_t BoUpSLP::getGatherNodeEffectiveScale(const TreeEntry &TE, return std::clamp(Avg, 1, BaseScale); } +uint64_t BoUpSLP::getEntryEffectiveScale(const TreeEntry &TE, Instruction *U) { + if (TE.isGather() || TE.State == TreeEntry::SplitVectorize) + return getGatherNodeEffectiveScale(TE, U); + return getScaleToLoopIterations(TE); +} + InstructionCost BoUpSLP::getVectorSpillReloadCost(const TreeEntry *E, Type *ScalarTy, Type *VecTy, Type *FinalVecTy, @@ -18977,8 +19008,6 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef VectorizedVals, // per-lane refined scale that accounts for LICM-hoistable insertelements // when an operand is invariant in the current loop nest but defined in // an outer loop. This prevents over-costing cross-loop-nest buildvectors. - const bool IsGatherLike = - TE.isGather() || TE.State == TreeEntry::SplitVectorize; if (!CostIsFree && !TE.isGather() && TE.hasState()) { if (PrevVecParent == TE.getMainOp()->getParent()) { Scale = PrevScale; @@ -18987,10 +19016,7 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef VectorizedVals, } } if (!CostIsFree && !Scale) { - Scale = - IsGatherLike - ? getGatherNodeEffectiveScale(TE, TE.Idx == 0 ? RdxRoot : nullptr) - : getScaleToLoopIterations(TE); + Scale = getEntryEffectiveScale(TE, TE.Idx == 0 ? RdxRoot : nullptr); C *= Scale; EntryToScale.try_emplace(&TE, Scale); if (!TE.isGather() && TE.hasState()) { @@ -19369,12 +19395,8 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef VectorizedVals, continue; } uint64_t Scale = EntryToScale.lookup(TE.get()); - if (!Scale) { - const bool IsGatherLike = - TE->isGather() || TE->State == TreeEntry::SplitVectorize; - Scale = IsGatherLike ? getGatherNodeEffectiveScale(*TE.get()) - : getScaleToLoopIterations(*TE.get()); - } + if (!Scale) + Scale = getEntryEffectiveScale(*TE); C *= Scale; NodesCosts.try_emplace(TE.get(), C); } @@ -19458,8 +19480,14 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost, (!SLPReVec || !isa( VectorizableTree.front()->Scalars.front()->getType()))) { - unsigned NumScalar = getNumScalarInsts(); - unsigned NumVector = getNumVectorInsts(); + // Loop containing the tree root; null for flat code or disabled + // loop-aware modeling. Shared by both calls below. + const Loop *TreeLoop = nullptr; + if (LoopAwareTripCount != 0 && VectorizableTree.front()->hasState()) + TreeLoop = + LI->getLoopFor(VectorizableTree.front()->getMainOp()->getParent()); + uint64_t NumScalar = getNumScalarInsts(TreeLoop); + uint64_t NumVector = getNumVectorInsts(TreeLoop); LLVM_DEBUG(dbgs() << "SLP: Inst count check: vector=" << NumVector << " scalar=" << NumScalar << "\n"); if (NumVector > NumScalar && !BypassesInstCountCheck()) { diff --git a/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll b/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll index ae6da0913cc24..94631869b15fe 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll @@ -7,7 +7,7 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) { ; CHECK-NEXT: [[BB:.*]]: ; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> , i32 [[ARG]], i64 3 ; CHECK-NEXT: [[TMP27:%.*]] = insertelement <2 x i32> poison, i32 [[ARG1]], i64 0 -; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x i32> [[TMP27]], <2 x i32> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> , i32 [[ARG]], i64 1 ; CHECK-NEXT: br label %[[BB3:.*]] ; CHECK: [[BB3]]: ; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[BB3]] ], [ 0, %[[BB]] ] @@ -28,9 +28,9 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) { ; CHECK-NEXT: [[OR11:%.*]] = or i32 [[TRUNC27]], 0 ; CHECK-NEXT: [[TMP8:%.*]] = or <4 x i32> zeroinitializer, [[TMP2]] ; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i32> [[TMP5]], [[TMP8]] -; CHECK-NEXT: [[XOR38:%.*]] = xor i32 [[ARG]], [[TRUNC28]] -; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> , i32 [[TRUNC19]], i64 0 +; CHECK-NEXT: [[TMP28:%.*]] = insertelement <2 x i32> [[TMP27]], i32 [[TRUNC28]], i64 1 ; CHECK-NEXT: [[TMP14:%.*]] = xor <2 x i32> [[TMP28]], [[TMP29]] +; CHECK-NEXT: [[XOR31:%.*]] = xor i32 [[ARG1]], [[TRUNC19]] ; CHECK-NEXT: [[SHL:%.*]] = shl i32 0, 1 ; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> poison, i32 [[SHL]], i64 0 ; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TRUNC10]], i64 1 @@ -38,9 +38,9 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) { ; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[TMP26:%.*]] = xor <4 x i32> [[TMP11]], [[TMP25]] ; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <4 x i32> [[TMP26]], <4 x i32> poison, <8 x i32> +; CHECK-NEXT: [[TMP32:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[XOR31]], i64 5 ; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <2 x i32> [[TMP14]], <2 x i32> poison, <8 x i32> -; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP24]], <8 x i32> [[TMP30]], <8 x i32> -; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x i32> [[TMP15]], i32 [[XOR38]], i64 7 +; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <8 x i32> [[TMP32]], <8 x i32> [[TMP30]], <8 x i32> ; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> poison, <8 x i32> ; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP17]], <8 x i32> poison, <12 x i32> ; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> poison, <12 x i32> diff --git a/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll b/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll index 923af5efa5d2e..9c7849d0de123 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll @@ -25,70 +25,70 @@ define void @test(ptr noalias readonly %0, ptr noalias readonly %1, i32 %2, doub ; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP19]] ; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP20]], i64 16 ; CHECK-NEXT: [[TMP22:%.*]] = load double, ptr [[TMP21]], align 8 -; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP15]], i64 8 -; CHECK-NEXT: [[TMP24:%.*]] = load double, ptr [[TMP23]], align 8 -; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP20]], i64 8 -; CHECK-NEXT: [[TMP26:%.*]] = load double, ptr [[TMP25]], align 8 -; CHECK-NEXT: [[TMP27:%.*]] = load double, ptr [[TMP15]], align 8 -; CHECK-NEXT: [[TMP28:%.*]] = load double, ptr [[TMP20]], align 8 -; CHECK-NEXT: [[TMP29:%.*]] = zext nneg i32 [[TMP11]] to i64 -; CHECK-NEXT: br label %[[BB30:.*]] -; CHECK: [[BB30]]: -; CHECK-NEXT: [[TMP31:%.*]] = phi i64 [ 2, %[[BB10]] ], [ [[TMP81:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP32:%.*]] = phi i32 [ 0, %[[BB10]] ], [ [[TMP80:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP33:%.*]] = phi i32 [ [[TMP18]], %[[BB10]] ], [ [[TMP54:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP34:%.*]] = phi i32 [ [[TMP13]], %[[BB10]] ], [ [[TMP56:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP35:%.*]] = phi double [ [[TMP28]], %[[BB10]] ], [ [[TMP59:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP36:%.*]] = phi double [ [[TMP27]], %[[BB10]] ], [ [[TMP62:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP37:%.*]] = phi double [ [[TMP17]], %[[BB10]] ], [ [[TMP70:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP38:%.*]] = phi double [ [[TMP22]], %[[BB10]] ], [ [[TMP68:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP39:%.*]] = phi double [ [[TMP24]], %[[BB10]] ], [ [[TMP66:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP40:%.*]] = phi double [ [[TMP26]], %[[BB10]] ], [ [[TMP64:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP41:%.*]] = fsub double [[TMP3]], [[TMP35]] -; CHECK-NEXT: [[TMP42:%.*]] = fsub double [[TMP3]], [[TMP36]] -; CHECK-NEXT: [[TMP43:%.*]] = fmul double [[TMP41]], [[TMP41]] -; CHECK-NEXT: [[TMP44:%.*]] = fmul double [[TMP42]], [[TMP42]] -; CHECK-NEXT: [[TMP45:%.*]] = fsub double [[TMP4]], [[TMP40]] -; CHECK-NEXT: [[TMP46:%.*]] = fsub double [[TMP4]], [[TMP39]] -; CHECK-NEXT: [[TMP47:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP45]], double [[TMP45]], double [[TMP43]]) -; CHECK-NEXT: [[TMP48:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP46]], double [[TMP46]], double [[TMP44]]) -; CHECK-NEXT: [[TMP49:%.*]] = fsub double [[TMP5]], [[TMP38]] -; CHECK-NEXT: [[TMP50:%.*]] = fsub double [[TMP5]], [[TMP37]] -; CHECK-NEXT: [[TMP51:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP49]], double [[TMP49]], double [[TMP47]]) -; CHECK-NEXT: [[TMP52:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP50]], double [[TMP50]], double [[TMP48]]) -; CHECK-NEXT: [[TMP53:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP1]], i64 [[TMP31]] -; CHECK-NEXT: [[TMP54]] = load i32, ptr [[TMP53]], align 4 -; CHECK-NEXT: [[TMP55:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP53]], i64 4 -; CHECK-NEXT: [[TMP56]] = load i32, ptr [[TMP55]], align 4 -; CHECK-NEXT: [[TMP57:%.*]] = sext i32 [[TMP54]] to i64 -; CHECK-NEXT: [[TMP58:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP57]] -; CHECK-NEXT: [[TMP59]] = load double, ptr [[TMP58]], align 8 -; CHECK-NEXT: [[TMP60:%.*]] = sext i32 [[TMP56]] to i64 -; CHECK-NEXT: [[TMP61:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP60]] -; CHECK-NEXT: [[TMP62]] = load double, ptr [[TMP61]], align 8 -; CHECK-NEXT: [[TMP63:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP58]], i64 8 -; CHECK-NEXT: [[TMP64]] = load double, ptr [[TMP63]], align 8 -; CHECK-NEXT: [[TMP65:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP61]], i64 8 -; CHECK-NEXT: [[TMP66]] = load double, ptr [[TMP65]], align 8 -; CHECK-NEXT: [[TMP67:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP58]], i64 16 -; CHECK-NEXT: [[TMP68]] = load double, ptr [[TMP67]], align 8 -; CHECK-NEXT: [[TMP69:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP61]], i64 16 -; CHECK-NEXT: [[TMP70]] = load double, ptr [[TMP69]], align 8 -; CHECK-NEXT: [[TMP71:%.*]] = fcmp olt double [[TMP51]], [[TMP6]] -; CHECK-NEXT: [[TMP72:%.*]] = fcmp olt double [[TMP52]], [[TMP6]] -; CHECK-NEXT: [[TMP73:%.*]] = zext nneg i32 [[TMP32]] to i64 -; CHECK-NEXT: [[TMP74:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP73]] -; CHECK-NEXT: store i32 [[TMP33]], ptr [[TMP74]], align 4 -; CHECK-NEXT: [[TMP75:%.*]] = zext i1 [[TMP71]] to i32 -; CHECK-NEXT: [[TMP76:%.*]] = add nuw nsw i32 [[TMP32]], [[TMP75]] -; CHECK-NEXT: [[TMP77:%.*]] = zext nneg i32 [[TMP76]] to i64 -; CHECK-NEXT: [[TMP78:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP77]] -; CHECK-NEXT: store i32 [[TMP34]], ptr [[TMP78]], align 4 -; CHECK-NEXT: [[TMP79:%.*]] = zext i1 [[TMP72]] to i32 -; CHECK-NEXT: [[TMP80]] = add nuw nsw i32 [[TMP76]], [[TMP79]] -; CHECK-NEXT: [[TMP81]] = add nuw nsw i64 [[TMP31]], 2 -; CHECK-NEXT: [[TMP82:%.*]] = icmp samesign ult i64 [[TMP81]], [[TMP29]] -; CHECK-NEXT: br i1 [[TMP82]], label %[[BB30]], label %[[BB83]] +; CHECK-NEXT: [[TMP23:%.*]] = load <2 x double>, ptr [[TMP15]], align 8 +; CHECK-NEXT: [[TMP24:%.*]] = load <2 x double>, ptr [[TMP20]], align 8 +; CHECK-NEXT: [[TMP25:%.*]] = zext nneg i32 [[TMP11]] to i64 +; CHECK-NEXT: [[TMP26:%.*]] = insertelement <2 x double> poison, double [[TMP17]], i64 0 +; CHECK-NEXT: [[TMP27:%.*]] = insertelement <2 x double> [[TMP26]], double [[TMP22]], i64 1 +; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x double> [[TMP23]], <2 x double> [[TMP24]], <2 x i32> +; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <2 x double> [[TMP23]], <2 x double> [[TMP24]], <2 x i32> +; CHECK-NEXT: [[TMP30:%.*]] = insertelement <2 x double> poison, double [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <2 x double> [[TMP30]], <2 x double> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP32:%.*]] = insertelement <2 x double> poison, double [[TMP4]], i64 0 +; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <2 x double> [[TMP32]], <2 x double> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP34:%.*]] = insertelement <2 x double> poison, double [[TMP3]], i64 0 +; CHECK-NEXT: [[TMP35:%.*]] = shufflevector <2 x double> [[TMP34]], <2 x double> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP36:%.*]] = insertelement <2 x double> poison, double [[TMP6]], i64 0 +; CHECK-NEXT: [[TMP37:%.*]] = shufflevector <2 x double> [[TMP36]], <2 x double> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: br label %[[BB38:.*]] +; CHECK: [[BB38]]: +; CHECK-NEXT: [[TMP39:%.*]] = phi i64 [ 2, %[[BB10]] ], [ [[TMP77:%.*]], %[[BB38]] ] +; CHECK-NEXT: [[TMP40:%.*]] = phi i32 [ 0, %[[BB10]] ], [ [[TMP76:%.*]], %[[BB38]] ] +; CHECK-NEXT: [[TMP41:%.*]] = phi i32 [ [[TMP18]], %[[BB10]] ], [ [[TMP50:%.*]], %[[BB38]] ] +; CHECK-NEXT: [[TMP42:%.*]] = phi i32 [ [[TMP13]], %[[BB10]] ], [ [[TMP52:%.*]], %[[BB38]] ] +; CHECK-NEXT: [[TMP43:%.*]] = phi <2 x double> [ [[TMP27]], %[[BB10]] ], [ [[TMP80:%.*]], %[[BB38]] ] +; CHECK-NEXT: [[TMP44:%.*]] = phi <2 x double> [ [[TMP28]], %[[BB10]] ], [ [[TMP81:%.*]], %[[BB38]] ] +; CHECK-NEXT: [[TMP45:%.*]] = phi <2 x double> [ [[TMP29]], %[[BB10]] ], [ [[TMP82:%.*]], %[[BB38]] ] +; CHECK-NEXT: [[TMP46:%.*]] = fsub <2 x double> [[TMP35]], [[TMP45]] +; CHECK-NEXT: [[TMP47:%.*]] = fsub <2 x double> [[TMP33]], [[TMP44]] +; CHECK-NEXT: [[TMP48:%.*]] = fsub <2 x double> [[TMP31]], [[TMP43]] +; CHECK-NEXT: [[TMP49:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP1]], i64 [[TMP39]] +; CHECK-NEXT: [[TMP50]] = load i32, ptr [[TMP49]], align 4 +; CHECK-NEXT: [[TMP51:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP49]], i64 4 +; CHECK-NEXT: [[TMP52]] = load i32, ptr [[TMP51]], align 4 +; CHECK-NEXT: [[TMP53:%.*]] = sext i32 [[TMP50]] to i64 +; CHECK-NEXT: [[TMP54:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP53]] +; CHECK-NEXT: [[TMP55:%.*]] = sext i32 [[TMP52]] to i64 +; CHECK-NEXT: [[TMP56:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP55]] +; CHECK-NEXT: [[TMP57:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP54]], i64 16 +; CHECK-NEXT: [[TMP58:%.*]] = load double, ptr [[TMP57]], align 8 +; CHECK-NEXT: [[TMP59:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP56]], i64 16 +; CHECK-NEXT: [[TMP60:%.*]] = load double, ptr [[TMP59]], align 8 +; CHECK-NEXT: [[TMP61:%.*]] = fmul <2 x double> [[TMP46]], [[TMP46]] +; CHECK-NEXT: [[TMP62:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP47]], <2 x double> [[TMP47]], <2 x double> [[TMP61]]) +; CHECK-NEXT: [[TMP63:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP48]], <2 x double> [[TMP48]], <2 x double> [[TMP62]]) +; CHECK-NEXT: [[TMP64:%.*]] = load <2 x double>, ptr [[TMP54]], align 8 +; CHECK-NEXT: [[TMP65:%.*]] = load <2 x double>, ptr [[TMP56]], align 8 +; CHECK-NEXT: [[TMP66:%.*]] = fcmp olt <2 x double> [[TMP63]], [[TMP37]] +; CHECK-NEXT: [[TMP67:%.*]] = zext nneg i32 [[TMP40]] to i64 +; CHECK-NEXT: [[TMP68:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP67]] +; CHECK-NEXT: store i32 [[TMP41]], ptr [[TMP68]], align 4 +; CHECK-NEXT: [[TMP69:%.*]] = extractelement <2 x i1> [[TMP66]], i64 1 +; CHECK-NEXT: [[TMP70:%.*]] = zext i1 [[TMP69]] to i32 +; CHECK-NEXT: [[TMP71:%.*]] = add nuw nsw i32 [[TMP40]], [[TMP70]] +; CHECK-NEXT: [[TMP72:%.*]] = zext nneg i32 [[TMP71]] to i64 +; CHECK-NEXT: [[TMP73:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP72]] +; CHECK-NEXT: store i32 [[TMP42]], ptr [[TMP73]], align 4 +; CHECK-NEXT: [[TMP74:%.*]] = extractelement <2 x i1> [[TMP66]], i64 0 +; CHECK-NEXT: [[TMP75:%.*]] = zext i1 [[TMP74]] to i32 +; CHECK-NEXT: [[TMP76]] = add nuw nsw i32 [[TMP71]], [[TMP75]] +; CHECK-NEXT: [[TMP77]] = add nuw nsw i64 [[TMP39]], 2 +; CHECK-NEXT: [[TMP78:%.*]] = icmp samesign ult i64 [[TMP77]], [[TMP25]] +; CHECK-NEXT: [[TMP79:%.*]] = insertelement <2 x double> poison, double [[TMP60]], i64 0 +; CHECK-NEXT: [[TMP80]] = insertelement <2 x double> [[TMP79]], double [[TMP58]], i64 1 +; CHECK-NEXT: [[TMP81]] = shufflevector <2 x double> [[TMP65]], <2 x double> [[TMP64]], <2 x i32> +; CHECK-NEXT: [[TMP82]] = shufflevector <2 x double> [[TMP65]], <2 x double> [[TMP64]], <2 x i32> +; CHECK-NEXT: br i1 [[TMP78]], label %[[BB38]], label %[[BB83]] ; CHECK: [[BB83]]: ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll index b47242eceafc2..aefa17e9c6a6e 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll @@ -5,17 +5,18 @@ define i32 @test(double %mul321.i) { ; CHECK-LABEL: define i32 @test( ; CHECK-SAME: double [[MUL321_I:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[MUL321_I]], i64 0 ; CHECK-NEXT: br label %[[DO_BODY220_I:.*]] ; CHECK: [[DO_BODY220_I]]: -; CHECK-NEXT: [[C1_2_I:%.*]] = phi double [ [[MUL321_I]], %[[DO_BODY221_I:.*]] ], [ 0.000000e+00, %[[ENTRY]] ] -; CHECK-NEXT: [[S1_1_I:%.*]] = phi double [ [[ADD318_I:%.*]], %[[DO_BODY221_I]] ], [ 0.000000e+00, %[[ENTRY]] ] +; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x double> [ [[TMP6:%.*]], %[[DO_BODY221_I:.*]] ], [ zeroinitializer, %[[ENTRY]] ] ; CHECK-NEXT: br label %[[DO_BODY221_I]] ; CHECK: [[DO_BODY221_I]]: -; CHECK-NEXT: [[SUB311_I1:%.*]] = fadd double [[C1_2_I]], 0.000000e+00 -; CHECK-NEXT: [[ADD315_I:%.*]] = fadd double [[S1_1_I]], 0.000000e+00 -; CHECK-NEXT: [[TMP4:%.*]] = fmul double [[SUB311_I1]], 0.000000e+00 -; CHECK-NEXT: [[TMP5:%.*]] = fmul double [[ADD315_I]], 0.000000e+00 -; CHECK-NEXT: [[ADD318_I]] = fadd double [[TMP4]], [[TMP5]] +; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x double> [[TMP1]], zeroinitializer +; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP2]], zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i64 0 +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i64 1 +; CHECK-NEXT: [[ADD318_I:%.*]] = fadd double [[TMP4]], [[TMP5]] +; CHECK-NEXT: [[TMP6]] = insertelement <2 x double> [[TMP0]], double [[ADD318_I]], i64 1 ; CHECK-NEXT: br label %[[DO_BODY220_I]] ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll b/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll index eb649f700bda6..8d3a93bb8d49c 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll @@ -23,25 +23,23 @@ target triple = "i386-apple-macosx10.9.0" define float @foo(ptr nocapture readonly %A) { ; CHECK-LABEL: @foo( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[A:%.*]], align 4 -; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1 -; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX1]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A:%.*]], align 4 ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2 ; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP0]], i64 0 ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[TMP4:%.*]] = phi float [ [[TMP3]], [[ENTRY:%.*]] ], [ [[DOTPRE:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE:%.*]] ] ; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ] ; CHECK-NEXT: [[B_032:%.*]] = phi float [ [[TMP2]], [[ENTRY]] ], [ [[ADD14:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ] -; CHECK-NEXT: [[G_031:%.*]] = phi float [ [[TMP1]], [[ENTRY]] ], [ [[TMP16:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ] -; CHECK-NEXT: [[R_030:%.*]] = phi float [ [[TMP3]], [[ENTRY]] ], [ [[TMP15:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[TMP4]], 7.000000e+00 -; CHECK-NEXT: [[TMP15]] = fadd float [[R_030]], [[MUL]] +; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x float> [ [[TMP0]], [[ENTRY]] ], [ [[TMP10:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ] ; CHECK-NEXT: [[TMP6:%.*]] = add nsw i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP6]] ; CHECK-NEXT: [[TMP7:%.*]] = load float, ptr [[ARRAYIDX7]], align 4 -; CHECK-NEXT: [[MUL8:%.*]] = fmul float [[TMP7]], 8.000000e+00 -; CHECK-NEXT: [[TMP16]] = fadd float [[G_031]], [[MUL8]] +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x float> poison, float [[TMP4]], i64 0 +; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x float> [[TMP11]], float [[TMP7]], i64 1 +; CHECK-NEXT: [[TMP9:%.*]] = fmul <2 x float> [[TMP8]], +; CHECK-NEXT: [[TMP10]] = fadd <2 x float> [[TMP5]], [[TMP9]] ; CHECK-NEXT: [[TMP12:%.*]] = add nsw i64 [[INDVARS_IV]], 2 ; CHECK-NEXT: [[ARRAYIDX12:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP12]] ; CHECK-NEXT: [[TMP13:%.*]] = load float, ptr [[ARRAYIDX12]], align 4 @@ -56,6 +54,8 @@ define float @foo(ptr nocapture readonly %A) { ; CHECK-NEXT: [[DOTPRE]] = load float, ptr [[ARRAYIDX3_PHI_TRANS_INSERT]], align 4 ; CHECK-NEXT: br label [[FOR_BODY]] ; CHECK: for.end: +; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x float> [[TMP10]], i64 0 +; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x float> [[TMP10]], i64 1 ; CHECK-NEXT: [[ADD16:%.*]] = fadd float [[TMP15]], [[TMP16]] ; CHECK-NEXT: [[ADD17:%.*]] = fadd float [[ADD16]], [[ADD14]] ; CHECK-NEXT: ret float [[ADD17]] From 1f1fcb9518cbe324d2af2aca51e32c93b324690f Mon Sep 17 00:00:00 2001 From: Alexis Engelke Date: Tue, 21 Jul 2026 20:55:03 +0200 Subject: [PATCH 15/50] [Analysis][NFC] Remove LoopBlock in BranchProbabilityInfo (#211083) After removing SccInfo, this data structure is largely pointless. --- llvm/lib/Analysis/BranchProbabilityInfo.cpp | 137 ++++++++------------ 1 file changed, 53 insertions(+), 84 deletions(-) diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp index c409c68909e71..ce3d58611d643 100644 --- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp +++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp @@ -170,28 +170,8 @@ class BPIConstruction { PostDominatorTree *PDT); private: - /// Helper class to keep basic block along with its loop data information. - class LoopBlock { - public: - explicit LoopBlock(const BasicBlock *BB, const CycleInfo &CI) - : BB(BB), C(CI.getCycle(BB)) {} - - const BasicBlock *getBlock() const { return BB; } - BasicBlock *getBlock() { return const_cast(BB); } - CycleRef getCycle() const { return C; } - - private: - const BasicBlock *const BB; - CycleRef C; - }; - - // Pair of LoopBlocks representing an edge from first to second block. - using LoopEdge = std::pair; - - /// Helper to construct LoopBlock for \p BB. - LoopBlock getLoopBlock(const BasicBlock *BB) const { - return LoopBlock(BB, *CI); - } + // Pair representing an edge from first to second block. + using LoopEdge = std::pair; /// Returns true if destination block belongs to some loop and source block is /// either doesn't belong to any loop or belongs to a loop which is not inner @@ -205,8 +185,8 @@ class BPIConstruction { /// in all other cases. bool isLoopEnteringExitingEdge(const LoopEdge &Edge) const; // Fills in \p Enters vector with all "enter" blocks to a loop \LB belongs to. - void getLoopEnterBlocks(const LoopBlock &LB, - SmallVectorImpl &Enters) const; + void getLoopEnterBlocks(const BasicBlock *LB, + SmallVectorImpl &Enters) const; /// Returns estimated weight for \p BB. std::nullopt if \p BB has no estimated /// weight. @@ -226,23 +206,24 @@ class BPIConstruction { /// estimated weight std::nullopt is returned. template std::optional - getMaxEstimatedEdgeWeight(const LoopBlock &SrcBB, + getMaxEstimatedEdgeWeight(const BasicBlock *SrcBB, iterator_range Successors) const; /// If \p LoopBB has no estimated weight then set it to \p BBWeight and /// return true. Otherwise \p BB's weight remains unchanged and false is /// returned. In addition all blocks/loops that might need their weight to be /// re-estimated are put into BlockWorkList/LoopWorkList. - bool updateEstimatedBlockWeight(LoopBlock &LoopBB, uint32_t BBWeight, - SmallVectorImpl &BlockWorkList, - SmallVectorImpl &LoopWorkList); + bool + updateEstimatedBlockWeight(const BasicBlock *BB, uint32_t BBWeight, + SmallVectorImpl &BlockWorkList, + SmallVectorImpl &LoopWorkList); /// Starting from \p LoopBB (including \p LoopBB itself) propagate \p BBWeight /// up the domination tree. - void propagateEstimatedBlockWeight(const LoopBlock &LoopBB, DominatorTree *DT, - PostDominatorTree *PDT, uint32_t BBWeight, - SmallVectorImpl &WorkList, - SmallVectorImpl &LoopWorkList); + void propagateEstimatedBlockWeight( + const BasicBlock *BB, DominatorTree *DT, PostDominatorTree *PDT, + uint32_t BBWeight, SmallVectorImpl &WorkList, + SmallVectorImpl &LoopWorkList); /// Returns block's weight encoded in the IR. std::optional getInitialEstimatedBlockWeight(const BasicBlock *BB); @@ -271,13 +252,13 @@ class BPIConstruction { }; bool BPIConstruction::isLoopEnteringEdge(const LoopEdge &Edge) const { - const auto &SrcBlock = Edge.first; - const auto &DstBlock = Edge.second; - if (!DstBlock.getCycle()) // Edge into no-cycle is not entering. + CycleRef SrcCycle = CI->getCycle(Edge.first); + CycleRef DstCycle = CI->getCycle(Edge.second); + if (!DstCycle) // Edge into no-cycle is not entering. return false; - if (!SrcBlock.getCycle()) // Edge from no-cycle into cycle is entering. + if (!SrcCycle) // Edge from no-cycle into cycle is entering. return true; - return !CI->contains(DstBlock.getCycle(), SrcBlock.getCycle()); + return !CI->contains(DstCycle, SrcCycle); } bool BPIConstruction::isLoopExitingEdge(const LoopEdge &Edge) const { @@ -289,12 +270,12 @@ bool BPIConstruction::isLoopEnteringExitingEdge(const LoopEdge &Edge) const { } void BPIConstruction::getLoopEnterBlocks( - const LoopBlock &LB, SmallVectorImpl &Enters) const { - CycleRef C = LB.getCycle(); + const BasicBlock *BB, SmallVectorImpl &Enters) const { + CycleRef C = CI->getCycle(BB); for (BasicBlock *Entry : CI->getEntries(C)) for (const auto *Pred : predecessors(Entry)) if (!CI->contains(C, Pred)) - Enters.push_back(const_cast(Pred)); + Enters.push_back(Pred); } // Propagate existing explicit probabilities from either profile data or @@ -327,9 +308,7 @@ bool BPIConstruction::calcMetadataWeights(const BasicBlock *BB) { auto Succs = succ_begin(TI); for (unsigned I = 0, E = Weights.size(); I != E; ++I) { WeightSum += Weights[I]; - const LoopBlock SrcLoopBB = getLoopBlock(BB); - const LoopBlock DstLoopBB = getLoopBlock(*Succs++); - auto EstimatedWeight = getEstimatedEdgeWeight({SrcLoopBB, DstLoopBB}); + auto EstimatedWeight = getEstimatedEdgeWeight({BB, *Succs++}); if (EstimatedWeight && *EstimatedWeight <= static_cast(BlockExecWeight::UNREACHABLE)) UnreachableIdxs.push_back(I); @@ -593,21 +572,18 @@ BPIConstruction::getEstimatedEdgeWeight(const LoopEdge &Edge) const { // For edges entering a loop take weight of a loop rather than an individual // block in the loop. return isLoopEnteringEdge(Edge) - ? getEstimatedLoopWeight(Edge.second.getCycle()) - : getEstimatedBlockWeight(Edge.second.getBlock()); + ? getEstimatedLoopWeight(CI->getCycle(Edge.second)) + : getEstimatedBlockWeight(Edge.second); } template std::optional BPIConstruction::getMaxEstimatedEdgeWeight( - const LoopBlock &SrcLoopBB, iterator_range Successors) const { + const BasicBlock *SrcBB, iterator_range Successors) const { std::optional MaxWeight; for (const BasicBlock *DstBB : Successors) { - const LoopBlock DstLoopBB = getLoopBlock(DstBB); - auto Weight = getEstimatedEdgeWeight({SrcLoopBB, DstLoopBB}); - + auto Weight = getEstimatedEdgeWeight({SrcBB, DstBB}); if (!Weight) return std::nullopt; - if (!MaxWeight || *MaxWeight < *Weight) MaxWeight = Weight; } @@ -621,11 +597,9 @@ std::optional BPIConstruction::getMaxEstimatedEdgeWeight( // Please note by the algorithm the weight is not expected to change once set // thus 'false' status is used to track visited blocks. bool BPIConstruction::updateEstimatedBlockWeight( - LoopBlock &LoopBB, uint32_t BBWeight, - SmallVectorImpl &BlockWorkList, - SmallVectorImpl &LoopWorkList) { - BasicBlock *BB = LoopBB.getBlock(); - + const BasicBlock *BB, uint32_t BBWeight, + SmallVectorImpl &BlockWorkList, + SmallVectorImpl &LoopWorkList) { // In general, weight is assigned to a block when it has final value and // can't/shouldn't be changed. However, there are cases when a block // inherently has several (possibly "contradicting") weights. For example, @@ -634,12 +608,11 @@ bool BPIConstruction::updateEstimatedBlockWeight( if (!EstimatedBlockWeight.insert({BB, BBWeight}).second) return false; - for (BasicBlock *PredBlock : predecessors(BB)) { - LoopBlock PredLoop = getLoopBlock(PredBlock); + for (const BasicBlock *PredBlock : predecessors(BB)) { // Add affected block/loop to a working list. - if (isLoopExitingEdge({PredLoop, LoopBB})) { - if (!EstimatedLoopWeight.count(PredLoop.getCycle())) - LoopWorkList.push_back(PredLoop); + if (isLoopExitingEdge({PredBlock, BB})) { + if (!EstimatedLoopWeight.count(CI->getCycle(PredBlock))) + LoopWorkList.push_back(PredBlock); } else if (!EstimatedBlockWeight.count(PredBlock)) BlockWorkList.push_back(PredBlock); } @@ -659,10 +632,9 @@ bool BPIConstruction::updateEstimatedBlockWeight( // In addition, \p WorkList is populated with basic blocks if at leas one // successor has updated estimated weight. void BPIConstruction::propagateEstimatedBlockWeight( - const LoopBlock &LoopBB, DominatorTree *DT, PostDominatorTree *PDT, - uint32_t BBWeight, SmallVectorImpl &BlockWorkList, - SmallVectorImpl &LoopWorkList) { - const BasicBlock *BB = LoopBB.getBlock(); + const BasicBlock *BB, DominatorTree *DT, PostDominatorTree *PDT, + uint32_t BBWeight, SmallVectorImpl &BlockWorkList, + SmallVectorImpl &LoopWorkList) { const auto *DTStartNode = DT->getNode(BB); const auto *PDTStartNode = PDT->getNode(BB); @@ -676,17 +648,16 @@ void BPIConstruction::propagateEstimatedBlockWeight( // of DomBB as well. break; - LoopBlock DomLoopBB = getLoopBlock(DomBB); - const LoopEdge Edge{DomLoopBB, LoopBB}; + const LoopEdge Edge{DomBB, BB}; // Don't propagate weight to blocks belonging to different loops. if (!isLoopEnteringExitingEdge(Edge)) { - if (!updateEstimatedBlockWeight(DomLoopBB, BBWeight, BlockWorkList, + if (!updateEstimatedBlockWeight(DomBB, BBWeight, BlockWorkList, LoopWorkList)) // If DomBB has weight set then all it's predecessors are already // processed (since we propagate weight up to the top of IR each time). break; } else if (isLoopExitingEdge(Edge)) { - LoopWorkList.push_back(DomLoopBB); + LoopWorkList.push_back(DomBB); } } } @@ -734,8 +705,8 @@ BPIConstruction::getInitialEstimatedBlockWeight(const BasicBlock *BB) { // best to propagate the weight to up/down the IR. void BPIConstruction::estimateBlockWeights(const Function &F, DominatorTree *DT, PostDominatorTree *PDT) { - SmallVector BlockWorkList; - SmallVector LoopWorkList; + SmallVector BlockWorkList; + SmallVector LoopWorkList; SmallDenseMap> LoopExitBlocks; // By doing RPO we make sure that all predecessors already have weights @@ -745,8 +716,8 @@ void BPIConstruction::estimateBlockWeights(const Function &F, DominatorTree *DT, if (auto BBWeight = getInitialEstimatedBlockWeight(BB)) // If we were able to find estimated weight for the block set it to this // block and propagate up the IR. - propagateEstimatedBlockWeight(getLoopBlock(BB), DT, PDT, *BBWeight, - BlockWorkList, LoopWorkList); + propagateEstimatedBlockWeight(BB, DT, PDT, *BBWeight, BlockWorkList, + LoopWorkList); // BlockWorklist/LoopWorkList contains blocks/loops with at least one // successor/exit having estimated weight. Try to propagate weight to such @@ -754,8 +725,8 @@ void BPIConstruction::estimateBlockWeights(const Function &F, DominatorTree *DT, // Process loops and blocks. Order is not important. do { while (!LoopWorkList.empty()) { - const LoopBlock LoopBB = LoopWorkList.pop_back_val(); - CycleRef C = LoopBB.getCycle(); + const BasicBlock *LoopBB = LoopWorkList.pop_back_val(); + CycleRef C = CI->getCycle(LoopBB); if (EstimatedLoopWeight.count(C)) continue; @@ -789,12 +760,11 @@ void BPIConstruction::estimateBlockWeights(const Function &F, DominatorTree *DT, // can't // think of any right now. And I doubt it will make any difference in // practice. - const LoopBlock LoopBB = getLoopBlock(BB); - auto MaxWeight = getMaxEstimatedEdgeWeight(LoopBB, successors(BB)); + auto MaxWeight = getMaxEstimatedEdgeWeight(BB, successors(BB)); if (MaxWeight) - propagateEstimatedBlockWeight(LoopBB, DT, PDT, *MaxWeight, - BlockWorkList, LoopWorkList); + propagateEstimatedBlockWeight(BB, DT, PDT, *MaxWeight, BlockWorkList, + LoopWorkList); } } while (!BlockWorkList.empty() || !LoopWorkList.empty()); } @@ -806,12 +776,12 @@ bool BPIConstruction::calcEstimatedHeuristics(const BasicBlock *BB) { assert(BB->getTerminator()->getNumSuccessors() > 1 && "expected more than one successor!"); - const LoopBlock LoopBB = getLoopBlock(BB); + CycleRef BBCycle = CI->getCycle(BB); SmallPtrSet UnlikelyBlocks; uint32_t TC = LBH_TAKEN_WEIGHT / LBH_NONTAKEN_WEIGHT; - if (LoopBB.getCycle()) - computeUnlikelySuccessors(BB, *CI, LoopBB.getCycle(), UnlikelyBlocks); + if (BBCycle) + computeUnlikelySuccessors(BB, *CI, BBCycle, UnlikelyBlocks); // Changed to 'true' if at least one successor has estimated weight. bool FoundEstimatedWeight = false; @@ -820,8 +790,7 @@ bool BPIConstruction::calcEstimatedHeuristics(const BasicBlock *BB) { // Go over all successors of BB and put their weights into SuccWeights. for (const BasicBlock *SuccBB : successors(BB)) { std::optional Weight; - const LoopBlock SuccLoopBB = getLoopBlock(SuccBB); - const LoopEdge Edge{LoopBB, SuccLoopBB}; + const LoopEdge Edge{BB, SuccBB}; Weight = getEstimatedEdgeWeight(Edge); @@ -834,7 +803,7 @@ bool BPIConstruction::calcEstimatedHeuristics(const BasicBlock *BB) { Weight.value_or(static_cast(BlockExecWeight::DEFAULT)) / TC); } - bool IsUnlikelyEdge = LoopBB.getCycle() && UnlikelyBlocks.contains(SuccBB); + bool IsUnlikelyEdge = BBCycle && UnlikelyBlocks.contains(SuccBB); if (IsUnlikelyEdge && // Avoid adjustment of ZERO weight since it should remain unchanged. Weight != static_cast(BlockExecWeight::ZERO)) { From 4bf0d282b9afd8f1a46a5bb0403c8363eca3deb4 Mon Sep 17 00:00:00 2001 From: Erich Keane Date: Tue, 21 Jul 2026 11:57:00 -0700 Subject: [PATCH 16/50] [OpenACC/CIR] Remove std::transform_inclusive_scan use (#211076) This seemingly is only added for GCC libstdcxx 11.1's C++17 support, but our support matrix is 7.4. Replace it with a loop that implements the same requirement. --- clang/lib/CIR/CodeGen/CIRGenOpenACCRecipe.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/clang/lib/CIR/CodeGen/CIRGenOpenACCRecipe.cpp b/clang/lib/CIR/CodeGen/CIRGenOpenACCRecipe.cpp index b82c854d2c03e..2c6365af613c6 100644 --- a/clang/lib/CIR/CodeGen/CIRGenOpenACCRecipe.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenOpenACCRecipe.cpp @@ -155,10 +155,11 @@ mlir::Value OpenACCRecipeBuilderBase::makeBoundsAlloca( // Collect the 'do we have any allocas needed after this type' list. llvm::SmallVector allocasLeftArr; llvm::ArrayRef resultTypes = boundTypes.drop_front(); - std::transform_inclusive_scan( - resultTypes.begin(), resultTypes.end(), - std::back_inserter(allocasLeftArr), std::plus{}, - [](QualType ty) { return !ty->isConstantArrayType(); }, false); + bool accumulator = false; + for (QualType ty : resultTypes) { + accumulator = accumulator || !ty->isConstantArrayType(); + allocasLeftArr.push_back(accumulator); + } // Keep track of the number of 'elements' that we're allocating. Individual // allocas should multiply this by the size of its current allocation. From 5009f54dd420f3c9df3f7e54022fc2234e2f0ebe Mon Sep 17 00:00:00 2001 From: Caroline Newcombe Date: Tue, 21 Jul 2026 15:09:57 -0400 Subject: [PATCH 17/50] [flang][OpenMP] Reland implement collapse for imperfectly nested loops (#211000) Reland of combined #208528 and #210753. Fixes #199092 - Flang previously rejected intervening code between associated loops in a collapsed nest (e.g. collapse(2) with statements between the outer and inner DO). This patch removes that restriction and implements correct lowering. --- flang/include/flang/Semantics/openmp-utils.h | 6 + flang/lib/Lower/OpenMP/OpenMP.cpp | 220 +++++- flang/lib/Lower/OpenMP/Utils.cpp | 33 +- flang/lib/Semantics/check-omp-loop.cpp | 10 + flang/lib/Semantics/openmp-utils.cpp | 87 ++- .../Lower/OpenMP/collapse-imperfect-nest.f90 | 648 ++++++++++++++++++ .../OpenMP/collapse-labeled-do-host-eval.f90 | 45 ++ .../Lower/OpenMP/collapse-loop-transform.f90 | 21 + .../collapse-target-intervening-todo.f90 | 28 + flang/test/Semantics/OpenMP/do-collapse.f90 | 8 +- .../OpenMP/do-concurrent-collapse-60.f90 | 4 +- .../OpenMP/do-concurrent-collapse.f90 | 4 +- flang/test/Semantics/OpenMP/do08.f90 | 15 - flang/test/Semantics/OpenMP/do10.f90 | 2 +- flang/test/Semantics/OpenMP/do13.f90 | 10 +- flang/test/Semantics/OpenMP/do15.f90 | 9 - flang/test/Semantics/OpenMP/do16.f90 | 6 - flang/test/Semantics/OpenMP/do22.f90 | 127 +++- .../OpenMP/doacross-nesting-omp60.f90 | 148 ++++ .../metadirective-loop-applicability.f90 | 10 +- .../OpenMP/metadirective-loop-nest.f90 | 10 +- .../OpenMP/ordered-nesting-omp50.f90 | 50 ++ .../OpenMP/ordered-nesting-omp51.f90 | 36 + 23 files changed, 1465 insertions(+), 72 deletions(-) create mode 100644 flang/test/Lower/OpenMP/collapse-imperfect-nest.f90 create mode 100644 flang/test/Lower/OpenMP/collapse-labeled-do-host-eval.f90 create mode 100644 flang/test/Lower/OpenMP/collapse-loop-transform.f90 create mode 100644 flang/test/Lower/OpenMP/collapse-target-intervening-todo.f90 create mode 100644 flang/test/Semantics/OpenMP/doacross-nesting-omp60.f90 create mode 100644 flang/test/Semantics/OpenMP/ordered-nesting-omp50.f90 create mode 100644 flang/test/Semantics/OpenMP/ordered-nesting-omp51.f90 diff --git a/flang/include/flang/Semantics/openmp-utils.h b/flang/include/flang/Semantics/openmp-utils.h index 086871b867a3a..496cad0b3bd57 100644 --- a/flang/include/flang/Semantics/openmp-utils.h +++ b/flang/include/flang/Semantics/openmp-utils.h @@ -404,6 +404,12 @@ std::optional> CollectAffectedDoLoops( const parser::OpenMPLoopConstruct &x, unsigned version, SemanticsContext *semaCtx = nullptr); +/// Returns whether the loop nest associated with `x` is a doacross loop nest, +/// i.e. its body contains an `ordered` directive carrying a doacross +/// dependence (the `doacross` clause, or the pre-5.2 `depend(sink/source)` +/// equivalent) that binds to `x`. Such a nest must be perfectly nested. +bool IsDoacrossAffected(const parser::OpenMPLoopConstruct &x); + struct LoopSequence { LoopSequence(const parser::ExecutionPartConstruct &root, unsigned version, bool allowAllLoops = false, SemanticsContext *semaCtx = nullptr); diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index 04df4f6e69377..a3acc2e991307 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -36,6 +36,7 @@ #include "flang/Optimizer/Builder/BoxValue.h" #include "flang/Optimizer/Builder/FIRBuilder.h" #include "flang/Optimizer/Builder/Todo.h" +#include "flang/Optimizer/Dialect/FIROpsSupport.h" #include "flang/Optimizer/Dialect/FIRType.h" #include "flang/Optimizer/HLFIR/HLFIROps.h" #include "flang/Parser/openmp-utils.h" @@ -933,6 +934,200 @@ static void genNestedEvaluations(lower::AbstractConverter &converter, converter.genEval(e); } +static mlir::Operation *setLoopVar(lower::AbstractConverter &converter, + mlir::Location loc, mlir::Value indexVal, + const semantics::Symbol *sym); + +/// Emit the body of a collapsed loop nest, including any intervening code +/// from imperfect nesting at intermediate levels (CLN relaxation, applied +/// retroactively for all OMP versions). +/// +/// Because omp.loop_nest places its entire body at the innermost nesting +/// level, intervening code must be guarded so that it only executes on the +/// iterations where the corresponding inner induction variables are at their +/// initial (for intervening code before nested loop) or final (for intervening +/// code after nested loop) values. +/// +/// \param [in] converter - PFT to MLIR conversion interface. +/// \param [in] outerEval - the evaluation containing the outermost loop +/// (typically the OpenMP construct evaluation). +/// \param [in] collapseValue - number of loops being collapsed (>= 1). +static void genCollapsedLoopNestBody(lower::AbstractConverter &converter, + lower::pft::Evaluation &outerEval, + int collapseValue) { + assert(collapseValue >= 1); + if (collapseValue == 1) { + genNestedEvaluations(converter, outerEval, /*collapseValue=*/1); + return; + } + + fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); + const mlir::Location loc = converter.getCurrentLocation(); + + // Get the enclosing omp.loop_nest to access induction variables and bounds. + auto loopNestOp = mlir::dyn_cast( + firOpBuilder.getInsertionBlock()->getParentOp()); + assert(loopNestOp && "expected to be inside omp.loop_nest"); + + // Collect before/after evaluations at each intermediate level. + struct LevelInfo { + llvm::SmallVector before; + llvm::SmallVector after; + + // Whether this level carries intervening code (i.e. imperfect nesting). + bool hasInterveningCode() const { + return !before.empty() || !after.empty(); + } + }; + llvm::SmallVector levels; + + // DO-variable symbol of each collapsed level (index 0 = outermost). Used to + // restore an inner loop's variable to its Fortran terminal value before + // emitting "after" intervening code (see below). + llvm::SmallVector ivSyms; + + lower::pft::Evaluation *curEval = &outerEval; + for (int i = 0; i < collapseValue - 1; ++i) { + lower::pft::Evaluation *doEval = getNestedDoConstruct(*curEval); + const semantics::Symbol *ivSym = getIterationVariableSymbol(*doEval); + assert(ivSym && "expected iteration variable on collapsed DO loop"); + ivSyms.push_back(ivSym); + LevelInfo level; + bool pastDo = false; + for (lower::pft::Evaluation &e : doEval->getNestedEvaluations()) { + // A labeled DO loop leaves a no-op ContinueStmt for its terminating + // labeled statement (alongside the EndDoStmt). Skip it so it isn't + // misclassified as intervening code. + if (e.getIf() || e.getIf() || + e.getIf() || + e.getIf()) + continue; + // Semantics guarantees the only DoConstruct here is the next associated + // loop (non-associated DO loops are rejected as intervening code). + if (e.getIf()) { + pastDo = true; + continue; + } + if (!pastDo) + level.before.push_back(&e); + else + level.after.push_back(&e); + } + levels.push_back(std::move(level)); + curEval = doEval; + } + // DO-variable symbol of the innermost collapsed loop must be restored + // inside enclosing "after" regions. + const semantics::Symbol *innermostIvSym = + getIterationVariableSymbol(*getNestedDoConstruct(*curEval)); + assert(innermostIvSym && "expected iteration variable on collapsed DO loop"); + ivSyms.push_back(innermostIvSym); + + // Build a guard condition: all induction variables from + // startLevel..endLevel-1 equal their respective bound values. + // For "before" guards (useLowerBound=true), compare iv == lb (first iter). + // For "after" guards (useLowerBound=false), compare iv == last_iv. + const auto lbs = loopNestOp.getLoopLowerBounds(); + const auto ubs = loopNestOp.getLoopUpperBounds(); + const auto steps = loopNestOp.getLoopSteps(); + + // The intervening-code guards and terminal-value restoration do arithmetic + // on the collapsed loop bounds. If those bounds are host_eval block arguments + // of an enclosing omp.target region, such uses are illegal, so diagnose + // instead of emitting IR the omp.target verifier rejects. + const bool hasInterveningCode = llvm::any_of( + levels, [](const LevelInfo &l) { return l.hasInterveningCode(); }); + if (hasInterveningCode) { + auto isHostEvalValue = [](mlir::Value v) { + auto blockArg = mlir::dyn_cast(v); + if (!blockArg) + return false; + auto iface = mlir::dyn_cast( + blockArg.getOwner()->getParentOp()); + return iface && + llvm::is_contained(iface.getHostEvalBlockArgs(), blockArg); + }; + if (llvm::any_of(lbs, isHostEvalValue) || + llvm::any_of(ubs, isHostEvalValue) || + llvm::any_of(steps, isHostEvalValue)) + TODO(loc, "collapsed loop nest with intervening code whose loop bounds " + "are evaluated on the host for an enclosing 'target' region"); + } + + // Last value the induction variable at \p lvl actually takes: + // lb + ((ub - lb) / step) * step. For unit steps this is exactly ub. + auto computeLastIV = [&](const int lvl) -> mlir::Value { + const auto constStep = fir::getIntIfConstant(steps[lvl]); + if (constStep && (*constStep == 1 || *constStep == -1)) + return ubs[lvl]; + const mlir::Value lb = lbs[lvl]; + const mlir::Value ub = ubs[lvl]; + const mlir::Value step = steps[lvl]; + const mlir::Value range = + mlir::arith::SubIOp::create(firOpBuilder, loc, ub, lb); + const mlir::Value tripMinus1 = + mlir::arith::DivSIOp::create(firOpBuilder, loc, range, step); + const mlir::Value lastOffset = + mlir::arith::MulIOp::create(firOpBuilder, loc, tripMinus1, step); + return mlir::arith::AddIOp::create(firOpBuilder, loc, lb, lastOffset); + }; + + auto buildGuard = [&](const int startLevel, const int endLevel, + const bool useLowerBound) -> mlir::Value { + mlir::Value cond; + for (int lvl = startLevel; lvl < endLevel; ++lvl) { + const mlir::Value iv = loopNestOp.getRegion().getArgument(lvl); + const mlir::Value target = useLowerBound ? lbs[lvl] : computeLastIV(lvl); + const mlir::Value cmp = mlir::arith::CmpIOp::create( + firOpBuilder, loc, mlir::arith::CmpIPredicate::eq, iv, target); + if (!cond) + cond = cmp; + else + cond = mlir::arith::AndIOp::create(firOpBuilder, loc, cond, cmp); + } + return cond; + }; + + // Emit "before" code at each level, guarded by inner IVs == lower bounds. + for (int i = 0; i < static_cast(levels.size()); ++i) { + if (levels[i].before.empty()) + continue; + const mlir::Value guard = + buildGuard(i + 1, collapseValue, /*useLowerBound=*/true); + auto ifOp = fir::IfOp::create(firOpBuilder, loc, guard, /*else*/ false); + firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front()); + for (auto *e : levels[i].before) + converter.genEval(*e); + firOpBuilder.setInsertionPointAfter(ifOp); + } + + // Emit innermost loop body. + genNestedEvaluations(converter, *curEval, /*collapseValue=*/1); + + // Emit "after" code at each level (innermost first), guarded by + // inner IVs == last iteration values (accounts for non-unit steps). + for (int i = static_cast(levels.size()) - 1; i >= 0; --i) { + if (levels[i].after.empty()) + continue; + const mlir::Value guard = + buildGuard(i + 1, collapseValue, /*useLowerBound=*/false); + auto ifOp = fir::IfOp::create(firOpBuilder, loc, guard, /*else*/ false); + firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front()); + // A normally-terminated Fortran DO loop leaves its variable one step past + // the last executed value, but the flattened nest leaves each at its last + // executed value. Restore the terminal value before running "after" code + // that may read it. + for (int lvl = i + 1; lvl < collapseValue; ++lvl) { + const mlir::Value terminal = mlir::arith::AddIOp::create( + firOpBuilder, loc, computeLastIV(lvl), steps[lvl]); + setLoopVar(converter, loc, terminal, ivSyms[lvl]); + } + for (auto *e : levels[i].after) + converter.genEval(*e); + firOpBuilder.setInsertionPointAfter(ifOp); + } +} + static fir::GlobalOp globalInitialization(lower::AbstractConverter &converter, fir::FirOpBuilder &firOpBuilder, const semantics::Symbol &sym, @@ -1624,6 +1819,13 @@ struct OpWithBodyGenInfo { return *this; } + OpWithBodyGenInfo &setCollapseInfo(int value, + lower::pft::Evaluation &outerEval) { + collapseValue = value; + outerCollapseEval = &outerEval; + return *this; + } + /// [inout] converter to use for the clauses. lower::AbstractConverter &converter; /// [in] Symbol table @@ -1652,6 +1854,10 @@ struct OpWithBodyGenInfo { bool genSkeletonOnly = false; /// [in] enables handling of privatized variable unless set to `false`. bool privatize = true; + /// [in] if set, outermost evaluation and collapse depth for emitting + /// intervening code from imperfect collapsed loop nests. + lower::pft::Evaluation *outerCollapseEval = nullptr; + int collapseValue = 0; }; static mlir::Value getReductionOverrideValue(fir::FirOpBuilder &builder, @@ -1951,7 +2157,11 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info, firOpBuilder.setInsertionPointToEnd(&op.getRegion(0).back()); auto *temp = lower::genOpenMPTerminator(firOpBuilder, &op, info.loc); firOpBuilder.setInsertionPointAfter(marker); - genNestedEvaluations(info.converter, info.eval); + if (info.outerCollapseEval) + genCollapsedLoopNestBody(info.converter, *info.outerCollapseEval, + info.collapseValue); + else + genNestedEvaluations(info.converter, info.eval); temp->erase(); } @@ -2831,7 +3041,8 @@ genLoopNestOp(lower::AbstractConverter &converter, lower::SymMap &symTable, directive) .setClauses(&item->clauses) .setDataSharingProcessor(&dsp) - .setGenRegionEntryCb(ivCallback), + .setGenRegionEntryCb(ivCallback) + .setCollapseInfo(nestValue, eval), queue, item, clauseOps); converter.overrideExprValues(oldOverrides); return loopNestOp; @@ -2923,7 +3134,7 @@ static void genCanonicalLoopNest( // Step 1: Loop prologues // Computing the trip count must happen before entering the outermost loop lower::pft::Evaluation *innermostEval = nestedEval; - for ([[maybe_unused]] auto iv : ivs) { + for (std::size_t i = 0; i < ivs.size(); ++i) { if (innermostEval->getIf()->IsDoConcurrent()) { // OpenMP specifies DO CONCURRENT only with the `!omp loop` construct. // Will need to add special cases for this combination. @@ -3005,7 +3216,8 @@ static void genCanonicalLoopNest( mlir::Value cli = newcli.getResult(); clis.push_back(cli); - innermostEval = &*std::next(innermostEval->getNestedEvaluations().begin()); + if (i + 1 < ivs.size()) + innermostEval = getNestedDoConstruct(*innermostEval); } // Step 2: Create nested canoncial loops diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp index b51f08ec89e4e..8f57f00d59c58 100644 --- a/flang/lib/Lower/OpenMP/Utils.cpp +++ b/flang/lib/Lower/OpenMP/Utils.cpp @@ -702,7 +702,12 @@ static void processTileSizesFromOpenMPConstruct( } } -pft::Evaluation *getNestedDoConstruct(pft::Evaluation &eval) { +// Searches the nested evaluations of \p eval for the next DO construct of a +// loop nest, returning nullptr when there is none left to descend into. This +// can happen when COLLAPSE counts loops that a transforming construct such as +// TILE generates from the source DO loops. getNestedDoConstruct wraps this for +// callers that require a DO construct and asserts when none is found. +static pft::Evaluation *tryGetNestedDoConstruct(pft::Evaluation &eval) { for (pft::Evaluation &nested : eval.getNestedEvaluations()) { // In an OpenMPConstruct there can be compiler directives: // 1 <> @@ -722,10 +727,18 @@ pft::Evaluation *getNestedDoConstruct(pft::Evaluation &eval) { // Loop transformations can introduce nested OpenMP // constructs between the directive and the actual do-loop nest. if (nested.getIf()) - return getNestedDoConstruct(nested); - assert(false && "Unexpected construct in the nested evaluations"); + return tryGetNestedDoConstruct(nested); + // Skip valid intervening code in imperfect loop nests + continue; } - llvm_unreachable("Expected do loop to be in the nested evaluations"); + return nullptr; +} + +pft::Evaluation *getNestedDoConstruct(pft::Evaluation &eval) { + pft::Evaluation *doConstruct{tryGetNestedDoConstruct(eval)}; + if (!doConstruct) + llvm_unreachable("Expected do loop to be in the nested evaluations"); + return doConstruct; } /// Populates the sizes vector with values if the given OpenMPConstruct @@ -850,8 +863,16 @@ void collectLoopRelatedInfo( iv.push_back(bounds->Name().thing.symbol); loopVarTypeSize = std::max( loopVarTypeSize, bounds->Name().thing.symbol->GetUltimate().size()); - if (--collapseValue) - doConstructEval = getNestedDoConstruct(*doConstructEval); + if (--collapseValue) { + doConstructEval = tryGetNestedDoConstruct(*doConstructEval); + // No DO construct left to descend into: COLLAPSE (or a tile SIZES list) + // counts loops generated by a nested loop-transforming construct. Not + // yet supported in lowering. + if (!doConstructEval) + TODO(currentLocation, + "Collapsing a loop nest that contains a loop-transforming " + "construct"); + } } while (collapseValue > 0); convertLoopBounds(converter, currentLocation, result, loopVarTypeSize); diff --git a/flang/lib/Semantics/check-omp-loop.cpp b/flang/lib/Semantics/check-omp-loop.cpp index b05ae4d0d8946..e827994e7c6d6 100644 --- a/flang/lib/Semantics/check-omp-loop.cpp +++ b/flang/lib/Semantics/check-omp-loop.cpp @@ -326,6 +326,16 @@ void OmpStructureChecker::CheckNestedConstruct( // Check requirements on nest depth. auto [needDepth, needPerfect]{ GetAffectedNestDepthWithReason(beginSpec, version)}; + + // Perfect nesting for doacross loop nests is handled differently across + // versions. Only in 6.0+ is the requirement keyed off the body + // actually containing an ORDERED directive with a doacross dependence + // rather than the ORDERED clause, so the body scan applies only to those + // later versions. + if (!needPerfect && version > 52 && IsDoacrossAffected(x)) { + needPerfect = true; + } + auto &[haveSema, havePerf]{sequence.depth()}; auto haveDepth{needPerfect ? havePerf : haveSema}; diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp index e5cdade2cbca0..27d23a9ffde30 100644 --- a/flang/lib/Semantics/openmp-utils.cpp +++ b/flang/lib/Semantics/openmp-utils.cpp @@ -1300,11 +1300,33 @@ std::pair, bool> GetAffectedNestDepthWithReason( ocount = std::nullopt; oreason = Reason(); } + bool hasOrdered{parser::omp::FindClause( + spec, llvm::omp::Clause::OMPC_ordered) != nullptr}; + // Perfect-nesting requirement for the ORDERED clause, by version: + // + // 5.0: Any ORDERED clause makes the associated loops a doacross loop + // nest that must be perfectly nested, whether or not the clause + // has an argument. + // 5.1/5.2: Only an ORDERED clause *with* an argument requires perfect + // nesting; a bare ORDERED clause does not. + // 6.0: Perfect nesting is required only when the body actually + // contains an ORDERED directive with a doacross dependence; + // that is detected separately by the caller via + // IsDoacrossAffected, so ORDERED(n) alone does not force + // perfect nesting here. if (ccount < ocount) { - // `ocount` cannot be std::nullopt here (C++ std guarantee). - return {{ocount.value_or(1), std::move(oreason)}, true}; + return {{ocount.value_or(1), std::move(oreason)}, version <= 52}; } - return {{ccount.value_or(1), std::move(creason)}, true}; + // Same rule as above when COLLAPSE drives the depth: ORDERED(n) requires a + // perfect nest through 5.2, while > 5.2 defers to IsDoacrossAffected. In + // 5.0, an ORDERED clause without argument also requires perfect nesting. + // The CLN relaxation for COLLAPSE is applied retroactively for all + // versions. + bool needPerfect{false}; + if (version <= 52) { + needPerfect = ocount.has_value() || (version == 50 && hasOrdered); + } + return {{ccount.value_or(1), std::move(creason)}, needPerfect}; } if (IsLoopTransforming(dir)) { @@ -1522,6 +1544,65 @@ std::optional GetMinimumSequenceCount( return GetMinimumSequenceCount(std::nullopt, std::nullopt); } +namespace { +/// Visitor that detects an `ordered` directive carrying a doacross dependence +/// (the `doacross` clause, or the pre-5.2 `depend(sink/source)` equivalent) +/// that binds to the loop construct being checked. Prunes nested constructs +/// that start their own associated loop nest, but descends into +/// loop-transforming constructs (e.g. tile, unroll), whose generated loops +/// extend the current nest. +struct DoacrossFinder { + bool found{false}; + bool inOrdered{false}; + template bool Pre(const T &) { return !found; } + template void Post(const T &) {} + + // Prune nested constructs that start their own associated loop nest; a + // doacross inside them binds there, not here. Loop-transforming constructs + // are the exception: their generated loops extend the current nest, so a + // doacross inside one still binds to the construct being checked. + bool Pre(const parser::OmpBlockConstruct &) { return false; } + bool Pre(const parser::OpenMPLoopConstruct &x) { + if (IsLoopTransforming(x.BeginDir().DirId())) { + return !found; + } + return false; + } + + bool Pre(const parser::OpenMPSimpleStandaloneConstruct &x) { + inOrdered = x.v.DirId() == llvm::omp::Directive::OMPD_ordered; + return !found; + } + void Post(const parser::OpenMPSimpleStandaloneConstruct &) { + inOrdered = false; + } + + bool Pre(const parser::OmpDoacross &) { + if (inOrdered) { + found = true; + } + return false; + } +}; + +static bool ContainsOrderedDoacross(const parser::Block &block) { + DoacrossFinder finder; + parser::Walk(block, finder); + return finder.found; +} +} // namespace + +bool IsDoacrossAffected(const parser::OpenMPLoopConstruct &x) { + // A loop nest is doacross-affected when it has an `ordered` clause and a + // stand-alone `ordered` construct carrying a doacross dependence is closely + // nested in its body. + const parser::OmpDirectiveSpecification &spec{x.BeginDir()}; + if (!parser::omp::FindClause(spec, llvm::omp::Clause::OMPC_ordered)) { + return false; + } + return ContainsOrderedDoacross(std::get(x.t)); +} + /// Collect the DO loops that are affected directly by the given loop /// transformation. Not all DO loops nested in the associated nest are /// affected by the top-level loop transformation, e.g. diff --git a/flang/test/Lower/OpenMP/collapse-imperfect-nest.f90 b/flang/test/Lower/OpenMP/collapse-imperfect-nest.f90 new file mode 100644 index 0000000000000..e963049090f49 --- /dev/null +++ b/flang/test/Lower/OpenMP/collapse-imperfect-nest.f90 @@ -0,0 +1,648 @@ +! Test lowering of imperfectly nested collapse loops (CLN relaxation). +! Intervening code is guarded by IV comparisons to restore correct +! execution frequency and ordering within the flat omp.loop_nest body. + +! RUN: %flang_fc1 -fopenmp -emit-hlfir %s -o - | FileCheck %s + +! CHECK-LABEL: func.func @_QPcollapse2_imperfect +subroutine collapse2_imperfect(n, x) + integer, intent(in) :: n + integer, intent(inout) :: x + integer :: i, j + + !$omp do collapse(2) + do i = 1, n + x = x + 1 + do j = 1, n + x = x + j + end do + end do + !$omp end do +end subroutine + +! CHECK: omp.wsloop +! CHECK-NEXT: omp.loop_nest (%[[I:.*]], %[[J:.*]]) : i32 = +! CHECK-SAME: (%{{.*}}, %[[LB_J:.*]]) to +! CHECK: hlfir.assign %[[I]] +! CHECK: hlfir.assign %[[J]] +! Guard: j == lower_bound (before code executes once per i) +! CHECK: %[[CMP:.*]] = arith.cmpi eq, %[[J]], %[[LB_J]] : i32 +! CHECK: fir.if %[[CMP]] { +! Intervening code: x = x + 1 +! CHECK: %[[X1:.*]] = fir.load %{{.*}} : !fir.ref +! CHECK: %[[C1:.*]] = arith.constant 1 : i32 +! CHECK: %[[ADD1:.*]] = arith.addi %[[X1]], %[[C1]] : i32 +! CHECK: hlfir.assign %[[ADD1]] to %{{.*}} : i32, !fir.ref +! CHECK: } +! Innermost body: x = x + j +! CHECK: %[[X2:.*]] = fir.load %{{.*}} : !fir.ref +! CHECK: %[[JVAL:.*]] = fir.load %{{.*}} : !fir.ref +! CHECK: %[[ADD2:.*]] = arith.addi %[[X2]], %[[JVAL]] : i32 +! CHECK: hlfir.assign %[[ADD2]] to %{{.*}} : i32, !fir.ref +! CHECK: omp.yield + +! CHECK-LABEL: func.func @_QPcollapse3_imperfect +subroutine collapse3_imperfect(n, x) + integer, intent(in) :: n + integer, intent(inout) :: x + integer :: i, j, k + + !$omp do collapse(3) + do i = 1, n + x = x + i + do j = 1, n + x = x + j + do k = 1, n + x = x + k + end do + end do + end do + !$omp end do +end subroutine + +! CHECK: omp.wsloop +! CHECK-NEXT: omp.loop_nest (%[[I3:.*]], %[[J3:.*]], %[[K3:.*]]) : i32 = +! CHECK-SAME: (%{{.*}}, %[[LB_J3:.*]], %[[LB_K3:.*]]) to +! CHECK: hlfir.assign %[[I3]] +! CHECK: hlfir.assign %[[J3]] +! CHECK: hlfir.assign %[[K3]] +! Guard: j == lb_j AND k == lb_k (level 0 before code, once per i) +! CHECK: %[[CMP_J:.*]] = arith.cmpi eq, %[[J3]], %[[LB_J3]] : i32 +! CHECK: %[[CMP_K1:.*]] = arith.cmpi eq, %[[K3]], %[[LB_K3]] : i32 +! CHECK: %[[AND1:.*]] = arith.andi %[[CMP_J]], %[[CMP_K1]] : i1 +! CHECK: fir.if %[[AND1]] { +! Intervening code at level 0: x = x + i +! CHECK: %[[XI:.*]] = fir.load %{{.*}} : !fir.ref +! CHECK: %[[IVAL:.*]] = fir.load %{{.*}} : !fir.ref +! CHECK: %[[ADDI:.*]] = arith.addi %[[XI]], %[[IVAL]] : i32 +! CHECK: hlfir.assign %[[ADDI]] to %{{.*}} : i32, !fir.ref +! CHECK: } +! Guard: k == lb_k (level 1 before code, once per (i,j)) +! CHECK: %[[CMP_K2:.*]] = arith.cmpi eq, %[[K3]], %[[LB_K3]] : i32 +! CHECK: fir.if %[[CMP_K2]] { +! Intervening code at level 1: x = x + j +! CHECK: %[[XJ:.*]] = fir.load %{{.*}} : !fir.ref +! CHECK: %[[JVAL3:.*]] = fir.load %{{.*}} : !fir.ref +! CHECK: %[[ADDJ:.*]] = arith.addi %[[XJ]], %[[JVAL3]] : i32 +! CHECK: hlfir.assign %[[ADDJ]] to %{{.*}} : i32, !fir.ref +! CHECK: } +! Innermost body: x = x + k +! CHECK: %[[XK:.*]] = fir.load %{{.*}} : !fir.ref +! CHECK: %[[KVAL:.*]] = fir.load %{{.*}} : !fir.ref +! CHECK: %[[ADDK:.*]] = arith.addi %[[XK]], %[[KVAL]] : i32 +! CHECK: hlfir.assign %[[ADDK]] to %{{.*}} : i32, !fir.ref +! CHECK: omp.yield + +! CHECK-LABEL: func.func @_QPcollapse2_both_sides +subroutine collapse2_both_sides(n, x) + integer, intent(in) :: n + integer, intent(inout) :: x + integer :: i, j + + !$omp simd collapse(2) + do i = 1, n + x = x + 1 + do j = 1, n + x = x + j + end do + call ext_sub(x) + end do + !$omp end simd +end subroutine + +! CHECK: omp.simd +! CHECK-NEXT: omp.loop_nest (%[[I4:.*]], %[[J4:.*]]) : i32 = +! CHECK-SAME: (%{{[^)]*}}, %[[LB_J4:[^)]*]]) to (%{{[^)]*}}, %[[UB_J4:[^)]*]]) +! CHECK: hlfir.assign %[[I4]] +! CHECK: hlfir.assign %[[J4]] +! Guard: j == lower_bound (before code) +! CHECK: %[[CMP_B:.*]] = arith.cmpi eq, %[[J4]], %[[LB_J4]] : i32 +! CHECK: fir.if %[[CMP_B]] { +! Intervening code before inner loop: x = x + 1 +! CHECK: %[[XB:.*]] = fir.load %{{.*}} : !fir.ref +! CHECK: %[[CB:.*]] = arith.constant 1 : i32 +! CHECK: %[[ADDB:.*]] = arith.addi %[[XB]], %[[CB]] : i32 +! CHECK: hlfir.assign %[[ADDB]] to %{{.*}} : i32, !fir.ref +! CHECK: } +! Innermost body: x = x + j +! CHECK: %[[XIN:.*]] = fir.load %{{.*}} : !fir.ref +! CHECK: %[[JIN:.*]] = fir.load %{{.*}} : !fir.ref +! CHECK: %[[ADDIN:.*]] = arith.addi %[[XIN]], %[[JIN]] : i32 +! CHECK: hlfir.assign %[[ADDIN]] to %{{.*}} : i32, !fir.ref +! Guard: j == upper_bound (after code) +! CHECK: %[[CMP_A:.*]] = arith.cmpi eq, %[[J4]], %[[UB_J4]] : i32 +! CHECK: fir.if %[[CMP_A]] { +! Intervening code after inner loop: call ext_sub(x) +! CHECK: fir.call @_QPext_sub +! CHECK: } +! CHECK: omp.yield + +! Test collapse(3) with both before and after code at multiple levels. +! CHECK-LABEL: func.func @_QPcollapse3_both_sides +subroutine collapse3_both_sides(n, x) + integer, intent(in) :: n + integer, intent(inout) :: x + integer :: i, j, k + + !$omp do collapse(3) + do i = 1, n + x = x + i + do j = 1, n + x = x + j + do k = 1, n + x = x + k + end do + x = x - j + end do + x = x - i + end do + !$omp end do +end subroutine + +! CHECK: omp.wsloop +! CHECK-NEXT: omp.loop_nest (%[[I:.*]], %[[J:.*]], %[[K:.*]]) : i32 = +! CHECK-SAME: (%{{[^)]*}}, %[[LB_J:[^)]*]], %[[LB_K:[^)]*]]) to (%{{[^)]*}}, %[[UB_J:[^)]*]], %[[UB_K:[^)]*]]) +! CHECK: hlfir.assign %[[I]] +! CHECK: hlfir.assign %[[J]] +! CHECK: hlfir.assign %[[K]] +! +! --- "before" guards (outermost level first) --- +! +! Guard level 0 before: j == lb_j AND k == lb_k (once per i) +! CHECK: %[[CJ1:.*]] = arith.cmpi eq, %[[J]], %[[LB_J]] : i32 +! CHECK: %[[CK1:.*]] = arith.cmpi eq, %[[K]], %[[LB_K]] : i32 +! CHECK: %[[AND1:.*]] = arith.andi %[[CJ1]], %[[CK1]] : i1 +! CHECK: fir.if %[[AND1]] { +! CHECK: arith.addi +! CHECK: hlfir.assign +! CHECK: } +! Guard level 1 before: k == lb_k (once per (i,j)) +! CHECK: %[[CK2:.*]] = arith.cmpi eq, %[[K]], %[[LB_K]] : i32 +! CHECK: fir.if %[[CK2]] { +! CHECK: arith.addi +! CHECK: hlfir.assign +! CHECK: } +! +! --- innermost body: x = x + k --- +! CHECK: arith.addi +! CHECK: hlfir.assign +! +! --- "after" guards (innermost level first) --- +! +! Guard level 1 after: k == ub_k (once per (i,j)) +! CHECK: %[[CK3:.*]] = arith.cmpi eq, %[[K]], %[[UB_K]] : i32 +! CHECK: fir.if %[[CK3]] { +! CHECK: arith.subi +! CHECK: hlfir.assign +! CHECK: } +! Guard level 0 after: j == ub_j AND k == ub_k (once per i) +! CHECK: %[[CJ2:.*]] = arith.cmpi eq, %[[J]], %[[UB_J]] : i32 +! CHECK: %[[CK4:.*]] = arith.cmpi eq, %[[K]], %[[UB_K]] : i32 +! CHECK: %[[AND2:.*]] = arith.andi %[[CJ2]], %[[CK4]] : i1 +! CHECK: fir.if %[[AND2]] { +! CHECK: arith.subi +! CHECK: hlfir.assign +! CHECK: } +! CHECK: omp.yield + +! Test collapse(4) with imperfect nesting at some levels and perfectly nested +! innermost loops. Level 0 (i->j) has before+after, level 1 (j->k) has before +! only, level 2 (k->l) is perfectly nested. This exercises skipping empty levels. +! CHECK-LABEL: func.func @_QPcollapse4_mixed +subroutine collapse4_mixed(n, x) + integer, intent(in) :: n + integer, intent(inout) :: x + integer :: i, j, k, l + + !$omp do collapse(4) + do i = 1, n + x = x + i + do j = 1, n + x = x + j + do k = 1, n + do l = 1, n + x = x + l + end do + end do + end do + x = x - i + end do + !$omp end do +end subroutine + +! CHECK: omp.wsloop +! CHECK-NEXT: omp.loop_nest (%[[I:.*]], %[[J:.*]], %[[K:.*]], %[[L:.*]]) : i32 = +! CHECK-SAME: (%{{[^)]*}}, %[[LB_J:[^)]*]], %[[LB_K:[^)]*]], %[[LB_L:[^)]*]]) to (%{{[^)]*}}, %[[UB_J:[^)]*]], %[[UB_K:[^)]*]], %[[UB_L:[^)]*]]) +! CHECK: hlfir.assign %[[I]] +! CHECK: hlfir.assign %[[J]] +! CHECK: hlfir.assign %[[K]] +! CHECK: hlfir.assign %[[L]] +! +! --- "before" guards --- +! +! Guard level 0 before: j == lb_j AND k == lb_k AND l == lb_l (once per i) +! CHECK: %[[CJ1:.*]] = arith.cmpi eq, %[[J]], %[[LB_J]] : i32 +! CHECK: %[[CK1:.*]] = arith.cmpi eq, %[[K]], %[[LB_K]] : i32 +! CHECK: %[[A1:.*]] = arith.andi %[[CJ1]], %[[CK1]] : i1 +! CHECK: %[[CL1:.*]] = arith.cmpi eq, %[[L]], %[[LB_L]] : i32 +! CHECK: %[[A2:.*]] = arith.andi %[[A1]], %[[CL1]] : i1 +! CHECK: fir.if %[[A2]] { +! CHECK: arith.addi +! CHECK: hlfir.assign +! CHECK: } +! Guard level 1 before: k == lb_k AND l == lb_l (once per (i,j)) +! CHECK: %[[CK2:.*]] = arith.cmpi eq, %[[K]], %[[LB_K]] : i32 +! CHECK: %[[CL2:.*]] = arith.cmpi eq, %[[L]], %[[LB_L]] : i32 +! CHECK: %[[A3:.*]] = arith.andi %[[CK2]], %[[CL2]] : i1 +! CHECK: fir.if %[[A3]] { +! CHECK: arith.addi +! CHECK: hlfir.assign +! CHECK: } +! Level 2 (k->l) is perfectly nested: no guard emitted. +! +! --- innermost body: x = x + l --- +! CHECK: arith.addi +! CHECK: hlfir.assign +! +! --- "after" guards (innermost first) --- +! +! Level 2 after: empty (perfectly nested), no guard emitted. +! Level 1 after: empty, no guard emitted. +! Guard level 0 after: j == ub_j AND k == ub_k AND l == ub_l (once per i) +! CHECK: %[[CJ2:.*]] = arith.cmpi eq, %[[J]], %[[UB_J]] : i32 +! CHECK: %[[CK3:.*]] = arith.cmpi eq, %[[K]], %[[UB_K]] : i32 +! CHECK: %[[A4:.*]] = arith.andi %[[CJ2]], %[[CK3]] : i1 +! CHECK: %[[CL3:.*]] = arith.cmpi eq, %[[L]], %[[UB_L]] : i32 +! CHECK: %[[A5:.*]] = arith.andi %[[A4]], %[[CL3]] : i1 +! CHECK: fir.if %[[A5]] { +! CHECK: arith.subi +! CHECK: hlfir.assign +! CHECK: } +! CHECK: omp.yield + +! Test collapse(2) with only after-code (no before-code). Exercises the path +! where levels[i].before.empty() is true and the "before" loop is entirely skipped. +! CHECK-LABEL: func.func @_QPcollapse2_after_only +subroutine collapse2_after_only(n, x) + integer, intent(in) :: n + integer, intent(inout) :: x + integer :: i, j + + !$omp do collapse(2) + do i = 1, n + do j = 1, n + x = x + j + end do + x = x - i + end do + !$omp end do +end subroutine + +! CHECK: omp.wsloop +! CHECK-NEXT: omp.loop_nest (%[[I:.*]], %[[J:.*]]) : i32 = +! CHECK-SAME: (%{{[^)]*}}, %{{[^)]*}}) to (%{{[^)]*}}, %[[UB_J:[^)]*]]) +! CHECK: hlfir.assign %[[I]] +! CHECK: hlfir.assign %[[J]] +! No "before" guard emitted (level 0 before is empty). +! Innermost body: x = x + j +! CHECK: arith.addi +! CHECK: hlfir.assign +! Guard: j == upper_bound (after code) +! CHECK: %[[CMP:.*]] = arith.cmpi eq, %[[J]], %[[UB_J]] : i32 +! CHECK: fir.if %[[CMP]] { +! CHECK: arith.subi +! CHECK: hlfir.assign +! CHECK: } +! CHECK: omp.yield + +! Test collapse(2) with multiple statements inside a single guard. Verifies +! that all evals in level.before land inside the same fir.if block. +! CHECK-LABEL: func.func @_QPcollapse2_multi_stmt +subroutine collapse2_multi_stmt(n, x) + integer, intent(in) :: n + integer, intent(inout) :: x + integer :: i, j + + !$omp do collapse(2) + do i = 1, n + x = x + 1 + x = x + i + do j = 1, n + x = x + j + end do + end do + !$omp end do +end subroutine + +! CHECK: omp.wsloop +! CHECK-NEXT: omp.loop_nest (%[[I:.*]], %[[J:.*]]) : i32 = +! CHECK-SAME: (%{{[^)]*}}, %[[LB_J:[^)]*]]) to +! CHECK: hlfir.assign %[[I]] +! CHECK: hlfir.assign %[[J]] +! Guard: j == lower_bound (before code, multiple statements in one guard) +! CHECK: %[[CMP:.*]] = arith.cmpi eq, %[[J]], %[[LB_J]] : i32 +! CHECK: fir.if %[[CMP]] { +! First intervening statement: x = x + 1 +! CHECK: arith.addi +! CHECK: hlfir.assign +! Second intervening statement: x = x + i +! CHECK: arith.addi +! CHECK: hlfir.assign +! CHECK: } +! Innermost body: x = x + j +! CHECK: arith.addi +! CHECK: hlfir.assign +! CHECK: omp.yield + +! Test collapse(2) with non-unit lower bound on inner loop. Verifies the guard +! compares against the actual loop lower bound operand (3, not 1). +! CHECK-LABEL: func.func @_QPcollapse2_nonunit_lb +subroutine collapse2_nonunit_lb(n, x) + integer, intent(in) :: n + integer, intent(inout) :: x + integer :: i, j + + !$omp do collapse(2) + do i = 1, n + x = x + i + do j = 3, n + x = x + j + end do + end do + !$omp end do +end subroutine + +! CHECK: omp.wsloop +! CHECK-NEXT: omp.loop_nest (%[[I:.*]], %[[J:.*]]) : i32 = +! CHECK-SAME: (%{{[^)]*}}, %[[LB_J:[^)]*]]) to +! CHECK: hlfir.assign %[[I]] +! CHECK: hlfir.assign %[[J]] +! Guard: j == lb_j (lb_j is 3, not 1) +! CHECK: %[[CMP:.*]] = arith.cmpi eq, %[[J]], %[[LB_J]] : i32 +! CHECK: fir.if %[[CMP]] { +! CHECK: arith.addi +! CHECK: hlfir.assign +! CHECK: } +! Innermost body: x = x + j +! CHECK: arith.addi +! CHECK: hlfir.assign +! CHECK: omp.yield + +! Test collapse(3) with after-only at level 0 and before-only at level 1. +! Exercises the independent skip logic at each level in both emission loops. +! CHECK-LABEL: func.func @_QPcollapse3_mixed_sides +subroutine collapse3_mixed_sides(n, x) + integer, intent(in) :: n + integer, intent(inout) :: x + integer :: i, j, k + + !$omp do collapse(3) + do i = 1, n + do j = 1, n + x = x + j + do k = 1, n + x = x + k + end do + end do + x = x - i + end do + !$omp end do +end subroutine + +! CHECK: omp.wsloop +! CHECK-NEXT: omp.loop_nest (%[[I:.*]], %[[J:.*]], %[[K:.*]]) : i32 = +! CHECK-SAME: (%{{[^)]*}}, %{{[^)]*}}, %[[LB_K:[^)]*]]) to (%{{[^)]*}}, %[[UB_J:[^)]*]], %[[UB_K:[^)]*]]) +! CHECK: hlfir.assign %[[I]] +! CHECK: hlfir.assign %[[J]] +! CHECK: hlfir.assign %[[K]] +! Level 0 before: empty (skipped). +! Guard level 1 before: k == lb_k (once per (i,j)) +! CHECK: %[[CK:.*]] = arith.cmpi eq, %[[K]], %[[LB_K]] : i32 +! CHECK: fir.if %[[CK]] { +! CHECK: arith.addi +! CHECK: hlfir.assign +! CHECK: } +! Innermost body: x = x + k +! CHECK: arith.addi +! CHECK: hlfir.assign +! Level 1 after: empty (skipped). +! Guard level 0 after: j == ub_j AND k == ub_k (once per i) +! CHECK: %[[CJ:.*]] = arith.cmpi eq, %[[J]], %[[UB_J]] : i32 +! CHECK: %[[CK2:.*]] = arith.cmpi eq, %[[K]], %[[UB_K]] : i32 +! CHECK: %[[AND:.*]] = arith.andi %[[CJ]], %[[CK2]] : i1 +! CHECK: fir.if %[[AND]] { +! CHECK: arith.subi +! CHECK: hlfir.assign +! CHECK: } +! CHECK: omp.yield + +! Test collapse(2) with non-unit positive step and after-code. +! The after guard must compare iv against the last executed value +! (lb + ((ub - lb) / step) * step), not the upper bound directly. +! For do j = 1, 10, 4: last_iv = 1 + ((10-1)/4)*4 = 1 + 8 = 9. +! CHECK-LABEL: func.func @_QPcollapse2_nonunit_step_after +subroutine collapse2_nonunit_step_after(n, x) + integer, intent(in) :: n + integer, intent(inout) :: x + integer :: i, j + + !$omp do collapse(2) + do i = 1, n + do j = 1, 10, 4 + x = x + j + end do + x = x - i + end do + !$omp end do +end subroutine + +! CHECK: omp.wsloop +! CHECK-NEXT: omp.loop_nest (%[[I:.*]], %[[J:.*]]) : i32 = +! CHECK-SAME: (%{{[^)]*}}, %[[LB_J:[^)]*]]) to (%{{[^)]*}}, %[[UB_J:[^)]*]]) inclusive step (%{{[^)]*}}, %[[STEP_J:[^)]*]]) +! CHECK: hlfir.assign %[[I]] +! CHECK: hlfir.assign %[[J]] +! Innermost body: x = x + j +! CHECK: arith.addi +! CHECK: hlfir.assign +! After guard: compute last_iv = lb + ((ub - lb) / step) * step +! CHECK: %[[RANGE:.*]] = arith.subi %[[UB_J]], %[[LB_J]] : i32 +! CHECK: %[[DIV:.*]] = arith.divsi %[[RANGE]], %[[STEP_J]] : i32 +! CHECK: %[[MUL:.*]] = arith.muli %[[DIV]], %[[STEP_J]] : i32 +! CHECK: %[[LAST:.*]] = arith.addi %[[LB_J]], %[[MUL]] : i32 +! CHECK: %[[CMP:.*]] = arith.cmpi eq, %[[J]], %[[LAST]] : i32 +! CHECK: fir.if %[[CMP]] { +! CHECK: arith.subi +! CHECK: hlfir.assign +! CHECK: } +! CHECK: omp.yield + +! Test collapse(2) with negative step and after-code. +! For do j = 10, 1, -4: last_iv = 10 + ((1-10)/(-4))*(-4) = 10 + (2*-4) = 2. +! CHECK-LABEL: func.func @_QPcollapse2_negative_step_after +subroutine collapse2_negative_step_after(n, x) + integer, intent(in) :: n + integer, intent(inout) :: x + integer :: i, j + + !$omp do collapse(2) + do i = 1, n + do j = 10, 1, -4 + x = x + j + end do + x = x - i + end do + !$omp end do +end subroutine + +! CHECK: omp.wsloop +! CHECK-NEXT: omp.loop_nest (%[[I:.*]], %[[J:.*]]) : i32 = +! CHECK-SAME: (%{{[^)]*}}, %[[LB_J:[^)]*]]) to (%{{[^)]*}}, %[[UB_J:[^)]*]]) inclusive step (%{{[^)]*}}, %[[STEP_J:[^)]*]]) +! CHECK: hlfir.assign %[[I]] +! CHECK: hlfir.assign %[[J]] +! Innermost body: x = x + j +! CHECK: arith.addi +! CHECK: hlfir.assign +! After guard: compute last_iv for negative step +! CHECK: %[[RANGE:.*]] = arith.subi %[[UB_J]], %[[LB_J]] : i32 +! CHECK: %[[DIV:.*]] = arith.divsi %[[RANGE]], %[[STEP_J]] : i32 +! CHECK: %[[MUL:.*]] = arith.muli %[[DIV]], %[[STEP_J]] : i32 +! CHECK: %[[LAST:.*]] = arith.addi %[[LB_J]], %[[MUL]] : i32 +! CHECK: %[[CMP:.*]] = arith.cmpi eq, %[[J]], %[[LAST]] : i32 +! CHECK: fir.if %[[CMP]] { +! CHECK: arith.subi +! CHECK: hlfir.assign +! CHECK: } +! CHECK: omp.yield + +! Test collapse(3) with non-unit step on the middle loop (not innermost). +! For do j = 1, n, 3: last_iv = lb + ((ub - lb) / step) * step (runtime). +! CHECK-LABEL: func.func @_QPcollapse3_nonunit_step_middle +subroutine collapse3_nonunit_step_middle(n, x) + integer, intent(in) :: n + integer, intent(inout) :: x + integer :: i, j, k + + !$omp do collapse(3) + do i = 1, n + do j = 1, n, 3 + x = x + j + do k = 1, n + x = x + k + end do + end do + x = x - i + end do + !$omp end do +end subroutine + +! CHECK: omp.wsloop +! CHECK-NEXT: omp.loop_nest (%[[I:.*]], %[[J:.*]], %[[K:.*]]) : i32 = +! CHECK-SAME: (%{{[^)]*}}, %[[LB_J:[^)]*]], %[[LB_K:[^)]*]]) to (%{{[^)]*}}, %[[UB_J:[^)]*]], %[[UB_K:[^)]*]]) inclusive step (%{{[^)]*}}, %[[STEP_J:[^)]*]], %{{[^)]*}}) +! CHECK: hlfir.assign %[[I]] +! CHECK: hlfir.assign %[[J]] +! CHECK: hlfir.assign %[[K]] +! Guard level 1 before: k == lb_k (once per (i,j)) +! CHECK: %[[CK1:.*]] = arith.cmpi eq, %[[K]], %[[LB_K]] : i32 +! CHECK: fir.if %[[CK1]] { +! CHECK: arith.addi +! CHECK: hlfir.assign +! CHECK: } +! Innermost body: x = x + k +! CHECK: arith.addi +! CHECK: hlfir.assign +! Guard level 0 after: must compute last_iv for j (non-unit step) AND k == ub_k +! CHECK: %[[RANGE:.*]] = arith.subi %[[UB_J]], %[[LB_J]] : i32 +! CHECK: %[[DIV:.*]] = arith.divsi %[[RANGE]], %[[STEP_J]] : i32 +! CHECK: %[[MUL:.*]] = arith.muli %[[DIV]], %[[STEP_J]] : i32 +! CHECK: %[[LASTJ:.*]] = arith.addi %[[LB_J]], %[[MUL]] : i32 +! CHECK: %[[CJ:.*]] = arith.cmpi eq, %[[J]], %[[LASTJ]] : i32 +! CHECK: %[[CK2:.*]] = arith.cmpi eq, %[[K]], %[[UB_K]] : i32 +! CHECK: %[[AND:.*]] = arith.andi %[[CJ]], %[[CK2]] : i1 +! CHECK: fir.if %[[AND]] { +! CHECK: arith.subi +! CHECK: hlfir.assign +! CHECK: } +! CHECK: omp.yield + +! Test collapse(2) with a dynamic (runtime) step value. +! The step is not a compile-time constant, so the last_iv computation +! cannot be folded away and must remain as arith ops in the IR. +! CHECK-LABEL: func.func @_QPcollapse2_dynamic_step_after +subroutine collapse2_dynamic_step_after(n, s, x) + integer, intent(in) :: n, s + integer, intent(inout) :: x + integer :: i, j + + !$omp do collapse(2) + do i = 1, n + do j = 1, n, s + x = x + j + end do + x = x - i + end do + !$omp end do +end subroutine + +! CHECK: omp.wsloop +! CHECK-NEXT: omp.loop_nest (%[[I:.*]], %[[J:.*]]) : i32 = +! CHECK-SAME: (%{{[^)]*}}, %[[LB_J:[^)]*]]) to (%{{[^)]*}}, %[[UB_J:[^)]*]]) inclusive step (%{{[^)]*}}, %[[STEP_J:[^)]*]]) +! CHECK: hlfir.assign %[[I]] +! CHECK: hlfir.assign %[[J]] +! Innermost body: x = x + j +! CHECK: arith.addi +! CHECK: hlfir.assign +! After guard: dynamic step forces last_iv computation to stay in IR +! CHECK: %[[RANGE:.*]] = arith.subi %[[UB_J]], %[[LB_J]] : i32 +! CHECK: %[[DIV:.*]] = arith.divsi %[[RANGE]], %[[STEP_J]] : i32 +! CHECK: %[[MUL:.*]] = arith.muli %[[DIV]], %[[STEP_J]] : i32 +! CHECK: %[[LAST:.*]] = arith.addi %[[LB_J]], %[[MUL]] : i32 +! CHECK: %[[CMP:.*]] = arith.cmpi eq, %[[J]], %[[LAST]] : i32 +! CHECK: fir.if %[[CMP]] { +! CHECK: arith.subi +! CHECK: hlfir.assign +! CHECK: } +! CHECK: omp.yield + +! Test that "after" code reading an inner DO variable sees the Fortran +! terminal value (lb + tripcount*step, i.e. one past the last executed +! value), not the last executed value the flattened nest leaves it at. +! The after guard must restore j to ub + step before the after code runs. +! CHECK-LABEL: func.func @_QPcollapse2_after_reads_inner +subroutine collapse2_after_reads_inner(n, x) + integer, intent(in) :: n + integer, intent(inout) :: x + integer :: i, j + + !$omp do collapse(2) + do i = 1, n + do j = 1, n + x = x + 1 + end do + x = x + j + end do + !$omp end do +end subroutine + +! CHECK: omp.wsloop +! CHECK-NEXT: omp.loop_nest (%[[I:.*]], %[[J:.*]]) : i32 = +! CHECK-SAME: (%{{[^)]*}}, %{{[^)]*}}) to (%{{[^)]*}}, %[[UB_J:[^)]*]]) inclusive step (%{{[^)]*}}, %[[STEP_J:[^)]*]]) +! CHECK: hlfir.assign %[[I]] +! Capture j's storage from the loop-variable store so the restore and the +! after-code read can be tied to the same address. +! CHECK: hlfir.assign %[[J]] to %[[J_ADDR:.*]] : i32, !fir.ref +! Innermost body: x = x + 1 +! CHECK: arith.addi +! CHECK: hlfir.assign +! Guard: j == ub_j (unit-step fast path compares against the upper bound) +! CHECK: %[[CMP:.*]] = arith.cmpi eq, %[[J]], %[[UB_J]] : i32 +! CHECK: fir.if %[[CMP]] { +! Restore j's storage to its Fortran terminal value (ub + step). +! CHECK: %[[TERM:.*]] = arith.addi %[[UB_J]], %[[STEP_J]] : i32 +! CHECK: hlfir.assign %[[TERM]] to %[[J_ADDR]] : i32, !fir.ref +! After code reads the restored j: x = x + j. +! CHECK: %[[XLD:.*]] = fir.load %{{.*}} : !fir.ref +! CHECK: %[[JLD:.*]] = fir.load %[[J_ADDR]] : !fir.ref +! CHECK: arith.addi %[[XLD]], %[[JLD]] : i32 +! CHECK: hlfir.assign +! CHECK: } +! CHECK: omp.yield diff --git a/flang/test/Lower/OpenMP/collapse-labeled-do-host-eval.f90 b/flang/test/Lower/OpenMP/collapse-labeled-do-host-eval.f90 new file mode 100644 index 0000000000000..f731f39eaadc4 --- /dev/null +++ b/flang/test/Lower/OpenMP/collapse-labeled-do-host-eval.f90 @@ -0,0 +1,45 @@ +! Regression test: a collapsed loop nest written with labeled DO loops must +! lower successfully when the loop bounds are host-evaluated under omp.target. +! +! A labeled DO loop leaves a no-op ContinueStmt for its terminating labeled +! statement (in addition to the EndDoStmt). That CONTINUE must not be treated +! as intervening code by the collapsed-loop-nest lowering: doing so emitted a +! guarded "after" region containing arithmetic (arith.addi) on the loop upper +! bound, which is an illegal use of an omp.target host_eval block argument and +! tripped the MLIR verifier ("host_eval argument illegal use in 'arith.addi'"). +! +! Before the fix, each of the three target regions below failed to verify. + +! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s + +! CHECK-LABEL: func.func @_QPinital +subroutine inital(n, m) + implicit none + integer, intent(in) :: n, m + integer :: i, j + + ! Exercises the labeled-DO collapsed-nest lowering path; verified via its + ! collapsed loop_nest below. + ! CHECK: omp.loop_nest {{.*}} collapse(2) + !$omp target parallel do collapse(2) + do 10 j = 1, n + do 11 i = 1, m + 11 end do + 10 end do + + ! CHECK: omp.target {{.*}}host_eval( + ! CHECK: omp.loop_nest {{.*}} collapse(2) + !$omp target teams distribute parallel do collapse(2) + do 20 j = 1, n + do 21 i = 1, m + 21 end do + 20 end do + + ! CHECK: omp.target {{.*}}host_eval( + ! CHECK: omp.loop_nest {{.*}} collapse(2) + !$omp target teams distribute collapse(2) + do 30 j = 1, n + do 31 i = 1, m + 31 end do + 30 end do +end subroutine inital diff --git a/flang/test/Lower/OpenMP/collapse-loop-transform.f90 b/flang/test/Lower/OpenMP/collapse-loop-transform.f90 new file mode 100644 index 0000000000000..c394ed9f53d21 --- /dev/null +++ b/flang/test/Lower/OpenMP/collapse-loop-transform.f90 @@ -0,0 +1,21 @@ +! COLLAPSE over a nested loop-transforming construct (e.g. TILE) is accepted by +! semantics but not yet supported in lowering. Check for a "not yet +! implemented" message instead of a compiler crash. + +! RUN: not %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=60 -o - %s 2>&1 | FileCheck %s + +subroutine collapse_tile(n, x) + integer, intent(in) :: n + integer, intent(inout) :: x + integer :: i, j + + !$omp do collapse(4) + !$omp tile sizes(2, 2) + do i = 1, n + do j = 1, n + x = x + i + j + end do + end do +end subroutine + +! CHECK: not yet implemented: Collapsing a loop nest that contains a loop-transforming construct diff --git a/flang/test/Lower/OpenMP/collapse-target-intervening-todo.f90 b/flang/test/Lower/OpenMP/collapse-target-intervening-todo.f90 new file mode 100644 index 0000000000000..48072802db9ff --- /dev/null +++ b/flang/test/Lower/OpenMP/collapse-target-intervening-todo.f90 @@ -0,0 +1,28 @@ +! Regression test: a collapsed imperfect loop nest with genuine intervening +! code, whose loop bounds are host-evaluated for an enclosing omp.target SPMD +! region, is not yet supported and must be diagnosed cleanly rather than +! producing IR that fails the omp.target verifier. +! +! The intervening statement "x = x + j" runs at intermediate nest level. Its +! guard and terminal-IV restoration would perform arith.cmpi/arith.addi on the +! collapsed loop bounds, which are omp.target host_eval block arguments in this +! context -- an illegal use. Until that path is implemented, emit a "not yet +! implemented" message. + +! RUN: not %flang_fc1 -emit-hlfir -fopenmp %s 2>&1 | FileCheck %s + +! CHECK: not yet implemented: collapsed loop nest with intervening code whose loop bounds are evaluated on the host for an enclosing 'target' region +subroutine repro(n, m, x) + implicit none + integer, intent(in) :: n, m + integer, intent(inout) :: x + integer :: i, j + + !$omp target teams distribute parallel do collapse(2) map(tofrom:x) + do i = 1, n + do j = 1, m + x = x + 1 + end do + x = x + j + end do +end subroutine diff --git a/flang/test/Semantics/OpenMP/do-collapse.f90 b/flang/test/Semantics/OpenMP/do-collapse.f90 index 2804ae00ad7be..9148f6558b000 100644 --- a/flang/test/Semantics/OpenMP/do-collapse.f90 +++ b/flang/test/Semantics/OpenMP/do-collapse.f90 @@ -3,7 +3,7 @@ ! 2.7.1 Collapse Clause program omp_doCollapse integer:: i,j - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 + !ERROR: This construct requires a nest of depth 3, but the associated nest is a nest of depth 2 !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp do collapse(3) do i = 1,10 @@ -15,7 +15,7 @@ program omp_doCollapse do i = 1,10 do j = 1, 10 - !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 + !ERROR: This construct requires a nest of depth 2, but the associated nest is a nest of depth 1 !BECAUSE: COLLAPSE clause was specified with argument 2 !$omp do collapse(2) do k = 1, 10 @@ -25,7 +25,7 @@ program omp_doCollapse end do end do - !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 + !ERROR: This construct requires a nest of depth 2, but the associated nest is a nest of depth 1 !BECAUSE: COLLAPSE clause was specified with argument 2 !$omp parallel do collapse(2) do i = 1, 3 @@ -35,7 +35,7 @@ program omp_doCollapse end do end do - !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 + !ERROR: This construct requires a nest of depth 2, but the associated nest is a nest of depth 1 !BECAUSE: COLLAPSE clause was specified with argument 2 !ERROR: At most one COLLAPSE clause can appear on SIMD directive !$omp simd collapse(2) collapse(1) diff --git a/flang/test/Semantics/OpenMP/do-concurrent-collapse-60.f90 b/flang/test/Semantics/OpenMP/do-concurrent-collapse-60.f90 index 4ba8e71a26323..75351cdf7190c 100644 --- a/flang/test/Semantics/OpenMP/do-concurrent-collapse-60.f90 +++ b/flang/test/Semantics/OpenMP/do-concurrent-collapse-60.f90 @@ -3,7 +3,7 @@ subroutine f integer :: i - !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 + !ERROR: This construct requires a nest of depth 2, but the associated nest is a nest of depth 1 !BECAUSE: COLLAPSE clause was specified with argument 2 !$omp parallel do collapse(2) do i = 1, 1 @@ -42,7 +42,7 @@ subroutine f print *, j end do - !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 + !ERROR: This construct requires a nest of depth 2, but the associated nest is a nest of depth 1 !BECAUSE: COLLAPSE clause was specified with argument 2 !$omp loop collapse(2) do i = 1, 1 diff --git a/flang/test/Semantics/OpenMP/do-concurrent-collapse.f90 b/flang/test/Semantics/OpenMP/do-concurrent-collapse.f90 index 7cd10518d845a..0a6906b0e2e63 100644 --- a/flang/test/Semantics/OpenMP/do-concurrent-collapse.f90 +++ b/flang/test/Semantics/OpenMP/do-concurrent-collapse.f90 @@ -1,7 +1,7 @@ !RUN: %python %S/../test_errors.py %s %flang -fopenmp integer :: i, j -! ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 +! ERROR: This construct requires a nest of depth 2, but the associated nest is a nest of depth 1 ! BECAUSE: COLLAPSE clause was specified with argument 2 !$omp parallel do collapse(2) do i = 1, 1 @@ -33,7 +33,7 @@ print *, j end do -! ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 +! ERROR: This construct requires a nest of depth 2, but the associated nest is a nest of depth 1 ! BECAUSE: COLLAPSE clause was specified with argument 2 !$omp loop collapse(2) do i = 1, 1 diff --git a/flang/test/Semantics/OpenMP/do08.f90 b/flang/test/Semantics/OpenMP/do08.f90 index 300485f067b1d..24755022f0296 100644 --- a/flang/test/Semantics/OpenMP/do08.f90 +++ b/flang/test/Semantics/OpenMP/do08.f90 @@ -7,11 +7,8 @@ program omp logical cond(10,10,10) cond = .false. - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 1 - !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp do collapse(3) do i = 0, 10 - !BECAUSE: This code prevents perfect nesting !ERROR: CYCLE statement to non-innermost associated loop of an OpenMP DO construct if (i .lt. 1) cycle do j = 0, 10 @@ -22,12 +19,9 @@ program omp end do !$omp end do - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 - !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp do collapse(3) do i = 0, 10 do j = 0, 10 - !BECAUSE: This code prevents perfect nesting !ERROR: CYCLE statement to non-innermost associated loop of an OpenMP DO construct if (i .lt. 1) cycle do k = 0, 10 @@ -37,11 +31,8 @@ program omp end do !$omp end do - !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 - !BECAUSE: COLLAPSE clause was specified with argument 2 !$omp do collapse(2) do i = 0, 10 - !BECAUSE: This code prevents perfect nesting !ERROR: CYCLE statement to non-innermost associated loop of an OpenMP DO construct if (i .lt. 1) cycle do j = 0, 10 @@ -53,11 +44,8 @@ program omp !$omp end do - !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 - !BECAUSE: COLLAPSE clause was specified with argument 2 !$omp do collapse(2) foo: do i = 0, 10 - !BECAUSE: This code prevents perfect nesting !ERROR: CYCLE statement to non-innermost associated loop of an OpenMP DO construct if (i .lt. 1) cycle foo do j = 0, 10 @@ -69,12 +57,9 @@ program omp !$omp end do - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 - !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp do collapse(3) do 60 i=2,200,2 do j=1,10 - !BECAUSE: This code prevents perfect nesting !ERROR: CYCLE statement to non-innermost associated loop of an OpenMP DO construct if (i == 100) cycle do k = 1, 10 diff --git a/flang/test/Semantics/OpenMP/do10.f90 b/flang/test/Semantics/OpenMP/do10.f90 index 1878864a4a5db..1fae5e098dc56 100644 --- a/flang/test/Semantics/OpenMP/do10.f90 +++ b/flang/test/Semantics/OpenMP/do10.f90 @@ -14,7 +14,7 @@ program omp_do end do !$omp end do - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 + !ERROR: This construct requires a nest of depth 3, but the associated nest is a nest of depth 2 !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp do collapse(3) do i = 1, 10 diff --git a/flang/test/Semantics/OpenMP/do13.f90 b/flang/test/Semantics/OpenMP/do13.f90 index 6d5e799e951b0..dd9f4be03e7e1 100644 --- a/flang/test/Semantics/OpenMP/do13.f90 +++ b/flang/test/Semantics/OpenMP/do13.f90 @@ -5,7 +5,7 @@ program omp integer i, j, k - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 1 + !ERROR: This construct requires a nest of depth 3, but the associated nest is a nest of depth 1 !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp do collapse(3) do i = 0, 10 @@ -20,7 +20,7 @@ program omp end do !$omp end do - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 + !ERROR: This construct requires a nest of depth 3, but the associated nest is a nest of depth 2 !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp do collapse(3) do i = 0, 10 @@ -35,7 +35,7 @@ program omp end do !$omp end do - !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 + !ERROR: This construct requires a nest of depth 2, but the associated nest is a nest of depth 1 !BECAUSE: COLLAPSE clause was specified with argument 2 !$omp do collapse(2) do i = 0, 10 @@ -51,7 +51,7 @@ program omp !$omp end do - !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 + !ERROR: This construct requires a nest of depth 2, but the associated nest is a nest of depth 1 !BECAUSE: COLLAPSE clause was specified with argument 2 !$omp do collapse(2) foo: do i = 0, 10 @@ -67,7 +67,7 @@ program omp !$omp end do - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 + !ERROR: This construct requires a nest of depth 3, but the associated nest is a nest of depth 2 !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp do collapse(3) do 60 i=1,10 diff --git a/flang/test/Semantics/OpenMP/do15.f90 b/flang/test/Semantics/OpenMP/do15.f90 index 00baa0c431c5f..96c195b2d1306 100644 --- a/flang/test/Semantics/OpenMP/do15.f90 +++ b/flang/test/Semantics/OpenMP/do15.f90 @@ -5,11 +5,8 @@ program omp integer i, j, k - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 1 - !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp do collapse(3) do i = 0, 10 - !BECAUSE: This code prevents perfect nesting if (i .lt. 1) then !ERROR: CYCLE statement to non-innermost associated loop of an OpenMP DO construct cycle @@ -22,12 +19,9 @@ program omp end do !$omp end do - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 - !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp do collapse(3) do i = 0, 10 do j = 0, 10 - !BECAUSE: This code prevents perfect nesting if (i .lt. 1) then !ERROR: CYCLE statement to non-innermost associated loop of an OpenMP DO construct cycle @@ -56,12 +50,9 @@ program omp !$omp end do - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 - !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp do collapse(3) foo: do i = 0, 10 foo1: do j = 0, 10 - !BECAUSE: This code prevents perfect nesting if (i .lt. 1) then !ERROR: CYCLE statement to non-innermost associated loop of an OpenMP DO construct cycle foo diff --git a/flang/test/Semantics/OpenMP/do16.f90 b/flang/test/Semantics/OpenMP/do16.f90 index 35d94b17f3c68..2a871156de75f 100644 --- a/flang/test/Semantics/OpenMP/do16.f90 +++ b/flang/test/Semantics/OpenMP/do16.f90 @@ -5,11 +5,8 @@ program omp integer i, j, k - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 1 - !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp do collapse(3) do i = 0, 10 - !BECAUSE: This code prevents perfect nesting select case (i) case(1) !ERROR: CYCLE statement to non-innermost associated loop of an OpenMP DO construct @@ -23,12 +20,9 @@ program omp end do !$omp end do - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 - !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp do collapse(3) do i = 0, 10 do j = 0, 10 - !BECAUSE: This code prevents perfect nesting select case (i) case(1) !ERROR: CYCLE statement to non-innermost associated loop of an OpenMP DO construct diff --git a/flang/test/Semantics/OpenMP/do22.f90 b/flang/test/Semantics/OpenMP/do22.f90 index 2ced881a2af8b..818de56aa1093 100644 --- a/flang/test/Semantics/OpenMP/do22.f90 +++ b/flang/test/Semantics/OpenMP/do22.f90 @@ -4,11 +4,9 @@ subroutine do_imperfectly_nested_before integer i, j - !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 - !BECAUSE: COLLAPSE clause was specified with argument 2 + ! Valid: print is allowed as CLN intervening code with collapse !$omp do collapse(2) do i = 1, 10 - !BECAUSE: This code prevents perfect nesting print *, i do j = 1, 10 print *, i, j @@ -21,15 +19,134 @@ subroutine do_imperfectly_nested_before subroutine do_imperfectly_nested_behind integer i, j - !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 - !BECAUSE: COLLAPSE clause was specified with argument 2 + ! Valid: print is allowed as CLN intervening code with collapse + !$omp do collapse(2) + do i = 1, 10 + do j = 1, 10 + print *, i, j + end do + print *, i + end do + !$omp end do +end subroutine + + +subroutine do_imperfectly_nested_scalar_assign + integer i, j, x + + ! Valid: scalar assignment is allowed as CLN intervening code with collapse + !$omp do collapse(2) + do i = 1, 10 + x = i + 1 + do j = 1, 10 + print *, i, j, x + end do + end do + !$omp end do +end subroutine + + +subroutine do_imperfectly_nested_call + integer i, j + + ! Valid: subroutine call is allowed as CLN intervening code with collapse !$omp do collapse(2) do i = 1, 10 + call sub(i) + do j = 1, 10 + print *, i, j + end do + end do + !$omp end do +end subroutine + + +subroutine do_imperfectly_nested_multiple + integer i, j, x + + ! Valid: multiple scalar statements are allowed as CLN intervening code + !$omp do collapse(2) + do i = 1, 10 + x = i * 2 + print *, x + call sub(x) + do j = 1, 10 + print *, i, j + end do + end do + !$omp end do +end subroutine + + +subroutine do_imperfect_collapse_bare_ordered + integer i, j, x + + ! Valid: bare ORDERED does not require a perfect nest. + !$omp do collapse(2) ordered + do i = 1, 10 + x = 0 + do j = 1, 10 + !$omp ordered + print *, i, j, x + !$omp end ordered + end do + end do + !$omp end do +end subroutine + + +subroutine do_imperfect_ordered_requires_perfect + integer i, j + + ! ordered(2) still requires perfect nesting at default OpenMP version + !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 + !BECAUSE: ORDERED clause was specified with argument 2 + !$omp do ordered(2) + do i = 1, 10 + !BECAUSE: This code prevents perfect nesting + print *, i do j = 1, 10 print *, i, j end do + end do + !$omp end do +end subroutine + + +subroutine do_imperfect_collapse_ordered_requires_perfect + integer i, j, k + + ! collapse(2) ordered(3) requires perfect nesting at default OpenMP version because ordered(3) > collapse(2) + !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 1 + !BECAUSE: ORDERED clause was specified with argument 3 + !$omp do collapse(2) ordered(3) + do i = 1, 10 !BECAUSE: This code prevents perfect nesting print *, i + do j = 1, 10 + do k = 1, 10 + print *, i, j, k + end do + end do + end do + !$omp end do +end subroutine + + +subroutine do_imperfect_array_assign_invalid + integer i, j + integer :: a(10) + + ! Array assignment is invalid CLN intervening code + !ERROR: This construct requires a nest of depth 2, but the associated nest is a nest of depth 1 + !BECAUSE: COLLAPSE clause was specified with argument 2 + !$omp do collapse(2) + do i = 1, 10 + !BECAUSE: The nest contains code that prevents it from being canonical at this nesting level + a(:) = 0 + do j = 1, 10 + print *, i, j + end do end do !$omp end do end subroutine diff --git a/flang/test/Semantics/OpenMP/doacross-nesting-omp60.f90 b/flang/test/Semantics/OpenMP/doacross-nesting-omp60.f90 new file mode 100644 index 0000000000000..d11be5a60e072 --- /dev/null +++ b/flang/test/Semantics/OpenMP/doacross-nesting-omp60.f90 @@ -0,0 +1,148 @@ +! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=60 +! OpenMP 6.0: doacross loop nests (those with ordered doacross(sink/source) +! constructs) require perfect nesting. + +! ordered(2) without doacross directives: imperfect nesting is valid in 6.0. +subroutine ordered_no_doacross_imperfect + integer i, j + + !$omp do ordered(2) + do i = 1, 10 + print *, i + do j = 1, 10 + print *, i, j + end do + end do + !$omp end do +end subroutine + +! Bare ORDERED (no argument) carries no doacross intent and does not require +! perfect nesting. +subroutine bare_ordered_no_doacross + integer i, j, x + + !$omp do ordered + do i = 1, 10 + x = 0 + do j = 1, 10 + !$omp ordered + print *, i, j, x + !$omp end ordered + end do + end do + !$omp end do +end subroutine + +! Perfectly nested doacross: valid. +subroutine doacross_perfect + integer i, j + + !$omp do ordered(2) + do i = 1, 10 + do j = 1, 10 + !$omp ordered doacross(sink: i-1, j) + print *, i, j + !$omp ordered doacross(source) + end do + end do + !$omp end do +end subroutine + +! Imperfectly nested doacross: invalid in 6.0. +subroutine doacross_imperfect + integer i, j + + !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 + !BECAUSE: ORDERED clause was specified with argument 2 + !$omp do ordered(2) + do i = 1, 10 + !BECAUSE: This code prevents perfect nesting + print *, i + do j = 1, 10 + !$omp ordered doacross(sink: i-1, j) + print *, i, j + !$omp ordered doacross(source) + end do + end do + !$omp end do +end subroutine + +! collapse(2) ordered(3) without doacross: imperfect nesting is valid. +subroutine collapse_ordered_no_doacross_imperfect + integer i, j, k + + !$omp do collapse(2) ordered(3) + do i = 1, 10 + print *, i + do j = 1, 10 + do k = 1, 10 + print *, i, j, k + end do + end do + end do + !$omp end do +end subroutine + +! Doacross with collapse: ordered(N) controls depth when N > collapse. +subroutine doacross_collapse + integer i, j, k + + !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 1 + !BECAUSE: ORDERED clause was specified with argument 3 + !$omp do collapse(2) ordered(3) + do i = 1, 10 + !BECAUSE: This code prevents perfect nesting + print *, i + do j = 1, 10 + do k = 1, 10 + !$omp ordered doacross(sink: i-1, j, k) + print *, i, j, k + !$omp ordered doacross(source) + end do + end do + end do + !$omp end do +end subroutine + +! ORDERED with no argument: the number of doacross-affected loops defaults to +! the COLLAPSE value (2 here). An imperfectly nested doacross is therefore +! invalid, and the diagnostic is keyed off COLLAPSE rather than ORDERED. +subroutine doacross_collapse_ordered_default + integer i, j + + !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 + !BECAUSE: COLLAPSE clause was specified with argument 2 + !$omp do collapse(2) ordered + do i = 1, 10 + !BECAUSE: This code prevents perfect nesting + print *, i + do j = 1, 10 + !$omp ordered doacross(sink: i-1, j) + print *, i, j + !$omp ordered doacross(source) + end do + end do + !$omp end do +end subroutine + +! Doacross inside a nested OpenMP region should not force perfect nesting on +! the outer loop. The doacross binds to the inner loop, not the outer one. +subroutine doacross_in_nested_region + integer i, j, k + + !$omp do collapse(2) + do i = 1, 10 + print *, i + do j = 1, 10 + !$omp parallel + !$omp do ordered(1) + do k = 1, 10 + !$omp ordered doacross(source) + print *, k + end do + !$omp end do + !$omp end parallel + end do + end do + !$omp end do +end subroutine diff --git a/flang/test/Semantics/OpenMP/metadirective-loop-applicability.f90 b/flang/test/Semantics/OpenMP/metadirective-loop-applicability.f90 index b90dc17fbbdf4..0665ab304aea5 100644 --- a/flang/test/Semantics/OpenMP/metadirective-loop-applicability.f90 +++ b/flang/test/Semantics/OpenMP/metadirective-loop-applicability.f90 @@ -16,7 +16,7 @@ subroutine f01(n, a) subroutine f02(n, a) integer :: n, a(n, n), i, j - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 + !ERROR: This construct requires a nest of depth 3, but the associated nest is a nest of depth 2 !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp metadirective when(implementation={vendor(llvm)}: do collapse(3)) default(nothing) do i = 1, n @@ -30,7 +30,7 @@ subroutine f02(n, a) subroutine f03(n, a, flag) integer :: n, a(n, n), i, j logical :: flag - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 + !ERROR: This construct requires a nest of depth 3, but the associated nest is a nest of depth 2 !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp metadirective when(user={condition(flag)}: do collapse(3)) default(nothing) do i = 1, n @@ -43,7 +43,7 @@ subroutine f03(n, a, flag) ! A dead WHEN clause must not suppress the unguarded DEFAULT variant. subroutine f04(n, a) integer :: n, a(n, n), i, j - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 + !ERROR: This construct requires a nest of depth 3, but the associated nest is a nest of depth 2 !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp metadirective when(device={kind(nohost)}: nothing) default(do collapse(3)) do i = 1, n @@ -71,7 +71,7 @@ subroutine f05(n, a) subroutine f06(n, a) integer :: n, a(n, n), i, j logical, parameter :: use_variant = .true. - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 + !ERROR: This construct requires a nest of depth 3, but the associated nest is a nest of depth 2 !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp metadirective when(user={condition(use_variant)}: do collapse(3)) default(nothing) do i = 1, n @@ -107,7 +107,7 @@ subroutine f08(n, a) ! stays selectable and its loop must still be checked. subroutine f09(n, a) integer :: n, a(n, n), i, j - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 + !ERROR: This construct requires a nest of depth 3, but the associated nest is a nest of depth 2 !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp metadirective when(implementation={vendor(bogus_vendor), extension(match_none)}: do collapse(3)) default(nothing) do i = 1, n diff --git a/flang/test/Semantics/OpenMP/metadirective-loop-nest.f90 b/flang/test/Semantics/OpenMP/metadirective-loop-nest.f90 index da712f65e7dff..9a19cbcc916a1 100644 --- a/flang/test/Semantics/OpenMP/metadirective-loop-nest.f90 +++ b/flang/test/Semantics/OpenMP/metadirective-loop-nest.f90 @@ -2,7 +2,7 @@ subroutine collapse_too_deep(n, a) integer :: n, a(n, n), i, j - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 + !ERROR: This construct requires a nest of depth 3, but the associated nest is a nest of depth 2 !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp metadirective when(implementation={vendor(llvm)}: do collapse(3)) default(nothing) do i = 1, n @@ -27,7 +27,7 @@ subroutine ordered_too_deep(n, a) subroutine collapse_too_deep_exec(n, a) integer :: n, a(n, n), i, j a = 0 - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 + !ERROR: This construct requires a nest of depth 3, but the associated nest is a nest of depth 2 !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp metadirective when(implementation={vendor(llvm)}: do collapse(3)) default(nothing) do i = 1, n @@ -40,7 +40,7 @@ subroutine collapse_too_deep_exec(n, a) subroutine collapse_too_deep_compiler_directive(n, a) integer :: n, a(n, n), i, j a = 0 - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 + !ERROR: This construct requires a nest of depth 3, but the associated nest is a nest of depth 2 !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp metadirective when(implementation={vendor(llvm)}: do collapse(3)) default(nothing) !dir$ ivdep @@ -86,7 +86,7 @@ subroutine noncanonical_no_control(n) subroutine collapse_too_deep_interface(n, a) integer :: n, a(n, n), i, j - !ERROR: This construct requires a perfect nest of depth 3, but the associated nest is a perfect nest of depth 2 + !ERROR: This construct requires a nest of depth 3, but the associated nest is a nest of depth 2 !BECAUSE: COLLAPSE clause was specified with argument 3 !$omp metadirective when(implementation={vendor(llvm)}: do collapse(3)) default(nothing) interface @@ -199,7 +199,7 @@ subroutine iface() ! subprogram for its execution part. subroutine no_loop_in_interface_body_preserves_outer(n, a) integer :: n, a(n), i - !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 + !ERROR: This construct requires a nest of depth 2, but the associated nest is a nest of depth 1 !BECAUSE: COLLAPSE clause was specified with argument 2 !$omp metadirective when(implementation={vendor(llvm)}: do collapse(2)) default(nothing) interface diff --git a/flang/test/Semantics/OpenMP/ordered-nesting-omp50.f90 b/flang/test/Semantics/OpenMP/ordered-nesting-omp50.f90 new file mode 100644 index 0000000000000..f2b82c343cf9c --- /dev/null +++ b/flang/test/Semantics/OpenMP/ordered-nesting-omp50.f90 @@ -0,0 +1,50 @@ +! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=50 +! OpenMP 5.0: an ORDERED clause requires perfect nesting, whether or not the +! clause has an argument. This is stricter than 5.1/5.2, where only ORDERED(n) +! (with an argument) requires perfect nesting (see ordered-nesting-omp51.f90). + +! Bare ORDERED with COLLAPSE(2): the two associated loops must be perfectly +! nested in 5.0, so intervening code between them is an error. +subroutine bare_ordered_collapse_imperfect + integer i, j + + !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 + !BECAUSE: COLLAPSE clause was specified with argument 2 + !$omp do collapse(2) ordered + do i = 1, 10 + !BECAUSE: This code prevents perfect nesting + print *, i + do j = 1, 10 + print *, i, j + end do + end do + !$omp end do +end subroutine + +! Bare ORDERED with COLLAPSE(2), perfectly nested: valid. +subroutine bare_ordered_collapse_perfect + integer i, j + + !$omp do collapse(2) ordered + do i = 1, 10 + do j = 1, 10 + print *, i, j + end do + end do + !$omp end do +end subroutine + +! Bare ORDERED with no COLLAPSE: only one loop is associated (depth 1), so +! perfect nesting is trivially satisfied and intervening code is allowed. +subroutine bare_ordered_no_collapse + integer i, j + + !$omp do ordered + do i = 1, 10 + print *, i + do j = 1, 10 + print *, i, j + end do + end do + !$omp end do +end subroutine diff --git a/flang/test/Semantics/OpenMP/ordered-nesting-omp51.f90 b/flang/test/Semantics/OpenMP/ordered-nesting-omp51.f90 new file mode 100644 index 0000000000000..a5f282dbdcd24 --- /dev/null +++ b/flang/test/Semantics/OpenMP/ordered-nesting-omp51.f90 @@ -0,0 +1,36 @@ +! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=51 +! OpenMP 5.1/5.2: only an ORDERED clause *with* an argument requires perfect +! nesting. A bare ORDERED clause does not, unlike in 5.0 (see +! ordered-nesting-omp50.f90). + +! Bare ORDERED with COLLAPSE(2), imperfectly nested: valid in 5.1, since the +! bare ORDERED clause does not impose perfect nesting. +subroutine bare_ordered_collapse_imperfect + integer i, j + + !$omp do collapse(2) ordered + do i = 1, 10 + print *, i + do j = 1, 10 + print *, i, j + end do + end do + !$omp end do +end subroutine + +! ORDERED(2) with an argument still requires perfect nesting. +subroutine ordered_arg_imperfect + integer i, j + + !ERROR: This construct requires a perfect nest of depth 2, but the associated nest is a perfect nest of depth 1 + !BECAUSE: ORDERED clause was specified with argument 2 + !$omp do ordered(2) + do i = 1, 10 + !BECAUSE: This code prevents perfect nesting + print *, i + do j = 1, 10 + print *, i, j + end do + end do + !$omp end do +end subroutine From 12e1c7f70f34d59c023d05d0fe2c228a4e9aac11 Mon Sep 17 00:00:00 2001 From: Daniil Kovalev Date: Tue, 21 Jul 2026 22:12:00 +0300 Subject: [PATCH 18/50] [PAC][libc++] Fix build with `ptrauth_calls` feature (#211033) After partial revert of #208330 in #209928, the libcxx build started failing because of missing `` include required for `uintptr_t` declaration used only by code behind `ptrauth_calls` feature check. See https://lab.llvm.org/buildbot/#/builders/227/builds/3358 This patch adds the missing include. --- libcxx/src/include/overridable_function.h | 1 + 1 file changed, 1 insertion(+) diff --git a/libcxx/src/include/overridable_function.h b/libcxx/src/include/overridable_function.h index e430ccda29f65..307405812f9af 100644 --- a/libcxx/src/include/overridable_function.h +++ b/libcxx/src/include/overridable_function.h @@ -114,6 +114,7 @@ _LIBCPP_END_NAMESPACE_STD # else // __has_feature(ptrauth_calls) +# include # include # if defined(_LIBCPP_OBJECT_FORMAT_MACHO) From c7e1531885e787e909ee06e1b625ae04197350bf Mon Sep 17 00:00:00 2001 From: Farzon Lotfi Date: Tue, 21 Jul 2026 15:20:45 -0400 Subject: [PATCH 19/50] [HLSL][SPIRV] Support Flat and Location decorators in the frontend (#210116) fixes #194293 fixes #194432 In HLSL we need both Location and and Flat must be emitted as a single `spirv.Decorations` node. Also we need to apply Flat decoration to pixe/fragment shades with inputs of integer or double because only 32 bit floats can be interpolated. Assisted with Claude Opus 4.8 via Copilot --- clang/lib/CodeGen/CGHLSLRuntime.cpp | 63 ++++++++++++++++--- clang/lib/CodeGen/CGHLSLRuntime.h | 7 ++- .../semantics/flat-decoration.ps.hlsl | 29 +++++++++ llvm/test/CodeGen/SPIRV/semantics/flat.ps.ll | 51 +++++++++++++++ 4 files changed, 138 insertions(+), 12 deletions(-) create mode 100644 clang/test/CodeGenHLSL/semantics/flat-decoration.ps.hlsl create mode 100644 llvm/test/CodeGen/SPIRV/semantics/flat.ps.ll diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp index ea5c1a24d726e..f86cb00c204f8 100644 --- a/clang/lib/CodeGen/CGHLSLRuntime.cpp +++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp @@ -1064,6 +1064,26 @@ static void addLocationDecoration(llvm::GlobalVariable *GV, unsigned Location) { GV->addMetadata("spirv.Decorations", *Decoration); } +// A fragment shader input interface variable whose base type is an integer or +// a 64-bit float (double) cannot be interpolated by the rasterizer. The Vulkan +// specification requires these variables to be decorated with Flat (see +// VUID-StandaloneSpirv-Flat-04744). Arrays and vectors are unwrapped to inspect +// their base scalar type. +static bool inputRequiresFlatDecoration(llvm::Type *Ty) { + while (true) { + if (auto *AT = dyn_cast(Ty)) { + Ty = AT->getElementType(); + continue; + } + if (auto *VT = dyn_cast(Ty)) { + Ty = VT->getElementType(); + continue; + } + break; + } + return Ty->isIntegerTy() || Ty->isDoubleTy(); +} + static llvm::Value *createSPIRVBuiltinLoad(IRBuilder<> &B, llvm::Module &M, llvm::Type *Ty, const Twine &Name, unsigned BuiltInID) { @@ -1079,20 +1099,36 @@ static llvm::Value *createSPIRVBuiltinLoad(IRBuilder<> &B, llvm::Module &M, static llvm::Value *createSPIRVLocationLoad(IRBuilder<> &B, llvm::Module &M, llvm::Type *Ty, unsigned Location, - StringRef Name) { + StringRef Name, bool NeedsFlat) { auto *GV = new llvm::GlobalVariable( M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage, /* Initializer= */ nullptr, /* Name= */ Name, /* insertBefore= */ nullptr, llvm::GlobalVariable::GeneralDynamicTLSModel, /* AddressSpace */ 7, /* isExternallyInitialized= */ true); GV->setVisibility(llvm::GlobalValue::HiddenVisibility); - addLocationDecoration(GV, Location); + + // Emit all decorations as a single `spirv.Decorations` node. Attaching + // multiple `spirv.Decorations` metadata nodes to the same global is not + // supported by the SPIR-V backend and results in all but one being dropped. + LLVMContext &Ctx = GV->getContext(); + SmallVector Decorations; + Decorations.push_back( + MDNode::get(Ctx, {ConstantAsMetadata::get( + B.getInt32(/* SPIRV::Decoration::Location */ 30)), + ConstantAsMetadata::get(B.getInt32(Location))})); + if (NeedsFlat) + Decorations.push_back( + MDNode::get(Ctx, {ConstantAsMetadata::get( + B.getInt32(/* SPIRV::Decoration::Flat */ 14))})); + GV->addMetadata("spirv.Decorations", *MDNode::get(Ctx, Decorations)); + return B.CreateLoad(Ty, GV); } llvm::Value *CGHLSLRuntime::emitSPIRVUserSemanticLoad( - llvm::IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl, - HLSLAppliedSemanticAttr *Semantic, std::optional Index) { + llvm::IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type, + const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic, + std::optional Index) { Twine BaseName = Twine(Semantic->getAttrName()->getName()); Twine VariableName = BaseName.concat(Twine(Index.value_or(0))); @@ -1106,8 +1142,14 @@ llvm::Value *CGHLSLRuntime::emitSPIRVUserSemanticLoad( unsigned ElementCount = AT ? AT->getNumElements() : 1; SPIRVLastAssignedInputSemanticLocation += ElementCount; + const auto *ShaderAttr = FD->getAttr(); + bool NeedsFlat = + ShaderAttr && + ShaderAttr->getType() == llvm::Triple::EnvironmentType::Pixel && + inputRequiresFlatDecoration(Type); + return createSPIRVLocationLoad(B, CGM.getModule(), Type, Location, - VariableName.str()); + VariableName.str(), NeedsFlat); } static void createSPIRVLocationStore(IRBuilder<> &B, llvm::Module &M, @@ -1199,10 +1241,11 @@ void CGHLSLRuntime::emitDXILUserSemanticStore(llvm::IRBuilder<> &B, } llvm::Value *CGHLSLRuntime::emitUserSemanticLoad( - IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl, - HLSLAppliedSemanticAttr *Semantic, std::optional Index) { + IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type, + const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic, + std::optional Index) { if (CGM.getTarget().getTriple().isSPIRV()) - return emitSPIRVUserSemanticLoad(B, Type, Decl, Semantic, Index); + return emitSPIRVUserSemanticLoad(B, FD, Type, Decl, Semantic, Index); if (CGM.getTarget().getTriple().isDXIL()) return emitDXILUserSemanticLoad(B, Type, Semantic, Index); @@ -1277,7 +1320,7 @@ llvm::Value *CGHLSLRuntime::emitSystemSemanticLoad( } if (ST == Triple::EnvironmentType::Vertex) { - return emitUserSemanticLoad(B, Type, Decl, Semantic, Index); + return emitUserSemanticLoad(B, FD, Type, Decl, Semantic, Index); } } @@ -1346,7 +1389,7 @@ llvm::Value *CGHLSLRuntime::handleScalarSemanticLoad( std::optional Index = Semantic->getSemanticIndex(); if (Semantic->getAttrName()->getName().starts_with_insensitive("SV_")) return emitSystemSemanticLoad(B, FD, Type, Decl, Semantic, Index); - return emitUserSemanticLoad(B, Type, Decl, Semantic, Index); + return emitUserSemanticLoad(B, FD, Type, Decl, Semantic, Index); } void CGHLSLRuntime::handleScalarSemanticStore( diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h index cf47b1633fd3c..0664eef464f98 100644 --- a/clang/lib/CodeGen/CGHLSLRuntime.h +++ b/clang/lib/CodeGen/CGHLSLRuntime.h @@ -335,14 +335,17 @@ class CGHLSLRuntime { llvm::GlobalVariable *GV, HLSLResourceBindingAttr *RBA); - llvm::Value *emitSPIRVUserSemanticLoad(llvm::IRBuilder<> &B, llvm::Type *Type, + llvm::Value *emitSPIRVUserSemanticLoad(llvm::IRBuilder<> &B, + const FunctionDecl *FD, + llvm::Type *Type, const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic, std::optional Index); llvm::Value *emitDXILUserSemanticLoad(llvm::IRBuilder<> &B, llvm::Type *Type, HLSLAppliedSemanticAttr *Semantic, std::optional Index); - llvm::Value *emitUserSemanticLoad(llvm::IRBuilder<> &B, llvm::Type *Type, + llvm::Value *emitUserSemanticLoad(llvm::IRBuilder<> &B, + const FunctionDecl *FD, llvm::Type *Type, const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic, std::optional Index); diff --git a/clang/test/CodeGenHLSL/semantics/flat-decoration.ps.hlsl b/clang/test/CodeGenHLSL/semantics/flat-decoration.ps.hlsl new file mode 100644 index 0000000000000..5e97be23386bd --- /dev/null +++ b/clang/test/CodeGenHLSL/semantics/flat-decoration.ps.hlsl @@ -0,0 +1,29 @@ +// RUN: %clang_cc1 -triple spirv-pc-vulkan1.3-pixel -x hlsl -emit-llvm -finclude-default-header -disable-llvm-passes -o - %s | FileCheck %s --check-prefix=CHECK-SPIRV + +// CHECK-SPIRV-DAG: @A0 = external hidden thread_local addrspace(7) externally_initialized constant float, !spirv.Decorations ![[#FLOAT_MD:]] +// CHECK-SPIRV-DAG: @B0 = external hidden thread_local addrspace(7) externally_initialized constant i32, !spirv.Decorations ![[#INT_MD:]] +// CHECK-SPIRV-DAG: @C0 = external hidden thread_local addrspace(7) externally_initialized constant double, !spirv.Decorations ![[#DOUBLE_MD:]] +// CHECK-SPIRV-DAG: @D0 = external hidden thread_local addrspace(7) externally_initialized constant <2 x i32>, !spirv.Decorations ![[#IVEC_MD:]] + +float main(float a : A, int b : B, double c : C, int2 d : D) : SV_Target { + return a + b + (float)c + d.x + d.y; +} + +// The float input only carries a Location decoration (no Flat). +// CHECK-SPIRV-DAG: ![[#FLOAT_MD]] = !{![[#FLOAT_LOC:]]} +// CHECK-SPIRV-DAG: ![[#FLOAT_LOC]] = !{i32 30, i32 0} + +// The integer input carries both Location and Flat decorations in one node. +// CHECK-SPIRV-DAG: ![[#INT_MD]] = !{![[#INT_LOC:]], ![[#FLAT:]]} +// CHECK-SPIRV-DAG: ![[#INT_LOC]] = !{i32 30, i32 1} + +// The double input carries both Location and Flat decorations in one node. +// CHECK-SPIRV-DAG: ![[#DOUBLE_MD]] = !{![[#DOUBLE_LOC:]], ![[#FLAT]]} +// CHECK-SPIRV-DAG: ![[#DOUBLE_LOC]] = !{i32 30, i32 2} + +// The integer vector input carries both Location and Flat decorations in one node. +// CHECK-SPIRV-DAG: ![[#IVEC_MD]] = !{![[#IVEC_LOC:]], ![[#FLAT]]} +// CHECK-SPIRV-DAG: ![[#IVEC_LOC]] = !{i32 30, i32 3} + +// CHECK-SPIRV-DAG: ![[#FLAT]] = !{i32 14} +// `-> SPIR-V decoration 'Flat' diff --git a/llvm/test/CodeGen/SPIRV/semantics/flat.ps.ll b/llvm/test/CodeGen/SPIRV/semantics/flat.ps.ll new file mode 100644 index 0000000000000..82934a85f9949 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/semantics/flat.ps.ll @@ -0,0 +1,51 @@ +; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv-unknown-vulkan1.3-pixel %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-pixel %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %} + +; This test is the simplified llvm IR of flat-decoration.ps.hlsl. It exists +; because we can't do end to end testing but this gets us close & shows that +; the generated code will now pass the spirv validator by adding the Flat & +; Location decorators. + +; CHECK-DAG: OpEntryPoint Fragment %[[#entry:]] "main" +; CHECK-DAG: OpExecutionMode %[[#entry]] OriginUpperLeft + +; CHECK-DAG: OpDecorate %[[#INT_IN:]] Location 0 +; CHECK-DAG: OpDecorate %[[#INT_IN]] Flat +; CHECK-DAG: OpDecorate %[[#DOUBLE_IN:]] Location 1 +; CHECK-DAG: OpDecorate %[[#DOUBLE_IN]] Flat +; CHECK-DAG: OpDecorate %[[#FLOAT_IN:]] Location 2 +; CHECK-NOT: OpDecorate %[[#FLOAT_IN]] Flat + +@B0 = external hidden thread_local addrspace(7) externally_initialized constant i32, !spirv.Decorations !0 +@C0 = external hidden thread_local addrspace(7) externally_initialized constant double, !spirv.Decorations !3 +@A0 = external hidden thread_local addrspace(7) externally_initialized constant float, !spirv.Decorations !5 +@SV_Target = external hidden thread_local addrspace(8) global float, !spirv.Decorations !7 + +define void @main() #0 { +entry: + %0 = load i32, ptr addrspace(7) @B0, align 4 + %1 = load double, ptr addrspace(7) @C0, align 8 + %2 = load float, ptr addrspace(7) @A0, align 4 + %conv.i = sitofp i32 %0 to float + %conv.d = fptrunc double %1 to float + %add1 = fadd float %2, %conv.i + %add2 = fadd float %add1, %conv.d + store float %add2, ptr addrspace(8) @SV_Target, align 4 + ret void +} + +attributes #0 = { "hlsl.shader"="pixel" } + +; Integer input: Location 0 and Flat. +!0 = !{!1, !2} +!1 = !{i32 30, i32 0} +!2 = !{i32 14} +; Double input: Location 1 and Flat. +!3 = !{!4, !2} +!4 = !{i32 30, i32 1} +; Float input: Location 2 only (interpolated, no Flat). +!5 = !{!6} +!6 = !{i32 30, i32 2} +; SV_Target output: Location 0. +!7 = !{!8} +!8 = !{i32 30, i32 0} From bf42992272b66002ffe2a814c916a6b74e477ada Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Tue, 21 Jul 2026 15:24:26 -0400 Subject: [PATCH 20/50] Revert "[SLP]Make the instruction-count check loop-aware" This reverts commit 3f928aec54e28dd470a3f0fbbe2ebafb73691dcc to fix buildbots https://lab.llvm.org/buildbot/#/builders/11/builds/44691 Reviewers: Pull Request: https://github.com/llvm/llvm-project/pull/211091 --- .../Transforms/Vectorize/SLPVectorizer.cpp | 100 +++++--------- .../X86/deleted-instructions-clear.ll | 10 +- .../X86/loop-invariant-gather-inst-count.ll | 128 +++++++++--------- .../phi-removed-on-operand-vectorization.ll | 15 +- .../Transforms/SLPVectorizer/X86/rgb_phi.ll | 18 +-- 5 files changed, 121 insertions(+), 150 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 8121e695698c8..7adac62541fe0 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -3434,15 +3434,12 @@ class slpvectorizer::BoUpSLP { Instruction *I, const SmallDenseSet *VectorizedVals = nullptr) const; - /// Estimates the number of scalar instructions in the tree, each weighted by - /// its loop-nest trip count (nest-invariant entries are dropped when - /// \p TreeLoop is non-null). - uint64_t getNumScalarInsts(bool HasTreeLoop); + /// Estimates the number of scalar instructions in the tree. + unsigned getNumScalarInsts() const; /// Estimates the number of vector instructions (including buildvectors, - /// shuffles, and extracts) the tree produces, weighted like - /// getNumScalarInsts(). - uint64_t getNumVectorInsts(bool HasTreeLoop); + /// shuffles, and extracts) that the tree will produce. + unsigned getNumVectorInsts() const; /// Return information about the vector formed for the specified index /// of a vector of (the same) instruction. @@ -3492,10 +3489,6 @@ class slpvectorizer::BoUpSLP { uint64_t getGatherNodeEffectiveScale(const TreeEntry &TE, Instruction *U = nullptr); - /// \returns the loop-nest execution scale of \p TE. - uint64_t getEntryEffectiveScale(const TreeEntry &TE, - Instruction *U = nullptr); - /// Get the loop nest for the given loop \p L. ArrayRef getLoopNest(const Loop *L); @@ -13512,16 +13505,12 @@ static InstructionCost canConvertToFMA(ArrayRef VL, TargetTransformInfo &TTI, const TargetLibraryInfo &TLI); -uint64_t BoUpSLP::getNumScalarInsts(bool HasTreeLoop) { - uint64_t Total = 0; +unsigned BoUpSLP::getNumScalarInsts() const { + unsigned Count = 0; for (const std::unique_ptr &Ptr : VectorizableTree) { const TreeEntry &TE = *Ptr; if (DeletedNodes.contains(&TE)) continue; - uint64_t Scale = getEntryEffectiveScale(TE); - if (HasTreeLoop && Scale <= 1) - continue; - unsigned Count = 0; if (TE.isGather() || TransformedToGatherNodes.contains(&TE)) { // Count extractelement scalars in gathers - they exist in the scalar // code regardless of vectorization. ExtractElement instructions @@ -13529,13 +13518,12 @@ uint64_t BoUpSLP::getNumScalarInsts(bool HasTreeLoop) { for (Value *V : TE.Scalars) if (isa(V)) ++Count; - Total = SaturatingMultiplyAdd(Count, Scale, Total); continue; } // CombinedVectorize entries (e.g. the fmul child of an FMulAdd, or the // cmp child of a MinMax select) are absorbed into the parent on both - // scalar and vector sides. The backend fuses fadd+fmul -> fma and - // select+cmp -> smin/smax even for scalar code, so skip to avoid + // scalar and vector sides. The backend fuses fadd+fmul → fma and + // select+cmp → smin/smax even for scalar code, so skip to avoid // double-counting. if (TE.State == TreeEntry::CombinedVectorize) continue; @@ -13560,7 +13548,7 @@ uint64_t BoUpSLP::getNumScalarInsts(bool HasTreeLoop) { } // Even when the whole node is not combined, individual scalar // instructions may be fused by the backend. Each fused pair (e.g. - // fadd+fmul -> fma, select+cmp -> smin/smax) becomes a single scalar + // fadd+fmul → fma, select+cmp → smin/smax) becomes a single scalar // instruction, absorbing the operand instruction. Subtract 1 for each // such match to avoid over-counting the scalar side. if (TE.CombinedOp == TreeEntry::NotCombinedOp && TE.hasState()) { @@ -13594,27 +13582,22 @@ uint64_t BoUpSLP::getNumScalarInsts(bool HasTreeLoop) { } } } - Total = SaturatingMultiplyAdd(Count, Scale, Total); } - return Total; + return Count; } -uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop) { - uint64_t Total = 0; - // Source vector -> max scale among the gather entries sharing it, so the - // combined shufflevector is still weighted like an in-loop entry below. - SmallDenseMap GatherExtractSourceVecs; +unsigned BoUpSLP::getNumVectorInsts() const { + unsigned Count = 0; + SmallPtrSet GatherExtractSourceVecs; for (const std::unique_ptr &Ptr : VectorizableTree) { const TreeEntry &TE = *Ptr; if (DeletedNodes.contains(&TE)) continue; if (TE.State == TreeEntry::CombinedVectorize) continue; - uint64_t Scale = getEntryEffectiveScale(TE); - if (HasTreeLoop && Scale <= 1) - continue; - unsigned Count = 0; - if (TE.isGather() || TransformedToGatherNodes.contains(&TE)) { + bool IsGatherOrTransformed = + TE.isGather() || TransformedToGatherNodes.contains(&TE); + if (IsGatherOrTransformed) { if (TE.hasState()) { if (const TreeEntry *E = getSameValuesTreeEntry(TE.getMainOp(), TE.Scalars); @@ -13624,7 +13607,7 @@ uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop) { if (const TreeEntry *E = getSameValuesTreeEntry(TE.getMainOp(), RevScalars); E && E->getVectorFactor() == TE.getVectorFactor()) { - Total = SaturatingAdd(Total, Scale); + ++Count; continue; } } @@ -13634,19 +13617,14 @@ uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop) { if (all_of(TE.Scalars, IsaPred)) { for (Value *V : TE.Scalars) - if (auto *EE = dyn_cast(V)) { - uint64_t &VecScale = - GatherExtractSourceVecs.try_emplace(EE->getVectorOperand(), 0) - .first->second; - VecScale = std::max(VecScale, Scale); - } + if (auto *EE = dyn_cast(V)) + GatherExtractSourceVecs.insert(EE->getVectorOperand()); } else { for (Value *V : TE.Scalars) { if (!isConstant(V)) ++Count; } } - Total = SaturatingMultiplyAdd(Count, Scale, Total); continue; } // InsertElement/ExtractElement vectorize entries don't produce real @@ -13662,7 +13640,6 @@ uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop) { Count += 2; if (!TE.ReorderIndices.empty() || !TE.ReuseShuffleIndices.empty()) ++Count; - Total = SaturatingMultiplyAdd(Count, Scale, Total); continue; } if (TE.State == TreeEntry::SplitVectorize) @@ -13671,10 +13648,8 @@ uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop) { ++Count; if (!TE.ReorderIndices.empty() || !TE.ReuseShuffleIndices.empty()) ++Count; - Total = SaturatingMultiplyAdd(Count, Scale, Total); } - for (const auto &VecAndScale : GatherExtractSourceVecs) - Total = SaturatingAdd(Total, VecAndScale.second); + Count += GatherExtractSourceVecs.size(); // Count extract instructions from ExternalUses, skipping insertelements // (those get folded into shuffles, not real extracts). SmallPtrSet CountedExtracts; @@ -13687,9 +13662,9 @@ uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop) { continue; if (!CountedExtracts.insert(EU.Scalar).second) continue; - ++Total; + ++Count; } - return Total; + return Count; } void BoUpSLP::TreeEntry::buildAltOpShuffleMask( @@ -16466,12 +16441,6 @@ uint64_t BoUpSLP::getGatherNodeEffectiveScale(const TreeEntry &TE, return std::clamp(Avg, 1, BaseScale); } -uint64_t BoUpSLP::getEntryEffectiveScale(const TreeEntry &TE, Instruction *U) { - if (TE.isGather() || TE.State == TreeEntry::SplitVectorize) - return getGatherNodeEffectiveScale(TE, U); - return getScaleToLoopIterations(TE); -} - InstructionCost BoUpSLP::getVectorSpillReloadCost(const TreeEntry *E, Type *ScalarTy, Type *VecTy, Type *FinalVecTy, @@ -19008,6 +18977,8 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef VectorizedVals, // per-lane refined scale that accounts for LICM-hoistable insertelements // when an operand is invariant in the current loop nest but defined in // an outer loop. This prevents over-costing cross-loop-nest buildvectors. + const bool IsGatherLike = + TE.isGather() || TE.State == TreeEntry::SplitVectorize; if (!CostIsFree && !TE.isGather() && TE.hasState()) { if (PrevVecParent == TE.getMainOp()->getParent()) { Scale = PrevScale; @@ -19016,7 +18987,10 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef VectorizedVals, } } if (!CostIsFree && !Scale) { - Scale = getEntryEffectiveScale(TE, TE.Idx == 0 ? RdxRoot : nullptr); + Scale = + IsGatherLike + ? getGatherNodeEffectiveScale(TE, TE.Idx == 0 ? RdxRoot : nullptr) + : getScaleToLoopIterations(TE); C *= Scale; EntryToScale.try_emplace(&TE, Scale); if (!TE.isGather() && TE.hasState()) { @@ -19395,8 +19369,12 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef VectorizedVals, continue; } uint64_t Scale = EntryToScale.lookup(TE.get()); - if (!Scale) - Scale = getEntryEffectiveScale(*TE); + if (!Scale) { + const bool IsGatherLike = + TE->isGather() || TE->State == TreeEntry::SplitVectorize; + Scale = IsGatherLike ? getGatherNodeEffectiveScale(*TE.get()) + : getScaleToLoopIterations(*TE.get()); + } C *= Scale; NodesCosts.try_emplace(TE.get(), C); } @@ -19480,14 +19458,8 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost, (!SLPReVec || !isa( VectorizableTree.front()->Scalars.front()->getType()))) { - // Loop containing the tree root; null for flat code or disabled - // loop-aware modeling. Shared by both calls below. - const Loop *TreeLoop = nullptr; - if (LoopAwareTripCount != 0 && VectorizableTree.front()->hasState()) - TreeLoop = - LI->getLoopFor(VectorizableTree.front()->getMainOp()->getParent()); - uint64_t NumScalar = getNumScalarInsts(TreeLoop); - uint64_t NumVector = getNumVectorInsts(TreeLoop); + unsigned NumScalar = getNumScalarInsts(); + unsigned NumVector = getNumVectorInsts(); LLVM_DEBUG(dbgs() << "SLP: Inst count check: vector=" << NumVector << " scalar=" << NumScalar << "\n"); if (NumVector > NumScalar && !BypassesInstCountCheck()) { diff --git a/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll b/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll index 94631869b15fe..ae6da0913cc24 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll @@ -7,7 +7,7 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) { ; CHECK-NEXT: [[BB:.*]]: ; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> , i32 [[ARG]], i64 3 ; CHECK-NEXT: [[TMP27:%.*]] = insertelement <2 x i32> poison, i32 [[ARG1]], i64 0 -; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> , i32 [[ARG]], i64 1 +; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x i32> [[TMP27]], <2 x i32> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: br label %[[BB3:.*]] ; CHECK: [[BB3]]: ; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[BB3]] ], [ 0, %[[BB]] ] @@ -28,9 +28,9 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) { ; CHECK-NEXT: [[OR11:%.*]] = or i32 [[TRUNC27]], 0 ; CHECK-NEXT: [[TMP8:%.*]] = or <4 x i32> zeroinitializer, [[TMP2]] ; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i32> [[TMP5]], [[TMP8]] -; CHECK-NEXT: [[TMP28:%.*]] = insertelement <2 x i32> [[TMP27]], i32 [[TRUNC28]], i64 1 +; CHECK-NEXT: [[XOR38:%.*]] = xor i32 [[ARG]], [[TRUNC28]] +; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> , i32 [[TRUNC19]], i64 0 ; CHECK-NEXT: [[TMP14:%.*]] = xor <2 x i32> [[TMP28]], [[TMP29]] -; CHECK-NEXT: [[XOR31:%.*]] = xor i32 [[ARG1]], [[TRUNC19]] ; CHECK-NEXT: [[SHL:%.*]] = shl i32 0, 1 ; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> poison, i32 [[SHL]], i64 0 ; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TRUNC10]], i64 1 @@ -38,9 +38,9 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) { ; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[TMP26:%.*]] = xor <4 x i32> [[TMP11]], [[TMP25]] ; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <4 x i32> [[TMP26]], <4 x i32> poison, <8 x i32> -; CHECK-NEXT: [[TMP32:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[XOR31]], i64 5 ; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <2 x i32> [[TMP14]], <2 x i32> poison, <8 x i32> -; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <8 x i32> [[TMP32]], <8 x i32> [[TMP30]], <8 x i32> +; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP24]], <8 x i32> [[TMP30]], <8 x i32> +; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x i32> [[TMP15]], i32 [[XOR38]], i64 7 ; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> poison, <8 x i32> ; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP17]], <8 x i32> poison, <12 x i32> ; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> poison, <12 x i32> diff --git a/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll b/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll index 9c7849d0de123..923af5efa5d2e 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll @@ -25,70 +25,70 @@ define void @test(ptr noalias readonly %0, ptr noalias readonly %1, i32 %2, doub ; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP19]] ; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP20]], i64 16 ; CHECK-NEXT: [[TMP22:%.*]] = load double, ptr [[TMP21]], align 8 -; CHECK-NEXT: [[TMP23:%.*]] = load <2 x double>, ptr [[TMP15]], align 8 -; CHECK-NEXT: [[TMP24:%.*]] = load <2 x double>, ptr [[TMP20]], align 8 -; CHECK-NEXT: [[TMP25:%.*]] = zext nneg i32 [[TMP11]] to i64 -; CHECK-NEXT: [[TMP26:%.*]] = insertelement <2 x double> poison, double [[TMP17]], i64 0 -; CHECK-NEXT: [[TMP27:%.*]] = insertelement <2 x double> [[TMP26]], double [[TMP22]], i64 1 -; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x double> [[TMP23]], <2 x double> [[TMP24]], <2 x i32> -; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <2 x double> [[TMP23]], <2 x double> [[TMP24]], <2 x i32> -; CHECK-NEXT: [[TMP30:%.*]] = insertelement <2 x double> poison, double [[TMP5]], i64 0 -; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <2 x double> [[TMP30]], <2 x double> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP32:%.*]] = insertelement <2 x double> poison, double [[TMP4]], i64 0 -; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <2 x double> [[TMP32]], <2 x double> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP34:%.*]] = insertelement <2 x double> poison, double [[TMP3]], i64 0 -; CHECK-NEXT: [[TMP35:%.*]] = shufflevector <2 x double> [[TMP34]], <2 x double> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP36:%.*]] = insertelement <2 x double> poison, double [[TMP6]], i64 0 -; CHECK-NEXT: [[TMP37:%.*]] = shufflevector <2 x double> [[TMP36]], <2 x double> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: br label %[[BB38:.*]] -; CHECK: [[BB38]]: -; CHECK-NEXT: [[TMP39:%.*]] = phi i64 [ 2, %[[BB10]] ], [ [[TMP77:%.*]], %[[BB38]] ] -; CHECK-NEXT: [[TMP40:%.*]] = phi i32 [ 0, %[[BB10]] ], [ [[TMP76:%.*]], %[[BB38]] ] -; CHECK-NEXT: [[TMP41:%.*]] = phi i32 [ [[TMP18]], %[[BB10]] ], [ [[TMP50:%.*]], %[[BB38]] ] -; CHECK-NEXT: [[TMP42:%.*]] = phi i32 [ [[TMP13]], %[[BB10]] ], [ [[TMP52:%.*]], %[[BB38]] ] -; CHECK-NEXT: [[TMP43:%.*]] = phi <2 x double> [ [[TMP27]], %[[BB10]] ], [ [[TMP80:%.*]], %[[BB38]] ] -; CHECK-NEXT: [[TMP44:%.*]] = phi <2 x double> [ [[TMP28]], %[[BB10]] ], [ [[TMP81:%.*]], %[[BB38]] ] -; CHECK-NEXT: [[TMP45:%.*]] = phi <2 x double> [ [[TMP29]], %[[BB10]] ], [ [[TMP82:%.*]], %[[BB38]] ] -; CHECK-NEXT: [[TMP46:%.*]] = fsub <2 x double> [[TMP35]], [[TMP45]] -; CHECK-NEXT: [[TMP47:%.*]] = fsub <2 x double> [[TMP33]], [[TMP44]] -; CHECK-NEXT: [[TMP48:%.*]] = fsub <2 x double> [[TMP31]], [[TMP43]] -; CHECK-NEXT: [[TMP49:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP1]], i64 [[TMP39]] -; CHECK-NEXT: [[TMP50]] = load i32, ptr [[TMP49]], align 4 -; CHECK-NEXT: [[TMP51:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP49]], i64 4 -; CHECK-NEXT: [[TMP52]] = load i32, ptr [[TMP51]], align 4 -; CHECK-NEXT: [[TMP53:%.*]] = sext i32 [[TMP50]] to i64 -; CHECK-NEXT: [[TMP54:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP53]] -; CHECK-NEXT: [[TMP55:%.*]] = sext i32 [[TMP52]] to i64 -; CHECK-NEXT: [[TMP56:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP55]] -; CHECK-NEXT: [[TMP57:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP54]], i64 16 -; CHECK-NEXT: [[TMP58:%.*]] = load double, ptr [[TMP57]], align 8 -; CHECK-NEXT: [[TMP59:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP56]], i64 16 -; CHECK-NEXT: [[TMP60:%.*]] = load double, ptr [[TMP59]], align 8 -; CHECK-NEXT: [[TMP61:%.*]] = fmul <2 x double> [[TMP46]], [[TMP46]] -; CHECK-NEXT: [[TMP62:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP47]], <2 x double> [[TMP47]], <2 x double> [[TMP61]]) -; CHECK-NEXT: [[TMP63:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP48]], <2 x double> [[TMP48]], <2 x double> [[TMP62]]) -; CHECK-NEXT: [[TMP64:%.*]] = load <2 x double>, ptr [[TMP54]], align 8 -; CHECK-NEXT: [[TMP65:%.*]] = load <2 x double>, ptr [[TMP56]], align 8 -; CHECK-NEXT: [[TMP66:%.*]] = fcmp olt <2 x double> [[TMP63]], [[TMP37]] -; CHECK-NEXT: [[TMP67:%.*]] = zext nneg i32 [[TMP40]] to i64 -; CHECK-NEXT: [[TMP68:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP67]] -; CHECK-NEXT: store i32 [[TMP41]], ptr [[TMP68]], align 4 -; CHECK-NEXT: [[TMP69:%.*]] = extractelement <2 x i1> [[TMP66]], i64 1 -; CHECK-NEXT: [[TMP70:%.*]] = zext i1 [[TMP69]] to i32 -; CHECK-NEXT: [[TMP71:%.*]] = add nuw nsw i32 [[TMP40]], [[TMP70]] -; CHECK-NEXT: [[TMP72:%.*]] = zext nneg i32 [[TMP71]] to i64 -; CHECK-NEXT: [[TMP73:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP72]] -; CHECK-NEXT: store i32 [[TMP42]], ptr [[TMP73]], align 4 -; CHECK-NEXT: [[TMP74:%.*]] = extractelement <2 x i1> [[TMP66]], i64 0 -; CHECK-NEXT: [[TMP75:%.*]] = zext i1 [[TMP74]] to i32 -; CHECK-NEXT: [[TMP76]] = add nuw nsw i32 [[TMP71]], [[TMP75]] -; CHECK-NEXT: [[TMP77]] = add nuw nsw i64 [[TMP39]], 2 -; CHECK-NEXT: [[TMP78:%.*]] = icmp samesign ult i64 [[TMP77]], [[TMP25]] -; CHECK-NEXT: [[TMP79:%.*]] = insertelement <2 x double> poison, double [[TMP60]], i64 0 -; CHECK-NEXT: [[TMP80]] = insertelement <2 x double> [[TMP79]], double [[TMP58]], i64 1 -; CHECK-NEXT: [[TMP81]] = shufflevector <2 x double> [[TMP65]], <2 x double> [[TMP64]], <2 x i32> -; CHECK-NEXT: [[TMP82]] = shufflevector <2 x double> [[TMP65]], <2 x double> [[TMP64]], <2 x i32> -; CHECK-NEXT: br i1 [[TMP78]], label %[[BB38]], label %[[BB83]] +; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP15]], i64 8 +; CHECK-NEXT: [[TMP24:%.*]] = load double, ptr [[TMP23]], align 8 +; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP20]], i64 8 +; CHECK-NEXT: [[TMP26:%.*]] = load double, ptr [[TMP25]], align 8 +; CHECK-NEXT: [[TMP27:%.*]] = load double, ptr [[TMP15]], align 8 +; CHECK-NEXT: [[TMP28:%.*]] = load double, ptr [[TMP20]], align 8 +; CHECK-NEXT: [[TMP29:%.*]] = zext nneg i32 [[TMP11]] to i64 +; CHECK-NEXT: br label %[[BB30:.*]] +; CHECK: [[BB30]]: +; CHECK-NEXT: [[TMP31:%.*]] = phi i64 [ 2, %[[BB10]] ], [ [[TMP81:%.*]], %[[BB30]] ] +; CHECK-NEXT: [[TMP32:%.*]] = phi i32 [ 0, %[[BB10]] ], [ [[TMP80:%.*]], %[[BB30]] ] +; CHECK-NEXT: [[TMP33:%.*]] = phi i32 [ [[TMP18]], %[[BB10]] ], [ [[TMP54:%.*]], %[[BB30]] ] +; CHECK-NEXT: [[TMP34:%.*]] = phi i32 [ [[TMP13]], %[[BB10]] ], [ [[TMP56:%.*]], %[[BB30]] ] +; CHECK-NEXT: [[TMP35:%.*]] = phi double [ [[TMP28]], %[[BB10]] ], [ [[TMP59:%.*]], %[[BB30]] ] +; CHECK-NEXT: [[TMP36:%.*]] = phi double [ [[TMP27]], %[[BB10]] ], [ [[TMP62:%.*]], %[[BB30]] ] +; CHECK-NEXT: [[TMP37:%.*]] = phi double [ [[TMP17]], %[[BB10]] ], [ [[TMP70:%.*]], %[[BB30]] ] +; CHECK-NEXT: [[TMP38:%.*]] = phi double [ [[TMP22]], %[[BB10]] ], [ [[TMP68:%.*]], %[[BB30]] ] +; CHECK-NEXT: [[TMP39:%.*]] = phi double [ [[TMP24]], %[[BB10]] ], [ [[TMP66:%.*]], %[[BB30]] ] +; CHECK-NEXT: [[TMP40:%.*]] = phi double [ [[TMP26]], %[[BB10]] ], [ [[TMP64:%.*]], %[[BB30]] ] +; CHECK-NEXT: [[TMP41:%.*]] = fsub double [[TMP3]], [[TMP35]] +; CHECK-NEXT: [[TMP42:%.*]] = fsub double [[TMP3]], [[TMP36]] +; CHECK-NEXT: [[TMP43:%.*]] = fmul double [[TMP41]], [[TMP41]] +; CHECK-NEXT: [[TMP44:%.*]] = fmul double [[TMP42]], [[TMP42]] +; CHECK-NEXT: [[TMP45:%.*]] = fsub double [[TMP4]], [[TMP40]] +; CHECK-NEXT: [[TMP46:%.*]] = fsub double [[TMP4]], [[TMP39]] +; CHECK-NEXT: [[TMP47:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP45]], double [[TMP45]], double [[TMP43]]) +; CHECK-NEXT: [[TMP48:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP46]], double [[TMP46]], double [[TMP44]]) +; CHECK-NEXT: [[TMP49:%.*]] = fsub double [[TMP5]], [[TMP38]] +; CHECK-NEXT: [[TMP50:%.*]] = fsub double [[TMP5]], [[TMP37]] +; CHECK-NEXT: [[TMP51:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP49]], double [[TMP49]], double [[TMP47]]) +; CHECK-NEXT: [[TMP52:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP50]], double [[TMP50]], double [[TMP48]]) +; CHECK-NEXT: [[TMP53:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP1]], i64 [[TMP31]] +; CHECK-NEXT: [[TMP54]] = load i32, ptr [[TMP53]], align 4 +; CHECK-NEXT: [[TMP55:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP53]], i64 4 +; CHECK-NEXT: [[TMP56]] = load i32, ptr [[TMP55]], align 4 +; CHECK-NEXT: [[TMP57:%.*]] = sext i32 [[TMP54]] to i64 +; CHECK-NEXT: [[TMP58:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP57]] +; CHECK-NEXT: [[TMP59]] = load double, ptr [[TMP58]], align 8 +; CHECK-NEXT: [[TMP60:%.*]] = sext i32 [[TMP56]] to i64 +; CHECK-NEXT: [[TMP61:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP60]] +; CHECK-NEXT: [[TMP62]] = load double, ptr [[TMP61]], align 8 +; CHECK-NEXT: [[TMP63:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP58]], i64 8 +; CHECK-NEXT: [[TMP64]] = load double, ptr [[TMP63]], align 8 +; CHECK-NEXT: [[TMP65:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP61]], i64 8 +; CHECK-NEXT: [[TMP66]] = load double, ptr [[TMP65]], align 8 +; CHECK-NEXT: [[TMP67:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP58]], i64 16 +; CHECK-NEXT: [[TMP68]] = load double, ptr [[TMP67]], align 8 +; CHECK-NEXT: [[TMP69:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP61]], i64 16 +; CHECK-NEXT: [[TMP70]] = load double, ptr [[TMP69]], align 8 +; CHECK-NEXT: [[TMP71:%.*]] = fcmp olt double [[TMP51]], [[TMP6]] +; CHECK-NEXT: [[TMP72:%.*]] = fcmp olt double [[TMP52]], [[TMP6]] +; CHECK-NEXT: [[TMP73:%.*]] = zext nneg i32 [[TMP32]] to i64 +; CHECK-NEXT: [[TMP74:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP73]] +; CHECK-NEXT: store i32 [[TMP33]], ptr [[TMP74]], align 4 +; CHECK-NEXT: [[TMP75:%.*]] = zext i1 [[TMP71]] to i32 +; CHECK-NEXT: [[TMP76:%.*]] = add nuw nsw i32 [[TMP32]], [[TMP75]] +; CHECK-NEXT: [[TMP77:%.*]] = zext nneg i32 [[TMP76]] to i64 +; CHECK-NEXT: [[TMP78:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP77]] +; CHECK-NEXT: store i32 [[TMP34]], ptr [[TMP78]], align 4 +; CHECK-NEXT: [[TMP79:%.*]] = zext i1 [[TMP72]] to i32 +; CHECK-NEXT: [[TMP80]] = add nuw nsw i32 [[TMP76]], [[TMP79]] +; CHECK-NEXT: [[TMP81]] = add nuw nsw i64 [[TMP31]], 2 +; CHECK-NEXT: [[TMP82:%.*]] = icmp samesign ult i64 [[TMP81]], [[TMP29]] +; CHECK-NEXT: br i1 [[TMP82]], label %[[BB30]], label %[[BB83]] ; CHECK: [[BB83]]: ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll index aefa17e9c6a6e..b47242eceafc2 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll @@ -5,18 +5,17 @@ define i32 @test(double %mul321.i) { ; CHECK-LABEL: define i32 @test( ; CHECK-SAME: double [[MUL321_I:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: [[ENTRY:.*]]: -; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[MUL321_I]], i64 0 ; CHECK-NEXT: br label %[[DO_BODY220_I:.*]] ; CHECK: [[DO_BODY220_I]]: -; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x double> [ [[TMP6:%.*]], %[[DO_BODY221_I:.*]] ], [ zeroinitializer, %[[ENTRY]] ] +; CHECK-NEXT: [[C1_2_I:%.*]] = phi double [ [[MUL321_I]], %[[DO_BODY221_I:.*]] ], [ 0.000000e+00, %[[ENTRY]] ] +; CHECK-NEXT: [[S1_1_I:%.*]] = phi double [ [[ADD318_I:%.*]], %[[DO_BODY221_I]] ], [ 0.000000e+00, %[[ENTRY]] ] ; CHECK-NEXT: br label %[[DO_BODY221_I]] ; CHECK: [[DO_BODY221_I]]: -; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x double> [[TMP1]], zeroinitializer -; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP2]], zeroinitializer -; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i64 0 -; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i64 1 -; CHECK-NEXT: [[ADD318_I:%.*]] = fadd double [[TMP4]], [[TMP5]] -; CHECK-NEXT: [[TMP6]] = insertelement <2 x double> [[TMP0]], double [[ADD318_I]], i64 1 +; CHECK-NEXT: [[SUB311_I1:%.*]] = fadd double [[C1_2_I]], 0.000000e+00 +; CHECK-NEXT: [[ADD315_I:%.*]] = fadd double [[S1_1_I]], 0.000000e+00 +; CHECK-NEXT: [[TMP4:%.*]] = fmul double [[SUB311_I1]], 0.000000e+00 +; CHECK-NEXT: [[TMP5:%.*]] = fmul double [[ADD315_I]], 0.000000e+00 +; CHECK-NEXT: [[ADD318_I]] = fadd double [[TMP4]], [[TMP5]] ; CHECK-NEXT: br label %[[DO_BODY220_I]] ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll b/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll index 8d3a93bb8d49c..eb649f700bda6 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll @@ -23,23 +23,25 @@ target triple = "i386-apple-macosx10.9.0" define float @foo(ptr nocapture readonly %A) { ; CHECK-LABEL: @foo( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A:%.*]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[A:%.*]], align 4 +; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1 +; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX1]], align 4 ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2 ; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[ARRAYIDX2]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP0]], i64 0 ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[TMP4:%.*]] = phi float [ [[TMP3]], [[ENTRY:%.*]] ], [ [[DOTPRE:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE:%.*]] ] ; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ] ; CHECK-NEXT: [[B_032:%.*]] = phi float [ [[TMP2]], [[ENTRY]] ], [ [[ADD14:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ] -; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x float> [ [[TMP0]], [[ENTRY]] ], [ [[TMP10:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ] +; CHECK-NEXT: [[G_031:%.*]] = phi float [ [[TMP1]], [[ENTRY]] ], [ [[TMP16:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ] +; CHECK-NEXT: [[R_030:%.*]] = phi float [ [[TMP3]], [[ENTRY]] ], [ [[TMP15:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ] +; CHECK-NEXT: [[MUL:%.*]] = fmul float [[TMP4]], 7.000000e+00 +; CHECK-NEXT: [[TMP15]] = fadd float [[R_030]], [[MUL]] ; CHECK-NEXT: [[TMP6:%.*]] = add nsw i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP6]] ; CHECK-NEXT: [[TMP7:%.*]] = load float, ptr [[ARRAYIDX7]], align 4 -; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x float> poison, float [[TMP4]], i64 0 -; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x float> [[TMP11]], float [[TMP7]], i64 1 -; CHECK-NEXT: [[TMP9:%.*]] = fmul <2 x float> [[TMP8]], -; CHECK-NEXT: [[TMP10]] = fadd <2 x float> [[TMP5]], [[TMP9]] +; CHECK-NEXT: [[MUL8:%.*]] = fmul float [[TMP7]], 8.000000e+00 +; CHECK-NEXT: [[TMP16]] = fadd float [[G_031]], [[MUL8]] ; CHECK-NEXT: [[TMP12:%.*]] = add nsw i64 [[INDVARS_IV]], 2 ; CHECK-NEXT: [[ARRAYIDX12:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP12]] ; CHECK-NEXT: [[TMP13:%.*]] = load float, ptr [[ARRAYIDX12]], align 4 @@ -54,8 +56,6 @@ define float @foo(ptr nocapture readonly %A) { ; CHECK-NEXT: [[DOTPRE]] = load float, ptr [[ARRAYIDX3_PHI_TRANS_INSERT]], align 4 ; CHECK-NEXT: br label [[FOR_BODY]] ; CHECK: for.end: -; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x float> [[TMP10]], i64 0 -; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x float> [[TMP10]], i64 1 ; CHECK-NEXT: [[ADD16:%.*]] = fadd float [[TMP15]], [[TMP16]] ; CHECK-NEXT: [[ADD17:%.*]] = fadd float [[ADD16]], [[ADD14]] ; CHECK-NEXT: ret float [[ADD17]] From 16e009f63499bb3c8e62388521593dc11f17c8f9 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 21 Jul 2026 20:45:03 +0100 Subject: [PATCH 21/50] [ConstraintElim] Derive signed facts for post-increment inductions. (#210079) Extend logic added in https://github.com/llvm/llvm-project/pull/209199 to also derive signed facts. To do so, we keep track of whether LowerBound = Start + Step overflows signed/unsigned. Code that previously bailed out if we were handling a post-increment induction has been updated to use the computed LowerBound, if it does not signed-wrap. Alive2 Proof: https://alive2.llvm.org/ce/z/frskVt PR: https://github.com/llvm/llvm-project/pull/210079 --- .../Scalar/ConstraintElimination.cpp | 45 +++++++------- ...nduction-condition-in-loop-exit-postinc.ll | 60 +++++++++++++++++-- .../monotonic-int-phis-signed.ll | 3 +- 3 files changed, 78 insertions(+), 30 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp index 71997a6c9e5f1..da92f978cc08c 100644 --- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp +++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp @@ -1074,38 +1074,40 @@ void State::addInfoForInductions(BasicBlock &BB) { } Value *LowerBound = StartValue; + bool LowerBoundNUW = true, LowerBoundNSW = true; if (IncStep) { - // Adjust lower bound when dealing with a post-increment value. auto *StartC = dyn_cast(StartValue); if (!StartC) return; - bool Overflow = false; - APInt Sum = StartC->getValue().uadd_ov(*StepOffset, Overflow); - if (Overflow) - return; + bool UOverflow = false, SOverflow = false; + APInt Sum = StartC->getValue().uadd_ov(*StepOffset, UOverflow); + (void)StartC->getValue().sadd_ov(*StepOffset, SOverflow); LowerBound = ConstantInt::get(StartValue->getType(), Sum); + LowerBoundNUW = !UOverflow; + LowerBoundNSW = !SOverflow; } - // AR may wrap. Add PN >= StartValue conditional on LowerBound <= B which + // AR may wrap. Add PN >= StartValue conditional on LowerBound <= B, which // guarantees that the loop exits before wrapping in combination with the // restrictions on B and the step above. - if (!MonotonicallyIncreasingUnsigned) + ConditionTy StartBeforeBoundULE = {CmpInst::ICMP_ULE, LowerBound, B}; + ConditionTy StartBeforeBoundSLE = {CmpInst::ICMP_SLE, LowerBound, B}; + if (!MonotonicallyIncreasingUnsigned && LowerBoundNUW) WorkList.push_back(FactOrCheck::getConditionFact( - DTN, CmpInst::ICMP_UGE, PN, StartValue, - ConditionTy(CmpInst::ICMP_ULE, LowerBound, B))); - // Only unsigned facts are derived for the post-increment path. - if (!MonotonicallyIncreasingSigned && !IncStep) + DTN, CmpInst::ICMP_UGE, PN, StartValue, StartBeforeBoundULE)); + if (!MonotonicallyIncreasingSigned && LowerBoundNSW) WorkList.push_back(FactOrCheck::getConditionFact( - DTN, CmpInst::ICMP_SGE, PN, StartValue, - ConditionTy(CmpInst::ICMP_SLE, StartValue, B))); + DTN, CmpInst::ICMP_SGE, PN, StartValue, StartBeforeBoundSLE)); - WorkList.push_back(FactOrCheck::getConditionFact( - DTN, CmpInst::ICMP_ULT, PN, B, - ConditionTy(CmpInst::ICMP_ULE, LowerBound, B))); - if (!IncStep) - WorkList.push_back(FactOrCheck::getConditionFact( - DTN, CmpInst::ICMP_SLT, PN, B, - ConditionTy(CmpInst::ICMP_SLE, StartValue, B))); + if (LowerBoundNSW) + WorkList.push_back(FactOrCheck::getConditionFact(DTN, CmpInst::ICMP_SLT, PN, + B, StartBeforeBoundSLE)); + + if (!LowerBoundNUW) + return; + + WorkList.push_back(FactOrCheck::getConditionFact(DTN, CmpInst::ICMP_ULT, PN, + B, StartBeforeBoundULE)); // Try to add condition from header to the dedicated exit blocks. When exiting // either with EQ or NE in the header, we know that the induction value must @@ -1113,14 +1115,13 @@ void State::addInfoForInductions(BasicBlock &BB) { assert(!StepOffset->isNegative() && "induction must be increasing"); assert((Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) && "unsupported predicate"); - ConditionTy Precond = {CmpInst::ICMP_ULE, LowerBound, B}; SmallVector ExitBBs; L->getExitBlocks(ExitBBs); for (BasicBlock *EB : ExitBBs) { // Bail out on non-dedicated exits. if (DT.dominates(&BB, EB)) { WorkList.emplace_back(FactOrCheck::getConditionFact( - DT.getNode(EB), CmpInst::ICMP_ULE, A, B, Precond)); + DT.getNode(EB), CmpInst::ICMP_ULE, A, B, StartBeforeBoundULE)); } } } diff --git a/llvm/test/Transforms/ConstraintElimination/induction-condition-in-loop-exit-postinc.ll b/llvm/test/Transforms/ConstraintElimination/induction-condition-in-loop-exit-postinc.ll index 103fb73e4d37a..b1ac763592750 100644 --- a/llvm/test/Transforms/ConstraintElimination/induction-condition-in-loop-exit-postinc.ll +++ b/llvm/test/Transforms/ConstraintElimination/induction-condition-in-loop-exit-postinc.ll @@ -283,8 +283,8 @@ exit: } ; Post-increment condition with signed predicates in body. -define i1 @postinc_negative_start_signed_body_not_folded(i1 %c) { -; CHECK-LABEL: define i1 @postinc_negative_start_signed_body_not_folded( +define i1 @postinc_negative_start_signed_body_folded(i1 %c) { +; CHECK-LABEL: define i1 @postinc_negative_start_signed_body_folded( ; CHECK-SAME: i1 [[C:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] @@ -294,10 +294,9 @@ define i1 @postinc_negative_start_signed_body_not_folded(i1 %c) { ; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], 10 ; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP_LATCH]] ; CHECK: [[LOOP_LATCH]]: -; CHECK-NEXT: [[RES:%.*]] = icmp slt i64 [[IV]], 10 ; CHECK-NEXT: br i1 [[C]], label %[[EXIT_0:.*]], label %[[LOOP_HEADER]] ; CHECK: [[EXIT_0]]: -; CHECK-NEXT: ret i1 [[RES]] +; CHECK-NEXT: ret i1 true ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret i1 false ; @@ -540,10 +539,9 @@ define void @postinc_signed_facts_when_unsigned_overflows(ptr %p, i1 %c) { ; CHECK-NEXT: [[DONE:%.*]] = icmp eq i8 [[IV_NEXT]], 100 ; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP_LATCH]] ; CHECK: [[LOOP_LATCH]]: -; CHECK-NEXT: [[UB:%.*]] = icmp slt i8 [[IV]], 100 ; CHECK-NEXT: [[UNS:%.*]] = icmp ult i8 [[IV]], 100 ; CHECK-NEXT: [[Z0:%.*]] = zext i1 false to i8 -; CHECK-NEXT: [[Z1:%.*]] = zext i1 [[UB]] to i8 +; CHECK-NEXT: [[Z1:%.*]] = zext i1 true to i8 ; CHECK-NEXT: [[Z2:%.*]] = zext i1 [[UNS]] to i8 ; CHECK-NEXT: store i8 [[Z0]], ptr [[P]], align 1 ; CHECK-NEXT: store i8 [[Z1]], ptr [[P]], align 1 @@ -576,3 +574,53 @@ loop.latch: exit: ret void } + +; Similar to @postinc_signed_facts_when_unsigned_overflows: here the signed sum +; StartValue + Step (127 + 1) overflows signed but not unsigned. +define void @postinc_unsigned_facts_when_signed_overflows(ptr %p) { +; CHECK-LABEL: define void @postinc_unsigned_facts_when_signed_overflows( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i8 [ 127, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[IV_NEXT]] = add i8 [[IV]], 1 +; CHECK-NEXT: [[DONE:%.*]] = icmp eq i8 [[IV_NEXT]], -100 +; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[SLB:%.*]] = icmp slt i8 [[IV]], 127 +; CHECK-NEXT: [[SUB:%.*]] = icmp slt i8 [[IV]], -100 +; CHECK-NEXT: [[Z0:%.*]] = zext i1 [[SLB]] to i8 +; CHECK-NEXT: [[Z1:%.*]] = zext i1 [[SUB]] to i8 +; CHECK-NEXT: [[Z2:%.*]] = zext i1 true to i8 +; CHECK-NEXT: store i8 [[Z0]], ptr [[P]], align 1 +; CHECK-NEXT: store i8 [[Z1]], ptr [[P]], align 1 +; CHECK-NEXT: store i8 [[Z2]], ptr [[P]], align 1 +; CHECK-NEXT: br label %[[LOOP_HEADER]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop.header + +loop.header: + %iv = phi i8 [ 127, %entry ], [ %iv.next, %loop.latch ] + %iv.next = add i8 %iv, 1 + %done = icmp eq i8 %iv.next, -100 + br i1 %done, label %exit, label %loop.latch + +loop.latch: + %slb = icmp slt i8 %iv, 127 + %sub = icmp slt i8 %iv, -100 + %uns = icmp ult i8 %iv, -100 + %z0 = zext i1 %slb to i8 + %z1 = zext i1 %sub to i8 + %z2 = zext i1 %uns to i8 + store i8 %z0, ptr %p, align 1 + store i8 %z1, ptr %p, align 1 + store i8 %z2, ptr %p, align 1 + br label %loop.header + +exit: + ret void +} diff --git a/llvm/test/Transforms/ConstraintElimination/monotonic-int-phis-signed.ll b/llvm/test/Transforms/ConstraintElimination/monotonic-int-phis-signed.ll index 5aa179a25ccac..a5c0702896efd 100644 --- a/llvm/test/Transforms/ConstraintElimination/monotonic-int-phis-signed.ll +++ b/llvm/test/Transforms/ConstraintElimination/monotonic-int-phis-signed.ll @@ -302,8 +302,7 @@ define void @signed_postinc_iv_step_1(i64 %end) { ; CHECK-NEXT: [[CMP_I_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[END]] ; CHECK-NEXT: br i1 [[CMP_I_NOT]], label [[EXIT]], label [[LOOP_LATCH]] ; CHECK: loop.latch: -; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i64 [[IV]], [[END]] -; CHECK-NEXT: call void @use(i1 [[CMP2]]) +; CHECK-NEXT: call void @use(i1 true) ; CHECK-NEXT: call void @use(i1 true) ; CHECK-NEXT: br label [[LOOP]] ; CHECK: exit: From e328d6e08979f163da46f2a12bf74d632b95f168 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Tue, 21 Jul 2026 15:46:53 -0400 Subject: [PATCH 22/50] [libc++] Use github-script instead of pygithub for benchmark jobs, and report failure (#211074) Switch GitHub API interactions to use actions/github-script instead of pygithub, as is done in other Github workflows in the repository. This is simpler and it also resolves the issue that if setting up the Python virtual environment fails, there's no way to communicate the issue back since doing so would require pygithub. As a drive-by, also report the failure when a step in the libc++ PR benchmark job fails, instead of leaving a "Running benchmarks in ..." comment indefinitely. Assisted by Claude Fixes #210985 --- .github/workflows/libcxx-pr-benchmark.yml | 140 ++++++++++++---------- 1 file changed, 79 insertions(+), 61 deletions(-) diff --git a/.github/workflows/libcxx-pr-benchmark.yml b/.github/workflows/libcxx-pr-benchmark.yml index 6737f87cfb3ae..1603962b6883f 100644 --- a/.github/workflows/libcxx-pr-benchmark.yml +++ b/.github/workflows/libcxx-pr-benchmark.yml @@ -48,43 +48,38 @@ jobs: permissions: pull-requests: write steps: - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: '3.14' - - - name: Setup virtual environment - run: | - python3 -m venv .venv - source .venv/bin/activate - python -m pip install pygithub==2.8.1 - - name: Extract information from the PR id: vars - env: - COMMENT_BODY: ${{ github.event.comment.body }} - run: | - source .venv/bin/activate - cat <> ${GITHUB_OUTPUT} - import github - repo = github.Github(auth=github.Auth.Token("${{ github.token }}")).get_repo("${{ github.repository }}") - pr = repo.get_pull(${{ github.event.issue.number }}) - print(f"pr_base={pr.base.sha}") - print(f"pr_head={pr.head.sha}") - EOF - BENCHMARKS=$(echo "$COMMENT_BODY" | sed -nE 's/\/libcxx-bot benchmark (.+)/\1/p') - echo "benchmarks=${BENCHMARKS}" >> ${GITHUB_OUTPUT} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const pr = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + core.setOutput('pr_base', pr.data.base.sha); + core.setOutput('pr_head', pr.data.head.sha); + const match = context.payload.comment.body.match(/\/libcxx-bot benchmark (.+)/); + core.setOutput('benchmarks', match ? match[1] : ''); - name: Update comment with link to the run - run: | - source .venv/bin/activate - cat < _Running benchmarks in ${run_url}_`; + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + body: [comment.body, note].join('\n\n'), + }); outputs: pr_base: ${{ steps.vars.outputs.pr_base }} pr_head: ${{ steps.vars.outputs.pr_head }} @@ -139,7 +134,6 @@ jobs: python3 -m venv .venv source .venv/bin/activate python -m pip install -r libcxx/utils/requirements.txt - python -m pip install pygithub==2.8.1 - name: Build the baseline and the candidate env: @@ -178,30 +172,54 @@ jobs: ./libcxx/utils/compare-benchmarks baseline.lnt candidate.lnt | tee results.txt - name: Update comment with results - run: | - source .venv/bin/activate - cat < - - Benchmark results for ${{ matrix.platform }}: - - - \`\`\` - {benchmark_results} - \`\`\` - - - """ - - comment.edit(new_comment_text) - EOF + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + PLATFORM: ${{ matrix.platform }} + with: + script: | + const fs = require('fs'); + const results = fs.readFileSync('results.txt', 'utf8'); + const { data: comment } = await github.rest.issues.getComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + }); + const details = [ + '
', + '', + `Benchmark results for ${process.env.PLATFORM}:`, + '', + '', + '```', + results, + '```', + '', + '
', + ].join('\n'); + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + body: [comment.body, details].join('\n\n'), + }); + + - name: Report failure in the comment + if: failure() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + PLATFORM: ${{ matrix.platform }} + with: + script: | + const { data: comment } = await github.rest.issues.getComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + }); + const run_url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const note = `> _:x: Benchmarks for ${process.env.PLATFORM} failed. See ${run_url} for details._`; + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + body: [comment.body, note].join('\n\n'), + }); From 5b6ebc25e08c35d30edbe6aa7b64d06c46686b18 Mon Sep 17 00:00:00 2001 From: nvptm Date: Tue, 21 Jul 2026 12:47:25 -0700 Subject: [PATCH 23/50] [flang][cuda][openacc] Support CUDA calls to ACC routines (#210165) Enable CUDA device procedures to call !$acc routine procedures. - Treat ACC routines as device-callable during CUDA semantic checking. - Apply implicit-device dummy argument handling to ACC routines called from CUDA device code. - Defer CUDA cloning of ACC routines until ACC lowering materializes and moves the specialized routine into the GPU module. - Add a CUF device-function transform regression test. --- .../include/flang/Evaluate/characteristics.h | 2 ++ flang/lib/Evaluate/characteristics.cpp | 5 +++++ .../CUDA/CUFDeviceFuncTransform.cpp | 5 +++++ flang/lib/Semantics/check-call.cpp | 16 +++++++++------ flang/lib/Semantics/check-cuda.cpp | 3 +++ .../Fir/CUDA/cuda-device-func-transform.mlir | 20 +++++++++++++++++++ 6 files changed, 45 insertions(+), 6 deletions(-) diff --git a/flang/include/flang/Evaluate/characteristics.h b/flang/include/flang/Evaluate/characteristics.h index af0d429b6aa3b..bc37f3d39c2ec 100644 --- a/flang/include/flang/Evaluate/characteristics.h +++ b/flang/include/flang/Evaluate/characteristics.h @@ -418,6 +418,8 @@ struct Procedure { DummyArguments dummyArguments; Attrs attrs; std::optional cudaSubprogramAttrs; + // Used only by CUDA semantic checks; this is not a CUDA procedure attribute. + bool hasOpenACCRoutine{false}; }; } // namespace Fortran::evaluate::characteristics diff --git a/flang/lib/Evaluate/characteristics.cpp b/flang/lib/Evaluate/characteristics.cpp index b679914f429c4..8750cd3c3ebb6 100644 --- a/flang/lib/Evaluate/characteristics.cpp +++ b/flang/lib/Evaluate/characteristics.cpp @@ -668,6 +668,7 @@ static std::optional CharacterizeProcedure( } } result.cudaSubprogramAttrs = subp.cudaSubprogramAttrs(); + result.hasOpenACCRoutine = !subp.openACCRoutineInfos().empty(); return std::move(result); }, [&](const semantics::ProcEntityDetails &proc) @@ -695,6 +696,9 @@ static std::optional CharacterizeProcedure( // functions as their interfaces. result->attrs.reset(Procedure::Attr::Elemental); } + if (result && !proc.openACCRoutineInfos().empty()) { + result->hasOpenACCRoutine = true; + } return result; } else { Procedure result; @@ -716,6 +720,7 @@ static std::optional CharacterizeProcedure( } else if (symbol.test(semantics::Symbol::Flag::Function)) { return std::nullopt; } + result.hasOpenACCRoutine = !proc.openACCRoutineInfos().empty(); // The PASS name, if any, is not a characteristic. return std::move(result); } diff --git a/flang/lib/Optimizer/Transforms/CUDA/CUFDeviceFuncTransform.cpp b/flang/lib/Optimizer/Transforms/CUDA/CUFDeviceFuncTransform.cpp index 2576dee47d079..307d41cb62a21 100644 --- a/flang/lib/Optimizer/Transforms/CUDA/CUFDeviceFuncTransform.cpp +++ b/flang/lib/Optimizer/Transforms/CUDA/CUFDeviceFuncTransform.cpp @@ -18,6 +18,7 @@ #include "mlir/Dialect/GPU/IR/GPUDialect.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/Dialect/LLVMIR/NVVMDialect.h" +#include "mlir/Dialect/OpenACC/OpenACC.h" #include "mlir/IR/IRMapping.h" #include "mlir/Pass/Pass.h" #include "llvm/ADT/SetVector.h" @@ -216,6 +217,10 @@ class CUFDeviceFuncTransform if (op.getCallee()) { auto func = symbolTable.lookup( op.getCallee()->getLeafReference()); + // ACCRoutineToGPUFunc moves the materialized specialized routine into + // the GPU module later in the pipeline. + if (mlir::acc::isAccRoutine(func)) + return; if (deviceFuncs.count(func) == 0) funcsToClone.insert(func); } diff --git a/flang/lib/Semantics/check-call.cpp b/flang/lib/Semantics/check-call.cpp index b59053ec0481d..0680a48bac2be 100644 --- a/flang/lib/Semantics/check-call.cpp +++ b/flang/lib/Semantics/check-call.cpp @@ -1142,10 +1142,13 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy, } dummyDataAttr = dummy.cudaDataAttr; // Treat MANAGED like DEVICE for nonallocatable nonpointer arguments to - // device subprograms - if (procedure.cudaSubprogramAttrs.value_or( - common::CUDASubprogramAttrs::Host) != - common::CUDASubprogramAttrs::Host && + // device subprograms. An OpenACC routine called from CUDA device code has + // the same implicit-device dummy-argument behavior. + bool isDeviceCallee{procedure.cudaSubprogramAttrs.value_or( + common::CUDASubprogramAttrs::Host) != + common::CUDASubprogramAttrs::Host || + (procedure.hasOpenACCRoutine && FindCUDADeviceContext(scope))}; + if (isDeviceCallee && !dummy.attrs.test( characteristics::DummyDataObject::Attr::Allocatable) && !dummy.attrs.test(characteristics::DummyDataObject::Attr::Pointer)) { @@ -1162,8 +1165,9 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy, if (!actualDataAttr && (!actualFirstSymbol || IsValue(*actualFirstSymbol) || IsFunctionResult(*actualFirstSymbol)) && - (*procedure.cudaSubprogramAttrs == - common::CUDASubprogramAttrs::Device)) { + (procedure.cudaSubprogramAttrs && + *procedure.cudaSubprogramAttrs == + common::CUDASubprogramAttrs::Device)) { actualDataAttr = common::CUDADataAttr::Device; } } diff --git a/flang/lib/Semantics/check-cuda.cpp b/flang/lib/Semantics/check-cuda.cpp index a257a47d92ca9..260e17262dcc0 100644 --- a/flang/lib/Semantics/check-cuda.cpp +++ b/flang/lib/Semantics/check-cuda.cpp @@ -98,6 +98,9 @@ struct DeviceExprChecker "not yet implemented: CUDA dynamic parallelism"_err_en_US); } } + if (!subp->openACCRoutineInfos().empty()) { + return {}; + } } const Symbol &ultimate{sym->GetUltimate()}; diff --git a/flang/test/Fir/CUDA/cuda-device-func-transform.mlir b/flang/test/Fir/CUDA/cuda-device-func-transform.mlir index fae01500edd72..427381732630d 100644 --- a/flang/test/Fir/CUDA/cuda-device-func-transform.mlir +++ b/flang/test/Fir/CUDA/cuda-device-func-transform.mlir @@ -201,3 +201,23 @@ func.func @_QPsub_maxtnid_only() attributes {cuf.launch_bounds = #cuf.launch_bou // CHECK: gpu.func @_QPsub_maxtnid_only() kernel attributes {nvvm.maxntid = array} // CHECK-NOT: nvvm.minctasm + +// ----- + +// ACC routines are materialized and moved to the GPU module later by the +// OpenACC pipeline. Do not clone them while collecting CUDA device callees. +func.func @acc_routine() attributes {acc.routine_info = #acc.routine_info<[@acc_routine_info]>} { + return +} + +func.func @cuda_global() attributes {cuf.proc_attr = #cuf.cuda_proc} { + fir.call @acc_routine() : () -> () + return +} + +// CHECK-LABEL: func.func @acc_routine() +// CHECK-LABEL: gpu.module @cuda_device_mod +// CHECK-NOT: func.func @acc_routine() +// CHECK-NOT: gpu.func @acc_routine() +// CHECK: gpu.func @cuda_global() kernel +// CHECK: fir.call @acc_routine() : () -> () From 5df4aa19aa1e6308509356db19b5e46884d47361 Mon Sep 17 00:00:00 2001 From: nvptm Date: Tue, 21 Jul 2026 12:59:01 -0700 Subject: [PATCH 24/50] [flang][acc] Emit unified declare globals as extern. (#210757) Emit external device symbols for OpenACC declare create and copyin globals in -gpu=mem:unified; preserve device definitions for device_resident globals. --- .../acc-declare-gpu-module-insertion.fir | 23 +++++++++++++++++++ .../mlir/Dialect/OpenACC/Transforms/Passes.td | 4 ++++ .../ACCDeclareGPUModuleInsertion.cpp | 21 +++++++++++++++-- 3 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 flang/test/Fir/OpenACC/acc-declare-gpu-module-insertion.fir diff --git a/flang/test/Fir/OpenACC/acc-declare-gpu-module-insertion.fir b/flang/test/Fir/OpenACC/acc-declare-gpu-module-insertion.fir new file mode 100644 index 0000000000000..b27094467f9cd --- /dev/null +++ b/flang/test/Fir/OpenACC/acc-declare-gpu-module-insertion.fir @@ -0,0 +1,23 @@ +// RUN: fir-opt %s --pass-pipeline="builtin.module(acc-declare-gpu-module-insertion{cuda-unified=true},acc-declare-gpu-module-insertion{cuda-unified=true})" | FileCheck %s + +// Globals registered as host variables must be declarations in device code. +// device_resident globals remain device definitions. +module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module} { + fir.global internal @shared {acc.declare = #acc.declare} : !fir.array<7xf32> { + %0 = fir.zero_bits !fir.array<7xf32> + fir.has_value %0 : !fir.array<7xf32> + } + fir.global @initialized_shared(dense<1.0> : vector<1xf32>) {acc.declare = #acc.declare} : !fir.array<1xf32> + fir.global @resident {acc.declare = #acc.declare} : !fir.array<7xf32> { + %0 = fir.zero_bits !fir.array<7xf32> + fir.has_value %0 : !fir.array<7xf32> + } +} + +// CHECK-LABEL: gpu.module @acc_gpu_module { +// CHECK: fir.global @shared {acc.declare = #acc.declare} : !fir.array<7xf32> +// CHECK-NOT: fir.has_value +// CHECK: fir.global @initialized_shared {acc.declare = #acc.declare} : !fir.array<1xf32> +// CHECK-NOT: fir.has_value +// CHECK: fir.global @resident {{.*}} : !fir.array<7xf32> { +// CHECK: fir.has_value diff --git a/mlir/include/mlir/Dialect/OpenACC/Transforms/Passes.td b/mlir/include/mlir/Dialect/OpenACC/Transforms/Passes.td index 04723c5db2d12..fbc832a37f478 100644 --- a/mlir/include/mlir/Dialect/OpenACC/Transforms/Passes.td +++ b/mlir/include/mlir/Dialect/OpenACC/Transforms/Passes.td @@ -157,6 +157,10 @@ def ACCDeclareGPUModuleInsertion : Pass<"acc-declare-gpu-module-insertion", "mli that device code can reference them. }]; let dependentDialects = ["mlir::acc::OpenACCDialect", "mlir::gpu::GPUDialect"]; + let options = [ + Option<"cudaUnified", "cuda-unified", "bool", "false", + "Emit declaration-only device globals for CUDA unified memory."> + ]; } def ACCLegalizeSerial : Pass<"acc-legalize-serial", "mlir::func::FuncOp"> { diff --git a/mlir/lib/Dialect/OpenACC/Transforms/ACCDeclareGPUModuleInsertion.cpp b/mlir/lib/Dialect/OpenACC/Transforms/ACCDeclareGPUModuleInsertion.cpp index 705b4f9df7770..51e2222208d0d 100644 --- a/mlir/lib/Dialect/OpenACC/Transforms/ACCDeclareGPUModuleInsertion.cpp +++ b/mlir/lib/Dialect/OpenACC/Transforms/ACCDeclareGPUModuleInsertion.cpp @@ -77,6 +77,15 @@ static bool hasAccDeclareGlobals(ModuleOp mod) { return false; } +static void makeDeviceGlobalDeclaration(Operation &globalOp) { + globalOp.removeAttr("initVal"); + globalOp.removeAttr("linkName"); + for (Region ®ion : globalOp.getRegions()) { + region.dropAllReferences(); + region.getBlocks().clear(); + } +} + class ACCDeclareGPUModuleInsertion : public acc::impl::ACCDeclareGPUModuleInsertionBase< ACCDeclareGPUModuleInsertion> { @@ -97,6 +106,12 @@ class ACCDeclareGPUModuleInsertion continue; StringAttr name = symOp.getNameAttr(); + Operation *deviceGlobal = globalOp.clone(); + auto declareAttr = + globalOp.getAttrOfType(acc::getDeclareAttrName()); + if (cudaUnified && declareAttr.getDataClause().getValue() != + acc::DataClause::acc_declare_device_resident) + makeDeviceGlobalDeclaration(*deviceGlobal); if (Operation *existing = gpuSymTable.lookup(name.getValue())) { // Reuse when structurally equivalent ignoring locations and discardable @@ -104,11 +119,12 @@ class ACCDeclareGPUModuleInsertion // true definition mismatch is a conflict. if (existing->getName() != globalOp.getName() || !OperationEquivalence::isEquivalentTo( - existing, &globalOp, + existing, deviceGlobal, OperationEquivalence::ignoreValueEquivalence, /*markEquivalent=*/nullptr, OperationEquivalence::IgnoreLocations | OperationEquivalence::IgnoreDiscardableAttrs)) { + deviceGlobal->destroy(); accSupport.emitNYI(globalOp.getLoc(), llvm::Twine("duplicate global symbol '") + name.getValue() + "' in gpu module"); @@ -120,10 +136,11 @@ class ACCDeclareGPUModuleInsertion if (Attribute declareAttr = globalOp.getAttr(acc::getDeclareAttrName())) existing->setAttr(acc::getDeclareAttrName(), declareAttr); + deviceGlobal->destroy(); continue; } - gpuSymTable.insert(globalOp.clone()); + gpuSymTable.insert(deviceGlobal); } return success(); } From 5697319382fb2ea2f5a15883304edc643ead8fcd Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Tue, 21 Jul 2026 13:09:11 -0700 Subject: [PATCH 25/50] [NVPTX] Simplify BRX emission avoiding pseduo-instruction chain (#209952) This is much more idiomatic and simpler as well and avoids a series of instructions which would cause syntax errors if somehow something were scheduled between them. --- llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp | 25 ++++++++++++ llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h | 3 ++ llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 38 ------------------- llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h | 1 - llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 25 ++++++------ llvm/test/CodeGen/NVPTX/jump-table.ll | 28 +++++++------- .../test/CodeGen/NVPTX/switch-loop-header.mir | 18 +-------- llvm/test/CodeGen/NVPTX/switch.ll | 18 ++++----- 8 files changed, 65 insertions(+), 91 deletions(-) diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp index a22e2cd05a931..e656c70c7c7fa 100644 --- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp @@ -46,6 +46,7 @@ #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineOperand.h" @@ -210,6 +211,11 @@ MCOperand NVPTXAsmPrinter::lowerOperand(const MachineOperand &MO) { MCSymbolRefExpr::create(MO.getMBB()->getSymbol(), OutContext)); case MachineOperand::MO_ExternalSymbol: return GetSymbolRef(GetExternalSymbolSymbol(MO.getSymbolName())); + case MachineOperand::MO_JumpTableIndex: + // The jump table index names the .branchtargets list emitted for a brx.idx + // (see emitFunctionBodyStart); reference it by that label. + return GetSymbolRef( + OutContext.getOrCreateSymbol("$L_brx_" + Twine(MO.getIndex()))); case MachineOperand::MO_GlobalAddress: return GetSymbolRef(getSymbol(MO.getGlobal())); case MachineOperand::MO_FPImmediate: { @@ -413,6 +419,21 @@ void NVPTXAsmPrinter::emitCallPrototype(const CallBase &CB, O << ";\n"; } +void NVPTXAsmPrinter::emitJumpTable(const MachineJumpTableEntry &MJT, + unsigned MJTI, raw_ostream &O) const { + O << "$L_brx_" << MJTI << ":\n"; + + if (MJT.MBBs.empty()) + return; + + O << "\t.branchtargets\n\t\t"; + interleave( + MJT.MBBs, O, + [&](const MachineBasicBlock *MBB) { MBB->getSymbol()->print(O, MAI); }, + ",\n\t\t"); + O << ";\n"; +} + // Return true if MBB is the header of a loop marked with // llvm.loop.unroll.disable or llvm.loop.unroll.count=1. bool NVPTXAsmPrinter::isLoopHeaderOfNoUnroll( @@ -515,6 +536,10 @@ void NVPTXAsmPrinter::emitFunctionBodyStart() { for (const auto &[Id, CB] : MFI->getCallPrototypes()) emitCallPrototype(*CB, Id, O); + if (const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo()) + for (const auto &[Idx, JT] : enumerate(MJTI->getJumpTables())) + emitJumpTable(JT, Idx, O); + OutStreamer->emitRawText(O.str()); } diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h index 8a9b811fec200..170515a57ab1b 100644 --- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h +++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h @@ -22,6 +22,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugLoc.h" @@ -190,6 +191,8 @@ class LLVM_LIBRARY_VISIBILITY NVPTXAsmPrinter : public AsmPrinter { void printReturnValStr(const MachineFunction &MF, raw_ostream &O); void emitCallPrototype(const CallBase &CB, unsigned UniqueCallSite, raw_ostream &O) const; + void emitJumpTable(const MachineJumpTableEntry &MJT, unsigned MJTI, + raw_ostream &O) const; bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &) override; void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O); diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp index ede1deb5400b0..7ffbcff8b5141 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp @@ -16,7 +16,6 @@ #include "llvm/ADT/APInt.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/ISDOpcodes.h" -#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/IR/GlobalValue.h" @@ -191,8 +190,6 @@ void NVPTXDAGToDAGISel::Select(SDNode *N) { if (tryBF16ArithToFMA(N)) return; break; - case ISD::BR_JT: - return selectBR_JT(N); default: break; } @@ -2284,38 +2281,3 @@ void NVPTXDAGToDAGISel::selectAtomicSwap128(SDNode *N) { ReplaceNode(N, ATOM); } - -void NVPTXDAGToDAGISel::selectBR_JT(SDNode *N) { - assert(Subtarget->hasBrx() && - "BR_JT should be expanded during legalization on unsupported targets"); - - SDLoc DL(N); - const SDValue InChain = N->getOperand(0); - const auto *JT = cast(N->getOperand(1)); - const SDValue Index = N->getOperand(2); - - unsigned JId = JT->getIndex(); - MachineJumpTableInfo *MJTI = CurDAG->getMachineFunction().getJumpTableInfo(); - ArrayRef MBBs = MJTI->getJumpTables()[JId].MBBs; - - SDValue IdV = getI32Imm(JId, DL); - - // Generate BrxStart node - MachineSDNode *Chain = CurDAG->getMachineNode( - NVPTX::BRX_START, DL, {MVT::Other, MVT::Glue}, {IdV, InChain}); - - // Generate BrxItem nodes - assert(!MBBs.empty()); - for (MachineBasicBlock *MBB : MBBs.drop_back()) - Chain = CurDAG->getMachineNode( - NVPTX::BRX_ITEM, DL, {MVT::Other, MVT::Glue}, - {CurDAG->getBasicBlock(MBB), SDValue(Chain, 0), SDValue(Chain, 1)}); - - // Generate BrxEnd nodes - MachineSDNode *BrxEnd = - CurDAG->getMachineNode(NVPTX::BRX_END, DL, MVT::Other, - {CurDAG->getBasicBlock(MBBs.back()), Index, IdV, - SDValue(Chain, 0), SDValue(Chain, 1)}); - - ReplaceNode(N, BrxEnd); -} diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h index fcb5700dcb6d4..055f1ff47306d 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h +++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h @@ -93,7 +93,6 @@ class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel { void SelectTcgen05Ld(SDNode *N, bool hasOffset = false); void SelectTcgen05St(SDNode *N, bool hasOffset = false); void selectAtomicSwap128(SDNode *N); - void selectBR_JT(SDNode *N); inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) { return CurDAG->getTargetConstant(Imm, DL, MVT::i32); diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td index 45e2935020d71..405be7d8a68fe 100644 --- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td +++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td @@ -1773,6 +1773,10 @@ def to_texternsym : SDNodeXFormgetTargetFlags()); }]>; +def to_tjumptable : SDNodeXFormgetTargetJumpTable(N->getIndex(), N->getValueType(0)); +}]>; + def : Pat<(i32 globaladdr:$dst), (MOV_B32_sym (to_tglobaladdr $dst))>; def : Pat<(i64 globaladdr:$dst), (MOV_B64_sym (to_tglobaladdr $dst))>; @@ -2489,21 +2493,16 @@ foreach t = [I32RT, I64RT] in { // // BRX // +def brx_jt : SDNode<"ISD::BR_JT", + SDTypeProfile<0, 2, [SDTCisPtrTy<0>, SDTCisVT<1, i32>]>, + [SDNPHasChain]>; -let isTerminator = 1, isBranch = 1, isIndirectBranch = 1, isNotDuplicable = 1 in { - - def BRX_START : - NVPTXInst<(outs), (ins i32imm:$id), "$$L_brx_$id: .branchtargets">; +let isTerminator = 1, isBarrier = 1, isBranch = 1, isIndirectBranch = 1, + isNotDuplicable = 1 in + def BRX_IDX : BasicNVPTXInst<(outs), (ins B32:$val, i32imm:$tgt), + "brx.idx">; - def BRX_ITEM : - NVPTXInst<(outs), (ins brtarget:$target), "\t$target,">; - - def BRX_END : - NVPTXInst<(outs), (ins brtarget:$target, B32:$val, i32imm:$id), - "\t$target;\n\tbrx.idx \t$val, $$L_brx_$id;"> { - let isBarrier = 1; - } -} +def : Pat<(brx_jt jumptable:$jt, i32:$idx), (BRX_IDX $idx, (to_tjumptable $jt))>; foreach a_type = ["s", "u"] in { diff --git a/llvm/test/CodeGen/NVPTX/jump-table.ll b/llvm/test/CodeGen/NVPTX/jump-table.ll index 8f253f5861aea..8ccd62dec9a48 100644 --- a/llvm/test/CodeGen/NVPTX/jump-table.ll +++ b/llvm/test/CodeGen/NVPTX/jump-table.ll @@ -13,17 +13,17 @@ define void @foo(i32 %i) { ; PTX60: { ; PTX60-NEXT: .reg .pred %p<2>; ; PTX60-NEXT: .reg .b32 %r<2>; -; PTX60-EMPTY: +; PTX60-NEXT: $L_brx_0: +; PTX60-NEXT: .branchtargets +; PTX60-NEXT: $L__BB0_2, +; PTX60-NEXT: $L__BB0_3, +; PTX60-NEXT: $L__BB0_4, +; PTX60-NEXT: $L__BB0_5; ; PTX60-NEXT: // %bb.0: // %entry ; PTX60-NEXT: ld.param.b32 %r1, [foo_param_0]; ; PTX60-NEXT: setp.gt.u32 %p1, %r1, 3; ; PTX60-NEXT: @%p1 bra $L__BB0_6; ; PTX60-NEXT: // %bb.1: // %entry -; PTX60-NEXT: $L_brx_0: .branchtargets -; PTX60-NEXT: $L__BB0_2, -; PTX60-NEXT: $L__BB0_3, -; PTX60-NEXT: $L__BB0_4, -; PTX60-NEXT: $L__BB0_5; ; PTX60-NEXT: brx.idx %r1, $L_brx_0; ; PTX60-NEXT: $L__BB0_2: // %case0 ; PTX60-NEXT: st.global.b32 [out], 0; @@ -107,7 +107,14 @@ define i32 @test2(i32 %tmp158) { ; PTX60: { ; PTX60-NEXT: .reg .pred %p<6>; ; PTX60-NEXT: .reg .b32 %r<3>; -; PTX60-EMPTY: +; PTX60-NEXT: $L_brx_0: +; PTX60-NEXT: .branchtargets +; PTX60-NEXT: $L__BB1_6, +; PTX60-NEXT: $L__BB1_7, +; PTX60-NEXT: $L__BB1_8, +; PTX60-NEXT: $L__BB1_9, +; PTX60-NEXT: $L__BB1_10, +; PTX60-NEXT: $L__BB1_11; ; PTX60-NEXT: // %bb.0: // %entry ; PTX60-NEXT: ld.param.b32 %r1, [test2_param_0]; ; PTX60-NEXT: setp.gt.s32 %p1, %r1, 119; @@ -124,13 +131,6 @@ define i32 @test2(i32 %tmp158) { ; PTX60-NEXT: setp.gt.u32 %p2, %r2, 5; ; PTX60-NEXT: @%p2 bra $L__BB1_5; ; PTX60-NEXT: // %bb.12: // %entry -; PTX60-NEXT: $L_brx_0: .branchtargets -; PTX60-NEXT: $L__BB1_6, -; PTX60-NEXT: $L__BB1_7, -; PTX60-NEXT: $L__BB1_8, -; PTX60-NEXT: $L__BB1_9, -; PTX60-NEXT: $L__BB1_10, -; PTX60-NEXT: $L__BB1_11; ; PTX60-NEXT: brx.idx %r2, $L_brx_0; ; PTX60-NEXT: $L__BB1_7: // %bb339 ; PTX60-NEXT: st.param.b32 [func_retval0], 12; diff --git a/llvm/test/CodeGen/NVPTX/switch-loop-header.mir b/llvm/test/CodeGen/NVPTX/switch-loop-header.mir index 1a1f39aab8dc4..eecb3b9618871 100644 --- a/llvm/test/CodeGen/NVPTX/switch-loop-header.mir +++ b/llvm/test/CodeGen/NVPTX/switch-loop-header.mir @@ -133,14 +133,7 @@ body: | ; CHECK-NEXT: bb.5: ; CHECK-NEXT: successors: %bb.3(0x3e000000), %bb.1(0x04000000), %bb.6(0x00000000), %bb.2(0x3e000000) ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: BRX_START 0 - ; CHECK-NEXT: BRX_ITEM %bb.3 - ; CHECK-NEXT: BRX_ITEM %bb.1 - ; CHECK-NEXT: BRX_ITEM %bb.6 - ; CHECK-NEXT: BRX_ITEM %bb.6 - ; CHECK-NEXT: BRX_ITEM %bb.2 - ; CHECK-NEXT: BRX_ITEM %bb.6 - ; CHECK-NEXT: BRX_END %bb.1, undef [[DEF]], 0 + ; CHECK-NEXT: BRX_IDX undef [[DEF]], %jump-table.0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.6: bb.0: @@ -169,14 +162,7 @@ body: | bb.5: successors: %bb.3(0x3e000000), %bb.1(0x04000000), %bb.6(0x00000000), %bb.2(0x3e000000) - BRX_START 0 - BRX_ITEM %bb.3 - BRX_ITEM %bb.1 - BRX_ITEM %bb.6 - BRX_ITEM %bb.6 - BRX_ITEM %bb.2 - BRX_ITEM %bb.6 - BRX_END %bb.1, undef %10, 0 + BRX_IDX undef %10, %jump-table.0 bb.6: ... diff --git a/llvm/test/CodeGen/NVPTX/switch.ll b/llvm/test/CodeGen/NVPTX/switch.ll index 328f69c59a800..f3186874135b3 100644 --- a/llvm/test/CodeGen/NVPTX/switch.ll +++ b/llvm/test/CodeGen/NVPTX/switch.ll @@ -8,7 +8,15 @@ define void @pr170051(i32 %cond) { ; CHECK: { ; CHECK-NEXT: .reg .pred %p<2>; ; CHECK-NEXT: .reg .b32 %r<4>; -; CHECK-EMPTY: +; CHECK-NEXT: $L_brx_0: +; CHECK-NEXT: .branchtargets +; CHECK-NEXT: $L__BB0_2, +; CHECK-NEXT: $L__BB0_3, +; CHECK-NEXT: $L__BB0_5, +; CHECK-NEXT: $L__BB0_5, +; CHECK-NEXT: $L__BB0_1, +; CHECK-NEXT: $L__BB0_5, +; CHECK-NEXT: $L__BB0_3; ; CHECK-NEXT: // %bb.0: // %entry ; CHECK-NEXT: mov.b32 %r2, 0; ; CHECK-NEXT: ld.param.b32 %r1, [pr170051_param_0]; @@ -26,14 +34,6 @@ define void @pr170051(i32 %cond) { ; CHECK-NEXT: // %bb.4: // %BS_LABEL_1 ; CHECK-NEXT: // in Loop: Header=BB0_3 Depth=1 ; CHECK-NEXT: mov.b32 %r3, %r1; -; CHECK-NEXT: $L_brx_0: .branchtargets -; CHECK-NEXT: $L__BB0_2, -; CHECK-NEXT: $L__BB0_3, -; CHECK-NEXT: $L__BB0_5, -; CHECK-NEXT: $L__BB0_5, -; CHECK-NEXT: $L__BB0_1, -; CHECK-NEXT: $L__BB0_5, -; CHECK-NEXT: $L__BB0_3; ; CHECK-NEXT: brx.idx %r1, $L_brx_0; ; CHECK-NEXT: $L__BB0_5: // %unreachable ; CHECK-NEXT: // begin inline asm From e8b684a64aa5176a1da5ff18c712343af695e456 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Tue, 21 Jul 2026 16:18:24 -0400 Subject: [PATCH 26/50] [SLP]Make the instruction-count check loop-aware Raw getNum{Scalar,Vector}Insts() counted one-time, LICM-hoisted broadcasts/buildvectors against the loop body, rejecting profitable loop trees (508.namd_r). Weight each entry by its loop-nest trip count and drop nest-invariant ones; flat code is unchanged (scale 1). Fixes #207572 Original Pull Request: https://github.com/llvm/llvm-project/pull/210074 Recommit after revert in bf42992272b66002ffe2a814c916a6b74e477ada Reviewers: Pull Request: https://github.com/llvm/llvm-project/pull/211096 --- .../Transforms/Vectorize/SLPVectorizer.cpp | 112 +++++++++------ .../X86/deleted-instructions-clear.ll | 10 +- .../X86/loop-invariant-gather-inst-count.ll | 128 +++++++++--------- .../phi-removed-on-operand-vectorization.ll | 15 +- .../Transforms/SLPVectorizer/X86/rgb_phi.ll | 18 +-- 5 files changed, 157 insertions(+), 126 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 7adac62541fe0..e8c400d98d6fb 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -3434,12 +3434,15 @@ class slpvectorizer::BoUpSLP { Instruction *I, const SmallDenseSet *VectorizedVals = nullptr) const; - /// Estimates the number of scalar instructions in the tree. - unsigned getNumScalarInsts() const; + /// Estimates the number of scalar instructions in the tree, each weighted by + /// its loop-nest trip count (nest-invariant entries are dropped when + /// \p TreeLoop is non-null). + uint64_t getNumScalarInsts(bool HasTreeLoop); /// Estimates the number of vector instructions (including buildvectors, - /// shuffles, and extracts) that the tree will produce. - unsigned getNumVectorInsts() const; + /// shuffles, and extracts) the tree produces, weighted like + /// getNumScalarInsts(). + uint64_t getNumVectorInsts(bool HasTreeLoop); /// Return information about the vector formed for the specified index /// of a vector of (the same) instruction. @@ -3489,6 +3492,10 @@ class slpvectorizer::BoUpSLP { uint64_t getGatherNodeEffectiveScale(const TreeEntry &TE, Instruction *U = nullptr); + /// \returns the loop-nest execution scale of \p TE. + uint64_t getEntryEffectiveScale(const TreeEntry &TE, + Instruction *U = nullptr); + /// Get the loop nest for the given loop \p L. ArrayRef getLoopNest(const Loop *L); @@ -13505,12 +13512,25 @@ static InstructionCost canConvertToFMA(ArrayRef VL, TargetTransformInfo &TTI, const TargetLibraryInfo &TLI); -unsigned BoUpSLP::getNumScalarInsts() const { - unsigned Count = 0; +uint64_t BoUpSLP::getNumScalarInsts(bool HasTreeLoop) { + uint64_t Total = 0; for (const std::unique_ptr &Ptr : VectorizableTree) { const TreeEntry &TE = *Ptr; if (DeletedNodes.contains(&TE)) continue; + // CombinedVectorize entries (e.g. the fmul child of an FMulAdd, or the + // cmp child of a MinMax select) are absorbed into the parent on both + // scalar and vector sides. The backend fuses fadd+fmul -> fma and + // select+cmp -> smin/smax even for scalar code, so skip to avoid + // double-counting. Checked before the scale lookup below: some of + // these entries started out as constant-only gather nodes and have no + // valid instruction state to key the loop-nest scale off of. + if (TE.State == TreeEntry::CombinedVectorize) + continue; + uint64_t Scale = getEntryEffectiveScale(TE); + if (HasTreeLoop && Scale <= 1) + continue; + unsigned Count = 0; if (TE.isGather() || TransformedToGatherNodes.contains(&TE)) { // Count extractelement scalars in gathers - they exist in the scalar // code regardless of vectorization. ExtractElement instructions @@ -13518,15 +13538,9 @@ unsigned BoUpSLP::getNumScalarInsts() const { for (Value *V : TE.Scalars) if (isa(V)) ++Count; + Total = SaturatingMultiplyAdd(Count, Scale, Total); continue; } - // CombinedVectorize entries (e.g. the fmul child of an FMulAdd, or the - // cmp child of a MinMax select) are absorbed into the parent on both - // scalar and vector sides. The backend fuses fadd+fmul → fma and - // select+cmp → smin/smax even for scalar code, so skip to avoid - // double-counting. - if (TE.State == TreeEntry::CombinedVectorize) - continue; // Each vectorize entry represents a bundle of scalar instructions. // Count per-entry without cross-entry deduplication, since shared // scalars across entries still represent separate work in scalar code. @@ -13548,7 +13562,7 @@ unsigned BoUpSLP::getNumScalarInsts() const { } // Even when the whole node is not combined, individual scalar // instructions may be fused by the backend. Each fused pair (e.g. - // fadd+fmul → fma, select+cmp → smin/smax) becomes a single scalar + // fadd+fmul -> fma, select+cmp -> smin/smax) becomes a single scalar // instruction, absorbing the operand instruction. Subtract 1 for each // such match to avoid over-counting the scalar side. if (TE.CombinedOp == TreeEntry::NotCombinedOp && TE.hasState()) { @@ -13582,22 +13596,27 @@ unsigned BoUpSLP::getNumScalarInsts() const { } } } + Total = SaturatingMultiplyAdd(Count, Scale, Total); } - return Count; + return Total; } -unsigned BoUpSLP::getNumVectorInsts() const { - unsigned Count = 0; - SmallPtrSet GatherExtractSourceVecs; +uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop) { + uint64_t Total = 0; + // Source vector -> max scale among the gather entries sharing it, so the + // combined shufflevector is still weighted like an in-loop entry below. + SmallDenseMap GatherExtractSourceVecs; for (const std::unique_ptr &Ptr : VectorizableTree) { const TreeEntry &TE = *Ptr; if (DeletedNodes.contains(&TE)) continue; if (TE.State == TreeEntry::CombinedVectorize) continue; - bool IsGatherOrTransformed = - TE.isGather() || TransformedToGatherNodes.contains(&TE); - if (IsGatherOrTransformed) { + uint64_t Scale = getEntryEffectiveScale(TE); + if (HasTreeLoop && Scale <= 1) + continue; + unsigned Count = 0; + if (TE.isGather() || TransformedToGatherNodes.contains(&TE)) { if (TE.hasState()) { if (const TreeEntry *E = getSameValuesTreeEntry(TE.getMainOp(), TE.Scalars); @@ -13607,7 +13626,7 @@ unsigned BoUpSLP::getNumVectorInsts() const { if (const TreeEntry *E = getSameValuesTreeEntry(TE.getMainOp(), RevScalars); E && E->getVectorFactor() == TE.getVectorFactor()) { - ++Count; + Total = SaturatingAdd(Total, Scale); continue; } } @@ -13617,14 +13636,19 @@ unsigned BoUpSLP::getNumVectorInsts() const { if (all_of(TE.Scalars, IsaPred)) { for (Value *V : TE.Scalars) - if (auto *EE = dyn_cast(V)) - GatherExtractSourceVecs.insert(EE->getVectorOperand()); + if (auto *EE = dyn_cast(V)) { + uint64_t &VecScale = + GatherExtractSourceVecs.try_emplace(EE->getVectorOperand(), 0) + .first->second; + VecScale = std::max(VecScale, Scale); + } } else { for (Value *V : TE.Scalars) { if (!isConstant(V)) ++Count; } } + Total = SaturatingMultiplyAdd(Count, Scale, Total); continue; } // InsertElement/ExtractElement vectorize entries don't produce real @@ -13640,6 +13664,7 @@ unsigned BoUpSLP::getNumVectorInsts() const { Count += 2; if (!TE.ReorderIndices.empty() || !TE.ReuseShuffleIndices.empty()) ++Count; + Total = SaturatingMultiplyAdd(Count, Scale, Total); continue; } if (TE.State == TreeEntry::SplitVectorize) @@ -13648,8 +13673,10 @@ unsigned BoUpSLP::getNumVectorInsts() const { ++Count; if (!TE.ReorderIndices.empty() || !TE.ReuseShuffleIndices.empty()) ++Count; + Total = SaturatingMultiplyAdd(Count, Scale, Total); } - Count += GatherExtractSourceVecs.size(); + for (const auto &VecAndScale : GatherExtractSourceVecs) + Total = SaturatingAdd(Total, VecAndScale.second); // Count extract instructions from ExternalUses, skipping insertelements // (those get folded into shuffles, not real extracts). SmallPtrSet CountedExtracts; @@ -13662,9 +13689,9 @@ unsigned BoUpSLP::getNumVectorInsts() const { continue; if (!CountedExtracts.insert(EU.Scalar).second) continue; - ++Count; + ++Total; } - return Count; + return Total; } void BoUpSLP::TreeEntry::buildAltOpShuffleMask( @@ -16441,6 +16468,12 @@ uint64_t BoUpSLP::getGatherNodeEffectiveScale(const TreeEntry &TE, return std::clamp(Avg, 1, BaseScale); } +uint64_t BoUpSLP::getEntryEffectiveScale(const TreeEntry &TE, Instruction *U) { + if (TE.isGather() || TE.State == TreeEntry::SplitVectorize) + return getGatherNodeEffectiveScale(TE, U); + return getScaleToLoopIterations(TE); +} + InstructionCost BoUpSLP::getVectorSpillReloadCost(const TreeEntry *E, Type *ScalarTy, Type *VecTy, Type *FinalVecTy, @@ -18977,8 +19010,6 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef VectorizedVals, // per-lane refined scale that accounts for LICM-hoistable insertelements // when an operand is invariant in the current loop nest but defined in // an outer loop. This prevents over-costing cross-loop-nest buildvectors. - const bool IsGatherLike = - TE.isGather() || TE.State == TreeEntry::SplitVectorize; if (!CostIsFree && !TE.isGather() && TE.hasState()) { if (PrevVecParent == TE.getMainOp()->getParent()) { Scale = PrevScale; @@ -18987,10 +19018,7 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef VectorizedVals, } } if (!CostIsFree && !Scale) { - Scale = - IsGatherLike - ? getGatherNodeEffectiveScale(TE, TE.Idx == 0 ? RdxRoot : nullptr) - : getScaleToLoopIterations(TE); + Scale = getEntryEffectiveScale(TE, TE.Idx == 0 ? RdxRoot : nullptr); C *= Scale; EntryToScale.try_emplace(&TE, Scale); if (!TE.isGather() && TE.hasState()) { @@ -19369,12 +19397,8 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef VectorizedVals, continue; } uint64_t Scale = EntryToScale.lookup(TE.get()); - if (!Scale) { - const bool IsGatherLike = - TE->isGather() || TE->State == TreeEntry::SplitVectorize; - Scale = IsGatherLike ? getGatherNodeEffectiveScale(*TE.get()) - : getScaleToLoopIterations(*TE.get()); - } + if (!Scale) + Scale = getEntryEffectiveScale(*TE); C *= Scale; NodesCosts.try_emplace(TE.get(), C); } @@ -19458,8 +19482,14 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost, (!SLPReVec || !isa( VectorizableTree.front()->Scalars.front()->getType()))) { - unsigned NumScalar = getNumScalarInsts(); - unsigned NumVector = getNumVectorInsts(); + // Loop containing the tree root; null for flat code or disabled + // loop-aware modeling. Shared by both calls below. + const Loop *TreeLoop = nullptr; + if (LoopAwareTripCount != 0 && VectorizableTree.front()->hasState()) + TreeLoop = + LI->getLoopFor(VectorizableTree.front()->getMainOp()->getParent()); + uint64_t NumScalar = getNumScalarInsts(TreeLoop); + uint64_t NumVector = getNumVectorInsts(TreeLoop); LLVM_DEBUG(dbgs() << "SLP: Inst count check: vector=" << NumVector << " scalar=" << NumScalar << "\n"); if (NumVector > NumScalar && !BypassesInstCountCheck()) { diff --git a/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll b/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll index ae6da0913cc24..94631869b15fe 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll @@ -7,7 +7,7 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) { ; CHECK-NEXT: [[BB:.*]]: ; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> , i32 [[ARG]], i64 3 ; CHECK-NEXT: [[TMP27:%.*]] = insertelement <2 x i32> poison, i32 [[ARG1]], i64 0 -; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x i32> [[TMP27]], <2 x i32> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> , i32 [[ARG]], i64 1 ; CHECK-NEXT: br label %[[BB3:.*]] ; CHECK: [[BB3]]: ; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[BB3]] ], [ 0, %[[BB]] ] @@ -28,9 +28,9 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) { ; CHECK-NEXT: [[OR11:%.*]] = or i32 [[TRUNC27]], 0 ; CHECK-NEXT: [[TMP8:%.*]] = or <4 x i32> zeroinitializer, [[TMP2]] ; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i32> [[TMP5]], [[TMP8]] -; CHECK-NEXT: [[XOR38:%.*]] = xor i32 [[ARG]], [[TRUNC28]] -; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> , i32 [[TRUNC19]], i64 0 +; CHECK-NEXT: [[TMP28:%.*]] = insertelement <2 x i32> [[TMP27]], i32 [[TRUNC28]], i64 1 ; CHECK-NEXT: [[TMP14:%.*]] = xor <2 x i32> [[TMP28]], [[TMP29]] +; CHECK-NEXT: [[XOR31:%.*]] = xor i32 [[ARG1]], [[TRUNC19]] ; CHECK-NEXT: [[SHL:%.*]] = shl i32 0, 1 ; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> poison, i32 [[SHL]], i64 0 ; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TRUNC10]], i64 1 @@ -38,9 +38,9 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) { ; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[TMP26:%.*]] = xor <4 x i32> [[TMP11]], [[TMP25]] ; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <4 x i32> [[TMP26]], <4 x i32> poison, <8 x i32> +; CHECK-NEXT: [[TMP32:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[XOR31]], i64 5 ; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <2 x i32> [[TMP14]], <2 x i32> poison, <8 x i32> -; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP24]], <8 x i32> [[TMP30]], <8 x i32> -; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x i32> [[TMP15]], i32 [[XOR38]], i64 7 +; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <8 x i32> [[TMP32]], <8 x i32> [[TMP30]], <8 x i32> ; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> poison, <8 x i32> ; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP17]], <8 x i32> poison, <12 x i32> ; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> poison, <12 x i32> diff --git a/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll b/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll index 923af5efa5d2e..9c7849d0de123 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll @@ -25,70 +25,70 @@ define void @test(ptr noalias readonly %0, ptr noalias readonly %1, i32 %2, doub ; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP19]] ; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP20]], i64 16 ; CHECK-NEXT: [[TMP22:%.*]] = load double, ptr [[TMP21]], align 8 -; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP15]], i64 8 -; CHECK-NEXT: [[TMP24:%.*]] = load double, ptr [[TMP23]], align 8 -; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP20]], i64 8 -; CHECK-NEXT: [[TMP26:%.*]] = load double, ptr [[TMP25]], align 8 -; CHECK-NEXT: [[TMP27:%.*]] = load double, ptr [[TMP15]], align 8 -; CHECK-NEXT: [[TMP28:%.*]] = load double, ptr [[TMP20]], align 8 -; CHECK-NEXT: [[TMP29:%.*]] = zext nneg i32 [[TMP11]] to i64 -; CHECK-NEXT: br label %[[BB30:.*]] -; CHECK: [[BB30]]: -; CHECK-NEXT: [[TMP31:%.*]] = phi i64 [ 2, %[[BB10]] ], [ [[TMP81:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP32:%.*]] = phi i32 [ 0, %[[BB10]] ], [ [[TMP80:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP33:%.*]] = phi i32 [ [[TMP18]], %[[BB10]] ], [ [[TMP54:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP34:%.*]] = phi i32 [ [[TMP13]], %[[BB10]] ], [ [[TMP56:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP35:%.*]] = phi double [ [[TMP28]], %[[BB10]] ], [ [[TMP59:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP36:%.*]] = phi double [ [[TMP27]], %[[BB10]] ], [ [[TMP62:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP37:%.*]] = phi double [ [[TMP17]], %[[BB10]] ], [ [[TMP70:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP38:%.*]] = phi double [ [[TMP22]], %[[BB10]] ], [ [[TMP68:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP39:%.*]] = phi double [ [[TMP24]], %[[BB10]] ], [ [[TMP66:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP40:%.*]] = phi double [ [[TMP26]], %[[BB10]] ], [ [[TMP64:%.*]], %[[BB30]] ] -; CHECK-NEXT: [[TMP41:%.*]] = fsub double [[TMP3]], [[TMP35]] -; CHECK-NEXT: [[TMP42:%.*]] = fsub double [[TMP3]], [[TMP36]] -; CHECK-NEXT: [[TMP43:%.*]] = fmul double [[TMP41]], [[TMP41]] -; CHECK-NEXT: [[TMP44:%.*]] = fmul double [[TMP42]], [[TMP42]] -; CHECK-NEXT: [[TMP45:%.*]] = fsub double [[TMP4]], [[TMP40]] -; CHECK-NEXT: [[TMP46:%.*]] = fsub double [[TMP4]], [[TMP39]] -; CHECK-NEXT: [[TMP47:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP45]], double [[TMP45]], double [[TMP43]]) -; CHECK-NEXT: [[TMP48:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP46]], double [[TMP46]], double [[TMP44]]) -; CHECK-NEXT: [[TMP49:%.*]] = fsub double [[TMP5]], [[TMP38]] -; CHECK-NEXT: [[TMP50:%.*]] = fsub double [[TMP5]], [[TMP37]] -; CHECK-NEXT: [[TMP51:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP49]], double [[TMP49]], double [[TMP47]]) -; CHECK-NEXT: [[TMP52:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP50]], double [[TMP50]], double [[TMP48]]) -; CHECK-NEXT: [[TMP53:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP1]], i64 [[TMP31]] -; CHECK-NEXT: [[TMP54]] = load i32, ptr [[TMP53]], align 4 -; CHECK-NEXT: [[TMP55:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP53]], i64 4 -; CHECK-NEXT: [[TMP56]] = load i32, ptr [[TMP55]], align 4 -; CHECK-NEXT: [[TMP57:%.*]] = sext i32 [[TMP54]] to i64 -; CHECK-NEXT: [[TMP58:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP57]] -; CHECK-NEXT: [[TMP59]] = load double, ptr [[TMP58]], align 8 -; CHECK-NEXT: [[TMP60:%.*]] = sext i32 [[TMP56]] to i64 -; CHECK-NEXT: [[TMP61:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP60]] -; CHECK-NEXT: [[TMP62]] = load double, ptr [[TMP61]], align 8 -; CHECK-NEXT: [[TMP63:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP58]], i64 8 -; CHECK-NEXT: [[TMP64]] = load double, ptr [[TMP63]], align 8 -; CHECK-NEXT: [[TMP65:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP61]], i64 8 -; CHECK-NEXT: [[TMP66]] = load double, ptr [[TMP65]], align 8 -; CHECK-NEXT: [[TMP67:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP58]], i64 16 -; CHECK-NEXT: [[TMP68]] = load double, ptr [[TMP67]], align 8 -; CHECK-NEXT: [[TMP69:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP61]], i64 16 -; CHECK-NEXT: [[TMP70]] = load double, ptr [[TMP69]], align 8 -; CHECK-NEXT: [[TMP71:%.*]] = fcmp olt double [[TMP51]], [[TMP6]] -; CHECK-NEXT: [[TMP72:%.*]] = fcmp olt double [[TMP52]], [[TMP6]] -; CHECK-NEXT: [[TMP73:%.*]] = zext nneg i32 [[TMP32]] to i64 -; CHECK-NEXT: [[TMP74:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP73]] -; CHECK-NEXT: store i32 [[TMP33]], ptr [[TMP74]], align 4 -; CHECK-NEXT: [[TMP75:%.*]] = zext i1 [[TMP71]] to i32 -; CHECK-NEXT: [[TMP76:%.*]] = add nuw nsw i32 [[TMP32]], [[TMP75]] -; CHECK-NEXT: [[TMP77:%.*]] = zext nneg i32 [[TMP76]] to i64 -; CHECK-NEXT: [[TMP78:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP77]] -; CHECK-NEXT: store i32 [[TMP34]], ptr [[TMP78]], align 4 -; CHECK-NEXT: [[TMP79:%.*]] = zext i1 [[TMP72]] to i32 -; CHECK-NEXT: [[TMP80]] = add nuw nsw i32 [[TMP76]], [[TMP79]] -; CHECK-NEXT: [[TMP81]] = add nuw nsw i64 [[TMP31]], 2 -; CHECK-NEXT: [[TMP82:%.*]] = icmp samesign ult i64 [[TMP81]], [[TMP29]] -; CHECK-NEXT: br i1 [[TMP82]], label %[[BB30]], label %[[BB83]] +; CHECK-NEXT: [[TMP23:%.*]] = load <2 x double>, ptr [[TMP15]], align 8 +; CHECK-NEXT: [[TMP24:%.*]] = load <2 x double>, ptr [[TMP20]], align 8 +; CHECK-NEXT: [[TMP25:%.*]] = zext nneg i32 [[TMP11]] to i64 +; CHECK-NEXT: [[TMP26:%.*]] = insertelement <2 x double> poison, double [[TMP17]], i64 0 +; CHECK-NEXT: [[TMP27:%.*]] = insertelement <2 x double> [[TMP26]], double [[TMP22]], i64 1 +; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x double> [[TMP23]], <2 x double> [[TMP24]], <2 x i32> +; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <2 x double> [[TMP23]], <2 x double> [[TMP24]], <2 x i32> +; CHECK-NEXT: [[TMP30:%.*]] = insertelement <2 x double> poison, double [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <2 x double> [[TMP30]], <2 x double> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP32:%.*]] = insertelement <2 x double> poison, double [[TMP4]], i64 0 +; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <2 x double> [[TMP32]], <2 x double> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP34:%.*]] = insertelement <2 x double> poison, double [[TMP3]], i64 0 +; CHECK-NEXT: [[TMP35:%.*]] = shufflevector <2 x double> [[TMP34]], <2 x double> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP36:%.*]] = insertelement <2 x double> poison, double [[TMP6]], i64 0 +; CHECK-NEXT: [[TMP37:%.*]] = shufflevector <2 x double> [[TMP36]], <2 x double> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: br label %[[BB38:.*]] +; CHECK: [[BB38]]: +; CHECK-NEXT: [[TMP39:%.*]] = phi i64 [ 2, %[[BB10]] ], [ [[TMP77:%.*]], %[[BB38]] ] +; CHECK-NEXT: [[TMP40:%.*]] = phi i32 [ 0, %[[BB10]] ], [ [[TMP76:%.*]], %[[BB38]] ] +; CHECK-NEXT: [[TMP41:%.*]] = phi i32 [ [[TMP18]], %[[BB10]] ], [ [[TMP50:%.*]], %[[BB38]] ] +; CHECK-NEXT: [[TMP42:%.*]] = phi i32 [ [[TMP13]], %[[BB10]] ], [ [[TMP52:%.*]], %[[BB38]] ] +; CHECK-NEXT: [[TMP43:%.*]] = phi <2 x double> [ [[TMP27]], %[[BB10]] ], [ [[TMP80:%.*]], %[[BB38]] ] +; CHECK-NEXT: [[TMP44:%.*]] = phi <2 x double> [ [[TMP28]], %[[BB10]] ], [ [[TMP81:%.*]], %[[BB38]] ] +; CHECK-NEXT: [[TMP45:%.*]] = phi <2 x double> [ [[TMP29]], %[[BB10]] ], [ [[TMP82:%.*]], %[[BB38]] ] +; CHECK-NEXT: [[TMP46:%.*]] = fsub <2 x double> [[TMP35]], [[TMP45]] +; CHECK-NEXT: [[TMP47:%.*]] = fsub <2 x double> [[TMP33]], [[TMP44]] +; CHECK-NEXT: [[TMP48:%.*]] = fsub <2 x double> [[TMP31]], [[TMP43]] +; CHECK-NEXT: [[TMP49:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP1]], i64 [[TMP39]] +; CHECK-NEXT: [[TMP50]] = load i32, ptr [[TMP49]], align 4 +; CHECK-NEXT: [[TMP51:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP49]], i64 4 +; CHECK-NEXT: [[TMP52]] = load i32, ptr [[TMP51]], align 4 +; CHECK-NEXT: [[TMP53:%.*]] = sext i32 [[TMP50]] to i64 +; CHECK-NEXT: [[TMP54:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP53]] +; CHECK-NEXT: [[TMP55:%.*]] = sext i32 [[TMP52]] to i64 +; CHECK-NEXT: [[TMP56:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP55]] +; CHECK-NEXT: [[TMP57:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP54]], i64 16 +; CHECK-NEXT: [[TMP58:%.*]] = load double, ptr [[TMP57]], align 8 +; CHECK-NEXT: [[TMP59:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP56]], i64 16 +; CHECK-NEXT: [[TMP60:%.*]] = load double, ptr [[TMP59]], align 8 +; CHECK-NEXT: [[TMP61:%.*]] = fmul <2 x double> [[TMP46]], [[TMP46]] +; CHECK-NEXT: [[TMP62:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP47]], <2 x double> [[TMP47]], <2 x double> [[TMP61]]) +; CHECK-NEXT: [[TMP63:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP48]], <2 x double> [[TMP48]], <2 x double> [[TMP62]]) +; CHECK-NEXT: [[TMP64:%.*]] = load <2 x double>, ptr [[TMP54]], align 8 +; CHECK-NEXT: [[TMP65:%.*]] = load <2 x double>, ptr [[TMP56]], align 8 +; CHECK-NEXT: [[TMP66:%.*]] = fcmp olt <2 x double> [[TMP63]], [[TMP37]] +; CHECK-NEXT: [[TMP67:%.*]] = zext nneg i32 [[TMP40]] to i64 +; CHECK-NEXT: [[TMP68:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP67]] +; CHECK-NEXT: store i32 [[TMP41]], ptr [[TMP68]], align 4 +; CHECK-NEXT: [[TMP69:%.*]] = extractelement <2 x i1> [[TMP66]], i64 1 +; CHECK-NEXT: [[TMP70:%.*]] = zext i1 [[TMP69]] to i32 +; CHECK-NEXT: [[TMP71:%.*]] = add nuw nsw i32 [[TMP40]], [[TMP70]] +; CHECK-NEXT: [[TMP72:%.*]] = zext nneg i32 [[TMP71]] to i64 +; CHECK-NEXT: [[TMP73:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP72]] +; CHECK-NEXT: store i32 [[TMP42]], ptr [[TMP73]], align 4 +; CHECK-NEXT: [[TMP74:%.*]] = extractelement <2 x i1> [[TMP66]], i64 0 +; CHECK-NEXT: [[TMP75:%.*]] = zext i1 [[TMP74]] to i32 +; CHECK-NEXT: [[TMP76]] = add nuw nsw i32 [[TMP71]], [[TMP75]] +; CHECK-NEXT: [[TMP77]] = add nuw nsw i64 [[TMP39]], 2 +; CHECK-NEXT: [[TMP78:%.*]] = icmp samesign ult i64 [[TMP77]], [[TMP25]] +; CHECK-NEXT: [[TMP79:%.*]] = insertelement <2 x double> poison, double [[TMP60]], i64 0 +; CHECK-NEXT: [[TMP80]] = insertelement <2 x double> [[TMP79]], double [[TMP58]], i64 1 +; CHECK-NEXT: [[TMP81]] = shufflevector <2 x double> [[TMP65]], <2 x double> [[TMP64]], <2 x i32> +; CHECK-NEXT: [[TMP82]] = shufflevector <2 x double> [[TMP65]], <2 x double> [[TMP64]], <2 x i32> +; CHECK-NEXT: br i1 [[TMP78]], label %[[BB38]], label %[[BB83]] ; CHECK: [[BB83]]: ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll index b47242eceafc2..aefa17e9c6a6e 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll @@ -5,17 +5,18 @@ define i32 @test(double %mul321.i) { ; CHECK-LABEL: define i32 @test( ; CHECK-SAME: double [[MUL321_I:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[MUL321_I]], i64 0 ; CHECK-NEXT: br label %[[DO_BODY220_I:.*]] ; CHECK: [[DO_BODY220_I]]: -; CHECK-NEXT: [[C1_2_I:%.*]] = phi double [ [[MUL321_I]], %[[DO_BODY221_I:.*]] ], [ 0.000000e+00, %[[ENTRY]] ] -; CHECK-NEXT: [[S1_1_I:%.*]] = phi double [ [[ADD318_I:%.*]], %[[DO_BODY221_I]] ], [ 0.000000e+00, %[[ENTRY]] ] +; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x double> [ [[TMP6:%.*]], %[[DO_BODY221_I:.*]] ], [ zeroinitializer, %[[ENTRY]] ] ; CHECK-NEXT: br label %[[DO_BODY221_I]] ; CHECK: [[DO_BODY221_I]]: -; CHECK-NEXT: [[SUB311_I1:%.*]] = fadd double [[C1_2_I]], 0.000000e+00 -; CHECK-NEXT: [[ADD315_I:%.*]] = fadd double [[S1_1_I]], 0.000000e+00 -; CHECK-NEXT: [[TMP4:%.*]] = fmul double [[SUB311_I1]], 0.000000e+00 -; CHECK-NEXT: [[TMP5:%.*]] = fmul double [[ADD315_I]], 0.000000e+00 -; CHECK-NEXT: [[ADD318_I]] = fadd double [[TMP4]], [[TMP5]] +; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x double> [[TMP1]], zeroinitializer +; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP2]], zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i64 0 +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i64 1 +; CHECK-NEXT: [[ADD318_I:%.*]] = fadd double [[TMP4]], [[TMP5]] +; CHECK-NEXT: [[TMP6]] = insertelement <2 x double> [[TMP0]], double [[ADD318_I]], i64 1 ; CHECK-NEXT: br label %[[DO_BODY220_I]] ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll b/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll index eb649f700bda6..8d3a93bb8d49c 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll @@ -23,25 +23,23 @@ target triple = "i386-apple-macosx10.9.0" define float @foo(ptr nocapture readonly %A) { ; CHECK-LABEL: @foo( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[A:%.*]], align 4 -; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1 -; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX1]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A:%.*]], align 4 ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2 ; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP0]], i64 0 ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[TMP4:%.*]] = phi float [ [[TMP3]], [[ENTRY:%.*]] ], [ [[DOTPRE:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE:%.*]] ] ; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ] ; CHECK-NEXT: [[B_032:%.*]] = phi float [ [[TMP2]], [[ENTRY]] ], [ [[ADD14:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ] -; CHECK-NEXT: [[G_031:%.*]] = phi float [ [[TMP1]], [[ENTRY]] ], [ [[TMP16:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ] -; CHECK-NEXT: [[R_030:%.*]] = phi float [ [[TMP3]], [[ENTRY]] ], [ [[TMP15:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[TMP4]], 7.000000e+00 -; CHECK-NEXT: [[TMP15]] = fadd float [[R_030]], [[MUL]] +; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x float> [ [[TMP0]], [[ENTRY]] ], [ [[TMP10:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ] ; CHECK-NEXT: [[TMP6:%.*]] = add nsw i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP6]] ; CHECK-NEXT: [[TMP7:%.*]] = load float, ptr [[ARRAYIDX7]], align 4 -; CHECK-NEXT: [[MUL8:%.*]] = fmul float [[TMP7]], 8.000000e+00 -; CHECK-NEXT: [[TMP16]] = fadd float [[G_031]], [[MUL8]] +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x float> poison, float [[TMP4]], i64 0 +; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x float> [[TMP11]], float [[TMP7]], i64 1 +; CHECK-NEXT: [[TMP9:%.*]] = fmul <2 x float> [[TMP8]], +; CHECK-NEXT: [[TMP10]] = fadd <2 x float> [[TMP5]], [[TMP9]] ; CHECK-NEXT: [[TMP12:%.*]] = add nsw i64 [[INDVARS_IV]], 2 ; CHECK-NEXT: [[ARRAYIDX12:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP12]] ; CHECK-NEXT: [[TMP13:%.*]] = load float, ptr [[ARRAYIDX12]], align 4 @@ -56,6 +54,8 @@ define float @foo(ptr nocapture readonly %A) { ; CHECK-NEXT: [[DOTPRE]] = load float, ptr [[ARRAYIDX3_PHI_TRANS_INSERT]], align 4 ; CHECK-NEXT: br label [[FOR_BODY]] ; CHECK: for.end: +; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x float> [[TMP10]], i64 0 +; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x float> [[TMP10]], i64 1 ; CHECK-NEXT: [[ADD16:%.*]] = fadd float [[TMP15]], [[TMP16]] ; CHECK-NEXT: [[ADD17:%.*]] = fadd float [[ADD16]], [[ADD14]] ; CHECK-NEXT: ret float [[ADD17]] From f1685cb3d97ca73e88e9435a5700bc9f6f47f2fb Mon Sep 17 00:00:00 2001 From: Eli Friedman Date: Tue, 21 Jul 2026 13:30:31 -0700 Subject: [PATCH 27/50] workflows/issue-release-workflow: Allow multiple URLs in /cherry-pick commands. (#211050) f0ae26c9 allowed specifying multiple commits to /cherry-pick, but that only works for plain commit hashes; it doesn't work for the URL form. Adjust the regex to allow multiple URLs. --- .github/workflows/issue-release-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/issue-release-workflow.yml b/.github/workflows/issue-release-workflow.yml index 923fef547177a..d8be3c491f530 100644 --- a/.github/workflows/issue-release-workflow.yml +++ b/.github/workflows/issue-release-workflow.yml @@ -46,7 +46,7 @@ jobs: # If there are multiple /cherry-pick commands they will all be included in the output. # This is fine since the github-automation.py script can handle multiple cherry-pick commands. validated_comment=$(\ - grep -o -e '/cherry-pick \(https://github.com/llvm/llvm-project/commit/\)\?[a-f0-9 ]\+' <<< $COMMENT_BODY \ + grep -o -e '/cherry-pick \(\(https://github.com/llvm/llvm-project/commit/\)\?[a-f0-9]\+ *\)\+' <<< $COMMENT_BODY \ ) echo "Validated Comment is:" echo ${validated_comment} From 8dc94a5d51bff16c61dd6596a6f25dbf93bc545f Mon Sep 17 00:00:00 2001 From: Andy Kaylor Date: Tue, 21 Jul 2026 13:37:59 -0700 Subject: [PATCH 28/50] [CIR] Fix avx512vlbw builtin test run lines (#211063) The avx512vlbw-builtins.c test was using the now-nonexistant avx10.1-512 target feature in several of its run lines. Because the feature no longer exists with that spelling it was being silently ignored. Apparently it isn't needed for the test upstream because the test has been passing, but I received reports from a downstream user that the test was failing for them. There was a recent change to more strictly enforce always-inlining if target features didn't match. I suspect that this downstream project has a different version of some intrinsic header files so that it requires "avx10.1" (the correct spelling of the feature). This change updates the test run lines to use the correct feature spelling. --- clang/test/CIR/CodeGenBuiltins/X86/avx512vlbw-builtins.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clang/test/CIR/CodeGenBuiltins/X86/avx512vlbw-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx512vlbw-builtins.c index b3a786a328117..08f11707f8d58 100644 --- a/clang/test/CIR/CodeGenBuiltins/X86/avx512vlbw-builtins.c +++ b/clang/test/CIR/CodeGenBuiltins/X86/avx512vlbw-builtins.c @@ -3,13 +3,13 @@ // RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw -target-feature +avx512vl -fclangir -emit-llvm -o %t.ll -Wall -Werror -Wsign-conversion // RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s -// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx10.1-512 -target-feature +avx512vl -fclangir -emit-cir -o %t.cir -Wall -Werror -Wsign-conversion +// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx10.1 -target-feature +avx512vl -fclangir -emit-cir -o %t.cir -Wall -Werror -Wsign-conversion // RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s -// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx10.1-512 -target-feature +avx512vl -fclangir -emit-llvm -o %t.ll -Wall -Werror -Wsign-conversion +// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx10.1 -target-feature +avx512vl -fclangir -emit-llvm -o %t.ll -Wall -Werror -Wsign-conversion // RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s // RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -target-feature +avx512vl -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s --check-prefixes=OGCG -// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx10.1-512 -target-feature +avx512bw -target-feature +avx512vl -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s --check-prefixes=OGCG +// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx10.1 -target-feature +avx512bw -target-feature +avx512vl -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s --check-prefixes=OGCG #include @@ -119,4 +119,4 @@ __mmask16 test_mm256_movepi16_mask(__m256i __A) { // OGCG: [[CMP:%.*]] = icmp slt <16 x i16> %{{.*}}, zeroinitializer // OGCG: bitcast <16 x i1> [[CMP]] to i16 return _mm256_movepi16_mask(__A); -} \ No newline at end of file +} From ded4902fe10242e4581860f92056719fa4591b2c Mon Sep 17 00:00:00 2001 From: Henry Jiang Date: Tue, 21 Jul 2026 13:38:33 -0700 Subject: [PATCH 29/50] Revert "[dsymutil] Add support for pseudo probes (#186877)" (#206789) This reverts commit bc3386c1b9a5bff82ce4baaf4254fc4268f69ff4. Let's back this out in favor of emitting pseudoprobe data in favor of emitting in the `__LLVM` segment, and let `dsymutil` collect the debug map objects and merge them into the final sections. --- .../tools/dsymutil/AArch64/pseudo-probe.test | 319 ------------------ llvm/tools/dsymutil/MachOUtils.cpp | 72 +--- 2 files changed, 7 insertions(+), 384 deletions(-) delete mode 100644 llvm/test/tools/dsymutil/AArch64/pseudo-probe.test diff --git a/llvm/test/tools/dsymutil/AArch64/pseudo-probe.test b/llvm/test/tools/dsymutil/AArch64/pseudo-probe.test deleted file mode 100644 index 0bc97eb574eae..0000000000000 --- a/llvm/test/tools/dsymutil/AArch64/pseudo-probe.test +++ /dev/null @@ -1,319 +0,0 @@ -## Test that dsymutil correctly copies pseudoprobe sections and segments - -# https://github.com/llvm/llvm-project/issues/190481 -# REQUIRES: host-byteorder-little-endian - -# RUN: rm -rf %t && mkdir -p %t -# RUN: yaml2obj %s -o %t/main.o -# RUN: dsymutil --linker classic %t/main.o -o %t/main.dSYM -# RUN: obj2yaml %t/main.dSYM/Contents/Resources/DWARF/main.o -o %t/out.yaml -# RUN: FileCheck %s < %t/out.yaml -# RUN: dsymutil --linker parallel %t/main.o -o %t/main.dSYM -# RUN: obj2yaml %t/main.dSYM/Contents/Resources/DWARF/main.o -o %t/out.yaml -# RUN: FileCheck %s < %t/out.yaml - -# CHECK: segname: __PSEUDO_PROBE -# CHECK: Sections: -# CHECK-NEXT: - sectname: __probe_descs -# CHECK-NEXT: segname: __PSEUDO_PROBE -# CHECK: content: ACBD18DB4CC2F85CFFFFFFFF0000000003666F6F37B51D194A7513E4FFFFFFFF000000000362617273FEFFA4B7F6BB68FFFFFFFF000002000362617AFAD58DE7366495DBFFFFFFFF00000100046D61696E -# CHECK-NEXT: - sectname: __probes -# CHECK-NEXT: segname: __PSEUDO_PROBE -# CHECK: content: 37B51D194A7513E402000020A9A7418A5E3E02B5018000FAD58DE7366495DB0200002001411FAE90F5722601800073FEFFA4B7F6BB6802000020DEDF76EC3AE9F50501800037B51D194A7513E402000020DEDF76EC3AE9F505018000ACBD18DB4CC2F85C020000208774C18D504B6E28018000 - ---- !mach-o -FileHeader: - magic: 0xFEEDFACF - cputype: 0x100000C - cpusubtype: 0x0 - filetype: 0x2 - ncmds: 17 - sizeofcmds: 976 - flags: 0x200085 - reserved: 0x0 -LoadCommands: - - cmd: LC_SEGMENT_64 - cmdsize: 72 - segname: __PAGEZERO - vmaddr: 0 - vmsize: 4294967296 - fileoff: 0 - filesize: 0 - maxprot: 0 - initprot: 0 - nsects: 0 - flags: 0 - - cmd: LC_SEGMENT_64 - cmdsize: 232 - segname: __TEXT - vmaddr: 4294967296 - vmsize: 16384 - fileoff: 0 - filesize: 16384 - maxprot: 5 - initprot: 5 - nsects: 2 - flags: 0 - Sections: - - sectname: __text - segname: __TEXT - addr: 0x100000410 - size: 20 - offset: 0x410 - align: 2 - reloff: 0x0 - nreloc: 0 - flags: 0x80000400 - reserved1: 0x0 - reserved2: 0x0 - reserved3: 0x0 - content: C0035FD6C0035FD6C0035FD600008052C0035FD6 - - sectname: __unwind_info - segname: __TEXT - addr: 0x100000424 - size: 88 - offset: 0x424 - align: 2 - reloff: 0x0 - nreloc: 0 - flags: 0x0 - reserved1: 0x0 - reserved2: 0x0 - reserved3: 0x0 - content: 010000001C000000000000001C000000000000001C00000002000000100400004000000040000000250400000000000040000000000000000000000000000000030000000C00010010000100000000000000000200000000 - - cmd: LC_SEGMENT_64 - cmdsize: 232 - segname: __PSEUDO_PROBE - vmaddr: 4294983680 - vmsize: 16384 - fileoff: 16384 - filesize: 16384 - maxprot: 3 - initprot: 3 - nsects: 2 - flags: 0 - Sections: - - sectname: __probe_descs - segname: __PSEUDO_PROBE - addr: 0x100004000 - size: 81 - offset: 0x4000 - align: 0 - reloff: 0x0 - nreloc: 0 - flags: 0x0 - reserved1: 0x0 - reserved2: 0x0 - reserved3: 0x0 - content: ACBD18DB4CC2F85CFFFFFFFF0000000003666F6F37B51D194A7513E4FFFFFFFF000000000362617273FEFFA4B7F6BB68FFFFFFFF000002000362617AFAD58DE7366495DBFFFFFFFF00000100046D61696E - - sectname: __probes - segname: __PSEUDO_PROBE - addr: 0x100004051 - size: 115 - offset: 0x4051 - align: 0 - reloff: 0x0 - nreloc: 0 - flags: 0x0 - reserved1: 0x0 - reserved2: 0x0 - reserved3: 0x0 - content: 37B51D194A7513E402000020A9A7418A5E3E02B5018000FAD58DE7366495DB0200002001411FAE90F5722601800073FEFFA4B7F6BB6802000020DEDF76EC3AE9F50501800037B51D194A7513E402000020DEDF76EC3AE9F505018000ACBD18DB4CC2F85C020000208774C18D504B6E28018000 - - cmd: LC_SEGMENT_64 - cmdsize: 72 - segname: __LINKEDIT - vmaddr: 4295000064 - vmsize: 16384 - fileoff: 32768 - filesize: 680 - maxprot: 1 - initprot: 1 - nsects: 0 - flags: 0 - - cmd: LC_DYLD_CHAINED_FIXUPS - cmdsize: 16 - dataoff: 32768 - datasize: 56 - - cmd: LC_DYLD_EXPORTS_TRIE - cmdsize: 16 - dataoff: 32824 - datasize: 80 - - cmd: LC_SYMTAB - cmdsize: 24 - symoff: 32912 - nsyms: 5 - stroff: 32992 - strsize: 48 - - cmd: LC_DYSYMTAB - cmdsize: 80 - ilocalsym: 0 - nlocalsym: 0 - iextdefsym: 0 - nextdefsym: 5 - iundefsym: 5 - nundefsym: 0 - tocoff: 0 - ntoc: 0 - modtaboff: 0 - nmodtab: 0 - extrefsymoff: 0 - nextrefsyms: 0 - indirectsymoff: 0 - nindirectsyms: 0 - extreloff: 0 - nextrel: 0 - locreloff: 0 - nlocrel: 0 - - cmd: LC_LOAD_DYLINKER - cmdsize: 32 - name: 12 - Content: '/usr/lib/dyld' - ZeroPadBytes: 7 - - cmd: LC_UUID - cmdsize: 24 - uuid: B95C17E1-7B8C-362B-B402-C636C09BB979 - - cmd: LC_BUILD_VERSION - cmdsize: 32 - platform: 1 - minos: 1703936 - sdk: 1704448 - ntools: 1 - Tools: - - tool: 3 - version: 86573312 - - cmd: LC_SOURCE_VERSION - cmdsize: 16 - version: 0 - - cmd: LC_MAIN - cmdsize: 24 - entryoff: 1052 - stacksize: 0 - - cmd: LC_LOAD_DYLIB - cmdsize: 56 - dylib: - name: 24 - timestamp: 2 - current_version: 88866816 - compatibility_version: 65536 - Content: '/usr/lib/libSystem.B.dylib' - ZeroPadBytes: 6 - - cmd: LC_FUNCTION_STARTS - cmdsize: 16 - dataoff: 32904 - datasize: 8 - - cmd: LC_DATA_IN_CODE - cmdsize: 16 - dataoff: 32912 - datasize: 0 - - cmd: LC_CODE_SIGNATURE - cmdsize: 16 - dataoff: 33040 - datasize: 408 -LinkEditData: - ExportTrie: - TerminalSize: 0 - NodeOffset: 0 - Name: '' - Flags: 0x0 - Address: 0x0 - Other: 0x0 - ImportName: '' - Children: - - TerminalSize: 0 - NodeOffset: 41 - Name: _ - Flags: 0x0 - Address: 0x0 - Other: 0x0 - ImportName: '' - Children: - - TerminalSize: 2 - NodeOffset: 9 - Name: _mh_execute_header - Flags: 0x0 - Address: 0x0 - Other: 0x0 - ImportName: '' - - TerminalSize: 0 - NodeOffset: 23 - Name: ba - Flags: 0x0 - Address: 0x0 - Other: 0x0 - ImportName: '' - Children: - - TerminalSize: 3 - NodeOffset: 13 - Name: r - Flags: 0x0 - Address: 0x414 - Other: 0x0 - ImportName: '' - - TerminalSize: 3 - NodeOffset: 18 - Name: z - Flags: 0x0 - Address: 0x418 - Other: 0x0 - ImportName: '' - - TerminalSize: 3 - NodeOffset: 31 - Name: foo - Flags: 0x0 - Address: 0x410 - Other: 0x0 - ImportName: '' - - TerminalSize: 3 - NodeOffset: 36 - Name: main - Flags: 0x0 - Address: 0x41C - Other: 0x0 - ImportName: '' - NameList: - - n_strx: 2 - n_type: 0xF - n_sect: 1 - n_desc: 16 - n_value: 4294967296 - - n_strx: 22 - n_type: 0xF - n_sect: 1 - n_desc: 0 - n_value: 4294968340 - - n_strx: 27 - n_type: 0xF - n_sect: 1 - n_desc: 0 - n_value: 4294968344 - - n_strx: 32 - n_type: 0xF - n_sect: 1 - n_desc: 0 - n_value: 4294968336 - - n_strx: 37 - n_type: 0xF - n_sect: 1 - n_desc: 0 - n_value: 4294968348 - StringTable: - - ' ' - - __mh_execute_header - - _bar - - _baz - - _foo - - _main - - '' - - '' - - '' - - '' - - '' - FunctionStarts: [ 0x410, 0x414, 0x418, 0x41C ] - ChainedFixups: [ 0x0, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x34, 0x0, - 0x0, 0x0, 0x34, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] -... - diff --git a/llvm/tools/dsymutil/MachOUtils.cpp b/llvm/tools/dsymutil/MachOUtils.cpp index a6dd3afa223e7..9b143157ac773 100644 --- a/llvm/tools/dsymutil/MachOUtils.cpp +++ b/llvm/tools/dsymutil/MachOUtils.cpp @@ -269,19 +269,13 @@ getSection(const object::MachOObjectFile &Obj, // // When the eh_frame section is transferred, its offset and size are set resp. // to \a EHFrameOffset and \a EHFrameSize. -// -// When the __PSEUDO_PROBE segment is transferred, its offset and size are set -// resp. to \a PseudoProbeOffset and \a PseudoProbeSize, and its sections' -// offsets are updated using \a PseudoProbeProbesOffset for __probes and -// \a PseudoProbeDescsOffset for __probe_descs. template static void transferSegmentAndSections( const object::MachOObjectFile::LoadCommandInfo &LCI, SegmentTy Segment, const object::MachOObjectFile &Obj, MachObjectWriter &Writer, uint64_t LinkeditOffset, uint64_t LinkeditSize, uint64_t EHFrameOffset, - uint64_t EHFrameSize, uint64_t PseudoProbeOffset, uint64_t PseudoProbeSize, - uint64_t PseudoProbeProbesOffset, uint64_t PseudoProbeDescsOffset, - uint64_t DwarfSegmentSize, uint64_t &GapForDwarf, uint64_t &EndAddress) { + uint64_t EHFrameSize, uint64_t DwarfSegmentSize, uint64_t &GapForDwarf, + uint64_t &EndAddress) { if (StringRef("__DWARF") == Segment.segname) return; @@ -293,10 +287,6 @@ static void transferSegmentAndSections( Segment.filesize = LinkeditSize; // Resize vmsize by rounding to the page size. Segment.vmsize = alignTo(LinkeditSize, 0x1000); - } else if (StringRef("__PSEUDO_PROBE") == Segment.segname && - PseudoProbeSize > 0) { - Segment.fileoff = PseudoProbeOffset; - Segment.filesize = PseudoProbeSize; } else { Segment.fileoff = Segment.filesize = 0; } @@ -322,14 +312,6 @@ static void transferSegmentAndSections( if (StringRef("__eh_frame") == Sect.sectname) { Sect.offset = EHFrameOffset; Sect.reloff = Sect.nreloc = 0; - } else if (StringRef("__probes") == Sect.sectname && - PseudoProbeProbesOffset > 0) { - Sect.offset = PseudoProbeProbesOffset; - Sect.reloff = Sect.nreloc = 0; - } else if (StringRef("__probe_descs") == Sect.sectname && - PseudoProbeDescsOffset > 0) { - Sect.offset = PseudoProbeDescsOffset; - Sect.reloff = Sect.nreloc = 0; } else { Sect.offset = Sect.reloff = Sect.nreloc = 0; } @@ -481,10 +463,6 @@ bool generateDsymCompanion( // If we have a valid eh_frame to copy, do it. uint64_t EHFrameSize = 0; StringRef EHFrameData; - StringRef PseudoProbeProbesData; - uint64_t PseudoProbeProbesSize = 0; - StringRef PseudoProbeDescsData; - uint64_t PseudoProbeDescsSize = 0; for (const object::SectionRef &Section : InputBinary.sections()) { Expected NameOrErr = Section.getName(); if (!NameOrErr) { @@ -500,23 +478,8 @@ bool generateDsymCompanion( } else { consumeError(ContentsOrErr.takeError()); } - } else if (SectionName == "probes") { - if (Expected ContentsOrErr = Section.getContents()) { - PseudoProbeProbesData = *ContentsOrErr; - PseudoProbeProbesSize = Section.getSize(); - } else { - consumeError(ContentsOrErr.takeError()); - } - } else if (SectionName == "probe_descs") { - if (Expected ContentsOrErr = Section.getContents()) { - PseudoProbeDescsData = *ContentsOrErr; - PseudoProbeDescsSize = Section.getSize(); - } else { - consumeError(ContentsOrErr.takeError()); - } } } - uint64_t PseudoProbeSize = PseudoProbeProbesSize + PseudoProbeDescsSize; unsigned HeaderSize = Is64Bit ? sizeof(MachO::mach_header_64) : sizeof(MachO::mach_header); @@ -598,16 +561,8 @@ bool generateDsymCompanion( uint64_t EHFrameStart = StringStart + NewStringsSize; EHFrameStart = alignTo(EHFrameStart, 0x1000); - // Place pseudo probe data after the EH frame. - uint64_t PseudoProbeStart = PseudoProbeSize > 0 - ? alignTo(EHFrameStart + EHFrameSize, 0x1000) - : EHFrameStart + EHFrameSize; - - uint64_t PseudoProbeProbesStart = PseudoProbeStart; - uint64_t PseudoProbeDescsStart = PseudoProbeStart + PseudoProbeProbesSize; - - uint64_t DwarfSegmentStart = - alignTo(PseudoProbeStart + PseudoProbeSize, 0x1000); + uint64_t DwarfSegmentStart = EHFrameStart + EHFrameSize; + DwarfSegmentStart = alignTo(DwarfSegmentStart, 0x1000); // Write the load commands for the segments and sections we 'import' from // the original binary. @@ -618,16 +573,12 @@ bool generateDsymCompanion( transferSegmentAndSections( LCI, InputBinary.getSegmentLoadCommand(LCI), InputBinary, Writer, SymtabStart, StringStart + NewStringsSize - SymtabStart, EHFrameStart, - EHFrameSize, PseudoProbeStart, PseudoProbeSize, - PseudoProbeProbesStart, PseudoProbeDescsStart, DwarfSegmentSize, - GapForDwarf, EndAddress); + EHFrameSize, DwarfSegmentSize, GapForDwarf, EndAddress); else if (LCI.C.cmd == MachO::LC_SEGMENT_64) transferSegmentAndSections( LCI, InputBinary.getSegment64LoadCommand(LCI), InputBinary, Writer, SymtabStart, StringStart + NewStringsSize - SymtabStart, EHFrameStart, - EHFrameSize, PseudoProbeStart, PseudoProbeSize, - PseudoProbeProbesStart, PseudoProbeDescsStart, DwarfSegmentSize, - GapForDwarf, EndAddress); + EHFrameSize, DwarfSegmentSize, GapForDwarf, EndAddress); } uint64_t DwarfVMAddr = alignTo(EndAddress, 0x1000); @@ -681,17 +632,8 @@ bool generateDsymCompanion( OutFile << EHFrameData; assert(OutFile.tell() == EHFrameStart + EHFrameSize); - // Transfer pseudo probe. - if (PseudoProbeSize > 0) { - OutFile.write_zeros(PseudoProbeStart - (EHFrameStart + EHFrameSize)); - assert(OutFile.tell() == PseudoProbeStart); - OutFile << PseudoProbeProbesData; - OutFile << PseudoProbeDescsData; - assert(OutFile.tell() == PseudoProbeStart + PseudoProbeSize); - } - // Pad till the Dwarf segment start. - OutFile.write_zeros(DwarfSegmentStart - OutFile.tell()); + OutFile.write_zeros(DwarfSegmentStart - (EHFrameStart + EHFrameSize)); assert(OutFile.tell() == DwarfSegmentStart); // Emit the Dwarf sections contents. From 009e841dbde021aa59bd7357269aeabfcc4f6128 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Tue, 21 Jul 2026 13:41:14 -0700 Subject: [PATCH 30/50] [flang][cuda] Add option for allocator registration function to CUFAddConstructor (#211098) This allows to register different allocators when needed. --- flang/include/flang/Optimizer/Transforms/Passes.td | 4 ++++ .../Optimizer/Transforms/CUDA/CUFAddConstructor.cpp | 10 +++++++--- flang/test/Fir/CUDA/cuda-constructor.f90 | 8 ++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td index d6cbb26882df5..8de944277e758 100644 --- a/flang/include/flang/Optimizer/Transforms/Passes.td +++ b/flang/include/flang/Optimizer/Transforms/Passes.td @@ -546,6 +546,10 @@ def CUFAddConstructor : Pass<"cuf-add-constructor", "mlir::ModuleOp"> { "cuf::CUFDialect", "mlir::func::FuncDialect", "mlir::DLTIDialect" ]; let options = [ + Option<"allocatorRegistrationFunction", + "allocator-registration-function", "std::string", /*default=*/"", + "Name of the function to call to register CUDA Fortran allocators. " + "Defaults to CUFRegisterAllocator.">, Option<"cudaUnified", "cuda-unified", "bool", /*default=*/"false", "Treat host module globals as unified memory (-gpu=mem:unified): " "register every plain host module global that is mirrored in the " diff --git a/flang/lib/Optimizer/Transforms/CUDA/CUFAddConstructor.cpp b/flang/lib/Optimizer/Transforms/CUDA/CUFAddConstructor.cpp index d53f1729ea6fc..6f2c17db5eb1d 100644 --- a/flang/lib/Optimizer/Transforms/CUDA/CUFAddConstructor.cpp +++ b/flang/lib/Optimizer/Transforms/CUDA/CUFAddConstructor.cpp @@ -302,13 +302,17 @@ struct CUFAddConstructor builder.setInsertionPointToStart(entryBlock); if (needAllocatorRegistration) { - // Symbol reference to CUFRegisterAllocator. + llvm::StringRef allocatorRegistrationFunctionName = + RTNAME_STRING(CUFRegisterAllocator); + if (!allocatorRegistrationFunction.empty()) + allocatorRegistrationFunctionName = allocatorRegistrationFunction; + // Symbol reference to the allocator registration function. builder.setInsertionPointToEnd(mod.getBody()); auto registerFuncOp = mlir::LLVM::LLVMFuncOp::create( - builder, loc, RTNAME_STRING(CUFRegisterAllocator), funcTy); + builder, loc, allocatorRegistrationFunctionName, funcTy); registerFuncOp.setVisibility(mlir::SymbolTable::Visibility::Private); auto cufRegisterAllocatorRef = mlir::SymbolRefAttr::get( - mod.getContext(), RTNAME_STRING(CUFRegisterAllocator)); + mod.getContext(), allocatorRegistrationFunctionName); builder.setInsertionPointToStart(entryBlock); mlir::LLVM::CallOp::create(builder, loc, funcTy, cufRegisterAllocatorRef); } diff --git a/flang/test/Fir/CUDA/cuda-constructor.f90 b/flang/test/Fir/CUDA/cuda-constructor.f90 index e2f2ba88e9205..f81d26ccd78b5 100644 --- a/flang/test/Fir/CUDA/cuda-constructor.f90 +++ b/flang/test/Fir/CUDA/cuda-constructor.f90 @@ -1,4 +1,7 @@ ! RUN: bbc -fcuda -emit-fir %s -o - | fir-opt --cuf-add-constructor | FileCheck %s +! RUN: bbc -fcuda -emit-fir %s -o - | fir-opt \ +! RUN: --cuf-add-constructor="allocator-registration-function=custom_register" \ +! RUN: | FileCheck %s --check-prefix=CUSTOM program main real, allocatable, device :: ahost(:) @@ -10,3 +13,8 @@ program main ! CHECK: } ! CHECK: llvm.func @_FortranACUFRegisterAllocator() attributes {sym_visibility = "private"} ! CHECK: llvm.mlir.global_ctors ctors = [@__cudaFortranConstructor], priorities = [0 : i32], data = [#llvm.zero] + +! CUSTOM-LABEL: llvm.func internal @__cudaFortranConstructor() { +! CUSTOM: llvm.call @custom_register() : () -> () +! CUSTOM: } +! CUSTOM: llvm.func @custom_register() attributes {sym_visibility = "private"} From db816fe2ed4e0be51192d3030d90a176d5fec9d5 Mon Sep 17 00:00:00 2001 From: Vladislav Dzhidzhoev Date: Tue, 21 Jul 2026 22:41:48 +0200 Subject: [PATCH 31/50] [DirectX][Driver] Add /Qpdb_in_private flag support (#204903) In DXC, when `/Qpdb_in_private` flag is specified, debug info PDB is emitted into PRIV part of the output DXContainer (as well as into separate PDB file, if its name is specified with `/Fd`). This patch reimplements similar behavior in llc and Clang. MC is modified to be able to emit PRIV part. Depends on https://github.com/llvm/llvm-project/pull/204166. --- clang/include/clang/Options/Options.td | 3 + clang/lib/Driver/ToolChains/Clang.cpp | 4 + clang/test/Driver/dxc_debug.hlsl | 2 + clang/test/Driver/dxc_section_emission.hlsl | 11 +++ llvm/lib/MC/MCDXContainerWriter.cpp | 22 +++-- .../lib/Target/DirectX/DXContainerGlobals.cpp | 34 ++++---- llvm/lib/Target/DirectX/DXContainerPDB.cpp | 46 +++++++++- .../ContainerData/PdbInPrivate-no-fd.test | 17 ++++ .../ContainerData/PdbInPrivate-no-flag.test | 5 ++ .../ContainerData/PdbInPrivate-no-pdbname.ll | 10 +++ .../DirectX/ContainerData/PdbInPrivate.ll | 18 ++++ .../DirectX/ContainerData/PdbInPrivate.test | 33 ++++++++ llvm/unittests/MC/CMakeLists.txt | 2 + llvm/unittests/MC/DXContainerWriterTest.cpp | 83 +++++++++++++++++++ 14 files changed, 267 insertions(+), 23 deletions(-) create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate-no-fd.test create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate-no-flag.test create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate-no-pdbname.ll create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate.ll create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate.test create mode 100644 llvm/unittests/MC/DXContainerWriterTest.cpp diff --git a/clang/include/clang/Options/Options.td b/clang/include/clang/Options/Options.td index c306179a084d3..669b6e64725c9 100644 --- a/clang/include/clang/Options/Options.td +++ b/clang/include/clang/Options/Options.td @@ -9923,6 +9923,9 @@ def dxc_gis : DXCFlag<"Gis">, def : Option<["/", "-"], "Qembed_debug", KIND_FLAG>, Group, Flags<[Ignored]>, Visibility<[DXCOption]>, HelpText<"Embed PDB in shader container (ignored)">; +def dxc_Qpdb_in_private : DXCFlag<"Qpdb_in_private">, + Flags<[HelpHidden]>, + HelpText<"Store PDB in private user data">; def spirv : DXCFlag<"spirv">, HelpText<"Generate SPIR-V code">; def metal : DXCFlag<"metal">, HelpText<"Generate Metal library">; diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index 10a1c0e8dec3f..1178f625d1172 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -4003,6 +4003,10 @@ static void RenderHLSLOptions(const Driver &D, const ArgList &Args, CmdArgs.push_back("-mllvm"); CmdArgs.push_back("--dx-strip-debug"); } + if (Args.hasArg(options::OPT_dxc_Qpdb_in_private)) { + CmdArgs.push_back("-mllvm"); + CmdArgs.push_back("--dx-pdb-in-private"); + } } static void RenderOpenACCOptions(const Driver &D, const ArgList &Args, diff --git a/clang/test/Driver/dxc_debug.hlsl b/clang/test/Driver/dxc_debug.hlsl index 90a02f1d1c1c4..b4a491fa6e1b1 100644 --- a/clang/test/Driver/dxc_debug.hlsl +++ b/clang/test/Driver/dxc_debug.hlsl @@ -3,6 +3,7 @@ // RUN: %clang_dxc -Tlib_6_7 -### /Zi /Qembed_debug %s 2>&1 | FileCheck %s // RUN: %clang_dxc -Tlib_6_7 -### -Zi %s 2>&1 | FileCheck %s // RUN: %clang_dxc -Tlib_6_7 -### -Zi -Qembed_debug %s 2>&1 | FileCheck %s +// RUN: %clang_dxc -Tlib_6_7 -### /Zi /Qpdb_in_private %s 2>&1 | FileCheck %s -check-prefixes=CHECK,CHECK-PRIV // RUN: %clang_dxc -Tlib_6_7 -### -Zi -Zss %s 2>&1 | FileCheck %s --check-prefix=CHECK,CHECK-ZSS // RUN: %clang_dxc -Tlib_6_7 -### /Zi -Qsource_in_debug_module %s 2>&1 | FileCheck %s --check-prefix=CHECK,CHECK-SIDM // RUN: %clang_dxc -Tlib_6_7 -### -Zi -Qstrip_debug %s 2>&1 | FileCheck %s --check-prefix=CHECK,CHECK-STRIP @@ -17,6 +18,7 @@ // Make sure dwarf-version is 4. // CHECK-DWARF-SAME: -dwarf-version=4 // Check that the flags are converted to their llc equivalents. +// CHECK-PRIV-SAME: --dx-pdb-in-private // CHECK-ZSS-SAME: -dx-Zss // CHECK-SIDM-SAME: --dx-source-in-debug-module // CHECK-STRIP-SAME: --dx-strip-debug diff --git a/clang/test/Driver/dxc_section_emission.hlsl b/clang/test/Driver/dxc_section_emission.hlsl index b27232f6b7d27..be8081e884c78 100644 --- a/clang/test/Driver/dxc_section_emission.hlsl +++ b/clang/test/Driver/dxc_section_emission.hlsl @@ -22,4 +22,15 @@ // CHECK: - Name: ILDN +// Check that /Qpdb_in_private emits a PRIV part in the output container. +// RUN: %clang_dxc -Tlib_6_7 /Fo %t-priv.dxbc /Zi /Qpdb_in_private %s 2>&1 +// RUN: obj2yaml %t-priv.dxbc | FileCheck %s --check-prefix=CHECK-PRIV + +// Without /Qpdb_in_private, PRIV is not emitted. +// RUN: %clang_dxc -Tlib_6_7 /Fo %t-no-priv.dxbc /Zi %s 2>&1 +// RUN: obj2yaml %t-no-priv.dxbc | FileCheck %s --check-prefix=CHECK-NO-PRIV + +// CHECK-PRIV: - Name: PRIV +// CHECK-NO-PRIV-NOT: - Name: PRIV + [numthreads(1, 1, 1)] void main() {} diff --git a/llvm/lib/MC/MCDXContainerWriter.cpp b/llvm/lib/MC/MCDXContainerWriter.cpp index d15086e101fb3..1bbdf52920900 100644 --- a/llvm/lib/MC/MCDXContainerWriter.cpp +++ b/llvm/lib/MC/MCDXContainerWriter.cpp @@ -49,14 +49,22 @@ void MCDXContainerBaseWriter::write(raw_ostream &OS, const Triple &TT) { // 16 part offsets gives us a little room for growth. llvm::SmallVector PartOffsets; uint64_t PartOffset = 0; + bool HasPrivate = false; for (const MCDXContainerPart &Part : Parts) { + if (HasPrivate) + reportFatalInternalError( + "PRIV must be the last section in a DXContainer"); + if (Part.Name == "PRIV") + HasPrivate = true; + uint64_t SectionSize = Part.Data.size(); assert(SectionSize < std::numeric_limits::max() && "Section size too large for DXContainer"); PartOffsets.push_back(PartOffset); PartOffset += sizeof(dxbc::PartHeader) + SectionSize; - PartOffset = alignTo(PartOffset, Align(4ul)); + if (!HasPrivate) + PartOffset = alignTo(PartOffset, Align(4ul)); // The DXIL part also writes a program header, so we need to include its // size when computing the offset for a part after the DXIL part. if (dxbc::isProgramPart(Part.Name)) @@ -96,8 +104,10 @@ void MCDXContainerBaseWriter::write(raw_ostream &OS, const Triple &TT) { if (dxbc::isProgramPart(Part.Name)) PartSize += sizeof(dxbc::ProgramHeader); - // DXContainer parts should be 4-byte aligned. - PartSize = alignTo(PartSize, Align(4)); + // DXContainer part should be 4-byte aligned, unless it is PRIV part. + bool IsPrivate = Part.Name == "PRIV"; + if (!IsPrivate) + PartSize = alignTo(PartSize, Align(4)); W.write(static_cast(PartSize)); if (dxbc::isProgramPart(Part.Name)) { dxbc::ProgramHeader Header; @@ -127,8 +137,10 @@ void MCDXContainerBaseWriter::write(raw_ostream &OS, const Triple &TT) { sizeof(dxbc::ProgramHeader))); } W.write(Part.Data); - unsigned Size = W.OS.tell() - Start; - W.OS.write_zeros(offsetToAlignment(Size, Align(4))); + if (!IsPrivate) { + unsigned Size = W.OS.tell() - Start; + W.OS.write_zeros(offsetToAlignment(Size, Align(4))); + } } } diff --git a/llvm/lib/Target/DirectX/DXContainerGlobals.cpp b/llvm/lib/Target/DirectX/DXContainerGlobals.cpp index f9cf43e67f1fb..1753b3e3f3a1a 100644 --- a/llvm/lib/Target/DirectX/DXContainerGlobals.cpp +++ b/llvm/lib/Target/DirectX/DXContainerGlobals.cpp @@ -43,6 +43,8 @@ static cl::opt ShaderHashDependsOnSource( "dx-Zss", cl::desc("Compute Shader Hash considering source information")); extern cl::opt PdbDebugPath; extern cl::opt SourceInDebugModule; +cl::opt PdbInPrivate("dx-pdb-in-private", + cl::desc("Store PDB in private user data")); namespace { class DXContainerGlobals : public llvm::ModulePass { @@ -159,22 +161,26 @@ void DXContainerGlobals::computeShaderHashAndDebugName( SmallString<40> DebugNameStr; Digest.stringifyResult(MD5, DebugNameStr); DebugNameStr += ".pdb"; - if (!PdbDebugPath.empty()) { - StringRef DebugFile = PdbDebugPath.getValue(); - SmallString<256> AbsoluteDebugName; - if (sys::path::is_separator(DebugFile.back())) { - // If /Fd was specified as a directory, put the MD5.pdb file there. - AbsoluteDebugName = DebugFile; - sys::path::append(AbsoluteDebugName, DebugNameStr); - } else { - // Otherwise, use /Fd value as a user-provided PDB file name. - DebugNameStr = DebugFile; - AbsoluteDebugName = DebugNameStr; + if (!PdbDebugPath.empty() || PdbInPrivate) { + if (!PdbDebugPath.empty()) { + StringRef DebugFile = PdbDebugPath.getValue(); + SmallString<256> AbsoluteDebugName; + if (sys::path::is_separator(DebugFile.back())) { + // If PDB output path was specified as a directory, put the MD5.pdb file + // there. + AbsoluteDebugName = DebugFile; + sys::path::append(AbsoluteDebugName, DebugNameStr); + } else { + // Otherwise, use PDB output path as a user-provided PDB file name. + DebugNameStr = DebugFile; + AbsoluteDebugName = DebugNameStr; + } + + // Pass PDB name to DXContainerPDBPass via PDBNAME section. + addSection(M, Globals, AbsoluteDebugName, "dx.pdb.name", + PdbFileNameSectionName); } - // Pass PDB name to DXContainerPDBPass via PDBNAME section. - addSection(M, Globals, AbsoluteDebugName, "dx.pdb.name", - PdbFileNameSectionName); // Pass module hash to DXContainerPDBPass. Globals.emplace_back(buildContainerGlobal( M, ConstantDataArray::get(M.getContext(), ArrayRef(HashData.Digest)), diff --git a/llvm/lib/Target/DirectX/DXContainerPDB.cpp b/llvm/lib/Target/DirectX/DXContainerPDB.cpp index dc3c05376b2c1..13f57baaaa032 100644 --- a/llvm/lib/Target/DirectX/DXContainerPDB.cpp +++ b/llvm/lib/Target/DirectX/DXContainerPDB.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "DirectX.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/StringSet.h" #include "llvm/BinaryFormat/DXContainer.h" #include "llvm/DebugInfo/CodeView/GUID.h" @@ -17,10 +18,15 @@ #include "llvm/IR/Module.h" #include "llvm/MC/MCDXContainerWriter.h" #include "llvm/Pass.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" #include "llvm/Support/IOSandbox.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" using namespace llvm; +extern cl::opt PdbInPrivate; + namespace { class DXContainerPDB : public ModulePass, MCDXContainerBaseWriter { @@ -85,10 +91,22 @@ ArrayRef DXContainerPDB::collectParts() { return Parts; } +static GlobalVariable *createPrivateDataGlobal(Module &M, StringRef Data) { + Constant *Content = + ConstantDataArray::getString(M.getContext(), Data, /*AddNull*/ false); + auto *GV = + new GlobalVariable(M, Content->getType(), true, + GlobalValue::PrivateLinkage, Content, "dx.priv"); + GV->setSection("PRIV"); + GV->setAlignment(Align(1)); + return GV; +} + bool DXContainerPDB::runOnModule(Module &M) { + llvm::scope_exit Cleanup([&]() { reset(); }); this->M = &M; - StringRef DebugFileName; + SmallString<128> DebugFileName; ArrayRef ModuleHash; for (const GlobalVariable &GV : M.globals()) { if (GV.getSection() == PdbFileNameSectionName) { @@ -102,11 +120,23 @@ bool DXContainerPDB::runOnModule(Module &M) { } // PDB emission was not requested. - if (DebugFileName.empty()) + if (DebugFileName.empty() && !PdbInPrivate) return false; if (ModuleHash.empty()) report_fatal_error("Module hash for PDB not found"); + bool DeleteAfterRead = false; + if (DebugFileName.empty()) { + if (std::error_code EC = + sys::fs::createTemporaryFile("dxil", "pdb", DebugFileName)) + reportFatalInternalError("Failed to create temporary PDB file"); + DeleteAfterRead = true; + } + llvm::scope_exit FileCleanup([&]() { + if (DeleteAfterRead) + sys::fs::remove(DebugFileName); + }); + BumpPtrAllocator Allocator; pdb::PDBFileBuilder Builder(Allocator); @@ -151,9 +181,17 @@ bool DXContainerPDB::runOnModule(Module &M) { reportFatalUsageError("Couldn't write to PDB file: " + Twine(toString(std::move(Err)))); - reset(); + if (!PdbInPrivate) + return false; + + ErrorOr> Buf = MemoryBuffer::getFile( + DebugFileName, /*IsText=*/false, /*RequiresNullTerminator=*/false); + if (!Buf) + reportFatalInternalError("Failed to read PDB for PRIV embedding"); + + appendToCompilerUsed(M, createPrivateDataGlobal(M, (*Buf)->getBuffer())); - return false; + return true; } char DXContainerPDB::ID = 0; diff --git a/llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate-no-fd.test b/llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate-no-fd.test new file mode 100644 index 0000000000000..c7e5cf3e340e2 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate-no-fd.test @@ -0,0 +1,17 @@ +## Check that --dx-pdb-in-private generates a PDB in PRIV without --dx-pdb-path. + +# RUN: llc %S/Inputs/SourceInfo.ll --filetype=obj -o %t.dxbc --dx-pdb-in-private +# RUN: obj2yaml %t.dxbc | FileCheck %s +# RUN: llvm-objcopy --dump-section=PRIV=%t.priv %t.dxbc +# RUN: llvm-pdbutil pdb2yaml --dxcontainer %t.priv | FileCheck %s --check-prefix=PDB + +# CHECK: - Name: PRIV +# CHECK-NEXT: Size: + +# PDB: PartCount: 5 +# PDB: Parts: +# PDB-DAG: - Name: ILDB +# PDB-DAG: - Name: HASH +# PDB-DAG: - Name: ILDN +# PDB-DAG: - Name: VERS +# PDB-DAG: - Name: SRCI diff --git a/llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate-no-flag.test b/llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate-no-flag.test new file mode 100644 index 0000000000000..4a185ed7842dd --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate-no-flag.test @@ -0,0 +1,5 @@ +## Check that PRIV is not emitted unless --dx-pdb-in-private is specified. + +# RUN: llc %S/Inputs/SourceInfo.ll --filetype=obj -o - | obj2yaml | FileCheck %s --implicit-check-not='PRIV' + +# CHECK: - Name: DXIL diff --git a/llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate-no-pdbname.ll b/llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate-no-pdbname.ll new file mode 100644 index 0000000000000..f07d19416b267 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate-no-pdbname.ll @@ -0,0 +1,10 @@ +;; Check that dxil-pdb can emit PRIV using a temporary PDB when PDBNAME is absent. +; RUN: opt %s -dxil-pdb --dx-pdb-in-private -S -o - | FileCheck %s + +;; CHECK: @dx.priv = private constant {{.*}} section "PRIV" + +target triple = "dxilv1.3-pc-shadermodel6.3-library" + +@dx.ildb = private constant [4 x i8] c"BC\C0\DE", section "ILDB", align 4 +@dx.pdb.hash = private constant [16 x i8] c"dummymodulehash!", section "PDBHASH", align 4 +@llvm.compiler.used = appending global [2 x ptr] [ptr @dx.ildb, ptr @dx.pdb.hash], section "llvm.metadata" diff --git a/llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate.ll b/llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate.ll new file mode 100644 index 0000000000000..7edc3fd8918d1 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate.ll @@ -0,0 +1,18 @@ +; RUN: opt %s -dxil-pdb --dx-pdb-in-private -S -o - | FileCheck %s --check-prefix=CHECK-IR +; RUN: opt %s -dxil-pdb --dx-pdb-in-private -o /dev/null +; RUN: llvm-pdbutil pdb2yaml --dxcontainer PdbInPrivateTest.pdb | FileCheck %s --check-prefix=CHECK-PDB + +;; Check that dxil-pdb pass emits a PRIV global when --dx-pdb-in-private is set. +; CHECK-IR: @dx.priv = private constant {{.*}} section "PRIV" + +; Check that the companion PDB file is still written when PDBNAME is present. +; CHECK-PDB: PartCount: 1 +; CHECK-PDB: Parts: +; CHECK-PDB: - Name: ILDB + +target triple = "dxilv1.3-pc-shadermodel6.3-library" + +@dx.ildb = private constant [4 x i8] c"BC\C0\DE", section "ILDB", align 4 +@dx.pdb.name = private constant [20 x i8] c"PdbInPrivateTest.pdb", section "PDBNAME", align 4 +@dx.pdb.hash = private constant [16 x i8] c"dummymodulehash!", section "PDBHASH", align 4 +@llvm.compiler.used = appending global [3 x ptr] [ptr @dx.ildb, ptr @dx.pdb.name, ptr @dx.pdb.hash], section "llvm.metadata" diff --git a/llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate.test b/llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate.test new file mode 100644 index 0000000000000..7c370ea0da02a --- /dev/null +++ b/llvm/test/CodeGen/DirectX/ContainerData/PdbInPrivate.test @@ -0,0 +1,33 @@ +## Check that --dx-pdb-in-private stores the generated PDB in the PRIV part. + +## File: --dx-pdb-path specifies the PDB output path. + +# RUN: llc %S/Inputs/SourceInfo.ll --filetype=obj -o %t.dxbc --dx-pdb-in-private --dx-pdb-path=%t.pdb +# RUN: obj2yaml %t.dxbc | FileCheck %s +# RUN: llvm-objcopy --dump-section=PRIV=%t.priv %t.dxbc +# RUN: diff %t.pdb %t.priv +# RUN: llvm-pdbutil pdb2yaml --dxcontainer %t.priv | FileCheck %s --check-prefix=PDB + +## Directory: --dx-pdb-path ending in '/' writes .pdb under that directory. + +# RUN: rm -rf %t.dir && mkdir %t.dir +# RUN: llc %S/Inputs/SourceInfo.ll --filetype=obj -o %t.dir.dxbc --dx-pdb-in-private --dx-pdb-path=%t.dir/ +# RUN: obj2yaml %t.dir.dxbc | FileCheck %s +# RUN: llvm-objcopy --dump-section=PRIV=%t.dir.priv %t.dir.dxbc +# RUN: llvm-objcopy --dump-section=DXIL=%t.dir.bc %t.dir.dxbc +# RUN: %md5sum %t.dir.bc >%t.dir.bc.md5 +# RUN: %python %S/Inputs/check_pdb_exists.py %t.dir %t.dir.bc.md5 +# RUN: cat %t.dir/*.pdb > %t.dir.pdb +# RUN: diff %t.dir.pdb %t.dir.priv +# RUN: llvm-pdbutil pdb2yaml --dxcontainer %t.dir.priv | FileCheck %s --check-prefix=PDB + +# CHECK: - Name: PRIV +# CHECK-NEXT: Size: + +# PDB: PartCount: 5 +# PDB: Parts: +# PDB-DAG: - Name: ILDB +# PDB-DAG: - Name: HASH +# PDB-DAG: - Name: ILDN +# PDB-DAG: - Name: VERS +# PDB-DAG: - Name: SRCI diff --git a/llvm/unittests/MC/CMakeLists.txt b/llvm/unittests/MC/CMakeLists.txt index 4958396c731b8..789fdb1722197 100644 --- a/llvm/unittests/MC/CMakeLists.txt +++ b/llvm/unittests/MC/CMakeLists.txt @@ -6,6 +6,7 @@ endforeach() set(LLVM_LINK_COMPONENTS ${LLVM_TARGETS_TO_BUILD} + BinaryFormat DebugInfoDWARF MC MCDisassembler @@ -19,6 +20,7 @@ add_llvm_unittest(MCTests DwarfDebugFrameCIE.cpp DwarfLineTables.cpp DwarfLineTableHeaders.cpp + DXContainerWriterTest.cpp MCInstPrinter.cpp StringTableBuilderTest.cpp TargetRegistry.cpp diff --git a/llvm/unittests/MC/DXContainerWriterTest.cpp b/llvm/unittests/MC/DXContainerWriterTest.cpp new file mode 100644 index 0000000000000..3641e873dbf4f --- /dev/null +++ b/llvm/unittests/MC/DXContainerWriterTest.cpp @@ -0,0 +1,83 @@ +//===- DXContainerWriterTest.cpp - MCDXContainerWriter tests --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/MC/MCDXContainerWriter.h" +#include "llvm/Object/DXContainer.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBufferRef.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TargetParser/Triple.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::object; + +namespace { + +class TestDXContainerWriter : public MCDXContainerBaseWriter { + SmallVector> PartNames; + SmallVector> PartData; + SmallVector Parts; + +protected: + ArrayRef collectParts() override { return Parts; } + +public: + void addPart(StringRef Name, ArrayRef Data) { + PartNames.emplace_back(Name); + PartData.emplace_back(Data.begin(), Data.end()); + Parts.push_back({PartNames.back(), PartData.back()}); + } +}; + +static Triple getTestTriple() { + return Triple("dxilv1.3-pc-shadermodel6.3-library"); +} + +static std::string writeContainer(TestDXContainerWriter &Writer) { + std::string Buffer; + raw_string_ostream OS(Buffer); + Writer.write(OS, getTestTriple()); + return Buffer; +} + +static DXContainer parseContainer(StringRef Buffer) { + return llvm::cantFail(DXContainer::create(MemoryBufferRef(Buffer, ""))); +} + +TEST(MCDXContainerWriterTest, PrivUnaligned) { + TestDXContainerWriter Writer; + const uint8_t PrivData[] = {0xDE, 0xAD, 0xBE, 0xEF, 0x42}; + Writer.addPart("PRIV", PrivData); + + std::string Buffer = writeContainer(Writer); + DXContainer C = parseContainer(Buffer); + + EXPECT_EQ(C.getHeader().PartCount, 1u); + EXPECT_EQ(C.getHeader().FileSize, 49u); + EXPECT_EQ(C.getData().size(), 49u); + + ASSERT_TRUE(C.getPrivateData()); + EXPECT_EQ(C.getPrivateData()->size(), 5u); + EXPECT_EQ( + *C.getPrivateData(), + StringRef(reinterpret_cast(PrivData), sizeof(PrivData))); +} + +TEST(MCDXContainerWriterTest, PrivMustBeLast) { + TestDXContainerWriter Writer; + const uint8_t PrivData[] = {0x42}; + const uint8_t DxilData[] = {0xBC, 0xC0, 0xDE, 0x00}; + Writer.addPart("PRIV", PrivData); + Writer.addPart("DXIL", DxilData); + + EXPECT_DEATH(writeContainer(Writer), + "PRIV must be the last section in a DXContainer"); +} + +} // namespace From 9a729e7fbbb6e1327e6af2148f0c941401b44cbf Mon Sep 17 00:00:00 2001 From: "forking-google-bazel-bot[bot]" <265904573+forking-google-bazel-bot[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:56:19 -0500 Subject: [PATCH 32/50] [Bazel] Fixes db816fe (#211109) This fixes db816fe2ed4e0be51192d3030d90a176d5fec9d5. Buildkite error link: https://buildkite.com/llvm-project/upstream-bazel/builds?commit=db816fe2ed4e0be51192d3030d90a176d5fec9d5 Co-authored-by: Google Bazel Bot --- utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel index aa61a743a5521..2786a9d2bd813 100644 --- a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel @@ -252,6 +252,7 @@ cc_test( "//llvm:Plugins", "//llvm:Support", "//llvm:Target", + "//llvm:TargetParser", "//llvm:TestingSupport", "//third-party/unittest:gtest", ], @@ -280,6 +281,7 @@ cc_test( "//llvm:MC", "//llvm:Support", "//llvm:Target", + "//llvm:TargetParser", "//third-party/unittest:gtest", ], ) @@ -317,6 +319,7 @@ cc_test( "//llvm:ObjectYAML", "//llvm:Support", "//llvm:Target", + "//llvm:TargetParser", "//llvm:TestingSupport", "//llvm:config", "//third-party/unittest:gtest", @@ -556,6 +559,7 @@ cc_test( "//llvm:Object", "//llvm:Support", "//llvm:Target", + "//llvm:TargetParser", "//third-party/unittest:gtest", "//third-party/unittest:gtest_main", ], @@ -578,6 +582,7 @@ cc_test( "//llvm:MC", "//llvm:Support", "//llvm:Target", + "//llvm:TargetParser", "//third-party/unittest:gtest", ], ) @@ -856,6 +861,7 @@ cc_test( "//llvm:MC", "//llvm:Support", "//llvm:Target", + "//llvm:TargetParser", "//third-party/unittest:gtest", "//third-party/unittest:gtest_main", ], From 3421dc247e4d6e87c6ab9b5db44c11873c28814f Mon Sep 17 00:00:00 2001 From: Jon Roelofs Date: Tue, 21 Jul 2026 14:22:36 -0700 Subject: [PATCH 33/50] [MC][AArch64] Add .cfi_set_ra_state for PAuth_LR (#209949) The new CFI directive directly assigns an RA signing state to the RA_SIGN_STATE DWARF pseudo-register for use when unwinding, to indicate whether the value of PC has been used as a diversifier for return address signing. The new directive subsumes and replaces .cfi_negate_ra_state_with_pc, which was found to be unsuitable for descibing some block layouts [1], particularly in hot-cold-split functions. 1: https://github.com/ARM-software/abi-aa/pull/346 --- bolt/lib/Core/BinaryFunction.cpp | 3 + llvm/include/llvm/BinaryFormat/Dwarf.def | 1 + llvm/include/llvm/CodeGen/CFIInstBuilder.h | 4 + .../DWARF/LowLevel/DWARFCFIProgram.h | 10 +- llvm/include/llvm/MC/MCDwarf.h | 46 +- llvm/include/llvm/MC/MCStreamer.h | 4 + .../CodeGen/AsmPrinter/AsmPrinterDwarf.cpp | 8 + llvm/lib/CodeGen/CFIInstrInserter.cpp | 1 + llvm/lib/CodeGen/MIRParser/MILexer.cpp | 1 + llvm/lib/CodeGen/MIRParser/MILexer.h | 1 + llvm/lib/CodeGen/MIRParser/MIParser.cpp | 23 + llvm/lib/CodeGen/MachineOperand.cpp | 11 + llvm/lib/DWARFCFIChecker/DWARFCFIState.cpp | 6 + llvm/lib/DebugInfo/DWARF/DWARFCFIPrinter.cpp | 9 + .../DWARF/LowLevel/DWARFCFIProgram.cpp | 16 + .../DWARF/LowLevel/DWARFUnwindTable.cpp | 11 + llvm/lib/MC/MCAsmStreamer.cpp | 22 + llvm/lib/MC/MCDwarf.cpp | 21 + llvm/lib/MC/MCSFrame.cpp | 1 + llvm/lib/MC/MCStreamer.cpp | 22 + .../lib/Target/AArch64/AArch64PointerAuth.cpp | 160 +++- .../AArch64/AsmParser/AArch64AsmParser.cpp | 31 + .../MCTargetDesc/AArch64AsmBackend.cpp | 3 + .../AArch64/sign-return-address-pauth-lr.ll | 822 +++++++++++++++++- .../CodeGen/AArch64/sign-return-address.ll | 346 ++++++++ llvm/test/CodeGen/MIR/AArch64/cfi.mir | 4 + .../MIR/AArch64/return-address-signing.mir | 4 +- .../MC/AArch64/cfi_set_ra_state-shrinkwrap.s | 142 +++ llvm/test/MC/AArch64/cfi_set_ra_state.s | 45 + .../MC/AArch64/directives-case_insensitive.s | 2 + .../DebugInfo/DWARF/DWARFDebugFrameTest.cpp | 1 + 31 files changed, 1736 insertions(+), 45 deletions(-) create mode 100644 llvm/test/MC/AArch64/cfi_set_ra_state-shrinkwrap.s create mode 100644 llvm/test/MC/AArch64/cfi_set_ra_state.s diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp index 3300b59066c16..a6722389e5d50 100644 --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -2874,6 +2874,7 @@ struct CFISnapshot { case MCCFIInstruction::OpLabel: case MCCFIInstruction::OpValOffset: case MCCFIInstruction::OpNegateRAState: + case MCCFIInstruction::OpLLVMSetRAState: reportFatalUsageError("unsupported CFI opcode"); case MCCFIInstruction::OpLLVMRegisterPair: case MCCFIInstruction::OpLLVMVectorRegisters: @@ -3019,6 +3020,7 @@ struct CFISnapshotDiff : public CFISnapshot { case MCCFIInstruction::OpLabel: case MCCFIInstruction::OpValOffset: case MCCFIInstruction::OpNegateRAState: + case MCCFIInstruction::OpLLVMSetRAState: reportFatalUsageError("unsupported CFI opcode"); case MCCFIInstruction::OpLLVMRegisterPair: case MCCFIInstruction::OpLLVMVectorRegisters: @@ -3174,6 +3176,7 @@ BinaryFunction::unwindCFIState(int32_t FromState, int32_t ToState, case MCCFIInstruction::OpLabel: case MCCFIInstruction::OpValOffset: case MCCFIInstruction::OpNegateRAState: + case MCCFIInstruction::OpLLVMSetRAState: reportFatalUsageError("unsupported CFI opcode"); case MCCFIInstruction::OpLLVMRegisterPair: case MCCFIInstruction::OpLLVMVectorRegisters: diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.def b/llvm/include/llvm/BinaryFormat/Dwarf.def index 8bb1766bcc259..c7ec6b673976a 100644 --- a/llvm/include/llvm/BinaryFormat/Dwarf.def +++ b/llvm/include/llvm/BinaryFormat/Dwarf.def @@ -1301,6 +1301,7 @@ HANDLE_DW_CFA(0x16, val_expression) // Vendor extensions: HANDLE_DW_CFA_PRED(0x1d, MIPS_advance_loc8, SELECT_MIPS64) HANDLE_DW_CFA_PRED(0x2d, GNU_window_save, SELECT_SPARC) +HANDLE_DW_CFA_PRED(0x2b, AARCH64_set_ra_state, SELECT_AARCH64) HANDLE_DW_CFA_PRED(0x2c, AARCH64_negate_ra_state_with_pc, SELECT_AARCH64) HANDLE_DW_CFA_PRED(0x2d, AARCH64_negate_ra_state, SELECT_AARCH64) HANDLE_DW_CFA_PRED(0x2e, GNU_args_size, SELECT_X86) diff --git a/llvm/include/llvm/CodeGen/CFIInstBuilder.h b/llvm/include/llvm/CodeGen/CFIInstBuilder.h index 8e22a5bacef6e..fa64ab6d6c23f 100644 --- a/llvm/include/llvm/CodeGen/CFIInstBuilder.h +++ b/llvm/include/llvm/CodeGen/CFIInstBuilder.h @@ -88,6 +88,10 @@ class CFIInstBuilder { insertCFIInst(MCCFIInstruction::createNegateRAStateWithPC(nullptr)); } + void buildSetRAState(unsigned State, MCSymbol *PACSym) const { + insertCFIInst(MCCFIInstruction::createSetRAState(nullptr, State, PACSym)); + } + void buildRegister(MCRegister Reg1, MCRegister Reg2) const { insertCFIInst(MCCFIInstruction::createRegister( nullptr, TRI.getDwarfRegNum(Reg1, IsEH), diff --git a/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.h b/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.h index 398da49aa861a..3500d5dd9d168 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.h +++ b/llvm/include/llvm/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.h @@ -122,6 +122,12 @@ class CFIProgram { // No operands addInstruction(Opcode); break; + case DW_CFA_AARCH64_set_ra_state: { + uint64_t RAState = Data.getULEB128(C); + uint64_t FactoredOffset = static_cast(Data.getSLEB128(C)); + addInstruction(Opcode, RAState, FactoredOffset); + break; + } case DW_CFA_set_loc: // Operands: Address addInstruction(Opcode, Data.getRelocatedAddress(C)); @@ -236,11 +242,13 @@ class CFIProgram { OT_Address, OT_Offset, OT_FactoredCodeOffset, + OT_SignedFactCodeOffset, OT_SignedFactDataOffset, OT_UnsignedFactDataOffset, OT_Register, OT_AddressSpace, - OT_Expression + OT_Expression, + OT_RAState, }; /// Get the OperandType as a "const char *". diff --git a/llvm/include/llvm/MC/MCDwarf.h b/llvm/include/llvm/MC/MCDwarf.h index 10a7035326036..3534a4f349331 100644 --- a/llvm/include/llvm/MC/MCDwarf.h +++ b/llvm/include/llvm/MC/MCDwarf.h @@ -527,6 +527,7 @@ class MCCFIInstruction { OpWindowSave, OpNegateRAState, OpNegateRAStateWithPC, + OpLLVMSetRAState, OpGnuArgsSize, OpLabel, OpValOffset, @@ -594,12 +595,24 @@ class MCCFIInstruction { unsigned MaskRegister; unsigned MaskRegisterSizeInBits; }; + /// Held in ExtraFields when OpLLVMSetRAState. + struct LLVMSetRAStateFields { + /// The ra_state value (DW_AARCH64_RA_NOT_SIGNED, DW_AARCH64_RA_SIGNED_SP, + /// or DW_AARCH64_RA_SIGNED_SP_PC). + unsigned State; + /// Symbol pointing to the signing instruction. + /// Precisely one of \p PACSym xor \p Offset should be set. + MCSymbol *PACSym; + /// Factored offset to the signing instruction. + /// Precisely one of \p PACSym xor \p Offset should be set. + int64_t Offset; + }; private: MCSymbol *Label; std::variant + VectorRegisterMaskFields, LLVMSetRAStateFields> ExtraFields; OpType Operation; SMLoc Loc; @@ -690,6 +703,22 @@ class MCCFIInstruction { return {OpNegateRAStateWithPC, L, CommonFields{}, Loc}; } + /// .cfi_set_ra_state AArch64 set RA sign state, + // with a symbolic offset to the signing instruction. + static MCCFIInstruction createSetRAState(MCSymbol *L, unsigned State, + MCSymbol *PACSym = nullptr, + SMLoc Loc = {}) { + return {OpLLVMSetRAState, L, LLVMSetRAStateFields{State, PACSym, 0}, Loc}; + } + + /// .cfi_set_ra_state AArch64 set RA sign state, + /// with a pre-computed factored offset to the signing instruction. + static MCCFIInstruction createSetRAState(MCSymbol *L, unsigned State, + int64_t Offset, SMLoc Loc = {}) { + return {OpLLVMSetRAState, L, LLVMSetRAStateFields{State, nullptr, Offset}, + Loc}; + } + /// .cfi_restore says that the rule for Register is now the same as it /// was at the beginning of the function, after all initial instructions added /// by .cfi_startproc were executed. @@ -831,6 +860,21 @@ class MCCFIInstruction { return std::get(ExtraFields).Offset; } + unsigned getRASignState() const { + assert(Operation == OpLLVMSetRAState); + return std::get(ExtraFields).State; + } + + MCSymbol *getRASignSymbol() const { + assert(Operation == OpLLVMSetRAState); + return std::get(ExtraFields).PACSym; + } + + int64_t getRASignOffset() const { + assert(Operation == OpLLVMSetRAState); + return std::get(ExtraFields).Offset; + } + MCSymbol *getCfiLabel() const { assert(Operation == OpLabel); return std::get(ExtraFields).CfiLabel; diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h index f1479f860e885..0facd10e92832 100644 --- a/llvm/include/llvm/MC/MCStreamer.h +++ b/llvm/include/llvm/MC/MCStreamer.h @@ -1062,6 +1062,10 @@ class LLVM_ABI MCStreamer { int64_t MaskRegisterSizeInBits, SMLoc Loc = {}); virtual void emitCFINegateRAStateWithPC(SMLoc Loc = {}); + virtual void emitCFILLVMSetRAState(unsigned State, MCSymbol *PACSym, + SMLoc Loc = {}); + virtual void emitCFILLVMSetRAState(unsigned State, int64_t Offset, + SMLoc Loc = {}); virtual void emitCFILabelDirective(SMLoc Loc, StringRef Name); virtual void emitCFIValOffset(int64_t Register, int64_t Offset, SMLoc Loc = {}); diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp index 36f186d312e9a..b0de5f563ac29 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp @@ -238,6 +238,14 @@ void AsmPrinter::emitCFIInstruction(const MCCFIInstruction &Inst) const { case MCCFIInstruction::OpNegateRAStateWithPC: OutStreamer->emitCFINegateRAStateWithPC(Loc); break; + case MCCFIInstruction::OpLLVMSetRAState: { + if (MCSymbol *Symbol = Inst.getRASignSymbol()) + OutStreamer->emitCFILLVMSetRAState(Inst.getRASignState(), Symbol, Loc); + else + OutStreamer->emitCFILLVMSetRAState(Inst.getRASignState(), + Inst.getRASignOffset(), Loc); + break; + } case MCCFIInstruction::OpSameValue: OutStreamer->emitCFISameValue(Inst.getRegister(), Loc); break; diff --git a/llvm/lib/CodeGen/CFIInstrInserter.cpp b/llvm/lib/CodeGen/CFIInstrInserter.cpp index b9a48188931de..ad81aaa5e276e 100644 --- a/llvm/lib/CodeGen/CFIInstrInserter.cpp +++ b/llvm/lib/CodeGen/CFIInstrInserter.cpp @@ -323,6 +323,7 @@ void CFIInstrInserter::calculateOutgoingCFAInfo(MBBCFAInfo &MBBInfo) { case MCCFIInstruction::OpWindowSave: case MCCFIInstruction::OpNegateRAState: case MCCFIInstruction::OpNegateRAStateWithPC: + case MCCFIInstruction::OpLLVMSetRAState: case MCCFIInstruction::OpGnuArgsSize: case MCCFIInstruction::OpLLVMRegisterPair: case MCCFIInstruction::OpLLVMVectorRegisters: diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.cpp b/llvm/lib/CodeGen/MIRParser/MILexer.cpp index cabe910ef28ca..720ec3f0f3295 100644 --- a/llvm/lib/CodeGen/MIRParser/MILexer.cpp +++ b/llvm/lib/CodeGen/MIRParser/MILexer.cpp @@ -242,6 +242,7 @@ static MIToken::TokenKind getIdentifierKind(StringRef Identifier) { MIToken::kw_cfi_aarch64_negate_ra_sign_state) .Case("negate_ra_sign_state_with_pc", MIToken::kw_cfi_aarch64_negate_ra_sign_state_with_pc) + .Case("llvm_set_ra_state", MIToken::kw_cfi_set_ra_state) .Case("llvm_register_pair", MIToken::kw_cfi_llvm_register_pair) .Case("llvm_vector_registers", MIToken::kw_cfi_llvm_vector_registers) .Case("llvm_vector_offset", MIToken::kw_cfi_llvm_vector_offset) diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.h b/llvm/lib/CodeGen/MIRParser/MILexer.h index 3922caf4220d3..9bca4af5187d6 100644 --- a/llvm/lib/CodeGen/MIRParser/MILexer.h +++ b/llvm/lib/CodeGen/MIRParser/MILexer.h @@ -99,6 +99,7 @@ struct MIToken { kw_cfi_window_save, kw_cfi_aarch64_negate_ra_sign_state, kw_cfi_aarch64_negate_ra_sign_state_with_pc, + kw_cfi_set_ra_state, kw_cfi_llvm_register_pair, kw_cfi_llvm_vector_registers, kw_cfi_llvm_vector_offset, diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp index f38c0d9e8baca..f14e2cd8ce3dc 100644 --- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp +++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp @@ -2856,6 +2856,28 @@ bool MIParser::parseCFIOperand(MachineOperand &Dest) { CFIIndex = MF.addFrameInst(MCCFIInstruction::createNegateRAStateWithPC(nullptr)); break; + case MIToken::kw_cfi_set_ra_state: { + unsigned State; + MCSymbol *PACSym = nullptr; + if (parseCFIUnsigned(State) || expectAndConsume(MIToken::comma)) + return true; + if (Token.is(MIToken::MCSymbol)) { + PACSym = getOrCreateMCSymbol(Token.stringValue()); + lex(); + CFIIndex = MF.addFrameInst( + MCCFIInstruction::createSetRAState(nullptr, State, PACSym)); + } else if (Token.is(MIToken::IntegerLiteral)) { + int Offset; + if (parseCFIOffset(Offset)) + return true; + CFIIndex = MF.addFrameInst( + MCCFIInstruction::createSetRAState(nullptr, State, Offset)); + } else { + return error("expected '' or integer offset for " + "cfi_set_ra_state"); + } + break; + } case MIToken::kw_cfi_llvm_register_pair: { unsigned Reg, R1, R2; unsigned R1Size, R2Size; @@ -3301,6 +3323,7 @@ bool MIParser::parseMachineOperand(const unsigned OpCode, const unsigned OpIdx, case MIToken::kw_cfi_window_save: case MIToken::kw_cfi_aarch64_negate_ra_sign_state: case MIToken::kw_cfi_aarch64_negate_ra_sign_state_with_pc: + case MIToken::kw_cfi_set_ra_state: case MIToken::kw_cfi_llvm_register_pair: case MIToken::kw_cfi_llvm_vector_registers: case MIToken::kw_cfi_llvm_vector_offset: diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp index 518759952667e..bf48bc7a1cbbb 100644 --- a/llvm/lib/CodeGen/MachineOperand.cpp +++ b/llvm/lib/CodeGen/MachineOperand.cpp @@ -790,6 +790,17 @@ static void printCFI(raw_ostream &OS, const MCCFIInstruction &CFI, if (MCSymbol *Label = CFI.getLabel()) MachineOperand::printSymbol(OS, *Label); break; + case MCCFIInstruction::OpLLVMSetRAState: { + OS << "llvm_set_ra_state "; + if (MCSymbol *Label = CFI.getLabel()) + MachineOperand::printSymbol(OS, *Label); + OS << CFI.getRASignState() << ", "; + if (MCSymbol *PACSym = CFI.getRASignSymbol()) + MachineOperand::printSymbol(OS, *PACSym); + else + OS << CFI.getRASignOffset(); + break; + } case MCCFIInstruction::OpLLVMRegisterPair: { const auto &Fields = CFI.getExtraFields(); diff --git a/llvm/lib/DWARFCFIChecker/DWARFCFIState.cpp b/llvm/lib/DWARFCFIChecker/DWARFCFIState.cpp index 606f5a520fd7d..4812ee35ddd65 100644 --- a/llvm/lib/DWARFCFIChecker/DWARFCFIState.cpp +++ b/llvm/lib/DWARFCFIChecker/DWARFCFIState.cpp @@ -150,6 +150,12 @@ dwarf::CFIProgram DWARFCFIState::convert(MCCFIInstruction Directive) { case MCCFIInstruction::OpNegateRAStateWithPC: CFIP.addInstruction(dwarf::DW_CFA_AARCH64_negate_ra_state_with_pc); break; + case MCCFIInstruction::OpLLVMSetRAState: { + CFIP.addInstruction(dwarf::DW_CFA_AARCH64_set_ra_state, + Directive.getRASignState(), + static_cast(Directive.getRASignOffset())); + break; + } case MCCFIInstruction::OpGnuArgsSize: CFIP.addInstruction(dwarf::DW_CFA_GNU_args_size); break; diff --git a/llvm/lib/DebugInfo/DWARF/DWARFCFIPrinter.cpp b/llvm/lib/DebugInfo/DWARF/DWARFCFIPrinter.cpp index 182b4956e5792..adc1c1ac48a15 100644 --- a/llvm/lib/DebugInfo/DWARF/DWARFCFIPrinter.cpp +++ b/llvm/lib/DebugInfo/DWARF/DWARFCFIPrinter.cpp @@ -83,6 +83,12 @@ static void printOperand(raw_ostream &OS, const DIDumpOptions &DumpOpts, else OS << formatv(" {0}*data_alignment_factor", int64_t(Operand)); break; + case CFIProgram::OT_SignedFactCodeOffset: + if (P.codeAlign()) + OS << formatv(" {0}", int64_t(Operand) * (int64_t)P.codeAlign()); + else + OS << formatv(" {0}*code_alignment_factor", int64_t(Operand)); + break; case CFIProgram::OT_UnsignedFactDataOffset: if (P.dataAlign()) OS << formatv(" {0}", int64_t(Operand * P.dataAlign())); @@ -93,6 +99,9 @@ static void printOperand(raw_ostream &OS, const DIDumpOptions &DumpOpts, OS << ' '; printRegister(OS, DumpOpts, Operand); break; + case CFIProgram::OT_RAState: + OS << ' ' << Operand; + break; case CFIProgram::OT_AddressSpace: OS << formatv(" in addrspace{0}", Operand); break; diff --git a/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.cpp b/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.cpp index eaf56b3071592..f8bc7dd2623f3 100644 --- a/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.cpp +++ b/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFCFIProgram.cpp @@ -32,11 +32,13 @@ const char *CFIProgram::operandTypeString(CFIProgram::OperandType OT) { ENUM_TO_CSTR(OT_Address); ENUM_TO_CSTR(OT_Offset); ENUM_TO_CSTR(OT_FactoredCodeOffset); + ENUM_TO_CSTR(OT_SignedFactCodeOffset); ENUM_TO_CSTR(OT_SignedFactDataOffset); ENUM_TO_CSTR(OT_UnsignedFactDataOffset); ENUM_TO_CSTR(OT_Register); ENUM_TO_CSTR(OT_AddressSpace); ENUM_TO_CSTR(OT_Expression); + ENUM_TO_CSTR(OT_RAState); } return ""; } @@ -61,6 +63,7 @@ CFIProgram::Instruction::getOperandAsUnsigned(const CFIProgram &CFIP, case OT_Offset: case OT_SignedFactDataOffset: case OT_UnsignedFactDataOffset: + case OT_SignedFactCodeOffset: return createStringError( errc::invalid_argument, "op[%" PRIu32 "] has OperandType OT_Offset which produces a signed " @@ -70,6 +73,7 @@ CFIProgram::Instruction::getOperandAsUnsigned(const CFIProgram &CFIP, case OT_Address: case OT_Register: case OT_AddressSpace: + case OT_RAState: return Operand; case OT_FactoredCodeOffset: { @@ -106,6 +110,7 @@ CFIProgram::Instruction::getOperandAsSigned(const CFIProgram &CFIP, case OT_Address: case OT_Register: case OT_AddressSpace: + case OT_RAState: return createStringError( errc::invalid_argument, "op[%" PRIu32 "] has OperandType %s which produces an unsigned result, " @@ -126,6 +131,16 @@ CFIProgram::Instruction::getOperandAsSigned(const CFIProgram &CFIP, return int64_t(Operand) * DataAlignmentFactor; } + case OT_SignedFactCodeOffset: { + const int64_t CodeAlignmentFactor = CFIP.codeAlign(); + if (CodeAlignmentFactor == 0) + return createStringError(errc::invalid_argument, + "op[%" PRIu32 "] has type %s but code " + "alignment is zero", + OperandIdx, CFIProgram::operandTypeString(Type)); + return int64_t(Operand) * CodeAlignmentFactor; + } + case OT_UnsignedFactDataOffset: { const int64_t DataAlignmentFactor = CFIP.dataAlign(); if (DataAlignmentFactor == 0) @@ -192,6 +207,7 @@ CFIProgram::getOperandTypes() { DECLARE_OP0(DW_CFA_restore_state); DECLARE_OP0(DW_CFA_GNU_window_save); DECLARE_OP0(DW_CFA_AARCH64_negate_ra_state_with_pc); + DECLARE_OP2(DW_CFA_AARCH64_set_ra_state, OT_RAState, OT_SignedFactCodeOffset); DECLARE_OP1(DW_CFA_GNU_args_size, OT_Offset); DECLARE_OP0(DW_CFA_nop); diff --git a/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFUnwindTable.cpp b/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFUnwindTable.cpp index 20f52673eefe1..94d62d176e638 100644 --- a/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFUnwindTable.cpp +++ b/llvm/lib/DebugInfo/DWARF/LowLevel/DWARFUnwindTable.cpp @@ -257,6 +257,17 @@ llvm::dwarf::parseRows(const CFIProgram &CFIP, UnwindRow &Row, break; } + case dwarf::DW_CFA_AARCH64_set_ra_state: { + constexpr uint32_t AArch64DWARFPAuthRaState = 34; + llvm::Expected State = Inst.getOperandAsUnsigned(CFIP, 0); + if (!State) + return State.takeError(); + Row.getRegisterLocations().setRegisterLocation( + AArch64DWARFPAuthRaState, + UnwindLocation::createIsConstant((int64_t)*State)); + break; + } + case dwarf::DW_CFA_undefined: { llvm::Expected RegNum = Inst.getOperandAsUnsigned(CFIP, 0); if (!RegNum) diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp index 037b44762875e..1a2c89c8cc055 100644 --- a/llvm/lib/MC/MCAsmStreamer.cpp +++ b/llvm/lib/MC/MCAsmStreamer.cpp @@ -493,6 +493,10 @@ class MCAsmStreamer final : public MCAsmBaseStreamer { void emitCFIWindowSave(SMLoc Loc) override; void emitCFINegateRAState(SMLoc Loc) override; void emitCFINegateRAStateWithPC(SMLoc Loc) override; + void emitCFILLVMSetRAState(unsigned State, MCSymbol *PACSym, + SMLoc Loc) override; + void emitCFILLVMSetRAState(unsigned State, int64_t Offset, + SMLoc Loc) override; void emitCFIReturnColumn(int64_t Register) override; void emitCFILLVMRegisterPair(int64_t Register, int64_t R1, int64_t R1Size, int64_t R2, int64_t R2Size, SMLoc Loc) override; @@ -2380,6 +2384,24 @@ void MCAsmStreamer::emitCFINegateRAStateWithPC(SMLoc Loc) { EmitEOL(); } +void MCAsmStreamer::emitCFILLVMSetRAState(unsigned State, MCSymbol *PACSym, + SMLoc Loc) { + MCStreamer::emitCFILLVMSetRAState(State, PACSym, Loc); + OS << "\t.cfi_set_ra_state " << State << ", "; + if (PACSym) + PACSym->print(OS, MAI); + else + OS << "0"; + EmitEOL(); +} + +void MCAsmStreamer::emitCFILLVMSetRAState(unsigned State, int64_t Offset, + SMLoc Loc) { + MCStreamer::emitCFILLVMSetRAState(State, Offset, Loc); + OS << "\t.cfi_set_ra_state " << State << ", " << Offset; + EmitEOL(); +} + void MCAsmStreamer::emitCFIReturnColumn(int64_t Register) { MCStreamer::emitCFIReturnColumn(Register); OS << "\t.cfi_return_column "; diff --git a/llvm/lib/MC/MCDwarf.cpp b/llvm/lib/MC/MCDwarf.cpp index 37ec324cdc499..b06ab36d1d66d 100644 --- a/llvm/lib/MC/MCDwarf.cpp +++ b/llvm/lib/MC/MCDwarf.cpp @@ -1330,6 +1330,9 @@ void MCCFIInstruction::replaceRegister(unsigned FromReg, unsigned ToReg) { ReplaceReg(F.Register); ReplaceReg(F.SpillRegister); ReplaceReg(F.MaskRegister); + }, + [](LLVMSetRAStateFields &) { + llvm_unreachable(".cfi_set_ra_state does not have registers"); }); std::visit(Visitor, ExtraFields); } @@ -1456,6 +1459,24 @@ void FrameEmitterImpl::emitCFIInstruction(const MCCFIInstruction &Instr) { Streamer.emitInt8(dwarf::DW_CFA_AARCH64_negate_ra_state_with_pc); return; + case MCCFIInstruction::OpLLVMSetRAState: { + Streamer.emitInt8(dwarf::DW_CFA_AARCH64_set_ra_state); + Streamer.emitULEB128IntValue(Instr.getRASignState()); + if (MCSymbol *PACSym = Instr.getRASignSymbol()) { + MCContext &Ctx = Streamer.getContext(); + const MCExpr *Diff = MCBinaryExpr::createSub( + MCSymbolRefExpr::create(PACSym, Ctx), + MCSymbolRefExpr::create(Instr.getLabel(), Ctx), Ctx); + const MCExpr *Factored = MCBinaryExpr::createDiv( + Diff, + MCConstantExpr::create(Ctx.getAsmInfo().getMinInstAlignment(), Ctx), + Ctx); + Streamer.emitSLEB128Value(Factored); + } else { + Streamer.emitSLEB128IntValue(Instr.getRASignOffset()); + } + return; + } case MCCFIInstruction::OpUndefined: { unsigned Reg = Instr.getRegister(); Streamer.emitInt8(dwarf::DW_CFA_undefined); diff --git a/llvm/lib/MC/MCSFrame.cpp b/llvm/lib/MC/MCSFrame.cpp index bdf2c95d5c071..5e5a6aaeb19f0 100644 --- a/llvm/lib/MC/MCSFrame.cpp +++ b/llvm/lib/MC/MCSFrame.cpp @@ -339,6 +339,7 @@ class SFrameEmitterImpl { case dwarf::DW_CFA_MIPS_advance_loc8: case dwarf::DW_CFA_AARCH64_negate_ra_state_with_pc: case dwarf::DW_CFA_AARCH64_negate_ra_state: + case dwarf::DW_CFA_AARCH64_set_ra_state: case dwarf::DW_CFA_LLVM_def_aspace_cfa: case dwarf::DW_CFA_LLVM_def_aspace_cfa_sf: Streamer.getContext().reportWarning( diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp index e8e88ce7b83c9..8798f093b24b5 100644 --- a/llvm/lib/MC/MCStreamer.cpp +++ b/llvm/lib/MC/MCStreamer.cpp @@ -748,6 +748,28 @@ void MCStreamer::emitCFINegateRAStateWithPC(SMLoc Loc) { CurFrame->Instructions.push_back(std::move(Instruction)); } +void MCStreamer::emitCFILLVMSetRAState(unsigned State, MCSymbol *PACSym, + SMLoc Loc) { + MCSymbol *Label = emitCFILabel(); + MCCFIInstruction Instruction = + MCCFIInstruction::createSetRAState(Label, State, PACSym, Loc); + MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo(); + if (!CurFrame) + return; + CurFrame->Instructions.push_back(std::move(Instruction)); +} + +void MCStreamer::emitCFILLVMSetRAState(unsigned State, int64_t Offset, + SMLoc Loc) { + MCSymbol *Label = emitCFILabel(); + MCCFIInstruction Instruction = + MCCFIInstruction::createSetRAState(Label, State, Offset, Loc); + MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo(); + if (!CurFrame) + return; + CurFrame->Instructions.push_back(std::move(Instruction)); +} + void MCStreamer::emitCFIReturnColumn(int64_t Register) { MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo(); if (!CurFrame) diff --git a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp index 7165c3bfed422..f527f7c844c22 100644 --- a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp +++ b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp @@ -25,6 +25,40 @@ using namespace llvm::AArch64PAuth; namespace { +/// Control the emission of .cfi_set_ra_state, which replaces the +/// deprecated .cfi_negate_ra_state_with_pc [1]. +/// +/// The latter is fundamentally unable to express some program orders [2], as +/// the dwarf 'program' reads functions in a linear scan of their addresses to +/// reconstruct the state of the frame, whereas control flow may enter and exit +/// such regions arbitrarily (such as in hot-cold-split, and shrinkwrapped +/// fucntions), and thus the negate-based cfi is unable to encode the address of +/// the signing instruciton in all program orders. +/// +/// Since .cfi_negate_ra_state is still sufficient for describing +/// ptrauth-returns=pauth, we default to using the new CFI only for PAuth_LR, as +/// DW_CFA_AARCH64_negate_ra_state has a smaller encoding than +/// DW_CFA_AARCH64_set_ra_state. +/// +/// 1: https://github.com/ARM-software/abi-aa/pull/346 +/// 2: https://github.com/ARM-software/abi-aa/issues/327 +enum class SetRAStateMode { + Never, // Always use .cfi_negate_ra_state(_with_pc) + PAuthLR, // Use .cfi_set_ra_state only for PAuth_LR + Always, // Use .cfi_set_ra_state for both PAuth and PAuth_LR +}; +cl::opt CFILLVMSetRASignStateMode( + "aarch64-cfi-llvm-set-ra-sign-state", cl::init(SetRAStateMode::Never), + cl::desc("Control emission of .cfi_set_ra_state for PAC return address " + "signing CFI"), + cl::values(clEnumValN(SetRAStateMode::Never, "never", + "Always use legacy .cfi_negate_ra_state[_with_pc]"), + clEnumValN(SetRAStateMode::PAuthLR, "pauth-lr", + "Use new CFI only for PAuth_LR (default)"), + clEnumValN(SetRAStateMode::Always, "always", + "Use new CFI for both PAuth and PAuth_LR")), + cl::Hidden); + class AArch64PointerAuthImpl { public: bool run(MachineFunction &MF); @@ -77,9 +111,17 @@ static void emitEpiloguePACSymOffsetIntoReg(const TargetInstrInfo &TII, } // Wrap a given PAC instruction in CFI that describes it. +// // Depending on the type of CFI required, we may need to emit the directive // either before or after the instruction, so that unwinders can correctly // interpret the location of the signing instruction. +// +// As a general rule, CFI opcodes describe the actions needed to recover the +// register state leading up to a not-yet-retired instruction, with one +// exception: .cfi_negate_ra_state_with_pc always comes before the paci[ab]sppc, +// since the unwinder uses the location of the CFI itself to derive the address +// of the signing instruction [1]. +// 1: https://github.com/llvm/llvm-project/pull/137795#issuecomment-2838779129 template static void decoratePACWithCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool EmitCFI, @@ -91,15 +133,37 @@ static void decoratePACWithCFI(MachineBasicBlock &MBB, auto &MF = *MBB.getParent(); auto &MFnI = *MF.getInfo(); - CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup); + const Triple &TT = MF.getTarget().getTargetTriple(); + if (MFnI.branchProtectionPAuthLR()) { - CFIBuilder.buildNegateRAStateWithPC(); - BuildPACMI(); + switch (CFILLVMSetRASignStateMode) { + case SetRAStateMode::Never: + CFIBuilder.buildNegateRAStateWithPC(); + BuildPACMI(); + break; + case SetRAStateMode::PAuthLR: + case SetRAStateMode::Always: { + BuildPACMI(); + MCSymbol *PACSym = MFnI.getSigningInstrLabel(); + assert(PACSym && "No PAC instruction to refer to"); + CFIBuilder.buildSetRAState(2, PACSym); + break; + } + } } else { - BuildPACMI(); - if (!MF.getTarget().getTargetTriple().isOSBinFormatMachO()) { - CFIBuilder.buildNegateRAState(); + switch (CFILLVMSetRASignStateMode) { + case SetRAStateMode::Never: + case SetRAStateMode::PAuthLR: + BuildPACMI(); + if (!TT.isOSBinFormatMachO()) { + CFIBuilder.buildNegateRAState(); + } + break; + case SetRAStateMode::Always: + BuildPACMI(); + CFIBuilder.buildSetRAState(1, nullptr); + break; } } } @@ -115,41 +179,57 @@ static void emitAUTCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const Triple &TT = MF.getTarget().getTargetTriple(); if (MFnI.branchProtectionPAuthLR()) { - // DW_CFA_AARCH64_negate_ra_state_with_pc is semantically broken for - // functions where shrinkwrapping places signing/authenticating pairs on - // distinct CFG paths. - // - // DWARF CFI is evaluated linearly over the byte stream, not along control - // flow edges. The toggle semantics of this directive therefore cannot - // faithfully represent the signed/unsigned RA state for all possible CFG - // paths. The added complexity versus DW_CFA_AARCH64_negate_ra_state is - // that an unwinder must also reconstruct the PC of the PACI[AB]SPPC in - // order to verify the signed LR, and that address is derived from the - // location of this directive in the linear CFI stream. - // - // The correct fix is to use DW_CFA_AARCH64_set_ra_state_with_pc, which - // sets the RA state and signing address absolutely rather than toggling - // them. An unwinder that supports this directive can reconstruct the - // correct state on any CFG path, regardless of how many - // signing/authenticating pairs exist in the function. However, not all - // unwinders support this directive, so we cannot rely on it exclusively. - // - // For unwinders that only support DW_CFA_AARCH64_negate_ra_state_with_pc, - // libunwind exploits a loophole: it records the address at the - // DW_CFA_AARCH64_negate_ra_state_with_pc site to authenticate the LR, but - // does not care that the CFI state remains "signed with pc" after - // authentication has occurred. This means we can safely omit the - // FrameDestroy emission of this directive, treating it solely as a marker - // for the signing site, as long as each function has at most one such - // signing location. That invariant holds today because shrinkwrapping - // does not yet hoist or sink PAuth_LR frame code across CFG join/split - // points; once it does, we must avoid those transformations on platforms - // that have this limitation. - // - // https://github.com/ARM-software/abi-aa/issues/327 - // https://github.com/ARM-software/abi-aa/pull/346 + switch (CFILLVMSetRASignStateMode) { + case SetRAStateMode::Never: + // DW_CFA_AARCH64_negate_ra_state_with_pc is semantically broken for + // functions where shrinkwrapping places signing/authenticating pairs on + // distinct CFG paths. + // + // DWARF CFI is evaluated linearly over the byte stream, not along control + // flow edges. The toggle semantics of this directive therefore cannot + // faithfully represent the signed/unsigned RA state for all possible CFG + // paths. The added complexity versus DW_CFA_AARCH64_negate_ra_state is + // that an unwinder must also reconstruct the PC of the PACI[AB]SPPC in + // order to verify the signed LR, and that address is derived from the + // location of this directive in the linear CFI stream. + // + // The correct fix is to use DW_CFA_AARCH64_set_ra_state_with_pc, which + // sets the RA state and signing address absolutely rather than toggling + // them. An unwinder that supports this directive can reconstruct the + // correct state on any CFG path, regardless of how many + // signing/authenticating pairs exist in the function. However, not all + // unwinders support this directive, so we cannot rely on it exclusively. + // + // For unwinders that only support DW_CFA_AARCH64_negate_ra_state_with_pc, + // libunwind exploits a loophole: it records the address at the + // DW_CFA_AARCH64_negate_ra_state_with_pc site to authenticate the LR, but + // does not care that the CFI state remains "signed with pc" after + // authentication has occurred. This means we can safely omit the + // FrameDestroy emission of this directive, treating it solely as a marker + // for the signing site, as long as each function has at most one such + // signing location. That invariant holds today because shrinkwrapping + // does not yet hoist or sink PAuth_LR frame code across CFG join/split + // points; once it does, we must avoid those transformations on platforms + // that have this limitation. + // + // https://github.com/ARM-software/abi-aa/issues/327 + // https://github.com/ARM-software/abi-aa/pull/346 + break; + case SetRAStateMode::PAuthLR: + case SetRAStateMode::Always: + CFIBuilder.buildSetRAState(0, nullptr); + break; + } } else if (!TT.isOSBinFormatMachO()) { - CFIBuilder.buildNegateRAState(); + switch (CFILLVMSetRASignStateMode) { + case SetRAStateMode::Never: + case SetRAStateMode::PAuthLR: + CFIBuilder.buildNegateRAState(); + break; + case SetRAStateMode::Always: + CFIBuilder.buildSetRAState(0, nullptr); + break; + } } } diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp index 4bb51c2e21934..b282f196d13f7 100644 --- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp +++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp @@ -201,6 +201,7 @@ class AArch64AsmParser : public MCTargetAsmParser { bool parseDirectiveUnreq(SMLoc L); bool parseDirectiveCFINegateRAState(); bool parseDirectiveCFINegateRAStateWithPC(); + bool parseDirectiveCFILLVMSetRAState(); bool parseDirectiveCFIBKeyFrame(); bool parseDirectiveCFIMTETaggedFrame(); @@ -7082,6 +7083,8 @@ bool AArch64AsmParser::ParseDirective(AsmToken DirectiveID) { parseDirectiveCFINegateRAState(); else if (IDVal == ".cfi_negate_ra_state_with_pc") parseDirectiveCFINegateRAStateWithPC(); + else if (IDVal == ".cfi_set_ra_state") + parseDirectiveCFILLVMSetRAState(); else if (IDVal == ".cfi_b_key_frame") parseDirectiveCFIBKeyFrame(); else if (IDVal == ".cfi_mte_tagged_frame") @@ -7557,6 +7560,34 @@ bool AArch64AsmParser::parseDirectiveCFINegateRAStateWithPC() { return false; } +/// parseDirectiveCFILLVMSetRAState +/// ::= .cfi_set_ra_state ra_state, offset +/// ::= .cfi_set_ra_state ra_state, pac_sym +bool AArch64AsmParser::parseDirectiveCFILLVMSetRAState() { + int64_t State; + if (getParser().parseAbsoluteExpression(State)) + return true; + if (parseToken(AsmToken::Comma, "expected ','")) + return true; + const MCExpr *Expr; + SMLoc ExprLoc = getLoc(); + if (getParser().parseExpression(Expr)) + return true; + if (parseEOL()) + return true; + if (auto *SymRef = dyn_cast(Expr)) { + getStreamer().emitCFILLVMSetRAState( + (unsigned)State, const_cast(&SymRef->getSymbol())); + } else if (auto *CE = dyn_cast(Expr)) { + getStreamer().emitCFILLVMSetRAState((unsigned)State, CE->getValue()); + } else { + return Error( + ExprLoc, + "expected an integer offset or a symbol for .cfi_set_ra_state"); + } + return false; +} + /// parseDirectiveCFIBKeyFrame /// ::= .cfi_b_key bool AArch64AsmParser::parseDirectiveCFIBKeyFrame() { diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp index 98fd4fd4a451b..6e2b30078b0f6 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp @@ -724,6 +724,9 @@ class DarwinAArch64AsmBackend : public AArch64AsmBackend { break; } + case MCCFIInstruction::OpLLVMSetRAState: + // .cfi_set_ra_state is not yet supported in Compact Unwind. + return CU::UNWIND_ARM64_MODE_DWARF; } } diff --git a/llvm/test/CodeGen/AArch64/sign-return-address-pauth-lr.ll b/llvm/test/CodeGen/AArch64/sign-return-address-pauth-lr.ll index 7f7c021995bb2..0d412d5df8223 100644 --- a/llvm/test/CodeGen/AArch64/sign-return-address-pauth-lr.ll +++ b/llvm/test/CodeGen/AArch64/sign-return-address-pauth-lr.ll @@ -4,6 +4,10 @@ ; -mbranch-protection=+pc enables branch protection. If the feature +pauth-lr ; is available (v9.5a onwards) then non-NOP instructions are used; otherwise ; NOP instructions are used. +; +; Independently, -aarch64-cfi-llvm-set-ra-sign-state picks which CFI directive +; describes that signing: the legacy .cfi_negate_ra_state[_with_pc] ("never", +; the default), or the new .cfi_set_ra_state ("pauth-lr"/"always"). ; There are 6 cases to cover: @@ -17,11 +21,19 @@ ; sign-return-address-pauth-lr.ll is identical, with the addition of the function ; attribute, which enables -mbranch-protection=pac-ret+pc, and therefore tests -; the remaining parameter combinations in the table: +; the remaining parameter combinations in the table, each with the legacy +; negate-based CFI, and the new set-based CFI. ; RUN: llc -mtriple=aarch64 < %s | FileCheck --check-prefixes=CHECK,COMPAT %s ; RUN: llc -mtriple=aarch64 -mattr=v8.3a < %s | FileCheck --check-prefixes=CHECK,V83A %s ; RUN: llc -mtriple=aarch64 -mattr=v9a -mattr=pauth-lr < %s | FileCheck --check-prefixes=PAUTHLR %s +; +; RUN: llc -mtriple=aarch64 -aarch64-cfi-llvm-set-ra-sign-state=pauth-lr < %s | FileCheck --check-prefixes=CFISET-CHECK,CFISET-COMPAT %s +; RUN: llc -mtriple=aarch64 -mattr=v8.3a -aarch64-cfi-llvm-set-ra-sign-state=pauth-lr < %s | FileCheck --check-prefixes=CFISET-CHECK,CFISET-V83A %s +; +; RUN: llc -mtriple=aarch64 -aarch64-cfi-llvm-set-ra-sign-state=never < %s | FileCheck --check-prefixes=NEGATE-CHECK,NEGATE-COMPAT %s +; RUN: llc -mtriple=aarch64 -mattr=v8.3a -aarch64-cfi-llvm-set-ra-sign-state=never < %s | FileCheck --check-prefixes=NEGATE-CHECK,NEGATE-V83A %s +; RUN: llc -mtriple=aarch64 -mattr=v9a,pauth-lr -aarch64-cfi-llvm-set-ra-sign-state=never < %s | FileCheck --check-prefixes=NEGATE-PAUTHLR %s ; See also: sign-return-address-pauth-lr-mir.ll @@ -33,6 +45,18 @@ define i32 @leaf(i32 %x) "branch-protection-pauth-lr" { ; PAUTHLR-LABEL: leaf: ; PAUTHLR: // %bb.0: ; PAUTHLR-NEXT: ret +; +; CFISET-CHECK-LABEL: leaf: +; CFISET-CHECK: // %bb.0: +; CFISET-CHECK-NEXT: ret +; +; NEGATE-CHECK-LABEL: leaf: +; NEGATE-CHECK: // %bb.0: +; NEGATE-CHECK-NEXT: ret +; +; NEGATE-PAUTHLR-LABEL: leaf: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: ret ret i32 %x } @@ -44,6 +68,18 @@ define i32 @leaf_sign_none(i32 %x) "branch-protection-pauth-lr" { ; PAUTHLR-LABEL: leaf_sign_none: ; PAUTHLR: // %bb.0: ; PAUTHLR-NEXT: ret +; +; CFISET-CHECK-LABEL: leaf_sign_none: +; CFISET-CHECK: // %bb.0: +; CFISET-CHECK-NEXT: ret +; +; NEGATE-CHECK-LABEL: leaf_sign_none: +; NEGATE-CHECK: // %bb.0: +; NEGATE-CHECK-NEXT: ret +; +; NEGATE-PAUTHLR-LABEL: leaf_sign_none: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: ret ret i32 %x } @@ -55,6 +91,18 @@ define i32 @leaf_sign_non_leaf(i32 %x) "branch-protection-pauth-lr" "sign-return ; PAUTHLR-LABEL: leaf_sign_non_leaf: ; PAUTHLR: // %bb.0: ; PAUTHLR-NEXT: ret +; +; CFISET-CHECK-LABEL: leaf_sign_non_leaf: +; CFISET-CHECK: // %bb.0: +; CFISET-CHECK-NEXT: ret +; +; NEGATE-CHECK-LABEL: leaf_sign_non_leaf: +; NEGATE-CHECK: // %bb.0: +; NEGATE-CHECK-NEXT: ret +; +; NEGATE-PAUTHLR-LABEL: leaf_sign_non_leaf: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: ret ret i32 %x } @@ -88,6 +136,59 @@ define i32 @leaf_sign_all(i32 %x) "branch-protection-pauth-lr" "sign-return-addr ; PAUTHLR-NEXT: .Ltmp0: ; PAUTHLR-NEXT: paciasppc ; PAUTHLR-NEXT: retaasppc .Ltmp0 +; +; CFISET-COMPAT-LABEL: leaf_sign_all: +; CFISET-COMPAT: // %bb.0: +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: .Ltmp0: +; CFISET-COMPAT-NEXT: hint #25 +; CFISET-COMPAT-NEXT: .cfi_set_ra_state 2, .Ltmp0 +; CFISET-COMPAT-NEXT: adrp x16, .Ltmp0 +; CFISET-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp0 +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: hint #29 +; CFISET-COMPAT-NEXT: ret +; +; CFISET-V83A-LABEL: leaf_sign_all: +; CFISET-V83A: // %bb.0: +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: .Ltmp0: +; CFISET-V83A-NEXT: paciasp +; CFISET-V83A-NEXT: .cfi_set_ra_state 2, .Ltmp0 +; CFISET-V83A-NEXT: adrp x16, .Ltmp0 +; CFISET-V83A-NEXT: add x16, x16, :lo12:.Ltmp0 +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: retaa +; +; NEGATE-COMPAT-LABEL: leaf_sign_all: +; NEGATE-COMPAT: // %bb.0: +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-COMPAT-NEXT: .Ltmp0: +; NEGATE-COMPAT-NEXT: hint #25 +; NEGATE-COMPAT-NEXT: adrp x16, .Ltmp0 +; NEGATE-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp0 +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: hint #29 +; NEGATE-COMPAT-NEXT: ret +; +; NEGATE-V83A-LABEL: leaf_sign_all: +; NEGATE-V83A: // %bb.0: +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-V83A-NEXT: .Ltmp0: +; NEGATE-V83A-NEXT: paciasp +; NEGATE-V83A-NEXT: adrp x16, .Ltmp0 +; NEGATE-V83A-NEXT: add x16, x16, :lo12:.Ltmp0 +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: retaa +; +; NEGATE-PAUTHLR-LABEL: leaf_sign_all: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-PAUTHLR-NEXT: .Ltmp0: +; NEGATE-PAUTHLR-NEXT: paciasppc +; NEGATE-PAUTHLR-NEXT: retaasppc .Ltmp0 ret i32 %x } @@ -142,6 +243,94 @@ define i64 @leaf_clobbers_lr(i64 %x) "branch-protection-pauth-lr" "sign-return-a ; PAUTHLR-NEXT: //NO_APP ; PAUTHLR-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload ; PAUTHLR-NEXT: retaasppc .Ltmp1 +; +; CFISET-COMPAT-LABEL: leaf_clobbers_lr: +; CFISET-COMPAT: // %bb.0: +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: .Ltmp1: +; CFISET-COMPAT-NEXT: hint #25 +; CFISET-COMPAT-NEXT: .cfi_set_ra_state 2, .Ltmp1 +; CFISET-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; CFISET-COMPAT-NEXT: .cfi_def_cfa_offset 16 +; CFISET-COMPAT-NEXT: .cfi_offset w30, -16 +; CFISET-COMPAT-NEXT: //APP +; CFISET-COMPAT-NEXT: mov x30, x0 +; CFISET-COMPAT-NEXT: //NO_APP +; CFISET-COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CFISET-COMPAT-NEXT: adrp x16, .Ltmp1 +; CFISET-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp1 +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: hint #29 +; CFISET-COMPAT-NEXT: ret +; +; CFISET-V83A-LABEL: leaf_clobbers_lr: +; CFISET-V83A: // %bb.0: +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: .Ltmp1: +; CFISET-V83A-NEXT: paciasp +; CFISET-V83A-NEXT: .cfi_set_ra_state 2, .Ltmp1 +; CFISET-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; CFISET-V83A-NEXT: .cfi_def_cfa_offset 16 +; CFISET-V83A-NEXT: .cfi_offset w30, -16 +; CFISET-V83A-NEXT: //APP +; CFISET-V83A-NEXT: mov x30, x0 +; CFISET-V83A-NEXT: //NO_APP +; CFISET-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CFISET-V83A-NEXT: adrp x16, .Ltmp1 +; CFISET-V83A-NEXT: add x16, x16, :lo12:.Ltmp1 +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: retaa +; +; NEGATE-COMPAT-LABEL: leaf_clobbers_lr: +; NEGATE-COMPAT: // %bb.0: +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-COMPAT-NEXT: .Ltmp1: +; NEGATE-COMPAT-NEXT: hint #25 +; NEGATE-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-COMPAT-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-COMPAT-NEXT: .cfi_offset w30, -16 +; NEGATE-COMPAT-NEXT: //APP +; NEGATE-COMPAT-NEXT: mov x30, x0 +; NEGATE-COMPAT-NEXT: //NO_APP +; NEGATE-COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-COMPAT-NEXT: adrp x16, .Ltmp1 +; NEGATE-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp1 +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: hint #29 +; NEGATE-COMPAT-NEXT: ret +; +; NEGATE-V83A-LABEL: leaf_clobbers_lr: +; NEGATE-V83A: // %bb.0: +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-V83A-NEXT: .Ltmp1: +; NEGATE-V83A-NEXT: paciasp +; NEGATE-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-V83A-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-V83A-NEXT: .cfi_offset w30, -16 +; NEGATE-V83A-NEXT: //APP +; NEGATE-V83A-NEXT: mov x30, x0 +; NEGATE-V83A-NEXT: //NO_APP +; NEGATE-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-V83A-NEXT: adrp x16, .Ltmp1 +; NEGATE-V83A-NEXT: add x16, x16, :lo12:.Ltmp1 +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: retaa +; +; NEGATE-PAUTHLR-LABEL: leaf_clobbers_lr: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-PAUTHLR-NEXT: .Ltmp1: +; NEGATE-PAUTHLR-NEXT: paciasppc +; NEGATE-PAUTHLR-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-PAUTHLR-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-PAUTHLR-NEXT: .cfi_offset w30, -16 +; NEGATE-PAUTHLR-NEXT: //APP +; NEGATE-PAUTHLR-NEXT: mov x30, x0 +; NEGATE-PAUTHLR-NEXT: //NO_APP +; NEGATE-PAUTHLR-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-PAUTHLR-NEXT: retaasppc .Ltmp1 call void asm sideeffect "mov x30, $0", "r,~{lr}"(i64 %x) #1 ret i64 %x } @@ -193,6 +382,84 @@ define i32 @non_leaf_sign_all(i32 %x) "branch-protection-pauth-lr" "sign-return- ; PAUTHLR-NEXT: bl foo ; PAUTHLR-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload ; PAUTHLR-NEXT: retaasppc .Ltmp2 +; +; CFISET-COMPAT-LABEL: non_leaf_sign_all: +; CFISET-COMPAT: // %bb.0: +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: .Ltmp2: +; CFISET-COMPAT-NEXT: hint #25 +; CFISET-COMPAT-NEXT: .cfi_set_ra_state 2, .Ltmp2 +; CFISET-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; CFISET-COMPAT-NEXT: .cfi_def_cfa_offset 16 +; CFISET-COMPAT-NEXT: .cfi_offset w30, -16 +; CFISET-COMPAT-NEXT: bl foo +; CFISET-COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CFISET-COMPAT-NEXT: adrp x16, .Ltmp2 +; CFISET-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp2 +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: hint #29 +; CFISET-COMPAT-NEXT: ret +; +; CFISET-V83A-LABEL: non_leaf_sign_all: +; CFISET-V83A: // %bb.0: +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: .Ltmp2: +; CFISET-V83A-NEXT: paciasp +; CFISET-V83A-NEXT: .cfi_set_ra_state 2, .Ltmp2 +; CFISET-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; CFISET-V83A-NEXT: .cfi_def_cfa_offset 16 +; CFISET-V83A-NEXT: .cfi_offset w30, -16 +; CFISET-V83A-NEXT: bl foo +; CFISET-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CFISET-V83A-NEXT: adrp x16, .Ltmp2 +; CFISET-V83A-NEXT: add x16, x16, :lo12:.Ltmp2 +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: retaa +; +; NEGATE-COMPAT-LABEL: non_leaf_sign_all: +; NEGATE-COMPAT: // %bb.0: +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-COMPAT-NEXT: .Ltmp2: +; NEGATE-COMPAT-NEXT: hint #25 +; NEGATE-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-COMPAT-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-COMPAT-NEXT: .cfi_offset w30, -16 +; NEGATE-COMPAT-NEXT: bl foo +; NEGATE-COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-COMPAT-NEXT: adrp x16, .Ltmp2 +; NEGATE-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp2 +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: hint #29 +; NEGATE-COMPAT-NEXT: ret +; +; NEGATE-V83A-LABEL: non_leaf_sign_all: +; NEGATE-V83A: // %bb.0: +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-V83A-NEXT: .Ltmp2: +; NEGATE-V83A-NEXT: paciasp +; NEGATE-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-V83A-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-V83A-NEXT: .cfi_offset w30, -16 +; NEGATE-V83A-NEXT: bl foo +; NEGATE-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-V83A-NEXT: adrp x16, .Ltmp2 +; NEGATE-V83A-NEXT: add x16, x16, :lo12:.Ltmp2 +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: retaa +; +; NEGATE-PAUTHLR-LABEL: non_leaf_sign_all: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-PAUTHLR-NEXT: .Ltmp2: +; NEGATE-PAUTHLR-NEXT: paciasppc +; NEGATE-PAUTHLR-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-PAUTHLR-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-PAUTHLR-NEXT: .cfi_offset w30, -16 +; NEGATE-PAUTHLR-NEXT: bl foo +; NEGATE-PAUTHLR-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-PAUTHLR-NEXT: retaasppc .Ltmp2 %call = call i32 @foo(i32 %x) ret i32 %call } @@ -242,6 +509,84 @@ define i32 @non_leaf_sign_non_leaf(i32 %x) "branch-protection-pauth-lr" "sign-re ; PAUTHLR-NEXT: bl foo ; PAUTHLR-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload ; PAUTHLR-NEXT: retaasppc .Ltmp3 +; +; CFISET-COMPAT-LABEL: non_leaf_sign_non_leaf: +; CFISET-COMPAT: // %bb.0: +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: .Ltmp3: +; CFISET-COMPAT-NEXT: hint #25 +; CFISET-COMPAT-NEXT: .cfi_set_ra_state 2, .Ltmp3 +; CFISET-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; CFISET-COMPAT-NEXT: .cfi_def_cfa_offset 16 +; CFISET-COMPAT-NEXT: .cfi_offset w30, -16 +; CFISET-COMPAT-NEXT: bl foo +; CFISET-COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CFISET-COMPAT-NEXT: adrp x16, .Ltmp3 +; CFISET-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp3 +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: hint #29 +; CFISET-COMPAT-NEXT: ret +; +; CFISET-V83A-LABEL: non_leaf_sign_non_leaf: +; CFISET-V83A: // %bb.0: +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: .Ltmp3: +; CFISET-V83A-NEXT: paciasp +; CFISET-V83A-NEXT: .cfi_set_ra_state 2, .Ltmp3 +; CFISET-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; CFISET-V83A-NEXT: .cfi_def_cfa_offset 16 +; CFISET-V83A-NEXT: .cfi_offset w30, -16 +; CFISET-V83A-NEXT: bl foo +; CFISET-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CFISET-V83A-NEXT: adrp x16, .Ltmp3 +; CFISET-V83A-NEXT: add x16, x16, :lo12:.Ltmp3 +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: retaa +; +; NEGATE-COMPAT-LABEL: non_leaf_sign_non_leaf: +; NEGATE-COMPAT: // %bb.0: +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-COMPAT-NEXT: .Ltmp3: +; NEGATE-COMPAT-NEXT: hint #25 +; NEGATE-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-COMPAT-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-COMPAT-NEXT: .cfi_offset w30, -16 +; NEGATE-COMPAT-NEXT: bl foo +; NEGATE-COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-COMPAT-NEXT: adrp x16, .Ltmp3 +; NEGATE-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp3 +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: hint #29 +; NEGATE-COMPAT-NEXT: ret +; +; NEGATE-V83A-LABEL: non_leaf_sign_non_leaf: +; NEGATE-V83A: // %bb.0: +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-V83A-NEXT: .Ltmp3: +; NEGATE-V83A-NEXT: paciasp +; NEGATE-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-V83A-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-V83A-NEXT: .cfi_offset w30, -16 +; NEGATE-V83A-NEXT: bl foo +; NEGATE-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-V83A-NEXT: adrp x16, .Ltmp3 +; NEGATE-V83A-NEXT: add x16, x16, :lo12:.Ltmp3 +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: retaa +; +; NEGATE-PAUTHLR-LABEL: non_leaf_sign_non_leaf: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-PAUTHLR-NEXT: .Ltmp3: +; NEGATE-PAUTHLR-NEXT: paciasppc +; NEGATE-PAUTHLR-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-PAUTHLR-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-PAUTHLR-NEXT: .cfi_offset w30, -16 +; NEGATE-PAUTHLR-NEXT: bl foo +; NEGATE-PAUTHLR-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-PAUTHLR-NEXT: retaasppc .Ltmp3 %call = call i32 @foo(i32 %x) ret i32 %call } @@ -283,6 +628,62 @@ define i32 @non_leaf_scs(i32 %x) "branch-protection-pauth-lr" "sign-return-addre ; PAUTHLR-NEXT: ldr x30, [x18, #-8]! ; PAUTHLR-NEXT: autiasppc .Ltmp4 ; PAUTHLR-NEXT: ret +; +; CFISET-CHECK-LABEL: non_leaf_scs: +; CFISET-CHECK: // %bb.0: +; CFISET-CHECK-NEXT: hint #39 +; CFISET-CHECK-NEXT: .Ltmp4: +; CFISET-CHECK-NEXT: paciasp +; CFISET-CHECK-NEXT: .cfi_set_ra_state 2, .Ltmp4 +; CFISET-CHECK-NEXT: str x30, [x18], #8 +; CFISET-CHECK-NEXT: .cfi_escape 0x16, 0x12, 0x02, 0x82, 0x78 // +; CFISET-CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; CFISET-CHECK-NEXT: .cfi_def_cfa_offset 16 +; CFISET-CHECK-NEXT: .cfi_offset w30, -16 +; CFISET-CHECK-NEXT: bl foo +; CFISET-CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CFISET-CHECK-NEXT: ldr x30, [x18, #-8]! +; CFISET-CHECK-NEXT: adrp x16, .Ltmp4 +; CFISET-CHECK-NEXT: add x16, x16, :lo12:.Ltmp4 +; CFISET-CHECK-NEXT: hint #39 +; CFISET-CHECK-NEXT: autiasp +; CFISET-CHECK-NEXT: ret +; +; NEGATE-CHECK-LABEL: non_leaf_scs: +; NEGATE-CHECK: // %bb.0: +; NEGATE-CHECK-NEXT: hint #39 +; NEGATE-CHECK-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-CHECK-NEXT: .Ltmp4: +; NEGATE-CHECK-NEXT: paciasp +; NEGATE-CHECK-NEXT: str x30, [x18], #8 +; NEGATE-CHECK-NEXT: .cfi_escape 0x16, 0x12, 0x02, 0x82, 0x78 // +; NEGATE-CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-CHECK-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-CHECK-NEXT: .cfi_offset w30, -16 +; NEGATE-CHECK-NEXT: bl foo +; NEGATE-CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-CHECK-NEXT: ldr x30, [x18, #-8]! +; NEGATE-CHECK-NEXT: adrp x16, .Ltmp4 +; NEGATE-CHECK-NEXT: add x16, x16, :lo12:.Ltmp4 +; NEGATE-CHECK-NEXT: hint #39 +; NEGATE-CHECK-NEXT: autiasp +; NEGATE-CHECK-NEXT: ret +; +; NEGATE-PAUTHLR-LABEL: non_leaf_scs: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-PAUTHLR-NEXT: .Ltmp4: +; NEGATE-PAUTHLR-NEXT: paciasppc +; NEGATE-PAUTHLR-NEXT: str x30, [x18], #8 +; NEGATE-PAUTHLR-NEXT: .cfi_escape 0x16, 0x12, 0x02, 0x82, 0x78 // +; NEGATE-PAUTHLR-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-PAUTHLR-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-PAUTHLR-NEXT: .cfi_offset w30, -16 +; NEGATE-PAUTHLR-NEXT: bl foo +; NEGATE-PAUTHLR-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-PAUTHLR-NEXT: ldr x30, [x18, #-8]! +; NEGATE-PAUTHLR-NEXT: autiasppc .Ltmp4 +; NEGATE-PAUTHLR-NEXT: ret %call = call i32 @foo(i32 %x) ret i32 %call } @@ -305,6 +706,35 @@ define i32 @leaf_sign_all_v83(i32 %x) "branch-protection-pauth-lr" "sign-return- ; PAUTHLR-NEXT: .Ltmp5: ; PAUTHLR-NEXT: paciasppc ; PAUTHLR-NEXT: retaasppc .Ltmp5 +; +; CFISET-CHECK-LABEL: leaf_sign_all_v83: +; CFISET-CHECK: // %bb.0: +; CFISET-CHECK-NEXT: hint #39 +; CFISET-CHECK-NEXT: .Ltmp5: +; CFISET-CHECK-NEXT: paciasp +; CFISET-CHECK-NEXT: .cfi_set_ra_state 2, .Ltmp5 +; CFISET-CHECK-NEXT: adrp x16, .Ltmp5 +; CFISET-CHECK-NEXT: add x16, x16, :lo12:.Ltmp5 +; CFISET-CHECK-NEXT: hint #39 +; CFISET-CHECK-NEXT: retaa +; +; NEGATE-CHECK-LABEL: leaf_sign_all_v83: +; NEGATE-CHECK: // %bb.0: +; NEGATE-CHECK-NEXT: hint #39 +; NEGATE-CHECK-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-CHECK-NEXT: .Ltmp5: +; NEGATE-CHECK-NEXT: paciasp +; NEGATE-CHECK-NEXT: adrp x16, .Ltmp5 +; NEGATE-CHECK-NEXT: add x16, x16, :lo12:.Ltmp5 +; NEGATE-CHECK-NEXT: hint #39 +; NEGATE-CHECK-NEXT: retaa +; +; NEGATE-PAUTHLR-LABEL: leaf_sign_all_v83: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-PAUTHLR-NEXT: .Ltmp5: +; NEGATE-PAUTHLR-NEXT: paciasppc +; NEGATE-PAUTHLR-NEXT: retaasppc .Ltmp5 ret i32 %x } @@ -363,6 +793,97 @@ define fastcc void @spill_lr_and_tail_call(i64 %x) "branch-protection-pauth-lr" ; PAUTHLR-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload ; PAUTHLR-NEXT: autiasppc .Ltmp6 ; PAUTHLR-NEXT: b bar +; +; CFISET-COMPAT-LABEL: spill_lr_and_tail_call: +; CFISET-COMPAT: // %bb.0: +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: .Ltmp6: +; CFISET-COMPAT-NEXT: hint #25 +; CFISET-COMPAT-NEXT: .cfi_set_ra_state 2, .Ltmp6 +; CFISET-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; CFISET-COMPAT-NEXT: .cfi_def_cfa_offset 16 +; CFISET-COMPAT-NEXT: .cfi_offset w30, -16 +; CFISET-COMPAT-NEXT: //APP +; CFISET-COMPAT-NEXT: mov x30, x0 +; CFISET-COMPAT-NEXT: //NO_APP +; CFISET-COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CFISET-COMPAT-NEXT: adrp x16, .Ltmp6 +; CFISET-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp6 +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: hint #29 +; CFISET-COMPAT-NEXT: b bar +; +; CFISET-V83A-LABEL: spill_lr_and_tail_call: +; CFISET-V83A: // %bb.0: +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: .Ltmp6: +; CFISET-V83A-NEXT: paciasp +; CFISET-V83A-NEXT: .cfi_set_ra_state 2, .Ltmp6 +; CFISET-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; CFISET-V83A-NEXT: .cfi_def_cfa_offset 16 +; CFISET-V83A-NEXT: .cfi_offset w30, -16 +; CFISET-V83A-NEXT: //APP +; CFISET-V83A-NEXT: mov x30, x0 +; CFISET-V83A-NEXT: //NO_APP +; CFISET-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CFISET-V83A-NEXT: adrp x16, .Ltmp6 +; CFISET-V83A-NEXT: add x16, x16, :lo12:.Ltmp6 +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: autiasp +; CFISET-V83A-NEXT: b bar +; +; NEGATE-COMPAT-LABEL: spill_lr_and_tail_call: +; NEGATE-COMPAT: // %bb.0: +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-COMPAT-NEXT: .Ltmp6: +; NEGATE-COMPAT-NEXT: hint #25 +; NEGATE-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-COMPAT-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-COMPAT-NEXT: .cfi_offset w30, -16 +; NEGATE-COMPAT-NEXT: //APP +; NEGATE-COMPAT-NEXT: mov x30, x0 +; NEGATE-COMPAT-NEXT: //NO_APP +; NEGATE-COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-COMPAT-NEXT: adrp x16, .Ltmp6 +; NEGATE-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp6 +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: hint #29 +; NEGATE-COMPAT-NEXT: b bar +; +; NEGATE-V83A-LABEL: spill_lr_and_tail_call: +; NEGATE-V83A: // %bb.0: +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-V83A-NEXT: .Ltmp6: +; NEGATE-V83A-NEXT: paciasp +; NEGATE-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-V83A-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-V83A-NEXT: .cfi_offset w30, -16 +; NEGATE-V83A-NEXT: //APP +; NEGATE-V83A-NEXT: mov x30, x0 +; NEGATE-V83A-NEXT: //NO_APP +; NEGATE-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-V83A-NEXT: adrp x16, .Ltmp6 +; NEGATE-V83A-NEXT: add x16, x16, :lo12:.Ltmp6 +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: autiasp +; NEGATE-V83A-NEXT: b bar +; +; NEGATE-PAUTHLR-LABEL: spill_lr_and_tail_call: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-PAUTHLR-NEXT: .Ltmp6: +; NEGATE-PAUTHLR-NEXT: paciasppc +; NEGATE-PAUTHLR-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-PAUTHLR-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-PAUTHLR-NEXT: .cfi_offset w30, -16 +; NEGATE-PAUTHLR-NEXT: //APP +; NEGATE-PAUTHLR-NEXT: mov x30, x0 +; NEGATE-PAUTHLR-NEXT: //NO_APP +; NEGATE-PAUTHLR-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-PAUTHLR-NEXT: autiasppc .Ltmp6 +; NEGATE-PAUTHLR-NEXT: b bar call void asm sideeffect "mov x30, $0", "r,~{lr}"(i64 %x) #1 tail call fastcc i64 @bar(i64 %x) ret void @@ -398,6 +919,59 @@ define i32 @leaf_sign_all_a_key(i32 %x) "branch-protection-pauth-lr" "sign-retur ; PAUTHLR-NEXT: .Ltmp7: ; PAUTHLR-NEXT: paciasppc ; PAUTHLR-NEXT: retaasppc .Ltmp7 +; +; CFISET-COMPAT-LABEL: leaf_sign_all_a_key: +; CFISET-COMPAT: // %bb.0: +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: .Ltmp7: +; CFISET-COMPAT-NEXT: hint #25 +; CFISET-COMPAT-NEXT: .cfi_set_ra_state 2, .Ltmp7 +; CFISET-COMPAT-NEXT: adrp x16, .Ltmp7 +; CFISET-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp7 +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: hint #29 +; CFISET-COMPAT-NEXT: ret +; +; CFISET-V83A-LABEL: leaf_sign_all_a_key: +; CFISET-V83A: // %bb.0: +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: .Ltmp7: +; CFISET-V83A-NEXT: paciasp +; CFISET-V83A-NEXT: .cfi_set_ra_state 2, .Ltmp7 +; CFISET-V83A-NEXT: adrp x16, .Ltmp7 +; CFISET-V83A-NEXT: add x16, x16, :lo12:.Ltmp7 +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: retaa +; +; NEGATE-COMPAT-LABEL: leaf_sign_all_a_key: +; NEGATE-COMPAT: // %bb.0: +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-COMPAT-NEXT: .Ltmp7: +; NEGATE-COMPAT-NEXT: hint #25 +; NEGATE-COMPAT-NEXT: adrp x16, .Ltmp7 +; NEGATE-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp7 +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: hint #29 +; NEGATE-COMPAT-NEXT: ret +; +; NEGATE-V83A-LABEL: leaf_sign_all_a_key: +; NEGATE-V83A: // %bb.0: +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-V83A-NEXT: .Ltmp7: +; NEGATE-V83A-NEXT: paciasp +; NEGATE-V83A-NEXT: adrp x16, .Ltmp7 +; NEGATE-V83A-NEXT: add x16, x16, :lo12:.Ltmp7 +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: retaa +; +; NEGATE-PAUTHLR-LABEL: leaf_sign_all_a_key: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-PAUTHLR-NEXT: .Ltmp7: +; NEGATE-PAUTHLR-NEXT: paciasppc +; NEGATE-PAUTHLR-NEXT: retaasppc .Ltmp7 ret i32 %x } @@ -434,6 +1008,64 @@ define i32 @leaf_sign_all_b_key(i32 %x) "branch-protection-pauth-lr" "sign-retur ; PAUTHLR-NEXT: .Ltmp8: ; PAUTHLR-NEXT: pacibsppc ; PAUTHLR-NEXT: retabsppc .Ltmp8 +; +; CFISET-COMPAT-LABEL: leaf_sign_all_b_key: +; CFISET-COMPAT: // %bb.0: +; CFISET-COMPAT-NEXT: .cfi_b_key_frame +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: .Ltmp8: +; CFISET-COMPAT-NEXT: hint #27 +; CFISET-COMPAT-NEXT: .cfi_set_ra_state 2, .Ltmp8 +; CFISET-COMPAT-NEXT: adrp x16, .Ltmp8 +; CFISET-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp8 +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: hint #31 +; CFISET-COMPAT-NEXT: ret +; +; CFISET-V83A-LABEL: leaf_sign_all_b_key: +; CFISET-V83A: // %bb.0: +; CFISET-V83A-NEXT: .cfi_b_key_frame +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: .Ltmp8: +; CFISET-V83A-NEXT: pacibsp +; CFISET-V83A-NEXT: .cfi_set_ra_state 2, .Ltmp8 +; CFISET-V83A-NEXT: adrp x16, .Ltmp8 +; CFISET-V83A-NEXT: add x16, x16, :lo12:.Ltmp8 +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: retab +; +; NEGATE-COMPAT-LABEL: leaf_sign_all_b_key: +; NEGATE-COMPAT: // %bb.0: +; NEGATE-COMPAT-NEXT: .cfi_b_key_frame +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-COMPAT-NEXT: .Ltmp8: +; NEGATE-COMPAT-NEXT: hint #27 +; NEGATE-COMPAT-NEXT: adrp x16, .Ltmp8 +; NEGATE-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp8 +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: hint #31 +; NEGATE-COMPAT-NEXT: ret +; +; NEGATE-V83A-LABEL: leaf_sign_all_b_key: +; NEGATE-V83A: // %bb.0: +; NEGATE-V83A-NEXT: .cfi_b_key_frame +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-V83A-NEXT: .Ltmp8: +; NEGATE-V83A-NEXT: pacibsp +; NEGATE-V83A-NEXT: adrp x16, .Ltmp8 +; NEGATE-V83A-NEXT: add x16, x16, :lo12:.Ltmp8 +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: retab +; +; NEGATE-PAUTHLR-LABEL: leaf_sign_all_b_key: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: .cfi_b_key_frame +; NEGATE-PAUTHLR-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-PAUTHLR-NEXT: .Ltmp8: +; NEGATE-PAUTHLR-NEXT: pacibsppc +; NEGATE-PAUTHLR-NEXT: retabsppc .Ltmp8 ret i32 %x } @@ -457,6 +1089,38 @@ define i32 @leaf_sign_all_v83_b_key(i32 %x) "branch-protection-pauth-lr" "sign-r ; PAUTHLR-NEXT: .Ltmp9: ; PAUTHLR-NEXT: pacibsppc ; PAUTHLR-NEXT: retabsppc .Ltmp9 +; +; CFISET-CHECK-LABEL: leaf_sign_all_v83_b_key: +; CFISET-CHECK: // %bb.0: +; CFISET-CHECK-NEXT: .cfi_b_key_frame +; CFISET-CHECK-NEXT: hint #39 +; CFISET-CHECK-NEXT: .Ltmp9: +; CFISET-CHECK-NEXT: pacibsp +; CFISET-CHECK-NEXT: .cfi_set_ra_state 2, .Ltmp9 +; CFISET-CHECK-NEXT: adrp x16, .Ltmp9 +; CFISET-CHECK-NEXT: add x16, x16, :lo12:.Ltmp9 +; CFISET-CHECK-NEXT: hint #39 +; CFISET-CHECK-NEXT: retab +; +; NEGATE-CHECK-LABEL: leaf_sign_all_v83_b_key: +; NEGATE-CHECK: // %bb.0: +; NEGATE-CHECK-NEXT: .cfi_b_key_frame +; NEGATE-CHECK-NEXT: hint #39 +; NEGATE-CHECK-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-CHECK-NEXT: .Ltmp9: +; NEGATE-CHECK-NEXT: pacibsp +; NEGATE-CHECK-NEXT: adrp x16, .Ltmp9 +; NEGATE-CHECK-NEXT: add x16, x16, :lo12:.Ltmp9 +; NEGATE-CHECK-NEXT: hint #39 +; NEGATE-CHECK-NEXT: retab +; +; NEGATE-PAUTHLR-LABEL: leaf_sign_all_v83_b_key: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: .cfi_b_key_frame +; NEGATE-PAUTHLR-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-PAUTHLR-NEXT: .Ltmp9: +; NEGATE-PAUTHLR-NEXT: pacibsppc +; NEGATE-PAUTHLR-NEXT: retabsppc .Ltmp9 ret i32 %x } @@ -494,6 +1158,64 @@ define i32 @leaf_sign_all_a_key_bti(i32 %x) "branch-protection-pauth-lr" "sign-r ; PAUTHLR-NEXT: .Ltmp10: ; PAUTHLR-NEXT: paciasppc ; PAUTHLR-NEXT: retaasppc .Ltmp10 +; +; CFISET-COMPAT-LABEL: leaf_sign_all_a_key_bti: +; CFISET-COMPAT: // %bb.0: +; CFISET-COMPAT-NEXT: hint #34 +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: .Ltmp10: +; CFISET-COMPAT-NEXT: hint #25 +; CFISET-COMPAT-NEXT: .cfi_set_ra_state 2, .Ltmp10 +; CFISET-COMPAT-NEXT: adrp x16, .Ltmp10 +; CFISET-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp10 +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: hint #29 +; CFISET-COMPAT-NEXT: ret +; +; CFISET-V83A-LABEL: leaf_sign_all_a_key_bti: +; CFISET-V83A: // %bb.0: +; CFISET-V83A-NEXT: hint #34 +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: .Ltmp10: +; CFISET-V83A-NEXT: paciasp +; CFISET-V83A-NEXT: .cfi_set_ra_state 2, .Ltmp10 +; CFISET-V83A-NEXT: adrp x16, .Ltmp10 +; CFISET-V83A-NEXT: add x16, x16, :lo12:.Ltmp10 +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: retaa +; +; NEGATE-COMPAT-LABEL: leaf_sign_all_a_key_bti: +; NEGATE-COMPAT: // %bb.0: +; NEGATE-COMPAT-NEXT: hint #34 +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-COMPAT-NEXT: .Ltmp10: +; NEGATE-COMPAT-NEXT: hint #25 +; NEGATE-COMPAT-NEXT: adrp x16, .Ltmp10 +; NEGATE-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp10 +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: hint #29 +; NEGATE-COMPAT-NEXT: ret +; +; NEGATE-V83A-LABEL: leaf_sign_all_a_key_bti: +; NEGATE-V83A: // %bb.0: +; NEGATE-V83A-NEXT: hint #34 +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-V83A-NEXT: .Ltmp10: +; NEGATE-V83A-NEXT: paciasp +; NEGATE-V83A-NEXT: adrp x16, .Ltmp10 +; NEGATE-V83A-NEXT: add x16, x16, :lo12:.Ltmp10 +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: retaa +; +; NEGATE-PAUTHLR-LABEL: leaf_sign_all_a_key_bti: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-PAUTHLR-NEXT: bti c +; NEGATE-PAUTHLR-NEXT: .Ltmp10: +; NEGATE-PAUTHLR-NEXT: paciasppc +; NEGATE-PAUTHLR-NEXT: retaasppc .Ltmp10 ret i32 %x } @@ -534,6 +1256,69 @@ define i32 @leaf_sign_all_b_key_bti(i32 %x) "branch-protection-pauth-lr" "sign-r ; PAUTHLR-NEXT: .Ltmp11: ; PAUTHLR-NEXT: pacibsppc ; PAUTHLR-NEXT: retabsppc .Ltmp11 +; +; CFISET-COMPAT-LABEL: leaf_sign_all_b_key_bti: +; CFISET-COMPAT: // %bb.0: +; CFISET-COMPAT-NEXT: .cfi_b_key_frame +; CFISET-COMPAT-NEXT: hint #34 +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: .Ltmp11: +; CFISET-COMPAT-NEXT: hint #27 +; CFISET-COMPAT-NEXT: .cfi_set_ra_state 2, .Ltmp11 +; CFISET-COMPAT-NEXT: adrp x16, .Ltmp11 +; CFISET-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp11 +; CFISET-COMPAT-NEXT: hint #39 +; CFISET-COMPAT-NEXT: hint #31 +; CFISET-COMPAT-NEXT: ret +; +; CFISET-V83A-LABEL: leaf_sign_all_b_key_bti: +; CFISET-V83A: // %bb.0: +; CFISET-V83A-NEXT: .cfi_b_key_frame +; CFISET-V83A-NEXT: hint #34 +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: .Ltmp11: +; CFISET-V83A-NEXT: pacibsp +; CFISET-V83A-NEXT: .cfi_set_ra_state 2, .Ltmp11 +; CFISET-V83A-NEXT: adrp x16, .Ltmp11 +; CFISET-V83A-NEXT: add x16, x16, :lo12:.Ltmp11 +; CFISET-V83A-NEXT: hint #39 +; CFISET-V83A-NEXT: retab +; +; NEGATE-COMPAT-LABEL: leaf_sign_all_b_key_bti: +; NEGATE-COMPAT: // %bb.0: +; NEGATE-COMPAT-NEXT: .cfi_b_key_frame +; NEGATE-COMPAT-NEXT: hint #34 +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-COMPAT-NEXT: .Ltmp11: +; NEGATE-COMPAT-NEXT: hint #27 +; NEGATE-COMPAT-NEXT: adrp x16, .Ltmp11 +; NEGATE-COMPAT-NEXT: add x16, x16, :lo12:.Ltmp11 +; NEGATE-COMPAT-NEXT: hint #39 +; NEGATE-COMPAT-NEXT: hint #31 +; NEGATE-COMPAT-NEXT: ret +; +; NEGATE-V83A-LABEL: leaf_sign_all_b_key_bti: +; NEGATE-V83A: // %bb.0: +; NEGATE-V83A-NEXT: .cfi_b_key_frame +; NEGATE-V83A-NEXT: hint #34 +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-V83A-NEXT: .Ltmp11: +; NEGATE-V83A-NEXT: pacibsp +; NEGATE-V83A-NEXT: adrp x16, .Ltmp11 +; NEGATE-V83A-NEXT: add x16, x16, :lo12:.Ltmp11 +; NEGATE-V83A-NEXT: hint #39 +; NEGATE-V83A-NEXT: retab +; +; NEGATE-PAUTHLR-LABEL: leaf_sign_all_b_key_bti: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: .cfi_b_key_frame +; NEGATE-PAUTHLR-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-PAUTHLR-NEXT: bti c +; NEGATE-PAUTHLR-NEXT: .Ltmp11: +; NEGATE-PAUTHLR-NEXT: pacibsppc +; NEGATE-PAUTHLR-NEXT: retabsppc .Ltmp11 ret i32 %x } @@ -560,5 +1345,40 @@ define i32 @leaf_sign_all_v83_b_key_bti(i32 %x) "branch-protection-pauth-lr" "si ; PAUTHLR-NEXT: .Ltmp12: ; PAUTHLR-NEXT: pacibsppc ; PAUTHLR-NEXT: retabsppc .Ltmp12 +; +; CFISET-CHECK-LABEL: leaf_sign_all_v83_b_key_bti: +; CFISET-CHECK: // %bb.0: +; CFISET-CHECK-NEXT: .cfi_b_key_frame +; CFISET-CHECK-NEXT: hint #34 +; CFISET-CHECK-NEXT: hint #39 +; CFISET-CHECK-NEXT: .Ltmp12: +; CFISET-CHECK-NEXT: pacibsp +; CFISET-CHECK-NEXT: .cfi_set_ra_state 2, .Ltmp12 +; CFISET-CHECK-NEXT: adrp x16, .Ltmp12 +; CFISET-CHECK-NEXT: add x16, x16, :lo12:.Ltmp12 +; CFISET-CHECK-NEXT: hint #39 +; CFISET-CHECK-NEXT: retab +; +; NEGATE-CHECK-LABEL: leaf_sign_all_v83_b_key_bti: +; NEGATE-CHECK: // %bb.0: +; NEGATE-CHECK-NEXT: .cfi_b_key_frame +; NEGATE-CHECK-NEXT: hint #34 +; NEGATE-CHECK-NEXT: hint #39 +; NEGATE-CHECK-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-CHECK-NEXT: .Ltmp12: +; NEGATE-CHECK-NEXT: pacibsp +; NEGATE-CHECK-NEXT: adrp x16, .Ltmp12 +; NEGATE-CHECK-NEXT: add x16, x16, :lo12:.Ltmp12 +; NEGATE-CHECK-NEXT: hint #39 +; NEGATE-CHECK-NEXT: retab +; +; NEGATE-PAUTHLR-LABEL: leaf_sign_all_v83_b_key_bti: +; NEGATE-PAUTHLR: // %bb.0: +; NEGATE-PAUTHLR-NEXT: .cfi_b_key_frame +; NEGATE-PAUTHLR-NEXT: .cfi_negate_ra_state_with_pc +; NEGATE-PAUTHLR-NEXT: bti c +; NEGATE-PAUTHLR-NEXT: .Ltmp12: +; NEGATE-PAUTHLR-NEXT: pacibsppc +; NEGATE-PAUTHLR-NEXT: retabsppc .Ltmp12 ret i32 %x } diff --git a/llvm/test/CodeGen/AArch64/sign-return-address.ll b/llvm/test/CodeGen/AArch64/sign-return-address.ll index 755ba860c7ccc..5003ef3822b19 100644 --- a/llvm/test/CodeGen/AArch64/sign-return-address.ll +++ b/llvm/test/CodeGen/AArch64/sign-return-address.ll @@ -4,6 +4,10 @@ ; v9.5-A is not expected to change codegen without -mbranch-protection=+pc, so reuse DWARFCFI-V83A. ; RUN: llc -mtriple=aarch64 -mattr=v9.5a < %s | FileCheck --check-prefixes=CHECK,DWARFCFI,DWARFCFI-V83A %s +; RUN: llc -mtriple=aarch64 -aarch64-cfi-llvm-set-ra-sign-state=never < %s | FileCheck --check-prefixes=CHECK,NEGATE-CFI,NEGATE-CFI-COMPAT %s +; RUN: llc -mtriple=aarch64 -mattr=v8.3a -aarch64-cfi-llvm-set-ra-sign-state=never < %s | FileCheck --check-prefixes=CHECK,NEGATE-CFI,NEGATE-CFI-V83A %s +; RUN: llc -mtriple=aarch64 -aarch64-cfi-llvm-set-ra-sign-state=always < %s | FileCheck --check-prefixes=CHECK,ALWAYS-CFI,ALWAYS-CFI-COMPAT %s +; RUN: llc -mtriple=aarch64 -mattr=v8.3a -aarch64-cfi-llvm-set-ra-sign-state=always < %s | FileCheck --check-prefixes=CHECK,ALWAYS-CFI,ALWAYS-CFI-V83A %s ; RUN: sed -e '/^define i32 @leaf_sign_all_a_key(/,/^}/d' -e '/^define i32 @leaf_sign_all_a_key_bti(/,/^}/d' %s > %t.win-valid.ll ; RUN: llc -mtriple=aarch64-windows < %t.win-valid.ll | FileCheck --check-prefixes=CHECK,WINCFI,WINCFI-COMPAT %s @@ -44,18 +48,35 @@ define i32 @leaf_sign_non_leaf(i32 %x) "sign-return-address"="non-leaf" { define i32 @leaf_sign_all(i32 %x) "sign-return-address"="all" { ; DWARFCFI-COMPAT-LABEL: leaf_sign_all: +; ALWAYS-CFI-COMPAT-LABEL: leaf_sign_all: ; DWARFCFI-COMPAT: // %bb.0: ; DWARFCFI-COMPAT-NEXT: hint #25 ; DWARFCFI-COMPAT-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-COMPAT: .cfi_set_ra_state 1, 0 ; DWARFCFI-COMPAT-NEXT: hint #29 ; DWARFCFI-COMPAT-NEXT: ret ; ; DWARFCFI-V83A-LABEL: leaf_sign_all: +; ALWAYS-CFI-V83A-LABEL: leaf_sign_all: ; DWARFCFI-V83A: // %bb.0: ; DWARFCFI-V83A-NEXT: paciasp ; DWARFCFI-V83A-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-V83A: .cfi_set_ra_state 1, 0 ; DWARFCFI-V83A-NEXT: retaa ; +; NEGATE-CFI-COMPAT-LABEL: leaf_sign_all: +; NEGATE-CFI-COMPAT: // %bb.0: +; NEGATE-CFI-COMPAT-NEXT: hint #25 +; NEGATE-CFI-COMPAT-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-COMPAT-NEXT: hint #29 +; NEGATE-CFI-COMPAT-NEXT: ret +; +; NEGATE-CFI-V83A-LABEL: leaf_sign_all: +; NEGATE-CFI-V83A: // %bb.0: +; NEGATE-CFI-V83A-NEXT: paciasp +; NEGATE-CFI-V83A-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-V83A-NEXT: retaa +; ; WINCFI-COMPAT-LABEL: leaf_sign_all: ; WINCFI-COMPAT: .seh_proc leaf_sign_all ; WINCFI-COMPAT-NEXT: // %bb.0: @@ -88,9 +109,11 @@ define i32 @leaf_sign_all(i32 %x) "sign-return-address"="all" { define i64 @leaf_clobbers_lr(i64 %x) "sign-return-address"="non-leaf" { ; DWARFCFI-COMPAT-LABEL: leaf_clobbers_lr: +; ALWAYS-CFI-COMPAT-LABEL: leaf_clobbers_lr: ; DWARFCFI-COMPAT: // %bb.0: ; DWARFCFI-COMPAT-NEXT: hint #25 ; DWARFCFI-COMPAT-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-COMPAT: .cfi_set_ra_state 1, 0 ; DWARFCFI-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; DWARFCFI-COMPAT-NEXT: .cfi_def_cfa_offset 16 ; DWARFCFI-COMPAT-NEXT: .cfi_offset w30, -16 @@ -102,9 +125,11 @@ define i64 @leaf_clobbers_lr(i64 %x) "sign-return-address"="non-leaf" { ; DWARFCFI-COMPAT-NEXT: ret ; ; DWARFCFI-V83A-LABEL: leaf_clobbers_lr: +; ALWAYS-CFI-V83A-LABEL: leaf_clobbers_lr: ; DWARFCFI-V83A: // %bb.0: ; DWARFCFI-V83A-NEXT: paciasp ; DWARFCFI-V83A-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-V83A: .cfi_set_ra_state 1, 0 ; DWARFCFI-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; DWARFCFI-V83A-NEXT: .cfi_def_cfa_offset 16 ; DWARFCFI-V83A-NEXT: .cfi_offset w30, -16 @@ -114,6 +139,33 @@ define i64 @leaf_clobbers_lr(i64 %x) "sign-return-address"="non-leaf" { ; DWARFCFI-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload ; DWARFCFI-V83A-NEXT: retaa ; +; NEGATE-CFI-COMPAT-LABEL: leaf_clobbers_lr: +; NEGATE-CFI-COMPAT: // %bb.0: +; NEGATE-CFI-COMPAT-NEXT: hint #25 +; NEGATE-CFI-COMPAT-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-CFI-COMPAT-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-CFI-COMPAT-NEXT: .cfi_offset w30, -16 +; NEGATE-CFI-COMPAT-NEXT: //APP +; NEGATE-CFI-COMPAT-NEXT: mov x30, x0 +; NEGATE-CFI-COMPAT-NEXT: //NO_APP +; NEGATE-CFI-COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-CFI-COMPAT-NEXT: hint #29 +; NEGATE-CFI-COMPAT-NEXT: ret +; +; NEGATE-CFI-V83A-LABEL: leaf_clobbers_lr: +; NEGATE-CFI-V83A: // %bb.0: +; NEGATE-CFI-V83A-NEXT: paciasp +; NEGATE-CFI-V83A-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-CFI-V83A-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-CFI-V83A-NEXT: .cfi_offset w30, -16 +; NEGATE-CFI-V83A-NEXT: //APP +; NEGATE-CFI-V83A-NEXT: mov x30, x0 +; NEGATE-CFI-V83A-NEXT: //NO_APP +; NEGATE-CFI-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-CFI-V83A-NEXT: retaa +; ; WINCFI-COMPAT-LABEL: leaf_clobbers_lr: ; WINCFI-COMPAT: .seh_proc leaf_clobbers_lr ; WINCFI-COMPAT-NEXT: // %bb.0: @@ -163,9 +215,11 @@ declare i32 @foo(i32) define i32 @non_leaf_sign_all(i32 %x) "sign-return-address"="all" { ; DWARFCFI-COMPAT-LABEL: non_leaf_sign_all: +; ALWAYS-CFI-COMPAT-LABEL: non_leaf_sign_all: ; DWARFCFI-COMPAT: // %bb.0: ; DWARFCFI-COMPAT-NEXT: hint #25 ; DWARFCFI-COMPAT-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-COMPAT: .cfi_set_ra_state 1, 0 ; DWARFCFI-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; DWARFCFI-COMPAT-NEXT: .cfi_def_cfa_offset 16 ; DWARFCFI-COMPAT-NEXT: .cfi_offset w30, -16 @@ -175,9 +229,11 @@ define i32 @non_leaf_sign_all(i32 %x) "sign-return-address"="all" { ; DWARFCFI-COMPAT-NEXT: ret ; ; DWARFCFI-V83A-LABEL: non_leaf_sign_all: +; ALWAYS-CFI-V83A-LABEL: non_leaf_sign_all: ; DWARFCFI-V83A: // %bb.0: ; DWARFCFI-V83A-NEXT: paciasp ; DWARFCFI-V83A-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-V83A: .cfi_set_ra_state 1, 0 ; DWARFCFI-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; DWARFCFI-V83A-NEXT: .cfi_def_cfa_offset 16 ; DWARFCFI-V83A-NEXT: .cfi_offset w30, -16 @@ -185,6 +241,29 @@ define i32 @non_leaf_sign_all(i32 %x) "sign-return-address"="all" { ; DWARFCFI-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload ; DWARFCFI-V83A-NEXT: retaa ; +; NEGATE-CFI-COMPAT-LABEL: non_leaf_sign_all: +; NEGATE-CFI-COMPAT: // %bb.0: +; NEGATE-CFI-COMPAT-NEXT: hint #25 +; NEGATE-CFI-COMPAT-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-CFI-COMPAT-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-CFI-COMPAT-NEXT: .cfi_offset w30, -16 +; NEGATE-CFI-COMPAT-NEXT: bl foo +; NEGATE-CFI-COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-CFI-COMPAT-NEXT: hint #29 +; NEGATE-CFI-COMPAT-NEXT: ret +; +; NEGATE-CFI-V83A-LABEL: non_leaf_sign_all: +; NEGATE-CFI-V83A: // %bb.0: +; NEGATE-CFI-V83A-NEXT: paciasp +; NEGATE-CFI-V83A-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-CFI-V83A-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-CFI-V83A-NEXT: .cfi_offset w30, -16 +; NEGATE-CFI-V83A-NEXT: bl foo +; NEGATE-CFI-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-CFI-V83A-NEXT: retaa +; ; WINCFI-COMPAT-LABEL: non_leaf_sign_all: ; WINCFI-COMPAT: .seh_proc non_leaf_sign_all ; WINCFI-COMPAT-NEXT: // %bb.0: @@ -228,9 +307,11 @@ define i32 @non_leaf_sign_all(i32 %x) "sign-return-address"="all" { define i32 @non_leaf_sign_non_leaf(i32 %x) "sign-return-address"="non-leaf" { ; DWARFCFI-COMPAT-LABEL: non_leaf_sign_non_leaf: +; ALWAYS-CFI-COMPAT-LABEL: non_leaf_sign_non_leaf: ; DWARFCFI-COMPAT: // %bb.0: ; DWARFCFI-COMPAT-NEXT: hint #25 ; DWARFCFI-COMPAT-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-COMPAT: .cfi_set_ra_state 1, 0 ; DWARFCFI-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; DWARFCFI-COMPAT-NEXT: .cfi_def_cfa_offset 16 ; DWARFCFI-COMPAT-NEXT: .cfi_offset w30, -16 @@ -240,9 +321,11 @@ define i32 @non_leaf_sign_non_leaf(i32 %x) "sign-return-address"="non-leaf" { ; DWARFCFI-COMPAT-NEXT: ret ; ; DWARFCFI-V83A-LABEL: non_leaf_sign_non_leaf: +; ALWAYS-CFI-V83A-LABEL: non_leaf_sign_non_leaf: ; DWARFCFI-V83A: // %bb.0: ; DWARFCFI-V83A-NEXT: paciasp ; DWARFCFI-V83A-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-V83A: .cfi_set_ra_state 1, 0 ; DWARFCFI-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; DWARFCFI-V83A-NEXT: .cfi_def_cfa_offset 16 ; DWARFCFI-V83A-NEXT: .cfi_offset w30, -16 @@ -250,6 +333,29 @@ define i32 @non_leaf_sign_non_leaf(i32 %x) "sign-return-address"="non-leaf" { ; DWARFCFI-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload ; DWARFCFI-V83A-NEXT: retaa ; +; NEGATE-CFI-COMPAT-LABEL: non_leaf_sign_non_leaf: +; NEGATE-CFI-COMPAT: // %bb.0: +; NEGATE-CFI-COMPAT-NEXT: hint #25 +; NEGATE-CFI-COMPAT-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-CFI-COMPAT-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-CFI-COMPAT-NEXT: .cfi_offset w30, -16 +; NEGATE-CFI-COMPAT-NEXT: bl foo +; NEGATE-CFI-COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-CFI-COMPAT-NEXT: hint #29 +; NEGATE-CFI-COMPAT-NEXT: ret +; +; NEGATE-CFI-V83A-LABEL: non_leaf_sign_non_leaf: +; NEGATE-CFI-V83A: // %bb.0: +; NEGATE-CFI-V83A-NEXT: paciasp +; NEGATE-CFI-V83A-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-CFI-V83A-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-CFI-V83A-NEXT: .cfi_offset w30, -16 +; NEGATE-CFI-V83A-NEXT: bl foo +; NEGATE-CFI-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-CFI-V83A-NEXT: retaa +; ; WINCFI-COMPAT-LABEL: non_leaf_sign_non_leaf: ; WINCFI-COMPAT: .seh_proc non_leaf_sign_non_leaf ; WINCFI-COMPAT-NEXT: // %bb.0: @@ -294,9 +400,11 @@ define i32 @non_leaf_sign_non_leaf(i32 %x) "sign-return-address"="non-leaf" { ; Should not use the RETAA instruction. define i32 @non_leaf_scs(i32 %x) "sign-return-address"="non-leaf" shadowcallstack "target-features"="+v8.3a,+reserve-x18" { ; DWARFCFI-LABEL: non_leaf_scs: +; ALWAYS-CFI-LABEL: non_leaf_scs: ; DWARFCFI: // %bb.0: ; DWARFCFI-NEXT: paciasp ; DWARFCFI-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI: .cfi_set_ra_state 1, 0 ; DWARFCFI-NEXT: str x30, [x18], #8 ; DWARFCFI-NEXT: .cfi_escape 0x16, 0x12, 0x02, 0x82, 0x78 // ; DWARFCFI-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill @@ -308,6 +416,21 @@ define i32 @non_leaf_scs(i32 %x) "sign-return-address"="non-leaf" shadowcallstac ; DWARFCFI-NEXT: autiasp ; DWARFCFI-NEXT: ret ; +; NEGATE-CFI-LABEL: non_leaf_scs: +; NEGATE-CFI: // %bb.0: +; NEGATE-CFI-NEXT: paciasp +; NEGATE-CFI-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-NEXT: str x30, [x18], #8 +; NEGATE-CFI-NEXT: .cfi_escape 0x16, 0x12, 0x02, 0x82, 0x78 // +; NEGATE-CFI-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-CFI-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-CFI-NEXT: .cfi_offset w30, -16 +; NEGATE-CFI-NEXT: bl foo +; NEGATE-CFI-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-CFI-NEXT: ldr x30, [x18, #-8]! +; NEGATE-CFI-NEXT: autiasp +; NEGATE-CFI-NEXT: ret +; ; WINCFI-LABEL: non_leaf_scs: ; WINCFI: .seh_proc non_leaf_scs ; WINCFI-NEXT: // %bb.0: @@ -339,11 +462,13 @@ define i32 @non_leaf_scs(i32 %x) "sign-return-address"="non-leaf" shadowcallstac ; By default, pac-ret hardening respects shrink-wrapping optimization. define void @shrink_wrap_sign_non_leaf(i32 %x, i32 %cond) "sign-return-address"="non-leaf" uwtable(async) { ; DWARFCFI-COMPAT-LABEL: shrink_wrap_sign_non_leaf: +; ALWAYS-CFI-COMPAT-LABEL: shrink_wrap_sign_non_leaf: ; DWARFCFI-COMPAT: // %bb.0: // %entry ; DWARFCFI-COMPAT-NEXT: cbnz w1, .LBB8_2 ; DWARFCFI-COMPAT-NEXT: // %bb.1: // %if.then ; DWARFCFI-COMPAT-NEXT: hint #25 ; DWARFCFI-COMPAT-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-COMPAT: .cfi_set_ra_state 1, 0 ; DWARFCFI-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; DWARFCFI-COMPAT-NEXT: .cfi_def_cfa_offset 16 ; DWARFCFI-COMPAT-NEXT: .cfi_offset w30, -16 @@ -353,6 +478,7 @@ define void @shrink_wrap_sign_non_leaf(i32 %x, i32 %cond) "sign-return-address"= ; DWARFCFI-COMPAT-NEXT: .cfi_restore w30 ; DWARFCFI-COMPAT-NEXT: hint #29 ; DWARFCFI-COMPAT-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-COMPAT: .cfi_set_ra_state 0, 0 ; DWARFCFI-COMPAT-NEXT: .LBB8_2: // %exit ; DWARFCFI-COMPAT-NEXT: adrp x8, var ; DWARFCFI-COMPAT-NEXT: mov w9, #42 // =0x2a @@ -360,11 +486,13 @@ define void @shrink_wrap_sign_non_leaf(i32 %x, i32 %cond) "sign-return-address"= ; DWARFCFI-COMPAT-NEXT: ret ; ; DWARFCFI-V83A-LABEL: shrink_wrap_sign_non_leaf: +; ALWAYS-CFI-V83A-LABEL: shrink_wrap_sign_non_leaf: ; DWARFCFI-V83A: // %bb.0: // %entry ; DWARFCFI-V83A-NEXT: cbnz w1, .LBB8_2 ; DWARFCFI-V83A-NEXT: // %bb.1: // %if.then ; DWARFCFI-V83A-NEXT: paciasp ; DWARFCFI-V83A-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-V83A: .cfi_set_ra_state 1, 0 ; DWARFCFI-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; DWARFCFI-V83A-NEXT: .cfi_def_cfa_offset 16 ; DWARFCFI-V83A-NEXT: .cfi_offset w30, -16 @@ -374,12 +502,55 @@ define void @shrink_wrap_sign_non_leaf(i32 %x, i32 %cond) "sign-return-address"= ; DWARFCFI-V83A-NEXT: .cfi_restore w30 ; DWARFCFI-V83A-NEXT: autiasp ; DWARFCFI-V83A-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-V83A: .cfi_set_ra_state 0, 0 ; DWARFCFI-V83A-NEXT: .LBB8_2: // %exit ; DWARFCFI-V83A-NEXT: adrp x8, var ; DWARFCFI-V83A-NEXT: mov w9, #42 // =0x2a ; DWARFCFI-V83A-NEXT: str x9, [x8, :lo12:var] ; DWARFCFI-V83A-NEXT: ret ; +; NEGATE-CFI-COMPAT-LABEL: shrink_wrap_sign_non_leaf: +; NEGATE-CFI-COMPAT: // %bb.0: // %entry +; NEGATE-CFI-COMPAT-NEXT: cbnz w1, .LBB8_2 +; NEGATE-CFI-COMPAT-NEXT: // %bb.1: // %if.then +; NEGATE-CFI-COMPAT-NEXT: hint #25 +; NEGATE-CFI-COMPAT-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-CFI-COMPAT-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-CFI-COMPAT-NEXT: .cfi_offset w30, -16 +; NEGATE-CFI-COMPAT-NEXT: bl foo +; NEGATE-CFI-COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-CFI-COMPAT-NEXT: .cfi_def_cfa_offset 0 +; NEGATE-CFI-COMPAT-NEXT: .cfi_restore w30 +; NEGATE-CFI-COMPAT-NEXT: hint #29 +; NEGATE-CFI-COMPAT-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-COMPAT-NEXT: .LBB8_2: // %exit +; NEGATE-CFI-COMPAT-NEXT: adrp x8, var +; NEGATE-CFI-COMPAT-NEXT: mov w9, #42 // =0x2a +; NEGATE-CFI-COMPAT-NEXT: str x9, [x8, :lo12:var] +; NEGATE-CFI-COMPAT-NEXT: ret +; +; NEGATE-CFI-V83A-LABEL: shrink_wrap_sign_non_leaf: +; NEGATE-CFI-V83A: // %bb.0: // %entry +; NEGATE-CFI-V83A-NEXT: cbnz w1, .LBB8_2 +; NEGATE-CFI-V83A-NEXT: // %bb.1: // %if.then +; NEGATE-CFI-V83A-NEXT: paciasp +; NEGATE-CFI-V83A-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-CFI-V83A-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-CFI-V83A-NEXT: .cfi_offset w30, -16 +; NEGATE-CFI-V83A-NEXT: bl foo +; NEGATE-CFI-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-CFI-V83A-NEXT: .cfi_def_cfa_offset 0 +; NEGATE-CFI-V83A-NEXT: .cfi_restore w30 +; NEGATE-CFI-V83A-NEXT: autiasp +; NEGATE-CFI-V83A-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-V83A-NEXT: .LBB8_2: // %exit +; NEGATE-CFI-V83A-NEXT: adrp x8, var +; NEGATE-CFI-V83A-NEXT: mov w9, #42 // =0x2a +; NEGATE-CFI-V83A-NEXT: str x9, [x8, :lo12:var] +; NEGATE-CFI-V83A-NEXT: ret +; ; WINCFI-COMPAT-LABEL: shrink_wrap_sign_non_leaf: ; WINCFI-COMPAT: .seh_proc shrink_wrap_sign_non_leaf ; WINCFI-COMPAT-NEXT: // %bb.0: // %entry @@ -444,9 +615,11 @@ exit: ; function, ignoring shrink-wrapping. define void @shrink_wrap_sign_all(i32 %x, i32 %cond) "sign-return-address"="all" uwtable(async) { ; DWARFCFI-COMPAT-LABEL: shrink_wrap_sign_all: +; ALWAYS-CFI-COMPAT-LABEL: shrink_wrap_sign_all: ; DWARFCFI-COMPAT: // %bb.0: // %entry ; DWARFCFI-COMPAT-NEXT: hint #25 ; DWARFCFI-COMPAT-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-COMPAT: .cfi_set_ra_state 1, 0 ; DWARFCFI-COMPAT-NEXT: cbnz w1, .LBB9_2 ; DWARFCFI-COMPAT-NEXT: // %bb.1: // %if.then ; DWARFCFI-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill @@ -462,12 +635,15 @@ define void @shrink_wrap_sign_all(i32 %x, i32 %cond) "sign-return-address"="all" ; DWARFCFI-COMPAT-NEXT: str x9, [x8, :lo12:var] ; DWARFCFI-COMPAT-NEXT: hint #29 ; DWARFCFI-COMPAT-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-COMPAT: .cfi_set_ra_state 0, 0 ; DWARFCFI-COMPAT-NEXT: ret ; ; DWARFCFI-V83A-LABEL: shrink_wrap_sign_all: +; ALWAYS-CFI-V83A-LABEL: shrink_wrap_sign_all: ; DWARFCFI-V83A: // %bb.0: // %entry ; DWARFCFI-V83A-NEXT: paciasp ; DWARFCFI-V83A-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-V83A: .cfi_set_ra_state 1, 0 ; DWARFCFI-V83A-NEXT: cbnz w1, .LBB9_2 ; DWARFCFI-V83A-NEXT: // %bb.1: // %if.then ; DWARFCFI-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill @@ -483,6 +659,46 @@ define void @shrink_wrap_sign_all(i32 %x, i32 %cond) "sign-return-address"="all" ; DWARFCFI-V83A-NEXT: str x9, [x8, :lo12:var] ; DWARFCFI-V83A-NEXT: retaa ; +; NEGATE-CFI-COMPAT-LABEL: shrink_wrap_sign_all: +; NEGATE-CFI-COMPAT: // %bb.0: // %entry +; NEGATE-CFI-COMPAT-NEXT: hint #25 +; NEGATE-CFI-COMPAT-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-COMPAT-NEXT: cbnz w1, .LBB9_2 +; NEGATE-CFI-COMPAT-NEXT: // %bb.1: // %if.then +; NEGATE-CFI-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-CFI-COMPAT-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-CFI-COMPAT-NEXT: .cfi_offset w30, -16 +; NEGATE-CFI-COMPAT-NEXT: bl foo +; NEGATE-CFI-COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-CFI-COMPAT-NEXT: .cfi_def_cfa_offset 0 +; NEGATE-CFI-COMPAT-NEXT: .cfi_restore w30 +; NEGATE-CFI-COMPAT-NEXT: .LBB9_2: // %exit +; NEGATE-CFI-COMPAT-NEXT: adrp x8, var +; NEGATE-CFI-COMPAT-NEXT: mov w9, #42 // =0x2a +; NEGATE-CFI-COMPAT-NEXT: str x9, [x8, :lo12:var] +; NEGATE-CFI-COMPAT-NEXT: hint #29 +; NEGATE-CFI-COMPAT-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-COMPAT-NEXT: ret +; +; NEGATE-CFI-V83A-LABEL: shrink_wrap_sign_all: +; NEGATE-CFI-V83A: // %bb.0: // %entry +; NEGATE-CFI-V83A-NEXT: paciasp +; NEGATE-CFI-V83A-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-V83A-NEXT: cbnz w1, .LBB9_2 +; NEGATE-CFI-V83A-NEXT: // %bb.1: // %if.then +; NEGATE-CFI-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-CFI-V83A-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-CFI-V83A-NEXT: .cfi_offset w30, -16 +; NEGATE-CFI-V83A-NEXT: bl foo +; NEGATE-CFI-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-CFI-V83A-NEXT: .cfi_def_cfa_offset 0 +; NEGATE-CFI-V83A-NEXT: .cfi_restore w30 +; NEGATE-CFI-V83A-NEXT: .LBB9_2: // %exit +; NEGATE-CFI-V83A-NEXT: adrp x8, var +; NEGATE-CFI-V83A-NEXT: mov w9, #42 // =0x2a +; NEGATE-CFI-V83A-NEXT: str x9, [x8, :lo12:var] +; NEGATE-CFI-V83A-NEXT: retaa +; ; WINCFI-COMPAT-LABEL: shrink_wrap_sign_all: ; WINCFI-COMPAT: .seh_proc shrink_wrap_sign_all ; WINCFI-COMPAT-NEXT: // %bb.0: // %entry @@ -545,11 +761,19 @@ exit: define i32 @leaf_sign_all_v83(i32 %x) "sign-return-address"="all" "target-features"="+v8.3a" { ; DWARFCFI-LABEL: leaf_sign_all_v83: +; ALWAYS-CFI-LABEL: leaf_sign_all_v83: ; DWARFCFI: // %bb.0: ; DWARFCFI-NEXT: paciasp ; DWARFCFI-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI: .cfi_set_ra_state 1, 0 ; DWARFCFI-NEXT: retaa ; +; NEGATE-CFI-LABEL: leaf_sign_all_v83: +; NEGATE-CFI: // %bb.0: +; NEGATE-CFI-NEXT: paciasp +; NEGATE-CFI-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-NEXT: retaa +; ; WINCFI-LABEL: leaf_sign_all_v83: ; WINCFI: .seh_proc leaf_sign_all_v83 ; WINCFI-NEXT: // %bb.0: @@ -570,9 +794,11 @@ declare fastcc i64 @bar(i64) define fastcc void @spill_lr_and_tail_call(i64 %x) "sign-return-address"="all" { ; DWARFCFI-COMPAT-LABEL: spill_lr_and_tail_call: +; ALWAYS-CFI-COMPAT-LABEL: spill_lr_and_tail_call: ; DWARFCFI-COMPAT: // %bb.0: ; DWARFCFI-COMPAT-NEXT: hint #25 ; DWARFCFI-COMPAT-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-COMPAT: .cfi_set_ra_state 1, 0 ; DWARFCFI-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; DWARFCFI-COMPAT-NEXT: .cfi_def_cfa_offset 16 ; DWARFCFI-COMPAT-NEXT: .cfi_offset w30, -16 @@ -584,9 +810,11 @@ define fastcc void @spill_lr_and_tail_call(i64 %x) "sign-return-address"="all" { ; DWARFCFI-COMPAT-NEXT: b bar ; ; DWARFCFI-V83A-LABEL: spill_lr_and_tail_call: +; ALWAYS-CFI-V83A-LABEL: spill_lr_and_tail_call: ; DWARFCFI-V83A: // %bb.0: ; DWARFCFI-V83A-NEXT: paciasp ; DWARFCFI-V83A-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-V83A: .cfi_set_ra_state 1, 0 ; DWARFCFI-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; DWARFCFI-V83A-NEXT: .cfi_def_cfa_offset 16 ; DWARFCFI-V83A-NEXT: .cfi_offset w30, -16 @@ -597,6 +825,34 @@ define fastcc void @spill_lr_and_tail_call(i64 %x) "sign-return-address"="all" { ; DWARFCFI-V83A-NEXT: autiasp ; DWARFCFI-V83A-NEXT: b bar ; +; NEGATE-CFI-COMPAT-LABEL: spill_lr_and_tail_call: +; NEGATE-CFI-COMPAT: // %bb.0: +; NEGATE-CFI-COMPAT-NEXT: hint #25 +; NEGATE-CFI-COMPAT-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-COMPAT-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-CFI-COMPAT-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-CFI-COMPAT-NEXT: .cfi_offset w30, -16 +; NEGATE-CFI-COMPAT-NEXT: //APP +; NEGATE-CFI-COMPAT-NEXT: mov x30, x0 +; NEGATE-CFI-COMPAT-NEXT: //NO_APP +; NEGATE-CFI-COMPAT-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-CFI-COMPAT-NEXT: hint #29 +; NEGATE-CFI-COMPAT-NEXT: b bar +; +; NEGATE-CFI-V83A-LABEL: spill_lr_and_tail_call: +; NEGATE-CFI-V83A: // %bb.0: +; NEGATE-CFI-V83A-NEXT: paciasp +; NEGATE-CFI-V83A-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-V83A-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; NEGATE-CFI-V83A-NEXT: .cfi_def_cfa_offset 16 +; NEGATE-CFI-V83A-NEXT: .cfi_offset w30, -16 +; NEGATE-CFI-V83A-NEXT: //APP +; NEGATE-CFI-V83A-NEXT: mov x30, x0 +; NEGATE-CFI-V83A-NEXT: //NO_APP +; NEGATE-CFI-V83A-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; NEGATE-CFI-V83A-NEXT: autiasp +; NEGATE-CFI-V83A-NEXT: b bar +; ; WINCFI-COMPAT-LABEL: spill_lr_and_tail_call: ; WINCFI-COMPAT: .seh_proc spill_lr_and_tail_call ; WINCFI-COMPAT-NEXT: // %bb.0: @@ -645,36 +901,72 @@ define fastcc void @spill_lr_and_tail_call(i64 %x) "sign-return-address"="all" { define i32 @leaf_sign_all_a_key(i32 %x) "sign-return-address"="all" "sign-return-address-key"="a_key" { ; DWARFCFI-COMPAT-LABEL: leaf_sign_all_a_key: +; ALWAYS-CFI-COMPAT-LABEL: leaf_sign_all_a_key: ; DWARFCFI-COMPAT: // %bb.0: ; DWARFCFI-COMPAT-NEXT: hint #25 ; DWARFCFI-COMPAT-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-COMPAT: .cfi_set_ra_state 1, 0 ; DWARFCFI-COMPAT-NEXT: hint #29 ; DWARFCFI-COMPAT-NEXT: ret ; ; DWARFCFI-V83A-LABEL: leaf_sign_all_a_key: +; ALWAYS-CFI-V83A-LABEL: leaf_sign_all_a_key: ; DWARFCFI-V83A: // %bb.0: ; DWARFCFI-V83A-NEXT: paciasp ; DWARFCFI-V83A-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-V83A: .cfi_set_ra_state 1, 0 ; DWARFCFI-V83A-NEXT: retaa +; +; NEGATE-CFI-COMPAT-LABEL: leaf_sign_all_a_key: +; NEGATE-CFI-COMPAT: // %bb.0: +; NEGATE-CFI-COMPAT-NEXT: hint #25 +; NEGATE-CFI-COMPAT-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-COMPAT-NEXT: hint #29 +; NEGATE-CFI-COMPAT-NEXT: ret +; +; NEGATE-CFI-V83A-LABEL: leaf_sign_all_a_key: +; NEGATE-CFI-V83A: // %bb.0: +; NEGATE-CFI-V83A-NEXT: paciasp +; NEGATE-CFI-V83A-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-V83A-NEXT: retaa ret i32 %x } define i32 @leaf_sign_all_b_key(i32 %x) "sign-return-address"="all" "sign-return-address-key"="b_key" { ; DWARFCFI-COMPAT-LABEL: leaf_sign_all_b_key: +; ALWAYS-CFI-COMPAT-LABEL: leaf_sign_all_b_key: ; DWARFCFI-COMPAT: // %bb.0: ; DWARFCFI-COMPAT-NEXT: .cfi_b_key_frame ; DWARFCFI-COMPAT-NEXT: hint #27 ; DWARFCFI-COMPAT-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-COMPAT: .cfi_set_ra_state 1, 0 ; DWARFCFI-COMPAT-NEXT: hint #31 ; DWARFCFI-COMPAT-NEXT: ret ; ; DWARFCFI-V83A-LABEL: leaf_sign_all_b_key: +; ALWAYS-CFI-V83A-LABEL: leaf_sign_all_b_key: ; DWARFCFI-V83A: // %bb.0: ; DWARFCFI-V83A-NEXT: .cfi_b_key_frame ; DWARFCFI-V83A-NEXT: pacibsp ; DWARFCFI-V83A-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-V83A: .cfi_set_ra_state 1, 0 ; DWARFCFI-V83A-NEXT: retab ; +; NEGATE-CFI-COMPAT-LABEL: leaf_sign_all_b_key: +; NEGATE-CFI-COMPAT: // %bb.0: +; NEGATE-CFI-COMPAT-NEXT: .cfi_b_key_frame +; NEGATE-CFI-COMPAT-NEXT: hint #27 +; NEGATE-CFI-COMPAT-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-COMPAT-NEXT: hint #31 +; NEGATE-CFI-COMPAT-NEXT: ret +; +; NEGATE-CFI-V83A-LABEL: leaf_sign_all_b_key: +; NEGATE-CFI-V83A: // %bb.0: +; NEGATE-CFI-V83A-NEXT: .cfi_b_key_frame +; NEGATE-CFI-V83A-NEXT: pacibsp +; NEGATE-CFI-V83A-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-V83A-NEXT: retab +; ; WINCFI-COMPAT-LABEL: leaf_sign_all_b_key: ; WINCFI-COMPAT: .seh_proc leaf_sign_all_b_key ; WINCFI-COMPAT-NEXT: // %bb.0: @@ -707,12 +999,21 @@ define i32 @leaf_sign_all_b_key(i32 %x) "sign-return-address"="all" "sign-return define i32 @leaf_sign_all_v83_b_key(i32 %x) "sign-return-address"="all" "target-features"="+v8.3a" "sign-return-address-key"="b_key" { ; DWARFCFI-LABEL: leaf_sign_all_v83_b_key: +; ALWAYS-CFI-LABEL: leaf_sign_all_v83_b_key: ; DWARFCFI: // %bb.0: ; DWARFCFI-NEXT: .cfi_b_key_frame ; DWARFCFI-NEXT: pacibsp ; DWARFCFI-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI: .cfi_set_ra_state 1, 0 ; DWARFCFI-NEXT: retab ; +; NEGATE-CFI-LABEL: leaf_sign_all_v83_b_key: +; NEGATE-CFI: // %bb.0: +; NEGATE-CFI-NEXT: .cfi_b_key_frame +; NEGATE-CFI-NEXT: pacibsp +; NEGATE-CFI-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-NEXT: retab +; ; WINCFI-LABEL: leaf_sign_all_v83_b_key: ; WINCFI: .seh_proc leaf_sign_all_v83_b_key ; WINCFI-NEXT: // %bb.0: @@ -732,37 +1033,73 @@ define i32 @leaf_sign_all_v83_b_key(i32 %x) "sign-return-address"="all" "target- ; Note that BTI instruction is not needed before PACIASP. define i32 @leaf_sign_all_a_key_bti(i32 %x) "sign-return-address"="all" "sign-return-address-key"="a_key" "branch-target-enforcement"{ ; DWARFCFI-COMPAT-LABEL: leaf_sign_all_a_key_bti: +; ALWAYS-CFI-COMPAT-LABEL: leaf_sign_all_a_key_bti: ; DWARFCFI-COMPAT: // %bb.0: ; DWARFCFI-COMPAT-NEXT: hint #25 ; DWARFCFI-COMPAT-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-COMPAT: .cfi_set_ra_state 1, 0 ; DWARFCFI-COMPAT-NEXT: hint #29 ; DWARFCFI-COMPAT-NEXT: ret ; ; DWARFCFI-V83A-LABEL: leaf_sign_all_a_key_bti: +; ALWAYS-CFI-V83A-LABEL: leaf_sign_all_a_key_bti: ; DWARFCFI-V83A: // %bb.0: ; DWARFCFI-V83A-NEXT: paciasp ; DWARFCFI-V83A-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-V83A: .cfi_set_ra_state 1, 0 ; DWARFCFI-V83A-NEXT: retaa +; +; NEGATE-CFI-COMPAT-LABEL: leaf_sign_all_a_key_bti: +; NEGATE-CFI-COMPAT: // %bb.0: +; NEGATE-CFI-COMPAT-NEXT: hint #25 +; NEGATE-CFI-COMPAT-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-COMPAT-NEXT: hint #29 +; NEGATE-CFI-COMPAT-NEXT: ret +; +; NEGATE-CFI-V83A-LABEL: leaf_sign_all_a_key_bti: +; NEGATE-CFI-V83A: // %bb.0: +; NEGATE-CFI-V83A-NEXT: paciasp +; NEGATE-CFI-V83A-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-V83A-NEXT: retaa ret i32 %x } ; Note that BTI instruction is not needed before PACIBSP. define i32 @leaf_sign_all_b_key_bti(i32 %x) "sign-return-address"="all" "sign-return-address-key"="b_key" "branch-target-enforcement"{ ; DWARFCFI-COMPAT-LABEL: leaf_sign_all_b_key_bti: +; ALWAYS-CFI-COMPAT-LABEL: leaf_sign_all_b_key_bti: ; DWARFCFI-COMPAT: // %bb.0: ; DWARFCFI-COMPAT-NEXT: .cfi_b_key_frame ; DWARFCFI-COMPAT-NEXT: hint #27 ; DWARFCFI-COMPAT-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-COMPAT: .cfi_set_ra_state 1, 0 ; DWARFCFI-COMPAT-NEXT: hint #31 ; DWARFCFI-COMPAT-NEXT: ret ; ; DWARFCFI-V83A-LABEL: leaf_sign_all_b_key_bti: +; ALWAYS-CFI-V83A-LABEL: leaf_sign_all_b_key_bti: ; DWARFCFI-V83A: // %bb.0: ; DWARFCFI-V83A-NEXT: .cfi_b_key_frame ; DWARFCFI-V83A-NEXT: pacibsp ; DWARFCFI-V83A-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI-V83A: .cfi_set_ra_state 1, 0 ; DWARFCFI-V83A-NEXT: retab ; +; NEGATE-CFI-COMPAT-LABEL: leaf_sign_all_b_key_bti: +; NEGATE-CFI-COMPAT: // %bb.0: +; NEGATE-CFI-COMPAT-NEXT: .cfi_b_key_frame +; NEGATE-CFI-COMPAT-NEXT: hint #27 +; NEGATE-CFI-COMPAT-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-COMPAT-NEXT: hint #31 +; NEGATE-CFI-COMPAT-NEXT: ret +; +; NEGATE-CFI-V83A-LABEL: leaf_sign_all_b_key_bti: +; NEGATE-CFI-V83A: // %bb.0: +; NEGATE-CFI-V83A-NEXT: .cfi_b_key_frame +; NEGATE-CFI-V83A-NEXT: pacibsp +; NEGATE-CFI-V83A-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-V83A-NEXT: retab +; ; WINCFI-COMPAT-LABEL: leaf_sign_all_b_key_bti: ; WINCFI-COMPAT: .seh_proc leaf_sign_all_b_key_bti ; WINCFI-COMPAT-NEXT: // %bb.0: @@ -796,12 +1133,21 @@ define i32 @leaf_sign_all_b_key_bti(i32 %x) "sign-return-address"="all" "sign-re ; Note that BTI instruction is not needed before PACIBSP. define i32 @leaf_sign_all_v83_b_key_bti(i32 %x) "sign-return-address"="all" "target-features"="+v8.3a" "sign-return-address-key"="b_key" "branch-target-enforcement" { ; DWARFCFI-LABEL: leaf_sign_all_v83_b_key_bti: +; ALWAYS-CFI-LABEL: leaf_sign_all_v83_b_key_bti: ; DWARFCFI: // %bb.0: ; DWARFCFI-NEXT: .cfi_b_key_frame ; DWARFCFI-NEXT: pacibsp ; DWARFCFI-NEXT: .cfi_negate_ra_state +; ALWAYS-CFI: .cfi_set_ra_state 1, 0 ; DWARFCFI-NEXT: retab ; +; NEGATE-CFI-LABEL: leaf_sign_all_v83_b_key_bti: +; NEGATE-CFI: // %bb.0: +; NEGATE-CFI-NEXT: .cfi_b_key_frame +; NEGATE-CFI-NEXT: pacibsp +; NEGATE-CFI-NEXT: .cfi_negate_ra_state +; NEGATE-CFI-NEXT: retab +; ; WINCFI-LABEL: leaf_sign_all_v83_b_key_bti: ; WINCFI: .seh_proc leaf_sign_all_v83_b_key_bti ; WINCFI-NEXT: // %bb.0: diff --git a/llvm/test/CodeGen/MIR/AArch64/cfi.mir b/llvm/test/CodeGen/MIR/AArch64/cfi.mir index efa053699b491..7e08c4040b9bf 100644 --- a/llvm/test/CodeGen/MIR/AArch64/cfi.mir +++ b/llvm/test/CodeGen/MIR/AArch64/cfi.mir @@ -49,4 +49,8 @@ body: | ; CHECK: CFI_INSTRUCTION window_save CFI_INSTRUCTION negate_ra_sign_state ; CHECK: CFI_INSTRUCTION negate_ra_sign_state + CFI_INSTRUCTION negate_ra_sign_state_with_pc + ; CHECK: CFI_INSTRUCTION negate_ra_sign_state_with_pc + CFI_INSTRUCTION llvm_set_ra_state 2, 0 + ; CHECK: CFI_INSTRUCTION llvm_set_ra_state 2, 0 RET_ReallyLR diff --git a/llvm/test/CodeGen/MIR/AArch64/return-address-signing.mir b/llvm/test/CodeGen/MIR/AArch64/return-address-signing.mir index b2bce451dff91..1fc4aa25b0512 100644 --- a/llvm/test/CodeGen/MIR/AArch64/return-address-signing.mir +++ b/llvm/test/CodeGen/MIR/AArch64/return-address-signing.mir @@ -1,4 +1,4 @@ -# RUN: llc -mtriple=aarch64 -run-pass=prolog-epilog -run-pass=aarch64-ptrauth -o - %s 2>&1 | FileCheck --strict-whitespace %s +# RUN: llc -mtriple=aarch64 -run-pass=prolog-epilog -run-pass=aarch64-ptrauth -aarch64-cfi-llvm-set-ra-sign-state=pauth-lr -o - %s 2>&1 | FileCheck --strict-whitespace %s --- | target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64" @@ -59,8 +59,8 @@ tracksRegLiveness: true frameInfo: maxCallFrameSize: 0 #CHECK: frame-setup PACM -#CHECK: frame-setup CFI_INSTRUCTION negate_ra_sign_state_with_pc #CHECK: frame-setup PACIASP implicit-def $lr, implicit $lr, implicit $sp, pre-instr-symbol +#CHECK: frame-setup CFI_INSTRUCTION llvm_set_ra_state 2, #CHECK: frame-destroy PACM #CHECK: frame-destroy AUTIASP implicit-def $lr, implicit $lr, implicit $sp body: | diff --git a/llvm/test/MC/AArch64/cfi_set_ra_state-shrinkwrap.s b/llvm/test/MC/AArch64/cfi_set_ra_state-shrinkwrap.s new file mode 100644 index 0000000000000..49354febaf538 --- /dev/null +++ b/llvm/test/MC/AArch64/cfi_set_ra_state-shrinkwrap.s @@ -0,0 +1,142 @@ +# RUN: llvm-mc -triple arm64e-apple-darwin -mattr=+pauth-lr %s -filetype=obj -o %t.o +# RUN: llvm-objdump --macho --unwind-info --dwarf=frames %t.o | FileCheck %s + +# Vanilla ptrauth-returns=pauth frame. + .globl _pauth +_pauth: + .cfi_startproc + .cfi_set_ra_state 1, 0 + pacibsp + + .cfi_set_ra_state 0, -4 + retab + .cfi_endproc + + +# Vanilla ptrauth-returns=pauth-lr frame, with .cfi_set_ra_state instead of +# the deprecated .cfi_negate_ra_state_with_pc + .globl _pauthlr +_pauthlr: + .cfi_startproc + .cfi_set_ra_state 2, 0 + pacibsppc + + .cfi_set_ra_state 0, -4 + retabsppc _pauthlr + .cfi_endproc + + +# Shrink-wrapped frame, with a strange block ordering that puts the aut before +# the pac, which is not representable via .cfi_negate_ra_state_with_pc +# See: https://github.com/ARM-software/abi-aa/issues/327 + .globl _pauthlr_shrinkwrap +_pauthlr_shrinkwrap: + .cfi_startproc + cmp w0, #42 + b.lt LBB0 + ret + +LBB1: + ldp x29, x30, [sp], #16 + .cfi_def_cfa_offset 0 + .cfi_restore x29 + .cfi_restore x30 + + adrp x16, LBB0@PAGE + add x16, x16, LBB0@PAGEOFF + .cfi_set_ra_state 0, 16 + autibsppcr x16 + + mov w0, #42 + ret + +LBB0: + pacibsppc + .cfi_set_ra_state 2, LBB0 + + stp x29, x30, [sp, #-16]! + .cfi_def_cfa_offset 16 + .cfi_offset x29, -16 + .cfi_offset x30, -8 + + b LBB1 + .cfi_endproc + + +# CHECK-LABEL: Contents of __compact_unwind section: +# CHECK-NEXT: Entry at offset 0x0: +# CHECK-NEXT: start: 0x0 ltmp0 +# CHECK-NEXT: length: 0x8 +# CHECK-NEXT: compact encoding: 0x03000000 +# CHECK-NEXT: Entry at offset 0x20: +# CHECK-NEXT: start: 0x8 _pauthlr +# CHECK-NEXT: length: 0x8 +# CHECK-NEXT: compact encoding: 0x03000000 +# CHECK-NEXT: Entry at offset 0x40: +# CHECK-NEXT: start: 0x10 _pauthlr_shrinkwrap +# CHECK-NEXT: length: 0x30 +# CHECK-NEXT: compact encoding: 0x03000000 +# CHECK-EMPTY: +# CHECK-LABEL: .debug_frame contents: +# CHECK-EMPTY: +# CHECK-EMPTY: +# CHECK-LABEL: .eh_frame contents: +# CHECK-EMPTY: +# CHECK-NEXT: 00000000 00000010 00000000 CIE +# CHECK-NEXT: Format: DWARF32 +# CHECK-NEXT: Version: 1 +# CHECK-NEXT: Augmentation: "zR" +# CHECK-NEXT: Code alignment factor: 1 +# CHECK-NEXT: Data alignment factor: -8 +# CHECK-NEXT: Return address column: 30 +# CHECK-NEXT: Augmentation data: 10 +# CHECK-EMPTY: +# CHECK-NEXT: DW_CFA_def_cfa: reg31 +0 +# CHECK-EMPTY: +# CHECK-NEXT: CFA=reg31 +# CHECK-EMPTY: +# CHECK-NEXT: 00000014 00000020 00000018 FDE cie=00000000 pc=000000a0...000000a8 +# CHECK-NEXT: Format: DWARF32 +# CHECK-NEXT: DW_CFA_AARCH64_set_ra_state: 1 0 +# CHECK-NEXT: DW_CFA_advance_loc: 4 to 0xa4 +# CHECK-NEXT: DW_CFA_AARCH64_set_ra_state: 0 -4 +# CHECK-NEXT: DW_CFA_nop: +# CHECK-NEXT: DW_CFA_nop: +# CHECK-NEXT: DW_CFA_nop: +# CHECK-NEXT: DW_CFA_nop: +# CHECK-EMPTY: +# CHECK-NEXT: 0xa0: CFA=reg31: reg34=1 +# CHECK-NEXT: 0xa4: CFA=reg31: reg34=0 +# CHECK-EMPTY: +# CHECK-NEXT: 00000038 0000001c 0000003c FDE cie=00000000 pc=000000a0...000000a8 +# CHECK-NEXT: Format: DWARF32 +# CHECK-NEXT: DW_CFA_AARCH64_set_ra_state: 2 0 +# CHECK-NEXT: DW_CFA_advance_loc: 4 to 0xa4 +# CHECK-NEXT: DW_CFA_AARCH64_set_ra_state: 0 -4 +# CHECK-EMPTY: +# CHECK-NEXT: 0xa0: CFA=reg31: reg34=2 +# CHECK-NEXT: 0xa4: CFA=reg31: reg34=0 +# CHECK-EMPTY: +# CHECK-NEXT: 00000058 0000002c 0000005c FDE cie=00000000 pc=000000a0...000000d0 +# CHECK-NEXT: Format: DWARF32 +# CHECK-NEXT: DW_CFA_advance_loc: 16 to 0xb0 +# CHECK-NEXT: DW_CFA_def_cfa_offset: +0 +# CHECK-NEXT: DW_CFA_restore: reg29 +# CHECK-NEXT: DW_CFA_restore: reg30 +# CHECK-NEXT: DW_CFA_advance_loc: 8 to 0xb8 +# CHECK-NEXT: DW_CFA_AARCH64_set_ra_state: 0 16 +# CHECK-NEXT: DW_CFA_advance_loc: 16 to 0xc8 +# CHECK-NEXT: DW_CFA_AARCH64_set_ra_state: 2 -4 +# CHECK-NEXT: DW_CFA_advance_loc: 4 to 0xcc +# CHECK-NEXT: DW_CFA_def_cfa_offset: +16 +# CHECK-NEXT: DW_CFA_offset: reg29 -16 +# CHECK-NEXT: DW_CFA_offset: reg30 -8 +# CHECK-NEXT: DW_CFA_nop: +# CHECK-NEXT: DW_CFA_nop: +# CHECK-NEXT: DW_CFA_nop: +# CHECK-EMPTY: +# CHECK-NEXT: 0xa0: CFA=reg31 +# CHECK-NEXT: 0xb0: CFA=reg31 +# CHECK-NEXT: 0xb8: CFA=reg31: reg34=0 +# CHECK-NEXT: 0xc8: CFA=reg31: reg34=2 +# CHECK-NEXT: 0xcc: CFA=reg31+16: reg29=[CFA-16], reg30=[CFA-8], reg34=2 diff --git a/llvm/test/MC/AArch64/cfi_set_ra_state.s b/llvm/test/MC/AArch64/cfi_set_ra_state.s new file mode 100644 index 0000000000000..f42a2f4cbb2bb --- /dev/null +++ b/llvm/test/MC/AArch64/cfi_set_ra_state.s @@ -0,0 +1,45 @@ +# RUN: llvm-mc -triple aarch64-pc-linux-gnu -mattr=+pauth-lr %s -filetype=asm -o - | FileCheck %s --check-prefix=ASM + +# RUN: llvm-mc -triple aarch64-pc-linux-gnu -mattr=+pauth-lr %s -filetype=obj -o %t.o +# RUN: llvm-dwarfdump --eh-frame %t.o | FileCheck %s --check-prefix=DWARF +# RUN: llvm-readobj --hex-dump=.eh_frame %t.o | FileCheck %s --check-prefix=HEX + + .cfi_startproc + .cfi_set_ra_state 1, 0 + .cfi_set_ra_state 2, 0 + .cfi_set_ra_state 0, -4 + .cfi_set_ra_state 2, LSigningInstrAfter +LSigningInstrBefore: + nop + nop +LSigningInstrAfter: + .cfi_set_ra_state 2, LSigningInstrBefore + .cfi_endproc + +# ASM: .cfi_startproc +# ASM-NEXT: .cfi_set_ra_state 1, 0 +# ASM-NEXT: .cfi_set_ra_state 2, 0 +# ASM-NEXT: .cfi_set_ra_state 0, -4 +# ASM-NEXT: .cfi_set_ra_state 2, LSigningInstrAfter +# ASM: .cfi_set_ra_state 2, LSigningInstrBefore +# ASM-NEXT: .cfi_endproc + +# DWARF: DW_CFA_AARCH64_set_ra_state: 1 0 +# DWARF-NEXT: DW_CFA_AARCH64_set_ra_state: 2 0 +# DWARF-NEXT: DW_CFA_AARCH64_set_ra_state: 0 -4 +# DWARF-NEXT: DW_CFA_AARCH64_set_ra_state: 2 8 +# DWARF: DW_CFA_AARCH64_set_ra_state: 2 -8 + +# Verify the raw encoding: +# +# opcode 0x2b, then ULEB128(state), then SLEB128(offset). +# +# state=1, offset= 0 => 2b 01 00 +# state=2, offset= 0 => 2b 02 00 +# state=0, offset= -4 => 2b 00 7c +# state=2, offset= +8 => 2b 02 08 +# (advance_loc 8) => 48 +# state=2, offset= -8 => 2b 02 78 +# +# HEX: 002b0100 2b02002b 007c2b02 +# HEX: 08482b02 78 diff --git a/llvm/test/MC/AArch64/directives-case_insensitive.s b/llvm/test/MC/AArch64/directives-case_insensitive.s index c2bdec73e349e..88f70521543f9 100644 --- a/llvm/test/MC/AArch64/directives-case_insensitive.s +++ b/llvm/test/MC/AArch64/directives-case_insensitive.s @@ -33,11 +33,13 @@ fred .REQ x5 .CFI_STARTPROC .CFI_NEGATE_RA_STATE .CFI_NEGATE_RA_STATE_WITH_PC +.CFI_SET_RA_STATE 2, 0 .CFI_B_KEY_FRAME .CFI_ENDPROC // CHECK: .cfi_startproc // CHECK: .cfi_negate_ra_state // CHECK: .cfi_negate_ra_state_with_pc +// CHECK: .cfi_set_ra_state 2, 0 // CHECK: .cfi_b_key_frame // CHECK: .cfi_endproc diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp index 5b7e90092744c..f225f30de61c8 100644 --- a/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp +++ b/llvm/unittests/DebugInfo/DWARF/DWARFDebugFrameTest.cpp @@ -176,6 +176,7 @@ TEST(DWARFDebugFrame, InvalidCFIOpcodesTest) { dwarf::DW_CFA_GNU_window_save, dwarf::DW_CFA_AARCH64_negate_ra_state, dwarf::DW_CFA_AARCH64_negate_ra_state_with_pc, + dwarf::DW_CFA_AARCH64_set_ra_state, dwarf::DW_CFA_GNU_args_size}; dwarf::CIE TestCIE = createCIE(/*IsDWARF64=*/false, From 866832a59612e569c8d18bc8e846cc32e9a5730b Mon Sep 17 00:00:00 2001 From: Farzon Lotfi Date: Tue, 21 Jul 2026 17:24:57 -0400 Subject: [PATCH 34/50] [SPIRV] Fix legalization of widened cbuffer vector loads (#209912) fixes #191070 In the SPIR-V backend an optimization has widened a narrow cbuffer load into a wider vector load that reads trailing padding (e.g. a <1 x float> float1x4 column loaded as <4 x float>). loadVectorFromVector only handled down-casts and asserts when the target has more elements than the source. We need to handle the widening case by loading the available lanes and inserting them into a poison target vector, leaving the padding lanes as poison. Assisted by Claude Opus 4.8 --- .../Target/SPIRV/SPIRVLegalizePointerCast.cpp | 40 +++++++++++++++++-- .../SPIRV/passes/SPIRVLegalizePointerCast.ll | 16 ++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp index 38960b23b26db..847eda7880c03 100644 --- a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp @@ -138,9 +138,27 @@ class SPIRVLegalizePointerCastImpl { return AssignValue; } - assert(TargetType->getNumElements() < SourceType->getNumElements()); - SmallVector Mask(/* Size= */ TargetType->getNumElements()); - for (unsigned I = 0; I < TargetType->getNumElements(); ++I) + auto *AssignVecTy = cast(AssignValue->getType()); + const unsigned NumTarget = TargetType->getNumElements(); + const unsigned NumSource = AssignVecTy->getNumElements(); + + // Optimizations may widen a narrow load to cover padding (e.g., loading a + // <1 x float> column as <4 x float>). Since extra lanes read trailing + // padding, insert only the valid lanes into a poison vector to avoid poison + // scalars. + if (NumTarget > NumSource) { + Value *Result = PoisonValue::get(TargetType); + buildAssignType(B, TargetType, Result); + for (unsigned I = 0; I < NumSource; ++I) { + Value *Scalar = extractScalarFromVector(B, AssignValue, I); + Result = makeInsertElement(B, Result, Scalar, I); + } + return Result; + } + + assert(NumTarget < NumSource); + SmallVector Mask(/* Size= */ NumTarget); + for (unsigned I = 0; I < NumTarget; ++I) Mask[I] = I; Value *Output = B.CreateShuffleVector(AssignValue, AssignValue, Mask); buildAssignType(B, TargetType, Output); @@ -427,6 +445,22 @@ class SPIRVLegalizePointerCastImpl { return NewI; } + // Extracts scalar element |Index| from |Vector|. A <1 x T> vector shares its + // SPIR-V type with the scalar T, so a plain extractelement would be invalid; + // bridge it with spv_bitcast instead. + Value *extractScalarFromVector(IRBuilder<> &B, Value *Vector, + unsigned Index) { + auto *VecTy = cast(Vector->getType()); + Type *ElemTy = VecTy->getElementType(); + if (VecTy->getNumElements() == 1) { + Value *Scalar = + B.CreateIntrinsic(Intrinsic::spv_bitcast, {ElemTy, VecTy}, {Vector}); + buildAssignType(B, ElemTy, Scalar); + return Scalar; + } + return makeExtractElement(B, ElemTy, Vector, Index); + } + // Stores the given Src vector operand into the Dst vector, adjusting the size // if required. Value *storeVectorFromVector(IRBuilder<> &B, Value *Src, Value *Dst, diff --git a/llvm/test/CodeGen/SPIRV/passes/SPIRVLegalizePointerCast.ll b/llvm/test/CodeGen/SPIRV/passes/SPIRVLegalizePointerCast.ll index e4ad402c1ec95..a03dc58c5a5b6 100644 --- a/llvm/test/CodeGen/SPIRV/passes/SPIRVLegalizePointerCast.ll +++ b/llvm/test/CodeGen/SPIRV/passes/SPIRVLegalizePointerCast.ll @@ -59,4 +59,20 @@ entry: ret void } +@WIDEN = external addrspace(12) global <{ <1 x float>, target("spirv.Padding", 12), <1 x float> }>, align 4 + +define spir_func void @widen() #0 { +; CHECK-LABEL: define spir_func void @widen( +; CHECK-NOT: call {{.*}}@llvm.spv.ptrcast +; CHECK: call ptr addrspace(12) {{.*}}@llvm.spv.gep.p12.p12(i1 false, ptr addrspace(12) @WIDEN, i32 0, i32 0) +; CHECK: load <1 x float>, ptr addrspace(12) +; CHECK: call float @llvm.spv.bitcast.f32.v1f32(<1 x float> +; CHECK: call <4 x float> @llvm.spv.insertelt.v4f32.v4f32.f32.i32(<4 x float> poison, float +entry: + %v = load <4 x float>, ptr addrspace(12) @WIDEN, align 4 + %x = extractelement <4 x float> %v, i32 0 + store float %x, ptr addrspace(10) @OUT, align 4 + ret void +} + attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" } From e55d3bca36ff8a23ef598f6f452ec1e3f399dc31 Mon Sep 17 00:00:00 2001 From: Henry Jiang Date: Tue, 21 Jul 2026 14:27:22 -0700 Subject: [PATCH 35/50] [MC] Emit Mach-O pseudo-probe sections in the __LLVM segment (#206793) Let's back this out in favor of emitting pseudoprobe data in favor of emitting in the __LLVM segment, and let dsymutil collect the debug map objects and merge them into the final sections. This is to prevent older ld toolchains that don't ignore the new `__PSEUDO_PROBE` segments from accidentally leaking the probe metadata into the final image. --- llvm/lib/MC/MCObjectFileInfo.cpp | 12 +++++++----- .../SampleProfile/pseudo-probe-emit-macho.ll | 10 +++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp index e9a8612046ff5..fdf634526d469 100644 --- a/llvm/lib/MC/MCObjectFileInfo.cpp +++ b/llvm/lib/MC/MCObjectFileInfo.cpp @@ -322,12 +322,14 @@ void MCObjectFileInfo::initMachOMCObjectFileInfo(const Triple &T) { RemarksSection = Ctx->getMachOSection( "__LLVM", "__remarks", MachO::S_ATTR_DEBUG, SectionKind::getMetadata()); - PseudoProbeSection = - Ctx->getMachOSection("__PSEUDO_PROBE", "__probes", - MachO::S_ATTR_DEBUG | MachO::S_ATTR_NO_DEAD_STRIP, - SectionKind::getMetadata()); + // Emit the pseudoprobe sections in the __LLVM segment. Current ld ignores the + // __LLVM and __DWARF segment, so the probe metadata is dropped from the final + // linked image on every toolchain. + PseudoProbeSection = Ctx->getMachOSection( + "__LLVM", "__probes", MachO::S_ATTR_DEBUG | MachO::S_ATTR_NO_DEAD_STRIP, + SectionKind::getMetadata()); PseudoProbeDescSection = - Ctx->getMachOSection("__PSEUDO_PROBE", "__probe_descs", + Ctx->getMachOSection("__LLVM", "__probe_descs", MachO::S_ATTR_DEBUG | MachO::S_ATTR_NO_DEAD_STRIP, SectionKind::getMetadata()); diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-emit-macho.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit-macho.ll index 3a9bd93a98adc..12178c4597da2 100644 --- a/llvm/test/Transforms/SampleProfile/pseudo-probe-emit-macho.ll +++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit-macho.ll @@ -83,7 +83,7 @@ entry: ; CHECK-IL: ![[#SCOPE1]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 455082015) ; Check the generation of pseudo_probe_desc section for MachO -; CHECK-ASM-MACHO: .section __PSEUDO_PROBE,__probe_descs,regular,no_dead_strip+debug +; CHECK-ASM-MACHO: .section __LLVM,__probe_descs,regular,no_dead_strip+debug ; CHECK-ASM-MACHO-NEXT: .quad [[#GUID]] ; CHECK-ASM-MACHO-NEXT: .quad [[#HASH:]] ; CHECK-ASM-MACHO-NEXT: .byte 3 @@ -95,24 +95,24 @@ entry: ; CHECK-SEC-MACHO-LABEL: Sections [ ; CHECK-SEC-MACHO: Name: __probe_descs -; CHECK-SEC-MACHO-NEXT: Segment: __PSEUDO_PROBE +; CHECK-SEC-MACHO-NEXT: Segment: __LLVM ; CHECK-SEC-MACHO: Attributes [ (0x120000) ; CHECK-SEC-MACHO-NEXT: Debug (0x20000) ; CHECK-SEC-MACHO-NEXT: NoDeadStrip (0x100000) ; CHECK-SEC-MACHO: Name: __probes -; CHECK-SEC-MACHO-NEXT: Segment: __PSEUDO_PROBE +; CHECK-SEC-MACHO-NEXT: Segment: __LLVM ; CHECK-SEC-MACHO: Attributes [ (0x120000) ; CHECK-SEC-MACHO-NEXT: Debug (0x20000) ; CHECK-SEC-MACHO-NEXT: NoDeadStrip (0x100000) ; CHECK-SEC-MACHO-MC-LABEL: Sections [ ; CHECK-SEC-MACHO-MC: Name: __probe_descs -; CHECK-SEC-MACHO-MC-NEXT: Segment: __PSEUDO_PROBE +; CHECK-SEC-MACHO-MC-NEXT: Segment: __LLVM ; CHECK-SEC-MACHO-MC: Attributes [ (0x120000) ; CHECK-SEC-MACHO-MC-NEXT: Debug (0x20000) ; CHECK-SEC-MACHO-MC-NEXT: NoDeadStrip (0x100000) ; CHECK-SEC-MACHO-MC: Name: __probes -; CHECK-SEC-MACHO-MC-NEXT: Segment: __PSEUDO_PROBE +; CHECK-SEC-MACHO-MC-NEXT: Segment: __LLVM ; CHECK-SEC-MACHO-MC: Attributes [ (0x120000) ; CHECK-SEC-MACHO-MC-NEXT: Debug (0x20000) ; CHECK-SEC-MACHO-MC-NEXT: NoDeadStrip (0x100000) From c3373183d2fbb2184c249f93549c300104c994e0 Mon Sep 17 00:00:00 2001 From: Alex Strelnikov Date: Tue, 21 Jul 2026 17:35:24 -0400 Subject: [PATCH 36/50] [libc] Migrate printf_core from stdio/ to __support/. (#208303) The code will be updated and used for wide character printf functions implemented in wchar/, so __support/ is a more consistent shared location. Assisted-by: Automated tooling, human reviewed. --- libc/fuzzing/stdio/CMakeLists.txt | 2 +- libc/fuzzing/stdio/printf_parser_fuzz.cpp | 2 +- libc/src/__support/CMakeLists.txt | 2 + .../printf_core/CMakeLists.txt | 17 +- .../printf_core/char_converter.h | 12 +- .../printf_core/converter.h | 18 +- .../printf_core/converter_atlas.h | 26 +- .../printf_core/converter_utils.h | 8 +- .../printf_core/core_structs.h | 8 +- .../printf_core/error_mapper.h | 6 +- .../printf_core/fixed_converter.h | 12 +- .../printf_core/float_dec_converter.h | 14 +- .../printf_core/float_dec_converter_limited.h | 12 +- .../printf_core/float_hex_converter.h | 14 +- .../printf_core/float_impl.cpp | 2 +- .../printf_core/float_inf_nan_converter.h | 12 +- .../printf_core/generic}/CMakeLists.txt | 2 +- .../printf_core/generic/error_mapper.h | 10 +- .../printf_core/int_converter.h | 12 +- .../printf_core/linux}/CMakeLists.txt | 2 +- .../printf_core/linux/error_mapper.h | 10 +- .../{stdio => __support}/printf_core/parser.h | 10 +- .../printf_core/printf_config.h | 6 +- .../printf_core/printf_main.h | 14 +- .../printf_core/ptr_converter.h | 14 +- .../printf_core/strerror_converter.h | 14 +- .../printf_core/string_converter.h | 12 +- .../printf_core/vasprintf_internal.h | 11 +- .../printf_core/vfprintf_internal.h | 12 +- .../printf_core/write_int_converter.h | 10 +- .../printf_core/write_modes.def | 0 .../{stdio => __support}/printf_core/writer.h | 10 +- libc/src/err/CMakeLists.txt | 4 +- libc/src/err/report.cpp | 4 +- libc/src/stdio/CMakeLists.txt | 45 +- libc/src/stdio/asprintf.cpp | 6 +- libc/src/stdio/asprintf_modular.cpp | 6 +- libc/src/stdio/baremetal/CMakeLists.txt | 8 +- libc/src/stdio/baremetal/vfprintf_internal.h | 8 +- libc/src/stdio/generic/CMakeLists.txt | 6 +- libc/src/stdio/generic/fprintf.cpp | 6 +- libc/src/stdio/generic/printf.cpp | 6 +- libc/src/stdio/generic/vfprintf.cpp | 6 +- libc/src/stdio/generic/vprintf.cpp | 6 +- libc/src/stdio/snprintf.cpp | 8 +- libc/src/stdio/snprintf_modular.cpp | 8 +- libc/src/stdio/sprintf.cpp | 8 +- libc/src/stdio/sprintf_modular.cpp | 8 +- libc/src/stdio/vasprintf.cpp | 6 +- libc/src/stdio/vasprintf_modular.cpp | 6 +- libc/src/stdio/vsnprintf.cpp | 8 +- libc/src/stdio/vsnprintf_modular.cpp | 8 +- libc/src/stdio/vsprintf.cpp | 8 +- libc/src/stdio/vsprintf_modular.cpp | 8 +- libc/src/stdlib/CMakeLists.txt | 12 +- libc/src/stdlib/str_from_util.h | 6 +- libc/src/stdlib/strfromd.cpp | 4 +- libc/src/stdlib/strfromf.cpp | 4 +- libc/src/stdlib/strfroml.cpp | 4 +- libc/src/time/CMakeLists.txt | 4 +- libc/src/time/strftime.cpp | 2 +- libc/src/time/strftime_core/CMakeLists.txt | 4 +- .../time/strftime_core/composite_converter.h | 2 +- libc/src/time/strftime_core/converter.h | 2 +- libc/src/time/strftime_core/num_converter.h | 2 +- libc/src/time/strftime_core/str_converter.h | 2 +- libc/src/time/strftime_core/strftime_main.h | 2 +- libc/src/time/strftime_l.cpp | 2 +- libc/test/UnitTest/CMakeLists.txt | 4 +- libc/test/UnitTest/PrintfMatcher.cpp | 4 +- libc/test/UnitTest/PrintfMatcher.h | 2 +- libc/test/shared/CMakeLists.txt | 6 +- libc/test/src/__support/CMakeLists.txt | 1 + .../printf_core/CMakeLists.txt | 18 +- .../printf_core/converter_test.cpp | 6 +- .../printf_core/parser_test.cpp | 2 +- .../printf_core/writer_test.cpp | 2 +- libc/test/src/stdio/CMakeLists.txt | 3 +- libc/test/src/stdio/sprintf_test.cpp | 2 +- .../llvm-project-overlay/libc/BUILD.bazel | 436 +++++++++--------- .../libc/test/UnitTest/BUILD.bazel | 4 +- .../libc/test/src/__support/BUILD.bazel | 37 ++ .../libc/test/src/stdio/BUILD.bazel | 39 +- 83 files changed, 566 insertions(+), 555 deletions(-) rename libc/src/{stdio => __support}/printf_core/CMakeLists.txt (91%) rename libc/src/{stdio => __support}/printf_core/char_converter.h (87%) rename libc/src/{stdio => __support}/printf_core/converter.h (89%) rename libc/src/{stdio => __support}/printf_core/converter_atlas.h (62%) rename libc/src/{stdio => __support}/printf_core/converter_utils.h (92%) rename libc/src/{stdio => __support}/printf_core/core_structs.h (95%) rename libc/src/{stdio => __support}/printf_core/error_mapper.h (76%) rename libc/src/{stdio => __support}/printf_core/fixed_converter.h (97%) rename libc/src/{stdio => __support}/printf_core/float_dec_converter.h (99%) rename libc/src/{stdio => __support}/printf_core/float_dec_converter_limited.h (98%) rename libc/src/{stdio => __support}/printf_core/float_hex_converter.h (96%) rename libc/src/{stdio => __support}/printf_core/float_impl.cpp (94%) rename libc/src/{stdio => __support}/printf_core/float_inf_nan_converter.h (90%) rename libc/src/{stdio/printf_core/linux => __support/printf_core/generic}/CMakeLists.txt (67%) rename libc/src/{stdio => __support}/printf_core/generic/error_mapper.h (79%) rename libc/src/{stdio => __support}/printf_core/int_converter.h (95%) rename libc/src/{stdio/printf_core/generic => __support/printf_core/linux}/CMakeLists.txt (67%) rename libc/src/{stdio => __support}/printf_core/linux/error_mapper.h (82%) rename libc/src/{stdio => __support}/printf_core/parser.h (99%) rename libc/src/{stdio => __support}/printf_core/printf_config.h (94%) rename libc/src/{stdio => __support}/printf_core/printf_main.h (81%) rename libc/src/{stdio => __support}/printf_core/ptr_converter.h (76%) rename libc/src/{stdio => __support}/printf_core/strerror_converter.h (87%) rename libc/src/{stdio => __support}/printf_core/string_converter.h (92%) rename libc/src/{stdio => __support}/printf_core/vasprintf_internal.h (87%) rename libc/src/{stdio => __support}/printf_core/vfprintf_internal.h (91%) rename libc/src/{stdio => __support}/printf_core/write_int_converter.h (88%) rename libc/src/{stdio => __support}/printf_core/write_modes.def (100%) rename libc/src/{stdio => __support}/printf_core/writer.h (97%) rename libc/test/src/{stdio => __support}/printf_core/CMakeLists.txt (56%) rename libc/test/src/{stdio => __support}/printf_core/converter_test.cpp (98%) rename libc/test/src/{stdio => __support}/printf_core/parser_test.cpp (99%) rename libc/test/src/{stdio => __support}/printf_core/writer_test.cpp (99%) diff --git a/libc/fuzzing/stdio/CMakeLists.txt b/libc/fuzzing/stdio/CMakeLists.txt index 401785a30469c..86f095e84737e 100644 --- a/libc/fuzzing/stdio/CMakeLists.txt +++ b/libc/fuzzing/stdio/CMakeLists.txt @@ -3,7 +3,7 @@ add_libc_fuzzer( SRCS printf_parser_fuzz.cpp DEPENDS - libc.src.stdio.printf_core.parser + libc.src.__support.printf_core.parser libc.src.errno.errno # needed for the strerror conversion ) diff --git a/libc/fuzzing/stdio/printf_parser_fuzz.cpp b/libc/fuzzing/stdio/printf_parser_fuzz.cpp index e745850bd6c3a..71da37d470fa8 100644 --- a/libc/fuzzing/stdio/printf_parser_fuzz.cpp +++ b/libc/fuzzing/stdio/printf_parser_fuzz.cpp @@ -11,7 +11,7 @@ //===----------------------------------------------------------------------===// #include "src/__support/arg_list.h" -#include "src/stdio/printf_core/parser.h" +#include "src/__support/printf_core/parser.h" #include #include diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt index f4cb283976bc3..12761e6a524b9 100644 --- a/libc/src/__support/CMakeLists.txt +++ b/libc/src/__support/CMakeLists.txt @@ -462,6 +462,8 @@ add_subdirectory(threads) add_subdirectory(File) +add_subdirectory(printf_core) + add_subdirectory(HashTable) add_subdirectory(fixed_point) diff --git a/libc/src/stdio/printf_core/CMakeLists.txt b/libc/src/__support/printf_core/CMakeLists.txt similarity index 91% rename from libc/src/stdio/printf_core/CMakeLists.txt rename to libc/src/__support/printf_core/CMakeLists.txt index 4d6f25a2ddd00..a2e47e778a8c7 100644 --- a/libc/src/stdio/printf_core/CMakeLists.txt +++ b/libc/src/__support/printf_core/CMakeLists.txt @@ -1,3 +1,8 @@ +if(NOT LLVM_LIBC_FULL_BUILD) + set(use_system_file "COMPILE_OPTIONS") + libc_add_definition(use_system_file "LIBC_COPT_STDIO_USE_SYSTEM_FILE") +endif() + if(LIBC_CONF_PRINTF_DISABLE_FLOAT) libc_add_definition(printf_config_copts "LIBC_COPT_PRINTF_DISABLE_FLOAT") endif() @@ -41,9 +46,9 @@ else() add_subdirectory(generic) endif() -set(target_error_mapper libc.src.stdio.printf_core.${LIBC_TARGET_OS}.error_mapper) +set(target_error_mapper libc.src.__support.printf_core.${LIBC_TARGET_OS}.error_mapper) if(NOT TARGET ${target_error_mapper}) - set(target_error_mapper libc.src.stdio.printf_core.generic.error_mapper) + set(target_error_mapper libc.src.__support.printf_core.generic.error_mapper) endif() if(LIBC_CONF_PRINTF_DISABLE_WIDE) @@ -178,8 +183,8 @@ add_header_library( libc.hdr.func.realloc libc.src.__support.arg_list libc.src.__support.error_or - libc.src.stdio.printf_core.printf_main - libc.src.stdio.printf_core.writer + libc.src.__support.printf_core.printf_main + libc.src.__support.printf_core.writer ) add_header_library( @@ -205,8 +210,8 @@ add_header_library( libc.src.__support.File.file libc.src.__support.error_or libc.src.__support.arg_list - libc.src.stdio.printf_core.printf_main - libc.src.stdio.printf_core.writer + libc.src.__support.printf_core.printf_main + libc.src.__support.printf_core.writer ${use_system_file} ) diff --git a/libc/src/stdio/printf_core/char_converter.h b/libc/src/__support/printf_core/char_converter.h similarity index 87% rename from libc/src/stdio/printf_core/char_converter.h rename to libc/src/__support/printf_core/char_converter.h index e4792c3b23b9f..59f3e22f7ce50 100644 --- a/libc/src/stdio/printf_core/char_converter.h +++ b/libc/src/__support/printf_core/char_converter.h @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_CHAR_CONVERTER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_CHAR_CONVERTER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_CHAR_CONVERTER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_CHAR_CONVERTER_H #ifndef LIBC_COPT_PRINTF_DISABLE_WIDE #include "hdr/types/wchar_t.h" @@ -19,9 +19,9 @@ #include "hdr/limits_macros.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/converter_utils.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/converter_utils.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/writer.h" namespace LIBC_NAMESPACE_DECL { namespace printf_core { @@ -85,4 +85,4 @@ LIBC_INLINE int convert_char(Writer *writer, } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_CHAR_CONVERTER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_CHAR_CONVERTER_H diff --git a/libc/src/stdio/printf_core/converter.h b/libc/src/__support/printf_core/converter.h similarity index 89% rename from libc/src/stdio/printf_core/converter.h rename to libc/src/__support/printf_core/converter.h index e931d6c34db68..67d0ee3d42b61 100644 --- a/libc/src/stdio/printf_core/converter.h +++ b/libc/src/__support/printf_core/converter.h @@ -6,19 +6,19 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_CONVERTER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_CONVERTER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_CONVERTER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_CONVERTER_H #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/printf_config.h" -#include "src/stdio/printf_core/strerror_converter.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/printf_config.h" +#include "src/__support/printf_core/strerror_converter.h" +#include "src/__support/printf_core/writer.h" // This option allows for replacing all of the conversion functions with custom // replacements. This allows conversions to be replaced at compile time. #ifndef LIBC_COPT_PRINTF_CONV_ATLAS -#include "src/stdio/printf_core/converter_atlas.h" +#include "src/__support/printf_core/converter_atlas.h" #else #include LIBC_COPT_PRINTF_CONV_ATLAS #endif @@ -55,7 +55,7 @@ LIBC_PRINTF_MODULE((template #define HANDLE_WRITE_MODE(MODE) \ template int convert_float( \ Writer * writer, const FormatSection &to_conv); -#include "src/stdio/printf_core/write_modes.def" +#include "src/__support/printf_core/write_modes.def" #undef HANDLE_WRITE_MODE #endif // LIBC_PRINTF_DEFINE_MODULES @@ -137,4 +137,4 @@ int convert(Writer *writer, const FormatSection &to_conv) { } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_CONVERTER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_CONVERTER_H diff --git a/libc/src/stdio/printf_core/converter_atlas.h b/libc/src/__support/printf_core/converter_atlas.h similarity index 62% rename from libc/src/stdio/printf_core/converter_atlas.h rename to libc/src/__support/printf_core/converter_atlas.h index dfb91b30e80f8..e60df06cbac05 100644 --- a/libc/src/stdio/printf_core/converter_atlas.h +++ b/libc/src/__support/printf_core/converter_atlas.h @@ -10,46 +10,46 @@ // just replace the #include, additionally it keeps the ifdefs out of the // converter header. -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_CONVERTER_ATLAS_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_CONVERTER_ATLAS_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_CONVERTER_ATLAS_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_CONVERTER_ATLAS_H // defines convert_string -#include "src/stdio/printf_core/string_converter.h" +#include "src/__support/printf_core/string_converter.h" // defines convert_char -#include "src/stdio/printf_core/char_converter.h" +#include "src/__support/printf_core/char_converter.h" // defines convert_int -#include "src/stdio/printf_core/int_converter.h" +#include "src/__support/printf_core/int_converter.h" #ifndef LIBC_COPT_PRINTF_DISABLE_FLOAT // defines convert_float_decimal // defines convert_float_dec_exp // defines convert_float_dec_auto #ifdef LIBC_COPT_FLOAT_TO_STR_USE_FLOAT320 -#include "src/stdio/printf_core/float_dec_converter_limited.h" +#include "src/__support/printf_core/float_dec_converter_limited.h" #else -#include "src/stdio/printf_core/float_dec_converter.h" +#include "src/__support/printf_core/float_dec_converter.h" #endif // defines convert_float_hex_exp -#include "src/stdio/printf_core/float_hex_converter.h" +#include "src/__support/printf_core/float_hex_converter.h" #endif // LIBC_COPT_PRINTF_DISABLE_FLOAT #ifdef LIBC_INTERNAL_PRINTF_HAS_FIXED_POINT // defines convert_fixed -#include "src/stdio/printf_core/fixed_converter.h" +#include "src/__support/printf_core/fixed_converter.h" #endif // LIBC_INTERNAL_PRINTF_HAS_FIXED_POINT #ifndef LIBC_COPT_PRINTF_DISABLE_WRITE_INT -#include "src/stdio/printf_core/write_int_converter.h" +#include "src/__support/printf_core/write_int_converter.h" #endif // LIBC_COPT_PRINTF_DISABLE_WRITE_INT // defines convert_pointer -#include "src/stdio/printf_core/ptr_converter.h" +#include "src/__support/printf_core/ptr_converter.h" #ifndef LIBC_COPT_PRINTF_DISABLE_STRERROR // defines convert_strerror -#include "src/stdio/printf_core/strerror_converter.h" +#include "src/__support/printf_core/strerror_converter.h" #endif // LIBC_COPT_PRINTF_DISABLE_STRERROR -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_CONVERTER_ATLAS_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_CONVERTER_ATLAS_H diff --git a/libc/src/stdio/printf_core/converter_utils.h b/libc/src/__support/printf_core/converter_utils.h similarity index 92% rename from libc/src/stdio/printf_core/converter_utils.h rename to libc/src/__support/printf_core/converter_utils.h index b27defddc354f..a21fb70f2f62a 100644 --- a/libc/src/stdio/printf_core/converter_utils.h +++ b/libc/src/__support/printf_core/converter_utils.h @@ -6,12 +6,12 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_CONVERTER_UTILS_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_CONVERTER_UTILS_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_CONVERTER_UTILS_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_CONVERTER_UTILS_H #include "src/__support/CPP/limits.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" +#include "src/__support/printf_core/core_structs.h" #include #include @@ -78,4 +78,4 @@ enum class RoundDirection { Up, Down, Even }; } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_CONVERTER_UTILS_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_CONVERTER_UTILS_H diff --git a/libc/src/stdio/printf_core/core_structs.h b/libc/src/__support/printf_core/core_structs.h similarity index 95% rename from libc/src/stdio/printf_core/core_structs.h rename to libc/src/__support/printf_core/core_structs.h index 9b74a0a391f95..3107b9ce46f00 100644 --- a/libc/src/stdio/printf_core/core_structs.h +++ b/libc/src/__support/printf_core/core_structs.h @@ -6,15 +6,15 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_CORE_STRUCTS_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_CORE_STRUCTS_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_CORE_STRUCTS_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_CORE_STRUCTS_H #include "src/__support/macros/config.h" #include "src/__support/CPP/string_view.h" #include "src/__support/CPP/type_traits.h" #include "src/__support/FPUtil/FPBits.h" -#include "src/stdio/printf_core/printf_config.h" +#include "src/__support/printf_core/printf_config.h" #include #include @@ -165,4 +165,4 @@ constexpr int MB_CONVERSION_ERROR = -1009; } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_CORE_STRUCTS_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_CORE_STRUCTS_H diff --git a/libc/src/stdio/printf_core/error_mapper.h b/libc/src/__support/printf_core/error_mapper.h similarity index 76% rename from libc/src/stdio/printf_core/error_mapper.h rename to libc/src/__support/printf_core/error_mapper.h index 23030930133a1..143ea94ce8963 100644 --- a/libc/src/stdio/printf_core/error_mapper.h +++ b/libc/src/__support/printf_core/error_mapper.h @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_ERROR_MAPPER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_ERROR_MAPPER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_ERROR_MAPPER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_ERROR_MAPPER_H #include "src/__support/macros/properties/architectures.h" @@ -18,4 +18,4 @@ #include "generic/error_mapper.h" #endif -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_ERROR_MAPPER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_ERROR_MAPPER_H diff --git a/libc/src/stdio/printf_core/fixed_converter.h b/libc/src/__support/printf_core/fixed_converter.h similarity index 97% rename from libc/src/stdio/printf_core/fixed_converter.h rename to libc/src/__support/printf_core/fixed_converter.h index e8a3314967562..76b2d0d63644f 100644 --- a/libc/src/stdio/printf_core/fixed_converter.h +++ b/libc/src/__support/printf_core/fixed_converter.h @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FIXED_CONVERTER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FIXED_CONVERTER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_FIXED_CONVERTER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_FIXED_CONVERTER_H #include "include/llvm-libc-macros/stdfix-macros.h" #include "src/__support/CPP/string_view.h" @@ -17,9 +17,9 @@ #include "src/__support/integer_to_string.h" #include "src/__support/libc_assert.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/converter_utils.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/converter_utils.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/writer.h" #include #include @@ -307,4 +307,4 @@ LIBC_INLINE int convert_fixed(Writer *writer, } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FIXED_CONVERTER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_FIXED_CONVERTER_H diff --git a/libc/src/stdio/printf_core/float_dec_converter.h b/libc/src/__support/printf_core/float_dec_converter.h similarity index 99% rename from libc/src/stdio/printf_core/float_dec_converter.h rename to libc/src/__support/printf_core/float_dec_converter.h index 7d86de3692115..b678cd29cdb81 100644 --- a/libc/src/stdio/printf_core/float_dec_converter.h +++ b/libc/src/__support/printf_core/float_dec_converter.h @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_DEC_CONVERTER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_DEC_CONVERTER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_FLOAT_DEC_CONVERTER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_FLOAT_DEC_CONVERTER_H #include "src/__support/CPP/string_view.h" #include "src/__support/FPUtil/FPBits.h" @@ -18,10 +18,10 @@ #include "src/__support/integer_to_string.h" #include "src/__support/libc_assert.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/converter_utils.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/float_inf_nan_converter.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/converter_utils.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/float_inf_nan_converter.h" +#include "src/__support/printf_core/writer.h" #include #include @@ -1245,4 +1245,4 @@ LIBC_INLINE int convert_float_dec_auto(Writer *writer, } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_DEC_CONVERTER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_FLOAT_DEC_CONVERTER_H diff --git a/libc/src/stdio/printf_core/float_dec_converter_limited.h b/libc/src/__support/printf_core/float_dec_converter_limited.h similarity index 98% rename from libc/src/stdio/printf_core/float_dec_converter_limited.h rename to libc/src/__support/printf_core/float_dec_converter_limited.h index 2693ebd531f96..2e34004ce92ca 100644 --- a/libc/src/stdio/printf_core/float_dec_converter_limited.h +++ b/libc/src/__support/printf_core/float_dec_converter_limited.h @@ -38,8 +38,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_DEC_CONVERTER_LIMITED_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_DEC_CONVERTER_LIMITED_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_FLOAT_DEC_CONVERTER_LIMITED_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_FLOAT_DEC_CONVERTER_LIMITED_H #include "src/__support/CPP/algorithm.h" #include "src/__support/CPP/string.h" @@ -50,9 +50,9 @@ #include "src/__support/integer_to_string.h" #include "src/__support/libc_assert.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/float_inf_nan_converter.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/float_inf_nan_converter.h" +#include "src/__support/printf_core/writer.h" #include "src/string/memory_utils/inline_memcpy.h" namespace LIBC_NAMESPACE_DECL { @@ -711,4 +711,4 @@ LIBC_INLINE int convert_float_dec_auto(Writer *writer, } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_DEC_CONVERTER_LIMITED_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_FLOAT_DEC_CONVERTER_LIMITED_H diff --git a/libc/src/stdio/printf_core/float_hex_converter.h b/libc/src/__support/printf_core/float_hex_converter.h similarity index 96% rename from libc/src/stdio/printf_core/float_hex_converter.h rename to libc/src/__support/printf_core/float_hex_converter.h index 6ac3bfac8a76e..640db7be2c0f4 100644 --- a/libc/src/stdio/printf_core/float_hex_converter.h +++ b/libc/src/__support/printf_core/float_hex_converter.h @@ -6,18 +6,18 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_HEX_CONVERTER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_HEX_CONVERTER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_FLOAT_HEX_CONVERTER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_FLOAT_HEX_CONVERTER_H #include "src/__support/CPP/string_view.h" #include "src/__support/FPUtil/FPBits.h" #include "src/__support/FPUtil/rounding_mode.h" #include "src/__support/ctype_utils.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/converter_utils.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/float_inf_nan_converter.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/converter_utils.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/float_inf_nan_converter.h" +#include "src/__support/printf_core/writer.h" #include #include @@ -298,4 +298,4 @@ LIBC_INLINE int convert_float_hex_exp(Writer *writer, } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_HEX_CONVERTER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_FLOAT_HEX_CONVERTER_H diff --git a/libc/src/stdio/printf_core/float_impl.cpp b/libc/src/__support/printf_core/float_impl.cpp similarity index 94% rename from libc/src/stdio/printf_core/float_impl.cpp rename to libc/src/__support/printf_core/float_impl.cpp index 92a616b955def..7e8fc6d983325 100644 --- a/libc/src/stdio/printf_core/float_impl.cpp +++ b/libc/src/__support/printf_core/float_impl.cpp @@ -16,7 +16,7 @@ #ifdef LIBC_COPT_PRINTF_MODULAR #define LIBC_PRINTF_DEFINE_MODULES -#include "src/stdio/printf_core/converter.h" +#include "src/__support/printf_core/converter.h" // Bring this file into the link if __printf_float is referenced. extern "C" void __printf_float() {} diff --git a/libc/src/stdio/printf_core/float_inf_nan_converter.h b/libc/src/__support/printf_core/float_inf_nan_converter.h similarity index 90% rename from libc/src/stdio/printf_core/float_inf_nan_converter.h rename to libc/src/__support/printf_core/float_inf_nan_converter.h index 2af2c7feee336..81769ae4b4913 100644 --- a/libc/src/stdio/printf_core/float_inf_nan_converter.h +++ b/libc/src/__support/printf_core/float_inf_nan_converter.h @@ -6,15 +6,15 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_INF_NAN_CONVERTER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_INF_NAN_CONVERTER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_FLOAT_INF_NAN_CONVERTER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_FLOAT_INF_NAN_CONVERTER_H #include "src/__support/FPUtil/FPBits.h" #include "src/__support/ctype_utils.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/converter_utils.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/converter_utils.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/writer.h" #include #include @@ -99,4 +99,4 @@ LIBC_INLINE int convert_inf_nan(Writer *writer, } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_INF_NAN_CONVERTER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_FLOAT_INF_NAN_CONVERTER_H diff --git a/libc/src/stdio/printf_core/linux/CMakeLists.txt b/libc/src/__support/printf_core/generic/CMakeLists.txt similarity index 67% rename from libc/src/stdio/printf_core/linux/CMakeLists.txt rename to libc/src/__support/printf_core/generic/CMakeLists.txt index 2f0143d992e31..a4533c816a684 100644 --- a/libc/src/stdio/printf_core/linux/CMakeLists.txt +++ b/libc/src/__support/printf_core/generic/CMakeLists.txt @@ -3,6 +3,6 @@ add_header_library( HDRS error_mapper.h DEPENDS - libc.src.stdio.printf_core.core_structs + libc.src.__support.printf_core.core_structs libc.hdr.errno_macros ) diff --git a/libc/src/stdio/printf_core/generic/error_mapper.h b/libc/src/__support/printf_core/generic/error_mapper.h similarity index 79% rename from libc/src/stdio/printf_core/generic/error_mapper.h rename to libc/src/__support/printf_core/generic/error_mapper.h index d8cdd2cc2dbaa..a3f355b531b33 100644 --- a/libc/src/stdio/printf_core/generic/error_mapper.h +++ b/libc/src/__support/printf_core/generic/error_mapper.h @@ -6,12 +6,12 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_GENERIC_ERROR_MAPPER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_GENERIC_ERROR_MAPPER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_GENERIC_ERROR_MAPPER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_GENERIC_ERROR_MAPPER_H #include "hdr/errno_macros.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" namespace LIBC_NAMESPACE_DECL { namespace printf_core { @@ -46,4 +46,4 @@ LIBC_INLINE static int internal_error_to_errno(int internal_error) { } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_GENERIC_ERROR_MAPPER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_GENERIC_ERROR_MAPPER_H diff --git a/libc/src/stdio/printf_core/int_converter.h b/libc/src/__support/printf_core/int_converter.h similarity index 95% rename from libc/src/stdio/printf_core/int_converter.h rename to libc/src/__support/printf_core/int_converter.h index 11234c32ce997..c40c742e41c35 100644 --- a/libc/src/stdio/printf_core/int_converter.h +++ b/libc/src/__support/printf_core/int_converter.h @@ -6,17 +6,17 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_INT_CONVERTER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_INT_CONVERTER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_INT_CONVERTER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_INT_CONVERTER_H #include "src/__support/CPP/span.h" #include "src/__support/CPP/string_view.h" #include "src/__support/ctype_utils.h" #include "src/__support/integer_to_string.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/converter_utils.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/converter_utils.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/writer.h" #include #include @@ -209,4 +209,4 @@ LIBC_INLINE int convert_int(Writer *writer, } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_INT_CONVERTER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_INT_CONVERTER_H diff --git a/libc/src/stdio/printf_core/generic/CMakeLists.txt b/libc/src/__support/printf_core/linux/CMakeLists.txt similarity index 67% rename from libc/src/stdio/printf_core/generic/CMakeLists.txt rename to libc/src/__support/printf_core/linux/CMakeLists.txt index 2f0143d992e31..a4533c816a684 100644 --- a/libc/src/stdio/printf_core/generic/CMakeLists.txt +++ b/libc/src/__support/printf_core/linux/CMakeLists.txt @@ -3,6 +3,6 @@ add_header_library( HDRS error_mapper.h DEPENDS - libc.src.stdio.printf_core.core_structs + libc.src.__support.printf_core.core_structs libc.hdr.errno_macros ) diff --git a/libc/src/stdio/printf_core/linux/error_mapper.h b/libc/src/__support/printf_core/linux/error_mapper.h similarity index 82% rename from libc/src/stdio/printf_core/linux/error_mapper.h rename to libc/src/__support/printf_core/linux/error_mapper.h index 3449f12593469..db8cddcd11f66 100644 --- a/libc/src/stdio/printf_core/linux/error_mapper.h +++ b/libc/src/__support/printf_core/linux/error_mapper.h @@ -6,12 +6,12 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_LINUX_ERROR_MAPPER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_LINUX_ERROR_MAPPER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_LINUX_ERROR_MAPPER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_LINUX_ERROR_MAPPER_H #include "hdr/errno_macros.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" namespace LIBC_NAMESPACE_DECL { namespace printf_core { @@ -54,4 +54,4 @@ LIBC_INLINE static int internal_error_to_errno(int internal_error) { } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_LINUX_ERROR_MAPPER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_LINUX_ERROR_MAPPER_H diff --git a/libc/src/stdio/printf_core/parser.h b/libc/src/__support/printf_core/parser.h similarity index 99% rename from libc/src/stdio/printf_core/parser.h rename to libc/src/__support/printf_core/parser.h index 369358882b621..bd7b78ebfabf3 100644 --- a/libc/src/stdio/printf_core/parser.h +++ b/libc/src/__support/printf_core/parser.h @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_PARSER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_PARSER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_PARSER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_PARSER_H #include "include/llvm-libc-macros/stdfix-macros.h" #include "src/__support/CPP/algorithm.h" // max @@ -16,9 +16,9 @@ #include "src/__support/CPP/type_traits.h" #include "src/__support/macros/config.h" #include "src/__support/macros/properties/types.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/printf_config.h" #include "src/__support/str_to_integer.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/printf_config.h" #include @@ -758,4 +758,4 @@ template class Parser { } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_PARSER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_PARSER_H diff --git a/libc/src/stdio/printf_core/printf_config.h b/libc/src/__support/printf_core/printf_config.h similarity index 94% rename from libc/src/stdio/printf_core/printf_config.h rename to libc/src/__support/printf_core/printf_config.h index 929b1c5b3931f..0cf983f10d096 100644 --- a/libc/src/stdio/printf_core/printf_config.h +++ b/libc/src/__support/printf_core/printf_config.h @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_PRINTF_CONFIG_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_PRINTF_CONFIG_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_PRINTF_CONFIG_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_PRINTF_CONFIG_H // The index array buffer is always initialized when printf is called. In cases // where index mode is necessary but memory is limited, or when index mode @@ -73,4 +73,4 @@ LIBC_PRINTF_MODULE_UNWRAP SIG LIBC_PRINTF_MODULE_DECL; #endif -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_PRINTF_CONFIG_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_PRINTF_CONFIG_H diff --git a/libc/src/stdio/printf_core/printf_main.h b/libc/src/__support/printf_core/printf_main.h similarity index 81% rename from libc/src/stdio/printf_core/printf_main.h rename to libc/src/__support/printf_core/printf_main.h index 874c24464ed52..b08e05be25303 100644 --- a/libc/src/stdio/printf_core/printf_main.h +++ b/libc/src/__support/printf_core/printf_main.h @@ -6,16 +6,16 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_PRINTF_MAIN_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_PRINTF_MAIN_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_PRINTF_MAIN_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_PRINTF_MAIN_H #include "src/__support/arg_list.h" #include "src/__support/error_or.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/converter.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/parser.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/converter.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/parser.h" +#include "src/__support/printf_core/writer.h" #include @@ -55,4 +55,4 @@ ErrorOr printf_main(Writer *writer, } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_PRINTF_MAIN_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_PRINTF_MAIN_H diff --git a/libc/src/stdio/printf_core/ptr_converter.h b/libc/src/__support/printf_core/ptr_converter.h similarity index 76% rename from libc/src/stdio/printf_core/ptr_converter.h rename to libc/src/__support/printf_core/ptr_converter.h index c2a74e3043e6f..692933a63f38b 100644 --- a/libc/src/stdio/printf_core/ptr_converter.h +++ b/libc/src/__support/printf_core/ptr_converter.h @@ -6,14 +6,14 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_PTR_CONVERTER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_PTR_CONVERTER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_PTR_CONVERTER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_PTR_CONVERTER_H #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/int_converter.h" -#include "src/stdio/printf_core/string_converter.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/int_converter.h" +#include "src/__support/printf_core/string_converter.h" +#include "src/__support/printf_core/writer.h" namespace LIBC_NAMESPACE_DECL { namespace printf_core { @@ -40,4 +40,4 @@ LIBC_INLINE int convert_pointer(Writer *writer, } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_PTR_CONVERTER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_PTR_CONVERTER_H diff --git a/libc/src/stdio/printf_core/strerror_converter.h b/libc/src/__support/printf_core/strerror_converter.h similarity index 87% rename from libc/src/stdio/printf_core/strerror_converter.h rename to libc/src/__support/printf_core/strerror_converter.h index 2cd6df0c01d45..ec02a9a52f99a 100644 --- a/libc/src/stdio/printf_core/strerror_converter.h +++ b/libc/src/__support/printf_core/strerror_converter.h @@ -6,15 +6,15 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_STRERROR_CONVERTER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_STRERROR_CONVERTER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_STRERROR_CONVERTER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_STRERROR_CONVERTER_H #include "src/__support/StringUtil/error_to_string.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/int_converter.h" -#include "src/stdio/printf_core/string_converter.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/int_converter.h" +#include "src/__support/printf_core/string_converter.h" +#include "src/__support/printf_core/writer.h" namespace LIBC_NAMESPACE_DECL { namespace printf_core { @@ -73,4 +73,4 @@ LIBC_INLINE int convert_strerror(Writer *writer, } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_STRERROR_CONVERTER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_STRERROR_CONVERTER_H diff --git a/libc/src/stdio/printf_core/string_converter.h b/libc/src/__support/printf_core/string_converter.h similarity index 92% rename from libc/src/stdio/printf_core/string_converter.h rename to libc/src/__support/printf_core/string_converter.h index 2f521544785c0..4cd1d52a2bab5 100644 --- a/libc/src/stdio/printf_core/string_converter.h +++ b/libc/src/__support/printf_core/string_converter.h @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_STRING_CONVERTER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_STRING_CONVERTER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_STRING_CONVERTER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_STRING_CONVERTER_H #ifndef LIBC_COPT_PRINTF_DISABLE_WIDE #include "hdr/types/char32_t.h" @@ -17,9 +17,9 @@ #endif // LIBC_COPT_PRINTF_DISABLE_WIDE #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/converter_utils.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/converter_utils.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/writer.h" #include "src/string/string_utils.h" // string_length #include @@ -142,4 +142,4 @@ LIBC_INLINE int convert_string(Writer *writer, } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_STRING_CONVERTER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_STRING_CONVERTER_H diff --git a/libc/src/stdio/printf_core/vasprintf_internal.h b/libc/src/__support/printf_core/vasprintf_internal.h similarity index 87% rename from libc/src/stdio/printf_core/vasprintf_internal.h rename to libc/src/__support/printf_core/vasprintf_internal.h index fe5428e6118dc..969da6ebc46bb 100644 --- a/libc/src/stdio/printf_core/vasprintf_internal.h +++ b/libc/src/__support/printf_core/vasprintf_internal.h @@ -6,14 +6,17 @@ // //===----------------------------------------------------------------------===// +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_VASPRINTF_INTERNAL_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_VASPRINTF_INTERNAL_H + #include "hdr/func/free.h" #include "hdr/func/malloc.h" #include "hdr/func/realloc.h" #include "src/__support/arg_list.h" #include "src/__support/error_or.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/printf_main.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/printf_main.h" +#include "src/__support/printf_core/writer.h" namespace LIBC_NAMESPACE_DECL { namespace printf_core { @@ -73,3 +76,5 @@ LIBC_INLINE ErrorOr vasprintf_internal(char **ret, } } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_VASPRINTF_INTERNAL_H diff --git a/libc/src/stdio/printf_core/vfprintf_internal.h b/libc/src/__support/printf_core/vfprintf_internal.h similarity index 91% rename from libc/src/stdio/printf_core/vfprintf_internal.h rename to libc/src/__support/printf_core/vfprintf_internal.h index 321b0693ad339..6a62e966f7ff6 100644 --- a/libc/src/stdio/printf_core/vfprintf_internal.h +++ b/libc/src/__support/printf_core/vfprintf_internal.h @@ -6,17 +6,17 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_VFPRINTF_INTERNAL_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_VFPRINTF_INTERNAL_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_VFPRINTF_INTERNAL_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_VFPRINTF_INTERNAL_H #include "src/__support/File/file.h" #include "src/__support/arg_list.h" #include "src/__support/error_or.h" #include "src/__support/macros/attributes.h" // For LIBC_INLINE #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/printf_main.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/printf_main.h" +#include "src/__support/printf_core/writer.h" #include "hdr/types/FILE.h" @@ -105,4 +105,4 @@ LIBC_INLINE ErrorOr vfprintf_internal(::FILE *__restrict stream, } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_VFPRINTF_INTERNAL_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_VFPRINTF_INTERNAL_H diff --git a/libc/src/stdio/printf_core/write_int_converter.h b/libc/src/__support/printf_core/write_int_converter.h similarity index 88% rename from libc/src/stdio/printf_core/write_int_converter.h rename to libc/src/__support/printf_core/write_int_converter.h index 2844a56dc9922..0d6c79015164d 100644 --- a/libc/src/stdio/printf_core/write_int_converter.h +++ b/libc/src/__support/printf_core/write_int_converter.h @@ -6,12 +6,12 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_WRITE_INT_CONVERTER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_WRITE_INT_CONVERTER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_WRITE_INT_CONVERTER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_WRITE_INT_CONVERTER_H #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/writer.h" #include #include @@ -75,4 +75,4 @@ LIBC_INLINE int convert_write_int(Writer *writer, } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_WRITE_INT_CONVERTER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_WRITE_INT_CONVERTER_H diff --git a/libc/src/stdio/printf_core/write_modes.def b/libc/src/__support/printf_core/write_modes.def similarity index 100% rename from libc/src/stdio/printf_core/write_modes.def rename to libc/src/__support/printf_core/write_modes.def diff --git a/libc/src/stdio/printf_core/writer.h b/libc/src/__support/printf_core/writer.h similarity index 97% rename from libc/src/stdio/printf_core/writer.h rename to libc/src/__support/printf_core/writer.h index b82792e4d5e02..e7d3fe530d7b0 100644 --- a/libc/src/stdio/printf_core/writer.h +++ b/libc/src/__support/printf_core/writer.h @@ -6,13 +6,13 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_WRITER_H -#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_WRITER_H +#ifndef LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_WRITER_H +#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_WRITER_H #include "src/__support/CPP/string_view.h" #include "src/__support/macros/config.h" #include "src/__support/macros/optimization.h" -#include "src/stdio/printf_core/core_structs.h" +#include "src/__support/printf_core/core_structs.h" #include "src/string/memory_utils/inline_memcpy.h" #include "src/string/memory_utils/inline_memset.h" @@ -23,7 +23,7 @@ namespace printf_core { #define HANDLE_WRITE_MODE(MODE) MODE, enum class WriteMode { -#include "src/stdio/printf_core/write_modes.def" +#include "src/__support/printf_core/write_modes.def" }; #undef HANDLE_WRITE_MODE @@ -254,4 +254,4 @@ Writer(WriteBuffer) } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL -#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_WRITER_H +#endif // LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_WRITER_H diff --git a/libc/src/err/CMakeLists.txt b/libc/src/err/CMakeLists.txt index a2c2c43f4fc3a..8f72d2178ea77 100644 --- a/libc/src/err/CMakeLists.txt +++ b/libc/src/err/CMakeLists.txt @@ -14,8 +14,8 @@ add_object_library( libc.src.__support.libc_errno libc.src.__support.OSUtil.osutil libc.src.__support.StringUtil.error_to_string - libc.src.stdio.printf_core.printf_main - libc.src.stdio.printf_core.writer + libc.src.__support.printf_core.printf_main + libc.src.__support.printf_core.writer libc.src.__support.CPP.string_view libc.src.__support.macros.config ) diff --git a/libc/src/err/report.cpp b/libc/src/err/report.cpp index bd4274fce77ed..1dd69dce3a5f0 100644 --- a/libc/src/err/report.cpp +++ b/libc/src/err/report.cpp @@ -20,8 +20,8 @@ #include "src/__support/arg_list.h" #include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/printf_main.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/printf_main.h" +#include "src/__support/printf_core/writer.h" #ifdef __linux__ extern "C" char *program_invocation_short_name; diff --git a/libc/src/stdio/CMakeLists.txt b/libc/src/stdio/CMakeLists.txt index feee8d60d1c60..0e3fe55462e03 100644 --- a/libc/src/stdio/CMakeLists.txt +++ b/libc/src/stdio/CMakeLists.txt @@ -129,10 +129,10 @@ add_entrypoint_object( HDRS sprintf.h DEPENDS - libc.src.stdio.printf_core.printf_main - libc.src.stdio.printf_core.writer - libc.src.stdio.printf_core.core_structs - libc.src.stdio.printf_core.error_mapper + libc.src.__support.printf_core.printf_main + libc.src.__support.printf_core.writer + libc.src.__support.printf_core.core_structs + libc.src.__support.printf_core.error_mapper libc.src.__support.libc_errno libc.src.__support.CPP.limits ) @@ -148,10 +148,10 @@ add_entrypoint_object( HDRS snprintf.h DEPENDS - libc.src.stdio.printf_core.printf_main - libc.src.stdio.printf_core.writer - libc.src.stdio.printf_core.core_structs - libc.src.stdio.printf_core.error_mapper + libc.src.__support.printf_core.printf_main + libc.src.__support.printf_core.writer + libc.src.__support.printf_core.core_structs + libc.src.__support.printf_core.error_mapper libc.src.__support.libc_errno libc.src.__support.CPP.limits ) @@ -167,9 +167,9 @@ add_entrypoint_object( HDRS asprintf.h DEPENDS - libc.src.stdio.printf_core.vasprintf_internal - libc.src.stdio.printf_core.core_structs - libc.src.stdio.printf_core.error_mapper + libc.src.__support.printf_core.vasprintf_internal + libc.src.__support.printf_core.core_structs + libc.src.__support.printf_core.error_mapper libc.src.__support.libc_errno libc.src.__support.CPP.limits ) @@ -186,10 +186,10 @@ add_entrypoint_object( HDRS vsprintf.h DEPENDS - libc.src.stdio.printf_core.printf_main - libc.src.stdio.printf_core.writer - libc.src.stdio.printf_core.core_structs - libc.src.stdio.printf_core.error_mapper + libc.src.__support.printf_core.printf_main + libc.src.__support.printf_core.writer + libc.src.__support.printf_core.core_structs + libc.src.__support.printf_core.error_mapper libc.src.__support.libc_errno libc.src.__support.CPP.limits ) @@ -206,10 +206,10 @@ add_entrypoint_object( HDRS vsnprintf.h DEPENDS - libc.src.stdio.printf_core.printf_main - libc.src.stdio.printf_core.writer - libc.src.stdio.printf_core.core_structs - libc.src.stdio.printf_core.error_mapper + libc.src.__support.printf_core.printf_main + libc.src.__support.printf_core.writer + libc.src.__support.printf_core.core_structs + libc.src.__support.printf_core.error_mapper libc.src.__support.libc_errno libc.src.__support.CPP.limits ) @@ -226,14 +226,13 @@ add_entrypoint_object( HDRS vasprintf.h DEPENDS - libc.src.stdio.printf_core.vasprintf_internal - libc.src.stdio.printf_core.core_structs - libc.src.stdio.printf_core.error_mapper + libc.src.__support.printf_core.vasprintf_internal + libc.src.__support.printf_core.core_structs + libc.src.__support.printf_core.error_mapper libc.src.__support.libc_errno libc.src.__support.CPP.limits ) -add_subdirectory(printf_core) add_subdirectory(scanf_core) add_entrypoint_object( diff --git a/libc/src/stdio/asprintf.cpp b/libc/src/stdio/asprintf.cpp index f80d2a5eb759d..433e300056203 100644 --- a/libc/src/stdio/asprintf.cpp +++ b/libc/src/stdio/asprintf.cpp @@ -11,9 +11,9 @@ #include "src/__support/arg_list.h" #include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/vasprintf_internal.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/vasprintf_internal.h" namespace LIBC_NAMESPACE_DECL { diff --git a/libc/src/stdio/asprintf_modular.cpp b/libc/src/stdio/asprintf_modular.cpp index 74b62cc0c2501..b1f4b86e712b1 100644 --- a/libc/src/stdio/asprintf_modular.cpp +++ b/libc/src/stdio/asprintf_modular.cpp @@ -10,10 +10,10 @@ #include "src/__support/arg_list.h" #include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/vasprintf_internal.h" #include "src/stdio/asprintf.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/vasprintf_internal.h" namespace LIBC_NAMESPACE_DECL { diff --git a/libc/src/stdio/baremetal/CMakeLists.txt b/libc/src/stdio/baremetal/CMakeLists.txt index 526400380941c..d26cdbb0f9d51 100644 --- a/libc/src/stdio/baremetal/CMakeLists.txt +++ b/libc/src/stdio/baremetal/CMakeLists.txt @@ -22,10 +22,10 @@ add_header_library( libc.src.__support.CPP.string_view libc.src.__support.libc_errno libc.src.__support.OSUtil.osutil - libc.src.stdio.printf_core.printf_main - libc.src.stdio.printf_core.writer - libc.src.stdio.printf_core.error_mapper - libc.src.stdio.printf_core.core_structs + libc.src.__support.printf_core.printf_main + libc.src.__support.printf_core.writer + libc.src.__support.printf_core.error_mapper + libc.src.__support.printf_core.core_structs ) add_header_library( diff --git a/libc/src/stdio/baremetal/vfprintf_internal.h b/libc/src/stdio/baremetal/vfprintf_internal.h index c4cd8b8ad8a83..c86c40a7cca0c 100644 --- a/libc/src/stdio/baremetal/vfprintf_internal.h +++ b/libc/src/stdio/baremetal/vfprintf_internal.h @@ -17,10 +17,10 @@ #include "src/__support/common.h" #include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/printf_main.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/printf_main.h" +#include "src/__support/printf_core/writer.h" namespace LIBC_NAMESPACE_DECL { diff --git a/libc/src/stdio/generic/CMakeLists.txt b/libc/src/stdio/generic/CMakeLists.txt index a9c5ae698c3bb..6b3b611ad731f 100644 --- a/libc/src/stdio/generic/CMakeLists.txt +++ b/libc/src/stdio/generic/CMakeLists.txt @@ -412,9 +412,9 @@ list(APPEND fprintf_deps libc.src.__support.arg_list libc.src.__support.CPP.limits libc.src.__support.libc_errno - libc.src.stdio.printf_core.vfprintf_internal - libc.src.stdio.printf_core.core_structs - libc.src.stdio.printf_core.error_mapper + libc.src.__support.printf_core.vfprintf_internal + libc.src.__support.printf_core.core_structs + libc.src.__support.printf_core.error_mapper ) if(LLVM_LIBC_FULL_BUILD) diff --git a/libc/src/stdio/generic/fprintf.cpp b/libc/src/stdio/generic/fprintf.cpp index b2033901557a0..11eeeb7dd4ade 100644 --- a/libc/src/stdio/generic/fprintf.cpp +++ b/libc/src/stdio/generic/fprintf.cpp @@ -12,9 +12,9 @@ #include "src/__support/File/file.h" #include "src/__support/arg_list.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/vfprintf_internal.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/vfprintf_internal.h" #include "hdr/types/FILE.h" #include diff --git a/libc/src/stdio/generic/printf.cpp b/libc/src/stdio/generic/printf.cpp index d02007dbd1471..5cb8820f8c84c 100644 --- a/libc/src/stdio/generic/printf.cpp +++ b/libc/src/stdio/generic/printf.cpp @@ -12,9 +12,9 @@ #include "src/__support/File/file.h" #include "src/__support/arg_list.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/vfprintf_internal.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/vfprintf_internal.h" #include "src/stdio/stdout.h" #include "hdr/types/FILE.h" diff --git a/libc/src/stdio/generic/vfprintf.cpp b/libc/src/stdio/generic/vfprintf.cpp index a26f082ed9347..fe70296a4b3af 100644 --- a/libc/src/stdio/generic/vfprintf.cpp +++ b/libc/src/stdio/generic/vfprintf.cpp @@ -12,9 +12,9 @@ #include "src/__support/File/file.h" #include "src/__support/arg_list.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/vfprintf_internal.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/vfprintf_internal.h" #include "hdr/types/FILE.h" #include diff --git a/libc/src/stdio/generic/vprintf.cpp b/libc/src/stdio/generic/vprintf.cpp index 2bd3b40a821ce..ee9e0d3059a22 100644 --- a/libc/src/stdio/generic/vprintf.cpp +++ b/libc/src/stdio/generic/vprintf.cpp @@ -12,9 +12,9 @@ #include "src/__support/File/file.h" #include "src/__support/arg_list.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/vfprintf_internal.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/vfprintf_internal.h" #include "src/stdio/stdout.h" #include "hdr/types/FILE.h" diff --git a/libc/src/stdio/snprintf.cpp b/libc/src/stdio/snprintf.cpp index d8fb1b1ad3577..ec1364ca6efc2 100644 --- a/libc/src/stdio/snprintf.cpp +++ b/libc/src/stdio/snprintf.cpp @@ -12,10 +12,10 @@ #include "src/__support/arg_list.h" #include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/printf_main.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/printf_main.h" +#include "src/__support/printf_core/writer.h" #include #include diff --git a/libc/src/stdio/snprintf_modular.cpp b/libc/src/stdio/snprintf_modular.cpp index 2e7683d56de75..86afdb9c5828e 100644 --- a/libc/src/stdio/snprintf_modular.cpp +++ b/libc/src/stdio/snprintf_modular.cpp @@ -12,10 +12,10 @@ #include "src/__support/arg_list.h" #include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/printf_main.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/printf_main.h" +#include "src/__support/printf_core/writer.h" #include #include diff --git a/libc/src/stdio/sprintf.cpp b/libc/src/stdio/sprintf.cpp index bbb3f64ea7b15..702ea5d43c69a 100644 --- a/libc/src/stdio/sprintf.cpp +++ b/libc/src/stdio/sprintf.cpp @@ -12,10 +12,10 @@ #include "src/__support/arg_list.h" #include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/printf_main.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/printf_main.h" +#include "src/__support/printf_core/writer.h" #include diff --git a/libc/src/stdio/sprintf_modular.cpp b/libc/src/stdio/sprintf_modular.cpp index cfed3fe4a4f4d..9003e2440d535 100644 --- a/libc/src/stdio/sprintf_modular.cpp +++ b/libc/src/stdio/sprintf_modular.cpp @@ -12,10 +12,10 @@ #include "src/__support/arg_list.h" #include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/printf_main.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/printf_main.h" +#include "src/__support/printf_core/writer.h" #include diff --git a/libc/src/stdio/vasprintf.cpp b/libc/src/stdio/vasprintf.cpp index 64d959c6edcae..444b5b1efffde 100644 --- a/libc/src/stdio/vasprintf.cpp +++ b/libc/src/stdio/vasprintf.cpp @@ -10,9 +10,9 @@ #include "src/__support/CPP/limits.h" #include "src/__support/arg_list.h" #include "src/__support/libc_errno.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/vasprintf_internal.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/vasprintf_internal.h" namespace LIBC_NAMESPACE_DECL { diff --git a/libc/src/stdio/vasprintf_modular.cpp b/libc/src/stdio/vasprintf_modular.cpp index 98620bb8a440c..87dda8e82813f 100644 --- a/libc/src/stdio/vasprintf_modular.cpp +++ b/libc/src/stdio/vasprintf_modular.cpp @@ -9,9 +9,9 @@ #include "src/__support/CPP/limits.h" #include "src/__support/arg_list.h" #include "src/__support/libc_errno.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/vasprintf_internal.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/vasprintf_internal.h" #include "src/stdio/vasprintf.h" namespace LIBC_NAMESPACE_DECL { diff --git a/libc/src/stdio/vsnprintf.cpp b/libc/src/stdio/vsnprintf.cpp index bea980d6b20ec..b25daed931fce 100644 --- a/libc/src/stdio/vsnprintf.cpp +++ b/libc/src/stdio/vsnprintf.cpp @@ -12,10 +12,10 @@ #include "src/__support/arg_list.h" #include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/printf_main.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/printf_main.h" +#include "src/__support/printf_core/writer.h" #include #include diff --git a/libc/src/stdio/vsnprintf_modular.cpp b/libc/src/stdio/vsnprintf_modular.cpp index 7225311601281..96f020839c3cc 100644 --- a/libc/src/stdio/vsnprintf_modular.cpp +++ b/libc/src/stdio/vsnprintf_modular.cpp @@ -12,10 +12,10 @@ #include "src/__support/arg_list.h" #include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/printf_main.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/printf_main.h" +#include "src/__support/printf_core/writer.h" #include #include diff --git a/libc/src/stdio/vsprintf.cpp b/libc/src/stdio/vsprintf.cpp index 3dae05fbbaadd..f1af2f6143ad2 100644 --- a/libc/src/stdio/vsprintf.cpp +++ b/libc/src/stdio/vsprintf.cpp @@ -12,10 +12,10 @@ #include "src/__support/arg_list.h" #include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/printf_main.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/printf_main.h" +#include "src/__support/printf_core/writer.h" #include diff --git a/libc/src/stdio/vsprintf_modular.cpp b/libc/src/stdio/vsprintf_modular.cpp index 844d1e3cdd550..34deab1bfe412 100644 --- a/libc/src/stdio/vsprintf_modular.cpp +++ b/libc/src/stdio/vsprintf_modular.cpp @@ -12,10 +12,10 @@ #include "src/__support/arg_list.h" #include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" -#include "src/stdio/printf_core/printf_main.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" +#include "src/__support/printf_core/printf_main.h" +#include "src/__support/printf_core/writer.h" #include diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt index 27a9934d0bd3b..7df0a2b0c4656 100644 --- a/libc/src/stdlib/CMakeLists.txt +++ b/libc/src/stdlib/CMakeLists.txt @@ -116,7 +116,7 @@ add_entrypoint_object( DEPENDS .str_from_util libc.src.__support.CPP.limits - libc.src.stdio.printf_core.error_mapper + libc.src.__support.printf_core.error_mapper ) add_entrypoint_object( @@ -128,7 +128,7 @@ add_entrypoint_object( DEPENDS .str_from_util libc.src.__support.CPP.limits - libc.src.stdio.printf_core.error_mapper + libc.src.__support.printf_core.error_mapper ) add_entrypoint_object( @@ -140,7 +140,7 @@ add_entrypoint_object( DEPENDS .str_from_util libc.src.__support.CPP.limits - libc.src.stdio.printf_core.error_mapper + libc.src.__support.printf_core.error_mapper ) add_header_library( @@ -148,9 +148,9 @@ add_header_library( HDRS str_from_util.h DEPENDS - libc.src.stdio.printf_core.converter - libc.src.stdio.printf_core.core_structs - libc.src.stdio.printf_core.writer + libc.src.__support.printf_core.converter + libc.src.__support.printf_core.core_structs + libc.src.__support.printf_core.writer libc.src.__support.str_to_integer libc.src.__support.CPP.type_traits ) diff --git a/libc/src/stdlib/str_from_util.h b/libc/src/stdlib/str_from_util.h index 61e6ba24b3817..8cd147580f719 100644 --- a/libc/src/stdlib/str_from_util.h +++ b/libc/src/stdlib/str_from_util.h @@ -19,10 +19,10 @@ #include "src/__support/CPP/type_traits.h" #include "src/__support/macros/config.h" +#include "src/__support/printf_core/converter_atlas.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/writer.h" #include "src/__support/str_to_integer.h" -#include "src/stdio/printf_core/converter_atlas.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/writer.h" #include diff --git a/libc/src/stdlib/strfromd.cpp b/libc/src/stdlib/strfromd.cpp index 5409a57c722d4..dbf46b7c0534b 100644 --- a/libc/src/stdlib/strfromd.cpp +++ b/libc/src/stdlib/strfromd.cpp @@ -9,8 +9,8 @@ #include "src/stdlib/strfromd.h" #include "src/__support/CPP/limits.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" #include "src/stdlib/str_from_util.h" namespace LIBC_NAMESPACE_DECL { diff --git a/libc/src/stdlib/strfromf.cpp b/libc/src/stdlib/strfromf.cpp index e68e50210dd4d..d1ea47d55a250 100644 --- a/libc/src/stdlib/strfromf.cpp +++ b/libc/src/stdlib/strfromf.cpp @@ -9,8 +9,8 @@ #include "src/stdlib/strfromf.h" #include "src/__support/CPP/limits.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" #include "src/stdlib/str_from_util.h" namespace LIBC_NAMESPACE_DECL { diff --git a/libc/src/stdlib/strfroml.cpp b/libc/src/stdlib/strfroml.cpp index 8dbd8d7fc4f2e..c23c12fc91c27 100644 --- a/libc/src/stdlib/strfroml.cpp +++ b/libc/src/stdlib/strfroml.cpp @@ -9,8 +9,8 @@ #include "src/stdlib/strfroml.h" #include "src/__support/CPP/limits.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/error_mapper.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/error_mapper.h" #include "src/stdlib/str_from_util.h" namespace LIBC_NAMESPACE_DECL { diff --git a/libc/src/time/CMakeLists.txt b/libc/src/time/CMakeLists.txt index 76178542c6c9a..f6b1c370f6d84 100644 --- a/libc/src/time/CMakeLists.txt +++ b/libc/src/time/CMakeLists.txt @@ -194,9 +194,9 @@ add_entrypoint_object( DEPENDS libc.hdr.types.size_t libc.hdr.types.struct_tm - libc.src.stdio.printf_core.writer libc.src.time.strftime_core.strftime_main libc.src.__support.macros.null_check + libc.src.__support.printf_core.writer ) add_entrypoint_object( @@ -209,9 +209,9 @@ add_entrypoint_object( libc.hdr.types.locale_t libc.hdr.types.size_t libc.hdr.types.struct_tm - libc.src.stdio.printf_core.writer libc.src.time.strftime_core.strftime_main libc.src.__support.macros.null_check + libc.src.__support.printf_core.writer ) add_entrypoint_object( diff --git a/libc/src/time/strftime.cpp b/libc/src/time/strftime.cpp index 71e1e99080e56..5522a2c1d2e4b 100644 --- a/libc/src/time/strftime.cpp +++ b/libc/src/time/strftime.cpp @@ -12,7 +12,7 @@ #include "src/__support/common.h" #include "src/__support/macros/config.h" #include "src/__support/macros/null_check.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/writer.h" #include "src/time/strftime_core/strftime_main.h" namespace LIBC_NAMESPACE_DECL { diff --git a/libc/src/time/strftime_core/CMakeLists.txt b/libc/src/time/strftime_core/CMakeLists.txt index 31eb41aa1647f..6996638128da5 100644 --- a/libc/src/time/strftime_core/CMakeLists.txt +++ b/libc/src/time/strftime_core/CMakeLists.txt @@ -30,7 +30,7 @@ add_header_library( .core_structs libc.src.time.time_utils libc.src.time.time_constants - libc.src.stdio.printf_core.writer + libc.src.__support.printf_core.writer libc.src.__support.CPP.string_view libc.src.__support.integer_to_string libc.src.__support.error_or @@ -45,6 +45,6 @@ add_header_library( .parser .converter libc.src.__support.error_or - libc.src.stdio.printf_core.writer + libc.src.__support.printf_core.writer libc.hdr.types.struct_tm ) diff --git a/libc/src/time/strftime_core/composite_converter.h b/libc/src/time/strftime_core/composite_converter.h index 9d343f3aef611..ec279615806e9 100644 --- a/libc/src/time/strftime_core/composite_converter.h +++ b/libc/src/time/strftime_core/composite_converter.h @@ -12,7 +12,7 @@ #include "hdr/types/struct_tm.h" #include "src/__support/CPP/string_view.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/writer.h" #include "src/time/strftime_core/core_structs.h" #include "src/time/strftime_core/num_converter.h" #include "src/time/strftime_core/str_converter.h" diff --git a/libc/src/time/strftime_core/converter.h b/libc/src/time/strftime_core/converter.h index 94e4366b4cb23..b6203ae8b72d0 100644 --- a/libc/src/time/strftime_core/converter.h +++ b/libc/src/time/strftime_core/converter.h @@ -16,7 +16,7 @@ #include "hdr/types/struct_tm.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/writer.h" #include "src/time/strftime_core/composite_converter.h" #include "src/time/strftime_core/core_structs.h" #include "src/time/strftime_core/num_converter.h" diff --git a/libc/src/time/strftime_core/num_converter.h b/libc/src/time/strftime_core/num_converter.h index c44e2058b6653..6bcc166fecf93 100644 --- a/libc/src/time/strftime_core/num_converter.h +++ b/libc/src/time/strftime_core/num_converter.h @@ -14,7 +14,7 @@ #include "src/__support/error_or.h" #include "src/__support/integer_to_string.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/writer.h" #include "src/time/strftime_core/core_structs.h" #include "src/time/time_constants.h" #include "src/time/time_utils.h" diff --git a/libc/src/time/strftime_core/str_converter.h b/libc/src/time/strftime_core/str_converter.h index 3273dcf9a6b86..4a636679c4a18 100644 --- a/libc/src/time/strftime_core/str_converter.h +++ b/libc/src/time/strftime_core/str_converter.h @@ -17,7 +17,7 @@ #include "hdr/types/struct_tm.h" #include "src/__support/CPP/string_view.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/writer.h" #include "src/time/strftime_core/core_structs.h" #include "src/time/time_constants.h" #include "src/time/time_utils.h" diff --git a/libc/src/time/strftime_core/strftime_main.h b/libc/src/time/strftime_core/strftime_main.h index 855a44107914c..4f958fb5bb294 100644 --- a/libc/src/time/strftime_core/strftime_main.h +++ b/libc/src/time/strftime_core/strftime_main.h @@ -12,7 +12,7 @@ #include "hdr/types/struct_tm.h" #include "src/__support/error_or.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/writer.h" #include "src/time/strftime_core/converter.h" #include "src/time/strftime_core/core_structs.h" #include "src/time/strftime_core/parser.h" diff --git a/libc/src/time/strftime_l.cpp b/libc/src/time/strftime_l.cpp index 8a8950fee91cc..faf3883c476b2 100644 --- a/libc/src/time/strftime_l.cpp +++ b/libc/src/time/strftime_l.cpp @@ -13,7 +13,7 @@ #include "src/__support/common.h" #include "src/__support/macros/config.h" #include "src/__support/macros/null_check.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/writer.h" #include "src/time/strftime_core/strftime_main.h" namespace LIBC_NAMESPACE_DECL { diff --git a/libc/test/UnitTest/CMakeLists.txt b/libc/test/UnitTest/CMakeLists.txt index 4a47597ae3a45..09e7655683fd5 100644 --- a/libc/test/UnitTest/CMakeLists.txt +++ b/libc/test/UnitTest/CMakeLists.txt @@ -183,8 +183,8 @@ add_unittest_framework_library( LibcTest libc.hdr.stdint_proxy libc.src.__support.FPUtil.fp_bits - libc.src.stdio.printf_core.core_structs - libc.src.stdio.printf_core.printf_config + libc.src.__support.printf_core.core_structs + libc.src.__support.printf_core.printf_config libc.test.UnitTest.string_utils ) diff --git a/libc/test/UnitTest/PrintfMatcher.cpp b/libc/test/UnitTest/PrintfMatcher.cpp index 886737f0dcb06..4fa8864b8744a 100644 --- a/libc/test/UnitTest/PrintfMatcher.cpp +++ b/libc/test/UnitTest/PrintfMatcher.cpp @@ -11,8 +11,8 @@ #include "hdr/stdint_proxy.h" #include "src/__support/FPUtil/FPBits.h" #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/printf_config.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/printf_config.h" #include "test/UnitTest/StringUtils.h" #include "test/UnitTest/Test.h" diff --git a/libc/test/UnitTest/PrintfMatcher.h b/libc/test/UnitTest/PrintfMatcher.h index 4f79832ab59fa..bb38aa43c36bd 100644 --- a/libc/test/UnitTest/PrintfMatcher.h +++ b/libc/test/UnitTest/PrintfMatcher.h @@ -10,7 +10,7 @@ #define LLVM_LIBC_UTILS_UNITTEST_PRINTF_MATCHER_H #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" +#include "src/__support/printf_core/core_structs.h" #include "test/UnitTest/Test.h" namespace LIBC_NAMESPACE_DECL { diff --git a/libc/test/shared/CMakeLists.txt b/libc/test/shared/CMakeLists.txt index 71b8bcd8d4ff1..9208dbcc5ade6 100644 --- a/libc/test/shared/CMakeLists.txt +++ b/libc/test/shared/CMakeLists.txt @@ -863,9 +863,9 @@ add_libc_test( SRCS shared_rpc_test.cpp DEPENDS - libc.src.stdio.printf_core.converter - libc.src.stdio.printf_core.parser - libc.src.stdio.printf_core.writer + libc.src.__support.printf_core.converter + libc.src.__support.printf_core.parser + libc.src.__support.printf_core.writer libc.src.__support.arg_list ) endif() diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt index 8233529266326..4e6784691d5e5 100644 --- a/libc/test/src/__support/CMakeLists.txt +++ b/libc/test/src/__support/CMakeLists.txt @@ -305,6 +305,7 @@ add_libc_test( add_subdirectory(CPP) add_subdirectory(File) +add_subdirectory(printf_core) add_subdirectory(RPC) add_subdirectory(OSUtil) add_subdirectory(FPUtil) diff --git a/libc/test/src/stdio/printf_core/CMakeLists.txt b/libc/test/src/__support/printf_core/CMakeLists.txt similarity index 56% rename from libc/test/src/stdio/printf_core/CMakeLists.txt rename to libc/test/src/__support/printf_core/CMakeLists.txt index ff7ebbc4f5fd0..b4db465666363 100644 --- a/libc/test/src/stdio/printf_core/CMakeLists.txt +++ b/libc/test/src/__support/printf_core/CMakeLists.txt @@ -1,14 +1,14 @@ add_libc_unittest( parser_test SUITE - libc_stdio_unittests + libc-support-tests SRCS parser_test.cpp LINK_LIBRARIES LibcPrintfHelpers DEPENDS - libc.src.stdio.printf_core.parser - libc.src.stdio.printf_core.core_structs + libc.src.__support.printf_core.parser + libc.src.__support.printf_core.core_structs libc.src.__support.CPP.string_view libc.src.__support.arg_list ) @@ -16,11 +16,11 @@ add_libc_unittest( add_libc_unittest( writer_test SUITE - libc_stdio_unittests + libc-support-tests SRCS writer_test.cpp DEPENDS - libc.src.stdio.printf_core.writer + libc.src.__support.printf_core.writer libc.src.string.memory_utils.inline_memcpy libc.src.__support.CPP.string_view ) @@ -28,11 +28,11 @@ add_libc_unittest( add_libc_unittest( converter_test SUITE - libc_stdio_unittests + libc-support-tests SRCS converter_test.cpp DEPENDS - libc.src.stdio.printf_core.converter - libc.src.stdio.printf_core.writer - libc.src.stdio.printf_core.core_structs + libc.src.__support.printf_core.converter + libc.src.__support.printf_core.writer + libc.src.__support.printf_core.core_structs ) diff --git a/libc/test/src/stdio/printf_core/converter_test.cpp b/libc/test/src/__support/printf_core/converter_test.cpp similarity index 98% rename from libc/test/src/stdio/printf_core/converter_test.cpp rename to libc/test/src/__support/printf_core/converter_test.cpp index 68e2fdac89d47..36037ab2ed4b0 100644 --- a/libc/test/src/stdio/printf_core/converter_test.cpp +++ b/libc/test/src/__support/printf_core/converter_test.cpp @@ -6,9 +6,9 @@ // //===----------------------------------------------------------------------===// -#include "src/stdio/printf_core/converter.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/converter.h" +#include "src/__support/printf_core/core_structs.h" +#include "src/__support/printf_core/writer.h" #include "test/UnitTest/Test.h" diff --git a/libc/test/src/stdio/printf_core/parser_test.cpp b/libc/test/src/__support/printf_core/parser_test.cpp similarity index 99% rename from libc/test/src/stdio/printf_core/parser_test.cpp rename to libc/test/src/__support/printf_core/parser_test.cpp index a73e3af990dd3..591bf2de87125 100644 --- a/libc/test/src/stdio/printf_core/parser_test.cpp +++ b/libc/test/src/__support/printf_core/parser_test.cpp @@ -9,7 +9,7 @@ #include "src/__support/CPP/bit.h" #include "src/__support/CPP/string_view.h" #include "src/__support/arg_list.h" -#include "src/stdio/printf_core/parser.h" +#include "src/__support/printf_core/parser.h" #include diff --git a/libc/test/src/stdio/printf_core/writer_test.cpp b/libc/test/src/__support/printf_core/writer_test.cpp similarity index 99% rename from libc/test/src/stdio/printf_core/writer_test.cpp rename to libc/test/src/__support/printf_core/writer_test.cpp index e0128505b2766..b1873d729bb3f 100644 --- a/libc/test/src/stdio/printf_core/writer_test.cpp +++ b/libc/test/src/__support/printf_core/writer_test.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "src/stdio/printf_core/writer.h" +#include "src/__support/printf_core/writer.h" #include "src/__support/CPP/string_view.h" #include "src/string/memory_utils/inline_memcpy.h" diff --git a/libc/test/src/stdio/CMakeLists.txt b/libc/test/src/stdio/CMakeLists.txt index 465e3c8698076..e0e5656f24db3 100644 --- a/libc/test/src/stdio/CMakeLists.txt +++ b/libc/test/src/stdio/CMakeLists.txt @@ -173,9 +173,9 @@ add_fp_unittest( SRCS sprintf_test.cpp DEPENDS - libc.src.stdio.printf_core.printf_config libc.src.stdio.sprintf libc.src.__support.FPUtil.fp_bits + libc.src.__support.printf_core.printf_config libc.include.inttypes ${wchar_deps} COMPILE_OPTIONS @@ -609,6 +609,5 @@ if(LIBC_TARGET_OS_IS_GPU) return() endif() -add_subdirectory(printf_core) add_subdirectory(scanf_core) add_subdirectory(testdata) diff --git a/libc/test/src/stdio/sprintf_test.cpp b/libc/test/src/stdio/sprintf_test.cpp index 3c0e9c74d0c06..a366920339110 100644 --- a/libc/test/src/stdio/sprintf_test.cpp +++ b/libc/test/src/stdio/sprintf_test.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// #include "src/__support/macros/config.h" -#include "src/stdio/printf_core/printf_config.h" +#include "src/__support/printf_core/printf_config.h" #include "src/stdio/sprintf.h" #ifndef LIBC_COPT_PRINTF_DISABLE_WIDE diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 6020f511d0d3b..bee6f7aa5766c 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -2064,6 +2064,180 @@ libc_support_library( ], ) +libc_support_library( + name = "__support_printf_core_printf_config", + hdrs = ["src/__support/printf_core/printf_config.h"], + deps = [ + ], +) + +libc_support_library( + name = "__support_printf_core_printf_core_structs", + hdrs = ["src/__support/printf_core/core_structs.h"], + deps = [ + ":__support_cpp_string_view", + ":__support_cpp_type_traits", + ":__support_fputil_fp_bits", + ":__support_macros_config", + ":__support_printf_core_printf_config", + ], +) + +libc_support_library( + name = "__support_printf_core_printf_parser", + hdrs = ["src/__support/printf_core/parser.h"], + deps = [ + ":__support_arg_list", + ":__support_common", + ":__support_cpp_algorithm", + ":__support_cpp_bit", + ":__support_cpp_limits", + ":__support_cpp_optional", + ":__support_cpp_string_view", + ":__support_cpp_type_traits", + ":__support_ctype_utils", + ":__support_fixed_point", + ":__support_fputil_fp_bits", + ":__support_libc_errno", + ":__support_macros_config", + ":__support_macros_properties_types", + ":__support_printf_core_printf_config", + ":__support_printf_core_printf_core_structs", + ":__support_str_to_integer", + ":errno", + ":llvm_libc_macros_stdfix_macros", + ":types_wint_t", + ], +) + +libc_support_library( + name = "__support_printf_core_printf_writer", + hdrs = ["src/__support/printf_core/writer.h"], + textual_hdrs = ["src/__support/printf_core/write_modes.def"], + deps = [ + ":__support_cpp_string_view", + ":__support_macros_config", + ":__support_macros_optimization", + ":__support_printf_core_printf_core_structs", + ":string_memory_utils", + ], +) + +libc_support_library( + name = "__support_printf_core_printf_converter", + hdrs = [ + "src/__support/printf_core/char_converter.h", + "src/__support/printf_core/converter.h", + "src/__support/printf_core/converter_atlas.h", + "src/__support/printf_core/converter_utils.h", + "src/__support/printf_core/float_dec_converter.h", + "src/__support/printf_core/float_hex_converter.h", + "src/__support/printf_core/float_inf_nan_converter.h", + "src/__support/printf_core/int_converter.h", + "src/__support/printf_core/ptr_converter.h", + "src/__support/printf_core/strerror_converter.h", + "src/__support/printf_core/string_converter.h", + "src/__support/printf_core/write_int_converter.h", + ], + deps = [ + ":__support_big_int", + ":__support_common", + ":__support_cpp_limits", + ":__support_cpp_span", + ":__support_cpp_string_view", + ":__support_ctype_utils", + ":__support_float_to_string", + ":__support_fputil_fenv_impl", + ":__support_fputil_fp_bits", + ":__support_fputil_rounding_mode", + ":__support_integer_to_string", + ":__support_libc_assert", + ":__support_macros_config", + ":__support_printf_core_printf_config", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_writer", + ":__support_stringutil", + ":__support_uint128", + ":__support_wchar_mbstate", + ":__support_wchar_string_converter", + ":__support_wchar_wcrtomb", + ":hdr_limits_macros", + ":hdr_wchar_macros", + ":string_utils", + ":types_char32_t", + ":types_char8_t", + ":types_wchar_t", + ":types_wint_t", + ], +) + +libc_support_library( + name = "__support_printf_core_printf_error_mapper", + hdrs = [ + "src/__support/printf_core/error_mapper.h", + ] + select({ + "@platforms//os:linux": [ + "src/__support/printf_core/linux/error_mapper.h", + ], + "//conditions:default": ["src/__support/printf_core/generic/error_mapper.h"], + }), + deps = [ + ":__support_cpp_type_traits", + ":__support_error_or", + ":__support_macros_properties_architectures", + ":__support_printf_core_printf_core_structs", + ":hdr_errno_macros", + ], +) + +libc_support_library( + name = "__support_printf_core_printf_main", + hdrs = ["src/__support/printf_core/printf_main.h"], + deps = [ + ":__support_arg_list", + ":__support_error_or", + ":__support_macros_config", + ":__support_printf_core_printf_converter", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_error_mapper", + ":__support_printf_core_printf_parser", + ":__support_printf_core_printf_writer", + ], +) + +libc_support_library( + name = "__support_printf_core_vfprintf_internal", + hdrs = ["src/__support/printf_core/vfprintf_internal.h"], + deps = [ + ":__support_arg_list", + ":__support_error_or", + ":__support_file_file", + ":__support_macros_attributes", + ":__support_macros_config", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_main", + ":__support_printf_core_printf_writer", + ":types_FILE", + ], +) + +libc_support_library( + name = "__support_printf_core_vasprintf_internal", + hdrs = ["src/__support/printf_core/vasprintf_internal.h"], + deps = [ + ":__support_arg_list", + ":__support_error_or", + ":__support_macros_attributes", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_main", + ":__support_printf_core_printf_writer", + ":func_free", + ":func_malloc", + ":func_realloc", + ":types_FILE", + ], +) + libc_support_library( name = "__support_math_extras", hdrs = ["src/__support/math_extras.h"], @@ -13908,10 +14082,10 @@ libc_support_library( ":__support_common", ":__support_cpp_type_traits", ":__support_macros_config", + ":__support_printf_core_printf_converter", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_writer", ":__support_str_to_integer", - ":printf_converter", - ":printf_core_structs", - ":printf_writer", ], ) @@ -13923,8 +14097,8 @@ libc_function( ":__support_common", ":__support_cpp_limits", ":__support_macros_config", - ":printf_core_structs", - ":printf_error_mapper", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_error_mapper", ":str_from_util", ], ) @@ -13937,8 +14111,8 @@ libc_function( ":__support_common", ":__support_cpp_limits", ":__support_macros_config", - ":printf_core_structs", - ":printf_error_mapper", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_error_mapper", ":str_from_util", ], ) @@ -13951,8 +14125,8 @@ libc_function( ":__support_common", ":__support_cpp_limits", ":__support_macros_config", - ":printf_core_structs", - ":printf_error_mapper", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_error_mapper", ":str_from_util", ], ) @@ -15298,180 +15472,6 @@ libc_support_library( ], ) -libc_support_library( - name = "printf_config", - hdrs = ["src/stdio/printf_core/printf_config.h"], - deps = [ - ], -) - -libc_support_library( - name = "printf_core_structs", - hdrs = ["src/stdio/printf_core/core_structs.h"], - deps = [ - ":__support_cpp_string_view", - ":__support_cpp_type_traits", - ":__support_fputil_fp_bits", - ":__support_macros_config", - ":printf_config", - ], -) - -libc_support_library( - name = "printf_parser", - hdrs = ["src/stdio/printf_core/parser.h"], - deps = [ - ":__support_arg_list", - ":__support_common", - ":__support_cpp_algorithm", - ":__support_cpp_bit", - ":__support_cpp_limits", - ":__support_cpp_optional", - ":__support_cpp_string_view", - ":__support_cpp_type_traits", - ":__support_ctype_utils", - ":__support_fixed_point", - ":__support_fputil_fp_bits", - ":__support_libc_errno", - ":__support_macros_config", - ":__support_macros_properties_types", - ":__support_str_to_integer", - ":errno", - ":llvm_libc_macros_stdfix_macros", - ":printf_config", - ":printf_core_structs", - ":types_wint_t", - ], -) - -libc_support_library( - name = "printf_writer", - hdrs = ["src/stdio/printf_core/writer.h"], - textual_hdrs = ["src/stdio/printf_core/write_modes.def"], - deps = [ - ":__support_cpp_string_view", - ":__support_macros_config", - ":__support_macros_optimization", - ":printf_core_structs", - ":string_memory_utils", - ], -) - -libc_support_library( - name = "printf_converter", - hdrs = [ - "src/stdio/printf_core/char_converter.h", - "src/stdio/printf_core/converter.h", - "src/stdio/printf_core/converter_atlas.h", - "src/stdio/printf_core/converter_utils.h", - "src/stdio/printf_core/float_dec_converter.h", - "src/stdio/printf_core/float_hex_converter.h", - "src/stdio/printf_core/float_inf_nan_converter.h", - "src/stdio/printf_core/int_converter.h", - "src/stdio/printf_core/ptr_converter.h", - "src/stdio/printf_core/strerror_converter.h", - "src/stdio/printf_core/string_converter.h", - "src/stdio/printf_core/write_int_converter.h", - ], - deps = [ - ":__support_big_int", - ":__support_common", - ":__support_cpp_limits", - ":__support_cpp_span", - ":__support_cpp_string_view", - ":__support_ctype_utils", - ":__support_float_to_string", - ":__support_fputil_fenv_impl", - ":__support_fputil_fp_bits", - ":__support_fputil_rounding_mode", - ":__support_integer_to_string", - ":__support_libc_assert", - ":__support_macros_config", - ":__support_stringutil", - ":__support_uint128", - ":__support_wchar_mbstate", - ":__support_wchar_string_converter", - ":__support_wchar_wcrtomb", - ":hdr_limits_macros", - ":hdr_wchar_macros", - ":printf_config", - ":printf_core_structs", - ":printf_writer", - ":string_utils", - ":types_char32_t", - ":types_char8_t", - ":types_wchar_t", - ":types_wint_t", - ], -) - -libc_support_library( - name = "printf_error_mapper", - hdrs = [ - "src/stdio/printf_core/error_mapper.h", - ] + select({ - "@platforms//os:linux": [ - "src/stdio/printf_core/linux/error_mapper.h", - ], - "//conditions:default": ["src/stdio/printf_core/generic/error_mapper.h"], - }), - deps = [ - ":__support_cpp_type_traits", - ":__support_error_or", - ":__support_macros_properties_architectures", - ":hdr_errno_macros", - ":printf_core_structs", - ], -) - -libc_support_library( - name = "printf_main", - hdrs = ["src/stdio/printf_core/printf_main.h"], - deps = [ - ":__support_arg_list", - ":__support_error_or", - ":__support_macros_config", - ":printf_converter", - ":printf_core_structs", - ":printf_error_mapper", - ":printf_parser", - ":printf_writer", - ], -) - -libc_support_library( - name = "vfprintf_internal", - hdrs = ["src/stdio/printf_core/vfprintf_internal.h"], - deps = [ - ":__support_arg_list", - ":__support_error_or", - ":__support_file_file", - ":__support_macros_attributes", - ":__support_macros_config", - ":printf_core_structs", - ":printf_main", - ":printf_writer", - ":types_FILE", - ], -) - -libc_support_library( - name = "vasprintf_internal", - hdrs = ["src/stdio/printf_core/vasprintf_internal.h"], - deps = [ - ":__support_arg_list", - ":__support_error_or", - ":__support_macros_attributes", - ":func_free", - ":func_malloc", - ":func_realloc", - ":printf_core_structs", - ":printf_main", - ":printf_writer", - ":types_FILE", - ], -) - libc_function( name = "asprintf", srcs = ["src/stdio/asprintf.cpp"], @@ -15481,9 +15481,9 @@ libc_function( ":__support_cpp_limits", ":__support_libc_errno", ":__support_macros_config", - ":printf_core_structs", - ":printf_error_mapper", - ":vasprintf_internal", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_error_mapper", + ":__support_printf_core_vasprintf_internal", ], ) @@ -15496,12 +15496,12 @@ libc_function( ":__support_cpp_limits", ":__support_libc_errno", ":__support_macros_config", + ":__support_printf_core_printf_config", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_error_mapper", + ":__support_printf_core_printf_main", + ":__support_printf_core_printf_writer", ":errno", - ":printf_config", - ":printf_core_structs", - ":printf_error_mapper", - ":printf_main", - ":printf_writer", ], ) @@ -15514,11 +15514,11 @@ libc_function( ":__support_cpp_limits", ":__support_libc_errno", ":__support_macros_config", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_error_mapper", + ":__support_printf_core_printf_main", + ":__support_printf_core_printf_writer", ":errno", - ":printf_core_structs", - ":printf_error_mapper", - ":printf_main", - ":printf_writer", ], ) @@ -15531,12 +15531,12 @@ libc_function( ":__support_cpp_limits", ":__support_file_file", ":__support_macros_config", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_error_mapper", + ":__support_printf_core_vfprintf_internal", ":errno", - ":printf_core_structs", - ":printf_error_mapper", ":stdout", ":types_FILE", - ":vfprintf_internal", ], ) @@ -15549,11 +15549,11 @@ libc_function( ":__support_cpp_limits", ":__support_file_file", ":__support_macros_config", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_error_mapper", + ":__support_printf_core_vfprintf_internal", ":errno", - ":printf_core_structs", - ":printf_error_mapper", ":types_FILE", - ":vfprintf_internal", ], ) @@ -15566,9 +15566,9 @@ libc_function( ":__support_cpp_limits", ":__support_libc_errno", ":__support_macros_config", - ":printf_core_structs", - ":printf_error_mapper", - ":vasprintf_internal", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_error_mapper", + ":__support_printf_core_vasprintf_internal", ], ) @@ -15581,11 +15581,11 @@ libc_function( ":__support_cpp_limits", ":__support_libc_errno", ":__support_macros_config", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_error_mapper", + ":__support_printf_core_printf_main", + ":__support_printf_core_printf_writer", ":errno", - ":printf_core_structs", - ":printf_error_mapper", - ":printf_main", - ":printf_writer", ], ) @@ -15598,11 +15598,11 @@ libc_function( ":__support_cpp_limits", ":__support_libc_errno", ":__support_macros_config", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_error_mapper", + ":__support_printf_core_printf_main", + ":__support_printf_core_printf_writer", ":errno", - ":printf_core_structs", - ":printf_error_mapper", - ":printf_main", - ":printf_writer", ], ) @@ -15615,12 +15615,12 @@ libc_function( ":__support_cpp_limits", ":__support_file_file", ":__support_macros_config", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_error_mapper", + ":__support_printf_core_vfprintf_internal", ":errno", - ":printf_core_structs", - ":printf_error_mapper", ":stdout", ":types_FILE", - ":vfprintf_internal", ], ) @@ -15633,11 +15633,11 @@ libc_function( ":__support_cpp_limits", ":__support_file_file", ":__support_macros_config", + ":__support_printf_core_printf_core_structs", + ":__support_printf_core_printf_error_mapper", + ":__support_printf_core_vfprintf_internal", ":errno", - ":printf_core_structs", - ":printf_error_mapper", ":types_FILE", - ":vfprintf_internal", ], ) diff --git a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel index 9eacb93043352..ebe367e3d8b6e 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel @@ -152,9 +152,9 @@ libc_test_library( ":string_utils", "//libc:__support_fputil_fp_bits", "//libc:__support_macros_config", + "//libc:__support_printf_core_printf_config", + "//libc:__support_printf_core_printf_core_structs", "//libc:hdr_stdint_proxy", - "//libc:printf_config", - "//libc:printf_core_structs", ], ) diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel index 0698f45ceabc9..d912c2f9e1d1e 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel @@ -121,3 +121,40 @@ libc_test( "//libc:__support_macros_properties_types", ], ) + +libc_test( + name = "printf_parser_test", + srcs = ["printf_core/parser_test.cpp"], + deps = [ + "//libc:__support_arg_list", + "//libc:__support_cpp_bit", + "//libc:__support_cpp_string_view", + "//libc:__support_printf_core_printf_core_structs", + "//libc:__support_printf_core_printf_parser", + "//libc/test/UnitTest:printf_matcher", + ], +) + +libc_test( + name = "printf_writer_test", + srcs = ["printf_core/writer_test.cpp"], + deps = [ + "//libc:__support_arg_list", + "//libc:__support_cpp_string_view", + "//libc:__support_printf_core_printf_core_structs", + "//libc:__support_printf_core_printf_writer", + "//libc:string_memory_utils", + ], +) + +libc_test( + name = "printf_converter_test", + srcs = ["printf_core/converter_test.cpp"], + deps = [ + "//libc:__support_arg_list", + "//libc:__support_cpp_string_view", + "//libc:__support_printf_core_printf_converter", + "//libc:__support_printf_core_printf_core_structs", + "//libc:__support_printf_core_printf_writer", + ], +) diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel index f9d3c4288b1f1..216e51dea58f8 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel @@ -10,43 +10,6 @@ package(default_visibility = ["//visibility:public"]) licenses(["notice"]) -libc_test( - name = "printf_parser_test", - srcs = ["printf_core/parser_test.cpp"], - deps = [ - "//libc:__support_arg_list", - "//libc:__support_cpp_bit", - "//libc:__support_cpp_string_view", - "//libc:printf_core_structs", - "//libc:printf_parser", - "//libc/test/UnitTest:printf_matcher", - ], -) - -libc_test( - name = "printf_writer_test", - srcs = ["printf_core/writer_test.cpp"], - deps = [ - "//libc:__support_arg_list", - "//libc:__support_cpp_string_view", - "//libc:printf_core_structs", - "//libc:printf_writer", - "//libc:string_memory_utils", - ], -) - -libc_test( - name = "printf_converter_test", - srcs = ["printf_core/converter_test.cpp"], - deps = [ - "//libc:__support_arg_list", - "//libc:__support_cpp_string_view", - "//libc:printf_converter", - "//libc:printf_core_structs", - "//libc:printf_writer", - ], -) - libc_test( name = "asprintf_test", srcs = ["asprintf_test.cpp"], @@ -62,8 +25,8 @@ libc_test( srcs = ["sprintf_test.cpp"], deps = [ "//libc:__support_fputil_fp_bits", + "//libc:__support_printf_core_printf_config", "//libc:hdr_wchar_macros", - "//libc:printf_config", "//libc:sprintf", "//libc:types_wint_t", "//libc/test/UnitTest:fp_test_helpers", From 96b100337b5b61e84ee07f6ae9168d3b502e574e Mon Sep 17 00:00:00 2001 From: Prabhu Rajasekaran Date: Tue, 21 Jul 2026 14:38:48 -0700 Subject: [PATCH 37/50] Reland "[Fuchsia] Stack analysis flags for runtimes" (#211072) Reverts llvm/llvm-project#192515 We have addressed the CFI related workflow by introducing a new MD_callgraph and by disambiguating it from MD_type used for CFI. --- clang/cmake/caches/Fuchsia-stage2.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake index 401258e9ae70e..a2c2a83be221b 100644 --- a/clang/cmake/caches/Fuchsia-stage2.cmake +++ b/clang/cmake/caches/Fuchsia-stage2.cmake @@ -205,7 +205,7 @@ if(FUCHSIA_SDK) set(FUCHSIA_x86_64-unknown-fuchsia_NAME x64) set(FUCHSIA_riscv64-unknown-fuchsia_NAME riscv64) foreach(target i386-unknown-fuchsia;x86_64-unknown-fuchsia;aarch64-unknown-fuchsia;arm-unknown-fuchsia;riscv64-unknown-fuchsia) - set(FUCHSIA_${target}_COMPILER_FLAGS "--target=${target} -I${FUCHSIA_SDK}/pkg/sync/include -I${FUCHSIA_SDK}/pkg/fdio/include") + set(FUCHSIA_${target}_COMPILER_FLAGS "--target=${target} -I${FUCHSIA_SDK}/pkg/sync/include -I${FUCHSIA_SDK}/pkg/fdio/include -fstack-size-section -fexperimental-call-graph-section") set(FUCHSIA_${target}_LINKER_FLAGS "-L${FUCHSIA_SDK}/arch/${FUCHSIA_${target}_NAME}/lib") set(FUCHSIA_${target}_SYSROOT "${FUCHSIA_SDK}/arch/${FUCHSIA_${target}_NAME}/sysroot") endforeach() @@ -326,7 +326,7 @@ foreach(target armv6m-none-eabi;armv7m-none-eabi;armv7em-none-eabi;armv8m.main-n if(${target} STREQUAL "armv8.1m.main-none-eabi") set(BUILTINS_${target}_CMAKE_${lang}_local_flags "${BUILTINS_${target}_CMAKE_${lang}_local_flags} -mfloat-abi=hard -march=armv8.1-m.main+mve.fp+fp.dp -mcpu=cortex-m55") endif() - set(BUILTINS_${target}_CMAKE_${lang}_FLAGS "${BUILTINS_${target}_CMAKE_${lang}_local_flags}" CACHE STRING "") + set(BUILTINS_${target}_CMAKE_${lang}_FLAGS "${BUILTINS_${target}_CMAKE_${lang}_local_flags} -fstack-size-section -fexperimental-call-graph-section" CACHE STRING "") endforeach() foreach(type SHARED;MODULE;EXE) set(BUILTINS_${target}_CMAKE_${type}_LINKER_FLAGS "-fuse-ld=lld" CACHE STRING "") @@ -352,7 +352,7 @@ foreach(target armv6m-none-eabi;armv7m-none-eabi;armv7em-none-eabi;armv8m.main-n if(${target} STREQUAL "armv8.1m.main-none-eabi") set(RUNTIMES_${target}_CMAKE_${lang}_local_flags "${RUNTIMES_${target}_CMAKE_${lang}_local_flags} -mfloat-abi=hard -march=armv8.1-m.main+mve.fp+fp.dp -mcpu=cortex-m55") endif() - set(RUNTIMES_${target}_CMAKE_${lang}_FLAGS "${RUNTIMES_${target}_CMAKE_${lang}_local_flags}" CACHE STRING "") + set(RUNTIMES_${target}_CMAKE_${lang}_FLAGS "${RUNTIMES_${target}_CMAKE_${lang}_local_flags} -fstack-size-section -fexperimental-call-graph-section" CACHE STRING "") endforeach() foreach(type SHARED;MODULE;EXE) set(RUNTIMES_${target}_CMAKE_${type}_LINKER_FLAGS "-fuse-ld=lld" CACHE STRING "") @@ -394,7 +394,7 @@ foreach(target riscv32-unknown-elf) set(BUILTINS_${target}_CMAKE_SYSROOT "" CACHE STRING "") set(BUILTINS_${target}_CMAKE_BUILD_TYPE MinSizeRel CACHE STRING "") foreach(lang C;CXX;ASM) - set(BUILTINS_${target}_CMAKE_${lang}_FLAGS "--target=${target} -march=rv32imafc -mabi=ilp32f" CACHE STRING "") + set(BUILTINS_${target}_CMAKE_${lang}_FLAGS "--target=${target} -march=rv32imafc -mabi=ilp32f -fstack-size-section -fexperimental-call-graph-section" CACHE STRING "") endforeach() foreach(type SHARED;MODULE;EXE) set(BUILTINS_${target}_CMAKE_${type}_LINKER_FLAGS "-fuse-ld=lld" CACHE STRING "") @@ -410,7 +410,7 @@ foreach(target riscv32-unknown-elf) foreach(lang C;CXX;ASM) # TODO: The preprocessor defines workaround various issues in libc and libc++ integration. # These should be addressed and removed over time. - set(RUNTIMES_${target}_CMAKE_${lang}_FLAGS "--target=${target} -march=rv32imafc -mabi=ilp32f -Wno-atomic-alignment" CACHE STRING "") + set(RUNTIMES_${target}_CMAKE_${lang}_FLAGS "--target=${target} -march=rv32imafc -mabi=ilp32f -Wno-atomic-alignment -fstack-size-section -fexperimental-call-graph-section" CACHE STRING "") endforeach() foreach(type SHARED;MODULE;EXE) set(RUNTIMES_${target}_CMAKE_${type}_LINKER_FLAGS "-fuse-ld=lld" CACHE STRING "") From 5503126b61dc0524da8cf4f20ba05f82685a960c Mon Sep 17 00:00:00 2001 From: Andy Kaylor Date: Tue, 21 Jul 2026 14:40:33 -0700 Subject: [PATCH 38/50] [CIR] Flatten loop ops with a cleanup region (#210472) This change adds support for CFG flattening of loop operations with cleanup regions. Rather than re-invent the entire branch through cleanup mechanism, I am first rewriting the loop by sinking the condition into the body (as a test + break-if-false) and hoisting the step into the body, and surrounding the entire body with a cir.cleanup.scope op. Then the greedy flattening algorithm runs the newly formed loop through the existing flattening mechanisms so that the loop cleanup handling ends up being done by the same code that flattens other cleanup scopes. Assisted-by: Cursor / various models --- clang/include/clang/CIR/Dialect/IR/CIROps.td | 6 + .../clang/CIR/Interfaces/CIRLoopOpInterface.h | 3 +- .../CIR/Interfaces/CIRLoopOpInterface.td | 11 + .../lib/CIR/Dialect/Transforms/FlattenCFG.cpp | 159 ++++++ .../CIR/Transforms/flatten-loop-cleanup.cir | 457 ++++++++++++++++++ 5 files changed, 635 insertions(+), 1 deletion(-) create mode 100644 clang/test/CIR/Transforms/flatten-loop-cleanup.cir diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td index 0f034a4f1f7fa..5a0922cdb1255 100644 --- a/clang/include/clang/CIR/Dialect/IR/CIROps.td +++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td @@ -2258,6 +2258,9 @@ def CIR_WhileOp : CIR_WhileOpBase<"while"> { mlir::Region *maybeGetCleanup() { return getCleanup().empty() ? nullptr : &getCleanup(); } + cir::CleanupKindAttr maybeGetCleanupKind() { + return getCleanupKindAttr(); + } llvm::SmallVector getRegionsInExecutionOrder() { llvm::SmallVector regions{&getCond(), &getBody()}; if (mlir::Region *cleanup = maybeGetCleanup()) @@ -2386,6 +2389,9 @@ def CIR_ForOp : CIR_LoopOpBase<"for"> { mlir::Region *maybeGetCleanup() { return getCleanup().empty() ? nullptr : &getCleanup(); } + cir::CleanupKindAttr maybeGetCleanupKind() { + return getCleanupKindAttr(); + } llvm::SmallVector getRegionsInExecutionOrder() { llvm::SmallVector regions{&getCond(), &getBody(), &getStep()}; diff --git a/clang/include/clang/CIR/Interfaces/CIRLoopOpInterface.h b/clang/include/clang/CIR/Interfaces/CIRLoopOpInterface.h index 90fd07919dc01..9c55e4ad74156 100644 --- a/clang/include/clang/CIR/Interfaces/CIRLoopOpInterface.h +++ b/clang/include/clang/CIR/Interfaces/CIRLoopOpInterface.h @@ -13,12 +13,13 @@ #ifndef CLANG_CIR_INTERFACES_CIRLOOPOPINTERFACE_H #define CLANG_CIR_INTERFACES_CIRLOOPOPINTERFACE_H -#include "llvm/ADT/APInt.h" #include "mlir/IR/BuiltinTypes.h" #include "mlir/IR/OpDefinition.h" #include "mlir/IR/Operation.h" #include "mlir/Interfaces/ControlFlowInterfaces.h" #include "mlir/Interfaces/LoopLikeInterface.h" +#include "clang/CIR/Dialect/IR/CIRAttrs.h" +#include "llvm/ADT/APInt.h" using llvm::APInt; namespace cir { diff --git a/clang/include/clang/CIR/Interfaces/CIRLoopOpInterface.td b/clang/include/clang/CIR/Interfaces/CIRLoopOpInterface.td index 01b78a2a8b413..ec4896dec36db 100644 --- a/clang/include/clang/CIR/Interfaces/CIRLoopOpInterface.td +++ b/clang/include/clang/CIR/Interfaces/CIRLoopOpInterface.td @@ -58,6 +58,17 @@ def LoopOpInterface : OpInterface<"LoopOpInterface", [ /*methodBody=*/"", /*defaultImplementation=*/"return nullptr;" >, + InterfaceMethod<[{ + Returns the cleanup-kind attribute for the loop's per-iteration + cleanup region, or a null attribute if the loop has no cleanup kind + (e.g. do-while, or a for/while without a cleanup region). + }], + /*retTy=*/"::cir::CleanupKindAttr", + /*methodName=*/"maybeGetCleanupKind", + /*args=*/(ins), + /*methodBody=*/"", + /*defaultImplementation=*/"return {};" + >, InterfaceMethod<[{ Returns the first region to be executed in the loop. }], diff --git a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp index a878199c03d99..96f8d778992d9 100644 --- a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp +++ b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp @@ -518,6 +518,154 @@ class CIRLoopOpInterfaceFlattening exit); } + // Rewrite a loop that has a per-iteration cleanup region into a loop whose + // condition is always 'true' and whose body is a cir.cleanup.scope enclosing + // the original condition, body, and step regions. + // + // The condition test is sunk to the top of the scope body (a false + // result becomes a cir.break out of the loop) and, for a for loop, the step + // is appended to the end of the scope body. The loop's cleanup region becomes + // the a cir.cleanup.scope enclosing the body region. + // + // For example, a while loop: + // + // cir.while { ; cir.condition(%c) } + // do { } + // cleanup all { } + // + // becomes: + // + // cir.while { cir.condition(%true) } do { + // cir.cleanup.scope { + // + // cir.brcond %c ^body, ^cond_false + // ^cond_false: + // cir.break + // ^body: + // + // } cleanup all { } + // cir.yield + // } + // + // A for loop is the same, except the step region is appended to the scope + // body (after the body) and both the body's normal end and any continue are + // redirected into the step so that the step runs before the cleanup. The init + // section of a for loop is already outside the loop op. + mlir::LogicalResult + rewriteLoopWithCleanup(cir::LoopOpInterface op, + mlir::PatternRewriter &rewriter) const { + mlir::Location loc = op.getLoc(); + mlir::Region *stepRegion = op.maybeGetStep(); + + cir::CleanupKindAttr cleanupKind = op.maybeGetCleanupKind(); + assert(cleanupKind && "loop cleanup region without a cleanup kind"); + + mlir::Region &condRegion = op.getCond(); + mlir::Region &bodyRegion = op.getBody(); + mlir::Region &cleanupRegion = *op.maybeGetCleanup(); + + // The cir.condition is the terminator of the condition region's last block + // (there can be more than one block if a nested scope was already + // flattened within the condition). + auto conditionOp = + cast(condRegion.back().getTerminator()); + mlir::Value condVal = conditionOp.getCondition(); + + // Capture block references and exit terminators before moving blocks + // around. Block pointers stay valid across region inlining because blocks + // are reparented, not recreated. + mlir::Block *bodyFront = &bodyRegion.front(); + mlir::Block *stepFront = stepRegion ? &stepRegion->front() : nullptr; + + // For a for loop the body's normal end and any continue must run the step + // before the cleanup, so collect them to redirect into the step region. For + // a while loop they are genuine cleanup-scope exits and are left untouched. + llvm::SmallVector bodyYieldsToStep; + llvm::SmallVector continuesToStep; + if (stepRegion) { + for (mlir::Block &blk : bodyRegion.getBlocks()) + if (auto y = dyn_cast(blk.getTerminator())) + bodyYieldsToStep.push_back(y); + op.walkBodySkippingNestedLoops([&](mlir::Operation *o) { + if (auto c = dyn_cast(o)) { + continuesToStep.push_back(c); + return mlir::WalkResult::skip(); + } + return mlir::WalkResult::advance(); + }); + } + + // Assemble the per-iteration blocks into the condition region, in + // execution order: [ cond blocks..., body blocks..., step blocks... ]. + rewriter.inlineRegionBefore(bodyRegion, condRegion, condRegion.end()); + if (stepRegion) + rewriter.inlineRegionBefore(*stepRegion, condRegion, condRegion.end()); + + // Replace cir.condition(%c) with a conditional branch into the body whose + // false edge breaks out of the loop. + mlir::Block *breakBlock = + rewriter.createBlock(&condRegion, condRegion.end()); + rewriter.setInsertionPointToEnd(breakBlock); + cir::BreakOp::create(rewriter, conditionOp.getLoc()); + + rewriter.setInsertionPoint(conditionOp); + rewriter.replaceOpWithNewOp(conditionOp, condVal, bodyFront, + breakBlock); + + // For a for loop, redirect the body's normal end and any continue to the + // step. The step's own yield remains the normal end of the iteration. + for (cir::YieldOp y : bodyYieldsToStep) + lowerTerminator(y, stepFront, rewriter); + for (cir::ContinueOp c : continuesToStep) + lowerTerminator(c, stepFront, rewriter); + + // Build the trivial loop body: a single cir.cleanup.scope followed by a + // yield, in a fresh block (the body region's blocks were moved out above). + mlir::Block *newBodyBlock = rewriter.createBlock(&bodyRegion); + rewriter.setInsertionPointToEnd(newBodyBlock); + auto emitYield = [](mlir::OpBuilder &b, mlir::Location l) { + cir::YieldOp::create(b, l); + }; + auto scope = cir::CleanupScopeOp::create( + rewriter, loc, cleanupKind.getValue(), emitYield, emitYield); + cir::YieldOp::create(rewriter, loc); + + // Move the assembled per-iteration blocks into the scope's body region and + // the loop's cleanup blocks into the scope's cleanup region, discarding the + // placeholder blocks the builder created. + mlir::Block *bodyPlaceholder = &scope.getBodyRegion().front(); + rewriter.inlineRegionBefore(condRegion, bodyPlaceholder); + rewriter.eraseBlock(bodyPlaceholder); + + mlir::Block *cleanupPlaceholder = &scope.getCleanupRegion().front(); + rewriter.inlineRegionBefore(cleanupRegion, cleanupPlaceholder); + rewriter.eraseBlock(cleanupPlaceholder); + + // Rebuild the loop condition region with an always-true condition. + mlir::Block *newCondBlock = rewriter.createBlock(&condRegion); + rewriter.setInsertionPointToEnd(newCondBlock); + mlir::Value trueVal = cir::ConstantOp::create( + rewriter, loc, cir::BoolAttr::get(rewriter.getContext(), true)); + cir::ConditionOp::create(rewriter, loc, trueVal); + + // For a for loop, rebuild the (now empty) step region with a trivial yield. + if (stepRegion) { + mlir::Block *newStepBlock = rewriter.createBlock(stepRegion); + rewriter.setInsertionPointToEnd(newStepBlock); + cir::YieldOp::create(rewriter, loc); + } + + // Drop the cleanup-kind attribute now that the loop's cleanup region is + // empty, so the trivial loop verifies and takes the no-cleanup flattening + // path when the greedy driver revisits it. + if (auto whileOp = mlir::dyn_cast(op.getOperation())) + whileOp.removeCleanupKindAttr(); + else if (auto forOp = mlir::dyn_cast(op.getOperation())) + forOp.removeCleanupKindAttr(); + + return mlir::success(); + } + mlir::LogicalResult matchAndRewrite(cir::LoopOpInterface op, mlir::PatternRewriter &rewriter) const final { @@ -529,6 +677,17 @@ class CIRLoopOpInterfaceFlattening if (hasNestedOpsToFlatten(region)) return mlir::failure(); + // Loops with a per-iteration cleanup region need every iteration-exit edge + // routed through that cleanup, including the false-condition exit and any + // exceptions that might be thrown from the step region. Rather than trying + // to figure out all of the cleanup routing here, we sink the condition into + // the body region, hoist the step region (if any) and create a new + // cir.cleanup.scope enclosing the body region. Subsequent passes of the + // greedy driver will flatten the cir.cleanup.scope and the loop reusing + // the normal handlers. + if (op.maybeGetCleanup()) + return rewriteLoopWithCleanup(op, rewriter); + // Setup CFG blocks. mlir::Block *entry = rewriter.getInsertionBlock(); mlir::Block *exit = diff --git a/clang/test/CIR/Transforms/flatten-loop-cleanup.cir b/clang/test/CIR/Transforms/flatten-loop-cleanup.cir new file mode 100644 index 0000000000000..40adb54633520 --- /dev/null +++ b/clang/test/CIR/Transforms/flatten-loop-cleanup.cir @@ -0,0 +1,457 @@ +// RUN: cir-opt %s -cir-flatten-cfg -o %t.cir +// RUN: FileCheck --input-file=%t.cir %s + +!u8i = !cir.int +!s32i = !cir.int +!rec_S = !cir.struct<"S" padded {!u8i}> + +module { + cir.func private @ctor(!cir.ptr) + cir.func private @operatorBool(!cir.ptr) -> !cir.bool + cir.func private @dtor(!cir.ptr) func_info<#cir.cxx_dtor> + attributes {nothrow} + cir.func private @doBodyStuff() + cir.func private @doStepStuff() + cir.func private @doBreakStuff() + cir.func private @doContinueStuff() + cir.func private @doReturnStuff() + cir.func private @maythrow() + + // In actual generated CIR, a cleanup-active flag will be used to guard + // against calling the destructor if a constructor in the condition region + // throws an exception. For the sake of simplicity, that behavior is omitted + // throughout this test. This test is focused only on flattening the CFG. + + cir.func @while_cleanup() { + %s = cir.alloca "s" align(1) init : !cir.ptr + cir.while { + cir.call @ctor(%s) : (!cir.ptr) -> () + %c = cir.call @operatorBool(%s) : (!cir.ptr) -> !cir.bool + cir.condition(%c) + } do { + cir.call @doBodyStuff() : () -> () + cir.yield + } cleanup normal { + cir.call @dtor(%s) nothrow : (!cir.ptr) -> () + cir.yield + } + cir.return + } + + // CHECK-LABEL: cir.func @while_cleanup() + // CHECK: %[[SLOT:.*]] = cir.alloca "__cleanup_dest_slot" {{.*}} cleanup_dest_slot : !cir.ptr + // CHECK: %[[S:.*]] = cir.alloca "s" {{.*}} : !cir.ptr + // CHECK: cir.br ^[[HEADER:bb[0-9]+]] + // CHECK: ^[[HEADER]]: + // CHECK: %[[TRUE:.*]] = cir.const #true + // CHECK: cir.brcond %[[TRUE]] ^[[LOOP_BODY:bb[0-9]+]], ^[[EXIT:bb[0-9]+]] + // CHECK: ^[[LOOP_BODY]]: + // CHECK: cir.call @ctor(%[[S]]) + // CHECK: %[[C:.*]] = cir.call @operatorBool(%[[S]]) + // CHECK: cir.brcond %[[C]] ^[[NORMAL:bb[0-9]+]], ^[[CONDFALSE:bb[0-9]+]] + // CHECK: ^[[NORMAL]]: + // CHECK: cir.call @doBodyStuff() + // CHECK: %[[ID0:.*]] = cir.const #cir.int<0> : !s32i + // CHECK: cir.store %[[ID0]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP:bb[0-9]+]] + // CHECK: ^[[CONDFALSE]]: + // CHECK: %[[ID1:.*]] = cir.const #cir.int<1> : !s32i + // CHECK: cir.store %[[ID1]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP]] + // CHECK: ^[[CLEANUP]]: + // CHECK: cir.call @dtor(%[[S]]) + // CHECK: %[[D:.*]] = cir.load %[[SLOT]] + // CHECK: cir.switch.flat %[[D]] : !s32i, ^[[UNREACHABLE:bb[0-9]+]] [ + // CHECK: 0: ^[[NORMAL_CONT:bb[0-9]+]], + // CHECK: 1: ^[[FALSE_CONT:bb[0-9]+]] + // CHECK: ] + // CHECK: ^[[NORMAL_CONT]]: + // CHECK: cir.br ^[[LATCH:bb[0-9]+]] + // CHECK: ^[[FALSE_CONT]]: + // CHECK: cir.br ^[[EXIT]] + // CHECK: ^[[UNREACHABLE]]: + // CHECK: cir.unreachable + // CHECK: ^[[LATCH]]: + // CHECK: cir.br ^[[HEADER]] + // CHECK: ^[[EXIT]]: + // CHECK: cir.return + + cir.func @for_cleanup() { + %s = cir.alloca "s" align(1) init : !cir.ptr + cir.for : cond { + cir.call @ctor(%s) : (!cir.ptr) -> () + %c = cir.call @operatorBool(%s) : (!cir.ptr) -> !cir.bool + cir.condition(%c) + } body { + cir.call @doBodyStuff() : () -> () + cir.yield + } step { + cir.call @doStepStuff() : () -> () + cir.yield + } cleanup normal { + cir.call @dtor(%s) nothrow : (!cir.ptr) -> () + cir.yield + } + cir.return + } + + // CHECK-LABEL: cir.func @for_cleanup() + // CHECK: %[[SLOT:.*]] = cir.alloca "__cleanup_dest_slot" {{.*}} cleanup_dest_slot : !cir.ptr + // CHECK: %[[S:.*]] = cir.alloca "s" {{.*}} : !cir.ptr + // CHECK: cir.br ^[[HEADER:bb[0-9]+]] + // CHECK: ^[[HEADER]]: + // CHECK: %[[TRUE:.*]] = cir.const #true + // CHECK: cir.brcond %[[TRUE]] ^[[LOOP_BODY:bb[0-9]+]], ^[[EXIT:bb[0-9]+]] + // CHECK: ^[[LOOP_BODY]]: + // CHECK: cir.call @ctor(%[[S]]) + // CHECK: %[[C:.*]] = cir.call @operatorBool(%[[S]]) + // CHECK: cir.brcond %[[C]] ^[[BODY:bb[0-9]+]], ^[[CONDFALSE:bb[0-9]+]] + // CHECK: ^[[BODY]]: + // CHECK: cir.call @doBodyStuff() + // CHECK: cir.br ^[[STEP:bb[0-9]+]] + // CHECK: ^[[STEP]]: + // CHECK: cir.call @doStepStuff() + // CHECK: %[[ID0:.*]] = cir.const #cir.int<0> : !s32i + // CHECK: cir.store %[[ID0]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP:bb[0-9]+]] + // CHECK: ^[[CONDFALSE]]: + // CHECK: %[[ID1:.*]] = cir.const #cir.int<1> : !s32i + // CHECK: cir.store %[[ID1]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP]] + // CHECK: ^[[CLEANUP]]: + // CHECK: cir.call @dtor(%[[S]]) + // CHECK: %[[D:.*]] = cir.load %[[SLOT]] + // CHECK: cir.switch.flat %[[D]] : !s32i, ^[[UNREACHABLE:bb[0-9]+]] [ + // CHECK: 0: ^[[STEP_CONT:bb[0-9]+]], + // CHECK: 1: ^[[FALSE_CONT:bb[0-9]+]] + // CHECK: ] + // CHECK: ^[[STEP_CONT]]: + // CHECK: cir.br ^[[FORWARD:bb[0-9]+]] + // CHECK: ^[[FALSE_CONT]]: + // CHECK: cir.br ^[[EXIT]] + // CHECK: ^[[UNREACHABLE]]: + // CHECK: cir.unreachable + // CHECK: ^[[FORWARD]]: + // CHECK: cir.br ^[[LATCH:bb[0-9]+]] + // CHECK: ^[[LATCH]]: + // CHECK: cir.br ^[[HEADER]] + // CHECK: ^[[EXIT]]: + // CHECK: cir.return + + cir.func @while_break_continue_return(%flag : !cir.bool) -> !s32i { + %s = cir.alloca "s" align(1) init : !cir.ptr + cir.while { + cir.call @ctor(%s) : (!cir.ptr) -> () + %c = cir.call @operatorBool(%s) : (!cir.ptr) -> !cir.bool + cir.condition(%c) + } do { + cir.call @doBodyStuff() : () -> () + cir.if %flag { + cir.call @doBreakStuff() : () -> () + cir.break + } + cir.if %flag { + cir.call @doContinueStuff() : () -> () + cir.continue + } + cir.call @doReturnStuff() : () -> () + %r = cir.const #cir.int<42> : !s32i + cir.return %r : !s32i + } cleanup normal { + cir.call @dtor(%s) nothrow : (!cir.ptr) -> () + cir.yield + } + %z = cir.const #cir.int<0> : !s32i + cir.return %z : !s32i + } + + // CHECK-LABEL: cir.func @while_break_continue_return + // CHECK: %[[SLOT:.*]] = cir.alloca "__cleanup_dest_slot" {{.*}} cleanup_dest_slot : !cir.ptr + // CHECK: %[[S:.*]] = cir.alloca "s" {{.*}} : !cir.ptr + // CHECK: cir.br ^[[HEADER:bb[0-9]+]] + // CHECK: ^[[HEADER]]: + // CHECK: %[[TRUE:.*]] = cir.const #true + // CHECK: cir.brcond %[[TRUE]] ^[[LOOP_BODY:bb[0-9]+]], ^[[EXIT:bb[0-9]+]] + // CHECK: ^[[LOOP_BODY]]: + // CHECK: cir.call @ctor(%[[S]]) + // CHECK: %[[C:.*]] = cir.call @operatorBool(%[[S]]) + // CHECK: cir.brcond %[[C]] ^[[BODY:bb[0-9]+]], ^[[CONDFALSE:bb[0-9]+]] + // CHECK: ^[[BODY]]: + // CHECK: cir.call @doBodyStuff() + // CHECK: cir.brcond %arg0 ^[[BREAK:bb[0-9]+]], ^[[CHECK_CONTINUE:bb[0-9]+]] + // CHECK: ^[[BREAK]]: + // CHECK: cir.call @doBreakStuff() + // CHECK: %[[IDB:.*]] = cir.const #cir.int<0> : !s32i + // CHECK: cir.store %[[IDB]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP:bb[0-9]+]] + // CHECK: ^[[CHECK_CONTINUE]]: + // CHECK: cir.brcond %arg0 ^[[CONTINUE:bb[0-9]+]], ^[[RETURN:bb[0-9]+]] + // CHECK: ^[[CONTINUE]]: + // CHECK: cir.call @doContinueStuff() + // CHECK: %[[IDC:.*]] = cir.const #cir.int<1> : !s32i + // CHECK: cir.store %[[IDC]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP]] + // CHECK: ^[[RETURN]]: + // CHECK: cir.call @doReturnStuff() + // CHECK: %[[IDR:.*]] = cir.const #cir.int<2> : !s32i + // CHECK: cir.store %[[IDR]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP]] + // CHECK: ^[[CONDFALSE]]: + // CHECK: %[[IDF:.*]] = cir.const #cir.int<3> : !s32i + // CHECK: cir.store %[[IDF]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP]] + // CHECK: ^[[CLEANUP]]: + // CHECK: cir.call @dtor(%[[S]]) + // CHECK: %[[D:.*]] = cir.load %[[SLOT]] + // CHECK: cir.switch.flat %[[D]] : !s32i, ^[[UNREACHABLE:bb[0-9]+]] [ + // CHECK: 0: ^[[BREAK_CONT:bb[0-9]+]], + // CHECK: 1: ^[[CONTINUE_CONT:bb[0-9]+]], + // CHECK: 2: ^[[RETURN_CONT:bb[0-9]+]], + // CHECK: 3: ^[[FALSE_CONT:bb[0-9]+]] + // CHECK: ] + // CHECK: ^[[BREAK_CONT]]: + // CHECK: cir.br ^[[EXIT]] + // CHECK: ^[[CONTINUE_CONT]]: + // CHECK: cir.br ^[[HEADER]] + // CHECK: ^[[RETURN_CONT]]: + // CHECK: %[[R:.*]] = cir.const #cir.int<42> : !s32i + // CHECK: cir.return %[[R]] : !s32i + // CHECK: ^[[FALSE_CONT]]: + // CHECK: cir.br ^[[EXIT]] + // CHECK: ^[[UNREACHABLE]]: + // CHECK: cir.unreachable + // CHECK: ^[[EXIT]]: + // CHECK: %[[Z:.*]] = cir.const #cir.int<0> : !s32i + // CHECK: cir.return %[[Z]] : !s32i + + cir.func @for_break_continue_return(%flag : !cir.bool) -> !s32i { + %s = cir.alloca "s" align(1) init : !cir.ptr + cir.for : cond { + cir.call @ctor(%s) : (!cir.ptr) -> () + %c = cir.call @operatorBool(%s) : (!cir.ptr) -> !cir.bool + cir.condition(%c) + } body { + cir.call @doBodyStuff() : () -> () + cir.if %flag { + cir.call @doBreakStuff() : () -> () + cir.break + } + cir.if %flag { + cir.call @doContinueStuff() : () -> () + cir.continue + } + cir.call @doReturnStuff() : () -> () + %r = cir.const #cir.int<42> : !s32i + cir.return %r : !s32i + } step { + cir.call @doStepStuff() : () -> () + cir.yield + } cleanup normal { + cir.call @dtor(%s) nothrow : (!cir.ptr) -> () + cir.yield + } + %z = cir.const #cir.int<0> : !s32i + cir.return %z : !s32i + } + + // CHECK-LABEL: cir.func @for_break_continue_return + // CHECK: %[[SLOT:.*]] = cir.alloca "__cleanup_dest_slot" {{.*}} cleanup_dest_slot : !cir.ptr + // CHECK: %[[S:.*]] = cir.alloca "s" {{.*}} : !cir.ptr + // CHECK: cir.br ^[[HEADER:bb[0-9]+]] + // CHECK: ^[[HEADER]]: + // CHECK: %[[TRUE:.*]] = cir.const #true + // CHECK: cir.brcond %[[TRUE]] ^[[LOOP_BODY:bb[0-9]+]], ^[[EXIT:bb[0-9]+]] + // CHECK: ^[[LOOP_BODY]]: + // CHECK: cir.call @ctor(%[[S]]) + // CHECK: %[[C:.*]] = cir.call @operatorBool(%[[S]]) + // CHECK: cir.brcond %[[C]] ^[[BODY:bb[0-9]+]], ^[[CONDFALSE:bb[0-9]+]] + // CHECK: ^[[BODY]]: + // CHECK: cir.call @doBodyStuff() + // CHECK: cir.brcond %arg0 ^[[BREAK:bb[0-9]+]], ^[[CHECK_CONTINUE:bb[0-9]+]] + // CHECK: ^[[BREAK]]: + // CHECK: cir.call @doBreakStuff() + // CHECK: %[[IDB:.*]] = cir.const #cir.int<1> : !s32i + // CHECK: cir.store %[[IDB]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP:bb[0-9]+]] + // CHECK: ^[[CHECK_CONTINUE]]: + // CHECK: cir.brcond %arg0 ^[[CONTINUE:bb[0-9]+]], ^[[RETURN:bb[0-9]+]] + // CHECK: ^[[CONTINUE]]: + // CHECK: cir.call @doContinueStuff() + // CHECK: cir.br ^[[STEP:bb[0-9]+]] + // CHECK: ^[[RETURN]]: + // CHECK: cir.call @doReturnStuff() + // CHECK: %[[IDR:.*]] = cir.const #cir.int<2> : !s32i + // CHECK: cir.store %[[IDR]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP]] + // CHECK: ^[[STEP]]: + // CHECK: cir.call @doStepStuff() + // CHECK: %[[IDS:.*]] = cir.const #cir.int<0> : !s32i + // CHECK: cir.store %[[IDS]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP]] + // CHECK: ^[[CONDFALSE]]: + // CHECK: %[[IDF:.*]] = cir.const #cir.int<3> : !s32i + // CHECK: cir.store %[[IDF]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP]] + // CHECK: ^[[CLEANUP]]: + // CHECK: cir.call @dtor(%[[S]]) + // CHECK: %[[D:.*]] = cir.load %[[SLOT]] + // CHECK: cir.switch.flat %[[D]] : !s32i, ^[[UNREACHABLE:bb[0-9]+]] [ + // CHECK: 0: ^[[STEP_CONT:bb[0-9]+]], + // CHECK: 1: ^[[BREAK_CONT:bb[0-9]+]], + // CHECK: 2: ^[[RETURN_CONT:bb[0-9]+]], + // CHECK: 3: ^[[FALSE_CONT:bb[0-9]+]] + // CHECK: ] + // CHECK: ^[[STEP_CONT]]: + // CHECK: cir.br ^[[FORWARD:bb[0-9]+]] + // CHECK: ^[[BREAK_CONT]]: + // CHECK: cir.br ^[[EXIT]] + // CHECK: ^[[RETURN_CONT]]: + // CHECK: %[[R:.*]] = cir.const #cir.int<42> : !s32i + // CHECK: cir.return %[[R]] : !s32i + // CHECK: ^[[FALSE_CONT]]: + // CHECK: cir.br ^[[EXIT]] + // CHECK: ^[[UNREACHABLE]]: + // CHECK: cir.unreachable + // CHECK: ^[[FORWARD]]: + // CHECK: cir.br ^[[LATCH:bb[0-9]+]] + // CHECK: ^[[LATCH]]: + // CHECK: cir.br ^[[HEADER]] + // CHECK: ^[[EXIT]]: + // CHECK: %[[Z:.*]] = cir.const #cir.int<0> : !s32i + // CHECK: cir.return %[[Z]] : !s32i + + cir.func @while_cleanup_eh() { + %s = cir.alloca "s" align(1) init : !cir.ptr + cir.while { + cir.call @ctor(%s) nothrow : (!cir.ptr) -> () + %c = cir.call @operatorBool(%s) : (!cir.ptr) -> !cir.bool + cir.condition(%c) + } do { + cir.call @maythrow() : () -> () + cir.yield + } cleanup all { + cir.call @dtor(%s) nothrow : (!cir.ptr) -> () + cir.yield + } + cir.return + } + + // CHECK-LABEL: cir.func @while_cleanup_eh() + // CHECK: %[[SLOT:.*]] = cir.alloca "__cleanup_dest_slot" {{.*}} cleanup_dest_slot : !cir.ptr + // CHECK: %[[S:.*]] = cir.alloca "s" {{.*}} : !cir.ptr + // CHECK: cir.br ^[[HEADER:bb[0-9]+]] + // CHECK: ^[[HEADER]]: + // CHECK: %[[TRUE:.*]] = cir.const #true + // CHECK: cir.brcond %[[TRUE]] ^[[LOOP_BODY:bb[0-9]+]], ^[[EXIT:bb[0-9]+]] + // CHECK: ^[[LOOP_BODY]]: + // CHECK: cir.call @ctor(%[[S]]) + // CHECK: %[[C:.*]] = cir.try_call @operatorBool(%[[S]]) ^[[COND_TEST:bb[0-9]+]], ^[[UNWIND:bb[0-9]+]] + // CHECK: ^[[COND_TEST]]: + // CHECK: cir.brcond %[[C]] ^[[BODY:bb[0-9]+]], ^[[CONDFALSE:bb[0-9]+]] + // CHECK: ^[[BODY]]: + // CHECK: cir.try_call @maythrow() ^[[NORMAL:bb[0-9]+]], ^[[UNWIND]] + // CHECK: ^[[NORMAL]]: + // CHECK: %[[ID0:.*]] = cir.const #cir.int<0> : !s32i + // CHECK: cir.store %[[ID0]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP:bb[0-9]+]] + // CHECK: ^[[CONDFALSE]]: + // CHECK: %[[ID1:.*]] = cir.const #cir.int<1> : !s32i + // CHECK: cir.store %[[ID1]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP]] + // CHECK: ^[[CLEANUP]]: + // CHECK: cir.call @dtor(%[[S]]) + // CHECK: %[[D:.*]] = cir.load %[[SLOT]] + // CHECK: cir.switch.flat %[[D]] : !s32i, ^[[UNREACHABLE:bb[0-9]+]] [ + // CHECK: 0: ^[[BODY_CONT:bb[0-9]+]], + // CHECK: 1: ^[[FALSE_CONT:bb[0-9]+]] + // CHECK: ] + // CHECK: ^[[BODY_CONT]]: + // CHECK: cir.br ^[[LATCH:bb[0-9]+]] + // CHECK: ^[[FALSE_CONT]]: + // CHECK: cir.br ^[[EXIT]] + // CHECK: ^[[UNREACHABLE]]: + // CHECK: cir.unreachable + // CHECK: ^[[UNWIND]]: + // CHECK: %[[TOK:.*]] = cir.eh.initiate cleanup : !cir.eh_token + // CHECK: cir.br ^[[EHCLEANUP:bb[0-9]+]](%[[TOK]] : !cir.eh_token) + // CHECK: ^[[EHCLEANUP]](%[[TOK2:.*]]: !cir.eh_token): + // CHECK: %[[CT:.*]] = cir.begin_cleanup %[[TOK2]] : !cir.eh_token -> !cir.cleanup_token + // CHECK: cir.call @dtor(%[[S]]) + // CHECK: cir.end_cleanup %[[CT]] : !cir.cleanup_token + // CHECK: cir.resume %[[TOK2]] : !cir.eh_token + // CHECK: ^[[LATCH]]: + // CHECK: cir.br ^[[HEADER]] + // CHECK: ^[[EXIT]]: + // CHECK: cir.return + + cir.func @for_cleanup_eh() { + %s = cir.alloca "s" align(1) init : !cir.ptr + cir.for : cond { + cir.call @ctor(%s) nothrow : (!cir.ptr) -> () + %c = cir.call @operatorBool(%s) : (!cir.ptr) -> !cir.bool + cir.condition(%c) + } body { + cir.call @maythrow() : () -> () + cir.yield + } step { + cir.call @doStepStuff() : () -> () + cir.yield + } cleanup all { + cir.call @dtor(%s) nothrow : (!cir.ptr) -> () + cir.yield + } + cir.return + } + + // CHECK-LABEL: cir.func @for_cleanup_eh() + // CHECK: %[[SLOT:.*]] = cir.alloca "__cleanup_dest_slot" {{.*}} cleanup_dest_slot : !cir.ptr + // CHECK: %[[S:.*]] = cir.alloca "s" {{.*}} : !cir.ptr + // CHECK: cir.br ^[[HEADER:bb[0-9]+]] + // CHECK: ^[[HEADER]]: + // CHECK: %[[TRUE:.*]] = cir.const #true + // CHECK: cir.brcond %[[TRUE]] ^[[LOOP_BODY:bb[0-9]+]], ^[[EXIT:bb[0-9]+]] + // CHECK: ^[[LOOP_BODY]]: + // CHECK: cir.call @ctor(%[[S]]) + // CHECK: %[[C:.*]] = cir.try_call @operatorBool(%[[S]]) ^[[COND_TEST:bb[0-9]+]], ^[[UNWIND:bb[0-9]+]] + // CHECK: ^[[COND_TEST]]: + // CHECK: cir.brcond %[[C]] ^[[BODY:bb[0-9]+]], ^[[CONDFALSE:bb[0-9]+]] + // CHECK: ^[[BODY]]: + // CHECK: cir.try_call @maythrow() ^[[NORMAL:bb[0-9]+]], ^[[UNWIND]] + // CHECK: ^[[NORMAL]]: + // CHECK: cir.try_call @doStepStuff() ^[[STEP_NORMAL:bb[0-9]+]], ^[[UNWIND]] + // CHECK: ^[[STEP_NORMAL]]: + // CHECK: %[[ID0:.*]] = cir.const #cir.int<0> : !s32i + // CHECK: cir.store %[[ID0]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP:bb[0-9]+]] + // CHECK: ^[[CONDFALSE]]: + // CHECK: %[[ID1:.*]] = cir.const #cir.int<1> : !s32i + // CHECK: cir.store %[[ID1]], %[[SLOT]] + // CHECK: cir.br ^[[CLEANUP]] + // CHECK: ^[[CLEANUP]]: + // CHECK: cir.call @dtor(%[[S]]) + // CHECK: %[[D:.*]] = cir.load %[[SLOT]] + // CHECK: cir.switch.flat %[[D]] : !s32i, ^[[UNREACHABLE:bb[0-9]+]] [ + // CHECK: 0: ^[[STEP_CONT:bb[0-9]+]], + // CHECK: 1: ^[[FALSE_CONT:bb[0-9]+]] + // CHECK: ] + // CHECK: ^[[STEP_CONT]]: + // CHECK: cir.br ^[[FORWARD:bb[0-9]+]] + // CHECK: ^[[FALSE_CONT]]: + // CHECK: cir.br ^[[EXIT]] + // CHECK: ^[[UNREACHABLE]]: + // CHECK: cir.unreachable + // CHECK: ^[[UNWIND]]: + // CHECK: %[[TOK:.*]] = cir.eh.initiate cleanup : !cir.eh_token + // CHECK: cir.br ^[[EHCLEANUP:bb[0-9]+]](%[[TOK]] : !cir.eh_token) + // CHECK: ^[[EHCLEANUP]](%[[TOK2:.*]]: !cir.eh_token): + // CHECK: %[[CT:.*]] = cir.begin_cleanup %[[TOK2]] : !cir.eh_token -> !cir.cleanup_token + // CHECK: cir.call @dtor(%[[S]]) + // CHECK: cir.end_cleanup %[[CT]] : !cir.cleanup_token + // CHECK: cir.resume %[[TOK2]] : !cir.eh_token + // CHECK: ^[[FORWARD]]: + // CHECK: cir.br ^[[LATCH:bb[0-9]+]] + // CHECK: ^[[LATCH]]: + // CHECK: cir.br ^[[HEADER]] + // CHECK: ^[[EXIT]]: + // CHECK: cir.return +} From 8bbaac66535c6b925dab34fb05f00505feac11d7 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Wed, 22 Jul 2026 00:02:12 +0200 Subject: [PATCH 39/50] TargetParser: Add Triple::getDefaultFloatABI (#210899) In order to eliminate TargetOptions ABI fields the front and middle end need to know what value the backend is going to choose for the ABI properties. This is similar to how we have getDefaultExceptionHandling and getDefaultWCharSize. The clang driver seems to have a different notion of which targets are default soft. This followed ARMTargetMachine as the authority. Co-authored-by: Claude (Claude-Opus-4.8) --- llvm/include/llvm/TargetParser/Triple.h | 14 ++-- llvm/lib/Target/ARM/ARMTargetMachine.h | 10 +-- llvm/lib/TargetParser/Triple.cpp | 25 ++++++++ llvm/unittests/TargetParser/TripleTest.cpp | 74 ++++++++++++++++++++++ 4 files changed, 109 insertions(+), 14 deletions(-) diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h index d819c4975b13e..141926473c9eb 100644 --- a/llvm/include/llvm/TargetParser/Triple.h +++ b/llvm/include/llvm/TargetParser/Triple.h @@ -10,6 +10,7 @@ #define LLVM_TARGETPARSER_TRIPLE_H #include "llvm/ADT/StringRef.h" +#include "llvm/Support/CodeGen.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/VersionTuple.h" @@ -1249,13 +1250,12 @@ class Triple { Env == Triple::GNUEABIHFT64; } - /// Tests if the target forces hardfloat. - bool isHardFloatABI() const { - EnvironmentType Env = getEnvironment(); - return Env == llvm::Triple::GNUEABIHF || - Env == llvm::Triple::GNUEABIHFT64 || - Env == llvm::Triple::MuslEABIHF || Env == llvm::Triple::EABIHF; - } + /// Returns the default floating-point ABI for this target triple, i.e. the + /// ABI the code generator will resolve FloatABI::Default to + LLVM_ABI FloatABI::ABIType getDefaultFloatABI() const; + + /// Tests if the target's default floating-point ABI is hard float. + bool isHardFloatABI() const { return getDefaultFloatABI() == FloatABI::Hard; } /// Tests whether the target supports comdat bool supportsCOMDAT() const { diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.h b/llvm/lib/Target/ARM/ARMTargetMachine.h index 109d6fd5e6671..689ff67d668be 100644 --- a/llvm/lib/Target/ARM/ARMTargetMachine.h +++ b/llvm/lib/Target/ARM/ARMTargetMachine.h @@ -79,13 +79,9 @@ class ARMBaseTargetMachine : public CodeGenTargetMachineImpl { } bool isTargetHardFloat() const { - return TargetTriple.getEnvironment() == Triple::GNUEABIHF || - TargetTriple.getEnvironment() == Triple::GNUEABIHFT64 || - TargetTriple.getEnvironment() == Triple::MuslEABIHF || - TargetTriple.getEnvironment() == Triple::EABIHF || - (TargetTriple.isOSBinFormatMachO() && - TargetTriple.getSubArch() == Triple::ARMSubArch_v7em) || - TargetTriple.isOSWindows() || TargetABI == ARM::ARM_ABI_AAPCS16; + // -target-abi=aapcs16 overrides the triple default. + return TargetABI == ARM::ARM_ABI_AAPCS16 || + TargetTriple.getDefaultFloatABI() == FloatABI::Hard; } bool targetSchedulesPostRAScheduling() const override { return true; }; diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp index 356782c1f6ad4..333cd0d6d68aa 100644 --- a/llvm/lib/TargetParser/Triple.cpp +++ b/llvm/lib/TargetParser/Triple.cpp @@ -2556,6 +2556,31 @@ ExceptionHandling Triple::getDefaultExceptionHandling() const { return ExceptionHandling::None; } +static FloatABI::ABIType getARMDefaultFloatABI(const Triple &T) { + Triple::EnvironmentType Env = T.getEnvironment(); + bool IsHard = + Env == Triple::GNUEABIHF || Env == Triple::GNUEABIHFT64 || + Env == Triple::MuslEABIHF || Env == Triple::EABIHF || + (T.isOSBinFormatMachO() && T.getSubArch() == Triple::ARMSubArch_v7em) || + T.isOSWindows() || ARM::computeTargetABI(T, "") == ARM::ARM_ABI_AAPCS16; + return IsHard ? FloatABI::Hard : FloatABI::Soft; +} + +FloatABI::ABIType Triple::getDefaultFloatABI() const { + if (isARM() || isThumb()) + return getARMDefaultFloatABI(*this); + + // MIPS defaults to hard float, except on FreeBSD which uses soft float. + if (isMIPS()) + return isOSFreeBSD() ? FloatABI::Soft : FloatABI::Hard; + + if (isCSKY() || isAVR() || getArch() == msp430) + return FloatABI::Soft; + + // Most targets use hard float unless soft float is explicitly requested. + return FloatABI::Hard; +} + // HLSL triple environment orders are relied on in the front end static_assert(Triple::Vertex - Triple::Pixel == 1, "incorrect HLSL stage order"); diff --git a/llvm/unittests/TargetParser/TripleTest.cpp b/llvm/unittests/TargetParser/TripleTest.cpp index cfe3bc6abf72f..c10d685c0668f 100644 --- a/llvm/unittests/TargetParser/TripleTest.cpp +++ b/llvm/unittests/TargetParser/TripleTest.cpp @@ -1611,6 +1611,80 @@ static std::string Join(StringRef A, StringRef B, StringRef C, StringRef D) { return Str; } +TEST(TripleTest, DefaultFloatABI) { + EXPECT_EQ(FloatABI::Hard, + Triple("arm-none-none-eabihf").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Hard, + Triple("arm-none-linux-gnueabihf").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Hard, + Triple("arm-none-linux-gnueabihft64").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Hard, + Triple("arm-none-linux-musleabihf").getDefaultFloatABI()); + + // Non-hard EABI environments are soft, regardless of OS. + EXPECT_EQ(FloatABI::Soft, Triple("arm-none-none-eabi").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Soft, + Triple("arm-none-linux-gnueabi").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Soft, + Triple("arm-none-linux-musleabi").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Soft, + Triple("arm-none-linux-android").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Soft, Triple("armv7-apple-ios").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Soft, Triple("armv7-apple-macosx").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Soft, + Triple("armv7-unknown-fuchsia").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Soft, Triple("arm-unknown-openbsd").getDefaultFloatABI()); + + // MachO M-profile v7em is hard. + EXPECT_EQ(FloatABI::Hard, + Triple("thumbv7em-apple-darwin").getDefaultFloatABI()); + + // Windows is hard. + EXPECT_EQ(FloatABI::Hard, + Triple("thumbv7-unknown-windows-msvc").getDefaultFloatABI()); + + // The AAPCS16 (watchOS) ABI is hard. + EXPECT_EQ(FloatABI::Hard, + Triple("thumbv7k-apple-watchos").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Hard, Triple("thumbv7k-apple-ios").getDefaultFloatABI()); + + // The Thumb arch goes through the same ARM logic. + EXPECT_EQ(FloatABI::Hard, + Triple("thumbv7-none-linux-gnueabihf").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Soft, + Triple("thumbv7-none-linux-gnueabi").getDefaultFloatABI()); + + // PowerPC, SystemZ and SPARC default to hard float. + EXPECT_EQ(FloatABI::Hard, + Triple("powerpc64le-unknown-linux-gnu").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Hard, + Triple("s390x-unknown-linux-gnu").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Hard, + Triple("sparc-unknown-linux-gnu").getDefaultFloatABI()); + + // MIPS defaults to hard float, except on FreeBSD which uses soft float. + EXPECT_EQ(FloatABI::Hard, + Triple("mips-unknown-linux-gnu").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Soft, + Triple("mips-unknown-freebsd").getDefaultFloatABI()); + + // Defaults to soft float. + EXPECT_EQ(FloatABI::Soft, Triple("avr-unknown-unknown").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Soft, + Triple("csky-unknown-linux-gnu").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Soft, + Triple("msp430-unknown-unknown").getDefaultFloatABI()); + + // Targets without a special case default to hard float. + EXPECT_EQ(FloatABI::Hard, + Triple("x86_64-unknown-linux-gnu").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Hard, + Triple("aarch64-unknown-linux-gnu").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Hard, + Triple("riscv64-unknown-linux-gnu").getDefaultFloatABI()); + EXPECT_EQ(FloatABI::Hard, Triple("amdgpu-amd-amdhsa").getDefaultFloatABI()); +} + TEST(TripleTest, Normalization) { EXPECT_EQ("unknown", Triple::normalize("")); From 8775feb5da4de07689b125702265321048ee13d7 Mon Sep 17 00:00:00 2001 From: Daniil Kovalev Date: Wed, 22 Jul 2026 01:06:06 +0300 Subject: [PATCH 40/50] [clang] Drop stale `*-buildattr` module flag emission (#211021) Module flags `sign-return-address-buildattr` and `tag-stack-memory-buildattr` are never exercised elsewhere by existing tests or code across the whole LLVM source tree, and there are no known cases of them being used. Moreover, the `sign-return-address-buildattr` flag emission was gated by `ptrauth` target feature which does not actually exist: see `AArch64TargetInfo::computeFeatureLookup()` and `ARMTargetInfo::hasFeature(StringRef Feature)`. So, it was never emitted. This patch removes code responsible for these stale module flags emission. Note: the flags was previously introduced in https://reviews.llvm.org/D112421 --- clang/lib/CodeGen/CodeGenModule.cpp | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 8ce4063a3b780..817b2b5e6a69d 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -1517,24 +1517,6 @@ void CodeGenModule::Release() { if (CodeGenOpts.IndirectBranchCSPrefix) getModule().addModuleFlag(llvm::Module::Override, "indirect_branch_cs_prefix", 1); - // Add module metadata for return address signing (ignoring - // non-leaf/all) and stack tagging. These are actually turned on by function - // attributes, but we use module metadata to emit build attributes. This is - // needed for LTO, where the function attributes are inside bitcode - // serialised into a global variable by the time build attributes are - // emitted, so we can't access them. LTO objects could be compiled with - // different flags therefore module flags are set to "Min" behavior to achieve - // the same end result of the normal build where e.g BTI is off if any object - // doesn't support it. - if (Context.getTargetInfo().hasFeature("ptrauth") && - LangOpts.getSignReturnAddressScope() != - LangOptions::SignReturnAddressScopeKind::None) - getModule().addModuleFlag(llvm::Module::Override, - "sign-return-address-buildattr", 1); - if (LangOpts.Sanitize.has(SanitizerKind::MemtagStack)) - getModule().addModuleFlag(llvm::Module::Override, - "tag-stack-memory-buildattr", 1); - if (T.isARM() || T.isThumb() || T.isAArch64()) { // Previously 1 is used and meant for the backed to derive the function // attribute form it. 2 now means function attributes already set for all From db4d75e13ddc64cfb3a2f3620b622b8c4376812d Mon Sep 17 00:00:00 2001 From: Alexey Gerenkov Date: Wed, 22 Jul 2026 01:06:25 +0300 Subject: [PATCH 41/50] [Xtensa] Add fp16 conversion support (#208206) Close https://github.com/llvm/llvm-project/issues/207505 --- llvm/lib/Target/Xtensa/XtensaISelLowering.cpp | 11 ++ llvm/test/CodeGen/Xtensa/fp16.ll | 159 ++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 llvm/test/CodeGen/Xtensa/fp16.ll diff --git a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp index 45409d405df6d..0fa9eea5af86c 100644 --- a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp +++ b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp @@ -244,8 +244,19 @@ XtensaTargetLowering::XtensaTargetLowering(const TargetMachine &TM, setOperationAction(ISD::FP_TO_SINT, MVT::i32, Expand); } + for (MVT VT : MVT::fp_valuetypes()) { + setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); + } + + setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); + setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand); + setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand); + setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand); + // Floating-point truncation and stores need to be done separately. setTruncStoreAction(MVT::f64, MVT::f32, Expand); + setTruncStoreAction(MVT::f64, MVT::f16, Expand); + setTruncStoreAction(MVT::f32, MVT::f16, Expand); if (Subtarget.hasS32C1I()) { setMaxAtomicSizeInBitsSupported(32); diff --git a/llvm/test/CodeGen/Xtensa/fp16.ll b/llvm/test/CodeGen/Xtensa/fp16.ll new file mode 100644 index 0000000000000..6ebc18ac8df08 --- /dev/null +++ b/llvm/test/CodeGen/Xtensa/fp16.ll @@ -0,0 +1,159 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 +; RUN: llc -mtriple=xtensa -mcpu=esp32 < %s | FileCheck --check-prefix=CHECK-ESP32 %s +; RUN: llc -mtriple=xtensa -mcpu=esp32s2 < %s | FileCheck --check-prefix=CHECK-ESP32S2 %s + +target datalayout = "e-m:e-p:32:32-v1:8:8-i64:64-i128:128-n32" +target triple = "xtensa" + +@x = global i16 12902 +@y = global i16 0 +@z = common global i16 0 + +define void @foo() nounwind { +; CHECK-ESP32-LABEL: foo: +; CHECK-ESP32: # %bb.0: # %entry +; CHECK-ESP32-NEXT: entry a1, 32 +; CHECK-ESP32-NEXT: l32r a6, .LCPI0_0 +; CHECK-ESP32-NEXT: l16ui a10, a6, 0 +; CHECK-ESP32-NEXT: l32r a8, .LCPI0_1 +; CHECK-ESP32-NEXT: callx8 a8 +; CHECK-ESP32-NEXT: or a7, a10, a10 +; CHECK-ESP32-NEXT: l32r a8, .LCPI0_2 +; CHECK-ESP32-NEXT: l16ui a10, a8, 0 +; CHECK-ESP32-NEXT: l32r a8, .LCPI0_3 +; CHECK-ESP32-NEXT: callx8 a8 +; CHECK-ESP32-NEXT: wfr f8, a10 +; CHECK-ESP32-NEXT: wfr f9, a7 +; CHECK-ESP32-NEXT: add.s f8, f9, f8 +; CHECK-ESP32-NEXT: rfr a10, f8 +; CHECK-ESP32-NEXT: l32r a8, .LCPI0_4 +; CHECK-ESP32-NEXT: callx8 a8 +; CHECK-ESP32-NEXT: s16i a10, a6, 0 +; CHECK-ESP32-NEXT: retw.n +; +; CHECK-ESP32S2-LABEL: foo: +; CHECK-ESP32S2: # %bb.0: # %entry +; CHECK-ESP32S2-NEXT: entry a1, 32 +; CHECK-ESP32S2-NEXT: l32r a6, .LCPI0_0 +; CHECK-ESP32S2-NEXT: l16ui a10, a6, 0 +; CHECK-ESP32S2-NEXT: l32r a8, .LCPI0_1 +; CHECK-ESP32S2-NEXT: callx8 a8 +; CHECK-ESP32S2-NEXT: or a7, a10, a10 +; CHECK-ESP32S2-NEXT: l32r a8, .LCPI0_2 +; CHECK-ESP32S2-NEXT: l16ui a10, a8, 0 +; CHECK-ESP32S2-NEXT: l32r a8, .LCPI0_3 +; CHECK-ESP32S2-NEXT: callx8 a8 +; CHECK-ESP32S2-NEXT: or a11, a10, a10 +; CHECK-ESP32S2-NEXT: l32r a8, .LCPI0_4 +; CHECK-ESP32S2-NEXT: or a10, a7, a7 +; CHECK-ESP32S2-NEXT: callx8 a8 +; CHECK-ESP32S2-NEXT: l32r a8, .LCPI0_5 +; CHECK-ESP32S2-NEXT: callx8 a8 +; CHECK-ESP32S2-NEXT: s16i a10, a6, 0 +; CHECK-ESP32S2-NEXT: retw.n +entry: + %fp16_v1 = load i16, ptr @x, align 2 + %fp16_v2 = load i16, ptr @y, align 2 + %f32_v1 = tail call float @llvm.convert.from.fp16.f32(i16 %fp16_v1) + %f32_v2 = tail call float @llvm.convert.from.fp16.f32(i16 %fp16_v2) + %f32_res = fadd float %f32_v1, %f32_v2 + %fp16_res = tail call i16 @llvm.convert.to.fp16.f32(float %f32_res) + store i16 %fp16_res, ptr @x, align 2 + ret void +} + +define double @test_from_fp16(i16 %in) { +; CHECK-ESP32-LABEL: test_from_fp16: +; CHECK-ESP32: .cfi_startproc +; CHECK-ESP32-NEXT: # %bb.0: +; CHECK-ESP32-NEXT: entry a1, 32 +; CHECK-ESP32-NEXT: .cfi_def_cfa_offset 32 +; CHECK-ESP32-NEXT: or a10, a2, a2 +; CHECK-ESP32-NEXT: l32r a8, .LCPI1_0 +; CHECK-ESP32-NEXT: callx8 a8 +; CHECK-ESP32-NEXT: l32r a8, .LCPI1_1 +; CHECK-ESP32-NEXT: callx8 a8 +; CHECK-ESP32-NEXT: or a2, a10, a10 +; CHECK-ESP32-NEXT: or a3, a11, a11 +; CHECK-ESP32-NEXT: retw.n +; +; CHECK-ESP32S2-LABEL: test_from_fp16: +; CHECK-ESP32S2: .cfi_startproc +; CHECK-ESP32S2-NEXT: # %bb.0: +; CHECK-ESP32S2-NEXT: entry a1, 32 +; CHECK-ESP32S2-NEXT: .cfi_def_cfa_offset 32 +; CHECK-ESP32S2-NEXT: l32r a8, .LCPI1_0 +; CHECK-ESP32S2-NEXT: and a10, a2, a8 +; CHECK-ESP32S2-NEXT: l32r a8, .LCPI1_1 +; CHECK-ESP32S2-NEXT: callx8 a8 +; CHECK-ESP32S2-NEXT: l32r a8, .LCPI1_2 +; CHECK-ESP32S2-NEXT: callx8 a8 +; CHECK-ESP32S2-NEXT: or a2, a10, a10 +; CHECK-ESP32S2-NEXT: or a3, a11, a11 +; CHECK-ESP32S2-NEXT: retw.n + %val = call double @llvm.convert.from.fp16.f64(i16 %in) + ret double %val +} + +define i16 @test_to_fp16(double %in) { +; CHECK-ESP32-LABEL: test_to_fp16: +; CHECK-ESP32: .cfi_startproc +; CHECK-ESP32-NEXT: # %bb.0: +; CHECK-ESP32-NEXT: entry a1, 32 +; CHECK-ESP32-NEXT: .cfi_def_cfa_offset 32 +; CHECK-ESP32-NEXT: or a11, a3, a3 +; CHECK-ESP32-NEXT: or a10, a2, a2 +; CHECK-ESP32-NEXT: l32r a8, .LCPI2_0 +; CHECK-ESP32-NEXT: callx8 a8 +; CHECK-ESP32-NEXT: or a2, a10, a10 +; CHECK-ESP32-NEXT: retw.n +; +; CHECK-ESP32S2-LABEL: test_to_fp16: +; CHECK-ESP32S2: .cfi_startproc +; CHECK-ESP32S2-NEXT: # %bb.0: +; CHECK-ESP32S2-NEXT: entry a1, 32 +; CHECK-ESP32S2-NEXT: .cfi_def_cfa_offset 32 +; CHECK-ESP32S2-NEXT: or a11, a3, a3 +; CHECK-ESP32S2-NEXT: or a10, a2, a2 +; CHECK-ESP32S2-NEXT: l32r a8, .LCPI2_0 +; CHECK-ESP32S2-NEXT: callx8 a8 +; CHECK-ESP32S2-NEXT: or a2, a10, a10 +; CHECK-ESP32S2-NEXT: retw.n + %val = call i16 @llvm.convert.to.fp16.f64(double %in) + ret i16 %val +} + +; Function Attrs: nounwind +define dso_local float @cvt(half %a) unnamed_addr #0 { +; CHECK-ESP32-LABEL: cvt: +; CHECK-ESP32: .cfi_startproc +; CHECK-ESP32-NEXT: # %bb.0: # %start +; CHECK-ESP32-NEXT: entry a1, 32 +; CHECK-ESP32-NEXT: .cfi_def_cfa_offset 32 +; CHECK-ESP32-NEXT: or a10, a2, a2 +; CHECK-ESP32-NEXT: l32r a8, .LCPI3_0 +; CHECK-ESP32-NEXT: callx8 a8 +; CHECK-ESP32-NEXT: or a2, a10, a10 +; CHECK-ESP32-NEXT: retw.n +; +; CHECK-ESP32S2-LABEL: cvt: +; CHECK-ESP32S2: .cfi_startproc +; CHECK-ESP32S2-NEXT: # %bb.0: # %start +; CHECK-ESP32S2-NEXT: entry a1, 32 +; CHECK-ESP32S2-NEXT: .cfi_def_cfa_offset 32 +; CHECK-ESP32S2-NEXT: l32r a8, .LCPI3_0 +; CHECK-ESP32S2-NEXT: and a10, a2, a8 +; CHECK-ESP32S2-NEXT: l32r a8, .LCPI3_1 +; CHECK-ESP32S2-NEXT: callx8 a8 +; CHECK-ESP32S2-NEXT: or a2, a10, a10 +; CHECK-ESP32S2-NEXT: retw.n +start: + %val = fpext half %a to float + ret float %val +} + +declare float @llvm.convert.from.fp16.f32(i16) nounwind readnone +declare double @llvm.convert.from.fp16.f64(i16) nounwind readnone + +declare i16 @llvm.convert.to.fp16.f32(float) nounwind readnone +declare i16 @llvm.convert.to.fp16.f64(double) nounwind readnone From aace063fda01504608e4c818a3694f8108da36cf Mon Sep 17 00:00:00 2001 From: Farzon Lotfi Date: Tue, 21 Jul 2026 18:10:52 -0400 Subject: [PATCH 42/50] [SPIR-V] Erase dead functions in shader modules (#209672) The LLVM offload test suite is current failing layout keyword orientation tests. While initially appearing to be a layout problem in the SPIR-V legalizer caused by matrix vector sizes exceeding 4, the root cause is actually persistent dead function definitions in the SPIR-V backend. This behavior is standard for OpenCL, which relies on a linker to clean up unused definitions; however, since shaders lack a linker and require full inlining, we must handle this in a special shader only pass. This change adds SPIRVFinalizeShaderLinkage (a shader-only analogue of DXILFinalizeLinkage): internalize non-entry, non-exported helpers and erase the dead ones. Globals are left alone since unreferenced externals become interface OpVariables. Fixes #201712 Assisted with Claude Opus 4.8 --- llvm/lib/Target/SPIRV/CMakeLists.txt | 1 + llvm/lib/Target/SPIRV/SPIRV.h | 2 + .../SPIRV/SPIRVFinalizeShaderLinkage.cpp | 94 +++++++++++++++++++ .../Target/SPIRV/SPIRVFinalizeShaderLinkage.h | 34 +++++++ llvm/lib/Target/SPIRV/SPIRVPassRegistry.def | 1 + llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp | 13 ++- .../CodeGen/SPIRV/finalize-shader-linkage.ll | 25 +++++ 7 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 llvm/lib/Target/SPIRV/SPIRVFinalizeShaderLinkage.cpp create mode 100644 llvm/lib/Target/SPIRV/SPIRVFinalizeShaderLinkage.h create mode 100644 llvm/test/CodeGen/SPIRV/finalize-shader-linkage.ll diff --git a/llvm/lib/Target/SPIRV/CMakeLists.txt b/llvm/lib/Target/SPIRV/CMakeLists.txt index cf77264a95d63..2879279925ffe 100644 --- a/llvm/lib/Target/SPIRV/CMakeLists.txt +++ b/llvm/lib/Target/SPIRV/CMakeLists.txt @@ -46,6 +46,7 @@ add_llvm_target(SPIRVCodeGen SPIRVRegisterBankInfo.cpp SPIRVRegisterInfo.cpp SPIRVRegularizer.cpp + SPIRVFinalizeShaderLinkage.cpp SPIRVSubtarget.cpp SPIRVTargetMachine.cpp SPIRVTargetTransformInfo.cpp diff --git a/llvm/lib/Target/SPIRV/SPIRV.h b/llvm/lib/Target/SPIRV/SPIRV.h index f352bb1da5dca..17559ba6ed080 100644 --- a/llvm/lib/Target/SPIRV/SPIRV.h +++ b/llvm/lib/Target/SPIRV/SPIRV.h @@ -27,6 +27,7 @@ ModulePass *createSPIRVPushConstantAccessLegacyPass(SPIRVTargetMachine *TM); FunctionPass *createSPIRVMergeRegionExitTargetsPass(); ModulePass *createSPIRVLegalizeImplicitBindingPass(); ModulePass *createSPIRVLegalizeZeroSizeArraysPass(const SPIRVTargetMachine &TM); +ModulePass *createSPIRVFinalizeShaderLinkagePass(const SPIRVTargetMachine &TM); FunctionPass *createSPIRVLegalizePointerCastPass(SPIRVTargetMachine *TM); FunctionPass *createSPIRVRegularizerPass(); FunctionPass *createSPIRVPreLegalizerCombiner(); @@ -57,6 +58,7 @@ void initializeSPIRVPrepareFunctionsLegacyPass(PassRegistry &); void initializeSPIRVPrepareGlobalsLegacyPass(PassRegistry &); void initializeSPIRVLegalizeImplicitBindingLegacyPass(PassRegistry &); void initializeSPIRVLegalizeZeroSizeArraysLegacyPass(PassRegistry &); +void initializeSPIRVFinalizeShaderLinkageLegacyPass(PassRegistry &); void initializeSPIRVCtorDtorLoweringLegacyPass(PassRegistry &); } // namespace llvm diff --git a/llvm/lib/Target/SPIRV/SPIRVFinalizeShaderLinkage.cpp b/llvm/lib/Target/SPIRV/SPIRVFinalizeShaderLinkage.cpp new file mode 100644 index 0000000000000..554d0f0e151d6 --- /dev/null +++ b/llvm/lib/Target/SPIRV/SPIRVFinalizeShaderLinkage.cpp @@ -0,0 +1,94 @@ +//===- SPIRVFinalizeShaderLinkage.cpp - Finalize shader linkage ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Shader-only analogue of DXILFinalizeLinkage: internalizes non-entry, +// non-exported HLSL helper functions and erases the resulting dead ones. +// +//===----------------------------------------------------------------------===// + +#include "SPIRVFinalizeShaderLinkage.h" +#include "SPIRV.h" +#include "SPIRVSubtarget.h" +#include "SPIRVTargetMachine.h" +#include "SPIRVUtils.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" + +#define DEBUG_TYPE "spirv-finalize-shader-linkage" + +using namespace llvm; + +namespace { + +bool finalizeShaderLinkage(const SPIRVTargetMachine &TM, Module &M) { + // This pass only applies to shader targets because shaders don't have + // linkers. + if (!TM.getSubtargetImpl()->isShader()) + return false; + + bool Changed = false; + + for (Function &F : M) { + if (F.isIntrinsic() || F.isDeclaration() || isEntryPoint(F)) + continue; + if (F.hasExternalLinkage() && !F.hasHiddenVisibility()) + continue; + if (!F.hasLocalLinkage()) { + F.setLinkage(GlobalValue::InternalLinkage); + Changed = true; + } + } + + // Erase dead helpers, iterating to a fixpoint for helper-calls-helper chains. + bool LocalChange = true; + while (LocalChange) { + LocalChange = false; + for (Function &F : make_early_inc_range(M)) + if (F.isDefTriviallyDead()) { + F.eraseFromParent(); + LocalChange = Changed = true; + } + } + return Changed; +} + +class SPIRVFinalizeShaderLinkageLegacy : public ModulePass { +public: + static char ID; + SPIRVFinalizeShaderLinkageLegacy(const SPIRVTargetMachine &TM) + : ModulePass(ID), TM(TM) {} + StringRef getPassName() const override { + return "SPIRV Finalize Shader Linkage"; + } + bool runOnModule(Module &M) override { return finalizeShaderLinkage(TM, M); } + +private: + const SPIRVTargetMachine &TM; +}; + +} // namespace + +PreservedAnalyses SPIRVFinalizeShaderLinkage::run(Module &M, + ModuleAnalysisManager &AM) { + return finalizeShaderLinkage(TM, M) ? PreservedAnalyses::none() + : PreservedAnalyses::all(); +} + +char SPIRVFinalizeShaderLinkageLegacy::ID = 0; + +INITIALIZE_PASS(SPIRVFinalizeShaderLinkageLegacy, + "spirv-finalize-shader-linkage", + "Finalize SPIR-V shader linkage", false, false) + +ModulePass * +llvm::createSPIRVFinalizeShaderLinkagePass(const SPIRVTargetMachine &TM) { + return new SPIRVFinalizeShaderLinkageLegacy(TM); +} diff --git a/llvm/lib/Target/SPIRV/SPIRVFinalizeShaderLinkage.h b/llvm/lib/Target/SPIRV/SPIRVFinalizeShaderLinkage.h new file mode 100644 index 0000000000000..c8ba6d513ca1c --- /dev/null +++ b/llvm/lib/Target/SPIRV/SPIRVFinalizeShaderLinkage.h @@ -0,0 +1,34 @@ +//===- SPIRVFinalizeShaderLinkage.h - Finalize shader linkage --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// Shader-only analogue of DXILFinalizeLinkage: internalizes non-entry, +/// non-exported HLSL helper functions and erases the resulting dead ones. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_SPIRV_SPIRVFINALIZESHADERLINKAGE_H +#define LLVM_LIB_TARGET_SPIRV_SPIRVFINALIZESHADERLINKAGE_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class SPIRVTargetMachine; + +class SPIRVFinalizeShaderLinkage + : public OptionalPassInfoMixin { + const SPIRVTargetMachine &TM; + +public: + SPIRVFinalizeShaderLinkage(const SPIRVTargetMachine &TM) : TM(TM) {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + +} // namespace llvm + +#endif // LLVM_LIB_TARGET_SPIRV_SPIRVFINALIZESHADERLINKAGE_H diff --git a/llvm/lib/Target/SPIRV/SPIRVPassRegistry.def b/llvm/lib/Target/SPIRV/SPIRVPassRegistry.def index bb9f1c63cba3b..efceace6d8413 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPassRegistry.def +++ b/llvm/lib/Target/SPIRV/SPIRVPassRegistry.def @@ -23,6 +23,7 @@ MODULE_PASS("spirv-lower-ctor-dtor", SPIRVCtorDtorLoweringPass()) MODULE_PASS("spirv-prepare-functions", SPIRVPrepareFunctions(*static_cast(this))) MODULE_PASS("spirv-prepare-globals", SPIRVPrepareGlobals()) MODULE_PASS("spirv-pushconstant-access", SPIRVPushConstantAccess(*static_cast(this))) +MODULE_PASS("spirv-finalize-shader-linkage", SPIRVFinalizeShaderLinkage(*static_cast(this))) MODULE_PASS("spirv-emit-intrinsics", SPIRVEmitIntrinsicsPass(*static_cast(this))) #undef MODULE_PASS diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp index 85ed1cba1ba30..07a664ad13216 100644 --- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp @@ -16,6 +16,7 @@ #include "SPIRVCBufferAccess.h" #include "SPIRVCtorDtorLowering.h" #include "SPIRVEmitIntrinsics.h" +#include "SPIRVFinalizeShaderLinkage.h" #include "SPIRVGlobalRegistry.h" #include "SPIRVLegalizeImplicitBinding.h" #include "SPIRVLegalizePointerCast.h" @@ -75,6 +76,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeSPIRVTarget() { initializeSPIRVPrepareGlobalsLegacyPass(PR); initializeSPIRVLegalizeImplicitBindingLegacyPass(PR); initializeSPIRVCtorDtorLoweringLegacyPass(PR); + initializeSPIRVFinalizeShaderLinkageLegacyPass(PR); } static Reloc::Model getEffectiveRelocModel(std::optional RM) { @@ -186,10 +188,13 @@ void SPIRVPassConfig::addIRPasses() { TargetPassConfig::addIRPasses(); - // Variadic function calls aren't supported in shader code. - // This needs to come before SPIRVPrepareFunctions because this - // may introduce intrinsic calls. - if (!TM.getSubtargetImpl()->isShader()) { + if (TM.getSubtargetImpl()->isShader()) { + if (getOptLevel() != CodeGenOptLevel::None) + addPass(createSPIRVFinalizeShaderLinkagePass(TM)); + } else { + // Variadic function calls aren't supported in shader code. + // This needs to come before SPIRVPrepareFunctions because this + // may introduce intrinsic calls. addPass(createExpandVariadicsPass(ExpandVariadicsMode::Lowering)); } diff --git a/llvm/test/CodeGen/SPIRV/finalize-shader-linkage.ll b/llvm/test/CodeGen/SPIRV/finalize-shader-linkage.ll new file mode 100644 index 0000000000000..e40a32a66e332 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/finalize-shader-linkage.ll @@ -0,0 +1,25 @@ +; RUN: opt -mtriple=spirv-unknown-vulkan1.3-compute -passes=spirv-finalize-shader-linkage -S %s | FileCheck %s --check-prefix=OPT +; RUN: llc -mtriple=spirv-unknown-vulkan1.3-compute %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -mtriple=spirv-unknown-vulkan1.3-compute %s -o - -filetype=obj | spirv-val %} + +; The pass erases the dead helper and keeps the entry point. +; OPT-NOT: @dead_matrix_helper +; OPT: define void @main() + +; CHECK-DAG: OpEntryPoint GLCompute %[[#entry:]] "main" +; The dead helper (and its illegal wide-vector parameter) must not be emitted. +; CHECK-NOT: OpFunctionParameter +; CHECK-NOT: dead_matrix_helper + +define hidden spir_func <6 x float> @dead_matrix_helper(<6 x float> %m) #1 { + %r = fadd <6 x float> %m, %m + ret <6 x float> %r +} + +define void @main() #0 { +entry: + ret void +} + +attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" } +attributes #1 = { alwaysinline } From 9cfbe4a4efb0cebf7c8f9d4fca3e864fc0ac86ed Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Tue, 21 Jul 2026 15:32:31 -0700 Subject: [PATCH 43/50] [llvm-reduce] Run AssignGUIDPass when loading BC Otherwise we run into crashes when loading BC that has a module summary. This is a no-op if we load existing GUIDs, so probably makes sense to run regardless. Reviewers: mtrofin Pull Request: https://github.com/llvm/llvm-project/pull/208965 --- .../tools/llvm-reduce/initialize-asm-parsers.ll | 13 +++++++++++++ llvm/tools/llvm-reduce/ReducerWorkItem.cpp | 5 +++++ 2 files changed, 18 insertions(+) create mode 100644 llvm/test/tools/llvm-reduce/initialize-asm-parsers.ll diff --git a/llvm/test/tools/llvm-reduce/initialize-asm-parsers.ll b/llvm/test/tools/llvm-reduce/initialize-asm-parsers.ll new file mode 100644 index 0000000000000..a8a321ca66749 --- /dev/null +++ b/llvm/test/tools/llvm-reduce/initialize-asm-parsers.ll @@ -0,0 +1,13 @@ +; Check that we can parse assembly. +; REQUIRES: x86-registered-target + +; RUN: llvm-as %s -o %t.bc +; RUN: llvm-reduce --test=llvm-dis %t.bc + +target triple = "x86_64-grtev4-linux-gnu" + +define void @foo(ptr , ptr ) { + ret void +} +^0 = module: (path: "/tmp/test.bc", hash: (1638052558, 3438078092, 2356962112, 743220300, 68728585)) +^1 = gv: (name: "foo", summaries: (function: (module: ^0, flags: (linkage: linkonce_odr), insts: 1))) diff --git a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp index 2a16c749dc567..110c0e75eed07 100644 --- a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp +++ b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp @@ -39,6 +39,8 @@ #include "llvm/Target/TargetMachine.h" #include "llvm/TargetParser/Host.h" #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h" +#include "llvm/Transforms/IPO/WholeProgramDevirt.h" +#include "llvm/Transforms/Utils/AssignGUID.h" #include "llvm/Transforms/Utils/Cloning.h" #include @@ -890,6 +892,9 @@ llvm::parseReducerWorkItem(StringRef ToolName, StringRef Filename, if (MMM->LTOInfo->IsThinLTO && MMM->LTOInfo->EnableSplitLTOUnit) initializeTargetInfo(); } + + if (MMM->LTOInfo) + AssignGUIDPass::runOnModule(MMM->getModule()); } if (MMM->verify(&errs())) { WithColor::error(errs(), ToolName) From 24586dad1e0b035ced84ef934a310b620fd5fd7f Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Tue, 21 Jul 2026 15:34:27 -0700 Subject: [PATCH 44/50] [llvm-reduce] Unconditionally initialize targets It is not expensive to do, and doing it unconditionally avoids edge cases like the one in the added test case. Reviewers: arsenm, mtrofin Pull Request: https://github.com/llvm/llvm-project/pull/208966 --- .../tools/llvm-reduce/initialize-asm-parsers.ll | 2 ++ llvm/tools/llvm-reduce/ReducerWorkItem.cpp | 13 ------------- llvm/tools/llvm-reduce/llvm-reduce.cpp | 6 ++++++ 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/llvm/test/tools/llvm-reduce/initialize-asm-parsers.ll b/llvm/test/tools/llvm-reduce/initialize-asm-parsers.ll index a8a321ca66749..a25b285acdb58 100644 --- a/llvm/test/tools/llvm-reduce/initialize-asm-parsers.ll +++ b/llvm/test/tools/llvm-reduce/initialize-asm-parsers.ll @@ -6,6 +6,8 @@ target triple = "x86_64-grtev4-linux-gnu" +module asm + "" define void @foo(ptr , ptr ) { ret void } diff --git a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp index 110c0e75eed07..08bb57adc07a1 100644 --- a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp +++ b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp @@ -33,7 +33,6 @@ #include "llvm/Passes/PassBuilder.h" #include "llvm/Support/MemoryBufferRef.h" #include "llvm/Support/SourceMgr.h" -#include "llvm/Support/TargetSelect.h" #include "llvm/Support/ToolOutputFile.h" #include "llvm/Support/WithColor.h" #include "llvm/Target/TargetMachine.h" @@ -440,13 +439,6 @@ static std::unique_ptr cloneMF(MachineFunction *SrcMF, return DstMF; } -static void initializeTargetInfo() { - InitializeAllTargets(); - InitializeAllTargetMCs(); - InitializeAllAsmPrinters(); - InitializeAllAsmParsers(); -} - void ReducerWorkItem::print(raw_ostream &ROS, void *p) const { if (MMI) { printMIR(ROS, *M); @@ -831,8 +823,6 @@ llvm::parseReducerWorkItem(StringRef ToolName, StringRef Filename, auto MMM = std::make_unique(); if (IsMIR) { - initializeTargetInfo(); - auto FileOrErr = MemoryBuffer::getFileOrSTDIN(Filename, /*IsText=*/true); if (std::error_code EC = FileOrErr.getError()) { WithColor::error(errs(), ToolName) << EC.message() << '\n'; @@ -888,9 +878,6 @@ llvm::parseReducerWorkItem(StringRef ToolName, StringRef Filename, } else { IsBitcode = true; MMM->readBitcode(MemoryBufferRef(**MB), Ctxt, ToolName); - - if (MMM->LTOInfo->IsThinLTO && MMM->LTOInfo->EnableSplitLTOUnit) - initializeTargetInfo(); } if (MMM->LTOInfo) diff --git a/llvm/tools/llvm-reduce/llvm-reduce.cpp b/llvm/tools/llvm-reduce/llvm-reduce.cpp index 94bc7e14159d6..ffda00d1a5785 100644 --- a/llvm/tools/llvm-reduce/llvm-reduce.cpp +++ b/llvm/tools/llvm-reduce/llvm-reduce.cpp @@ -22,6 +22,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/InitLLVM.h" #include "llvm/Support/Process.h" +#include "llvm/Support/TargetSelect.h" #include "llvm/Support/WithColor.h" #include "llvm/Support/raw_ostream.h" @@ -138,6 +139,11 @@ int main(int Argc, char **Argv) { InitLLVM X(Argc, Argv); const StringRef ToolName(Argv[0]); + InitializeAllTargets(); + InitializeAllTargetMCs(); + InitializeAllAsmPrinters(); + InitializeAllAsmParsers(); + cl::HideUnrelatedOptions({&LLVMReduceOptions, &getColorCategory()}); cl::ParseCommandLineOptions( Argc, Argv, From fcbf28ffc812cfba9db0899256aa8757815dfcf9 Mon Sep 17 00:00:00 2001 From: adams381 Date: Tue, 21 Jul 2026 17:38:20 -0500 Subject: [PATCH 45/50] [CIR] Report NYI for defaulted union copy/move assignment A defaulted union copy or move assignment operator has an empty synthesized body because Sema skips union fields, leaving no AST expression for the implied whole-object copy. CIRGen emitted that empty body and silently dropped the assignment. Report NYI for defaulted union assignment instead. Struct and array assignments are unaffected because their synthesized bodies contain the memberwise copies needed by the existing body-emission path. --- clang/lib/CIR/CodeGen/CIRGenClass.cpp | 14 ++++++++++++++ .../test/CIR/CodeGen/trivial-union-assign-nyi.cpp | 15 +++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 clang/test/CIR/CodeGen/trivial-union-assign-nyi.cpp diff --git a/clang/lib/CIR/CodeGen/CIRGenClass.cpp b/clang/lib/CIR/CodeGen/CIRGenClass.cpp index 96ece6703a512..2e50956e1c2ee 100644 --- a/clang/lib/CIR/CodeGen/CIRGenClass.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenClass.cpp @@ -901,6 +901,20 @@ void CIRGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &args) { assert(!cir::MissingFeatures::incrementProfileCounter()); assert(!cir::MissingFeatures::runCleanupsScope()); + // A defaulted union copy/move assignment has an empty synthesized body: + // Sema skips union fields (the FIXME in SemaDeclCXX::buildSingleCopyAssign), + // so there is no AST expression for the implied whole-object memcpy. + // Emitting that body would silently drop the copy, so report NYI instead. + // Struct/array memcpy-equivalent assignments carry the implicit memberwise + // copies in the AST (per-field assignment expressions, or a builtin memcpy + // call for array members) and lower correctly through the loop below. + if (assignOp->isMemcpyEquivalentSpecialMember(getContext()) && + assignOp->getParent()->isUnion()) { + cgm.errorNYI(assignOp->getSourceRange(), + "defaulted union copy/move assignment operator"); + return; + } + // Classic codegen uses a special class to attempt to replace member // initializers with memcpy. We could possibly defer that to the // lowering or optimization phases to keep the memory accesses more diff --git a/clang/test/CIR/CodeGen/trivial-union-assign-nyi.cpp b/clang/test/CIR/CodeGen/trivial-union-assign-nyi.cpp new file mode 100644 index 0000000000000..e457dca4fd23d --- /dev/null +++ b/clang/test/CIR/CodeGen/trivial-union-assign-nyi.cpp @@ -0,0 +1,15 @@ +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir -verify %s + +// The defaulted copy/move assignment operator of a union has an empty +// synthesized body -- Sema skips union fields, leaving no AST expression for +// the implied whole-object copy. Emitting that body would silently drop the +// copy, so CIRGen reports NYI instead. + +// expected-error@+1 2 {{ClangIR code gen Not Yet Implemented: defaulted union copy/move assignment operator}} +union U { + void *p; + int i; +}; + +void copy_assign(U &a, U &b) { a = b; } +void move_assign(U &a, U &b) { a = static_cast(b); } From 4fff13bfb9648a2e1c34cded0065b493de9cec1b Mon Sep 17 00:00:00 2001 From: adams381 Date: Tue, 21 Jul 2026 17:58:48 -0500 Subject: [PATCH 46/50] [CIR] Classify empty records as Ignore in x86_64 callconv (#211078) The x86_64 aggregate calling-convention bridge rejects a zero-field record as NYI, even though the SysV classifier already treats an empty record as NoClass/NoClass (so it returns Ignore) and the rewriter already drops arguments and returns classified Ignore. Dropping the zero-field reject in `isSupportedType` lets a C empty struct classify as Ignore: the argument slot is removed, the remaining arguments shift down, and an empty-record return lowers to void. The C++ empty class stays NYI. CIRGen lays it out as a single padded byte, which the padded reject still catches. Unions, packed, and all-float aggregates remain NYI as before. --------- Co-authored-by: Andy Kaylor --- .../Transforms/CallConvLoweringPass.cpp | 12 ++-- .../abi-lowering/x86_64-aggregate-nyi.cir | 9 --- .../abi-lowering/x86_64-empty-record.cir | 64 +++++++++++++++++++ 3 files changed, 70 insertions(+), 15 deletions(-) create mode 100644 clang/test/CIR/Transforms/abi-lowering/x86_64-empty-record.cir diff --git a/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp b/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp index e1414d3b311e2..b8c98e448cf1a 100644 --- a/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp +++ b/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp @@ -115,13 +115,13 @@ static bool isSupportedType(mlir::Type ty) { // Unions and packed / padded records each need classification this bridge // does not implement (a union widen fixup and pad-aware eightbyte // classification), so reject them here and report NYI rather than - // misclassify. Empty-for-ABI records classify as Ignore, which is also - // deferred: a C empty struct is a zero-field record, and CIRGen lays out - // an empty C++ class as a single padded byte (caught by the padded check). + // misclassify. A zero-field record (a C empty struct) classifies as + // Ignore and is dropped from the lowered signature. CIRGen lays out an + // empty C++ class as a single padded byte, which the padded check rejects. // A real one-byte struct such as `{char[1]}` has a field and is not // padded, so it is classified normally. if (recTy.isUnion() || !recTy.isComplete() || recTy.getPacked() || - recTy.getPadded() || recTy.getMembers().empty()) + recTy.getPadded()) return false; return llvm::all_of(recTy.getMembers(), [](mlir::Type m) { return isSupportedType(m); }); @@ -257,8 +257,8 @@ static const llvm::abi::Type *mapCIRType(mlir::Type type, /// Indirect: an aggregate that does not fit in registers is passed via a /// pointer (sret for returns, byval for arguments). /// -/// Ignore: a void return has no register or stack slot. (Empty-for-ABI -/// records are rejected by isSupportedType, so they never reach here.) +/// Ignore: a void return has no register or stack slot, and a zero-field +/// (empty) record is dropped from the signature. static std::optional convertABIArgInfo(const llvm::abi::ArgInfo &info, MLIRContext *ctx, mlir::Type origTy) { diff --git a/clang/test/CIR/Transforms/abi-lowering/x86_64-aggregate-nyi.cir b/clang/test/CIR/Transforms/abi-lowering/x86_64-aggregate-nyi.cir index 87692e1a3e5a2..85f05fef9f96b 100644 --- a/clang/test/CIR/Transforms/abi-lowering/x86_64-aggregate-nyi.cir +++ b/clang/test/CIR/Transforms/abi-lowering/x86_64-aggregate-nyi.cir @@ -8,7 +8,6 @@ !rec_P = !cir.struct<"P" packed {!s8i, !s32i}> !rec_Ov = !cir.struct<"Ov" padded {!s32i, !cir.array}> !rec_E = !cir.struct<"E" padded {!u8i}> -!rec_E0 = !cir.struct<"E0" {}> !rec_FF = !cir.struct<"FF" {!cir.float, !cir.float}> !rec_RetFF = !cir.struct<"RetFF" {!cir.float, !cir.float}> @@ -50,14 +49,6 @@ module attributes { // CHECK: not yet implemented for type '!cir.struct<"E" padded - // A zero-field record (a C empty struct) also classifies as Ignore, which is - // deferred, so it is rejected rather than passed with the argument dropped. - cir.func @take_e0(%arg0: !rec_E0) { - cir.return - } - - // CHECK: not yet implemented for type '!cir.struct<"E0" - // An all-float struct classifies to an SSE vector coerce this bridge does // not represent, so it is reported NYI rather than passed unchanged. cir.func @take_ff(%arg0: !rec_FF) { diff --git a/clang/test/CIR/Transforms/abi-lowering/x86_64-empty-record.cir b/clang/test/CIR/Transforms/abi-lowering/x86_64-empty-record.cir new file mode 100644 index 0000000000000..23254734f27e5 --- /dev/null +++ b/clang/test/CIR/Transforms/abi-lowering/x86_64-empty-record.cir @@ -0,0 +1,64 @@ +// RUN: cir-opt %s -cir-call-conv-lowering=target=x86_64 | FileCheck %s + +!s32i = !cir.int +!rec_E0 = !cir.struct<"E0" {}> + +module attributes { + dlti.dl_spec = #dlti.dl_spec< + #dlti.dl_entry: vector<2xi64>>, + #dlti.dl_entry: vector<2xi64>>> +} { + + // A zero-field record (a C empty struct) classifies as Ignore: the argument + // is dropped from the lowered signature. + cir.func @take_empty(%arg0: !rec_E0) { + cir.return + } + + // CHECK: cir.func{{.*}} @take_empty() + // CHECK-NEXT: cir.return + + // The empty argument is dropped even when a real argument remains; the real + // argument shifts down to the first slot. + cir.func @take_mixed(%arg0: !rec_E0, %arg1: !s32i) -> !s32i { + cir.return %arg1 : !s32i + } + + // CHECK: cir.func{{.*}} @take_mixed(%arg0: !s32i) -> !s32i + // CHECK-NEXT: cir.return %arg0 : !s32i + + // An empty-record return is dropped: the function returns void. The + // local storage slot survives the rewrite; only the return is dropped. + cir.func @ret_empty() -> !rec_E0 { + %0 = cir.alloca "r" align(1) : !cir.ptr + %1 = cir.load %0 : !cir.ptr, !rec_E0 + cir.return %1 : !rec_E0 + } + + // CHECK: cir.func{{.*}} @ret_empty() + // CHECK: cir.alloca {{.*}} !cir.ptr + // CHECK: cir.return{{$}} + + // The empty argument is also dropped at call sites: the caller's own empty + // argument is dropped, and the call to take_mixed drops it too. + cir.func @caller(%arg0: !rec_E0, %arg1: !s32i) -> !s32i { + %0 = cir.call @take_mixed(%arg0, %arg1) : (!rec_E0, !s32i) -> !s32i + cir.return %0 : !s32i + } + + // CHECK: cir.func{{.*}} @caller(%arg0: !s32i) -> !s32i + // CHECK: cir.call @take_mixed(%arg0) : (!s32i) -> !s32i + + // A local empty struct lives in an alloca; the alloca survives the rewrite + // while the empty operand is dropped from the call. + cir.func @caller_local(%arg0: !s32i) -> !s32i { + %0 = cir.alloca "e" align(1) : !cir.ptr + %1 = cir.load %0 : !cir.ptr, !rec_E0 + %2 = cir.call @take_mixed(%1, %arg0) : (!rec_E0, !s32i) -> !s32i + cir.return %2 : !s32i + } + + // CHECK: cir.func{{.*}} @caller_local(%arg0: !s32i) -> !s32i + // CHECK: cir.alloca "e" align(1) : !cir.ptr + // CHECK: cir.call @take_mixed(%arg0) : (!s32i) -> !s32i +} From 3782c00acdde18c0fb1a0946a26898dc2e7b3141 Mon Sep 17 00:00:00 2001 From: Alexey Bader Date: Tue, 21 Jul 2026 15:59:12 -0700 Subject: [PATCH 47/50] [clang-sycl-linker] Forward all --ocloc-options occurrences to ocloc (#211075) getLastArgValue() only returned the final --ocloc-options= occurrence, silently dropping earlier ones when the option is passed multiple times (one token per occurrence). Use getAllArgValues() so every occurrence is forwarded to ocloc. --- clang/test/OffloadTools/clang-sycl-linker/basic.ll | 6 ++++++ clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/clang/test/OffloadTools/clang-sycl-linker/basic.ll b/clang/test/OffloadTools/clang-sycl-linker/basic.ll index 448a166e0a5e1..00e3bba8d297e 100644 --- a/clang/test/OffloadTools/clang-sycl-linker/basic.ll +++ b/clang/test/OffloadTools/clang-sycl-linker/basic.ll @@ -118,6 +118,12 @@ ; AOT-INTEL-GPU-NEXT: sycl-bundle: image kind: o, triple: spirv64, arch: bmg_g21 ; AOT-INTEL-GPU-NOT: {{.+}} ; +; Test that all --ocloc-options are passed to ocloc, even if they contain spaces or quotes. +; RUN: clang-sycl-linker --dry-run -v --module-split-mode=link_unit -arch=bmg_g21 %t/input1.bc -o %t/aot-gpu.out 2>&1 \ +; RUN: --ocloc-options="-a -b" --ocloc-options=-c --ocloc-options="d" --ocloc-options='"-e -f"' \ +; RUN: | FileCheck %s --check-prefix=AOT-INTEL-OCLOC-OPTIONS +; AOT-INTEL-OCLOC-OPTIONS: "{{.*}}ocloc{{.*}}" {{.*}}-device bmg_g21 -a -b -c d "-e -f" +; ; Test AOT compilation for an Intel CPU. ; Test that IMG_Object image kind is set for AOT compilation (Intel CPU). ; RUN: clang-sycl-linker --dry-run -v --module-split-mode=link_unit -arch=graniterapids %t/input1.bc %t/input2.bc -o %t/aot-cpu.out 2>&1 \ diff --git a/clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp b/clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp index 5e937507289ec..acb529faa0b51 100644 --- a/clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp +++ b/clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp @@ -725,8 +725,11 @@ static Error runAOTCompileIntelGPU(StringRef InputFile, StringRef OutputFile, CmdArgs.push_back("-device"); CmdArgs.push_back(Arch); - StringRef ExtraArgs = Args.getLastArgValue(OPT_ocloc_options_EQ); - ExtraArgs.split(CmdArgs, " ", /*MaxSplit=*/-1, /*KeepEmpty=*/false); + // getAllArgValues returns a temporary vector; retain it so the StringRefs + // remain valid through the executeCommands call below. + std::vector ExtraArgsStorage = + Args.getAllArgValues(OPT_ocloc_options_EQ); + llvm::append_range(CmdArgs, ExtraArgsStorage); CmdArgs.push_back("-output"); CmdArgs.push_back(OutputFile); From aef1e4f249856ce1b9e8362fee75619f3fa8efcf Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Tue, 21 Jul 2026 16:10:13 -0700 Subject: [PATCH 48/50] [NVPTX] Support short entry param pointers (#206512) --- clang/test/CodeGen/target-data.c | 2 +- llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 11 +- llvm/lib/TargetParser/TargetDataLayout.cpp | 30 +- .../CodeGen/NVPTX/lower-args-gridconstant.ll | 687 ++++++++++++------ llvm/unittests/TargetParser/TripleTest.cpp | 32 + 5 files changed, 518 insertions(+), 244 deletions(-) diff --git a/clang/test/CodeGen/target-data.c b/clang/test/CodeGen/target-data.c index a5e0b814c7042..e74454ce95f81 100644 --- a/clang/test/CodeGen/target-data.c +++ b/clang/test/CodeGen/target-data.c @@ -144,7 +144,7 @@ // RUN: %clang_cc1 -triple nvptx-unknown -o - -emit-llvm %s | \ // RUN: FileCheck %s -check-prefix=NVPTX -// NVPTX: target datalayout = "e-p:32:32-p6:32:32-p7:32:32-i64:64-i128:128-i256:256-v16:16-v32:32-n16:32:64" +// NVPTX: target datalayout = "e-p:32:32-i64:64-i128:128-i256:256-v16:16-v32:32-n16:32:64" // RUN: %clang_cc1 -triple nvptx64-unknown -o - -emit-llvm %s | \ // RUN: FileCheck %s -check-prefix=NVPTX64 diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp index 55b01f1383f44..8a622396e7f6d 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -4071,11 +4071,13 @@ SDValue NVPTXTargetLowering::LowerFormalArguments( SelectionDAG &DAG, SmallVectorImpl &InVals) const { const DataLayout &DL = DAG.getDataLayout(); LLVMContext &Ctx = *DAG.getContext(); - auto PtrVT = getPointerTy(DAG.getDataLayout()); const Function &F = DAG.getMachineFunction().getFunction(); const bool IsKernel = isKernelFunction(F); + const MVT PtrVT = getPointerTy(DL, IsKernel ? ADDRESS_SPACE_ENTRY_PARAM + : ADDRESS_SPACE_LOCAL); + SDValue Root = DAG.getRoot(); SmallVector OutChains; @@ -4129,18 +4131,17 @@ SDValue NVPTXTargetLowering::LowerFormalArguments( const auto &ByvalIn = ArgIns[0]; assert(getValueType(DL, Ty) == ByvalIn.VT && "Ins type did not match function type"); - assert(ByvalIn.VT == PtrVT && "ByVal argument must be a pointer"); SDValue P; if (IsKernel) { - assert(Arg.getType()->getPointerAddressSpace() == - ADDRESS_SPACE_ENTRY_PARAM && + assert(Ty->getPointerAddressSpace() == ADDRESS_SPACE_ENTRY_PARAM && "Kernel ByVal argument must be lowered to the param address " "space by NVPTXLowerArgs"); P = ArgSymbol; P.getNode()->setIROrder(Arg.getArgNo() + 1); } else { - P = DAG.getNode(NVPTXISD::MoveParam, dl, ByvalIn.VT, ArgSymbol); + P = DAG.getNode(NVPTXISD::MoveParam, dl, ArgSymbol.getValueType(), + ArgSymbol); P.getNode()->setIROrder(Arg.getArgNo() + 1); P = DAG.getAddrSpaceCast(dl, ByvalIn.VT, P, ADDRESS_SPACE_LOCAL, ADDRESS_SPACE_GENERIC); diff --git a/llvm/lib/TargetParser/TargetDataLayout.cpp b/llvm/lib/TargetParser/TargetDataLayout.cpp index 18f0c13907e52..8b6f46642e4fa 100644 --- a/llvm/lib/TargetParser/TargetDataLayout.cpp +++ b/llvm/lib/TargetParser/TargetDataLayout.cpp @@ -456,19 +456,31 @@ static std::string computeX86DataLayout(const Triple &TT) { } static std::string computeNVPTXDataLayout(const Triple &T, StringRef ABIName) { - bool Is64Bit = T.getArch() == Triple::nvptx64; + const bool Is32Bit = T.getArch() == Triple::nvptx; + const bool IsShortPtr = ABIName == "shortptr"; std::string Ret = "e"; - // Tensor Memory (addrspace:6) is always 32-bits. - // Distributed Shared Memory (addrspace:7) follows shared memory - // (addrspace:3). - if (!Is64Bit) - Ret += "-p:32:32-p6:32:32-p7:32:32"; - else if (ABIName == "shortptr") - Ret += "-p3:32:32-p4:32:32-p5:32:32-p6:32:32-p7:32:32"; - else + if (Is32Bit) { + Ret += "-p:32:32"; + } else { + // Keep the pointer specifications sorted by address space. + // + // In shortptr mode, specify the following address spaces as 32-bits: + // - shared (addrspace:3) + // - constant (addrspace:4) + // - local (addrspace:5) + // - shared cluster (addrspace:7) + // - entry parameter (addrspace:101) + if (IsShortPtr) + Ret += "-p3:32:32-p4:32:32-p5:32:32"; + + // Tensor Memory (addrspace:6) is always 32-bits. Ret += "-p6:32:32"; + if (IsShortPtr) + Ret += "-p7:32:32-p101:32:32"; + } + Ret += "-i64:64-i128:128-i256:256-v16:16-v32:32-n16:32:64"; return Ret; diff --git a/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll b/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll index da8c440512af3..33264a1447872 100644 --- a/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll +++ b/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll @@ -1,7 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 -; RUN: opt < %s -S -nvptx-lower-args --mtriple nvptx64-nvidia-cuda -mcpu=sm_70 -mattr=+ptx77 | FileCheck %s --check-prefixes OPT -; RUN: llc < %s --mtriple nvptx64-nvidia-cuda -mcpu=sm_70 -mattr=+ptx77 -O1 | FileCheck %s --check-prefixes PTX +; RUN: opt < %s -S -nvptx-lower-args -mcpu=sm_70 -mattr=+ptx77 | FileCheck %s --check-prefixes OPT +; RUN: llc < %s -mcpu=sm_70 -mattr=+ptx77 -O1 | FileCheck %s --check-prefixes PTX,PTX-DEFAULT +; RUN: llc < %s -mcpu=sm_70 -mattr=+ptx77 -O1 --nvptx-short-ptr | FileCheck %s --check-prefixes PTX,PTX-SHORT-PTR +target triple = "nvptx64-nvidia-cuda" %struct.uint4 = type { i32, i32, i32, i32 } @gi = dso_local addrspace(1) externally_initialized global %struct.uint4 { i32 50462976, i32 117835012, i32 185207048, i32 252579084 }, align 16 @@ -18,36 +20,68 @@ define dso_local noundef i32 @non_kernel_function(ptr nocapture noundef readonly ; OPT-NEXT: [[TMP0:%.*]] = load i32, ptr [[ADD_PTR]], align 1 ; OPT-NEXT: ret i32 [[TMP0]] ; -; PTX-LABEL: non_kernel_function( -; PTX: { -; PTX-NEXT: .reg .pred %p<2>; -; PTX-NEXT: .reg .b16 %rs<3>; -; PTX-NEXT: .reg .b32 %r<11>; -; PTX-NEXT: .reg .b64 %rd<8>; -; PTX-EMPTY: -; PTX-NEXT: // %bb.0: // %entry -; PTX-NEXT: mov.b64 %rd1, non_kernel_function_param_0; -; PTX-NEXT: cvta.local.u64 %rd2, %rd1; -; PTX-NEXT: ld.param.b8 %rs1, [non_kernel_function_param_1]; -; PTX-NEXT: and.b16 %rs2, %rs1, 1; -; PTX-NEXT: setp.ne.b16 %p1, %rs2, 0; -; PTX-NEXT: mov.b64 %rd3, gi; -; PTX-NEXT: cvta.global.u64 %rd4, %rd3; -; PTX-NEXT: selp.b64 %rd5, %rd2, %rd4, %p1; -; PTX-NEXT: ld.param.s32 %rd6, [non_kernel_function_param_2]; -; PTX-NEXT: add.s64 %rd7, %rd5, %rd6; -; PTX-NEXT: ld.b8 %r1, [%rd7]; -; PTX-NEXT: ld.b8 %r2, [%rd7+1]; -; PTX-NEXT: shl.b32 %r3, %r2, 8; -; PTX-NEXT: or.b32 %r4, %r3, %r1; -; PTX-NEXT: ld.b8 %r5, [%rd7+2]; -; PTX-NEXT: shl.b32 %r6, %r5, 16; -; PTX-NEXT: ld.b8 %r7, [%rd7+3]; -; PTX-NEXT: shl.b32 %r8, %r7, 24; -; PTX-NEXT: or.b32 %r9, %r8, %r6; -; PTX-NEXT: or.b32 %r10, %r9, %r4; -; PTX-NEXT: st.param.b32 [func_retval0], %r10; -; PTX-NEXT: ret; +; PTX-DEFAULT-LABEL: non_kernel_function( +; PTX-DEFAULT: { +; PTX-DEFAULT-NEXT: .reg .pred %p<2>; +; PTX-DEFAULT-NEXT: .reg .b16 %rs<3>; +; PTX-DEFAULT-NEXT: .reg .b32 %r<11>; +; PTX-DEFAULT-NEXT: .reg .b64 %rd<8>; +; PTX-DEFAULT-EMPTY: +; PTX-DEFAULT-NEXT: // %bb.0: // %entry +; PTX-DEFAULT-NEXT: mov.b64 %rd1, non_kernel_function_param_0; +; PTX-DEFAULT-NEXT: cvta.local.u64 %rd2, %rd1; +; PTX-DEFAULT-NEXT: ld.param.b8 %rs1, [non_kernel_function_param_1]; +; PTX-DEFAULT-NEXT: and.b16 %rs2, %rs1, 1; +; PTX-DEFAULT-NEXT: setp.ne.b16 %p1, %rs2, 0; +; PTX-DEFAULT-NEXT: mov.b64 %rd3, gi; +; PTX-DEFAULT-NEXT: cvta.global.u64 %rd4, %rd3; +; PTX-DEFAULT-NEXT: selp.b64 %rd5, %rd2, %rd4, %p1; +; PTX-DEFAULT-NEXT: ld.param.s32 %rd6, [non_kernel_function_param_2]; +; PTX-DEFAULT-NEXT: add.s64 %rd7, %rd5, %rd6; +; PTX-DEFAULT-NEXT: ld.b8 %r1, [%rd7]; +; PTX-DEFAULT-NEXT: ld.b8 %r2, [%rd7+1]; +; PTX-DEFAULT-NEXT: shl.b32 %r3, %r2, 8; +; PTX-DEFAULT-NEXT: or.b32 %r4, %r3, %r1; +; PTX-DEFAULT-NEXT: ld.b8 %r5, [%rd7+2]; +; PTX-DEFAULT-NEXT: shl.b32 %r6, %r5, 16; +; PTX-DEFAULT-NEXT: ld.b8 %r7, [%rd7+3]; +; PTX-DEFAULT-NEXT: shl.b32 %r8, %r7, 24; +; PTX-DEFAULT-NEXT: or.b32 %r9, %r8, %r6; +; PTX-DEFAULT-NEXT: or.b32 %r10, %r9, %r4; +; PTX-DEFAULT-NEXT: st.param.b32 [func_retval0], %r10; +; PTX-DEFAULT-NEXT: ret; +; +; PTX-SHORT-PTR-LABEL: non_kernel_function( +; PTX-SHORT-PTR: { +; PTX-SHORT-PTR-NEXT: .reg .pred %p<2>; +; PTX-SHORT-PTR-NEXT: .reg .b16 %rs<3>; +; PTX-SHORT-PTR-NEXT: .reg .b32 %r<12>; +; PTX-SHORT-PTR-NEXT: .reg .b64 %rd<8>; +; PTX-SHORT-PTR-EMPTY: +; PTX-SHORT-PTR-NEXT: // %bb.0: // %entry +; PTX-SHORT-PTR-NEXT: ld.param.b8 %rs1, [non_kernel_function_param_1]; +; PTX-SHORT-PTR-NEXT: and.b16 %rs2, %rs1, 1; +; PTX-SHORT-PTR-NEXT: setp.ne.b16 %p1, %rs2, 0; +; PTX-SHORT-PTR-NEXT: mov.b32 %r1, non_kernel_function_param_0; +; PTX-SHORT-PTR-NEXT: cvt.u64.u32 %rd1, %r1; +; PTX-SHORT-PTR-NEXT: cvta.local.u64 %rd2, %rd1; +; PTX-SHORT-PTR-NEXT: mov.b64 %rd3, gi; +; PTX-SHORT-PTR-NEXT: cvta.global.u64 %rd4, %rd3; +; PTX-SHORT-PTR-NEXT: selp.b64 %rd5, %rd2, %rd4, %p1; +; PTX-SHORT-PTR-NEXT: ld.param.s32 %rd6, [non_kernel_function_param_2]; +; PTX-SHORT-PTR-NEXT: add.s64 %rd7, %rd5, %rd6; +; PTX-SHORT-PTR-NEXT: ld.b8 %r2, [%rd7]; +; PTX-SHORT-PTR-NEXT: ld.b8 %r3, [%rd7+1]; +; PTX-SHORT-PTR-NEXT: shl.b32 %r4, %r3, 8; +; PTX-SHORT-PTR-NEXT: or.b32 %r5, %r4, %r2; +; PTX-SHORT-PTR-NEXT: ld.b8 %r6, [%rd7+2]; +; PTX-SHORT-PTR-NEXT: shl.b32 %r7, %r6, 16; +; PTX-SHORT-PTR-NEXT: ld.b8 %r8, [%rd7+3]; +; PTX-SHORT-PTR-NEXT: shl.b32 %r9, %r8, 24; +; PTX-SHORT-PTR-NEXT: or.b32 %r10, %r9, %r7; +; PTX-SHORT-PTR-NEXT: or.b32 %r11, %r10, %r5; +; PTX-SHORT-PTR-NEXT: st.param.b32 [func_retval0], %r11; +; PTX-SHORT-PTR-NEXT: ret; entry: %a. = select i1 %b, ptr %a, ptr addrspacecast (ptr addrspace(1) @gi to ptr) %idx.ext = sext i32 %c to i64 @@ -76,7 +110,6 @@ define ptx_kernel void @grid_const_int(ptr byval(i32) align 4 "nvvm.grid_constan ; OPT-NEXT: [[ADD:%.*]] = add i32 [[TMP]], [[INPUT2]] ; OPT-NEXT: store i32 [[ADD]], ptr [[OUT]], align 4 ; OPT-NEXT: ret void -; %tmp = load i32, ptr %input1, align 4 %add = add i32 %tmp, %input2 store i32 %add, ptr %out @@ -108,7 +141,6 @@ define ptx_kernel void @grid_const_struct(ptr byval(%struct.s) align 4 "nvvm.gri ; OPT-NEXT: [[ADD:%.*]] = add i32 [[TMP1]], [[TMP2]] ; OPT-NEXT: store i32 [[ADD]], ptr [[OUT]], align 4 ; OPT-NEXT: ret void -; %gep1 = getelementptr inbounds %struct.s, ptr %input, i32 0, i32 0 %gep2 = getelementptr inbounds %struct.s, ptr %input, i32 0, i32 1 %int1 = load i32, ptr %gep1 @@ -119,63 +151,115 @@ define ptx_kernel void @grid_const_struct(ptr byval(%struct.s) align 4 "nvvm.gri } define ptx_kernel void @grid_const_escape(ptr byval(%struct.s) align 4 "nvvm.grid_constant" %input) { -; PTX-LABEL: grid_const_escape( -; PTX: { -; PTX-NEXT: .reg .b64 %rd<4>; -; PTX-NEXT: prototype_0 : .callprototype (.param .b32 _) _ (.param .b64 _); -; PTX-NEXT: // %bb.0: -; PTX-NEXT: mov.b64 %rd1, grid_const_escape_param_0; -; PTX-NEXT: cvta.param.u64 %rd2, %rd1; -; PTX-NEXT: { // callseq 0, 0 -; PTX-NEXT: .param .b64 param0; -; PTX-NEXT: .param .b32 retval0; -; PTX-NEXT: st.param.b64 [param0], %rd2; -; PTX-NEXT: mov.b64 %rd3, escape; -; PTX-NEXT: call (retval0), %rd3, (param0), prototype_0; -; PTX-NEXT: } // callseq 0 -; PTX-NEXT: ret; +; PTX-DEFAULT-LABEL: grid_const_escape( +; PTX-DEFAULT: { +; PTX-DEFAULT-NEXT: .reg .b64 %rd<4>; +; PTX-DEFAULT-NEXT: prototype_0 : .callprototype (.param .b32 _) _ (.param .b64 _); +; PTX-DEFAULT-NEXT: // %bb.0: +; PTX-DEFAULT-NEXT: mov.b64 %rd1, grid_const_escape_param_0; +; PTX-DEFAULT-NEXT: cvta.param.u64 %rd2, %rd1; +; PTX-DEFAULT-NEXT: { // callseq 0, 0 +; PTX-DEFAULT-NEXT: .param .b64 param0; +; PTX-DEFAULT-NEXT: .param .b32 retval0; +; PTX-DEFAULT-NEXT: st.param.b64 [param0], %rd2; +; PTX-DEFAULT-NEXT: mov.b64 %rd3, escape; +; PTX-DEFAULT-NEXT: call (retval0), %rd3, (param0), prototype_0; +; PTX-DEFAULT-NEXT: } // callseq 0 +; PTX-DEFAULT-NEXT: ret; +; +; PTX-SHORT-PTR-LABEL: grid_const_escape( +; PTX-SHORT-PTR: { +; PTX-SHORT-PTR-NEXT: .reg .b32 %r<2>; +; PTX-SHORT-PTR-NEXT: .reg .b64 %rd<4>; +; PTX-SHORT-PTR-NEXT: prototype_0 : .callprototype (.param .b32 _) _ (.param .b64 _); +; PTX-SHORT-PTR-NEXT: // %bb.0: +; PTX-SHORT-PTR-NEXT: mov.b32 %r1, grid_const_escape_param_0; +; PTX-SHORT-PTR-NEXT: cvt.u64.u32 %rd1, %r1; +; PTX-SHORT-PTR-NEXT: cvta.param.u64 %rd2, %rd1; +; PTX-SHORT-PTR-NEXT: { // callseq 0, 0 +; PTX-SHORT-PTR-NEXT: .param .b64 param0; +; PTX-SHORT-PTR-NEXT: .param .b32 retval0; +; PTX-SHORT-PTR-NEXT: st.param.b64 [param0], %rd2; +; PTX-SHORT-PTR-NEXT: mov.b64 %rd3, escape; +; PTX-SHORT-PTR-NEXT: call (retval0), %rd3, (param0), prototype_0; +; PTX-SHORT-PTR-NEXT: } // callseq 0 +; PTX-SHORT-PTR-NEXT: ret; ; OPT-LABEL: define ptx_kernel void @grid_const_escape( ; OPT-SAME: ptr addrspace(101) readonly byval([[STRUCT_S:%.*]]) align 4 "nvvm.grid_constant" [[INPUT:%.*]]) #[[ATTR0]] { ; OPT-NEXT: [[INPUT_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[INPUT]] to ptr ; OPT-NEXT: [[CALL:%.*]] = call i32 @escape(ptr [[INPUT_PARAM_GEN]]) ; OPT-NEXT: ret void -; %call = call i32 @escape(ptr %input) ret void } define ptx_kernel void @multiple_grid_const_escape(ptr byval(%struct.s) align 4 "nvvm.grid_constant" %input, i32 %a, ptr byval(i32) align 4 "nvvm.grid_constant" %b) { -; PTX-LABEL: multiple_grid_const_escape( -; PTX: { -; PTX-NEXT: .local .align 4 .b8 __local_depot4[4]; -; PTX-NEXT: .reg .b64 %SP; -; PTX-NEXT: .reg .b64 %SPL; -; PTX-NEXT: .reg .b32 %r<2>; -; PTX-NEXT: .reg .b64 %rd<8>; -; PTX-NEXT: prototype_1 : .callprototype (.param .b32 _) _ (.param .b64 _, .param .b64 _, .param .b64 _); -; PTX-NEXT: // %bb.0: -; PTX-NEXT: mov.b64 %SPL, __local_depot4; -; PTX-NEXT: cvta.local.u64 %SP, %SPL; -; PTX-NEXT: mov.b64 %rd1, multiple_grid_const_escape_param_0; -; PTX-NEXT: mov.b64 %rd2, multiple_grid_const_escape_param_2; -; PTX-NEXT: cvta.param.u64 %rd3, %rd2; -; PTX-NEXT: ld.param.b32 %r1, [multiple_grid_const_escape_param_1]; -; PTX-NEXT: cvta.param.u64 %rd4, %rd1; -; PTX-NEXT: add.u64 %rd5, %SP, 0; -; PTX-NEXT: add.u64 %rd6, %SPL, 0; -; PTX-NEXT: st.local.b32 [%rd6], %r1; -; PTX-NEXT: { // callseq 1, 0 -; PTX-NEXT: .param .b64 param0; -; PTX-NEXT: .param .b64 param1; -; PTX-NEXT: .param .b64 param2; -; PTX-NEXT: .param .b32 retval0; -; PTX-NEXT: st.param.b64 [param2], %rd3; -; PTX-NEXT: st.param.b64 [param1], %rd5; -; PTX-NEXT: st.param.b64 [param0], %rd4; -; PTX-NEXT: mov.b64 %rd7, escape3; -; PTX-NEXT: call (retval0), %rd7, (param0, param1, param2), prototype_1; -; PTX-NEXT: } // callseq 1 -; PTX-NEXT: ret; +; PTX-DEFAULT-LABEL: multiple_grid_const_escape( +; PTX-DEFAULT: { +; PTX-DEFAULT-NEXT: .local .align 4 .b8 __local_depot4[4]; +; PTX-DEFAULT-NEXT: .reg .b64 %SP; +; PTX-DEFAULT-NEXT: .reg .b64 %SPL; +; PTX-DEFAULT-NEXT: .reg .b32 %r<2>; +; PTX-DEFAULT-NEXT: .reg .b64 %rd<8>; +; PTX-DEFAULT-NEXT: prototype_1 : .callprototype (.param .b32 _) _ (.param .b64 _, .param .b64 _, .param .b64 _); +; PTX-DEFAULT-NEXT: // %bb.0: +; PTX-DEFAULT-NEXT: mov.b64 %SPL, __local_depot4; +; PTX-DEFAULT-NEXT: cvta.local.u64 %SP, %SPL; +; PTX-DEFAULT-NEXT: mov.b64 %rd1, multiple_grid_const_escape_param_0; +; PTX-DEFAULT-NEXT: mov.b64 %rd2, multiple_grid_const_escape_param_2; +; PTX-DEFAULT-NEXT: cvta.param.u64 %rd3, %rd2; +; PTX-DEFAULT-NEXT: ld.param.b32 %r1, [multiple_grid_const_escape_param_1]; +; PTX-DEFAULT-NEXT: cvta.param.u64 %rd4, %rd1; +; PTX-DEFAULT-NEXT: add.u64 %rd5, %SP, 0; +; PTX-DEFAULT-NEXT: add.u64 %rd6, %SPL, 0; +; PTX-DEFAULT-NEXT: st.local.b32 [%rd6], %r1; +; PTX-DEFAULT-NEXT: { // callseq 1, 0 +; PTX-DEFAULT-NEXT: .param .b64 param0; +; PTX-DEFAULT-NEXT: .param .b64 param1; +; PTX-DEFAULT-NEXT: .param .b64 param2; +; PTX-DEFAULT-NEXT: .param .b32 retval0; +; PTX-DEFAULT-NEXT: st.param.b64 [param2], %rd3; +; PTX-DEFAULT-NEXT: st.param.b64 [param1], %rd5; +; PTX-DEFAULT-NEXT: st.param.b64 [param0], %rd4; +; PTX-DEFAULT-NEXT: mov.b64 %rd7, escape3; +; PTX-DEFAULT-NEXT: call (retval0), %rd7, (param0, param1, param2), prototype_1; +; PTX-DEFAULT-NEXT: } // callseq 1 +; PTX-DEFAULT-NEXT: ret; +; +; PTX-SHORT-PTR-LABEL: multiple_grid_const_escape( +; PTX-SHORT-PTR: { +; PTX-SHORT-PTR-NEXT: .local .align 4 .b8 __local_depot4[4]; +; PTX-SHORT-PTR-NEXT: .reg .b64 %SP; +; PTX-SHORT-PTR-NEXT: .reg .b64 %SPL; +; PTX-SHORT-PTR-NEXT: .reg .b32 %r<5>; +; PTX-SHORT-PTR-NEXT: .reg .b64 %rd<8>; +; PTX-SHORT-PTR-NEXT: prototype_1 : .callprototype (.param .b32 _) _ (.param .b64 _, .param .b64 _, .param .b64 _); +; PTX-SHORT-PTR-NEXT: // %bb.0: +; PTX-SHORT-PTR-NEXT: mov.b64 %SPL, __local_depot4; +; PTX-SHORT-PTR-NEXT: cvta.local.u64 %SP, %SPL; +; PTX-SHORT-PTR-NEXT: mov.b32 %r1, multiple_grid_const_escape_param_0; +; PTX-SHORT-PTR-NEXT: mov.b32 %r2, multiple_grid_const_escape_param_2; +; PTX-SHORT-PTR-NEXT: cvt.u64.u32 %rd1, %r2; +; PTX-SHORT-PTR-NEXT: cvta.param.u64 %rd2, %rd1; +; PTX-SHORT-PTR-NEXT: ld.param.b32 %r3, [multiple_grid_const_escape_param_1]; +; PTX-SHORT-PTR-NEXT: cvt.u64.u32 %rd3, %r1; +; PTX-SHORT-PTR-NEXT: cvta.param.u64 %rd4, %rd3; +; PTX-SHORT-PTR-NEXT: add.u64 %rd5, %SP, 0; +; PTX-SHORT-PTR-NEXT: add.u64 %rd6, %SPL, 0; +; PTX-SHORT-PTR-NEXT: cvt.u32.u64 %r4, %rd6; +; PTX-SHORT-PTR-NEXT: st.local.b32 [%r4], %r3; +; PTX-SHORT-PTR-NEXT: { // callseq 1, 0 +; PTX-SHORT-PTR-NEXT: .param .b64 param0; +; PTX-SHORT-PTR-NEXT: .param .b64 param1; +; PTX-SHORT-PTR-NEXT: .param .b64 param2; +; PTX-SHORT-PTR-NEXT: .param .b32 retval0; +; PTX-SHORT-PTR-NEXT: st.param.b64 [param2], %rd2; +; PTX-SHORT-PTR-NEXT: st.param.b64 [param1], %rd5; +; PTX-SHORT-PTR-NEXT: st.param.b64 [param0], %rd4; +; PTX-SHORT-PTR-NEXT: mov.b64 %rd7, escape3; +; PTX-SHORT-PTR-NEXT: call (retval0), %rd7, (param0, param1, param2), prototype_1; +; PTX-SHORT-PTR-NEXT: } // callseq 1 +; PTX-SHORT-PTR-NEXT: ret; ; OPT-LABEL: define ptx_kernel void @multiple_grid_const_escape( ; OPT-SAME: ptr addrspace(101) readonly byval([[STRUCT_S:%.*]]) align 4 "nvvm.grid_constant" [[INPUT:%.*]], i32 [[A:%.*]], ptr addrspace(101) readonly byval(i32) align 4 "nvvm.grid_constant" [[B:%.*]]) #[[ATTR0]] { ; OPT-NEXT: [[B_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[B]] to ptr @@ -184,7 +268,6 @@ define ptx_kernel void @multiple_grid_const_escape(ptr byval(%struct.s) align 4 ; OPT-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4 ; OPT-NEXT: [[CALL:%.*]] = call i32 @escape3(ptr [[INPUT_PARAM_GEN]], ptr [[A_ADDR]], ptr [[B_PARAM_GEN]]) ; OPT-NEXT: ret void -; %a.addr = alloca i32, align 4 store i32 %a, ptr %a.addr, align 4 %call = call i32 @escape3(ptr %input, ptr %a.addr, ptr %b) @@ -192,43 +275,74 @@ define ptx_kernel void @multiple_grid_const_escape(ptr byval(%struct.s) align 4 } define ptx_kernel void @grid_const_memory_escape(ptr byval(%struct.s) align 4 "nvvm.grid_constant" %input, ptr %addr) { -; PTX-LABEL: grid_const_memory_escape( -; PTX: { -; PTX-NEXT: .reg .b64 %rd<5>; -; PTX-EMPTY: -; PTX-NEXT: // %bb.0: -; PTX-NEXT: mov.b64 %rd1, grid_const_memory_escape_param_0; -; PTX-NEXT: ld.param.b64 %rd2, [grid_const_memory_escape_param_1]; -; PTX-NEXT: cvta.to.global.u64 %rd3, %rd2; -; PTX-NEXT: cvta.param.u64 %rd4, %rd1; -; PTX-NEXT: st.global.b64 [%rd3], %rd4; -; PTX-NEXT: ret; +; PTX-DEFAULT-LABEL: grid_const_memory_escape( +; PTX-DEFAULT: { +; PTX-DEFAULT-NEXT: .reg .b64 %rd<5>; +; PTX-DEFAULT-EMPTY: +; PTX-DEFAULT-NEXT: // %bb.0: +; PTX-DEFAULT-NEXT: mov.b64 %rd1, grid_const_memory_escape_param_0; +; PTX-DEFAULT-NEXT: ld.param.b64 %rd2, [grid_const_memory_escape_param_1]; +; PTX-DEFAULT-NEXT: cvta.to.global.u64 %rd3, %rd2; +; PTX-DEFAULT-NEXT: cvta.param.u64 %rd4, %rd1; +; PTX-DEFAULT-NEXT: st.global.b64 [%rd3], %rd4; +; PTX-DEFAULT-NEXT: ret; +; +; PTX-SHORT-PTR-LABEL: grid_const_memory_escape( +; PTX-SHORT-PTR: { +; PTX-SHORT-PTR-NEXT: .reg .b32 %r<2>; +; PTX-SHORT-PTR-NEXT: .reg .b64 %rd<5>; +; PTX-SHORT-PTR-EMPTY: +; PTX-SHORT-PTR-NEXT: // %bb.0: +; PTX-SHORT-PTR-NEXT: mov.b32 %r1, grid_const_memory_escape_param_0; +; PTX-SHORT-PTR-NEXT: ld.param.b64 %rd1, [grid_const_memory_escape_param_1]; +; PTX-SHORT-PTR-NEXT: cvta.to.global.u64 %rd2, %rd1; +; PTX-SHORT-PTR-NEXT: cvt.u64.u32 %rd3, %r1; +; PTX-SHORT-PTR-NEXT: cvta.param.u64 %rd4, %rd3; +; PTX-SHORT-PTR-NEXT: st.global.b64 [%rd2], %rd4; +; PTX-SHORT-PTR-NEXT: ret; ; OPT-LABEL: define ptx_kernel void @grid_const_memory_escape( ; OPT-SAME: ptr addrspace(101) readonly byval([[STRUCT_S:%.*]]) align 4 "nvvm.grid_constant" [[INPUT:%.*]], ptr [[ADDR:%.*]]) #[[ATTR0]] { ; OPT-NEXT: [[INPUT1:%.*]] = addrspacecast ptr addrspace(101) [[INPUT]] to ptr ; OPT-NEXT: store ptr [[INPUT1]], ptr [[ADDR]], align 8 ; OPT-NEXT: ret void -; store ptr %input, ptr %addr, align 8 ret void } define ptx_kernel void @grid_const_inlineasm_escape(ptr byval(%struct.s) align 4 "nvvm.grid_constant" %input, ptr %result) { -; PTX-LABEL: grid_const_inlineasm_escape( -; PTX: { -; PTX-NEXT: .reg .b64 %rd<7>; -; PTX-EMPTY: -; PTX-NEXT: // %bb.0: -; PTX-NEXT: mov.b64 %rd4, grid_const_inlineasm_escape_param_0; -; PTX-NEXT: ld.param.b64 %rd5, [grid_const_inlineasm_escape_param_1]; -; PTX-NEXT: cvta.to.global.u64 %rd6, %rd5; -; PTX-NEXT: cvta.param.u64 %rd2, %rd4; -; PTX-NEXT: add.s64 %rd3, %rd2, 4; -; PTX-NEXT: // begin inline asm -; PTX-NEXT: add.s64 %rd1, %rd2, %rd3; -; PTX-NEXT: // end inline asm -; PTX-NEXT: st.global.b64 [%rd6], %rd1; -; PTX-NEXT: ret; +; PTX-DEFAULT-LABEL: grid_const_inlineasm_escape( +; PTX-DEFAULT: { +; PTX-DEFAULT-NEXT: .reg .b64 %rd<7>; +; PTX-DEFAULT-EMPTY: +; PTX-DEFAULT-NEXT: // %bb.0: +; PTX-DEFAULT-NEXT: mov.b64 %rd4, grid_const_inlineasm_escape_param_0; +; PTX-DEFAULT-NEXT: ld.param.b64 %rd5, [grid_const_inlineasm_escape_param_1]; +; PTX-DEFAULT-NEXT: cvta.to.global.u64 %rd6, %rd5; +; PTX-DEFAULT-NEXT: cvta.param.u64 %rd2, %rd4; +; PTX-DEFAULT-NEXT: add.s64 %rd3, %rd2, 4; +; PTX-DEFAULT-NEXT: // begin inline asm +; PTX-DEFAULT-NEXT: add.s64 %rd1, %rd2, %rd3; +; PTX-DEFAULT-NEXT: // end inline asm +; PTX-DEFAULT-NEXT: st.global.b64 [%rd6], %rd1; +; PTX-DEFAULT-NEXT: ret; +; +; PTX-SHORT-PTR-LABEL: grid_const_inlineasm_escape( +; PTX-SHORT-PTR: { +; PTX-SHORT-PTR-NEXT: .reg .b32 %r<2>; +; PTX-SHORT-PTR-NEXT: .reg .b64 %rd<7>; +; PTX-SHORT-PTR-EMPTY: +; PTX-SHORT-PTR-NEXT: // %bb.0: +; PTX-SHORT-PTR-NEXT: mov.b32 %r1, grid_const_inlineasm_escape_param_0; +; PTX-SHORT-PTR-NEXT: ld.param.b64 %rd4, [grid_const_inlineasm_escape_param_1]; +; PTX-SHORT-PTR-NEXT: cvta.to.global.u64 %rd5, %rd4; +; PTX-SHORT-PTR-NEXT: cvt.u64.u32 %rd6, %r1; +; PTX-SHORT-PTR-NEXT: cvta.param.u64 %rd2, %rd6; +; PTX-SHORT-PTR-NEXT: add.s64 %rd3, %rd2, 4; +; PTX-SHORT-PTR-NEXT: // begin inline asm +; PTX-SHORT-PTR-NEXT: add.s64 %rd1, %rd2, %rd3; +; PTX-SHORT-PTR-NEXT: // end inline asm +; PTX-SHORT-PTR-NEXT: st.global.b64 [%rd5], %rd1; +; PTX-SHORT-PTR-NEXT: ret; ; PTX-NOT .local ; OPT-LABEL: define ptx_kernel void @grid_const_inlineasm_escape( ; OPT-SAME: ptr addrspace(101) readonly byval([[STRUCT_S:%.*]]) align 4 "nvvm.grid_constant" [[INPUT:%.*]], ptr [[RESULT:%.*]]) #[[ATTR0]] { @@ -238,7 +352,6 @@ define ptx_kernel void @grid_const_inlineasm_escape(ptr byval(%struct.s) align 4 ; OPT-NEXT: [[TMP2:%.*]] = call i64 asm "add.s64 $0, $1, $2 ; OPT-NEXT: store i64 [[TMP2]], ptr [[RESULT]], align 8 ; OPT-NEXT: ret void -; %tmpptr1 = getelementptr inbounds %struct.s, ptr %input, i32 0, i32 0 %tmpptr2 = getelementptr inbounds %struct.s, ptr %input, i32 0, i32 1 %1 = call i64 asm "add.s64 $0, $1, $2;", "=l,l,l"(ptr %tmpptr1, ptr %tmpptr2) #1 @@ -247,27 +360,50 @@ define ptx_kernel void @grid_const_inlineasm_escape(ptr byval(%struct.s) align 4 } define ptx_kernel void @grid_const_partial_escape(ptr byval(i32) align 4 "nvvm.grid_constant" %input, ptr %output) { -; PTX-LABEL: grid_const_partial_escape( -; PTX: { -; PTX-NEXT: .reg .b32 %r<3>; -; PTX-NEXT: .reg .b64 %rd<6>; -; PTX-NEXT: prototype_2 : .callprototype (.param .b32 _) _ (.param .b64 _); -; PTX-NEXT: // %bb.0: -; PTX-NEXT: mov.b64 %rd1, grid_const_partial_escape_param_0; -; PTX-NEXT: ld.param.b64 %rd2, [grid_const_partial_escape_param_1]; -; PTX-NEXT: cvta.to.global.u64 %rd3, %rd2; -; PTX-NEXT: cvta.param.u64 %rd4, %rd1; -; PTX-NEXT: ld.param.b32 %r1, [grid_const_partial_escape_param_0]; -; PTX-NEXT: add.s32 %r2, %r1, %r1; -; PTX-NEXT: st.global.b32 [%rd3], %r2; -; PTX-NEXT: { // callseq 2, 0 -; PTX-NEXT: .param .b64 param0; -; PTX-NEXT: .param .b32 retval0; -; PTX-NEXT: st.param.b64 [param0], %rd4; -; PTX-NEXT: mov.b64 %rd5, escape; -; PTX-NEXT: call (retval0), %rd5, (param0), prototype_2; -; PTX-NEXT: } // callseq 2 -; PTX-NEXT: ret; +; PTX-DEFAULT-LABEL: grid_const_partial_escape( +; PTX-DEFAULT: { +; PTX-DEFAULT-NEXT: .reg .b32 %r<3>; +; PTX-DEFAULT-NEXT: .reg .b64 %rd<6>; +; PTX-DEFAULT-NEXT: prototype_2 : .callprototype (.param .b32 _) _ (.param .b64 _); +; PTX-DEFAULT-NEXT: // %bb.0: +; PTX-DEFAULT-NEXT: mov.b64 %rd1, grid_const_partial_escape_param_0; +; PTX-DEFAULT-NEXT: ld.param.b64 %rd2, [grid_const_partial_escape_param_1]; +; PTX-DEFAULT-NEXT: cvta.to.global.u64 %rd3, %rd2; +; PTX-DEFAULT-NEXT: cvta.param.u64 %rd4, %rd1; +; PTX-DEFAULT-NEXT: ld.param.b32 %r1, [grid_const_partial_escape_param_0]; +; PTX-DEFAULT-NEXT: add.s32 %r2, %r1, %r1; +; PTX-DEFAULT-NEXT: st.global.b32 [%rd3], %r2; +; PTX-DEFAULT-NEXT: { // callseq 2, 0 +; PTX-DEFAULT-NEXT: .param .b64 param0; +; PTX-DEFAULT-NEXT: .param .b32 retval0; +; PTX-DEFAULT-NEXT: st.param.b64 [param0], %rd4; +; PTX-DEFAULT-NEXT: mov.b64 %rd5, escape; +; PTX-DEFAULT-NEXT: call (retval0), %rd5, (param0), prototype_2; +; PTX-DEFAULT-NEXT: } // callseq 2 +; PTX-DEFAULT-NEXT: ret; +; +; PTX-SHORT-PTR-LABEL: grid_const_partial_escape( +; PTX-SHORT-PTR: { +; PTX-SHORT-PTR-NEXT: .reg .b32 %r<4>; +; PTX-SHORT-PTR-NEXT: .reg .b64 %rd<6>; +; PTX-SHORT-PTR-NEXT: prototype_2 : .callprototype (.param .b32 _) _ (.param .b64 _); +; PTX-SHORT-PTR-NEXT: // %bb.0: +; PTX-SHORT-PTR-NEXT: mov.b32 %r1, grid_const_partial_escape_param_0; +; PTX-SHORT-PTR-NEXT: ld.param.b64 %rd1, [grid_const_partial_escape_param_1]; +; PTX-SHORT-PTR-NEXT: cvta.to.global.u64 %rd2, %rd1; +; PTX-SHORT-PTR-NEXT: cvt.u64.u32 %rd3, %r1; +; PTX-SHORT-PTR-NEXT: cvta.param.u64 %rd4, %rd3; +; PTX-SHORT-PTR-NEXT: ld.param.b32 %r2, [grid_const_partial_escape_param_0]; +; PTX-SHORT-PTR-NEXT: add.s32 %r3, %r2, %r2; +; PTX-SHORT-PTR-NEXT: st.global.b32 [%rd2], %r3; +; PTX-SHORT-PTR-NEXT: { // callseq 2, 0 +; PTX-SHORT-PTR-NEXT: .param .b64 param0; +; PTX-SHORT-PTR-NEXT: .param .b32 retval0; +; PTX-SHORT-PTR-NEXT: st.param.b64 [param0], %rd4; +; PTX-SHORT-PTR-NEXT: mov.b64 %rd5, escape; +; PTX-SHORT-PTR-NEXT: call (retval0), %rd5, (param0), prototype_2; +; PTX-SHORT-PTR-NEXT: } // callseq 2 +; PTX-SHORT-PTR-NEXT: ret; ; OPT-LABEL: define ptx_kernel void @grid_const_partial_escape( ; OPT-SAME: ptr addrspace(101) readonly byval(i32) align 4 "nvvm.grid_constant" [[INPUT:%.*]], ptr [[OUTPUT:%.*]]) #[[ATTR0]] { ; OPT-NEXT: [[INPUT1_GEN:%.*]] = addrspacecast ptr addrspace(101) [[INPUT]] to ptr @@ -276,7 +412,6 @@ define ptx_kernel void @grid_const_partial_escape(ptr byval(i32) align 4 "nvvm.g ; OPT-NEXT: store i32 [[TWICE]], ptr [[OUTPUT]], align 4 ; OPT-NEXT: [[CALL:%.*]] = call i32 @escape(ptr [[INPUT1_GEN]]) ; OPT-NEXT: ret void -; %val = load i32, ptr %input %twice = add i32 %val, %val store i32 %twice, ptr %output @@ -285,29 +420,54 @@ define ptx_kernel void @grid_const_partial_escape(ptr byval(i32) align 4 "nvvm.g } define ptx_kernel i32 @grid_const_partial_escapemem(ptr byval(%struct.s) align 4 "nvvm.grid_constant" %input, ptr %output) { -; PTX-LABEL: grid_const_partial_escapemem( -; PTX: { -; PTX-NEXT: .reg .b32 %r<4>; -; PTX-NEXT: .reg .b64 %rd<6>; -; PTX-NEXT: prototype_3 : .callprototype (.param .b32 _) _ (.param .b64 _); -; PTX-NEXT: // %bb.0: -; PTX-NEXT: mov.b64 %rd1, grid_const_partial_escapemem_param_0; -; PTX-NEXT: ld.param.b64 %rd2, [grid_const_partial_escapemem_param_1]; -; PTX-NEXT: cvta.to.global.u64 %rd3, %rd2; -; PTX-NEXT: cvta.param.u64 %rd4, %rd1; -; PTX-NEXT: ld.param.b32 %r1, [grid_const_partial_escapemem_param_0]; -; PTX-NEXT: ld.param.b32 %r2, [grid_const_partial_escapemem_param_0+4]; -; PTX-NEXT: st.global.b64 [%rd3], %rd4; -; PTX-NEXT: add.s32 %r3, %r1, %r2; -; PTX-NEXT: { // callseq 3, 0 -; PTX-NEXT: .param .b64 param0; -; PTX-NEXT: .param .b32 retval0; -; PTX-NEXT: st.param.b64 [param0], %rd4; -; PTX-NEXT: mov.b64 %rd5, escape; -; PTX-NEXT: call (retval0), %rd5, (param0), prototype_3; -; PTX-NEXT: } // callseq 3 -; PTX-NEXT: st.param.b32 [func_retval0], %r3; -; PTX-NEXT: ret; +; PTX-DEFAULT-LABEL: grid_const_partial_escapemem( +; PTX-DEFAULT: { +; PTX-DEFAULT-NEXT: .reg .b32 %r<4>; +; PTX-DEFAULT-NEXT: .reg .b64 %rd<6>; +; PTX-DEFAULT-NEXT: prototype_3 : .callprototype (.param .b32 _) _ (.param .b64 _); +; PTX-DEFAULT-NEXT: // %bb.0: +; PTX-DEFAULT-NEXT: mov.b64 %rd1, grid_const_partial_escapemem_param_0; +; PTX-DEFAULT-NEXT: ld.param.b64 %rd2, [grid_const_partial_escapemem_param_1]; +; PTX-DEFAULT-NEXT: cvta.to.global.u64 %rd3, %rd2; +; PTX-DEFAULT-NEXT: cvta.param.u64 %rd4, %rd1; +; PTX-DEFAULT-NEXT: ld.param.b32 %r1, [grid_const_partial_escapemem_param_0]; +; PTX-DEFAULT-NEXT: ld.param.b32 %r2, [grid_const_partial_escapemem_param_0+4]; +; PTX-DEFAULT-NEXT: st.global.b64 [%rd3], %rd4; +; PTX-DEFAULT-NEXT: add.s32 %r3, %r1, %r2; +; PTX-DEFAULT-NEXT: { // callseq 3, 0 +; PTX-DEFAULT-NEXT: .param .b64 param0; +; PTX-DEFAULT-NEXT: .param .b32 retval0; +; PTX-DEFAULT-NEXT: st.param.b64 [param0], %rd4; +; PTX-DEFAULT-NEXT: mov.b64 %rd5, escape; +; PTX-DEFAULT-NEXT: call (retval0), %rd5, (param0), prototype_3; +; PTX-DEFAULT-NEXT: } // callseq 3 +; PTX-DEFAULT-NEXT: st.param.b32 [func_retval0], %r3; +; PTX-DEFAULT-NEXT: ret; +; +; PTX-SHORT-PTR-LABEL: grid_const_partial_escapemem( +; PTX-SHORT-PTR: { +; PTX-SHORT-PTR-NEXT: .reg .b32 %r<5>; +; PTX-SHORT-PTR-NEXT: .reg .b64 %rd<6>; +; PTX-SHORT-PTR-NEXT: prototype_3 : .callprototype (.param .b32 _) _ (.param .b64 _); +; PTX-SHORT-PTR-NEXT: // %bb.0: +; PTX-SHORT-PTR-NEXT: mov.b32 %r1, grid_const_partial_escapemem_param_0; +; PTX-SHORT-PTR-NEXT: ld.param.b64 %rd1, [grid_const_partial_escapemem_param_1]; +; PTX-SHORT-PTR-NEXT: cvta.to.global.u64 %rd2, %rd1; +; PTX-SHORT-PTR-NEXT: cvt.u64.u32 %rd3, %r1; +; PTX-SHORT-PTR-NEXT: cvta.param.u64 %rd4, %rd3; +; PTX-SHORT-PTR-NEXT: ld.param.b32 %r2, [grid_const_partial_escapemem_param_0]; +; PTX-SHORT-PTR-NEXT: ld.param.b32 %r3, [grid_const_partial_escapemem_param_0+4]; +; PTX-SHORT-PTR-NEXT: st.global.b64 [%rd2], %rd4; +; PTX-SHORT-PTR-NEXT: add.s32 %r4, %r2, %r3; +; PTX-SHORT-PTR-NEXT: { // callseq 3, 0 +; PTX-SHORT-PTR-NEXT: .param .b64 param0; +; PTX-SHORT-PTR-NEXT: .param .b32 retval0; +; PTX-SHORT-PTR-NEXT: st.param.b64 [param0], %rd4; +; PTX-SHORT-PTR-NEXT: mov.b64 %rd5, escape; +; PTX-SHORT-PTR-NEXT: call (retval0), %rd5, (param0), prototype_3; +; PTX-SHORT-PTR-NEXT: } // callseq 3 +; PTX-SHORT-PTR-NEXT: st.param.b32 [func_retval0], %r4; +; PTX-SHORT-PTR-NEXT: ret; ; OPT-LABEL: define ptx_kernel i32 @grid_const_partial_escapemem( ; OPT-SAME: ptr addrspace(101) readonly byval([[STRUCT_S:%.*]]) align 4 "nvvm.grid_constant" [[INPUT:%.*]], ptr [[OUTPUT:%.*]]) #[[ATTR0]] { ; OPT-NEXT: [[INPUT1:%.*]] = addrspacecast ptr addrspace(101) [[INPUT]] to ptr @@ -319,7 +479,6 @@ define ptx_kernel i32 @grid_const_partial_escapemem(ptr byval(%struct.s) align 4 ; OPT-NEXT: [[ADD:%.*]] = add i32 [[VAL1]], [[VAL2]] ; OPT-NEXT: [[CALL2:%.*]] = call i32 @escape(ptr [[PTR1]]) ; OPT-NEXT: ret i32 [[ADD]] -; %ptr1 = getelementptr inbounds %struct.s, ptr %input, i32 0, i32 0 %val1 = load i32, ptr %ptr1 %ptr2 = getelementptr inbounds %struct.s, ptr %input, i32 0, i32 1 @@ -331,25 +490,45 @@ define ptx_kernel i32 @grid_const_partial_escapemem(ptr byval(%struct.s) align 4 } define ptx_kernel void @grid_const_phi(ptr byval(%struct.s) align 4 "nvvm.grid_constant" %input1, ptr %inout) { -; PTX-LABEL: grid_const_phi( -; PTX: { -; PTX-NEXT: .reg .pred %p<2>; -; PTX-NEXT: .reg .b32 %r<3>; -; PTX-NEXT: .reg .b64 %rd<4>; -; PTX-EMPTY: -; PTX-NEXT: // %bb.0: -; PTX-NEXT: mov.b64 %rd3, grid_const_phi_param_0; -; PTX-NEXT: ld.param.b64 %rd2, [grid_const_phi_param_1]; -; PTX-NEXT: cvta.to.global.u64 %rd1, %rd2; -; PTX-NEXT: ld.global.b32 %r1, [%rd1]; -; PTX-NEXT: setp.lt.s32 %p1, %r1, 0; -; PTX-NEXT: @%p1 bra $L__BB9_2; -; PTX-NEXT: // %bb.1: // %second -; PTX-NEXT: add.s64 %rd3, %rd3, 4; -; PTX-NEXT: $L__BB9_2: // %merge -; PTX-NEXT: ld.param.b32 %r2, [%rd3]; -; PTX-NEXT: st.global.b32 [%rd1], %r2; -; PTX-NEXT: ret; +; PTX-DEFAULT-LABEL: grid_const_phi( +; PTX-DEFAULT: { +; PTX-DEFAULT-NEXT: .reg .pred %p<2>; +; PTX-DEFAULT-NEXT: .reg .b32 %r<3>; +; PTX-DEFAULT-NEXT: .reg .b64 %rd<4>; +; PTX-DEFAULT-EMPTY: +; PTX-DEFAULT-NEXT: // %bb.0: +; PTX-DEFAULT-NEXT: mov.b64 %rd3, grid_const_phi_param_0; +; PTX-DEFAULT-NEXT: ld.param.b64 %rd2, [grid_const_phi_param_1]; +; PTX-DEFAULT-NEXT: cvta.to.global.u64 %rd1, %rd2; +; PTX-DEFAULT-NEXT: ld.global.b32 %r1, [%rd1]; +; PTX-DEFAULT-NEXT: setp.lt.s32 %p1, %r1, 0; +; PTX-DEFAULT-NEXT: @%p1 bra $L__BB9_2; +; PTX-DEFAULT-NEXT: // %bb.1: // %second +; PTX-DEFAULT-NEXT: add.s64 %rd3, %rd3, 4; +; PTX-DEFAULT-NEXT: $L__BB9_2: // %merge +; PTX-DEFAULT-NEXT: ld.param.b32 %r2, [%rd3]; +; PTX-DEFAULT-NEXT: st.global.b32 [%rd1], %r2; +; PTX-DEFAULT-NEXT: ret; +; +; PTX-SHORT-PTR-LABEL: grid_const_phi( +; PTX-SHORT-PTR: { +; PTX-SHORT-PTR-NEXT: .reg .pred %p<2>; +; PTX-SHORT-PTR-NEXT: .reg .b32 %r<4>; +; PTX-SHORT-PTR-NEXT: .reg .b64 %rd<3>; +; PTX-SHORT-PTR-EMPTY: +; PTX-SHORT-PTR-NEXT: // %bb.0: +; PTX-SHORT-PTR-NEXT: mov.b32 %r3, grid_const_phi_param_0; +; PTX-SHORT-PTR-NEXT: ld.param.b64 %rd2, [grid_const_phi_param_1]; +; PTX-SHORT-PTR-NEXT: cvta.to.global.u64 %rd1, %rd2; +; PTX-SHORT-PTR-NEXT: ld.global.b32 %r1, [%rd1]; +; PTX-SHORT-PTR-NEXT: setp.lt.s32 %p1, %r1, 0; +; PTX-SHORT-PTR-NEXT: @%p1 bra $L__BB9_2; +; PTX-SHORT-PTR-NEXT: // %bb.1: // %second +; PTX-SHORT-PTR-NEXT: add.s32 %r3, %r3, 4; +; PTX-SHORT-PTR-NEXT: $L__BB9_2: // %merge +; PTX-SHORT-PTR-NEXT: ld.param.b32 %r2, [%r3]; +; PTX-SHORT-PTR-NEXT: st.global.b32 [%rd1], %r2; +; PTX-SHORT-PTR-NEXT: ret; ; OPT-LABEL: define ptx_kernel void @grid_const_phi( ; OPT-SAME: ptr addrspace(101) readonly byval([[STRUCT_S:%.*]]) align 4 "nvvm.grid_constant" [[INPUT1:%.*]], ptr [[INOUT:%.*]]) #[[ATTR0]] { ; OPT-NEXT: [[INPUT1_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[INPUT1]] to ptr @@ -367,7 +546,6 @@ define ptx_kernel void @grid_const_phi(ptr byval(%struct.s) align 4 "nvvm.grid_c ; OPT-NEXT: [[VALLOADED:%.*]] = load i32, ptr [[PTRNEW]], align 4 ; OPT-NEXT: store i32 [[VALLOADED]], ptr [[INOUT]], align 4 ; OPT-NEXT: ret void -; %val = load i32, ptr %inout %less = icmp slt i32 %val, 0 @@ -387,26 +565,47 @@ merge: ; NOTE: %input2 is *not* grid_constant define ptx_kernel void @grid_const_phi_ngc(ptr byval(%struct.s) align 4 "nvvm.grid_constant" %input1, ptr byval(%struct.s) align 4 %input2, ptr %inout) { -; PTX-LABEL: grid_const_phi_ngc( -; PTX: { -; PTX-NEXT: .reg .pred %p<2>; -; PTX-NEXT: .reg .b32 %r<3>; -; PTX-NEXT: .reg .b64 %rd<5>; -; PTX-EMPTY: -; PTX-NEXT: // %bb.0: -; PTX-NEXT: mov.b64 %rd4, grid_const_phi_ngc_param_0; -; PTX-NEXT: ld.param.b64 %rd3, [grid_const_phi_ngc_param_2]; -; PTX-NEXT: cvta.to.global.u64 %rd1, %rd3; -; PTX-NEXT: ld.global.b32 %r1, [%rd1]; -; PTX-NEXT: setp.lt.s32 %p1, %r1, 0; -; PTX-NEXT: @%p1 bra $L__BB10_2; -; PTX-NEXT: // %bb.1: // %second -; PTX-NEXT: mov.b64 %rd2, grid_const_phi_ngc_param_1; -; PTX-NEXT: add.s64 %rd4, %rd2, 4; -; PTX-NEXT: $L__BB10_2: // %merge -; PTX-NEXT: ld.param.b32 %r2, [%rd4]; -; PTX-NEXT: st.global.b32 [%rd1], %r2; -; PTX-NEXT: ret; +; PTX-DEFAULT-LABEL: grid_const_phi_ngc( +; PTX-DEFAULT: { +; PTX-DEFAULT-NEXT: .reg .pred %p<2>; +; PTX-DEFAULT-NEXT: .reg .b32 %r<3>; +; PTX-DEFAULT-NEXT: .reg .b64 %rd<5>; +; PTX-DEFAULT-EMPTY: +; PTX-DEFAULT-NEXT: // %bb.0: +; PTX-DEFAULT-NEXT: mov.b64 %rd4, grid_const_phi_ngc_param_0; +; PTX-DEFAULT-NEXT: ld.param.b64 %rd3, [grid_const_phi_ngc_param_2]; +; PTX-DEFAULT-NEXT: cvta.to.global.u64 %rd1, %rd3; +; PTX-DEFAULT-NEXT: ld.global.b32 %r1, [%rd1]; +; PTX-DEFAULT-NEXT: setp.lt.s32 %p1, %r1, 0; +; PTX-DEFAULT-NEXT: @%p1 bra $L__BB10_2; +; PTX-DEFAULT-NEXT: // %bb.1: // %second +; PTX-DEFAULT-NEXT: mov.b64 %rd2, grid_const_phi_ngc_param_1; +; PTX-DEFAULT-NEXT: add.s64 %rd4, %rd2, 4; +; PTX-DEFAULT-NEXT: $L__BB10_2: // %merge +; PTX-DEFAULT-NEXT: ld.param.b32 %r2, [%rd4]; +; PTX-DEFAULT-NEXT: st.global.b32 [%rd1], %r2; +; PTX-DEFAULT-NEXT: ret; +; +; PTX-SHORT-PTR-LABEL: grid_const_phi_ngc( +; PTX-SHORT-PTR: { +; PTX-SHORT-PTR-NEXT: .reg .pred %p<2>; +; PTX-SHORT-PTR-NEXT: .reg .b32 %r<5>; +; PTX-SHORT-PTR-NEXT: .reg .b64 %rd<3>; +; PTX-SHORT-PTR-EMPTY: +; PTX-SHORT-PTR-NEXT: // %bb.0: +; PTX-SHORT-PTR-NEXT: mov.b32 %r4, grid_const_phi_ngc_param_0; +; PTX-SHORT-PTR-NEXT: ld.param.b64 %rd2, [grid_const_phi_ngc_param_2]; +; PTX-SHORT-PTR-NEXT: cvta.to.global.u64 %rd1, %rd2; +; PTX-SHORT-PTR-NEXT: ld.global.b32 %r2, [%rd1]; +; PTX-SHORT-PTR-NEXT: setp.lt.s32 %p1, %r2, 0; +; PTX-SHORT-PTR-NEXT: @%p1 bra $L__BB10_2; +; PTX-SHORT-PTR-NEXT: // %bb.1: // %second +; PTX-SHORT-PTR-NEXT: mov.b32 %r1, grid_const_phi_ngc_param_1; +; PTX-SHORT-PTR-NEXT: add.s32 %r4, %r1, 4; +; PTX-SHORT-PTR-NEXT: $L__BB10_2: // %merge +; PTX-SHORT-PTR-NEXT: ld.param.b32 %r3, [%r4]; +; PTX-SHORT-PTR-NEXT: st.global.b32 [%rd1], %r3; +; PTX-SHORT-PTR-NEXT: ret; ; OPT-LABEL: define ptx_kernel void @grid_const_phi_ngc( ; OPT-SAME: ptr addrspace(101) readonly byval([[STRUCT_S:%.*]]) align 4 "nvvm.grid_constant" [[INPUT1:%.*]], ptr addrspace(101) readonly byval([[STRUCT_S]]) align 4 [[INPUT2:%.*]], ptr [[INOUT:%.*]]) #[[ATTR0]] { ; OPT-NEXT: [[INPUT2_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[INPUT2]] to ptr @@ -425,7 +624,6 @@ define ptx_kernel void @grid_const_phi_ngc(ptr byval(%struct.s) align 4 "nvvm.gr ; OPT-NEXT: [[VALLOADED:%.*]] = load i32, ptr [[PTRNEW]], align 4 ; OPT-NEXT: store i32 [[VALLOADED]], ptr [[INOUT]], align 4 ; OPT-NEXT: ret void -; %val = load i32, ptr %inout %less = icmp slt i32 %val, 0 br i1 %less, label %first, label %second @@ -444,23 +642,41 @@ merge: ; NOTE: %input2 is *not* grid_constant define ptx_kernel void @grid_const_select(ptr byval(i32) align 4 "nvvm.grid_constant" %input1, ptr byval(i32) align 4 %input2, ptr %inout) { -; PTX-LABEL: grid_const_select( -; PTX: { -; PTX-NEXT: .reg .pred %p<2>; -; PTX-NEXT: .reg .b32 %r<3>; -; PTX-NEXT: .reg .b64 %rd<6>; -; PTX-EMPTY: -; PTX-NEXT: // %bb.0: -; PTX-NEXT: mov.b64 %rd1, grid_const_select_param_0; -; PTX-NEXT: ld.param.b64 %rd2, [grid_const_select_param_2]; -; PTX-NEXT: cvta.to.global.u64 %rd3, %rd2; -; PTX-NEXT: mov.b64 %rd4, grid_const_select_param_1; -; PTX-NEXT: ld.global.b32 %r1, [%rd3]; -; PTX-NEXT: setp.lt.s32 %p1, %r1, 0; -; PTX-NEXT: selp.b64 %rd5, %rd1, %rd4, %p1; -; PTX-NEXT: ld.param.b32 %r2, [%rd5]; -; PTX-NEXT: st.global.b32 [%rd3], %r2; -; PTX-NEXT: ret; +; PTX-DEFAULT-LABEL: grid_const_select( +; PTX-DEFAULT: { +; PTX-DEFAULT-NEXT: .reg .pred %p<2>; +; PTX-DEFAULT-NEXT: .reg .b32 %r<3>; +; PTX-DEFAULT-NEXT: .reg .b64 %rd<6>; +; PTX-DEFAULT-EMPTY: +; PTX-DEFAULT-NEXT: // %bb.0: +; PTX-DEFAULT-NEXT: mov.b64 %rd1, grid_const_select_param_0; +; PTX-DEFAULT-NEXT: ld.param.b64 %rd2, [grid_const_select_param_2]; +; PTX-DEFAULT-NEXT: cvta.to.global.u64 %rd3, %rd2; +; PTX-DEFAULT-NEXT: mov.b64 %rd4, grid_const_select_param_1; +; PTX-DEFAULT-NEXT: ld.global.b32 %r1, [%rd3]; +; PTX-DEFAULT-NEXT: setp.lt.s32 %p1, %r1, 0; +; PTX-DEFAULT-NEXT: selp.b64 %rd5, %rd1, %rd4, %p1; +; PTX-DEFAULT-NEXT: ld.param.b32 %r2, [%rd5]; +; PTX-DEFAULT-NEXT: st.global.b32 [%rd3], %r2; +; PTX-DEFAULT-NEXT: ret; +; +; PTX-SHORT-PTR-LABEL: grid_const_select( +; PTX-SHORT-PTR: { +; PTX-SHORT-PTR-NEXT: .reg .pred %p<2>; +; PTX-SHORT-PTR-NEXT: .reg .b32 %r<6>; +; PTX-SHORT-PTR-NEXT: .reg .b64 %rd<3>; +; PTX-SHORT-PTR-EMPTY: +; PTX-SHORT-PTR-NEXT: // %bb.0: +; PTX-SHORT-PTR-NEXT: mov.b32 %r1, grid_const_select_param_0; +; PTX-SHORT-PTR-NEXT: ld.param.b64 %rd1, [grid_const_select_param_2]; +; PTX-SHORT-PTR-NEXT: cvta.to.global.u64 %rd2, %rd1; +; PTX-SHORT-PTR-NEXT: mov.b32 %r2, grid_const_select_param_1; +; PTX-SHORT-PTR-NEXT: ld.global.b32 %r3, [%rd2]; +; PTX-SHORT-PTR-NEXT: setp.lt.s32 %p1, %r3, 0; +; PTX-SHORT-PTR-NEXT: selp.b32 %r4, %r1, %r2, %p1; +; PTX-SHORT-PTR-NEXT: ld.param.b32 %r5, [%r4]; +; PTX-SHORT-PTR-NEXT: st.global.b32 [%rd2], %r5; +; PTX-SHORT-PTR-NEXT: ret; ; OPT-LABEL: define ptx_kernel void @grid_const_select( ; OPT-SAME: ptr addrspace(101) readonly byval(i32) align 4 "nvvm.grid_constant" [[INPUT1:%.*]], ptr addrspace(101) readonly byval(i32) align 4 [[INPUT2:%.*]], ptr [[INOUT:%.*]]) #[[ATTR0]] { ; OPT-NEXT: [[INPUT2_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[INPUT2]] to ptr @@ -471,7 +687,6 @@ define ptx_kernel void @grid_const_select(ptr byval(i32) align 4 "nvvm.grid_cons ; OPT-NEXT: [[VALLOADED:%.*]] = load i32, ptr [[PTRNEW]], align 4 ; OPT-NEXT: store i32 [[VALLOADED]], ptr [[INOUT]], align 4 ; OPT-NEXT: ret void -; %val = load i32, ptr %inout %less = icmp slt i32 %val, 0 %ptrnew = select i1 %less, ptr %input1, ptr %input2 @@ -481,19 +696,34 @@ define ptx_kernel void @grid_const_select(ptr byval(i32) align 4 "nvvm.grid_cons } define ptx_kernel i32 @grid_const_ptrtoint(ptr byval(i32) align 4 "nvvm.grid_constant" %input) { -; PTX-LABEL: grid_const_ptrtoint( -; PTX: { -; PTX-NEXT: .reg .b32 %r<4>; -; PTX-NEXT: .reg .b64 %rd<3>; -; PTX-EMPTY: -; PTX-NEXT: // %bb.0: -; PTX-NEXT: mov.b64 %rd1, grid_const_ptrtoint_param_0; -; PTX-NEXT: cvta.param.u64 %rd2, %rd1; -; PTX-NEXT: ld.param.b32 %r1, [grid_const_ptrtoint_param_0]; -; PTX-NEXT: cvt.u32.u64 %r2, %rd2; -; PTX-NEXT: add.s32 %r3, %r1, %r2; -; PTX-NEXT: st.param.b32 [func_retval0], %r3; -; PTX-NEXT: ret; +; PTX-DEFAULT-LABEL: grid_const_ptrtoint( +; PTX-DEFAULT: { +; PTX-DEFAULT-NEXT: .reg .b32 %r<4>; +; PTX-DEFAULT-NEXT: .reg .b64 %rd<3>; +; PTX-DEFAULT-EMPTY: +; PTX-DEFAULT-NEXT: // %bb.0: +; PTX-DEFAULT-NEXT: mov.b64 %rd1, grid_const_ptrtoint_param_0; +; PTX-DEFAULT-NEXT: cvta.param.u64 %rd2, %rd1; +; PTX-DEFAULT-NEXT: ld.param.b32 %r1, [grid_const_ptrtoint_param_0]; +; PTX-DEFAULT-NEXT: cvt.u32.u64 %r2, %rd2; +; PTX-DEFAULT-NEXT: add.s32 %r3, %r1, %r2; +; PTX-DEFAULT-NEXT: st.param.b32 [func_retval0], %r3; +; PTX-DEFAULT-NEXT: ret; +; +; PTX-SHORT-PTR-LABEL: grid_const_ptrtoint( +; PTX-SHORT-PTR: { +; PTX-SHORT-PTR-NEXT: .reg .b32 %r<5>; +; PTX-SHORT-PTR-NEXT: .reg .b64 %rd<3>; +; PTX-SHORT-PTR-EMPTY: +; PTX-SHORT-PTR-NEXT: // %bb.0: +; PTX-SHORT-PTR-NEXT: mov.b32 %r1, grid_const_ptrtoint_param_0; +; PTX-SHORT-PTR-NEXT: cvt.u64.u32 %rd1, %r1; +; PTX-SHORT-PTR-NEXT: cvta.param.u64 %rd2, %rd1; +; PTX-SHORT-PTR-NEXT: ld.param.b32 %r2, [grid_const_ptrtoint_param_0]; +; PTX-SHORT-PTR-NEXT: cvt.u32.u64 %r3, %rd2; +; PTX-SHORT-PTR-NEXT: add.s32 %r4, %r2, %r3; +; PTX-SHORT-PTR-NEXT: st.param.b32 [func_retval0], %r4; +; PTX-SHORT-PTR-NEXT: ret; ; OPT-LABEL: define ptx_kernel i32 @grid_const_ptrtoint( ; OPT-SAME: ptr addrspace(101) readonly byval(i32) align 4 "nvvm.grid_constant" [[INPUT:%.*]]) #[[ATTR0]] { ; OPT-NEXT: [[INPUT1:%.*]] = addrspacecast ptr addrspace(101) [[INPUT]] to ptr @@ -501,7 +731,6 @@ define ptx_kernel i32 @grid_const_ptrtoint(ptr byval(i32) align 4 "nvvm.grid_con ; OPT-NEXT: [[PTRVAL:%.*]] = ptrtoint ptr [[INPUT1]] to i32 ; OPT-NEXT: [[KEEPALIVE:%.*]] = add i32 [[INPUT3]], [[PTRVAL]] ; OPT-NEXT: ret i32 [[KEEPALIVE]] -; %val = load i32, ptr %input %ptrval = ptrtoint ptr %input to i32 %keepalive = add i32 %val, %ptrval diff --git a/llvm/unittests/TargetParser/TripleTest.cpp b/llvm/unittests/TargetParser/TripleTest.cpp index c10d685c0668f..28d541a703417 100644 --- a/llvm/unittests/TargetParser/TripleTest.cpp +++ b/llvm/unittests/TargetParser/TripleTest.cpp @@ -7,6 +7,8 @@ //===----------------------------------------------------------------------===// #include "llvm/TargetParser/Triple.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/FormatVariadic.h" #include "llvm/Support/VersionTuple.h" @@ -3821,6 +3823,36 @@ TEST(TripleTest, DefaultWCharSize) { EXPECT_EQ(1u, Triple("xcore-unknown-unknown").getDefaultWCharSize()); } +TEST(DataLayoutTest, NVPTX) { + Triple TT32 = Triple("nvptx-nvidia-cuda"); + Triple TT64 = Triple("nvptx64-nvidia-cuda"); + + auto PointerLayoutSpecs = [](StringRef DataLayout) { + SmallVector Specs; + for (StringRef Spec : split(DataLayout, '-')) + if (Spec.starts_with("p")) + Specs.emplace_back(Spec); + return Specs; + }; + + // The 32-bit target uses a single 32-bit pointer specification and is + // unaffected by the ABI name. + EXPECT_THAT(PointerLayoutSpecs(TT32.computeDataLayout("")), + testing::ElementsAre("p:32:32")); + EXPECT_THAT(PointerLayoutSpecs(TT32.computeDataLayout("shortptr")), + testing::ElementsAre("p:32:32")); + + // The default 64-bit target only shrinks Tensor Memory (addrspace:6). + EXPECT_THAT(PointerLayoutSpecs(TT64.computeDataLayout("")), + testing::ElementsAre("p6:32:32")); + + // In shortptr mode the extra address spaces become 32-bit. The pointer + // specifications must remain sorted by address space. + EXPECT_THAT(PointerLayoutSpecs(TT64.computeDataLayout("shortptr")), + testing::ElementsAre("p3:32:32", "p4:32:32", "p5:32:32", + "p6:32:32", "p7:32:32", "p101:32:32")); +} + TEST(DataLayoutTest, CheriRISCV32) { Triple TT = Triple("riscv32-unknown-unknown"); From b11881e5daa280d322127d4c51dcf472dbcc1dd5 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Tue, 21 Jul 2026 16:21:20 -0700 Subject: [PATCH 49/50] [BranchFolding] Fold away subsequent identical branches If we have a BB that has a single conditional branch instruction it that is identical to the previous block's branch instruction, we can delete the BB as it is redundant. This doesn't directly impact performance as such instructions are never executed, but this can help decrease code size which can help with overall icache pressure (though likely only slightly). The biggest impact would probably be fitting more instructions into a single cache line. This is probably almost a no-op with PLO but definitely doesn't hurt. Fixes #202763. Reviewers: RKSimon, arsenm, krzysz00, topperc, lei137 Pull Request: https://github.com/llvm/llvm-project/pull/203110 --- llvm/lib/CodeGen/BranchFolding.cpp | 21 ++ llvm/test/CodeGen/AArch64/pr166870.ll | 18 +- .../CodeGen/AMDGPU/mdt-preserving-crash.ll | 30 ++- .../CodeGen/AMDGPU/subreg-coalescer-crash.ll | 4 +- llvm/test/CodeGen/PowerPC/bdzlr.ll | 2 - llvm/test/CodeGen/PowerPC/p10-spill-crun.ll | 92 ++++----- llvm/test/CodeGen/PowerPC/pr45448.ll | 18 +- llvm/test/CodeGen/PowerPC/subreg-postra.ll | 190 +++++++++--------- llvm/test/CodeGen/Thumb2/thumb2-cbnz.ll | 2 +- .../CodeGen/X86/2008-04-17-CoalescerBug.ll | 109 +++++----- .../X86/2009-11-17-UpdateTerminator.ll | 32 ++- .../CodeGen/X86/2011-09-14-valcoalesce.ll | 48 ++--- 12 files changed, 275 insertions(+), 291 deletions(-) diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp index 79fd1e833a4c5..13b52d1a829e3 100644 --- a/llvm/lib/CodeGen/BranchFolding.cpp +++ b/llvm/lib/CodeGen/BranchFolding.cpp @@ -1551,6 +1551,27 @@ bool BranchFolder::OptimizeBlock(MachineBasicBlock *MBB) { } } + // If we have a block that consists of a single conditional branch + // instruction that is exactly identical to the terminator in the previous + // block, we can remove this block. + bool AreConditionalsEqual = + CurCond.size() > 0 && + llvm::equal(CurCond, PriorCond, + [](const MachineOperand &LHS, const MachineOperand &RHS) { + return LHS.isIdenticalTo(RHS); + }); + if (MBB->size() == 1 && PrevBB.canFallThrough() && CurTBB == PriorTBB && + AreConditionalsEqual) { + // We remove the branch from the previous basic block rather than this + // one in case there are other blocks that specifically branch to this + // one. + TII->removeBranch(PrevBB); + PrevBB.removeSuccessor(CurTBB); + MadeChange = true; + ++NumBranchOpts; + goto ReoptimizeBlock; + } + // If this block has no successors (e.g. it is a return block or ends with // a call to a no-return function like abort or __cxa_throw) and if the pred // falls through into this block, and if it would otherwise fall through diff --git a/llvm/test/CodeGen/AArch64/pr166870.ll b/llvm/test/CodeGen/AArch64/pr166870.ll index 4b3daef3b233d..fd597dbec4a90 100644 --- a/llvm/test/CodeGen/AArch64/pr166870.ll +++ b/llvm/test/CodeGen/AArch64/pr166870.ll @@ -26,24 +26,22 @@ define i32 @widget(i32 %arg, i32 %arg1, i1 %arg2, ptr %arg3, i1 %arg4) #0 nounwi ; CHECK-NEXT: mov x22, x1 ; CHECK-NEXT: bl baz ; CHECK-NEXT: mov w8, #1 // =0x1 -; CHECK-NEXT: cbnz w8, .LBB0_9 -; CHECK-NEXT: // %bb.5: // %bb7 -; CHECK-NEXT: cbnz w8, .LBB0_9 -; CHECK-NEXT: // %bb.6: // %bb8 +; CHECK-NEXT: cbnz w8, .LBB0_8 +; CHECK-NEXT: // %bb.5: // %bb8 ; CHECK-NEXT: mov w20, #0 // =0x0 ; CHECK-NEXT: mov w8, w21 ; CHECK-NEXT: mov x21, x8 ; CHECK-NEXT: mov w8, w22 ; CHECK-NEXT: mov x22, x8 -; CHECK-NEXT: .LBB0_7: // %bb10 +; CHECK-NEXT: .LBB0_6: // %bb10 ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: strb w20, [x19] -; CHECK-NEXT: cbnz x21, .LBB0_7 -; CHECK-NEXT: // %bb.8: // %bb12 -; CHECK-NEXT: // in Loop: Header=BB0_7 Depth=1 +; CHECK-NEXT: cbnz x21, .LBB0_6 +; CHECK-NEXT: // %bb.7: // %bb12 +; CHECK-NEXT: // in Loop: Header=BB0_6 Depth=1 ; CHECK-NEXT: bl snork -; CHECK-NEXT: cbnz x22, .LBB0_7 -; CHECK-NEXT: .LBB0_9: +; CHECK-NEXT: cbnz x22, .LBB0_6 +; CHECK-NEXT: .LBB0_8: ; CHECK-NEXT: mov w0, #0 // =0x0 ; CHECK-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload diff --git a/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll b/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll index 3b8106b3c6d2d..62712dc8b8842 100644 --- a/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll +++ b/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll @@ -16,28 +16,24 @@ define protected amdgpu_kernel void @_RSENC_PRInit______________________________ ; CHECK-NEXT: v_lshl_add_u32 v0, v0, 1, v0 ; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, s4, v0 ; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc -; CHECK-NEXT: s_cbranch_execz .LBB0_13 +; CHECK-NEXT: s_cbranch_execz .LBB0_11 ; CHECK-NEXT: ; %bb.1: ; %if.end15 ; CHECK-NEXT: s_load_dword s4, s[8:9], 0x0 ; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: s_bitcmp1_b32 s4, 0 ; CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0 ; CHECK-NEXT: s_and_b64 vcc, exec, s[4:5] -; CHECK-NEXT: s_cbranch_vccnz .LBB0_13 +; CHECK-NEXT: s_cbranch_vccnz .LBB0_11 ; CHECK-NEXT: ; %bb.2: ; %lor.lhs.false17 ; CHECK-NEXT: s_cmp_eq_u32 s4, 0 ; CHECK-NEXT: .LBB0_3: ; %while.cond.i ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: s_cbranch_scc1 .LBB0_3 ; CHECK-NEXT: ; %bb.4: ; %if.end60 -; CHECK-NEXT: s_cbranch_execz .LBB0_12 +; CHECK-NEXT: s_cbranch_execz .LBB0_10 ; CHECK-NEXT: ; %bb.5: ; %if.end5.i -; CHECK-NEXT: s_cbranch_scc0 .LBB0_12 -; CHECK-NEXT: ; %bb.6: ; %if.end5.i314 -; CHECK-NEXT: s_cbranch_scc0 .LBB0_12 -; CHECK-NEXT: ; %bb.7: ; %if.end5.i338 -; CHECK-NEXT: s_cbranch_scc0 .LBB0_12 -; CHECK-NEXT: ; %bb.8: ; %if.end5.i362 +; CHECK-NEXT: s_cbranch_scc0 .LBB0_10 +; CHECK-NEXT: ; %bb.6: ; %if.end5.i362 ; CHECK-NEXT: v_mov_b32_e32 v0, 0 ; CHECK-NEXT: s_getpc_b64 s[4:5] ; CHECK-NEXT: s_add_u32 s4, s4, _RSENC_gDcd_______________________________@rel32@lo+1157 @@ -47,23 +43,23 @@ define protected amdgpu_kernel void @_RSENC_PRInit______________________________ ; CHECK-NEXT: buffer_store_byte v0, v0, s[0:3], 0 offen ; CHECK-NEXT: s_waitcnt vmcnt(1) ; CHECK-NEXT: buffer_store_byte v1, off, s[0:3], 0 offset:257 -; CHECK-NEXT: s_cbranch_scc0 .LBB0_12 -; CHECK-NEXT: ; %bb.9: ; %if.end5.i400 +; CHECK-NEXT: s_cbranch_scc0 .LBB0_10 +; CHECK-NEXT: ; %bb.7: ; %if.end5.i400 ; CHECK-NEXT: flat_load_ubyte v0, v[0:1] ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; CHECK-NEXT: v_cmp_eq_u16_e32 vcc, 0, v0 ; CHECK-NEXT: s_and_b64 exec, exec, vcc -; CHECK-NEXT: s_cbranch_execz .LBB0_12 -; CHECK-NEXT: ; %bb.10: ; %if.then404 +; CHECK-NEXT: s_cbranch_execz .LBB0_10 +; CHECK-NEXT: ; %bb.8: ; %if.then404 ; CHECK-NEXT: s_movk_i32 s4, 0x1000 -; CHECK-NEXT: .LBB0_11: ; %for.body564 +; CHECK-NEXT: .LBB0_9: ; %for.body564 ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: s_sub_i32 s4, s4, 32 ; CHECK-NEXT: s_cmp_lg_u32 s4, 0 -; CHECK-NEXT: s_cbranch_scc1 .LBB0_11 -; CHECK-NEXT: .LBB0_12: ; %UnifiedUnreachableBlock +; CHECK-NEXT: s_cbranch_scc1 .LBB0_9 +; CHECK-NEXT: .LBB0_10: ; %UnifiedUnreachableBlock ; CHECK-NEXT: ; divergent unreachable -; CHECK-NEXT: .LBB0_13: ; %UnifiedReturnBlock +; CHECK-NEXT: .LBB0_11: ; %UnifiedReturnBlock ; CHECK-NEXT: s_endpgm entry: %runtimeVersionCopy = alloca [128 x i8], align 16, addrspace(5) diff --git a/llvm/test/CodeGen/AMDGPU/subreg-coalescer-crash.ll b/llvm/test/CodeGen/AMDGPU/subreg-coalescer-crash.ll index 04d1512ec7f04..150978386f40f 100644 --- a/llvm/test/CodeGen/AMDGPU/subreg-coalescer-crash.ll +++ b/llvm/test/CodeGen/AMDGPU/subreg-coalescer-crash.ll @@ -5,9 +5,7 @@ define amdgpu_kernel void @row_filter_C1_D0() #0 { ; GCN-LABEL: row_filter_C1_D0: ; GCN: ; %bb.0: ; %entry -; GCN-NEXT: s_cbranch_scc1 .LBB0_2 -; GCN-NEXT: ; %bb.1: ; %do.body.preheader -; GCN-NEXT: .LBB0_2: ; %for.inc.1 +; GCN-NEXT: ; %bb.1: ; %for.inc.1 entry: br i1 poison, label %for.inc.1, label %do.body.preheader diff --git a/llvm/test/CodeGen/PowerPC/bdzlr.ll b/llvm/test/CodeGen/PowerPC/bdzlr.ll index e7979bc0c779b..3b04cbe1b9640 100644 --- a/llvm/test/CodeGen/PowerPC/bdzlr.ll +++ b/llvm/test/CodeGen/PowerPC/bdzlr.ll @@ -53,13 +53,11 @@ for.end: ; preds = %for.body, %if.end, ; CHECK: @lua_xmove ; CHECK: bnelr -; CHECK: bnelr ; CHECK: bdzlr ; CHECK-NOT: blr ; CHECK-CRB: @lua_xmove ; CHECK-CRB: bclr 12, -; CHECK-CRB: bclr 12, ; CHECK-CRB: bdzlr ; CHECK-CRB-NOT: blr } diff --git a/llvm/test/CodeGen/PowerPC/p10-spill-crun.ll b/llvm/test/CodeGen/PowerPC/p10-spill-crun.ll index 3c72272209329..b622e469292dd 100644 --- a/llvm/test/CodeGen/PowerPC/p10-spill-crun.ll +++ b/llvm/test/CodeGen/PowerPC/p10-spill-crun.ll @@ -77,13 +77,11 @@ define dso_local void @P10_Spill_CR_UN(ptr %arg, ptr %arg1, i32 %arg2) local_unn ; CHECK-NEXT: srwi r3, r28, 7 ; CHECK-NEXT: andi. r3, r3, 1 ; CHECK-NEXT: crmove 4*cr2+un, gt -; CHECK-NEXT: bc 12, 4*cr2+eq, .LBB0_7 +; CHECK-NEXT: bc 12, 4*cr2+eq, .LBB0_6 ; CHECK-NEXT: # %bb.3: # %bb37 ; CHECK-NEXT: lwz r28, 0(r3) -; CHECK-NEXT: bc 12, 4*cr5+lt, .LBB0_5 -; CHECK-NEXT: # %bb.4: # %bb37 -; CHECK-NEXT: bc 4, 4*cr5+lt, .LBB0_14 -; CHECK-NEXT: .LBB0_5: # %bb42 +; CHECK-NEXT: bc 4, 4*cr5+lt, .LBB0_11 +; CHECK-NEXT: # %bb.4: # %bb42 ; CHECK-NEXT: paddi r3, 0, global_1@PCREL, 1 ; CHECK-NEXT: li r4, 0 ; CHECK-NEXT: cmpwi r28, 0 @@ -96,10 +94,10 @@ define dso_local void @P10_Spill_CR_UN(ptr %arg, ptr %arg1, i32 %arg2) local_unn ; CHECK-NEXT: vextsh2d v2, v2 ; CHECK-NEXT: xscvsxdsp f0, v2 ; CHECK-NEXT: bc 12, 4*cr2+lt, .LBB0_12 -; CHECK-NEXT: # %bb.6: # %bb42 +; CHECK-NEXT: # %bb.5: # %bb42 ; CHECK-NEXT: xxspltidp vs1, 1069547520 ; CHECK-NEXT: b .LBB0_13 -; CHECK-NEXT: .LBB0_7: # %bb19 +; CHECK-NEXT: .LBB0_6: # %bb19 ; CHECK-NEXT: setnbc r3, 4*cr2+un ; CHECK-NEXT: paddi r4, 0, global_4@PCREL, 1 ; CHECK-NEXT: stw r3, 176(r1) @@ -114,36 +112,36 @@ define dso_local void @P10_Spill_CR_UN(ptr %arg, ptr %arg1, i32 %arg2) local_unn ; CHECK-NEXT: cmpwi cr2, r27, 0 ; CHECK-NEXT: mcrf cr3, cr0 ; CHECK-NEXT: .p2align 5 -; CHECK-NEXT: .LBB0_8: # %bb27 +; CHECK-NEXT: .LBB0_7: # %bb27 ; CHECK-NEXT: # ; CHECK-NEXT: mr r3, r30 ; CHECK-NEXT: li r4, 0 ; CHECK-NEXT: bl call_6@notoc -; CHECK-NEXT: bc 4, 4*cr4+eq, .LBB0_18 -; CHECK-NEXT: # %bb.9: # %bb31 +; CHECK-NEXT: bc 4, 4*cr4+eq, .LBB0_17 +; CHECK-NEXT: # %bb.8: # %bb31 ; CHECK-NEXT: # -; CHECK-NEXT: bc 4, 4*cr3+eq, .LBB0_18 -; CHECK-NEXT: # %bb.10: # %bb33 +; CHECK-NEXT: bc 4, 4*cr3+eq, .LBB0_17 +; CHECK-NEXT: # %bb.9: # %bb33 ; CHECK-NEXT: # -; CHECK-NEXT: bc 4, 4*cr2+eq, .LBB0_8 -; CHECK-NEXT: # %bb.11: # %bb36 +; CHECK-NEXT: bc 4, 4*cr2+eq, .LBB0_7 +; CHECK-NEXT: # %bb.10: # %bb36 ; CHECK-NEXT: stb r3, 181(r1) ; CHECK-NEXT: # implicit-def: $cr2un ; CHECK-NEXT: mfocrf r3, 32 ; CHECK-NEXT: lwz r4, 176(r1) ; CHECK-NEXT: rlwimi r3, r4, 21, 11, 11 ; CHECK-NEXT: mtocrf 32, r3 -; CHECK-NEXT: b .LBB0_16 +; CHECK-NEXT: b .LBB0_15 +; CHECK-NEXT: .LBB0_11: # %bb41 +; CHECK-NEXT: # implicit-def: $r3 +; CHECK-NEXT: b .LBB0_14 ; CHECK-NEXT: .LBB0_12: ; CHECK-NEXT: xxspltidp vs1, 1071644672 ; CHECK-NEXT: .LBB0_13: # %bb42 ; CHECK-NEXT: xsmulsp f0, f1, f0 ; CHECK-NEXT: xscvdpsxws f0, f0 ; CHECK-NEXT: mffprwz r3, f0 -; CHECK-NEXT: b .LBB0_15 -; CHECK-NEXT: .LBB0_14: # %bb41 -; CHECK-NEXT: # implicit-def: $r3 -; CHECK-NEXT: .LBB0_15: # %bb50 +; CHECK-NEXT: .LBB0_14: # %bb50 ; CHECK-NEXT: li r4, 0 ; CHECK-NEXT: xxspltidp vs3, -1082130432 ; CHECK-NEXT: xxspltidp vs4, -1082130432 @@ -162,9 +160,9 @@ define dso_local void @P10_Spill_CR_UN(ptr %arg, ptr %arg1, i32 %arg2) local_unn ; CHECK-NEXT: std r4, 112(r1) ; CHECK-NEXT: li r4, 1024 ; CHECK-NEXT: bl call_4@notoc -; CHECK-NEXT: .LBB0_16: # %bb54 -; CHECK-NEXT: bc 12, 4*cr2+un, .LBB0_19 -; CHECK-NEXT: # %bb.17: # %bb56 +; CHECK-NEXT: .LBB0_15: # %bb54 +; CHECK-NEXT: bc 12, 4*cr2+un, .LBB0_18 +; CHECK-NEXT: # %bb.16: # %bb56 ; CHECK-NEXT: ld r30, 208(r1) # 8-byte Folded Reload ; CHECK-NEXT: ld r29, 200(r1) # 8-byte Folded Reload ; CHECK-NEXT: ld r28, 192(r1) # 8-byte Folded Reload @@ -175,9 +173,9 @@ define dso_local void @P10_Spill_CR_UN(ptr %arg, ptr %arg1, i32 %arg2) local_unn ; CHECK-NEXT: mtlr r0 ; CHECK-NEXT: mtcrf 56, r12 ; CHECK-NEXT: blr -; CHECK-NEXT: .LBB0_18: # %bb30 +; CHECK-NEXT: .LBB0_17: # %bb30 ; CHECK-NEXT: stb r3, 181(r1) -; CHECK-NEXT: .LBB0_19: # %bb55 +; CHECK-NEXT: .LBB0_18: # %bb55 ; ; CHECK-BE-LABEL: P10_Spill_CR_UN: ; CHECK-BE: # %bb.0: # %bb @@ -225,14 +223,12 @@ define dso_local void @P10_Spill_CR_UN(ptr %arg, ptr %arg1, i32 %arg2) local_unn ; CHECK-BE-NEXT: srwi r3, r28, 7 ; CHECK-BE-NEXT: andi. r3, r3, 1 ; CHECK-BE-NEXT: crmove 4*cr2+un, gt -; CHECK-BE-NEXT: bc 12, 4*cr2+eq, .LBB0_7 +; CHECK-BE-NEXT: bc 12, 4*cr2+eq, .LBB0_6 ; CHECK-BE-NEXT: # %bb.3: # %bb37 ; CHECK-BE-NEXT: lwz r28, 0(r3) ; CHECK-BE-NEXT: addis r3, r2, global_1@toc@ha -; CHECK-BE-NEXT: bc 12, 4*cr5+lt, .LBB0_5 -; CHECK-BE-NEXT: # %bb.4: # %bb37 -; CHECK-BE-NEXT: bc 4, 4*cr5+lt, .LBB0_14 -; CHECK-BE-NEXT: .LBB0_5: # %bb42 +; CHECK-BE-NEXT: bc 4, 4*cr5+lt, .LBB0_11 +; CHECK-BE-NEXT: # %bb.4: # %bb42 ; CHECK-BE-NEXT: li r4, 0 ; CHECK-BE-NEXT: addi r3, r3, global_1@toc@l ; CHECK-BE-NEXT: cmpwi r28, 0 @@ -247,10 +243,10 @@ define dso_local void @P10_Spill_CR_UN(ptr %arg, ptr %arg1, i32 %arg2) local_unn ; CHECK-BE-NEXT: vextsh2d v2, v2 ; CHECK-BE-NEXT: xscvsxdsp f0, v2 ; CHECK-BE-NEXT: bc 12, 4*cr2+lt, .LBB0_12 -; CHECK-BE-NEXT: # %bb.6: # %bb42 +; CHECK-BE-NEXT: # %bb.5: # %bb42 ; CHECK-BE-NEXT: xxspltidp vs1, 1069547520 ; CHECK-BE-NEXT: b .LBB0_13 -; CHECK-BE-NEXT: .LBB0_7: # %bb19 +; CHECK-BE-NEXT: .LBB0_6: # %bb19 ; CHECK-BE-NEXT: setnbc r3, 4*cr2+un ; CHECK-BE-NEXT: addis r4, r2, global_4@toc@ha ; CHECK-BE-NEXT: stw r3, 192(r1) @@ -270,37 +266,37 @@ define dso_local void @P10_Spill_CR_UN(ptr %arg, ptr %arg1, i32 %arg2) local_unn ; CHECK-BE-NEXT: cmpwi cr2, r27, 0 ; CHECK-BE-NEXT: mcrf cr3, cr0 ; CHECK-BE-NEXT: .p2align 5 -; CHECK-BE-NEXT: .LBB0_8: # %bb27 +; CHECK-BE-NEXT: .LBB0_7: # %bb27 ; CHECK-BE-NEXT: # ; CHECK-BE-NEXT: mr r3, r30 ; CHECK-BE-NEXT: li r4, 0 ; CHECK-BE-NEXT: bl call_6 ; CHECK-BE-NEXT: nop -; CHECK-BE-NEXT: bc 4, 4*cr4+eq, .LBB0_18 -; CHECK-BE-NEXT: # %bb.9: # %bb31 +; CHECK-BE-NEXT: bc 4, 4*cr4+eq, .LBB0_17 +; CHECK-BE-NEXT: # %bb.8: # %bb31 ; CHECK-BE-NEXT: # -; CHECK-BE-NEXT: bc 4, 4*cr3+eq, .LBB0_18 -; CHECK-BE-NEXT: # %bb.10: # %bb33 +; CHECK-BE-NEXT: bc 4, 4*cr3+eq, .LBB0_17 +; CHECK-BE-NEXT: # %bb.9: # %bb33 ; CHECK-BE-NEXT: # -; CHECK-BE-NEXT: bc 4, 4*cr2+eq, .LBB0_8 -; CHECK-BE-NEXT: # %bb.11: # %bb36 +; CHECK-BE-NEXT: bc 4, 4*cr2+eq, .LBB0_7 +; CHECK-BE-NEXT: # %bb.10: # %bb36 ; CHECK-BE-NEXT: stb r3, 197(r1) ; CHECK-BE-NEXT: # implicit-def: $cr2un ; CHECK-BE-NEXT: mfocrf r3, 32 ; CHECK-BE-NEXT: lwz r4, 192(r1) ; CHECK-BE-NEXT: rlwimi r3, r4, 21, 11, 11 ; CHECK-BE-NEXT: mtocrf 32, r3 -; CHECK-BE-NEXT: b .LBB0_16 +; CHECK-BE-NEXT: b .LBB0_15 +; CHECK-BE-NEXT: .LBB0_11: # %bb41 +; CHECK-BE-NEXT: # implicit-def: $r3 +; CHECK-BE-NEXT: b .LBB0_14 ; CHECK-BE-NEXT: .LBB0_12: ; CHECK-BE-NEXT: xxspltidp vs1, 1071644672 ; CHECK-BE-NEXT: .LBB0_13: # %bb42 ; CHECK-BE-NEXT: xsmulsp f0, f1, f0 ; CHECK-BE-NEXT: xscvdpsxws f0, f0 ; CHECK-BE-NEXT: mffprwz r3, f0 -; CHECK-BE-NEXT: b .LBB0_15 -; CHECK-BE-NEXT: .LBB0_14: # %bb41 -; CHECK-BE-NEXT: # implicit-def: $r3 -; CHECK-BE-NEXT: .LBB0_15: # %bb50 +; CHECK-BE-NEXT: .LBB0_14: # %bb50 ; CHECK-BE-NEXT: li r4, 0 ; CHECK-BE-NEXT: xxspltidp vs3, -1082130432 ; CHECK-BE-NEXT: xxspltidp vs4, -1082130432 @@ -320,9 +316,9 @@ define dso_local void @P10_Spill_CR_UN(ptr %arg, ptr %arg1, i32 %arg2) local_unn ; CHECK-BE-NEXT: li r4, 1024 ; CHECK-BE-NEXT: bl call_4 ; CHECK-BE-NEXT: nop -; CHECK-BE-NEXT: .LBB0_16: # %bb54 -; CHECK-BE-NEXT: bc 12, 4*cr2+un, .LBB0_19 -; CHECK-BE-NEXT: # %bb.17: # %bb56 +; CHECK-BE-NEXT: .LBB0_15: # %bb54 +; CHECK-BE-NEXT: bc 12, 4*cr2+un, .LBB0_18 +; CHECK-BE-NEXT: # %bb.16: # %bb56 ; CHECK-BE-NEXT: ld r30, 224(r1) # 8-byte Folded Reload ; CHECK-BE-NEXT: ld r29, 216(r1) # 8-byte Folded Reload ; CHECK-BE-NEXT: ld r28, 208(r1) # 8-byte Folded Reload @@ -333,9 +329,9 @@ define dso_local void @P10_Spill_CR_UN(ptr %arg, ptr %arg1, i32 %arg2) local_unn ; CHECK-BE-NEXT: mtlr r0 ; CHECK-BE-NEXT: mtcrf 56, r12 ; CHECK-BE-NEXT: blr -; CHECK-BE-NEXT: .LBB0_18: # %bb30 +; CHECK-BE-NEXT: .LBB0_17: # %bb30 ; CHECK-BE-NEXT: stb r3, 197(r1) -; CHECK-BE-NEXT: .LBB0_19: # %bb55 +; CHECK-BE-NEXT: .LBB0_18: # %bb55 bb: %tmp = alloca [3 x i8], align 1 %tmp3 = tail call zeroext i8 @call_1(ptr %arg1) diff --git a/llvm/test/CodeGen/PowerPC/pr45448.ll b/llvm/test/CodeGen/PowerPC/pr45448.ll index 0edbae47e9378..3ed059ee7d6ba 100644 --- a/llvm/test/CodeGen/PowerPC/pr45448.ll +++ b/llvm/test/CodeGen/PowerPC/pr45448.ll @@ -7,18 +7,16 @@ define hidden void @julia_tryparse_internal_45896() #0 { ; CHECK: # %bb.0: # %top ; CHECK-NEXT: ld r3, 0(r3) ; CHECK-NEXT: cmpldi r3, 0 -; CHECK-NEXT: beq cr0, .LBB0_6 +; CHECK-NEXT: beq cr0, .LBB0_5 ; CHECK-NEXT: # %bb.1: # %top ; CHECK-NEXT: cmpldi r3, 10 ; CHECK-NEXT: beq cr0, .LBB0_3 ; CHECK-NEXT: # %bb.2: # %top ; CHECK-NEXT: .LBB0_3: # %L294 -; CHECK-NEXT: bc 12, 4*cr5+lt, .LBB0_5 -; CHECK-NEXT: # %bb.4: # %L294 -; CHECK-NEXT: bc 4, 4*cr5+lt, .LBB0_7 -; CHECK-NEXT: .LBB0_5: # %L1057.preheader -; CHECK-NEXT: .LBB0_6: # %fail194 -; CHECK-NEXT: .LBB0_7: # %L670 +; CHECK-NEXT: bc 4, 4*cr5+lt, .LBB0_6 +; CHECK-NEXT: # %bb.4: # %L1057.preheader +; CHECK-NEXT: .LBB0_5: # %fail194 +; CHECK-NEXT: .LBB0_6: # %L670 ; CHECK-NEXT: li r5, -3 ; CHECK-NEXT: sradi r4, r3, 63 ; CHECK-NEXT: rldic r5, r5, 4, 32 @@ -29,9 +27,9 @@ define hidden void @julia_tryparse_internal_45896() #0 { ; CHECK-NEXT: li r3, 0 ; CHECK-NEXT: addze r3, r3 ; CHECK-NEXT: or. r3, r4, r3 -; CHECK-NEXT: beq cr0, .LBB0_9 -; CHECK-NEXT: # %bb.8: # %L917 -; CHECK-NEXT: .LBB0_9: # %L994 +; CHECK-NEXT: beq cr0, .LBB0_8 +; CHECK-NEXT: # %bb.7: # %L917 +; CHECK-NEXT: .LBB0_8: # %L994 top: %0 = load i64, ptr undef, align 8 %1 = icmp ne i64 %0, 0 diff --git a/llvm/test/CodeGen/PowerPC/subreg-postra.ll b/llvm/test/CodeGen/PowerPC/subreg-postra.ll index a315da545ba0f..38d259aad5c07 100644 --- a/llvm/test/CodeGen/PowerPC/subreg-postra.ll +++ b/llvm/test/CodeGen/PowerPC/subreg-postra.ll @@ -25,54 +25,52 @@ define void @jbd2_journal_commit_transaction(ptr %journal, i64 %inp1, i32 %inp2, ; CHECK-NEXT: andi. 6, 10, 1 ; CHECK-NEXT: crmove 8, 1 ; CHECK-NEXT: andi. 6, 9, 1 -; CHECK-NEXT: bc 4, 20, .LBB0_24 +; CHECK-NEXT: bc 4, 20, .LBB0_23 ; CHECK-NEXT: # %bb.1: # %do.body -; CHECK-NEXT: bc 4, 20, .LBB0_25 +; CHECK-NEXT: bc 4, 20, .LBB0_24 ; CHECK-NEXT: # %bb.2: # %trace_jbd2_start_commit.exit ; CHECK-NEXT: mr 30, 8 ; CHECK-NEXT: mr 29, 7 -; CHECK-NEXT: bc 12, 20, .LBB0_4 -; CHECK-NEXT: # %bb.3: # %do.body.i1116 +; CHECK-NEXT: bc 4, 20, .LBB0_25 +; CHECK-NEXT: # %bb.3: # %trace_jbd2_commit_locking.exit ; CHECK-NEXT: bc 4, 20, .LBB0_26 -; CHECK-NEXT: .LBB0_4: # %trace_jbd2_commit_locking.exit +; CHECK-NEXT: # %bb.4: # %spin_unlock.exit1146 ; CHECK-NEXT: bc 4, 20, .LBB0_27 -; CHECK-NEXT: # %bb.5: # %spin_unlock.exit1146 +; CHECK-NEXT: # %bb.5: # %trace_jbd2_commit_flushing.exit ; CHECK-NEXT: bc 4, 20, .LBB0_28 -; CHECK-NEXT: # %bb.6: # %trace_jbd2_commit_flushing.exit -; CHECK-NEXT: bc 4, 20, .LBB0_29 -; CHECK-NEXT: # %bb.7: # %for.end.i +; CHECK-NEXT: # %bb.6: # %for.end.i +; CHECK-NEXT: bc 4, 20, .LBB0_30 +; CHECK-NEXT: # %bb.7: # %journal_submit_data_buffers.exit ; CHECK-NEXT: bc 4, 20, .LBB0_31 -; CHECK-NEXT: # %bb.8: # %journal_submit_data_buffers.exit +; CHECK-NEXT: # %bb.8: # %if.end103 ; CHECK-NEXT: bc 4, 20, .LBB0_32 -; CHECK-NEXT: # %bb.9: # %if.end103 +; CHECK-NEXT: # %bb.9: # %trace_jbd2_commit_logging.exit ; CHECK-NEXT: bc 4, 20, .LBB0_33 -; CHECK-NEXT: # %bb.10: # %trace_jbd2_commit_logging.exit +; CHECK-NEXT: # %bb.10: # %for.end.i1287 ; CHECK-NEXT: bc 4, 20, .LBB0_34 -; CHECK-NEXT: # %bb.11: # %for.end.i1287 +; CHECK-NEXT: # %bb.11: # %journal_finish_inode_data_buffers.exit ; CHECK-NEXT: bc 4, 20, .LBB0_35 -; CHECK-NEXT: # %bb.12: # %journal_finish_inode_data_buffers.exit -; CHECK-NEXT: bc 4, 20, .LBB0_36 -; CHECK-NEXT: # %bb.13: # %if.end256 +; CHECK-NEXT: # %bb.12: # %if.end256 ; CHECK-NEXT: cmpdi 1, 4, 0 ; CHECK-NEXT: .p2align 4 -; CHECK-NEXT: .LBB0_14: # %while.body318 +; CHECK-NEXT: .LBB0_13: # %while.body318 ; CHECK-NEXT: # -; CHECK-NEXT: bc 4, 6, .LBB0_19 -; CHECK-NEXT: # %bb.15: # %wait_on_buffer.exit +; CHECK-NEXT: bc 4, 6, .LBB0_18 +; CHECK-NEXT: # %bb.14: # %wait_on_buffer.exit ; CHECK-NEXT: # -; CHECK-NEXT: bc 4, 1, .LBB0_14 -; CHECK-NEXT: # %bb.16: # %do.body378 -; CHECK-NEXT: bc 4, 8, .LBB0_20 -; CHECK-NEXT: # %bb.17: # %while.end418 -; CHECK-NEXT: bc 4, 8, .LBB0_23 -; CHECK-NEXT: .LBB0_18: # %if.end421 -; CHECK-NEXT: .LBB0_19: # %if.then.i1296 -; CHECK-NEXT: .LBB0_20: # %while.body392.lr.ph +; CHECK-NEXT: bc 4, 1, .LBB0_13 +; CHECK-NEXT: # %bb.15: # %do.body378 +; CHECK-NEXT: bc 4, 8, .LBB0_19 +; CHECK-NEXT: # %bb.16: # %while.end418 +; CHECK-NEXT: bc 4, 8, .LBB0_22 +; CHECK-NEXT: .LBB0_17: # %if.end421 +; CHECK-NEXT: .LBB0_18: # %if.then.i1296 +; CHECK-NEXT: .LBB0_19: # %while.body392.lr.ph ; CHECK-NEXT: lis 26, 4 ; CHECK-NEXT: mr 27, 5 ; CHECK-NEXT: mr 28, 3 ; CHECK-NEXT: .p2align 4 -; CHECK-NEXT: .LBB0_21: # %while.body392 +; CHECK-NEXT: .LBB0_20: # %while.body392 ; CHECK-NEXT: # ; CHECK-NEXT: ld 3, 0(3) ; CHECK-NEXT: ldu 25, -72(3) @@ -88,33 +86,33 @@ define void @jbd2_journal_commit_transaction(ptr %journal, i64 %inp1, i32 %inp2, ; CHECK-NEXT: std 3, 0(30) ; CHECK-NEXT: bl __brelse ; CHECK-NEXT: nop -; CHECK-NEXT: bc 4, 9, .LBB0_21 -; CHECK-NEXT: # %bb.22: # %while.end418.loopexit +; CHECK-NEXT: bc 4, 9, .LBB0_20 +; CHECK-NEXT: # %bb.21: # %while.end418.loopexit ; CHECK-NEXT: andi. 3, 25, 1 ; CHECK-NEXT: li 3, -5 ; CHECK-NEXT: mr 5, 27 ; CHECK-NEXT: iselgt 5, 5, 3 ; CHECK-NEXT: mr 3, 28 -; CHECK-NEXT: bc 12, 8, .LBB0_18 -; CHECK-NEXT: .LBB0_23: # %if.then420 +; CHECK-NEXT: bc 12, 8, .LBB0_17 +; CHECK-NEXT: .LBB0_22: # %if.then420 ; CHECK-NEXT: extsw 4, 5 ; CHECK-NEXT: bl jbd2_journal_abort ; CHECK-NEXT: nop -; CHECK-NEXT: .LBB0_24: # %if.then5 -; CHECK-NEXT: .LBB0_25: # %do.body.i -; CHECK-NEXT: .LBB0_26: # %do.body5.i1122 -; CHECK-NEXT: .LBB0_27: # %if.then.i.i.i.i1144 -; CHECK-NEXT: .LBB0_28: # %do.body.i1159 -; CHECK-NEXT: .LBB0_29: # %for.body.lr.ph.i -; CHECK-NEXT: bc 4, 20, .LBB0_37 -; CHECK-NEXT: # %bb.30: # %spin_unlock.exit.i -; CHECK-NEXT: .LBB0_31: # %if.then.i.i.i.i31.i -; CHECK-NEXT: .LBB0_32: # %if.then102 -; CHECK-NEXT: .LBB0_33: # %do.body.i1182 -; CHECK-NEXT: .LBB0_34: # %for.body.i1277 -; CHECK-NEXT: .LBB0_35: # %if.then.i.i.i.i84.i -; CHECK-NEXT: .LBB0_36: # %if.then249 -; CHECK-NEXT: .LBB0_37: # %if.then.i.i.i.i.i +; CHECK-NEXT: .LBB0_23: # %if.then5 +; CHECK-NEXT: .LBB0_24: # %do.body.i +; CHECK-NEXT: .LBB0_25: # %do.body5.i1122 +; CHECK-NEXT: .LBB0_26: # %if.then.i.i.i.i1144 +; CHECK-NEXT: .LBB0_27: # %do.body.i1159 +; CHECK-NEXT: .LBB0_28: # %for.body.lr.ph.i +; CHECK-NEXT: bc 4, 20, .LBB0_36 +; CHECK-NEXT: # %bb.29: # %spin_unlock.exit.i +; CHECK-NEXT: .LBB0_30: # %if.then.i.i.i.i31.i +; CHECK-NEXT: .LBB0_31: # %if.then102 +; CHECK-NEXT: .LBB0_32: # %do.body.i1182 +; CHECK-NEXT: .LBB0_33: # %for.body.i1277 +; CHECK-NEXT: .LBB0_34: # %if.then.i.i.i.i84.i +; CHECK-NEXT: .LBB0_35: # %if.then249 +; CHECK-NEXT: .LBB0_36: # %if.then.i.i.i.i.i ; ; CHECK-NO-ISEL-LABEL: jbd2_journal_commit_transaction: ; CHECK-NO-ISEL: # %bb.0: # %entry @@ -135,54 +133,52 @@ define void @jbd2_journal_commit_transaction(ptr %journal, i64 %inp1, i32 %inp2, ; CHECK-NO-ISEL-NEXT: andi. 6, 10, 1 ; CHECK-NO-ISEL-NEXT: crmove 8, 1 ; CHECK-NO-ISEL-NEXT: andi. 6, 9, 1 -; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_26 +; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_25 ; CHECK-NO-ISEL-NEXT: # %bb.1: # %do.body -; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_27 +; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_26 ; CHECK-NO-ISEL-NEXT: # %bb.2: # %trace_jbd2_start_commit.exit ; CHECK-NO-ISEL-NEXT: mr 30, 8 ; CHECK-NO-ISEL-NEXT: mr 29, 7 -; CHECK-NO-ISEL-NEXT: bc 12, 20, .LBB0_4 -; CHECK-NO-ISEL-NEXT: # %bb.3: # %do.body.i1116 +; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_27 +; CHECK-NO-ISEL-NEXT: # %bb.3: # %trace_jbd2_commit_locking.exit ; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_28 -; CHECK-NO-ISEL-NEXT: .LBB0_4: # %trace_jbd2_commit_locking.exit +; CHECK-NO-ISEL-NEXT: # %bb.4: # %spin_unlock.exit1146 ; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_29 -; CHECK-NO-ISEL-NEXT: # %bb.5: # %spin_unlock.exit1146 +; CHECK-NO-ISEL-NEXT: # %bb.5: # %trace_jbd2_commit_flushing.exit ; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_30 -; CHECK-NO-ISEL-NEXT: # %bb.6: # %trace_jbd2_commit_flushing.exit -; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_31 -; CHECK-NO-ISEL-NEXT: # %bb.7: # %for.end.i +; CHECK-NO-ISEL-NEXT: # %bb.6: # %for.end.i +; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_32 +; CHECK-NO-ISEL-NEXT: # %bb.7: # %journal_submit_data_buffers.exit ; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_33 -; CHECK-NO-ISEL-NEXT: # %bb.8: # %journal_submit_data_buffers.exit +; CHECK-NO-ISEL-NEXT: # %bb.8: # %if.end103 ; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_34 -; CHECK-NO-ISEL-NEXT: # %bb.9: # %if.end103 +; CHECK-NO-ISEL-NEXT: # %bb.9: # %trace_jbd2_commit_logging.exit ; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_35 -; CHECK-NO-ISEL-NEXT: # %bb.10: # %trace_jbd2_commit_logging.exit +; CHECK-NO-ISEL-NEXT: # %bb.10: # %for.end.i1287 ; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_36 -; CHECK-NO-ISEL-NEXT: # %bb.11: # %for.end.i1287 +; CHECK-NO-ISEL-NEXT: # %bb.11: # %journal_finish_inode_data_buffers.exit ; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_37 -; CHECK-NO-ISEL-NEXT: # %bb.12: # %journal_finish_inode_data_buffers.exit -; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_38 -; CHECK-NO-ISEL-NEXT: # %bb.13: # %if.end256 +; CHECK-NO-ISEL-NEXT: # %bb.12: # %if.end256 ; CHECK-NO-ISEL-NEXT: cmpdi 1, 4, 0 ; CHECK-NO-ISEL-NEXT: .p2align 4 -; CHECK-NO-ISEL-NEXT: .LBB0_14: # %while.body318 +; CHECK-NO-ISEL-NEXT: .LBB0_13: # %while.body318 ; CHECK-NO-ISEL-NEXT: # -; CHECK-NO-ISEL-NEXT: bc 4, 6, .LBB0_19 -; CHECK-NO-ISEL-NEXT: # %bb.15: # %wait_on_buffer.exit +; CHECK-NO-ISEL-NEXT: bc 4, 6, .LBB0_18 +; CHECK-NO-ISEL-NEXT: # %bb.14: # %wait_on_buffer.exit ; CHECK-NO-ISEL-NEXT: # -; CHECK-NO-ISEL-NEXT: bc 4, 1, .LBB0_14 -; CHECK-NO-ISEL-NEXT: # %bb.16: # %do.body378 -; CHECK-NO-ISEL-NEXT: bc 4, 8, .LBB0_20 -; CHECK-NO-ISEL-NEXT: # %bb.17: # %while.end418 -; CHECK-NO-ISEL-NEXT: bc 4, 8, .LBB0_25 -; CHECK-NO-ISEL-NEXT: .LBB0_18: # %if.end421 -; CHECK-NO-ISEL-NEXT: .LBB0_19: # %if.then.i1296 -; CHECK-NO-ISEL-NEXT: .LBB0_20: # %while.body392.lr.ph +; CHECK-NO-ISEL-NEXT: bc 4, 1, .LBB0_13 +; CHECK-NO-ISEL-NEXT: # %bb.15: # %do.body378 +; CHECK-NO-ISEL-NEXT: bc 4, 8, .LBB0_19 +; CHECK-NO-ISEL-NEXT: # %bb.16: # %while.end418 +; CHECK-NO-ISEL-NEXT: bc 4, 8, .LBB0_24 +; CHECK-NO-ISEL-NEXT: .LBB0_17: # %if.end421 +; CHECK-NO-ISEL-NEXT: .LBB0_18: # %if.then.i1296 +; CHECK-NO-ISEL-NEXT: .LBB0_19: # %while.body392.lr.ph ; CHECK-NO-ISEL-NEXT: lis 26, 4 ; CHECK-NO-ISEL-NEXT: mr 27, 5 ; CHECK-NO-ISEL-NEXT: mr 28, 3 ; CHECK-NO-ISEL-NEXT: .p2align 4 -; CHECK-NO-ISEL-NEXT: .LBB0_21: # %while.body392 +; CHECK-NO-ISEL-NEXT: .LBB0_20: # %while.body392 ; CHECK-NO-ISEL-NEXT: # ; CHECK-NO-ISEL-NEXT: ld 3, 0(3) ; CHECK-NO-ISEL-NEXT: ldu 25, -72(3) @@ -198,35 +194,35 @@ define void @jbd2_journal_commit_transaction(ptr %journal, i64 %inp1, i32 %inp2, ; CHECK-NO-ISEL-NEXT: std 3, 0(30) ; CHECK-NO-ISEL-NEXT: bl __brelse ; CHECK-NO-ISEL-NEXT: nop -; CHECK-NO-ISEL-NEXT: bc 4, 9, .LBB0_21 -; CHECK-NO-ISEL-NEXT: # %bb.22: # %while.end418.loopexit +; CHECK-NO-ISEL-NEXT: bc 4, 9, .LBB0_20 +; CHECK-NO-ISEL-NEXT: # %bb.21: # %while.end418.loopexit ; CHECK-NO-ISEL-NEXT: andi. 3, 25, 1 ; CHECK-NO-ISEL-NEXT: mr 5, 27 -; CHECK-NO-ISEL-NEXT: bc 12, 1, .LBB0_24 -; CHECK-NO-ISEL-NEXT: # %bb.23: # %while.end418.loopexit +; CHECK-NO-ISEL-NEXT: bc 12, 1, .LBB0_23 +; CHECK-NO-ISEL-NEXT: # %bb.22: # %while.end418.loopexit ; CHECK-NO-ISEL-NEXT: li 5, -5 -; CHECK-NO-ISEL-NEXT: .LBB0_24: # %while.end418.loopexit +; CHECK-NO-ISEL-NEXT: .LBB0_23: # %while.end418.loopexit ; CHECK-NO-ISEL-NEXT: mr 3, 28 -; CHECK-NO-ISEL-NEXT: bc 12, 8, .LBB0_18 -; CHECK-NO-ISEL-NEXT: .LBB0_25: # %if.then420 +; CHECK-NO-ISEL-NEXT: bc 12, 8, .LBB0_17 +; CHECK-NO-ISEL-NEXT: .LBB0_24: # %if.then420 ; CHECK-NO-ISEL-NEXT: extsw 4, 5 ; CHECK-NO-ISEL-NEXT: bl jbd2_journal_abort ; CHECK-NO-ISEL-NEXT: nop -; CHECK-NO-ISEL-NEXT: .LBB0_26: # %if.then5 -; CHECK-NO-ISEL-NEXT: .LBB0_27: # %do.body.i -; CHECK-NO-ISEL-NEXT: .LBB0_28: # %do.body5.i1122 -; CHECK-NO-ISEL-NEXT: .LBB0_29: # %if.then.i.i.i.i1144 -; CHECK-NO-ISEL-NEXT: .LBB0_30: # %do.body.i1159 -; CHECK-NO-ISEL-NEXT: .LBB0_31: # %for.body.lr.ph.i -; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_39 -; CHECK-NO-ISEL-NEXT: # %bb.32: # %spin_unlock.exit.i -; CHECK-NO-ISEL-NEXT: .LBB0_33: # %if.then.i.i.i.i31.i -; CHECK-NO-ISEL-NEXT: .LBB0_34: # %if.then102 -; CHECK-NO-ISEL-NEXT: .LBB0_35: # %do.body.i1182 -; CHECK-NO-ISEL-NEXT: .LBB0_36: # %for.body.i1277 -; CHECK-NO-ISEL-NEXT: .LBB0_37: # %if.then.i.i.i.i84.i -; CHECK-NO-ISEL-NEXT: .LBB0_38: # %if.then249 -; CHECK-NO-ISEL-NEXT: .LBB0_39: # %if.then.i.i.i.i.i +; CHECK-NO-ISEL-NEXT: .LBB0_25: # %if.then5 +; CHECK-NO-ISEL-NEXT: .LBB0_26: # %do.body.i +; CHECK-NO-ISEL-NEXT: .LBB0_27: # %do.body5.i1122 +; CHECK-NO-ISEL-NEXT: .LBB0_28: # %if.then.i.i.i.i1144 +; CHECK-NO-ISEL-NEXT: .LBB0_29: # %do.body.i1159 +; CHECK-NO-ISEL-NEXT: .LBB0_30: # %for.body.lr.ph.i +; CHECK-NO-ISEL-NEXT: bc 4, 20, .LBB0_38 +; CHECK-NO-ISEL-NEXT: # %bb.31: # %spin_unlock.exit.i +; CHECK-NO-ISEL-NEXT: .LBB0_32: # %if.then.i.i.i.i31.i +; CHECK-NO-ISEL-NEXT: .LBB0_33: # %if.then102 +; CHECK-NO-ISEL-NEXT: .LBB0_34: # %do.body.i1182 +; CHECK-NO-ISEL-NEXT: .LBB0_35: # %for.body.i1277 +; CHECK-NO-ISEL-NEXT: .LBB0_36: # %if.then.i.i.i.i84.i +; CHECK-NO-ISEL-NEXT: .LBB0_37: # %if.then249 +; CHECK-NO-ISEL-NEXT: .LBB0_38: # %if.then.i.i.i.i.i ptr %inp3, ptr %inp4, ptr %inp5, i1 %inp6, i1 %inp7, i1 %inp8) #0 { diff --git a/llvm/test/CodeGen/Thumb2/thumb2-cbnz.ll b/llvm/test/CodeGen/Thumb2/thumb2-cbnz.ll index 6ff9470483deb..e9cd167cef60b 100644 --- a/llvm/test/CodeGen/Thumb2/thumb2-cbnz.ll +++ b/llvm/test/CodeGen/Thumb2/thumb2-cbnz.ll @@ -5,7 +5,7 @@ declare double @foo(double) nounwind readnone define void @t(i32 %c, double %b) { entry: -; CHECK: cmp r0, #0 +; CHECK: cbz %cmp1 = icmp ne i32 %c, 0 br i1 %cmp1, label %bb3, label %bb1 diff --git a/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll b/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll index 3913e93b83a66..e4c9bbef81b2e 100644 --- a/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll +++ b/llvm/test/CodeGen/X86/2008-04-17-CoalescerBug.ll @@ -34,74 +34,69 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(ptr noalias sret(%struct ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %esi ; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %ebx ; CHECK-NEXT: testb $1, %bl -; CHECK-NEXT: je LBB0_25 -; CHECK-NEXT: ## %bb.1: ## %bb116.i -; CHECK-NEXT: je LBB0_25 -; CHECK-NEXT: ## %bb.2: ## %bb52.i.i -; CHECK-NEXT: je LBB0_25 -; CHECK-NEXT: ## %bb.3: ## %bb142.i -; CHECK-NEXT: je LBB0_25 -; CHECK-NEXT: ## %bb.4: +; CHECK-NEXT: je LBB0_22 +; CHECK-NEXT: ## %bb.1: ; CHECK-NEXT: movl L_.str89$non_lazy_ptr, %edi ; CHECK-NEXT: movb $1, %bh ; CHECK-NEXT: movl L_.str$non_lazy_ptr, %ebp -; CHECK-NEXT: jmp LBB0_5 -; CHECK-NEXT: LBB0_21: ## %bb7806 -; CHECK-NEXT: ## in Loop: Header=BB0_5 Depth=1 +; CHECK-NEXT: jmp LBB0_2 +; CHECK-NEXT: .p2align 4 +; CHECK-NEXT: LBB0_18: ## %bb7806 +; CHECK-NEXT: ## in Loop: Header=BB0_2 Depth=1 ; CHECK-NEXT: Ltmp16: ## EH_LABEL ; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp) ; CHECK-NEXT: movl $1, {{[0-9]+}}(%esp) ; CHECK-NEXT: movl $0, (%esp) ; CHECK-NEXT: calll __ZN12wxStringBase6appendEmw ; CHECK-NEXT: Ltmp17: ## EH_LABEL -; CHECK-NEXT: LBB0_5: ## %bb3261 +; CHECK-NEXT: LBB0_2: ## %bb3261 ; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: cmpl $37, 0 -; CHECK-NEXT: jne LBB0_25 -; CHECK-NEXT: ## %bb.6: ## %bb3306 -; CHECK-NEXT: ## in Loop: Header=BB0_5 Depth=1 +; CHECK-NEXT: jne LBB0_22 +; CHECK-NEXT: ## %bb.3: ## %bb3306 +; CHECK-NEXT: ## in Loop: Header=BB0_2 Depth=1 ; CHECK-NEXT: Ltmp0: ## EH_LABEL ; CHECK-NEXT: movl %edi, {{[0-9]+}}(%esp) ; CHECK-NEXT: movl $0, (%esp) ; CHECK-NEXT: calll __ZN12wxStringBaseaSEPKw ; CHECK-NEXT: Ltmp1: ## EH_LABEL -; CHECK-NEXT: ## %bb.7: ## %bb3314 -; CHECK-NEXT: ## in Loop: Header=BB0_5 Depth=1 +; CHECK-NEXT: ## %bb.4: ## %bb3314 +; CHECK-NEXT: ## in Loop: Header=BB0_2 Depth=1 ; CHECK-NEXT: movl 0, %eax ; CHECK-NEXT: cmpl $121, %eax -; CHECK-NEXT: ja LBB0_25 -; CHECK-NEXT: ## %bb.8: ## %bb3314 -; CHECK-NEXT: ## in Loop: Header=BB0_5 Depth=1 +; CHECK-NEXT: ja LBB0_22 +; CHECK-NEXT: ## %bb.5: ## %bb3314 +; CHECK-NEXT: ## in Loop: Header=BB0_2 Depth=1 ; CHECK-NEXT: jmpl *LJTI0_0(,%eax,4) -; CHECK-NEXT: LBB0_10: ## %bb5809 -; CHECK-NEXT: ## in Loop: Header=BB0_5 Depth=1 +; CHECK-NEXT: LBB0_7: ## %bb5809 +; CHECK-NEXT: ## in Loop: Header=BB0_2 Depth=1 ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: jne LBB0_25 -; CHECK-NEXT: ## %bb.11: ## %bb5809 -; CHECK-NEXT: ## in Loop: Header=BB0_5 Depth=1 +; CHECK-NEXT: jne LBB0_22 +; CHECK-NEXT: ## %bb.8: ## %bb5809 +; CHECK-NEXT: ## in Loop: Header=BB0_2 Depth=1 ; CHECK-NEXT: testb %bh, %bh -; CHECK-NEXT: je LBB0_25 -; CHECK-NEXT: ## %bb.12: ## %bb91.i8504 -; CHECK-NEXT: ## in Loop: Header=BB0_5 Depth=1 +; CHECK-NEXT: je LBB0_22 +; CHECK-NEXT: ## %bb.9: ## %bb91.i8504 +; CHECK-NEXT: ## in Loop: Header=BB0_2 Depth=1 ; CHECK-NEXT: testb $1, %bl -; CHECK-NEXT: je LBB0_14 -; CHECK-NEXT: ## %bb.13: ## %bb155.i8541 -; CHECK-NEXT: ## in Loop: Header=BB0_5 Depth=1 +; CHECK-NEXT: je LBB0_11 +; CHECK-NEXT: ## %bb.10: ## %bb155.i8541 +; CHECK-NEXT: ## in Loop: Header=BB0_2 Depth=1 ; CHECK-NEXT: Ltmp4: ## EH_LABEL ; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp) ; CHECK-NEXT: movl $0, (%esp) ; CHECK-NEXT: calll _gmtime_r ; CHECK-NEXT: Ltmp5: ## EH_LABEL -; CHECK-NEXT: LBB0_14: ## %bb182.i8560 -; CHECK-NEXT: ## in Loop: Header=BB0_5 Depth=1 +; CHECK-NEXT: LBB0_11: ## %bb182.i8560 +; CHECK-NEXT: ## in Loop: Header=BB0_2 Depth=1 ; CHECK-NEXT: testb $1, %bl +; CHECK-NEXT: je LBB0_12 +; CHECK-NEXT: ## %bb.13: ## %bb278.i8617 +; CHECK-NEXT: ## in Loop: Header=BB0_2 Depth=1 ; CHECK-NEXT: je LBB0_15 -; CHECK-NEXT: ## %bb.16: ## %bb278.i8617 -; CHECK-NEXT: ## in Loop: Header=BB0_5 Depth=1 -; CHECK-NEXT: je LBB0_18 -; CHECK-NEXT: ## %bb.17: ## %bb440.i8663 -; CHECK-NEXT: ## in Loop: Header=BB0_5 Depth=1 +; CHECK-NEXT: ## %bb.14: ## %bb440.i8663 +; CHECK-NEXT: ## in Loop: Header=BB0_2 Depth=1 ; CHECK-NEXT: Ltmp6: ## EH_LABEL ; CHECK-NEXT: movl L_.str4$non_lazy_ptr, %eax ; CHECK-NEXT: movl %eax, {{[0-9]+}}(%esp) @@ -113,12 +108,12 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(ptr noalias sret(%struct ; CHECK-NEXT: movl $1717, {{[0-9]+}}(%esp) ## imm = 0x6B5 ; CHECK-NEXT: calll __Z10wxOnAssertPKwiPKcS0_S0_ ; CHECK-NEXT: Ltmp7: ## EH_LABEL -; CHECK-NEXT: jmp LBB0_18 -; CHECK-NEXT: LBB0_15: ## %bb187.i8591 -; CHECK-NEXT: ## in Loop: Header=BB0_5 Depth=1 -; CHECK-NEXT: jne LBB0_25 -; CHECK-NEXT: LBB0_18: ## %invcont5814 -; CHECK-NEXT: ## in Loop: Header=BB0_5 Depth=1 +; CHECK-NEXT: jmp LBB0_15 +; CHECK-NEXT: LBB0_12: ## %bb187.i8591 +; CHECK-NEXT: ## in Loop: Header=BB0_2 Depth=1 +; CHECK-NEXT: jne LBB0_22 +; CHECK-NEXT: LBB0_15: ## %invcont5814 +; CHECK-NEXT: ## in Loop: Header=BB0_2 Depth=1 ; CHECK-NEXT: Ltmp8: ## EH_LABEL ; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp) ; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp) @@ -126,8 +121,8 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(ptr noalias sret(%struct ; CHECK-NEXT: calll __ZN8wxString6FormatEPKwz ; CHECK-NEXT: subl $4, %esp ; CHECK-NEXT: Ltmp9: ## EH_LABEL -; CHECK-NEXT: ## %bb.19: ## %invcont5831 -; CHECK-NEXT: ## in Loop: Header=BB0_5 Depth=1 +; CHECK-NEXT: ## %bb.16: ## %invcont5831 +; CHECK-NEXT: ## in Loop: Header=BB0_2 Depth=1 ; CHECK-NEXT: Ltmp10: ## EH_LABEL ; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp) ; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp) @@ -135,8 +130,8 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(ptr noalias sret(%struct ; CHECK-NEXT: movl $0, (%esp) ; CHECK-NEXT: calll __ZN12wxStringBase10ConcatSelfEmPKwm ; CHECK-NEXT: Ltmp11: ## EH_LABEL -; CHECK-NEXT: jmp LBB0_5 -; CHECK-NEXT: LBB0_9: ## %bb5657 +; CHECK-NEXT: jmp LBB0_2 +; CHECK-NEXT: LBB0_6: ## %bb5657 ; CHECK-NEXT: Ltmp13: ## EH_LABEL ; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK-NEXT: movl %eax, {{[0-9]+}}(%esp) @@ -144,8 +139,8 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(ptr noalias sret(%struct ; CHECK-NEXT: movl %eax, (%esp) ; CHECK-NEXT: calll __ZNK10wxDateTime12GetDayOfYearERKNS_8TimeZoneE ; CHECK-NEXT: Ltmp14: ## EH_LABEL -; CHECK-NEXT: jmp LBB0_25 -; CHECK-NEXT: LBB0_20: ## %bb5968 +; CHECK-NEXT: jmp LBB0_22 +; CHECK-NEXT: LBB0_17: ## %bb5968 ; CHECK-NEXT: Ltmp2: ## EH_LABEL ; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp) ; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp) @@ -153,7 +148,7 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(ptr noalias sret(%struct ; CHECK-NEXT: calll __ZN8wxString6FormatEPKwz ; CHECK-NEXT: subl $4, %esp ; CHECK-NEXT: Ltmp3: ## EH_LABEL -; CHECK-NEXT: LBB0_25: ## %bb115.critedge.i +; CHECK-NEXT: LBB0_22: ## %bb115.critedge.i ; CHECK-NEXT: movl %esi, %eax ; CHECK-NEXT: addl $28, %esp ; CHECK-NEXT: popl %esi @@ -161,15 +156,15 @@ define void @_ZNK10wxDateTime6FormatEPKwRKNS_8TimeZoneE(ptr noalias sret(%struct ; CHECK-NEXT: popl %ebx ; CHECK-NEXT: popl %ebp ; CHECK-NEXT: retl $4 -; CHECK-NEXT: LBB0_23: ## %lpad.loopexit.split-lp +; CHECK-NEXT: LBB0_20: ## %lpad.loopexit.split-lp ; CHECK-NEXT: Ltmp15: ## EH_LABEL -; CHECK-NEXT: jmp LBB0_25 -; CHECK-NEXT: LBB0_24: ## %lpad8185 +; CHECK-NEXT: jmp LBB0_22 +; CHECK-NEXT: LBB0_21: ## %lpad8185 ; CHECK-NEXT: Ltmp12: ## EH_LABEL -; CHECK-NEXT: jmp LBB0_25 -; CHECK-NEXT: LBB0_22: ## %lpad.loopexit +; CHECK-NEXT: jmp LBB0_22 +; CHECK-NEXT: LBB0_19: ## %lpad.loopexit ; CHECK-NEXT: Ltmp18: ## EH_LABEL -; CHECK-NEXT: jmp LBB0_25 +; CHECK-NEXT: jmp LBB0_22 ; CHECK-NEXT: Lfunc_end0: entry: br i1 %foo, label %bb116.i, label %bb115.critedge.i diff --git a/llvm/test/CodeGen/X86/2009-11-17-UpdateTerminator.ll b/llvm/test/CodeGen/X86/2009-11-17-UpdateTerminator.ll index 56b20a519c8e6..ca24f84e9f13a 100644 --- a/llvm/test/CodeGen/X86/2009-11-17-UpdateTerminator.ll +++ b/llvm/test/CodeGen/X86/2009-11-17-UpdateTerminator.ll @@ -21,33 +21,27 @@ define zeroext i8 @_ZN4llvm9InlineAsm14ConstraintInfo5ParseENS_9StringRefERSt6ve ; CHECK: ## %bb.0: ## %entry ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: jne LBB0_8 +; CHECK-NEXT: jne LBB0_6 ; CHECK-NEXT: ## %bb.1: ## %bb27.outer.preheader ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: movb $1, %cl ; CHECK-NEXT: .p2align 4 -; CHECK-NEXT: LBB0_4: ## %bb27.outer108 +; CHECK-NEXT: LBB0_3: ## %bb27.outer108 ; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: je LBB0_5 -; CHECK-NEXT: ## %bb.2: ## %bb10 -; CHECK-NEXT: ## in Loop: Header=BB0_4 Depth=1 -; CHECK-NEXT: jne LBB0_8 -; CHECK-NEXT: ## %bb.3: ## %bb10 -; CHECK-NEXT: ## in Loop: Header=BB0_4 Depth=1 ; CHECK-NEXT: je LBB0_4 -; CHECK-NEXT: jmp LBB0_8 -; CHECK-NEXT: LBB0_5: ## %bb27.outer108 -; CHECK-NEXT: ## in Loop: Header=BB0_4 Depth=1 -; CHECK-NEXT: jne LBB0_8 -; CHECK-NEXT: ## %bb.6: ## %bb27.outer108 -; CHECK-NEXT: ## in Loop: Header=BB0_4 Depth=1 -; CHECK-NEXT: jne LBB0_8 -; CHECK-NEXT: ## %bb.7: ## %bb27.outer108 -; CHECK-NEXT: ## in Loop: Header=BB0_4 Depth=1 +; CHECK-NEXT: ## %bb.2: ## %bb10 +; CHECK-NEXT: ## in Loop: Header=BB0_3 Depth=1 +; CHECK-NEXT: je LBB0_3 +; CHECK-NEXT: jmp LBB0_6 +; CHECK-NEXT: LBB0_4: ## %bb27.outer108 +; CHECK-NEXT: ## in Loop: Header=BB0_3 Depth=1 +; CHECK-NEXT: jne LBB0_6 +; CHECK-NEXT: ## %bb.5: ## %bb27.outer108 +; CHECK-NEXT: ## in Loop: Header=BB0_3 Depth=1 ; CHECK-NEXT: testb %cl, %cl -; CHECK-NEXT: jne LBB0_4 -; CHECK-NEXT: LBB0_8: ## %bb14 +; CHECK-NEXT: jne LBB0_3 +; CHECK-NEXT: LBB0_6: ## %bb14 ; CHECK-NEXT: movl $1, %eax ; CHECK-NEXT: retq entry: diff --git a/llvm/test/CodeGen/X86/2011-09-14-valcoalesce.ll b/llvm/test/CodeGen/X86/2011-09-14-valcoalesce.ll index 3b3b4d72a0064..7ecfca759d4d6 100644 --- a/llvm/test/CodeGen/X86/2011-09-14-valcoalesce.ll +++ b/llvm/test/CodeGen/X86/2011-09-14-valcoalesce.ll @@ -29,7 +29,7 @@ define void @BZ2_compressBlock() nounwind ssp { ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: jne .LBB0_24 +; CHECK-NEXT: jne .LBB0_22 ; CHECK-NEXT: # %bb.1: # %if.then68 ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al @@ -54,7 +54,7 @@ define void @BZ2_compressBlock() nounwind ssp { ; CHECK-NEXT: movb $1, %dl ; CHECK-NEXT: testb %dl, %dl ; CHECK-NEXT: movl $0, %esi -; CHECK-NEXT: jne .LBB0_14 +; CHECK-NEXT: jne .LBB0_12 ; CHECK-NEXT: # %bb.7: # %while.body85.i.preheader ; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1 ; CHECK-NEXT: xorl %ecx, %ecx @@ -73,57 +73,51 @@ define void @BZ2_compressBlock() nounwind ssp { ; CHECK-NEXT: .LBB0_10: # %while.end.i ; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1 ; CHECK-NEXT: testb %bl, %bl -; CHECK-NEXT: jne .LBB0_14 -; CHECK-NEXT: # %bb.11: # %land.lhs.true.i -; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1 -; CHECK-NEXT: jne .LBB0_14 -; CHECK-NEXT: # %bb.12: # %land.lhs.true103.i -; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1 -; CHECK-NEXT: jne .LBB0_14 -; CHECK-NEXT: # %bb.13: # %if.then108.i +; CHECK-NEXT: jne .LBB0_12 +; CHECK-NEXT: # %bb.11: # %if.then108.i ; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1 ; CHECK-NEXT: xorl %esi, %esi ; CHECK-NEXT: movl %edx, %ecx -; CHECK-NEXT: .LBB0_14: # %if.end117.i +; CHECK-NEXT: .LBB0_12: # %if.end117.i ; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1 ; CHECK-NEXT: testb %bl, %bl -; CHECK-NEXT: jne .LBB0_16 -; CHECK-NEXT: # %bb.15: # %if.then122.i +; CHECK-NEXT: jne .LBB0_14 +; CHECK-NEXT: # %bb.13: # %if.then122.i ; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1 ; CHECK-NEXT: movl %ecx, {{[0-9]+}}(%esp) ; CHECK-NEXT: movl %esi, {{[0-9]+}}(%esp) ; CHECK-NEXT: movl %eax, {{[0-9]+}}(%esp) ; CHECK-NEXT: calll fprintf@PLT -; CHECK-NEXT: .LBB0_16: # %for.cond138.preheader.i +; CHECK-NEXT: .LBB0_14: # %for.cond138.preheader.i ; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1 ; CHECK-NEXT: testb %bl, %bl ; CHECK-NEXT: incl %esi ; CHECK-NEXT: testb %bl, %bl ; CHECK-NEXT: movl %esi, %eax ; CHECK-NEXT: jne .LBB0_6 -; CHECK-NEXT: # %bb.17: # %for.cond182.preheader.i +; CHECK-NEXT: # %bb.15: # %for.cond182.preheader.i ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al ; CHECK-NEXT: leal {{[0-9]+}}(%esp), %esp ; CHECK-NEXT: popl %esi ; CHECK-NEXT: popl %ebx -; CHECK-NEXT: jne .LBB0_18 -; CHECK-NEXT: jmp .LBB0_19 -; CHECK-NEXT: .LBB0_18: # %for.inc220.us.i -; CHECK-NEXT: .LBB0_19: # %while.body300.preheader.i +; CHECK-NEXT: jne .LBB0_16 +; CHECK-NEXT: jmp .LBB0_17 +; CHECK-NEXT: .LBB0_16: # %for.inc220.us.i +; CHECK-NEXT: .LBB0_17: # %while.body300.preheader.i ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: jne .LBB0_21 -; CHECK-NEXT: .LBB0_20: # %for.end335.i +; CHECK-NEXT: jne .LBB0_19 +; CHECK-NEXT: .LBB0_18: # %for.end335.i ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: jmp .LBB0_20 -; CHECK-NEXT: .LBB0_21: # %while.end2742.i +; CHECK-NEXT: jmp .LBB0_18 +; CHECK-NEXT: .LBB0_19: # %while.end2742.i ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: jne .LBB0_23 -; CHECK-NEXT: # %bb.22: # %if.then2748.i -; CHECK-NEXT: .LBB0_23: # %for.body2778.i -; CHECK-NEXT: .LBB0_24: # %if.end85 +; CHECK-NEXT: jne .LBB0_21 +; CHECK-NEXT: # %bb.20: # %if.then2748.i +; CHECK-NEXT: .LBB0_21: # %for.body2778.i +; CHECK-NEXT: .LBB0_22: # %if.end85 ; CHECK-NEXT: retl entry: br i1 undef, label %if.then68, label %if.end85 From 8b9cce358bef26ae4cb9275dd6a43f903bafbaa0 Mon Sep 17 00:00:00 2001 From: jimingham Date: Tue, 21 Jul 2026 16:31:38 -0700 Subject: [PATCH 50/50] Make result variables obey their dynamic values in subsequent expressions This is a resubmit of the original patch: 6344e3aa8106dfdfb30cac36c8ca02bc4c52ce24: Make result variables obey their dynamic values in subsequent expressions (#168611) When I originally submitted this, it caused intermittent flakey failures on systems I didn't have access to, and I didn't have time to sort them out, so I reverted the patch. I'm resubmitting this so I can run the bots on it a few rounds to see if I can reproduce and diagnose those intermittent failures. Here's the commit log from the original submission describing the change: When you run an expression and the result has a dynamic type that is different from the expression's static result type, we print the result variable using the dynamic type, but at present when you use the result variable in an expression later on, we only give you access to the static type. For instance: ``` (lldb) expr MakeADerivedReportABase() (Derived *) $0 = 0x00000001007e93e0 (lldb) expr $0->method_from_derived() ^ error: no member named 'method_from_derived' in 'Base' (lldb) ``` The static return type of that function is `Base *`, but we printed that the result was a `Derived *` and then only used the `Base *` part of it in subsequent expressions. That's not very helpful, and forces you to guess and then cast the result types to their dynamic type in order to be able to access the full type you were returned, which is inconvenient. This patch makes lldb retain the dynamic type of the result variable (and ditto for persistent result variables). It also adds more testing of expression result variables with various types of dynamic values, to ensure we can access both the ivars and methods of the type we print the result as. --- .../lldb/Expression/ExpressionVariable.h | 64 +++++-- .../Python/lldbsuite/test/lldbtest.py | 1 - lldb/source/Expression/ExpressionVariable.cpp | 78 +++++++- lldb/source/Expression/LLVMUserExpression.cpp | 14 +- lldb/source/Expression/Materializer.cpp | 49 ++--- .../Clang/ClangExpressionVariable.cpp | 2 +- lldb/source/Target/ABI.cpp | 2 +- .../functionalities/expr-result-var/Makefile | 3 + .../expr-result-var/TestCPPExprResult.py | 174 ++++++++++++++++++ .../expr-result-var/two-bases.cpp | 55 ++++++ 10 files changed, 385 insertions(+), 57 deletions(-) create mode 100644 lldb/test/API/functionalities/expr-result-var/Makefile create mode 100644 lldb/test/API/functionalities/expr-result-var/TestCPPExprResult.py create mode 100644 lldb/test/API/functionalities/expr-result-var/two-bases.cpp diff --git a/lldb/include/lldb/Expression/ExpressionVariable.h b/lldb/include/lldb/Expression/ExpressionVariable.h index 991faf17daef3..62100fcb5f527 100644 --- a/lldb/include/lldb/Expression/ExpressionVariable.h +++ b/lldb/include/lldb/Expression/ExpressionVariable.h @@ -33,11 +33,19 @@ class ExpressionVariable virtual ~ExpressionVariable() = default; - llvm::Expected GetByteSize() { return m_frozen_sp->GetByteSize(); } + llvm::Expected GetByteSize() { + return GetValueObject()->GetByteSize(); + } ConstString GetName() { return m_frozen_sp->GetName(); } - lldb::ValueObjectSP GetValueObject() { return m_frozen_sp; } + lldb::ValueObjectSP GetValueObject() { + lldb::ValueObjectSP dyn_sp = + m_frozen_sp->GetDynamicValue(lldb::eDynamicDontRunTarget); + if (dyn_sp && dyn_sp->UpdateValueIfNeeded()) + return dyn_sp; + return m_frozen_sp; + } uint8_t *GetValueBytes(); @@ -52,7 +60,7 @@ class ExpressionVariable Value::ContextType::RegisterInfo, const_cast(reg_info)); } - CompilerType GetCompilerType() { return m_frozen_sp->GetCompilerType(); } + CompilerType GetCompilerType() { return GetValueObject()->GetCompilerType(); } void SetCompilerType(const CompilerType &compiler_type) { m_frozen_sp->GetValue().SetCompilerType(compiler_type); @@ -60,23 +68,31 @@ class ExpressionVariable void SetName(llvm::StringRef name) { m_frozen_sp->SetName(name); } - // this function is used to copy the address-of m_live_sp into m_frozen_sp - // this is necessary because the results of certain cast and pointer- - // arithmetic operations (such as those described in bugzilla issues 11588 - // and 11618) generate frozen objects that do not have a valid address-of, - // which can be troublesome when using synthetic children providers. - // Transferring the address-of the live object solves these issues and - // provides the expected user-level behavior - void TransferAddress(bool force = false) { - if (m_live_sp.get() == nullptr) - return; - - if (m_frozen_sp.get() == nullptr) - return; - - if (force || (m_frozen_sp->GetLiveAddress() == LLDB_INVALID_ADDRESS)) - m_frozen_sp->SetLiveAddress(m_live_sp->GetLiveAddress()); + /// This function is used to copy the address-of m_live_sp into m_frozen_sp. + /// It is necessary because the results of certain cast and pointer- + /// arithmetic operations (such as those described in bugzilla issues 11588 + /// and 11618) generate frozen objects that do not have a valid address-of, + /// which can be troublesome when using synthetic children providers. + /// Transferring the address-of the live object solves these issues and + /// provides the expected user-level behavior. + /// The other job we do in TransferAddress is adjust the value in the live + /// address slot in the target for the "offset to top" in multiply inherited + /// class hierarchies. + void TransferAddress(bool force = false); + + /// When we build an expression variable we know whether we're going to use + /// the static or dynamic result. If we present the dynamic value once, we + /// should use the dynamic value in future references to the variable, so we + /// record that fact here. + void PreserveDynamicOption(lldb::DynamicValueType dyn_type) { + m_dyn_option = dyn_type; } + /// We don't try to get the dynamic value of the live object when we fetch + /// it here. The live object describes the container of the value in the + /// target, but it's type is of the object for convenience. So it can't + /// produce the dynamic value. Instead, we use TransferAddress to adjust the + /// value held by the LiveObject. + lldb::ValueObjectSP GetLiveObject() { return m_live_sp; } enum Flags { EVNone = 0, @@ -110,6 +126,14 @@ class ExpressionVariable /// These members should be private. /// @{ /// A value object whose value's data lives in host (lldb's) memory. + /// The m_frozen_sp holds the data & type of the expression variable or result + /// in the host. The m_frozen_sp also can present a dynamic value if one is + /// available. + /// The m_frozen_sp manages the copy of this value in m_frozen_sp that we + /// insert in the target so that it can be referred to in future expressions. + /// We don't actually use the contents of the live_sp to create the value in + /// the target, that comes from the frozen sp. The live_sp is mostly to track + /// the target-side of the value. lldb::ValueObjectSP m_frozen_sp; /// The ValueObject counterpart to m_frozen_sp that tracks the value in /// inferior memory. This object may not always exist; its presence depends on @@ -119,6 +143,8 @@ class ExpressionVariable /// track. lldb::ValueObjectSP m_live_sp; /// @} + + lldb::DynamicValueType m_dyn_option = lldb::eNoDynamicValues; }; /// \class ExpressionVariableList ExpressionVariable.h diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py index 72429b8ba511b..6bf44a0964696 100644 --- a/lldb/packages/Python/lldbsuite/test/lldbtest.py +++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py @@ -2967,7 +2967,6 @@ def expect_expr( ) frame = self.frame() - if not options: options = lldb.SBExpressionOptions() diff --git a/lldb/source/Expression/ExpressionVariable.cpp b/lldb/source/Expression/ExpressionVariable.cpp index 9e8ea60f8e052..4c9568106b346 100644 --- a/lldb/source/Expression/ExpressionVariable.cpp +++ b/lldb/source/Expression/ExpressionVariable.cpp @@ -20,15 +20,15 @@ char ExpressionVariable::ID; ExpressionVariable::ExpressionVariable() : m_flags(0) {} uint8_t *ExpressionVariable::GetValueBytes() { + lldb::ValueObjectSP valobj_sp = GetValueObject(); std::optional byte_size = - llvm::expectedToOptional(m_frozen_sp->GetByteSize()); + llvm::expectedToOptional(valobj_sp->GetByteSize()); if (byte_size && *byte_size) { - if (m_frozen_sp->GetDataExtractor().GetByteSize() < *byte_size) { - m_frozen_sp->GetValue().ResizeData(*byte_size); - m_frozen_sp->GetValue().GetData(m_frozen_sp->GetDataExtractor()); + if (valobj_sp->GetDataExtractor().GetByteSize() < *byte_size) { + valobj_sp->GetValue().ResizeData(*byte_size); + valobj_sp->GetValue().GetData(valobj_sp->GetDataExtractor()); } - return const_cast( - m_frozen_sp->GetDataExtractor().GetDataStart()); + return const_cast(valobj_sp->GetDataExtractor().GetDataStart()); } return nullptr; } @@ -37,6 +37,72 @@ char PersistentExpressionState::ID; PersistentExpressionState::PersistentExpressionState() = default; +void ExpressionVariable::TransferAddress(bool force) { + if (!m_live_sp) + return; + + if (!m_frozen_sp) + return; + + if (force || (m_frozen_sp->GetLiveAddress() == LLDB_INVALID_ADDRESS)) { + lldb::addr_t live_addr = m_live_sp->GetLiveAddress(); + m_frozen_sp->SetLiveAddress(live_addr); + // One more detail, if there's an offset_to_top in the frozen_sp, then we + // need to appy that offset by hand. The live_sp can't compute this + // itself as its type is the type of the contained object which confuses + // the dynamic type calculation. So we have to update the contents of the + // m_live_sp with the dynamic value. + // Note: We could get this right when we originally write the address, but + // that happens in different ways for the various flavors of + // Entity*::Materialize, but everything comes through here, and it's just + // one extra memory write. + + // You can only have an "offset_to_top" with pointers or references: + if (!m_frozen_sp->GetCompilerType().IsPointerOrReferenceType()) + return; + + lldb::ProcessSP process_sp = m_frozen_sp->GetProcessSP(); + // If there's no dynamic value, then there can't be an offset_to_top: + if (!process_sp || + !process_sp->IsPossibleDynamicValue(*(m_frozen_sp.get()))) + return; + + lldb::ValueObjectSP dyn_sp = m_frozen_sp->GetDynamicValue(m_dyn_option); + if (!dyn_sp) + return; + ValueObject::AddrAndType static_addr = m_frozen_sp->GetPointerValue(); + if (static_addr.type != eAddressTypeLoad) + return; + + ValueObject::AddrAndType dynamic_addr = dyn_sp->GetPointerValue(); + if (dynamic_addr.type != eAddressTypeLoad || + static_addr.address == dynamic_addr.address) + return; + + Status error; + Log *log = GetLog(LLDBLog::Expressions); + lldb::addr_t cur_value = + process_sp->ReadPointerFromMemory(live_addr, error); + if (error.Fail()) + return; + + if (cur_value != static_addr.address) { + LLDB_LOG(log, + "Stored value: {0} read from {1} doesn't " + "match static addr: {2}", + cur_value, live_addr, static_addr.address); + return; + } + + if (!process_sp->WritePointerToMemory(live_addr, dynamic_addr.address, + error)) { + LLDB_LOG(log, "Got error: {0} writing dynamic value: {1} to {2}", error, + dynamic_addr.address, live_addr); + return; + } + } +} + PersistentExpressionState::~PersistentExpressionState() = default; lldb::addr_t PersistentExpressionState::LookupSymbol(ConstString name) { diff --git a/lldb/source/Expression/LLVMUserExpression.cpp b/lldb/source/Expression/LLVMUserExpression.cpp index d2c06cbf3ba72..eaecb3dbfe726 100644 --- a/lldb/source/Expression/LLVMUserExpression.cpp +++ b/lldb/source/Expression/LLVMUserExpression.cpp @@ -66,7 +66,7 @@ LLVMUserExpression::DoExecute(DiagnosticManager &diagnostic_manager, ExecutionContext &exe_ctx, const EvaluateExpressionOptions &options, lldb::UserExpressionSP &shared_ptr_to_me, - lldb::ExpressionVariableSP &result) { + lldb::ExpressionVariableSP &result_sp) { // The expression log is quite verbose, and if you're just tracking the // execution of the expression, it's quite convenient to have these logs come // out with the STEP log as well. @@ -254,10 +254,9 @@ LLVMUserExpression::DoExecute(DiagnosticManager &diagnostic_manager, } } - if (FinalizeJITExecution(diagnostic_manager, exe_ctx, result, - function_stack_bottom, function_stack_top)) { + if (FinalizeJITExecution(diagnostic_manager, exe_ctx, result_sp, + function_stack_bottom, function_stack_top)) return lldb::eExpressionCompleted; - } return lldb::eExpressionResultUnavailable; } @@ -293,8 +292,13 @@ bool LLVMUserExpression::FinalizeJITExecution( result = GetResultAfterDematerialization(exe_ctx.GetBestExecutionContextScope()); - if (result) + if (result) { + // TransferAddress also does the offset_to_top calculation, so record the + // dynamic option before we do that. + if (EvaluateExpressionOptions *options = GetOptions()) + result->PreserveDynamicOption(options->GetUseDynamic()); result->TransferAddress(); + } m_dematerializer_sp.reset(); diff --git a/lldb/source/Expression/Materializer.cpp b/lldb/source/Expression/Materializer.cpp index ea01b0d7fc56d..42c303a0a747d 100644 --- a/lldb/source/Expression/Materializer.cpp +++ b/lldb/source/Expression/Materializer.cpp @@ -76,10 +76,11 @@ class EntityPersistentVariable : public Materializer::Entity { const bool zero_memory = false; IRMemoryMap::AllocationPolicy used_policy; - auto address_or_error = map.Malloc( + const uint64_t malloc_size = llvm::expectedToOptional(m_persistent_variable_sp->GetByteSize()) - .value_or(0), - 8, lldb::ePermissionsReadable | lldb::ePermissionsWritable, + .value_or(0); + auto address_or_error = map.Malloc( + malloc_size, 8, lldb::ePermissionsReadable | lldb::ePermissionsWritable, IRMemoryMap::eAllocationPolicyMirror, zero_memory, &used_policy); if (!address_or_error) { err = Status::FromErrorStringWithFormat( @@ -90,8 +91,9 @@ class EntityPersistentVariable : public Materializer::Entity { } lldb::addr_t mem = *address_or_error; - LLDB_LOGF(log, "Allocated %s (0x%" PRIx64 ") successfully", - m_persistent_variable_sp->GetName().GetCString(), mem); + LLDB_LOGF( + log, "Allocated 0x%" PRIx64 "bytes for %s (0x%" PRIx64 ") successfully", + malloc_size, m_persistent_variable_sp->GetName().GetCString(), mem); // Put the location of the spare memory into the live data of the // ValueObject. @@ -142,12 +144,12 @@ class EntityPersistentVariable : public Materializer::Entity { void DestroyAllocation(IRMemoryMap &map, Status &err) { Status deallocate_error; - map.Free((lldb::addr_t)m_persistent_variable_sp->m_live_sp->GetValue() - .GetScalar() - .ULongLong(), + lldb::ValueObjectSP live_valobj_sp = + m_persistent_variable_sp->GetLiveObject(); + map.Free((lldb::addr_t)live_valobj_sp->GetValue().GetScalar().ULongLong(), deallocate_error); - m_persistent_variable_sp->m_live_sp.reset(); + live_valobj_sp.reset(); if (!deallocate_error.Success()) { err = Status::FromErrorStringWithFormat( @@ -179,17 +181,17 @@ class EntityPersistentVariable : public Materializer::Entity { return; } + lldb::ValueObjectSP live_valobj_sp = + m_persistent_variable_sp->GetLiveObject(); if ((m_persistent_variable_sp->m_flags & ExpressionVariable::EVIsProgramReference && - m_persistent_variable_sp->m_live_sp) || + live_valobj_sp) || m_persistent_variable_sp->m_flags & ExpressionVariable::EVIsLLDBAllocated) { Status write_error; - map.WriteScalarToMemory( - load_addr, - m_persistent_variable_sp->m_live_sp->GetValue().GetScalar(), - map.GetAddressByteSize(), write_error); + map.WriteScalarToMemory(load_addr, live_valobj_sp->GetValue().GetScalar(), + map.GetAddressByteSize(), write_error); if (!write_error.Success()) { err = Status::FromErrorStringWithFormatv( @@ -222,13 +224,15 @@ class EntityPersistentVariable : public Materializer::Entity { m_delegate->DidDematerialize(m_persistent_variable_sp); } + lldb::ValueObjectSP live_valobj_sp = + m_persistent_variable_sp->GetLiveObject(); if ((m_persistent_variable_sp->m_flags & ExpressionVariable::EVIsLLDBAllocated) || (m_persistent_variable_sp->m_flags & ExpressionVariable::EVIsProgramReference)) { if (m_persistent_variable_sp->m_flags & ExpressionVariable::EVIsProgramReference && - !m_persistent_variable_sp->m_live_sp) { + !live_valobj_sp) { // If the reference comes from the program, then the // ClangExpressionVariable's live variable data hasn't been set up yet. // Do this now. @@ -248,7 +252,7 @@ class EntityPersistentVariable : public Materializer::Entity { m_persistent_variable_sp->m_live_sp = ValueObjectConstResult::Create( map.GetBestExecutionContextScope(), - m_persistent_variable_sp.get()->GetCompilerType(), + m_persistent_variable_sp->GetCompilerType(), m_persistent_variable_sp->GetName(), location, eAddressTypeLoad, llvm::expectedToOptional(m_persistent_variable_sp->GetByteSize()) .value_or(0)); @@ -270,19 +274,17 @@ class EntityPersistentVariable : public Materializer::Entity { } } - lldb::addr_t mem = m_persistent_variable_sp->m_live_sp->GetValue() - .GetScalar() - .ULongLong(); - - if (!m_persistent_variable_sp->m_live_sp) { + if (!live_valobj_sp) { err = Status::FromErrorStringWithFormat( "couldn't find the memory area used to store %s", m_persistent_variable_sp->GetName().GetCString()); return; } - if (m_persistent_variable_sp->m_live_sp->GetValue() - .GetValueAddressType() != eAddressTypeLoad) { + lldb::addr_t mem = live_valobj_sp->GetValue().GetScalar().ULongLong(); + + if (live_valobj_sp->GetValue().GetValueAddressType() != + eAddressTypeLoad) { err = Status::FromErrorStringWithFormat( "the address of the memory area for %s is in an incorrect format", m_persistent_variable_sp->GetName().GetCString()); @@ -319,7 +321,6 @@ class EntityPersistentVariable : public Materializer::Entity { read_error.AsCString()); return; } - m_persistent_variable_sp->m_flags &= ~ExpressionVariable::EVNeedsFreezeDry; } diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionVariable.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionVariable.cpp index e2fb4a054daf3..d7b3fe0167299 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionVariable.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionVariable.cpp @@ -59,6 +59,6 @@ ClangExpressionVariable::ClangExpressionVariable( } TypeFromUser ClangExpressionVariable::GetTypeFromUser() { - TypeFromUser tfu(m_frozen_sp->GetCompilerType()); + TypeFromUser tfu(GetValueObject()->GetCompilerType()); return tfu; } diff --git a/lldb/source/Target/ABI.cpp b/lldb/source/Target/ABI.cpp index b7b45f9f4c44b..90d7430e53738 100644 --- a/lldb/source/Target/ABI.cpp +++ b/lldb/source/Target/ABI.cpp @@ -127,7 +127,7 @@ ValueObjectSP ABI::GetReturnValueObject(Thread &thread, CompilerType &ast_type, ExpressionVariable::EVNeedsAllocation; break; case Value::ValueType::LoadAddress: - expr_variable_sp->m_live_sp = live_valobj_sp; + expr_variable_sp->GetLiveObject() = live_valobj_sp; expr_variable_sp->m_flags |= ExpressionVariable::EVIsProgramReference; break; diff --git a/lldb/test/API/functionalities/expr-result-var/Makefile b/lldb/test/API/functionalities/expr-result-var/Makefile new file mode 100644 index 0000000000000..6c259307ef229 --- /dev/null +++ b/lldb/test/API/functionalities/expr-result-var/Makefile @@ -0,0 +1,3 @@ +CXX_SOURCES := two-bases.cpp + +include Makefile.rules diff --git a/lldb/test/API/functionalities/expr-result-var/TestCPPExprResult.py b/lldb/test/API/functionalities/expr-result-var/TestCPPExprResult.py new file mode 100644 index 0000000000000..7bd4bb582a4a3 --- /dev/null +++ b/lldb/test/API/functionalities/expr-result-var/TestCPPExprResult.py @@ -0,0 +1,174 @@ +""" +Test the reuse of C++ result variables, particularly making sure +that the dynamic typing is preserved. +""" + + +import lldb +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil + + +class TestCPPResultVariables(TestBase): + NO_DEBUG_INFO_TESTCASE = True + SHARED_BUILD_TESTCASE = False + + def setUp(self): + TestBase.setUp(self) + self.main_source_file = lldb.SBFileSpec("two-bases.cpp") + + def check_dereference(self, result_varname, frame, expr_options): + deref_expr = "*{0}".format(result_varname) + base_children = ValueCheck( + name="Base", value="", children=[ValueCheck(name="base_int", value="100")] + ) + base_1_arr_children = [ + ValueCheck(name="[0]", value="100"), + ValueCheck(name="[1]", value="101"), + ValueCheck(name="[2]", value="102"), + ValueCheck(name="[3]", value="103"), + ValueCheck(name="[4]", value="104"), + ValueCheck(name="[5]", value="105"), + ValueCheck(name="[6]", value="106"), + ValueCheck(name="[7]", value="107"), + ValueCheck(name="[8]", value="108"), + ValueCheck(name="[9]", value="109"), + ] + base_2_arr_children = [ + ValueCheck(name="[0]", value="200"), + ValueCheck(name="[1]", value="201"), + ValueCheck(name="[2]", value="202"), + ValueCheck(name="[3]", value="203"), + ValueCheck(name="[4]", value="204"), + ValueCheck(name="[5]", value="205"), + ValueCheck(name="[6]", value="206"), + ValueCheck(name="[7]", value="207"), + ValueCheck(name="[8]", value="208"), + ValueCheck(name="[9]", value="209"), + ] + deref_children = [ + ValueCheck( + name="Base_1", + value="", + children=[ + base_children, + ValueCheck( + name="base_1_arr", value="", children=base_1_arr_children + ), + ], + ), + ValueCheck( + name="Base_2", + value="", + children=[ + base_children, + ValueCheck( + name="base_2_arr", value="", children=base_2_arr_children + ), + ], + ), + ValueCheck(name="derived_int", value="1000"), + ] + result_var_deref = self.expect_expr( + deref_expr, + result_type="Derived", + result_children=deref_children, + options=expr_options, + ) + + direct_access_expr = "{0}->derived_int".format(result_varname) + self.expect_expr(direct_access_expr, result_type="int", result_value="1000") + + # Also check this by directly accessing the result variable: + result_value = frame.FindValue(result_varname, lldb.eValueTypeConstResult, True) + self.assertTrue(result_value.error.success, "Found my result variable") + value_check = ValueCheck(children=deref_children) + value_check.check_value( + self, result_value, f"{result_varname} children are correct" + ) + + # Make sure we can also call a function through the derived type: + method_result = self.expect_expr( + f"{result_varname}->method_of_derived()", + result_type="int", + options=expr_options, + ) + self.assertEqual(method_result.signed, 500, "Got the right result value") + + @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24663") + def test_virtual_dynamic_results(self): + self.do_test_dynamic_results(True) + + @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24663") + def test_non_virtual_dynamic_results(self): + self.do_test_dynamic_results(False) + + def do_test_dynamic_results(self, virtual): + """Test that when we uses a result variable in a subsequent expression it + uses the dynamic value - if that was requested when the result variable was made. + """ + if virtual: + self.build(dictionary={"CFLAGS_EXTRAS": "-DVIRTUAL=''"}) + else: + self.build(dictionary={"CFLAGS_EXTRAS": "-DVIRTUAL='virtual'"}) + + (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint( + self, "Set a breakpoint here", self.main_source_file + ) + + frame = thread.GetFrameAtIndex(0) + expr_options = lldb.SBExpressionOptions() + expr_options.SetFetchDynamicValue(lldb.eDynamicDontRunTarget) + base_1_ptr = self.expect_expr( + "base_1_ptr", result_type="Derived *", options=expr_options + ) + result_varname = base_1_ptr.GetName() + self.check_dereference(result_varname, frame, expr_options) + + # Now do the same thing, but use a persistent result variable: + empty_var = frame.EvaluateExpression( + "void *$base_1_ptr = base_1_ptr", expr_options + ) + self.assertIn( + empty_var.error.description, + "unknown error", + "Expressions that don't have results return this error", + ) + persist_base_1_ptr = frame.FindValue( + "$base_1_ptr", lldb.eValueTypeConstResult, True + ) + self.assertTrue(persist_base_1_ptr.error.success, "Got the persistent variable") + self.check_dereference("$base_1_ptr", frame, expr_options) + + # Now check the second of the multiply inherited bases, this one will have an offset_to_top + # that we need to calculate: + base_2_ptr = self.expect_expr( + "base_2_ptr", result_type="Derived *", options=expr_options + ) + self.check_dereference(base_2_ptr.GetName(), frame, expr_options) + + # Again, do the same thing for a persistent expression variable: + empty_var = frame.EvaluateExpression( + "void *$base_2_ptr = base_2_ptr", expr_options + ) + self.check_dereference("$base_2_ptr", frame, expr_options) + + # Now try starting from a virtual base class of both our bases: + base_through_1 = self.expect_expr( + "base_through_1", result_type="Derived *", options=expr_options + ) + self.check_dereference(base_through_1.GetName(), frame, expr_options) + + # Now try starting from a virtual base class of both our bases: + base_through_2 = self.expect_expr( + "base_through_2", result_type="Derived *", options=expr_options + ) + self.check_dereference(base_through_2.GetName(), frame, expr_options) + + # Now check that we get the right results when we run an + # expression to get the base class object: + base_through_expr = self.expect_expr( + "MakeADerivedReportABase()", result_type="Derived *", options=expr_options + ) + self.check_dereference(base_through_expr.GetName(), frame, expr_options) diff --git a/lldb/test/API/functionalities/expr-result-var/two-bases.cpp b/lldb/test/API/functionalities/expr-result-var/two-bases.cpp new file mode 100644 index 0000000000000..55af757d02bd8 --- /dev/null +++ b/lldb/test/API/functionalities/expr-result-var/two-bases.cpp @@ -0,0 +1,55 @@ +#include +#include + +struct Base { + virtual ~Base() = default; + int base_int = 100; + Base *return_me() { return this; } +}; + +struct Base_1 : public VIRTUAL Base { + virtual ~Base_1() = default; + int base_1_arr[10] = {100, 101, 102, 103, 104, 105, 106, 107, 108, 109}; + Base *return_base_1() { return return_me(); } +}; + +struct Base_2 : public VIRTUAL Base { + virtual ~Base_2() = default; + int base_2_arr[10] = {200, 201, 202, 203, 204, 205, 206, 207, 208, 209}; + Base *return_base_2() { return return_me(); } +}; + +struct Derived : public Base_1, Base_2 { + virtual ~Derived() = default; + int derived_int = 1000; + int method_of_derived() { return 500; } +}; + +Base *MakeADerivedReportABase() { return (Base *)((Base_1 *)new Derived()); } + +int main() { + Derived my_derived; + int call_it = my_derived.method_of_derived(); + + Base_1 *base_1_ptr = (Base_1 *)&my_derived; + + Base_2 *base_2_ptr = (Base_2 *)&my_derived; + + Base *base_through_1 = my_derived.return_base_1(); + Base *base_through_2 = my_derived.return_base_2(); + + // Call this to make sure the compiler makes it. + Base *fake_base = MakeADerivedReportABase(); + + uint64_t base_through_1_addr = (uint64_t)base_through_1; + uint64_t base_through_2_addr = (uint64_t)base_through_2; + int64_t base_offset = base_through_2_addr - base_through_1_addr; + printf("Base offset (should be 0): 0x%llx.\n", base_offset); + uint64_t base_1_addr = (uint64_t)base_1_ptr; + uint64_t base_2_addr = (uint64_t)base_2_ptr; + int64_t offset = base_2_addr - base_1_addr; + + // Set a breakpoint here + return my_derived.derived_int + base_1_ptr->base_1_arr[0] + + base_2_ptr->base_2_arr[0] + my_derived.return_base_1()->base_int; +}