From fc0a8108a55ae5db3aa0e71a9877bd56f0581728 Mon Sep 17 00:00:00 2001 From: Dimitry Andric Date: Thu, 16 Nov 2023 19:35:55 +0100 Subject: [PATCH] Vendor import of llvm-project branch release/17.x llvmorg-17.0.5-0-g98bfdac5ce82. --- .../include/clang/Basic/DiagnosticASTKinds.td | 2 +- clang/include/clang/Sema/Sema.h | 7 +- clang/lib/AST/ExprConstant.cpp | 13 +- clang/lib/AST/Interp/Interp.cpp | 3 +- clang/lib/CodeGen/CGExprConstant.cpp | 7 +- clang/lib/Driver/ToolChains/Solaris.cpp | 9 +- clang/lib/Format/TokenAnnotator.cpp | 6 + clang/lib/Format/WhitespaceManager.cpp | 2 +- clang/lib/Parse/ParseDecl.cpp | 11 +- libcxx/include/__config | 60 ++++-- libcxx/include/__expected/expected.h | 182 ++++++++---------- llvm/lib/CodeGen/BranchFolding.cpp | 6 - llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 1 + .../SelectionDAG/LegalizeVectorTypes.cpp | 10 + llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 2 +- .../AArch64/AArch64TargetTransformInfo.cpp | 24 +++ .../AArch64/AArch64TargetTransformInfo.h | 3 + .../AArch64/GISel/AArch64CallLowering.cpp | 7 +- llvm/lib/Target/LoongArch/LoongArch.h | 2 + .../LoongArch/LoongArchExpandPseudoInsts.cpp | 121 ++++++++++++ .../LoongArch/LoongArchFloat32InstrInfo.td | 17 ++ .../LoongArch/LoongArchFloatInstrFormats.td | 12 ++ .../Target/LoongArch/LoongArchInstrInfo.cpp | 6 + .../LoongArch/LoongArchRegisterInfo.cpp | 7 - .../LoongArch/LoongArchTargetMachine.cpp | 1 + llvm/lib/Target/Mips/MipsISelLowering.cpp | 14 +- llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 19 +- llvm/lib/Transforms/IPO/GlobalOpt.cpp | 30 ++- .../Scalar/ConstraintElimination.cpp | 16 +- .../Scalar/CorrelatedValuePropagation.cpp | 68 +++---- llvm/lib/Transforms/Scalar/GVN.cpp | 11 +- .../lib/Transforms/Scalar/MemCpyOptimizer.cpp | 1 + .../Transforms/Vectorize/SLPVectorizer.cpp | 3 +- openmp/runtime/src/kmp_runtime.cpp | 3 +- openmp/runtime/src/kmp_wrapper_getpid.h | 2 +- 35 files changed, 475 insertions(+), 213 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticASTKinds.td b/clang/include/clang/Basic/DiagnosticASTKinds.td index 0794ed7ba683..694e6a5840bc 100644 --- a/clang/include/clang/Basic/DiagnosticASTKinds.td +++ b/clang/include/clang/Basic/DiagnosticASTKinds.td @@ -69,7 +69,7 @@ def note_consteval_address_accessible : Note< "%select{pointer|reference}0 to a consteval declaration " "is not a constant expression">; def note_constexpr_uninitialized : Note< - "subobject %0 is not initialized">; + "subobject %select{of type |}0%1 is not initialized">; def note_constexpr_uninitialized_base : Note< "constructor of base class %0 is not called">; def note_constexpr_static_local : Note< diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index cfd1c0f977c0..3752a23faa85 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -710,9 +710,13 @@ class Sema final { return result; } + // Saves the current floating-point pragma stack and clear it in this Sema. class FpPragmaStackSaveRAII { public: - FpPragmaStackSaveRAII(Sema &S) : S(S), SavedStack(S.FpPragmaStack) {} + FpPragmaStackSaveRAII(Sema &S) + : S(S), SavedStack(std::move(S.FpPragmaStack)) { + S.FpPragmaStack.Stack.clear(); + } ~FpPragmaStackSaveRAII() { S.FpPragmaStack = std::move(SavedStack); } private: @@ -722,7 +726,6 @@ class Sema final { void resetFPOptions(FPOptions FPO) { CurFPFeatures = FPO; - FpPragmaStack.Stack.clear(); FpPragmaStack.CurrentValue = FPO.getChangesFrom(FPOptions(LangOpts)); } diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index c62044f36194..99ae88a6cd69 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -2379,10 +2379,15 @@ static bool CheckEvaluationResult(CheckEvaluationResultKind CERK, const FieldDecl *SubobjectDecl, CheckedTemporaries &CheckedTemps) { if (!Value.hasValue()) { - assert(SubobjectDecl && "SubobjectDecl shall be non-null"); - Info.FFDiag(DiagLoc, diag::note_constexpr_uninitialized) << SubobjectDecl; - Info.Note(SubobjectDecl->getLocation(), - diag::note_constexpr_subobject_declared_here); + if (SubobjectDecl) { + Info.FFDiag(DiagLoc, diag::note_constexpr_uninitialized) + << /*(name)*/ 1 << SubobjectDecl; + Info.Note(SubobjectDecl->getLocation(), + diag::note_constexpr_subobject_declared_here); + } else { + Info.FFDiag(DiagLoc, diag::note_constexpr_uninitialized) + << /*of type*/ 0 << Type; + } return false; } diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/Interp/Interp.cpp index 4917f43f9512..59108b857577 100644 --- a/clang/lib/AST/Interp/Interp.cpp +++ b/clang/lib/AST/Interp/Interp.cpp @@ -382,7 +382,8 @@ bool CheckPure(InterpState &S, CodePtr OpPC, const CXXMethodDecl *MD) { static void DiagnoseUninitializedSubobject(InterpState &S, const SourceInfo &SI, const FieldDecl *SubObjDecl) { assert(SubObjDecl && "Subobject declaration does not exist"); - S.FFDiag(SI, diag::note_constexpr_uninitialized) << SubObjDecl; + S.FFDiag(SI, diag::note_constexpr_uninitialized) + << /*(name)*/ 1 << SubObjDecl; S.Note(SubObjDecl->getLocation(), diag::note_constexpr_subobject_declared_here); } diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp index 942daa4aa577..91369b7d8804 100644 --- a/clang/lib/CodeGen/CGExprConstant.cpp +++ b/clang/lib/CodeGen/CGExprConstant.cpp @@ -1736,9 +1736,10 @@ llvm::Constant *ConstantEmitter::tryEmitPrivate(const Expr *E, QualType destType) { assert(!destType->isVoidType() && "can't emit a void constant"); - if (llvm::Constant *C = - ConstExprEmitter(*this).Visit(const_cast(E), destType)) - return C; + if (!destType->isReferenceType()) + if (llvm::Constant *C = + ConstExprEmitter(*this).Visit(const_cast(E), destType)) + return C; Expr::EvalResult Result; diff --git a/clang/lib/Driver/ToolChains/Solaris.cpp b/clang/lib/Driver/ToolChains/Solaris.cpp index de5a69e4ca3f..85619a91554e 100644 --- a/clang/lib/Driver/ToolChains/Solaris.cpp +++ b/clang/lib/Driver/ToolChains/Solaris.cpp @@ -64,6 +64,7 @@ void solaris::Linker::ConstructJob(Compilation &C, const JobAction &JA, const InputInfoList &Inputs, const ArgList &Args, const char *LinkingOutput) const { + const Driver &D = getToolChain().getDriver(); const bool IsPIE = getPIE(Args, getToolChain()); ArgStringList CmdArgs; @@ -152,8 +153,11 @@ void solaris::Linker::ConstructJob(Compilation &C, const JobAction &JA, if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs, options::OPT_r)) { - if (getToolChain().ShouldLinkCXXStdlib(Args)) - getToolChain().AddCXXStdlibLibArgs(Args, CmdArgs); + if (D.CCCIsCXX()) { + if (getToolChain().ShouldLinkCXXStdlib(Args)) + getToolChain().AddCXXStdlibLibArgs(Args, CmdArgs); + CmdArgs.push_back("-lm"); + } if (Args.hasArg(options::OPT_fstack_protector) || Args.hasArg(options::OPT_fstack_protector_strong) || Args.hasArg(options::OPT_fstack_protector_all)) { @@ -172,7 +176,6 @@ void solaris::Linker::ConstructJob(Compilation &C, const JobAction &JA, CmdArgs.push_back("-lc"); if (!Args.hasArg(options::OPT_shared)) { CmdArgs.push_back("-lgcc"); - CmdArgs.push_back("-lm"); } const SanitizerArgs &SA = getToolChain().getSanitizerArgs(Args); if (NeedsSanitizerDeps) { diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 73840332e22c..4a1fc08455e5 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -3178,6 +3178,12 @@ static bool isFunctionDeclarationName(bool IsCpp, const FormatToken &Current, !Previous->isOneOf(tok::kw_return, tok::kw_co_return)) { return true; } + if (Previous->is(tok::r_paren) && Previous->is(TT_TypeDeclarationParen)) { + assert(Previous->MatchingParen); + assert(Previous->MatchingParen->is(tok::l_paren)); + assert(Previous->MatchingParen->is(TT_TypeDeclarationParen)); + return true; + } if (!Previous->isOneOf(tok::star, tok::amp, tok::ampamp, TT_TemplateCloser)) return false; Next = skipOperatorName(Next); diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp index 668ca38ad683..c1016c44a74a 100644 --- a/clang/lib/Format/WhitespaceManager.cpp +++ b/clang/lib/Format/WhitespaceManager.cpp @@ -965,7 +965,7 @@ void WhitespaceManager::alignConsecutiveDeclarations() { AlignTokens( Style, [](Change const &C) { - if (C.Tok->isOneOf(TT_FunctionDeclarationName, TT_FunctionTypeLParen)) + if (C.Tok->is(TT_FunctionDeclarationName)) return true; if (C.Tok->isNot(TT_StartOfName)) return false; diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp index 43b2a32cce71..cf1e3a94de7f 100644 --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -3998,8 +3998,15 @@ void Parser::ParseDeclarationSpecifiers( case tok::kw_thread_local: if (getLangOpts().C2x) Diag(Tok, diag::warn_c2x_compat_keyword) << Tok.getName(); - isInvalid = DS.SetStorageClassSpecThread(DeclSpec::TSCS_thread_local, Loc, - PrevSpec, DiagID); + // We map thread_local to _Thread_local in C23 mode so it retains the C + // semantics rather than getting the C++ semantics. + // FIXME: diagnostics will show _Thread_local when the user wrote + // thread_local in source in C23 mode; we need some general way to + // identify which way the user spelled the keyword in source. + isInvalid = DS.SetStorageClassSpecThread( + getLangOpts().C2x ? DeclSpec::TSCS__Thread_local + : DeclSpec::TSCS_thread_local, + Loc, PrevSpec, DiagID); isStorageClass = true; break; case tok::kw__Thread_local: diff --git a/libcxx/include/__config b/libcxx/include/__config index 3859c027cd29..10c056f313ff 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -40,15 +40,11 @@ // _LIBCPP_VERSION represents the version of libc++, which matches the version of LLVM. // Given a LLVM release LLVM XX.YY.ZZ (e.g. LLVM 17.0.1 == 17.00.01), _LIBCPP_VERSION is // defined to XXYYZZ. -# define _LIBCPP_VERSION 170003 +# define _LIBCPP_VERSION 170005 # define _LIBCPP_CONCAT_IMPL(_X, _Y) _X##_Y # define _LIBCPP_CONCAT(_X, _Y) _LIBCPP_CONCAT_IMPL(_X, _Y) -// Valid C++ identifier that revs with every libc++ version. This can be used to -// generate identifiers that must be unique for every released libc++ version. -# define _LIBCPP_VERSIONED_IDENTIFIER _LIBCPP_CONCAT(v, _LIBCPP_VERSION) - # if __STDC_HOSTED__ == 0 # define _LIBCPP_FREESTANDING # endif @@ -754,22 +750,54 @@ typedef __char32_t char32_t; # define _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION _LIBCPP_ALWAYS_INLINE # endif +# if _LIBCPP_ENABLE_HARDENED_MODE +# define _LIBCPP_HARDENING_SIG h +# elif _LIBCPP_ENABLE_ASSERTIONS +# define _LIBCPP_HARDENING_SIG s +# elif _LIBCPP_ENABLE_DEBUG_MODE +# define _LIBCPP_HARDENING_SIG d +# else +# define _LIBCPP_HARDENING_SIG u // for unchecked +# endif + +# ifdef _LIBCPP_HAS_NO_EXCEPTIONS +# define _LIBCPP_EXCEPTIONS_SIG n +# else +# define _LIBCPP_EXCEPTIONS_SIG e +# endif + +# define _LIBCPP_ODR_SIGNATURE \ + _LIBCPP_CONCAT(_LIBCPP_CONCAT(_LIBCPP_HARDENING_SIG, _LIBCPP_EXCEPTIONS_SIG), _LIBCPP_VERSION) + // This macro marks a symbol as being hidden from libc++'s ABI. This is achieved // on two levels: // 1. The symbol is given hidden visibility, which ensures that users won't start exporting // symbols from their dynamic library by means of using the libc++ headers. This ensures // that those symbols stay private to the dynamic library in which it is defined. // -// 2. The symbol is given an ABI tag that changes with each version of libc++. This ensures -// that no ODR violation can arise from mixing two TUs compiled with different versions -// of libc++ where we would have changed the definition of a symbol. If the symbols shared -// the same name, the ODR would require that their definitions be token-by-token equivalent, -// which basically prevents us from being able to make any change to any function in our -// headers. Using this ABI tag ensures that the symbol name is "bumped" artificially at -// each release, which lets us change the definition of these symbols at our leisure. -// Note that historically, this has been achieved in various ways, including force-inlining -// all functions or giving internal linkage to all functions. Both these (previous) solutions -// suffer from drawbacks that lead notably to code bloat. +// 2. The symbol is given an ABI tag that encodes the ODR-relevant properties of the library. +// This ensures that no ODR violation can arise from mixing two TUs compiled with different +// versions or configurations of libc++ (such as exceptions vs no-exceptions). Indeed, if the +// program contains two definitions of a function, the ODR requires them to be token-by-token +// equivalent, and the linker is allowed to pick either definition and discard the other one. +// +// For example, if a program contains a copy of `vector::at()` compiled with exceptions enabled +// *and* a copy of `vector::at()` compiled with exceptions disabled (by means of having two TUs +// compiled with different settings), the two definitions are both visible by the linker and they +// have the same name, but they have a meaningfully different implementation (one throws an exception +// and the other aborts the program). This violates the ODR and makes the program ill-formed, and in +// practice what will happen is that the linker will pick one of the definitions at random and will +// discard the other one. This can quite clearly lead to incorrect program behavior. +// +// A similar reasoning holds for many other properties that are ODR-affecting. Essentially any +// property that causes the code of a function to differ from the code in another configuration +// can be considered ODR-affecting. In practice, we don't encode all such properties in the ABI +// tag, but we encode the ones that we think are most important: library version, exceptions, and +// hardening mode. +// +// Note that historically, solving this problem has been achieved in various ways, including +// force-inlining all functions or giving internal linkage to all functions. Both these previous +// solutions suffer from drawbacks that lead notably to code bloat. // // Note that we use _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION to ensure that we don't depend // on _LIBCPP_HIDE_FROM_ABI methods of classes explicitly instantiated in the dynamic library. @@ -789,7 +817,7 @@ typedef __char32_t char32_t; # ifndef _LIBCPP_NO_ABI_TAG # define _LIBCPP_HIDE_FROM_ABI \ _LIBCPP_HIDDEN _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION \ - __attribute__((__abi_tag__(_LIBCPP_TOSTRING(_LIBCPP_VERSIONED_IDENTIFIER)))) + __attribute__((__abi_tag__(_LIBCPP_TOSTRING(_LIBCPP_ODR_SIGNATURE)))) # else # define _LIBCPP_HIDE_FROM_ABI _LIBCPP_HIDDEN _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION # endif diff --git a/libcxx/include/__expected/expected.h b/libcxx/include/__expected/expected.h index 7d57aa4db5f9..5836600312de 100644 --- a/libcxx/include/__expected/expected.h +++ b/libcxx/include/__expected/expected.h @@ -119,9 +119,7 @@ class expected { _LIBCPP_HIDE_FROM_ABI constexpr expected() noexcept(is_nothrow_default_constructible_v<_Tp>) // strengthened requires is_default_constructible_v<_Tp> - : __has_val_(true) { - std::construct_at(std::addressof(__union_.__val_)); - } + : __union_(std::in_place), __has_val_(true) {} _LIBCPP_HIDE_FROM_ABI constexpr expected(const expected&) = delete; @@ -136,14 +134,7 @@ class expected { noexcept(is_nothrow_copy_constructible_v<_Tp> && is_nothrow_copy_constructible_v<_Err>) // strengthened requires(is_copy_constructible_v<_Tp> && is_copy_constructible_v<_Err> && !(is_trivially_copy_constructible_v<_Tp> && is_trivially_copy_constructible_v<_Err>)) - : __has_val_(__other.__has_val_) { - if (__has_val_) { - std::construct_at(std::addressof(__union_.__val_), __other.__union_.__val_); - } else { - std::construct_at(std::addressof(__union_.__unex_), __other.__union_.__unex_); - } - } - + : __union_(__other.__has_val_, __other.__union_), __has_val_(__other.__has_val_) { } _LIBCPP_HIDE_FROM_ABI constexpr expected(expected&&) requires(is_move_constructible_v<_Tp> && is_move_constructible_v<_Err> @@ -154,13 +145,7 @@ class expected { noexcept(is_nothrow_move_constructible_v<_Tp> && is_nothrow_move_constructible_v<_Err>) requires(is_move_constructible_v<_Tp> && is_move_constructible_v<_Err> && !(is_trivially_move_constructible_v<_Tp> && is_trivially_move_constructible_v<_Err>)) - : __has_val_(__other.__has_val_) { - if (__has_val_) { - std::construct_at(std::addressof(__union_.__val_), std::move(__other.__union_.__val_)); - } else { - std::construct_at(std::addressof(__union_.__unex_), std::move(__other.__union_.__unex_)); - } - } + : __union_(__other.__has_val_, std::move(__other.__union_)), __has_val_(__other.__has_val_) { } private: template @@ -198,36 +183,21 @@ class expected { expected(const expected<_Up, _OtherErr>& __other) noexcept(is_nothrow_constructible_v<_Tp, const _Up&> && is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened - : __has_val_(__other.__has_val_) { - if (__has_val_) { - std::construct_at(std::addressof(__union_.__val_), __other.__union_.__val_); - } else { - std::construct_at(std::addressof(__union_.__unex_), __other.__union_.__unex_); - } - } + : __union_(__other.__has_val_, __other.__union_), __has_val_(__other.__has_val_) {} template requires __can_convert<_Up, _OtherErr, _Up, _OtherErr>::value _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_Up, _Tp> || !is_convertible_v<_OtherErr, _Err>) expected(expected<_Up, _OtherErr>&& __other) noexcept(is_nothrow_constructible_v<_Tp, _Up> && is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened - : __has_val_(__other.__has_val_) { - if (__has_val_) { - std::construct_at(std::addressof(__union_.__val_), std::move(__other.__union_.__val_)); - } else { - std::construct_at(std::addressof(__union_.__unex_), std::move(__other.__union_.__unex_)); - } - } + : __union_(__other.__has_val_, std::move(__other.__union_)), __has_val_(__other.__has_val_) {} template requires(!is_same_v, in_place_t> && !is_same_v> && !__is_std_unexpected>::value && is_constructible_v<_Tp, _Up>) _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_Up, _Tp>) - expected(_Up&& __u) - noexcept(is_nothrow_constructible_v<_Tp, _Up>) // strengthened - : __has_val_(true) { - std::construct_at(std::addressof(__union_.__val_), std::forward<_Up>(__u)); - } + expected(_Up&& __u) noexcept(is_nothrow_constructible_v<_Tp, _Up>) // strengthened + : __union_(std::in_place, std::forward<_Up>(__u)), __has_val_(true) {} template @@ -235,52 +205,40 @@ class expected { _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v) expected(const unexpected<_OtherErr>& __unex) noexcept(is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened - : __has_val_(false) { - std::construct_at(std::addressof(__union_.__unex_), __unex.error()); - } + : __union_(std::unexpect, __unex.error()), __has_val_(false) {} template requires is_constructible_v<_Err, _OtherErr> _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_OtherErr, _Err>) expected(unexpected<_OtherErr>&& __unex) noexcept(is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened - : __has_val_(false) { - std::construct_at(std::addressof(__union_.__unex_), std::move(__unex.error())); - } + : __union_(std::unexpect, std::move(__unex.error())), __has_val_(false) {} template requires is_constructible_v<_Tp, _Args...> _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(in_place_t, _Args&&... __args) noexcept(is_nothrow_constructible_v<_Tp, _Args...>) // strengthened - : __has_val_(true) { - std::construct_at(std::addressof(__union_.__val_), std::forward<_Args>(__args)...); - } + : __union_(std::in_place, std::forward<_Args>(__args)...), __has_val_(true) {} template requires is_constructible_v< _Tp, initializer_list<_Up>&, _Args... > _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(in_place_t, initializer_list<_Up> __il, _Args&&... __args) noexcept(is_nothrow_constructible_v<_Tp, initializer_list<_Up>&, _Args...>) // strengthened - : __has_val_(true) { - std::construct_at(std::addressof(__union_.__val_), __il, std::forward<_Args>(__args)...); - } + : __union_(std::in_place, __il, std::forward<_Args>(__args)...), __has_val_(true) {} template requires is_constructible_v<_Err, _Args...> _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(unexpect_t, _Args&&... __args) - noexcept(is_nothrow_constructible_v<_Err, _Args...>) // strengthened - : __has_val_(false) { - std::construct_at(std::addressof(__union_.__unex_), std::forward<_Args>(__args)...); - } + noexcept(is_nothrow_constructible_v<_Err, _Args...>) // strengthened + : __union_(std::unexpect, std::forward<_Args>(__args)...), __has_val_(false) {} template requires is_constructible_v< _Err, initializer_list<_Up>&, _Args... > _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(unexpect_t, initializer_list<_Up> __il, _Args&&... __args) noexcept(is_nothrow_constructible_v<_Err, initializer_list<_Up>&, _Args...>) // strengthened - : __has_val_(false) { - std::construct_at(std::addressof(__union_.__unex_), __il, std::forward<_Args>(__args)...); - } + : __union_(std::unexpect, __il, std::forward<_Args>(__args)...), __has_val_(false) {} // [expected.object.dtor], destructor @@ -439,9 +397,10 @@ class expected { std::destroy_at(std::addressof(__union_.__val_)); } else { std::destroy_at(std::addressof(__union_.__unex_)); - __has_val_ = true; } - return *std::construct_at(std::addressof(__union_.__val_), std::forward<_Args>(__args)...); + std::construct_at(std::addressof(__union_.__val_), std::forward<_Args>(__args)...); + __has_val_ = true; + return __union_.__val_; } template @@ -451,9 +410,10 @@ class expected { std::destroy_at(std::addressof(__union_.__val_)); } else { std::destroy_at(std::addressof(__union_.__unex_)); - __has_val_ = true; } - return *std::construct_at(std::addressof(__union_.__val_), __il, std::forward<_Args>(__args)...); + std::construct_at(std::addressof(__union_.__val_), __il, std::forward<_Args>(__args)...); + __has_val_ = true; + return __union_.__val_; } @@ -892,11 +852,15 @@ class expected { } private: - struct __empty_t {}; - template union __union_t { - _LIBCPP_HIDE_FROM_ABI constexpr __union_t() {} + template + _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(std::in_place_t, _Args&&... __args) + : __val_(std::forward<_Args>(__args)...) {} + + template + _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(std::unexpect_t, _Args&&... __args) + : __unex_(std::forward<_Args>(__args)...) {} template _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t( @@ -908,6 +872,14 @@ class expected { std::__expected_construct_unexpected_from_invoke_tag, _Func&& __f, _Args&&... __args) : __unex_(std::invoke(std::forward<_Func>(__f), std::forward<_Args>(__args)...)) {} + template + _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(bool __has_val, _Union&& __other) { + if (__has_val) + std::construct_at(std::addressof(__val_), std::forward<_Union>(__other).__val_); + else + std::construct_at(std::addressof(__unex_), std::forward<_Union>(__other).__unex_); + } + _LIBCPP_HIDE_FROM_ABI constexpr ~__union_t() requires(is_trivially_destructible_v<_ValueType> && is_trivially_destructible_v<_ErrorType>) = default; @@ -926,7 +898,16 @@ class expected { template requires(is_trivially_move_constructible_v<_ValueType> && is_trivially_move_constructible_v<_ErrorType>) union __union_t<_ValueType, _ErrorType> { - _LIBCPP_HIDE_FROM_ABI constexpr __union_t() : __empty_() {} + _LIBCPP_HIDE_FROM_ABI constexpr __union_t(const __union_t&) = default; + _LIBCPP_HIDE_FROM_ABI constexpr __union_t& operator=(const __union_t&) = default; + + template + _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(std::in_place_t, _Args&&... __args) + : __val_(std::forward<_Args>(__args)...) {} + + template + _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(std::unexpect_t, _Args&&... __args) + : __unex_(std::forward<_Args>(__args)...) {} template _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t( @@ -938,6 +919,14 @@ class expected { std::__expected_construct_unexpected_from_invoke_tag, _Func&& __f, _Args&&... __args) : __unex_(std::invoke(std::forward<_Func>(__f), std::forward<_Args>(__args)...)) {} + template + _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(bool __has_val, _Union&& __other) { + if (__has_val) + std::construct_at(std::addressof(__val_), std::forward<_Union>(__other).__val_); + else + std::construct_at(std::addressof(__unex_), std::forward<_Union>(__other).__unex_); + } + _LIBCPP_HIDE_FROM_ABI constexpr ~__union_t() requires(is_trivially_destructible_v<_ValueType> && is_trivially_destructible_v<_ErrorType>) = default; @@ -947,7 +936,6 @@ class expected { requires(!is_trivially_destructible_v<_ValueType> || !is_trivially_destructible_v<_ErrorType>) {} - _LIBCPP_NO_UNIQUE_ADDRESS __empty_t __empty_; _LIBCPP_NO_UNIQUE_ADDRESS _ValueType __val_; _LIBCPP_NO_UNIQUE_ADDRESS _ErrorType __unex_; }; @@ -995,11 +983,7 @@ class expected<_Tp, _Err> { _LIBCPP_HIDE_FROM_ABI constexpr expected(const expected& __rhs) noexcept(is_nothrow_copy_constructible_v<_Err>) // strengthened requires(is_copy_constructible_v<_Err> && !is_trivially_copy_constructible_v<_Err>) - : __has_val_(__rhs.__has_val_) { - if (!__rhs.__has_val_) { - std::construct_at(std::addressof(__union_.__unex_), __rhs.__union_.__unex_); - } - } + : __union_(__rhs.__has_val_, __rhs.__union_), __has_val_(__rhs.__has_val_) {} _LIBCPP_HIDE_FROM_ABI constexpr expected(expected&&) requires(is_move_constructible_v<_Err> && is_trivially_move_constructible_v<_Err>) @@ -1008,51 +992,35 @@ class expected<_Tp, _Err> { _LIBCPP_HIDE_FROM_ABI constexpr expected(expected&& __rhs) noexcept(is_nothrow_move_constructible_v<_Err>) requires(is_move_constructible_v<_Err> && !is_trivially_move_constructible_v<_Err>) - : __has_val_(__rhs.__has_val_) { - if (!__rhs.__has_val_) { - std::construct_at(std::addressof(__union_.__unex_), std::move(__rhs.__union_.__unex_)); - } - } + : __union_(__rhs.__has_val_, std::move(__rhs.__union_)), __has_val_(__rhs.__has_val_) {} template requires __can_convert<_Up, _OtherErr, const _OtherErr&>::value _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v) expected(const expected<_Up, _OtherErr>& __rhs) noexcept(is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened - : __has_val_(__rhs.__has_val_) { - if (!__rhs.__has_val_) { - std::construct_at(std::addressof(__union_.__unex_), __rhs.__union_.__unex_); - } - } + : __union_(__rhs.__has_val_, __rhs.__union_), __has_val_(__rhs.__has_val_) {} template requires __can_convert<_Up, _OtherErr, _OtherErr>::value _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_OtherErr, _Err>) expected(expected<_Up, _OtherErr>&& __rhs) noexcept(is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened - : __has_val_(__rhs.__has_val_) { - if (!__rhs.__has_val_) { - std::construct_at(std::addressof(__union_.__unex_), std::move(__rhs.__union_.__unex_)); - } - } + : __union_(__rhs.__has_val_, std::move(__rhs.__union_)), __has_val_(__rhs.__has_val_) {} template requires is_constructible_v<_Err, const _OtherErr&> _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v) expected(const unexpected<_OtherErr>& __unex) noexcept(is_nothrow_constructible_v<_Err, const _OtherErr&>) // strengthened - : __has_val_(false) { - std::construct_at(std::addressof(__union_.__unex_), __unex.error()); - } + : __union_(std::unexpect, __unex.error()), __has_val_(false) {} template requires is_constructible_v<_Err, _OtherErr> _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_OtherErr, _Err>) expected(unexpected<_OtherErr>&& __unex) noexcept(is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened - : __has_val_(false) { - std::construct_at(std::addressof(__union_.__unex_), std::move(__unex.error())); - } + : __union_(std::unexpect, std::move(__unex.error())), __has_val_(false) {} _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(in_place_t) noexcept : __has_val_(true) {} @@ -1060,17 +1028,13 @@ class expected<_Tp, _Err> { requires is_constructible_v<_Err, _Args...> _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(unexpect_t, _Args&&... __args) noexcept(is_nothrow_constructible_v<_Err, _Args...>) // strengthened - : __has_val_(false) { - std::construct_at(std::addressof(__union_.__unex_), std::forward<_Args>(__args)...); - } + : __union_(std::unexpect, std::forward<_Args>(__args)...), __has_val_(false) {} template requires is_constructible_v< _Err, initializer_list<_Up>&, _Args... > _LIBCPP_HIDE_FROM_ABI constexpr explicit expected(unexpect_t, initializer_list<_Up> __il, _Args&&... __args) noexcept(is_nothrow_constructible_v<_Err, initializer_list<_Up>&, _Args...>) // strengthened - : __has_val_(false) { - std::construct_at(std::addressof(__union_.__unex_), __il, std::forward<_Args>(__args)...); - } + : __union_(std::unexpect, __il, std::forward<_Args>(__args)...), __has_val_(false) {} private: template @@ -1504,11 +1468,23 @@ class expected<_Tp, _Err> { union __union_t { _LIBCPP_HIDE_FROM_ABI constexpr __union_t() : __empty_() {} + template + _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(std::unexpect_t, _Args&&... __args) + : __unex_(std::forward<_Args>(__args)...) {} + template _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t( __expected_construct_unexpected_from_invoke_tag, _Func&& __f, _Args&&... __args) : __unex_(std::invoke(std::forward<_Func>(__f), std::forward<_Args>(__args)...)) {} + template + _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(bool __has_val, _Union&& __other) { + if (__has_val) + std::construct_at(std::addressof(__empty_)); + else + std::construct_at(std::addressof(__unex_), std::forward<_Union>(__other).__unex_); + } + _LIBCPP_HIDE_FROM_ABI constexpr ~__union_t() requires(is_trivially_destructible_v<_ErrorType>) = default; @@ -1529,11 +1505,23 @@ class expected<_Tp, _Err> { union __union_t<_ErrorType> { _LIBCPP_HIDE_FROM_ABI constexpr __union_t() : __empty_() {} + template + _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(std::unexpect_t, _Args&&... __args) + : __unex_(std::forward<_Args>(__args)...) {} + template _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t( __expected_construct_unexpected_from_invoke_tag, _Func&& __f, _Args&&... __args) : __unex_(std::invoke(std::forward<_Func>(__f), std::forward<_Args>(__args)...)) {} + template + _LIBCPP_HIDE_FROM_ABI constexpr explicit __union_t(bool __has_val, _Union&& __other) { + if (__has_val) + std::construct_at(std::addressof(__empty_)); + else + std::construct_at(std::addressof(__unex_), std::forward<_Union>(__other).__unex_); + } + _LIBCPP_HIDE_FROM_ABI constexpr ~__union_t() requires(is_trivially_destructible_v<_ErrorType>) = default; diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp index 3830f25debaf..0801296cab49 100644 --- a/llvm/lib/CodeGen/BranchFolding.cpp +++ b/llvm/lib/CodeGen/BranchFolding.cpp @@ -485,13 +485,7 @@ BranchFolder::MergePotentialsElt::operator<(const MergePotentialsElt &o) const { return true; if (getBlock()->getNumber() > o.getBlock()->getNumber()) return false; - // _GLIBCXX_DEBUG checks strict weak ordering, which involves comparing - // an object with itself. -#ifndef _GLIBCXX_DEBUG - llvm_unreachable("Predecessor appears twice"); -#else return false; -#endif } /// CountTerminators - Count the number of terminators in the given diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index db8f61eee606..ad70655de349 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -1000,6 +1000,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer { SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N); SDValue WidenVecOp_INSERT_SUBVECTOR(SDNode *N); SDValue WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N); + SDValue WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N); SDValue WidenVecOp_STORE(SDNode* N); SDValue WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo); SDValue WidenVecOp_VP_STRIDED_STORE(SDNode *N, unsigned OpNo); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index 8c117c1c74dc..9c1839f2576e 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -5946,6 +5946,11 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { case ISD::EXPERIMENTAL_VP_STRIDED_STORE: Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo); break; + case ISD::ANY_EXTEND_VECTOR_INREG: + case ISD::SIGN_EXTEND_VECTOR_INREG: + case ISD::ZERO_EXTEND_VECTOR_INREG: + Res = WidenVecOp_EXTEND_VECTOR_INREG(N); + break; case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break; case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break; case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break; @@ -6338,6 +6343,11 @@ SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { N->getValueType(0), InOp, N->getOperand(1)); } +SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) { + SDValue InOp = GetWidenedVector(N->getOperand(0)); + return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), InOp); +} + SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { // We have to widen the value, but we want only to store the original // vector type. diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 30bd580ad86a..0691e07a639b 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -4764,7 +4764,7 @@ int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI, Offset = Remainder; else { NewOffset = NewOffset < 0 ? MinOff : MaxOff; - Offset = Offset - NewOffset * Scale + Remainder; + Offset = Offset - NewOffset * Scale; } if (EmittableOffset) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 353e96856b8f..bee9ec4c7132 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -212,6 +212,30 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, return (CallerBits & CalleeBits) == CalleeBits; } +bool AArch64TTIImpl::areTypesABICompatible( + const Function *Caller, const Function *Callee, + const ArrayRef &Types) const { + if (!BaseT::areTypesABICompatible(Caller, Callee, Types)) + return false; + + // We need to ensure that argument promotion does not attempt to promote + // pointers to fixed-length vector types larger than 128 bits like + // <8 x float> (and pointers to aggregate types which have such fixed-length + // vector type members) into the values of the pointees. Such vector types + // are used for SVE VLS but there is no ABI for SVE VLS arguments and the + // backend cannot lower such value arguments. The 128-bit fixed-length SVE + // types can be safely treated as 128-bit NEON types and they cannot be + // distinguished in IR. + if (ST->useSVEForFixedLengthVectors() && llvm::any_of(Types, [](Type *Ty) { + auto FVTy = dyn_cast(Ty); + return FVTy && + FVTy->getScalarSizeInBits() * FVTy->getNumElements() > 128; + })) + return false; + + return true; +} + bool AArch64TTIImpl::shouldMaximizeVectorBandwidth( TargetTransformInfo::RegisterKind K) const { assert(K != TargetTransformInfo::RGK_Scalar); diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index 787cb3c5d34b..d1977a62a76d 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -77,6 +77,9 @@ class AArch64TTIImpl : public BasicTTIImplBase { bool areInlineCompatible(const Function *Caller, const Function *Callee) const; + bool areTypesABICompatible(const Function *Caller, const Function *Callee, + const ArrayRef &Types) const; + /// \name Scalar TTI Implementations /// @{ diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp index e78d8bb487a9..c56e3373d3a7 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp @@ -829,9 +829,9 @@ bool AArch64CallLowering::doCallerAndCalleePassArgsTheSameWay( bool AArch64CallLowering::areCalleeOutgoingArgsTailCallable( CallLoweringInfo &Info, MachineFunction &MF, - SmallVectorImpl &OutArgs) const { + SmallVectorImpl &OrigOutArgs) const { // If there are no outgoing arguments, then we are done. - if (OutArgs.empty()) + if (OrigOutArgs.empty()) return true; const Function &CallerF = MF.getFunction(); @@ -851,6 +851,9 @@ bool AArch64CallLowering::areCalleeOutgoingArgsTailCallable( AArch64OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg, Subtarget, /*IsReturn*/ false); + // determineAssignments() may modify argument flags, so make a copy. + SmallVector OutArgs; + append_range(OutArgs, OrigOutArgs); if (!determineAssignments(CalleeAssigner, OutArgs, OutInfo)) { LLVM_DEBUG(dbgs() << "... Could not analyze call operands.\n"); return false; diff --git a/llvm/lib/Target/LoongArch/LoongArch.h b/llvm/lib/Target/LoongArch/LoongArch.h index 05f4ac8c9255..09ca089c9115 100644 --- a/llvm/lib/Target/LoongArch/LoongArch.h +++ b/llvm/lib/Target/LoongArch/LoongArch.h @@ -36,9 +36,11 @@ bool lowerLoongArchMachineOperandToMCOperand(const MachineOperand &MO, FunctionPass *createLoongArchExpandAtomicPseudoPass(); FunctionPass *createLoongArchISelDag(LoongArchTargetMachine &TM); FunctionPass *createLoongArchPreRAExpandPseudoPass(); +FunctionPass *createLoongArchExpandPseudoPass(); void initializeLoongArchDAGToDAGISelPass(PassRegistry &); void initializeLoongArchExpandAtomicPseudoPass(PassRegistry &); void initializeLoongArchPreRAExpandPseudoPass(PassRegistry &); +void initializeLoongArchExpandPseudoPass(PassRegistry &); } // end namespace llvm #endif // LLVM_LIB_TARGET_LOONGARCH_LOONGARCH_H diff --git a/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp b/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp index dd0b2cfde544..72c1f1cec198 100644 --- a/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp @@ -29,6 +29,8 @@ using namespace llvm; #define LOONGARCH_PRERA_EXPAND_PSEUDO_NAME \ "LoongArch Pre-RA pseudo instruction expansion pass" +#define LOONGARCH_EXPAND_PSEUDO_NAME \ + "LoongArch pseudo instruction expansion pass" namespace { @@ -513,15 +515,134 @@ bool LoongArchPreRAExpandPseudo::expandFunctionCALL( return true; } +class LoongArchExpandPseudo : public MachineFunctionPass { +public: + const LoongArchInstrInfo *TII; + static char ID; + + LoongArchExpandPseudo() : MachineFunctionPass(ID) { + initializeLoongArchExpandPseudoPass(*PassRegistry::getPassRegistry()); + } + + bool runOnMachineFunction(MachineFunction &MF) override; + + StringRef getPassName() const override { + return LOONGARCH_EXPAND_PSEUDO_NAME; + } + +private: + bool expandMBB(MachineBasicBlock &MBB); + bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI); + bool expandCopyCFR(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI); +}; + +char LoongArchExpandPseudo::ID = 0; + +bool LoongArchExpandPseudo::runOnMachineFunction(MachineFunction &MF) { + TII = + static_cast(MF.getSubtarget().getInstrInfo()); + + bool Modified = false; + for (auto &MBB : MF) + Modified |= expandMBB(MBB); + + return Modified; +} + +bool LoongArchExpandPseudo::expandMBB(MachineBasicBlock &MBB) { + bool Modified = false; + + MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); + while (MBBI != E) { + MachineBasicBlock::iterator NMBBI = std::next(MBBI); + Modified |= expandMI(MBB, MBBI, NMBBI); + MBBI = NMBBI; + } + + return Modified; +} + +bool LoongArchExpandPseudo::expandMI(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI) { + switch (MBBI->getOpcode()) { + case LoongArch::PseudoCopyCFR: + return expandCopyCFR(MBB, MBBI, NextMBBI); + } + + return false; +} + +bool LoongArchExpandPseudo::expandCopyCFR( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI) { + MachineFunction *MF = MBB.getParent(); + MachineInstr &MI = *MBBI; + DebugLoc DL = MI.getDebugLoc(); + + // Expand: + // MBB: + // fcmp.caf.s $dst, $fa0, $fa0 # set $dst 0(false) + // bceqz $src, SinkBB + // FalseBB: + // fcmp.cueq.s $dst, $fa0, $fa0 # set $dst 1(true) + // SinkBB: + // fallthrough + + const BasicBlock *LLVM_BB = MBB.getBasicBlock(); + auto *FalseBB = MF->CreateMachineBasicBlock(LLVM_BB); + auto *SinkBB = MF->CreateMachineBasicBlock(LLVM_BB); + + MF->insert(++MBB.getIterator(), FalseBB); + MF->insert(++FalseBB->getIterator(), SinkBB); + + Register DestReg = MI.getOperand(0).getReg(); + Register SrcReg = MI.getOperand(1).getReg(); + // DestReg = 0 + BuildMI(MBB, MBBI, DL, TII->get(LoongArch::SET_CFR_FALSE), DestReg); + // Insert branch instruction. + BuildMI(MBB, MBBI, DL, TII->get(LoongArch::BCEQZ)) + .addReg(SrcReg) + .addMBB(SinkBB); + // DestReg = 1 + BuildMI(FalseBB, DL, TII->get(LoongArch::SET_CFR_TRUE), DestReg); + + FalseBB->addSuccessor(SinkBB); + + SinkBB->splice(SinkBB->end(), &MBB, MI, MBB.end()); + SinkBB->transferSuccessors(&MBB); + + MBB.addSuccessor(FalseBB); + MBB.addSuccessor(SinkBB); + + NextMBBI = MBB.end(); + MI.eraseFromParent(); + + // Make sure live-ins are correctly attached to this new basic block. + LivePhysRegs LiveRegs; + computeAndAddLiveIns(LiveRegs, *FalseBB); + computeAndAddLiveIns(LiveRegs, *SinkBB); + + return true; +} + } // end namespace INITIALIZE_PASS(LoongArchPreRAExpandPseudo, "loongarch-prera-expand-pseudo", LOONGARCH_PRERA_EXPAND_PSEUDO_NAME, false, false) +INITIALIZE_PASS(LoongArchExpandPseudo, "loongarch-expand-pseudo", + LOONGARCH_EXPAND_PSEUDO_NAME, false, false) + namespace llvm { FunctionPass *createLoongArchPreRAExpandPseudoPass() { return new LoongArchPreRAExpandPseudo(); } +FunctionPass *createLoongArchExpandPseudoPass() { + return new LoongArchExpandPseudo(); +} } // end namespace llvm diff --git a/llvm/lib/Target/LoongArch/LoongArchFloat32InstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchFloat32InstrInfo.td index eb49ae329ebe..826db54febd3 100644 --- a/llvm/lib/Target/LoongArch/LoongArchFloat32InstrInfo.td +++ b/llvm/lib/Target/LoongArch/LoongArchFloat32InstrInfo.td @@ -126,6 +126,23 @@ def PseudoST_CFR : Pseudo<(outs), let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in def PseudoLD_CFR : Pseudo<(outs CFR:$ccd), (ins GPR:$rj, grlenimm:$imm)>; + +// SET_CFR_{FALSE,TRUE} +// These instructions are defined in order to avoid expensive check error if +// regular instruction patterns are used. +// fcmp.caf.s $dst, $fa0, $fa0 +def SET_CFR_FALSE : SET_CFR<0x0c100000, "fcmp.caf.s">; +// fcmp.cueq.s $dst, $fa0, $fa0 +def SET_CFR_TRUE : SET_CFR<0x0c160000, "fcmp.cueq.s">; + +// Pseudo instruction for copying CFRs. +def PseudoCopyCFR : Pseudo<(outs CFR:$dst), (ins CFR:$src)> { + let mayLoad = 0; + let mayStore = 0; + let hasSideEffects = 0; + let Size = 12; +} + } // Predicates = [HasBasicF] //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/LoongArch/LoongArchFloatInstrFormats.td b/llvm/lib/Target/LoongArch/LoongArchFloatInstrFormats.td index f853fca5c8b6..f66f620ca8b2 100644 --- a/llvm/lib/Target/LoongArch/LoongArchFloatInstrFormats.td +++ b/llvm/lib/Target/LoongArch/LoongArchFloatInstrFormats.td @@ -218,3 +218,15 @@ class FP_STORE_2RI12 op, RegisterClass rc = FPR32> : FPFmt2RI12; } // hasSideEffects = 0, mayLoad = 0, mayStore = 1 + +// This class is used to define `SET_CFR_{FALSE,TRUE}` instructions which are +// used to expand `PseudoCopyCFR`. +class SET_CFR op, string opcstr> + : FP_CMP { + let isCodeGenOnly = 1; + let fj = 0; // fa0 + let fk = 0; // fa0 + let AsmString = opcstr # "\t$cd, $$fa0, $$fa0"; + let OutOperandList = (outs CFR:$cd); + let InOperandList = (ins); +} diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp index f5e32c452933..ef79b8a0dcd3 100644 --- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp @@ -61,6 +61,12 @@ void LoongArchInstrInfo::copyPhysReg(MachineBasicBlock &MBB, .addReg(SrcReg, getKillRegState(KillSrc)); return; } + // CFR->CFR copy. + if (LoongArch::CFRRegClass.contains(DstReg, SrcReg)) { + BuildMI(MBB, MBBI, DL, get(LoongArch::PseudoCopyCFR), DstReg) + .addReg(SrcReg, getKillRegState(KillSrc)); + return; + } // FPR->FPR copies. unsigned Opc; diff --git a/llvm/lib/Target/LoongArch/LoongArchRegisterInfo.cpp b/llvm/lib/Target/LoongArch/LoongArchRegisterInfo.cpp index 4037c4d370bb..257b947a3ce4 100644 --- a/llvm/lib/Target/LoongArch/LoongArchRegisterInfo.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchRegisterInfo.cpp @@ -98,13 +98,6 @@ LoongArchRegisterInfo::getReservedRegs(const MachineFunction &MF) const { if (TFI->hasBP(MF)) markSuperRegs(Reserved, LoongArchABI::getBPReg()); // bp - // FIXME: To avoid generating COPY instructions between CFRs, only use $fcc0. - // This is required to work around the fact that COPY instruction between CFRs - // is not provided in LoongArch. - if (MF.getSubtarget().hasBasicF()) - for (size_t Reg = LoongArch::FCC1; Reg <= LoongArch::FCC7; ++Reg) - markSuperRegs(Reserved, Reg); - assert(checkAllSuperRegsMarked(Reserved)); return Reserved; } diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp index 46e4a06f6bc0..d0a4e9375048 100644 --- a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp @@ -180,6 +180,7 @@ LoongArchTargetMachine::getTargetTransformInfo(const Function &F) const { void LoongArchPassConfig::addPreEmitPass() { addPass(&BranchRelaxationPassID); } void LoongArchPassConfig::addPreEmitPass2() { + addPass(createLoongArchExpandPseudoPass()); // Schedule the expansion of AtomicPseudos at the last possible moment, // avoiding the possibility for other passes to break the requirements for // forward progress in the LL/SC block. diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp index 18d7773067f1..3c69ec4912b1 100644 --- a/llvm/lib/Target/Mips/MipsISelLowering.cpp +++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp @@ -2593,12 +2593,13 @@ SDValue MipsTargetLowering::lowerShiftLeftParts(SDValue Op, SDValue Shamt = Op.getOperand(2); // if shamt < (VT.bits): // lo = (shl lo, shamt) - // hi = (or (shl hi, shamt) (srl (srl lo, 1), ~shamt)) + // hi = (or (shl hi, shamt) (srl (srl lo, 1), (xor shamt, (VT.bits-1)))) // else: // lo = 0 // hi = (shl lo, shamt[4:0]) - SDValue Not = DAG.getNode(ISD::XOR, DL, MVT::i32, Shamt, - DAG.getConstant(-1, DL, MVT::i32)); + SDValue Not = + DAG.getNode(ISD::XOR, DL, MVT::i32, Shamt, + DAG.getConstant(VT.getSizeInBits() - 1, DL, MVT::i32)); SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, DAG.getConstant(1, DL, VT)); SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, Not); @@ -2623,7 +2624,7 @@ SDValue MipsTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG, MVT VT = Subtarget.isGP64bit() ? MVT::i64 : MVT::i32; // if shamt < (VT.bits): - // lo = (or (shl (shl hi, 1), ~shamt) (srl lo, shamt)) + // lo = (or (shl (shl hi, 1), (xor shamt, (VT.bits-1))) (srl lo, shamt)) // if isSRA: // hi = (sra hi, shamt) // else: @@ -2635,8 +2636,9 @@ SDValue MipsTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG, // else: // lo = (srl hi, shamt[4:0]) // hi = 0 - SDValue Not = DAG.getNode(ISD::XOR, DL, MVT::i32, Shamt, - DAG.getConstant(-1, DL, MVT::i32)); + SDValue Not = + DAG.getNode(ISD::XOR, DL, MVT::i32, Shamt, + DAG.getConstant(VT.getSizeInBits() - 1, DL, MVT::i32)); SDValue ShiftLeft1Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, DAG.getConstant(1, DL, VT)); SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, ShiftLeft1Hi, Not); diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index e0cbca6dc1c2..c1065f73000f 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -299,11 +299,6 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB, MCRegister SrcReg, bool KillSrc) const { const TargetRegisterInfo *TRI = STI.getRegisterInfo(); - if (RISCV::GPRPF64RegClass.contains(DstReg)) - DstReg = TRI->getSubReg(DstReg, RISCV::sub_32); - if (RISCV::GPRPF64RegClass.contains(SrcReg)) - SrcReg = TRI->getSubReg(SrcReg, RISCV::sub_32); - if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) { BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), DstReg) .addReg(SrcReg, getKillRegState(KillSrc)) @@ -311,6 +306,20 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB, return; } + if (RISCV::GPRPF64RegClass.contains(DstReg, SrcReg)) { + // Emit an ADDI for both parts of GPRPF64. + BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), + TRI->getSubReg(DstReg, RISCV::sub_32)) + .addReg(TRI->getSubReg(SrcReg, RISCV::sub_32), getKillRegState(KillSrc)) + .addImm(0); + BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), + TRI->getSubReg(DstReg, RISCV::sub_32_hi)) + .addReg(TRI->getSubReg(SrcReg, RISCV::sub_32_hi), + getKillRegState(KillSrc)) + .addImm(0); + return; + } + // Handle copy from csr if (RISCV::VCSRRegClass.contains(SrcReg) && RISCV::GPRRegClass.contains(DstReg)) { diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp index 1ccc523ead8a..8012e1e650a0 100644 --- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp +++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp @@ -1701,13 +1701,16 @@ static void RemoveAttribute(Function *F, Attribute::AttrKind A) { /// idea here is that we don't want to mess with the convention if the user /// explicitly requested something with performance implications like coldcc, /// GHC, or anyregcc. -static bool hasChangeableCC(Function *F) { +static bool hasChangeableCCImpl(Function *F) { CallingConv::ID CC = F->getCallingConv(); // FIXME: Is it worth transforming x86_stdcallcc and x86_fastcallcc? if (CC != CallingConv::C && CC != CallingConv::X86_ThisCall) return false; + if (F->isVarArg()) + return false; + // FIXME: Change CC for the whole chain of musttail calls when possible. // // Can't change CC of the function that either has musttail calls, or is a @@ -1727,7 +1730,16 @@ static bool hasChangeableCC(Function *F) { if (BB.getTerminatingMustTailCall()) return false; - return true; + return !F->hasAddressTaken(); +} + +using ChangeableCCCacheTy = SmallDenseMap; +static bool hasChangeableCC(Function *F, + ChangeableCCCacheTy &ChangeableCCCache) { + auto Res = ChangeableCCCache.try_emplace(F, false); + if (Res.second) + Res.first->second = hasChangeableCCImpl(F); + return Res.first->second; } /// Return true if the block containing the call site has a BlockFrequency of @@ -1781,7 +1793,8 @@ static void changeCallSitesToColdCC(Function *F) { // coldcc calling convention. static bool hasOnlyColdCalls(Function &F, - function_ref GetBFI) { + function_ref GetBFI, + ChangeableCCCacheTy &ChangeableCCCache) { for (BasicBlock &BB : F) { for (Instruction &I : BB) { if (CallInst *CI = dyn_cast(&I)) { @@ -1800,8 +1813,7 @@ hasOnlyColdCalls(Function &F, if (!CalledFn->hasLocalLinkage()) return false; // Check if it's valid to use coldcc calling convention. - if (!hasChangeableCC(CalledFn) || CalledFn->isVarArg() || - CalledFn->hasAddressTaken()) + if (!hasChangeableCC(CalledFn, ChangeableCCCache)) return false; BlockFrequencyInfo &CallerBFI = GetBFI(F); if (!isColdCallSite(*CI, CallerBFI)) @@ -1931,9 +1943,10 @@ OptimizeFunctions(Module &M, bool Changed = false; + ChangeableCCCacheTy ChangeableCCCache; std::vector AllCallsCold; for (Function &F : llvm::make_early_inc_range(M)) - if (hasOnlyColdCalls(F, GetBFI)) + if (hasOnlyColdCalls(F, GetBFI, ChangeableCCCache)) AllCallsCold.push_back(&F); // Optimize functions. @@ -1995,7 +2008,7 @@ OptimizeFunctions(Module &M, continue; } - if (hasChangeableCC(&F) && !F.isVarArg() && !F.hasAddressTaken()) { + if (hasChangeableCC(&F, ChangeableCCCache)) { NumInternalFunc++; TargetTransformInfo &TTI = GetTTI(F); // Change the calling convention to coldcc if either stress testing is @@ -2005,6 +2018,7 @@ OptimizeFunctions(Module &M, if (EnableColdCCStressTest || (TTI.useColdCCForColdCall(F) && isValidCandidateForColdCC(F, GetBFI, AllCallsCold))) { + ChangeableCCCache.erase(&F); F.setCallingConv(CallingConv::Cold); changeCallSitesToColdCC(&F); Changed = true; @@ -2012,7 +2026,7 @@ OptimizeFunctions(Module &M, } } - if (hasChangeableCC(&F) && !F.isVarArg() && !F.hasAddressTaken()) { + if (hasChangeableCC(&F, ChangeableCCCache)) { // If this function has a calling convention worth changing, is not a // varargs function, and is only called directly, promote it to use the // Fast calling convention. diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp index 2b88dd08d88b..5365bca0ab47 100644 --- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp +++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp @@ -412,6 +412,19 @@ static Decomposition decompose(Value *V, return ResA; }; + Type *Ty = V->getType()->getScalarType(); + if (Ty->isPointerTy() && !IsSigned) { + if (auto *GEP = dyn_cast(V)) + return decomposeGEP(*GEP, Preconditions, IsSigned, DL); + return V; + } + + // Don't handle integers > 64 bit. Our coefficients are 64-bit large, so + // coefficient add/mul may wrap, while the operation in the full bit width + // would not. + if (!Ty->isIntegerTy() || Ty->getIntegerBitWidth() > 64) + return V; + // Decompose \p V used with a signed predicate. if (IsSigned) { if (auto *CI = dyn_cast(V)) { @@ -439,9 +452,6 @@ static Decomposition decompose(Value *V, return int64_t(CI->getZExtValue()); } - if (auto *GEP = dyn_cast(V)) - return decomposeGEP(*GEP, Preconditions, IsSigned, DL); - Value *Op0; bool IsKnownNonNegative = false; if (match(V, m_ZExt(m_Value(Op0)))) { diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp index 48b27a1ea0a2..523196e5e6ea 100644 --- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp +++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp @@ -470,17 +470,17 @@ static bool processBinOp(BinaryOperator *BinOp, LazyValueInfo *LVI); // because it is negation-invariant. static bool processAbsIntrinsic(IntrinsicInst *II, LazyValueInfo *LVI) { Value *X = II->getArgOperand(0); - bool IsIntMinPoison = cast(II->getArgOperand(1))->isOne(); - Type *Ty = X->getType(); - Constant *IntMin = - ConstantInt::get(Ty, APInt::getSignedMinValue(Ty->getScalarSizeInBits())); - LazyValueInfo::Tristate Result; + if (!Ty->isIntegerTy()) + return false; + + bool IsIntMinPoison = cast(II->getArgOperand(1))->isOne(); + APInt IntMin = APInt::getSignedMinValue(Ty->getScalarSizeInBits()); + ConstantRange Range = LVI->getConstantRangeAtUse( + II->getOperandUse(0), /*UndefAllowed*/ IsIntMinPoison); // Is X in [0, IntMin]? NOTE: INT_MIN is fine! - Result = LVI->getPredicateAt(CmpInst::Predicate::ICMP_ULE, X, IntMin, II, - /*UseBlockValue=*/true); - if (Result == LazyValueInfo::True) { + if (Range.icmp(CmpInst::ICMP_ULE, IntMin)) { ++NumAbs; II->replaceAllUsesWith(X); II->eraseFromParent(); @@ -488,40 +488,30 @@ static bool processAbsIntrinsic(IntrinsicInst *II, LazyValueInfo *LVI) { } // Is X in [IntMin, 0]? NOTE: INT_MIN is fine! - Constant *Zero = ConstantInt::getNullValue(Ty); - Result = LVI->getPredicateAt(CmpInst::Predicate::ICMP_SLE, X, Zero, II, - /*UseBlockValue=*/true); - assert(Result != LazyValueInfo::False && "Should have been handled already."); + if (Range.getSignedMax().isNonPositive()) { + IRBuilder<> B(II); + Value *NegX = B.CreateNeg(X, II->getName(), /*HasNUW=*/false, + /*HasNSW=*/IsIntMinPoison); + ++NumAbs; + II->replaceAllUsesWith(NegX); + II->eraseFromParent(); - if (Result == LazyValueInfo::Unknown) { - // Argument's range crosses zero. - bool Changed = false; - if (!IsIntMinPoison) { - // Can we at least tell that the argument is never INT_MIN? - Result = LVI->getPredicateAt(CmpInst::Predicate::ICMP_NE, X, IntMin, II, - /*UseBlockValue=*/true); - if (Result == LazyValueInfo::True) { - ++NumNSW; - ++NumSubNSW; - II->setArgOperand(1, ConstantInt::getTrue(II->getContext())); - Changed = true; - } - } - return Changed; + // See if we can infer some no-wrap flags. + if (auto *BO = dyn_cast(NegX)) + processBinOp(BO, LVI); + + return true; } - IRBuilder<> B(II); - Value *NegX = B.CreateNeg(X, II->getName(), /*HasNUW=*/false, - /*HasNSW=*/IsIntMinPoison); - ++NumAbs; - II->replaceAllUsesWith(NegX); - II->eraseFromParent(); - - // See if we can infer some no-wrap flags. - if (auto *BO = dyn_cast(NegX)) - processBinOp(BO, LVI); - - return true; + // Argument's range crosses zero. + // Can we at least tell that the argument is never INT_MIN? + if (!IsIntMinPoison && !Range.contains(IntMin)) { + ++NumNSW; + ++NumSubNSW; + II->setArgOperand(1, ConstantInt::getTrue(II->getContext())); + return true; + } + return false; } // See if this min/max intrinsic always picks it's one specific operand. diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp index 4cfc0bacefbc..1ede4e7932af 100644 --- a/llvm/lib/Transforms/Scalar/GVN.cpp +++ b/llvm/lib/Transforms/Scalar/GVN.cpp @@ -946,9 +946,14 @@ static void replaceValuesPerBlockEntry( SmallVectorImpl &ValuesPerBlock, Value *OldValue, Value *NewValue) { for (AvailableValueInBlock &V : ValuesPerBlock) { - if ((V.AV.isSimpleValue() && V.AV.getSimpleValue() == OldValue) || - (V.AV.isCoercedLoadValue() && V.AV.getCoercedLoadValue() == OldValue)) - V = AvailableValueInBlock::get(V.BB, NewValue); + if (V.AV.Val == OldValue) + V.AV.Val = NewValue; + if (V.AV.isSelectValue()) { + if (V.AV.V1 == OldValue) + V.AV.V1 = NewValue; + if (V.AV.V2 == OldValue) + V.AV.V2 = NewValue; + } } } diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp index 91dc5018d232..a95d6adf36d6 100644 --- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp +++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp @@ -1603,6 +1603,7 @@ bool MemCpyOptPass::processByValArgument(CallBase &CB, unsigned ArgNo) { << " " << CB << "\n"); // Otherwise we're good! Update the byval argument. + combineAAMetadata(&CB, MDep); CB.setArgOperand(ArgNo, MDep->getSource()); ++NumMemCpyInstr; return true; diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 821a3fa22a85..9870ffbb586c 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -10118,7 +10118,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) { } if (E->State == TreeEntry::NeedToGather) { - if (E->getMainOp() && E->Idx == 0) + // Set insert point for non-reduction initial nodes. + if (E->getMainOp() && E->Idx == 0 && !UserIgnoreList) setInsertPointAfterBundle(E); Value *Vec = createBuildVector(E); E->VectorizedValue = Vec; diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp index e55798df610c..c63bd1c63bfd 100644 --- a/openmp/runtime/src/kmp_runtime.cpp +++ b/openmp/runtime/src/kmp_runtime.cpp @@ -8858,7 +8858,8 @@ __kmp_determine_reduction_method( #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS -#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_WINDOWS || KMP_OS_HURD +#if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ + KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_HURD // basic tuning diff --git a/openmp/runtime/src/kmp_wrapper_getpid.h b/openmp/runtime/src/kmp_wrapper_getpid.h index 32ede3ed715b..82b1fe767507 100644 --- a/openmp/runtime/src/kmp_wrapper_getpid.h +++ b/openmp/runtime/src/kmp_wrapper_getpid.h @@ -30,7 +30,7 @@ #include #define __kmp_gettid() _lwp_self() #elif KMP_OS_OPENBSD -#define __kmp_gettid() syscall(SYS_getthrid) +#define __kmp_gettid() getthrid() #elif defined(SYS_gettid) // Hopefully other Unix systems define SYS_gettid syscall for getting os thread // id