diff --git a/flang/include/flang/Lower/AbstractConverter.h b/flang/include/flang/Lower/AbstractConverter.h index 1d1323642bf9c..8ae68e143cd2f 100644 --- a/flang/include/flang/Lower/AbstractConverter.h +++ b/flang/include/flang/Lower/AbstractConverter.h @@ -348,6 +348,10 @@ class AbstractConverter { virtual Fortran::lower::SymbolBox lookupOneLevelUpSymbol(const Fortran::semantics::Symbol &sym) = 0; + /// Find the symbol in the inner-most level of the local map or return null. + virtual Fortran::lower::SymbolBox + shallowLookupSymbol(const Fortran::semantics::Symbol &sym) = 0; + /// Return the mlir::SymbolTable associated to the ModuleOp. /// Look-ups are faster using it than using module.lookup<>, /// but the module op should be queried in case of failure diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.h b/flang/include/flang/Optimizer/Dialect/FIROps.h index 1bed227afb50d..62ef8b4b502f2 100644 --- a/flang/include/flang/Optimizer/Dialect/FIROps.h +++ b/flang/include/flang/Optimizer/Dialect/FIROps.h @@ -147,6 +147,10 @@ class CoordinateIndicesAdaptor { mlir::ValueRange values; }; +struct LocalitySpecifierOperands { + llvm::SmallVector<::mlir::Value> privateVars; + llvm::SmallVector<::mlir::Attribute> privateSyms; +}; } // namespace fir #endif // FORTRAN_OPTIMIZER_DIALECT_FIROPS_H diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td index dc66885f776f0..f4b17ef7eed09 100644 --- a/flang/include/flang/Optimizer/Dialect/FIROps.td +++ b/flang/include/flang/Optimizer/Dialect/FIROps.td @@ -3605,6 +3605,21 @@ def fir_LocalitySpecifierOp : fir_Op<"local", [IsolatedFromAbove]> { ]; let extraClassDeclaration = [{ + mlir::BlockArgument getInitMoldArg() { + auto ®ion = getInitRegion(); + return region.empty() ? nullptr : region.getArgument(0); + } + mlir::BlockArgument getInitPrivateArg() { + auto ®ion = getInitRegion(); + return region.empty() ? nullptr : region.getArgument(1); + } + + /// Returns true if the init region might read from the mold argument + bool initReadsFromMold() { + mlir::BlockArgument moldArg = getInitMoldArg(); + return moldArg && !moldArg.use_empty(); + } + /// Get the type for arguments to nested regions. This should /// generally be either the same as getType() or some pointer /// type (pointing to the type allocated by this op). diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp index c9e91cf3e8042..49675d34215a9 100644 --- a/flang/lib/Lower/Bridge.cpp +++ b/flang/lib/Lower/Bridge.cpp @@ -12,6 +12,8 @@ #include "flang/Lower/Bridge.h" +#include "OpenMP/DataSharingProcessor.h" +#include "OpenMP/Utils.h" #include "flang/Lower/Allocatable.h" #include "flang/Lower/CallInterface.h" #include "flang/Lower/Coarray.h" @@ -1142,6 +1144,14 @@ class FirConverter : public Fortran::lower::AbstractConverter { return name; } + /// Find the symbol in the inner-most level of the local map or return null. + Fortran::lower::SymbolBox + shallowLookupSymbol(const Fortran::semantics::Symbol &sym) override { + if (Fortran::lower::SymbolBox v = localSymbols.shallowLookupSymbol(sym)) + return v; + return {}; + } + private: FirConverter() = delete; FirConverter(const FirConverter &) = delete; @@ -1216,14 +1226,6 @@ class FirConverter : public Fortran::lower::AbstractConverter { return {}; } - /// Find the symbol in the inner-most level of the local map or return null. - Fortran::lower::SymbolBox - shallowLookupSymbol(const Fortran::semantics::Symbol &sym) { - if (Fortran::lower::SymbolBox v = localSymbols.shallowLookupSymbol(sym)) - return v; - return {}; - } - /// Find the symbol in one level up of symbol map such as for host-association /// in OpenMP code or return null. Fortran::lower::SymbolBox @@ -2027,9 +2029,34 @@ class FirConverter : public Fortran::lower::AbstractConverter { void handleLocalitySpecs(const IncrementLoopInfo &info) { Fortran::semantics::SemanticsContext &semanticsContext = bridge.getSemanticsContext(); - for (const Fortran::semantics::Symbol *sym : info.localSymList) + // TODO Extract `DataSharingProcessor` from omp to a more general location. + Fortran::lower::omp::DataSharingProcessor dsp( + *this, semanticsContext, getEval(), + /*useDelayedPrivatization=*/true, localSymbols); + fir::LocalitySpecifierOperands privateClauseOps; + auto doConcurrentLoopOp = + mlir::dyn_cast_if_present(info.loopOp); + // TODO Promote to using `enableDelayedPrivatization` (which is enabled by + // default unlike the staging flag) once the implementation of this is more + // complete. + bool useDelayedPriv = + enableDelayedPrivatizationStaging && doConcurrentLoopOp; + + for (const Fortran::semantics::Symbol *sym : info.localSymList) { + if (useDelayedPriv) { + dsp.privatizeSymbol(sym, &privateClauseOps); + continue; + } + createHostAssociateVarClone(*sym, /*skipDefaultInit=*/false); + } + for (const Fortran::semantics::Symbol *sym : info.localInitSymList) { + if (useDelayedPriv) { + dsp.privatizeSymbol(sym, &privateClauseOps); + continue; + } + createHostAssociateVarClone(*sym, /*skipDefaultInit=*/true); const auto *hostDetails = sym->detailsIf(); @@ -2048,6 +2075,24 @@ class FirConverter : public Fortran::lower::AbstractConverter { sym->detailsIf(); copySymbolBinding(hostDetails->symbol(), *sym); } + + if (useDelayedPriv) { + doConcurrentLoopOp.getLocalVarsMutable().assign( + privateClauseOps.privateVars); + doConcurrentLoopOp.setLocalSymsAttr( + builder->getArrayAttr(privateClauseOps.privateSyms)); + + for (auto [sym, privateVar] : llvm::zip_equal( + dsp.getAllSymbolsToPrivatize(), privateClauseOps.privateVars)) { + auto arg = doConcurrentLoopOp.getRegion().begin()->addArgument( + privateVar.getType(), doConcurrentLoopOp.getLoc()); + bindSymbol(*sym, hlfir::translateToExtendedValue( + privateVar.getLoc(), *builder, hlfir::Entity{arg}, + /*contiguousHint=*/true) + .first); + } + } + // Note that allocatable, types with ultimate components, and type // requiring finalization are forbidden in LOCAL/LOCAL_INIT (F2023 C1130), // so no clean-up needs to be generated for these entities. diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp index 20dc46e4710fb..629478294ef5b 100644 --- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp +++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp @@ -20,6 +20,7 @@ #include "flang/Optimizer/Builder/BoxValue.h" #include "flang/Optimizer/Builder/HLFIRTools.h" #include "flang/Optimizer/Builder/Todo.h" +#include "flang/Optimizer/Dialect/FIROps.h" #include "flang/Optimizer/HLFIR/HLFIRDialect.h" #include "flang/Optimizer/HLFIR/HLFIROps.h" #include "flang/Semantics/attr.h" @@ -53,6 +54,15 @@ DataSharingProcessor::DataSharingProcessor( }); } +DataSharingProcessor::DataSharingProcessor(lower::AbstractConverter &converter, + semantics::SemanticsContext &semaCtx, + lower::pft::Evaluation &eval, + bool useDelayedPrivatization, + lower::SymMap &symTable) + : DataSharingProcessor(converter, semaCtx, {}, eval, + /*shouldCollectPreDeterminedSymols=*/false, + useDelayedPrivatization, symTable) {} + void DataSharingProcessor::processStep1( mlir::omp::PrivateClauseOps *clauseOps) { collectSymbolsForPrivatization(); @@ -174,7 +184,8 @@ void DataSharingProcessor::cloneSymbol(const semantics::Symbol *sym) { void DataSharingProcessor::copyFirstPrivateSymbol( const semantics::Symbol *sym, mlir::OpBuilder::InsertPoint *copyAssignIP) { - if (sym->test(semantics::Symbol::Flag::OmpFirstPrivate)) + if (sym->test(semantics::Symbol::Flag::OmpFirstPrivate) || + sym->test(semantics::Symbol::Flag::LocalityLocalInit)) converter.copyHostAssociateVar(*sym, copyAssignIP); } @@ -497,9 +508,9 @@ void DataSharingProcessor::privatize(mlir::omp::PrivateClauseOps *clauseOps) { if (const auto *commonDet = sym->detailsIf()) { for (const auto &mem : commonDet->objects()) - doPrivatize(&*mem, clauseOps); + privatizeSymbol(&*mem, clauseOps); } else - doPrivatize(sym, clauseOps); + privatizeSymbol(sym, clauseOps); } } @@ -516,22 +527,30 @@ void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) { } } -void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym, - mlir::omp::PrivateClauseOps *clauseOps) { +template +void DataSharingProcessor::privatizeSymbol( + const semantics::Symbol *symToPrivatize, OperandsStructType *clauseOps) { if (!useDelayedPrivatization) { - cloneSymbol(sym); - copyFirstPrivateSymbol(sym); + cloneSymbol(symToPrivatize); + copyFirstPrivateSymbol(symToPrivatize); return; } - lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym); + const semantics::Symbol *sym = symToPrivatize->HasLocalLocality() + ? &symToPrivatize->GetUltimate() + : symToPrivatize; + lower::SymbolBox hsb = symToPrivatize->HasLocalLocality() + ? converter.shallowLookupSymbol(*sym) + : converter.lookupOneLevelUpSymbol(*sym); assert(hsb && "Host symbol box not found"); hlfir::Entity entity{hsb.getAddr()}; bool cannotHaveNonDefaultLowerBounds = !entity.mayHaveNonDefaultLowerBounds(); mlir::Location symLoc = hsb.getAddr().getLoc(); std::string privatizerName = sym->name().ToString() + ".privatizer"; - bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate); + bool isFirstPrivate = + symToPrivatize->test(semantics::Symbol::Flag::OmpFirstPrivate) || + symToPrivatize->test(semantics::Symbol::Flag::LocalityLocalInit); mlir::Value privVal = hsb.getAddr(); mlir::Type allocType = privVal.getType(); @@ -565,7 +584,7 @@ void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym, mlir::Type argType = privVal.getType(); - mlir::omp::PrivateClauseOp privatizerOp = [&]() { + OpType privatizerOp = [&]() { auto moduleOp = firOpBuilder.getModule(); auto uniquePrivatizerName = fir::getTypeAsString( allocType, converter.getKindMap(), @@ -573,16 +592,25 @@ void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym, (isFirstPrivate ? "_firstprivate" : "_private")); if (auto existingPrivatizer = - moduleOp.lookupSymbol( - uniquePrivatizerName)) + moduleOp.lookupSymbol(uniquePrivatizerName)) return existingPrivatizer; mlir::OpBuilder::InsertionGuard guard(firOpBuilder); firOpBuilder.setInsertionPointToStart(moduleOp.getBody()); - auto result = firOpBuilder.create( - symLoc, uniquePrivatizerName, allocType, - isFirstPrivate ? mlir::omp::DataSharingClauseType::FirstPrivate - : mlir::omp::DataSharingClauseType::Private); + OpType result; + + if constexpr (std::is_same_v) { + result = firOpBuilder.create( + symLoc, uniquePrivatizerName, allocType, + isFirstPrivate ? mlir::omp::DataSharingClauseType::FirstPrivate + : mlir::omp::DataSharingClauseType::Private); + } else { + result = firOpBuilder.create( + symLoc, uniquePrivatizerName, allocType, + isFirstPrivate ? fir::LocalitySpecifierType::LocalInit + : fir::LocalitySpecifierType::Local); + } + fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym); lower::SymMapScope outerScope(symTable); @@ -625,27 +653,36 @@ void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym, ©Region, /*insertPt=*/{}, {argType, argType}, {symLoc, symLoc}); firOpBuilder.setInsertionPointToEnd(copyEntryBlock); - auto addSymbol = [&](unsigned argIdx, bool force = false) { + auto addSymbol = [&](unsigned argIdx, const semantics::Symbol *symToMap, + bool force = false) { symExV.match( [&](const fir::MutableBoxValue &box) { symTable.addSymbol( - *sym, fir::substBase(box, copyRegion.getArgument(argIdx)), - force); + *symToMap, + fir::substBase(box, copyRegion.getArgument(argIdx)), force); }, [&](const auto &box) { - symTable.addSymbol(*sym, copyRegion.getArgument(argIdx), force); + symTable.addSymbol(*symToMap, copyRegion.getArgument(argIdx), + force); }); }; - addSymbol(0, true); + addSymbol(0, sym, true); lower::SymMapScope innerScope(symTable); - addSymbol(1); + addSymbol(1, symToPrivatize); auto ip = firOpBuilder.saveInsertionPoint(); - copyFirstPrivateSymbol(sym, &ip); - - firOpBuilder.create( - hsb.getAddr().getLoc(), symTable.shallowLookupSymbol(*sym).getAddr()); + copyFirstPrivateSymbol(symToPrivatize, &ip); + + if constexpr (std::is_same_v) { + firOpBuilder.create( + hsb.getAddr().getLoc(), + symTable.shallowLookupSymbol(*symToPrivatize).getAddr()); + } else { + firOpBuilder.create( + hsb.getAddr().getLoc(), + symTable.shallowLookupSymbol(*symToPrivatize).getAddr()); + } } return result; @@ -656,9 +693,22 @@ void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym, clauseOps->privateVars.push_back(privVal); } - symToPrivatizer[sym] = privatizerOp; + if (symToPrivatize->HasLocalLocality()) + allPrivatizedSymbols.insert(symToPrivatize); } +template void +DataSharingProcessor::privatizeSymbol( + const semantics::Symbol *symToPrivatize, + mlir::omp::PrivateClauseOps *clauseOps); + +template void +DataSharingProcessor::privatizeSymbol( + const semantics::Symbol *symToPrivatize, + fir::LocalitySpecifierOperands *clauseOps); + } // namespace omp } // namespace lower } // namespace Fortran diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h index 7787e4ffb03c2..ae759bfef566b 100644 --- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h +++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h @@ -77,8 +77,6 @@ class DataSharingProcessor { llvm::SetVector preDeterminedSymbols; llvm::SetVector allPrivatizedSymbols; - llvm::DenseMap - symToPrivatizer; lower::AbstractConverter &converter; semantics::SemanticsContext &semaCtx; fir::FirOpBuilder &firOpBuilder; @@ -105,8 +103,6 @@ class DataSharingProcessor { void collectImplicitSymbols(); void collectPreDeterminedSymbols(); void privatize(mlir::omp::PrivateClauseOps *clauseOps); - void doPrivatize(const semantics::Symbol *sym, - mlir::omp::PrivateClauseOps *clauseOps); void copyLastPrivatize(mlir::Operation *op); void insertLastPrivateCompare(mlir::Operation *op); void cloneSymbol(const semantics::Symbol *sym); @@ -125,6 +121,11 @@ class DataSharingProcessor { bool shouldCollectPreDeterminedSymbols, bool useDelayedPrivatization, lower::SymMap &symTable); + DataSharingProcessor(lower::AbstractConverter &converter, + semantics::SemanticsContext &semaCtx, + lower::pft::Evaluation &eval, + bool useDelayedPrivatization, lower::SymMap &symTable); + // Privatisation is split into two steps. // Step1 performs cloning of all privatisation clauses and copying for // firstprivates. Step1 is performed at the place where process/processStep1 @@ -151,6 +152,11 @@ class DataSharingProcessor { ? allPrivatizedSymbols.getArrayRef() : llvm::ArrayRef(); } + + template + void privatizeSymbol(const semantics::Symbol *symToPrivatize, + OperandsStructType *clauseOps); }; } // namespace omp diff --git a/flang/test/Lower/do_concurrent_delayed_locality.f90 b/flang/test/Lower/do_concurrent_delayed_locality.f90 new file mode 100644 index 0000000000000..9b234087ed4be --- /dev/null +++ b/flang/test/Lower/do_concurrent_delayed_locality.f90 @@ -0,0 +1,49 @@ +! RUN: %flang_fc1 -emit-hlfir -mmlir --openmp-enable-delayed-privatization-staging=true -o - %s | FileCheck %s + +subroutine do_concurrent_with_locality_specs + implicit none + integer :: i, local_var, local_init_var + + do concurrent (i=1:10) local(local_var) local_init(local_init_var) + if (i < 5) then + local_var = 42 + else + local_init_var = 84 + end if + end do +end subroutine + +! CHECK: fir.local {type = local_init} @[[LOCAL_INIT_SYM:.*]] : i32 copy { +! CHECK: ^bb0(%[[ORIG_VAL:.*]]: !fir.ref, %[[LOCAL_VAL:.*]]: !fir.ref): +! CHECK: %[[ORIG_VAL_LD:.*]] = fir.load %[[ORIG_VAL]] : !fir.ref +! CHECK: hlfir.assign %[[ORIG_VAL_LD]] to %[[LOCAL_VAL]] : i32, !fir.ref +! CHECK: fir.yield(%[[LOCAL_VAL]] : !fir.ref) +! CHECK: } + +! CHECK: fir.local {type = local} @[[LOCAL_SYM:.*]] : i32 + +! CHECK-LABEL: func.func @_QPdo_concurrent_with_locality_specs() { +! CHECK: %[[ORIG_LOCAL_INIT_ALLOC:.*]] = fir.alloca i32 {bindc_name = "local_init_var", {{.*}}} +! CHECK: %[[ORIG_LOCAL_INIT_DECL:.*]]:2 = hlfir.declare %[[ORIG_LOCAL_INIT_ALLOC]] + +! CHECK: %[[ORIG_LOCAL_ALLOC:.*]] = fir.alloca i32 {bindc_name = "local_var", {{.*}}} +! CHECK: %[[ORIG_LOCAL_DECL:.*]]:2 = hlfir.declare %[[ORIG_LOCAL_ALLOC]] + +! CHECK: fir.do_concurrent { +! CHECK: %[[IV_DECL:.*]]:2 = hlfir.declare %{{.*}} + +! CHECK: fir.do_concurrent.loop (%{{.*}}) = (%{{.*}}) to (%{{.*}}) step (%{{.*}}) local(@[[LOCAL_SYM]] %[[ORIG_LOCAL_DECL]]#0 -> %[[LOCAL_ARG:.*]], @[[LOCAL_INIT_SYM]] %[[ORIG_LOCAL_INIT_DECL]]#0 -> %[[LOCAL_INIT_ARG:.*]] : !fir.ref, !fir.ref) { +! CHECK: %[[LOCAL_DECL:.*]]:2 = hlfir.declare %[[LOCAL_ARG]] +! CHECK: %[[LOCAL_INIT_DECL:.*]]:2 = hlfir.declare %[[LOCAL_INIT_ARG]] + +! CHECK: fir.if %{{.*}} { +! CHECK: %[[C42:.*]] = arith.constant 42 : i32 +! CHECK: hlfir.assign %[[C42]] to %[[LOCAL_DECL]]#0 : i32, !fir.ref +! CHECK: } else { +! CHECK: %[[C84:.*]] = arith.constant 84 : i32 +! CHECK: hlfir.assign %[[C84]] to %[[LOCAL_INIT_DECL]]#0 : i32, !fir.ref +! CHECK: } +! CHECK: } +! CHECK: } +! CHECK: return +! CHECK: }