From 69d6252069c0a22edaa13e4ceef27c69bceb20c6 Mon Sep 17 00:00:00 2001 From: Felix Schlepper Date: Wed, 12 Aug 2026 15:07:36 +0200 Subject: [PATCH 1/2] ITS: slab allocator idea Signed-off-by: Felix Schlepper --- Detectors/ITSMFT/ITS/tracking/CMakeLists.txt | 1 + .../include/ITStracking/CapacityEstimator.h | 110 +++ .../include/ITStracking/SlabBumpAllocator.h | 397 +++++++++++ .../tracking/include/ITStracking/TimeFrame.h | 13 +- .../include/ITStracking/TrackerTraits.h | 22 +- .../include/ITStracking/TrackingTopology.h | 29 +- .../tracking/include/ITStracking/Vertexer.h | 2 + .../ITS/tracking/src/CapacityEstimator.cxx | 141 ++++ .../ITSMFT/ITS/tracking/src/TimeFrame.cxx | 6 + Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx | 1 + .../ITSMFT/ITS/tracking/src/TrackerTraits.cxx | 655 ++++++++++-------- .../ITS/tracking/src/TrackingInterface.cxx | 1 + .../ITSMFT/ITS/tracking/src/Vertexer.cxx | 3 +- .../ITSMFT/ITS/tracking/test/CMakeLists.txt | 6 + .../tracking/test/testSlabBumpAllocator.cxx | 526 ++++++++++++++ .../tracking/test/testTrackingTopology.cxx | 59 ++ 16 files changed, 1670 insertions(+), 302 deletions(-) create mode 100644 Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h create mode 100644 Detectors/ITSMFT/ITS/tracking/include/ITStracking/SlabBumpAllocator.h create mode 100644 Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx create mode 100644 Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx diff --git a/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt b/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt index 1dd64b6f1874b..17420d47a2732 100644 --- a/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt +++ b/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt @@ -13,6 +13,7 @@ o2_add_library(ITStracking TARGETVARNAME targetName SOURCES src/ClusterLines.cxx src/Cluster.cxx + src/CapacityEstimator.cxx src/Configuration.cxx src/FastMultEstConfig.cxx src/FastMultEst.cxx diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h new file mode 100644 index 0000000000000..aa37de186b910 --- /dev/null +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h @@ -0,0 +1,110 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +/// +/// \file CapacityEstimator.h +/// \brief Cross-timeframe output-size prediction. +/// + +#ifndef TRACKINGITSU_INCLUDE_CAPACITYESTIMATOR_H_ +#define TRACKINGITSU_INCLUDE_CAPACITYESTIMATOR_H_ + +#include +#include +#include + +namespace o2::its +{ + +enum SlabSite : uint8_t { + Tracklets = 0, + Cells, + Neighbours, + Roads, + NSlabSite, +}; +constexpr const char* const SlabSiteNames[SlabSite::NSlabSite]{"Tracklets", "Cells", "Neighbours", "Roads"}; + +class CapacityEstimator +{ + public: + struct Config { + float alpha{0.2f}; + float marginInit{1.30f}; + float marginMin{1.10f}; + float marginMax{4.00f}; + float marginUp{1.50f}; + float marginOverflowSlack{1.05f}; + float marginDown{0.98f}; + float lowWatermark{0.60f}; + uint32_t decayAfter{2}; + size_t floorSlots{1024}; + }; + + using KeyType = uint64_t; + + struct Decoded { + SlabSite site; + int iteration; + int variant; + int slot; + }; + + static constexpr KeyType makeKey(SlabSite site, int iteration, int variant, int slot) noexcept + { + return (static_cast(site) << 56) | + (static_cast(iteration & 0xFF) << 48) | + (static_cast(variant & 0xFFFF) << 32) | + static_cast(static_cast(slot)); + } + + static constexpr Decoded decodeKey(KeyType key) noexcept + { + return { + .site = static_cast((key >> 56) & 0xFF), + .iteration = static_cast((key >> 48) & 0xFF), + .variant = static_cast((key >> 32) & 0xFFFF), + .slot = static_cast(static_cast(key & 0xFFFFFFFF))}; + } + + static constexpr int makeVariant(int high, int low) noexcept + { + return ((high & 0xFF) << 8) | (low & 0xFF); + } + + static constexpr int getVariantHigh(int variant) noexcept + { + return (variant >> 8) & 0xFF; + } + + static constexpr int getVariantLow(int variant) noexcept + { + return variant & 0xFF; + } + + CapacityEstimator(); + explicit CapacityEstimator(Config cfg); + ~CapacityEstimator(); + CapacityEstimator(const CapacityEstimator&) = delete; + CapacityEstimator& operator=(const CapacityEstimator&) = delete; + + void reset(); + size_t capacity(uint64_t key, double scale) const; + void update(uint64_t key, double scale, size_t emitted, size_t capacityUsed, bool overflowed, bool memoryLimited); + void print() const; + + private: + struct Impl; + std::unique_ptr mImpl; +}; + +} // namespace o2::its + +#endif /* TRACKINGITSU_INCLUDE_CAPACITYESTIMATOR_H_ */ diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/SlabBumpAllocator.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/SlabBumpAllocator.h new file mode 100644 index 0000000000000..e32516ea1e0e0 --- /dev/null +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/SlabBumpAllocator.h @@ -0,0 +1,397 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +/// +/// \file SlabBumpAllocator.h +/// \brief Lock-free slot allocator and single-pass sink. +/// + +#ifndef TRACKINGITSU_INCLUDE_SLABBUMPALLOCATOR_H_ +#define TRACKINGITSU_INCLUDE_SLABBUMPALLOCATOR_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "ITStracking/BoundedAllocator.h" + +namespace o2::its +{ + +class SlabBumpAllocator +{ + public: + struct Range { + size_t base{0}; + size_t n{0}; + bool valid() const noexcept { return n != 0; } + }; + + SlabBumpAllocator(size_t capacity, size_t slab) noexcept + : mCapacity{capacity}, mSlab{slab ? slab : size_t{1}} {} + + Range grab() noexcept + { + if (mExhausted.load(std::memory_order_relaxed)) { + return {}; + } + const size_t base = mCursor.fetch_add(mSlab, std::memory_order_relaxed); + if (base >= mCapacity) { + mExhausted.store(true, std::memory_order_relaxed); + return {}; + } + return {.base = base, .n = std::min(mSlab, mCapacity - base)}; + } + + [[nodiscard]] size_t capacity() const noexcept { return mCapacity; } + [[nodiscard]] size_t slab() const noexcept { return mSlab; } + [[nodiscard]] size_t watermark() const noexcept + { + return std::min(mCursor.load(std::memory_order_relaxed), mCapacity); + } + + static size_t suggestSlab(size_t capacity, int nThreads, size_t minSlab = 256, size_t maxSlab = 4096) noexcept + { + const size_t t = static_cast(std::max(1, nThreads)); + const size_t fairShare = std::max(1, capacity / t); + return std::clamp(std::max(1, capacity / (8 * t)), + std::min(minSlab, fairShare), + std::min(maxSlab, fairShare)); + } + + void resetCapacity(size_t capacity) noexcept + { + assert(mCursor.load(std::memory_order_relaxed) == 0); + mCapacity = capacity; + mExhausted.store(capacity == 0, std::memory_order_relaxed); + } + + private: + std::atomic mCursor{0}; + std::atomic mExhausted{false}; + size_t mCapacity; + size_t mSlab; +}; + +enum class SlabMode : uint8_t { + Unordered, + GroupedByProducer +}; + +struct SlabSinkStats { + size_t requested{0}; ///< slots the caller predicted it would need + size_t capacity{0}; ///< slots the memory pool actually granted + size_t emitted{0}; + size_t spilled{0}; + bool overflowed{false}; ///< something did not fit into the staging area + bool memoryLimited{false}; ///< the pool granted less than was requested +}; + +template +class SlabSink +{ + static constexpr int32_t NoProducer = -1; + + public: + struct Config { + size_t capacity{0}; ///< predicted number of slots + int nThreads{1}; ///< workers that will feed this sink + int nConcurrentSinks{1}; ///< sinks that may be alive on the same pool at the same time + size_t slabOverride{0}; ///< 0: derive the slab size from the granted capacity + }; + + static constexpr size_t BytesPerSlot = Mode == SlabMode::GroupedByProducer ? (2 * sizeof(T)) + sizeof(int32_t) : sizeof(T); + + struct Run { + size_t begin{0}; + size_t end{0}; + }; + + class Handle + { + public: + explicit Handle(SlabSink* sink) + : mSink{sink}, mRuns{sink->memoryResource()}, mSpill{sink->memoryResource()}, mSpillProducer{sink->memoryResource()} {} + + void beginProducer(int32_t p) noexcept { mProducer = p; } + + template + void emplace(Args&&... args) + { + if constexpr (Mode == SlabMode::GroupedByProducer) { + assert(mProducer != NoProducer); + } + ++mEmitted; + if (mSlot == mSlotEnd && !refill()) { + mSpill.emplace_back(std::forward(args)...); + if constexpr (Mode == SlabMode::GroupedByProducer) { + mSpillProducer.push_back(mProducer); + } + return; + } + mSink->store(mSlot++, mProducer, std::forward(args)...); + } + + [[nodiscard]] size_t emitted() const noexcept { return mEmitted; } + [[nodiscard]] size_t spilled() const noexcept { return mSpill.size(); } + + private: + friend class SlabSink; + + bool refill() + { + if (mDrained) { // the arena is gone, do not touch the shared cursor again + return false; + } + closeRun(); + const auto r = mSink->mAlloc.grab(); + if (!r.valid()) { + mDrained = true; + return false; + } + mRunBegin = r.base; + mSlot = r.base; + mSlotEnd = r.base + r.n; + return true; + } + + void closeRun() + { + if constexpr (Mode == SlabMode::Unordered) { + if (mSlot > mRunBegin) { + mRuns.push_back(Run{.begin = mRunBegin, .end = mSlot}); + mRunBegin = mSlot; // only advanced once push_back succeeded, so a throw can be retried + } + } + } + + SlabSink* mSink{nullptr}; + size_t mSlot{0}; + size_t mSlotEnd{0}; + size_t mRunBegin{0}; + int32_t mProducer{NoProducer}; + bool mDrained{false}; + size_t mEmitted{0}; + bounded_vector mRuns; + bounded_vector mSpill; + bounded_vector mSpillProducer; + }; + + SlabSink(const Config& cfg, std::pmr::memory_resource* mr) + : SlabSink{cfg, grantedCapacity(cfg.capacity, cfg.nConcurrentSinks, mr), mr} {} + + SlabSink(SlabSink&&) = delete; + SlabSink(const SlabSink&) = delete; + SlabSink& operator=(SlabSink&&) = delete; + SlabSink& operator=(const SlabSink&) = delete; + ~SlabSink() = default; + + Handle& local() { return mHandles.local(); } + + [[nodiscard]] std::pmr::memory_resource* memoryResource() const noexcept { return mMR; } + + [[nodiscard]] SlabSinkStats stats() const + { + SlabSinkStats s; + s.requested = mRequested; + s.capacity = mAlloc.capacity(); + s.memoryLimited = s.capacity < s.requested; + for (const auto& h : mHandles) { + s.emitted += h.emitted(); + s.spilled += h.spilled(); + } + s.overflowed = s.spilled != 0; + return s; + } + + void finalizeUnordered(bounded_vector& dest) + { + static_assert(Mode == SlabMode::Unordered); + assert(!mFinalized); + assert(dest.get_allocator().resource()->is_equal(*mMR)); + mFinalized = true; + + bounded_vector runs{mMR}; + size_t nRuns{0}; + for (auto& h : mHandles) { + h.closeRun(); + nRuns += h.mRuns.size(); + } + runs.reserve(nRuns); + for (const auto& h : mHandles) { + runs.insert(runs.end(), h.mRuns.begin(), h.mRuns.end()); + } + std::sort(runs.begin(), runs.end(), [](const Run& a, const Run& b) { return a.begin < b.begin; }); + + // Runs are disjoint and now ordered, so the compaction target never runs ahead of the source. + size_t outputSize{0}; + for (const auto& run : runs) { + for (size_t slot{run.begin}; slot < run.end; ++slot) { + if (outputSize != slot) { + mStaging[outputSize] = std::move(mStaging[slot]); + } + ++outputSize; + } + } + deepVectorClear(runs, mMR); + mStaging.resize(outputSize); + dest.swap(mStaging); + + for (auto& h : mHandles) { + dest.insert(dest.end(), std::make_move_iterator(h.mSpill.begin()), std::make_move_iterator(h.mSpill.end())); + deepVectorClear(h.mSpill, mMR); + } + shrinkIfWasteful(dest); + deepVectorClear(mStaging, mMR); + } + + void finalizeGrouped(size_t nProducers, bounded_vector& lut, bounded_vector& dest) + { + static_assert(Mode == SlabMode::GroupedByProducer); + assert(!mFinalized); + mFinalized = true; + const size_t wm = mAlloc.watermark(); + + lut.assign(nProducers + 1, 0); + + for (size_t s = 0; s < wm; ++s) { + const int32_t p = mProducerOf[s]; + if (p != NoProducer) { + ++lut[p + 1]; + } + } + for (const auto& h : mHandles) { + for (const int32_t p : h.mSpillProducer) { + ++lut[p + 1]; + } + } + std::inclusive_scan(lut.begin(), lut.end(), lut.begin()); + + bounded_vector cursor(lut.begin(), lut.begin() + static_cast(nProducers), mMR); + for (size_t s = 0; s < wm; ++s) { + const int32_t p = mProducerOf[s]; + mProducerOf[s] = (p != NoProducer) ? cursor[p]++ : -1; + } + + const auto total = static_cast(lut.back()); + dest.resize(total); + for (auto& h : mHandles) { + for (size_t i = 0; i < h.mSpill.size(); ++i) { + dest[cursor[h.mSpillProducer[i]]++] = std::move(h.mSpill[i]); + } + deepVectorClear(h.mSpill, mMR); + deepVectorClear(h.mSpillProducer, mMR); + } + deepVectorClear(cursor, mMR); + + T* const staging = mStaging.data(); + tbb::parallel_for(tbb::blocked_range(0, wm, 4096), [&](const tbb::blocked_range& r) { + for (size_t s = r.begin(); s != r.end(); ++s) { + const int d = mProducerOf[s]; + if (d < 0) { + continue; + } + dest[d] = std::move(staging[s]); + } + }); + + deepVectorClear(mStaging, mMR); + deepVectorClear(mProducerOf, mMR); + } + + private: + SlabSink(const Config& cfg, size_t granted, std::pmr::memory_resource* mr) + : mMR{mr}, + mRequested{cfg.capacity}, + mAlloc{granted, cfg.slabOverride ? cfg.slabOverride : SlabBumpAllocator::suggestSlab(granted, cfg.nThreads)}, + mStaging{mr}, + mProducerOf{mr}, + mHandles{[this]() { return Handle{this}; }} + { + try { + mStaging.resize(granted); + if constexpr (Mode == SlabMode::GroupedByProducer) { + mProducerOf.assign(granted, NoProducer); + } + } catch (const std::bad_alloc&) { + discardPreallocation(); + } catch (const std::length_error&) { + discardPreallocation(); + } + } + + static size_t grantedCapacity(size_t requested, int nConcurrentSinks, const std::pmr::memory_resource* mr) noexcept + { + const auto* bounded = dynamic_cast(mr); + if (bounded == nullptr) { + return requested; + } + const size_t used = bounded->getUsedMemory(); + const size_t limit = bounded->getMaxMemory(); + const size_t remaining = used < limit ? limit - used : 0; + // Keep half of what is left for the spill vectors and whatever else is still live, then + // split the rest between the sinks that may be running on this pool at the same time. + const size_t budget = (remaining / 2) / static_cast(std::max(1, nConcurrentSinks)); + return std::min(requested, budget / BytesPerSlot); + } + + static void shrinkIfWasteful(bounded_vector& v) + { + if (v.capacity() > v.size() + (v.size() / 4)) { + v.shrink_to_fit(); + } + } + + void discardPreallocation() + { + // Capacity prediction is only an optimization; spilling preserves the output. + deepVectorClear(mStaging, mMR); + deepVectorClear(mProducerOf, mMR); + mAlloc.resetCapacity(0); + } + + template + void store(size_t slot, [[maybe_unused]] int32_t producer, Args&&... args) + { + mStaging[slot] = T(std::forward(args)...); + if constexpr (Mode == SlabMode::GroupedByProducer) { + mProducerOf[slot] = producer; + } + } + + std::pmr::memory_resource* mMR{nullptr}; + size_t mRequested{0}; + SlabBumpAllocator mAlloc; + bounded_vector mStaging; + bounded_vector mProducerOf; + tbb::enumerable_thread_specific mHandles; + bool mFinalized{false}; +}; + +template +using UnorderedSlabSink = SlabSink; + +template +using GroupedSlabSink = SlabSink; + +} // namespace o2::its + +#endif /* TRACKINGITSU_INCLUDE_SLABBUMPALLOCATOR_H_ */ diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h index ae466a32bfc89..11246fa0ee3b0 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h @@ -23,6 +23,7 @@ #include "DataFormatsITS/TrackITS.h" #include "DataFormatsITS/Vertex.h" +#include "ITStracking/CapacityEstimator.h" #include "ITStracking/Cell.h" #include "ITStracking/Cluster.h" #include "ITStracking/Configuration.h" @@ -55,6 +56,7 @@ class ROFRecord; namespace its { + namespace gpu { template @@ -71,8 +73,10 @@ struct TimeFrame { using TrackSeedN = TrackSeed; friend class gpu::TimeFrameGPU; - TimeFrame() = default; - virtual ~TimeFrame() = default; + TimeFrame(); + virtual ~TimeFrame(); + TimeFrame(const TimeFrame&) = delete; + TimeFrame& operator=(const TimeFrame&) = delete; const Vertex& getPrimaryVertex(const int ivtx) const { return mPrimaryVertices[ivtx]; } auto& getPrimaryVertices() { return mPrimaryVertices; }; @@ -227,6 +231,9 @@ struct TimeFrame { /// staggering void setIsStaggered(bool b) noexcept { mIsStaggered = b; } + CapacityEstimator& getCapacityEstimator() noexcept { return mCapacityEstimator; } + const CapacityEstimator& getCapacityEstimator() const noexcept { return mCapacityEstimator; } + // Vertexer void computeTrackletsPerROFScans(); void computeTracletsPerClusterScans(); @@ -318,6 +325,8 @@ struct TimeFrame { std::vector> mCellsNeighboursLUT; bounded_vector mBogusClusters; /// keep track of clusters with wild coordinates + CapacityEstimator mCapacityEstimator; + // Vertexer bounded_vector mPrimaryVertices; bounded_vector mPrimaryVerticesLabels; diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h index 4d6378aded0e8..276e06dc94e6f 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h @@ -17,15 +17,18 @@ #define TRACKINGITSU_INCLUDE_TRACKERTRAITS_H_ #include +#include #include #include "DetectorsBase/Propagator.h" #include "ITStracking/Configuration.h" #include "ITStracking/IndexTableUtils.h" +#include "ITStracking/CapacityEstimator.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/Cell.h" #include "ITStracking/BoundedAllocator.h" #include "ITStracking/TrackExtensionHypothesis.h" +#include "ITStracking/TrackFollower.h" #include "ITStracking/TrackITSInternal.h" // #define OPTIMISATION_OUTPUT @@ -40,12 +43,24 @@ namespace its { class TrackITSExt; +template +struct RoadSeed { + TrackSeed seed; + int cellId{constants::UnusedIndex}; + int cellTopologyId{constants::UnusedIndex}; + + RoadSeed() = default; + RoadSeed(TrackSeed&& inputSeed, int inputCellId, int inputCellTopologyId) + : seed{std::move(inputSeed)}, cellId{inputCellId}, cellTopologyId{inputCellTopologyId} {} +}; + template class TrackerTraits { public: using IndexTableUtilsN = IndexTableUtils; using TrackSeedN = TrackSeed; + using RoadSeedN = RoadSeed; virtual ~TrackerTraits() = default; virtual void adoptTimeFrame(TimeFrame* tf) { mTimeFrame = tf; } @@ -57,7 +72,7 @@ class TrackerTraits virtual void findRoads(const int iteration); template - void processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, const bounded_vector& currentCellSeed, const bounded_vector& currentCellId, const bounded_vector& currentCellTopologyId, bounded_vector& updatedCellSeed, bounded_vector& updatedCellId, bounded_vector& updatedCellTopologyId); + void processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, uint64_t capacityKey, const bounded_vector& currentSeeds, bounded_vector& updatedSeeds); void acceptTracks(int iteration, bounded_vector& tracks, const bounded_vector& trackIndices, bounded_vector>& firstClusters); void markTracks(int iteration); @@ -66,7 +81,6 @@ class TrackerTraits { mTrkParams = trkPars; } - TimeFrame* getTimeFrame() { return mTimeFrame; } virtual void setBz(float bz); float getBz() const { return mBz; } @@ -104,7 +118,9 @@ class TrackerTraits const int iteration, const TrackingFrameInfo* const* tfInfos, const Cluster* const* unsortedClusters, - const o2::base::Propagator* propagator); + const o2::base::Propagator* propagator, + const TrackFollowContext& followCtx, + TrackFollowerScratch& scratch); o2::gpu::GPUChainITS* mChain = nullptr; TimeFrame* mTimeFrame; diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingTopology.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingTopology.h index 16f5e6f01e873..80432ebc4151c 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingTopology.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingTopology.h @@ -70,6 +70,7 @@ class TrackingTopology const CellTopology* cells{nullptr}; const Range* cellsByFirstLinkIndex{nullptr}; const Id* cellsByFirstLink{nullptr}; + const Id* maxCellLevel{nullptr}; ///< host only, see getDeviceView Mask seedingLayerMask{0}; Id nLinks{0}; Id nCells{0}; @@ -78,6 +79,7 @@ class TrackingTopology GPUhdi() const LayerLink& getLink(Id id) const { return links[id]; } GPUhdi() const CellTopology& getCell(Id id) const { return cells[id]; } GPUhdi() Range getCellsStartingWithLink(Id linkId) const { return cellsByFirstLinkIndex[linkId]; } + GPUhdi() Id getMaxCellLevel(Id id) const { return maxCellLevel[id]; } #ifndef GPUCA_GPUCODE std::string asString() const @@ -93,7 +95,7 @@ class TrackingTopology const auto& c = cells[cellId]; const auto& first = links[c.firstLink]; const auto& second = links[c.secondLink]; - out += fmt::format("\n {}: {} -> {} -> {} hitMask={} links=({}, {})", cellId, first.fromLayer, first.toLayer, second.toLayer, c.hitLayerMask.asString(), c.firstLink, c.secondLink); + out += fmt::format("\n {}: {} -> {} -> {} hitMask={} links=({}, {}) maxLevel={}", cellId, first.fromLayer, first.toLayer, second.toLayer, c.hitLayerMask.asString(), c.firstLink, c.secondLink, maxCellLevel != nullptr ? int(maxCellLevel[cellId]) : -1); } return out; } @@ -143,6 +145,7 @@ class TrackingTopology } fillCellsByLink(); + fillMaxCellLevels(); } View getView() const @@ -151,6 +154,7 @@ class TrackingTopology mCells.data(), mCellsByFirstLinkIndex.data(), mCellsByFirstLink.data(), + mMaxCellLevel.data(), mSeedingLayerMask, mNLinks, mNCells, @@ -166,6 +170,7 @@ class TrackingTopology deviceCells, deviceCellsByFirstLinkIndex, deviceCellsByFirstLink, + nullptr, mSeedingLayerMask, mNLinks, mNCells, @@ -176,6 +181,7 @@ class TrackingTopology const auto& getCells() const noexcept { return mCells; } const auto& getCellsByFirstLinkIndex() const noexcept { return mCellsByFirstLinkIndex; } const auto& getCellsByFirstLink() const noexcept { return mCellsByFirstLink; } + const auto& getMaxCellLevels() const noexcept { return mMaxCellLevel; } Id getNLinks() const noexcept { return mNLinks; } Id getNCells() const noexcept { return mNCells; } Id getNCellsByFirstLink() const noexcept { return mNCellsByFirstLink; } @@ -190,6 +196,26 @@ class TrackingTopology mCells.fill({}); mCellsByFirstLinkIndex.fill(Range{0, 0}); mCellsByFirstLink.fill(0); + mMaxCellLevel.fill(0); + } + + void fillMaxCellLevels() + { + for (Id cellId = 0; cellId < mNCells; ++cellId) { + mMaxCellLevel[cellId] = 1; + } + for (int outerLayer = 0; outerLayer < mMaxLayers; ++outerLayer) { + for (Id cellId = 0; cellId < mNCells; ++cellId) { + if (mCells[cellId].hitLayerMask.last() != outerLayer) { + continue; + } + const auto& successors = mCellsByFirstLinkIndex[mCells[cellId].secondLink]; + for (Id i = 0; i < successors.getEntries(); ++i) { + const Id next = mCellsByFirstLink[successors.getFirstEntry() + i]; + mMaxCellLevel[next] = o2::gpu::CAMath::Max(mMaxCellLevel[next], static_cast(mMaxCellLevel[cellId] + 1)); + } + } + } } void fillCellsByLink() @@ -230,6 +256,7 @@ class TrackingTopology std::array mCells{}; std::array mCellsByFirstLinkIndex{}; std::array mCellsByFirstLink{}; + std::array mMaxCellLevel{}; }; } // namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h index c899dde24ed44..59c22b505bc94 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h @@ -99,6 +99,7 @@ class Vertexer private: std::uint32_t mTimeFrameCounter = 0; + double mTotalTime{0}; VertexerTraitsN* mTraits = nullptr; /// Observer pointer, not owned by this class TimeFrameN* mTimeFrame = nullptr; /// Observer pointer, not owned by this class @@ -164,6 +165,7 @@ float Vertexer::evaluateTask(void (Vertexer::*task)(T...), std LOGP(info, "iter:{}:{}: {}", iteration, StateNames[mCurStep], mMemoryPool->asString()); } + mTotalTime += diff; return diff; } diff --git a/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx b/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx new file mode 100644 index 0000000000000..6405e2aa0fb74 --- /dev/null +++ b/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx @@ -0,0 +1,141 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "ITStracking/CapacityEstimator.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "Framework/Logger.h" + +namespace o2::its +{ + +struct CapacityEstimator::Impl { + struct Entry { + float ratio{0.f}; + float margin{0.f}; + uint32_t nSamples{0}; + uint32_t nLowStreak{0}; + }; + + explicit Impl(Config config) : cfg{config} {} + + Config cfg; + mutable std::mutex mutex; + std::unordered_map entries; +}; + +CapacityEstimator::CapacityEstimator() : CapacityEstimator{Config{}} {} + +CapacityEstimator::CapacityEstimator(Config cfg) : mImpl{std::make_unique(cfg)} {} + +CapacityEstimator::~CapacityEstimator() = default; + +void CapacityEstimator::reset() +{ + std::lock_guard lock{mImpl->mutex}; + mImpl->entries.clear(); +} + +size_t CapacityEstimator::capacity(uint64_t key, double scale) const +{ + if (!(scale > 0.)) { + return 0; + } + std::lock_guard lock{mImpl->mutex}; + const auto it = mImpl->entries.find(key); + if (it == mImpl->entries.end() || it->second.nSamples == 0) { + return mImpl->cfg.floorSlots; + } + const auto& e = it->second; + const double raw = double(e.ratio) * scale * double(e.margin); + if (!std::isfinite(raw) || raw < 0.) { + return mImpl->cfg.floorSlots; + } + if (raw >= static_cast(std::numeric_limits::max())) { + return std::numeric_limits::max(); + } + return std::max(mImpl->cfg.floorSlots, static_cast(std::ceil(raw))); +} + +void CapacityEstimator::update(uint64_t key, double scale, size_t emitted, size_t capacityUsed, bool overflowed, bool memoryLimited) +{ + if (!(scale > 0.)) { + return; + } + std::lock_guard lock{mImpl->mutex}; + auto& e = mImpl->entries[key]; + const auto& cfg = mImpl->cfg; + + const bool firstSample = e.nSamples == 0; + if (firstSample) { + e.margin = cfg.marginInit; + } + const auto sample = static_cast(double(emitted) / scale); + e.ratio = firstSample ? sample : (cfg.alpha * sample) + ((1.f - cfg.alpha) * e.ratio); + ++e.nSamples; + + if (memoryLimited) { + e.nLowStreak = 0; + e.margin = std::max(cfg.marginMin, e.margin * cfg.marginDown); + return; + } + if (overflowed) { + e.nLowStreak = 0; + if (!firstSample) { + const float shortfall = capacityUsed ? static_cast(double(emitted) / double(capacityUsed)) : cfg.marginUp; + e.margin = std::min(cfg.marginMax, e.margin * std::clamp(shortfall * cfg.marginOverflowSlack, 1.02f, cfg.marginUp)); + } + return; + } + const float util = capacityUsed ? float(double(emitted) / double(capacityUsed)) : 1.f; + if (util < cfg.lowWatermark) { + if (++e.nLowStreak >= cfg.decayAfter) { + e.margin = std::max(cfg.marginMin, e.margin * cfg.marginDown); + e.nLowStreak = 0; + } + } else if (e.nLowStreak > 0) { + --e.nLowStreak; + } +} + +void CapacityEstimator::print() const +{ + std::lock_guard lock{mImpl->mutex}; + std::vector keys; + keys.reserve(mImpl->entries.size()); + for (const auto& [key, _] : mImpl->entries) { + keys.push_back(key); + } + std::sort(keys.begin(), keys.end(), [](KeyType a, KeyType b) { + const auto da = decodeKey(a); + const auto db = decodeKey(b); + return std::tie(da.site, da.iteration, da.variant, da.slot) < + std::tie(db.site, db.iteration, db.variant, db.slot); + }); + if (keys.empty()) { + return; + } + LOGP(info, "Printing CapacityEstimators:"); + for (const auto key : keys) { + const auto& value = mImpl->entries.at(key); + const auto decoded = decodeKey(key); + LOGP(info, "\tSite:{} | iter:{} | var:({},{}) | slot:{} | ratio:{} | margin:{} | sam:{} | low:{}", SlabSiteNames[decoded.site], decoded.iteration, getVariantHigh(decoded.variant), getVariantLow(decoded.variant), decoded.slot, value.ratio, value.margin, value.nSamples, value.nLowStreak); + } +} + +} // namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx index 2ef2f7b724337..497857295a269 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx @@ -43,6 +43,12 @@ constexpr float DefClusErrorCol = o2::itsmft::SegmentationAlpide::PitchCol * 0.5 constexpr float DefClusError2Row = DefClusErrorRow * DefClusErrorRow; constexpr float DefClusError2Col = DefClusErrorCol * DefClusErrorCol; +template +TimeFrame::TimeFrame() = default; + +template +TimeFrame::~TimeFrame() = default; + template void TimeFrame::addPrimaryVertex(const Vertex& vert) { diff --git a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx index 4862c3add5893..28e967386e984 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx @@ -59,6 +59,7 @@ float Tracker::clustersToTracks(const LogFunc& logger, const LogFunc& e if (mTrkParams[iteration].DropTFUponFailure) { mMemoryPool->print(); mTimeFrame->wipe(); + mTimeFrame->getCapacityEstimator().reset(); ++mNumberOfDroppedTFs; error(std::format("...Dropping TimeSlice {} (out of {} dropped {})...", mTimeSlice, mTimeFrameCounter, mNumberOfDroppedTFs)); } else { diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx index 7489e334996a0..589702e735118 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx @@ -25,6 +25,7 @@ #include #include #include +#include #include "DetectorsBase/Propagator.h" #include "GPUCommonMath.h" @@ -34,6 +35,7 @@ #include "ITStracking/IndexTableUtils.h" #include "ITStracking/LayerMask.h" #include "ITStracking/ROFLookupTables.h" +#include "ITStracking/SlabBumpAllocator.h" #include "ITStracking/TrackerTraits.h" #include "ITStracking/TrackFollower.h" #include "ITStracking/TrackHelpers.h" @@ -42,12 +44,6 @@ namespace o2::its { -struct PassMode { - using OnePass = std::integral_constant; - using TwoPassCount = std::integral_constant; - using TwoPassInsert = std::integral_constant; -}; - template void TrackerTraits::computeLayerTracklets(const int iteration, int iVertex) { @@ -62,31 +58,29 @@ void TrackerTraits::computeLayerTracklets(const int iteration, int iVer gsl::span diamondSpan(&diamondVert, 1); mTaskArena->execute([&] { - auto forTracklets = [&](auto Tag, int linkId, int pivotROF, int base, int& offset) -> int { + auto forTracklets = [&](int linkId, int pivotROF, auto&& emit) { const auto& link = topology.getLink(linkId); if (!mTimeFrame->getROFMaskView().isROFEnabled(link.fromLayer, pivotROF)) { - return 0; + return; } gsl::span primaryVertices = mTrkParams[iteration].UseDiamond ? diamondSpan : mTimeFrame->getPrimaryVertices(link.fromLayer, pivotROF); if (primaryVertices.empty()) { - return 0; + return; } const int startVtx = iVertex >= 0 ? iVertex : 0; const int endVtx = iVertex >= 0 ? o2::gpu::CAMath::Min(iVertex + 1, int(primaryVertices.size())) : int(primaryVertices.size()); if (endVtx <= startVtx || (iVertex + 1) > primaryVertices.size()) { - return 0; + return; } const auto& rofOverlap = mTimeFrame->getROFOverlapTableView().getOverlap(link.fromLayer, link.toLayer, pivotROF); if (!rofOverlap.getEntries()) { - return 0; + return; } - int localCount = 0; - auto& tracklets = mTimeFrame->getTracklets()[linkId]; auto layer0 = mTimeFrame->getClustersOnLayer(pivotROF, link.fromLayer); if (layer0.empty()) { - return 0; + return; } const float meanDeltaR = mTrkParams[iteration].LayerRadii[link.toLayer] - mTrkParams[iteration].LayerRadii[link.fromLayer]; @@ -160,64 +154,58 @@ void TrackerTraits::computeLayerTracklets(const int iteration, int iVer math_utils::isPhiDifferenceBelow(currentCluster.phi, nextCluster.phi, phiCut)) { const float phi{o2::gpu::CAMath::ATan2(currentCluster.yCoordinate - nextCluster.yCoordinate, currentCluster.xCoordinate - nextCluster.xCoordinate)}; const float tanL = (currentCluster.zCoordinate - nextCluster.zCoordinate) / (currentCluster.radius - nextCluster.radius); - if constexpr (decltype(Tag)::value == PassMode::OnePass::value) { - tracklets.emplace_back(currentSortedIndex, mTimeFrame->getSortedIndex(targetROF, link.toLayer, iNext), tanL, phi, ts); - } else if constexpr (decltype(Tag)::value == PassMode::TwoPassCount::value) { - ++localCount; - } else if constexpr (decltype(Tag)::value == PassMode::TwoPassInsert::value) { - const int idx = base + offset++; - tracklets[idx] = Tracklet(currentSortedIndex, mTimeFrame->getSortedIndex(targetROF, link.toLayer, iNext), tanL, phi, ts); - } + emit(currentSortedIndex, mTimeFrame->getSortedIndex(targetROF, link.toLayer, iNext), tanL, phi, ts); } } } } } } - return localCount; }; - int dummy{0}; if (mTaskArena->max_concurrency() <= 1) { for (int linkId{0}; linkId < topology.nLinks; ++linkId) { const int fromLayer = topology.getLink(linkId).fromLayer; - const int startROF = 0, endROF = mTimeFrame->getROFOverlapTableView().getLayer(fromLayer).mNROFsTF; - for (int pivotROF{startROF}; pivotROF < endROF; ++pivotROF) { - forTracklets(PassMode::OnePass{}, linkId, pivotROF, 0, dummy); + const int endROF = mTimeFrame->getROFOverlapTableView().getLayer(fromLayer).mNROFsTF; + auto& tracklets = mTimeFrame->getTracklets()[linkId]; + for (int pivotROF{0}; pivotROF < endROF; ++pivotROF) { + forTracklets(linkId, pivotROF, [&tracklets](auto&&... args) { tracklets.emplace_back(std::forward(args)...); }); } } } else { + const int maxConcurrency = std::max(1, mTaskArena->max_concurrency()); + const int nConcurrentSinks = std::min(static_cast(topology.nLinks), maxConcurrency); tbb::parallel_for(0, static_cast(topology.nLinks), [&](const int linkId) { const int fromLayer = topology.getLink(linkId).fromLayer; const int startROF = 0, endROF = mTimeFrame->getROFOverlapTableView().getLayer(fromLayer).mNROFsTF; - bounded_vector perROFCount((endROF - startROF) + 1, mMemoryPool.get()); - tbb::parallel_for(startROF, endROF, [&](const int pivotROF) { - perROFCount[pivotROF - startROF] = forTracklets(PassMode::TwoPassCount{}, linkId, pivotROF, 0, dummy); - }); - std::exclusive_scan(perROFCount.begin(), perROFCount.end(), perROFCount.begin(), 0); - const int nTracklets = perROFCount.back(); - mTimeFrame->getTracklets()[linkId].resize(nTracklets); - if (nTracklets == 0) { - return; - } + auto& tracklets = mTimeFrame->getTracklets()[linkId]; + const auto key = CapacityEstimator::makeKey(SlabSite::Tracklets, iteration, iVertex + 1, linkId); + const auto scale = static_cast(mTimeFrame->getClusters()[fromLayer].size()); + const size_t capacity = mTimeFrame->getCapacityEstimator().capacity(key, scale); + + UnorderedSlabSink sink{{.capacity = capacity, .nThreads = maxConcurrency, .nConcurrentSinks = nConcurrentSinks}, mMemoryPool.get()}; tbb::parallel_for(startROF, endROF, [&](const int pivotROF) { - int baseIdx = perROFCount[pivotROF - startROF]; - if (baseIdx == perROFCount[pivotROF + 1 - startROF]) { - return; - } - int localIdx = 0; - forTracklets(PassMode::TwoPassInsert{}, linkId, pivotROF, baseIdx, localIdx); + auto& handle = sink.local(); + forTracklets(linkId, pivotROF, [&handle](auto&&... args) { handle.emplace(std::forward(args)...); }); }); + const auto st = sink.stats(); + sink.finalizeUnordered(tracklets); + mTimeFrame->getCapacityEstimator().update(key, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); }); } tbb::parallel_for(0, static_cast(topology.nLinks), [&](const int linkId) { /// Sort tracklets & remove duplicates - // duplicates can exist simply since we evaluate per vertex auto& trkl{mTimeFrame->getTracklets()[linkId]}; - std::sort(trkl.begin(), trkl.end()); - trkl.erase(std::unique(trkl.begin(), trkl.end()), trkl.end()); - trkl.shrink_to_fit(); + if (mTaskArena->max_concurrency() > 1) { + tbb::parallel_sort(trkl.begin(), trkl.end()); + } else { + std::sort(trkl.begin(), trkl.end()); + } + if (iVertex < 0) { // duplicates can exist simply since we evaluate for all vertices if we do perVertex duplicates cannot exist + trkl.erase(std::unique(trkl.begin(), trkl.end()), trkl.end()); + trkl.shrink_to_fit(); + } auto& lut{mTimeFrame->getTrackletsLookupTable()[linkId]}; if (!trkl.empty()) { for (const auto& tkl : trkl) { @@ -257,16 +245,26 @@ template void TrackerTraits::computeLayerCells(const int iteration) { const auto topology = mTimeFrame->getTrackingTopologyView(); - for (int cellTopologyId = 0; cellTopologyId < topology.nCells; ++cellTopologyId) { - deepVectorClear(mTimeFrame->getCells()[cellTopologyId]); - deepVectorClear(mTimeFrame->getCellsLookupTable()[cellTopologyId]); - if (mTimeFrame->hasMCinformation() && mTrkParams[iteration].CreateArtefactLabels) { - deepVectorClear(mTimeFrame->getCellsLabel(cellTopologyId)); - } - } + const bool createLabels = mTimeFrame->hasMCinformation() && mTrkParams[iteration].CreateArtefactLabels; mTaskArena->execute([&] { - auto forTrackletCells = [&](auto Tag, int cellTopologyId, bounded_vector& layerCells, int iTracklet, int offset = 0) -> int { + const int maxConcurrency = std::max(1, mTaskArena->max_concurrency()); + auto clearTopology = [&](const int cellTopologyId) { + deepVectorClear(mTimeFrame->getCells()[cellTopologyId]); + deepVectorClear(mTimeFrame->getCellsLookupTable()[cellTopologyId]); + if (createLabels) { + deepVectorClear(mTimeFrame->getCellsLabel(cellTopologyId)); + } + }; + if (maxConcurrency > 1) { + tbb::parallel_for(0, static_cast(topology.nCells), clearTopology); + } else { + for (int cellTopologyId{0}; cellTopologyId < topology.nCells; ++cellTopologyId) { + clearTopology(cellTopologyId); + } + } + + auto forTrackletCells = [&](int cellTopologyId, int iTracklet, auto&& emit) { const auto& cellTopology = topology.getCell(cellTopologyId); const auto& firstLink = topology.getLink(cellTopology.firstLink); const auto& secondLink = topology.getLink(cellTopology.secondLink); @@ -274,7 +272,6 @@ void TrackerTraits::computeLayerCells(const int iteration) const int nextLayerClusterIndex{currentTracklet.secondClusterIndex}; const int nextLayerFirstTrackletIndex{mTimeFrame->getTrackletsLookupTable()[cellTopology.secondLink][nextLayerClusterIndex]}; const int nextLayerLastTrackletIndex{mTimeFrame->getTrackletsLookupTable()[cellTopology.secondLink][nextLayerClusterIndex + 1]}; - int foundCells{0}; for (int iNextTracklet{nextLayerFirstTrackletIndex}; iNextTracklet < nextLayerLastTrackletIndex; ++iNextTracklet) { const Tracklet& nextTracklet{mTimeFrame->getTracklets()[cellTopology.secondLink][iNextTracklet]}; if (nextTracklet.firstClusterIndex != nextLayerClusterIndex) { @@ -293,10 +290,10 @@ void TrackerTraits::computeLayerCells(const int iteration) mTimeFrame->getClusters()[firstLink.toLayer][nextTracklet.firstClusterIndex].clusterId, mTimeFrame->getClusters()[secondLink.toLayer][nextTracklet.secondClusterIndex].clusterId}; const int hitLayers[3]{firstLink.fromLayer, firstLink.toLayer, secondLink.toLayer}; - const auto& cluster1_glo = mTimeFrame->getUnsortedClusters()[firstLink.fromLayer][clusId[0]]; - const auto& cluster2_glo = mTimeFrame->getUnsortedClusters()[firstLink.toLayer][clusId[1]]; - const auto& cluster3_tf = mTimeFrame->getTrackingFrameInfoOnLayer(secondLink.toLayer)[clusId[2]]; - auto track{o2::its::track::buildTrackSeed(cluster1_glo, cluster2_glo, cluster3_tf, mBz)}; + const auto& cluster1Glo = mTimeFrame->getUnsortedClusters()[firstLink.fromLayer][clusId[0]]; + const auto& cluster2Glo = mTimeFrame->getUnsortedClusters()[firstLink.toLayer][clusId[1]]; + const auto& cluster3Tf = mTimeFrame->getTrackingFrameInfoOnLayer(secondLink.toLayer)[clusId[2]]; + auto track{o2::its::track::buildTrackSeed(cluster1Glo, cluster2Glo, cluster3Tf, mBz)}; float chi2{0.f}; bool good{false}; @@ -331,67 +328,56 @@ void TrackerTraits::computeLayerCells(const int iteration) if (good) { TimeEstBC ts = currentTracklet.getTimeStamp(); ts += nextTracklet.getTimeStamp(); - if constexpr (decltype(Tag)::value == PassMode::OnePass::value) { - layerCells.emplace_back(cellTopology.hitLayerMask, clusId[0], clusId[1], clusId[2], iTracklet, iNextTracklet, track, chi2, ts); - ++foundCells; - } else if constexpr (decltype(Tag)::value == PassMode::TwoPassCount::value) { - ++foundCells; - } else if constexpr (decltype(Tag)::value == PassMode::TwoPassInsert::value) { - layerCells[offset++] = CellSeed(cellTopology.hitLayerMask, clusId[0], clusId[1], clusId[2], iTracklet, iNextTracklet, track, chi2, ts); - ++foundCells; - } else { - static_assert(false, "Unknown mode!"); - } + emit(cellTopology.hitLayerMask, clusId[0], clusId[1], clusId[2], iTracklet, iNextTracklet, track, chi2, ts); } } } - return foundCells; }; + bounded_vector activeTopologies(mMemoryPool.get()); + activeTopologies.reserve(topology.nCells); for (int cellTopologyId = 0; cellTopologyId < topology.nCells; ++cellTopologyId) { const auto& cellTopology = topology.getCell(cellTopologyId); - if (mTimeFrame->getTracklets()[cellTopology.firstLink].empty() || - mTimeFrame->getTracklets()[cellTopology.secondLink].empty()) { - continue; + if (!mTimeFrame->getTracklets()[cellTopology.firstLink].empty() && + !mTimeFrame->getTracklets()[cellTopology.secondLink].empty()) { + activeTopologies.push_back(cellTopologyId); } + } + + const int nConcurrentSinks = std::min(maxConcurrency, static_cast(activeTopologies.size())); + auto processTopology = [&](const int cellTopologyId) { + const auto& cellTopology = topology.getCell(cellTopologyId); auto& layerCells = mTimeFrame->getCells()[cellTopologyId]; + auto& lut = mTimeFrame->getCellsLookupTable()[cellTopologyId]; const int currentLayerTrackletsNum{static_cast(mTimeFrame->getTracklets()[cellTopology.firstLink].size())}; - bounded_vector perTrackletCount(currentLayerTrackletsNum + 1, 0, mMemoryPool.get()); - if (mTaskArena->max_concurrency() <= 1) { - for (int iTracklet{0}; iTracklet < currentLayerTrackletsNum; ++iTracklet) { - perTrackletCount[iTracklet] = forTrackletCells(PassMode::OnePass{}, cellTopologyId, layerCells, iTracklet); - } - std::exclusive_scan(perTrackletCount.begin(), perTrackletCount.end(), perTrackletCount.begin(), 0); - } else { - tbb::parallel_for(0, currentLayerTrackletsNum, [&](const int iTracklet) { - perTrackletCount[iTracklet] = forTrackletCells(PassMode::TwoPassCount{}, cellTopologyId, layerCells, iTracklet); - }); - std::exclusive_scan(perTrackletCount.begin(), perTrackletCount.end(), perTrackletCount.begin(), 0); - auto totalCells{perTrackletCount.back()}; - if (totalCells == 0) { - auto& lut = mTimeFrame->getCellsLookupTable()[cellTopologyId]; - lut.resize(currentLayerTrackletsNum + 1); - std::fill(lut.begin(), lut.end(), 0); - continue; - } - layerCells.resize(totalCells); + const auto key = CapacityEstimator::makeKey(SlabSite::Cells, iteration, 0, cellTopologyId); + const auto scale = static_cast(currentLayerTrackletsNum); + if (maxConcurrency > 1) { + const size_t capacity = mTimeFrame->getCapacityEstimator().capacity(key, scale); + GroupedSlabSink sink{{.capacity = capacity, .nThreads = maxConcurrency, .nConcurrentSinks = nConcurrentSinks}, mMemoryPool.get()}; tbb::parallel_for(0, currentLayerTrackletsNum, [&](const int iTracklet) { - int offset = perTrackletCount[iTracklet]; - if (offset == perTrackletCount[iTracklet + 1]) { - return; - } - forTrackletCells(PassMode::TwoPassInsert{}, cellTopologyId, layerCells, iTracklet, offset); + auto& handle = sink.local(); + handle.beginProducer(iTracklet); + forTrackletCells(cellTopologyId, iTracklet, [&handle](auto&&... args) { handle.emplace(std::forward(args)...); }); }); + const auto st = sink.stats(); + sink.finalizeGrouped(size_t(currentLayerTrackletsNum), lut, layerCells); + mTimeFrame->getCapacityEstimator().update(key, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); + } else { + lut.resize(currentLayerTrackletsNum + 1); + for (int iTracklet{0}; iTracklet < currentLayerTrackletsNum; ++iTracklet) { + lut[iTracklet] = static_cast(layerCells.size()); + forTrackletCells(cellTopologyId, iTracklet, [&](auto&&... args) { + layerCells.emplace_back(std::forward(args)...); + }); + } + lut.back() = static_cast(layerCells.size()); } - auto& lut = mTimeFrame->getCellsLookupTable()[cellTopologyId]; - lut.resize(currentLayerTrackletsNum + 1); - std::copy_n(perTrackletCount.begin(), currentLayerTrackletsNum + 1, lut.begin()); - - if (mTimeFrame->hasMCinformation() && mTrkParams[iteration].CreateArtefactLabels) { + if (createLabels) { auto& labels = mTimeFrame->getCellsLabel(cellTopologyId); labels.reserve(layerCells.size()); for (const auto& cell : layerCells) { @@ -400,13 +386,30 @@ void TrackerTraits::computeLayerCells(const int iteration) labels.emplace_back(currentLab == nextLab ? currentLab : MCCompLabel()); } } + }; + + if (maxConcurrency > 1) { + tbb::parallel_for(0, static_cast(activeTopologies.size()), [&](const int i) { + processTopology(activeTopologies[i]); + }); + } else { + for (const int cellTopologyId : activeTopologies) { + processTopology(cellTopologyId); + } } - }); - for (int linkId = 0; linkId < topology.nLinks; ++linkId) { - deepVectorClear(mTimeFrame->getTracklets()[linkId]); - deepVectorClear(mTimeFrame->getTrackletsLabel(linkId)); - } + auto clearTracklets = [&](const int linkId) { + deepVectorClear(mTimeFrame->getTracklets()[linkId]); + deepVectorClear(mTimeFrame->getTrackletsLabel(linkId)); + }; + if (maxConcurrency > 1) { + tbb::parallel_for(0, static_cast(topology.nLinks), clearTracklets); + } else { + for (int linkId{0}; linkId < topology.nLinks; ++linkId) { + clearTracklets(linkId); + } + } + }); } template @@ -414,16 +417,29 @@ void TrackerTraits::findCellsNeighbours(const int iteration) { const auto topology = mTimeFrame->getTrackingTopologyView(); mTaskArena->execute([&] { - std::vector> cellsNeighboursByTarget; - cellsNeighboursByTarget.reserve(topology.nCells); - for (int cellTopologyId{0}; cellTopologyId < topology.nCells; ++cellTopologyId) { + const int maxConcurrency = std::max(1, mTaskArena->max_concurrency()); + auto clearNeighbours = [&](const int cellTopologyId) { deepVectorClear(mTimeFrame->getCellsNeighbours()[cellTopologyId]); deepVectorClear(mTimeFrame->getCellsNeighboursTopology()[cellTopologyId]); deepVectorClear(mTimeFrame->getCellsNeighboursLUT()[cellTopologyId]); - cellsNeighboursByTarget.emplace_back(mMemoryPool.get()); + }; + if (maxConcurrency > 1) { + tbb::parallel_for(0, static_cast(topology.nCells), clearNeighbours); + } else { + for (int cellTopologyId{0}; cellTopologyId < topology.nCells; ++cellTopologyId) { + clearNeighbours(cellTopologyId); + } } + auto neighbourLess = [](const CellNeighbour& a, const CellNeighbour& b) { + return std::tie(a.nextCellTopology, a.nextCell, a.cellTopology, a.cell) < + std::tie(b.nextCellTopology, b.nextCell, b.cellTopology, b.cell); + }; + for (int outerLayer{0}; outerLayer < NLayers; ++outerLayer) { + bounded_vector activeTopologies(mMemoryPool.get()); + activeTopologies.reserve(topology.nCells); + size_t sourceCellCount{0}; for (int cellTopologyId{0}; cellTopologyId < topology.nCells; ++cellTopologyId) { const auto& cellTopology = topology.getCell(cellTopologyId); if (cellTopology.hitLayerMask.last() != outerLayer || @@ -434,129 +450,197 @@ void TrackerTraits::findCellsNeighbours(const int iteration) if (!successors.getEntries()) { continue; } + activeTopologies.push_back(cellTopologyId); + sourceCellCount += mTimeFrame->getCells()[cellTopologyId].size(); + } - tbb::enumerable_thread_specific> sourceNeighbours([&]() { return bounded_vector{mMemoryPool.get()}; }); - tbb::parallel_for(0, static_cast(mTimeFrame->getCells()[cellTopologyId].size()), [&](const int iCell) { - auto& localNeighbours = sourceNeighbours.local(); - const auto& currentCellSeed{mTimeFrame->getCells()[cellTopologyId][iCell]}; - const int nextLayerTrackletIndex{currentCellSeed.getSecondTrackletIndex()}; - for (int iSuccessor{0}; iSuccessor < successors.getEntries(); ++iSuccessor) { - const int nextCellTopologyId = topology.cellsByFirstLink[successors.getFirstEntry() + iSuccessor]; - if (mTimeFrame->getCells()[nextCellTopologyId].empty() || - mTimeFrame->getCellsLookupTable()[nextCellTopologyId].empty()) { - continue; + if (activeTopologies.empty()) { + continue; + } + + auto forSourceCell = [&](const int cellTopologyId, const int iCell, auto&& emit) { + const auto& cellTopology = topology.getCell(cellTopologyId); + const auto successors = topology.getCellsStartingWithLink(cellTopology.secondLink); + const auto& currentCellSeed{mTimeFrame->getCells()[cellTopologyId][iCell]}; + const int nextLayerTrackletIndex{currentCellSeed.getSecondTrackletIndex()}; + for (int iSuccessor{0}; iSuccessor < successors.getEntries(); ++iSuccessor) { + const int nextCellTopologyId = topology.cellsByFirstLink[successors.getFirstEntry() + iSuccessor]; + if (mTimeFrame->getCells()[nextCellTopologyId].empty() || + mTimeFrame->getCellsLookupTable()[nextCellTopologyId].empty()) { + continue; + } + const auto& nextCellLUT = mTimeFrame->getCellsLookupTable()[nextCellTopologyId]; + if (nextLayerTrackletIndex + 1 >= static_cast(nextCellLUT.size())) { + continue; + } + const int nextLayerFirstCellIndex{nextCellLUT[nextLayerTrackletIndex]}; + const int nextLayerLastCellIndex{nextCellLUT[nextLayerTrackletIndex + 1]}; + for (int iNextCell{nextLayerFirstCellIndex}; iNextCell < nextLayerLastCellIndex; ++iNextCell) { + const auto& nextCellSeedRef{mTimeFrame->getCells()[nextCellTopologyId][iNextCell]}; + if (nextCellSeedRef.getFirstTrackletIndex() != nextLayerTrackletIndex || !currentCellSeed.getTimeStamp().isCompatible(nextCellSeedRef.getTimeStamp())) { + break; } - const auto& nextCellLUT = mTimeFrame->getCellsLookupTable()[nextCellTopologyId]; - if (nextLayerTrackletIndex + 1 >= static_cast(nextCellLUT.size())) { + + auto nextCellSeed{mTimeFrame->getCells()[nextCellTopologyId][iNextCell]}; /// copy + if (!nextCellSeed.rotate(currentCellSeed.getAlpha()) || + !nextCellSeed.propagateTo(currentCellSeed.getX(), getBz())) { continue; } - const int nextLayerFirstCellIndex{nextCellLUT[nextLayerTrackletIndex]}; - const int nextLayerLastCellIndex{nextCellLUT[nextLayerTrackletIndex + 1]}; - for (int iNextCell{nextLayerFirstCellIndex}; iNextCell < nextLayerLastCellIndex; ++iNextCell) { - const auto& nextCellSeedRef{mTimeFrame->getCells()[nextCellTopologyId][iNextCell]}; - if (nextCellSeedRef.getFirstTrackletIndex() != nextLayerTrackletIndex || !currentCellSeed.getTimeStamp().isCompatible(nextCellSeedRef.getTimeStamp())) { - break; - } - auto nextCellSeed{mTimeFrame->getCells()[nextCellTopologyId][iNextCell]}; /// copy - if (!nextCellSeed.rotate(currentCellSeed.getAlpha()) || - !nextCellSeed.propagateTo(currentCellSeed.getX(), getBz())) { - continue; - } - - float chi2 = currentCellSeed.getPredictedChi2(nextCellSeed); - if (chi2 > mTrkParams[iteration].MaxChi2ClusterAttachment) { - continue; - } - - const int nextLevel = currentCellSeed.getLevel() + 1; - localNeighbours.emplace_back(cellTopologyId, iCell, nextCellTopologyId, iNextCell, nextLevel); + float chi2 = currentCellSeed.getPredictedChi2(nextCellSeed); + if (chi2 > mTrkParams[iteration].MaxChi2ClusterAttachment) { + continue; } - } - }); - bounded_vector count(topology.nCells, 0, mMemoryPool.get()); - for (const auto& localNeighbours : sourceNeighbours) { - for (const auto& neigh : localNeighbours) { - ++count[neigh.nextCellTopology]; + const int nextLevel = currentCellSeed.getLevel() + 1; + emit(cellTopologyId, iCell, nextCellTopologyId, iNextCell, nextLevel); } } - for (size_t i{0}; i < topology.nCells; ++i) { - cellsNeighboursByTarget[i].reserve(count[i]); - } - for (const auto& localNeighbours : sourceNeighbours) { - for (const auto& neigh : localNeighbours) { - cellsNeighboursByTarget[neigh.nextCellTopology].emplace_back(neigh); - if (neigh.level > mTimeFrame->getCells()[neigh.nextCellTopology][neigh.nextCell].getLevel()) { - mTimeFrame->getCells()[neigh.nextCellTopology][neigh.nextCell].setLevel(neigh.level); - } + }; + + bounded_vector waveNeighbours{mMemoryPool.get()}; + const auto key = CapacityEstimator::makeKey(SlabSite::Neighbours, iteration, 0, outerLayer); + const auto scale = static_cast(sourceCellCount); + if (maxConcurrency > 1) { + const size_t capacity = mTimeFrame->getCapacityEstimator().capacity(key, scale); + UnorderedSlabSink sink{{.capacity = capacity, .nThreads = maxConcurrency}, mMemoryPool.get()}; + tbb::parallel_for(0, static_cast(activeTopologies.size()), [&](const int i) { + const int cellTopologyId = activeTopologies[i]; + tbb::parallel_for(0, static_cast(mTimeFrame->getCells()[cellTopologyId].size()), [&](const int iCell) { + auto& handle = sink.local(); + forSourceCell(cellTopologyId, iCell, [&handle](auto&&... args) { + handle.emplace(std::forward(args)...); + }); + }); + }); + const auto st = sink.stats(); + sink.finalizeUnordered(waveNeighbours); + mTimeFrame->getCapacityEstimator().update(key, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); + tbb::parallel_sort(waveNeighbours.begin(), waveNeighbours.end(), neighbourLess); + } else { + for (const int cellTopologyId : activeTopologies) { + for (int iCell{0}; iCell < static_cast(mTimeFrame->getCells()[cellTopologyId].size()); ++iCell) { + forSourceCell(cellTopologyId, iCell, [&](auto&&... args) { + waveNeighbours.emplace_back(std::forward(args)...); + }); } } + std::sort(waveNeighbours.begin(), waveNeighbours.end(), neighbourLess); } - } - for (int cellTopologyId{0}; cellTopologyId < topology.nCells; ++cellTopologyId) { - auto& cellsNeighbours = cellsNeighboursByTarget[cellTopologyId]; - if (cellsNeighbours.empty()) { - continue; + struct TargetSpan { + int topologyId; + size_t begin; + size_t end; + }; + bounded_vector targetSpans{mMemoryPool.get()}; + targetSpans.reserve(topology.nCells); + for (int targetTopologyId{0}; targetTopologyId < topology.nCells; ++targetTopologyId) { + const auto first = std::lower_bound(waveNeighbours.begin(), waveNeighbours.end(), targetTopologyId, + [](const CellNeighbour& neighbour, int id) { return neighbour.nextCellTopology < id; }); + const auto last = std::upper_bound(first, waveNeighbours.end(), targetTopologyId, + [](int id, const CellNeighbour& neighbour) { return id < neighbour.nextCellTopology; }); + if (first != last) { + targetSpans.push_back({targetTopologyId, static_cast(first - waveNeighbours.begin()), static_cast(last - waveNeighbours.begin())}); + } } - std::sort(cellsNeighbours.begin(), cellsNeighbours.end(), [](const auto& a, const auto& b) { - return a.nextCell < b.nextCell; - }); - - auto& cellsNeighbourLUT = mTimeFrame->getCellsNeighboursLUT()[cellTopologyId]; - cellsNeighbourLUT.assign(mTimeFrame->getCells()[cellTopologyId].size(), 0); - for (const auto& neigh : cellsNeighbours) { - ++cellsNeighbourLUT[neigh.nextCell]; + auto finalizeTarget = [&](const int i) { + const auto [targetTopologyId, begin, end] = targetSpans[i]; + auto& cellsNeighbourLUT = mTimeFrame->getCellsNeighboursLUT()[targetTopologyId]; + cellsNeighbourLUT.assign(mTimeFrame->getCells()[targetTopologyId].size(), 0); + for (size_t j{begin}; j < end; ++j) { + const auto& neighbour = waveNeighbours[j]; + ++cellsNeighbourLUT[neighbour.nextCell]; + auto& targetCell = mTimeFrame->getCells()[targetTopologyId][neighbour.nextCell]; + if (neighbour.level > targetCell.getLevel()) { + targetCell.setLevel(neighbour.level); + } + } + std::inclusive_scan(cellsNeighbourLUT.begin(), cellsNeighbourLUT.end(), cellsNeighbourLUT.begin()); + + auto& cellsNeighbours = mTimeFrame->getCellsNeighbours()[targetTopologyId]; + auto& cellsNeighboursTopology = mTimeFrame->getCellsNeighboursTopology()[targetTopologyId]; + cellsNeighbours.resize(end - begin); + cellsNeighboursTopology.resize(end - begin); + for (size_t j{begin}; j < end; ++j) { + cellsNeighbours[j - begin] = waveNeighbours[j].cell; + cellsNeighboursTopology[j - begin] = waveNeighbours[j].cellTopology; + } + }; + if (maxConcurrency > 1) { + tbb::parallel_for(0, static_cast(targetSpans.size()), finalizeTarget); + } else { + for (int i{0}; i < static_cast(targetSpans.size()); ++i) { + finalizeTarget(i); + } } - std::inclusive_scan(cellsNeighbourLUT.begin(), cellsNeighbourLUT.end(), cellsNeighbourLUT.begin()); - - mTimeFrame->getCellsNeighbours()[cellTopologyId].reserve(cellsNeighbours.size()); - mTimeFrame->getCellsNeighboursTopology()[cellTopologyId].reserve(cellsNeighbours.size()); - std::ranges::transform(cellsNeighbours, std::back_inserter(mTimeFrame->getCellsNeighbours()[cellTopologyId]), [](const auto& neigh) { return neigh.cell; }); - std::ranges::transform(cellsNeighbours, std::back_inserter(mTimeFrame->getCellsNeighboursTopology()[cellTopologyId]), [](const auto& neigh) { return neigh.cellTopology; }); } // clean up LUTs - for (auto& cellLUT : mTimeFrame->getCellsLookupTable()) { - deepVectorClear(cellLUT); + auto clearCellLUT = [&](const int cellTopologyId) { + deepVectorClear(mTimeFrame->getCellsLookupTable()[cellTopologyId]); + }; + if (maxConcurrency > 1) { + tbb::parallel_for(0, static_cast(topology.nCells), clearCellLUT); + } else { + for (int cellTopologyId{0}; cellTopologyId < topology.nCells; ++cellTopologyId) { + clearCellLUT(cellTopologyId); + } } }); } template template -void TrackerTraits::processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, const bounded_vector& currentCellSeed, const bounded_vector& currentCellId, const bounded_vector& currentCellTopologyId, bounded_vector& updatedCellSeeds, bounded_vector& updatedCellsIds, bounded_vector& updatedCellsTopologyIds) +void TrackerTraits::processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, uint64_t capacityKey, const bounded_vector& currentSeeds, bounded_vector& updatedSeeds) { + constexpr bool IsInitial = std::is_same_v; + static_assert(IsInitial || std::is_same_v); auto propagator = o2::base::Propagator::Instance(); mTaskArena->execute([&] { - auto forCellNeighbours = [&](auto Tag, int iCell, int offset = 0) -> int { - const auto& currentCell{currentCellSeed[iCell]}; - const int cellTopologyId = currentCellTopologyId.empty() ? defaultCellTopologyId : currentCellTopologyId[iCell]; - - if constexpr (decltype(Tag)::value != PassMode::TwoPassInsert::value) { - if (currentCell.getLevel() != iLevel) { - return 0; + auto forCellNeighbours = [&](int iCell, auto&& emit) { + const auto& inputSeed = currentSeeds[iCell]; + const auto& currentCell = [&]() -> const auto& { + if constexpr (IsInitial) { + return inputSeed; + } else { + return inputSeed.seed; } - if (currentCellId.empty()) { - for (int layer = 0; layer < NLayers; ++layer) { - const int clusterIndex = currentCell.getCluster(layer); - if (clusterIndex != constants::UnusedIndex && mTimeFrame->isClusterUsed(layer, clusterIndex)) { - return 0; /// this we do only on the first iteration, hence the check on currentCellId - } + }(); + const int cellTopologyId = [&]() { + if constexpr (IsInitial) { + return defaultCellTopologyId; + } else { + return inputSeed.cellTopologyId; + } + }(); + const int cellId = [&]() { + if constexpr (IsInitial) { + return iCell; + } else { + return inputSeed.cellId; + } + }(); + + if (currentCell.getLevel() != iLevel) { + return; + } + if constexpr (IsInitial) { + for (int layer = 0; layer < NLayers; ++layer) { + const int clusterIndex = currentCell.getCluster(layer); + if (clusterIndex != constants::UnusedIndex && mTimeFrame->isClusterUsed(layer, clusterIndex)) { + return; } } } - const int cellId = currentCellId.empty() ? iCell : currentCellId[iCell]; if (cellTopologyId < 0 || mTimeFrame->getCellsNeighboursLUT()[cellTopologyId].empty()) { - return 0; + return; } const int startNeighbourId{cellId ? mTimeFrame->getCellsNeighboursLUT()[cellTopologyId][cellId - 1] : 0}; const int endNeighbourId{mTimeFrame->getCellsNeighboursLUT()[cellTopologyId][cellId]}; - int foundSeeds{0}; for (int iNeighbourCell{startNeighbourId}; iNeighbourCell < endNeighbourId; ++iNeighbourCell) { const int neighbourCellTopologyId = mTimeFrame->getCellsNeighboursTopology()[cellTopologyId][iNeighbourCell]; const int neighbourCellId = mTimeFrame->getCellsNeighbours()[cellTopologyId][iNeighbourCell]; @@ -605,60 +689,34 @@ void TrackerTraits::processNeighbours(int iteration, int defaultCellTop continue; } - if constexpr (decltype(Tag)::value != PassMode::TwoPassCount::value) { - seed.getClusters()[neighbourLayer] = neighbourCluster; - auto mask = seed.getHitLayerMask(); - mask.set(neighbourLayer); - seed.setHitLayerMask(mask); - seed.setLevel(neighbourCell.getLevel()); - seed.setFirstTrackletIndex(neighbourCell.getFirstTrackletIndex()); - seed.setSecondTrackletIndex(neighbourCell.getSecondTrackletIndex()); - } - - if constexpr (decltype(Tag)::value == PassMode::OnePass::value) { - updatedCellSeeds.push_back(seed); - updatedCellsIds.push_back(neighbourCellId); - updatedCellsTopologyIds.push_back(neighbourCellTopologyId); - } else if constexpr (decltype(Tag)::value == PassMode::TwoPassCount::value) { - ++foundSeeds; - } else if constexpr (decltype(Tag)::value == PassMode::TwoPassInsert::value) { - updatedCellSeeds[offset] = seed; - updatedCellsIds[offset] = neighbourCellId; - updatedCellsTopologyIds[offset++] = neighbourCellTopologyId; - } else { - static_assert(false, "Unknown mode!"); - } + seed.getClusters()[neighbourLayer] = neighbourCluster; + auto mask = seed.getHitLayerMask(); + mask.set(neighbourLayer); + seed.setHitLayerMask(mask); + seed.setLevel(neighbourCell.getLevel()); + seed.setFirstTrackletIndex(neighbourCell.getFirstTrackletIndex()); + seed.setSecondTrackletIndex(neighbourCell.getSecondTrackletIndex()); + emit(std::move(seed), neighbourCellId, neighbourCellTopologyId); } - return foundSeeds; }; - const int nCells = static_cast(currentCellSeed.size()); + const int nCells = static_cast(currentSeeds.size()); if (mTaskArena->max_concurrency() <= 1) { for (int iCell{0}; iCell < nCells; ++iCell) { - forCellNeighbours(PassMode::OnePass{}, iCell); + forCellNeighbours(iCell, [&](auto&&... args) { updatedSeeds.emplace_back(std::forward(args)...); }); } } else { - bounded_vector perCellCount(nCells + 1, 0, mMemoryPool.get()); - tbb::parallel_for(0, nCells, [&](const int iCell) { - perCellCount[iCell] = forCellNeighbours(PassMode::TwoPassCount{}, iCell); - }); - - std::exclusive_scan(perCellCount.begin(), perCellCount.end(), perCellCount.begin(), 0); - auto totalNeighbours{perCellCount.back()}; - if (totalNeighbours == 0) { - return; - } - updatedCellSeeds.resize(totalNeighbours); - updatedCellsIds.resize(totalNeighbours); - updatedCellsTopologyIds.resize(totalNeighbours); + const auto scale = static_cast(nCells); + const size_t capacity = mTimeFrame->getCapacityEstimator().capacity(capacityKey, scale); + UnorderedSlabSink sink{{.capacity = capacity, .nThreads = mTaskArena->max_concurrency()}, mMemoryPool.get()}; tbb::parallel_for(0, nCells, [&](const int iCell) { - int offset = perCellCount[iCell]; - if (offset == perCellCount[iCell + 1]) { - return; - } - forCellNeighbours(PassMode::TwoPassInsert{}, iCell, offset); + auto& handle = sink.local(); + forCellNeighbours(iCell, [&](auto&&... args) { handle.emplace(std::forward(args)...); }); }); + const auto st = sink.stats(); + sink.finalizeUnordered(updatedSeeds); + mTimeFrame->getCapacityEstimator().update(capacityKey, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); } }); } @@ -669,7 +727,9 @@ bool TrackerTraits::finaliseTrackSeed(const TrackSeedN& seed, const int iteration, const TrackingFrameInfo* const* tfInfos, const Cluster* const* unsortedClusters, - const o2::base::Propagator* propagator) + const o2::base::Propagator* propagator, + const TrackFollowContext& followCtx, + TrackFollowerScratch& scratch) { const auto& trkParams = mTrkParams[iteration]; const track::TrackFitContext fitCtx{ @@ -703,32 +763,12 @@ bool TrackerTraits::finaliseTrackSeed(const TrackSeedN& seed, return passesFinalLengthCut(track); } - const int maxHypotheses = std::max(1, trkParams.TrackFollowerMaxHypotheses); - TrackFollowerScratch scratch{mMemoryPool.get()}; - if (static_cast(scratch.activeHypotheses.size()) < maxHypotheses) { - scratch.activeHypotheses.resize(maxHypotheses); - } - if (static_cast(scratch.nextHypotheses.size()) < maxHypotheses) { - scratch.nextHypotheses.resize(maxHypotheses); + if (static_cast(scratch.activeHypotheses.size()) < followCtx.maxHypotheses) { + scratch.activeHypotheses.resize(followCtx.maxHypotheses); } - - const Cluster* clustersPtrs[NLayers]{}; - const unsigned char* usedClustersPtrs[NLayers]{}; - const int* clustersIndexTablesPtrs[NLayers]{}; - const int* rofClustersPtrs[NLayers]{}; - for (int iLayer{0}; iLayer < NLayers; ++iLayer) { - clustersPtrs[iLayer] = mTimeFrame->getClusters()[iLayer].data(); - usedClustersPtrs[iLayer] = mTimeFrame->getUsedClusters(iLayer).data(); - clustersIndexTablesPtrs[iLayer] = mTimeFrame->getIndexTable(0, iLayer).data(); - rofClustersPtrs[iLayer] = mTimeFrame->getROFrameClusters(iLayer).data(); + if (static_cast(scratch.nextHypotheses.size()) < followCtx.maxHypotheses) { + scratch.nextHypotheses.resize(followCtx.maxHypotheses); } - const TrackFollowContext followCtx{ - &mTimeFrame->getIndexTableUtils(), - mTimeFrame->getROFMaskView(), - mTimeFrame->getROFOverlapTableView(), - clustersPtrs, usedClustersPtrs, clustersIndexTablesPtrs, rofClustersPtrs, - trkParams.LayerRadii.data(), trkParams.PhiBins, maxHypotheses, - trkParams.TrackFollowerNSigmaCutPhi, trkParams.TrackFollowerNSigmaCutZ}; const auto backup = internalTrack; auto best = internalTrack; @@ -768,6 +808,8 @@ void TrackerTraits::findRoads(const int iteration) unsortedClusters[iLayer] = mTimeFrame->getUnsortedClusters()[iLayer].data(); } const auto topology = mTimeFrame->getTrackingTopologyView(); + tbb::enumerable_thread_specific followerScratch{ + [mr = mMemoryPool.get()]() { return TrackFollowerScratch{mr}; }}; for (int startLevel{mTrkParams[iteration].CellsPerRoad()}; startLevel >= mTrkParams[iteration].CellMinimumLevel(); --startLevel) { const track::TrackSeedSelector seedFilter{constants::MaxTrackSeedQ2Pt, mTrkParams[iteration].MaxChi2NDF, startLevel, mTrkParams[iteration].MaxHoles, mTrkParams[iteration].getMinSeedingClusters(), mTrkParams[iteration].HoleLayerMask, mTrkParams[iteration].getNonSeedingLayerMask()}; @@ -775,33 +817,36 @@ void TrackerTraits::findRoads(const int iteration) bounded_vector trackSeeds(mMemoryPool.get()); for (int startCellTopologyId{0}; startCellTopologyId < topology.nCells; ++startCellTopologyId) { const int startLayer = topology.getCell(startCellTopologyId).hitLayerMask.last(); - if (!(mTrkParams[iteration].StartLayerMask.has(startLayer)) || mTimeFrame->getCells()[startCellTopologyId].empty()) { + if (!(mTrkParams[iteration].StartLayerMask.has(startLayer)) || + mTimeFrame->getCells()[startCellTopologyId].empty() || + topology.getMaxCellLevel(startCellTopologyId) < startLevel) { continue; } - bounded_vector lastCellId(mMemoryPool.get()), updatedCellId(mMemoryPool.get()); - bounded_vector lastCellTopologyId(mMemoryPool.get()), updatedCellTopologyId(mMemoryPool.get()); - bounded_vector lastCellSeed(mMemoryPool.get()), updatedCellSeed(mMemoryPool.get()); + bounded_vector lastSeeds(mMemoryPool.get()), updatedSeeds(mMemoryPool.get()); + + auto roadKey = [&](int level) { + return CapacityEstimator::makeKey(SlabSite::Roads, iteration, CapacityEstimator::makeVariant(startLevel, level), startCellTopologyId); + }; - processNeighbours(iteration, startCellTopologyId, startLevel, mTimeFrame->getCells()[startCellTopologyId], lastCellId, lastCellTopologyId, updatedCellSeed, updatedCellId, updatedCellTopologyId); + processNeighbours(iteration, startCellTopologyId, startLevel, roadKey(startLevel), mTimeFrame->getCells()[startCellTopologyId], updatedSeeds); int level = startLevel; - while (level > 2 && !updatedCellSeed.empty()) { - lastCellSeed.swap(updatedCellSeed); - lastCellId.swap(updatedCellId); - lastCellTopologyId.swap(updatedCellTopologyId); - deepVectorClear(updatedCellSeed); /// tame the memory peaks - deepVectorClear(updatedCellId); /// tame the memory peaks - deepVectorClear(updatedCellTopologyId); - processNeighbours(iteration, constants::UnusedIndex, --level, lastCellSeed, lastCellId, lastCellTopologyId, updatedCellSeed, updatedCellId, updatedCellTopologyId); + while (level > 2 && !updatedSeeds.empty()) { + lastSeeds.swap(updatedSeeds); + deepVectorClear(updatedSeeds); + --level; + processNeighbours(iteration, constants::UnusedIndex, level, roadKey(level), lastSeeds, updatedSeeds); } - deepVectorClear(lastCellId); /// tame the memory peaks - deepVectorClear(lastCellTopologyId); /// tame the memory peaks - deepVectorClear(lastCellSeed); /// tame the memory peaks + deepVectorClear(lastSeeds); - if (!updatedCellSeed.empty()) { - trackSeeds.reserve(trackSeeds.size() + std::count_if(updatedCellSeed.begin(), updatedCellSeed.end(), seedFilter)); - std::copy_if(updatedCellSeed.begin(), updatedCellSeed.end(), std::back_inserter(trackSeeds), seedFilter); + if (!updatedSeeds.empty()) { + trackSeeds.reserve(trackSeeds.size() + std::count_if(updatedSeeds.begin(), updatedSeeds.end(), [&](const auto& road) { return seedFilter(road.seed); })); + for (auto& road : updatedSeeds) { + if (seedFilter(road.seed)) { + trackSeeds.emplace_back(std::move(road.seed)); + } + } } } @@ -809,6 +854,25 @@ void TrackerTraits::findRoads(const int iteration) continue; } + const Cluster* clustersPtrs[NLayers]{}; + const unsigned char* usedClustersPtrs[NLayers]{}; + const int* clustersIndexTablesPtrs[NLayers]{}; + const int* rofClustersPtrs[NLayers]{}; + for (int iLayer{0}; iLayer < NLayers; ++iLayer) { + clustersPtrs[iLayer] = mTimeFrame->getClusters()[iLayer].data(); + usedClustersPtrs[iLayer] = mTimeFrame->getUsedClusters(iLayer).data(); + clustersIndexTablesPtrs[iLayer] = mTimeFrame->getIndexTable(0, iLayer).data(); + rofClustersPtrs[iLayer] = mTimeFrame->getROFrameClusters(iLayer).data(); + } + const TrackFollowContext followCtx{ + &mTimeFrame->getIndexTableUtils(), + mTimeFrame->getROFMaskView(), + mTimeFrame->getROFOverlapTableView(), + clustersPtrs, usedClustersPtrs, clustersIndexTablesPtrs, rofClustersPtrs, + mTrkParams[iteration].LayerRadii.data(), mTrkParams[iteration].PhiBins, + std::max(1, mTrkParams[iteration].TrackFollowerMaxHypotheses), + mTrkParams[iteration].TrackFollowerNSigmaCutPhi, mTrkParams[iteration].TrackFollowerNSigmaCutZ}; + bounded_vector tracks(mMemoryPool.get()); mTaskArena->execute([&] { const int nSeeds = static_cast(trackSeeds.size()); @@ -830,10 +894,11 @@ void TrackerTraits::findRoads(const int iteration) tbb::parallel_for(tbb::blocked_range(0, nSeeds, chunkSize), [&](const auto& range) { bounded_vector localTracks(mMemoryPool.get()); localTracks.reserve(std::min(chunkSize, static_cast(range.size()))); + auto& scratch = followerScratch.local(); for (int iSeed{range.begin()}; iSeed < range.end(); ++iSeed) { - TrackITSExt temporaryTrack; - if (finaliseTrackSeed(trackSeeds[iSeed], temporaryTrack, iteration, tfInfos, unsortedClusters, propagator)) { - localTracks.push_back(temporaryTrack); + localTracks.emplace_back(); + if (!finaliseTrackSeed(trackSeeds[iSeed], localTracks.back(), iteration, tfInfos, unsortedClusters, propagator, followCtx, scratch)) { + localTracks.pop_back(); } if (static_cast(localTracks.size()) == chunkSize) { flushTracks(localTracks); @@ -1010,16 +1075,16 @@ void TrackerTraits::setNThreads(int n, std::shared_ptr } template class TrackerTraits<7>; -template void TrackerTraits<7>::processNeighbours(int, int, int, const bounded_vector&, const bounded_vector&, const bounded_vector&, bounded_vector>&, bounded_vector&, bounded_vector&); -template void TrackerTraits<7>::processNeighbours>(int, int, int, const bounded_vector>&, const bounded_vector&, const bounded_vector&, bounded_vector>&, bounded_vector&, bounded_vector&); +template void TrackerTraits<7>::processNeighbours(int, int, int, uint64_t, const bounded_vector&, bounded_vector>&); +template void TrackerTraits<7>::processNeighbours>(int, int, int, uint64_t, const bounded_vector>&, bounded_vector>&); // ALICE3 upgrade #ifdef ENABLE_UPGRADES template class TrackerTraits<11>; -template void TrackerTraits<11>::processNeighbours(int, int, int, const bounded_vector&, const bounded_vector&, const bounded_vector&, bounded_vector>&, bounded_vector&, bounded_vector&); -template void TrackerTraits<11>::processNeighbours>(int, int, int, const bounded_vector>&, const bounded_vector&, const bounded_vector&, bounded_vector>&, bounded_vector&, bounded_vector&); +template void TrackerTraits<11>::processNeighbours(int, int, int, uint64_t, const bounded_vector&, bounded_vector>&); +template void TrackerTraits<11>::processNeighbours>(int, int, int, uint64_t, const bounded_vector>&, bounded_vector>&); template class TrackerTraits<13>; -template void TrackerTraits<13>::processNeighbours(int, int, int, const bounded_vector&, const bounded_vector&, const bounded_vector&, bounded_vector>&, bounded_vector&, bounded_vector&); -template void TrackerTraits<13>::processNeighbours>(int, int, int, const bounded_vector>&, const bounded_vector&, const bounded_vector&, bounded_vector>&, bounded_vector&, bounded_vector&); +template void TrackerTraits<13>::processNeighbours(int, int, int, uint64_t, const bounded_vector&, bounded_vector>&); +template void TrackerTraits<13>::processNeighbours>(int, int, int, uint64_t, const bounded_vector>&, bounded_vector>&); #endif } // namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx index 3f98e146996cf..83a1086ec5263 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx @@ -472,6 +472,7 @@ void ITSTrackingInterface::printSummary() const { mVertexer->printSummary(); mTracker->printSummary(); + mTimeFrame->getCapacityEstimator().print(); } void ITSTrackingInterface::setTraitsFromProvider(VertexerTraitsN* vertexerTraits, diff --git a/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx b/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx index ba37275f87688..d25d5efbec262 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx @@ -158,7 +158,8 @@ void Vertexer::addTimingStatCurStep(int iteration, double timeMs) template void Vertexer::printSummary() const { - LOGP(info, "Vertexer summary: Processed {} TFs", mTimeFrameCounter); + auto avgTF = mTotalTime * 1.e-3 / ((mTimeFrameCounter > 0) ? (double)mTimeFrameCounter : -1.0); + LOGP(info, "Vertexer summary: Processed {} TFs in TOT={:.2f} s, AVG/TF={:.2f} s", mTimeFrameCounter, mTotalTime * 1.e-3, avgTF); for (size_t iteration = 0; iteration < mTimingStats.size(); ++iteration) { for (size_t state = 0; state < NSteps; ++state) { const auto& stats = mTimingStats[iteration][state]; diff --git a/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt b/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt index f8fce10b78602..c7c4d6dc101a2 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt +++ b/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt @@ -15,6 +15,12 @@ o2_add_test(boundedmemoryresource LABELS "its;tracking" PUBLIC_LINK_LIBRARIES O2::ITStracking) +o2_add_test(slabbumpallocator + SOURCES testSlabBumpAllocator.cxx + COMPONENT_NAME its-tracking + LABELS "its;tracking" + PUBLIC_LINK_LIBRARIES O2::ITStracking TBB::tbb) + o2_add_test(roflookuptables SOURCES testROFLookupTables.cxx COMPONENT_NAME its-tracking diff --git a/Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx b/Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx new file mode 100644 index 0000000000000..9cae6fd11a132 --- /dev/null +++ b/Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx @@ -0,0 +1,526 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#define BOOST_TEST_MODULE Test SlabBumpAllocator +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ITStracking/BoundedAllocator.h" +#include "ITStracking/CapacityEstimator.h" +#include "ITStracking/SlabBumpAllocator.h" + +using namespace o2::its; + +namespace +{ + +struct Rec { + int a{-1}; + int b{-1}; + float payload{0.f}; + Rec() = default; + Rec(int aa, int bb, float p) : a{aa}, b{bb}, payload{p} {} + bool operator<(const Rec& o) const + { + if ((a < 0) != (o.a < 0)) { + return o.a < 0; + } + return a != o.a ? a < o.a : b < o.b; + } + bool operator==(const Rec& o) const { return a == o.a && b == o.b; } +}; + +std::ostream& operator<<(std::ostream& os, const Rec& r) +{ + return os << "Rec{" << r.a << ',' << r.b << ',' << r.payload << '}'; +} + +class StingyResource final : public std::pmr::memory_resource +{ + public: + explicit StingyResource(size_t maxBytes) : mMax{maxBytes} {} + + private: + void* do_allocate(size_t bytes, size_t alignment) final + { + if (bytes > mMax) { + throw std::bad_alloc{}; + } + return std::pmr::new_delete_resource()->allocate(bytes, alignment); + } + void do_deallocate(void* p, size_t bytes, size_t alignment) final + { + std::pmr::new_delete_resource()->deallocate(p, bytes, alignment); + } + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept final { return this == &other; } + + size_t mMax; +}; + +template +void runConcurrently(F&& f) +{ + tbb::task_arena arena{4}; + arena.execute(std::forward(f)); +} + +template +void produce(int i, uint32_t seed, Emit&& emit) +{ + std::mt19937 rng(seed + (uint32_t(i) * 2654435761u)); + const int n = int(rng() % 12); + for (int k = 0; k < n; ++k) { + emit(i, k, float((i * 100) + k)); + } +} + +std::vector> reference(int nProducers, uint32_t seed) +{ + std::vector> out(nProducers); + for (int i = 0; i < nProducers; ++i) { + produce(i, seed, [&](int a, int b, float p) { out[i].emplace_back(a, b, p); }); + } + return out; +} + +void checkGrouped(int nProducers, size_t capacity, size_t slab, size_t maxMemory = std::numeric_limits::max()) +{ + constexpr uint32_t seed = 7u; + BoundedMemoryResource mr{maxMemory}; + + const auto ref = reference(nProducers, seed); + std::vector flat; + std::vector refLut(nProducers + 1, 0); + for (int i = 0; i < nProducers; ++i) { + refLut[i + 1] = refLut[i] + int(ref[i].size()); + flat.insert(flat.end(), ref[i].begin(), ref[i].end()); + } + + GroupedSlabSink sink{{.capacity = capacity, .nThreads = 4, .slabOverride = slab}, &mr}; + runConcurrently([&] { + tbb::parallel_for(0, nProducers, [&](int i) { + auto& h = sink.local(); + h.beginProducer(i); + produce(i, seed, [&](int a, int b, float p) { h.emplace(a, b, p); }); + }); + }); + + const auto st = sink.stats(); + BOOST_TEST(st.emitted == flat.size()); + + bounded_vector lut{&mr}; + bounded_vector dest{&mr}; + sink.finalizeGrouped(size_t(nProducers), lut, dest); + + BOOST_REQUIRE(lut.size() == size_t(nProducers) + 1); + BOOST_TEST(std::equal(lut.begin(), lut.end(), refLut.begin())); + BOOST_REQUIRE(dest.size() == flat.size()); + for (size_t i = 0; i < flat.size(); ++i) { + BOOST_TEST(dest[i] == flat[i]); + BOOST_TEST(dest[i].payload == flat[i].payload); + } +} + +void checkUnordered(int nProducers, size_t capacity, size_t slab, size_t maxMemory = std::numeric_limits::max()) +{ + constexpr uint32_t seed = 11u; + BoundedMemoryResource mr{maxMemory}; + + const auto ref = reference(nProducers, seed); + std::vector flat; + for (const auto& v : ref) { + flat.insert(flat.end(), v.begin(), v.end()); + } + std::sort(flat.begin(), flat.end()); + flat.erase(std::unique(flat.begin(), flat.end()), flat.end()); + + UnorderedSlabSink sink{{.capacity = capacity, .nThreads = 4, .slabOverride = slab}, &mr}; + runConcurrently([&] { + tbb::parallel_for(0, nProducers, [&](int i) { + auto& h = sink.local(); + produce(i, seed, [&](int a, int b, float p) { h.emplace(a, b, p); }); + }); + }); + + const auto st = sink.stats(); + BOOST_TEST(st.emitted == flat.size()); + + bounded_vector dest{&mr}; + sink.finalizeUnordered(dest); + + std::sort(dest.begin(), dest.end()); + + BOOST_REQUIRE(dest.size() == flat.size()); + for (size_t i = 0; i < flat.size(); ++i) { + BOOST_TEST(dest[i] == flat[i]); + BOOST_TEST(dest[i].payload == flat[i].payload); + } +} + +} // namespace + +BOOST_AUTO_TEST_CASE(slab_hands_out_disjoint_ranges) +{ + SlabBumpAllocator alloc{1000, 256}; + std::vector seen(1000, 0); + size_t got{0}; + while (true) { + const auto r = alloc.grab(); + if (!r.valid()) { + break; + } + BOOST_REQUIRE(r.base + r.n <= 1000); + for (size_t s = r.base; s < r.base + r.n; ++s) { + BOOST_REQUIRE(seen[s] == 0); + seen[s] = 1; + } + got += r.n; + } + BOOST_TEST(got == 1000u); + BOOST_TEST(alloc.watermark() <= 1000u); +} + +BOOST_AUTO_TEST_CASE(slab_never_exceeds_a_threads_fair_share) +{ + BOOST_TEST(SlabBumpAllocator::suggestSlab(64, 8) <= 8u); + BOOST_TEST(SlabBumpAllocator::suggestSlab(0, 8) >= 1u); + BOOST_TEST(SlabBumpAllocator::suggestSlab(1u << 20, 8) == 4096u); +} + +BOOST_AUTO_TEST_CASE(grouped_reproduces_two_pass_layout) +{ + checkGrouped(2000, 40000, 512); + checkGrouped(300, 20000, 4096); +} + +BOOST_AUTO_TEST_CASE(grouped_survives_capacity_underestimate) +{ + checkGrouped(2000, 3000, 256); + checkGrouped(500, 0, 1, 1u << 20); +} + +BOOST_AUTO_TEST_CASE(grouped_survives_capacity_overestimate) +{ + checkGrouped(20, 1u << 20, 256, 1u << 16); +} + +BOOST_AUTO_TEST_CASE(grouped_keeps_order_across_slab_and_spill_boundaries) +{ + BoundedMemoryResource mr; + const std::vector counts{3, 5, 6, 0, 2}; + GroupedSlabSink sink{{.capacity = 10, .nThreads = 1, .slabOverride = 4}, &mr}; + + auto& h = sink.local(); + for (size_t p = 0; p < counts.size(); ++p) { + h.beginProducer(int(p)); + for (int k = 0; k < counts[p]; ++k) { + h.emplace(int(p), k, float(k)); + } + } + const auto st = sink.stats(); + BOOST_TEST(st.emitted == 16u); + BOOST_TEST(st.spilled == 6u); // capacity 10 of 16 + BOOST_TEST(st.overflowed); + + bounded_vector lut{&mr}; + bounded_vector dest{&mr}; + sink.finalizeGrouped(counts.size(), lut, dest); + + BOOST_REQUIRE(lut.size() == counts.size() + 1); + BOOST_REQUIRE(dest.size() == 16u); + int expected{0}; + for (size_t p = 0; p < counts.size(); ++p) { + BOOST_TEST(lut[p] == expected); + for (int k = 0; k < counts[p]; ++k) { + BOOST_TEST(dest[expected + k] == Rec(int(p), k, 0.f)); + } + expected += counts[p]; + } + BOOST_TEST(lut.back() == expected); +} + +BOOST_AUTO_TEST_CASE(unordered_reproduces_emitted_records) +{ + checkUnordered(2000, 40000, 512); + checkUnordered(300, 20000, 4096); +} + +BOOST_AUTO_TEST_CASE(unordered_survives_capacity_underestimate) +{ + checkUnordered(2000, 3000, 256); + checkUnordered(500, 0, 1, 1u << 20); +} + +BOOST_AUTO_TEST_CASE(unordered_keeps_records_across_slab_and_spill_boundaries) +{ + BoundedMemoryResource mr; + UnorderedSlabSink sink{{.capacity = 10, .nThreads = 1, .slabOverride = 4}, &mr}; + + auto& h = sink.local(); + for (int i = 0; i < 14; ++i) { + h.emplace(i, i + 1, float(i)); + } + const auto st = sink.stats(); + BOOST_TEST(st.emitted == 14u); + BOOST_TEST(st.spilled == 4u); + + bounded_vector dest{&mr}; + sink.finalizeUnordered(dest); + + BOOST_REQUIRE(dest.size() == 14u); + for (int i = 0; i < 14; ++i) { + BOOST_TEST(dest[i] == Rec(i, i + 1, float(i))); + } +} + +BOOST_AUTO_TEST_CASE(unordered_removes_unused_slots) +{ + BoundedMemoryResource mr; + UnorderedSlabSink sink{{.capacity = 10, .nThreads = 1, .slabOverride = 4}, &mr}; + sink.local().emplace(1, 2, 3.f); + sink.local().emplace(); + + bounded_vector dest{&mr}; + sink.finalizeUnordered(dest); + + BOOST_REQUIRE(dest.size() == 2u); + BOOST_TEST(dest.front() == Rec(1, 2, 3.f)); + BOOST_TEST(dest.front().payload == 3.f); + BOOST_TEST(dest.back() == Rec{}); +} + +BOOST_AUTO_TEST_CASE(unordered_does_not_hand_back_an_oversized_buffer) +{ + BoundedMemoryResource mr; + UnorderedSlabSink sink{{.capacity = 100000, .nThreads = 1, .slabOverride = 256}, &mr}; + + auto& h = sink.local(); + for (int i = 0; i < 100; ++i) { + h.emplace(i, i + 1, float(i)); + } + bounded_vector dest{&mr}; + sink.finalizeUnordered(dest); + + BOOST_REQUIRE(dest.size() == 100u); + BOOST_TEST(dest.capacity() < 1000u); +} + +BOOST_AUTO_TEST_CASE(capacity_is_clamped_to_what_the_pool_can_spare) +{ + constexpr size_t maxMemory = 1u << 16; + BoundedMemoryResource mr{maxMemory}; + UnorderedSlabSink sink{{.capacity = 1u << 20, .nThreads = 4}, &mr}; + + const auto st = sink.stats(); + BOOST_TEST(st.requested == size_t{1u << 20}); + BOOST_TEST(st.capacity > 0u); + BOOST_TEST(st.capacity < st.requested); + BOOST_TEST(st.memoryLimited); + BOOST_TEST(st.capacity * sizeof(Rec) <= maxMemory / 2); +} + +BOOST_AUTO_TEST_CASE(capacity_is_split_between_concurrent_sinks) +{ + size_t alone{0}, shared{0}; + { + BoundedMemoryResource mr{1u << 16}; + UnorderedSlabSink sink{{.capacity = 1u << 20, .nThreads = 4, .nConcurrentSinks = 1}, &mr}; + alone = sink.stats().capacity; + } + { + BoundedMemoryResource mr{1u << 16}; + UnorderedSlabSink sink{{.capacity = 1u << 20, .nThreads = 4, .nConcurrentSinks = 4}, &mr}; + shared = sink.stats().capacity; + } + BOOST_TEST(shared > 0u); + BOOST_TEST(shared < alone); + BOOST_TEST(shared * 4 <= alone + 8); // integer division slack +} + +BOOST_AUTO_TEST_CASE(unordered_survives_a_failed_preallocation) +{ + StingyResource mr{1u << 12}; + UnorderedSlabSink sink{{.capacity = 1u << 20, .nThreads = 1}, &mr}; + + const auto st = sink.stats(); + BOOST_TEST(st.capacity == 0u); + BOOST_TEST(st.memoryLimited); + + auto& handle = sink.local(); + for (int i = 0; i < 10; ++i) { + handle.emplace(i, i + 1, float(i)); + } + + bounded_vector dest{&mr}; + sink.finalizeUnordered(dest); + BOOST_REQUIRE(dest.size() == 10u); + for (int i = 0; i < 10; ++i) { + BOOST_TEST(dest[i] == Rec(i, i + 1, float(i))); + } +} + +BOOST_AUTO_TEST_CASE(estimator_cold_start_has_capacity) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Cells, 0, 0, 3); + BOOST_TEST(est.capacity(key, 1000.) == 1024u); + + est.update(key, 0., 0, 0, false, false); + BOOST_TEST(est.capacity(key, 0.) == 0u); + BOOST_TEST(est.capacity(key, 1000.) == 1024u); + + est.update(key, 1000., 0, 1024, false, false); + BOOST_TEST(est.capacity(key, 1000.) == 1024u); +} + +BOOST_AUTO_TEST_CASE(estimator_converges_and_reacts_to_overflow) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Tracklets, 1, 0, 0); + constexpr double scale = 1000.; + constexpr double rate = 5.; + + for (int tf = 0; tf < 12; ++tf) { + const size_t cap = est.capacity(key, scale); + const auto emitted = size_t(scale * rate); + est.update(key, scale, emitted, cap != 0 ? cap : emitted, cap != 0 && emitted > cap, false); + } + + const size_t cap = est.capacity(key, scale); + BOOST_TEST(cap >= size_t(scale * rate)); + BOOST_TEST(cap <= size_t(scale * rate * 1.35)); + + const size_t bigger = est.capacity(key, 2. * scale); + BOOST_TEST(bigger > size_t(2. * scale * rate)); + BOOST_TEST(bigger <= size_t(2. * scale * rate * 1.35)); + + est.update(key, scale, size_t(scale * rate * 4.), size_t(scale * rate), true, false); + BOOST_TEST(est.capacity(key, scale) > cap); +} + +BOOST_AUTO_TEST_CASE(estimator_backs_off_when_the_pool_refuses) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Roads, 2, 0, 0); + constexpr double scale = 1000.; + constexpr double rate = 5.; + const auto emitted = size_t(scale * rate); + + for (int tf = 0; tf < 12; ++tf) { + const size_t cap = est.capacity(key, scale); + est.update(key, scale, emitted, cap, emitted > cap, false); + } + const size_t settled = est.capacity(key, scale); + + for (int tf = 0; tf < 12; ++tf) { + est.update(key, scale, emitted, 100, true, true); + } + BOOST_TEST(est.capacity(key, scale) < settled); +} + +BOOST_AUTO_TEST_CASE(estimator_grows_in_proportion_to_the_miss) +{ + CapacityEstimator est; + constexpr double scale = 1000.; + const auto nearMiss = CapacityEstimator::makeKey(SlabSite::Cells, 3, 0, 0); + const auto wayOff = CapacityEstimator::makeKey(SlabSite::Cells, 3, 0, 1); + + for (const auto key : {nearMiss, wayOff}) { + est.update(key, scale, 2000, 2000, false, false); + } + const size_t settled = est.capacity(nearMiss, scale); + + est.update(nearMiss, scale, 2000, 1900, true, false); // overran by 5% + est.update(wayOff, scale, 2000, 500, true, false); // overran by 4x + + const size_t afterNearMiss = est.capacity(nearMiss, scale); + const size_t afterWayOff = est.capacity(wayOff, scale); + BOOST_TEST(afterNearMiss > settled); + BOOST_TEST(afterNearMiss < afterWayOff); + BOOST_TEST(afterNearMiss < size_t(1.25 * double(settled))); + BOOST_TEST(afterWayOff > size_t(1.4 * double(settled))); +} + +BOOST_AUTO_TEST_CASE(estimator_recovers_from_a_single_overflow) +{ + CapacityEstimator::Config cfg; + cfg.decayAfter = 1; + CapacityEstimator est{cfg}; + const auto key = CapacityEstimator::makeKey(SlabSite::Tracklets, 4, 0, 0); + constexpr double scale = 1000.; + + est.update(key, scale, 2000, 2000, false, false); + est.update(key, scale, 2000, 500, true, false); + const size_t inflated = est.capacity(key, scale); + + for (int tf = 0; tf < 30; ++tf) { + est.update(key, scale, 2000, 20000, false, false); // 10% utilisation + } + const size_t recovered = est.capacity(key, scale); + BOOST_TEST(recovered < inflated); + BOOST_TEST(recovered <= size_t(2. * scale * double(cfg.marginMin)) + 2); +} + +BOOST_AUTO_TEST_CASE(estimator_decay_survives_interleaved_busy_timeframes) +{ + CapacityEstimator::Config cfg; + cfg.decayAfter = 4; + CapacityEstimator est{cfg}; + const auto key = CapacityEstimator::makeKey(SlabSite::Roads, 5, 0, 0); + constexpr double scale = 1000.; + + est.update(key, scale, 2000, 2000, false, false); + est.update(key, scale, 2000, 500, true, false); + const size_t inflated = est.capacity(key, scale); + + for (int tf = 0; tf < 80; ++tf) { + const bool quiet = (tf % 4) != 3; + est.update(key, scale, 2000, quiet ? 20000 : 2000, false, false); + } + BOOST_TEST(est.capacity(key, scale) < inflated); +} + +BOOST_AUTO_TEST_CASE(estimator_reset_forgets_inflated_margins) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Cells, 0, 0, 0); + constexpr double scale = 1000.; + + for (int tf = 0; tf < 6; ++tf) { + est.update(key, scale, size_t(scale * 5.), 10, true, false); + } + BOOST_TEST(est.capacity(key, scale) > 5000u); + + est.reset(); + BOOST_TEST(est.capacity(key, scale) == 1024u); +} + +BOOST_AUTO_TEST_CASE(estimator_keys_separate_the_road_walk_steps) +{ + const auto a = CapacityEstimator::makeKey(SlabSite::Roads, 0, CapacityEstimator::makeVariant(6, 4), 1); + const auto b = CapacityEstimator::makeKey(SlabSite::Roads, 0, CapacityEstimator::makeVariant(5, 4), 1); + const auto c = CapacityEstimator::makeKey(SlabSite::Roads, 0, CapacityEstimator::makeVariant(6, 4), 2); + BOOST_TEST(a != b); + BOOST_TEST(a != c); + BOOST_TEST(b != c); +} diff --git a/Detectors/ITSMFT/ITS/tracking/test/testTrackingTopology.cxx b/Detectors/ITSMFT/ITS/tracking/test/testTrackingTopology.cxx index d3f249650a287..6c76bcd193ec8 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/testTrackingTopology.cxx +++ b/Detectors/ITSMFT/ITS/tracking/test/testTrackingTopology.cxx @@ -72,6 +72,65 @@ BOOST_AUTO_TEST_CASE(trackingtopology_basic) } } +/// Without holes the cell graph is a single chain, so cell i - spanning layers i, i+1, i+2 - +/// can only ever be reached by the i cells below it. +BOOST_AUTO_TEST_CASE(trackingtopology_max_cell_level_is_the_chain_depth) +{ + o2::its::TrackingTopology<7> topo; + topo.init(7, 0, 0); + const auto view = topo.getView(); + view.print(); + + BOOST_REQUIRE_EQUAL(view.nLinks, 6); + BOOST_REQUIRE_EQUAL(view.nCells, 5); + for (int i{0}; i < view.nCells; ++i) { + BOOST_CHECK_EQUAL(int(view.getMaxCellLevel(i)), i + 1); + } +} + +/// With a hole allowed the graph branches, and the depth is the longest path ending on a cell +/// rather than its index. Every cell must still be reachable by at least one chain, and no cell +/// may claim a level deeper than the number of cells that could precede it. +BOOST_AUTO_TEST_CASE(trackingtopology_max_cell_level_follows_the_longest_path) +{ + o2::its::TrackingTopology<5> topo; + topo.init(5, 1, 1 << 2); + const auto view = topo.getView(); + view.print(); + + bool sawBranching = false; + for (int i{0}; i < view.nCells; ++i) { + const auto level = int(view.getMaxCellLevel(i)); + BOOST_CHECK_GE(level, 1); + BOOST_CHECK_LE(level, int(view.nCells)); + // A cell reached by a chain of n predecessors needs n+2 layers below its outer one. + BOOST_CHECK_LE(level, view.getCell(i).hitLayerMask.last() - 1); + sawBranching |= level != i + 1; + } + BOOST_CHECK(sawBranching); // otherwise this is just the chain case again +} + +/// Neighbour construction can finalize a target after one source-layer wave: every predecessor +/// of a target ends on the destination layer of the target's first link. +BOOST_AUTO_TEST_CASE(trackingtopology_predecessors_belong_to_one_layer_wave) +{ + o2::its::TrackingTopology<7> topo; + topo.init(7, 2, (1 << 2) | (1 << 4)); + const auto view = topo.getView(); + + for (int sourceId{0}; sourceId < view.nCells; ++sourceId) { + const auto& source = view.getCell(sourceId); + const int sourceWave = source.hitLayerMask.last(); + const auto successors = view.getCellsStartingWithLink(source.secondLink); + for (int i{0}; i < successors.getEntries(); ++i) { + const int targetId = view.cellsByFirstLink[successors.getFirstEntry() + i]; + const auto& target = view.getCell(targetId); + BOOST_CHECK_EQUAL(target.firstLink, source.secondLink); + BOOST_CHECK_EQUAL(sourceWave, view.getLink(target.firstLink).toLayer); + } + } +} + BOOST_AUTO_TEST_CASE(trackingtopology_single_allowed_hole) { o2::its::TrackingTopology<5> topo; From 1c38a93a9363f632da8157937b9acbc4bcbd8ede Mon Sep 17 00:00:00 2001 From: Felix Schlepper Date: Sat, 15 Aug 2026 08:26:36 +0200 Subject: [PATCH 2/2] ITSGPU: use size estimates Signed-off-by: Felix Schlepper --- .../GPU/ITStrackingGPU/LaunchGeometry.h | 68 + .../GPU/ITStrackingGPU/TimeFrameGPU.h | 128 +- .../GPU/ITStrackingGPU/TrackerTraitsGPU.h | 3 - .../GPU/ITStrackingGPU/TrackingKernels.h | 349 ++-- .../ITS/tracking/GPU/ITStrackingGPU/Utils.h | 95 - .../ITS/tracking/GPU/cuda/CMakeLists.txt | 3 + .../ITS/tracking/GPU/cuda/TimeFrameGPU.cu | 724 +++---- .../tracking/GPU/cuda/TrackerTraitsGPU.cxx | 438 ++-- .../ITS/tracking/GPU/cuda/TrackingKernels.cu | 1802 +++++------------ .../ITS/tracking/GPU/hip/CMakeLists.txt | 4 + .../include/ITStracking/CapacityEstimator.h | 28 +- .../tracking/include/ITStracking/Constants.h | 27 +- .../include/ITStracking/ExternalAllocator.h | 1 + .../ITS/tracking/src/CapacityEstimator.cxx | 42 +- Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx | 15 +- .../tracking/test/testSlabBumpAllocator.cxx | 65 + .../include/GPUWorkflow/GPUWorkflowSpec.h | 1 + GPU/Workflow/src/GPUWorkflowITS.cxx | 5 + GPU/Workflow/src/GPUWorkflowSpec.cxx | 9 + cmake/O2AddHipifiedExecutable.cmake | 5 + cmake/O2AddHipifiedLibrary.cmake | 5 + 21 files changed, 1487 insertions(+), 2330 deletions(-) create mode 100644 Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/LaunchGeometry.h diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/LaunchGeometry.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/LaunchGeometry.h new file mode 100644 index 0000000000000..078f78dc49e76 --- /dev/null +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/LaunchGeometry.h @@ -0,0 +1,68 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +/// +/// \file LaunchGeometry.h +/// \brief Compile-time launch geometry of the ITS tracking kernels, per GPU family. +/// + +#ifndef ITSTRACKINGGPU_LAUNCHGEOMETRY_H_ +#define ITSTRACKINGGPU_LAUNCHGEOMETRY_H_ + +namespace o2::its::gpu +{ + +#if defined(GPUCA_GPUTYPE_VEGA) // gfx906: MI50, Radeon VII +constexpr int ComputeUnits = 60; +constexpr int WarpSize = 64; +#elif defined(GPUCA_GPUTYPE_MI100) // gfx908 +constexpr int ComputeUnits = 120; +constexpr int WarpSize = 64; +#elif defined(GPUCA_GPUTYPE_MI210) // gfx90a +constexpr int ComputeUnits = 104; +constexpr int WarpSize = 64; +#elif defined(GPUCA_GPUTYPE_MI300) // gfx942: MI300X (MI300A has 228) +constexpr int ComputeUnits = 304; +constexpr int WarpSize = 64; +#elif defined(GPUCA_GPUTYPE_RDNA) // gfx10xx/11xx consumer parts, wave32 +constexpr int ComputeUnits = 60; +constexpr int WarpSize = 32; +#elif defined(GPUCA_GPUTYPE_BLACKWELL) // sm_120: RTX 5080 +constexpr int ComputeUnits = 84; +constexpr int WarpSize = 32; +#elif defined(GPUCA_GPUTYPE_HOPPER) // sm_90: H100 +constexpr int ComputeUnits = 132; +constexpr int WarpSize = 32; +#elif defined(GPUCA_GPUTYPE_ADA) // sm_89: RTX 4090 +constexpr int ComputeUnits = 128; +constexpr int WarpSize = 32; +#elif defined(GPUCA_GPUTYPE_AMPERE) // sm_80/86: A100 has 108, RTX 3090 has 82 +constexpr int ComputeUnits = 108; +constexpr int WarpSize = 32; +#elif defined(GPUCA_GPUTYPE_TURING) // sm_75: RTX 2080 Ti +constexpr int ComputeUnits = 68; +constexpr int WarpSize = 32; +#else +#warning "GPU architecture not available setting fallback" +constexpr int ComputeUnits = 60; +constexpr int WarpSize = 64; +#endif + +constexpr int GPUThreads = 256; +constexpr int BlocksPerComputeUnit = 4; +constexpr int GPUBlocks = ComputeUnits * BlocksPerComputeUnit; +constexpr int GPUThreadsTotal = GPUBlocks * GPUThreads; + +static_assert(GPUThreads % WarpSize == 0, "block size must be a whole number of warps/waves"); +static_assert(GPUBlocks > 0 && GPUThreads > 0, "degenerate launch geometry"); + +} // namespace o2::its::gpu + +#endif // ITSTRACKINGGPU_LAUNCHGEOMETRY_H_ diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h index 38de1e712108d..d6ad28ac6c361 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h @@ -45,68 +45,55 @@ class TimeFrameGPU : public TimeFrame /// Most relevant operations void pushMemoryStack(const int); void popMemoryStack(const int); - void registerHostMemory(const int); - void unregisterHostMemory(const int); + void unregisterHostMemory(); void initialise(const TrackingParameters&, int maxLayers); void initialise(const TrackingParameters&, int maxLayers, int iteration); void loadIndexTableUtils(); void loadTrackingTopologies(); void loadTrackingFrameInfoDevice(const int); - void createTrackingFrameInfoDeviceArray(); + void createTrackingFrameInfoDeviceArray(const int = NLayers); void loadUnsortedClustersDevice(const int); void createUnsortedClustersDeviceArray(const int = NLayers); void loadClustersDevice(const int); void createClustersDeviceArray(const int = NLayers); void loadClustersIndexTables(const int); - void createClustersIndexTablesArray(); + void createClustersIndexTablesArray(const int = NLayers); void createUsedClustersDevice(const int); void createUsedClustersDeviceArray(const int = NLayers); void loadUsedClustersDevice(); void loadROFrameClustersDevice(const int); - void createROFrameClustersDeviceArray(); + void createROFrameClustersDeviceArray(const int = NLayers); void loadROFCutMask(const int); void loadVertices(); void loadROFOverlapTable(); void loadROFVertexLookupTable(); - void updateROFVertexLookupTable(); + void uploadROFVertexLookupTable(); + void loadIterationParameters(const TrackingParameters&); /// void createTrackletsLUTDevice(bool, const int); void createTrackletsLUTDeviceArray(); - void loadTrackletsDevice(); - void loadTrackletsLUTDevice(); - void loadCellsDevice(); - void loadCellsLUTDevice(); - void loadTrackSeedsDevice(); - void loadTrackSeedsChi2Device(); - void loadTrackSeedsDevice(bounded_vector&); - void createTrackletsBuffers(const int); + void createTrackSeedsDevice(const size_t capacity); + void createTrackletsBuffers(const int, size_t capacity); void createTrackletsBuffersArray(); - void createCellsBuffers(const int); + void createCellsBuffers(const int, size_t capacity); void createCellsBuffersArray(); - void createCellsDevice(); void createCellsLUTDevice(const int); void createCellsLUTDeviceArray(); - void createNeighboursIndexTablesDevice(const int); - void createNeighboursDevice(const unsigned int layer); + void createNeighboursDevice(const unsigned int layer, size_t capacity); void createNeighboursLUTDevice(const int, const unsigned int); - void createTrackITSExtDevice(const size_t); + void createTrackITSExtDevice(const size_t capacity); + void createTrackITSExtHost(const size_t nTracks); void createTrackExtensionScratchDevice(const int nThreads, const int maxHypotheses); void downloadTrackITSExtDevice(); void downloadTrackIndicesDevice(); - void downloadCellsNeighboursDevice(std::vector>&, const int); - void downloadNeighboursLUTDevice(bounded_vector&, const int); - void downloadCellsDevice(); - void downloadCellsLUTDevice(); /// synchronization auto& getStream(const size_t stream) { return mGpuStreams[stream]; } auto& getStreams() { return mGpuStreams; } - void syncStream(const size_t stream); void syncStreams(const bool = true); void waitEvent(const int, const int); void recordEvent(const int); - void recordEvents(const int = 0, const int = NLayers); /// cleanup virtual void wipe() final; @@ -115,17 +102,16 @@ class TimeFrameGPU : public TimeFrame virtual bool isGPU() const noexcept final { return true; } virtual const char* getName() const noexcept override final { return "GPU"; } IndexTableUtilsN* getDeviceIndexTableUtils() { return mIndexTableUtilsDevice; } + const float* getDeviceLayerRadii() const { return mLayerRadiiDevice; } + const float* getDeviceMinPts() const { return mMinPtsDevice; } + const float* getDeviceLayerxX0() const { return mLayerxX0Device; } const auto getDeviceROFOverlapTableView() { return mDeviceROFOverlapTableView; } const auto getDeviceROFVertexLookupTableView() { return mDeviceROFVertexLookupTableView; } const auto getDeviceROFMaskTableView() { return mDeviceROFMaskTableView; } const auto getDeviceTrackingTopologyView() const { return mDeviceTrackingTopologyView; } - int* getDeviceROFramesClusters(const int layer) { return mROFramesClustersDevice[layer]; } auto& getTrackITSExt() { return mTrackITSExt; } auto& getTrackIndices() { return mTrackIndices; } Vertex* getDeviceVertices() { return mPrimaryVerticesDevice; } - int* getDeviceROFramesPV() { return mROFramesPVDevice; } - unsigned char* getDeviceUsedClusters(const int); - const o2::base::Propagator* getChainPropagator(); // Hybrid TrackITSExt* getDeviceTrackITSExt() { return mTrackITSExtDevice; } @@ -133,11 +119,8 @@ class TimeFrameGPU : public TimeFrame TrackExtensionHypothesis* getDeviceActiveTrackExtensionHypotheses() { return mActiveTrackExtensionHypothesesDevice; } TrackExtensionHypothesis* getDeviceNextTrackExtensionHypotheses() { return mNextTrackExtensionHypothesesDevice; } int* getDeviceNeighboursLUT(const int layer) { return mNeighboursLUTDevice[layer]; } - gsl::span getDeviceNeighboursLUTs() { return mNeighboursLUTDevice; } CellNeighbour** getDeviceArrayNeighbours() { return mNeighboursDeviceArray; } - std::array& getDeviceNeighboursAll() { return mNeighboursDevice; } CellNeighbour* getDeviceNeighbours(const int layer) { return mNeighboursDevice[layer]; } - TrackingFrameInfo* getDeviceTrackingFrameInfo(const int); const TrackingFrameInfo** getDeviceArrayTrackingFrameInfo() const { return mTrackingFrameInfoDeviceArray; } const Cluster** getDeviceArrayClusters() const { return mClustersDeviceArray; } const Cluster** getDeviceArrayUnsortedClusters() const { return mUnsortedClustersDeviceArray; } @@ -151,11 +134,9 @@ class TimeFrameGPU : public TimeFrame int** getDeviceArrayNeighboursCellLUT() const { return mNeighboursCellLUTDeviceArray; } CellSeed** getDeviceArrayCells() { return mCellsDeviceArray; } TrackSeedN* getDeviceTrackSeeds() { return mTrackSeedsDevice; } - int* getDeviceTrackSeedsLUT() { return mTrackSeedsLUTDevice; } + int* getDeviceTrackSeedIndices() { return mTrackSeedIndicesDevice; } + int* getDeviceTrackCounter() { return mTrackCounterDevice; } auto getNTrackSeeds() const { return mNTracks; } - o2::track::TrackParCovF** getDeviceArrayTrackSeeds() { return mCellSeedsDeviceArray; } - float** getDeviceArrayTrackSeedsChi2() { return mCellSeedsChi2DeviceArray; } - int* getDeviceNeighboursIndexTables(const int layer) { return mNeighboursIndexTablesDevice[layer]; } void setDevicePropagator(const o2::base::PropagatorImpl* p) final { this->mPropagatorDevice = p; } @@ -164,7 +145,6 @@ class TimeFrameGPU : public TimeFrame gsl::span getNCells() { return {mNCells.data(), static_cast::size_type>(this->mTrackingTopologyView.nCells)}; } auto& getArrayNCells() { return mNCells; } gsl::span getNNeighbours() { return {mNNeighbours.data(), static_cast::size_type>(this->mTrackingTopologyView.nCells)}; } - auto& getArrayNNeighbours() { return mNNeighbours; } // Host-available device getters gsl::span getDeviceTrackletsLUTs() { return mTrackletsLUTDevice; } @@ -178,8 +158,33 @@ class TimeFrameGPU : public TimeFrame size_t getNumberOfNeighbours() const final; private: - void allocMemAsync(void**, size_t, Stream&, bool, int32_t = o2::gpu::GPUMemoryResource::MEMORY_GPU); // Abstract owned and unowned memory allocations on specific stream - void allocMem(void**, size_t, bool, int32_t = o2::gpu::GPUMemoryResource::MEMORY_GPU); // Abstract owned and unowned memory allocations on default stream + enum class SlotInit { + Raw, ///< whatever the allocator handed back + Zero ///< cleared on the slot's stream + }; + + template + T* allocDevice(size_t n, int32_t type = o2::gpu::GPUMemoryResource::MEMORY_GPU); + template + T* allocDeviceAsync(size_t n, Stream&, int32_t type = o2::gpu::GPUMemoryResource::MEMORY_GPU); + template + SlotPtr* allocSlotArray(size_t n); + template + void copyToDevice(T* dst, const T* src, size_t n); + template + void copyFromDevice(T* dst, const T* src, size_t n); + template + void publishSlot(ArrayT deviceArray, int slot, T* const& devicePtr, Stream&); + template + T* createSlot(std::array& slots, ArrayT deviceArray, int slot, size_t n, const char* what, SlotInit init = SlotInit::Raw, int32_t type = o2::gpu::GPUMemoryResource::MEMORY_GPU); + template + void uploadSlot(std::array& slots, ArrayT deviceArray, int slot, const Container& host, const char* what); + template + void createPinnedSlotArray(ArrayT& deviceArray, std::array& slots, std::bitset& pinned); + template + void pinHostLayers(Layers& layers, std::bitset& pinned, int maxLayers); + template + typename Table::View uploadNavigationTable(const Table& table, const typename Table::View& hostView); // Host-available device buffer sizes std::array mNTracklets{}; @@ -187,7 +192,11 @@ class TimeFrameGPU : public TimeFrame std::array mNNeighbours{}; // Device pointers - IndexTableUtilsN* mIndexTableUtilsDevice; + IndexTableUtilsN* mIndexTableUtilsDevice{nullptr}; + float* mIterationParametersDevice{nullptr}; + const float* mLayerRadiiDevice{nullptr}; + const float* mMinPtsDevice{nullptr}; + const float* mLayerxX0Device{nullptr}; // device navigation views ROFOverlapTableN::View mDeviceROFOverlapTableView; ROFVertexLookupTableN::View mDeviceROFVertexLookupTableView; @@ -196,18 +205,17 @@ class TimeFrameGPU : public TimeFrame typename TrackingTopologyN::View mDeviceTrackingTopologyView; // Hybrid pref - Vertex* mPrimaryVerticesDevice; - int* mROFramesPVDevice; - std::array mClustersDevice; - std::array mUnsortedClustersDevice; - std::array mClustersIndexTablesDevice; - std::array mUsedClustersDevice; - std::array mROFramesClustersDevice; - const Cluster** mClustersDeviceArray; - const Cluster** mUnsortedClustersDeviceArray; - const int** mClustersIndexTablesDeviceArray; - uint8_t** mUsedClustersDeviceArray; - const int** mROFramesClustersDeviceArray; + Vertex* mPrimaryVerticesDevice{nullptr}; + std::array mClustersDevice{}; + std::array mUnsortedClustersDevice{}; + std::array mClustersIndexTablesDevice{}; + std::array mUsedClustersDevice{}; + std::array mROFramesClustersDevice{}; + const Cluster** mClustersDeviceArray{nullptr}; + const Cluster** mUnsortedClustersDeviceArray{nullptr}; + const int** mClustersIndexTablesDeviceArray{nullptr}; + uint8_t** mUsedClustersDeviceArray{nullptr}; + const int** mROFramesClustersDeviceArray{nullptr}; std::array mTrackletsDevice{}; std::array mTrackletsLUTDevice{}; std::array mCellsLUTDevice{}; @@ -218,24 +226,20 @@ class TimeFrameGPU : public TimeFrame int** mNeighboursCellLUTDeviceArray{nullptr}; int** mTrackletsLUTDeviceArray{nullptr}; std::array mCellsDevice{}; - CellSeed** mCellsDeviceArray; - std::array mNeighboursIndexTablesDevice{}; + CellSeed** mCellsDeviceArray{nullptr}; TrackSeedN* mTrackSeedsDevice{nullptr}; - int* mTrackSeedsLUTDevice{nullptr}; + int* mTrackSeedIndicesDevice{nullptr}; ///< which seed each emitted track was fitted from + int* mTrackCounterDevice{nullptr}; unsigned int mNTracks{0}; - std::array mCellSeedsDevice{}; - o2::track::TrackParCovF** mCellSeedsDeviceArray; - std::array mCellSeedsChi2Device{}; - float** mCellSeedsChi2DeviceArray; - TrackITSExt* mTrackITSExtDevice; + TrackITSExt* mTrackITSExtDevice{nullptr}; int* mTrackIndicesDevice{nullptr}; TrackExtensionHypothesis* mActiveTrackExtensionHypothesesDevice{nullptr}; TrackExtensionHypothesis* mNextTrackExtensionHypothesesDevice{nullptr}; std::array mNeighboursDevice{}; CellNeighbour** mNeighboursDeviceArray{nullptr}; - std::array mTrackingFrameInfoDevice; - const TrackingFrameInfo** mTrackingFrameInfoDeviceArray; + std::array mTrackingFrameInfoDevice{}; + const TrackingFrameInfo** mTrackingFrameInfoDeviceArray{nullptr}; // State Streams mGpuStreams; diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackerTraitsGPU.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackerTraitsGPU.h index 81d870c5b46c2..0d84662666632 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackerTraitsGPU.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackerTraitsGPU.h @@ -22,8 +22,6 @@ namespace o2::its template class TrackerTraitsGPU final : public TrackerTraits { - using typename TrackerTraits::IndexTableUtilsN; - public: TrackerTraitsGPU() = default; ~TrackerTraitsGPU() final = default; @@ -47,7 +45,6 @@ class TrackerTraitsGPU final : public TrackerTraits int getTFNumberOfCells() const override; private: - IndexTableUtilsN* mDeviceIndexTableUtils; gpu::TimeFrameGPU* mTimeFrameGPU; }; diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h index 34ac3e564e26d..b31d9400657c1 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h @@ -17,6 +17,7 @@ #include #include "ITStracking/BoundedAllocator.h" +#include "ITStracking/CapacityEstimator.h" #include "ITStracking/ROFLookupTables.h" #include "ITStracking/TrackingTopology.h" #include "ITStracking/TrackExtensionHypothesis.h" @@ -38,225 +39,145 @@ class TrackITSExt; class ExternalAllocator; template -void countTrackletsInROFsHandler(const IndexTableUtils* utils, - const typename ROFMaskTable::View& rofMask, - const int linkId, - const int fromLayer, - const int toLayer, - const typename ROFOverlapTable::View& rofOverlaps, - const typename ROFVertexLookupTable::View& vertexLUT, - const int vertexId, - const Vertex* vertices, - const int* rofPV, - const Cluster** clusters, - std::vector nClusters, - const int** ROFClusters, - const unsigned char** usedClusters, - const int** clustersIndexTables, - int** trackletsLUTs, - gsl::span trackletsLUTsHost, - const bool selectUPCVertices, - const float NSigmaCut, +struct TrackingKernels { + static int computeTrackletsInROFsHandler(const IndexTableUtils* utils, + const typename ROFMaskTable::View& rofMask, + const int linkId, + const int fromLayer, + const int toLayer, + const typename ROFOverlapTable::View& rofOverlaps, + const typename ROFVertexLookupTable::View& vertexLUT, + const int vertexId, + const Vertex* vertices, + const Cluster** clusters, + const std::vector& nClusters, + const int** ROFClusters, + const unsigned char** usedClusters, + const int** clustersIndexTables, + Tracklet** tracklets, + gsl::span spanTracklets, + gsl::span nTracklets, + const int capacity, + gsl::span trackletsLUTsHost, + const bool selectUPCVertices, + const float NSigmaCut, + const typename TrackingTopology::View topology, + bounded_vector& linkPhiCuts, + const float resolutionPV, + std::array& minR, + std::array& maxR, + bounded_vector& resolutions, + std::vector& radii, + bounded_vector& linkMSAngles, + o2::its::ExternalAllocator* alloc, + gpu::Streams& streams); + + static int computeCellsHandler(const Cluster** sortedClusters, + const Cluster** unsortedClusters, + const TrackingFrameInfo** tfInfo, + Tracklet** tracklets, + int** trackletsLUT, + const int nTracklets, + const int cellTopologyId, const typename TrackingTopology::View topology, - bounded_vector& linkPhiCuts, - const float resolutionPV, - std::array& minR, - std::array& maxR, - bounded_vector& resolutions, - std::vector& radii, - bounded_vector& linkMSAngles, + CellSeed* cells, + const int capacity, + int* cellsLUTsHost, + const float bz, + const float maxChi2ClusterAttachment, + const float cellDeltaTanLambdaSigma, + const float nSigmaCut, + const float* layerxX0, o2::its::ExternalAllocator* alloc, gpu::Streams& streams); -template -void computeTrackletsInROFsHandler(const IndexTableUtils* utils, - const typename ROFMaskTable::View& rofMask, - const int linkId, - const int fromLayer, - const int toLayer, - const typename ROFOverlapTable::View& rofOverlaps, - const typename ROFVertexLookupTable::View& vertexLUT, - const int vertexId, - const Vertex* vertices, - const int* rofPV, - const Cluster** clusters, - std::vector nClusters, - const int** ROFClusters, - const unsigned char** usedClusters, - const int** clustersIndexTables, - Tracklet** tracklets, - gsl::span spanTracklets, - gsl::span nTracklets, - int** trackletsLUTs, - gsl::span trackletsLUTsHost, - const bool selectUPCVertices, - const float NSigmaCut, - const typename TrackingTopology::View topology, - bounded_vector& linkPhiCuts, - const float resolutionPV, - std::array& minR, - std::array& maxR, - bounded_vector& resolutions, - std::vector& radii, - bounded_vector& linkMSAngles, - o2::its::ExternalAllocator* alloc, - gpu::Streams& streams); - -template -void countCellsHandler(const Cluster** sortedClusters, - const Cluster** unsortedClusters, - const TrackingFrameInfo** tfInfo, - Tracklet** tracklets, - int** trackletsLUT, - const int nTracklets, - const int cellTopologyId, - const typename TrackingTopology::View topology, - CellSeed* cells, - int** cellsLUTsDeviceArray, - int* cellsLUTsHost, - const float bz, - const float maxChi2ClusterAttachment, - const float cellDeltaTanLambdaSigma, - const float nSigmaCut, - const std::vector& layerxX0Host, - o2::its::ExternalAllocator* alloc, - gpu::Streams& streams); - -template -void computeCellsHandler(const Cluster** sortedClusters, - const Cluster** unsortedClusters, - const TrackingFrameInfo** tfInfo, - Tracklet** tracklets, - int** trackletsLUT, - const int nTracklets, - const int cellTopologyId, - const typename TrackingTopology::View topology, - CellSeed* cells, - int** cellsLUTsDeviceArray, - int* cellsLUTsHost, - const float bz, - const float maxChi2ClusterAttachment, - const float cellDeltaTanLambdaSigma, - const float nSigmaCut, - const std::vector& layerxX0Host, - gpu::Streams& streams); - -template -void countCellNeighboursHandler(CellSeed** cellsLayersDevice, - int* neighboursCursor, - int** cellsLUTs, - const int sourceCellTopologyId, - const int targetCellTopologyId, - const float maxChi2ClusterAttachment, - const float bz, - const unsigned int nCells, - gpu::Stream& stream); - -void scanCellNeighboursHandler(int* neighboursCursor, - int* neighboursLUT, - const unsigned int nCells, - o2::its::ExternalAllocator* alloc, - gpu::Stream& stream); - -template -void computeCellNeighboursHandler(CellSeed** cellsLayersDevice, - int* neighboursCursor, - int** cellsLUTs, - CellNeighbour* cellNeighbours, - const int sourceCellTopologyId, - const int targetCellTopologyId, - const float maxChi2ClusterAttachment, - const float bz, - const unsigned int nCells, + static void computeCellNeighboursHandler(CellSeed** cellsLayersDevice, + int** cellsLUTs, + CellNeighbour* cellNeighbours, + int* outputCounter, + const int capacity, + const int sourceCellTopologyId, + const int targetCellTopologyId, + const float maxChi2ClusterAttachment, + const float bz, + const unsigned int nCells, + gpu::Stream& stream); + + static void processNeighboursHandler(const int startLevel, + const int startCellTopologyId, + CellSeed** allCellSeeds, + CellSeed* currentCellSeeds, + const int* currentCellTopologyIds, + const int* currentCellIds, + const int* nCells, + const unsigned char** usedClusters, + CellNeighbour** neighbours, + int** neighboursDeviceLUTs, + const TrackingFrameInfo** foundTrackingFrameInfo, + TrackSeed* seedsDevice, + const int seedsCapacity, + int& seedsCursor, + CapacityEstimator& estimator, + const int iteration, + const float bz, + const float MaxChi2ClusterAttachment, + const float maxChi2NDF, + const int maxHoles, + const int minSeedingClusters, + const LayerMask holeLayerMask, + const LayerMask nonSeedingLayerMask, + const float* layerxX0, + const o2::base::Propagator* propagator, + const o2::base::PropagatorF::MatCorrType matCorrType, + o2::its::ExternalAllocator* alloc); + + static int computeTrackSeedHandler(TrackSeed* trackSeeds, + const TrackingFrameInfo** foundTrackingFrameInfo, + const Cluster** unsortedClusters, + const IndexTableUtils* utils, + const typename ROFMaskTable::View& rofMask, + const typename ROFOverlapTable::View& rofOverlaps, + const Cluster** clusters, + const unsigned char** usedClusters, + const int** clustersIndexTables, + const int** ROFClusters, + o2::its::TrackITSExt* tracks, + int* trackIndices, + int* trackSeedIndices, + int* outputCounter, + const int trackCapacity, + TrackExtensionHypothesis* activeHypotheses, + TrackExtensionHypothesis* nextHypotheses, + const float* layerRadii, + const float* minPts, + const float* layerxX0, + const unsigned int nSeeds, + const float Bz, + const float maxChi2ClusterAttachment, + const float maxChi2NDF, + const int reseedIfShorter, + const bool repeatRefitOut, + const bool shiftRefToCluster, + const int nLayers, + const int phiBins, + const int maxHypotheses, + const bool extendTop, + const bool extendBot, + const float nSigmaCutPhi, + const float nSigmaCutZ, + const o2::base::Propagator* propagator, + const o2::base::PropagatorF::MatCorrType matCorrType, + o2::its::ExternalAllocator* alloc); +}; + +void resetOutputCounterHandler(int* outputCounter, gpu::Stream& stream); + +int finalizeCellNeighboursHandler(CellNeighbour* cellNeighbours, + int* neighboursLUT, + const int nTargetCells, + const int capacity, + o2::its::ExternalAllocator* alloc, gpu::Stream& stream); -int filterCellNeighboursHandler(gpuPair*, - int*, - unsigned int, - gpu::Stream&, - o2::its::ExternalAllocator* = nullptr); - -template -void processNeighboursHandler(const int startLevel, - const int defaultCellTopologyId, - CellSeed** allCellSeeds, - CellSeed* currentCellSeeds, - const int* currentCellTopologyIds, - const int* currentCellIds, - const int* nCells, - const unsigned char** usedClusters, - CellNeighbour** neighbours, - int** neighboursDeviceLUTs, - const TrackingFrameInfo** foundTrackingFrameInfo, - bounded_vector>& seedsHost, - const float bz, - const float MaxChi2ClusterAttachment, - const float maxChi2NDF, - const int maxHoles, - const int minSeedingClusters, - const LayerMask holeLayerMask, - const LayerMask nonSeedingLayerMask, - const std::vector& layerxX0Host, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType, - o2::its::ExternalAllocator* alloc); - -template -void countTrackSeedHandler(TrackSeed* trackSeeds, - const TrackingFrameInfo** foundTrackingFrameInfo, - const Cluster** unsortedClusters, - int* seedLUT, - const std::vector& layerRadiiHost, - const std::vector& minPtsHost, - const std::vector& layerxX0Host, - const unsigned int nSeeds, - const float Bz, - const float maxChi2ClusterAttachment, - const float maxChi2NDF, - const int reseedIfShorter, - const bool repeatRefitOut, - const bool shiftRefToCluster, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType, - o2::its::ExternalAllocator* alloc); - -template -void computeTrackSeedHandler(TrackSeed* trackSeeds, - const TrackingFrameInfo** foundTrackingFrameInfo, - const Cluster** unsortedClusters, - const IndexTableUtils* utils, - const typename ROFMaskTable::View& rofMask, - const typename ROFOverlapTable::View& rofOverlaps, - const Cluster** clusters, - const unsigned char** usedClusters, - const int** clustersIndexTables, - const int** ROFClusters, - o2::its::TrackITSExt* tracks, - int* trackIndices, - const int* seedLUT, - TrackExtensionHypothesis* activeHypotheses, - TrackExtensionHypothesis* nextHypotheses, - const std::vector& layerRadiiHost, - const std::vector& minPtsHost, - const std::vector& layerxX0Host, - const unsigned int nSeeds, - const unsigned int nTracks, - const float Bz, - const float maxChi2ClusterAttachment, - const float maxChi2NDF, - const int reseedIfShorter, - const bool repeatRefitOut, - const bool shiftRefToCluster, - const int nLayers, - const int phiBins, - const int maxHypotheses, - const bool extendTop, - const bool extendBot, - const float nSigmaCutPhi, - const float nSigmaCutZ, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType, - o2::its::ExternalAllocator* alloc); - } // namespace o2::its #endif // ITSTRACKINGGPU_TRACKINGKERNELS_H_ diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/Utils.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/Utils.h index bcc20ace7bbc2..e6909b28a687a 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/Utils.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/Utils.h @@ -343,29 +343,6 @@ struct TypedAllocator { ExternalAllocator* mInternalAllocator; }; -GPUdii() gpuSpan getPrimaryVertices(const int rof, - const int* roframesPV, - const int nROF, - const uint8_t* mask, - const Vertex* vertices) -{ - const int start_pv_id = roframesPV[rof]; - const int stop_rof = rof >= nROF - 1 ? nROF : rof + 1; - size_t delta = mask[rof] ? roframesPV[stop_rof] - start_pv_id : 0; // return empty span if ROF is excluded - return gpuSpan(&vertices[start_pv_id], delta); -}; - -GPUdii() gpuSpan getPrimaryVertices(const int romin, - const int romax, - const int* roframesPV, - const int nROF, - const Vertex* vertices) -{ - const int start_pv_id = roframesPV[romin]; - const int stop_rof = romax >= nROF - 1 ? nROF : romax + 1; - return gpuSpan(&vertices[start_pv_id], roframesPV[stop_rof] - roframesPV[romin]); -}; - GPUdii() gpuSpan getClustersOnLayer(const int rof, const int totROFs, const int layer, @@ -381,78 +358,6 @@ GPUdii() gpuSpan getClustersOnLayer(const int rof, return gpuSpan(&(clusters[layer][start_clus_id]), delta); } -GPUdii() gpuSpan getTrackletsPerCluster(const int rof, - const int totROFs, - const int mode, - const int** roframesClus, - const Tracklet** tracklets) -{ - if (rof < 0 || rof >= totROFs) { - return gpuSpan(); - } - const int start_clus_id{roframesClus[1][rof]}; - const int stop_rof = rof >= totROFs - 1 ? totROFs : rof + 1; - const unsigned int delta = roframesClus[1][stop_rof] - start_clus_id; - return gpuSpan(&(tracklets[mode][start_clus_id]), delta); -} - -GPUdii() gpuSpan getNTrackletsPerCluster(const int rof, - const int totROFs, - const int mode, - const int** roframesClus, - int** ntracklets) -{ - if (rof < 0 || rof >= totROFs) { - return gpuSpan(); - } - const int start_clus_id{roframesClus[1][rof]}; - const int stop_rof = rof >= totROFs - 1 ? totROFs : rof + 1; - const unsigned int delta = roframesClus[1][stop_rof] - start_clus_id; - return gpuSpan(&(ntracklets[mode][start_clus_id]), delta); -} - -GPUdii() gpuSpan getNTrackletsPerCluster(const int rof, - const int totROFs, - const int mode, - const int** roframesClus, - const int** ntracklets) -{ - if (rof < 0 || rof >= totROFs) { - return gpuSpan(); - } - const int start_clus_id{roframesClus[1][rof]}; - const int stop_rof = rof >= totROFs - 1 ? totROFs : rof + 1; - const unsigned int delta = roframesClus[1][stop_rof] - start_clus_id; - return gpuSpan(&(ntracklets[mode][start_clus_id]), delta); -} - -GPUdii() gpuSpan getNLinesPerCluster(const int rof, - const int totROFs, - const int** roframesClus, - int* nlines) -{ - if (rof < 0 || rof >= totROFs) { - return gpuSpan(); - } - const int start_clus_id{roframesClus[1][rof]}; - const int stop_rof = rof >= totROFs - 1 ? totROFs : rof + 1; - const unsigned int delta = roframesClus[1][stop_rof] - start_clus_id; - return gpuSpan(&(nlines[start_clus_id]), delta); -} - -GPUdii() gpuSpan getNLinesPerCluster(const int rof, - const int totROFs, - const int** roframesClus, - const int* nlines) -{ - if (rof < 0 || rof >= totROFs) { - return gpuSpan(); - } - const int start_clus_id{roframesClus[1][rof]}; - const int stop_rof = rof >= totROFs - 1 ? totROFs : rof + 1; - const unsigned int delta = roframesClus[1][stop_rof] - start_clus_id; - return gpuSpan(&(nlines[start_clus_id]), delta); -} #endif } // namespace gpu } // namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/CMakeLists.txt b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/CMakeLists.txt index 38f11265682ce..c6c2507956203 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/CMakeLists.txt +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/CMakeLists.txt @@ -37,4 +37,7 @@ if(CUDA_ENABLED) ) # target_compile_definitions(${targetName} PRIVATE ITS_MEASURE_GPU_TIME ITS_GPU_LOG) target_compile_definitions(${targetName} PRIVATE $) + if(GPUCA_DETERMINISTIC_MODE GREATER_EQUAL ${GPUCA_DETERMINISTIC_MODE_MAP_GPU}) + target_compile_definitions(${targetName} PRIVATE GPUCA_DETERMINISTIC_MODE) + endif() endif() diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu index 2d467b0d5e151..b37ff6ddbab5c 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu @@ -13,6 +13,8 @@ #include #include +#include +#include #include #include @@ -31,153 +33,230 @@ namespace o2::its::gpu { template -void TimeFrameGPU::allocMemAsync(void** ptr, size_t size, Stream& stream, bool extAllocator, int32_t type) +template +T* TimeFrameGPU::allocDevice(const size_t n, const int32_t type) { - if (extAllocator) { - *ptr = (this->mExternalAllocator)->allocate(size, type); + if (n == 0) { + return nullptr; + } + void* ptr{nullptr}; + if (this->hasFrameworkAllocator()) { + ptr = (this->mExternalAllocator)->allocate(n * sizeof(T), type); } else { GPULog("Calling default CUDA allocator"); - GPUChkErrS(cudaMallocAsync(reinterpret_cast(ptr), size, stream.get())); + GPUChkErrS(cudaMalloc(&ptr, n * sizeof(T))); } + return static_cast(ptr); } template -void TimeFrameGPU::allocMem(void** ptr, size_t size, bool extAllocator, int32_t type) +template +T* TimeFrameGPU::allocDeviceAsync(const size_t n, Stream& stream, const int32_t type) { - if (extAllocator) { - *ptr = (this->mExternalAllocator)->allocate(size, type); + if (n == 0) { + return nullptr; + } + void* ptr{nullptr}; + if (this->hasFrameworkAllocator()) { + ptr = (this->mExternalAllocator)->allocate(n * sizeof(T), type); } else { GPULog("Calling default CUDA allocator"); - GPUChkErrS(cudaMalloc(reinterpret_cast(ptr), size)); + GPUChkErrS(cudaMallocAsync(&ptr, n * sizeof(T), stream.get())); } + return static_cast(ptr); } template -void TimeFrameGPU::loadIndexTableUtils() +template +SlotPtr* TimeFrameGPU::allocSlotArray(const size_t n) { - GPUTimer timer("loading indextable utils"); - { - GPULog("gpu-allocation: allocating IndexTableUtils buffer, for {:.2f} MB.", sizeof(IndexTableUtilsN) / constants::MB); - allocMem(reinterpret_cast(&mIndexTableUtilsDevice), sizeof(IndexTableUtilsN), this->hasFrameworkAllocator()); + auto* array = allocDevice(n); + if (array != nullptr) { + GPUChkErrS(cudaMemset(array, 0, n * sizeof(SlotPtr))); } - GPULog("gpu-transfer: loading IndexTableUtils object, for {:.2f} MB.", sizeof(IndexTableUtilsN) / constants::MB); - GPUChkErrS(cudaMemcpy(mIndexTableUtilsDevice, &(this->mIndexTableUtils), sizeof(IndexTableUtilsN), cudaMemcpyHostToDevice)); + return array; } template -void TimeFrameGPU::createUnsortedClustersDeviceArray(const int maxLayers) +template +void TimeFrameGPU::copyToDevice(T* dst, const T* src, const size_t n) { - { - GPUTimer timer("creating unsorted clusters array"); - allocMem(reinterpret_cast(&mUnsortedClustersDeviceArray), NLayers * sizeof(Cluster*), this->hasFrameworkAllocator()); - GPUChkErrS(cudaHostRegister(mUnsortedClustersDevice.data(), NLayers * sizeof(Cluster*), cudaHostRegisterPortable)); - mPinnedUnsortedClusters.set(NLayers); - if (!this->hasFrameworkAllocator()) { - for (auto iLayer{0}; iLayer < o2::gpu::CAMath::Min(maxLayers, NLayers); ++iLayer) { - GPUChkErrS(cudaHostRegister(this->mUnsortedClusters[iLayer].data(), this->mUnsortedClusters[iLayer].size() * sizeof(Cluster), cudaHostRegisterPortable)); - mPinnedUnsortedClusters.set(iLayer); - } - } + if (n > 0) { + GPUChkErrS(cudaMemcpy(dst, src, n * sizeof(T), cudaMemcpyHostToDevice)); } } template -void TimeFrameGPU::loadUnsortedClustersDevice(const int layer) +template +void TimeFrameGPU::copyFromDevice(T* dst, const T* src, const size_t n) { - { - GPUTimer timer(mGpuStreams[layer], "loading unsorted clusters", layer); - GPULog("gpu-transfer: loading {} unsorted clusters on layer {}, for {:.2f} MB.", this->mUnsortedClusters[layer].size(), layer, this->mUnsortedClusters[layer].size() * sizeof(Cluster) / constants::MB); - allocMemAsync(reinterpret_cast(&mUnsortedClustersDevice[layer]), this->mUnsortedClusters[layer].size() * sizeof(Cluster), mGpuStreams[layer], this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpyAsync(mUnsortedClustersDevice[layer], this->mUnsortedClusters[layer].data(), this->mUnsortedClusters[layer].size() * sizeof(Cluster), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); - GPUChkErrS(cudaMemcpyAsync(&mUnsortedClustersDeviceArray[layer], &mUnsortedClustersDevice[layer], sizeof(Cluster*), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); + if (n > 0) { + GPUChkErrS(cudaMemcpy(dst, src, n * sizeof(T), cudaMemcpyDeviceToHost)); } } template -void TimeFrameGPU::createClustersDeviceArray(const int maxLayers) +template +void TimeFrameGPU::publishSlot(ArrayT deviceArray, const int slot, T* const& devicePtr, Stream& stream) { - { - GPUTimer timer("creating sorted clusters array"); - allocMem(reinterpret_cast(&mClustersDeviceArray), NLayers * sizeof(Cluster*), this->hasFrameworkAllocator()); - GPUChkErrS(cudaHostRegister(mClustersDevice.data(), NLayers * sizeof(Cluster*), cudaHostRegisterPortable)); - mPinnedClusters.set(NLayers); - if (!this->hasFrameworkAllocator()) { - for (auto iLayer{0}; iLayer < o2::gpu::CAMath::Min(maxLayers, NLayers); ++iLayer) { - GPUChkErrS(cudaHostRegister(this->mClusters[iLayer].data(), this->mClusters[iLayer].size() * sizeof(Cluster), cudaHostRegisterPortable)); - mPinnedClusters.set(iLayer); - } - } + GPUChkErrS(cudaMemcpyAsync(&deviceArray[slot], &devicePtr, sizeof(T*), cudaMemcpyHostToDevice, stream.get())); +} + +template +template +T* TimeFrameGPU::createSlot(std::array& slots, ArrayT deviceArray, const int slot, const size_t n, + const char* what, const SlotInit init, const int32_t type) +{ + auto& stream = mGpuStreams[slot]; + GPULog("gpu-allocation: creating {} for {} elements on slot {}, for {:.2f} MB.", what, n, slot, n * sizeof(T) / constants::MB); + slots[slot] = allocDeviceAsync(n, stream, type); + if (init == SlotInit::Zero && n > 0) { + GPUChkErrS(cudaMemsetAsync(slots[slot], 0, n * sizeof(T), stream.get())); } + publishSlot(deviceArray, slot, slots[slot], stream); + return slots[slot]; } template -void TimeFrameGPU::loadClustersDevice(const int layer) +template +void TimeFrameGPU::uploadSlot(std::array& slots, ArrayT deviceArray, const int slot, const Container& host, const char* what) { - { - GPUTimer timer(mGpuStreams[layer], "loading sorted clusters", layer); - GPULog("gpu-transfer: loading {} clusters on layer {}, for {:.2f} MB.", this->mClusters[layer].size(), layer, this->mClusters[layer].size() * sizeof(Cluster) / constants::MB); - allocMemAsync(reinterpret_cast(&mClustersDevice[layer]), this->mClusters[layer].size() * sizeof(Cluster), mGpuStreams[layer], this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpyAsync(mClustersDevice[layer], this->mClusters[layer].data(), this->mClusters[layer].size() * sizeof(Cluster), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); - GPUChkErrS(cudaMemcpyAsync(&mClustersDeviceArray[layer], &mClustersDevice[layer], sizeof(Cluster*), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); + auto& stream = mGpuStreams[slot]; + GPULog("gpu-transfer: loading {} {} on slot {}, for {:.2f} MB.", host.size(), what, slot, host.size() * sizeof(T) / constants::MB); + slots[slot] = allocDeviceAsync(host.size(), stream); + if (!host.empty()) { + GPUChkErrS(cudaMemcpyAsync(slots[slot], host.data(), host.size() * sizeof(T), cudaMemcpyHostToDevice, stream.get())); } + publishSlot(deviceArray, slot, slots[slot], stream); } template -void TimeFrameGPU::createClustersIndexTablesArray() +template +void TimeFrameGPU::createPinnedSlotArray(ArrayT& deviceArray, std::array& slots, std::bitset& pinned) { - { - GPUTimer timer("creating clustersindextable array"); - allocMem(reinterpret_cast(&mClustersIndexTablesDeviceArray), NLayers * sizeof(int*), this->hasFrameworkAllocator()); - GPUChkErrS(cudaHostRegister(mClustersIndexTablesDevice.data(), NLayers * sizeof(int*), cudaHostRegisterPortable)); - mPinnedClustersIndexTables.set(NLayers); - if (!this->hasFrameworkAllocator()) { - for (auto iLayer{0}; iLayer < NLayers; ++iLayer) { - GPUChkErrS(cudaHostRegister(this->mIndexTables[iLayer].data(), this->mIndexTables[iLayer].size() * sizeof(int), cudaHostRegisterPortable)); - mPinnedClustersIndexTables.set(iLayer); - } + deviceArray = allocSlotArray>(N); + GPUChkErrS(cudaHostRegister(slots.data(), N * sizeof(T*), cudaHostRegisterPortable)); + pinned.set(NLayers); +} + +template +template +void TimeFrameGPU::pinHostLayers(Layers& layers, std::bitset& pinned, const int maxLayers) +{ + if (this->hasFrameworkAllocator()) { // the framework already hands out registered memory + return; + } + for (auto iLayer{0}; iLayer < o2::gpu::CAMath::Min(maxLayers, NLayers); ++iLayer) { + auto& host = layers[iLayer]; + if (host.empty()) { // registering an empty range fails, and the bit must stay clear for wipe() + continue; } + GPUChkErrS(cudaHostRegister(host.data(), host.size() * sizeof(typename std::decay_t::value_type), cudaHostRegisterPortable)); + pinned.set(iLayer); } } +template +template +typename Table::View TimeFrameGPU::uploadNavigationTable(const Table& table, const typename Table::View& hostView) +{ + auto* dFlatTable = allocDevice(table.getFlatTableSize()); + auto* dIndices = allocDevice(table.getIndicesSize()); + auto* dLayers = allocDevice(NLayers); + copyToDevice(dFlatTable, hostView.mFlatTable, table.getFlatTableSize()); + copyToDevice(dIndices, hostView.mIndices, table.getIndicesSize()); + copyToDevice(dLayers, hostView.mLayers, NLayers); + return table.getDeviceView(dFlatTable, dIndices, dLayers); +} + +template +void TimeFrameGPU::loadIndexTableUtils() +{ + GPUTimer timer("loading indextable utils"); + GPULog("gpu-transfer: loading IndexTableUtils object, for {:.2f} MB.", sizeof(IndexTableUtilsN) / constants::MB); + mIndexTableUtilsDevice = allocDevice(1); + copyToDevice(mIndexTableUtilsDevice, &(this->mIndexTableUtils), 1); +} + +template +void TimeFrameGPU::loadIterationParameters(const TrackingParameters& params) +{ + GPUTimer timer("loading iteration parameters"); + const auto& radii = params.LayerRadii; + const auto& minPts = params.MinPt; + const auto& xX0 = params.LayerxX0; + const size_t n = radii.size() + minPts.size() + xX0.size(); + GPULog("gpu-transfer: loading {} iteration parameters, for {:.2f} MB.", n, n * sizeof(float) / constants::MB); + std::vector staging; + staging.reserve(n); + staging.insert(staging.end(), radii.begin(), radii.end()); + staging.insert(staging.end(), minPts.begin(), minPts.end()); + staging.insert(staging.end(), xX0.begin(), xX0.end()); + mIterationParametersDevice = allocDevice(n); + copyToDevice(mIterationParametersDevice, staging.data(), n); + mLayerRadiiDevice = mIterationParametersDevice; + mMinPtsDevice = mLayerRadiiDevice + radii.size(); + mLayerxX0Device = mMinPtsDevice + minPts.size(); +} + +template +void TimeFrameGPU::createUnsortedClustersDeviceArray(const int maxLayers) +{ + GPUTimer timer("creating unsorted clusters array"); + createPinnedSlotArray(mUnsortedClustersDeviceArray, mUnsortedClustersDevice, mPinnedUnsortedClusters); + pinHostLayers(this->mUnsortedClusters, mPinnedUnsortedClusters, maxLayers); +} + +template +void TimeFrameGPU::loadUnsortedClustersDevice(const int layer) +{ + GPUTimer timer(mGpuStreams[layer], "loading unsorted clusters", layer); + uploadSlot(mUnsortedClustersDevice, mUnsortedClustersDeviceArray, layer, this->mUnsortedClusters[layer], "unsorted clusters"); +} + +template +void TimeFrameGPU::createClustersDeviceArray(const int maxLayers) +{ + GPUTimer timer("creating sorted clusters array"); + createPinnedSlotArray(mClustersDeviceArray, mClustersDevice, mPinnedClusters); + pinHostLayers(this->mClusters, mPinnedClusters, maxLayers); +} + +template +void TimeFrameGPU::loadClustersDevice(const int layer) +{ + GPUTimer timer(mGpuStreams[layer], "loading sorted clusters", layer); + uploadSlot(mClustersDevice, mClustersDeviceArray, layer, this->mClusters[layer], "sorted clusters"); +} + +template +void TimeFrameGPU::createClustersIndexTablesArray(const int maxLayers) +{ + GPUTimer timer("creating clustersindextable array"); + createPinnedSlotArray(mClustersIndexTablesDeviceArray, mClustersIndexTablesDevice, mPinnedClustersIndexTables); + pinHostLayers(this->mIndexTables, mPinnedClustersIndexTables, maxLayers); +} + template void TimeFrameGPU::loadClustersIndexTables(const int layer) { - { - GPUTimer timer(mGpuStreams[layer], "loading sorted clusters", layer); - GPULog("gpu-transfer: loading clusters indextable for layer {} with {} elements, for {:.2f} MB.", layer, this->mIndexTables[layer].size(), this->mIndexTables[layer].size() * sizeof(int) / constants::MB); - allocMemAsync(reinterpret_cast(&mClustersIndexTablesDevice[layer]), this->mIndexTables[layer].size() * sizeof(int), mGpuStreams[layer], this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpyAsync(mClustersIndexTablesDevice[layer], this->mIndexTables[layer].data(), this->mIndexTables[layer].size() * sizeof(int), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); - GPUChkErrS(cudaMemcpyAsync(&mClustersIndexTablesDeviceArray[layer], &mClustersIndexTablesDevice[layer], sizeof(int*), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); - } + GPUTimer timer(mGpuStreams[layer], "loading clusters indextables", layer); + uploadSlot(mClustersIndexTablesDevice, mClustersIndexTablesDeviceArray, layer, this->mIndexTables[layer], "clusters indextable entries"); } template void TimeFrameGPU::createUsedClustersDeviceArray(const int maxLayers) { - { - GPUTimer timer("creating used clusters flags"); - allocMem(reinterpret_cast(&mUsedClustersDeviceArray), NLayers * sizeof(uint8_t*), this->hasFrameworkAllocator()); - GPUChkErrS(cudaHostRegister(mUsedClustersDevice.data(), NLayers * sizeof(uint8_t*), cudaHostRegisterPortable)); - mPinnedUsedClusters.set(NLayers); - if (!this->hasFrameworkAllocator()) { - for (auto iLayer{0}; iLayer < o2::gpu::CAMath::Min(maxLayers, NLayers); ++iLayer) { - GPUChkErrS(cudaHostRegister(this->mUsedClusters[iLayer].data(), this->mUsedClusters[iLayer].size() * sizeof(uint8_t), cudaHostRegisterPortable)); - mPinnedUsedClusters.set(iLayer); - } - } - } + GPUTimer timer("creating used clusters flags"); + createPinnedSlotArray(mUsedClustersDeviceArray, mUsedClustersDevice, mPinnedUsedClusters); + pinHostLayers(this->mUsedClusters, mPinnedUsedClusters, maxLayers); } template void TimeFrameGPU::createUsedClustersDevice(const int layer) { - { - GPUTimer timer(mGpuStreams[layer], "creating used clusters flags", layer); - GPULog("gpu-transfer: creating {} used clusters flags on layer {}, for {:.2f} MB.", this->mUsedClusters[layer].size(), layer, this->mUsedClusters[layer].size() * sizeof(unsigned char) / constants::MB); - allocMemAsync(reinterpret_cast(&mUsedClustersDevice[layer]), this->mUsedClusters[layer].size() * sizeof(unsigned char), mGpuStreams[layer], this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemsetAsync(mUsedClustersDevice[layer], 0, this->mUsedClusters[layer].size() * sizeof(unsigned char), mGpuStreams[layer].get())); - GPUChkErrS(cudaMemcpyAsync(&mUsedClustersDeviceArray[layer], &mUsedClustersDevice[layer], sizeof(unsigned char*), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); - } + GPUTimer timer(mGpuStreams[layer], "creating used clusters flags", layer); + createSlot(mUsedClustersDevice, mUsedClustersDeviceArray, layer, this->mUsedClusters[layer].size(), "used clusters flags", SlotInit::Zero); } template @@ -185,148 +264,83 @@ void TimeFrameGPU::loadUsedClustersDevice() { for (auto iLayer{0}; iLayer < NLayers; ++iLayer) { GPUTimer timer(mGpuStreams[iLayer], "loading used clusters flags", iLayer); - GPULog("gpu-transfer: loading {} used clusters flags on layer {}, for {:.2f} MB.", this->mUsedClusters[iLayer].size(), iLayer, this->mUsedClusters[iLayer].size() * sizeof(unsigned char) / constants::MB); - GPUChkErrS(cudaMemcpyAsync(mUsedClustersDevice[iLayer], this->mUsedClusters[iLayer].data(), this->mUsedClusters[iLayer].size() * sizeof(unsigned char), cudaMemcpyHostToDevice, mGpuStreams[iLayer].get())); + const auto& used = this->mUsedClusters[iLayer]; + GPULog("gpu-transfer: loading {} used clusters flags on layer {}, for {:.2f} MB.", used.size(), iLayer, used.size() * sizeof(unsigned char) / constants::MB); + if (!used.empty()) { + GPUChkErrS(cudaMemcpyAsync(mUsedClustersDevice[iLayer], used.data(), used.size() * sizeof(unsigned char), cudaMemcpyHostToDevice, mGpuStreams[iLayer].get())); + } } } template -void TimeFrameGPU::createROFrameClustersDeviceArray() +void TimeFrameGPU::createROFrameClustersDeviceArray(const int maxLayers) { - { - GPUTimer timer("creating ROFrame clusters array"); - allocMem(reinterpret_cast(&mROFramesClustersDeviceArray), NLayers * sizeof(int*), this->hasFrameworkAllocator()); - GPUChkErrS(cudaHostRegister(mROFramesClustersDevice.data(), NLayers * sizeof(int*), cudaHostRegisterPortable)); - mPinnedROFramesClusters.set(NLayers); - if (!this->hasFrameworkAllocator()) { - for (auto iLayer{0}; iLayer < NLayers; ++iLayer) { - GPUChkErrS(cudaHostRegister(this->mROFramesClusters[iLayer].data(), this->mROFramesClusters[iLayer].size() * sizeof(int), cudaHostRegisterPortable)); - mPinnedROFramesClusters.set(iLayer); - } - } - } + GPUTimer timer("creating ROFrame clusters array"); + createPinnedSlotArray(mROFramesClustersDeviceArray, mROFramesClustersDevice, mPinnedROFramesClusters); + pinHostLayers(this->mROFramesClusters, mPinnedROFramesClusters, maxLayers); } template void TimeFrameGPU::loadROFrameClustersDevice(const int layer) { - { - GPUTimer timer(mGpuStreams[layer], "loading ROframe clusters", layer); - GPULog("gpu-transfer: loading {} ROframe clusters info on layer {}, for {:.2f} MB.", this->mROFramesClusters[layer].size(), layer, this->mROFramesClusters[layer].size() * sizeof(int) / constants::MB); - allocMemAsync(reinterpret_cast(&mROFramesClustersDevice[layer]), this->mROFramesClusters[layer].size() * sizeof(int), mGpuStreams[layer], this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpyAsync(mROFramesClustersDevice[layer], this->mROFramesClusters[layer].data(), this->mROFramesClusters[layer].size() * sizeof(int), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); - GPUChkErrS(cudaMemcpyAsync(&mROFramesClustersDeviceArray[layer], &mROFramesClustersDevice[layer], sizeof(int*), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); - } + GPUTimer timer(mGpuStreams[layer], "loading ROframe clusters", layer); + uploadSlot(mROFramesClustersDevice, mROFramesClustersDeviceArray, layer, this->mROFramesClusters[layer], "ROframe clusters"); } template -void TimeFrameGPU::createTrackingFrameInfoDeviceArray() +void TimeFrameGPU::createTrackingFrameInfoDeviceArray(const int maxLayers) { - { - GPUTimer timer("creating trackingframeinfo array"); - allocMem(reinterpret_cast(&mTrackingFrameInfoDeviceArray), NLayers * sizeof(TrackingFrameInfo*), this->hasFrameworkAllocator()); - GPUChkErrS(cudaHostRegister(mTrackingFrameInfoDevice.data(), NLayers * sizeof(TrackingFrameInfo*), cudaHostRegisterPortable)); - mPinnedTrackingFrameInfo.set(NLayers); - if (!this->hasFrameworkAllocator()) { - for (auto iLayer{0}; iLayer < NLayers; ++iLayer) { - GPUChkErrS(cudaHostRegister(this->mTrackingFrameInfo[iLayer].data(), this->mTrackingFrameInfo[iLayer].size() * sizeof(TrackingFrameInfo), cudaHostRegisterPortable)); - mPinnedTrackingFrameInfo.set(iLayer); - } - } - } + GPUTimer timer("creating trackingframeinfo array"); + createPinnedSlotArray(mTrackingFrameInfoDeviceArray, mTrackingFrameInfoDevice, mPinnedTrackingFrameInfo); + pinHostLayers(this->mTrackingFrameInfo, mPinnedTrackingFrameInfo, maxLayers); } template void TimeFrameGPU::loadTrackingFrameInfoDevice(const int layer) { - { - GPUTimer timer(mGpuStreams[layer], "loading trackingframeinfo", layer); - GPULog("gpu-transfer: loading {} tfinfo on layer {}, for {:.2f} MB.", this->mTrackingFrameInfo[layer].size(), layer, this->mTrackingFrameInfo[layer].size() * sizeof(TrackingFrameInfo) / constants::MB); - allocMemAsync(reinterpret_cast(&mTrackingFrameInfoDevice[layer]), this->mTrackingFrameInfo[layer].size() * sizeof(TrackingFrameInfo), mGpuStreams[layer], this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpyAsync(mTrackingFrameInfoDevice[layer], this->mTrackingFrameInfo[layer].data(), this->mTrackingFrameInfo[layer].size() * sizeof(TrackingFrameInfo), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); - GPUChkErrS(cudaMemcpyAsync(&mTrackingFrameInfoDeviceArray[layer], &mTrackingFrameInfoDevice[layer], sizeof(TrackingFrameInfo*), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); - } + GPUTimer timer(mGpuStreams[layer], "loading trackingframeinfo", layer); + uploadSlot(mTrackingFrameInfoDevice, mTrackingFrameInfoDeviceArray, layer, this->mTrackingFrameInfo[layer], "tfinfo"); } template void TimeFrameGPU::loadROFCutMask(const int iteration) { - { - GPUTimer timer("loading multiplicity cut mask"); - const auto& hostTable = *(this->mROFMask); - const auto hostView = hostTable.getView(); - using TableEntry = ROFMaskTable::TableEntry; - using TableIndex = ROFMaskTable::TableIndex; - TableEntry* d_flatTable{nullptr}; - TableIndex* d_indices{nullptr}; - GPULog("gpu-transfer: iteration {} loading multiplicity cut mask with {} elements, for {:.2f} MB.", - iteration, hostTable.getFlatMaskSize(), hostTable.getFlatMaskSize() * sizeof(TableEntry) / constants::MB); - allocMem(reinterpret_cast(&d_flatTable), hostTable.getFlatMaskSize() * sizeof(TableEntry), this->hasFrameworkAllocator()); - allocMem(reinterpret_cast(&d_indices), NLayers * sizeof(uint32_t), this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpy(d_indices, hostView.mLayerROFOffsets, NLayers * sizeof(TableIndex), cudaMemcpyHostToDevice)); - // Re-copy the flat mask on every qualifying iteration (e.g. after swapMasks() for UPC) - GPUChkErrS(cudaMemcpy(d_flatTable, hostView.mFlatMask, hostTable.getFlatMaskSize() * sizeof(TableEntry), cudaMemcpyHostToDevice)); - mDeviceROFMaskTableView = hostTable.getDeviceView(d_flatTable, d_indices); - } + GPUTimer timer("loading multiplicity cut mask"); + const auto& hostTable = *(this->mROFMask); + const auto hostView = hostTable.getView(); + using TableEntry = ROFMaskTable::TableEntry; + using TableIndex = ROFMaskTable::TableIndex; + GPULog("gpu-transfer: iteration {} loading multiplicity cut mask with {} elements, for {:.2f} MB.", + iteration, hostTable.getFlatMaskSize(), hostTable.getFlatMaskSize() * sizeof(TableEntry) / constants::MB); + auto* dFlatMask = allocDevice(hostTable.getFlatMaskSize()); + auto* dOffsets = allocDevice(NLayers + 1); // the view reads the sentinel past the last layer + copyToDevice(dOffsets, hostView.mLayerROFOffsets, NLayers + 1); + // Re-copy the flat mask on every qualifying iteration (e.g. after swapMasks() for UPC) + copyToDevice(dFlatMask, hostView.mFlatMask, hostTable.getFlatMaskSize()); + mDeviceROFMaskTableView = hostTable.getDeviceView(dFlatMask, dOffsets); } template void TimeFrameGPU::loadVertices() { - { - GPUTimer timer("loading seeding vertices"); - GPULog("gpu-transfer: loading {} seeding vertices, for {:.2f} MB.", this->mPrimaryVertices.size(), this->mPrimaryVertices.size() * sizeof(Vertex) / constants::MB); - allocMem(reinterpret_cast(&mPrimaryVerticesDevice), this->mPrimaryVertices.size() * sizeof(Vertex), this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpy(mPrimaryVerticesDevice, this->mPrimaryVertices.data(), this->mPrimaryVertices.size() * sizeof(Vertex), cudaMemcpyHostToDevice)); - } + GPUTimer timer("loading seeding vertices"); + GPULog("gpu-transfer: loading {} seeding vertices, for {:.2f} MB.", this->mPrimaryVertices.size(), this->mPrimaryVertices.size() * sizeof(Vertex) / constants::MB); + mPrimaryVerticesDevice = allocDevice(this->mPrimaryVertices.size()); + copyToDevice(mPrimaryVerticesDevice, this->mPrimaryVertices.data(), this->mPrimaryVertices.size()); } template void TimeFrameGPU::loadROFOverlapTable() { - { - GPUTimer timer("initialising device view of ROFOverlapTable"); - const auto& hostTable = this->getROFOverlapTable(); - const auto& hostView = this->getROFOverlapTableView(); - using TableEntry = ROFOverlapTable::TableEntry; - using TableIndex = ROFOverlapTable::TableIndex; - using LayerTiming = o2::its::LayerTiming; - TableEntry* d_flatTable{nullptr}; - TableIndex* d_indices{nullptr}; - LayerTiming* d_layers{nullptr}; - size_t flatTableSize = hostTable.getFlatTableSize(); - allocMem(reinterpret_cast(&d_flatTable), flatTableSize * sizeof(TableEntry), this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpy(d_flatTable, hostView.mFlatTable, flatTableSize * sizeof(TableEntry), cudaMemcpyHostToDevice)); - allocMem(reinterpret_cast(&d_indices), hostTable.getIndicesSize() * sizeof(TableIndex), this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpy(d_indices, hostView.mIndices, hostTable.getIndicesSize() * sizeof(TableIndex), cudaMemcpyHostToDevice)); - allocMem(reinterpret_cast(&d_layers), NLayers * sizeof(LayerTiming), this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpy(d_layers, hostView.mLayers, NLayers * sizeof(LayerTiming), cudaMemcpyHostToDevice)); - mDeviceROFOverlapTableView = hostTable.getDeviceView(d_flatTable, d_indices, d_layers); - } + GPUTimer timer("initialising device view of ROFOverlapTable"); + mDeviceROFOverlapTableView = uploadNavigationTable(this->getROFOverlapTable(), this->getROFOverlapTableView()); } template void TimeFrameGPU::loadROFVertexLookupTable() { - { - GPUTimer timer("initialising device view of ROFVertexLookupTable"); - const auto& hostTable = this->getROFVertexLookupTable(); - const auto& hostView = this->getROFVertexLookupTableView(); - using TableEntry = ROFVertexLookupTable::TableEntry; - using TableIndex = ROFVertexLookupTable::TableIndex; - using LayerTiming = o2::its::LayerTiming; - TableEntry* d_flatTable{nullptr}; - TableIndex* d_indices{nullptr}; - LayerTiming* d_layers{nullptr}; - size_t flatTableSize = hostTable.getFlatTableSize(); - allocMem(reinterpret_cast(&d_flatTable), flatTableSize * sizeof(TableEntry), this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpy(d_flatTable, hostView.mFlatTable, flatTableSize * sizeof(TableEntry), cudaMemcpyHostToDevice)); - allocMem(reinterpret_cast(&d_indices), hostTable.getIndicesSize() * sizeof(TableIndex), this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpy(d_indices, hostView.mIndices, hostTable.getIndicesSize() * sizeof(TableIndex), cudaMemcpyHostToDevice)); - allocMem(reinterpret_cast(&d_layers), NLayers * sizeof(LayerTiming), this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpy(d_layers, hostView.mLayers, NLayers * sizeof(LayerTiming), cudaMemcpyHostToDevice)); - mDeviceROFVertexLookupTableView = hostTable.getDeviceView(d_flatTable, d_indices, d_layers); - } + GPUTimer timer("initialising device view of ROFVertexLookupTable"); + mDeviceROFVertexLookupTableView = uploadNavigationTable(this->getROFVertexLookupTable(), this->getROFVertexLookupTableView()); } template @@ -341,18 +355,14 @@ void TimeFrameGPU::loadTrackingTopologies() using Id = typename TrackingTopologyN::Id; for (size_t iteration = 0; iteration < hostTopologies.size(); ++iteration) { const auto& topology = hostTopologies[iteration]; - LayerLink* dLinks{nullptr}; - CellTopology* dCells{nullptr}; - Range* dCellsByFirstLinkIndex{nullptr}; - Id* dCellsByFirstLink{nullptr}; - allocMem(reinterpret_cast(&dLinks), topology.getNLinks() * sizeof(LayerLink), this->hasFrameworkAllocator()); - allocMem(reinterpret_cast(&dCells), topology.getNCells() * sizeof(CellTopology), this->hasFrameworkAllocator()); - allocMem(reinterpret_cast(&dCellsByFirstLinkIndex), topology.getNLinks() * sizeof(Range), this->hasFrameworkAllocator()); - allocMem(reinterpret_cast(&dCellsByFirstLink), topology.getNCellsByFirstLink() * sizeof(Id), this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpy(dLinks, topology.getLinks().data(), topology.getNLinks() * sizeof(LayerLink), cudaMemcpyHostToDevice)); - GPUChkErrS(cudaMemcpy(dCells, topology.getCells().data(), topology.getNCells() * sizeof(CellTopology), cudaMemcpyHostToDevice)); - GPUChkErrS(cudaMemcpy(dCellsByFirstLinkIndex, topology.getCellsByFirstLinkIndex().data(), topology.getNLinks() * sizeof(Range), cudaMemcpyHostToDevice)); - GPUChkErrS(cudaMemcpy(dCellsByFirstLink, topology.getCellsByFirstLink().data(), topology.getNCellsByFirstLink() * sizeof(Id), cudaMemcpyHostToDevice)); + auto* dLinks = allocDevice(topology.getNLinks()); + auto* dCells = allocDevice(topology.getNCells()); + auto* dCellsByFirstLinkIndex = allocDevice(topology.getNLinks()); + auto* dCellsByFirstLink = allocDevice(topology.getNCellsByFirstLink()); + copyToDevice(dLinks, topology.getLinks().data(), topology.getNLinks()); + copyToDevice(dCells, topology.getCells().data(), topology.getNCells()); + copyToDevice(dCellsByFirstLinkIndex, topology.getCellsByFirstLinkIndex().data(), topology.getNLinks()); + copyToDevice(dCellsByFirstLink, topology.getCellsByFirstLink().data(), topology.getNCellsByFirstLink()); mDeviceTrackerTopologyViews[iteration] = topology.getDeviceView(dLinks, dCells, dCellsByFirstLinkIndex, dCellsByFirstLink); } if (!mDeviceTrackerTopologyViews.empty()) { @@ -361,27 +371,21 @@ void TimeFrameGPU::loadTrackingTopologies() } template -void TimeFrameGPU::updateROFVertexLookupTable() +void TimeFrameGPU::uploadROFVertexLookupTable() { + GPUTimer timer("updating device view of ROFVertexLookupTable"); const auto& hostTable = this->getROFVertexLookupTable(); - { - GPUTimer timer("updating device view of ROFVertexLookupTable"); - const auto& hostView = this->getROFVertexLookupTableView(); - using TableEntry = ROFVertexLookupTable::TableEntry; - TableEntry* d_flatTable{nullptr}; - size_t flatTableSize = hostTable.getFlatTableSize(); - allocMem(reinterpret_cast(&d_flatTable), flatTableSize * sizeof(TableEntry), this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpy(d_flatTable, hostView.mFlatTable, flatTableSize * sizeof(TableEntry), cudaMemcpyHostToDevice)); - mDeviceROFVertexLookupTableView = hostTable.getDeviceView(d_flatTable, hostView.mIndices, hostView.mLayers); - } + const auto& hostView = this->getROFVertexLookupTableView(); + using TableEntry = ROFVertexLookupTable::TableEntry; + auto* dFlatTable = allocDevice(hostTable.getFlatTableSize()); + copyToDevice(dFlatTable, hostView.mFlatTable, hostTable.getFlatTableSize()); + mDeviceROFVertexLookupTableView = hostTable.getDeviceView(dFlatTable, mDeviceROFVertexLookupTableView.mIndices, mDeviceROFVertexLookupTableView.mLayers); } template void TimeFrameGPU::createTrackletsLUTDeviceArray() { - { - allocMem(reinterpret_cast(&mTrackletsLUTDeviceArray), MaxLinks * sizeof(int*), this->hasFrameworkAllocator()); - } + mTrackletsLUTDeviceArray = allocSlotArray(MaxLinks); } template @@ -389,11 +393,9 @@ void TimeFrameGPU::createTrackletsLUTDevice(bool allocate, const int la { GPUTimer timer(mGpuStreams[layer], "creating tracklets LUTs", layer); const int fromLayer = this->mTrackingTopologyView.getLink(layer).fromLayer; - const int ncls = this->mClusters[fromLayer].size() + 1; + const size_t ncls = this->mClusters[fromLayer].size() + 1; if (allocate || mTrackletsLUTDevice[layer] == nullptr) { - GPULog("gpu-allocation: creating tracklets LUT for {} elements on layer {}, for {:.2f} MB.", ncls, layer, ncls * sizeof(int) / constants::MB); - allocMemAsync(reinterpret_cast(&mTrackletsLUTDevice[layer]), ncls * sizeof(int), mGpuStreams[layer], this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemcpyAsync(&mTrackletsLUTDeviceArray[layer], &mTrackletsLUTDevice[layer], sizeof(int*), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); + createSlot(mTrackletsLUTDevice, mTrackletsLUTDeviceArray, layer, ncls, "tracklets LUT"); } GPUChkErrS(cudaMemsetAsync(mTrackletsLUTDevice[layer], 0, ncls * sizeof(int), mGpuStreams[layer].get())); } @@ -401,90 +403,31 @@ void TimeFrameGPU::createTrackletsLUTDevice(bool allocate, const int la template void TimeFrameGPU::createTrackletsBuffersArray() { - { - GPUTimer timer("creating tracklet buffers array"); - allocMem(reinterpret_cast(&mTrackletsDeviceArray), MaxLinks * sizeof(Tracklet*), this->hasFrameworkAllocator()); - } + GPUTimer timer("creating tracklet buffers array"); + mTrackletsDeviceArray = allocSlotArray(MaxLinks); } template -void TimeFrameGPU::createTrackletsBuffers(const int layer) +void TimeFrameGPU::createTrackletsBuffers(const int layer, size_t capacity) { GPUTimer timer(mGpuStreams[layer], "creating tracklet buffers", layer); mNTracklets[layer] = 0; - const int fromLayer = this->mTrackingTopologyView.getLink(layer).fromLayer; - GPUChkErrS(cudaMemcpyAsync(&mNTracklets[layer], mTrackletsLUTDevice[layer] + this->mClusters[fromLayer].size(), sizeof(int), cudaMemcpyDeviceToHost, mGpuStreams[layer].get())); - mGpuStreams[layer].sync(); // ensure number of tracklets is correct - GPULog("gpu-transfer: creating tracklets buffer for {} elements on layer {}, for {:.2f} MB.", mNTracklets[layer], layer, mNTracklets[layer] * sizeof(Tracklet) / constants::MB); - allocMemAsync(reinterpret_cast(&mTrackletsDevice[layer]), mNTracklets[layer] * sizeof(Tracklet), mGpuStreams[layer], this->hasFrameworkAllocator(), (o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK)); - GPUChkErrS(cudaMemsetAsync(mTrackletsDevice[layer], 0, mNTracklets[layer] * sizeof(Tracklet), mGpuStreams[layer].get())); - GPUChkErrS(cudaMemcpyAsync(&mTrackletsDeviceArray[layer], &mTrackletsDevice[layer], sizeof(Tracklet*), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); -} - -template -void TimeFrameGPU::loadTrackletsDevice() -{ - GPUTimer timer(mGpuStreams, "loading tracklets", NLayers - 1); - for (auto iLayer{0}; iLayer < NLayers - 1; ++iLayer) { - GPULog("gpu-transfer: loading {} tracklets on layer {}, for {:.2f} MB.", this->mTracklets[iLayer].size(), iLayer, this->mTracklets[iLayer].size() * sizeof(Tracklet) / constants::MB); - GPUChkErrS(cudaHostRegister(this->mTracklets[iLayer].data(), this->mTracklets[iLayer].size() * sizeof(Tracklet), cudaHostRegisterPortable)); - GPUChkErrS(cudaMemcpyAsync(mTrackletsDevice[iLayer], this->mTracklets[iLayer].data(), this->mTracklets[iLayer].size() * sizeof(Tracklet), cudaMemcpyHostToDevice, mGpuStreams[iLayer].get())); - } -} - -template -void TimeFrameGPU::loadTrackletsLUTDevice() -{ - GPUTimer timer("loading tracklets"); - for (auto iLayer{0}; iLayer < NLayers - 2; ++iLayer) { - GPULog("gpu-transfer: loading tracklets LUT for {} elements on layer {}, for {:.2f} MB", this->mTrackletsLookupTable[iLayer].size(), iLayer + 1, this->mTrackletsLookupTable[iLayer].size() * sizeof(int) / constants::MB); - GPUChkErrS(cudaMemcpyAsync(mTrackletsLUTDevice[iLayer + 1], this->mTrackletsLookupTable[iLayer].data(), this->mTrackletsLookupTable[iLayer].size() * sizeof(int), cudaMemcpyHostToDevice, mGpuStreams[iLayer].get())); - } - mGpuStreams.sync(); - GPUChkErrS(cudaMemcpy(mTrackletsLUTDeviceArray, mTrackletsLUTDevice.data(), (NLayers - 1) * sizeof(int*), cudaMemcpyHostToDevice)); -} - -template -void TimeFrameGPU::createNeighboursIndexTablesDevice(const int layer) -{ - GPUTimer timer(mGpuStreams[layer], "creating cells neighbours", layer); - GPULog("gpu-transfer: reserving neighbours LUT for {} elements on layer {}, for {:.2f} MB.", mNCells[layer] + 1, layer, (mNCells[layer] + 1) * sizeof(int) / constants::MB); - allocMemAsync(reinterpret_cast(&mNeighboursIndexTablesDevice[layer]), (mNCells[layer] + 1) * sizeof(int), mGpuStreams[layer], this->hasFrameworkAllocator(), (o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK)); - GPUChkErrS(cudaMemsetAsync(mNeighboursIndexTablesDevice[layer], 0, (mNCells[layer] + 1) * sizeof(int), mGpuStreams[layer].get())); + createSlot(mTrackletsDevice, mTrackletsDeviceArray, layer, capacity, "tracklets buffer", SlotInit::Raw, o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK); } template void TimeFrameGPU::createNeighboursLUTDevice(const int layer, const unsigned int nCells) { GPUTimer timer(mGpuStreams[layer], "reserving neighboursLUT"); - GPULog("gpu-allocation: reserving neighbours LUT for {} elements on layer {} , for {:.2f} MB.", nCells + 1, layer, (nCells + 1) * sizeof(int) / constants::MB); - allocMemAsync(reinterpret_cast(&mNeighboursLUTDevice[layer]), (nCells + 1) * sizeof(int), mGpuStreams[layer], this->hasFrameworkAllocator(), (o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK)); // We need one element more to move exc -> inc - GPUChkErrS(cudaMemsetAsync(mNeighboursLUTDevice[layer], 0, (nCells + 1) * sizeof(int), mGpuStreams[layer].get())); - GPUChkErrS(cudaMemcpyAsync(&mNeighboursCellLUTDeviceArray[layer], &mNeighboursLUTDevice[layer], sizeof(int*), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); -} - -template -void TimeFrameGPU::loadCellsDevice() -{ - GPUTimer timer(mGpuStreams, "loading cell seeds", NLayers - 2); - for (auto iLayer{0}; iLayer < NLayers - 2; ++iLayer) { - GPULog("gpu-transfer: loading {} cell seeds on layer {}, for {:.2f} MB.", this->mCells[iLayer].size(), iLayer, this->mCells[iLayer].size() * sizeof(CellSeed) / constants::MB); - allocMemAsync(reinterpret_cast(&mCellsDevice[iLayer]), this->mCells[iLayer].size() * sizeof(CellSeed), mGpuStreams[iLayer], this->hasFrameworkAllocator()); - allocMemAsync(reinterpret_cast(&mNeighboursIndexTablesDevice[iLayer]), (this->mCells[iLayer].size() + 1) * sizeof(int), mGpuStreams[iLayer], this->hasFrameworkAllocator()); // accessory for the neigh. finding. - GPUChkErrS(cudaMemsetAsync(mNeighboursIndexTablesDevice[iLayer], 0, (this->mCells[iLayer].size() + 1) * sizeof(int), mGpuStreams[iLayer].get())); - GPUChkErrS(cudaMemcpyAsync(mCellsDevice[iLayer], this->mCells[iLayer].data(), this->mCells[iLayer].size() * sizeof(CellSeed), cudaMemcpyHostToDevice, mGpuStreams[iLayer].get())); - } + createSlot(mNeighboursLUTDevice, mNeighboursCellLUTDeviceArray, layer, nCells + 1, "neighbours LUT", SlotInit::Zero, o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK); } template void TimeFrameGPU::createCellsLUTDeviceArray() { - { - GPUTimer timer("creating cells LUTs array"); - allocMem(reinterpret_cast(&mCellsLUTDeviceArray), MaxCells * sizeof(int*), this->hasFrameworkAllocator()); - allocMem(reinterpret_cast(&mNeighboursCellLUTDeviceArray), MaxCells * sizeof(int*), this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemset(mNeighboursCellLUTDeviceArray, 0, MaxCells * sizeof(int*))); - } + GPUTimer timer("creating cells LUTs array"); + mCellsLUTDeviceArray = allocSlotArray(MaxCells); + mNeighboursCellLUTDeviceArray = allocSlotArray(MaxCells); } template @@ -492,150 +435,74 @@ void TimeFrameGPU::createCellsLUTDevice(const int layer) { GPUTimer timer(mGpuStreams[layer], "creating cells LUTs", layer); const int firstLink = this->mTrackingTopologyView.getCell(layer).firstLink; - GPULog("gpu-transfer: creating cell LUT for {} elements on layer {}, for {:.2f} MB.", mNTracklets[firstLink] + 1, layer, (mNTracklets[firstLink] + 1) * sizeof(int) / constants::MB); - allocMemAsync(reinterpret_cast(&mCellsLUTDevice[layer]), (mNTracklets[firstLink] + 1) * sizeof(int), mGpuStreams[layer], this->hasFrameworkAllocator(), (o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK)); - GPUChkErrS(cudaMemsetAsync(mCellsLUTDevice[layer], 0, (mNTracklets[firstLink] + 1) * sizeof(int), mGpuStreams[layer].get())); - GPUChkErrS(cudaMemcpyAsync(&mCellsLUTDeviceArray[layer], &mCellsLUTDevice[layer], sizeof(int*), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); + createSlot(mCellsLUTDevice, mCellsLUTDeviceArray, layer, mNTracklets[firstLink] + 1, "cells LUT", SlotInit::Zero, o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK); } template void TimeFrameGPU::createCellsBuffersArray() { - { - GPUTimer timer("creating cells buffers array"); - allocMem(reinterpret_cast(&mCellsDeviceArray), MaxCells * sizeof(CellSeed*), this->hasFrameworkAllocator()); - allocMem(reinterpret_cast(&mNeighboursDeviceArray), MaxCells * sizeof(CellNeighbour*), this->hasFrameworkAllocator()); - GPUChkErrS(cudaMemset(mNeighboursDeviceArray, 0, MaxCells * sizeof(CellNeighbour*))); - GPUChkErrS(cudaMemcpy(mCellsDeviceArray, mCellsDevice.data(), mCellsDevice.size() * sizeof(CellSeed*), cudaMemcpyHostToDevice)); - } + GPUTimer timer("creating cells buffers array"); + mCellsDeviceArray = allocSlotArray(MaxCells); + mNeighboursDeviceArray = allocSlotArray(MaxCells); } template -void TimeFrameGPU::createCellsBuffers(const int layer) +void TimeFrameGPU::createCellsBuffers(const int layer, size_t capacity) { GPUTimer timer(mGpuStreams[layer], "creating cells buffers"); mNCells[layer] = 0; - const int firstLink = this->mTrackingTopologyView.getCell(layer).firstLink; - GPUChkErrS(cudaMemcpyAsync(&mNCells[layer], mCellsLUTDevice[layer] + mNTracklets[firstLink], sizeof(int), cudaMemcpyDeviceToHost, mGpuStreams[layer].get())); - mGpuStreams[layer].sync(); // ensure number of cells is correct - GPULog("gpu-transfer: creating cell buffer for {} elements on layer {}, for {:.2f} MB.", mNCells[layer], layer, mNCells[layer] * sizeof(CellSeed) / constants::MB); - allocMemAsync(reinterpret_cast(&mCellsDevice[layer]), mNCells[layer] * sizeof(CellSeed), mGpuStreams[layer], this->hasFrameworkAllocator(), (o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK)); - GPUChkErrS(cudaMemsetAsync(mCellsDevice[layer], 0, mNCells[layer] * sizeof(CellSeed), mGpuStreams[layer].get())); - GPUChkErrS(cudaMemcpyAsync(&mCellsDeviceArray[layer], &mCellsDevice[layer], sizeof(CellSeed*), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); + createSlot(mCellsDevice, mCellsDeviceArray, layer, capacity, "cells buffer", SlotInit::Raw, o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK); } template -void TimeFrameGPU::loadCellsLUTDevice() +void TimeFrameGPU::createNeighboursDevice(const unsigned int layer, size_t capacity) { - GPUTimer timer(mGpuStreams, "loading cells LUTs", NLayers - 3); - for (auto iLayer{0}; iLayer < NLayers - 3; ++iLayer) { - GPULog("gpu-transfer: loading cell LUT for {} elements on layer {}, for {:.2f} MB.", this->mCellsLookupTable[iLayer].size(), iLayer, this->mCellsLookupTable[iLayer].size() * sizeof(int) / constants::MB); - GPUChkErrS(cudaHostRegister(this->mCellsLookupTable[iLayer].data(), this->mCellsLookupTable[iLayer].size() * sizeof(int), cudaHostRegisterPortable)); - GPUChkErrS(cudaMemcpyAsync(mCellsLUTDevice[iLayer + 1], this->mCellsLookupTable[iLayer].data(), this->mCellsLookupTable[iLayer].size() * sizeof(int), cudaMemcpyHostToDevice, mGpuStreams[iLayer].get())); - } + GPUTimer timer(mGpuStreams[layer], "reserving neighbours", layer); + this->mNNeighbours[layer] = 0; + createSlot(mNeighboursDevice, mNeighboursDeviceArray, layer, capacity, "neighbours buffer", SlotInit::Raw, o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK); } template -void TimeFrameGPU::loadTrackSeedsDevice(bounded_vector& seeds) +void TimeFrameGPU::createTrackSeedsDevice(const size_t capacity) { - GPUTimer timer("loading track seeds"); - GPULog("gpu-transfer: loading {} track seeds, for {:.2f} MB.", seeds.size(), seeds.size() * sizeof(TrackSeedN) / constants::MB); - allocMem(reinterpret_cast(&mTrackSeedsDevice), seeds.size() * sizeof(TrackSeedN), this->hasFrameworkAllocator(), (o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK)); - GPUChkErrS(cudaMemcpy(mTrackSeedsDevice, seeds.data(), seeds.size() * sizeof(TrackSeedN), cudaMemcpyHostToDevice)); - GPULog("gpu-transfer: creating {} track seeds LUT, for {:.2f} MB.", seeds.size() + 1, (seeds.size() + 1) * sizeof(int) / constants::MB); - allocMem(reinterpret_cast(&mTrackSeedsLUTDevice), (seeds.size() + 1) * sizeof(int), this->hasFrameworkAllocator(), (o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK)); - GPUChkErrS(cudaMemset(mTrackSeedsLUTDevice, 0, (seeds.size() + 1) * sizeof(int))); + GPUTimer timer("reserving track seeds"); + GPULog("gpu-allocation: reserving {} track seeds, for {:.2f} MB.", capacity, capacity * sizeof(TrackSeedN) / constants::MB); + mTrackSeedsDevice = allocDevice(capacity, o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK); } template -void TimeFrameGPU::createNeighboursDevice(const unsigned int layer) +void TimeFrameGPU::createTrackITSExtDevice(const size_t capacity) { - GPUTimer timer(mGpuStreams[layer], "reserving neighbours", layer); - this->mNNeighbours[layer] = 0; - if (this->mNCells[layer] == 0) { - mNeighboursDevice[layer] = nullptr; - GPUChkErrS(cudaMemcpyAsync(&mNeighboursDeviceArray[layer], &mNeighboursDevice[layer], sizeof(CellNeighbour*), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); - return; - } - GPUChkErrS(cudaMemcpyAsync(&(this->mNNeighbours[layer]), &(mNeighboursLUTDevice[layer][this->mNCells[layer]]), sizeof(unsigned int), cudaMemcpyDeviceToHost, mGpuStreams[layer].get())); - mGpuStreams[layer].sync(); // ensure number of neighbours is correct - if (this->mNNeighbours[layer] == 0) { - mNeighboursDevice[layer] = nullptr; - GPUChkErrS(cudaMemcpyAsync(&mNeighboursDeviceArray[layer], &mNeighboursDevice[layer], sizeof(CellNeighbour*), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); - return; + GPUTimer timer("reserving tracks"); + GPULog("gpu-allocation: reserving {} tracks, for {:.2f} MB.", capacity, capacity * sizeof(o2::its::TrackITSExt) / constants::MB); + mTrackITSExtDevice = allocDevice(capacity, o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK); + if (capacity > 0) { + GPUChkErrS(cudaMemsetAsync(mTrackITSExtDevice, 0, capacity * sizeof(o2::its::TrackITSExt), Stream::DefaultStream)); } - GPULog("gpu-allocation: reserving {} neighbours, for {:.2f} MB.", this->mNNeighbours[layer], (this->mNNeighbours[layer]) * sizeof(CellNeighbour) / constants::MB); - allocMemAsync(reinterpret_cast(&mNeighboursDevice[layer]), (this->mNNeighbours[layer]) * sizeof(CellNeighbour), mGpuStreams[layer], this->hasFrameworkAllocator(), (o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK)); - GPUChkErrS(cudaMemsetAsync(mNeighboursDevice[layer], -1, (this->mNNeighbours[layer]) * sizeof(CellNeighbour), mGpuStreams[layer].get())); - GPUChkErrS(cudaMemcpyAsync(&mNeighboursDeviceArray[layer], &mNeighboursDevice[layer], sizeof(CellNeighbour*), cudaMemcpyHostToDevice, mGpuStreams[layer].get())); + GPULog("gpu-allocation: reserving {} track indices, for {:.2f} MB.", capacity, capacity * sizeof(int) / constants::MB); + mTrackIndicesDevice = allocDevice(capacity, o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK); + mTrackSeedIndicesDevice = allocDevice(capacity, o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK); + mTrackCounterDevice = allocDevice(1, o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK); } template -void TimeFrameGPU::createTrackITSExtDevice(const size_t nSeeds) +void TimeFrameGPU::createTrackITSExtHost(const size_t nTracks) { - GPUTimer timer("reserving tracks"); - mNTracks = 0; - GPUChkErrS(cudaMemcpy(&mNTracks, mTrackSeedsLUTDevice + nSeeds, sizeof(int), cudaMemcpyDeviceToHost)); - GPULog("gpu-allocation: reserving {} tracks, for {:.2f} MB.", mNTracks, mNTracks * sizeof(o2::its::TrackITSExt) / constants::MB); - mTrackITSExt = bounded_vector(mNTracks, {}, this->getMemoryPool().get()); - mTrackIndices = bounded_vector(mNTracks, 0, this->getMemoryPool().get()); - allocMem(reinterpret_cast(&mTrackITSExtDevice), mNTracks * sizeof(o2::its::TrackITSExt), this->hasFrameworkAllocator(), (o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK)); - GPUChkErrS(cudaMemset(mTrackITSExtDevice, 0, mNTracks * sizeof(o2::its::TrackITSExt))); - GPULog("gpu-allocation: reserving {} track indices, for {:.2f} MB.", mNTracks, mNTracks * sizeof(int) / constants::MB); - allocMem(reinterpret_cast(&mTrackIndicesDevice), mNTracks * sizeof(int), this->hasFrameworkAllocator(), (o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK)); + GPUTimer timer("reserving host tracks"); + mNTracks = nTracks; + mTrackITSExt = bounded_vector(nTracks, {}, this->getMemoryPool().get()); + mTrackIndices = bounded_vector(nTracks, 0, this->getMemoryPool().get()); } template void TimeFrameGPU::createTrackExtensionScratchDevice(const int nThreads, const int maxHypotheses) { GPUTimer timer("reserving track extension scratch"); + using Hypothesis = o2::its::TrackExtensionHypothesis; const size_t nHypotheses = static_cast(std::max(1, nThreads)) * std::max(1, maxHypotheses); - GPULog("gpu-allocation: reserving {} track extension hypotheses per scratch buffer, for {:.2f} MB each.", nHypotheses, nHypotheses * sizeof(o2::its::TrackExtensionHypothesis) / constants::MB); - mActiveTrackExtensionHypothesesDevice = nullptr; - mNextTrackExtensionHypothesesDevice = nullptr; - if (nHypotheses == 0) { - return; - } - allocMem(reinterpret_cast(&mActiveTrackExtensionHypothesesDevice), nHypotheses * sizeof(o2::its::TrackExtensionHypothesis), this->hasFrameworkAllocator(), (o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK)); - allocMem(reinterpret_cast(&mNextTrackExtensionHypothesesDevice), nHypotheses * sizeof(o2::its::TrackExtensionHypothesis), this->hasFrameworkAllocator(), (o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK)); -} - -template -void TimeFrameGPU::downloadCellsDevice() -{ - GPUTimer timer(mGpuStreams, "downloading cells", NLayers - 2); - for (int iLayer{0}; iLayer < NLayers - 2; ++iLayer) { - GPULog("gpu-transfer: downloading {} cells on layer: {}, for {:.2f} MB.", mNCells[iLayer], iLayer, mNCells[iLayer] * sizeof(CellSeed) / constants::MB); - this->mCells[iLayer].resize(mNCells[iLayer]); - GPUChkErrS(cudaMemcpyAsync(this->mCells[iLayer].data(), this->mCellsDevice[iLayer], mNCells[iLayer] * sizeof(CellSeed), cudaMemcpyDeviceToHost, mGpuStreams[iLayer].get())); - } -} - -template -void TimeFrameGPU::downloadCellsLUTDevice() -{ - GPUTimer timer(mGpuStreams, "downloading cell luts", NLayers - 3); - for (auto iLayer{0}; iLayer < NLayers - 3; ++iLayer) { - GPULog("gpu-transfer: downloading cells lut on layer {} for {} elements", iLayer, (mNTracklets[iLayer + 1] + 1)); - this->mCellsLookupTable[iLayer].resize(mNTracklets[iLayer + 1] + 1); - GPUChkErrS(cudaMemcpyAsync(this->mCellsLookupTable[iLayer].data(), mCellsLUTDevice[iLayer + 1], (mNTracklets[iLayer + 1] + 1) * sizeof(int), cudaMemcpyDeviceToHost, mGpuStreams[iLayer].get())); - } -} - -template -void TimeFrameGPU::downloadCellsNeighboursDevice(std::vector>& neighbours, const int layer) -{ - GPUTimer timer(mGpuStreams[layer], "downloading neighbours from layer", layer); - GPULog("gpu-transfer: downloading {} neighbours, for {:.2f} MB.", neighbours[layer].size(), neighbours[layer].size() * sizeof(CellNeighbour) / constants::MB); - GPUChkErrS(cudaMemcpyAsync(neighbours[layer].data(), mNeighboursDevice[layer], neighbours[layer].size() * sizeof(CellNeighbour), cudaMemcpyDeviceToHost, mGpuStreams[layer].get())); -} - -template -void TimeFrameGPU::downloadNeighboursLUTDevice(bounded_vector& lut, const int layer) -{ - GPUTimer timer(mGpuStreams[layer], "downloading neighbours LUT from layer", layer); - GPULog("gpu-transfer: downloading neighbours LUT for {} elements on layer {}, for {:.2f} MB.", lut.size(), layer, lut.size() * sizeof(int) / constants::MB); - GPUChkErrS(cudaMemcpyAsync(lut.data(), mNeighboursLUTDevice[layer], lut.size() * sizeof(int), cudaMemcpyDeviceToHost, mGpuStreams[layer].get())); + GPULog("gpu-allocation: reserving {} track extension hypotheses per scratch buffer, for {:.2f} MB each.", nHypotheses, nHypotheses * sizeof(Hypothesis) / constants::MB); + mActiveTrackExtensionHypothesesDevice = allocDevice(nHypotheses, o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK); + mNextTrackExtensionHypothesesDevice = allocDevice(nHypotheses, o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK); } template @@ -643,7 +510,7 @@ void TimeFrameGPU::downloadTrackITSExtDevice() { GPUTimer timer("downloading tracks"); GPULog("gpu-transfer: downloading {} tracks, for {:.2f} MB.", mTrackITSExt.size(), mTrackITSExt.size() * sizeof(o2::its::TrackITSExt) / constants::MB); - GPUChkErrS(cudaMemcpy(mTrackITSExt.data(), mTrackITSExtDevice, mTrackITSExt.size() * sizeof(o2::its::TrackITSExt), cudaMemcpyDeviceToHost)); + copyFromDevice(mTrackITSExt.data(), mTrackITSExtDevice, mTrackITSExt.size()); } template @@ -651,42 +518,33 @@ void TimeFrameGPU::downloadTrackIndicesDevice() { GPUTimer timer("downloading track indices"); GPULog("gpu-transfer: downloading {} track indices, for {:.2f} MB.", mTrackIndices.size(), mTrackIndices.size() * sizeof(int) / constants::MB); - GPUChkErrS(cudaMemcpy(mTrackIndices.data(), mTrackIndicesDevice, mTrackIndices.size() * sizeof(int), cudaMemcpyDeviceToHost)); + copyFromDevice(mTrackIndices.data(), mTrackIndicesDevice, mTrackIndices.size()); } template -void TimeFrameGPU::unregisterHostMemory(const int maxLayers) +void TimeFrameGPU::unregisterHostMemory() { GPUTimer timer("unregistering host memory"); GPULog("unregistering host memory"); - auto checkedUnregisterEntry = [](auto& bits, auto& vec, int layer) { - if (bits.test(layer)) { - GPUChkErrS(cudaHostUnregister(vec[layer].data())); - bits.reset(layer); + auto unpin = [](auto& pinned, auto& layers, auto& slots) { + for (auto iLayer{0}; iLayer < NLayers; ++iLayer) { + if (pinned.test(iLayer)) { + GPUChkErrS(cudaHostUnregister(layers[iLayer].data())); + } } - }; - auto checkedUnregisterArray = [](auto& bits, auto& vec) { - if (bits.test(NLayers)) { - GPUChkErrS(cudaHostUnregister(vec.data())); - bits.reset(NLayers); + if (pinned.test(NLayers)) { + GPUChkErrS(cudaHostUnregister(slots.data())); } + pinned.reset(); }; - for (auto iLayer{0}; iLayer < NLayers; ++iLayer) { - checkedUnregisterEntry(mPinnedUsedClusters, this->mUsedClusters, iLayer); - checkedUnregisterEntry(mPinnedUnsortedClusters, this->mUnsortedClusters, iLayer); - checkedUnregisterEntry(mPinnedClusters, this->mClusters, iLayer); - checkedUnregisterEntry(mPinnedClustersIndexTables, this->mIndexTables, iLayer); - checkedUnregisterEntry(mPinnedTrackingFrameInfo, this->mTrackingFrameInfo, iLayer); - checkedUnregisterEntry(mPinnedROFramesClusters, this->mROFramesClusters, iLayer); - } - checkedUnregisterArray(mPinnedUsedClusters, mUsedClustersDevice); - checkedUnregisterArray(mPinnedUnsortedClusters, mUnsortedClustersDevice); - checkedUnregisterArray(mPinnedClusters, mClustersDevice); - checkedUnregisterArray(mPinnedClustersIndexTables, mClustersIndexTablesDevice); - checkedUnregisterArray(mPinnedTrackingFrameInfo, mTrackingFrameInfoDevice); - checkedUnregisterArray(mPinnedROFramesClusters, mROFramesClustersDevice); + unpin(mPinnedUsedClusters, this->mUsedClusters, mUsedClustersDevice); + unpin(mPinnedUnsortedClusters, this->mUnsortedClusters, mUnsortedClustersDevice); + unpin(mPinnedClusters, this->mClusters, mClustersDevice); + unpin(mPinnedClustersIndexTables, this->mIndexTables, mClustersIndexTablesDevice); + unpin(mPinnedTrackingFrameInfo, this->mTrackingFrameInfo, mTrackingFrameInfoDevice); + unpin(mPinnedROFramesClusters, this->mROFramesClusters, mROFramesClustersDevice); } namespace detail @@ -738,12 +596,6 @@ void TimeFrameGPU::initialise(const TrackingParameters& trkParam, int m } } -template -void TimeFrameGPU::syncStream(const size_t stream) -{ - mGpuStreams[stream].sync(); -} - template void TimeFrameGPU::syncStreams(const bool device) { @@ -762,18 +614,10 @@ void TimeFrameGPU::recordEvent(const int event) mGpuStreams[event].record(); } -template -void TimeFrameGPU::recordEvents(const int start, const int end) -{ - for (int i{start}; i < end; ++i) { - recordEvent(i); - } -} - template void TimeFrameGPU::wipe() { - unregisterHostMemory(0); + unregisterHostMemory(); o2::its::TimeFrame::wipe(); } diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx index b773553bf3849..0392bf9fd2dbd 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx @@ -12,11 +12,9 @@ #include -#include -#include - #include "ITStrackingGPU/TrackerTraitsGPU.h" #include "ITStrackingGPU/TrackingKernels.h" +#include "ITStrackingGPU/LaunchGeometry.h" #include "ITStracking/Configuration.h" namespace o2::its @@ -25,6 +23,10 @@ template void TrackerTraitsGPU::initialiseTimeFrame(const int iteration) { mTimeFrameGPU->initialise(this->mTrkParams[iteration], NLayers, iteration); + // Small per-iteration float arrays the fitting kernels read. Uploaded here rather than + // rebuilt inside every handler call, where the allocate/copy/free cycle cost more host time + // than the transfer. + mTimeFrameGPU->loadIterationParameters(this->mTrkParams[iteration]); if (this->mTrkParams[iteration].PassFlags[IterationStep::FirstPass]) { // on default stream @@ -33,8 +35,8 @@ void TrackerTraitsGPU::initialiseTimeFrame(const int iteration) mTimeFrameGPU->loadROFOverlapTable(); // this can be put in constant memory actually mTimeFrameGPU->loadROFVertexLookupTable(); mTimeFrameGPU->loadTrackingTopologies(); - // once the tables are in persistent memory just update the vertex one - // mTimeFrameGPU->updateROFVertexLookupTable(); + // once the tables are in persistent memory just re-upload the vertex one + // mTimeFrameGPU->uploadROFVertexLookupTable(); mTimeFrameGPU->loadIndexTableUtils(); // pinned on host mTimeFrameGPU->createUsedClustersDeviceArray(); @@ -85,76 +87,47 @@ void TrackerTraitsGPU::computeLayerTracklets(const int iteration, int i // With per-primary-vertex processing, the chain is called once per vertex while initialisation is only done once. mTimeFrameGPU->pushMemoryStack(iteration); + const auto nClusters = mTimeFrameGPU->getClusterSizes(); for (int linkId{0}; linkId < hostTopology.nLinks; ++linkId) { const auto link = hostTopology.getLink(linkId); mTimeFrameGPU->waitEvent(linkId, link.fromLayer); mTimeFrameGPU->waitEvent(linkId, link.toLayer); - countTrackletsInROFsHandler(mTimeFrameGPU->getDeviceIndexTableUtils(), - mTimeFrameGPU->getDeviceROFMaskTableView(), - linkId, - link.fromLayer, - link.toLayer, - mTimeFrameGPU->getDeviceROFOverlapTableView(), - mTimeFrameGPU->getDeviceROFVertexLookupTableView(), - iVertex, - mTimeFrameGPU->getDeviceVertices(), - mTimeFrameGPU->getDeviceROFramesPV(), - mTimeFrameGPU->getDeviceArrayClusters(), - mTimeFrameGPU->getClusterSizes(), - mTimeFrameGPU->getDeviceROFrameClusters(), - (const uint8_t**)mTimeFrameGPU->getDeviceArrayUsedClusters(), - mTimeFrameGPU->getDeviceArrayClustersIndexTables(), - mTimeFrameGPU->getDeviceArrayTrackletsLUT(), - mTimeFrameGPU->getDeviceTrackletsLUTs(), - this->mTrkParams[iteration].PassFlags[IterationStep::SelectUPCVertices], - this->mTrkParams[iteration].NSigmaCut, - topology, - mTimeFrameGPU->getLinkPhiCuts(), - this->mTrkParams[iteration].PVres, - mTimeFrameGPU->getMinRs(), - mTimeFrameGPU->getMaxRs(), - mTimeFrameGPU->getPositionResolutions(), - this->mTrkParams[iteration].LayerRadii, - mTimeFrameGPU->getLinkMSAngles(), - mTimeFrameGPU->getFrameworkAllocator(), - mTimeFrameGPU->getStreams()); - mTimeFrameGPU->createTrackletsBuffers(linkId); - if (mTimeFrameGPU->getNTracklets()[linkId] == 0) { - mTimeFrameGPU->recordEvent(linkId); - continue; - } - computeTrackletsInROFsHandler(mTimeFrameGPU->getDeviceIndexTableUtils(), - mTimeFrameGPU->getDeviceROFMaskTableView(), - linkId, - link.fromLayer, - link.toLayer, - mTimeFrameGPU->getDeviceROFOverlapTableView(), - mTimeFrameGPU->getDeviceROFVertexLookupTableView(), - iVertex, - mTimeFrameGPU->getDeviceVertices(), - mTimeFrameGPU->getDeviceROFramesPV(), - mTimeFrameGPU->getDeviceArrayClusters(), - mTimeFrameGPU->getClusterSizes(), - mTimeFrameGPU->getDeviceROFrameClusters(), - (const uint8_t**)mTimeFrameGPU->getDeviceArrayUsedClusters(), - mTimeFrameGPU->getDeviceArrayClustersIndexTables(), - mTimeFrameGPU->getDeviceArrayTracklets(), - mTimeFrameGPU->getDeviceTracklets(), - mTimeFrameGPU->getNTracklets(), - mTimeFrameGPU->getDeviceArrayTrackletsLUT(), - mTimeFrameGPU->getDeviceTrackletsLUTs(), - this->mTrkParams[iteration].PassFlags[IterationStep::SelectUPCVertices], - this->mTrkParams[iteration].NSigmaCut, - topology, - mTimeFrameGPU->getLinkPhiCuts(), - this->mTrkParams[iteration].PVres, - mTimeFrameGPU->getMinRs(), - mTimeFrameGPU->getMaxRs(), - mTimeFrameGPU->getPositionResolutions(), - this->mTrkParams[iteration].LayerRadii, - mTimeFrameGPU->getLinkMSAngles(), - mTimeFrameGPU->getFrameworkAllocator(), - mTimeFrameGPU->getStreams()); + const auto key = CapacityEstimator::makeKey(SlabSite::Tracklets, iteration, iVertex + 1, linkId); + const auto scale = static_cast(nClusters[link.fromLayer]); + runOnSlab(mTimeFrameGPU->getCapacityEstimator(), key, scale, [&](const int capacity) { + mTimeFrameGPU->createTrackletsBuffers(linkId, capacity); + return TrackingKernels::computeTrackletsInROFsHandler(mTimeFrameGPU->getDeviceIndexTableUtils(), + mTimeFrameGPU->getDeviceROFMaskTableView(), + linkId, + link.fromLayer, + link.toLayer, + mTimeFrameGPU->getDeviceROFOverlapTableView(), + mTimeFrameGPU->getDeviceROFVertexLookupTableView(), + iVertex, + mTimeFrameGPU->getDeviceVertices(), + mTimeFrameGPU->getDeviceArrayClusters(), + nClusters, + mTimeFrameGPU->getDeviceROFrameClusters(), + (const uint8_t**)mTimeFrameGPU->getDeviceArrayUsedClusters(), + mTimeFrameGPU->getDeviceArrayClustersIndexTables(), + mTimeFrameGPU->getDeviceArrayTracklets(), + mTimeFrameGPU->getDeviceTracklets(), + mTimeFrameGPU->getNTracklets(), + capacity, + mTimeFrameGPU->getDeviceTrackletsLUTs(), + this->mTrkParams[iteration].PassFlags[IterationStep::SelectUPCVertices], + this->mTrkParams[iteration].NSigmaCut, + topology, + mTimeFrameGPU->getLinkPhiCuts(), + this->mTrkParams[iteration].PVres, + mTimeFrameGPU->getMinRs(), + mTimeFrameGPU->getMaxRs(), + mTimeFrameGPU->getPositionResolutions(), + this->mTrkParams[iteration].LayerRadii, + mTimeFrameGPU->getLinkMSAngles(), + mTimeFrameGPU->getFrameworkAllocator(), + mTimeFrameGPU->getStreams()); + }); mTimeFrameGPU->recordEvent(linkId); } } @@ -188,46 +161,30 @@ void TrackerTraitsGPU::computeLayerCells(const int iteration) mTimeFrameGPU->waitEvent(cellTopologyId, first.fromLayer); mTimeFrameGPU->waitEvent(cellTopologyId, first.toLayer); mTimeFrameGPU->waitEvent(cellTopologyId, second.toLayer); - countCellsHandler(mTimeFrameGPU->getDeviceArrayClusters(), - mTimeFrameGPU->getDeviceArrayUnsortedClusters(), - mTimeFrameGPU->getDeviceArrayTrackingFrameInfo(), - mTimeFrameGPU->getDeviceArrayTracklets(), - mTimeFrameGPU->getDeviceArrayTrackletsLUT(), - currentLayerTrackletsNum, - cellTopologyId, - topology, - nullptr, - mTimeFrameGPU->getDeviceArrayCellsLUT(), - mTimeFrameGPU->getDeviceCellLUTs()[cellTopologyId], - this->mBz, - this->mTrkParams[iteration].MaxChi2ClusterAttachment, - this->mTrkParams[iteration].CellDeltaTanLambdaSigma, - this->mTrkParams[iteration].NSigmaCut, - this->mTrkParams[iteration].LayerxX0, - mTimeFrameGPU->getFrameworkAllocator(), - mTimeFrameGPU->getStreams()); - mTimeFrameGPU->createCellsBuffers(cellTopologyId); - if (mTimeFrameGPU->getNCells()[cellTopologyId] == 0) { - mTimeFrameGPU->recordEvent(cellTopologyId); - continue; - } - computeCellsHandler(mTimeFrameGPU->getDeviceArrayClusters(), - mTimeFrameGPU->getDeviceArrayUnsortedClusters(), - mTimeFrameGPU->getDeviceArrayTrackingFrameInfo(), - mTimeFrameGPU->getDeviceArrayTracklets(), - mTimeFrameGPU->getDeviceArrayTrackletsLUT(), - currentLayerTrackletsNum, - cellTopologyId, - topology, - mTimeFrameGPU->getDeviceCells()[cellTopologyId], - mTimeFrameGPU->getDeviceArrayCellsLUT(), - mTimeFrameGPU->getDeviceCellLUTs()[cellTopologyId], - this->mBz, - this->mTrkParams[iteration].MaxChi2ClusterAttachment, - this->mTrkParams[iteration].CellDeltaTanLambdaSigma, - this->mTrkParams[iteration].NSigmaCut, - this->mTrkParams[iteration].LayerxX0, - mTimeFrameGPU->getStreams()); + const auto key = CapacityEstimator::makeKey(SlabSite::Cells, iteration, 0, cellTopologyId); + const auto scale = static_cast(currentLayerTrackletsNum); + const int emitted = runOnSlab(mTimeFrameGPU->getCapacityEstimator(), key, scale, [&](const int capacity) { + mTimeFrameGPU->createCellsBuffers(cellTopologyId, capacity); + return TrackingKernels::computeCellsHandler(mTimeFrameGPU->getDeviceArrayClusters(), + mTimeFrameGPU->getDeviceArrayUnsortedClusters(), + mTimeFrameGPU->getDeviceArrayTrackingFrameInfo(), + mTimeFrameGPU->getDeviceArrayTracklets(), + mTimeFrameGPU->getDeviceArrayTrackletsLUT(), + currentLayerTrackletsNum, + cellTopologyId, + topology, + mTimeFrameGPU->getDeviceCells()[cellTopologyId], + capacity, + mTimeFrameGPU->getDeviceCellLUTs()[cellTopologyId], + this->mBz, + this->mTrkParams[iteration].MaxChi2ClusterAttachment, + this->mTrkParams[iteration].CellDeltaTanLambdaSigma, + this->mTrkParams[iteration].NSigmaCut, + mTimeFrameGPU->getDeviceLayerxX0(), + mTimeFrameGPU->getFrameworkAllocator(), + mTimeFrameGPU->getStreams()); + }); + mTimeFrameGPU->getNCells()[cellTopologyId] = emitted; mTimeFrameGPU->recordEvent(cellTopologyId); } mTimeFrameGPU->syncStreams(false); @@ -237,6 +194,8 @@ template void TrackerTraitsGPU::findCellsNeighbours(const int iteration) { const auto hostTopology = mTimeFrameGPU->getTrackingTopologyView(); + bounded_vector sourceTopologies(this->getMemoryPool().get()); + sourceTopologies.reserve(hostTopology.nCells); for (int outerLayer{0}; outerLayer < NLayers; ++outerLayer) { for (int targetCellTopologyId{0}; targetCellTopologyId < hostTopology.nCells; ++targetCellTopologyId) { const auto targetCellTopology = hostTopology.getCell(targetCellTopologyId); @@ -244,61 +203,54 @@ void TrackerTraitsGPU::findCellsNeighbours(const int iteration) continue; } const int targetCellsNum{static_cast(mTimeFrameGPU->getNCells()[targetCellTopologyId])}; - if (!targetCellsNum) { - mTimeFrameGPU->getNNeighbours()[targetCellTopologyId] = 0; - mTimeFrameGPU->recordEvent(targetCellTopologyId); - continue; - } - mTimeFrameGPU->createNeighboursIndexTablesDevice(targetCellTopologyId); - mTimeFrameGPU->createNeighboursLUTDevice(targetCellTopologyId, targetCellsNum); - + sourceTopologies.clear(); + size_t sourceCellCount{0}; for (int sourceCellTopologyId{0}; sourceCellTopologyId < hostTopology.nCells; ++sourceCellTopologyId) { const auto sourceCellTopology = hostTopology.getCell(sourceCellTopologyId); const int sourceCellsNum{static_cast(mTimeFrameGPU->getNCells()[sourceCellTopologyId])}; if (!sourceCellsNum || sourceCellTopology.secondLink != targetCellTopology.firstLink) { continue; } - mTimeFrameGPU->waitEvent(targetCellTopologyId, sourceCellTopologyId); - countCellNeighboursHandler(mTimeFrameGPU->getDeviceArrayCells(), - mTimeFrameGPU->getDeviceNeighboursIndexTables(targetCellTopologyId), - mTimeFrameGPU->getDeviceArrayCellsLUT(), - sourceCellTopologyId, - targetCellTopologyId, - this->mTrkParams[iteration].MaxChi2ClusterAttachment, - this->mBz, - sourceCellsNum, - mTimeFrameGPU->getStream(targetCellTopologyId)); + sourceTopologies.push_back(sourceCellTopologyId); + sourceCellCount += sourceCellsNum; } - - scanCellNeighboursHandler(mTimeFrameGPU->getDeviceNeighboursIndexTables(targetCellTopologyId), - mTimeFrameGPU->getDeviceNeighboursLUT(targetCellTopologyId), - targetCellsNum, - mTimeFrameGPU->getFrameworkAllocator(), - mTimeFrameGPU->getStream(targetCellTopologyId)); - - mTimeFrameGPU->createNeighboursDevice(targetCellTopologyId); - if (mTimeFrameGPU->getNNeighbours()[targetCellTopologyId] == 0) { + if (!targetCellsNum || sourceTopologies.empty()) { + mTimeFrameGPU->getNNeighbours()[targetCellTopologyId] = 0; + mTimeFrameGPU->createNeighboursDevice(targetCellTopologyId, 0); mTimeFrameGPU->recordEvent(targetCellTopologyId); continue; } + mTimeFrameGPU->createNeighboursLUTDevice(targetCellTopologyId, targetCellsNum); + auto& stream = mTimeFrameGPU->getStream(targetCellTopologyId); + int* outputCounter = mTimeFrameGPU->getDeviceNeighboursLUT(targetCellTopologyId) + targetCellsNum; - for (int sourceCellTopologyId{0}; sourceCellTopologyId < hostTopology.nCells; ++sourceCellTopologyId) { - const auto sourceCellTopology = hostTopology.getCell(sourceCellTopologyId); - const int sourceCellsNum{static_cast(mTimeFrameGPU->getNCells()[sourceCellTopologyId])}; - if (!sourceCellsNum || sourceCellTopology.secondLink != targetCellTopology.firstLink) { - continue; + const auto key = CapacityEstimator::makeKey(SlabSite::Neighbours, iteration, 0, targetCellTopologyId); + const auto scale = static_cast(sourceCellCount); + const int emitted = runOnSlab(mTimeFrameGPU->getCapacityEstimator(), key, scale, [&](const int capacity) { + mTimeFrameGPU->createNeighboursDevice(targetCellTopologyId, capacity); + resetOutputCounterHandler(outputCounter, stream); + for (const int sourceCellTopologyId : sourceTopologies) { + mTimeFrameGPU->waitEvent(targetCellTopologyId, sourceCellTopologyId); + TrackingKernels::computeCellNeighboursHandler(mTimeFrameGPU->getDeviceArrayCells(), + mTimeFrameGPU->getDeviceArrayCellsLUT(), + mTimeFrameGPU->getDeviceNeighbours(targetCellTopologyId), + outputCounter, + capacity, + sourceCellTopologyId, + targetCellTopologyId, + this->mTrkParams[iteration].MaxChi2ClusterAttachment, + this->mBz, + mTimeFrameGPU->getNCells()[sourceCellTopologyId], + stream); } - computeCellNeighboursHandler(mTimeFrameGPU->getDeviceArrayCells(), - mTimeFrameGPU->getDeviceNeighboursIndexTables(targetCellTopologyId), - mTimeFrameGPU->getDeviceArrayCellsLUT(), - mTimeFrameGPU->getDeviceNeighbours(targetCellTopologyId), - sourceCellTopologyId, - targetCellTopologyId, - this->mTrkParams[iteration].MaxChi2ClusterAttachment, - this->mBz, - sourceCellsNum, - mTimeFrameGPU->getStream(targetCellTopologyId)); - } + return finalizeCellNeighboursHandler(mTimeFrameGPU->getDeviceNeighbours(targetCellTopologyId), + mTimeFrameGPU->getDeviceNeighboursLUT(targetCellTopologyId), + targetCellsNum, + capacity, + mTimeFrameGPU->getFrameworkAllocator(), + stream); + }); + mTimeFrameGPU->getNNeighbours()[targetCellTopologyId] = emitted; mTimeFrameGPU->recordEvent(targetCellTopologyId); } } @@ -315,102 +267,104 @@ void TrackerTraitsGPU::findRoads(const int iteration) const bool extendBot = this->mTrkParams[iteration].PassFlags[IterationStep::TrackFollowerBot]; const bool extendTracks = extendTop || extendBot; for (int startLevel{this->mTrkParams[iteration].CellsPerRoad()}; startLevel >= this->mTrkParams[iteration].CellMinimumLevel(); --startLevel) { - bounded_vector> trackSeeds(this->getMemoryPool().get()); + // The cells that may start a road at this level, as the scale the estimator predicts from. + size_t startCells{0}; for (int startCellTopologyId{0}; startCellTopologyId < hostTopology.nCells; ++startCellTopologyId) { const int startLayer = hostTopology.getCell(startCellTopologyId).hitLayerMask.last(); - if (!(this->mTrkParams[iteration].StartLayerMask.has(startLayer)) || mTimeFrameGPU->getNCells()[startCellTopologyId] == 0) { - continue; + if (this->mTrkParams[iteration].StartLayerMask.has(startLayer)) { + startCells += mTimeFrameGPU->getNCells()[startCellTopologyId]; } - processNeighboursHandler(startLevel, - startCellTopologyId, - mTimeFrameGPU->getDeviceArrayCells(), - mTimeFrameGPU->getDeviceCells()[startCellTopologyId], - nullptr, - nullptr, - mTimeFrameGPU->getArrayNCells().data(), - (const uint8_t**)mTimeFrameGPU->getDeviceArrayUsedClusters(), - mTimeFrameGPU->getDeviceArrayNeighbours(), - mTimeFrameGPU->getDeviceArrayNeighboursCellLUT(), - mTimeFrameGPU->getDeviceArrayTrackingFrameInfo(), - trackSeeds, - this->mBz, - this->mTrkParams[iteration].MaxChi2ClusterAttachment, - this->mTrkParams[iteration].MaxChi2NDF, - this->mTrkParams[iteration].MaxHoles, - this->mTrkParams[iteration].getMinSeedingClusters(), - this->mTrkParams[iteration].HoleLayerMask, - this->mTrkParams[iteration].getNonSeedingLayerMask(), - this->mTrkParams[iteration].LayerxX0, - mTimeFrameGPU->getDevicePropagator(), - this->mTrkParams[iteration].CorrType, - mTimeFrameGPU->getFrameworkAllocator()); } - // fixme: I don't want to move tracks back and forth, but I need a way to use a thrust::allocator that is aware of our managed memory. - if (trackSeeds.empty()) { + if (!startCells) { + continue; + } + const auto key = CapacityEstimator::makeKey(SlabSite::TrackSeeds, iteration, startLevel, 0); + auto& estimator = mTimeFrameGPU->getCapacityEstimator(); + const int nSeeds = runOnSlab(estimator, key, static_cast(startCells), [&](const int capacity) { + mTimeFrameGPU->createTrackSeedsDevice(capacity); + int cursor{0}; + for (int startCellTopologyId{0}; startCellTopologyId < hostTopology.nCells; ++startCellTopologyId) { + const int startLayer = hostTopology.getCell(startCellTopologyId).hitLayerMask.last(); + if (!(this->mTrkParams[iteration].StartLayerMask.has(startLayer)) || mTimeFrameGPU->getNCells()[startCellTopologyId] == 0) { + continue; + } + TrackingKernels::processNeighboursHandler(startLevel, + startCellTopologyId, + mTimeFrameGPU->getDeviceArrayCells(), + mTimeFrameGPU->getDeviceCells()[startCellTopologyId], + nullptr, + nullptr, + mTimeFrameGPU->getArrayNCells().data(), + (const uint8_t**)mTimeFrameGPU->getDeviceArrayUsedClusters(), + mTimeFrameGPU->getDeviceArrayNeighbours(), + mTimeFrameGPU->getDeviceArrayNeighboursCellLUT(), + mTimeFrameGPU->getDeviceArrayTrackingFrameInfo(), + mTimeFrameGPU->getDeviceTrackSeeds(), + capacity, + cursor, + mTimeFrameGPU->getCapacityEstimator(), + iteration, + this->mBz, + this->mTrkParams[iteration].MaxChi2ClusterAttachment, + this->mTrkParams[iteration].MaxChi2NDF, + this->mTrkParams[iteration].MaxHoles, + this->mTrkParams[iteration].getMinSeedingClusters(), + this->mTrkParams[iteration].HoleLayerMask, + this->mTrkParams[iteration].getNonSeedingLayerMask(), + mTimeFrameGPU->getDeviceLayerxX0(), + mTimeFrameGPU->getDevicePropagator(), + this->mTrkParams[iteration].CorrType, + mTimeFrameGPU->getFrameworkAllocator()); + } + return cursor; }, estimator.peakCapacity(key)); + if (!nSeeds) { LOGP(debug, "No track seeds found, skipping track finding"); continue; } - mTimeFrameGPU->loadTrackSeedsDevice(trackSeeds); - - // Since TrackITSExt is an enourmous class it is better to first count how many - // successfull fits we do and only then allocate - countTrackSeedHandler(mTimeFrameGPU->getDeviceTrackSeeds(), - mTimeFrameGPU->getDeviceArrayTrackingFrameInfo(), - mTimeFrameGPU->getDeviceArrayUnsortedClusters(), - mTimeFrameGPU->getDeviceTrackSeedsLUT(), - this->mTrkParams[iteration].LayerRadii, - this->mTrkParams[iteration].MinPt, - this->mTrkParams[iteration].LayerxX0, - trackSeeds.size(), - this->mBz, - this->mTrkParams[iteration].MaxChi2ClusterAttachment, - this->mTrkParams[iteration].MaxChi2NDF, - this->mTrkParams[iteration].ReseedIfShorter, - this->mTrkParams[iteration].RepeatRefitOut, - this->mTrkParams[iteration].ShiftRefToCluster, - mTimeFrameGPU->getDevicePropagator(), - this->mTrkParams[iteration].CorrType, - mTimeFrameGPU->getFrameworkAllocator()); - mTimeFrameGPU->createTrackITSExtDevice(trackSeeds.size()); - if (extendTracks) { - mTimeFrameGPU->createTrackExtensionScratchDevice(constants::GPUThreadsTotal, this->mTrkParams[iteration].TrackFollowerMaxHypotheses); + if (extendTracks) { // independent of the slab size, so it must not be redone on a retry + mTimeFrameGPU->createTrackExtensionScratchDevice(gpu::GPUThreadsTotal, this->mTrkParams[iteration].TrackFollowerMaxHypotheses); } - computeTrackSeedHandler(mTimeFrameGPU->getDeviceTrackSeeds(), - mTimeFrameGPU->getDeviceArrayTrackingFrameInfo(), - mTimeFrameGPU->getDeviceArrayUnsortedClusters(), - mTimeFrameGPU->getDeviceIndexTableUtils(), - mTimeFrameGPU->getDeviceROFMaskTableView(), - mTimeFrameGPU->getDeviceROFOverlapTableView(), - mTimeFrameGPU->getDeviceArrayClusters(), - (const unsigned char**)mTimeFrameGPU->getDeviceArrayUsedClusters(), - mTimeFrameGPU->getDeviceArrayClustersIndexTables(), - mTimeFrameGPU->getDeviceROFrameClusters(), - mTimeFrameGPU->getDeviceTrackITSExt(), - mTimeFrameGPU->getDeviceTrackIndices(), - mTimeFrameGPU->getDeviceTrackSeedsLUT(), - extendTracks ? mTimeFrameGPU->getDeviceActiveTrackExtensionHypotheses() : nullptr, - extendTracks ? mTimeFrameGPU->getDeviceNextTrackExtensionHypotheses() : nullptr, - this->mTrkParams[iteration].LayerRadii, - this->mTrkParams[iteration].MinPt, - this->mTrkParams[iteration].LayerxX0, - trackSeeds.size(), - mTimeFrameGPU->getNTrackSeeds(), - this->mBz, - this->mTrkParams[iteration].MaxChi2ClusterAttachment, - this->mTrkParams[iteration].MaxChi2NDF, - this->mTrkParams[iteration].ReseedIfShorter, - this->mTrkParams[iteration].RepeatRefitOut, - this->mTrkParams[iteration].ShiftRefToCluster, - this->mTrkParams[iteration].NLayers, - this->mTrkParams[iteration].PhiBins, - this->mTrkParams[iteration].TrackFollowerMaxHypotheses, - extendTop, - extendBot, - this->mTrkParams[iteration].TrackFollowerNSigmaCutPhi, - this->mTrkParams[iteration].TrackFollowerNSigmaCutZ, - mTimeFrameGPU->getDevicePropagator(), - this->mTrkParams[iteration].CorrType, - mTimeFrameGPU->getFrameworkAllocator()); + const auto trackKey = CapacityEstimator::makeKey(SlabSite::Tracks, iteration, startLevel, 0); + const int nTracks = runOnSlab(estimator, trackKey, static_cast(nSeeds), [&](const int capacity) { + mTimeFrameGPU->createTrackITSExtDevice(capacity); + return TrackingKernels::computeTrackSeedHandler(mTimeFrameGPU->getDeviceTrackSeeds(), + mTimeFrameGPU->getDeviceArrayTrackingFrameInfo(), + mTimeFrameGPU->getDeviceArrayUnsortedClusters(), + mTimeFrameGPU->getDeviceIndexTableUtils(), + mTimeFrameGPU->getDeviceROFMaskTableView(), + mTimeFrameGPU->getDeviceROFOverlapTableView(), + mTimeFrameGPU->getDeviceArrayClusters(), + (const unsigned char**)mTimeFrameGPU->getDeviceArrayUsedClusters(), + mTimeFrameGPU->getDeviceArrayClustersIndexTables(), + mTimeFrameGPU->getDeviceROFrameClusters(), + mTimeFrameGPU->getDeviceTrackITSExt(), + mTimeFrameGPU->getDeviceTrackIndices(), + mTimeFrameGPU->getDeviceTrackSeedIndices(), + mTimeFrameGPU->getDeviceTrackCounter(), + capacity, + extendTracks ? mTimeFrameGPU->getDeviceActiveTrackExtensionHypotheses() : nullptr, + extendTracks ? mTimeFrameGPU->getDeviceNextTrackExtensionHypotheses() : nullptr, + mTimeFrameGPU->getDeviceLayerRadii(), + mTimeFrameGPU->getDeviceMinPts(), + mTimeFrameGPU->getDeviceLayerxX0(), + static_cast(nSeeds), + this->mBz, + this->mTrkParams[iteration].MaxChi2ClusterAttachment, + this->mTrkParams[iteration].MaxChi2NDF, + this->mTrkParams[iteration].ReseedIfShorter, + this->mTrkParams[iteration].RepeatRefitOut, + this->mTrkParams[iteration].ShiftRefToCluster, + this->mTrkParams[iteration].NLayers, + this->mTrkParams[iteration].PhiBins, + this->mTrkParams[iteration].TrackFollowerMaxHypotheses, + extendTop, + extendBot, + this->mTrkParams[iteration].TrackFollowerNSigmaCutPhi, + this->mTrkParams[iteration].TrackFollowerNSigmaCutZ, + mTimeFrameGPU->getDevicePropagator(), + this->mTrkParams[iteration].CorrType, + mTimeFrameGPU->getFrameworkAllocator()); }, estimator.peakCapacity(trackKey)); + mTimeFrameGPU->createTrackITSExtHost(nTracks); mTimeFrameGPU->downloadTrackITSExtDevice(); mTimeFrameGPU->downloadTrackIndicesDevice(); diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu index bf443e410e7d8..2dbb83bbf3f03 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu @@ -12,22 +12,25 @@ #include #include +#include #include #include #include #include +#include #include #include #include #include #include +#include #include -#include #include "ITStracking/Constants.h" #include "ITStracking/Definitions.h" #include "ITStracking/IndexTableUtils.h" +#include "ITStrackingGPU/LaunchGeometry.h" #include "ITStracking/MathUtils.h" #include "ITStracking/ExternalAllocator.h" #include "ITStracking/Tracklet.h" @@ -50,49 +53,19 @@ namespace o2::its namespace gpu { -template -struct sort_by_second { - GPUhd() bool operator()(const gpuPair& a, const gpuPair& b) const { return a.second < b.second; } -}; - -template -struct pair_to_first { - GPUhd() int operator()(const gpuPair& a) const - { - return a.first; - } -}; - -template -struct pair_to_second { - GPUhd() int operator()(const gpuPair& a) const - { - return a.second; - } -}; - -template -struct is_invalid_pair { - GPUhd() bool operator()(const gpuPair& p) const - { - return p.first == -1 && p.second == -1; - } -}; - -template -struct is_valid_pair { - GPUhd() bool operator()(const gpuPair& p) const - { - return !(p.first == -1 && p.second == -1); - } -}; - struct compare_track_index_chi2 { const TrackITSExt* tracks; + const int* seedIndices; GPUhd() bool operator()(const int a, const int b) const { - return o2::its::track::isBetter(tracks[a], tracks[b]); + if (o2::its::track::isBetter(tracks[a], tracks[b])) { + return true; + } + if (o2::its::track::isBetter(tracks[b], tracks[a])) { + return false; + } + return seedIndices[a] < seedIndices[b]; } }; @@ -117,44 +90,7 @@ struct TrackExtensionDirectionFollowerDevice { }; template -GPUg() void __launch_bounds__(constants::GPUThreads, 1) countTrackSeedsKernel( - TrackSeed* trackSeeds, - const TrackingFrameInfo** foundTrackingFrameInfo, - const Cluster** unsortedClusters, - int* seedLUT, - const float* layerRadii, - const float* minPts, - const float* layerxX0, - const unsigned int nSeeds, - const float bz, - const float maxChi2ClusterAttachment, - const float maxChi2NDF, - const int reseedIfShorter, - const bool repeatRefitOut, - const bool shiftRefToCluster, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType) -{ - const o2::its::track::TrackFitContext fitCtx{ - foundTrackingFrameInfo, layerxX0, NLayers, bz, - maxChi2ClusterAttachment, maxChi2NDF, - propagator, matCorrType, shiftRefToCluster, repeatRefitOut}; - for (int iCurrentTrackSeedIndex = blockIdx.x * blockDim.x + threadIdx.x; iCurrentTrackSeedIndex < nSeeds; iCurrentTrackSeedIndex += blockDim.x * gridDim.x) { - TrackITSInternal temporaryTrack; - if (o2::its::track::refitTrackSeed(trackSeeds[iCurrentTrackSeedIndex], - temporaryTrack, - fitCtx, - unsortedClusters, - layerRadii, - minPts, - reseedIfShorter)) { - seedLUT[iCurrentTrackSeedIndex] = 1; - } - } -} - -template -GPUg() void __launch_bounds__(constants::GPUThreads, 1) fitTrackSeedsKernel( +GPUg() void __launch_bounds__(GPUThreads, 1) fitTrackSeedsKernel( TrackSeed* trackSeeds, const TrackingFrameInfo** foundTrackingFrameInfo, const Cluster** unsortedClusters, @@ -166,7 +102,9 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) fitTrackSeedsKernel( const int** clustersIndexTables, const int** ROFClusters, o2::its::TrackITSExt* tracks, - const int* seedLUT, + int* trackSeedIndices, + int* outputCounter, + const int trackCapacity, TrackExtensionHypothesis* activeHypothesesScratch, TrackExtensionHypothesis* nextHypothesesScratch, const float* layerRadii, @@ -198,9 +136,6 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) fitTrackSeedsKernel( clusters, usedClusters, clustersIndexTables, ROFClusters, layerRadii, phiBins, maxHypothesesConfig, nSigmaCutPhi, nSigmaCutZ}; for (int iCurrentTrackSeedIndex = blockIdx.x * blockDim.x + threadIdx.x; iCurrentTrackSeedIndex < nSeeds; iCurrentTrackSeedIndex += blockDim.x * gridDim.x) { - if (seedLUT[iCurrentTrackSeedIndex] == seedLUT[iCurrentTrackSeedIndex + 1]) { - continue; - } TrackITSInternal temporaryTrack; bool refitSuccess = o2::its::track::refitTrackSeed(trackSeeds[iCurrentTrackSeedIndex], temporaryTrack, @@ -209,36 +144,41 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) fitTrackSeedsKernel( layerRadii, minPts, reseedIfShorter); - if (refitSuccess) { - if ((extendTop || extendBot) && activeHypothesesScratch && nextHypothesesScratch) { - const int maxHypotheses = o2::gpu::CAMath::Max(maxHypothesesConfig, 1); - const int threadIndex = blockIdx.x * blockDim.x + threadIdx.x; - auto* activeHypotheses = activeHypothesesScratch + threadIndex * maxHypotheses; - auto* nextHypotheses = nextHypothesesScratch + threadIndex * maxHypotheses; - const auto backup = temporaryTrack; - auto best = temporaryTrack; - uint32_t bestDiff{0}; - TrackExtensionDirectionFollowerDevice followDirection{&fitCtx, &followCtx, activeHypotheses, nextHypotheses}; - TrackExtensionBestTrial bestTrial{backup.getPattern(), fitCtx}; - followTrackExtensionBranches(backup, extendTop, extendBot, nLayers, followDirection, bestTrial, best, bestDiff); - temporaryTrack = best; - tracks[seedLUT[iCurrentTrackSeedIndex]] = makeTrackITSExt(temporaryTrack); - if (bestDiff) { - tracks[seedLUT[iCurrentTrackSeedIndex]].setExtendedLayerPattern(bestDiff); - } - continue; - } - tracks[seedLUT[iCurrentTrackSeedIndex]] = makeTrackITSExt(temporaryTrack); + if (!refitSuccess) { + continue; } + uint32_t bestDiff{0}; + if ((extendTop || extendBot) && activeHypothesesScratch && nextHypothesesScratch) { + const int maxHypotheses = o2::gpu::CAMath::Max(maxHypothesesConfig, 1); + const int threadIndex = blockIdx.x * blockDim.x + threadIdx.x; + auto* activeHypotheses = activeHypothesesScratch + threadIndex * maxHypotheses; + auto* nextHypotheses = nextHypothesesScratch + threadIndex * maxHypotheses; + const auto backup = temporaryTrack; + auto best = temporaryTrack; + TrackExtensionDirectionFollowerDevice followDirection{&fitCtx, &followCtx, activeHypotheses, nextHypotheses}; + TrackExtensionBestTrial bestTrial{backup.getPattern(), fitCtx}; + followTrackExtensionBranches(backup, extendTop, extendBot, nLayers, followDirection, bestTrial, best, bestDiff); + temporaryTrack = best; + } + const int slot = atomicAdd(outputCounter, 1); + if (slot >= trackCapacity) { + continue; + } + tracks[slot] = makeTrackITSExt(temporaryTrack); + if (bestDiff) { + tracks[slot].setExtendedLayerPattern(bestDiff); + } + trackSeedIndices[slot] = iCurrentTrackSeedIndex; } } -template -GPUg() void __launch_bounds__(constants::GPUThreads, 1) computeLayerCellNeighboursKernel( +template +GPUg() void __launch_bounds__(GPUThreads, 1) computeLayerCellNeighboursKernel( CellSeed** cellSeedArray, - int* neighboursCursor, int** cellsLUTs, CellNeighbour* cellNeighbours, + int* outputCounter, + const int outputCapacity, const int sourceCellTopologyId, const int targetCellTopologyId, const float maxChi2ClusterAttachment, @@ -266,22 +206,20 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) computeLayerCellNeighbou continue; } - if constexpr (initRun) { - atomicAdd(neighboursCursor + iNextCell, 1); - } else { - const int offset = atomicAdd(neighboursCursor + iNextCell, 1); - cellNeighbours[offset] = {sourceCellTopologyId, iCurrentCellIndex, targetCellTopologyId, iNextCell, currentCellSeed.getLevel() + 1}; - const int currentCellLevel{currentCellSeed.getLevel()}; - if (currentCellLevel >= nextCellSeed.getLevel()) { - atomicMax(cellSeedArray[targetCellTopologyId][iNextCell].getLevelPtr(), currentCellLevel + 1); - } + const int currentCellLevel{currentCellSeed.getLevel()}; + const int outputIndex = atomicAdd(outputCounter, 1); + if (outputIndex < outputCapacity) { + cellNeighbours[outputIndex] = {sourceCellTopologyId, iCurrentCellIndex, targetCellTopologyId, iNextCell, currentCellLevel + 1}; + } + if (currentCellLevel >= nextCellSeed.getLevel()) { + atomicMax(cellSeedArray[targetCellTopologyId][iNextCell].getLevelPtr(), currentCellLevel + 1); } } } } -template -GPUg() void __launch_bounds__(constants::GPUThreads, 1) computeLayerCellsKernel( +template +GPUg() void __launch_bounds__(GPUThreads, 1) computeLayerCellsKernel( const Cluster** sortedClusters, const Cluster** unsortedClusters, const TrackingFrameInfo** tfInfo, @@ -291,7 +229,8 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) computeLayerCellsKernel( const int cellTopologyId, const typename TrackingTopology::View topology, CellSeed* cells, - int** cellsLUTs, + int* outputCounter, + const int outputCapacity, const float* layerxX0, const float bz, const float maxChi2ClusterAttachment, @@ -303,11 +242,6 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) computeLayerCellsKernel( const auto second = topology.getLink(cellTopology.secondLink); const int layers[3] = {first.fromLayer, first.toLayer, second.toLayer}; for (int iCurrentTrackletIndex = blockIdx.x * blockDim.x + threadIdx.x; iCurrentTrackletIndex < nTrackletsCurrent; iCurrentTrackletIndex += blockDim.x * gridDim.x) { - if constexpr (!initRun) { - if (cellsLUTs[cellTopologyId][iCurrentTrackletIndex] == cellsLUTs[cellTopologyId][iCurrentTrackletIndex + 1]) { - continue; - } - } const Tracklet& currentTracklet = tracklets[cellTopology.firstLink][iCurrentTrackletIndex]; const int nextLayerClusterIndex{currentTracklet.secondClusterIndex}; const int nextLayerFirstTrackletIndex{trackletsLUT[cellTopology.secondLink][nextLayerClusterIndex]}; @@ -315,7 +249,6 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) computeLayerCellsKernel( if (nextLayerFirstTrackletIndex == nextLayerLastTrackletIndex) { continue; } - int foundCells{0}; for (int iNextTrackletIndex{nextLayerFirstTrackletIndex}; iNextTrackletIndex < nextLayerLastTrackletIndex; ++iNextTrackletIndex) { if (tracklets[cellTopology.secondLink][iNextTrackletIndex].firstClusterIndex != nextLayerClusterIndex) { break; @@ -332,10 +265,10 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) computeLayerCellsKernel( sortedClusters[layers[1]][nextTracklet.firstClusterIndex].clusterId, sortedClusters[layers[2]][nextTracklet.secondClusterIndex].clusterId}; - const auto& cluster1_glo = unsortedClusters[layers[0]][clusId[0]]; - const auto& cluster2_glo = unsortedClusters[layers[1]][clusId[1]]; - const auto& cluster3_tf = tfInfo[layers[2]][clusId[2]]; - auto track{o2::its::track::buildTrackSeed(cluster1_glo, cluster2_glo, cluster3_tf, bz)}; + const auto& cluster1Glo = unsortedClusters[layers[0]][clusId[0]]; + const auto& cluster2Glo = unsortedClusters[layers[1]][clusId[1]]; + const auto& cluster3Tf = tfInfo[layers[2]][clusId[2]]; + auto track{o2::its::track::buildTrackSeed(cluster1Glo, cluster2Glo, cluster3Tf, bz)}; float chi2{0.f}; bool good{false}; for (int iC{2}; iC--;) { @@ -364,22 +297,19 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) computeLayerCellsKernel( if (!good) { continue; } - if constexpr (!initRun) { - TimeEstBC ts = currentTracklet.getTimeStamp(); - ts += nextTracklet.getTimeStamp(); - new (cells + cellsLUTs[cellTopologyId][iCurrentTrackletIndex] + foundCells) CellSeed{cellTopology.hitLayerMask, clusId[0], clusId[1], clusId[2], iCurrentTrackletIndex, iNextTrackletIndex, track, chi2, ts}; + TimeEstBC ts = currentTracklet.getTimeStamp(); + ts += nextTracklet.getTimeStamp(); + const int outputIndex = atomicAdd(outputCounter, 1); + if (outputIndex < outputCapacity) { + new (cells + outputIndex) CellSeed{cellTopology.hitLayerMask, clusId[0], clusId[1], clusId[2], iCurrentTrackletIndex, iNextTrackletIndex, track, chi2, ts}; } - ++foundCells; } } - if constexpr (initRun) { - cellsLUTs[cellTopologyId][iCurrentTrackletIndex] = foundCells; - } } } -template -GPUg() void __launch_bounds__(constants::GPUThreads, 1) computeLayerTrackletsMultiROFKernel( +template +GPUg() void __launch_bounds__(GPUThreads, 1) computeLayerTrackletsMultiROFKernel( const IndexTableUtils* utils, const typename ROFMaskTable::View rofMask, const int linkId, @@ -387,14 +317,14 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) computeLayerTrackletsMul const typename ROFOverlapTable::View rofOverlaps, const typename ROFVertexLookupTable::View vertexLUT, const Vertex* vertices, - const int* rofPV, const int vertexId, const Cluster** clusters, const int** ROFClusters, const unsigned char** usedClusters, const int** indexTables, Tracklet** tracklets, - int** trackletsLUT, + int* outputCounter, + const int outputCapacity, const bool selectUPCVertices, const float NSigmaCut, const float phiCut, @@ -441,17 +371,11 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) computeLayerTrackletsMul for (int currentClusterIndex = threadIdx.x; currentClusterIndex < clustersCurrentLayer.size(); currentClusterIndex += blockDim.x) { - unsigned int storedTracklets{0}; const auto& currentCluster{clustersCurrentLayer[currentClusterIndex]}; const int currentSortedIndex{ROFClusters[fromLayer][pivotROF] + currentClusterIndex}; if (usedClusters[fromLayer][currentCluster.clusterId]) { continue; } - if constexpr (!initRun) { - if (trackletsLUT[linkId][currentSortedIndex] == trackletsLUT[linkId][currentSortedIndex + 1]) { - continue; - } - } const float inverseR0{1.f / currentCluster.radius}; for (int iV{startVtx}; iV < endVtx; ++iV) { @@ -508,15 +432,13 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) computeLayerTrackletsMul const float deltaPhi{o2::gpu::CAMath::Abs(currentCluster.phi - nextCluster.phi)}; const float deltaZ{o2::gpu::CAMath::Abs(tanLambda * (nextCluster.radius - currentCluster.radius) + currentCluster.zCoordinate - nextCluster.zCoordinate)}; if (deltaZ / sigmaZ < NSigmaCut && (deltaPhi < phiCut || o2::gpu::CAMath::Abs(deltaPhi - o2::constants::math::TwoPI) < phiCut)) { - if constexpr (initRun) { - trackletsLUT[linkId][currentSortedIndex]++; // we need l0 as well for usual exclusive sums. - } else { - const float phi{o2::gpu::CAMath::ATan2(currentCluster.yCoordinate - nextCluster.yCoordinate, currentCluster.xCoordinate - nextCluster.xCoordinate)}; - const float tanL{(currentCluster.zCoordinate - nextCluster.zCoordinate) / (currentCluster.radius - nextCluster.radius)}; - const int nextSortedIndex{ROFClusters[toLayer][targetROF] + nextClusterIndex}; - new (tracklets[linkId] + trackletsLUT[linkId][currentSortedIndex] + storedTracklets) Tracklet{currentSortedIndex, nextSortedIndex, tanL, phi, ts}; + const float phi{o2::gpu::CAMath::ATan2(currentCluster.yCoordinate - nextCluster.yCoordinate, currentCluster.xCoordinate - nextCluster.xCoordinate)}; + const float tanL{(currentCluster.zCoordinate - nextCluster.zCoordinate) / (currentCluster.radius - nextCluster.radius)}; + const int nextSortedIndex{ROFClusters[toLayer][targetROF] + nextClusterIndex}; + const int outputIndex = atomicAdd(outputCounter, 1); + if (outputIndex < outputCapacity) { + new (tracklets[linkId] + outputIndex) Tracklet{currentSortedIndex, nextSortedIndex, tanL, phi, ts}; } - ++storedTracklets; } } } @@ -526,7 +448,7 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) computeLayerTrackletsMul } } -GPUg() void __launch_bounds__(constants::GPUThreads, 1) compileTrackletsLookupTableKernel( +GPUg() void __launch_bounds__(GPUThreads, 1) compileTrackletsLookupTableKernel( const Tracklet* tracklets, int* trackletsLookUpTable, const int nTracklets) @@ -536,8 +458,61 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) compileTrackletsLookupTa } } -template -GPUg() void __launch_bounds__(constants::GPUThreads, 1) processNeighboursKernel( +/// Counts how many entries fall into each lookup table slot, the table becomes the offsets of +/// the sorted entries once exclusively scanned. +GPUg() void __launch_bounds__(GPUThreads, 1) compileLookupTableKernel( + const int* keys, + int* lookUpTable, + const int nEntries) +{ + for (int currentEntry = blockIdx.x * blockDim.x + threadIdx.x; currentEntry < nEntries; currentEntry += blockDim.x * gridDim.x) { + atomicAdd(&lookUpTable[keys[currentEntry]], 1); + } +} + +GPUg() void __launch_bounds__(GPUThreads, 1) compileCellNeighboursLookupTableKernel( + const CellNeighbour* neighbours, + int* neighboursLookUpTable, + const int nNeighbours) +{ + for (int currentNeighbourIndex = blockIdx.x * blockDim.x + threadIdx.x; currentNeighbourIndex < nNeighbours; currentNeighbourIndex += blockDim.x * gridDim.x) { + atomicAdd(&neighboursLookUpTable[neighbours[currentNeighbourIndex].nextCell], 1); + } +} + +struct trackletClusterKey { + GPUhd() uint64_t operator()(const Tracklet& tracklet) const + { + return (static_cast(tracklet.firstClusterIndex) << 32) | static_cast(tracklet.secondClusterIndex); + } +}; + +struct cellFirstTrackletIndex { + GPUhd() int operator()(const CellSeed& cell) const { return cell.getFirstTrackletIndex(); } +}; + +struct cellNeighbourNextCell { + GPUhd() int operator()(const CellNeighbour& neighbour) const { return neighbour.nextCell; } +}; + +struct cellNeighbourLess { + GPUhd() bool operator()(const CellNeighbour& a, const CellNeighbour& b) const + { + if (a.nextCellTopology != b.nextCellTopology) { + return a.nextCellTopology < b.nextCellTopology; + } + if (a.nextCell != b.nextCell) { + return a.nextCell < b.nextCell; + } + if (a.cellTopology != b.cellTopology) { + return a.cellTopology < b.cellTopology; + } + return a.cell < b.cell; + } +}; + +template +GPUg() void __launch_bounds__(GPUThreads, 1) processNeighboursKernel( const int defaultCellTopologyId, const int level, CellSeed** allCellSeeds, @@ -548,8 +523,10 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) processNeighboursKernel( TrackSeed* updatedCellSeeds, int* updatedCellsIds, int* updatedCellTopologyIds, - int* foundSeedsTable, // auxiliary only in GPU code to compute the number of cells per iteration - const unsigned char** usedClusters, // Used clusters + int* updatedSourceSeeds, + int* outputCounter, + const int outputCapacity, + const unsigned char** usedClusters, CellNeighbour** neighbours, int** neighboursLUT, const TrackingFrameInfo** foundTrackingFrameInfo, @@ -560,12 +537,6 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) processNeighboursKernel( const o2::base::PropagatorF::MatCorrType matCorrType) { for (unsigned int iCurrentCell = blockIdx.x * blockDim.x + threadIdx.x; iCurrentCell < nCurrentCells; iCurrentCell += blockDim.x * gridDim.x) { - if constexpr (!dryRun) { - if (foundSeedsTable[iCurrentCell] == foundSeedsTable[iCurrentCell + 1]) { - continue; - } - } - int foundSeeds{0}; const auto& currentCell{currentCellSeeds[iCurrentCell]}; const int cellTopologyId = currentCellTopologyIds == nullptr ? defaultCellTopologyId : currentCellTopologyIds[iCurrentCell]; if (currentCell.getLevel() != level) { @@ -634,21 +605,22 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) processNeighboursKernel( if (!seed.o2::track::TrackParCov::update(trHit.positionTrackingFrame, trHit.covarianceTrackingFrame)) { continue; } - if constexpr (dryRun) { - foundSeedsTable[iCurrentCell]++; - } else { - seed.getClusters()[neighbourLayer] = neighbourCluster; - auto mask = seed.getHitLayerMask(); - mask.set(neighbourLayer); - seed.setHitLayerMask(mask); - seed.setLevel(neighbourCell.getLevel()); - seed.setFirstTrackletIndex(neighbourCell.getFirstTrackletIndex()); - seed.setSecondTrackletIndex(neighbourCell.getSecondTrackletIndex()); - updatedCellsIds[foundSeedsTable[iCurrentCell] + foundSeeds] = neighbourCellId; - updatedCellTopologyIds[foundSeedsTable[iCurrentCell] + foundSeeds] = neighbourCellTopologyId; - updatedCellSeeds[foundSeedsTable[iCurrentCell] + foundSeeds] = seed; + seed.getClusters()[neighbourLayer] = neighbourCluster; + auto mask = seed.getHitLayerMask(); + mask.set(neighbourLayer); + seed.setHitLayerMask(mask); + seed.setLevel(neighbourCell.getLevel()); + seed.setFirstTrackletIndex(neighbourCell.getFirstTrackletIndex()); + seed.setSecondTrackletIndex(neighbourCell.getSecondTrackletIndex()); + const int outputIndex = atomicAdd(outputCounter, 1); + if (outputIndex < outputCapacity) { + updatedCellsIds[outputIndex] = neighbourCellId; + updatedCellTopologyIds[outputIndex] = neighbourCellTopologyId; + updatedCellSeeds[outputIndex] = seed; + if (updatedSourceSeeds != nullptr) { + updatedSourceSeeds[outputIndex] = static_cast(iCurrentCell); + } } - foundSeeds++; } } } @@ -656,37 +628,42 @@ GPUg() void __launch_bounds__(constants::GPUThreads, 1) processNeighboursKernel( } // namespace gpu template -void countTrackletsInROFsHandler(const IndexTableUtils* utils, - const typename ROFMaskTable::View& rofMask, - const int linkId, - const int fromLayer, - const int toLayer, - const typename ROFOverlapTable::View& rofOverlaps, - const typename ROFVertexLookupTable::View& vertexLUT, - const int vertexId, - const Vertex* vertices, - const int* rofPV, - const Cluster** clusters, - std::vector nClusters, - const int** ROFClusters, - const unsigned char** usedClusters, - const int** clustersIndexTables, - int** trackletsLUTs, - gsl::span trackletsLUTsHost, - const bool selectUPCVertices, - const float NSigmaCut, - const typename TrackingTopology::View topology, - bounded_vector& linkPhiCuts, - const float resolutionPV, - std::array& minRs, - std::array& maxRs, - bounded_vector& resolutions, - std::vector& radii, - bounded_vector& linkMSAngles, - o2::its::ExternalAllocator* alloc, - gpu::Streams& streams) +int TrackingKernels::computeTrackletsInROFsHandler(const IndexTableUtils* utils, + const typename ROFMaskTable::View& rofMask, + const int linkId, + const int fromLayer, + const int toLayer, + const typename ROFOverlapTable::View& rofOverlaps, + const typename ROFVertexLookupTable::View& vertexLUT, + const int vertexId, + const Vertex* vertices, + const Cluster** clusters, + const std::vector& nClusters, + const int** ROFClusters, + const unsigned char** usedClusters, + const int** clustersIndexTables, + Tracklet** tracklets, + gsl::span spanTracklets, + gsl::span nTracklets, + const int capacity, + gsl::span trackletsLUTsHost, + const bool selectUPCVertices, + const float NSigmaCut, + const typename TrackingTopology::View topology, + bounded_vector& linkPhiCuts, + const float resolutionPV, + std::array& minRs, + std::array& maxRs, + bounded_vector& resolutions, + std::vector& radii, + bounded_vector& linkMSAngles, + o2::its::ExternalAllocator* alloc, + gpu::Streams& streams) { - gpu::computeLayerTrackletsMultiROFKernel<<>>( + int emitted = 0; + int* outputCounter = trackletsLUTsHost[linkId] + nClusters[fromLayer]; + GPUChkErrS(cudaMemsetAsync(outputCounter, 0, sizeof(int), streams[linkId].get())); + gpu::computeLayerTrackletsMultiROFKernel<<>>( utils, rofMask, linkId, @@ -694,77 +671,14 @@ void countTrackletsInROFsHandler(const IndexTableUtils* utils, rofOverlaps, vertexLUT, vertices, - rofPV, - vertexId, - clusters, - ROFClusters, - usedClusters, - clustersIndexTables, - nullptr, - trackletsLUTs, - selectUPCVertices, - NSigmaCut, - linkPhiCuts[linkId], - resolutionPV, - minRs[toLayer], - maxRs[toLayer], - resolutions[fromLayer], - radii[toLayer] - radii[fromLayer], - linkMSAngles[linkId]); - auto nosync_policy = THRUST_NAMESPACE::par_nosync(gpu::TypedAllocator(alloc)).on(streams[linkId].get()); - thrust::exclusive_scan(nosync_policy, trackletsLUTsHost[linkId], trackletsLUTsHost[linkId] + nClusters[fromLayer] + 1, trackletsLUTsHost[linkId]); -} - -template -void computeTrackletsInROFsHandler(const IndexTableUtils* utils, - const typename ROFMaskTable::View& rofMask, - const int linkId, - const int fromLayer, - const int toLayer, - const typename ROFOverlapTable::View& rofOverlaps, - const typename ROFVertexLookupTable::View& vertexLUT, - const int vertexId, - const Vertex* vertices, - const int* rofPV, - const Cluster** clusters, - std::vector nClusters, - const int** ROFClusters, - const unsigned char** usedClusters, - const int** clustersIndexTables, - Tracklet** tracklets, - gsl::span spanTracklets, - gsl::span nTracklets, - int** trackletsLUTs, - gsl::span trackletsLUTsHost, - const bool selectUPCVertices, - const float NSigmaCut, - const typename TrackingTopology::View topology, - bounded_vector& linkPhiCuts, - const float resolutionPV, - std::array& minRs, - std::array& maxRs, - bounded_vector& resolutions, - std::vector& radii, - bounded_vector& linkMSAngles, - o2::its::ExternalAllocator* alloc, - gpu::Streams& streams) -{ - gpu::computeLayerTrackletsMultiROFKernel<<>>( - utils, - rofMask, - linkId, - topology, - rofOverlaps, - vertexLUT, - vertices, - rofPV, vertexId, clusters, ROFClusters, usedClusters, clustersIndexTables, tracklets, - trackletsLUTs, + outputCounter, + capacity, selectUPCVertices, NSigmaCut, linkPhiCuts[linkId], @@ -774,23 +688,41 @@ void computeTrackletsInROFsHandler(const IndexTableUtils* utils, resolutions[fromLayer], radii[toLayer] - radii[fromLayer], linkMSAngles[linkId]); - thrust::device_ptr tracklets_ptr(spanTracklets[linkId]); + GPUChkErrS(cudaMemcpyAsync(&emitted, outputCounter, sizeof(int), cudaMemcpyDeviceToHost, streams[linkId].get())); + streams[linkId].sync(); + if (emitted > capacity) { + return emitted; + } + nTracklets[linkId] = emitted; auto nosync_policy = THRUST_NAMESPACE::par_nosync(gpu::TypedAllocator(alloc)).on(streams[linkId].get()); - thrust::sort(nosync_policy, tracklets_ptr, tracklets_ptr + nTracklets[linkId]); - auto unique_end = thrust::unique(nosync_policy, tracklets_ptr, tracklets_ptr + nTracklets[linkId]); - nTracklets[linkId] = unique_end - tracklets_ptr; - if (fromLayer > 0) { - GPUChkErrS(cudaMemsetAsync(trackletsLUTsHost[linkId], 0, (nClusters[fromLayer] + 1) * sizeof(int), streams[linkId].get())); - gpu::compileTrackletsLookupTableKernel<<>>( - spanTracklets[linkId], - trackletsLUTsHost[linkId], - nTracklets[linkId]); - thrust::exclusive_scan(nosync_policy, trackletsLUTsHost[linkId], trackletsLUTsHost[linkId] + nClusters[fromLayer] + 1, trackletsLUTsHost[linkId]); + if (emitted > 0) { + thrust::device_ptr trackletsPtr(spanTracklets[linkId]); + constexpr uint64_t SortTag = qStr2Tag("ITSTRKSR"); + alloc->pushTagOnStack(SortTag); + auto keys = gpu::TypedAllocator(alloc).allocate(emitted); + thrust::transform(nosync_policy, trackletsPtr, trackletsPtr + emitted, keys, gpu::trackletClusterKey{}); + thrust::sort_by_key(nosync_policy, keys, keys + emitted, trackletsPtr); + if (vertexId < 0) { + auto uniqueEnd = thrust::unique_by_key(nosync_policy, keys, keys + emitted, trackletsPtr); + nTracklets[linkId] = uniqueEnd.first - keys; + } + streams[linkId].sync(); + alloc->popTagOffStack(SortTag); + } + GPUChkErrS(cudaMemsetAsync(trackletsLUTsHost[linkId], 0, (nClusters[fromLayer] + 1) * sizeof(int), streams[linkId].get())); + if (nTracklets[linkId] == 0) { + return emitted; } + gpu::compileTrackletsLookupTableKernel<<>>( + spanTracklets[linkId], + trackletsLUTsHost[linkId], + nTracklets[linkId]); + thrust::exclusive_scan(nosync_policy, trackletsLUTsHost[linkId], trackletsLUTsHost[linkId] + nClusters[fromLayer] + 1, trackletsLUTsHost[linkId]); + return emitted; } template -void countCellsHandler( +int TrackingKernels::computeCellsHandler( const Cluster** sortedClusters, const Cluster** unsortedClusters, const TrackingFrameInfo** tfInfo, @@ -800,18 +732,21 @@ void countCellsHandler( const int cellTopologyId, const typename TrackingTopology::View topology, CellSeed* cells, - int** cellsLUTsArrayDevice, + const int capacity, int* cellsLUTsHost, const float bz, const float maxChi2ClusterAttachment, const float cellDeltaTanLambdaSigma, const float nSigmaCut, - const std::vector& layerxX0Host, + const float* layerxX0, o2::its::ExternalAllocator* alloc, gpu::Streams& streams) { - thrust::device_vector layerxX0(layerxX0Host); - gpu::computeLayerCellsKernel<<>>( + int emitted = 0; + auto& stream = streams[cellTopologyId]; + int* outputCounter = cellsLUTsHost + nTracklets; + GPUChkErrS(cudaMemsetAsync(outputCounter, 0, sizeof(int), stream.get())); + gpu::computeLayerCellsKernel<<>>( sortedClusters, // const Cluster** unsortedClusters, // const Cluster** tfInfo, // const TrackingFrameInfo** @@ -820,72 +755,71 @@ void countCellsHandler( nTracklets, // const int cellTopologyId, // const int topology, - cells, // CellSeed* - cellsLUTsArrayDevice, // int** - thrust::raw_pointer_cast(&layerxX0[0]), + cells, // CellSeed* + outputCounter, // int* + capacity, // const int + layerxX0, bz, // const float maxChi2ClusterAttachment, // const float cellDeltaTanLambdaSigma, // const float nSigmaCut); // const float - auto nosync_policy = THRUST_NAMESPACE::par_nosync(gpu::TypedAllocator(alloc)).on(streams[cellTopologyId].get()); + GPUChkErrS(cudaMemcpyAsync(&emitted, outputCounter, sizeof(int), cudaMemcpyDeviceToHost, stream.get())); + stream.sync(); + if (emitted > capacity) { + return emitted; + } + auto nosync_policy = THRUST_NAMESPACE::par_nosync(gpu::TypedAllocator(alloc)).on(stream.get()); + GPUChkErrS(cudaMemsetAsync(cellsLUTsHost, 0, (nTracklets + 1) * sizeof(int), stream.get())); + if (emitted == 0) { + return emitted; + } + constexpr uint64_t SortTag = qStr2Tag("ITSCELSR"); + alloc->pushTagOnStack(SortTag); + gpu::TypedAllocator keyAllocator(alloc); + gpu::TypedAllocator cellAllocator(alloc); + auto keys = keyAllocator.allocate(emitted); + auto permutation = keyAllocator.allocate(emitted); + thrust::device_ptr cellsPtr(cells); + thrust::transform(nosync_policy, cellsPtr, cellsPtr + emitted, keys, gpu::cellFirstTrackletIndex{}); + thrust::sequence(nosync_policy, permutation, permutation + emitted); + thrust::stable_sort_by_key(nosync_policy, keys, keys + emitted, permutation); + gpu::compileLookupTableKernel<<>>( + thrust::raw_pointer_cast(keys), + cellsLUTsHost, + emitted); thrust::exclusive_scan(nosync_policy, cellsLUTsHost, cellsLUTsHost + nTracklets + 1, cellsLUTsHost); + auto sortedCells = cellAllocator.allocate(emitted); + thrust::gather(nosync_policy, permutation, permutation + emitted, cellsPtr, sortedCells); + GPUChkErrS(cudaMemcpyAsync(cells, thrust::raw_pointer_cast(sortedCells), emitted * sizeof(CellSeed), cudaMemcpyDeviceToDevice, stream.get())); + stream.sync(); + alloc->popTagOffStack(SortTag); + return emitted; } -template -void computeCellsHandler( - const Cluster** sortedClusters, - const Cluster** unsortedClusters, - const TrackingFrameInfo** tfInfo, - Tracklet** tracklets, - int** trackletsLUT, - const int nTracklets, - const int cellTopologyId, - const typename TrackingTopology::View topology, - CellSeed* cells, - int** cellsLUTsArrayDevice, - int* cellsLUTsHost, - const float bz, - const float maxChi2ClusterAttachment, - const float cellDeltaTanLambdaSigma, - const float nSigmaCut, - const std::vector& layerxX0Host, - gpu::Streams& streams) +void resetOutputCounterHandler(int* outputCounter, gpu::Stream& stream) { - thrust::device_vector layerxX0(layerxX0Host); - gpu::computeLayerCellsKernel<<>>( - sortedClusters, // const Cluster** - unsortedClusters, // const Cluster** - tfInfo, // const TrackingFrameInfo** - tracklets, // const Tracklets** - trackletsLUT, // const int** - nTracklets, // const int - cellTopologyId, // const int - topology, - cells, // CellSeed* - cellsLUTsArrayDevice, // int** - thrust::raw_pointer_cast(&layerxX0[0]), - bz, // const float - maxChi2ClusterAttachment, // const float - cellDeltaTanLambdaSigma, // const float - nSigmaCut); // const float + GPUChkErrS(cudaMemsetAsync(outputCounter, 0, sizeof(int), stream.get())); } template -void countCellNeighboursHandler(CellSeed** cellsLayersDevice, - int* neighboursCursor, - int** cellsLUTs, - const int sourceCellTopologyId, - const int targetCellTopologyId, - const float maxChi2ClusterAttachment, - const float bz, - const unsigned int nCells, - gpu::Stream& stream) +void TrackingKernels::computeCellNeighboursHandler(CellSeed** cellsLayersDevice, + int** cellsLUTs, + CellNeighbour* cellNeighbours, + int* outputCounter, + const int capacity, + const int sourceCellTopologyId, + const int targetCellTopologyId, + const float maxChi2ClusterAttachment, + const float bz, + const unsigned int nCells, + gpu::Stream& stream) { - gpu::computeLayerCellNeighboursKernel<<>>( + gpu::computeLayerCellNeighboursKernel<<>>( cellsLayersDevice, - neighboursCursor, cellsLUTs, - nullptr, + cellNeighbours, + outputCounter, + capacity, sourceCellTopologyId, targetCellTopologyId, maxChi2ClusterAttachment, @@ -893,945 +827,309 @@ void countCellNeighboursHandler(CellSeed** cellsLayersDevice, nCells); } -void scanCellNeighboursHandler(int* neighboursCursor, - int* neighboursLUT, - const unsigned int nCells, - o2::its::ExternalAllocator* alloc, - gpu::Stream& stream) -{ - auto nosync_policy = THRUST_NAMESPACE::par_nosync(gpu::TypedAllocator(alloc)).on(stream.get()); - thrust::exclusive_scan(nosync_policy, neighboursCursor, neighboursCursor + nCells + 1, neighboursCursor); - GPUChkErrS(cudaMemcpyAsync(neighboursLUT, neighboursCursor, (nCells + 1) * sizeof(int), cudaMemcpyDeviceToDevice, stream.get())); -} - -template -void computeCellNeighboursHandler(CellSeed** cellsLayersDevice, - int* neighboursCursor, - int** cellsLUTs, - CellNeighbour* cellNeighbours, - const int sourceCellTopologyId, - const int targetCellTopologyId, - const float maxChi2ClusterAttachment, - const float bz, - const unsigned int nCells, +int finalizeCellNeighboursHandler(CellNeighbour* cellNeighbours, + int* neighboursLUT, + const int nTargetCells, + const int capacity, + o2::its::ExternalAllocator* alloc, gpu::Stream& stream) { - gpu::computeLayerCellNeighboursKernel<<>>( - cellsLayersDevice, - neighboursCursor, - cellsLUTs, + int emitted = 0; + int* outputCounter = neighboursLUT + nTargetCells; + GPUChkErrS(cudaMemcpyAsync(&emitted, outputCounter, sizeof(int), cudaMemcpyDeviceToHost, stream.get())); + stream.sync(); + if (emitted > capacity) { + return emitted; + } + auto nosync_policy = THRUST_NAMESPACE::par_nosync(gpu::TypedAllocator(alloc)).on(stream.get()); + if (emitted > 0) { + thrust::device_ptr neighboursPtr(cellNeighbours); + constexpr uint64_t SortTag = qStr2Tag("ITSNGHSR"); + alloc->pushTagOnStack(SortTag); +#ifdef GPUCA_DETERMINISTIC_MODE + thrust::sort(nosync_policy, neighboursPtr, neighboursPtr + emitted, gpu::cellNeighbourLess{}); +#else + auto keys = gpu::TypedAllocator(alloc).allocate(emitted); + thrust::transform(nosync_policy, neighboursPtr, neighboursPtr + emitted, keys, gpu::cellNeighbourNextCell{}); + thrust::sort_by_key(nosync_policy, keys, keys + emitted, neighboursPtr); +#endif + stream.sync(); + alloc->popTagOffStack(SortTag); + } + GPUChkErrS(cudaMemsetAsync(neighboursLUT, 0, (nTargetCells + 1) * sizeof(int), stream.get())); + if (emitted == 0) { + return emitted; + } + gpu::compileCellNeighboursLookupTableKernel<<>>( cellNeighbours, - sourceCellTopologyId, - targetCellTopologyId, - maxChi2ClusterAttachment, - bz, - nCells); -} - -int filterCellNeighboursHandler(gpuPair* cellNeighbourPairs, - int* cellNeighbours, - unsigned int nNeigh, - gpu::Stream& stream, - o2::its::ExternalAllocator* allocator) -{ - auto nosync_policy = THRUST_NAMESPACE::par_nosync(gpu::TypedAllocator(allocator)).on(stream.get()); - thrust::device_ptr> neighVectorPairs(cellNeighbourPairs); - thrust::device_ptr validNeighs(cellNeighbours); - auto updatedEnd = thrust::remove_if(nosync_policy, neighVectorPairs, neighVectorPairs + nNeigh, gpu::is_invalid_pair()); - size_t newSize = updatedEnd - neighVectorPairs; - thrust::stable_sort(nosync_policy, neighVectorPairs, neighVectorPairs + newSize, gpu::sort_by_second()); - thrust::transform(nosync_policy, neighVectorPairs, neighVectorPairs + newSize, validNeighs, gpu::pair_to_first()); - return newSize; + neighboursLUT, + emitted); + thrust::exclusive_scan(nosync_policy, neighboursLUT, neighboursLUT + nTargetCells + 1, neighboursLUT); + return emitted; } template -void processNeighboursHandler(const int startLevel, - const int defaultCellTopologyId, - CellSeed** allCellSeeds, - CellSeed* currentCellSeeds, - const int* currentCellTopologyIds, - const int* currentCellIds, - const int* nCells, - const unsigned char** usedClusters, - CellNeighbour** neighbours, - int** neighboursDeviceLUTs, - const TrackingFrameInfo** foundTrackingFrameInfo, - bounded_vector>& seedsHost, - const float bz, - const float maxChi2ClusterAttachment, - const float maxChi2NDF, - const int maxHoles, - const int minSeedingClusters, - const LayerMask holeLayerMask, - const LayerMask nonSeedingLayerMask, - const std::vector& layerxX0Host, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType, - o2::its::ExternalAllocator* alloc) +void TrackingKernels::processNeighboursHandler(const int startLevel, + const int startCellTopologyId, + CellSeed** allCellSeeds, + CellSeed* currentCellSeeds, + const int* currentCellTopologyIds, + const int* currentCellIds, + const int* nCells, + const unsigned char** usedClusters, + CellNeighbour** neighbours, + int** neighboursDeviceLUTs, + const TrackingFrameInfo** foundTrackingFrameInfo, + TrackSeed* seedsDevice, + const int seedsCapacity, + int& seedsCursor, + CapacityEstimator& estimator, + const int iteration, + const float bz, + const float maxChi2ClusterAttachment, + const float maxChi2NDF, + const int maxHoles, + const int minSeedingClusters, + const LayerMask holeLayerMask, + const LayerMask nonSeedingLayerMask, + const float* layerxX0, + const o2::base::Propagator* propagator, + const o2::base::PropagatorF::MatCorrType matCorrType, + o2::its::ExternalAllocator* alloc) { constexpr uint64_t Tag = qStr2Tag("ITS_PNH1"); alloc->pushTagOnStack(Tag); auto allocInt = gpu::TypedAllocator(alloc); auto allocTrackSeed = gpu::TypedAllocator>(alloc); - thrust::device_vector layerxX0(layerxX0Host); - thrust::device_vector> foundSeedsTable(nCells[defaultCellTopologyId] + 1, 0, allocInt); auto nosync_policy = THRUST_NAMESPACE::par_nosync(gpu::TypedAllocator(alloc)).on(gpu::Stream::DefaultStream); + auto outputCounter = allocInt.allocate(1); + + auto roadKey = [&](const int level) { + return CapacityEstimator::makeKey(SlabSite::Roads, iteration, CapacityEstimator::makeVariant(startLevel, level), startCellTopologyId); + }; + + struct Slab { + thrust::device_ptr> seeds{}; + thrust::device_ptr cellIds{}; + thrust::device_ptr cellTopologyIds{}; + int capacity{0}; + }; + Slab slabs[2]; + auto ensureCapacity = [&](Slab& slab, const int capacity) { + if (slab.capacity >= capacity) { + return; + } + slab.seeds = allocTrackSeed.allocate(capacity); + slab.cellIds = allocInt.allocate(capacity); + slab.cellTopologyIds = allocInt.allocate(capacity); + slab.capacity = capacity; + }; + + constexpr double SlabHeadroom = 1.3; // deliberately tighter than the estimator's adaptive margin + size_t peak = 0; + double waveScale = static_cast(nCells[startCellTopologyId]); + for (int level = startLevel; level >= 2 && waveScale > 0.; --level) { + const double expected = estimator.expected(roadKey(level), waveScale); + peak = std::max(peak, static_cast(std::ceil(expected * SlabHeadroom))); + waveScale = expected; + } + if (peak == 0) { + peak = estimator.peakCapacity(roadKey(startLevel)); + } + const int slabCapacity = static_cast(std::min(peak, static_cast(std::numeric_limits::max()))); + ensureCapacity(slabs[0], slabCapacity); + ensureCapacity(slabs[1], slabCapacity); + + int filled = -1; // slab holding the wave that was produced last + int nWaveSeeds = 0; + + auto processLevel = [&](auto* levelSeeds, const int* levelCellIds, const int* levelCellTopologyIds, + const unsigned int nLevelSeeds, const int level, const int topologyId) { + const int outIdx = filled == 0 ? 1 : 0; + Slab& out = slabs[outIdx]; + thrust::device_ptr> staged{}; + thrust::device_ptr stagedCellIds{}, stagedCellTopologyIds{}, sourceSeeds{}; + const int emitted = runOnSlab( + estimator, roadKey(level), static_cast(nLevelSeeds), [&](const int capacity) { + ensureCapacity(out, capacity); +#ifdef GPUCA_DETERMINISTIC_MODE + staged = allocTrackSeed.allocate(out.capacity); + stagedCellIds = allocInt.allocate(out.capacity); + stagedCellTopologyIds = allocInt.allocate(out.capacity); + sourceSeeds = allocInt.allocate(out.capacity); +#else + staged = out.seeds; + stagedCellIds = out.cellIds; + stagedCellTopologyIds = out.cellTopologyIds; +#endif + GPUChkErrS(cudaMemsetAsync(thrust::raw_pointer_cast(outputCounter), 0, sizeof(int), gpu::Stream::DefaultStream)); + gpu::processNeighboursKernel><<>>( + topologyId, + level, + allCellSeeds, + levelSeeds, + levelCellIds, + levelCellTopologyIds, + nLevelSeeds, + thrust::raw_pointer_cast(staged), + thrust::raw_pointer_cast(stagedCellIds), + thrust::raw_pointer_cast(stagedCellTopologyIds), + thrust::raw_pointer_cast(sourceSeeds), + thrust::raw_pointer_cast(outputCounter), + out.capacity, + usedClusters, + neighbours, + neighboursDeviceLUTs, + foundTrackingFrameInfo, + layerxX0, + bz, + maxChi2ClusterAttachment, + propagator, + matCorrType); + int wanted{0}; + GPUChkErrS(cudaMemcpyAsync(&wanted, thrust::raw_pointer_cast(outputCounter), sizeof(int), cudaMemcpyDeviceToHost, gpu::Stream::DefaultStream)); + GPUChkErrS(cudaStreamSynchronize(gpu::Stream::DefaultStream)); + return wanted; + }, + static_cast(out.capacity)); + + nWaveSeeds = emitted; + filled = outIdx; +#ifdef GPUCA_DETERMINISTIC_MODE + if (emitted > 0) { + auto permutation = allocInt.allocate(emitted); + thrust::sequence(nosync_policy, permutation, permutation + emitted); + thrust::stable_sort_by_key(nosync_policy, sourceSeeds, sourceSeeds + emitted, permutation); + thrust::gather(nosync_policy, permutation, permutation + emitted, staged, out.seeds); + thrust::gather(nosync_policy, permutation, permutation + emitted, stagedCellIds, out.cellIds); + thrust::gather(nosync_policy, permutation, permutation + emitted, stagedCellTopologyIds, out.cellTopologyIds); + GPUChkErrS(cudaStreamSynchronize(gpu::Stream::DefaultStream)); + } +#endif + }; - gpu::processNeighboursKernel<<>>( - defaultCellTopologyId, - startLevel, - allCellSeeds, - currentCellSeeds, - nullptr, - nullptr, - nCells[defaultCellTopologyId], - nullptr, - nullptr, - nullptr, - thrust::raw_pointer_cast(&foundSeedsTable[0]), - usedClusters, - neighbours, - neighboursDeviceLUTs, - foundTrackingFrameInfo, - thrust::raw_pointer_cast(&layerxX0[0]), - bz, - maxChi2ClusterAttachment, - propagator, - matCorrType); - thrust::exclusive_scan(nosync_policy, foundSeedsTable.begin(), foundSeedsTable.end(), foundSeedsTable.begin()); - - thrust::device_vector> updatedCellId(foundSeedsTable.back(), 0, allocInt); - thrust::device_vector> updatedCellTopologyId(foundSeedsTable.back(), 0, allocInt); - thrust::device_vector, gpu::TypedAllocator>> updatedCellSeed(foundSeedsTable.back(), allocTrackSeed); - gpu::processNeighboursKernel<<>>( - defaultCellTopologyId, - startLevel, - allCellSeeds, - currentCellSeeds, - nullptr, - nullptr, - nCells[defaultCellTopologyId], - thrust::raw_pointer_cast(&updatedCellSeed[0]), - thrust::raw_pointer_cast(&updatedCellId[0]), - thrust::raw_pointer_cast(&updatedCellTopologyId[0]), - thrust::raw_pointer_cast(&foundSeedsTable[0]), - usedClusters, - neighbours, - neighboursDeviceLUTs, - foundTrackingFrameInfo, - thrust::raw_pointer_cast(&layerxX0[0]), - bz, - maxChi2ClusterAttachment, - propagator, - matCorrType); - GPUChkErrS(cudaStreamSynchronize(gpu::Stream::DefaultStream)); + processLevel(currentCellSeeds, currentCellIds, currentCellTopologyIds, nCells[startCellTopologyId], startLevel, startCellTopologyId); int level = startLevel; - thrust::device_vector> lastCellId(allocInt); - thrust::device_vector> lastCellTopologyId(allocInt); - thrust::device_vector, gpu::TypedAllocator>> lastCellSeed(allocTrackSeed); - while (level > 2 && !updatedCellSeed.empty()) { - lastCellSeed.swap(updatedCellSeed); - lastCellId.swap(updatedCellId); - lastCellTopologyId.swap(updatedCellTopologyId); - thrust::device_vector, gpu::TypedAllocator>>(allocTrackSeed).swap(updatedCellSeed); - thrust::device_vector>(allocInt).swap(updatedCellId); - thrust::device_vector>(allocInt).swap(updatedCellTopologyId); - auto lastCellSeedSize{lastCellSeed.size()}; - foundSeedsTable.resize(lastCellSeedSize + 1); - thrust::fill(nosync_policy, foundSeedsTable.begin(), foundSeedsTable.end(), 0); - + while (level > 2 && nWaveSeeds > 0) { + const Slab& in = slabs[filled]; + const int nLastSeeds = nWaveSeeds; --level; - gpu::processNeighboursKernel><<>>( - constants::UnusedIndex, - level, - allCellSeeds, - thrust::raw_pointer_cast(&lastCellSeed[0]), - thrust::raw_pointer_cast(&lastCellId[0]), - thrust::raw_pointer_cast(&lastCellTopologyId[0]), - lastCellSeedSize, - nullptr, - nullptr, - nullptr, - thrust::raw_pointer_cast(&foundSeedsTable[0]), - usedClusters, - neighbours, - neighboursDeviceLUTs, - foundTrackingFrameInfo, - thrust::raw_pointer_cast(&layerxX0[0]), - bz, - maxChi2ClusterAttachment, - propagator, - matCorrType); - thrust::exclusive_scan(nosync_policy, foundSeedsTable.begin(), foundSeedsTable.end(), foundSeedsTable.begin()); - - auto foundSeeds{foundSeedsTable.back()}; - updatedCellId.resize(foundSeeds); - thrust::fill(nosync_policy, updatedCellId.begin(), updatedCellId.end(), 0); - updatedCellTopologyId.resize(foundSeeds); - thrust::fill(nosync_policy, updatedCellTopologyId.begin(), updatedCellTopologyId.end(), 0); - updatedCellSeed.resize(foundSeeds); - thrust::fill(nosync_policy, updatedCellSeed.begin(), updatedCellSeed.end(), TrackSeed()); + processLevel(thrust::raw_pointer_cast(in.seeds), thrust::raw_pointer_cast(in.cellIds), thrust::raw_pointer_cast(in.cellTopologyIds), + nLastSeeds, level, constants::UnusedIndex); + } - gpu::processNeighboursKernel><<>>( - constants::UnusedIndex, - level, - allCellSeeds, - thrust::raw_pointer_cast(&lastCellSeed[0]), - thrust::raw_pointer_cast(&lastCellId[0]), - thrust::raw_pointer_cast(&lastCellTopologyId[0]), - lastCellSeedSize, - thrust::raw_pointer_cast(&updatedCellSeed[0]), - thrust::raw_pointer_cast(&updatedCellId[0]), - thrust::raw_pointer_cast(&updatedCellTopologyId[0]), - thrust::raw_pointer_cast(&foundSeedsTable[0]), - usedClusters, - neighbours, - neighboursDeviceLUTs, - foundTrackingFrameInfo, - thrust::raw_pointer_cast(&layerxX0[0]), - bz, - maxChi2ClusterAttachment, - propagator, - matCorrType); + if (nWaveSeeds > 0) { + Slab& spare = slabs[filled == 0 ? 1 : 0]; + ensureCapacity(spare, nWaveSeeds); + const auto& last = slabs[filled]; + auto end = thrust::copy_if(nosync_policy, last.seeds, last.seeds + nWaveSeeds, spare.seeds, track::TrackSeedSelector{constants::MaxTrackSeedQ2Pt, maxChi2NDF, startLevel, maxHoles, minSeedingClusters, holeLayerMask, nonSeedingLayerMask}); + const int nSelected = static_cast(end - spare.seeds); + if (nSelected > 0 && seedsCursor + nSelected <= seedsCapacity) { + GPUChkErrS(cudaMemcpyAsync(seedsDevice + seedsCursor, thrust::raw_pointer_cast(spare.seeds), + nSelected * sizeof(TrackSeed), cudaMemcpyDeviceToDevice, + gpu::Stream::DefaultStream)); + GPUChkErrS(cudaStreamSynchronize(gpu::Stream::DefaultStream)); + } + seedsCursor += nSelected; } - GPUChkErrS(cudaStreamSynchronize(gpu::Stream::DefaultStream)); - thrust::device_vector, gpu::TypedAllocator>> outSeeds(updatedCellSeed.size(), allocTrackSeed); - auto end = thrust::copy_if(nosync_policy, updatedCellSeed.begin(), updatedCellSeed.end(), outSeeds.begin(), track::TrackSeedSelector{constants::MaxTrackSeedQ2Pt, maxChi2NDF, startLevel, maxHoles, minSeedingClusters, holeLayerMask, nonSeedingLayerMask}); - auto s{end - outSeeds.begin()}; - seedsHost.reserve(seedsHost.size() + s); - thrust::copy(outSeeds.begin(), outSeeds.begin() + s, std::back_inserter(seedsHost)); alloc->popTagOffStack(Tag); } template -void countTrackSeedHandler(TrackSeed* trackSeeds, - const TrackingFrameInfo** foundTrackingFrameInfo, - const Cluster** unsortedClusters, - int* seedLUT, - const std::vector& layerRadiiHost, - const std::vector& minPtsHost, - const std::vector& layerxX0Host, - const unsigned int nSeeds, - const float bz, - const float maxChi2ClusterAttachment, - const float maxChi2NDF, - const int reseedIfShorter, - const bool repeatRefitOut, - const bool shiftRefToCluster, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType, - o2::its::ExternalAllocator* alloc) +int TrackingKernels::computeTrackSeedHandler(TrackSeed* trackSeeds, + const TrackingFrameInfo** foundTrackingFrameInfo, + const Cluster** unsortedClusters, + const IndexTableUtils* utils, + const typename ROFMaskTable::View& rofMask, + const typename ROFOverlapTable::View& rofOverlaps, + const Cluster** clusters, + const unsigned char** usedClusters, + const int** clustersIndexTables, + const int** ROFClusters, + o2::its::TrackITSExt* tracks, + int* trackIndices, + int* trackSeedIndices, + int* outputCounter, + const int trackCapacity, + TrackExtensionHypothesis* activeHypotheses, + TrackExtensionHypothesis* nextHypotheses, + const float* layerRadii, + const float* minPts, + const float* layerxX0, + const unsigned int nSeeds, + const float bz, + const float maxChi2ClusterAttachment, + const float maxChi2NDF, + const int reseedIfShorter, + const bool repeatRefitOut, + const bool shiftRefToCluster, + const int nLayers, + const int phiBins, + const int maxHypotheses, + const bool extendTop, + const bool extendBot, + const float nSigmaCutPhi, + const float nSigmaCutZ, + const o2::base::Propagator* propagator, + const o2::base::PropagatorF::MatCorrType matCorrType, + o2::its::ExternalAllocator* alloc) { - // TODO: the minPts&layerRadii is transfered twice - // we should allocate this in constant memory and stop these - // small transferes! - thrust::device_vector minPts(minPtsHost); - thrust::device_vector layerRadii(layerRadiiHost); - thrust::device_vector layerxX0(layerxX0Host); - gpu::countTrackSeedsKernel<<>>( - trackSeeds, // CellSeed* - foundTrackingFrameInfo, // TrackingFrameInfo** - unsortedClusters, // Cluster** - seedLUT, // int* - thrust::raw_pointer_cast(&layerRadii[0]), // const float* - thrust::raw_pointer_cast(&minPts[0]), // const float* - thrust::raw_pointer_cast(&layerxX0[0]), // const float* - nSeeds, // const unsigned int - bz, // const float - maxChi2ClusterAttachment, // float - maxChi2NDF, // float - reseedIfShorter, // int - repeatRefitOut, // bool - shiftRefToCluster, // bool - propagator, // const o2::base::Propagator* - matCorrType); // o2::base::PropagatorF::MatCorrType - auto sync_policy = THRUST_NAMESPACE::par(gpu::TypedAllocator(alloc)); - thrust::exclusive_scan(sync_policy, seedLUT, seedLUT + nSeeds + 1, seedLUT); -} - -template -void computeTrackSeedHandler(TrackSeed* trackSeeds, - const TrackingFrameInfo** foundTrackingFrameInfo, - const Cluster** unsortedClusters, - const IndexTableUtils* utils, - const typename ROFMaskTable::View& rofMask, - const typename ROFOverlapTable::View& rofOverlaps, - const Cluster** clusters, - const unsigned char** usedClusters, - const int** clustersIndexTables, - const int** ROFClusters, - o2::its::TrackITSExt* tracks, - int* trackIndices, - const int* seedLUT, - TrackExtensionHypothesis* activeHypotheses, - TrackExtensionHypothesis* nextHypotheses, - const std::vector& layerRadiiHost, - const std::vector& minPtsHost, - const std::vector& layerxX0Host, - const unsigned int nSeeds, - const unsigned int nTracks, - const float bz, - const float maxChi2ClusterAttachment, - const float maxChi2NDF, - const int reseedIfShorter, - const bool repeatRefitOut, - const bool shiftRefToCluster, - const int nLayers, - const int phiBins, - const int maxHypotheses, - const bool extendTop, - const bool extendBot, - const float nSigmaCutPhi, - const float nSigmaCutZ, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType, - o2::its::ExternalAllocator* alloc) -{ - thrust::device_vector minPts(minPtsHost); - thrust::device_vector layerRadii(layerRadiiHost); - thrust::device_vector layerxX0(layerxX0Host); - gpu::fitTrackSeedsKernel<<>>( - trackSeeds, // CellSeed* - foundTrackingFrameInfo, // TrackingFrameInfo** - unsortedClusters, // Cluster** - utils, // IndexTableUtils* - rofMask, // ROFMaskTable::View - rofOverlaps, // ROFOverlapTable::View - clusters, // Cluster** - usedClusters, // unsigned char** - clustersIndexTables, // int** - ROFClusters, // int** - tracks, // TrackITSExt* - seedLUT, // const int* - activeHypotheses, // TrackExtensionHypothesis* - nextHypotheses, // TrackExtensionHypothesis* - thrust::raw_pointer_cast(&layerRadii[0]), // const float* - thrust::raw_pointer_cast(&minPts[0]), // const float* - thrust::raw_pointer_cast(&layerxX0[0]), // const float* - nSeeds, // const unsigned int - bz, // const float - maxChi2ClusterAttachment, // float - maxChi2NDF, // float - reseedIfShorter, // int - repeatRefitOut, // bool - shiftRefToCluster, // bool - nLayers, // int - phiBins, // int - maxHypotheses, // int - extendTop, // bool - extendBot, // bool - nSigmaCutPhi, // float - nSigmaCutZ, // float - propagator, // const o2::base::Propagator* - matCorrType); // o2::base::PropagatorF::MatCorrType + GPUChkErrS(cudaMemsetAsync(outputCounter, 0, sizeof(int), gpu::Stream::DefaultStream)); + gpu::fitTrackSeedsKernel<<>>( + trackSeeds, // CellSeed* + foundTrackingFrameInfo, // TrackingFrameInfo** + unsortedClusters, // Cluster** + utils, // IndexTableUtils* + rofMask, // ROFMaskTable::View + rofOverlaps, // ROFOverlapTable::View + clusters, // Cluster** + usedClusters, // unsigned char** + clustersIndexTables, // int** + ROFClusters, // int** + tracks, // TrackITSExt* + trackSeedIndices, // int* + outputCounter, // int* + trackCapacity, // const int + activeHypotheses, // TrackExtensionHypothesis* + nextHypotheses, // TrackExtensionHypothesis* + layerRadii, // const float* + minPts, // const float* + layerxX0, // const float* + nSeeds, // const unsigned int + bz, // const float + maxChi2ClusterAttachment, // float + maxChi2NDF, // float + reseedIfShorter, // int + repeatRefitOut, // bool + shiftRefToCluster, // bool + nLayers, // int + phiBins, // int + maxHypotheses, // int + extendTop, // bool + extendBot, // bool + nSigmaCutPhi, // float + nSigmaCutZ, // float + propagator, // const o2::base::Propagator* + matCorrType); // o2::base::PropagatorF::MatCorrType + int emitted{0}; + GPUChkErrS(cudaMemcpyAsync(&emitted, outputCounter, sizeof(int), cudaMemcpyDeviceToHost, gpu::Stream::DefaultStream)); + GPUChkErrS(cudaStreamSynchronize(gpu::Stream::DefaultStream)); + if (emitted > trackCapacity) { // the slab was too small, the caller resizes and calls again + return emitted; + } auto sync_policy = THRUST_NAMESPACE::par(gpu::TypedAllocator(alloc)); thrust::device_ptr trackIndicesPtr(trackIndices); - thrust::sequence(sync_policy, trackIndicesPtr, trackIndicesPtr + nTracks); - thrust::sort(sync_policy, trackIndicesPtr, trackIndicesPtr + nTracks, gpu::compare_track_index_chi2{tracks}); + thrust::sequence(sync_policy, trackIndicesPtr, trackIndicesPtr + emitted); + thrust::sort(sync_policy, trackIndicesPtr, trackIndicesPtr + emitted, gpu::compare_track_index_chi2{tracks, trackSeedIndices}); + return emitted; } -/// Explicit instantiation of ITS2 handlers -template void countTrackletsInROFsHandler<7>(const IndexTableUtils<7>* utils, - const ROFMaskTable<7>::View& rofMask, - const int linkId, - const int fromLayer, - const int toLayer, - const ROFOverlapTable<7>::View& rofOverlaps, - const ROFVertexLookupTable<7>::View& vertexLUT, - const int vertexId, - const Vertex* vertices, - const int* rofPV, - const Cluster** clusters, - std::vector nClusters, - const int** ROFClusters, - const unsigned char** usedClusters, - const int** clustersIndexTables, - int** trackletsLUTs, - gsl::span trackletsLUTsHost, - const bool selectUPCVertices, - const float NSigmaCut, - const TrackingTopology<7>::View topology, - bounded_vector& linkPhiCuts, - const float resolutionPV, - std::array& minRs, - std::array& maxRs, - bounded_vector& resolutions, - std::vector& radii, - bounded_vector& linkMSAngles, - o2::its::ExternalAllocator* alloc, - gpu::Streams& streams); - -template void computeTrackletsInROFsHandler<7>(const IndexTableUtils<7>* utils, - const ROFMaskTable<7>::View& rofMask, - const int linkId, - const int fromLayer, - const int toLayer, - const ROFOverlapTable<7>::View& rofOverlaps, - const ROFVertexLookupTable<7>::View& vertexLUT, - const int vertexId, - const Vertex* vertices, - const int* rofPV, - const Cluster** clusters, - std::vector nClusters, - const int** ROFClusters, - const unsigned char** usedClusters, - const int** clustersIndexTables, - Tracklet** tracklets, - gsl::span spanTracklets, - gsl::span nTracklets, - int** trackletsLUTs, - gsl::span trackletsLUTsHost, - const bool selectUPCVertices, - const float NSigmaCut, - const TrackingTopology<7>::View topology, - bounded_vector& linkPhiCuts, - const float resolutionPV, - std::array& minRs, - std::array& maxRs, - bounded_vector& resolutions, - std::vector& radii, - bounded_vector& linkMSAngles, - o2::its::ExternalAllocator* alloc, - gpu::Streams& streams); - -template void countCellsHandler<7>(const Cluster** sortedClusters, - const Cluster** unsortedClusters, - const TrackingFrameInfo** tfInfo, - Tracklet** tracklets, - int** trackletsLUT, - const int nTracklets, - const int cellTopologyId, - const TrackingTopology<7>::View topology, - CellSeed* cells, - int** cellsLUTsArrayDevice, - int* cellsLUTsHost, - const float bz, - const float maxChi2ClusterAttachment, - const float cellDeltaTanLambdaSigma, - const float nSigmaCut, - const std::vector& layerxX0Host, - o2::its::ExternalAllocator* alloc, - gpu::Streams& streams); - -template void computeCellsHandler<7>(const Cluster** sortedClusters, - const Cluster** unsortedClusters, - const TrackingFrameInfo** tfInfo, - Tracklet** tracklets, - int** trackletsLUT, - const int nTracklets, - const int cellTopologyId, - const TrackingTopology<7>::View topology, - CellSeed* cells, - int** cellsLUTsArrayDevice, - int* cellsLUTsHost, - const float bz, - const float maxChi2ClusterAttachment, - const float cellDeltaTanLambdaSigma, - const float nSigmaCut, - const std::vector& layerxX0Host, - gpu::Streams& streams); - -template void countCellNeighboursHandler<7>(CellSeed** cellsLayersDevice, - int* neighboursCursor, - int** cellsLUTs, - const int sourceCellTopologyId, - const int targetCellTopologyId, - const float maxChi2ClusterAttachment, - const float bz, - const unsigned int nCells, - gpu::Stream& stream); - -template void computeCellNeighboursHandler<7>(CellSeed** cellsLayersDevice, - int* neighboursCursor, - int** cellsLUTs, - CellNeighbour* cellNeighbours, - const int sourceCellTopologyId, - const int targetCellTopologyId, - const float maxChi2ClusterAttachment, - const float bz, - const unsigned int nCells, - gpu::Stream& stream); - -template void processNeighboursHandler<7>(const int startLevel, - const int defaultCellTopologyId, - CellSeed** allCellSeeds, - CellSeed* currentCellSeeds, - const int* currentCellTopologyIds, - const int* currentCellIds, - const int* nCells, - const unsigned char** usedClusters, - CellNeighbour** neighbours, - int** neighboursDeviceLUTs, - const TrackingFrameInfo** foundTrackingFrameInfo, - bounded_vector>& seedsHost, - const float bz, - const float maxChi2ClusterAttachment, - const float maxChi2NDF, - const int maxHoles, - const int minSeedingClusters, - const LayerMask holeLayerMask, - const LayerMask nonSeedingLayerMask, - const std::vector& layerxX0Host, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType, - o2::its::ExternalAllocator* alloc); - -template void countTrackSeedHandler(TrackSeed<7>* trackSeeds, - const TrackingFrameInfo** foundTrackingFrameInfo, - const Cluster** unsortedClusters, - int* seedLUT, - const std::vector& layerRadiiHost, - const std::vector& minPtsHost, - const std::vector& layerxX0Host, - const unsigned int nSeeds, - const float bz, - const float maxChi2ClusterAttachment, - const float maxChi2NDF, - const int reseedIfShorter, - const bool repeatRefitOut, - const bool shiftRefToCluster, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType, - o2::its::ExternalAllocator* alloc); - -template void computeTrackSeedHandler(TrackSeed<7>* trackSeeds, - const TrackingFrameInfo** foundTrackingFrameInfo, - const Cluster** unsortedClusters, - const IndexTableUtils<7>* utils, - const ROFMaskTable<7>::View& rofMask, - const ROFOverlapTable<7>::View& rofOverlaps, - const Cluster** clusters, - const unsigned char** usedClusters, - const int** clustersIndexTables, - const int** ROFClusters, - o2::its::TrackITSExt* tracks, - int* trackIndices, - const int* seedLUT, - TrackExtensionHypothesis<7>* activeHypotheses, - TrackExtensionHypothesis<7>* nextHypotheses, - const std::vector& layerRadiiHost, - const std::vector& minPtsHost, - const std::vector& layerxX0Host, - const unsigned int nSeeds, - const unsigned int nTracks, - const float bz, - const float maxChi2ClusterAttachment, - const float maxChi2NDF, - const int reseedIfShorter, - const bool repeatRefitOut, - const bool shiftRefToCluster, - const int nLayers, - const int phiBins, - const int maxHypotheses, - const bool extendTop, - const bool extendBot, - const float nSigmaCutPhi, - const float nSigmaCutZ, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType, - o2::its::ExternalAllocator* alloc); - -/// Explicit instantiation of ALICE3 handlers +/// One instantiation per detector layout emits every handler above. +template struct TrackingKernels<7>; #ifdef ENABLE_UPGRADES -template void countTrackletsInROFsHandler<11>(const IndexTableUtils<11>* utils, - const ROFMaskTable<11>::View& rofMask, - const int linkId, - const int fromLayer, - const int toLayer, - const ROFOverlapTable<11>::View& rofOverlaps, - const ROFVertexLookupTable<11>::View& vertexLUT, - const int vertexId, - const Vertex* vertices, - const int* rofPV, - const Cluster** clusters, - std::vector nClusters, - const int** ROFClusters, - const unsigned char** usedClusters, - const int** clustersIndexTables, - int** trackletsLUTs, - gsl::span trackletsLUTsHost, - const bool selectUPCVertices, - const float NSigmaCut, - const TrackingTopology<11>::View topology, - bounded_vector& linkPhiCuts, - const float resolutionPV, - std::array& minRs, - std::array& maxRs, - bounded_vector& resolutions, - std::vector& radii, - bounded_vector& linkMSAngles, - o2::its::ExternalAllocator* alloc, - gpu::Streams& streams); - -template void computeTrackletsInROFsHandler<11>(const IndexTableUtils<11>* utils, - const ROFMaskTable<11>::View& rofMask, - const int linkId, - const int fromLayer, - const int toLayer, - const ROFOverlapTable<11>::View& rofOverlaps, - const ROFVertexLookupTable<11>::View& vertexLUT, - const int vertexId, - const Vertex* vertices, - const int* rofPV, - const Cluster** clusters, - std::vector nClusters, - const int** ROFClusters, - const unsigned char** usedClusters, - const int** clustersIndexTables, - Tracklet** tracklets, - gsl::span spanTracklets, - gsl::span nTracklets, - int** trackletsLUTs, - gsl::span trackletsLUTsHost, - const bool selectUPCVertices, - const float NSigmaCut, - const TrackingTopology<11>::View topology, - bounded_vector& linkPhiCuts, - const float resolutionPV, - std::array& minRs, - std::array& maxRs, - bounded_vector& resolutions, - std::vector& radii, - bounded_vector& linkMSAngles, - o2::its::ExternalAllocator* alloc, - gpu::Streams& streams); - -template void countCellsHandler<11>(const Cluster** sortedClusters, - const Cluster** unsortedClusters, - const TrackingFrameInfo** tfInfo, - Tracklet** tracklets, - int** trackletsLUT, - const int nTracklets, - const int cellTopologyId, - const TrackingTopology<11>::View topology, - CellSeed* cells, - int** cellsLUTsArrayDevice, - int* cellsLUTsHost, - const float bz, - const float maxChi2ClusterAttachment, - const float cellDeltaTanLambdaSigma, - const float nSigmaCut, - const std::vector& layerxX0Host, - o2::its::ExternalAllocator* alloc, - gpu::Streams& streams); - -template void computeCellsHandler<11>(const Cluster** sortedClusters, - const Cluster** unsortedClusters, - const TrackingFrameInfo** tfInfo, - Tracklet** tracklets, - int** trackletsLUT, - const int nTracklets, - const int cellTopologyId, - const TrackingTopology<11>::View topology, - CellSeed* cells, - int** cellsLUTsArrayDevice, - int* cellsLUTsHost, - const float bz, - const float maxChi2ClusterAttachment, - const float cellDeltaTanLambdaSigma, - const float nSigmaCut, - const std::vector& layerxX0Host, - gpu::Streams& streams); - -template void countCellNeighboursHandler<11>(CellSeed** cellsLayersDevice, - int* neighboursCursor, - int** cellsLUTs, - const int sourceCellTopologyId, - const int targetCellTopologyId, - const float maxChi2ClusterAttachment, - const float bz, - const unsigned int nCells, - gpu::Stream& stream); - -template void computeCellNeighboursHandler<11>(CellSeed** cellsLayersDevice, - int* neighboursCursor, - int** cellsLUTs, - CellNeighbour* cellNeighbours, - const int sourceCellTopologyId, - const int targetCellTopologyId, - const float maxChi2ClusterAttachment, - const float bz, - const unsigned int nCells, - gpu::Stream& stream); - -template void processNeighboursHandler<11>(const int startLevel, - const int defaultCellTopologyId, - CellSeed** allCellSeeds, - CellSeed* currentCellSeeds, - const int* currentCellTopologyIds, - const int* currentCellIds, - const int* nCells, - const unsigned char** usedClusters, - CellNeighbour** neighbours, - int** neighboursDeviceLUTs, - const TrackingFrameInfo** foundTrackingFrameInfo, - bounded_vector>& seedsHost, - const float bz, - const float maxChi2ClusterAttachment, - const float maxChi2NDF, - const int maxHoles, - const int minSeedingClusters, - const LayerMask holeLayerMask, - const LayerMask nonSeedingLayerMask, - const std::vector& layerxX0Host, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType, - o2::its::ExternalAllocator* alloc); - -template void countTrackSeedHandler(TrackSeed<11>* trackSeeds, - const TrackingFrameInfo** foundTrackingFrameInfo, - const Cluster** unsortedClusters, - int* seedLUT, - const std::vector& layerRadiiHost, - const std::vector& minPtsHost, - const std::vector& layerxX0Host, - const unsigned int nSeeds, - const float bz, - const float maxChi2ClusterAttachment, - const float maxChi2NDF, - const int reseedIfShorter, - const bool repeatRefitOut, - const bool shiftRefToCluster, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType, - o2::its::ExternalAllocator* alloc); - -template void computeTrackSeedHandler(TrackSeed<11>* trackSeeds, - const TrackingFrameInfo** foundTrackingFrameInfo, - const Cluster** unsortedClusters, - const IndexTableUtils<11>* utils, - const ROFMaskTable<11>::View& rofMask, - const ROFOverlapTable<11>::View& rofOverlaps, - const Cluster** clusters, - const unsigned char** usedClusters, - const int** clustersIndexTables, - const int** ROFClusters, - o2::its::TrackITSExt* tracks, - int* trackIndices, - const int* seedLUT, - TrackExtensionHypothesis<11>* activeHypotheses, - TrackExtensionHypothesis<11>* nextHypotheses, - const std::vector& layerRadiiHost, - const std::vector& minPtsHost, - const std::vector& layerxX0Host, - const unsigned int nSeeds, - const unsigned int nTracks, - const float bz, - const float maxChi2ClusterAttachment, - const float maxChi2NDF, - const int reseedIfShorter, - const bool repeatRefitOut, - const bool shiftRefToCluster, - const int nLayers, - const int phiBins, - const int maxHypotheses, - const bool extendTop, - const bool extendBot, - const float nSigmaCutPhi, - const float nSigmaCutZ, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType, - o2::its::ExternalAllocator* alloc); - -template void countTrackletsInROFsHandler<13>(const IndexTableUtils<13>* utils, - const ROFMaskTable<13>::View& rofMask, - const int linkId, - const int fromLayer, - const int toLayer, - const ROFOverlapTable<13>::View& rofOverlaps, - const ROFVertexLookupTable<13>::View& vertexLUT, - const int vertexId, - const Vertex* vertices, - const int* rofPV, - const Cluster** clusters, - std::vector nClusters, - const int** ROFClusters, - const unsigned char** usedClusters, - const int** clustersIndexTables, - int** trackletsLUTs, - gsl::span trackletsLUTsHost, - const bool selectUPCVertices, - const float NSigmaCut, - const TrackingTopology<13>::View topology, - bounded_vector& linkPhiCuts, - const float resolutionPV, - std::array& minRs, - std::array& maxRs, - bounded_vector& resolutions, - std::vector& radii, - bounded_vector& linkMSAngles, - o2::its::ExternalAllocator* alloc, - gpu::Streams& streams); - -template void computeTrackletsInROFsHandler<13>(const IndexTableUtils<13>* utils, - const ROFMaskTable<13>::View& rofMask, - const int linkId, - const int fromLayer, - const int toLayer, - const ROFOverlapTable<13>::View& rofOverlaps, - const ROFVertexLookupTable<13>::View& vertexLUT, - const int vertexId, - const Vertex* vertices, - const int* rofPV, - const Cluster** clusters, - std::vector nClusters, - const int** ROFClusters, - const unsigned char** usedClusters, - const int** clustersIndexTables, - Tracklet** tracklets, - gsl::span spanTracklets, - gsl::span nTracklets, - int** trackletsLUTs, - gsl::span trackletsLUTsHost, - const bool selectUPCVertices, - const float NSigmaCut, - const TrackingTopology<13>::View topology, - bounded_vector& linkPhiCuts, - const float resolutionPV, - std::array& minRs, - std::array& maxRs, - bounded_vector& resolutions, - std::vector& radii, - bounded_vector& linkMSAngles, - o2::its::ExternalAllocator* alloc, - gpu::Streams& streams); - -template void countCellsHandler<13>(const Cluster** sortedClusters, - const Cluster** unsortedClusters, - const TrackingFrameInfo** tfInfo, - Tracklet** tracklets, - int** trackletsLUT, - const int nTracklets, - const int cellTopologyId, - const TrackingTopology<13>::View topology, - CellSeed* cells, - int** cellsLUTsArrayDevice, - int* cellsLUTsHost, - const float bz, - const float maxChi2ClusterAttachment, - const float cellDeltaTanLambdaSigma, - const float nSigmaCut, - const std::vector& layerxX0Host, - o2::its::ExternalAllocator* alloc, - gpu::Streams& streams); - -template void computeCellsHandler<13>(const Cluster** sortedClusters, - const Cluster** unsortedClusters, - const TrackingFrameInfo** tfInfo, - Tracklet** tracklets, - int** trackletsLUT, - const int nTracklets, - const int cellTopologyId, - const TrackingTopology<13>::View topology, - CellSeed* cells, - int** cellsLUTsArrayDevice, - int* cellsLUTsHost, - const float bz, - const float maxChi2ClusterAttachment, - const float cellDeltaTanLambdaSigma, - const float nSigmaCut, - const std::vector& layerxX0Host, - gpu::Streams& streams); - -template void countCellNeighboursHandler<13>(CellSeed** cellsLayersDevice, - int* neighboursCursor, - int** cellsLUTs, - const int sourceCellTopologyId, - const int targetCellTopologyId, - const float maxChi2ClusterAttachment, - const float bz, - const unsigned int nCells, - gpu::Stream& stream); - -template void computeCellNeighboursHandler<13>(CellSeed** cellsLayersDevice, - int* neighboursCursor, - int** cellsLUTs, - CellNeighbour* cellNeighbours, - const int sourceCellTopologyId, - const int targetCellTopologyId, - const float maxChi2ClusterAttachment, - const float bz, - const unsigned int nCells, - gpu::Stream& stream); - -template void processNeighboursHandler<13>(const int startLevel, - const int defaultCellTopologyId, - CellSeed** allCellSeeds, - CellSeed* currentCellSeeds, - const int* currentCellTopologyIds, - const int* currentCellIds, - const int* nCells, - const unsigned char** usedClusters, - CellNeighbour** neighbours, - int** neighboursDeviceLUTs, - const TrackingFrameInfo** foundTrackingFrameInfo, - bounded_vector>& seedsHost, - const float bz, - const float maxChi2ClusterAttachment, - const float maxChi2NDF, - const int maxHoles, - const int minSeedingClusters, - const LayerMask holeLayerMask, - const LayerMask nonSeedingLayerMask, - const std::vector& layerxX0Host, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType, - o2::its::ExternalAllocator* alloc); - -template void countTrackSeedHandler(TrackSeed<13>* trackSeeds, - const TrackingFrameInfo** foundTrackingFrameInfo, - const Cluster** unsortedClusters, - int* seedLUT, - const std::vector& layerRadiiHost, - const std::vector& minPtsHost, - const std::vector& layerxX0Host, - const unsigned int nSeeds, - const float bz, - const float maxChi2ClusterAttachment, - const float maxChi2NDF, - const int reseedIfShorter, - const bool repeatRefitOut, - const bool shiftRefToCluster, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType, - o2::its::ExternalAllocator* alloc); - -template void computeTrackSeedHandler(TrackSeed<13>* trackSeeds, - const TrackingFrameInfo** foundTrackingFrameInfo, - const Cluster** unsortedClusters, - const IndexTableUtils<13>* utils, - const ROFMaskTable<13>::View& rofMask, - const ROFOverlapTable<13>::View& rofOverlaps, - const Cluster** clusters, - const unsigned char** usedClusters, - const int** clustersIndexTables, - const int** ROFClusters, - o2::its::TrackITSExt* tracks, - int* trackIndices, - const int* seedLUT, - TrackExtensionHypothesis<13>* activeHypotheses, - TrackExtensionHypothesis<13>* nextHypotheses, - const std::vector& layerRadiiHost, - const std::vector& minPtsHost, - const std::vector& layerxX0Host, - const unsigned int nSeeds, - const unsigned int nTracks, - const float bz, - const float maxChi2ClusterAttachment, - const float maxChi2NDF, - const int reseedIfShorter, - const bool repeatRefitOut, - const bool shiftRefToCluster, - const int nLayers, - const int phiBins, - const int maxHypotheses, - const bool extendTop, - const bool extendBot, - const float nSigmaCutPhi, - const float nSigmaCutZ, - const o2::base::Propagator* propagator, - const o2::base::PropagatorF::MatCorrType matCorrType, - o2::its::ExternalAllocator* alloc); +template struct TrackingKernels<11>; +template struct TrackingKernels<13>; #endif + } // namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/hip/CMakeLists.txt b/Detectors/ITSMFT/ITS/tracking/GPU/hip/CMakeLists.txt index e28fe04c06772..c582d1d8ee396 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/hip/CMakeLists.txt +++ b/Detectors/ITSMFT/ITS/tracking/GPU/hip/CMakeLists.txt @@ -27,4 +27,8 @@ if(HIP_ENABLED) hip::host PRIVATE_LINK_LIBRARIES O2::GPUTrackingHIPExternalProvider TARGETVARNAME targetName) + set_target_gpu_arch("HIP" ${targetName}) + if(GPUCA_DETERMINISTIC_MODE GREATER_EQUAL ${GPUCA_DETERMINISTIC_MODE_MAP_GPU}) + target_compile_definitions(${targetName} PRIVATE GPUCA_DETERMINISTIC_MODE) + endif() endif() diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h index aa37de186b910..71daa38efabe0 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h @@ -16,8 +16,10 @@ #ifndef TRACKINGITSU_INCLUDE_CAPACITYESTIMATOR_H_ #define TRACKINGITSU_INCLUDE_CAPACITYESTIMATOR_H_ +#include #include #include +#include #include namespace o2::its @@ -28,9 +30,11 @@ enum SlabSite : uint8_t { Cells, Neighbours, Roads, + TrackSeeds, + Tracks, NSlabSite, }; -constexpr const char* const SlabSiteNames[SlabSite::NSlabSite]{"Tracklets", "Cells", "Neighbours", "Roads"}; +constexpr const char* const SlabSiteNames[SlabSite::NSlabSite]{"Tracklets", "Cells", "Neighbours", "Roads", "TrackSeeds", "Tracks"}; class CapacityEstimator { @@ -97,6 +101,8 @@ class CapacityEstimator void reset(); size_t capacity(uint64_t key, double scale) const; + size_t peakCapacity(uint64_t key) const; + double expected(uint64_t key, double scale) const; void update(uint64_t key, double scale, size_t emitted, size_t capacityUsed, bool overflowed, bool memoryLimited); void print() const; @@ -105,6 +111,26 @@ class CapacityEstimator std::unique_ptr mImpl; }; +template +int runOnSlab(CapacityEstimator& estimator, const CapacityEstimator::KeyType key, const double scale, Emit&& emit, const size_t floorCapacity = 0) +{ + const auto toInt = [](const size_t v) { return static_cast(std::min(v, static_cast(std::numeric_limits::max()))); }; + const int initialCapacity = toInt(estimator.capacity(key, scale)); + int capacity = std::max(initialCapacity, toInt(floorCapacity)); + int emitted = 0; + bool overflowed = false; + bool needsRetry = false; + do { + const int attemptCapacity = capacity; + emitted = emit(attemptCapacity); + needsRetry = emitted > attemptCapacity; + overflowed |= needsRetry; + capacity = emitted; + } while (needsRetry); + estimator.update(key, scale, emitted, initialCapacity, overflowed, false); + return emitted; +} + } // namespace o2::its #endif /* TRACKINGITSU_INCLUDE_CAPACITYESTIMATOR_H_ */ diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Constants.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Constants.h index a85578deea9a2..6c97c7fd69172 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Constants.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Constants.h @@ -27,21 +27,18 @@ constexpr float MB = KB * KB; constexpr float GB = MB * KB; constexpr bool DoTimeBenchmarks = true; constexpr bool SaveTimeBenchmarks = false; -constexpr float Tolerance = 1e-12; // numerical tolerance -constexpr int ClustersPerCell = 3; // number of clusters for a cell -constexpr int UnusedIndex = -1; // global unused flag -constexpr float UnsetValue = -999.f; // global unset value -constexpr float Radl = 9.36f; // Radiation length of Si [cm] -constexpr float Rho = 2.33f; // Density of Si [g/cm^3] -constexpr int MaxIter = 4; // Max. supported iterations -constexpr int MaxSelectedTrackletsPerCluster = 100; // vertexer: max lines per cluster -constexpr int NumberOfConcurrentSeeds = 16; // default split per worker for the final track fit/extraploation step -constexpr int MinNumberOfConcurrentSeeds = (1 << 8); // minimum chunk size for a worker for the final track fit/extraploation step -constexpr int MaxNumberOfConcurrentSeeds = (1 << 12); // maximum chunk size for a worker for the final track fit/extraploation step -constexpr float MaxTrackSeedQ2Pt = 1.e3f; // maximum q/pt for track seeds -constexpr int GPUBlocks = 60; // default CUDA/HIP launch blocks -constexpr int GPUThreads = 256; // default CUDA/HIP launch threads -constexpr int GPUThreadsTotal = GPUBlocks * GPUThreads; // default CUDA/HIP total launched threads +constexpr float Tolerance = 1e-12; // numerical tolerance +constexpr int ClustersPerCell = 3; // number of clusters for a cell +constexpr int UnusedIndex = -1; // global unused flag +constexpr float UnsetValue = -999.f; // global unset value +constexpr float Radl = 9.36f; // Radiation length of Si [cm] +constexpr float Rho = 2.33f; // Density of Si [g/cm^3] +constexpr int MaxIter = 4; // Max. supported iterations +constexpr int MaxSelectedTrackletsPerCluster = 100; // vertexer: max lines per cluster +constexpr int NumberOfConcurrentSeeds = 16; // default split per worker for the final track fit/extraploation step +constexpr int MinNumberOfConcurrentSeeds = (1 << 8); // minimum chunk size for a worker for the final track fit/extraploation step +constexpr int MaxNumberOfConcurrentSeeds = (1 << 12); // maximum chunk size for a worker for the final track fit/extraploation step +constexpr float MaxTrackSeedQ2Pt = 1.e3f; // maximum q/pt for track seeds namespace helpers { diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ExternalAllocator.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ExternalAllocator.h index 7d1e98736db2c..e858c4bb476f9 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ExternalAllocator.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ExternalAllocator.h @@ -38,6 +38,7 @@ class ExternalAllocator mType = old; return p; } + void* allocateStack(size_t s) { return allocate(s, (o2::gpu::GPUMemoryResource::MEMORY_GPU | o2::gpu::GPUMemoryResource::MEMORY_STACK)); diff --git a/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx b/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx index 6405e2aa0fb74..c1d3d275342aa 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx @@ -28,8 +28,10 @@ struct CapacityEstimator::Impl { struct Entry { float ratio{0.f}; float margin{0.f}; + size_t maxEmitted{0}; uint32_t nSamples{0}; uint32_t nLowStreak{0}; + uint32_t nOverflows{0}; ///< times the slab was too small and the work had to be redone }; explicit Impl(Config config) : cfg{config} {} @@ -66,12 +68,46 @@ size_t CapacityEstimator::capacity(uint64_t key, double scale) const if (!std::isfinite(raw) || raw < 0.) { return mImpl->cfg.floorSlots; } - if (raw >= static_cast(std::numeric_limits::max())) { + // A ratio is only meaningful at the scale it was measured at. Learned on a handful of inputs it + // can be arbitrarily large, and applying it to a scale orders of magnitude bigger asks for a slab + // nobody can allocate. Bound the request by what this site has ever actually emitted: overshooting + // burns memory that a bump allocator cannot give back, undershooting only costs one retry. + const size_t ceiling = std::max(mImpl->cfg.floorSlots, static_cast(double(e.maxEmitted) * double(mImpl->cfg.marginMax))); + if (raw >= static_cast(ceiling)) { + return ceiling; + } + return std::max(mImpl->cfg.floorSlots, static_cast(std::ceil(raw))); +} + +size_t CapacityEstimator::peakCapacity(uint64_t key) const +{ + std::lock_guard lock{mImpl->mutex}; + const auto it = mImpl->entries.find(key); + if (it == mImpl->entries.end() || it->second.maxEmitted == 0) { + return mImpl->cfg.floorSlots; + } + const auto& e = it->second; + const double raw = double(e.maxEmitted) * double(e.margin); + if (!std::isfinite(raw) || raw >= static_cast(std::numeric_limits::max())) { return std::numeric_limits::max(); } return std::max(mImpl->cfg.floorSlots, static_cast(std::ceil(raw))); } +double CapacityEstimator::expected(uint64_t key, double scale) const +{ + if (!(scale > 0.)) { + return 0.; + } + std::lock_guard lock{mImpl->mutex}; + const auto it = mImpl->entries.find(key); + if (it == mImpl->entries.end() || it->second.nSamples == 0) { + return 0.; + } + const double raw = double(it->second.ratio) * scale; + return std::isfinite(raw) && raw > 0. ? raw : 0.; +} + void CapacityEstimator::update(uint64_t key, double scale, size_t emitted, size_t capacityUsed, bool overflowed, bool memoryLimited) { if (!(scale > 0.)) { @@ -87,6 +123,7 @@ void CapacityEstimator::update(uint64_t key, double scale, size_t emitted, size_ } const auto sample = static_cast(double(emitted) / scale); e.ratio = firstSample ? sample : (cfg.alpha * sample) + ((1.f - cfg.alpha) * e.ratio); + e.maxEmitted = std::max(e.maxEmitted, emitted); ++e.nSamples; if (memoryLimited) { @@ -95,6 +132,7 @@ void CapacityEstimator::update(uint64_t key, double scale, size_t emitted, size_ return; } if (overflowed) { + ++e.nOverflows; e.nLowStreak = 0; if (!firstSample) { const float shortfall = capacityUsed ? static_cast(double(emitted) / double(capacityUsed)) : cfg.marginUp; @@ -134,7 +172,7 @@ void CapacityEstimator::print() const for (const auto key : keys) { const auto& value = mImpl->entries.at(key); const auto decoded = decodeKey(key); - LOGP(info, "\tSite:{} | iter:{} | var:({},{}) | slot:{} | ratio:{} | margin:{} | sam:{} | low:{}", SlabSiteNames[decoded.site], decoded.iteration, getVariantHigh(decoded.variant), getVariantLow(decoded.variant), decoded.slot, value.ratio, value.margin, value.nSamples, value.nLowStreak); + LOGP(info, "\tSite:{} | iter:{} | var:({},{}) | slot:{} | ratio:{} | margin:{} | maxEmitted:{} | sam:{} | low:{} | overflows:{}", SlabSiteNames[decoded.site], decoded.iteration, getVariantHigh(decoded.variant), getVariantLow(decoded.variant), decoded.slot, value.ratio, value.margin, value.maxEmitted, value.nSamples, value.nLowStreak, value.nOverflows); } } diff --git a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx index 28e967386e984..d8ff8442f908f 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -49,10 +50,16 @@ float Tracker::clustersToTracks(const LogFunc& logger, const LogFunc& e int iteration{0}, iVertex{0}; auto handleException = [&](const auto& err) { - LOGP(error, "Too much memory in {} in iteration {} iVtx={}: {:.2f} GB. Current limit is {:.2f} GB, check the detector status and/or the selections.", - StateNames[mCurStep], iteration, iVertex, - (double)mTimeFrame->getArtefactsMemory() / GB, - (double)mTrkParams[iteration].MaxMemory / GB); + if (mTrkParams[iteration].MaxMemory == std::numeric_limits::max()) { + LOGP(error, "Allocation failed in {} in iteration {} iVtx={} ({:.2f} GB of host artefacts, no host limit set), check the detector status and/or the selections.", + StateNames[mCurStep], iteration, iVertex, + (double)mTimeFrame->getArtefactsMemory() / GB); + } else { + LOGP(error, "Too much memory in {} in iteration {} iVtx={}: {:.2f} GB. Current limit is {:.2f} GB, check the detector status and/or the selections.", + StateNames[mCurStep], iteration, iVertex, + (double)mTimeFrame->getArtefactsMemory() / GB, + (double)mTrkParams[iteration].MaxMemory / GB); + } if (typeid(err) != typeid(std::bad_alloc)) { // only print if the exceptions is different from what is expected LOGP(error, "Exception: {}", err.what()); } diff --git a/Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx b/Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx index 9cae6fd11a132..d72ede1a01c09 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx +++ b/Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx @@ -418,6 +418,71 @@ BOOST_AUTO_TEST_CASE(estimator_converges_and_reacts_to_overflow) BOOST_TEST(est.capacity(key, scale) > cap); } +BOOST_AUTO_TEST_CASE(estimator_does_not_extrapolate_a_low_statistics_ratio) +{ + // A first sample taken on a handful of inputs sets the ratio outright, so without a ceiling the + // next timeframe would ask for a slab orders of magnitude past anything the site ever emitted. + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Roads, 2, CapacityEstimator::makeVariant(3, 3), 5); + constexpr size_t emitted = 100000; + + est.update(key, 2., emitted, est.capacity(key, 2.), true, false); // ratio of 50000, from two inputs + + const size_t asked = est.capacity(key, 500000.); + BOOST_TEST(asked <= emitted * 4u); // bounded by what this site has ever actually produced + BOOST_TEST(asked >= emitted); // but still enough headroom not to force a pointless retry +} + +BOOST_AUTO_TEST_CASE(estimator_reports_a_scale_independent_peak) +{ + // Sizing a buffer that has to serve several differently sized runs cannot use capacity(), which + // needs the scale of one particular run. + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Roads, 0, CapacityEstimator::makeVariant(5, 3), 2); + BOOST_TEST(est.peakCapacity(key) == 1024u); // cold start falls back to the floor + + est.update(key, 1000., 50000, 60000, false, false); + BOOST_TEST(est.peakCapacity(key) >= 50000u); + + est.update(key, 10., 700, 1024, false, false); // a much smaller run must not shrink the peak + BOOST_TEST(est.peakCapacity(key) >= 50000u); + BOOST_TEST(est.peakCapacity(key) <= 50000u * 4u); +} + +BOOST_AUTO_TEST_CASE(estimator_expected_tracks_the_current_input) +{ + // Chaining sites whose input is the previous one's output needs a margin-free prediction that + // follows this timeframe, not the largest one ever seen. + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Roads, 0, CapacityEstimator::makeVariant(5, 4), 4); + BOOST_TEST(est.expected(key, 1000.) == 0.); // nothing learned yet + + est.update(key, 1000., 2000, 2600, false, false); // ratio of 2 + BOOST_TEST(est.expected(key, 1000.) == 2000.); + BOOST_TEST(est.expected(key, 250.) == 500.); // a smaller timeframe predicts proportionally less + BOOST_TEST(est.expected(key, 0.) == 0.); + + // ... while the all-time peak stays where it was, which is why it cannot size a shared buffer. + BOOST_TEST(est.peakCapacity(key) >= 2000u); +} + +BOOST_AUTO_TEST_CASE(estimator_ceiling_follows_real_growth) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Cells, 0, 0, 1); + constexpr double scale = 1000.; + size_t need = 10000; + + for (int tf = 0; tf < 6; ++tf) { + const size_t cap = est.capacity(key, scale); + est.update(key, scale, need, cap, need > cap, false); + need *= 2; + } + // Each timeframe doubled the output; the ceiling has to have followed, or every one of them + // would have paid for a retry. + BOOST_TEST(est.capacity(key, scale) >= need / 2); +} + BOOST_AUTO_TEST_CASE(estimator_backs_off_when_the_pool_refuses) { CapacityEstimator est; diff --git a/GPU/Workflow/include/GPUWorkflow/GPUWorkflowSpec.h b/GPU/Workflow/include/GPUWorkflow/GPUWorkflowSpec.h index 978c5f312cfe7..4f9ca7138f500 100644 --- a/GPU/Workflow/include/GPUWorkflow/GPUWorkflowSpec.h +++ b/GPU/Workflow/include/GPUWorkflow/GPUWorkflowSpec.h @@ -167,6 +167,7 @@ class GPURecoWorkflowSpec : public o2::framework::Task /// initialize TPC options from command line void initFunctionTPCCalib(o2::framework::InitContext& ic); void initFunctionITS(o2::framework::InitContext& ic); + void exitFunctionITS(); /// storing new calib objects in buffer void finaliseCCDBTPC(o2::framework::ConcreteDataMatcher& matcher, void* obj); void finaliseCCDBITS(o2::framework::ConcreteDataMatcher& matcher, void* obj); diff --git a/GPU/Workflow/src/GPUWorkflowITS.cxx b/GPU/Workflow/src/GPUWorkflowITS.cxx index 2a0e36d65bb7a..94c51ed2d183e 100644 --- a/GPU/Workflow/src/GPUWorkflowITS.cxx +++ b/GPU/Workflow/src/GPUWorkflowITS.cxx @@ -67,6 +67,11 @@ void GPURecoWorkflowSpec::initFunctionITS(o2::framework::InitContext& ic) mITSTrackingInterface->setTraitsFromProvider(vtxTraits, trkTraits, mITSTimeFrame); } +void GPURecoWorkflowSpec::exitFunctionITS() +{ + mITSTrackingInterface.reset(nullptr); +} + void GPURecoWorkflowSpec::finaliseCCDBITS(o2::framework::ConcreteDataMatcher& matcher, void* obj) { mITSTrackingInterface->finaliseCCDB(matcher, obj); diff --git a/GPU/Workflow/src/GPUWorkflowSpec.cxx b/GPU/Workflow/src/GPUWorkflowSpec.cxx index e51a45044da2b..15c789a2c1e09 100644 --- a/GPU/Workflow/src/GPUWorkflowSpec.cxx +++ b/GPU/Workflow/src/GPUWorkflowSpec.cxx @@ -371,6 +371,12 @@ void GPURecoWorkflowSpec::stop() void GPURecoWorkflowSpec::endOfStream(EndOfStreamContext& ec) { + if (mSpecConfig.runITSTracking && mITSTrackingInterface != nullptr) { + if (static bool printOnce{false}; !printOnce) { + printOnce = true; + mITSTrackingInterface->printSummary(); + } + } handlePipelineEndOfStream(ec); } @@ -1417,6 +1423,9 @@ Outputs GPURecoWorkflowSpec::outputs() void GPURecoWorkflowSpec::deinitialize() { + if (mSpecConfig.runITSTracking) { + exitFunctionITS(); + } ExitPipeline(); mQA.reset(nullptr); mDisplayFrontend.reset(nullptr); diff --git a/cmake/O2AddHipifiedExecutable.cmake b/cmake/O2AddHipifiedExecutable.cmake index c7354fd989e76..14ce37ec526b2 100644 --- a/cmake/O2AddHipifiedExecutable.cmake +++ b/cmake/O2AddHipifiedExecutable.cmake @@ -78,4 +78,9 @@ function(o2_add_hipified_executable baseTargetName) o2_add_executable("${baseTargetName}" SOURCES ${HIP_SOURCES} ${FORWARD_ARGS}) + + # Export architecture name + if(A_TARGETVARNAME) + set(${A_TARGETVARNAME} ${${A_TARGETVARNAME}} PARENT_SCOPE) + endif() endfunction() diff --git a/cmake/O2AddHipifiedLibrary.cmake b/cmake/O2AddHipifiedLibrary.cmake index a9d8602bf87e3..df4f35353a9fc 100644 --- a/cmake/O2AddHipifiedLibrary.cmake +++ b/cmake/O2AddHipifiedLibrary.cmake @@ -72,4 +72,9 @@ function(o2_add_hipified_library baseTargetName) o2_add_library("${baseTargetName}" SOURCES ${HIP_SOURCES} ${FORWARD_ARGS}) + + # Export architecture name + if(A_TARGETVARNAME) + set(${A_TARGETVARNAME} ${${A_TARGETVARNAME}} PARENT_SCOPE) + endif() endfunction()