From 203b86a6abeb3aa1956b5a36a20b62c931ac3fa3 Mon Sep 17 00:00:00 2001 From: jesgum Date: Thu, 1 Oct 2026 18:29:29 +0200 Subject: [PATCH] Refactor alice3 configuration parser --- ALICE3/Core/CMakeLists.txt | 1 + ALICE3/Core/ConfigurationParser.cxx | 211 ++++++++++++++++++++++++++++ ALICE3/Core/ConfigurationParser.h | 53 +++++++ ALICE3/Core/FlatTrackSmearer.cxx | 3 +- ALICE3/Core/GeometryContainer.cxx | 185 +----------------------- ALICE3/Core/GeometryContainer.h | 24 +--- 6 files changed, 272 insertions(+), 205 deletions(-) create mode 100644 ALICE3/Core/ConfigurationParser.cxx create mode 100644 ALICE3/Core/ConfigurationParser.h diff --git a/ALICE3/Core/CMakeLists.txt b/ALICE3/Core/CMakeLists.txt index 55f916861a6..4bfb95fba00 100644 --- a/ALICE3/Core/CMakeLists.txt +++ b/ALICE3/Core/CMakeLists.txt @@ -11,6 +11,7 @@ o2physics_add_library(ALICE3Core SOURCES TrackUtilities.cxx + ConfigurationParser.cxx FlatLutEntry.cxx FlatTrackSmearer.cxx GeometryContainer.cxx diff --git a/ALICE3/Core/ConfigurationParser.cxx b/ALICE3/Core/ConfigurationParser.cxx new file mode 100644 index 00000000000..0682090d08f --- /dev/null +++ b/ALICE3/Core/ConfigurationParser.cxx @@ -0,0 +1,211 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +/// +/// \file ConfigurationParser.cxx +/// \brief Utilities to access and parse TEnv configuration files for the ALICE3 fast simulation +/// \author Nicolò Jacazio, Università del Piemonte Orientale (IT) +/// + +#include "ConfigurationParser.h" + +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace o2::fastsim +{ + +std::map> ConfigurationParser::parseTEnvConfiguration(std::string& filename, std::vector& entries) +{ + std::map> configMap; + filename = gSystem->ExpandPathName(filename.c_str()); + LOG(info) << "Parsing TEnv configuration file: " << filename; + TEnv env(filename.c_str()); + THashList* table = env.GetTable(); + entries.clear(); + for (int i = 0; i < table->GetEntries(); ++i) { + const std::string key = table->At(i)->GetName(); + // key should contain exactly one dot + if (key.find('.') == std::string::npos || key.find('.') != key.rfind('.')) { + LOG(fatal) << "Key " << key << " does not contain exactly one dot"; + continue; + } + const std::string firstPart = key.substr(0, key.find('.')); + if (std::find(entries.begin(), entries.end(), firstPart) == entries.end()) { + entries.push_back(firstPart); + } + } + env.Print(); + // Entries + for (const auto& entry : entries) { + LOG(info) << " Reading entry " << entry; + for (int i = 0; i < table->GetEntries(); ++i) { + const std::string key = table->At(i)->GetName(); + if (key.find(entry + ".") == 0) { + const std::string paramName = key.substr(key.find('.') + 1); + const std::string value = env.GetValue(key.c_str(), ""); + configMap[entry][paramName] = value; + } + } + } + return configMap; +} + +std::string ConfigurationParser::accessFile(const std::string& path, const std::string& downloadPath, o2::ccdb::BasicCCDBManager* ccdb, int timeoutSeconds) +{ + if (path.rfind("ccdb:", 0) == 0) { + const std::string ccdbPath = path.substr(5); // remove "ccdb:" prefix + const std::string localPath = Form("%s/%s/snapshot.root", downloadPath.c_str(), ccdbPath.c_str()); + const std::string lockFile = localPath + ".lock"; + const std::string doneFile = localPath + ".done"; + + // Create directory structure if it doesn't exist + std::string dirPath = localPath.substr(0, localPath.find_last_of('/')); + gSystem->mkdir(dirPath.c_str(), true); + + // Check if file is already fully downloaded + struct stat buffer; + if (stat(doneFile.c_str(), &buffer) == 0) { + LOG(info) << " --- Geometry configuration file already exists: " << localPath << ". Skipping download."; + return localPath; + } + + // Acquire file lock for inter-process synchronization + int lockFd = open(lockFile.c_str(), O_CREAT | O_RDWR, 0666); + if (lockFd == -1) { + LOG(error) << " --- Failed to create lock file: " << lockFile; + return localPath; + } + + // Try to acquire exclusive lock (non-blocking) + LOG(info) << " --- Attempting to acquire lock for: " << localPath; + int lockResult = flock(lockFd, LOCK_EX | LOCK_NB); + + if (lockResult == -1 && errno == EWOULDBLOCK) { + // Lock is held by another process - wait up to 10 minutes for download to complete + LOG(info) << " --- Lock is held by another process. Waiting for download to complete (up to 10 minutes)..."; + close(lockFd); + + const auto startTime = std::chrono::steady_clock::now(); + const auto timeout = std::chrono::minutes(10); + const auto checkInterval = std::chrono::seconds(5); + + while (true) { + // Check if download is complete + if (stat(doneFile.c_str(), &buffer) == 0) { + LOG(info) << " --- Geometry configuration file was downloaded by another process: " << localPath; + return localPath; + } + + // Check timeout + auto elapsed = std::chrono::steady_clock::now() - startTime; + if (elapsed >= timeout) { + LOG(fatal) << " --- Timeout waiting for file download: " << localPath << ". Waited for 10 minutes."; + return localPath; + } + + // Wait before checking again + std::this_thread::sleep_for(checkInterval); + } + } else if (lockResult == -1) { + LOG(error) << " --- Failed to acquire lock for: " << lockFile; + close(lockFd); + return localPath; + } + + // Lock acquired successfully - double-check if file was downloaded while we were trying + if (stat(doneFile.c_str(), &buffer) == 0) { + LOG(info) << " --- Geometry configuration file was downloaded by another process: " << localPath; + flock(lockFd, LOCK_UN); + close(lockFd); + return localPath; + } + + // File does not exist, retrieve from CCDB + LOG(info) << " --- CCDB source detected for " << path; + std::map metadata; + bool status = ccdb->getCCDBAccessor().retrieveBlob(ccdbPath, downloadPath, metadata, 1); + if (!status) { + flock(lockFd, LOCK_UN); + close(lockFd); + LOG(fatal) << " --- Failed to retrieve configuration from CCDB for path: " << ccdbPath; + return ""; + } + LOG(info) << " --- Retrieved configuration from CCDB to: " << localPath; + + // Verify the integrity of the downloaded file + if (stat(localPath.c_str(), &buffer) != 0) { + flock(lockFd, LOCK_UN); + close(lockFd); + LOG(fatal) << " --- Downloaded file does not exist or is not accessible: " << localPath; + return ""; + } + if (buffer.st_size == 0) { + flock(lockFd, LOCK_UN); + close(lockFd); + LOG(fatal) << " --- Downloaded file is empty: " << localPath; + return ""; + } + LOG(info) << " --- File integrity verified. Size: " << buffer.st_size << " bytes"; + + // Create done marker file to indicate successful download + std::ofstream doneMarker(doneFile); + doneMarker.close(); + + // Release lock + flock(lockFd, LOCK_UN); + close(lockFd); + + // If timeout is specified, schedule file deletion after timeout + if (timeoutSeconds > 0) { + LOG(info) << " --- Deleting configuration file after timeout: " << localPath; + std::thread deletionThread([localPath, doneFile, timeoutSeconds]() { + LOG(info) << " --- Operating deletion of configuration file after timeout: " << localPath; + std::this_thread::sleep_for(std::chrono::seconds(timeoutSeconds)); + if (std::remove(localPath.c_str()) == 0) { + LOG(info) << " --- File deleted successfully: " << localPath; + } else { + LOG(warning) << " --- Failed to delete file: " << localPath; + } + // Also remove the done marker file + if (std::remove(doneFile.c_str()) == 0) { + LOG(info) << " --- Done marker deleted: " << doneFile; + } + }); + deletionThread.detach(); + } + + return localPath; + } + return path; +} + +} // namespace o2::fastsim diff --git a/ALICE3/Core/ConfigurationParser.h b/ALICE3/Core/ConfigurationParser.h new file mode 100644 index 00000000000..b92a02e59ec --- /dev/null +++ b/ALICE3/Core/ConfigurationParser.h @@ -0,0 +1,53 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +/// +/// \file ConfigurationParser.h +/// \brief Utilities to access and parse TEnv configuration files for the ALICE3 fast simulation +/// \author Nicolò Jacazio, Università del Piemonte Orientale (IT) +/// + +#ifndef ALICE3_CORE_CONFIGURATIONPARSER_H_ +#define ALICE3_CORE_CONFIGURATIONPARSER_H_ + +#include + +#include +#include +#include + +namespace o2::fastsim +{ + +class ConfigurationParser +{ + public: + /** + * @brief Parses a TEnv configuration file with keys of the form "." and returns the key-value pairs split per entry + * @param filename Path to the TEnv configuration file + * @param entries Vector to store the order of the entries as they appear in the file + * @return A map where each key is an entry name and the value is another map of key-value pairs for that entry + */ + static std::map> parseTEnvConfiguration(std::string& filename, std::vector& entries); + + /** + * @brief Accesses a file given its path, which can be either a local path or a ccdb path (starting with "ccdb:"). In the first case it returns the local path, in the second it retrieves the file from ccdb and returns the local path to the retrieved file. + * @param path The path to the file, either local or ccdb (starting with "ccdb:") + * @param downloadPath The local path where to download the file if it's a ccdb path. Default is "/tmp/GeometryContainer/" + * @param ccdb Pointer to the CCDB manager to use for retrieving the file if it's a ccdb path. Must be set when path is a ccdb path. + * @param timeoutSeconds If positive, then this function will wait for these seconds after download before removing the downloaded file. + * @return The local path to the file, either the original local path or the path to the retrieved file from ccdb + */ + static std::string accessFile(const std::string& path, const std::string& downloadPath = "/tmp/GeometryContainer/", o2::ccdb::BasicCCDBManager* ccdb = nullptr, int timeoutSeconds = 0); +}; + +} // namespace o2::fastsim + +#endif // ALICE3_CORE_CONFIGURATIONPARSER_H_ diff --git a/ALICE3/Core/FlatTrackSmearer.cxx b/ALICE3/Core/FlatTrackSmearer.cxx index 64c370d52a1..1bd79f088a8 100644 --- a/ALICE3/Core/FlatTrackSmearer.cxx +++ b/ALICE3/Core/FlatTrackSmearer.cxx @@ -11,6 +11,7 @@ #include "FlatTrackSmearer.h" +#include "ALICE3/Core/ConfigurationParser.h" #include "ALICE3/Core/FlatLutEntry.h" #include "ALICE3/Core/GeometryContainer.h" @@ -96,7 +97,7 @@ bool TrackSmearer::loadTable(int pdg, const char* filename, bool forceReload) } LOGF(info, "Loading %s LUT file: '%s'", getParticleName(pdg), filename); - const std::string localFilename = o2::fastsim::GeometryEntry::accessFile(filename, "./.ALICE3/LUTs/", mCcdbManager, 10); + const std::string localFilename = o2::fastsim::ConfigurationParser::accessFile(filename, "./.ALICE3/LUTs/", mCcdbManager, o2::fastsim::GeometryContainer::cleanLutWhenLoaded() ? 10 : 0); std::ifstream lutFile(localFilename, std::ifstream::binary); if (!lutFile.is_open()) { diff --git a/ALICE3/Core/GeometryContainer.cxx b/ALICE3/Core/GeometryContainer.cxx index 79f1a2d67c5..7936e66b629 100644 --- a/ALICE3/Core/GeometryContainer.cxx +++ b/ALICE3/Core/GeometryContainer.cxx @@ -9,7 +9,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. /// -/// \file GeometryContainer.h +/// \file GeometryContainer.cxx /// \author Nicolò Jacazio, Università del Piemonte Orientale (IT) /// \brief Set of utilities for the ALICE3 geometry handling /// \since February 13, 2026 @@ -23,66 +23,13 @@ #include #include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include #include #include -#include #include -#include -#include - namespace o2::fastsim { -std::map> GeometryEntry::parseTEnvConfiguration(std::string& filename, std::vector& layers) -{ - std::map> configMap; - filename = gSystem->ExpandPathName(filename.c_str()); - LOG(info) << "Parsing TEnv configuration file: " << filename; - TEnv env(filename.c_str()); - THashList* table = env.GetTable(); - layers.clear(); - for (int i = 0; i < table->GetEntries(); ++i) { - const std::string key = table->At(i)->GetName(); - // key should contain exactly one dot - if (key.find('.') == std::string::npos || key.find('.') != key.rfind('.')) { - LOG(fatal) << "Key " << key << " does not contain exactly one dot"; - continue; - } - const std::string firstPart = key.substr(0, key.find('.')); - if (std::find(layers.begin(), layers.end(), firstPart) == layers.end()) { - layers.push_back(firstPart); - } - } - env.Print(); - // Layers - for (const auto& layer : layers) { - LOG(info) << " Reading layer " << layer; - for (int i = 0; i < table->GetEntries(); ++i) { - const std::string key = table->At(i)->GetName(); - if (key.find(layer + ".") == 0) { - const std::string paramName = key.substr(key.find('.') + 1); - const std::string value = env.GetValue(key.c_str(), ""); - configMap[layer][paramName] = value; - } - } - } - return configMap; -} - bool GeometryContainer::mCleanLutWhenLoaded = true; void GeometryContainer::init(o2::framework::InitContext& initContext) { @@ -159,134 +106,4 @@ void GeometryEntry::replaceValue(const std::string& layerName, const std::string setValue(layerName, key, value); } -std::string GeometryEntry::accessFile(const std::string& path, const std::string& downloadPath, o2::ccdb::BasicCCDBManager* ccdb, int timeoutSeconds) -{ - - if (path.rfind("ccdb:", 0) == 0) { - const std::string ccdbPath = path.substr(5); // remove "ccdb:" prefix - const std::string localPath = Form("%s/%s/snapshot.root", downloadPath.c_str(), ccdbPath.c_str()); - const std::string lockFile = localPath + ".lock"; - const std::string doneFile = localPath + ".done"; - - // Create directory structure if it doesn't exist - std::string dirPath = localPath.substr(0, localPath.find_last_of('/')); - gSystem->mkdir(dirPath.c_str(), true); - - // Check if file is already fully downloaded - struct stat buffer; - if (stat(doneFile.c_str(), &buffer) == 0) { - LOG(info) << " --- Geometry configuration file already exists: " << localPath << ". Skipping download."; - return localPath; - } - - // Acquire file lock for inter-process synchronization - int lockFd = open(lockFile.c_str(), O_CREAT | O_RDWR, 0666); - if (lockFd == -1) { - LOG(error) << " --- Failed to create lock file: " << lockFile; - return localPath; - } - - // Try to acquire exclusive lock (non-blocking) - LOG(info) << " --- Attempting to acquire lock for: " << localPath; - int lockResult = flock(lockFd, LOCK_EX | LOCK_NB); - - if (lockResult == -1 && errno == EWOULDBLOCK) { - // Lock is held by another process - wait up to 10 minutes for download to complete - LOG(info) << " --- Lock is held by another process. Waiting for download to complete (up to 10 minutes)..."; - close(lockFd); - - const auto startTime = std::chrono::steady_clock::now(); - const auto timeout = std::chrono::minutes(10); - const auto checkInterval = std::chrono::seconds(5); - - while (true) { - // Check if download is complete - if (stat(doneFile.c_str(), &buffer) == 0) { - LOG(info) << " --- Geometry configuration file was downloaded by another process: " << localPath; - return localPath; - } - - // Check timeout - auto elapsed = std::chrono::steady_clock::now() - startTime; - if (elapsed >= timeout) { - LOG(fatal) << " --- Timeout waiting for geometry file download: " << localPath << ". Waited for 10 minutes."; - return localPath; - } - - // Wait before checking again - std::this_thread::sleep_for(checkInterval); - } - } else if (lockResult == -1) { - LOG(error) << " --- Failed to acquire lock for: " << lockFile; - close(lockFd); - return localPath; - } - - // Lock acquired successfully - double-check if file was downloaded while we were trying - if (stat(doneFile.c_str(), &buffer) == 0) { - LOG(info) << " --- Geometry configuration file was downloaded by another process: " << localPath; - flock(lockFd, LOCK_UN); - close(lockFd); - return localPath; - } - - // File does not exist, retrieve from CCDB - LOG(info) << " --- CCDB source detected for detector geometry " << path; - std::map metadata; - bool status = ccdb->getCCDBAccessor().retrieveBlob(ccdbPath, downloadPath, metadata, 1); - if (!status) { - flock(lockFd, LOCK_UN); - close(lockFd); - LOG(fatal) << " --- Failed to retrieve geometry configuration from CCDB for path: " << ccdbPath; - return ""; - } - LOG(info) << " --- Retrieved geometry configuration from CCDB to: " << localPath; - - // Verify the integrity of the downloaded file - if (stat(localPath.c_str(), &buffer) != 0) { - flock(lockFd, LOCK_UN); - close(lockFd); - LOG(fatal) << " --- Downloaded file does not exist or is not accessible: " << localPath; - return ""; - } - if (buffer.st_size == 0) { - flock(lockFd, LOCK_UN); - close(lockFd); - LOG(fatal) << " --- Downloaded file is empty: " << localPath; - return ""; - } - LOG(info) << " --- File integrity verified. Size: " << buffer.st_size << " bytes"; - - // Create done marker file to indicate successful download - std::ofstream doneMarker(doneFile); - doneMarker.close(); - - // Release lock - flock(lockFd, LOCK_UN); - close(lockFd); - - // If timeout is specified, schedule file deletion after timeout - if (timeoutSeconds > 0 && GeometryContainer::cleanLutWhenLoaded()) { - LOG(info) << " --- Deleting geometry configuration file after timeout: " << localPath; - std::thread deletionThread([localPath, doneFile, timeoutSeconds]() { - LOG(info) << " --- Operating deletion of geometry configuration file after timeout: " << localPath; - std::this_thread::sleep_for(std::chrono::seconds(timeoutSeconds)); - if (std::remove(localPath.c_str()) == 0) { - LOG(info) << " --- File deleted successfully: " << localPath; - } else { - LOG(warning) << " --- Failed to delete file: " << localPath; - } - // Also remove the done marker file - if (std::remove(doneFile.c_str()) == 0) { - LOG(info) << " --- Done marker deleted: " << doneFile; - } - }); - deletionThread.detach(); - } - - return localPath; - } - return path; -} - } // namespace o2::fastsim diff --git a/ALICE3/Core/GeometryContainer.h b/ALICE3/Core/GeometryContainer.h index 67037c1b183..c343ecedbb0 100644 --- a/ALICE3/Core/GeometryContainer.h +++ b/ALICE3/Core/GeometryContainer.h @@ -18,6 +18,8 @@ #ifndef ALICE3_CORE_GEOMETRYCONTAINER_H_ #define ALICE3_CORE_GEOMETRYCONTAINER_H_ +#include "ALICE3/Core/ConfigurationParser.h" + #include #include #include @@ -35,32 +37,14 @@ struct GeometryEntry { GeometryEntry() = default; explicit GeometryEntry(const std::string& filename, o2::ccdb::BasicCCDBManager* ccdb = nullptr) { - mFileName = accessFile(filename, "./.ALICE3/Configuration/", ccdb); - mConfigurations = GeometryEntry::parseTEnvConfiguration(mFileName, mLayerNames); + mFileName = ConfigurationParser::accessFile(filename, "./.ALICE3/Configuration/", ccdb); + mConfigurations = ConfigurationParser::parseTEnvConfiguration(mFileName, mLayerNames); LOG(info) << "Loaded geometry configuration from file: " << mFileName << " with " << mLayerNames.size() << " layers."; if (mLayerNames.empty()) { LOG(warning) << "No layers found in geometry configuration file: " << filename; } } - /** - * @brief Parses a TEnv configuration file and returns the key-value pairs split per entry - * @param filename Path to the TEnv configuration file - * @param layers Vector to store the order of the layers as they appear in the file - * @return A map where each key is a layer name and the value is another map of key-value pairs for that layer - */ - static std::map> parseTEnvConfiguration(std::string& filename, std::vector& layers); - - /** - * @brief Accesses a file given its path, which can be either a local path or a ccdb path (starting with "ccdb:"). In the first case it returns the local path, in the second it retrieves the file from ccdb and returns the local path to the retrieved file. - * @param path The path to the file, either local or ccdb (starting with "ccdb:") - * @param downloadPath The local path where to download the file if it's a ccdb path. Default is "/tmp/GeometryContainer/" - * @param ccdb Pointer to the CCDB manager to use for retrieving the file if it's a ccdb path. If nullptr, the function will create a temporary CCDB manager instance. Default is nullptr. - * @param timeoutSeconds If positive, then this function will wait for these seconds after download before removing the downloaded file. - * @return The local path to the file, either the original local path or the path to the retrieved file from ccdb - */ - static std::string accessFile(const std::string& path, const std::string& downloadPath = "/tmp/GeometryContainer/", o2::ccdb::BasicCCDBManager* ccdb = nullptr, int timeoutSeconds = 0); - std::map> getConfigurations() const { return mConfigurations; } std::map getConfiguration(const std::string& layerName) const; std::vector getLayerNames() const { return mLayerNames; }