diff --git a/Detectors/gconfig/CMakeLists.txt b/Detectors/gconfig/CMakeLists.txt index d35bbed3deb36..cb4af087cef05 100644 --- a/Detectors/gconfig/CMakeLists.txt +++ b/Detectors/gconfig/CMakeLists.txt @@ -74,6 +74,12 @@ o2_target_root_dictionary(SimSetup include/SimSetup/MCReplayParam.h LINKDEF src/GConfLinkDef.h) +o2_add_test(StaticTLSReserve + SOURCES test/testStaticTLSReserve.cxx + COMPONENT_NAME SimSetup + PUBLIC_LINK_LIBRARIES O2::SimSetup + LABELS simsetup) + o2_add_test_root_macro(UserDecay.C PUBLIC_LINK_LIBRARIES O2::SimSetup LABELS simsetup) diff --git a/Detectors/gconfig/include/SimSetup/StaticTLSReserve.h b/Detectors/gconfig/include/SimSetup/StaticTLSReserve.h new file mode 100644 index 0000000000000..5c6c8c98a1961 --- /dev/null +++ b/Detectors/gconfig/include/SimSetup/StaticTLSReserve.h @@ -0,0 +1,127 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef O2_SIMSETUP_STATICTLSRESERVE_H +#define O2_SIMSETUP_STATICTLSRESERVE_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace o2 +{ + +/// Static TLS reserve in bytes for a Geant4 built with the initial-exec TLS model and loaded by dlopen +constexpr unsigned long Geant4StaticTLSReserve = 32768; + +/// The GLIBC_TUNABLES value with a static TLS reserve of at least `reserve` bytes, or an empty string if +/// `tunables` already has one. As in glibc, the last reserve counts and numbers may be hex (0x) or octal (0). +/// Other tunables are kept; entries without a value are dropped, since glibc then ignores the whole string. +inline std::string withStaticTLSReserve(const char* tunables, unsigned long reserve = Geant4StaticTLSReserve) +{ + constexpr std::string_view name = "glibc.rtld.optional_static_tls"; + std::string kept; + bool haveReserve = false; + bool malformed = false; + unsigned long current = 0; + std::string_view rest = tunables ? tunables : ""; + while (!rest.empty()) { + const auto colon = rest.find(':'); + const std::string_view entry = rest.substr(0, colon); + rest = colon == std::string_view::npos ? std::string_view{} : rest.substr(colon + 1); + const auto equal = entry.find('='); + if (equal == std::string_view::npos || equal == 0) { + malformed = true; // also the truncated reserve as it arrives in GRID jobs + continue; + } + if (entry.substr(0, equal) == name) { + std::string_view value = entry.substr(equal + 1); + int base = 10; + if (value.size() > 2 && value[0] == '0' && (value[1] == 'x' || value[1] == 'X')) { + base = 16; + value.remove_prefix(2); + } else if (value.size() > 1 && value[0] == '0') { + base = 8; + value.remove_prefix(1); + } + unsigned long bytes = 0; + const auto [end, ec] = std::from_chars(value.data(), value.data() + value.size(), bytes, base); + if (!value.empty() && ec == std::errc() && end == value.data() + value.size()) { + haveReserve = true; + current = bytes; + } else { + haveReserve = false; // a broken last value leaves no reserve + } + continue; // our own entry replaces all of them + } + kept += (kept.empty() ? "" : ":") + std::string(entry); + } + if (!malformed && haveReserve && current >= reserve) { + return {}; + } + return std::string(name) + "=" + std::to_string(reserve) + (kept.empty() ? "" : ":" + kept); +} + +/// Sets the static TLS reserve in the environment, for the programs started from here. Returns true if it changed. +inline bool setStaticTLSReserve(unsigned long reserve = Geant4StaticTLSReserve) +{ + const auto tunables = withStaticTLSReserve(getenv("GLIBC_TUNABLES"), reserve); + if (tunables.empty()) { + return false; + } + setenv("GLIBC_TUNABLES", tunables.c_str(), 1); + return true; +} + +/// Restarts the program once with the static TLS reserve if it lacks one. glibc reads GLIBC_TUNABLES only at +/// program start, so the reserve cannot be set later. Call this first in main, before any thread is started. +inline void ensureStaticTLSReserve([[maybe_unused]] char* argv[], + [[maybe_unused]] unsigned long reserve = Geant4StaticTLSReserve) +{ +#ifdef __linux__ + constexpr const char* guard = "O2_STATIC_TLS_REEXEC"; + if (getenv(guard)) { + unsetenv(guard); // restarted already; do not loop if glibc ignores the tunable + return; + } + if (withStaticTLSReserve(getenv("GLIBC_TUNABLES"), reserve).empty()) { + return; + } + // the real path keeps the process name; /proc/self/exe would rename it to "exe" + char self[PATH_MAX]; + const auto length = readlink("/proc/self/exe", self, sizeof(self) - 1); + if (length <= 0) { + return; + } + self[length] = '\0'; + const char* base = strrchr(self, '/'); + if (strncmp(base ? base + 1 : self, "ld-", 3) == 0) { + return; // started through the dynamic loader, which a restart would not reproduce + } + setStaticTLSReserve(reserve); + setenv(guard, "1", 1); + fprintf(stderr, "Restarting %s with GLIBC_TUNABLES=%s for the static TLS of Geant4\n", self, getenv("GLIBC_TUNABLES")); + execv(self, argv); + const int error = errno; + unsetenv(guard); + fprintf(stderr, "Restart of %s failed: %s\n", self, strerror(error)); +#endif +} + +} // namespace o2 + +#endif diff --git a/Detectors/gconfig/test/testStaticTLSReserve.cxx b/Detectors/gconfig/test/testStaticTLSReserve.cxx new file mode 100644 index 0000000000000..9d19e7a111392 --- /dev/null +++ b/Detectors/gconfig/test/testStaticTLSReserve.cxx @@ -0,0 +1,70 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#define BOOST_TEST_MODULE Test SimSetup StaticTLSReserve +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK +#include +#include "SimSetup/StaticTLSReserve.h" + +using o2::withStaticTLSReserve; + +BOOST_AUTO_TEST_CASE(missing_or_broken_reserve_is_added) +{ + const std::string reserve = "glibc.rtld.optional_static_tls=32768"; + BOOST_CHECK_EQUAL(withStaticTLSReserve(nullptr, 32768), reserve); + BOOST_CHECK_EQUAL(withStaticTLSReserve("", 32768), reserve); + // the value as it arrives in GRID jobs + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls", 32768), reserve); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=", 32768), reserve); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=512", 32768), reserve); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=-1", 32768), reserve); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=32768x", 32768), reserve); +} + +BOOST_AUTO_TEST_CASE(sufficient_reserve_is_kept) +{ + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=32768", 32768), ""); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=1048576", 32768), ""); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.malloc.arena_max=4:glibc.rtld.optional_static_tls=65536", 32768), ""); + // glibc also takes hex and octal + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=0x100000", 32768), ""); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=0100000", 32768), ""); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=0x200", 32768), "glibc.rtld.optional_static_tls=32768"); +} + +BOOST_AUTO_TEST_CASE(malformed_entries_are_dropped) +{ + // glibc ignores the whole string if one entry has no value, so a good reserve next to it does not count + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=32768:glibc.malloc.arena_max", 32768), + "glibc.rtld.optional_static_tls=32768"); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.malloc.arena_max:glibc.malloc.tcache_count=0", 32768), + "glibc.rtld.optional_static_tls=32768:glibc.malloc.tcache_count=0"); +} + +BOOST_AUTO_TEST_CASE(other_tunables_are_kept) +{ + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.malloc.arena_max=4", 32768), + "glibc.rtld.optional_static_tls=32768:glibc.malloc.arena_max=4"); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls:glibc.malloc.arena_max=4", 32768), + "glibc.rtld.optional_static_tls=32768:glibc.malloc.arena_max=4"); + // a longer name is a different tunable + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tlsx=1", 32768), + "glibc.rtld.optional_static_tls=32768:glibc.rtld.optional_static_tlsx=1"); +} + +BOOST_AUTO_TEST_CASE(last_reserve_counts) +{ + // glibc applies the entries in order, so a later small reserve wins over an earlier large one + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=65536:glibc.rtld.optional_static_tls=512", 32768), + "glibc.rtld.optional_static_tls=32768"); + BOOST_CHECK_EQUAL(withStaticTLSReserve("glibc.rtld.optional_static_tls=512:glibc.rtld.optional_static_tls=65536", 32768), ""); +} diff --git a/run/O2SimDeviceRunner.cxx b/run/O2SimDeviceRunner.cxx index 11da3792d0ac5..96742d1a6976e 100644 --- a/run/O2SimDeviceRunner.cxx +++ b/run/O2SimDeviceRunner.cxx @@ -39,6 +39,7 @@ #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" #include "rapidjson/filereadstream.h" +#include namespace bpo = boost::program_options; std::vector gChildProcesses; // global vector of child pids @@ -267,6 +268,8 @@ bool waitForControlInput(int workerID) int main(int argc, char* argv[]) { + // Geant4 is loaded by dlopen and may need a static TLS reserve, which glibc only takes at program start + o2::ensureStaticTLSReserve(argv); struct sigaction act; memset(&act, 0, sizeof act); sigemptyset(&act.sa_mask); diff --git a/run/o2sim.cxx b/run/o2sim.cxx index 4a9cdbb4bdf90..c82df14b3ac45 100644 --- a/run/o2sim.cxx +++ b/run/o2sim.cxx @@ -13,9 +13,12 @@ #include #include #include +#include int main(int argc, char* argv[]) { + // Geant4 is loaded by dlopen and may need a static TLS reserve, which glibc only takes at program start + o2::ensureStaticTLSReserve(argv); TStopwatch timer; timer.Start(); auto& conf = o2::conf::SimConfig::Instance(); diff --git a/run/o2sim_evalmat.cxx b/run/o2sim_evalmat.cxx index 77ce4ace9ec84..dec95a1e87acf 100644 --- a/run/o2sim_evalmat.cxx +++ b/run/o2sim_evalmat.cxx @@ -14,9 +14,12 @@ #include #include #include +#include int main(int argc, char* argv[]) { + // Geant4 is loaded by dlopen and may need a static TLS reserve, which glibc only takes at program start + o2::ensureStaticTLSReserve(argv); TStopwatch timer; timer.Start(); diff --git a/run/o2sim_parallel.cxx b/run/o2sim_parallel.cxx index 08e3e24935d0c..2a44592df117a 100644 --- a/run/o2sim_parallel.cxx +++ b/run/o2sim_parallel.cxx @@ -48,6 +48,7 @@ #include "SimPublishChannelHelper.h" #include #include +#include std::string getServerLogName() { @@ -435,6 +436,8 @@ std::vector checkArgs(int argc, char* argv[]) // for parallel simulation int main(int argc, char* argv[]) { + // for the workers, which load Geant4 by dlopen; glibc takes the static TLS reserve only at program start + o2::setStaticTLSReserve(); LOG(info) << "This is o2-sim version " << o2::fullVersion() << " (" << o2::gitRevision() << ")"; LOG(info) << o2::getBuildInfo();