diff --git a/collector/CMakeLists.txt b/collector/CMakeLists.txt index adc11c1735..4b1ec9a2e1 100644 --- a/collector/CMakeLists.txt +++ b/collector/CMakeLists.txt @@ -49,8 +49,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/lib/CollectorVersion.h.in ${CMAKE_CUR set(FALCO_DIR ${PROJECT_SOURCE_DIR}/../falcosecurity-libs) -add_subdirectory(container-plugin) - add_subdirectory(${PROJECT_SOURCE_DIR}/proto) include_directories(${PROJECT_SOURCE_DIR}/lib) diff --git a/collector/Makefile b/collector/Makefile index 2288e1466f..d92738a389 100644 --- a/collector/Makefile +++ b/collector/Makefile @@ -9,9 +9,9 @@ COLLECTOR_BIN_DIR = $(CMAKE_DIR)/collector LIBSINSP_BIN_DIR = $(CMAKE_DIR)/collector/EXCLUDE_FROM_DEFAULT_BUILD/libsinsp SRC_MOUNT_DIR = /tmp/collector -HDRS := $(wildcard lib/*.h) $(wildcard container-plugin/*.h) $(shell find $(BASE_PATH)/falcosecurity-libs/userspace -name '*.h') +HDRS := $(wildcard lib/*.h) $(shell find $(BASE_PATH)/falcosecurity-libs/userspace -name '*.h') -SRCS := $(wildcard lib/*.cpp) $(wildcard container-plugin/*.cpp) collector.cpp +SRCS := $(wildcard lib/*.cpp) collector.cpp COLLECTOR_BUILD_DEPS := $(HDRS) $(SRCS) $(shell find $(BASE_PATH)/falcosecurity-libs -name '*.h' -o -name '*.cpp' -o -name '*.c') @@ -40,7 +40,6 @@ container/bin/collector: cmake-build/collector mkdir -p container/libs cp "$(COLLECTOR_BIN_DIR)/collector" container/bin/collector cp "$(COLLECTOR_BIN_DIR)/self-checks" container/bin/self-checks - cp "$(COLLECTOR_BIN_DIR)/collector-container-plugin.so" container/libs/collector-container-plugin.so .PHONY: collector collector: container/bin/collector txt-files diff --git a/collector/container-plugin/CMakeLists.txt b/collector/container-plugin/CMakeLists.txt deleted file mode 100644 index 2ade8f5f25..0000000000 --- a/collector/container-plugin/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -add_library(collector-container-plugin SHARED ContainerPlugin.cpp) - -set_target_properties(collector-container-plugin PROPERTIES - PREFIX "" - OUTPUT_NAME "collector-container-plugin" - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/collector" -) - -target_include_directories(collector-container-plugin PRIVATE - ${FALCO_DIR}/userspace - ${FALCO_DIR}/driver -) diff --git a/collector/container-plugin/ContainerID.h b/collector/container-plugin/ContainerID.h deleted file mode 100644 index 5afe569081..0000000000 --- a/collector/container-plugin/ContainerID.h +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -namespace collector::container_plugin { - -constexpr size_t kContainerIDLength = 64; -constexpr size_t kShortContainerIDLength = 12; - -inline std::optional ExtractContainerIDFromCgroup(std::string_view cgroup) { - const auto scope = cgroup.rfind(".scope"); - if (scope != std::string_view::npos) { - cgroup.remove_suffix(cgroup.size() - scope); - } - if (cgroup.size() < kContainerIDLength + 1) { - return {}; - } - const auto id_start = cgroup.size() - kContainerIDLength; - const char separator = cgroup[id_start - 1]; - if (separator != '/' && separator != '-' && separator != ':') { - return {}; - } - const std::string_view parent = cgroup.substr(0, id_start - 1); - constexpr std::string_view kConmonSuffix = "-conmon"; - if (parent.size() >= kConmonSuffix.size() && - parent.substr(parent.size() - kConmonSuffix.size()) == kConmonSuffix) { - return {}; - } - const std::string_view id = cgroup.substr(id_start); - if (!std::all_of(id.begin(), id.end(), [](char c) { return std::isxdigit(static_cast(c)); })) { - return {}; - } - return id.substr(0, kShortContainerIDLength); -} - -} // namespace collector::container_plugin diff --git a/collector/container-plugin/ContainerPlugin.cpp b/collector/container-plugin/ContainerPlugin.cpp deleted file mode 100644 index 0cbaa6c89a..0000000000 --- a/collector/container-plugin/ContainerPlugin.cpp +++ /dev/null @@ -1,260 +0,0 @@ -#include -#include -#include -#include -#include - -#include - -#include "ContainerID.h" - -namespace { - -constexpr std::string_view kPluginName = "collector-container"; -constexpr std::string_view kPluginVersion = "0.1.0"; -constexpr std::string_view kHostContainerID = "host"; - -struct plugin_state { - std::string last_error; - ss_plugin_table_t* threads = nullptr; - ss_plugin_table_field_t* cgroups = nullptr; - ss_plugin_table_field_t* cgroup_path = nullptr; - ss_plugin_table_field_t* container_id = nullptr; - std::string extracted_container_id; - const char* extracted_container_id_ptr = nullptr; -}; - -std::string ContainerIDFromCgroup(std::string_view cgroup) { - return std::string(collector::container_plugin::ExtractContainerIDFromCgroup(cgroup).value_or(std::string_view{})); -} - -struct cgroup_iteration_state { - plugin_state* plugin; - ss_plugin_table_reader_vtable_ext* reader; - ss_plugin_table_t* cgroup_table; - std::string container_id; -}; - -ss_plugin_bool FindContainerID(ss_plugin_table_iterator_state_t* data, ss_plugin_table_entry_t* entry) { - auto* state = reinterpret_cast(data); - ss_plugin_state_data value{}; - if (state->reader->read_entry_field(state->cgroup_table, entry, state->plugin->cgroup_path, &value) != SS_PLUGIN_SUCCESS) { - return 0; - } - if (value.str != nullptr) { - state->container_id = ContainerIDFromCgroup(value.str); - } - return 1; -} - -ss_plugin_rc CacheContainerID(plugin_state* state, - ss_plugin_table_entry_t* thread, - ss_plugin_table_reader_vtable_ext* reader, - ss_plugin_table_writer_vtable_ext* writer) { - ss_plugin_state_data cgroups{}; - if (reader->read_entry_field(state->threads, thread, state->cgroups, &cgroups) != SS_PLUGIN_SUCCESS || - cgroups.table == nullptr) { - state->last_error = "failed to read thread cgroups"; - return SS_PLUGIN_FAILURE; - } - - cgroup_iteration_state iteration{state, reader, cgroups.table, {}}; - if (!reader->iterate_entries(cgroups.table, FindContainerID, - reinterpret_cast(&iteration))) { - state->last_error = "failed to inspect thread cgroups"; - return SS_PLUGIN_FAILURE; - } - - ss_plugin_state_data value{}; - const std::string id = iteration.container_id.empty() ? std::string(kHostContainerID) : iteration.container_id; - value.str = id.c_str(); - if (writer->write_entry_field(state->threads, thread, state->container_id, &value) != SS_PLUGIN_SUCCESS) { - state->last_error = "failed to cache thread container ID"; - return SS_PLUGIN_FAILURE; - } - return SS_PLUGIN_SUCCESS; -} - -struct thread_iteration_state { - plugin_state* plugin; - ss_plugin_table_reader_vtable_ext* reader; - ss_plugin_table_writer_vtable_ext* writer; -}; - -ss_plugin_bool CacheInitialThread(ss_plugin_table_iterator_state_t* data, ss_plugin_table_entry_t* entry) { - auto* state = reinterpret_cast(data); - return CacheContainerID(state->plugin, entry, state->reader, state->writer) == SS_PLUGIN_SUCCESS; -} - -} // namespace - -extern "C" const char* plugin_get_required_api_version() { - return PLUGIN_API_VERSION_STR; -} - -extern "C" const char* plugin_get_version() { - return kPluginVersion.data(); -} - -extern "C" const char* plugin_get_name() { - return kPluginName.data(); -} - -extern "C" const char* plugin_get_description() { - return "Caches Collector container IDs in Falco thread state"; -} - -extern "C" const char* plugin_get_contact() { - return "https://github.com/stackrox/collector"; -} - -extern "C" const char* plugin_get_required_event_schema_version(ss_plugin_t*) { - return "4.1.0"; -} - -extern "C" ss_plugin_t* plugin_init(const ss_plugin_init_input* input, ss_plugin_rc* rc) { - auto* state = new plugin_state; - *rc = SS_PLUGIN_FAILURE; - if (input == nullptr || input->tables == nullptr || input->tables->fields_ext == nullptr || - input->tables->reader_ext == nullptr || input->tables->writer_ext == nullptr) { - state->last_error = "Falco table API is unavailable"; - return reinterpret_cast(state); - } - - state->threads = input->tables->get_table(input->owner, "threads", SS_PLUGIN_ST_INT64); - if (state->threads == nullptr) { - state->last_error = "failed to access Falco threads table"; - return reinterpret_cast(state); - } - state->cgroups = input->tables->fields_ext->get_table_field(state->threads, "cgroups", SS_PLUGIN_ST_TABLE); - state->container_id = input->tables->fields_ext->add_table_field(state->threads, "container_id", SS_PLUGIN_ST_STRING); - if (state->cgroups == nullptr || state->container_id == nullptr) { - state->last_error = "failed to access Falco thread cgroup or container ID fields"; - return reinterpret_cast(state); - } - - ss_plugin_table_entry_t* entry = input->tables->writer_ext->create_table_entry(state->threads); - ss_plugin_state_data cgroups{}; - if (entry == nullptr || input->tables->reader_ext->read_entry_field(state->threads, entry, state->cgroups, &cgroups) != SS_PLUGIN_SUCCESS || - cgroups.table == nullptr) { - if (entry != nullptr) { - input->tables->writer_ext->destroy_table_entry(state->threads, entry); - } - state->last_error = "failed to access Falco cgroup table"; - return reinterpret_cast(state); - } - state->cgroup_path = input->tables->fields_ext->get_table_field(cgroups.table, "second", SS_PLUGIN_ST_STRING); - input->tables->writer_ext->destroy_table_entry(state->threads, entry); - if (state->cgroup_path == nullptr) { - state->last_error = "failed to access Falco cgroup path field"; - return reinterpret_cast(state); - } - - *rc = SS_PLUGIN_SUCCESS; - return reinterpret_cast(state); -} - -extern "C" void plugin_destroy(ss_plugin_t* plugin) { - delete reinterpret_cast(plugin); -} - -extern "C" const char* plugin_get_last_error(ss_plugin_t* plugin) { - return reinterpret_cast(plugin)->last_error.c_str(); -} - -extern "C" const char* plugin_get_parse_event_sources() { - return "[\"syscall\"]"; -} - -extern "C" uint16_t* plugin_get_parse_event_types(uint32_t* count, ss_plugin_t*) { - static uint16_t event_types[] = { - PPME_SYSCALL_CLONE_20_X, - PPME_SYSCALL_FORK_20_X, - PPME_SYSCALL_VFORK_20_X, - PPME_SYSCALL_CLONE3_X, - PPME_SYSCALL_EXECVE_16_X, - PPME_SYSCALL_EXECVE_17_X, - PPME_SYSCALL_EXECVE_18_X, - PPME_SYSCALL_EXECVE_19_X, - PPME_SYSCALL_EXECVEAT_X, - PPME_SYSCALL_CHROOT_X, - }; - *count = sizeof(event_types) / sizeof(event_types[0]); - return event_types; -} - -extern "C" const char* plugin_get_fields() { - return R"([{"type":"string","name":"container.id","desc":"Cached container ID for the event thread"}])"; -} - -extern "C" const char* plugin_get_extract_event_sources() { - return "[\"syscall\"]"; -} - -extern "C" ss_plugin_rc plugin_extract_fields(ss_plugin_t* plugin, - const ss_plugin_event_input* event, - const ss_plugin_field_extract_input* input) { - auto* state = reinterpret_cast(plugin); - ss_plugin_state_data key{}; - key.s64 = static_cast(event->evt->tid); - ss_plugin_table_entry_t* thread = input->table_reader_ext->get_table_entry(state->threads, &key); - if (thread == nullptr) { - for (uint32_t i = 0; i < input->num_fields; ++i) { - input->fields[i].res_len = 0; - } - return SS_PLUGIN_SUCCESS; - } - - ss_plugin_state_data value{}; - const ss_plugin_rc read_rc = input->table_reader_ext->read_entry_field(state->threads, thread, state->container_id, &value); - input->table_reader_ext->release_table_entry(state->threads, thread); - if (read_rc != SS_PLUGIN_SUCCESS || value.str == nullptr) { - state->last_error = "failed to read cached thread container ID"; - return SS_PLUGIN_FAILURE; - } - - state->extracted_container_id = value.str; - state->extracted_container_id_ptr = state->extracted_container_id.c_str(); - for (uint32_t i = 0; i < input->num_fields; ++i) { - if (input->fields[i].field_id != 0) { - input->fields[i].res_len = 0; - continue; - } - input->fields[i].res.str = &state->extracted_container_id_ptr; - input->fields[i].res_len = 1; - } - return SS_PLUGIN_SUCCESS; -} - -extern "C" ss_plugin_rc plugin_parse_event(ss_plugin_t* plugin, - const ss_plugin_event_input* event, - const ss_plugin_event_parse_input* input) { - auto* state = reinterpret_cast(plugin); - ss_plugin_state_data key{}; - key.s64 = static_cast(event->evt->tid); - ss_plugin_table_entry_t* thread = input->table_reader_ext->get_table_entry(state->threads, &key); - if (thread == nullptr) { - return SS_PLUGIN_SUCCESS; - } - const ss_plugin_rc rc = CacheContainerID(state, thread, input->table_reader_ext, input->table_writer_ext); - input->table_reader_ext->release_table_entry(state->threads, thread); - return rc; -} - -extern "C" ss_plugin_rc plugin_capture_open(ss_plugin_t* plugin, const ss_plugin_capture_listen_input* input) { - auto* state = reinterpret_cast(plugin); - thread_iteration_state iteration{state, input->table_reader_ext, input->table_writer_ext}; - if (!input->table_reader_ext->iterate_entries( - state->threads, CacheInitialThread, - reinterpret_cast(&iteration))) { - if (state->last_error.empty()) { - state->last_error = "failed to cache initial thread container IDs"; - } - return SS_PLUGIN_FAILURE; - } - return SS_PLUGIN_SUCCESS; -} - -extern "C" ss_plugin_rc plugin_capture_close(ss_plugin_t*, const ss_plugin_capture_listen_input*) { - return SS_PLUGIN_SUCCESS; -} diff --git a/collector/container/Dockerfile b/collector/container/Dockerfile index e2c88c4e29..3cf7945a6e 100644 --- a/collector/container/Dockerfile +++ b/collector/container/Dockerfile @@ -39,7 +39,6 @@ COPY container/THIRD_PARTY_NOTICES/ /THIRD_PARTY_NOTICES/ COPY kernel-modules /kernel-modules COPY container/bin/collector /usr/local/bin/ COPY container/bin/self-checks /usr/local/bin/self-checks -COPY container/libs/collector-container-plugin.so /usr/local/lib/collector/collector-container-plugin.so COPY container/status-check.sh /usr/local/bin/status-check.sh EXPOSE 8080 9090 diff --git a/collector/container/dev.Dockerfile b/collector/container/dev.Dockerfile index 6fe4958c7a..148fd8bbd1 100644 --- a/collector/container/dev.Dockerfile +++ b/collector/container/dev.Dockerfile @@ -23,7 +23,6 @@ COPY container/THIRD_PARTY_NOTICES/ /THIRD_PARTY_NOTICES/ COPY kernel-modules /kernel-modules COPY container/bin/collector /usr/local/bin/ COPY container/bin/self-checks /usr/local/bin/self-checks -COPY container/libs/collector-container-plugin.so /usr/local/lib/collector/collector-container-plugin.so COPY container/status-check.sh /usr/local/bin/status-check.sh EXPOSE 8080 9090 diff --git a/collector/container/konflux.Dockerfile b/collector/container/konflux.Dockerfile index 583d6cb48e..02086c988b 100644 --- a/collector/container/konflux.Dockerfile +++ b/collector/container/konflux.Dockerfile @@ -135,7 +135,6 @@ COPY --from=package_installer /out/ / COPY --from=builder ${CMAKE_BUILD_DIR}/collector/collector /usr/local/bin/ COPY --from=builder ${CMAKE_BUILD_DIR}/collector/self-checks /usr/local/bin/ -COPY --from=builder ${CMAKE_BUILD_DIR}/collector/collector-container-plugin.so /usr/local/lib/collector/collector-container-plugin.so COPY LICENSE /licenses/LICENSE diff --git a/collector/lib/CollectorConfig.cpp b/collector/lib/CollectorConfig.cpp index 1f22f26e0f..294e25bb4d 100644 --- a/collector/lib/CollectorConfig.cpp +++ b/collector/lib/CollectorConfig.cpp @@ -82,7 +82,6 @@ PathEnvVar tls_client_cert_path("ROX_COLLECTOR_TLS_CLIENT_CERT"); PathEnvVar tls_client_key_path("ROX_COLLECTOR_TLS_CLIENT_KEY"); BoolEnvVar disable_process_arguments("ROX_COLLECTOR_NO_PROCESS_ARGUMENTS", false); -StringEnvVar container_plugin_path("ROX_COLLECTOR_CONTAINER_PLUGIN_PATH", "/usr/local/lib/collector/collector-container-plugin.so"); } // namespace constexpr bool CollectorConfig::kTurnOffScrape; @@ -98,7 +97,6 @@ CollectorConfig::CollectorConfig() { scrape_interval_ = kScrapeInterval; turn_off_scrape_ = kTurnOffScrape; collection_method_ = kCollectionMethod; - container_plugin_path_ = container_plugin_path.value(); } void CollectorConfig::InitCollectorConfig(CollectorArgs* args) { diff --git a/collector/lib/CollectorConfig.h b/collector/lib/CollectorConfig.h index 6172d5c4fa..2a5244c29e 100644 --- a/collector/lib/CollectorConfig.h +++ b/collector/lib/CollectorConfig.h @@ -161,7 +161,6 @@ class CollectorConfig { unsigned int GetSinspTotalBufferSize() const { return sinsp_total_buffer_size_; } unsigned int GetSinspThreadCacheSize() const { return sinsp_thread_cache_size_; } bool DisableProcessArguments() const { return disable_process_arguments_; } - const std::string& ContainerPluginPath() const { return container_plugin_path_; } static std::pair CheckConfiguration(const char* config, Json::Value* root); @@ -226,7 +225,6 @@ class CollectorConfig { std::optional grpc_server_; bool disable_process_arguments_ = false; - std::string container_plugin_path_ = "/usr/local/lib/collector/collector-container-plugin.so"; // One ring buffer will be initialized for this many CPUs unsigned int sinsp_cpu_per_buffer_ = 0; diff --git a/collector/lib/CollectorService.cpp b/collector/lib/CollectorService.cpp index 1d6243d591..4c7330d879 100644 --- a/collector/lib/CollectorService.cpp +++ b/collector/lib/CollectorService.cpp @@ -53,7 +53,7 @@ CollectorService::CollectorService(CollectorConfig& config, std::atomic(system_inspector_.GetInspector(), conn_tracker_, system_inspector_.GetUserspaceStats()); + auto network_signal_handler = std::make_unique(system_inspector_.GetInspector(), conn_tracker_, system_inspector_.GetUserspaceStats(), system_inspector_.GetContainerIDCache()); network_signal_handler->SetCollectConnectionStatus(config_.CollectConnectionStatus()); network_signal_handler->SetTrackSendRecv(config_.TrackingSendRecv()); system_inspector_.AddSignalHandler(std::move(network_signal_handler)); diff --git a/collector/lib/NetworkSignalHandler.cpp b/collector/lib/NetworkSignalHandler.cpp index c109470a7f..e3b39e3116 100644 --- a/collector/lib/NetworkSignalHandler.cpp +++ b/collector/lib/NetworkSignalHandler.cpp @@ -6,6 +6,7 @@ #include "EventMap.h" #include "Utility.h" +#include "system-inspector/ContainerIDCache.h" #include "system-inspector/EventExtractor.h" namespace collector { @@ -43,8 +44,8 @@ EventMap modifiers = { } // namespace -NetworkSignalHandler::NetworkSignalHandler(sinsp* inspector, std::shared_ptr conn_tracker, system_inspector::Stats* stats) - : event_extractor_(std::make_unique()), conn_tracker_(std::move(conn_tracker)), stats_(stats), collect_connection_status_(true), track_send_recv_(false) { +NetworkSignalHandler::NetworkSignalHandler(sinsp* inspector, std::shared_ptr conn_tracker, system_inspector::Stats* stats, system_inspector::ContainerIDCache* container_id_cache) + : event_extractor_(std::make_unique()), conn_tracker_(std::move(conn_tracker)), stats_(stats), container_id_cache_(container_id_cache), collect_connection_status_(true), track_send_recv_(false) { event_extractor_->Init(inspector); } @@ -152,7 +153,7 @@ std::optional NetworkSignalHandler::GetConnection(sinsp_evt* evt) { const Endpoint* local = is_server ? &server : &client; const Endpoint* remote = is_server ? &client : &server; - auto container_id = GetContainerID(evt); + auto container_id = container_id_cache_->Get(*evt->get_thread_info()); if (container_id.empty()) { return std::nullopt; } diff --git a/collector/lib/NetworkSignalHandler.h b/collector/lib/NetworkSignalHandler.h index 36e10d3d60..e4e18cae50 100644 --- a/collector/lib/NetworkSignalHandler.h +++ b/collector/lib/NetworkSignalHandler.h @@ -7,6 +7,10 @@ #include "SignalHandler.h" #include "system-inspector/SystemInspector.h" +namespace collector::system_inspector { +class ContainerIDCache; +} + // forward declarations class sinsp; class sinsp_evt; @@ -18,7 +22,7 @@ class EventExtractor; class NetworkSignalHandler final : public SignalHandler { public: - explicit NetworkSignalHandler(sinsp* inspector, std::shared_ptr conn_tracker, system_inspector::Stats* stats); + explicit NetworkSignalHandler(sinsp* inspector, std::shared_ptr conn_tracker, system_inspector::Stats* stats, system_inspector::ContainerIDCache* container_id_cache); ~NetworkSignalHandler() override; std::string GetName() override { return "NetworkSignalHandler"; } @@ -35,6 +39,7 @@ class NetworkSignalHandler final : public SignalHandler { std::unique_ptr event_extractor_; std::shared_ptr conn_tracker_; system_inspector::Stats* stats_; + system_inspector::ContainerIDCache* container_id_cache_; bool collect_connection_status_; bool track_send_recv_; diff --git a/collector/lib/Process.cpp b/collector/lib/Process.cpp index 738cb3058f..7a168ed5ab 100644 --- a/collector/lib/Process.cpp +++ b/collector/lib/Process.cpp @@ -6,6 +6,7 @@ #include "CollectorStats.h" #include "Utility.h" +#include "system-inspector/ContainerIDCache.h" #include "system-inspector/Service.h" namespace collector { @@ -33,7 +34,7 @@ std::string Process::container_id() const { WaitForProcessInfo(); if (system_inspector_ && system_inspector_threadinfo_) { - auto id = GetContainerID(*system_inspector_->GetInspector(), *system_inspector_threadinfo_); + auto id = system_inspector_->GetContainerIDCache()->Get(*system_inspector_threadinfo_); if (!id.empty()) { return id; } diff --git a/collector/lib/ProcessSignalFormatter.cpp b/collector/lib/ProcessSignalFormatter.cpp index 8295b6fc7f..3cfe98436a 100644 --- a/collector/lib/ProcessSignalFormatter.cpp +++ b/collector/lib/ProcessSignalFormatter.cpp @@ -10,6 +10,7 @@ #include "EventMap.h" #include "Logging.h" #include "Utility.h" +#include "system-inspector/ContainerIDCache.h" #include "system-inspector/EventExtractor.h" namespace collector { @@ -24,6 +25,8 @@ using TimeUtil = google::protobuf::util::TimeUtil; namespace { +system_inspector::ContainerIDCache empty_container_id_cache; + enum ProcessSignalType { EXECVE, UNKNOWN_PROCESS_TYPE @@ -58,10 +61,12 @@ std::string extract_proc_args(sinsp_threadinfo* tinfo) { ProcessSignalFormatter::ProcessSignalFormatter( sinsp* inspector, - const CollectorConfig& config) : event_names_(EventNames::GetInstance()), - inspector_(inspector), - event_extractor_(std::make_unique()), - config_(config) { + const CollectorConfig& config, + system_inspector::ContainerIDCache* container_id_cache) : event_names_(EventNames::GetInstance()), + inspector_(inspector), + event_extractor_(std::make_unique()), + container_id_cache_(container_id_cache == nullptr ? &empty_container_id_cache : container_id_cache), + config_(config) { event_extractor_->Init(inspector); } @@ -176,7 +181,7 @@ ProcessSignal* ProcessSignalFormatter::CreateProcessSignal(sinsp_evt* event) { signal->set_allocated_time(timestamp); // set container_id - auto container_id = GetContainerID(event); + auto container_id = container_id_cache_->Get(*event->get_thread_info()); if (!container_id.empty()) { signal->set_container_id(container_id); } @@ -242,7 +247,7 @@ ProcessSignal* ProcessSignalFormatter::CreateProcessSignal(sinsp_threadinfo* tin signal->set_allocated_time(timestamp); // set container_id - signal->set_container_id(GetContainerID(*inspector_, *tinfo)); + signal->set_container_id(container_id_cache_->Get(*tinfo)); // set process lineage std::vector lineage; @@ -266,7 +271,7 @@ std::string ProcessSignalFormatter::ProcessDetails(sinsp_evt* event) { std::stringstream ss; const std::string* path = event_extractor_->get_exepath(event); const std::string* name = event_extractor_->get_comm(event); - auto container_id = GetContainerID(event); + auto container_id = container_id_cache_->Get(*event->get_thread_info()); const char* args = event_extractor_->get_proc_args(event); const int64_t* pid = event_extractor_->get_pid(event); @@ -348,7 +353,7 @@ void ProcessSignalFormatter::GetProcessLineage(sinsp_threadinfo* tinfo, // all platforms. // if (pt->m_vpid == 0) { - if (GetContainerID(*inspector_, *pt).empty()) { + if (container_id_cache_->Get(*pt).empty()) { return false; } } else if (pt->m_pid == pt->m_vpid) { diff --git a/collector/lib/ProcessSignalFormatter.h b/collector/lib/ProcessSignalFormatter.h index ceeeb98dea..34dd61da1d 100644 --- a/collector/lib/ProcessSignalFormatter.h +++ b/collector/lib/ProcessSignalFormatter.h @@ -19,6 +19,7 @@ class sinsp_threadinfo; namespace collector { namespace system_inspector { class EventExtractor; +class ContainerIDCache; } } // namespace collector @@ -26,7 +27,7 @@ namespace collector { class ProcessSignalFormatter : public ProtoSignalFormatter { public: - ProcessSignalFormatter(sinsp* inspector, const CollectorConfig& config); + ProcessSignalFormatter(sinsp* inspector, const CollectorConfig& config, system_inspector::ContainerIDCache* container_id_cache = nullptr); ~ProcessSignalFormatter(); using Signal = v1::Signal; @@ -56,6 +57,7 @@ class ProcessSignalFormatter : public ProtoSignalFormatter event_extractor_; + system_inspector::ContainerIDCache* container_id_cache_; const CollectorConfig& config_; }; diff --git a/collector/lib/ProcessSignalHandler.h b/collector/lib/ProcessSignalHandler.h index b6c2797a17..d2e10aa45b 100644 --- a/collector/lib/ProcessSignalHandler.h +++ b/collector/lib/ProcessSignalHandler.h @@ -21,11 +21,12 @@ class ProcessSignalHandler : public SignalHandler { public: ProcessSignalHandler( sinsp* inspector, - ISignalServiceClient* client, - system_inspector::Stats* stats, - const CollectorConfig& config) + ISignalServiceClient* client, + system_inspector::Stats* stats, + const CollectorConfig& config, + system_inspector::ContainerIDCache* container_id_cache) : client_(client), - formatter_(inspector, config), + formatter_(inspector, config, container_id_cache), stats_(stats), config_(config) {} diff --git a/collector/lib/Utility.cpp b/collector/lib/Utility.cpp index efa1d7b7a5..58ea1aa2f4 100644 --- a/collector/lib/Utility.cpp +++ b/collector/lib/Utility.cpp @@ -25,7 +25,9 @@ extern "C" { #include "Logging.h" #include "Utility.h" -#include "../container-plugin/ContainerID.h" +#include +#include + namespace collector { @@ -59,34 +61,6 @@ const char* SignalName(int signum) { } } -std::string GetContainerID(sinsp& inspector, const sinsp_threadinfo& tinfo) { - const auto& fields = inspector.m_thread_manager->dynamic_fields()->fields(); - const auto field = fields.find("container_id"); - if (field == fields.end()) { - return {}; - } - auto accessor = field->second.new_accessor(); - std::string container_id; - // libsinsp's dynamic-field read API is not const-qualified. - const_cast(tinfo).get_dynamic_field(accessor, container_id); - return container_id == "host" ? std::string{} : container_id; -} - -std::string GetContainerID(sinsp_evt* event) { - if (!event) { - return {}; - } - sinsp_threadinfo* tinfo = event->get_thread_info(); - if (!tinfo) { - return {}; - } - sinsp* inspector = event->get_inspector(); - if (!inspector) { - return {}; - } - return GetContainerID(*inspector, *tinfo); -} - std::ostream& operator<<(std::ostream& os, const sinsp_threadinfo* t) { if (t) { os << "Name: " << t->m_comm << ", PID: " << t->m_pid << ", Args: " << t->m_exe; @@ -206,7 +180,30 @@ void TryUnlink(const char* path) { } std::optional ExtractContainerIDFromCgroup(std::string_view cgroup) { - return container_plugin::ExtractContainerIDFromCgroup(cgroup); + constexpr size_t kContainerIDLength = 64; + constexpr size_t kShortContainerIDLength = 12; + const auto scope = cgroup.rfind(".scope"); + if (scope != std::string_view::npos) { + cgroup.remove_suffix(cgroup.size() - scope); + } + if (cgroup.size() < kContainerIDLength + 1) { + return {}; + } + const auto id_start = cgroup.size() - kContainerIDLength; + const char separator = cgroup[id_start - 1]; + if (separator != '/' && separator != '-' && separator != ':') { + return {}; + } + const std::string_view parent = cgroup.substr(0, id_start - 1); + constexpr std::string_view kConmonSuffix = "-conmon"; + if (parent.size() >= kConmonSuffix.size() && parent.substr(parent.size() - kConmonSuffix.size()) == kConmonSuffix) { + return {}; + } + const std::string_view id = cgroup.substr(id_start); + if (!std::all_of(id.begin(), id.end(), [](char c) { return std::isxdigit(static_cast(c)); })) { + return {}; + } + return id.substr(0, kShortContainerIDLength); } std::optional SanitizedUTF8(std::string_view str) { diff --git a/collector/lib/Utility.h b/collector/lib/Utility.h index 193a08e19a..fe6437a2b7 100644 --- a/collector/lib/Utility.h +++ b/collector/lib/Utility.h @@ -67,14 +67,6 @@ std::string Str(Args&&... args) { std::ostream& operator<<(std::ostream& os, const sinsp_threadinfo* t); -// Return the cached container ID from a threadinfo. -// Returns an empty string for host processes. -std::string GetContainerID(sinsp& inspector, const sinsp_threadinfo& tinfo); - -// Extract container ID from an event's thread info cgroups. -// Returns an empty string if no container ID found. -std::string GetContainerID(sinsp_evt* event); - // UUIDStr returns UUID in string format. const char* UUIDStr(); diff --git a/collector/lib/system-inspector/ContainerIDCache.cpp b/collector/lib/system-inspector/ContainerIDCache.cpp new file mode 100644 index 0000000000..707be0fcbe --- /dev/null +++ b/collector/lib/system-inspector/ContainerIDCache.cpp @@ -0,0 +1,76 @@ +#include "ContainerIDCache.h" + +#include + +#include +#include +#include + +#include "Utility.h" + +namespace collector::system_inspector { + +void ContainerIDCache::Cache(const sinsp_threadinfo& tinfo) { + std::string container_id; + for (const auto& cgroup : tinfo.cgroups()) { + if (const auto id = ExtractContainerIDFromCgroup(cgroup.second)) { + container_id = *id; + break; + } + } + + std::lock_guard lock(mutex_); + entries_[tinfo.m_tid] = {tinfo.m_clone_ts, std::move(container_id)}; +} + +void ContainerIDCache::Initialise(sinsp& inspector) { + inspector.m_thread_manager->get_threads()->loop([this](sinsp_threadinfo& tinfo) { + Cache(tinfo); + return true; + }); +} + +void ContainerIDCache::Prune(sinsp& inspector, uint64_t now_us) { + if (now_us - last_prune_us_ < 60'000'000) { + return; + } + last_prune_us_ = now_us; + + std::unordered_set live_tids; + inspector.m_thread_manager->get_threads()->loop([&live_tids](sinsp_threadinfo& tinfo) { + live_tids.insert(tinfo.m_tid); + return true; + }); + + std::lock_guard lock(mutex_); + for (auto it = entries_.begin(); it != entries_.end();) { + if (live_tids.count(it->first) == 0) { + it = entries_.erase(it); + } else { + ++it; + } + } +} + +std::string ContainerIDCache::Get(const sinsp_threadinfo& tinfo) const { + std::lock_guard lock(mutex_); + const auto entry = entries_.find(tinfo.m_tid); + if (entry == entries_.end() || entry->second.clone_ts != tinfo.m_clone_ts) { + return {}; + } + return entry->second.container_id; +} + +void ContainerIDCache::on_clone(sinsp_evt*, sinsp_threadinfo* newtinfo, int64_t) { + if (newtinfo != nullptr) { + Cache(*newtinfo); + } +} + +void ContainerIDCache::on_execve(sinsp_evt* evt) { + if (evt != nullptr && evt->get_thread_info() != nullptr) { + Cache(*evt->get_thread_info()); + } +} + +} // namespace collector::system_inspector diff --git a/collector/lib/system-inspector/ContainerIDCache.h b/collector/lib/system-inspector/ContainerIDCache.h new file mode 100644 index 0000000000..41f3b55682 --- /dev/null +++ b/collector/lib/system-inspector/ContainerIDCache.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include + +#include + +class sinsp; +class sinsp_threadinfo; + +namespace collector::system_inspector { + +class ContainerIDCache final : public sinsp_observer { + public: + void Initialise(sinsp& inspector); + void Prune(sinsp& inspector, uint64_t now_us); + std::string Get(const sinsp_threadinfo& tinfo) const; + + void on_read(sinsp_evt*, int64_t, int64_t, sinsp_fdinfo*, const char*, uint32_t, uint32_t) override {} + void on_write(sinsp_evt*, int64_t, int64_t, sinsp_fdinfo*, const char*, uint32_t, uint32_t) override {} + void on_sendfile(sinsp_evt*, int64_t, uint32_t) override {} + void on_connect(sinsp_evt*, uint8_t*) override {} + void on_accept(sinsp_evt*, int64_t, uint8_t*, sinsp_fdinfo*) override {} + void on_file_open(sinsp_evt*, const std::string&, uint32_t) override {} + void on_error(sinsp_evt*) override {} + void on_erase_fd(erase_fd_params*) override {} + void on_socket_shutdown(sinsp_evt*) override {} + void on_execve(sinsp_evt* evt) override; + void on_clone(sinsp_evt*, sinsp_threadinfo* newtinfo, int64_t) override; + void on_bind(sinsp_evt*) override {} + void on_socket_status_changed(sinsp_evt*) override {} + + private: + struct Entry { + uint64_t clone_ts; + std::string container_id; + }; + + void Cache(const sinsp_threadinfo& tinfo); + + mutable std::mutex mutex_; + std::unordered_map entries_; + uint64_t last_prune_us_ = 0; +}; + +} // namespace collector::system_inspector diff --git a/collector/lib/system-inspector/ContainerIDFilterCheck.cpp b/collector/lib/system-inspector/ContainerIDFilterCheck.cpp new file mode 100644 index 0000000000..8fbb15dd3f --- /dev/null +++ b/collector/lib/system-inspector/ContainerIDFilterCheck.cpp @@ -0,0 +1,65 @@ +#include "ContainerIDFilterCheck.h" + +#include +#include + +#include + +#include "ContainerIDCache.h" + +namespace collector::system_inspector { + +namespace { + +constexpr char kHostContainerID[] = "host"; + +const filtercheck_field_info kFields[] = { + {PT_CHARBUF, EPF_NONE, PF_NA, "container.id", "Cached container ID for the event thread", ""}, +}; + +} // namespace + +ContainerIDFilterCheck::ContainerIDFilterCheck(const ContainerIDCache* container_id_cache) + : container_id_cache_(container_id_cache) { + static const filter_check_info info = { + "container", + "Container fields", + "Container fields", + sizeof(kFields) / sizeof(kFields[0]), + kFields, + filter_check_info::FL_NONE, + }; + m_info = &info; +} + +std::unique_ptr ContainerIDFilterCheck::allocate_new() { + return std::make_unique(container_id_cache_); +} + +uint8_t* ContainerIDFilterCheck::extract_single(sinsp_evt* event, uint32_t* len, bool) { + *len = 0; + if (event == nullptr || m_field_id != 0) { + return nullptr; + } + + sinsp_threadinfo* tinfo = event->get_thread_info(); + if (tinfo == nullptr) { + return nullptr; + } + + result_ = container_id_cache_->Get(*tinfo); + if (!result_.empty()) { + *len = result_.size(); + return reinterpret_cast(result_.data()); + } + + // Match the bundled container filter: an empty ID identifies the host only + // when the process is outside a PID namespace. + if (!tinfo->is_in_pid_namespace()) { + *len = sizeof(kHostContainerID) - 1; + return reinterpret_cast(const_cast(kHostContainerID)); + } + return reinterpret_cast(result_.data()); +} + +} // namespace collector::system_inspector diff --git a/collector/lib/system-inspector/ContainerIDFilterCheck.h b/collector/lib/system-inspector/ContainerIDFilterCheck.h new file mode 100644 index 0000000000..8c5fa1c740 --- /dev/null +++ b/collector/lib/system-inspector/ContainerIDFilterCheck.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include + +#include + +namespace collector::system_inspector { + +class ContainerIDCache; + +class ContainerIDFilterCheck final : public sinsp_filter_check { + public: + explicit ContainerIDFilterCheck(const ContainerIDCache* container_id_cache); + + std::unique_ptr allocate_new() override; + + protected: + uint8_t* extract_single(sinsp_evt* event, uint32_t* len, bool sanitize_strings) override; + + private: + const ContainerIDCache* container_id_cache_; + std::string result_; +}; + +} // namespace collector::system_inspector diff --git a/collector/lib/system-inspector/Service.cpp b/collector/lib/system-inspector/Service.cpp index 6150d5ac75..4eb308dd09 100644 --- a/collector/lib/system-inspector/Service.cpp +++ b/collector/lib/system-inspector/Service.cpp @@ -14,8 +14,10 @@ #include #include "CollectionMethod.h" +#include "ContainerIDCache.h" #include "CollectorException.h" #include "CollectorStats.h" +#include "ContainerIDFilterCheck.h" #include "EventExtractor.h" #include "EventNames.h" #include "HostInfo.h" @@ -34,10 +36,13 @@ namespace collector::system_inspector { namespace { } // namespace -Service::~Service() = default; +Service::~Service() { + inspector_->set_observer(nullptr); +} Service::Service(const CollectorConfig& config) : inspector_(std::make_unique(true)), + container_id_cache_(std::make_unique()), default_formatter_(std::make_unique( inspector_.get(), DEFAULT_OUTPUT_STR, @@ -51,11 +56,7 @@ Service::Service(const CollectorConfig& config) inspector_->disable_log_timestamps(); inspector_->set_log_callback(logging::InspectorLogCallback); - container_plugin_ = inspector_->register_plugin(config.ContainerPluginPath()); - std::string plugin_error; - if (!container_plugin_->init("{}", plugin_error)) { - CLOG(FATAL) << "Failed to initialise container plugin: " << plugin_error; - } + inspector_->set_observer(container_id_cache_.get()); inspector_->set_import_users(config.ImportUsers()); inspector_->set_thread_timeout_s(30); @@ -82,9 +83,10 @@ Service::Service(const CollectorConfig& config) signal_client_ = std::make_unique(); } AddSignalHandler(std::make_unique(inspector_.get(), - signal_client_.get(), - &userspace_stats_, - config)); + signal_client_.get(), + &userspace_stats_, + config, + container_id_cache_.get())); if (signal_handlers_.size() == 2) { // self-check handlers do not count towards this check, because they @@ -99,12 +101,13 @@ bool Service::InitKernel(const CollectorConfig& config) { CLOG(ERROR) << "Failed to setup " << config.GetCollectionMethod() << " driver."; return false; } + container_id_cache_->Initialise(*inspector_); sinsp_filter_check_list filter_list; filter_list.add_filter_check(inspector_->new_generic_filtercheck()); - filter_list.add_filter_check(sinsp_plugin::new_filtercheck(container_plugin_)); + filter_list.add_filter_check(std::make_unique(container_id_cache_.get())); auto filter_factory = std::make_shared(inspector_.get(), filter_list); - sinsp_filter_compiler filter_compiler(filter_factory, "container.id != host"); + sinsp_filter_compiler filter_compiler(filter_factory, "proc.pid != val(proc.vpid) or container.id != host"); inspector_->set_filter(filter_compiler.compile(), "container.id != host"); return true; @@ -119,6 +122,7 @@ sinsp_evt* Service::GetNext() { if (res != SCAP_SUCCESS || event == nullptr) { return nullptr; } + container_id_cache_->Prune(*inspector_, NowMicros()); #ifdef TRACE_SINSP_EVENTS // Do not allow to change sinsp events tracing at runtime, as the output @@ -277,7 +281,7 @@ bool Service::SendExistingProcesses(SignalHandler* handler) { } return threads->loop([&](sinsp_threadinfo& tinfo) { - if (!GetContainerID(*inspector_, tinfo).empty() && tinfo.is_main_thread()) { + if (!container_id_cache_->Get(tinfo).empty() && tinfo.is_main_thread()) { auto result = handler->HandleExistingProcess(&tinfo); if (result == SignalHandler::ERROR || result == SignalHandler::NEEDS_REFRESH) { CLOG(WARNING) << "Failed to write existing process signal: " << &tinfo; diff --git a/collector/lib/system-inspector/Service.h b/collector/lib/system-inspector/Service.h index 915b330820..fd05d6ae1e 100644 --- a/collector/lib/system-inspector/Service.h +++ b/collector/lib/system-inspector/Service.h @@ -17,11 +17,11 @@ class sinsp; class sinsp_evt; class sinsp_evt_formatter; -class sinsp_plugin; class sinsp_threadinfo; namespace collector::system_inspector { +class ContainerIDCache; class Service : public SystemInspector { public: Service(const Service&) = delete; @@ -44,6 +44,7 @@ class Service : public SystemInspector { void GetProcessInformation(uint64_t pid, ProcessInfoCallbackRef callback); sinsp* GetInspector() { return inspector_.get(); } + ContainerIDCache* GetContainerIDCache() { return container_id_cache_.get(); } Stats* GetUserspaceStats() { return &userspace_stats_; } void AddSignalHandler(std::unique_ptr signal_handler); @@ -69,7 +70,7 @@ class Service : public SystemInspector { mutable std::mutex libsinsp_mutex_; std::unique_ptr inspector_; - std::shared_ptr container_plugin_; + std::unique_ptr container_id_cache_; std::unique_ptr default_formatter_; std::unique_ptr signal_client_; std::vector signal_handlers_; diff --git a/collector/test/CMakeLists.txt b/collector/test/CMakeLists.txt index 956530b947..d4e9b12bda 100644 --- a/collector/test/CMakeLists.txt +++ b/collector/test/CMakeLists.txt @@ -20,8 +20,6 @@ foreach(test_file ${TEST_SRC_FILES}) endif() add_test(${test_name} ${test_name}) - add_dependencies(${test_name} collector-container-plugin) - set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT "ROX_COLLECTOR_CONTAINER_PLUGIN_PATH=${CMAKE_BINARY_DIR}/collector/collector-container-plugin.so") if(USE_VALGRIND) # TODO: This test has a deadlock when running on valgrind. Further investigation needed. diff --git a/collector/test/ProcessSignalFormatterTest.cpp b/collector/test/ProcessSignalFormatterTest.cpp index 233c3bfed3..a754bc64c2 100644 --- a/collector/test/ProcessSignalFormatterTest.cpp +++ b/collector/test/ProcessSignalFormatterTest.cpp @@ -5,6 +5,7 @@ #include "CollectorStats.h" #include "ProcessSignalFormatter.h" +#include "system-inspector/ContainerIDCache.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -27,7 +28,7 @@ TEST(ProcessSignalFormatterTest, NoProcessTest) { CollectorStats& collector_stats = CollectorStats::GetOrCreate(); CollectorConfig config; - ProcessSignalFormatter processSignalFormatter(inspector, config); + ProcessSignalFormatter processSignalFormatter(inspector, config, nullptr); sinsp_threadinfo* tinfo = NULL; std::vector lineage; @@ -638,8 +639,9 @@ TEST(ProcessSignalFormatterTest, Rox3377ProcessLineageWithNoVPidTest) { TEST(ProcessSignalFormatterTest, ProcessArguments) { std::unique_ptr inspector(new sinsp()); MockCollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, &container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; @@ -666,9 +668,10 @@ TEST(ProcessSignalFormatterTest, ProcessArguments) { TEST(ProcessSignalFormatterTest, NoProcessArguments) { std::unique_ptr inspector(new sinsp()); MockCollectorConfig config; + system_inspector::ContainerIDCache container_id_cache; config.SetDisableProcessArguments(true); - ProcessSignalFormatter processSignalFormatter(inspector.get(), config); + ProcessSignalFormatter processSignalFormatter(inspector.get(), config, &container_id_cache); auto tinfo = inspector->get_threadinfo_factory().create(); tinfo->m_pid = 3; diff --git a/collector/test/SystemInspectorServiceTest.cpp b/collector/test/SystemInspectorServiceTest.cpp index ea5a1dac85..e68e788426 100644 --- a/collector/test/SystemInspectorServiceTest.cpp +++ b/collector/test/SystemInspectorServiceTest.cpp @@ -1,39 +1,32 @@ -#include - #include -#include #include #include "Utility.h" #include "gtest/gtest.h" +#include "system-inspector/ContainerIDCache.h" +#include "system-inspector/ContainerIDFilterCheck.h" #include "system-inspector/Service.h" namespace collector::system_inspector { TEST(SystemInspectorServiceTest, FilterEvent) { std::unique_ptr inspector(new sinsp()); - const char* plugin_path = std::getenv("ROX_COLLECTOR_CONTAINER_PLUGIN_PATH"); - ASSERT_NE(plugin_path, nullptr); - auto plugin = inspector->register_plugin(plugin_path); - std::string error; - ASSERT_TRUE(plugin->init("{}", error)) << error; + ContainerIDCache container_id_cache; sinsp_filter_check_list filter_list; filter_list.add_filter_check(inspector->new_generic_filtercheck()); - filter_list.add_filter_check(sinsp_plugin::new_filtercheck(plugin)); + + filter_list.add_filter_check(std::make_unique(&container_id_cache)); auto filter_factory = std::make_shared(inspector.get(), filter_list); sinsp_filter_compiler filter_compiler(filter_factory, "container.id != host"); - ASSERT_NO_THROW(filter_compiler.compile()); - - const auto& fields = inspector->m_thread_manager->dynamic_fields()->fields(); - const auto container_id_field = fields.find("container_id"); - ASSERT_NE(container_id_field, fields.end()); - const auto container_id_accessor = container_id_field->second.new_accessor(); + auto filter = filter_compiler.compile(); const auto& factory = inspector->get_threadinfo_factory(); auto regular_process = factory.create(); + regular_process->m_tid = 1; regular_process->m_exepath = "/bin/busybox"; regular_process->m_comm = "sleep"; - regular_process->set_dynamic_field(container_id_accessor, std::string("aaaaaaaaaaaa")); + regular_process->set_cgroups({"cpu:/docker/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}); + container_id_cache.on_clone(nullptr, regular_process.get(), -1); auto runc_process = factory.create(); runc_process->m_exepath = "runc"; @@ -43,6 +36,12 @@ TEST(SystemInspectorServiceTest, FilterEvent) { host_process->m_exepath = "/usr/bin/bash"; host_process->m_comm = "bash"; + auto pid_namespace_process = factory.create(); + pid_namespace_process->m_tid = 42; + pid_namespace_process->m_vtid = 1; + + sinsp_evt event(inspector.get()); + struct test_t { const sinsp_threadinfo* tinfo; bool expected; @@ -59,9 +58,12 @@ TEST(SystemInspectorServiceTest, FilterEvent) { << "Failed for: " << t.name; } - EXPECT_EQ(GetContainerID(*inspector, *regular_process), "aaaaaaaaaaaa"); - regular_process->set_dynamic_field(container_id_accessor, std::string("host")); - EXPECT_TRUE(GetContainerID(*inspector, *regular_process).empty()); + event.set_tinfo(regular_process.get()); + EXPECT_FALSE(filter->run(&event)); + event.set_tinfo(host_process.get()); + EXPECT_FALSE(filter->run(&event)); + event.set_tinfo(pid_namespace_process.get()); + EXPECT_TRUE(filter->run(&event)); } } // namespace collector::system_inspector diff --git a/falcosecurity-libs b/falcosecurity-libs index e61430ac73..fdb27236a5 160000 --- a/falcosecurity-libs +++ b/falcosecurity-libs @@ -1 +1 @@ -Subproject commit e61430ac73d78137344eda9a21e663cd897b4f59 +Subproject commit fdb27236a5b589fb2f658cb3453b6e52388eb22c diff --git a/integration-tests/Makefile b/integration-tests/Makefile index 6a8e992bce..a15bd1d021 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -105,7 +105,7 @@ ci-benchmarks: benchmark .PHONY: docker-clean docker-clean: - docker rm -fv container-stats benchmark collector grpc-server 2>/dev/null || true + docker rm -fv container-stats benchmark collector grpc-server cpu-profile benchmark-connections 2>/dev/null || true .PHONY: clean clean: docker-clean diff --git a/integration-tests/container/berserker/Dockerfile b/integration-tests/container/berserker/Dockerfile index ea60cbc931..6562f01f70 100644 --- a/integration-tests/container/berserker/Dockerfile +++ b/integration-tests/container/berserker/Dockerfile @@ -1,6 +1,7 @@ -FROM quay.io/rhacs-eng/qa:berserker-1.0-59-g87ad0d870e +FROM quay.io/stackrox-io/berserker:network-1.0-85-g1b7ab034aa COPY workloads/ /etc/berserker/ +COPY network/workloads/ /etc/berserker/ ENV PATH="${PATH}:/usr/local/bin" diff --git a/integration-tests/container/berserker/network/workloads/network-client.toml b/integration-tests/container/berserker/network/workloads/network-client.toml new file mode 100644 index 0000000000..022474db5a --- /dev/null +++ b/integration-tests/container/berserker/network/workloads/network-client.toml @@ -0,0 +1,16 @@ +restart_interval = 10 +workers = 1 +per_core = false + +[workload] +type = "network" +server = false +address = "223.42.0.1" +target_port = 1337 +nconnections = 100 +arrival_rate = 0.1 +departure_rate = 0.1 +connections_static = 100 +connections_dyn_max = 1000 +preempt = true +conns_per_addr = 1 diff --git a/integration-tests/container/berserker/network/workloads/network-server.toml b/integration-tests/container/berserker/network/workloads/network-server.toml new file mode 100644 index 0000000000..d241aa3449 --- /dev/null +++ b/integration-tests/container/berserker/network/workloads/network-server.toml @@ -0,0 +1,14 @@ +restart_interval = 10 +workers = 1 +per_core = false + +[workload] +type = "network" +server = true +address = "223.42.0.1" +target_port = 1337 +nconnections = 100 +connections_static = 100 +connections_dyn_max = 100 +preempt = false +conns_per_addr = 1 diff --git a/integration-tests/container/perf/scripts/run-tool.sh b/integration-tests/container/perf/scripts/run-tool.sh index 1e1910aef1..19a6d30e92 100755 --- a/integration-tests/container/perf/scripts/run-tool.sh +++ b/integration-tests/container/perf/scripts/run-tool.sh @@ -4,11 +4,18 @@ set -eo pipefail TOOL_PID=0 +function postrun() { + if [[ -n "${PERF_OUTPUT_FILE:-}" && -e "${PERF_OUTPUT_FILE}" ]]; then + chmod go+r "${PERF_OUTPUT_FILE}" + fi +} + function exit_trap() { if [[ $TOOL_PID -ne 0 ]]; then - kill -INT $TOOL_PID - wait $TOOL_PID + kill -INT $TOOL_PID || true + wait $TOOL_PID || true fi + postrun exit 0 } @@ -22,6 +29,7 @@ function preinit() { function run_tool() { TOOL="$1" shift + umask 022 # make sure to background the task so we can set up the pid # and handle signals from docker eval "$TOOL $* 2>&1 &" @@ -33,6 +41,8 @@ function run_tool() { # When docker tries to stop this container, this is interrupted # and the exit_trap is run, which handles cleaning up the tool process. wait $! + TOOL_PID=0 + postrun } trap exit_trap EXIT diff --git a/integration-tests/pkg/config/config.go b/integration-tests/pkg/config/config.go index 47febb7cd4..11e763ff2d 100644 --- a/integration-tests/pkg/config/config.go +++ b/integration-tests/pkg/config/config.go @@ -91,6 +91,10 @@ type Benchmarks struct { BccCommand string BpftraceCommand string PerfCommand string + CPUProfile bool + CPUProfileFreq string + Workloads []string + EnableScrape bool SkipInit bool } @@ -161,6 +165,10 @@ func BenchmarksInfo() *Benchmarks { BccCommand: ReadEnvVar(envBccCommand), BpftraceCommand: ReadEnvVar(envBpftraceCommand), PerfCommand: ReadEnvVar(envPerfCommand), + CPUProfile: ReadBoolEnvVar(envCPUProfile), + CPUProfileFreq: ReadEnvVarWithDefault(envCPUProfileFreq, "199"), + Workloads: strings.Split(ReadEnvVarWithDefault(envBenchmarkWorkloads, "processes,endpoints"), ","), + EnableScrape: ReadBoolEnvVar(envBenchmarkEnableScrape), SkipInit: ReadBoolEnvVar(envSkipHeadersInit), } } diff --git a/integration-tests/pkg/config/env.go b/integration-tests/pkg/config/env.go index d2ae4eadda..46634115fc 100644 --- a/integration-tests/pkg/config/env.go +++ b/integration-tests/pkg/config/env.go @@ -22,10 +22,14 @@ const ( envQATag = "COLLECTOR_QA_TAG" - envPerfCommand = "COLLECTOR_PERF_COMMAND" - envBpftraceCommand = "COLLECTOR_BPFTRACE_COMMAND" - envBccCommand = "COLLECTOR_BCC_COMMAND" - envSkipHeadersInit = "COLLECTOR_SKIP_HEADERS_INIT" + envPerfCommand = "COLLECTOR_PERF_COMMAND" + envCPUProfile = "COLLECTOR_CPU_PROFILE" + envCPUProfileFreq = "COLLECTOR_CPU_PROFILE_FREQUENCY" + envBenchmarkWorkloads = "COLLECTOR_BENCHMARK_WORKLOADS" + envBenchmarkEnableScrape = "COLLECTOR_BENCHMARK_ENABLE_SCRAPE" + envBpftraceCommand = "COLLECTOR_BPFTRACE_COMMAND" + envBccCommand = "COLLECTOR_BCC_COMMAND" + envSkipHeadersInit = "COLLECTOR_SKIP_HEADERS_INIT" envStopTimeout = "STOP_TIMEOUT" ) diff --git a/integration-tests/pkg/executor/executor.go b/integration-tests/pkg/executor/executor.go index c772120ed0..6c84d2bb4c 100644 --- a/integration-tests/pkg/executor/executor.go +++ b/integration-tests/pkg/executor/executor.go @@ -57,6 +57,8 @@ type Executor interface { StopContainer(name string) (string, error) StartContainer(config config.ContainerStartConfig) (string, error) GetContainerHealthCheck(containerID string) (string, error) + GetContainerPID(containerID string) (int, error) + CopyFileFromContainer(containerID string, sourcePath string, destinationPath string) error GetContainerStats(ctx context.Context, containerID string) (*ContainerStat, error) GetContainerIP(containerID string) (string, error) GetContainerLogs(containerID string) (ContainerLogs, error) diff --git a/integration-tests/pkg/executor/executor_cri.go b/integration-tests/pkg/executor/executor_cri.go index 18f81ff848..334e98b2d8 100644 --- a/integration-tests/pkg/executor/executor_cri.go +++ b/integration-tests/pkg/executor/executor_cri.go @@ -313,6 +313,14 @@ func (c *criExecutor) GetContainerStats(ctx context.Context, containerID string) }, nil } +func (c *criExecutor) GetContainerPID(string) (int, error) { + return 0, fmt.Errorf("CPU profiling is supported only with the Docker runtime") +} + +func (c *criExecutor) CopyFileFromContainer(string, string, string) error { + return fmt.Errorf("CPU profiling is supported only with the Docker runtime") +} + func (c *criExecutor) GetContainerIP(name string) (string, error) { container, err := c.getContainer(name) if err != nil { diff --git a/integration-tests/pkg/executor/executor_docker_api.go b/integration-tests/pkg/executor/executor_docker_api.go index 3e2736bbcc..404e190c1a 100644 --- a/integration-tests/pkg/executor/executor_docker_api.go +++ b/integration-tests/pkg/executor/executor_docker_api.go @@ -1,11 +1,14 @@ package executor import ( + "archive/tar" "bytes" "context" "encoding/json" "fmt" "io" + "os" + "path/filepath" "strings" "time" @@ -92,6 +95,50 @@ func (d *dockerAPIExecutor) GetContainerHealthCheck(containerID string) (string, return strings.Join(inspectResp.Config.Healthcheck.Test, " "), nil } +func (d *dockerAPIExecutor) GetContainerPID(containerID string) (int, error) { + inspectResp, err := d.inspectContainer(containerID) + if err != nil { + return 0, fmt.Errorf("error inspecting container: %w", err) + } + return inspectResp.State.Pid, nil +} + +func (d *dockerAPIExecutor) CopyFileFromContainer(containerID string, sourcePath string, destinationPath string) error { + archive, _, err := d.client.CopyFromContainer(context.Background(), containerID, sourcePath) + if err != nil { + return fmt.Errorf("copying %s from container: %w", sourcePath, err) + } + defer archive.Close() + + tarReader := tar.NewReader(archive) + for { + header, err := tarReader.Next() + if err == io.EOF { + return fmt.Errorf("copied archive does not contain %s", sourcePath) + } + if err != nil { + return fmt.Errorf("reading copied archive: %w", err) + } + if header.Typeflag != tar.TypeReg { + continue + } + + if err := os.MkdirAll(filepath.Dir(destinationPath), 0755); err != nil { + return fmt.Errorf("creating destination directory: %w", err) + } + file, err := os.OpenFile(destinationPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, header.FileInfo().Mode()) + if err != nil { + return fmt.Errorf("creating destination file: %w", err) + } + _, copyErr := io.Copy(file, tarReader) + closeErr := file.Close() + if copyErr != nil { + return fmt.Errorf("writing destination file: %w", copyErr) + } + return closeErr + } +} + func (d *dockerAPIExecutor) GetContainerStats(ctx context.Context, containerID string) ( *ContainerStat, error) { diff --git a/integration-tests/pkg/executor/executor_k8s.go b/integration-tests/pkg/executor/executor_k8s.go index e6c7b38da4..8183d94ad6 100644 --- a/integration-tests/pkg/executor/executor_k8s.go +++ b/integration-tests/pkg/executor/executor_k8s.go @@ -25,6 +25,14 @@ type K8sExecutor struct { clientset *kubernetes.Clientset } +func (e *K8sExecutor) GetContainerPID(string) (int, error) { + return 0, fmt.Errorf("CPU profiling is supported only with the Docker runtime") +} + +func (e *K8sExecutor) CopyFileFromContainer(string, string, string) error { + return fmt.Errorf("CPU profiling is supported only with the Docker runtime") +} + func NewK8sExecutor() (*K8sExecutor, error) { log.Info("Creating k8s configuration") config, err := rest.InClusterConfig() diff --git a/integration-tests/suites/base.go b/integration-tests/suites/base.go index ff03a50dd6..0cca619e3c 100644 --- a/integration-tests/suites/base.go +++ b/integration-tests/suites/base.go @@ -7,7 +7,6 @@ import ( "fmt" "os" "os/exec" - "path/filepath" "strconv" "strings" "time" @@ -19,6 +18,7 @@ import ( "google.golang.org/grpc/status" "github.com/stackrox/collector/integration-tests/pkg/collector" + "github.com/stackrox/collector/integration-tests/pkg/common" "github.com/stackrox/collector/integration-tests/pkg/config" "github.com/stackrox/collector/integration-tests/pkg/executor" "github.com/stackrox/collector/integration-tests/pkg/log" @@ -298,11 +298,8 @@ func (s *IntegrationTestSuiteBase) WritePerfResults() { LoadStopTs: s.stop.Format("2006-01-02 15:04:05"), } + f, err := common.PrepareLog(s.T().Name(), "perf.json") perfJson, _ := json.Marshal(perf) - perfFilename := filepath.Join(config.LogPath(), "perf.json") - - log.Info("Writing %s\n", perfFilename) - f, err := os.OpenFile(perfFilename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) s.Require().NoError(err) defer f.Close() diff --git a/integration-tests/suites/benchmark.go b/integration-tests/suites/benchmark.go index 4e877cf8a4..108d13bc2c 100644 --- a/integration-tests/suites/benchmark.go +++ b/integration-tests/suites/benchmark.go @@ -3,6 +3,9 @@ package suites import ( "fmt" "os" + "path/filepath" + "strconv" + "strings" "time" "github.com/google/shlex" @@ -10,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stackrox/collector/integration-tests/pkg/collector" "github.com/stackrox/collector/integration-tests/pkg/common" "github.com/stackrox/collector/integration-tests/pkg/config" ) @@ -25,8 +29,9 @@ type BenchmarkCollectorTestSuite struct { type BenchmarkTestSuiteBase struct { IntegrationTestSuiteBase - perfContainers []string - loadContainers []string + perfContainers []string + loadContainers []string + profileContainerID string } func (b *BenchmarkTestSuiteBase) StartPerfTools() { @@ -65,6 +70,52 @@ func (b *BenchmarkTestSuiteBase) StartPerfTools() { } } +func (b *BenchmarkTestSuiteBase) StartCPUProfile() { + benchmarkOptions := config.BenchmarksInfo() + if !benchmarkOptions.CPUProfile { + return + } + + resultDir, err := filepath.Abs(filepath.Join(config.LogPath(), strings.SplitN(b.T().Name(), "/", 2)[0])) + b.Require().NoError(err) + b.Require().NoError(os.MkdirAll(resultDir, os.ModePerm)) + b.Require().NoError(b.Executor().CopyFileFromContainer( + b.Collector().ContainerID(), + "/usr/local/bin/collector", + filepath.Join(resultDir, "rootfs/usr/local/bin/collector"))) + collectorPID, err := b.Executor().GetContainerPID(b.Collector().ContainerID()) + b.Require().NoError(err) + containerID, err := b.Executor().StartContainer(config.ContainerStartConfig{ + Name: "cpu-profile", + Image: config.Images().QaImageByKey("performance-perf"), + Privileged: true, + PidMode: "host", + Mounts: map[string]string{"/results": resultDir}, + Env: map[string]string{ + "PERF_FREQUENCY": benchmarkOptions.CPUProfileFreq, + "PERF_OUTPUT_FILE": "/results/perf.data", + }, + Command: []string{"record", "--buildid-all", "-e", "cpu-clock", "-F", benchmarkOptions.CPUProfileFreq, "-g", "--call-graph", "dwarf", "-p", strconv.Itoa(collectorPID), "-o", "/results/perf.data", "--", "sleep", "70"}, + }) + b.Require().NoError(err) + b.profileContainerID = containerID + b.perfContainers = append(b.perfContainers, containerID) + +} + +func (b *BenchmarkTestSuiteBase) StartCPUProfileCapture() { + b.StartCPUProfile() +} + +func (b *BenchmarkTestSuiteBase) StopCPUProfileCapture() { + if b.profileContainerID == "" { + return + } + finished, err := b.waitForContainerToExit("cpu-profile", b.profileContainerID, 100*time.Millisecond, 5*time.Minute) + b.Require().NoError(err) + b.Require().True(finished, "CPU profiler did not finish") +} + func (b *BenchmarkTestSuiteBase) StartPerfContainer(name string, image string, args string) { argsList, err := shlex.Split(args) require.NoError(b.T(), err) @@ -132,6 +183,10 @@ func (b *BenchmarkTestSuiteBase) StopPerfTools() { require.NoError(b.T(), err) fmt.Println(log) + if container == b.profileContainerID { + _, err = b.Executor().CaptureLogs(strings.SplitN(b.T().Name(), "/", 2)[0], "cpu-profile") + require.NoError(b.T(), err) + } } b.removeContainers(b.perfContainers...) @@ -139,12 +194,24 @@ func (b *BenchmarkTestSuiteBase) StopPerfTools() { } func (s *BenchmarkCollectorTestSuite) SetupSuite() { - s.RegisterCleanup("perf", "bcc", "bpftrace", "init", - "benchmark-processes", "benchmark-endpoints") + s.RegisterCleanup("perf", "cpu-profile", "bcc", "bpftrace", "init", + "benchmark-processes", "benchmark-endpoints", "benchmark-connections") s.StartPerfTools() - s.StartCollector(false, nil) + var collectorOptions *collector.StartupOptions + if config.BenchmarksInfo().EnableScrape { + collectorOptions = &collector.StartupOptions{ + Config: map[string]any{ + "turnOffScrape": false, + "scrapeInterval": 1, + }, + Env: map[string]string{ + "ROX_PROCESSES_LISTENING_ON_PORT": "true", + }, + } + } + s.StartCollector(false, collectorOptions) } func (s *BenchmarkTestSuiteBase) SpinBerserker(workload string) (string, error) { @@ -168,14 +235,47 @@ func (s *BenchmarkTestSuiteBase) SpinBerserker(workload string) (string, error) return containerID, nil } -func (s *BenchmarkTestSuiteBase) RunCollectorBenchmark() { - procContainerID, err := s.SpinBerserker("processes") - s.Require().NoError(err) +func (s *BenchmarkTestSuiteBase) SpinNetworkBerserker() (string, error) { + benchmarkImage := config.Images().QaImageByKey("performance-berserker") + if err := s.Executor().PullImage(benchmarkImage); err != nil { + return "", err + } - endpointsContainerID, err := s.SpinBerserker("endpoints") - s.Require().NoError(err) + containerID, err := s.Executor().StartContainer(config.ContainerStartConfig{ + Name: "benchmark-connections", + Image: benchmarkImage, + Privileged: true, + Entrypoint: []string{"/scripts/init.sh"}, + Env: map[string]string{ + "BERSERKER__DURATION": "60", + "IP_BASE": "223.42.0.1/16", + }, + }) + if err != nil { + return "", err + } + s.loadContainers = append(s.loadContainers, containerID) + return containerID, nil +} +func (s *BenchmarkTestSuiteBase) RunCollectorBenchmark() { s.start = time.Now().UTC() + s.StartCPUProfileCapture() + + benchmarkContainers := make([]string, 0, len(config.BenchmarksInfo().Workloads)) + var networkContainerID string + for _, workload := range config.BenchmarksInfo().Workloads { + if workload == "connections" { + containerID, err := s.SpinNetworkBerserker() + s.Require().NoError(err) + benchmarkContainers = append(benchmarkContainers, containerID) + networkContainerID = common.ContainerShortID(containerID) + continue + } + containerID, err := s.SpinBerserker(workload) + s.Require().NoError(err) + benchmarkContainers = append(benchmarkContainers, containerID) + } // The assumption is that the benchmark is short, and to get better // resolution into when relevant metrics start and stop, tick more @@ -183,14 +283,16 @@ func (s *BenchmarkTestSuiteBase) RunCollectorBenchmark() { waitTick := 1 * time.Second // Container name here is used only for reporting - _, err = s.waitForContainerToExit("berserker", procContainerID, waitTick, 0) - s.Require().NoError(err) - - _, err = s.waitForContainerToExit("berserker", endpointsContainerID, waitTick, 0) - - s.Require().NoError(err) + for _, containerID := range benchmarkContainers { + _, err := s.waitForContainerToExit("berserker", containerID, waitTick, 0) + s.Require().NoError(err) + } + if networkContainerID != "" { + s.Require().NotEmpty(s.Sensor().Connections(networkContainerID), "network workload produced no Collector connection signals") + } s.stop = time.Now().UTC() + s.StopCPUProfileCapture() } func (s *BenchmarkCollectorTestSuite) TestBenchmarkCollector() { diff --git a/perf/run-benchmarks.sh b/perf/run-benchmarks.sh new file mode 100755 index 0000000000..c09bf434ba --- /dev/null +++ b/perf/run-benchmarks.sh @@ -0,0 +1,217 @@ +#!/usr/bin/env bash +# Run process and connection benchmarks against explicit Collector images, then +# preserve profiles, workload results, and run context for later comparison. +set -euo pipefail + +PERF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CURRENT_WORKTREE="$(git -C "${PERF_DIR}/.." rev-parse --show-toplevel)" +RESULT_DIR="${CURRENT_WORKTREE}/integration-tests/container-logs/core-bpf/TestBenchmarkCollector" +INTEGRATION_TEST_LOG="${CURRENT_WORKTREE}/integration-tests/integration-test.log" +SAMPLER_PID= +COLLECTOR_LOG_LEVEL="${COLLECTOR_LOG_LEVEL:-info}" + +if [[ $# -ne 1 || ! $1 =~ ^[[:alnum:]_.-]+$ || $1 == "." || $1 == ".." ]]; then + printf 'Usage: %s RUN_NAME\n' "$0" >&2 + exit 1 +fi + +OUTPUT_DIR="${PERF_DIR}/$1" +mkdir -p "$OUTPUT_DIR" + +declare -a WORKLOADS=(processes connections) + +# Set these to the immutable Quay image references you want to compare. +declare -A IMAGES=( + [3.25.0]="${COLLECTOR_IMAGE_3_25_0:-quay.io/stackrox-io/collector:3.25.7-3-g284a332178}" + [master]="${COLLECTOR_IMAGE_MASTER:?Set COLLECTOR_IMAGE_MASTER to the Quay master image}" + [current]="${COLLECTOR_IMAGE_CURRENT:?Set COLLECTOR_IMAGE_CURRENT to the Quay current-branch image}" +) + +stop_sampler() { + if [[ -n "$SAMPLER_PID" ]]; then + kill "$SAMPLER_PID" 2>/dev/null || true + wait "$SAMPLER_PID" 2>/dev/null || true + SAMPLER_PID= + fi +} + +trap stop_sampler EXIT INT TERM + +capture_command() { + local output=$1 + shift + + { + printf '$' + printf ' %q' "$@" + printf '\n' + "$@" + } >> "$output" 2>&1 || true +} + +capture_metadata() { + local destination=$1 + local version=$2 + local workload=$3 + local image=$4 + local metadata="${destination}/metadata.txt" + + { + printf 'version=%s\n' "$version" + printf 'workload=%s\n' "$workload" + printf 'image=%s\n' "$image" + printf 'capture_started_utc=%s\n' "$(date --utc --iso-8601=seconds)" + printf 'working_tree=%s\n' "$CURRENT_WORKTREE" + printf 'perf_frequency=%s\n' "${COLLECTOR_CPU_PROFILE_FREQUENCY:-199}" + } > "$metadata" + + capture_command "$metadata" uname -a + capture_command "$metadata" lscpu + capture_command "$metadata" perf version + capture_command "$metadata" docker version + capture_command "$metadata" docker info + capture_command "$metadata" git -C "$CURRENT_WORKTREE" rev-parse HEAD + capture_command "$metadata" git -C "$CURRENT_WORKTREE" status --short --branch + capture_command "$metadata" git -C "$CURRENT_WORKTREE" submodule status + + cp /proc/cpuinfo "${destination}/cpuinfo.txt" 2>/dev/null || true + cp /proc/meminfo "${destination}/meminfo.txt" 2>/dev/null || true + cp /proc/cmdline "${destination}/kernel-cmdline.txt" 2>/dev/null || true + cp /proc/sys/kernel/perf_event_paranoid "${destination}/perf_event_paranoid.txt" 2>/dev/null || true + cp /proc/sys/kernel/perf_event_max_sample_rate "${destination}/perf_event_max_sample_rate.txt" 2>/dev/null || true + + docker image inspect "$image" > "${destination}/image-inspect.json" 2> "${destination}/image-inspect.err" || true +} + +sample_run() { + local destination=$1 + local sequence=0 + + while true; do + local timestamp + timestamp=$(date --utc +%s.%N) + + curl --silent --show-error --max-time 2 http://localhost:9090/metrics \ + > "${destination}/prometheus/${sequence}-${timestamp}.prom" 2>/dev/null || \ + rm -f "${destination}/prometheus/${sequence}-${timestamp}.prom" + + while IFS= read -r stats; do + printf '%s\t%s\n' "$timestamp" "$stats" + done < <(docker stats --no-stream --format '{{json .}}' \ + collector cpu-profile benchmark-processes benchmark-connections 2>/dev/null || true) \ + >> "${destination}/docker-stats.jsonl" + + sequence=$((sequence + 1)) + sleep 1 + done +} + +save_artefacts() { + local version=$1 + local workload=$2 + local run_source=$3 + local data_destination="${OUTPUT_DIR}/${version}-${workload}.data" + local root_destination="${OUTPUT_DIR}/root-${version}-${workload}" + + [[ ! -e "$data_destination" ]] || { printf 'Refusing to overwrite %s\n' "$data_destination" >&2; return 1; } + [[ ! -e "$root_destination" ]] || { printf 'Refusing to overwrite %s\n' "$root_destination" >&2; return 1; } + [[ -r "${RESULT_DIR}/perf.data" ]] || { printf 'Missing readable profile at %s\n' "${RESULT_DIR}/perf.data" >&2; return 1; } + [[ -d "${RESULT_DIR}/rootfs" ]] || { printf 'Missing Collector rootfs at %s\n' "${RESULT_DIR}/rootfs" >&2; return 1; } + + cp "${RESULT_DIR}/perf.data" "$data_destination" + cp -a "${RESULT_DIR}/rootfs" "$root_destination" + cp -a "${RESULT_DIR}/." "${run_source}/results/" + cp "$INTEGRATION_TEST_LOG" "${run_source}/integration-test.log" 2>/dev/null || true + perf buildid-list -i "$data_destination" > "${run_source}/perf-buildids.txt" 2>&1 || true + perf evlist -i "$data_destination" > "${run_source}/perf-events.txt" 2>&1 || true + printf 'capture_finished_utc=%s\n' "$(date --utc --iso-8601=seconds)" >> "${run_source}/metadata.txt" + perf buildid-cache --add "${root_destination}/usr/local/bin/collector" +} + +run_version() { + local version=$1 + local image=$2 + + for workload in "${WORKLOADS[@]}"; do + local run_destination="${OUTPUT_DIR}/${version}-${workload}-artefacts" + [[ ! -e "$run_destination" ]] || { printf 'Refusing to overwrite %s\n' "$run_destination" >&2; return 1; } + mkdir -p "${run_destination}/prometheus" "${run_destination}/results" + capture_metadata "$run_destination" "$version" "$workload" "$image" + + : > "$INTEGRATION_TEST_LOG" + sample_run "$run_destination" & + SAMPLER_PID=$! + + set +e + COLLECTOR_LOG_LEVEL="$COLLECTOR_LOG_LEVEL" \ + COLLECTOR_IMAGE="$image" \ + COLLECTOR_BENCHMARK_WORKLOADS="$workload" \ + COLLECTOR_CPU_PROFILE=true \ + COLLECTOR_CPU_PROFILE_FREQUENCY="${COLLECTOR_CPU_PROFILE_FREQUENCY:-199}" \ + make -C "${CURRENT_WORKTREE}/integration-tests" TestBenchmarkCollector + local test_status=$? + set -e + + stop_sampler + if [[ $test_status -ne 0 ]]; then + cp "$INTEGRATION_TEST_LOG" "${run_destination}/integration-test.log" 2>/dev/null || true + printf 'test_exit_status=%d\n' "$test_status" >> "${run_destination}/metadata.txt" + return "$test_status" + fi + + printf 'test_exit_status=0\n' >> "${run_destination}/metadata.txt" + save_artefacts "$version" "$workload" "$run_destination" + done +} + +fold_profile() { + local version=$1 + local workload=$2 + local profile="${OUTPUT_DIR}/${version}-${workload}.data" + local folded="${OUTPUT_DIR}/${version}-${workload}.folded" + local flamegraph="${OUTPUT_DIR}/${version}-${workload}.svg" + + perf script -i "$profile" | stackcollapse-perf.pl > "$folded" + flamegraph.pl \ + --title "Collector ${version}: ${workload}" \ + --countname samples \ + --width 2400 \ + --minwidth 0.5 \ + "$folded" > "$flamegraph" +} + +diff_flamegraph() { + local version=$1 + local workload=$2 + local baseline="${OUTPUT_DIR}/3.25.0-${workload}.folded" + local comparison="${OUTPUT_DIR}/${version}-${workload}.folded" + local output="${OUTPUT_DIR}/3.25.0-vs-${version}-${workload}.svg" + + difffolded.pl -n "$baseline" "$comparison" \ + | flamegraph.pl \ + --title "Collector ${workload}: 3.25.0 vs ${version}" \ + --subtitle "Normalised CPU samples: red = more CPU in ${version}; blue = less" \ + --countname samples \ + --width 2400 \ + --minwidth 0.5 \ + > "$output" +} + +generate_graphs() { + command -v stackcollapse-perf.pl >/dev/null || { printf 'Missing stackcollapse-perf.pl in PATH\n' >&2; return 1; } + command -v flamegraph.pl >/dev/null || { printf 'Missing flamegraph.pl in PATH\n' >&2; return 1; } + command -v difffolded.pl >/dev/null || { printf 'Missing difffolded.pl in PATH\n' >&2; return 1; } + + for workload in "${WORKLOADS[@]}"; do + fold_profile "3.25.0" "$workload" + fold_profile "master" "$workload" + fold_profile "current" "$workload" + diff_flamegraph "master" "$workload" + diff_flamegraph "current" "$workload" + done +} + +run_version "3.25.0" "${IMAGES[3.25.0]}" +run_version "master" "${IMAGES[master]}" +run_version "current" "${IMAGES[current]}" +generate_graphs