From c25b42e6c88104dfc9da3dba5f126ef891aff06f Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Thu, 19 Nov 2026 13:04:31 +0800 Subject: [PATCH 1/2] fix: export a series again when it is used after del del removed the key from the key index but left its full name in the per-metric lookup table. The next inc or set with the same label values took the cached name and never re-added the key, so the value was written to the dict but left out of the exposition. Reset the lookup table in del, as upstream does since knyar/nginx-lua-prometheus@9eeaf9a. --- prometheus.lua | 5 +++++ prometheus_test.lua | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/prometheus.lua b/prometheus.lua index 5da4441..83bbaa5 100644 --- a/prometheus.lua +++ b/prometheus.lua @@ -559,6 +559,11 @@ local function del(self, label_values) if err then self._log_error("Error deleting key: ".. k .. ": " .. err) end + + -- Clean up the full metric name lookup table as well, so that the series + -- is added back to the key index when it is used again. + self.lookup = {} + self.lookup_size = 0 end -- Set the value of a gauge metric. diff --git a/prometheus_test.lua b/prometheus_test.lua index 853b6a9..c783383 100644 --- a/prometheus_test.lua +++ b/prometheus_test.lua @@ -581,6 +581,26 @@ function TestPrometheus:testGaugeDel() luaunit.assertEquals(self.dict:get('gauge2{f2="f2value",f1="f1value"}'), nil) luaunit.assertEquals(self.dict:get("nginx_metric_errors_total"), 1) end +function TestPrometheus:testGaugeSetAfterDel() + self.gauge2:set(1, {"a", "x"}) + self.gauge2:del({"a", "x"}) + self.gauge2:set(2, {"a", "x"}) + + local out = table.concat(self.p:metric_data()) + luaunit.assertStrContains(out, 'gauge2{f2="a",f1="x"} 2\n') + luaunit.assertEquals(self.dict:get("nginx_metric_errors_total"), 0) +end +function TestPrometheus:testCounterSetAfterDel() + self.counter2:inc(1, {"a", "x"}) + self.p._counter:sync() + self.counter2:del({"a", "x"}) + self.counter2:inc(3, {"a", "x"}) + self.p._counter:sync() + + local out = table.concat(self.p:metric_data()) + luaunit.assertStrContains(out, 'metric2{f2="a",f1="x"} 3\n') + luaunit.assertEquals(self.dict:get("nginx_metric_errors_total"), 0) +end function TestPrometheus:testCounterDel() self.counter1:inc(1) self.p._counter:sync() From ece8e6d402e6daf2de6ff1f59ee5f6d3f714bb6f Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Thu, 19 Nov 2026 13:59:26 +0800 Subject: [PATCH 2/2] fix: reset the lookup table when another worker deletes a series del and reset on one worker left the series' full name cached in the lookup tables of the other workers. Using the same labels there again took the cached name without adding the key back to the index, so the series was never exported again. The key index now takes a callback that a sync calls for every key it finds removed, and the callback resets the lookup table of that metric. Metrics with an exptime are skipped: they add the series on every lookup. Every worker also syncs the key index every sync_interval, so the reset does not wait for a scrape or a new series. Ports the rest of knyar/nginx-lua-prometheus@9eeaf9a3. --- README.md | 6 +-- prometheus.lua | 45 ++++++++++++++-- prometheus_keys.lua | 10 +++- prometheus_test.lua | 123 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a887689..6a58c43 100644 --- a/README.md +++ b/README.md @@ -103,9 +103,9 @@ section of nginx configuration. metric names on output. * `error_metric_name` (string): Can be used to change the default name of error metric (see [Built-in metrics](#built-in-metrics) for details). - * `sync_interval` (number): sets per-worker counter sync interval in seconds. - This sets the boundary on eventual consistency of counter metrics. Defaults - to 1. + * `sync_interval` (number): sets the sync interval for per-worker counters and + key index (in seconds). This sets the boundary on eventual consistency of + counter metric increments, and metric resets/deletions. Defaults to 1. * `lookup_max_size` (number): maximum size of a per-metric lookup table maintained by each worker to cache full metric names. Defaults to 1000. If you have metrics with extremely high cardinality and lots of available diff --git a/prometheus.lua b/prometheus.lua index 83bbaa5..e9f8c0f 100644 --- a/prometheus.lua +++ b/prometheus.lua @@ -387,8 +387,7 @@ local function lookup_or_create(self, label_values) end if self.lookup_size >= self.lookup_max_size then - self.lookup_size = 0 - self.lookup = {} + self:reset_lookup() end local t = self.lookup @@ -562,8 +561,7 @@ local function del(self, label_values) -- Clean up the full metric name lookup table as well, so that the series -- is added back to the key index when it is used again. - self.lookup = {} - self.lookup_size = 0 + self:reset_lookup() end -- Set the value of a gauge metric. @@ -708,6 +706,12 @@ local function reset(self) end -- Clean up the full metric name lookup table as well. + self:reset_lookup() +end + +-- Reset the metric name lookup table for a given metric. +local function reset_lookup(self) + self.lookup_size = 0 self.lookup = {} end @@ -763,7 +767,24 @@ function Prometheus.init(dict_name, options_or_prefix) self.registry = {} self.key_index = key_index_lib.new(self.dict, KEY_INDEX_PREFIX, - self.remove_expired_keys_interval, self.auto_flush_expired) + self.remove_expired_keys_interval, self.auto_flush_expired, + function(metric_key) + -- When another worker calls reset or del on a metric, reset that + -- metric's local lookup table, so that the series is added back to the + -- key index when it is used again. A metric with an exptime adds the + -- series on every lookup anyway. + local metric_name = ngx_re_gsub(metric_key, "{.*", "", "jo") + local m = self.registry[metric_name] + if not m then + m = self.registry[ngx_re_gsub(metric_name, "_sum$", "", "jo")] + if not m or m.typ ~= TYPE_HISTOGRAM then + return + end + end + if not m.exptime or m.exptime <= 0 then + m:reset_lookup() + end + end) self.initialized = true @@ -780,6 +801,12 @@ function Prometheus.init(dict_name, options_or_prefix) return self end +local function sync_key_index(premature, self) + if not premature then + self.key_index:sync() + end +end + -- Initialize the worker counter. -- -- This can call this function from the `init_worker_by_lua` if you are calling @@ -807,6 +834,13 @@ function Prometheus:init_worker(sync_interval) error(err, 2) end self._counter = counter_instance + + -- Sync the key index on every worker, so that deletions by other workers + -- reach the local lookup tables (see the delete callback in init()). + local ok, timer_err = ngx.timer.every(self.sync_interval, sync_key_index, self) + if not ok then + ngx.log(ngx.ERR, "failed to start the key index sync timer: ", timer_err) + end end -- Register a new metric. @@ -869,6 +903,7 @@ local function register(self, name, help, label_names, buckets, typ, exptime) lookup = {}, lookup_size = 0, lookup_max_size = self.lookup_max_size, + reset_lookup = reset_lookup, parent = self, -- Store a reference for logging functions for faster lookup. _log_error = function(...) self:log_error(...) end, diff --git a/prometheus_keys.lua b/prometheus_keys.lua index 43c5536..40cb600 100644 --- a/prometheus_keys.lua +++ b/prometheus_keys.lua @@ -64,10 +64,13 @@ end -- remove_expired_keys_interval: seconds between cleanup rounds; 0 or less -- starts no cleanup timer. auto_flush_expired: false leaves --- dict:flush_expired() to the caller. +-- dict:flush_expired() to the caller. delete_callback: called with the key +-- when a sync finds its slot empty, which mostly means that another worker +-- removed it. function KeyIndex.new(shared_dict, prefix, remove_expired_keys_interval, - auto_flush_expired) + auto_flush_expired, delete_callback) local self = setmetatable({}, KeyIndex) + self.delete_callback = delete_callback self.dict = shared_dict self.key_prefix = prefix .. "key_" self.delete_count = prefix .. "delete_count" @@ -263,6 +266,9 @@ function KeyIndex:sync_range(first, last) end end elseif self.keys[i] then + if self.delete_callback then + self.delete_callback(self.keys[i]) + end self:forget_slot(i) end end diff --git a/prometheus_test.lua b/prometheus_test.lua index c783383..2baf31a 100644 --- a/prometheus_test.lua +++ b/prometheus_test.lua @@ -601,6 +601,109 @@ function TestPrometheus:testCounterSetAfterDel() luaunit.assertStrContains(out, 'metric2{f2="a",f1="x"} 3\n') luaunit.assertEquals(self.dict:get("nginx_metric_errors_total"), 0) end +-- A second library instance on the same dict stands in for another worker. +function TestPrometheus:testSeriesExportedAfterDelByAnotherWorker() + local p2 = require('prometheus').init('metrics') + local gauge2_p2 = p2:gauge("gauge2", "Gauge 2", {"f2", "f1"}) + + -- The label set goes A -> B -> A: this worker still holds the A series in + -- its lookup table when the other one retires it. + self.gauge2:set(1, {"a", "x"}) + self.gauge2:set(5, {"a", "y"}) + gauge2_p2:del({"a", "x"}) + self.p.key_index:sync() + luaunit.assertEquals(self.gauge2.lookup, {}) + luaunit.assertEquals(self.gauge2.lookup_size, 0) + + self.gauge2:set(2, {"a", "x"}) + local out = table.concat(self.p:metric_data()) + luaunit.assertStrContains(out, 'gauge2{f2="a",f1="x"} 2\n') + luaunit.assertStrContains(out, 'gauge2{f2="a",f1="y"} 5\n') + luaunit.assertEquals(self.dict:get("nginx_metric_errors_total"), 0) +end +function TestPrometheus:testHistogramExportedAfterResetByAnotherWorker() + local p2 = require('prometheus').init('metrics') + local hist1_p2 = p2:histogram("l1", "Histogram 1") + + self.hist1:observe(0.4) + self.p._counter:sync() + hist1_p2:reset() + self.p.key_index:sync() + luaunit.assertEquals(self.hist1.lookup, {}) + + self.hist1:observe(0.6) + self.p._counter:sync() + local out = table.concat(self.p:metric_data()) + luaunit.assertStrContains(out, 'l1_count 1\n') + luaunit.assertStrContains(out, 'l1_sum 0.6\n') + luaunit.assertEquals(self.dict:get("nginx_metric_errors_total"), 0) +end +-- A metric with an exptime adds the series to the key index on every lookup, +-- so its lookup table is kept. +function TestPrometheus:testExptimeLookupKeptAfterDelByAnotherWorker() + local p2 = require('prometheus').init('metrics') + local gauge_exp_p2 = p2:gauge("gauge_exp", "Gauge expire", nil, 1) + + self.gauge_exp:set(1) + gauge_exp_p2:del() + self.p.key_index:sync() + luaunit.assertNotEquals(self.gauge_exp.lookup, {}) + + self.gauge_exp:set(3) + local out = table.concat(self.p:metric_data()) + luaunit.assertStrContains(out, 'gauge_exp 3\n') + luaunit.assertEquals(self.dict:get("nginx_metric_errors_total"), 0) +end +-- Every worker syncs the key index on a timer, so a deletion by another worker +-- reaches its lookup tables without waiting for a scrape. +function TestPrometheus:testKeyIndexSyncTimer() + local every = ngx.timer.every + local timers = {} + ngx.timer.every = function(delay, fn, context) + timers[#timers + 1] = {delay = delay, fn = fn, context = context} + return true + end + local p = require('prometheus').init('metrics', {sync_interval = 2}) + ngx.timer.every = every + + local sync_timer + for _, timer in ipairs(timers) do + if timer.context == p then + sync_timer = timer + end + end + luaunit.assertNotNil(sync_timer) + luaunit.assertEquals(sync_timer.delay, 2) + + local gauge2 = p:gauge("gauge2", "Gauge 2", {"f2", "f1"}) + gauge2:set(1, {"a", "x"}) + self.gauge2:del({"a", "x"}) + + sync_timer.fn(true, sync_timer.context) + luaunit.assertNotEquals(gauge2.lookup, {}) + sync_timer.fn(false, sync_timer.context) + luaunit.assertEquals(gauge2.lookup, {}) + + gauge2:set(2, {"a", "x"}) + local out = table.concat(p:metric_data()) + luaunit.assertStrContains(out, 'gauge2{f2="a",f1="x"} 2\n') + luaunit.assertEquals(self.dict:get("nginx_metric_errors_total"), 0) +end +function TestPrometheus.testKeyIndexSyncTimerFailure() + local every = ngx.timer.every + ngx.timer.every = function() return nil, "too many pending timers" end + require('prometheus').init('metrics') + ngx.timer.every = every + + local logged = false + for _, log in ipairs(ngx.logs or {}) do + if log:find("failed to start the key index sync timer", 1, true) and + log:find("too many pending timers", 1, true) then + logged = true + end + end + luaunit.assertTrue(logged) +end function TestPrometheus:testCounterDel() self.counter1:inc(1) self.p._counter:sync() @@ -2267,6 +2370,26 @@ function TestKeyIndex:testSync() luaunit.assertEquals(keys[2], "key3") end +-- The callback reports keys that another worker removed, not local removals. +function TestKeyIndex:testDeleteCallback() + local deleted = {} + local key_index = require('prometheus_keys').new(self.dict, "_cb_", 1, nil, + function(key) deleted[#deleted + 1] = key end) + key_index:add({"key1", "key2", "key3"}, "eviction") + + -- removed by another worker + self.dict:set("_cb_delete_count", 1) + self.dict:delete("_cb_key_2") + key_index:sync() + luaunit.assertEquals(deleted, {"key2"}) + + key_index:remove("key3", "eviction") + key_index:sync() + luaunit.assertEquals(deleted, {"key2"}) + luaunit.assertEquals(key_index:list(), {"key1"}) + luaunit.assertEquals(ngx.logs, nil) +end + function TestPrometheus:testLookupMaxSize() local c = self.p:counter("testc", "Counter", {"tag"}) local g = self.p:gauge("testg", "Gauge", {"tag"})