diff --git a/README.md b/README.md index a887689..6a58c43 100644 --- a/README.md +++ b/README.md @@ -103,9 +103,9 @@ section of nginx configuration. metric names on output. * `error_metric_name` (string): Can be used to change the default name of error metric (see [Built-in metrics](#built-in-metrics) for details). - * `sync_interval` (number): sets per-worker counter sync interval in seconds. - This sets the boundary on eventual consistency of counter metrics. Defaults - to 1. + * `sync_interval` (number): sets the sync interval for per-worker counters and + key index (in seconds). This sets the boundary on eventual consistency of + counter metric increments, and metric resets/deletions. Defaults to 1. * `lookup_max_size` (number): maximum size of a per-metric lookup table maintained by each worker to cache full metric names. Defaults to 1000. If you have metrics with extremely high cardinality and lots of available diff --git a/prometheus.lua b/prometheus.lua index 5da4441..e9f8c0f 100644 --- a/prometheus.lua +++ b/prometheus.lua @@ -387,8 +387,7 @@ local function lookup_or_create(self, label_values) end if self.lookup_size >= self.lookup_max_size then - self.lookup_size = 0 - self.lookup = {} + self:reset_lookup() end local t = self.lookup @@ -559,6 +558,10 @@ local function del(self, label_values) if err then self._log_error("Error deleting key: ".. k .. ": " .. err) end + + -- Clean up the full metric name lookup table as well, so that the series + -- is added back to the key index when it is used again. + self:reset_lookup() end -- Set the value of a gauge metric. @@ -703,6 +706,12 @@ local function reset(self) end -- Clean up the full metric name lookup table as well. + self:reset_lookup() +end + +-- Reset the metric name lookup table for a given metric. +local function reset_lookup(self) + self.lookup_size = 0 self.lookup = {} end @@ -758,7 +767,24 @@ function Prometheus.init(dict_name, options_or_prefix) self.registry = {} self.key_index = key_index_lib.new(self.dict, KEY_INDEX_PREFIX, - self.remove_expired_keys_interval, self.auto_flush_expired) + self.remove_expired_keys_interval, self.auto_flush_expired, + function(metric_key) + -- When another worker calls reset or del on a metric, reset that + -- metric's local lookup table, so that the series is added back to the + -- key index when it is used again. A metric with an exptime adds the + -- series on every lookup anyway. + local metric_name = ngx_re_gsub(metric_key, "{.*", "", "jo") + local m = self.registry[metric_name] + if not m then + m = self.registry[ngx_re_gsub(metric_name, "_sum$", "", "jo")] + if not m or m.typ ~= TYPE_HISTOGRAM then + return + end + end + if not m.exptime or m.exptime <= 0 then + m:reset_lookup() + end + end) self.initialized = true @@ -775,6 +801,12 @@ function Prometheus.init(dict_name, options_or_prefix) return self end +local function sync_key_index(premature, self) + if not premature then + self.key_index:sync() + end +end + -- Initialize the worker counter. -- -- This can call this function from the `init_worker_by_lua` if you are calling @@ -802,6 +834,13 @@ function Prometheus:init_worker(sync_interval) error(err, 2) end self._counter = counter_instance + + -- Sync the key index on every worker, so that deletions by other workers + -- reach the local lookup tables (see the delete callback in init()). + local ok, timer_err = ngx.timer.every(self.sync_interval, sync_key_index, self) + if not ok then + ngx.log(ngx.ERR, "failed to start the key index sync timer: ", timer_err) + end end -- Register a new metric. @@ -864,6 +903,7 @@ local function register(self, name, help, label_names, buckets, typ, exptime) lookup = {}, lookup_size = 0, lookup_max_size = self.lookup_max_size, + reset_lookup = reset_lookup, parent = self, -- Store a reference for logging functions for faster lookup. _log_error = function(...) self:log_error(...) end, diff --git a/prometheus_keys.lua b/prometheus_keys.lua index 43c5536..40cb600 100644 --- a/prometheus_keys.lua +++ b/prometheus_keys.lua @@ -64,10 +64,13 @@ end -- remove_expired_keys_interval: seconds between cleanup rounds; 0 or less -- starts no cleanup timer. auto_flush_expired: false leaves --- dict:flush_expired() to the caller. +-- dict:flush_expired() to the caller. delete_callback: called with the key +-- when a sync finds its slot empty, which mostly means that another worker +-- removed it. function KeyIndex.new(shared_dict, prefix, remove_expired_keys_interval, - auto_flush_expired) + auto_flush_expired, delete_callback) local self = setmetatable({}, KeyIndex) + self.delete_callback = delete_callback self.dict = shared_dict self.key_prefix = prefix .. "key_" self.delete_count = prefix .. "delete_count" @@ -263,6 +266,9 @@ function KeyIndex:sync_range(first, last) end end elseif self.keys[i] then + if self.delete_callback then + self.delete_callback(self.keys[i]) + end self:forget_slot(i) end end diff --git a/prometheus_test.lua b/prometheus_test.lua index 853b6a9..2baf31a 100644 --- a/prometheus_test.lua +++ b/prometheus_test.lua @@ -581,6 +581,129 @@ function TestPrometheus:testGaugeDel() luaunit.assertEquals(self.dict:get('gauge2{f2="f2value",f1="f1value"}'), nil) luaunit.assertEquals(self.dict:get("nginx_metric_errors_total"), 1) end +function TestPrometheus:testGaugeSetAfterDel() + self.gauge2:set(1, {"a", "x"}) + self.gauge2:del({"a", "x"}) + self.gauge2:set(2, {"a", "x"}) + + local out = table.concat(self.p:metric_data()) + luaunit.assertStrContains(out, 'gauge2{f2="a",f1="x"} 2\n') + luaunit.assertEquals(self.dict:get("nginx_metric_errors_total"), 0) +end +function TestPrometheus:testCounterSetAfterDel() + self.counter2:inc(1, {"a", "x"}) + self.p._counter:sync() + self.counter2:del({"a", "x"}) + self.counter2:inc(3, {"a", "x"}) + self.p._counter:sync() + + local out = table.concat(self.p:metric_data()) + luaunit.assertStrContains(out, 'metric2{f2="a",f1="x"} 3\n') + luaunit.assertEquals(self.dict:get("nginx_metric_errors_total"), 0) +end +-- A second library instance on the same dict stands in for another worker. +function TestPrometheus:testSeriesExportedAfterDelByAnotherWorker() + local p2 = require('prometheus').init('metrics') + local gauge2_p2 = p2:gauge("gauge2", "Gauge 2", {"f2", "f1"}) + + -- The label set goes A -> B -> A: this worker still holds the A series in + -- its lookup table when the other one retires it. + self.gauge2:set(1, {"a", "x"}) + self.gauge2:set(5, {"a", "y"}) + gauge2_p2:del({"a", "x"}) + self.p.key_index:sync() + luaunit.assertEquals(self.gauge2.lookup, {}) + luaunit.assertEquals(self.gauge2.lookup_size, 0) + + self.gauge2:set(2, {"a", "x"}) + local out = table.concat(self.p:metric_data()) + luaunit.assertStrContains(out, 'gauge2{f2="a",f1="x"} 2\n') + luaunit.assertStrContains(out, 'gauge2{f2="a",f1="y"} 5\n') + luaunit.assertEquals(self.dict:get("nginx_metric_errors_total"), 0) +end +function TestPrometheus:testHistogramExportedAfterResetByAnotherWorker() + local p2 = require('prometheus').init('metrics') + local hist1_p2 = p2:histogram("l1", "Histogram 1") + + self.hist1:observe(0.4) + self.p._counter:sync() + hist1_p2:reset() + self.p.key_index:sync() + luaunit.assertEquals(self.hist1.lookup, {}) + + self.hist1:observe(0.6) + self.p._counter:sync() + local out = table.concat(self.p:metric_data()) + luaunit.assertStrContains(out, 'l1_count 1\n') + luaunit.assertStrContains(out, 'l1_sum 0.6\n') + luaunit.assertEquals(self.dict:get("nginx_metric_errors_total"), 0) +end +-- A metric with an exptime adds the series to the key index on every lookup, +-- so its lookup table is kept. +function TestPrometheus:testExptimeLookupKeptAfterDelByAnotherWorker() + local p2 = require('prometheus').init('metrics') + local gauge_exp_p2 = p2:gauge("gauge_exp", "Gauge expire", nil, 1) + + self.gauge_exp:set(1) + gauge_exp_p2:del() + self.p.key_index:sync() + luaunit.assertNotEquals(self.gauge_exp.lookup, {}) + + self.gauge_exp:set(3) + local out = table.concat(self.p:metric_data()) + luaunit.assertStrContains(out, 'gauge_exp 3\n') + luaunit.assertEquals(self.dict:get("nginx_metric_errors_total"), 0) +end +-- Every worker syncs the key index on a timer, so a deletion by another worker +-- reaches its lookup tables without waiting for a scrape. +function TestPrometheus:testKeyIndexSyncTimer() + local every = ngx.timer.every + local timers = {} + ngx.timer.every = function(delay, fn, context) + timers[#timers + 1] = {delay = delay, fn = fn, context = context} + return true + end + local p = require('prometheus').init('metrics', {sync_interval = 2}) + ngx.timer.every = every + + local sync_timer + for _, timer in ipairs(timers) do + if timer.context == p then + sync_timer = timer + end + end + luaunit.assertNotNil(sync_timer) + luaunit.assertEquals(sync_timer.delay, 2) + + local gauge2 = p:gauge("gauge2", "Gauge 2", {"f2", "f1"}) + gauge2:set(1, {"a", "x"}) + self.gauge2:del({"a", "x"}) + + sync_timer.fn(true, sync_timer.context) + luaunit.assertNotEquals(gauge2.lookup, {}) + sync_timer.fn(false, sync_timer.context) + luaunit.assertEquals(gauge2.lookup, {}) + + gauge2:set(2, {"a", "x"}) + local out = table.concat(p:metric_data()) + luaunit.assertStrContains(out, 'gauge2{f2="a",f1="x"} 2\n') + luaunit.assertEquals(self.dict:get("nginx_metric_errors_total"), 0) +end +function TestPrometheus.testKeyIndexSyncTimerFailure() + local every = ngx.timer.every + ngx.timer.every = function() return nil, "too many pending timers" end + require('prometheus').init('metrics') + ngx.timer.every = every + + local logged = false + for _, log in ipairs(ngx.logs or {}) do + if log:find("failed to start the key index sync timer", 1, true) and + log:find("too many pending timers", 1, true) then + logged = true + end + end + luaunit.assertTrue(logged) +end function TestPrometheus:testCounterDel() self.counter1:inc(1) self.p._counter:sync() @@ -2247,6 +2370,26 @@ function TestKeyIndex:testSync() luaunit.assertEquals(keys[2], "key3") end +-- The callback reports keys that another worker removed, not local removals. +function TestKeyIndex:testDeleteCallback() + local deleted = {} + local key_index = require('prometheus_keys').new(self.dict, "_cb_", 1, nil, + function(key) deleted[#deleted + 1] = key end) + key_index:add({"key1", "key2", "key3"}, "eviction") + + -- removed by another worker + self.dict:set("_cb_delete_count", 1) + self.dict:delete("_cb_key_2") + key_index:sync() + luaunit.assertEquals(deleted, {"key2"}) + + key_index:remove("key3", "eviction") + key_index:sync() + luaunit.assertEquals(deleted, {"key2"}) + luaunit.assertEquals(key_index:list(), {"key1"}) + luaunit.assertEquals(ngx.logs, nil) +end + function TestPrometheus:testLookupMaxSize() local c = self.p:counter("testc", "Counter", {"tag"}) local g = self.p:gauge("testg", "Gauge", {"tag"})