Repository navigation
fix: export a series again when it is used after del or reset #32
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -387,8 +387,7 @@ local function lookup_or_create(self, label_values) | |
| end | ||
|
|
||
| if self.lookup_size >= self.lookup_max_size then | ||
| self.lookup_size = 0 | ||
| self.lookup = {} | ||
| self:reset_lookup() | ||
| end | ||
|
|
||
| local t = self.lookup | ||
|
|
@@ -559,6 +558,10 @@ local function del(self, label_values) | |
| if err then | ||
| self._log_error("Error deleting key: ".. k .. ": " .. err) | ||
| end | ||
|
|
||
| -- Clean up the full metric name lookup table as well, so that the series | ||
| -- is added back to the key index when it is used again. | ||
| self:reset_lookup() | ||
| end | ||
|
|
||
| -- Set the value of a gauge metric. | ||
|
|
@@ -703,6 +706,12 @@ local function reset(self) | |
| end | ||
|
|
||
| -- Clean up the full metric name lookup table as well. | ||
| self:reset_lookup() | ||
| end | ||
|
|
||
| -- Reset the metric name lookup table for a given metric. | ||
| local function reset_lookup(self) | ||
| self.lookup_size = 0 | ||
| self.lookup = {} | ||
| end | ||
|
|
||
|
|
@@ -758,7 +767,24 @@ function Prometheus.init(dict_name, options_or_prefix) | |
|
|
||
| self.registry = {} | ||
| self.key_index = key_index_lib.new(self.dict, KEY_INDEX_PREFIX, | ||
| self.remove_expired_keys_interval, self.auto_flush_expired) | ||
| self.remove_expired_keys_interval, self.auto_flush_expired, | ||
| function(metric_key) | ||
| -- When another worker calls reset or del on a metric, reset that | ||
| -- metric's local lookup table, so that the series is added back to the | ||
| -- key index when it is used again. A metric with an exptime adds the | ||
| -- series on every lookup anyway. | ||
| local metric_name = ngx_re_gsub(metric_key, "{.*", "", "jo") | ||
| local m = self.registry[metric_name] | ||
| if not m then | ||
| m = self.registry[ngx_re_gsub(metric_name, "_sum$", "", "jo")] | ||
| if not m or m.typ ~= TYPE_HISTOGRAM then | ||
| return | ||
| end | ||
| end | ||
| if not m.exptime or m.exptime <= 0 then | ||
| m:reset_lookup() | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Recover writes made before the next synchronization. If worker B deletes a cached gauge series, worker A can set that series before its next timer runs. That cache hit writes the value without restoring its index slot. This callback then clears the cache and forgets the slot. Scrapes continue to omit the value until another write occurs. This exceeds the documented consistency interval. Validate or re-register the cached key before accepting the write. Add a regression that writes before synchronization and performs no later write. (raw.githubusercontent.com)
|
||
| end | ||
| end) | ||
|
|
||
| self.initialized = true | ||
|
|
||
|
|
@@ -775,6 +801,12 @@ function Prometheus.init(dict_name, options_or_prefix) | |
| return self | ||
| end | ||
|
|
||
| local function sync_key_index(premature, self) | ||
| if not premature then | ||
| self.key_index:sync() | ||
| end | ||
| end | ||
|
|
||
| -- Initialize the worker counter. | ||
| -- | ||
| -- This can call this function from the `init_worker_by_lua` if you are calling | ||
|
|
@@ -802,6 +834,13 @@ function Prometheus:init_worker(sync_interval) | |
| error(err, 2) | ||
| end | ||
| self._counter = counter_instance | ||
|
|
||
| -- Sync the key index on every worker, so that deletions by other workers | ||
| -- reach the local lookup tables (see the delete callback in init()). | ||
| local ok, timer_err = ngx.timer.every(self.sync_interval, sync_key_index, self) | ||
| if not ok then | ||
| ngx.log(ngx.ERR, "failed to start the key index sync timer: ", timer_err) | ||
| end | ||
| end | ||
|
|
||
| -- Register a new metric. | ||
|
|
@@ -864,6 +903,7 @@ local function register(self, name, help, label_names, buckets, typ, exptime) | |
| lookup = {}, | ||
| lookup_size = 0, | ||
| lookup_max_size = self.lookup_max_size, | ||
| reset_lookup = reset_lookup, | ||
| parent = self, | ||
| -- Store a reference for logging functions for faster lookup. | ||
| _log_error = function(...) self:log_error(...) end, | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.