Add a public reachability status page
asxpio now probes public hosts tied to each datacenter once a minute and stores 5-minute rows (bingo_monitor_* tables, granted to the read-only bingo role). Show them: /:locale/status lists every monitored site with its current state and a 24-hour bar, /status.json has the same data, and monitored datacenter pages get a small block. A site is down only when nothing answered for two buckets. Everything reads as unknown when the worker is stale or fewer than half of the control hosts in Russia answer, since probes run from Finland and filtering can look like an outage. The page says plainly that this is not an incident record. Refresh test/schema.sql for asxpio's migration 017. Assisted-by: Claude:opus-5-5
This commit is contained in:
parent
2fe297fce1
commit
1a5506c77d
10 changed files with 414 additions and 9 deletions
|
|
@ -1,6 +1,7 @@
|
|||
-- bingo tables as created by the asxpio migrations (004–016), for tests.
|
||||
-- bingo tables as created by the asxpio migrations (004–017), for tests.
|
||||
-- Refresh when asxpio adds a bingo migration: run them into an empty DB and
|
||||
-- pg_dump -s -O -x -t bingo_datacenters -t bingo_incidents.
|
||||
-- pg_dump -s -O -x -t bingo_datacenters -t bingo_incidents -t 'bingo_monitor_*'.
|
||||
|
||||
CREATE TABLE public.bingo_datacenters (
|
||||
id integer NOT NULL,
|
||||
slug text NOT NULL,
|
||||
|
|
@ -61,6 +62,45 @@ ALTER TABLE public.bingo_incidents ALTER COLUMN id ADD GENERATED BY DEFAULT AS I
|
|||
CACHE 1
|
||||
);
|
||||
|
||||
CREATE TABLE public.bingo_monitor_buckets (
|
||||
bucket_at timestamp without time zone NOT NULL,
|
||||
rounds integer DEFAULT 0 NOT NULL,
|
||||
control_checks integer DEFAULT 0 NOT NULL,
|
||||
control_ok integer DEFAULT 0 NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE public.bingo_monitor_samples (
|
||||
datacenter_id integer NOT NULL,
|
||||
bucket_at timestamp without time zone NOT NULL,
|
||||
checks integer DEFAULT 0 NOT NULL,
|
||||
ok integer DEFAULT 0 NOT NULL,
|
||||
rtt_ms_sum integer DEFAULT 0 NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE public.bingo_monitor_targets (
|
||||
id integer NOT NULL,
|
||||
datacenter_id integer,
|
||||
host text NOT NULL,
|
||||
port integer DEFAULT 443 NOT NULL,
|
||||
label text,
|
||||
source_url text,
|
||||
active boolean DEFAULT true NOT NULL,
|
||||
last_ok boolean,
|
||||
last_rtt_ms integer,
|
||||
last_checked_at timestamp without time zone,
|
||||
last_change_at timestamp without time zone,
|
||||
created_at timestamp without time zone NOT NULL
|
||||
);
|
||||
|
||||
ALTER TABLE public.bingo_monitor_targets ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
|
||||
SEQUENCE NAME public.bingo_monitor_targets_id_seq
|
||||
START WITH 1
|
||||
INCREMENT BY 1
|
||||
NO MINVALUE
|
||||
NO MAXVALUE
|
||||
CACHE 1
|
||||
);
|
||||
|
||||
ALTER TABLE ONLY public.bingo_datacenters
|
||||
ADD CONSTRAINT bingo_datacenters_pkey PRIMARY KEY (id);
|
||||
|
||||
|
|
@ -70,9 +110,25 @@ ALTER TABLE ONLY public.bingo_datacenters
|
|||
ALTER TABLE ONLY public.bingo_incidents
|
||||
ADD CONSTRAINT bingo_incidents_pkey PRIMARY KEY (id);
|
||||
|
||||
ALTER TABLE ONLY public.bingo_monitor_buckets
|
||||
ADD CONSTRAINT bingo_monitor_buckets_pkey PRIMARY KEY (bucket_at);
|
||||
|
||||
ALTER TABLE ONLY public.bingo_monitor_samples
|
||||
ADD CONSTRAINT bingo_monitor_samples_pkey PRIMARY KEY (datacenter_id, bucket_at);
|
||||
|
||||
ALTER TABLE ONLY public.bingo_monitor_targets
|
||||
ADD CONSTRAINT bingo_monitor_targets_pkey PRIMARY KEY (id);
|
||||
|
||||
CREATE INDEX bingo_incidents_datacenter_id_index ON public.bingo_incidents USING btree (datacenter_id);
|
||||
|
||||
CREATE INDEX bingo_monitor_targets_datacenter_id_index ON public.bingo_monitor_targets USING btree (datacenter_id);
|
||||
|
||||
ALTER TABLE ONLY public.bingo_incidents
|
||||
ADD CONSTRAINT bingo_incidents_datacenter_id_fkey FOREIGN KEY (datacenter_id) REFERENCES public.bingo_datacenters(id) ON DELETE CASCADE;
|
||||
|
||||
ALTER TABLE ONLY public.bingo_monitor_samples
|
||||
ADD CONSTRAINT bingo_monitor_samples_datacenter_id_fkey FOREIGN KEY (datacenter_id) REFERENCES public.bingo_datacenters(id) ON DELETE CASCADE;
|
||||
|
||||
ALTER TABLE ONLY public.bingo_monitor_targets
|
||||
ADD CONSTRAINT bingo_monitor_targets_datacenter_id_fkey FOREIGN KEY (datacenter_id) REFERENCES public.bingo_datacenters(id) ON DELETE CASCADE;
|
||||
|
||||
|
|
|
|||
93
test/status_test.rb
Normal file
93
test/status_test.rb
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
require_relative 'test_helper'
|
||||
|
||||
class StatusTest < Minitest::Test
|
||||
include Rack::Test::Methods
|
||||
|
||||
def app = BingoWeb
|
||||
|
||||
NOW = Time.utc(2026, 10, 18, 12, 2)
|
||||
B = Time.utc(2026, 10, 18, 12, 0) # newest bucket at NOW
|
||||
|
||||
def setup
|
||||
skip 'TEST_DATABASE_URL not set' unless TestDb.available?
|
||||
@db = DB.connection
|
||||
%i[bingo_monitor_buckets bingo_monitor_samples bingo_monitor_targets bingo_incidents bingo_datacenters].each { |t| @db[t].delete }
|
||||
@up, @down, @part, @none = %w[up down part none].each_with_index.map do |s, i|
|
||||
Bingo::Datacenter.create(slug: "dc-#{s}", position: i + 1, name_ru: "ЦОД #{s}", city_ru: 'Москва',
|
||||
operator: 'Op', status: 'operational')
|
||||
end
|
||||
[@up, @down, @part].each { |dc| @db[:bingo_monitor_targets].insert(datacenter_id: dc.id, host: "h-#{dc.slug}", port: 443, created_at: B) }
|
||||
@db[:bingo_monitor_targets].insert(datacenter_id: nil, host: 'ctl', port: 80, created_at: B)
|
||||
end
|
||||
|
||||
def bucket(at, control_ok: 4, control_checks: 4)
|
||||
@db[:bingo_monitor_buckets].insert(bucket_at: at, rounds: 5, control_checks: control_checks, control_ok: control_ok)
|
||||
end
|
||||
|
||||
def sample(dc, at, ok, checks = 5)
|
||||
@db[:bingo_monitor_samples].insert(datacenter_id: dc.id, bucket_at: at, checks: checks, ok: ok, rtt_ms_sum: ok * 40)
|
||||
end
|
||||
|
||||
def two_buckets(base = B, **controls)
|
||||
[base - 300, base].each do |at|
|
||||
bucket(at, **controls)
|
||||
sample(@up, at, 5)
|
||||
sample(@down, at, 0)
|
||||
sample(@part, at, 2)
|
||||
end
|
||||
end
|
||||
|
||||
def states(report) = report.sites.to_h { |s| [s.dc.slug, s.state] }
|
||||
|
||||
def test_states_from_newest_buckets
|
||||
two_buckets
|
||||
r = Bingo::Status.load(NOW)
|
||||
assert_equal({ 'dc-up' => 'up', 'dc-down' => 'down', 'dc-part' => 'degraded' }, states(r))
|
||||
assert_equal ['dc-none'], r.unmonitored.map(&:slug)
|
||||
assert_equal 40, r.sites.first.rtt_ms
|
||||
assert_equal 'ok', r.state
|
||||
end
|
||||
|
||||
def test_one_dark_bucket_after_a_good_one_is_not_yet_down
|
||||
bucket(B - 300); sample(@down, B - 300, 5)
|
||||
bucket(B); sample(@down, B, 0)
|
||||
assert_equal 'degraded', states(Bingo::Status.load(NOW))['dc-down']
|
||||
end
|
||||
|
||||
def test_lost_vantage_makes_everything_unknown
|
||||
two_buckets(control_ok: 1, control_checks: 4)
|
||||
r = Bingo::Status.load(NOW)
|
||||
assert_equal %w[unknown], states(r).values.uniq
|
||||
assert_equal 'vantage', r.state
|
||||
assert_nil r.sites.first.history.last, 'untrusted hour shows no data'
|
||||
end
|
||||
|
||||
def test_stale_worker_makes_everything_unknown
|
||||
two_buckets
|
||||
r = Bingo::Status.load(NOW + 3600)
|
||||
assert_equal %w[unknown], states(r).values.uniq
|
||||
assert_equal 'stale', r.state
|
||||
end
|
||||
|
||||
def test_status_page_and_json
|
||||
two_buckets(Time.at(Time.now.to_i / 300 * 300).utc) # pages use the real clock
|
||||
get '/ru/status'
|
||||
assert last_response.ok?
|
||||
assert_includes last_response.body, 'не отвечает'
|
||||
assert_includes last_response.body, 'ЦОД none' # unmonitored list
|
||||
get '/status.json'
|
||||
data = JSON.parse(last_response.body)
|
||||
assert_equal %w[dc-up dc-down dc-part], data['datacenters'].map { |d| d['slug'] }
|
||||
assert_equal ['dc-none'], data['unmonitored']
|
||||
get '/en/dc/dc-up'
|
||||
assert_includes last_response.body, 'Network reachability'
|
||||
get '/en/dc/dc-none'
|
||||
refute_includes last_response.body, 'Network reachability'
|
||||
end
|
||||
|
||||
def test_status_page_before_any_round
|
||||
get '/en/status'
|
||||
assert last_response.ok?
|
||||
assert_includes last_response.body, 'No checks yet.'
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Add a link
Reference in a new issue