Add a public reachability status page
All checks were successful
Build and Deploy to Production / test (push) Successful in 18s
Build and Deploy to Production / build (push) Successful in 49s
Build and Deploy to Production / deploy (push) Successful in 33s

asxpio now probes public hosts tied to each datacenter once a minute
and stores 5-minute rows (bingo_monitor_* tables, granted to the
read-only bingo role). Show them: /:locale/status lists every monitored
site with its current state and a 24-hour bar, /status.json has the
same data, and monitored datacenter pages get a small block.

A site is down only when nothing answered for two buckets. Everything
reads as unknown when the worker is stale or fewer than half of the
control hosts in Russia answer, since probes run from Finland and
filtering can look like an outage. The page says plainly that this is
not an incident record.

Refresh test/schema.sql for asxpio's migration 017.

Assisted-by: Claude:opus-5-5
This commit is contained in:
Sergei Poljanski 2026-10-18 10:33:52 +04:00
commit 1a5506c77d
Signed by: asxpi
GPG key ID: 4F8851660FA4121B
10 changed files with 414 additions and 9 deletions

View file

@ -1,6 +1,7 @@
-- bingo tables as created by the asxpio migrations (004–016), for tests.
-- bingo tables as created by the asxpio migrations (004–017), for tests.
-- Refresh when asxpio adds a bingo migration: run them into an empty DB and
-- pg_dump -s -O -x -t bingo_datacenters -t bingo_incidents.
-- pg_dump -s -O -x -t bingo_datacenters -t bingo_incidents -t 'bingo_monitor_*'.
CREATE TABLE public.bingo_datacenters (
id integer NOT NULL,
slug text NOT NULL,
@ -61,6 +62,45 @@ ALTER TABLE public.bingo_incidents ALTER COLUMN id ADD GENERATED BY DEFAULT AS I
CACHE 1
);
CREATE TABLE public.bingo_monitor_buckets (
bucket_at timestamp without time zone NOT NULL,
rounds integer DEFAULT 0 NOT NULL,
control_checks integer DEFAULT 0 NOT NULL,
control_ok integer DEFAULT 0 NOT NULL
);
CREATE TABLE public.bingo_monitor_samples (
datacenter_id integer NOT NULL,
bucket_at timestamp without time zone NOT NULL,
checks integer DEFAULT 0 NOT NULL,
ok integer DEFAULT 0 NOT NULL,
rtt_ms_sum integer DEFAULT 0 NOT NULL
);
CREATE TABLE public.bingo_monitor_targets (
id integer NOT NULL,
datacenter_id integer,
host text NOT NULL,
port integer DEFAULT 443 NOT NULL,
label text,
source_url text,
active boolean DEFAULT true NOT NULL,
last_ok boolean,
last_rtt_ms integer,
last_checked_at timestamp without time zone,
last_change_at timestamp without time zone,
created_at timestamp without time zone NOT NULL
);
ALTER TABLE public.bingo_monitor_targets ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
SEQUENCE NAME public.bingo_monitor_targets_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1
);
ALTER TABLE ONLY public.bingo_datacenters
ADD CONSTRAINT bingo_datacenters_pkey PRIMARY KEY (id);
@ -70,9 +110,25 @@ ALTER TABLE ONLY public.bingo_datacenters
ALTER TABLE ONLY public.bingo_incidents
ADD CONSTRAINT bingo_incidents_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.bingo_monitor_buckets
ADD CONSTRAINT bingo_monitor_buckets_pkey PRIMARY KEY (bucket_at);
ALTER TABLE ONLY public.bingo_monitor_samples
ADD CONSTRAINT bingo_monitor_samples_pkey PRIMARY KEY (datacenter_id, bucket_at);
ALTER TABLE ONLY public.bingo_monitor_targets
ADD CONSTRAINT bingo_monitor_targets_pkey PRIMARY KEY (id);
CREATE INDEX bingo_incidents_datacenter_id_index ON public.bingo_incidents USING btree (datacenter_id);
CREATE INDEX bingo_monitor_targets_datacenter_id_index ON public.bingo_monitor_targets USING btree (datacenter_id);
ALTER TABLE ONLY public.bingo_incidents
ADD CONSTRAINT bingo_incidents_datacenter_id_fkey FOREIGN KEY (datacenter_id) REFERENCES public.bingo_datacenters(id) ON DELETE CASCADE;
ALTER TABLE ONLY public.bingo_monitor_samples
ADD CONSTRAINT bingo_monitor_samples_datacenter_id_fkey FOREIGN KEY (datacenter_id) REFERENCES public.bingo_datacenters(id) ON DELETE CASCADE;
ALTER TABLE ONLY public.bingo_monitor_targets
ADD CONSTRAINT bingo_monitor_targets_datacenter_id_fkey FOREIGN KEY (datacenter_id) REFERENCES public.bingo_datacenters(id) ON DELETE CASCADE;

93
test/status_test.rb Normal file
View file

@ -0,0 +1,93 @@
require_relative 'test_helper'
class StatusTest < Minitest::Test
include Rack::Test::Methods
def app = BingoWeb
NOW = Time.utc(2026, 10, 18, 12, 2)
B = Time.utc(2026, 10, 18, 12, 0) # newest bucket at NOW
def setup
skip 'TEST_DATABASE_URL not set' unless TestDb.available?
@db = DB.connection
%i[bingo_monitor_buckets bingo_monitor_samples bingo_monitor_targets bingo_incidents bingo_datacenters].each { |t| @db[t].delete }
@up, @down, @part, @none = %w[up down part none].each_with_index.map do |s, i|
Bingo::Datacenter.create(slug: "dc-#{s}", position: i + 1, name_ru: "ЦОД #{s}", city_ru: 'Москва',
operator: 'Op', status: 'operational')
end
[@up, @down, @part].each { |dc| @db[:bingo_monitor_targets].insert(datacenter_id: dc.id, host: "h-#{dc.slug}", port: 443, created_at: B) }
@db[:bingo_monitor_targets].insert(datacenter_id: nil, host: 'ctl', port: 80, created_at: B)
end
def bucket(at, control_ok: 4, control_checks: 4)
@db[:bingo_monitor_buckets].insert(bucket_at: at, rounds: 5, control_checks: control_checks, control_ok: control_ok)
end
def sample(dc, at, ok, checks = 5)
@db[:bingo_monitor_samples].insert(datacenter_id: dc.id, bucket_at: at, checks: checks, ok: ok, rtt_ms_sum: ok * 40)
end
def two_buckets(base = B, **controls)
[base - 300, base].each do |at|
bucket(at, **controls)
sample(@up, at, 5)
sample(@down, at, 0)
sample(@part, at, 2)
end
end
def states(report) = report.sites.to_h { |s| [s.dc.slug, s.state] }
def test_states_from_newest_buckets
two_buckets
r = Bingo::Status.load(NOW)
assert_equal({ 'dc-up' => 'up', 'dc-down' => 'down', 'dc-part' => 'degraded' }, states(r))
assert_equal ['dc-none'], r.unmonitored.map(&:slug)
assert_equal 40, r.sites.first.rtt_ms
assert_equal 'ok', r.state
end
def test_one_dark_bucket_after_a_good_one_is_not_yet_down
bucket(B - 300); sample(@down, B - 300, 5)
bucket(B); sample(@down, B, 0)
assert_equal 'degraded', states(Bingo::Status.load(NOW))['dc-down']
end
def test_lost_vantage_makes_everything_unknown
two_buckets(control_ok: 1, control_checks: 4)
r = Bingo::Status.load(NOW)
assert_equal %w[unknown], states(r).values.uniq
assert_equal 'vantage', r.state
assert_nil r.sites.first.history.last, 'untrusted hour shows no data'
end
def test_stale_worker_makes_everything_unknown
two_buckets
r = Bingo::Status.load(NOW + 3600)
assert_equal %w[unknown], states(r).values.uniq
assert_equal 'stale', r.state
end
def test_status_page_and_json
two_buckets(Time.at(Time.now.to_i / 300 * 300).utc) # pages use the real clock
get '/ru/status'
assert last_response.ok?
assert_includes last_response.body, 'не отвечает'
assert_includes last_response.body, 'ЦОД none' # unmonitored list
get '/status.json'
data = JSON.parse(last_response.body)
assert_equal %w[dc-up dc-down dc-part], data['datacenters'].map { |d| d['slug'] }
assert_equal ['dc-none'], data['unmonitored']
get '/en/dc/dc-up'
assert_includes last_response.body, 'Network reachability'
get '/en/dc/dc-none'
refute_includes last_response.body, 'Network reachability'
end
def test_status_page_before_any_round
get '/en/status'
assert last_response.ok?
assert_includes last_response.body, 'No checks yet.'
end
end