bingo: add a reachability monitor for the status page
datacenterbingo gets a public page showing whether each datacenter's network answers. The probing lives here, next to the schema and the admin, since the public site only has a read-only role. bin/monitor runs as a second container from the same image. Once a minute it opens one TCP connection to every active target (a handshake or a refusal both count as answered) and folds the results into 5-minute rows per datacenter, plus a per-bucket row for control hosts in Russia, so a lost vantage point isn't mistaken for dark sites. Rows are kept 30 days. Targets are managed at /admin/bingo/monitor; only hosts a public source ties to that one site belong there. Migration 017 also grants the new tables to the bingo role, so the public site can read them without a storage redeploy. Assisted-by: Claude:opus-5-5
This commit is contained in:
parent
e0038ef3a4
commit
0cc3e1dece
10 changed files with 415 additions and 1 deletions
55
db/migrations/017_bingo_monitor.rb
Normal file
55
db/migrations/017_bingo_monitor.rb
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
Sequel.migration do
|
||||
up do
|
||||
# Reachability monitor behind datacenterbingo's status page (bin/monitor
|
||||
# writes, the public site reads). A target is a host a public source ties
|
||||
# to one datacenter; datacenter_id NULL marks a control host in Russia
|
||||
# that belongs to no listed site, so "site down" can be told apart from
|
||||
# "our vantage point lost Russia".
|
||||
create_table :bingo_monitor_targets do
|
||||
primary_key :id
|
||||
foreign_key :datacenter_id, :bingo_datacenters, on_delete: :cascade
|
||||
String :host, null: false
|
||||
Integer :port, null: false, default: 443
|
||||
String :label
|
||||
String :source_url
|
||||
TrueClass :active, null: false, default: true
|
||||
TrueClass :last_ok
|
||||
Integer :last_rtt_ms
|
||||
DateTime :last_checked_at
|
||||
DateTime :last_change_at
|
||||
DateTime :created_at, null: false
|
||||
index :datacenter_id
|
||||
end
|
||||
|
||||
# Per datacenter per 5-minute bucket: probes sent, probes answered and
|
||||
# the summed handshake time of the answered ones.
|
||||
create_table :bingo_monitor_samples do
|
||||
foreign_key :datacenter_id, :bingo_datacenters, null: false, on_delete: :cascade
|
||||
DateTime :bucket_at, null: false
|
||||
Integer :checks, null: false, default: 0
|
||||
Integer :ok, null: false, default: 0
|
||||
Integer :rtt_ms_sum, null: false, default: 0
|
||||
primary_key %i[datacenter_id bucket_at]
|
||||
end
|
||||
|
||||
# One row per bucket the worker ran in: its rounds (a heartbeat) and the
|
||||
# control hosts' results.
|
||||
create_table :bingo_monitor_buckets do
|
||||
DateTime :bucket_at, primary_key: true
|
||||
Integer :rounds, null: false, default: 0
|
||||
Integer :control_checks, null: false, default: 0
|
||||
Integer :control_ok, null: false, default: 0
|
||||
end
|
||||
|
||||
# The public site's read-only role (provisioned by the storage repo).
|
||||
if from(:pg_roles).where(rolname: 'bingo').any?
|
||||
%w[bingo_monitor_targets bingo_monitor_samples bingo_monitor_buckets].each do |t|
|
||||
run "GRANT SELECT ON #{t} TO bingo"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
down do
|
||||
drop_table :bingo_monitor_buckets, :bingo_monitor_samples, :bingo_monitor_targets
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Add a link
Reference in a new issue