diff --git a/CLAUDE.md b/CLAUDE.md index e08a259..2140069 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -139,6 +139,7 @@ The public site (datacenterbingo.xyz) lives in its own repo and container: `git. - Served publicly by the bingo site at `/p/:file`; previewed here at `/admin/bingo/p/:file`. - IndexNow (`Bingo::IndexNow`): admin actions ping api.indexnow.org with the changed URLs. The key comes from `INDEXNOW` → `INDEXNOW_KEY`; the bingo site has the same key and serves `/.txt`. Unset = no-op. - Photos are third-party (operator sites, Habr, Wikimedia Commons CC BY-SA) — keep `photo_credit` filled; renders/illustrations are marked as such. +- Monitor (`lib/bingo_monitor.rb`, `bin/monitor`, migration 017): the `asxpio-monitor` container (same image, compose healthcheck on a heartbeat file) opens one TCP connection per minute to each active `bingo_monitor_targets` row; a handshake or a refusal counts as answered. Results fold into 5-minute `bingo_monitor_samples` per datacenter (upserts) and `bingo_monitor_buckets` (rounds + control hosts, i.e. targets with `datacenter_id` NULL); kept 30 days. Targets are managed at `/admin/bingo/monitor` (bulk lines `slug host[:port] source_url label`, slug `_control` for controls). Only add hosts a public source ties to that one site. The public status page in datacenterbingo reads these tables; migration 017 grants them to the `bingo` role. - No street addresses or coordinates on the page — city level only. ## Operational notes diff --git a/asxpio.rb b/asxpio.rb index 06f3b37..8dfcf01 100644 --- a/asxpio.rb +++ b/asxpio.rb @@ -38,6 +38,7 @@ if ENV['DATABASE_URL'] require_relative 'lib/invoice_pdf' require_relative 'lib/invoice_stats' require_relative 'lib/bingo' + require_relative 'lib/bingo_monitor' Bingo.seed! require_relative 'lib/user' require_relative 'lib/audit' diff --git a/bin/monitor b/bin/monitor new file mode 100755 index 0000000..92b020d --- /dev/null +++ b/bin/monitor @@ -0,0 +1,42 @@ +#!/usr/bin/env ruby +# Reachability worker for datacenterbingo's status page (Bingo::Monitor). +# Runs as its own container from the asxpio image; the web container owns +# migrations, so this waits until the monitor tables exist. +require 'logger' +require 'fileutils' + +$root = File.expand_path('..', __dir__) +$logger = Logger.new($stdout) +$env = ENV.fetch('RACK_ENV', 'development') +$stdout.sync = true + +require_relative '../lib/db' +DB.connect! +until DB.connection.table_exists?(:bingo_monitor_buckets) + $logger.info('monitor: waiting for migrations') + sleep 10 +end +require_relative '../lib/bingo' +require_relative '../lib/bingo_monitor' + +HEARTBEAT = ENV.fetch('MONITOR_HEARTBEAT', '/tmp/monitor.heartbeat') +pruned_at = nil + +loop do + started = Time.now.utc + begin + targets = Bingo::Monitor::Target.where(active: true).all + results = Bingo::Monitor.round(targets) + Bingo::Monitor.record!(results, at: started) + if pruned_at.nil? || started - pruned_at > 3600 + Bingo::Monitor.prune!(started) + pruned_at = started + end + FileUtils.touch(HEARTBEAT) + up = results.count { |_, ms| ms } + $logger.info("monitor: #{up}/#{results.size} answered in #{(Time.now.utc - started).round(1)}s") + rescue StandardError => e # a DB blip must not end the worker + $logger.error("monitor: #{e.class}: #{e.message}") + end + sleep [Bingo::Monitor::INTERVAL - (Time.now.utc - started), 1].max +end diff --git a/db/migrations/017_bingo_monitor.rb b/db/migrations/017_bingo_monitor.rb new file mode 100644 index 0000000..46daece --- /dev/null +++ b/db/migrations/017_bingo_monitor.rb @@ -0,0 +1,55 @@ +Sequel.migration do + up do + # Reachability monitor behind datacenterbingo's status page (bin/monitor + # writes, the public site reads). A target is a host a public source ties + # to one datacenter; datacenter_id NULL marks a control host in Russia + # that belongs to no listed site, so "site down" can be told apart from + # "our vantage point lost Russia". + create_table :bingo_monitor_targets do + primary_key :id + foreign_key :datacenter_id, :bingo_datacenters, on_delete: :cascade + String :host, null: false + Integer :port, null: false, default: 443 + String :label + String :source_url + TrueClass :active, null: false, default: true + TrueClass :last_ok + Integer :last_rtt_ms + DateTime :last_checked_at + DateTime :last_change_at + DateTime :created_at, null: false + index :datacenter_id + end + + # Per datacenter per 5-minute bucket: probes sent, probes answered and + # the summed handshake time of the answered ones. + create_table :bingo_monitor_samples do + foreign_key :datacenter_id, :bingo_datacenters, null: false, on_delete: :cascade + DateTime :bucket_at, null: false + Integer :checks, null: false, default: 0 + Integer :ok, null: false, default: 0 + Integer :rtt_ms_sum, null: false, default: 0 + primary_key %i[datacenter_id bucket_at] + end + + # One row per bucket the worker ran in: its rounds (a heartbeat) and the + # control hosts' results. + create_table :bingo_monitor_buckets do + DateTime :bucket_at, primary_key: true + Integer :rounds, null: false, default: 0 + Integer :control_checks, null: false, default: 0 + Integer :control_ok, null: false, default: 0 + end + + # The public site's read-only role (provisioned by the storage repo). + if from(:pg_roles).where(rolname: 'bingo').any? + %w[bingo_monitor_targets bingo_monitor_samples bingo_monitor_buckets].each do |t| + run "GRANT SELECT ON #{t} TO bingo" + end + end + end + + down do + drop_table :bingo_monitor_buckets, :bingo_monitor_samples, :bingo_monitor_targets + end +end diff --git a/docker-compose.yml b/docker-compose.yml index 46e9c74..24243a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,25 @@ services: - "traefik.http.routers.asxpio.service=asxpio" - "traefik.http.services.asxpio.loadbalancer.server.port=3000" + # Reachability worker for datacenterbingo's status page (bin/monitor). + # Same image; the deploy's sed sets both image lines. + asxpio-monitor: + image: asxpio:latest + container_name: asxpio-monitor + restart: unless-stopped + env_file: .env + command: ["ruby", "bin/monitor"] + networks: + - storage # postgres; not internal, so probes reach the internet + mem_limit: 256m + healthcheck: + # The worker touches its heartbeat after every round (60 s). + test: ["CMD", "ruby", "-e", "exit(Time.now - File.mtime('/tmp/monitor.heartbeat') < 300 ? 0 : 1)"] + interval: 60s + timeout: 10s + start_period: 120s + retries: 3 + networks: traefik: external: true diff --git a/lib/bingo_admin.rb b/lib/bingo_admin.rb index 41f629f..be26ec1 100644 --- a/lib/bingo_admin.rb +++ b/lib/bingo_admin.rb @@ -226,6 +226,46 @@ module BingoAdmin redirect '/admin/bingo' end + # Reachability targets for the public status page (Bingo::Monitor). + app.get '/admin/bingo/monitor' do + @page_title = 'Monitor · bingo admin' + @noindex = true + @targets = Bingo::Monitor::Target.eager(:datacenter).all + .sort_by { |t| [t.datacenter&.position || 0, t.host, t.port] } + @unmonitored = Bingo::Datacenter.where(status: 'operational') + .exclude(id: Bingo::Monitor::Target.exclude(datacenter_id: nil).select(:datacenter_id)) + .order(:position).all + @last_round = DB.connection[:bingo_monitor_buckets].order(Sequel.desc(:bucket_at)).first + @flash = session.delete(:bingo_flash) + erb :'admin/bingo/monitor' + end + + app.post '/admin/bingo/monitor' do + attrs, errors = Bingo::Monitor.parse(params[:targets]) + if errors.empty? + attrs.each { |a| Bingo::Monitor::Target.create(a) } + audit!('bingo.monitor_add', nil, "#{attrs.size} target(s)") + session[:bingo_flash] = "Added #{attrs.size} target(s)." + else + session[:bingo_flash] = "Nothing added: #{errors.join('; ')}" + end + redirect '/admin/bingo/monitor' + end + + app.post '/admin/bingo/monitor/:id/toggle' do + t = Bingo::Monitor::Target[params[:id].to_i] or halt 404 + t.update(active: !t.active) + audit!('bingo.monitor_toggle', t.endpoint, t.active ? 'on' : 'off') + redirect '/admin/bingo/monitor' + end + + app.post '/admin/bingo/monitor/:id/delete' do + t = Bingo::Monitor::Target[params[:id].to_i] or halt 404 + t.destroy + audit!('bingo.monitor_delete', t.endpoint, t.datacenter&.slug) + redirect '/admin/bingo/monitor' + end + app.post '/admin/bingo/incidents/:id/delete' do inc = Bingo::Incident[params[:id].to_i] inc&.destroy # hook bumps the datacenter's updated_at diff --git a/lib/bingo_monitor.rb b/lib/bingo_monitor.rb new file mode 100644 index 0000000..31dc3d9 --- /dev/null +++ b/lib/bingo_monitor.rb @@ -0,0 +1,122 @@ +require 'socket' + +module Bingo + # Reachability monitor for datacenterbingo's status page. Every minute + # bin/monitor opens one TCP connection to each active target; a handshake + # or a refusal both prove the site's network answers. Results fold into + # 5-minute rows per datacenter (upserts, so a restart mid-bucket merges). + module Monitor + INTERVAL = 60 # seconds between rounds + BUCKET = 300 # seconds per sample row + TIMEOUT = 3 # connect and DNS timeout per probe + THREADS = 32 + KEEP_DAYS = 30 + CONTROL = '_control'.freeze # slug for control hosts in the bulk form + + class Target < Sequel::Model(:bingo_monitor_targets) + many_to_one :datacenter, class: 'Bingo::Datacenter' + + def before_create + self.created_at ||= Time.now.utc + super + end + + def control? = datacenter_id.nil? + def endpoint = "#{host}:#{port}" + end + + module_function + + def bucket_of(time) = Time.at(time.to_i - (time.to_i % BUCKET)).utc + + # Handshake time in ms, or nil when nothing answered (timeout, no route, + # DNS failure). A refusal is an answer: the host is up, the port closed. + def probe(host, port) + t = Process.clock_gettime(Process::CLOCK_MONOTONIC) + begin + Socket.tcp(host, port, connect_timeout: TIMEOUT, resolv_timeout: TIMEOUT, &:close) + rescue Errno::ECONNREFUSED + nil # fall through to the elapsed time + end + ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - t) * 1000).round + rescue SystemCallError, SocketError, IOError + nil + end + + # [[target, ms or nil], ...], probed concurrently. + def round(targets) + queue = Queue.new + targets.each { |t| queue << t } + results = Queue.new + Array.new([THREADS, targets.size].min) do + Thread.new do + while (t = (queue.pop(true) rescue nil)) + results << [t, probe(t.host, t.port)] + end + end + end.each(&:join) + Array.new(results.size) { results.pop } + end + + def record!(results, at: Time.now.utc) + db = DB.connection + bucket = bucket_of(at) + db.transaction do + sites, controls = results.partition { |t, _| t.datacenter_id } + sites.group_by { |t, _| t.datacenter_id }.each do |dc_id, rs| + answered = rs.filter_map { |_, ms| ms } + upsert(db[:bingo_monitor_samples], %i[datacenter_id bucket_at], + { datacenter_id: dc_id, bucket_at: bucket }, + checks: rs.size, ok: answered.size, rtt_ms_sum: answered.sum) + end + upsert(db[:bingo_monitor_buckets], %i[bucket_at], { bucket_at: bucket }, + rounds: 1, control_checks: controls.size, control_ok: controls.count { |_, ms| ms }) + results.each do |t, ms| + change = t.last_ok.nil? || t.last_ok != !ms.nil? + db[:bingo_monitor_targets].where(id: t.id).update( + { last_ok: !ms.nil?, last_rtt_ms: ms, last_checked_at: at }.merge(change ? { last_change_at: at } : {}) + ) + end + end + end + + # Insert key+counters, or add the counters to the existing row. + def upsert(ds, key_cols, key, counters) + t = ds.first_source_table + ds.insert_conflict(target: key_cols, + update: counters.keys.to_h { |c| [c, Sequel[t][c] + Sequel[:excluded][c]] }) + .insert(key.merge(counters)) + end + + def prune!(now = Time.now.utc) + cutoff = now - (KEEP_DAYS * 86_400) + DB.connection[:bingo_monitor_samples].where { bucket_at < cutoff }.delete + DB.connection[:bingo_monitor_buckets].where { bucket_at < cutoff }.delete + end + + # Bulk form, one target per line: "slug host[:port] source_url label…". + # slug "_control" adds a control host. Returns [targets_attrs, errors]. + def parse(text) + slugs = Datacenter.select_hash(:slug, :id) + attrs, errors = [], [] + text.to_s.each_line.with_index(1) do |line, n| + line = line.strip + next if line.empty? || line.start_with?('#') + slug, hostport, url, *label = line.split(/\s+/) + host, port = hostport.to_s.match(/\A\[?([^\]]+?)\]?(?::(\d+))?\z/)&.captures + dc_id = slug == CONTROL ? nil : slugs[slug] + err = if slug != CONTROL && !dc_id then "unknown slug #{slug.inspect}" + elsif host.to_s.empty? then 'no host' + elsif url && !url.match?(%r{\Ahttps?://}) then 'source must be an http(s) URL' + end + if err + errors << "line #{n}: #{err}" + else + attrs << { datacenter_id: dc_id, host: host, port: (port || 443).to_i, source_url: url, + label: (label.join(' ') unless label.empty?) } + end + end + [attrs, errors] + end + end +end diff --git a/test/bingo_monitor_test.rb b/test/bingo_monitor_test.rb new file mode 100644 index 0000000..7987108 --- /dev/null +++ b/test/bingo_monitor_test.rb @@ -0,0 +1,74 @@ +require_relative 'test_helper' +require 'socket' + +class BingoMonitorTest < Minitest::Test + include Rack::Test::Methods + include AdminLogin + + def app = AsxpioWeb + + def setup + skip 'TEST_DATABASE_URL not set' unless TestDb.available? + db = DB.connection + %i[bingo_monitor_buckets bingo_monitor_samples bingo_monitor_targets bingo_incidents bingo_datacenters].each { |t| db[t].delete } + @dc = Bingo::Datacenter.create(slug: 'dc-a', position: 1, name_ru: 'ЦОД А', city_ru: 'Москва', operator: 'Op') + end + + def test_probe_counts_handshake_and_refusal_as_answered + server = TCPServer.new('127.0.0.1', 0) + open_port = server.addr[1] + assert_kind_of Integer, Bingo::Monitor.probe('127.0.0.1', open_port) + server.close + assert_kind_of Integer, Bingo::Monitor.probe('127.0.0.1', open_port), 'refused = host answered' + assert_nil Bingo::Monitor.probe('no-such-host.invalid', 443) + end + + def test_record_folds_rounds_into_one_bucket + a = Bingo::Monitor::Target.create(datacenter_id: @dc.id, host: 'a', port: 443) + b = Bingo::Monitor::Target.create(datacenter_id: @dc.id, host: 'b', port: 443) + c = Bingo::Monitor::Target.create(datacenter_id: nil, host: 'ctl', port: 80) + t0 = Time.utc(2026, 10, 18, 4, 0, 30) + Bingo::Monitor.record!([[a, 10], [b, nil], [c, 5]], at: t0) + Bingo::Monitor.record!([[a.refresh, 20], [b.refresh, 30], [c.refresh, nil]], at: t0 + 60) + + s = DB.connection[:bingo_monitor_samples].first + assert_equal [Time.utc(2026, 10, 18, 4, 0), 4, 3, 60], s.values_at(:bucket_at, :checks, :ok, :rtt_ms_sum) + bk = DB.connection[:bingo_monitor_buckets].first + assert_equal [2, 2, 1], bk.values_at(:rounds, :control_checks, :control_ok) + b.refresh + assert b.last_ok + assert_equal t0 + 60, b.last_change_at + assert_equal t0, a.refresh.last_change_at, 'unchanged state keeps its time' + end + + def test_prune_drops_old_buckets + Bingo::Monitor.record!([], at: Time.utc(2026, 1, 1)) + Bingo::Monitor.record!([], at: Time.utc(2026, 10, 18)) + Bingo::Monitor.prune!(Time.utc(2026, 10, 18)) + assert_equal [Time.utc(2026, 10, 18)], DB.connection[:bingo_monitor_buckets].select_map(:bucket_at) + end + + def test_parse_bulk_lines + attrs, errors = Bingo::Monitor.parse(<<~TXT) + # comment + dc-a 192.0.2.1:80 https://example.com/ips Test IP one + _control lg.example.ru + dc-zzz 192.0.2.2 + dc-a 192.0.2.3 not-a-url + TXT + assert_equal({ datacenter_id: @dc.id, host: '192.0.2.1', port: 80, source_url: 'https://example.com/ips', label: 'Test IP one' }, attrs[0]) + assert_equal({ datacenter_id: nil, host: 'lg.example.ru', port: 443, source_url: nil, label: nil }, attrs[1]) + assert_equal ['line 4: unknown slug "dc-zzz"', 'line 5: source must be an http(s) URL'], errors + end + + def test_admin_adds_and_lists_targets + admin_login! + get '/admin/bingo/monitor' + token = last_response.body[/name="authenticity_token" value="([^"]+)"/, 1] + post '/admin/bingo/monitor', authenticity_token: token, targets: "dc-a 192.0.2.1:80 https://example.com/ips Test IP\n" + assert_equal 1, Bingo::Monitor::Target.count + get '/admin/bingo/monitor' + assert_includes last_response.body, '192.0.2.1:80' + refute_includes last_response.body, 'dc-a

', 'monitored site leaves the unmonitored list' + end +end diff --git a/views/admin/bingo/index.erb b/views/admin/bingo/index.erb index 4cf3f0b..8edd8ff 100644 --- a/views/admin/bingo/index.erb +++ b/views/admin/bingo/index.erb @@ -2,7 +2,7 @@

Bingo

- <% if bingo_public_url %>public card ↗<% end %> + <% if bingo_public_url %>public card ↗ · <% end %>monitor

diff --git a/views/admin/bingo/monitor.erb b/views/admin/bingo/monitor.erb new file mode 100644 index 0000000..2535e97 --- /dev/null +++ b/views/admin/bingo/monitor.erb @@ -0,0 +1,60 @@ +<%== erb :'admin/_nav', layout: false %> +
+

Bingo monitor

+

+ ← bingo + · last round: <%= @last_round ? "#{@last_round[:bucket_at].strftime('%Y-%m-%d %H:%M')} UTC bucket, controls #{@last_round[:control_ok]}/#{@last_round[:control_checks]}" : 'never (is bin/monitor running?)' %> +

+
+ +<% if @flash %>
<%= @flash %>
<% end %> + +<% if can?('bingo', 'write') %> +
+

Add targets

+

One per line: slug host[:port] source_url label…. Port defaults to 443. Slug <%= Bingo::Monitor::CONTROL %> adds a control host (in Russia, not in any listed site). Only hosts a public source ties to that one datacenter; one TCP connect per minute each.

+
+ +
+ +
+
+<% end %> + +
+

Targets (<%= @targets.size %>)

+ + + + <% @targets.each do |t| %> + > + + + + + + + + + <% end %> + +
DatacenterEndpointLabelLastmsSince
<%= t.control? ? 'control' : "##{t.datacenter.position} #{t.datacenter.name_ru}" %><%= t.endpoint %><% if t.source_url %><%= t.label || 'source' %><% else %><%= t.label %><% end %><%= { true => '✅', false => '❌', nil => '—' }[t.last_ok] %><%= t.last_rtt_ms %><%= t.last_change_at&.strftime('%m-%d %H:%M') %> + <% if can?('bingo', 'write') %> +
+ + +
+ <% end %> + <% if can?('bingo', 'delete') %> +
+ + +
+ <% end %> +
+
+ +
+

Operational, not monitored (<%= @unmonitored.size %>)

+

<%= @unmonitored.map { |dc| "##{dc.position} #{dc.slug}" }.join(', ') %>

+