From 26e862ff04f95462cc94031ef9af6b7675c276ce Mon Sep 17 00:00:00 2001 From: Sergei Poljanski Date: Sat, 10 Oct 2026 09:42:46 +0400 Subject: [PATCH] bingo: ping IndexNow when the card changes A newly recorded incident only showed up in search once a crawler happened to come back. IndexNow lets Bing, Yandex and the other participating engines know about changed URLs right away. Bingo::IndexNow posts the card URLs plus the affected datacenter pages to api.indexnow.org, in a background thread so admin saves don't wait. Incidents ping their datacenter. Edits ping the datacenter, or every page when positions move (create, delete, reorder, sort). A deleted datacenter's pages are included so the engines see the 404. BingoWeb serves the key at /.txt. The key comes from the INDEXNOW Forgejo secret, written to .env as INDEXNOW_KEY. With no key set this does nothing. Assisted-by: Claude:opus-5 --- .env.example | 2 ++ .forgejo/workflows/deploy.yaml | 4 ++- bingo_web.rb | 7 +++++ lib/bingo.rb | 39 ++++++++++++++++++++++++++++ lib/bingo_admin.rb | 9 ++++++- test/bingo_test.rb | 47 ++++++++++++++++++++++++++++++++++ 6 files changed, 106 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 8cb0bfe..6921f07 100644 --- a/.env.example +++ b/.env.example @@ -22,3 +22,5 @@ MAIL_TO=ie@asxp.io # BINGO_HOST=bingo.localhost # Optional second hostname serving the same card. # BINGO_ALT_HOST=bingo2.localhost +# IndexNow key (8-128 chars, A-Z a-z 0-9 -); unset = no pings, no key file. +# INDEXNOW_KEY= diff --git a/.forgejo/workflows/deploy.yaml b/.forgejo/workflows/deploy.yaml index 2bc2ed3..00d4209 100644 --- a/.forgejo/workflows/deploy.yaml +++ b/.forgejo/workflows/deploy.yaml @@ -110,11 +110,12 @@ jobs: ASXPIO_S3_SECRET_KEY: ${{ secrets.ASXPIO_S3_SECRET_KEY }} LTC_ADDRESS: ${{ secrets.LTC_ADDRESS }} CRYPTO_ADDRESSES: ${{ secrets.CRYPTO_ADDRESSES }} + INDEXNOW: ${{ secrets.INDEXNOW }} with: host: ${{ secrets.DEPLOY_IP }} username: ${{ secrets.DEPLOY_USER }} key: ${{ secrets.DEPLOY_SSH_KEY }} - envs: SESSION_SECRET,SMTP_PASSWORD,ADMIN_USER,ADMIN_PASSWORD,ASXPIO_DB_PASSWORD,ASXPIO_S3_ACCESS_KEY,ASXPIO_S3_SECRET_KEY,LTC_ADDRESS,CRYPTO_ADDRESSES + envs: SESSION_SECRET,SMTP_PASSWORD,ADMIN_USER,ADMIN_PASSWORD,ASXPIO_DB_PASSWORD,ASXPIO_S3_ACCESS_KEY,ASXPIO_S3_SECRET_KEY,LTC_ADDRESS,CRYPTO_ADDRESSES,INDEXNOW script_stop: true script: | umask 077 @@ -140,6 +141,7 @@ jobs: LTC_ADDRESS=${LTC_ADDRESS} CRYPTO_ADDRESSES=${CRYPTO_ADDRESSES} BINGO_HOST=datacenterbingo.xyz + INDEXNOW_KEY=${INDEXNOW} EOF chmod 600 /opt/asxpio/.env diff --git a/bingo_web.rb b/bingo_web.rb index 5d7c567..e5701da 100644 --- a/bingo_web.rb +++ b/bingo_web.rb @@ -188,6 +188,13 @@ class BingoWeb < Sinatra::Base erb :dc end + # IndexNow key file: proves to the engines that pings come from us. + get %r{/([A-Za-z0-9-]{8,128})\.txt} do |name| + halt 404 unless name == Bingo::IndexNow.key + content_type 'text/plain' + name + end + # Icons from assets/bingo/ (public/ is asxp.io's). Search engines want # a crawlable favicon URL; a data: URI doesn't count. ICONS = { 'favicon.svg' => 'image/svg+xml', 'favicon.ico' => 'image/x-icon', diff --git a/lib/bingo.rb b/lib/bingo.rb index 050f2d7..0dcfe8c 100644 --- a/lib/bingo.rb +++ b/lib/bingo.rb @@ -397,6 +397,45 @@ module Bingo end end + # Tells IndexNow engines (Bing, Yandex, Seznam, ...) which bingo URLs + # changed, so new incidents get recrawled within minutes. The key comes + # from INDEXNOW_KEY and BingoWeb serves it at /.txt; without the key + # or BINGO_HOST (dev, tests) this does nothing. + module IndexNow + ENDPOINT = URI('https://api.indexnow.org/indexnow') + + module_function + + def key = ENV['INDEXNOW_KEY'].to_s[/\A[A-Za-z0-9-]{8,128}\z/] + def host = ENV['BINGO_HOST'].to_s[/\A\S+\z/] + + # The card in every locale plus the pages of the given datacenters. + def urls(dcs = []) + paths = LOCALES.map { |l| "/#{l}" } + dcs.flat_map { |dc| LOCALES.map { |l| "/#{l}/dc/#{dc.slug}" } } + paths.map { |p| "https://#{host}#{p}" } + end + + def payload(dcs = []) + { host: host, key: key, keyLocation: "https://#{host}/#{key}.txt", urlList: urls(dcs) } + end + + # Fire and forget: an admin save doesn't wait on a third party. + def ping(dcs = []) + return unless key && host + body = JSON.generate(payload(dcs)) + Thread.new { deliver(body) } + end + + def deliver(body) + res = Net::HTTP.start(ENDPOINT.host, ENDPOINT.port, use_ssl: true, open_timeout: 5, read_timeout: 10) do |http| + http.post(ENDPOINT.path, body, 'Content-Type' => 'application/json; charset=utf-8') + end + $logger.warn("bingo: IndexNow answered #{res.code}") unless res.is_a?(Net::HTTPSuccess) + rescue StandardError => e + $logger.warn("bingo: IndexNow #{e.class}: #{e.message}") + end + end + # Positions stay dense 1..N; the card shows the first SIZE*SIZE, the rest # wait in reserve. Placing a datacenter inserts it at pos (default: last) # and shifts everyone after it. diff --git a/lib/bingo_admin.rb b/lib/bingo_admin.rb index 9be3839..3f0fff5 100644 --- a/lib/bingo_admin.rb +++ b/lib/bingo_admin.rb @@ -73,6 +73,7 @@ module BingoAdmin @dc.save Bingo.place!(@dc, position_param) audit!('bingo.dc_create', @dc.slug, "##{@dc.refresh.position} #{@dc.name_ru}") + Bingo::IndexNow.ping(Bingo::Datacenter.all) # positions shifted session[:bingo_flash] = "Added #{@dc.name_ru} at ##{@dc.refresh.position}." redirect "/admin/bingo/dc/#{@dc.id}" end @@ -90,8 +91,10 @@ module BingoAdmin errs = assign_dc(dc) if errs.empty? dc.save_changes - Bingo.place!(dc, position_param) if position_param && position_param != dc.position + moved = position_param && position_param != dc.position + Bingo.place!(dc, position_param) if moved audit!('bingo.dc_update', dc.slug, dc.name_ru) + Bingo::IndexNow.ping(moved ? Bingo::Datacenter.all : [dc]) session[:bingo_flash] = 'Saved.' else session[:bingo_flash] = errs.join('; ') @@ -109,6 +112,7 @@ module BingoAdmin Bingo.renumber! end audit!('bingo.dc_delete', dc.slug, "#{dc.name_ru}, #{n} incident(s)") + Bingo::IndexNow.ping([dc, *Bingo::Datacenter.all]) # its pages now 404, the rest moved up session[:bingo_flash] = "Deleted #{dc.name_ru}#{" and #{n} incident(s)" if n.positive?}." redirect '/admin/bingo' end @@ -116,6 +120,7 @@ module BingoAdmin app.post '/admin/bingo/sort' do Bingo.sort_by_mw! audit!('bingo.sort_by_mw') + Bingo::IndexNow.ping(Bingo::Datacenter.all) session[:bingo_flash] = 'Sorted by MW (unknown last).' redirect '/admin/bingo' end @@ -212,6 +217,7 @@ module BingoAdmin title_uk: blank_nil(params[:title_uk]), url: blank_nil(url) ) audit!('bingo.incident_create', dc.slug, "#{date} #{params[:kind]}: #{params[:title_ru].strip}") + Bingo::IndexNow.ping([dc]) session[:bingo_flash] = "Incident recorded for #{dc.name_ru}." else session[:bingo_flash] = errs.join('; ') @@ -223,6 +229,7 @@ module BingoAdmin inc = Bingo::Incident[params[:id].to_i] inc&.destroy # hook bumps the datacenter's updated_at audit!('bingo.incident_delete', inc&.datacenter&.slug, inc && "#{inc.occurred_on} #{inc.title_ru}") + Bingo::IndexNow.ping([inc.datacenter]) if inc session[:bingo_flash] = 'Incident deleted.' redirect '/admin/bingo' end diff --git a/test/bingo_test.rb b/test/bingo_test.rb index 8699fd4..07cb36f 100644 --- a/test/bingo_test.rb +++ b/test/bingo_test.rb @@ -1,4 +1,5 @@ require_relative 'test_helper' +require 'minitest/mock' class BingoTest < Minitest::Test include Rack::Test::Methods @@ -288,6 +289,52 @@ class BingoTest < Minitest::Test assert_includes body, '
DC 1' end + def with_env(vars) + old = vars.to_h { |k, _| [k, ENV[k]] } + vars.each { |k, v| ENV[k] = v } + yield + ensure + old.each { |k, v| ENV[k] = v } + end + + def test_indexnow_key_file + get '/0123456789abcdef.txt' + assert_equal 404, last_response.status + with_env('INDEXNOW_KEY' => '0123456789abcdef') do + get '/0123456789abcdef.txt' + assert last_response.ok? + assert_equal '0123456789abcdef', last_response.body + get '/fedcba9876543210.txt' + assert_equal 404, last_response.status + end + end + + def test_indexnow_ping + assert_nil Bingo::IndexNow.ping([@dcs[0]]) # no key: nothing sent + sent = [] + with_env('INDEXNOW_KEY' => '0123456789abcdef', 'BINGO_HOST' => 'bingo.example') do + Bingo::IndexNow.stub(:deliver, ->(body) { sent << JSON.parse(body) }) do + Bingo::IndexNow.ping([@dcs[0]]).join + end + end + body = sent.fetch(0) + assert_equal 'bingo.example', body['host'] + assert_equal 'https://bingo.example/0123456789abcdef.txt', body['keyLocation'] + assert_equal %w[/ru /en /uk /ru/dc/dc-1 /en/dc/dc-1 /uk/dc/dc-1].map { |p| "https://bingo.example#{p}" }, body['urlList'] + end + + def test_admin_incident_pings_indexnow + @app = AsxpioWeb + token = admin_token + pinged = [] + Bingo::IndexNow.stub(:ping, ->(dcs) { pinged << dcs.map(&:slug) }) do + post '/admin/bingo/incidents', { authenticity_token: token, datacenter_id: @dcs[2].id, occurred_on: '2026-10-01', + kind: 'fire', title_ru: 'Пожар' }, admin_env + post "/admin/bingo/incidents/#{Bingo::Incident.first.id}/delete", { authenticity_token: token }, admin_env + end + assert_equal [['dc-3'], ['dc-3']], pinged + end + def test_root_varies_on_language get '/' assert_equal 'Accept-Language', last_response.headers['Vary']